containers
/
automation-tests
mirror of https://github.com/containers/automation-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove devicemapper storage driver 

Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>
This commit is contained in:
Kir Kolyshkin 2023-05-26 16:42:39 -07:00
parent 0ac7916be7
commit 1caec4ab84
31 changed files with 10 additions and 5596 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
storage/.cirrus.yml
View File

@ -126,7 +126,7 @@ lint_task:
folder: $GOPATH/pkg/mod
build_script: |
apt-get update
apt-get install -y libbtrfs-dev libdevmapper-dev
apt-get install -y libbtrfs-dev
test_script: |
make TAGS=regex_precompile local-validate
make lint

2
storage/Makefile
View File

@ -27,7 +27,7 @@
vendor-in-container
NATIVETAGS :=
AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh) $(shell ./hack/libsubid_tag.sh)
AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libsubid_tag.sh)
BUILDFLAGS := -tags "$(AUTOTAGS) $(TAGS)" $(FLAGS)
GO ?= go
TESTFLAGS := $(shell $(GO) test -race $(BUILDFLAGS) ./pkg/stringutils 2>&1 > /dev/null && echo -race)

62
storage/docs/containers-storage.conf.5.md
View File

@ -27,7 +27,7 @@ No bare options are used. The format of TOML can be simplified to:
The `storage` table supports the following options:
**driver**=""
Copy On Write (COW) container storage driver. Valid drivers are "overlay", "vfs", "devmapper", "aufs", "btrfs", and "zfs". Some drivers (for example, "zfs", "btrfs", and "aufs") may not work if your kernel lacks support for the filesystem.
Copy On Write (COW) container storage driver. Valid drivers are "overlay", "vfs", "aufs", "btrfs", and "zfs". Some drivers (for example, "zfs", "btrfs", and "aufs") may not work if your kernel lacks support for the filesystem.
This field is required to guarantee proper operation.
Valid rootless drivers are "btrfs", "overlay", and "vfs".
Rootless users default to the driver defined in the system configuration when possible.
@ -158,66 +158,6 @@ The `storage.options.btrfs` table supports the following options:
**size**=""
Maximum size of a container image. This flag can be used to set quota on the size of container images. (format: <number>[<unit>], where unit = b (bytes), k (kilobytes), m (megabytes), or g (gigabytes))
### STORAGE OPTIONS FOR THINPOOL (devicemapper) TABLE
The `storage.options.thinpool` table supports the following options for the `devicemapper` driver:
**autoextend_percent**=""
Tells the thinpool driver the amount by which the thinpool needs to be grown. This is specified in terms of % of pool size. So a value of 20 means that when threshold is hit, pool will be grown by 20% of existing pool size. (default: 20%)
**autoextend_threshold**=""
Tells the driver the thinpool extension threshold in terms of percentage of pool size. For example, if threshold is 60, that means when pool is 60% full, threshold has been hit. (default: 80%)
**basesize**=""
Specifies the size to use when creating the base device, which limits the size of images and containers. (default: 10g)
**blocksize**=""
Specifies a custom blocksize to use for the thin pool. (default: 64k)
**directlvm_device**=""
Specifies a custom block storage device to use for the thin pool. Required for using graphdriver `devicemapper`.
**directlvm_device_force**=""
Tells driver to wipe device (directlvm_device) even if device already has a filesystem. (default: false)
**fs**="xfs"
Specifies the filesystem type to use for the base device. (default: xfs)
**log_level**=""
Sets the log level of devicemapper.
0: LogLevelSuppress 0 (default)
2: LogLevelFatal
3: LogLevelErr
4: LogLevelWarn
5: LogLevelNotice
6: LogLevelInfo
7: LogLevelDebug
**metadata_size**=""
metadata_size is used to set the `pvcreate --metadatasize` options when creating thin devices. (Default 128k)
**min_free_space**=""
Specifies the min free space percent in a thin pool required for new device creation to succeed. Valid values are from 0% - 99%. Value 0% disables. (default: 10%)
**mkfsarg**=""
Specifies extra mkfs arguments to be used when creating the base device.
**mountopt**=""
Comma separated list of default options to be used to mount container images. Suggested value "nodev". Mount options are documented in the mount(8) man page.
**size**=""
Maximum size of a container image. This flag can be used to set quota on the size of container images. (format: <number>[<unit>], where unit = b (bytes), k (kilobytes), m (megabytes), or g (gigabytes))
**use_deferred_deletion**=""
Marks thinpool device for deferred deletion. If the thinpool is in use when the driver attempts to delete it, the driver will attempt to delete device every 30 seconds until successful, or when it restarts. Deferred deletion permanently deletes the device and all data stored in the device will be lost. (default: true).
**use_deferred_removal**=""
Marks devicemapper block device for deferred removal. If the device is in use when its driver attempts to remove it, the driver tells the kernel to remove the device as soon as possible. Note this does not free up the disk space, use deferred deletion to fully remove the thinpool. (default: true).
**xfs_nospace_max_retries**=""
Specifies the maximum number of retries XFS should attempt to complete IO when ENOSPC (no space) error is returned by underlying storage device. (default: 0, which means to try continuously.)
### STORAGE OPTIONS FOR OVERLAY TABLE
The `storage.options.overlay` table supports the following options:

254
storage/drivers/devmapper/device_setup.go
View File

@ -1,254 +0,0 @@
//go:build linux && cgo
// +build linux,cgo
package devmapper
import (
"bufio"
"bytes"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/sirupsen/logrus"
)
type directLVMConfig struct {
Device string
ThinpPercent uint64
ThinpMetaPercent uint64
AutoExtendPercent uint64
AutoExtendThreshold uint64
MetaDataSize string
}
var (
errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified")
errThinpPercentTooBig = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100")
errMissingSetupDevice = errors.New("must provide device path in `dm.directlvm_device` in order to configure direct-lvm")
)
func validateLVMConfig(cfg directLVMConfig) error {
if cfg.Device == "" {
return errMissingSetupDevice
}
if (cfg.ThinpPercent > 0 && cfg.ThinpMetaPercent == 0) || cfg.ThinpMetaPercent > 0 && cfg.ThinpPercent == 0 {
return errThinpPercentMissing
}
if cfg.ThinpPercent+cfg.ThinpMetaPercent > 100 {
return errThinpPercentTooBig
}
return nil
}
func checkDevAvailable(dev string) error {
lvmScan, err := exec.LookPath("lvmdiskscan")
if err != nil {
logrus.Debugf("could not find lvmdiskscan: %v", err)
return nil
}
out, err := exec.Command(lvmScan).CombinedOutput()
if err != nil {
logrus.WithError(err).Error(string(out))
return nil
}
if !bytes.Contains(out, []byte(dev)) {
return fmt.Errorf("%s is not available for use with devicemapper", dev)
}
return nil
}
func checkDevInVG(dev string) error {
pvDisplay, err := exec.LookPath("pvdisplay")
if err != nil {
logrus.Debugf("could not find pvdisplay: %v", err)
return nil
}
out, err := exec.Command(pvDisplay, dev).CombinedOutput()
if err != nil {
logrus.WithError(err).Error(string(out))
return nil
}
scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimSpace(out)))
for scanner.Scan() {
fields := strings.SplitAfter(strings.TrimSpace(scanner.Text()), "VG Name")
if len(fields) > 1 {
// got "VG Name" line"
vg := strings.TrimSpace(fields[1])
if len(vg) > 0 {
return fmt.Errorf("%s is already part of a volume group %q: must remove this device from any volume group or provide a different device", dev, vg)
}
logrus.Error(fields)
break
}
}
return nil
}
func checkDevHasFS(dev string) error {
blkid, err := exec.LookPath("blkid")
if err != nil {
logrus.Debugf("could not find blkid %v", err)
return nil
}
out, err := exec.Command(blkid, dev).CombinedOutput()
if err != nil {
logrus.WithError(err).Error(string(out))
return nil
}
fields := bytes.Fields(out)
for _, f := range fields {
kv := bytes.Split(f, []byte{'='})
if bytes.Equal(kv[0], []byte("TYPE")) {
v := bytes.Trim(kv[1], "\"")
if len(v) > 0 {
return fmt.Errorf("%s has a filesystem already, use dm.directlvm_device_force=true if you want to wipe the device", dev)
}
return nil
}
}
return nil
}
func verifyBlockDevice(dev string, force bool) error {
absPath, err := filepath.Abs(dev)
if err != nil {
return fmt.Errorf("unable to get absolute path for %s: %s", dev, err)
}
realPath, err := filepath.EvalSymlinks(absPath)
if err != nil {
return fmt.Errorf("failed to canonicalise path for %s: %s", dev, err)
}
if err := checkDevAvailable(absPath); err != nil {
logrus.Infof("block device '%s' not available, checking '%s'", absPath, realPath)
if err := checkDevAvailable(realPath); err != nil {
return fmt.Errorf("neither '%s' nor '%s' are in the output of lvmdiskscan, can't use device", absPath, realPath)
}
}
if err := checkDevInVG(realPath); err != nil {
return err
}
if force {
return nil
}
if err := checkDevHasFS(realPath); err != nil {
return err
}
return nil
}
func readLVMConfig(root string) (directLVMConfig, error) {
var cfg directLVMConfig
p := filepath.Join(root, "setup-config.json")
b, err := os.ReadFile(p)
if err != nil {
if os.IsNotExist(err) {
return cfg, nil
}
return cfg, fmt.Errorf("reading existing setup config: %w", err)
}
// check if this is just an empty file, no need to produce a json error later if so
if len(b) == 0 {
return cfg, nil
}
if err := json.Unmarshal(b, &cfg); err != nil {
return cfg, fmt.Errorf("unmarshaling previous device setup config: %w", err)
}
return cfg, nil
}
func writeLVMConfig(root string, cfg directLVMConfig) error {
p := filepath.Join(root, "setup-config.json")
b, err := json.Marshal(cfg)
if err != nil {
return fmt.Errorf("marshalling direct lvm config: %w", err)
}
if err := os.WriteFile(p, b, 0o600); err != nil {
return fmt.Errorf("writing direct lvm config to file: %w", err)
}
return nil
}
func setupDirectLVM(cfg directLVMConfig) error {
lvmProfileDir := "/etc/lvm/profile"
binaries := []string{"pvcreate", "vgcreate", "lvcreate", "lvconvert", "lvchange", "thin_check"}
for _, bin := range binaries {
if _, err := exec.LookPath(bin); err != nil {
return fmt.Errorf("looking up command `"+bin+"` while setting up direct lvm: %w", err)
}
}
err := os.MkdirAll(lvmProfileDir, 0o755)
if err != nil {
return fmt.Errorf("creating lvm profile directory: %w", err)
}
if cfg.AutoExtendPercent == 0 {
cfg.AutoExtendPercent = 20
}
if cfg.AutoExtendThreshold == 0 {
cfg.AutoExtendThreshold = 80
}
if cfg.ThinpPercent == 0 {
cfg.ThinpPercent = 95
}
if cfg.ThinpMetaPercent == 0 {
cfg.ThinpMetaPercent = 1
}
if cfg.MetaDataSize == "" {
cfg.MetaDataSize = "128k"
}
out, err := exec.Command("pvcreate", "--metadatasize", cfg.MetaDataSize, "-f", cfg.Device).CombinedOutput()
if err != nil {
return fmt.Errorf("%v: %w", string(out), err)
}
out, err = exec.Command("vgcreate", "storage", cfg.Device).CombinedOutput()
if err != nil {
return fmt.Errorf("%v: %w", string(out), err)
}
out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpool", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput()
if err != nil {
return fmt.Errorf("%v: %w", string(out), err)
}
out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpoolmeta", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput()
if err != nil {
return fmt.Errorf("%v: %w", string(out), err)
}
out, err = exec.Command("lvconvert", "-y", "--zero", "n", "-c", "512K", "--thinpool", "storage/thinpool", "--poolmetadata", "storage/thinpoolmeta").CombinedOutput()
if err != nil {
return fmt.Errorf("%v: %w", string(out), err)
}
profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent)
err = os.WriteFile(lvmProfileDir+"/storage-thinpool.profile", []byte(profile), 0o600)
if err != nil {
return fmt.Errorf("writing storage thinp autoextend profile: %w", err)
}
out, err = exec.Command("lvchange", "--metadataprofile", "storage-thinpool", "storage/thinpool").CombinedOutput()
if err != nil {
return fmt.Errorf("%s: %w", string(out), err)
}
return nil
}

2889
storage/drivers/devmapper/deviceset.go
View File

File diff suppressed because it is too large Load Diff

109
storage/drivers/devmapper/devmapper_doc.go
View File

@ -1,109 +0,0 @@
//go:build linux && cgo
// +build linux,cgo
package devmapper
// Definition of struct dm_task and sub structures (from lvm2)
//
// struct dm_ioctl {
// /*
// * The version number is made up of three parts:
// * major - no backward or forward compatibility,
// * minor - only backwards compatible,
// * patch - both backwards and forwards compatible.
// *
// * All clients of the ioctl interface should fill in the
// * version number of the interface that they were
// * compiled with.
// *
// * All recognized ioctl commands (ie. those that don't
// * return -ENOTTY) fill out this field, even if the
// * command failed.
// */
// uint32_t version[3]; /* in/out */
// uint32_t data_size; /* total size of data passed in
// * including this struct */
// uint32_t data_start; /* offset to start of data
// * relative to start of this struct */
// uint32_t target_count; /* in/out */
// int32_t open_count; /* out */
// uint32_t flags; /* in/out */
// /*
// * event_nr holds either the event number (input and output) or the
// * udev cookie value (input only).
// * The DM_DEV_WAIT ioctl takes an event number as input.
// * The DM_SUSPEND, DM_DEV_REMOVE and DM_DEV_RENAME ioctls
// * use the field as a cookie to return in the DM_COOKIE
// * variable with the uevents they issue.
// * For output, the ioctls return the event number, not the cookie.
// */
// uint32_t event_nr; /* in/out */
// uint32_t padding;
// uint64_t dev; /* in/out */
// char name[DM_NAME_LEN]; /* device name */
// char uuid[DM_UUID_LEN]; /* unique identifier for
// * the block device */
// char data[7]; /* padding or data */
// };
// struct target {
// uint64_t start;
// uint64_t length;
// char *type;
// char *params;
// struct target *next;
// };
// typedef enum {
// DM_ADD_NODE_ON_RESUME, /* add /dev/mapper node with dmsetup resume */
// DM_ADD_NODE_ON_CREATE /* add /dev/mapper node with dmsetup create */
// } dm_add_node_t;
// struct dm_task {
// int type;
// char *dev_name;
// char *mangled_dev_name;
// struct target *head, *tail;
// int read_only;
// uint32_t event_nr;
// int major;
// int minor;
// int allow_default_major_fallback;
// uid_t uid;
// gid_t gid;
// mode_t mode;
// uint32_t read_ahead;
// uint32_t read_ahead_flags;
// union {
// struct dm_ioctl *v4;
// } dmi;
// char *newname;
// char *message;
// char *geometry;
// uint64_t sector;
// int no_flush;
// int no_open_count;
// int skip_lockfs;
// int query_inactive_table;
// int suppress_identical_reload;
// dm_add_node_t add_node;
// uint64_t existing_table_size;
// int cookie_set;
// int new_uuid;
// int secure_data;
// int retry_remove;
// int enable_checks;
// int expected_errno;
// char *uuid;
// char *mangled_uuid;
// };
//

220
storage/drivers/devmapper/devmapper_test.go
View File

@ -1,220 +0,0 @@
//go:build linux && cgo
// +build linux,cgo
package devmapper
import (
"fmt"
"os"
"os/exec"
"syscall"
"testing"
"time"
graphdriver "github.com/containers/storage/drivers"
"github.com/containers/storage/drivers/graphtest"
"github.com/containers/storage/pkg/parsers/kernel"
"golang.org/x/sys/unix"
)
func init() {
// Reduce the size of the base fs and loopback for the tests
defaultDataLoopbackSize = 300 * 1024 * 1024
defaultMetaDataLoopbackSize = 200 * 1024 * 1024
defaultBaseFsSize = 300 * 1024 * 1024
defaultUdevSyncOverride = true
if err := initLoopbacks(); err != nil {
panic(err)
}
}
// initLoopbacks ensures that the loopback devices are properly created within
// the system running the device mapper tests.
func initLoopbacks() error {
statT, err := getBaseLoopStats()
if err != nil {
return err
}
// create at least 128 loopback files, since a few first ones
// might be already in use by the host OS
for i := 0; i < 128; i++ {
loopPath := fmt.Sprintf("/dev/loop%d", i)
// only create new loopback files if they don't exist
if _, err := os.Stat(loopPath); err != nil {
if mkerr := syscall.Mknod(loopPath,
uint32(statT.Mode|syscall.S_IFBLK), int((7<<8)|(i&0xff)|((i&0xfff00)<<12))); mkerr != nil {
return mkerr
}
os.Chown(loopPath, int(statT.Uid), int(statT.Gid))
}
}
return nil
}
// getBaseLoopStats inspects /dev/loop0 to collect uid,gid, and mode for the
// loop0 device on the system. If it does not exist we assume 0,0,0660 for the
// stat data
func getBaseLoopStats() (*syscall.Stat_t, error) {
loop0, err := os.Stat("/dev/loop0")
if err != nil {
if os.IsNotExist(err) {
return &syscall.Stat_t{
Uid: 0,
Gid: 0,
Mode: 0o660,
}, nil
}
return nil, err
}
return loop0.Sys().(*syscall.Stat_t), nil
}
// This avoids creating a new driver for each test if all tests are run
// Make sure to put new tests between TestDevmapperSetup and TestDevmapperTeardown
func TestDevmapperSetup(t *testing.T) {
graphtest.GetDriver(t, "devicemapper", "test=1")
}
func TestDevmapperCreateEmpty(t *testing.T) {
graphtest.DriverTestCreateEmpty(t, "devicemapper", "test=1")
}
func TestDevmapperCreateBase(t *testing.T) {
graphtest.DriverTestCreateBase(t, "devicemapper", "test=1")
}
func TestDevmapperCreateSnap(t *testing.T) {
graphtest.DriverTestCreateSnap(t, "devicemapper", "test=1")
}
func TestDevmapperCreateFromTemplate(t *testing.T) {
graphtest.DriverTestCreateFromTemplate(t, "devicemapper", "test=1")
}
func TestDevmapperEcho(t *testing.T) {
graphtest.DriverTestEcho(t, "devicemapper", "test=1")
}
func TestDevmapperListLayers(t *testing.T) {
graphtest.DriverTestListLayers(t, "devicemapper", "test=1")
}
func TestDevmapperTeardown(t *testing.T) {
graphtest.PutDriver(t)
}
func TestDevmapperReduceLoopBackSize(t *testing.T) {
tenMB := int64(10 * 1024 * 1024)
testChangeLoopBackSize(t, -tenMB, defaultDataLoopbackSize, defaultMetaDataLoopbackSize)
}
func TestDevmapperIncreaseLoopBackSize(t *testing.T) {
tenMB := int64(10 * 1024 * 1024)
testChangeLoopBackSize(t, tenMB, defaultDataLoopbackSize+tenMB, defaultMetaDataLoopbackSize+tenMB)
}
func testChangeLoopBackSize(t *testing.T, delta, expectDataSize, expectMetaDataSize int64) {
driver := graphtest.GetDriver(t, "devicemapper", "test=1").(*graphtest.Driver).Driver.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
defer graphtest.PutDriver(t)
// make sure data or metadata loopback size are the default size
if s := driver.DeviceSet.Status(); s.Data.Total != uint64(defaultDataLoopbackSize) || s.Metadata.Total != uint64(defaultMetaDataLoopbackSize) {
t.Fatal("data or metadata loop back size is incorrect")
}
if err := driver.Cleanup(); err != nil {
t.Fatal(err)
}
// Reload
d, err := Init(driver.home, graphdriver.Options{DriverOptions: []string{
fmt.Sprintf("dm.loopdatasize=%d", defaultDataLoopbackSize+delta),
fmt.Sprintf("dm.loopmetadatasize=%d", defaultMetaDataLoopbackSize+delta),
"test=1",
}})
if err != nil {
t.Fatalf("error creating devicemapper driver: %v", err)
}
driver = d.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
if s := driver.DeviceSet.Status(); s.Data.Total != uint64(expectDataSize) || s.Metadata.Total != uint64(expectMetaDataSize) {
t.Fatalf("data or metadata loop back size is incorrect (data actual/expected=%d/%d, metadata actual/expected = %d/%d)", s.Data.Total, expectDataSize, s.Metadata.Total, expectMetaDataSize)
}
if err := driver.Cleanup(); err != nil {
t.Fatal(err)
}
}
// Make sure devices.Lock() has been release upon return from cleanupDeletedDevices() function
func TestDevmapperLockReleasedDeviceDeletion(t *testing.T) {
driver := graphtest.GetDriver(t, "devicemapper", "test=1").(*graphtest.Driver).Driver.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
defer graphtest.PutDriver(t)
// Call cleanupDeletedDevices() and after the call take and release
// DeviceSet Lock. If lock has not been released, this will hang.
driver.DeviceSet.cleanupDeletedDevices()
doneChan := make(chan bool)
go func() {
driver.DeviceSet.Lock()
defer driver.DeviceSet.Unlock()
doneChan <- true
}()
select {
case <-time.After(time.Second * 5):
// Timer expired. That means lock was not released upon
// function return and we are deadlocked. Release lock
// here so that cleanup could succeed and fail the test.
driver.DeviceSet.Unlock()
t.Fatal("Could not acquire devices lock after call to cleanupDeletedDevices()")
case <-doneChan:
}
}
// Ensure that mounts aren't leakedriver. It's non-trivial for us to test the full
// reproducer of #34573 in a unit test, but we can at least make sure that a
// simple command run in a new namespace doesn't break things horribly.
func TestDevmapperMountLeaks(t *testing.T) {
if !kernel.CheckKernelVersion(3, 18, 0) {
t.Skipf("kernel version <3.18.0 and so is missing torvalds/linux@8ed936b5671bfb33d89bc60bdcc7cf0470ba52fe.")
}
driver := graphtest.GetDriver(t, "devicemapper", "test=1", "dm.use_deferred_removal=false", "dm.use_deferred_deletion=false").(*graphtest.Driver).Driver.(*graphdriver.NaiveDiffDriver).ProtoDriver.(*Driver)
defer graphtest.PutDriver(t)
// We need to create a new (dummy) device.
if err := driver.Create("some-layer", "", nil); err != nil {
t.Fatalf("setting up some-layer: %v", err)
}
// Mount the device.
_, err := driver.Get("some-layer", graphdriver.MountOpts{})
if err != nil {
t.Fatalf("mounting some-layer: %v", err)
}
// Create a new subprocess which will inherit our mountpoint, then
// intentionally leak it and stick around. We can't do this entirely within
// Go because forking and namespaces in Go are really not handled well at
// all.
cmd := exec.Cmd{
Path: "/bin/sh",
Args: []string{
"/bin/sh", "-c",
"mount --make-rprivate / && sleep 1000s",
},
SysProcAttr: &syscall.SysProcAttr{
Unshareflags: syscall.CLONE_NEWNS,
},
}
if err := cmd.Start(); err != nil {
t.Fatalf("starting sub-command: %v", err)
}
defer func() {
unix.Kill(cmd.Process.Pid, unix.SIGKILL)
cmd.Wait()
}()
// Now try to "drop" the device.
if err := driver.Put("some-layer"); err != nil {
t.Fatalf("unmounting some-layer: %v", err)
}
}

272
storage/drivers/devmapper/driver.go
View File

@ -1,272 +0,0 @@
//go:build linux && cgo
// +build linux,cgo
package devmapper
import (
"fmt"
"os"
"path"
"strconv"
graphdriver "github.com/containers/storage/drivers"
"github.com/containers/storage/pkg/devicemapper"
"github.com/containers/storage/pkg/directory"
"github.com/containers/storage/pkg/fileutils"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/pkg/locker"
"github.com/containers/storage/pkg/mount"
units "github.com/docker/go-units"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
)
const defaultPerms = os.FileMode(0o555)
func init() {
graphdriver.MustRegister("devicemapper", Init)
}
// Driver contains the device set mounted and the home directory
type Driver struct {
*DeviceSet
home string
uidMaps []idtools.IDMap
gidMaps []idtools.IDMap
ctr *graphdriver.RefCounter
locker *locker.Locker
}
// Init creates a driver with the given home and the set of options.
func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) {
deviceSet, err := NewDeviceSet(home, true, options.DriverOptions, options.UIDMaps, options.GIDMaps)
if err != nil {
return nil, err
}
if err := mount.MakePrivate(home); err != nil {
return nil, err
}
d := &Driver{
DeviceSet: deviceSet,
home: home,
uidMaps: options.UIDMaps,
gidMaps: options.GIDMaps,
ctr: graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()),
locker: locker.New(),
}
return graphdriver.NewNaiveDiffDriver(d, graphdriver.NewNaiveLayerIDMapUpdater(d)), nil
}
func (d *Driver) String() string {
return "devicemapper"
}
// Status returns the status about the driver in a printable format.
// Information returned contains Pool Name, Data File, Metadata file, disk usage by
// the data and metadata, etc.
func (d *Driver) Status() [][2]string {
s := d.DeviceSet.Status()
status := [][2]string{
{"Pool Name", s.PoolName},
{"Pool Blocksize", units.HumanSize(float64(s.SectorSize))},
{"Base Device Size", units.HumanSize(float64(s.BaseDeviceSize))},
{"Backing Filesystem", s.BaseDeviceFS},
{"Data file", s.DataFile},
{"Metadata file", s.MetadataFile},
{"Data Space Used", units.HumanSize(float64(s.Data.Used))},
{"Data Space Total", units.HumanSize(float64(s.Data.Total))},
{"Data Space Available", units.HumanSize(float64(s.Data.Available))},
{"Metadata Space Used", units.HumanSize(float64(s.Metadata.Used))},
{"Metadata Space Total", units.HumanSize(float64(s.Metadata.Total))},
{"Metadata Space Available", units.HumanSize(float64(s.Metadata.Available))},
{"Thin Pool Minimum Free Space", units.HumanSize(float64(s.MinFreeSpace))},
{"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)},
{"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)},
{"Deferred Deletion Enabled", fmt.Sprintf("%v", s.DeferredDeleteEnabled)},
{"Deferred Deleted Device Count", fmt.Sprintf("%v", s.DeferredDeletedDeviceCount)},
}
if len(s.DataLoopback) > 0 {
status = append(status, [2]string{"Data loop file", s.DataLoopback})
}
if len(s.MetadataLoopback) > 0 {
status = append(status, [2]string{"Metadata loop file", s.MetadataLoopback})
}
if vStr, err := devicemapper.GetLibraryVersion(); err == nil {
status = append(status, [2]string{"Library Version", vStr})
}
return status
}
// Metadata returns a map of information about the device.
func (d *Driver) Metadata(id string) (map[string]string, error) {
m, err := d.DeviceSet.exportDeviceMetadata(id)
if err != nil {
return nil, err
}
metadata := make(map[string]string)
metadata["DeviceId"] = strconv.Itoa(m.deviceID)
metadata["DeviceSize"] = strconv.FormatUint(m.deviceSize, 10)
metadata["DeviceName"] = m.deviceName
return metadata, nil
}
// Cleanup unmounts a device.
func (d *Driver) Cleanup() error {
err := d.DeviceSet.Shutdown(d.home)
umountErr := mount.Unmount(d.home)
// in case we have two errors, prefer the one from Shutdown()
if err != nil {
return err
}
return umountErr
}
// CreateFromTemplate creates a layer with the same contents and parent as another layer.
func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error {
return d.Create(id, template, opts)
}
// CreateReadWrite creates a layer that is writable for use as a container
// file system.
func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error {
return d.Create(id, parent, opts)
}
// Create adds a device with a given id and the parent.
func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
var storageOpt map[string]string
if opts != nil {
storageOpt = opts.StorageOpt
}
if err := d.DeviceSet.AddDevice(id, parent, storageOpt); err != nil {
return err
}
return nil
}
// Remove removes a device with a given id, unmounts the filesystem, and removes the mount point.
func (d *Driver) Remove(id string) error {
d.locker.Lock(id)
defer d.locker.Unlock(id)
if !d.DeviceSet.HasDevice(id) {
// Consider removing a non-existing device a no-op
// This is useful to be able to progress on container removal
// if the underlying device has gone away due to earlier errors
return nil
}
// This assumes the device has been properly Get/Put:ed and thus is unmounted
if err := d.DeviceSet.DeleteDevice(id, false); err != nil {
return fmt.Errorf("failed to remove device %s: %v", id, err)
}
// Most probably the mount point is already removed on Put()
// (see DeviceSet.UnmountDevice()), but just in case it was not
// let's try to remove it here as well, ignoring errors as
// an older kernel can return EBUSY if e.g. the mount was leaked
// to other mount namespaces. A failure to remove the container's
// mount point is not important and should not be treated
// as a failure to remove the container.
mp := path.Join(d.home, "mnt", id)
err := unix.Rmdir(mp)
if err != nil && !os.IsNotExist(err) {
logrus.WithField("storage-driver", "devicemapper").Warnf("unable to remove mount point %q: %s", mp, err)
}
return nil
}
// Get mounts a device with given id into the root filesystem
func (d *Driver) Get(id string, options graphdriver.MountOpts) (string, error) {
d.locker.Lock(id)
defer d.locker.Unlock(id)
mp := path.Join(d.home, "mnt", id)
rootFs := path.Join(mp, "rootfs")
if count := d.ctr.Increment(mp); count > 1 {
return rootFs, nil
}
uid, gid, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
if err != nil {
d.ctr.Decrement(mp)
return "", err
}
// Create the target directories if they don't exist
if err := idtools.MkdirAllAs(path.Join(d.home, "mnt"), 0o755, uid, gid); err != nil {
d.ctr.Decrement(mp)
return "", err
}
if err := idtools.MkdirAs(mp, 0o755, uid, gid); err != nil && !os.IsExist(err) {
d.ctr.Decrement(mp)
return "", err
}
// Mount the device
if err := d.DeviceSet.MountDevice(id, mp, options); err != nil {
d.ctr.Decrement(mp)
return "", err
}
if err := idtools.MkdirAllAs(rootFs, defaultPerms, uid, gid); err != nil {
d.ctr.Decrement(mp)
d.DeviceSet.UnmountDevice(id, mp)
return "", err
}
idFile := path.Join(mp, "id")
if err := fileutils.Exists(idFile); err != nil && os.IsNotExist(err) {
// Create an "id" file with the container/image id in it to help reconstruct this in case
// of later problems
if err := os.WriteFile(idFile, []byte(id), 0o600); err != nil {
d.ctr.Decrement(mp)
d.DeviceSet.UnmountDevice(id, mp)
return "", err
}
}
return rootFs, nil
}
// Put unmounts a device and removes it.
func (d *Driver) Put(id string) error {
d.locker.Lock(id)
defer d.locker.Unlock(id)
mp := path.Join(d.home, "mnt", id)
if count := d.ctr.Decrement(mp); count > 0 {
return nil
}
err := d.DeviceSet.UnmountDevice(id, mp)
if err != nil {
logrus.Errorf("devmapper: Error unmounting device %s: %v", id, err)
}
return err
}
// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID.
// For devmapper, it queries the mnt path for this ID.
func (d *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) {
d.locker.Lock(id)
defer d.locker.Unlock(id)
return directory.Usage(path.Join(d.home, "mnt", id))
}
// Exists checks to see if the device exists.
func (d *Driver) Exists(id string) bool {
return d.DeviceSet.HasDevice(id)
}
// AdditionalImageStores returns additional image stores supported by the driver
func (d *Driver) AdditionalImageStores() []string {
return nil
}

8
storage/drivers/devmapper/jsoniter.go
View File

@ -1,8 +0,0 @@
//go:build linux && cgo
// +build linux,cgo
package devmapper
import jsoniter "github.com/json-iterator/go"
var json = jsoniter.ConfigCompatibleWithStandardLibrary

89
storage/drivers/devmapper/mount.go
View File

@ -1,89 +0,0 @@
//go:build linux && cgo
// +build linux,cgo
package devmapper
import (
"bytes"
"fmt"
"os"
"path/filepath"
"golang.org/x/sys/unix"
)
// FIXME: this is copy-pasted from the aufs driver.
// It should be moved into the core.
// Mounted returns true if a mount point exists.
func Mounted(mountpoint string) (bool, error) {
var mntpointSt unix.Stat_t
if err := unix.Stat(mountpoint, &mntpointSt); err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
var parentSt unix.Stat_t
if err := unix.Stat(filepath.Join(mountpoint, ".."), &parentSt); err != nil {
return false, err
}
return mntpointSt.Dev != parentSt.Dev, nil
}
type probeData struct {
fsName string
magic string
offset uint64
}
// ProbeFsType returns the filesystem name for the given device id.
func ProbeFsType(device string) (string, error) {
probes := []probeData{
{"btrfs", "_BHRfS_M", 0x10040},
{"ext4", "\123\357", 0x438},
{"xfs", "XFSB", 0},
}
maxLen := uint64(0)
for _, p := range probes {
l := p.offset + uint64(len(p.magic))
if l > maxLen {
maxLen = l
}
}
file, err := os.Open(device)
if err != nil {
return "", err
}
defer file.Close()
buffer := make([]byte, maxLen)
l, err := file.Read(buffer)
if err != nil {
return "", err
}
if uint64(l) != maxLen {
return "", fmt.Errorf("devmapper: unable to detect filesystem type of %s, short read", device)
}
for _, p := range probes {
if bytes.Equal([]byte(p.magic), buffer[p.offset:p.offset+uint64(len(p.magic))]) {
return p.fsName, nil
}
}
return "", fmt.Errorf("devmapper: Unknown filesystem type on %s", device)
}
func joinMountOptions(a, b string) string {
if a == "" {
return b
}
if b == "" {
return a
}
return a + "," + b
}

2
storage/drivers/driver_linux.go
View File

@ -94,8 +94,6 @@ var (
// Slice of drivers that should be used in an order
Priority = []string{
"overlay",
// We don't support devicemapper without configuration
// "devicemapper",
"aufs",
"btrfs",
"zfs",

9
storage/drivers/register/register_devicemapper.go
View File

@ -1,9 +0,0 @@
//go:build !exclude_graphdriver_devicemapper && linux && cgo
// +build !exclude_graphdriver_devicemapper,linux,cgo
package register
import (
// register the devmapper graphdriver
_ "github.com/containers/storage/drivers/devmapper"
)

18
storage/hack/libdm_tag.sh
View File

@ -1,18 +0,0 @@
#!/usr/bin/env bash
if test $(${GO:-go} env GOOS) != "linux" ; then
exit 0
fi
tmpdir="$PWD/tmp.$RANDOM"
mkdir -p "$tmpdir"
trap 'rm -fr "$tmpdir"' EXIT
cc -o "$tmpdir"/libdm_tag -ldevmapper -x c - > /dev/null 2> /dev/null << EOF
#include <libdevmapper.h>
int main() {
struct dm_task *task;
dm_task_deferred_remove(task);
return 0;
}
EOF
if test $? -ne 0 ; then
echo libdm_no_deferred_remove
fi

125
storage/pkg/config/config.go
View File

@ -5,72 +5,6 @@ import (
"os"
)
// ThinpoolOptionsConfig represents the "storage.options.thinpool"
// TOML config table.
type ThinpoolOptionsConfig struct {
// AutoExtendPercent determines the amount by which pool needs to be
// grown. This is specified in terms of % of pool size. So a value of
// 20 means that when threshold is hit, pool will be grown by 20% of
// existing pool size.
AutoExtendPercent string `toml:"autoextend_percent,omitempty"`
// AutoExtendThreshold determines the pool extension threshold in terms
// of percentage of pool size. For example, if threshold is 60, that
// means when pool is 60% full, threshold has been hit.
AutoExtendThreshold string `toml:"autoextend_threshold,omitempty"`
// BaseSize specifies the size to use when creating the base device,
// which limits the size of images and containers.
BaseSize string `toml:"basesize,omitempty"`
// BlockSize specifies a custom blocksize to use for the thin pool.
BlockSize string `toml:"blocksize,omitempty"`
// DirectLvmDevice specifies a custom block storage device to use for
// the thin pool.
DirectLvmDevice string `toml:"directlvm_device,omitempty"`
// DirectLvmDeviceForcewipes device even if device already has a
// filesystem
DirectLvmDeviceForce string `toml:"directlvm_device_force,omitempty"`
// Fs specifies the filesystem type to use for the base device.
Fs string `toml:"fs,omitempty"`
// log_level sets the log level of devicemapper.
LogLevel string `toml:"log_level,omitempty"`
// MetadataSize specifies the size of the metadata for the thinpool
// It will be used with the `pvcreate --metadata` option.
MetadataSize string `toml:"metadatasize,omitempty"`
// MinFreeSpace specifies the min free space percent in a thin pool
// require for new device creation to
MinFreeSpace string `toml:"min_free_space,omitempty"`
// MkfsArg specifies extra mkfs arguments to be used when creating the
// basedevice.
MkfsArg string `toml:"mkfsarg,omitempty"`
// MountOpt specifies extra mount options used when mounting the thin
// devices.
MountOpt string `toml:"mountopt,omitempty"`
// Size
Size string `toml:"size,omitempty"`
// UseDeferredDeletion marks device for deferred deletion
UseDeferredDeletion string `toml:"use_deferred_deletion,omitempty"`
// UseDeferredRemoval marks device for deferred removal
UseDeferredRemoval string `toml:"use_deferred_removal,omitempty"`
// XfsNoSpaceMaxRetriesFreeSpace specifies the maximum number of
// retries XFS should attempt to complete IO when ENOSPC (no space)
// error is returned by underlying storage device.
XfsNoSpaceMaxRetries string `toml:"xfs_nospace_max_retries,omitempty"`
}
type AufsOptionsConfig struct {
// MountOpt specifies extra mount options used when mounting
MountOpt string `toml:"mountopt,omitempty"`
@ -181,9 +115,6 @@ type OptionsConfig struct {
// Btrfs container options to be handed to btrfs drivers
Btrfs struct{ BtrfsOptionsConfig } `toml:"btrfs,omitempty"`
// Thinpool container options to be handed to thinpool drivers
Thinpool struct{ ThinpoolOptionsConfig } `toml:"thinpool,omitempty"`
// Overlay container options to be handed to overlay drivers
Overlay struct{ OverlayOptionsConfig } `toml:"overlay,omitempty"`
@ -231,62 +162,6 @@ func GetGraphDriverOptions(driverName string, options OptionsConfig) []string {
doptions = append(doptions, fmt.Sprintf("%s.size=%s", driverName, options.Size))
}
case "devicemapper":
if options.Thinpool.AutoExtendPercent != "" {
doptions = append(doptions, fmt.Sprintf("dm.thinp_autoextend_percent=%s", options.Thinpool.AutoExtendPercent))
}
if options.Thinpool.AutoExtendThreshold != "" {
doptions = append(doptions, fmt.Sprintf("dm.thinp_autoextend_threshold=%s", options.Thinpool.AutoExtendThreshold))
}
if options.Thinpool.BaseSize != "" {
doptions = append(doptions, fmt.Sprintf("dm.basesize=%s", options.Thinpool.BaseSize))
}
if options.Thinpool.BlockSize != "" {
doptions = append(doptions, fmt.Sprintf("dm.blocksize=%s", options.Thinpool.BlockSize))
}
if options.Thinpool.DirectLvmDevice != "" {
doptions = append(doptions, fmt.Sprintf("dm.directlvm_device=%s", options.Thinpool.DirectLvmDevice))
}
if options.Thinpool.DirectLvmDeviceForce != "" {
doptions = append(doptions, fmt.Sprintf("dm.directlvm_device_force=%s", options.Thinpool.DirectLvmDeviceForce))
}
if options.Thinpool.Fs != "" {
doptions = append(doptions, fmt.Sprintf("dm.fs=%s", options.Thinpool.Fs))
}
if options.Thinpool.LogLevel != "" {
doptions = append(doptions, fmt.Sprintf("dm.libdm_log_level=%s", options.Thinpool.LogLevel))
}
if options.Thinpool.MetadataSize != "" {
doptions = append(doptions, fmt.Sprintf("dm.metadata_size=%s", options.Thinpool.MetadataSize))
}
if options.Thinpool.MinFreeSpace != "" {
doptions = append(doptions, fmt.Sprintf("dm.min_free_space=%s", options.Thinpool.MinFreeSpace))
}
if options.Thinpool.MkfsArg != "" {
doptions = append(doptions, fmt.Sprintf("dm.mkfsarg=%s", options.Thinpool.MkfsArg))
}
if options.Thinpool.MountOpt != "" {
doptions = append(doptions, fmt.Sprintf("%s.mountopt=%s", driverName, options.Thinpool.MountOpt))
} else if options.MountOpt != "" {
doptions = append(doptions, fmt.Sprintf("%s.mountopt=%s", driverName, options.MountOpt))
}
if options.Thinpool.Size != "" {
doptions = append(doptions, fmt.Sprintf("%s.size=%s", driverName, options.Thinpool.Size))
} else if options.Size != "" {
doptions = append(doptions, fmt.Sprintf("%s.size=%s", driverName, options.Size))
}
if options.Thinpool.UseDeferredDeletion != "" {
doptions = append(doptions, fmt.Sprintf("dm.use_deferred_deletion=%s", options.Thinpool.UseDeferredDeletion))
}
if options.Thinpool.UseDeferredRemoval != "" {
doptions = append(doptions, fmt.Sprintf("dm.use_deferred_removal=%s", options.Thinpool.UseDeferredRemoval))
}
if options.Thinpool.XfsNoSpaceMaxRetries != "" {
doptions = append(doptions, fmt.Sprintf("dm.xfs_nospace_max_retries=%s", options.Thinpool.XfsNoSpaceMaxRetries))
}
case "overlay", "overlay2":
// Specify whether composefs must be used to mount the data layers
if options.Overlay.IgnoreChownErrors != "" {

65
storage/pkg/config/config_test.go
View File

@ -60,71 +60,6 @@ func TestAufsOptions(t *testing.T) {
}
}
func TestDeviceMapperOptions(t *testing.T) {
var (
doptions []string
options OptionsConfig
)
doptions = GetGraphDriverOptions("devicemapper", options)
if len(doptions) != 0 {
t.Fatalf("Expected 0 options, got %v", doptions)
}
// Make sure legacy mountopt still works
options = OptionsConfig{}
options.MountOpt = foobar
doptions = GetGraphDriverOptions("devicemapper", options)
if len(doptions) == 0 {
t.Fatalf("Expected 0 options, got %v", doptions)
}
if !searchOptions(doptions, "mountopt=foobar") {
t.Fatalf("Expected to find 'foobar' options, got %v", doptions)
}
// Make sure Devicemapper ignores other drivers mountpoints takes presedence
options.Zfs.MountOpt = nodev
doptions = GetGraphDriverOptions("devicemapper", options)
if searchOptions(doptions, "mountopt=nodev") {
t.Fatalf("Expected to find 'nodev' options, got %v", doptions)
}
// Make sure DevicemapperMountOpt takes precedence
options.Thinpool.MountOpt = nodev
doptions = GetGraphDriverOptions("devicemapper", options)
if len(doptions) == 0 {
t.Fatalf("Expected 0 options, got %v", doptions)
}
if !searchOptions(doptions, "mountopt=nodev") {
t.Fatalf("Expected to find 'nodev' options, got %v", doptions)
}
options = OptionsConfig{}
options.Thinpool.AutoExtendPercent = "50"
doptions = GetGraphDriverOptions("devicemapper", options)
if len(doptions) == 0 {
t.Fatalf("Expected 0 options, got %v", doptions)
}
if !searchOptions(doptions, "50") {
t.Fatalf("Expected to find '50' options, got %v", doptions)
}
options.Size = s200
doptions = GetGraphDriverOptions("devicemapper", options)
if len(doptions) == 0 {
t.Fatalf("Expected 0 options, got %v", doptions)
}
if !searchOptions(doptions, s200) {
t.Fatalf("Expected to find size %q options, got %v", s200, doptions)
}
// Make sure Thinpool.Size takes precedence
options.Thinpool.Size = s100
doptions = GetGraphDriverOptions("devicemapper", options)
if len(doptions) == 0 {
t.Fatalf("Expected 0 options, got %v", doptions)
}
if !searchOptions(doptions, s100) {
t.Fatalf("Expected to find size %q, got %v", s100, doptions)
}
}
func TestBtrfsOptions(t *testing.T) {
var (
doptions []string

813
storage/pkg/devicemapper/devmapper.go
View File

@ -1,813 +0,0 @@
//go:build linux && cgo
// +build linux,cgo
package devicemapper
import (
"errors"
"fmt"
"os"
"runtime"
"unsafe"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
)
// Same as DM_DEVICE_* enum values from libdevmapper.h
// nolint: unused
const (
deviceCreate TaskType = iota
deviceReload
deviceRemove
deviceRemoveAll
deviceSuspend
deviceResume
deviceInfo
deviceDeps
deviceRename
deviceVersion
deviceStatus
deviceTable
deviceWaitevent
deviceList
deviceClear
deviceMknodes
deviceListVersions
deviceTargetMsg
deviceSetGeometry
)
const (
addNodeOnResume AddNodeType = iota
addNodeOnCreate
)
// List of errors returned when using devicemapper.
var (
ErrTaskRun = errors.New("dm_task_run failed")
ErrTaskSetName = errors.New("dm_task_set_name failed")
ErrTaskSetMessage = errors.New("dm_task_set_message failed")
ErrTaskSetAddNode = errors.New("dm_task_set_add_node failed")
ErrTaskSetRo = errors.New("dm_task_set_ro failed")
ErrTaskAddTarget = errors.New("dm_task_add_target failed")
ErrTaskSetSector = errors.New("dm_task_set_sector failed")
ErrTaskGetDeps = errors.New("dm_task_get_deps failed")
ErrTaskGetInfo = errors.New("dm_task_get_info failed")
ErrTaskGetDriverVersion = errors.New("dm_task_get_driver_version failed")
ErrTaskDeferredRemove = errors.New("dm_task_deferred_remove failed")
ErrTaskSetCookie = errors.New("dm_task_set_cookie failed")
ErrNilCookie = errors.New("cookie ptr can't be nil")
ErrGetBlockSize = errors.New("Can't get block size")
ErrUdevWait = errors.New("wait on udev cookie failed")
ErrSetDevDir = errors.New("dm_set_dev_dir failed")
ErrGetLibraryVersion = errors.New("dm_get_library_version failed")
ErrCreateRemoveTask = errors.New("Can't create task of type deviceRemove")
ErrRunRemoveDevice = errors.New("running RemoveDevice failed")
ErrInvalidAddNode = errors.New("Invalid AddNode type")
ErrBusy = errors.New("Device is Busy")
ErrDeviceIDExists = errors.New("Device Id Exists")
ErrEnxio = errors.New("No such device or address")
)
var (
dmSawBusy bool
dmSawExist bool
dmSawEnxio bool // No Such Device or Address
)
type (
// Task represents a devicemapper task (like lvcreate, etc.) ; a task is needed for each ioctl
// command to execute.
Task struct {
unmanaged *cdmTask
}
// Deps represents dependents (layer) of a device.
Deps struct {
Count uint32
Filler uint32
Device []uint64
}
// Info represents information about a device.
Info struct {
Exists int
Suspended int
LiveTable int
InactiveTable int
OpenCount int32
EventNr uint32
Major uint32
Minor uint32
ReadOnly int
TargetCount int32
DeferredRemove int
}
// TaskType represents a type of task
TaskType int
// AddNodeType represents a type of node to be added
AddNodeType int
)
// DeviceIDExists returns whether error conveys the information about device Id already
// exist or not. This will be true if device creation or snap creation
// operation fails if device or snap device already exists in pool.
// Current implementation is little crude as it scans the error string
// for exact pattern match. Replacing it with more robust implementation
// is desirable.
func DeviceIDExists(err error) bool {
return fmt.Sprint(err) == fmt.Sprint(ErrDeviceIDExists)
}
func (t *Task) destroy() {
if t != nil {
DmTaskDestroy(t.unmanaged)
runtime.SetFinalizer(t, nil)
}
}
// TaskCreateNamed is a convenience function for TaskCreate when a name
// will be set on the task as well
func TaskCreateNamed(t TaskType, name string) (*Task, error) {
task := TaskCreate(t)
if task == nil {
return nil, fmt.Errorf("devicemapper: Can't create task of type %d", int(t))
}
if err := task.setName(name); err != nil {
return nil, fmt.Errorf("devicemapper: Can't set task name %s", name)
}
return task, nil
}
// TaskCreate initializes a devicemapper task of tasktype
func TaskCreate(tasktype TaskType) *Task {
Ctask := DmTaskCreate(int(tasktype))
if Ctask == nil {
return nil
}
task := &Task{unmanaged: Ctask}
runtime.SetFinalizer(task, (*Task).destroy)
return task
}
func (t *Task) run() error {
if res := DmTaskRun(t.unmanaged); res != 1 {
return ErrTaskRun
}
runtime.KeepAlive(t)
return nil
}
func (t *Task) setName(name string) error {
if res := DmTaskSetName(t.unmanaged, name); res != 1 {
return ErrTaskSetName
}
return nil
}
func (t *Task) setMessage(message string) error {
if res := DmTaskSetMessage(t.unmanaged, message); res != 1 {
return ErrTaskSetMessage
}
return nil
}
func (t *Task) setSector(sector uint64) error {
if res := DmTaskSetSector(t.unmanaged, sector); res != 1 {
return ErrTaskSetSector
}
return nil
}
func (t *Task) setCookie(cookie *uint, flags uint16) error {
if cookie == nil {
return ErrNilCookie
}
if res := DmTaskSetCookie(t.unmanaged, cookie, flags); res != 1 {
return ErrTaskSetCookie
}
return nil
}
func (t *Task) setAddNode(addNode AddNodeType) error {
if addNode != addNodeOnResume && addNode != addNodeOnCreate {
return ErrInvalidAddNode
}
if res := DmTaskSetAddNode(t.unmanaged, addNode); res != 1 {
return ErrTaskSetAddNode
}
return nil
}
func (t *Task) addTarget(start, size uint64, ttype, params string) error {
if res := DmTaskAddTarget(t.unmanaged, start, size,
ttype, params); res != 1 {
return ErrTaskAddTarget
}
return nil
}
func (t *Task) getDeps() (*Deps, error) { //nolint:unused
var deps *Deps
if deps = DmTaskGetDeps(t.unmanaged); deps == nil {
return nil, ErrTaskGetDeps
}
return deps, nil
}
func (t *Task) getInfo() (*Info, error) {
info := &Info{}
if res := DmTaskGetInfo(t.unmanaged, info); res != 1 {
return nil, ErrTaskGetInfo
}
return info, nil
}
func (t *Task) getInfoWithDeferred() (*Info, error) {
info := &Info{}
if res := DmTaskGetInfoWithDeferred(t.unmanaged, info); res != 1 {
return nil, ErrTaskGetInfo
}
return info, nil
}
func (t *Task) getDriverVersion() (string, error) {
res := DmTaskGetDriverVersion(t.unmanaged)
if res == "" {
return "", ErrTaskGetDriverVersion
}
return res, nil
}
func (t *Task) getNextTarget(next unsafe.Pointer) (nextPtr unsafe.Pointer, start uint64,
length uint64, targetType string, params string,
) {
return DmGetNextTarget(t.unmanaged, next, &start, &length,
&targetType, &params),
start, length, targetType, params
}
// UdevWait waits for any processes that are waiting for udev to complete the specified cookie.
func UdevWait(cookie *uint) error {
if res := DmUdevWait(*cookie); res != 1 {
logrus.Debugf("devicemapper: Failed to wait on udev cookie %d, %d", *cookie, res)
return ErrUdevWait
}
return nil
}
// SetDevDir sets the dev folder for the device mapper library (usually /dev).
func SetDevDir(dir string) error {
if res := DmSetDevDir(dir); res != 1 {
logrus.Debug("devicemapper: Error dm_set_dev_dir")
return ErrSetDevDir
}
return nil
}
// GetLibraryVersion returns the device mapper library version.
func GetLibraryVersion() (string, error) {
var version string
if res := DmGetLibraryVersion(&version); res != 1 {
return "", ErrGetLibraryVersion
}
return version, nil
}
// UdevSyncSupported returns whether device-mapper is able to sync with udev
//
// This is essential otherwise race conditions can arise where both udev and
// device-mapper attempt to create and destroy devices.
func UdevSyncSupported() bool {
return DmUdevGetSyncSupport() != 0
}
// UdevSetSyncSupport allows setting whether the udev sync should be enabled.
// The return bool indicates the state of whether the sync is enabled.
func UdevSetSyncSupport(enable bool) bool {
if enable {
DmUdevSetSyncSupport(1)
} else {
DmUdevSetSyncSupport(0)
}
return UdevSyncSupported()
}
// CookieSupported returns whether the version of device-mapper supports the
// use of cookie's in the tasks.
// This is largely a lower level call that other functions use.
func CookieSupported() bool {
return DmCookieSupported() != 0
}
// RemoveDevice is a useful helper for cleaning up a device.
func RemoveDevice(name string) error {
task, err := TaskCreateNamed(deviceRemove, name)
if task == nil {
return err
}
cookie := new(uint)
if err := task.setCookie(cookie, 0); err != nil {
return fmt.Errorf("devicemapper: Can not set cookie: %s", err)
}
defer UdevWait(cookie)
dmSawBusy = false // reset before the task is run
dmSawEnxio = false
if err = task.run(); err != nil {
if dmSawBusy {
return ErrBusy
}
if dmSawEnxio {
return ErrEnxio
}
return fmt.Errorf("devicemapper: Error running RemoveDevice %s", err)
}
return nil
}
// RemoveDeviceDeferred is a useful helper for cleaning up a device, but deferred.
func RemoveDeviceDeferred(name string) error {
logrus.Debugf("devicemapper: RemoveDeviceDeferred START(%s)", name)
defer logrus.Debugf("devicemapper: RemoveDeviceDeferred END(%s)", name)
task, err := TaskCreateNamed(deviceRemove, name)
if task == nil {
return err
}
if err := DmTaskDeferredRemove(task.unmanaged); err != 1 {
return ErrTaskDeferredRemove
}
// set a task cookie and disable library fallback, or else libdevmapper will
// disable udev dm rules and delete the symlink under /dev/mapper by itself,
// even if the removal is deferred by the kernel.
cookie := new(uint)
flags := uint16(DmUdevDisableLibraryFallback)
if err := task.setCookie(cookie, flags); err != nil {
return fmt.Errorf("devicemapper: Can not set cookie: %s", err)
}
// libdevmapper and udev relies on System V semaphore for synchronization,
// semaphores created in `task.setCookie` will be cleaned up in `UdevWait`.
// So these two function call must come in pairs, otherwise semaphores will
// be leaked, and the limit of number of semaphores defined in `/proc/sys/kernel/sem`
// will be reached, which will eventually make all following calls to 'task.SetCookie'
// fail.
// this call will not wait for the deferred removal's final executing, since no
// udev event will be generated, and the semaphore's value will not be incremented
// by udev, what UdevWait is just cleaning up the semaphore.
defer UdevWait(cookie)
dmSawEnxio = false
if err = task.run(); err != nil {
if dmSawEnxio {
return ErrEnxio
}
return fmt.Errorf("devicemapper: Error running RemoveDeviceDeferred %s", err)
}
return nil
}
// CancelDeferredRemove cancels a deferred remove for a device.
func CancelDeferredRemove(deviceName string) error {
task, err := TaskCreateNamed(deviceTargetMsg, deviceName)
if task == nil {
return err
}
if err := task.setSector(0); err != nil {
return fmt.Errorf("devicemapper: Can't set sector %s", err)
}
if err := task.setMessage("@cancel_deferred_remove"); err != nil {
return fmt.Errorf("devicemapper: Can't set message %s", err)
}
dmSawBusy = false
dmSawEnxio = false
if err := task.run(); err != nil {
// A device might be being deleted already
if dmSawBusy {
return ErrBusy
} else if dmSawEnxio {
return ErrEnxio
}
return fmt.Errorf("devicemapper: Error running CancelDeferredRemove %s", err)
}
return nil
}
// GetBlockDeviceSize returns the size of a block device identified by the specified file.
func GetBlockDeviceSize(file *os.File) (uint64, error) {
size, err := ioctlBlkGetSize64(file.Fd())
if err != nil {
logrus.Errorf("devicemapper: Error getblockdevicesize: %s", err)
return 0, ErrGetBlockSize
}
return uint64(size), nil
}
// BlockDeviceDiscard runs discard for the given path.
// This is used as a workaround for the kernel not discarding block so
// on the thin pool when we remove a thinp device, so we do it
// manually
func BlockDeviceDiscard(path string) error {
file, err := os.OpenFile(path, os.O_RDWR, 0)
if err != nil {
return err
}
defer file.Close()
size, err := GetBlockDeviceSize(file)
if err != nil {
return err
}
if err := ioctlBlkDiscard(file.Fd(), 0, size); err != nil {
return err
}
// Without this sometimes the remove of the device that happens after
// discard fails with EBUSY.
unix.Sync()
return nil
}
// CreatePool is the programmatic example of "dmsetup create".
// It creates a device with the specified poolName, data and metadata file and block size.
func CreatePool(poolName string, dataFile, metadataFile *os.File, poolBlockSize uint32) error {
task, err := TaskCreateNamed(deviceCreate, poolName)
if task == nil {
return err
}
size, err := GetBlockDeviceSize(dataFile)
if err != nil {
return fmt.Errorf("devicemapper: Can't get data size %s", err)
}
params := fmt.Sprintf("%s %s %d 32768 1 skip_block_zeroing", metadataFile.Name(), dataFile.Name(), poolBlockSize)
if err := task.addTarget(0, size/512, "thin-pool", params); err != nil {
return fmt.Errorf("devicemapper: Can't add target %s", err)
}
cookie := new(uint)
flags := uint16(DmUdevDisableSubsystemRulesFlag | DmUdevDisableDiskRulesFlag | DmUdevDisableOtherRulesFlag)
if err := task.setCookie(cookie, flags); err != nil {
return fmt.Errorf("devicemapper: Can't set cookie %s", err)
}
defer UdevWait(cookie)
if err := task.run(); err != nil {
return fmt.Errorf("devicemapper: Error running deviceCreate (CreatePool) %s", err)
}
return nil
}
// ReloadPool is the programmatic example of "dmsetup reload".
// It reloads the table with the specified poolName, data and metadata file and block size.
func ReloadPool(poolName string, dataFile, metadataFile *os.File, poolBlockSize uint32) error {
task, err := TaskCreateNamed(deviceReload, poolName)
if task == nil {
return err
}
size, err := GetBlockDeviceSize(dataFile)
if err != nil {
return fmt.Errorf("devicemapper: Can't get data size %s", err)
}
params := fmt.Sprintf("%s %s %d 32768 1 skip_block_zeroing", metadataFile.Name(), dataFile.Name(), poolBlockSize)
if err := task.addTarget(0, size/512, "thin-pool", params); err != nil {
return fmt.Errorf("devicemapper: Can't add target %s", err)
}
if err := task.run(); err != nil {
return fmt.Errorf("devicemapper: Error running ReloadPool %s", err)
}
return nil
}
// GetDeps is the programmatic example of "dmsetup deps".
// It outputs a list of devices referenced by the live table for the specified device.
func GetDeps(name string) (*Deps, error) {
task, err := TaskCreateNamed(deviceDeps, name)
if task == nil {
return nil, err
}
if err := task.run(); err != nil {
return nil, err
}
return task.getDeps()
}
// GetInfo is the programmatic example of "dmsetup info".
// It outputs some brief information about the device.
func GetInfo(name string) (*Info, error) {
task, err := TaskCreateNamed(deviceInfo, name)
if task == nil {
return nil, err
}
if err := task.run(); err != nil {
return nil, err
}
return task.getInfo()
}
// GetInfoWithDeferred is the programmatic example of "dmsetup info", but deferred.
// It outputs some brief information about the device.
func GetInfoWithDeferred(name string) (*Info, error) {
task, err := TaskCreateNamed(deviceInfo, name)
if task == nil {
return nil, err
}
if err := task.run(); err != nil {
return nil, err
}
return task.getInfoWithDeferred()
}
// GetDriverVersion is the programmatic example of "dmsetup version".
// It outputs version information of the driver.
func GetDriverVersion() (string, error) {
task := TaskCreate(deviceVersion)
if task == nil {
return "", fmt.Errorf("devicemapper: Can't create deviceVersion task")
}
if err := task.run(); err != nil {
return "", err
}
return task.getDriverVersion()
}
// GetStatus is the programmatic example of "dmsetup status".
// It outputs status information for the specified device name.
func GetStatus(name string) (uint64, uint64, string, string, error) {
task, err := TaskCreateNamed(deviceStatus, name)
if task == nil {
logrus.Debugf("devicemapper: GetStatus() Error TaskCreateNamed: %s", err)
return 0, 0, "", "", err
}
if err := task.run(); err != nil {
logrus.Debugf("devicemapper: GetStatus() Error Run: %s", err)
return 0, 0, "", "", err
}
devinfo, err := task.getInfo()
if err != nil {
logrus.Debugf("devicemapper: GetStatus() Error GetInfo: %s", err)
return 0, 0, "", "", err
}
if devinfo.Exists == 0 {
logrus.Debugf("devicemapper: GetStatus() Non existing device %s", name)
return 0, 0, "", "", fmt.Errorf("devicemapper: Non existing device %s", name)
}
_, start, length, targetType, params := task.getNextTarget(unsafe.Pointer(nil))
return start, length, targetType, params, nil
}
// GetTable is the programmatic example for "dmsetup table".
// It outputs the current table for the specified device name.
func GetTable(name string) (uint64, uint64, string, string, error) {
task, err := TaskCreateNamed(deviceTable, name)
if task == nil {
logrus.Debugf("devicemapper: GetTable() Error TaskCreateNamed: %s", err)
return 0, 0, "", "", err
}
if err := task.run(); err != nil {
logrus.Debugf("devicemapper: GetTable() Error Run: %s", err)
return 0, 0, "", "", err
}
devinfo, err := task.getInfo()
if err != nil {
logrus.Debugf("devicemapper: GetTable() Error GetInfo: %s", err)
return 0, 0, "", "", err
}
if devinfo.Exists == 0 {
logrus.Debugf("devicemapper: GetTable() Non existing device %s", name)
return 0, 0, "", "", fmt.Errorf("devicemapper: Non existing device %s", name)
}
_, start, length, targetType, params := task.getNextTarget(unsafe.Pointer(nil))
return start, length, targetType, params, nil
}
// SetTransactionID sets a transaction id for the specified device name.
func SetTransactionID(poolName string, oldID uint64, newID uint64) error {
task, err := TaskCreateNamed(deviceTargetMsg, poolName)
if task == nil {
return err
}
if err := task.setSector(0); err != nil {
return fmt.Errorf("devicemapper: Can't set sector %s", err)
}
if err := task.setMessage(fmt.Sprintf("set_transaction_id %d %d", oldID, newID)); err != nil {
return fmt.Errorf("devicemapper: Can't set message %s", err)
}
if err := task.run(); err != nil {
return fmt.Errorf("devicemapper: Error running SetTransactionID %s", err)
}
return nil
}
// SuspendDevice is the programmatic example of "dmsetup suspend".
// It suspends the specified device.
func SuspendDevice(name string) error {
task, err := TaskCreateNamed(deviceSuspend, name)
if task == nil {
return err
}
if err := task.run(); err != nil {
return fmt.Errorf("devicemapper: Error running deviceSuspend %s", err)
}
return nil
}
// ResumeDevice is the programmatic example of "dmsetup resume".
// It un-suspends the specified device.
func ResumeDevice(name string) error {
task, err := TaskCreateNamed(deviceResume, name)
if task == nil {
return err
}
cookie := new(uint)
if err := task.setCookie(cookie, 0); err != nil {
return fmt.Errorf("devicemapper: Can't set cookie %s", err)
}
defer UdevWait(cookie)
if err := task.run(); err != nil {
return fmt.Errorf("devicemapper: Error running deviceResume %s", err)
}
return nil
}
// CreateDevice creates a device with the specified poolName with the specified device id.
func CreateDevice(poolName string, deviceID int) error {
logrus.Debugf("devicemapper: CreateDevice(poolName=%v, deviceID=%v)", poolName, deviceID)
task, err := TaskCreateNamed(deviceTargetMsg, poolName)
if task == nil {
return err
}
if err := task.setSector(0); err != nil {
return fmt.Errorf("devicemapper: Can't set sector %s", err)
}
if err := task.setMessage(fmt.Sprintf("create_thin %d", deviceID)); err != nil {
return fmt.Errorf("devicemapper: Can't set message %s", err)
}
dmSawExist = false // reset before the task is run
if err := task.run(); err != nil {
// Caller wants to know about ErrDeviceIDExists so that it can try with a different device id.
if dmSawExist {
return ErrDeviceIDExists
}
return fmt.Errorf("devicemapper: Error running CreateDevice %s", err)
}
return nil
}
// DeleteDevice deletes a device with the specified poolName with the specified device id.
func DeleteDevice(poolName string, deviceID int) error {
task, err := TaskCreateNamed(deviceTargetMsg, poolName)
if task == nil {
return err
}
if err := task.setSector(0); err != nil {
return fmt.Errorf("devicemapper: Can't set sector %s", err)
}
if err := task.setMessage(fmt.Sprintf("delete %d", deviceID)); err != nil {
return fmt.Errorf("devicemapper: Can't set message %s", err)
}
dmSawBusy = false
if err := task.run(); err != nil {
if dmSawBusy {
return ErrBusy
}
return fmt.Errorf("devicemapper: Error running DeleteDevice %s", err)
}
return nil
}
// ActivateDevice activates the device identified by the specified
// poolName, name and deviceID with the specified size.
func ActivateDevice(poolName string, name string, deviceID int, size uint64) error {
return activateDevice(poolName, name, deviceID, size, "")
}
// ActivateDeviceWithExternal activates the device identified by the specified
// poolName, name and deviceID with the specified size.
func ActivateDeviceWithExternal(poolName string, name string, deviceID int, size uint64, external string) error {
return activateDevice(poolName, name, deviceID, size, external)
}
func activateDevice(poolName string, name string, deviceID int, size uint64, external string) error {
task, err := TaskCreateNamed(deviceCreate, name)
if task == nil {
return err
}
var params string
if len(external) > 0 {
params = fmt.Sprintf("%s %d %s", poolName, deviceID, external)
} else {
params = fmt.Sprintf("%s %d", poolName, deviceID)
}
if err := task.addTarget(0, size/512, "thin", params); err != nil {
return fmt.Errorf("devicemapper: Can't add target %s", err)
}
if err := task.setAddNode(addNodeOnCreate); err != nil {
return fmt.Errorf("devicemapper: Can't add node %s", err)
}
cookie := new(uint)
if err := task.setCookie(cookie, 0); err != nil {
return fmt.Errorf("devicemapper: Can't set cookie %s", err)
}
defer UdevWait(cookie)
if err := task.run(); err != nil {
return fmt.Errorf("devicemapper: Error running deviceCreate (ActivateDevice) %s", err)
}
return nil
}
// CreateSnapDeviceRaw creates a snapshot device. Caller needs to suspend and resume the origin device if it is active.
func CreateSnapDeviceRaw(poolName string, deviceID int, baseDeviceID int) error {
task, err := TaskCreateNamed(deviceTargetMsg, poolName)
if task == nil {
return err
}
if err := task.setSector(0); err != nil {
return fmt.Errorf("devicemapper: Can't set sector %s", err)
}
if err := task.setMessage(fmt.Sprintf("create_snap %d %d", deviceID, baseDeviceID)); err != nil {
return fmt.Errorf("devicemapper: Can't set message %s", err)
}
dmSawExist = false // reset before the task is run
if err := task.run(); err != nil {
// Caller wants to know about ErrDeviceIDExists so that it can try with a different device id.
if dmSawExist {
return ErrDeviceIDExists
}
return fmt.Errorf("devicemapper: Error running deviceCreate (CreateSnapDeviceRaw) %s", err)
}
return nil
}
// CreateSnapDevice creates a snapshot based on the device identified by the baseName and baseDeviceId,
func CreateSnapDevice(poolName string, deviceID int, baseName string, baseDeviceID int) error {
devinfo, _ := GetInfo(baseName)
doSuspend := devinfo != nil && devinfo.Exists != 0
if doSuspend {
if err := SuspendDevice(baseName); err != nil {
return err
}
}
if err := CreateSnapDeviceRaw(poolName, deviceID, baseDeviceID); err != nil {
if doSuspend {
if err2 := ResumeDevice(baseName); err2 != nil {
return fmt.Errorf("CreateSnapDeviceRaw Error: (%v): ResumeDevice Error: %w", err, err2)
}
}
return err
}
if doSuspend {
if err := ResumeDevice(baseName); err != nil {
return err
}
}
return nil
}

123
storage/pkg/devicemapper/devmapper_log.go
View File

@ -1,123 +0,0 @@
//go:build linux && cgo
// +build linux,cgo
package devicemapper
import "C"
import (
"fmt"
"strings"
"github.com/sirupsen/logrus"
)
// DevmapperLogger defines methods required to register as a callback for
// logging events received from devicemapper. Note that devicemapper will send
// *all* logs regardless to callbacks (including debug logs) so it's
// recommended to not spam the console with the outputs.
type DevmapperLogger interface {
// DMLog is the logging callback containing all of the information from
// devicemapper. The interface is identical to the C libdm counterpart.
DMLog(level int, file string, line int, dmError int, message string)
}
// dmLogger is the current logger in use that is being forwarded our messages.
var dmLogger DevmapperLogger
// LogInit changes the logging callback called after processing libdm logs for
// error message information. The default logger simply forwards all logs to
// logrus. Calling LogInit(nil) disables the calling of callbacks.
func LogInit(logger DevmapperLogger) {
dmLogger = logger
}
// Due to the way cgo works this has to be in a separate file, as devmapper.go has
// definitions in the cgo block, which is incompatible with using "//export"
// StorageDevmapperLogCallback exports the devmapper log callback for cgo. Note that
// because we are using callbacks, this function will be called for *every* log
// in libdm (even debug ones because there's no way of setting the verbosity
// level for an external logging callback).
//
//export StorageDevmapperLogCallback
func StorageDevmapperLogCallback(level C.int, file *C.char, line, dmErrnoOrClass C.int, message *C.char) {
msg := C.GoString(message)
// Track what errno libdm saw, because the library only gives us 0 or 1.
if level < LogLevelDebug {
if strings.Contains(msg, "busy") {
dmSawBusy = true
}
if strings.Contains(msg, "File exists") {
dmSawExist = true
}
if strings.Contains(msg, "No such device or address") {
dmSawEnxio = true
}
}
if dmLogger != nil {
dmLogger.DMLog(int(level), C.GoString(file), int(line), int(dmErrnoOrClass), msg)
}
}
// DefaultLogger is the default logger used by pkg/devicemapper. It forwards
// all logs that are of higher or equal priority to the given level to the
// corresponding logrus level.
type DefaultLogger struct {
// Level corresponds to the highest libdm level that will be forwarded to
// logrus. In order to change this, register a new DefaultLogger.
Level int
}
// DMLog is the logging callback containing all of the information from
// devicemapper. The interface is identical to the C libdm counterpart.
func (l DefaultLogger) DMLog(level int, file string, line, dmError int, message string) {
if level <= l.Level {
// Forward the log to the correct logrus level, if allowed by dmLogLevel.
logMsg := fmt.Sprintf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
switch level {
case LogLevelFatal, LogLevelErr:
logrus.Error(logMsg)
case LogLevelWarn:
logrus.Warn(logMsg)
case LogLevelNotice, LogLevelInfo:
logrus.Info(logMsg)
case LogLevelDebug:
logrus.Debug(logMsg)
default:
// Don't drop any "unknown" levels.
logrus.Info(logMsg)
}
}
}
// registerLogCallback registers our own logging callback function for libdm
// (which is StorageDevmapperLogCallback).
//
// Because libdm only gives us {0,1} error codes we need to parse the logs
// produced by libdm (to set dmSawBusy and so on). Note that by registering a
// callback using StorageDevmapperLogCallback, libdm will no longer output logs to
// stderr so we have to log everything ourselves. None of this handling is
// optional because we depend on log callbacks to parse the logs, and if we
// don't forward the log information we'll be in a lot of trouble when
// debugging things.
func registerLogCallback() {
LogWithErrnoInit()
}
func init() {
// Use the default logger by default. We only allow LogLevelFatal by
// default, because internally we mask a lot of libdm errors by retrying
// and similar tricks. Also, libdm is very chatty and we don't want to
// worry users for no reason.
dmLogger = DefaultLogger{
Level: LogLevelFatal,
}
// Register as early as possible so we don't miss anything.
registerLogCallback()
}

252
storage/pkg/devicemapper/devmapper_wrapper.go
View File

@ -1,252 +0,0 @@
//go:build linux && cgo
// +build linux,cgo
package devicemapper
/*
#define _GNU_SOURCE
#include <libdevmapper.h>
#include <linux/fs.h> // FIXME: present only for BLKGETSIZE64, maybe we can remove it?
// FIXME: Can't we find a way to do the logging in pure Go?
extern void StorageDevmapperLogCallback(int level, char *file, int line, int dm_errno_or_class, char *str);
static void log_cb(int level, const char *file, int line, int dm_errno_or_class, const char *f, ...)
{
char *buffer = NULL;
va_list ap;
int ret;
va_start(ap, f);
ret = vasprintf(&buffer, f, ap);
va_end(ap);
if (ret < 0) {
// memory allocation failed -- should never happen?
return;
}
StorageDevmapperLogCallback(level, (char *)file, line, dm_errno_or_class, buffer);
free(buffer);
}
static void log_with_errno_init()
{
dm_log_with_errno_init(log_cb);
}
*/
import "C"
import (
"reflect"
"unsafe"
)
type (
cdmTask C.struct_dm_task
)
// IOCTL consts
const (
BlkGetSize64 = C.BLKGETSIZE64
BlkDiscard = C.BLKDISCARD
)
// Devicemapper cookie flags.
const (
DmUdevDisableSubsystemRulesFlag = C.DM_UDEV_DISABLE_SUBSYSTEM_RULES_FLAG
DmUdevDisableDiskRulesFlag = C.DM_UDEV_DISABLE_DISK_RULES_FLAG
DmUdevDisableOtherRulesFlag = C.DM_UDEV_DISABLE_OTHER_RULES_FLAG
DmUdevDisableLibraryFallback = C.DM_UDEV_DISABLE_LIBRARY_FALLBACK
)
// DeviceMapper mapped functions.
var (
DmGetLibraryVersion = dmGetLibraryVersionFct
DmGetNextTarget = dmGetNextTargetFct
DmSetDevDir = dmSetDevDirFct
DmTaskAddTarget = dmTaskAddTargetFct
DmTaskCreate = dmTaskCreateFct
DmTaskDestroy = dmTaskDestroyFct
DmTaskGetDeps = dmTaskGetDepsFct
DmTaskGetInfo = dmTaskGetInfoFct
DmTaskGetDriverVersion = dmTaskGetDriverVersionFct
DmTaskRun = dmTaskRunFct
DmTaskSetAddNode = dmTaskSetAddNodeFct
DmTaskSetCookie = dmTaskSetCookieFct
DmTaskSetMessage = dmTaskSetMessageFct
DmTaskSetName = dmTaskSetNameFct
DmTaskSetRo = dmTaskSetRoFct
DmTaskSetSector = dmTaskSetSectorFct
DmUdevWait = dmUdevWaitFct
DmUdevSetSyncSupport = dmUdevSetSyncSupportFct
DmUdevGetSyncSupport = dmUdevGetSyncSupportFct
DmCookieSupported = dmCookieSupportedFct
LogWithErrnoInit = logWithErrnoInitFct
DmTaskDeferredRemove = dmTaskDeferredRemoveFct
DmTaskGetInfoWithDeferred = dmTaskGetInfoWithDeferredFct
)
func free(p *C.char) {
C.free(unsafe.Pointer(p))
}
func dmTaskDestroyFct(task *cdmTask) {
C.dm_task_destroy((*C.struct_dm_task)(task))
}
func dmTaskCreateFct(taskType int) *cdmTask {
return (*cdmTask)(C.dm_task_create(C.int(taskType)))
}
func dmTaskRunFct(task *cdmTask) int {
ret, _ := C.dm_task_run((*C.struct_dm_task)(task))
return int(ret)
}
func dmTaskSetNameFct(task *cdmTask, name string) int {
Cname := C.CString(name)
defer free(Cname)
return int(C.dm_task_set_name((*C.struct_dm_task)(task), Cname))
}
func dmTaskSetMessageFct(task *cdmTask, message string) int {
Cmessage := C.CString(message)
defer free(Cmessage)
return int(C.dm_task_set_message((*C.struct_dm_task)(task), Cmessage))
}
func dmTaskSetSectorFct(task *cdmTask, sector uint64) int {
return int(C.dm_task_set_sector((*C.struct_dm_task)(task), C.uint64_t(sector)))
}
func dmTaskSetCookieFct(task *cdmTask, cookie *uint, flags uint16) int {
cCookie := C.uint32_t(*cookie)
defer func() {
*cookie = uint(cCookie)
}()
return int(C.dm_task_set_cookie((*C.struct_dm_task)(task), &cCookie, C.uint16_t(flags)))
}
func dmTaskSetAddNodeFct(task *cdmTask, addNode AddNodeType) int {
return int(C.dm_task_set_add_node((*C.struct_dm_task)(task), C.dm_add_node_t(addNode)))
}
func dmTaskSetRoFct(task *cdmTask) int {
return int(C.dm_task_set_ro((*C.struct_dm_task)(task)))
}
func dmTaskAddTargetFct(task *cdmTask,
start, size uint64, ttype, params string,
) int {
Cttype := C.CString(ttype)
defer free(Cttype)
Cparams := C.CString(params)
defer free(Cparams)
return int(C.dm_task_add_target((*C.struct_dm_task)(task), C.uint64_t(start), C.uint64_t(size), Cttype, Cparams))
}
func dmTaskGetDepsFct(task *cdmTask) *Deps {
Cdeps := C.dm_task_get_deps((*C.struct_dm_task)(task))
if Cdeps == nil {
return nil
}
// golang issue: https://github.com/golang/go/issues/11925
var devices []C.uint64_t
devicesHdr := (*reflect.SliceHeader)(unsafe.Pointer(&devices))
devicesHdr.Data = uintptr(unsafe.Pointer(uintptr(unsafe.Pointer(Cdeps)) + unsafe.Sizeof(*Cdeps)))
devicesHdr.Len = int(Cdeps.count)
devicesHdr.Cap = int(Cdeps.count)
deps := &Deps{
Count: uint32(Cdeps.count),
Filler: uint32(Cdeps.filler),
}
for _, device := range devices {
deps.Device = append(deps.Device, uint64(device))
}
return deps
}
func dmTaskGetInfoFct(task *cdmTask, info *Info) int {
Cinfo := C.struct_dm_info{}
defer func() {
info.Exists = int(Cinfo.exists)
info.Suspended = int(Cinfo.suspended)
info.LiveTable = int(Cinfo.live_table)
info.InactiveTable = int(Cinfo.inactive_table)
info.OpenCount = int32(Cinfo.open_count)
info.EventNr = uint32(Cinfo.event_nr)
info.Major = uint32(Cinfo.major)
info.Minor = uint32(Cinfo.minor)
info.ReadOnly = int(Cinfo.read_only)
info.TargetCount = int32(Cinfo.target_count)
}()
return int(C.dm_task_get_info((*C.struct_dm_task)(task), &Cinfo))
}
func dmTaskGetDriverVersionFct(task *cdmTask) string {
buffer := C.malloc(128)
defer C.free(buffer)
res := C.dm_task_get_driver_version((*C.struct_dm_task)(task), (*C.char)(buffer), 128)
if res == 0 {
return ""
}
return C.GoString((*C.char)(buffer))
}
func dmGetNextTargetFct(task *cdmTask, next unsafe.Pointer, start, length *uint64, target, params *string) unsafe.Pointer {
var (
Cstart, Clength C.uint64_t
CtargetType, Cparams *C.char
)
defer func() {
*start = uint64(Cstart)
*length = uint64(Clength)
*target = C.GoString(CtargetType)
*params = C.GoString(Cparams)
}()
nextp := C.dm_get_next_target((*C.struct_dm_task)(task), next, &Cstart, &Clength, &CtargetType, &Cparams)
return nextp
}
func dmUdevSetSyncSupportFct(syncWithUdev int) {
(C.dm_udev_set_sync_support(C.int(syncWithUdev)))
}
func dmUdevGetSyncSupportFct() int {
return int(C.dm_udev_get_sync_support())
}
func dmUdevWaitFct(cookie uint) int {
return int(C.dm_udev_wait(C.uint32_t(cookie)))
}
func dmCookieSupportedFct() int {
return int(C.dm_cookie_supported())
}
func logWithErrnoInitFct() {
C.log_with_errno_init()
}
func dmSetDevDirFct(dir string) int {
Cdir := C.CString(dir)
defer free(Cdir)
return int(C.dm_set_dev_dir(Cdir))
}
func dmGetLibraryVersionFct(version *string) int {
buffer := C.CString(string(make([]byte, 128)))
defer free(buffer)
defer func() {
*version = C.GoString(buffer)
}()
return int(C.dm_get_library_version(buffer, 128))
}

32
storage/pkg/devicemapper/devmapper_wrapper_deferred_remove.go
View File

@ -1,32 +0,0 @@
//go:build linux && cgo && !libdm_no_deferred_remove
// +build linux,cgo,!libdm_no_deferred_remove
package devicemapper
// #include <libdevmapper.h>
import "C"
// LibraryDeferredRemovalSupport tells if the feature is enabled in the build
const LibraryDeferredRemovalSupport = true
func dmTaskDeferredRemoveFct(task *cdmTask) int {
return int(C.dm_task_deferred_remove((*C.struct_dm_task)(task)))
}
func dmTaskGetInfoWithDeferredFct(task *cdmTask, info *Info) int {
Cinfo := C.struct_dm_info{}
defer func() {
info.Exists = int(Cinfo.exists)
info.Suspended = int(Cinfo.suspended)
info.LiveTable = int(Cinfo.live_table)
info.InactiveTable = int(Cinfo.inactive_table)
info.OpenCount = int32(Cinfo.open_count)
info.EventNr = uint32(Cinfo.event_nr)
info.Major = uint32(Cinfo.major)
info.Minor = uint32(Cinfo.minor)
info.ReadOnly = int(Cinfo.read_only)
info.TargetCount = int32(Cinfo.target_count)
info.DeferredRemove = int(Cinfo.deferred_remove)
}()
return int(C.dm_task_get_info((*C.struct_dm_task)(task), &Cinfo))
}

7
storage/pkg/devicemapper/devmapper_wrapper_dynamic.go
View File

@ -1,7 +0,0 @@
//go:build linux && cgo && !static_build
// +build linux,cgo,!static_build
package devicemapper
// #cgo pkg-config: devmapper
import "C"

16
storage/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go
View File

@ -1,16 +0,0 @@
//go:build linux && cgo && libdm_no_deferred_remove
// +build linux,cgo,libdm_no_deferred_remove
package devicemapper
// LibraryDeferredRemovalSupport tells if the feature is enabled in the build
const LibraryDeferredRemovalSupport = false
func dmTaskDeferredRemoveFct(task *cdmTask) int {
// Error. Nobody should be calling it.
return -1
}
func dmTaskGetInfoWithDeferredFct(task *cdmTask, info *Info) int {
return -1
}

7
storage/pkg/devicemapper/devmapper_wrapper_static.go
View File

@ -1,7 +0,0 @@
//go:build linux && cgo && static_build
// +build linux,cgo,static_build
package devicemapper
// #cgo pkg-config: --static devmapper
import "C"

29
storage/pkg/devicemapper/ioctl.go
View File

@ -1,29 +0,0 @@
//go:build linux && cgo
// +build linux,cgo
package devicemapper
import (
"unsafe"
"golang.org/x/sys/unix"
)
func ioctlBlkGetSize64(fd uintptr) (int64, error) {
var size int64
if _, _, err := unix.Syscall(unix.SYS_IOCTL, fd, BlkGetSize64, uintptr(unsafe.Pointer(&size))); err != 0 {
return 0, err
}
return size, nil
}
func ioctlBlkDiscard(fd uintptr, offset, length uint64) error {
var r [2]uint64
r[0] = offset
r[1] = length
if _, _, err := unix.Syscall(unix.SYS_IOCTL, fd, BlkDiscard, uintptr(unsafe.Pointer(&r[0]))); err != 0 {
return err
}
return nil
}

11
storage/pkg/devicemapper/log.go
View File

@ -1,11 +0,0 @@
package devicemapper
// definitions from lvm2 lib/log/log.h
const (
LogLevelFatal = 2 + iota // _LOG_FATAL
LogLevelErr // _LOG_ERR
LogLevelWarn // _LOG_WARN
LogLevelNotice // _LOG_NOTICE
LogLevelInfo // _LOG_INFO
LogLevelDebug // _LOG_DEBUG
)

76
storage/storage.conf
View File

@ -173,79 +173,3 @@ mountopt = "nodev"
# "force_mask" permissions.
#
# force_mask = ""
[storage.options.thinpool]
# Storage Options for thinpool
# autoextend_percent determines the amount by which pool needs to be
# grown. This is specified in terms of % of pool size. So a value of 20 means
# that when threshold is hit, pool will be grown by 20% of existing
# pool size.
# autoextend_percent = "20"
# autoextend_threshold determines the pool extension threshold in terms
# of percentage of pool size. For example, if threshold is 60, that means when
# pool is 60% full, threshold has been hit.
# autoextend_threshold = "80"
# basesize specifies the size to use when creating the base device, which
# limits the size of images and containers.
# basesize = "10G"
# blocksize specifies a custom blocksize to use for the thin pool.
# blocksize="64k"
# directlvm_device specifies a custom block storage device to use for the
# thin pool. Required if you setup devicemapper.
# directlvm_device = ""
# directlvm_device_force wipes device even if device already has a filesystem.
# directlvm_device_force = "True"
# fs specifies the filesystem type to use for the base device.
# fs="xfs"
# log_level sets the log level of devicemapper.
# 0: LogLevelSuppress 0 (Default)
# 2: LogLevelFatal
# 3: LogLevelErr
# 4: LogLevelWarn
# 5: LogLevelNotice
# 6: LogLevelInfo
# 7: LogLevelDebug
# log_level = "7"
# min_free_space specifies the min free space percent in a thin pool require for
# new device creation to succeed. Valid values are from 0% - 99%.
# Value 0% disables
# min_free_space = "10%"
# mkfsarg specifies extra mkfs arguments to be used when creating the base
# device.
# mkfsarg = ""
# metadata_size is used to set the `pvcreate --metadatasize` options when
# creating thin devices. Default is 128k
# metadata_size = ""
# Size is used to set a maximum size of the container image.
# size = ""
# use_deferred_removal marks devicemapper block device for deferred removal.
# If the thinpool is in use when the driver attempts to remove it, the driver
# tells the kernel to remove it as soon as possible. Note this does not free
# up the disk space, use deferred deletion to fully remove the thinpool.
# use_deferred_removal = "True"
# use_deferred_deletion marks thinpool device for deferred deletion.
# If the device is busy when the driver attempts to delete it, the driver
# will attempt to delete device every 30 seconds until successful.
# If the program using the driver exits, the driver will continue attempting
# to cleanup the next time the driver is used. Deferred deletion permanently
# deletes the device and all data stored in device will be lost.
# use_deferred_deletion = "True"
# xfs_nospace_max_retries specifies the maximum number of retries XFS should
# attempt to complete IO when ENOSPC (no space) error is returned by
# underlying storage device.
# xfs_nospace_max_retries = "0"

76
storage/storage.conf-freebsd
View File

@ -134,79 +134,3 @@ mountopt = "nodev"
# "force_mask" permissions.
#
# force_mask = ""
[storage.options.thinpool]
# Storage Options for thinpool
# autoextend_percent determines the amount by which pool needs to be
# grown. This is specified in terms of % of pool size. So a value of 20 means
# that when threshold is hit, pool will be grown by 20% of existing
# pool size.
# autoextend_percent = "20"
# autoextend_threshold determines the pool extension threshold in terms
# of percentage of pool size. For example, if threshold is 60, that means when
# pool is 60% full, threshold has been hit.
# autoextend_threshold = "80"
# basesize specifies the size to use when creating the base device, which
# limits the size of images and containers.
# basesize = "10G"
# blocksize specifies a custom blocksize to use for the thin pool.
# blocksize="64k"
# directlvm_device specifies a custom block storage device to use for the
# thin pool. Required if you setup devicemapper.
# directlvm_device = ""
# directlvm_device_force wipes device even if device already has a filesystem.
# directlvm_device_force = "True"
# fs specifies the filesystem type to use for the base device.
# fs="xfs"
# log_level sets the log level of devicemapper.
# 0: LogLevelSuppress 0 (Default)
# 2: LogLevelFatal
# 3: LogLevelErr
# 4: LogLevelWarn
# 5: LogLevelNotice
# 6: LogLevelInfo
# 7: LogLevelDebug
# log_level = "7"
# min_free_space specifies the min free space percent in a thin pool require for
# new device creation to succeed. Valid values are from 0% - 99%.
# Value 0% disables
# min_free_space = "10%"
# mkfsarg specifies extra mkfs arguments to be used when creating the base
# device.
# mkfsarg = ""
# metadata_size is used to set the `pvcreate --metadatasize` options when
# creating thin devices. Default is 128k
# metadata_size = ""
# Size is used to set a maximum size of the container image.
# size = ""
# use_deferred_removal marks devicemapper block device for deferred removal.
# If the thinpool is in use when the driver attempts to remove it, the driver
# tells the kernel to remove it as soon as possible. Note this does not free
# up the disk space, use deferred deletion to fully remove the thinpool.
# use_deferred_removal = "True"
# use_deferred_deletion marks thinpool device for deferred deletion.
# If the device is busy when the driver attempts to delete it, the driver
# will attempt to delete device every 30 seconds until successful.
# If the program using the driver exits, the driver will continue attempting
# to cleanup the next time the driver is used. Deferred deletion permanently
# deletes the device and all data stored in device will be lost.
# use_deferred_deletion = "True"
# xfs_nospace_max_retries specifies the maximum number of retries XFS should
# attempt to complete IO when ENOSPC (no space) error is returned by
# underlying storage device.
# xfs_nospace_max_retries = "0"

2
storage/tests/delete-layer.bats
View File

@ -70,7 +70,7 @@ load helpers
@test "delete-layer-with-mappings" {
case "$STORAGE_DRIVER" in
btrfs|devicemapper|overlay*|vfs|zfs)
btrfs|overlay*|vfs|zfs)
;;
*)
skip "not supported by driver $STORAGE_DRIVER"

12
storage/tests/idmaps.bats
View File

@ -12,7 +12,7 @@ load helpers
skip "not supported on $OS"
fi
case "$STORAGE_DRIVER" in
btrfs|devicemapper|overlay*|vfs|zfs)
btrfs|overlay*|vfs|zfs)
;;
*)
skip "not supported by driver $STORAGE_DRIVER"
@ -126,7 +126,7 @@ load helpers
skip "not supported on $OS"
fi
case "$STORAGE_DRIVER" in
btrfs|devicemapper|overlay*|vfs|zfs)
btrfs|overlay*|vfs|zfs)
;;
*)
skip "not supported by driver $STORAGE_DRIVER"
@ -220,7 +220,7 @@ load helpers
skip "not supported on $OS"
fi
case "$STORAGE_DRIVER" in
btrfs|devicemapper|overlay*|vfs|zfs)
btrfs|overlay*|vfs|zfs)
;;
*)
skip "not supported by driver $STORAGE_DRIVER"
@ -354,7 +354,7 @@ load helpers
skip "not supported on $OS"
fi
case "$STORAGE_DRIVER" in
btrfs|devicemapper|overlay*|vfs|zfs)
btrfs|overlay*|vfs|zfs)
;;
*)
skip "not supported by driver $STORAGE_DRIVER"
@ -419,7 +419,7 @@ load helpers
skip "not supported on $OS"
fi
case "$STORAGE_DRIVER" in
btrfs|devicemapper|overlay*|vfs|zfs)
btrfs|overlay*|vfs|zfs)
;;
*)
skip "not supported by driver $STORAGE_DRIVER"
@ -593,7 +593,7 @@ load helpers
skip "not supported on $OS"
fi
case "$STORAGE_DRIVER" in
btrfs|devicemapper|overlay*|vfs|zfs)
btrfs|overlay*|vfs|zfs)
;;
*)
skip "not supported by driver $STORAGE_DRIVER"

12
storage/tests/test_drivers.bash
View File

@ -11,15 +11,6 @@ btrfs() {
[ $(stat -f -c %T ${TMPDIR}) = btrfs ]
}
devicemapper() {
for binary in pvcreate vgcreate lvcreate lvconvert lvchange thin_check ; do
if ! which $binary > /dev/null 2> /dev/null ; then
return 1
fi
done
pkg-config devmapper 2> /dev/null
}
overlay() {
modprobe overlay 2> /dev/null
grep -E -q ' overlay$' /proc/filesystems
@ -37,9 +28,6 @@ if [ "$STORAGE_DRIVER" = "" ] ; then
if btrfs; then
drivers="$drivers btrfs"
fi
if devicemapper; then
drivers="$drivers devicemapper"
fi
if overlay; then
drivers="$drivers overlay"
fi

8
storage/types/options_test.go
View File

@ -83,14 +83,6 @@ func TestGetRootlessStorageOpts(t *testing.T) {
assert.Assert(t, storageOpts.GraphDriverName == overlayDriver || storageOpts.GraphDriverName == vfsDriver, fmt.Sprintf("The rootless driver should be set to 'overlay' or 'vfs' not '%v'", storageOpts.GraphDriverName))
})
t.Run("systemDriver=devmapper", func(t *testing.T) {
systemOpts := StoreOptions{}
systemOpts.GraphDriverName = "devmapper"
storageOpts, err := getRootlessStorageOpts(systemOpts)
assert.NilError(t, err)
assert.Assert(t, storageOpts.GraphDriverName == overlayDriver || storageOpts.GraphDriverName == vfsDriver, fmt.Sprintf("The rootless driver should be set to 'overlay' or 'vfs' not '%v'", storageOpts.GraphDriverName))
})
t.Run("systemDriver=zfs", func(t *testing.T) {
systemOpts := StoreOptions{}
systemOpts.GraphDriverName = "zfs"

4
storage/types/storage_test.conf
View File

@ -39,7 +39,3 @@ remap-gids = "0:1500000000:60000"
# mountopt specifies comma separated list of extra mount options
mountopt = "nodev"
[storage.options.thinpool]
# Storage Options for thinpool