From ad6467f9e17205fa76a3b916efe51ba5c1b37506 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 30 Oct 2014 20:26:39 -0400 Subject: [PATCH 1/5] devmapper: use proper DM_UDEV_DISABLE_*_FLAG when creating the thin-pool Otherwise udev can unecessarily execute various rules (and issue scanning IO, etc) against the thin-pool -- which can never be a top-level device. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- pkg/devicemapper/devmapper.go | 3 ++- pkg/devicemapper/devmapper_wrapper.go | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/devicemapper/devmapper.go b/pkg/devicemapper/devmapper.go index c0b931c3fb..3de42ba9e8 100644 --- a/pkg/devicemapper/devmapper.go +++ b/pkg/devicemapper/devmapper.go @@ -361,7 +361,8 @@ func CreatePool(poolName string, dataFile, metadataFile *os.File, poolBlockSize } var cookie uint = 0 - if err := task.SetCookie(&cookie, 0); err != nil { + var flags uint16 = DmUdevDisableSubsystemRulesFlag | DmUdevDisableDiskRulesFlag | DmUdevDisableOtherRulesFlag + if err := task.SetCookie(&cookie, flags); err != nil { return fmt.Errorf("Can't set cookie %s", err) } diff --git a/pkg/devicemapper/devmapper_wrapper.go b/pkg/devicemapper/devmapper_wrapper.go index c7e96a1617..499405a10d 100644 --- a/pkg/devicemapper/devmapper_wrapper.go +++ b/pkg/devicemapper/devmapper_wrapper.go @@ -82,6 +82,12 @@ const ( LoNameSize = C.LO_NAME_SIZE ) +const ( + DmUdevDisableSubsystemRulesFlag = C.DM_UDEV_DISABLE_SUBSYSTEM_RULES_FLAG + DmUdevDisableDiskRulesFlag = C.DM_UDEV_DISABLE_DISK_RULES_FLAG + DmUdevDisableOtherRulesFlag = C.DM_UDEV_DISABLE_OTHER_RULES_FLAG +) + var ( DmGetLibraryVersion = dmGetLibraryVersionFct DmGetNextTarget = dmGetNextTargetFct From 2b10749cdd0939e4b9e6e18e160984129d733663 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 24 Oct 2014 19:25:24 -0400 Subject: [PATCH 2/5] devmapper: Add option for specifying an lvm2 created thin-pool device Ideally lvm2 would be used to create/manage the thin-pool volume that is then handed to docker to exclusively create/manage the thin and thin snapshot volumes needed for it's containers. Managing the thin-pool outside of docker makes for the most feature-rich method of having docker utilize device mapper thin provisioning as the backing storage for docker's containers. lvm2-based thin-pool management feature highlights include: automatic or interactive thin-pool resize support, dynamically change thin-pool features, automatic thinp metadata checking when lvm2 activates the thin-pool, etc. Docker will not activate/deactivate the specified thin-pool device but it will exclusively manage/create thin and thin snapshot volumes in it. Docker will not take ownership of the specified thin-pool device unless it has 0 data blocks used and a transaction id of 0. This should help guard against using a thin-pool that is already in use. Also fix typos in setupBaseImage() relative to the thin volume type of the base image. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/README.md | 19 ++++++++++++ daemon/graphdriver/devmapper/deviceset.go | 37 ++++++++++++++++++----- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/daemon/graphdriver/devmapper/README.md b/daemon/graphdriver/devmapper/README.md index c42620247b..3b69cef84f 100644 --- a/daemon/graphdriver/devmapper/README.md +++ b/daemon/graphdriver/devmapper/README.md @@ -100,6 +100,25 @@ Here is the list of supported options: ``docker -d --storage-opt dm.mountopt=nodiscard`` + * `dm.thinpooldev` + + Specifies a custom blockdevice to use for the thin pool. + + If using a block device for device mapper storage, ideally lvm2 + would be used to create/manage the thin-pool volume that is then + handed to docker to exclusively create/manage the thin and thin + snapshot volumes needed for it's containers. Managing the thin-pool + outside of docker makes for the most feature-rich method of having + docker utilize device mapper thin provisioning as the backing + storage for docker's containers. lvm2-based thin-pool management + feature highlights include: automatic or interactive thin-pool + resize support, dynamically change thin-pool features, automatic + thinp metadata checking when lvm2 activates the thin-pool, etc. + + Example use: + + ``docker -d --storage-opt dm.thinpooldev=/dev/mapper/thin-pool`` + * `dm.datadev` Specifies a custom blockdevice to use for data for the thin pool. diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index 79d9dc6d06..0e94ca7a03 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -84,6 +84,7 @@ type DeviceSet struct { metadataDevice string doBlkDiscard bool thinpBlockSize uint32 + thinPoolDevice string } type DiskUsage struct { @@ -150,7 +151,11 @@ func (devices *DeviceSet) oldMetadataFile() string { } func (devices *DeviceSet) getPoolName() string { - return devices.devicePrefix + "-pool" + if devices.thinPoolDevice == "" { + return devices.devicePrefix + "-pool" + } else { + return devices.thinPoolDevice + } } func (devices *DeviceSet) getPoolDevName() string { @@ -411,7 +416,21 @@ func (devices *DeviceSet) setupBaseImage() error { } } - log.Debugf("Initializing base device-manager snapshot") + if devices.thinPoolDevice != "" && oldInfo == nil { + if _, transactionId, dataUsed, _, _, _, err := devices.poolStatus(); err != nil { + return err + } else { + if dataUsed != 0 { + return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks", + devices.thinPoolDevice) + } else if transactionId != 0 { + return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction Id", + devices.thinPoolDevice) + } + } + } + + log.Debugf("Initializing base device-mapper thin volume") id := devices.NextDeviceId @@ -430,7 +449,7 @@ func (devices *DeviceSet) setupBaseImage() error { return err } - log.Debugf("Creating filesystem on base device-manager snapshot") + log.Debugf("Creating filesystem on base device-mapper thin volume") if err = devices.activateDeviceIfNeeded(info); err != nil { return err @@ -605,7 +624,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino) log.Debugf("Generated prefix: %s", devices.devicePrefix) - // Check for the existence of the device -pool + // Check for the existence of the thin-pool device log.Debugf("Checking for existence of the pool '%s'", devices.getPoolName()) info, err := devicemapper.GetInfo(devices.getPoolName()) if info == nil { @@ -624,7 +643,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { createdLoopback := false // If the pool doesn't exist, create it - if info.Exists == 0 { + if info.Exists == 0 && devices.thinPoolDevice == "" { log.Debugf("Pool doesn't exist. Creating it.") var ( @@ -988,8 +1007,10 @@ func (devices *DeviceSet) Shutdown() error { } devices.Lock() - if err := devices.deactivatePool(); err != nil { - log.Debugf("Shutdown deactivate pool , error: %s", err) + if devices.thinPoolDevice == "" { + if err := devices.deactivatePool(); err != nil { + log.Debugf("Shutdown deactivate pool , error: %s", err) + } } devices.saveDeviceSetMetaData() @@ -1275,6 +1296,8 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error devices.metadataDevice = val case "dm.datadev": devices.dataDevice = val + case "dm.thinpooldev": + devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/") case "dm.blkdiscard": foundBlkDiscard = true devices.doBlkDiscard, err = strconv.ParseBool(val) From e49567ba729001c31fe71e4b715eed8f50d7ded9 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 13 Nov 2014 13:37:47 -0500 Subject: [PATCH 3/5] devmapper: disable discards by default if dm.thinpooldev was specified User may still enable discards by setting dm.blkdiscard=true Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/deviceset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index 0e94ca7a03..f28dc982bd 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -1317,7 +1317,7 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error } // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive - if !foundBlkDiscard && devices.dataDevice != "" { + if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") { devices.doBlkDiscard = false } From 553b50bd37ade60bfafe5d5cc10f984251741f44 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 21 Nov 2014 21:36:23 -0500 Subject: [PATCH 4/5] devmapper: remove unnecessary else branch in getPoolName() Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/deviceset.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index f28dc982bd..e4fa3e7195 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -153,9 +153,8 @@ func (devices *DeviceSet) oldMetadataFile() string { func (devices *DeviceSet) getPoolName() string { if devices.thinPoolDevice == "" { return devices.devicePrefix + "-pool" - } else { - return devices.thinPoolDevice } + return devices.thinPoolDevice } func (devices *DeviceSet) getPoolDevName() string { From b9f1b0a7514c6e40e7048fb9206001259eb7c33c Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 21 Nov 2014 22:26:09 -0500 Subject: [PATCH 5/5] devmapper: cleanup some extraneous branching in setupBaseImage() Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/deviceset.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index e4fa3e7195..b9d6e7616d 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -416,16 +416,17 @@ func (devices *DeviceSet) setupBaseImage() error { } if devices.thinPoolDevice != "" && oldInfo == nil { - if _, transactionId, dataUsed, _, _, _, err := devices.poolStatus(); err != nil { + _, transactionId, dataUsed, _, _, _, err := devices.poolStatus() + if err != nil { return err - } else { - if dataUsed != 0 { - return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks", - devices.thinPoolDevice) - } else if transactionId != 0 { - return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction Id", - devices.thinPoolDevice) - } + } + if dataUsed != 0 { + return fmt.Errorf("Unable to take ownership of thin-pool (%s) that already has used data blocks", + devices.thinPoolDevice) + } + if transactionId != 0 { + return fmt.Errorf("Unable to take ownership of thin-pool (%s) with non-zero transaction Id", + devices.thinPoolDevice) } }