From d420134fa2ea8ceedf2a9a64263d09b308263c2c Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 20 Jun 2014 11:45:42 -0400 Subject: [PATCH 1/6] devmapper: fix reloadPool() to also specify '1 skip_block_zeroing' createPool() and reloadPool() should be consistent with the thin-pool table params they use. Since createPool() specifies '1 skip_block_zeroing' reloadPool() should too. Otherwise, if the pool is reloaded (as is done when resizing loopback devices) block zeroing will be enabled after the reload completes. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/devmapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/graphdriver/devmapper/devmapper.go b/daemon/graphdriver/devmapper/devmapper.go index a6602c276e..2590ec0fc1 100644 --- a/daemon/graphdriver/devmapper/devmapper.go +++ b/daemon/graphdriver/devmapper/devmapper.go @@ -369,7 +369,7 @@ func reloadPool(poolName string, dataFile, metadataFile *os.File) error { return fmt.Errorf("Can't get data size %s", err) } - params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768" + params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768 1 skip_block_zeroing" if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil { return fmt.Errorf("Can't add target %s", err) } From 2470a5ed999d9dc8d4b9bd250727ac13964db5b3 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 23 Jun 2014 15:36:08 -0400 Subject: [PATCH 2/6] devmapper: use RAMInBytes() rather than FromHumanSize() Device Mapper needs device sizes in binary (1024) multiples. Otherwise kernel checks can find that the specified thin-pool device sizes aren't a multiple of the specified thin-pool blocksize. The name for "RAMInBytes" is likely too narrow given the new consumers but... Also add "tebibyte" support to RAMInBytes. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/deviceset.go | 6 +++--- pkg/units/size.go | 8 +++++--- pkg/units/size_test.go | 3 +++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index 31c3f391e4..b0e0819ba8 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -1170,19 +1170,19 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error key = strings.ToLower(key) switch key { case "dm.basesize": - size, err := units.FromHumanSize(val) + size, err := units.RAMInBytes(val) if err != nil { return nil, err } devices.baseFsSize = uint64(size) case "dm.loopdatasize": - size, err := units.FromHumanSize(val) + size, err := units.RAMInBytes(val) if err != nil { return nil, err } devices.dataLoopbackSize = size case "dm.loopmetadatasize": - size, err := units.FromHumanSize(val) + size, err := units.RAMInBytes(val) if err != nil { return nil, err } diff --git a/pkg/units/size.go b/pkg/units/size.go index 480ec2f141..e0eec3ec7d 100644 --- a/pkg/units/size.go +++ b/pkg/units/size.go @@ -58,11 +58,11 @@ func FromHumanSize(size string) (int64, error) { } // Parses a human-readable string representing an amount of RAM -// in bytes, kibibytes, mebibytes or gibibytes, and returns the -// number of bytes, or -1 if the string is unparseable. +// in bytes, kibibytes, mebibytes, gibibytes, or tebibytes and +// returns the number of bytes, or -1 if the string is unparseable. // Units are case-insensitive, and the 'b' suffix is optional. func RAMInBytes(size string) (bytes int64, err error) { - re, error := regexp.Compile("^(\\d+)([kKmMgG])?[bB]?$") + re, error := regexp.Compile("^(\\d+)([kKmMgGtT])?[bB]?$") if error != nil { return -1, error } @@ -86,6 +86,8 @@ func RAMInBytes(size string) (bytes int64, err error) { memLimit *= 1024 * 1024 } else if unit == "g" { memLimit *= 1024 * 1024 * 1024 + } else if unit == "t" { + memLimit *= 1024 * 1024 * 1024 * 1024 } return memLimit, nil diff --git a/pkg/units/size_test.go b/pkg/units/size_test.go index 5240bbd9f0..2c4982b16f 100644 --- a/pkg/units/size_test.go +++ b/pkg/units/size_test.go @@ -64,7 +64,10 @@ func TestRAMInBytes(t *testing.T) { assertRAMInBytes(t, "32kb", false, 32*1024) assertRAMInBytes(t, "32Kb", false, 32*1024) assertRAMInBytes(t, "32Mb", false, 32*1024*1024) + assertRAMInBytes(t, "32MB", false, 32*1024*1024) assertRAMInBytes(t, "32Gb", false, 32*1024*1024*1024) + assertRAMInBytes(t, "32G", false, 32*1024*1024*1024) + assertRAMInBytes(t, "32Tb", false, 32*1024*1024*1024*1024) assertRAMInBytes(t, "", true, -1) assertRAMInBytes(t, "hello", true, -1) From 09ee269d998ad04733ef577739fa051df9d3f12e Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 20 Jun 2014 11:53:18 -0400 Subject: [PATCH 3/6] devmapper: Add option for specifying the thin pool blocksize Add dm.blocksize option that you can use with --storage-opt to set a specific blocksize for the thin provisioning pool. Also change the default dm-thin-pool blocksize from 64K to 512K. This strikes a balance between the desire to have smaller blocksize given docker's use of snapshots versus the desire to have more performance that comes with using a larger blocksize. But if very small files will be used on average the user is encouraged to override this default. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/README.md | 8 ++++++++ daemon/graphdriver/devmapper/deviceset.go | 14 ++++++++++++-- daemon/graphdriver/devmapper/devmapper.go | 8 ++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/daemon/graphdriver/devmapper/README.md b/daemon/graphdriver/devmapper/README.md index c8ab1d1ee1..849cfce64c 100644 --- a/daemon/graphdriver/devmapper/README.md +++ b/daemon/graphdriver/devmapper/README.md @@ -126,6 +126,14 @@ Here is the list of supported options: ``docker -d --storage-opt dm.datadev=/dev/sdb1 --storage-opt dm.metadatadev=/dev/sdc1`` + * `dm.blocksize` + + Specifies a custom blocksize to use for the thin pool. + + Example use: + + ``docker -d --storage-opt dm.blocksize=64K`` + * `dm.blkdiscard` Enables or disables the use of blkdiscard when removing diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index b0e0819ba8..c42d9c5a72 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -28,6 +28,7 @@ var ( DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 + DefaultThinpBlockSize uint32 = 1024 // 512K = 1024 512b sectors ) type DevInfo struct { @@ -78,6 +79,7 @@ type DeviceSet struct { dataDevice string metadataDevice string doBlkDiscard bool + thinpBlockSize uint32 } type DiskUsage struct { @@ -510,7 +512,7 @@ func (devices *DeviceSet) ResizePool(size int64) error { } // Reload with the new block sizes - if err := reloadPool(devices.getPoolName(), dataloopback, metadataloopback); err != nil { + if err := reloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil { return fmt.Errorf("Unable to reload pool: %s", err) } @@ -640,7 +642,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { } defer metadataFile.Close() - if err := createPool(devices.getPoolName(), dataFile, metadataFile); err != nil { + if err := createPool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { return err } } @@ -1159,6 +1161,7 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error baseFsSize: DefaultBaseFsSize, filesystem: "ext4", doBlkDiscard: true, + thinpBlockSize: DefaultThinpBlockSize, } foundBlkDiscard := false @@ -1206,6 +1209,13 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error if err != nil { return nil, err } + case "dm.blocksize": + size, err := units.RAMInBytes(val) + if err != nil { + return nil, err + } + // convert to 512b sectors + devices.thinpBlockSize = uint32(size) >> 9 default: return nil, fmt.Errorf("Unknown option %s\n", key) } diff --git a/daemon/graphdriver/devmapper/devmapper.go b/daemon/graphdriver/devmapper/devmapper.go index 2590ec0fc1..ee4f2a8159 100644 --- a/daemon/graphdriver/devmapper/devmapper.go +++ b/daemon/graphdriver/devmapper/devmapper.go @@ -328,7 +328,7 @@ func BlockDeviceDiscard(path string) error { } // This is the programmatic example of "dmsetup create" -func createPool(poolName string, dataFile, metadataFile *os.File) error { +func createPool(poolName string, dataFile, metadataFile *os.File, poolBlockSize uint32) error { task, err := createTask(DeviceCreate, poolName) if task == nil { return err @@ -339,7 +339,7 @@ func createPool(poolName string, dataFile, metadataFile *os.File) error { return fmt.Errorf("Can't get data size %s", err) } - params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768 1 skip_block_zeroing" + params := fmt.Sprintf("%s %s %d 32768 1 skip_block_zeroing", metadataFile.Name(), dataFile.Name(), poolBlockSize) if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil { return fmt.Errorf("Can't add target %s", err) } @@ -358,7 +358,7 @@ func createPool(poolName string, dataFile, metadataFile *os.File) error { return nil } -func reloadPool(poolName string, dataFile, metadataFile *os.File) error { +func reloadPool(poolName string, dataFile, metadataFile *os.File, poolBlockSize uint32) error { task, err := createTask(DeviceReload, poolName) if task == nil { return err @@ -369,7 +369,7 @@ func reloadPool(poolName string, dataFile, metadataFile *os.File) error { return fmt.Errorf("Can't get data size %s", err) } - params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768 1 skip_block_zeroing" + params := fmt.Sprintf("%s %s %d 32768 1 skip_block_zeroing", metadataFile.Name(), dataFile.Name(), poolBlockSize) if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil { return fmt.Errorf("Can't add target %s", err) } From f9c078ef38106d00de7774374f3ac71bb0d562d3 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 24 Jun 2014 12:43:45 -0400 Subject: [PATCH 4/6] devmapper: remove extra space in DefaultThinpBlockSize assignment Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/deviceset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index c42d9c5a72..61ba81852d 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -28,7 +28,7 @@ var ( DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 - DefaultThinpBlockSize uint32 = 1024 // 512K = 1024 512b sectors + DefaultThinpBlockSize uint32 = 1024 // 512K = 1024 512b sectors ) type DevInfo struct { From 79f217e3501a43326bd532e8eb03182f7488f61c Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 26 Jun 2014 12:06:41 -0400 Subject: [PATCH 5/6] devmapper: document the default DM thin pool blocksize Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon/graphdriver/devmapper/README.md b/daemon/graphdriver/devmapper/README.md index 849cfce64c..ed99d85a93 100644 --- a/daemon/graphdriver/devmapper/README.md +++ b/daemon/graphdriver/devmapper/README.md @@ -128,7 +128,8 @@ Here is the list of supported options: * `dm.blocksize` - Specifies a custom blocksize to use for the thin pool. + Specifies a custom blocksize to use for the thin pool. The default + blocksize is 512K. Example use: From a2f3ce2294bbe998df24edd50f2b571d0b21bac8 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 26 Jun 2014 12:39:16 -0400 Subject: [PATCH 6/6] devmapper: add thin-pool blocksize to the 'docker info' output Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/driver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/graphdriver/devmapper/driver.go b/daemon/graphdriver/devmapper/driver.go index cf82ad62ed..29335377f6 100644 --- a/daemon/graphdriver/devmapper/driver.go +++ b/daemon/graphdriver/devmapper/driver.go @@ -54,6 +54,7 @@ func (d *Driver) Status() [][2]string { status := [][2]string{ {"Pool Name", s.PoolName}, + {"Pool Blocksize", fmt.Sprintf("%d Kb", s.SectorSize/1024)}, {"Data file", s.DataLoopback}, {"Metadata file", s.MetadataLoopback}, {"Data Space Used", fmt.Sprintf("%.1f Mb", float64(s.Data.Used)/(1024*1024))},