From ba1688ccef63f087f492800fa846b097bf81100a Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 18 Dec 2014 01:56:13 +0000 Subject: [PATCH 1/7] add matching to constraints Signed-off-by: Victor Vieux --- scheduler/filter/label.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scheduler/filter/label.go b/scheduler/filter/label.go index dbd3f80cb5..041e0a7fe4 100644 --- a/scheduler/filter/label.go +++ b/scheduler/filter/label.go @@ -2,8 +2,10 @@ package filter import ( "fmt" + "regexp" "strings" + log "github.com/Sirupsen/logrus" "github.com/docker/swarm/cluster" "github.com/samalba/dockerclient" ) @@ -27,18 +29,32 @@ func (f *LabelFilter) extractConstraints(env []string) map[string]string { func (f *LabelFilter) Filter(config *dockerclient.ContainerConfig, nodes []*cluster.Node) ([]*cluster.Node, error) { constraints := f.extractConstraints(config.Env) for k, v := range constraints { + regex := "^" + strings.Replace(v, "*", ".*", -1) + "$" + log.Debugf("matching constraint: %s=%s", k, regex) candidates := []*cluster.Node{} for _, node := range nodes { switch k { case "node": // "node" label is a special case pinning a container to a specific node. - if strings.ToLower(node.ID) == v || strings.ToLower(node.Name) == v { + matchedID, err := regexp.MatchString(regex, strings.ToLower(node.ID)) + if err != nil { + log.Error(err) + } + matchedName, err := regexp.MatchString(regex, strings.ToLower(node.Name)) + if err != nil { + log.Error(err) + } + if matchedID || matchedName { candidates = append(candidates, node) } default: // By default match the node labels. if label, ok := node.Labels[k]; ok { - if strings.Contains(strings.ToLower(label), v) { + matched, err := regexp.MatchString(regex, strings.ToLower(label)) + if err != nil { + log.Error(err) + } + if matched { candidates = append(candidates, node) } } From 28a5063168571df324742ff2b434fca7e0cda9bb Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 18 Dec 2014 02:02:32 +0000 Subject: [PATCH 2/7] add tests and move from label to constraint Signed-off-by: Victor Vieux --- contrib/demo.sh | 2 +- flags.go | 4 +-- scheduler/filter/{label.go => constraint.go} | 8 ++--- .../{label_test.go => constraint_test.go} | 32 ++++++++++++++----- scheduler/filter/filter.go | 6 ++-- 5 files changed, 34 insertions(+), 18 deletions(-) rename scheduler/filter/{label.go => constraint.go} (84%) rename scheduler/filter/{label_test.go => constraint_test.go} (79%) diff --git a/contrib/demo.sh b/contrib/demo.sh index 4bf2ca9d88..0d04da2e03 100644 --- a/contrib/demo.sh +++ b/contrib/demo.sh @@ -43,7 +43,7 @@ docker run -d -p 80:80 nginx # clean up cluster docker rm -f `docker ps -aq` -docker run -d -e constraint:operatingsystem=fedora redis +docker run -d -e "constraint:operatingsystem=fedora*" redis docker ps docker run -d -e constraint:storagedriver=devicemapper redis diff --git a/flags.go b/flags.go index f929bf34ce..f357c3c276 100644 --- a/flags.go +++ b/flags.go @@ -57,7 +57,7 @@ var ( } flFilter = cli.StringSliceFlag{ Name: "filter, f", - Usage: "Filter to use [health, label, port]", - Value: &cli.StringSlice{"health", "label", "port"}, + Usage: "Filter to use [constraint, health, port]", + Value: &cli.StringSlice{"constraint", "health", "port"}, } ) diff --git a/scheduler/filter/label.go b/scheduler/filter/constraint.go similarity index 84% rename from scheduler/filter/label.go rename to scheduler/filter/constraint.go index 041e0a7fe4..b4d1782cd5 100644 --- a/scheduler/filter/label.go +++ b/scheduler/filter/constraint.go @@ -10,11 +10,11 @@ import ( "github.com/samalba/dockerclient" ) -// LabelFilter selects only nodes that match certain labels. -type LabelFilter struct { +// ConstraintFilter selects only nodes that match certain labels. +type ConstraintFilter struct { } -func (f *LabelFilter) extractConstraints(env []string) map[string]string { +func (f *ConstraintFilter) extractConstraints(env []string) map[string]string { constraints := make(map[string]string) for _, e := range env { if strings.HasPrefix(e, "constraint:") { @@ -26,7 +26,7 @@ func (f *LabelFilter) extractConstraints(env []string) map[string]string { return constraints } -func (f *LabelFilter) Filter(config *dockerclient.ContainerConfig, nodes []*cluster.Node) ([]*cluster.Node, error) { +func (f *ConstraintFilter) Filter(config *dockerclient.ContainerConfig, nodes []*cluster.Node) ([]*cluster.Node, error) { constraints := f.extractConstraints(config.Env) for k, v := range constraints { regex := "^" + strings.Replace(v, "*", ".*", -1) + "$" diff --git a/scheduler/filter/label_test.go b/scheduler/filter/constraint_test.go similarity index 79% rename from scheduler/filter/label_test.go rename to scheduler/filter/constraint_test.go index fcab6185f2..a13c41e0bc 100644 --- a/scheduler/filter/label_test.go +++ b/scheduler/filter/constraint_test.go @@ -8,9 +8,9 @@ import ( "github.com/stretchr/testify/assert" ) -func TestLabeleFilter(t *testing.T) { +func TestConstrainteFilter(t *testing.T) { var ( - f = LabelFilter{} + f = ConstraintFilter{} nodes = []*cluster.Node{ cluster.NewNode("node-0"), cluster.NewNode("node-1"), @@ -23,22 +23,25 @@ func TestLabeleFilter(t *testing.T) { nodes[0].ID = "node-0-id" nodes[0].Name = "node-0-name" nodes[0].Labels = map[string]string{ - "name": "node0", - "group": "1", + "name": "node0", + "group": "1", + "region": "us-west", } nodes[1].ID = "node-1-id" nodes[1].Name = "node-1-name" nodes[1].Labels = map[string]string{ - "name": "node1", - "group": "1", + "name": "node1", + "group": "1", + "region": "us-east", } nodes[2].ID = "node-2-id" nodes[2].Name = "node-2-name" nodes[2].Labels = map[string]string{ - "name": "node2", - "group": "2", + "name": "node2", + "group": "2", + "region": "eu", } // Without constraints we should get the unfiltered list of nodes back. @@ -91,4 +94,17 @@ func TestLabeleFilter(t *testing.T) { assert.NoError(t, err) assert.Len(t, result, 1) assert.Equal(t, result[0], nodes[0]) + + // Check matching + result, err = f.Filter(&dockerclient.ContainerConfig{ + Env: []string{"constraint:region=us"}, + }, nodes) + assert.Error(t, err) + assert.Len(t, result, 0) + + result, err = f.Filter(&dockerclient.ContainerConfig{ + Env: []string{"constraint:region=us*"}, + }, nodes) + assert.NoError(t, err) + assert.Len(t, result, 2) } diff --git a/scheduler/filter/filter.go b/scheduler/filter/filter.go index 21fdf721a4..aacf8081ea 100644 --- a/scheduler/filter/filter.go +++ b/scheduler/filter/filter.go @@ -20,9 +20,9 @@ var ( func init() { filters = map[string]Filter{ - "health": &HealthFilter{}, - "label": &LabelFilter{}, - "port": &PortFilter{}, + "health": &HealthFilter{}, + "constraint": &ConstraintFilter{}, + "port": &PortFilter{}, } } From e4dfa66b625a24155b184f7160f24c8057d97193 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 18 Dec 2014 22:02:20 +0000 Subject: [PATCH 3/7] add README.md Signed-off-by: Victor Vieux --- scheduler/filter/README.md | 126 +++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 scheduler/filter/README.md diff --git a/scheduler/filter/README.md b/scheduler/filter/README.md new file mode 100644 index 0000000000..dea93604d2 --- /dev/null +++ b/scheduler/filter/README.md @@ -0,0 +1,126 @@ +Filters +======= + +The `Docker Swarm` scheduler comes with multiple filters. + +Thoses filters are used to schedule containers on a subset of nodes. + +`Docker Swarm` currently supports 3 filters: +* [Constraint](README.md#constraint-filter) +* [Port](README.md#port-filter) +* [Healty](README.md#healthy-filter) + +## Constraint Filter + +Constraints are key/value pairs associated to particular nodes. You can see them as *node tags*. + +When creating a container, the user can select a subset of nodes that should be considered for scheduling by specifying one or more sets of matching key/value pairs. + +This approach has several practical use cases such as: +* Selecting specific host properties (such as `storage=ssd`, in order to schedule containers on specific hardware). +* Tagging nodes based on their physical location (`region=us-east`, to force containers to run on a given location). +* Logical cluster partioning (`environment=production`, to split a cluster into sub-clusters with different properties). + +To tag a node with a specific set of key/value pairs, one must pass a list of `--label` options at docker startup time. + +For instance, let's start `node-1` with the `storage=ssd` label: + +```bash +$ docker -d --label storage=ssd +$ swarm join --discovery token://XXXXXXXXXXXXXXXXXX --addr=192.168.0.42:2375 +``` + +Again, but this time `node-2` with `storage=disk`: + +```bash +$ docker -d --label storage=disk +$ swarm join --discovery token://XXXXXXXXXXXXXXXXXX --addr=192.168.0.43:2375 +``` + +Once the nodes are registered with the cluster, the master pulls their respective tags and will take them into account when scheduling new containers. + +Let's start a MySQL server and make sure it gets good I/O performance by selecting nodes with flash drives: + +``` +$ docker run -d -P -e constraint:storage=ssd --name db mysql +f8b693db9cd6 + +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NODE NAMES +f8b693db9cd6 mysql:latest "mysqld" Less than a second ago running 192.168.0.42:49178->3306/tcp node-1 db +``` + +In this case, the master selected all nodes that met the `storage=ssd` constraint and applied resource management on top of them, as discussed earlier. +`node-1` was selected in this example since it's the only host running flash. + +Now we want to run an `nginx` frontend in our cluster. However, we don't want *flash* drives since we'll mostly write logs to disk. + +``` +$ docker run -d -P -e constraint:storage=disk --name frontend nginx +f8b693db9cd6 + +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NODE NAMES +963841b138d8 nginx:latest "nginx" Less than a second ago running 192.168.0.43:49177->80/tcp node-2 frontend +f8b693db9cd6 mysql:latest "mysqld" Up About a minute running 192.168.0.42:49178->3306/tcp node-1 db +``` + +The scheduler selected `node-2` since it was started with the `storage=disk` label. + +#### Standard Constraints + +Additionally, a standard set of constraints can be used when scheduling containers without specifying them when starting the node. +Those tags are sourced from `docker info` and currently include: + +* OperatingSystem +* KernelVersion +* Driver +* ExecutionDriver + +## Port Filter + +With this filter, `ports` are considered as a unique resource. + +``` +$ docker run -d -p 80:80 nginx +87c4376856a8 + +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NODE NAMES +87c4376856a8 nginx:latest "nginx" Less than a second ago running 192.168.0.42:80->80/tcp node-1 prickly_engelbart +``` + +Docker cluster selects a node where the public `80` port is available and schedules a container on it, in this case `node-1`. + +Attempting to run another container with the public `80` port will result in clustering selecting a different node, since that port is already occupied on `node-1`: +``` +$ docker run -d -p 80:80 nginx +963841b138d8 + +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NODE NAMES +963841b138d8 nginx:latest "nginx" Less than a second ago running 192.168.0.43:80->80/tcp node-2 dreamy_turing +87c4376856a8 nginx:latest "nginx" Up About a minute running 192.168.0.42:80->80/tcp node-1 prickly_engelbart +``` + +Again, repeating the same command will result in the selection of `node-3`, since port `80` is neither available on `node-1` nor `node-2`: +``` +$ docker run -d -p 80:80 nginx +963841b138d8 + +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NODE NAMES +f8b693db9cd6 nginx:latest "nginx" Less than a second ago running 192.168.0.44:80->80/tcp node-3 stoic_albattani +963841b138d8 nginx:latest "nginx" Up About a minute running 192.168.0.43:80->80/tcp node-2 dreamy_turing +87c4376856a8 nginx:latest "nginx" Up About a minute running 192.168.0.42:80->80/tcp node-1 prickly_engelbart +``` + +Finally, Docker Cluster will refuse to run another container that requires port `80` since not a single node in the cluster has it available: +``` +$ docker run -d -p 80:80 nginx +2014/10/29 00:33:20 Error response from daemon: no resources availalble to schedule container +``` + +## Health Filter + +This filter will prevent scheduling containers and unhealthy nodes. From 2f6531d3768d77990ce420c922ecde98c8c725a8 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Dec 2014 00:47:51 +0000 Subject: [PATCH 4/7] update docs Signed-off-by: Victor Vieux --- scheduler/filter/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scheduler/filter/README.md b/scheduler/filter/README.md index dea93604d2..e476182640 100644 --- a/scheduler/filter/README.md +++ b/scheduler/filter/README.md @@ -72,10 +72,10 @@ The scheduler selected `node-2` since it was started with the `storage=disk` lab Additionally, a standard set of constraints can be used when scheduling containers without specifying them when starting the node. Those tags are sourced from `docker info` and currently include: -* OperatingSystem -* KernelVersion -* Driver -* ExecutionDriver +* storagedriver +* executiondriver +* kernelversion +* operatingsystem ## Port Filter @@ -123,4 +123,4 @@ $ docker run -d -p 80:80 nginx ## Health Filter -This filter will prevent scheduling containers and unhealthy nodes. +This filter will prevent scheduling containers on unhealthy nodes. From 037f4c379bf9c4b114df97cb8376f1564a354937 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Dec 2014 00:59:57 +0000 Subject: [PATCH 5/7] simplify code Signed-off-by: Victor Vieux --- scheduler/filter/constraint.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/scheduler/filter/constraint.go b/scheduler/filter/constraint.go index b4d1782cd5..ad34f23128 100644 --- a/scheduler/filter/constraint.go +++ b/scheduler/filter/constraint.go @@ -26,35 +26,33 @@ func (f *ConstraintFilter) extractConstraints(env []string) map[string]string { return constraints } +// Create the regex for globbing (ex: ub*t* -> ^ub.*t.*$) +// and match. +func (f *ConstraintFilter) match(pattern, s string) bool { + regex := "^" + strings.Replace(pattern, "*", ".*", -1) + "$" + matched, err := regexp.MatchString(regex, strings.ToLower(s)) + if err != nil { + log.Error(err) + } + return matched +} + func (f *ConstraintFilter) Filter(config *dockerclient.ContainerConfig, nodes []*cluster.Node) ([]*cluster.Node, error) { constraints := f.extractConstraints(config.Env) for k, v := range constraints { - regex := "^" + strings.Replace(v, "*", ".*", -1) + "$" - log.Debugf("matching constraint: %s=%s", k, regex) + log.Debugf("matching constraint: %s=%s", k, v) candidates := []*cluster.Node{} for _, node := range nodes { switch k { case "node": // "node" label is a special case pinning a container to a specific node. - matchedID, err := regexp.MatchString(regex, strings.ToLower(node.ID)) - if err != nil { - log.Error(err) - } - matchedName, err := regexp.MatchString(regex, strings.ToLower(node.Name)) - if err != nil { - log.Error(err) - } - if matchedID || matchedName { + if f.match(v, node.ID) || f.match(v, node.Name) { candidates = append(candidates, node) } default: // By default match the node labels. if label, ok := node.Labels[k]; ok { - matched, err := regexp.MatchString(regex, strings.ToLower(label)) - if err != nil { - log.Error(err) - } - if matched { + if f.match(v, label) { candidates = append(candidates, node) } } From 156f16c05e457992a1ed47fc5637c5fee449adc5 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Dec 2014 01:02:46 +0000 Subject: [PATCH 6/7] update main README.md Signed-off-by: Victor Vieux --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index b10d36f114..7ffe93557f 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,10 @@ http:// See [here](https://github.com/docker/swarm/discovery) for more information about other discovery services. +### Advanced Scheduling + +See [filters](scheduler/filter) and [strategies](scheduler/strategy) to learn more about advanced scheduling. + ### TLS Swarm supports TLS authentication between the CLI and Swarm but also between Swarm and the Docker nodes. From e5873381980d6fc3713f5cab48a1bd92e926f297 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Dec 2014 01:04:42 +0000 Subject: [PATCH 7/7] update README.md Signed-off-by: Victor Vieux --- scheduler/filter/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scheduler/filter/README.md b/scheduler/filter/README.md index e476182640..2fceb1bbb4 100644 --- a/scheduler/filter/README.md +++ b/scheduler/filter/README.md @@ -10,6 +10,8 @@ Thoses filters are used to schedule containers on a subset of nodes. * [Port](README.md#port-filter) * [Healty](README.md#healthy-filter) +You can choose the filter(s) you want to use with the `--filter` flag of `swarm manage` + ## Constraint Filter Constraints are key/value pairs associated to particular nodes. You can see them as *node tags*.