From d12ea7ff6d66988e7e0a6ad776a8733dd061d4bd Mon Sep 17 00:00:00 2001
From: Isabel Jimenez <contact@isabeljimenez.com>
Date: Thu, 14 Jan 2016 04:18:38 -0500
Subject: [PATCH 1/2] Adding test
Signed-off-by: Isabel Jimenez <contact@isabeljimenez.com>
---
test/integration/mesos/api/run.bats | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/test/integration/mesos/api/run.bats b/test/integration/mesos/api/run.bats
index 28ebe269e4..aeaac12519 100644
--- a/test/integration/mesos/api/run.bats
+++ b/test/integration/mesos/api/run.bats
@@ -64,3 +64,19 @@ function teardown() {
[ "$status" -ne 0 ]
[[ "${output}" == *'resources constraints (-c and/or -m) are required by mesos'* ]]
}
+
+@test "mesos - docker run with long pull" {
+ start_docker 2
+ start_mesos
+ swarm_manage --cluster-driver mesos-experimental --cluster-opt mesos.tasktimeout=1s 127.0.0.1:$MESOS_MASTER_PORT
+
+ # make sure no container exist
+ run docker_swarm ps -qa
+ [ "${#lines[@]}" -eq 0 ]
+
+ # run
+ docker_swarm run -m 20m -d --name test_container busybox sleep 100
+
+ # verify, container is running
+ [ -n $(docker_swarm ps -q --filter=name=test_container --filter=status=running) ]
+}
\ No newline at end of file
From a99ceeb9c17a989c6e98c899b5dbf3258368427c Mon Sep 17 00:00:00 2001
From: Isabel Jimenez <contact@isabeljimenez.com>
Date: Thu, 14 Jan 2016 04:55:45 -0500
Subject: [PATCH 2/2] Adding suicide logic for tasks so as to prevent false
timeout for tasks having a long image pull
Signed-off-by: Isabel Jimenez <contact@isabeljimenez.com>
---
cluster/mesos/agent_test.go | 9 +++++----
cluster/mesos/cluster.go | 6 ++----
cluster/mesos/task/task.go | 13 ++++++++++++-
cluster/mesos/task/task_test.go | 7 ++++---
cluster/mesos/task/tasks_test.go | 9 +++++----
test/integration/mesos/api/run.bats | 2 +-
6 files changed, 29 insertions(+), 17 deletions(-)
diff --git a/cluster/mesos/agent_test.go b/cluster/mesos/agent_test.go
index 60aea81ed0..d252082024 100644
--- a/cluster/mesos/agent_test.go
+++ b/cluster/mesos/agent_test.go
@@ -2,6 +2,7 @@ package mesos
import (
"testing"
+ "time"
"github.com/docker/swarm/cluster"
"github.com/docker/swarm/cluster/mesos/task"
@@ -42,11 +43,11 @@ func TestAddTask(t *testing.T) {
assert.Empty(t, s.tasks)
assert.True(t, s.empty())
- t1, err := task.NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "task1")
+ t1, err := task.NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "task1", 5*time.Second)
assert.NoError(t, err)
s.addTask(t1)
- t2, err := task.NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "task1")
+ t2, err := task.NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "task1", 5*time.Second)
assert.NoError(t, err)
s.addTask(t2)
assert.Equal(t, len(s.tasks), 2)
@@ -80,11 +81,11 @@ func TestRemoveTask(t *testing.T) {
assert.Empty(t, s.tasks)
- t1, err := task.NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "task1")
+ t1, err := task.NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "task1", 5*time.Second)
assert.NoError(t, err)
s.addTask(t1)
- t2, err := task.NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "task1")
+ t2, err := task.NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "task1", 5*time.Second)
assert.NoError(t, err)
s.addTask(t2)
assert.Equal(t, len(s.tasks), 2)
diff --git a/cluster/mesos/cluster.go b/cluster/mesos/cluster.go
index b487c6c1e6..e2bb9638f5 100644
--- a/cluster/mesos/cluster.go
+++ b/cluster/mesos/cluster.go
@@ -183,7 +183,7 @@ func (c *Cluster) CreateContainer(config *cluster.ContainerConfig, name string,
return nil, errResourcesNeeded
}
- task, err := task.NewTask(config, name)
+ task, err := task.NewTask(config, name, c.taskCreationTimeout)
if err != nil {
return nil, err
}
@@ -194,10 +194,8 @@ func (c *Cluster) CreateContainer(config *cluster.ContainerConfig, name string,
case container := <-task.GetContainer():
return formatContainer(container), nil
case err := <-task.Error:
- return nil, err
- case <-time.After(c.taskCreationTimeout):
c.pendingTasks.Remove(task)
- return nil, fmt.Errorf("container failed to start after %s", c.taskCreationTimeout)
+ return nil, err
}
}
diff --git a/cluster/mesos/task/task.go b/cluster/mesos/task/task.go
index ebdeb7f3e0..191852eb24 100644
--- a/cluster/mesos/task/task.go
+++ b/cluster/mesos/task/task.go
@@ -5,6 +5,7 @@ import (
"fmt"
"strconv"
"strings"
+ "time"
log "github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/stringid"
@@ -162,7 +163,7 @@ func (t *Task) Build(slaveID string, offers map[string]*mesosproto.Offer) {
}
// NewTask fucntion creates a task
-func NewTask(config *cluster.ContainerConfig, name string) (*Task, error) {
+func NewTask(config *cluster.ContainerConfig, name string, timeout time.Duration) (*Task, error) {
id := stringid.TruncateID(stringid.GenerateRandomID())
if name != "" {
@@ -183,9 +184,19 @@ func NewTask(config *cluster.ContainerConfig, name string) (*Task, error) {
task.Name = &name
task.TaskId = &mesosproto.TaskID{Value: &id}
task.Labels = &mesosproto.Labels{Labels: []*mesosproto.Label{{Key: proto.String("SWARM_CONTAINER_NAME"), Value: &name}}}
+
+ go task.suicide(timeout)
+
return task, nil
}
+func (t *Task) suicide(timeout time.Duration) {
+ <-time.After(timeout)
+ if !t.Stopped() && t.SlaveId == nil {
+ t.Error <- fmt.Errorf("container failed to start after %s", timeout)
+ }
+}
+
// SendStatus method writes the task status in the updates channel
func (t *Task) SendStatus(status *mesosproto.TaskStatus) {
t.updates <- status
diff --git a/cluster/mesos/task/task_test.go b/cluster/mesos/task/task_test.go
index bd4154a840..4066489fe0 100644
--- a/cluster/mesos/task/task_test.go
+++ b/cluster/mesos/task/task_test.go
@@ -4,6 +4,7 @@ import (
"sort"
"strings"
"testing"
+ "time"
"github.com/docker/swarm/cluster"
"github.com/mesos/mesos-go/mesosproto"
@@ -20,7 +21,7 @@ func TestBuild(t *testing.T) {
CpuShares: 42,
Memory: 2097152,
Cmd: []string{"ls", "foo", "bar"},
- }), name)
+ }), name, 5*time.Second)
assert.NoError(t, err)
task.Build("slave-id", nil)
@@ -47,7 +48,7 @@ func TestBuild(t *testing.T) {
}
func TestNewTask(t *testing.T) {
- task, err := NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), name)
+ task, err := NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), name, 5*time.Second)
assert.NoError(t, err)
assert.Equal(t, *task.Name, name)
@@ -56,7 +57,7 @@ func TestNewTask(t *testing.T) {
}
func TestSendGetStatus(t *testing.T) {
- task, err := NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "")
+ task, err := NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{}), "", 5*time.Second)
assert.NoError(t, err)
status := mesosutil.NewTaskStatus(nil, mesosproto.TaskState_TASK_RUNNING)
diff --git a/cluster/mesos/task/tasks_test.go b/cluster/mesos/task/tasks_test.go
index 3a92480996..d30e6e64be 100644
--- a/cluster/mesos/task/tasks_test.go
+++ b/cluster/mesos/task/tasks_test.go
@@ -2,6 +2,7 @@ package task
import (
"testing"
+ "time"
"github.com/docker/swarm/cluster"
"github.com/samalba/dockerclient"
@@ -25,14 +26,14 @@ func TestAdd(t *testing.T) {
CpuShares: 42,
Memory: 2097152,
Cmd: []string{"ls", "foo", "bar"},
- }), "name1")
+ }), "name1", 5*time.Second)
task2, _ := NewTask(cluster.BuildContainerConfig(dockerclient.ContainerConfig{
Image: "test-image",
CpuShares: 42,
Memory: 2097152,
Cmd: []string{"ls", "foo", "bar"},
- }), "name2")
+ }), "name2", 5*time.Second)
q.Add(task1)
assert.Equal(t, len(q.Tasks), 0)
@@ -48,7 +49,7 @@ func TestRemove(t *testing.T) {
CpuShares: 42,
Memory: 2097152,
Cmd: []string{"ls", "foo", "bar"},
- }), "name1")
+ }), "name1", 5*time.Second)
q.Add(task1)
assert.Equal(t, len(q.Tasks), 1)
@@ -64,7 +65,7 @@ func TestProcess(t *testing.T) {
CpuShares: 42,
Memory: 2097152,
Cmd: []string{"ls", "foo", "bar"},
- }), "name1")
+ }), "name1", 5*time.Second)
q.Add(task1)
assert.Equal(t, len(q.Tasks), 1)
diff --git a/test/integration/mesos/api/run.bats b/test/integration/mesos/api/run.bats
index aeaac12519..63ba068637 100644
--- a/test/integration/mesos/api/run.bats
+++ b/test/integration/mesos/api/run.bats
@@ -79,4 +79,4 @@ function teardown() {
# verify, container is running
[ -n $(docker_swarm ps -q --filter=name=test_container --filter=status=running) ]
-}
\ No newline at end of file
+}