Increase timeout for swarm requests

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
(cherry picked from commit 85b1fdf15ce2ad1b373748554d3aa218e2eb5a5f)
Signed-off-by: Tibor Vass <tibor@docker.com>
This commit is contained in:
Tonis Tiigi 2016-07-15 10:58:21 -07:00 committed by Tibor Vass
parent 4118e0c9ee
commit abf9cfe22f
1 changed files with 80 additions and 28 deletions

View File

@ -32,6 +32,7 @@ import (
const swarmDirName = "swarm" const swarmDirName = "swarm"
const controlSocket = "control.sock" const controlSocket = "control.sock"
const swarmConnectTimeout = 20 * time.Second const swarmConnectTimeout = 20 * time.Second
const swarmRequestTimeout = 20 * time.Second
const stateFile = "docker-state.json" const stateFile = "docker-state.json"
const defaultAddr = "0.0.0.0:2377" const defaultAddr = "0.0.0.0:2377"
@ -459,9 +460,8 @@ func (c *Cluster) clearState() error {
return nil return nil
} }
func (c *Cluster) getRequestContext() context.Context { // TODO: not needed when requests don't block on qourum lost func (c *Cluster) getRequestContext() (context.Context, func()) { // TODO: not needed when requests don't block on qourum lost
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second) return context.WithTimeout(context.Background(), swarmRequestTimeout)
return ctx
} }
// Inspect retrieves the configuration properties of a managed swarm cluster. // Inspect retrieves the configuration properties of a managed swarm cluster.
@ -473,7 +473,10 @@ func (c *Cluster) Inspect() (types.Swarm, error) {
return types.Swarm{}, c.errNoManager() return types.Swarm{}, c.errNoManager()
} }
swarm, err := getSwarm(c.getRequestContext(), c.client) ctx, cancel := c.getRequestContext()
defer cancel()
swarm, err := getSwarm(ctx, c.client)
if err != nil { if err != nil {
return types.Swarm{}, err return types.Swarm{}, err
} }
@ -494,7 +497,10 @@ func (c *Cluster) Update(version uint64, spec types.Spec) error {
return c.errNoManager() return c.errNoManager()
} }
swarm, err := getSwarm(c.getRequestContext(), c.client) ctx, cancel := c.getRequestContext()
defer cancel()
swarm, err := getSwarm(ctx, c.client)
if err != nil { if err != nil {
return err return err
} }
@ -505,7 +511,7 @@ func (c *Cluster) Update(version uint64, spec types.Spec) error {
} }
_, err = c.client.UpdateCluster( _, err = c.client.UpdateCluster(
c.getRequestContext(), ctx,
&swarmapi.UpdateClusterRequest{ &swarmapi.UpdateClusterRequest{
ClusterID: swarm.ID, ClusterID: swarm.ID,
Spec: &swarmSpec, Spec: &swarmSpec,
@ -591,9 +597,13 @@ func (c *Cluster) Info() types.Info {
if c.err != nil { if c.err != nil {
info.Error = c.err.Error() info.Error = c.err.Error()
} }
ctx, cancel := c.getRequestContext()
defer cancel()
if c.isActiveManager() { if c.isActiveManager() {
info.ControlAvailable = true info.ControlAvailable = true
if r, err := c.client.ListNodes(c.getRequestContext(), &swarmapi.ListNodesRequest{}); err == nil { if r, err := c.client.ListNodes(ctx, &swarmapi.ListNodesRequest{}); err == nil {
info.Nodes = len(r.Nodes) info.Nodes = len(r.Nodes)
for _, n := range r.Nodes { for _, n := range r.Nodes {
if n.ManagerStatus != nil { if n.ManagerStatus != nil {
@ -602,7 +612,7 @@ func (c *Cluster) Info() types.Info {
} }
} }
if swarm, err := getSwarm(c.getRequestContext(), c.client); err == nil && swarm != nil { if swarm, err := getSwarm(ctx, c.client); err == nil && swarm != nil {
info.CACertHash = swarm.RootCA.CACertHash info.CACertHash = swarm.RootCA.CACertHash
} }
} }
@ -647,8 +657,11 @@ func (c *Cluster) GetServices(options apitypes.ServiceListOptions) ([]types.Serv
if err != nil { if err != nil {
return nil, err return nil, err
} }
ctx, cancel := c.getRequestContext()
defer cancel()
r, err := c.client.ListServices( r, err := c.client.ListServices(
c.getRequestContext(), ctx,
&swarmapi.ListServicesRequest{Filters: filters}) &swarmapi.ListServicesRequest{Filters: filters})
if err != nil { if err != nil {
return nil, err return nil, err
@ -672,7 +685,8 @@ func (c *Cluster) CreateService(s types.ServiceSpec, encodedAuth string) (string
return "", c.errNoManager() return "", c.errNoManager()
} }
ctx := c.getRequestContext() ctx, cancel := c.getRequestContext()
defer cancel()
err := populateNetworkID(ctx, c.client, &s) err := populateNetworkID(ctx, c.client, &s)
if err != nil { if err != nil {
@ -709,7 +723,10 @@ func (c *Cluster) GetService(input string) (types.Service, error) {
return types.Service{}, c.errNoManager() return types.Service{}, c.errNoManager()
} }
service, err := getService(c.getRequestContext(), c.client, input) ctx, cancel := c.getRequestContext()
defer cancel()
service, err := getService(ctx, c.client, input)
if err != nil { if err != nil {
return types.Service{}, err return types.Service{}, err
} }
@ -725,7 +742,8 @@ func (c *Cluster) UpdateService(serviceID string, version uint64, spec types.Ser
return c.errNoManager() return c.errNoManager()
} }
ctx := c.getRequestContext() ctx, cancel := c.getRequestContext()
defer cancel()
err := populateNetworkID(ctx, c.client, &spec) err := populateNetworkID(ctx, c.client, &spec)
if err != nil { if err != nil {
@ -746,7 +764,7 @@ func (c *Cluster) UpdateService(serviceID string, version uint64, spec types.Ser
} else { } else {
// this is needed because if the encodedAuth isn't being updated then we // this is needed because if the encodedAuth isn't being updated then we
// shouldn't lose it, and continue to use the one that was already present // shouldn't lose it, and continue to use the one that was already present
currentService, err := getService(c.getRequestContext(), c.client, serviceID) currentService, err := getService(ctx, c.client, serviceID)
if err != nil { if err != nil {
return err return err
} }
@ -758,7 +776,7 @@ func (c *Cluster) UpdateService(serviceID string, version uint64, spec types.Ser
} }
_, err = c.client.UpdateService( _, err = c.client.UpdateService(
c.getRequestContext(), ctx,
&swarmapi.UpdateServiceRequest{ &swarmapi.UpdateServiceRequest{
ServiceID: serviceID, ServiceID: serviceID,
Spec: &serviceSpec, Spec: &serviceSpec,
@ -779,12 +797,15 @@ func (c *Cluster) RemoveService(input string) error {
return c.errNoManager() return c.errNoManager()
} }
service, err := getService(c.getRequestContext(), c.client, input) ctx, cancel := c.getRequestContext()
defer cancel()
service, err := getService(ctx, c.client, input)
if err != nil { if err != nil {
return err return err
} }
if _, err := c.client.RemoveService(c.getRequestContext(), &swarmapi.RemoveServiceRequest{ServiceID: service.ID}); err != nil { if _, err := c.client.RemoveService(ctx, &swarmapi.RemoveServiceRequest{ServiceID: service.ID}); err != nil {
return err return err
} }
return nil return nil
@ -803,8 +824,12 @@ func (c *Cluster) GetNodes(options apitypes.NodeListOptions) ([]types.Node, erro
if err != nil { if err != nil {
return nil, err return nil, err
} }
ctx, cancel := c.getRequestContext()
defer cancel()
r, err := c.client.ListNodes( r, err := c.client.ListNodes(
c.getRequestContext(), ctx,
&swarmapi.ListNodesRequest{Filters: filters}) &swarmapi.ListNodesRequest{Filters: filters})
if err != nil { if err != nil {
return nil, err return nil, err
@ -827,7 +852,10 @@ func (c *Cluster) GetNode(input string) (types.Node, error) {
return types.Node{}, c.errNoManager() return types.Node{}, c.errNoManager()
} }
node, err := getNode(c.getRequestContext(), c.client, input) ctx, cancel := c.getRequestContext()
defer cancel()
node, err := getNode(ctx, c.client, input)
if err != nil { if err != nil {
return types.Node{}, err return types.Node{}, err
} }
@ -848,8 +876,11 @@ func (c *Cluster) UpdateNode(nodeID string, version uint64, spec types.NodeSpec)
return err return err
} }
ctx, cancel := c.getRequestContext()
defer cancel()
_, err = c.client.UpdateNode( _, err = c.client.UpdateNode(
c.getRequestContext(), ctx,
&swarmapi.UpdateNodeRequest{ &swarmapi.UpdateNodeRequest{
NodeID: nodeID, NodeID: nodeID,
Spec: &nodeSpec, Spec: &nodeSpec,
@ -870,7 +901,8 @@ func (c *Cluster) RemoveNode(input string) error {
return c.errNoManager() return c.errNoManager()
} }
ctx := c.getRequestContext() ctx, cancel := c.getRequestContext()
defer cancel()
node, err := getNode(ctx, c.client, input) node, err := getNode(ctx, c.client, input)
if err != nil { if err != nil {
@ -922,8 +954,12 @@ func (c *Cluster) GetTasks(options apitypes.TaskListOptions) ([]types.Task, erro
if err != nil { if err != nil {
return nil, err return nil, err
} }
ctx, cancel := c.getRequestContext()
defer cancel()
r, err := c.client.ListTasks( r, err := c.client.ListTasks(
c.getRequestContext(), ctx,
&swarmapi.ListTasksRequest{Filters: filters}) &swarmapi.ListTasksRequest{Filters: filters})
if err != nil { if err != nil {
return nil, err return nil, err
@ -946,7 +982,10 @@ func (c *Cluster) GetTask(input string) (types.Task, error) {
return types.Task{}, c.errNoManager() return types.Task{}, c.errNoManager()
} }
task, err := getTask(c.getRequestContext(), c.client, input) ctx, cancel := c.getRequestContext()
defer cancel()
task, err := getTask(ctx, c.client, input)
if err != nil { if err != nil {
return types.Task{}, err return types.Task{}, err
} }
@ -962,7 +1001,10 @@ func (c *Cluster) GetNetwork(input string) (apitypes.NetworkResource, error) {
return apitypes.NetworkResource{}, c.errNoManager() return apitypes.NetworkResource{}, c.errNoManager()
} }
network, err := getNetwork(c.getRequestContext(), c.client, input) ctx, cancel := c.getRequestContext()
defer cancel()
network, err := getNetwork(ctx, c.client, input)
if err != nil { if err != nil {
return apitypes.NetworkResource{}, err return apitypes.NetworkResource{}, err
} }
@ -978,7 +1020,10 @@ func (c *Cluster) GetNetworks() ([]apitypes.NetworkResource, error) {
return nil, c.errNoManager() return nil, c.errNoManager()
} }
r, err := c.client.ListNetworks(c.getRequestContext(), &swarmapi.ListNetworksRequest{}) ctx, cancel := c.getRequestContext()
defer cancel()
r, err := c.client.ListNetworks(ctx, &swarmapi.ListNetworksRequest{})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1006,8 +1051,11 @@ func (c *Cluster) CreateNetwork(s apitypes.NetworkCreateRequest) (string, error)
return "", errors.NewRequestForbiddenError(err) return "", errors.NewRequestForbiddenError(err)
} }
ctx, cancel := c.getRequestContext()
defer cancel()
networkSpec := convert.BasicNetworkCreateToGRPC(s) networkSpec := convert.BasicNetworkCreateToGRPC(s)
r, err := c.client.CreateNetwork(c.getRequestContext(), &swarmapi.CreateNetworkRequest{Spec: &networkSpec}) r, err := c.client.CreateNetwork(ctx, &swarmapi.CreateNetworkRequest{Spec: &networkSpec})
if err != nil { if err != nil {
return "", err return "", err
} }
@ -1024,12 +1072,15 @@ func (c *Cluster) RemoveNetwork(input string) error {
return c.errNoManager() return c.errNoManager()
} }
network, err := getNetwork(c.getRequestContext(), c.client, input) ctx, cancel := c.getRequestContext()
defer cancel()
network, err := getNetwork(ctx, c.client, input)
if err != nil { if err != nil {
return err return err
} }
if _, err := c.client.RemoveNetwork(c.getRequestContext(), &swarmapi.RemoveNetworkRequest{NetworkID: network.ID}); err != nil { if _, err := c.client.RemoveNetwork(ctx, &swarmapi.RemoveNetworkRequest{NetworkID: network.ID}); err != nil {
return err return err
} }
return nil return nil
@ -1095,7 +1146,8 @@ func (c *Cluster) Cleanup() {
} }
func (c *Cluster) managerStats() (current bool, reachable int, unreachable int, err error) { func (c *Cluster) managerStats() (current bool, reachable int, unreachable int, err error) {
ctx, _ := context.WithTimeout(context.Background(), 3*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
nodes, err := c.client.ListNodes(ctx, &swarmapi.ListNodesRequest{}) nodes, err := c.client.ListNodes(ctx, &swarmapi.ListNodesRequest{})
if err != nil { if err != nil {
return false, 0, 0, err return false, 0, 0, err