Merge pull request #1098 from aluzzardi/leadership-integration

Leadership: integration tests

test/integration/helpers.bash

@@ -17,8 +17,8 @@ DOCKER_IMAGE=${DOCKER_IMAGE:-dockerswarm/dind-master}
 DOCKER_VERSION=${DOCKER_VERSION:-latest}
 DOCKER_BINARY=${DOCKER_BINARY:-`command -v docker`}
 
-# Host on which the manager will listen to (random port between 6000 and 7000).
-SWARM_HOST=127.0.0.1:$(( ( RANDOM % 1000 )  + 6000 ))
+# Port on which the manager will listen to (random port between 6000 and 7000).
+SWARM_BASE_PORT=$(( ( RANDOM % 1000 )  + 6000 ))
 
 # Use a random base port (for engines) between 5000 and 6000.
 BASE_PORT=$(( ( RANDOM % 1000 )  + 5000 ))
@@ -50,7 +50,7 @@ function docker_host() {
 
 # Run the docker CLI against swarm.
 function docker_swarm() {
-	docker -H $SWARM_HOST "$@"
+	docker -H ${SWARM_HOSTS[0]} "$@"
 }
 
 # Run the swarm binary. You must NOT fork this command (swarm foo &) as the PID
@@ -94,9 +94,15 @@ function swarm_manage() {
 		discovery="$@"
 	fi
 
-	"$SWARM_BINARY" -l debug manage -H "$SWARM_HOST" --heartbeat=1s "$discovery" &
-	SWARM_PID=$!
-	wait_until_reachable "$SWARM_HOST"
+	local i=${#SWARM_MANAGE_PID[@]}
+	local port=$(($SWARM_BASE_PORT + $i))
+	local host=127.0.0.1:$port
+
+	echo "$SWARM_BINARY" -l debug manage -H "$host" --heartbeat=1s $discovery
+	"$SWARM_BINARY" -l debug manage -H "$host" --heartbeat=1s $discovery &
+	SWARM_MANAGE_PID[$i]=$!
+	SWARM_HOSTS[$i]=$host
+	wait_until_reachable "$host"
 }
 
 # swarm join every engine created with `start_docker`.
@@ -124,7 +130,9 @@ function swarm_join() {
 
 # Stops the manager.
 function swarm_manage_cleanup() {
-	kill $SWARM_PID || true
+	for pid in ${SWARM_MANAGE_PID[@]}; do
+		kill $pid || true
+	done
 }
 
 # Clean up Swarm join processes

test/integration/replication.bats

@@ -0,0 +1,60 @@
+#!/usr/bin/env bats
+
+load helpers
+
+# Address on which the store will listen (random port between 8000 and 9000).
+STORE_HOST=127.0.0.1:$(( ( RANDOM % 1000 )  + 8000 ))
+
+# Discovery parameter for Swarm
+DISCOVERY="consul://${STORE_HOST}/test"
+
+# Container name for integration test
+CONTAINER_NAME=swarm_leader
+
+function start_store() {
+	docker_host run -v $(pwd)/discovery/consul/config:/config --name=$CONTAINER_NAME -h $CONTAINER_NAME -p $STORE_HOST:8500 -d progrium/consul -server -bootstrap-expect 1 -config-file=/config/consul.json
+	# FIXME: We have to wait a few seconds for the store to come up.
+	sleep 3
+}
+
+function stop_store() {
+	docker_host rm -f -v $CONTAINER_NAME
+}
+
+function setup() {
+	start_store
+}
+
+function teardown() {
+	swarm_manage_cleanup
+	swarm_join_cleanup
+	stop_docker
+	stop_store
+}
+
+@test "leader election" {
+	local i=${#SWARM_MANAGE_PID[@]}
+	local port=$(($SWARM_BASE_PORT + $i))
+	local host=127.0.0.1:$port
+
+	# Bring up one manager, make sure it becomes primary.
+	swarm_manage --replication --advertise 127.0.0.1:$SWARM_BASE_PORT "$DISCOVERY"
+	run docker -H ${SWARM_HOSTS[0]} info
+	[[ "${output}" == *"Role: primary"* ]]
+
+	# Fire up a second manager. Ensure it's a replica forwarding to the right primary.
+	swarm_manage --replication --advertise 127.0.0.1:$(($SWARM_BASE_PORT + 1)) "$DISCOVERY"
+	run docker -H ${SWARM_HOSTS[1]} info
+	[[ "${output}" == *"Role: replica"* ]]
+	[[ "${output}" == *"Primary: ${SWARM_HOSTS[0]}"* ]]
+
+	# Kill the leader and ensure the replica takes over.
+	kill "${SWARM_MANAGE_PID[0]}"
+	retry 120 1 eval "docker -H ${SWARM_HOSTS[1]} info | grep -q 'Role: primary'"
+
+	# Add a new replica and make sure it sees the new leader as primary.
+	swarm_manage --replication --advertise 127.0.0.1:$(($SWARM_BASE_PORT + 2)) "$DISCOVERY"
+	run docker -H ${SWARM_HOSTS[2]} info
+	[[ "${output}" == *"Role: replica"* ]]
+	[[ "${output}" == *"Primary: ${SWARM_HOSTS[1]}"* ]]
+}