ci: run integration tests & fix race condition (#2947)

* Fix integration tests race condition
* Run integration tests on CI
* Use existing DIND version

Signed-off-by: Leonard Kinday <leonard@kinday.ru>

Co-authored-by: Milas Bowman <milas.bowman@docker.com>

.github/workflows/ci.yml

@@ -14,10 +14,9 @@ jobs:
             - name: Run flake8
               run: flake8 docker/ tests/
 
-    build:
+    unit-tests:
         runs-on: ubuntu-latest
         strategy:
-            max-parallel: 1
             matrix:
                 python-version: ["3.7", "3.8", "3.9", "3.10", "3.11.0-alpha - 3.11.0"]
 
@@ -26,13 +25,27 @@ jobs:
             - name: Set up Python ${{ matrix.python-version }}
               uses: actions/setup-python@v4
               with:
-                  python-version: ${{ matrix.python-version }}
+                python-version: ${{ matrix.python-version }}
             - name: Install dependencies
               run: |
                   python3 -m pip install --upgrade pip
                   pip3 install -r test-requirements.txt -r requirements.txt
-            - name: Test with pytest
+            - name: Run unit tests
               run: |
                   docker logout
                   rm -rf ~/.docker
                   py.test -v --cov=docker tests/unit
+
+    integration-tests:
+        runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                variant: [ "integration-dind", "integration-dind-ssl" ]
+
+        steps:
+            - uses: actions/checkout@v3
+            - name: make ${{ matrix.variant }}
+              run: |
+                  docker logout
+                  rm -rf ~/.docker
+                  make ${{ matrix.variant }}

Makefile

@@ -1,5 +1,5 @@
 TEST_API_VERSION ?= 1.41
-TEST_ENGINE_VERSION ?= 20.10.05
+TEST_ENGINE_VERSION ?= 20.10
 
 .PHONY: all
 all: test
@@ -46,10 +46,32 @@ integration-dind: integration-dind-py3
 .PHONY: integration-dind-py3
 integration-dind-py3: build-py3 setup-network
 	docker rm -vf dpy-dind-py3 || :
-	docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\
+
-		docker:${TEST_ENGINE_VERSION}-dind dockerd -H tcp://0.0.0.0:2375 --experimental
+	docker run \
-	docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\
+		--detach \
-		--network dpy-tests docker-sdk-python3 py.test tests/integration/${file}
+		--name dpy-dind-py3 \
+		--network dpy-tests \
+		--privileged \
+		docker:${TEST_ENGINE_VERSION}-dind \
+		dockerd -H tcp://0.0.0.0:2375 --experimental
+
+	# Wait for Docker-in-Docker to come to life
+	docker run \
+		--network dpy-tests \
+		--rm \
+		--tty \
+		busybox \
+		sh -c 'while ! nc -z dpy-dind-py3 2375; do sleep 1; done'
+
+	docker run \
+		--env="DOCKER_HOST=tcp://dpy-dind-py3:2375" \
+		--env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}" \
+		--network dpy-tests \
+		--rm \
+		--tty \
+		docker-sdk-python3 \
+		py.test tests/integration/${file}
+
 	docker rm -vf dpy-dind-py3
 
 
@@ -66,18 +88,50 @@ integration-ssh-py3: build-dind-ssh build-py3 setup-network
 
 
 .PHONY: integration-dind-ssl
-integration-dind-ssl: build-dind-certs build-py3
+integration-dind-ssl: build-dind-certs build-py3 setup-network
 	docker rm -vf dpy-dind-certs dpy-dind-ssl || :
 	docker run -d --name dpy-dind-certs dpy-dind-certs
-	docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\
+
-		--env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\
+	docker run \
-		--network dpy-tests --network-alias docker -v /tmp --privileged\
+		--detach \
-		docker:${TEST_ENGINE_VERSION}-dind\
+		--env="DOCKER_CERT_PATH=/certs" \
-		dockerd --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\
+		--env="DOCKER_HOST=tcp://localhost:2375" \
-		--tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental
+		--env="DOCKER_TLS_VERIFY=1" \
-	docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\
+		--name dpy-dind-ssl \
-		--env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\
+		--network dpy-tests \
-		--network dpy-tests docker-sdk-python3 py.test tests/integration/${file}
+		--network-alias docker \
+		--privileged \
+		--volume /tmp \
+		--volumes-from dpy-dind-certs \
+		docker:${TEST_ENGINE_VERSION}-dind \
+		dockerd \
+			--tlsverify \
+			--tlscacert=/certs/ca.pem \
+			--tlscert=/certs/server-cert.pem \
+			--tlskey=/certs/server-key.pem \
+			-H tcp://0.0.0.0:2375 \
+			--experimental
+
+	# Wait for Docker-in-Docker to come to life
+	docker run \
+		--network dpy-tests \
+		--rm \
+		--tty \
+		busybox \
+		sh -c 'while ! nc -z dpy-dind-ssl 2375; do sleep 1; done'
+
+	docker run \
+		--env="DOCKER_CERT_PATH=/certs" \
+		--env="DOCKER_HOST=tcp://docker:2375" \
+		--env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}" \
+		--env="DOCKER_TLS_VERIFY=1" \
+		--network dpy-tests \
+		--rm \
+		--volumes-from dpy-dind-ssl \
+		--tty \
+		docker-sdk-python3 \
+		py.test tests/integration/${file}
+
 	docker rm -vf dpy-dind-ssl dpy-dind-certs
 
 .PHONY: flake8