ci: use single dynamic build-test action

Add a GitHub action that dynamically generates the matrix of Dockerfiles
to test based on what files have changed. This avoids having to have
generated workflows for each version x variant combo.

It looks for any changed Dockerfiles, or docker-entrypoint.sh. If any of
the testing files are changed, it will test all Dockerfiles
This commit is contained in:
Travis Shivers 2020-09-24 21:58:22 -05:00
parent f17e9e7b72
commit ee169b2fdd
No known key found for this signature in database
GPG Key ID: EE4CC2891B8FCD33
28 changed files with 171 additions and 696 deletions

View File

@ -1,24 +0,0 @@
name: 10 on alpine3.10
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/alpine3.10/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/alpine3.10/Dockerfile
jobs:
build:
name: 10 on alpine3.10
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 10 alpine3.10

View File

@ -1,24 +0,0 @@
name: 10 on alpine3.11
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/alpine3.11/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/alpine3.11/Dockerfile
jobs:
build:
name: 10 on alpine3.11
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 10 alpine3.11

View File

@ -1,24 +0,0 @@
name: 10 on alpine3.9
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/alpine3.9/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/alpine3.9/Dockerfile
jobs:
build:
name: 10 on alpine3.9
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 10 alpine3.9

View File

@ -1,24 +0,0 @@
name: 10 on buster-slim
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/buster-slim/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/buster-slim/Dockerfile
jobs:
build:
name: 10 on buster-slim
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 10 buster-slim

View File

@ -1,24 +0,0 @@
name: 10 on buster
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/buster/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/buster/Dockerfile
jobs:
build:
name: 10 on buster
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 10 buster

View File

@ -1,24 +0,0 @@
name: 10 on stretch-slim
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/stretch-slim/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/stretch-slim/Dockerfile
jobs:
build:
name: 10 on stretch-slim
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 10 stretch-slim

View File

@ -1,24 +0,0 @@
name: 10 on stretch
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/stretch/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 10/stretch/Dockerfile
jobs:
build:
name: 10 on stretch
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 10 stretch

View File

@ -1,24 +0,0 @@
name: 12 on alpine3.10
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/alpine3.10/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/alpine3.10/Dockerfile
jobs:
build:
name: 12 on alpine3.10
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 12 alpine3.10

View File

@ -1,24 +0,0 @@
name: 12 on alpine3.11
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/alpine3.11/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/alpine3.11/Dockerfile
jobs:
build:
name: 12 on alpine3.11
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 12 alpine3.11

View File

@ -1,24 +0,0 @@
name: 12 on alpine3.12
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/alpine3.12/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/alpine3.12/Dockerfile
jobs:
build:
name: 12 on alpine3.12
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 12 alpine3.12

View File

@ -1,24 +0,0 @@
name: 12 on alpine3.9
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/alpine3.9/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/alpine3.9/Dockerfile
jobs:
build:
name: 12 on alpine3.9
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 12 alpine3.9

View File

@ -1,24 +0,0 @@
name: 12 on buster-slim
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/buster-slim/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/buster-slim/Dockerfile
jobs:
build:
name: 12 on buster-slim
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 12 buster-slim

View File

@ -1,24 +0,0 @@
name: 12 on buster
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/buster/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/buster/Dockerfile
jobs:
build:
name: 12 on buster
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 12 buster

View File

@ -1,24 +0,0 @@
name: 12 on stretch-slim
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/stretch-slim/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/stretch-slim/Dockerfile
jobs:
build:
name: 12 on stretch-slim
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 12 stretch-slim

View File

@ -1,24 +0,0 @@
name: 12 on stretch
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/stretch/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 12/stretch/Dockerfile
jobs:
build:
name: 12 on stretch
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 12 stretch

View File

@ -1,24 +0,0 @@
name: 14 on alpine3.10
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/alpine3.10/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/alpine3.10/Dockerfile
jobs:
build:
name: 14 on alpine3.10
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 14 alpine3.10

View File

@ -1,24 +0,0 @@
name: 14 on alpine3.11
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/alpine3.11/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/alpine3.11/Dockerfile
jobs:
build:
name: 14 on alpine3.11
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 14 alpine3.11

View File

@ -1,24 +0,0 @@
name: 14 on alpine3.12
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/alpine3.12/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/alpine3.12/Dockerfile
jobs:
build:
name: 14 on alpine3.12
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 14 alpine3.12

View File

@ -1,24 +0,0 @@
name: 14 on buster-slim
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/buster-slim/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/buster-slim/Dockerfile
jobs:
build:
name: 14 on buster-slim
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 14 buster-slim

View File

@ -1,24 +0,0 @@
name: 14 on buster
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/buster/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/buster/Dockerfile
jobs:
build:
name: 14 on buster
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 14 buster

View File

@ -1,24 +0,0 @@
name: 14 on stretch-slim
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/stretch-slim/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/stretch-slim/Dockerfile
jobs:
build:
name: 14 on stretch-slim
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 14 stretch-slim

View File

@ -1,24 +0,0 @@
name: 14 on stretch
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/stretch/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- 14/stretch/Dockerfile
jobs:
build:
name: 14 on stretch
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh 14 stretch

89
.github/workflows/build-test.yml vendored Normal file
View File

@ -0,0 +1,89 @@
name: build-test
on:
push:
paths:
- "**/Dockerfile"
- "**/docker-entrypoint.sh"
- genMatrix.js
- ".github/workflows/build-test.yml"
pull_request:
paths:
- "**/Dockerfile"
- "**/docker-entrypoint.sh"
- genMatrix.js
- ".github/workflows/build-test.yml"
jobs:
gen-matrix:
name: generate-matrix
runs-on: ubuntu-latest
steps:
- name: Calculate file differences
uses: lots0logs/gh-action-get-changed-files@2.1.4
id: diff
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Checkout
uses: actions/checkout@v2
- name: Generate testing matrix
uses: actions/github-script@v3
id: generator
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const script = require(`${process.env.GITHUB_WORKSPACE}/genMatrix.js`)
return script(
${{ steps.diff.outputs.added }},
${{ steps.diff.outputs.modified }},
${{ steps.diff.outputs.renamed }},
);
outputs:
matrix: ${{ steps.generator.outputs.result }}
build:
if: ${{ fromJson(needs.gen-matrix.outputs.matrix) }}
needs: gen-matrix
name: build
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.gen-matrix.outputs.matrix) }}
steps:
- name: Get short node version
uses: actions/github-script@v3
id: short-version
with:
result-encoding: string
script: return "${{ matrix.version }}".split('.')[0]
- name: Checkout
uses: actions/checkout@v2
- name: Build image
uses: docker/build-push-action@v2
with:
push: false
load: true
context: .
file: ./${{ steps.short-version.outputs.result }}/${{ matrix.variant }}/Dockerfile
tags: node:${{ matrix.version }}-${{ matrix.variant }}
- name: Test for node version
run: |
image_node_version=$(docker run --rm node:${{ matrix.version }}-${{ matrix.variant }} node --print "process.versions.node")
echo "Expected: \"${{ matrix.version }}\", Got: \"${image_node_version}\""
[ "${image_node_version}" == "${{ matrix.version }}" ]
- name: Test for npm
run: docker run --rm node:${{ matrix.version }}-${{ matrix.variant }} npm --version
- name: Test for yarn
run: docker run --rm node:${{ matrix.version }}-${{ matrix.variant }} yarn --version

View File

@ -1,11 +0,0 @@
name: Check for out of sync YAML pipeline files
on: [pull_request]
jobs:
regen:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: ./update.sh -t
- run: git diff --color --exit-code

80
genMatrix.js Normal file
View File

@ -0,0 +1,80 @@
'use strict';
const path = require('path');
const fs = require('fs');
const testFiles = [
'genMatrix.js',
'.github/workflows/build-test.yml',
];
const nodeDirRegex = /^\d+$/;
const areTestFilesChanged = (changedFiles) => changedFiles
.some((file) => testFiles.includes(file));
// Returns a list of the child directories in the given path
const getChildDirectories = (parent) => fs.readdirSync(parent, { withFileTypes: true })
.filter((dirent) => dirent.isDirectory())
.map(({ name }) => path.resolve(parent, name));
const getNodeVerionDirs = (base) => getChildDirectories(base)
.filter((childPath) => nodeDirRegex.test(path.basename(childPath)));
// Returns the paths of Dockerfiles that are at: base/*/Dockerfile
const getDockerfilesInChildDirs = (base) => getChildDirectories(base)
.map((childDir) => path.resolve(childDir, 'Dockerfile'));
const getAllDockerfiles = (base) => getNodeVerionDirs(base).flatMap(getDockerfilesInChildDirs);
const getAffectedDockerfiles = (filesAdded, filesModified, filesRenamed) => {
const files = [
...filesAdded,
...filesModified,
...filesRenamed,
];
// If the test files were changed, include everything
if (areTestFilesChanged(files)) {
console.log('Test files changed so scheduling all Dockerfiles');
return getAllDockerfiles(__dirname);
}
const modifiedDockerfiles = files.filter((file) => file.endsWith('/Dockerfile'));
// Get Dockerfiles affected by modified docker-entrypoint.sh files
const entrypointAffectedDockerfiles = files
.filter((file) => file.endsWith('/docker-entrypoint.sh'))
.map((file) => path.resolve(path.dirname(file), 'Dockerfile'));
return [
...modifiedDockerfiles,
...entrypointAffectedDockerfiles,
];
};
const getFullNodeVersionFromDockerfile = (file) => fs.readFileSync(file, 'utf8')
.match(/^ENV NODE_VERSION (\d*\.*\d*\.\d*)/m)[1];
const getDockerfileMatrixEntry = (file) => {
const [variant] = path.dirname(file).split(path.sep).slice(-1);
const version = getFullNodeVersionFromDockerfile(file);
return {
version,
variant,
};
};
const generateBuildMatrix = (filesAdded, filesModified, filesRenamed) => {
const dockerfiles = [...new Set(getAffectedDockerfiles(filesAdded, filesModified, filesRenamed))];
const entries = dockerfiles.map(getDockerfileMatrixEntry);
// Return null if there are no entries so we can skip the matrix step
return entries.length
? { include: entries }
: null;
};
module.exports = generateBuildMatrix;

View File

@ -1,93 +0,0 @@
#!/usr/bin/env bash
#
# Run a test build for all images.
set -euo pipefail
. functions.sh
# Convert comma delimited cli arguments to arrays
# E.g. ./test-build.sh 10,12 slim,alpine
# "10,12" becomes "10 12" and "slim,alpine" becomes "slim alpine"
IFS=',' read -ra versions_arg <<< "${1:-}"
IFS=',' read -ra variant_arg <<< "${2:-}"
default_variant=$(get_config "./" "default_variant")
function build() {
local version
local tag
local variant
local full_tag
local path
version="$1"
shift
variant="$1"
shift
tag="$1"
shift
full_tag=$(get_full_tag "${variant}" "${tag}")
path=$(get_path "${version}" "${variant}")
info "Building ${full_tag}..."
if ! docker build --cpuset-cpus="0,1" -t node:"${full_tag}" "${path}"; then
fatal "Build of ${full_tag} failed!"
fi
info "Build of ${full_tag} succeeded."
}
function test_image() {
local full_version
local variant
local tag
local full_tag
full_version="$1"
shift
variant="$1"
shift
tag="$1"
shift
full_tag=$(get_full_tag "${variant}" "${tag}")
info "Testing ${full_tag}"
(
export full_version=${full_version}
export full_tag=${full_tag}
bats test-image.bats
)
}
cd "$(cd "${0%/*}" && pwd -P)" || exit
IFS=' ' read -ra versions <<< "$(get_versions . "${versions_arg[@]}")"
if [ ${#versions[@]} -eq 0 ]; then
fatal "No valid versions found!"
fi
for version in "${versions[@]}"; do
# Skip "docs" and other non-docker directories
[ -f "${version}/Dockerfile" ] || [ -a "${version}/${default_variant}/Dockerfile" ] || continue
tag=$(get_tag "${version}")
full_version=$(get_full_version "${version}")
# Get supported variants according to the target architecture.
# See details in function.sh
IFS=' ' read -ra variants <<< "$(get_variants "$(dirname "${version}")" "${variant_arg[@]}")"
for variant in "${variants[@]}"; do
# Skip non-docker directories
[ -f "${version}/${variant}/Dockerfile" ] || continue
build "${version}" "${variant}" "${tag}"
test_image "${full_version}" "${variant}" "${tag}"
done
done
info "All builds successful!"
exit 0

View File

@ -1,17 +0,0 @@
#!/usr/bin/env bats
@test "Test for node and version" {
run docker run --rm node:"$full_tag" node -e "process.stdout.write(process.versions.node)"
[ "$status" -eq 0 ]
[ "$output" == "${full_version}" ]
}
@test "Test for npm" {
run docker run --rm node:"$full_tag" npm --version
[ "$status" -eq 0 ]
}
@test "Test for yarn" {
run docker run --rm node:"$full_tag" yarn --version
[ "$status" -eq 0 ]
}

View File

@ -18,7 +18,6 @@ function usage() {
- update.sh 8 buster-slim,buster # Update only buster's slim and buster variants for version 8
- update.sh -s 8 stretch # Update only stretch variant for version 8, skip updating Alpine and Yarn
- update.sh . alpine # Update the alpine variant for all versions
- update.sh -b # Update CI files only
OPTIONS:
-s Security update; skip updating the yarn and alpine versions.
@ -29,17 +28,12 @@ EOF
}
SKIP=false
CI_ONLY=false
while getopts "sbh" opt; do
while getopts "sh" opt; do
case "${opt}" in
s)
SKIP=true
shift
;;
b)
CI_ONLY=true
shift
;;
h)
usage
exit
@ -197,56 +191,19 @@ function update_node_version() {
)
}
function add_stage() {
local baseuri=${1}
shift
local version=${1}
shift
local variant=${1}
shift
echo "name: ${version} on ${variant}
on:
push:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- ${version}/${variant}/Dockerfile
pull_request:
paths:
- functions.sh
- test-build.sh
- test-image.bats
- ${version}/${variant}/Dockerfile
jobs:
build:
name: ${version} on ${variant}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: sudo apt-get install bats
- run: ./test-build.sh ${version} ${variant}" > ".github/workflows/${version}-${variant}.yml"
}
for version in "${versions[@]}"; do
parentpath=$(dirname "${version}")
versionnum=$(basename "${version}")
baseuri=$(get_config "${parentpath}" "baseuri")
update_version=$(in_versions_to_update "${version}")
[ "${update_version}" -eq 0 ] && [ true != "$CI_ONLY" ] && info "Updating version ${version}..."
[ "${update_version}" -eq 0 ] && info "Updating version ${version}..."
# Get supported variants according the target architecture
# See details in function.sh
IFS=' ' read -ra variants <<< "$(get_variants "${parentpath}")"
if [ -f "${version}/Dockerfile" ]; then
add_stage "${baseuri}" "${version}" "default"
[ true = "$CI_ONLY" ] && continue
if [ "${update_version}" -eq 0 ]; then
update_node_version "${baseuri}" "${versionnum}" "${parentpath}/Dockerfile.template" "${version}/Dockerfile" &
fi
@ -255,8 +212,6 @@ for version in "${versions[@]}"; do
for variant in "${variants[@]}"; do
# Skip non-docker directories
[ -f "${version}/${variant}/Dockerfile" ] || continue
add_stage "${baseuri}" "${version}" "${variant}"
[ true = "$CI_ONLY" ] && continue
update_variant=$(in_variants_to_update "${variant}")
template_file="${parentpath}/Dockerfile-${variant}.template"