From 9eb9bd48c41a3fa85aef128f45989321d3339dc5 Mon Sep 17 00:00:00 2001 From: Saj Goonatilleke Date: Thu, 25 Jan 2024 09:31:12 +1100 Subject: [PATCH] init --- .../containers/branch-transformer/Dockerfile | 24 +++ .../containers/branch-transformer/test.py | 119 +++++++++++++ .../containers/branch-transformer/transform | 43 +++++ concourse/plumb-branch.yaml | 29 +++ concourse/plumb.yaml | 166 ++++++++++++++++++ 5 files changed, 381 insertions(+) create mode 100644 concourse/containers/branch-transformer/Dockerfile create mode 100755 concourse/containers/branch-transformer/test.py create mode 100755 concourse/containers/branch-transformer/transform create mode 100644 concourse/plumb-branch.yaml create mode 100644 concourse/plumb.yaml diff --git a/concourse/containers/branch-transformer/Dockerfile b/concourse/containers/branch-transformer/Dockerfile new file mode 100644 index 0000000..f6baa74 --- /dev/null +++ b/concourse/containers/branch-transformer/Dockerfile @@ -0,0 +1,24 @@ +# syntax=docker/dockerfile:1 + +ARG BASE_IMAGE_ALPINE=alpine:3 + + +FROM ${BASE_IMAGE_ALPINE} AS py + +RUN apk -v --no-progress --no-cache add --upgrade python3 + + +FROM py AS test + +RUN apk -v --no-progress --no-cache add --upgrade py3-pytest + +WORKDIR /src +COPY test.py transform . +RUN PYTHONDONTWRITEBYTECODE=1 ./test.py ./transform + + +FROM py + +COPY --from=test /src/transform /usr/local/bin/ + +ENTRYPOINT ["/usr/local/bin/transform"] diff --git a/concourse/containers/branch-transformer/test.py b/concourse/containers/branch-transformer/test.py new file mode 100755 index 0000000..73aacff --- /dev/null +++ b/concourse/containers/branch-transformer/test.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +""" +Exercise the transform program. + +Sample inputs are fed to the transform program on stdin. +The transform program writes its output to stdout. +This test program compares actual output with expected output, +and exits with non-zero status upon any mismatch. +""" + +# This test program is automatically executed on container build. +# Test dependencies, namely pytest, are supplied by the container image. +# +# This file must have a .py extension. +# pytest maintainers refuse to allow test collection otherwise. + +# pylint: disable=missing-function-docstring + +import argparse +import json +import os +import sys +import tempfile +import subprocess +from contextlib import ExitStack + +import pytest + + +def test_null(): + assert run_prog([]) == [] + + +def test_ignores_plumbing_branch(): + assert run_prog([{"name": "plumbing"}]) == [] + + +def test_ignores_trunkish_branch(): + assert (run_prog([{"name": "main"}, + {"name": "master"}]) + == []) + + +def test_topic_branch(): + assert (run_prog([{"name": "bobtest"}]) + == + [{"name": "bobtest", + "tag": "bobtest"}]) + assert (run_prog([{"name": "bob-test"}]) + == + [{"name": "bob-test", + "tag": "bob-test"}]) + + # use - as a substitute for characters that would not be valid in a tag + assert (run_prog([{"name": "bob/frob"}]) + == + [{"name": "bob/frob", + "tag": "bob-frob"}]) + assert (run_prog([{"name": "bob/-frob"}]) + == + [{"name": "bob/-frob", + "tag": "bob-frob"}]) + + +PROG = os.environ.get("TEST_PROG") # see bottom + + +def run_prog(input_obj): + with ExitStack() as st: + fi = st.enter_context(tempfile.TemporaryFile(mode="w+", encoding="utf8")) + fo = st.enter_context(tempfile.TemporaryFile(mode="w+", encoding="utf8")) + json.dump(input_obj, fi) + fi.seek(0) + subprocess.run([PROG], stdin=fi, stdout=fo, check=True) + fo.seek(0) + return json.load(fo) + + +class Args: + + @classmethod + def parse(cls, argv=None): + if argv is None: + argv = sys.argv + parser = argparse.ArgumentParser() + parser.add_argument("transform-prog", + help="Path to the program under test.") + args = parser.parse_args(argv[1:]) + return cls(args) + + def __init__(self, args): + self._args = args + + def __getattr__(self, name): + return getattr(self._args, name) + + @property + def prog(self): + return getattr(self, "transform-prog") + + +def main(argv=None): + if not argv: + argv = sys.argv + # Global bindings we make here are not visible when pytest is executing, + # probably because it (re)imports the module. ¯\_(ツ)_/¯ + # Funnel whatever we need through the process environment instead. + args = Args.parse(argv) + os.environ["TEST_PROG"] = args.prog + return pytest.main(["-vv", # dump full structured diffs upon any mismatch + "-o", "console_output_style=classic", + # suppress useless progress markers + "--tb=short", # suppress outrageously verbose tracebacks + argv[0]]) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/concourse/containers/branch-transformer/transform b/concourse/containers/branch-transformer/transform new file mode 100755 index 0000000..858396a --- /dev/null +++ b/concourse/containers/branch-transformer/transform @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +import json +import re +import sys + + +def main(unused_argv=None): + # https://github.com/aoldershaw/git-branches-resource + # see branches.json + # JSON array of objects containing the field 'name' + input_obj = json.load(sys.stdin) + + all_branches = [b.get("name") for b in input_obj] + + topic_branches = [name for name in all_branches + if name not in ("main", "master", "plumbing")] + + output_obj = [{"name": name, "tag": tag_from_branch(name)} + for name in topic_branches] + + json.dump(output_obj, sys.stdout) + + +PAT_UNSAFE_TAG = re.compile(r"[^0-9a-zA-Z.-]", flags=re.ASCII) +PAT_HYPHEN_MINUS_RUN = re.compile(r"--+") + +def tag_from_branch(branch): + return patsubsts(branch, ((PAT_UNSAFE_TAG, "-"), + (PAT_HYPHEN_MINUS_RUN, "-"))) + + +def patsubsts(s, patsubs): + for pat, sub in patsubs: + s = pat.sub(sub, s) + return s + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/concourse/plumb-branch.yaml b/concourse/plumb-branch.yaml new file mode 100644 index 0000000..43bb7b3 --- /dev/null +++ b/concourse/plumb-branch.yaml @@ -0,0 +1,29 @@ +# SECURITY +# This is a public repository. Mind what you write. +# Do not accept modifications from people outside CDCK. +# Seek infra security review if unsure. + +--- +resources: +- name: discourse-auth-proxy + type: git + icon: github + source: + uri: git@github.com:discourse/discourse-auth-proxy.git + branch: ((branch)) + paths: [dist/concourse/pipeline-branch.yaml] + private_key: ((github-discoursebuild)) + webhook_token: unused-but-some-value-required + +jobs: +- name: set-branch-pipeline + plan: + - get: discourse-auth-proxy + trigger: true + - set_pipeline: auth-proxy-branch + file: discourse-auth-proxy/dist/concourse/pipeline-branch.yaml + instance_vars: + branch: ((branch)) + vars: + image_repository: auth-proxy/test + image_tag: ((image_tag)) diff --git a/concourse/plumb.yaml b/concourse/plumb.yaml new file mode 100644 index 0000000..49e307d --- /dev/null +++ b/concourse/plumb.yaml @@ -0,0 +1,166 @@ +# SECURITY +# This is a public repository. Mind what you write. +# Do not accept modifications from people outside CDCK. +# Seek infra security review if unsure. + +--- +var_sources: +- name: xacco + type: vault + config: + url: http://127.0.0.1:8200 + path_prefix: /aws-xacc-obfuscate + client_token: unused-but-some-value-required + +resource_types: +- name: git-branches + type: registry-image + source: + repository: practical-concourse/resource-types/git-branches + aws_access_key_id: ((xacco:machine/concourse-ecr-pull/docker-registry.AWS_ACCESS_KEY_ID)) + aws_secret_access_key: ((xacco:machine/concourse-ecr-pull/docker-registry.AWS_SECRET_ACCESS_KEY)) + aws_session_token: ((xacco:machine/concourse-ecr-pull/docker-registry.AWS_SESSION_TOKEN)) + aws_region: ((obfuscate-aws-docker-registry.region)) + +resources: +- name: branches + type: git-branches + icon: github + source: + uri: git@github.com:discourse/discourse-auth-proxy.git + private_key: ((github-discoursebuild)) + webhook_token: unused-but-some-value-required + +- name: trunk + type: git + icon: github + source: + uri: git@github.com:discourse/discourse-auth-proxy.git + paths: [dist/concourse/pipeline-trunk.yaml] + private_key: ((github-discoursebuild)) + webhook_token: unused-but-some-value-required + +- name: plumbing + type: git + icon: github + source: + uri: git@github.com:discourse/discourse-auth-proxy.git + branch: plumbing + private_key: ((github-discoursebuild)) + webhook_token: unused-but-some-value-required + +- name: alpine + type: registry-image + icon: docker + check_every: 24h + source: + repository: alpine + tag: "3" + username: ((docker-hub.username)) + password: ((docker-hub.password)) + +- name: branch-transformer + type: registry-image + icon: docker + source: + repository: auth-proxy/concourse/branch-transformer + tag: latest + aws_access_key_id: ((xacco:machine/concourse-ecr-push/docker-registry.AWS_ACCESS_KEY_ID)) + aws_secret_access_key: ((xacco:machine/concourse-ecr-push/docker-registry.AWS_SECRET_ACCESS_KEY)) + aws_session_token: ((xacco:machine/concourse-ecr-push/docker-registry.AWS_SESSION_TOKEN)) + aws_region: ((obfuscate-aws-docker-registry.region)) + +jobs: +- name: set-self-pipeline + plan: + - get: plumbing + trigger: true + - set_pipeline: self + file: plumbing/concourse/plumb.yaml + +- name: set-trunk-pipeline + plan: + - in_parallel: + - get: plumbing + trigger: true + passed: [set-self-pipeline] + - get: trunk + trigger: true + - set_pipeline: auth-proxy + file: trunk/dist/concourse/pipeline-trunk.yaml + +- name: build-pipeline-helpers + plan: + - in_parallel: + - get: plumbing + trigger: true + passed: [set-self-pipeline] + - get: alpine + params: {format: oci} + - task: build + privileged: true + output_mapping: + image: branch-transformer + config: + platform: linux + image_resource: + type: registry-image + source: + repository: concourse/oci-build-task + username: ((docker-hub.username)) + password: ((docker-hub.password)) + inputs: + - name: alpine + - name: plumbing + outputs: + - name: image + caches: + - path: cache + params: + CONTEXT: plumbing/concourse/containers/branch-transformer + DOCKERFILE: plumbing/concourse/containers/branch-transformer/Dockerfile + IMAGE_ARG_BASE_IMAGE_ALPINE: alpine/image.tar + OUTPUT_OCI: true + run: + path: build + - put: branch-transformer + inputs: + - branch-transformer + params: {image: branch-transformer/image} + +- name: set-branch-pipelines + plan: + - in_parallel: + - get: branches + trigger: true + - get: plumbing + trigger: true + passed: + - build-pipeline-helpers + - get: branch-transformer + passed: [build-pipeline-helpers] + - task: transform + image: branch-transformer + config: + platform: linux + inputs: + - name: branches + outputs: + - name: transformed + run: + path: sh + args: + - -exc + - | + exec /usr/local/bin/transform < branches/branches.json > transformed/branches.json + - load_var: branches + file: transformed/branches.json + - across: + - var: branch + values: ((.:branches)) + set_pipeline: plumb-auth-proxy-branch + file: plumbing/concourse/plumb-branch.yaml + instance_vars: + branch: ((.:branch.name)) + vars: + image_tag: ((.:branch.tag))