From b4cc609ef317fd15185d44ef26e5faca9df9d1f3 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 10 Nov 2022 10:32:26 +0100 Subject: [PATCH] build(ci/gha): move builder configuration in a dedicated page Signed-off-by: CrazyMax --- _data/toc.yaml | 2 + _includes/gha-tutorial.md | 29 +- build/ci/github-actions/configure-builder.md | 300 +++++++++++++++++++ build/ci/github-actions/examples.md | 299 +----------------- build/ci/github-actions/index.md | 5 +- 5 files changed, 322 insertions(+), 313 deletions(-) create mode 100644 build/ci/github-actions/configure-builder.md diff --git a/_data/toc.yaml b/_data/toc.yaml index 1727a524d8..373157db3d 100644 --- a/_data/toc.yaml +++ b/_data/toc.yaml @@ -1597,6 +1597,8 @@ manuals: title: Introduction - path: /build/ci/github-actions/examples/ title: Examples + - path: /build/ci/github-actions/configure-builder/ + title: Configuring your builder - path: /build/release-notes/ title: Release notes - sectiontitle: Docker Compose diff --git a/_includes/gha-tutorial.md b/_includes/gha-tutorial.md index c401d77cad..c25b6701ca 100644 --- a/_includes/gha-tutorial.md +++ b/_includes/gha-tutorial.md @@ -73,22 +73,25 @@ target="blank" rel="noopener"}. Now the essentials: what steps to run, and in what order to run them. {% raw %} - ```yaml jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout + - + name: Checkout uses: actions/checkout@v3 - - name: Login to Docker Hub + - + name: Login to Docker Hub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - name: Set up Docker Buildx + - + name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Build and push + - + name: Build and push uses: docker/build-push-action@v3 with: context: . @@ -96,7 +99,6 @@ jobs: push: true tags: ${{ secrets.DOCKER_HUB_USERNAME }}/clockbox:latest ``` - {% endraw %} The previous YAML snippet contains a sequence of steps that: @@ -124,7 +126,6 @@ Add these steps to your workflow file. The full workflow configuration should look as follows: {% raw %} - ```yaml name: ci @@ -137,16 +138,20 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout + - + name: Checkout uses: actions/checkout@v3 - - name: Login to Docker Hub + - + name: Login to Docker Hub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - name: Set up Docker Buildx + - + name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Build and push + - + name: Build and push uses: docker/build-push-action@v3 with: context: . @@ -154,7 +159,6 @@ jobs: push: true tags: ${{ secrets.DOCKER_HUB_USERNAME }}/clockbox:latest ``` - {% endraw %} ### Run the workflow @@ -175,4 +179,3 @@ Save the workflow file and run the job. If you see the new repository in that list, it means the GitHub Actions successfully pushed the image to Docker Hub! - diff --git a/build/ci/github-actions/configure-builder.md b/build/ci/github-actions/configure-builder.md new file mode 100644 index 0000000000..f82b56c9f0 --- /dev/null +++ b/build/ci/github-actions/configure-builder.md @@ -0,0 +1,300 @@ +--- +title: Configuring your builder +description: Configuring BuildKit instances with GitHub Actions. +keywords: ci, github actions, gha, buildkit, buildx +--- + +This page contains instructions on configuring your BuildKit instances when +using our [Setup Buildx Action](https://github.com/docker/setup-buildx-action){: target="_blank" rel="noopener" class="_" }. + +## Daemon configuration + +You can provide a [BuildKit configuration](../../buildkit/toml-configuration.md) +to your builder if you're using the [`docker-container` driver](../../building/drivers/docker-container.md) +(default) with the `config` or `config-inline` inputs: + +### Registry mirror + +You can configure a registry mirror using an inline block directly in your +workflow with the `config-inline` input: + +```yaml +name: ci + +on: + push: + +jobs: + buildx: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + config-inline: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] +``` + +For more information about using a registry mirror, see [Registry mirror](../../buildkit/configure.md#registry-mirror). + +### Max parallelism + +You can limit the parallelism of the BuildKit solver which is particularly +useful for low-powered machines. + +You can use the `config-inline` input like the previous example, or you can use +a dedicated BuildKit config file from your repository if you want with the +`config` input: + +```toml +# .github/buildkitd.toml +[worker.oci] + max-parallelism = 4 +``` + +```yaml +name: ci + +on: + push: + +jobs: + buildx: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + config: .github/buildkitd.toml +``` + +## Append additional nodes to the builder + +Buildx supports running builds on multiple machines. This is useful for building +[multi-platform images](../../building/multi-platform.md) on native nodes for +more complicated cases that aren't handled by QEMU. Building on native nodes +generally has better performance, and allows you to distribute the build across +multiple machines. + +You can append nodes to the builder you're creating using the `append` option. +It takes input in the form of a YAML string document to remove limitations +intrinsically linked to GitHub Actions: you can only use strings in the input +fields: + +| Name | Type | Description | +|-------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `name` | String | [Name of the node](../../../engine/reference/commandline/buildx_create.md#node). If empty, it's the name of the builder it belongs to, with an index number suffix. This is useful to set it if you want to modify/remove a node in an underlying step of you workflow. | +| `endpoint` | String | [Docker context or endpoint](../../../engine/reference/commandline/buildx_create.md#description) of the node to add to the builder | +| `driver-opts` | List | List of additional [driver-specific options](../../../engine/reference/commandline/buildx_create.md#driver-opt) | +| `buildkitd-flags` | String | [Flags for buildkitd](../../../engine/reference/commandline/buildx_create.md#buildkitd-flags) daemon | +| `platforms` | String | Fixed [platforms](../../../engine/reference/commandline/buildx_create.md#platform) for the node. If not empty, values take priority over the detected ones. | + +Here is an example using remote nodes with the [`remote` driver](../../building/drivers/remote.md) +and [TLS authentication](#tls-authentication): + +{% raw %} +```yaml +name: ci + +on: + push: + +jobs: + buildx: + runs-on: ubuntu-latest + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver: remote + endpoint: tcp://oneprovider:1234 + append: | + - endpoint: tcp://graviton2:1234 + platforms: linux/arm64 + - endpoint: tcp://linuxone:1234 + platforms: linux/s390x + env: + BUILDER_NODE_0_AUTH_TLS_CACERT: ${{ secrets.ONEPROVIDER_CA }} + BUILDER_NODE_0_AUTH_TLS_CERT: ${{ secrets.ONEPROVIDER_CERT }} + BUILDER_NODE_0_AUTH_TLS_KEY: ${{ secrets.ONEPROVIDER_KEY }} + BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.GRAVITON2_CA }} + BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.GRAVITON2_CERT }} + BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.GRAVITON2_KEY }} + BUILDER_NODE_2_AUTH_TLS_CACERT: ${{ secrets.LINUXONE_CA }} + BUILDER_NODE_2_AUTH_TLS_CERT: ${{ secrets.LINUXONE_CERT }} + BUILDER_NODE_2_AUTH_TLS_KEY: ${{ secrets.LINUXONE_KEY }} +``` +{% endraw %} + +## Authentication for remote builders + +The following examples show how to handle authentication for remote builders, +using SSH or TLS. + +### SSH authentication + +To be able to connect to an SSH endpoint using the [`docker-container` driver](../../building/drivers/docker-container.md), +you have to set up the SSH private key and configuration on the GitHub Runner: + +{% raw %} +```yaml +name: ci + +on: + push: + +jobs: + buildx: + runs-on: ubuntu-latest + steps: + - + name: Set up SSH + uses: MrSquaare/ssh-setup-action@523473d91581ccbf89565e12b40faba93f2708bd # v1.1.0 + with: + host: graviton2 + private-key: ${{ secrets.SSH_PRIVATE_KEY }} + private-key-name: aws_graviton2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + endpoint: ssh://me@graviton2 +``` +{% endraw %} + +### TLS authentication + +You can also [set up a remote BuildKit instance](../../building/drivers/remote.md#example-remote-buildkit-in-docker-container) +using the remote driver. To ease the integration in your workflow, you can use +an environment variables that sets up authentication using the BuildKit client +certificates for the `tcp://`: + +- `BUILDER_NODE__AUTH_TLS_CACERT` +- `BUILDER_NODE__AUTH_TLS_CERT` +- `BUILDER_NODE__AUTH_TLS_KEY` + +The `` placeholder is the position of the node in the list of nodes. + +{% raw %} +```yaml +name: ci + +on: + push: + +jobs: + buildx: + runs-on: ubuntu-latest + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver: remote + endpoint: tcp://graviton2:1234 + env: + BUILDER_NODE_0_AUTH_TLS_CACERT: ${{ secrets.GRAVITON2_CA }} + BUILDER_NODE_0_AUTH_TLS_CERT: ${{ secrets.GRAVITON2_CERT }} + BUILDER_NODE_0_AUTH_TLS_KEY: ${{ secrets.GRAVITON2_KEY }} +``` +{% endraw %} + +## Standalone mode + +If you don't have the Docker CLI installed on the GitHub Runner, the Buildx +binary gets invoked directly, instead of calling it as a Docker CLI plugin. This +can be useful if you want to use the `kubernetes` driver in your self-hosted +runner: + +```yaml +name: ci + +on: + push: + +jobs: + buildx: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver: kubernetes + - + name: Build + run: | + buildx build . +``` + +## Isolated builders + +The following example shows how you can select different builders for different +jobs. + +An example scenario where this might be useful is when you are using a monorepo, +and you want to pinpoint different packages to specific builders. For example, +some packages may be particularly resource-intensive to build and require more +compute. Or they require a builder equipped with a particular capability or +hardware. + +For more information about remote builder, see [`remote` driver](../../building/drivers/remote.md) +and the [append builder nodes example](#append-additional-nodes-to-the-builder). + +{% raw %} +```yaml +name: ci + +on: + push: + branches: + - "main" + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + uses: docker/setup-buildx-action@v2 + id: builder1 + - + uses: docker/setup-buildx-action@v2 + id: builder2 + - + name: Builder 1 name + run: echo ${{ steps.builder1.outputs.name }} + - + name: Builder 2 name + run: echo ${{ steps.builder2.outputs.name }} + - + name: Build against builder1 + uses: docker/build-push-action@v3 + with: + builder: ${{ steps.builder1.outputs.name }} + context: . + target: mytarget1 + - + name: Build against builder2 + uses: docker/build-push-action@v3 + with: + builder: ${{ steps.builder2.outputs.name }} + context: . + target: mytarget2 +``` +{% endraw %} diff --git a/build/ci/github-actions/examples.md b/build/ci/github-actions/examples.md index 928a973c04..663a977b47 100644 --- a/build/ci/github-actions/examples.md +++ b/build/ci/github-actions/examples.md @@ -1,7 +1,7 @@ --- title: Example workflows description: Docker GitHub Actions workflow examples. -keywords: CI, GitHub Actions, examples +keywords: ci, github actions, gha, examples --- This page showcases different examples of how you can customize and use the @@ -816,303 +816,6 @@ jobs: tags: myimage:latest ``` -## Builder configuration - -This section contains instructions on configuring your BuildKit build instances -when using GitHub Actions. - -### Append additional nodes to the builder - -Buildx supports running builds on multiple machines. This is useful for building -[multi-platform images](../../building/multi-platform.md) on native nodes for -more complicated cases that aren't handled by QEMU. Building on native nodes -generally has better performance, and allows you to distribute the build across -multiple machines. - -You can append nodes to the builder you're creating using the `append` option. -It takes input in the form of a YAML string document to remove limitations -intrinsically linked to GitHub Actions: you can only use strings in the input -fields: - -| Name | Type | Description | -|-------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `name` | String | [Name of the node](../../../engine/reference/commandline/buildx_create.md#node). If empty, it's the name of the builder it belongs to, with an index number suffix. This is useful to set it if you want to modify/remove a node in an underlying step of you workflow. | -| `endpoint` | String | [Docker context or endpoint](../../../engine/reference/commandline/buildx_create.md#description) of the node to add to the builder | -| `driver-opts` | List | List of additional [driver-specific options](../../../engine/reference/commandline/buildx_create.md#driver-opt) | -| `buildkitd-flags` | String | [Flags for buildkitd](../../../engine/reference/commandline/buildx_create.md#buildkitd-flags) daemon | -| `platforms` | String | Fixed [platforms](../../../engine/reference/commandline/buildx_create.md#platform) for the node. If not empty, values take priority over the detected ones. | - -Here is an example using remote nodes with the [`remote` driver](../../building/drivers/remote.md) -and [TLS authentication](#tls-authentication): - -{% raw %} -```yaml -name: ci - -on: - push: - -jobs: - buildx: - runs-on: ubuntu-latest - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - driver: remote - endpoint: tcp://oneprovider:1234 - append: | - - endpoint: tcp://graviton2:1234 - platforms: linux/arm64 - - endpoint: tcp://linuxone:1234 - platforms: linux/s390x - env: - BUILDER_NODE_0_AUTH_TLS_CACERT: ${{ secrets.ONEPROVIDER_CA }} - BUILDER_NODE_0_AUTH_TLS_CERT: ${{ secrets.ONEPROVIDER_CERT }} - BUILDER_NODE_0_AUTH_TLS_KEY: ${{ secrets.ONEPROVIDER_KEY }} - BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.GRAVITON2_CA }} - BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.GRAVITON2_CERT }} - BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.GRAVITON2_KEY }} - BUILDER_NODE_2_AUTH_TLS_CACERT: ${{ secrets.LINUXONE_CA }} - BUILDER_NODE_2_AUTH_TLS_CERT: ${{ secrets.LINUXONE_CERT }} - BUILDER_NODE_2_AUTH_TLS_KEY: ${{ secrets.LINUXONE_KEY }} -``` -{% endraw %} - -### Authentication for remote builders - -The following examples show how to handle authentication for remote builders, -using SSH or TLS. - -#### SSH authentication - -To be able to connect to an SSH endpoint using the [`docker-container` driver](../../building/drivers/docker-container.md), -you have to set up the SSH private key and configuration on the GitHub Runner: - -{% raw %} -```yaml -name: ci - -on: - push: - -jobs: - buildx: - runs-on: ubuntu-latest - steps: - - - name: Set up SSH - uses: MrSquaare/ssh-setup-action@523473d91581ccbf89565e12b40faba93f2708bd # v1.1.0 - with: - host: graviton2 - private-key: ${{ secrets.SSH_PRIVATE_KEY }} - private-key-name: aws_graviton2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - endpoint: ssh://me@graviton2 -``` -{% endraw %} - -#### TLS authentication - -You can also [set up a remote BuildKit instance](../../building/drivers/remote.md#example-remote-buildkit-in-docker-container) -using the remote driver. To ease the integration in your workflow, you can use -an environment variables that sets up authentication using the BuildKit client -certificates for the `tcp://`: - -- `BUILDER_NODE__AUTH_TLS_CACERT` -- `BUILDER_NODE__AUTH_TLS_CERT` -- `BUILDER_NODE__AUTH_TLS_KEY` - -The `` placeholder is the position of the node in the list of nodes. - -{% raw %} -```yaml -name: ci - -on: - push: - -jobs: - buildx: - runs-on: ubuntu-latest - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - driver: remote - endpoint: tcp://graviton2:1234 - env: - BUILDER_NODE_0_AUTH_TLS_CACERT: ${{ secrets.GRAVITON2_CA }} - BUILDER_NODE_0_AUTH_TLS_CERT: ${{ secrets.GRAVITON2_CERT }} - BUILDER_NODE_0_AUTH_TLS_KEY: ${{ secrets.GRAVITON2_KEY }} -``` -{% endraw %} - -### Daemon configuration - -You can provide a [BuildKit configuration](../../buildkit/toml-configuration.md) -to your builder if you're using the [`docker-container` driver](../../building/drivers/docker-container.md) -(default) with the `config` or `config-inline` inputs: - -### Registry mirror - -You can configure a registry mirror using an inline block directly in your -workflow with the `config-inline` input: - -```yaml -name: ci - -on: - push: - -jobs: - buildx: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config-inline: | - [registry."docker.io"] - mirrors = ["mirror.gcr.io"] -``` - -For more information about using a registry mirror, see [Registry mirror](../../buildkit/configure.md#registry-mirror). - -#### Max parallelism - -You can limit the parallelism of the BuildKit solver which is particularly -useful for low-powered machines. - -You can use the `config-inline` input like the previous example, or you can use -a dedicated BuildKit config file from your repository if you want with the -`config` input: - -```toml -# .github/buildkitd.toml -[worker.oci] - max-parallelism = 4 -``` - -```yaml -name: ci - -on: - push: - -jobs: - buildx: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config: .github/buildkitd.toml -``` - -### Standalone mode - -If you don't have the Docker CLI installed on the GitHub Runner, the Buildx -binary gets invoked directly, instead of calling it as a Docker CLI plugin. This -can be useful if you want to use the `kubernetes` driver in your self-hosted -runner: - -```yaml -name: ci - -on: - push: - -jobs: - buildx: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - driver: kubernetes - - - name: Build - run: | - buildx build . -``` - -## Isolated builders - -The following example shows how you can select different builders for different -jobs. - -An example scenario where this might be useful is when you are using a monorepo, -and you want to pinpoint different packages to specific builders. For example, -some packages may be particularly resource-intensive to build and require more -compute. Or they require a builder equipped with a particular capability or -hardware. - -For more information about remote builder, see [`remote` driver](../../building/drivers/remote.md) -and the [append builder nodes example](#append-additional-nodes-to-the-builder). - -{% raw %} -```yaml -name: ci - -on: - push: - branches: - - "main" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - uses: docker/setup-buildx-action@v2 - id: builder1 - - - uses: docker/setup-buildx-action@v2 - id: builder2 - - - name: Builder 1 name - run: echo ${{ steps.builder1.outputs.name }} - - - name: Builder 2 name - run: echo ${{ steps.builder2.outputs.name }} - - - name: Build against builder1 - uses: docker/build-push-action@v3 - with: - builder: ${{ steps.builder1.outputs.name }} - context: . - target: mytarget1 - - - name: Build against builder2 - uses: docker/build-push-action@v3 - with: - builder: ${{ steps.builder2.outputs.name }} - context: . - target: mytarget2 -``` -{% endraw %} - ## Copy images between registries [Multi-platform images](../../building/multi-platform.md) built using Buildx can diff --git a/build/ci/github-actions/index.md b/build/ci/github-actions/index.md index 15b8410176..44c6ce9f10 100644 --- a/build/ci/github-actions/index.md +++ b/build/ci/github-actions/index.md @@ -2,7 +2,7 @@ title: Introduction to GitHub Actions description: > Docker maintains a set of official GitHub Actions for building Docker images. -keywords: github, actions, gha, ci, build, introduction, tutorial +keywords: ci, github actions, gha, build, introduction, tutorial redirect_from: - /ci-cd/github-actions/ --- @@ -43,4 +43,5 @@ using the official Docker actions, to build and push an image to Docker Hub. There are many more things you can do to customize your workflow to better suit your needs. To learn more about some of the more advanced use cases, take a look at the advanced examples, such as [building multi-platform images](examples.md#multi-platform-images), -or [using cache storage backends](examples.md#cache). +or [using cache storage backends](examples.md#cache) and also how to +[configure your builder](configure-builder.md).