From 39963cc984f21827b5174db2ffe9dfa72f007f0f Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Tue, 1 Oct 2024 11:04:44 +0200 Subject: [PATCH 1/8] cirrus: use fastvm for builds Builds now take over 10 mins, given golang compilation is parallelized by default we can give more cores to speed it up. Signed-off-by: Paul Holzinger --- .cirrus.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 1022e58cfe..f8159b4045 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -129,10 +129,10 @@ validate-source_task: build_task: alias: 'build' name: 'Build for $DISTRO_NV' # N/B: Referenced by URLencoded strings elsewhere - gce_instance: &standardvm + gce_instance: &fastvm image_project: libpod-218412 zone: "us-central1-a" - cpu: 2 + cpu: 4 memory: "4Gb" # Required to be 200gig, do not modify - has i/o performance impact # according to gcloud CLI tool warning messages. @@ -232,7 +232,7 @@ alt_build_task: env: <<: *stdenvars TEST_FLAVOR: "altbuild" - gce_instance: *standardvm + gce_instance: *fastvm matrix: - env: ALT_NAME: 'Build Each Commit' @@ -380,7 +380,9 @@ bindings_task: (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**')) depends_on: &build - build_success - gce_instance: *standardvm + gce_instance: &standardvm + <<: *fastvm + cpu: 2 env: <<: *stdenvars TEST_FLAVOR: bindings @@ -626,9 +628,7 @@ local_integration_test_task: &local_integration_test_task matrix: *platform_axis # integration tests scale well with cpu as they are parallelized # so we give these tests 4 cores to make them faster - gce_instance: &fastvm - <<: *standardvm - cpu: 4 + gce_instance: *fastvm env: TEST_FLAVOR: int clone_script: *get_gosrc From bc3db7c95b4dff59646f835ad69f413bda6b1380 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 7 Oct 2024 14:23:36 +0200 Subject: [PATCH 2/8] Makefile: allow man-page-check to be run in parallel This target runs several scripts in serial but they do not have any dependencies so we can split them all into their own target so that make -j can run the targets in parallel to speed this up. Signed-off-by: Paul Holzinger --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e46c591945..dc66703fc9 100644 --- a/Makefile +++ b/Makefile @@ -590,10 +590,18 @@ podman-remote-%-docs: podman-remote $(if $(findstring windows,$*),docs/source/markdown,docs/build/man) .PHONY: man-page-check -man-page-check: bin/podman docs +man-page-check: man-page-checker xref-helpmsgs-manpages xref-quadlet-docs xref-quadlet-docs + +man-page-checker: bin/podman docs hack/man-page-checker + +xref-helpmsgs-manpages: bin/podman docs hack/xref-helpmsgs-manpages + +man-page-table-check: docs hack/man-page-table-check + +xref-quadlet-docs: docs hack/xref-quadlet-docs .PHONY: swagger-check From 9082a14d767c5f0c84bf35cb47b98610a777079b Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 7 Oct 2024 14:28:22 +0200 Subject: [PATCH 3/8] cirrus: run make with parallel jobs where useful The doc generation and the validate-binaries target can be run in parallel as they do not depend on each other and a specific ordering. As such we pass -j $(nproc) but also --output-sync=target to ensure the output is not intermixed between several targets which could be harder to read in case of errors. Hower dus the complex podman-release target we can run podman-release and validate-binaries at the same time as the dependencies are not right and we run podman-release first in order to get the correct binaires build. Signed-off-by: Paul Holzinger --- contrib/cirrus/runner.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index 8d4dddd69d..9ffa3b5fed 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -210,19 +210,17 @@ eof } function _run_build() { - local vb_target + # Ensure always start from clean-slate with all vendor modules downloaded + showrun make clean + showrun make vendor + showrun make -j $(nproc) --output-sync=target podman-release # includes podman, podman-remote, and docs # There's no reason to validate-binaries across multiple linux platforms # shellcheck disable=SC2154 if [[ "$DISTRO_NV" =~ $FEDORA_NAME ]]; then - vb_target=validate-binaries + showrun make -j $(nproc) --output-sync=target validate-binaries fi - # Ensure always start from clean-slate with all vendor modules downloaded - showrun make clean - showrun make vendor - showrun make podman-release $vb_target # includes podman, podman-remote, and docs - # Last-minute confirmation that we're testing the desired runtime. # This Can't Possibly Fail™ in regular CI; only when updating VMs. # $CI_DESIRED_RUNTIME must be defined in .cirrus.yml. From 04dffbc2c725493d89d7e95cd3d0e7416b45ba45 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 7 Oct 2024 14:50:12 +0200 Subject: [PATCH 4/8] cirrus: change alt arch task to only compile binaries The current podman-release-%.tar.gz target does a lot more then just checking if we can build for the given arch, in particular it first builds a local podman-remote for the remote-docs.sh script. This makes things slow as we compile several things and then builda and package the docs. Given the docs are not arch specific there is realy no point in doing all that work. All we care about is if the bianries can build on other arches to catch compile issue for otherwise untested arches. This should make the CI Alt Arch. tasks much faster. Signed-off-by: Paul Holzinger --- Makefile | 8 ++++++++ contrib/cirrus/runner.sh | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index dc66703fc9..b0e604f407 100644 --- a/Makefile +++ b/Makefile @@ -500,6 +500,14 @@ local-cross: $(CROSS_BUILD_TARGETS) ## Cross compile podman binary for multiple .PHONY: cross cross: local-cross +# Simple target to check that we can build all binaries for another arch, +# the resulting binaries are not meant to be usable this is just for +# testing if it builds, it depends on the caller to set GOOS/GOARCH. +.PHONY: cross-binaries +cross-binaries: + $(MAKE) CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) \ + BUILDTAGS="$(BUILDTAGS_CROSS)" clean-binaries binaries + .PHONY: completions completions: podman podman-remote # key = shell, value = completion filename diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index 9ffa3b5fed..d69067b025 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -310,7 +310,7 @@ function _run_altbuild() { function _build_altbuild_archs() { for arch in "$@"; do msg "Building release archive for $arch" - showrun make podman-release-${arch}.tar.gz GOARCH=$arch + showrun make cross-binaries GOARCH=$arch done } From 7eb4a6fe2b2e3b22bdd8d19cb15512b692353fbd Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 7 Oct 2024 17:54:08 +0200 Subject: [PATCH 5/8] cirrus: speed up postbuild In particular the main build task already did a make vendor and a regeneration of the completion scripts. This means the first tre_status would pick up both changes so the suggestion would be off. And rerunning the same thing again here just makes thing slower than they need to be. In particular there was the bug that make completion even rebuild podman because generate-bindings obviously updates the timestamps of the files as they are overwritten. We do however must run generate-bindings as it was not run before. Signed-off-by: Paul Holzinger --- contrib/cirrus/postbuild.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/cirrus/postbuild.sh b/contrib/cirrus/postbuild.sh index ef6e89797f..18150e9dac 100755 --- a/contrib/cirrus/postbuild.sh +++ b/contrib/cirrus/postbuild.sh @@ -30,13 +30,13 @@ source $AUTOMATION_LIB_PATH/common_lib.sh # shellcheck disable=SC2154 cd $CIRRUS_WORKING_DIR +# Note, make completions and make vendor will already be run in _run_build() +# so do not run them again for no reason. This just makes CI slower. +SUGGESTION="run 'make vendor' or 'make completions' and commit all changes" ./hack/tree_status.sh + showrun make .install.goimports -showrun make vendor -SUGGESTION="run 'make vendor' and commit all changes" ./hack/tree_status.sh showrun make generate-bindings SUGGESTION="run 'make generate-bindings' and commit all changes" ./hack/tree_status.sh -showrun make completions -SUGGESTION="run 'make completions' and commit all changes" ./hack/tree_status.sh # Defined in Cirrus-CI config. # shellcheck disable=SC2154 From 6e69978d2e119b79047dd29c4e535f50e615baf4 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 7 Oct 2024 18:13:37 +0200 Subject: [PATCH 6/8] cirrus: use shared repo_prep/repo_artifacts scripts The script for aarch is exactly the same so there doesn't seem to be a reason to duplciate it. Signed-off-by: Paul Holzinger --- .cirrus.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index f8159b4045..07279b6dae 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -210,11 +210,8 @@ build_aarch64_task: main_script: *main # Cirrus-CI is very slow uploading one file at time, and the repo contains # thousands of files. Speed this up by archiving into tarball first. - repo_prep_script: &repo_prep_aarch64 >- - tar cjf /tmp/repo.tbz -C $GOSRC . && mv /tmp/repo.tbz $GOSRC/ - repo_artifacts: &repo_artifacts_aarch64 - path: ./repo.tbz - type: application/octet-stream + repo_prep_script: *repo_prep + repo_artifacts: *repo_artifacts always: *runner_stats From 9e35fead036b1591f4446b29e2d0b3f62b267f75 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 7 Oct 2024 19:27:06 +0200 Subject: [PATCH 7/8] cirrus: use zstd over bzip2 for repo archive The repo tar process took over 1:20 min, with zstd it takes less than 10s so we safe over a minute by doing this. Signed-off-by: Paul Holzinger --- .cirrus.yml | 60 ++++++++++++---------- contrib/cirrus/win-installer-main.ps1 | 2 +- contrib/cirrus/win-podman-machine-test.ps1 | 2 +- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 07279b6dae..5e9ec3b486 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -178,9 +178,9 @@ build_task: # Cirrus-CI is very slow uploading one file at time, and the repo contains # thousands of files. Speed this up by archiving into tarball first. repo_prep_script: &repo_prep >- - tar cjf /tmp/repo.tbz -C $GOSRC . && mv /tmp/repo.tbz $GOSRC/ + tar --zstd -cf /tmp/repo.tar.zst -C $GOSRC . && mv /tmp/repo.tar.zst $GOSRC/ repo_artifacts: &repo_artifacts - path: ./repo.tbz + path: ./repo.tar.zst type: application/octet-stream always: &runner_stats runner_stats_artifacts: @@ -247,11 +247,11 @@ alt_build_task: ALT_NAME: 'Alt Arch. MIPS64 Cross' - env: ALT_NAME: 'Alt Arch. Other Cross' - # This task cannot make use of the shared repo.tbz artifact. + # This task cannot make use of the shared repo.tar.zst artifact. clone_script: *full_clone setup_script: *setup main_script: *main - # Produce a new repo.tbz artifact for consumption by 'artifacts' task. + # Produce a new repo.tar.zst artifact for consumption by 'artifacts' task. repo_prep_script: *repo_prep repo_artifacts: *repo_artifacts always: *runner_stats @@ -298,7 +298,7 @@ osx_alt_build_task: # The Mac tests rely this Podman binary to run, and the CI Mac is ARM-based build_arm64_script: - make podman-remote-release-darwin_arm64.zip - # Produce a new repo.tbz artifact for consumption by dependent tasks. + # Produce a new repo.tar.zst artifact for consumption by dependent tasks. repo_prep_script: *repo_prep repo_artifacts: *repo_artifacts # This host is/was shared with potentially many other CI tasks. @@ -326,11 +326,11 @@ freebsd_alt_build_task: freebsd_instance: image_family: freebsd-13-3 setup_script: - - pkg install -y gpgme bash go-md2man gmake gsed gnugrep go pkgconf + - pkg install -y gpgme bash go-md2man gmake gsed gnugrep go pkgconf zstd build_amd64_script: - gmake podman-release - # This task cannot make use of the shared repo.tbz artifact and must - # produce a new repo.tbz artifact for consumption by 'artifacts' task. + # This task cannot make use of the shared repo.tar.zst artifact and must + # produce a new repo.tar.zst artifact for consumption by 'artifacts' task. repo_prep_script: *repo_prep repo_artifacts: *repo_artifacts @@ -386,9 +386,9 @@ bindings_task: # N/B: This script depends on ${DISTRO_NV} being defined for the task. clone_script: &get_gosrc | cd /tmp - echo "$ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tbz" - time $ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tbz - time tar xjf /tmp/repo.tbz -C $GOSRC + echo "$ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tar.zst" + time $ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tar.zst + time tar --zstd -xf /tmp/repo.tar.zst -C $GOSRC setup_script: *setup main_script: *main always: &logs_artifacts @@ -465,11 +465,11 @@ win_installer_task: $ProgressPreference = 'SilentlyContinue' New-Item -ItemType Directory -Force -Path "$ENV:CIRRUS_WORKING_DIR" Set-Location "$ENV:CIRRUS_WORKING_DIR" - $uri = "${ENV:ART_URL}/Windows Cross/repo/repo.tbz" + $uri = "${ENV:ART_URL}/Windows Cross/repo/repo.tar.zst" Write-Host "Downloading $uri" For($i = 0;;) { Try { - Invoke-WebRequest -UseBasicParsing -ErrorAction Stop -OutFile "repo.tbz2" ` + Invoke-WebRequest -UseBasicParsing -ErrorAction Stop -OutFile "repo.tar.zst" ` -Uri "$uri" Break } Catch { @@ -480,11 +480,19 @@ win_installer_task: Start-Sleep -Seconds 10 } } - arc unarchive repo.tbz2 .\ + Write-Host "zstd -d repo.tar.zst" + zstd -d repo.tar.zst if ($LASTEXITCODE -ne 0) { - throw "Unarchive repo.tbz2 failed" + throw "Extract repo.tar.zst failed" Exit 1 } + Write-Host "arc unarchive repo.tar .\" + arc unarchive repo.tar .\repo + if ($LASTEXITCODE -ne 0) { + throw "Unarchive repo.tar failed" + Exit 1 + } + Get-ChildItem -Path . Get-ChildItem -Path .\repo main_script: ".\\repo\\contrib\\cirrus\\win-installer-main.ps1" @@ -731,9 +739,9 @@ podman_machine_aarch64_task: VM_IMAGE_NAME: "${FEDORA_AARCH64_AMI}" clone_script: &get_gosrc_aarch64 | cd /tmp - echo "$ARTCURL/build_aarch64/repo/repo.tbz" - time $ARTCURL/build_aarch64/repo/repo.tbz - time tar xjf /tmp/repo.tbz -C $GOSRC + echo "$ARTCURL/build_aarch64/repo/repo.tar.zst" + time $ARTCURL/build_aarch64/repo/repo.tar.zst + time tar --zstd -xf /tmp/repo.tar.zst -C $GOSRC setup_script: *setup main_script: *main always: *int_logs_artifacts @@ -795,8 +803,8 @@ podman_machine_mac_task: clone_script: # artifacts from osx_alt_build_task - mkdir -p $CIRRUS_WORKING_DIR - cd $CIRRUS_WORKING_DIR - - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tbz - - tar xjf repo.tbz + - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tar.zst + - tar --zstd -xf repo.tar.zst # This host is/was shared with potentially many other CI tasks. # The previous task may have been canceled or aborted. prep_script: *mac_cleanup @@ -1105,20 +1113,20 @@ artifacts_task: fedora_binaries_script: - mkdir -p /tmp/fed - cd /tmp/fed - - $ARTCURL/Build%20for%20${FEDORA_NAME}/repo/repo.tbz - - tar xjf repo.tbz + - $ARTCURL/Build%20for%20${FEDORA_NAME}/repo/repo.tar.zst + - tar --zstd -xf repo.tar.zst - cp ./bin/* $CIRRUS_WORKING_DIR/ win_binaries_script: - mkdir -p /tmp/win - cd /tmp/win - - $ARTCURL/Windows%20Cross/repo/repo.tbz - - tar xjf repo.tbz + - $ARTCURL/Windows%20Cross/repo/repo.tar.zst + - tar --zstd -xf repo.tar.zst - mv ./podman-remote*.zip $CIRRUS_WORKING_DIR/ osx_binaries_script: - mkdir -p /tmp/osx - cd /tmp/osx - - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tbz - - tar xjf repo.tbz + - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tar.zst + - tar --zstd -xf repo.tar.zst - mv ./podman-remote-release-darwin_*.zip $CIRRUS_WORKING_DIR/ - mv ./contrib/pkginstaller/out/podman-installer-macos-*.pkg $CIRRUS_WORKING_DIR/ always: diff --git a/contrib/cirrus/win-installer-main.ps1 b/contrib/cirrus/win-installer-main.ps1 index 619d722429..033cf0df5e 100644 --- a/contrib/cirrus/win-installer-main.ps1 +++ b/contrib/cirrus/win-installer-main.ps1 @@ -15,7 +15,7 @@ if ($Env:CI -eq "true") { Push-Location $WIN_INST_FOLDER # Build Installer -# Note: consumes podman-remote-release-windows_amd64.zip from repo.tbz2 +# Note: consumes podman-remote-release-windows_amd64.zip from repo.tar.zst Run-Command ".\build.ps1 $Env:WIN_INST_VER dev `"$RELEASE_DIR`"" Pop-Location diff --git a/contrib/cirrus/win-podman-machine-test.ps1 b/contrib/cirrus/win-podman-machine-test.ps1 index 4d8ff59667..965163b6a5 100644 --- a/contrib/cirrus/win-podman-machine-test.ps1 +++ b/contrib/cirrus/win-podman-machine-test.ps1 @@ -24,7 +24,7 @@ if ($Env:TEST_FLAVOR -eq "machine-wsl") { Write-Host " CONTAINERS_MACHINE_PROVIDER = $Env:CONTAINERS_MACHINE_PROVIDER" Write-Host "`n" -# The repo.tbz artifact was extracted here +# The repo.tar.zst artifact was extracted here Set-Location "$ENV:CIRRUS_WORKING_DIR\repo" # Tests hard-code this location for podman-remote binary, make sure it actually runs. Run-Command ".\bin\windows\podman.exe --version" From 5d10e6e16b9465f046341b9c2a9f84d4fbb9ad04 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 14 Oct 2024 17:08:34 +0200 Subject: [PATCH 8/8] cirrus: let tar extract figure out the compression tar should be smart enough to check the magic byte and use the correct decompression algo based on that so there is no need to spell it out explictly. Signed-off-by: Paul Holzinger --- .cirrus.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 5e9ec3b486..401c8817a0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -388,7 +388,7 @@ bindings_task: cd /tmp echo "$ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tar.zst" time $ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tar.zst - time tar --zstd -xf /tmp/repo.tar.zst -C $GOSRC + time tar -xf /tmp/repo.tar.zst -C $GOSRC setup_script: *setup main_script: *main always: &logs_artifacts @@ -741,7 +741,7 @@ podman_machine_aarch64_task: cd /tmp echo "$ARTCURL/build_aarch64/repo/repo.tar.zst" time $ARTCURL/build_aarch64/repo/repo.tar.zst - time tar --zstd -xf /tmp/repo.tar.zst -C $GOSRC + time tar -xf /tmp/repo.tar.zst -C $GOSRC setup_script: *setup main_script: *main always: *int_logs_artifacts @@ -804,7 +804,7 @@ podman_machine_mac_task: - mkdir -p $CIRRUS_WORKING_DIR - cd $CIRRUS_WORKING_DIR - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tar.zst - - tar --zstd -xf repo.tar.zst + - tar -xf repo.tar.zst # This host is/was shared with potentially many other CI tasks. # The previous task may have been canceled or aborted. prep_script: *mac_cleanup @@ -1114,19 +1114,19 @@ artifacts_task: - mkdir -p /tmp/fed - cd /tmp/fed - $ARTCURL/Build%20for%20${FEDORA_NAME}/repo/repo.tar.zst - - tar --zstd -xf repo.tar.zst + - tar -xf repo.tar.zst - cp ./bin/* $CIRRUS_WORKING_DIR/ win_binaries_script: - mkdir -p /tmp/win - cd /tmp/win - $ARTCURL/Windows%20Cross/repo/repo.tar.zst - - tar --zstd -xf repo.tar.zst + - tar -xf repo.tar.zst - mv ./podman-remote*.zip $CIRRUS_WORKING_DIR/ osx_binaries_script: - mkdir -p /tmp/osx - cd /tmp/osx - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tar.zst - - tar --zstd -xf repo.tar.zst + - tar -xf repo.tar.zst - mv ./podman-remote-release-darwin_*.zip $CIRRUS_WORKING_DIR/ - mv ./contrib/pkginstaller/out/podman-installer-macos-*.pkg $CIRRUS_WORKING_DIR/ always: