Compare commits

..

No commits in common. "main" and "20240821t171500z-f40f39d13" have entirely different histories.

24 changed files with 207 additions and 185 deletions

View File

@ -233,6 +233,9 @@ cache_images_task:
- <<: *cache_image
env:
PACKER_BUILDS: "fedora-netavark"
- <<: *cache_image
env:
PACKER_BUILDS: "fedora-podman-py"
- <<: *cache_image
env:
PACKER_BUILDS: "fedora-aws"
@ -337,6 +340,7 @@ test_imgts_task: &imgts
fedora-c${IMG_SFX}
prior-fedora-c${IMG_SFX}
fedora-netavark-c${IMG_SFX}
fedora-podman-py-c${IMG_SFX}
rawhide-c${IMG_SFX}
debian-c${IMG_SFX}
build-push-c${IMG_SFX}

View File

@ -14,9 +14,4 @@ jobs:
# Ref: https://docs.github.com/en/actions/using-workflows/reusing-workflows
call_cron_failures:
uses: containers/podman/.github/workflows/check_cirrus_cron.yml@main
secrets:
SECRET_CIRRUS_API_KEY: ${{secrets.SECRET_CIRRUS_API_KEY}}
ACTION_MAIL_SERVER: ${{secrets.ACTION_MAIL_SERVER}}
ACTION_MAIL_USERNAME: ${{secrets.ACTION_MAIL_USERNAME}}
ACTION_MAIL_PASSWORD: ${{secrets.ACTION_MAIL_PASSWORD}}
ACTION_MAIL_SENDER: ${{secrets.ACTION_MAIL_SENDER}}
secrets: inherit

View File

@ -132,10 +132,12 @@ jobs:
- if: steps.manifests.outputs.count > 0
name: Post PR comment with image name/id table
uses: thollander/actions-comment-pull-request@v3
uses: jungwinter/comment@v1.1.0
with:
pr-number: '${{ steps.retro.outputs.prn }}'
message: |
issue_number: '${{ steps.retro.outputs.prn }}'
type: 'create'
token: '${{ secrets.GITHUB_TOKEN }}'
body: |
${{ env.IMAGE_TABLE }}
# Ref: https://github.com/marketplace/actions/deploy-to-gist

View File

@ -1 +1 @@
20250812t173301z-f42f41d13
20240821t171500z-f40f39d13

View File

@ -22,11 +22,11 @@ export CENTOS_STREAM_RELEASE = 9
# Warning: Beta Fedora releases are not supported. Verifiy EC2 AMI availability
# here: https://fedoraproject.org/cloud/download
export FEDORA_RELEASE = 42
export PRIOR_FEDORA_RELEASE = 41
export FEDORA_RELEASE = 40
export PRIOR_FEDORA_RELEASE = 39
# This should always be one-greater than $FEDORA_RELEASE (assuming it's actually the latest)
export RAWHIDE_RELEASE = 43
export RAWHIDE_RELEASE = 41
# Automation assumes the actual release number (after SID upgrade)
# is always one-greater than the latest DEBIAN_BASE_FAMILY (GCE image).
@ -132,17 +132,17 @@ help: ## Default target, parses special in-line comments as documentation.
# names and a max-length of 63.
.PHONY: IMG_SFX
IMG_SFX: timebomb-check ## Generate a new date-based image suffix, store in the file IMG_SFX
@echo "$$(date -u +%Y%m%dt%H%M%Sz)-f$(FEDORA_RELEASE)f$(PRIOR_FEDORA_RELEASE)d$(subst .,,$(DEBIAN_RELEASE))" > "$@"
@cat IMG_SFX
$(file >$@,$(shell date --utc +%Y%m%dt%H%M%Sz)-f$(FEDORA_RELEASE)f$(PRIOR_FEDORA_RELEASE)d$(subst .,,$(DEBIAN_RELEASE)))
@echo "$(file <IMG_SFX)"
# Prevent us from wasting CI time when we have expired timebombs
.PHONY: timebomb-check
timebomb-check:
@now=$$(date -u +%Y%m%d); \
@now=$$(date --utc +%Y%m%d); \
found=; \
while read -r bomb; do \
when=$$(echo "$$bomb" | sed -E -e 's/^.*timebomb ([0-9]+).*/\1/'); \
if [ "$$when" -le "$$now" ]; then \
when=$$(echo "$$bomb" | sed -e 's/^.*timebomb \([0-9]\+\).*/\1/'); \
if [ $$when -le $$now ]; then \
echo "$$bomb"; \
found=found; \
fi; \

View File

@ -1,108 +0,0 @@
The README here is waaaaaay too complicated for Ed. So here is a
simplified version of the typical things you need to do.
Super Duper Simplest Case
=========================
This is by far the most common case, and the simplest to understand.
You do this when you want to build VMs with newer package versions than
whatever VMs are currently set up in CI. You really need to
understand this before you get into anything more complicated.
```
$ git checkout -b lets-see-what-happens
$ make IMG_SFX
$ git commit -asm"Let's just see what happens"
```
...and push that as a PR.
If you're lucky, in about an hour you will get an email from `github-actions[bot]`
with a nice table of base and cache images, with links. I strongly encourage you
to try to get Ed's
[cirrus-vm-get-versions](https://github.com/edsantiago/containertools/tree/main/cirrus-vm-get-versions)
script working, because this will give you a very quick easy reliable
list of what packages have changed. You don't need this, but life will be painful
for you without it.
(If you're not lucky, the build will break. There are infinite ways for
this to happen, so you're on your own here. Ask for help! This is a great
team, and one or more people may quickly realize the problem.)
Once you have new VMs built, **test in an actual project**! Usually podman
and buildah, but you may want the varks too:
```
$ cd ~/src/github/containers/podman ! or wherever
$ git checkout -b test-new-vms
$ vim .cirrus.yml
[ search for "c202", and replace with your new IMG_SFX.]
[ Don't forget the leading "c"! ]
$ git commit -as
[ Please include a link to the automation_images PR! ]
```
Push this PR and see what happens. If you're very lucky, it will
pass on this and other repos. Get your podman/buildah/vark PRs
reviewed and merged, and then review-merge the automation_images one.
Pushing (har har) Your Luck
---------------------------
Feel lucky? Tag this VM build, so `dependabot` will create PRs
on all the myriad container repos:
```
$ git tag $(<IMG_SFX)
$ git push --no-verify upstream $(<IMG_SFX)
```
Within a few hours you'll see a ton of PRs. It is very likely that
something will go wrong in one or two, and if so, it's impossible to
cover all possibilities. As above, ask for help.
More Complicated Cases
======================
These are the next two most common.
Bumping One Package
-------------------
Quite often we need an emergency bump of only one package that
is not yet stable. Here are examples of the two most typical
cases,
[crun](https://github.com/containers/automation_images/pull/386/files) and
[pasta](https://github.com/containers/automation_images/pull/383/files).
Note the `timebomb` directives. Please use these: the time you save
may be your own, one future day. And please use 2-6 week times.
A timebomb that expires in a year is going to be hard to understand
when it goes off.
Bumping Distros
---------------
Like Fedora 40 to 41. Edit `Makefile`. Change `FEDORA`, `PRIOR_FEDORA`,
and `RAWHIDE`, then proceed with Simple Case.
There is almost zero chance that this will work on the first try.
Sorry, that's just the way it is. See the
[F40 to F41 PR](https://github.com/containers/automation_images/pull/392/files)
for a not-atypical example.
STRONG RECOMMENDATION
=====================
Read [check-imgsfx.sh](check-imgsfx.sh) and follow its instructions. Ed
likes to copy that to `.git/hooks/pre-push`, Chris likes using some
external tool that Ed doesn't trust. Use your judgment.
The reason for this is that you are going to forget to `make IMG_SFX`
one day, and then you're going to `git push --force` an update and walk
away, and come back to a failed run because `IMG_SFX` must always
always always be brand new.
Weak Recommendation
-------------------
Ed likes to fiddle with `IMG_SFX`, zeroing out to the nearest
quarter hour. Absolutely unnecessary, but easier on the eyes
when trying to see which VMs are in use or when comparing
diffs.

View File

@ -118,7 +118,7 @@ builders:
architecture: 'x86_64'
image-type: 'machine'
is-public: 'true'
name: 'Fedora-Cloud-Base*-{{user `FEDORA_RELEASE`}}-*'
name: 'Fedora-Cloud-Base*-{{user `FEDORA_RELEASE`}}-*us-east-1*'
root-device-type: 'ebs'
state: 'available'
virtualization-type: 'hvm'

View File

@ -52,6 +52,19 @@ install_automation_tooling
# Ensure automation library is loaded
source "$REPO_DIRPATH/lib.sh"
# 2024-01-02 found debian 13 tar 1.35+dfsg-2
# which has the horrible duplicate-path bug:
# https://github.com/containers/podman/issues/19407
# https://bugzilla.redhat.com/show_bug.cgi?id=2230127
# 2024-01-25 dfsg-3 also has the bug
# 2024-05-01 trixy still has 1.35+dfsg-3
timebomb 20240901 "prevent us from getting broken tar-1.35+dfsg-3"
$SUDO tee /etc/apt/preferences.d/$(date +%Y%m%d)-tar <<EOF
Package: tar
Pin: version 1.35+dfsg-[23]
Pin-Priority: -1
EOF
# Workaround 12->13 forward-incompatible change in grub scripts.
# Without this, updating to the SID kernel may fail.
echo "Upgrading grub-common"

View File

@ -90,9 +90,7 @@ if ! ((CONTAINER)); then
# This is necessary to prevent permission-denied errors on service-start
# and also on the off-chance the package gets updated and context reset.
$SUDO semanage fcontext --add --type bin_t /usr/bin/cloud-init
# This used restorecon before so we don't have to specify the file_contexts.local
# manually, however with f42 that stopped working: https://bugzilla.redhat.com/show_bug.cgi?id=2360183
$SUDO setfiles -v /etc/selinux/targeted/contexts/files/file_contexts.local /usr/bin/cloud-init
$SUDO restorecon -v /usr/bin/cloud-init
else # GCP Image
echo "Setting GCP startup service (for Cirrus-CI agent) SELinux unconfined"
# ref: https://cloud.google.com/compute/docs/startupscript

View File

@ -75,6 +75,9 @@ builders:
source_image_family: 'fedora-base'
labels: *fedora_gce_labels
- <<: *aux_fed_img
name: 'fedora-podman-py'
- <<: *aux_fed_img
name: 'fedora-netavark'

View File

@ -44,7 +44,7 @@ INSTALL_PACKAGES=(\
fuse-overlayfs
gcc
gettext
git
git-daemon-run
gnupg2
go-md2man
golang
@ -103,8 +103,6 @@ INSTALL_PACKAGES=(\
skopeo
slirp4netns
socat
libsqlite3-0
libsqlite3-dev
systemd-container
sudo
time

View File

@ -47,11 +47,6 @@ req_env_vars PACKER_BUILD_NAME
bash $SCRIPT_DIRPATH/debian_packaging.sh
# dnsmasq is set to bind 0.0.0.0:53, that will conflict with our dns tests.
# We don't need a local resolver.
$SUDO systemctl disable dnsmasq.service
$SUDO systemctl mask dnsmasq.service
if ! ((CONTAINER)); then
warn "Making Debian kernel enable cgroup swap accounting"
SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/'

View File

@ -0,0 +1,98 @@
#!/bin/bash
# This script is called from fedora_setup.sh and various Dockerfiles.
# It's not intended to be used outside of those contexts. It assumes the lib.sh
# library has already been sourced, and that all "ground-up" package-related activity
# needs to be done, including repository setup and initial update.
set -e
SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}")
SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH")
REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../")
# shellcheck source=./lib.sh
source "$REPO_DIRPATH/lib.sh"
# shellcheck disable=SC2154
warn "Enabling updates-testing repository for $PACKER_BUILD_NAME"
lilto ooe.sh $SUDO dnf install -y 'dnf-command(config-manager)'
lilto ooe.sh $SUDO dnf config-manager --set-enabled updates-testing
msg "Updating/Installing repos and packages for $OS_REL_VER"
bigto ooe.sh $SUDO dnf update -y
INSTALL_PACKAGES=(\
bash-completion
bridge-utils
buildah
bzip2
curl
findutils
fuse3
gcc
git
git-daemon
glib2-devel
glibc-devel
hostname
httpd-tools
iproute
iptables
jq
libtool
lsof
make
nmap-ncat
openssl
openssl-devel
pkgconfig
podman
policycoreutils
protobuf
protobuf-devel
python-pip-wheel
python-setuptools-wheel
python-toml
python-wheel-wheel
python3-PyYAML
python3-coverage
python3-dateutil
python3-docker
python3-fixtures
python3-libselinux
python3-libsemanage
python3-libvirt
python3-pip
python3-psutil
python3-pylint
python3-pytest
python3-pyxdg
python3-requests
python3-requests-mock
python3-virtualenv
python3.6
python3.8
python3.9
redhat-rpm-config
rsync
sed
skopeo
socat
tar
time
tox
unzip
vim
wget
xz
zip
zstd
)
echo "Installing general build/test dependencies"
bigto $SUDO dnf install -y "${INSTALL_PACKAGES[@]}"
# It was observed in F33, dnf install doesn't always get you the latest/greatest
lilto $SUDO dnf update -y

View File

@ -28,7 +28,7 @@ req_env_vars PACKER_BUILD_NAME
if [[ "$PACKER_BUILD_NAME" == "fedora" ]] && [[ ! "$PACKER_BUILD_NAME" =~ "prior" ]]; then
warn "Enabling updates-testing repository for $PACKER_BUILD_NAME"
lilto ooe.sh $SUDO dnf install -y 'dnf-command(config-manager)'
lilto ooe.sh $SUDO dnf config-manager setopt updates-testing.enabled=1
lilto ooe.sh $SUDO dnf config-manager --set-enabled updates-testing
else
warn "NOT enabling updates-testing repository for $PACKER_BUILD_NAME"
fi
@ -56,7 +56,6 @@ INSTALL_PACKAGES=(\
curl
device-mapper-devel
dnsmasq
docker-distribution
e2fsprogs-devel
emacs-nox
fakeroot
@ -65,7 +64,6 @@ INSTALL_PACKAGES=(\
fuse3
fuse3-devel
gcc
gh
git
git-daemon
glib2-devel
@ -83,7 +81,6 @@ INSTALL_PACKAGES=(\
iproute
iptables
jq
koji
krb5-workstation
libassuan
libassuan-devel
@ -115,29 +112,22 @@ INSTALL_PACKAGES=(\
passt
perl-Clone
perl-FindBin
pigz
pkgconfig
podman
podman-remote
pre-commit
procps-ng
protobuf
protobuf-c
protobuf-c-devel
protobuf-devel
python3-fedora-distro-aliases
python3-koji-cli-plugins
redhat-rpm-config
rpcbind
rsync
runc
sed
ShellCheck
skopeo
slirp4netns
socat
sqlite-libs
sqlite-devel
squashfs-tools
tar
time
@ -154,10 +144,12 @@ INSTALL_PACKAGES=(\
# Rawhide images don't need these packages
if [[ "$PACKER_BUILD_NAME" =~ fedora ]]; then
INSTALL_PACKAGES+=( \
docker-compose
python-pip-wheel
python-setuptools-wheel
python-toml
python-wheel-wheel
python2
python3-PyYAML
python3-coverage
python3-dateutil
@ -174,11 +166,6 @@ if [[ "$PACKER_BUILD_NAME" =~ fedora ]]; then
python3-requests
python3-requests-mock
)
else # podman-sequoia is only available in Rawhide
timebomb 20251101 "Also install the package in future Fedora releases, and enable Sequoia support in users of the images."
INSTALL_PACKAGES+=( \
podman-sequoia
)
fi
# When installing during a container-build, having this present
@ -189,23 +176,11 @@ fi
if ! ((CONTAINER)); then
INSTALL_PACKAGES+=( \
bpftrace
composefs
container-selinux
fuse-overlayfs
libguestfs-tools
selinux-policy-devel
policycoreutils
)
# Extra packages needed by podman-machine-os
INSTALL_PACKAGES+=( \
podman-machine
osbuild
osbuild-tools
osbuild-ostree
xfsprogs
e2fsprogs
)
fi
@ -223,6 +198,18 @@ DOWNLOAD_PACKAGES=(\
msg "Installing general build/test dependencies"
bigto $SUDO dnf install -y "${INSTALL_PACKAGES[@]}"
# 2024-08-21 fixes CI system test problems
timebomb 20240901 "pasta not yet in stable for all arches"
if [[ "$OS_RELEASE_VER" -eq 39 ]]; then
arch=$(uname -m)
n=passt
v=0%5E20240814.g61c0b0d
r=1.fc$OS_RELEASE_VER
bigto $SUDO dnf install -y \
https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$arch/$n-$v-$r.$arch.rpm \
https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/noarch/$n-selinux-$v-$r.noarch.rpm
fi
msg "Downloading packages for optional installation at runtime, as needed."
$SUDO mkdir -p "$PACKAGE_DOWNLOAD_DIR"
cd "$PACKAGE_DOWNLOAD_DIR"
@ -238,3 +225,11 @@ cd -
# Occasionally following an install, there are more updates available.
# This may be due to activation of suggested/recommended dependency resolution.
lilto $SUDO dnf update -y
# Gah. FIXME 2024-06-20: rawhide now includes rpm-plugin-ima,
# which causes rootless podman pods to fail.
# https://github.com/containers/podman/issues/18543
if ! ((CONTAINER)); then
timebomb 20240901 "Temporary workaround for signed rpms (ima) in rawhide"
$SUDO setfattr -x security.ima /usr/libexec/catatonit/catatonit || true
fi

View File

@ -30,6 +30,8 @@ req_env_vars PACKER_BUILD_NAME
# shellcheck disable=SC2154
if [[ "$PACKER_BUILD_NAME" =~ "netavark" ]]; then
bash $SCRIPT_DIRPATH/fedora-netavark_packaging.sh
elif [[ "$PACKER_BUILD_NAME" =~ "podman-py" ]]; then
bash $SCRIPT_DIRPATH/fedora-podman-py_packaging.sh
elif [[ "$PACKER_BUILD_NAME" =~ "build-push" ]]; then
bash $SCRIPT_DIRPATH/build-push_packaging.sh
# Registers qemu emulation for non-native execution

View File

@ -76,16 +76,12 @@ declare -a IMAGELIST=(
registry:2.8.2
registry:volume_omitted
systemd-image:20240124
testartifact:20250206-single
testartifact:20250206-multi
testartifact:20250206-multi-no-title
testartifact:20250206-evil
testdigest_v2s2
testdigest_v2s2:20200210
testimage:00000000
testimage:00000004
testimage:20221018
testimage:20241011
testimage:20240123
testimage:multiimage
testimage@sha256:1385ce282f3a959d0d6baf45636efe686c1e14c3e7240eb31907436f7bc531fa
testdigest_v2s2:20200210

View File

@ -16,9 +16,18 @@ source "$REPO_DIRPATH/lib.sh"
# for both VM and container image build workflows.
req_env_vars PACKER_BUILD_NAME
warn "Upgrading Fedora '$OS_RELEASE_VER' to rawhide, this might break."
# shellcheck disable=SC2154
warn "If so, this script may be found in the repo. as '$SCRIPT_DIRPATH/$SCRIPT_FILENAME'."
# Going from F38 -> rawhide requires some special handling WRT DNF upgrade to DNF5
if [[ "$OS_RELEASE_VER" -ge 38 ]]; then
warn "Upgrading dnf -> dnf5"
showrun $SUDO dnf update -y dnf
showrun $SUDO dnf install -y dnf5
# Even dnf5 refuses to remove the 'dnf' package.
showrun $SUDO rpm -e yum dnf
else
warn "Upgrading Fedora '$OS_RELEASE_VER' to rawhide, this might break."
# shellcheck disable=SC2154
warn "If so, this script may be found in the repo. as '$SCRIPT_DIRPATH/$SCRIPT_FILENAME'."
fi
# Show what's happening
set -x
@ -30,6 +39,10 @@ $SUDO sed -i -r -e 's/^gpgcheck=.+/gpgcheck=0/' /etc/yum.repos.d/*.repo
$SUDO dnf5 -y distro-sync --releasever=rawhide --allowerasing
$SUDO dnf5 upgrade -y
# As of May 2024 composefs is heating up
timebomb 20241231 "At some point, composefs should be available on all fedoras"
$SUDO dnf5 -y install composefs
# A shared fedora_packaging.sh script is called next that doesn't always support dnf5
$SUDO ln -s $(type -P dnf5) /usr/local/bin/dnf

View File

@ -0,0 +1,17 @@
{
"builds": [
{
"name": "fedora-podman-py",
"builder_type": "googlecompute",
"build_time": 1658176090,
"files": null,
"artifact_id": "fedora-podman-py-c5419329914142720",
"packer_run_uuid": "e5b1e6ab-37a5-a695-624d-47bf0060b272",
"custom_data": {
"IMG_SFX": "5419329914142720",
"STAGE": "cache"
}
}
],
"last_run_uuid": "e5b1e6ab-37a5-a695-624d-47bf0060b272"
}

View File

@ -17,7 +17,7 @@ ENV CIRRUS_WORKING_DIR=/var/tmp/automation_images \
# to rely on COPY or ADD instructions. See documentation for warning.
RUN test -n "$PACKER_VERSION"
RUN dnf update -y && \
dnf -y mark dependency $(rpm -qa | grep -Ev '(gpg-pubkey)|(dnf)|(sudo)') && \
dnf mark remove $(rpm -qa | grep -Ev '(gpg-pubkey)|(dnf)|(sudo)') && \
dnf install -y \
ShellCheck \
bash-completion \
@ -38,7 +38,7 @@ RUN dnf update -y && \
util-linux \
unzip \
&& \
dnf -y mark user dnf sudo $_ && \
dnf mark install dnf sudo $_ && \
dnf autoremove -y && \
dnf clean all

View File

@ -35,14 +35,6 @@ if [[ -n "$AWS_INI" ]]; then
set_aws_filepath
fi
id
# FIXME: ssh-keygen seems to fail to create keys with Permission denied
# in the base_images make target, I have no idea why but all CI jobs are
# broken because of this. Let's try without selinux.
if [[ "$(getenforce)" == "Enforcing" ]]; then
setenforce 0
fi
set -x
cd "$REPO_DIRPATH"
export IMG_SFX=$IMG_SFX

View File

@ -8,6 +8,7 @@ containers-build-source-image
libpod-218412
netavark-2021
oci-seccomp-bpf-hook
podman-py
skopeo
storage-240716
udica-247612

View File

@ -12,6 +12,7 @@ RUN dnf -y update && \
dnf clean all
ENV REG_REPO="https://github.com/docker/distribution.git" \
REG_COMMIT="b5ca020cfbe998e5af3457fda087444cf5116496" \
REG_COMMIT_SCHEMA1="ec87e9b6971d831f0eff752ddb54fb64693e51cd" \
OSO_REPO="https://github.com/openshift/origin.git" \
OSO_TAG="v1.5.0-alpha.3"

View File

@ -9,6 +9,7 @@ set -e
declare -a req_vars
req_vars=(\
REG_REPO
REG_COMMIT
REG_COMMIT_SCHEMA1
OSO_REPO
OSO_TAG
@ -42,6 +43,12 @@ cd "$REG_GOSRC"
(
# This is required to be set like this by the build system
export GOPATH="$PWD/Godeps/_workspace:$GOPATH"
# This comes in from the Containerfile
# shellcheck disable=SC2154
git checkout -q "$REG_COMMIT"
go build -o /usr/local/bin/registry-v2 \
github.com/docker/distribution/cmd/registry
# This comes in from the Containerfile
# shellcheck disable=SC2154
git checkout -q "$REG_COMMIT_SCHEMA1"

View File

@ -9,7 +9,7 @@ iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocola
# Install basic required tooling.
# psexec needed to workaround session 0 WSL bug
retryInstall 7zip git archiver psexec golang mingw StrawberryPerl zstandard; Check-Exit
retryInstall git archiver psexec golang mingw StrawberryPerl; Check-Exit
# Update service is required for dotnet
Set-Service -Name wuauserv -StartupType "Manual"; Check-Exit