Cirrus: Update VM images & modernize scripts
Recent VM images now include installation of many shared/common script functions from the containers/automation repository. In the interest of re-use and maintainability, migrate local (duplicate) function calls to make use of the shared versions. Signed-off-by: Chris Evich <cevich@redhat.com>
This commit is contained in:
Chris Evich
parent
a4cc7aa79e
commit
a62ac10e90
|
|
@ -25,7 +25,7 @@ env:
|
|||
# GCE project where images live
|
||||
IMAGE_PROJECT: "libpod-218412"
|
||||
# VM Image built in containers/automation_images
|
||||
_BUILT_IMAGE_SUFFIX: "c6110627968057344"
|
||||
_BUILT_IMAGE_SUFFIX: "c4948709391728640"
|
||||
FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
|
||||
PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
|
||||
UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"
|
||||
|
|
|
|||
|
|
@ -23,10 +23,10 @@ then
|
|||
fi
|
||||
|
||||
[[ -n "type -P parted" ]] || \
|
||||
die 2 "The parted command is required."
|
||||
die "The parted command is required."
|
||||
|
||||
[[ ! -b ${SLASH_DEVICE}2 ]] || \
|
||||
die 5 "Found unexpected block device ${SLASH_DEVICE}2"
|
||||
die "Found unexpected block device ${SLASH_DEVICE}2"
|
||||
|
||||
PPRINTCMD="parted --script ${SLASH_DEVICE} print"
|
||||
FINDMNTCMD="findmnt --source=${SLASH_DEVICE}1 --mountpoint=/ --canonicalize --evaluate --first-only --noheadings"
|
||||
|
|
@ -38,7 +38,7 @@ then
|
|||
echo "Repartitioning original partition table:"
|
||||
$PPRINTCMD
|
||||
else
|
||||
die 6 "Unexpected output from '$FINDMNTCMD': $(<$TMPF)"
|
||||
die "Unexpected output from '$FINDMNTCMD': $(<$TMPF)"
|
||||
fi
|
||||
|
||||
echo "Adding partition offset within unpartitioned space."
|
||||
|
|
@ -54,7 +54,7 @@ FSTYPE=$(findmnt --first-only --noheadings --output FSTYPE ${SLASH_DEVICE}1)
|
|||
echo "Expanding $FSTYPE filesystem on ${SLASH_DEVICE}1"
|
||||
case $FSTYPE in
|
||||
ext*) resize2fs ${SLASH_DEVICE}1 ;;
|
||||
*) die 11 "Script $(basename $0) doesn't know how to resize a $FSTYPE filesystem." ;;
|
||||
*) die "Script $(basename $0) doesn't know how to resize a $FSTYPE filesystem." ;;
|
||||
esac
|
||||
|
||||
# Must happen last - signals completion to other tooling
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ case $TEST_DRIVER in
|
|||
devicemapper)
|
||||
# Setup by devicemapper_setup in lib.sh
|
||||
DM_DEVICE=$(< $DM_REF_FILEPATH)
|
||||
echo "WARNING: Performing destructive testing against $DM_DEVICE"
|
||||
warn "Performing destructive testing against $DM_DEVICE"
|
||||
showrun make STORAGE_DRIVER=devicemapper STORAGE_OPTION=dm.directlvm_device=$DM_DEVICE local-test-integration
|
||||
;;
|
||||
vfs)
|
||||
|
|
@ -32,6 +32,6 @@ case $TEST_DRIVER in
|
|||
showrun make STORAGE_DRIVER=aufs local-test-integration
|
||||
;;
|
||||
*)
|
||||
die 11 "Unknown/Unsupported \$TEST_DRIVER=$TEST_DRIVER (see .cirrus.yml and $(basename $0))"
|
||||
die "Unknown/Unsupported \$TEST_DRIVER=$TEST_DRIVER (see .cirrus.yml and $(basename $0))"
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
|
|
@ -6,12 +6,35 @@
|
|||
# Global details persist here
|
||||
source /etc/environment # not always loaded under all circumstances
|
||||
|
||||
# Under some contexts these values are not set, make sure they are.
|
||||
# Due to differences across platforms and runtime execution environments,
|
||||
# handling of the (otherwise) default shell setup is non-uniform. Rather
|
||||
# than attempt to workaround differences, simply force-load/set required
|
||||
# items every time this library is utilized.
|
||||
source /etc/profile
|
||||
source /etc/environment
|
||||
USER="$(whoami)"
|
||||
export HOME="$(getent passwd $USER | cut -d : -f 6)"
|
||||
[[ -n "$UID" ]] || UID=$(getent passwd $USER | cut -d : -f 3)
|
||||
GID=$(getent passwd $USER | cut -d : -f 4)
|
||||
|
||||
# During VM Image build, the 'containers/automation' installation
|
||||
# was performed. The final step of that installation sets the
|
||||
# installation location in $AUTOMATION_LIB_PATH in /etc/environment
|
||||
# or in the default shell profile.
|
||||
if [[ -n "$AUTOMATION_LIB_PATH" ]]; then
|
||||
for libname in defaults anchors console_output utils; do
|
||||
# There's no way shellcheck can process this location
|
||||
# shellcheck disable=SC1090
|
||||
source $AUTOMATION_LIB_PATH/${libname}.sh
|
||||
done
|
||||
else
|
||||
(
|
||||
echo "WARNING: It does not appear that containers/automation was installed."
|
||||
echo " Functionality of most of this library will be negatively impacted"
|
||||
echo " This ${BASH_SOURCE[0]} was loaded by ${BASH_SOURCE[1]}"
|
||||
) > /dev/stderr
|
||||
fi
|
||||
|
||||
# Essential default paths, many are overridden when executing under Cirrus-CI
|
||||
# others are duplicated here, to assist in debugging.
|
||||
export GOPATH="${GOPATH:-/var/tmp/go}"
|
||||
|
|
@ -57,17 +80,17 @@ OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
|
|||
OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
|
||||
|
||||
# Working with dnf + timeout/retry
|
||||
export SHORT_DNFY='timeout_attempt_delay_command 120s 5 30s dnf -y'
|
||||
export LONG_DNFY='timeout_attempt_delay_command 300s 5 60s dnf -y'
|
||||
export SHORT_DNFY='lilto dnf -y'
|
||||
export LONG_DNFY='bigto dnf -y'
|
||||
# Working with apt under Debian/Ubuntu automation is a PITA, make it easy
|
||||
# Avoid some ways of getting stuck waiting for user input
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
# Short-cut for frequently used base command
|
||||
export SUDOAPTGET='sudo -E apt-get -q --yes'
|
||||
# Short list of packages or quick-running command
|
||||
SHORT_APTGET="timeout_attempt_delay_command 120s 5 60s $SUDOAPTGET"
|
||||
SHORT_APTGET="lilto $SUDOAPTGET"
|
||||
# Long list / long-running command
|
||||
LONG_APTGET="timeout_attempt_delay_command 300s 5 60s $SUDOAPTGET"
|
||||
LONG_APTGET="bigto $SUDOAPTGET"
|
||||
|
||||
# Packaging adjustments needed to:
|
||||
# https://github.com/containers/libpod/blob/master/contrib/cirrus/packer/fedora_setup.sh
|
||||
|
|
@ -85,91 +108,12 @@ else
|
|||
unset DM_LVM_VG_NAME DM_REF_FILEPATH
|
||||
fi
|
||||
|
||||
# Pass in a list of one or more envariable names; exit non-zero with
|
||||
# helpful error message if any value is empty
|
||||
req_env_var() {
|
||||
# Provide context. If invoked from function use its name; else script name
|
||||
local caller=${FUNCNAME[1]}
|
||||
if [[ -n "$caller" ]]; then
|
||||
# Indicate that it's a function name
|
||||
caller="$caller()"
|
||||
else
|
||||
# Not called from a function: use script name
|
||||
caller=$(basename $0)
|
||||
fi
|
||||
|
||||
# Usage check
|
||||
[[ -n "$1" ]] || die 1 "FATAL: req_env_var: invoked without arguments"
|
||||
|
||||
# Each input arg is an envariable name, e.g. HOME PATH etc. Expand each.
|
||||
# If any is empty, bail out and explain why.
|
||||
for i; do
|
||||
if [[ -z "${!i}" ]]; then
|
||||
die 9 "FATAL: $caller requires \$$i to be non-empty"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
show_env_vars() {
|
||||
echo "Showing selection of environment variable definitions:"
|
||||
_ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
|
||||
egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
|
||||
sort -u)
|
||||
for _env_var_name in $_ENV_VAR_NAMES
|
||||
do
|
||||
# Supports older BASH versions
|
||||
printf " ${_env_var_name}=%q\n" "$(printenv $_env_var_name)"
|
||||
done
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "************************************************"
|
||||
echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}"
|
||||
echo "************************************************"
|
||||
exit ${1:-1}
|
||||
}
|
||||
|
||||
bad_os_id_ver() {
|
||||
echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)"
|
||||
exit 42
|
||||
die "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)"
|
||||
}
|
||||
|
||||
timeout_attempt_delay_command() {
|
||||
TIMEOUT=$1
|
||||
ATTEMPTS=$2
|
||||
DELAY=$3
|
||||
shift 3
|
||||
CMD=$(echo "$@" | tr --squeeze-repeats '\r\n\v\t' ' ')
|
||||
STDOUTERR=$(mktemp -p '' $(basename $0)_XXXXX)
|
||||
req_env_var ATTEMPTS DELAY
|
||||
echo "Retrying $ATTEMPTS times with a $DELAY delay, and $TIMEOUT timeout for command: $CMD"
|
||||
for (( COUNT=1 ; COUNT <= $ATTEMPTS ; COUNT++ ))
|
||||
do
|
||||
echo "##### (attempt #$COUNT)" &>> "$STDOUTERR"
|
||||
if timeout --foreground $TIMEOUT $CMD &>> "$STDOUTERR"
|
||||
then
|
||||
echo "##### (success after #$COUNT attempts)" &>> "$STDOUTERR"
|
||||
break
|
||||
else
|
||||
echo "##### (failed with exit: $?)" &>> "$STDOUTERR"
|
||||
sleep $DELAY
|
||||
fi
|
||||
done
|
||||
cat "$STDOUTERR"
|
||||
rm -f "$STDOUTERR"
|
||||
if (( COUNT > $ATTEMPTS ))
|
||||
then
|
||||
echo "##### (exceeded $ATTEMPTS attempts)"
|
||||
exit 125
|
||||
fi
|
||||
}
|
||||
|
||||
# Helper/wrapper script to only show stderr/stdout on non-zero exit
|
||||
install_ooe() {
|
||||
req_env_var SCRIPT_BASE
|
||||
echo "Installing script to mask stdout/stderr unless non-zero exit."
|
||||
sudo install -D -m 755 "$SCRIPT_BASE/ooe.sh" /usr/local/bin/ooe.sh
|
||||
}
|
||||
lilto() { err_retry 8 1000 "" "$@"; } # just over 4 minutes max
|
||||
bigto() { err_retry 7 5670 "" "$@"; } # 12 minutes max
|
||||
|
||||
install_fuse_overlayfs_from_git(){
|
||||
wd=$(pwd)
|
||||
|
|
@ -198,34 +142,34 @@ showrun() {
|
|||
then
|
||||
shift
|
||||
# Properly escape any nested spaces, so command can be copy-pasted
|
||||
echo '+ '$(printf " %q" "$@")' &' > /dev/stderr
|
||||
msg '+ '$(printf " %q" "$@")' &'
|
||||
"$@" &
|
||||
echo -e "${RED}<backgrounded>${NOR}"
|
||||
msg -e "${RED}<backgrounded>${NOR}"
|
||||
else
|
||||
echo '--------------------------------------------------'
|
||||
echo '+ '$(printf " %q" "$@") > /dev/stderr
|
||||
msg '--------------------------------------------------'
|
||||
msg '+ '$(printf " %q" "$@") > /dev/stderr
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
devicemapper_setup() {
|
||||
req_env_var TEST_DRIVER DM_LVM_VG_NAME DM_REF_FILEPATH
|
||||
req_env_vars TEST_DRIVER DM_LVM_VG_NAME DM_REF_FILEPATH
|
||||
# Requires add_second_partition.sh to have already run successfully
|
||||
if [[ -r "/root/second_partition_ready" ]]
|
||||
then
|
||||
device=$(< /root/second_partition_ready)
|
||||
if [[ -n "$device" ]] # LVM setup should only ever happen once
|
||||
then
|
||||
echo "Setting up LVM PV on $device to validate it's functional"
|
||||
msg "Setting up LVM PV on $device to validate it's functional"
|
||||
showrun pvcreate --force --yes "$device"
|
||||
echo "Wiping LVM signatures from $device to prepare it for testing use"
|
||||
msg "Wiping LVM signatures from $device to prepare it for testing use"
|
||||
showrun pvremove --force --yes "$device"
|
||||
# Block setup from happening ever again
|
||||
truncate --size=0 /root/second_partition_ready # mark completion|in-use
|
||||
echo "$device" > "$DM_REF_FILEPATH"
|
||||
fi
|
||||
echo "Test device $(cat $DM_REF_FILEPATH) is ready to go."
|
||||
msg "Test device $(cat $DM_REF_FILEPATH) is ready to go."
|
||||
else
|
||||
echo "WARNING: Can't read /root/second_partition_ready, created by $(dirname $0)/add_second_partition.sh"
|
||||
warn "Can't read /root/second_partition_ready, created by $(dirname $0)/add_second_partition.sh"
|
||||
fi
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,88 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Unit tests for some functions in lib.sh
|
||||
#
|
||||
source $(dirname $0)/lib.sh
|
||||
|
||||
# Iterator and return code; updated in test functions
|
||||
testnum=0
|
||||
rc=0
|
||||
|
||||
function check_result {
|
||||
testnum=$(expr $testnum + 1)
|
||||
MSG=$(echo "$1" | tr -d '*>\012'|sed -e 's/^ \+//')
|
||||
if [ "$MSG" = "$2" ]; then
|
||||
echo "ok $testnum $3 = $MSG"
|
||||
else
|
||||
echo "not ok $testnum $3"
|
||||
echo "# expected: $2"
|
||||
echo "# actual: $MSG"
|
||||
rc=1
|
||||
fi
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# tests for die()
|
||||
|
||||
function test_die() {
|
||||
local input_status=$1
|
||||
local input_msg=$2
|
||||
local expected_status=$3
|
||||
local expected_msg=$4
|
||||
|
||||
local msg
|
||||
msg=$(die $input_status "$input_msg")
|
||||
local status=$?
|
||||
|
||||
check_result "$msg" "$expected_msg" "die $input_status $input_msg"
|
||||
}
|
||||
|
||||
test_die 1 "a message" 1 "a message"
|
||||
test_die 2 "" 2 "FATAL ERROR (but no message given!) in test_die()"
|
||||
test_die '' '' 1 "FATAL ERROR (but no message given!) in test_die()"
|
||||
|
||||
###############################################################################
|
||||
# tests for req_env_var()
|
||||
|
||||
function test_rev() {
|
||||
local input_args=$1
|
||||
local expected_status=$2
|
||||
local expected_msg=$3
|
||||
|
||||
# bash gotcha: doing 'local msg=...' on one line loses exit status
|
||||
local msg
|
||||
msg=$(req_env_var $input_args)
|
||||
local status=$?
|
||||
|
||||
check_result "$msg" "$expected_msg" "req_env_var $input_args"
|
||||
check_result "$status" "$expected_status" "req_env_var $input_args (rc)"
|
||||
}
|
||||
|
||||
# error if called with no args
|
||||
test_rev '' 1 'FATAL: req_env_var: invoked without arguments'
|
||||
|
||||
# error if desired envariable is unset
|
||||
unset FOO BAR
|
||||
test_rev FOO 9 'FATAL: test_rev() requires $FOO to be non-empty'
|
||||
test_rev BAR 9 'FATAL: test_rev() requires $BAR to be non-empty'
|
||||
# OK if desired envariable was unset
|
||||
FOO=1
|
||||
test_rev FOO 0 ''
|
||||
|
||||
# OK if multiple vars are non-empty
|
||||
FOO="stuff"
|
||||
BAR="things"
|
||||
ENV_VARS="FOO BAR"
|
||||
test_rev "$ENV_VARS" 0 ''
|
||||
unset BAR
|
||||
|
||||
# ...but error if any single desired one is unset
|
||||
test_rev "FOO BAR" 9 'FATAL: test_rev() requires $BAR to be non-empty'
|
||||
|
||||
# ...and OK if all args are set
|
||||
BAR=1
|
||||
test_rev "FOO BAR" 0 ''
|
||||
|
||||
###############################################################################
|
||||
|
||||
exit $rc
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# This script executes a command while logging all output to a temporary
|
||||
# file. If the command exits non-zero, then all output is sent to the console,
|
||||
# before returning the exit code. If the script itself fails, the exit code 121
|
||||
# is returned.
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
SCRIPT_BASEDIR="$(basename $0)"
|
||||
|
||||
badusage() {
|
||||
echo "Incorrect usage: $SCRIPT_BASEDIR) <command> [options]" > /dev/stderr
|
||||
echo "ERROR: $1"
|
||||
exit 121
|
||||
}
|
||||
|
||||
COMMAND="$@"
|
||||
[[ -n "$COMMAND" ]] || badusage "No command specified"
|
||||
|
||||
OUTPUT_TMPFILE="$(mktemp -p '' ${SCRIPT_BASEDIR}_output_XXXX)"
|
||||
output_on_error() {
|
||||
RET=$?
|
||||
set +e
|
||||
if [[ "$RET" -ne "0" ]]
|
||||
then
|
||||
echo "---------------------------"
|
||||
cat "$OUTPUT_TMPFILE"
|
||||
echo "[$(date --iso-8601=second)] <exit $RET> $COMMAND"
|
||||
fi
|
||||
rm -f "$OUTPUT_TMPFILE"
|
||||
}
|
||||
trap "output_on_error" EXIT
|
||||
|
||||
"$@" 2>&1 | while IFS='' read LINE # Preserve leading/trailing whitespace
|
||||
do
|
||||
# Every stdout and (copied) stderr line
|
||||
echo "[$(date --iso-8601=second)] $LINE"
|
||||
done >> "$OUTPUT_TMPFILE"
|
||||
|
|
@ -4,15 +4,12 @@ set -e
|
|||
|
||||
source $(dirname $0)/lib.sh
|
||||
|
||||
req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER SHORT_APTGET TEST_DRIVER
|
||||
|
||||
install_ooe
|
||||
req_env_vars GOSRC OS_RELEASE_ID OS_RELEASE_VER SHORT_APTGET TEST_DRIVER
|
||||
|
||||
show_env_vars
|
||||
|
||||
cd $GOSRC
|
||||
|
||||
echo "Setting up $OS_RELEASE_ID $OS_RELEASE_VER"
|
||||
msg "Setting up $OS_RELEASE_ID $OS_RELEASE_VER"
|
||||
case "$OS_RELEASE_ID" in
|
||||
fedora)
|
||||
$LONG_DNFY update # install latest packages
|
||||
|
|
|
|||
Loading…
Reference in New Issue