diff --git a/.github/workflows/boulder-ci.yml b/.github/workflows/boulder-ci.yml index a8e25821b..eac3bbbc9 100644 --- a/.github/workflows/boulder-ci.yml +++ b/.github/workflows/boulder-ci.yml @@ -39,9 +39,12 @@ jobs: - "docker-compose run --use-aliases boulder ./test.sh --integration" # Config changes that have landed in main but not yet been applied to # production can be made in boulder-config-next.json. - - "docker-compose run --use-aliases boulder ./test.sh --integration --config-next" + # Database migrations in `sa/_db-next/migrations` are only performed + # when `docker-compose` is called using `-f docker-compose.yml -f + # docker-compose.next.yml` + - "docker-compose -f docker-compose.yml -f docker-compose.next.yml run --use-aliases boulder ./test.sh --integration" - "docker-compose run --use-aliases boulder ./test.sh --unit --enable-race-detection" - - "docker-compose run --use-aliases boulder ./test.sh --unit --enable-race-detection --config-next" + - "docker-compose -f docker-compose.yml -f docker-compose.next.yml run --use-aliases boulder ./test.sh --unit --enable-race-detection" - "docker-compose run --use-aliases boulder ./test.sh --start-py" # gomod-vendor runs with a separate network access definition # because it needs to fetch packages from GitHub et. al., which diff --git a/.travis.yml b/.travis.yml index 622b57b7c..f07450bb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,9 +26,9 @@ env: - TESTFLAGS="--lints --integration --generate --rpm" # Config changes that have landed in main but not yet been applied to # production can be made in boulder-config-next.json. - - TESTFLAGS="--integration --config-next" + - TESTFLAGS="--integration" OVERRIDES="-f docker-compose.yml -f docker-compose.next.yml" - TESTFLAGS="--unit --enable-race-detection" - - TESTFLAGS="--unit --enable-race-detection --config-next" + - TESTFLAGS="--unit --enable-race-detection" OVERRIDES="-f docker-compose.yml -f docker-compose.next.yml" - TESTFLAGS="--start-py" # gomod-vendor runs with a separate container because it needs to fetch # packages from GitHub et. al., which is incompatible with the DNS server @@ -51,7 +51,7 @@ before_script: script: - >- - docker-compose run --use-aliases + docker-compose ${OVERRIDES} run --use-aliases -e TRAVIS_BRANCH -e TRAVIS_JOB_ID -e TRAVIS_PULL_REQUEST diff --git a/docker-compose.next.yml b/docker-compose.next.yml new file mode 100644 index 000000000..3cdf9ee8b --- /dev/null +++ b/docker-compose.next.yml @@ -0,0 +1,16 @@ +version: '3' +services: + boulder: + environment: + FAKE_DNS: 10.77.77.77 + BOULDER_CONFIG_DIR: test/config-next + GOFLAGS: -mod=vendor + # This is required so Python doesn't throw an error when printing + # non-ASCII to stdout. + PYTHONIOENCODING: utf-8 + # These are variables you can set to affect what tests get run or + # how they are run. Including them here with no value means they are + # passed through from the environment. + RUN: "" + INT_FILTER: "" + RACE: "" diff --git a/docker-compose.yml b/docker-compose.yml index b7c98b811..6b05ba1e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -80,6 +80,7 @@ services: environment: GO111MODULE: "on" GOFLAGS: "-mod=vendor" + BOULDER_CONFIG_DIR: test/config networks: - bluenet volumes: diff --git a/sa/_db-next/dbconf.yml b/sa/_db-next/dbconf.yml deleted file mode 100644 index 46a87734a..000000000 --- a/sa/_db-next/dbconf.yml +++ /dev/null @@ -1,10 +0,0 @@ -test: - driver: mysql - open: root@tcp(boulder-mysql:3306)/boulder_sa_test -integration: - driver: mysql - open: root@tcp(boulder-mysql:3306)/boulder_sa_integration -# what goose uses by default, even during migration creation -development: - driver: mysql - open: root@tcp(boulder-mysql:3306)/boulder_sa_integration diff --git a/sa/_db-next/dbconf.yml b/sa/_db-next/dbconf.yml new file mode 120000 index 000000000..557b467ad --- /dev/null +++ b/sa/_db-next/dbconf.yml @@ -0,0 +1 @@ +../_db/dbconf.yml \ No newline at end of file diff --git a/sa/_db-next/migrations/20210223140000_CombinedSchema.sql b/sa/_db-next/migrations/20210223140000_CombinedSchema.sql new file mode 120000 index 000000000..918ac6eff --- /dev/null +++ b/sa/_db-next/migrations/20210223140000_CombinedSchema.sql @@ -0,0 +1 @@ +../../_db/migrations/20210223140000_CombinedSchema.sql \ No newline at end of file diff --git a/sa/_db-next/migrations/20190423125755_DropCertStatusSubscriberApproved.sql b/sa/_db-next/migrations/20210223140001_DropCertStatusSubscriberApproved.sql similarity index 100% rename from sa/_db-next/migrations/20190423125755_DropCertStatusSubscriberApproved.sql rename to sa/_db-next/migrations/20210223140001_DropCertStatusSubscriberApproved.sql diff --git a/sa/_db-next/migrations/20190423132049_DropCertStatusLockCol.sql b/sa/_db-next/migrations/20210223140002_DropCertStatusLockCol.sql similarity index 100% rename from sa/_db-next/migrations/20190423132049_DropCertStatusLockCol.sql rename to sa/_db-next/migrations/20210223140002_DropCertStatusLockCol.sql diff --git a/sa/_db-next/migrations/20210216114200_IssuedNamesDropIndex.sql b/sa/_db-next/migrations/20210223140003_IssuedNamesDropIndex.sql similarity index 100% rename from sa/_db-next/migrations/20210216114200_IssuedNamesDropIndex.sql rename to sa/_db-next/migrations/20210223140003_IssuedNamesDropIndex.sql diff --git a/sa/migrations.sh b/sa/migrations.sh new file mode 100755 index 000000000..4f2b4e2bc --- /dev/null +++ b/sa/migrations.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash + +set -eu + +if type realpath >/dev/null 2>&1 ; then + cd "$(realpath -- $(dirname -- "$0"))" +fi + +# posix compliant escape sequence +esc=$'\033'"[" +res="${esc}0m" + +# +# Defaults +# +DB_NEXT_PATH="_db-next/migrations" +DB_PATH="_db/migrations" +OUTCOME="ERROR" +PROMOTE=() +RUN=() + +# +# Print Functions +# +function print_outcome() { + if [ "${OUTCOME}" == OK ] + then + echo -e "${esc}0;32;1m${OUTCOME}${res}" + else + echo -e "${esc}0;31;1m${OUTCOME}${res}" + fi +} + +function print_usage_exit() { + echo "${USAGE}" + exit 0 +} + +# newline + bold magenta +function print_heading() { + echo + echo -e "${esc}0;34;1m${1}${res}" +} + +# bold cyan +function print_moving() { + local src=${1} + local dest=${2} + echo -e "moving: ${esc}0;36;1m${src}${res}" + echo -e "to: ${esc}0;32;1m${dest}${res}" +} + +# bold yellow +function print_unlinking() { + echo -e "unlinking: ${esc}0;33;1m${1}${res}" +} + +# bold magenta +function print_linking () { + local from=${1} + local to=${2} + echo -e "linking: ${esc}0;35;1m${from} ->${res}" + echo -e "to: ${esc}0;39;1m${to}${res}" +} + +function print_migrations(){ + iter=1 + for file in "${migrations[@]}" + do + echo "${iter}) $(basename -- ${file})" + iter=$(expr "${iter}" + 1) + done +} + +function exit_msg() { + # complain to STDERR and exit with error + echo "${*}" >&2 + exit 2 +} + +# +# Utility Functions +# +function get_promotable_migrations() { + local migrations=() + for file in "${DB_NEXT_PATH}"/*.sql; do + [[ -f "${file}" && ! -L "${file}" ]] || continue + migrations+=("${file}") + done + if [[ "${migrations[@]}" ]]; then + echo "${migrations[@]}" + else + exit_msg "There are no promotable migrations at path: "\"${DB_NEXT_PATH}\""" + fi +} + +function get_demotable_migrations() { + local migrations=() + for file in "${DB_NEXT_PATH}"/*.sql; do + [[ -L "${file}" ]] || continue + migrations+=("${file}") + done + if [[ "${migrations[@]}" ]]; then + echo "${migrations[@]}" + else + exit_msg "There are no demotable migrations at path: "\"${DB_NEXT_PATH}\""" + fi +} + +# +# CLI Parser +# +USAGE="$(cat -- <<-EOM + +Usage: + + Boulder DB Migrations CLI + + Helper for listing, promoting, and demoting Boulder schema files + + ./$(basename "${0}") [OPTION]... + + -l, --list-next Lists schema files present in sa/_db-next + -c, --list-current Lists schema files promoted from sa/_db-next to sa/_db + -p, --promote Select and promote a schema from sa/_db-next to sa/_db + -d, --demote Select and demote a schema from sa/_db to sa/_db-next + -h, --help Shows this help message + +EOM +)" + +while getopts nchpd-: OPT; do + if [ "$OPT" = - ]; then # long option: reformulate OPT and OPTARG + OPT="${OPTARG%%=*}" # extract long option name + OPTARG="${OPTARG#$OPT}" # extract long option argument (may be empty) + OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=` + fi + case "${OPT}" in + n | list-next ) RUN+=("list_next") ;; + c | list-current ) RUN+=("list_current") ;; + p | promote ) RUN+=("promote") ;; + d | demote ) RUN+=("demote") ;; + h | help ) print_usage_exit ;; + ??* ) exit_msg "Illegal option --${OPT}" ;; # bad long option + ? ) exit 2 ;; # bad short option (error reported via getopts) + esac +done +shift $((OPTIND-1)) # remove parsed opts and args from $@ list + +# On EXIT, trap and print outcome +trap "print_outcome" EXIT + +STEP="list_next" +if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then + print_heading "Next Schemas" + migrations=($(get_promotable_migrations)) + print_migrations "${migrations[@]}" +fi + +STEP="list_current" +if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then + print_heading "Current Schemas" + migrations=($(get_demotable_migrations)) + print_migrations "${migrations[@]}" +fi + +STEP="promote" +if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then + print_heading "Promote Schema" + migrations=($(get_promotable_migrations)) + declare -a mig_index=() + declare -A mig_file=() + for i in "${!migrations[@]}"; do + mig_index["$i"]="${migrations[$i]%% *}" + mig_file["${mig_index[$i]}"]="${migrations[$i]#* }" + done + + promote="" + PS3='Which schema would you like to promote? (q to cancel): ' + + select opt in "${mig_index[@]}"; do + case "${opt}" in + "") echo "Invalid option or cancelled, exiting..." ; break ;; + *) mig_file_path="${mig_file[$opt]}" ; break ;; + esac + done + if [[ "${mig_file_path}" ]] + then + print_heading "Promoting Schema" + promote_mig_name="$(basename -- "${mig_file_path}")" + promoted_mig_file_path="${DB_PATH}/${promote_mig_name}" + symlink_relpath="$(realpath --relative-to=${DB_NEXT_PATH} ${promoted_mig_file_path})" + + print_moving "${mig_file_path}" "${promoted_mig_file_path}" + mv "${mig_file_path}" "${promoted_mig_file_path}" + + print_linking "${mig_file_path}" "${symlink_relpath}" + ln -s "${symlink_relpath}" "${DB_NEXT_PATH}" + fi +fi + +STEP="demote" +if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then + print_heading "Demote Schema" + migrations=($(get_demotable_migrations)) + declare -a mig_index=() + declare -A mig_file=() + for i in "${!migrations[@]}"; do + mig_index["$i"]="${migrations[$i]%% *}" + mig_file["${mig_index[$i]}"]="${migrations[$i]#* }" + done + + demote_mig="" + PS3='Which schema would you like to demote? (q to cancel): ' + + select opt in "${mig_index[@]}"; do + case "${opt}" in + "") echo "Invalid option or cancelled, exiting..." ; break ;; + *) mig_link_path="${mig_file[$opt]}" ; break ;; + esac + done + if [[ "${mig_link_path}" ]] + then + print_heading "Demoting Schema" + demote_mig_name="$(basename -- "${mig_link_path}")" + demote_mig_from="${DB_PATH}/${demote_mig_name}" + + print_unlinking "${mig_link_path}" + rm "${mig_link_path}" + print_moving "${demote_mig_from}" "${mig_link_path}" + mv "${demote_mig_from}" "${mig_link_path}" + fi +fi + +OUTCOME="OK" diff --git a/test.sh b/test.sh index 50e9a6c09..660ccb159 100755 --- a/test.sh +++ b/test.sh @@ -14,7 +14,6 @@ fi # Defaults # export RACE="false" -export BOULDER_CONFIG_DIR="test/config" STAGE="starting" STATUS="FAILURE" RUN=() diff --git a/test/create_db.sh b/test/create_db.sh index f1cfcc5f7..adf26f75d 100755 --- a/test/create_db.sh +++ b/test/create_db.sh @@ -3,6 +3,59 @@ set -o errexit cd $(dirname $0)/.. source test/db-common.sh +# posix compliant escape sequence +esc=$'\033'"[" +res="${esc}0m" + + +function print_heading() { + echo + # newline + bold magenta + echo -e "${esc}0;34;1m${1}${res}" +} + +function exit_msg() { + # complain to STDERR and exit with error + echo "${*}" >&2 + exit 2 +} + +function get_migrations() { + local db_schemas_path="${1}" + local migrations=() + for file in "${db_schemas_path}"/*.sql; do + [[ -f "${file}" ]] || continue + migrations+=("${file}") + done + if [[ "${migrations[@]}" ]]; then + echo "${migrations[@]}" + else + exit_msg "There are no migrations at path: "\"${db_schemas_path}\""" + fi +} + +function create_empty_db() { + local db="${1}" + local dbconn="${2}" + create_script="drop database if exists \`${db}\`; create database if not exists \`${db}\`;" + mysql ${dbconn} -e "${create_script}" || die "unable to create ${db}" + echo "created empty "$db" database" +} + +function apply_migrations() { + local migrations="${1}" + local dbpath="${2}" + local dbenv="${3}" + local db="${4}" + if [[ "${migrations[@]}" ]] + then + echo "applying migrations from ${db_mig_path}" + goose -path="${dbpath}" -env="${dbenv}" up || die "unable to migrate ${db} with ${dbpath}" + else + echo "no migrations at ${dbpath}" + fi +} + # set db connection for if running in a separate container or not dbconn="-u root" if [[ $MYSQL_CONTAINER ]]; then @@ -21,34 +74,51 @@ mysql $dbconn -e "SET GLOBAL max_connections = 500;" for dbenv in $DBENVS; do db="boulder_sa_${dbenv}" - - if mysql $dbconn -e 'show databases;' | grep $db > /dev/null; then - echo "Database $db already exists - skipping create" + print_heading "Checking if ${db} exists" + if mysql ${dbconn} -e 'show databases;' | grep "${db}" > /dev/null; then + echo "${db} already exists - skipping create" else - create_script="drop database if exists \`${db}\`; create database if not exists \`${db}\`;" - - mysql $dbconn -e "$create_script" || die "unable to create ${db}" - - echo "created empty ${db} database" + echo "${db} doesn't exist - creating" + create_empty_db "${db}" "${dbconn}" fi - goose -path=./sa/_db/ -env=$dbenv up || die "unable to migrate ${db} with ./sa/_db/" - echo "migrated ${db} database with ./sa/_db/" + # Determine which $dbpath and $db_mig_path to use. + if [[ "${BOULDER_CONFIG_DIR}" == "test/config-next" ]] + then + dbpath="./sa/_db-next" + else + dbpath="./sa/_db" + fi + db_mig_path="${dbpath}/migrations" - if [[ "$BOULDER_CONFIG_DIR" = "test/config-next" ]]; then - nextDir="./sa/_db-next/" + # Populate an array with schema files present at $dbpath. + migrations=($(get_migrations "${db_mig_path}")) - # Goose exits non-zero if there are no migrations to apply with the error - # message: - # "2016/09/26 15:43:38 no valid version found" - # so we only want to run goose with the nextDir if there is a migrations - # directory present with at least one migration - if [ $(find "$nextDir/migrations" -maxdepth 0 -type d -not -empty 2>/dev/null) ]; then - goose -path=${nextDir} -env=$dbenv up || die "unable to migrate ${db} with ${nextDir}" - echo "migrated ${db} database with ${nextDir}" - else - echo "no ${nextDir} migrations to apply" - fi + # Goose up, this will work if there are schema files present at + # $dbpath with a newer timestamp than the current goose dbversion. + apply_migrations "${migrations}" "${dbpath}" "${dbenv}" "${db}" + + # The (actual) latest migration should always be the last file or + # symlink at $db_mig_path. + latest_mig_path_filename="$(basename -- "${migrations[-1]}")" + + # Goose's dbversion is the timestamp (first 14 characters) of the file + # that it last migrated to. We can figure out which goose dbversion we + # should be on by parsing the timestamp of the latest file at + # $db_mig_path. + latest_db_mig_version="${latest_mig_path_filename:0:14}" + + # Ask Goose the timestamp (dbversion) our database is currently + # migrated to. + goose_dbversion="$(goose -path=${dbpath} -env=${dbenv} dbversion | sed 's/goose: dbversion //')" + + # If the $goose_dbversion does not match the $latest_in_db_mig_path, + # trigger recreate + if [[ "${latest_db_mig_version}" != "${goose_dbversion}" ]]; then + print_heading "Detected latest migration version mismatch" + echo "dropping and recreating from migrations at ${db_mig_path}" + create_empty_db "${db}" "${dbconn}" + apply_migrations "${migrations}" "${dbpath}" "${dbenv}" "${db}" fi # With MYSQL_CONTAINER, patch the GRANT statements to @@ -68,4 +138,5 @@ for dbenv in $DBENVS; do echo "added users to ${db}" done -echo "created all databases" +echo +echo "database setup complete"