Run db-next migrations with config-next configuration (#5320)

Docker container should load the appropriate schema (`sa/_db` or
`sa/_db-next`) for the given configuration.

- Add `docker-compose.next.yml` docker-compose overrides
- Detect when to apply `sa/_db-next/migrations`
- Detect mismatch between `goose dbversion` and the latest migration
- Symlink `promoted` schema back to `sa/_db-next/migrations`
- Add tooling to consistently promote/demote schema migrations

Fixes #5300

.github/workflows/boulder-ci.yml

@@ -39,9 +39,12 @@ jobs:
           - "docker-compose run --use-aliases boulder ./test.sh --integration"
           # Config changes that have landed in main but not yet been applied to
           # production can be made in boulder-config-next.json.
-          - "docker-compose run --use-aliases boulder ./test.sh --integration --config-next"
+          # Database migrations in `sa/_db-next/migrations` are only performed 
+          # when `docker-compose` is called using `-f docker-compose.yml -f 
+          # docker-compose.next.yml`
+          - "docker-compose -f docker-compose.yml -f docker-compose.next.yml run --use-aliases boulder ./test.sh --integration"
           - "docker-compose run --use-aliases boulder ./test.sh --unit --enable-race-detection"
-          - "docker-compose run --use-aliases boulder ./test.sh --unit --enable-race-detection --config-next"
+          - "docker-compose -f docker-compose.yml -f docker-compose.next.yml run --use-aliases boulder ./test.sh --unit --enable-race-detection"
           - "docker-compose run --use-aliases boulder ./test.sh --start-py"
           # gomod-vendor runs with a separate network access definition
           # because it needs to fetch packages from GitHub et. al., which

.travis.yml

@@ -26,9 +26,9 @@ env:
     - TESTFLAGS="--lints --integration --generate --rpm"
     # Config changes that have landed in main but not yet been applied to
     # production can be made in boulder-config-next.json.
-    - TESTFLAGS="--integration --config-next"
+    - TESTFLAGS="--integration" OVERRIDES="-f docker-compose.yml -f docker-compose.next.yml"
     - TESTFLAGS="--unit --enable-race-detection"
-    - TESTFLAGS="--unit --enable-race-detection --config-next"
+    - TESTFLAGS="--unit --enable-race-detection" OVERRIDES="-f docker-compose.yml -f docker-compose.next.yml"
     - TESTFLAGS="--start-py"
     # gomod-vendor runs with a separate container because it needs to fetch
     # packages from GitHub et. al., which is incompatible with the DNS server
@@ -51,7 +51,7 @@ before_script:
 
 script:
   - >-
-    docker-compose run --use-aliases
+    docker-compose ${OVERRIDES} run --use-aliases
     -e TRAVIS_BRANCH
     -e TRAVIS_JOB_ID
     -e TRAVIS_PULL_REQUEST

docker-compose.next.yml

@@ -0,0 +1,16 @@
+version: '3'
+services:
+    boulder:
+      environment:
+          FAKE_DNS: 10.77.77.77
+          BOULDER_CONFIG_DIR: test/config-next
+          GOFLAGS: -mod=vendor
+          # This is required so Python doesn't throw an error when printing
+          # non-ASCII to stdout.
+          PYTHONIOENCODING: utf-8
+          # These are variables you can set to affect what tests get run or
+          # how they are run. Including them here with no value means they are
+          # passed through from the environment.
+          RUN: ""
+          INT_FILTER: ""
+          RACE: ""

docker-compose.yml

@@ -80,6 +80,7 @@ services:
         environment:
             GO111MODULE: "on"
             GOFLAGS: "-mod=vendor"
+            BOULDER_CONFIG_DIR: test/config
         networks:
           - bluenet
         volumes:

sa/_db-next/dbconf.yml

@@ -1,10 +0,0 @@
-test:
-  driver: mysql
-  open: root@tcp(boulder-mysql:3306)/boulder_sa_test
-integration:
-  driver: mysql
-  open: root@tcp(boulder-mysql:3306)/boulder_sa_integration
-# what goose uses by default, even during migration creation
-development:
-  driver: mysql
-  open: root@tcp(boulder-mysql:3306)/boulder_sa_integration

sa/_db-next/dbconf.yml

@@ -0,0 +1 @@
+../_db/dbconf.yml

sa/_db-next/migrations/20210223140000_CombinedSchema.sql

@@ -0,0 +1 @@
+../../_db/migrations/20210223140000_CombinedSchema.sql

sa/migrations.sh

@@ -0,0 +1,235 @@
+#!/usr/bin/env bash
+
+set -eu
+
+if type realpath >/dev/null 2>&1 ; then
+  cd "$(realpath -- $(dirname -- "$0"))"
+fi
+
+# posix compliant escape sequence
+esc=$'\033'"["
+res="${esc}0m"
+
+#
+# Defaults
+#
+DB_NEXT_PATH="_db-next/migrations"
+DB_PATH="_db/migrations"
+OUTCOME="ERROR"
+PROMOTE=()
+RUN=()
+
+#
+# Print Functions
+#
+function print_outcome() {
+  if [ "${OUTCOME}" == OK ]
+  then
+    echo -e "${esc}0;32;1m${OUTCOME}${res}"
+  else
+    echo -e "${esc}0;31;1m${OUTCOME}${res}"
+  fi
+}
+
+function print_usage_exit() {
+  echo "${USAGE}"
+  exit 0
+}
+
+# newline + bold magenta
+function print_heading() {
+  echo
+  echo -e "${esc}0;34;1m${1}${res}"
+}
+
+# bold cyan
+function print_moving() {
+  local src=${1}
+  local dest=${2}
+  echo -e "moving:    ${esc}0;36;1m${src}${res}"
+  echo -e "to:        ${esc}0;32;1m${dest}${res}"
+}
+
+# bold yellow
+function print_unlinking() {
+  echo -e "unlinking: ${esc}0;33;1m${1}${res}"
+}
+
+# bold magenta
+function print_linking () {
+  local from=${1}
+  local to=${2}
+  echo -e "linking:   ${esc}0;35;1m${from} ->${res}"
+  echo -e "to:        ${esc}0;39;1m${to}${res}"
+}
+
+function print_migrations(){
+  iter=1
+  for file in "${migrations[@]}"
+  do
+    echo "${iter}) $(basename -- ${file})"
+    iter=$(expr "${iter}" + 1)
+  done
+}
+
+function exit_msg() {
+  # complain to STDERR and exit with error
+  echo "${*}" >&2
+  exit 2
+}
+
+#
+# Utility Functions
+#
+function get_promotable_migrations() {
+  local migrations=()
+  for file in "${DB_NEXT_PATH}"/*.sql; do
+    [[ -f "${file}" && ! -L "${file}" ]] || continue
+    migrations+=("${file}")
+  done
+  if [[ "${migrations[@]}" ]]; then
+    echo "${migrations[@]}"
+  else
+    exit_msg "There are no promotable migrations at path: "\"${DB_NEXT_PATH}\"""
+  fi
+}
+
+function get_demotable_migrations() {
+  local migrations=()
+  for file in "${DB_NEXT_PATH}"/*.sql; do
+    [[ -L "${file}" ]] || continue
+    migrations+=("${file}")
+  done
+  if [[ "${migrations[@]}" ]]; then
+    echo "${migrations[@]}"
+  else
+    exit_msg "There are no demotable migrations at path: "\"${DB_NEXT_PATH}\"""
+  fi
+}
+
+#
+# CLI Parser
+#
+USAGE="$(cat -- <<-EOM
+
+Usage:
+  
+  Boulder DB Migrations CLI
+
+  Helper for listing, promoting, and demoting Boulder schema files
+
+  ./$(basename "${0}") [OPTION]...
+
+  -l, --list-next           Lists schema files present in sa/_db-next
+  -c, --list-current        Lists schema files promoted from sa/_db-next to sa/_db 
+  -p, --promote             Select and promote a schema from sa/_db-next to sa/_db
+  -d, --demote              Select and demote a schema from sa/_db to sa/_db-next
+  -h, --help                Shows this help message
+
+EOM
+)"
+
+while getopts nchpd-: OPT; do
+  if [ "$OPT" = - ]; then     # long option: reformulate OPT and OPTARG
+    OPT="${OPTARG%%=*}"       # extract long option name
+    OPTARG="${OPTARG#$OPT}"   # extract long option argument (may be empty)
+    OPTARG="${OPTARG#=}"      # if long option argument, remove assigning `=`
+  fi
+  case "${OPT}" in
+    n | list-next )           RUN+=("list_next") ;;
+    c | list-current )        RUN+=("list_current") ;;
+    p | promote )             RUN+=("promote") ;;
+    d | demote )              RUN+=("demote") ;;
+    h | help )                print_usage_exit ;;
+    ??* )                     exit_msg "Illegal option --${OPT}" ;;  # bad long option
+    ? )                       exit 2 ;;  # bad short option (error reported via getopts)
+  esac
+done
+shift $((OPTIND-1)) # remove parsed opts and args from $@ list
+
+# On EXIT, trap and print outcome
+trap "print_outcome" EXIT
+
+STEP="list_next"
+if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then
+  print_heading "Next Schemas"
+  migrations=($(get_promotable_migrations))
+  print_migrations "${migrations[@]}"
+fi
+
+STEP="list_current"
+if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then
+  print_heading "Current Schemas"
+  migrations=($(get_demotable_migrations))
+  print_migrations "${migrations[@]}"
+fi
+
+STEP="promote"
+if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then
+  print_heading "Promote Schema"
+  migrations=($(get_promotable_migrations))
+  declare -a mig_index=()
+  declare -A mig_file=()
+  for i in "${!migrations[@]}"; do
+    mig_index["$i"]="${migrations[$i]%% *}"
+    mig_file["${mig_index[$i]}"]="${migrations[$i]#* }"
+  done
+
+  promote=""
+  PS3='Which schema would you like to promote? (q to cancel): '
+  
+  select opt in "${mig_index[@]}"; do
+    case "${opt}" in
+      "") echo "Invalid option or cancelled, exiting..." ; break ;;
+      *)  mig_file_path="${mig_file[$opt]}" ; break ;;
+    esac
+  done
+  if [[ "${mig_file_path}" ]]
+  then
+    print_heading "Promoting Schema"
+    promote_mig_name="$(basename -- "${mig_file_path}")"
+    promoted_mig_file_path="${DB_PATH}/${promote_mig_name}"
+    symlink_relpath="$(realpath --relative-to=${DB_NEXT_PATH} ${promoted_mig_file_path})"
+
+    print_moving "${mig_file_path}" "${promoted_mig_file_path}"
+    mv "${mig_file_path}" "${promoted_mig_file_path}"
+    
+    print_linking "${mig_file_path}" "${symlink_relpath}"
+    ln -s "${symlink_relpath}" "${DB_NEXT_PATH}"
+  fi
+fi
+
+STEP="demote"
+if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then
+  print_heading "Demote Schema"
+  migrations=($(get_demotable_migrations))
+  declare -a mig_index=()
+  declare -A mig_file=()
+  for i in "${!migrations[@]}"; do
+    mig_index["$i"]="${migrations[$i]%% *}"
+    mig_file["${mig_index[$i]}"]="${migrations[$i]#* }"
+  done
+
+  demote_mig=""
+  PS3='Which schema would you like to demote? (q to cancel): '
+  
+  select opt in "${mig_index[@]}"; do
+    case "${opt}" in
+      "") echo "Invalid option or cancelled, exiting..." ; break ;;
+      *)  mig_link_path="${mig_file[$opt]}" ; break ;;
+    esac
+  done
+  if [[ "${mig_link_path}" ]]
+  then
+    print_heading "Demoting Schema"
+    demote_mig_name="$(basename -- "${mig_link_path}")"
+    demote_mig_from="${DB_PATH}/${demote_mig_name}"
+
+    print_unlinking "${mig_link_path}"
+    rm "${mig_link_path}"
+    print_moving "${demote_mig_from}" "${mig_link_path}"
+    mv "${demote_mig_from}" "${mig_link_path}"
+  fi
+fi
+
+OUTCOME="OK"

test.sh

@@ -14,7 +14,6 @@ fi
 # Defaults
 #
 export RACE="false"
-export BOULDER_CONFIG_DIR="test/config"
 STAGE="starting"
 STATUS="FAILURE"
 RUN=()

test/create_db.sh

@@ -3,6 +3,59 @@ set -o errexit
 cd $(dirname $0)/..
 source test/db-common.sh
 
+# posix compliant escape sequence
+esc=$'\033'"["
+res="${esc}0m"
+
+
+function print_heading() {
+  echo
+  # newline + bold magenta
+  echo -e "${esc}0;34;1m${1}${res}"
+}
+
+function exit_msg() {
+  # complain to STDERR and exit with error
+  echo "${*}" >&2
+  exit 2
+}
+
+function get_migrations() {
+  local db_schemas_path="${1}"
+  local migrations=()
+  for file in "${db_schemas_path}"/*.sql; do
+    [[ -f "${file}" ]] || continue
+    migrations+=("${file}")
+  done
+  if [[ "${migrations[@]}" ]]; then
+    echo "${migrations[@]}"
+  else
+    exit_msg "There are no migrations at path: "\"${db_schemas_path}\"""
+  fi
+}
+
+function create_empty_db() {
+  local db="${1}"
+  local dbconn="${2}"
+  create_script="drop database if exists \`${db}\`; create database if not exists \`${db}\`;"
+  mysql ${dbconn} -e "${create_script}" || die "unable to create ${db}"
+  echo "created empty "$db" database"
+}
+
+function apply_migrations() {
+  local migrations="${1}"
+  local dbpath="${2}"
+  local dbenv="${3}"
+  local db="${4}"
+  if [[ "${migrations[@]}" ]]
+  then
+    echo "applying migrations from ${db_mig_path}"
+    goose -path="${dbpath}" -env="${dbenv}" up || die "unable to migrate ${db} with ${dbpath}"
+  else
+    echo "no migrations at ${dbpath}"
+  fi
+}
+
 # set db connection for if running in a separate container or not
 dbconn="-u root"
 if [[ $MYSQL_CONTAINER ]]; then
@@ -21,34 +74,51 @@ mysql $dbconn -e "SET GLOBAL max_connections = 500;"
 
 for dbenv in $DBENVS; do
   db="boulder_sa_${dbenv}"
-
-  if mysql $dbconn -e 'show databases;' | grep $db > /dev/null; then
-    echo "Database $db already exists - skipping create"
+  print_heading "Checking if ${db} exists"
+  if mysql ${dbconn} -e 'show databases;' | grep "${db}" > /dev/null; then
+    echo "${db} already exists - skipping create"
   else
-    create_script="drop database if exists \`${db}\`; create database if not exists \`${db}\`;"
-
-    mysql $dbconn -e "$create_script" || die "unable to create ${db}"
-
-    echo "created empty ${db} database"
+    echo "${db} doesn't exist - creating"
+    create_empty_db "${db}" "${dbconn}"
   fi
 
-  goose -path=./sa/_db/ -env=$dbenv up || die "unable to migrate ${db} with ./sa/_db/"
-  echo "migrated ${db} database with ./sa/_db/"
+  # Determine which $dbpath and $db_mig_path to use.
+  if [[ "${BOULDER_CONFIG_DIR}" == "test/config-next" ]]
+  then
+    dbpath="./sa/_db-next"
+  else
+    dbpath="./sa/_db"
+  fi
+  db_mig_path="${dbpath}/migrations"
 
-  if [[ "$BOULDER_CONFIG_DIR" = "test/config-next" ]]; then
-    nextDir="./sa/_db-next/"
+  # Populate an array with schema files present at $dbpath.
+  migrations=($(get_migrations "${db_mig_path}"))
 
-    # Goose exits non-zero if there are no migrations to apply with the error
-    # message:
-    #   "2016/09/26 15:43:38 no valid version found"
-    # so we only want to run goose with the nextDir if there is a migrations
-    # directory present with at least one migration
-    if [ $(find "$nextDir/migrations" -maxdepth 0 -type d -not -empty 2>/dev/null) ]; then
-      goose -path=${nextDir} -env=$dbenv up || die "unable to migrate ${db} with ${nextDir}"
-      echo "migrated ${db} database with ${nextDir}"
-    else
-      echo "no ${nextDir} migrations to apply"
-    fi
+  # Goose up, this will work if there are schema files present at
+  # $dbpath with a newer timestamp than the current goose dbversion.
+  apply_migrations "${migrations}" "${dbpath}" "${dbenv}" "${db}"
+
+  # The (actual) latest migration should always be the last file or
+  # symlink at $db_mig_path.
+  latest_mig_path_filename="$(basename -- "${migrations[-1]}")"
+
+  # Goose's dbversion is the timestamp (first 14 characters) of the file
+  # that it last migrated to. We can figure out which goose dbversion we
+  # should be on by parsing the timestamp of the latest file at
+  # $db_mig_path.
+  latest_db_mig_version="${latest_mig_path_filename:0:14}"
+  
+  # Ask Goose the timestamp (dbversion) our database is currently
+  # migrated to.
+  goose_dbversion="$(goose -path=${dbpath} -env=${dbenv} dbversion | sed 's/goose: dbversion //')"
+
+  # If the $goose_dbversion does not match the $latest_in_db_mig_path,
+  # trigger recreate
+  if [[ "${latest_db_mig_version}" != "${goose_dbversion}" ]]; then
+    print_heading "Detected latest migration version mismatch"
+    echo "dropping and recreating from migrations at ${db_mig_path}"
+    create_empty_db "${db}" "${dbconn}"
+    apply_migrations "${migrations}" "${dbpath}" "${dbenv}" "${db}"
   fi
 
   # With MYSQL_CONTAINER, patch the GRANT statements to
@@ -68,4 +138,5 @@ for dbenv in $DBENVS; do
   echo "added users to ${db}"
 done
 
-echo "created all databases"
+echo
+echo "database setup complete"