diff --git a/Makefile b/Makefile index e3c8e6d9d..d9beb6a34 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ rpm: build --package "$(ARCHIVEDIR)/boulder-$(VERSION)-$(COMMIT_ID).x86_64.rpm" \ --description "Boulder is an ACME-compatible X.509 Certificate Authority" \ --maintainer "$(MAINTAINER)" \ - test/config/ sa/_db data/ $(OBJECTS) + test/config/ sa/db data/ $(OBJECTS) deb: build fpm -f -s dir -t deb --name "boulder" \ @@ -64,10 +64,10 @@ deb: build --package "$(ARCHIVEDIR)/boulder-$(VERSION)-$(COMMIT_ID).x86_64.deb" \ --description "Boulder is an ACME-compatible X.509 Certificate Authority" \ --maintainer "$(MAINTAINER)" \ - test/config/ sa/_db data/ $(OBJECTS) bin/ct-test-srv + test/config/ sa/db data/ $(OBJECTS) bin/ct-test-srv tar: build fpm -f -s dir -t tar --name "boulder" --prefix=/opt/boulder \ --package "$(ARCHIVEDIR)/boulder-$(VERSION)-$(COMMIT_ID).amd64.tar" \ - test/config/ sa/_db data/ $(OBJECTS) + test/config/ sa/db data/ $(OBJECTS) gzip -f "$(ARCHIVEDIR)/boulder-$(VERSION)-$(COMMIT_ID).amd64.tar" diff --git a/docker-compose.yml b/docker-compose.yml index 02a97d1bb..7e74833fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: # used for release builds. make-deb.sh relies on being able to parse the # numeric version between 'go' and the underscore-prefixed date. If you make # changes to these tokens, please update this parsing logic. - image: &boulder_image letsencrypt/boulder-tools:${BOULDER_TOOLS_TAG:-go1.18.4_2022-08-11} + image: &boulder_image letsencrypt/boulder-tools:${BOULDER_TOOLS_TAG:-go1.18.6_2022-09-06} environment: FAKE_DNS: 10.77.77.77 BOULDER_CONFIG_DIR: test/config diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 899f6f899..91eca8bf9 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -289,18 +289,22 @@ func initTables(dbMap *gorp.DbMap) { } ``` -You can then add a migration with: +New migrations should be added at `./sa/db-next`: -`$ goose -path ./sa/_db/ create AddWizards sql` +```shell +$ cd sa/db +$ sql-migrate new -env="boulder_sa_test" AddWizards +Created migration boulder_sa/20220906165519-AddWizards.sql +``` Finally, edit the resulting file -(`sa/_db/migrations/20160915101011_AddWizards.sql`) to define your migration: +(`sa/db-next/boulder_sa/20220906165519-AddWizards.sql`) to define your migration: ```mysql --- +goose Up +-- +migrate Up ALTER TABLE people ADD isWizard BOOLEAN SET DEFAULT false; --- +goose Down +-- +migrate Down ALTER TABLE people DROP isWizard BOOLEAN SET DEFAULT false; ``` diff --git a/sa/_db-next/dbconf.yml b/sa/_db-next/dbconf.yml deleted file mode 120000 index 557b467ad..000000000 --- a/sa/_db-next/dbconf.yml +++ /dev/null @@ -1 +0,0 @@ -../_db/dbconf.yml \ No newline at end of file diff --git a/sa/_db-next/migrations/20210223140000_CombinedSchema.sql b/sa/_db-next/migrations/20210223140000_CombinedSchema.sql deleted file mode 120000 index 918ac6eff..000000000 --- a/sa/_db-next/migrations/20210223140000_CombinedSchema.sql +++ /dev/null @@ -1 +0,0 @@ -../../_db/migrations/20210223140000_CombinedSchema.sql \ No newline at end of file diff --git a/sa/_db-next/migrations/20210308140000_SimplePartitioning.sql b/sa/_db-next/migrations/20210308140000_SimplePartitioning.sql deleted file mode 120000 index 222cdb159..000000000 --- a/sa/_db-next/migrations/20210308140000_SimplePartitioning.sql +++ /dev/null @@ -1 +0,0 @@ -../../_db/migrations/20210308140000_SimplePartitioning.sql \ No newline at end of file diff --git a/sa/_db-next/migrations/20220328100000_Incidents.sql b/sa/_db-next/migrations/20220328100000_Incidents.sql deleted file mode 120000 index d56df031f..000000000 --- a/sa/_db-next/migrations/20220328100000_Incidents.sql +++ /dev/null @@ -1 +0,0 @@ -../../_db/migrations/20220328100000_Incidents.sql \ No newline at end of file diff --git a/sa/_db/dbconf.yml b/sa/_db/dbconf.yml deleted file mode 100644 index 46a87734a..000000000 --- a/sa/_db/dbconf.yml +++ /dev/null @@ -1,10 +0,0 @@ -test: - driver: mysql - open: root@tcp(boulder-mysql:3306)/boulder_sa_test -integration: - driver: mysql - open: root@tcp(boulder-mysql:3306)/boulder_sa_integration -# what goose uses by default, even during migration creation -development: - driver: mysql - open: root@tcp(boulder-mysql:3306)/boulder_sa_integration diff --git a/sa/db-next/boulder_sa/20210223140000_CombinedSchema.sql b/sa/db-next/boulder_sa/20210223140000_CombinedSchema.sql new file mode 120000 index 000000000..817437886 --- /dev/null +++ b/sa/db-next/boulder_sa/20210223140000_CombinedSchema.sql @@ -0,0 +1 @@ +../../db/boulder_sa/20210223140000_CombinedSchema.sql \ No newline at end of file diff --git a/sa/_db-next/migrations/20210223140001_DropCertStatusSubscriberApproved.sql b/sa/db-next/boulder_sa/20210223140001_DropCertStatusSubscriberApproved.sql similarity index 90% rename from sa/_db-next/migrations/20210223140001_DropCertStatusSubscriberApproved.sql rename to sa/db-next/boulder_sa/20210223140001_DropCertStatusSubscriberApproved.sql index b7a41a974..f1dfadabb 100644 --- a/sa/_db-next/migrations/20210223140001_DropCertStatusSubscriberApproved.sql +++ b/sa/db-next/boulder_sa/20210223140001_DropCertStatusSubscriberApproved.sql @@ -1,10 +1,10 @@ --- +goose Up +-- +migrate Up -- SQL in section 'Up' is executed when this migration is applied ALTER TABLE `certificateStatus` DROP COLUMN `subscriberApproved`; --- +goose Down +-- +migrate Down -- SQL section 'Down' is executed when this migration is rolled back ALTER TABLE `certificateStatus` ADD COLUMN `subscriberApproved` TINYINT(1) DEFAULT 0; diff --git a/sa/_db-next/migrations/20210223140002_DropCertStatusLockCol.sql b/sa/db-next/boulder_sa/20210223140002_DropCertStatusLockCol.sql similarity index 89% rename from sa/_db-next/migrations/20210223140002_DropCertStatusLockCol.sql rename to sa/db-next/boulder_sa/20210223140002_DropCertStatusLockCol.sql index 3e6459e96..f634cac25 100644 --- a/sa/_db-next/migrations/20210223140002_DropCertStatusLockCol.sql +++ b/sa/db-next/boulder_sa/20210223140002_DropCertStatusLockCol.sql @@ -1,10 +1,10 @@ --- +goose Up +-- +migrate Up -- SQL in section 'Up' is executed when this migration is applied ALTER TABLE `certificateStatus` DROP COLUMN `LockCol`; --- +goose Down +-- +migrate Down -- SQL section 'Down' is executed when this migration is rolled back ALTER TABLE `certificateStatus` ADD COLUMN `LockCol` BIGINT(20) DEFAULT 0; diff --git a/sa/_db-next/migrations/20210223140003_IssuedNamesDropIndex.sql b/sa/db-next/boulder_sa/20210223140003_IssuedNamesDropIndex.sql similarity index 90% rename from sa/_db-next/migrations/20210223140003_IssuedNamesDropIndex.sql rename to sa/db-next/boulder_sa/20210223140003_IssuedNamesDropIndex.sql index f52d70d8b..92724bc9c 100644 --- a/sa/_db-next/migrations/20210223140003_IssuedNamesDropIndex.sql +++ b/sa/db-next/boulder_sa/20210223140003_IssuedNamesDropIndex.sql @@ -1,10 +1,10 @@ --- +goose Up +-- +migrate Up -- SQL in section 'Up' is executed when this migration is applied ALTER TABLE issuedNames DROP INDEX `reversedName_renewal_notBefore_Idx`; --- +goose Down +-- +migrate Down -- SQL section 'Down' is executed when this migration is rolled back ALTER TABLE issuedNames ADD INDEX `reversedName_renewal_notBefore_Idx` (`reversedName`,`renewal`,`notBefore`); diff --git a/sa/db-next/boulder_sa/20210308140000_SimplePartitioning.sql b/sa/db-next/boulder_sa/20210308140000_SimplePartitioning.sql new file mode 120000 index 000000000..c1d45be5b --- /dev/null +++ b/sa/db-next/boulder_sa/20210308140000_SimplePartitioning.sql @@ -0,0 +1 @@ +../../db/boulder_sa/20210308140000_SimplePartitioning.sql \ No newline at end of file diff --git a/sa/db-next/boulder_sa/20220328100000_Incidents.sql b/sa/db-next/boulder_sa/20220328100000_Incidents.sql new file mode 120000 index 000000000..2f7718557 --- /dev/null +++ b/sa/db-next/boulder_sa/20220328100000_Incidents.sql @@ -0,0 +1 @@ +../../db/boulder_sa/20220328100000_Incidents.sql \ No newline at end of file diff --git a/sa/db-next/dbconfig.yml b/sa/db-next/dbconfig.yml new file mode 120000 index 000000000..cb034f30b --- /dev/null +++ b/sa/db-next/dbconfig.yml @@ -0,0 +1 @@ +../db/dbconfig.yml \ No newline at end of file diff --git a/sa/db-next/incidents_sa/20220328100000_Incidents.sql b/sa/db-next/incidents_sa/20220328100000_Incidents.sql new file mode 120000 index 000000000..957d64710 --- /dev/null +++ b/sa/db-next/incidents_sa/20220328100000_Incidents.sql @@ -0,0 +1 @@ +../../db/incidents_sa/20220328100000_Incidents.sql \ No newline at end of file diff --git a/test/sa_db_users.sql b/sa/db-users/boulder_sa.sql similarity index 98% rename from test/sa_db_users.sql rename to sa/db-users/boulder_sa.sql index 39a490f66..7e6adc8e6 100644 --- a/test/sa_db_users.sql +++ b/sa/db-users/boulder_sa.sql @@ -1,4 +1,4 @@ --- sa_db_users.sql is run by test/create_db.sh to create users for each +-- this file is run by test/create_db.sh to create users for each -- component with the appropriate permissions. -- These lines require MariaDB 10.1+ diff --git a/sa/db-users/incidents_sa.sql b/sa/db-users/incidents_sa.sql new file mode 100644 index 000000000..1f5193608 --- /dev/null +++ b/sa/db-users/incidents_sa.sql @@ -0,0 +1,13 @@ +-- this file is run by test/create_db.sh to create users for each +-- component with the appropriate permissions. + +-- These lines require MariaDB 10.1+ +CREATE USER IF NOT EXISTS 'sa'@'localhost'; +CREATE USER IF NOT EXISTS 'test_setup'@'localhost'; + +-- Storage Authority +GRANT SELECT,INSERT,UPDATE ON incident_foo TO 'sa'@'localhost'; +GRANT SELECT,INSERT,UPDATE ON incident_bar TO 'sa'@'localhost'; + +-- Test setup and teardown +GRANT ALL PRIVILEGES ON * to 'test_setup'@'localhost'; diff --git a/sa/_db/migrations/20210223140000_CombinedSchema.sql b/sa/db/boulder_sa/20210223140000_CombinedSchema.sql similarity index 94% rename from sa/_db/migrations/20210223140000_CombinedSchema.sql rename to sa/db/boulder_sa/20210223140000_CombinedSchema.sql index f9705d018..21030b33d 100644 --- a/sa/_db/migrations/20210223140000_CombinedSchema.sql +++ b/sa/db/boulder_sa/20210223140000_CombinedSchema.sql @@ -1,4 +1,4 @@ --- +goose Up +-- +migrate Up -- SQL in section 'Up' is executed when this migration is applied CREATE TABLE `authz2` ( @@ -213,25 +213,25 @@ CREATE TABLE `serials` ( CONSTRAINT `regId_serials` FOREIGN KEY (`registrationID`) REFERENCES `registrations` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION ) ENGINE=InnoDB DEFAULT CHARSET=utf8; --- +goose Down +-- +migrate Down -- SQL section 'Down' is executed when this migration is rolled back -- First set of tables have foreign key constraints, so are dropped first. -DROP TABLE `certificates` -DROP TABLE `orderFqdnSets` -DROP TABLE `precertificates` -DROP TABLE `requestedNames` -DROP TABLE `serials` +DROP TABLE `certificates`; +DROP TABLE `orderFqdnSets`; +DROP TABLE `precertificates`; +DROP TABLE `requestedNames`; +DROP TABLE `serials`; -DROP TABLE `authz2` -DROP TABLE `blockedKeys` -DROP TABLE `certificateStatus` -DROP TABLE `certificatesPerName` -DROP TABLE `crls` -DROP TABLE `fqdnSets` -DROP TABLE `issuedNames` -DROP TABLE `keyHashToSerial` -DROP TABLE `newOrdersRL` -DROP TABLE `orderToAuthz2` -DROP TABLE `orders` -DROP TABLE `registrations` +DROP TABLE `authz2`; +DROP TABLE `blockedKeys`; +DROP TABLE `certificateStatus`; +DROP TABLE `certificatesPerName`; +DROP TABLE `crls`; +DROP TABLE `fqdnSets`; +DROP TABLE `issuedNames`; +DROP TABLE `keyHashToSerial`; +DROP TABLE `newOrdersRL`; +DROP TABLE `orderToAuthz2`; +DROP TABLE `orders`; +DROP TABLE `registrations`; diff --git a/sa/_db/migrations/20210308140000_SimplePartitioning.sql b/sa/db/boulder_sa/20210308140000_SimplePartitioning.sql similarity index 98% rename from sa/_db/migrations/20210308140000_SimplePartitioning.sql rename to sa/db/boulder_sa/20210308140000_SimplePartitioning.sql index 3b1d16aac..10aec1c68 100644 --- a/sa/_db/migrations/20210308140000_SimplePartitioning.sql +++ b/sa/db/boulder_sa/20210308140000_SimplePartitioning.sql @@ -1,5 +1,5 @@ --- +goose Up +-- +migrate Up -- SQL in section 'Up' is executed when this migration is applied ALTER TABLE authz2 DROP INDEX IF EXISTS token; @@ -39,7 +39,7 @@ ALTER TABLE precertificates DROP INDEX IF EXISTS serial, ADD INDEX serial (seria ALTER TABLE precertificates PARTITION BY RANGE(id) ( PARTITION p_start VALUES LESS THAN MAXVALUE); --- +goose Down +-- +migrate Down -- SQL section 'Down' is executed when this migration is rolled back ALTER TABLE authz2 REMOVE PARTITIONING; diff --git a/sa/_db/migrations/20220328100000_Incidents.sql b/sa/db/boulder_sa/20220328100000_Incidents.sql similarity index 90% rename from sa/_db/migrations/20220328100000_Incidents.sql rename to sa/db/boulder_sa/20220328100000_Incidents.sql index 619257ca8..d489b43b5 100644 --- a/sa/_db/migrations/20220328100000_Incidents.sql +++ b/sa/db/boulder_sa/20220328100000_Incidents.sql @@ -1,4 +1,4 @@ --- +goose Up +-- +migrate Up -- SQL in section 'Up' is executed when this migration is applied CREATE TABLE `incidents` ( @@ -30,9 +30,9 @@ CREATE TABLE `incident_bar` ( KEY `orderID_idx` (`orderID`) ) CHARSET=utf8mb4; --- +goose Down +-- +migrate Down -- SQL section 'Down' is executed when this migration is rolled back -DROP TABLE `incidents` -DROP TABLE `incident_foo` -DROP TABLE `incident_bar` +DROP TABLE `incidents`; +DROP TABLE `incident_foo`; +DROP TABLE `incident_bar`; diff --git a/sa/db/dbconfig.yml b/sa/db/dbconfig.yml new file mode 100644 index 000000000..0a1bd8201 --- /dev/null +++ b/sa/db/dbconfig.yml @@ -0,0 +1,20 @@ +# https://github.com/rubenv/sql-migrate#readme +boulder_sa_test: + dialect: mysql + datasource: root@tcp(boulder-mysql:3306)/boulder_sa_test?parseTime=true + dir: boulder_sa + +boulder_sa_integration: + dialect: mysql + datasource: root@tcp(boulder-mysql:3306)/boulder_sa_integration?parseTime=true + dir: boulder_sa + +incidents_sa_test: + dialect: mysql + datasource: root@tcp(boulder-mysql:3306)/incidents_sa_test?parseTime=true + dir: incidents_sa + +incidents_sa_integration: + dialect: mysql + datasource: root@tcp(boulder-mysql:3306)/incidents_sa_integration?parseTime=true + dir: incidents_sa diff --git a/sa/db/incidents_sa/20220328100000_Incidents.sql b/sa/db/incidents_sa/20220328100000_Incidents.sql new file mode 100644 index 000000000..dec39f18e --- /dev/null +++ b/sa/db/incidents_sa/20220328100000_Incidents.sql @@ -0,0 +1,28 @@ +-- +migrate Up +-- SQL in section 'Up' is executed when this migration is applied + +CREATE TABLE `incident_foo` ( + `serial` varchar(255) NOT NULL, + `registrationID` bigint(20) unsigned NULL, + `orderID` bigint(20) unsigned NULL, + `lastNoticeSent` datetime NULL, + PRIMARY KEY (`serial`), + KEY `registrationID_idx` (`registrationID`), + KEY `orderID_idx` (`orderID`) +) CHARSET=utf8mb4; + +CREATE TABLE `incident_bar` ( + `serial` varchar(255) NOT NULL, + `registrationID` bigint(20) unsigned NULL, + `orderID` bigint(20) unsigned NULL, + `lastNoticeSent` datetime NULL, + PRIMARY KEY (`serial`), + KEY `registrationID_idx` (`registrationID`), + KEY `orderID_idx` (`orderID`) +) CHARSET=utf8mb4; + +-- +migrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE `incident_foo`; +DROP TABLE `incident_bar`; diff --git a/sa/migrations.sh b/sa/migrations.sh index 4f2b4e2bc..f849934e0 100755 --- a/sa/migrations.sh +++ b/sa/migrations.sh @@ -13,11 +13,12 @@ res="${esc}0m" # # Defaults # -DB_NEXT_PATH="_db-next/migrations" -DB_PATH="_db/migrations" +DB_NEXT_PATH="db-next" +DB_PATH="db" OUTCOME="ERROR" PROMOTE=() RUN=() +DB="" # # Print Functions @@ -63,7 +64,14 @@ function print_linking () { echo -e "to: ${esc}0;39;1m${to}${res}" } -function print_migrations(){ +function check_arg() { + if [ -z "${OPTARG}" ] + then + exit_msg "No arg for --${OPT} option, use: -h for help">&2 + fi +} + +function print_migrations() { iter=1 for file in "${migrations[@]}" do @@ -83,27 +91,29 @@ function exit_msg() { # function get_promotable_migrations() { local migrations=() - for file in "${DB_NEXT_PATH}"/*.sql; do + local migpath="${DB_NEXT_PATH}/${1}" + for file in "${migpath}"/*.sql; do [[ -f "${file}" && ! -L "${file}" ]] || continue migrations+=("${file}") done if [[ "${migrations[@]}" ]]; then echo "${migrations[@]}" else - exit_msg "There are no promotable migrations at path: "\"${DB_NEXT_PATH}\""" + exit_msg "There are no promotable migrations at path: "\"${migpath}\""" fi } function get_demotable_migrations() { local migrations=() - for file in "${DB_NEXT_PATH}"/*.sql; do + local migpath="${DB_NEXT_PATH}/${1}" + for file in "${migpath}"/*.sql; do [[ -L "${file}" ]] || continue migrations+=("${file}") done if [[ "${migrations[@]}" ]]; then echo "${migrations[@]}" else - exit_msg "There are no demotable migrations at path: "\"${DB_NEXT_PATH}\""" + exit_msg "There are no demotable migrations at path: "\"${migpath}\""" fi } @@ -116,26 +126,27 @@ Usage: Boulder DB Migrations CLI - Helper for listing, promoting, and demoting Boulder schema files + Helper for listing, promoting, and demoting migration files ./$(basename "${0}") [OPTION]... - - -l, --list-next Lists schema files present in sa/_db-next - -c, --list-current Lists schema files promoted from sa/_db-next to sa/_db - -p, --promote Select and promote a schema from sa/_db-next to sa/_db - -d, --demote Select and demote a schema from sa/_db to sa/_db-next + -b --db Name of the database, this is required (e.g. boulder_sa or incidents_sa) + -n, --list-next Lists migration files present in sa/db-next/ + -c, --list-current Lists migration files promoted from sa/db-next/ to sa/db/ + -p, --promote Select and promote a migration from sa/db-next/ to sa/db/ + -d, --demote Select and demote a migration from sa/db/ to sa/db-next/ -h, --help Shows this help message EOM )" -while getopts nchpd-: OPT; do +while getopts nchpd-:b:-: OPT; do if [ "$OPT" = - ]; then # long option: reformulate OPT and OPTARG OPT="${OPTARG%%=*}" # extract long option name OPTARG="${OPTARG#$OPT}" # extract long option argument (may be empty) OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=` fi case "${OPT}" in + b | db ) check_arg; DB="${OPTARG}" ;; n | list-next ) RUN+=("list_next") ;; c | list-current ) RUN+=("list_current") ;; p | promote ) RUN+=("promote") ;; @@ -150,24 +161,26 @@ shift $((OPTIND-1)) # remove parsed opts and args from $@ list # On EXIT, trap and print outcome trap "print_outcome" EXIT +[ -z "${DB}" ] && exit_msg "You must specify a database with flag -b \"foo\" or --db=\"foo\"" + STEP="list_next" if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then - print_heading "Next Schemas" - migrations=($(get_promotable_migrations)) + print_heading "Next Migrations" + migrations=($(get_promotable_migrations "${DB}")) print_migrations "${migrations[@]}" fi STEP="list_current" if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then - print_heading "Current Schemas" - migrations=($(get_demotable_migrations)) + print_heading "Current Migrations" + migrations=($(get_demotable_migrations "${DB}")) print_migrations "${migrations[@]}" fi STEP="promote" if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then - print_heading "Promote Schema" - migrations=($(get_promotable_migrations)) + print_heading "Promote Migration" + migrations=($(get_promotable_migrations "${DB}")) declare -a mig_index=() declare -A mig_file=() for i in "${!migrations[@]}"; do @@ -176,7 +189,7 @@ if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then done promote="" - PS3='Which schema would you like to promote? (q to cancel): ' + PS3='Which migration would you like to promote? (q to cancel): ' select opt in "${mig_index[@]}"; do case "${opt}" in @@ -186,23 +199,23 @@ if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then done if [[ "${mig_file_path}" ]] then - print_heading "Promoting Schema" + print_heading "Promoting Migration" promote_mig_name="$(basename -- "${mig_file_path}")" - promoted_mig_file_path="${DB_PATH}/${promote_mig_name}" - symlink_relpath="$(realpath --relative-to=${DB_NEXT_PATH} ${promoted_mig_file_path})" + promoted_mig_file_path="${DB_PATH}/${DB}/${promote_mig_name}" + symlink_relpath="$(realpath --relative-to=${DB_NEXT_PATH}/${DB} ${promoted_mig_file_path})" print_moving "${mig_file_path}" "${promoted_mig_file_path}" mv "${mig_file_path}" "${promoted_mig_file_path}" print_linking "${mig_file_path}" "${symlink_relpath}" - ln -s "${symlink_relpath}" "${DB_NEXT_PATH}" + ln -s "${symlink_relpath}" "${DB_NEXT_PATH}/${DB}" fi fi STEP="demote" if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then - print_heading "Demote Schema" - migrations=($(get_demotable_migrations)) + print_heading "Demote Migration" + migrations=($(get_demotable_migrations "${DB}")) declare -a mig_index=() declare -A mig_file=() for i in "${!migrations[@]}"; do @@ -211,7 +224,7 @@ if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then done demote_mig="" - PS3='Which schema would you like to demote? (q to cancel): ' + PS3='Which migration would you like to demote? (q to cancel): ' select opt in "${mig_index[@]}"; do case "${opt}" in @@ -221,9 +234,9 @@ if [[ "${RUN[@]}" =~ "${STEP}" ]] ; then done if [[ "${mig_link_path}" ]] then - print_heading "Demoting Schema" + print_heading "Demoting Migration" demote_mig_name="$(basename -- "${mig_link_path}")" - demote_mig_from="${DB_PATH}/${demote_mig_name}" + demote_mig_from="${DB_PATH}/${DB}/${demote_mig_name}" print_unlinking "${mig_link_path}" rm "${mig_link_path}" diff --git a/test/boulder-tools/build.sh b/test/boulder-tools/build.sh index 3138a1f6c..affdc1059 100755 --- a/test/boulder-tools/build.sh +++ b/test/boulder-tools/build.sh @@ -42,7 +42,7 @@ export GOBIN=/usr/local/bin GOCACHE=/tmp/gocache # and vice versa. go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 -go install bitbucket.org/liamstask/goose/cmd/goose@latest +go install github.com/rubenv/sql-migrate/...@v1.1.2 go install golang.org/x/tools/cmd/stringer@latest go install github.com/letsencrypt/pebble/cmd/pebble-challtestsrv@master go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.47.1 diff --git a/test/create_db.sh b/test/create_db.sh index fdbdc06cb..919b34d70 100755 --- a/test/create_db.sh +++ b/test/create_db.sh @@ -1,10 +1,20 @@ -#!/bin/bash +#!/usr/bin/env bash set -o errexit cd $(dirname $0)/.. -DBENVS="test + +# If you modify DBS or ENVS, you must also modify the corresponding keys in +# sa/db/dbconfig.yml, see: https://github.com/rubenv/sql-migrate#readme + +DBS="boulder_sa +incidents_sa" + +ENVS="test integration" +# /path/to/boulder/repo +root_dir=$(dirname $(dirname $(readlink -f "$0"))) + # posix compliant escape sequence esc=$'\033'"[" res="${esc}0m" @@ -16,134 +26,82 @@ function print_heading() { } function exit_err() { - if [ ! -z "$1" ]; then + if [ ! -z "$1" ] + then echo $1 > /dev/stderr fi exit 1 } -function exit_msg() { - # complain to STDERR and exit with error - echo "${*}" >&2 - exit 2 -} - -function get_migrations() { - local db_schemas_path="${1}" - local migrations=() - for file in "${db_schemas_path}"/*.sql; do - [[ -f "${file}" ]] || continue - migrations+=("${file}") - done - if [[ "${migrations[@]}" ]]; then - echo "${migrations[@]}" - else - exit_msg "There are no migrations at path: "\"${db_schemas_path}\""" - fi -} - function create_empty_db() { local db="${1}" local dbconn="${2}" create_script="drop database if exists \`${db}\`; create database if not exists \`${db}\`;" mysql ${dbconn} -e "${create_script}" || exit_err "unable to create ${db}" - echo "created empty "$db" database" -} - -function apply_migrations() { - local migrations="${1}" - local dbpath="${2}" - local dbenv="${3}" - local db="${4}" - if [[ "${migrations[@]}" ]] - then - echo "applying migrations from ${db_mig_path}" - goose -path="${dbpath}" -env="${dbenv}" up || exit_err "unable to migrate ${db} with ${dbpath}" - else - echo "no migrations at ${dbpath}" - fi } # set db connection for if running in a separate container or not dbconn="-u root" -if [[ $MYSQL_CONTAINER ]]; then +if [[ $MYSQL_CONTAINER ]] +then dbconn="-u root -h boulder-mysql --port 3306" fi # MariaDB sets the default binlog_format to STATEMENT, # which causes warnings that fail tests. Instead set it # to the format we use in production, MIXED. -mysql $dbconn -e "SET GLOBAL binlog_format = 'MIXED';" +mysql ${dbconn} -e "SET GLOBAL binlog_format = 'MIXED';" # MariaDB sets the default @@max_connections value to 100. The SA alone is # configured to use up to 100 connections. We increase the max connections here # to give headroom for other components (ocsp-updater for example). -mysql $dbconn -e "SET GLOBAL max_connections = 500;" +mysql ${dbconn} -e "SET GLOBAL max_connections = 500;" -for dbenv in $DBENVS; do - db="boulder_sa_${dbenv}" - print_heading "Checking if ${db} exists" - if mysql ${dbconn} -e 'show databases;' | grep "${db}" > /dev/null; then - echo "${db} already exists - skipping create" - else - echo "${db} doesn't exist - creating" - create_empty_db "${db}" "${dbconn}" - fi +for db in $DBS; do + for env in $ENVS; do + dbname="${db}_${env}" + print_heading "${dbname}" + if mysql ${dbconn} -e 'show databases;' | grep "${dbname}" > /dev/null; then + echo "Already exists - skipping create" + else + echo "Doesn't exist - creating" + create_empty_db "${dbname}" "${dbconn}" + fi - # Determine which $dbpath and $db_mig_path to use. - if [[ "${BOULDER_CONFIG_DIR}" == "test/config-next" ]] - then - dbpath="./sa/_db-next" - else - dbpath="./sa/_db" - fi - db_mig_path="${dbpath}/migrations" + if [[ "${BOULDER_CONFIG_DIR}" == "test/config-next" ]] + then + dbpath="./sa/db-next" + else + dbpath="./sa/db" + fi - # Populate an array with schema files present at $dbpath. - migrations=($(get_migrations "${db_mig_path}")) + # sql-migrate will default to ./dbconfig.yml and treat all configured dirs + # as relative. + cd "${dbpath}" + r=`sql-migrate up -env="${dbname}" | xargs echo` + if [[ "${r}" == "Migration failed"* ]] + then + echo "Migration failed - dropping and recreating" + create_empty_db "${dbname}" "${dbconn}" + sql-migrate up -env="${dbname}" || exit_err "Migration failed after dropping and recreating" + else + echo "${r}" + fi - # Goose up, this will work if there are schema files present at - # $dbpath with a newer timestamp than the current goose dbversion. - apply_migrations "${migrations}" "${dbpath}" "${dbenv}" "${db}" - - # The (actual) latest migration should always be the last file or - # symlink at $db_mig_path. - latest_mig_path_filename="$(basename -- "${migrations[-1]}")" - - # Goose's dbversion is the timestamp (first 14 characters) of the file - # that it last migrated to. We can figure out which goose dbversion we - # should be on by parsing the timestamp of the latest file at - # $db_mig_path. - latest_db_mig_version="${latest_mig_path_filename:0:14}" - - # Ask Goose the timestamp (dbversion) our database is currently - # migrated to. - goose_dbversion="$(goose -path=${dbpath} -env=${dbenv} dbversion | sed 's/goose: dbversion //')" - - # If the $goose_dbversion does not match the $latest_in_db_mig_path, - # trigger recreate - if [[ "${latest_db_mig_version}" != "${goose_dbversion}" ]]; then - print_heading "Detected latest migration version mismatch" - echo "dropping and recreating from migrations at ${db_mig_path}" - create_empty_db "${db}" "${dbconn}" - apply_migrations "${migrations}" "${dbpath}" "${dbenv}" "${db}" - fi - - # With MYSQL_CONTAINER, patch the GRANT statements to - # use 127.0.0.1, not localhost, as MySQL may interpret - # 'username'@'localhost' to mean only users for UNIX - # socket connections. Use '-f' to ignore errors while - # we have migrations that haven't been applied but - # add new tables (TODO(#2931): remove -f). - USERS_SQL=test/sa_db_users.sql - if [[ ${MYSQL_CONTAINER} ]]; then - sed -e "s/'localhost'/'%'/g" < ${USERS_SQL} | \ - mysql $dbconn -D $db -f || exit_err "unable to add users to ${db}" - else - sed -e "s/'localhost'/'127.%'/g" < $USERS_SQL | \ - mysql $dbconn -D $db -f < $USERS_SQL || exit_err "unable to add users to ${db}" - fi - echo "added users to ${db}" + USERS_SQL="../db-users/${db}.sql" + if [[ ${MYSQL_CONTAINER} ]] + then + sed -e "s/'localhost'/'%'/g" < ${USERS_SQL} | \ + mysql ${dbconn} -D "${dbname}" -f || exit_err "Unable to add users from ${USERS_SQL}" + else + sed -e "s/'localhost'/'127.%'/g" < $USERS_SQL | \ + mysql ${dbconn} -D "${dbname}" -f < $USERS_SQL || exit_err "Unable to add users from ${USERS_SQL}" + fi + echo "Added users from ${USERS_SQL}" + + # return to the root directory + cd "${root_dir}" + done done echo