From 3768297b23c8a9b4027f8cc88e8529c04b83f06d Mon Sep 17 00:00:00 2001 From: fuxinji9527 <1992666531@qq.com> Date: Fri, 1 Nov 2024 16:15:28 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E6=9C=80=E6=96=B0discourse?= =?UTF-8?q?=5Fdocker=E4=BB=A3=E7=A0=81=E4=BB=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- discourse-doctor | 17 +-- discourse-setup | 108 +++++++++++----- image/auto_build.rb | 112 ++++++++++++++-- image/base/install-imagemagick | 22 ++-- image/base/install-jemalloc | 53 +++++--- image/base/install-nginx | 11 +- image/base/install-oxipng | 33 ++--- image/base/install-redis | 7 +- image/base/install-ruby | 16 --- image/base/install-rust | 2 +- image/base/release.Dockerfile | 15 --- image/base/slim.Dockerfile | 121 ------------------ image/discourse_bench/Dockerfile | 2 +- image/discourse_bench/run_bench.sh | 2 +- image/discourse_dev/Dockerfile | 48 +++++-- image/discourse_dev/postgres_dev.template.yml | 31 +---- image/discourse_fast_switch/create_switch.rb | 13 +- image/discourse_test/Dockerfile | 17 +-- launcher | 8 +- samples/mail-receiver.yml | 10 +- samples/standalone.yml | 7 +- samples/web_only.yml | 7 +- scripts/mk_swapfile | 51 -------- templates/import/mbox.template.yml | 2 +- templates/import/mssql-dep.template.yml | 2 +- templates/import/mysql-dep.template.yml | 2 +- templates/import/phpbb3.template.yml | 2 +- templates/import/vanilla.template.yml | 30 +++-- templates/postgres.13.template.yml | 3 +- templates/postgres.template.yml | 116 ++++++++++------- templates/redis.template.yml | 16 ++- templates/web.china.template.yml | 18 ++- templates/web.ssl.template.yml | 8 +- templates/web.template.yml | 61 ++++++--- 34 files changed, 502 insertions(+), 471 deletions(-) delete mode 100755 image/base/install-ruby delete mode 100644 image/base/release.Dockerfile delete mode 100644 image/base/slim.Dockerfile delete mode 100755 scripts/mk_swapfile diff --git a/discourse-doctor b/discourse-doctor index 0c7ea18..9f89236 100755 --- a/discourse-doctor +++ b/discourse-doctor @@ -209,7 +209,7 @@ dump_yaml() { ## read a variable from the config file and stick it in read_config_result ## read_config() { - config_line=$(egrep "^ #?$1:" $web_file) + config_line=$(grep -E "^ #?$1:" $web_file) read_config_result=$(echo $config_line | awk -F ":" '{print $2}') read_config_result=$(echo $read_config_result | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") } @@ -314,18 +314,19 @@ print_done() { log "==================== DONE! ====================" DOCTOR_FILE=$(date +%s | sha256sum | base64 | head -c 20).txt - if [ $app_name == 'app' ] && [ "$NO_CONTAINER" != 'y' ] - then - read -p "Would you like to serve a publicly available version of this file? (Y/n)" serve - if [ $serve == 'Y' ] - then + if [ $app_name == 'app' ] && [ "$NO_CONTAINER" != 'y' ]; then + read -p "Would you like to serve a publicly available version of this file? (Y/n) " serve + case "${serve:-Y}" in + y*|Y*) cp $LOG_FILE shared/standalone/log/var-log/$DOCTOR_FILE sudo docker exec -w /var/www/discourse -i $app_name cp /var/log/$DOCTOR_FILE public log "The output of this program may be available at http://$DISCOURSE_HOSTNAME/$DOCTOR_FILE" log "You should inspect that file carefully before sharing the URL." - else + ;; + *) log "Publicly available log not generated." - fi + ;; + esac fi # The following is not in the web log file since it was copied above, which seems correct log diff --git a/discourse-setup b/discourse-setup index 0daafcb..209c622 100755 --- a/discourse-setup +++ b/discourse-setup @@ -257,7 +257,8 @@ scale_ram_and_cpu() { avail_cores=`sysctl hw.ncpu | awk '/hw.ncpu:/ {print $2}'` else avail_gb=$(check_linux_memory) - avail_cores=$((`awk '/cpu cores/ {print $4;exit}' /proc/cpuinfo`*`sort /proc/cpuinfo | uniq | grep -c "physical id"`)) + threads_per_core=$(lscpu | awk 'BEGIN {FS=":"} /Thread\(s\) per core/ {print $2}') + avail_cores=$((`lscpu | awk '/^CPU\(s\)/ {print $2}'`*${threads_per_core})) fi echo "Found ${avail_gb}GB of memory and $avail_cores physical CPU cores" @@ -343,32 +344,40 @@ check_port() { ## read a variable from the config file ## read_config() { - config_line=`egrep "^ #?$1:" $web_file` + config_line=`grep -E "^ #?$1:" $web_file` read_config_result=`echo $config_line | awk -F":" '{print $2}'` read_config_result=`echo $read_config_result | sed "s/^\([\"']\)\(.*\)\1\$/\2/g"` } read_default() { - config_line=`egrep "^ #?$1:" samples/standalone.yml` + config_line=`grep -E "^ #?$1:" samples/standalone.yml` read_default_result=`echo $config_line | awk -F":" '{print $2}'` read_default_result=`echo $read_config_result | sed "s/^\([\"']\)\(.*\)\1\$/\2/g"` } -assert_maxmind_license_key() { +assert_maxmind_envs() { if ! grep DISCOURSE_MAXMIND_LICENSE_KEY $web_file >/dev/null 2>&1 then - echo "Adding MAXMIND placeholder to $web_file" + echo "Adding MAXMIND_LICENSE_KEY placeholder to $web_file" sed -i '/^.*LETSENCRYPT_ACCOUNT_EMAIL.*/a \ \ #DISCOURSE_MAXMIND_LICENSE_KEY: 1234567890123456' $web_file fi - if ! grep DISCOURSE_MAXMIND_LICENSE_KEY $web_file >/dev/null 2>&1 + + if ! grep DISCOURSE_MAXMIND_ACCOUNT_ID $web_file >/dev/null 2>&1 + then + echo "Adding MAXMIND_ACCOUNT_ID placeholder to $web_file" + sed -i '/^.*LETSENCRYPT_ACCOUNT_EMAIL.*/a \ \ #DISCOURSE_MAXMIND_ACCOUNT_ID: 123456' $web_file + fi + + + if ! grep -e DISCOURSE_MAXMIND_LICENSE_KEY -e DISCOURSE_MAXMIND_ACCOUNT_ID $web_file >/dev/null 2>&1 then cat < discourse_dev/postgres.template.yml") + run( + "sed -e 's/\(db_name: discourse\)/\1_development/' ../templates/postgres.template.yml > discourse_dev/postgres.template.yml", + ) run("cp ../templates/redis.template.yml discourse_dev/redis.template.yml") + run("cp base/install-rust discourse_dev/install-rust") end -if ARGV.length != 1 +if ARGV.length == 0 puts <<~TEXT Usage: ruby auto_build.rb IMAGE @@ -57,7 +143,7 @@ else end puts "Building #{images[image]}" - dev_deps() if image == :discourse_dev + dev_deps() if image == :discourse_dev_amd64 || image == :discourse_dev_arm64 - build(images[image]) + build(images[image], ARGV[1..-1].join(" ")) end diff --git a/image/base/install-imagemagick b/image/base/install-imagemagick index 911e719..44dbea4 100755 --- a/image/base/install-imagemagick +++ b/image/base/install-imagemagick @@ -9,7 +9,7 @@ IMAGE_MAGICK_HASH="d282117bc6d0e91ad1ad685d096623b96ed8e229f911c891d83277b350ef8 LIBJPEGTURBO=$(cat /etc/issue | grep -qi Debian && echo 'libjpeg62-turbo libjpeg62-turbo-dev' || echo 'libjpeg-turbo8 libjpeg-turbo8-dev') # Ubuntu 22.04/22.10 doesn't have libwebp6 -LIBWEBP=$(cat /etc/issue | grep -qi 'Ubuntu 22' && echo 'libwebp7' || echo 'libwebp6') +LIBWEBP=$(cat /etc/issue | grep -qiE 'Debian GNU/Linux 12|Ubuntu 22' && echo 'libwebp7' || echo 'libwebp6') PREFIX=/usr/local WDIR=/tmp/imagemagick @@ -17,17 +17,17 @@ WDIR=/tmp/imagemagick # Install build deps apt -y -q remove imagemagick apt -y -q install git make gcc pkg-config autoconf curl g++ yasm cmake \ - libde265-0 libde265-dev ${LIBJPEGTURBO} x265 libx265-dev libtool \ - libpng16-16 libpng-dev ${LIBJPEGTURBO} ${LIBWEBP} libwebp-dev libgomp1 \ + libde265-0 libde265-dev ${LIBJPEGTURBO} ${LIBWEBP} x265 libx265-dev libtool \ + libpng16-16 libpng-dev libwebp-dev libgomp1 \ libwebpmux3 libwebpdemux2 ghostscript libxml2-dev libxml2-utils librsvg2-dev \ - libltdl7-dev libbz2-dev gsfonts libtiff-dev libfreetype6-dev libjpeg-dev + libltdl7-dev libbz2-dev gsfonts libtiff-dev libfreetype6-dev libjpeg-dev libheif1 libheif-dev -# Ubuntu doesn't like `bullseye-backports` -if cat /etc/issue | grep -qi 'Ubuntu 22'; then - apt -y install libheif1 libaom-dev libheif-dev +# Ubuntu doesn't like backports +if cat /etc/issue | grep -qiE 'Debian GNU/Linux 12|Ubuntu 22'; then + apt -y install libaom-dev else # Use backports instead of compiling it - apt -y -q install -t bullseye-backports libheif1 libaom-dev libheif-dev + apt -y -q install -t bullseye-backports libaom-dev fi mkdir -p $WDIR @@ -40,7 +40,7 @@ echo "$IMAGE_MAGICK_HASH $WDIR/ImageMagick.tar.gz" | sha256sum -c IMDIR=$WDIR/$(tar tzf $WDIR/ImageMagick.tar.gz --wildcards "ImageMagick-*/configure" |cut -d/ -f1) tar zxf $WDIR/ImageMagick.tar.gz -C $WDIR cd $IMDIR -PKG_CONF_LIBDIR=$PREFIX/lib LDFLAGS=-L$PREFIX/lib CFLAGS=-I$PREFIX/include ./configure \ +PKG_CONF_LIBDIR=$PREFIX/lib LDFLAGS=-L$PREFIX/lib CFLAGS='-O2 -I$PREFIX/include' ./configure \ --prefix=$PREFIX \ --enable-static \ --enable-bounds-checking \ @@ -62,11 +62,11 @@ PKG_CONF_LIBDIR=$PREFIX/lib LDFLAGS=-L$PREFIX/lib CFLAGS=-I$PREFIX/include ./con --with-heic \ --with-rsvg \ --with-webp -make all && make install +make all -j"$(nproc)" && make install cd $HOME rm -rf $WDIR ldconfig /usr/local/lib # Validate ImageMagick install -test $(convert -version | grep -o -e png -e tiff -e jpeg -e freetype -e heic -e webp | wc -l) -eq 6 +test $(magick -version | grep -o -e png -e tiff -e jpeg -e freetype -e heic -e webp | wc -l) -eq 6 diff --git a/image/base/install-jemalloc b/image/base/install-jemalloc index b075978..311c969 100755 --- a/image/base/install-jemalloc +++ b/image/base/install-jemalloc @@ -3,24 +3,41 @@ set -e # version check: https://github.com/jemalloc/jemalloc/releases -# jemalloc stable -mkdir /jemalloc-stable -cd /jemalloc-stable +# Newer aarch64 platforms, like Raspberry Pi 5 with Debian Bookworm, are +# shipping with PAGESIZE=16K. Setting it here is retrocompatible with older +# systems, so it's safe to set it unconditionally for arm. +# This means aarch64 will use the latest jemalloc, where we can configure the +# page size, while x64 will keep using our pinned 3.6.0 jemalloc +if uname -m | grep -qi 'aarch64'; then + mkdir /jemalloc-new + cd /jemalloc-new -wget -q https://github.com/jemalloc/jemalloc/releases/download/3.6.0/jemalloc-3.6.0.tar.bz2 -sha256sum jemalloc-3.6.0.tar.bz2 -echo "e16c2159dd3c81ca2dc3b5c9ef0d43e1f2f45b04548f42db12e7c12d7bdf84fe jemalloc-3.6.0.tar.bz2" | sha256sum -c -tar --strip-components=1 -xjf jemalloc-3.6.0.tar.bz2 -./configure --prefix=/usr && make && make install -cd / && rm -rf /jemalloc-stable + wget -q https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 + sha256sum jemalloc-5.3.0.tar.bz2 + echo "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa jemalloc-5.3.0.tar.bz2" | sha256sum -c + tar --strip-components=1 -xjf jemalloc-5.3.0.tar.bz2 + ./configure --prefix=/usr --with-lg-page=16 && make build_lib -j"$(nproc)" && make install_lib + cd / && rm -rf /jemalloc-new +else + # jemalloc stable + mkdir /jemalloc-stable + cd /jemalloc-stable -# jemalloc new -mkdir /jemalloc-new -cd /jemalloc-new + wget -q https://github.com/jemalloc/jemalloc/releases/download/3.6.0/jemalloc-3.6.0.tar.bz2 + sha256sum jemalloc-3.6.0.tar.bz2 + echo "e16c2159dd3c81ca2dc3b5c9ef0d43e1f2f45b04548f42db12e7c12d7bdf84fe jemalloc-3.6.0.tar.bz2" | sha256sum -c + tar --strip-components=1 -xjf jemalloc-3.6.0.tar.bz2 + ./configure --prefix=/usr $EXTRA_CONF && make -j"$(nproc)" && make install + cd / && rm -rf /jemalloc-stable -wget -q https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 -sha256sum jemalloc-5.3.0.tar.bz2 -echo "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa jemalloc-5.3.0.tar.bz2" | sha256sum -c -tar --strip-components=1 -xjf jemalloc-5.3.0.tar.bz2 -./configure --prefix=/usr --with-install-suffix=5.3.0 && make build_lib && make install_lib -cd / && rm -rf /jemalloc-new + # jemalloc new + mkdir /jemalloc-new + cd /jemalloc-new + + wget -q https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 + sha256sum jemalloc-5.3.0.tar.bz2 + echo "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa jemalloc-5.3.0.tar.bz2" | sha256sum -c + tar --strip-components=1 -xjf jemalloc-5.3.0.tar.bz2 + ./configure --prefix=/usr --with-install-suffix=5.3.0 && make build_lib -j"$(nproc)" && make install_lib + cd / && rm -rf /jemalloc-new +fi diff --git a/image/base/install-nginx b/image/base/install-nginx index aa24338..7abe2b6 100755 --- a/image/base/install-nginx +++ b/image/base/install-nginx @@ -2,13 +2,12 @@ set -e # version check: https://nginx.org/en/download.html -VERSION=1.23.3 -HASH="75cb5787dbb9fae18b14810f91cc4343f64ce4c24e27302136fb52498042ba54" +VERSION=1.26.1 cd /tmp wget -q https://nginx.org/download/nginx-$VERSION.tar.gz -sha256sum nginx-$VERSION.tar.gz -echo "$HASH nginx-$VERSION.tar.gz" | sha256sum -c +wget -q https://nginx.org/download/nginx-$VERSION.tar.gz.asc +gpg --verify nginx-$VERSION.tar.gz.asc nginx-$VERSION.tar.gz tar zxf nginx-$VERSION.tar.gz cd nginx-$VERSION @@ -31,7 +30,9 @@ make install mv /usr/share/nginx/sbin/nginx /usr/sbin cd / -rm -fr /tmp/nginx +rm -fr /tmp/nginx-$VERSION +rm -f /tmp/nginx-$VERSION.tar.gz +rm -f /tmp/nginx-$VERSION.tar.gz.asc rm -fr /tmp/libbrotli rm -fr /tmp/ngx_brotli rm -fr /etc/nginx/modules-enabled/* diff --git a/image/base/install-oxipng b/image/base/install-oxipng index a6933ab..63cad97 100755 --- a/image/base/install-oxipng +++ b/image/base/install-oxipng @@ -2,23 +2,24 @@ set -e # version check: https://github.com/shssoichiro/oxipng/releases -OXIPNG_VERSION="8.0.0" -OXIPNG_HASH="ef96d6340e70900de0a38ace8f5f20878f6c256b18b0c59cd87f2b515437b87b" -OXIPNG_ARCHIVE="v${OXIPNG_VERSION}.tar.gz" -OXIPNG_DIR="oxipng-${OXIPNG_VERSION}" +OXIPNG_VERSION="9.1.2" +dpkgArch="$(dpkg --print-architecture)" + +case "${dpkgArch##*-}" in + amd64) OXIPNG_FILE="oxipng-${OXIPNG_VERSION}-x86_64-unknown-linux-musl.tar.gz"; OXIPNG_HASH='211d53f3781be4a71566fbaad6611a3da018ac9b22d500651b091c2b42ebe318' ;; + arm64) OXIPNG_FILE="oxipng-${OXIPNG_VERSION}-aarch64-unknown-linux-musl.tar.gz"; OXIPNG_HASH='818d47d7195e1e0c4d58a9f3b6fd84aa3cd21770c60c876e73e2e6a17ca69b52' ;; + *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; +esac # Install other deps -apt-get -y install advancecomp jhead jpegoptim libjpeg-turbo-progs optipng +apt -y -q install advancecomp jhead jpegoptim libjpeg-turbo-progs optipng -cd /tmp -wget -q https://github.com/shssoichiro/oxipng/archive/refs/tags/${OXIPNG_ARCHIVE} -sha256sum ${OXIPNG_ARCHIVE} -echo "${OXIPNG_HASH} ${OXIPNG_ARCHIVE}" | sha256sum -c +mkdir /oxipng-install +cd /oxipng-install -tar -zxf ${OXIPNG_ARCHIVE} -cd ${OXIPNG_DIR} - -CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse /usr/local/cargo/bin/cargo build --release -cp target/release/oxipng /usr/local/bin - -cd / && rm -fr /tmp/${OXIPNG_DIR} +wget -q https://github.com/shssoichiro/oxipng/releases/download/v${OXIPNG_VERSION}/${OXIPNG_FILE} +sha256sum ${OXIPNG_FILE} +echo "${OXIPNG_HASH} ${OXIPNG_FILE}" | sha256sum -c +tar --strip-components=1 -xzf $OXIPNG_FILE +cp -v ./oxipng /usr/local/bin +cd / && rm -fr /oxipng-install \ No newline at end of file diff --git a/image/base/install-redis b/image/base/install-redis index 690b264..4a0d6fd 100755 --- a/image/base/install-redis +++ b/image/base/install-redis @@ -14,8 +14,13 @@ echo "$REDIS_HASH redis-$REDIS_VERSION.tar.gz" | sha256sum -c tar zxf redis-$REDIS_VERSION.tar.gz cd redis-$REDIS_VERSION +# aarch64 compatibility +if uname -m | grep -qi 'aarch64'; then + export JEMALLOC_CONFIGURE_OPTS="--with-lg-page=16" +fi + # Building and installing binaries. -make BUILD_TLS=yes && make install PREFIX=/usr +make -j"$(nproc)" BUILD_TLS=yes && make install PREFIX=/usr # Add `redis` user and group. adduser --system --home /var/lib/redis --quiet --group redis || true diff --git a/image/base/install-ruby b/image/base/install-ruby deleted file mode 100755 index 4be5a5f..0000000 --- a/image/base/install-ruby +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -RUBY_VERSION="3.2.2" -export CONFIGURE_OPTS="--enable-yjit" - -apt-get -y install --no-install-recommends ruby bison libffi-dev - -mkdir /src -git -C /src clone https://github.com/rbenv/ruby-build.git -cd /src/ruby-build && ./install.sh -cd / && rm -fr /src - -ruby-build ${RUBY_VERSION} /usr/local - -apt-get -y purge ruby diff --git a/image/base/install-rust b/image/base/install-rust index b452637..968899a 100755 --- a/image/base/install-rust +++ b/image/base/install-rust @@ -5,7 +5,7 @@ set -e export RUSTUP_HOME=/usr/local/rustup export CARGO_HOME=/usr/local/cargo export PATH=/usr/local/cargo/bin:$PATH -export RUST_VERSION=1.68.0 +export RUST_VERSION=1.75.0 export RUSTUP_VERSION=1.25.2 dpkgArch="$(dpkg --print-architecture)" diff --git a/image/base/release.Dockerfile b/image/base/release.Dockerfile deleted file mode 100644 index 95c0c9c..0000000 --- a/image/base/release.Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -ARG from=discourse/base -ARG tag=build_slim - -FROM $from:$tag - -ENV RAILS_ENV=production - -RUN cd /var/www/discourse &&\ - sudo -u discourse bundle config --local deployment true &&\ - sudo -u discourse bundle config --local path ./vendor/bundle &&\ - sudo -u discourse bundle config --local without test development &&\ - sudo -u discourse bundle install --jobs 4 &&\ - sudo -u discourse yarn install --frozen-lockfile &&\ - sudo -u discourse yarn cache clean &&\ - find /var/www/discourse/vendor/bundle -name tmp -type d -exec rm -rf {} + diff --git a/image/base/slim.Dockerfile b/image/base/slim.Dockerfile deleted file mode 100644 index f36c593..0000000 --- a/image/base/slim.Dockerfile +++ /dev/null @@ -1,121 +0,0 @@ -# NAME: discourse/base -# VERSION: release -FROM debian:bullseye-slim - -ENV PG_MAJOR=13 \ - RUBY_ALLOCATOR=/usr/lib/libjemalloc.so.1 \ - RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH \ - LEFTHOOK=0 - -#LABEL maintainer="Sam Saffron \"https://twitter.com/samsaffron\"" - -RUN echo 2.0.`date +%Y%m%d` > /VERSION - -RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' > /etc/apt/sources.list.d/bullseye-backports.list -RUN echo "debconf debconf/frontend select Teletype" | debconf-set-selections -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install gnupg sudo curl fping -RUN sh -c "fping proxy && echo 'Acquire { Retries \"0\"; HTTP { Proxy \"http://proxy:3128\";}; };' > /etc/apt/apt.conf.d/40proxy && apt-get update || true" -RUN apt-mark hold initscripts -RUN apt-get -y upgrade - -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y locales locales-all -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 - -RUN curl https://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | apt-key add - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" | \ - tee /etc/apt/sources.list.d/postgres.list -RUN curl --silent --location https://deb.nodesource.com/setup_18.x | sudo bash - -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list -RUN apt-get -y update -# install these without recommends to avoid pulling in e.g. -# X11 libraries, mailutils -RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends git rsyslog logrotate cron ssh-client less -RUN DEBIAN_FRONTEND=noninteractive apt-get -y install autoconf build-essential ca-certificates rsync \ - libxslt-dev libcurl4-openssl-dev \ - libssl-dev libyaml-dev libtool \ - libpcre3 libpcre3-dev zlib1g zlib1g-dev \ - libxml2-dev gawk parallel \ - postgresql-${PG_MAJOR} postgresql-client-${PG_MAJOR} \ - postgresql-contrib-${PG_MAJOR} libpq-dev postgresql-${PG_MAJOR}-pgvector \ - libreadline-dev anacron wget \ - psmisc whois brotli libunwind-dev \ - libtcmalloc-minimal4 cmake \ - pngcrush pngquant -RUN sed -i -e 's/start -q anacron/anacron -s/' /etc/cron.d/anacron -RUN sed -i.bak 's/$ModLoad imklog/#$ModLoad imklog/' /etc/rsyslog.conf -RUN sed -i.bak 's/module(load="imklog")/#module(load="imklog")/' /etc/rsyslog.conf -RUN dpkg-divert --local --rename --add /sbin/initctl -RUN sh -c "test -f /sbin/initctl || ln -s /bin/true /sbin/initctl" -RUN cd / &&\ - DEBIAN_FRONTEND=noninteractive apt-get -y install runit socat &&\ - mkdir -p /etc/runit/1.d &&\ - apt-get clean &&\ - rm -f /etc/apt/apt.conf.d/40proxy &&\ - locale-gen en_US &&\ - DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs yarn &&\ - npm install -g terser uglify-js pnpm - -ADD install-imagemagick /tmp/install-imagemagick -RUN /tmp/install-imagemagick - -ADD install-jemalloc /tmp/install-jemalloc -RUN /tmp/install-jemalloc - -ADD install-nginx /tmp/install-nginx -RUN /tmp/install-nginx - -ADD install-redis /tmp/install-redis -RUN /tmp/install-redis - -ADD install-rust /tmp/install-rust -ADD install-ruby /tmp/install-ruby -ADD install-oxipng /tmp/install-oxipng -RUN /tmp/install-rust && /tmp/install-ruby && /tmp/install-oxipng && rustup self uninstall -y - -RUN echo 'gem: --no-document' >> /usr/local/etc/gemrc &&\ - gem update --system - -RUN gem install bundler pups --force &&\ - mkdir -p /pups/bin/ &&\ - ln -s /usr/local/bin/pups /pups/bin/pups - -# This tool allows us to disable huge page support for our current process -# since the flag is preserved through forks and execs it can be used on any -# process -ADD thpoff.c /src/thpoff.c -RUN gcc -o /usr/local/sbin/thpoff /src/thpoff.c && rm /src/thpoff.c - -# clean up for docker squash -RUN rm -fr /usr/share/man &&\ - rm -fr /usr/share/doc &&\ - rm -fr /usr/share/vim/vim74/doc &&\ - rm -fr /usr/share/vim/vim74/lang &&\ - rm -fr /usr/share/vim/vim74/spell/en* &&\ - rm -fr /usr/share/vim/vim74/tutor &&\ - rm -fr /usr/local/share/doc &&\ - rm -fr /usr/local/share/ri &&\ - rm -fr /usr/local/share/ruby-build &&\ - rm -fr /var/lib/apt/lists/* &&\ - rm -fr /root/.gem &&\ - rm -fr /root/.npm &&\ - rm -fr /tmp/* - -# this can probably be done, but I worry that people changing PG locales will have issues -# cd /usr/share/locale && rm -fr `ls -d */ | grep -v en` - -# this is required for aarch64 which uses buildx -# see https://github.com/docker/buildx/issues/150 -RUN rm -f /etc/service - -COPY etc/ /etc -COPY sbin/ /sbin - -# Discourse specific bits -RUN useradd discourse -s /bin/bash -m -U &&\ - install -dm 0755 -o discourse -g discourse /var/www/discourse &&\ - sudo -u discourse git clone --filter=tree:0 https://github.com/discourse/discourse.git /var/www/discourse diff --git a/image/discourse_bench/Dockerfile b/image/discourse_bench/Dockerfile index 5b5d50e..8dd9ef7 100644 --- a/image/discourse_bench/Dockerfile +++ b/image/discourse_bench/Dockerfile @@ -13,7 +13,7 @@ RUN git config --global user.email "you@example.com" &&\ git config --global user.name "Your Name" RUN git pull &&\ - sudo -u discourse bundle install --standalone + sudo -u discourse bundle install --jobs $(($(nproc) - 1)) --standalone RUN gem install facter &&\ gem install mailcatcher diff --git a/image/discourse_bench/run_bench.sh b/image/discourse_bench/run_bench.sh index d1f9c6a..a9a90f6 100644 --- a/image/discourse_bench/run_bench.sh +++ b/image/discourse_bench/run_bench.sh @@ -10,7 +10,7 @@ redis-server /etc/redis/redis.conf git pull # install needed gems -sudo -E -u discourse bundle install +sudo -E -u discourse bundle install --jobs $(($(nproc) - 1)) # start mailcatcher mailcatcher --http-ip 0.0.0.0 diff --git a/image/discourse_dev/Dockerfile b/image/discourse_dev/Dockerfile index 4bbd9e4..57545f3 100644 --- a/image/discourse_dev/Dockerfile +++ b/image/discourse_dev/Dockerfile @@ -1,6 +1,15 @@ # NAME: discourse/discourse_dev # VERSION: release -FROM discourse/base:slim + +ARG from_tag=slim + +FROM discourse/base:$from_tag AS repo-fetcher +WORKDIR /repo +RUN chown discourse . +USER discourse +RUN git clone https://github.com/discourse/discourse . --depth 1 + +FROM discourse/base:$from_tag #LABEL maintainer="Sam Saffron \"https://twitter.com/samsaffron\"" @@ -12,14 +21,20 @@ ADD sudoers.discourse /etc/sudoers.d/discourse RUN sudo -u discourse bundle config set --global path /home/discourse/.bundle/gems +# Add user-install ruby gems to PATH +RUN echo 'PATH="$(ruby -r rubygems -e "puts Gem.user_dir")/bin:$PATH"' >> /home/discourse/.profile + # get redis going ADD redis.template.yml /pups/redis.yml RUN /pups/bin/pups /pups/redis.yml RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 \ + RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH # get postgres going ADD postgres.template.yml /pups/postgres.yml @@ -36,13 +51,28 @@ RUN mv /shared/postgres_data /shared/postgres_data_orig # invocations when used with a mounted volume) ADD ensure-database /etc/runit/1.d/ensure-database -RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - &&\ - echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list &&\ - apt update &&\ - apt install -y google-chrome-stable firefox-esr +ADD install-rust /tmp/install-rust +ADD install-selenium /tmp/install-selenium +RUN /tmp/install-selenium # Install & Configure MailHog (https://github.com/mailhog/MailHog) RUN wget -qO /tmp/mailhog https://github.com/mailhog/MailHog/releases/download/v1.0.1/MailHog_linux_amd64\ && echo "e2ed634ded49929f089b20045581955ed217672078fd86082dd7a6c67c5d09c7 /tmp/mailhog" | sha256sum -c -\ && mv /tmp/mailhog /usr/local/bin/mailhog\ - && chmod +x /usr/local/bin/mailhog + && chmod +x /usr/local/bin/mailhog\ + && rm -rf /tmp/* + +USER discourse + +# Warm global bundle cache, then delete the compressed `cache/` versions (`/gem/` are enough) +RUN --mount=type=bind,src=/repo,from=repo-fetcher,target=/tmp/discourse-clone,readwrite \ + cd /tmp/discourse-clone \ + && bundle install --deployment \ + && rm -rf /home/discourse/.bundle/gems/ruby/*/cache/* + +# Warm global yarn cache +RUN --mount=type=bind,src=/repo,from=repo-fetcher,target=/tmp/discourse-clone,readwrite \ + cd /tmp/discourse-clone \ + && (if [ -f yarn.lock ]; then yarn install; else CI=1 pnpm install; fi) + +USER root diff --git a/image/discourse_dev/postgres_dev.template.yml b/image/discourse_dev/postgres_dev.template.yml index fd257f2..6c84045 100644 --- a/image/discourse_dev/postgres_dev.template.yml +++ b/image/discourse_dev/postgres_dev.template.yml @@ -23,33 +23,4 @@ run: # give db a few secs to start up - exec: "sleep 5" - - exec: su postgres -c 'createdb discourse_development' || true - - exec: su postgres -c 'psql discourse_development -c "grant all privileges on database discourse_development to discourse;"' || true - - exec: su postgres -c 'psql discourse_development -c "alter schema public owner to discourse;"' - - exec: su postgres -c 'psql discourse_development -c "create extension if not exists hstore;"' - - exec: su postgres -c 'psql discourse_development -c "create extension if not exists pg_trgm;"' - - exec: su postgres -c 'psql discourse_development -c "create extension if not exists vector;"' - - - exec: su postgres -c 'createdb discourse_test' || true - - exec: su postgres -c 'psql discourse_test -c "grant all privileges on database discourse_test to discourse;"' || true - - exec: su postgres -c 'psql discourse_test -c "alter schema public owner to discourse;"' - - exec: su postgres -c 'psql discourse_test -c "create extension if not exists hstore;"' - - exec: su postgres -c 'psql discourse_test -c "create extension if not exists pg_trgm;"' - - exec: su postgres -c 'psql discourse_test -c "create extension if not exists vector;"' - - - exec: su postgres -c 'createdb discourse_test_multisite' || true - - exec: su postgres -c 'psql discourse_test_multisite -c "grant all privileges on database discourse_test_multisite to discourse;"' || true - - exec: su postgres -c 'psql discourse_test_multisite -c "alter schema public owner to discourse;"' - - exec: su postgres -c 'psql discourse_test_multisite -c "create extension if not exists hstore;"' - - exec: su postgres -c 'psql discourse_test_multisite -c "create extension if not exists pg_trgm;"' - - exec: su postgres -c 'psql discourse_test_multisite -c "create extension if not exists vector;"' - - - exec: cd tmp && git clone https://github.com/discourse/discourse.git --depth=1 - - exec: chown -R discourse /tmp/discourse - - exec: cd /tmp/discourse && sudo -u discourse bundle config --local path ./vendor/bundle - - exec: cd /tmp/discourse && sudo -u discourse bundle install - - exec: cd /tmp/discourse && sudo -u discourse yarn install - - exec: cd /tmp/discourse && sudo -u discourse yarn cache clean - - exec: cd /tmp/discourse && sudo -u discourse bundle exec rake db:migrate - - exec: cd /tmp/discourse && sudo -u discourse RAILS_ENV=test bundle exec rake db:migrate - - exec: rm -fr /tmp/discourse + - exec: su postgres -c 'psql -c "ALTER USER discourse WITH SUPERUSER;"' diff --git a/image/discourse_fast_switch/create_switch.rb b/image/discourse_fast_switch/create_switch.rb index 24006d3..294fb78 100644 --- a/image/discourse_fast_switch/create_switch.rb +++ b/image/discourse_fast_switch/create_switch.rb @@ -1,13 +1,13 @@ -require 'fileutils' +require "fileutils" -puts "-"*100,"creating switch","-"*100 +puts "-" * 100, "creating switch", "-" * 100 system("cd /var/www/discourse && git pull") -['24', '25'].each do |v| +%w[24 25].each do |v| bin = "/usr/local/bin/use_#{v}" -File.write(bin, <> /etc/apt/sources.list.d/google.list &&\ +ENV TESTEM_DEFAULT_BROWSER Chrome +ADD install-chrome /tmp/install-chrome +RUN /tmp/install-chrome &&\ apt update &&\ - apt install -y libgconf-2-4 libxss1 google-chrome-stable firefox-esr &&\ + apt install -y libgconf-2-4 libxss1 firefox-esr &&\ cd /tmp && wget -q "https://download.mozilla.org/?product=firefox-latest-ssl&os=linux64&lang=en-US" -O firefox.tar.bz2 &&\ tar xjvf firefox.tar.bz2 && mv /tmp/firefox /opt/firefox-evergreen &&\ apt clean @@ -28,11 +29,11 @@ RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo ap FROM with_browsers AS release RUN cd /var/www/discourse &&\ - sudo -u discourse bundle install --jobs=4 &&\ - sudo -E -u discourse -H yarn install &&\ - sudo -u discourse yarn cache clean + sudo -u discourse bundle install --jobs $(($(nproc) - 1)) &&\ + sudo -E -u discourse -H /bin/bash -c 'if [ -f yarn.lock ]; then (yarn install && yarn cache clean); else CI=1 pnpm install; fi' RUN cd /var/www/discourse && sudo -E -u discourse -H bundle exec rake plugin:install_all_official &&\ - sudo -E -u discourse -H bundle exec rake plugin:install_all_gems + LOAD_PLUGINS=1 sudo -E -u discourse -H bundle exec rake plugin:install_all_gems &&\ + sudo -E -u discourse -H bundle exec ruby script/install_minio_binaries.rb -ENTRYPOINT sudo -E -u discourse -H ruby script/docker_test.rb +ENTRYPOINT ["sudo", "-E", "-u", "discourse", "-H", "ruby", "script/docker_test.rb"] diff --git a/launcher b/launcher index ec62983..3a4f21a 100755 --- a/launcher +++ b/launcher @@ -83,8 +83,8 @@ fi cd "$(dirname "$0")" pups_version='v1.0.3' -docker_min_version='17.03.1' -docker_rec_version='17.06.2' +docker_min_version='20.10.0' +docker_rec_version='24.0.7' git_min_version='1.8.0' git_rec_version='1.8.0' kernel_min_version='4.4.0' @@ -92,7 +92,7 @@ kernel_min_version='4.4.0' config_file=containers/"$config".yml cidbootstrap=cids/"$config"_bootstrap.cid local_discourse=local_discourse -image="discourse/base:2.0.20230711-0100" +image="discourse/base:2.0.20240825-0027" docker_path=`which docker.io 2> /dev/null || which docker` git_path=`which git` @@ -173,7 +173,7 @@ check_prereqs() { fi # 2. running an approved storage driver? - if ! $docker_path info 2> /dev/null | egrep -q 'Storage Driver: (btrfs|aufs|zfs|overlay2)$'; then + if ! $docker_path info 2> /dev/null | grep -E -q 'Storage Driver: (btrfs|aufs|zfs|overlay2)$'; then echo "Your Docker installation is not using a supported storage driver. If we were to proceed you may have a broken install." echo "overlay2 is the recommended storage driver, although zfs and aufs may work as well." echo "Other storage drivers are known to be problematic." diff --git a/samples/mail-receiver.yml b/samples/mail-receiver.yml index df5e3cd..1828f76 100644 --- a/samples/mail-receiver.yml +++ b/samples/mail-receiver.yml @@ -27,11 +27,11 @@ env: # POSTCONF_smtpd_tls_security_level: may - ## The URL of the mail processing endpoint of your Discourse forum. - ## This is simply your forum's base URL, with `/admin/email/handle_mail` - ## appended. Be careful if you're running a subfolder setup -- in that case, - ## the URL needs to have the subfolder included! - DISCOURSE_MAIL_ENDPOINT: 'https://discourse.example.com/admin/email/handle_mail' + ## The base URL for this Discourse instance. + ## This will be whatever your Discourse site URL is. For example, + ## https://discourse.example.com. If you're running a subfolder setup, + ## be sure to account for that (ie https://example.com/forum). + DISCOURSE_BASE_URL: 'https://discourse.example.com' ## The master API key of your Discourse forum. You can get this from ## the "API" tab of your admin panel. diff --git a/samples/standalone.yml b/samples/standalone.yml index 2ba6d15..e6d946f 100644 --- a/samples/standalone.yml +++ b/samples/standalone.yml @@ -77,9 +77,10 @@ env: ## The http or https CDN address for this Discourse instance (configured to pull) ## see https://meta.discourse.org/t/14857 for details #DISCOURSE_CDN_URL: https://discourse-cdn.example.com - - ## The maxmind geolocation IP address key for IP address lookup - ## see https://meta.discourse.org/t/-/137387/23 for details + + ## The maxmind geolocation IP account ID and license key for IP address lookups + ## see https://meta.discourse.org/t/-/173941 for details + #DISCOURSE_MAXMIND_ACCOUNT_ID: 123456 #DISCOURSE_MAXMIND_LICENSE_KEY: 1234567890123456 ## The Docker container is stateless; all data is stored in /shared diff --git a/samples/web_only.yml b/samples/web_only.yml index ea83093..c4753dd 100644 --- a/samples/web_only.yml +++ b/samples/web_only.yml @@ -75,9 +75,10 @@ env: ## The http or https CDN address for this Discourse instance (configured to pull) ## see https://meta.discourse.org/t/14857 for details #DISCOURSE_CDN_URL: https://discourse-cdn.example.com - - ## The maxmind geolocation IP address key for IP address lookup - ## see https://meta.discourse.org/t/-/137387/23 for details + + ## The maxmind geolocation IP account ID and license key for IP address lookups + ## see https://meta.discourse.org/t/-/173941 for details + #DISCOURSE_MAXMIND_ACCOUNT_ID: 123456 #DISCOURSE_MAXMIND_LICENSE_KEY: 1234567890123456 volumes: diff --git a/scripts/mk_swapfile b/scripts/mk_swapfile deleted file mode 100755 index 3240b44..0000000 --- a/scripts/mk_swapfile +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -e -# This script adds a 1GB swapfile to the system - -function do_err() { - code=$? - echo "Command failed with code $code: $BASH_COMMAND" - exit $code - -} -trap do_err ERR - - -function set_swappiness() { - if ! grep -q '^vm.swappiness' /etc/sysctl.conf; then - echo -n 'Setting ' - sysctl -w vm.swappiness=10 - echo vm.swappiness = 10 >> /etc/sysctl.conf - fi -} - -function get_new_swapfile() { - for i in `seq 0 99`; do - if [ ! -e /swapfile.$i ]; then - echo /swapfile.$i - return - fi - done - # Seriously? 100 swapfiles already exist? - echo "too many swapfiles" - exit 1 -} - -[ `id -u` -eq 0 ] || { echo "You must be root to run this script"; exit 1; } - -# how big? default 1GB -declare -i num_gb -num_gb="${1-1}" -[ $num_gb -lt 1 ] && { echo "Please specify an integer >= 1"; exit 1; } -echo "Creating a ${num_gb}GB swapfile..." - -set_swappiness - -SWAPFILE=$(get_new_swapfile) - -umask 077 -dd if=/dev/zero of=$SWAPFILE bs=1k count=$(($num_gb * 1024 * 1024)) conv=excl -mkswap $SWAPFILE -swapon $SWAPFILE -echo "$SWAPFILE swap swap auto 0 0" >> /etc/fstab - -echo 1GiB swapfile successfully added diff --git a/templates/import/mbox.template.yml b/templates/import/mbox.template.yml index ff6580f..c96055c 100644 --- a/templates/import/mbox.template.yml +++ b/templates/import/mbox.template.yml @@ -34,4 +34,4 @@ hooks: - apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y libsqlite3-dev - echo "gem 'sqlite3'" >> Gemfile - su discourse -c 'bundle config unset deployment' - - su discourse -c 'bundle install --no-deployment --path vendor/bundle --jobs 4 --without test development' + - su discourse -c 'bundle install --no-deployment --path vendor/bundle --jobs $(($(nproc) - 1)) --without test development' diff --git a/templates/import/mssql-dep.template.yml b/templates/import/mssql-dep.template.yml index 8df6890..f657855 100644 --- a/templates/import/mssql-dep.template.yml +++ b/templates/import/mssql-dep.template.yml @@ -22,4 +22,4 @@ hooks: cmd: - echo "gem 'tiny_tds'" >> Gemfile - su discourse -c 'bundle config unset deployment' - - su discourse -c 'bundle install --no-deployment --path vendor/bundle --jobs 4 --without test development' + - su discourse -c 'bundle install --no-deployment --path vendor/bundle --jobs $(($(nproc) - 1)) --without test development' diff --git a/templates/import/mysql-dep.template.yml b/templates/import/mysql-dep.template.yml index 27b394d..5c9b4c1 100644 --- a/templates/import/mysql-dep.template.yml +++ b/templates/import/mysql-dep.template.yml @@ -11,4 +11,4 @@ hooks: - echo "gem 'mysql2'" >> Gemfile - apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y libmariadb-dev - su discourse -c 'bundle config unset deployment' - - su discourse -c 'bundle install --no-deployment --path vendor/bundle --jobs 4 --without test development' + - su discourse -c 'bundle install --no-deployment --path vendor/bundle --jobs $(($(nproc) - 1)) --without test development' diff --git a/templates/import/phpbb3.template.yml b/templates/import/phpbb3.template.yml index 9a3024f..3b70cf4 100644 --- a/templates/import/phpbb3.template.yml +++ b/templates/import/phpbb3.template.yml @@ -114,4 +114,4 @@ hooks: - echo "gem 'mysql2'" >> Gemfile - echo "gem 'ruby-bbcode-to-md', :github => 'nlalonde/ruby-bbcode-to-md'" >> Gemfile - su discourse -c 'bundle config unset deployment' - - su discourse -c 'bundle install --no-deployment --path vendor/bundle --jobs 4 --without test development' + - su discourse -c 'bundle install --no-deployment --path vendor/bundle --jobs $(($(nproc) - 1)) --without test development' diff --git a/templates/import/vanilla.template.yml b/templates/import/vanilla.template.yml index 78802a2..753ce40 100644 --- a/templates/import/vanilla.template.yml +++ b/templates/import/vanilla.template.yml @@ -1,17 +1,13 @@ # This template installs MariaDB and all dependencies needed for importing from vanilla. +env: + UNICORN_SIDEKIQS: 0 + params: home: /var/www/discourse hooks: after_web_config: - - exec: - cd: /etc/service - cmd: - - rm -R unicorn - - rm -R nginx - - rm -R cron - - exec: cd: /etc/runit/3.d cmd: @@ -36,7 +32,7 @@ hooks: - exec: cmd: - mkdir -p /shared/import/mysql/data - - apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y nano libmariadbclient-dev mariadb-server + - apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y nano libmariadb-dev mariadb-server - sed -Ei 's/^log/#&/' /etc/mysql/my.cnf - file: @@ -95,10 +91,7 @@ hooks: sv stop mysql fi - cd $home - echo "The Vanilla import is starting..." - echo - su discourse -c 'bundle exec ruby script/import_scripts/vanilla.rb' + echo "import_vanilla.sh completed" - exec: cd: $home @@ -106,11 +99,22 @@ hooks: - mkdir -p /shared/import/data - chown discourse -R /shared/import + before_code: + - exec: + cd: $home + cmd: + # Add your discourse core fork and pull custom code + - su discourse -c 'git remote set-url origin https://github.com/{github_username}/discourse.git' + after_bundle_exec: - exec: cd: $home cmd: + # Add the gems used in the script - echo "gem 'mysql2'" >> Gemfile - echo "gem 'ruby-bbcode-to-md', :github => 'nlalonde/ruby-bbcode-to-md'" >> Gemfile - su discourse -c 'bundle config unset deployment' - - su discourse -c 'bundle install --no-deployment --path vendor/bundle --jobs 4 --without test development' + - su discourse -c 'bundle install --no-deployment --path vendor/bundle --jobs $(($(nproc) - 1)) --without test development' + - service mariadb start + # imports the DB into mysql + - sh /usr/local/bin/import_flarum_test.sh diff --git a/templates/postgres.13.template.yml b/templates/postgres.13.template.yml index bb2c0a2..c5d0814 100644 --- a/templates/postgres.13.template.yml +++ b/templates/postgres.13.template.yml @@ -17,7 +17,8 @@ hooks: to: sv start postgres || exit 1 run: - - exec: locale-gen $LANG && update-locale + - exec: sed -i "s/^# $LANG/$LANG/" /etc/locale.gen + - exec: locale-gen && update-locale - exec: mkdir -p /shared/postgres_run - exec: chown postgres:postgres /shared/postgres_run - exec: chmod 775 /shared/postgres_run diff --git a/templates/postgres.template.yml b/templates/postgres.template.yml index bb2c0a2..95001a0 100644 --- a/templates/postgres.template.yml +++ b/templates/postgres.template.yml @@ -14,20 +14,14 @@ hooks: - replace: filename: /etc/service/unicorn/run from: "# postgres" - to: sv start postgres || exit 1 + to: | + if [ -f /root/install_postgres ]; then + /root/install_postgres + rm /root/install_postgres + fi + sv start postgres || exit 1 run: - - exec: locale-gen $LANG && update-locale - - exec: mkdir -p /shared/postgres_run - - exec: chown postgres:postgres /shared/postgres_run - - exec: chmod 775 /shared/postgres_run - - exec: rm -fr /var/run/postgresql - - exec: ln -s /shared/postgres_run /var/run/postgresql - - exec: socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1 - - exec: rm -fr /shared/postgres_run/.s* - - exec: rm -fr /shared/postgres_run/*.pid - - exec: mkdir -p /shared/postgres_run/13-main.pg_stat_tmp - - exec: chown postgres:postgres /shared/postgres_run/13-main.pg_stat_tmp - file: path: /etc/service/postgres/run chmod: "+x" @@ -51,6 +45,36 @@ run: #!/bin/bash sv stop postgres + - file: + path: /root/install_postgres + chmod: "+x" + contents: | + #!/bin/bash + sed -i "s/^# $LANG/$LANG/" /etc/locale.gen + locale-gen && update-locale + mkdir -p /shared/postgres_run + chown postgres:postgres /shared/postgres_run + chmod 775 /shared/postgres_run + rm -fr /var/run/postgresql + ln -s /shared/postgres_run /var/run/postgresql + if [ -e /shared/postgres_run/.s.PGSQL.5432 ]; then + socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1 + fi + rm -fr /shared/postgres_run/.s* + rm -fr /shared/postgres_run/*.pid + mkdir -p /shared/postgres_run/13-main.pg_stat_tmp + chown postgres:postgres /shared/postgres_run/13-main.pg_stat_tmp + chown -R root /var/lib/postgresql/13/main + [ ! -e /shared/postgres_data ] && install -d -m 0755 -o postgres -g postgres /shared/postgres_data && sudo -E -u postgres /usr/lib/postgresql/13/bin/initdb -D /shared/postgres_data || exit 0 + chown -R postgres:postgres /shared/postgres_data + chown -R postgres:postgres /var/run/postgresql + if [ -f /root/upgrade_postgres ]; then + /root/upgrade_postgres + rm /root/upgrade_postgres + fi + # Necessary to enable backups + install -d -m 0755 -o postgres -g postgres /shared/postgres_backup + - file: path: /root/upgrade_postgres chmod: "+x" @@ -116,16 +140,6 @@ run: exit 77 fi - - exec: - cmd: - - chown -R root /var/lib/postgresql/13/main - - "[ ! -e /shared/postgres_data ] && install -d -m 0755 -o postgres -g postgres /shared/postgres_data && sudo -E -u postgres /usr/lib/postgresql/13/bin/initdb -D /shared/postgres_data || exit 0" - - chown -R postgres:postgres /shared/postgres_data - - chown -R postgres:postgres /var/run/postgresql - - - exec: /root/upgrade_postgres - - exec: rm /root/upgrade_postgres - - replace: filename: "/etc/postgresql/13/main/postgresql.conf" from: "data_directory = '/var/lib/postgresql/13/main'" @@ -161,11 +175,6 @@ run: from: /#?default_text_search_config *=.*/ to: "default_text_search_config = '$db_default_text_search_config'" - # Necessary to enable backups - - exec: - cmd: - - install -d -m 0755 -o postgres -g postgres /shared/postgres_backup - - replace: filename: "/etc/postgresql/13/main/postgresql.conf" from: /#?checkpoint_segments *=.*/ @@ -199,29 +208,39 @@ run: to: "host all all ::/0 md5" - exec: + tag: db + cmd: | + if [ -f /root/install_postgres ]; then + /root/install_postgres && rm -f /root/install_postgres + elif [ -e /shared/postgres_run/.s.PGSQL.5432 ]; then + socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1 + fi + + - exec: + tag: db background: true # use fast shutdown for pg stop_signal: INT cmd: HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/13/bin/postmaster -D /etc/postgresql/13/main - # give db a few secs to start up - - exec: "sleep 5" - - - exec: su postgres -c 'createdb $db_name' || true - - exec: su postgres -c 'psql $db_name -c "create user $db_user;"' || true - - exec: su postgres -c 'psql $db_name -c "grant all privileges on database $db_name to $db_user;"' || true - - exec: su postgres -c 'psql $db_name -c "alter schema public owner to $db_user;"' - - exec: su postgres -c 'psql template1 -c "create extension if not exists hstore;"' - - exec: su postgres -c 'psql template1 -c "create extension if not exists pg_trgm;"' - - exec: su postgres -c 'psql template1 -c "create extension if not exists vector;"' - - exec: su postgres -c 'psql $db_name -c "create extension if not exists hstore;"' - - exec: su postgres -c 'psql $db_name -c "create extension if not exists pg_trgm;"' - - exec: su postgres -c 'psql $db_name -c "create extension if not exists vector;"' - - exec: - stdin: | - update pg_database set encoding = pg_char_to_encoding('UTF8') where datname = '$db_name' AND encoding = pg_char_to_encoding('SQL_ASCII'); - cmd: sudo -u postgres psql $db_name - raise_on_fail: false + - file: + path: /usr/local/bin/create_db + chmod: +x + contents: | + #!/bin/bash + su postgres -c 'createdb $db_name' || true + su postgres -c 'psql $db_name -c "create user $db_user;"' || true + su postgres -c 'psql $db_name -c "grant all privileges on database $db_name to $db_user;"' || true + su postgres -c 'psql $db_name -c "alter schema public owner to $db_user;"' + su postgres -c 'psql template1 -c "create extension if not exists hstore;"' + su postgres -c 'psql template1 -c "create extension if not exists pg_trgm;"' + su postgres -c 'psql template1 -c "create extension if not exists vector;"' + su postgres -c 'psql template1 -c "alter extension vector update;"' || true + su postgres -c 'psql $db_name -c "create extension if not exists hstore;"' + su postgres -c 'psql $db_name -c "create extension if not exists pg_trgm;"' + su postgres -c 'psql $db_name -c "create extension if not exists vector;"' + su postgres -c 'psql $db_name -c "alter extension vector update;"' || true + sudo -u postgres psql $db_name <<< "update pg_database set encoding = pg_char_to_encoding('UTF8') where datname = '$db_name' AND encoding = pg_char_to_encoding('SQL_ASCII');" || true - file: path: /var/lib/postgresql/take-database-backup @@ -242,5 +261,10 @@ run: #0 */4 * * * /var/lib/postgresql/take-database-backup - exec: + tag: db hook: postgres - cmd: "echo postgres installed!" + cmd: + # give db a few secs to start up + - "sleep 5" + - /usr/local/bin/create_db + - "echo postgres installed!" diff --git a/templates/redis.template.yml b/templates/redis.template.yml index b5d2a2b..9044869 100644 --- a/templates/redis.template.yml +++ b/templates/redis.template.yml @@ -33,6 +33,7 @@ run: to: "" - exec: + tag: db cmd: - install -d -m 0755 -o redis -g redis /shared/redis_data @@ -68,18 +69,25 @@ run: - exec: background: true + tag: db cmd: exec chpst -u redis -U redis /usr/bin/redis-server /etc/redis/redis.conf - - - exec: sleep 10 + - exec: + tag: db + cmd: sleep 10 # we can not migrate without redis, launch it if needed hooks: - before_code: + before_db_migrate: - exec: background: true cmd: exec chpst -u redis -U redis /usr/bin/redis-server /etc/redis/redis.conf + - exec: sleep 10 after_code: - replace: filename: /etc/service/unicorn/run from: "# redis" - to: sv start redis || exit 1 + to: | + if [ ! -d /shared/redis_data ]; then + install -d -m 0755 -o redis -g redis /shared/redis_data + fi + sv start redis || exit 1 diff --git a/templates/web.china.template.yml b/templates/web.china.template.yml index 973647f..fc95a0d 100644 --- a/templates/web.china.template.yml +++ b/templates/web.china.template.yml @@ -1,10 +1,20 @@ hooks: + before_code: + - exec: + cmd: + - su discourse -c 'git config --global url."https://mirror.ghproxy.com/https://github.com/".insteadOf "https://github.com/"' + before_web: - exec: - cmd: - - gem sources --add https://gems.ruby-china.com/ --remove https://rubygems.org/ + cmd: + - su discourse -c 'gem sources --add https://mirrors.tuna.tsinghua.edu.cn/rubygems/ --remove https://rubygems.org/' + + before_yarn: + - exec: + cmd: + - su discourse -c 'pnpm config set registry https://registry.npmmirror.com --global' before_bundle_exec: - exec: - cmd: - - su discourse -c 'bundle config mirror.https://rubygems.org https://gems.ruby-china.com/' + cmd: + - su discourse -c 'bundle config mirror.https://rubygems.org https://mirrors.tuna.tsinghua.edu.cn/rubygems' diff --git a/templates/web.ssl.template.yml b/templates/web.ssl.template.yml index cefd1aa..cc6a66f 100644 --- a/templates/web.ssl.template.yml +++ b/templates/web.ssl.template.yml @@ -15,14 +15,16 @@ run: filename: "/etc/nginx/conf.d/discourse.conf" from: /listen 80;\s+gzip on;/m to: | - listen 443 ssl http2; + listen 443 ssl; + http2 on; SSL_TEMPLATE_SSL_BLOCK - replace: filename: "/etc/nginx/conf.d/discourse.conf" from: /listen 80;\s+listen \[::\]:80;\s+gzip on;/m to: | - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + http2 on; SSL_TEMPLATE_SSL_BLOCK - replace: hook: ssl diff --git a/templates/web.template.yml b/templates/web.template.yml index 9fb801d..a30ac02 100644 --- a/templates/web.template.yml +++ b/templates/web.template.yml @@ -12,19 +12,20 @@ env: DISCOURSE_DB_HOST: DISCOURSE_DB_PORT: - params: version: tests-passed - home: /var/www/discourse upload_size: 10m nginx_worker_connections: 4000 run: - exec: thpoff echo "thpoff is installed!" - - exec: /usr/local/bin/ruby -e 'if ENV["DISCOURSE_SMTP_ADDRESS"] == "smtp.example.com"; puts "Aborting! Mail is not configured!"; exit 1; end' - - exec: /usr/local/bin/ruby -e 'if ENV["DISCOURSE_HOSTNAME"] == "discourse.example.com"; puts "Aborting! Domain is not configured!"; exit 1; end' - - exec: /usr/local/bin/ruby -e 'if (ENV["DISCOURSE_CDN_URL"] || "")[0..1] == "//"; puts "Aborting! CDN must have a protocol specified. Once fixed you should rebake your posts now to correct all posts."; exit 1; end' + - exec: + tag: precompile + cmd: + - /usr/local/bin/ruby -e 'if ENV["DISCOURSE_SMTP_ADDRESS"] == "smtp.example.com"; puts "Aborting! Mail is not configured!"; exit 1; end' + - /usr/local/bin/ruby -e 'if ENV["DISCOURSE_HOSTNAME"] == "discourse.example.com"; puts "Aborting! Domain is not configured!"; exit 1; end' + - /usr/local/bin/ruby -e 'if (ENV["DISCOURSE_CDN_URL"] || "")[0..1] == "//"; puts "Aborting! CDN must have a protocol specified. Once fixed you should rebake your posts now to correct all posts."; exit 1; end' # TODO: move to base image (anacron can not be fired up using rc.d) - exec: rm -f /etc/cron.d/anacron - file: @@ -55,6 +56,13 @@ run: # postgres cd $home chown -R discourse:www-data /shared/log/rails + # before precompile + if [[ -z "$PRECOMPILE_ON_BOOT" ]]; then + PRECOMPILE_ON_BOOT=1 + fi + if [ -f /usr/local/bin/create_db ] && [ "$CREATE_DB_ON_BOOT" = "1" ]; then /usr/local/bin/create_db; fi; + if [ "$MIGRATE_ON_BOOT" = "1" ]; then su discourse -c 'bundle exec rake db:migrate'; fi + if [ "$PRECOMPILE_ON_BOOT" = "1" ]; then SKIP_EMBER_CLI_COMPILE=1 su discourse -c 'bundle exec rake assets:precompile'; fi LD_PRELOAD=$RUBY_ALLOCATOR HOME=/home/discourse USER=discourse exec thpoff chpst -u discourse:www-data -U discourse:www-data bundle exec config/unicorn_launcher -E production -c config/unicorn.conf.rb - file: @@ -170,29 +178,35 @@ run: - exec: cd: $home + hook: yarn cmd: - |- - if [ "$version" != "tests-passed" ]; then - rm -rf app/assets/javascripts/node_modules + if [ -f yarn.lock ]; then + if [ -d node_modules/.pnpm ]; then + echo "This version of Discourse uses yarn, but pnpm node_modules are preset. Cleaning up..." + find ./node_modules ./app/assets/javascripts/*/node_modules -mindepth 1 -maxdepth 1 -exec rm -rf {} + + fi + su discourse -c 'yarn install --frozen-lockfile && yarn cache clean' + else + su discourse -c 'CI=1 pnpm install --frozen-lockfile' fi - - su discourse -c 'yarn install --frozen-lockfile && yarn cache clean' - exec: cd: $home hook: bundle_exec cmd: - - su discourse -c 'bundle config --local deployment true' - - su discourse -c 'bundle config --local without "development test"' - - su discourse -c 'bundle install --retry 3 --jobs 4' + - su discourse -c 'bundle install --jobs $(($(nproc) - 1)) --retry 3' - exec: cd: $home cmd: - su discourse -c 'LOAD_PLUGINS=0 bundle exec rake plugin:pull_compatible_all' + hook: plugin_compatibility raise_on_fail: false - exec: cd: $home + tag: migrate hook: db_migrate cmd: - su discourse -c 'bundle exec rake db:migrate' @@ -202,18 +216,24 @@ run: hook: maxminddb_get cmd: - su discourse -c 'bundle exec rake maxminddb:get' - - - exec: - cd: $home - hook: assets_precompile - cmd: - - su discourse -c 'bundle exec rake themes:update assets:precompile' - exec: cd: $home + tag: build + hook: assets_precompile_build cmd: - - su discourse -c 'sed -i "s/timeout 30/timeout 60/" config/unicorn.conf.rb' - - echo "Unicorn timeout changed to 60 seconds" + - su discourse -c 'bundle exec rake assets:precompile:build' + - exec: + cd: $home + tag: precompile + hook: assets_precompile + cmd: + - su discourse -c 'SKIP_EMBER_CLI_COMPILE=1 bundle exec rake themes:update assets:precompile' + - replace: + tag: precompile + filename: /etc/service/unicorn/run + from: "# before precompile" + to: "PRECOMPILE_ON_BOOT=0" - file: path: /usr/local/bin/discourse @@ -276,6 +296,7 @@ run: missingok delaycompress compress + sharedscripts postrotate sv 1 unicorn endscript @@ -420,7 +441,7 @@ run: - +