discourse
/
docker_manager
mirror of https://github.com/discourse/docker_manager.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DEV: Run stree

This commit is contained in:
Jarek Radosz 2023-01-15 19:56:52 +01:00
parent 96211d622e
commit 4ab63aca32
7 changed files with 169 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
app/controllers/docker_manager/admin_controller.rb
View File

@ -1,30 +1,36 @@
# frozen_string_literal: true # frozen_string_literal: true
require_dependency 'docker_manager/git_repo' require_dependency "docker_manager/git_repo"
require_dependency 'docker_manager/upgrader' require_dependency "docker_manager/upgrader"
module DockerManager module DockerManager
class AdminController < DockerManager::ApplicationController class AdminController < Admin::AdminController
layout nil helper DockerManager::ApplicationHelper
def index def index
return if Rails.env.development? return if Rails.env.development?
version = File.read('/VERSION') rescue '1.0.0' version =
begin
File.read("/VERSION")
rescue StandardError
"1.0.0"
end
version = Gem::Version.new(version) version = Gem::Version.new(version)
expected_version = Gem::Version.new('2.0.20220128-1817') expected_version = Gem::Version.new("2.0.20220128-1817")
ruby_version = Gem::Version.new(RUBY_VERSION) ruby_version = Gem::Version.new(RUBY_VERSION)
expected_ruby_version = Gem::Version.new('3.1.3') expected_ruby_version = Gem::Version.new("3.1.3")
min_stable_version = Gem::Version.new('2.8.7') min_stable_version = Gem::Version.new("2.8.7")
min_beta_version = Gem::Version.new('2.9.0.beta8') min_beta_version = Gem::Version.new("2.9.0.beta8")
upgrade_image = version < expected_version upgrade_image = version < expected_version
upgrade_ruby = ruby_version < expected_ruby_version upgrade_ruby = ruby_version < expected_ruby_version
upgrade_discourse = discourse_upgrade_required?(min_stable_version, min_beta_version) upgrade_discourse =
discourse_upgrade_required?(min_stable_version, min_beta_version)
if upgrade_image || upgrade_ruby || upgrade_discourse if upgrade_image || upgrade_ruby || upgrade_discourse
render 'upgrade_required', layout: false render "upgrade_required", layout: false
else else
render render
end end
@ -40,10 +46,16 @@ module DockerManager
official: Plugin::Metadata::OFFICIAL_PLUGINS.include?(r.name) official: Plugin::Metadata::OFFICIAL_PLUGINS.include?(r.name)
} }
result[:fork] = true if result[:official] && !r.url.starts_with?("https://github.com/discourse/") result[:fork] = true if result[:official] &&
!r.url.starts_with?("https://github.com/discourse/")
if r.valid? if r.valid?
result[:id] = r.name.downcase.gsub(/[^a-z]/, '_').gsub(/_+/, '_').sub(/_$/, '') result[:id] = r
.name
.downcase
.gsub(/[^a-z]/, "_")
.gsub(/_+/, "_")
.sub(/_$/, "")
result[:version] = r.latest_local_commit result[:version] = r.latest_local_commit
result[:pretty_version] = r.latest_local_tag_version.presence result[:pretty_version] = r.latest_local_tag_version.presence
result[:url] = r.url result[:url] = r.url
@ -63,53 +75,60 @@ module DockerManager
return respond_progress if repo.blank? return respond_progress if repo.blank?
upgrader = Upgrader.new(current_user.id, repo, repo_version(repo)) upgrader = Upgrader.new(current_user.id, repo, repo_version(repo))
respond_progress(logs: upgrader.find_logs, percentage: upgrader.last_percentage) respond_progress(
logs: upgrader.find_logs,
percentage: upgrader.last_percentage
)
end end
def latest def latest
proc = Proc.new do |repo| proc =
repo.update_remote! if Rails.env == 'production' Proc.new do |repo|
{ repo.update_remote! if Rails.env == "production"
path: repo.path, {
version: repo.latest_origin_commit, path: repo.path,
pretty_version: repo.latest_origin_tag_version.presence, version: repo.latest_origin_commit,
commits_behind: repo.commits_behind, pretty_version: repo.latest_origin_tag_version.presence,
date: repo.latest_origin_commit_date commits_behind: repo.commits_behind,
} date: repo.latest_origin_commit_date
end }
end
if all_repos? if all_repos?
return render json: { return(
repos: DockerManager::GitRepo.find_all.map(&proc) render json: { repos: DockerManager::GitRepo.find_all.map(&proc) }
} )
end end
repo = DockerManager::GitRepo.find(params[:path]) repo = DockerManager::GitRepo.find(params[:path])
raise Discourse::NotFound unless repo.present? raise Discourse::NotFound unless repo.present?
render json: { render json: { latest: proc.call(repo) }
latest: proc.call(repo)
}
end end
def upgrade def upgrade
repo = find_repos(params[:path]) repo = find_repos(params[:path])
raise Discourse::NotFound unless repo.present? raise Discourse::NotFound unless repo.present?
script_path = File.expand_path(File.join(__dir__, '../../../scripts/docker_manager_upgrade.rb')) script_path =
File.expand_path(
File.join(__dir__, "../../../scripts/docker_manager_upgrade.rb")
)
env_vars = { env_vars = {
'UPGRADE_USER_ID' => current_user.id.to_s, "UPGRADE_USER_ID" => current_user.id.to_s,
'UPGRADE_PATH' => params[:path].to_s, "UPGRADE_PATH" => params[:path].to_s,
'UPGRADE_REPO_VERSION' => repo_version(repo).to_s, "UPGRADE_REPO_VERSION" => repo_version(repo).to_s,
'RAILS_ENV' => Rails.env "RAILS_ENV" => Rails.env
} }
["http_proxy", "https_proxy", "no_proxy", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"].each do |p| %w[
env_vars[p] = ENV[p] if ! ENV[p].nil? http_proxy
end https_proxy
pid = spawn( no_proxy
env_vars, HTTP_PROXY
"bundle exec rails runner #{script_path}" HTTPS_PROXY
) NO_PROXY
].each { |p| env_vars[p] = ENV[p] if !ENV[p].nil? }
pid = spawn(env_vars, "bundle exec rails runner #{script_path}")
Process.detach(pid) Process.detach(pid)
render plain: "OK" render plain: "OK"
end end
@ -138,9 +157,7 @@ module DockerManager
end end
def self.find_repos(path, upgrading: false, all: false) def self.find_repos(path, upgrading: false, all: false)
unless all_repos?(path) return DockerManager::GitRepo.find(path) unless all_repos?(path)
return DockerManager::GitRepo.find(path)
end
repos = DockerManager::GitRepo.find_all repos = DockerManager::GitRepo.find_all
return repos if all return repos if all
@ -149,7 +166,8 @@ module DockerManager
if upgrading if upgrading
repo.upgrading? repo.upgrading?
else else
!repo.upgrading? && (repo.latest_local_commit != repo.latest_origin_commit) !repo.upgrading? &&
(repo.latest_local_commit != repo.latest_origin_commit)
end end
end end
end end
@ -157,12 +175,7 @@ module DockerManager
private private
def respond_progress(logs: nil, percentage: nil) def respond_progress(logs: nil, percentage: nil)
render json: { render json: { progress: { logs: logs, percentage: percentage } }
progress: {
logs: logs,
percentage: percentage
}
}
end end
def all_repos? def all_repos?
@ -174,7 +187,11 @@ module DockerManager
end end
def repo_version(repo) def repo_version(repo)
repo.is_a?(Array) && params[:version].blank? ? concat_repos_versions(repo) : params[:version] if repo.is_a?(Array) && params[:version].blank?
concat_repos_versions(repo)
else
params[:version]
end
end end
def concat_repos_versions(repos) def concat_repos_versions(repos)

36
lib/docker_manager/git_repo.rb
View File

@ -75,20 +75,26 @@ class DockerManager::GitRepo
def self.find_all def self.find_all
repos = [ repos = [
DockerManager::GitRepo.new(Rails.root.to_s, 'discourse'), DockerManager::GitRepo.new(Rails.root.to_s, "discourse"),
DockerManager::GitRepo.new("#{Rails.root}/plugins/docker_manager", "docker_manager") DockerManager::GitRepo.new(
"#{Rails.root}/plugins/docker_manager",
"docker_manager"
)
] ]
p = Proc.new { |x|
next if x.name == "docker_manager" p =
repos << DockerManager::GitRepo.new(File.dirname(x.path), x.name) Proc.new do |x|
} next if x.name == "docker_manager"
repos << DockerManager::GitRepo.new(File.dirname(x.path), x.name)
end
if Discourse.respond_to?(:visible_plugins) if Discourse.respond_to?(:visible_plugins)
Discourse.visible_plugins.each(&p) Discourse.visible_plugins.each(&p)
else else
Discourse.plugins.each(&p) Discourse.plugins.each(&p)
end end
repos
repos
end end
def self.find(path) def self.find(path)
@ -96,7 +102,10 @@ class DockerManager::GitRepo
end end
def upstream_branch def upstream_branch
@upstream_branch ||= run("for-each-ref --format='%(upstream:short)' $(git symbolic-ref -q HEAD)") @upstream_branch ||=
run(
"for-each-ref --format='%(upstream:short)' $(git symbolic-ref -q HEAD)"
)
end end
def has_local_main? def has_local_main?
@ -109,9 +118,7 @@ class DockerManager::GitRepo
result = run(command) result = run(command)
return unless result.present? return unless result.present?
if result =~ /-(\d+)-/ result = result.gsub(/-(\d+)-.*/, " +#{$1}") if result =~ /-(\d+)-/
result = result.gsub(/-(\d+)-.*/, " +#{$1}")
end
result result
end end
@ -125,7 +132,11 @@ class DockerManager::GitRepo
end end
def has_origin_main? def has_origin_main?
run("branch -a").match?(/remotes\/origin\/main$/) rescue false begin
run("branch -a").match?(%r{remotes/origin/main$})
rescue StandardError
false
end
end end
def tracking_branch def tracking_branch
@ -150,5 +161,4 @@ class DockerManager::GitRepo
rescue => e rescue => e
p e p e
end end
end end

74
lib/docker_manager/upgrader.rb
View File

@ -1,7 +1,6 @@
# frozen_string_literal: true # frozen_string_literal: true
class DockerManager::Upgrader class DockerManager::Upgrader
def initialize(user_id, repos, from_version) def initialize(user_id, repos, from_version)
@user_id = user_id @user_id = user_id
@repos = repos.is_a?(Array) ? repos : [repos] @repos = repos.is_a?(Array) ? repos : [repos]
@ -19,9 +18,7 @@ class DockerManager::Upgrader
end end
def upgrade def upgrade
@repos.each do |repo| @repos.each { |repo| return unless repo.start_upgrading }
return unless repo.start_upgrading
end
percent(0) percent(0)
@ -68,7 +65,6 @@ class DockerManager::Upgrader
# HEAD@{upstream} is just a fancy way how to say origin/master (in normal case) # HEAD@{upstream} is just a fancy way how to say origin/master (in normal case)
# see http://stackoverflow.com/a/12699604/84283 # see http://stackoverflow.com/a/12699604/84283
@repos.each_with_index do |repo, index| @repos.each_with_index do |repo, index|
# We automatically handle renames from `master` -> `main` # We automatically handle renames from `master` -> `main`
if repo.upstream_branch == "origin/master" && repo.branch == "origin/main" if repo.upstream_branch == "origin/master" && repo.branch == "origin/main"
log "Branch has changed to #{repo.branch}" log "Branch has changed to #{repo.branch}"
@ -84,7 +80,9 @@ class DockerManager::Upgrader
run "cd #{repo.path} && git branch -u origin/main main" run "cd #{repo.path} && git branch -u origin/main main"
run("cd #{repo.path} && git reset --hard HEAD@{upstream}") run("cd #{repo.path} && git reset --hard HEAD@{upstream}")
else else
run("cd #{repo.path} && git fetch --tags --force && git reset --hard HEAD@{upstream}") run(
"cd #{repo.path} && git fetch --tags --force && git reset --hard HEAD@{upstream}"
)
end end
percent(20 * (index + 1) / @repos.size) percent(20 * (index + 1) / @repos.size)
@ -101,10 +99,13 @@ class DockerManager::Upgrader
run("SKIP_POST_DEPLOYMENT_MIGRATIONS=1 bundle exec rake multisite:migrate") run("SKIP_POST_DEPLOYMENT_MIGRATIONS=1 bundle exec rake multisite:migrate")
percent(40) percent(40)
log("*** Bundling assets. This will take a while *** ") log("*** Bundling assets. This will take a while *** ")
less_memory_flags = "RUBY_GC_MALLOC_LIMIT_MAX=20971520 RUBY_GC_OLDMALLOC_LIMIT_MAX=20971520 RUBY_GC_HEAP_GROWTH_MAX_SLOTS=50000 RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR=0.9 " less_memory_flags =
"RUBY_GC_MALLOC_LIMIT_MAX=20971520 RUBY_GC_OLDMALLOC_LIMIT_MAX=20971520 RUBY_GC_HEAP_GROWTH_MAX_SLOTS=50000 RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR=0.9 "
run("#{less_memory_flags} bundle exec rake themes:update assets:precompile") run("#{less_memory_flags} bundle exec rake themes:update assets:precompile")
using_s3_assets = ENV["DISCOURSE_USE_S3"] && ENV["DISCOURSE_S3_BUCKET"] && ENV["DISCOURSE_S3_CDN_URL"] using_s3_assets =
ENV["DISCOURSE_USE_S3"] && ENV["DISCOURSE_S3_BUCKET"] &&
ENV["DISCOURSE_S3_CDN_URL"]
if using_s3_assets if using_s3_assets
run("#{less_memory_flags} bundle exec rake s3:upload_assets") run("#{less_memory_flags} bundle exec rake s3:upload_assets")
@ -133,16 +134,15 @@ class DockerManager::Upgrader
log_version_upgrade log_version_upgrade
percent(100) percent(100)
log("DONE") log("DONE")
publish('status', 'complete') publish("status", "complete")
rescue => ex rescue => ex
publish('status', 'failed') publish("status", "failed")
[ [
"Docker Manager: FAILED TO UPGRADE", "Docker Manager: FAILED TO UPGRADE",
ex.inspect, ex.inspect,
ex.backtrace.join("\n"), ex.backtrace.join("\n")
].each do |message| ].each do |message|
STDERR.puts(message) STDERR.puts(message)
log(message) log(message)
end end
@ -158,7 +158,8 @@ class DockerManager::Upgrader
end end
def publish(type, value) def publish(type, value)
MessageBus.publish("/docker/upgrade", MessageBus.publish(
"/docker/upgrade",
{ type: type, value: value }, { type: type, value: value },
user_ids: [@user_id] user_ids: [@user_id]
) )
@ -168,7 +169,7 @@ class DockerManager::Upgrader
log "$ #{cmd}" log "$ #{cmd}"
msg = +"" msg = +""
allowed_env = %w{ allowed_env = %w[
PWD PWD
HOME HOME
SHELL SHELL
@ -181,20 +182,24 @@ class DockerManager::Upgrader
http_proxy http_proxy
https_proxy https_proxy
no_proxy no_proxy
} ]
clear_env = Hash[*ENV.map { |k, v| [k, nil] } clear_env =
.reject { |k, v| Hash[
allowed_env.include?(k) || *ENV
k =~ /^DISCOURSE_/ .map { |k, v| [k, nil] }
} .reject { |k, v| allowed_env.include?(k) || k =~ /^DISCOURSE_/ }
.flatten] .flatten
]
clear_env["RAILS_ENV"] = "production" clear_env["RAILS_ENV"] = "production"
clear_env["TERM"] = 'dumb' # claim we have a terminal clear_env["TERM"] = "dumb" # claim we have a terminal
retval = nil retval = nil
Open3.popen2e(clear_env, "cd #{Rails.root} && #{cmd} 2>&1") do |_in, out, wait_thread| Open3.popen2e(
clear_env,
"cd #{Rails.root} && #{cmd} 2>&1"
) do |_in, out, wait_thread|
out.each do |line| out.each do |line|
line.rstrip! # the client adds newlines, so remove the one we're given line.rstrip! # the client adds newlines, so remove the one we're given
log(line) log(line)
@ -233,18 +238,18 @@ class DockerManager::Upgrader
def percent(val) def percent(val)
Discourse.redis.set(percent_key, val) Discourse.redis.set(percent_key, val)
Discourse.redis.expire(percent_key, 30.minutes) Discourse.redis.expire(percent_key, 30.minutes)
publish('percent', val) publish("percent", val)
end end
def log(message) def log(message)
Discourse.redis.append logs_key, message + "\n" Discourse.redis.append logs_key, message + "\n"
Discourse.redis.expire(logs_key, 30.minutes) Discourse.redis.expire(logs_key, 30.minutes)
publish 'log', message publish "log", message
end end
def log_version_upgrade def log_version_upgrade
StaffActionLogger.new(User.find(@user_id)).log_custom( StaffActionLogger.new(User.find(@user_id)).log_custom(
'discourse_upgrade', "discourse_upgrade",
from_version: @from_version, from_version: @from_version,
repository: @repos.map(&:path).join(", ") repository: @repos.map(&:path).join(", ")
) )
@ -267,13 +272,13 @@ class DockerManager::Upgrader
end end
def unicorn_workers(master_pid) def unicorn_workers(master_pid)
`ps -f --ppid #{master_pid} | grep worker | awk '{ print $2 }'` `ps -f --ppid #{master_pid} | grep worker | awk '{ print $2 }'`.split(
.split("\n") "\n"
.map(&:to_i) ).map(&:to_i)
end end
def local_web_url def local_web_url
"http://127.0.0.1:#{ENV['UNICORN_PORT'] || 3000}/srv/status" "http://127.0.0.1:#{ENV["UNICORN_PORT"] || 3000}/srv/status"
end end
def reload_unicorn(launcher_pid) def reload_unicorn(launcher_pid)
@ -282,20 +287,19 @@ class DockerManager::Upgrader
Process.kill("USR2", launcher_pid) Process.kill("USR2", launcher_pid)
iterations = 0 iterations = 0
while pid_exists?(original_master_pid) do while pid_exists?(original_master_pid)
iterations += 1 iterations += 1
break if iterations >= 60 break if iterations >= 60
log("Waiting for Unicorn to reload#{'.' * iterations}") log("Waiting for Unicorn to reload#{"." * iterations}")
sleep 2 sleep 2
end end
iterations = 0 iterations = 0
while `curl -s #{local_web_url}` != "ok" do while `curl -s #{local_web_url}` != "ok"
iterations += 1 iterations += 1
break if iterations >= 60 break if iterations >= 60
log("Waiting for Unicorn workers to start up#{'.' * iterations}") log("Waiting for Unicorn workers to start up#{"." * iterations}")
sleep 2 sleep 2
end end
end end
end end

4
plugin.rb
View File

@ -9,7 +9,9 @@
module ::DockerManager module ::DockerManager
# should be automatic, but something is weird # should be automatic, but something is weird
load File.expand_path(File.dirname(__FILE__)) + '/app/helpers/application_helper.rb' load File.expand_path(File.dirname(__FILE__)) +
"/app/helpers/application_helper.rb"
class Engine < ::Rails::Engine class Engine < ::Rails::Engine
engine_name "docker_manager" engine_name "docker_manager"
isolate_namespace DockerManager isolate_namespace DockerManager

8
scripts/docker_manager_upgrade.rb
View File

@ -5,11 +5,11 @@
# process from getting killed if/when unicorn gets killed. # process from getting killed if/when unicorn gets killed.
fork do fork do
Process.setsid Process.setsid
require_relative '../lib/docker_manager/upgrader.rb' require_relative "../lib/docker_manager/upgrader.rb"
user_id = ENV['UPGRADE_USER_ID'].to_i user_id = ENV["UPGRADE_USER_ID"].to_i
path = ENV['UPGRADE_PATH'] path = ENV["UPGRADE_PATH"]
repo_version = ENV['UPGRADE_REPO_VERSION'] repo_version = ENV["UPGRADE_REPO_VERSION"]
raise "user_id is required" if user_id <= 0 raise "user_id is required" if user_id <= 0
raise "path is required" if path.blank? raise "path is required" if path.blank?

25
spec/lib/git_repo_spec.rb
View File

@ -1,10 +1,9 @@
# frozen_string_literal: true # frozen_string_literal: true
require 'rails_helper' require "rails_helper"
require 'docker_manager/git_repo' require "docker_manager/git_repo"
RSpec.describe DockerManager::GitRepo do RSpec.describe DockerManager::GitRepo do
describe ".find_all" do describe ".find_all" do
it "returns a list of repos" do it "returns a list of repos" do
expect(described_class.find_all).to be_present expect(described_class.find_all).to be_present
@ -23,27 +22,37 @@ RSpec.describe DockerManager::GitRepo do
end end
describe "#branch" do describe "#branch" do
it "returns origin/master if a repo hasn't been renamed" do it "returns origin/master if a repo hasn't been renamed" do
described_class.any_instance.stubs(:upstream_branch).returns("origin/master") described_class
.any_instance
.stubs(:upstream_branch)
.returns("origin/master")
described_class.any_instance.stubs(:has_origin_main?).returns(false) described_class.any_instance.stubs(:has_origin_main?).returns(false)
repo = described_class.new("dummy", "dummy") repo = described_class.new("dummy", "dummy")
expect(repo.branch).to eq("origin/master") expect(repo.branch).to eq("origin/master")
end end
it "returns origin/main if a repo has been renamed but still tracks master" do it "returns origin/main if a repo has been renamed but still tracks master" do
described_class.any_instance.stubs(:upstream_branch).returns("origin/master") described_class
.any_instance
.stubs(:upstream_branch)
.returns("origin/master")
described_class.any_instance.stubs(:has_origin_main?).returns(true) described_class.any_instance.stubs(:has_origin_main?).returns(true)
repo = described_class.new("dummy", "dummy") repo = described_class.new("dummy", "dummy")
expect(repo.branch).to eq("origin/main") expect(repo.branch).to eq("origin/main")
end end
it "returns origin/main if a repo points at origin/main" do it "returns origin/main if a repo points at origin/main" do
described_class.any_instance.stubs(:upstream_branch).returns("origin/main") described_class
.any_instance
.stubs(:upstream_branch)
.returns("origin/main")
described_class.any_instance.stubs(:has_origin_main?).returns(true) described_class.any_instance.stubs(:has_origin_main?).returns(true)
repo = described_class.new("dummy", "dummy") repo = described_class.new("dummy", "dummy")
expect(repo.branch).to eq("origin/main") expect(repo.branch).to eq("origin/main")
end end
end end
end end

26
spec/requests/admin_controller_spec.rb
View File

@ -1,39 +1,39 @@
# frozen_string_literal: true # frozen_string_literal: true
require 'rails_helper' require "rails_helper"
RSpec.describe DockerManager::AdminController do RSpec.describe DockerManager::AdminController do
describe 'anonymous user' do describe "anonymous user" do
it 'should be a 404' do it "should be a 404" do
get '/admin/upgrade' get "/admin/upgrade"
expect(response.status).to eq(404) expect(response.status).to eq(404)
end end
end end
describe 'when user is not an admin' do describe "when user is not an admin" do
it 'should 404' do it "should 404" do
sign_in(Fabricate(:user)) sign_in(Fabricate(:user))
get '/admin/upgrade' get "/admin/upgrade"
expect(response.status).to eq(404) expect(response.status).to eq(404)
end end
end end
describe 'when user is an admin' do describe "when user is an admin" do
it 'should return the right response' do it "should return the right response" do
sign_in(Fabricate(:admin)) sign_in(Fabricate(:admin))
get '/admin/upgrade' get "/admin/upgrade"
expect(response.status).to eq(200) expect(response.status).to eq(200)
end end
end end
describe '#repos' do describe "#repos" do
it 'should return the right response' do it "should return the right response" do
sign_in(Fabricate(:admin)) sign_in(Fabricate(:admin))
get '/admin/docker/repos' get "/admin/docker/repos"
expect(response.status).to eq(200) expect(response.status).to eq(200)
body = JSON.parse(response.body) body = JSON.parse(response.body)
expect(body["repos"].first["official"]).to eq(false) expect(body["repos"].first["official"]).to eq(false)