discourse
/
docker_manager
mirror of https://github.com/discourse/docker_manager.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DEV: Run stree

This commit is contained in:
Jarek Radosz 2023-01-15 19:56:52 +01:00
parent 96211d622e
commit 4ab63aca32
7 changed files with 169 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
app/controllers/docker_manager/admin_controller.rb
View File

@ -1,30 +1,36 @@
# frozen_string_literal: true
require_dependency 'docker_manager/git_repo'
require_dependency 'docker_manager/upgrader'
require_dependency "docker_manager/git_repo"
require_dependency "docker_manager/upgrader"
module DockerManager
class AdminController < DockerManager::ApplicationController
layout nil
class AdminController < Admin::AdminController
helper DockerManager::ApplicationHelper
def index
return if Rails.env.development?
version = File.read('/VERSION') rescue '1.0.0'
version =
begin
File.read("/VERSION")
rescue StandardError
"1.0.0"
end
version = Gem::Version.new(version)
expected_version = Gem::Version.new('2.0.20220128-1817')
expected_version = Gem::Version.new("2.0.20220128-1817")
ruby_version = Gem::Version.new(RUBY_VERSION)
expected_ruby_version = Gem::Version.new('3.1.3')
min_stable_version = Gem::Version.new('2.8.7')
min_beta_version = Gem::Version.new('2.9.0.beta8')
expected_ruby_version = Gem::Version.new("3.1.3")
min_stable_version = Gem::Version.new("2.8.7")
min_beta_version = Gem::Version.new("2.9.0.beta8")
upgrade_image = version < expected_version
upgrade_ruby = ruby_version < expected_ruby_version
upgrade_discourse = discourse_upgrade_required?(min_stable_version, min_beta_version)
upgrade_discourse =
discourse_upgrade_required?(min_stable_version, min_beta_version)
if upgrade_image || upgrade_ruby || upgrade_discourse
render 'upgrade_required', layout: false
render "upgrade_required", layout: false
else
render
end
@ -40,10 +46,16 @@ module DockerManager
official: Plugin::Metadata::OFFICIAL_PLUGINS.include?(r.name)
}
result[:fork] = true if result[:official] && !r.url.starts_with?("https://github.com/discourse/")
result[:fork] = true if result[:official] &&
!r.url.starts_with?("https://github.com/discourse/")
if r.valid?
result[:id] = r.name.downcase.gsub(/[^a-z]/, '_').gsub(/_+/, '_').sub(/_$/, '')
result[:id] = r
.name
.downcase
.gsub(/[^a-z]/, "_")
.gsub(/_+/, "_")
.sub(/_$/, "")
result[:version] = r.latest_local_commit
result[:pretty_version] = r.latest_local_tag_version.presence
result[:url] = r.url
@ -63,12 +75,16 @@ module DockerManager
return respond_progress if repo.blank?
upgrader = Upgrader.new(current_user.id, repo, repo_version(repo))
respond_progress(logs: upgrader.find_logs, percentage: upgrader.last_percentage)
respond_progress(
logs: upgrader.find_logs,
percentage: upgrader.last_percentage
)
end
def latest
proc = Proc.new do |repo|
repo.update_remote! if Rails.env == 'production'
proc =
Proc.new do |repo|
repo.update_remote! if Rails.env == "production"
{
path: repo.path,
version: repo.latest_origin_commit,
@ -79,37 +95,40 @@ module DockerManager
end
if all_repos?
return render json: {
repos: DockerManager::GitRepo.find_all.map(&proc)
}
return(
render json: { repos: DockerManager::GitRepo.find_all.map(&proc) }
)
end
repo = DockerManager::GitRepo.find(params[:path])
raise Discourse::NotFound unless repo.present?
render json: {
latest: proc.call(repo)
}
render json: { latest: proc.call(repo) }
end
def upgrade
repo = find_repos(params[:path])
raise Discourse::NotFound unless repo.present?
script_path = File.expand_path(File.join(__dir__, '../../../scripts/docker_manager_upgrade.rb'))
script_path =
File.expand_path(
File.join(__dir__, "../../../scripts/docker_manager_upgrade.rb")
)
env_vars = {
'UPGRADE_USER_ID' => current_user.id.to_s,
'UPGRADE_PATH' => params[:path].to_s,
'UPGRADE_REPO_VERSION' => repo_version(repo).to_s,
'RAILS_ENV' => Rails.env
"UPGRADE_USER_ID" => current_user.id.to_s,
"UPGRADE_PATH" => params[:path].to_s,
"UPGRADE_REPO_VERSION" => repo_version(repo).to_s,
"RAILS_ENV" => Rails.env
}
["http_proxy", "https_proxy", "no_proxy", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"].each do |p|
env_vars[p] = ENV[p] if ! ENV[p].nil?
end
pid = spawn(
env_vars,
"bundle exec rails runner #{script_path}"
)
%w[
http_proxy
https_proxy
no_proxy
HTTP_PROXY
HTTPS_PROXY
NO_PROXY
].each { |p| env_vars[p] = ENV[p] if !ENV[p].nil? }
pid = spawn(env_vars, "bundle exec rails runner #{script_path}")
Process.detach(pid)
render plain: "OK"
end
@ -138,9 +157,7 @@ module DockerManager
end
def self.find_repos(path, upgrading: false, all: false)
unless all_repos?(path)
return DockerManager::GitRepo.find(path)
end
return DockerManager::GitRepo.find(path) unless all_repos?(path)
repos = DockerManager::GitRepo.find_all
return repos if all
@ -149,7 +166,8 @@ module DockerManager
if upgrading
repo.upgrading?
else
!repo.upgrading? && (repo.latest_local_commit != repo.latest_origin_commit)
!repo.upgrading? &&
(repo.latest_local_commit != repo.latest_origin_commit)
end
end
end
@ -157,12 +175,7 @@ module DockerManager
private
def respond_progress(logs: nil, percentage: nil)
render json: {
progress: {
logs: logs,
percentage: percentage
}
}
render json: { progress: { logs: logs, percentage: percentage } }
end
def all_repos?
@ -174,7 +187,11 @@ module DockerManager
end
def repo_version(repo)
repo.is_a?(Array) && params[:version].blank? ? concat_repos_versions(repo) : params[:version]
if repo.is_a?(Array) && params[:version].blank?
concat_repos_versions(repo)
else
params[:version]
end
end
def concat_repos_versions(repos)

32
lib/docker_manager/git_repo.rb
View File

@ -75,20 +75,26 @@ class DockerManager::GitRepo
def self.find_all
repos = [
DockerManager::GitRepo.new(Rails.root.to_s, 'discourse'),
DockerManager::GitRepo.new("#{Rails.root}/plugins/docker_manager", "docker_manager")
DockerManager::GitRepo.new(Rails.root.to_s, "discourse"),
DockerManager::GitRepo.new(
"#{Rails.root}/plugins/docker_manager",
"docker_manager"
)
]
p = Proc.new { |x|
p =
Proc.new do |x|
next if x.name == "docker_manager"
repos << DockerManager::GitRepo.new(File.dirname(x.path), x.name)
}
end
if Discourse.respond_to?(:visible_plugins)
Discourse.visible_plugins.each(&p)
else
Discourse.plugins.each(&p)
end
repos
repos
end
def self.find(path)
@ -96,7 +102,10 @@ class DockerManager::GitRepo
end
def upstream_branch
@upstream_branch ||= run("for-each-ref --format='%(upstream:short)' $(git symbolic-ref -q HEAD)")
@upstream_branch ||=
run(
"for-each-ref --format='%(upstream:short)' $(git symbolic-ref -q HEAD)"
)
end
def has_local_main?
@ -109,9 +118,7 @@ class DockerManager::GitRepo
result = run(command)
return unless result.present?
if result =~ /-(\d+)-/
result = result.gsub(/-(\d+)-.*/, " +#{$1}")
end
result = result.gsub(/-(\d+)-.*/, " +#{$1}") if result =~ /-(\d+)-/
result
end
@ -125,7 +132,11 @@ class DockerManager::GitRepo
end
def has_origin_main?
run("branch -a").match?(/remotes\/origin\/main$/) rescue false
begin
run("branch -a").match?(%r{remotes/origin/main$})
rescue StandardError
false
end
end
def tracking_branch
@ -150,5 +161,4 @@ class DockerManager::GitRepo
rescue => e
p e
end
end

74
lib/docker_manager/upgrader.rb
View File

@ -1,7 +1,6 @@
# frozen_string_literal: true
class DockerManager::Upgrader
def initialize(user_id, repos, from_version)
@user_id = user_id
@repos = repos.is_a?(Array) ? repos : [repos]
@ -19,9 +18,7 @@ class DockerManager::Upgrader
end
def upgrade
@repos.each do |repo|
return unless repo.start_upgrading
end
@repos.each { |repo| return unless repo.start_upgrading }
percent(0)
@ -68,7 +65,6 @@ class DockerManager::Upgrader
# HEAD@{upstream} is just a fancy way how to say origin/master (in normal case)
# see http://stackoverflow.com/a/12699604/84283
@repos.each_with_index do |repo, index|
# We automatically handle renames from `master` -> `main`
if repo.upstream_branch == "origin/master" && repo.branch == "origin/main"
log "Branch has changed to #{repo.branch}"
@ -84,7 +80,9 @@ class DockerManager::Upgrader
run "cd #{repo.path} && git branch -u origin/main main"
run("cd #{repo.path} && git reset --hard HEAD@{upstream}")
else
run("cd #{repo.path} && git fetch --tags --force && git reset --hard HEAD@{upstream}")
run(
"cd #{repo.path} && git fetch --tags --force && git reset --hard HEAD@{upstream}"
)
end
percent(20 * (index + 1) / @repos.size)
@ -101,10 +99,13 @@ class DockerManager::Upgrader
run("SKIP_POST_DEPLOYMENT_MIGRATIONS=1 bundle exec rake multisite:migrate")
percent(40)
log("*** Bundling assets. This will take a while *** ")
less_memory_flags = "RUBY_GC_MALLOC_LIMIT_MAX=20971520 RUBY_GC_OLDMALLOC_LIMIT_MAX=20971520 RUBY_GC_HEAP_GROWTH_MAX_SLOTS=50000 RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR=0.9 "
less_memory_flags =
"RUBY_GC_MALLOC_LIMIT_MAX=20971520 RUBY_GC_OLDMALLOC_LIMIT_MAX=20971520 RUBY_GC_HEAP_GROWTH_MAX_SLOTS=50000 RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR=0.9 "
run("#{less_memory_flags} bundle exec rake themes:update assets:precompile")
using_s3_assets = ENV["DISCOURSE_USE_S3"] && ENV["DISCOURSE_S3_BUCKET"] && ENV["DISCOURSE_S3_CDN_URL"]
using_s3_assets =
ENV["DISCOURSE_USE_S3"] && ENV["DISCOURSE_S3_BUCKET"] &&
ENV["DISCOURSE_S3_CDN_URL"]
if using_s3_assets
run("#{less_memory_flags} bundle exec rake s3:upload_assets")
@ -133,16 +134,15 @@ class DockerManager::Upgrader
log_version_upgrade
percent(100)
log("DONE")
publish('status', 'complete')
publish("status", "complete")
rescue => ex
publish('status', 'failed')
publish("status", "failed")
[
"Docker Manager: FAILED TO UPGRADE",
ex.inspect,
ex.backtrace.join("\n"),
ex.backtrace.join("\n")
].each do |message|
STDERR.puts(message)
log(message)
end
@ -158,7 +158,8 @@ class DockerManager::Upgrader
end
def publish(type, value)
MessageBus.publish("/docker/upgrade",
MessageBus.publish(
"/docker/upgrade",
{ type: type, value: value },
user_ids: [@user_id]
)
@ -168,7 +169,7 @@ class DockerManager::Upgrader
log "$ #{cmd}"
msg = +""
allowed_env = %w{
allowed_env = %w[
PWD
HOME
SHELL
@ -181,20 +182,24 @@ class DockerManager::Upgrader
http_proxy
https_proxy
no_proxy
}
]
clear_env = Hash[*ENV.map { |k, v| [k, nil] }
.reject { |k, v|
allowed_env.include?(k) ||
k =~ /^DISCOURSE_/
}
.flatten]
clear_env =
Hash[
*ENV
.map { |k, v| [k, nil] }
.reject { |k, v| allowed_env.include?(k) || k =~ /^DISCOURSE_/ }
.flatten
]
clear_env["RAILS_ENV"] = "production"
clear_env["TERM"] = 'dumb' # claim we have a terminal
clear_env["TERM"] = "dumb" # claim we have a terminal
retval = nil
Open3.popen2e(clear_env, "cd #{Rails.root} && #{cmd} 2>&1") do |_in, out, wait_thread|
Open3.popen2e(
clear_env,
"cd #{Rails.root} && #{cmd} 2>&1"
) do |_in, out, wait_thread|
out.each do |line|
line.rstrip! # the client adds newlines, so remove the one we're given
log(line)
@ -233,18 +238,18 @@ class DockerManager::Upgrader
def percent(val)
Discourse.redis.set(percent_key, val)
Discourse.redis.expire(percent_key, 30.minutes)
publish('percent', val)
publish("percent", val)
end
def log(message)
Discourse.redis.append logs_key, message + "\n"
Discourse.redis.expire(logs_key, 30.minutes)
publish 'log', message
publish "log", message
end
def log_version_upgrade
StaffActionLogger.new(User.find(@user_id)).log_custom(
'discourse_upgrade',
"discourse_upgrade",
from_version: @from_version,
repository: @repos.map(&:path).join(", ")
)
@ -267,13 +272,13 @@ class DockerManager::Upgrader
end
def unicorn_workers(master_pid)
`ps -f --ppid #{master_pid} | grep worker | awk '{ print $2 }'`
.split("\n")
.map(&:to_i)
`ps -f --ppid #{master_pid} | grep worker | awk '{ print $2 }'`.split(
"\n"
).map(&:to_i)
end
def local_web_url
"http://127.0.0.1:#{ENV['UNICORN_PORT'] || 3000}/srv/status"
"http://127.0.0.1:#{ENV["UNICORN_PORT"] || 3000}/srv/status"
end
def reload_unicorn(launcher_pid)
@ -282,20 +287,19 @@ class DockerManager::Upgrader
Process.kill("USR2", launcher_pid)
iterations = 0
while pid_exists?(original_master_pid) do
while pid_exists?(original_master_pid)
iterations += 1
break if iterations >= 60
log("Waiting for Unicorn to reload#{'.' * iterations}")
log("Waiting for Unicorn to reload#{"." * iterations}")
sleep 2
end
iterations = 0
while `curl -s #{local_web_url}` != "ok" do
while `curl -s #{local_web_url}` != "ok"
iterations += 1
break if iterations >= 60
log("Waiting for Unicorn workers to start up#{'.' * iterations}")
log("Waiting for Unicorn workers to start up#{"." * iterations}")
sleep 2
end
end
end

4
plugin.rb
View File

@ -9,7 +9,9 @@
module ::DockerManager
# should be automatic, but something is weird
load File.expand_path(File.dirname(__FILE__)) + '/app/helpers/application_helper.rb'
load File.expand_path(File.dirname(__FILE__)) +
"/app/helpers/application_helper.rb"
class Engine < ::Rails::Engine
engine_name "docker_manager"
isolate_namespace DockerManager

8
scripts/docker_manager_upgrade.rb
View File

@ -5,11 +5,11 @@
# process from getting killed if/when unicorn gets killed.
fork do
Process.setsid
require_relative '../lib/docker_manager/upgrader.rb'
require_relative "../lib/docker_manager/upgrader.rb"
user_id = ENV['UPGRADE_USER_ID'].to_i
path = ENV['UPGRADE_PATH']
repo_version = ENV['UPGRADE_REPO_VERSION']
user_id = ENV["UPGRADE_USER_ID"].to_i
path = ENV["UPGRADE_PATH"]
repo_version = ENV["UPGRADE_REPO_VERSION"]
raise "user_id is required" if user_id <= 0
raise "path is required" if path.blank?

25
spec/lib/git_repo_spec.rb
View File

@ -1,10 +1,9 @@
# frozen_string_literal: true
require 'rails_helper'
require 'docker_manager/git_repo'
require "rails_helper"
require "docker_manager/git_repo"
RSpec.describe DockerManager::GitRepo do
describe ".find_all" do
it "returns a list of repos" do
expect(described_class.find_all).to be_present
@ -23,27 +22,37 @@ RSpec.describe DockerManager::GitRepo do
end
describe "#branch" do
it "returns origin/master if a repo hasn't been renamed" do
described_class.any_instance.stubs(:upstream_branch).returns("origin/master")
described_class
.any_instance
.stubs(:upstream_branch)
.returns("origin/master")
described_class.any_instance.stubs(:has_origin_main?).returns(false)
repo = described_class.new("dummy", "dummy")
expect(repo.branch).to eq("origin/master")
end
it "returns origin/main if a repo has been renamed but still tracks master" do
described_class.any_instance.stubs(:upstream_branch).returns("origin/master")
described_class
.any_instance
.stubs(:upstream_branch)
.returns("origin/master")
described_class.any_instance.stubs(:has_origin_main?).returns(true)
repo = described_class.new("dummy", "dummy")
expect(repo.branch).to eq("origin/main")
end
it "returns origin/main if a repo points at origin/main" do
described_class.any_instance.stubs(:upstream_branch).returns("origin/main")
described_class
.any_instance
.stubs(:upstream_branch)
.returns("origin/main")
described_class.any_instance.stubs(:has_origin_main?).returns(true)
repo = described_class.new("dummy", "dummy")
expect(repo.branch).to eq("origin/main")
end
end
end

26
spec/requests/admin_controller_spec.rb
View File

@ -1,39 +1,39 @@
# frozen_string_literal: true
require 'rails_helper'
require "rails_helper"
RSpec.describe DockerManager::AdminController do
describe 'anonymous user' do
it 'should be a 404' do
get '/admin/upgrade'
describe "anonymous user" do
it "should be a 404" do
get "/admin/upgrade"
expect(response.status).to eq(404)
end
end
describe 'when user is not an admin' do
it 'should 404' do
describe "when user is not an admin" do
it "should 404" do
sign_in(Fabricate(:user))
get '/admin/upgrade'
get "/admin/upgrade"
expect(response.status).to eq(404)
end
end
describe 'when user is an admin' do
it 'should return the right response' do
describe "when user is an admin" do
it "should return the right response" do
sign_in(Fabricate(:admin))
get '/admin/upgrade'
get "/admin/upgrade"
expect(response.status).to eq(200)
end
end
describe '#repos' do
it 'should return the right response' do
describe "#repos" do
it "should return the right response" do
sign_in(Fabricate(:admin))
get '/admin/docker/repos'
get "/admin/docker/repos"
expect(response.status).to eq(200)
body = JSON.parse(response.body)
expect(body["repos"].first["official"]).to eq(false)