diff --git a/put-multiarch/.dockerignore b/put-multiarch/.dockerignore deleted file mode 100644 index 0ef2312..0000000 --- a/put-multiarch/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -!*.pl diff --git a/put-multiarch/Dockerfile b/put-multiarch/Dockerfile deleted file mode 100644 index a46047b..0000000 --- a/put-multiarch/Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -FROM perl:5.28-slim - -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - wget \ - ; \ - rm -rf /var/lib/apt/lists/* - -# secure by default ♥ (thanks to sri!) -ENV PERL_CPANM_OPT --verbose --mirror https://cpan.metacpan.org -# TODO find a way to make --mirror-only / SSL work with backpan too :( -#RUN cpanm Digest::SHA Module::Signature -# TODO find a way to make --verify work with backpan as well :'( -#ENV PERL_CPANM_OPT $PERL_CPANM_OPT --verify - -# reinstall cpanm itself, for good measure -RUN cpanm App::cpanminus - -RUN set -eux; \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - gcc \ - libc-dev \ - libssl-dev \ - zlib1g-dev \ - ; \ - rm -rf /var/lib/apt/lists/*; \ - cpanm \ - EV \ - IO::Socket::IP \ - IO::Socket::Socks \ - Net::DNS::Native \ - ; \ -# the tests for IO::Socket::SSL like to hang... :( - cpanm --notest IO::Socket::SSL; \ - apt-mark auto '.*' > /dev/null; \ - apt-mark manual $savedAptMark > /dev/null; \ - apt-get purge -y --auto-remove - -# https://metacpan.org/pod/release/SRI/Mojolicious-7.94/lib/Mojo/IOLoop.pm#DESCRIPTION -ENV LIBEV_FLAGS 4 -# epoll (Linux) - -RUN cpanm Mojolicious@8.15 - -COPY put-multiarch.pl /usr/local/bin/ -ENTRYPOINT ["put-multiarch.pl"] diff --git a/put-multiarch/put-multiarch.pl b/put-multiarch/put-multiarch.pl deleted file mode 100755 index 4822542..0000000 --- a/put-multiarch/put-multiarch.pl +++ /dev/null @@ -1,465 +0,0 @@ -#!/usr/bin/env perl -use Mojo::Base -strict, -signatures; - -# this is a replacement for "bashbrew put-shared" (without "--single-arch") to combine many architecture-specific repositories into manifest lists in a separate repository -# for example, combining amd64/bash:latest, arm32v5/bash:latest, ..., s390x/bash:latest into a single library/bash:latest manifest list -# (in a more efficient way than manifest-tool can do generically such that we can reasonably do 3700+ no-op tag pushes individually in ~9 minutes) - -use Digest::SHA; -use Mojo::Promise; -use Mojo::UserAgent; -use Mojo::Util; - -my $publicProxy = $ENV{DOCKERHUB_PUBLIC_PROXY} || die 'missing DOCKERHUB_PUBLIC_PROXY env (https://github.com/tianon/dockerhub-public-proxy)'; -my $dryRun = ($ARGV[0] || '') eq '--dry-run'; -shift @ARGV if $dryRun; - -my $ua = Mojo::UserAgent->new->max_redirects(10)->connect_timeout(120)->inactivity_timeout(120); -$ua->transactor->name(join ' ', - # https://github.com/docker/docker/blob/v1.11.2/dockerversion/useragent.go#L13-L34 - 'docker/1.11.2', - 'go/1.6.2', - 'git-commit/v1.11.2', - 'kernel/4.4.11', - 'os/linux', - 'arch/amd64', - # BOGUS USER AGENTS FOR THE BOGUS USER AGENT THRONE -); - -sub ua_retry_simple_req_p ($method, $url, $tries = 10) { - --$tries; - my $lastTry = $tries < 1; - - my $methodP = lc($method) . '_p'; - my $prom = $ua->$methodP($url); - if (!$lastTry) { - $prom = $prom->then(sub ($tx) { - return $tx if !$tx->error || $tx->res->code == 404 || $tx->res->code == 401; - return ua_retry_simple_req_p($method, $url, $tries); - }, sub { - return ua_retry_simple_req_p($method, $url, $tries); - }); - } - return $prom; -} - -sub split_image_name ($image) { - if ($image =~ m{ - ^ - (?: ([^/:]+) / )? # optional namespace - ([^/:]+) # image name - (?: : ([^/:]+) )? # optional tag - $ - }x) { - my ($namespace, $name, $tag) = ( - $1 // 'library', # namespace - $2, # image name - $3 // 'latest', # tag - ); - return ($namespace, $name, $tag); - } - die "unrecognized image name format in: $image"; -} - -sub arch_to_platform ($arch) { - if ($arch =~ m{ - ^ - (?: ([^-]+) - )? # optional "os" prefix ("windows-", etc) - ([^-]+?) # "architecture" bit ("arm64", "s390x", etc) - (v[0-9]+)? # optional "variant" suffix ("v7", "v6", etc) - $ - }x) { - return ( - os => $1 // 'linux', - architecture => ( - $2 eq 'i386' - ? '386' - : ( - $2 eq 'arm32' - ? 'arm' - : $2 - ) - ), - ($3 ? (variant => $3) : ()), - ); - } - die "unrecognized architecture format in: $arch"; -} - -# TODO make this promise-based and non-blocking? -# https://github.com/jberger/Mojolicious-Plugin-TailLog/blob/master/lib/Mojolicious/Plugin/TailLog.pm#L16-L22 -# https://metacpan.org/pod/Capture::Tiny -# https://metacpan.org/pod/Mojo::IOLoop#subprocess -# https://metacpan.org/pod/IO::Async::Process -sub bashbrew (@) { - open my $fh, '-|', 'bashbrew', @_ or die "failed to run 'bashbrew': $!"; - local $/; - my $output = <$fh>; - close $fh or die "failed to close 'bashbrew'"; - chomp $output; - return $output; -} - -sub get_manifest_p ($org, $repo, $ref, $tries = 10) { - --$tries; - my $lastTry = $tries < 1; - - state %cache; - if ($ref =~ m!^sha256:! && $cache{$ref}) { - return Mojo::Promise->resolve($cache{$ref}); - } - - return ua_retry_simple_req_p(GET => "$publicProxy/v2/$org/$repo/manifests/$ref")->then(sub ($tx) { - return if $tx->res->code == 404 || $tx->res->code == 401; - - if (!$lastTry && $tx->res->code != 200) { - return get_manifest_p($org, $repo, $ref, $tries); - } - die "unexpected exit code fetching '$org/$repo:$ref': " . $tx->res->code unless $tx->res->code == 200; - - my $digest = $tx->res->headers->header('docker-content-digest') or die "'$org/$repo:$ref' is missing 'docker-content-digest' header"; - die "malformed 'docker-content-digest' header in '$org/$repo:$ref': '$digest'" unless $digest =~ m!^sha256:!; - - my $manifest = $tx->res->json or die "'$org/$repo:$ref' has bad or missing JSON"; - my $size = int($tx->res->headers->content_length); - my $verbatim = $tx->res->body; - - return $cache{$digest} = { - digest => $digest, - manifest => $manifest, - size => $size, - verbatim => $verbatim, - - mediaType => ( - $manifest->{schemaVersion} == 1 - ? 'application/vnd.docker.distribution.manifest.v1+json' - : ( - $manifest->{schemaVersion} == 2 - ? $manifest->{mediaType} - : die "unknown schemaVersion for '$org/$repo' at '$ref'" - ) - ), - }; - }); -} - -sub get_blob_p ($org, $repo, $ref, $tries = 10) { - die "unexpected blob reference for '$org/$repo': '$ref'" unless $ref =~ m!^sha256:!; - - --$tries; - my $lastTry = $tries < 1; - - state %cache; - return Mojo::Promise->resolve($cache{$ref}) if $cache{$ref}; - - return ua_retry_simple_req_p(GET => "$publicProxy/v2/$org/$repo/blobs/$ref")->then(sub ($tx) { - return if $tx->res->code == 404; - - if (!$lastTry && $tx->res->code != 200) { - return get_blob_p($org, $repo, $ref, $tries); - } - die "unexpected exit code fetching blob from '$org/$repo:$ref'': " . $tx->res->code unless $tx->res->code == 200; - - return $cache{$ref} = $tx->res->json; - }); -} - -sub head_manifest_p ($org, $repo, $ref) { - die "unexpected manifest reference for HEAD '$org/$repo': '$ref'" unless $ref =~ m!^sha256:!; - - my $cacheKey = "$org/$repo:$ref"; - state %cache; - return Mojo::Promise->resolve($cache{$cacheKey}) if $cache{$cacheKey}; - - return ua_retry_simple_req_p(HEAD => "$publicProxy/v2/$org/$repo/manifests/$ref")->then(sub ($tx) { - return 0 if $tx->res->code == 404 || $tx->res->code == 401; - die "unexpected exit code HEADing manifest '$cacheKey': " . $tx->res->code unless $tx->res->code == 200; - return $cache{$cacheKey} = 1; - }); -} - -sub head_blob_p ($org, $repo, $ref) { - die "unexpected blob reference for HEAD '$org/$repo': '$ref'" unless $ref =~ m!^sha256:!; - - my $cacheKey = "$org/$repo:$ref"; - state %cache; - return Mojo::Promise->resolve($cache{$cacheKey}) if $cache{$cacheKey}; - - return ua_retry_simple_req_p(HEAD => "$publicProxy/v2/$org/$repo/blobs/$ref")->then(sub ($tx) { - return 0 if $tx->res->code == 404 || $tx->res->code == 401; - die "unexpected exit code HEADing blob '$cacheKey': " . $tx->res->code unless $tx->res->code == 200; - return $cache{$cacheKey} = 1; - }); -} - -# get list of manifest list items and necessary blobs for a particular architecture -sub get_arch_p ($targetNamespace, $arch, $archNamespace, $repo, $tag) { - return get_manifest_p($archNamespace, $repo, $tag)->then(sub ($manifestData = undef) { - return unless $manifestData; - my ($digest, $manifest, $size) = ($manifestData->{digest}, $manifestData->{manifest}, $manifestData->{size}); - - my $mediaType = $manifestData->{mediaType}; - if ($mediaType eq 'application/vnd.docker.distribution.manifest.list.v2+json') { - # jackpot -- if it's already a manifest list, the hard work is done! - return ($archNamespace, $manifest->{manifests}); - } - if ($mediaType eq 'application/vnd.docker.distribution.manifest.v1+json' || $mediaType eq 'application/vnd.docker.distribution.manifest.v2+json') { - my $manifestListItem = { - mediaType => $mediaType, - size => $size, - digest => $digest, - platform => { - arch_to_platform($arch), - ($manifest->{'os.version'} ? ('os.version' => $manifest->{'os.version'}) : ()), - }, - }; - if ($manifestListItem->{platform}{os} eq 'windows' && !$manifestListItem->{platform}{'os.version'} && $mediaType eq 'application/vnd.docker.distribution.manifest.v2+json') { - # if we're on Windows, we need to make an effort to fetch the "os.version" value from the config for the platform object - return get_blob_p($archNamespace, $repo, $manifest->{config}{digest})->then(sub ($config = undef) { - if ($config && $config->{'os.version'}) { - $manifestListItem->{platform}{'os.version'} = $config->{'os.version'}; - } - return ($archNamespace, [ $manifestListItem ]); - }); - } - else { - return ($archNamespace, [ $manifestListItem ]); - } - } - - die "unknown mediaType '$mediaType' for '$archNamespace/$repo:$tag'"; - }); -} - -sub needed_artifacts_p ($targetNamespace, $sourceNamespace, $repo, $manifestDigest) { - return head_manifest_p($targetNamespace, $repo, $manifestDigest)->then(sub ($exists) { - return if $exists; - - return get_manifest_p($sourceNamespace, $repo, $manifestDigest)->then(sub ($manifestData = undef) { - return unless $manifestData; - - my $manifest = $manifestData->{manifest}; - my $schemaVersion = $manifest->{schemaVersion}; - my @blobs; - if ($schemaVersion == 1) { - push @blobs, map { $_->{blobSum} } @{ $manifest->{fsLayers} }; - } - elsif ($schemaVersion == 2) { - die "this should never happen: $manifest->{mediaType}" unless $manifest->{mediaType} eq 'application/vnd.docker.distribution.manifest.v2+json'; # sanity check - push @blobs, $manifest->{config}{digest}, map { $_->{urls} ? () : $_->{digest} } @{ $manifest->{layers} }; - } - else { - die "this should never happen: $schemaVersion"; # sanity check - } - - return Mojo::Promise->all( - Mojo::Promise->resolve([ $sourceNamespace, $repo, 'manifest', $manifestDigest ]), - Mojo::Promise->map({ concurrency => 3 }, sub ($blob) { - return head_blob_p($targetNamespace, $repo, $blob)->then(sub ($exists) { - return if $exists; - return $sourceNamespace, $repo, 'blob', $blob; - }); - }, @blobs), - )->then(sub { map { @$_ } @_ }); - }); - }); -} - -sub get_dockerhub_creds { - die 'missing DOCKER_CONFIG or HOME environment variable' unless $ENV{DOCKER_CONFIG} or $ENV{HOME}; - my $config = Mojo::File->new(($ENV{DOCKER_CONFIG} || ($ENV{HOME} . '/.docker')) . '/config.json')->slurp; - die 'missing or empty ".docker/config.json" file' unless $config; - my $json = Mojo::JSON::decode_json($config); - die 'invalid ".docker/config.json" file' unless $json && $json->{auths}; - for my $registry (keys %{ $json->{auths} }) { - my $auth = $json->{auths}{$registry}{auth}; - next unless $auth; - if ($registry eq 'https://index.docker.io/v1/' || $registry eq 'index.docker.io') { - $auth = Mojo::Util::b64_decode($auth); - return $auth if $auth && $auth =~ m!:!; - } - } - die 'failed to find credentials for Docker Hub in ".docker/config.json" file'; -} - -sub authenticated_registry_req_p ($method, $repos, $url, $contentType = undef, $payload = undef, $tries = 10) { - --$tries; - my $lastTry = $tries < 1; - - my %headers = ($contentType ? ('Content-Type' => $contentType) : ()); - - state %tokens; - if (my $token = $tokens{$repos}) { - $headers{Authorization} = "Bearer $token"; - } - - my $methodP = lc($method) . '_p'; - my $fullUrl = "https://registry-1.docker.io/v2/$url"; - my $prom = $ua->$methodP($fullUrl, \%headers, ($payload ? $payload : ())); - if (!$lastTry) { - $prom = $prom->then(sub ($tx) { - if (!$lastTry && $tx->res->code == 401) { - # "Unauthorized" -- we must need to go fetch a token for this registry request (so let's go do that, then retry the original registry request) - my $auth = $tx->res->headers->www_authenticate; - die "unexpected WWW-Authenticate header ('$url'): $auth" unless $auth =~ m{ ^ Bearer \s+ (\S.*) $ }x; - my $realm = $1; - my $authUrl = Mojo::URL->new; - while ($realm =~ m{ - # key="val", - ([^=]+) - = - "([^"]+)" - ,? - }xg) { - my ($key, $val) = ($1, $2); - next if $key eq 'error' and $val eq 'invalid_token'; # just ignore the error if it's "invalid_token" because it likely means our token expired mid-push so we just need to renew - die "WWW-Authenticate header error ('$url'): $val ($auth)" if $key eq 'error'; - if ($key eq 'realm') { - $authUrl->base(Mojo::URL->new($val)); - } - else { - $authUrl->query->append($key => $_) for split / /, $val; # Docker's auth server expects "scope=xxx&scope=yyy" instead of "scope=xxx%20yyy" - } - } - $authUrl = $authUrl->to_abs; - say {*STDERR} "Note: grabbing auth token from $authUrl (for $fullUrl; $tries tries remain)"; - my $dockerhubCreds = get_dockerhub_creds(); - return ua_retry_simple_req_p(GET => $authUrl->userinfo($dockerhubCreds)->to_unsafe_string)->then(sub ($tx) { - if (my $error = $tx->error) { - die "registry authentication error ('$url'): " . ($error->{code} ? $error->{code} . ' -- ' : '') . $error->{message}; - } - - $tokens{$repos} = $tx->res->json->{token}; - return authenticated_registry_req_p($method, $repos, $url, $contentType, $payload, $tries); - }); - } - - if (!$lastTry && $tx->res->code != 200) { - return authenticated_registry_req_p($method, $repos, $url, $contentType, $payload, $tries); - } - - if (my $error = $tx->error) { - $tx->req->headers->authorization('REDATCTED') if $tx->req->headers->authorization; - die "registry request error ('$url'): " . ($error->{code} ? $error->{code} . ' -- ' : '') . $error->{message} . "\n\nREQUEST:\n" . $tx->req->headers->to_string . "\n\n" . $tx->req->body . "\n\nRESPONSE:\n" . $tx->res->to_string . "\n"; - } - - return $tx; - }, sub { - return authenticated_registry_req_p($method, $repos, $url, $contentType, $payload, $tries); - }); - } - return $prom; -} - -Mojo::Promise->map({ concurrency => 8 }, sub ($img) { - my ($org, $repo, $tag) = split_image_name($img); - - die "image '$img' is missing explict namespace -- bailing to avoid accidental push to '$org'" unless $img =~ m!/!; - - my @tags = ( - $img =~ m/:/ - ? ( "$repo:$tag" ) - : ( List::Util::uniq sort split /\n/, bashbrew('list', $repo) ) - ); - return Mojo::Promise->resolve unless @tags; # no tags, nothing to do! (opensuse, etc) - - return Mojo::Promise->map({ concurrency => 1 }, sub ($repoTag) { - my (undef, $repo, $tag) = split_image_name($repoTag); - - my @arches = List::Util::uniq sort split /\n/, bashbrew('cat', '--format', '{{ range .Entries }}{{ range .Architectures }}{{ . }}={{ archNamespace . }}{{ "\n" }}{{ end }}{{ end }}', "$repo:$tag"); - return Mojo::Promise->resolve unless @arches; # no arches, nothing to do! - - return Mojo::Promise->map({ concurrency => 1 }, sub ($archData) { - my ($arch, $archNamespace) = split /=/, $archData; - return get_arch_p($org, $arch, $archNamespace, $repo, $tag); - }, @arches)->then(sub (@archResponses) { - my @manifestListItems; - my @neededArtifactPromises; - for my $archResponse (@archResponses) { - next unless @$archResponse; - my ($archNamespace, $manifestListItems) = @$archResponse; - push @manifestListItems, @$manifestListItems; - push @neededArtifactPromises, map { my $digest = $_->{digest}; sub { needed_artifacts_p($org, $archNamespace, $repo, $digest) } } @$manifestListItems; - } - - my $manifestList = { - schemaVersion => 2, - mediaType => 'application/vnd.docker.distribution.manifest.list.v2+json', - manifests => \@manifestListItems, - }; - my $manifestListJson = Mojo::JSON::encode_json($manifestList); - my $manifestListDigest = 'sha256:' . Digest::SHA::sha256_hex($manifestListJson); - - return head_manifest_p($org, $repo, $manifestListDigest)->then(sub ($exists) { - # if we already have the manifest we're planning to push in the namespace where we plan to push it, we can skip all blob mounts! \m/ - return if $exists; - # (we can also skip if we're in "dry run" mode since we only care about the final manifest matching in that case) - return if $dryRun; - - return ( - @neededArtifactPromises - ? Mojo::Promise->map({ concurrency => 1 }, sub { $_->() }, @neededArtifactPromises) - : Mojo::Promise->resolve - )->then(sub (@neededArtifacts) { - @neededArtifacts = map { @$_ } @neededArtifacts; - # now "@neededArtifacts" is a list of tuples of the format [ sourceNamespace, sourceRepo, type, digest ], ready for cross-repo mounting / PUTing (where type is "blob" or "manifest") - my @mountBlobPromises; - my @putManifestPromises; - for my $neededArtifact (@neededArtifacts) { - next unless @$neededArtifact; - my ($sourceNamespace, $sourceRepo, $type, $digest) = @$neededArtifact; - if ($type eq 'blob') { - # https://docs.docker.com/registry/spec/api/#mount-blob - push @mountBlobPromises, sub { authenticated_registry_req_p(POST => "$org/$repo:push,$sourceNamespace/$sourceRepo:pull" => "$org/$repo/blobs/uploads/?mount=$digest&from=$sourceNamespace/$sourceRepo") }; - } - elsif ($type eq 'manifest') { - push @putManifestPromises, sub { get_manifest_p($sourceNamespace, $sourceRepo, $digest)->then(sub ($manifestData = undef) { - return unless $manifestData; - return authenticated_registry_req_p(PUT => "$org/$repo:push" => "$org/$repo/manifests/$digest" => $manifestData->{mediaType} => $manifestData->{verbatim}); - }) }; - } - else { - die "this should never happen: $type"; # sanity check - } - } - - # mount any necessary blobs - return ( - @mountBlobPromises - ? Mojo::Promise->map({ concurrency => 1 }, sub { $_->() }, @mountBlobPromises) - : Mojo::Promise->resolve - )->then(sub { - # ... *then* push any missing image manifests (because they'll fail to push if the blobs aren't pushed first) - if (@putManifestPromises) { - return Mojo::Promise->map({ concurrency => 1 }, sub { $_->() }, @putManifestPromises); - } - return; - }); - }); - })->then(sub { - # let's do one final check of the tag we're pushing to see if it's already the manifest we expect it to be (to avoid making literally every image constantly "Updated a few seconds ago" all the time) - return get_manifest_p($org, $repo, $tag)->then(sub ($manifestData = undef) { - if ($manifestData && $manifestData->{digest} eq $manifestListDigest) { - say "Skipping $org/$repo:$tag ($manifestListDigest)" unless $dryRun; # if we're in "dry run" mode, we need clean output - return; - } - - if ($dryRun) { - say "Would push $org/$repo:$tag ($manifestListDigest)"; - return; - } - - # finally, all necessary blobs and manifests are pushed, we've verified that we do in fact need to push this manifest, so we should be golden to push it! - return authenticated_registry_req_p(PUT => "$org/$repo:push" => "$org/$repo/manifests/$tag" => $manifestList->{mediaType} => $manifestListJson)->then(sub ($tx) { - my $digest = $tx->res->headers->header('Docker-Content-Digest'); - say "Pushed $org/$repo:$tag ($digest)"; - say {*STDERR} "WARNING: expected '$manifestListDigest', got '$digest' (for '$org/$repo:$tag')" unless $manifestListDigest eq $digest; - }); - }); - }); - }); - }, @tags); -}, @ARGV)->catch(sub { - say {*STDERR} "ERROR: $_" for @_; - exit scalar @_; -})->wait; diff --git a/put-multiarch/put-multiarch.sh b/put-multiarch/put-multiarch.sh deleted file mode 100755 index 0d24e90..0000000 --- a/put-multiarch/put-multiarch.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -bashbrew="$(which bashbrew)" -bashbrewLibrary="${BASHBREW_LIBRARY:-$HOME/docker/official-images/library}" -[ -n "$BASHBREW_ARCH_NAMESPACES" ] - -dockerConfig="${DOCKER_CONFIG:-$HOME/.docker}" -[ -s "$dockerConfig/config.json" ] - -args=( - -v "$bashbrew":/usr/local/bin/bashbrew:ro - -v "$bashbrewLibrary":/library:ro - -e BASHBREW_LIBRARY=/library - -e BASHBREW_ARCH_NAMESPACES - - -v "$dockerConfig":/.docker:ro - -e DOCKER_CONFIG='/.docker' - - -e DOCKERHUB_PUBLIC_PROXY - - #-e MOJO_CLIENT_DEBUG=1 - #-e MOJO_IOLOOP_DEBUG=1 -) - -if [ -t 1 ]; then - args+=( -it ) -fi - -dir="$(dirname "$BASH_SOURCE")" -img="$(docker build -q -t oi/put-multiarch --cache-from oi/put-multiarch "$dir")" - -exec docker run --rm --init "${args[@]}" "$img" "$@"