Merge pull request #336 from infosiftr/generate-tag-details.pl
Rewrite "generate-tag-details.sh" in Perl for better correctness and cooler features
This commit is contained in:
yosifkit
commit
51e19f679f
|
|
@ -0,0 +1,192 @@
|
|||
#!/usr/bin/env perl
|
||||
use strict;
|
||||
use warnings;
|
||||
use 5.010;
|
||||
use open ':encoding(utf8)';
|
||||
|
||||
use Mojo::UserAgent;
|
||||
|
||||
die 'no images specified' unless @ARGV;
|
||||
|
||||
my $ua = Mojo::UserAgent->new->max_redirects(10);
|
||||
$ua->transactor->name('Docker');
|
||||
|
||||
sub split_image_name {
|
||||
my $image = shift;
|
||||
if ($image =~ m{
|
||||
^
|
||||
(?: ([^/:]+) / )? # optional namespace
|
||||
([^/:]+) # image name
|
||||
(?: : ([^/:]+) )? # optional tag
|
||||
$
|
||||
}x) {
|
||||
my ($namespace, $name, $tag) = (
|
||||
$1 // 'library', # namespace
|
||||
$2, # image name
|
||||
$3 // 'latest', # tag
|
||||
);
|
||||
return ("$namespace/$name", $tag);
|
||||
}
|
||||
die "unrecognized image name format in: $image";
|
||||
}
|
||||
|
||||
sub get_token {
|
||||
my $repo = shift;
|
||||
state %tokens;
|
||||
return $tokens{$repo} if $tokens{$repo};
|
||||
my $realmTx = $ua->get("https://registry-1.docker.io/v2/$repo/tags/list");
|
||||
my $auth = $realmTx->res->headers->www_authenticate;
|
||||
die "unexpected WWW-Authenticate header: $auth" unless $auth =~ m{ ^ Bearer \s+ (\S.*) $ }x;
|
||||
my $realm = $1;
|
||||
my $url = Mojo::URL->new;
|
||||
while ($realm =~ m{
|
||||
# key="val",
|
||||
([^=]+)
|
||||
=
|
||||
"([^"]+)"
|
||||
,?
|
||||
}xg) {
|
||||
my ($key, $val) = ($1, $2);
|
||||
if ($key eq 'realm') {
|
||||
$url->base(Mojo::URL->new($val));
|
||||
} else {
|
||||
$url->query->append($key => $val);
|
||||
}
|
||||
}
|
||||
$url = $url->to_abs;
|
||||
my $tokenTx = $ua->get($url);
|
||||
die "failed to fetch token for $repo" unless $tokenTx->success;
|
||||
return $tokens{$repo} = $tokenTx->res->json->{token};
|
||||
}
|
||||
|
||||
sub get_manifest {
|
||||
my ($repo, $tag) = @_;
|
||||
my $image = "$repo:$tag";
|
||||
state %manifests;
|
||||
return $manifests{$image} if $manifests{$image};
|
||||
|
||||
my $token = get_token($repo);
|
||||
my $authorizationHeader = { Authorization => "Bearer $token" };
|
||||
|
||||
my $manifestTx = $ua->get("https://registry-1.docker.io/v2/$repo/manifests/$tag" => $authorizationHeader);
|
||||
die "failed to get manifest for $image" unless $manifestTx->success;
|
||||
return $manifests{$image} = $manifestTx->res->json;
|
||||
}
|
||||
|
||||
sub get_blob_headers {
|
||||
my ($repo, $blob) = @_;
|
||||
my $key = $repo . '@' . $blob;
|
||||
state %headers;
|
||||
return $headers{$key} if $headers{$key};
|
||||
|
||||
my $token = get_token($repo);
|
||||
my $authorizationHeader = { Authorization => "Bearer $token" };
|
||||
|
||||
my $headersTx = $ua->head("https://registry-1.docker.io/v2/$repo/blobs/$blob" => $authorizationHeader);
|
||||
die "failed to get headers for $key" unless $headersTx->success;
|
||||
return $headers{$key} = $headersTx->res->headers;
|
||||
}
|
||||
|
||||
sub get_layer_data {
|
||||
my ($repo, $id, $blob, $v1) = @_;
|
||||
$id //= $v1->{id};
|
||||
state %layers;
|
||||
return $layers{$id} if $layers{$id};
|
||||
die "missing v1/blob data for layer $id" unless $blob and $v1;
|
||||
my $data = {
|
||||
map({ $_ => $v1->{$_} } qw(id created parent docker_version)),
|
||||
container_command => $v1->{container_config}{Cmd},
|
||||
virtual_size => $v1->{Size},
|
||||
blob => $blob,
|
||||
};
|
||||
my $blobHeaders = get_blob_headers($repo, $blob);
|
||||
$data->{blob_content_length} = $blobHeaders->content_length;
|
||||
$data->{blob_last_modified} = $blobHeaders->last_modified;
|
||||
return $layers{$id} = $data;
|
||||
}
|
||||
|
||||
sub cmd_to_dockerfile {
|
||||
my ($cmd) = @_;
|
||||
if (scalar(@$cmd) == 3 && $cmd->[0] eq '/bin/sh' && $cmd->[1] eq '-c') {
|
||||
$cmd = $cmd->[2];
|
||||
if ($cmd =~ s{^(#[(]nop[)] )}{}) {
|
||||
return $cmd;
|
||||
}
|
||||
# prefix tabs and 4-space-indents with \ and a newline (for readability), but only if we don't already have any newlines
|
||||
$cmd =~ s/ ( (?:\t|[ ]{4})+ ) /\\\n$1/xg unless $cmd =~ m!\n!;
|
||||
return 'RUN ' . $cmd;
|
||||
}
|
||||
return 'RUN ' . Mojo::JSON::encode_json($cmd);
|
||||
}
|
||||
|
||||
my @humanSizeUnits = qw( B KB MB GB TB );
|
||||
my $humanSizeScale = 1000;
|
||||
sub human_size {
|
||||
my ($bytes) = @_;
|
||||
my $unit = 0;
|
||||
my $unitBytes = $bytes;
|
||||
while (($unitBytes = int($bytes / ($humanSizeScale ** $unit))) > $humanSizeScale) {
|
||||
last unless $humanSizeUnits[$unit + 1];
|
||||
++$unit;
|
||||
}
|
||||
return sprintf '%.1f %s', $bytes / ($humanSizeScale ** $unit), $humanSizeUnits[$unit];
|
||||
}
|
||||
|
||||
sub size {
|
||||
my $text = human_size(@_);
|
||||
$text .= " ($_[0] bytes)" unless $text =~ m! \s+ B $ !x;
|
||||
return $text;
|
||||
}
|
||||
|
||||
sub date {
|
||||
my $date = Mojo::Date->new(@_);
|
||||
return $date->to_string;
|
||||
}
|
||||
|
||||
while (my $image = shift) {
|
||||
print "\n";
|
||||
say '## `' . $image . '`';
|
||||
my ($repo, $tag) = split_image_name($image);
|
||||
|
||||
my $manifest = get_manifest($repo, $tag);
|
||||
my %parentChild;
|
||||
my %totals = (
|
||||
virtual_size => 0,
|
||||
blob_content_length => 0,
|
||||
);
|
||||
for my $i (0 .. $#{ $manifest->{fsLayers} }) {
|
||||
my $data = get_layer_data(
|
||||
$repo, undef,
|
||||
$manifest->{fsLayers}[$i]{blobSum},
|
||||
Mojo::JSON::decode_json($manifest->{history}[$i]{v1Compatibility}),
|
||||
);
|
||||
$parentChild{$data->{parent} // ''} = $data->{id};
|
||||
$totals{$_} += $data->{$_} for keys %totals;
|
||||
}
|
||||
print "\n";
|
||||
say "-\t" . 'Total Virtual Size: ' . size($totals{virtual_size});
|
||||
say "-\t" . 'Total v2 Content-Length: ' . size($totals{blob_content_length});
|
||||
print "\n";
|
||||
say '### Layers (' . scalar(keys %parentChild) . ')';
|
||||
my $cur = $parentChild{''};
|
||||
while ($cur) {
|
||||
print "\n";
|
||||
say '#### `' . $cur . '`';
|
||||
my $data = get_layer_data($repo, $cur);
|
||||
if ($data->{container_command}) {
|
||||
print "\n";
|
||||
say '```dockerfile';
|
||||
say cmd_to_dockerfile($data->{container_command});
|
||||
say '```';
|
||||
}
|
||||
print "\n";
|
||||
say "-\t" . 'Created: ' . date($data->{created}) if $data->{created};
|
||||
say "-\t" . 'Parent Layer: `' . $data->{parent} . '`' if $data->{parent};
|
||||
say "-\t" . 'Docker Version: ' . $data->{docker_version} if $data->{docker_version};
|
||||
say "-\t" . 'Virtual Size: ' . size($data->{virtual_size});
|
||||
say "-\t" . 'v2 Blob: `' . $data->{blob} . '`';
|
||||
say "-\t" . 'v2 Content-Length: ' . size($data->{blob_content_length});
|
||||
say "-\t" . 'v2 Last-Modified: ' . date($data->{blob_last_modified});
|
||||
$cur = $parentChild{$cur};
|
||||
}
|
||||
}
|
||||
|
|
@ -1,166 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
repo="$1"
|
||||
if [ -z "$repo" ]; then
|
||||
echo >&2 "usage: $0 repo"
|
||||
echo >&2 " ie: $0 hylang"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
lines="$(curl -fsSL 'https://raw.githubusercontent.com/docker-library/official-images/master/library/'"$repo" | grep -vE '^$|^#')"
|
||||
if [ -z "$lines" ]; then
|
||||
echo >&2 "Failed to read manifest file for $repo"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
IFS=$'\n'
|
||||
tags=( $(echo "$lines" | awk -F ': +' '{ print $1 }') )
|
||||
unset IFS
|
||||
|
||||
tokenUrl="$(curl -sSLD- "https://registry-1.docker.io/v2/library/$repo/tags/list" -o /dev/null | tr -d '\r' | awk -F ': +' 'tolower($1) == "www-authenticate" && gsub(/^Bearer\s+realm="/, "", $2) { sub(/",/, "?", $2); gsub(/"/, "", $2); gsub(/,/, "\\&", $2); print $2 }')"
|
||||
token="$(curl -fsSL "$tokenUrl" | tr -d '\r' | sed -r 's/^[{]|[}]$//g' | awk -v RS=',' -F '"' '$2 == "token" { print $4 }')"
|
||||
authHeader="Authorization: Bearer $token"
|
||||
|
||||
get_digest() {
|
||||
local tag="$1"
|
||||
curl -sSLD- "https://registry-1.docker.io/v2/library/$repo/manifests/$tag" --header "$authHeader" -o /dev/null | tr -d '\r' | awk -F ': +' 'tolower($1) == "docker-content-digest" { print $2 }'
|
||||
}
|
||||
|
||||
get_manifest() {
|
||||
local tag="$1"
|
||||
curl -sSL "https://registry-1.docker.io/v2/library/$repo/manifests/$tag" --header "$authHeader" | tr -d '\r'
|
||||
}
|
||||
|
||||
get_blob_headers() {
|
||||
local blob="$1"
|
||||
curl -sSL "https://registry-1.docker.io/v2/library/$repo/blobs/$blob" --header "$authHeader" --head | tr -d '\r'
|
||||
}
|
||||
|
||||
json_get_data() {
|
||||
local json="$1"; shift
|
||||
local data="$1"; shift
|
||||
local strip="$1"; shift || strip='^"|"$'
|
||||
echo "$json" | awk -F '\t' '$1 ~ /^'"$data"'$/ { gsub(/'"$strip"'/, "", $2); print $2 }'
|
||||
}
|
||||
|
||||
humanSizeUnits=( B KB MB GB TB )
|
||||
humanSizeScale=1
|
||||
human_size() {
|
||||
local bytes="$1"
|
||||
local unit=0
|
||||
local unitBytes="$1"
|
||||
while unitBytes="$(echo "scale=0; $bytes / (1000 ^ $unit)" | bc -l)" && [ "$unitBytes" -gt 1000 ]; do
|
||||
if [ ! "${humanSizeUnits[$(( unit + 1 ))]}" ]; then
|
||||
break
|
||||
fi
|
||||
unit="$(( unit + 1 ))"
|
||||
done
|
||||
echo "$(echo "scale=$humanSizeScale; $bytes / (1000 ^ $unit)" | bc -l) ${humanSizeUnits[$unit]}"
|
||||
}
|
||||
|
||||
size() {
|
||||
text="$(human_size "$1")"
|
||||
if [[ "$text" != *' B' ]]; then
|
||||
text+=" ($1 bytes)"
|
||||
fi
|
||||
echo "$text"
|
||||
}
|
||||
|
||||
pdate() {
|
||||
TZ=America/Los_Angeles date --date="$1" --rfc-2822
|
||||
}
|
||||
|
||||
jsonSh="$(curl -fsSL 'https://raw.githubusercontent.com/dominictarr/JSON.sh/ed3f9dd285ebd4183934adb54ea5a2fda6b25a98/JSON.sh')"
|
||||
|
||||
echo "# Tags of \`$repo\`"
|
||||
|
||||
# add a simple ToC
|
||||
echo
|
||||
for tag in "${tags[@]}"; do
|
||||
# GitHub heading anchors are strange
|
||||
href="${repo}:${tag}"
|
||||
href="${href//./}"
|
||||
href="${href//:/}"
|
||||
href="${href,,}"
|
||||
echo "- [\`$repo:$tag\`](#${href})"
|
||||
done
|
||||
|
||||
declare -A layerData=()
|
||||
|
||||
for tag in "${tags[@]}"; do
|
||||
echo
|
||||
echo "## \`$repo:$tag\`"
|
||||
digest="$(get_digest "$tag")"
|
||||
if [ "$digest" ]; then
|
||||
echo
|
||||
echo '```console'
|
||||
echo "$ docker pull $repo@$digest"
|
||||
echo '```'
|
||||
fi
|
||||
manifest="$(get_manifest "$tag")"
|
||||
if [ "$manifest" ]; then
|
||||
parsedManifest="$(echo "$manifest" | bash <(echo "$jsonSh") -l)"
|
||||
eval "declare -A layerBlobs=( $(echo "$parsedManifest" | awk -F '\t' 'gsub(/^\["fsLayers",/, "", $1) && gsub(/,"blobSum"\]$/, "", $1) { print "[" $1 "]=" $2 }') )"
|
||||
eval "declare -A layerV1s=( $(echo "$parsedManifest" | awk -F '\t' 'gsub(/^\["history",/, "", $1) && gsub(/,"v1Compatibility"\]$/, "", $1) { print "[" $1 "]=" $2 }') )"
|
||||
declare -A parentChild=()
|
||||
declare -A totals=(
|
||||
[Size]=0
|
||||
[content-length]=0
|
||||
)
|
||||
for i in "${!layerV1s[@]}"; do
|
||||
layerV1="${layerV1s[$i]%'\n'}" # lol \n
|
||||
parsedLayerV1="$(echo "$layerV1" | bash <(echo "$jsonSh"))"
|
||||
layerId="$(json_get_data "$parsedLayerV1" '\["id"\]')"
|
||||
if [ -z "${layerData[$layerId]}" ]; then
|
||||
layerData["$layerId"]=1
|
||||
for field in created parent docker_version Size; do
|
||||
layerData["${layerId}_${field}"]="$(json_get_data "$parsedLayerV1" '\["'"$field"'"\]')"
|
||||
done
|
||||
layerData["${layerId}_container_cmd"]="$(json_get_data "$parsedLayerV1" '\["container_config","Cmd"\]' '')"
|
||||
layerData["${layerId}_blob"]="${layerBlobs[$i]}"
|
||||
blobHeaders="$(get_blob_headers "${layerData["${layerId}_blob"]}")"
|
||||
for header in content-length last-modified; do
|
||||
layerData["${layerId}_${header}"]="$(echo "$blobHeaders" | awk -F ': +' 'tolower($1) == "'"$header"'" { print $2 }')"
|
||||
done
|
||||
fi
|
||||
parentChild["${layerData[${layerId}_parent]:-none}"]="$layerId"
|
||||
for field in Size content-length; do
|
||||
totals["$field"]="$(echo "${totals[$field]} + ${layerData[${layerId}_${field}]}" | bc)"
|
||||
done
|
||||
done
|
||||
echo
|
||||
echo "- Total Virtual Size: $(size "${totals[Size]}")"
|
||||
echo "- Total v2 Content-Length: $(size "${totals[content-length]}"); compressed"
|
||||
if [ "${#parentChild[@]}" -gt 0 ]; then
|
||||
echo
|
||||
echo "### Layers (${#parentChild[@]})"
|
||||
cur="${parentChild[none]}"
|
||||
while [ "$cur" ]; do
|
||||
echo
|
||||
echo "#### \`$cur\`"
|
||||
cmd="${layerData[${cur}_container_cmd]}"
|
||||
if [ "$cmd" ]; then
|
||||
cmd="${cmd//'\u0026'/'&'}"
|
||||
cmd="${cmd//'\u003c'/'<'}"
|
||||
cmd="${cmd//'\u003e'/'>'}"
|
||||
echo
|
||||
echo '```json'
|
||||
echo "$cmd"
|
||||
echo '```'
|
||||
fi
|
||||
echo
|
||||
echo "- Created: $(pdate "${layerData[${cur}_created]}")"
|
||||
if [ "${layerData[${cur}_parent]}" ]; then
|
||||
echo "- Parent Layer: \`${layerData[${cur}_parent]}\`"
|
||||
fi
|
||||
echo "- Docker Version: ${layerData[${cur}_docker_version]}"
|
||||
echo "- Virtual Size: $(size "${layerData[${cur}_Size]}")"
|
||||
echo "- v2 Blob: \`${layerData[${cur}_blob]}\`"
|
||||
echo "- v2 Content-Length: $(size "${layerData[${cur}_content-length]}"); compressed"
|
||||
echo "- v2 Last-Modified: $(pdate "${layerData[${cur}_last-modified]}")"
|
||||
cur="${parentChild[$cur]}"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
<!-- THIS FILE IS GENERATED VIA '.template-helpers/generate-tag-details.pl' -->
|
||||
|
||||
# Tags of `hello-world`
|
||||
|
||||
- [`hello-world:latest`](#hello-worldlatest)
|
||||
|
||||
## `hello-world:latest`
|
||||
|
||||
- Total Virtual Size: 960.0 B
|
||||
- Total v2 Content-Length: 633.0 B
|
||||
|
||||
### Layers (2)
|
||||
|
||||
#### `535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912`
|
||||
|
||||
```dockerfile
|
||||
COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /
|
||||
```
|
||||
|
||||
- Created: Thu, 06 Aug 2015 23:53:22 GMT
|
||||
- Docker Version: 1.7.1
|
||||
- Virtual Size: 960.0 B
|
||||
- v2 Blob: `sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb`
|
||||
- v2 Content-Length: 601.0 B
|
||||
- v2 Last-Modified: Fri, 07 Aug 2015 00:38:43 GMT
|
||||
|
||||
#### `af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c`
|
||||
|
||||
```dockerfile
|
||||
CMD ["/hello"]
|
||||
```
|
||||
|
||||
- Created: Thu, 06 Aug 2015 23:53:22 GMT
|
||||
- Parent Layer: `535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912`
|
||||
- Docker Version: 1.7.1
|
||||
- Virtual Size: 0.0 B
|
||||
- v2 Blob: `sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4`
|
||||
- v2 Content-Length: 32.0 B
|
||||
- v2 Last-Modified: Fri, 27 Mar 2015 17:18:47 GMT
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -4,18 +4,37 @@ set -eo pipefail
|
|||
cd "$(dirname "$(readlink -f "$BASH_SOURCE")")"
|
||||
helperDir='.template-helpers'
|
||||
|
||||
# going for "bashbrew: command not found"
|
||||
bashbrew --help > /dev/null
|
||||
|
||||
docker build -t docker-library-docs .
|
||||
|
||||
repos=( "$@" )
|
||||
if [ ${#repos[@]} -eq 0 ]; then
|
||||
repos=( */ )
|
||||
fi
|
||||
repos=( "${repos[@]%/}" )
|
||||
|
||||
script="$helperDir/generate-tag-details.pl"
|
||||
for repo in "${repos[@]}"; do
|
||||
echo -n "$repo ... "
|
||||
{
|
||||
echo "<!-- THIS FILE IS GENERATED BY '$helperDir/generate-tag-details.sh' -->"
|
||||
IFS=$'\n'
|
||||
tags=( $(bashbrew list "$repo") )
|
||||
unset IFS
|
||||
echo "<!-- THIS FILE IS GENERATED VIA '$script' -->"
|
||||
echo
|
||||
"$helperDir/generate-tag-details.sh" "$repo"
|
||||
echo "# Tags of \`$repo\`"
|
||||
echo
|
||||
# add a simple ToC
|
||||
for tag in "${tags[@]}"; do
|
||||
# GitHub heading anchors are strange
|
||||
href="${tag//./}"
|
||||
href="${href//:/}"
|
||||
href="#${href,,}"
|
||||
echo $'-\t[`'"$tag"'`]('"$href"')'
|
||||
done
|
||||
docker run -i --rm -v "$PWD":/wtf:ro -w /wtf --entrypoint "$script" docker-library-docs "${tags[@]}"
|
||||
} > "$repo/tag-details.md"
|
||||
echo 'done'
|
||||
done
|
||||
|
|
|
|||
Loading…
Reference in New Issue