Add Docker Hub categories (#2446)

* Add repo metadata, start with categories!

scripts for checking repo categories, updating the canonical set
added categories to push.pl

* Add initial set of semi-acurate categories

* Adjustments following tianon's review

* Simplify metadata.sh use cases (CI or interactive); just diff and check all the time

Update README.md about metadata.sh usage

* Unify the categories checks into one jq expressions

* Update initial categories

* Link to Docker docs for categories; minor categories script adjustments

.ci/check-metadata.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+cd "$(dirname "$(readlink -f "$BASH_SOURCE")")/.."
+
+# metadata.sh takes directories with a 'metadata.json' in them
+# metadata.json is expected in every repo
+# "." so that the canonical source metadata.json is checked too
+./metadata.sh */ .

.github/workflows/ci.yml

@@ -46,3 +46,8 @@ jobs:
           fetch-depth: 0
       - run: .ci/check-pr-no-readme.sh
     if: ${{ github.event_name == 'pull_request' }}
+  metadata:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: .ci/check-metadata.sh

README.md

@@ -61,6 +61,7 @@ After opening your Pull Request the changes will be checked by an automated `mar
 -	Create a `license.md` (required)
 -	Create a `maintainer.md` (required)
 -	Create a `github-repo` (required)
+-	Create a `metadata.json` (required)
 -	Add a `logo.png` (recommended)
 
 Optionally:
@@ -128,6 +129,18 @@ The image is automatically scaled to a 120 pixel square for the top of the Docke
 
 This file should contain a link to the maintainers of the Dockerfile.
 
+## `metadata.json`
+
+This file contains data about the repo for Docker Hub. The minimum file is defined below. `./metadata.sh [repo-name]` must be used to correctly format it (use `-w` to apply its suggested format changes). Only three sorted unique Docker Hub categories are allowed. `metadata.json` in the root contains the list of categories to choose from. See descriptions for the categories on the [Docker docs site](https://docs.docker.com/docker-hub/repos/categories/).
+
+```json
+{
+    "hub": {
+         "categories": []
+    }
+}
+```
+
 ## `README-short.txt`
 
 This is the short description for the Docker Hub, limited to 100 characters in a single line.

adminer/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

aerospike/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

almalinux/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

alpine/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

alt/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

amazoncorretto/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

amazonlinux/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

api-firewall/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "api-management"
+    ]
+  }
+}

arangodb/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

archlinux/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

backdrop/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "content-management-system"
+    ]
+  }
+}

bash/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "developer-tools"
+    ]
+  }
+}

bonita/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

buildpack-deps/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "developer-tools"
+    ]
+  }
+}

busybox/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

caddy/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "web-servers"
+    ]
+  }
+}

cassandra/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

centos/metadata.json

@@ -0,0 +1,5 @@
+{
+  "hub": {
+    "categories": []
+  }
+}

chronograf/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "monitoring-and-observability"
+    ]
+  }
+}

cirros/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

clearlinux/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

clefos/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

clojure/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

composer/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "developer-tools"
+    ]
+  }
+}

consul/metadata.json

@@ -0,0 +1,5 @@
+{
+  "hub": {
+    "categories": []
+  }
+}

convertigo/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

couchbase/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

couchdb/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

crate/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

dart/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

debian/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

docker/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "developer-tools"
+    ]
+  }
+}

drupal/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "content-management-system"
+    ]
+  }
+}

eclipse-mosquitto/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "message-queues"
+    ]
+  }
+}

eclipse-temurin/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

eggdrop/metadata.json

@@ -0,0 +1,5 @@
+{
+  "hub": {
+    "categories": []
+  }
+}

elasticsearch/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

elixir/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

emqx/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "message-queues"
+    ]
+  }
+}

erlang/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

express-gateway/metadata.json

@@ -0,0 +1,5 @@
+{
+  "hub": {
+    "categories": []
+  }
+}

fedora/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

flink/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "data-science"
+    ]
+  }
+}

fluentd/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "monitoring-and-observability"
+    ]
+  }
+}

friendica/metadata.json

@@ -0,0 +1,5 @@
+{
+  "hub": {
+    "categories": []
+  }
+}

gazebo/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

gcc/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

geonetwork/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "data-science"
+    ]
+  }
+}

get-categories.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+workdir="$(readlink -f "$BASH_SOURCE")"
+workdir="$(dirname "$workdir")"
+
+jsonFile='metadata.json'
+canonicalMetadataFile="$workdir/$jsonFile"
+
+# add categories slugs to canonicalMetadataFile without losing other keys there
+curl -fsSL https://hub.docker.com/v2/categories | jq -s --sort-keys '
+	.[0] as $allCategories
+	| .[1]
+	| .hub.categories = ( [ $allCategories[].slug ] | sort )
+' - "$canonicalMetadataFile" | tee "$canonicalMetadataFile.new"
+mv "$canonicalMetadataFile.new" "$canonicalMetadataFile"

ghost/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "content-management-system"
+    ]
+  }
+}

golang/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

gradle/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "developer-tools"
+    ]
+  }
+}

groovy/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

haproxy/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "web-servers"
+    ]
+  }
+}

haskell/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

haxe/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

hello-world/metadata.json

@@ -0,0 +1,5 @@
+{
+  "hub": {
+    "categories": []
+  }
+}

hitch/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "web-servers"
+    ]
+  }
+}

httpd/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "web-servers"
+    ]
+  }
+}

hylang/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

ibm-semeru-runtimes/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

ibmjava/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

influxdb/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

irssi/metadata.json

@@ -0,0 +1,5 @@
+{
+  "hub": {
+    "categories": []
+  }
+}

jetty/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "web-servers"
+    ]
+  }
+}

jobber/metadata.json

@@ -0,0 +1,5 @@
+{
+  "hub": {
+    "categories": []
+  }
+}

joomla/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "content-management-system"
+    ]
+  }
+}

jruby/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

julia/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

kapacitor/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "monitoring-and-observability"
+    ]
+  }
+}

kibana/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "monitoring-and-observability"
+    ]
+  }
+}

kong/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "api-management"
+    ]
+  }
+}

lightstreamer/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "message-queues"
+    ]
+  }
+}

liquibase/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "developer-tools"
+    ]
+  }
+}

logstash/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "monitoring-and-observability"
+    ]
+  }
+}

mageia/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "operating-systems"
+    ]
+  }
+}

mariadb/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

matomo/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "monitoring-and-observability"
+    ]
+  }
+}

maven/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

mediawiki/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "content-management-system"
+    ]
+  }
+}

memcached/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

metadata.json

@@ -0,0 +1,22 @@
+{
+  "hub": {
+    "categories": [
+      "api-management",
+      "content-management-system",
+      "data-science",
+      "databases-and-storage",
+      "developer-tools",
+      "integration-and-delivery",
+      "internet-of-things",
+      "languages-and-frameworks",
+      "machine-learning-and-ai",
+      "message-queues",
+      "monitoring-and-observability",
+      "networking",
+      "operating-systems",
+      "security",
+      "web-analytics",
+      "web-servers"
+    ]
+  }
+}

metadata.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+workdir="$(readlink -f "$BASH_SOURCE")"
+workdir="$(dirname "$workdir")"
+cd "$workdir"
+
+jsonFile='metadata.json'
+canonicalMetadataFile="./$jsonFile"
+export maxCategories=3
+
+self="$(basename "$0")"
+
+usage() {
+	cat <<EOUSAGE
+
+usage: $self [--write] REPO[...]
+   eg: $self debian
+       $self -w python
+
+This script checks a givens repo's metadata.json. It checks formating (providing a diff), checks categories, and can write the formatting changes.
+
+-h, --help          Print this help output and exit.
+-w, --write         Apply json formatting (run without to see the diff that would be applied).
+
+Arguments are the list of repos with a 'metadata.json' in them. 'metadata.json' is expected in every repo.
+'.' can also be passed to check the format of the canonical './metadata.json' at
+the root of the repo, but the max categories of '-c' is skipped for it.
+EOUSAGE
+}
+
+# arg handling
+opts="$(getopt -o 'hw' --long 'help,write' -- "$@" || { usage >&2 && false; })"
+eval set -- "$opts"
+
+write=
+
+while :; do
+	flag="$1"
+	shift
+	case "$flag" in
+		--help | -h) usage && exit 0 ;;
+		--write | -w) write=1 ;;
+		--) break ;;
+		*)
+			{
+				echo "error: unknown flag: $flag"
+				usage
+			} >&2
+			exit 1
+			;;
+	esac
+done
+
+repos=( "$@" )
+if [ "${#repos[@]}" -eq 0 ]; then
+	repos=( */ )
+fi
+repos=( "${repos[@]%/}" )
+
+failures=0
+for repo in "${repos[@]}"; do
+	repoFile="$repo/$jsonFile"
+	if [ ! -s "$repoFile" ]; then
+		echo >&2 "error: $repoFile does not exist or is empty"
+		(( failures++ )) || :
+		continue
+	fi
+
+	# sort object keys and pretty print with jq as our "cannonical json"
+	# sort categories array, no duplicates
+	if ! repoFileJson="$(jq -s --sort-keys '.[0] | .hub.categories |= unique' "$repoFile")"; then
+		echo >&2 "error parsing '$repoFile'; invalid JSON?"
+		(( failures++ )) || :
+		continue
+	fi
+	if ! filediff="$(diff -u "$repoFile" <(cat <<<"$repoFileJson"))"; then
+		cat <<<"$filediff"
+		if [ -n "$write" ]; then
+			cat <<<"$repoFileJson" > "$repoFile"
+		else
+			(( failures++ )) || :
+		fi
+	fi
+
+	# TODO also check for required keys and/or types?
+	# the canonicalMetadataFile doesn't have too many categories since it is the source of categories
+	# all other metadata.json files must not be more than maxCategories or have categories that aren't in the canonical set
+	if [ "$repoFile" != "$canonicalMetadataFile" ]; then
+		export repoFile
+		if errorText="$(jq -r --slurpfile canonical "$canonicalMetadataFile" '
+			.hub.categories
+			| (
+				length
+				| if . > (env.maxCategories | tonumber) then
+					"error: \(env.repoFile): too many categories: \(.) (max \(env.maxCategories))"
+				else empty end
+			),
+			(
+				. - $canonical[0].hub.categories
+				| if length > 0 then
+					"error: \(env.repoFile): unknown categories \(.)"
+				else empty end
+			)
+		' "$repoFile")"; then
+			if [ -n "$errorText" ]; then
+				echo >&2 "$errorText"
+				(( failures++ )) || :
+			fi
+		else
+			echo >&2 "error parsing '$repoFile'; invalid JSON?"
+			(( failures++ )) || :
+			continue
+		fi
+	fi
+done
+
+exit "$failures"

mongo-express/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

mongo/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

monica/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "content-management-system"
+    ]
+  }
+}

mono/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

mysql/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

nats-streaming/metadata.json

@@ -0,0 +1,5 @@
+{
+  "hub": {
+    "categories": []
+  }
+}

nats/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "message-queues"
+    ]
+  }
+}

neo4j/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "databases-and-storage"
+    ]
+  }
+}

neurodebian/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "data-science"
+    ]
+  }
+}

nextcloud/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "content-management-system"
+    ]
+  }
+}

nginx/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "web-servers"
+    ]
+  }
+}

node/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "languages-and-frameworks"
+    ]
+  }
+}

notary/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "security"
+    ]
+  }
+}

odoo/metadata.json

@@ -0,0 +1,7 @@
+{
+  "hub": {
+    "categories": [
+      "content-management-system"
+    ]
+  }
+}