Use the Hub v2 API for a dramatic no-op push speedup

This also has the benefit of only triggering automated builds when there have been actual changes to the image.

This works by comparing the Hub's `last_updated` timestamp to the `Created` timestamp of local images (in the case of `bashbrew push`), or to the `last_updated` timestamp of the images which make up the `SharedTags` and/or multiarch images (in the case of `bashbrew put-shared`).

This is a dramatic speedup because the Hub v2 API is faster (and has no auth for public repos) than the registry API.  If there are any errors in the fetching of `last_updated` from the Hub, we massage the timestamps such that the push simply always happens, as before (which will be the case for pushes to non-Hub registries, for example).

go/src/bashbrew/cmd-push.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/codegangsta/cli"
 )
@@ -31,10 +32,16 @@ func cmdPush(c *cli.Context) error {
 			}
 
 			for _, tag := range r.Tags(namespace, uniq, entry) {
-				fmt.Printf("Pushing %s\n", tag)
+				created := dockerCreated(tag)
-				err = dockerPush(tag)
+				lastUpdated := fetchDockerHubTagMeta(tag).lastUpdatedTime()
-				if err != nil {
+				if created.After(lastUpdated) {
-					return cli.NewMultiError(fmt.Errorf(`failed pushing %q`, tag), err)
+					fmt.Printf("Pushing %s\n", tag)
+					err = dockerPush(tag)
+					if err != nil {
+						return cli.NewMultiError(fmt.Errorf(`failed pushing %q`, tag), err)
+					}
+				} else {
+					fmt.Printf("Skipping %s (created %s, last updated %s)\n", tag, created.Local().Format(time.RFC3339), lastUpdated.Local().Format(time.RFC3339))
 				}
 			}
 		}

go/src/bashbrew/cmd-put-shared.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"time"
 
 	"github.com/codegangsta/cli"
 
@@ -11,8 +12,9 @@ import (
 	"github.com/docker-library/go-dockerlibrary/manifest"
 )
 
-func entriesToManifestToolYaml(r Repo, entries ...*manifest.Manifest2822Entry) (string, error) {
+func entriesToManifestToolYaml(r Repo, entries ...*manifest.Manifest2822Entry) (string, time.Time, error) {
 	yaml := ""
+	mru := time.Time{}
 	entryIdentifiers := []string{}
 	for _, entry := range entries {
 		entryIdentifiers = append(entryIdentifiers, r.EntryIdentifier(*entry))
@@ -32,7 +34,13 @@ func entriesToManifestToolYaml(r Repo, entries ...*manifest.Manifest2822Entry) (
 				continue
 			}
 
-			yaml += fmt.Sprintf("  - image: %s/%s:%s\n    platform:\n", archNamespace, r.RepoName, entry.Tags[0])
+			archImage := fmt.Sprintf("%s/%s:%s", archNamespace, r.RepoName, entry.Tags[0])
+			archImageMeta := fetchDockerHubTagMeta(archImage)
+			if archU := archImageMeta.lastUpdatedTime(); archU.After(mru) {
+				mru = archU
+			}
+
+			yaml += fmt.Sprintf("  - image: %s\n    platform:\n", archImage)
 			yaml += fmt.Sprintf("      os: %s\n", ociArch.OS)
 			yaml += fmt.Sprintf("      architecture: %s\n", ociArch.Architecture)
 			if ociArch.Variant != "" {
@@ -41,10 +49,10 @@ func entriesToManifestToolYaml(r Repo, entries ...*manifest.Manifest2822Entry) (
 		}
 	}
 	if yaml == "" {
-		return "", fmt.Errorf("failed gathering images for creating %q", entryIdentifiers)
+		return "", time.Time{}, fmt.Errorf("failed gathering images for creating %q", entryIdentifiers)
 	}
 
-	return "manifests:\n" + yaml, nil
+	return "manifests:\n" + yaml, mru, nil
 }
 
 func tagsToManifestToolYaml(repo string, tags ...string) string {
@@ -79,40 +87,50 @@ func cmdPutShared(c *cli.Context) error {
 		targetRepo := path.Join(namespace, r.RepoName)
 
 		// handle all multi-architecture tags first (regardless of whether they have SharedTags)
+		// turn them into SharedTagGroup objects so all manifest-tool invocations can be handled by a single process/loop
+		sharedTagGroups := []manifest.SharedTagGroup{}
 		for _, entry := range r.Entries() {
-			// "image:" will be added later so we don't have to regenerate the entire "manifests" section every time
+			sharedTagGroups = append(sharedTagGroups, manifest.SharedTagGroup{
-			yaml, err := entriesToManifestToolYaml(*r, &entry)
+				SharedTags: entry.Tags,
-			if err != nil {
+				Entries:    []*manifest.Manifest2822Entry{&entry},
-				return err
+			})
-			}
-
-			entryIdentifier := fmt.Sprintf("%s:%s", targetRepo, entry.Tags[0])
-			fmt.Printf("Putting %s\n", entryIdentifier)
-			tagYaml := tagsToManifestToolYaml(targetRepo, entry.Tags...) + yaml
-			if err := manifestToolPushFromSpec(tagYaml); err != nil {
-				return fmt.Errorf("failed pushing %q (%q)", entryIdentifier, entry.TagsString())
-			}
 		}
 
-		// TODO do something better with r.TagName (ie, the user has done something crazy like "bashbrew put-shared single-repo:single-tag")
+		// TODO do something smarter with r.TagName (ie, the user has done something crazy like "bashbrew put-shared single-repo:single-tag")
-		sharedTagGroups := r.Manifest.GetSharedTagGroups()
+		if r.TagName == "" {
-		if len(sharedTagGroups) == 0 {
+			sharedTagGroups = append(sharedTagGroups, r.Manifest.GetSharedTagGroups()...)
-			continue
+		} else {
-		}
-		if r.TagName != "" {
 			fmt.Fprintf(os.Stderr, "warning: a single tag was requested -- skipping SharedTags\n")
+		}
+
+		if len(sharedTagGroups) == 0 {
 			continue
 		}
 
 		for _, group := range sharedTagGroups {
-			yaml, err := entriesToManifestToolYaml(*r, group.Entries...)
+			yaml, mostRecentPush, err := entriesToManifestToolYaml(*r, group.Entries...)
 			if err != nil {
 				return err
 			}
 
-			groupIdentifier := fmt.Sprintf("%s:%s", targetRepo, group.SharedTags[0])
+			tagsToPush := []string{}
-			fmt.Printf("Putting shared %s\n", groupIdentifier)
+			for _, tag := range group.SharedTags {
-			tagYaml := tagsToManifestToolYaml(targetRepo, group.SharedTags...) + yaml
+				image := fmt.Sprintf("%s:%s", targetRepo, tag)
+				tagUpdated := fetchDockerHubTagMeta(image).lastUpdatedTime()
+				if mostRecentPush.After(tagUpdated) {
+					tagsToPush = append(tagsToPush, tag)
+				} else {
+					fmt.Printf("Skipping %s (created %s, last updated %s)\n", image, mostRecentPush.Local().Format(time.RFC3339), tagUpdated.Local().Format(time.RFC3339))
+				}
+			}
+
+			if len(tagsToPush) == 0 {
+				continue
+			}
+
+			groupIdentifier := fmt.Sprintf("%s:%s", targetRepo, tagsToPush[0])
+			fmt.Printf("Putting %s\n", groupIdentifier)
+			tagYaml := tagsToManifestToolYaml(targetRepo, tagsToPush...) + yaml
 			if err := manifestToolPushFromSpec(tagYaml); err != nil {
 				return fmt.Errorf("failed pushing %s", groupIdentifier)
 			}

go/src/bashbrew/hub.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+type dockerHubTagMeta struct {
+	LastUpdated string `json:"last_updated"`
+}
+
+func (meta dockerHubTagMeta) lastUpdatedTime() time.Time {
+	t, err := time.Parse(time.RFC3339Nano, meta.LastUpdated)
+	if err != nil {
+		return time.Time{}
+	}
+	return t
+}
+
+func fetchDockerHubTagMeta(repoTag string) dockerHubTagMeta {
+	repoTag = latestizeRepoTag(repoTag)
+	parts := strings.SplitN(repoTag, ":", 2)
+	repo, tag := parts[0], parts[1]
+
+	var meta dockerHubTagMeta
+
+	resp, err := http.Get(fmt.Sprintf("https://hub.docker.com/v2/repositories/%s/tags/%s/", repo, tag))
+	if err != nil {
+		return meta
+	}
+	defer resp.Body.Close()
+
+	err = json.NewDecoder(resp.Body).Decode(&meta)
+	if err != nil {
+		return meta
+	}
+
+	return meta
+}
+
+func dockerCreated(image string) time.Time {
+	created, err := dockerInspect("{{.Created}}", image)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "warning: error while fetching creation time of %q: %v\n", image, err)
+		return time.Now()
+	}
+	created = strings.TrimSpace(created)
+
+	t, err := time.Parse(time.RFC3339Nano, created)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "warning: error while parsing creation time of %q (%q): %v\n", image, created, err)
+		return time.Now()
+	}
+
+	return t
+}