Build all boulder binaries into a single binary (#5693)

The resulting `boulder` binary can be invoked by different names to
trigger the behavior of the relevant subcommand. For instance, symlinking
and invoking as `boulder-ca` acts as the CA. Symlinking and invoking as
`boulder-va` acts as the VA.

This reduces the .deb file size from about 200MB to about 20MB.

This works by creating a registry that maps subcommand names to `main`
functions. Each subcommand registers itself in an `init()` function. The
monolithic `boulder` binary then checks what name it was invoked with
(`os.Args[0]`), looks it up in the registry, and invokes the appropriate
`main`. To avoid conflicts, all of the old `package main` are replaced
with `package notmain`.

To get the list of registered subcommands, run `boulder --list`. This
is used when symlinking all the variants into place, to ensure the set
of symlinked names matches the entries in the registry.

Fixes #5692

Makefile

@@ -38,7 +38,7 @@ $(CMD_BINS): build_cmds
 build_cmds: | $(OBJDIR)
 	echo $(OBJECTS)
 	GOBIN=$(OBJDIR) GO111MODULE=on go install -mod=vendor $(GO_BUILD_FLAGS) ./...
-	cp $(OBJDIR)/boulder-va $(OBJDIR)/boulder-remoteva
+	./link.sh
 
 # Building an RPM requires `fpm` from https://github.com/jordansissel/fpm
 # which you can install with `gem install fpm`.

cmd/admin-revoker/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"
@@ -274,3 +274,7 @@ func main() {
 		usage()
 	}
 }
+
+func init() {
+	cmd.RegisterCommand("admin-revoker", main)
+}

cmd/admin-revoker/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"

cmd/akamai-purger/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"
@@ -215,3 +215,7 @@ func main() {
 	// Once that's done, CatchSignals will call os.Exit().
 	select {}
 }
+
+func init() {
+	cmd.RegisterCommand("akamai-purger", main)
+}

cmd/bad-key-revoker/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bytes"
@@ -539,3 +539,7 @@ func (bkr *badKeyRevoker) backoff() {
 func (bkr *badKeyRevoker) backoffReset() {
 	bkr.backoffTicker = 0
 }
+
+func init() {
+	cmd.RegisterCommand("bad-key-revoker", main)
+}

cmd/bad-key-revoker/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"

cmd/boulder-ca/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"flag"
@@ -311,3 +311,7 @@ func main() {
 
 	select {}
 }
+
+func init() {
+	cmd.RegisterCommand("boulder-ca", main)
+}

cmd/boulder-ca/main_test.go

@@ -1 +1 @@
-package main
+package notmain

cmd/boulder-observer/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"flag"
@@ -33,3 +33,7 @@ func main() {
 	// Start the `Observer` daemon.
 	observer.Start()
 }
+
+func init() {
+	cmd.RegisterCommand("boulder-observer", main)
+}

cmd/boulder-publisher/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"flag"
@@ -112,3 +112,7 @@ func main() {
 	err = cmd.FilterShutdownErrors(grpcSrv.Serve(l))
 	cmd.FailOnError(err, "Publisher gRPC service failed")
 }
+
+func init() {
+	cmd.RegisterCommand("boulder-publisher", main)
+}

cmd/boulder-publisher/main_test.go

@@ -1 +1 @@
-package main
+package notmain

cmd/boulder-ra/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"flag"
@@ -269,3 +269,7 @@ func main() {
 	err = cmd.FilterShutdownErrors(grpcSrv.Serve(listener))
 	cmd.FailOnError(err, "RA gRPC service failed")
 }
+
+func init() {
+	cmd.RegisterCommand("boulder-ra", main)
+}

cmd/boulder-ra/main_test.go

@@ -1 +1 @@
-package main
+package notmain

cmd/boulder-sa/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"flag"
@@ -127,3 +127,7 @@ func main() {
 	err = cmd.FilterShutdownErrors(grpcSrv.Serve(listener))
 	cmd.FailOnError(err, "SA gRPC service failed")
 }
+
+func init() {
+	cmd.RegisterCommand("boulder-sa", main)
+}

cmd/boulder-sa/main_test.go

@@ -1 +1 @@
-package main
+package notmain

cmd/boulder-va/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"flag"
@@ -203,3 +203,10 @@ func main() {
 	err = cmd.FilterShutdownErrors(grpcSrv.Serve(l))
 	cmd.FailOnError(err, "VA gRPC service failed")
 }
+
+func init() {
+	cmd.RegisterCommand("boulder-va", main)
+	// We register under two different names, because it's convenient for the
+	// remote VAs to show up under a different program name when looking at logs.
+	cmd.RegisterCommand("boulder-remoteva", main)
+}

cmd/boulder-va/main_test.go

@@ -1 +1 @@
-package main
+package notmain

cmd/boulder-wfe/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bytes"
@@ -223,3 +223,7 @@ func main() {
 	// waits instead for Shutdown to return.
 	<-done
 }
+
+func init() {
+	cmd.RegisterCommand("boulder-wfe", main)
+}

cmd/boulder-wfe/main_test.go

@@ -1 +1 @@
-package main
+package notmain

cmd/boulder-wfe2/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bytes"
@@ -459,3 +459,7 @@ func main() {
 	// waits instead for Shutdown to return.
 	<-done
 }
+
+func init() {
+	cmd.RegisterCommand("boulder-wfe2", main)
+}

cmd/boulder-wfe2/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bytes"

cmd/boulder/main.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	_ "github.com/letsencrypt/boulder/cmd/admin-revoker"
+	_ "github.com/letsencrypt/boulder/cmd/akamai-purger"
+	_ "github.com/letsencrypt/boulder/cmd/bad-key-revoker"
+	_ "github.com/letsencrypt/boulder/cmd/boulder-ca"
+	_ "github.com/letsencrypt/boulder/cmd/boulder-observer"
+	_ "github.com/letsencrypt/boulder/cmd/boulder-publisher"
+	_ "github.com/letsencrypt/boulder/cmd/boulder-ra"
+	_ "github.com/letsencrypt/boulder/cmd/boulder-sa"
+	_ "github.com/letsencrypt/boulder/cmd/boulder-va"
+	_ "github.com/letsencrypt/boulder/cmd/boulder-wfe"
+	_ "github.com/letsencrypt/boulder/cmd/boulder-wfe2"
+	_ "github.com/letsencrypt/boulder/cmd/caa-log-checker"
+	_ "github.com/letsencrypt/boulder/cmd/ceremony"
+	_ "github.com/letsencrypt/boulder/cmd/cert-checker"
+	_ "github.com/letsencrypt/boulder/cmd/contact-auditor"
+	_ "github.com/letsencrypt/boulder/cmd/expiration-mailer"
+	_ "github.com/letsencrypt/boulder/cmd/id-exporter"
+	_ "github.com/letsencrypt/boulder/cmd/log-validator"
+	_ "github.com/letsencrypt/boulder/cmd/nonce-service"
+	_ "github.com/letsencrypt/boulder/cmd/notify-mailer"
+	_ "github.com/letsencrypt/boulder/cmd/ocsp-responder"
+	_ "github.com/letsencrypt/boulder/cmd/ocsp-updater"
+	_ "github.com/letsencrypt/boulder/cmd/orphan-finder"
+
+	"github.com/letsencrypt/boulder/cmd"
+)
+
+func main() {
+	cmd.LookupCommand(path.Base(os.Args[0]))()
+}
+
+func init() {
+	cmd.RegisterCommand("boulder", func() {
+		if len(os.Args) > 1 && os.Args[1] == "--list" {
+			for _, c := range cmd.AvailableCommands() {
+				if c != "boulder" {
+					fmt.Println(c)
+				}
+			}
+		} else {
+			fmt.Fprintf(os.Stderr, "Call with --list to list available subcommands. Symlink and run as a subcommand to run that subcommand.\n")
+		}
+	})
+}

cmd/caa-log-checker/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bufio"
@@ -255,3 +255,7 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func init() {
+	cmd.RegisterCommand("caa-log-checker", main)
+}

cmd/caa-log-checker/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"compress/gzip"

cmd/ceremony/cert.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto"

cmd/ceremony/cert_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bytes"

cmd/ceremony/crl.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto"

cmd/ceremony/crl_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto"

cmd/ceremony/ecdsa.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto"

cmd/ceremony/ecdsa_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto/ecdsa"

cmd/ceremony/file.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import "os"
 

cmd/ceremony/file_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"io/ioutil"

cmd/ceremony/key.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto"

cmd/ceremony/key_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto"

cmd/ceremony/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bytes"
@@ -15,6 +15,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/letsencrypt/boulder/cmd"
 	"github.com/letsencrypt/boulder/linter"
 	"github.com/letsencrypt/boulder/pkcs11helpers"
 	"golang.org/x/crypto/ocsp"
@@ -790,3 +791,7 @@ func main() {
 		log.Fatalf("unknown ceremony-type, must be one of: root, intermediate, ocsp-signer, crl-signer, key, ocsp-response")
 	}
 }
+
+func init() {
+	cmd.RegisterCommand("ceremony", main)
+}

cmd/ceremony/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"io/ioutil"

cmd/ceremony/ocsp.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto"

cmd/ceremony/ocsp_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto/ecdsa"

cmd/ceremony/rsa.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto/rsa"

cmd/ceremony/rsa_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto"

cmd/cert-checker/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bytes"
@@ -447,3 +447,7 @@ func main() {
 	err = checker.issuedReport.dump()
 	cmd.FailOnError(err, "Failed to dump results: %s\n")
 }
+
+func init() {
+	cmd.RegisterCommand("cert-checker", main)
+}

cmd/cert-checker/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"

cmd/contact-auditor/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"database/sql"
@@ -223,3 +223,7 @@ func main() {
 	}
 
 }
+
+func init() {
+	cmd.RegisterCommand("contact-auditor", main)
+}

cmd/contact-auditor/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"

cmd/expiration-mailer/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bytes"
@@ -631,3 +631,7 @@ func main() {
 		cmd.FailOnError(err, "expiration-mailer has failed")
 	}
 }
+
+func init() {
+	cmd.RegisterCommand("expiration-mailer", main)
+}

cmd/expiration-mailer/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"

cmd/expiration-mailer/send_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"crypto/x509"

cmd/id-exporter/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bufio"
@@ -322,3 +322,7 @@ func main() {
 	err = results.writeToFile(*outFile)
 	cmd.FailOnError(err, fmt.Sprintf("Could not write result to outfile %q", *outFile))
 }
+
+func init() {
+	cmd.RegisterCommand("id-exporter", main)
+}

cmd/id-exporter/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"

cmd/log-validator/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"encoding/json"
@@ -200,3 +200,7 @@ func main() {
 		}
 	})
 }
+
+func init() {
+	cmd.RegisterCommand("log-validator", main)
+}

cmd/log-validator/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"testing"

cmd/nonce-service/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"
@@ -97,3 +97,7 @@ func main() {
 	err = cmd.FilterShutdownErrors(grpcSrv.Serve(l))
 	cmd.FailOnError(err, "Nonce service gRPC server failed")
 }
+
+func init() {
+	cmd.RegisterCommand("nonce-service", main)
+}

cmd/notify-mailer/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"encoding/csv"
@@ -573,3 +573,7 @@ func main() {
 
 	log.Info("Completed successfully")
 }
+
+func init() {
+	cmd.RegisterCommand("notify-mailer", main)
+}

cmd/notify-mailer/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"database/sql"

cmd/ocsp-responder/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bytes"
@@ -372,3 +372,7 @@ func mux(stats prometheus.Registerer, responderPath string, source bocsp.Source,
 	})
 	return hnynethttp.WrapHandler(measured_http.New(&ocspMux{h}, cmd.Clock(), stats))
 }
+
+func init() {
+	cmd.RegisterCommand("ocsp-responder", main)
+}

cmd/ocsp-responder/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"bytes"

cmd/ocsp-updater/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"
@@ -596,3 +596,7 @@ func main() {
 		updater.tick()
 	}
 }
+
+func init() {
+	cmd.RegisterCommand("ocsp-updater", main)
+}

cmd/ocsp-updater/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"

cmd/orphan-finder/main.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"
@@ -440,3 +440,7 @@ func main() {
 		usage()
 	}
 }
+
+func init() {
+	cmd.RegisterCommand("orphan-finder", main)
+}

cmd/orphan-finder/main_test.go

@@ -1,4 +1,4 @@
-package main
+package notmain
 
 import (
 	"context"

cmd/registry.go

@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+)
+
+var registry struct {
+	sync.Mutex
+	commands map[string]func()
+}
+
+// Register a boulder subcommand to be run when the binary name matches `name`.
+func RegisterCommand(name string, f func()) {
+	registry.Lock()
+	defer registry.Unlock()
+
+	if registry.commands == nil {
+		registry.commands = make(map[string]func())
+	}
+
+	if registry.commands[name] != nil {
+		panic(fmt.Sprintf("command %q was registered twice", name))
+	}
+	registry.commands[name] = f
+}
+
+func LookupCommand(name string) func() {
+	registry.Lock()
+	defer registry.Unlock()
+	return registry.commands[name]
+}
+
+func AvailableCommands() []string {
+	registry.Lock()
+	defer registry.Unlock()
+	var avail []string
+	for name := range registry.commands {
+		avail = append(avail, name)
+	}
+	sort.Strings(avail)
+	return avail
+}

link.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+#
+# Symlink the various boulder subcommands into place.
+#
+BINDIR="$PWD/bin"
+for n in `"${BINDIR}/boulder" --list` ; do
+  ln -sf boulder "${BINDIR}/$n"
+done