Introduce multi-stage upgrade (#2723)

`linkerd install` supports a 2-stage install process, `linkerd upgrade`
did not.

Add 2-stage support for `linkerd upgrade`. Also exercise multi-stage
functionality during upgrade integration tests.

Part of #2337

Signed-off-by: Andrew Seigner <siggy@buoyant.io>

chart/templates/prometheus-rbac.yaml

@@ -4,12 +4,6 @@
 ### Prometheus RBAC
 ###
 ---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: linkerd-prometheus
-  namespace: {{.Namespace}}
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
@@ -31,4 +25,10 @@ subjects:
 - kind: ServiceAccount
   name: linkerd-prometheus
   namespace: {{.Namespace}}
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: linkerd-prometheus
+  namespace: {{.Namespace}}
 {{- end}}

chart/templates/proxy_injector-rbac.yaml

@@ -4,12 +4,6 @@
 ### Proxy Injector RBAC
 ###
 ---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: linkerd-proxy-injector
-  namespace: {{.Namespace}}
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -41,4 +35,10 @@ roleRef:
   kind: ClusterRole
   name: linkerd-{{.Namespace}}-proxy-injector
   apiGroup: rbac.authorization.k8s.io
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: linkerd-proxy-injector
+  namespace: {{.Namespace}}
 {{end -}}

chart/templates/sp_validator-rbac.yaml

@@ -4,12 +4,6 @@
 ### Service Profile Validator RBAC
 ###
 ---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: linkerd-sp-validator
-  namespace: {{.Namespace}}
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -35,4 +29,10 @@ roleRef:
   kind: ClusterRole
   name: linkerd-{{.Namespace}}-sp-validator
   apiGroup: rbac.authorization.k8s.io
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: linkerd-sp-validator
+  namespace: {{.Namespace}}
 {{end -}}

cli/cmd/install.go

@@ -896,7 +896,7 @@ func (idvals *installIdentityValues) toIdentityContext() *pb.IdentityContext {
 	}
 }
 
-func validateArgs(args []string, flags *pflag.FlagSet, installOnlyFlags *pflag.FlagSet) (string, error) {
+func validateArgs(args []string, flags *pflag.FlagSet, additionalFlags *pflag.FlagSet) (string, error) {
 	if len(args) > 1 {
 		return "", fmt.Errorf("only zero or one argument permitted, received: %s", args)
 	} else if len(args) == 0 {
@@ -909,12 +909,17 @@ func validateArgs(args []string, flags *pflag.FlagSet, installOnlyFlags *pflag.F
 	if stage == configStage {
 		combinedFlags := pflag.NewFlagSet("combined-flags", pflag.ExitOnError)
 		combinedFlags.AddFlagSet(flags)
-		combinedFlags.AddFlagSet(installOnlyFlags)
+		combinedFlags.AddFlagSet(additionalFlags)
 
 		invalidFlags := make([]string, 0)
 		combinedFlags.VisitAll(func(f *pflag.Flag) {
 			if f.Changed {
-				invalidFlags = append(invalidFlags, f.Name)
+				switch f.Name {
+				case "ignore-cluster":
+					// allow `linkerd install config --ignore-cluster`
+				default:
+					invalidFlags = append(invalidFlags, f.Name)
+				}
 			}
 		})
 		if len(invalidFlags) > 0 {

cli/cmd/testdata/install_config.golden

@@ -192,12 +192,6 @@ spec:
 ### Prometheus RBAC
 ###
 ---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: linkerd-prometheus
-  namespace: linkerd
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
@@ -220,16 +214,16 @@ subjects:
   name: linkerd-prometheus
   namespace: linkerd
 ---
-###
-### Proxy Injector RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-proxy-injector
+  name: linkerd-prometheus
   namespace: linkerd
 ---
+###
+### Proxy Injector RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -262,16 +256,16 @@ roleRef:
   name: linkerd-linkerd-proxy-injector
   apiGroup: rbac.authorization.k8s.io
 ---
-###
-### Service Profile Validator RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-sp-validator
+  name: linkerd-proxy-injector
   namespace: linkerd
 ---
+###
+### Service Profile Validator RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -298,3 +292,9 @@ roleRef:
   name: linkerd-linkerd-sp-validator
   apiGroup: rbac.authorization.k8s.io
 ---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: linkerd-sp-validator
+  namespace: linkerd
+---

cli/cmd/testdata/install_default.golden

@@ -192,12 +192,6 @@ spec:
 ### Prometheus RBAC
 ###
 ---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: linkerd-prometheus
-  namespace: linkerd
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
@@ -220,16 +214,16 @@ subjects:
   name: linkerd-prometheus
   namespace: linkerd
 ---
-###
-### Proxy Injector RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-proxy-injector
+  name: linkerd-prometheus
   namespace: linkerd
 ---
+###
+### Proxy Injector RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -262,16 +256,16 @@ roleRef:
   name: linkerd-linkerd-proxy-injector
   apiGroup: rbac.authorization.k8s.io
 ---
-###
-### Service Profile Validator RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-sp-validator
+  name: linkerd-proxy-injector
   namespace: linkerd
 ---
+###
+### Service Profile Validator RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -298,6 +292,12 @@ roleRef:
   name: linkerd-linkerd-sp-validator
   apiGroup: rbac.authorization.k8s.io
 ---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: linkerd-sp-validator
+  namespace: linkerd
+---
 kind: ConfigMap
 apiVersion: v1
 metadata:

cli/cmd/testdata/install_ha_output.golden

@@ -192,12 +192,6 @@ spec:
 ### Prometheus RBAC
 ###
 ---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: linkerd-prometheus
-  namespace: linkerd
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
@@ -220,16 +214,16 @@ subjects:
   name: linkerd-prometheus
   namespace: linkerd
 ---
-###
-### Proxy Injector RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-proxy-injector
+  name: linkerd-prometheus
   namespace: linkerd
 ---
+###
+### Proxy Injector RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -262,16 +256,16 @@ roleRef:
   name: linkerd-linkerd-proxy-injector
   apiGroup: rbac.authorization.k8s.io
 ---
-###
-### Service Profile Validator RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-sp-validator
+  name: linkerd-proxy-injector
   namespace: linkerd
 ---
+###
+### Service Profile Validator RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -298,6 +292,12 @@ roleRef:
   name: linkerd-linkerd-sp-validator
   apiGroup: rbac.authorization.k8s.io
 ---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: linkerd-sp-validator
+  namespace: linkerd
+---
 kind: ConfigMap
 apiVersion: v1
 metadata:

cli/cmd/testdata/install_ha_with_overrides_output.golden

@@ -192,12 +192,6 @@ spec:
 ### Prometheus RBAC
 ###
 ---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: linkerd-prometheus
-  namespace: linkerd
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
@@ -220,16 +214,16 @@ subjects:
   name: linkerd-prometheus
   namespace: linkerd
 ---
-###
-### Proxy Injector RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-proxy-injector
+  name: linkerd-prometheus
   namespace: linkerd
 ---
+###
+### Proxy Injector RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -262,16 +256,16 @@ roleRef:
   name: linkerd-linkerd-proxy-injector
   apiGroup: rbac.authorization.k8s.io
 ---
-###
-### Service Profile Validator RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-sp-validator
+  name: linkerd-proxy-injector
   namespace: linkerd
 ---
+###
+### Service Profile Validator RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -298,6 +292,12 @@ roleRef:
   name: linkerd-linkerd-sp-validator
   apiGroup: rbac.authorization.k8s.io
 ---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: linkerd-sp-validator
+  namespace: linkerd
+---
 kind: ConfigMap
 apiVersion: v1
 metadata:

cli/cmd/testdata/install_no_init_container.golden

@@ -192,12 +192,6 @@ spec:
 ### Prometheus RBAC
 ###
 ---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: linkerd-prometheus
-  namespace: linkerd
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
@@ -220,16 +214,16 @@ subjects:
   name: linkerd-prometheus
   namespace: linkerd
 ---
-###
-### Proxy Injector RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-proxy-injector
+  name: linkerd-prometheus
   namespace: linkerd
 ---
+###
+### Proxy Injector RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -262,16 +256,16 @@ roleRef:
   name: linkerd-linkerd-proxy-injector
   apiGroup: rbac.authorization.k8s.io
 ---
-###
-### Service Profile Validator RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-sp-validator
+  name: linkerd-proxy-injector
   namespace: linkerd
 ---
+###
+### Service Profile Validator RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -298,6 +292,12 @@ roleRef:
   name: linkerd-linkerd-sp-validator
   apiGroup: rbac.authorization.k8s.io
 ---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: linkerd-sp-validator
+  namespace: linkerd
+---
 kind: ConfigMap
 apiVersion: v1
 metadata:

cli/cmd/testdata/install_output.golden

@@ -192,12 +192,6 @@ spec:
 ### Prometheus RBAC
 ###
 ---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: linkerd-prometheus
-  namespace: Namespace
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
@@ -220,16 +214,16 @@ subjects:
   name: linkerd-prometheus
   namespace: Namespace
 ---
-###
-### Proxy Injector RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-proxy-injector
+  name: linkerd-prometheus
   namespace: Namespace
 ---
+###
+### Proxy Injector RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -262,16 +256,16 @@ roleRef:
   name: linkerd-Namespace-proxy-injector
   apiGroup: rbac.authorization.k8s.io
 ---
-###
-### Service Profile Validator RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-sp-validator
+  name: linkerd-proxy-injector
   namespace: Namespace
 ---
+###
+### Service Profile Validator RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -298,6 +292,12 @@ roleRef:
   name: linkerd-Namespace-sp-validator
   apiGroup: rbac.authorization.k8s.io
 ---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: linkerd-sp-validator
+  namespace: Namespace
+---
 kind: ConfigMap
 apiVersion: v1
 metadata:

cli/cmd/testdata/upgrade_default.golden

@@ -192,12 +192,6 @@ spec:
 ### Prometheus RBAC
 ###
 ---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: linkerd-prometheus
-  namespace: linkerd
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
@@ -220,16 +214,16 @@ subjects:
   name: linkerd-prometheus
   namespace: linkerd
 ---
-###
-### Proxy Injector RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-proxy-injector
+  name: linkerd-prometheus
   namespace: linkerd
 ---
+###
+### Proxy Injector RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -262,16 +256,16 @@ roleRef:
   name: linkerd-linkerd-proxy-injector
   apiGroup: rbac.authorization.k8s.io
 ---
-###
-### Service Profile Validator RBAC
-###
----
 kind: ServiceAccount
 apiVersion: v1
 metadata:
-  name: linkerd-sp-validator
+  name: linkerd-proxy-injector
   namespace: linkerd
 ---
+###
+### Service Profile Validator RBAC
+###
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -298,6 +292,12 @@ roleRef:
   name: linkerd-linkerd-sp-validator
   apiGroup: rbac.authorization.k8s.io
 ---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: linkerd-sp-validator
+  namespace: linkerd
+---
 kind: ConfigMap
 apiVersion: v1
 metadata:

cli/cmd/upgrade.go

@@ -18,8 +18,10 @@ import (
 )
 
 const (
-	okMessage   = "You're on your way to upgrading Linkerd!\nVisit this URL for further instructions: https://linkerd.io/upgrade/#nextsteps\n"
-	failMessage = "For troubleshooting help, visit: https://linkerd.io/upgrade/#troubleshooting\n"
+	okMessage           = "You're on your way to upgrading Linkerd!"
+	controlPlaneMessage = "Don't forget to run `linkerd upgrade control-plane`!"
+	visitMessage        = "Visit this URL for further instructions: https://linkerd.io/upgrade/#nextsteps"
+	failMessage         = "For troubleshooting help, visit: https://linkerd.io/upgrade/#troubleshooting\n"
 )
 
 type upgradeOptions struct {
@@ -34,26 +36,57 @@ func newUpgradeOptionsWithDefaults() *upgradeOptions {
 	}
 }
 
+// upgradeOnlyFlagSet includes flags that are only accessible at upgrade-time
+// and not at install-time. also these flags are not intended to be persisted
+// via linkerd-config ConfigMap, unlike recordableFlagSet
+func (options *upgradeOptions) upgradeOnlyFlagSet() *pflag.FlagSet {
+	flags := pflag.NewFlagSet("upgrade-only", pflag.ExitOnError)
+
+	flags.StringVar(
+		&options.manifests, "from-manifests", options.manifests,
+		"Read config from a Linkerd install YAML rather than from Kubernetes",
+	)
+
+	return flags
+}
+
 func newCmdUpgrade() *cobra.Command {
 	options := newUpgradeOptionsWithDefaults()
 	flags := options.recordableFlagSet()
+	upgradeOnlyFlags := options.upgradeOnlyFlagSet()
 
 	cmd := &cobra.Command{
-		Use:   "upgrade [flags]",
-		Short: "Output Kubernetes configs to upgrade an existing Linkerd control plane",
+		Use:       fmt.Sprintf("upgrade [%s|%s] [flags]", configStage, controlPlaneStage),
+		Args:      cobra.OnlyValidArgs,
+		ValidArgs: []string{configStage, controlPlaneStage},
+		Short:     "Output Kubernetes configs to upgrade an existing Linkerd control plane",
 		Long: `Output Kubernetes configs to upgrade an existing Linkerd control plane.
 
 Note that the default flag values for this command come from the Linkerd control
 plane. The default values displayed in the Flags section below only apply to the
 install command.`,
+
+		Example: `  # Default upgrade.
+  linkerd upgrade | kubectl apply -f -
+
+  # Similar to install, upgrade may also be broken up into two stages, by user
+  # privilege.
+  # First stage requires cluster-level privileges.
+  linkerd upgrade config | kubectl apply -f -
+  # Second stage requires namespace-level privileges.
+  linkerd upgrade control-plane | kubectl apply -f -`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if options.ignoreCluster {
 				panic("ignore cluster must be unset") // Programmer error.
 			}
 
+			stage, err := validateArgs(args, flags, upgradeOnlyFlags)
+			if err != nil {
+				return err
+			}
+
 			// We need a Kubernetes client to fetch configs and issuer secrets.
 			var k kubernetes.Interface
-			var err error
 			if options.manifests != "" {
 				readers, err := read(options.manifests)
 				if err != nil {
@@ -71,7 +104,7 @@ install command.`,
 				}
 			}
 
-			values, configs, err := options.validateAndBuild(k, flags)
+			values, configs, err := options.validateAndBuild(stage, k, flags)
 			if err != nil {
 				upgradeErrorf("Failed to build upgrade configuration: %s", err)
 			}
@@ -86,23 +119,21 @@ install command.`,
 			buf.WriteTo(os.Stdout)
 
 			fmt.Fprintf(os.Stderr, "\n%s %s\n", okStatus, okMessage)
+			if stage == configStage {
+				fmt.Fprintf(os.Stderr, "%s\n", controlPlaneMessage)
+			}
+			fmt.Fprintf(os.Stderr, "%s\n\n", visitMessage)
 
 			return nil
 		},
 	}
 
-	// add this flag directly rather than as part of the FlagSet because we do not
-	// want it persisted into linkerd-config/install ConfigMap
-	cmd.PersistentFlags().StringVar(
-		&options.manifests, "from-manifests", options.manifests,
-		"Read config from a Linkerd install YAML rather than from Kubernetes",
-	)
-
 	cmd.PersistentFlags().AddFlagSet(flags)
+	cmd.PersistentFlags().AddFlagSet(upgradeOnlyFlags)
 	return cmd
 }
 
-func (options *upgradeOptions) validateAndBuild(k kubernetes.Interface, flags *pflag.FlagSet) (*installValues, *pb.All, error) {
+func (options *upgradeOptions) validateAndBuild(stage string, k kubernetes.Interface, flags *pflag.FlagSet) (*installValues, *pb.All, error) {
 	if err := options.validate(); err != nil {
 		return nil, nil, err
 	}
@@ -162,6 +193,7 @@ func (options *upgradeOptions) validateAndBuild(k kubernetes.Interface, flags *p
 		return nil, nil, fmt.Errorf("could not build install configuration: %s", err)
 	}
 	values.Identity = identity
+	values.stage = stage
 
 	return values, configs, nil
 }

cli/cmd/upgrade_test.go

@@ -87,7 +87,7 @@ data:
 				t.Fatalf("Error mocking k8s client: %s", err)
 			}
 
-			values, configs, err := options.validateAndBuild(clientset, flags)
+			values, configs, err := options.validateAndBuild("", clientset, flags)
 			if !reflect.DeepEqual(err, tc.err) {
 				t.Fatalf("Expected \"%s\", got \"%s\"", tc.err, err)
 			} else if err == nil {
@@ -134,7 +134,7 @@ data:
 		t.Fatalf("Error mocking k8s client: %s", err)
 	}
 
-	values, configs, err := options.validateAndBuild(clientset, flags)
+	values, configs, err := options.validateAndBuild("", clientset, flags)
 	if err != nil {
 		t.Fatalf("validateAndBuild failed with %s", err)
 	}

test/install_test.go

@@ -124,6 +124,21 @@ func TestInstallOrUpgrade(t *testing.T) {
 
 	if TestHelper.UpgradeFromVersion() != "" {
 		cmd = "upgrade"
+
+		// test 2-stage install during upgrade
+		out, _, err := TestHelper.LinkerdRun(cmd, "config")
+		if err != nil {
+			t.Fatalf("linkerd upgrade config command failed\n%s", out)
+		}
+
+		// apply stage 1
+		out, err = TestHelper.KubectlApply(out, TestHelper.GetLinkerdNamespace())
+		if err != nil {
+			t.Fatalf("kubectl apply command failed\n%s", out)
+		}
+
+		// prepare for stage 2
+		args = append([]string{"control-plane"}, args...)
 	}
 
 	exec := append([]string{cmd}, args...)