Merge pull request #16940 from justinsb/use_kops_toolbox_dump_in_e2e

tests: run kops toolbox dump in bare-metal e2e

cmd/kops/toolbox_dump.go

@@ -70,6 +70,9 @@ type ToolboxDumpOptions struct {
 	SSHUser      string
 	MaxNodes     int
 	K8sResources bool
+
+	// CloudResources controls whether we dump the cloud resources
+	CloudResources bool
 }
 
 func (o *ToolboxDumpOptions) InitDefaults() {
@@ -78,6 +81,7 @@ func (o *ToolboxDumpOptions) InitDefaults() {
 	o.SSHUser = "ubuntu"
 	o.MaxNodes = 500
 	o.K8sResources = k8sResources != ""
+	o.CloudResources = true
 }
 
 func NewCmdToolboxDump(f commandutils.Factory, out io.Writer) *cobra.Command {
@@ -104,6 +108,7 @@ func NewCmdToolboxDump(f commandutils.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringVar(&options.Dir, "dir", options.Dir, "Target directory; if specified will collect logs and other information.")
 	cmd.MarkFlagDirname("dir")
 	cmd.Flags().BoolVar(&options.K8sResources, "k8s-resources", options.K8sResources, "Include k8s resources in the dump")
+	cmd.Flags().BoolVar(&options.CloudResources, "cloud-resources", options.CloudResources, "Include cloud resources in the dump")
 	cmd.Flags().IntVar(&options.MaxNodes, "max-nodes", options.MaxNodes, "The maximum number of nodes from which to dump logs")
 	cmd.Flags().StringVar(&options.PrivateKey, "private-key", options.PrivateKey, "File containing private key to use for SSH access to instances")
 	cmd.Flags().StringVar(&options.SSHUser, "ssh-user", options.SSHUser, "The remote user for SSH access to instances")
@@ -132,6 +137,8 @@ func RunToolboxDump(ctx context.Context, f commandutils.Factory, out io.Writer,
 		return err
 	}
 
+	var cloudResources *resources.Dump
+	if options.CloudResources {
 		resourceMap, err := resourceops.ListResources(cloud, cluster)
 		if err != nil {
 			return err
@@ -140,6 +147,8 @@ func RunToolboxDump(ctx context.Context, f commandutils.Factory, out io.Writer,
 		if err != nil {
 			return err
 		}
+		cloudResources = d
+	}
 
 	if options.Dir != "" {
 		privateKeyPath := options.PrivateKey
@@ -213,23 +222,26 @@ func RunToolboxDump(ctx context.Context, f commandutils.Factory, out io.Writer,
 		// look for a bastion instance and use it if exists
 		// Prefer a bastion load balancer if exists
 		bastionAddress := ""
-		for _, lb := range d.LoadBalancers {
+		if cloudResources != nil {
+			for _, lb := range cloudResources.LoadBalancers {
 				if strings.Contains(lb.Name, "bastion") && lb.DNSName != "" {
 					bastionAddress = lb.DNSName
 				}
 			}
 			if bastionAddress == "" {
-			for _, instance := range d.Instances {
+				for _, instance := range cloudResources.Instances {
 					if strings.Contains(instance.Name, "bastion") {
 						bastionAddress = instance.PublicAddresses[0]
 					}
 				}
 			}
+		}
 		dumper := dump.NewLogDumper(bastionAddress, sshConfig, keyRing, options.Dir)
 
 		var additionalIPs []string
 		var additionalPrivateIPs []string
-		for _, instance := range d.Instances {
+		if cloudResources != nil {
+			for _, instance := range cloudResources.Instances {
 				if len(instance.PublicAddresses) != 0 {
 					additionalIPs = append(additionalIPs, instance.PublicAddresses[0])
 				} else if len(instance.PrivateAddresses) != 0 {
@@ -238,6 +250,7 @@ func RunToolboxDump(ctx context.Context, f commandutils.Factory, out io.Writer,
 					klog.Warningf("no IP for instance %q", instance.Name)
 				}
 			}
+		}
 
 		if err := dumper.DumpAllNodes(ctx, nodes, options.MaxNodes, additionalIPs, additionalPrivateIPs); err != nil {
 			klog.Warningf("error dumping nodes: %v", err)
@@ -262,9 +275,10 @@ func RunToolboxDump(ctx context.Context, f commandutils.Factory, out io.Writer,
 		}
 	}
 
+	if cloudResources != nil {
 		switch options.Output {
 		case OutputYaml:
-		b, err := kops.ToRawYaml(d)
+			b, err := kops.ToRawYaml(cloudResources)
 			if err != nil {
 				return fmt.Errorf("error marshaling yaml: %v", err)
 			}
@@ -275,7 +289,7 @@ func RunToolboxDump(ctx context.Context, f commandutils.Factory, out io.Writer,
 			return nil
 
 		case OutputJSON:
-		b, err := json.MarshalIndent(d, "", "  ")
+			b, err := json.MarshalIndent(cloudResources, "", "  ")
 			if err != nil {
 				return fmt.Errorf("error marshaling json: %v", err)
 			}
@@ -288,6 +302,8 @@ func RunToolboxDump(ctx context.Context, f commandutils.Factory, out io.Writer,
 		default:
 			return fmt.Errorf("unsupported output format: %q", options.Output)
 		}
+	}
+	return nil
 }
 
 func truncateNodeList(nodes *corev1.NodeList, max int) error {

docs/cli/kops_toolbox_dump.md

@@ -23,6 +23,7 @@ kops toolbox dump [CLUSTER] [flags]
 ### Options
 
 ```
+      --cloud-resources      Include cloud resources in the dump (default true)
       --dir string           Target directory; if specified will collect logs and other information.
   -h, --help                 help for dump
       --k8s-resources        Include k8s resources in the dump

tests/e2e/scenarios/bare-metal/dump-artifacts

@@ -71,3 +71,14 @@ for ns in kube-system; do
     kubectl logs -n ${ns} ${pod} > ${ARTIFACTS}/logs/${ns}/${pod}.log || true
   done
 done
+
+# Use `kops toolbox dump` to dump a lot of useful information
+# We pass --cloud-resources=false because dumping cloud resources is not implemented on metal
+mkdir -p ${ARTIFACTS}/dump
+${KOPS} toolbox dump \
+  --dir ${ARTIFACTS}/dump \
+  --k8s-resources \
+  --cloud-resources=false \
+  --private-key ${REPO_ROOT}/.build/.ssh/id_ed25519 \
+  --ssh-user root \
+  --name metal.k8s.local

tests/e2e/scenarios/bare-metal/run-test

@@ -28,7 +28,7 @@ BINDIR=${WORKDIR}/bin
 mkdir -p "${BINDIR}"
 go build -o ${BINDIR}/kops ./cmd/kops
 
-KOPS=${BINDIR}/kops
+export KOPS=${BINDIR}/kops
 
 function cleanup() {
   echo "running dump-artifacts"