package kube import ( "bytes" "errors" "fmt" "io" "net" "net/http" "os" "os/signal" "strings" "syscall" buildahParse "github.com/containers/buildah/pkg/parse" "github.com/containers/common/pkg/auth" "github.com/containers/common/pkg/completion" "github.com/containers/image/v5/types" "github.com/containers/podman/v5/cmd/podman/common" "github.com/containers/podman/v5/cmd/podman/parse" "github.com/containers/podman/v5/cmd/podman/registry" "github.com/containers/podman/v5/cmd/podman/utils" "github.com/containers/podman/v5/libpod/define" "github.com/containers/podman/v5/libpod/shutdown" "github.com/containers/podman/v5/pkg/annotations" "github.com/containers/podman/v5/pkg/domain/entities" "github.com/containers/podman/v5/pkg/errorhandling" "github.com/containers/podman/v5/pkg/util" "github.com/spf13/cobra" ) // playKubeOptionsWrapper allows for separating CLI-only fields from API-only // fields. type playKubeOptionsWrapper struct { entities.PlayKubeOptions TLSVerifyCLI bool CredentialsCLI string StartCLI bool BuildCLI bool annotations []string macs []string } var ( // https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ defaultSeccompRoot = "/var/lib/kubelet/seccomp" playOptions = playKubeOptionsWrapper{} playDescription = `Reads in a structured file of Kubernetes YAML. Creates pods or volumes based on the Kubernetes kind described in the YAML. Supported kinds are Pods, Deployments, DaemonSets and PersistentVolumeClaims.` playCmd = &cobra.Command{ Use: "play [options] KUBEFILE|-", Short: "Play a pod or volume based on Kubernetes YAML", Long: playDescription, RunE: play, Args: cobra.ExactArgs(1), ValidArgsFunction: common.AutocompleteDefaultOneArg, Example: `podman kube play nginx.yml cat nginx.yml | podman kube play - podman kube play --creds user:password --seccomp-profile-root /custom/path apache.yml podman kube play https://example.com/nginx.yml`, } ) var ( playKubeCmd = &cobra.Command{ Use: "kube [options] KUBEFILE|-", Short: "Play a pod or volume based on Kubernetes YAML", Long: playDescription, Hidden: true, RunE: playKube, Args: cobra.ExactArgs(1), ValidArgsFunction: common.AutocompleteDefaultOneArg, Example: `podman play kube nginx.yml cat nginx.yml | podman play kube - podman play kube --creds user:password --seccomp-profile-root /custom/path apache.yml podman play kube https://example.com/nginx.yml`, } logDriverFlagName = "log-driver" ) func init() { registry.Commands = append(registry.Commands, registry.CliCommand{ Command: playCmd, Parent: kubeCmd, }) playFlags(playCmd) registry.Commands = append(registry.Commands, registry.CliCommand{ Command: playKubeCmd, Parent: playKubeParentCmd, }) playFlags(playKubeCmd) } func playFlags(cmd *cobra.Command) { flags := cmd.Flags() flags.SetNormalizeFunc(utils.AliasFlags) annotationFlagName := "annotation" flags.StringArrayVar( &playOptions.annotations, annotationFlagName, []string{}, "Add annotations to pods (key=value)", ) _ = cmd.RegisterFlagCompletionFunc(annotationFlagName, completion.AutocompleteNone) credsFlagName := "creds" flags.StringVar(&playOptions.CredentialsCLI, credsFlagName, "", "`Credentials` (USERNAME:PASSWORD) to use for authenticating to a registry") _ = cmd.RegisterFlagCompletionFunc(credsFlagName, completion.AutocompleteNone) staticMACFlagName := "mac-address" flags.StringSliceVar(&playOptions.macs, staticMACFlagName, nil, "Static MAC addresses to assign to the pods") _ = cmd.RegisterFlagCompletionFunc(staticMACFlagName, completion.AutocompleteNone) networkFlagName := "network" flags.StringArrayVar(&playOptions.Networks, networkFlagName, nil, "Connect pod to network(s) or network mode") _ = cmd.RegisterFlagCompletionFunc(networkFlagName, common.AutocompleteNetworkFlag) staticIPFlagName := "ip" flags.IPSliceVar(&playOptions.StaticIPs, staticIPFlagName, nil, "Static IP addresses to assign to the pods") _ = cmd.RegisterFlagCompletionFunc(staticIPFlagName, completion.AutocompleteNone) flags.StringVar(&playOptions.LogDriver, logDriverFlagName, common.LogDriver(), "Logging driver for the container") _ = cmd.RegisterFlagCompletionFunc(logDriverFlagName, common.AutocompleteLogDriver) logOptFlagName := "log-opt" flags.StringArrayVar( &playOptions.LogOptions, logOptFlagName, []string{}, "Logging driver options", ) _ = cmd.RegisterFlagCompletionFunc(logOptFlagName, common.AutocompleteLogOpt) usernsFlagName := "userns" flags.StringVar(&playOptions.Userns, usernsFlagName, os.Getenv("PODMAN_USERNS"), "User namespace to use", ) _ = cmd.RegisterFlagCompletionFunc(usernsFlagName, common.AutocompleteUserNamespace) flags.BoolVar(&playOptions.NoHosts, "no-hosts", false, "Do not create /etc/hosts within the pod's containers, instead use the version from the image") flags.BoolVarP(&playOptions.Quiet, "quiet", "q", false, "Suppress output information when pulling images") flags.BoolVar(&playOptions.TLSVerifyCLI, "tls-verify", true, "Require HTTPS and verify certificates when contacting registries") flags.BoolVar(&playOptions.StartCLI, "start", true, "Start the pod after creating it") flags.BoolVar(&playOptions.Force, "force", false, "Remove volumes as part of --down") authfileFlagName := "authfile" flags.StringVar(&playOptions.Authfile, authfileFlagName, auth.GetDefaultAuthFile(), "Path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override") _ = cmd.RegisterFlagCompletionFunc(authfileFlagName, completion.AutocompleteDefault) downFlagName := "down" flags.BoolVar(&playOptions.Down, downFlagName, false, "Stop pods defined in the YAML file") _ = flags.MarkHidden("down") replaceFlagName := "replace" flags.BoolVar(&playOptions.Replace, replaceFlagName, false, "Delete and recreate pods defined in the YAML file") publishPortsFlagName := "publish" flags.StringSliceVar(&playOptions.PublishPorts, publishPortsFlagName, []string{}, "Publish a container's port, or a range of ports, to the host") _ = cmd.RegisterFlagCompletionFunc(publishPortsFlagName, completion.AutocompleteNone) publishAllPortsFlagName := "publish-all" flags.BoolVar(&playOptions.PublishAllPorts, publishAllPortsFlagName, false, "Whether to publish all ports defined in the K8S YAML file (containerPort, hostPort), if false only hostPort will be published") waitFlagName := "wait" flags.BoolVarP(&playOptions.Wait, waitFlagName, "w", false, "Clean up all objects created when a SIGTERM is received or pods exit") configmapFlagName := "configmap" flags.StringArrayVar(&playOptions.ConfigMaps, configmapFlagName, []string{}, "`Pathname` of a YAML file containing a kubernetes configmap") _ = cmd.RegisterFlagCompletionFunc(configmapFlagName, completion.AutocompleteDefault) noTruncFlagName := "no-trunc" flags.BoolVar(&playOptions.UseLongAnnotations, noTruncFlagName, false, "Use annotations that are not truncated to the Kubernetes maximum length of 63 characters") _ = flags.MarkHidden(noTruncFlagName) if !registry.IsRemote() { certDirFlagName := "cert-dir" flags.StringVar(&playOptions.CertDir, certDirFlagName, "", "`Pathname` of a directory containing TLS certificates and keys") _ = cmd.RegisterFlagCompletionFunc(certDirFlagName, completion.AutocompleteDefault) seccompProfileRootFlagName := "seccomp-profile-root" flags.StringVar(&playOptions.SeccompProfileRoot, seccompProfileRootFlagName, defaultSeccompRoot, "Directory path for seccomp profiles") _ = cmd.RegisterFlagCompletionFunc(seccompProfileRootFlagName, completion.AutocompleteDefault) buildFlagName := "build" flags.BoolVar(&playOptions.BuildCLI, buildFlagName, false, "Build all images in a YAML (given Containerfiles exist)") contextDirFlagName := "context-dir" flags.StringVar(&playOptions.ContextDir, contextDirFlagName, "", "Path to top level of context directory") _ = cmd.RegisterFlagCompletionFunc(contextDirFlagName, completion.AutocompleteDefault) flags.StringVar(&playOptions.SignaturePolicy, "signature-policy", "", "`Pathname` of signature policy file (not usually used)") _ = flags.MarkHidden("signature-policy") // Below flags are local-only and hidden since they are used in // kube-play's systemd integration only and hence hidden from // users. serviceFlagName := "service-container" flags.BoolVar(&playOptions.ServiceContainer, serviceFlagName, false, "Starts a service container before all pods") _ = flags.MarkHidden(serviceFlagName) exitFlagName := "service-exit-code-propagation" flags.StringVar(&playOptions.ExitCodePropagation, exitFlagName, "", "Exit-code propagation of the service container") _ = flags.MarkHidden(exitFlagName) } } func play(cmd *cobra.Command, args []string) error { if playOptions.ServiceContainer && !playOptions.StartCLI { // Sanity check to be future proof return fmt.Errorf("--service-container does not work with --start=stop") } // TLS verification in c/image is controlled via a `types.OptionalBool` // which allows for distinguishing among set-true, set-false, unspecified // which is important to implement a sane way of dealing with defaults of // boolean CLI flags. if cmd.Flags().Changed("tls-verify") { playOptions.SkipTLSVerify = types.NewOptionalBool(!playOptions.TLSVerifyCLI) } if cmd.Flags().Changed("start") { playOptions.Start = types.NewOptionalBool(playOptions.StartCLI) } if cmd.Flags().Changed("build") { playOptions.Build = types.NewOptionalBool(playOptions.BuildCLI) if playOptions.Build == types.OptionalBoolTrue { systemContext, err := buildahParse.SystemContextFromOptions(cmd) if err != nil { return err } playOptions.SystemContext = systemContext } } if cmd.Flags().Changed("authfile") { if err := auth.CheckAuthFile(playOptions.Authfile); err != nil { return err } } if playOptions.ContextDir != "" && playOptions.Build != types.OptionalBoolTrue { return errors.New("--build must be specified when using --context-dir option") } if playOptions.CredentialsCLI != "" { creds, err := util.ParseRegistryCreds(playOptions.CredentialsCLI) if err != nil { return err } playOptions.Username = creds.Username playOptions.Password = creds.Password } for _, annotation := range playOptions.annotations { key, val, hasVal := strings.Cut(annotation, "=") if !hasVal { return fmt.Errorf("annotation %q must include an '=' sign", annotation) } if playOptions.Annotations == nil { playOptions.Annotations = make(map[string]string) } playOptions.Annotations[key] = val } if err := annotations.ValidateAnnotations(playOptions.Annotations); err != nil { return err } for _, mac := range playOptions.macs { m, err := net.ParseMAC(mac) if err != nil { return err } playOptions.StaticMACs = append(playOptions.StaticMACs, m) } if playOptions.Force && !playOptions.Down { return errors.New("--force may be specified only with --down") } reader, err := readerFromArg(args[0]) if err != nil { return err } if playOptions.Down { return teardown(reader, entities.PlayKubeDownOptions{Force: playOptions.Force}) } if playOptions.Replace { if err := teardown(reader, entities.PlayKubeDownOptions{Force: playOptions.Force}); err != nil && !errorhandling.Contains(err, define.ErrNoSuchPod) { return err } if _, err := reader.Seek(0, 0); err != nil { return err } } // Create a channel to catch an interrupt or SIGTERM signal ch := make(chan os.Signal, 1) var teardownReader *bytes.Reader if playOptions.Wait { // Stop the shutdown signal handler so we can actually clean up after a SIGTERM or interrupt if err := shutdown.Stop(); err != nil && err != shutdown.ErrNotStarted { return err } signal.Notify(ch, os.Interrupt, syscall.SIGTERM) playOptions.ServiceContainer = true // Read the kube yaml file again so that a reader can be passed down to the teardown function teardownReader, err = readerFromArg(args[0]) if err != nil { return err } fmt.Println("Use ctrl+c to clean up or wait for pods to exit") } var teardownErr error cancelled := false if playOptions.Wait { // use a goroutine to wait for a sigterm or interrupt go func() { <-ch // clean up any volumes that were created as well fmt.Println("\nCleaning up containers, pods, and volumes...") cancelled = true if err := teardown(teardownReader, entities.PlayKubeDownOptions{Force: true}); err != nil && !errorhandling.Contains(err, define.ErrNoSuchPod) { teardownErr = fmt.Errorf("error during cleanup: %v", err) } }() } if playErr := kubeplay(reader); playErr != nil { // FIXME: The cleanup logic below must be fixed to only remove // resources that were created before a failure. Otherwise, // rerunning the same YAML file will cause an error and remove // the previously created workload. // // teardown any containers, pods, and volumes that might have been created before we hit the error // reader, err := readerFromArg(args[0]) // if err != nil { // return err // } // if err := teardown(reader, entities.PlayKubeDownOptions{Force: true}, true); err != nil && !errorhandling.Contains(err, define.ErrNoSuchPod) { // return fmt.Errorf("error tearing down workloads %q after kube play error %q", err, playErr) // } return playErr } if teardownErr != nil { return teardownErr } // cleanup if --wait=true and the pods have exited if playOptions.Wait && !cancelled { fmt.Println("Cleaning up containers, pods, and volumes...") // clean up any volumes that were created as well if err := teardown(teardownReader, entities.PlayKubeDownOptions{Force: true}); err != nil && !errorhandling.Contains(err, define.ErrNoSuchPod) { return err } } return nil } func playKube(cmd *cobra.Command, args []string) error { return play(cmd, args) } func readerFromArg(fileName string) (*bytes.Reader, error) { var reader io.Reader switch { case fileName == "-": // Read from stdin reader = os.Stdin case parse.ValidURL(fileName) == nil: response, err := http.Get(fileName) if err != nil { return nil, err } defer response.Body.Close() reader = response.Body default: f, err := os.Open(fileName) if err != nil { return nil, err } defer f.Close() reader = f } data, err := io.ReadAll(reader) if err != nil { return nil, err } return bytes.NewReader(data), nil } func teardown(body io.Reader, options entities.PlayKubeDownOptions) error { var ( podStopErrors utils.OutputErrors podRmErrors utils.OutputErrors volRmErrors utils.OutputErrors secRmErrors utils.OutputErrors ) reports, err := registry.ContainerEngine().PlayKubeDown(registry.GetContext(), body, options) if err != nil { return err } // Output stopped pods fmt.Println("Pods stopped:") for _, stopped := range reports.StopReport { switch { case len(stopped.Errs) > 0: podStopErrors = append(podStopErrors, stopped.Errs...) default: fmt.Println(stopped.Id) } } // Dump any stop errors lastStopError := podStopErrors.PrintErrors() if lastStopError != nil { fmt.Fprintf(os.Stderr, "Error: %s\n", lastStopError) } // Output rm'd pods fmt.Println("Pods removed:") for _, removed := range reports.RmReport { switch { case removed.Err != nil: podRmErrors = append(podRmErrors, removed.Err) default: fmt.Println(removed.Id) } } lastPodRmError := podRmErrors.PrintErrors() if lastPodRmError != nil { fmt.Fprintf(os.Stderr, "Error: %s\n", lastPodRmError) } // Output rm'd volumes fmt.Println("Secrets removed:") for _, removed := range reports.SecretRmReport { switch { case removed.Err != nil: secRmErrors = append(secRmErrors, removed.Err) default: fmt.Println(removed.ID) } } lastSecretRmError := secRmErrors.PrintErrors() if lastPodRmError != nil { fmt.Fprintf(os.Stderr, "Error: %s\n", lastSecretRmError) } // Output rm'd volumes fmt.Println("Volumes removed:") for _, removed := range reports.VolumeRmReport { switch { case removed.Err != nil: volRmErrors = append(volRmErrors, removed.Err) default: fmt.Println(removed.Id) } } return volRmErrors.PrintErrors() } func kubeplay(body io.Reader) error { report, err := registry.ContainerEngine().PlayKube(registry.GetContext(), body, playOptions.PlayKubeOptions) if err != nil { return err } if report.ExitCode != nil { registry.SetExitCode(int(*report.ExitCode)) } if err := printPlayReport(report); err != nil { return err } // If --wait=true, we need wait for the service container to exit so that we know that the pod has exited and we can clean up if playOptions.Wait { _, err := registry.ContainerEngine().ContainerWait(registry.GetContext(), []string{report.ServiceContainerID}, entities.WaitOptions{}) if err != nil { return err } } return nil } // printPlayReport goes through the report returned by KubePlay and prints it out in a human // friendly format. func printPlayReport(report *entities.PlayKubeReport) error { // Print volumes report for i, volume := range report.Volumes { if i == 0 { fmt.Println("Volumes:") } fmt.Println(volume.Name) } // Print secrets report for i, secret := range report.Secrets { if i == 0 { fmt.Println("Secrets:") } fmt.Println(secret.CreateReport.ID) } // Print pods report for _, pod := range report.Pods { for _, l := range pod.Logs { fmt.Fprint(os.Stderr, l) } } ctrsFailed := 0 for _, pod := range report.Pods { fmt.Println("Pod:") fmt.Println(pod.ID) switch len(pod.Containers) { case 0: continue case 1: fmt.Println("Container:") default: fmt.Println("Containers:") } for _, ctr := range pod.Containers { fmt.Println(ctr) } ctrsFailed += len(pod.ContainerErrors) // If We have errors, add a newline if len(pod.ContainerErrors) > 0 { fmt.Println() } for _, err := range pod.ContainerErrors { fmt.Fprintln(os.Stderr, err) } // Empty line for space for next block fmt.Println() } if ctrsFailed > 0 { return fmt.Errorf("failed to start %d containers", ctrsFailed) } return nil }