From b94862171b29dbef4cd780e4b1746d97f62f7a94 Mon Sep 17 00:00:00 2001 From: Sujil02 Date: Wed, 29 Apr 2020 22:41:56 -0400 Subject: [PATCH] And system prune feature for v2. Adds podman system prune for v2. Refactoring for code reuse from pods containers images and volume prune. Adds and enables testcases to support the added feature. Signed-off-by: Sujil02 --- cmd/podman/containers/prune.go | 9 +- cmd/podman/images/prune.go | 15 +-- cmd/podman/pods/prune.go | 13 +- cmd/podman/system/prune.go | 103 +++++++++++++++ cmd/podman/utils/utils.go | 55 ++++++++- cmd/podman/volumes/prune.go | 12 +- pkg/domain/entities/engine_container.go | 1 + pkg/domain/infra/abi/containers.go | 4 + pkg/domain/infra/abi/images.go | 6 +- pkg/domain/infra/abi/pods.go | 4 + pkg/domain/infra/abi/system.go | 38 ++++++ pkg/domain/infra/abi/volumes.go | 6 +- pkg/domain/infra/tunnel/system.go | 7 ++ test/e2e/prune_test.go | 158 +++++++++++++++++++++++- test/e2e/volume_prune_test.go | 1 - 15 files changed, 381 insertions(+), 51 deletions(-) create mode 100644 cmd/podman/system/prune.go diff --git a/cmd/podman/containers/prune.go b/cmd/podman/containers/prune.go index df627259cb..d4bea48f98 100644 --- a/cmd/podman/containers/prune.go +++ b/cmd/podman/containers/prune.go @@ -43,7 +43,6 @@ func init() { func prune(cmd *cobra.Command, args []string) error { var ( - errs utils.OutputErrors pruneOptions = entities.ContainerPruneOptions{} ) if len(args) > 0 { @@ -76,11 +75,5 @@ func prune(cmd *cobra.Command, args []string) error { if err != nil { return err } - for k := range responses.ID { - fmt.Println(k) - } - for _, v := range responses.Err { - errs = append(errs, v) - } - return errs.PrintErrors() + return utils.PrintContainerPruneResults(responses) } diff --git a/cmd/podman/images/prune.go b/cmd/podman/images/prune.go index 53a1966c17..7c9e3eb610 100644 --- a/cmd/podman/images/prune.go +++ b/cmd/podman/images/prune.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/containers/libpod/cmd/podman/registry" + "github.com/containers/libpod/cmd/podman/utils" "github.com/containers/libpod/cmd/podman/validate" "github.com/containers/libpod/pkg/domain/entities" "github.com/pkg/errors" @@ -71,17 +72,5 @@ Are you sure you want to continue? [y/N] `) return err } - for _, i := range results.Report.Id { - fmt.Println(i) - } - - for _, e := range results.Report.Err { - fmt.Fprint(os.Stderr, e.Error()+"\n") - } - - if results.Size > 0 { - fmt.Fprintf(os.Stdout, "Size: %d\n", results.Size) - } - - return nil + return utils.PrintImagePruneResults(results) } diff --git a/cmd/podman/pods/prune.go b/cmd/podman/pods/prune.go index 091e3375b2..bc15d80359 100644 --- a/cmd/podman/pods/prune.go +++ b/cmd/podman/pods/prune.go @@ -41,9 +41,6 @@ func init() { } func prune(cmd *cobra.Command, args []string) error { - var ( - errs utils.OutputErrors - ) if len(args) > 0 { return errors.Errorf("`%s` takes no arguments", cmd.CommandPath()) } @@ -60,16 +57,8 @@ func prune(cmd *cobra.Command, args []string) error { } } responses, err := registry.ContainerEngine().PodPrune(context.Background(), pruneOptions) - if err != nil { return err } - for _, r := range responses { - if r.Err == nil { - fmt.Println(r.Id) - } else { - errs = append(errs, r.Err) - } - } - return errs.PrintErrors() + return utils.PrintPodPruneResults(responses) } diff --git a/cmd/podman/system/prune.go b/cmd/podman/system/prune.go new file mode 100644 index 0000000000..cc9b473f1d --- /dev/null +++ b/cmd/podman/system/prune.go @@ -0,0 +1,103 @@ +package system + +import ( + "bufio" + "context" + "fmt" + "os" + "strings" + + "github.com/containers/libpod/cmd/podman/registry" + "github.com/containers/libpod/cmd/podman/utils" + "github.com/containers/libpod/cmd/podman/validate" + "github.com/containers/libpod/pkg/domain/entities" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var ( + pruneOptions = entities.SystemPruneOptions{} + + pruneDescription = fmt.Sprintf(` + podman system prune + + Remove unused data +`) + + pruneCommand = &cobra.Command{ + Use: "prune [flags]", + Short: "Remove unused data", + Args: validate.NoArgs, + Long: pruneDescription, + RunE: prune, + Example: `podman system prune`, + } + force bool +) + +func init() { + registry.Commands = append(registry.Commands, registry.CliCommand{ + Mode: []entities.EngineMode{entities.ABIMode, entities.TunnelMode}, + Command: pruneCommand, + Parent: systemCmd, + }) + flags := pruneCommand.Flags() + flags.BoolVarP(&force, "force", "f", false, "Do not prompt for confirmation. The default is false") + flags.BoolVarP(&pruneOptions.All, "all", "a", false, "Remove all unused data") + flags.BoolVar(&pruneOptions.Volume, "volumes", false, "Prune volumes") + +} + +func prune(cmd *cobra.Command, args []string) error { + // Prompt for confirmation if --force is not set + if !force { + reader := bufio.NewReader(os.Stdin) + volumeString := "" + if pruneOptions.Volume { + volumeString = ` + - all volumes not used by at least one container` + } + fmt.Printf(` +WARNING! This will remove: + - all stopped containers%s + - all stopped pods + - all dangling images + - all build cache +Are you sure you want to continue? [y/N] `, volumeString) + answer, err := reader.ReadString('\n') + if err != nil { + return errors.Wrapf(err, "error reading input") + } + if strings.ToLower(answer)[0] != 'y' { + return nil + } + } + // TODO: support for filters in system prune + response, err := registry.ContainerEngine().SystemPrune(context.Background(), pruneOptions) + if err != nil { + return err + } + // Print pod prune results + fmt.Println("Deleted Pods") + err = utils.PrintPodPruneResults(response.PodPruneReport) + if err != nil { + return err + } + // Print container prune results + fmt.Println("Deleted Containers") + err = utils.PrintContainerPruneResults(response.ContainerPruneReport) + if err != nil { + return err + } + // Print Volume prune results + if pruneOptions.Volume { + fmt.Println("Deleted Volumes") + err = utils.PrintVolumePruneResults(response.VolumePruneReport) + if err != nil { + return err + } + } + // Print Images prune results + fmt.Println("Deleted Images") + return utils.PrintImagePruneResults(response.ImagePruneReport) +} diff --git a/cmd/podman/utils/utils.go b/cmd/podman/utils/utils.go index c7d105ba48..f4c7046283 100644 --- a/cmd/podman/utils/utils.go +++ b/cmd/podman/utils/utils.go @@ -1,6 +1,11 @@ package utils -import "os" +import ( + "fmt" + "os" + + "github.com/containers/libpod/pkg/domain/entities" +) // IsDir returns true if the specified path refers to a directory. func IsDir(path string) bool { @@ -20,3 +25,51 @@ func FileExists(path string) bool { } return !file.IsDir() } + +func PrintPodPruneResults(podPruneReports []*entities.PodPruneReport) error { + var errs OutputErrors + for _, r := range podPruneReports { + if r.Err == nil { + fmt.Println(r.Id) + } else { + errs = append(errs, r.Err) + } + } + return errs.PrintErrors() +} + +func PrintContainerPruneResults(containerPruneReport *entities.ContainerPruneReport) error { + var errs OutputErrors + for k := range containerPruneReport.ID { + fmt.Println(k) + } + for _, v := range containerPruneReport.Err { + errs = append(errs, v) + } + return errs.PrintErrors() +} + +func PrintVolumePruneResults(volumePruneReport []*entities.VolumePruneReport) error { + var errs OutputErrors + for _, r := range volumePruneReport { + if r.Err == nil { + fmt.Println(r.Id) + } else { + errs = append(errs, r.Err) + } + } + return errs.PrintErrors() +} + +func PrintImagePruneResults(imagePruneReport *entities.ImagePruneReport) error { + for _, i := range imagePruneReport.Report.Id { + fmt.Println(i) + } + for _, e := range imagePruneReport.Report.Err { + fmt.Fprint(os.Stderr, e.Error()+"\n") + } + if imagePruneReport.Size > 0 { + fmt.Fprintf(os.Stdout, "Size: %d\n", imagePruneReport.Size) + } + return nil +} diff --git a/cmd/podman/volumes/prune.go b/cmd/podman/volumes/prune.go index 2c3ed88f37..57344385b5 100644 --- a/cmd/podman/volumes/prune.go +++ b/cmd/podman/volumes/prune.go @@ -44,9 +44,6 @@ func init() { } func prune(cmd *cobra.Command, args []string) error { - var ( - errs utils.OutputErrors - ) // Prompt for confirmation if --force is not set if !pruneOptions.Force { reader := bufio.NewReader(os.Stdin) @@ -64,12 +61,5 @@ func prune(cmd *cobra.Command, args []string) error { if err != nil { return err } - for _, r := range responses { - if r.Err == nil { - fmt.Println(r.Id) - } else { - errs = append(errs, r.Err) - } - } - return errs.PrintErrors() + return utils.PrintVolumePruneResults(responses) } diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go index eebf4c0333..d98aee6df4 100644 --- a/pkg/domain/entities/engine_container.go +++ b/pkg/domain/entities/engine_container.go @@ -42,6 +42,7 @@ type ContainerEngine interface { ContainerWait(ctx context.Context, namesOrIds []string, options WaitOptions) ([]WaitReport, error) Events(ctx context.Context, opts EventsOptions) error GenerateSystemd(ctx context.Context, nameOrID string, opts GenerateSystemdOptions) (*GenerateSystemdReport, error) + SystemPrune(ctx context.Context, options SystemPruneOptions) (*SystemPruneReport, error) HealthCheckRun(ctx context.Context, nameOrId string, options HealthCheckOptions) (*define.HealthCheckResults, error) Info(ctx context.Context) (*define.Info, error) PodCreate(ctx context.Context, opts PodCreateOptions) (*PodCreateReport, error) diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 82bf82bf06..cb61176b85 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -184,6 +184,10 @@ func (ic *ContainerEngine) ContainerPrune(ctx context.Context, options entities. filterFuncs = append(filterFuncs, generatedFunc) } } + return ic.pruneContainersHelper(ctx, filterFuncs) +} + +func (ic *ContainerEngine) pruneContainersHelper(ctx context.Context, filterFuncs []libpod.ContainerFilter) (*entities.ContainerPruneReport, error) { prunedContainers, pruneErrors, err := ic.Libpod.PruneContainers(filterFuncs) if err != nil { return nil, err diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index be788b2bf6..0af06fb894 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -36,7 +36,11 @@ func (ir *ImageEngine) Exists(_ context.Context, nameOrId string) (*entities.Boo } func (ir *ImageEngine) Prune(ctx context.Context, opts entities.ImagePruneOptions) (*entities.ImagePruneReport, error) { - results, err := ir.Libpod.ImageRuntime().PruneImages(ctx, opts.All, opts.Filter) + return ir.pruneImagesHelper(ctx, opts.All, opts.Filter) +} + +func (ir *ImageEngine) pruneImagesHelper(ctx context.Context, all bool, filters []string) (*entities.ImagePruneReport, error) { + results, err := ir.Libpod.ImageRuntime().PruneImages(ctx, all, filters) if err != nil { return nil, err } diff --git a/pkg/domain/infra/abi/pods.go b/pkg/domain/infra/abi/pods.go index b286bcf0d2..16c222cbd4 100644 --- a/pkg/domain/infra/abi/pods.go +++ b/pkg/domain/infra/abi/pods.go @@ -243,6 +243,10 @@ func (ic *ContainerEngine) PodRm(ctx context.Context, namesOrIds []string, optio } func (ic *ContainerEngine) PodPrune(ctx context.Context, options entities.PodPruneOptions) ([]*entities.PodPruneReport, error) { + return ic.prunePodHelper(ctx) +} + +func (ic *ContainerEngine) prunePodHelper(ctx context.Context) ([]*entities.PodPruneReport, error) { var ( reports []*entities.PodPruneReport ) diff --git a/pkg/domain/infra/abi/system.go b/pkg/domain/infra/abi/system.go index e5c109ee66..ab1b282d86 100644 --- a/pkg/domain/infra/abi/system.go +++ b/pkg/domain/infra/abi/system.go @@ -175,3 +175,41 @@ func setUMask() { // nolint:deadcode,unused func checkInput() error { // nolint:deadcode,unused return nil } + +// SystemPrune removes unsed data from the system. Pruning pods, containers, volumes and images. +func (ic *ContainerEngine) SystemPrune(ctx context.Context, options entities.SystemPruneOptions) (*entities.SystemPruneReport, error) { + var systemPruneReport = new(entities.SystemPruneReport) + podPruneReport, err := ic.prunePodHelper(ctx) + if err != nil { + return nil, err + } + systemPruneReport.PodPruneReport = podPruneReport + + containerPruneReport, err := ic.pruneContainersHelper(ctx, nil) + if err != nil { + return nil, err + } + systemPruneReport.ContainerPruneReport = containerPruneReport + + results, err := ic.Libpod.ImageRuntime().PruneImages(ctx, options.All, nil) + if err != nil { + return nil, err + } + report := entities.ImagePruneReport{ + Report: entities.Report{ + Id: results, + Err: nil, + }, + } + + systemPruneReport.ImagePruneReport = &report + + if options.Volume { + volumePruneReport, err := ic.pruneVolumesHelper(ctx) + if err != nil { + return nil, err + } + systemPruneReport.VolumePruneReport = volumePruneReport + } + return systemPruneReport, nil +} diff --git a/pkg/domain/infra/abi/volumes.go b/pkg/domain/infra/abi/volumes.go index bdae4359dd..91b2440dfb 100644 --- a/pkg/domain/infra/abi/volumes.go +++ b/pkg/domain/infra/abi/volumes.go @@ -1,5 +1,3 @@ -// +build ABISupport - package abi import ( @@ -113,6 +111,10 @@ func (ic *ContainerEngine) VolumeInspect(ctx context.Context, namesOrIds []strin } func (ic *ContainerEngine) VolumePrune(ctx context.Context, opts entities.VolumePruneOptions) ([]*entities.VolumePruneReport, error) { + return ic.pruneVolumesHelper(ctx) +} + +func (ic *ContainerEngine) pruneVolumesHelper(ctx context.Context) ([]*entities.VolumePruneReport, error) { var ( reports []*entities.VolumePruneReport ) diff --git a/pkg/domain/infra/tunnel/system.go b/pkg/domain/infra/tunnel/system.go index 97bf885e77..18cb6c75a6 100644 --- a/pkg/domain/infra/tunnel/system.go +++ b/pkg/domain/infra/tunnel/system.go @@ -3,6 +3,7 @@ package tunnel import ( "context" "errors" + "fmt" "github.com/containers/libpod/libpod/define" "github.com/containers/libpod/pkg/bindings/system" @@ -21,3 +22,9 @@ func (ic *ContainerEngine) VarlinkService(_ context.Context, _ entities.ServiceO func (ic *ContainerEngine) SetupRootless(_ context.Context, cmd *cobra.Command) error { panic(errors.New("rootless engine mode is not supported when tunneling")) } + +// SystemPrune prunes unused data from the system. +func (ic *ContainerEngine) SystemPrune(ctx context.Context, options entities.SystemPruneOptions) (*entities.SystemPruneReport, error) { + fmt.Println("in tunnel") + return system.Prune(ic.ClientCxt, &options.All, &options.Volume) +} diff --git a/test/e2e/prune_test.go b/test/e2e/prune_test.go index 466a4f739e..a09b6f37aa 100644 --- a/test/e2e/prune_test.go +++ b/test/e2e/prune_test.go @@ -148,7 +148,6 @@ var _ = Describe("Podman prune", func() { It("podman system image prune unused images", func() { SkipIfRemote() - Skip(v2fail) podmanTest.RestoreAllArtifacts() podmanTest.BuildImage(pruneImage, "alpine_bash:latest", "true") prune := podmanTest.PodmanNoCache([]string{"system", "prune", "-a", "--force"}) @@ -162,7 +161,6 @@ var _ = Describe("Podman prune", func() { }) It("podman system prune pods", func() { - Skip(v2fail) session := podmanTest.Podman([]string{"pod", "create"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -193,4 +191,160 @@ var _ = Describe("Podman prune", func() { Expect(pods.ExitCode()).To(Equal(0)) Expect(len(pods.OutputToStringArray())).To(Equal(2)) }) + + It("podman system prune - pod,container stopped", func() { + session := podmanTest.Podman([]string{"pod", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Start and stop a pod to get it in exited state. + session = podmanTest.Podman([]string{"pod", "start", "-l"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"pod", "stop", "-l"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Create a container. This container should be pruned. + create := podmanTest.Podman([]string{"create", "--name", "test", BB}) + create.WaitWithDefaultTimeout() + Expect(create.ExitCode()).To(Equal(0)) + + prune := podmanTest.Podman([]string{"system", "prune", "-f"}) + prune.WaitWithDefaultTimeout() + Expect(prune.ExitCode()).To(Equal(0)) + + pods := podmanTest.Podman([]string{"pod", "ps"}) + pods.WaitWithDefaultTimeout() + Expect(pods.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfPods()).To(Equal(0)) + + Expect(podmanTest.NumberOfContainers()).To(Equal(0)) + }) + + It("podman system prune with running, exited pod and volume prune set true", func() { + + // Start and stop a pod to get it in exited state. + session := podmanTest.Podman([]string{"pod", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"pod", "start", "-l"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"pod", "stop", "-l"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Start a pod and leave it running + session = podmanTest.Podman([]string{"pod", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + session = podmanTest.Podman([]string{"pod", "start", "-l"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Number of pod should be 2. One exited one running. + Expect(podmanTest.NumberOfPods()).To(Equal(2)) + + // Create a container. This container should be pruned. + _, ec, _ := podmanTest.RunLsContainer("test1") + Expect(ec).To(Equal(0)) + + // Number of containers should be three now. + // Two as pods infra container and one newly created. + Expect(podmanTest.NumberOfContainers()).To(Equal(3)) + + // image list current count should not be pruned if all flag isnt enabled + session = podmanTest.Podman([]string{"images"}) + session.WaitWithDefaultTimeout() + numberOfImages := len(session.OutputToStringArray()) + + // Adding unused volume should be pruned + session = podmanTest.Podman([]string{"volume", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"create", "-v", "myvol:/myvol", ALPINE, "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(3)) + + session = podmanTest.Podman([]string{"system", "prune", "--force", "--volumes"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Volumes should be pruned. + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(0)) + + // One Pod should not be pruned as it was running + Expect(podmanTest.NumberOfPods()).To(Equal(1)) + + // Running pods infra container should not be pruned. + Expect(podmanTest.NumberOfContainers()).To(Equal(1)) + + // Image should not be pruned and number should be same. + images := podmanTest.Podman([]string{"images"}) + images.WaitWithDefaultTimeout() + Expect(len(images.OutputToStringArray())).To(Equal(numberOfImages)) + }) + + It("podman system prune - with dangling images true", func() { + session := podmanTest.Podman([]string{"pod", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Start and stop a pod to get it in exited state. + session = podmanTest.Podman([]string{"pod", "start", "-l"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"pod", "stop", "-l"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Create a container. This container should be pruned. + create := podmanTest.Podman([]string{"create", "--name", "test", BB}) + create.WaitWithDefaultTimeout() + Expect(create.ExitCode()).To(Equal(0)) + + // Adding images should be pruned + podmanTest.BuildImage(pruneImage, "alpine_bash:latest", "true") + + // Adding unused volume should not be pruned as volumes not set + session = podmanTest.Podman([]string{"volume", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + prune := podmanTest.Podman([]string{"system", "prune", "-f", "-a"}) + prune.WaitWithDefaultTimeout() + Expect(prune.ExitCode()).To(Equal(0)) + + pods := podmanTest.Podman([]string{"pod", "ps"}) + pods.WaitWithDefaultTimeout() + Expect(pods.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfPods()).To(Equal(0)) + + Expect(podmanTest.NumberOfContainers()).To(Equal(0)) + + // Volumes should not be pruned + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(2)) + + images := podmanTest.PodmanNoCache([]string{"images", "-aq"}) + images.WaitWithDefaultTimeout() + // all images are unused, so they all should be deleted! + Expect(len(images.OutputToStringArray())).To(Equal(0)) + }) }) diff --git a/test/e2e/volume_prune_test.go b/test/e2e/volume_prune_test.go index b9ea90568e..3049646b08 100644 --- a/test/e2e/volume_prune_test.go +++ b/test/e2e/volume_prune_test.go @@ -65,7 +65,6 @@ var _ = Describe("Podman volume prune", func() { }) It("podman system prune --volume", func() { - Skip(v2fail) session := podmanTest.Podman([]string{"volume", "create"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0))