Merge pull request #3551 from mheon/fix_memory_leak

Fix memory leak with exit files
This commit is contained in:
OpenShift Merge Robot 2019-08-02 03:44:43 +02:00 committed by GitHub
commit e3240daa47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 278 additions and 236 deletions

View File

@ -20,7 +20,7 @@ RUN apt-get update && apt-get install -y \
libnl-3-dev \
libostree-dev \
libprotobuf-dev \
libprotobuf-c0-dev \
libprotobuf-c-dev \
libseccomp2 \
libseccomp-dev \
libtool \

View File

@ -16,6 +16,7 @@ type MainFlags struct {
CniConfigDir string
ConmonPath string
DefaultMountsFile string
EventsBackend string
HooksDir []string
MaxWorks int
Namespace string

View File

@ -118,6 +118,10 @@ func getRuntime(ctx context.Context, c *cliconfig.PodmanCommand, renumber, migra
options = append(options, libpod.WithNetworkCmdPath(c.GlobalFlags.NetworkCmdPath))
}
if c.Flags().Changed("events-backend") {
options = append(options, libpod.WithEventsLogger(c.GlobalFlags.EventsBackend))
}
if c.Flags().Changed("cgroup-manager") {
options = append(options, libpod.WithCgroupManager(c.GlobalFlags.CGroupManager))
} else {

View File

@ -48,6 +48,7 @@ func init() {
if err := rootCmd.PersistentFlags().MarkHidden("default-mounts-file"); err != nil {
logrus.Error("unable to mark default-mounts-file flag as hidden")
}
rootCmd.PersistentFlags().StringVar(&MainGlobalOpts.EventsBackend, "events-backend", "", "Events backend to use")
// Override default --help information of `--help` global flag
var dummyHelp bool
rootCmd.PersistentFlags().BoolVar(&dummyHelp, "help", false, "Help for podman")

View File

@ -99,7 +99,7 @@ libpod to manage containers.
a slirp4netns network. If "" is used then the binary is looked up using the $PATH environment variable.
**events_logger**=""
Default method to use when logging events. Valid values are "journald" and "file".
Default method to use when logging events. Valid values are "file", "journald", and "none".
**detach_keys**=""
Keys sequence used for detaching a container

View File

@ -36,6 +36,10 @@ Note: CGroup manager is not supported in rootless mode when using CGroups Versio
Path to where the cpu performance results should be written
**--events-logger**=*type*
Backend to use for storing events. Allowed values are **file**, **journald**, and **none**.
**--hooks-dir**=*path*
Each `*.json` file in the path configures a hook for Podman containers. For more details on the syntax of the JSON files and the semantics of hook injection, see `oci-hooks(5)`. Podman and libpod currently support both the 1.0.0 and 0.1.0 hook schemas, although the 0.1.0 schema is deprecated.

View File

@ -187,7 +187,7 @@ func (c *Container) StopWithTimeout(timeout uint) error {
c.state.State == define.ContainerStateExited {
return define.ErrCtrStopped
}
defer c.newContainerEvent(events.Stop)
return c.stop(timeout)
}

View File

@ -634,19 +634,15 @@ func (c *Container) removeConmonFiles() error {
return errors.Wrapf(err, "error removing container %s OOM file", c.ID())
}
// Instead of outright deleting the exit file, rename it (if it exists).
// We want to retain it so we can get the exit code of containers which
// are removed (at least until we have a workable events system)
// Remove the exit file so we don't leak memory in tmpfs
exitFile := filepath.Join(c.ociRuntime.exitsDir, c.ID())
oldExitFile := filepath.Join(c.ociRuntime.exitsDir, fmt.Sprintf("%s-old", c.ID()))
if _, err := os.Stat(exitFile); err != nil {
if !os.IsNotExist(err) {
return errors.Wrapf(err, "error running stat on container %s exit file", c.ID())
}
} else {
// Rename should replace the old exit file (if it exists)
if err := os.Rename(exitFile, oldExitFile); err != nil {
return errors.Wrapf(err, "error renaming container %s exit file", c.ID())
if err := os.Remove(exitFile); err != nil {
return errors.Wrapf(err, "error removing container %s exit file", c.ID())
}
}
@ -1112,7 +1108,13 @@ func (c *Container) stop(timeout uint) error {
}
// Wait until we have an exit file, and sync once we do
return c.waitForExitFileAndSync()
if err := c.waitForExitFileAndSync(); err != nil {
return err
}
c.newContainerEvent(events.Stop)
return nil
}
// Internal, non-locking function to pause a container

View File

@ -14,6 +14,8 @@ const (
LogFile EventerType = iota
// Journald indicates journald should be used to log events
Journald EventerType = iota
// Null is a no-op events logger. It does not read or write events.
Null EventerType = iota
)
// Event describes the attributes of a libpod event

View File

@ -16,11 +16,30 @@ var ErrNoJournaldLogging = errors.New("No support for journald logging")
// String returns a string representation of EventerType
func (et EventerType) String() string {
if et == LogFile {
switch et {
case LogFile:
return "file"
}
case Journald:
return "journald"
case Null:
return "none"
default:
return "invalid"
}
}
// IsValidEventer checks if the given string is a valid eventer type.
func IsValidEventer(eventer string) bool {
switch eventer {
case LogFile.String():
return true
case Journald.String():
return true
case Null.String():
return true
default:
return false
}
}
// NewEvent creates a event struct and populates with

View File

@ -18,8 +18,10 @@ func NewEventer(options EventerOptions) (eventer Eventer, err error) {
}
case strings.ToUpper(LogFile.String()):
eventer = EventLogFile{options}
case strings.ToUpper(Null.String()):
eventer = NewNullEventer()
default:
return eventer, errors.Errorf("unknown event logger type: %s", strings.ToUpper(options.EventerType))
return nil, errors.Errorf("unknown event logger type: %s", strings.ToUpper(options.EventerType))
}
return eventer, nil
}

View File

@ -55,7 +55,7 @@ func (e EventLogFile) Read(options ReadOptions) error {
return err
}
switch event.Type {
case Image, Volume, Pod, Container:
case Image, Volume, Pod, System, Container:
// no-op
default:
return errors.Errorf("event type %s is not valid in %s", event.Type.String(), e.options.LogFilePath)

File diff suppressed because it is too large Load Diff

View File

@ -394,14 +394,9 @@ func (r *Runtime) removeContainer(ctx context.Context, c *Container, force bool,
// Check that the container's in a good state to be removed
if c.state.State == config2.ContainerStateRunning {
if err := c.ociRuntime.stopContainer(c, c.StopTimeout()); err != nil {
if err := c.stop(c.StopTimeout()); err != nil {
return errors.Wrapf(err, "cannot remove container %s as it could not be stopped", c.ID())
}
// Need to update container state to make sure we know it's stopped
if err := c.waitForExitFileAndSync(); err != nil {
return err
}
}
// Check that all of our exec sessions have finished

View File

@ -9,7 +9,6 @@ import (
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
@ -23,6 +22,7 @@ import (
"github.com/containers/libpod/cmd/podman/shared/parse"
"github.com/containers/libpod/libpod"
"github.com/containers/libpod/libpod/define"
"github.com/containers/libpod/libpod/events"
"github.com/containers/libpod/libpod/image"
"github.com/containers/libpod/libpod/logs"
"github.com/containers/libpod/pkg/adapter/shortcuts"
@ -418,14 +418,13 @@ func (r *LocalRuntime) Run(ctx context.Context, c *cliconfig.RunValues, exitCode
if ecode, err := ctr.Wait(); err != nil {
if errors.Cause(err) == define.ErrNoSuchCtr {
// The container may have been removed
// Go looking for an exit file
ctrExitCode, err := ReadExitFile(config.TmpDir, ctr.ID())
// Check events
event, err := r.Runtime.GetLastContainerEvent(ctr.ID(), events.Exited)
if err != nil {
logrus.Errorf("Cannot get exit code: %v", err)
exitCode = 127
} else {
exitCode = ctrExitCode
exitCode = event.ContainerExitCode
}
}
} else {
@ -441,31 +440,6 @@ func (r *LocalRuntime) Run(ctx context.Context, c *cliconfig.RunValues, exitCode
return exitCode, nil
}
// ReadExitFile reads a container's exit file
func ReadExitFile(runtimeTmp, ctrID string) (int, error) {
exitFile := filepath.Join(runtimeTmp, "exits", fmt.Sprintf("%s-old", ctrID))
logrus.Debugf("Attempting to read container %s exit code from file %s", ctrID, exitFile)
// Check if it exists
if _, err := os.Stat(exitFile); err != nil {
return 0, errors.Wrapf(err, "error getting exit file for container %s", ctrID)
}
// File exists, read it in and convert to int
statusStr, err := ioutil.ReadFile(exitFile)
if err != nil {
return 0, errors.Wrapf(err, "error reading exit file for container %s", ctrID)
}
exitCode, err := strconv.Atoi(string(statusStr))
if err != nil {
return 0, errors.Wrapf(err, "error parsing exit code for container %s", ctrID)
}
return exitCode, nil
}
// Ps ...
func (r *LocalRuntime) Ps(c *cliconfig.PsValues, opts shared.PsOptions) ([]shared.PsContainerOutput, error) {
maxWorkers := shared.Parallelize("ps")
@ -655,18 +629,13 @@ func (r *LocalRuntime) Start(ctx context.Context, c *cliconfig.StartValues, sigP
if ecode, err := ctr.Wait(); err != nil {
if errors.Cause(err) == define.ErrNoSuchCtr {
// The container may have been removed
// Go looking for an exit file
rtc, err := r.GetConfig()
if err != nil {
return 0, err
}
ctrExitCode, err := ReadExitFile(rtc.TmpDir, ctr.ID())
// Check events
event, err := r.Runtime.GetLastContainerEvent(ctr.ID(), events.Exited)
if err != nil {
logrus.Errorf("Cannot get exit code: %v", err)
exitCode = 127
} else {
exitCode = ctrExitCode
exitCode = event.ContainerExitCode
}
}
} else {

View File

@ -168,6 +168,9 @@ func (c *CreateConfig) createExitCommand(runtime *libpod.Runtime) ([]string, err
for _, opt := range config.StorageConfig.GraphDriverOptions {
command = append(command, []string{"--storage-opt", opt}...)
}
if config.EventsLogger != "" {
command = append(command, []string{"--events-backend", config.EventsLogger}...)
}
if c.Syslog {
command = append(command, "--syslog", "true")

View File

@ -412,7 +412,7 @@ func (p *PodmanTestIntegration) BuildImage(dockerfile, imageName string, layers
// PodmanPID execs podman and returns its PID
func (p *PodmanTestIntegration) PodmanPID(args []string) (*PodmanSessionIntegration, int) {
podmanOptions := p.MakeOptions(args)
podmanOptions := p.MakeOptions(args, false)
fmt.Printf("Running: %s %s\n", p.PodmanBinary, strings.Join(podmanOptions, " "))
command := exec.Command(p.PodmanBinary, podmanOptions...)
session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)

View File

@ -30,13 +30,20 @@ func SkipIfRootless() {
// Podman is the exec call to podman on the filesystem
func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration {
podmanSession := p.PodmanBase(args, false)
podmanSession := p.PodmanBase(args, false, false)
return &PodmanSessionIntegration{podmanSession}
}
// PodmanNoCache calls podman with out adding the imagecache
func (p *PodmanTestIntegration) PodmanNoCache(args []string) *PodmanSessionIntegration {
podmanSession := p.PodmanBase(args, true)
podmanSession := p.PodmanBase(args, true, false)
return &PodmanSessionIntegration{podmanSession}
}
// PodmanNoEvents calls the Podman command without an imagecache and without an
// events backend. It is used mostly for caching and uncaching images.
func (p *PodmanTestIntegration) PodmanNoEvents(args []string) *PodmanSessionIntegration {
podmanSession := p.PodmanBase(args, true, true)
return &PodmanSessionIntegration{podmanSession}
}
@ -135,7 +142,7 @@ func (p *PodmanTestIntegration) StopVarlink() {
}
//MakeOptions assembles all the podman main options
func (p *PodmanTestIntegration) makeOptions(args []string) []string {
func (p *PodmanTestIntegration) makeOptions(args []string, noEvents bool) []string {
return args
}
@ -156,7 +163,7 @@ func (p *PodmanTestIntegration) RestoreArtifactToCache(image string) error {
dest := strings.Split(image, "/")
destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1))
p.CrioRoot = p.ImageCacheDir
restore := p.PodmanNoCache([]string{"load", "-q", "-i", destName})
restore := p.PodmanNoEvents([]string{"load", "-q", "-i", destName})
restore.WaitWithDefaultTimeout()
return nil
}

View File

@ -23,19 +23,26 @@ func SkipIfRootless() {
// Podman is the exec call to podman on the filesystem
func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration {
podmanSession := p.PodmanBase(args, false)
podmanSession := p.PodmanBase(args, false, false)
return &PodmanSessionIntegration{podmanSession}
}
// PodmanNoCache calls the podman command with no configured imagecache
func (p *PodmanTestIntegration) PodmanNoCache(args []string) *PodmanSessionIntegration {
podmanSession := p.PodmanBase(args, true)
podmanSession := p.PodmanBase(args, true, false)
return &PodmanSessionIntegration{podmanSession}
}
// PodmanNoEvents calls the Podman command without an imagecache and without an
// events backend. It is used mostly for caching and uncaching images.
func (p *PodmanTestIntegration) PodmanNoEvents(args []string) *PodmanSessionIntegration {
podmanSession := p.PodmanBase(args, true, true)
return &PodmanSessionIntegration{podmanSession}
}
// PodmanAsUser is the exec call to podman on the filesystem with the specified uid/gid and environment
func (p *PodmanTestIntegration) PodmanAsUser(args []string, uid, gid uint32, cwd string, env []string) *PodmanSessionIntegration {
podmanSession := p.PodmanAsUserBase(args, uid, gid, cwd, env, false)
podmanSession := p.PodmanAsUserBase(args, uid, gid, cwd, env, false, false)
return &PodmanSessionIntegration{podmanSession}
}
@ -59,14 +66,19 @@ func PodmanTestCreate(tempDir string) *PodmanTestIntegration {
}
// MakeOptions assembles all the podman main options
func (p *PodmanTestIntegration) makeOptions(args []string) []string {
func (p *PodmanTestIntegration) makeOptions(args []string, noEvents bool) []string {
var debug string
if _, ok := os.LookupEnv("DEBUG"); ok {
debug = "--log-level=debug --syslog=true "
}
podmanOptions := strings.Split(fmt.Sprintf("%s--root %s --runroot %s --runtime %s --conmon %s --cni-config-dir %s --cgroup-manager %s --tmpdir %s",
debug, p.CrioRoot, p.RunRoot, p.OCIRuntime, p.ConmonBinary, p.CNIConfigDir, p.CgroupManager, p.TmpDir), " ")
eventsType := "file"
if noEvents {
eventsType = "none"
}
podmanOptions := strings.Split(fmt.Sprintf("%s--root %s --runroot %s --runtime %s --conmon %s --cni-config-dir %s --cgroup-manager %s --tmpdir %s --events-backend %s",
debug, p.CrioRoot, p.RunRoot, p.OCIRuntime, p.ConmonBinary, p.CNIConfigDir, p.CgroupManager, p.TmpDir, eventsType), " ")
if os.Getenv("HOOK_OPTION") != "" {
podmanOptions = append(podmanOptions, os.Getenv("HOOK_OPTION"))
}
@ -81,7 +93,7 @@ func (p *PodmanTestIntegration) RestoreArtifact(image string) error {
fmt.Printf("Restoring %s...\n", image)
dest := strings.Split(image, "/")
destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1))
restore := p.PodmanNoCache([]string{"load", "-q", "-i", destName})
restore := p.PodmanNoEvents([]string{"load", "-q", "-i", destName})
restore.Wait(90)
return nil
}
@ -93,7 +105,7 @@ func (p *PodmanTestIntegration) RestoreArtifactToCache(image string) error {
destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1))
p.CrioRoot = p.ImageCacheDir
restore := p.PodmanNoCache([]string{"load", "-q", "-i", destName})
restore := p.PodmanNoEvents([]string{"load", "-q", "-i", destName})
restore.WaitWithDefaultTimeout()
return nil
}

View File

@ -789,9 +789,10 @@ USER mail`
match, _ := session.GrepString("1.2.3.4")
Expect(match).Should(BeTrue())
session = podmanTest.Podman([]string{"run", "--rm", "--http-proxy=false", ALPINE, "printenv", "http_proxy"})
session = podmanTest.Podman([]string{"run", "--http-proxy=false", ALPINE, "printenv", "http_proxy"})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(1))
Expect(session.OutputToString()).To(Equal(""))
os.Unsetenv("http_proxy")
})

View File

@ -23,7 +23,7 @@ var _ = Describe("PodmanTest test", func() {
FakeOutputs["check"] = []string{"check"}
os.Setenv("HOOK_OPTION", "hook_option")
env := os.Environ()
session := podmanTest.PodmanAsUserBase([]string{"check"}, 1000, 1000, "", env, true)
session := podmanTest.PodmanAsUserBase([]string{"check"}, 1000, 1000, "", env, true, false)
os.Unsetenv("HOOK_OPTION")
session.WaitWithDefaultTimeout()
Expect(session.Command.Process).ShouldNot(BeNil())

View File

@ -26,14 +26,14 @@ var (
// PodmanTestCommon contains common functions will be updated later in
// the inheritance structs
type PodmanTestCommon interface {
MakeOptions(args []string) []string
MakeOptions(args []string, noEvents bool) []string
WaitForContainer() bool
WaitContainerReady(id string, expStr string, timeout int, step int) bool
}
// PodmanTest struct for command line options
type PodmanTest struct {
PodmanMakeOptions func(args []string) []string
PodmanMakeOptions func(args []string, noEvents bool) []string
PodmanBinary string
ArtifactPath string
TempDir string
@ -59,15 +59,15 @@ type HostOS struct {
}
// MakeOptions assembles all podman options
func (p *PodmanTest) MakeOptions(args []string) []string {
return p.PodmanMakeOptions(args)
func (p *PodmanTest) MakeOptions(args []string, noEvents bool) []string {
return p.PodmanMakeOptions(args, noEvents)
}
// PodmanAsUserBase exec podman as user. uid and gid is set for credentials usage. env is used
// to record the env for debugging
func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string, env []string, nocache bool) *PodmanSession {
func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string, env []string, nocache, noEvents bool) *PodmanSession {
var command *exec.Cmd
podmanOptions := p.MakeOptions(args)
podmanOptions := p.MakeOptions(args, noEvents)
podmanBinary := p.PodmanBinary
if p.RemoteTest {
podmanBinary = p.RemotePodmanBinary
@ -105,8 +105,8 @@ func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string
}
// PodmanBase exec podman with default env.
func (p *PodmanTest) PodmanBase(args []string, nocache bool) *PodmanSession {
return p.PodmanAsUserBase(args, 0, 0, "", nil, nocache)
func (p *PodmanTest) PodmanBase(args []string, nocache, noEvents bool) *PodmanSession {
return p.PodmanAsUserBase(args, 0, 0, "", nil, nocache, noEvents)
}
// WaitForContainer waits on a started container
@ -124,7 +124,7 @@ func (p *PodmanTest) WaitForContainer() bool {
// containers are currently running.
func (p *PodmanTest) NumberOfContainersRunning() int {
var containers []string
ps := p.PodmanBase([]string{"ps", "-q"}, true)
ps := p.PodmanBase([]string{"ps", "-q"}, true, false)
ps.WaitWithDefaultTimeout()
Expect(ps.ExitCode()).To(Equal(0))
for _, i := range ps.OutputToStringArray() {
@ -139,7 +139,7 @@ func (p *PodmanTest) NumberOfContainersRunning() int {
// containers are currently defined.
func (p *PodmanTest) NumberOfContainers() int {
var containers []string
ps := p.PodmanBase([]string{"ps", "-aq"}, true)
ps := p.PodmanBase([]string{"ps", "-aq"}, true, false)
ps.WaitWithDefaultTimeout()
Expect(ps.ExitCode()).To(Equal(0))
for _, i := range ps.OutputToStringArray() {
@ -154,7 +154,7 @@ func (p *PodmanTest) NumberOfContainers() int {
// pods are currently defined.
func (p *PodmanTest) NumberOfPods() int {
var pods []string
ps := p.PodmanBase([]string{"pod", "ps", "-q"}, true)
ps := p.PodmanBase([]string{"pod", "ps", "-q"}, true, false)
ps.WaitWithDefaultTimeout()
Expect(ps.ExitCode()).To(Equal(0))
for _, i := range ps.OutputToStringArray() {
@ -170,7 +170,7 @@ func (p *PodmanTest) NumberOfPods() int {
func (p *PodmanTest) GetContainerStatus() string {
var podmanArgs = []string{"ps"}
podmanArgs = append(podmanArgs, "--all", "--format={{.Status}}")
session := p.PodmanBase(podmanArgs, true)
session := p.PodmanBase(podmanArgs, true, false)
session.WaitWithDefaultTimeout()
return session.OutputToString()
}
@ -178,7 +178,7 @@ func (p *PodmanTest) GetContainerStatus() string {
// WaitContainerReady waits process or service inside container start, and ready to be used.
func (p *PodmanTest) WaitContainerReady(id string, expStr string, timeout int, step int) bool {
startTime := time.Now()
s := p.PodmanBase([]string{"logs", id}, true)
s := p.PodmanBase([]string{"logs", id}, true, false)
s.WaitWithDefaultTimeout()
for {
@ -191,7 +191,7 @@ func (p *PodmanTest) WaitContainerReady(id string, expStr string, timeout int, s
return true
}
time.Sleep(time.Duration(step) * time.Second)
s = p.PodmanBase([]string{"logs", id}, true)
s = p.PodmanBase([]string{"logs", id}, true, false)
s.WaitWithDefaultTimeout()
}
}

View File

@ -32,7 +32,7 @@ func FakePodmanTestCreate() *FakePodmanTest {
return p
}
func (p *FakePodmanTest) makeOptions(args []string) []string {
func (p *FakePodmanTest) makeOptions(args []string, noEvents bool) []string {
return FakeOutputs[strings.Join(args, " ")]
}