From 6b06e9b77c8191096eeb82ac54c59b894f87da8c Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Wed, 4 Aug 2021 15:24:29 -0400 Subject: [PATCH 1/6] Switch eventlogger to journald by default [NO TESTS NEEDED] Since we are just testing the default. Signed-off-by: Daniel J Walsh --- go.mod | 2 +- go.sum | 9 +++- libpod/container_log_linux.go | 16 +++++++- libpod/container_log_unsupported.go | 4 ++ libpod/runtime_ctr.go | 11 ++++- .../containers/common/libimage/image.go | 17 +++++++- .../containers/common/pkg/auth/auth.go | 2 - .../containers/common/pkg/config/config.go | 7 +++- .../common/pkg/config/containers.conf | 13 +++--- .../containers/common/pkg/config/default.go | 10 ++++- .../containers/common/pkg/config/nosystemd.go | 7 +++- .../containers/common/pkg/config/systemd.go | 41 ++++++++++++++----- .../containers/common/version/version.go | 2 +- vendor/modules.txt | 2 +- 14 files changed, 112 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index f64b850a1f..e8cb0ed51d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/containernetworking/cni v0.8.1 github.com/containernetworking/plugins v0.9.1 github.com/containers/buildah v1.22.3 - github.com/containers/common v0.42.1 + github.com/containers/common v0.43.0 github.com/containers/conmon v2.0.20+incompatible github.com/containers/image/v5 v5.15.2 github.com/containers/ocicrypt v1.1.2 diff --git a/go.sum b/go.sum index 359bfee73a..69c1e0c341 100644 --- a/go.sum +++ b/go.sum @@ -240,11 +240,13 @@ github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9 github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= github.com/containers/buildah v1.22.3 h1:RomxwUa24jMcqzXQetpw4wGMfNlNZLhc9qwyoWHblwc= github.com/containers/buildah v1.22.3/go.mod h1:JVXRyx5Rkp5w5jwvaXe45kuHtyoxpERMjXrR45+3Wfg= -github.com/containers/common v0.42.1 h1:ADOZrVAS8ZY5hBAvr/GoRoPv5Z7TBkxWgxQEXQjlqac= github.com/containers/common v0.42.1/go.mod h1:AaF3ipZfgezsctDuhzLkq4Vl+LkEy7J74ikh2HSXDsg= +github.com/containers/common v0.43.0 h1:CeDdfhLyPfsG6TAKJneT/4RWOhHfWF2Yv+Wz6SuMusU= +github.com/containers/common v0.43.0/go.mod h1:BAoVyRYlxKZKAYpHcFMdrXlIZyzbJp9NwKTgadTd/Dg= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/image/v5 v5.14.0/go.mod h1:SxiBKOcKuT+4yTjD0AskjO+UwFvNcVOJ9qlAw1HNSPU= +github.com/containers/image/v5 v5.15.0/go.mod h1:gzdBcooi6AFdiqfzirUqv90hUyHyI0MMdaqKzACKr2s= github.com/containers/image/v5 v5.15.2 h1:DKicmVr0h1HGkzs9muoErX+fVbV9sV9W5TyMy5perLE= github.com/containers/image/v5 v5.15.2/go.mod h1:8jejVSzTDfyPwr/HXp9rri34n/vbdavYk6IzTiB3TBw= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE= @@ -258,7 +260,9 @@ github.com/containers/psgo v1.5.2 h1:3aoozst/GIwsrr/5jnFy3FrJay98uujPCu9lTuSZ/Cw github.com/containers/psgo v1.5.2/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzPUWfawVU= github.com/containers/storage v1.23.5/go.mod h1:ha26Q6ngehFNhf3AWoXldvAvwI4jFe3ETQAf/CeZPyM= github.com/containers/storage v1.32.6/go.mod h1:mdB+b89p+jU8zpzLTVXA0gWMmIo0WrkfGMh1R8O2IQw= +github.com/containers/storage v1.33.0/go.mod h1:FUZPF4nJijX8ixdhByZJXf02cvbyLi6dyDwXdIe8QVY= github.com/containers/storage v1.33.1/go.mod h1:FUZPF4nJijX8ixdhByZJXf02cvbyLi6dyDwXdIe8QVY= +github.com/containers/storage v1.34.0/go.mod h1:t6I+hTgPU0/tVxQ75vw406wDi/TXwYBqZp4QZV9N7b8= github.com/containers/storage v1.34.1 h1:PsBGMH7hwuQ3MOr7qTgPznFrE8ebfIbwQbg2gKvg0lE= github.com/containers/storage v1.34.1/go.mod h1:FY2TcbfgCLMU4lYoKnlZeZXeH353TOTbpDEA+sAcqAY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -582,6 +586,7 @@ github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= @@ -713,6 +718,7 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -751,6 +757,7 @@ github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwy github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opencontainers/selinux v1.8.3/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= github.com/opencontainers/selinux v1.8.4 h1:krlgQ6/j9CkCXT5oW0yVXdQFOME3NjKuuAZXuR6O7P4= github.com/opencontainers/selinux v1.8.4/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656 h1:WaxyNFpmIDu4i6so9r6LVFIbSaXqsj8oitMitt86ae4= diff --git a/libpod/container_log_linux.go b/libpod/container_log_linux.go index 4eb600bfe4..748f80fa5c 100644 --- a/libpod/container_log_linux.go +++ b/libpod/container_log_linux.go @@ -12,6 +12,7 @@ import ( "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/events" "github.com/containers/podman/v3/libpod/logs" + "github.com/coreos/go-systemd/v22/journal" "github.com/coreos/go-systemd/v22/sdjournal" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -29,6 +30,17 @@ func init() { logDrivers = append(logDrivers, define.JournaldLogging) } +// initializeJournal will write an empty string to the journal +// when a journal is created. This solves a problem when people +// attempt to read logs from a container that has never had stdout/stderr +func (c *Container) initializeJournal(ctx context.Context) error { + m := make(map[string]string) + m["SYSLOG_IDENTIFIER"] = "podman" + m["PODMAN_ID"] = c.ID() + m["CONTAINER_ID_FULL"] = c.ID() + return journal.Send("", journal.PriInfo, m) +} + func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOptions, logChannel chan *logs.LogLine) error { journal, err := sdjournal.NewJournal() if err != nil { @@ -63,12 +75,12 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption } // API requires Next() immediately after SeekHead(). if _, err := journal.Next(); err != nil { - return errors.Wrap(err, "initial journal cursor") + return errors.Wrap(err, "next journal") } // API requires a next|prev before getting a cursor. if _, err := journal.Previous(); err != nil { - return errors.Wrap(err, "initial journal cursor") + return errors.Wrap(err, "previous journal") } // Note that the initial cursor may not yet be ready, so we'll do an diff --git a/libpod/container_log_unsupported.go b/libpod/container_log_unsupported.go index d10082141e..a551df9426 100644 --- a/libpod/container_log_unsupported.go +++ b/libpod/container_log_unsupported.go @@ -13,3 +13,7 @@ import ( func (c *Container) readFromJournal(_ context.Context, _ *logs.LogOptions, _ chan *logs.LogLine) error { return errors.Wrapf(define.ErrOSNotSupported, "Journald logging only enabled with systemd on linux") } + +func (c *Container) initializeJournal(ctx context.Context) error { + return errors.Wrapf(define.ErrOSNotSupported, "Journald logging only enabled with systemd on linux") +} diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 02bbb6981f..52072b0f32 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -462,8 +462,15 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai ctrNamedVolumes = append(ctrNamedVolumes, newVol) } - if ctr.config.LogPath == "" && ctr.config.LogDriver != define.JournaldLogging && ctr.config.LogDriver != define.NoLogging { - ctr.config.LogPath = filepath.Join(ctr.config.StaticDir, "ctr.log") + switch ctr.config.LogDriver { + case define.NoLogging: + break + case define.JournaldLogging: + ctr.initializeJournal(ctx) + default: + if ctr.config.LogPath == "" { + ctr.config.LogPath = filepath.Join(ctr.config.StaticDir, "ctr.log") + } } if !MountExists(ctr.config.Spec.Mounts, "/dev/shm") && ctr.config.ShmDir == "" { diff --git a/vendor/github.com/containers/common/libimage/image.go b/vendor/github.com/containers/common/libimage/image.go index c47e633394..b02bf9783a 100644 --- a/vendor/github.com/containers/common/libimage/image.go +++ b/vendor/github.com/containers/common/libimage/image.go @@ -448,14 +448,24 @@ func (i *Image) removeRecursive(ctx context.Context, rmMap map[string]*RemoveIma return parent.removeRecursive(ctx, rmMap, processedIDs, "", options) } +var errTagDigest = errors.New("tag by digest not supported") + // Tag the image with the specified name and store it in the local containers // storage. The name is normalized according to the rules of NormalizeName. func (i *Image) Tag(name string) error { + if strings.HasPrefix(name, "sha256:") { // ambiguous input + return errors.Wrap(errTagDigest, name) + } + ref, err := NormalizeName(name) if err != nil { return errors.Wrapf(err, "error normalizing name %q", name) } + if _, isDigested := ref.(reference.Digested); isDigested { + return errors.Wrap(errTagDigest, name) + } + logrus.Debugf("Tagging image %s with %q", i.ID(), ref.String()) if i.runtime.eventChannel != nil { defer i.runtime.writeEvent(&Event{ID: i.ID(), Name: name, Time: time.Now(), Type: EventTypeImageTag}) @@ -480,7 +490,7 @@ var errUntagDigest = errors.New("untag by digest not supported") // the local containers storage. The name is normalized according to the rules // of NormalizeName. func (i *Image) Untag(name string) error { - if strings.HasPrefix(name, "sha256:") { + if strings.HasPrefix(name, "sha256:") { // ambiguous input return errors.Wrap(errUntagDigest, name) } @@ -488,6 +498,11 @@ func (i *Image) Untag(name string) error { if err != nil { return errors.Wrapf(err, "error normalizing name %q", name) } + + if _, isDigested := ref.(reference.Digested); isDigested { + return errors.Wrap(errUntagDigest, name) + } + name = ref.String() logrus.Debugf("Untagging %q from image %s", ref.String(), i.ID()) diff --git a/vendor/github.com/containers/common/pkg/auth/auth.go b/vendor/github.com/containers/common/pkg/auth/auth.go index 093da02990..0934b155f8 100644 --- a/vendor/github.com/containers/common/pkg/auth/auth.go +++ b/vendor/github.com/containers/common/pkg/auth/auth.go @@ -104,7 +104,6 @@ func Login(ctx context.Context, systemContext *types.SystemContext, opts *LoginO return errors.Wrap(err, "get credentials for repository") } } else { - // nolint: staticcheck authConfig, err = config.GetCredentials(systemContext, registry) if err != nil { return errors.Wrap(err, "get credentials") @@ -321,7 +320,6 @@ func Logout(systemContext *types.SystemContext, opts *LogoutOptions, args []stri return errors.Wrap(err, "get credentials for repository") } } else { - // nolint: staticcheck authConfig, err = config.GetCredentials(systemContext, registry) if err != nil { return errors.Wrap(err, "get credentials") diff --git a/vendor/github.com/containers/common/pkg/config/config.go b/vendor/github.com/containers/common/pkg/config/config.go index 008cfb6425..e554bac707 100644 --- a/vendor/github.com/containers/common/pkg/config/config.go +++ b/vendor/github.com/containers/common/pkg/config/config.go @@ -274,6 +274,9 @@ type EngineConfig struct { // MachineEnabled indicates if Podman is running in a podman-machine VM MachineEnabled bool `toml:"machine_enabled,omitempty"` + // MachineImage is the image used when creating a podman-machine VM + MachineImage string `toml:"machine_image,omitempty"` + // MultiImageArchive - if true, the container engine allows for storing // archives (e.g., of the docker-archive transport) with multiple // images. By default, Podman creates single-image archives. @@ -691,8 +694,8 @@ func (c *Config) Validate() error { } func (c *EngineConfig) findRuntime() string { - // Search for crun first followed by runc and kata - for _, name := range []string{"crun", "runc", "kata"} { + // Search for crun first followed by runc, kata, runsc + for _, name := range []string{"crun", "runc", "kata", "runsc"} { for _, v := range c.OCIRuntimes[name] { if _, err := os.Stat(v); err == nil { return name diff --git a/vendor/github.com/containers/common/pkg/config/containers.conf b/vendor/github.com/containers/common/pkg/config/containers.conf index a83aa94075..023297501d 100644 --- a/vendor/github.com/containers/common/pkg/config/containers.conf +++ b/vendor/github.com/containers/common/pkg/config/containers.conf @@ -60,23 +60,23 @@ default_capabilities = [ ] # A list of sysctls to be set in containers by default, -# specified as "name=value", +# specified as "name = value", # for example:"net.ipv4.ping_group_range = 0 0". # default_sysctls = [ - "net.ipv4.ping_group_range=0 0", + "net.ipv4.ping_group_range = 0 0", ] # A list of ulimits to be set in containers by default, specified as -# "=:", for example: -# "nofile=1024:2048" +# " = :", for example: +# "nofile = 1024:2048" # See setrlimit(2) for a list of resource names. # Any limit not specified here will be inherited from the process launching the # container engine. # Ulimits has limits for non privileged container engines. # #default_ulimits = [ -# "nofile=1280:2560", +# "nofile = 1280:2560", #] # List of devices. Specified as @@ -381,6 +381,9 @@ default_sysctls = [ # #machine_enabled = false +# The image used when creating a podman-machine VM. +# machine_image = "testing" + # MultiImageArchive - if true, the container engine allows for storing archives # (e.g., of the docker-archive transport) with multiple images. By default, # Podman creates single-image archives. diff --git a/vendor/github.com/containers/common/pkg/config/default.go b/vendor/github.com/containers/common/pkg/config/default.go index a16dd0e023..66531a2ba1 100644 --- a/vendor/github.com/containers/common/pkg/config/default.go +++ b/vendor/github.com/containers/common/pkg/config/default.go @@ -105,8 +105,6 @@ const ( DefaultApparmorProfile = apparmor.Profile // SystemdCgroupsManager represents systemd native cgroup manager SystemdCgroupsManager = "systemd" - // DefaultLogDriver is the default type of log files - DefaultLogDriver = "k8s-file" // DefaultLogSizeMax is the default value for the maximum log size // allowed for a container. Negative values mean that no limit is imposed. DefaultLogSizeMax = -1 @@ -339,6 +337,7 @@ func defaultConfigFromMemory() (*EngineConfig, error) { // constants. c.LockType = "shm" c.MachineEnabled = false + c.MachineImage = "testing" c.ChownCopiedFiles = true @@ -549,6 +548,7 @@ func (c *Config) LogDriver() string { return c.Containers.LogDriver } +// MachineEnabled returns if podman is running inside a VM or not func (c *Config) MachineEnabled() bool { return c.Engine.MachineEnabled } @@ -558,3 +558,9 @@ func (c *Config) MachineEnabled() bool { func (c *Config) RootlessNetworking() string { return c.Containers.RootlessNetworking } + +// MachineImage returns the image to be +// used when creating a podman-machine VM +func (c *Config) MachineImage() string { + return c.Engine.MachineImage +} diff --git a/vendor/github.com/containers/common/pkg/config/nosystemd.go b/vendor/github.com/containers/common/pkg/config/nosystemd.go index 6e39a6ccdb..2a3b6fb35f 100644 --- a/vendor/github.com/containers/common/pkg/config/nosystemd.go +++ b/vendor/github.com/containers/common/pkg/config/nosystemd.go @@ -1,7 +1,12 @@ -// +build !systemd +// +build !systemd !cgo package config +const ( + // DefaultLogDriver is the default type of log files + DefaultLogDriver = "k8s-file" +) + func defaultCgroupManager() string { return CgroupfsCgroupsManager } diff --git a/vendor/github.com/containers/common/pkg/config/systemd.go b/vendor/github.com/containers/common/pkg/config/systemd.go index ed014126b9..fab3ea437a 100644 --- a/vendor/github.com/containers/common/pkg/config/systemd.go +++ b/vendor/github.com/containers/common/pkg/config/systemd.go @@ -1,4 +1,4 @@ -// +build systemd +// +build systemd,cgo package config @@ -9,11 +9,19 @@ import ( "github.com/containers/common/pkg/cgroupv2" "github.com/containers/storage/pkg/unshare" + "github.com/coreos/go-systemd/v22/sdjournal" ) var ( - systemdOnce sync.Once - usesSystemd bool + systemdOnce sync.Once + usesSystemd bool + journaldOnce sync.Once + usesJournald bool +) + +const ( + // DefaultLogDriver is the default type of log files + DefaultLogDriver = "journald" ) func defaultCgroupManager() string { @@ -29,20 +37,17 @@ func defaultCgroupManager() string { } func defaultEventsLogger() string { - if useSystemd() { + if useJournald() { return "journald" } return "file" } func defaultLogDriver() string { - // If we decide to change the default for logdriver, it should be done here. - if useSystemd() { - return DefaultLogDriver + if useJournald() { + return "journald" } - - return DefaultLogDriver - + return "k8s-file" } func useSystemd() bool { @@ -56,3 +61,19 @@ func useSystemd() bool { }) return usesSystemd } + +func useJournald() bool { + journaldOnce.Do(func() { + if !useSystemd() { + return + } + journal, err := sdjournal.NewJournal() + if err != nil { + return + } + journal.Close() + usesJournald = true + return + }) + return usesJournald +} diff --git a/vendor/github.com/containers/common/version/version.go b/vendor/github.com/containers/common/version/version.go index 1e9e48f330..ae372bee73 100644 --- a/vendor/github.com/containers/common/version/version.go +++ b/vendor/github.com/containers/common/version/version.go @@ -1,4 +1,4 @@ package version // Version is the version of the build. -const Version = "0.42.1" +const Version = "0.43.0" diff --git a/vendor/modules.txt b/vendor/modules.txt index c1cfbe76da..2b8db54ff3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -94,7 +94,7 @@ github.com/containers/buildah/pkg/overlay github.com/containers/buildah/pkg/parse github.com/containers/buildah/pkg/rusage github.com/containers/buildah/util -# github.com/containers/common v0.42.1 +# github.com/containers/common v0.43.0 github.com/containers/common/libimage github.com/containers/common/libimage/manifests github.com/containers/common/pkg/apparmor From 1411fa5f23e3890fe870ec4ffdf2e6f114688030 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Wed, 18 Aug 2021 12:13:03 -0400 Subject: [PATCH 2/6] libpod/Container.readFromJournal(): don't skip the first entry When reading log entries from the journal, don't skip past the first matching entry after we've positioned the cursor at it. Make the first blank-line entry that we logged so that the container would always have at least one log entry for us to find (until it gets vacuumed out, at least) a fake history entry, so that `logs` doesn't pass it on for display. CI already has tests that exercise journal-based logging, so [NO TESTS NEEDED] Signed-off-by: Nalin Dahyabhai --- libpod/container_log_linux.go | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/libpod/container_log_linux.go b/libpod/container_log_linux.go index 748f80fa5c..ca1e11ef5e 100644 --- a/libpod/container_log_linux.go +++ b/libpod/container_log_linux.go @@ -38,6 +38,8 @@ func (c *Container) initializeJournal(ctx context.Context) error { m["SYSLOG_IDENTIFIER"] = "podman" m["PODMAN_ID"] = c.ID() m["CONTAINER_ID_FULL"] = c.ID() + history := events.History + m["PODMAN_EVENT"] = history.String() return journal.Send("", journal.PriInfo, m) } @@ -89,10 +91,10 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption var cursorError error for i := 1; i <= 3; i++ { cursor, cursorError = journal.GetCursor() - if err != nil { + if cursorError != nil { + time.Sleep(time.Duration(i*100) * time.Millisecond) continue } - time.Sleep(time.Duration(i*100) * time.Millisecond) break } if cursorError != nil { @@ -116,6 +118,7 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption tailQueue := []*logs.LogLine{} // needed for options.Tail doTail := options.Tail > 0 + lastReadCursor := "" for { select { case <-ctx.Done(): @@ -125,18 +128,25 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption // Fallthrough } - if _, err := journal.Next(); err != nil { - logrus.Errorf("Failed to move journal cursor to next entry: %v", err) - return + if lastReadCursor != "" { + // Advance to next entry if we read this one. + if _, err := journal.Next(); err != nil { + logrus.Errorf("Failed to move journal cursor to next entry: %v", err) + return + } } - latestCursor, err := journal.GetCursor() + + // Fetch the location of this entry, presumably either + // the one that follows the last one we read, or that + // same last one, if there is no next entry (yet). + cursor, err = journal.GetCursor() if err != nil { logrus.Errorf("Failed to get journal cursor: %v", err) return } - // Hit the end of the journal. - if cursor == latestCursor { + // Hit the end of the journal (so far?). + if cursor == lastReadCursor { if doTail { // Flush *once* we hit the end of the journal. startIndex := int64(len(tailQueue)-1) - options.Tail @@ -157,8 +167,9 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption journal.Wait(sdjournal.IndefiniteWait) continue } - cursor = latestCursor + lastReadCursor = cursor + // Read the journal entry. entry, err := journal.GetEntry() if err != nil { logrus.Errorf("Failed to get journal entry: %v", err) From fc6a020330e211ab15671a282198e39562213cec Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Wed, 18 Aug 2021 12:42:20 -0400 Subject: [PATCH 3/6] bump github.com/containers/common Update github.com/containers/common from 0.43.0 to 0.43.2. Signed-off-by: Nalin Dahyabhai --- go.mod | 2 +- go.sum | 4 ++-- .../containers/common/libimage/image.go | 12 +++++++++--- .../containers/common/pkg/config/containers.conf | 15 ++++++++------- .../containers/common/pkg/parse/parse.go | 3 ++- .../containers/common/version/version.go | 2 +- vendor/modules.txt | 2 +- 7 files changed, 24 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index e8cb0ed51d..e277867945 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/containernetworking/cni v0.8.1 github.com/containernetworking/plugins v0.9.1 github.com/containers/buildah v1.22.3 - github.com/containers/common v0.43.0 + github.com/containers/common v0.43.2 github.com/containers/conmon v2.0.20+incompatible github.com/containers/image/v5 v5.15.2 github.com/containers/ocicrypt v1.1.2 diff --git a/go.sum b/go.sum index 69c1e0c341..c1e4acd017 100644 --- a/go.sum +++ b/go.sum @@ -241,8 +241,8 @@ github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRD github.com/containers/buildah v1.22.3 h1:RomxwUa24jMcqzXQetpw4wGMfNlNZLhc9qwyoWHblwc= github.com/containers/buildah v1.22.3/go.mod h1:JVXRyx5Rkp5w5jwvaXe45kuHtyoxpERMjXrR45+3Wfg= github.com/containers/common v0.42.1/go.mod h1:AaF3ipZfgezsctDuhzLkq4Vl+LkEy7J74ikh2HSXDsg= -github.com/containers/common v0.43.0 h1:CeDdfhLyPfsG6TAKJneT/4RWOhHfWF2Yv+Wz6SuMusU= -github.com/containers/common v0.43.0/go.mod h1:BAoVyRYlxKZKAYpHcFMdrXlIZyzbJp9NwKTgadTd/Dg= +github.com/containers/common v0.43.2 h1:oSP5d5sDrq7OkoqLPVrLpi1LZOAwpTwOZXgPDHfmD0E= +github.com/containers/common v0.43.2/go.mod h1:BAoVyRYlxKZKAYpHcFMdrXlIZyzbJp9NwKTgadTd/Dg= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/image/v5 v5.14.0/go.mod h1:SxiBKOcKuT+4yTjD0AskjO+UwFvNcVOJ9qlAw1HNSPU= diff --git a/vendor/github.com/containers/common/libimage/image.go b/vendor/github.com/containers/common/libimage/image.go index b02bf9783a..b4623a8709 100644 --- a/vendor/github.com/containers/common/libimage/image.go +++ b/vendor/github.com/containers/common/libimage/image.go @@ -499,9 +499,15 @@ func (i *Image) Untag(name string) error { return errors.Wrapf(err, "error normalizing name %q", name) } - if _, isDigested := ref.(reference.Digested); isDigested { - return errors.Wrap(errUntagDigest, name) - } + // FIXME: this is breaking Podman CI but must be re-enabled once + // c/storage supports alterting the digests of an image. Then, + // Podman will do the right thing. + // + // !!! Also make sure to re-enable the tests !!! + // + // if _, isDigested := ref.(reference.Digested); isDigested { + // return errors.Wrap(errUntagDigest, name) + // } name = ref.String() diff --git a/vendor/github.com/containers/common/pkg/config/containers.conf b/vendor/github.com/containers/common/pkg/config/containers.conf index 023297501d..0068a9a174 100644 --- a/vendor/github.com/containers/common/pkg/config/containers.conf +++ b/vendor/github.com/containers/common/pkg/config/containers.conf @@ -60,23 +60,23 @@ default_capabilities = [ ] # A list of sysctls to be set in containers by default, -# specified as "name = value", -# for example:"net.ipv4.ping_group_range = 0 0". +# specified as "name=value", +# for example:"net.ipv4.ping_group_range=0 0". # default_sysctls = [ - "net.ipv4.ping_group_range = 0 0", + "net.ipv4.ping_group_range=0 0", ] # A list of ulimits to be set in containers by default, specified as -# " = :", for example: -# "nofile = 1024:2048" +# "=:", for example: +# "nofile=1024:2048" # See setrlimit(2) for a list of resource names. # Any limit not specified here will be inherited from the process launching the # container engine. # Ulimits has limits for non privileged container engines. # #default_ulimits = [ -# "nofile = 1280:2560", +# "nofile=1280:2560", #] # List of devices. Specified as @@ -382,7 +382,8 @@ default_sysctls = [ #machine_enabled = false # The image used when creating a podman-machine VM. -# machine_image = "testing" +# +#machine_image = "testing" # MultiImageArchive - if true, the container engine allows for storing archives # (e.g., of the docker-archive transport) with multiple images. By default, diff --git a/vendor/github.com/containers/common/pkg/parse/parse.go b/vendor/github.com/containers/common/pkg/parse/parse.go index 1a25957d60..02e670c50c 100644 --- a/vendor/github.com/containers/common/pkg/parse/parse.go +++ b/vendor/github.com/containers/common/pkg/parse/parse.go @@ -5,6 +5,7 @@ package parse import ( "os" + "path" "path/filepath" "strings" @@ -155,7 +156,7 @@ func ValidateVolumeCtrDir(ctrDir string) error { if ctrDir == "" { return errors.New("container directory cannot be empty") } - if !filepath.IsAbs(ctrDir) { + if !path.IsAbs(ctrDir) { return errors.Errorf("invalid container path %q, must be an absolute path", ctrDir) } return nil diff --git a/vendor/github.com/containers/common/version/version.go b/vendor/github.com/containers/common/version/version.go index ae372bee73..572fe9bbd3 100644 --- a/vendor/github.com/containers/common/version/version.go +++ b/vendor/github.com/containers/common/version/version.go @@ -1,4 +1,4 @@ package version // Version is the version of the build. -const Version = "0.43.0" +const Version = "0.43.2" diff --git a/vendor/modules.txt b/vendor/modules.txt index 2b8db54ff3..c9e56cdc06 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -94,7 +94,7 @@ github.com/containers/buildah/pkg/overlay github.com/containers/buildah/pkg/parse github.com/containers/buildah/pkg/rusage github.com/containers/buildah/util -# github.com/containers/common v0.43.0 +# github.com/containers/common v0.43.2 github.com/containers/common/libimage github.com/containers/common/libimage/manifests github.com/containers/common/pkg/apparmor From b20a5470597897931699b976a9ed9ad1b2651c42 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Wed, 18 Aug 2021 16:19:36 -0400 Subject: [PATCH 4/6] 330-corrupt-images: don't try to tag with a canonical name In these tests, don't try to tag an image using a canonical ("with digest") image name. Signed-off-by: Nalin Dahyabhai --- test/system/330-corrupt-images.bats | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/system/330-corrupt-images.bats b/test/system/330-corrupt-images.bats index 2ee5eee9c9..eeffff3ec2 100644 --- a/test/system/330-corrupt-images.bats +++ b/test/system/330-corrupt-images.bats @@ -13,7 +13,8 @@ if [ -z "${PODMAN_CORRUPT_TEST_WORKDIR}" ]; then export PODMAN_CORRUPT_TEST_WORKDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-${TMPDIR:-/tmp}} podman_corrupt_test.XXXXXX) fi -PODMAN_CORRUPT_TEST_IMAGE_FQIN=quay.io/libpod/alpine@sha256:634a8f35b5f16dcf4aaa0822adc0b1964bb786fca12f6831de8ddc45e5986a00 +PODMAN_CORRUPT_TEST_IMAGE_CANONICAL_FQIN=quay.io/libpod/alpine@sha256:634a8f35b5f16dcf4aaa0822adc0b1964bb786fca12f6831de8ddc45e5986a00 +PODMAN_CORRUPT_TEST_IMAGE_TAGGED_FQIN=${PODMAN_CORRUPT_TEST_IMAGE_CANONICAL_FQIN%%@sha256:*}:test PODMAN_CORRUPT_TEST_IMAGE_ID=961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4 # All tests in this file (and ONLY in this file) run with a custom rootdir @@ -59,7 +60,7 @@ function _corrupt_image_test() { run_podman load -i ${PODMAN_CORRUPT_TEST_WORKDIR}/img.tar # "podman load" restores it without a tag, which (a) causes rmi-by-name # to fail, and (b) causes "podman images" to exit 0 instead of 125 - run_podman tag ${PODMAN_CORRUPT_TEST_IMAGE_ID} ${PODMAN_CORRUPT_TEST_IMAGE_FQIN} + run_podman tag ${PODMAN_CORRUPT_TEST_IMAGE_ID} ${PODMAN_CORRUPT_TEST_IMAGE_TAGGED_FQIN} # shortcut variable name local id=${PODMAN_CORRUPT_TEST_IMAGE_ID} @@ -91,9 +92,9 @@ function _corrupt_image_test() { @test "podman corrupt images - initialize" { # Pull once, save cached copy. - run_podman pull $PODMAN_CORRUPT_TEST_IMAGE_FQIN + run_podman pull $PODMAN_CORRUPT_TEST_IMAGE_CANONICAL_FQIN run_podman save -o ${PODMAN_CORRUPT_TEST_WORKDIR}/img.tar \ - $PODMAN_CORRUPT_TEST_IMAGE_FQIN + $PODMAN_CORRUPT_TEST_IMAGE_CANONICAL_FQIN } # END first "test" does a one-time pull of our desired image @@ -104,8 +105,8 @@ function _corrupt_image_test() { _corrupt_image_test "rmi -f ${PODMAN_CORRUPT_TEST_IMAGE_ID}" } -@test "podman corrupt images - rmi -f " { - _corrupt_image_test "rmi -f ${PODMAN_CORRUPT_TEST_IMAGE_FQIN}" +@test "podman corrupt images - rmi -f " { + _corrupt_image_test "rmi -f ${PODMAN_CORRUPT_TEST_IMAGE_TAGGED_FQIN}" } @test "podman corrupt images - rmi -f -a" { From 21f396de6f5024abbf6edd2ca63edcb1525eefcc Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Thu, 19 Aug 2021 18:24:47 -0400 Subject: [PATCH 5/6] logs: adjust handling around partial log messages In libpod/logs.LogLine.Write(), don't write a newline to stdout/stderr when the log message is only part of a line. In libpod.ConmonOCIRuntime.HTTPAttach(), don't send a newline over the HTTP connection when the log message is only part of a line. In pkg/api/handlers/compat.LogsFromContainer(), don't send a newline over the HTTP connection when the log message is only part of a line, and don't make doing so conditional on whether or not the client used the docker or podman endpoint. In pkg/domain/infra/tunnel.ContainerEngine.ContainerLogs(), don't add our own newline to log messages, since they already come through from the server when they need to. Signed-off-by: Nalin Dahyabhai --- libpod/logs/log.go | 12 ++++++++++-- libpod/oci_conmon_linux.go | 8 +++++--- pkg/api/handlers/compat/containers_logs.go | 4 +--- pkg/domain/infra/tunnel/containers.go | 4 ++-- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/libpod/logs/log.go b/libpod/logs/log.go index 1a0223edc0..a584de0ee7 100644 --- a/libpod/logs/log.go +++ b/libpod/logs/log.go @@ -251,11 +251,19 @@ func (l *LogLine) Write(stdout io.Writer, stderr io.Writer, logOpts *LogOptions) switch l.Device { case "stdout": if stdout != nil { - fmt.Fprintln(stdout, l.String(logOpts)) + if l.Partial() { + fmt.Fprint(stdout, l.String(logOpts)) + } else { + fmt.Fprintln(stdout, l.String(logOpts)) + } } case "stderr": if stderr != nil { - fmt.Fprintln(stderr, l.String(logOpts)) + if l.Partial() { + fmt.Fprint(stderr, l.String(logOpts)) + } else { + fmt.Fprintln(stderr, l.String(logOpts)) + } } default: // Warn the user if the device type does not match. Most likely the file is corrupted. diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go index ff25be2342..c149119808 100644 --- a/libpod/oci_conmon_linux.go +++ b/libpod/oci_conmon_linux.go @@ -625,9 +625,11 @@ func (r *ConmonOCIRuntime) HTTPAttach(ctr *Container, req *http.Request, w http. if err != nil { break } - _, err = httpBuf.Write([]byte("\n")) - if err != nil { - break + if !logLine.Partial() { + _, err = httpBuf.Write([]byte("\n")) + if err != nil { + break + } } err = httpBuf.Flush() if err != nil { diff --git a/pkg/api/handlers/compat/containers_logs.go b/pkg/api/handlers/compat/containers_logs.go index 656e2c627c..0c10ce75e1 100644 --- a/pkg/api/handlers/compat/containers_logs.go +++ b/pkg/api/handlers/compat/containers_logs.go @@ -152,9 +152,7 @@ func LogsFromContainer(w http.ResponseWriter, r *http.Request) { } frame.WriteString(line.Msg) - // Log lines in the compat layer require adding EOL - // https://github.com/containers/podman/issues/8058 - if !utils.IsLibpodRequest(r) { + if !line.Partial() { frame.WriteString("\n") } diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index b638bfe24c..81ddce42f6 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -404,11 +404,11 @@ func (ic *ContainerEngine) ContainerLogs(_ context.Context, nameOrIDs []string, return err case line := <-stdoutCh: if opts.StdoutWriter != nil { - _, _ = io.WriteString(opts.StdoutWriter, line+"\n") + _, _ = io.WriteString(opts.StdoutWriter, line) } case line := <-stderrCh: if opts.StderrWriter != nil { - _, _ = io.WriteString(opts.StderrWriter, line+"\n") + _, _ = io.WriteString(opts.StderrWriter, line) } } } From 3007bd4a9939c0ed8160a319f70788c19e419435 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Mon, 23 Aug 2021 17:32:46 -0400 Subject: [PATCH 6/6] 130-kill.bats: increase timeouts from 10s to 60s Increase the amount of time we're willing to wait for a log message that a container should be printing to show up in the output of `logs -f`, since on at least one CI configuration we're seeing a turnaround as high as 46s, but it's not something we can directly control, so that's not a hard maximum. Signed-off-by: Nalin Dahyabhai --- test/system/130-kill.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/system/130-kill.bats b/test/system/130-kill.bats index 3770eac27c..d85f0a6a99 100644 --- a/test/system/130-kill.bats +++ b/test/system/130-kill.bats @@ -33,7 +33,7 @@ load helpers exec 5<$fifo # First container emits READY when ready; wait for it. - read -t 10 -u 5 ready + read -t 60 -u 5 ready is "$ready" "READY" "first log message from container" # Helper function: send the given signal, verify that it's received. @@ -42,7 +42,7 @@ load helpers local signum=${2:-$1} # e.g. if signal=HUP, we expect to see '1' run_podman kill -s $signal $cid - read -t 10 -u 5 actual || die "Timed out: no ACK for kill -s $signal" + read -t 60 -u 5 actual || die "Timed out: no ACK for kill -s $signal" is "$actual" "got: $signum" "Signal $signal handled by container" }