podman image tree: restore previous behavior

The initial version of libimage changed the order of layers which has
now been restored to remain backwards compatible.

Further changes:

 * Fix a bug in the journald logging which requires to strip trailing
   new lines from the message.  The system tests did not pass due to
   empty new lines.  Triggered by changing the default logger to
   journald in containers/common.

 * Fix another bug in the journald logging which embedded the container
   ID inside the message rather than the specifid field.  That surfaced
   in a preceeding whitespace of each log line which broke the system
   tests.

 * Alter the system tests to make sure that the k8s-file and the
   journald logging drivers are executed.

 * A number of e2e tests have been changed to force the k8s-file driver
   to make them pass when running inside a root container.

 * Increase the timeout in a kill test which seems to take longer now.
   Reasons are unknown.  Tests passed earlier and no signal-related
   changes happend.  It may be CI VM flake since some system tests but
   other flaked.

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>
This commit is contained in:
Valentin Rothberg 2021-05-05 17:08:17 +02:00
parent 59dd357509
commit d32863bbb4
32 changed files with 278 additions and 123 deletions

2
go.mod
View File

@ -12,7 +12,7 @@ require (
github.com/containernetworking/cni v0.8.1
github.com/containernetworking/plugins v0.9.1
github.com/containers/buildah v1.20.2-0.20210504130217-903dc56408ac
github.com/containers/common v0.37.2-0.20210503193405-42134aa138ce
github.com/containers/common v0.38.1-0.20210510140555-24645399a050
github.com/containers/conmon v2.0.20+incompatible
github.com/containers/image/v5 v5.12.0
github.com/containers/ocicrypt v1.1.1

3
go.sum
View File

@ -197,8 +197,9 @@ github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9
github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
github.com/containers/buildah v1.20.2-0.20210504130217-903dc56408ac h1:rPQTF+1lz+F4uTZgfk2pwqGcEEg9mPSWK58UncsqsrA=
github.com/containers/buildah v1.20.2-0.20210504130217-903dc56408ac/go.mod h1:0hqcxPCNk/lit/SwBQoXXymCbp2LUa07U0cwrn/T1c0=
github.com/containers/common v0.37.2-0.20210503193405-42134aa138ce h1:e7VNmGqwfUQkw+D5bms262x1HYqxfN9/+t5SoaFnwTk=
github.com/containers/common v0.37.2-0.20210503193405-42134aa138ce/go.mod h1:JjU+yvzIGyx8ZsY8nyf7snzs4VSNh1eIaYsqoSKBoRw=
github.com/containers/common v0.38.1-0.20210510140555-24645399a050 h1:o3UdnXpzCmJwto4y+addBH2NXZObuZ0tXA7COZXNMnQ=
github.com/containers/common v0.38.1-0.20210510140555-24645399a050/go.mod h1:64dWQkAgrd2cxVh9eDOxJ/IgPOulVFg6G7WLRDTrAuA=
github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
github.com/containers/image/v5 v5.11.1/go.mod h1:HC9lhJ/Nz5v3w/5Co7H431kLlgzlVlOC+auD/er3OqE=

View File

@ -8,12 +8,12 @@ import (
"fmt"
"io"
"math"
"strings"
"time"
"github.com/containers/podman/v3/libpod/define"
"github.com/containers/podman/v3/libpod/logs"
journal "github.com/coreos/go-systemd/v22/sdjournal"
"github.com/hpcloud/tail/watch"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@ -89,21 +89,19 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption
}
}()
go func() {
for {
state, err := c.State()
if err != nil {
until <- time.Time{}
logrus.Error(err)
break
}
time.Sleep(watch.POLL_DURATION)
if state != define.ContainerStateRunning && state != define.ContainerStatePaused {
until <- time.Time{}
break
}
}
// FIXME (#10323): we are facing a terrible
// race condition here. At the time the
// container dies and `c.Wait()` has returned,
// we may not have received all journald logs.
// So far there is no other way than waiting
// for a second. Ultimately, `r.Follow` is
// racy and we may have to implement our custom
// logic here.
c.Wait(ctx)
time.Sleep(time.Second)
until <- time.Time{}
}()
follower := FollowBuffer{logChannel}
follower := journaldFollowBuffer{logChannel, options.Multi}
err := r.Follow(until, follower)
if err != nil {
logrus.Debugf(err.Error())
@ -124,7 +122,7 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption
// because we are reusing bytes, we need to make
// sure the old data doesn't get into the new line
bytestr := string(bytes[:ec])
logLine, err2 := logs.NewLogLine(bytestr)
logLine, err2 := logs.NewJournaldLogLine(bytestr, options.Multi)
if err2 != nil {
logrus.Error(err2)
continue
@ -210,16 +208,18 @@ func formatterMessage(entry *journal.JournalEntry) (string, error) {
if !ok {
return "", fmt.Errorf("no MESSAGE field present in journal entry")
}
msg = strings.TrimSuffix(msg, "\n")
return msg, nil
}
type FollowBuffer struct {
type journaldFollowBuffer struct {
logChannel chan *logs.LogLine
withID bool
}
func (f FollowBuffer) Write(p []byte) (int, error) {
func (f journaldFollowBuffer) Write(p []byte) (int, error) {
bytestr := string(p)
logLine, err := logs.NewLogLine(bytestr)
logLine, err := logs.NewJournaldLogLine(bytestr, f.withID)
if err != nil {
return -1, err
}

View File

@ -206,6 +206,36 @@ func NewLogLine(line string) (*LogLine, error) {
return &l, nil
}
// NewJournaldLogLine creates a LogLine from the specified line from journald.
// Note that if withID is set, the first item of the message is considerred to
// be the container ID and set as such.
func NewJournaldLogLine(line string, withID bool) (*LogLine, error) {
splitLine := strings.Split(line, " ")
if len(splitLine) < 4 {
return nil, errors.Errorf("'%s' is not a valid container log line", line)
}
logTime, err := time.Parse(LogTimeFormat, splitLine[0])
if err != nil {
return nil, errors.Wrapf(err, "unable to convert time %s from container log", splitLine[0])
}
var msg, id string
if withID {
id = splitLine[3]
msg = strings.Join(splitLine[4:], " ")
} else {
msg = strings.Join(splitLine[3:], " ")
// NO ID
}
l := LogLine{
Time: logTime,
Device: splitLine[1],
ParseLogType: splitLine[2],
Msg: msg,
CID: id,
}
return &l, nil
}
// Partial returns a bool if the log line is a partial log type
func (l *LogLine) Partial() bool {
return l.ParseLogType == PartialLogType

View File

@ -60,8 +60,10 @@ var _ = Describe("Podman create with --ip flag", func() {
})
It("Podman create with specified static IP has correct IP", func() {
// NOTE: we force the k8s-file log driver to make sure the
// tests are passing inside a container.
ip := GetRandomIPAddress()
result := podmanTest.Podman([]string{"create", "--name", "test", "--ip", ip, ALPINE, "ip", "addr"})
result := podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test", "--ip", ip, ALPINE, "ip", "addr"})
result.WaitWithDefaultTimeout()
// Rootless static ip assignment without network should error
if rootless.IsRootless() {
@ -83,10 +85,10 @@ var _ = Describe("Podman create with --ip flag", func() {
It("Podman create two containers with the same IP", func() {
SkipIfRootless("--ip not supported without network in rootless mode")
ip := GetRandomIPAddress()
result := podmanTest.Podman([]string{"create", "--name", "test1", "--ip", ip, ALPINE, "sleep", "999"})
result := podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test1", "--ip", ip, ALPINE, "sleep", "999"})
result.WaitWithDefaultTimeout()
Expect(result.ExitCode()).To(Equal(0))
result = podmanTest.Podman([]string{"create", "--name", "test2", "--ip", ip, ALPINE, "ip", "addr"})
result = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test2", "--ip", ip, ALPINE, "ip", "addr"})
result.WaitWithDefaultTimeout()
Expect(result.ExitCode()).To(Equal(0))
result = podmanTest.Podman([]string{"start", "test1"})

View File

@ -160,9 +160,12 @@ var _ = Describe("Podman create", func() {
if podmanTest.Host.Arch == "ppc64le" {
Skip("skip failing test on ppc64le")
}
// NOTE: we force the k8s-file log driver to make sure the
// tests are passing inside a container.
mountPath := filepath.Join(podmanTest.TempDir, "secrets")
os.Mkdir(mountPath, 0755)
session := podmanTest.Podman([]string{"create", "--name", "test", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
session := podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
session = podmanTest.Podman([]string{"start", "test"})
@ -173,7 +176,7 @@ var _ = Describe("Podman create", func() {
Expect(session.ExitCode()).To(Equal(0))
Expect(session.OutputToString()).To(ContainSubstring("/create/test rw"))
session = podmanTest.Podman([]string{"create", "--name", "test_ro", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test,ro", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test_ro", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test,ro", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
session = podmanTest.Podman([]string{"start", "test_ro"})
@ -184,7 +187,7 @@ var _ = Describe("Podman create", func() {
Expect(session.ExitCode()).To(Equal(0))
Expect(session.OutputToString()).To(ContainSubstring("/create/test ro"))
session = podmanTest.Podman([]string{"create", "--name", "test_shared", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test,shared", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test_shared", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test,shared", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
session = podmanTest.Podman([]string{"start", "test_shared"})
@ -200,7 +203,7 @@ var _ = Describe("Podman create", func() {
mountPath = filepath.Join(podmanTest.TempDir, "scratchpad")
os.Mkdir(mountPath, 0755)
session = podmanTest.Podman([]string{"create", "--name", "test_tmpfs", "--mount", "type=tmpfs,target=/create/test", ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test_tmpfs", "--mount", "type=tmpfs,target=/create/test", ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
session = podmanTest.Podman([]string{"start", "test_tmpfs"})

View File

@ -649,11 +649,13 @@ var _ = Describe("Podman run networking", func() {
defer podmanTest.removeCNINetwork(netName)
name := "nc-server"
run := podmanTest.Podman([]string{"run", "-d", "--name", name, "--net", netName, ALPINE, "nc", "-l", "-p", "8080"})
run := podmanTest.Podman([]string{"run", "--log-driver", "k8s-file", "-d", "--name", name, "--net", netName, ALPINE, "nc", "-l", "-p", "8080"})
run.WaitWithDefaultTimeout()
Expect(run.ExitCode()).To(Equal(0))
run = podmanTest.Podman([]string{"run", "--rm", "--net", netName, "--uidmap", "0:1:4096", ALPINE, "sh", "-c", fmt.Sprintf("echo podman | nc -w 1 %s.dns.podman 8080", name)})
// NOTE: we force the k8s-file log driver to make sure the
// tests are passing inside a container.
run = podmanTest.Podman([]string{"run", "--log-driver", "k8s-file", "--rm", "--net", netName, "--uidmap", "0:1:4096", ALPINE, "sh", "-c", fmt.Sprintf("echo podman | nc -w 1 %s.dns.podman 8080", name)})
run.WaitWithDefaultTimeout()
Expect(run.ExitCode()).To(Equal(0))

View File

@ -712,7 +712,7 @@ USER bin`, BB)
It("podman run log-opt", func() {
log := filepath.Join(podmanTest.TempDir, "/container.log")
session := podmanTest.Podman([]string{"run", "--rm", "--log-opt", fmt.Sprintf("path=%s", log), ALPINE, "ls"})
session := podmanTest.Podman([]string{"run", "--rm", "--log-driver", "k8s-file", "--log-opt", fmt.Sprintf("path=%s", log), ALPINE, "ls"})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
_, err := os.Stat(log)

View File

@ -215,7 +215,7 @@ var _ = Describe("Toolbox-specific testing", func() {
useradd := fmt.Sprintf("useradd --home-dir %s --shell %s --uid %s %s",
homeDir, shell, uid, username)
passwd := fmt.Sprintf("passwd --delete %s", username)
session = podmanTest.Podman([]string{"create", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
fmt.Sprintf("%s; %s; echo READY; sleep 1000", useradd, passwd)})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
@ -250,7 +250,7 @@ var _ = Describe("Toolbox-specific testing", func() {
groupadd := fmt.Sprintf("groupadd --gid %s %s", gid, groupName)
session = podmanTest.Podman([]string{"create", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
fmt.Sprintf("%s; echo READY; sleep 1000", groupadd)})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
@ -294,7 +294,7 @@ var _ = Describe("Toolbox-specific testing", func() {
usermod := fmt.Sprintf("usermod --append --groups wheel --home %s --shell %s --uid %s --gid %s %s",
homeDir, shell, uid, gid, username)
session = podmanTest.Podman([]string{"create", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
fmt.Sprintf("%s; %s; %s; echo READY; sleep 1000", useradd, groupadd, usermod)})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
@ -339,6 +339,7 @@ var _ = Describe("Toolbox-specific testing", func() {
// These should be most of the switches that Toolbox uses to create a "toolbox" container
// https://github.com/containers/toolbox/blob/master/src/cmd/create.go
session = podmanTest.Podman([]string{"create",
"--log-driver", "k8s-file",
"--dns", "none",
"--hostname", "toolbox",
"--ipc", "host",

View File

@ -27,13 +27,22 @@ load helpers
run_podman rm $cid
}
@test "podman logs - multi" {
function _log_test_multi() {
local driver=$1
skip_if_remote "logs does not support multiple containers when run remotely"
# Under k8s file, 'podman logs' returns just the facts, Ma'am.
# Under journald, there may be other cruft (e.g. container removals)
local etc=
if [[ $driver =~ journal ]]; then
etc='.*'
fi
# Simple helper to make the container starts, below, easier to read
local -a cid
doit() {
run_podman run --rm -d --name "$1" $IMAGE sh -c "$2";
run_podman run --log-driver=$driver --rm -d --name "$1" $IMAGE sh -c "$2";
cid+=($(echo "${output:0:12}"))
}
@ -47,24 +56,21 @@ load helpers
run_podman logs -f c1 c2
is "$output" \
"${cid[0]} a
${cid[1]} b
${cid[1]} c
"${cid[0]} a$etc
${cid[1]} b$etc
${cid[1]} c$etc
${cid[0]} d" "Sequential output from logs"
}
@test "podman logs over journald" {
@test "podman logs - multi k8s-file" {
_log_test_multi k8s-file
}
@test "podman logs - multi journald" {
# We can't use journald on RHEL as rootless: rhbz#1895105
skip_if_journald_unavailable
msg=$(random_string 20)
run_podman run --name myctr --log-driver journald $IMAGE echo $msg
run_podman logs myctr
is "$output" "$msg" "check that log output equals the container output"
run_podman rm myctr
_log_test_multi journald
}
# vim: filetype=sh

View File

@ -393,9 +393,9 @@ Labels.$label_name | $label_value
"image tree: third line"
is "${lines[3]}" "Image Layers" \
"image tree: fourth line"
is "${lines[4]}" ".* ID: [0-9a-f]\{12\} Size: .* Top Layer of: \[localhost/build_test:latest]" \
is "${lines[4]}" ".* ID: [0-9a-f]\{12\} Size: .* Top Layer of: \[$IMAGE]" \
"image tree: first layer line"
is "${lines[-1]}" ".* ID: [0-9a-f]\{12\} Size: .* Top Layer of: \[$IMAGE]" \
is "${lines[-1]}" ".* ID: [0-9a-f]\{12\} Size: .* Top Layer of: \[localhost/build_test:latest]" \
"image tree: last layer line"
# FIXME: 'image tree --whatrequires' does not work via remote

View File

@ -8,7 +8,8 @@ load helpers
@test "podman kill - test signal handling in containers" {
# Start a container that will handle all signals by emitting 'got: N'
local -a signals=(1 2 3 4 5 6 8 10 12 13 14 15 16 20 21 22 23 24 25 26 64)
run_podman run -d $IMAGE sh -c \
# Force the k8s-file driver until #10323 is fixed.
run_podman run --log-driver=k8s-file -d $IMAGE sh -c \
"for i in ${signals[*]}; do trap \"echo got: \$i\" \$i; done;
echo READY;
while ! test -e /stop; do sleep 0.05; done;

View File

@ -1,14 +1,18 @@
package libimage
import "time"
import (
"time"
// EventType indicates the type of an event. Currrently, there is only one
"github.com/sirupsen/logrus"
)
// EventType indicates the type of an event. Currently, there is only one
// supported type for container image but we may add more (e.g., for manifest
// lists) in the future.
type EventType int
const (
// EventTypeUnknow is an unitialized EventType.
// EventTypeUnknown is an uninitialized EventType.
EventTypeUnknown EventType = iota
// EventTypeImagePull represents an image pull.
EventTypeImagePull
@ -26,7 +30,7 @@ const (
EventTypeImageUntag
// EventTypeImageMount represents an image being mounted.
EventTypeImageMount
// EventTypeImageUnmounted represents an image being unmounted.
// EventTypeImageUnmount represents an image being unmounted.
EventTypeImageUnmount
)
@ -41,3 +45,18 @@ type Event struct {
// Type of the event.
Type EventType
}
// writeEvent writes the specified event to the Runtime's event channel. The
// event is discarded if no event channel has been registered (yet).
func (r *Runtime) writeEvent(event *Event) {
select {
case r.eventChannel <- event:
// Done
case <-time.After(2 * time.Second):
// The Runtime's event channel has a buffer of size 100 which
// should be enough even under high load. However, we
// shouldn't block too long in case the buffer runs full (could
// be an honest user error or bug).
logrus.Warnf("Discarding libimage event which was not read within 2 seconds: %v", event)
}
}

View File

@ -277,6 +277,10 @@ func (i *Image) remove(ctx context.Context, rmMap map[string]*RemoveImageReport,
return errors.Errorf("cannot remove read-only image %q", i.ID())
}
if i.runtime.eventChannel != nil {
i.runtime.writeEvent(&Event{ID: i.ID(), Name: referencedBy, Time: time.Now(), Type: EventTypeImageRemove})
}
// Check if already visisted this image.
report, exists := rmMap[i.ID()]
if exists {
@ -423,6 +427,9 @@ func (i *Image) Tag(name string) error {
}
logrus.Debugf("Tagging image %s with %q", i.ID(), ref.String())
if i.runtime.eventChannel != nil {
i.runtime.writeEvent(&Event{ID: i.ID(), Name: name, Time: time.Now(), Type: EventTypeImageTag})
}
newNames := append(i.Names(), ref.String())
if err := i.runtime.store.SetNames(i.ID(), newNames); err != nil {
@ -454,6 +461,9 @@ func (i *Image) Untag(name string) error {
name = ref.String()
logrus.Debugf("Untagging %q from image %s", ref.String(), i.ID())
if i.runtime.eventChannel != nil {
i.runtime.writeEvent(&Event{ID: i.ID(), Name: name, Time: time.Now(), Type: EventTypeImageUntag})
}
removedName := false
newNames := []string{}
@ -593,6 +603,10 @@ func (i *Image) RepoDigests() ([]string, error) {
// are directly passed down to the containers storage. Returns the fully
// evaluated path to the mount point.
func (i *Image) Mount(ctx context.Context, mountOptions []string, mountLabel string) (string, error) {
if i.runtime.eventChannel != nil {
i.runtime.writeEvent(&Event{ID: i.ID(), Name: "", Time: time.Now(), Type: EventTypeImageMount})
}
mountPoint, err := i.runtime.store.MountImage(i.ID(), mountOptions, mountLabel)
if err != nil {
return "", err
@ -634,6 +648,9 @@ func (i *Image) Mountpoint() (string, error) {
// Unmount the image. Use force to ignore the reference counter and forcefully
// unmount.
func (i *Image) Unmount(force bool) error {
if i.runtime.eventChannel != nil {
i.runtime.writeEvent(&Event{ID: i.ID(), Name: "", Time: time.Now(), Type: EventTypeImageUnmount})
}
logrus.Debugf("Unmounted image %s", i.ID())
_, err := i.runtime.store.UnmountImage(i.ID(), force)
return err

View File

@ -35,36 +35,45 @@ func (i *Image) Tree(traverseChildren bool) (string, error) {
fmt.Fprintf(sb, "No Image Layers")
}
tree := gotree.New(sb.String())
layerTree, err := i.runtime.layerTree()
if err != nil {
return "", err
}
imageNode := layerTree.node(i.TopLayer())
// Traverse the entire tree down to all children.
if traverseChildren {
tree := gotree.New(sb.String())
if err := imageTreeTraverseChildren(imageNode, tree); err != nil {
return "", err
}
} else {
// Walk all layers of the image and assemlbe their data.
for parentNode := imageNode; parentNode != nil; parentNode = parentNode.parent {
if parentNode.layer == nil {
break // we're done
}
var tags string
repoTags, err := parentNode.repoTags()
if err != nil {
return "", err
}
if len(repoTags) > 0 {
tags = fmt.Sprintf(" Top Layer of: %s", repoTags)
}
tree.Add(fmt.Sprintf("ID: %s Size: %7v%s", parentNode.layer.ID[:12], units.HumanSizeWithPrecision(float64(parentNode.layer.UncompressedSize), 4), tags))
return tree.Print(), nil
}
// Walk all layers of the image and assemlbe their data. Note that the
// tree is constructed in reverse order to remain backwards compatible
// with Podman.
contents := []string{}
for parentNode := imageNode; parentNode != nil; parentNode = parentNode.parent {
if parentNode.layer == nil {
break // we're done
}
var tags string
repoTags, err := parentNode.repoTags()
if err != nil {
return "", err
}
if len(repoTags) > 0 {
tags = fmt.Sprintf(" Top Layer of: %s", repoTags)
}
content := fmt.Sprintf("ID: %s Size: %7v%s", parentNode.layer.ID[:12], units.HumanSizeWithPrecision(float64(parentNode.layer.UncompressedSize), 4), tags)
contents = append(contents, content)
}
contents = append(contents, sb.String())
tree := gotree.New(contents[len(contents)-1])
for i := len(contents) - 2; i >= 0; i-- {
tree.Add(contents[i])
}
return tree.Print(), nil
@ -80,14 +89,22 @@ func imageTreeTraverseChildren(node *layerNode, parent gotree.Tree) error {
tags = fmt.Sprintf(" Top Layer of: %s", repoTags)
}
newNode := parent.Add(fmt.Sprintf("ID: %s Size: %7v%s", node.layer.ID[:12], units.HumanSizeWithPrecision(float64(node.layer.UncompressedSize), 4), tags))
content := fmt.Sprintf("ID: %s Size: %7v%s", node.layer.ID[:12], units.HumanSizeWithPrecision(float64(node.layer.UncompressedSize), 4), tags)
if len(node.children) <= 1 {
newNode = parent
var newTree gotree.Tree
if node.parent == nil || len(node.parent.children) <= 1 {
// No parent or no siblings, so we can go linear.
parent.Add(content)
newTree = parent
} else {
// Each siblings gets a new tree, so we can branch.
newTree = gotree.New(content)
parent.AddTree(newTree)
}
for i := range node.children {
child := node.children[i]
if err := imageTreeTraverseChildren(child, newNode); err != nil {
if err := imageTreeTraverseChildren(child, newTree); err != nil {
return err
}
}

View File

@ -59,7 +59,7 @@ func (l *layerNode) repoTags() ([]string, error) {
return nil, err
}
for _, tag := range repoTags {
if _, visted := visitedTags[tag]; visted {
if _, visited := visitedTags[tag]; visited {
continue
}
visitedTags[tag] = true

View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"os"
"time"
dirTransport "github.com/containers/image/v5/directory"
dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
@ -23,6 +24,10 @@ type LoadOptions struct {
func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) ([]string, error) {
logrus.Debugf("Loading image from %q", path)
if r.eventChannel != nil {
r.writeEvent(&Event{ID: "", Name: path, Time: time.Now(), Type: EventTypeImageLoad})
}
var (
loadedImages []string
loadError error

View File

@ -3,6 +3,7 @@ package libimage
import (
"context"
"fmt"
"time"
"github.com/containers/common/libimage/manifests"
imageCopy "github.com/containers/image/v5/copy"
@ -364,6 +365,10 @@ func (m *ManifestList) Push(ctx context.Context, destination string, options *Ma
}
}
if m.image.runtime.eventChannel != nil {
m.image.runtime.writeEvent(&Event{ID: m.ID(), Name: destination, Time: time.Now(), Type: EventTypeImagePush})
}
// NOTE: we're using the logic in copier to create a proper
// types.SystemContext. This prevents us from having an error prone
// code duplicate here.

View File

@ -5,10 +5,10 @@ import (
"fmt"
"io"
"strings"
"time"
"github.com/containers/common/pkg/config"
dirTransport "github.com/containers/image/v5/directory"
dockerTransport "github.com/containers/image/v5/docker"
registryTransport "github.com/containers/image/v5/docker"
dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
"github.com/containers/image/v5/docker/reference"
ociArchiveTransport "github.com/containers/image/v5/oci/archive"
@ -42,7 +42,7 @@ type PullOptions struct {
// policies (e.g., buildah-bud versus podman-build). Making the pull-policy
// choice explicit is an attempt to prevent silent regressions.
//
// The errror is storage.ErrImageUnknown iff the pull policy is set to "never"
// The error is storage.ErrImageUnknown iff the pull policy is set to "never"
// and no local image has been found. This allows for an easier integration
// into some users of this package (e.g., Buildah).
func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullPolicy, options *PullOptions) ([]*Image, error) {
@ -56,7 +56,7 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
if err != nil {
// If the image clearly refers to a local one, we can look it up directly.
// In fact, we need to since they are not parseable.
if strings.HasPrefix(name, "sha256:") || (len(name) == 64 && !strings.Contains(name, "/.:@")) {
if strings.HasPrefix(name, "sha256:") || (len(name) == 64 && !strings.ContainsAny(name, "/.:@")) {
if pullPolicy == config.PullPolicyAlways {
return nil, errors.Errorf("pull policy is always but image has been referred to by ID (%s)", name)
}
@ -76,10 +76,14 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
ref = dockerRef
}
if options.AllTags && ref.Transport().Name() != dockerTransport.Transport.Name() {
if options.AllTags && ref.Transport().Name() != registryTransport.Transport.Name() {
return nil, errors.Errorf("pulling all tags is not supported for %s transport", ref.Transport().Name())
}
if r.eventChannel != nil {
r.writeEvent(&Event{ID: "", Name: name, Time: time.Now(), Type: EventTypeImagePull})
}
var (
pulledImages []string
pullError error
@ -88,29 +92,17 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
// Dispatch the copy operation.
switch ref.Transport().Name() {
// DOCKER/REGISTRY
case dockerTransport.Transport.Name():
// DOCKER REGISTRY
case registryTransport.Transport.Name():
pulledImages, pullError = r.copyFromRegistry(ctx, ref, strings.TrimPrefix(name, "docker://"), pullPolicy, options)
// DOCKER ARCHIVE
case dockerArchiveTransport.Transport.Name():
pulledImages, pullError = r.copyFromDockerArchive(ctx, ref, &options.CopyOptions)
// OCI
case ociTransport.Transport.Name():
pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
// OCI ARCHIVE
case ociArchiveTransport.Transport.Name():
pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
// DIR
case dirTransport.Transport.Name():
pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
// UNSUPPORTED
// ALL OTHER TRANSPORTS
default:
return nil, errors.Errorf("unsupported transport %q for pulling", ref.Transport().Name())
pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
}
if pullError != nil {
@ -162,7 +154,12 @@ func (r *Runtime) copyFromDefault(ctx context.Context, ref types.ImageReference,
imageName = "sha256:" + storageName[1:]
} else {
storageName = manifest.Annotations["org.opencontainers.image.ref.name"]
imageName = storageName
named, err := NormalizeName(storageName)
if err != nil {
return nil, err
}
imageName = named.String()
storageName = imageName
}
default:
@ -275,7 +272,7 @@ func (r *Runtime) copyFromRegistry(ctx context.Context, ref types.ImageReference
}
named := reference.TrimNamed(ref.DockerReference())
tags, err := dockerTransport.GetRepositoryTags(ctx, &r.systemContext, ref)
tags, err := registryTransport.GetRepositoryTags(ctx, &r.systemContext, ref)
if err != nil {
return nil, err
}
@ -399,7 +396,7 @@ func (r *Runtime) copySingleImageFromRegistry(ctx context.Context, imageName str
for _, candidate := range resolved.PullCandidates {
candidateString := candidate.Value.String()
logrus.Debugf("Attempting to pull candidate %s for %s", candidateString, imageName)
srcRef, err := dockerTransport.NewReference(candidate.Value)
srcRef, err := registryTransport.NewReference(candidate.Value)
if err != nil {
return nil, err
}

View File

@ -2,6 +2,7 @@ package libimage
import (
"context"
"time"
dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
"github.com/containers/image/v5/docker/reference"
@ -61,8 +62,12 @@ func (r *Runtime) Push(ctx context.Context, source, destination string, options
destRef = dockerRef
}
if r.eventChannel != nil {
r.writeEvent(&Event{ID: image.ID(), Name: destination, Time: time.Now(), Type: EventTypeImagePush})
}
// Buildah compat: Make sure to tag the destination image if it's a
// Docker archive. This way, we preseve the image name.
// Docker archive. This way, we preserve the image name.
if destRef.Transport().Name() == dockerArchiveTransport.Transport.Name() {
if named, err := reference.ParseNamed(resolvedSource); err == nil {
tagged, isTagged := named.(reference.NamedTagged)

View File

@ -20,6 +20,9 @@ import (
// RuntimeOptions allow for creating a customized Runtime.
type RuntimeOptions struct {
// The base system context of the runtime which will be used throughout
// the entire lifespan of the Runtime. Certain options in some
// functions may override specific fields.
SystemContext *types.SystemContext
}
@ -41,6 +44,8 @@ func setRegistriesConfPath(systemContext *types.SystemContext) {
// Runtime is responsible for image management and storing them in a containers
// storage.
type Runtime struct {
// Use to send events out to users.
eventChannel chan *Event
// Underlying storage store.
store storage.Store
// Global system context. No pointer to simplify copying and modifying
@ -55,6 +60,18 @@ func (r *Runtime) systemContextCopy() *types.SystemContext {
return &sys
}
// EventChannel creates a buffered channel for events that the Runtime will use
// to write events to. Callers are expected to read from the channel in a
// timely manner.
// Can be called once for a given Runtime.
func (r *Runtime) EventChannel() chan *Event {
if r.eventChannel != nil {
return r.eventChannel
}
r.eventChannel = make(chan *Event, 100)
return r.eventChannel
}
// RuntimeFromStore returns a Runtime for the specified store.
func RuntimeFromStore(store storage.Store, options *RuntimeOptions) (*Runtime, error) {
if options == nil {
@ -99,6 +116,9 @@ func RuntimeFromStoreOptions(runtimeOptions *RuntimeOptions, storeOptions *stora
// is considered to be an error condition.
func (r *Runtime) Shutdown(force bool) error {
_, err := r.store.Shutdown(force)
if r.eventChannel != nil {
close(r.eventChannel)
}
return err
}

View File

@ -3,6 +3,7 @@ package libimage
import (
"context"
"strings"
"time"
dirTransport "github.com/containers/image/v5/directory"
dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
@ -46,7 +47,7 @@ func (r *Runtime) Save(ctx context.Context, names []string, format, path string,
// All formats support saving 1.
default:
if format != "docker-archive" {
return errors.Errorf("unspported format %q for saving multiple images (only docker-archive)", format)
return errors.Errorf("unsupported format %q for saving multiple images (only docker-archive)", format)
}
if len(options.AdditionalTags) > 0 {
return errors.Errorf("cannot save multiple images with multiple tags")
@ -62,7 +63,7 @@ func (r *Runtime) Save(ctx context.Context, names []string, format, path string,
return r.saveDockerArchive(ctx, names, path, options)
}
return errors.Errorf("unspported format %q for saving images", format)
return errors.Errorf("unsupported format %q for saving images", format)
}
@ -74,6 +75,10 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string
return err
}
if r.eventChannel != nil {
r.writeEvent(&Event{ID: image.ID(), Name: path, Time: time.Now(), Type: EventTypeImageSave})
}
// Unless the image was referenced by ID, use the resolved name as a
// tag.
var tag string
@ -101,7 +106,7 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string
options.ManifestMIMEType = manifest.DockerV2Schema2MediaType
default:
return errors.Errorf("unspported format %q for saving images", format)
return errors.Errorf("unsupported format %q for saving images", format)
}
if err != nil {
@ -160,6 +165,9 @@ func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path st
}
}
localImages[image.ID()] = local
if r.eventChannel != nil {
r.writeEvent(&Event{ID: image.ID(), Name: path, Time: time.Now(), Type: EventTypeImageSave})
}
}
writer, err := dockerArchiveTransport.NewWriter(r.systemContextCopy(), path)

View File

@ -7,7 +7,7 @@ import (
"strings"
"sync"
dockerTransport "github.com/containers/image/v5/docker"
registryTransport "github.com/containers/image/v5/docker"
"github.com/containers/image/v5/pkg/sysregistriesv2"
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
@ -193,7 +193,7 @@ func (r *Runtime) searchImageInRegistry(ctx context.Context, term, registry stri
return results, nil
}
results, err := dockerTransport.SearchRegistry(ctx, sys, registry, term, limit)
results, err := registryTransport.SearchRegistry(ctx, sys, registry, term, limit)
if err != nil {
return []SearchResult{}, err
}
@ -255,7 +255,7 @@ func (r *Runtime) searchImageInRegistry(ctx context.Context, term, registry stri
func searchRepositoryTags(ctx context.Context, sys *types.SystemContext, registry, term string, options *SearchOptions) ([]SearchResult, error) {
dockerPrefix := "docker://"
imageRef, err := alltransports.ParseImageName(fmt.Sprintf("%s/%s", registry, term))
if err == nil && imageRef.Transport().Name() != dockerTransport.Transport.Name() {
if err == nil && imageRef.Transport().Name() != registryTransport.Transport.Name() {
return nil, errors.Errorf("reference %q must be a docker reference", term)
} else if err != nil {
imageRef, err = alltransports.ParseImageName(fmt.Sprintf("%s%s", dockerPrefix, fmt.Sprintf("%s/%s", registry, term)))
@ -263,7 +263,7 @@ func searchRepositoryTags(ctx context.Context, sys *types.SystemContext, registr
return nil, errors.Errorf("reference %q must be a docker reference", term)
}
}
tags, err := dockerTransport.GetRepositoryTags(ctx, sys, imageRef)
tags, err := registryTransport.GetRepositoryTags(ctx, sys, imageRef)
if err != nil {
return nil, errors.Errorf("error getting repository tags: %v", err)
}
@ -288,18 +288,18 @@ func searchRepositoryTags(ctx context.Context, sys *types.SystemContext, registr
return paramsArr, nil
}
func (f *SearchFilter) matchesStarFilter(result dockerTransport.SearchResult) bool {
func (f *SearchFilter) matchesStarFilter(result registryTransport.SearchResult) bool {
return result.StarCount >= f.Stars
}
func (f *SearchFilter) matchesAutomatedFilter(result dockerTransport.SearchResult) bool {
func (f *SearchFilter) matchesAutomatedFilter(result registryTransport.SearchResult) bool {
if f.IsAutomated != types.OptionalBoolUndefined {
return result.IsAutomated == (f.IsAutomated == types.OptionalBoolTrue)
}
return true
}
func (f *SearchFilter) matchesOfficialFilter(result dockerTransport.SearchResult) bool {
func (f *SearchFilter) matchesOfficialFilter(result registryTransport.SearchResult) bool {
if f.IsOfficial != types.OptionalBoolUndefined {
return result.IsOfficial == (f.IsOfficial == types.OptionalBoolTrue)
}

View File

@ -232,7 +232,7 @@ type EngineConfig struct {
// will fall back to containers/image defaults.
ImageParallelCopies uint `toml:"image_parallel_copies,omitempty"`
// ImageDefaultFormat sepecified the manifest Type (oci, v2s2, or v2s1)
// ImageDefaultFormat specified the manifest Type (oci, v2s2, or v2s1)
// to use when pulling, pushing, building container images. By default
// image pulled and pushed match the format of the source image.
// Building/committing defaults to OCI.
@ -425,6 +425,12 @@ type NetworkConfig struct {
// to attach pods to.
DefaultNetwork string `toml:"default_network,omitempty"`
// DefaultSubnet is the subnet to be used for the default CNI network.
// If a network with the name given in DefaultNetwork is not present
// then a new network using this subnet will be created.
// Must be a valid IPv4 CIDR block.
DefaultSubnet string `toml:"default_subnet,omitempty"`
// NetworkConfigDir is where CNI network configuration files are stored.
NetworkConfigDir string `toml:"network_config_dir,omitempty"`
}

View File

@ -157,7 +157,7 @@ default_sysctls = [
# Logging driver for the container. Available options: k8s-file and journald.
#
# log_driver = "k8s-file"
# log_driver = "journald"
# Maximum size allowed for the container log file. Negative numbers indicate
# that no size limit is imposed. If positive, it must be >= 8192 to match or
@ -243,6 +243,12 @@ default_sysctls = [
# The network name of the default CNI network to attach pods to.
# default_network = "podman"
# The default subnet for the default CNI network given in default_network.
# If a network with that name does not exist, a new network using that name and
# this subnet will be created.
# Must be a valid IPv4 CIDR prefix.
#default_subnet = "10.88.0.0/16"
# Path to the directory where CNI configuration files are located.
#
# network_config_dir = "/etc/cni/net.d/"
@ -254,7 +260,7 @@ default_sysctls = [
# Manifest Type (oci, v2s2, or v2s1) to use when pulling, pushing, building
# container images. By default image pulled and pushed match the format of the
# source image. Building/commiting defaults to OCI.
# source image. Building/committing defaults to OCI.
# image_default_format = ""
# Cgroup management implementation used for the runtime.

View File

@ -102,7 +102,7 @@ const (
// SystemdCgroupsManager represents systemd native cgroup manager
SystemdCgroupsManager = "systemd"
// DefaultLogDriver is the default type of log files
DefaultLogDriver = "k8s-file"
DefaultLogDriver = "journald"
// DefaultLogSizeMax is the default value for the maximum log size
// allowed for a container. Negative values mean that no limit is imposed.
DefaultLogSizeMax = -1
@ -114,6 +114,9 @@ const (
// DefaultSignaturePolicyPath is the default value for the
// policy.json file.
DefaultSignaturePolicyPath = "/etc/containers/policy.json"
// DefaultSubnet is the subnet that will be used for the default CNI
// network.
DefaultSubnet = "10.88.0.0/16"
// DefaultRootlessSignaturePolicyPath is the location within
// XDG_CONFIG_HOME of the rootless policy.json file.
DefaultRootlessSignaturePolicyPath = "containers/policy.json"
@ -204,6 +207,7 @@ func DefaultConfig() (*Config, error) {
},
Network: NetworkConfig{
DefaultNetwork: "podman",
DefaultSubnet: DefaultSubnet,
NetworkConfigDir: cniConfig,
CNIPluginDirs: cniBinDir,
},

View File

@ -96,7 +96,7 @@ func PrepareFilters(r *http.Request) (map[string][]string, error) {
return filterMap, nil
}
// MatchLabelFilters matches labels and returs true if they are valid
// MatchLabelFilters matches labels and returns true if they are valid
func MatchLabelFilters(filterValues []string, labels map[string]string) bool {
outer:
for _, filterValue := range filterValues {

View File

@ -33,7 +33,7 @@ type Driver struct {
func NewDriver(rootPath string) (*Driver, error) {
fileDriver := new(Driver)
fileDriver.secretsDataFilePath = filepath.Join(rootPath, secretsDataFile)
// the lockfile functions requre that the rootPath dir is executable
// the lockfile functions require that the rootPath dir is executable
if err := os.MkdirAll(rootPath, 0700); err != nil {
return nil, err
}

View File

@ -99,7 +99,7 @@ func NewManager(rootPath string) (*SecretsManager, error) {
if !filepath.IsAbs(rootPath) {
return nil, errors.Wrapf(errInvalidPath, "path must be absolute: %s", rootPath)
}
// the lockfile functions requre that the rootPath dir is executable
// the lockfile functions require that the rootPath dir is executable
if err := os.MkdirAll(rootPath, 0700); err != nil {
return nil, err
}

View File

@ -76,7 +76,7 @@ func (s *SecretsManager) getNameAndID(nameOrID string) (name, id string, err err
}
// ID prefix may have been given, iterate through all IDs.
// ID and partial ID has a max lenth of 25, so we return if its greater than that.
// ID and partial ID has a max length of 25, so we return if its greater than that.
if len(nameOrID) > secretIDLength {
return "", "", errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
}

View File

@ -1,4 +1,4 @@
package version
// Version is the version of the build.
const Version = "0.37.2-dev"
const Version = "0.38.1-dev"

2
vendor/modules.txt vendored
View File

@ -90,7 +90,7 @@ github.com/containers/buildah/pkg/overlay
github.com/containers/buildah/pkg/parse
github.com/containers/buildah/pkg/rusage
github.com/containers/buildah/util
# github.com/containers/common v0.37.2-0.20210503193405-42134aa138ce
# github.com/containers/common v0.38.1-0.20210510140555-24645399a050
github.com/containers/common/libimage
github.com/containers/common/libimage/manifests
github.com/containers/common/pkg/apparmor