From 6e0decbe03bfc819c018d435e8d908170e32ad29 Mon Sep 17 00:00:00 2001 From: Matt Heon Date: Tue, 6 Feb 2024 08:24:28 -0500 Subject: [PATCH] Send container stats over API on a per-interface basis This mirrors how the Docker API handles things, allowing us to be more compatible with Docker and more verbose on the Libpod API. Stats are given as per network interface in the container, but still aggregated for `podman stats` and `podman pod stats` display (so the CLI does not change, only the Libpod and Compat APIs). Signed-off-by: Matt Heon --- cmd/podman/containers/stats.go | 10 +++- docs/source/markdown/podman-stats.1.md.in | 3 +- libpod/define/containerstate.go | 26 +++++++--- libpod/networking_freebsd.go | 51 +++++-------------- libpod/networking_linux.go | 33 +++++++----- libpod/stats_common.go | 6 +++ libpod/stats_freebsd.go | 14 ----- libpod/stats_linux.go | 12 ----- .../handlers/compat/containers_stats_linux.go | 29 +++++------ pkg/domain/infra/abi/pods_stats.go | 9 +++- test/apiv2/19-stats.at | 18 +++++++ 11 files changed, 107 insertions(+), 104 deletions(-) diff --git a/cmd/podman/containers/stats.go b/cmd/podman/containers/stats.go index ed462a7af6..a0467957b9 100644 --- a/cmd/podman/containers/stats.go +++ b/cmd/podman/containers/stats.go @@ -215,7 +215,15 @@ func (s *containerStats) MemPerc() string { } func (s *containerStats) NetIO() string { - return combineHumanValues(s.NetInput, s.NetOutput) + var netInput uint64 + var netOutput uint64 + + for _, net := range s.Network { + netInput += net.RxBytes + netOutput += net.TxBytes + } + + return combineHumanValues(netInput, netOutput) } func (s *containerStats) BlockIO() string { diff --git a/docs/source/markdown/podman-stats.1.md.in b/docs/source/markdown/podman-stats.1.md.in index 5e96224da1..f44a7f8ef5 100644 --- a/docs/source/markdown/podman-stats.1.md.in +++ b/docs/source/markdown/podman-stats.1.md.in @@ -50,9 +50,8 @@ Valid placeholders for the Go template are listed below: | .MemUsage | Memory usage | | .MemUsageBytes | Memory usage (IEC) | | .Name | Container Name | -| .NetInput | Network Input | | .NetIO | Network IO | -| .NetOutput | Network Output | +| .Network | Network I/O, separated by network interface | | .PerCPU | CPU time consumed by all tasks [1] | | .PIDs | Number of PIDs | | .PIDS | Number of PIDs (yes, we know this is a dup) | diff --git a/libpod/define/containerstate.go b/libpod/define/containerstate.go index 4520dc41db..e967988861 100644 --- a/libpod/define/containerstate.go +++ b/libpod/define/containerstate.go @@ -141,11 +141,23 @@ type ContainerStats struct { MemUsage uint64 MemLimit uint64 MemPerc float64 - NetInput uint64 - NetOutput uint64 - BlockInput uint64 - BlockOutput uint64 - PIDs uint64 - UpTime time.Duration - Duration uint64 + // Map of interface name to network statistics for that interface. + Network map[string]ContainerNetworkStats + BlockInput uint64 + BlockOutput uint64 + PIDs uint64 + UpTime time.Duration + Duration uint64 +} + +// Statistics for an individual container network interface +type ContainerNetworkStats struct { + RxBytes uint64 + RxDropped uint64 + RxErrors uint64 + RxPackets uint64 + TxBytes uint64 + TxDropped uint64 + TxErrors uint64 + TxPackets uint64 } diff --git a/libpod/networking_freebsd.go b/libpod/networking_freebsd.go index ff894803bb..c5f4667ba1 100644 --- a/libpod/networking_freebsd.go +++ b/libpod/networking_freebsd.go @@ -13,6 +13,7 @@ import ( "github.com/containers/buildah/pkg/jail" "github.com/containers/common/libnetwork/types" + "github.com/containers/podman/v4/libpod/define" "github.com/containers/storage/pkg/lockfile" "github.com/sirupsen/logrus" ) @@ -45,33 +46,6 @@ type NetstatAddress struct { Collisions uint64 `json:"collisions"` } -// copied from github.com/vishvanada/netlink which does not build on freebsd -type LinkStatistics64 struct { - RxPackets uint64 - TxPackets uint64 - RxBytes uint64 - TxBytes uint64 - RxErrors uint64 - TxErrors uint64 - RxDropped uint64 - TxDropped uint64 - Multicast uint64 - Collisions uint64 - RxLengthErrors uint64 - RxOverErrors uint64 - RxCrcErrors uint64 - RxFrameErrors uint64 - RxFifoErrors uint64 - RxMissedErrors uint64 - TxAbortedErrors uint64 - TxCarrierErrors uint64 - TxFifoErrors uint64 - TxHeartbeatErrors uint64 - TxWindowErrors uint64 - RxCompressed uint64 - TxCompressed uint64 -} - type RootlessNetNS struct { dir string Lock *lockfile.LockFile @@ -223,7 +197,7 @@ func (r *Runtime) teardownNetNS(ctr *Container) error { // TODO (5.0): return the statistics per network interface // This would allow better compat with docker. -func getContainerNetIO(ctr *Container) (*LinkStatistics64, error) { +func getContainerNetIO(ctr *Container) (map[string]define.ContainerNetworkStats, error) { if ctr.state.NetNS == "" { // If NetNS is nil, it was set as none, and no netNS // was set up this is a valid state and thus return no @@ -249,8 +223,9 @@ func getContainerNetIO(ctr *Container) (*LinkStatistics64, error) { return nil, err } + res := make(map[string]define.ContainerNetworkStats) + // Sum all the interface stats - in practice only Tx/TxBytes are needed - res := &LinkStatistics64{} for _, ifaddr := range stats.Statistics.Interface { // Each interface has two records, one for link-layer which has // an MTU field and one for IP which doesn't. We only want the @@ -260,14 +235,16 @@ func getContainerNetIO(ctr *Container) (*LinkStatistics64, error) { // if we move to per-interface stats in future, this can be // reported separately. if ifaddr.Mtu > 0 { - res.RxPackets += ifaddr.ReceivedPackets - res.TxPackets += ifaddr.SentPackets - res.RxBytes += ifaddr.ReceivedBytes - res.TxBytes += ifaddr.SentBytes - res.RxErrors += ifaddr.ReceivedErrors - res.TxErrors += ifaddr.SentErrors - res.RxDropped += ifaddr.DroppedPackets - res.Collisions += ifaddr.Collisions + linkStats := define.ContainerNetworkStats{ + RxPackets: ifaddr.ReceivedPackets, + TxPackets: ifaddr.SentPackets, + RxBytes: ifaddr.ReceivedBytes, + TxBytes: ifaddr.SentBytes, + RxErrors: ifaddr.ReceivedErrors, + TxErrors: ifaddr.SentErrors, + RxDropped: ifaddr.DroppedPackets, + } + res[ifaddr.Name] = linkStats } } diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index 977fdf108a..c19631a445 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -13,6 +13,7 @@ import ( "github.com/containers/common/libnetwork/types" netUtil "github.com/containers/common/libnetwork/util" "github.com/containers/common/pkg/netns" + "github.com/containers/podman/v4/libpod/define" "github.com/containers/podman/v4/pkg/rootless" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -186,10 +187,9 @@ func getContainerNetNS(ctr *Container) (string, *Container, error) { return "", nil, nil } -// TODO (5.0): return the statistics per network interface -// This would allow better compat with docker. -func getContainerNetIO(ctr *Container) (*netlink.LinkStatistics, error) { - var netStats *netlink.LinkStatistics +// Returns a map of interface name to statistics for that interface. +func getContainerNetIO(ctr *Container) (map[string]define.ContainerNetworkStats, error) { + perNetworkStats := make(map[string]define.ContainerNetworkStats) netNSPath, otherCtr, netPathErr := getContainerNetNS(ctr) if netPathErr != nil { @@ -222,21 +222,26 @@ func getContainerNetIO(ctr *Container) (*netlink.LinkStatistics, error) { if err != nil { return err } - if netStats == nil { - netStats = link.Attrs().Statistics - continue - } - // Currently only Tx/RxBytes are used. - // In the future we should return all stats per interface so that - // api users have a better options. stats := link.Attrs().Statistics - netStats.TxBytes += stats.TxBytes - netStats.RxBytes += stats.RxBytes + if stats != nil { + newStats := define.ContainerNetworkStats{ + RxBytes: stats.RxBytes, + RxDropped: stats.RxDropped, + RxErrors: stats.RxErrors, + RxPackets: stats.RxPackets, + TxBytes: stats.TxBytes, + TxDropped: stats.TxDropped, + TxErrors: stats.TxErrors, + TxPackets: stats.TxPackets, + } + + perNetworkStats[dev] = newStats + } } } return nil }) - return netStats, err + return perNetworkStats, err } // joinedNetworkNSPath returns netns path and bool if netns was set diff --git a/libpod/stats_common.go b/libpod/stats_common.go index 338f67a4ea..015192dc72 100644 --- a/libpod/stats_common.go +++ b/libpod/stats_common.go @@ -41,6 +41,12 @@ func (c *Container) GetContainerStats(previousStats *define.ContainerStats) (*de } } + netStats, err := getContainerNetIO(c) + if err != nil { + return nil, err + } + stats.Network = netStats + if err := c.getPlatformContainerStats(stats, previousStats); err != nil { return nil, err } diff --git a/libpod/stats_freebsd.go b/libpod/stats_freebsd.go index 3945e977ee..30bb5acad0 100644 --- a/libpod/stats_freebsd.go +++ b/libpod/stats_freebsd.go @@ -80,20 +80,6 @@ func (c *Container) getPlatformContainerStats(stats *define.ContainerStats, prev stats.MemLimit = c.getMemLimit() stats.SystemNano = now - netStats, err := getContainerNetIO(c) - if err != nil { - return err - } - - // Handle case where the container is not in a network namespace - if netStats != nil { - stats.NetInput = netStats.RxBytes - stats.NetOutput = netStats.TxBytes - } else { - stats.NetInput = 0 - stats.NetOutput = 0 - } - return nil } diff --git a/libpod/stats_linux.go b/libpod/stats_linux.go index ad8dfc45f4..839c04fc09 100644 --- a/libpod/stats_linux.go +++ b/libpod/stats_linux.go @@ -39,10 +39,6 @@ func (c *Container) getPlatformContainerStats(stats *define.ContainerStats, prev return fmt.Errorf("unable to obtain cgroup stats: %w", err) } conState := c.state.State - netStats, err := getContainerNetIO(c) - if err != nil { - return err - } // If the current total usage in the cgroup is less than what was previously // recorded then it means the container was restarted and runs in a new cgroup @@ -69,14 +65,6 @@ func (c *Container) getPlatformContainerStats(stats *define.ContainerStats, prev stats.CPUSystemNano = cgroupStats.CpuStats.CpuUsage.UsageInKernelmode stats.SystemNano = now stats.PerCPU = cgroupStats.CpuStats.CpuUsage.PercpuUsage - // Handle case where the container is not in a network namespace - if netStats != nil { - stats.NetInput = netStats.RxBytes - stats.NetOutput = netStats.TxBytes - } else { - stats.NetInput = 0 - stats.NetOutput = 0 - } return nil } diff --git a/pkg/api/handlers/compat/containers_stats_linux.go b/pkg/api/handlers/compat/containers_stats_linux.go index f94fe65f1f..79f8186614 100644 --- a/pkg/api/handlers/compat/containers_stats_linux.go +++ b/pkg/api/handlers/compat/containers_stats_linux.go @@ -119,23 +119,20 @@ streamLabel: // A label to flatten the scope return } - // FIXME: network inspection does not yet work entirely net := make(map[string]docker.NetworkStats) - networkName := inspect.NetworkSettings.EndpointID - if networkName == "" { - networkName = "network" - } - net[networkName] = docker.NetworkStats{ - RxBytes: stats.NetInput, - RxPackets: 0, - RxErrors: 0, - RxDropped: 0, - TxBytes: stats.NetOutput, - TxPackets: 0, - TxErrors: 0, - TxDropped: 0, - EndpointID: inspect.NetworkSettings.EndpointID, - InstanceID: "", + for netName, netStats := range stats.Network { + net[netName] = docker.NetworkStats{ + RxBytes: netStats.RxBytes, + RxPackets: netStats.RxPackets, + RxErrors: netStats.RxErrors, + RxDropped: netStats.RxDropped, + TxBytes: netStats.TxBytes, + TxPackets: netStats.TxPackets, + TxErrors: netStats.TxErrors, + TxDropped: netStats.TxDropped, + EndpointID: inspect.NetworkSettings.EndpointID, + InstanceID: "", + } } resources := ctnr.LinuxResources() diff --git a/pkg/domain/infra/abi/pods_stats.go b/pkg/domain/infra/abi/pods_stats.go index 447ceab9de..1a18c18e6f 100644 --- a/pkg/domain/infra/abi/pods_stats.go +++ b/pkg/domain/infra/abi/pods_stats.go @@ -44,12 +44,19 @@ func (ic *ContainerEngine) podsToStatsReport(pods []*libpod.Pod) ([]*entities.Po } podID := pods[i].ID()[:12] for j := range podStats { + var podNetInput uint64 + var podNetOutput uint64 + for _, stats := range podStats[j].Network { + podNetInput += stats.RxBytes + podNetOutput += stats.TxBytes + } + r := entities.PodStatsReport{ CPU: floatToPercentString(podStats[j].CPU), MemUsage: combineHumanValues(podStats[j].MemUsage, podStats[j].MemLimit), MemUsageBytes: combineBytesValues(podStats[j].MemUsage, podStats[j].MemLimit), Mem: floatToPercentString(podStats[j].MemPerc), - NetIO: combineHumanValues(podStats[j].NetInput, podStats[j].NetOutput), + NetIO: combineHumanValues(podNetInput, podNetOutput), BlockIO: combineHumanValues(podStats[j].BlockInput, podStats[j].BlockOutput), PIDS: pidsToString(podStats[j].PIDs), CID: podStats[j].ContainerID[:12], diff --git a/test/apiv2/19-stats.at b/test/apiv2/19-stats.at index 8d8a9ef590..2ca4b02daf 100644 --- a/test/apiv2/19-stats.at +++ b/test/apiv2/19-stats.at @@ -9,3 +9,21 @@ if root; then # regression for https://github.com/containers/podman/issues/15754 t GET libpod/containers/container1/stats?stream=false 200 .cpu_stats.online_cpus=1 fi + +podman run -dt --name testctr1 $IMAGE top &>/dev/null + +t GET libpod/containers/testctr1/stats?stream=false 200 '.networks | length'=1 + +podman rm -f testctr1 + +podman network create testnet1 +podman network create testnet2 + +podman run -dt --name testctr2 --net testnet1,testnet2 $IMAGE top &>/dev/null + +t GET libpod/containers/testctr2/stats?stream=false 200 '.networks | length'=2 + +podman rm -f testctr2 + +podman network rm testnet1 +podman network rm testnet2