containers
/
podman
mirror of https://github.com/containers/podman.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14400 from cdoern/scp 

podman image scp remote support & podman image scp tagging
This commit is contained in:
openshift-ci[bot] 2022-06-28 17:46:12 +00:00 committed by GitHub
parent 50fa651a4e 6d3520e8b7
commit d8f197cc14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 898 additions and 675 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

304
cmd/podman/images/scp.go
View File

@ -1,28 +1,12 @@
package images
import (
"context"
"fmt"
"io/ioutil"
urlP "net/url"
"os"
"os/exec"
"os/user"
"strconv"
"strings"
"github.com/containers/common/pkg/config"
"github.com/containers/podman/v4/cmd/podman/common"
"github.com/containers/podman/v4/cmd/podman/registry"
"github.com/containers/podman/v4/cmd/podman/system/connection"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/containers/podman/v4/utils"
scpD "github.com/dtylman/scp"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/crypto/ssh"
)
var (
@ -32,7 +16,6 @@ var (
Annotations: map[string]string{
registry.UnshareNSRequired: "",
registry.ParentNSRequired: "",
registry.EngineMode: registry.ABIMode,
},
Long: saveScpDescription,
Short: "securely copy images",
@ -46,9 +29,6 @@ var (
var (
parentFlags []string
quiet bool
source entities.ImageScpOptions
dest entities.ImageScpOptions
sshInfo entities.ImageScpConnections
)
func init() {
@ -66,7 +46,6 @@ func scpFlags(cmd *cobra.Command) {
func scp(cmd *cobra.Command, args []string) (finalErr error) {
var (
// TODO add tag support for images
err error
)
for i, val := range os.Args {
@ -81,288 +60,17 @@ func scp(cmd *cobra.Command, args []string) (finalErr error) {
}
parentFlags = append(parentFlags, val)
}
podman, err := os.Executable()
if err != nil {
return err
}
f, err := ioutil.TempFile("", "podman") // open temp file for load/save output
if err != nil {
return err
}
confR, err := config.NewConfig("") // create a hand made config for the remote engine since we might use remote and native at once
if err != nil {
return errors.Wrapf(err, "could not make config")
src := args[0]
dst := ""
if len(args) > 1 {
dst = args[1]
}
abiEng, err := registry.NewImageEngine(cmd, args) // abi native engine
err = registry.ImageEngine().Scp(registry.Context(), src, dst, parentFlags, quiet)
if err != nil {
return err
}
cfg, err := config.ReadCustomConfig() // get ready to set ssh destination if necessary
if err != nil {
return err
}
locations := []*entities.ImageScpOptions{}
cliConnections := []string{}
var flipConnections bool
for _, arg := range args {
loc, connect, err := parseImageSCPArg(arg)
if err != nil {
return err
}
locations = append(locations, loc)
cliConnections = append(cliConnections, connect...)
}
source = *locations[0]
switch {
case len(locations) > 1:
if flipConnections, err = validateSCPArgs(locations); err != nil {
return err
}
if flipConnections { // the order of cliConnections matters, we need to flip both arrays since the args are parsed separately sometimes.
cliConnections[0], cliConnections[1] = cliConnections[1], cliConnections[0]
locations[0], locations[1] = locations[1], locations[0]
}
dest = *locations[1]
case len(locations) == 1:
switch {
case len(locations[0].Image) == 0:
return errors.Wrapf(define.ErrInvalidArg, "no source image specified")
case len(locations[0].Image) > 0 && !locations[0].Remote && len(locations[0].User) == 0: // if we have podman image scp $IMAGE
return errors.Wrapf(define.ErrInvalidArg, "must specify a destination")
}
}
source.Quiet = quiet
source.File = f.Name() // after parsing the arguments, set the file for the save/load
dest.File = source.File
if err = os.Remove(source.File); err != nil { // remove the file and simply use its name so podman creates the file upon save. avoids umask errors
return err
}
allLocal := true // if we are all localhost, do not validate connections but if we are using one localhost and one non we need to use sshd
for _, val := range cliConnections {
if !strings.Contains(val, "@localhost::") {
allLocal = false
break
}
}
if allLocal {
cliConnections = []string{}
}
var serv map[string]config.Destination
serv, err = GetServiceInformation(cliConnections, cfg)
if err != nil {
return err
}
// TODO: Add podman remote support
confR.Engine = config.EngineConfig{Remote: true, CgroupManager: "cgroupfs", ServiceDestinations: serv} // pass the service dest (either remote or something else) to engine
saveCmd, loadCmd := createCommands(podman)
switch {
case source.Remote: // if we want to load FROM the remote, dest can either be local or remote in this case
err = saveToRemote(source.Image, source.File, "", sshInfo.URI[0], sshInfo.Identities[0])
if err != nil {
return err
}
if dest.Remote { // we want to load remote -> remote, both source and dest are remote
rep, err := loadToRemote(dest.File, "", sshInfo.URI[1], sshInfo.Identities[1])
if err != nil {
return err
}
fmt.Println(rep)
break
}
err = execPodman(podman, loadCmd)
if err != nil {
return err
}
case dest.Remote: // remote host load, implies source is local
err = execPodman(podman, saveCmd)
if err != nil {
return err
}
rep, err := loadToRemote(source.File, "", sshInfo.URI[0], sshInfo.Identities[0])
if err != nil {
return err
}
fmt.Println(rep)
if err = os.Remove(source.File); err != nil {
return err
}
// TODO: Add podman remote support
default: // else native load, both source and dest are local and transferring between users
if source.User == "" { // source user has to be set, destination does not
source.User = os.Getenv("USER")
if source.User == "" {
u, err := user.Current()
if err != nil {
return errors.Wrapf(err, "could not obtain user, make sure the environmental variable $USER is set")
}
source.User = u.Username
}
}
err := abiEng.Transfer(context.Background(), source, dest, parentFlags)
if err != nil {
return err
}
}
return nil
}
// loadToRemote takes image and remote connection information. it connects to the specified client
// and copies the saved image dir over to the remote host and then loads it onto the machine
// returns a string containing output or an error
func loadToRemote(localFile string, tag string, url *urlP.URL, iden string) (string, error) {
dial, remoteFile, err := createConnection(url, iden)
if err != nil {
return "", err
}
defer dial.Close()
n, err := scpD.CopyTo(dial, localFile, remoteFile)
if err != nil {
errOut := strconv.Itoa(int(n)) + " Bytes copied before error"
return " ", errors.Wrapf(err, errOut)
}
var run string
if tag != "" {
return "", errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
}
podman := os.Args[0]
run = podman + " image load --input=" + remoteFile + ";rm " + remoteFile // run ssh image load of the file copied via scp
out, err := connection.ExecRemoteCommand(dial, run)
if err != nil {
return "", err
}
return strings.TrimSuffix(string(out), "\n"), nil
}
// saveToRemote takes image information and remote connection information. it connects to the specified client
// and saves the specified image on the remote machine and then copies it to the specified local location
// returns an error if one occurs.
func saveToRemote(image, localFile string, tag string, uri *urlP.URL, iden string) error {
dial, remoteFile, err := createConnection(uri, iden)
if err != nil {
return err
}
defer dial.Close()
if tag != "" {
return errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
}
podman := os.Args[0]
run := podman + " image save " + image + " --format=oci-archive --output=" + remoteFile // run ssh image load of the file copied via scp. Files are reverse in this case...
_, err = connection.ExecRemoteCommand(dial, run)
if err != nil {
return err
}
n, err := scpD.CopyFrom(dial, remoteFile, localFile)
if _, conErr := connection.ExecRemoteCommand(dial, "rm "+remoteFile); conErr != nil {
logrus.Errorf("Removing file on endpoint: %v", conErr)
}
if err != nil {
errOut := strconv.Itoa(int(n)) + " Bytes copied before error"
return errors.Wrapf(err, errOut)
}
return nil
}
// makeRemoteFile creates the necessary remote file on the host to
// save or load the image to. returns a string with the file name or an error
func makeRemoteFile(dial *ssh.Client) (string, error) {
run := "mktemp"
remoteFile, err := connection.ExecRemoteCommand(dial, run)
if err != nil {
return "", err
}
return strings.TrimSuffix(string(remoteFile), "\n"), nil
}
// createConnections takes a boolean determining which ssh client to dial
// and returns the dials client, its newly opened remote file, and an error if applicable.
func createConnection(url *urlP.URL, iden string) (*ssh.Client, string, error) {
cfg, err := connection.ValidateAndConfigure(url, iden)
if err != nil {
return nil, "", err
}
dialAdd, err := ssh.Dial("tcp", url.Host, cfg) // dial the client
if err != nil {
return nil, "", errors.Wrapf(err, "failed to connect")
}
file, err := makeRemoteFile(dialAdd)
if err != nil {
return nil, "", err
}
return dialAdd, file, nil
}
// GetSerivceInformation takes the parsed list of hosts to connect to and validates the information
func GetServiceInformation(cliConnections []string, cfg *config.Config) (map[string]config.Destination, error) {
var serv map[string]config.Destination
var url string
var iden string
for i, val := range cliConnections {
splitEnv := strings.SplitN(val, "::", 2)
sshInfo.Connections = append(sshInfo.Connections, splitEnv[0])
if len(splitEnv[1]) != 0 {
err := validateImageName(splitEnv[1])
if err != nil {
return nil, err
}
source.Image = splitEnv[1]
//TODO: actually use the new name given by the user
}
conn, found := cfg.Engine.ServiceDestinations[sshInfo.Connections[i]]
if found {
url = conn.URI
iden = conn.Identity
} else { // no match, warn user and do a manual connection.
url = "ssh://" + sshInfo.Connections[i]
iden = ""
logrus.Warnf("Unknown connection name given. Please use system connection add to specify the default remote socket location")
}
urlT, err := urlP.Parse(url) // create an actual url to pass to exec command
if err != nil {
return nil, err
}
if urlT.User.Username() == "" {
if urlT.User, err = connection.GetUserInfo(urlT); err != nil {
return nil, err
}
}
sshInfo.URI = append(sshInfo.URI, urlT)
sshInfo.Identities = append(sshInfo.Identities, iden)
}
return serv, nil
}
// execPodman executes the podman save/load command given the podman binary
func execPodman(podman string, command []string) error {
cmd := exec.Command(podman)
utils.CreateSCPCommand(cmd, command[1:])
logrus.Debugf("Executing podman command: %q", cmd)
return cmd.Run()
}
// createCommands forms the podman save and load commands used by SCP
func createCommands(podman string) ([]string, []string) {
var parentString string
quiet := ""
if source.Quiet {
quiet = "-q "
}
if len(parentFlags) > 0 {
parentString = strings.Join(parentFlags, " ") + " " // if there are parent args, an extra space needs to be added
} else {
parentString = strings.Join(parentFlags, " ")
}
loadCmd := strings.Split(fmt.Sprintf("%s %sload %s--input %s", podman, parentString, quiet, dest.File), " ")
saveCmd := strings.Split(fmt.Sprintf("%s %vsave %s--output %s %s", podman, parentString, quiet, source.File, source.Image), " ")
return saveCmd, loadCmd
}

46
cmd/podman/images/scp_test.go
View File

@ -1,46 +0,0 @@
package images
import (
"testing"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/stretchr/testify/assert"
)
func TestParseSCPArgs(t *testing.T) {
args := []string{"alpine", "root@localhost::"}
var source *entities.ImageScpOptions
var dest *entities.ImageScpOptions
var err error
source, _, err = parseImageSCPArg(args[0])
assert.Nil(t, err)
assert.Equal(t, source.Image, "alpine")
dest, _, err = parseImageSCPArg(args[1])
assert.Nil(t, err)
assert.Equal(t, dest.Image, "")
assert.Equal(t, dest.User, "root")
args = []string{"root@localhost::alpine"}
source, _, err = parseImageSCPArg(args[0])
assert.Nil(t, err)
assert.Equal(t, source.User, "root")
assert.Equal(t, source.Image, "alpine")
args = []string{"charliedoern@192.168.68.126::alpine", "foobar@192.168.68.126::"}
source, _, err = parseImageSCPArg(args[0])
assert.Nil(t, err)
assert.True(t, source.Remote)
assert.Equal(t, source.Image, "alpine")
dest, _, err = parseImageSCPArg(args[1])
assert.Nil(t, err)
assert.True(t, dest.Remote)
assert.Equal(t, dest.Image, "")
args = []string{"charliedoern@192.168.68.126::alpine"}
source, _, err = parseImageSCPArg(args[0])
assert.Nil(t, err)
assert.True(t, source.Remote)
assert.Equal(t, source.Image, "alpine")
}

88
cmd/podman/images/scp_utils.go
View File

@ -1,88 +0,0 @@
package images
import (
"strings"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/pkg/errors"
)
// parseImageSCPArg returns the valid connection, and source/destination data based off of the information provided by the user
// arg is a string containing one of the cli arguments returned is a filled out source/destination options structs as well as a connections array and an error if applicable
func parseImageSCPArg(arg string) (*entities.ImageScpOptions, []string, error) {
location := entities.ImageScpOptions{}
var err error
cliConnections := []string{}
switch {
case strings.Contains(arg, "@localhost::"): // image transfer between users
location.User = strings.Split(arg, "@")[0]
location, err = validateImagePortion(location, arg)
if err != nil {
return nil, nil, err
}
cliConnections = append(cliConnections, arg)
case strings.Contains(arg, "::"):
location, err = validateImagePortion(location, arg)
if err != nil {
return nil, nil, err
}
location.Remote = true
cliConnections = append(cliConnections, arg)
default:
location.Image = arg
}
return &location, cliConnections, nil
}
// validateImagePortion is a helper function to validate the image name in an SCP argument
func validateImagePortion(location entities.ImageScpOptions, arg string) (entities.ImageScpOptions, error) {
if remoteArgLength(arg, 1) > 0 {
err := validateImageName(strings.Split(arg, "::")[1])
if err != nil {
return location, err
}
location.Image = strings.Split(arg, "::")[1] // this will get checked/set again once we validate connections
}
return location, nil
}
// validateSCPArgs takes the array of source and destination options and checks for common errors
func validateSCPArgs(locations []*entities.ImageScpOptions) (bool, error) {
if len(locations) > 2 {
return false, errors.Wrapf(define.ErrInvalidArg, "cannot specify more than two arguments")
}
switch {
case len(locations[0].Image) > 0 && len(locations[1].Image) > 0:
return false, errors.Wrapf(define.ErrInvalidArg, "cannot specify an image rename")
case len(locations[0].Image) == 0 && len(locations[1].Image) == 0:
return false, errors.Wrapf(define.ErrInvalidArg, "a source image must be specified")
case len(locations[0].Image) == 0 && len(locations[1].Image) != 0:
if locations[0].Remote && locations[1].Remote {
return true, nil // we need to flip the cliConnections array so the save/load connections are in the right place
}
}
return false, nil
}
// validateImageName makes sure that the image given is valid and no injections are occurring
// we simply use this for error checking, bot setting the image
func validateImageName(input string) error {
// ParseNormalizedNamed transforms a shortname image into its
// full name reference so busybox => docker.io/library/busybox
// we want to keep our shortnames, so only return an error if
// we cannot parse what the user has given us
_, err := reference.ParseNormalizedNamed(input)
return err
}
// remoteArgLength is a helper function to simplify the extracting of host argument data
// returns an int which contains the length of a specified index in a host::image string
func remoteArgLength(input string, side int) int {
if strings.Contains(input, "::") {
return len((strings.Split(input, "::"))[side])
}
return -1
}

111
cmd/podman/system/connection/add.go
View File

@ -6,21 +6,18 @@ import (
"net"
"net/url"
"os"
"os/user"
"regexp"
"time"
"github.com/containers/common/pkg/completion"
"github.com/containers/common/pkg/config"
"github.com/containers/podman/v4/cmd/podman/registry"
"github.com/containers/podman/v4/cmd/podman/system"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/terminal"
"github.com/containers/podman/v4/pkg/domain/utils"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
)
var (
@ -95,7 +92,7 @@ func add(cmd *cobra.Command, args []string) error {
switch uri.Scheme {
case "ssh":
if uri.User.Username() == "" {
if uri.User, err = GetUserInfo(uri); err != nil {
if uri.User, err = utils.GetUserInfo(uri); err != nil {
return err
}
}
@ -180,32 +177,8 @@ func add(cmd *cobra.Command, args []string) error {
return cfg.Write()
}
func GetUserInfo(uri *url.URL) (*url.Userinfo, error) {
var (
usr *user.User
err error
)
if u, found := os.LookupEnv("_CONTAINERS_ROOTLESS_UID"); found {
usr, err = user.LookupId(u)
if err != nil {
return nil, errors.Wrapf(err, "failed to look up rootless user")
}
} else {
usr, err = user.Current()
if err != nil {
return nil, errors.Wrapf(err, "failed to obtain current user")
}
}
pw, set := uri.User.Password()
if set {
return url.UserPassword(usr.Username, pw), nil
}
return url.User(usr.Username), nil
}
func getUDS(uri *url.URL, iden string) (string, error) {
cfg, err := ValidateAndConfigure(uri, iden)
cfg, err := utils.ValidateAndConfigure(uri, iden)
if err != nil {
return "", errors.Wrapf(err, "failed to validate")
}
@ -226,7 +199,7 @@ func getUDS(uri *url.URL, iden string) (string, error) {
if v, found := os.LookupEnv("PODMAN_BINARY"); found {
podman = v
}
infoJSON, err := ExecRemoteCommand(dial, podman+" info --format=json")
infoJSON, err := utils.ExecRemoteCommand(dial, podman+" info --format=json")
if err != nil {
return "", err
}
@ -241,79 +214,3 @@ func getUDS(uri *url.URL, iden string) (string, error) {
}
return info.Host.RemoteSocket.Path, nil
}
// ValidateAndConfigure will take a ssh url and an identity key (rsa and the like) and ensure the information given is valid
// iden iden can be blank to mean no identity key
// once the function validates the information it creates and returns an ssh.ClientConfig.
func ValidateAndConfigure(uri *url.URL, iden string) (*ssh.ClientConfig, error) {
var signers []ssh.Signer
passwd, passwdSet := uri.User.Password()
if iden != "" { // iden might be blank if coming from image scp or if no validation is needed
value := iden
s, err := terminal.PublicKey(value, []byte(passwd))
if err != nil {
return nil, errors.Wrapf(err, "failed to read identity %q", value)
}
signers = append(signers, s)
logrus.Debugf("SSH Ident Key %q %s %s", value, ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found { // validate ssh information, specifically the unix file socket used by the ssh agent.
logrus.Debugf("Found SSH_AUTH_SOCK %q, ssh-agent signer enabled", sock)
c, err := net.Dial("unix", sock)
if err != nil {
return nil, err
}
agentSigners, err := agent.NewClient(c).Signers()
if err != nil {
return nil, err
}
signers = append(signers, agentSigners...)
if logrus.IsLevelEnabled(logrus.DebugLevel) {
for _, s := range agentSigners {
logrus.Debugf("SSH Agent Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
}
}
var authMethods []ssh.AuthMethod // now we validate and check for the authorization methods, most notaibly public key authorization
if len(signers) > 0 {
var dedup = make(map[string]ssh.Signer)
for _, s := range signers {
fp := ssh.FingerprintSHA256(s.PublicKey())
if _, found := dedup[fp]; found {
logrus.Debugf("Dedup SSH Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
dedup[fp] = s
}
var uniq []ssh.Signer
for _, s := range dedup {
uniq = append(uniq, s)
}
authMethods = append(authMethods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) {
return uniq, nil
}))
}
if passwdSet { // if password authentication is given and valid, add to the list
authMethods = append(authMethods, ssh.Password(passwd))
}
if len(authMethods) == 0 {
authMethods = append(authMethods, ssh.PasswordCallback(func() (string, error) {
pass, err := terminal.ReadPassword(fmt.Sprintf("%s's login password:", uri.User.Username()))
return string(pass), err
}))
}
tick, err := time.ParseDuration("40s")
if err != nil {
return nil, err
}
cfg := &ssh.ClientConfig{
User: uri.User.Username(),
Auth: authMethods,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: tick,
}
return cfg, nil
}

27
cmd/podman/system/connection/shared.go
View File

@ -1,27 +0,0 @@
package connection
import (
"bytes"
"github.com/pkg/errors"
"golang.org/x/crypto/ssh"
)
// ExecRemoteCommand takes a ssh client connection and a command to run and executes the
// command on the specified client. The function returns the Stdout from the client or the Stderr
func ExecRemoteCommand(dial *ssh.Client, run string) ([]byte, error) {
sess, err := dial.NewSession() // new ssh client session
if err != nil {
return nil, err
}
defer sess.Close()
var buffer bytes.Buffer
var bufferErr bytes.Buffer
sess.Stdout = &buffer // output from client funneled into buffer
sess.Stderr = &bufferErr // err form client funneled into buffer
if err := sess.Run(run); err != nil { // run the command on the ssh client
return nil, errors.Wrapf(err, bufferErr.String())
}
return buffer.Bytes(), nil
}

2
contrib/cirrus/lib.sh
View File

@ -135,6 +135,7 @@ setup_rootless() {
req_env_vars GOPATH GOSRC SECRET_ENV_RE
ROOTLESS_USER="${ROOTLESS_USER:-some${RANDOM}dude}"
ROOTLESS_UID=""
local rootless_uid
local rootless_gid
@ -158,6 +159,7 @@ setup_rootless() {
cd $GOSRC || exit 1
# Guarantee independence from specific values
rootless_uid=$[RANDOM+1000]
ROOTLESS_UID=$rootless_uid
rootless_gid=$[RANDOM+1000]
msg "creating $rootless_uid:$rootless_gid $ROOTLESS_USER user"
groupadd -g $rootless_gid $ROOTLESS_USER

4
contrib/cirrus/setup_environment.sh
View File

@ -186,10 +186,11 @@ esac
# Required to be defined by caller: Are we testing as root or a regular user
case "$PRIV_NAME" in
root)
if [[ "$TEST_FLAVOR" = "sys" ]]; then
if [[ "$TEST_FLAVOR" = "sys" || "$TEST_FLAVOR" = "apiv2" ]]; then
# Used in local image-scp testing
setup_rootless
echo "PODMAN_ROOTLESS_USER=$ROOTLESS_USER" >> /etc/ci_environment
echo "PODMAN_ROOTLESS_UID=$ROOTLESS_UID" >> /etc/ci_environment
fi
;;
rootless)
@ -203,6 +204,7 @@ esac
if [[ -n "$ROOTLESS_USER" ]]; then
echo "ROOTLESS_USER=$ROOTLESS_USER" >> /etc/ci_environment
echo "ROOTLESS_UID=$ROOTLESS_UID" >> /etc/ci_environment
fi
# Required to be defined by caller: Are we testing podman or podman-remote client

31
pkg/api/handlers/libpod/images.go
View File

@ -21,7 +21,9 @@ import (
api "github.com/containers/podman/v4/pkg/api/types"
"github.com/containers/podman/v4/pkg/auth"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/containers/podman/v4/pkg/domain/entities/reports"
"github.com/containers/podman/v4/pkg/domain/infra/abi"
domainUtils "github.com/containers/podman/v4/pkg/domain/utils"
"github.com/containers/podman/v4/pkg/errorhandling"
"github.com/containers/podman/v4/pkg/util"
utils2 "github.com/containers/podman/v4/utils"
@ -670,3 +672,32 @@ func ImagesRemove(w http.ResponseWriter, r *http.Request) {
utils.Error(w, http.StatusInternalServerError, errorhandling.JoinErrors(rmErrors))
}
}
func ImageScp(w http.ResponseWriter, r *http.Request) {
decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder)
query := struct {
Destination string `schema:"destination"`
Quiet bool `schema:"quiet"`
}{
// This is where you can override the golang default value for one of fields
}
if err := decoder.Decode(&query, r.URL.Query()); err != nil {
utils.Error(w, http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String()))
return
}
sourceArg := utils.GetName(r)
rep, source, dest, _, err := domainUtils.ExecuteTransfer(sourceArg, query.Destination, []string{}, query.Quiet)
if err != nil {
utils.Error(w, http.StatusInternalServerError, err)
return
}
if source != nil || dest != nil {
utils.Error(w, http.StatusBadRequest, errors.Wrapf(define.ErrInvalidArg, "cannot use the user transfer function on the remote client"))
return
}
utils.WriteResponse(w, http.StatusOK, &reports.ScpReport{Id: rep.Names[0]})
}

7
pkg/api/handlers/swagger/responses.go
View File

@ -41,6 +41,13 @@ type imagesLoadResponseLibpod struct {
Body entities.ImageLoadReport
}
// Image Scp
// swagger:response
type imagesScpResponseLibpod struct {
// in:body
Body reports.ScpReport
}
// Image Import
// swagger:response
type imagesImportResponseLibpod struct {

34
pkg/api/server/register_images.go
View File

@ -1615,5 +1615,39 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error {
// 500:
// $ref: "#/responses/internalError"
r.Handle(VersionedPath("/libpod/build"), s.APIHandler(compat.BuildImage)).Methods(http.MethodPost)
// swagger:operation POST /libpod/images/scp/{name} libpod ImageScpLibpod
// ---
// tags:
// - images
// summary: Copy an image from one host to another
// description: Copy an image from one host to another
// parameters:
// - in: path
// name: name
// required: true
// description: source connection/image
// type: string
// - in: query
// name: destination
// required: false
// description: dest connection/image
// type: string
// - in: query
// name: quiet
// required: false
// description: quiet output
// type: boolean
// default: false
// produces:
// - application/json
// responses:
// 200:
// $ref: "#/responses/imagesScpResponseLibpod"
// 400:
// $ref: "#/responses/badParamError"
// 500:
// $ref: '#/responses/internalError'
r.Handle(VersionedPath("/libpod/images/scp/{name:.*}"), s.APIHandler(libpod.ImageScp)).Methods(http.MethodPost)
return nil
}

20
pkg/bindings/images/images.go
View File

@ -346,3 +346,23 @@ func Search(ctx context.Context, term string, options *SearchOptions) ([]entitie
return results, nil
}
func Scp(ctx context.Context, source, destination *string, options ScpOptions) (reports.ScpReport, error) {
rep := reports.ScpReport{}
conn, err := bindings.GetClient(ctx)
if err != nil {
return rep, err
}
params, err := options.ToParams()
if err != nil {
return rep, err
}
response, err := conn.DoRequest(ctx, nil, http.MethodPost, fmt.Sprintf("/images/scp/%s", *source), params, nil)
if err != nil {
return rep, err
}
defer response.Body.Close()
return rep, response.Process(&rep)
}

5
pkg/bindings/images/types.go
View File

@ -188,3 +188,8 @@ type BuildOptions struct {
// ExistsOptions are optional options for checking if an image exists
type ExistsOptions struct {
}
type ScpOptions struct {
Quiet *bool
Destination *string
}

12
pkg/bindings/images/types_scp_options.go Normal file
View File

@ -0,0 +1,12 @@
package images
import (
"net/url"
"github.com/containers/podman/v4/pkg/bindings/internal/util"
)
// ToParams formats struct fields to be passed to API service
func (o *ScpOptions) ToParams() (url.Values, error) {
return util.ToParams(o)
}

2
pkg/domain/entities/engine_image.go
View File

@ -22,12 +22,12 @@ type ImageEngine interface {
Push(ctx context.Context, source string, destination string, opts ImagePushOptions) error
Remove(ctx context.Context, images []string, opts ImageRemoveOptions) (*ImageRemoveReport, []error)
Save(ctx context.Context, nameOrID string, tags []string, options ImageSaveOptions) error
Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool) error
Search(ctx context.Context, term string, opts ImageSearchOptions) ([]ImageSearchReport, error)
SetTrust(ctx context.Context, args []string, options SetTrustOptions) error
ShowTrust(ctx context.Context, args []string, options ShowTrustOptions) (*ShowTrustReport, error)
Shutdown(ctx context.Context)
Tag(ctx context.Context, nameOrID string, tags []string, options ImageTagOptions) error
Transfer(ctx context.Context, source ImageScpOptions, dest ImageScpOptions, parentFlags []string) error
Tree(ctx context.Context, nameOrID string, options ImageTreeOptions) (*ImageTreeReport, error)
Unmount(ctx context.Context, images []string, options ImageUnmountOptions) ([]*ImageUnmountReport, error)
Untag(ctx context.Context, nameOrID string, tags []string, options ImageUntagOptions) error

2
pkg/domain/entities/images.go
View File

@ -325,6 +325,8 @@ type ImageScpOptions struct {
Image string `json:"image,omitempty"`
// User is used in conjunction with Transfer to determine if a valid user was given to save from/load into
User string `json:"user,omitempty"`
// Tag is the name to be used for the image on the destination
Tag string `json:"tag,omitempty"`
}
// ImageScpConnections provides the ssh related information used in remote image transfer

5
pkg/domain/entities/reports/scp.go Normal file
View File

@ -0,0 +1,5 @@
package reports
type ScpReport struct {
Id string `json:"Id"` //nolint:revive,stylecheck
}

157
pkg/domain/infra/abi/images.go
View File

@ -3,6 +3,7 @@ package abi
import (
"context"
"fmt"
"io/fs"
"io/ioutil"
"net/url"
"os"
@ -29,7 +30,6 @@ import (
domainUtils "github.com/containers/podman/v4/pkg/domain/utils"
"github.com/containers/podman/v4/pkg/errorhandling"
"github.com/containers/podman/v4/pkg/rootless"
"github.com/containers/podman/v4/utils"
"github.com/containers/storage"
dockerRef "github.com/docker/distribution/reference"
"github.com/opencontainers/go-digest"
@ -350,22 +350,6 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri
}
return pushError
}
// Transfer moves images between root and rootless storage so the user specified in the scp call can access and use the image modified by root
func (ir *ImageEngine) Transfer(ctx context.Context, source entities.ImageScpOptions, dest entities.ImageScpOptions, parentFlags []string) error {
if source.User == "" {
return errors.Wrapf(define.ErrInvalidArg, "you must define a user when transferring from root to rootless storage")
}
podman, err := os.Executable()
if err != nil {
return err
}
if rootless.IsRootless() && (len(dest.User) == 0 || dest.User == "root") { // if we are rootless and do not have a destination user we can just use sudo
return transferRootless(source, dest, podman, parentFlags)
}
return transferRootful(source, dest, podman, parentFlags)
}
func (ir *ImageEngine) Tag(ctx context.Context, nameOrID string, tags []string, options entities.ImageTagOptions) error {
// Allow tagging manifest list instead of resolving instances from manifest
lookupOptions := &libimage.LookupImageOptions{ManifestList: true}
@ -694,55 +678,34 @@ func (ir *ImageEngine) Sign(ctx context.Context, names []string, options entitie
return nil, nil
}
func getSigFilename(sigStoreDirPath string) (string, error) {
sigFileSuffix := 1
sigFiles, err := ioutil.ReadDir(sigStoreDirPath)
if err != nil {
return "", err
}
sigFilenames := make(map[string]bool)
for _, file := range sigFiles {
sigFilenames[file.Name()] = true
}
for {
sigFilename := "signature-" + strconv.Itoa(sigFileSuffix)
if _, exists := sigFilenames[sigFilename]; !exists {
return sigFilename, nil
}
sigFileSuffix++
}
}
func localPathFromURI(url *url.URL) (string, error) {
if url.Scheme != "file" {
return "", errors.Errorf("writing to %s is not supported. Use a supported scheme", url.String())
}
return url.Path, nil
}
// putSignature creates signature and saves it to the signstore file
func putSignature(manifestBlob []byte, mech signature.SigningMechanism, sigStoreDir string, instanceDigest digest.Digest, dockerReference dockerRef.Reference, options entities.SignOptions) error {
newSig, err := signature.SignDockerManifest(manifestBlob, dockerReference.String(), mech, options.SignBy)
func (ir *ImageEngine) Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool) error {
rep, source, dest, flags, err := domainUtils.ExecuteTransfer(src, dst, parentFlags, quiet)
if err != nil {
return err
}
signatureDir := fmt.Sprintf("%s@%s=%s", sigStoreDir, instanceDigest.Algorithm(), instanceDigest.Hex())
if err := os.MkdirAll(signatureDir, 0751); err != nil {
// The directory is allowed to exist
if !os.IsExist(err) {
return err
}
}
sigFilename, err := getSigFilename(signatureDir)
if (rep == nil && err == nil) && (source != nil && dest != nil) { // we need to execute the transfer
err := Transfer(ctx, *source, *dest, flags)
if err != nil {
return err
}
if err = ioutil.WriteFile(filepath.Join(signatureDir, sigFilename), newSig, 0644); err != nil {
return err
}
return nil
}
func Transfer(ctx context.Context, source entities.ImageScpOptions, dest entities.ImageScpOptions, parentFlags []string) error {
if source.User == "" {
return errors.Wrapf(define.ErrInvalidArg, "you must define a user when transferring from root to rootless storage")
}
podman, err := os.Executable()
if err != nil {
return err
}
if rootless.IsRootless() && (len(dest.User) == 0 || dest.User == "root") { // if we are rootless and do not have a destination user we can just use sudo
return transferRootless(source, dest, podman, parentFlags)
}
return transferRootful(source, dest, podman, parentFlags)
}
// TransferRootless creates new podman processes using exec.Command and sudo, transferring images between the given source and destination users
func transferRootless(source entities.ImageScpOptions, dest entities.ImageScpOptions, podman string, parentFlags []string) error {
var cmdSave *exec.Cmd
@ -763,7 +726,7 @@ func transferRootless(source entities.ImageScpOptions, dest entities.ImageScpOpt
} else {
cmdSave = exec.Command(podman)
}
cmdSave = utils.CreateSCPCommand(cmdSave, saveCommand)
cmdSave = domainUtils.CreateSCPCommand(cmdSave, saveCommand)
logrus.Debugf("Executing save command: %q", cmdSave)
err := cmdSave.Run()
if err != nil {
@ -776,8 +739,11 @@ func transferRootless(source entities.ImageScpOptions, dest entities.ImageScpOpt
} else {
cmdLoad = exec.Command(podman)
}
cmdLoad = utils.CreateSCPCommand(cmdLoad, loadCommand)
cmdLoad = domainUtils.CreateSCPCommand(cmdLoad, loadCommand)
logrus.Debugf("Executing load command: %q", cmdLoad)
if len(dest.Tag) > 0 {
return domainUtils.ScpTag(cmdLoad, podman, dest)
}
return cmdLoad.Run()
}
@ -833,11 +799,20 @@ func transferRootful(source entities.ImageScpOptions, dest entities.ImageScpOpti
return err
}
}
err = execPodman(uSave, saveCommand)
_, err = execTransferPodman(uSave, saveCommand, false)
if err != nil {
return err
}
return execPodman(uLoad, loadCommand)
out, err := execTransferPodman(uLoad, loadCommand, (len(dest.Tag) > 0))
if err != nil {
return err
}
if out != nil {
image := domainUtils.ExtractImage(out)
_, err := execTransferPodman(uLoad, []string{podman, "tag", image, dest.Tag}, false)
return err
}
return nil
}
func lookupUser(u string) (*user.User, error) {
@ -847,10 +822,10 @@ func lookupUser(u string) (*user.User, error) {
return user.Lookup(u)
}
func execPodman(execUser *user.User, command []string) error {
cmdLogin, err := utils.LoginUser(execUser.Username)
func execTransferPodman(execUser *user.User, command []string, needToTag bool) ([]byte, error) {
cmdLogin, err := domainUtils.LoginUser(execUser.Username)
if err != nil {
return err
return nil, err
}
defer func() {
@ -864,11 +839,11 @@ func execPodman(execUser *user.User, command []string) error {
cmd.Stdout = os.Stdout
uid, err := strconv.ParseInt(execUser.Uid, 10, 32)
if err != nil {
return err
return nil, err
}
gid, err := strconv.ParseInt(execUser.Gid, 10, 32)
if err != nil {
return err
return nil, err
}
cmd.SysProcAttr = &syscall.SysProcAttr{
Credential: &syscall.Credential{
@ -878,5 +853,55 @@ func execPodman(execUser *user.User, command []string) error {
NoSetGroups: false,
},
}
return cmd.Run()
if needToTag {
cmd.Stdout = nil
return cmd.Output()
}
return nil, cmd.Run()
}
func getSigFilename(sigStoreDirPath string) (string, error) {
sigFileSuffix := 1
sigFiles, err := ioutil.ReadDir(sigStoreDirPath)
if err != nil {
return "", err
}
sigFilenames := make(map[string]bool)
for _, file := range sigFiles {
sigFilenames[file.Name()] = true
}
for {
sigFilename := "signature-" + strconv.Itoa(sigFileSuffix)
if _, exists := sigFilenames[sigFilename]; !exists {
return sigFilename, nil
}
sigFileSuffix++
}
}
func localPathFromURI(url *url.URL) (string, error) {
if url.Scheme != "file" {
return "", errors.Errorf("writing to %s is not supported. Use a supported scheme", url.String())
}
return url.Path, nil
}
// putSignature creates signature and saves it to the signstore file
func putSignature(manifestBlob []byte, mech signature.SigningMechanism, sigStoreDir string, instanceDigest digest.Digest, dockerReference dockerRef.Reference, options entities.SignOptions) error {
newSig, err := signature.SignDockerManifest(manifestBlob, dockerReference.String(), mech, options.SignBy)
if err != nil {
return err
}
signatureDir := fmt.Sprintf("%s@%s=%s", sigStoreDir, instanceDigest.Algorithm(), instanceDigest.Hex())
if err := os.MkdirAll(signatureDir, 0751); err != nil {
// The directory is allowed to exist
if !errors.Is(err, fs.ErrExist) {
return err
}
}
sigFilename, err := getSigFilename(signatureDir)
if err != nil {
return err
}
return ioutil.WriteFile(filepath.Join(signatureDir, sigFilename), newSig, 0644)
}

26
pkg/domain/infra/tunnel/images.go
View File

@ -2,6 +2,7 @@ package tunnel
import (
"context"
"fmt"
"io/ioutil"
"os"
"strconv"
@ -12,7 +13,6 @@ import (
"github.com/containers/common/pkg/config"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/types"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/bindings/images"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/containers/podman/v4/pkg/domain/entities/reports"
@ -123,10 +123,6 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, opts entities.
return &entities.ImagePullReport{Images: pulledImages}, nil
}
func (ir *ImageEngine) Transfer(ctx context.Context, source entities.ImageScpOptions, dest entities.ImageScpOptions, parentFlags []string) error {
return errors.Wrapf(define.ErrNotImplemented, "cannot use the remote client to transfer images between root and rootless storage")
}
func (ir *ImageEngine) Tag(ctx context.Context, nameOrID string, tags []string, opt entities.ImageTagOptions) error {
options := new(images.TagOptions)
for _, newTag := range tags {
@ -367,3 +363,23 @@ func (ir *ImageEngine) Shutdown(_ context.Context) {
func (ir *ImageEngine) Sign(ctx context.Context, names []string, options entities.SignOptions) (*entities.SignReport, error) {
return nil, errors.New("not implemented yet")
}
func (ir *ImageEngine) Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool) error {
options := new(images.ScpOptions)
var destination *string
if len(dst) > 1 {
destination = &dst
}
options.Quiet = &quiet
options.Destination = destination
rep, err := images.Scp(ir.ClientCtx, &src, destination, *options)
if err != nil {
return err
}
fmt.Println("Loaded Image(s):", rep.Id)
return nil
}

581
pkg/domain/utils/scp.go Normal file
View File

@ -0,0 +1,581 @@
package utils
import (
"bytes"
"fmt"
"io/ioutil"
"net"
"net/url"
"os"
"os/exec"
"os/user"
"strconv"
"strings"
"time"
scpD "github.com/dtylman/scp"
"github.com/containers/common/pkg/config"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/containers/podman/v4/pkg/terminal"
"github.com/docker/distribution/reference"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
)
func ExecuteTransfer(src, dst string, parentFlags []string, quiet bool) (*entities.ImageLoadReport, *entities.ImageScpOptions, *entities.ImageScpOptions, []string, error) {
source := entities.ImageScpOptions{}
dest := entities.ImageScpOptions{}
sshInfo := entities.ImageScpConnections{}
report := entities.ImageLoadReport{Names: []string{}}
podman, err := os.Executable()
if err != nil {
return nil, nil, nil, nil, err
}
f, err := ioutil.TempFile("", "podman") // open temp file for load/save output
if err != nil {
return nil, nil, nil, nil, err
}
confR, err := config.NewConfig("") // create a hand made config for the remote engine since we might use remote and native at once
if err != nil {
return nil, nil, nil, nil, errors.Wrapf(err, "could not make config")
}
cfg, err := config.ReadCustomConfig() // get ready to set ssh destination if necessary
if err != nil {
return nil, nil, nil, nil, err
}
locations := []*entities.ImageScpOptions{}
cliConnections := []string{}
args := []string{src}
if len(dst) > 0 {
args = append(args, dst)
}
for _, arg := range args {
loc, connect, err := ParseImageSCPArg(arg)
if err != nil {
return nil, nil, nil, nil, err
}
locations = append(locations, loc)
cliConnections = append(cliConnections, connect...)
}
source = *locations[0]
switch {
case len(locations) > 1:
if err = ValidateSCPArgs(locations); err != nil {
return nil, nil, nil, nil, err
}
dest = *locations[1]
case len(locations) == 1:
switch {
case len(locations[0].Image) == 0:
return nil, nil, nil, nil, errors.Wrapf(define.ErrInvalidArg, "no source image specified")
case len(locations[0].Image) > 0 && !locations[0].Remote && len(locations[0].User) == 0: // if we have podman image scp $IMAGE
return nil, nil, nil, nil, errors.Wrapf(define.ErrInvalidArg, "must specify a destination")
}
}
source.Quiet = quiet
source.File = f.Name() // after parsing the arguments, set the file for the save/load
dest.File = source.File
if err = os.Remove(source.File); err != nil { // remove the file and simply use its name so podman creates the file upon save. avoids umask errors
return nil, nil, nil, nil, err
}
allLocal := true // if we are all localhost, do not validate connections but if we are using one localhost and one non we need to use sshd
for _, val := range cliConnections {
if !strings.Contains(val, "@localhost::") {
allLocal = false
break
}
}
if allLocal {
cliConnections = []string{}
}
var serv map[string]config.Destination
serv, err = GetServiceInformation(&sshInfo, cliConnections, cfg)
if err != nil {
return nil, nil, nil, nil, err
}
confR.Engine = config.EngineConfig{Remote: true, CgroupManager: "cgroupfs", ServiceDestinations: serv} // pass the service dest (either remote or something else) to engine
saveCmd, loadCmd := CreateCommands(source, dest, parentFlags, podman)
switch {
case source.Remote: // if we want to load FROM the remote, dest can either be local or remote in this case
err = SaveToRemote(source.Image, source.File, "", sshInfo.URI[0], sshInfo.Identities[0])
if err != nil {
return nil, nil, nil, nil, err
}
if dest.Remote { // we want to load remote -> remote, both source and dest are remote
rep, id, err := LoadToRemote(dest, dest.File, "", sshInfo.URI[1], sshInfo.Identities[1])
if err != nil {
return nil, nil, nil, nil, err
}
if len(rep) > 0 {
fmt.Println(rep)
}
if len(id) > 0 {
report.Names = append(report.Names, id)
}
break
}
id, err := ExecPodman(dest, podman, loadCmd)
if err != nil {
return nil, nil, nil, nil, err
}
if len(id) > 0 {
report.Names = append(report.Names, id)
}
case dest.Remote: // remote host load, implies source is local
_, err = ExecPodman(dest, podman, saveCmd)
if err != nil {
return nil, nil, nil, nil, err
}
rep, id, err := LoadToRemote(dest, source.File, "", sshInfo.URI[0], sshInfo.Identities[0])
if err != nil {
return nil, nil, nil, nil, err
}
if len(rep) > 0 {
fmt.Println(rep)
}
if len(id) > 0 {
report.Names = append(report.Names, id)
}
if err = os.Remove(source.File); err != nil {
return nil, nil, nil, nil, err
}
default: // else native load, both source and dest are local and transferring between users
if source.User == "" { // source user has to be set, destination does not
source.User = os.Getenv("USER")
if source.User == "" {
u, err := user.Current()
if err != nil {
return nil, nil, nil, nil, errors.Wrapf(err, "could not obtain user, make sure the environmental variable $USER is set")
}
source.User = u.Username
}
}
return nil, &source, &dest, parentFlags, nil // transfer needs to be done in ABI due to cross issues
}
return &report, nil, nil, nil, nil
}
// CreateSCPCommand takes an existing command, appends the given arguments and returns a configured podman command for image scp
func CreateSCPCommand(cmd *exec.Cmd, command []string) *exec.Cmd {
cmd.Args = append(cmd.Args, command...)
cmd.Env = os.Environ()
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
return cmd
}
// ScpTag is a helper function for native podman to tag an image after a local load from image SCP
func ScpTag(cmd *exec.Cmd, podman string, dest entities.ImageScpOptions) error {
cmd.Stdout = nil
out, err := cmd.Output() // this function captures the output temporarily in order to execute the next command
if err != nil {
return err
}
image := ExtractImage(out)
if cmd.Args[0] == "sudo" { // transferRootless will need the sudo since we are loading to sudo from a user acct
cmd = exec.Command("sudo", podman, "tag", image, dest.Tag)
} else {
cmd = exec.Command(podman, "tag", image, dest.Tag)
}
cmd.Stdout = os.Stdout
return cmd.Run()
}
// ExtractImage pulls out the last line of output from save/load (image id)
func ExtractImage(out []byte) string {
fmt.Println(strings.TrimSuffix(string(out), "\n")) // print output
stringOut := string(out) // get all output
arrOut := strings.Split(stringOut, " ") // split it into an array
return strings.ReplaceAll(arrOut[len(arrOut)-1], "\n", "") // replace the trailing \n
}
// LoginUser starts the user process on the host so that image scp can use systemd-run
func LoginUser(user string) (*exec.Cmd, error) {
sleep, err := exec.LookPath("sleep")
if err != nil {
return nil, err
}
machinectl, err := exec.LookPath("machinectl")
if err != nil {
return nil, err
}
cmd := exec.Command(machinectl, "shell", "-q", user+"@.host", sleep, "inf")
err = cmd.Start()
return cmd, err
}
// loadToRemote takes image and remote connection information. it connects to the specified client
// and copies the saved image dir over to the remote host and then loads it onto the machine
// returns a string containing output or an error
func LoadToRemote(dest entities.ImageScpOptions, localFile string, tag string, url *url.URL, iden string) (string, string, error) {
dial, remoteFile, err := CreateConnection(url, iden)
if err != nil {
return "", "", err
}
defer dial.Close()
n, err := scpD.CopyTo(dial, localFile, remoteFile)
if err != nil {
errOut := strconv.Itoa(int(n)) + " Bytes copied before error"
return " ", "", errors.Wrapf(err, errOut)
}
var run string
if tag != "" {
return "", "", errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
}
podman := os.Args[0]
run = podman + " image load --input=" + remoteFile + ";rm " + remoteFile // run ssh image load of the file copied via scp
out, err := ExecRemoteCommand(dial, run)
if err != nil {
return "", "", err
}
rep := strings.TrimSuffix(string(out), "\n")
outArr := strings.Split(rep, " ")
id := outArr[len(outArr)-1]
if len(dest.Tag) > 0 { // tag the remote image using the output ID
run = podman + " tag " + id + " " + dest.Tag
_, err = ExecRemoteCommand(dial, run)
if err != nil {
return "", "", err
}
}
return rep, id, nil
}
// saveToRemote takes image information and remote connection information. it connects to the specified client
// and saves the specified image on the remote machine and then copies it to the specified local location
// returns an error if one occurs.
func SaveToRemote(image, localFile string, tag string, uri *url.URL, iden string) error {
dial, remoteFile, err := CreateConnection(uri, iden)
if err != nil {
return err
}
defer dial.Close()
if tag != "" {
return errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
}
podman := os.Args[0]
run := podman + " image save " + image + " --format=oci-archive --output=" + remoteFile // run ssh image load of the file copied via scp. Files are reverse in this case...
_, err = ExecRemoteCommand(dial, run)
if err != nil {
return err
}
n, err := scpD.CopyFrom(dial, remoteFile, localFile)
if _, conErr := ExecRemoteCommand(dial, "rm "+remoteFile); conErr != nil {
logrus.Errorf("Removing file on endpoint: %v", conErr)
}
if err != nil {
errOut := strconv.Itoa(int(n)) + " Bytes copied before error"
return errors.Wrapf(err, errOut)
}
return nil
}
// makeRemoteFile creates the necessary remote file on the host to
// save or load the image to. returns a string with the file name or an error
func MakeRemoteFile(dial *ssh.Client) (string, error) {
run := "mktemp"
remoteFile, err := ExecRemoteCommand(dial, run)
if err != nil {
return "", err
}
return strings.TrimSuffix(string(remoteFile), "\n"), nil
}
// createConnections takes a boolean determining which ssh client to dial
// and returns the dials client, its newly opened remote file, and an error if applicable.
func CreateConnection(url *url.URL, iden string) (*ssh.Client, string, error) {
cfg, err := ValidateAndConfigure(url, iden)
if err != nil {
return nil, "", err
}
dialAdd, err := ssh.Dial("tcp", url.Host, cfg) // dial the client
if err != nil {
return nil, "", errors.Wrapf(err, "failed to connect")
}
file, err := MakeRemoteFile(dialAdd)
if err != nil {
return nil, "", err
}
return dialAdd, file, nil
}
// GetSerivceInformation takes the parsed list of hosts to connect to and validates the information
func GetServiceInformation(sshInfo *entities.ImageScpConnections, cliConnections []string, cfg *config.Config) (map[string]config.Destination, error) {
var serv map[string]config.Destination
var urlS string
var iden string
for i, val := range cliConnections {
splitEnv := strings.SplitN(val, "::", 2)
sshInfo.Connections = append(sshInfo.Connections, splitEnv[0])
conn, found := cfg.Engine.ServiceDestinations[sshInfo.Connections[i]]
if found {
urlS = conn.URI
iden = conn.Identity
} else { // no match, warn user and do a manual connection.
urlS = "ssh://" + sshInfo.Connections[i]
iden = ""
logrus.Warnf("Unknown connection name given. Please use system connection add to specify the default remote socket location")
}
urlFinal, err := url.Parse(urlS) // create an actual url to pass to exec command
if err != nil {
return nil, err
}
if urlFinal.User.Username() == "" {
if urlFinal.User, err = GetUserInfo(urlFinal); err != nil {
return nil, err
}
}
sshInfo.URI = append(sshInfo.URI, urlFinal)
sshInfo.Identities = append(sshInfo.Identities, iden)
}
return serv, nil
}
// execPodman executes the podman save/load command given the podman binary
func ExecPodman(dest entities.ImageScpOptions, podman string, command []string) (string, error) {
cmd := exec.Command(podman)
CreateSCPCommand(cmd, command[1:])
logrus.Debugf("Executing podman command: %q", cmd)
if strings.Contains(strings.Join(command, " "), "load") { // need to tag
if len(dest.Tag) > 0 {
return "", ScpTag(cmd, podman, dest)
}
cmd.Stdout = nil
out, err := cmd.Output()
if err != nil {
return "", err
}
img := ExtractImage(out)
return img, nil
}
return "", cmd.Run()
}
// createCommands forms the podman save and load commands used by SCP
func CreateCommands(source entities.ImageScpOptions, dest entities.ImageScpOptions, parentFlags []string, podman string) ([]string, []string) {
var parentString string
quiet := ""
if source.Quiet {
quiet = "-q "
}
if len(parentFlags) > 0 {
parentString = strings.Join(parentFlags, " ") + " " // if there are parent args, an extra space needs to be added
} else {
parentString = strings.Join(parentFlags, " ")
}
loadCmd := strings.Split(fmt.Sprintf("%s %sload %s--input %s", podman, parentString, quiet, dest.File), " ")
saveCmd := strings.Split(fmt.Sprintf("%s %vsave %s--output %s %s", podman, parentString, quiet, source.File, source.Image), " ")
return saveCmd, loadCmd
}
// parseImageSCPArg returns the valid connection, and source/destination data based off of the information provided by the user
// arg is a string containing one of the cli arguments returned is a filled out source/destination options structs as well as a connections array and an error if applicable
func ParseImageSCPArg(arg string) (*entities.ImageScpOptions, []string, error) {
location := entities.ImageScpOptions{}
var err error
cliConnections := []string{}
switch {
case strings.Contains(arg, "@localhost::"): // image transfer between users
location.User = strings.Split(arg, "@")[0]
location, err = ValidateImagePortion(location, arg)
if err != nil {
return nil, nil, err
}
cliConnections = append(cliConnections, arg)
case strings.Contains(arg, "::"):
location, err = ValidateImagePortion(location, arg)
if err != nil {
return nil, nil, err
}
location.Remote = true
cliConnections = append(cliConnections, arg)
default:
location.Image = arg
}
return &location, cliConnections, nil
}
// validateImagePortion is a helper function to validate the image name in an SCP argument
func ValidateImagePortion(location entities.ImageScpOptions, arg string) (entities.ImageScpOptions, error) {
if RemoteArgLength(arg, 1) > 0 {
err := ValidateImageName(strings.Split(arg, "::")[1])
if err != nil {
return location, err
}
location.Image = strings.Split(arg, "::")[1] // this will get checked/set again once we validate connections
}
return location, nil
}
// validateSCPArgs takes the array of source and destination options and checks for common errors
func ValidateSCPArgs(locations []*entities.ImageScpOptions) error {
if len(locations) > 2 {
return errors.Wrapf(define.ErrInvalidArg, "cannot specify more than two arguments")
}
switch {
case len(locations[0].Image) > 0 && len(locations[1].Image) > 0:
locations[1].Tag = locations[1].Image
locations[1].Image = ""
case len(locations[0].Image) == 0 && len(locations[1].Image) == 0:
return errors.Wrapf(define.ErrInvalidArg, "a source image must be specified")
}
return nil
}
// validateImageName makes sure that the image given is valid and no injections are occurring
// we simply use this for error checking, bot setting the image
func ValidateImageName(input string) error {
// ParseNormalizedNamed transforms a shortname image into its
// full name reference so busybox => docker.io/library/busybox
// we want to keep our shortnames, so only return an error if
// we cannot parse what the user has given us
_, err := reference.ParseNormalizedNamed(input)
return err
}
// remoteArgLength is a helper function to simplify the extracting of host argument data
// returns an int which contains the length of a specified index in a host::image string
func RemoteArgLength(input string, side int) int {
if strings.Contains(input, "::") {
return len((strings.Split(input, "::"))[side])
}
return -1
}
// ExecRemoteCommand takes a ssh client connection and a command to run and executes the
// command on the specified client. The function returns the Stdout from the client or the Stderr
func ExecRemoteCommand(dial *ssh.Client, run string) ([]byte, error) {
sess, err := dial.NewSession() // new ssh client session
if err != nil {
return nil, err
}
defer sess.Close()
var buffer bytes.Buffer
var bufferErr bytes.Buffer
sess.Stdout = &buffer // output from client funneled into buffer
sess.Stderr = &bufferErr // err form client funneled into buffer
if err := sess.Run(run); err != nil { // run the command on the ssh client
return nil, errors.Wrapf(err, bufferErr.String())
}
return buffer.Bytes(), nil
}
func GetUserInfo(uri *url.URL) (*url.Userinfo, error) {
var (
usr *user.User
err error
)
if u, found := os.LookupEnv("_CONTAINERS_ROOTLESS_UID"); found {
usr, err = user.LookupId(u)
if err != nil {
return nil, errors.Wrapf(err, "failed to lookup rootless user")
}
} else {
usr, err = user.Current()
if err != nil {
return nil, errors.Wrapf(err, "failed to obtain current user")
}
}
pw, set := uri.User.Password()
if set {
return url.UserPassword(usr.Username, pw), nil
}
return url.User(usr.Username), nil
}
// ValidateAndConfigure will take a ssh url and an identity key (rsa and the like) and ensure the information given is valid
// iden iden can be blank to mean no identity key
// once the function validates the information it creates and returns an ssh.ClientConfig.
func ValidateAndConfigure(uri *url.URL, iden string) (*ssh.ClientConfig, error) {
var signers []ssh.Signer
passwd, passwdSet := uri.User.Password()
if iden != "" { // iden might be blank if coming from image scp or if no validation is needed
value := iden
s, err := terminal.PublicKey(value, []byte(passwd))
if err != nil {
return nil, errors.Wrapf(err, "failed to read identity %q", value)
}
signers = append(signers, s)
logrus.Debugf("SSH Ident Key %q %s %s", value, ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found { // validate ssh information, specifically the unix file socket used by the ssh agent.
logrus.Debugf("Found SSH_AUTH_SOCK %q, ssh-agent signer enabled", sock)
c, err := net.Dial("unix", sock)
if err != nil {
return nil, err
}
agentSigners, err := agent.NewClient(c).Signers()
if err != nil {
return nil, err
}
signers = append(signers, agentSigners...)
if logrus.IsLevelEnabled(logrus.DebugLevel) {
for _, s := range agentSigners {
logrus.Debugf("SSH Agent Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
}
}
var authMethods []ssh.AuthMethod // now we validate and check for the authorization methods, most notaibly public key authorization
if len(signers) > 0 {
var dedup = make(map[string]ssh.Signer)
for _, s := range signers {
fp := ssh.FingerprintSHA256(s.PublicKey())
if _, found := dedup[fp]; found {
logrus.Debugf("Dedup SSH Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
dedup[fp] = s
}
var uniq []ssh.Signer
for _, s := range dedup {
uniq = append(uniq, s)
}
authMethods = append(authMethods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) {
return uniq, nil
}))
}
if passwdSet { // if password authentication is given and valid, add to the list
authMethods = append(authMethods, ssh.Password(passwd))
}
if len(authMethods) == 0 {
authMethods = append(authMethods, ssh.PasswordCallback(func() (string, error) {
pass, err := terminal.ReadPassword(fmt.Sprintf("%s's login password:", uri.User.Username()))
return string(pass), err
}))
}
tick, err := time.ParseDuration("40s")
if err != nil {
return nil, err
}
cfg := &ssh.ClientConfig{
User: uri.User.Username(),
Auth: authMethods,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: tick,
}
return cfg, nil
}

39
pkg/domain/utils/utils_test.go
View File

@ -5,6 +5,7 @@ import (
"sort"
"testing"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/stretchr/testify/assert"
)
@ -74,3 +75,41 @@ func TestToURLValues(t *testing.T) {
})
}
}
func TestParseSCPArgs(t *testing.T) {
args := []string{"alpine", "root@localhost::"}
var source *entities.ImageScpOptions
var dest *entities.ImageScpOptions
var err error
source, _, err = ParseImageSCPArg(args[0])
assert.Nil(t, err)
assert.Equal(t, source.Image, "alpine")
dest, _, err = ParseImageSCPArg(args[1])
assert.Nil(t, err)
assert.Equal(t, dest.Image, "")
assert.Equal(t, dest.User, "root")
args = []string{"root@localhost::alpine"}
source, _, err = ParseImageSCPArg(args[0])
assert.Nil(t, err)
assert.Equal(t, source.User, "root")
assert.Equal(t, source.Image, "alpine")
args = []string{"charliedoern@192.168.68.126::alpine", "foobar@192.168.68.126::"}
source, _, err = ParseImageSCPArg(args[0])
assert.Nil(t, err)
assert.True(t, source.Remote)
assert.Equal(t, source.Image, "alpine")
dest, _, err = ParseImageSCPArg(args[1])
assert.Nil(t, err)
assert.True(t, dest.Remote)
assert.Equal(t, dest.Image, "")
args = []string{"charliedoern@192.168.68.126::alpine"}
source, _, err = ParseImageSCPArg(args[0])
assert.Nil(t, err)
assert.True(t, source.Remote)
assert.Equal(t, source.Image, "alpine")
}

13
test/apiv2/12-imagesMore.at
View File

@ -56,4 +56,17 @@ t GET libpod/images/$IMAGE/json 200 \
t DELETE libpod/images/$IMAGE 200 \
.ExitCode=0
podman pull -q $IMAGE
# test podman image SCP
# ssh needs to work so we can validate that the failure is past argument parsing
podman system connection add --default test ssh://$USER@localhost/run/user/$UID/podman/podman.sock
# should fail but need to check the output...
# status 125 here means that the save/load fails due to
# cirrus weirdness with exec.Command. All of the args have been parsed sucessfully.
t POST "libpod/images/scp/$IMAGE?destination=QA::" 500 \
.cause="exit status 125"
t DELETE libpod/images/$IMAGE 200 \
.ExitCode=0
stop_registry

2
test/apiv2/test-apiv2
View File

@ -23,6 +23,8 @@ REGISTRY_IMAGE="${PODMAN_TEST_IMAGE_REGISTRY}/${PODMAN_TEST_IMAGE_USER}/registry
###############################################################################
# BEGIN setup
USER=$PODMAN_ROOTLESS_USER
UID=$PODMAN_ROOTLESS_UID
TMPDIR=${TMPDIR:-/tmp}
WORKDIR=$(mktemp --tmpdir -d $ME.tmp.XXXXXX)

9
test/e2e/image_scp_test.go
View File

@ -50,18 +50,12 @@ var _ = Describe("podman image scp", func() {
})
It("podman image scp bogus image", func() {
if IsRemote() {
Skip("this test is only for non-remote")
}
scp := podmanTest.Podman([]string{"image", "scp", "FOOBAR"})
scp.WaitWithDefaultTimeout()
Expect(scp).Should(ExitWithError())
})
It("podman image scp with proper connection", func() {
if IsRemote() {
Skip("this test is only for non-remote")
}
cmd := []string{"system", "connection", "add",
"--default",
"QA",
@ -86,7 +80,10 @@ var _ = Describe("podman image scp", func() {
// This tests that the input we are given is validated and prepared correctly
// The error given should either be a missing image (due to testing suite complications) or a no such host timeout on ssh
Expect(scp).Should(ExitWithError())
// podman-remote exits with a different error
if !IsRemote() {
Expect(scp.ErrorToString()).Should(ContainSubstring("no such host"))
}
})

24
test/system/120-load.bats
View File

@ -128,8 +128,24 @@ verify_iid_and_name() {
run_podman image inspect --format '{{.Digest}}' $newname
is "$output" "$src_digest" "Digest of re-fetched image matches original"
# Clean up
# test tagging capability
run_podman untag $IMAGE $newname
run_podman image scp ${notme}@localhost::$newname foobar:123
run_podman image inspect --format '{{.Digest}}' foobar:123
is "$output" "$src_digest" "Digest of re-fetched image matches original"
# remove root img for transfer back with another name
_sudo $PODMAN image rm $newname
# get foobar's ID, for an ID transfer test
run_podman image inspect --format '{{.ID}}' foobar:123
run_podman image scp $output ${notme}@localhost::foobartwo
_sudo $PODMAN image exists foobartwo
# Clean up
_sudo $PODMAN image rm foobartwo
run_podman untag $IMAGE $newname
# Negative test for nonexistent image.
@ -142,12 +158,6 @@ verify_iid_and_name() {
run_podman 125 image scp $nope ${notme}@localhost::
is "$output" "Error: $nope: image not known.*" "Pushing nonexistent image"
# Negative test for copying to a different name
run_podman 125 image scp $IMAGE ${notme}@localhost::newname:newtag
is "$output" "Error: cannot specify an image rename: invalid argument" \
"Pushing with a different name: not allowed"
# FIXME: any point in copying by image ID? What else should we test?
}

24
utils/utils.go
View File

@ -243,27 +243,3 @@ func MovePauseProcessToScope(pausePidPath string) {
}
}
}
// CreateSCPCommand takes an existing command, appends the given arguments and returns a configured podman command for image scp
func CreateSCPCommand(cmd *exec.Cmd, command []string) *exec.Cmd {
cmd.Args = append(cmd.Args, command...)
cmd.Env = os.Environ()
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
return cmd
}
// LoginUser starts the user process on the host so that image scp can use systemd-run
func LoginUser(user string) (*exec.Cmd, error) {
sleep, err := exec.LookPath("sleep")
if err != nil {
return nil, err
}
machinectl, err := exec.LookPath("machinectl")
if err != nil {
return nil, err
}
cmd := exec.Command(machinectl, "shell", "-q", user+"@.host", sleep, "inf")
err = cmd.Start()
return cmd, err
}