kubernetes
/
git-sync
mirror of https://github.com/kubernetes/git-sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add --credential flag to spec multiple user/pass 

* New flag is a JSON-encoded struct.
* Add test case for multiple HTTP passwd.
This commit is contained in:
Tim Hockin 2023-09-11 09:56:17 -07:00
parent 7da482e001
commit 162e543e05
3 changed files with 354 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
README.md
View File

@ -161,6 +161,26 @@ OPTIONS
Use a git cookiefile (/etc/git-secret/cookie_file) for
authentication.
--credential <string>, $GITSYNC_CREDENTIAL
Make one or more credentials available for authentication (see git
help credential). This is similar to --username and --password or
--password-file, but for specific URLs, for example when using
submodules. The value for this flag is either a JSON-encoded
object (see the schema below) or a JSON-encoded list of that same
object type. This flag may be specified more than once.
Object schema:
- url: string, required
- username: string, required
- password: string, optional
- password-file: string, optional
One of password or password-file must be specified. Users should
prefer password-file for better security.
Example:
--credential='{"url":"https://github.com", "username":"myname", "password-file":"/creds/mypass"}'
--depth <int>, $GITSYNC_DEPTH
Create a shallow clone with history truncated to the specified
number of commits. If not specified, this defaults to syncing a
@ -358,7 +378,8 @@ OPTIONS
--username <string>, $GITSYNC_USERNAME
The username to use for git authentication (see --password-file or
--password).
--password). If more than one username and password is required
(e.g. with submodules), use --credential.
-v, --verbose <int>, $GITSYNC_VERBOSE
Set the log verbosity level. Logs at this level and lower will be
@ -426,6 +447,12 @@ AUTHENTICATION
consults a URL (e.g. http://metadata) to get credentials on each
sync.
When using submodules it may be necessary to specify more than one
username and password, which can be done with --credential
(GITSYNC_CREDENTIAL). All of the username+password pairs, from
both --username/--password and --credential are fed into 'git
credential approve'.
SSH
When an SSH transport is specified, the key(s) defined in
--ssh-key-file (GITSYNC_SSH_KEY_FILE) will be used. Users are

263
main.go
View File

@ -21,6 +21,7 @@ package main // import "k8s.io/git-sync/cmd/git-sync"
import (
"context"
"crypto/md5"
"encoding/json"
"errors"
"fmt"
"io"
@ -105,6 +106,146 @@ const (
const defaultDirMode = os.FileMode(0775) // subject to umask
type credential struct {
URL string `json:"url"`
Username string `json:"username"`
Password string `json:"password,omitempty"`
PasswordFile string `json:"password-file,omitempty"`
}
func (c credential) String() string {
jb, err := json.Marshal(c)
if err != nil {
return fmt.Sprintf("<encoding error: %v>", err)
}
return string(jb)
}
// credentialSliceValue is for flags.
type credentialSliceValue struct {
value []credential
changed bool
}
var _ pflag.Value = &credentialSliceValue{}
var _ pflag.SliceValue = &credentialSliceValue{}
// pflagCredentialSlice is like pflag.StringSlice()
func pflagCredentialSlice(name, def, usage string) *[]credential {
p := &credentialSliceValue{}
_ = p.Set(def)
pflag.Var(p, name, usage)
return &p.value
}
// unmarshal is like json.Unmarshal, but fails on unknown fields.
func (cs credentialSliceValue) unmarshal(val string, out any) error {
dec := json.NewDecoder(strings.NewReader(val))
dec.DisallowUnknownFields()
return dec.Decode(out)
}
// decodeList handles a string-encoded JSON object.
func (cs credentialSliceValue) decodeObject(val string) (credential, error) {
var cred credential
if err := cs.unmarshal(val, &cred); err != nil {
return credential{}, err
}
return cred, nil
}
// decodeList handles a string-encoded JSON list.
func (cs credentialSliceValue) decodeList(val string) ([]credential, error) {
var creds []credential
if err := cs.unmarshal(val, &creds); err != nil {
return nil, err
}
return creds, nil
}
// decode handles a string-encoded JSON object or list.
func (cs credentialSliceValue) decode(val string) ([]credential, error) {
s := strings.TrimSpace(val)
if s == "" {
return nil, nil
}
// If it tastes like an object...
if s[0] == '{' {
cred, err := cs.decodeObject(s)
return []credential{cred}, err
}
// If it tastes like a list...
if s[0] == '[' {
return cs.decodeList(s)
}
// Otherwise, bad
return nil, fmt.Errorf("not a JSON object or list")
}
func (cs *credentialSliceValue) Set(val string) error {
v, err := cs.decode(val)
if err != nil {
return err
}
if !cs.changed {
cs.value = v
} else {
cs.value = append(cs.value, v...)
}
cs.changed = true
return nil
}
func (cs credentialSliceValue) Type() string {
return "credentialSlice"
}
func (cs credentialSliceValue) String() string {
if len(cs.value) == 0 {
return "[]"
}
jb, err := json.Marshal(cs.value)
if err != nil {
return fmt.Sprintf("<encoding error: %v>", err)
}
return string(jb)
}
func (cs *credentialSliceValue) Append(val string) error {
v, err := cs.decodeObject(val)
if err != nil {
return err
}
cs.value = append(cs.value, v)
return nil
}
func (cs *credentialSliceValue) Replace(val []string) error {
creds := []credential{}
for _, s := range val {
v, err := cs.decodeObject(s)
if err != nil {
return err
}
creds = append(creds, v)
}
cs.value = creds
return nil
}
func (cs credentialSliceValue) GetSlice() []string {
if len(cs.value) == 0 {
return nil
}
ret := []string{}
for _, cred := range cs.value {
ret = append(ret, cred.String())
}
return ret
}
func envString(def string, key string, alts ...string) string {
if val := os.Getenv(key); val != "" {
return val
@ -450,6 +591,7 @@ func main() {
flPasswordFile := pflag.String("password-file",
envString("", "GITSYNC_PASSWORD_FILE", "GIT_SYNC_PASSWORD_FILE"),
"the file from which the password or personal access token for git auth will be sourced")
flCredentials := pflagCredentialSlice("credential", envString("", "GITSYNC_CREDENTIAL"), "one or more credentials (see --man for details) available for authentication")
flSSHKeyFiles := pflag.StringArray("ssh-key-file",
envStringArray("/etc/git-secret/ssh", "GITSYNC_SSH_KEY_FILE", "GIT_SYNC_SSH_KEY_FILE", "GIT_SSH_KEY_FILE"),
@ -689,12 +831,36 @@ func main() {
}
}
if *flPassword != "" && *flPasswordFile != "" {
handleConfigError(log, true, "ERROR: only one of --password and --password-file may be specified")
}
if *flUsername != "" {
if *flPassword == "" && *flPasswordFile == "" {
handleConfigError(log, true, "ERROR: --password or --password-file must be set when --username is specified")
handleConfigError(log, true, "ERROR: --password or --password-file must be specified when --username is specified")
}
if *flPassword != "" && *flPasswordFile != "" {
handleConfigError(log, true, "ERROR: only one of --password and --password-file may be specified")
}
} else {
if *flPassword != "" {
handleConfigError(log, true, "ERROR: --password may only be specified when --username is specified")
}
if *flPasswordFile != "" {
handleConfigError(log, true, "ERROR: --password-file may only be specified when --username is specified")
}
}
if len(*flCredentials) > 0 {
for _, cred := range *flCredentials {
if cred.URL == "" {
handleConfigError(log, true, "ERROR: --credential URL must be specified")
}
if cred.Username == "" {
handleConfigError(log, true, "ERROR: --credential username must be specified")
}
if cred.Password == "" && cred.PasswordFile == "" {
handleConfigError(log, true, "ERROR: --credential password or password-file must be set")
}
if cred.Password != "" && cred.PasswordFile != "" {
handleConfigError(log, true, "ERROR: only one of --credential password and password-file may be specified")
}
}
}
@ -754,6 +920,17 @@ func main() {
absLink := makeAbsPath(*flLink, absRoot)
absTouchFile := makeAbsPath(*flTouchFile, absRoot)
// Merge credential sources.
if *flUsername != "" {
cred := credential{
URL: *flRepo,
Username: *flUsername,
Password: *flPassword,
PasswordFile: *flPasswordFile,
}
*flCredentials = append([]credential{cred}, (*flCredentials)...)
}
if *flAddUser {
if err := addUser(); err != nil {
log.Error(err, "ERROR: can't add user")
@ -800,14 +977,16 @@ func main() {
os.Exit(1)
}
if *flUsername != "" {
if *flPasswordFile != "" {
passwordFileBytes, err := os.ReadFile(*flPasswordFile)
// Finish populating credentials.
for i := range *flCredentials {
cred := &(*flCredentials)[i]
if cred.PasswordFile != "" {
passwordFileBytes, err := os.ReadFile(cred.PasswordFile)
if err != nil {
log.Error(err, "can't read password file", "file", *flPasswordFile)
log.Error(err, "can't read password file", "file", cred.PasswordFile)
os.Exit(1)
}
*flPassword = string(passwordFileBytes)
cred.Password = string(passwordFileBytes)
}
}
@ -931,8 +1110,8 @@ func main() {
// Craft a function that can be called to refresh credentials when needed.
refreshCreds := func(ctx context.Context) error {
// These should all be mutually-exclusive configs.
if *flUsername != "" {
if err := git.StoreCredentials(ctx, *flUsername, *flPassword); err != nil {
for _, cred := range *flCredentials {
if err := git.StoreCredentials(ctx, cred.URL, cred.Username, cred.Password); err != nil {
return err
}
}
@ -1109,6 +1288,11 @@ func logSafeFlags() []string {
arg := fl.Name
val := fl.Value.String()
// Don't log empty values
if val == "" {
return
}
// Handle --password
if arg == "password" {
val = redactedString
@ -1117,9 +1301,19 @@ func logSafeFlags() []string {
if arg == "repo" {
val = redactURL(val)
}
// Don't log empty values
if val == "" {
return
// Handle --credential
if arg == "credential" {
orig := fl.Value.(*credentialSliceValue)
sl := []credential{} // make a copy of the slice so we can mutate it
for _, cred := range orig.value {
if cred.Password != "" {
cred.Password = redactedString
}
sl = append(sl, cred)
}
tmp := *orig // make a copy
tmp.value = sl
val = tmp.String()
}
ret = append(ret, "--"+arg+"="+val)
@ -1904,12 +2098,12 @@ func md5sum(s string) string {
return fmt.Sprintf("%x", h.Sum(nil))
}
// StoreCredentials stores the username and password for later use.
func (git *repoSync) StoreCredentials(ctx context.Context, username, password string) error {
git.log.V(1).Info("storing git credentials")
git.log.V(9).Info("md5 of credentials", "username", md5sum(username), "password", md5sum(password))
// StoreCredentials stores a username and password for later use.
func (git *repoSync) StoreCredentials(ctx context.Context, url, username, password string) error {
git.log.V(1).Info("storing git credential", "url", url)
git.log.V(9).Info("md5 of credential", "url", url, "username", md5sum(username), "password", md5sum(password))
creds := fmt.Sprintf("url=%v\nusername=%v\npassword=%v\n", git.repo, username, password)
creds := fmt.Sprintf("url=%v\nusername=%v\npassword=%v\n", url, username, password)
_, _, err := git.RunWithStdin(ctx, "", creds, "credential", "approve")
if err != nil {
return fmt.Errorf("can't configure git credentials: %w", err)
@ -2017,7 +2211,7 @@ func (git *repoSync) CallAskPassURL(ctx context.Context) error {
}
}
if err := git.StoreCredentials(ctx, username, password); err != nil {
if err := git.StoreCredentials(ctx, git.repo, username, password); err != nil {
return err
}
@ -2297,6 +2491,26 @@ OPTIONS
Use a git cookiefile (/etc/git-secret/cookie_file) for
authentication.
--credential <string>, $GITSYNC_CREDENTIAL
Make one or more credentials available for authentication (see git
help credential). This is similar to --username and --password or
--password-file, but for specific URLs, for example when using
submodules. The value for this flag is either a JSON-encoded
object (see the schema below) or a JSON-encoded list of that same
object type. This flag may be specified more than once.
Object schema:
- url: string, required
- username: string, required
- password: string, optional
- password-file: string, optional
One of password or password-file must be specified. Users should
prefer password-file for better security.
Example:
--credential='{"url":"https://github.com", "username":"myname", "password-file":"/creds/mypass"}'
--depth <int>, $GITSYNC_DEPTH
Create a shallow clone with history truncated to the specified
number of commits. If not specified, this defaults to syncing a
@ -2494,7 +2708,8 @@ OPTIONS
--username <string>, $GITSYNC_USERNAME
The username to use for git authentication (see --password-file or
--password).
--password). If more than one username and password is required
(e.g. with submodules), use --credential.
-v, --verbose <int>, $GITSYNC_VERBOSE
Set the log verbosity level. Logs at this level and lower will be
@ -2562,6 +2777,12 @@ AUTHENTICATION
consults a URL (e.g. http://metadata) to get credentials on each
sync.
When using submodules it may be necessary to specify more than one
username and password, which can be done with --credential
(GITSYNC_CREDENTIAL). All of the username+password pairs, from
both --username/--password and --credential are fed into 'git
credential approve'.
SSH
When an SSH transport is specified, the key(s) defined in
--ssh-key-file (GITSYNC_SSH_KEY_FILE) will be used. Users are

84
test_e2e.sh
View File

@ -2738,6 +2738,87 @@ function e2e::submodule_sync_over_ssh_different_keys() {
rm -rf $NESTED_SUBMODULE
}
##############################################
# Test submodules over HTTP with different passwords
##############################################
function e2e::submodule_sync_over_http_different_passwords() {
# Init nested submodule repo
NESTED_SUBMODULE_REPO_NAME="nested-sub"
NESTED_SUBMODULE="$WORK/$NESTED_SUBMODULE_REPO_NAME"
mkdir "$NESTED_SUBMODULE"
git -C "$NESTED_SUBMODULE" init -q -b "$MAIN_BRANCH"
config_repo "$NESTED_SUBMODULE"
echo "nested-submodule" > "$NESTED_SUBMODULE/nested-submodule.file"
git -C "$NESTED_SUBMODULE" add nested-submodule.file
git -C "$NESTED_SUBMODULE" commit -aqm "init nested-submodule.file"
# Run a git-over-SSH server. Use password "test1".
echo 'test:$apr1$cXiFWR90$Pmoz7T8kEmlpC9Bpj4MX3.' > "$WORK/htpasswd.1"
CTR_SUBSUB=$(docker_run \
-v "$NESTED_SUBMODULE":/git/repo:ro \
-v "$WORK/htpasswd.1":/etc/htpasswd:ro \
e2e/test/httpd)
IP_SUBSUB=$(docker_ip "$CTR_SUBSUB")
# Init submodule repo
SUBMODULE_REPO_NAME="sub"
SUBMODULE="$WORK/$SUBMODULE_REPO_NAME"
mkdir "$SUBMODULE"
git -C "$SUBMODULE" init -q -b "$MAIN_BRANCH"
config_repo "$SUBMODULE"
echo "submodule" > "$SUBMODULE/submodule.file"
git -C "$SUBMODULE" add submodule.file
git -C "$SUBMODULE" commit -aqm "init submodule.file"
# Add nested submodule to submodule repo
echo -ne "url=http://$IP_SUBSUB/repo\nusername=test\npassword=test1\n" | git credential approve
git -C "$SUBMODULE" submodule add -q "http://$IP_SUBSUB/repo" "$NESTED_SUBMODULE_REPO_NAME"
git -C "$SUBMODULE" commit -aqm "add nested submodule"
# Run a git-over-SSH server. Use password "test2".
echo 'test:$apr1$vWBoWUBS$2H.WFxF8T7rH/gZF99Edl/' > "$WORK/htpasswd.2"
CTR_SUB=$(docker_run \
-v "$SUBMODULE":/git/repo:ro \
-v "$WORK/htpasswd.2":/etc/htpasswd:ro \
e2e/test/httpd)
IP_SUB=$(docker_ip "$CTR_SUB")
# Add the submodule to the main repo
echo -ne "url=http://$IP_SUB/repo\nusername=test\npassword=test2\n" | git credential approve
git -C "$REPO" submodule add -q "http://$IP_SUB/repo" "$SUBMODULE_REPO_NAME"
git -C "$REPO" commit -aqm "add submodule"
git -C "$REPO" submodule update --recursive --remote > /dev/null 2>&1
# Run a git-over-SSH server. Use password "test3".
echo 'test:$apr1$oKP2oGwp$ESJ4FESEP/8Sisy02B/vM/' > "$WORK/htpasswd.3"
CTR=$(docker_run \
-v "$REPO":/git/repo:ro \
-v "$WORK/htpasswd.3":/etc/htpasswd:ro \
e2e/test/httpd)
IP=$(docker_ip "$CTR")
GIT_SYNC \
--period=100ms \
--repo="http://$IP/repo" \
--root="$ROOT" \
--link="link" \
--credential="{ \"url\": \"http://$IP_SUBSUB/repo\", \"username\": \"test\", \"password\": \"test1\" }" \
--credential="{ \"url\": \"http://$IP_SUB/repo\", \"username\": \"test\", \"password\": \"test2\" }" \
--credential="{ \"url\": \"http://$IP/repo\", \"username\": \"test\", \"password\": \"test3\" }" \
&
wait_for_sync "${MAXWAIT}"
assert_link_exists "$ROOT/link"
assert_file_exists "$ROOT/link/file"
assert_file_exists "$ROOT/link/$SUBMODULE_REPO_NAME/submodule.file"
assert_file_exists "$ROOT/link/$SUBMODULE_REPO_NAME/$NESTED_SUBMODULE_REPO_NAME/nested-submodule.file"
assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
rm -rf $SUBMODULE
rm -rf $NESTED_SUBMODULE
}
##############################################
# Test sparse-checkout files
##############################################
@ -3215,6 +3296,9 @@ umask 0002
# Mark all repos as safe, to avoid "dubious ownership".
git config --global --add safe.directory '*'
# Store credentials for the test.
git config --global credential.helper "store --file $DIR/gitcreds"
FAILS=()
FINAL_RET=0
RUNS="${RUNS:-1}"