Merge pull request #216 from cydu-cloud/master

Support GIT_ASKPASS_URL callback

cmd/git-sync/main.go

@@ -98,6 +98,9 @@ var flSSHKnownHostsFile = flag.String("ssh-known-hosts-file", envString("GIT_SSH
 var flCookieFile = flag.Bool("cookie-file", envBool("GIT_COOKIE_FILE", false),
 	"use git cookiefile")
 
+var flAskPassURL = flag.String("askpass-url", envString("GIT_ASKPASS_URL", ""),
+	"the URL for GIT_ASKPASS callback")
+
 var flGitCmd = flag.String("git", envString("GIT_SYNC_GIT", "git"),
 	"the git command to run (subject to PATH search, mostly for testing)")
 
@@ -233,8 +236,8 @@ func main() {
 		os.Exit(1)
 	}
 
-	if (*flUsername != "" || *flPassword != "" || *flCookieFile) && *flSSH {
+	if (*flUsername != "" || *flPassword != "" || *flCookieFile || *flAskPassURL != "") && *flSSH {
-		fmt.Fprintf(os.Stderr, "ERROR: --ssh is set but --username, --password, or --cookie-file were provided\n")
+		fmt.Fprintf(os.Stderr, "ERROR: --ssh is set but --username, --password, --askpass-url, or --cookie-file were provided\n")
 		os.Exit(1)
 	}
 
@@ -263,6 +266,13 @@ func main() {
 		}
 	}
 
+	if *flAskPassURL != "" {
+		if err := setupGitAskPassURL(ctx); err != nil {
+			fmt.Fprintf(os.Stderr, "ERROR: failed to call ASKPASS callback URL: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
 	// The scope of the initialization context ends here, so we call cancel to release resources associated with it.
 	cancel()
 
@@ -315,7 +325,7 @@ func main() {
 	for {
 		start := time.Now()
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(*flSyncTimeout))
-		if changed, hash, err := syncRepo(ctx, *flRepo, *flBranch, *flRev, *flDepth, *flRoot, *flDest); err != nil {
+		if changed, hash, err := syncRepo(ctx, *flRepo, *flBranch, *flRev, *flDepth, *flRoot, *flDest, *flAskPassURL); err != nil {
 			syncDuration.WithLabelValues("error").Observe(time.Since(start).Seconds())
 			syncCount.WithLabelValues("error").Inc()
 			if *flMaxSyncFailures != -1 && failCount >= *flMaxSyncFailures {
@@ -571,7 +581,15 @@ func revIsHash(ctx context.Context, rev, gitRoot string) (bool, error) {
 
 // syncRepo syncs the branch of a given repository to the destination at the given rev.
 // returns (1) whether a change occured, (2) the new hash, and (3) an error if one happened
-func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot, dest string) (bool, string, error) {
+func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot, dest string, authUrl string) (bool, string, error) {
+	if authUrl != "" {
+		// For ASKPASS Callback URL, the credentials behind is dynamic, it needs to be
+		// re-fetched each time.
+		if err := setupGitAskPassURL(ctx); err != nil {
+			return false, "", fmt.Errorf("failed to call GIT_ASKPASS_URL: %v", err)
+		}
+	}
+
 	target := path.Join(gitRoot, dest)
 	gitRepoPath := path.Join(target, ".git")
 	var hash string
@@ -756,3 +774,55 @@ func setupGitCookieFile(ctx context.Context) error {
 
 	return nil
 }
+
+// The expected ASKPASS callback output are below,
+// see https://git-scm.com/docs/gitcredentials for more examples:
+// username=xxx@example.com
+// password=ya29.xxxyyyzzz
+func setupGitAskPassURL(ctx context.Context) error {
+	log.V(1).Info("configuring GIT_ASKPASS_URL")
+
+	var netClient = &http.Client{
+		Timeout: time.Second * 1,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+	httpReq, err := http.NewRequestWithContext(ctx, "GET", *flAskPassURL, nil)
+	if err != nil {
+		return fmt.Errorf("error create auth request: %v", err)
+	}
+	resp, err := netClient.Do(httpReq)
+	if err != nil {
+		return fmt.Errorf("error access auth url: %v", err)
+	}
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("access auth url: %v", err)
+	}
+	authData, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		return fmt.Errorf("error read auth response: %v", err)
+	}
+
+	username := ""
+	password := ""
+	for _, line := range strings.Split(string(authData), "\n") {
+		keyValues := strings.SplitN(line, "=", 2)
+		if len(keyValues) != 2 {
+			continue
+		}
+		switch keyValues[0] {
+		case "username":
+			username = keyValues[1]
+		case "password":
+			password = keyValues[1]
+		}
+	}
+
+	if err := setupGitAuth(ctx, username, password, *flRepo); err != nil {
+		return fmt.Errorf("error setup git auth: %v", err)
+	}
+
+	return nil
+}

docs/askpass-url.md

@@ -0,0 +1,31 @@
+# Using an Http Auth URL with git-sync
+
+## Step 1: Create a GIT_ASKPASS HTTP Service
+
+The GIT ASKPASS Service is exposed via HTTP and provide the answer to GIT_ASKPASS.
+
+Example of the service's output, see more at <https://git-scm.com/docs/gitcredentials>
+
+```
+username=xxx@example.com
+password=ya29.mysecret
+```
+
+## Step 2: Configure git-sync container
+
+In your git-sync container configuration, specify the GIT_ASKPASS_URL
+
+The credentials will pass in plain text, make sure the connection between git-sync
+and GIT ASKPASS Service are secure.
+
+See askpass_url e2e test as an example.
+
+```yaml
+name: "git-sync"
+...
+env:
+  - name: "GIT_SYNC_REPO",
+    value: "https://source.developers.google.com/p/[GCP PROJECT ID]/r/[REPO NAME]"
+  - name: "GIT_ASKPASS_URL",
+    value: "http://localhost:9102/git_askpass",
+```

docs/cookie-file.md

@@ -2,39 +2,35 @@
 
 Git-sync supports use of an HTTP Cookie File for accessing git content.
 
-# Step 1: Create Secret
+## Step 1: Create Secret
 
 First, create a secret file from the git cookie file you wish to
 use.
 
 Example: if the cookie-file is `~/.gitcookies`:
 
-```
+```bash
 kubectl create secret generic git-cookie-file --from-file=cookie_file=~/.gitcookies
 ```
 
-Note that the key is `cookie_file`.  This is the filename that git-sync will look
+Note that the key is `cookie_file`. This is the filename that git-sync will look
 for.
 
-# Step 2: Configure Pod/Deployment Volume
+## Step 2: Configure Pod/Deployment Volume
 
 In your Pod or Deployment configuration, specify a Volume for mounting the
 cookie-file Secret. Make sure to set `secretName` to the same name you used to
 create the secret (`git-cookie-file` in the example above).
 
-```
+```yaml
-volumes: [
+volumes:
-    {
+  - name: git-secret
-        "name": "git-secret",
+    secret:
-        "secret": {
+      secretName: git-cookie-file
-          "secretName": "git-cookie-file",
+      defaultMode: 0440
-        }
-    },
-    ...
-],
 ```
 
-# Step 2: Configure git-sync container
+## Step 3: Configure git-sync container
 
 In your git-sync container configuration, mount your volume at
 "/etc/git-secret". Make sure to pass the `--cookie-file` flag or set the
@@ -42,26 +38,16 @@ environment variable `GIT_COOKIE_FILE` to "true", and to use a git repo
 (`--repo` flag or `GIT_SYNC_REPO` env) is set to use a URL with the HTTP
 protocol.
 
-```
+```yaml
-{
+name: "git-sync"
-    name: "git-sync",
+...
-    ...
+env:
-    env: [
+  - name: GIT_SYNC_REPO
-        {
+    value: https://github.com/kubernetes/kubernetes.git
-            name: "GIT_SYNC_REPO",
+  - name: GIT_COOKIE_FILE
-            value: "https://github.com/kubernetes/kubernetes.git"
+    value: true
-        }, {
+volumeMounts:
-            name: "GIT_COOKIE_FILE",
+  - name: git-secret
-            value: "true",
+    mountPath: /etc/git-secret
-        },
+    readOnly: true
-    ...
-    ]
-    volumeMounts: [
-        {
-            "name": "git-secret",
-            "mountPath": "/etc/git-secret"
-        },
-        ...
-    ],
-}
 ```

docs/ssh.md

@@ -11,7 +11,7 @@ This can be done one of two ways:
 
 Obtain the host keys for your git server:
 
-```
+```bash
 ssh-keyscan $YOUR_GIT_HOST > /tmp/known_hosts
 ```
 
@@ -19,8 +19,7 @@ Use the `kubectl create secret` command and point to the file on your
 filesystem that stores the key. Ensure that the file is mapped to "ssh" as
 shown (the file can be located anywhere).
 
-
+```bash
-```
 kubectl create secret generic git-creds \
     --from-file=ssh=$HOME/.ssh/id_rsa \
     --from-file=known_hosts=/tmp/known_hosts
@@ -31,7 +30,7 @@ kubectl create secret generic git-creds \
 Write a config file for a Secret that holds your SSH private key, with the key
 (pasted in base64 encoded plaintext) mapped to the "ssh" field.
 
-```
+```json
 {
   "kind": "Secret",
   "apiVersion": "v1",
@@ -47,7 +46,7 @@ Write a config file for a Secret that holds your SSH private key, with the key
 
 Create the Secret using `kubectl create -f`.
 
-```
+```bash
 kubectl create -f /path/to/secret-config.json
 ```
 
@@ -57,7 +56,7 @@ In your Pod or Deployment configuration, specify a volume for mounting the
 Secret. Ensure that secretName matches the name you used when creating the
 Secret (e.g. "git-creds" used in both above examples).
 
-```
+```yaml
       # ...
       volumes:
       - name: git-secret
@@ -76,7 +75,7 @@ git@github.com/foo/bar) , and set the `-ssh` flags (or set GIT_SYNC_SSH to
 "true").  You will also need to set your container's `securityContext` to run
 as user ID "65533" which is created for running git-sync as non-root.
 
-```
+```yaml
       # ...
       containers:
       - name: git-sync
@@ -97,7 +96,7 @@ as user ID "65533" which is created for running git-sync as non-root.
 Lastly, you need to tell your Pod to run with the git-sync FS group.  Note
 that this is a Pod-wide setting, unlike the container `securityContext` above.
 
-```
+```yaml
       # ...
       securityContext:
         fsGroup: 65533 # to make SSH key readable
@@ -113,7 +112,7 @@ restrictive enough to be used as an SSH key), so make sure you set the
 In case the above YAML snippets are confusing (because whitespace matters in
 YAML), here is a full example:
 
-```
+```yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -131,7 +130,7 @@ spec:
       - name: git-secret
         secret:
           secretName: git-creds
-          defaultMode: 288 # = mode 0440
+          defaultMode: 0440
       containers:
       - name: git-sync
         image: k8s.gcr.io/git-sync:v3.1.1
@@ -146,6 +145,7 @@ spec:
         volumeMounts:
         - name: git-secret
           mountPath: /etc/git-secret
+          readOnly: true
       securityContext:
         fsGroup: 65533 # to make SSH key readable
 ```

test_e2e.sh

@@ -52,6 +52,14 @@ function assert_file_eq() {
     fail "file $1 does not contain '$2': $(cat $1)"
 }
 
+NCPORT=8888
+function freencport() {
+  while :; do
+    NCPORT=$((RANDOM+2000))
+    ss -lpn | grep -q ":$NCPORT " || break
+  done
+}
+
 # #####################
 # main
 # #####################
@@ -666,11 +674,54 @@ assert_file_eq "$ROOT"/link/file "$TESTCASE 1"
 # Wrap up
 pass
 
+##############################################
+# Test askpass_url
+##############################################
+testcase "askpass_url"
+echo "$TESTCASE 1" > "$REPO"/file
+freencport
+git -C "$REPO" commit -qam "$TESTCASE 1"
+# run the askpass_url service with wrong password
+{ (for i in 1 2; do echo -e 'HTTP/1.1 200 OK\r\n\r\nusername=you@example.com\npassword=dummypw' | nc -N -l $NCPORT > /dev/null; done) &}
+GIT_SYNC \
+    --git=$ASKPASS_GIT \
+    --askpass-url="http://localhost:$NCPORT/git_askpass" \
+    --logtostderr \
+    --v=5 \
+    --one-time \
+    --repo="file://$REPO" \
+    --branch=master \
+    --rev=HEAD \
+    --root="$ROOT" \
+    --dest="link" \
+    > "$DIR"/log."$TESTCASE" 2>&1 || true
+# check for failure
+assert_file_absent "$ROOT"/link/file
+# run with askpass_url service with correct password
+{ (for i in 1 2; do echo -e 'HTTP/1.1 200 OK\r\n\r\nusername=you@example.com\npassword=Lov3!k0os' | nc -N -l $NCPORT > /dev/null; done) &}
+GIT_SYNC \
+    --git=$ASKPASS_GIT \
+    --askpass-url="http://localhost:$NCPORT/git_askpass" \
+    --logtostderr \
+    --v=5 \
+    --one-time \
+    --repo="file://$REPO" \
+    --branch=master \
+    --rev=HEAD \
+    --root="$ROOT" \
+    --dest="link" \
+    > "$DIR"/log."$TESTCASE" 2>&1
+assert_link_exists "$ROOT"/link
+assert_file_exists "$ROOT"/link/file
+assert_file_eq "$ROOT"/link/file "$TESTCASE 1"
+# Wrap up
+pass
+
 ##############################################
 # Test webhook
 ##############################################
 testcase "webhook"
-NCPORT=8888
+freencport
 # First sync
 echo "$TESTCASE 1" > "$REPO"/file
 git -C "$REPO" commit -qam "$TESTCASE 1"