Allow multiple vault paths in CASC_VAULT_PATHS (#763)

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## 1.8 (not released yet)
 
+- [#763](https://github.com/jenkinsci/configuration-as-code-plugin/issues/763) Introduce CASC_VAULT_PATHS to allow multiple vault paths to read from. CASC_VAULT_PATH kept for backwards compatibility and offering multi path too now.
+
 ## 1.7
 
 - fix revealing secret with default defined, where environment variable is defined, it would always default.

README.md

@@ -244,14 +244,14 @@ Prerequisites:
 - The environment variable `CASC_VAULT_APPROLE` must be present, if token is not used and U/P not used. (Vault AppRole ID.)
 - The environment variable `CASC_VAULT_APPROLE_SECRET` must be present, it token is not used and U/P not used. (Value AppRole Secret ID.)
 - The environment variable `CASC_VAULT_TOKEN` must be present, if U/P is not used. (Vault token.)
-- The environment variable `CASC_VAULT_PATH` must be present. (Vault key path. For example, `/secrets/jenkins`.)
+- The environment variable `CASC_VAULT_PATHS` must be present. (Comma separated vault key paths. For example, `secret/jenkins,secret/admin`.)
 - The environment variable `CASC_VAULT_URL` must be present. (Vault url, including port number.)
 - The environment variable `CASC_VAULT_MOUNT` is optional. (Vault auth mount. For example, `ldap` or another username & password authentication type, defaults to `userpass`.)
 - The environment variable `CASC_VAULT_NAMESPACE` is optional. If used, sets the Vault namespace for Enterprise Vaults.
 - The environment variable `CASC_VAULT_FILE` is optional, provides a way for the other variables to be read from a file instead of environment variables.
 - The environment variable `CASC_VAULT_ENGINE_VERSION` is optional. If unset, your vault path is assumed to be using kv version 2. If your vault path uses engine version 1, set this variable to `1`.
 
-If the environment variables `CASC_VAULT_URL` and `CASC_VAULT_PATH` are present, Configuration-as-Code will try to gather initial secrets from Vault. However for it to work properly there is a need for authentication by either the combination of `CASC_VAULT_USER` and `CASC_VAULT_PW`, a `CASC_VAULT_TOKEN`, or the combination of `CASC_VAULT_APPROLE` and `CASC_VAULT_APPROLE_SECRET`. The authenticated user must have at least read access.
+If the environment variables `CASC_VAULT_URL` and `CASC_VAULT_PATHS` are present, Configuration-as-Code will try to gather initial secrets from Vault. However for it to work properly there is a need for authentication by either the combination of `CASC_VAULT_USER` and `CASC_VAULT_PW`, a `CASC_VAULT_TOKEN`, or the combination of `CASC_VAULT_APPROLE` and `CASC_VAULT_APPROLE_SECRET`. The authenticated user must have at least read access.
 
 You can also provide a `CASC_VAULT_FILE` environment variable where you load the secrets from file.
 
@@ -261,7 +261,7 @@ File should be in a Java Properties format
 CASC_VAULT_PW=PASSWORD
 CASC_VAULT_USER=USER
 CASC_VAULT_TOKEN=TOKEN
-CASC_VAULT_PATH=secret/jenkins/master
+CASC_VAULT_PATHS=secret/jenkins/master,secret/admin
 CASC_VAULT_URL=https://vault.dot.com
 CASC_VAULT_MOUNT=ldap
 ```

plugin/src/main/java/io/jenkins/plugins/casc/impl/secrets/VaultSecretSource.java

@@ -18,7 +18,7 @@ import java.util.logging.Logger;
 
 /**
  * Requires either CASC_VAULT_USER and CASC_VAULT_PW, or CASC_VAULT_TOKEN environment variables set
- * alongside with CASC_VAULT_PATH and CASC_VAULT_URL
+ * alongside with CASC_VAULT_PATHS and CASC_VAULT_URL
  */
 @Extension
 
@@ -27,66 +27,138 @@ public class VaultSecretSource extends SecretSource {
     private final static Logger LOGGER = Logger.getLogger(VaultSecretSource.class.getName());
     private Map<String, String> secrets = new HashMap<>();
 
+    private static final String CASC_VAULT_FILE = "CASC_VAULT_FILE";
+    private static final String CASC_VAULT_PW = "CASC_VAULT_PW";
+    private static final String CASC_VAULT_USER = "CASC_VAULT_USER";
+    private static final String CASC_VAULT_URL = "CASC_VAULT_URL";
+    private static final String CASC_VAULT_MOUNT = "CASC_VAULT_MOUNT";
+    private static final String CASC_VAULT_TOKEN = "CASC_VAULT_TOKEN";
+    private static final String CASC_VAULT_APPROLE = "CASC_VAULT_APPROLE";
+    private static final String CASC_VAULT_APPROLE_SECRET = "CASC_VAULT_APPROLE_SECRET";
+    private static final String CASC_VAULT_NAMESPACE = "CASC_VAULT_NAMESPACE";
+    private static final String CASC_VAULT_ENGINE_VERSION = "CASC_VAULT_ENGINE_VERSION";
+    private static final String CASC_VAULT_PATHS = "CASC_VAULT_PATHS";
+    private static final String CASC_VAULT_PATH = "CASC_VAULT_PATH"; // TODO: deprecate!
+
+    private static final String DEFAULT_ENGINE_VERSION = "2";
+    private static final String DEFAULT_USER_BACKEND = "userpass";
+
+
     public VaultSecretSource() {
-        String vaultFile = System.getenv("CASC_VAULT_FILE");
+        Optional<String> vaultFile = Optional.ofNullable(System.getenv(CASC_VAULT_FILE));
         Properties prop = new Properties();
-        if (vaultFile != null) {
+        vaultFile.ifPresent(file -> readPropertiesFromVaultFile(file, prop));
-            try (FileInputStream input = new FileInputStream(vaultFile)) {
+
-                prop.load(input);
+        // Parse variables
-                if (prop.isEmpty()) {
+        Optional<String> vaultPw = getVariable(CASC_VAULT_PW, prop);
-                    LOGGER.log(Level.WARNING, "Vault secret file is empty");
+        Optional<String> vaultUser = getVariable(CASC_VAULT_USER, prop);
-                }
+        Optional<String> vaultUrl = getVariable(CASC_VAULT_URL, prop);
-            } catch (IOException ex) {
+        Optional<String> vaultMount = getVariable(CASC_VAULT_MOUNT, prop);
-                LOGGER.log(Level.WARNING, "Failed to load Vault secrets from file", ex);
+        Optional<String> vaultToken = getVariable(CASC_VAULT_TOKEN, prop);
+        Optional<String> vaultAppRole = getVariable(CASC_VAULT_APPROLE, prop);
+        Optional<String> vaultAppRoleSecret = getVariable(CASC_VAULT_APPROLE_SECRET, prop);
+        Optional<String> vaultNamespace = getVariable(CASC_VAULT_NAMESPACE, prop);
+        Optional<String> vaultEngineVersion = getVariable(CASC_VAULT_ENGINE_VERSION, prop);
+
+        Optional<String[]> vaultPaths = getCommaSeparatedVariables(CASC_VAULT_PATHS, prop)
+                .map(Optional::of)
+                .orElse(getCommaSeparatedVariables(CASC_VAULT_PATH, prop)); // TODO: deprecate!
+
+        // Check mandatory variables are set
+        if (!vaultUrl.isPresent() || !vaultPaths.isPresent()) return;
+
+        // Check defaults
+        if (!vaultMount.isPresent()) vaultMount = Optional.of(DEFAULT_USER_BACKEND);
+        if (!vaultEngineVersion.isPresent()) vaultEngineVersion = Optional.of(DEFAULT_ENGINE_VERSION);
+
+        // configure vault client
+        VaultConfig vaultConfig = new VaultConfig().address(vaultUrl.get());
+        try {
+            LOGGER.log(Level.FINE, "Attempting to connect to Vault: {0}", vaultUrl.get());
+            if (vaultNamespace.isPresent()) {
+                vaultConfig.nameSpace(vaultNamespace.get());
+                LOGGER.log(Level.FINE, "Using namespace with Vault: {0}", vaultNamespace);
+            }
+
+            vaultConfig.engineVersion(Integer.parseInt(vaultEngineVersion.get()));
+            LOGGER.log(Level.FINE, "Using engine version: {0}", vaultEngineVersion);
+
+            vaultConfig = vaultConfig.build();
+        } catch (VaultException e) {
+            LOGGER.log(Level.WARNING, "Could not configure vault connection", e);
+        }
+
+        Vault vault = new Vault(vaultConfig);
+        Optional<String> authToken = Optional.empty();
+
+        // attempt token login
+        if (vaultToken.isPresent()) {
+            authToken = vaultToken;
+            LOGGER.log(Level.FINE, "Using supplied token to access Vault");
+        }
+
+        // attempt AppRole login
+        if (vaultAppRole.isPresent() && vaultAppRoleSecret.isPresent() && !authToken.isPresent()) {
+            try {
+                authToken = Optional.ofNullable(
+                        vault.auth().loginByAppRole(vaultAppRole.get(), vaultAppRoleSecret.get()).getAuthClientToken()
+                );
+                LOGGER.log(Level.FINE, "Login to Vault using AppRole/SecretID successful");
+            } catch (VaultException e) {
+                LOGGER.log(Level.WARNING, "Could not login with AppRole", e);
             }
         }
-        String vaultPw = getVariable("CASC_VAULT_PW", prop);
-        String vaultUsr = getVariable("CASC_VAULT_USER", prop);
-        String vaultPth = getVariable("CASC_VAULT_PATH", prop);
-        String vaultUrl = getVariable("CASC_VAULT_URL", prop);
-        String vaultMount = getVariable("CASC_VAULT_MOUNT", prop);
-        String vaultToken = getVariable("CASC_VAULT_TOKEN", prop);
-        String vaultAppRole = getVariable("CASC_VAULT_APPROLE", prop);
-        String vaultAppRoleSecret = getVariable("CASC_VAULT_APPROLE_SECRET", prop);
-        String vaultNamespace = getVariable("CASC_VAULT_NAMESPACE", prop);
-        String vaultEngineVersion = getVariable("CASC_VAULT_ENGINE_VERSION", prop);
 
-        if(((vaultPw != null && vaultUsr != null) || 
+        // attempt User/Pass login
-            vaultToken != null || 
+        if (vaultUser.isPresent() && vaultPw.isPresent() && !authToken.isPresent()) {
-            (vaultAppRole != null && vaultAppRoleSecret != null)) && vaultPth != null && vaultUrl != null) {
-            LOGGER.log(Level.FINE, "Attempting to connect to Vault: {0}", vaultUrl);
             try {
-                VaultConfig config = new VaultConfig().address(vaultUrl);
+                authToken = Optional.ofNullable(
-                if (vaultNamespace != null) {
+                        vault.auth().loginByUserPass(vaultUser.get(), vaultPw.get(), vaultMount.get()).getAuthClientToken()
-                    // optionally set namespace
+                );
-                    config = config.nameSpace(vaultNamespace);
+                LOGGER.log(Level.FINE, "Login to Vault using User/Pass successful");
-                    LOGGER.log(Level.FINE, "Using namespace with Vault: {0}", vaultNamespace);
+            } catch (VaultException e) {
-                }
+                LOGGER.log(Level.WARNING, "Could not login with User/Pass", e);
-                if (vaultEngineVersion != null) {
-                    // optionally set vault engine version
-                    config = config.engineVersion( Integer.parseInt(vaultEngineVersion) );
-                    LOGGER.log(Level.FINE, "Using engine version: {0}", vaultEngineVersion);
-                }
-                config = config.build();
-                Vault vault = new Vault(config);
-                //Obtain a login token
-                final String token;
-                if (vaultToken != null) {
-                    token = vaultToken;
-                    LOGGER.log(Level.FINE, "Using supplied token to access Vault");
-                } else if (vaultAppRole != null && vaultAppRoleSecret != null) {
-                    token = vault.auth().loginByAppRole(vaultAppRole, vaultAppRoleSecret).getAuthClientToken();
-                    LOGGER.log(Level.FINE, "Login to Vault using AppRole/SecretID successful");
-                } else {
-                    token = vault.auth().loginByUserPass(vaultUsr, vaultPw, vaultMount).getAuthClientToken();
-                    LOGGER.log(Level.FINE, "Login to Vault using U/P successful");
-                }
-                config.token(token).build();
-                secrets = vault.logical().read(vaultPth).getData();
-            } catch (VaultException ve) {
-                LOGGER.log(Level.WARNING, "Unable to connect to Vault", ve);
             }
         }
+
+        // Use authToken to read secrets from vault
+        if (authToken.isPresent()) {
+            readSecretsFromVault(authToken.get(), vaultConfig, vault, vaultPaths.get());
+        } else {
+            LOGGER.log(Level.WARNING, "Vault auth token missing. Cannot read from vault");
+        }
+    }
+
+    private void readSecretsFromVault(String token, VaultConfig vaultConfig, Vault vault, String[] vaultPaths) {
+        try {
+            vaultConfig.token(token).build();
+            for (String vaultPath : vaultPaths) {
+
+                // check if we overwrite an existing key from another path
+                Map<String, String> nextSecrets = vault.logical().read(vaultPath).getData();
+                for (String key : nextSecrets.keySet()) {
+                    if (secrets.containsKey(key)) {
+                        LOGGER.log(Level.WARNING, "Key {0} exists in multiple vault paths.", key);
+                    }
+                }
+
+                // merge
+                secrets.putAll(nextSecrets);
+            }
+        } catch (VaultException e) {
+            LOGGER.log(Level.WARNING, "Unable to fetch secret from Vault", e);
+        }
+    }
+
+    private void readPropertiesFromVaultFile(String vaultFile, Properties prop) {
+        try (FileInputStream input = new FileInputStream(vaultFile)) {
+            prop.load(input);
+            if (prop.isEmpty()) {
+                LOGGER.log(Level.WARNING, "Vault secret file is empty");
+            }
+        } catch (IOException ex) {
+            LOGGER.log(Level.WARNING, "Failed to load Vault secrets from file", ex);
+        }
     }
 
     @Override
@@ -103,11 +175,13 @@ public class VaultSecretSource extends SecretSource {
         this.secrets = secrets;
     }
 
-    private String getVariable(String key, Properties prop) {
+    private Optional<String> getVariable(String key, Properties prop) {
-        if (prop != null && !prop.isEmpty()) {
+        return Optional.ofNullable(prop.getProperty(key, System.getenv(key)));
-            return prop.getProperty(key, System.getenv(key));
+    }
-        } else {
+
-            return System.getenv(key);
+    private Optional<String[]> getCommaSeparatedVariables(String key, Properties prop) {
-        }
+        if (key.equals(CASC_VAULT_PATH)) LOGGER.log(Level.WARNING, "[Deprecation Warning] CASC_VAULT_PATH will be deprecated. " +
+                "Please use CASC_VAULT_PATHS instead."); // TODO: deprecate!
+        return getVariable(key, prop).map(str -> str.split(","));
     }
 }

plugin/src/test/java/io/jenkins/plugins/casc/vault/VaultSecretSourceTest.java

@@ -49,7 +49,7 @@ public class VaultSecretSourceTest {
     public void kv1WithUser() {
         envVars.set("CASC_VAULT_USER", VAULT_USER);
         envVars.set("CASC_VAULT_PW", VAULT_PW);
-        envVars.set("CASC_VAULT_PATH", VAULT_PATH_V1);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV1_1 + "," + VAULT_PATH_KV1_2);
         envVars.set("CASC_VAULT_ENGINE_VERSION", "1");
         assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
     }
@@ -58,7 +58,7 @@ public class VaultSecretSourceTest {
     public void kv2WithUser() {
         envVars.set("CASC_VAULT_USER", VAULT_USER);
         envVars.set("CASC_VAULT_PW", VAULT_PW);
-        envVars.set("CASC_VAULT_PATH", VAULT_PATH_V2);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1 + "," + VAULT_PATH_KV2_2);
         envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
         assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
     }
@@ -67,7 +67,7 @@ public class VaultSecretSourceTest {
     public void kv2WithWrongUser() {
         envVars.set("CASC_VAULT_USER", "1234");
         envVars.set("CASC_VAULT_PW", VAULT_PW);
-        envVars.set("CASC_VAULT_PATH", VAULT_PATH_V2);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1);
         envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
         assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo(""));
     }
@@ -75,7 +75,7 @@ public class VaultSecretSourceTest {
     @Test
     public void kv1WithToken() {
         envVars.set("CASC_VAULT_TOKEN", VAULT_ROOT_TOKEN);
-        envVars.set("CASC_VAULT_PATH", VAULT_PATH_V1);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV1_1 + "," + VAULT_PATH_KV1_2);
         envVars.set("CASC_VAULT_ENGINE_VERSION", "1");
         assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
     }
@@ -83,7 +83,7 @@ public class VaultSecretSourceTest {
     @Test
     public void kv2WithToken() {
         envVars.set("CASC_VAULT_TOKEN", VAULT_ROOT_TOKEN);
-        envVars.set("CASC_VAULT_PATH", VAULT_PATH_V2);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1 + "," + VAULT_PATH_KV2_2);
         envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
         assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
     }
@@ -91,7 +91,7 @@ public class VaultSecretSourceTest {
     @Test
     public void kv1WithWrongToken() {
         envVars.set("CASC_VAULT_TOKEN", "1234");
-        envVars.set("CASC_VAULT_PATH", VAULT_PATH_V1);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV1_1);
         envVars.set("CASC_VAULT_ENGINE_VERSION", "1");
         assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo(""));
     }
@@ -100,7 +100,7 @@ public class VaultSecretSourceTest {
     public void kv1WithApprole() {
         envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
         envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
-        envVars.set("CASC_VAULT_PATH", VAULT_PATH_V1);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV1_1 + "," + VAULT_PATH_KV1_2);
         envVars.set("CASC_VAULT_ENGINE_VERSION", "1");
         assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
     }
@@ -109,7 +109,7 @@ public class VaultSecretSourceTest {
     public void kv2WithApprole() {
         envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
         envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
-        envVars.set("CASC_VAULT_PATH", VAULT_PATH_V2);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1 + "," + VAULT_PATH_KV2_2);
         envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
         assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
     }
@@ -118,8 +118,38 @@ public class VaultSecretSourceTest {
     public void kv2WithWrongApprole() {
         envVars.set("CASC_VAULT_APPROLE", "1234");
         envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
-        envVars.set("CASC_VAULT_PATH", VAULT_PATH_V2);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1);
         envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
         assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo(""));
     }
+
+    @Test
+    public void kv2WithApproleMultipleKeys() {
+        envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
+        envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1 + "," + VAULT_PATH_KV2_2);
+        envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
+        assertThat(SecretSourceResolver.resolve(context, "${key2}"), equalTo("456"));
+        assertThat(SecretSourceResolver.resolve(context, "${key3}"), equalTo("789"));
+    }
+
+    @Test
+    public void kv2WithApproleMultipleKeysOverriden() {
+        envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
+        envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
+        envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1 + "," + VAULT_PATH_KV2_2 + "," + VAULT_PATH_KV2_3);
+        envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
+        assertThat(SecretSourceResolver.resolve(context, "${key2}"), equalTo("321"));
+        assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
+    }
+
+    // TODO: used to check for backwards compatibility. Deprecate!
+    @Test
+    public void kv2WithUserDeprecatedPath() {
+        envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
+        envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
+        envVars.set("CASC_VAULT_PATH", VAULT_PATH_KV1_1 + "," + VAULT_PATH_KV1_2);
+        envVars.set("CASC_VAULT_ENGINE_VERSION", "1");
+        assertThat(SecretSourceResolver.resolve(context, "${key3}"), equalTo("789"));
+    }
 }

plugin/src/test/java/io/jenkins/plugins/casc/vault/VaultTestUtil.java

@@ -21,8 +21,11 @@ class VaultTestUtil {
     public static final String VAULT_ROOT_TOKEN = "root-token";
     public static final String VAULT_USER = "admin";
     public static final String VAULT_PW = "admin";
-    public static final String VAULT_PATH_V1 = "kv-v1/admin";
+    public static final String VAULT_PATH_KV1_1 = "kv-v1/admin";
-    public static final String VAULT_PATH_V2 = "kv-v2/admin";
+    public static final String VAULT_PATH_KV1_2 = "kv-v1/dev";
+    public static final String VAULT_PATH_KV2_1 = "kv-v2/admin";
+    public static final String VAULT_PATH_KV2_2 = "kv-v2/dev";
+    public static final String VAULT_PATH_KV2_3 = "kv-v2/qa";
     public static String VAULT_APPROLE_ID = "";
     public static String VAULT_APPROLE_SECRET = "";
 
@@ -83,8 +86,11 @@ class VaultTestUtil {
                     new HashMap<>()).getData().get("secret_id");
 
             // add secrets for v1 and v2
-            runCommand(container, "vault", "kv", "put", VAULT_PATH_V1, "key1=123", "key2=456");
+            runCommand(container, "vault", "kv", "put", VAULT_PATH_KV1_1, "key1=123", "key2=456");
-            runCommand(container, "vault", "kv", "put", VAULT_PATH_V2, "key1=123", "key2=456");
+            runCommand(container, "vault", "kv", "put", VAULT_PATH_KV1_2, "key3=789");
+            runCommand(container, "vault", "kv", "put", VAULT_PATH_KV2_1, "key1=123", "key2=456");
+            runCommand(container, "vault", "kv", "put", VAULT_PATH_KV2_2, "key3=789");
+            runCommand(container, "vault", "kv", "put", VAULT_PATH_KV2_3, "key2=321");
 
         } catch (Exception e) {
             LOGGER.log(Level.WARNING, e.getMessage());