Allow multiple vault paths in CASC_VAULT_PATHS (#763)

This commit is contained in:
Karl Fischer 2019-03-04 18:17:25 +00:00 committed by Joseph Petersen
parent 79f1f67c43
commit d97e07370b
5 changed files with 187 additions and 75 deletions

View File

@ -2,6 +2,8 @@
## 1.8 (not released yet)
- [#763](https://github.com/jenkinsci/configuration-as-code-plugin/issues/763) Introduce CASC_VAULT_PATHS to allow multiple vault paths to read from. CASC_VAULT_PATH kept for backwards compatibility and offering multi path too now.
## 1.7
- fix revealing secret with default defined, where environment variable is defined, it would always default.

View File

@ -244,14 +244,14 @@ Prerequisites:
- The environment variable `CASC_VAULT_APPROLE` must be present, if token is not used and U/P not used. (Vault AppRole ID.)
- The environment variable `CASC_VAULT_APPROLE_SECRET` must be present, it token is not used and U/P not used. (Value AppRole Secret ID.)
- The environment variable `CASC_VAULT_TOKEN` must be present, if U/P is not used. (Vault token.)
- The environment variable `CASC_VAULT_PATH` must be present. (Vault key path. For example, `/secrets/jenkins`.)
- The environment variable `CASC_VAULT_PATHS` must be present. (Comma separated vault key paths. For example, `secret/jenkins,secret/admin`.)
- The environment variable `CASC_VAULT_URL` must be present. (Vault url, including port number.)
- The environment variable `CASC_VAULT_MOUNT` is optional. (Vault auth mount. For example, `ldap` or another username & password authentication type, defaults to `userpass`.)
- The environment variable `CASC_VAULT_NAMESPACE` is optional. If used, sets the Vault namespace for Enterprise Vaults.
- The environment variable `CASC_VAULT_FILE` is optional, provides a way for the other variables to be read from a file instead of environment variables.
- The environment variable `CASC_VAULT_ENGINE_VERSION` is optional. If unset, your vault path is assumed to be using kv version 2. If your vault path uses engine version 1, set this variable to `1`.
If the environment variables `CASC_VAULT_URL` and `CASC_VAULT_PATH` are present, Configuration-as-Code will try to gather initial secrets from Vault. However for it to work properly there is a need for authentication by either the combination of `CASC_VAULT_USER` and `CASC_VAULT_PW`, a `CASC_VAULT_TOKEN`, or the combination of `CASC_VAULT_APPROLE` and `CASC_VAULT_APPROLE_SECRET`. The authenticated user must have at least read access.
If the environment variables `CASC_VAULT_URL` and `CASC_VAULT_PATHS` are present, Configuration-as-Code will try to gather initial secrets from Vault. However for it to work properly there is a need for authentication by either the combination of `CASC_VAULT_USER` and `CASC_VAULT_PW`, a `CASC_VAULT_TOKEN`, or the combination of `CASC_VAULT_APPROLE` and `CASC_VAULT_APPROLE_SECRET`. The authenticated user must have at least read access.
You can also provide a `CASC_VAULT_FILE` environment variable where you load the secrets from file.
@ -261,7 +261,7 @@ File should be in a Java Properties format
CASC_VAULT_PW=PASSWORD
CASC_VAULT_USER=USER
CASC_VAULT_TOKEN=TOKEN
CASC_VAULT_PATH=secret/jenkins/master
CASC_VAULT_PATHS=secret/jenkins/master,secret/admin
CASC_VAULT_URL=https://vault.dot.com
CASC_VAULT_MOUNT=ldap
```

View File

@ -18,7 +18,7 @@ import java.util.logging.Logger;
/**
* Requires either CASC_VAULT_USER and CASC_VAULT_PW, or CASC_VAULT_TOKEN environment variables set
* alongside with CASC_VAULT_PATH and CASC_VAULT_URL
* alongside with CASC_VAULT_PATHS and CASC_VAULT_URL
*/
@Extension
@ -27,66 +27,138 @@ public class VaultSecretSource extends SecretSource {
private final static Logger LOGGER = Logger.getLogger(VaultSecretSource.class.getName());
private Map<String, String> secrets = new HashMap<>();
private static final String CASC_VAULT_FILE = "CASC_VAULT_FILE";
private static final String CASC_VAULT_PW = "CASC_VAULT_PW";
private static final String CASC_VAULT_USER = "CASC_VAULT_USER";
private static final String CASC_VAULT_URL = "CASC_VAULT_URL";
private static final String CASC_VAULT_MOUNT = "CASC_VAULT_MOUNT";
private static final String CASC_VAULT_TOKEN = "CASC_VAULT_TOKEN";
private static final String CASC_VAULT_APPROLE = "CASC_VAULT_APPROLE";
private static final String CASC_VAULT_APPROLE_SECRET = "CASC_VAULT_APPROLE_SECRET";
private static final String CASC_VAULT_NAMESPACE = "CASC_VAULT_NAMESPACE";
private static final String CASC_VAULT_ENGINE_VERSION = "CASC_VAULT_ENGINE_VERSION";
private static final String CASC_VAULT_PATHS = "CASC_VAULT_PATHS";
private static final String CASC_VAULT_PATH = "CASC_VAULT_PATH"; // TODO: deprecate!
private static final String DEFAULT_ENGINE_VERSION = "2";
private static final String DEFAULT_USER_BACKEND = "userpass";
public VaultSecretSource() {
String vaultFile = System.getenv("CASC_VAULT_FILE");
Optional<String> vaultFile = Optional.ofNullable(System.getenv(CASC_VAULT_FILE));
Properties prop = new Properties();
if (vaultFile != null) {
try (FileInputStream input = new FileInputStream(vaultFile)) {
prop.load(input);
if (prop.isEmpty()) {
LOGGER.log(Level.WARNING, "Vault secret file is empty");
}
} catch (IOException ex) {
LOGGER.log(Level.WARNING, "Failed to load Vault secrets from file", ex);
vaultFile.ifPresent(file -> readPropertiesFromVaultFile(file, prop));
// Parse variables
Optional<String> vaultPw = getVariable(CASC_VAULT_PW, prop);
Optional<String> vaultUser = getVariable(CASC_VAULT_USER, prop);
Optional<String> vaultUrl = getVariable(CASC_VAULT_URL, prop);
Optional<String> vaultMount = getVariable(CASC_VAULT_MOUNT, prop);
Optional<String> vaultToken = getVariable(CASC_VAULT_TOKEN, prop);
Optional<String> vaultAppRole = getVariable(CASC_VAULT_APPROLE, prop);
Optional<String> vaultAppRoleSecret = getVariable(CASC_VAULT_APPROLE_SECRET, prop);
Optional<String> vaultNamespace = getVariable(CASC_VAULT_NAMESPACE, prop);
Optional<String> vaultEngineVersion = getVariable(CASC_VAULT_ENGINE_VERSION, prop);
Optional<String[]> vaultPaths = getCommaSeparatedVariables(CASC_VAULT_PATHS, prop)
.map(Optional::of)
.orElse(getCommaSeparatedVariables(CASC_VAULT_PATH, prop)); // TODO: deprecate!
// Check mandatory variables are set
if (!vaultUrl.isPresent() || !vaultPaths.isPresent()) return;
// Check defaults
if (!vaultMount.isPresent()) vaultMount = Optional.of(DEFAULT_USER_BACKEND);
if (!vaultEngineVersion.isPresent()) vaultEngineVersion = Optional.of(DEFAULT_ENGINE_VERSION);
// configure vault client
VaultConfig vaultConfig = new VaultConfig().address(vaultUrl.get());
try {
LOGGER.log(Level.FINE, "Attempting to connect to Vault: {0}", vaultUrl.get());
if (vaultNamespace.isPresent()) {
vaultConfig.nameSpace(vaultNamespace.get());
LOGGER.log(Level.FINE, "Using namespace with Vault: {0}", vaultNamespace);
}
vaultConfig.engineVersion(Integer.parseInt(vaultEngineVersion.get()));
LOGGER.log(Level.FINE, "Using engine version: {0}", vaultEngineVersion);
vaultConfig = vaultConfig.build();
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Could not configure vault connection", e);
}
Vault vault = new Vault(vaultConfig);
Optional<String> authToken = Optional.empty();
// attempt token login
if (vaultToken.isPresent()) {
authToken = vaultToken;
LOGGER.log(Level.FINE, "Using supplied token to access Vault");
}
// attempt AppRole login
if (vaultAppRole.isPresent() && vaultAppRoleSecret.isPresent() && !authToken.isPresent()) {
try {
authToken = Optional.ofNullable(
vault.auth().loginByAppRole(vaultAppRole.get(), vaultAppRoleSecret.get()).getAuthClientToken()
);
LOGGER.log(Level.FINE, "Login to Vault using AppRole/SecretID successful");
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Could not login with AppRole", e);
}
}
String vaultPw = getVariable("CASC_VAULT_PW", prop);
String vaultUsr = getVariable("CASC_VAULT_USER", prop);
String vaultPth = getVariable("CASC_VAULT_PATH", prop);
String vaultUrl = getVariable("CASC_VAULT_URL", prop);
String vaultMount = getVariable("CASC_VAULT_MOUNT", prop);
String vaultToken = getVariable("CASC_VAULT_TOKEN", prop);
String vaultAppRole = getVariable("CASC_VAULT_APPROLE", prop);
String vaultAppRoleSecret = getVariable("CASC_VAULT_APPROLE_SECRET", prop);
String vaultNamespace = getVariable("CASC_VAULT_NAMESPACE", prop);
String vaultEngineVersion = getVariable("CASC_VAULT_ENGINE_VERSION", prop);
if(((vaultPw != null && vaultUsr != null) ||
vaultToken != null ||
(vaultAppRole != null && vaultAppRoleSecret != null)) && vaultPth != null && vaultUrl != null) {
LOGGER.log(Level.FINE, "Attempting to connect to Vault: {0}", vaultUrl);
// attempt User/Pass login
if (vaultUser.isPresent() && vaultPw.isPresent() && !authToken.isPresent()) {
try {
VaultConfig config = new VaultConfig().address(vaultUrl);
if (vaultNamespace != null) {
// optionally set namespace
config = config.nameSpace(vaultNamespace);
LOGGER.log(Level.FINE, "Using namespace with Vault: {0}", vaultNamespace);
}
if (vaultEngineVersion != null) {
// optionally set vault engine version
config = config.engineVersion( Integer.parseInt(vaultEngineVersion) );
LOGGER.log(Level.FINE, "Using engine version: {0}", vaultEngineVersion);
}
config = config.build();
Vault vault = new Vault(config);
//Obtain a login token
final String token;
if (vaultToken != null) {
token = vaultToken;
LOGGER.log(Level.FINE, "Using supplied token to access Vault");
} else if (vaultAppRole != null && vaultAppRoleSecret != null) {
token = vault.auth().loginByAppRole(vaultAppRole, vaultAppRoleSecret).getAuthClientToken();
LOGGER.log(Level.FINE, "Login to Vault using AppRole/SecretID successful");
} else {
token = vault.auth().loginByUserPass(vaultUsr, vaultPw, vaultMount).getAuthClientToken();
LOGGER.log(Level.FINE, "Login to Vault using U/P successful");
}
config.token(token).build();
secrets = vault.logical().read(vaultPth).getData();
} catch (VaultException ve) {
LOGGER.log(Level.WARNING, "Unable to connect to Vault", ve);
authToken = Optional.ofNullable(
vault.auth().loginByUserPass(vaultUser.get(), vaultPw.get(), vaultMount.get()).getAuthClientToken()
);
LOGGER.log(Level.FINE, "Login to Vault using User/Pass successful");
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Could not login with User/Pass", e);
}
}
// Use authToken to read secrets from vault
if (authToken.isPresent()) {
readSecretsFromVault(authToken.get(), vaultConfig, vault, vaultPaths.get());
} else {
LOGGER.log(Level.WARNING, "Vault auth token missing. Cannot read from vault");
}
}
private void readSecretsFromVault(String token, VaultConfig vaultConfig, Vault vault, String[] vaultPaths) {
try {
vaultConfig.token(token).build();
for (String vaultPath : vaultPaths) {
// check if we overwrite an existing key from another path
Map<String, String> nextSecrets = vault.logical().read(vaultPath).getData();
for (String key : nextSecrets.keySet()) {
if (secrets.containsKey(key)) {
LOGGER.log(Level.WARNING, "Key {0} exists in multiple vault paths.", key);
}
}
// merge
secrets.putAll(nextSecrets);
}
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Unable to fetch secret from Vault", e);
}
}
private void readPropertiesFromVaultFile(String vaultFile, Properties prop) {
try (FileInputStream input = new FileInputStream(vaultFile)) {
prop.load(input);
if (prop.isEmpty()) {
LOGGER.log(Level.WARNING, "Vault secret file is empty");
}
} catch (IOException ex) {
LOGGER.log(Level.WARNING, "Failed to load Vault secrets from file", ex);
}
}
@Override
@ -103,11 +175,13 @@ public class VaultSecretSource extends SecretSource {
this.secrets = secrets;
}
private String getVariable(String key, Properties prop) {
if (prop != null && !prop.isEmpty()) {
return prop.getProperty(key, System.getenv(key));
} else {
return System.getenv(key);
}
private Optional<String> getVariable(String key, Properties prop) {
return Optional.ofNullable(prop.getProperty(key, System.getenv(key)));
}
private Optional<String[]> getCommaSeparatedVariables(String key, Properties prop) {
if (key.equals(CASC_VAULT_PATH)) LOGGER.log(Level.WARNING, "[Deprecation Warning] CASC_VAULT_PATH will be deprecated. " +
"Please use CASC_VAULT_PATHS instead."); // TODO: deprecate!
return getVariable(key, prop).map(str -> str.split(","));
}
}

View File

@ -49,7 +49,7 @@ public class VaultSecretSourceTest {
public void kv1WithUser() {
envVars.set("CASC_VAULT_USER", VAULT_USER);
envVars.set("CASC_VAULT_PW", VAULT_PW);
envVars.set("CASC_VAULT_PATH", VAULT_PATH_V1);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV1_1 + "," + VAULT_PATH_KV1_2);
envVars.set("CASC_VAULT_ENGINE_VERSION", "1");
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
}
@ -58,7 +58,7 @@ public class VaultSecretSourceTest {
public void kv2WithUser() {
envVars.set("CASC_VAULT_USER", VAULT_USER);
envVars.set("CASC_VAULT_PW", VAULT_PW);
envVars.set("CASC_VAULT_PATH", VAULT_PATH_V2);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1 + "," + VAULT_PATH_KV2_2);
envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
}
@ -67,7 +67,7 @@ public class VaultSecretSourceTest {
public void kv2WithWrongUser() {
envVars.set("CASC_VAULT_USER", "1234");
envVars.set("CASC_VAULT_PW", VAULT_PW);
envVars.set("CASC_VAULT_PATH", VAULT_PATH_V2);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1);
envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo(""));
}
@ -75,7 +75,7 @@ public class VaultSecretSourceTest {
@Test
public void kv1WithToken() {
envVars.set("CASC_VAULT_TOKEN", VAULT_ROOT_TOKEN);
envVars.set("CASC_VAULT_PATH", VAULT_PATH_V1);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV1_1 + "," + VAULT_PATH_KV1_2);
envVars.set("CASC_VAULT_ENGINE_VERSION", "1");
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
}
@ -83,7 +83,7 @@ public class VaultSecretSourceTest {
@Test
public void kv2WithToken() {
envVars.set("CASC_VAULT_TOKEN", VAULT_ROOT_TOKEN);
envVars.set("CASC_VAULT_PATH", VAULT_PATH_V2);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1 + "," + VAULT_PATH_KV2_2);
envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
}
@ -91,7 +91,7 @@ public class VaultSecretSourceTest {
@Test
public void kv1WithWrongToken() {
envVars.set("CASC_VAULT_TOKEN", "1234");
envVars.set("CASC_VAULT_PATH", VAULT_PATH_V1);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV1_1);
envVars.set("CASC_VAULT_ENGINE_VERSION", "1");
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo(""));
}
@ -100,7 +100,7 @@ public class VaultSecretSourceTest {
public void kv1WithApprole() {
envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
envVars.set("CASC_VAULT_PATH", VAULT_PATH_V1);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV1_1 + "," + VAULT_PATH_KV1_2);
envVars.set("CASC_VAULT_ENGINE_VERSION", "1");
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
}
@ -109,7 +109,7 @@ public class VaultSecretSourceTest {
public void kv2WithApprole() {
envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
envVars.set("CASC_VAULT_PATH", VAULT_PATH_V2);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1 + "," + VAULT_PATH_KV2_2);
envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
}
@ -118,8 +118,38 @@ public class VaultSecretSourceTest {
public void kv2WithWrongApprole() {
envVars.set("CASC_VAULT_APPROLE", "1234");
envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
envVars.set("CASC_VAULT_PATH", VAULT_PATH_V2);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1);
envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo(""));
}
@Test
public void kv2WithApproleMultipleKeys() {
envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1 + "," + VAULT_PATH_KV2_2);
envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
assertThat(SecretSourceResolver.resolve(context, "${key2}"), equalTo("456"));
assertThat(SecretSourceResolver.resolve(context, "${key3}"), equalTo("789"));
}
@Test
public void kv2WithApproleMultipleKeysOverriden() {
envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_1 + "," + VAULT_PATH_KV2_2 + "," + VAULT_PATH_KV2_3);
envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
assertThat(SecretSourceResolver.resolve(context, "${key2}"), equalTo("321"));
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
}
// TODO: used to check for backwards compatibility. Deprecate!
@Test
public void kv2WithUserDeprecatedPath() {
envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
envVars.set("CASC_VAULT_PATH", VAULT_PATH_KV1_1 + "," + VAULT_PATH_KV1_2);
envVars.set("CASC_VAULT_ENGINE_VERSION", "1");
assertThat(SecretSourceResolver.resolve(context, "${key3}"), equalTo("789"));
}
}

View File

@ -21,8 +21,11 @@ class VaultTestUtil {
public static final String VAULT_ROOT_TOKEN = "root-token";
public static final String VAULT_USER = "admin";
public static final String VAULT_PW = "admin";
public static final String VAULT_PATH_V1 = "kv-v1/admin";
public static final String VAULT_PATH_V2 = "kv-v2/admin";
public static final String VAULT_PATH_KV1_1 = "kv-v1/admin";
public static final String VAULT_PATH_KV1_2 = "kv-v1/dev";
public static final String VAULT_PATH_KV2_1 = "kv-v2/admin";
public static final String VAULT_PATH_KV2_2 = "kv-v2/dev";
public static final String VAULT_PATH_KV2_3 = "kv-v2/qa";
public static String VAULT_APPROLE_ID = "";
public static String VAULT_APPROLE_SECRET = "";
@ -83,8 +86,11 @@ class VaultTestUtil {
new HashMap<>()).getData().get("secret_id");
// add secrets for v1 and v2
runCommand(container, "vault", "kv", "put", VAULT_PATH_V1, "key1=123", "key2=456");
runCommand(container, "vault", "kv", "put", VAULT_PATH_V2, "key1=123", "key2=456");
runCommand(container, "vault", "kv", "put", VAULT_PATH_KV1_1, "key1=123", "key2=456");
runCommand(container, "vault", "kv", "put", VAULT_PATH_KV1_2, "key3=789");
runCommand(container, "vault", "kv", "put", VAULT_PATH_KV2_1, "key1=123", "key2=456");
runCommand(container, "vault", "kv", "put", VAULT_PATH_KV2_2, "key3=789");
runCommand(container, "vault", "kv", "put", VAULT_PATH_KV2_3, "key2=321");
} catch (Exception e) {
LOGGER.log(Level.WARNING, e.getMessage());