VaultSecretSource: re-issue stale authToken (#770)

This commit is contained in:
Karl Fischer 2019-03-12 18:51:37 +01:00 committed by Joseph Petersen
parent e86574c839
commit a187e81ec1
11 changed files with 267 additions and 73 deletions

View File

@ -3,6 +3,7 @@
## 1.8 (not released yet)
- [#763](https://github.com/jenkinsci/configuration-as-code-plugin/issues/763) Introduce CASC_VAULT_PATHS to allow multiple vault paths to read from. CASC_VAULT_PATH kept for backwards compatibility and offering multi path too now.
- [#770](https://github.com/jenkinsci/configuration-as-code-plugin/issues/770) Re-issue TTL expired vault tokens for user/pass and appRole/secret authentication. Refresh secrets from vault on each `configure()` call.
## 1.7

View File

@ -252,6 +252,7 @@ Prerequisites:
- The environment variable `CASC_VAULT_NAMESPACE` is optional. If used, sets the Vault namespace for Enterprise Vaults.
- The environment variable `CASC_VAULT_FILE` is optional, provides a way for the other variables to be read from a file instead of environment variables.
- The environment variable `CASC_VAULT_ENGINE_VERSION` is optional. If unset, your vault path is assumed to be using kv version 2. If your vault path uses engine version 1, set this variable to `1`.
- The issued token should have read access to vault path `auth/token/lookup-self` in order to determine its expiration time. JCasC will re-issue a token if its expiration is reached (except for `CASC_VAULT_TOKEN`).
If the environment variables `CASC_VAULT_URL` and `CASC_VAULT_PATHS` are present, JCasC will try to gather initial secrets from Vault. However for it to work properly there is a need for authentication by either the combination of `CASC_VAULT_USER` and `CASC_VAULT_PW`, a `CASC_VAULT_TOKEN`, or the combination of `CASC_VAULT_APPROLE` and `CASC_VAULT_APPROLE_SECRET`. The authenticated user must have at least read access.

View File

@ -5,6 +5,10 @@ import com.bettercloud.vault.VaultConfig;
import com.bettercloud.vault.VaultException;
import hudson.Extension;
import io.jenkins.plugins.casc.SecretSource;
import io.jenkins.plugins.casc.impl.secrets.vault.VaultAppRoleAuthenticator;
import io.jenkins.plugins.casc.impl.secrets.vault.VaultAuthenticator;
import io.jenkins.plugins.casc.impl.secrets.vault.VaultSingleTokenAuthenticator;
import io.jenkins.plugins.casc.impl.secrets.vault.VaultUserPassAuthenticator;
import org.apache.commons.lang.StringUtils;
import java.io.FileInputStream;
@ -17,15 +21,15 @@ import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Requires either CASC_VAULT_USER and CASC_VAULT_PW, or CASC_VAULT_TOKEN environment variables set
* alongside with CASC_VAULT_PATHS and CASC_VAULT_URL
* Requires either CASC_VAULT_USER and CASC_VAULT_PW, or CASC_VAULT_TOKEN,
* or CASC_VAULT_APPROLE and CASC_VAULT_APPROLE_SECRET
* environment variables set alongside with CASC_VAULT_PATHS and CASC_VAULT_URL
*/
@Extension
public class VaultSecretSource extends SecretSource {
private static final Logger LOGGER = Logger.getLogger(VaultSecretSource.class.getName());
private Map<String, String> secrets = new HashMap<>();
private final static Logger LOGGER = Logger.getLogger(VaultSecretSource.class.getName());
private static final String CASC_VAULT_FILE = "CASC_VAULT_FILE";
private static final String CASC_VAULT_PW = "CASC_VAULT_PW";
@ -39,10 +43,15 @@ public class VaultSecretSource extends SecretSource {
private static final String CASC_VAULT_ENGINE_VERSION = "CASC_VAULT_ENGINE_VERSION";
private static final String CASC_VAULT_PATHS = "CASC_VAULT_PATHS";
private static final String CASC_VAULT_PATH = "CASC_VAULT_PATH"; // TODO: deprecate!
private static final String DEFAULT_ENGINE_VERSION = "2";
private static final String DEFAULT_USER_BACKEND = "userpass";
private Map<String, String> secrets = new HashMap<>();
private Vault vault;
private VaultConfig vaultConfig;
private VaultAuthenticator vaultAuthenticator;
private String[] vaultPaths;
public VaultSecretSource() {
Optional<String> vaultFile = Optional.ofNullable(System.getenv(CASC_VAULT_FILE));
@ -50,16 +59,9 @@ public class VaultSecretSource extends SecretSource {
vaultFile.ifPresent(file -> readPropertiesFromVaultFile(file, prop));
// Parse variables
Optional<String> vaultPw = getVariable(CASC_VAULT_PW, prop);
Optional<String> vaultUser = getVariable(CASC_VAULT_USER, prop);
Optional<String> vaultEngineVersionOpt = getVariable(CASC_VAULT_ENGINE_VERSION, prop);
Optional<String> vaultUrl = getVariable(CASC_VAULT_URL, prop);
Optional<String> vaultMount = getVariable(CASC_VAULT_MOUNT, prop);
Optional<String> vaultToken = getVariable(CASC_VAULT_TOKEN, prop);
Optional<String> vaultAppRole = getVariable(CASC_VAULT_APPROLE, prop);
Optional<String> vaultAppRoleSecret = getVariable(CASC_VAULT_APPROLE_SECRET, prop);
Optional<String> vaultNamespace = getVariable(CASC_VAULT_NAMESPACE, prop);
Optional<String> vaultEngineVersion = getVariable(CASC_VAULT_ENGINE_VERSION, prop);
Optional<String[]> vaultPaths = getCommaSeparatedVariables(CASC_VAULT_PATHS, prop)
.map(Optional::of)
.orElse(getCommaSeparatedVariables(CASC_VAULT_PATH, prop)); // TODO: deprecate!
@ -67,86 +69,88 @@ public class VaultSecretSource extends SecretSource {
// Check mandatory variables are set
if (!vaultUrl.isPresent() || !vaultPaths.isPresent()) return;
// Check defaults
if (!vaultMount.isPresent()) vaultMount = Optional.of(DEFAULT_USER_BACKEND);
if (!vaultEngineVersion.isPresent()) vaultEngineVersion = Optional.of(DEFAULT_ENGINE_VERSION);
String vaultEngineVersion = vaultEngineVersionOpt.orElse(DEFAULT_ENGINE_VERSION);
this.vaultPaths = vaultPaths.get();
determineAuthenticator(prop);
// configure vault client
VaultConfig vaultConfig = new VaultConfig().address(vaultUrl.get());
vaultConfig = new VaultConfig().address(vaultUrl.get());
try {
LOGGER.log(Level.FINE, "Attempting to connect to Vault: {0}", vaultUrl.get());
LOGGER.log(Level.FINE, "Attempting to connect to Vault: {0}", vaultUrl);
if (vaultNamespace.isPresent()) {
vaultConfig.nameSpace(vaultNamespace.get());
LOGGER.log(Level.FINE, "Using namespace with Vault: {0}", vaultNamespace);
}
vaultConfig.engineVersion(Integer.parseInt(vaultEngineVersion.get()));
vaultConfig.engineVersion(Integer.parseInt(vaultEngineVersion));
LOGGER.log(Level.FINE, "Using engine version: {0}", vaultEngineVersion);
vaultConfig = vaultConfig.build();
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Could not configure vault connection", e);
}
Vault vault = new Vault(vaultConfig);
Optional<String> authToken = Optional.empty();
// attempt token login
if (vaultToken.isPresent()) {
authToken = vaultToken;
LOGGER.log(Level.FINE, "Using supplied token to access Vault");
try {
vaultConfig.build();
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Could not configure vault client", e);
}
// attempt AppRole login
if (vaultAppRole.isPresent() && vaultAppRoleSecret.isPresent() && !authToken.isPresent()) {
try {
authToken = Optional.ofNullable(
vault.auth().loginByAppRole(vaultAppRole.get(), vaultAppRoleSecret.get()).getAuthClientToken()
);
LOGGER.log(Level.FINE, "Login to Vault using AppRole/SecretID successful");
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Could not login with AppRole", e);
}
}
vault = new Vault(vaultConfig);
}
// attempt User/Pass login
if (vaultUser.isPresent() && vaultPw.isPresent() && !authToken.isPresent()) {
try {
authToken = Optional.ofNullable(
vault.auth().loginByUserPass(vaultUser.get(), vaultPw.get(), vaultMount.get()).getAuthClientToken()
);
LOGGER.log(Level.FINE, "Login to Vault using User/Pass successful");
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Could not login with User/Pass", e);
}
}
private void determineAuthenticator(Properties prop) {
Optional<String> vaultPw = getVariable(CASC_VAULT_PW, prop);
Optional<String> vaultUser = getVariable(CASC_VAULT_USER, prop);
Optional<String> vaultMount = getVariable(CASC_VAULT_MOUNT, prop);
Optional<String> vaultToken = getVariable(CASC_VAULT_TOKEN, prop);
Optional<String> vaultAppRole = getVariable(CASC_VAULT_APPROLE, prop);
Optional<String> vaultAppRoleSecret = getVariable(CASC_VAULT_APPROLE_SECRET, prop);
// Use authToken to read secrets from vault
if (authToken.isPresent()) {
readSecretsFromVault(authToken.get(), vaultConfig, vault, vaultPaths.get());
} else {
LOGGER.log(Level.WARNING, "Vault auth token missing. Cannot read from vault");
vaultToken.ifPresent(
token -> vaultAuthenticator = new VaultSingleTokenAuthenticator(token)
);
vaultUser.ifPresent(
user -> vaultPw.ifPresent(
pw -> vaultAuthenticator = new VaultUserPassAuthenticator(user, pw, vaultMount.orElse(DEFAULT_USER_BACKEND))
)
);
vaultAppRole.ifPresent(
approle -> vaultAppRoleSecret.ifPresent(
approleSecret -> vaultAuthenticator = new VaultAppRoleAuthenticator(approle, approleSecret)
)
);
if (vaultAuthenticator == null) {
LOGGER.log(Level.WARNING, "Could not determine vault authentication method. Not able to read secrets from vault.");
}
}
private void readSecretsFromVault(String token, VaultConfig vaultConfig, Vault vault, String[] vaultPaths) {
try {
vaultConfig.token(token).build();
for (String vaultPath : vaultPaths) {
private void readSecretsFromVault() {
Optional<String[]> vaultPathsOpt = Optional.ofNullable(vaultPaths);
// check if we overwrite an existing key from another path
Map<String, String> nextSecrets = vault.logical().read(vaultPath).getData();
for (String key : nextSecrets.keySet()) {
if (secrets.containsKey(key)) {
LOGGER.log(Level.WARNING, "Key {0} exists in multiple vault paths.", key);
if (vaultPathsOpt.isPresent()) {
try {
// refresh map
secrets = new HashMap<>();
// Parse secrets
for (String vaultPath : vaultPathsOpt.get()) {
// check if we overwrite an existing key from another path
Map<String, String> nextSecrets = vault.logical().read(vaultPath).getData();
for (String key : nextSecrets.keySet()) {
if (secrets.containsKey(key)) {
LOGGER.log(Level.WARNING, "Key {0} exists in multiple vault paths.", key);
}
}
}
// merge
secrets.putAll(nextSecrets);
// merge
secrets.putAll(nextSecrets);
}
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Unable to fetch secret from Vault", e);
}
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Unable to fetch secret from Vault", e);
}
}
@ -164,6 +168,19 @@ public class VaultSecretSource extends SecretSource {
@Override
public Optional<String> reveal(String secret) {
if (StringUtils.isBlank(secret)) return Optional.empty();
// TODO: move this to SecretSource.init() function which gets called only once when CasC.configure() is run
// Ensure secrets are up-to-date
if (vaultAuthenticator != null) {
try {
vaultAuthenticator.authenticate(vault, vaultConfig);
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Could not authenticate with vault client", e);
}
readSecretsFromVault();
}
return Optional.ofNullable(secrets.get(secret));
}
@ -180,8 +197,9 @@ public class VaultSecretSource extends SecretSource {
}
private Optional<String[]> getCommaSeparatedVariables(String key, Properties prop) {
if (key.equals(CASC_VAULT_PATH)) LOGGER.log(Level.WARNING, "[Deprecation Warning] CASC_VAULT_PATH will be deprecated. " +
"Please use CASC_VAULT_PATHS instead."); // TODO: deprecate!
if (key.equals(CASC_VAULT_PATH))
LOGGER.log(Level.WARNING, "[Deprecation Warning] CASC_VAULT_PATH will be deprecated. " +
"Please use CASC_VAULT_PATHS instead."); // TODO: deprecate!
return getVariable(key, prop).map(str -> str.split(","));
}
}

View File

@ -0,0 +1,30 @@
package io.jenkins.plugins.casc.impl.secrets.vault;
import com.bettercloud.vault.Vault;
import com.bettercloud.vault.VaultConfig;
import com.bettercloud.vault.VaultException;
import java.util.logging.Level;
import java.util.logging.Logger;
public class VaultAppRoleAuthenticator extends VaultAuthenticatorWithExpiration {
private final static Logger LOGGER = Logger.getLogger(VaultAppRoleAuthenticator.class.getName());
private String approle;
private String approleSecret;
public VaultAppRoleAuthenticator(String approle, String approleSecret) {
this.approle = approle;
this.approleSecret = approleSecret;
}
public void authenticate(Vault vault, VaultConfig config) throws VaultException {
if (isTokenTTLExpired()) {
// authenticate
String authToken = vault.auth().loginByAppRole(approle, approleSecret).getAuthClientToken();
config.token(authToken).build();
LOGGER.log(Level.FINE, "Login to Vault using AppRole/SecretID successful");
getTTLExpiryOfCurrentToken(vault);
}
}
}

View File

@ -0,0 +1,9 @@
package io.jenkins.plugins.casc.impl.secrets.vault;
import com.bettercloud.vault.Vault;
import com.bettercloud.vault.VaultConfig;
import com.bettercloud.vault.VaultException;
public interface VaultAuthenticator {
void authenticate(Vault vault, VaultConfig config) throws VaultException;
}

View File

@ -0,0 +1,48 @@
package io.jenkins.plugins.casc.impl.secrets.vault;
import com.bettercloud.vault.Vault;
import com.bettercloud.vault.VaultException;
import java.util.Calendar;
import java.util.logging.Level;
import java.util.logging.Logger;
abstract class VaultAuthenticatorWithExpiration implements VaultAuthenticator {
private final static Logger LOGGER = Logger.getLogger(VaultAuthenticatorWithExpiration.class.getName());
private Calendar tokenExpiration;
public boolean isTokenTTLExpired() {
if (tokenExpiration == null) return true;
boolean result = true;
Calendar now = Calendar.getInstance();
long timeDiffInMillis = now.getTimeInMillis() - tokenExpiration.getTimeInMillis();
if (timeDiffInMillis < -2000L) {
// token will be valid for at least another 2s
result = false;
LOGGER.log(Level.FINE, "Auth token is still valid");
} else {
LOGGER.log(Level.FINE, "Auth token has to be re-issued" + timeDiffInMillis);
}
return result;
}
public void getTTLExpiryOfCurrentToken(Vault vault) {
int tokenTTL = 0;
try {
// save token TTL
tokenTTL = (int)vault.auth().lookupSelf().getTTL();
} catch (VaultException e) {
LOGGER.log(Level.WARNING, "Could not determine token expiration. " +
"Check if token is allowed to access auth/token/lookup-self. " +
"Assuming token TTL expired.", e);
}
tokenExpiration = Calendar.getInstance();
tokenExpiration.add(Calendar.SECOND, tokenTTL);
}
}

View File

@ -0,0 +1,18 @@
package io.jenkins.plugins.casc.impl.secrets.vault;
import com.bettercloud.vault.Vault;
import com.bettercloud.vault.VaultConfig;
import com.bettercloud.vault.VaultException;
public class VaultSingleTokenAuthenticator implements VaultAuthenticator {
private String token;
public VaultSingleTokenAuthenticator(String token) {
this.token = token;
}
public void authenticate(Vault vault, VaultConfig config) throws VaultException {
// No special mechanism - token already exists
config.token(token).build();
}
}

View File

@ -0,0 +1,32 @@
package io.jenkins.plugins.casc.impl.secrets.vault;
import com.bettercloud.vault.Vault;
import com.bettercloud.vault.VaultConfig;
import com.bettercloud.vault.VaultException;
import java.util.logging.Level;
import java.util.logging.Logger;
public class VaultUserPassAuthenticator extends VaultAuthenticatorWithExpiration {
private final static Logger LOGGER = Logger.getLogger(VaultUserPassAuthenticator.class.getName());
private String user;
private String pass;
private String mountPath;
public VaultUserPassAuthenticator(String user, String pass, String mountPath) {
this.user = user;
this.pass = pass;
this.mountPath = mountPath;
}
public void authenticate(Vault vault, VaultConfig config) throws VaultException {
if (isTokenTTLExpired()) {
// authenticate
String authToken = vault.auth().loginByUserPass(user, pass, mountPath).getAuthClientToken();
config.token(authToken).build();
LOGGER.log(Level.FINE, "Login to Vault using AppRole/SecretID successful");
getTTLExpiryOfCurrentToken(vault);
}
}
}

View File

@ -9,6 +9,10 @@ import org.junit.contrib.java.lang.system.EnvironmentVariables;
import org.jvnet.hudson.test.JenkinsRule;
import org.testcontainers.vault.VaultContainer;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import static org.hamcrest.CoreMatchers.equalTo;
import static io.jenkins.plugins.casc.vault.VaultTestUtil.*;
import static org.junit.Assert.assertThat;
@ -17,6 +21,8 @@ import static org.junit.Assume.assumeTrue;
// Inspired by https://github.com/BetterCloud/vault-java-driver/blob/master/src/test-integration/java/com/bettercloud/vault/util/VaultContainer.java
public class VaultSecretSourceTest {
private final static Logger LOGGER = Logger.getLogger(VaultSecretSourceTest.class.getName());
@ClassRule
public static VaultContainer vaultContainer = createVaultContainer();
@ -143,6 +149,31 @@ public class VaultSecretSourceTest {
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("123"));
}
@Test
public void kv2WithApproleWithReauth() {
envVars.set("CASC_VAULT_APPROLE", VAULT_APPROLE_ID);
envVars.set("CASC_VAULT_APPROLE_SECRET", VAULT_APPROLE_SECRET);
envVars.set("CASC_VAULT_PATHS", VAULT_PATH_KV2_AUTH_TEST);
envVars.set("CASC_VAULT_ENGINE_VERSION", "2");
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("auth-test"));
try {
// Wait for auth token to become stale
Thread.sleep(2000);
// Update secret
runCommand(vaultContainer, "vault", "kv", "put", VAULT_PATH_KV2_AUTH_TEST, "key1=re-auth-test");
} catch (InterruptedException e) {
LOGGER.log(Level.WARNING, "Test got interrupted", e);
assert false;
} catch (IOException eio) {
LOGGER.log(Level.WARNING, "Could not update vault secret for test", eio);
assert false;
}
assertThat(SecretSourceResolver.resolve(context, "${key1}"), equalTo("re-auth-test"));
}
// TODO: used to check for backwards compatibility. Deprecate!
@Test
public void kv2WithUserDeprecatedPath() {

View File

@ -26,10 +26,11 @@ class VaultTestUtil {
public static final String VAULT_PATH_KV2_1 = "kv-v2/admin";
public static final String VAULT_PATH_KV2_2 = "kv-v2/dev";
public static final String VAULT_PATH_KV2_3 = "kv-v2/qa";
public static final String VAULT_PATH_KV2_AUTH_TEST = "kv-v2/auth-test";
public static String VAULT_APPROLE_ID = "";
public static String VAULT_APPROLE_SECRET = "";
private static void runCommand(VaultContainer container, final String... command)
public static void runCommand(VaultContainer container, final String... command)
throws IOException, InterruptedException {
LOGGER.log(Level.FINE, String.join(" ", command));
container.execInContainer(command);
@ -73,7 +74,7 @@ class VaultTestUtil {
// Create AppRole
runCommand(container, "vault", "auth", "enable", "approle");
runCommand(container, "vault", "write", "auth/approle/role/admin",
"secret_id_ttl=10m", "token_num_uses=0", "token_ttl=20m", "token_max_ttl=20m",
"secret_id_ttl=10m", "token_num_uses=0", "token_ttl=4s", "token_max_ttl=4s",
"secret_id_num_uses=1000", "policies=admin");
// Retrieve AppRole credentials
@ -91,6 +92,7 @@ class VaultTestUtil {
runCommand(container, "vault", "kv", "put", VAULT_PATH_KV2_1, "key1=123", "key2=456");
runCommand(container, "vault", "kv", "put", VAULT_PATH_KV2_2, "key3=789");
runCommand(container, "vault", "kv", "put", VAULT_PATH_KV2_3, "key2=321");
runCommand(container, "vault", "kv", "put", VAULT_PATH_KV2_AUTH_TEST, "key1=auth-test");
} catch (Exception e) {
LOGGER.log(Level.WARNING, e.getMessage());

View File

@ -9,3 +9,7 @@ path "kv-v1/*" {
path "kv-v2/*" {
capabilities = ["create", "read", "list"]
}
path "auth/token/lookup-self" {
capabilities = ["read"]
}