mirror of https://github.com/kubernetes/kops.git
e2e test runner
This commit is contained in:
Justin Santa Barbara
parent
b5b173c0e3
commit
68719704ef
|
|
@ -39,4 +39,10 @@ spec:
|
|||
zone: us-east-1c
|
||||
name: main
|
||||
...
|
||||
```
|
||||
```
|
||||
|
||||
|
||||
### Running the kubernetes e2e test suite
|
||||
|
||||
The [e2e](../e2e/README.md) directory has a docker image and some scripts which make it easy to run
|
||||
the kubernetes e2e tests, using kops.
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
FROM debian:jessie
|
||||
|
||||
RUN apt-get update && apt-get install --yes curl python-pip openssh-client
|
||||
RUN pip install awscli
|
||||
|
||||
RUN curl https://sdk.cloud.google.com | bash
|
||||
|
||||
RUN curl https://storage.googleapis.com/golang/go1.6.3.linux-amd64.tar.gz | tar -C /usr/local -xzf -
|
||||
|
||||
ENV PATH /root/google-cloud-sdk/bin:/usr/local/go/bin:$PATH
|
||||
|
||||
ADD runtests.sh /
|
||||
|
||||
ADD conf /conf
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
#JOB_NAME=kubernetes-e2e-kops-aws
|
||||
#KUBERNETES_VERSION=v1.3.5
|
||||
#DNS_DOMAIN="mydomain.com"
|
||||
#JENKINS_GCS_LOGS_PATH=gs://kopeio-kubernetes-e2e/logs
|
||||
#KOPS_STATE_STORE=s3://mys3bucket
|
||||
|
||||
test: image
|
||||
docker run -v ${HOME}/.ssh/id_rsa.pub:/root/.ssh/id_rsa.pub \
|
||||
-v ${HOME}/.aws:/root/.aws \
|
||||
-v ${HOME}/.gsutil:/root/.gsutil \
|
||||
-v ${HOME}/.config/gcloud:/root/.config/gcloud \
|
||||
-e "DNS_DOMAIN=${DNS_DOMAIN}" \
|
||||
-e "JENKINS_GCS_LOGS_PATH=${JENKINS_GCS_LOGS_PATH}" \
|
||||
-e "KOPS_STATE_STORE=${KOPS_STATE_STORE}" \
|
||||
-e "JOB_NAME=${JOB_NAME}" \
|
||||
-e "KUBERNETES_VERSION=${KUBERNETES_VERSION}" \
|
||||
kubernetes-e2e-runner-kops \
|
||||
/runtests.sh
|
||||
|
||||
image:
|
||||
docker build -t kubernetes-e2e-runner-kops .
|
||||
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
## Run kubernetes e2e tests
|
||||
|
||||
This docker image lets you run the kubernetes e2e tests very easily, using kops to create the cluster.
|
||||
|
||||
You simply call make, specifying some variables that controls the build.
|
||||
|
||||
An example:
|
||||
|
||||
`make JOB_NAME=kubernetes-e2e-kops-aws KUBERNETES_VERSION=v1.3.5 DNS_DOMAIN=e2e.mydomain.com JENKINS_GCS_LOGS_PATH=gs://kopeio-kubernetes-e2e/logs KOPS_STATE_STORE=s3://clusters.mydomain.com`
|
||||
|
||||
Variables:
|
||||
|
||||
* `JOB_NAME` the e2e job to run. Corresponds to a conf file in the conf directory.
|
||||
* `KUBERNETES_VERSION` the version of kubernetes to run. Either a version like `v1.3.5`, or a URL prefix like `https://storage.googleapis.com/kubernetes-release-dev/ci/v1.4.0-alpha.2.677+ea69570f61af8e/`. See [testing docs](../docs/testing.md)
|
||||
* `DNS_DOMAIN` the dns domain name to use for the cluster. Must be a real domain name, with a zone registered in DNS (route53)
|
||||
* `JENKINS_GCS_LOGS_PATH` the gs bucket where we should upload the results of the build. Note these will be publicly readable.
|
||||
* `KOPS_STATE_STORE` the url where the kops registry (store of cluster information) lives.
|
||||
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
##=============================================================
|
||||
# AWS Settings
|
||||
export AWS_DEFAULT_REGION=${AWS_REGION}
|
||||
#export KUBE_AWS_ZONE="us-west-2a"
|
||||
#export PROJECT="k8s-jkns-e2e-aws"
|
||||
export AWS_CONFIG_FILE=~/.aws/credentials
|
||||
export AWS_SSH_KEY=~/.ssh/kube_aws_rsa
|
||||
export KUBE_SSH_USER=admin
|
||||
# This is needed to be able to create PD from the e2e test
|
||||
export AWS_SHARED_CREDENTIALS_FILE=~/.aws/credentials
|
||||
|
||||
export AWS_PROFILE=kubernetes-e2e
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
# GCE variables
|
||||
#export INSTANCE_PREFIX="${E2E_NAME:-jenkins-e2e}"
|
||||
#export KUBE_GCE_NETWORK="${E2E_NAME:-jenkins-e2e}"
|
||||
#export KUBE_GCE_INSTANCE_PREFIX="${E2E_NAME:-jenkins-e2e}"
|
||||
#export GCE_SERVICE_ACCOUNT=$(gcloud auth list 2> /dev/null | grep active | cut -f3 -d' ')
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
export KUBERNETES_PROVIDER="aws"
|
||||
|
||||
export E2E_MIN_STARTUP_PODS="1"
|
||||
export MASTER_SIZE="m3.large"
|
||||
export NODE_SIZE="m3.large"
|
||||
export NUM_NODES="3"
|
||||
export GINKGO_TEST_ARGS="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
|
||||
export GINKGO_PARALLEL="y"
|
||||
|
||||
# Central only has 2 AZs, so we use it for the non-HA test
|
||||
export NODE_ZONES="eu-central-1a"
|
||||
export AWS_REGION="eu-central-1"
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
. conf/kubernetes-e2e-upup-aws
|
||||
|
||||
export NODE_ZONES="eu-west-1a,eu-west-1b,eu-west-1c"
|
||||
export AWS_REGION="eu-west-1"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
# Used to set site-specific settings
|
||||
# Most of these probably are better in the Makefile
|
||||
|
|
@ -0,0 +1,215 @@
|
|||
#!/bin/bash -ex
|
||||
|
||||
if [[ -z "${JOB_NAME}" ]]; then
|
||||
echo "Must specify JOB_NAME env var"
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "${KUBERNETES_VERSION}" ]]; then
|
||||
echo "Must specify KUBERNETES_VERSION env var"
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "${DNS_DOMAIN}" ]]; then
|
||||
echo "Must specify DNS_DOMAIN env var"
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "${KOPS_STATE_STORE}" ]]; then
|
||||
echo "Must specify KOPS_STATE_STORE env var"
|
||||
exit 1
|
||||
fi
|
||||
# TODO: Maybe skip if we don't want to upload logs?
|
||||
if [[ -z "${JENKINS_GCS_LOGS_PATH}" ]]; then
|
||||
echo "Must specify JENKINS_GCS_LOGS_PATH env var"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "JOB_NAME=${JOB_NAME}"
|
||||
echo "Loading conf/${JOB_NAME}"
|
||||
|
||||
. conf/${JOB_NAME}
|
||||
|
||||
echo "Loading conf/cloud/${KUBERNETES_PROVIDER}"
|
||||
. conf/cloud/${KUBERNETES_PROVIDER}
|
||||
|
||||
echo "Loading conf/site"
|
||||
. conf/site
|
||||
|
||||
##=============================================================
|
||||
# Global settings
|
||||
export KUBE_GCS_RELEASE_BUCKET=kubernetes-release
|
||||
|
||||
# We download the binaries ourselves
|
||||
# TODO: No way to tell e2e to use a particular release?
|
||||
# TODO: Maybe download and then bring up the cluster?
|
||||
export JENKINS_USE_EXISTING_BINARIES=y
|
||||
|
||||
# This actually just skips kube-up master detection
|
||||
export KUBERNETES_CONFORMANCE_TEST=y
|
||||
|
||||
##=============================================================
|
||||
# System settings (emulate jenkins)
|
||||
export USER=root
|
||||
export WORKSPACE=$HOME
|
||||
# Nothing should want Jenkins $HOME
|
||||
export HOME=${WORKSPACE}
|
||||
export BUILD_NUMBER=`date -u +%Y%m%d%H%M%S`
|
||||
export JENKINS_HOME=${HOME}
|
||||
|
||||
# We'll directly up & down the cluster
|
||||
export E2E_UP="${E2E_UP:-false}"
|
||||
export E2E_TEST="${E2E_TEST:-true}"
|
||||
export E2E_DOWN="${E2E_DOWN:-false}"
|
||||
|
||||
# Skip gcloud update checking
|
||||
export CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK=true
|
||||
|
||||
|
||||
##=============================================================
|
||||
|
||||
branch=master
|
||||
|
||||
build_dir=${JENKINS_HOME}/jobs/${JOB_NAME}/builds/${BUILD_NUMBER}/
|
||||
rm -rf ${build_dir}
|
||||
mkdir -p ${build_dir}/workspace
|
||||
|
||||
cd ${build_dir}/workspace
|
||||
|
||||
# Sanity check
|
||||
#gsutil ls ${JENKINS_GCS_LOGS_PATH}
|
||||
|
||||
exit_code=0
|
||||
SECONDS=0 # magic bash timer variable
|
||||
curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh" > /tmp/e2e.sh
|
||||
chmod +x /tmp/e2e.sh
|
||||
|
||||
# We need kubectl to write kubecfg from kops
|
||||
curl -fsS --retry 3 "https://storage.googleapis.com/kubernetes-release/release/v1.3.5/bin/linux/amd64/kubectl" > /usr/local/bin/kubectl
|
||||
chmod +x /usr/local/bin/kubectl
|
||||
|
||||
curl -fsS --retry 3 "https://kubeupv2.s3.amazonaws.com/kops/kops-1.3.tar.gz" > /tmp/kops.tar.gz
|
||||
tar zxf /tmp/kops.tar.gz -C /opt
|
||||
|
||||
if [[ ! -e ${AWS_SSH_KEY} ]]; then
|
||||
echo "Creating ssh key ${AWS_SSH_KEY}"
|
||||
ssh-keygen -N "" -t rsa -f ${AWS_SSH_KEY}
|
||||
fi
|
||||
|
||||
function fetch_tars_from_gcs() {
|
||||
local -r bucket="${1}"
|
||||
local -r build_version="${2}"
|
||||
echo "Pulling binaries from GCS; using server version ${bucket}/${build_version}."
|
||||
gsutil -mq cp \
|
||||
"gs://${KUBE_GCS_RELEASE_BUCKET}/${bucket}/${build_version}/kubernetes.tar.gz" \
|
||||
"gs://${KUBE_GCS_RELEASE_BUCKET}/${bucket}/${build_version}/kubernetes-test.tar.gz" \
|
||||
.
|
||||
}
|
||||
|
||||
function unpack_binaries() {
|
||||
md5sum kubernetes*.tar.gz
|
||||
tar -xzf kubernetes.tar.gz
|
||||
tar -xzf kubernetes-test.tar.gz
|
||||
}
|
||||
|
||||
|
||||
fetch_tars_from_gcs release ${KUBERNETES_VERSION}
|
||||
unpack_binaries
|
||||
|
||||
# Clean up everything when we're done
|
||||
function finish {
|
||||
/opt/kops/kops delete cluster \
|
||||
--name ${JOB_NAME}.${DNS_DOMAIN} \
|
||||
--yes 2>&1 | tee -a ${build_dir}/build-log.txt
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
set -e
|
||||
|
||||
# Create the cluster spec
|
||||
pushd /opt/kops
|
||||
/opt/kops/kops create cluster \
|
||||
--name ${JOB_NAME}.${DNS_DOMAIN} \
|
||||
--cloud ${KUBERNETES_PROVIDER} \
|
||||
--zones ${NODE_ZONES} \
|
||||
--node-size ${NODE_SIZE} \
|
||||
--master-size ${MASTER_SIZE} \
|
||||
--ssh-public-key ${AWS_SSH_KEY}.pub \
|
||||
--kubernetes-version ${KUBERNETES_VERSION} \
|
||||
--v=4 2>&1 | tee -a ${build_dir}/build-log.txt
|
||||
exit_code=${PIPESTATUS[0]}
|
||||
popd
|
||||
|
||||
# Apply the cluster spec
|
||||
if [[ ${exit_code} == 0 ]]; then
|
||||
pushd /opt/kops
|
||||
/opt/kops/kops update cluster \
|
||||
--name ${JOB_NAME}.${DNS_DOMAIN} \
|
||||
--yes --v=4 2>&1 | tee -a ${build_dir}/build-log.txt
|
||||
exit_code=${PIPESTATUS[0]}
|
||||
popd
|
||||
fi
|
||||
|
||||
# Wait for kubectl to begin responding (at least master up)
|
||||
if [[ ${exit_code} == 0 ]]; then
|
||||
attempt=0
|
||||
while true; do
|
||||
kubectl get nodes --show-labels 2>&1 | tee -a ${build_dir}/build-log.txt
|
||||
exit_code=${PIPESTATUS[0]}
|
||||
|
||||
if [[ ${exit_code} == 0 ]]; then
|
||||
break
|
||||
fi
|
||||
if (( attempt > 60 )); then
|
||||
echo "Unable to connect to API in 15 minutes (master did not launch?)"
|
||||
break
|
||||
fi
|
||||
attempt=$(($attempt+1))
|
||||
sleep 15
|
||||
done
|
||||
fi
|
||||
|
||||
# TODO: can we get rid of this?
|
||||
echo "API responded; waiting 450 seconds for DNS to settle"
|
||||
for ((i=1;i<=15;i++)); do
|
||||
kubectl get nodes --show-labels 2>&1 | tee -a ${build_dir}/build-log.txt
|
||||
sleep 30
|
||||
done
|
||||
|
||||
|
||||
# Run e2e tests
|
||||
if [[ ${exit_code} == 0 ]]; then
|
||||
/tmp/e2e.sh 2>&1 | tee -a ${build_dir}/build-log.txt
|
||||
exit_code=${PIPESTATUS[0]}
|
||||
fi
|
||||
|
||||
# Try to clean up normally so it goes into the logs
|
||||
# (we have an exit hook for abnormal termination, but that does not get logged)
|
||||
finish
|
||||
|
||||
duration=$SECONDS
|
||||
set +e
|
||||
|
||||
if [[ ${exit_code} == 0 ]]; then
|
||||
success="true"
|
||||
else
|
||||
success="false"
|
||||
fi
|
||||
|
||||
version=`cat kubernetes/version`
|
||||
|
||||
gcs_acl="public-read"
|
||||
gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}"
|
||||
gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}"
|
||||
|
||||
gsutil -q cp -a "${gcs_acl}" -z txt "${build_dir}/build-log.txt" "${gcs_build_path}/"
|
||||
|
||||
curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash -
|
||||
|
||||
|
||||
curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-finished.sh" > upload-finished.sh
|
||||
chmod +x upload-finished.sh
|
||||
|
||||
if [[ ${exit_code} == 0 ]]; then
|
||||
./upload-finished.sh SUCCESS
|
||||
else
|
||||
./upload-finished.sh UNSTABLE
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue