diff --git a/docs/testing.md b/docs/testing.md index 67524c2062..0d1d4ed46f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -39,4 +39,10 @@ spec: zone: us-east-1c name: main ... -``` \ No newline at end of file +``` + + +### Running the kubernetes e2e test suite + +The [e2e](../e2e/README.md) directory has a docker image and some scripts which make it easy to run +the kubernetes e2e tests, using kops. \ No newline at end of file diff --git a/e2e/Dockerfile b/e2e/Dockerfile new file mode 100644 index 0000000000..e337661224 --- /dev/null +++ b/e2e/Dockerfile @@ -0,0 +1,14 @@ +FROM debian:jessie + +RUN apt-get update && apt-get install --yes curl python-pip openssh-client +RUN pip install awscli + +RUN curl https://sdk.cloud.google.com | bash + +RUN curl https://storage.googleapis.com/golang/go1.6.3.linux-amd64.tar.gz | tar -C /usr/local -xzf - + +ENV PATH /root/google-cloud-sdk/bin:/usr/local/go/bin:$PATH + +ADD runtests.sh / + +ADD conf /conf diff --git a/e2e/Makefile b/e2e/Makefile new file mode 100644 index 0000000000..1cdb13f5a6 --- /dev/null +++ b/e2e/Makefile @@ -0,0 +1,22 @@ +#JOB_NAME=kubernetes-e2e-kops-aws +#KUBERNETES_VERSION=v1.3.5 +#DNS_DOMAIN="mydomain.com" +#JENKINS_GCS_LOGS_PATH=gs://kopeio-kubernetes-e2e/logs +#KOPS_STATE_STORE=s3://mys3bucket + +test: image + docker run -v ${HOME}/.ssh/id_rsa.pub:/root/.ssh/id_rsa.pub \ + -v ${HOME}/.aws:/root/.aws \ + -v ${HOME}/.gsutil:/root/.gsutil \ + -v ${HOME}/.config/gcloud:/root/.config/gcloud \ + -e "DNS_DOMAIN=${DNS_DOMAIN}" \ + -e "JENKINS_GCS_LOGS_PATH=${JENKINS_GCS_LOGS_PATH}" \ + -e "KOPS_STATE_STORE=${KOPS_STATE_STORE}" \ + -e "JOB_NAME=${JOB_NAME}" \ + -e "KUBERNETES_VERSION=${KUBERNETES_VERSION}" \ + kubernetes-e2e-runner-kops \ + /runtests.sh + +image: + docker build -t kubernetes-e2e-runner-kops . + diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000000..6028cf646f --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,18 @@ +## Run kubernetes e2e tests + +This docker image lets you run the kubernetes e2e tests very easily, using kops to create the cluster. + +You simply call make, specifying some variables that controls the build. + +An example: + +`make JOB_NAME=kubernetes-e2e-kops-aws KUBERNETES_VERSION=v1.3.5 DNS_DOMAIN=e2e.mydomain.com JENKINS_GCS_LOGS_PATH=gs://kopeio-kubernetes-e2e/logs KOPS_STATE_STORE=s3://clusters.mydomain.com` + +Variables: + +* `JOB_NAME` the e2e job to run. Corresponds to a conf file in the conf directory. +* `KUBERNETES_VERSION` the version of kubernetes to run. Either a version like `v1.3.5`, or a URL prefix like `https://storage.googleapis.com/kubernetes-release-dev/ci/v1.4.0-alpha.2.677+ea69570f61af8e/`. See [testing docs](../docs/testing.md) +* `DNS_DOMAIN` the dns domain name to use for the cluster. Must be a real domain name, with a zone registered in DNS (route53) +* `JENKINS_GCS_LOGS_PATH` the gs bucket where we should upload the results of the build. Note these will be publicly readable. +* `KOPS_STATE_STORE` the url where the kops registry (store of cluster information) lives. + diff --git a/e2e/conf/cloud/aws b/e2e/conf/cloud/aws new file mode 100644 index 0000000000..6d221141da --- /dev/null +++ b/e2e/conf/cloud/aws @@ -0,0 +1,12 @@ +##============================================================= +# AWS Settings +export AWS_DEFAULT_REGION=${AWS_REGION} +#export KUBE_AWS_ZONE="us-west-2a" +#export PROJECT="k8s-jkns-e2e-aws" +export AWS_CONFIG_FILE=~/.aws/credentials +export AWS_SSH_KEY=~/.ssh/kube_aws_rsa +export KUBE_SSH_USER=admin +# This is needed to be able to create PD from the e2e test +export AWS_SHARED_CREDENTIALS_FILE=~/.aws/credentials + +export AWS_PROFILE=kubernetes-e2e diff --git a/e2e/conf/cloud/gce b/e2e/conf/cloud/gce new file mode 100644 index 0000000000..f793b710a2 --- /dev/null +++ b/e2e/conf/cloud/gce @@ -0,0 +1,5 @@ +# GCE variables +#export INSTANCE_PREFIX="${E2E_NAME:-jenkins-e2e}" +#export KUBE_GCE_NETWORK="${E2E_NAME:-jenkins-e2e}" +#export KUBE_GCE_INSTANCE_PREFIX="${E2E_NAME:-jenkins-e2e}" +#export GCE_SERVICE_ACCOUNT=$(gcloud auth list 2> /dev/null | grep active | cut -f3 -d' ') diff --git a/e2e/conf/kubernetes-e2e-kops-aws b/e2e/conf/kubernetes-e2e-kops-aws new file mode 100644 index 0000000000..90f1e384b2 --- /dev/null +++ b/e2e/conf/kubernetes-e2e-kops-aws @@ -0,0 +1,12 @@ +export KUBERNETES_PROVIDER="aws" + +export E2E_MIN_STARTUP_PODS="1" +export MASTER_SIZE="m3.large" +export NODE_SIZE="m3.large" +export NUM_NODES="3" +export GINKGO_TEST_ARGS="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]" +export GINKGO_PARALLEL="y" + +# Central only has 2 AZs, so we use it for the non-HA test +export NODE_ZONES="eu-central-1a" +export AWS_REGION="eu-central-1" diff --git a/e2e/conf/kubernetes-e2e-kops-aws-ha b/e2e/conf/kubernetes-e2e-kops-aws-ha new file mode 100644 index 0000000000..22a2c3f16d --- /dev/null +++ b/e2e/conf/kubernetes-e2e-kops-aws-ha @@ -0,0 +1,4 @@ +. conf/kubernetes-e2e-upup-aws + +export NODE_ZONES="eu-west-1a,eu-west-1b,eu-west-1c" +export AWS_REGION="eu-west-1" diff --git a/e2e/conf/site b/e2e/conf/site new file mode 100644 index 0000000000..4c8e6af5d3 --- /dev/null +++ b/e2e/conf/site @@ -0,0 +1,2 @@ +# Used to set site-specific settings +# Most of these probably are better in the Makefile \ No newline at end of file diff --git a/e2e/runtests.sh b/e2e/runtests.sh new file mode 100755 index 0000000000..0763083f23 --- /dev/null +++ b/e2e/runtests.sh @@ -0,0 +1,215 @@ +#!/bin/bash -ex + +if [[ -z "${JOB_NAME}" ]]; then + echo "Must specify JOB_NAME env var" + exit 1 +fi +if [[ -z "${KUBERNETES_VERSION}" ]]; then + echo "Must specify KUBERNETES_VERSION env var" + exit 1 +fi +if [[ -z "${DNS_DOMAIN}" ]]; then + echo "Must specify DNS_DOMAIN env var" + exit 1 +fi +if [[ -z "${KOPS_STATE_STORE}" ]]; then + echo "Must specify KOPS_STATE_STORE env var" + exit 1 +fi +# TODO: Maybe skip if we don't want to upload logs? +if [[ -z "${JENKINS_GCS_LOGS_PATH}" ]]; then + echo "Must specify JENKINS_GCS_LOGS_PATH env var" + exit 1 +fi + +echo "JOB_NAME=${JOB_NAME}" +echo "Loading conf/${JOB_NAME}" + +. conf/${JOB_NAME} + +echo "Loading conf/cloud/${KUBERNETES_PROVIDER}" +. conf/cloud/${KUBERNETES_PROVIDER} + +echo "Loading conf/site" +. conf/site + +##============================================================= +# Global settings +export KUBE_GCS_RELEASE_BUCKET=kubernetes-release + +# We download the binaries ourselves +# TODO: No way to tell e2e to use a particular release? +# TODO: Maybe download and then bring up the cluster? +export JENKINS_USE_EXISTING_BINARIES=y + +# This actually just skips kube-up master detection +export KUBERNETES_CONFORMANCE_TEST=y + +##============================================================= +# System settings (emulate jenkins) +export USER=root +export WORKSPACE=$HOME +# Nothing should want Jenkins $HOME +export HOME=${WORKSPACE} +export BUILD_NUMBER=`date -u +%Y%m%d%H%M%S` +export JENKINS_HOME=${HOME} + +# We'll directly up & down the cluster +export E2E_UP="${E2E_UP:-false}" +export E2E_TEST="${E2E_TEST:-true}" +export E2E_DOWN="${E2E_DOWN:-false}" + +# Skip gcloud update checking +export CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK=true + + +##============================================================= + +branch=master + +build_dir=${JENKINS_HOME}/jobs/${JOB_NAME}/builds/${BUILD_NUMBER}/ +rm -rf ${build_dir} +mkdir -p ${build_dir}/workspace + +cd ${build_dir}/workspace + +# Sanity check +#gsutil ls ${JENKINS_GCS_LOGS_PATH} + +exit_code=0 +SECONDS=0 # magic bash timer variable +curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh" > /tmp/e2e.sh +chmod +x /tmp/e2e.sh + +# We need kubectl to write kubecfg from kops +curl -fsS --retry 3 "https://storage.googleapis.com/kubernetes-release/release/v1.3.5/bin/linux/amd64/kubectl" > /usr/local/bin/kubectl +chmod +x /usr/local/bin/kubectl + +curl -fsS --retry 3 "https://kubeupv2.s3.amazonaws.com/kops/kops-1.3.tar.gz" > /tmp/kops.tar.gz +tar zxf /tmp/kops.tar.gz -C /opt + +if [[ ! -e ${AWS_SSH_KEY} ]]; then + echo "Creating ssh key ${AWS_SSH_KEY}" + ssh-keygen -N "" -t rsa -f ${AWS_SSH_KEY} +fi + +function fetch_tars_from_gcs() { + local -r bucket="${1}" + local -r build_version="${2}" + echo "Pulling binaries from GCS; using server version ${bucket}/${build_version}." + gsutil -mq cp \ + "gs://${KUBE_GCS_RELEASE_BUCKET}/${bucket}/${build_version}/kubernetes.tar.gz" \ + "gs://${KUBE_GCS_RELEASE_BUCKET}/${bucket}/${build_version}/kubernetes-test.tar.gz" \ + . +} + +function unpack_binaries() { + md5sum kubernetes*.tar.gz + tar -xzf kubernetes.tar.gz + tar -xzf kubernetes-test.tar.gz +} + + +fetch_tars_from_gcs release ${KUBERNETES_VERSION} +unpack_binaries + +# Clean up everything when we're done +function finish { + /opt/kops/kops delete cluster \ + --name ${JOB_NAME}.${DNS_DOMAIN} \ + --yes 2>&1 | tee -a ${build_dir}/build-log.txt +} +trap finish EXIT + +set -e + +# Create the cluster spec +pushd /opt/kops +/opt/kops/kops create cluster \ + --name ${JOB_NAME}.${DNS_DOMAIN} \ + --cloud ${KUBERNETES_PROVIDER} \ + --zones ${NODE_ZONES} \ + --node-size ${NODE_SIZE} \ + --master-size ${MASTER_SIZE} \ + --ssh-public-key ${AWS_SSH_KEY}.pub \ + --kubernetes-version ${KUBERNETES_VERSION} \ + --v=4 2>&1 | tee -a ${build_dir}/build-log.txt +exit_code=${PIPESTATUS[0]} +popd + +# Apply the cluster spec +if [[ ${exit_code} == 0 ]]; then + pushd /opt/kops + /opt/kops/kops update cluster \ + --name ${JOB_NAME}.${DNS_DOMAIN} \ + --yes --v=4 2>&1 | tee -a ${build_dir}/build-log.txt + exit_code=${PIPESTATUS[0]} + popd +fi + +# Wait for kubectl to begin responding (at least master up) +if [[ ${exit_code} == 0 ]]; then + attempt=0 + while true; do + kubectl get nodes --show-labels 2>&1 | tee -a ${build_dir}/build-log.txt + exit_code=${PIPESTATUS[0]} + + if [[ ${exit_code} == 0 ]]; then + break + fi + if (( attempt > 60 )); then + echo "Unable to connect to API in 15 minutes (master did not launch?)" + break + fi + attempt=$(($attempt+1)) + sleep 15 + done +fi + +# TODO: can we get rid of this? +echo "API responded; waiting 450 seconds for DNS to settle" +for ((i=1;i<=15;i++)); do + kubectl get nodes --show-labels 2>&1 | tee -a ${build_dir}/build-log.txt + sleep 30 +done + + +# Run e2e tests +if [[ ${exit_code} == 0 ]]; then + /tmp/e2e.sh 2>&1 | tee -a ${build_dir}/build-log.txt + exit_code=${PIPESTATUS[0]} +fi + +# Try to clean up normally so it goes into the logs +# (we have an exit hook for abnormal termination, but that does not get logged) +finish + +duration=$SECONDS +set +e + +if [[ ${exit_code} == 0 ]]; then + success="true" +else + success="false" +fi + +version=`cat kubernetes/version` + +gcs_acl="public-read" +gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}" +gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}" + +gsutil -q cp -a "${gcs_acl}" -z txt "${build_dir}/build-log.txt" "${gcs_build_path}/" + +curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash - + + +curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-finished.sh" > upload-finished.sh +chmod +x upload-finished.sh + +if [[ ${exit_code} == 0 ]]; then + ./upload-finished.sh SUCCESS +else + ./upload-finished.sh UNSTABLE +fi +