diff --git a/vitess/README.md b/vitess/README.md new file mode 100644 index 00000000..d2a5bbf1 --- /dev/null +++ b/vitess/README.md @@ -0,0 +1,133 @@ + + + + +WARNING +WARNING +WARNING +WARNING +WARNING + +

PLEASE NOTE: This document applies to the HEAD of the source tree

+ +If you are using a released version of Kubernetes, you should +refer to the docs that go with that version. + + +The latest 1.0.x release of this document can be found +[here](http://releases.k8s.io/release-1.0/examples/vitess/README.md). + +Documentation for other releases can be found at +[releases.k8s.io](http://releases.k8s.io). + +-- + + + + + +## Vitess Example + +This example shows how to run a [Vitess](http://vitess.io) cluster in Kubernetes. +Vitess is a MySQL clustering system developed at YouTube that makes sharding +transparent to the application layer. It also makes scaling MySQL within +Kubernetes as simple as launching more pods. + +The example brings up a database with 2 shards, and then runs a pool of +[sharded guestbook](https://github.com/youtube/vitess/tree/master/examples/kubernetes/guestbook) +pods. The guestbook app was ported from the original +[guestbook](../../examples/guestbook-go/) +example found elsewhere in this tree, modified to use Vitess as the backend. + +For a more detailed, step-by-step explanation of this example setup, see the +[Vitess on Kubernetes](http://vitess.io/getting-started/) guide. + +### Prerequisites + +You'll need to install [Go 1.4+](https://golang.org/doc/install) to build +`vtctlclient`, the command-line admin tool for Vitess. + +We also assume you have a running Kubernetes cluster with `kubectl` pointing to +it by default. See the [Getting Started guides](../../docs/getting-started-guides/) +for how to get to that point. Note that your Kubernetes cluster needs to have +enough resources (CPU+RAM) to schedule all the pods. By default, this example +requires a cluster-wide total of at least 6 virtual CPUs and 10GiB RAM. You can +tune these requirements in the +[resource limits](../../docs/user-guide/compute-resources.md) +section of each YAML file. + +Lastly, you need to open ports 30000 (for the Vitess admin daemon) and 80 (for +the guestbook app) in your firewall. See the +[Services and Firewalls](../../docs/user-guide/services-firewalls.md) +guide for examples of how to do that. + +### Start Vitess + +``` console +./vitess-up.sh +``` + +This will run through the steps to bring up Vitess. At the end, you should see +something like this: + +``` console +**************************** +* Complete! +* Use the following line to make an alias to kvtctl: +* alias kvtctl='$GOPATH/bin/vtctlclient -server 104.197.47.173:30000' +* See the vtctld UI at: http://104.197.47.173:30000 +**************************** +``` + +### Start the Guestbook app + +``` console +./guestbook-up.sh +``` + +The guestbook service is configured with `type: LoadBalancer` to tell Kubernetes +to expose it on an external IP. It may take a minute to set up, but you should +soon see the external IP show up under the internal one like this: + +``` console +$ kubectl get service guestbook +NAME LABELS SELECTOR IP(S) PORT(S) +guestbook name=guestbook 10.67.253.173 80/TCP + 104.197.151.132 +``` + +Visit the external IP in your browser to view the guestbook. Note that in this +modified guestbook, there are multiple pages to demonstrate range-based sharding +in Vitess. Each page number is assigned to one of the shards using a +[consistent hashing](https://en.wikipedia.org/wiki/Consistent_hashing) scheme. + +### Tear down + +``` console +./guestbook-down.sh +./vitess-down.sh +``` + +You may also want to remove any firewall rules you created. + +### Limitations + +Currently this example cluster is not configured to use the built-in +[Backup/Restore](http://vitess.io/user-guide/backup-and-restore.html) feature of +Vitess, because as of +[Vitess v2.0.0-alpha2](https://github.com/youtube/vitess/releases) that feature +requires a network-mounted directory. Usually this system is used to restore +from the latest backup when a pod is moved or added in an existing deployment. +As part of the final Vitess v2.0.0 release, we plan to provide support for +saving backups in a cloud-based blob store (such as Google Cloud Storage or +Amazon S3), which we believe will be better suited to running in Kubernetes. + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/vitess/README.md?pixel)]() + diff --git a/vitess/create_test_table.sql b/vitess/create_test_table.sql new file mode 100644 index 00000000..0a6ef360 --- /dev/null +++ b/vitess/create_test_table.sql @@ -0,0 +1,8 @@ +CREATE TABLE messages ( + page BIGINT(20) UNSIGNED, + time_created_ns BIGINT(20) UNSIGNED, + keyspace_id BIGINT(20) UNSIGNED, + message VARCHAR(10000), + PRIMARY KEY (page, time_created_ns) +) ENGINE=InnoDB + diff --git a/vitess/env.sh b/vitess/env.sh new file mode 100644 index 00000000..710cf343 --- /dev/null +++ b/vitess/env.sh @@ -0,0 +1,55 @@ +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an include file used by the other scripts in this directory. + +# Most clusters will just be accessed with 'kubectl' on $PATH. +# However, some might require a different command. For example, GKE required +# KUBECTL='gcloud beta container kubectl' for a while. Now that most of our +# use cases just need KUBECTL=kubectl, we'll make that the default. +KUBECTL=${KUBECTL:-kubectl} + +# This should match the nodePort in vtctld-service.yaml +VTCTLD_PORT=${VTCTLD_PORT:-30000} + +# Customizable parameters +SHARDS=${SHARDS:-'-80,80-'} +TABLETS_PER_SHARD=${TABLETS_PER_SHARD:-2} +RDONLY_COUNT=${RDONLY_COUNT:-0} +MAX_TASK_WAIT_RETRIES=${MAX_TASK_WAIT_RETRIES:-300} +MAX_VTTABLET_TOPO_WAIT_RETRIES=${MAX_VTTABLET_TOPO_WAIT_RETRIES:-180} +VTTABLET_TEMPLATE=${VTTABLET_TEMPLATE:-'vttablet-pod-template.yaml'} +VTGATE_TEMPLATE=${VTGATE_TEMPLATE:-'vtgate-controller-template.yaml'} +VTGATE_COUNT=${VTGATE_COUNT:-1} +CELLS=${CELLS:-'test'} +ETCD_REPLICAS=3 + +VTGATE_REPLICAS=$VTGATE_COUNT + +# Get the ExternalIP of any node. +get_node_ip() { + $KUBECTL get -o template -t '{{range (index .items 0).status.addresses}}{{if eq .type "ExternalIP"}}{{.address}}{{end}}{{end}}' nodes +} + +# Try to find vtctld address if not provided. +get_vtctld_addr() { + if [ -z "$VTCTLD_ADDR" ]; then + node_ip=$(get_node_ip) + if [ -n "$node_ip" ]; then + VTCTLD_ADDR="$node_ip:$VTCTLD_PORT" + fi + fi + echo "$VTCTLD_ADDR" +} + diff --git a/vitess/etcd-controller-template.yaml b/vitess/etcd-controller-template.yaml new file mode 100644 index 00000000..dcd7980b --- /dev/null +++ b/vitess/etcd-controller-template.yaml @@ -0,0 +1,54 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: etcd-{{cell}} +spec: + replicas: {{replicas}} + template: + metadata: + labels: + component: etcd + cell: {{cell}} + app: vitess + spec: + volumes: + - name: certs + hostPath: {path: /etc/ssl/certs} + containers: + - name: etcd + image: vitess/etcd:v2.0.13-lite + volumeMounts: + - name: certs + readOnly: true + mountPath: /etc/ssl/certs + resources: + limits: + memory: "128Mi" + cpu: "100m" + command: + - bash + - "-c" + - >- + ipaddr=$(hostname -i) + + global_etcd=$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT + + cell="{{cell}}" && + local_etcd_host_var="ETCD_${cell^^}_SERVICE_HOST" && + local_etcd_port_var="ETCD_${cell^^}_SERVICE_PORT" && + local_etcd=${!local_etcd_host_var}:${!local_etcd_port_var} + + if [ "{{cell}}" != "global" ]; then + until etcdctl -C "http://$global_etcd" + set "/vt/cells/{{cell}}" "http://$local_etcd"; do + echo "[$(date)] waiting for global etcd to register cell '{{cell}}'"; + sleep 1; + done; + fi + + etcd -name $HOSTNAME -discovery {{discovery}} + -advertise-client-urls http://$ipaddr:4001 + -initial-advertise-peer-urls http://$ipaddr:7001 + -listen-client-urls http://$ipaddr:4001 + -listen-peer-urls http://$ipaddr:7001 + diff --git a/vitess/etcd-down.sh b/vitess/etcd-down.sh new file mode 100755 index 00000000..b301a469 --- /dev/null +++ b/vitess/etcd-down.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that tears down the etcd servers started by +# etcd-up.sh. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +CELLS=${CELLS:-'test'} +cells=`echo $CELLS | tr ',' ' '` + +# Delete replication controllers +for cell in 'global' $cells; do + echo "Stopping etcd replicationcontroller for $cell cell..." + $KUBECTL stop replicationcontroller etcd-$cell + + echo "Deleting etcd service for $cell cell..." + $KUBECTL delete service etcd-$cell +done + diff --git a/vitess/etcd-service-template.yaml b/vitess/etcd-service-template.yaml new file mode 100644 index 00000000..817c3e13 --- /dev/null +++ b/vitess/etcd-service-template.yaml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + name: etcd-{{cell}} + labels: + component: etcd + cell: {{cell}} + app: vitess +spec: + ports: + - port: 4001 + selector: + component: etcd + cell: {{cell}} + app: vitess + diff --git a/vitess/etcd-up.sh b/vitess/etcd-up.sh new file mode 100755 index 00000000..9a3c6b6e --- /dev/null +++ b/vitess/etcd-up.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates etcd clusters. +# Vitess requires a global cluster, as well as one for each cell. +# +# For automatic discovery, an etcd cluster can be bootstrapped from an +# existing cluster. In this example, we use an externally-run discovery +# service, but you can use your own. See the etcd docs for more: +# https://github.com/coreos/etcd/blob/v2.0.13/Documentation/clustering.md + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +replicas=${ETCD_REPLICAS:-3} + +CELLS=${CELLS:-'test'} +cells=`echo $CELLS | tr ',' ' '` + +for cell in 'global' $cells; do + # Generate a discovery token. + echo "Generating discovery token for $cell cell..." + discovery=$(curl -sL https://discovery.etcd.io/new?size=$replicas) + if [ -z "$discovery" ]; then + echo "Failed to get etcd discovery token for cell '$cell'." + exit 1 + fi + + # Create the client service, which will load-balance across all replicas. + echo "Creating etcd service for $cell cell..." + cat etcd-service-template.yaml | \ + sed -e "s/{{cell}}/$cell/g" | \ + $KUBECTL create -f - + + # Expand template variables + sed_script="" + for var in cell discovery replicas; do + sed_script+="s,{{$var}},${!var},g;" + done + + # Create the replication controller. + echo "Creating etcd replicationcontroller for $cell cell..." + cat etcd-controller-template.yaml | sed -e "$sed_script" | $KUBECTL create -f - +done + diff --git a/vitess/guestbook-controller.yaml b/vitess/guestbook-controller.yaml new file mode 100644 index 00000000..c615b559 --- /dev/null +++ b/vitess/guestbook-controller.yaml @@ -0,0 +1,23 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: guestbook +spec: + replicas: 3 + template: + metadata: + labels: + component: guestbook + app: vitess + spec: + containers: + - name: guestbook + image: vitess/guestbook:v2.0 + ports: + - name: http-server + containerPort: 8080 + resources: + limits: + memory: "128Mi" + cpu: "100m" + diff --git a/vitess/guestbook-down.sh b/vitess/guestbook-down.sh new file mode 100755 index 00000000..57a6978c --- /dev/null +++ b/vitess/guestbook-down.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops guestbook. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +echo "Stopping guestbook replicationcontroller..." +$KUBECTL stop replicationcontroller guestbook + +echo "Deleting guestbook service..." +$KUBECTL delete service guestbook diff --git a/vitess/guestbook-service.yaml b/vitess/guestbook-service.yaml new file mode 100644 index 00000000..5435f7fa --- /dev/null +++ b/vitess/guestbook-service.yaml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + name: guestbook + labels: + component: guestbook + app: vitess +spec: + ports: + - port: 80 + targetPort: http-server + selector: + component: guestbook + app: vitess + type: LoadBalancer + diff --git a/vitess/guestbook-up.sh b/vitess/guestbook-up.sh new file mode 100755 index 00000000..1e21e01b --- /dev/null +++ b/vitess/guestbook-up.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that starts a guestbook replicationcontroller. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +echo "Creating guestbook service..." +$KUBECTL create -f guestbook-service.yaml + +echo "Creating guestbook replicationcontroller..." +$KUBECTL create -f guestbook-controller.yaml diff --git a/vitess/vitess-down.sh b/vitess/vitess-down.sh new file mode 100755 index 00000000..fbda4a1a --- /dev/null +++ b/vitess/vitess-down.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +./vtgate-down.sh +SHARDS=$SHARDS CELLS=$CELLS TABLETS_PER_SHARD=$TABLETS_PER_SHARD ./vttablet-down.sh +./vtctld-down.sh +./etcd-down.sh diff --git a/vitess/vitess-up.sh b/vitess/vitess-up.sh new file mode 100755 index 00000000..888230ba --- /dev/null +++ b/vitess/vitess-up.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a fully functional vitess cluster. +# It performs the following steps: +# - Create etcd clusters +# - Create vtctld pod +# - Create vttablet pods +# - Perform vtctl initialization: +# SetKeyspaceShardingInfo, Rebuild Keyspace, Reparent Shard, Apply Schema +# - Create vtgate pods + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +cells=`echo $CELLS | tr ',' ' '` +num_cells=`echo $cells | wc -w` + +function update_spinner_value () { + spinner='-\|/' + cur_spinner=${spinner:$(($1%${#spinner})):1} +} + +function wait_for_running_tasks () { + # This function waits for pods to be in the "Running" state + # 1. task_name: Name that the desired task begins with + # 2. num_tasks: Number of tasks to wait for + # Returns: + # 0 if successful, -1 if timed out + task_name=$1 + num_tasks=$2 + counter=0 + + echo "Waiting for ${num_tasks}x $task_name to enter state Running" + + while [ $counter -lt $MAX_TASK_WAIT_RETRIES ]; do + # Get status column of pods with name starting with $task_name, + # count how many are in state Running + num_running=`$KUBECTL get pods | grep ^$task_name | grep Running | wc -l` + + echo -en "\r$task_name: $num_running out of $num_tasks in state Running..." + if [ $num_running -eq $num_tasks ] + then + echo Complete + return 0 + fi + update_spinner_value $counter + echo -n $cur_spinner + let counter=counter+1 + sleep 1 + done + echo Timed out + return -1 +} + +if [ -z "$GOPATH" ]; then + echo "ERROR: GOPATH undefined, can't obtain vtctlclient" + exit -1 +fi + +export KUBECTL='kubectl' + +echo "Downloading and installing vtctlclient..." +go get github.com/youtube/vitess/go/cmd/vtctlclient +num_shards=`echo $SHARDS | tr "," " " | wc -w` +total_tablet_count=$(($num_shards*$TABLETS_PER_SHARD*$num_cells)) +vtgate_count=$VTGATE_COUNT +if [ $vtgate_count -eq 0 ]; then + vtgate_count=$(($total_tablet_count/4>3?$total_tablet_count/4:3)) +fi + +echo "****************************" +echo "*Creating vitess cluster:" +echo "* Shards: $SHARDS" +echo "* Tablets per shard: $TABLETS_PER_SHARD" +echo "* Rdonly per shard: $RDONLY_COUNT" +echo "* VTGate count: $vtgate_count" +echo "* Cells: $cells" +echo "****************************" + +echo 'Running etcd-up.sh' && CELLS=$CELLS ./etcd-up.sh +wait_for_running_tasks etcd-global 3 +for cell in $cells; do + wait_for_running_tasks etcd-$cell 3 +done + +echo 'Running vtctld-up.sh' && ./vtctld-up.sh +echo 'Running vttablet-up.sh' && CELLS=$CELLS ./vttablet-up.sh +echo 'Running vtgate-up.sh' && ./vtgate-up.sh + +wait_for_running_tasks vtctld 1 +wait_for_running_tasks vttablet $total_tablet_count +wait_for_running_tasks vtgate $vtgate_count + +vtctld_port=30000 +vtctld_ip=`kubectl get -o yaml nodes | grep 'type: ExternalIP' -B 1 | head -1 | awk '{print $NF}'` +vtctl_server="$vtctld_ip:$vtctld_port" +kvtctl="$GOPATH/bin/vtctlclient -server $vtctl_server" + +echo Waiting for tablets to be visible in the topology +counter=0 +while [ $counter -lt $MAX_VTTABLET_TOPO_WAIT_RETRIES ]; do + num_tablets=0 + for cell in $cells; do + num_tablets=$(($num_tablets+`$kvtctl ListAllTablets $cell | wc -l`)) + done + echo -en "\r$num_tablets out of $total_tablet_count in topology..." + if [ $num_tablets -eq $total_tablet_count ] + then + echo Complete + break + fi + update_spinner_value $counter + echo -n $cur_spinner + let counter=counter+1 + sleep 1 + if [ $counter -eq $MAX_VTTABLET_TOPO_WAIT_RETRIES ] + then + echo Timed out + fi +done + +# split_shard_count = num_shards for sharded keyspace, 0 for unsharded +split_shard_count=$num_shards +if [ $split_shard_count -eq 1 ]; then + split_shard_count=0 +fi + +echo -n Setting Keyspace Sharding Info... +$kvtctl SetKeyspaceShardingInfo -force -split_shard_count $split_shard_count test_keyspace keyspace_id uint64 +echo Done +echo -n Rebuilding Keyspace Graph... +$kvtctl RebuildKeyspaceGraph test_keyspace +echo Done +echo -n Reparenting... +shard_num=1 +for shard in $(echo $SHARDS | tr "," " "); do + $kvtctl InitShardMaster -force test_keyspace/$shard `echo $cells | awk '{print $1}'`-0000000${shard_num}00 + let shard_num=shard_num+1 +done +echo Done +echo -n Applying Schema... +$kvtctl ApplySchema -sql "$(cat create_test_table.sql)" test_keyspace +echo Done + +echo "****************************" +echo "* Complete!" +echo "* Use the following line to make an alias to kvtctl:" +echo "* alias kvtctl='\$GOPATH/bin/vtctlclient -server $vtctl_server'" +echo "* See the vtctld UI at: http://${vtctl_server}" +echo "****************************" diff --git a/vitess/vtctld-controller.yaml b/vitess/vtctld-controller.yaml new file mode 100644 index 00000000..be97f073 --- /dev/null +++ b/vitess/vtctld-controller.yaml @@ -0,0 +1,45 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: vtctld +spec: + replicas: 1 + template: + metadata: + labels: + component: vtctld + app: vitess + spec: + containers: + - name: vtctld + image: vitess/lite:v2.0 + volumeMounts: + - name: syslog + mountPath: /dev/log + - name: vtdataroot + mountPath: /vt/vtdataroot + resources: + limits: + memory: "128Mi" + cpu: "100m" + command: + - sh + - "-c" + - >- + mkdir -p $VTDATAROOT/tmp && + chown -R vitess /vt && + su -p -c "/vt/bin/vtctld + -debug + -templates $VTTOP/go/cmd/vtctld/templates + -web_dir $VTTOP/web/vtctld + -log_dir $VTDATAROOT/tmp + -alsologtostderr + -port 15000 + -topo_implementation etcd + -tablet_protocol grpc + -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT" vitess + volumes: + - name: syslog + hostPath: {path: /dev/log} + - name: vtdataroot + emptyDir: {} diff --git a/vitess/vtctld-down.sh b/vitess/vtctld-down.sh new file mode 100755 index 00000000..0ff1a66c --- /dev/null +++ b/vitess/vtctld-down.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops vtctld. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +echo "Stopping vtctld replicationcontroller..." +$KUBECTL stop replicationcontroller vtctld + +echo "Deleting vtctld service..." +$KUBECTL delete service vtctld diff --git a/vitess/vtctld-service.yaml b/vitess/vtctld-service.yaml new file mode 100644 index 00000000..5c8ba593 --- /dev/null +++ b/vitess/vtctld-service.yaml @@ -0,0 +1,17 @@ +kind: Service +apiVersion: v1 +metadata: + name: vtctld + labels: + component: vtctld + app: vitess +spec: + ports: + - port: 15000 + targetPort: 15000 + nodePort: 30000 + selector: + component: vtctld + app: vitess + type: NodePort + diff --git a/vitess/vtctld-up.sh b/vitess/vtctld-up.sh new file mode 100755 index 00000000..6622d0ca --- /dev/null +++ b/vitess/vtctld-up.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that starts vtctld. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +echo "Creating vtctld service..." +$KUBECTL create -f vtctld-service.yaml + +echo "Creating vtctld replicationcontroller..." +$KUBECTL create -f vtctld-controller.yaml + +server=$(get_vtctld_addr) +echo +echo "vtctld address: http://$server" + diff --git a/vitess/vtgate-controller-template.yaml b/vitess/vtgate-controller-template.yaml new file mode 100644 index 00000000..b33a108c --- /dev/null +++ b/vitess/vtgate-controller-template.yaml @@ -0,0 +1,44 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: vtgate +spec: + replicas: {{replicas}} + template: + metadata: + labels: + component: vtgate + app: vitess + spec: + containers: + - name: vtgate + image: vitess/lite:v2.0 + volumeMounts: + - name: syslog + mountPath: /dev/log + - name: vtdataroot + mountPath: /vt/vtdataroot + resources: + limits: + memory: "512Mi" + cpu: "500m" + command: + - sh + - "-c" + - >- + mkdir -p $VTDATAROOT/tmp && + chown -R vitess /vt && + su -p -c "/vt/bin/vtgate + -topo_implementation etcd + -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT + -log_dir $VTDATAROOT/tmp + -alsologtostderr + -port 15001 + -tablet_protocol grpc + -cell test" vitess + volumes: + - name: syslog + hostPath: {path: /dev/log} + - name: vtdataroot + emptyDir: {} + diff --git a/vitess/vtgate-down.sh b/vitess/vtgate-down.sh new file mode 100755 index 00000000..0ac15d6c --- /dev/null +++ b/vitess/vtgate-down.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops vtgate. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +echo "Stopping vtgate replicationcontroller..." +$KUBECTL stop replicationcontroller vtgate + +echo "Deleting vtgate service..." +$KUBECTL delete service vtgate diff --git a/vitess/vtgate-service.yaml b/vitess/vtgate-service.yaml new file mode 100644 index 00000000..192968aa --- /dev/null +++ b/vitess/vtgate-service.yaml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: vtgate + labels: + component: vtgate + app: vitess +spec: + ports: + - port: 15001 + selector: + component: vtgate + app: vitess + type: LoadBalancer + diff --git a/vitess/vtgate-up.sh b/vitess/vtgate-up.sh new file mode 100755 index 00000000..556aa35f --- /dev/null +++ b/vitess/vtgate-up.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that starts a vtgate replicationcontroller. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +VTGATE_REPLICAS=${VTGATE_REPLICAS:-3} +VTGATE_TEMPLATE=${VTGATE_TEMPLATE:-'vtgate-controller-template.yaml'} + +replicas=$VTGATE_REPLICAS + +echo "Creating vtgate service..." +$KUBECTL create -f vtgate-service.yaml + +sed_script="" +for var in replicas; do + sed_script+="s,{{$var}},${!var},g;" +done + +echo "Creating vtgate replicationcontroller..." +cat $VTGATE_TEMPLATE | sed -e "$sed_script" | $KUBECTL create -f - diff --git a/vitess/vttablet-down.sh b/vitess/vttablet-down.sh new file mode 100755 index 00000000..9ce3d33b --- /dev/null +++ b/vitess/vttablet-down.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that tears down the vttablet pods started by +# vttablet-up.sh. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +server=$(get_vtctld_addr) + +# Delete the pods for all shards +CELLS=${CELLS:-'test'} +keyspace='test_keyspace' +SHARDS=${SHARDS:-'0'} +TABLETS_PER_SHARD=${TABLETS_PER_SHARD:-5} +UID_BASE=${UID_BASE:-100} + +num_shards=`echo $SHARDS | tr "," " " | wc -w` +uid_base=$UID_BASE + +for shard in `seq 1 $num_shards`; do + cell_index=0 + for cell in `echo $CELLS | tr "," " "`; do + for uid_index in `seq 0 $(($TABLETS_PER_SHARD-1))`; do + uid=$[$uid_base + $uid_index + $cell_index] + printf -v alias '%s-%010d' $cell $uid + + if [ -n "$server" ]; then + echo "Removing tablet $alias from Vitess topology..." + vtctlclient -server $server ScrapTablet -force $alias + vtctlclient -server $server DeleteTablet $alias + fi + + echo "Deleting pod for tablet $alias..." + $KUBECTL delete pod vttablet-$uid + done + let cell_index=cell_index+100000000 + done + let uid_base=uid_base+100 +done diff --git a/vitess/vttablet-pod-template.yaml b/vitess/vttablet-pod-template.yaml new file mode 100644 index 00000000..00b71f17 --- /dev/null +++ b/vitess/vttablet-pod-template.yaml @@ -0,0 +1,120 @@ +kind: Pod +apiVersion: v1 +metadata: + name: vttablet-{{uid}} + labels: + component: vttablet + keyspace: "{{keyspace}}" + shard: "{{shard_label}}" + tablet: "{{alias}}" + app: vitess +spec: + containers: + - name: vttablet + image: vitess/lite:v2.0 + volumeMounts: + - name: syslog + mountPath: /dev/log + - name: vtdataroot + mountPath: /vt/vtdataroot + resources: + limits: + memory: "1Gi" + cpu: "500m" + command: + - bash + - "-c" + - >- + set -e + + mysql_socket="$VTDATAROOT/{{tablet_subdir}}/mysql.sock" + + mkdir -p $VTDATAROOT/tmp + + chown -R vitess /vt + + while [ ! -e $mysql_socket ]; do + echo "[$(date)] waiting for $mysql_socket" ; + sleep 1 ; + done + + su -p -s /bin/bash -c "mysql -u vt_dba -S $mysql_socket + -e 'CREATE DATABASE IF NOT EXISTS vt_{{keyspace}}'" vitess + + su -p -s /bin/bash -c "/vt/bin/vttablet + -topo_implementation etcd + -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT + -log_dir $VTDATAROOT/tmp + -alsologtostderr + -port {{port}} + -grpc_port {{grpc_port}} + -service_map 'grpc-queryservice' + -tablet-path {{alias}} + -tablet_hostname $(hostname -i) + -init_keyspace {{keyspace}} + -init_shard {{shard}} + -target_tablet_type {{tablet_type}} + -mysqlctl_socket $VTDATAROOT/mysqlctl.sock + -db-config-app-uname vt_app + -db-config-app-dbname vt_{{keyspace}} + -db-config-app-charset utf8 + -db-config-dba-uname vt_dba + -db-config-dba-dbname vt_{{keyspace}} + -db-config-dba-charset utf8 + -db-config-repl-uname vt_repl + -db-config-repl-dbname vt_{{keyspace}} + -db-config-repl-charset utf8 + -db-config-filtered-uname vt_filtered + -db-config-filtered-dbname vt_{{keyspace}} + -db-config-filtered-charset utf8 + -enable-rowcache + -rowcache-bin /usr/bin/memcached + -rowcache-socket $VTDATAROOT/{{tablet_subdir}}/memcache.sock" vitess + - name: mysql + image: vitess/lite:v2.0 + volumeMounts: + - name: syslog + mountPath: /dev/log + - name: vtdataroot + mountPath: /vt/vtdataroot + resources: + limits: + memory: "1Gi" + cpu: "500m" + command: + - sh + - "-c" + - >- + mkdir -p $VTDATAROOT/tmp && + chown -R vitess /vt + + su -p -c "/vt/bin/mysqlctld + -log_dir $VTDATAROOT/tmp + -alsologtostderr + -tablet_uid {{uid}} + -socket_file $VTDATAROOT/mysqlctl.sock + -db-config-app-uname vt_app + -db-config-app-dbname vt_{{keyspace}} + -db-config-app-charset utf8 + -db-config-dba-uname vt_dba + -db-config-dba-dbname vt_{{keyspace}} + -db-config-dba-charset utf8 + -db-config-repl-uname vt_repl + -db-config-repl-dbname vt_{{keyspace}} + -db-config-repl-charset utf8 + -db-config-filtered-uname vt_filtered + -db-config-filtered-dbname vt_{{keyspace}} + -db-config-filtered-charset utf8 + -bootstrap_archive mysql-db-dir_10.0.13-MariaDB.tbz" vitess + # The bootstrap archive above contains an empty mysql data dir + # with user permissions set up as required by Vitess. The archive is + # included in the Docker image. + env: + - name: EXTRA_MY_CNF + value: /vt/config/mycnf/master_mariadb.cnf + volumes: + - name: syslog + hostPath: {path: /dev/log} + - name: vtdataroot + emptyDir: {} + diff --git a/vitess/vttablet-up.sh b/vitess/vttablet-up.sh new file mode 100755 index 00000000..8b4bff61 --- /dev/null +++ b/vitess/vttablet-up.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a vttablet deployment. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +# Create the pods for shard-0 +CELLS=${CELLS:-'test'} +keyspace='test_keyspace' +SHARDS=${SHARDS:-'0'} +TABLETS_PER_SHARD=${TABLETS_PER_SHARD:-5} +port=15002 +grpc_port=16002 +UID_BASE=${UID_BASE:-100} +VTTABLET_TEMPLATE=${VTTABLET_TEMPLATE:-'vttablet-pod-template.yaml'} +RDONLY_COUNT=${RDONLY_COUNT:-2} + +uid_base=$UID_BASE +for shard in $(echo $SHARDS | tr "," " "); do + cell_index=0 + for cell in `echo $CELLS | tr ',' ' '`; do + echo "Creating $keyspace.shard-$shard pods in cell $CELL..." + for uid_index in `seq 0 $(($TABLETS_PER_SHARD-1))`; do + uid=$[$uid_base + $uid_index + $cell_index] + printf -v alias '%s-%010d' $cell $uid + printf -v tablet_subdir 'vt_%010d' $uid + + echo "Creating pod for tablet $alias..." + + # Add xx to beginning or end if there is a dash. K8s does not allow for + # leading or trailing dashes for labels + shard_label=`echo $shard | sed s'/[-]$/-xx/' | sed s'/^-/xx-/'` + + tablet_type=replica + if [ $uid_index -gt $(($TABLETS_PER_SHARD-$RDONLY_COUNT-1)) ]; then + tablet_type=rdonly + fi + + # Expand template variables + sed_script="" + for var in alias cell uid keyspace shard shard_label port grpc_port tablet_subdir tablet_type; do + sed_script+="s,{{$var}},${!var},g;" + done + + # Instantiate template and send to kubectl. + cat $VTTABLET_TEMPLATE | sed -e "$sed_script" | $KUBECTL create -f - + done + let cell_index=cell_index+100000000 + done + let uid_base=uid_base+100 +done