diff --git a/examples/vitess/README.md b/examples/vitess/README.md index d2a5bbf1388..1794510d081 100644 --- a/examples/vitess/README.md +++ b/examples/vitess/README.md @@ -61,11 +61,36 @@ tune these requirements in the [resource limits](../../docs/user-guide/compute-resources.md) section of each YAML file. -Lastly, you need to open ports 30000 (for the Vitess admin daemon) and 80 (for +Lastly, you need to open ports 30000-30001 (for the Vitess admin daemon) and 80 (for the guestbook app) in your firewall. See the [Services and Firewalls](../../docs/user-guide/services-firewalls.md) guide for examples of how to do that. +### Configure site-local settings + +Run the `configure.sh` script to generate a `config.sh` file, which will be used +to customize your cluster settings. + +``` console +./configure.sh +``` + +Currently, we have out-of-the-box support for storing +[backups](http://vitess.io/user-guide/backup-and-restore.html) in +[Google Cloud Storage](https://cloud.google.com/storage/). +If you're using GCS, fill in the fields requested by the configure script. +Note that your Kubernetes cluster must be running on instances with the +`storage-rw` scope for this to work. With Container Engine, you can do this by +passing `--scopes storage-rw` to the `glcoud container clusters create` command. + +For other platforms, you'll need to choose the `file` backup storage plugin, +and mount a read-write network volume into the `vttablet` and `vtctld` pods. +For example, you can mount any storage service accessible through NFS into a +Kubernetes volume. Then provide the mount path to the configure script here. + +If you prefer to skip setting up a backup volume for the purpose of this example, +you can choose `file` mode and set the path to `/tmp`. + ### Start Vitess ``` console @@ -79,7 +104,7 @@ something like this: **************************** * Complete! * Use the following line to make an alias to kvtctl: -* alias kvtctl='$GOPATH/bin/vtctlclient -server 104.197.47.173:30000' +* alias kvtctl='$GOPATH/bin/vtctlclient -server 104.197.47.173:30001' * See the vtctld UI at: http://104.197.47.173:30000 **************************** ``` @@ -115,18 +140,6 @@ in Vitess. Each page number is assigned to one of the shards using a You may also want to remove any firewall rules you created. -### Limitations - -Currently this example cluster is not configured to use the built-in -[Backup/Restore](http://vitess.io/user-guide/backup-and-restore.html) feature of -Vitess, because as of -[Vitess v2.0.0-alpha2](https://github.com/youtube/vitess/releases) that feature -requires a network-mounted directory. Usually this system is used to restore -from the latest backup when a pod is moved or added in an existing deployment. -As part of the final Vitess v2.0.0 release, we plan to provide support for -saving backups in a cloud-based blob store (such as Google Cloud Storage or -Amazon S3), which we believe will be better suited to running in Kubernetes. - [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/vitess/README.md?pixel)]() diff --git a/examples/vitess/configure.sh b/examples/vitess/configure.sh new file mode 100755 index 00000000000..6391b0db3d3 --- /dev/null +++ b/examples/vitess/configure.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script generates config.sh, which is a site-local config file that is not +# checked into source control. + +# Select and configure Backup Storage Implementation. +storage=gcs +read -p "Backup Storage (file, gcs) [gcs]: " +if [ -n "$REPLY" ]; then storage="$REPLY"; fi + +case "$storage" in +gcs) + # Google Cloud Storage + project=$(gcloud config list project | grep 'project\s*=' | sed -r 's/^.*=\s*(.*)$/\1/') + read -p "Google Developers Console Project [$project]: " + if [ -n "$REPLY" ]; then project="$REPLY"; fi + if [ -z "$project" ]; then + echo "ERROR: Project name must not be empty." + exit 1 + fi + + read -p "Google Cloud Storage bucket for Vitess backups: " bucket + if [ -z "$bucket" ]; then + echo "ERROR: Bucket name must not be empty." + exit 1 + fi + echo + echo "NOTE: If you haven't already created this bucket, you can do so by running:" + echo " gsutil mb gs://$bucket" + echo + + backup_flags=$(echo -backup_storage_implementation gcs \ + -gcs_backup_storage_project "'$project'" \ + -gcs_backup_storage_bucket "'$bucket'") + ;; +file) + # Mounted volume (e.g. NFS) + read -p "Root directory for backups (usually an NFS mount): " file_root + if [ -z "$file_root" ]; then + echo "ERROR: Root directory must not be empty." + exit 1 + fi + echo + echo "NOTE: You must add your NFS mount to the vtctld-controller-template" + echo " and vttablet-pod-template as described in the Kubernetes docs:" + echo " http://kubernetes.io/v1.0/docs/user-guide/volumes.html#nfs" + echo + + backup_flags=$(echo -backup_storage_implementation file \ + -file_backup_storage_root "'$file_root'") + ;; +*) + echo "ERROR: Unsupported backup storage implementation: $storage" + exit 1 +esac + +echo "Saving config.sh..." +echo "backup_flags=\"$backup_flags\"" > config.sh + diff --git a/examples/vitess/env.sh b/examples/vitess/env.sh index 710cf343ea1..efb276a0c15 100644 --- a/examples/vitess/env.sh +++ b/examples/vitess/env.sh @@ -21,7 +21,7 @@ KUBECTL=${KUBECTL:-kubectl} # This should match the nodePort in vtctld-service.yaml -VTCTLD_PORT=${VTCTLD_PORT:-30000} +VTCTLD_PORT=${VTCTLD_PORT:-30001} # Customizable parameters SHARDS=${SHARDS:-'-80,80-'} @@ -53,3 +53,11 @@ get_vtctld_addr() { echo "$VTCTLD_ADDR" } +config_file=`dirname "${BASH_SOURCE}"`/config.sh +if [ ! -f $config_file ]; then + echo "Please run ./configure.sh first to generate config.sh file." + exit 1 +fi + +source $config_file + diff --git a/examples/vitess/guestbook-controller.yaml b/examples/vitess/guestbook-controller.yaml index cece7e232d8..1c5ca5a18b8 100644 --- a/examples/vitess/guestbook-controller.yaml +++ b/examples/vitess/guestbook-controller.yaml @@ -12,7 +12,7 @@ spec: spec: containers: - name: guestbook - image: vitess/guestbook:v2.0.0-alpha3 + image: vitess/guestbook:v2.0.0-alpha5 ports: - name: http-server containerPort: 8080 diff --git a/examples/vitess/vitess-up.sh b/examples/vitess/vitess-up.sh index 888230ba354..b38d7567dcb 100755 --- a/examples/vitess/vitess-up.sh +++ b/examples/vitess/vitess-up.sh @@ -74,7 +74,7 @@ fi export KUBECTL='kubectl' echo "Downloading and installing vtctlclient..." -go get github.com/youtube/vitess/go/cmd/vtctlclient +go get -u github.com/youtube/vitess/go/cmd/vtctlclient num_shards=`echo $SHARDS | tr "," " " | wc -w` total_tablet_count=$(($num_shards*$TABLETS_PER_SHARD*$num_cells)) vtgate_count=$VTGATE_COUNT @@ -105,7 +105,7 @@ wait_for_running_tasks vtctld 1 wait_for_running_tasks vttablet $total_tablet_count wait_for_running_tasks vtgate $vtgate_count -vtctld_port=30000 +vtctld_port=30001 vtctld_ip=`kubectl get -o yaml nodes | grep 'type: ExternalIP' -B 1 | head -1 | awk '{print $NF}'` vtctl_server="$vtctld_ip:$vtctld_port" kvtctl="$GOPATH/bin/vtctlclient -server $vtctl_server" @@ -160,5 +160,6 @@ echo "****************************" echo "* Complete!" echo "* Use the following line to make an alias to kvtctl:" echo "* alias kvtctl='\$GOPATH/bin/vtctlclient -server $vtctl_server'" -echo "* See the vtctld UI at: http://${vtctl_server}" +echo "* See the vtctld UI at: http://${vtctld_ip}:30000" echo "****************************" + diff --git a/examples/vitess/vtctld-controller.yaml b/examples/vitess/vtctld-controller-template.yaml similarity index 77% rename from examples/vitess/vtctld-controller.yaml rename to examples/vitess/vtctld-controller-template.yaml index 0f86613725a..72fe245a212 100644 --- a/examples/vitess/vtctld-controller.yaml +++ b/examples/vitess/vtctld-controller-template.yaml @@ -12,12 +12,15 @@ spec: spec: containers: - name: vtctld - image: vitess/lite:v2.0.0-alpha3 + image: vitess/lite:v2.0.0-alpha5 volumeMounts: - name: syslog mountPath: /dev/log - name: vtdataroot mountPath: /vt/vtdataroot + - name: certs + readOnly: true + mountPath: /etc/ssl/certs resources: limits: memory: "128Mi" @@ -35,13 +38,18 @@ spec: -log_dir $VTDATAROOT/tmp -alsologtostderr -port 15000 - -service_map 'bsonrpc-vt-vtctl' + -grpc_port 15001 + -service_map 'grpc-vtctl' -topo_implementation etcd -tablet_protocol grpc -tablet_manager_protocol grpc - -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT" vitess + -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT + {{backup_flags}}" vitess volumes: - name: syslog hostPath: {path: /dev/log} - name: vtdataroot emptyDir: {} + - name: certs + hostPath: {path: /etc/ssl/certs} + diff --git a/examples/vitess/vtctld-service.yaml b/examples/vitess/vtctld-service.yaml index 5c8ba5935cf..70d619a873d 100644 --- a/examples/vitess/vtctld-service.yaml +++ b/examples/vitess/vtctld-service.yaml @@ -8,8 +8,13 @@ metadata: spec: ports: - port: 15000 + name: web targetPort: 15000 nodePort: 30000 + - port: 15001 + name: grpc + targetPort: 15001 + nodePort: 30001 selector: component: vtctld app: vitess diff --git a/examples/vitess/vtctld-up.sh b/examples/vitess/vtctld-up.sh index 6622d0cae2b..dd2a63b70de 100755 --- a/examples/vitess/vtctld-up.sh +++ b/examples/vitess/vtctld-up.sh @@ -25,7 +25,14 @@ echo "Creating vtctld service..." $KUBECTL create -f vtctld-service.yaml echo "Creating vtctld replicationcontroller..." -$KUBECTL create -f vtctld-controller.yaml +# Expand template variables +sed_script="" +for var in backup_flags; do + sed_script+="s,{{$var}},${!var},g;" +done + +# Instantiate template and send to kubectl. +cat vtctld-controller-template.yaml | sed -e "$sed_script" | $KUBECTL create -f - server=$(get_vtctld_addr) echo diff --git a/examples/vitess/vtgate-controller-template.yaml b/examples/vitess/vtgate-controller-template.yaml index 98f18d3e135..03c9665b2df 100644 --- a/examples/vitess/vtgate-controller-template.yaml +++ b/examples/vitess/vtgate-controller-template.yaml @@ -12,7 +12,7 @@ spec: spec: containers: - name: vtgate - image: vitess/lite:v2.0.0-alpha3 + image: vitess/lite:v2.0.0-alpha5 volumeMounts: - name: syslog mountPath: /dev/log diff --git a/examples/vitess/vttablet-down.sh b/examples/vitess/vttablet-down.sh index 9ce3d33b7d5..af739f277c1 100755 --- a/examples/vitess/vttablet-down.sh +++ b/examples/vitess/vttablet-down.sh @@ -41,12 +41,6 @@ for shard in `seq 1 $num_shards`; do uid=$[$uid_base + $uid_index + $cell_index] printf -v alias '%s-%010d' $cell $uid - if [ -n "$server" ]; then - echo "Removing tablet $alias from Vitess topology..." - vtctlclient -server $server ScrapTablet -force $alias - vtctlclient -server $server DeleteTablet $alias - fi - echo "Deleting pod for tablet $alias..." $KUBECTL delete pod vttablet-$uid done @@ -54,3 +48,4 @@ for shard in `seq 1 $num_shards`; do done let uid_base=uid_base+100 done + diff --git a/examples/vitess/vttablet-pod-template.yaml b/examples/vitess/vttablet-pod-template.yaml index bca3050e454..d3d097e8fcf 100644 --- a/examples/vitess/vttablet-pod-template.yaml +++ b/examples/vitess/vttablet-pod-template.yaml @@ -11,12 +11,15 @@ metadata: spec: containers: - name: vttablet - image: vitess/lite:v2.0.0-alpha3 + image: vitess/lite:v2.0.0-alpha5 volumeMounts: - name: syslog mountPath: /dev/log - name: vtdataroot mountPath: /vt/vtdataroot + - name: certs + readOnly: true + mountPath: /etc/ssl/certs resources: limits: memory: "1Gi" @@ -70,9 +73,11 @@ spec: -db-config-filtered-charset utf8 -enable-rowcache -rowcache-bin /usr/bin/memcached - -rowcache-socket $VTDATAROOT/{{tablet_subdir}}/memcache.sock" vitess + -rowcache-socket $VTDATAROOT/{{tablet_subdir}}/memcache.sock + -health_check_interval 5s + -restore_from_backup {{backup_flags}}" vitess - name: mysql - image: vitess/lite:v2.0.0-alpha3 + image: vitess/lite:v2.0.0-alpha5 volumeMounts: - name: syslog mountPath: /dev/log @@ -118,4 +123,6 @@ spec: hostPath: {path: /dev/log} - name: vtdataroot emptyDir: {} + - name: certs + hostPath: {path: /etc/ssl/certs} diff --git a/examples/vitess/vttablet-up.sh b/examples/vitess/vttablet-up.sh index 8b4bff61873..98556c27c8e 100755 --- a/examples/vitess/vttablet-up.sh +++ b/examples/vitess/vttablet-up.sh @@ -55,7 +55,7 @@ for shard in $(echo $SHARDS | tr "," " "); do # Expand template variables sed_script="" - for var in alias cell uid keyspace shard shard_label port grpc_port tablet_subdir tablet_type; do + for var in alias cell uid keyspace shard shard_label port grpc_port tablet_subdir tablet_type backup_flags; do sed_script+="s,{{$var}},${!var},g;" done