diff --git a/examples/storage/cassandra/README.md b/examples/storage/cassandra/README.md index be6d2908d46..e39f1d3d88d 100644 --- a/examples/storage/cassandra/README.md +++ b/examples/storage/cassandra/README.md @@ -47,7 +47,7 @@ computer. ## Cassandra Docker -The pods use the [```gcr.io/google-samples/cassandra:v11```](image/Dockerfile) +The pods use the [```gcr.io/google-samples/cassandra:v12```](image/Dockerfile) image from Google's [container registry](https://cloud.google.com/container-registry/docs/). The docker is based on `debian:jessie` and includes OpenJDK 8. This image includes a standard Cassandra installation from the Apache Debian repo. Through the use of environment variables you are able to change values that are inserted into the `cassandra.yaml`. @@ -167,15 +167,18 @@ If an error is returned the service create failed. ## Step 2: Use a StatefulSet to create Cassandra Ring -StatefulSets (previously PetSets) are a new feature that was added as an Alpha component in -Kubernetes 1.3. Deploying stateful distributed applications, like Cassandra, within a clustered +StatefulSets (previously PetSets) are a feature that was upgraded to a Beta component in +Kubernetes 1.5. Deploying stateful distributed applications, like Cassandra, within a clustered environment can be challenging. We implemented StatefulSet to greatly simplify this process. Multiple StatefulSet features are used within this example, but is out of -scope of this documentation. [Please refer to the PetSet documentation.](http://kubernetes.io/docs/user-guide/petset/) +scope of this documentation. [Please refer to the Stateful Set documentation.](https://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/) The StatefulSet manifest that is included below, creates a Cassandra ring that consists of three pods. +This example includes using a GCE Storage Class, please update appropriately depending +on the cloud you are working with. + ```yaml @@ -193,7 +196,7 @@ spec: spec: containers: - name: cassandra - image: gcr.io/google-samples/cassandra:v11 + image: gcr.io/google-samples/cassandra:v12 imagePullPolicy: Always ports: - containerPort: 7000 @@ -215,6 +218,10 @@ spec: capabilities: add: - IPC_LOCK + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "PID=$(pidof java) && kill $PID && while ps -p $PID > /dev/null; do sleep 1; done"] env: - name: MAX_HEAP_SIZE value: 512M @@ -255,12 +262,20 @@ spec: - metadata: name: cassandra-data annotations: - volume.alpha.kubernetes.io/storage-class: anything + volume.beta.kubernetes.io/storage-class: fast spec: accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 1Gi +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: fast +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-ssd ``` [Download example](cassandra-statefulset.yaml?raw=true) @@ -466,7 +481,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: gcr.io/google-samples/cassandra:v11 + image: gcr.io/google-samples/cassandra:v12 name: cassandra ports: - containerPort: 7000 @@ -515,7 +530,7 @@ You can list the new controller: $ kubectl get rc -o wide NAME DESIRED CURRENT AGE CONTAINER(S) IMAGE(S) SELECTOR -cassandra 2 2 11s cassandra gcr.io/google-samples/cassandra:v11 app=cassandra +cassandra 2 2 11s cassandra gcr.io/google-samples/cassandra:v12 app=cassandra ``` @@ -692,7 +707,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: gcr.io/google-samples/cassandra:v11 + image: gcr.io/google-samples/cassandra:v12 name: cassandra ports: - containerPort: 7000 diff --git a/examples/storage/cassandra/cassandra-controller.yaml b/examples/storage/cassandra/cassandra-controller.yaml index bd351742edc..f2344df12af 100644 --- a/examples/storage/cassandra/cassandra-controller.yaml +++ b/examples/storage/cassandra/cassandra-controller.yaml @@ -38,7 +38,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: gcr.io/google-samples/cassandra:v11 + image: gcr.io/google-samples/cassandra:v12 name: cassandra ports: - containerPort: 7000 diff --git a/examples/storage/cassandra/cassandra-daemonset.yaml b/examples/storage/cassandra/cassandra-daemonset.yaml index 1b3d6bf987d..8305ebf88bd 100644 --- a/examples/storage/cassandra/cassandra-daemonset.yaml +++ b/examples/storage/cassandra/cassandra-daemonset.yaml @@ -31,7 +31,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: gcr.io/google-samples/cassandra:v11 + image: gcr.io/google-samples/cassandra:v12 name: cassandra ports: - containerPort: 7000 diff --git a/examples/storage/cassandra/cassandra-statefulset.yaml b/examples/storage/cassandra/cassandra-statefulset.yaml index 489094a2765..4a1981efaba 100644 --- a/examples/storage/cassandra/cassandra-statefulset.yaml +++ b/examples/storage/cassandra/cassandra-statefulset.yaml @@ -12,7 +12,7 @@ spec: spec: containers: - name: cassandra - image: gcr.io/google-samples/cassandra:v11 + image: gcr.io/google-samples/cassandra:v12 imagePullPolicy: Always ports: - containerPort: 7000 @@ -78,9 +78,17 @@ spec: - metadata: name: cassandra-data annotations: - volume.alpha.kubernetes.io/storage-class: anything + volume.beta.kubernetes.io/storage-class: fast spec: accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 1Gi +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: fast +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-ssd diff --git a/examples/storage/cassandra/image/Dockerfile b/examples/storage/cassandra/image/Dockerfile index 248db00d7e5..d14401b8f76 100644 --- a/examples/storage/cassandra/image/Dockerfile +++ b/examples/storage/cassandra/image/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2016 The Kubernetes Authors. +# Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,67 +12,119 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM google/debian:jessie +FROM gcr.io/google_containers/ubuntu-slim:0.6 + +ARG BUILD_DATE +ARG VCS_REF +ARG CASSANDRA_VERSION +ARG DEV_CONTAINER + +LABEL \ + org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.docker.dockerfile="/Dockerfile" \ + org.label-schema.license="Apache License 2.0" \ + org.label-schema.name="k8s-for-greeks/docker-cassandra-k8s" \ + org.label-schema.url="https://github.com/k8s-for-greeks/" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-type="Git" \ + org.label-schema.vcs-url="https://github.com/k8s-for-greeks/docker-cassandra-k8s" + +ENV CASSANDRA_HOME=/usr/local/apache-cassandra-${CASSANDRA_VERSION} \ + CASSANDRA_CONF=/etc/cassandra \ + CASSANDRA_DATA=/cassandra_data \ + CASSANDRA_LOGS=/var/log/cassandra \ + JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ + PATH=${PATH}:/usr/lib/jvm/java-8-openjdk-amd64/bin:/usr/local/apache-cassandra-${CASSANDRA_VERSION}/bin \ + DI_VERSION=1.2.0 \ + DI_SHA=81231da1cd074fdc81af62789fead8641ef3f24b6b07366a1c34e5b059faf363 ADD files / -ENV DI_VERSION="1.1.1" DI_SHA="dec8167091671df0dd3748a8938102479db5fffc" - -RUN mv /java.list /etc/apt/sources.list.d/java.list \ - && apt-get update \ - && apt-get -qq -y --force-yes install --no-install-recommends procps openjdk-8-jre-headless libjemalloc1 curl localepurge \ - && curl -L https://github.com/Yelp/dumb-init/releases/download/v${DI_VERSION}/dumb-init_${DI_VERSION}_amd64 > /sbin/dumb-init \ - && echo "$DI_SHA /sbin/dumb-init" | sha1sum -c - \ - && mv /cassandra.list /etc/apt/sources.list.d/cassandra.list \ - && gpg --keyserver pgp.mit.edu --recv-keys F758CE318D77295D \ - && gpg --export --armor F758CE318D77295D | apt-key add - \ - && gpg --keyserver pgp.mit.edu --recv-keys 2B5C1B00 \ - && gpg --export --armor 2B5C1B00 | apt-key add - \ - && gpg --keyserver pgp.mit.edu --recv-keys 0353B12C \ - && gpg --export --armor 0353B12C | apt-key add - \ - && gpg --keyserver pool.sks-keyservers.net --recv-keys A278B781FE4B2BDA \ - && gpg --export --armor A278B781FE4B2BDA | apt-key add - \ - && apt-get update \ - && apt-get -qq -y --force-yes install --no-install-recommends curl cassandra localepurge \ - && chmod a+rx /run.sh /sbin/dumb-init /ready-probe.sh \ - && mkdir -p /cassandra_data/data \ - && mv /logback.xml /cassandra.yaml /jvm.options /etc/cassandra/ \ - - # Not able to run as cassandra until https://github.com/kubernetes/kubernetes/issues/2630 is resolved - # && chown -R cassandra: /etc/cassandra /cassandra_data /run.sh /kubernetes-cassandra.jar \ - # && chmod o+w -R /etc/cassandra /cassandra_data \ - - && apt-get -y purge curl localepurge \ - && apt-get clean \ - && rm -rf \ - doc \ - man \ - info \ - locale \ +RUN set -e && echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \ + && apt-get update && apt-get -qq -y --force-yes install --no-install-recommends \ + openjdk-8-jre-headless \ + libjemalloc1 \ + localepurge \ + wget && \ + mirror_url=$( wget -q -O - http://www.apache.org/dyn/closer.cgi/cassandra/ \ + | sed -n 's#.*href="\(http://ftp.[^"]*\)".*#\1#p' \ + | head -n 1 \ + ) \ + && wget -q -O - ${mirror_url}/${CASSANDRA_VERSION}/apache-cassandra-${CASSANDRA_VERSION}-bin.tar.gz \ + | tar -xzf - -C /usr/local \ + && wget -q -O - https://github.com/Yelp/dumb-init/releases/download/v${DI_VERSION}/dumb-init_${DI_VERSION}_amd64 > /sbin/dumb-init \ + && echo "$DI_SHA /sbin/dumb-init" | sha256sum -c - \ + && chmod +x /sbin/dumb-init \ + && chmod +x /ready-probe.sh \ + && mkdir -p /cassandra_data/data \ + && mkdir -p /etc/cassandra \ + && mv /logback.xml /cassandra.yaml /jvm.options /etc/cassandra/ \ + && mv /usr/local/apache-cassandra-${CASSANDRA_VERSION}/conf/cassandra-env.sh /etc/cassandra/ \ + && adduser --disabled-password --no-create-home --gecos '' --disabled-login cassandra \ + && chown cassandra: /ready-probe.sh \ + && if [ -n "$DEV_CONTAINER" ]; then apt-get -y --no-install-recommends install python; else rm -rf $CASSANDRA_HOME/pylib; fi \ + && apt-get -y purge wget localepurge \ + && apt-get autoremove \ + && apt-get clean \ + && rm -rf \ + $CASSANDRA_HOME/*.txt \ + $CASSANDRA_HOME/doc \ + $CASSANDRA_HOME/javadoc \ + $CASSANDRA_HOME/tools/*.yaml \ + $CASSANDRA_HOME/tools/bin/*.bat \ + $CASSANDRA_HOME/bin/*.bat \ + doc \ + man \ + info \ + locale \ + common-licenses \ + ~/.bashrc \ /var/lib/apt/lists/* \ /var/log/* \ /var/cache/debconf/* \ - common-licenses \ - ~/.bashrc \ /etc/systemd \ /lib/lsb \ /lib/udev \ /usr/share/doc/ \ /usr/share/doc-base/ \ /usr/share/man/ \ - /tmp/* + /tmp/* \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/plugin \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/javaws \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/jjs \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/orbd \ + /usr/lib/jvm/java-8-openjdk-amd64/bin/pack200 \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/policytool \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/rmid \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/rmiregistry \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/servertool \ + /usr/lib/jvm/java-8-openjdk-amd64/bin/tnameserv \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/unpack200 \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/javaws.jar \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/deploy* \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/desktop \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/*javafx* \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/*jfx* \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libdecora_sse.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libprism_*.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libfxplugins.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libglass.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libgstreamer-lite.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjavafx*.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjfx*.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext/jfxrt.jar \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext/nashorn.jar \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/oblique-fonts \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/plugin.jar \ + /usr/lib/jvm/java-8-openjdk-amd64/man -VOLUME ["/cassandra_data"] +VOLUME ["/$CASSANDRA_DATA"] # 7000: intra-node communication # 7001: TLS intra-node communication # 7199: JMX # 9042: CQL -# 9160: thrift service not included cause it is going away -EXPOSE 7000 7001 7199 9042 - -# Not able to do this until https://github.com/kubernetes/kubernetes/issues/2630 is resolved -# if you are using attached storage -# USER cassandra +# 9160: thrift service +EXPOSE 7000 7001 7199 9042 9160 CMD ["/sbin/dumb-init", "/bin/bash", "/run.sh"] diff --git a/examples/storage/cassandra/image/Makefile b/examples/storage/cassandra/image/Makefile index 4c992457c66..3143d5ce24b 100644 --- a/examples/storage/cassandra/image/Makefile +++ b/examples/storage/cassandra/image/Makefile @@ -13,22 +13,28 @@ # limitations under the License. # build the cassandra image. - -VERSION=v11 -PROJECT_ID=google_samples +VERSION=v12 +PROJECT_ID?=google_samples PROJECT=gcr.io/${PROJECT_ID} +CASSANDRA_VERSION=3.9 -all: build +all: kubernetes-cassandra.jar build kubernetes-cassandra.jar: ../java/* ../java/src/main/java/io/k8s/cassandra/*.java cd ../java && mvn clean && mvn package mv ../java/target/kubernetes-cassandra*.jar files/kubernetes-cassandra.jar cd ../java && mvn clean -build: kubernetes-cassandra.jar - docker build --pull -t ${PROJECT}/cassandra:${VERSION} . +container: + docker build --pull --build-arg "CASSANDRA_VERSION=${CASSANDRA_VERSION}" -t ${PROJECT}/cassandra:${VERSION} . + +container-dev: + docker build --pull --build-arg "CASSANDRA_VERSION=${CASSANDRA_VERSION}" --build-arg "DEV_CONTAINER=true" -t ${PROJECT}/cassandra:${VERSION}-dev . + +build: container container-dev push: build gcloud docker -- push ${PROJECT}/cassandra:${VERSION} + gcloud docker -- push ${PROJECT}/cassandra:${VERSION}-dev .PHONY: all build push diff --git a/examples/storage/cassandra/image/files/cassandra.list b/examples/storage/cassandra/image/files/cassandra.list deleted file mode 100644 index c2f5cd12eea..00000000000 --- a/examples/storage/cassandra/image/files/cassandra.list +++ /dev/null @@ -1,2 +0,0 @@ -deb http://www.apache.org/dist/cassandra/debian 39x main -deb-src http://www.apache.org/dist/cassandra/debian 39x main diff --git a/examples/storage/cassandra/image/files/cassandra.yaml b/examples/storage/cassandra/image/files/cassandra.yaml index e0b7107cb04..7df40351e23 100644 --- a/examples/storage/cassandra/image/files/cassandra.yaml +++ b/examples/storage/cassandra/image/files/cassandra.yaml @@ -68,7 +68,7 @@ max_hints_delivery_threads: 2 # Directory where Cassandra should store hints. # If not set, the default directory is $CASSANDRA_HOME/data/hints. -# hints_directory: /var/lib/cassandra/hints +hints_directory: /cassandra_data/hints # How often hints should be flushed from the internal buffers to disk. # Will *not* trigger fsync. diff --git a/examples/storage/cassandra/image/files/java.list b/examples/storage/cassandra/image/files/java.list deleted file mode 100644 index 5bf6d5647df..00000000000 --- a/examples/storage/cassandra/image/files/java.list +++ /dev/null @@ -1,2 +0,0 @@ -# for jre8 -deb http://http.debian.net/debian jessie-backports main diff --git a/examples/storage/cassandra/image/files/kubernetes-cassandra.jar b/examples/storage/cassandra/image/files/kubernetes-cassandra.jar index 4c9415139be..c639d56d0e5 100644 Binary files a/examples/storage/cassandra/image/files/kubernetes-cassandra.jar and b/examples/storage/cassandra/image/files/kubernetes-cassandra.jar differ diff --git a/examples/storage/cassandra/image/files/ready-probe.sh b/examples/storage/cassandra/image/files/ready-probe.sh index 2bf981b6dd2..6601b1a495d 100644 --- a/examples/storage/cassandra/image/files/ready-probe.sh +++ b/examples/storage/cassandra/image/files/ready-probe.sh @@ -16,12 +16,12 @@ if [[ $(nodetool status | grep $POD_IP) == *"UN"* ]]; then if [[ $DEBUG ]]; then - echo "Not Up"; + echo "UN"; fi exit 0; else if [[ $DEBUG ]]; then - echo "UN"; + echo "Not Up"; fi exit 1; fi diff --git a/examples/storage/cassandra/image/files/run.sh b/examples/storage/cassandra/image/files/run.sh old mode 100644 new mode 100755 index 6d057fc0a7d..4f7ae651de4 --- a/examples/storage/cassandra/image/files/run.sh +++ b/examples/storage/cassandra/image/files/run.sh @@ -21,6 +21,7 @@ CASSANDRA_CFG=$CASSANDRA_CONF_DIR/cassandra.yaml # we are doing StatefulSet or just setting our seeds if [ -z "$CASSANDRA_SEEDS" ]; then HOSTNAME=$(hostname -f) + CASSANDRA_SEEDS=$(hostname -f) fi # The following vars relate to there counter parts in $CASSANDRA_CFG @@ -167,5 +168,9 @@ if [[ $CASSANDRA_OPEN_JMX == 'true' ]]; then sed -ri 's/ -Dcom\.sun\.management\.jmxremote\.password\.file=\/etc\/cassandra\/jmxremote\.password//' $CASSANDRA_CONF_DIR/cassandra-env.sh fi +chmod 700 "${CASSANDRA_DATA}" +chown -c -R cassandra "${CASSANDRA_DATA}" "${CASSANDRA_CONF_DIR}" + export CLASSPATH=/kubernetes-cassandra.jar -cassandra -R -f + +su cassandra -c "$CASSANDRA_HOME/bin/cassandra -f"