diff --git a/federation/Makefile b/federation/Makefile new file mode 100644 index 00000000000..2cac8af9b6b --- /dev/null +++ b/federation/Makefile @@ -0,0 +1,26 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DBG_MAKEFILE ?= +ifeq ($(DBG_MAKEFILE),1) + $(warning ***** starting makefile for goal(s) "$(MAKECMDGOALS)") + $(warning ***** $(shell date)) +else + # If we're not debugging the Makefile, don't echo recipes. + MAKEFLAGS += -s +endif + +.PHONY: build +build: + ./build.sh $(do) diff --git a/federation/README.md b/federation/README.md new file mode 100644 index 00000000000..fcee30ed745 --- /dev/null +++ b/federation/README.md @@ -0,0 +1,67 @@ +# Cluster Federation + +Kubernetes Cluster Federation enables users to federate multiple +Kubernetes clusters. Please see the [user guide](http://kubernetes.io/docs/admin/federation/) +and the [admin guide](http://kubernetes.io/docs/user-guide/federation/federated-services/) +for more details about setting up and using the Cluster Federation. + +# Building Kubernetes Cluster Federation + +Please see the [Kubernetes Development Guide](https://github.com/kubernetes/kubernetes/blob/master/docs/devel/development.md) +for initial setup. Once you have the development environment setup +as explained in that guide, you also need to install [`jq`](https://stedolan.github.io/jq/download/) + + +Building cluster federation should be as simple as running: + +```shell +make build do=gen +``` + +To deploy clusters and install federation components, edit the +`config.default.json` file to describe your clusters and run + +```shell +make build do=deploy +``` + +To turn down the federation components and tear down the clusters run: + +```shell +make build do=destroy +``` + +# Ideas for improvement + +1. Split the `build` phase (make recipe) into multiple phases: + 1. `init`: pull installer images + 2. `build-binaries` + 3. `build-docker` + 4. `build`: build-binary + build-docker + 5. `push`: to push the built images + 6. `genconfig` + 7. `deploy-clusters` + 8. `deploy-federation` + 9. `deploy`: deploy-clusters + deploy-federation + 10. `destroy-federation` + 11. `destroy-clusters` + 12. `destroy`: destroy-federation + destroy-clusters + 13. `redeploy-federation`: just redeploys the federation components. + +2. Add a `release` phase to run as part of Kubernetes release process + that copies only a part of the `build.sh` script that's relevant to + the users into the release. + +3. Continue with `destroy` phase even in the face of errors. + + The bash script sets `set -e errexit` which causes the script to exit + at the very first error. This should be the default mode for deploying + components but not for destroying/cleanup. + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/federation/README.md?pixel)]() diff --git a/federation/build.sh b/federation/build.sh new file mode 100755 index 00000000000..0b708f64412 --- /dev/null +++ b/federation/build.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script will build the hyperkube image and push it to the repository +# referred to by KUBE_REGISTRY. The image will be given a version tag with +# the value from KUBE_VERSION. It also turns up/turns down Kubernetes +# clusters and federation components using the built hyperkube image. +# e.g. run as: +# KUBE_REGISTRY=localhost:5000/anushku \ +# KUBE_VERSION=1.3.0-dev ./build.sh +# +# will deploy the components using +# localhost:5000/anushku/hyperkube-amd64:1.3.0-dev image. + +# Everything in this script is expected to be executed from the $KUBE_ROOT +# directory. + +# TODO(madhusudancs): Separate the dev functions from the deployment +# functions. A lot of code here is to make this work in dev environments. +# The script that we ship to the users as part of a release should be +# much simpler (about 80% of the code here could be removed for non-dev +# environments). + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +CUR_ROOT=$(dirname "${BASH_SOURCE}") + +source "${KUBE_ROOT}/build/common.sh" +source "${KUBE_ROOT}/build/util.sh" +# Provides the $KUBERNETES_PROVIDER variable and detect-project function +source "${KUBE_ROOT}/cluster/kube-util.sh" +source "${KUBE_ROOT}/cluster/lib/logging.sh" + +readonly ACTION="${1:-gen}" + +readonly TMP_DIR="$(mktemp -d)" +readonly FEDERATION_OUTPUT_ROOT="${LOCAL_OUTPUT_ROOT}/federation" + +readonly KUBE_ANYWHERE_FEDERATION_IMAGE="gcr.io/madhusudancs-containers/kubernetes-anywhere-federation" +readonly KUBE_ANYWHERE_FEDERATION_VERSION="v0.9.0" +readonly KUBE_ANYWHERE_FEDERATION_CHARTS_IMAGE="gcr.io/madhusudancs-containers/federation-charts" +readonly KUBE_ANYWHERE_FEDERATION_CHARTS_VERSION="v0.9.0" + +readonly GOOGLE_APPLICATION_CREDENTIALS="${GOOGLE_APPLICATION_CREDENTIALS:-${HOME}/.config/gcloud/application_default_credentials.json}" +readonly KUBE_CONFIG_DIR="${KUBE_CONFIG_DIR:-${HOME}/.kube}" +readonly KUBE_CONFIG="${KUBE_CONFIG:-${HOME}/.kube/config}" + +detect-project +readonly KUBE_PROJECT="${KUBE_PROJECT:-${PROJECT:-}}" + +readonly KUBE_REGISTRY="${KUBE_REGISTRY:-gcr.io/${KUBE_PROJECT}}" +# In dev environments this value must be recomputed after build. See +# the build() function. Not making it readonly +KUBE_VERSION="${KUBE_VERSION:-}" + + +function cleanup { + rm -rf "${TMP_DIR}" + cd "${CUR_ROOT}" +} +trap cleanup EXIT + +function dirty_sha() { + local -r index="${KUBE_ROOT}/.git/index" + local -r objects_dir="${KUBE_ROOT}/.git/objects" + + local -r tmp_dir="${TMP_DIR}/.git" + local -r tmp_index="${tmp_dir}/index" + local -r tmp_objects_dir="${tmp_dir}/objects" + + mkdir -p "${tmp_objects_dir}" + cp "${index}" "${tmp_index}" + + local -r files=$(git ls-files -m -o -d --exclude-standard) + GIT_INDEX_FILE="${tmp_index}" git add ${files} + GIT_ALTERNATE_OBJECT_DIRECTORIES="${objects_dir}" GIT_OBJECT_DIRECTORY="${tmp_objects_dir}" GIT_INDEX_FILE="${tmp_index}" git write-tree +} + +function update_config() { + local -r q="${1:-}" + local -r cfile="${2:-}" + local -r bname="$(basename ${cfile})" + + jq "${q}" "${cfile}" > "${TMP_DIR}/${bname}" + mv "${TMP_DIR}/${bname}" "${cfile}" +} + +function build() { + kube::build::verify_prereqs + kube::build::build_image + kube::build::run_build_command make WHAT="cmd/kubectl cmd/hyperkube" + + # Recompute KUBE_VERSION because it might have changed after rebuild. + KUBE_VERSION="${KUBE_VERSION:-$(kube::release::semantic_image_tag_version)}" + + # Also append the dirty tree SHA to keep the versions unique across + # builds. + if [[ "${KUBE_VERSION}" == *-dirty ]]; then + KUBE_VERSION+=".$(dirty_sha)" + fi + + BASEIMAGE="ubuntu:16.04" \ + REGISTRY="${KUBE_REGISTRY}" \ + VERSION="${KUBE_VERSION}" \ + make -C "${KUBE_ROOT}/cluster/images/hyperkube" build +} + +function push() { + kube::log::status "Pushing hyperkube image to the registry" + gcloud docker push "${KUBE_REGISTRY}/hyperkube-amd64:${KUBE_VERSION}" +} + +function pull_installer() { + kube::log::status "Pulling installer images" + docker pull "${KUBE_ANYWHERE_FEDERATION_IMAGE}:${KUBE_ANYWHERE_FEDERATION_VERSION}" + docker pull "${KUBE_ANYWHERE_FEDERATION_CHARTS_IMAGE}:${KUBE_ANYWHERE_FEDERATION_CHARTS_VERSION}" +} + +function ensure_files() { + kube::log::status "Ensure provider is supported..." + if [[ "${KUBERNETES_PROVIDER:-}" != "gce" ]]; then + echo "Supported providers: \"gce\"" + exit 1 + fi + + kube::log::status "Ensure credential files exist..." + if [[ ! -f "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then + echo "Please ensure Google credentials file \""${GOOGLE_APPLICATION_CREDENTIALS}"\" exists." + exit 1 + fi + + if [[ ! -f "${KUBE_CONFIG}" ]]; then + echo "Please ensure kubeconfig file \""${KUBE_CONFIG}"\" exists." + exit 1 + fi +} + +function kube_action() { + kube::log::status "${ACTION} clusters" + docker run \ + --user="$(id -u):$(id -g)" \ + -m 12G \ + -v "${GOOGLE_APPLICATION_CREDENTIALS}:/.config/gcloud/application_default_credentials.json:ro" \ + -v "${KUBE_CONFIG_DIR}:/.kube" \ + -v "${FEDERATION_OUTPUT_ROOT}:/_output" \ + "${KUBE_ANYWHERE_FEDERATION_IMAGE}:${KUBE_ANYWHERE_FEDERATION_VERSION}" \ + "${ACTION}" +} + +function federation_action() { + kube::log::status "${ACTION} federation components" + docker run \ + -m 12G \ + -v "${KUBE_CONFIG}:/root/.kube/config:ro" \ + -v "${FEDERATION_OUTPUT_ROOT}:/_output" \ + "${KUBE_ANYWHERE_FEDERATION_CHARTS_IMAGE}:${KUBE_ANYWHERE_FEDERATION_CHARTS_VERSION}" \ + "${ACTION}" +} + +function gen_or_update_config() { + mkdir -p "${FEDERATION_OUTPUT_ROOT}" + cp "federation/config.default.json" "${FEDERATION_OUTPUT_ROOT}/config.json" + + update_config \ + '[.[] | .phase1.gce.project |= "'"${KUBE_PROJECT}"'"]' \ + "${FEDERATION_OUTPUT_ROOT}/config.json" + + # Not chaining for readability + update_config \ + '[.[] | .phase2 = { docker_registry: "'"${KUBE_REGISTRY}"'", kubernetes_version: "'"${KUBE_VERSION}"'" } ]' \ + "${FEDERATION_OUTPUT_ROOT}/config.json" + + cat < "${FEDERATION_OUTPUT_ROOT}/values.yaml" +apiserverRegistry: "${KUBE_REGISTRY}" +apiserverVersion: "${KUBE_VERSION}" +controllerManagerRegistry: "${KUBE_REGISTRY}" +controllerManagerVersion: "${KUBE_VERSION}" +EOF +} + +if [[ "${ACTION}" == "gen" || "${ACTION}" == "deploy" ]]; then + ensure_files + + cd "${KUBE_ROOT}" + build + push + + pull_installer + + # Update config after build and push, but before turning up the clusters + # to ensure the config has the right image version tags. + gen_or_update_config + + kube_action + federation_action +else + federation_action + kube_action +fi + +kube::log::status "Successfully completed!" diff --git a/federation/config.default.json b/federation/config.default.json new file mode 100644 index 00000000000..8bcb1f832d4 --- /dev/null +++ b/federation/config.default.json @@ -0,0 +1,83 @@ +[ + { + "phase1": { + "num_nodes": 3, + "cluster_name": "cluster1-kubernetes", + "cloud_provider": "gce", + "cluster_cidr": "10.180.0.0/14", + "gce": { + "os_image": "ubuntu-1604-xenial-v20160420c", + "instance_type": "n1-standard-2", + "project": "", + "region": "us-central1", + "zone": "us-central1-a", + "network": "federation" + } + }, + "phase2": { + "docker_registry": "gcr.io/google-containers", + "kubernetes_version": "v1.3.0" + }, + "phase3": { + "run_addons": true, + "kube_proxy": true, + "dashboard": true, + "heapster": true, + "kube_dns": true + } + }, + { + "phase1": { + "num_nodes": 3, + "cluster_name": "cluster2-kubernetes", + "cloud_provider": "gce", + "cluster_cidr": "10.184.0.0/14", + "gce": { + "os_image": "ubuntu-1604-xenial-v20160420c", + "instance_type": "n1-standard-2", + "project": "", + "region": "us-central1", + "zone": "us-central1-b", + "network": "federation" + } + }, + "phase2": { + "docker_registry": "gcr.io/google-containers", + "kubernetes_version": "v1.3.0" + }, + "phase3": { + "run_addons": true, + "kube_proxy": true, + "dashboard": true, + "heapster": true, + "kube_dns": true + } + }, + { + "phase1": { + "num_nodes": 3, + "cluster_name": "cluster3-kubernetes", + "cloud_provider": "gce", + "cluster_cidr": "10.188.0.0/14", + "gce": { + "os_image": "ubuntu-1604-xenial-v20160420c", + "instance_type": "n1-standard-2", + "project": "", + "region": "us-central1", + "zone": "us-central1-f", + "network": "federation" + } + }, + "phase2": { + "docker_registry": "gcr.io/google-containers", + "kubernetes_version": "v1.3.0" + }, + "phase3": { + "run_addons": true, + "kube_proxy": true, + "dashboard": true, + "heapster": true, + "kube_dns": true + } + } +] diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index ab1a6e1de0c..c90ce6f2398 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -64,6 +64,18 @@ cluster/vsphere/templates/salt-minion.sh: hostname_override: $(ip route get 1.1 examples/cluster-dns/images/frontend/client.py: service_address = socket.gethostbyname(hostname) examples/storage/cassandra/image/run.sh: cluster_name \ examples/storage/vitess/env.sh: node_ip=$(get_node_ip) +federation/config.default.json: "cloud_provider": "gce", +federation/config.default.json: "cloud_provider": "gce", +federation/config.default.json: "cloud_provider": "gce", +federation/config.default.json: "cluster_cidr": "10.180.0.0/14", +federation/config.default.json: "cluster_cidr": "10.184.0.0/14", +federation/config.default.json: "cluster_cidr": "10.188.0.0/14", +federation/config.default.json: "cluster_name": "cluster1-kubernetes", +federation/config.default.json: "cluster_name": "cluster2-kubernetes", +federation/config.default.json: "cluster_name": "cluster3-kubernetes", +federation/config.default.json: "num_nodes": 3, +federation/config.default.json: "num_nodes": 3, +federation/config.default.json: "num_nodes": 3, hack/local-up-cluster.sh: advertise_address="--advertise_address=${API_HOST}" hack/local-up-cluster.sh: runtime_config="--runtime-config=${RUNTIME_CONFIG}" hack/local-up-cluster.sh: advertise_address=""