diff --git a/cluster/libvirt-coreos/.gitignore b/cluster/libvirt-coreos/.gitignore deleted file mode 100644 index f26d8e4020a..00000000000 --- a/cluster/libvirt-coreos/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/libvirt_storage_pool/ -/coreos_production_qemu_image.img.bz2 diff --git a/cluster/libvirt-coreos/README.md b/cluster/libvirt-coreos/README.md deleted file mode 100644 index b1b0a3e490c..00000000000 --- a/cluster/libvirt-coreos/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Kubernetes CoreOS cluster - -With this tutorial one creates a Kubernetes CoreOS cluster containing of one -master and three nodes (workers) running on `192.168.10.1`-`192.168.10.4`. - -For working correctly you need to create the directory addressed as `POOL_PATH` in -`util.sh`: -``` -$ sudo mkdir /var/lib/libvirt/images/kubernetes -$ sudo chown -R $USER:$USER /var/lib/libvirt/images/kubernetes/ -``` - -Then we follow the instructions in the main `kubernetes` directory. - -For debugging set `export UTIL_SH_DEBUG=1`. -``` -$ export KUBERNETES_PROVIDER=libvirt-coreos -$ make release-skip-tests -$ ./cluster/kube-up.sh -``` - -To bring the cluster down again, execute: -``` -$ ./cluster/kube-down.sh -``` - -Have fun! - - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/libvirt-coreos/README.md?pixel)]() diff --git a/cluster/libvirt-coreos/config-default.sh b/cluster/libvirt-coreos/config-default.sh deleted file mode 100644 index 44ba5ce6aac..00000000000 --- a/cluster/libvirt-coreos/config-default.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## Contains configuration values for interacting with the libvirt CoreOS cluster - -# Number of minions in the cluster -NUM_NODES=${NUM_NODES:-3} -export NUM_NODES - -# The IP of the master -export MASTER_IP="192.168.10.1" - -export INSTANCE_PREFIX=kubernetes -export MASTER_NAME="${INSTANCE_PREFIX}-master" - -# Map out the IPs, names and container subnets of each node -export NODE_IP_BASE="192.168.10." -NODE_CONTAINER_SUBNET_BASE="10.10" -MASTER_CONTAINER_NETMASK="255.255.255.0" -MASTER_CONTAINER_ADDR="${NODE_CONTAINER_SUBNET_BASE}.0.1" -MASTER_CONTAINER_SUBNET="${NODE_CONTAINER_SUBNET_BASE}.0.1/24" -CONTAINER_SUBNET="${NODE_CONTAINER_SUBNET_BASE}.0.0/16" -if [[ "$NUM_NODES" -gt 253 ]]; then - echo "ERROR: Because of how IPs are allocated in ${BASH_SOURCE}, you cannot create more than 253 nodes" - exit 1 -fi -for ((i=0; i < NUM_NODES; i++)) do - NODE_IPS[$i]="${NODE_IP_BASE}$((i+2))" - NODE_NAMES[$i]="${INSTANCE_PREFIX}-node-$((i+1))" - NODE_CONTAINER_SUBNETS[$i]="${NODE_CONTAINER_SUBNET_BASE}.$((i+1)).1/24" - NODE_CONTAINER_ADDRS[$i]="${NODE_CONTAINER_SUBNET_BASE}.$((i+1)).1" - NODE_CONTAINER_NETMASKS[$i]="255.255.255.0" -done -NODE_CONTAINER_SUBNETS[$NUM_NODES]=$MASTER_CONTAINER_SUBNET - -SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-10.11.0.0/16}" # formerly PORTAL_NET - - -# Optional: Enable node logging. -ENABLE_NODE_LOGGING=false -LOGGING_DESTINATION=elasticsearch - -# Optional: Install cluster DNS. -ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}" -DNS_SERVER_IP="${SERVICE_CLUSTER_IP_RANGE%.*}.254" -DNS_DOMAIN="cluster.local" - -# Optional: Install cluster registry -ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-true}" - -# Optional: Enable DNS horizontal autoscaler -ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" - -#Generate dns files -sed -f "${KUBE_ROOT}/cluster/addons/dns/transforms2sed.sed" < "${KUBE_ROOT}/cluster/addons/dns/kube-dns.yaml.base" | sed -f "${KUBE_ROOT}/cluster/libvirt-coreos/forShellEval.sed" > "${KUBE_ROOT}/cluster/libvirt-coreos/kube-dns.yaml" - - -#Generate registry files -sed -f "${KUBE_ROOT}/cluster/libvirt-coreos/forEmptyDirRegistry.sed" < "${KUBE_ROOT}/cluster/addons/registry/registry-rc.yaml" > "${KUBE_ROOT}/cluster/libvirt-coreos/registry-rc.yaml" diff --git a/cluster/libvirt-coreos/config-test.sh b/cluster/libvirt-coreos/config-test.sh deleted file mode 100644 index af3633c1be7..00000000000 --- a/cluster/libvirt-coreos/config-test.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## Contains configuration values for interacting with the Vagrant cluster in test mode -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. -source "${KUBE_ROOT}/cluster/libvirt-coreos/config-default.sh" diff --git a/cluster/libvirt-coreos/coreos.xml b/cluster/libvirt-coreos/coreos.xml deleted file mode 100644 index 041f3a01510..00000000000 --- a/cluster/libvirt-coreos/coreos.xml +++ /dev/null @@ -1,71 +0,0 @@ - - ${name} - 512 - 512 - 2 - - hvm - - - - - - - - - destroy - restart - restart - - $(which qemu-system-$(uname -m)) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/cluster/libvirt-coreos/forEmptyDirRegistry.sed b/cluster/libvirt-coreos/forEmptyDirRegistry.sed deleted file mode 100644 index a00392576b3..00000000000 --- a/cluster/libvirt-coreos/forEmptyDirRegistry.sed +++ /dev/null @@ -1,2 +0,0 @@ -s/persistentVolumeClaim:/emptyDir: {}/g -s/claimName: kube-registry-pvc//g diff --git a/cluster/libvirt-coreos/forShellEval.sed b/cluster/libvirt-coreos/forShellEval.sed deleted file mode 100644 index c1efc974831..00000000000 --- a/cluster/libvirt-coreos/forShellEval.sed +++ /dev/null @@ -1,3 +0,0 @@ -s/\"/\\"/g -s/DNS_SERVER_IP/{DNS_SERVER_IP}/g -s/DNS_DOMAIN/{DNS_DOMAIN}/g diff --git a/cluster/libvirt-coreos/namespace.yaml b/cluster/libvirt-coreos/namespace.yaml deleted file mode 100644 index 986f4b48221..00000000000 --- a/cluster/libvirt-coreos/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: kube-system diff --git a/cluster/libvirt-coreos/network_kubernetes_global.xml b/cluster/libvirt-coreos/network_kubernetes_global.xml deleted file mode 100644 index b22cb262fc0..00000000000 --- a/cluster/libvirt-coreos/network_kubernetes_global.xml +++ /dev/null @@ -1,11 +0,0 @@ - - kubernetes_global - - - - - - - - - diff --git a/cluster/libvirt-coreos/network_kubernetes_pods.xml b/cluster/libvirt-coreos/network_kubernetes_pods.xml deleted file mode 100644 index 13bb1d566a5..00000000000 --- a/cluster/libvirt-coreos/network_kubernetes_pods.xml +++ /dev/null @@ -1,6 +0,0 @@ - - kubernetes_pods - - - - diff --git a/cluster/libvirt-coreos/node-openssl.cnf b/cluster/libvirt-coreos/node-openssl.cnf deleted file mode 100644 index acf03fcfbcf..00000000000 --- a/cluster/libvirt-coreos/node-openssl.cnf +++ /dev/null @@ -1,10 +0,0 @@ -[req] -req_extensions = v3_req -distinguished_name = req_distinguished_name -[req_distinguished_name] -[ v3_req ] -basicConstraints = CA:FALSE -keyUsage = nonRepudiation, digitalSignature, keyEncipherment -subjectAltName = @alt_names -[alt_names] -IP.1 = $ENV::WORKER_IP diff --git a/cluster/libvirt-coreos/openssl.cnf b/cluster/libvirt-coreos/openssl.cnf deleted file mode 100644 index 5b1c8e894c8..00000000000 --- a/cluster/libvirt-coreos/openssl.cnf +++ /dev/null @@ -1,15 +0,0 @@ -[req] -req_extensions = v3_req -distinguished_name = req_distinguished_name -[req_distinguished_name] -[ v3_req ] -basicConstraints = CA:FALSE -keyUsage = nonRepudiation, digitalSignature, keyEncipherment -subjectAltName = @alt_names -[alt_names] -DNS.1 = kubernetes -DNS.2 = kubernetes.default -DNS.3 = kubernetes.default.svc -DNS.4 = kubernetes.default.svc.cluster.local -IP.1 = $ENV::KUBERNETES_SVC -IP.2 = $ENV::MASTER_IP diff --git a/cluster/libvirt-coreos/user_data.yml b/cluster/libvirt-coreos/user_data.yml deleted file mode 100644 index 3288c44bb4f..00000000000 --- a/cluster/libvirt-coreos/user_data.yml +++ /dev/null @@ -1,116 +0,0 @@ -#cloud-config - -hostname: ${name} - -ssh_authorized_keys: -${ssh_keys} - -write_files: - - path: /etc/systemd/journald.conf - permissions: 0644 - content: | - [Journal] - SystemMaxUse=50M - RuntimeMaxUse=50M - -coreos: - etcd2: - advertise-client-urls: http://${public_ip}:2379 - initial-advertise-peer-urls: http://${public_ip}:2380 - listen-client-urls: http://0.0.0.0:2379 - listen-peer-urls: http://${public_ip}:2380 - initial-cluster-state: new - initial-cluster: ${etcd2_initial_cluster} - units: - - name: static.network - command: start - content: | - [Match] - # Name=eth0 - MACAddress=52:54:00:00:00:${i} - - [Network] - Address=${public_ip}/24 - DNS=192.168.10.254 - Gateway=192.168.10.254 - - name: cbr0.netdev - command: start - content: | - [NetDev] - Kind=bridge - Name=cbr0 - - name: cbr0.network - command: start - content: | - [Match] - Name=cbr0 - - [Network] - Address=${NODE_CONTAINER_SUBNETS[$i]} - - [Route] - Destination=${CONTAINER_SUBNET} - - name: cbr0-interface.network - command: start - content: | - [Match] - # Name=eth1 - MACAddress=52:54:00:00:01:${i} - - [Network] - Bridge=cbr0 - - name: nat.service - command: start - content: | - [Unit] - Description=NAT non container traffic - - [Service] - ExecStart=/usr/sbin/iptables -w -t nat -A POSTROUTING -o eth0 -j MASQUERADE ! -d ${CONTAINER_SUBNET} - RemainAfterExit=yes - Type=oneshot - - name: etcd2.service - command: start - drop-ins: - - name: 10-override-name.conf - content: | - [Service] - Environment=ETCD_NAME=%H - - name: docker.service - command: start - drop-ins: - - name: 50-opts.conf - content: | - [Service] - Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false' - - name: docker-tcp.socket - command: start - enable: yes - content: | - [Unit] - Description=Docker Socket for the API - - [Socket] - ListenStream=2375 - BindIPv6Only=both - Service=docker.service - - [Install] - WantedBy=sockets.target - - name: opt-kubernetes.mount - command: start - content: | - [Unit] - ConditionVirtualization=|vm - - [Mount] - What=kubernetes - Where=/opt/kubernetes - Options=ro,trans=virtio,version=9p2000.L - Type=9p - update: - group: ${COREOS_CHANNEL:-alpha} - reboot-strategy: off - -$( [[ ${type} =~ "master" ]] && render-template "$ROOT/user_data_master.yml" ) -$( [[ ${type} =~ "node" ]] && render-template "$ROOT/user_data_minion.yml" ) diff --git a/cluster/libvirt-coreos/user_data_master.yml b/cluster/libvirt-coreos/user_data_master.yml deleted file mode 100644 index 4f277db6c99..00000000000 --- a/cluster/libvirt-coreos/user_data_master.yml +++ /dev/null @@ -1,93 +0,0 @@ -#cloud-config - -coreos: - units: - - name: kube-apiserver.service - command: start - content: | - [Unit] - After=opt-kubernetes.mount etcd2.service - ConditionFileIsExecutable=/opt/kubernetes/bin/kube-apiserver - Description=Kubernetes API Server - Documentation=https://github.com/kubernetes/kubernetes - Requires=opt-kubernetes.mount etcd2.service - - [Service] - ExecStart=/opt/kubernetes/bin/kube-apiserver \ - --tls-cert-file=./opt/kubernetes/certs/apiserver.pem \ - --tls-private-key-file=/opt/kubernetes/certs/apiserver-key.pem \ - --client-ca-file=/opt/kubernetes/certs/ca.pem \ - --service-account-key-file=/opt/kubernetes/certs/apiserver-key.pem \ - --service-account-lookup=${SERVICE_ACCOUNT_LOOKUP} \ - --admission-control=${ADMISSION_CONTROL} \ - --insecure-bind-address=0.0.0.0 \ - --insecure-port=8080 \ - --etcd-servers=http://127.0.0.1:2379 \ - --kubelet-port=10250 \ - --v=4 \ - --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE} - Restart=always - RestartSec=2 - - [Install] - WantedBy=multi-user.target - - name: kube-controller-manager.service - command: start - content: | - [Unit] - After=opt-kubernetes.mount kube-apiserver.service - ConditionFileIsExecutable=/opt/kubernetes/bin/kube-controller-manager - Description=Kubernetes Controller Manager - Documentation=https://github.com/kubernetes/kubernetes - Requires=opt-kubernetes.mount kube-apiserver.service - - [Service] - ExecStart=/opt/kubernetes/bin/kube-controller-manager \ - --master=127.0.0.1:8080 \ - --service-account-private-key-file=/opt/kubernetes/certs/apiserver-key.pem \ - --root-ca-file=/opt/kubernetes/certs/ca.pem \ - --v=4 - Restart=always - RestartSec=2 - - [Install] - WantedBy=multi-user.target - - name: kube-scheduler.service - command: start - content: | - [Unit] - After=opt-kubernetes.mount kube-apiserver.service - ConditionFileIsExecutable=/opt/kubernetes/bin/kube-scheduler - Description=Kubernetes Scheduler - Documentation=https://github.com/kubernetes/kubernetes - Requires=opt-kubernetes.mount kube-apiserver.service - - [Service] - ExecStart=/opt/kubernetes/bin/kube-scheduler \ - --master=127.0.0.1:8080 - Restart=always - RestartSec=2 - - [Install] - WantedBy=multi-user.target - - name: kube-addons.service - command: start - content: | - [Unit] - After=opt-kubernetes.mount kube-apiserver.service - ConditionPathIsDirectory=/opt/kubernetes/addons - Description=Kubernetes addons - Documentation=https://github.com/kubernetes/kubernetes - Requires=opt-kubernetes.mount - Wants=kube-apiserver.service - - [Service] - Type=oneshot - ExecStartPre=/bin/bash -c 'while [[ \"\$(curl -s http://127.0.0.1:8080/healthz)\" != \"ok\" ]]; do sleep 1; done' - ExecStartPre=/bin/sleep 10 - ExecStart=/opt/kubernetes/bin/kubectl create -f /opt/kubernetes/addons - ExecStop=/opt/kubernetes/bin/kubectl delete -f /opt/kubernetes/addons - RemainAfterExit=yes - - [Install] - WantedBy=multi-user.target diff --git a/cluster/libvirt-coreos/user_data_minion.yml b/cluster/libvirt-coreos/user_data_minion.yml deleted file mode 100644 index a28bdded378..00000000000 --- a/cluster/libvirt-coreos/user_data_minion.yml +++ /dev/null @@ -1,49 +0,0 @@ -#cloud-config - -coreos: - units: - - name: kubelet.service - command: start - content: | - [Unit] - After=opt-kubernetes.mount docker.socket - ConditionFileIsExecutable=/opt/kubernetes/bin/kubelet - Description=Kubernetes Kubelet - Documentation=https://github.com/kubernetes/kubernetes - Requires=opt-kubernetes.mount docker.socket - - [Service] - ExecStart=/opt/kubernetes/bin/kubelet \ - --address=0.0.0.0 \ - --hostname-override=${NODE_IPS[$i]} \ - --cluster-domain=cluster.local \ - --kubeconfig=/opt/kubernetes/kubeconfig/kubelet.kubeconfig \ - --tls-cert-file=/opt/kubernetes/certs/${NODE_NAMES[$i]}-node.pem \ \ - --tls-private-key-file=/opt/kubernetes/certs/${NODE_NAMES[$i]}-node-key.pem \ - $( [[ "$ENABLE_CLUSTER_DNS" == "true" ]] && echo "--cluster-dns=${DNS_SERVER_IP}" ) \ - $( [[ "$ENABLE_CLUSTER_DNS" == "true" ]] && echo "--cluster-domain=${DNS_DOMAIN}" ) \ - --pod-manifest-path=/opt/kubernetes/manifests - Restart=always - RestartSec=2 - - [Install] - WantedBy=multi-user.target - - name: kube-proxy.service - command: start - content: | - [Unit] - After=opt-kubernetes.mount - ConditionFileIsExecutable=/opt/kubernetes/bin/kube-proxy - Description=Kubernetes Proxy - Documentation=https://github.com/kubernetes/kubernetes - Requires=opt-kubernetes.mount - - [Service] - ExecStart=/opt/kubernetes/bin/kube-proxy \ - --master=http://${MASTER_IP}:8080 \ - --hostname-override=${NODE_IPS[$i]} - Restart=always - RestartSec=2 - - [Install] - WantedBy=multi-user.target diff --git a/cluster/libvirt-coreos/util.sh b/cluster/libvirt-coreos/util.sh deleted file mode 100644 index 545d9850453..00000000000 --- a/cluster/libvirt-coreos/util.sh +++ /dev/null @@ -1,459 +0,0 @@ -#!/bin/bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts. - -[ ! -z ${UTIL_SH_DEBUG+x} ] && set -x - -command -v kubectl >/dev/null 2>&1 || { echo >&2 "kubectl not found in path. Aborting."; exit 1; } - -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. -readonly ROOT=$(dirname "${BASH_SOURCE}") -source "$ROOT/${KUBE_CONFIG_FILE:-"config-default.sh"}" -source "$KUBE_ROOT/cluster/common.sh" - -export LIBVIRT_DEFAULT_URI=qemu:///system -export SERVICE_ACCOUNT_LOOKUP=${SERVICE_ACCOUNT_LOOKUP:-true} -export ADMISSION_CONTROL=${ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,PVCProtection,ResourceQuota} -readonly POOL=kubernetes -readonly POOL_PATH=/var/lib/libvirt/images/kubernetes - -[ ! -d "${POOL_PATH}" ] && (echo "$POOL_PATH" does not exist ; exit 1 ) - -# Creates a kubeconfig file for the kubelet. -# Args: address (e.g. "http://localhost:8080"), destination file path -function create-kubelet-kubeconfig() { - local apiserver_address="${1}" - local destination="${2}" - if [[ -z "${apiserver_address}" ]]; then - echo "Must provide API server address to create Kubelet kubeconfig file!" - exit 1 - fi - if [[ -z "${destination}" ]]; then - echo "Must provide destination path to create Kubelet kubeconfig file!" - exit 1 - fi - echo "Creating Kubelet kubeconfig file" - local dest_dir="$(dirname "${destination}")" - mkdir -p "${dest_dir}" &>/dev/null || sudo mkdir -p "${dest_dir}" - sudo=$(test -w "${dest_dir}" || echo "sudo -E") - cat < /dev/null -apiVersion: v1 -kind: Config -clusters: - - cluster: - server: ${apiserver_address} - name: local -contexts: - - context: - cluster: local - name: local -current-context: local -EOF -} - -# join -# Concatenates the list elements with the delimiter passed as first parameter -# -# Ex: join , a b c -# -> a,b,c -function join { - local IFS="$1" - shift - echo "$*" -} - -# Must ensure that the following ENV vars are set -function detect-master { - KUBE_MASTER_IP=$MASTER_IP - KUBE_MASTER=$MASTER_NAME - export KUBERNETES_MASTER=http://$KUBE_MASTER_IP:8080 - echo "KUBE_MASTER_IP: $KUBE_MASTER_IP" - echo "KUBE_MASTER: $KUBE_MASTER" -} - -# Get node IP addresses and store in KUBE_NODE_IP_ADDRESSES[] -function detect-nodes { - KUBE_NODE_IP_ADDRESSES=("${NODE_IPS[@]}") -} - -function generate_certs { - node_names=("${@}") - #Root-CA - tempdir=$(mktemp -d) - CA_KEY=${CA_KEY:-"$tempdir/ca-key.pem"} - CA_CERT=${CA_CERT:-"$tempdir/ca.pem"} - openssl genrsa -out "${CA_KEY}" 2048 2>/dev/null - openssl req -x509 -new -nodes -key "${CA_KEY}" -days 10000 -out "${CA_CERT}" -subj "/CN=kube-ca" 2>/dev/null - - #API server key pair - KUBE_KEY=${KUBE_KEY:-"$tempdir/apiserver-key.pem"} - API_SERVER_CERT_REQ=${API_SERVER_CERT_REQ:-"$tempdir/apiserver.csr"} - openssl genrsa -out "${KUBE_KEY}" 2048 2>/dev/null - KUBERNETES_SVC=${SERVICE_CLUSTER_IP_RANGE%.*}.1 openssl req -new -key "${KUBE_KEY}" -out "${API_SERVER_CERT_REQ}" -subj "/CN=kube-apiserver" -config cluster/libvirt-coreos/openssl.cnf 2>/dev/null - KUBE_CERT=${KUBE_CERT:-"$tempdir/apiserver.pem"} - KUBERNETES_SVC=${SERVICE_CLUSTER_IP_RANGE%.*}.1 openssl x509 -req -in "${API_SERVER_CERT_REQ}" -CA "${CA_CERT}" -CAkey "${CA_KEY}" -CAcreateserial -out "${KUBE_CERT}" -days 365 -extensions v3_req -extfile cluster/libvirt-coreos/openssl.cnf 2>/dev/null - - #Copy apiserver and controller tsl assets - mkdir -p "$POOL_PATH/kubernetes/certs" - cp "${KUBE_CERT}" "$POOL_PATH/kubernetes/certs" - cp "${KUBE_KEY}" "$POOL_PATH/kubernetes/certs" - cp "${CA_CERT}" "$POOL_PATH/kubernetes/certs" - - #Generate nodes certificate - for (( i = 0 ; i < $NUM_NODES ; i++ )); do - openssl genrsa -out $tempdir/${node_names[$i]}-node-key.pem 2048 2>/dev/null - cp "$tempdir/${node_names[$i]}-node-key.pem" "$POOL_PATH/kubernetes/certs" - WORKER_IP=${NODE_IPS[$i]} openssl req -new -key $tempdir/${node_names[$i]}-node-key.pem -out $tempdir/${node_names[$i]}-node.csr -subj "/CN=${node_names[$i]}" -config cluster/libvirt-coreos/node-openssl.cnf 2>/dev/null - WORKER_IP=${NODE_IPS[$i]} openssl x509 -req -in $tempdir/${node_names[$i]}-node.csr -CA "${CA_CERT}" -CAkey "${CA_KEY}" -CAcreateserial -out $tempdir/${node_names[$i]}-node.pem -days 365 -extensions v3_req -extfile cluster/libvirt-coreos/node-openssl.cnf 2>/dev/null - cp "$tempdir/${node_names[$i]}-node.pem" "$POOL_PATH/kubernetes/certs" - done - echo "TLS assets generated..." -} - -#Setup registry proxy -function setup_registry_proxy { - if [[ "$ENABLE_CLUSTER_REGISTRY" == "true" ]]; then - cp "./cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml" "$POOL_PATH/kubernetes/manifests" - fi -} - -# Verify prereqs on host machine -function verify-prereqs { - if ! which virsh >/dev/null; then - echo "Can't find virsh in PATH, please fix and retry." >&2 - exit 1 - fi - if ! virsh nodeinfo >/dev/null; then - exit 1 - fi - if [[ "$(&2 - echo "Enabling it would reduce the memory footprint of large clusters" >&2 - if [[ -t 0 ]]; then - read -t 5 -n 1 -p "Do you want to enable KSM (requires root password) (y/n)? " answer - echo "" - if [[ "$answer" == 'y' ]]; then - su -c 'echo 1 > /sys/kernel/mm/ksm/run' - fi - else - echo "You can enable it with (as root):" >&2 - echo "" >&2 - echo " echo 1 > /sys/kernel/mm/ksm/run" >&2 - echo "" >&2 - fi - fi -} - -# Destroy the libvirt storage pool and all the images inside -# -# If 'keep_base_image' is passed as first parameter, -# the base image is kept, as well as the storage pool. -# All the other images are deleted. -function destroy-pool { - virsh pool-info $POOL >/dev/null 2>&1 || return - - rm -rf "$POOL_PATH"/kubernetes/* - rm -rf "$POOL_PATH"/kubernetes_config*/* - local vol - virsh vol-list $POOL | awk 'NR>2 && !/^$/ && $1 ~ /^kubernetes/ {print $1}' | \ - while read vol; do - virsh vol-delete $vol --pool $POOL - done - - [[ "$1" == 'keep_base_image' ]] && return - - set +e - virsh vol-delete coreos_base.img --pool $POOL - virsh pool-destroy $POOL - rmdir "$POOL_PATH" - set -e -} - -# Creates the libvirt storage pool and populate it with -# - the CoreOS base image -# - the kubernetes binaries -function initialize-pool { - mkdir -p "$POOL_PATH" - if ! virsh pool-info $POOL >/dev/null 2>&1; then - virsh pool-create-as $POOL dir --target "$POOL_PATH" - fi - - wget -N -P "$ROOT" https://${COREOS_CHANNEL:-alpha}.release.core-os.net/amd64-usr/current/coreos_production_qemu_image.img.bz2 - if [[ "$ROOT/coreos_production_qemu_image.img.bz2" -nt "$POOL_PATH/coreos_base.img" ]]; then - bunzip2 -f -k "$ROOT/coreos_production_qemu_image.img.bz2" - virsh vol-delete coreos_base.img --pool $POOL 2> /dev/null || true - fi - if ! virsh vol-list $POOL | grep -q coreos_base.img; then - virsh vol-create-as $POOL coreos_base.img 10G --format qcow2 - virsh vol-upload coreos_base.img "$ROOT/coreos_production_qemu_image.img" --pool $POOL - fi - - mkdir -p "$POOL_PATH/kubernetes" - kube-push-internal - - mkdir -p "$POOL_PATH/kubernetes/manifests" - if [[ "$ENABLE_NODE_LOGGING" == "true" ]]; then - if [[ "$LOGGING_DESTINATION" == "elasticsearch" ]]; then - cp "$KUBE_ROOT/cluster/saltbase/salt/fluentd-es/fluentd-es.manifest" "$POOL_PATH/kubernetes/manifests" - elif [[ "$LOGGING_DESTINATION" == "gcp" ]]; then - cp "$KUBE_ROOT/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.manifest" "$POOL_PATH/kubernetes/manifests" - fi - fi - - mkdir -p "$POOL_PATH/kubernetes/addons" - if [[ "$ENABLE_CLUSTER_DNS" == "true" ]]; then - render-template "$ROOT/namespace.yaml" > "$POOL_PATH/kubernetes/addons/namespace.yaml" - render-template "$ROOT/kube-dns.yaml" > "$POOL_PATH/kubernetes/addons/kube-dns.yaml" - fi - - virsh pool-refresh $POOL -} - -function destroy-network { - set +e - virsh net-destroy kubernetes_global - virsh net-destroy kubernetes_pods - set -e -} - -function initialize-network { - virsh net-create "$ROOT/network_kubernetes_global.xml" - virsh net-create "$ROOT/network_kubernetes_pods.xml" -} - -function render-template { - eval "echo \"$(cat $1)\"" -} - -function wait-cluster-readiness { - echo "Wait for cluster readiness" - - local timeout=120 - while [[ $timeout -ne 0 ]]; do - nb_ready_nodes=$(kubectl get nodes -o go-template="{{range.items}}{{range.status.conditions}}{{.type}}{{end}}:{{end}}" 2>/dev/null | tr ':' '\n' | grep -c Ready || true) - echo "Nb ready nodes: $nb_ready_nodes / $NUM_NODES" - if [[ "$nb_ready_nodes" -eq "$NUM_NODES" ]]; then - return 0 - fi - - timeout=$(($timeout-1)) - sleep .5 - done - - return 1 -} - -# Instantiate a kubernetes cluster -function kube-up { - detect-master - detect-nodes - initialize-pool keep_base_image - generate_certs "${NODE_NAMES[@]}" - setup_registry_proxy - initialize-network - - readonly ssh_keys="$(cat ~/.ssh/*.pub | sed 's/^/ - /')" - readonly kubernetes_dir="$POOL_PATH/kubernetes" - - local i - for (( i = 0 ; i <= $NUM_NODES ; i++ )); do - if [[ $i -eq $NUM_NODES ]]; then - etcd2_initial_cluster[$i]="${MASTER_NAME}=http://${MASTER_IP}:2380" - else - etcd2_initial_cluster[$i]="${NODE_NAMES[$i]}=http://${NODE_IPS[$i]}:2380" - fi - done - etcd2_initial_cluster=$(join , "${etcd2_initial_cluster[@]}") - readonly machines=$(join , "${KUBE_NODE_IP_ADDRESSES[@]}") - - for (( i = 0 ; i <= $NUM_NODES ; i++ )); do - if [[ $i -eq $NUM_NODES ]]; then - type=master - name=$MASTER_NAME - public_ip=$MASTER_IP - else - type=node-$(printf "%02d" $i) - name=${NODE_NAMES[$i]} - public_ip=${NODE_IPS[$i]} - fi - image=$name.img - config=kubernetes_config_$type - - virsh vol-create-as $POOL $image 10G --format qcow2 --backing-vol coreos_base.img --backing-vol-format qcow2 - - mkdir -p "$POOL_PATH/$config/openstack/latest" - render-template "$ROOT/user_data.yml" > "$POOL_PATH/$config/openstack/latest/user_data" - virsh pool-refresh $POOL - - domain_xml=$(mktemp) - render-template $ROOT/coreos.xml > $domain_xml - virsh create $domain_xml - rm $domain_xml - done - - export KUBE_SERVER="http://192.168.10.1:8080" - export CONTEXT="libvirt-coreos" - create-kubeconfig - create-kubelet-kubeconfig "http://${MASTER_IP}:8080" "${POOL_PATH}/kubernetes/kubeconfig/kubelet.kubeconfig" - - wait-cluster-readiness - - echo "Kubernetes cluster is running. The master is running at:" - echo - echo " http://${KUBE_MASTER_IP}:8080" - echo - echo "You can control the Kubernetes cluster with: 'kubectl'" - echo "You can connect on the master with: 'ssh core@${KUBE_MASTER_IP}'" - - wait-registry-readiness - -} - -function create_registry_rc() { - echo " Create registry replication controller" - kubectl create -f $ROOT/registry-rc.yaml - local timeout=120 - while [[ $timeout -ne 0 ]]; do - phase=$(kubectl get pods -n kube-system -lk8s-app=kube-registry --output='jsonpath={.items..status.phase}') - if [ "$phase" = "Running" ]; then - return 0 - fi - timeout=$(($timeout-1)) - sleep .5 - done -} - - -function create_registry_svc() { - echo " Create registry service" - kubectl create -f "${KUBE_ROOT}/cluster/addons/registry/registry-svc.yaml" -} - -function wait-registry-readiness() { - if [[ "$ENABLE_CLUSTER_REGISTRY" != "true" ]]; then - return 0 - fi - echo "Wait for registry readiness..." - local timeout=120 - while [[ $timeout -ne 0 ]]; do - phase=$(kubectl get namespaces --output=jsonpath='{.items[?(@.metadata.name=="kube-system")].status.phase}') - if [ "$phase" = "Active" ]; then - create_registry_rc - create_registry_svc - return 0 - fi - echo "waiting for namespace kube-system" - timeout=$(($timeout-1)) - sleep .5 - done -} - -# Delete a kubernetes cluster -function kube-down { - virsh list | awk 'NR>2 && !/^$/ && $2 ~ /^kubernetes/ {print $2}' | \ - while read dom; do - virsh destroy $dom - done - destroy-pool keep_base_image - destroy-network -} - -# The kubernetes binaries are pushed to a host directory which is exposed to the VM -function upload-server-tars { - tar -x -C "$POOL_PATH/kubernetes" -f "$SERVER_BINARY_TAR" kubernetes - rm -rf "$POOL_PATH/kubernetes/bin" - mv "$POOL_PATH/kubernetes/kubernetes/server/bin" "$POOL_PATH/kubernetes/bin" - chmod -R 755 "$POOL_PATH/kubernetes/bin" - rm -fr "$POOL_PATH/kubernetes/kubernetes" -} - -# Update a kubernetes cluster with latest source -function kube-push { - kube-push-internal - ssh-to-node "$MASTER_NAME" "sudo systemctl restart kube-apiserver kube-controller-manager kube-scheduler" - for ((i=0; i < NUM_NODES; i++)); do - ssh-to-node "${NODE_NAMES[$i]}" "sudo systemctl restart kubelet kube-proxy" - done - wait-cluster-readiness -} - -function kube-push-internal { - case "${KUBE_PUSH:-release}" in - release) - kube-push-release;; - local) - kube-push-local;; - *) - echo "The only known push methods are \"release\" to use the release tarball or \"local\" to use the binaries built by make. KUBE_PUSH is set \"$KUBE_PUSH\"" >&2 - return 1;; - esac -} - -function kube-push-release { - find-release-tars - upload-server-tars -} - -function kube-push-local { - rm -rf "$POOL_PATH/kubernetes/bin/*" - mkdir -p "$POOL_PATH/kubernetes/bin" - cp "${KUBE_ROOT}/_output/local/go/bin"/* "$POOL_PATH/kubernetes/bin" -} - -# Execute prior to running tests to build a release if required for env -function test-build-release { - echo "TODO" -} - -# Execute prior to running tests to initialize required structure -function test-setup { - "${KUBE_ROOT}/cluster/kube-up.sh" -} - -# Execute after running tests to perform any required clean-up -function test-teardown { - kube-down -} - -# SSH to a node by name or IP ($1) and run a command ($2). -function ssh-to-node { - local node="$1" - local cmd="$2" - local machine - - if [[ "$node" == "$MASTER_IP" ]] || [[ "$node" =~ ^"$NODE_IP_BASE" ]]; then - machine="$node" - elif [[ "$node" == "$MASTER_NAME" ]]; then - machine="$MASTER_IP" - else - for ((i=0; i < NUM_NODES; i++)); do - if [[ "$node" == "${NODE_NAMES[$i]}" ]]; then - machine="${NODE_IPS[$i]}" - break - fi - done - fi - if [[ -z "$machine" ]]; then - echo "$node is an unknown machine to ssh to" >&2 - fi - ssh -o ConnectTimeout=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ControlMaster=no "core@$machine" "$cmd" -} - -# Perform preparations required to run e2e tests -function prepare-e2e() { - echo "libvirt-coreos doesn't need special preparations for e2e tests" 1>&2 -}