From 4646594f299ab893548a2f385ae4d38321061b68 Mon Sep 17 00:00:00 2001 From: Maciej Borsz Date: Wed, 5 Jun 2019 13:10:55 +0200 Subject: [PATCH] Migrate kubemark to e2e-up/e2e/down scripts. --- cluster/kubemark/gce/config-default.sh | 86 +---- test/kubemark/gce/util.sh | 174 +++------- .../resources/manifests/etcd-events.yaml | 51 --- test/kubemark/resources/manifests/etcd.yaml | 50 --- .../resources/manifests/kube-apiserver.yaml | 70 ---- .../manifests/kube-controller-manager.yaml | 54 --- .../resources/manifests/kube-scheduler.yaml | 43 --- test/kubemark/skeleton/util.sh | 56 +-- test/kubemark/start-kubemark.sh | 323 +----------------- test/kubemark/stop-kubemark.sh | 5 +- 10 files changed, 86 insertions(+), 826 deletions(-) delete mode 100644 test/kubemark/resources/manifests/etcd-events.yaml delete mode 100644 test/kubemark/resources/manifests/etcd.yaml delete mode 100644 test/kubemark/resources/manifests/kube-apiserver.yaml delete mode 100644 test/kubemark/resources/manifests/kube-controller-manager.yaml delete mode 100644 test/kubemark/resources/manifests/kube-scheduler.yaml diff --git a/cluster/kubemark/gce/config-default.sh b/cluster/kubemark/gce/config-default.sh index e0388e2c3b9..8fea98fcc08 100644 --- a/cluster/kubemark/gce/config-default.sh +++ b/cluster/kubemark/gce/config-default.sh @@ -23,92 +23,22 @@ source "${KUBE_ROOT}/cluster/gce/config-common.sh" -GCLOUD=gcloud -ZONE=${KUBE_GCE_ZONE:-us-central1-b} -REGION=${ZONE%-*} -NUM_NODES=${KUBEMARK_NUM_NODES:-10} -NUM_WINDOWS_NODES=${KUBEMARK_NUM_WINDOWS_NODES:-0} -MASTER_SIZE=${KUBEMARK_MASTER_SIZE:-n1-standard-$(get-master-size)} -MASTER_DISK_TYPE=pd-ssd -MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-$(get-master-disk-size)} -MASTER_ROOT_DISK_SIZE=${KUBEMARK_MASTER_ROOT_DISK_SIZE:-$(get-master-root-disk-size)} -REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false} -PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false} -NODE_ACCELERATORS=${NODE_ACCELERATORS:-""} -CREATE_CUSTOM_NETWORK=${CREATE_CUSTOM_NETWORK:-false} -EVENT_PD=${EVENT_PD:-false} - -MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-gci} -NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-gci} -MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-cos-beta-73-11647-64-0} -MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud} CLEANUP_KUBEMARK_IMAGE=${CLEANUP_KUBEMARK_IMAGE:-true} -# GPUs supported in GCE do not have compatible drivers in Debian 7. -if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then - NODE_ACCELERATORS="" -fi - -NETWORK=${KUBE_GCE_NETWORK:-e2e} -if [[ "${CREATE_CUSTOM_NETWORK}" == true ]]; then - SUBNETWORK="${SUBNETWORK:-${NETWORK}-custom-subnet}" -fi -INSTANCE_PREFIX="${INSTANCE_PREFIX:-"default"}" -MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master" -AGGREGATOR_MASTER_NAME="${INSTANCE_PREFIX}-kubemark-aggregator" -MASTER_TAG="kubemark-master" -EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store" -MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" -CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-$(get-cluster-ip-range)}" -RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" -TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100} -KUBE_APISERVER_REQUEST_TIMEOUT=300 -ETCD_COMPACTION_INTERVAL_SEC="${KUBEMARK_ETCD_COMPACTION_INTERVAL_SEC:-}" - -# Set etcd image (e.g. k8s.gcr.io/etcd) and version (e.g. 3.1.12-1) if you need -# non-default version. -ETCD_IMAGE="${TEST_ETCD_IMAGE:-}" -ETCD_VERSION="${TEST_ETCD_VERSION:-}" -ETCD_SERVERS="${KUBEMARK_ETCD_SERVERS:-}" -ETCD_SERVERS_OVERRIDES="${KUBEMARK_ETCD_SERVERS_OVERRIDES:-}" - -# Storage backend. 'etcd2' and 'etcd3' are supported. -STORAGE_BACKEND=${STORAGE_BACKEND:-} -# Storage media type: application/json and application/vnd.kubernetes.protobuf are supported. -STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-} - -# Default Log level for all components in test clusters and variables to override it in specific components. -TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:---v=4}" -API_SERVER_TEST_LOG_LEVEL="${API_SERVER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" -CONTROLLER_MANAGER_TEST_LOG_LEVEL="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" -SCHEDULER_TEST_LOG_LEVEL="${SCHEDULER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" +# NUM_NODES is used by start-kubemark.sh to determine a correct number of replicas. +NUM_NODES=${KUBEMARK_NUM_NODES:-10} +NUM_WINDOWS_NODES=${KUBEMARK_NUM_WINDOWS_NODES:-0} HOLLOW_KUBELET_TEST_LOG_LEVEL="${HOLLOW_KUBELET_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" HOLLOW_PROXY_TEST_LOG_LEVEL="${HOLLOW_PROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" -TEST_CLUSTER_DELETE_COLLECTION_WORKERS="${TEST_CLUSTER_DELETE_COLLECTION_WORKERS:---delete-collection-workers=16}" -TEST_CLUSTER_MAX_REQUESTS_INFLIGHT="${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT:-}" -TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:-}" - # ContentType used by all components to communicate with apiserver. TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}" -KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}" - -CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota}" - -# Master components' test arguments. -APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1,scheduling.k8s.io/v1alpha1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS}" -CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}" -SCHEDULER_TEST_ARGS="${KUBEMARK_SCHEDULER_TEST_ARGS:-} ${SCHEDULER_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}" - # Hollow-node components' test arguments. HOLLOW_KUBELET_TEST_ARGS="${HOLLOW_KUBELET_TEST_ARGS:-} ${HOLLOW_KUBELET_TEST_LOG_LEVEL}" HOLLOW_PROXY_TEST_ARGS="${HOLLOW_PROXY_TEST_ARGS:-} ${HOLLOW_PROXY_TEST_LOG_LEVEL}" -SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16" # formerly PORTAL_NET -ALLOCATE_NODE_CIDRS=true - # Optional: Enable cluster autoscaler. ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-false}" # When using Cluster Autoscaler, always start with one hollow-node replica. @@ -126,12 +56,4 @@ fi ENABLE_KUBEMARK_KUBE_DNS="${ENABLE_KUBEMARK_KUBE_DNS:-true}" KUBE_DNS_DOMAIN="${KUBE_DNS_DOMAIN:-cluster.local}" -# Optional: set feature gates -FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}" - -# Enable a simple "AdvancedAuditing" setup for testing. -ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-false}" - -# The number of services that are allowed to sync concurrently. Will be passed -# into kube-controller-manager via `--concurrent-service-syncs` -CONCURRENT_SERVICE_SYNCS="${CONCURRENT_SERVICE_SYNCS:-}" +CLEANUP_KUBEMARK_IMAGE=false diff --git a/test/kubemark/gce/util.sh b/test/kubemark/gce/util.sh index 760db164d52..f88b8ba3e92 100644 --- a/test/kubemark/gce/util.sh +++ b/test/kubemark/gce/util.sh @@ -31,127 +31,65 @@ function authenticate-docker { gcloud beta auth configure-docker -q } -# This function isn't too robust to race, but that should be ok given its one-off usage during setup. -function get-or-create-master-ip { - MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \ - --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') 2>/dev/null || true +function create-kubemark-master { + # We intentionally override env vars in subshell to preserve original values. + # shellcheck disable=SC2030,SC2031 + ( + export KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark" + export CLUSTER_NAME="${CLUSTER_NAME}-kubemark" + export KUBE_CREATE_NODES=false + export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark" + + # Even if the "real cluster" is private, we shouldn't manage cloud nat. + export KUBE_GCE_PRIVATE_CLUSTER=false - if [[ -z "${MASTER_IP:-}" ]]; then - run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \ - --project "${PROJECT}" \ - --region "${REGION}" -q + # Quite tricky cidr setup: we set KUBE_GCE_ENABLE_IP_ALIASES=true to avoid creating + # cloud routes and RangeAllocator to assign cidrs by kube-controller-manager. + export KUBE_GCE_ENABLE_IP_ALIASES=true + export KUBE_GCE_NODE_IPAM_MODE=RangeAllocator + + # Disable all addons. They are running outside of the kubemark cluster. + export KUBE_ENABLE_CLUSTER_AUTOSCALER=false + export KUBE_ENABLE_CLUSTER_DNS=false + export KUBE_ENABLE_NODE_LOGGING=false + export KUBE_ENABLE_METRICS_SERVER=false + export KUBE_ENABLE_CLUSTER_MONITORING="none" + export KUBE_ENABLE_L7_LOADBALANCING="none" - MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \ - --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') - fi + # Unset env variables set by kubetest for 'root cluster'. We need recompute them + # for kubemark master. + # TODO(mborsz): Figure out some better way to filter out such env variables than + # listing them here. + unset MASTER_SIZE MASTER_DISK_SIZE MASTER_ROOT_DISK_SIZE + + # Set kubemark-specific overrides: + # for each defined env KUBEMARK_X=Y call export X=Y. + for var in ${!KUBEMARK_*}; do + dst_var=${var#KUBEMARK_} + val=${!var} + echo "Setting ${dst_var} to '${val}'" + export "${dst_var}"="${val}" + done + + # Append kubemark-specific "MASTER_COMPONENTS_QPS_LIMITS". + # TODO(mborsz): Migrate all users of KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS to {SCHEDULER,CONTROLLER_MANAGER}_TEST_ARGS. + export CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_ARGS:-} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}" + export SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS:-} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}" + + "${KUBE_ROOT}/hack/e2e-internal/e2e-up.sh" + ) } -function create-master-instance-with-resources { - GCLOUD_COMMON_ARGS=(--project "${PROJECT}" --zone "${ZONE}") - # Override the master image project to cos-cloud for COS images staring with `cos` string prefix. - DEFAULT_GCI_PROJECT=google-containers - if [[ "${GCI_VERSION}" == "cos"* ]]; then - DEFAULT_GCI_PROJECT=cos-cloud - fi - MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-${DEFAULT_GCI_PROJECT}} +function delete-kubemark-master { + # We intentionally override env vars in subshell to preserve original values. + # shellcheck disable=SC2030,SC2031 + ( + export CLUSTER_NAME="${CLUSTER_NAME}-kubemark" + export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark" - run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \ - "${GCLOUD_COMMON_ARGS[@]}" \ - --type "${MASTER_DISK_TYPE}" \ - --size "${MASTER_DISK_SIZE}" & - - if [ "${EVENT_PD:-}" == "true" ]; then - run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \ - "${GCLOUD_COMMON_ARGS[@]}" \ - --type "${MASTER_DISK_TYPE}" \ - --size "${MASTER_DISK_SIZE}" & - fi - - get-or-create-master-ip & - - wait - - run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \ - "${GCLOUD_COMMON_ARGS[@]}" \ - --address "${MASTER_IP}" \ - --machine-type "${MASTER_SIZE}" \ - --image-project="${MASTER_IMAGE_PROJECT}" \ - --image "${MASTER_IMAGE}" \ - --tags "${MASTER_TAG}" \ - --subnet "${SUBNETWORK:-${NETWORK}}" \ - --scopes "storage-ro,logging-write" \ - --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \ - --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" - - run-gcloud-compute-with-retries instances add-metadata "${MASTER_NAME}" \ - "${GCLOUD_COMMON_ARGS[@]}" \ - --metadata-from-file startup-script="${KUBE_ROOT}/test/kubemark/resources/start-kubemark-master.sh" & - - if [ "${EVENT_PD:-}" == "true" ]; then - echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}" - run-gcloud-compute-with-retries instances attach-disk "${MASTER_NAME}" \ - "${GCLOUD_COMMON_ARGS[@]}" \ - --disk "${MASTER_NAME}-event-pd" \ - --device-name="master-event-pd" & - fi - - run-gcloud-compute-with-retries firewall-rules create "${MASTER_NAME}-https" \ - --project "${PROJECT}" \ - --network "${NETWORK}" \ - --source-ranges "0.0.0.0/0" \ - --target-tags "${MASTER_TAG}" \ - --allow "tcp:443" & - - run-gcloud-compute-with-retries firewall-rules create "${MASTER_NAME}-internal" \ - --project "${PROJECT}" \ - --network "${NETWORK}" \ - --source-ranges "10.0.0.0/8" \ - --target-tags "${MASTER_TAG}" \ - --allow "tcp:1-2379,tcp:2382-65535,udp:1-65535,icmp" & - - wait -} - -# Command to be executed is '$1'. -# No. of retries is '$2' (if provided) or 1 (default). -function execute-cmd-on-master-with-retries() { - RETRIES="${2:-1}" run-gcloud-compute-with-retries ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" --command="$1" -} - -function copy-files() { - run-gcloud-compute-with-retries scp --recurse --zone="${ZONE}" --project="${PROJECT}" "$@" -} - -function delete-master-instance-and-resources { - GCLOUD_COMMON_ARGS=(--project "${PROJECT}" --zone "${ZONE}" --quiet) - - gcloud compute instances delete "${MASTER_NAME}" \ - "${GCLOUD_COMMON_ARGS[@]}" || true - - gcloud compute disks delete "${MASTER_NAME}-pd" \ - "${GCLOUD_COMMON_ARGS[@]}" || true - - gcloud compute disks delete "${MASTER_NAME}-event-pd" \ - "${GCLOUD_COMMON_ARGS[@]}" &> /dev/null || true - - gcloud compute addresses delete "${MASTER_NAME}-ip" \ - --project "${PROJECT}" \ - --region "${REGION}" \ - --quiet || true - - gcloud compute firewall-rules delete "${MASTER_NAME}-https" \ - --project "${PROJECT}" \ - --quiet || true - - gcloud compute firewall-rules delete "${MASTER_NAME}-internal" \ - --project "${PROJECT}" \ - --quiet || true - - if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then - gcloud compute instances delete "${EVENT_STORE_NAME}" \ - "${GCLOUD_COMMON_ARGS[@]}" || true - - gcloud compute disks delete "${EVENT_STORE_NAME}-pd" \ - "${GCLOUD_COMMON_ARGS[@]}" || true - fi + export KUBE_DELETE_NETWORK=false + # Even if the "real cluster" is private, we shouldn't manage cloud nat. + export KUBE_GCE_PRIVATE_CLUSTER=false + "${KUBE_ROOT}/hack/e2e-internal/e2e-down.sh" + ) } diff --git a/test/kubemark/resources/manifests/etcd-events.yaml b/test/kubemark/resources/manifests/etcd-events.yaml deleted file mode 100644 index ea83b1b604b..00000000000 --- a/test/kubemark/resources/manifests/etcd-events.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: etcd-server-events - namespace: kube-system -spec: - hostNetwork: true - nodeName: {{instance_prefix}}-master - containers: - - name: etcd-container - image: {{kube_docker_registry}}/etcd:{{etcd_image}} - imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 100m - command: - - /bin/sh - - -c - - /usr/local/bin/etcd - {{params}} - 1>>/var/log/etcd-events.log 2>&1 - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /health - port: 4002 - scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 15 - ports: - - name: serverport - containerPort: 2381 - hostPort: 2381 - protocol: TCP - - name: clientport - containerPort: 4002 - hostPort: 4002 - protocol: TCP - volumeMounts: - - name: varetcd - mountPath: /var/etcd - - name: varlogetcd - mountPath: /var/log/etcd-events.log - volumes: - - name: varetcd - hostPath: - path: /var/etcd/events - - name: varlogetcd - hostPath: - path: /var/log/etcd-events.log - type: FileOrCreate diff --git a/test/kubemark/resources/manifests/etcd.yaml b/test/kubemark/resources/manifests/etcd.yaml deleted file mode 100644 index 2f7d88a633f..00000000000 --- a/test/kubemark/resources/manifests/etcd.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: etcd-server - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: etcd-container - image: {{kube_docker_registry}}/etcd:{{etcd_image}} - imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 200m - command: - - /bin/sh - - -c - - /usr/local/bin/etcd - {{params}} - 1>>/var/log/etcd.log 2>&1 - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /health - port: 2379 - scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 15 - ports: - - name: serverport - containerPort: 2380 - hostPort: 2380 - protocol: TCP - - name: clientport - containerPort: 2379 - hostPort: 2379 - protocol: TCP - volumeMounts: - - name: varetcd - mountPath: /var/etcd - - name: varlogetcd - mountPath: /var/log/etcd.log - volumes: - - name: varetcd - hostPath: - path: /var/etcd - - name: varlogetcd - hostPath: - path: /var/log/etcd.log - type: FileOrCreate diff --git a/test/kubemark/resources/manifests/kube-apiserver.yaml b/test/kubemark/resources/manifests/kube-apiserver.yaml deleted file mode 100644 index 162d2144208..00000000000 --- a/test/kubemark/resources/manifests/kube-apiserver.yaml +++ /dev/null @@ -1,70 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: kube-apiserver - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: kube-apiserver - image: {{kube_docker_registry}}/kube-apiserver:{{kube-apiserver_docker_tag}} - imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 250m - command: - - /bin/sh - - -c - - /usr/local/bin/kube-apiserver - {{params}} - 1>>/var/log/kube-apiserver.log 2>&1 - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 15 - ports: - - name: https - containerPort: 443 - hostPort: 443 - protocol: TCP - - name: local - containerPort: 8080 - hostPort: 8080 - protocol: TCP - volumeMounts: -{{audit_policy_config_mount}} - - name: srvkube - mountPath: /etc/srv/kubernetes - readOnly: true - - name: logfile - mountPath: /var/log/kube-apiserver.log - - name: etcssl - mountPath: /etc/ssl - readOnly: true - - name: usrsharecacerts - mountPath: /usr/share/ca-certificates - readOnly: true - - name: srvsshproxy - mountPath: /etc/srv/sshproxy - volumes: -{{audit_policy_config_volume}} - - name: srvkube - hostPath: - path: /etc/srv/kubernetes - - name: logfile - hostPath: - path: /var/log/kube-apiserver.log - type: FileOrCreate - - name: etcssl - hostPath: - path: /etc/ssl - - name: usrsharecacerts - hostPath: - path: /usr/share/ca-certificates - - name: srvsshproxy - hostPath: - path: /etc/srv/sshproxy diff --git a/test/kubemark/resources/manifests/kube-controller-manager.yaml b/test/kubemark/resources/manifests/kube-controller-manager.yaml deleted file mode 100644 index c465b9ecc9f..00000000000 --- a/test/kubemark/resources/manifests/kube-controller-manager.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: kube-controller-manager - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: kube-controller-manager - image: {{kube_docker_registry}}/kube-controller-manager:{{kube-controller-manager_docker_tag}} - imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 200m - command: - - /bin/sh - - -c - - /usr/local/bin/kube-controller-manager - {{params}} - 1>>/var/log/kube-controller-manager.log 2>&1 - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10252 - scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 15 - volumeMounts: - - name: srvkube - mountPath: /etc/srv/kubernetes - readOnly: true - - name: logfile - mountPath: /var/log/kube-controller-manager.log - - name: etcssl - mountPath: /etc/ssl - readOnly: true - - name: usrsharecacerts - mountPath: /usr/share/ca-certificates - readOnly: true - volumes: - - name: srvkube - hostPath: - path: /etc/srv/kubernetes - - name: logfile - hostPath: - path: /var/log/kube-controller-manager.log - type: FileOrCreate - - name: etcssl - hostPath: - path: /etc/ssl - - name: usrsharecacerts - hostPath: - path: /usr/share/ca-certificates diff --git a/test/kubemark/resources/manifests/kube-scheduler.yaml b/test/kubemark/resources/manifests/kube-scheduler.yaml deleted file mode 100644 index daf1728295a..00000000000 --- a/test/kubemark/resources/manifests/kube-scheduler.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: kube-scheduler - namespace: kube-system -spec: - hostNetwork: true - nodeName: {{instance_prefix}}-master - containers: - - name: kube-scheduler - image: {{kube_docker_registry}}/kube-scheduler:{{kube-scheduler_docker_tag}} - imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 100m - command: - - /bin/sh - - -c - - /usr/local/bin/kube-scheduler - {{params}} - 1>>/var/log/kube-scheduler.log 2>&1 - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10251 - scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 15 - volumeMounts: - - name: srvkube - mountPath: /etc/srv/kubernetes - readOnly: true - - name: logfile - mountPath: /var/log/kube-scheduler.log - volumes: - - name: srvkube - hostPath: - path: /etc/srv/kubernetes - - name: logfile - hostPath: - path: /var/log/kube-scheduler.log - type: FileOrCreate diff --git a/test/kubemark/skeleton/util.sh b/test/kubemark/skeleton/util.sh index 0620dc0496c..d64a2fe00d3 100644 --- a/test/kubemark/skeleton/util.sh +++ b/test/kubemark/skeleton/util.sh @@ -24,55 +24,15 @@ function authenticate-docker { echo "Configuring registry authentication" 1>&2 } -# This function should get master IP address (creating one if needed). -# ENV vars that should be defined by the end of this function: -# - MASTER_IP -# -# Recommended for this function to include retrying logic in case of failures. -function get-or-create-master-ip { - echo "MASTER_IP: $MASTER_IP" 1>&2 +# This function should create kubemark master and write kubeconfig to +# "${RESOURCE_DIRECTORY}/kubeconfig.kubemark". +function create-kubemark-master { + echo "Creating cluster..." } -# This function should create a machine instance for the master along -# with any/all of the following resources: -# - Attach a PD to the master (optionally 1 more for storing events) -# - A public IP address for the master ($MASTER_IP) -# - A network firewall rule allowing all TCP traffic on port 443 in master -# Note: This step is compulsory in order for kubemark to work -# -# ENV vars that should be defined by the end of this function: -# - MASTER_NAME -# -# Recommended for this function to include retrying logic for the above -# operations in case of failures. -function create-master-instance-with-resources { - echo "MASTER_IP: $MASTER_IP" 1>&2 - echo "MASTER_NAME: $MASTER_NAME" 1>&2 -} - -# This function should execute the command('$1') on the master machine -# (possibly through SSH), retrying in case of failure. The allowed number of -# retries would be '$2' (if not provided, default to single try). -function execute-cmd-on-master-with-retries() { - echo "Executing command on the master" 1>&2 -} - -# This function should act as an scp for the kubemark cluster, which copies -# the files given by the first n-1 arguments to the remote location given -# by the n^th argument. -# -# Recommended for this function to include retrying logic in case of failures. -function copy-files() { - echo "Copying files" 1>&2 -} - -# This function should delete the master instance along with all the -# resources that have been allocated inside the function -# 'create-master-instance-with-resources' above. -# -# Recommended for this function to include retrying logic in case of failures. -function delete-master-instance-and-resources { - echo "Deleting master instance and its allocated resources" 1>&2 +# This function should delete kubemark master. +function delete-kubemark-master { + echo "Deleting cluster..." } # Common colors used throughout the kubemark scripts @@ -90,4 +50,4 @@ if [[ -z "${color_start-}" ]]; then declare -r color_cyan="${color_start}1;36m" # shellcheck disable=SC2034 declare -r color_norm="${color_start}0m" -fi \ No newline at end of file +fi diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 933a39615d5..e5e21a50705 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -34,174 +34,15 @@ fi source "${KUBE_ROOT}/cluster/kubemark/util.sh" -# hack/lib/init.sh will ovewrite ETCD_VERSION if this is unset -# what what is default in hack/lib/etcd.sh -# To avoid it, if it is empty, we set it to 'avoid-overwrite' and -# clean it after that. -if [ -z "${ETCD_VERSION:-}" ]; then - ETCD_VERSION="avoid-overwrite" -fi -source "${KUBE_ROOT}/hack/lib/init.sh" -if [ "${ETCD_VERSION:-}" == "avoid-overwrite" ]; then - ETCD_VERSION="" -fi - KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" +LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark" # Generate a random 6-digit alphanumeric tag for the kubemark image. # Used to uniquify image builds across different invocations of this script. KUBEMARK_IMAGE_TAG=$(head /dev/urandom | tr -dc 'a-z0-9' | fold -w 6 | head -n 1) -# Write all environment variables that we need to pass to the kubemark master, -# locally to the file ${RESOURCE_DIRECTORY}/kubemark-master-env.sh. -function create-master-environment-file { - cat > "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" </dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - HEAPSTER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - CLUSTER_AUTOSCALER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - KUBE_DNS_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - echo "Generated PKI authentication data for kubemark." -} - -# Wait for the master to be reachable for executing commands on it. We do this by -# trying to run the bash noop(:) on the master, with 10 retries. -function wait-for-master-reachability { - execute-cmd-on-master-with-retries ":" 10 - echo "Checked master reachability for remote command execution." -} - -# Write all the relevant certs/keys/tokens to the master. -function write-pki-config-to-master { - PKI_SETUP_CMD="sudo mkdir /home/kubernetes/k8s_auth_data -p && \ - sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/ca.crt\" && \ - sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/server.cert\" && \ - sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/server.key\" && \ - sudo bash -c \"echo ${ETCD_APISERVER_CA_KEY_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/etcd-apiserver-ca.key\" && \ - sudo bash -c \"echo ${ETCD_APISERVER_CA_CERT_BASE64} | base64 --decode | gunzip > /home/kubernetes/k8s_auth_data/etcd-apiserver-ca.crt\" && \ - sudo bash -c \"echo ${ETCD_APISERVER_SERVER_KEY_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/etcd-apiserver-server.key\" && \ - sudo bash -c \"echo ${ETCD_APISERVER_SERVER_CERT_BASE64} | base64 --decode | gunzip > /home/kubernetes/k8s_auth_data/etcd-apiserver-server.crt\" && \ - sudo bash -c \"echo ${ETCD_APISERVER_CLIENT_KEY_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/etcd-apiserver-client.key\" && \ - sudo bash -c \"echo ${ETCD_APISERVER_CLIENT_CERT_BASE64} | base64 --decode | gunzip > /home/kubernetes/k8s_auth_data/etcd-apiserver-client.crt\" && \ - sudo bash -c \"echo ${REQUESTHEADER_CA_CERT_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/aggr_ca.crt\" && \ - sudo bash -c \"echo ${PROXY_CLIENT_CERT_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/proxy_client.crt\" && \ - sudo bash -c \"echo ${PROXY_CLIENT_KEY_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/proxy_client.key\" && \ - sudo bash -c \"echo ${KUBECFG_CERT_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/kubecfg.crt\" && \ - sudo bash -c \"echo ${KUBECFG_KEY_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/kubecfg.key\" && \ - sudo bash -c \"echo \"${KUBE_BEARER_TOKEN},admin,admin\" > /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ - sudo bash -c \"echo \"${KUBELET_TOKEN},system:node:node-name,uid:kubelet,system:nodes\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ - sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},system:kube-proxy,uid:kube_proxy\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ - sudo bash -c \"echo \"${HEAPSTER_TOKEN},system:heapster,uid:heapster\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ - sudo bash -c \"echo \"${CLUSTER_AUTOSCALER_TOKEN},system:cluster-autoscaler,uid:cluster-autoscaler\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ - sudo bash -c \"echo \"${NODE_PROBLEM_DETECTOR_TOKEN},system:node-problem-detector,uid:system:node-problem-detector\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ - sudo bash -c \"echo \"${KUBE_DNS_TOKEN},system:kube-dns,uid:kube-dns\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ - sudo bash -c \"echo ${KUBE_PASSWORD},admin,admin > /home/kubernetes/k8s_auth_data/basic_auth.csv\"" - execute-cmd-on-master-with-retries "${PKI_SETUP_CMD}" 3 - echo "Wrote PKI certs, keys, tokens and admin password to master." -} - -# Write kubeconfig to ${RESOURCE_DIRECTORY}/kubeconfig.kubemark in order to -# use kubectl locally. -function write-local-kubeconfig { - LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark" - cat > "${LOCAL_KUBECONFIG}" << EOF -apiVersion: v1 -kind: Config -users: -- name: kubecfg - user: - client-certificate-data: "${KUBECFG_CERT_BASE64}" - client-key-data: "${KUBECFG_KEY_BASE64}" - username: admin - password: admin -clusters: -- name: kubemark - cluster: - certificate-authority-data: "${CA_CERT_BASE64}" - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: kubecfg - name: kubemark-context -current-context: kubemark-context -EOF - echo "Kubeconfig file for kubemark master written to ${LOCAL_KUBECONFIG}." -} - -# Copy all the necessary resource files (scripts/configs/manifests) to the master. -function copy-resource-files-to-master { - copy-files \ - "${SERVER_BINARY_TAR}" \ - "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" \ - "${RESOURCE_DIRECTORY}/start-kubemark-master.sh" \ - "${RESOURCE_DIRECTORY}/kubeconfig.kubemark" \ - "${KUBEMARK_DIRECTORY}/configure-kubectl.sh" \ - "${RESOURCE_DIRECTORY}/manifests/etcd.yaml" \ - "${RESOURCE_DIRECTORY}/manifests/etcd-events.yaml" \ - "${RESOURCE_DIRECTORY}/manifests/kube-apiserver.yaml" \ - "${RESOURCE_DIRECTORY}/manifests/kube-scheduler.yaml" \ - "${RESOURCE_DIRECTORY}/manifests/kube-controller-manager.yaml" \ - "${RESOURCE_DIRECTORY}/manifests/kube-addon-manager.yaml" \ - "${RESOURCE_DIRECTORY}/manifests/addons/kubemark-rbac-bindings" \ - "kubernetes@${MASTER_NAME}":/home/kubernetes/ - echo "Copied server binary, master startup scripts, configs and resource manifests to master." -} - -# Make startup scripts executable and run start-kubemark-master.sh. -function start-master-components { - echo "" - MASTER_STARTUP_CMD="sudo bash /home/kubernetes/start-kubemark-master.sh" - execute-cmd-on-master-with-retries "${MASTER_STARTUP_CMD}" - echo "The master has started and is now live." -} - # Create a docker image for hollow-node and upload it to the appropriate docker registry. function create-and-upload-hollow-node-image { authenticate-docker @@ -243,121 +84,6 @@ function delete-kubemark-image { # manifests of the hollow-node and heapster replication controllers from # templates, and finally create these resources through kubectl. function create-kube-hollow-node-resources { - # Create kubeconfig for Kubelet. - KUBELET_KUBECONFIG_CONTENTS="apiVersion: v1 -kind: Config -users: -- name: kubelet - user: - client-certificate-data: ${KUBELET_CERT_BASE64} - client-key-data: ${KUBELET_KEY_BASE64} -clusters: -- name: kubemark - cluster: - certificate-authority-data: ${CA_CERT_BASE64} - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: kubelet - name: kubemark-context -current-context: kubemark-context" - - # Create kubeconfig for Kubeproxy. - KUBEPROXY_KUBECONFIG_CONTENTS="apiVersion: v1 -kind: Config -users: -- name: kube-proxy - user: - token: ${KUBE_PROXY_TOKEN} -clusters: -- name: kubemark - cluster: - insecure-skip-tls-verify: true - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: kube-proxy - name: kubemark-context -current-context: kubemark-context" - - # Create kubeconfig for Heapster. - HEAPSTER_KUBECONFIG_CONTENTS="apiVersion: v1 -kind: Config -users: -- name: heapster - user: - token: ${HEAPSTER_TOKEN} -clusters: -- name: kubemark - cluster: - insecure-skip-tls-verify: true - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: heapster - name: kubemark-context -current-context: kubemark-context" - - # Create kubeconfig for Cluster Autoscaler. - CLUSTER_AUTOSCALER_KUBECONFIG_CONTENTS="apiVersion: v1 -kind: Config -users: -- name: cluster-autoscaler - user: - token: ${CLUSTER_AUTOSCALER_TOKEN} -clusters: -- name: kubemark - cluster: - insecure-skip-tls-verify: true - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: cluster-autoscaler - name: kubemark-context -current-context: kubemark-context" - - # Create kubeconfig for NodeProblemDetector. - NPD_KUBECONFIG_CONTENTS="apiVersion: v1 -kind: Config -users: -- name: node-problem-detector - user: - token: ${NODE_PROBLEM_DETECTOR_TOKEN} -clusters: -- name: kubemark - cluster: - insecure-skip-tls-verify: true - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: node-problem-detector - name: kubemark-context -current-context: kubemark-context" - - # Create kubeconfig for Kube DNS. - KUBE_DNS_KUBECONFIG_CONTENTS="apiVersion: v1 -kind: Config -users: -- name: kube-dns - user: - token: ${KUBE_DNS_TOKEN} -clusters: -- name: kubemark - cluster: - insecure-skip-tls-verify: true - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: kube-dns - name: kubemark-context -current-context: kubemark-context" - # Create kubemark namespace. "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" @@ -367,13 +93,15 @@ current-context: kubemark-context" --from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json" # Create secret for passing kubeconfigs to kubelet, kubeproxy and npd. + # It's bad that all component shares the same kubeconfig. + # TODO(https://github.com/kubernetes/kubernetes/issues/79883): Migrate all components to separate credentials. "${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \ - --from-literal=kubelet.kubeconfig="${KUBELET_KUBECONFIG_CONTENTS}" \ - --from-literal=kubeproxy.kubeconfig="${KUBEPROXY_KUBECONFIG_CONTENTS}" \ - --from-literal=heapster.kubeconfig="${HEAPSTER_KUBECONFIG_CONTENTS}" \ - --from-literal=cluster_autoscaler.kubeconfig="${CLUSTER_AUTOSCALER_KUBECONFIG_CONTENTS}" \ - --from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}" \ - --from-literal=dns.kubeconfig="${KUBE_DNS_KUBECONFIG_CONTENTS}" + --from-file=kubelet.kubeconfig="${LOCAL_KUBECONFIG}" \ + --from-file=kubeproxy.kubeconfig="${LOCAL_KUBECONFIG}" \ + --from-file=npd.kubeconfig="${LOCAL_KUBECONFIG}" \ + --from-file=heapster.kubeconfig="${LOCAL_KUBECONFIG}" \ + --from-file=cluster_autoscaler.kubeconfig="${LOCAL_KUBECONFIG}" \ + --from-file=dns.kubeconfig="${LOCAL_KUBECONFIG}" # Create addon pods. # Heapster. @@ -474,42 +202,23 @@ function wait-for-hollow-nodes-to-run-or-timeout { } ############################### Main Function ######################################## -detect-project &> /dev/null -find-release-tars - -# We need master IP to generate PKI and kubeconfig for cluster. -get-or-create-master-ip -generate-pki-config -write-local-kubeconfig - -# Setup for master. -function start-master { - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_yellow}STARTING SETUP FOR MASTER${color_norm}" - create-master-environment-file - create-master-instance-with-resources - wait-for-master-reachability - write-pki-config-to-master - copy-resource-files-to-master - start-master-components -} -start-master & -start_master_pid=$! # Setup for hollow-nodes. function start-hollow-nodes { + # shellcheck disable=SC2154 # Color defined in sourced script echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}" create-and-upload-hollow-node-image create-kube-hollow-node-resources wait-for-hollow-nodes-to-run-or-timeout } -start-hollow-nodes & -start_hollow_nodes_pid=$! -wait $start_master_pid || { echo "Failed to start kubemark master" ; exit 1 ; } -wait $start_hollow_nodes_pid ||{ echo "Failed to start hollow nodes" ; exit 1 ; } +detect-project &> /dev/null +create-kubemark-master + +MASTER_IP=$(grep server "$LOCAL_KUBECONFIG" | awk -F "/" '{print $3}') + +start-hollow-nodes echo "" echo "Master IP: ${MASTER_IP}" -echo "Password to kubemark master: ${KUBE_PASSWORD}" echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}" diff --git a/test/kubemark/stop-kubemark.sh b/test/kubemark/stop-kubemark.sh index 75ef920b201..d064ef3e277 100755 --- a/test/kubemark/stop-kubemark.sh +++ b/test/kubemark/stop-kubemark.sh @@ -41,7 +41,6 @@ detect-project &> /dev/null rm -rf "${RESOURCE_DIRECTORY}/addons" \ "${RESOURCE_DIRECTORY}/kubeconfig.kubemark" \ - "${RESOURCE_DIRECTORY}/hollow-node.yaml" \ - "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" &> /dev/null || true + "${RESOURCE_DIRECTORY}/hollow-node.yaml" &> /dev/null || true -delete-master-instance-and-resources +delete-kubemark-master