diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2ac1d28d9ef..58451ec7131 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,8 +1,8 @@ **What this PR does / why we need it**: diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index a8ba43e223d..37c04ff3990 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -3299,35 +3299,35 @@ }, { "ImportPath": "k8s.io/kube-openapi/pkg/aggregator", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/builder", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/generators", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/handler", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto/validation", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/utils/clock", diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 16aae26af93..7a66405a6e8 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -75671,6 +75671,36 @@ } ] }, + "io.k8s.api.core.v1.ConfigMapNodeConfigSource": { + "description": "ConfigMapNodeConfigSource contains the information to reference a ConfigMap as a config source for the Node.", + "required": [ + "namespace", + "name", + "kubeletConfigKey" + ], + "properties": { + "kubeletConfigKey": { + "description": "KubeletConfigKey declares which key of the referenced ConfigMap corresponds to the KubeletConfiguration structure This field is required in all cases.", + "type": "string" + }, + "name": { + "description": "Name is the metadata.name of the referenced ConfigMap. This field is required in all cases.", + "type": "string" + }, + "namespace": { + "description": "Namespace is the metadata.namespace of the referenced ConfigMap. This field is required in all cases.", + "type": "string" + }, + "resourceVersion": { + "description": "ResourceVersion is the metadata.ResourceVersion of the referenced ConfigMap. This field is forbidden in Node.Spec.", + "type": "string" + }, + "uid": { + "description": "UID is the metadata.UID of the referenced ConfigMap. This field is currently reqired in Node.Spec.", + "type": "string" + } + } + }, "io.k8s.api.core.v1.ConfigMapProjection": { "description": "Adapts a ConfigMap into a projected volume.\n\nThe contents of the target ConfigMap's Data field will be presented in a projected volume as files using the keys in the Data field as the file names, unless the items element is populated with specific mappings of keys to paths. Note that this is identical to a configmap volume source without the default mode.", "properties": { @@ -77242,25 +77272,11 @@ "io.k8s.api.core.v1.NodeConfigSource": { "description": "NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil.", "properties": { - "apiVersion": { - "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", - "type": "string" - }, - "configMapRef": { - "$ref": "#/definitions/io.k8s.api.core.v1.ObjectReference" - }, - "kind": { - "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", - "type": "string" + "configMap": { + "description": "ConfigMap is a reference to a Node's ConfigMap", + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapNodeConfigSource" } - }, - "x-kubernetes-group-version-kind": [ - { - "group": "", - "kind": "NodeConfigSource", - "version": "v1" - } - ] + } }, "io.k8s.api.core.v1.NodeDaemonEndpoints": { "description": "NodeDaemonEndpoints lists ports opened by daemons running on the Node.", diff --git a/api/swagger-spec/v1.json b/api/swagger-spec/v1.json index f5d4303c78f..b1a691f2e01 100644 --- a/api/swagger-spec/v1.json +++ b/api/swagger-spec/v1.json @@ -18718,16 +18718,40 @@ "id": "v1.NodeConfigSource", "description": "NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil.", "properties": { - "kind": { + "configMap": { + "$ref": "v1.ConfigMapNodeConfigSource", + "description": "ConfigMap is a reference to a Node's ConfigMap" + } + } + }, + "v1.ConfigMapNodeConfigSource": { + "id": "v1.ConfigMapNodeConfigSource", + "description": "ConfigMapNodeConfigSource contains the information to reference a ConfigMap as a config source for the Node.", + "required": [ + "namespace", + "name", + "kubeletConfigKey" + ], + "properties": { + "namespace": { "type": "string", - "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds" + "description": "Namespace is the metadata.namespace of the referenced ConfigMap. This field is required in all cases." }, - "apiVersion": { + "name": { "type": "string", - "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources" + "description": "Name is the metadata.name of the referenced ConfigMap. This field is required in all cases." }, - "configMapRef": { - "$ref": "v1.ObjectReference" + "uid": { + "type": "string", + "description": "UID is the metadata.UID of the referenced ConfigMap. This field is currently reqired in Node.Spec." + }, + "resourceVersion": { + "type": "string", + "description": "ResourceVersion is the metadata.ResourceVersion of the referenced ConfigMap. This field is forbidden in Node.Spec." + }, + "kubeletConfigKey": { + "type": "string", + "description": "KubeletConfigKey declares which key of the referenced ConfigMap corresponds to the KubeletConfiguration structure This field is required in all cases." } } }, diff --git a/build/workspace_mirror.bzl b/build/workspace_mirror.bzl index bbc02c88a76..49224913082 100644 --- a/build/workspace_mirror.bzl +++ b/build/workspace_mirror.bzl @@ -15,31 +15,35 @@ prefix = "https://storage.googleapis.com/k8s-bazel-cache/" def mirror(url): - """Try downloading a URL from a GCS mirror first, then from the original. + """Try downloading a URL from a GCS mirror first, then from the original. - Update the GCS bucket using bazel run //hack:update-mirror""" - return [prefix + url, url] + Update the GCS bucket using bazel run //hack:update-mirror""" + return [prefix + url, url] -# This function only gives proper results when executed from WORKSPACE, -# but the data is needed in sh_binary, which can only be in a BUILD file. -# Thus, it is be exported by a repository_rule (which executes in WORKSPACE) -# to be used by the sh_binary. def mirror_urls(): - urls = [] - for k, v in native.existing_rules().items(): - us = list(v.get('urls', [])) - if 'url' in v: - us.append(v['url']) - for u in us: - if u and not u.startswith(prefix): - urls.append(u) - return sorted(urls) + # This function only gives proper results when executed from WORKSPACE, + # but the data is needed in sh_binary, which can only be in a BUILD file. + # Thus, it is be exported by a repository_rule (which executes in WORKSPACE) + # to be used by the sh_binary. + urls = [] + for k, v in native.existing_rules().items(): + us = list(v.get("urls", [])) + if "url" in v: + us.append(v["url"]) + for u in us: + if u and not u.startswith(prefix): + urls.append(u) + return sorted(urls) def export_urls_impl(repo_ctx): - repo_ctx.file(repo_ctx.path("BUILD.bazel"), """ + repo_ctx.file(repo_ctx.path("BUILD.bazel"), """ exports_files(glob(["**"]), visibility=["//visibility:public"]) """) - repo_ctx.file(repo_ctx.path("urls.txt"), content="\n".join(repo_ctx.attr.urls)) + repo_ctx.file( + repo_ctx.path("urls.txt"), + # Add a trailing newline, since the "while read" loop needs it + content = ("\n".join(repo_ctx.attr.urls) + "\n"), + ) _export_urls = repository_rule( attrs = { @@ -50,4 +54,4 @@ _export_urls = repository_rule( ) def export_urls(name): - return _export_urls(name=name, urls=mirror_urls()) + return _export_urls(name = name, urls = mirror_urls()) diff --git a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml index 4aa453e17ba..6bb8230dcb1 100644 --- a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml @@ -36,30 +36,30 @@ data: apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: heapster-v1.5.2 + name: heapster-v1.5.3 namespace: kube-system labels: k8s-app: heapster kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile - version: v1.5.2 + version: v1.5.3 spec: replicas: 1 selector: matchLabels: k8s-app: heapster - version: v1.5.2 + version: v1.5.3 template: metadata: labels: k8s-app: heapster - version: v1.5.2 + version: v1.5.3 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical containers: - - image: k8s.gcr.io/heapster-amd64:v1.5.2 + - image: k8s.gcr.io/heapster-amd64:v1.5.3 name: heapster livenessProbe: httpGet: @@ -72,7 +72,7 @@ spec: - /heapster - --source=kubernetes.summary_api:'' - --sink=gcm - - image: k8s.gcr.io/heapster-amd64:v1.5.2 + - image: k8s.gcr.io/heapster-amd64:v1.5.3 name: eventer command: - /eventer @@ -107,7 +107,7 @@ spec: - --memory={{ base_metrics_memory }} - --extra-memory={{metrics_memory_per_node}}Mi - --threshold=5 - - --deployment=heapster-v1.5.2 + - --deployment=heapster-v1.5.3 - --container=heapster - --poll-period=300000 - --estimator=exponential @@ -140,7 +140,7 @@ spec: - --memory={{base_eventer_memory}} - --extra-memory={{eventer_memory_per_node}}Ki - --threshold=5 - - --deployment=heapster-v1.5.2 + - --deployment=heapster-v1.5.3 - --container=eventer - --poll-period=300000 - --estimator=exponential diff --git a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml index 1ac542cc2a6..634861a53c9 100644 --- a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml +++ b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml @@ -36,30 +36,30 @@ data: apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: heapster-v1.5.2 + name: heapster-v1.5.3 namespace: kube-system labels: k8s-app: heapster kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile - version: v1.5.2 + version: v1.5.3 spec: replicas: 1 selector: matchLabels: k8s-app: heapster - version: v1.5.2 + version: v1.5.3 template: metadata: labels: k8s-app: heapster - version: v1.5.2 + version: v1.5.3 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical containers: - - image: k8s.gcr.io/heapster-amd64:v1.5.2 + - image: k8s.gcr.io/heapster-amd64:v1.5.3 name: heapster livenessProbe: httpGet: @@ -73,7 +73,7 @@ spec: - --source=kubernetes.summary_api:'' - --sink=influxdb:http://monitoring-influxdb:8086 - --sink=gcm:?metrics=autoscaling - - image: k8s.gcr.io/heapster-amd64:v1.5.2 + - image: k8s.gcr.io/heapster-amd64:v1.5.3 name: eventer command: - /eventer @@ -108,7 +108,7 @@ spec: - --memory={{ base_metrics_memory }} - --extra-memory={{ metrics_memory_per_node }}Mi - --threshold=5 - - --deployment=heapster-v1.5.2 + - --deployment=heapster-v1.5.3 - --container=heapster - --poll-period=300000 - --estimator=exponential @@ -141,7 +141,7 @@ spec: - --memory={{ base_eventer_memory }} - --extra-memory={{ eventer_memory_per_node }}Ki - --threshold=5 - - --deployment=heapster-v1.5.2 + - --deployment=heapster-v1.5.3 - --container=eventer - --poll-period=300000 - --estimator=exponential diff --git a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml index 722c842cf6a..92c306c1aa6 100644 --- a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml @@ -36,30 +36,30 @@ data: apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: heapster-v1.5.2 + name: heapster-v1.5.3 namespace: kube-system labels: k8s-app: heapster kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile - version: v1.5.2 + version: v1.5.3 spec: replicas: 1 selector: matchLabels: k8s-app: heapster - version: v1.5.2 + version: v1.5.3 template: metadata: labels: k8s-app: heapster - version: v1.5.2 + version: v1.5.3 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical containers: - - image: k8s.gcr.io/heapster-amd64:v1.5.2 + - image: k8s.gcr.io/heapster-amd64:v1.5.3 name: heapster livenessProbe: httpGet: @@ -72,7 +72,7 @@ spec: - /heapster - --source=kubernetes.summary_api:'' - --sink=influxdb:http://monitoring-influxdb:8086 - - image: k8s.gcr.io/heapster-amd64:v1.5.2 + - image: k8s.gcr.io/heapster-amd64:v1.5.3 name: eventer command: - /eventer @@ -107,7 +107,7 @@ spec: - --memory={{ base_metrics_memory }} - --extra-memory={{ metrics_memory_per_node }}Mi - --threshold=5 - - --deployment=heapster-v1.5.2 + - --deployment=heapster-v1.5.3 - --container=heapster - --poll-period=300000 - --estimator=exponential @@ -140,7 +140,7 @@ spec: - --memory={{ base_eventer_memory }} - --extra-memory={{ eventer_memory_per_node }}Ki - --threshold=5 - - --deployment=heapster-v1.5.2 + - --deployment=heapster-v1.5.3 - --container=eventer - --poll-period=300000 - --estimator=exponential diff --git a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml index af0c8c535ed..fb586492916 100644 --- a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml @@ -23,30 +23,30 @@ data: apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: heapster-v1.5.2 + name: heapster-v1.5.3 namespace: kube-system labels: k8s-app: heapster kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile - version: v1.5.2 + version: v1.5.3 spec: replicas: 1 selector: matchLabels: k8s-app: heapster - version: v1.5.2 + version: v1.5.3 template: metadata: labels: k8s-app: heapster - version: v1.5.2 + version: v1.5.3 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical containers: - - image: k8s.gcr.io/heapster-amd64:v1.5.2 + - image: k8s.gcr.io/heapster-amd64:v1.5.3 name: heapster livenessProbe: httpGet: @@ -108,7 +108,7 @@ spec: - --memory={{ base_metrics_memory }} - --extra-memory={{metrics_memory_per_node}}Mi - --threshold=5 - - --deployment=heapster-v1.5.2 + - --deployment=heapster-v1.5.3 - --container=heapster - --poll-period=300000 - --estimator=exponential diff --git a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml index 14b18d12dde..fa794c54af9 100644 --- a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml @@ -23,30 +23,30 @@ data: apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: heapster-v1.5.2 + name: heapster-v1.5.3 namespace: kube-system labels: k8s-app: heapster kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile - version: v1.5.2 + version: v1.5.3 spec: replicas: 1 selector: matchLabels: k8s-app: heapster - version: v1.5.2 + version: v1.5.3 template: metadata: labels: k8s-app: heapster - version: v1.5.2 + version: v1.5.3 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical containers: - - image: k8s.gcr.io/heapster-amd64:v1.5.2 + - image: k8s.gcr.io/heapster-amd64:v1.5.3 name: heapster livenessProbe: httpGet: @@ -87,7 +87,7 @@ spec: - --memory={{ base_metrics_memory }} - --extra-memory={{ metrics_memory_per_node }}Mi - --threshold=5 - - --deployment=heapster-v1.5.2 + - --deployment=heapster-v1.5.3 - --container=heapster - --poll-period=300000 - --estimator=exponential diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/run.sh b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/run.sh index 355701315c0..be0011b61d0 100755 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/run.sh +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/run.sh @@ -20,4 +20,4 @@ # For systems without journald mkdir -p /var/log/journal -/usr/local/bin/fluentd $@ +exec /usr/local/bin/fluentd $@ diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml index 1cb61a318be..1b28506cb1b 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml @@ -38,13 +38,13 @@ spec: - name: config-volume mountPath: /etc/google-fluentd/config.d env: - - name: STACKDRIVER_METADATA_AGENT_URL - value: {{ stackdriver_metadata_agent_url }} - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName + - name: STACKDRIVER_METADATA_AGENT_URL + value: http://$(NODE_NAME):8799 # Liveness probe is aimed to help in situarions where fluentd # silently hangs for no apparent reasons until manual restart. # The idea of this probe is that if fluentd is not queueing or diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index eb2bd973d7b..1ec7d832e20 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -405,7 +405,7 @@ HEAPSTER_GCP_BASE_CPU="${HEAPSTER_GCP_BASE_CPU:-80m}" HEAPSTER_GCP_CPU_PER_NODE="${HEAPSTER_GCP_CPU_PER_NODE:-0.5}" # Adding to PROVIDER_VARS, since this is GCP-specific. -PROVIDER_VARS="${PROVIDER_VARS:-} FLUENTD_GCP_VERSION FLUENTD_GCP_MEMORY_LIMIT FLUENTD_GCP_CPU_REQUEST FLUENTD_GCP_MEMORY_REQUEST HEAPSTER_GCP_BASE_MEMORY HEAPSTER_GCP_MEMORY_PER_NODE HEAPSTER_GCP_BASE_CPU HEAPSTER_GCP_CPU_PER_NODE CUSTOM_KUBE_DASHBOARD_BANNER LOGGING_STACKDRIVER_RESOURCE_TYPES STACKDRIVER_METADATA_AGENT_URL" +PROVIDER_VARS="${PROVIDER_VARS:-} FLUENTD_GCP_VERSION FLUENTD_GCP_MEMORY_LIMIT FLUENTD_GCP_CPU_REQUEST FLUENTD_GCP_MEMORY_REQUEST HEAPSTER_GCP_BASE_MEMORY HEAPSTER_GCP_MEMORY_PER_NODE HEAPSTER_GCP_BASE_CPU HEAPSTER_GCP_CPU_PER_NODE CUSTOM_KUBE_DASHBOARD_BANNER LOGGING_STACKDRIVER_RESOURCE_TYPES" # Fluentd configuration for node-journal ENABLE_NODE_JOURNAL="${ENABLE_NODE_JOURNAL:-false}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index b800e855f82..08b71c70042 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -76,7 +76,7 @@ ALLOWED_NOTREADY_NODES="${ALLOWED_NOTREADY_NODES:-$((NUM_NODES / 100))}" # Also please update corresponding image for node e2e at: # https://github.com/kubernetes/kubernetes/blob/master/test/e2e_node/jenkins/image-config.yaml CVM_VERSION=${CVM_VERSION:-container-vm-v20170627} -GCI_VERSION=${KUBE_GCI_VERSION:-cos-beta-66-10452-28-0} +GCI_VERSION=${KUBE_GCI_VERSION:-cos-stable-65-10323-64-0} MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-} MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud} NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}} @@ -421,7 +421,7 @@ HEAPSTER_GCP_BASE_CPU="${HEAPSTER_GCP_BASE_CPU:-80m}" HEAPSTER_GCP_CPU_PER_NODE="${HEAPSTER_GCP_CPU_PER_NODE:-0.5}" # Adding to PROVIDER_VARS, since this is GCP-specific. -PROVIDER_VARS="${PROVIDER_VARS:-} FLUENTD_GCP_VERSION FLUENTD_GCP_MEMORY_LIMIT FLUENTD_GCP_CPU_REQUEST FLUENTD_GCP_MEMORY_REQUEST HEAPSTER_GCP_BASE_MEMORY HEAPSTER_GCP_MEMORY_PER_NODE HEAPSTER_GCP_BASE_CPU HEAPSTER_GCP_CPU_PER_NODE CUSTOM_KUBE_DASHBOARD_BANNER LOGGING_STACKDRIVER_RESOURCE_TYPES STACKDRIVER_METADATA_AGENT_URL" +PROVIDER_VARS="${PROVIDER_VARS:-} FLUENTD_GCP_VERSION FLUENTD_GCP_MEMORY_LIMIT FLUENTD_GCP_CPU_REQUEST FLUENTD_GCP_MEMORY_REQUEST HEAPSTER_GCP_BASE_MEMORY HEAPSTER_GCP_MEMORY_PER_NODE HEAPSTER_GCP_BASE_CPU HEAPSTER_GCP_CPU_PER_NODE CUSTOM_KUBE_DASHBOARD_BANNER LOGGING_STACKDRIVER_RESOURCE_TYPES" # Fluentd configuration for node-journal ENABLE_NODE_JOURNAL="${ENABLE_NODE_JOURNAL:-false}" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 79f3432616b..31cbc34b75c 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -892,8 +892,9 @@ function create-kubelet-kubeconfig() { echo "Must provide API server address to create Kubelet kubeconfig file!" exit 1 fi - echo "Creating kubelet kubeconfig file" - cat </var/lib/kubelet/bootstrap-kubeconfig + if [[ "${CREATE_BOOTSTRAP_KUBECONFIG:-true}" == "true" ]]; then + echo "Creating kubelet bootstrap-kubeconfig file" + cat </var/lib/kubelet/bootstrap-kubeconfig apiVersion: v1 kind: Config users: @@ -913,6 +914,13 @@ contexts: name: service-account-context current-context: service-account-context EOF + elif [[ "${FETCH_BOOTSTRAP_KUBECONFIG:-false}" == "true" ]]; then + echo "Fetching kubelet bootstrap-kubeconfig file from metadata" + get-metadata-value "instance/attributes/bootstrap-kubeconfig" >/var/lib/kubelet/bootstrap-kubeconfig + else + echo "Fetching kubelet kubeconfig file from metadata" + get-metadata-value "instance/attributes/kubeconfig" >/var/lib/kubelet/kubeconfig + fi } # Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY @@ -1584,6 +1592,9 @@ function start-kube-apiserver { if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then params+=" --enable-logs-handler=false" fi + if [[ -n "${APISERVER_KUBELET_CA:-}" ]]; then + params+=" --kubelet-certificate-authority=${APISERVER_KUBELET_CA}" + fi local admission_controller_config_mount="" local admission_controller_config_volume="" @@ -1612,7 +1623,7 @@ function start-kube-apiserver { params+=" --feature-gates=${FEATURE_GATES}" fi if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then - local -r vm_external_ip=$(curl --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip") + local -r vm_external_ip=$(get-metadata-value "instance/network-interfaces/0/access-configs/0/external-ip") if [[ -n "${PROXY_SSH_USER:-}" ]]; then params+=" --advertise-address=${vm_external_ip}" params+=" --ssh-user=${PROXY_SSH_USER}" @@ -2008,6 +2019,20 @@ function download-extra-addons { "${curl_cmd[@]}" } +# A function that fetches a GCE metadata value and echoes it out. +# +# $1: URL path after /computeMetadata/v1/ (without heading slash). +function get-metadata-value { + curl \ + --retry 5 \ + --retry-delay 3 \ + ${CURL_RETRY_CONNREFUSED} \ + --fail \ + --silent \ + -H 'Metadata-Flavor: Google' \ + "http://metadata/computeMetadata/v1/${1}" +} + # A helper function for copying manifests and setting dir/files # permissions. # @@ -2161,12 +2186,6 @@ function setup-fluentd { sed -i -e "s@{{ fluentd_gcp_configmap_name }}@${fluentd_gcp_configmap_name}@g" "${fluentd_gcp_yaml}" fluentd_gcp_version="${FLUENTD_GCP_VERSION:-0.2-1.5.30-1-k8s}" sed -i -e "s@{{ fluentd_gcp_version }}@${fluentd_gcp_version}@g" "${fluentd_gcp_yaml}" - if [[ "${STACKDRIVER_METADATA_AGENT_URL:-}" != "" ]]; then - metadata_agent_url="${STACKDRIVER_METADATA_AGENT_URL}" - else - metadata_agent_url="http://${HOSTNAME}:8799" - fi - sed -i -e "s@{{ stackdriver_metadata_agent_url }}@${metadata_agent_url}@g" "${fluentd_gcp_yaml}" update-prometheus-to-sd-parameters ${fluentd_gcp_yaml} start-fluentd-resource-update ${fluentd_gcp_yaml} update-container-runtime ${fluentd_gcp_configmap_yaml} @@ -2590,4 +2609,4 @@ if [[ "$#" -eq 1 && "${1}" == "--source-only" ]]; then : else main "${@}" -fi \ No newline at end of file +fi diff --git a/cluster/gce/upgrade.sh b/cluster/gce/upgrade.sh index 2d7f7883464..0a44596d683 100755 --- a/cluster/gce/upgrade.sh +++ b/cluster/gce/upgrade.sh @@ -342,15 +342,21 @@ function do-single-node-upgrade() { sleep 1 done - # Wait for the node to not have SchedulingDisabled=True and also to have - # Ready=True. + # Uncordon the node. + echo "== Uncordon ${instance}. == " >&2 + local uncordon_rc + "${KUBE_ROOT}/cluster/kubectl.sh" uncordon "${instance}" \ + && uncordon_rc=$? || uncordon_rc=$? + if [[ "${uncordon_rc}" != 0 ]]; then + echo "== FAILED to uncordon ${instance} ==" + return ${uncordon_rc} + fi + + # Wait for the node to have Ready=True. echo "== Waiting for ${instance} to become ready. ==" >&2 while true; do - local cordoned=$("${KUBE_ROOT}/cluster/kubectl.sh" get node "${instance}" --output='jsonpath={.status.conditions[?(@.type == "SchedulingDisabled")].status}') local ready=$("${KUBE_ROOT}/cluster/kubectl.sh" get node "${instance}" --output='jsonpath={.status.conditions[?(@.type == "Ready")].status}') - if [[ "${cordoned}" == 'True' ]]; then - echo "Node ${instance} is still not ready: SchedulingDisabled=${ready}" - elif [[ "${ready}" != 'True' ]]; then + if [[ "${ready}" != 'True' ]]; then echo "Node ${instance} is still not ready: Ready=${ready}" else echo "Node ${instance} Ready=${ready}" diff --git a/cluster/juju/layers/kubernetes-master/layer.yaml b/cluster/juju/layers/kubernetes-master/layer.yaml index ed496bc31af..9a53451dce5 100644 --- a/cluster/juju/layers/kubernetes-master/layer.yaml +++ b/cluster/juju/layers/kubernetes-master/layer.yaml @@ -1,6 +1,7 @@ repo: https://github.com/kubernetes/kubernetes.git includes: - 'layer:basic' + - 'layer:status' - 'layer:snap' - 'layer:tls-client' - 'layer:leadership' @@ -15,6 +16,7 @@ includes: - 'interface:kube-dns' - 'interface:kube-control' - 'interface:public-address' + - 'interface:aws' options: basic: packages: diff --git a/cluster/juju/layers/kubernetes-master/metadata.yaml b/cluster/juju/layers/kubernetes-master/metadata.yaml index fa1f62b74c4..dfba03c99aa 100644 --- a/cluster/juju/layers/kubernetes-master/metadata.yaml +++ b/cluster/juju/layers/kubernetes-master/metadata.yaml @@ -40,6 +40,8 @@ requires: interface: public-address ceph-storage: interface: ceph-admin + aws: + interface: aws resources: kubectl: type: file diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 18aa3564b67..36207665d1a 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -39,6 +39,7 @@ from charms.reactive import hook from charms.reactive import remove_state from charms.reactive import set_state from charms.reactive import is_state +from charms.reactive import endpoint_from_flag from charms.reactive import when, when_any, when_not from charms.reactive.helpers import data_changed, any_file_changed from charms.kubernetes.common import get_version @@ -1224,6 +1225,9 @@ def configure_apiserver(etcd_connection_string, leader_etcd_version): api_opts['enable-aggregator-routing'] = 'true' api_opts['client-ca-file'] = ca_cert_path + if is_state('endpoint.aws.ready'): + api_opts['cloud-provider'] = 'aws' + configure_kubernetes_service('kube-apiserver', api_opts, 'api-extra-args') restart_apiserver() @@ -1245,6 +1249,9 @@ def configure_controller_manager(): controller_opts['service-account-private-key-file'] = \ '/root/cdk/serviceaccount.key' + if is_state('endpoint.aws.ready'): + controller_opts['cloud-provider'] = 'aws' + configure_kubernetes_service('kube-controller-manager', controller_opts, 'controller-manager-extra-args') restart_controller_manager() @@ -1377,3 +1384,61 @@ def getStorageBackend(): if storage_backend == 'auto': storage_backend = leader_get('auto_storage_backend') return storage_backend + + +@when('leadership.is_leader') +@when_not('leadership.set.cluster_tag') +def create_cluster_tag(): + cluster_tag = 'kubernetes-{}'.format(token_generator()) + leader_set(cluster_tag=cluster_tag) + + +@when('leadership.set.cluster_tag', + 'kube-control.connected') +@when_not('kubernetes-master.cluster-tag-sent') +def send_cluster_tag(): + cluster_tag = leader_get('cluster_tag') + kube_control = endpoint_from_flag('kube-control.connected') + kube_control.set_cluster_tag(cluster_tag) + set_state('kubernetes-master.cluster-tag-sent') + + +@when_not('kube-control.connected') +def clear_cluster_tag_sent(): + remove_state('kubernetes-master.cluster-tag-sent') + + +@when('endpoint.aws.joined', + 'leadership.set.cluster_tag') +@when_not('kubernetes-master.aws-request-sent') +def request_integration(): + hookenv.status_set('maintenance', 'requesting aws integration') + aws = endpoint_from_flag('endpoint.aws.joined') + cluster_tag = leader_get('cluster_tag') + aws.tag_instance({ + 'KubernetesCluster': cluster_tag, + 'k8s.io/role/master': 'true', + }) + aws.tag_instance_security_group({ + 'KubernetesCluster': cluster_tag, + }) + aws.enable_instance_inspection() + aws.enable_network_management() + aws.enable_dns_management() + aws.enable_load_balancer_management() + aws.enable_block_storage_management() + aws.enable_object_storage_management(['kubernetes-*']) + set_state('kubernetes-master.aws-request-sent') + hookenv.status_set('waiting', 'waiting for aws integration') + + +@when_not('endpoint.aws.joined') +def clear_requested_integration(): + remove_state('kubernetes-master.aws-request-sent') + + +@when('endpoint.aws.ready') +@when_not('kubernetes-master.restarted-for-aws') +def restart_for_aws(): + set_state('kubernetes-master.restarted-for-aws') + remove_state('kubernetes-master.components.started') # force restart diff --git a/cluster/juju/layers/kubernetes-worker/actions/registry b/cluster/juju/layers/kubernetes-worker/actions/registry index 7a0ee150abd..81c8bc03809 100755 --- a/cluster/juju/layers/kubernetes-worker/actions/registry +++ b/cluster/juju/layers/kubernetes-worker/actions/registry @@ -25,7 +25,7 @@ from base64 import b64encode from charmhelpers.core.hookenv import action_get from charmhelpers.core.hookenv import action_set from charms.templating.jinja2 import render -from subprocess import call +from subprocess import call, check_output os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin') @@ -33,6 +33,9 @@ deletion = action_get('delete') context = {} +arch = check_output(['dpkg', '--print-architecture']).rstrip() +context['arch'] = arch.decode('utf-8') + # These config options must be defined in the case of a creation param_error = False for param in ('tlscert', 'tlskey', 'domain', 'htpasswd', 'htpasswd-plain'): diff --git a/cluster/juju/layers/kubernetes-worker/layer.yaml b/cluster/juju/layers/kubernetes-worker/layer.yaml index b9f3768a4e5..26aaa4d40ae 100644 --- a/cluster/juju/layers/kubernetes-worker/layer.yaml +++ b/cluster/juju/layers/kubernetes-worker/layer.yaml @@ -1,6 +1,7 @@ repo: https://github.com/kubernetes/kubernetes.git includes: - 'layer:basic' + - 'layer:status' - 'layer:debug' - 'layer:snap' - 'layer:docker' @@ -12,6 +13,7 @@ includes: - 'interface:kubernetes-cni' - 'interface:kube-dns' - 'interface:kube-control' + - 'interface:aws' config: deletes: - install_from_upstream diff --git a/cluster/juju/layers/kubernetes-worker/metadata.yaml b/cluster/juju/layers/kubernetes-worker/metadata.yaml index 48715a4d988..cc09efb8a4d 100644 --- a/cluster/juju/layers/kubernetes-worker/metadata.yaml +++ b/cluster/juju/layers/kubernetes-worker/metadata.yaml @@ -28,6 +28,8 @@ requires: interface: kube-dns kube-control: interface: kube-control + aws: + interface: aws provides: cni: interface: kubernetes-cni diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py index 13d0e4a5f2b..6540b6250bc 100644 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py @@ -29,6 +29,7 @@ from socket import gethostname, getfqdn from charms import layer from charms.layer import snap from charms.reactive import hook +from charms.reactive import endpoint_from_flag from charms.reactive import set_state, remove_state, is_state from charms.reactive import when, when_any, when_not @@ -623,6 +624,9 @@ def configure_kubelet(dns, ingress_ip): 'to kubelet') kubelet_opts['feature-gates'] = 'DevicePlugins=true' + if is_state('endpoint.aws.ready'): + kubelet_opts['cloud-provider'] = 'aws' + configure_kubernetes_service('kubelet', kubelet_opts, 'kubelet-extra-args') @@ -738,7 +742,7 @@ def launch_default_ingress_controller(): "k8s.gcr.io/nginx-ingress-controller-arm64:0.9.0-beta.15" else: context['ingress_image'] = \ - "k8s.gcr.io/nginx-ingress-controller:0.9.0-beta.15" # noqa + "k8s.gcr.io/nginx-ingress-controller:0.9.0-beta.15" # noqa if get_version('kubelet') < (1, 9): context['daemonset_api_version'] = 'extensions/v1beta1' else: @@ -1067,3 +1071,39 @@ def remove_label(label): retry = 'Failed to remove label {0}. Will retry.'.format(label) if not persistent_call(cmd, retry): raise ApplyNodeLabelFailed(retry) + + +@when('endpoint.aws.joined', + 'kube-control.cluster_tag.available') +@when_not('kubernetes-worker.aws-request-sent') +def request_integration(): + kube_control = endpoint_from_flag('kube-control.cluster_tag.available') + hookenv.status_set('maintenance', 'requesting aws integration') + aws = endpoint_from_flag('endpoint.aws.joined') + cluster_tag = kube_control.get_cluster_tag() + aws.tag_instance({ + 'KubernetesCluster': cluster_tag, + }) + aws.tag_instance_security_group({ + 'KubernetesCluster': cluster_tag, + }) + aws.tag_instance_subnet({ + 'KubernetesCluster': cluster_tag, + }) + aws.enable_instance_inspection() + aws.enable_dns_management() + aws.enable_object_storage_management(['kubernetes-*']) + set_state('kubernetes-worker.aws-request-sent') + hookenv.status_set('waiting', 'waiting for aws integration') + + +@when_not('endpoint.aws.joined') +def clear_requested_integration(): + remove_state('kubernetes-worker.aws-request-sent') + + +@when('endpoint.aws.ready') +@when_not('kubernetes-worker.restarted-for-aws') +def restart_for_aws(): + set_state('kubernetes-worker.restarted-for-aws') + set_state('kubernetes-worker.restart-needed') diff --git a/cluster/juju/layers/kubernetes-worker/templates/registry.yaml b/cluster/juju/layers/kubernetes-worker/templates/registry.yaml index d24b713ce7f..2503f3abfb2 100644 --- a/cluster/juju/layers/kubernetes-worker/templates/registry.yaml +++ b/cluster/juju/layers/kubernetes-worker/templates/registry.yaml @@ -37,7 +37,7 @@ spec: spec: containers: - name: registry - image: registry:2 + image: cdkbot/registry-{{ arch }}:2.6 resources: # keep request = limit to keep this container in guaranteed class limits: diff --git a/cmd/clicheck/BUILD b/cmd/clicheck/BUILD index cafcaec3a05..db6e92a3c2d 100644 --- a/cmd/clicheck/BUILD +++ b/cmd/clicheck/BUILD @@ -17,7 +17,6 @@ go_library( importpath = "k8s.io/kubernetes/cmd/clicheck", deps = [ "//pkg/kubectl/cmd:go_default_library", - "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/cmd/util/sanity:go_default_library", ], ) diff --git a/cmd/clicheck/check_cli_conventions.go b/cmd/clicheck/check_cli_conventions.go index 524f9f05dc3..17d109d512d 100644 --- a/cmd/clicheck/check_cli_conventions.go +++ b/cmd/clicheck/check_cli_conventions.go @@ -22,7 +22,6 @@ import ( "os" "k8s.io/kubernetes/pkg/kubectl/cmd" - cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" cmdsanity "k8s.io/kubernetes/pkg/kubectl/cmd/util/sanity" ) @@ -33,7 +32,7 @@ var ( func main() { var errorCount int - kubectl := cmd.NewKubectlCommand(cmdutil.NewFactory(nil), os.Stdin, ioutil.Discard, ioutil.Discard) + kubectl := cmd.NewKubectlCommand(os.Stdin, ioutil.Discard, ioutil.Discard) errors := cmdsanity.RunCmdChecks(kubectl, cmdsanity.AllCmdChecks, []string{}) for _, err := range errors { errorCount++ diff --git a/cmd/gendocs/BUILD b/cmd/gendocs/BUILD index 44f9a9a2de4..04897fb3126 100644 --- a/cmd/gendocs/BUILD +++ b/cmd/gendocs/BUILD @@ -18,7 +18,6 @@ go_library( deps = [ "//cmd/genutils:go_default_library", "//pkg/kubectl/cmd:go_default_library", - "//pkg/kubectl/cmd/util:go_default_library", "//vendor/github.com/spf13/cobra/doc:go_default_library", ], ) diff --git a/cmd/gendocs/gen_kubectl_docs.go b/cmd/gendocs/gen_kubectl_docs.go index 39c1d8f39df..dc03a0e8260 100644 --- a/cmd/gendocs/gen_kubectl_docs.go +++ b/cmd/gendocs/gen_kubectl_docs.go @@ -24,7 +24,6 @@ import ( "github.com/spf13/cobra/doc" "k8s.io/kubernetes/cmd/genutils" "k8s.io/kubernetes/pkg/kubectl/cmd" - cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" ) func main() { @@ -47,6 +46,6 @@ func main() { // regardless of where we run. os.Setenv("HOME", "/home/username") // TODO os.Stdin should really be something like ioutil.Discard, but a Reader - kubectl := cmd.NewKubectlCommand(cmdutil.NewFactory(nil), os.Stdin, ioutil.Discard, ioutil.Discard) + kubectl := cmd.NewKubectlCommand(os.Stdin, ioutil.Discard, ioutil.Discard) doc.GenMarkdownTree(kubectl, outDir) } diff --git a/cmd/genman/BUILD b/cmd/genman/BUILD index e9ee6dcba47..e71a704e22f 100644 --- a/cmd/genman/BUILD +++ b/cmd/genman/BUILD @@ -25,7 +25,6 @@ go_library( "//cmd/kubeadm/app/cmd:go_default_library", "//cmd/kubelet/app:go_default_library", "//pkg/kubectl/cmd:go_default_library", - "//pkg/kubectl/cmd/util:go_default_library", "//vendor/github.com/cpuguy83/go-md2man/md2man:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", diff --git a/cmd/genman/gen_kube_man.go b/cmd/genman/gen_kube_man.go index 416d4a9f9fa..8ccc994f31c 100644 --- a/cmd/genman/gen_kube_man.go +++ b/cmd/genman/gen_kube_man.go @@ -35,7 +35,6 @@ import ( kubeadmapp "k8s.io/kubernetes/cmd/kubeadm/app/cmd" kubeletapp "k8s.io/kubernetes/cmd/kubelet/app" kubectlcmd "k8s.io/kubernetes/pkg/kubectl/cmd" - kubectlcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" ) func main() { @@ -106,7 +105,7 @@ func main() { case "kubectl": // generate manpage for kubectl // TODO os.Stdin should really be something like ioutil.Discard, but a Reader - kubectl := kubectlcmd.NewKubectlCommand(kubectlcmdutil.NewFactory(nil), os.Stdin, ioutil.Discard, ioutil.Discard) + kubectl := kubectlcmd.NewKubectlCommand(os.Stdin, ioutil.Discard, ioutil.Discard) genMarkdown(kubectl, "", outDir) for _, c := range kubectl.Commands() { genMarkdown(c, "kubectl", outDir) diff --git a/cmd/genyaml/BUILD b/cmd/genyaml/BUILD index 0571d54d89a..924e6144d2d 100644 --- a/cmd/genyaml/BUILD +++ b/cmd/genyaml/BUILD @@ -18,7 +18,6 @@ go_library( deps = [ "//cmd/genutils:go_default_library", "//pkg/kubectl/cmd:go_default_library", - "//pkg/kubectl/cmd/util:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/gopkg.in/yaml.v2:go_default_library", diff --git a/cmd/genyaml/gen_kubectl_yaml.go b/cmd/genyaml/gen_kubectl_yaml.go index 376a1c7122f..c95ee2a3d59 100644 --- a/cmd/genyaml/gen_kubectl_yaml.go +++ b/cmd/genyaml/gen_kubectl_yaml.go @@ -27,7 +27,6 @@ import ( "gopkg.in/yaml.v2" "k8s.io/kubernetes/cmd/genutils" "k8s.io/kubernetes/pkg/kubectl/cmd" - cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" ) type cmdOption struct { @@ -66,7 +65,7 @@ func main() { // regardless of where we run. os.Setenv("HOME", "/home/username") // TODO os.Stdin should really be something like ioutil.Discard, but a Reader - kubectl := cmd.NewKubectlCommand(cmdutil.NewFactory(nil), os.Stdin, ioutil.Discard, ioutil.Discard) + kubectl := cmd.NewKubectlCommand(os.Stdin, ioutil.Discard, ioutil.Discard) genYaml(kubectl, "", outDir) for _, c := range kubectl.Commands() { genYaml(c, "kubectl", outDir) diff --git a/cmd/importverifier/importverifier.go b/cmd/importverifier/importverifier.go index ee3f07c7bb0..c73dad68c2a 100644 --- a/cmd/importverifier/importverifier.go +++ b/cmd/importverifier/importverifier.go @@ -236,7 +236,7 @@ func resolvePackageTree(treeBase string) ([]Package, error) { if err != nil { var message string if ee, ok := err.(*exec.ExitError); ok { - message = fmt.Sprintf("%v\n%v", ee, ee.Stderr) + message = fmt.Sprintf("%v\n%v", ee, string(ee.Stderr)) } else { message = fmt.Sprintf("%v", err) } diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index 5e37b561d0f..9ea3abbf64e 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -78,11 +78,11 @@ go_library( "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/discovery/cached:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/tools/cache:go_default_library", "//vendor/k8s.io/client-go/util/cert:go_default_library", "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library", diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go index 461e4f14df0..864d8099aed 100644 --- a/cmd/kube-apiserver/app/aggregator.go +++ b/cmd/kube-apiserver/app/aggregator.go @@ -84,7 +84,7 @@ func createAggregatorConfig( if err := commandOptions.APIEnablement.ApplyTo( &genericConfig, aggregatorapiserver.DefaultAPIResourceConfigSource(), - aggregatorscheme.Registry); err != nil { + aggregatorscheme.Scheme); err != nil { return nil, err } diff --git a/cmd/kube-apiserver/app/apiextensions.go b/cmd/kube-apiserver/app/apiextensions.go index 95edfe3d4e8..375372a162e 100644 --- a/cmd/kube-apiserver/app/apiextensions.go +++ b/cmd/kube-apiserver/app/apiextensions.go @@ -58,7 +58,7 @@ func createAPIExtensionsConfig( if err := commandOptions.APIEnablement.ApplyTo( &genericConfig, apiextensionsapiserver.DefaultAPIResourceConfigSource(), - apiextensionsapiserver.Registry); err != nil { + apiextensionsapiserver.Scheme); err != nil { return nil, err } diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 3e931ca619c..8f9e3a92174 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -284,8 +284,8 @@ func TestAddFlags(t *testing.T) { CloudProvider: "azure", }, StorageSerialization: &kubeoptions.StorageSerializationOptions{ - StorageVersions: legacyscheme.Registry.AllPreferredGroupVersions(), - DefaultStorageVersions: legacyscheme.Registry.AllPreferredGroupVersions(), + StorageVersions: kubeoptions.ToPreferredVersionString(legacyscheme.Scheme.PreferredVersionAllGroups()), + DefaultStorageVersions: kubeoptions.ToPreferredVersionString(legacyscheme.Scheme.PreferredVersionAllGroups()), }, APIEnablement: &apiserveroptions.APIEnablementOptions{ RuntimeConfig: utilflag.ConfigurationMap{}, diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go index b69cd84ea4f..f1295756621 100644 --- a/cmd/kube-apiserver/app/options/validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -82,7 +82,7 @@ func (s *ServerRunOptions) Validate() []error { if s.MasterCount <= 0 { errors = append(errors, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount)) } - if errs := s.APIEnablement.Validate(legacyscheme.Registry, apiextensionsapiserver.Registry, aggregatorscheme.Registry); len(errs) > 0 { + if errs := s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme); len(errs) > 0 { errors = append(errors, errs...) } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index de7db7f19c1..ca557e69c5c 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -54,11 +54,11 @@ import ( serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/storage/etcd3/preflight" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/discovery" cacheddiscovery "k8s.io/client-go/discovery/cached" clientgoinformers "k8s.io/client-go/informers" clientgoclientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" certutil "k8s.io/client-go/util/cert" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" openapi "k8s.io/kube-openapi/pkg/common" @@ -428,7 +428,7 @@ func BuildGenericConfig( if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil { return } - if lastErr = s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Registry); lastErr != nil { + if lastErr = s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil { return } @@ -575,11 +575,11 @@ func BuildAdmissionPluginInitializers( // We have a functional client so we can use that to build our discovery backed REST mapper // Use a discovery client capable of being refreshed. discoveryClient := cacheddiscovery.NewMemCacheClient(client.Discovery()) - discoveryRESTMapper := discovery.NewDeferredDiscoveryRESTMapper(discoveryClient) + discoveryRESTMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) admissionPostStartHook := func(context genericapiserver.PostStartHookContext) error { discoveryRESTMapper.Reset() - go utilwait.Until(discoveryRESTMapper.Reset, 10*time.Second, context.StopCh) + go utilwait.Until(discoveryRESTMapper.Reset, 30*time.Second, context.StopCh) return nil } @@ -619,7 +619,7 @@ func BuildStorageFactory(s *options.ServerRunOptions, apiResourceConfig *servers } storageFactory, err := kubeapiserver.NewStorageFactory( s.Etcd.StorageConfig, s.Etcd.DefaultStorageMediaType, legacyscheme.Codecs, - serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Registry), storageGroupsToEncodingVersion, + serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme), storageGroupsToEncodingVersion, // The list includes resources that need to be stored in a different // group version than other resources in the groups. // FIXME (soltysh): this GroupVersionResource override should be configurable diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index ce9a02b6dbc..74462342012 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -115,12 +115,12 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/discovery/cached:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/scale:go_default_library", "//vendor/k8s.io/client-go/tools/leaderelection:go_default_library", "//vendor/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library", diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 3b4c645dca2..5ec454f0115 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -35,10 +35,10 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/discovery" cacheddiscovery "k8s.io/client-go/discovery/cached" "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/leaderelection/resourcelock" certutil "k8s.io/client-go/util/cert" @@ -228,7 +228,7 @@ type ControllerContext struct { // DeferredDiscoveryRESTMapper is a RESTMapper that will defer // initialization of the RESTMapper until the first mapping is // requested. - RESTMapper *discovery.DeferredDiscoveryRESTMapper + RESTMapper *restmapper.DeferredDiscoveryRESTMapper // AvailableResources is a map listing currently available resources AvailableResources map[schema.GroupVersionResource]bool @@ -399,7 +399,7 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien // Use a discovery client capable of being refreshed. discoveryClient := rootClientBuilder.ClientOrDie("controller-discovery") cachedClient := cacheddiscovery.NewMemCacheClient(discoveryClient.Discovery()) - restMapper := discovery.NewDeferredDiscoveryRESTMapper(cachedClient) + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedClient) go wait.Until(func() { restMapper.Reset() }, 30*time.Second, stop) diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index 393b218a453..141fc73d144 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -350,13 +350,10 @@ func startGarbageCollectorController(ctx ControllerContext) (bool, error) { discoveryClient := cacheddiscovery.NewMemCacheClient(gcClientset.Discovery()) config := ctx.ClientBuilder.ConfigOrDie("generic-garbage-collector") - config.ContentConfig = dynamic.ContentConfig() - // TODO: Make NewMetadataCodecFactory support arbitrary (non-compiled) - // resource types. Otherwise we'll be storing full Unstructured data in our - // caches for custom resources. Consider porting it to work with - // metav1beta1.PartialObjectMetadata. - metaOnlyClientPool := dynamic.NewClientPool(config, ctx.RESTMapper, dynamic.LegacyAPIPathResolverFunc) - clientPool := dynamic.NewClientPool(config, ctx.RESTMapper, dynamic.LegacyAPIPathResolverFunc) + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return true, err + } // Get an initial set of deletable resources to prime the garbage collector. deletableResources := garbagecollector.GetDeletableResources(discoveryClient) @@ -365,8 +362,7 @@ func startGarbageCollectorController(ctx ControllerContext) (bool, error) { ignoredResources[schema.GroupResource{Group: r.Group, Resource: r.Resource}] = struct{}{} } garbageCollector, err := garbagecollector.NewGarbageCollector( - metaOnlyClientPool, - clientPool, + dynamicClient, ctx.RESTMapper, deletableResources, ignoredResources, diff --git a/cmd/kubeadm/app/apis/kubeadm/install/BUILD b/cmd/kubeadm/app/apis/kubeadm/install/BUILD index 891b07141a4..32fc23fcb0c 100644 --- a/cmd/kubeadm/app/apis/kubeadm/install/BUILD +++ b/cmd/kubeadm/app/apis/kubeadm/install/BUILD @@ -17,9 +17,8 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", "//pkg/api/legacyscheme:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/cmd/kubeadm/app/apis/kubeadm/install/install.go b/cmd/kubeadm/app/apis/kubeadm/install/install.go index 5d50b746ff8..81bac336224 100644 --- a/cmd/kubeadm/app/apis/kubeadm/install/install.go +++ b/cmd/kubeadm/app/apis/kubeadm/install/install.go @@ -17,30 +17,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/pkg/api/legacyscheme" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: kubeadm.GroupName, - VersionPreferenceOrder: []string{v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: kubeadm.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(kubeadm.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion)) } diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index f45e5e42d8b..fb92450a7a7 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -300,6 +300,8 @@ type HostPathMount struct { MountPath string // Writable controls write access to the volume Writable bool + // PathType is the type of the HostPath. + PathType v1.HostPathType } // KubeProxy contains elements describing the proxy configuration. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index 5eb5eea73a4..c5f8ab9e21f 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -278,6 +278,8 @@ type HostPathMount struct { MountPath string `json:"mountPath"` // Writable controls write access to the volume Writable bool `json:"writable,omitempty"` + // PathType is the type of the HostPath. + PathType v1.HostPathType `json:"pathType,omitempty"` } // KubeProxy contains elements describing the proxy configuration. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go index d6d56ad086e..818edd4718b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go @@ -23,8 +23,8 @@ package v1alpha1 import ( unsafe "unsafe" - core_v1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -156,6 +156,7 @@ func autoConvert_v1alpha1_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMou out.HostPath = in.HostPath out.MountPath = in.MountPath out.Writable = in.Writable + out.PathType = v1.HostPathType(in.PathType) return nil } @@ -169,6 +170,7 @@ func autoConvert_kubeadm_HostPathMount_To_v1alpha1_HostPathMount(in *kubeadm.Hos out.HostPath = in.HostPath out.MountPath = in.MountPath out.Writable = in.Writable + out.PathType = v1.HostPathType(in.PathType) return nil } @@ -240,7 +242,7 @@ func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in out.NoTaintMaster = in.NoTaintMaster out.PrivilegedPods = in.PrivilegedPods out.Token = in.Token - out.TokenTTL = (*v1.Duration)(unsafe.Pointer(in.TokenTTL)) + out.TokenTTL = (*meta_v1.Duration)(unsafe.Pointer(in.TokenTTL)) out.TokenUsages = *(*[]string)(unsafe.Pointer(&in.TokenUsages)) out.TokenGroups = *(*[]string)(unsafe.Pointer(&in.TokenGroups)) out.CRISocket = in.CRISocket @@ -253,7 +255,7 @@ func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs)) out.CertificatesDir = in.CertificatesDir out.ImageRepository = in.ImageRepository - out.ImagePullPolicy = core_v1.PullPolicy(in.ImagePullPolicy) + out.ImagePullPolicy = v1.PullPolicy(in.ImagePullPolicy) out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage if err := Convert_v1alpha1_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil { return err @@ -291,7 +293,7 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in out.NoTaintMaster = in.NoTaintMaster out.PrivilegedPods = in.PrivilegedPods out.Token = in.Token - out.TokenTTL = (*v1.Duration)(unsafe.Pointer(in.TokenTTL)) + out.TokenTTL = (*meta_v1.Duration)(unsafe.Pointer(in.TokenTTL)) out.TokenUsages = *(*[]string)(unsafe.Pointer(&in.TokenUsages)) out.TokenGroups = *(*[]string)(unsafe.Pointer(&in.TokenGroups)) out.CRISocket = in.CRISocket @@ -303,7 +305,7 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in out.SchedulerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.SchedulerExtraVolumes)) out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs)) out.CertificatesDir = in.CertificatesDir - out.ImagePullPolicy = core_v1.PullPolicy(in.ImagePullPolicy) + out.ImagePullPolicy = v1.PullPolicy(in.ImagePullPolicy) out.ImageRepository = in.ImageRepository // INFO: in.CIImageRepository opted out of conversion generation out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage @@ -349,7 +351,7 @@ func autoConvert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration(in *Nod out.DiscoveryFile = in.DiscoveryFile out.DiscoveryToken = in.DiscoveryToken out.DiscoveryTokenAPIServers = *(*[]string)(unsafe.Pointer(&in.DiscoveryTokenAPIServers)) - out.DiscoveryTimeout = (*v1.Duration)(unsafe.Pointer(in.DiscoveryTimeout)) + out.DiscoveryTimeout = (*meta_v1.Duration)(unsafe.Pointer(in.DiscoveryTimeout)) out.NodeName = in.NodeName out.TLSBootstrapToken = in.TLSBootstrapToken out.Token = in.Token @@ -371,7 +373,7 @@ func autoConvert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in *kub out.DiscoveryFile = in.DiscoveryFile out.DiscoveryToken = in.DiscoveryToken out.DiscoveryTokenAPIServers = *(*[]string)(unsafe.Pointer(&in.DiscoveryTokenAPIServers)) - out.DiscoveryTimeout = (*v1.Duration)(unsafe.Pointer(in.DiscoveryTimeout)) + out.DiscoveryTimeout = (*meta_v1.Duration)(unsafe.Pointer(in.DiscoveryTimeout)) out.NodeName = in.NodeName out.TLSBootstrapToken = in.TLSBootstrapToken out.Token = in.Token diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 06d163c4e52..5624cc9705e 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -73,6 +73,7 @@ go_library( "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/tools/bootstrap/token/api:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/util/cert:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], @@ -102,6 +103,7 @@ go_test( "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library", "//vendor/k8s.io/client-go/testing:go_default_library", "//vendor/k8s.io/client-go/tools/bootstrap/token/api:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", ], @@ -124,3 +126,14 @@ filegroup( ], tags = ["automanaged"], ) + +go_test( + name = "go_default_xtest", + srcs = ["config_test.go"], + deps = [ + ":go_default_library", + "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", + "//vendor/github.com/renstrom/dedent:go_default_library", + ], +) diff --git a/cmd/kubeadm/app/cmd/config.go b/cmd/kubeadm/app/cmd/config.go index f518d1993b2..36cb40cb878 100644 --- a/cmd/kubeadm/app/cmd/config.go +++ b/cmd/kubeadm/app/cmd/config.go @@ -19,17 +19,21 @@ package cmd import ( "fmt" "io" + "strings" "github.com/golang/glog" "github.com/renstrom/dedent" "github.com/spf13/cobra" + flag "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" + "k8s.io/kubernetes/cmd/kubeadm/app/images" "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" @@ -62,7 +66,7 @@ func NewCmdConfig(out io.Writer) *cobra.Command { cmd.AddCommand(NewCmdConfigUpload(out, &kubeConfigFile)) cmd.AddCommand(NewCmdConfigView(out, &kubeConfigFile)) - + cmd.AddCommand(NewCmdConfigListImages(out)) return cmd } @@ -201,3 +205,70 @@ func uploadConfiguration(client clientset.Interface, cfgPath string, defaultcfg // Then just call the uploadconfig phase to do the rest of the work return uploadconfig.UploadConfiguration(internalcfg, client) } + +// NewCmdConfigListImages returns the "kubeadm images" command +func NewCmdConfigListImages(out io.Writer) *cobra.Command { + cfg := &kubeadmapiext.MasterConfiguration{} + kubeadmapiext.SetDefaults_MasterConfiguration(cfg) + var cfgPath, featureGatesString string + var err error + + cmd := &cobra.Command{ + Use: "list-images", + Short: "Print a list of images kubeadm will use. The configuration file is used in case any images or image repositories are customized.", + Run: func(_ *cobra.Command, _ []string) { + if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { + kubeadmutil.CheckErr(err) + } + listImages, err := NewListImages(cfgPath, cfg) + kubeadmutil.CheckErr(err) + kubeadmutil.CheckErr(listImages.Run(out)) + }, + } + AddListImagesConfigFlag(cmd.PersistentFlags(), cfg, &featureGatesString) + AddListImagesFlags(cmd.PersistentFlags(), &cfgPath) + + return cmd +} + +// NewListImages returns a "kubeadm images" command +func NewListImages(cfgPath string, cfg *kubeadmapiext.MasterConfiguration) (*ListImages, error) { + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) + if err != nil { + return nil, fmt.Errorf("could not convert cfg to an internal cfg: %v", err) + } + + return &ListImages{ + cfg: internalcfg, + }, nil +} + +// ListImages defines the struct used for "kubeadm images" +type ListImages struct { + cfg *kubeadmapi.MasterConfiguration +} + +// Run runs the images command and writes the result to the io.Writer passed in +func (i *ListImages) Run(out io.Writer) error { + imgs := images.GetAllImages(i.cfg) + for _, img := range imgs { + fmt.Fprintln(out, img) + } + + return nil +} + +// AddListImagesConfigFlag adds the flags that configure kubeadm +func AddListImagesConfigFlag(flagSet *flag.FlagSet, cfg *kubeadmapiext.MasterConfiguration, featureGatesString *string) { + flagSet.StringVar( + &cfg.KubernetesVersion, "kubernetes-version", cfg.KubernetesVersion, + `Choose a specific Kubernetes version for the control plane.`, + ) + flagSet.StringVar(featureGatesString, "feature-gates", *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ + "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) +} + +// AddListImagesFlags adds the flag that defines the location of the config file +func AddListImagesFlags(flagSet *flag.FlagSet, cfgPath *string) { + flagSet.StringVar(cfgPath, "config", *cfgPath, "Path to kubeadm config file.") +} diff --git a/cmd/kubeadm/app/cmd/config_test.go b/cmd/kubeadm/app/cmd/config_test.go new file mode 100644 index 00000000000..836fa5a8e5a --- /dev/null +++ b/cmd/kubeadm/app/cmd/config_test.go @@ -0,0 +1,172 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd_test + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/renstrom/dedent" + + kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd" + "k8s.io/kubernetes/cmd/kubeadm/app/features" +) + +const ( + defaultNumberOfImages = 8 +) + +func TestNewCmdConfigListImages(t *testing.T) { + var output bytes.Buffer + images := cmd.NewCmdConfigListImages(&output) + images.Run(nil, nil) + actual := strings.Split(output.String(), "\n") + if len(actual) != defaultNumberOfImages { + t.Fatalf("Expected %v but found %v images", defaultNumberOfImages, len(actual)) + } +} + +func TestListImagesRunWithCustomConfigPath(t *testing.T) { + testcases := []struct { + name string + expectedImageCount int + // each string provided here must appear in at least one image returned by Run + expectedImageSubstrings []string + configContents []byte + }{ + { + name: "empty config contents", + expectedImageCount: defaultNumberOfImages, + configContents: []byte{}, + }, + { + name: "set k8s version", + expectedImageCount: defaultNumberOfImages, + expectedImageSubstrings: []string{ + ":v1.9.1", + }, + configContents: []byte(dedent.Dedent(` + apiVersion: kubeadm.k8s.io/v1alpha1 + kind: MasterConfiguration + kubernetesVersion: 1.9.1 + `)), + }, + { + name: "use coredns", + expectedImageCount: defaultNumberOfImages, + expectedImageSubstrings: []string{ + "coredns", + }, + configContents: []byte(dedent.Dedent(` + apiVersion: kubeadm.k8s.io/v1alpha1 + kind: MasterConfiguration + featureGates: + CoreDNS: True + `)), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "kubeadm-images-test") + if err != nil { + t.Fatalf("Unable to create temporary directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + configFilePath := filepath.Join(tmpDir, "test-config-file") + err = ioutil.WriteFile(configFilePath, tc.configContents, 0644) + if err != nil { + t.Fatalf("Failed writing a config file: %v", err) + } + + i, err := cmd.NewListImages(configFilePath, &kubeadmapiext.MasterConfiguration{}) + if err != nil { + t.Fatalf("Failed getting the kubeadm images command: %v", err) + } + var output bytes.Buffer + if i.Run(&output) != nil { + t.Fatalf("Error from running the images command: %v", err) + } + actual := strings.Split(output.String(), "\n") + if len(actual) != tc.expectedImageCount { + t.Fatalf("did not get the same number of images: actual: %v expected: %v. Actual value: %v", len(actual), tc.expectedImageCount, actual) + } + + for _, substring := range tc.expectedImageSubstrings { + if !strings.Contains(output.String(), substring) { + t.Errorf("Expected to find %v but did not in this list of images: %v", substring, actual) + } + } + }) + } +} + +func TestConfigListImagesRunWithoutPath(t *testing.T) { + testcases := []struct { + name string + cfg kubeadmapiext.MasterConfiguration + expectedImages int + }{ + { + name: "empty config", + expectedImages: defaultNumberOfImages, + }, + { + name: "external etcd configuration", + cfg: kubeadmapiext.MasterConfiguration{ + Etcd: kubeadmapiext.Etcd{ + Endpoints: []string{"hi"}, + }, + }, + expectedImages: defaultNumberOfImages - 1, + }, + { + name: "coredns enabled", + cfg: kubeadmapiext.MasterConfiguration{ + FeatureGates: map[string]bool{ + features.CoreDNS: true, + }, + }, + expectedImages: defaultNumberOfImages, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + i, err := cmd.NewListImages("", &tc.cfg) + if err != nil { + t.Fatalf("did not expect an error while creating the Images command: %v", err) + } + + var output bytes.Buffer + if i.Run(&output) != nil { + t.Fatalf("did not expect an error running the Images command: %v", err) + } + + actual := strings.Split(output.String(), "\n") + if len(actual) != tc.expectedImages { + t.Fatalf("expected %v images but got %v", tc.expectedImages, actual) + } + }) + } +} diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 666de7e2f50..041cc673233 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -187,7 +187,7 @@ func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiext.MasterConfigur ) flagSet.StringVar( &cfg.Token, "token", cfg.Token, - "The token to use for establishing bidirectional trust between nodes and masters.", + "The token to use for establishing bidirectional trust between nodes and masters. The format is [a-z0-9]{6}\\.[a-z0-9]{16} - e.g. abcdef.0123456789abcdef", ) flagSet.DurationVar( &cfg.TokenTTL.Duration, "token-ttl", cfg.TokenTTL.Duration, diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index 4defa95bd80..a38c999fb1b 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -39,12 +39,6 @@ import ( utilsexec "k8s.io/utils/exec" ) -var ( - crictlSandboxesParamsFormat = "%s -r %s sandboxes --quiet | xargs -r" - crictlStopParamsFormat = "%s -r %s stops %s" - crictlRemoveParamsFormat = "%s -r %s rms %s" -) - // NewCmdReset returns the "kubeadm reset" command func NewCmdReset(in io.Reader, out io.Writer) *cobra.Command { var skipPreFlight bool @@ -213,28 +207,31 @@ func resetWithCrictl(execer utilsexec.Interface, dockerCheck preflight.Checker, glog.Infof("[reset] cleaning up running containers using crictl with socket %s\n", criSocketPath) glog.V(1).Infoln("[reset] listing running pods using crictl") - listcmd := fmt.Sprintf(crictlSandboxesParamsFormat, crictlPath, criSocketPath) - glog.V(1).Infof("[reset] executing comand %q", listcmd) - output, err := execer.Command(listcmd).CombinedOutput() + params := []string{"-r", criSocketPath, "pods", "--quiet"} + glog.V(1).Infof("[reset] executing command %s %s", crictlPath, strings.Join(params, " ")) + output, err := execer.Command(crictlPath, params...).CombinedOutput() if err != nil { - glog.Infoln("[reset] failed to list running pods using crictl. Trying using docker instead") + glog.Infof("[reset] failed to list running pods using crictl: %s. Trying using docker instead", err) resetWithDocker(execer, dockerCheck) return } sandboxes := strings.Split(string(output), " ") glog.V(1).Infoln("[reset] stopping and removing running containers using crictl") for _, s := range sandboxes { - stopcmd := fmt.Sprintf(crictlStopParamsFormat, crictlPath, criSocketPath, s) - glog.V(1).Infof("[reset] executing command %q", stopcmd) - if err := execer.Command(stopcmd).Run(); err != nil { - glog.Infoln("[reset] failed to stop the running containers using crictl. Trying using docker instead") + if strings.TrimSpace(s) == "" { + continue + } + params = []string{"-r", criSocketPath, "stop", s} + glog.V(1).Infof("[reset] executing command %s %s", crictlPath, strings.Join(params, " ")) + if err := execer.Command(crictlPath, params...).Run(); err != nil { + glog.Infof("[reset] failed to stop the running containers using crictl: %s. Trying using docker instead", err) resetWithDocker(execer, dockerCheck) return } - removecmd := fmt.Sprintf(crictlRemoveParamsFormat, crictlPath, criSocketPath, s) - glog.V(1).Infof("[reset] executing command %q", removecmd) - if err := execer.Command(removecmd).Run(); err != nil { - glog.Infoln("[reset] failed to remove the running containers using crictl. Trying using docker instead") + params = []string{"-r", criSocketPath, "rm", s} + glog.V(1).Infof("[reset] executing command %s %s", crictlPath, strings.Join(params, " ")) + if err := execer.Command(crictlPath, params...).Run(); err != nil { + glog.Infof("[reset] failed to remove the running containers using crictl: %s. Trying using docker instead", err) resetWithDocker(execer, dockerCheck) return } diff --git a/cmd/kubeadm/app/cmd/token.go b/cmd/kubeadm/app/cmd/token.go index e42ccd9dbac..ed95b39399f 100644 --- a/cmd/kubeadm/app/cmd/token.go +++ b/cmd/kubeadm/app/cmd/token.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/util/duration" clientset "k8s.io/client-go/kubernetes" bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api" + "k8s.io/client-go/tools/clientcmd" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" @@ -50,6 +51,8 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" ) +const defaultKubeConfig = "/etc/kubernetes/admin.conf" + // NewCmdToken returns cobra.Command for token management func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { var kubeConfigFile string @@ -85,7 +88,7 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { } tokenCmd.PersistentFlags().StringVar(&kubeConfigFile, - "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster") + "kubeconfig", defaultKubeConfig, "The KubeConfig file to use when talking to the cluster. If the flag is not set a set of standard locations are searched for an existing KubeConfig file") tokenCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", dryRun, "Whether to enable dry-run mode or not") @@ -121,6 +124,7 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { kubeadmutil.CheckErr(err) glog.V(1).Infoln("[token] getting Clientsets from KubeConfig file") + kubeConfigFile = findExistingKubeConfig(kubeConfigFile) client, err := getClientset(kubeConfigFile, dryRun) kubeadmutil.CheckErr(err) @@ -152,6 +156,7 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { This command will list all bootstrap tokens for you. `), Run: func(tokenCmd *cobra.Command, args []string) { + kubeConfigFile = findExistingKubeConfig(kubeConfigFile) client, err := getClientset(kubeConfigFile, dryRun) kubeadmutil.CheckErr(err) @@ -175,6 +180,7 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { if len(args) < 1 { kubeadmutil.CheckErr(fmt.Errorf("missing subcommand; 'token delete' is missing token of form [%q]", tokenutil.TokenIDRegexpString)) } + kubeConfigFile = findExistingKubeConfig(kubeConfigFile) client, err := getClientset(kubeConfigFile, dryRun) kubeadmutil.CheckErr(err) @@ -381,3 +387,16 @@ func getClientset(file string, dryRun bool) (clientset.Interface, error) { } return kubeconfigutil.ClientSetFromFile(file) } + +func findExistingKubeConfig(file string) string { + // The user did provide a --kubeconfig flag. Respect that and threat it as an + // explicit path without building a DefaultClientConfigLoadingRules object. + if file != defaultKubeConfig { + return file + } + // The user did not provide a --kubeconfig flag. Find a config in the standard + // locations using DefaultClientConfigLoadingRules, but also consider `defaultKubeConfig`. + rules := clientcmd.NewDefaultClientConfigLoadingRules() + rules.Precedence = append(rules.Precedence, defaultKubeConfig) + return rules.GetDefaultFilename() +} diff --git a/cmd/kubeadm/app/cmd/token_test.go b/cmd/kubeadm/app/cmd/token_test.go index e9d25088366..a5dcf59ead7 100644 --- a/cmd/kubeadm/app/cmd/token_test.go +++ b/cmd/kubeadm/app/cmd/token_test.go @@ -37,12 +37,13 @@ import ( "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api" + "k8s.io/client-go/tools/clientcmd" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" ) const ( - TokenExpectedRegex = "^\\S{6}\\.\\S{16}\n$" - TestConfig = `apiVersion: v1 + tokenExpectedRegex = "^\\S{6}\\.\\S{16}\n$" + testConfigToken = `apiVersion: v1 clusters: - cluster: certificate-authority-data: @@ -63,8 +64,8 @@ users: client-certificate-data: client-key-data: ` - TestConfigCertAuthorityData = "certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFM01USXhOREUxTlRFek1Gb1hEVEkzTVRJeE1qRTFOVEV6TUZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTlZrCnNkT0NjRDBIOG9ycXZ5djBEZ09jZEpjRGc4aTJPNGt3QVpPOWZUanJGRHJqbDZlVXRtdlMyZ1lZd0c4TGhPV2gKb0lkZ3AvbVkrbVlDakliUUJtTmE2Ums1V2JremhJRzM1c1lseE9NVUJJR0xXMzN0RTh4SlR1RVd3V0NmZnpLcQpyaU1UT1A3REF3MUxuM2xUNlpJNGRNM09NOE1IUk9Wd3lRMDVpbWo5eUx5R1lYdTlvSncwdTVXWVpFYmpUL3VpCjJBZ2QwVDMrZGFFb044aVBJOTlVQkQxMzRkc2VGSEJEY3hHcmsvVGlQdHBpSC9IOGoxRWZaYzRzTGlONzJmL2YKYUpacTROSHFiT2F5UkpITCtJejFNTW1DRkN3cjdHOHVENWVvWWp2dEdZN2xLc1pBTlUwK3VlUnJsTitxTzhQWQpxaTZNMDFBcmV1UzFVVHFuTkM4Q0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFNbXo4Nm9LMmFLa0owMnlLSC9ZcTlzaDZZcDEKYmhLS25mMFJCaTA1clRacWdhTi9oTnROdmQxSzJxZGRLNzhIT2pVdkpNRGp3NERieXF0Wll2V01XVFRCQnQrSgpPMGNyWkg5NXlqUW42YzRlcU1FTjFhOUFKNXRlclNnTDVhREJsK0FMTWxaNVpxTzBUOUJDdTJtNXV3dGNWaFZuCnh6cGpTT3V5WVdOQ3A5bW9mV2VPUTljNXhEcElWeUlMUkFvNmZ5Z2c3N25TSDN4ckVmd0VKUHFMd1RPYVk1bTcKeEZWWWJoR3dxUGU5V0I5aTR5cnNrZUFBWlpUSzdVbklKMXFkRmlHQk9aZlRtaDhYQ3BOTHZZcFBLQW9hWWlsRwpjOW1acVhpWVlESTV6R1IxMElpc2FWNXJUY2hDenNQVWRhQzRVbnpTZG01cTdKYTAyb0poQlU1TE1FMD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=" - TestConfigNoCluster = `apiVersion: v1 + testConfigTokenCertAuthorityData = "certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFM01USXhOREUxTlRFek1Gb1hEVEkzTVRJeE1qRTFOVEV6TUZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTlZrCnNkT0NjRDBIOG9ycXZ5djBEZ09jZEpjRGc4aTJPNGt3QVpPOWZUanJGRHJqbDZlVXRtdlMyZ1lZd0c4TGhPV2gKb0lkZ3AvbVkrbVlDakliUUJtTmE2Ums1V2JremhJRzM1c1lseE9NVUJJR0xXMzN0RTh4SlR1RVd3V0NmZnpLcQpyaU1UT1A3REF3MUxuM2xUNlpJNGRNM09NOE1IUk9Wd3lRMDVpbWo5eUx5R1lYdTlvSncwdTVXWVpFYmpUL3VpCjJBZ2QwVDMrZGFFb044aVBJOTlVQkQxMzRkc2VGSEJEY3hHcmsvVGlQdHBpSC9IOGoxRWZaYzRzTGlONzJmL2YKYUpacTROSHFiT2F5UkpITCtJejFNTW1DRkN3cjdHOHVENWVvWWp2dEdZN2xLc1pBTlUwK3VlUnJsTitxTzhQWQpxaTZNMDFBcmV1UzFVVHFuTkM4Q0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFNbXo4Nm9LMmFLa0owMnlLSC9ZcTlzaDZZcDEKYmhLS25mMFJCaTA1clRacWdhTi9oTnROdmQxSzJxZGRLNzhIT2pVdkpNRGp3NERieXF0Wll2V01XVFRCQnQrSgpPMGNyWkg5NXlqUW42YzRlcU1FTjFhOUFKNXRlclNnTDVhREJsK0FMTWxaNVpxTzBUOUJDdTJtNXV3dGNWaFZuCnh6cGpTT3V5WVdOQ3A5bW9mV2VPUTljNXhEcElWeUlMUkFvNmZ5Z2c3N25TSDN4ckVmd0VKUHFMd1RPYVk1bTcKeEZWWWJoR3dxUGU5V0I5aTR5cnNrZUFBWlpUSzdVbklKMXFkRmlHQk9aZlRtaDhYQ3BOTHZZcFBLQW9hWWlsRwpjOW1acVhpWVlESTV6R1IxMElpc2FWNXJUY2hDenNQVWRhQzRVbnpTZG01cTdKYTAyb0poQlU1TE1FMD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=" + testConfigTokenNoCluster = `apiVersion: v1 clusters: - cluster: server: @@ -89,12 +90,12 @@ func TestRunGenerateToken(t *testing.T) { output := buf.String() - matched, err := regexp.MatchString(TokenExpectedRegex, output) + matched, err := regexp.MatchString(tokenExpectedRegex, output) if err != nil { t.Fatalf("Encountered an error while trying to match RunGenerateToken's output: %v", err) } if !matched { - t.Errorf("RunGenerateToken's output did not match expected regex; wanted: [%s], got: [%s]", TokenExpectedRegex, output) + t.Errorf("RunGenerateToken's output did not match expected regex; wanted: [%s], got: [%s]", tokenExpectedRegex, output) } } @@ -211,14 +212,14 @@ func TestNewCmdTokenGenerate(t *testing.T) { func TestNewCmdToken(t *testing.T) { var buf, bufErr bytes.Buffer - testConfigFile := "test-config-file" + testConfigTokenFile := "test-config-file" tmpDir, err := ioutil.TempDir("", "kubeadm-token-test") if err != nil { t.Errorf("Unable to create temporary directory: %v", err) } defer os.RemoveAll(tmpDir) - fullPath := filepath.Join(tmpDir, testConfigFile) + fullPath := filepath.Join(tmpDir, testConfigTokenFile) f, err := os.Create(fullPath) if err != nil { @@ -230,6 +231,7 @@ func TestNewCmdToken(t *testing.T) { name string args []string configToWrite string + kubeConfigEnv string expectedError bool }{ { @@ -239,23 +241,39 @@ func TestNewCmdToken(t *testing.T) { expectedError: false, }, { - name: "valid: delete", + name: "valid: delete from --kubeconfig", args: []string{"delete", "abcdef.1234567890123456", "--dry-run", "--kubeconfig=" + fullPath}, - configToWrite: TestConfig, + configToWrite: testConfigToken, + expectedError: false, + }, + { + name: "valid: delete from " + clientcmd.RecommendedConfigPathEnvVar, + args: []string{"delete", "abcdef.1234567890123456", "--dry-run"}, + configToWrite: testConfigToken, + kubeConfigEnv: fullPath, expectedError: false, }, } - cmd := NewCmdToken(&buf, &bufErr) for _, tc := range testCases { + // the command is created for each test so that the kubeConfigFile + // variable in NewCmdToken() is reset. + cmd := NewCmdToken(&buf, &bufErr) if _, err = f.WriteString(tc.configToWrite); err != nil { t.Errorf("Unable to write test file %q: %v", fullPath, err) } + // store the current value of the environment variable. + storedEnv := os.Getenv(clientcmd.RecommendedConfigPathEnvVar) + if tc.kubeConfigEnv != "" { + os.Setenv(clientcmd.RecommendedConfigPathEnvVar, tc.kubeConfigEnv) + } cmd.SetArgs(tc.args) err := cmd.Execute() if (err != nil) != tc.expectedError { t.Errorf("Test case %q: NewCmdToken expected error: %v, saw: %v", tc.name, tc.expectedError, (err != nil)) } + // restore the environment variable. + os.Setenv(clientcmd.RecommendedConfigPathEnvVar, storedEnv) } } @@ -276,14 +294,14 @@ func TestGetSecretString(t *testing.T) { } func TestGetClientset(t *testing.T) { - testConfigFile := "test-config-file" + testConfigTokenFile := "test-config-file" tmpDir, err := ioutil.TempDir("", "kubeadm-token-test") if err != nil { t.Errorf("Unable to create temporary directory: %v", err) } defer os.RemoveAll(tmpDir) - fullPath := filepath.Join(tmpDir, testConfigFile) + fullPath := filepath.Join(tmpDir, testConfigTokenFile) // test dryRun = false on a non-exisiting file if _, err = getClientset(fullPath, false); err == nil { @@ -301,7 +319,7 @@ func TestGetClientset(t *testing.T) { } defer f.Close() - if _, err = f.WriteString(TestConfig); err != nil { + if _, err = f.WriteString(testConfigToken); err != nil { t.Errorf("Unable to write test file %q: %v", fullPath, err) } @@ -327,7 +345,7 @@ func TestRunDeleteToken(t *testing.T) { } defer f.Close() - if _, err = f.WriteString(TestConfig); err != nil { + if _, err = f.WriteString(testConfigToken); err != nil { t.Errorf("Unable to write test file %q: %v", fullPath, err) } @@ -369,7 +387,7 @@ func TestRunListTokens(t *testing.T) { defer f.Close() // test config without secrets; should fail - if _, err = f.WriteString(TestConfig); err != nil { + if _, err = f.WriteString(testConfigToken); err != nil { t.Errorf("Unable to write test file %q: %v", fullPath, err) } @@ -394,9 +412,9 @@ func TestRunListTokens(t *testing.T) { }() fmt.Printf("dummy API server listening on localhost:%s\n", portString) - testConfigOpenPort := strings.Replace(TestConfig, "server: localhost:8000", "server: localhost:"+portString, -1) + testConfigTokenOpenPort := strings.Replace(testConfigToken, "server: localhost:8000", "server: localhost:"+portString, -1) - if _, err = f.WriteString(testConfigOpenPort); err != nil { + if _, err = f.WriteString(testConfigTokenOpenPort); err != nil { t.Errorf("Unable to write test file %q: %v", fullPath, err) } diff --git a/cmd/kubeadm/app/images/BUILD b/cmd/kubeadm/app/images/BUILD index 2f9e30e518e..a7609f1295b 100644 --- a/cmd/kubeadm/app/images/BUILD +++ b/cmd/kubeadm/app/images/BUILD @@ -11,7 +11,10 @@ go_library( srcs = ["images.go"], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/images", deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", + "//cmd/kubeadm/app/phases/addons/dns:go_default_library", "//cmd/kubeadm/app/util:go_default_library", ], ) diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go index ef6ceb7eed5..3b25bd8bf25 100644 --- a/cmd/kubeadm/app/images/images.go +++ b/cmd/kubeadm/app/images/images.go @@ -20,7 +20,10 @@ import ( "fmt" "runtime" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) @@ -42,3 +45,25 @@ func GetCoreImage(image, repoPrefix, k8sVersion, overrideImage string) string { constants.KubeScheduler: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "kube-scheduler", runtime.GOARCH, kubernetesImageTag), }[image] } + +// GetAllImages returns a list of container images kubeadm expects to use on a control plane node +func GetAllImages(cfg *kubeadmapi.MasterConfiguration) []string { + imgs := []string{} + imgs = append(imgs, GetCoreImage(constants.KubeAPIServer, cfg.ImageRepository, cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage)) + imgs = append(imgs, GetCoreImage(constants.KubeControllerManager, cfg.ImageRepository, cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage)) + imgs = append(imgs, GetCoreImage(constants.KubeScheduler, cfg.ImageRepository, cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage)) + imgs = append(imgs, fmt.Sprintf("%v/%v-%v:%v", cfg.ImageRepository, constants.KubeProxy, runtime.GOARCH, kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion))) + imgs = append(imgs, fmt.Sprintf("%v/pause-%v:%v", cfg.ImageRepository, runtime.GOARCH, "3.1")) + + // if etcd is not external then add the image as it will be required + if len(cfg.Etcd.Endpoints) == 0 { + imgs = append(imgs, GetCoreImage(constants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Image)) + } + + dnsImage := fmt.Sprintf("%v/k8s-dns-kube-dns-%v:%v", cfg.ImageRepository, runtime.GOARCH, dns.GetDNSVersion(nil, constants.KubeDNS)) + if features.Enabled(cfg.FeatureGates, features.CoreDNS) { + dnsImage = fmt.Sprintf("coredns/coredns:%v", dns.GetDNSVersion(nil, constants.CoreDNS)) + } + imgs = append(imgs, dnsImage) + return imgs +} diff --git a/cmd/kubeadm/app/phases/controlplane/volumes.go b/cmd/kubeadm/app/phases/controlplane/volumes.go index e3761dbe989..144961540fd 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes.go @@ -107,9 +107,9 @@ func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.MasterConfiguration) c // Merge user defined mounts and ensure unique volume and volume mount // names - mounts.AddExtraHostPathMounts(kubeadmconstants.KubeAPIServer, cfg.APIServerExtraVolumes, &hostPathDirectoryOrCreate) - mounts.AddExtraHostPathMounts(kubeadmconstants.KubeControllerManager, cfg.ControllerManagerExtraVolumes, &hostPathDirectoryOrCreate) - mounts.AddExtraHostPathMounts(kubeadmconstants.KubeScheduler, cfg.SchedulerExtraVolumes, &hostPathDirectoryOrCreate) + mounts.AddExtraHostPathMounts(kubeadmconstants.KubeAPIServer, cfg.APIServerExtraVolumes) + mounts.AddExtraHostPathMounts(kubeadmconstants.KubeControllerManager, cfg.ControllerManagerExtraVolumes) + mounts.AddExtraHostPathMounts(kubeadmconstants.KubeScheduler, cfg.SchedulerExtraVolumes) return mounts } @@ -155,10 +155,11 @@ func (c *controlPlaneHostPathMounts) AddHostPathMounts(component string, vols [] // AddExtraHostPathMounts adds host path mounts and overwrites the default // paths in the case that a user specifies the same volume/volume mount name. -func (c *controlPlaneHostPathMounts) AddExtraHostPathMounts(component string, extraVols []kubeadmapi.HostPathMount, hostPathType *v1.HostPathType) { +func (c *controlPlaneHostPathMounts) AddExtraHostPathMounts(component string, extraVols []kubeadmapi.HostPathMount) { for _, extraVol := range extraVols { fmt.Printf("[controlplane] Adding extra host path mount %q to %q\n", extraVol.Name, component) - c.NewHostPathMount(component, extraVol.Name, extraVol.HostPath, extraVol.MountPath, !extraVol.Writable, hostPathType) + hostPathType := extraVol.PathType + c.NewHostPathMount(component, extraVol.Name, extraVol.HostPath, extraVol.MountPath, !extraVol.Writable, &hostPathType) } } diff --git a/cmd/kubeadm/app/phases/controlplane/volumes_test.go b/cmd/kubeadm/app/phases/controlplane/volumes_test.go index 585d3e2a78f..99992f97057 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes_test.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes_test.go @@ -615,19 +615,35 @@ func TestAddExtraHostPathMounts(t *testing.T) { mounts.AddHostPathMounts("component", vols, volMounts) hostPathMounts := []kubeadmapi.HostPathMount{ { - Name: "foo", - HostPath: "/tmp/qux", - MountPath: "/tmp/qux", + Name: "foo-0", + HostPath: "/tmp/qux-0", + MountPath: "/tmp/qux-0", Writable: false, + PathType: v1.HostPathFile, }, { - Name: "bar", - HostPath: "/tmp/asd", - MountPath: "/tmp/asd", + Name: "bar-0", + HostPath: "/tmp/asd-0", + MountPath: "/tmp/asd-0", Writable: true, + PathType: v1.HostPathDirectory, + }, + { + Name: "foo-1", + HostPath: "/tmp/qux-1", + MountPath: "/tmp/qux-1", + Writable: false, + PathType: v1.HostPathFileOrCreate, + }, + { + Name: "bar-1", + HostPath: "/tmp/asd-1", + MountPath: "/tmp/asd-1", + Writable: true, + PathType: v1.HostPathDirectoryOrCreate, }, } - mounts.AddExtraHostPathMounts("component", hostPathMounts, &hostPathDirectoryOrCreate) + mounts.AddExtraHostPathMounts("component", hostPathMounts) for _, hostMount := range hostPathMounts { volumeName := hostMount.Name if _, ok := mounts.volumes["component"][volumeName]; !ok { @@ -643,6 +659,9 @@ func TestAddExtraHostPathMounts(t *testing.T) { if _, ok := mounts.volumeMounts["component"][volumeName]; !ok { t.Errorf("Expected to find volume mount %q", volumeName) } + if *vol.HostPath.Type != v1.HostPathType(hostMount.PathType) { + t.Errorf("Expected to host path type %q", hostMount.PathType) + } volMount, _ := mounts.volumeMounts["component"][volumeName] if volMount.Name != volumeName { t.Errorf("Expected volume mount name %q", volumeName) diff --git a/cmd/kubeadm/app/phases/kubelet/kubelet.go b/cmd/kubeadm/app/phases/kubelet/kubelet.go index 13afaaa3325..4f0a51888a7 100644 --- a/cmd/kubeadm/app/phases/kubelet/kubelet.go +++ b/cmd/kubeadm/app/phases/kubelet/kubelet.go @@ -116,10 +116,11 @@ func updateNodeWithConfigMap(client clientset.Interface, nodeName string) error } node.Spec.ConfigSource = &v1.NodeConfigSource{ - ConfigMapRef: &v1.ObjectReference{ - Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, - Namespace: metav1.NamespaceSystem, - UID: kubeletCfg.UID, + ConfigMap: &v1.ConfigMapNodeConfigSource{ + Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, + Namespace: metav1.NamespaceSystem, + UID: kubeletCfg.UID, + KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey, }, } diff --git a/cmd/kubeadm/app/phases/kubelet/kubelet_test.go b/cmd/kubeadm/app/phases/kubelet/kubelet_test.go index 9c6935e4a88..8d211b052a7 100644 --- a/cmd/kubeadm/app/phases/kubelet/kubelet_test.go +++ b/cmd/kubeadm/app/phases/kubelet/kubelet_test.go @@ -50,7 +50,7 @@ func TestCreateBaseKubeletConfiguration(t *testing.T) { }, Spec: v1.NodeSpec{ ConfigSource: &v1.NodeConfigSource{ - ConfigMapRef: &v1.ObjectReference{ + ConfigMap: &v1.ConfigMapNodeConfigSource{ UID: "", }, }, @@ -94,7 +94,7 @@ func TestUpdateNodeWithConfigMap(t *testing.T) { }, Spec: v1.NodeSpec{ ConfigSource: &v1.NodeConfigSource{ - ConfigMapRef: &v1.ObjectReference{ + ConfigMap: &v1.ConfigMapNodeConfigSource{ UID: "", }, }, diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index 4336a019d0c..34cf9bdccf8 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -107,7 +107,7 @@ func (criCheck CRICheck) Check() (warnings, errors []error) { errors = append(errors, fmt.Errorf("unable to find command crictl: %s", err)) return warnings, errors } - if err := criCheck.exec.Command(fmt.Sprintf("%s -r %s info", crictlPath, criCheck.socket)).Run(); err != nil { + if err := criCheck.exec.Command(crictlPath, "-r", criCheck.socket, "info").Run(); err != nil { errors = append(errors, fmt.Errorf("unable to check if the container runtime at %q is running: %s", criCheck.socket, err)) return warnings, errors } diff --git a/cmd/kubeadm/app/util/apiclient/clientbacked_dryrun.go b/cmd/kubeadm/app/util/apiclient/clientbacked_dryrun.go index cf48237e79e..73bbde66efb 100644 --- a/cmd/kubeadm/app/util/apiclient/clientbacked_dryrun.go +++ b/cmd/kubeadm/app/util/apiclient/clientbacked_dryrun.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" @@ -34,7 +33,7 @@ import ( // ClientBackedDryRunGetter implements the DryRunGetter interface for use in NewDryRunClient() and proxies all GET and LIST requests to the backing API server reachable via rest.Config type ClientBackedDryRunGetter struct { client clientset.Interface - dynClientPool dynamic.ClientPool + dynamicClient dynamic.DynamicInterface } // InitDryRunGetter should implement the DryRunGetter interface @@ -46,10 +45,14 @@ func NewClientBackedDryRunGetter(config *rest.Config) (*ClientBackedDryRunGetter if err != nil { return nil, err } + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, err + } return &ClientBackedDryRunGetter{ client: client, - dynClientPool: dynamic.NewDynamicClientPool(config), + dynamicClient: dynamicClient, }, nil } @@ -68,22 +71,13 @@ func NewClientBackedDryRunGetterFromKubeconfig(file string) (*ClientBackedDryRun // HandleGetAction handles GET actions to the dryrun clientset this interface supports func (clg *ClientBackedDryRunGetter) HandleGetAction(action core.GetAction) (bool, runtime.Object, error) { - rc, err := clg.actionToResourceClient(action) + unstructuredObj, err := clg.dynamicClient.Resource(action.GetResource()).Namespace(action.GetNamespace()).Get(action.GetName(), metav1.GetOptions{}) if err != nil { return true, nil, err } - - unversionedObj, err := rc.Get(action.GetName(), metav1.GetOptions{}) + newObj, err := decodeUnstructuredIntoAPIObject(action, unstructuredObj) if err != nil { - return true, nil, err - } - // If the unversioned object does not have .apiVersion; the inner object is probably nil - if len(unversionedObj.GetAPIVersion()) == 0 { - return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), action.GetName()) - } - newObj, err := decodeUnversionedIntoAPIObject(action, unversionedObj) - if err != nil { - fmt.Printf("error after decode: %v %v\n", unversionedObj, err) + fmt.Printf("error after decode: %v %v\n", unstructuredObj, err) return true, nil, err } return true, newObj, err @@ -91,27 +85,18 @@ func (clg *ClientBackedDryRunGetter) HandleGetAction(action core.GetAction) (boo // HandleListAction handles LIST actions to the dryrun clientset this interface supports func (clg *ClientBackedDryRunGetter) HandleListAction(action core.ListAction) (bool, runtime.Object, error) { - rc, err := clg.actionToResourceClient(action) - if err != nil { - return true, nil, err - } - listOpts := metav1.ListOptions{ LabelSelector: action.GetListRestrictions().Labels.String(), FieldSelector: action.GetListRestrictions().Fields.String(), } - unversionedList, err := rc.List(listOpts) + unstructuredList, err := clg.dynamicClient.Resource(action.GetResource()).Namespace(action.GetNamespace()).List(listOpts) if err != nil { return true, nil, err } - // If the runtime.Object here is nil, we should return successfully with no result - if unversionedList == nil { - return true, unversionedList, nil - } - newObj, err := decodeUnversionedIntoAPIObject(action, unversionedList) + newObj, err := decodeUnstructuredIntoAPIObject(action, unstructuredList) if err != nil { - fmt.Printf("error after decode: %v %v\n", unversionedList, err) + fmt.Printf("error after decode: %v %v\n", unstructuredList, err) return true, nil, err } return true, newObj, err @@ -122,28 +107,10 @@ func (clg *ClientBackedDryRunGetter) Client() clientset.Interface { return clg.client } -// actionToResourceClient returns the ResourceInterface for the given action -// First; the function gets the right API group interface from the resource type. The API group struct behind the interface -// returned may be cached in the dynamic client pool. Then, an APIResource object is constructed so that it can be passed to -// dynamic.Interface's Resource() function, which will give us the final ResourceInterface to query -func (clg *ClientBackedDryRunGetter) actionToResourceClient(action core.Action) (dynamic.ResourceInterface, error) { - dynIface, err := clg.dynClientPool.ClientForGroupVersionResource(action.GetResource()) - if err != nil { - return nil, err - } - - apiResource := &metav1.APIResource{ - Name: action.GetResource().Resource, - Namespaced: action.GetNamespace() != "", - } - - return dynIface.Resource(apiResource, action.GetNamespace()), nil -} - // decodeUnversionedIntoAPIObject converts the *unversioned.Unversioned object returned from the dynamic client // to bytes; and then decodes it back _to an external api version (k8s.io/api vs k8s.io/kubernetes/pkg/api*)_ using the normal API machinery -func decodeUnversionedIntoAPIObject(action core.Action, unversionedObj runtime.Object) (runtime.Object, error) { - objBytes, err := json.Marshal(unversionedObj) +func decodeUnstructuredIntoAPIObject(action core.Action, unstructuredObj runtime.Unstructured) (runtime.Object, error) { + objBytes, err := json.Marshal(unstructuredObj) if err != nil { return nil, err } diff --git a/cmd/kubeadm/app/util/version.go b/cmd/kubeadm/app/util/version.go index b64857b61c1..286d0348e58 100644 --- a/cmd/kubeadm/app/util/version.go +++ b/cmd/kubeadm/app/util/version.go @@ -122,8 +122,8 @@ func splitVersion(version string) (string, string, error) { switch { case strings.HasPrefix(subs[0][2], "ci"): - // Special case. CI images populated only by ci-cross area - urlSuffix = "ci-cross" + // Just use whichever the user specified + urlSuffix = subs[0][2] default: urlSuffix = "release" } diff --git a/cmd/kubeadm/app/util/version_test.go b/cmd/kubeadm/app/util/version_test.go index 23591d2cf4b..017f25936a3 100644 --- a/cmd/kubeadm/app/util/version_test.go +++ b/cmd/kubeadm/app/util/version_test.go @@ -180,8 +180,10 @@ func TestSplitVersion(t *testing.T) { {"release/v1.7.0", "https://dl.k8s.io/release", "v1.7.0", true}, {"release/latest-1.7", "https://dl.k8s.io/release", "latest-1.7", true}, // CI builds area, lookup actual builds at ci-cross/*.txt + {"ci/latest", "https://dl.k8s.io/ci", "latest", true}, {"ci-cross/latest", "https://dl.k8s.io/ci-cross", "latest", true}, - {"ci/latest-1.7", "https://dl.k8s.io/ci-cross", "latest-1.7", true}, + {"ci/latest-1.7", "https://dl.k8s.io/ci", "latest-1.7", true}, + {"ci-cross/latest-1.7", "https://dl.k8s.io/ci-cross", "latest-1.7", true}, // unknown label in default (release) area: splitVersion validate only areas. {"unknown-1", "https://dl.k8s.io/release", "unknown-1", true}, // unknown area, not valid input. diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index eb4140bebc4..9dc3d5d5e2a 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -196,7 +196,7 @@ type KubeletFlags struct { // enable gathering custom metrics. EnableCustomMetrics bool // allowPrivileged enables containers to request privileged mode. - // Defaults to false. + // Defaults to true. AllowPrivileged bool // hostNetworkSources is a comma-separated list of sources from which the // Kubelet allows pods to use of host network. Defaults to "*". Valid @@ -244,6 +244,8 @@ func NewKubeletFlags() *KubeletFlags { HostIPCSources: []string{kubetypes.AllSource}, // TODO(#56523): default CAdvisorPort to 0 (disabled) and deprecate it CAdvisorPort: 4194, + // TODO(#58010:v1.13.0): Remove --allow-privileged, it is deprecated + AllowPrivileged: true, } } @@ -416,8 +418,8 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { // TODO(#54161:v1.11.0): Remove --enable-custom-metrics flag, it is deprecated. fs.BoolVar(&f.EnableCustomMetrics, "enable-custom-metrics", f.EnableCustomMetrics, "Support for gathering custom metrics.") fs.MarkDeprecated("enable-custom-metrics", "will be removed in a future version") - // TODO(#58010:v1.12.0): Remove --allow-privileged, it is deprecated - fs.BoolVar(&f.AllowPrivileged, "allow-privileged", f.AllowPrivileged, "If true, allow containers to request privileged mode.") + // TODO(#58010:v1.13.0): Remove --allow-privileged, it is deprecated + fs.BoolVar(&f.AllowPrivileged, "allow-privileged", f.AllowPrivileged, "If true, allow containers to request privileged mode. Default: true") fs.MarkDeprecated("allow-privileged", "will be removed in a future version") // TODO(#58010:v1.12.0): Remove --host-network-sources, it is deprecated fs.StringSliceVar(&f.HostNetworkSources, "host-network-sources", f.HostNetworkSources, "Comma-separated list of sources from which the Kubelet allows pods to use of host network.") diff --git a/docs/.generated_docs b/docs/.generated_docs index aa1c65e3da2..691f514a87b 100644 --- a/docs/.generated_docs +++ b/docs/.generated_docs @@ -58,6 +58,7 @@ docs/admin/kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md docs/admin/kubeadm_alpha_phase_upload-config.md docs/admin/kubeadm_completion.md docs/admin/kubeadm_config.md +docs/admin/kubeadm_config_list-images.md docs/admin/kubeadm_config_upload.md docs/admin/kubeadm_config_upload_from-file.md docs/admin/kubeadm_config_upload_from-flags.md @@ -132,6 +133,7 @@ docs/man/man1/kubeadm-alpha-phase-upload-config.1 docs/man/man1/kubeadm-alpha-phase.1 docs/man/man1/kubeadm-alpha.1 docs/man/man1/kubeadm-completion.1 +docs/man/man1/kubeadm-config-list-images.1 docs/man/man1/kubeadm-config-upload-from-file.1 docs/man/man1/kubeadm-config-upload-from-flags.1 docs/man/man1/kubeadm-config-upload.1 diff --git a/docs/admin/kubeadm_config_list-images.md b/docs/admin/kubeadm_config_list-images.md new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/admin/kubeadm_config_list-images.md @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/api-reference/v1/definitions.html b/docs/api-reference/v1/definitions.html index 8b923d3bf25..de1f986cb5b 100755 --- a/docs/api-reference/v1/definitions.html +++ b/docs/api-reference/v1/definitions.html @@ -403,9 +403,6 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }

v1.Node

  • -

    v1.NodeConfigSource

    -
    • -

  • v1.NodeList

  • @@ -3661,6 +3658,68 @@ Examples:
    + +
    +

    v1.ConfigMapNodeConfigSource

    +
    +

    ConfigMapNodeConfigSource contains the information to reference a ConfigMap as a config source for the Node.

    +
    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescriptionRequiredSchemaDefault

    namespace

    Namespace is the metadata.namespace of the referenced ConfigMap. This field is required in all cases.

    true

    string

    name

    Name is the metadata.name of the referenced ConfigMap. This field is required in all cases.

    true

    string

    uid

    UID is the metadata.UID of the referenced ConfigMap. This field is currently reqired in Node.Spec.

    false

    string

    resourceVersion

    ResourceVersion is the metadata.ResourceVersion of the referenced ConfigMap. This field is forbidden in Node.Spec.

    false

    string

    kubeletConfigKey

    KubeletConfigKey declares which key of the referenced ConfigMap corresponds to the KubeletConfiguration structure This field is required in all cases.

    true

    string

    +

    v1.EndpointPort

    @@ -5830,24 +5889,10 @@ Examples:
    -

    kind

    -

    Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds

    +

    configMap

    +

    ConfigMap is a reference to a Node’s ConfigMap

    false

    -

    string

    - - - -

    apiVersion

    -

    APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources

    -

    false

    -

    string

    - - - -

    configMapRef

    - -

    false

    -

    v1.ObjectReference

    +

    v1.ConfigMapNodeConfigSource

    diff --git a/docs/man/man1/kubeadm-config-list-images.1 b/docs/man/man1/kubeadm-config-list-images.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-config-list-images.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/hack/.golint_failures b/hack/.golint_failures index 21df2075e26..85b97d89884 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -1,3 +1,4 @@ + cluster/images/etcd-version-monitor cmd/hyperkube cmd/kube-apiserver/app @@ -134,7 +135,6 @@ pkg/kubeapiserver/authorizer/modes pkg/kubeapiserver/options pkg/kubeapiserver/server pkg/kubectl -pkg/kubectl/categories pkg/kubectl/cmd pkg/kubectl/cmd/auth pkg/kubectl/cmd/config @@ -148,8 +148,8 @@ pkg/kubectl/cmd/util pkg/kubectl/cmd/util/editor pkg/kubectl/cmd/util/jsonmerge pkg/kubectl/cmd/util/sanity +pkg/kubectl/genericclioptions/resource pkg/kubectl/metricsutil -pkg/kubectl/resource pkg/kubectl/util pkg/kubectl/util/crlf pkg/kubectl/util/slice @@ -461,8 +461,6 @@ staging/src/k8s.io/apimachinery/pkg/api/testing/fuzzer staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip staging/src/k8s.io/apimachinery/pkg/api/validation staging/src/k8s.io/apimachinery/pkg/api/validation/path -staging/src/k8s.io/apimachinery/pkg/apimachinery/announced -staging/src/k8s.io/apimachinery/pkg/apimachinery/registered staging/src/k8s.io/apimachinery/pkg/apis/meta/fuzzer staging/src/k8s.io/apimachinery/pkg/apis/meta/internalversion staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured @@ -698,6 +696,7 @@ test/e2e/ui test/e2e/upgrades test/e2e/upgrades/apps test/e2e/upgrades/storage +test/e2e_kubeadm test/e2e_node test/e2e_node/builder test/e2e_node/environment diff --git a/hack/import-restrictions.yaml b/hack/import-restrictions.yaml index 939bc4e4c74..35645ea7ebc 100644 --- a/hack/import-restrictions.yaml +++ b/hack/import-restrictions.yaml @@ -20,6 +20,9 @@ - baseImportPath: "./pkg/kubectl/genericclioptions/" allowedImports: - k8s.io/apimachinery + - k8s.io/client-go + # TODO this one should be tightened. We depend on it for testing, but we should instead create our own scheme + - k8s.io/api/core/v1 - baseImportPath: "./vendor/k8s.io/apimachinery/" allowedImports: diff --git a/hack/make-rules/test.sh b/hack/make-rules/test.sh index 88efb996bf0..639ee5397c7 100755 --- a/hack/make-rules/test.sh +++ b/hack/make-rules/test.sh @@ -57,6 +57,7 @@ kube::test::find_dirs() { -o -path './target/*' \ -o -path './test/e2e/*' \ -o -path './test/e2e_node/*' \ + -o -path './test/e2e_kubeadm/*' \ -o -path './test/integration/*' \ -o -path './third_party/*' \ -o -path './staging/*' \ diff --git a/hack/update-workspace-mirror.sh b/hack/update-workspace-mirror.sh index 9b7c456c994..98380974cd7 100755 --- a/hack/update-workspace-mirror.sh +++ b/hack/update-workspace-mirror.sh @@ -19,7 +19,7 @@ set -o nounset set -o pipefail if [[ $# -ne 1 ]]; then - echo 'use "bazel run //build:update-mirror"' + echo 'use "bazel run //hack:update-mirror"' echo "(usage: $0 )" exit 1 fi diff --git a/pkg/api/legacyscheme/BUILD b/pkg/api/legacyscheme/BUILD index 57bbde79725..0a8768895f2 100644 --- a/pkg/api/legacyscheme/BUILD +++ b/pkg/api/legacyscheme/BUILD @@ -6,7 +6,6 @@ go_library( importpath = "k8s.io/kubernetes/pkg/api/legacyscheme", visibility = ["//visibility:public"], deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", ], diff --git a/pkg/api/legacyscheme/scheme.go b/pkg/api/legacyscheme/scheme.go index 2ab5cff1cc1..acfcc8f8efa 100644 --- a/pkg/api/legacyscheme/scheme.go +++ b/pkg/api/legacyscheme/scheme.go @@ -17,15 +17,10 @@ limitations under the License. package legacyscheme import ( - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" ) -// Registry is an instance of an API registry. This is an interim step to start removing the idea of a global -// API registry. -var Registry = registered.NewAPIRegistrationManager() - // Scheme is the default instance of runtime.Scheme to which types in the Kubernetes API are already registered. // NOTE: If you are copying this file to start a new api group, STOP! Copy the // extensions group instead. This Scheme is special and should appear ONLY in diff --git a/pkg/api/testapi/testapi.go b/pkg/api/testapi/testapi.go index 5b58add4c17..53adb8ec391 100644 --- a/pkg/api/testapi/testapi.go +++ b/pkg/api/testapi/testapi.go @@ -136,13 +136,13 @@ func init() { } if _, ok := Groups[api.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: api.GroupName, Version: legacyscheme.Registry.GroupOrDie(api.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: api.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(api.GroupName)[0].Version} Groups[api.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[extensions.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: extensions.GroupName, Version: legacyscheme.Registry.GroupOrDie(extensions.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: extensions.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(extensions.GroupName)[0].Version} Groups[extensions.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } @@ -155,7 +155,7 @@ func init() { } internalTypes[k] = t } - externalGroupVersion := schema.GroupVersion{Group: autoscaling.GroupName, Version: legacyscheme.Registry.GroupOrDie(autoscaling.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: autoscaling.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(autoscaling.GroupName)[0].Version} Groups[autoscaling.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } @@ -168,91 +168,91 @@ func init() { break } } - externalGroupVersion := schema.GroupVersion{Group: autoscaling.GroupName, Version: legacyscheme.Registry.GroupOrDie(autoscaling.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: autoscaling.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(autoscaling.GroupName)[0].Version} Groups[autoscaling.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[batch.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: batch.GroupName, Version: legacyscheme.Registry.GroupOrDie(batch.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: batch.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(batch.GroupName)[0].Version} Groups[batch.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[apps.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: apps.GroupName, Version: legacyscheme.Registry.GroupOrDie(apps.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: apps.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(apps.GroupName)[0].Version} Groups[apps.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[policy.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: policy.GroupName, Version: legacyscheme.Registry.GroupOrDie(policy.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: policy.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(policy.GroupName)[0].Version} Groups[policy.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[rbac.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: rbac.GroupName, Version: legacyscheme.Registry.GroupOrDie(rbac.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: rbac.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(rbac.GroupName)[0].Version} Groups[rbac.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[scheduling.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: scheduling.GroupName, Version: legacyscheme.Registry.GroupOrDie(scheduling.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: scheduling.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(scheduling.GroupName)[0].Version} Groups[scheduling.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[settings.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: settings.GroupName, Version: legacyscheme.Registry.GroupOrDie(settings.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: settings.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(settings.GroupName)[0].Version} Groups[settings.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[storage.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: storage.GroupName, Version: legacyscheme.Registry.GroupOrDie(storage.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: storage.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(storage.GroupName)[0].Version} Groups[storage.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[certificates.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: certificates.GroupName, Version: legacyscheme.Registry.GroupOrDie(certificates.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: certificates.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(certificates.GroupName)[0].Version} Groups[certificates.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[imagepolicy.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: imagepolicy.GroupName, Version: legacyscheme.Registry.GroupOrDie(imagepolicy.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: imagepolicy.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(imagepolicy.GroupName)[0].Version} Groups[imagepolicy.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[authorization.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: authorization.GroupName, Version: legacyscheme.Registry.GroupOrDie(authorization.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: authorization.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(authorization.GroupName)[0].Version} Groups[authorization.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[admissionregistration.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: admissionregistration.GroupName, Version: legacyscheme.Registry.GroupOrDie(admissionregistration.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: admissionregistration.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(admissionregistration.GroupName)[0].Version} Groups[admissionregistration.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[admission.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: admission.GroupName, Version: legacyscheme.Registry.GroupOrDie(admission.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: admission.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(admission.GroupName)[0].Version} Groups[admission.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[networking.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: networking.GroupName, Version: legacyscheme.Registry.GroupOrDie(networking.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: networking.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(networking.GroupName)[0].Version} Groups[networking.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } } if _, ok := Groups[events.GroupName]; !ok { - externalGroupVersion := schema.GroupVersion{Group: events.GroupName, Version: legacyscheme.Registry.GroupOrDie(events.GroupName).GroupVersions[0].Version} + externalGroupVersion := schema.GroupVersion{Group: events.GroupName, Version: legacyscheme.Scheme.PrioritizedVersionsForGroup(events.GroupName)[0].Version} Groups[events.GroupName] = TestGroup{ externalGroupVersion: externalGroupVersion, } diff --git a/pkg/api/testing/meta_test.go b/pkg/api/testing/meta_test.go index d7f23567c78..384ad2906f2 100644 --- a/pkg/api/testing/meta_test.go +++ b/pkg/api/testing/meta_test.go @@ -30,7 +30,7 @@ import ( var _ metav1.Object = &metav1.ObjectMeta{} func TestAccessorImplementations(t *testing.T) { - for _, gv := range legacyscheme.Registry.RegisteredGroupVersions() { + for _, gv := range legacyscheme.Scheme.PrioritizedVersionsAllGroups() { internalGV := schema.GroupVersion{Group: gv.Group, Version: runtime.APIVersionInternal} for _, gv := range []schema.GroupVersion{gv, internalGV} { for kind, knownType := range legacyscheme.Scheme.KnownTypes(gv) { diff --git a/pkg/apis/admission/install/BUILD b/pkg/apis/admission/install/BUILD index 027f2ee1e07..d39ff9d17e9 100644 --- a/pkg/apis/admission/install/BUILD +++ b/pkg/apis/admission/install/BUILD @@ -13,9 +13,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/admission:go_default_library", "//pkg/apis/admission/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/admission/install/install.go b/pkg/apis/admission/install/install.go index 4330e590d82..82a1cb4a40a 100644 --- a/pkg/apis/admission/install/install.go +++ b/pkg/apis/admission/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/admission" "k8s.io/kubernetes/pkg/apis/admission/v1beta1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: admission.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: admission.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(admission.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion)) } diff --git a/pkg/apis/admissionregistration/install/BUILD b/pkg/apis/admissionregistration/install/BUILD index 472272fb389..76a85b29115 100644 --- a/pkg/apis/admissionregistration/install/BUILD +++ b/pkg/apis/admissionregistration/install/BUILD @@ -14,9 +14,8 @@ go_library( "//pkg/apis/admissionregistration:go_default_library", "//pkg/apis/admissionregistration/v1alpha1:go_default_library", "//pkg/apis/admissionregistration/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/admissionregistration/install/install.go b/pkg/apis/admissionregistration/install/install.go index a1d2d2cad5b..65a38b2fb34 100644 --- a/pkg/apis/admissionregistration/install/install.go +++ b/pkg/apis/admissionregistration/install/install.go @@ -17,9 +17,8 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/admissionregistration" "k8s.io/kubernetes/pkg/apis/admissionregistration/v1alpha1" @@ -27,22 +26,13 @@ import ( ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: admissionregistration.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version, v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: admissionregistration.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(admissionregistration.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion, v1alpha1.SchemeGroupVersion)) } diff --git a/pkg/apis/apps/install/BUILD b/pkg/apis/apps/install/BUILD index e33b4cb2422..622fb317519 100644 --- a/pkg/apis/apps/install/BUILD +++ b/pkg/apis/apps/install/BUILD @@ -15,9 +15,8 @@ go_library( "//pkg/apis/apps/v1:go_default_library", "//pkg/apis/apps/v1beta1:go_default_library", "//pkg/apis/apps/v1beta2:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/apps/install/install.go b/pkg/apis/apps/install/install.go index ac71542f42b..c53860afb7a 100644 --- a/pkg/apis/apps/install/install.go +++ b/pkg/apis/apps/install/install.go @@ -19,9 +19,8 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/apps/v1" @@ -30,23 +29,14 @@ import ( ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: apps.GroupName, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version, v1beta2.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: apps.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - v1beta2.SchemeGroupVersion.Version: v1beta2.AddToScheme, - v1.SchemeGroupVersion.Version: v1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(apps.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(v1beta2.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta2.SchemeGroupVersion, v1beta1.SchemeGroupVersion)) } diff --git a/pkg/apis/authentication/install/BUILD b/pkg/apis/authentication/install/BUILD index d5034f1c440..c65bab3292b 100644 --- a/pkg/apis/authentication/install/BUILD +++ b/pkg/apis/authentication/install/BUILD @@ -14,9 +14,8 @@ go_library( "//pkg/apis/authentication:go_default_library", "//pkg/apis/authentication/v1:go_default_library", "//pkg/apis/authentication/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/authentication/install/install.go b/pkg/apis/authentication/install/install.go index f968844bb59..7974cb91dca 100644 --- a/pkg/apis/authentication/install/install.go +++ b/pkg/apis/authentication/install/install.go @@ -19,9 +19,8 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/authentication" "k8s.io/kubernetes/pkg/apis/authentication/v1" @@ -29,22 +28,13 @@ import ( ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: authentication.GroupName, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: authentication.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - v1.SchemeGroupVersion.Version: v1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(authentication.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion)) } diff --git a/pkg/apis/authorization/install/BUILD b/pkg/apis/authorization/install/BUILD index 7132b4a8692..22dcec212d2 100644 --- a/pkg/apis/authorization/install/BUILD +++ b/pkg/apis/authorization/install/BUILD @@ -14,9 +14,8 @@ go_library( "//pkg/apis/authorization:go_default_library", "//pkg/apis/authorization/v1:go_default_library", "//pkg/apis/authorization/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/authorization/install/install.go b/pkg/apis/authorization/install/install.go index ecc111beb3a..f9af4928dae 100644 --- a/pkg/apis/authorization/install/install.go +++ b/pkg/apis/authorization/install/install.go @@ -19,9 +19,8 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/authorization" "k8s.io/kubernetes/pkg/apis/authorization/v1" @@ -29,22 +28,13 @@ import ( ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: authorization.GroupName, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: authorization.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - v1.SchemeGroupVersion.Version: v1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(authorization.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion)) } diff --git a/pkg/apis/autoscaling/install/BUILD b/pkg/apis/autoscaling/install/BUILD index d1cb4d5dcbf..968a92c2276 100644 --- a/pkg/apis/autoscaling/install/BUILD +++ b/pkg/apis/autoscaling/install/BUILD @@ -14,9 +14,8 @@ go_library( "//pkg/apis/autoscaling:go_default_library", "//pkg/apis/autoscaling/v1:go_default_library", "//pkg/apis/autoscaling/v2beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/autoscaling/install/install.go b/pkg/apis/autoscaling/install/install.go index 63853b8dc30..62236a0c779 100644 --- a/pkg/apis/autoscaling/install/install.go +++ b/pkg/apis/autoscaling/install/install.go @@ -19,9 +19,8 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/autoscaling" "k8s.io/kubernetes/pkg/apis/autoscaling/v1" @@ -29,22 +28,13 @@ import ( ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: autoscaling.GroupName, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version, v2beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: autoscaling.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1.SchemeGroupVersion.Version: v1.AddToScheme, - v2beta1.SchemeGroupVersion.Version: v2beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(autoscaling.AddToScheme(scheme)) + utilruntime.Must(v2beta1.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v2beta1.SchemeGroupVersion)) } diff --git a/pkg/apis/batch/install/BUILD b/pkg/apis/batch/install/BUILD index 3445e3f35ea..b0399ab6b27 100644 --- a/pkg/apis/batch/install/BUILD +++ b/pkg/apis/batch/install/BUILD @@ -15,9 +15,8 @@ go_library( "//pkg/apis/batch/v1:go_default_library", "//pkg/apis/batch/v1beta1:go_default_library", "//pkg/apis/batch/v2alpha1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/batch/install/install.go b/pkg/apis/batch/install/install.go index 043f559e48f..c87c186cc40 100644 --- a/pkg/apis/batch/install/install.go +++ b/pkg/apis/batch/install/install.go @@ -19,9 +19,8 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/batch" "k8s.io/kubernetes/pkg/apis/batch/v1" @@ -30,23 +29,14 @@ import ( ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: batch.GroupName, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version, v2alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: batch.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1.SchemeGroupVersion.Version: v1.AddToScheme, - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - v2alpha1.SchemeGroupVersion.Version: v2alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(batch.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(v2alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion, v2alpha1.SchemeGroupVersion)) } diff --git a/pkg/apis/certificates/install/BUILD b/pkg/apis/certificates/install/BUILD index 0c457073eac..0ef7bd71226 100644 --- a/pkg/apis/certificates/install/BUILD +++ b/pkg/apis/certificates/install/BUILD @@ -13,9 +13,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/certificates:go_default_library", "//pkg/apis/certificates/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/certificates/install/install.go b/pkg/apis/certificates/install/install.go index 88b8d854d5a..8f685da6091 100644 --- a/pkg/apis/certificates/install/install.go +++ b/pkg/apis/certificates/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/certificates" "k8s.io/kubernetes/pkg/apis/certificates/v1beta1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: certificates.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: certificates.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(certificates.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion)) } diff --git a/pkg/apis/componentconfig/install/BUILD b/pkg/apis/componentconfig/install/BUILD index fb58da06e40..847ba6e5cbc 100644 --- a/pkg/apis/componentconfig/install/BUILD +++ b/pkg/apis/componentconfig/install/BUILD @@ -13,9 +13,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/componentconfig:go_default_library", "//pkg/apis/componentconfig/v1alpha1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/componentconfig/install/install.go b/pkg/apis/componentconfig/install/install.go index 3911b06b158..74ed50129b0 100644 --- a/pkg/apis/componentconfig/install/install.go +++ b/pkg/apis/componentconfig/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: componentconfig.GroupName, - VersionPreferenceOrder: []string{v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: componentconfig.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(componentconfig.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion)) } diff --git a/pkg/apis/core/install/BUILD b/pkg/apis/core/install/BUILD index 44a6e23acef..a5b6505f127 100644 --- a/pkg/apis/core/install/BUILD +++ b/pkg/apis/core/install/BUILD @@ -14,9 +14,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/apis/core/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/core/install/install.go b/pkg/apis/core/install/install.go index c5987a996f2..d2d82e27d4d 100644 --- a/pkg/apis/core/install/install.go +++ b/pkg/apis/core/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/v1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: core.GroupName, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: core.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1.SchemeGroupVersion.Version: v1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(core.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion)) } diff --git a/pkg/apis/core/register.go b/pkg/apis/core/register.go index 2784cbe157e..c6cd8681d81 100644 --- a/pkg/apis/core/register.go +++ b/pkg/apis/core/register.go @@ -60,7 +60,6 @@ func addKnownTypes(scheme *runtime.Scheme) error { &ServiceProxyOptions{}, &NodeList{}, &Node{}, - &NodeConfigSource{}, &NodeProxyOptions{}, &Endpoints{}, &EndpointsList{}, diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index 4880b647437..00cb5c819c3 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -3257,12 +3257,48 @@ type NodeSpec struct { DoNotUse_ExternalID string } -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - // NodeConfigSource specifies a source of node configuration. Exactly one subfield must be non-nil. type NodeConfigSource struct { - metav1.TypeMeta - ConfigMapRef *ObjectReference + ConfigMap *ConfigMapNodeConfigSource +} + +type ConfigMapNodeConfigSource struct { + // Namespace is the metadata.namespace of the referenced ConfigMap. + // This field is required in all cases. + Namespace string + + // Name is the metadata.name of the referenced ConfigMap. + // This field is required in all cases. + Name string + + // UID is the metadata.UID of the referenced ConfigMap. + // This field is currently reqired in Node.Spec. + // TODO(#61643): This field will be forbidden in Node.Spec when #61643 is resolved. + // #61643 changes the behavior of dynamic Kubelet config to respect + // ConfigMap updates, and thus removes the ability to pin the Spec to a given UID. + // TODO(#56896): This field will be required in Node.Status when #56896 is resolved. + // #63314 (the PR that resolves #56896) adds a structured status to the Node + // object for reporting information about the config. This status requires UID + // and ResourceVersion, so that it represents a fully-explicit description of + // the configuration in use, while (see previous TODO) the Spec will be + // restricted to namespace/name in #61643. + // +optional + UID types.UID + + // ResourceVersion is the metadata.ResourceVersion of the referenced ConfigMap. + // This field is forbidden in Node.Spec. + // TODO(#56896): This field will be required in Node.Status when #56896 is resolved. + // #63314 (the PR that resolves #56896) adds a structured status to the Node + // object for reporting information about the config. This status requires UID + // and ResourceVersion, so that it represents a fully-explicit description of + // the configuration in use, while (see previous TODO) the Spec will be + // restricted to namespace/name in #61643. + // +optional + ResourceVersion string + + // KubeletConfigKey declares which key of the referenced ConfigMap corresponds to the KubeletConfiguration structure + // This field is required in all cases. + KubeletConfigKey string } // DaemonEndpoint contains information about a single Daemon endpoint. diff --git a/pkg/apis/core/v1/zz_generated.conversion.go b/pkg/apis/core/v1/zz_generated.conversion.go index 4762d90b8ef..4d207e69f5f 100644 --- a/pkg/apis/core/v1/zz_generated.conversion.go +++ b/pkg/apis/core/v1/zz_generated.conversion.go @@ -82,6 +82,8 @@ func RegisterConversions(scheme *runtime.Scheme) error { Convert_core_ConfigMapKeySelector_To_v1_ConfigMapKeySelector, Convert_v1_ConfigMapList_To_core_ConfigMapList, Convert_core_ConfigMapList_To_v1_ConfigMapList, + Convert_v1_ConfigMapNodeConfigSource_To_core_ConfigMapNodeConfigSource, + Convert_core_ConfigMapNodeConfigSource_To_v1_ConfigMapNodeConfigSource, Convert_v1_ConfigMapProjection_To_core_ConfigMapProjection, Convert_core_ConfigMapProjection_To_v1_ConfigMapProjection, Convert_v1_ConfigMapVolumeSource_To_core_ConfigMapVolumeSource, @@ -941,6 +943,34 @@ func Convert_core_ConfigMapList_To_v1_ConfigMapList(in *core.ConfigMapList, out return autoConvert_core_ConfigMapList_To_v1_ConfigMapList(in, out, s) } +func autoConvert_v1_ConfigMapNodeConfigSource_To_core_ConfigMapNodeConfigSource(in *v1.ConfigMapNodeConfigSource, out *core.ConfigMapNodeConfigSource, s conversion.Scope) error { + out.Namespace = in.Namespace + out.Name = in.Name + out.UID = types.UID(in.UID) + out.ResourceVersion = in.ResourceVersion + out.KubeletConfigKey = in.KubeletConfigKey + return nil +} + +// Convert_v1_ConfigMapNodeConfigSource_To_core_ConfigMapNodeConfigSource is an autogenerated conversion function. +func Convert_v1_ConfigMapNodeConfigSource_To_core_ConfigMapNodeConfigSource(in *v1.ConfigMapNodeConfigSource, out *core.ConfigMapNodeConfigSource, s conversion.Scope) error { + return autoConvert_v1_ConfigMapNodeConfigSource_To_core_ConfigMapNodeConfigSource(in, out, s) +} + +func autoConvert_core_ConfigMapNodeConfigSource_To_v1_ConfigMapNodeConfigSource(in *core.ConfigMapNodeConfigSource, out *v1.ConfigMapNodeConfigSource, s conversion.Scope) error { + out.Namespace = in.Namespace + out.Name = in.Name + out.UID = types.UID(in.UID) + out.ResourceVersion = in.ResourceVersion + out.KubeletConfigKey = in.KubeletConfigKey + return nil +} + +// Convert_core_ConfigMapNodeConfigSource_To_v1_ConfigMapNodeConfigSource is an autogenerated conversion function. +func Convert_core_ConfigMapNodeConfigSource_To_v1_ConfigMapNodeConfigSource(in *core.ConfigMapNodeConfigSource, out *v1.ConfigMapNodeConfigSource, s conversion.Scope) error { + return autoConvert_core_ConfigMapNodeConfigSource_To_v1_ConfigMapNodeConfigSource(in, out, s) +} + func autoConvert_v1_ConfigMapProjection_To_core_ConfigMapProjection(in *v1.ConfigMapProjection, out *core.ConfigMapProjection, s conversion.Scope) error { if err := Convert_v1_LocalObjectReference_To_core_LocalObjectReference(&in.LocalObjectReference, &out.LocalObjectReference, s); err != nil { return err @@ -2586,7 +2616,7 @@ func Convert_core_NodeCondition_To_v1_NodeCondition(in *core.NodeCondition, out } func autoConvert_v1_NodeConfigSource_To_core_NodeConfigSource(in *v1.NodeConfigSource, out *core.NodeConfigSource, s conversion.Scope) error { - out.ConfigMapRef = (*core.ObjectReference)(unsafe.Pointer(in.ConfigMapRef)) + out.ConfigMap = (*core.ConfigMapNodeConfigSource)(unsafe.Pointer(in.ConfigMap)) return nil } @@ -2596,7 +2626,7 @@ func Convert_v1_NodeConfigSource_To_core_NodeConfigSource(in *v1.NodeConfigSourc } func autoConvert_core_NodeConfigSource_To_v1_NodeConfigSource(in *core.NodeConfigSource, out *v1.NodeConfigSource, s conversion.Scope) error { - out.ConfigMapRef = (*v1.ObjectReference)(unsafe.Pointer(in.ConfigMapRef)) + out.ConfigMap = (*v1.ConfigMapNodeConfigSource)(unsafe.Pointer(in.ConfigMap)) return nil } diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index 4c9b3437783..d41cf195674 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -4116,7 +4116,7 @@ func ValidateNodeUpdate(node, oldNode *core.Node) field.ErrorList { // Allow updates to Node.Spec.ConfigSource if DynamicKubeletConfig feature gate is enabled if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) { if node.Spec.ConfigSource != nil { - allErrs = append(allErrs, validateNodeConfigSource(node.Spec.ConfigSource, field.NewPath("spec", "configSource"))...) + allErrs = append(allErrs, validateNodeConfigSourceSpec(node.Spec.ConfigSource, field.NewPath("spec", "configSource"))...) } oldNode.Spec.ConfigSource = node.Spec.ConfigSource } @@ -4131,15 +4131,13 @@ func ValidateNodeUpdate(node, oldNode *core.Node) field.ErrorList { return allErrs } -func validateNodeConfigSource(source *core.NodeConfigSource, fldPath *field.Path) field.ErrorList { +// validation specific to Node.Spec.ConfigSource +func validateNodeConfigSourceSpec(source *core.NodeConfigSource, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} count := int(0) - if ref := source.ConfigMapRef; ref != nil { + if source.ConfigMap != nil { count++ - // name, namespace, and UID must all be non-empty for ConfigMapRef - if ref.Name == "" || ref.Namespace == "" || string(ref.UID) == "" { - allErrs = append(allErrs, field.Invalid(fldPath.Child("configMapRef"), ref, "name, namespace, and UID must all be non-empty")) - } + allErrs = append(allErrs, validateConfigMapNodeConfigSourceSpec(source.ConfigMap, fldPath.Child("configMap"))...) } // add more subfields here in the future as they are added to NodeConfigSource @@ -4150,6 +4148,50 @@ func validateNodeConfigSource(source *core.NodeConfigSource, fldPath *field.Path return allErrs } +// validation specific to Node.Spec.ConfigSource.ConfigMap +func validateConfigMapNodeConfigSourceSpec(source *core.ConfigMapNodeConfigSource, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + // TODO(#61643): Prevent ref.UID from being set here when we switch from requiring UID to respecting all ConfigMap updates + if string(source.UID) == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("uid"), "uid must be set in spec")) + } + // resourceVersion must not be set in spec + if source.ResourceVersion != "" { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("resourceVersion"), "resourceVersion must not be set in spec")) + } + return append(allErrs, validateConfigMapNodeConfigSource(source, fldPath)...) +} + +// common validation +func validateConfigMapNodeConfigSource(source *core.ConfigMapNodeConfigSource, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + // validate target configmap namespace + if source.Namespace == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("namespace"), "namespace must be set in spec")) + } else { + for _, msg := range ValidateNameFunc(ValidateNamespaceName)(source.Namespace, false) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("namespace"), source.Namespace, msg)) + } + } + // validate target configmap name + if source.Name == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("name"), "name must be set in spec")) + } else { + for _, msg := range ValidateNameFunc(ValidateConfigMapName)(source.Name, false) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), source.Name, msg)) + } + } + // validate kubeletConfigKey against rules for configMap key names + if source.KubeletConfigKey == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("kubeletConfigKey"), "kubeletConfigKey must be set in spec")) + } else { + for _, msg := range validation.IsConfigMapKey(source.KubeletConfigKey) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("kubeletConfigKey"), source.KubeletConfigKey, msg)) + } + } + return allErrs +} + // Validate compute resource typename. // Refer to docs/design/resources.md for more details. func validateResourceName(value string, fldPath *field.Path) field.ErrorList { diff --git a/pkg/apis/core/zz_generated.deepcopy.go b/pkg/apis/core/zz_generated.deepcopy.go index b5ceff0d139..93246d3b9e7 100644 --- a/pkg/apis/core/zz_generated.deepcopy.go +++ b/pkg/apis/core/zz_generated.deepcopy.go @@ -631,6 +631,22 @@ func (in *ConfigMapList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigMapNodeConfigSource) DeepCopyInto(out *ConfigMapNodeConfigSource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigMapNodeConfigSource. +func (in *ConfigMapNodeConfigSource) DeepCopy() *ConfigMapNodeConfigSource { + if in == nil { + return nil + } + out := new(ConfigMapNodeConfigSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConfigMapProjection) DeepCopyInto(out *ConfigMapProjection) { *out = *in @@ -2364,13 +2380,12 @@ func (in *NodeCondition) DeepCopy() *NodeCondition { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeConfigSource) DeepCopyInto(out *NodeConfigSource) { *out = *in - out.TypeMeta = in.TypeMeta - if in.ConfigMapRef != nil { - in, out := &in.ConfigMapRef, &out.ConfigMapRef + if in.ConfigMap != nil { + in, out := &in.ConfigMap, &out.ConfigMap if *in == nil { *out = nil } else { - *out = new(ObjectReference) + *out = new(ConfigMapNodeConfigSource) **out = **in } } @@ -2387,14 +2402,6 @@ func (in *NodeConfigSource) DeepCopy() *NodeConfigSource { return out } -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *NodeConfigSource) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeDaemonEndpoints) DeepCopyInto(out *NodeDaemonEndpoints) { *out = *in diff --git a/pkg/apis/events/install/BUILD b/pkg/apis/events/install/BUILD index 01c67e2ed5a..a3150f73fa0 100644 --- a/pkg/apis/events/install/BUILD +++ b/pkg/apis/events/install/BUILD @@ -9,9 +9,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/events:go_default_library", "//pkg/apis/events/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/events/install/install.go b/pkg/apis/events/install/install.go index cb2cc66b237..28ac59e9170 100644 --- a/pkg/apis/events/install/install.go +++ b/pkg/apis/events/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/events" "k8s.io/kubernetes/pkg/apis/events/v1beta1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: events.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: events.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(events.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion)) } diff --git a/pkg/apis/extensions/install/BUILD b/pkg/apis/extensions/install/BUILD index 0f8ce5873b1..89384d6ed3d 100644 --- a/pkg/apis/extensions/install/BUILD +++ b/pkg/apis/extensions/install/BUILD @@ -13,9 +13,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/extensions:go_default_library", "//pkg/apis/extensions/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/extensions/install/install.go b/pkg/apis/extensions/install/install.go index 5f9e2e1974b..c22ad59012a 100644 --- a/pkg/apis/extensions/install/install.go +++ b/pkg/apis/extensions/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: extensions.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: extensions.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(extensions.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion)) } diff --git a/pkg/apis/imagepolicy/install/BUILD b/pkg/apis/imagepolicy/install/BUILD index 85e102dfe6b..620376bca71 100644 --- a/pkg/apis/imagepolicy/install/BUILD +++ b/pkg/apis/imagepolicy/install/BUILD @@ -13,9 +13,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/imagepolicy:go_default_library", "//pkg/apis/imagepolicy/v1alpha1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/imagepolicy/install/install.go b/pkg/apis/imagepolicy/install/install.go index 4e29db57c84..f4937181c9a 100644 --- a/pkg/apis/imagepolicy/install/install.go +++ b/pkg/apis/imagepolicy/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/imagepolicy" "k8s.io/kubernetes/pkg/apis/imagepolicy/v1alpha1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: imagepolicy.GroupName, - VersionPreferenceOrder: []string{v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: imagepolicy.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(imagepolicy.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion)) } diff --git a/pkg/apis/networking/install/BUILD b/pkg/apis/networking/install/BUILD index a5ce49004d4..687a53d3554 100644 --- a/pkg/apis/networking/install/BUILD +++ b/pkg/apis/networking/install/BUILD @@ -13,9 +13,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/networking:go_default_library", "//pkg/apis/networking/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/networking/install/install.go b/pkg/apis/networking/install/install.go index 1047c4379ac..4cef1acaffc 100644 --- a/pkg/apis/networking/install/install.go +++ b/pkg/apis/networking/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/networking" "k8s.io/kubernetes/pkg/apis/networking/v1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: networking.GroupName, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: networking.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1.SchemeGroupVersion.Version: v1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(networking.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion)) } diff --git a/pkg/apis/policy/install/BUILD b/pkg/apis/policy/install/BUILD index 6cecb39790c..de072f8edac 100644 --- a/pkg/apis/policy/install/BUILD +++ b/pkg/apis/policy/install/BUILD @@ -13,9 +13,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/policy:go_default_library", "//pkg/apis/policy/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/policy/install/install.go b/pkg/apis/policy/install/install.go index a5a2a79f4b2..0d91720e4d8 100644 --- a/pkg/apis/policy/install/install.go +++ b/pkg/apis/policy/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/policy" "k8s.io/kubernetes/pkg/apis/policy/v1beta1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: policy.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: policy.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(policy.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion)) } diff --git a/pkg/apis/rbac/install/BUILD b/pkg/apis/rbac/install/BUILD index e4299e41b50..bffb4033168 100644 --- a/pkg/apis/rbac/install/BUILD +++ b/pkg/apis/rbac/install/BUILD @@ -15,9 +15,8 @@ go_library( "//pkg/apis/rbac/v1:go_default_library", "//pkg/apis/rbac/v1alpha1:go_default_library", "//pkg/apis/rbac/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/rbac/install/install.go b/pkg/apis/rbac/install/install.go index be34472affa..545523d0a50 100644 --- a/pkg/apis/rbac/install/install.go +++ b/pkg/apis/rbac/install/install.go @@ -19,9 +19,8 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/rbac" "k8s.io/kubernetes/pkg/apis/rbac/v1" @@ -30,26 +29,14 @@ import ( ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: rbac.GroupName, - // Rollout plan: - // 1.10 (once all stored objects are at v1): - // * remove v1alpha1 (announced deprecated in 1.8) - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version, v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: rbac.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1.SchemeGroupVersion.Version: v1.AddToScheme, - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(rbac.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion, v1alpha1.SchemeGroupVersion)) } diff --git a/pkg/apis/scheduling/install/BUILD b/pkg/apis/scheduling/install/BUILD index 827b0e5738a..b704fa433f3 100644 --- a/pkg/apis/scheduling/install/BUILD +++ b/pkg/apis/scheduling/install/BUILD @@ -13,9 +13,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/scheduling:go_default_library", "//pkg/apis/scheduling/v1alpha1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/scheduling/install/install.go b/pkg/apis/scheduling/install/install.go index 65055255c27..a8cfd8e7e07 100644 --- a/pkg/apis/scheduling/install/install.go +++ b/pkg/apis/scheduling/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/scheduling" "k8s.io/kubernetes/pkg/apis/scheduling/v1alpha1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: scheduling.GroupName, - VersionPreferenceOrder: []string{v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: scheduling.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(scheduling.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion)) } diff --git a/pkg/apis/settings/install/BUILD b/pkg/apis/settings/install/BUILD index 9a6ffe726ac..78fc97fd7ac 100644 --- a/pkg/apis/settings/install/BUILD +++ b/pkg/apis/settings/install/BUILD @@ -13,9 +13,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/settings:go_default_library", "//pkg/apis/settings/v1alpha1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/settings/install/install.go b/pkg/apis/settings/install/install.go index 75f68644e11..c4cf5e788a5 100644 --- a/pkg/apis/settings/install/install.go +++ b/pkg/apis/settings/install/install.go @@ -19,30 +19,20 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/settings" "k8s.io/kubernetes/pkg/apis/settings/v1alpha1" ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: settings.GroupName, - VersionPreferenceOrder: []string{v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: settings.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(settings.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion)) } diff --git a/pkg/apis/storage/install/BUILD b/pkg/apis/storage/install/BUILD index 8007f900ae3..0e1c479f1cd 100644 --- a/pkg/apis/storage/install/BUILD +++ b/pkg/apis/storage/install/BUILD @@ -15,9 +15,8 @@ go_library( "//pkg/apis/storage/v1:go_default_library", "//pkg/apis/storage/v1alpha1:go_default_library", "//pkg/apis/storage/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/pkg/apis/storage/install/install.go b/pkg/apis/storage/install/install.go index a7203c3b29d..8ebe9e8b773 100644 --- a/pkg/apis/storage/install/install.go +++ b/pkg/apis/storage/install/install.go @@ -19,9 +19,8 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/storage" "k8s.io/kubernetes/pkg/apis/storage/v1" @@ -30,23 +29,14 @@ import ( ) func init() { - Install(legacyscheme.Registry, legacyscheme.Scheme) + Install(legacyscheme.Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: storage.GroupName, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version, v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: storage.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1.SchemeGroupVersion.Version: v1.AddToScheme, - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(storage.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion, v1alpha1.SchemeGroupVersion)) } diff --git a/pkg/client/clientset_generated/internalclientset/scheme/BUILD b/pkg/client/clientset_generated/internalclientset/scheme/BUILD index 8369575c0f9..46c2682a3e9 100644 --- a/pkg/client/clientset_generated/internalclientset/scheme/BUILD +++ b/pkg/client/clientset_generated/internalclientset/scheme/BUILD @@ -31,7 +31,6 @@ go_library( "//pkg/apis/scheduling/install:go_default_library", "//pkg/apis/settings/install:go_default_library", "//pkg/apis/storage/install:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/pkg/client/clientset_generated/internalclientset/scheme/register.go b/pkg/client/clientset_generated/internalclientset/scheme/register.go index 6ecf57e9fa9..6f7d8813a0b 100644 --- a/pkg/client/clientset_generated/internalclientset/scheme/register.go +++ b/pkg/client/clientset_generated/internalclientset/scheme/register.go @@ -19,7 +19,6 @@ limitations under the License. package scheme import ( - registered "k8s.io/apimachinery/pkg/apimachinery/registered" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -46,31 +45,29 @@ var Scheme = runtime.NewScheme() var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) -var Registry = registered.NewAPIRegistrationManager() - func init() { v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - Install(Registry, Scheme) + Install(Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - admissionregistration.Install(registry, scheme) - core.Install(registry, scheme) - apps.Install(registry, scheme) - authentication.Install(registry, scheme) - authorization.Install(registry, scheme) - autoscaling.Install(registry, scheme) - batch.Install(registry, scheme) - certificates.Install(registry, scheme) - events.Install(registry, scheme) - extensions.Install(registry, scheme) - networking.Install(registry, scheme) - policy.Install(registry, scheme) - rbac.Install(registry, scheme) - scheduling.Install(registry, scheme) - settings.Install(registry, scheme) - storage.Install(registry, scheme) +func Install(scheme *runtime.Scheme) { + admissionregistration.Install(scheme) + core.Install(scheme) + apps.Install(scheme) + authentication.Install(scheme) + authorization.Install(scheme) + autoscaling.Install(scheme) + batch.Install(scheme) + certificates.Install(scheme) + events.Install(scheme) + extensions.Install(scheme) + networking.Install(scheme) + policy.Install(scheme) + rbac.Install(scheme) + scheduling.Install(scheme) + settings.Install(scheme) + storage.Install(scheme) - ExtraInstall(registry, scheme) + ExtraInstall(scheme) } diff --git a/pkg/client/clientset_generated/internalclientset/scheme/register_custom.go b/pkg/client/clientset_generated/internalclientset/scheme/register_custom.go index 4649558a126..d00f12b9b8e 100644 --- a/pkg/client/clientset_generated/internalclientset/scheme/register_custom.go +++ b/pkg/client/clientset_generated/internalclientset/scheme/register_custom.go @@ -17,12 +17,11 @@ limitations under the License. package scheme import ( - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" componentconfig "k8s.io/kubernetes/pkg/apis/componentconfig/install" ) -func ExtraInstall(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { +func ExtraInstall(scheme *runtime.Scheme) { // componentconfig is an apigroup, but we don't have an API endpoint because its objects are just embedded in ConfigMaps. - componentconfig.Install(registry, scheme) + componentconfig.Install(scheme) } diff --git a/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/admissionregistration_client.go b/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/admissionregistration_client.go index ce5e4dc9fe6..9ecefad4c96 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/admissionregistration_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/admissionregistration/internalversion/admissionregistration_client.go @@ -76,17 +76,12 @@ func New(c rest.Interface) *AdmissionregistrationClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("admissionregistration.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("admissionregistration.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("admissionregistration.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/apps_client.go b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/apps_client.go index 475736c8ba9..abcdc639fa8 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/apps_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/apps/internalversion/apps_client.go @@ -71,17 +71,12 @@ func New(c rest.Interface) *AppsClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("apps") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("apps")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("apps")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/authentication/internalversion/authentication_client.go b/pkg/client/clientset_generated/internalclientset/typed/authentication/internalversion/authentication_client.go index 959a1b26216..086b38b4624 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/authentication/internalversion/authentication_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/authentication/internalversion/authentication_client.go @@ -66,17 +66,12 @@ func New(c rest.Interface) *AuthenticationClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("authentication.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("authentication.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("authentication.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/authorization/internalversion/authorization_client.go b/pkg/client/clientset_generated/internalclientset/typed/authorization/internalversion/authorization_client.go index 66fe2c9bbc2..f97f54644bc 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/authorization/internalversion/authorization_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/authorization/internalversion/authorization_client.go @@ -81,17 +81,12 @@ func New(c rest.Interface) *AuthorizationClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("authorization.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("authorization.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("authorization.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/autoscaling/internalversion/autoscaling_client.go b/pkg/client/clientset_generated/internalclientset/typed/autoscaling/internalversion/autoscaling_client.go index fad6c5a230f..e00bc91913a 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/autoscaling/internalversion/autoscaling_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/autoscaling/internalversion/autoscaling_client.go @@ -66,17 +66,12 @@ func New(c rest.Interface) *AutoscalingClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("autoscaling") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("autoscaling")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("autoscaling")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/batch_client.go b/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/batch_client.go index 361f657869e..ebfcb85a264 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/batch_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion/batch_client.go @@ -71,17 +71,12 @@ func New(c rest.Interface) *BatchClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("batch") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("batch")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("batch")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/certificates/internalversion/certificates_client.go b/pkg/client/clientset_generated/internalclientset/typed/certificates/internalversion/certificates_client.go index b1f96de3684..852a4297529 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/certificates/internalversion/certificates_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/certificates/internalversion/certificates_client.go @@ -66,17 +66,12 @@ func New(c rest.Interface) *CertificatesClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("certificates.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("certificates.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("certificates.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/core_client.go b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/core_client.go index 9c90de87768..6ba341b91b8 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/core_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/core/internalversion/core_client.go @@ -141,17 +141,12 @@ func New(c rest.Interface) *CoreClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("") - if err != nil { - return err - } - config.APIPath = "/api" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/events/internalversion/events_client.go b/pkg/client/clientset_generated/internalclientset/typed/events/internalversion/events_client.go index 7eeb04e0491..9eb8b7c523f 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/events/internalversion/events_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/events/internalversion/events_client.go @@ -61,17 +61,12 @@ func New(c rest.Interface) *EventsClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("events.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("events.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("events.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion/extensions_client.go b/pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion/extensions_client.go index 2ce8fcdad16..43ea79be323 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion/extensions_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/extensions/internalversion/extensions_client.go @@ -81,17 +81,12 @@ func New(c rest.Interface) *ExtensionsClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("extensions") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("extensions")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("extensions")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion/networking_client.go b/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion/networking_client.go index 3875aa215ae..1d2f9ab0eaf 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion/networking_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/networking/internalversion/networking_client.go @@ -66,17 +66,12 @@ func New(c rest.Interface) *NetworkingClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("networking.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("networking.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("networking.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/policy_client.go b/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/policy_client.go index b6a0d1f181b..8040244dd5b 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/policy_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/policy/internalversion/policy_client.go @@ -76,17 +76,12 @@ func New(c rest.Interface) *PolicyClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("policy") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("policy")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("policy")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/rbac_client.go b/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/rbac_client.go index 2b643b50dd0..269fe8791fe 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/rbac_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion/rbac_client.go @@ -81,17 +81,12 @@ func New(c rest.Interface) *RbacClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("rbac.authorization.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("rbac.authorization.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("rbac.authorization.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion/scheduling_client.go b/pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion/scheduling_client.go index 47fc65917e3..c30bb6f4d67 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion/scheduling_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/scheduling/internalversion/scheduling_client.go @@ -66,17 +66,12 @@ func New(c rest.Interface) *SchedulingClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("scheduling.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("scheduling.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("scheduling.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/settings/internalversion/settings_client.go b/pkg/client/clientset_generated/internalclientset/typed/settings/internalversion/settings_client.go index 1a84704bd3b..8c365dc6f9e 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/settings/internalversion/settings_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/settings/internalversion/settings_client.go @@ -66,17 +66,12 @@ func New(c rest.Interface) *SettingsClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("settings.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("settings.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("settings.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/storage_client.go b/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/storage_client.go index 1c98304e4f3..9272e2b9c5e 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/storage_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/storage/internalversion/storage_client.go @@ -71,17 +71,12 @@ func New(c rest.Interface) *StorageClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("storage.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("storage.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("storage.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/pkg/cloudprovider/plugins.go b/pkg/cloudprovider/plugins.go index 739c0961339..22fbf9150bd 100644 --- a/pkg/cloudprovider/plugins.go +++ b/pkg/cloudprovider/plugins.go @@ -33,8 +33,11 @@ type Factory func(config io.Reader) (Interface, error) // All registered cloud providers. var ( - providersMutex sync.Mutex - providers = make(map[string]Factory) + providersMutex sync.Mutex + providers = make(map[string]Factory) + deprecatedCloudProviders = []string{ + "openstack", + } ) const externalCloudProvider = "external" @@ -95,6 +98,14 @@ func InitCloudProvider(name string, configFilePath string) (Interface, error) { return nil, nil } + for _, provider := range deprecatedCloudProviders { + if provider == name { + glog.Warningf("WARNING: %s built-in cloud provider is now deprecated. "+ + "Please use 'external' cloud provider for %s", name, name) + break + } + } + if configFilePath != "" { var config *os.File config, err = os.Open(configFilePath) diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index 730f56c8eef..6ea14ef58ef 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -3683,7 +3683,11 @@ func findSecurityGroupForInstance(instance *ec2.Instance, taggedSecurityGroups m // We create instances with one SG // If users create multiple SGs, they must tag one of them as being k8s owned if len(tagged) != 1 { - return nil, fmt.Errorf("Multiple tagged security groups found for instance %s; ensure only the k8s security group is tagged", instanceID) + taggedGroups := "" + for _, v := range tagged { + taggedGroups += fmt.Sprintf("%s(%s) ", *v.GroupId, *v.GroupName) + } + return nil, fmt.Errorf("Multiple tagged security groups found for instance %s; ensure only the k8s security group is tagged; the tagged groups were %v", instanceID, taggedGroups) } return tagged[0], nil } diff --git a/pkg/cloudprovider/providers/aws/aws_test.go b/pkg/cloudprovider/providers/aws/aws_test.go index 1d307c6169b..770b908f063 100644 --- a/pkg/cloudprovider/providers/aws/aws_test.go +++ b/pkg/cloudprovider/providers/aws/aws_test.go @@ -1319,6 +1319,29 @@ func TestEnsureLoadBalancerHealthCheck(t *testing.T) { }) } +func TestFindSecurityGroupForInstance(t *testing.T) { + groups := map[string]*ec2.SecurityGroup{"sg123": {GroupId: aws.String("sg123")}} + id, err := findSecurityGroupForInstance(&ec2.Instance{SecurityGroups: []*ec2.GroupIdentifier{{GroupId: aws.String("sg123"), GroupName: aws.String("my_group")}}}, groups) + if err != nil { + t.Error() + } + assert.Equal(t, *id.GroupId, "sg123") + assert.Equal(t, *id.GroupName, "my_group") +} + +func TestFindSecurityGroupForInstanceMultipleTagged(t *testing.T) { + groups := map[string]*ec2.SecurityGroup{"sg123": {GroupId: aws.String("sg123")}} + _, err := findSecurityGroupForInstance(&ec2.Instance{ + SecurityGroups: []*ec2.GroupIdentifier{ + {GroupId: aws.String("sg123"), GroupName: aws.String("my_group")}, + {GroupId: aws.String("sg123"), GroupName: aws.String("another_group")}, + }, + }, groups) + require.Error(t, err) + assert.Contains(t, err.Error(), "sg123(my_group)") + assert.Contains(t, err.Error(), "sg123(another_group)") +} + func newMockedFakeAWSServices(id string) *FakeAWSServices { s := NewFakeAWSServices(id) s.ec2 = &MockedFakeEC2{FakeEC2Impl: s.ec2.(*FakeEC2Impl)} diff --git a/pkg/cloudprovider/providers/azure/azure_backoff.go b/pkg/cloudprovider/providers/azure/azure_backoff.go index 0d19ec349d5..8bf55f63fd2 100644 --- a/pkg/cloudprovider/providers/azure/azure_backoff.go +++ b/pkg/cloudprovider/providers/azure/azure_backoff.go @@ -111,11 +111,12 @@ func (az *Cloud) GetIPForMachineWithRetry(name types.NodeName) (string, string, // CreateOrUpdateSGWithRetry invokes az.SecurityGroupsClient.CreateOrUpdate with exponential backoff retry func (az *Cloud) CreateOrUpdateSGWithRetry(sg network.SecurityGroup) error { return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { - respChan, errChan := az.SecurityGroupsClient.CreateOrUpdate(az.ResourceGroup, *sg.Name, sg, nil) - resp := <-respChan - err := <-errChan + ctx, cancel := getContextWithCancel() + defer cancel() + + resp, err := az.SecurityGroupsClient.CreateOrUpdate(ctx, az.ResourceGroup, *sg.Name, sg) glog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%s): end", *sg.Name) - done, err := processRetryResponse(resp.Response, err) + done, err := processHTTPRetryResponse(resp, err) if done && err == nil { // Invalidate the cache right after updating az.nsgCache.Delete(*sg.Name) @@ -208,11 +209,12 @@ func (az *Cloud) CreateOrUpdatePIPWithRetry(pipResourceGroup string, pip network // CreateOrUpdateInterfaceWithRetry invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry func (az *Cloud) CreateOrUpdateInterfaceWithRetry(nic network.Interface) error { return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { - respChan, errChan := az.InterfacesClient.CreateOrUpdate(az.ResourceGroup, *nic.Name, nic, nil) - resp := <-respChan - err := <-errChan + ctx, cancel := getContextWithCancel() + defer cancel() + + resp, err := az.InterfacesClient.CreateOrUpdate(ctx, az.ResourceGroup, *nic.Name, nic) glog.V(10).Infof("InterfacesClient.CreateOrUpdate(%s): end", *nic.Name) - return processRetryResponse(resp.Response, err) + return processHTTPRetryResponse(resp, err) }) } @@ -246,32 +248,35 @@ func (az *Cloud) DeleteLBWithRetry(lbName string) error { // CreateOrUpdateRouteTableWithRetry invokes az.RouteTablesClient.CreateOrUpdate with exponential backoff retry func (az *Cloud) CreateOrUpdateRouteTableWithRetry(routeTable network.RouteTable) error { return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { - respChan, errChan := az.RouteTablesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, routeTable, nil) - resp := <-respChan - err := <-errChan - return processRetryResponse(resp.Response, err) + ctx, cancel := getContextWithCancel() + defer cancel() + + resp, err := az.RouteTablesClient.CreateOrUpdate(ctx, az.ResourceGroup, az.RouteTableName, routeTable) + return processHTTPRetryResponse(resp, err) }) } // CreateOrUpdateRouteWithRetry invokes az.RoutesClient.CreateOrUpdate with exponential backoff retry func (az *Cloud) CreateOrUpdateRouteWithRetry(route network.Route) error { return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { - respChan, errChan := az.RoutesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, *route.Name, route, nil) - resp := <-respChan - err := <-errChan + ctx, cancel := getContextWithCancel() + defer cancel() + + resp, err := az.RoutesClient.CreateOrUpdate(ctx, az.ResourceGroup, az.RouteTableName, *route.Name, route) glog.V(10).Infof("RoutesClient.CreateOrUpdate(%s): end", *route.Name) - return processRetryResponse(resp.Response, err) + return processHTTPRetryResponse(resp, err) }) } // DeleteRouteWithRetry invokes az.RoutesClient.Delete with exponential backoff retry func (az *Cloud) DeleteRouteWithRetry(routeName string) error { return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { - respChan, errChan := az.RoutesClient.Delete(az.ResourceGroup, az.RouteTableName, routeName, nil) - resp := <-respChan - err := <-errChan + ctx, cancel := getContextWithCancel() + defer cancel() + + resp, err := az.RoutesClient.Delete(ctx, az.ResourceGroup, az.RouteTableName, routeName) glog.V(10).Infof("RoutesClient.Delete(%s): end", az.RouteTableName) - return processRetryResponse(resp, err) + return processHTTPRetryResponse(resp, err) }) } diff --git a/pkg/cloudprovider/providers/azure/azure_client.go b/pkg/cloudprovider/providers/azure/azure_client.go index 14e1bcdf556..335a0ffafa9 100644 --- a/pkg/cloudprovider/providers/azure/azure_client.go +++ b/pkg/cloudprovider/providers/azure/azure_client.go @@ -41,45 +41,6 @@ func createARMRateLimitErr(isWrite bool, opName string) error { return fmt.Errorf("azure - ARM rate limited(%s) for operation:%s", opType, opName) } -func createARMRateLimitErrChannel(isWrite bool, opName string) chan error { - err := createARMRateLimitErr(isWrite, opName) - errChan := make(chan error, 1) - errChan <- err - return errChan -} - -// LoadBalancerListResultPage is for faking. -type LoadBalancerListResultPage interface { - Next() error - NotDone() bool - Response() network.LoadBalancerListResult - Values() []network.LoadBalancer -} - -// PublicIPAddressListResultPage is for faking. -type PublicIPAddressListResultPage interface { - Next() error - NotDone() bool - Response() network.PublicIPAddressListResult - Values() []network.PublicIPAddress -} - -// SecurityGroupListResultPage is for faking. -type SecurityGroupListResultPage interface { - Next() error - NotDone() bool - Response() network.SecurityGroupListResult - Values() []network.SecurityGroup -} - -// SubnetListResultPage is for faking. -type SubnetListResultPage interface { - Next() error - NotDone() bool - Response() network.SubnetListResult - Values() []network.Subnet -} - // VirtualMachinesClient defines needed functions for azure compute.VirtualMachinesClient type VirtualMachinesClient interface { CreateOrUpdate(ctx context.Context, resourceGroupName string, VMName string, parameters compute.VirtualMachine) (resp *http.Response, err error) @@ -89,9 +50,9 @@ type VirtualMachinesClient interface { // InterfacesClient defines needed functions for azure network.InterfacesClient type InterfacesClient interface { - CreateOrUpdate(resourceGroupName string, networkInterfaceName string, parameters network.Interface, cancel <-chan struct{}) (<-chan network.Interface, <-chan error) - Get(resourceGroupName string, networkInterfaceName string, expand string) (result network.Interface, err error) - GetVirtualMachineScaleSetNetworkInterface(resourceGroupName string, virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string, expand string) (result network.Interface, err error) + CreateOrUpdate(ctx context.Context, resourceGroupName string, networkInterfaceName string, parameters network.Interface) (resp *http.Response, err error) + Get(ctx context.Context, resourceGroupName string, networkInterfaceName string, expand string) (result network.Interface, err error) + GetVirtualMachineScaleSetNetworkInterface(ctx context.Context, resourceGroupName string, virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string, expand string) (result network.Interface, err error) } // LoadBalancersClient defines needed functions for azure network.LoadBalancersClient @@ -112,18 +73,18 @@ type PublicIPAddressesClient interface { // SubnetsClient defines needed functions for azure network.SubnetsClient type SubnetsClient interface { - CreateOrUpdate(resourceGroupName string, virtualNetworkName string, subnetName string, subnetParameters network.Subnet, cancel <-chan struct{}) (<-chan network.Subnet, <-chan error) - Delete(resourceGroupName string, virtualNetworkName string, subnetName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) - Get(resourceGroupName string, virtualNetworkName string, subnetName string, expand string) (result network.Subnet, err error) - List(resourceGroupName string, virtualNetworkName string) (result SubnetListResultPage, err error) + CreateOrUpdate(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string, subnetParameters network.Subnet) (resp *http.Response, err error) + Delete(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string) (resp *http.Response, err error) + Get(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string, expand string) (result network.Subnet, err error) + List(ctx context.Context, resourceGroupName string, virtualNetworkName string) (result []network.Subnet, err error) } // SecurityGroupsClient defines needed functions for azure network.SecurityGroupsClient type SecurityGroupsClient interface { - CreateOrUpdate(resourceGroupName string, networkSecurityGroupName string, parameters network.SecurityGroup, cancel <-chan struct{}) (<-chan network.SecurityGroup, <-chan error) - Delete(resourceGroupName string, networkSecurityGroupName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) - Get(resourceGroupName string, networkSecurityGroupName string, expand string) (result network.SecurityGroup, err error) - List(resourceGroupName string) (result SecurityGroupListResultPage, err error) + CreateOrUpdate(ctx context.Context, resourceGroupName string, networkSecurityGroupName string, parameters network.SecurityGroup) (resp *http.Response, err error) + Delete(ctx context.Context, resourceGroupName string, networkSecurityGroupName string) (resp *http.Response, err error) + Get(ctx context.Context, resourceGroupName string, networkSecurityGroupName string, expand string) (result network.SecurityGroup, err error) + List(ctx context.Context, resourceGroupName string) (result []network.SecurityGroup, err error) } // VirtualMachineScaleSetsClient defines needed functions for azure compute.VirtualMachineScaleSetsClient @@ -144,14 +105,14 @@ type VirtualMachineScaleSetVMsClient interface { // RoutesClient defines needed functions for azure network.RoutesClient type RoutesClient interface { - CreateOrUpdate(resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route, cancel <-chan struct{}) (<-chan network.Route, <-chan error) - Delete(resourceGroupName string, routeTableName string, routeName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) + CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route) (resp *http.Response, err error) + Delete(ctx context.Context, resourceGroupName string, routeTableName string, routeName string) (resp *http.Response, err error) } // RouteTablesClient defines needed functions for azure network.RouteTablesClient type RouteTablesClient interface { - CreateOrUpdate(resourceGroupName string, routeTableName string, parameters network.RouteTable, cancel <-chan struct{}) (<-chan network.RouteTable, <-chan error) - Get(resourceGroupName string, routeTableName string, expand string) (result network.RouteTable, err error) + CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, parameters network.RouteTable) (resp *http.Response, err error) + Get(ctx context.Context, resourceGroupName string, routeTableName string, expand string) (result network.RouteTable, err error) } // StorageAccountClient defines needed functions for azure storage.AccountsClient @@ -297,13 +258,11 @@ func newAzInterfacesClient(config *azClientConfig) *azInterfacesClient { } } -func (az *azInterfacesClient) CreateOrUpdate(resourceGroupName string, networkInterfaceName string, parameters network.Interface, cancel <-chan struct{}) (<-chan network.Interface, <-chan error) { +func (az *azInterfacesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, networkInterfaceName string, parameters network.Interface) (resp *http.Response, err error) { /* Write rate limiting */ if !az.rateLimiterWriter.TryAccept() { - errChan := createARMRateLimitErrChannel(true, "NiCreateOrUpdate") - resultChan := make(chan network.Interface, 1) - resultChan <- network.Interface{} - return resultChan, errChan + err = createARMRateLimitErr(true, "NiCreateOrUpdate") + return } glog.V(10).Infof("azInterfacesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, networkInterfaceName) @@ -311,31 +270,19 @@ func (az *azInterfacesClient) CreateOrUpdate(resourceGroupName string, networkIn glog.V(10).Infof("azInterfacesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, networkInterfaceName) }() - ctx := context.TODO() - errChan := make(chan error, 1) - resultChan := make(chan network.Interface, 1) mc := newMetricContext("interfaces", "create_or_update", resourceGroupName, az.client.SubscriptionID) future, err := az.client.CreateOrUpdate(ctx, resourceGroupName, networkInterfaceName, parameters) if err != nil { mc.Observe(err) - errChan <- err - return resultChan, errChan + return future.Response(), err } - go func() { - if err := future.WaitForCompletion(ctx, az.client.Client); err != nil { - mc.Observe(err) - errChan <- err - return - } - result, err := future.Result(az.client) - errChan <- err - resultChan <- result - }() - return resultChan, errChan + err = future.WaitForCompletion(ctx, az.client.Client) + mc.Observe(err) + return future.Response(), err } -func (az *azInterfacesClient) Get(resourceGroupName string, networkInterfaceName string, expand string) (result network.Interface, err error) { +func (az *azInterfacesClient) Get(ctx context.Context, resourceGroupName string, networkInterfaceName string, expand string) (result network.Interface, err error) { if !az.rateLimiterReader.TryAccept() { err = createARMRateLimitErr(false, "NicGet") return @@ -347,12 +294,12 @@ func (az *azInterfacesClient) Get(resourceGroupName string, networkInterfaceName }() mc := newMetricContext("interfaces", "get", resourceGroupName, az.client.SubscriptionID) - result, err = az.client.Get(context.TODO(), resourceGroupName, networkInterfaceName, expand) + result, err = az.client.Get(ctx, resourceGroupName, networkInterfaceName, expand) mc.Observe(err) return } -func (az *azInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(resourceGroupName string, virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string, expand string) (result network.Interface, err error) { +func (az *azInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(ctx context.Context, resourceGroupName string, virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string, expand string) (result network.Interface, err error) { if !az.rateLimiterReader.TryAccept() { err = createARMRateLimitErr(false, "NicGetVirtualMachineScaleSetNetworkInterface") return @@ -364,7 +311,7 @@ func (az *azInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(resource }() mc := newMetricContext("interfaces", "get_vmss_ni", resourceGroupName, az.client.SubscriptionID) - result, err = az.client.GetVirtualMachineScaleSetNetworkInterface(context.TODO(), resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName, expand) + result, err = az.client.GetVirtualMachineScaleSetNetworkInterface(ctx, resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName, expand) mc.Observe(err) return } @@ -621,13 +568,11 @@ func newAzSubnetsClient(config *azClientConfig) *azSubnetsClient { } } -func (az *azSubnetsClient) CreateOrUpdate(resourceGroupName string, virtualNetworkName string, subnetName string, subnetParameters network.Subnet, cancel <-chan struct{}) (<-chan network.Subnet, <-chan error) { +func (az *azSubnetsClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string, subnetParameters network.Subnet) (resp *http.Response, err error) { /* Write rate limiting */ if !az.rateLimiterWriter.TryAccept() { - errChan := createARMRateLimitErrChannel(true, "SubnetCreateOrUpdate") - resultChan := make(chan network.Subnet, 1) - resultChan <- network.Subnet{} - return resultChan, errChan + err = createARMRateLimitErr(true, "SubnetCreateOrUpdate") + return } glog.V(10).Infof("azSubnetsClient.CreateOrUpdate(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName) @@ -635,37 +580,23 @@ func (az *azSubnetsClient) CreateOrUpdate(resourceGroupName string, virtualNetwo glog.V(10).Infof("azSubnetsClient.CreateOrUpdate(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName) }() - ctx := context.TODO() - resultChan := make(chan network.Subnet, 1) - errChan := make(chan error, 1) mc := newMetricContext("subnets", "create_or_update", resourceGroupName, az.client.SubscriptionID) future, err := az.client.CreateOrUpdate(ctx, resourceGroupName, virtualNetworkName, subnetName, subnetParameters) if err != nil { mc.Observe(err) - errChan <- err - return resultChan, errChan + return future.Response(), err } - go func() { - if err := future.WaitForCompletion(ctx, az.client.Client); err != nil { - mc.Observe(err) - errChan <- err - return - } - result, err := future.Result(az.client) - mc.Observe(err) - errChan <- err - resultChan <- result - }() - return resultChan, errChan + + err = future.WaitForCompletion(ctx, az.client.Client) + mc.Observe(err) + return future.Response(), err } -func (az *azSubnetsClient) Delete(resourceGroupName string, virtualNetworkName string, subnetName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) { +func (az *azSubnetsClient) Delete(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string) (resp *http.Response, err error) { /* Write rate limiting */ if !az.rateLimiterWriter.TryAccept() { - errChan := createARMRateLimitErrChannel(true, "SubnetDelete") - resultChan := make(chan autorest.Response, 1) - resultChan <- autorest.Response{} - return resultChan, errChan + err = createARMRateLimitErr(true, "SubnetDelete") + return } glog.V(10).Infof("azSubnetsClient.Delete(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName) @@ -673,31 +604,19 @@ func (az *azSubnetsClient) Delete(resourceGroupName string, virtualNetworkName s glog.V(10).Infof("azSubnetsClient.Delete(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName) }() - ctx := context.TODO() - resultChan := make(chan autorest.Response, 1) - errChan := make(chan error, 1) mc := newMetricContext("subnets", "delete", resourceGroupName, az.client.SubscriptionID) future, err := az.client.Delete(ctx, resourceGroupName, virtualNetworkName, subnetName) if err != nil { mc.Observe(err) - errChan <- err - return resultChan, errChan + return future.Response(), err } - go func() { - if err := future.WaitForCompletion(ctx, az.client.Client); err != nil { - mc.Observe(err) - errChan <- err - return - } - result, err := future.Result(az.client) - mc.Observe(err) - errChan <- err - resultChan <- result - }() - return resultChan, errChan + + err = future.WaitForCompletion(ctx, az.client.Client) + mc.Observe(err) + return future.Response(), err } -func (az *azSubnetsClient) Get(resourceGroupName string, virtualNetworkName string, subnetName string, expand string) (result network.Subnet, err error) { +func (az *azSubnetsClient) Get(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string, expand string) (result network.Subnet, err error) { if !az.rateLimiterReader.TryAccept() { err = createARMRateLimitErr(false, "SubnetGet") return @@ -709,13 +628,12 @@ func (az *azSubnetsClient) Get(resourceGroupName string, virtualNetworkName stri }() mc := newMetricContext("subnets", "get", resourceGroupName, az.client.SubscriptionID) - ctx := context.TODO() result, err = az.client.Get(ctx, resourceGroupName, virtualNetworkName, subnetName, expand) mc.Observe(err) return } -func (az *azSubnetsClient) List(resourceGroupName string, virtualNetworkName string) (SubnetListResultPage, error) { +func (az *azSubnetsClient) List(ctx context.Context, resourceGroupName string, virtualNetworkName string) ([]network.Subnet, error) { if !az.rateLimiterReader.TryAccept() { return nil, createARMRateLimitErr(false, "SubnetList") } @@ -726,10 +644,22 @@ func (az *azSubnetsClient) List(resourceGroupName string, virtualNetworkName str }() mc := newMetricContext("subnets", "list", resourceGroupName, az.client.SubscriptionID) - ctx := context.TODO() - result, err := az.client.List(ctx, resourceGroupName, virtualNetworkName) - mc.Observe(err) - return &result, err + iterator, err := az.client.ListComplete(ctx, resourceGroupName, virtualNetworkName) + if err != nil { + mc.Observe(err) + return nil, err + } + + result := make([]network.Subnet, 0) + for ; iterator.NotDone(); err = iterator.Next() { + if err != nil { + return nil, err + } + + result = append(result, iterator.Value()) + } + + return result, nil } // azSecurityGroupsClient implements SecurityGroupsClient. @@ -753,13 +683,11 @@ func newAzSecurityGroupsClient(config *azClientConfig) *azSecurityGroupsClient { } } -func (az *azSecurityGroupsClient) CreateOrUpdate(resourceGroupName string, networkSecurityGroupName string, parameters network.SecurityGroup, cancel <-chan struct{}) (<-chan network.SecurityGroup, <-chan error) { +func (az *azSecurityGroupsClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, networkSecurityGroupName string, parameters network.SecurityGroup) (resp *http.Response, err error) { /* Write rate limiting */ if !az.rateLimiterWriter.TryAccept() { - errChan := createARMRateLimitErrChannel(true, "NSGCreateOrUpdate") - resultChan := make(chan network.SecurityGroup, 1) - resultChan <- network.SecurityGroup{} - return resultChan, errChan + err = createARMRateLimitErr(true, "NSGCreateOrUpdate") + return } glog.V(10).Infof("azSecurityGroupsClient.CreateOrUpdate(%q,%q): start", resourceGroupName, networkSecurityGroupName) @@ -767,37 +695,23 @@ func (az *azSecurityGroupsClient) CreateOrUpdate(resourceGroupName string, netwo glog.V(10).Infof("azSecurityGroupsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, networkSecurityGroupName) }() - ctx := context.TODO() - resultChan := make(chan network.SecurityGroup, 1) - errChan := make(chan error, 1) mc := newMetricContext("security_groups", "create_or_update", resourceGroupName, az.client.SubscriptionID) future, err := az.client.CreateOrUpdate(ctx, resourceGroupName, networkSecurityGroupName, parameters) if err != nil { mc.Observe(err) - errChan <- err - return resultChan, errChan + return future.Response(), err } - go func() { - if err := future.WaitForCompletion(ctx, az.client.Client); err != nil { - mc.Observe(err) - errChan <- err - return - } - result, err := future.Result(az.client) - mc.Observe(err) - errChan <- err - resultChan <- result - }() - return resultChan, errChan + + err = future.WaitForCompletion(ctx, az.client.Client) + mc.Observe(err) + return future.Response(), err } -func (az *azSecurityGroupsClient) Delete(resourceGroupName string, networkSecurityGroupName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) { +func (az *azSecurityGroupsClient) Delete(ctx context.Context, resourceGroupName string, networkSecurityGroupName string) (resp *http.Response, err error) { /* Write rate limiting */ if !az.rateLimiterWriter.TryAccept() { - errChan := createARMRateLimitErrChannel(true, "NSGDelete") - resultChan := make(chan autorest.Response, 1) - resultChan <- autorest.Response{} - return resultChan, errChan + err = createARMRateLimitErr(true, "NSGDelete") + return } glog.V(10).Infof("azSecurityGroupsClient.Delete(%q,%q): start", resourceGroupName, networkSecurityGroupName) @@ -805,31 +719,19 @@ func (az *azSecurityGroupsClient) Delete(resourceGroupName string, networkSecuri glog.V(10).Infof("azSecurityGroupsClient.Delete(%q,%q): end", resourceGroupName, networkSecurityGroupName) }() - ctx := context.TODO() - resultChan := make(chan autorest.Response, 1) - errChan := make(chan error, 1) mc := newMetricContext("security_groups", "delete", resourceGroupName, az.client.SubscriptionID) future, err := az.client.Delete(ctx, resourceGroupName, networkSecurityGroupName) if err != nil { mc.Observe(err) - errChan <- err - return resultChan, errChan + return future.Response(), err } - go func() { - if err := future.WaitForCompletion(ctx, az.client.Client); err != nil { - mc.Observe(err) - errChan <- err - return - } - result, err := future.Result(az.client) - mc.Observe(err) - errChan <- err - resultChan <- result - }() - return resultChan, errChan + + err = future.WaitForCompletion(ctx, az.client.Client) + mc.Observe(err) + return future.Response(), err } -func (az *azSecurityGroupsClient) Get(resourceGroupName string, networkSecurityGroupName string, expand string) (result network.SecurityGroup, err error) { +func (az *azSecurityGroupsClient) Get(ctx context.Context, resourceGroupName string, networkSecurityGroupName string, expand string) (result network.SecurityGroup, err error) { if !az.rateLimiterReader.TryAccept() { err = createARMRateLimitErr(false, "NSGGet") return @@ -841,13 +743,12 @@ func (az *azSecurityGroupsClient) Get(resourceGroupName string, networkSecurityG }() mc := newMetricContext("security_groups", "get", resourceGroupName, az.client.SubscriptionID) - ctx := context.TODO() result, err = az.client.Get(ctx, resourceGroupName, networkSecurityGroupName, expand) mc.Observe(err) return } -func (az *azSecurityGroupsClient) List(resourceGroupName string) (SecurityGroupListResultPage, error) { +func (az *azSecurityGroupsClient) List(ctx context.Context, resourceGroupName string) ([]network.SecurityGroup, error) { if !az.rateLimiterReader.TryAccept() { return nil, createARMRateLimitErr(false, "NSGList") } @@ -858,10 +759,22 @@ func (az *azSecurityGroupsClient) List(resourceGroupName string) (SecurityGroupL }() mc := newMetricContext("security_groups", "list", resourceGroupName, az.client.SubscriptionID) - ctx := context.TODO() - result, err := az.client.List(ctx, resourceGroupName) + iterator, err := az.client.ListComplete(ctx, resourceGroupName) mc.Observe(err) - return &result, err + if err != nil { + return nil, err + } + + result := make([]network.SecurityGroup, 0) + for ; iterator.NotDone(); err = iterator.Next() { + if err != nil { + return nil, err + } + + result = append(result, iterator.Value()) + } + + return result, nil } // azVirtualMachineScaleSetsClient implements VirtualMachineScaleSetsClient. @@ -1109,13 +1022,11 @@ func newAzRoutesClient(config *azClientConfig) *azRoutesClient { } } -func (az *azRoutesClient) CreateOrUpdate(resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route, cancel <-chan struct{}) (<-chan network.Route, <-chan error) { +func (az *azRoutesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route) (resp *http.Response, err error) { /* Write rate limiting */ if !az.rateLimiterWriter.TryAccept() { - errChan := createARMRateLimitErrChannel(true, "RouteCreateOrUpdate") - resultChan := make(chan network.Route, 1) - resultChan <- network.Route{} - return resultChan, errChan + err = createARMRateLimitErr(true, "RouteCreateOrUpdate") + return } glog.V(10).Infof("azRoutesClient.CreateOrUpdate(%q,%q,%q): start", resourceGroupName, routeTableName, routeName) @@ -1123,37 +1034,23 @@ func (az *azRoutesClient) CreateOrUpdate(resourceGroupName string, routeTableNam glog.V(10).Infof("azRoutesClient.CreateOrUpdate(%q,%q,%q): end", resourceGroupName, routeTableName, routeName) }() - ctx := context.TODO() - resultChan := make(chan network.Route, 1) - errChan := make(chan error, 1) mc := newMetricContext("routes", "create_or_update", resourceGroupName, az.client.SubscriptionID) future, err := az.client.CreateOrUpdate(ctx, resourceGroupName, routeTableName, routeName, routeParameters) if err != nil { mc.Observe(err) - errChan <- err - return resultChan, errChan + return future.Response(), err } - go func() { - if err := future.WaitForCompletion(ctx, az.client.Client); err != nil { - mc.Observe(err) - errChan <- err - return - } - result, err := future.Result(az.client) - mc.Observe(err) - errChan <- err - resultChan <- result - }() - return resultChan, errChan + + err = future.WaitForCompletion(ctx, az.client.Client) + mc.Observe(err) + return future.Response(), err } -func (az *azRoutesClient) Delete(resourceGroupName string, routeTableName string, routeName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) { +func (az *azRoutesClient) Delete(ctx context.Context, resourceGroupName string, routeTableName string, routeName string) (resp *http.Response, err error) { /* Write rate limiting */ if !az.rateLimiterWriter.TryAccept() { - errChan := createARMRateLimitErrChannel(true, "RouteDelete") - resultChan := make(chan autorest.Response, 1) - resultChan <- autorest.Response{} - return resultChan, errChan + err = createARMRateLimitErr(true, "RouteDelete") + return } glog.V(10).Infof("azRoutesClient.Delete(%q,%q,%q): start", resourceGroupName, routeTableName, routeName) @@ -1161,28 +1058,16 @@ func (az *azRoutesClient) Delete(resourceGroupName string, routeTableName string glog.V(10).Infof("azRoutesClient.Delete(%q,%q,%q): end", resourceGroupName, routeTableName, routeName) }() - ctx := context.TODO() - resultChan := make(chan autorest.Response, 1) - errChan := make(chan error, 1) mc := newMetricContext("routes", "delete", resourceGroupName, az.client.SubscriptionID) future, err := az.client.Delete(ctx, resourceGroupName, routeTableName, routeName) if err != nil { mc.Observe(err) - errChan <- err - return resultChan, errChan + return future.Response(), err } - go func() { - if err := future.WaitForCompletion(ctx, az.client.Client); err != nil { - mc.Observe(err) - errChan <- err - return - } - result, err := future.Result(az.client) - mc.Observe(err) - errChan <- err - resultChan <- result - }() - return resultChan, errChan + + err = future.WaitForCompletion(ctx, az.client.Client) + mc.Observe(err) + return future.Response(), err } // azRouteTablesClient implements RouteTablesClient. @@ -1206,13 +1091,11 @@ func newAzRouteTablesClient(config *azClientConfig) *azRouteTablesClient { } } -func (az *azRouteTablesClient) CreateOrUpdate(resourceGroupName string, routeTableName string, parameters network.RouteTable, cancel <-chan struct{}) (<-chan network.RouteTable, <-chan error) { +func (az *azRouteTablesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, parameters network.RouteTable) (resp *http.Response, err error) { /* Write rate limiting */ if !az.rateLimiterWriter.TryAccept() { - errChan := createARMRateLimitErrChannel(true, "RouteTableCreateOrUpdate") - resultChan := make(chan network.RouteTable, 1) - resultChan <- network.RouteTable{} - return resultChan, errChan + err = createARMRateLimitErr(true, "RouteTableCreateOrUpdate") + return } glog.V(10).Infof("azRouteTablesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, routeTableName) @@ -1220,31 +1103,19 @@ func (az *azRouteTablesClient) CreateOrUpdate(resourceGroupName string, routeTab glog.V(10).Infof("azRouteTablesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, routeTableName) }() - ctx := context.TODO() - resultChan := make(chan network.RouteTable, 1) - errChan := make(chan error, 1) mc := newMetricContext("route_tables", "create_or_update", resourceGroupName, az.client.SubscriptionID) future, err := az.client.CreateOrUpdate(ctx, resourceGroupName, routeTableName, parameters) + mc.Observe(err) if err != nil { - mc.Observe(err) - errChan <- err - return resultChan, errChan + return future.Response(), err } - go func() { - if err := future.WaitForCompletion(ctx, az.client.Client); err != nil { - mc.Observe(err) - errChan <- err - return - } - result, err := future.Result(az.client) - mc.Observe(err) - errChan <- err - resultChan <- result - }() - return resultChan, errChan + + err = future.WaitForCompletion(ctx, az.client.Client) + mc.Observe(err) + return future.Response(), err } -func (az *azRouteTablesClient) Get(resourceGroupName string, routeTableName string, expand string) (result network.RouteTable, err error) { +func (az *azRouteTablesClient) Get(ctx context.Context, resourceGroupName string, routeTableName string, expand string) (result network.RouteTable, err error) { if !az.rateLimiterReader.TryAccept() { err = createARMRateLimitErr(false, "GetRouteTable") return @@ -1256,7 +1127,6 @@ func (az *azRouteTablesClient) Get(resourceGroupName string, routeTableName stri }() mc := newMetricContext("route_tables", "get", resourceGroupName, az.client.SubscriptionID) - ctx := context.TODO() result, err = az.client.Get(ctx, resourceGroupName, routeTableName, expand) mc.Observe(err) return diff --git a/pkg/cloudprovider/providers/azure/azure_fakes.go b/pkg/cloudprovider/providers/azure/azure_fakes.go index 40d613fd084..a79e7e19011 100644 --- a/pkg/cloudprovider/providers/azure/azure_fakes.go +++ b/pkg/cloudprovider/providers/azure/azure_fakes.go @@ -220,33 +220,19 @@ func newFakeAzureInterfacesClient() *fakeAzureInterfacesClient { return fIC } -func (fIC *fakeAzureInterfacesClient) CreateOrUpdate(resourceGroupName string, networkInterfaceName string, parameters network.Interface, cancel <-chan struct{}) (<-chan network.Interface, <-chan error) { +func (fIC *fakeAzureInterfacesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, networkInterfaceName string, parameters network.Interface) (resp *http.Response, err error) { fIC.mutex.Lock() defer fIC.mutex.Unlock() - resultChan := make(chan network.Interface, 1) - errChan := make(chan error, 1) - var result network.Interface - var err error - defer func() { - resultChan <- result - errChan <- err - close(resultChan) - close(errChan) - }() + if _, ok := fIC.FakeStore[resourceGroupName]; !ok { fIC.FakeStore[resourceGroupName] = make(map[string]network.Interface) } fIC.FakeStore[resourceGroupName][networkInterfaceName] = parameters - result = fIC.FakeStore[resourceGroupName][networkInterfaceName] - result.Response.Response = &http.Response{ - StatusCode: http.StatusOK, - } - err = nil - return resultChan, errChan + return nil, nil } -func (fIC *fakeAzureInterfacesClient) Get(resourceGroupName string, networkInterfaceName string, expand string) (result network.Interface, err error) { +func (fIC *fakeAzureInterfacesClient) Get(ctx context.Context, resourceGroupName string, networkInterfaceName string, expand string) (result network.Interface, err error) { fIC.mutex.Lock() defer fIC.mutex.Unlock() if _, ok := fIC.FakeStore[resourceGroupName]; ok { @@ -260,7 +246,7 @@ func (fIC *fakeAzureInterfacesClient) Get(resourceGroupName string, networkInter } } -func (fIC *fakeAzureInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(resourceGroupName string, virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string, expand string) (result network.Interface, err error) { +func (fIC *fakeAzureInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(ctx context.Context, resourceGroupName string, virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string, expand string) (result network.Interface, err error) { return result, nil } @@ -328,67 +314,37 @@ func newFakeAzureSubnetsClient() *fakeAzureSubnetsClient { return fASC } -func (fASC *fakeAzureSubnetsClient) CreateOrUpdate(resourceGroupName string, virtualNetworkName string, subnetName string, subnetParameters network.Subnet, cancel <-chan struct{}) (<-chan network.Subnet, <-chan error) { +func (fASC *fakeAzureSubnetsClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string, subnetParameters network.Subnet) (resp *http.Response, err error) { fASC.mutex.Lock() defer fASC.mutex.Unlock() - resultChan := make(chan network.Subnet, 1) - errChan := make(chan error, 1) - var result network.Subnet - var err error - defer func() { - resultChan <- result - errChan <- err - close(resultChan) - close(errChan) - }() + rgVnet := strings.Join([]string{resourceGroupName, virtualNetworkName}, "AND") if _, ok := fASC.FakeStore[rgVnet]; !ok { fASC.FakeStore[rgVnet] = make(map[string]network.Subnet) } fASC.FakeStore[rgVnet][subnetName] = subnetParameters - result = fASC.FakeStore[rgVnet][subnetName] - result.Response.Response = &http.Response{ - StatusCode: http.StatusOK, - } - err = nil - return resultChan, errChan + + return nil, nil } -func (fASC *fakeAzureSubnetsClient) Delete(resourceGroupName string, virtualNetworkName string, subnetName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) { +func (fASC *fakeAzureSubnetsClient) Delete(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string) (resp *http.Response, err error) { fASC.mutex.Lock() defer fASC.mutex.Unlock() - respChan := make(chan autorest.Response, 1) - errChan := make(chan error, 1) - var resp autorest.Response - var err error - defer func() { - respChan <- resp - errChan <- err - close(respChan) - close(errChan) - }() rgVnet := strings.Join([]string{resourceGroupName, virtualNetworkName}, "AND") if rgSubnets, ok := fASC.FakeStore[rgVnet]; ok { if _, ok := rgSubnets[subnetName]; ok { delete(rgSubnets, subnetName) - resp.Response = &http.Response{ - StatusCode: http.StatusAccepted, - } - err = nil - return respChan, errChan + return nil, nil } } - resp.Response = &http.Response{ + + return &http.Response{ StatusCode: http.StatusNotFound, - } - err = autorest.DetailedError{ - StatusCode: http.StatusNotFound, - Message: "Not such Subnet", - } - return respChan, errChan + }, nil } -func (fASC *fakeAzureSubnetsClient) Get(resourceGroupName string, virtualNetworkName string, subnetName string, expand string) (result network.Subnet, err error) { + +func (fASC *fakeAzureSubnetsClient) Get(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string, expand string) (result network.Subnet, err error) { fASC.mutex.Lock() defer fASC.mutex.Unlock() rgVnet := strings.Join([]string{resourceGroupName, virtualNetworkName}, "AND") @@ -403,30 +359,10 @@ func (fASC *fakeAzureSubnetsClient) Get(resourceGroupName string, virtualNetwork } } -type fakeSubnetListResultPage struct { - next SubnetListResultPage - value network.SubnetListResult - values []network.Subnet - err error -} - -func (pg *fakeSubnetListResultPage) Next() error { - return nil -} -func (pg *fakeSubnetListResultPage) NotDone() bool { - return pg.next != nil -} - -func (pg *fakeSubnetListResultPage) Response() network.SubnetListResult { - return pg.value -} -func (pg *fakeSubnetListResultPage) Values() []network.Subnet { - return pg.values -} - -func (fASC *fakeAzureSubnetsClient) List(resourceGroupName string, virtualNetworkName string) (result SubnetListResultPage, err error) { +func (fASC *fakeAzureSubnetsClient) List(ctx context.Context, resourceGroupName string, virtualNetworkName string) (result []network.Subnet, err error) { fASC.mutex.Lock() defer fASC.mutex.Unlock() + rgVnet := strings.Join([]string{resourceGroupName, virtualNetworkName}, "AND") var value []network.Subnet if _, ok := fASC.FakeStore[rgVnet]; ok { @@ -434,12 +370,8 @@ func (fASC *fakeAzureSubnetsClient) List(resourceGroupName string, virtualNetwor value = append(value, v) } } - return &fakeSubnetListResultPage{ - value: network.SubnetListResult{ - Value: &value, - }, - values: value, - }, nil + + return value, nil } type fakeAzureNSGClient struct { @@ -454,65 +386,35 @@ func newFakeAzureNSGClient() *fakeAzureNSGClient { return fNSG } -func (fNSG *fakeAzureNSGClient) CreateOrUpdate(resourceGroupName string, networkSecurityGroupName string, parameters network.SecurityGroup, cancel <-chan struct{}) (<-chan network.SecurityGroup, <-chan error) { +func (fNSG *fakeAzureNSGClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, networkSecurityGroupName string, parameters network.SecurityGroup) (resp *http.Response, err error) { fNSG.mutex.Lock() defer fNSG.mutex.Unlock() - resultChan := make(chan network.SecurityGroup, 1) - errChan := make(chan error, 1) - var result network.SecurityGroup - var err error - defer func() { - resultChan <- result - errChan <- err - close(resultChan) - close(errChan) - }() + if _, ok := fNSG.FakeStore[resourceGroupName]; !ok { fNSG.FakeStore[resourceGroupName] = make(map[string]network.SecurityGroup) } fNSG.FakeStore[resourceGroupName][networkSecurityGroupName] = parameters - result = fNSG.FakeStore[resourceGroupName][networkSecurityGroupName] - result.Response.Response = &http.Response{ - StatusCode: http.StatusOK, - } - err = nil - return resultChan, errChan + + return nil, nil } -func (fNSG *fakeAzureNSGClient) Delete(resourceGroupName string, networkSecurityGroupName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) { +func (fNSG *fakeAzureNSGClient) Delete(ctx context.Context, resourceGroupName string, networkSecurityGroupName string) (resp *http.Response, err error) { fNSG.mutex.Lock() defer fNSG.mutex.Unlock() - respChan := make(chan autorest.Response, 1) - errChan := make(chan error, 1) - var resp autorest.Response - var err error - defer func() { - respChan <- resp - errChan <- err - close(respChan) - close(errChan) - }() + if rgSGs, ok := fNSG.FakeStore[resourceGroupName]; ok { if _, ok := rgSGs[networkSecurityGroupName]; ok { delete(rgSGs, networkSecurityGroupName) - resp.Response = &http.Response{ - StatusCode: http.StatusAccepted, - } - err = nil - return respChan, errChan + return nil, nil } } - resp.Response = &http.Response{ + + return &http.Response{ StatusCode: http.StatusNotFound, - } - err = autorest.DetailedError{ - StatusCode: http.StatusNotFound, - Message: "Not such NSG", - } - return respChan, errChan + }, nil } -func (fNSG *fakeAzureNSGClient) Get(resourceGroupName string, networkSecurityGroupName string, expand string) (result network.SecurityGroup, err error) { +func (fNSG *fakeAzureNSGClient) Get(ctx context.Context, resourceGroupName string, networkSecurityGroupName string, expand string) (result network.SecurityGroup, err error) { fNSG.mutex.Lock() defer fNSG.mutex.Unlock() if _, ok := fNSG.FakeStore[resourceGroupName]; ok { @@ -526,43 +428,18 @@ func (fNSG *fakeAzureNSGClient) Get(resourceGroupName string, networkSecurityGro } } -type fakeSecurityGroupListResultPage struct { - next SecurityGroupListResultPage - value network.SecurityGroupListResult - values []network.SecurityGroup - err error -} - -func (pg *fakeSecurityGroupListResultPage) Next() error { - return nil -} -func (pg *fakeSecurityGroupListResultPage) NotDone() bool { - return pg.next != nil -} - -func (pg *fakeSecurityGroupListResultPage) Response() network.SecurityGroupListResult { - return pg.value -} -func (pg *fakeSecurityGroupListResultPage) Values() []network.SecurityGroup { - return pg.values -} - -func (fNSG *fakeAzureNSGClient) List(resourceGroupName string) (result SecurityGroupListResultPage, err error) { +func (fNSG *fakeAzureNSGClient) List(ctx context.Context, resourceGroupName string) (result []network.SecurityGroup, err error) { fNSG.mutex.Lock() defer fNSG.mutex.Unlock() + var value []network.SecurityGroup if _, ok := fNSG.FakeStore[resourceGroupName]; ok { for _, v := range fNSG.FakeStore[resourceGroupName] { value = append(value, v) } } - result = &fakeSecurityGroupListResultPage{ - value: network.SecurityGroupListResult{ - Value: &value, - }, - values: value, - } - return result, nil + + return value, nil } func getRandomIPPtr() *string { @@ -722,66 +599,32 @@ func newFakeRoutesClient() *fakeRoutesClient { return fRC } -func (fRC *fakeRoutesClient) CreateOrUpdate(resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route, cancel <-chan struct{}) (<-chan network.Route, <-chan error) { +func (fRC *fakeRoutesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route) (resp *http.Response, err error) { fRC.mutex.Lock() defer fRC.mutex.Unlock() - resultChan := make(chan network.Route, 1) - errChan := make(chan error, 1) - var result network.Route - var err error - defer func() { - resultChan <- result - errChan <- err - close(resultChan) - close(errChan) - }() - if _, ok := fRC.FakeStore[routeTableName]; !ok { fRC.FakeStore[routeTableName] = make(map[string]network.Route) } fRC.FakeStore[routeTableName][routeName] = routeParameters - result = fRC.FakeStore[routeTableName][routeName] - result.Response.Response = &http.Response{ - StatusCode: http.StatusOK, - } - err = nil - return resultChan, errChan + + return nil, nil } -func (fRC *fakeRoutesClient) Delete(resourceGroupName string, routeTableName string, routeName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) { +func (fRC *fakeRoutesClient) Delete(ctx context.Context, resourceGroupName string, routeTableName string, routeName string) (resp *http.Response, err error) { fRC.mutex.Lock() defer fRC.mutex.Unlock() - respChan := make(chan autorest.Response, 1) - errChan := make(chan error, 1) - var resp autorest.Response - var err error - defer func() { - respChan <- resp - errChan <- err - close(respChan) - close(errChan) - }() if routes, ok := fRC.FakeStore[routeTableName]; ok { if _, ok := routes[routeName]; ok { delete(routes, routeName) - resp.Response = &http.Response{ - StatusCode: http.StatusAccepted, - } - - err = nil - return respChan, errChan + return nil, nil } } - resp.Response = &http.Response{ + + return &http.Response{ StatusCode: http.StatusNotFound, - } - err = autorest.DetailedError{ - StatusCode: http.StatusNotFound, - Message: "Not such Route", - } - return respChan, errChan + }, nil } type fakeRouteTablesClient struct { @@ -797,36 +640,21 @@ func newFakeRouteTablesClient() *fakeRouteTablesClient { return fRTC } -func (fRTC *fakeRouteTablesClient) CreateOrUpdate(resourceGroupName string, routeTableName string, parameters network.RouteTable, cancel <-chan struct{}) (<-chan network.RouteTable, <-chan error) { +func (fRTC *fakeRouteTablesClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, routeTableName string, parameters network.RouteTable) (resp *http.Response, err error) { fRTC.mutex.Lock() defer fRTC.mutex.Unlock() fRTC.Calls = append(fRTC.Calls, "CreateOrUpdate") - resultChan := make(chan network.RouteTable, 1) - errChan := make(chan error, 1) - var result network.RouteTable - var err error - defer func() { - resultChan <- result - errChan <- err - close(resultChan) - close(errChan) - }() - if _, ok := fRTC.FakeStore[resourceGroupName]; !ok { fRTC.FakeStore[resourceGroupName] = make(map[string]network.RouteTable) } fRTC.FakeStore[resourceGroupName][routeTableName] = parameters - result = fRTC.FakeStore[resourceGroupName][routeTableName] - result.Response.Response = &http.Response{ - StatusCode: http.StatusOK, - } - err = nil - return resultChan, errChan + + return nil, nil } -func (fRTC *fakeRouteTablesClient) Get(resourceGroupName string, routeTableName string, expand string) (result network.RouteTable, err error) { +func (fRTC *fakeRouteTablesClient) Get(ctx context.Context, resourceGroupName string, routeTableName string, expand string) (result network.RouteTable, err error) { fRTC.mutex.Lock() defer fRTC.mutex.Unlock() @@ -998,7 +826,7 @@ func (fDC *fakeDisksClient) Get(ctx context.Context, resourceGroupName string, d } type fakeVMSet struct { - NodeToIP map[string]map[string]string + NodeToIP map[string]string Err error } @@ -1010,19 +838,16 @@ func (f *fakeVMSet) GetInstanceTypeByNodeName(name string) (string, error) { return "", fmt.Errorf("unimplemented") } -func (f *fakeVMSet) GetIPByNodeName(name, vmSetName string) (string, string, error) { - nodes, found := f.NodeToIP[vmSetName] - if !found { - return "", "", fmt.Errorf("not found") - } - ip, found := nodes[name] +func (f *fakeVMSet) GetIPByNodeName(name string) (string, string, error) { + ip, found := f.NodeToIP[name] if !found { return "", "", fmt.Errorf("not found") } + return ip, "", nil } -func (f *fakeVMSet) GetPrimaryInterface(nodeName, vmSetName string) (network.Interface, error) { +func (f *fakeVMSet) GetPrimaryInterface(nodeName string) (network.Interface, error) { return network.Interface{}, fmt.Errorf("unimplemented") } diff --git a/pkg/cloudprovider/providers/azure/azure_routes.go b/pkg/cloudprovider/providers/azure/azure_routes.go index 96e5c824934..0230ed6f5dd 100644 --- a/pkg/cloudprovider/providers/azure/azure_routes.go +++ b/pkg/cloudprovider/providers/azure/azure_routes.go @@ -82,11 +82,11 @@ func (az *Cloud) createRouteTable() error { } glog.V(3).Infof("create: creating routetable. routeTableName=%q", az.RouteTableName) - respChan, errChan := az.RouteTablesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, routeTable, nil) - resp := <-respChan - err := <-errChan + ctx, cancel := getContextWithCancel() + defer cancel() + resp, err := az.RouteTablesClient.CreateOrUpdate(ctx, az.ResourceGroup, az.RouteTableName, routeTable) glog.V(10).Infof("RouteTablesClient.CreateOrUpdate(%q): end", az.RouteTableName) - if az.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) { + if az.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { glog.V(2).Infof("create backing off: creating routetable. routeTableName=%q", az.RouteTableName) retryErr := az.CreateOrUpdateRouteTableWithRetry(routeTable) if retryErr != nil { @@ -127,11 +127,11 @@ func (az *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint s } glog.V(3).Infof("create: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR) - respChan, errChan := az.RoutesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, *route.Name, route, nil) - resp := <-respChan - err = <-errChan + ctx, cancel := getContextWithCancel() + defer cancel() + resp, err := az.RoutesClient.CreateOrUpdate(ctx, az.ResourceGroup, az.RouteTableName, *route.Name, route) glog.V(10).Infof("RoutesClient.CreateOrUpdate(%q): end", az.RouteTableName) - if az.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) { + if az.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { glog.V(2).Infof("create backing off: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR) retryErr := az.CreateOrUpdateRouteWithRetry(route) if retryErr != nil { @@ -152,13 +152,13 @@ func (az *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint s func (az *Cloud) DeleteRoute(ctx context.Context, clusterName string, kubeRoute *cloudprovider.Route) error { glog.V(2).Infof("delete: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) + ctx, cancel := getContextWithCancel() + defer cancel() routeName := mapNodeNameToRouteName(kubeRoute.TargetNode) - respChan, errChan := az.RoutesClient.Delete(az.ResourceGroup, az.RouteTableName, routeName, nil) - resp := <-respChan - err := <-errChan + resp, err := az.RoutesClient.Delete(ctx, az.ResourceGroup, az.RouteTableName, routeName) glog.V(10).Infof("RoutesClient.Delete(%q): end", az.RouteTableName) - if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) { + if az.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { glog.V(2).Infof("delete backing off: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) retryErr := az.DeleteRouteWithRetry(routeName) if retryErr != nil { diff --git a/pkg/cloudprovider/providers/azure/azure_routes_test.go b/pkg/cloudprovider/providers/azure/azure_routes_test.go index c879abc3cd4..cc248ee52e6 100644 --- a/pkg/cloudprovider/providers/azure/azure_routes_test.go +++ b/pkg/cloudprovider/providers/azure/azure_routes_test.go @@ -94,10 +94,8 @@ func TestCreateRoute(t *testing.T) { route := cloudprovider.Route{TargetNode: "node", DestinationCIDR: "1.2.3.4/24"} nodeIP := "2.4.6.8" - fakeVM.NodeToIP = map[string]map[string]string{ - "": { - "node": nodeIP, - }, + fakeVM.NodeToIP = map[string]string{ + "node": nodeIP, } err := cloud.CreateRoute(context.TODO(), "cluster", "unused", &route) diff --git a/pkg/cloudprovider/providers/azure/azure_standard.go b/pkg/cloudprovider/providers/azure/azure_standard.go index a7f3b3efa25..e30579febc8 100644 --- a/pkg/cloudprovider/providers/azure/azure_standard.go +++ b/pkg/cloudprovider/providers/azure/azure_standard.go @@ -289,7 +289,7 @@ outer: } func (az *Cloud) getIPForMachine(nodeName types.NodeName) (string, string, error) { - return az.vmSet.GetIPByNodeName(string(nodeName), "") + return az.vmSet.GetIPByNodeName(string(nodeName)) } var polyTable = crc32.MakeTable(crc32.Koopman) @@ -429,8 +429,8 @@ func (as *availabilitySet) GetPrimaryVMSetName() string { } // GetIPByNodeName gets machine private IP and public IP by node name. -func (as *availabilitySet) GetIPByNodeName(name, vmSetName string) (string, string, error) { - nic, err := as.GetPrimaryInterface(name, vmSetName) +func (as *availabilitySet) GetIPByNodeName(name string) (string, string, error) { + nic, err := as.GetPrimaryInterface(name) if err != nil { return "", "", err } @@ -554,8 +554,13 @@ func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) return availabilitySetNames, nil } -// GetPrimaryInterface gets machine primary network interface by node name and vmSet. -func (as *availabilitySet) GetPrimaryInterface(nodeName, vmSetName string) (network.Interface, error) { +// GetPrimaryInterface gets machine primary network interface by node name. +func (as *availabilitySet) GetPrimaryInterface(nodeName string) (network.Interface, error) { + return as.getPrimaryInterfaceWithVMSet(nodeName, "") +} + +// getPrimaryInterfaceWithVMSet gets machine primary network interface by node name and vmSet. +func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName string) (network.Interface, error) { var machine compute.VirtualMachine machine, err := as.GetVirtualMachineWithRetry(types.NodeName(nodeName)) @@ -573,8 +578,13 @@ func (as *availabilitySet) GetPrimaryInterface(nodeName, vmSetName string) (netw return network.Interface{}, err } - // Check availability set. - // Backends of Standard load balancer could belong to multiple VMAS, so we don't check vmSet for it. + // Check availability set name. Note that vmSetName is empty string when getting + // the Node's IP address. While vmSetName is not empty, it should be checked with + // Node's real availability set name: + // - For basic SKU load balancer, errNotInVMSet should be returned if the node's + // availability set is mismatched with vmSetName. + // - For standard SKU load balancer, backend could belong to multiple VMAS, so we + // don't check vmSet for it. if vmSetName != "" && !as.useStandardLoadBalancer() { expectedAvailabilitySetName := as.getAvailabilitySetID(vmSetName) if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) { @@ -584,7 +594,9 @@ func (as *availabilitySet) GetPrimaryInterface(nodeName, vmSetName string) (netw } } - nic, err := as.InterfacesClient.Get(as.ResourceGroup, nicName, "") + ctx, cancel := getContextWithCancel() + defer cancel() + nic, err := as.InterfacesClient.Get(ctx, as.ResourceGroup, nicName, "") if err != nil { return network.Interface{}, err } @@ -596,7 +608,7 @@ func (as *availabilitySet) GetPrimaryInterface(nodeName, vmSetName string) (netw // participating in the specified LoadBalancer Backend Pool. func (as *availabilitySet) ensureHostInPool(serviceName string, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) error { vmName := mapNodeNameToVMName(nodeName) - nic, err := as.GetPrimaryInterface(vmName, vmSetName) + nic, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName) if err != nil { if err == errNotInVMSet { glog.V(3).Infof("ensureHostInPool skips node %s because it is not in the vmSet %s", nodeName, vmSetName) @@ -652,11 +664,11 @@ func (as *availabilitySet) ensureHostInPool(serviceName string, nodeName types.N nicName := *nic.Name glog.V(3).Infof("nicupdate(%s): nic(%s) - updating", serviceName, nicName) - respChan, errChan := as.InterfacesClient.CreateOrUpdate(as.ResourceGroup, *nic.Name, nic, nil) - resp := <-respChan - err := <-errChan + ctx, cancel := getContextWithCancel() + defer cancel() + resp, err := as.InterfacesClient.CreateOrUpdate(ctx, as.ResourceGroup, *nic.Name, nic) glog.V(10).Infof("InterfacesClient.CreateOrUpdate(%q): end", *nic.Name) - if as.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) { + if as.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { glog.V(2).Infof("nicupdate(%s) backing off: nic(%s) - updating, err=%v", serviceName, nicName, err) retryErr := as.CreateOrUpdateInterfaceWithRetry(nic) if retryErr != nil { diff --git a/pkg/cloudprovider/providers/azure/azure_test.go b/pkg/cloudprovider/providers/azure/azure_test.go index 043187ecd12..dfcf2ec92a7 100644 --- a/pkg/cloudprovider/providers/azure/azure_test.go +++ b/pkg/cloudprovider/providers/azure/azure_test.go @@ -1056,7 +1056,9 @@ func getClusterResources(az *Cloud, vmCount int, availabilitySetCount int) (clus }, }, } - az.InterfacesClient.CreateOrUpdate(az.Config.ResourceGroup, nicName, newNIC, nil) + ctx, cancel := getContextWithCancel() + defer cancel() + az.InterfacesClient.CreateOrUpdate(ctx, az.Config.ResourceGroup, nicName, newNIC) // create vm asID := az.getAvailabilitySetID(asName) @@ -1077,9 +1079,9 @@ func getClusterResources(az *Cloud, vmCount int, availabilitySetCount int) (clus }, } - ctx, cancel := getContextWithCancel() - defer cancel() - _, err := az.VirtualMachinesClient.CreateOrUpdate(ctx, az.Config.ResourceGroup, vmName, newVM) + vmCtx, vmCancel := getContextWithCancel() + defer vmCancel() + _, err := az.VirtualMachinesClient.CreateOrUpdate(vmCtx, az.Config.ResourceGroup, vmName, newVM) if err != nil { } // add to kubernetes @@ -1176,11 +1178,13 @@ func getTestSecurityGroup(az *Cloud, services ...v1.Service) *network.SecurityGr }, } + ctx, cancel := getContextWithCancel() + defer cancel() az.SecurityGroupsClient.CreateOrUpdate( + ctx, az.ResourceGroup, az.SecurityGroupName, - sg, - nil) + sg) return &sg } @@ -1854,13 +1858,15 @@ func addTestSubnet(t *testing.T, az *Cloud, svc *v1.Service) { az.VnetName, subName) - _, errChan := az.SubnetsClient.CreateOrUpdate(az.VnetResourceGroup, az.VnetName, subName, + ctx, cancel := getContextWithCancel() + defer cancel() + _, err := az.SubnetsClient.CreateOrUpdate(ctx, az.VnetResourceGroup, az.VnetName, subName, network.Subnet{ ID: &subnetID, Name: &subName, - }, nil) + }) - if err := <-errChan; err != nil { + if err != nil { t.Errorf("Subnet cannot be created or update, %v", err) } svc.Annotations[ServiceAnnotationLoadBalancerInternalSubnet] = subName diff --git a/pkg/cloudprovider/providers/azure/azure_vmsets.go b/pkg/cloudprovider/providers/azure/azure_vmsets.go index 47ea1c89154..34b69e8595c 100644 --- a/pkg/cloudprovider/providers/azure/azure_vmsets.go +++ b/pkg/cloudprovider/providers/azure/azure_vmsets.go @@ -35,9 +35,9 @@ type VMSet interface { // GetInstanceTypeByNodeName gets the instance type by node name. GetInstanceTypeByNodeName(name string) (string, error) // GetIPByNodeName gets machine private IP and public IP by node name. - GetIPByNodeName(name, vmSetName string) (string, string, error) - // GetPrimaryInterface gets machine primary network interface by node name and vmSet. - GetPrimaryInterface(nodeName, vmSetName string) (network.Interface, error) + GetIPByNodeName(name string) (string, string, error) + // GetPrimaryInterface gets machine primary network interface by node name. + GetPrimaryInterface(nodeName string) (network.Interface, error) // GetNodeNameByProviderID gets the node name by provider ID. GetNodeNameByProviderID(providerID string) (types.NodeName, error) diff --git a/pkg/cloudprovider/providers/azure/azure_vmss.go b/pkg/cloudprovider/providers/azure/azure_vmss.go index 4a46996d38f..c7fa190b164 100644 --- a/pkg/cloudprovider/providers/azure/azure_vmss.go +++ b/pkg/cloudprovider/providers/azure/azure_vmss.go @@ -247,10 +247,10 @@ func (ss *scaleSet) GetPrimaryVMSetName() string { // GetIPByNodeName gets machine private IP and public IP by node name. // TODO(feiskyer): Azure vmss doesn't support associating a public IP to single virtual machine yet, // fix this after it is supported. -func (ss *scaleSet) GetIPByNodeName(nodeName, vmSetName string) (string, string, error) { - nic, err := ss.GetPrimaryInterface(nodeName, vmSetName) +func (ss *scaleSet) GetIPByNodeName(nodeName string) (string, string, error) { + nic, err := ss.GetPrimaryInterface(nodeName) if err != nil { - glog.Errorf("error: ss.GetIPByNodeName(%s), GetPrimaryInterface(%q, %q), err=%v", nodeName, nodeName, vmSetName, err) + glog.Errorf("error: ss.GetIPByNodeName(%s), GetPrimaryInterface(%q), err=%v", nodeName, nodeName, err) return "", "", err } @@ -418,7 +418,7 @@ func (ss *scaleSet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) (vmSetN } // GetPrimaryInterface gets machine primary network interface by node name and vmSet. -func (ss *scaleSet) GetPrimaryInterface(nodeName, vmSetName string) (network.Interface, error) { +func (ss *scaleSet) GetPrimaryInterface(nodeName string) (network.Interface, error) { managedByAS, err := ss.isNodeManagedByAvailabilitySet(nodeName) if err != nil { glog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err) @@ -426,7 +426,7 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName, vmSetName string) (network.Int } if managedByAS { // vm is managed by availability set. - return ss.availabilitySet.GetPrimaryInterface(nodeName, "") + return ss.availabilitySet.GetPrimaryInterface(nodeName) } ssName, instanceID, vm, err := ss.getVmssVM(nodeName) @@ -435,12 +435,6 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName, vmSetName string) (network.Int return network.Interface{}, err } - // Check scale set name. - // Backends of Standard load balancer could belong to multiple VMSS, so we don't check vmSet for it. - if !strings.EqualFold(ssName, vmSetName) && !ss.useStandardLoadBalancer() { - return network.Interface{}, errNotInVMSet - } - primaryInterfaceID, err := ss.getPrimaryInterfaceID(vm) if err != nil { glog.Errorf("error: ss.GetPrimaryInterface(%s), ss.getPrimaryInterfaceID(), err=%v", nodeName, err) @@ -453,7 +447,9 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName, vmSetName string) (network.Int return network.Interface{}, err } - nic, err := ss.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface(ss.ResourceGroup, ssName, instanceID, nicName, "") + ctx, cancel := getContextWithCancel() + defer cancel() + nic, err := ss.InterfacesClient.GetVirtualMachineScaleSetNetworkInterface(ctx, ss.ResourceGroup, ssName, instanceID, nicName, "") if err != nil { glog.Errorf("error: ss.GetPrimaryInterface(%s), ss.GetVirtualMachineScaleSetNetworkInterface.Get(%s, %s, %s), err=%v", nodeName, ss.ResourceGroup, ssName, nicName, err) return network.Interface{}, err @@ -697,7 +693,7 @@ func (ss *scaleSet) EnsureHostsInPool(serviceName string, nodes []*v1.Node, back } for ssName, instanceIDs := range scalesets { - // Only add nodes belonging to specified vmSet to basic LB backends. + // Only add nodes belonging to specified vmSet for basic SKU LB. if !ss.useStandardLoadBalancer() && !strings.EqualFold(ssName, vmSetName) { continue } diff --git a/pkg/cloudprovider/providers/azure/azure_vmss_cache.go b/pkg/cloudprovider/providers/azure/azure_vmss_cache.go index b7f1552d7b7..e224845f30c 100644 --- a/pkg/cloudprovider/providers/azure/azure_vmss_cache.go +++ b/pkg/cloudprovider/providers/azure/azure_vmss_cache.go @@ -47,13 +47,19 @@ func (ss *scaleSet) makeVmssVMName(scaleSetName, instanceID string) string { } func extractVmssVMName(name string) (string, string, error) { - ret := strings.Split(name, vmssNameSeparator) - if len(ret) != 2 { + split := strings.SplitAfter(name, vmssNameSeparator) + if len(split) < 2 { glog.Errorf("Failed to extract vmssVMName %q", name) return "", "", ErrorNotVmssInstance } - return ret[0], ret[1], nil + ssName := strings.Join(split[0:len(split)-1], "") + // removing the trailing `vmssNameSeparator` since we used SplitAfter + ssName = ssName[:len(ssName)-1] + + instanceID := split[len(split)-1] + + return ssName, instanceID, nil } func (ss *scaleSet) newVmssCache() (*timedCache, error) { diff --git a/pkg/cloudprovider/providers/azure/azure_vmss_cache_test.go b/pkg/cloudprovider/providers/azure/azure_vmss_cache_test.go index ad8bc4798a2..b6609a6a2ad 100644 --- a/pkg/cloudprovider/providers/azure/azure_vmss_cache_test.go +++ b/pkg/cloudprovider/providers/azure/azure_vmss_cache_test.go @@ -46,6 +46,12 @@ func TestExtractVmssVMName(t *testing.T) { expectedScaleSet: "vm", expectedInstanceID: "1234", }, + { + description: "correct vmss VM name with Extra Separator should return correct scaleSet and instanceID", + vmName: "vm_test_1234", + expectedScaleSet: "vm_test", + expectedInstanceID: "1234", + }, } for _, c := range cases { diff --git a/pkg/cloudprovider/providers/azure/azure_wrap.go b/pkg/cloudprovider/providers/azure/azure_wrap.go index 5e359c295ee..2c30e287220 100644 --- a/pkg/cloudprovider/providers/azure/azure_wrap.go +++ b/pkg/cloudprovider/providers/azure/azure_wrap.go @@ -129,7 +129,9 @@ func (az *Cloud) getSubnet(virtualNetworkName string, subnetName string) (subnet rg = az.ResourceGroup } - subnet, err = az.SubnetsClient.Get(rg, virtualNetworkName, subnetName, "") + ctx, cancel := getContextWithCancel() + defer cancel() + subnet, err = az.SubnetsClient.Get(ctx, rg, virtualNetworkName, subnetName, "") exists, realErr = checkResourceExistsFromError(err) if realErr != nil { return subnet, false, realErr @@ -217,7 +219,9 @@ func (az *Cloud) newLBCache() (*timedCache, error) { func (az *Cloud) newNSGCache() (*timedCache, error) { getter := func(key string) (interface{}, error) { - nsg, err := az.SecurityGroupsClient.Get(az.ResourceGroup, key, "") + ctx, cancel := getContextWithCancel() + defer cancel() + nsg, err := az.SecurityGroupsClient.Get(ctx, az.ResourceGroup, key, "") exists, realErr := checkResourceExistsFromError(err) if realErr != nil { return nil, realErr @@ -235,7 +239,9 @@ func (az *Cloud) newNSGCache() (*timedCache, error) { func (az *Cloud) newRouteTableCache() (*timedCache, error) { getter := func(key string) (interface{}, error) { - rt, err := az.RouteTablesClient.Get(az.ResourceGroup, key, "") + ctx, cancel := getContextWithCancel() + defer cancel() + rt, err := az.RouteTablesClient.Get(ctx, az.ResourceGroup, key, "") exists, realErr := checkResourceExistsFromError(err) if realErr != nil { return nil, realErr diff --git a/pkg/cloudprovider/providers/gce/cloud/gen.go b/pkg/cloudprovider/providers/gce/cloud/gen.go index fb381b7bcd2..1a98eb32074 100644 --- a/pkg/cloudprovider/providers/gce/cloud/gen.go +++ b/pkg/cloudprovider/providers/gce/cloud/gen.go @@ -2526,6 +2526,7 @@ type BackendServices interface { Insert(ctx context.Context, key *meta.Key, obj *ga.BackendService) error Delete(ctx context.Context, key *meta.Key) error GetHealth(context.Context, *meta.Key, *ga.ResourceGroupReference) (*ga.BackendServiceGroupHealth, error) + Patch(context.Context, *meta.Key, *ga.BackendService) error Update(context.Context, *meta.Key, *ga.BackendService) error } @@ -2567,6 +2568,7 @@ type MockBackendServices struct { InsertHook func(ctx context.Context, key *meta.Key, obj *ga.BackendService, m *MockBackendServices) (bool, error) DeleteHook func(ctx context.Context, key *meta.Key, m *MockBackendServices) (bool, error) GetHealthHook func(context.Context, *meta.Key, *ga.ResourceGroupReference, *MockBackendServices) (*ga.BackendServiceGroupHealth, error) + PatchHook func(context.Context, *meta.Key, *ga.BackendService, *MockBackendServices) error UpdateHook func(context.Context, *meta.Key, *ga.BackendService, *MockBackendServices) error // X is extra state that can be used as part of the mock. Generated code @@ -2721,6 +2723,14 @@ func (m *MockBackendServices) GetHealth(ctx context.Context, key *meta.Key, arg0 return nil, fmt.Errorf("GetHealthHook must be set") } +// Patch is a mock for the corresponding method. +func (m *MockBackendServices) Patch(ctx context.Context, key *meta.Key, arg0 *ga.BackendService) error { + if m.PatchHook != nil { + return m.PatchHook(ctx, key, arg0, m) + } + return nil +} + // Update is a mock for the corresponding method. func (m *MockBackendServices) Update(ctx context.Context, key *meta.Key, arg0 *ga.BackendService) error { if m.UpdateHook != nil { @@ -2899,6 +2909,39 @@ func (g *GCEBackendServices) GetHealth(ctx context.Context, key *meta.Key, arg0 return v, err } +// Patch is a method on GCEBackendServices. +func (g *GCEBackendServices) Patch(ctx context.Context, key *meta.Key, arg0 *ga.BackendService) error { + glog.V(5).Infof("GCEBackendServices.Patch(%v, %v, ...): called", ctx, key) + + if !key.Valid() { + glog.V(2).Infof("GCEBackendServices.Patch(%v, %v, ...): key is invalid (%#v)", ctx, key, key) + return fmt.Errorf("invalid GCE key (%+v)", key) + } + projectID := g.s.ProjectRouter.ProjectID(ctx, "ga", "BackendServices") + rk := &RateLimitKey{ + ProjectID: projectID, + Operation: "Patch", + Version: meta.Version("ga"), + Service: "BackendServices", + } + glog.V(5).Infof("GCEBackendServices.Patch(%v, %v, ...): projectID = %v, rk = %+v", ctx, key, projectID, rk) + + if err := g.s.RateLimiter.Accept(ctx, rk); err != nil { + glog.V(4).Infof("GCEBackendServices.Patch(%v, %v, ...): RateLimiter error: %v", ctx, key, err) + return err + } + call := g.s.GA.BackendServices.Patch(projectID, key.Name, arg0) + call.Context(ctx) + op, err := call.Do() + if err != nil { + glog.V(4).Infof("GCEBackendServices.Patch(%v, %v, ...) = %+v", ctx, key, err) + return err + } + err = g.s.WaitForCompletion(ctx, op) + glog.V(4).Infof("GCEBackendServices.Patch(%v, %v, ...) = %+v", ctx, key, err) + return err +} + // Update is a method on GCEBackendServices. func (g *GCEBackendServices) Update(ctx context.Context, key *meta.Key, arg0 *ga.BackendService) error { glog.V(5).Infof("GCEBackendServices.Update(%v, %v, ...): called", ctx, key) diff --git a/pkg/cloudprovider/providers/gce/cloud/meta/meta.go b/pkg/cloudprovider/providers/gce/cloud/meta/meta.go index 852248beb80..dc0f5264e01 100644 --- a/pkg/cloudprovider/providers/gce/cloud/meta/meta.go +++ b/pkg/cloudprovider/providers/gce/cloud/meta/meta.go @@ -103,6 +103,7 @@ var AllServices = []*ServiceInfo{ serviceType: reflect.TypeOf(&ga.BackendServicesService{}), additionalMethods: []string{ "GetHealth", + "Patch", "Update", }, }, diff --git a/pkg/cloudprovider/providers/vsphere/BUILD b/pkg/cloudprovider/providers/vsphere/BUILD index 0566d806411..ef7c3435c11 100644 --- a/pkg/cloudprovider/providers/vsphere/BUILD +++ b/pkg/cloudprovider/providers/vsphere/BUILD @@ -39,6 +39,7 @@ go_test( deps = [ "//pkg/cloudprovider:go_default_library", "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library", + "//vendor/github.com/vmware/govmomi/simulator:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library", ], diff --git a/pkg/cloudprovider/providers/vsphere/nodemanager.go b/pkg/cloudprovider/providers/vsphere/nodemanager.go index 4979d605871..25f8d58c9e2 100644 --- a/pkg/cloudprovider/providers/vsphere/nodemanager.go +++ b/pkg/cloudprovider/providers/vsphere/nodemanager.go @@ -360,7 +360,7 @@ func (nm *NodeManager) renewNodeInfo(nodeInfo *NodeInfo, reconnect bool) (*NodeI return nil, err } } - vm := nodeInfo.vm.RenewVM(vsphereInstance.conn.GoVmomiClient) + vm := nodeInfo.vm.RenewVM(vsphereInstance.conn.Client) return &NodeInfo{vm: &vm, dataCenter: vm.Datacenter, vcServer: nodeInfo.vcServer}, nil } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/BUILD b/pkg/cloudprovider/providers/vsphere/vclib/BUILD index ca4eb530d7d..28efd2b3818 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/BUILD +++ b/pkg/cloudprovider/providers/vsphere/vclib/BUILD @@ -26,7 +26,6 @@ go_library( deps = [ "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", - "//vendor/github.com/vmware/govmomi:go_default_library", "//vendor/github.com/vmware/govmomi/find:go_default_library", "//vendor/github.com/vmware/govmomi/object:go_default_library", "//vendor/github.com/vmware/govmomi/pbm:go_default_library", diff --git a/pkg/cloudprovider/providers/vsphere/vclib/connection.go b/pkg/cloudprovider/providers/vsphere/vclib/connection.go index dda185ab0a3..6181eba3f72 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/connection.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/connection.go @@ -18,19 +18,19 @@ package vclib import ( "context" - "fmt" + "net" neturl "net/url" "sync" "github.com/golang/glog" - "github.com/vmware/govmomi" "github.com/vmware/govmomi/session" "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/soap" ) // VSphereConnection contains information for connecting to vCenter type VSphereConnection struct { - GoVmomiClient *govmomi.Client + Client *vim25.Client Username string Password string Hostname string @@ -43,23 +43,23 @@ var ( clientLock sync.Mutex ) -// Connect makes connection to vCenter and sets VSphereConnection.GoVmomiClient. -// If connection.GoVmomiClient is already set, it obtains the existing user session. -// if user session is not valid, connection.GoVmomiClient will be set to the new client. +// Connect makes connection to vCenter and sets VSphereConnection.Client. +// If connection.Client is already set, it obtains the existing user session. +// if user session is not valid, connection.Client will be set to the new client. func (connection *VSphereConnection) Connect(ctx context.Context) error { var err error clientLock.Lock() defer clientLock.Unlock() - if connection.GoVmomiClient == nil { - connection.GoVmomiClient, err = connection.NewClient(ctx) + if connection.Client == nil { + connection.Client, err = connection.NewClient(ctx) if err != nil { glog.Errorf("Failed to create govmomi client. err: %+v", err) return err } return nil } - m := session.NewManager(connection.GoVmomiClient.Client) + m := session.NewManager(connection.Client) userSession, err := m.UserSession(ctx) if err != nil { glog.Errorf("Error while obtaining user session. err: %+v", err) @@ -69,8 +69,8 @@ func (connection *VSphereConnection) Connect(ctx context.Context) error { return nil } glog.Warningf("Creating new client session since the existing session is not valid or not authenticated") - connection.GoVmomiClient.Logout(ctx) - connection.GoVmomiClient, err = connection.NewClient(ctx) + + connection.Client, err = connection.NewClient(ctx) if err != nil { glog.Errorf("Failed to create govmomi client. err: %+v", err) return err @@ -78,19 +78,36 @@ func (connection *VSphereConnection) Connect(ctx context.Context) error { return nil } +// Logout calls SessionManager.Logout for the given connection. +func (connection *VSphereConnection) Logout(ctx context.Context) { + m := session.NewManager(connection.Client) + if err := m.Logout(ctx); err != nil { + glog.Errorf("Logout failed: %s", err) + } +} + // NewClient creates a new govmomi client for the VSphereConnection obj -func (connection *VSphereConnection) NewClient(ctx context.Context) (*govmomi.Client, error) { - url, err := neturl.Parse(fmt.Sprintf("https://%s:%s/sdk", connection.Hostname, connection.Port)) +func (connection *VSphereConnection) NewClient(ctx context.Context) (*vim25.Client, error) { + url, err := soap.ParseURL(net.JoinHostPort(connection.Hostname, connection.Port)) if err != nil { glog.Errorf("Failed to parse URL: %s. err: %+v", url, err) return nil, err } - url.User = neturl.UserPassword(connection.Username, connection.Password) - client, err := govmomi.NewClient(ctx, url, connection.Insecure) + + sc := soap.NewClient(url, connection.Insecure) + client, err := vim25.NewClient(ctx, sc) if err != nil { glog.Errorf("Failed to create new client. err: %+v", err) return nil, err } + + m := session.NewManager(client) + + err = m.Login(ctx, neturl.UserPassword(connection.Username, connection.Password)) + if err != nil { + return nil, err + } + if connection.RoundTripperCount == 0 { connection.RoundTripperCount = RoundTripperDefaultCount } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/constants.go b/pkg/cloudprovider/providers/vsphere/vclib/constants.go index 451d9241180..522f308b8b3 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/constants.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/constants.go @@ -53,7 +53,8 @@ const ( // Test Constants const ( - testDefaultDatacenter = "DC0" - testDefaultDatastore = "LocalDS_0" + TestDefaultDatacenter = "DC0" + TestDefaultDatastore = "LocalDS_0" + TestDefaultNetwork = "VM Network" testNameNotFound = "enoent" ) diff --git a/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go b/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go index fa4f1c3d76a..d60448d1ccf 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go @@ -39,7 +39,7 @@ type Datacenter struct { // GetDatacenter returns the DataCenter Object for the given datacenterPath // If datacenter is located in a folder, include full path to datacenter else just provide the datacenter name func GetDatacenter(ctx context.Context, connection *VSphereConnection, datacenterPath string) (*Datacenter, error) { - finder := find.NewFinder(connection.GoVmomiClient.Client, false) + finder := find.NewFinder(connection.Client, false) datacenter, err := finder.Datacenter(ctx, datacenterPath) if err != nil { glog.Errorf("Failed to find the datacenter: %s. err: %+v", datacenterPath, err) @@ -52,7 +52,7 @@ func GetDatacenter(ctx context.Context, connection *VSphereConnection, datacente // GetAllDatacenter returns all the DataCenter Objects func GetAllDatacenter(ctx context.Context, connection *VSphereConnection) ([]*Datacenter, error) { var dc []*Datacenter - finder := find.NewFinder(connection.GoVmomiClient.Client, false) + finder := find.NewFinder(connection.Client, false) datacenters, err := finder.DatacenterList(ctx, "*") if err != nil { glog.Errorf("Failed to find the datacenter. err: %+v", err) diff --git a/pkg/cloudprovider/providers/vsphere/vclib/datacenter_test.go b/pkg/cloudprovider/providers/vsphere/vclib/datacenter_test.go index 284a3edafc9..ad7e13921ae 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/datacenter_test.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/datacenter_test.go @@ -47,14 +47,14 @@ func TestDatacenter(t *testing.T) { t.Fatal(err) } - vc := &VSphereConnection{GoVmomiClient: c} + vc := &VSphereConnection{Client: c.Client} _, err = GetDatacenter(ctx, vc, testNameNotFound) if err == nil { t.Error("expected error") } - dc, err := GetDatacenter(ctx, vc, testDefaultDatacenter) + dc, err := GetDatacenter(ctx, vc, TestDefaultDatacenter) if err != nil { t.Error(err) } @@ -74,7 +74,7 @@ func TestDatacenter(t *testing.T) { t.Error("expected error") } - vm, err := dc.GetVMByPath(ctx, testDefaultDatacenter+"/vm/"+avm.Name) + vm, err := dc.GetVMByPath(ctx, TestDefaultDatacenter+"/vm/"+avm.Name) if err != nil { t.Error(err) } @@ -103,7 +103,7 @@ func TestDatacenter(t *testing.T) { t.Error("expected error") } - ds, err := dc.GetDatastoreByName(ctx, testDefaultDatastore) + ds, err := dc.GetDatastoreByName(ctx, TestDefaultDatastore) if err != nil { t.Error(err) } @@ -113,7 +113,7 @@ func TestDatacenter(t *testing.T) { t.Error("expected error") } - _, err = dc.GetFolderByPath(ctx, testDefaultDatacenter+"/vm") + _, err = dc.GetFolderByPath(ctx, TestDefaultDatacenter+"/vm") if err != nil { t.Error(err) } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/datastore_test.go b/pkg/cloudprovider/providers/vsphere/vclib/datastore_test.go index 4300e4d6f8c..6c6e888cb3e 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/datastore_test.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/datastore_test.go @@ -45,9 +45,9 @@ func TestDatastore(t *testing.T) { t.Fatal(err) } - vc := &VSphereConnection{GoVmomiClient: c} + vc := &VSphereConnection{Client: c.Client} - dc, err := GetDatacenter(ctx, vc, testDefaultDatacenter) + dc, err := GetDatacenter(ctx, vc, TestDefaultDatacenter) if err != nil { t.Error(err) } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/folder_test.go b/pkg/cloudprovider/providers/vsphere/vclib/folder_test.go index 315d008dc09..be570d80dcd 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/folder_test.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/folder_test.go @@ -47,9 +47,9 @@ func TestFolder(t *testing.T) { t.Fatal(err) } - vc := &VSphereConnection{GoVmomiClient: c} + vc := &VSphereConnection{Client: c.Client} - dc, err := GetDatacenter(ctx, vc, testDefaultDatacenter) + dc, err := GetDatacenter(ctx, vc, TestDefaultDatacenter) if err != nil { t.Error(err) } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/utils_test.go b/pkg/cloudprovider/providers/vsphere/vclib/utils_test.go index ab4bd89df42..436055cf172 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/utils_test.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/utils_test.go @@ -46,9 +46,9 @@ func TestUtils(t *testing.T) { t.Fatal(err) } - vc := &VSphereConnection{GoVmomiClient: c} + vc := &VSphereConnection{Client: c.Client} - dc, err := GetDatacenter(ctx, vc, testDefaultDatacenter) + dc, err := GetDatacenter(ctx, vc, TestDefaultDatacenter) if err != nil { t.Error(err) } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go index 679d827adc3..01654b3d1ef 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go @@ -23,9 +23,9 @@ import ( "time" "github.com/golang/glog" - "github.com/vmware/govmomi" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" ) @@ -403,8 +403,8 @@ func (vm *VirtualMachine) deleteController(ctx context.Context, controllerDevice } // RenewVM renews this virtual machine with new client connection. -func (vm *VirtualMachine) RenewVM(client *govmomi.Client) VirtualMachine { - dc := Datacenter{Datacenter: object.NewDatacenter(client.Client, vm.Datacenter.Reference())} - newVM := object.NewVirtualMachine(client.Client, vm.VirtualMachine.Reference()) +func (vm *VirtualMachine) RenewVM(client *vim25.Client) VirtualMachine { + dc := Datacenter{Datacenter: object.NewDatacenter(client, vm.Datacenter.Reference())} + newVM := object.NewVirtualMachine(client, vm.VirtualMachine.Reference()) return VirtualMachine{VirtualMachine: newVM, Datacenter: &dc} } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine_test.go b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine_test.go index 0b38fd1be4f..ca2ef9665e4 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine_test.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine_test.go @@ -43,9 +43,9 @@ func TestVirtualMachine(t *testing.T) { t.Fatal(err) } - vc := &VSphereConnection{GoVmomiClient: c} + vc := &VSphereConnection{Client: c.Client} - dc, err := GetDatacenter(ctx, vc, testDefaultDatacenter) + dc, err := GetDatacenter(ctx, vc, TestDefaultDatacenter) if err != nil { t.Error(err) } diff --git a/pkg/cloudprovider/providers/vsphere/vsphere.go b/pkg/cloudprovider/providers/vsphere/vsphere.go index cac821eb556..c9390077849 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere.go @@ -360,7 +360,10 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance return vsphereInstanceMap, nil } -// Creates new Contreoller node interface and returns +// getVMUUID allows tests to override GetVMUUID +var getVMUUID = GetVMUUID + +// Creates new Controller node interface and returns func newControllerNode(cfg VSphereConfig) (*VSphere, error) { var err error @@ -399,7 +402,7 @@ func newControllerNode(cfg VSphereConfig) (*VSphere, error) { glog.Errorf("Failed to get hostname. err: %+v", err) return nil, err } - vs.vmUUID, err = GetVMUUID() + vs.vmUUID, err = getVMUUID() if err != nil { glog.Errorf("Failed to get uuid. err: %+v", err) return nil, err @@ -410,8 +413,8 @@ func newControllerNode(cfg VSphereConfig) (*VSphere, error) { func logout(vs *VSphere) { for _, vsphereIns := range vs.vsphereInstanceMap { - if vsphereIns.conn.GoVmomiClient != nil { - vsphereIns.conn.GoVmomiClient.Logout(context.TODO()) + if vsphereIns.conn.Client != nil { + vsphereIns.conn.Logout(context.TODO()) } } diff --git a/pkg/cloudprovider/providers/vsphere/vsphere_test.go b/pkg/cloudprovider/providers/vsphere/vsphere_test.go index 84b96644134..35e1903aea5 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere_test.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere_test.go @@ -18,12 +18,14 @@ package vsphere import ( "context" + "crypto/tls" "log" "os" "strconv" "strings" "testing" + "github.com/vmware/govmomi/simulator" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/kubernetes/pkg/cloudprovider" @@ -59,6 +61,50 @@ func configFromEnv() (cfg VSphereConfig, ok bool) { return } +// configFromEnvOrSim returns config from configFromEnv if set, +// otherwise starts a vcsim instance and returns config for use against the vcsim instance. +func configFromEnvOrSim() (VSphereConfig, func()) { + cfg, ok := configFromEnv() + if ok { + return cfg, func() {} + } + + model := simulator.VPX() + + err := model.Create() + if err != nil { + log.Fatal(err) + } + + model.Service.TLS = new(tls.Config) + s := model.Service.NewServer() + + cfg.Global.InsecureFlag = true + cfg.Global.VCenterIP = s.URL.Hostname() + cfg.Global.VCenterPort = s.URL.Port() + cfg.Global.User = s.URL.User.Username() + cfg.Global.Password, _ = s.URL.User.Password() + cfg.Global.Datacenter = vclib.TestDefaultDatacenter + cfg.Network.PublicNetwork = vclib.TestDefaultNetwork + cfg.Global.DefaultDatastore = vclib.TestDefaultDatastore + cfg.Disk.SCSIControllerType = os.Getenv("VSPHERE_SCSICONTROLLER_TYPE") + cfg.Global.WorkingDir = os.Getenv("VSPHERE_WORKING_DIR") + cfg.Global.VMName = os.Getenv("VSPHERE_VM_NAME") + + if cfg.Global.WorkingDir == "" { + cfg.Global.WorkingDir = "vm" // top-level Datacenter.VmFolder + } + + uuid := simulator.Map.Any("VirtualMachine").(*simulator.VirtualMachine).Config.Uuid + getVMUUID = func() (string, error) { return uuid, nil } + + return cfg, func() { + getVMUUID = GetVMUUID + s.Close() + model.Remove() + } +} + func TestReadConfig(t *testing.T) { _, err := readConfig(nil) if err == nil { @@ -110,10 +156,8 @@ func TestNewVSphere(t *testing.T) { } func TestVSphereLogin(t *testing.T) { - cfg, ok := configFromEnv() - if !ok { - t.Skipf("No config found in environment") - } + cfg, cleanup := configFromEnvOrSim() + defer cleanup() // Create vSphere configuration object vs, err := newControllerNode(cfg) @@ -126,8 +170,8 @@ func TestVSphereLogin(t *testing.T) { defer cancel() // Create vSphere client - var vcInstance *VSphereInstance - if vcInstance, ok = vs.vsphereInstanceMap[cfg.Global.VCenterIP]; !ok { + vcInstance, ok := vs.vsphereInstanceMap[cfg.Global.VCenterIP] + if !ok { t.Fatalf("Couldn't get vSphere instance: %s", cfg.Global.VCenterIP) } @@ -135,7 +179,7 @@ func TestVSphereLogin(t *testing.T) { if err != nil { t.Errorf("Failed to connect to vSphere: %s", err) } - defer vcInstance.conn.GoVmomiClient.Logout(ctx) + defer vcInstance.conn.Logout(ctx) } func TestZones(t *testing.T) { diff --git a/pkg/controller/endpoint/endpoints_controller.go b/pkg/controller/endpoint/endpoints_controller.go index ca7b50678bc..e5a04bceeb8 100644 --- a/pkg/controller/endpoint/endpoints_controller.go +++ b/pkg/controller/endpoint/endpoints_controller.go @@ -372,7 +372,7 @@ func (e *EndpointController) handleErr(err error, key interface{}) { } if e.queue.NumRequeues(key) < maxRetries { - glog.V(2).Infof("Error syncing endpoints for service %q: %v", key, err) + glog.V(2).Infof("Error syncing endpoints for service %q, retrying. Error: %v", key, err) e.queue.AddRateLimited(key) return } diff --git a/pkg/controller/garbagecollector/BUILD b/pkg/controller/garbagecollector/BUILD index 6fca5ce338e..37f439ed780 100644 --- a/pkg/controller/garbagecollector/BUILD +++ b/pkg/controller/garbagecollector/BUILD @@ -54,14 +54,12 @@ go_test( deps = [ "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core/install:go_default_library", - "//pkg/controller/garbagecollector/metaonly:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/meta/testrestmapper:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/json:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/pkg/controller/garbagecollector/garbagecollector.go b/pkg/controller/garbagecollector/garbagecollector.go index bce61b37904..4d05dee56cf 100644 --- a/pkg/controller/garbagecollector/garbagecollector.go +++ b/pkg/controller/garbagecollector/garbagecollector.go @@ -59,10 +59,8 @@ const ResourceResyncTime time.Duration = 0 // ensures that the garbage collector operates with a graph that is at least as // up to date as the notification is sent. type GarbageCollector struct { - restMapper resettableRESTMapper - // clientPool uses the regular dynamicCodec. We need it to update - // finalizers. It can be removed if we support patching finalizers. - clientPool dynamic.ClientPool + restMapper resettableRESTMapper + dynamicClient dynamic.DynamicInterface // garbage collector attempts to delete the items in attemptToDelete queue when the time is ripe. attemptToDelete workqueue.RateLimitingInterface // garbage collector attempts to orphan the dependents of the items in the attemptToOrphan queue, then deletes the items. @@ -76,8 +74,7 @@ type GarbageCollector struct { } func NewGarbageCollector( - metaOnlyClientPool dynamic.ClientPool, - clientPool dynamic.ClientPool, + dynamicClient dynamic.DynamicInterface, mapper resettableRESTMapper, deletableResources map[schema.GroupVersionResource]struct{}, ignoredResources map[schema.GroupResource]struct{}, @@ -88,17 +85,17 @@ func NewGarbageCollector( attemptToOrphan := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "garbage_collector_attempt_to_orphan") absentOwnerCache := NewUIDCache(500) gc := &GarbageCollector{ - clientPool: clientPool, + dynamicClient: dynamicClient, restMapper: mapper, attemptToDelete: attemptToDelete, attemptToOrphan: attemptToOrphan, absentOwnerCache: absentOwnerCache, } gb := &GraphBuilder{ - metaOnlyClientPool: metaOnlyClientPool, - informersStarted: informersStarted, - restMapper: mapper, - graphChanges: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "garbage_collector_graph_changes"), + dynamicClient: dynamicClient, + informersStarted: informersStarted, + restMapper: mapper, + graphChanges: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "garbage_collector_graph_changes"), uidToNode: &concurrentUIDToNode{ uidToNode: make(map[types.UID]*node), }, @@ -291,19 +288,15 @@ func (gc *GarbageCollector) isDangling(reference metav1.OwnerReference, item *no // ii) should update the object to remove such references. This is to // prevent objects having references to an old resource from being // deleted during a cluster upgrade. - fqKind := schema.FromAPIVersionAndKind(reference.APIVersion, reference.Kind) - client, err := gc.clientPool.ClientForGroupVersionKind(fqKind) - if err != nil { - return false, nil, err - } - resource, err := gc.apiResource(reference.APIVersion, reference.Kind) + resource, namespaced, err := gc.apiResource(reference.APIVersion, reference.Kind) if err != nil { return false, nil, err } + // TODO: It's only necessary to talk to the API server if the owner node // is a "virtual" node. The local graph could lag behind the real // status, but in practice, the difference is small. - owner, err = client.Resource(resource, resourceDefaultNamespace(resource, item.identity.Namespace)).Get(reference.Name, metav1.GetOptions{}) + owner, err = gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.identity.Namespace)).Get(reference.Name, metav1.GetOptions{}) switch { case errors.IsNotFound(err): gc.absentOwnerCache.Add(reference.UID) diff --git a/pkg/controller/garbagecollector/garbagecollector_test.go b/pkg/controller/garbagecollector/garbagecollector_test.go index aadbf179c6c..96682e4c6da 100644 --- a/pkg/controller/garbagecollector/garbagecollector_test.go +++ b/pkg/controller/garbagecollector/garbagecollector_test.go @@ -35,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/api/meta/testrestmapper" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/sets" @@ -48,7 +47,6 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/controller/garbagecollector/metaonly" ) type testRESTMapper struct { @@ -59,12 +57,13 @@ func (_ *testRESTMapper) Reset() {} func TestGarbageCollectorConstruction(t *testing.T) { config := &restclient.Config{} - config.ContentConfig.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: metaonly.NewMetadataCodecFactory()} tweakableRM := meta.NewDefaultRESTMapper(nil) - rm := &testRESTMapper{meta.MultiRESTMapper{tweakableRM, testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme)}} - metaOnlyClientPool := dynamic.NewClientPool(config, rm, dynamic.LegacyAPIPathResolverFunc) - config.ContentConfig.NegotiatedSerializer = nil - clientPool := dynamic.NewClientPool(config, rm, dynamic.LegacyAPIPathResolverFunc) + rm := &testRESTMapper{meta.MultiRESTMapper{tweakableRM, testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme)}} + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + t.Fatal(err) + } + podResource := map[schema.GroupVersionResource]struct{}{ {Version: "v1", Resource: "pods"}: {}, } @@ -79,7 +78,7 @@ func TestGarbageCollectorConstruction(t *testing.T) { // construction will not fail. alwaysStarted := make(chan struct{}) close(alwaysStarted) - gc, err := NewGarbageCollector(metaOnlyClientPool, clientPool, rm, twoResources, map[schema.GroupResource]struct{}{}, sharedInformers, alwaysStarted) + gc, err := NewGarbageCollector(dynamicClient, rm, twoResources, map[schema.GroupResource]struct{}{}, sharedInformers, alwaysStarted) if err != nil { t.Fatal(err) } @@ -190,16 +189,17 @@ type garbageCollector struct { } func setupGC(t *testing.T, config *restclient.Config) garbageCollector { - config.ContentConfig.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: metaonly.NewMetadataCodecFactory()} - metaOnlyClientPool := dynamic.NewClientPool(config, testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme), dynamic.LegacyAPIPathResolverFunc) - config.ContentConfig.NegotiatedSerializer = nil - clientPool := dynamic.NewClientPool(config, testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme), dynamic.LegacyAPIPathResolverFunc) + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + t.Fatal(err) + } + podResource := map[schema.GroupVersionResource]struct{}{{Version: "v1", Resource: "pods"}: {}} client := fake.NewSimpleClientset() sharedInformers := informers.NewSharedInformerFactory(client, 0) alwaysStarted := make(chan struct{}) close(alwaysStarted) - gc, err := NewGarbageCollector(metaOnlyClientPool, clientPool, &testRESTMapper{testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme)}, podResource, ignoredResources, sharedInformers, alwaysStarted) + gc, err := NewGarbageCollector(dynamicClient, &testRESTMapper{testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme)}, podResource, ignoredResources, sharedInformers, alwaysStarted) if err != nil { t.Fatal(err) } @@ -434,13 +434,13 @@ func TestGCListWatcher(t *testing.T) { testHandler := &fakeActionHandler{} srv, clientConfig := testServerAndClientConfig(testHandler.ServeHTTP) defer srv.Close() - clientPool := dynamic.NewClientPool(clientConfig, testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme), dynamic.LegacyAPIPathResolverFunc) podResource := schema.GroupVersionResource{Version: "v1", Resource: "pods"} - client, err := clientPool.ClientForGroupVersionResource(podResource) + dynamicClient, err := dynamic.NewForConfig(clientConfig) if err != nil { t.Fatal(err) } - lw := listWatcher(client, podResource) + + lw := listWatcher(dynamicClient, podResource) lw.DisableChunking = true if _, err := lw.Watch(metav1.ListOptions{ResourceVersion: "1"}); err != nil { t.Fatal(err) @@ -823,16 +823,19 @@ func TestGarbageCollectorSync(t *testing.T) { t.Fatal(err) } - rm := &testRESTMapper{testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme)} - metaOnlyClientPool := dynamic.NewClientPool(clientConfig, rm, dynamic.LegacyAPIPathResolverFunc) - clientPool := dynamic.NewClientPool(clientConfig, rm, dynamic.LegacyAPIPathResolverFunc) + rm := &testRESTMapper{testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme)} + dynamicClient, err := dynamic.NewForConfig(clientConfig) + if err != nil { + t.Fatal(err) + } + podResource := map[schema.GroupVersionResource]struct{}{ {Group: "", Version: "v1", Resource: "pods"}: {}, } sharedInformers := informers.NewSharedInformerFactory(client, 0) alwaysStarted := make(chan struct{}) close(alwaysStarted) - gc, err := NewGarbageCollector(metaOnlyClientPool, clientPool, rm, podResource, map[schema.GroupResource]struct{}{}, sharedInformers, alwaysStarted) + gc, err := NewGarbageCollector(dynamicClient, rm, podResource, map[schema.GroupResource]struct{}{}, sharedInformers, alwaysStarted) if err != nil { t.Fatal(err) } diff --git a/pkg/controller/garbagecollector/graph_builder.go b/pkg/controller/garbagecollector/graph_builder.go index b21548352f2..1fbb7a1ce7f 100644 --- a/pkg/controller/garbagecollector/graph_builder.go +++ b/pkg/controller/garbagecollector/graph_builder.go @@ -91,6 +91,7 @@ type GraphBuilder struct { // it is protected by monitorLock. running bool + dynamicClient dynamic.DynamicInterface // metaOnlyClientPool uses a special codec, which removes fields except for // apiVersion, kind, and metadata during decoding. metaOnlyClientPool dynamic.ClientPool @@ -127,27 +128,15 @@ func (m *monitor) Run() { type monitors map[schema.GroupVersionResource]*monitor -func listWatcher(client dynamic.Interface, resource schema.GroupVersionResource) *cache.ListWatch { +func listWatcher(client dynamic.DynamicInterface, resource schema.GroupVersionResource) *cache.ListWatch { return &cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - // APIResource.Kind is not used by the dynamic client, so - // leave it empty. We want to list this resource in all - // namespaces if it's namespace scoped, so leave - // APIResource.Namespaced as false is all right. - apiResource := metav1.APIResource{Name: resource.Resource} - return client. - Resource(&apiResource, metav1.NamespaceAll). - List(options) + // We want to list this resource in all namespaces if it's namespace scoped, so not passing namespace is ok. + return client.Resource(resource).List(options) }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - // APIResource.Kind is not used by the dynamic client, so - // leave it empty. We want to list this resource in all - // namespaces if it's namespace scoped, so leave - // APIResource.Namespaced as false is all right. - apiResource := metav1.APIResource{Name: resource.Resource} - return client. - Resource(&apiResource, metav1.NamespaceAll). - Watch(options) + // We want to list this resource in all namespaces if it's namespace scoped, so not passing namespace is ok. + return client.Resource(resource).Watch(options) }, } } @@ -199,12 +188,8 @@ func (gb *GraphBuilder) controllerFor(resource schema.GroupVersionResource, kind // TODO: consider store in one storage. glog.V(5).Infof("create storage for resource %s", resource) - client, err := gb.metaOnlyClientPool.ClientForGroupVersionKind(kind) - if err != nil { - return nil, err - } _, monitor := cache.NewInformer( - listWatcher(client, resource), + listWatcher(gb.dynamicClient, resource), nil, ResourceResyncTime, // don't need to clone because it's not from shared cache diff --git a/pkg/controller/garbagecollector/operations.go b/pkg/controller/garbagecollector/operations.go index f1cc538e1e4..29025692b89 100644 --- a/pkg/controller/garbagecollector/operations.go +++ b/pkg/controller/garbagecollector/operations.go @@ -31,8 +31,8 @@ import ( ) // cluster scoped resources don't have namespaces. Default to the item's namespace, but clear it for cluster scoped resources -func resourceDefaultNamespace(resource *metav1.APIResource, defaultNamespace string) string { - if resource.Namespaced { +func resourceDefaultNamespace(namespaced bool, defaultNamespace string) string { + if namespaced { return defaultNamespace } return "" @@ -40,74 +40,48 @@ func resourceDefaultNamespace(resource *metav1.APIResource, defaultNamespace str // apiResource consults the REST mapper to translate an tuple to a unversioned.APIResource struct. -func (gc *GarbageCollector) apiResource(apiVersion, kind string) (*metav1.APIResource, error) { +func (gc *GarbageCollector) apiResource(apiVersion, kind string) (schema.GroupVersionResource, bool, error) { fqKind := schema.FromAPIVersionAndKind(apiVersion, kind) mapping, err := gc.restMapper.RESTMapping(fqKind.GroupKind(), fqKind.Version) if err != nil { - return nil, newRESTMappingError(kind, apiVersion) + return schema.GroupVersionResource{}, false, newRESTMappingError(kind, apiVersion) } - glog.V(5).Infof("map kind %s, version %s to resource %s", kind, apiVersion, mapping.Resource) - resource := metav1.APIResource{ - Name: mapping.Resource.Resource, - Namespaced: mapping.Scope == meta.RESTScopeNamespace, - Kind: kind, - } - return &resource, nil + return mapping.Resource, mapping.Scope == meta.RESTScopeNamespace, nil } func (gc *GarbageCollector) deleteObject(item objectReference, policy *metav1.DeletionPropagation) error { - fqKind := schema.FromAPIVersionAndKind(item.APIVersion, item.Kind) - client, err := gc.clientPool.ClientForGroupVersionKind(fqKind) - if err != nil { - return err - } - resource, err := gc.apiResource(item.APIVersion, item.Kind) + resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind) if err != nil { return err } uid := item.UID preconditions := metav1.Preconditions{UID: &uid} deleteOptions := metav1.DeleteOptions{Preconditions: &preconditions, PropagationPolicy: policy} - return client.Resource(resource, resourceDefaultNamespace(resource, item.Namespace)).Delete(item.Name, &deleteOptions) + return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Delete(item.Name, &deleteOptions) } func (gc *GarbageCollector) getObject(item objectReference) (*unstructured.Unstructured, error) { - fqKind := schema.FromAPIVersionAndKind(item.APIVersion, item.Kind) - client, err := gc.clientPool.ClientForGroupVersionKind(fqKind) + resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind) if err != nil { return nil, err } - resource, err := gc.apiResource(item.APIVersion, item.Kind) - if err != nil { - return nil, err - } - return client.Resource(resource, resourceDefaultNamespace(resource, item.Namespace)).Get(item.Name, metav1.GetOptions{}) + return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Get(item.Name, metav1.GetOptions{}) } func (gc *GarbageCollector) updateObject(item objectReference, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { - fqKind := schema.FromAPIVersionAndKind(item.APIVersion, item.Kind) - client, err := gc.clientPool.ClientForGroupVersionKind(fqKind) + resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind) if err != nil { return nil, err } - resource, err := gc.apiResource(item.APIVersion, item.Kind) - if err != nil { - return nil, err - } - return client.Resource(resource, resourceDefaultNamespace(resource, item.Namespace)).Update(obj) + return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Update(obj) } func (gc *GarbageCollector) patchObject(item objectReference, patch []byte) (*unstructured.Unstructured, error) { - fqKind := schema.FromAPIVersionAndKind(item.APIVersion, item.Kind) - client, err := gc.clientPool.ClientForGroupVersionKind(fqKind) + resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind) if err != nil { return nil, err } - resource, err := gc.apiResource(item.APIVersion, item.Kind) - if err != nil { - return nil, err - } - return client.Resource(resource, resourceDefaultNamespace(resource, item.Namespace)).Patch(item.Name, types.StrategicMergePatchType, patch) + return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Patch(item.Name, types.StrategicMergePatchType, patch) } // TODO: Using Patch when strategicmerge supports deleting an entry from a diff --git a/pkg/controller/podautoscaler/horizontal_test.go b/pkg/controller/podautoscaler/horizontal_test.go index 9beca7a446f..86f9d714424 100644 --- a/pkg/controller/podautoscaler/horizontal_test.go +++ b/pkg/controller/podautoscaler/horizontal_test.go @@ -512,7 +512,7 @@ func (tc *testCase) prepareTestClient(t *testing.T) (*fake.Clientset, *metricsfa } name := getForAction.GetName() - mapper := testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme) + mapper := testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme) metrics := &cmapi.MetricValueList{} var matchedTarget *autoscalingv2.MetricSpec for i, target := range tc.metricsTarget { @@ -650,7 +650,7 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform eventClient.Core(), testScaleClient, testClient.Autoscaling(), - testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme), + testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme), replicaCalc, informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), controller.NoResyncPeriodFunc(), diff --git a/pkg/controller/podautoscaler/legacy_horizontal_test.go b/pkg/controller/podautoscaler/legacy_horizontal_test.go index 3229b2352c9..59d53447073 100644 --- a/pkg/controller/podautoscaler/legacy_horizontal_test.go +++ b/pkg/controller/podautoscaler/legacy_horizontal_test.go @@ -498,7 +498,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) { eventClient.Core(), testScaleClient, testClient.Autoscaling(), - testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme), + testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme), replicaCalc, informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), controller.NoResyncPeriodFunc(), diff --git a/pkg/controller/podautoscaler/metrics/rest_metrics_client_test.go b/pkg/controller/podautoscaler/metrics/rest_metrics_client_test.go index 81ef554e706..2558e2320ec 100644 --- a/pkg/controller/podautoscaler/metrics/rest_metrics_client_test.go +++ b/pkg/controller/podautoscaler/metrics/rest_metrics_client_test.go @@ -154,7 +154,7 @@ func (tc *restClientTestCase) prepareTestClient(t *testing.T) (*metricsfake.Clie return true, &metrics, nil } else { name := getForAction.GetName() - mapper := testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme) + mapper := testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme) assert.NotNil(t, tc.singleObject, "should have only requested a single-object metric when we asked for metrics for a single object") gk := schema.FromAPIVersionAndKind(tc.singleObject.APIVersion, tc.singleObject.Kind).GroupKind() mapping, err := mapper.RESTMapping(gk) diff --git a/pkg/controller/podautoscaler/replica_calculator_test.go b/pkg/controller/podautoscaler/replica_calculator_test.go index 3739c085b41..21af497a339 100644 --- a/pkg/controller/podautoscaler/replica_calculator_test.go +++ b/pkg/controller/podautoscaler/replica_calculator_test.go @@ -222,7 +222,7 @@ func (tc *replicaCalcTestCase) prepareTestClient(t *testing.T) (*fake.Clientset, return true, &metrics, nil } name := getForAction.GetName() - mapper := testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme) + mapper := testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme) metrics := &cmapi.MetricValueList{} assert.NotNil(t, tc.metric.singleObject, "should have only requested a single-object metric when calling GetObjectMetricReplicas") gk := schema.FromAPIVersionAndKind(tc.metric.singleObject.APIVersion, tc.metric.singleObject.Kind).GroupKind() diff --git a/pkg/kubeapiserver/options/storage_versions.go b/pkg/kubeapiserver/options/storage_versions.go index d9405d33453..714cc70afa1 100644 --- a/pkg/kubeapiserver/options/storage_versions.go +++ b/pkg/kubeapiserver/options/storage_versions.go @@ -22,6 +22,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kubernetes/pkg/api/legacyscheme" + "sort" + "github.com/spf13/pflag" ) @@ -40,8 +42,8 @@ type StorageSerializationOptions struct { func NewStorageSerializationOptions() *StorageSerializationOptions { return &StorageSerializationOptions{ - DefaultStorageVersions: legacyscheme.Registry.AllPreferredGroupVersions(), - StorageVersions: legacyscheme.Registry.AllPreferredGroupVersions(), + DefaultStorageVersions: ToPreferredVersionString(legacyscheme.Scheme.PreferredVersionAllGroups()), + StorageVersions: ToPreferredVersionString(legacyscheme.Scheme.PreferredVersionAllGroups()), } } @@ -104,3 +106,16 @@ func (s *StorageSerializationOptions) AddFlags(fs *pflag.FlagSet) { "It defaults to a list of preferred versions of all known groups.") } + +// ToPreferredVersionString returns the preferred versions of all registered +// groups in the form of "group1/version1,group2/version2,...". This is compatible +// with the flag format +func ToPreferredVersionString(versions []schema.GroupVersion) string { + var defaults []string + for _, version := range versions { + defaults = append(defaults, version.String()) + } + // sorting provides stable output for help. + sort.Strings(defaults) + return strings.Join(defaults, ",") +} diff --git a/pkg/kubectl/BUILD b/pkg/kubectl/BUILD index e20cb894a85..3ce1e6d876d 100644 --- a/pkg/kubectl/BUILD +++ b/pkg/kubectl/BUILD @@ -98,7 +98,6 @@ go_library( "generate.go", "history.go", "interfaces.go", - "kubectl.go", "namespace.go", "pdb.go", "priorityclass.go", @@ -136,7 +135,7 @@ go_library( "//pkg/credentialprovider:go_default_library", "//pkg/kubectl/apps:go_default_library", "//pkg/kubectl/cmd/scalejob:go_default_library", - "//pkg/kubectl/resource:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/util:go_default_library", "//pkg/kubectl/util/hash:go_default_library", "//pkg/kubectl/util/slice:go_default_library", @@ -201,14 +200,12 @@ filegroup( ":package-srcs", "//pkg/kubectl/apply:all-srcs", "//pkg/kubectl/apps:all-srcs", - "//pkg/kubectl/categories:all-srcs", "//pkg/kubectl/cmd:all-srcs", "//pkg/kubectl/explain:all-srcs", "//pkg/kubectl/genericclioptions:all-srcs", "//pkg/kubectl/metricsutil:all-srcs", "//pkg/kubectl/plugins:all-srcs", "//pkg/kubectl/proxy:all-srcs", - "//pkg/kubectl/resource:all-srcs", "//pkg/kubectl/scheme:all-srcs", "//pkg/kubectl/util:all-srcs", "//pkg/kubectl/validation:all-srcs", diff --git a/pkg/kubectl/bash_comp_utils.go b/pkg/kubectl/bash_comp_utils.go index 346200276fd..94df450a4a9 100644 --- a/pkg/kubectl/bash_comp_utils.go +++ b/pkg/kubectl/bash_comp_utils.go @@ -23,7 +23,7 @@ import ( "github.com/spf13/cobra" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" ) func AddJsonFilenameFlag(cmd *cobra.Command, value *[]string, usage string) { diff --git a/pkg/kubectl/cmd/BUILD b/pkg/kubectl/cmd/BUILD index 81dc8dcbbfc..a865578827b 100644 --- a/pkg/kubectl/cmd/BUILD +++ b/pkg/kubectl/cmd/BUILD @@ -79,10 +79,10 @@ go_library( "//pkg/kubectl/cmd/util/openapi:go_default_library", "//pkg/kubectl/explain:go_default_library", "//pkg/kubectl/genericclioptions:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/metricsutil:go_default_library", "//pkg/kubectl/plugins:go_default_library", "//pkg/kubectl/proxy:go_default_library", - "//pkg/kubectl/resource:go_default_library", "//pkg/kubectl/scheme:go_default_library", "//pkg/kubectl/util:go_default_library", "//pkg/kubectl/util/i18n:go_default_library", @@ -198,8 +198,8 @@ go_test( "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/cmd/util/openapi:go_default_library", "//pkg/kubectl/genericclioptions:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/plugins:go_default_library", - "//pkg/kubectl/resource:go_default_library", "//pkg/kubectl/scheme:go_default_library", "//pkg/kubectl/util/i18n:go_default_library", "//pkg/kubectl/util/term:go_default_library", diff --git a/pkg/kubectl/cmd/alpha.go b/pkg/kubectl/cmd/alpha.go index afbe7347dc0..bd7f34e66ac 100644 --- a/pkg/kubectl/cmd/alpha.go +++ b/pkg/kubectl/cmd/alpha.go @@ -17,17 +17,16 @@ limitations under the License. package cmd import ( - "io" - "github.com/spf13/cobra" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) // NewCmdAlpha creates a command that acts as an alternate root command for features in alpha -func NewCmdAlpha(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cobra.Command { +func NewCmdAlpha(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { cmd := &cobra.Command{ Use: "alpha", Short: i18n.T("Commands for features in alpha"), @@ -37,7 +36,7 @@ func NewCmdAlpha(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cobra.Com // Alpha commands should be added here. As features graduate from alpha they should move // from here to the CommandGroups defined by NewKubeletCommand() in cmd.go. //cmd.AddCommand(NewCmdDebug(f, in, out, err)) - cmd.AddCommand(NewCmdDiff(f, out, err)) + cmd.AddCommand(NewCmdDiff(f, streams)) // NewKubeletCommand() will hide the alpha command if it has no subcommands. Overriding // the help function ensures a reasonable message if someone types the hidden command anyway. diff --git a/pkg/kubectl/cmd/annotate.go b/pkg/kubectl/cmd/annotate.go index e9d1b88e5b6..cdeb95ef4cf 100644 --- a/pkg/kubectl/cmd/annotate.go +++ b/pkg/kubectl/cmd/annotate.go @@ -30,14 +30,15 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" - "k8s.io/kubernetes/pkg/printers" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" + "k8s.io/kubernetes/pkg/printers" ) // AnnotateOptions have the data required to perform the annotate operation @@ -110,7 +111,7 @@ var ( func NewAnnotateOptions(ioStreams genericclioptions.IOStreams) *AnnotateOptions { return &AnnotateOptions{ - PrintFlags: printers.NewPrintFlags("annotated"), + PrintFlags: printers.NewPrintFlags("annotated", legacyscheme.Scheme), RecordFlags: genericclioptions.NewRecordFlags(), Recorder: genericclioptions.NoopRecorder{}, diff --git a/pkg/kubectl/cmd/annotate_test.go b/pkg/kubectl/cmd/annotate_test.go index b7af6786450..f54fcf0ff11 100644 --- a/pkg/kubectl/cmd/annotate_test.go +++ b/pkg/kubectl/cmd/annotate_test.go @@ -456,7 +456,7 @@ func TestAnnotateObject(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ GroupVersion: schema.GroupVersion{Group: "testgroup", Version: "v1"}, @@ -510,7 +510,7 @@ func TestAnnotateObjectFromFile(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ GroupVersion: schema.GroupVersion{Group: "testgroup", Version: "v1"}, @@ -597,7 +597,7 @@ func TestAnnotateMultipleObjects(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ GroupVersion: schema.GroupVersion{Group: "testgroup", Version: "v1"}, NegotiatedSerializer: unstructuredSerializer, diff --git a/pkg/kubectl/cmd/apiresources.go b/pkg/kubectl/cmd/apiresources.go index b2ad4f0df9d..c427c2e438d 100644 --- a/pkg/kubectl/cmd/apiresources.go +++ b/pkg/kubectl/cmd/apiresources.go @@ -90,8 +90,10 @@ func NewCmdApiResources(f cmdutil.Factory, ioStreams genericclioptions.IOStreams cmdutil.CheckErr(o.RunApiResources(cmd, f)) }, } - cmdutil.AddOutputFlags(cmd) - cmdutil.AddNoHeadersFlags(cmd) + + cmd.Flags().Bool("no-headers", false, "When using the default or custom-column output format, don't print headers (default print headers).") + cmd.Flags().StringP("output", "o", "", "Output format. One of: wide|name.") + cmd.Flags().StringVar(&o.APIGroup, "api-group", "", "Limit to resources in the specified API group.") cmd.Flags().BoolVar(&o.Namespaced, "namespaced", true, "Namespaced indicates if a resource is namespaced or not.") cmd.Flags().StringSliceVar(&o.Verbs, "verbs", o.Verbs, "Limit to resources that support the specified verbs.") diff --git a/pkg/kubectl/cmd/apply.go b/pkg/kubectl/cmd/apply.go index 752b919d86b..fcdaaf09ab1 100644 --- a/pkg/kubectl/cmd/apply.go +++ b/pkg/kubectl/cmd/apply.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -41,6 +42,7 @@ import ( "k8s.io/client-go/dynamic" scaleclient "k8s.io/client-go/scale" oapi "k8s.io/kube-openapi/pkg/util/proto" + "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/kubectl" @@ -48,9 +50,10 @@ import ( cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" "k8s.io/kubernetes/pkg/kubectl/util/i18n" + "k8s.io/kubernetes/pkg/kubectl/validation" "k8s.io/kubernetes/pkg/printers" ) @@ -64,15 +67,27 @@ type ApplyOptions struct { DeleteFlags *DeleteFlags DeleteOptions *DeleteOptions - Selector string - DryRun bool - Prune bool - PruneResources []pruneResource - cmdBaseName string - All bool - Overwrite bool - OpenApiPatch bool - PruneWhitelist []string + Selector string + DryRun bool + Prune bool + PruneResources []pruneResource + cmdBaseName string + All bool + Overwrite bool + OpenApiPatch bool + PruneWhitelist []string + ShouldIncludeUninitialized bool + + Validator validation.Schema + Builder *resource.Builder + Mapper meta.RESTMapper + Scaler scaleclient.ScalesGetter + DynamicClient dynamic.DynamicInterface + ClientSetFunc func() (internalclientset.Interface, error) + OpenAPISchema openapi.Resources + + Namespace string + EnforceNamespace bool genericclioptions.IOStreams } @@ -117,7 +132,7 @@ func NewApplyOptions(ioStreams genericclioptions.IOStreams) *ApplyOptions { return &ApplyOptions{ RecordFlags: genericclioptions.NewRecordFlags(), DeleteFlags: NewDeleteFlags("that contains the configuration to apply"), - PrintFlags: printers.NewPrintFlags("created"), + PrintFlags: printers.NewPrintFlags("created", legacyscheme.Scheme), Overwrite: true, OpenApiPatch: true, @@ -145,7 +160,7 @@ func NewCmdApply(baseName string, f cmdutil.Factory, ioStreams genericclioptions cmdutil.CheckErr(o.Complete(f, cmd)) cmdutil.CheckErr(validateArgs(cmd, args)) cmdutil.CheckErr(validatePruneAll(o.Prune, o.All, o.Selector)) - cmdutil.CheckErr(o.Run(f, cmd)) + cmdutil.CheckErr(o.Run()) }, } @@ -197,7 +212,33 @@ func (o *ApplyOptions) Complete(f cmdutil.Factory, cmd *cobra.Command) error { return err } - o.DeleteOptions = o.DeleteFlags.ToOptions(o.Out, o.ErrOut) + o.DeleteOptions = o.DeleteFlags.ToOptions(o.IOStreams) + o.ShouldIncludeUninitialized = cmdutil.ShouldIncludeUninitialized(cmd, o.Prune) + + o.OpenAPISchema, _ = f.OpenAPISchema() + o.ClientSetFunc = f.ClientSet + o.Validator, err = f.Validator(cmdutil.GetFlagBool(cmd, "validate")) + o.Builder = f.NewBuilder() + o.Mapper, err = f.RESTMapper() + if err != nil { + return err + } + + o.Scaler, err = f.ScaleClient() + if err != nil { + return err + } + + o.DynamicClient, err = f.DynamicClient() + if err != nil { + return err + } + + o.Namespace, o.EnforceNamespace, err = f.DefaultNamespace() + if err != nil { + return err + } + return nil } @@ -249,61 +290,39 @@ func parsePruneResources(mapper meta.RESTMapper, gvks []string) ([]pruneResource return pruneResources, nil } -// TODO(juanvallejo): break dependency on factory and cmd -func (o *ApplyOptions) Run(f cmdutil.Factory, cmd *cobra.Command) error { - schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate")) - if err != nil { - return err - } - +func (o *ApplyOptions) Run() error { var openapiSchema openapi.Resources if o.OpenApiPatch { - openapiSchema, err = f.OpenAPISchema() - if err != nil { - openapiSchema = nil - } - } - - cmdNamespace, enforceNamespace, err := f.DefaultNamespace() - if err != nil { - return err + openapiSchema = o.OpenAPISchema } // include the uninitialized objects by default if --prune is true // unless explicitly set --include-uninitialized=false - includeUninitialized := cmdutil.ShouldIncludeUninitialized(cmd, o.Prune) - r := f.NewBuilder(). + r := o.Builder. Unstructured(). - Schema(schema). + Schema(o.Validator). ContinueOnError(). - NamespaceParam(cmdNamespace).DefaultNamespace(). - FilenameParam(enforceNamespace, &o.DeleteOptions.FilenameOptions). + NamespaceParam(o.Namespace).DefaultNamespace(). + FilenameParam(o.EnforceNamespace, &o.DeleteOptions.FilenameOptions). LabelSelectorParam(o.Selector). - IncludeUninitialized(includeUninitialized). + IncludeUninitialized(o.ShouldIncludeUninitialized). Flatten(). Do() if err := r.Err(); err != nil { return err } - mapper, err := f.RESTMapper() - if err != nil { - return err - } - + var err error if o.Prune { - o.PruneResources, err = parsePruneResources(mapper, o.PruneWhitelist) + o.PruneResources, err = parsePruneResources(o.Mapper, o.PruneWhitelist) if err != nil { return err } } - output := cmdutil.GetFlagString(cmd, "output") + output := *o.PrintFlags.OutputFormat shortOutput := output == "name" - encoder := scheme.DefaultJSONEncoder() - deserializer := scheme.Codecs.UniversalDeserializer() - visitedUids := sets.NewString() visitedNamespaces := sets.NewString() @@ -326,7 +345,7 @@ func (o *ApplyOptions) Run(f cmdutil.Factory, cmd *cobra.Command) error { // Get the modified configuration of the object. Embed the result // as an annotation in the modified configuration, so that it will appear // in the patch sent to the server. - modified, err := kubectl.GetModifiedConfiguration(info.Object, true, encoder) + modified, err := kubectl.GetModifiedConfiguration(info.Object, true, unstructured.UnstructuredJSONScheme) if err != nil { return cmdutil.AddSourceToErr(fmt.Sprintf("retrieving modified configuration from:\n%s\nfor:", info.String()), info.Source, err) } @@ -340,7 +359,7 @@ func (o *ApplyOptions) Run(f cmdutil.Factory, cmd *cobra.Command) error { } // Create the resource if it doesn't exist // First, update the annotation used by kubectl apply - if err := kubectl.CreateApplyAnnotation(info.Object, encoder); err != nil { + if err := kubectl.CreateApplyAnnotation(info.Object, unstructured.UnstructuredJSONScheme); err != nil { return cmdutil.AddSourceToErr("creating", info.Source, err) } @@ -382,22 +401,13 @@ func (o *ApplyOptions) Run(f cmdutil.Factory, cmd *cobra.Command) error { if _, ok := annotationMap[api.LastAppliedConfigAnnotation]; !ok { fmt.Fprintf(o.ErrOut, warningNoLastAppliedConfigAnnotation, o.cmdBaseName) } - scaler, err := f.ScaleClient() - if err != nil { - return err - } + helper := resource.NewHelper(info.Client, info.Mapping) - dynamicClient, err := f.DynamicClient() - if err != nil { - return err - } patcher := &patcher{ - encoder: encoder, - decoder: deserializer, mapping: info.Mapping, helper: helper, - dynamicClient: dynamicClient, - clientsetFunc: f.ClientSet, + dynamicClient: o.DynamicClient, + clientsetFunc: o.ClientSetFunc, overwrite: o.Overwrite, backOff: clockwork.NewRealClock(), force: o.DeleteOptions.ForceDeletion, @@ -405,7 +415,7 @@ func (o *ApplyOptions) Run(f cmdutil.Factory, cmd *cobra.Command) error { timeout: o.DeleteOptions.Timeout, gracePeriod: o.DeleteOptions.GracePeriod, openapiSchema: openapiSchema, - scaleClient: scaler, + scaleClient: o.Scaler, } patchBytes, patchedObject, err := patcher.patch(info.Object, modified, info.Source, info.Namespace, info.Name, o.ErrOut) @@ -479,15 +489,10 @@ func (o *ApplyOptions) Run(f cmdutil.Factory, cmd *cobra.Command) error { return nil } - dynamicClient, err := f.DynamicClient() - if err != nil { - return err - } - p := pruner{ - mapper: mapper, - dynamicClient: dynamicClient, - clientsetFunc: f.ClientSet, + mapper: o.Mapper, + dynamicClient: o.DynamicClient, + clientsetFunc: o.ClientSetFunc, labelSelector: o.Selector, visitedUids: visitedUids, @@ -495,26 +500,27 @@ func (o *ApplyOptions) Run(f cmdutil.Factory, cmd *cobra.Command) error { cascade: o.DeleteOptions.Cascade, dryRun: o.DryRun, gracePeriod: o.DeleteOptions.GracePeriod, + scaler: o.Scaler, toPrinter: o.ToPrinter, out: o.Out, } - namespacedRESTMappings, nonNamespacedRESTMappings, err := getRESTMappings(mapper, &(o.PruneResources)) + namespacedRESTMappings, nonNamespacedRESTMappings, err := getRESTMappings(o.Mapper, &(o.PruneResources)) if err != nil { return fmt.Errorf("error retrieving RESTMappings to prune: %v", err) } for n := range visitedNamespaces { for _, m := range namespacedRESTMappings { - if err := p.prune(f, n, m, includeUninitialized); err != nil { + if err := p.prune(n, m, o.ShouldIncludeUninitialized); err != nil { return fmt.Errorf("error pruning namespaced object %v: %v", m.GroupVersionKind, err) } } } for _, m := range nonNamespacedRESTMappings { - if err := p.prune(f, metav1.NamespaceNone, m, includeUninitialized); err != nil { + if err := p.prune(metav1.NamespaceNone, m, o.ShouldIncludeUninitialized); err != nil { return fmt.Errorf("error pruning nonNamespaced object %v: %v", m.GroupVersionKind, err) } } @@ -585,12 +591,14 @@ type pruner struct { dryRun bool gracePeriod int + scaler scaleclient.ScalesGetter + toPrinter func(string) (printers.ResourcePrinterFunc, error) out io.Writer } -func (p *pruner) prune(f cmdutil.Factory, namespace string, mapping *meta.RESTMapping, includeUninitialized bool) error { +func (p *pruner) prune(namespace string, mapping *meta.RESTMapping, includeUninitialized bool) error { objList, err := p.dynamicClient.Resource(mapping.Resource). Namespace(namespace). List(metav1.ListOptions{ @@ -606,10 +614,6 @@ func (p *pruner) prune(f cmdutil.Factory, namespace string, mapping *meta.RESTMa if err != nil { return err } - scaler, err := f.ScaleClient() - if err != nil { - return err - } for _, obj := range objs { metadata, err := meta.Accessor(obj) @@ -627,7 +631,7 @@ func (p *pruner) prune(f cmdutil.Factory, namespace string, mapping *meta.RESTMa } name := metadata.GetName() if !p.dryRun { - if err := p.delete(namespace, name, mapping, scaler); err != nil { + if err := p.delete(namespace, name, mapping, p.scaler); err != nil { return err } } @@ -675,9 +679,6 @@ func (p *patcher) delete(namespace, name string) error { } type patcher struct { - encoder runtime.Encoder - decoder runtime.Decoder - mapping *meta.RESTMapping helper *resource.Helper dynamicClient dynamic.DynamicInterface @@ -697,7 +698,7 @@ type patcher struct { func (p *patcher) patchSimple(obj runtime.Object, modified []byte, source, namespace, name string, errOut io.Writer) ([]byte, runtime.Object, error) { // Serialize the current configuration of the object from the server. - current, err := runtime.Encode(p.encoder, obj) + current, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj) if err != nil { return nil, nil, cmdutil.AddSourceToErr(fmt.Sprintf("serializing current configuration from:\n%v\nfor:", obj), source, err) } @@ -803,7 +804,7 @@ func (p *patcher) deleteAndCreate(original runtime.Object, modified []byte, name if err != nil { return modified, nil, err } - versionedObject, _, err := p.decoder.Decode(modified, nil, nil) + versionedObject, _, err := unstructured.UnstructuredJSONScheme.Decode(modified, nil, nil) if err != nil { return modified, nil, err } diff --git a/pkg/kubectl/cmd/apply_set_last_applied.go b/pkg/kubectl/cmd/apply_set_last_applied.go index f2291807f21..f7b6e8773a1 100644 --- a/pkg/kubectl/cmd/apply_set_last_applied.go +++ b/pkg/kubectl/cmd/apply_set_last_applied.go @@ -26,12 +26,13 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/cmd/util/editor" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) @@ -82,7 +83,7 @@ var ( func NewSetLastAppliedOptions(ioStreams genericclioptions.IOStreams) *SetLastAppliedOptions { return &SetLastAppliedOptions{ - PrintFlags: printers.NewPrintFlags("configured"), + PrintFlags: printers.NewPrintFlags("configured", legacyscheme.Scheme), IOStreams: ioStreams, } } @@ -102,8 +103,9 @@ func NewCmdApplySetLastApplied(f cmdutil.Factory, ioStreams genericclioptions.IO }, } + o.PrintFlags.AddFlags(cmd) + cmdutil.AddDryRunFlag(cmd) - cmdutil.AddPrinterFlags(cmd) cmd.Flags().BoolVar(&o.CreateAnnotation, "create-annotation", o.CreateAnnotation, "Will create 'last-applied-configuration' annotations if current objects doesn't have one") kubectl.AddJsonFilenameFlag(cmd, &o.FilenameOptions.Filenames, "Filename, directory, or URL to files that contains the last-applied-configuration annotations") diff --git a/pkg/kubectl/cmd/apply_test.go b/pkg/kubectl/cmd/apply_test.go index c7ebef5627d..283266f5381 100644 --- a/pkg/kubectl/cmd/apply_test.go +++ b/pkg/kubectl/cmd/apply_test.go @@ -359,7 +359,7 @@ func TestRunApplyViewLastApplied(t *testing.T) { expectedErr: "error: Unexpected -o output mode: wide, the flag 'output' must be one of yaml|json\nSee 'view-last-applied -h' for help and examples.", expectedOut: "", selector: "", - args: []string{"rc", "test-rc"}, + args: []string{"replicationcontroller", "test-rc"}, respBytes: rcBytesWithConfig, }, { @@ -369,7 +369,7 @@ func TestRunApplyViewLastApplied(t *testing.T) { expectedErr: "", expectedOut: "test: 1234\n", selector: "name=test-rc", - args: []string{"rc"}, + args: []string{"replicationcontroller"}, respBytes: rcBytesWithConfig, }, { @@ -379,7 +379,7 @@ func TestRunApplyViewLastApplied(t *testing.T) { expectedErr: "error: no last-applied-configuration annotation found on resource: test-rc", expectedOut: "", selector: "", - args: []string{"rc", "test-rc"}, + args: []string{"replicationcontroller", "test-rc"}, respBytes: rcBytes, }, { @@ -389,7 +389,7 @@ func TestRunApplyViewLastApplied(t *testing.T) { expectedErr: "Error from server (NotFound): the server could not find the requested resource (get replicationcontrollers no-match)", expectedOut: "", selector: "", - args: []string{"rc", "no-match"}, + args: []string{"replicationcontroller", "no-match"}, respBytes: nil, }, } @@ -398,7 +398,7 @@ func TestRunApplyViewLastApplied(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ GroupVersion: schema.GroupVersion{Version: "v1"}, @@ -1021,8 +1021,6 @@ func TestUnstructuredIdempotentApply(t *testing.T) { } path := "/namespaces/test/widgets/widget" - verifiedPatch := false - for _, fn := range testingOpenAPISchemaFns { t.Run("test repeated apply operations on an unstructured object", func(t *testing.T) { tf := cmdtesting.NewTestFactory() @@ -1039,41 +1037,15 @@ func TestUnstructuredIdempotentApply(t *testing.T) { Header: defaultHeader(), Body: body}, nil case p == path && m == "PATCH": - // In idempotent updates, kubectl sends a logically empty - // request body with the PATCH request. - // Should look like this: - // Request Body: {"metadata":{"annotations":{}}} + // In idempotent updates, kubectl will resolve to an empty patch and not send anything to the server + // Thus, if we reach this branch, kubectl is unnecessarily sending a patch. patch, err := ioutil.ReadAll(req.Body) if err != nil { t.Fatal(err) } - - contentType := req.Header.Get("Content-Type") - if contentType != "application/merge-patch+json" { - t.Fatalf("Unexpected Content-Type: %s", contentType) - } - - patchMap := map[string]interface{}{} - if err := json.Unmarshal(patch, &patchMap); err != nil { - t.Fatal(err) - } - if len(patchMap) != 1 { - t.Fatalf("Unexpected Patch. Has more than 1 entry. path: %s", patch) - } - - annotationsMap := walkMapPath(t, patchMap, []string{"metadata", "annotations"}) - if len(annotationsMap) != 0 { - t.Fatalf("Unexpected Patch. Found unexpected annotation: %s", patch) - } - - verifiedPatch = true - - body := ioutil.NopCloser(bytes.NewReader(serversideData)) - return &http.Response{ - StatusCode: 200, - Header: defaultHeader(), - Body: body}, nil + t.Fatalf("Unexpected Patch: %s", patch) + return nil, nil default: t.Fatalf("unexpected request: %#v\n%#v", req.URL, req) return nil, nil @@ -1096,9 +1068,6 @@ func TestUnstructuredIdempotentApply(t *testing.T) { if errBuf.String() != "" { t.Fatalf("unexpected error output: %s", errBuf.String()) } - if !verifiedPatch { - t.Fatal("No server-side patch call detected") - } }) } } @@ -1121,7 +1090,7 @@ func TestRunApplySetLastApplied(t *testing.T) { name: "set with exist object", filePath: filenameRC, expectedErr: "", - expectedOut: "replicationcontroller/test-rc configured\n", + expectedOut: "replicationcontroller/test-rc\n", output: "name", }, { @@ -1142,14 +1111,14 @@ func TestRunApplySetLastApplied(t *testing.T) { name: "set with exist object output json", filePath: filenameRCJSON, expectedErr: "", - expectedOut: "replicationcontroller/test-rc configured\n", + expectedOut: "replicationcontroller/test-rc\n", output: "name", }, { name: "set test for a directory of files", filePath: dirName, expectedErr: "", - expectedOut: "replicationcontroller/test-rc configured\nreplicationcontroller/test-rc configured\n", + expectedOut: "replicationcontroller/test-rc\nreplicationcontroller/test-rc\n", output: "name", }, } @@ -1158,7 +1127,7 @@ func TestRunApplySetLastApplied(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ GroupVersion: schema.GroupVersion{Version: "v1"}, diff --git a/pkg/kubectl/cmd/apply_view_last_applied.go b/pkg/kubectl/cmd/apply_view_last_applied.go index 1b1cb9c244c..ce5c7722e19 100644 --- a/pkg/kubectl/cmd/apply_view_last_applied.go +++ b/pkg/kubectl/cmd/apply_view_last_applied.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) diff --git a/pkg/kubectl/cmd/attach.go b/pkg/kubectl/cmd/attach.go index 569cfec9245..1171c307203 100644 --- a/pkg/kubectl/cmd/attach.go +++ b/pkg/kubectl/cmd/attach.go @@ -35,6 +35,7 @@ import ( coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -60,12 +61,10 @@ const ( defaultPodLogsTimeout = 20 * time.Second ) -func NewCmdAttach(f cmdutil.Factory, cmdIn io.Reader, cmdOut, cmdErr io.Writer) *cobra.Command { - options := &AttachOptions{ +func NewCmdAttach(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := &AttachOptions{ StreamOptions: StreamOptions{ - In: cmdIn, - Out: cmdOut, - Err: cmdErr, + IOStreams: streams, }, Attach: &DefaultRemoteAttach{}, @@ -77,15 +76,15 @@ func NewCmdAttach(f cmdutil.Factory, cmdIn io.Reader, cmdOut, cmdErr io.Writer) Long: "Attach to a process that is already running inside an existing container.", Example: attachExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(f, cmd, args)) - cmdutil.CheckErr(options.Validate()) - cmdutil.CheckErr(options.Run()) + cmdutil.CheckErr(o.Complete(f, cmd, args)) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) }, } cmdutil.AddPodRunningTimeoutFlag(cmd, defaultPodAttachTimeout) - cmd.Flags().StringVarP(&options.ContainerName, "container", "c", options.ContainerName, "Container name. If omitted, the first container in the pod will be chosen") - cmd.Flags().BoolVarP(&options.Stdin, "stdin", "i", options.Stdin, "Pass stdin to the container") - cmd.Flags().BoolVarP(&options.TTY, "tty", "t", options.TTY, "Stdin is a TTY") + cmd.Flags().StringVarP(&o.ContainerName, "container", "c", o.ContainerName, "Container name. If omitted, the first container in the pod will be chosen") + cmd.Flags().BoolVarP(&o.Stdin, "stdin", "i", o.Stdin, "Pass stdin to the container") + cmd.Flags().BoolVarP(&o.TTY, "tty", "t", o.TTY, "Stdin is a TTY") return cmd } @@ -203,7 +202,7 @@ func (p *AttachOptions) Validate() error { if len(p.PodName) == 0 { allErrs = append(allErrs, errors.New("pod name must be specified")) } - if p.Out == nil || p.Err == nil { + if p.Out == nil || p.ErrOut == nil { allErrs = append(allErrs, errors.New("both output and error output must be provided")) } if p.Attach == nil || p.PodClient == nil || p.Config == nil { @@ -236,8 +235,8 @@ func (p *AttachOptions) Run() error { } if p.TTY && !containerToAttach.TTY { p.TTY = false - if p.Err != nil { - fmt.Fprintf(p.Err, "Unable to use a TTY - container %s did not allocate one\n", containerToAttach.Name) + if p.ErrOut != nil { + fmt.Fprintf(p.ErrOut, "Unable to use a TTY - container %s did not allocate one\n", containerToAttach.Name) } } else if !p.TTY && containerToAttach.TTY { // the container was launched with a TTY, so we have to force a TTY here, otherwise you'll get @@ -249,7 +248,7 @@ func (p *AttachOptions) Run() error { t := p.setupTTY() // save p.Err so we can print the command prompt message below - stderr := p.Err + stderr := p.ErrOut var sizeQueue remotecommand.TerminalSizeQueue if t.Raw { @@ -266,7 +265,7 @@ func (p *AttachOptions) Run() error { // unset p.Err if it was previously set because both stdout and stderr go over p.Out when tty is // true - p.Err = nil + p.ErrOut = nil } fn := func() error { @@ -284,11 +283,11 @@ func (p *AttachOptions) Run() error { Container: containerToAttach.Name, Stdin: p.Stdin, Stdout: p.Out != nil, - Stderr: p.Err != nil, + Stderr: p.ErrOut != nil, TTY: t.Raw, }, legacyscheme.ParameterCodec) - return p.Attach.Attach("POST", req.URL(), p.Config, p.In, p.Out, p.Err, t.Raw, sizeQueue) + return p.Attach.Attach("POST", req.URL(), p.Config, p.In, p.Out, p.ErrOut, t.Raw, sizeQueue) } if !p.Quiet && stderr != nil { @@ -322,8 +321,8 @@ func (p *AttachOptions) containerToAttachTo(pod *api.Pod) (*api.Container, error } if len(p.SuggestedCmdUsage) > 0 { - fmt.Fprintf(p.Err, "Defaulting container name to %s.\n", pod.Spec.Containers[0].Name) - fmt.Fprintf(p.Err, "%s\n", p.SuggestedCmdUsage) + fmt.Fprintf(p.ErrOut, "Defaulting container name to %s.\n", pod.Spec.Containers[0].Name) + fmt.Fprintf(p.ErrOut, "%s\n", p.SuggestedCmdUsage) } glog.V(4).Infof("defaulting container name to %s", pod.Spec.Containers[0].Name) diff --git a/pkg/kubectl/cmd/attach_test.go b/pkg/kubectl/cmd/attach_test.go index 603487def6e..55e0aad17ed 100644 --- a/pkg/kubectl/cmd/attach_test.go +++ b/pkg/kubectl/cmd/attach_test.go @@ -17,7 +17,6 @@ limitations under the License. package cmd import ( - "bytes" "fmt" "io" "net/http" @@ -38,6 +37,7 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/scheme" ) @@ -143,7 +143,7 @@ func TestPodAndContainerAttach(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -227,7 +227,7 @@ func TestAttach(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -249,9 +249,6 @@ func TestAttach(t *testing.T) { } tf.Namespace = "test" tf.ClientConfigVal = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs, GroupVersion: &schema.GroupVersion{Version: test.version}}} - bufOut := bytes.NewBuffer([]byte{}) - bufErr := bytes.NewBuffer([]byte{}) - bufIn := bytes.NewBuffer([]byte{}) remoteAttach := &fakeRemoteAttach{} if test.remoteAttachErr { remoteAttach.err = fmt.Errorf("attach error") @@ -259,9 +256,7 @@ func TestAttach(t *testing.T) { params := &AttachOptions{ StreamOptions: StreamOptions{ ContainerName: test.container, - In: bufIn, - Out: bufOut, - Err: bufErr, + IOStreams: genericclioptions.NewTestIOStreamsDiscard(), }, Attach: remoteAttach, GetPodTimeout: 1000, @@ -320,7 +315,7 @@ func TestAttachWarnings(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -342,16 +337,12 @@ func TestAttachWarnings(t *testing.T) { } tf.Namespace = "test" tf.ClientConfigVal = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs, GroupVersion: &schema.GroupVersion{Version: test.version}}} - bufOut := bytes.NewBuffer([]byte{}) - bufErr := bytes.NewBuffer([]byte{}) - bufIn := bytes.NewBuffer([]byte{}) + streams, _, _, bufErr := genericclioptions.NewTestIOStreams() ex := &fakeRemoteAttach{} params := &AttachOptions{ StreamOptions: StreamOptions{ ContainerName: test.container, - In: bufIn, - Out: bufOut, - Err: bufErr, + IOStreams: streams, Stdin: test.stdin, TTY: test.tty, }, diff --git a/pkg/kubectl/cmd/auth/BUILD b/pkg/kubectl/cmd/auth/BUILD index 987683660a3..7d6a163f88e 100644 --- a/pkg/kubectl/cmd/auth/BUILD +++ b/pkg/kubectl/cmd/auth/BUILD @@ -24,7 +24,9 @@ go_library( "//pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion:go_default_library", "//pkg/kubectl/cmd/templates:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", - "//pkg/kubectl/resource:go_default_library", + "//pkg/kubectl/genericclioptions:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", + "//pkg/printers:go_default_library", "//pkg/registry/rbac/reconciliation:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", diff --git a/pkg/kubectl/cmd/auth/auth.go b/pkg/kubectl/cmd/auth/auth.go index eb70d3005df..b48fe56a772 100644 --- a/pkg/kubectl/cmd/auth/auth.go +++ b/pkg/kubectl/cmd/auth/auth.go @@ -17,24 +17,23 @@ limitations under the License. package auth import ( - "io" - "github.com/spf13/cobra" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" ) -func NewCmdAuth(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { +func NewCmdAuth(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { // Parent command to which all subcommands are added. cmds := &cobra.Command{ Use: "auth", Short: "Inspect authorization", Long: `Inspect authorization`, - Run: cmdutil.DefaultSubCommandRun(errOut), + Run: cmdutil.DefaultSubCommandRun(streams.ErrOut), } - cmds.AddCommand(NewCmdCanI(f, out, errOut)) - cmds.AddCommand(NewCmdReconcile(f, out, errOut)) + cmds.AddCommand(NewCmdCanI(f, streams)) + cmds.AddCommand(NewCmdReconcile(f, streams)) return cmds } diff --git a/pkg/kubectl/cmd/auth/cani.go b/pkg/kubectl/cmd/auth/cani.go index d0269f2f8a9..88a93d8d229 100644 --- a/pkg/kubectl/cmd/auth/cani.go +++ b/pkg/kubectl/cmd/auth/cani.go @@ -19,12 +19,12 @@ package auth import ( "errors" "fmt" - "io" "io/ioutil" "os" "strings" "github.com/spf13/cobra" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" @@ -48,8 +48,7 @@ type CanIOptions struct { Subresource string ResourceName string - Out io.Writer - Err io.Writer + genericclioptions.IOStreams } var ( @@ -81,10 +80,9 @@ var ( kubectl auth can-i get /logs/`) ) -func NewCmdCanI(f cmdutil.Factory, out, err io.Writer) *cobra.Command { +func NewCmdCanI(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &CanIOptions{ - Out: out, - Err: err, + IOStreams: streams, } cmd := &cobra.Command{ @@ -232,9 +230,9 @@ func (o *CanIOptions) resourceFor(mapper meta.RESTMapper, resourceArg string) sc gvr, err = mapper.ResourceFor(groupResource.WithVersion("")) if err != nil { if len(groupResource.Group) == 0 { - fmt.Fprintf(o.Err, "Warning: the server doesn't have a resource type '%s'\n", groupResource.Resource) + fmt.Fprintf(o.ErrOut, "Warning: the server doesn't have a resource type '%s'\n", groupResource.Resource) } else { - fmt.Fprintf(o.Err, "Warning: the server doesn't have a resource type '%s' in group '%s'\n", groupResource.Resource, groupResource.Group) + fmt.Fprintf(o.ErrOut, "Warning: the server doesn't have a resource type '%s' in group '%s'\n", groupResource.Resource, groupResource.Group) } return schema.GroupVersionResource{Resource: resourceArg} } diff --git a/pkg/kubectl/cmd/auth/cani_test.go b/pkg/kubectl/cmd/auth/cani_test.go index eb3cf9c5d84..84dc9826868 100644 --- a/pkg/kubectl/cmd/auth/cani_test.go +++ b/pkg/kubectl/cmd/auth/cani_test.go @@ -119,7 +119,7 @@ func TestRunAccessCheck(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { test.o.Out = ioutil.Discard - test.o.Err = ioutil.Discard + test.o.ErrOut = ioutil.Discard tf := cmdtesting.NewTestFactory() defer tf.Cleanup() diff --git a/pkg/kubectl/cmd/auth/reconcile.go b/pkg/kubectl/cmd/auth/reconcile.go index 26dd8069c4d..7e9a476981b 100644 --- a/pkg/kubectl/cmd/auth/reconcile.go +++ b/pkg/kubectl/cmd/auth/reconcile.go @@ -18,7 +18,6 @@ package auth import ( "errors" - "io" "github.com/golang/glog" "github.com/spf13/cobra" @@ -29,21 +28,25 @@ import ( internalrbacclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/rbac/internalversion" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" + "k8s.io/kubernetes/pkg/printers" "k8s.io/kubernetes/pkg/registry/rbac/reconciliation" ) // ReconcileOptions is the start of the data required to perform the operation. As new fields are added, add them here instead of // referencing the cmd.Flags() type ReconcileOptions struct { + PrintFlags *printers.PrintFlags + FilenameOptions *resource.FilenameOptions + Visitor resource.Visitor RBACClient internalrbacclient.RbacInterface NamespaceClient internalcoreclient.NamespaceInterface - Print func(*resource.Info) error + PrintObject printers.ResourcePrinterFunc - Out io.Writer - Err io.Writer + genericclioptions.IOStreams } var ( @@ -57,12 +60,16 @@ var ( kubectl auth reconcile -f my-rbac-rules.yaml`) ) -func NewCmdReconcile(f cmdutil.Factory, out, err io.Writer) *cobra.Command { - fileOptions := &resource.FilenameOptions{} - o := &ReconcileOptions{ - Out: out, - Err: err, +func NewReconcileOptions(ioStreams genericclioptions.IOStreams) *ReconcileOptions { + return &ReconcileOptions{ + FilenameOptions: &resource.FilenameOptions{}, + PrintFlags: printers.NewPrintFlags("reconciled", legacyscheme.Scheme), + IOStreams: ioStreams, } +} + +func NewCmdReconcile(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewReconcileOptions(streams) cmd := &cobra.Command{ Use: "reconcile -f FILENAME", @@ -71,21 +78,21 @@ func NewCmdReconcile(f cmdutil.Factory, out, err io.Writer) *cobra.Command { Long: reconcileLong, Example: reconcileExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(o.Complete(cmd, f, args, fileOptions)) + cmdutil.CheckErr(o.Complete(cmd, f, args)) cmdutil.CheckErr(o.Validate()) cmdutil.CheckErr(o.RunReconcile()) }, } - cmdutil.AddPrinterFlags(cmd) - usage := "identifying the resource to reconcile." - cmdutil.AddFilenameOptionFlags(cmd, fileOptions, usage) + o.PrintFlags.AddFlags(cmd) + + cmdutil.AddFilenameOptionFlags(cmd, o.FilenameOptions, "identifying the resource to reconcile.") cmd.MarkFlagRequired("filename") return cmd } -func (o *ReconcileOptions) Complete(cmd *cobra.Command, f cmdutil.Factory, args []string, options *resource.FilenameOptions) error { +func (o *ReconcileOptions) Complete(cmd *cobra.Command, f cmdutil.Factory, args []string) error { if len(args) > 0 { return errors.New("no arguments are allowed") } @@ -99,7 +106,7 @@ func (o *ReconcileOptions) Complete(cmd *cobra.Command, f cmdutil.Factory, args WithScheme(legacyscheme.Scheme). ContinueOnError(). NamespaceParam(namespace).DefaultNamespace(). - FilenameParam(enforceNamespace, options). + FilenameParam(enforceNamespace, o.FilenameOptions). Flatten(). Do() @@ -115,17 +122,12 @@ func (o *ReconcileOptions) Complete(cmd *cobra.Command, f cmdutil.Factory, args o.RBACClient = client.Rbac() o.NamespaceClient = client.Core().Namespaces() - dryRun := false - output := cmdutil.GetFlagString(cmd, "output") - shortOutput := output == "name" - o.Print = func(info *resource.Info) error { - if len(output) > 0 && !shortOutput { - return cmdutil.PrintObject(cmd, info.Object, o.Out) - } - cmdutil.PrintSuccess(shortOutput, o.Out, info.Object, dryRun, "reconciled") - return nil + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err } + o.PrintObject = printer.PrintObj return nil } @@ -139,13 +141,13 @@ func (o *ReconcileOptions) Validate() error { if o.NamespaceClient == nil { return errors.New("ReconcileOptions.NamespaceClient must be set") } - if o.Print == nil { + if o.PrintObject == nil { return errors.New("ReconcileOptions.Print must be set") } if o.Out == nil { return errors.New("ReconcileOptions.Out must be set") } - if o.Err == nil { + if o.ErrOut == nil { return errors.New("ReconcileOptions.Err must be set") } return nil @@ -157,10 +159,6 @@ func (o *ReconcileOptions) RunReconcile() error { return err } - // shallowInfoCopy this is used to later twiddle the Object for printing - // we really need more straightforward printing options - shallowInfoCopy := *info - switch t := info.Object.(type) { case *rbac.Role: reconcileOptions := reconciliation.ReconcileRoleOptions{ @@ -176,8 +174,7 @@ func (o *ReconcileOptions) RunReconcile() error { if err != nil { return err } - shallowInfoCopy.Object = result.Role.GetObject() - o.Print(&shallowInfoCopy) + o.PrintObject(result.Role.GetObject(), o.Out) case *rbac.ClusterRole: reconcileOptions := reconciliation.ReconcileRoleOptions{ @@ -192,8 +189,7 @@ func (o *ReconcileOptions) RunReconcile() error { if err != nil { return err } - shallowInfoCopy.Object = result.Role.GetObject() - o.Print(&shallowInfoCopy) + o.PrintObject(result.Role.GetObject(), o.Out) case *rbac.RoleBinding: reconcileOptions := reconciliation.ReconcileRoleBindingOptions{ @@ -209,8 +205,7 @@ func (o *ReconcileOptions) RunReconcile() error { if err != nil { return err } - shallowInfoCopy.Object = result.RoleBinding.GetObject() - o.Print(&shallowInfoCopy) + o.PrintObject(result.RoleBinding.GetObject(), o.Out) case *rbac.ClusterRoleBinding: reconcileOptions := reconciliation.ReconcileRoleBindingOptions{ @@ -225,8 +220,7 @@ func (o *ReconcileOptions) RunReconcile() error { if err != nil { return err } - shallowInfoCopy.Object = result.RoleBinding.GetObject() - o.Print(&shallowInfoCopy) + o.PrintObject(result.RoleBinding.GetObject(), o.Out) default: glog.V(1).Infof("skipping %#v", info.Object.GetObjectKind()) diff --git a/pkg/kubectl/cmd/autoscale.go b/pkg/kubectl/cmd/autoscale.go index a4a4aba2c09..81bef6cb227 100644 --- a/pkg/kubectl/cmd/autoscale.go +++ b/pkg/kubectl/cmd/autoscale.go @@ -31,7 +31,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) @@ -82,7 +82,7 @@ type AutoscaleOptions struct { func NewAutoscaleOptions(ioStreams genericclioptions.IOStreams) *AutoscaleOptions { return &AutoscaleOptions{ - PrintFlags: printers.NewPrintFlags("autoscaled"), + PrintFlags: printers.NewPrintFlags("autoscaled", legacyscheme.Scheme), FilenameOptions: &resource.FilenameOptions{}, RecordFlags: genericclioptions.NewRecordFlags(), Recorder: genericclioptions.NoopRecorder{}, @@ -95,7 +95,6 @@ func NewCmdAutoscale(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) * o := NewAutoscaleOptions(ioStreams) validArgs := []string{"deployment", "replicaset", "replicationcontroller"} - argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ Use: "autoscale (-f FILENAME | TYPE NAME | TYPE/NAME) [--min=MINPODS] --max=MAXPODS [--cpu-percent=CPU]", @@ -108,8 +107,7 @@ func NewCmdAutoscale(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) * cmdutil.CheckErr(o.Validate()) cmdutil.CheckErr(o.Run()) }, - ValidArgs: validArgs, - ArgAliases: argAliases, + ValidArgs: validArgs, } // bind flag structs diff --git a/pkg/kubectl/cmd/certificates.go b/pkg/kubectl/cmd/certificates.go index 169381a48a6..fa13bff51fa 100644 --- a/pkg/kubectl/cmd/certificates.go +++ b/pkg/kubectl/cmd/certificates.go @@ -28,7 +28,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" @@ -98,7 +98,7 @@ func (o *CertificateOptions) Validate() error { func NewCmdCertificateApprove(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { options := CertificateOptions{ - PrintFlags: printers.NewPrintFlags("approved"), + PrintFlags: printers.NewPrintFlags("approved", legacyscheme.Scheme), IOStreams: ioStreams, } cmd := &cobra.Command{ @@ -123,8 +123,10 @@ func NewCmdCertificateApprove(f cmdutil.Factory, ioStreams genericclioptions.IOS cmdutil.CheckErr(options.RunCertificateApprove(cmdutil.GetFlagBool(cmd, "force"))) }, } + + options.PrintFlags.AddFlags(cmd) + cmd.Flags().Bool("force", false, "Update the CSR even if it is already approved.") - cmdutil.AddOutputFlagsForMutation(cmd) cmdutil.AddFilenameOptionFlags(cmd, &options.FilenameOptions, "identifying the resource to update") return cmd @@ -153,7 +155,7 @@ func (o *CertificateOptions) RunCertificateApprove(force bool) error { func NewCmdCertificateDeny(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { options := CertificateOptions{ - PrintFlags: printers.NewPrintFlags("denied"), + PrintFlags: printers.NewPrintFlags("denied", legacyscheme.Scheme), IOStreams: ioStreams, } cmd := &cobra.Command{ @@ -173,8 +175,10 @@ func NewCmdCertificateDeny(f cmdutil.Factory, ioStreams genericclioptions.IOStre cmdutil.CheckErr(options.RunCertificateDeny(cmdutil.GetFlagBool(cmd, "force"))) }, } + + options.PrintFlags.AddFlags(cmd) + cmd.Flags().Bool("force", false, "Update the CSR even if it is already denied.") - cmdutil.AddOutputFlagsForMutation(cmd) cmdutil.AddFilenameOptionFlags(cmd, &options.FilenameOptions, "identifying the resource to update") return cmd diff --git a/pkg/kubectl/cmd/clusterinfo.go b/pkg/kubectl/cmd/clusterinfo.go index bd30c721955..da37d1143cc 100644 --- a/pkg/kubectl/cmd/clusterinfo.go +++ b/pkg/kubectl/cmd/clusterinfo.go @@ -29,7 +29,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ct "github.com/daviddengcn/go-colortext" diff --git a/pkg/kubectl/cmd/clusterinfo_dump.go b/pkg/kubectl/cmd/clusterinfo_dump.go index 759c5c2a57d..21dcf9534d4 100644 --- a/pkg/kubectl/cmd/clusterinfo_dump.go +++ b/pkg/kubectl/cmd/clusterinfo_dump.go @@ -28,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" @@ -56,7 +57,7 @@ type ClusterInfoDumpOptions struct { // NewCmdCreateSecret groups subcommands to create various types of secrets func NewCmdClusterInfoDump(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { o := &ClusterInfoDumpOptions{ - PrintFlags: printers.NewPrintFlags(""), + PrintFlags: printers.NewPrintFlags("", legacyscheme.Scheme), IOStreams: ioStreams, } diff --git a/pkg/kubectl/cmd/cmd.go b/pkg/kubectl/cmd/cmd.go index 4cbf38eb367..a1f7498ca5b 100644 --- a/pkg/kubectl/cmd/cmd.go +++ b/pkg/kubectl/cmd/cmd.go @@ -17,12 +17,13 @@ limitations under the License. package cmd import ( + "flag" "fmt" "io" "os" "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apiserver/pkg/util/flag" + utilflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/client-go/tools/clientcmd" "k8s.io/kubernetes/pkg/kubectl/cmd/auth" cmdconfig "k8s.io/kubernetes/pkg/kubectl/cmd/config" @@ -93,13 +94,15 @@ __kubectl_parse_config() fi } +# $1 is the name of resource (required) +# $2 is template string for kubectl get (optional) __kubectl_parse_get() { local template - template="{{ range .items }}{{ .metadata.name }} {{ end }}" + template="${2:-"{{ range .items }}{{ .metadata.name }} {{ end }}"}" local kubectl_out if kubectl_out=$(kubectl get $(__kubectl_override_flags) -o template --template="${template}" "$1" 2>/dev/null); then - COMPREPLY=( $( compgen -W "${kubectl_out[*]}" -- "$cur" ) ) + COMPREPLY+=( $( compgen -W "${kubectl_out[*]}" -- "$cur" ) ) fi } @@ -170,6 +173,36 @@ __kubectl_require_pod_and_container() return 0 } +__kubectl_cp() +{ + if [[ $(type -t compopt) = "builtin" ]]; then + compopt -o nospace + fi + + case "$cur" in + /*|[.~]*) # looks like a path + return + ;; + *:*) # TODO: complete remote files in the pod + return + ;; + */*) # complete / + local template namespace kubectl_out + template="{{ range .items }}{{ .metadata.namespace }}/{{ .metadata.name }}: {{ end }}" + namespace="${cur%%/*}" + if kubectl_out=( $(kubectl get $(__kubectl_override_flags) --namespace "${namespace}" -o template --template="${template}" pods 2>/dev/null) ); then + COMPREPLY=( $(compgen -W "${kubectl_out[*]}" -- "${cur}") ) + fi + return + ;; + *) # complete namespaces, pods, and filedirs + __kubectl_parse_get "namespace" "{{ range .items }}{{ .metadata.name }}/ {{ end }}" + __kubectl_parse_get "pod" "{{ range .items }}{{ .metadata.name }}: {{ end }}" + _filedir + ;; + esac +} + __custom_func() { case ${last_command} in kubectl_get | kubectl_describe | kubectl_delete | kubectl_label | kubectl_edit | kubectl_patch |\ @@ -202,6 +235,10 @@ __custom_func() { __kubectl_config_get_clusters return ;; + kubectl_cp) + __kubectl_cp + return + ;; *) ;; esac @@ -219,11 +256,11 @@ var ( ) func NewDefaultKubectlCommand() *cobra.Command { - return NewKubectlCommand(cmdutil.NewFactory(nil), os.Stdin, os.Stdout, os.Stderr) + return NewKubectlCommand(os.Stdin, os.Stdout, os.Stderr) } // NewKubectlCommand creates the `kubectl` command and its nested children. -func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cobra.Command { +func NewKubectlCommand(in io.Reader, out, err io.Writer) *cobra.Command { // Parent command to which all subcommands are added. cmds := &cobra.Command{ Use: "kubectl", @@ -237,8 +274,14 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob BashCompletionFunction: bashCompletionFunc, } - f.BindFlags(cmds.PersistentFlags()) - f.BindExternalFlags(cmds.PersistentFlags()) + kubeConfigFlags := cmdutil.NewConfigFlags() + kubeConfigFlags.AddFlags(cmds.PersistentFlags()) + matchVersionKubeConfigFlags := cmdutil.NewMatchVersionFlags(kubeConfigFlags) + matchVersionKubeConfigFlags.AddFlags(cmds.PersistentFlags()) + + cmds.PersistentFlags().AddGoFlagSet(flag.CommandLine) + + f := cmdutil.NewFactory(matchVersionKubeConfigFlags) // Sending in 'nil' for the getLanguageFn() results in using // the LANG environment variable. @@ -248,7 +291,7 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob i18n.LoadTranslations("kubectl", nil) // From this point and forward we get warnings on flags that contain "_" separators - cmds.SetGlobalNormalizationFunc(flag.WarnWordSepNormalizeFunc) + cmds.SetGlobalNormalizationFunc(utilflag.WarnWordSepNormalizeFunc) ioStreams := genericclioptions.IOStreams{In: in, Out: out, ErrOut: err} @@ -269,7 +312,7 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob NewCmdExplain("kubectl", f, ioStreams), get.NewCmdGet("kubectl", f, ioStreams), NewCmdEdit(f, ioStreams), - NewCmdDelete(f, out, err), + NewCmdDelete(f, ioStreams), }, }, { @@ -286,7 +329,7 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob Commands: []*cobra.Command{ NewCmdCertificate(f, ioStreams), NewCmdClusterInfo(f, ioStreams), - NewCmdTop(f, out, err), + NewCmdTop(f, ioStreams), NewCmdCordon(f, ioStreams), NewCmdUncordon(f, ioStreams), NewCmdDrain(f, ioStreams), @@ -297,13 +340,13 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob Message: "Troubleshooting and Debugging Commands:", Commands: []*cobra.Command{ NewCmdDescribe("kubectl", f, ioStreams), - NewCmdLogs(f, out, err), - NewCmdAttach(f, in, out, err), - NewCmdExec(f, in, out, err), - NewCmdPortForward(f, out, err), - NewCmdProxy(f, out), + NewCmdLogs(f, ioStreams), + NewCmdAttach(f, ioStreams), + NewCmdExec(f, ioStreams), + NewCmdPortForward(f, ioStreams), + NewCmdProxy(f, ioStreams), NewCmdCp(f, ioStreams), - auth.NewCmdAuth(f, out, err), + auth.NewCmdAuth(f, ioStreams), }, }, { @@ -311,7 +354,7 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob Commands: []*cobra.Command{ NewCmdApply("kubectl", f, ioStreams), NewCmdPatch(f, ioStreams), - NewCmdReplace(f, out, err), + NewCmdReplace(f, ioStreams), NewCmdConvert(f, ioStreams), }, }, @@ -320,7 +363,7 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob Commands: []*cobra.Command{ NewCmdLabel(f, ioStreams), NewCmdAnnotate("kubectl", f, ioStreams), - NewCmdCompletion(out, ""), + NewCmdCompletion(ioStreams.Out, ""), }, }, } @@ -329,7 +372,7 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob filters := []string{"options"} // Hide the "alpha" subcommand if there are no alpha commands in this build. - alpha := NewCmdAlpha(f, in, out, err) + alpha := NewCmdAlpha(f, ioStreams) if !alpha.HasSubCommands() { filters = append(filters, alpha.Name()) } @@ -349,12 +392,12 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob } cmds.AddCommand(alpha) - cmds.AddCommand(cmdconfig.NewCmdConfig(f, clientcmd.NewDefaultPathOptions(), out, err)) - cmds.AddCommand(NewCmdPlugin(f, in, out, err)) + cmds.AddCommand(cmdconfig.NewCmdConfig(f, clientcmd.NewDefaultPathOptions(), ioStreams)) + cmds.AddCommand(NewCmdPlugin(f, ioStreams)) cmds.AddCommand(NewCmdVersion(f, ioStreams)) cmds.AddCommand(NewCmdApiVersions(f, ioStreams)) cmds.AddCommand(NewCmdApiResources(f, ioStreams)) - cmds.AddCommand(NewCmdOptions(out)) + cmds.AddCommand(NewCmdOptions(ioStreams.Out)) return cmds } diff --git a/pkg/kubectl/cmd/cmd_test.go b/pkg/kubectl/cmd/cmd_test.go index 1dfe2295190..742d8926841 100644 --- a/pkg/kubectl/cmd/cmd_test.go +++ b/pkg/kubectl/cmd/cmd_test.go @@ -239,7 +239,7 @@ func newAllPhasePodList() *api.PodList { func TestNormalizationFuncGlobalExistence(t *testing.T) { // This test can be safely deleted when we will not support multiple flag formats - root := NewKubectlCommand(cmdutil.NewFactory(nil), os.Stdin, os.Stdout, os.Stderr) + root := NewKubectlCommand(os.Stdin, os.Stdout, os.Stderr) if root.Parent() != nil { t.Fatal("We expect the root command to be returned") diff --git a/pkg/kubectl/cmd/config/BUILD b/pkg/kubectl/cmd/config/BUILD index ba56eab4b34..494c871890f 100644 --- a/pkg/kubectl/cmd/config/BUILD +++ b/pkg/kubectl/cmd/config/BUILD @@ -14,6 +14,7 @@ go_library( "current_context.go", "delete_cluster.go", "delete_context.go", + "flags.go", "get_clusters.go", "get_contexts.go", "navigation_step_parser.go", @@ -28,11 +29,14 @@ go_library( "//build/visible_to:pkg_kubectl_cmd_config_CONSUMERS", ], deps = [ + "//pkg/api/legacyscheme:go_default_library", "//pkg/kubectl/cmd/templates:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", + "//pkg/kubectl/genericclioptions:go_default_library", "//pkg/kubectl/util/i18n:go_default_library", "//pkg/printers:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", @@ -64,6 +68,7 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/kubectl/cmd/util:go_default_library", + "//pkg/kubectl/genericclioptions:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", diff --git a/pkg/kubectl/cmd/config/config.go b/pkg/kubectl/cmd/config/config.go index 7984eb5e15f..dbfa53efb28 100644 --- a/pkg/kubectl/cmd/config/config.go +++ b/pkg/kubectl/cmd/config/config.go @@ -18,7 +18,6 @@ package config import ( "fmt" - "io" "path" "strconv" @@ -27,11 +26,12 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) // NewCmdConfig creates a command object for the "config" action, and adds all child commands to it. -func NewCmdConfig(f cmdutil.Factory, pathOptions *clientcmd.PathOptions, out, errOut io.Writer) *cobra.Command { +func NewCmdConfig(f cmdutil.Factory, pathOptions *clientcmd.PathOptions, streams genericclioptions.IOStreams) *cobra.Command { if len(pathOptions.ExplicitFileFlag) == 0 { pathOptions.ExplicitFileFlag = clientcmd.RecommendedConfigPathFlag } @@ -48,25 +48,26 @@ func NewCmdConfig(f cmdutil.Factory, pathOptions *clientcmd.PathOptions, out, er 1. If the --` + pathOptions.ExplicitFileFlag + ` flag is set, then only that file is loaded. The flag may only be set once and no merging takes place. 2. If $` + pathOptions.EnvVar + ` environment variable is set, then it is used a list of paths (normal path delimitting rules for your system). These paths are merged. When a value is modified, it is modified in the file that defines the stanza. When a value is created, it is created in the first file that exists. If no files in the chain exist, then it creates the last file in the list. 3. Otherwise, ` + path.Join("${HOME}", pathOptions.GlobalFileSubpath) + ` is used and no merging takes place.`), - Run: cmdutil.DefaultSubCommandRun(errOut), + Run: cmdutil.DefaultSubCommandRun(streams.ErrOut), } // file paths are common to all sub commands cmd.PersistentFlags().StringVar(&pathOptions.LoadingRules.ExplicitPath, pathOptions.ExplicitFileFlag, pathOptions.LoadingRules.ExplicitPath, "use a particular kubeconfig file") - cmd.AddCommand(NewCmdConfigView(f, out, errOut, pathOptions)) - cmd.AddCommand(NewCmdConfigSetCluster(out, pathOptions)) - cmd.AddCommand(NewCmdConfigSetAuthInfo(out, pathOptions)) - cmd.AddCommand(NewCmdConfigSetContext(out, pathOptions)) - cmd.AddCommand(NewCmdConfigSet(out, pathOptions)) - cmd.AddCommand(NewCmdConfigUnset(out, pathOptions)) - cmd.AddCommand(NewCmdConfigCurrentContext(out, pathOptions)) - cmd.AddCommand(NewCmdConfigUseContext(out, pathOptions)) - cmd.AddCommand(NewCmdConfigGetContexts(out, pathOptions)) - cmd.AddCommand(NewCmdConfigGetClusters(out, pathOptions)) - cmd.AddCommand(NewCmdConfigDeleteCluster(out, pathOptions)) - cmd.AddCommand(NewCmdConfigDeleteContext(out, errOut, pathOptions)) - cmd.AddCommand(NewCmdConfigRenameContext(out, pathOptions)) + // TODO(juanvallejo): update all subcommands to work with genericclioptions.IOStreams + cmd.AddCommand(NewCmdConfigView(f, streams, pathOptions)) + cmd.AddCommand(NewCmdConfigSetCluster(streams.Out, pathOptions)) + cmd.AddCommand(NewCmdConfigSetAuthInfo(streams.Out, pathOptions)) + cmd.AddCommand(NewCmdConfigSetContext(streams.Out, pathOptions)) + cmd.AddCommand(NewCmdConfigSet(streams.Out, pathOptions)) + cmd.AddCommand(NewCmdConfigUnset(streams.Out, pathOptions)) + cmd.AddCommand(NewCmdConfigCurrentContext(streams.Out, pathOptions)) + cmd.AddCommand(NewCmdConfigUseContext(streams.Out, pathOptions)) + cmd.AddCommand(NewCmdConfigGetContexts(streams, pathOptions)) + cmd.AddCommand(NewCmdConfigGetClusters(streams.Out, pathOptions)) + cmd.AddCommand(NewCmdConfigDeleteCluster(streams.Out, pathOptions)) + cmd.AddCommand(NewCmdConfigDeleteContext(streams.Out, streams.ErrOut, pathOptions)) + cmd.AddCommand(NewCmdConfigRenameContext(streams.Out, pathOptions)) return cmd } diff --git a/pkg/kubectl/cmd/config/config_test.go b/pkg/kubectl/cmd/config/config_test.go index 2fc8000278d..ca8d26c222c 100644 --- a/pkg/kubectl/cmd/config/config_test.go +++ b/pkg/kubectl/cmd/config/config_test.go @@ -17,7 +17,6 @@ limitations under the License. package config import ( - "bytes" "fmt" "io/ioutil" "os" @@ -31,6 +30,7 @@ import ( "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" ) func newRedFederalCowHammerConfig() clientcmdapi.Config { @@ -863,9 +863,8 @@ func testConfigCommand(args []string, startingConfig clientcmdapi.Config, t *tes argsToUse = append(argsToUse, "--kubeconfig="+fakeKubeFile.Name()) argsToUse = append(argsToUse, args...) - buf := bytes.NewBuffer([]byte{}) - - cmd := NewCmdConfig(cmdutil.NewFactory(nil), clientcmd.NewDefaultPathOptions(), buf, buf) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() + cmd := NewCmdConfig(cmdutil.NewFactory(cmdutil.NewTestConfigFlags()), clientcmd.NewDefaultPathOptions(), streams) cmd.SetArgs(argsToUse) cmd.Execute() diff --git a/pkg/kubectl/cmd/config/flags.go b/pkg/kubectl/cmd/config/flags.go new file mode 100644 index 00000000000..0473f9c0419 --- /dev/null +++ b/pkg/kubectl/cmd/config/flags.go @@ -0,0 +1,88 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "github.com/spf13/cobra" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubernetes/pkg/printers" +) + +// kubectlConfigPrintFlags composes common printer flag structs +// used across all config commands, and provides a method +// of retrieving a known printer based on flag values provided. +type kubectlConfigPrintFlags struct { + JSONYamlPrintFlags *printers.JSONYamlPrintFlags + NamePrintFlags *printers.NamePrintFlags + TemplateFlags *printers.KubeTemplatePrintFlags + + OutputFormat *string +} + +func (f *kubectlConfigPrintFlags) Complete(successTemplate string) error { + return f.NamePrintFlags.Complete(successTemplate) +} + +func (f *kubectlConfigPrintFlags) ToPrinter() (printers.ResourcePrinter, error) { + outputFormat := "" + if f.OutputFormat != nil { + outputFormat = *f.OutputFormat + } + + if p, err := f.JSONYamlPrintFlags.ToPrinter(outputFormat); !printers.IsNoCompatiblePrinterError(err) { + return p, err + } + + if p, err := f.NamePrintFlags.ToPrinter(outputFormat); !printers.IsNoCompatiblePrinterError(err) { + return p, err + } + + if p, err := f.TemplateFlags.ToPrinter(outputFormat); !printers.IsNoCompatiblePrinterError(err) { + return p, err + } + + return nil, printers.NoCompatiblePrinterError{Options: f} +} + +func (f *kubectlConfigPrintFlags) AddFlags(cmd *cobra.Command) { + f.JSONYamlPrintFlags.AddFlags(cmd) + f.NamePrintFlags.AddFlags(cmd) + f.TemplateFlags.AddFlags(cmd) + + if f.OutputFormat != nil { + cmd.Flags().StringVarP(f.OutputFormat, "output", "o", *f.OutputFormat, "Output format. One of: json|yaml|name|go-template=...|go-template-file=...|jsonpath=...|jsonpath-file=... See custom columns [http://kubernetes.io/docs/user-guide/kubectl-overview/#custom-columns], golang template [http://golang.org/pkg/text/template/#pkg-overview] and jsonpath template [http://kubernetes.io/docs/user-guide/jsonpath].") + } +} + +// WithDefaultOutput sets a default output format if one is not provided through a flag value +func (f *kubectlConfigPrintFlags) WithDefaultOutput(output string) *kubectlConfigPrintFlags { + f.OutputFormat = &output + return f +} + +func newKubeConfigPrintFlags(scheme runtime.ObjectConvertor) *kubectlConfigPrintFlags { + outputFormat := "" + + return &kubectlConfigPrintFlags{ + OutputFormat: &outputFormat, + + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(scheme), + NamePrintFlags: printers.NewNamePrintFlags("", scheme), + TemplateFlags: printers.NewKubeTemplatePrintFlags(), + } +} diff --git a/pkg/kubectl/cmd/config/get_contexts.go b/pkg/kubectl/cmd/config/get_contexts.go index b392bdb3961..791db8030c3 100644 --- a/pkg/kubectl/cmd/config/get_contexts.go +++ b/pkg/kubectl/cmd/config/get_contexts.go @@ -31,6 +31,7 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) @@ -41,7 +42,8 @@ type GetContextsOptions struct { nameOnly bool showHeaders bool contextNames []string - out io.Writer + + genericclioptions.IOStreams } var ( @@ -57,8 +59,12 @@ var ( // NewCmdConfigGetContexts creates a command object for the "get-contexts" action, which // retrieves one or more contexts from a kubeconfig. -func NewCmdConfigGetContexts(out io.Writer, configAccess clientcmd.ConfigAccess) *cobra.Command { - options := &GetContextsOptions{configAccess: configAccess} +func NewCmdConfigGetContexts(streams genericclioptions.IOStreams, configAccess clientcmd.ConfigAccess) *cobra.Command { + options := &GetContextsOptions{ + configAccess: configAccess, + + IOStreams: streams, + } cmd := &cobra.Command{ Use: "get-contexts [(-o|--output=)name)]", @@ -74,22 +80,22 @@ func NewCmdConfigGetContexts(out io.Writer, configAccess clientcmd.ConfigAccess) cmdutil.CheckErr(fmt.Errorf("output must be one of '' or 'name': %v", outputFormat)) } if !supportedOutputTypes.Has(outputFormat) { - fmt.Fprintf(out, "--output %v is not available in kubectl config get-contexts; resetting to default output format\n", outputFormat) + fmt.Fprintf(options.Out, "--output %v is not available in kubectl config get-contexts; resetting to default output format\n", outputFormat) cmd.Flags().Set("output", "") } - cmdutil.CheckErr(options.Complete(cmd, args, out)) + cmdutil.CheckErr(options.Complete(cmd, args)) cmdutil.CheckErr(options.RunGetContexts()) }, } - cmdutil.AddOutputFlags(cmd) - cmdutil.AddNoHeadersFlags(cmd) + + cmd.Flags().Bool("no-headers", false, "When using the default or custom-column output format, don't print headers (default print headers).") + cmd.Flags().StringP("output", "o", "", "Output format. One of: name") return cmd } // Complete assigns GetContextsOptions from the args. -func (o *GetContextsOptions) Complete(cmd *cobra.Command, args []string, out io.Writer) error { +func (o *GetContextsOptions) Complete(cmd *cobra.Command, args []string) error { o.contextNames = args - o.out = out o.nameOnly = false if cmdutil.GetFlagString(cmd, "output") == "name" { o.nameOnly = true @@ -109,9 +115,9 @@ func (o GetContextsOptions) RunGetContexts() error { return err } - out, found := o.out.(*tabwriter.Writer) + out, found := o.Out.(*tabwriter.Writer) if !found { - out = printers.GetNewTabWriter(o.out) + out = printers.GetNewTabWriter(o.Out) defer out.Flush() } diff --git a/pkg/kubectl/cmd/config/get_contexts_test.go b/pkg/kubectl/cmd/config/get_contexts_test.go index d21c4470a43..fe16f076647 100644 --- a/pkg/kubectl/cmd/config/get_contexts_test.go +++ b/pkg/kubectl/cmd/config/get_contexts_test.go @@ -17,13 +17,13 @@ limitations under the License. package config import ( - "bytes" "io/ioutil" "os" "testing" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" ) type getContextsTest struct { @@ -157,11 +157,11 @@ func (test getContextsTest) run(t *testing.T) { pathOptions := clientcmd.NewDefaultPathOptions() pathOptions.GlobalFile = fakeKubeFile.Name() pathOptions.EnvVar = "" - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() options := GetContextsOptions{ configAccess: pathOptions, } - cmd := NewCmdConfigGetContexts(buf, options.configAccess) + cmd := NewCmdConfigGetContexts(streams, options.configAccess) if test.nameOnly { cmd.Flags().Set("output", "name") } diff --git a/pkg/kubectl/cmd/config/set.go b/pkg/kubectl/cmd/config/set.go index 4374b74b201..184a987fc3f 100644 --- a/pkg/kubectl/cmd/config/set.go +++ b/pkg/kubectl/cmd/config/set.go @@ -33,11 +33,6 @@ import ( "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) -const ( - cannotHaveStepsAfterError = "Cannot have steps after %v. %v are remaining" - additionStepRequiredUnlessUnsettingError = "Must have additional steps after %v unless you are unsetting it" -) - type setOptions struct { configAccess clientcmd.ConfigAccess propertyName string diff --git a/pkg/kubectl/cmd/config/view.go b/pkg/kubectl/cmd/config/view.go index 92f46510980..5a0466126f1 100644 --- a/pkg/kubectl/cmd/config/view.go +++ b/pkg/kubectl/cmd/config/view.go @@ -18,8 +18,6 @@ package config import ( "errors" - "fmt" - "io" "github.com/spf13/cobra" @@ -27,18 +25,27 @@ import ( "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/client-go/tools/clientcmd/api/latest" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) type ViewOptions struct { + PrintFlags *kubectlConfigPrintFlags + PrintObject printers.ResourcePrinterFunc + ConfigAccess clientcmd.ConfigAccess Merge flag.Tristate Flatten bool Minify bool RawByteData bool + + OutputFormat string + + genericclioptions.IOStreams } var ( @@ -56,12 +63,17 @@ var ( # Get the password for the e2e user kubectl config view -o jsonpath='{.users[?(@.name == "e2e")].user.password}'`) + + defaultOutputFormat = "yaml" ) -func NewCmdConfigView(f cmdutil.Factory, out, errOut io.Writer, ConfigAccess clientcmd.ConfigAccess) *cobra.Command { - options := &ViewOptions{ConfigAccess: ConfigAccess} - // Default to yaml - defaultOutputFormat := "yaml" +func NewCmdConfigView(f cmdutil.Factory, streams genericclioptions.IOStreams, ConfigAccess clientcmd.ConfigAccess) *cobra.Command { + o := &ViewOptions{ + PrintFlags: newKubeConfigPrintFlags(legacyscheme.Scheme).WithDefaultOutput("yaml"), + ConfigAccess: ConfigAccess, + + IOStreams: streams, + } cmd := &cobra.Command{ Use: "view", @@ -69,40 +81,48 @@ func NewCmdConfigView(f cmdutil.Factory, out, errOut io.Writer, ConfigAccess cli Long: view_long, Example: view_example, Run: func(cmd *cobra.Command, args []string) { - options.Complete() - outputFormat := cmdutil.GetFlagString(cmd, "output") - if outputFormat == "wide" { - fmt.Fprintf(errOut, "--output wide is not available in kubectl config view; reset to default output format (%s)\n\n", defaultOutputFormat) - // TODO: once printing is abstracted, this should be handled at flag declaration time - cmd.Flags().Set("output", defaultOutputFormat) - } - if outputFormat == "" { - fmt.Fprintf(errOut, "Reset to default output format (%s) as --output is empty\n", defaultOutputFormat) - // TODO: once printing is abstracted, this should be handled at flag declaration time - cmd.Flags().Set("output", defaultOutputFormat) - } - - printOpts := cmdutil.ExtractCmdPrintOptions(cmd, false) - printer, err := cmdutil.PrinterForOptions(printOpts) - cmdutil.CheckErr(err) - - cmdutil.CheckErr(options.Run(out, printer)) + cmdutil.CheckErr(o.Complete(cmd)) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) }, } - cmdutil.AddPrinterFlags(cmd) - cmd.Flags().Set("output", defaultOutputFormat) + o.PrintFlags.AddFlags(cmd) - options.Merge.Default(true) - mergeFlag := cmd.Flags().VarPF(&options.Merge, "merge", "", "Merge the full hierarchy of kubeconfig files") + o.Merge.Default(true) + mergeFlag := cmd.Flags().VarPF(&o.Merge, "merge", "", "Merge the full hierarchy of kubeconfig files") mergeFlag.NoOptDefVal = "true" - cmd.Flags().BoolVar(&options.RawByteData, "raw", options.RawByteData, "Display raw byte data") - cmd.Flags().BoolVar(&options.Flatten, "flatten", options.Flatten, "Flatten the resulting kubeconfig file into self-contained output (useful for creating portable kubeconfig files)") - cmd.Flags().BoolVar(&options.Minify, "minify", options.Minify, "Remove all information not used by current-context from the output") + cmd.Flags().BoolVar(&o.RawByteData, "raw", o.RawByteData, "Display raw byte data") + cmd.Flags().BoolVar(&o.Flatten, "flatten", o.Flatten, "Flatten the resulting kubeconfig file into self-contained output (useful for creating portable kubeconfig files)") + cmd.Flags().BoolVar(&o.Minify, "minify", o.Minify, "Remove all information not used by current-context from the output") return cmd } -func (o ViewOptions) Run(out io.Writer, printer printers.ResourcePrinter) error { +func (o *ViewOptions) Complete(cmd *cobra.Command) error { + if o.ConfigAccess.IsExplicitFile() { + if !o.Merge.Provided() { + o.Merge.Set("false") + } + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + o.PrintObject = printer.PrintObj + + return nil +} + +func (o ViewOptions) Validate() error { + if !o.Merge.Value() && !o.ConfigAccess.IsExplicitFile() { + return errors.New("if merge==false a precise file must to specified") + } + + return nil +} + +func (o ViewOptions) Run() error { config, err := o.loadConfig() if err != nil { return err @@ -127,22 +147,7 @@ func (o ViewOptions) Run(out io.Writer, printer printers.ResourcePrinter) error return err } - err = printer.PrintObj(convertedObj, out) - if err != nil { - return err - } - - return nil -} - -func (o *ViewOptions) Complete() bool { - if o.ConfigAccess.IsExplicitFile() { - if !o.Merge.Provided() { - o.Merge.Set("false") - } - } - - return true + return o.PrintObject(convertedObj, o.Out) } func (o ViewOptions) loadConfig() (*clientcmdapi.Config, error) { @@ -155,14 +160,6 @@ func (o ViewOptions) loadConfig() (*clientcmdapi.Config, error) { return config, err } -func (o ViewOptions) Validate() error { - if !o.Merge.Value() && !o.ConfigAccess.IsExplicitFile() { - return errors.New("if merge==false a precise file must to specified") - } - - return nil -} - // getStartingConfig returns the Config object built from the sources specified by the options, the filename read (only if it was a single file), and an error if something goes wrong func (o *ViewOptions) getStartingConfig() (*clientcmdapi.Config, error) { switch { diff --git a/pkg/kubectl/cmd/config/view_test.go b/pkg/kubectl/cmd/config/view_test.go index 873e5f0797d..b3f05d48bf4 100644 --- a/pkg/kubectl/cmd/config/view_test.go +++ b/pkg/kubectl/cmd/config/view_test.go @@ -17,7 +17,6 @@ limitations under the License. package config import ( - "bytes" "io/ioutil" "os" "testing" @@ -25,6 +24,7 @@ import ( "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" ) type viewClusterTest struct { @@ -141,9 +141,8 @@ func (test viewClusterTest) run(t *testing.T) { pathOptions := clientcmd.NewDefaultPathOptions() pathOptions.GlobalFile = fakeKubeFile.Name() pathOptions.EnvVar = "" - buf := bytes.NewBuffer([]byte{}) - errBuf := bytes.NewBuffer([]byte{}) - cmd := NewCmdConfigView(cmdutil.NewFactory(nil), buf, errBuf, pathOptions) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() + cmd := NewCmdConfigView(cmdutil.NewFactory(cmdutil.NewTestConfigFlags()), streams, pathOptions) cmd.Flags().Parse(test.flags) if err := cmd.Execute(); err != nil { t.Fatalf("unexpected error executing command: %v,kubectl config view flags: %v", err, test.flags) diff --git a/pkg/kubectl/cmd/convert.go b/pkg/kubectl/cmd/convert.go index 264895023ca..66f4027877f 100644 --- a/pkg/kubectl/cmd/convert.go +++ b/pkg/kubectl/cmd/convert.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" @@ -103,7 +103,7 @@ type ConvertOptions struct { func NewConvertOptions(ioStreams genericclioptions.IOStreams) *ConvertOptions { return &ConvertOptions{ - PrintFlags: printers.NewPrintFlags("converted").WithDefaultOutput("yaml"), + PrintFlags: printers.NewPrintFlags("converted", scheme.Scheme).WithDefaultOutput("yaml"), local: true, IOStreams: ioStreams, } @@ -199,7 +199,7 @@ func objectListToVersionedObject(objects []runtime.Object, specifiedOutputVersio if !specifiedOutputVersion.Empty() { targetVersions = append(targetVersions, specifiedOutputVersion) } - targetVersions = append(targetVersions, scheme.Registry.GroupOrDie(api.GroupName).GroupVersions[0]) + targetVersions = append(targetVersions, schema.GroupVersion{Group: "", Version: "v1"}) converted, err := tryConvert(scheme.Scheme, objectList, targetVersions...) if err != nil { return nil, err @@ -226,7 +226,7 @@ func asVersionedObject(infos []*resource.Info, forceList bool, specifiedOutputVe if !specifiedOutputVersion.Empty() { targetVersions = append(targetVersions, specifiedOutputVersion) } - targetVersions = append(targetVersions, scheme.Registry.GroupOrDie(api.GroupName).GroupVersions[0]) + targetVersions = append(targetVersions, schema.GroupVersion{Group: "", Version: "v1"}) converted, err := tryConvert(scheme.Scheme, object, targetVersions...) if err != nil { @@ -275,7 +275,7 @@ func asVersionedObjects(infos []*resource.Info, specifiedOutputVersion schema.Gr gvks, _, err := scheme.Scheme.ObjectKinds(info.Object) if err == nil { for _, gvk := range gvks { - for _, version := range scheme.Registry.RegisteredVersionsForGroup(gvk.Group) { + for _, version := range scheme.Scheme.PrioritizedVersionsForGroup(gvk.Group) { targetVersions = append(targetVersions, version) } } diff --git a/pkg/kubectl/cmd/cp.go b/pkg/kubectl/cmd/cp.go index 258b4f50a78..5a919abfbbb 100644 --- a/pkg/kubectl/cmd/cp.go +++ b/pkg/kubectl/cmd/cp.go @@ -18,7 +18,6 @@ package cmd import ( "archive/tar" - "bytes" "errors" "fmt" "io" @@ -35,6 +34,8 @@ import ( "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/util/i18n" + "bytes" + "github.com/renstrom/dedent" "github.com/spf13/cobra" ) @@ -194,8 +195,10 @@ func (o *CopyOptions) Run(args []string) error { func (o *CopyOptions) checkDestinationIsDir(dest fileSpec) error { options := &ExecOptions{ StreamOptions: StreamOptions{ - Out: bytes.NewBuffer([]byte{}), - Err: bytes.NewBuffer([]byte{}), + IOStreams: genericclioptions.IOStreams{ + Out: bytes.NewBuffer([]byte{}), + ErrOut: bytes.NewBuffer([]byte{}), + }, Namespace: dest.PodNamespace, PodName: dest.PodName, @@ -240,9 +243,11 @@ func (o *CopyOptions) copyToPod(src, dest fileSpec) error { options := &ExecOptions{ StreamOptions: StreamOptions{ - In: reader, - Out: o.Out, - Err: o.ErrOut, + IOStreams: genericclioptions.IOStreams{ + In: reader, + Out: o.Out, + ErrOut: o.ErrOut, + }, Stdin: true, Namespace: dest.PodNamespace, @@ -263,9 +268,11 @@ func (o *CopyOptions) copyFromPod(src, dest fileSpec) error { reader, outStream := io.Pipe() options := &ExecOptions{ StreamOptions: StreamOptions{ - In: nil, - Out: outStream, - Err: o.Out, + IOStreams: genericclioptions.IOStreams{ + In: nil, + Out: outStream, + ErrOut: o.Out, + }, Namespace: src.PodNamespace, PodName: src.PodName, diff --git a/pkg/kubectl/cmd/cp_test.go b/pkg/kubectl/cmd/cp_test.go index 24b7d7d6e38..5b2fe9865a8 100644 --- a/pkg/kubectl/cmd/cp_test.go +++ b/pkg/kubectl/cmd/cp_test.go @@ -513,7 +513,7 @@ func TestCopyToPod(t *testing.T) { tf := cmdtesting.NewTestFactory() tf.Namespace = "test" ns := legacyscheme.Codecs - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.Client = &fake.RESTClient{ GroupVersion: schema.GroupVersion{Group: "", Version: "v1"}, diff --git a/pkg/kubectl/cmd/create/BUILD b/pkg/kubectl/cmd/create/BUILD index b14ab4ff9dd..898a9374bc8 100644 --- a/pkg/kubectl/cmd/create/BUILD +++ b/pkg/kubectl/cmd/create/BUILD @@ -29,7 +29,7 @@ go_library( "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/cmd/util/editor:go_default_library", "//pkg/kubectl/genericclioptions:go_default_library", - "//pkg/kubectl/resource:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/scheme:go_default_library", "//pkg/kubectl/util/i18n:go_default_library", "//pkg/printers:go_default_library", @@ -45,6 +45,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/batch/v1:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/rbac/v1:go_default_library", ], diff --git a/pkg/kubectl/cmd/create/create.go b/pkg/kubectl/cmd/create/create.go index 14b3ba7152e..cee2299f4ee 100644 --- a/pkg/kubectl/cmd/create/create.go +++ b/pkg/kubectl/cmd/create/create.go @@ -19,26 +19,26 @@ package create import ( "fmt" "io" + "net/url" "os" "runtime" "strings" "github.com/spf13/cobra" - "net/url" - "github.com/golang/glog" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" kruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/cmd/util/editor" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -78,7 +78,7 @@ var ( func NewCreateOptions(ioStreams genericclioptions.IOStreams) *CreateOptions { return &CreateOptions{ - PrintFlags: NewPrintFlags("created"), + PrintFlags: NewPrintFlags("created", legacyscheme.Scheme), RecordFlags: genericclioptions.NewRecordFlags(), Recorder: genericclioptions.NoopRecorder{}, @@ -284,12 +284,16 @@ func (o *CreateOptions) raw(f cmdutil.Factory) error { } } // TODO post content with stream. Right now it ignores body content - bytes, err := restClient.Post().RequestURI(o.Raw).Body(data).DoRaw() + result := restClient.Post().RequestURI(o.Raw).Body(data).Do() + if err := result.Error(); err != nil { + return err + } + body, err := result.Raw() if err != nil { return err } - fmt.Fprintf(o.Out, "%v", string(bytes)) + fmt.Fprintf(o.Out, "%v", string(body)) return nil } @@ -340,6 +344,12 @@ type CreateSubcommandOptions struct { DryRun bool CreateAnnotation bool + Namespace string + EnforceNamespace bool + + Mapper meta.RESTMapper + DynamicClient dynamic.DynamicInterface + PrintObj func(obj kruntime.Object) error genericclioptions.IOStreams @@ -347,12 +357,12 @@ type CreateSubcommandOptions struct { func NewCreateSubcommandOptions(ioStreams genericclioptions.IOStreams) *CreateSubcommandOptions { return &CreateSubcommandOptions{ - PrintFlags: NewPrintFlags("created"), + PrintFlags: NewPrintFlags("created", legacyscheme.Scheme), IOStreams: ioStreams, } } -func (o *CreateSubcommandOptions) Complete(cmd *cobra.Command, args []string, generator kubectl.StructuredGenerator) error { +func (o *CreateSubcommandOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string, generator kubectl.StructuredGenerator) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -375,38 +385,43 @@ func (o *CreateSubcommandOptions) Complete(cmd *cobra.Command, args []string, ge return printer.PrintObj(obj, o.Out) } + o.Namespace, o.EnforceNamespace, err = f.DefaultNamespace() + if err != nil { + return err + } + + o.DynamicClient, err = f.DynamicClient() + if err != nil { + return err + } + + o.Mapper, err = f.RESTMapper() + if err != nil { + return err + } + return nil } -// TODO(juanvallejo): remove dependency on factory here. Complete necessary bits -// from it in the Complete() method. // RunCreateSubcommand executes a create subcommand using the specified options -func RunCreateSubcommand(f cmdutil.Factory, options *CreateSubcommandOptions) error { - namespace, nsOverriden, err := f.DefaultNamespace() +func (o *CreateSubcommandOptions) Run() error { + obj, err := o.StructuredGenerator.StructuredGenerate() if err != nil { return err } - obj, err := options.StructuredGenerator.StructuredGenerate() - if err != nil { - return err - } - mapper, err := f.RESTMapper() - if err != nil { - return err - } - if !options.DryRun { + if !o.DryRun { // create subcommands have compiled knowledge of things they create, so type them directly gvks, _, err := legacyscheme.Scheme.ObjectKinds(obj) if err != nil { return err } gvk := gvks[0] - mapping, err := mapper.RESTMapping(schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}, gvk.Version) + mapping, err := o.Mapper.RESTMapping(schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}, gvk.Version) if err != nil { return err } - if err := kubectl.CreateOrUpdateAnnotation(options.CreateAnnotation, obj, cmdutil.InternalVersionJSONEncoder()); err != nil { + if err := kubectl.CreateOrUpdateAnnotation(o.CreateAnnotation, obj, cmdutil.InternalVersionJSONEncoder()); err != nil { return err } @@ -414,14 +429,10 @@ func RunCreateSubcommand(f cmdutil.Factory, options *CreateSubcommandOptions) er if err := legacyscheme.Scheme.Convert(obj, asUnstructured, nil); err != nil { return err } - dynamicClient, err := f.DynamicClient() - if err != nil { - return err - } if mapping.Scope.Name() == meta.RESTScopeNameRoot { - namespace = "" + o.Namespace = "" } - actualObject, err := dynamicClient.Resource(mapping.Resource).Namespace(namespace).Create(asUnstructured) + actualObject, err := o.DynamicClient.Resource(mapping.Resource).Namespace(o.Namespace).Create(asUnstructured) if err != nil { return err } @@ -429,10 +440,10 @@ func RunCreateSubcommand(f cmdutil.Factory, options *CreateSubcommandOptions) er // ensure we pass a versioned object to the printer obj = actualObject } else { - if meta, err := meta.Accessor(obj); err == nil && nsOverriden { - meta.SetNamespace(namespace) + if meta, err := meta.Accessor(obj); err == nil && o.EnforceNamespace { + meta.SetNamespace(o.Namespace) } } - return options.PrintObj(obj) + return o.PrintObj(obj) } diff --git a/pkg/kubectl/cmd/create/create_clusterrolebinding.go b/pkg/kubectl/cmd/create/create_clusterrolebinding.go index 2098457b40f..d0799239063 100644 --- a/pkg/kubectl/cmd/create/create_clusterrolebinding.go +++ b/pkg/kubectl/cmd/create/create_clusterrolebinding.go @@ -52,8 +52,8 @@ func NewCmdCreateClusterRoleBinding(f cmdutil.Factory, ioStreams genericclioptio Long: clusterRoleBindingLong, Example: clusterRoleBindingExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -70,7 +70,7 @@ func NewCmdCreateClusterRoleBinding(f cmdutil.Factory, ioStreams genericclioptio return cmd } -func (o *ClusterRoleBindingOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *ClusterRoleBindingOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -90,10 +90,10 @@ func (o *ClusterRoleBindingOpts) Complete(cmd *cobra.Command, args []string) err return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateClusterRoleBinding is the implementation of the create clusterrolebinding command. -func (o *ClusterRoleBindingOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *ClusterRoleBindingOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_configmap.go b/pkg/kubectl/cmd/create/create_configmap.go index b99b143f738..8abaa314fae 100644 --- a/pkg/kubectl/cmd/create/create_configmap.go +++ b/pkg/kubectl/cmd/create/create_configmap.go @@ -74,8 +74,8 @@ func NewCmdCreateConfigMap(f cmdutil.Factory, ioStreams genericclioptions.IOStre Long: configMapLong, Example: configMapExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -91,7 +91,7 @@ func NewCmdCreateConfigMap(f cmdutil.Factory, ioStreams genericclioptions.IOStre return cmd } -func (o *ConfigMapOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *ConfigMapOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -111,10 +111,10 @@ func (o *ConfigMapOpts) Complete(cmd *cobra.Command, args []string) error { return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateConfigMap is the implementation of the create configmap command. -func (o *ConfigMapOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *ConfigMapOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_configmap_test.go b/pkg/kubectl/cmd/create/create_configmap_test.go index 36884e0a1ca..65aac513988 100644 --- a/pkg/kubectl/cmd/create/create_configmap_test.go +++ b/pkg/kubectl/cmd/create/create_configmap_test.go @@ -39,7 +39,7 @@ func TestCreateConfigMap(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ diff --git a/pkg/kubectl/cmd/create/create_deployment.go b/pkg/kubectl/cmd/create/create_deployment.go index 08c1dc01e36..13ec3235d51 100644 --- a/pkg/kubectl/cmd/create/create_deployment.go +++ b/pkg/kubectl/cmd/create/create_deployment.go @@ -56,7 +56,7 @@ func NewCmdCreateDeployment(f cmdutil.Factory, ioStreams genericclioptions.IOStr Example: deploymentExample, Run: func(cmd *cobra.Command, args []string) { cmdutil.CheckErr(options.Complete(f, cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Run()) }, } @@ -143,13 +143,13 @@ func (o *DeploymentOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args [] return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // createDeployment // 1. Reads user config values from Cobra. // 2. Sets up the correct Generator object. // 3. Calls RunCreateSubcommand. -func (o *DeploymentOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *DeploymentOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_deployment_test.go b/pkg/kubectl/cmd/create/create_deployment_test.go index 1c56a299021..af1e48f081e 100644 --- a/pkg/kubectl/cmd/create/create_deployment_test.go +++ b/pkg/kubectl/cmd/create/create_deployment_test.go @@ -142,7 +142,7 @@ func TestCreateDeploymentNoImage(t *testing.T) { cmd.Flags().Set("output", "name") options := &DeploymentOpts{ CreateSubcommandOptions: &CreateSubcommandOptions{ - PrintFlags: NewPrintFlags("created"), + PrintFlags: NewPrintFlags("created", legacyscheme.Scheme), DryRun: true, IOStreams: ioStreams, }, @@ -153,6 +153,6 @@ func TestCreateDeploymentNoImage(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - err = options.Run(tf) + err = options.Run() assert.Error(t, err, "at least one image must be specified") } diff --git a/pkg/kubectl/cmd/create/create_job.go b/pkg/kubectl/cmd/create/create_job.go index 74462cc9ce0..9db9ff77bfa 100644 --- a/pkg/kubectl/cmd/create/create_job.go +++ b/pkg/kubectl/cmd/create/create_job.go @@ -26,10 +26,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientbatchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -63,7 +64,7 @@ type CreateJobOptions struct { func NewCreateJobOptions(ioStreams genericclioptions.IOStreams) *CreateJobOptions { return &CreateJobOptions{ - PrintFlags: NewPrintFlags("created"), + PrintFlags: NewPrintFlags("created", legacyscheme.Scheme), IOStreams: ioStreams, } } diff --git a/pkg/kubectl/cmd/create/create_job_test.go b/pkg/kubectl/cmd/create/create_job_test.go index af6fbcfa5f2..7aa14cc9f3f 100644 --- a/pkg/kubectl/cmd/create/create_job_test.go +++ b/pkg/kubectl/cmd/create/create_job_test.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" fake "k8s.io/client-go/kubernetes/fake" clienttesting "k8s.io/client-go/testing" + "k8s.io/kubernetes/pkg/api/legacyscheme" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" ) @@ -84,7 +85,7 @@ func TestCreateJobFromCronJob(t *testing.T) { f := cmdtesting.NewTestFactory() defer f.Cleanup() - printFlags := NewPrintFlags("created") + printFlags := NewPrintFlags("created", legacyscheme.Scheme) ioStreams, _, buf, _ := genericclioptions.NewTestIOStreams() cmdOptions := &CreateJobOptions{ diff --git a/pkg/kubectl/cmd/create/create_namespace.go b/pkg/kubectl/cmd/create/create_namespace.go index ddcd56116ec..3ab1a26d4f2 100644 --- a/pkg/kubectl/cmd/create/create_namespace.go +++ b/pkg/kubectl/cmd/create/create_namespace.go @@ -53,8 +53,8 @@ func NewCmdCreateNamespace(f cmdutil.Factory, ioStreams genericclioptions.IOStre Long: namespaceLong, Example: namespaceExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -67,7 +67,7 @@ func NewCmdCreateNamespace(f cmdutil.Factory, ioStreams genericclioptions.IOStre return cmd } -func (o *NamespaceOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *NamespaceOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -81,10 +81,10 @@ func (o *NamespaceOpts) Complete(cmd *cobra.Command, args []string) error { return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateNamespace implements the behavior to run the create namespace command -func (o *NamespaceOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *NamespaceOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_namespace_test.go b/pkg/kubectl/cmd/create/create_namespace_test.go index 785f6bda7a2..8bfd9bd2fb4 100644 --- a/pkg/kubectl/cmd/create/create_namespace_test.go +++ b/pkg/kubectl/cmd/create/create_namespace_test.go @@ -35,7 +35,7 @@ func TestCreateNamespace(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ diff --git a/pkg/kubectl/cmd/create/create_pdb.go b/pkg/kubectl/cmd/create/create_pdb.go index e830aaca2c2..c8027a1423d 100644 --- a/pkg/kubectl/cmd/create/create_pdb.go +++ b/pkg/kubectl/cmd/create/create_pdb.go @@ -58,8 +58,8 @@ func NewCmdCreatePodDisruptionBudget(f cmdutil.Factory, ioStreams genericcliopti Long: pdbLong, Example: pdbExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -75,7 +75,7 @@ func NewCmdCreatePodDisruptionBudget(f cmdutil.Factory, ioStreams genericcliopti return cmd } -func (o *PodDisruptionBudgetOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *PodDisruptionBudgetOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -100,10 +100,10 @@ func (o *PodDisruptionBudgetOpts) Complete(cmd *cobra.Command, args []string) er return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreatePodDisruptionBudget implements the behavior to run the create pdb command. -func (o *PodDisruptionBudgetOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *PodDisruptionBudgetOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_pdb_test.go b/pkg/kubectl/cmd/create/create_pdb_test.go index 93eb4cfad30..06a7fd80c80 100644 --- a/pkg/kubectl/cmd/create/create_pdb_test.go +++ b/pkg/kubectl/cmd/create/create_pdb_test.go @@ -59,7 +59,7 @@ func TestCreatePdb(t *testing.T) { cmd.Flags().Set("dry-run", "true") cmd.Flags().Set("output", outputFormat) - printFlags := NewPrintFlags("created") + printFlags := NewPrintFlags("created", legacyscheme.Scheme) printFlags.OutputFormat = &outputFormat options := &PodDisruptionBudgetOpts{ @@ -69,12 +69,12 @@ func TestCreatePdb(t *testing.T) { IOStreams: ioStreams, }, } - err := options.Complete(cmd, []string{pdbName}) + err := options.Complete(tf, cmd, []string{pdbName}) if err != nil { t.Fatalf("unexpected error: %v", err) } - err = options.Run(tf) + err = options.Run() if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/kubectl/cmd/create/create_priorityclass.go b/pkg/kubectl/cmd/create/create_priorityclass.go index b1c217f1e0e..58abe318006 100644 --- a/pkg/kubectl/cmd/create/create_priorityclass.go +++ b/pkg/kubectl/cmd/create/create_priorityclass.go @@ -56,8 +56,8 @@ func NewCmdCreatePriorityClass(f cmdutil.Factory, ioStreams genericclioptions.IO Long: pcLong, Example: pcExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -73,7 +73,7 @@ func NewCmdCreatePriorityClass(f cmdutil.Factory, ioStreams genericclioptions.IO return cmd } -func (o *PriorityClassOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *PriorityClassOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -92,10 +92,10 @@ func (o *PriorityClassOpts) Complete(cmd *cobra.Command, args []string) error { return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreatePriorityClass implements the behavior to run the create priorityClass command. -func (o *PriorityClassOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *PriorityClassOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_priorityclass_test.go b/pkg/kubectl/cmd/create/create_priorityclass_test.go index 44d30218278..c4048820ab3 100644 --- a/pkg/kubectl/cmd/create/create_priorityclass_test.go +++ b/pkg/kubectl/cmd/create/create_priorityclass_test.go @@ -59,7 +59,7 @@ func TestCreatePriorityClass(t *testing.T) { cmd.Flags().Set("dry-run", "true") cmd.Flags().Set("output", outputFormat) - printFlags := NewPrintFlags("created") + printFlags := NewPrintFlags("created", legacyscheme.Scheme) printFlags.OutputFormat = &outputFormat options := &PriorityClassOpts{ @@ -69,12 +69,12 @@ func TestCreatePriorityClass(t *testing.T) { IOStreams: ioStreams, }, } - err := options.Complete(cmd, []string{pcName}) + err := options.Complete(tf, cmd, []string{pcName}) if err != nil { t.Fatalf("unexpected error: %v", err) } - err = options.Run(tf) + err = options.Run() if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/kubectl/cmd/create/create_quota.go b/pkg/kubectl/cmd/create/create_quota.go index 2806e66d6a0..015c96431bb 100644 --- a/pkg/kubectl/cmd/create/create_quota.go +++ b/pkg/kubectl/cmd/create/create_quota.go @@ -56,8 +56,8 @@ func NewCmdCreateQuota(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) Long: quotaLong, Example: quotaExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -71,7 +71,7 @@ func NewCmdCreateQuota(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) return cmd } -func (o *QuotaOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *QuotaOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -89,10 +89,10 @@ func (o *QuotaOpts) Complete(cmd *cobra.Command, args []string) error { return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateQuota implements the behavior to run the create quota command -func (o *QuotaOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *QuotaOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_role.go b/pkg/kubectl/cmd/create/create_role.go index c3ba4130ae4..a2d2e2dccaf 100644 --- a/pkg/kubectl/cmd/create/create_role.go +++ b/pkg/kubectl/cmd/create/create_role.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" clientgorbacv1 "k8s.io/client-go/kubernetes/typed/rbac/v1" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" @@ -119,7 +120,7 @@ type CreateRoleOptions struct { func NewCreateRoleOptions(ioStreams genericclioptions.IOStreams) *CreateRoleOptions { return &CreateRoleOptions{ - PrintFlags: NewPrintFlags("created"), + PrintFlags: NewPrintFlags("created", legacyscheme.Scheme), IOStreams: ioStreams, } diff --git a/pkg/kubectl/cmd/create/create_role_test.go b/pkg/kubectl/cmd/create/create_role_test.go index bad465e4d48..5741a7fdc03 100644 --- a/pkg/kubectl/cmd/create/create_role_test.go +++ b/pkg/kubectl/cmd/create/create_role_test.go @@ -376,14 +376,14 @@ func TestComplete(t *testing.T) { "test-missing-name": { params: []string{}, roleOptions: &CreateRoleOptions{ - PrintFlags: NewPrintFlags("created"), + PrintFlags: NewPrintFlags("created", legacyscheme.Scheme), }, expectErr: true, }, "test-duplicate-verbs": { params: []string{roleName}, roleOptions: &CreateRoleOptions{ - PrintFlags: NewPrintFlags("created"), + PrintFlags: NewPrintFlags("created", legacyscheme.Scheme), Name: roleName, Verbs: []string{ "get", @@ -416,7 +416,7 @@ func TestComplete(t *testing.T) { "test-verball": { params: []string{roleName}, roleOptions: &CreateRoleOptions{ - PrintFlags: NewPrintFlags("created"), + PrintFlags: NewPrintFlags("created", legacyscheme.Scheme), Name: roleName, Verbs: []string{ "get", @@ -445,7 +445,7 @@ func TestComplete(t *testing.T) { "test-duplicate-resourcenames": { params: []string{roleName}, roleOptions: &CreateRoleOptions{ - PrintFlags: NewPrintFlags("created"), + PrintFlags: NewPrintFlags("created", legacyscheme.Scheme), Name: roleName, Verbs: []string{"*"}, ResourceNames: []string{"foo", "foo"}, @@ -470,7 +470,7 @@ func TestComplete(t *testing.T) { "test-valid-complete-case": { params: []string{roleName}, roleOptions: &CreateRoleOptions{ - PrintFlags: NewPrintFlags("created"), + PrintFlags: NewPrintFlags("created", legacyscheme.Scheme), Name: roleName, Verbs: []string{"*"}, ResourceNames: []string{"foo"}, diff --git a/pkg/kubectl/cmd/create/create_rolebinding.go b/pkg/kubectl/cmd/create/create_rolebinding.go index e6210285ef1..67566df41aa 100644 --- a/pkg/kubectl/cmd/create/create_rolebinding.go +++ b/pkg/kubectl/cmd/create/create_rolebinding.go @@ -52,8 +52,8 @@ func NewCmdCreateRoleBinding(f cmdutil.Factory, ioStreams genericclioptions.IOSt Long: roleBindingLong, Example: roleBindingExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -70,7 +70,7 @@ func NewCmdCreateRoleBinding(f cmdutil.Factory, ioStreams genericclioptions.IOSt return cmd } -func (o *RoleBindingOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *RoleBindingOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -91,9 +91,9 @@ func (o *RoleBindingOpts) Complete(cmd *cobra.Command, args []string) error { return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } -func (o *RoleBindingOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *RoleBindingOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_secret.go b/pkg/kubectl/cmd/create/create_secret.go index a1749d49eff..700e15bb1bf 100644 --- a/pkg/kubectl/cmd/create/create_secret.go +++ b/pkg/kubectl/cmd/create/create_secret.go @@ -89,8 +89,8 @@ func NewCmdCreateSecretGeneric(f cmdutil.Factory, ioStreams genericclioptions.IO Long: secretLong, Example: secretExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -107,7 +107,7 @@ func NewCmdCreateSecretGeneric(f cmdutil.Factory, ioStreams genericclioptions.IO return cmd } -func (o *SecretGenericOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *SecretGenericOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -128,12 +128,12 @@ func (o *SecretGenericOpts) Complete(cmd *cobra.Command, args []string) error { return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateSecretGeneric is the implementation of the create secret generic command -func (o *SecretGenericOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *SecretGenericOpts) Run() error { + return o.CreateSubcommandOptions.Run() } var ( @@ -174,8 +174,8 @@ func NewCmdCreateSecretDockerRegistry(f cmdutil.Factory, ioStreams genericcliopt Long: secretForDockerRegistryLong, Example: secretForDockerRegistryExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -196,7 +196,7 @@ func NewCmdCreateSecretDockerRegistry(f cmdutil.Factory, ioStreams genericcliopt return cmd } -func (o *SecretDockerRegistryOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *SecretDockerRegistryOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -228,12 +228,12 @@ func (o *SecretDockerRegistryOpts) Complete(cmd *cobra.Command, args []string) e return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateSecretDockerRegistry is the implementation of the create secret docker-registry command -func (o *SecretDockerRegistryOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *SecretDockerRegistryOpts) Run() error { + return o.CreateSubcommandOptions.Run() } var ( @@ -265,8 +265,8 @@ func NewCmdCreateSecretTLS(f cmdutil.Factory, ioStreams genericclioptions.IOStre Long: secretForTLSLong, Example: secretForTLSExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -281,7 +281,7 @@ func NewCmdCreateSecretTLS(f cmdutil.Factory, ioStreams genericclioptions.IOStre return cmd } -func (o *SecretTLSOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *SecretTLSOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -306,10 +306,10 @@ func (o *SecretTLSOpts) Complete(cmd *cobra.Command, args []string) error { return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateSecretTLS is the implementation of the create secret tls command -func (o *SecretTLSOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *SecretTLSOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_secret_test.go b/pkg/kubectl/cmd/create/create_secret_test.go index a563ef540b2..21b41a8520c 100644 --- a/pkg/kubectl/cmd/create/create_secret_test.go +++ b/pkg/kubectl/cmd/create/create_secret_test.go @@ -40,7 +40,7 @@ func TestCreateSecretGeneric(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -73,7 +73,7 @@ func TestCreateSecretDockerRegistry(t *testing.T) { secretObject := &v1.Secret{} secretObject.Name = "my-secret" tf := cmdtesting.NewTestFactory() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ diff --git a/pkg/kubectl/cmd/create/create_service.go b/pkg/kubectl/cmd/create/create_service.go index 3afce7207db..e645138d2e6 100644 --- a/pkg/kubectl/cmd/create/create_service.go +++ b/pkg/kubectl/cmd/create/create_service.go @@ -77,8 +77,8 @@ func NewCmdCreateServiceClusterIP(f cmdutil.Factory, ioStreams genericclioptions Long: serviceClusterIPLong, Example: serviceClusterIPExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -96,7 +96,7 @@ func errUnsupportedGenerator(cmd *cobra.Command, generatorName string) error { return cmdutil.UsageErrorf(cmd, "Generator %s not supported. ", generatorName) } -func (o *ServiceClusterIPOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *ServiceClusterIPOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -115,12 +115,12 @@ func (o *ServiceClusterIPOpts) Complete(cmd *cobra.Command, args []string) error return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateServiceClusterIP is the implementation of the create service clusterip command -func (o *ServiceClusterIPOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *ServiceClusterIPOpts) Run() error { + return o.CreateSubcommandOptions.Run() } var ( @@ -149,8 +149,8 @@ func NewCmdCreateServiceNodePort(f cmdutil.Factory, ioStreams genericclioptions. Long: serviceNodePortLong, Example: serviceNodePortExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -164,7 +164,7 @@ func NewCmdCreateServiceNodePort(f cmdutil.Factory, ioStreams genericclioptions. return cmd } -func (o *ServiceNodePortOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *ServiceNodePortOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -184,12 +184,12 @@ func (o *ServiceNodePortOpts) Complete(cmd *cobra.Command, args []string) error return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateServiceNodePort is the implementation of the create service nodeport command -func (o *ServiceNodePortOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *ServiceNodePortOpts) Run() error { + return o.CreateSubcommandOptions.Run() } var ( @@ -218,8 +218,8 @@ func NewCmdCreateServiceLoadBalancer(f cmdutil.Factory, ioStreams genericcliopti Long: serviceLoadBalancerLong, Example: serviceLoadBalancerExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -232,7 +232,7 @@ func NewCmdCreateServiceLoadBalancer(f cmdutil.Factory, ioStreams genericcliopti return cmd } -func (o *ServiceLoadBalancerOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *ServiceLoadBalancerOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -251,12 +251,12 @@ func (o *ServiceLoadBalancerOpts) Complete(cmd *cobra.Command, args []string) er return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateServiceLoadBalancer is the implementation of the create service loadbalancer command -func (o *ServiceLoadBalancerOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *ServiceLoadBalancerOpts) Run() error { + return o.CreateSubcommandOptions.Run() } var ( @@ -289,8 +289,8 @@ func NewCmdCreateServiceExternalName(f cmdutil.Factory, ioStreams genericcliopti Long: serviceExternalNameLong, Example: serviceExternalNameExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -305,7 +305,7 @@ func NewCmdCreateServiceExternalName(f cmdutil.Factory, ioStreams genericcliopti return cmd } -func (o *ServiceExternalNameOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *ServiceExternalNameOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -324,10 +324,10 @@ func (o *ServiceExternalNameOpts) Complete(cmd *cobra.Command, args []string) er return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateExternalNameService is the implementation of the create service externalname command -func (o *ServiceExternalNameOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *ServiceExternalNameOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_service_test.go b/pkg/kubectl/cmd/create/create_service_test.go index 059c5c0fcc3..9fd018f1852 100644 --- a/pkg/kubectl/cmd/create/create_service_test.go +++ b/pkg/kubectl/cmd/create/create_service_test.go @@ -35,7 +35,7 @@ func TestCreateService(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) negSer := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -69,7 +69,7 @@ func TestCreateServiceNodePort(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) negSer := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -103,7 +103,7 @@ func TestCreateServiceExternalName(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) negSer := legacyscheme.Codecs tf.Client = &fake.RESTClient{ diff --git a/pkg/kubectl/cmd/create/create_serviceaccount.go b/pkg/kubectl/cmd/create/create_serviceaccount.go index 3489376ef73..0399f8307a2 100644 --- a/pkg/kubectl/cmd/create/create_serviceaccount.go +++ b/pkg/kubectl/cmd/create/create_serviceaccount.go @@ -53,8 +53,8 @@ func NewCmdCreateServiceAccount(f cmdutil.Factory, ioStreams genericclioptions.I Long: serviceAccountLong, Example: serviceAccountExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(options.Complete(cmd, args)) - cmdutil.CheckErr(options.Run(f)) + cmdutil.CheckErr(options.Complete(f, cmd, args)) + cmdutil.CheckErr(options.Run()) }, } @@ -66,7 +66,7 @@ func NewCmdCreateServiceAccount(f cmdutil.Factory, ioStreams genericclioptions.I return cmd } -func (o *ServiceAccountOpts) Complete(cmd *cobra.Command, args []string) error { +func (o *ServiceAccountOpts) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { name, err := NameFromCommandArgs(cmd, args) if err != nil { return err @@ -80,10 +80,10 @@ func (o *ServiceAccountOpts) Complete(cmd *cobra.Command, args []string) error { return errUnsupportedGenerator(cmd, generatorName) } - return o.CreateSubcommandOptions.Complete(cmd, args, generator) + return o.CreateSubcommandOptions.Complete(f, cmd, args, generator) } // CreateServiceAccount implements the behavior to run the create service account command -func (o *ServiceAccountOpts) Run(f cmdutil.Factory) error { - return RunCreateSubcommand(f, o.CreateSubcommandOptions) +func (o *ServiceAccountOpts) Run() error { + return o.CreateSubcommandOptions.Run() } diff --git a/pkg/kubectl/cmd/create/create_serviceaccount_test.go b/pkg/kubectl/cmd/create/create_serviceaccount_test.go index 6fcba4e02f2..2f62c05fd3c 100644 --- a/pkg/kubectl/cmd/create/create_serviceaccount_test.go +++ b/pkg/kubectl/cmd/create/create_serviceaccount_test.go @@ -35,7 +35,7 @@ func TestCreateServiceAccount(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ diff --git a/pkg/kubectl/cmd/create/create_test.go b/pkg/kubectl/cmd/create/create_test.go index f83a51c6e52..0d5a956afac 100644 --- a/pkg/kubectl/cmd/create/create_test.go +++ b/pkg/kubectl/cmd/create/create_test.go @@ -54,7 +54,7 @@ func TestCreateObject(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ GroupVersion: schema.GroupVersion{Version: "v1"}, @@ -90,7 +90,7 @@ func TestCreateMultipleObject(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ GroupVersion: schema.GroupVersion{Version: "v1"}, @@ -130,7 +130,7 @@ func TestCreateDirectory(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ GroupVersion: schema.GroupVersion{Version: "v1"}, diff --git a/pkg/kubectl/cmd/create/flags.go b/pkg/kubectl/cmd/create/flags.go index 529bf67a98b..5d4133bab90 100644 --- a/pkg/kubectl/cmd/create/flags.go +++ b/pkg/kubectl/cmd/create/flags.go @@ -19,6 +19,7 @@ package create import ( "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/pkg/printers" ) @@ -68,14 +69,14 @@ func (f *PrintFlags) AddFlags(cmd *cobra.Command) { } } -func NewPrintFlags(operation string) *PrintFlags { +func NewPrintFlags(operation string, scheme runtime.ObjectConvertor) *PrintFlags { outputFormat := "" return &PrintFlags{ OutputFormat: &outputFormat, - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(operation), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(scheme), + NamePrintFlags: printers.NewNamePrintFlags(operation, scheme), TemplateFlags: printers.NewKubeTemplatePrintFlags(), } } diff --git a/pkg/kubectl/cmd/delete.go b/pkg/kubectl/cmd/delete.go index 39a45f4f9de..5d80ae4642e 100644 --- a/pkg/kubectl/cmd/delete.go +++ b/pkg/kubectl/cmd/delete.go @@ -18,7 +18,6 @@ package cmd import ( "fmt" - "io" "strings" "time" @@ -31,7 +30,8 @@ import ( "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -109,11 +109,10 @@ type DeleteOptions struct { Mapper meta.RESTMapper Result *resource.Result - Out io.Writer - ErrOut io.Writer + genericclioptions.IOStreams } -func NewCmdDelete(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { +func NewCmdDelete(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { deleteFlags := NewDeleteCommandFlags("containing the resource to delete.") cmd := &cobra.Command{ @@ -123,16 +122,14 @@ func NewCmdDelete(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { Long: delete_long, Example: delete_example, Run: func(cmd *cobra.Command, args []string) { - options := deleteFlags.ToOptions(out, errOut) - cmdutil.CheckErr(cmdutil.ValidateOutputArgs(cmd)) - - if err := options.Complete(f, out, errOut, args, cmd); err != nil { + o := deleteFlags.ToOptions(streams) + if err := o.Complete(f, args, cmd); err != nil { cmdutil.CheckErr(err) } - if err := options.Validate(cmd); err != nil { + if err := o.Validate(cmd); err != nil { cmdutil.CheckErr(cmdutil.UsageErrorf(cmd, err.Error())) } - if err := options.RunDelete(); err != nil { + if err := o.RunDelete(); err != nil { cmdutil.CheckErr(err) } }, @@ -145,12 +142,32 @@ func NewCmdDelete(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { return cmd } -func (o *DeleteOptions) Complete(f cmdutil.Factory, out, errOut io.Writer, args []string, cmd *cobra.Command) error { +func (o *DeleteOptions) Complete(f cmdutil.Factory, args []string, cmd *cobra.Command) error { cmdNamespace, enforceNamespace, err := f.DefaultNamespace() if err != nil { return err } + if o.DeleteAll || len(o.LabelSelector) > 0 || len(o.FieldSelector) > 0 { + if f := cmd.Flags().Lookup("ignore-not-found"); f != nil && !f.Changed { + // If the user didn't explicitly set the option, default to ignoring NotFound errors when used with --all, -l, or --field-selector + o.IgnoreNotFound = true + } + } + if o.DeleteNow { + if o.GracePeriod != -1 { + return fmt.Errorf("--now and --grace-period cannot be specified together") + } + o.GracePeriod = 1 + } + if o.GracePeriod == 0 && !o.ForceDeletion { + // To preserve backwards compatibility, but prevent accidental data loss, we convert --grace-period=0 + // into --grace-period=1 and wait until the object is successfully deleted. Users may provide --force + // to bypass this wait. + o.WaitForDeletion = true + o.GracePeriod = 1 + } + o.Reaper = f.Reaper includeUninitialized := cmdutil.ShouldIncludeUninitialized(cmd, false) @@ -177,48 +194,25 @@ func (o *DeleteOptions) Complete(f cmdutil.Factory, out, errOut io.Writer, args return err } - // Set up writer - o.Out = out - o.ErrOut = errOut - return nil } func (o *DeleteOptions) Validate(cmd *cobra.Command) error { + if o.Output != "" && o.Output != "name" { + return cmdutil.UsageErrorf(cmd, "Unexpected -o output mode: %v. We only support '-o name'.", o.Output) + } + if o.DeleteAll && len(o.LabelSelector) > 0 { return fmt.Errorf("cannot set --all and --selector at the same time") } if o.DeleteAll && len(o.FieldSelector) > 0 { return fmt.Errorf("cannot set --all and --field-selector at the same time") } - if o.DeleteAll { - f := cmd.Flags().Lookup("ignore-not-found") - // The flag should never be missing - if f == nil { - return fmt.Errorf("missing --ignore-not-found flag") - } - // If the user didn't explicitly set the option, default to ignoring NotFound errors when used with --all - if !f.Changed { - o.IgnoreNotFound = true - } - } - if o.DeleteNow { - if o.GracePeriod != -1 { - return fmt.Errorf("--now and --grace-period cannot be specified together") - } - o.GracePeriod = 1 - } - if o.GracePeriod == 0 { - if o.ForceDeletion { - fmt.Fprintf(o.ErrOut, "warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.\n") - } else { - // To preserve backwards compatibility, but prevent accidental data loss, we convert --grace-period=0 - // into --grace-period=1 and wait until the object is successfully deleted. Users may provide --force - // to bypass this wait. - o.WaitForDeletion = true - o.GracePeriod = 1 - } - } else if o.ForceDeletion { + + switch { + case o.GracePeriod == 0 && o.ForceDeletion: + fmt.Fprintf(o.ErrOut, "warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.\n") + case o.ForceDeletion: fmt.Fprintf(o.ErrOut, "warning: --force is ignored because --grace-period is not 0.\n") } return nil diff --git a/pkg/kubectl/cmd/delete_flags.go b/pkg/kubectl/cmd/delete_flags.go index 10cc3d85ea8..6e65f166964 100644 --- a/pkg/kubectl/cmd/delete_flags.go +++ b/pkg/kubectl/cmd/delete_flags.go @@ -17,13 +17,13 @@ limitations under the License. package cmd import ( - "io" "time" "github.com/spf13/cobra" "k8s.io/kubernetes/pkg/kubectl" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" ) type FileNameFlags struct { @@ -72,10 +72,9 @@ type DeleteFlags struct { Output *string } -func (f *DeleteFlags) ToOptions(out, errOut io.Writer) *DeleteOptions { +func (f *DeleteFlags) ToOptions(streams genericclioptions.IOStreams) *DeleteOptions { options := &DeleteOptions{ - Out: out, - ErrOut: errOut, + IOStreams: streams, } // add filename options diff --git a/pkg/kubectl/cmd/delete_test.go b/pkg/kubectl/cmd/delete_test.go index ca3a1846edb..233d0a00910 100644 --- a/pkg/kubectl/cmd/delete_test.go +++ b/pkg/kubectl/cmd/delete_test.go @@ -17,7 +17,6 @@ limitations under the License. package cmd import ( - "bytes" "encoding/json" "io" "io/ioutil" @@ -38,7 +37,8 @@ import ( "k8s.io/kubernetes/pkg/kubectl" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" ) @@ -48,9 +48,7 @@ func fakecmd() *cobra.Command { cmd := &cobra.Command{ Use: "delete ([-f FILENAME] | TYPE [(NAME | -l label | --all)])", DisableFlagsInUseLine: true, - Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(cmdutil.ValidateOutputArgs(cmd)) - }, + Run: func(cmd *cobra.Command, args []string) {}, } return cmd } @@ -62,7 +60,7 @@ func TestDeleteObjectByTuple(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -86,8 +84,8 @@ func TestDeleteObjectByTuple(t *testing.T) { } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) - cmd := NewCmdDelete(tf, buf, errBuf) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() + cmd := NewCmdDelete(tf, streams) cmd.Flags().Set("namespace", "test") cmd.Flags().Set("cascade", "false") cmd.Flags().Set("output", "name") @@ -97,8 +95,8 @@ func TestDeleteObjectByTuple(t *testing.T) { } // Test cascading delete of object without client-side reaper doesn't make GET requests - buf, errBuf = bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) - cmd = NewCmdDelete(tf, buf, errBuf) + streams, _, buf, _ = genericclioptions.NewTestIOStreams() + cmd = NewCmdDelete(tf, streams) cmd.Flags().Set("namespace", "test") cmd.Flags().Set("output", "name") cmd.Run(cmd, []string{"secrets/mysecret"}) @@ -128,7 +126,7 @@ func TestOrphanDependentsInDeleteObject(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) var expectedOrphanDependents *bool tf.UnstructuredClient = &fake.RESTClient{ @@ -149,8 +147,8 @@ func TestOrphanDependentsInDeleteObject(t *testing.T) { // DeleteOptions.OrphanDependents should be false, when cascade is true (default). falseVar := false expectedOrphanDependents = &falseVar - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) - cmd := NewCmdDelete(tf, buf, errBuf) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() + cmd := NewCmdDelete(tf, streams) cmd.Flags().Set("namespace", "test") cmd.Flags().Set("output", "name") cmd.Run(cmd, []string{"secrets/mysecret"}) @@ -161,8 +159,8 @@ func TestOrphanDependentsInDeleteObject(t *testing.T) { // Test that delete options should be set to orphan when cascade is false. trueVar := true expectedOrphanDependents = &trueVar - buf, errBuf = bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) - cmd = NewCmdDelete(tf, buf, errBuf) + streams, _, buf, _ = genericclioptions.NewTestIOStreams() + cmd = NewCmdDelete(tf, streams) cmd.Flags().Set("namespace", "test") cmd.Flags().Set("cascade", "false") cmd.Flags().Set("output", "name") @@ -180,7 +178,7 @@ func TestDeleteNamedObject(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -204,8 +202,8 @@ func TestDeleteNamedObject(t *testing.T) { } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) - cmd := NewCmdDelete(tf, buf, errBuf) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() + cmd := NewCmdDelete(tf, streams) cmd.Flags().Set("namespace", "test") cmd.Flags().Set("cascade", "false") cmd.Flags().Set("output", "name") @@ -215,8 +213,8 @@ func TestDeleteNamedObject(t *testing.T) { } // Test cascading delete of object without client-side reaper doesn't make GET requests - buf, errBuf = bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) - cmd = NewCmdDelete(tf, buf, errBuf) + streams, _, buf, _ = genericclioptions.NewTestIOStreams() + cmd = NewCmdDelete(tf, streams) cmd.Flags().Set("namespace", "test") cmd.Flags().Set("cascade", "false") cmd.Flags().Set("output", "name") @@ -233,7 +231,7 @@ func TestDeleteObject(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -248,9 +246,9 @@ func TestDeleteObject(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) - cmd := NewCmdDelete(tf, buf, errBuf) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() + cmd := NewCmdDelete(tf, streams) cmd.Flags().Set("filename", "../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml") cmd.Flags().Set("cascade", "false") cmd.Flags().Set("output", "name") @@ -294,7 +292,7 @@ func TestDeleteObjectGraceZero(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -320,11 +318,11 @@ func TestDeleteObjectGraceZero(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) reaper := &fakeReaper{} fake := &fakeReaperFactory{Factory: tf, reaper: reaper} - cmd := NewCmdDelete(fake, buf, errBuf) + streams, _, buf, errBuf := genericclioptions.NewTestIOStreams() + cmd := NewCmdDelete(fake, streams) cmd.Flags().Set("output", "name") cmd.Flags().Set("grace-period", "0") cmd.Run(cmd, []string{"pods/nginx"}) @@ -359,7 +357,6 @@ func TestDeleteObjectNotFound(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) options := &DeleteOptions{ FilenameOptions: resource.FilenameOptions{ @@ -368,8 +365,9 @@ func TestDeleteObjectNotFound(t *testing.T) { GracePeriod: -1, Cascade: false, Output: "name", + IOStreams: genericclioptions.NewTestIOStreamsDiscard(), } - err := options.Complete(tf, buf, errBuf, []string{}, fakecmd()) + err := options.Complete(tf, []string{}, fakecmd()) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -397,9 +395,9 @@ func TestDeleteObjectIgnoreNotFound(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdDelete(tf, buf, errBuf) + cmd := NewCmdDelete(tf, streams) cmd.Flags().Set("filename", "../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml") cmd.Flags().Set("cascade", "false") cmd.Flags().Set("ignore-not-found", "true") @@ -421,7 +419,7 @@ func TestDeleteAllNotFound(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -440,7 +438,6 @@ func TestDeleteAllNotFound(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) // Make sure we can explicitly choose to fail on NotFound errors, even with --all options := &DeleteOptions{ @@ -450,8 +447,9 @@ func TestDeleteAllNotFound(t *testing.T) { DeleteAll: true, IgnoreNotFound: false, Output: "name", + IOStreams: genericclioptions.NewTestIOStreamsDiscard(), } - err := options.Complete(tf, buf, errBuf, []string{"services"}, fakecmd()) + err := options.Complete(tf, []string{"services"}, fakecmd()) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -468,7 +466,7 @@ func TestDeleteAllIgnoreNotFound(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) // Add an item to the list which will result in a 404 on delete svc.Items = append(svc.Items, api.Service{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) @@ -491,9 +489,9 @@ func TestDeleteAllIgnoreNotFound(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdDelete(tf, buf, errBuf) + cmd := NewCmdDelete(tf, streams) cmd.Flags().Set("all", "true") cmd.Flags().Set("cascade", "false") cmd.Flags().Set("output", "name") @@ -511,7 +509,7 @@ func TestDeleteMultipleObject(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -528,9 +526,9 @@ func TestDeleteMultipleObject(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdDelete(tf, buf, errBuf) + cmd := NewCmdDelete(tf, streams) cmd.Flags().Set("filename", "../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml") cmd.Flags().Set("filename", "../../../test/e2e/testing-manifests/guestbook/frontend-service.yaml") cmd.Flags().Set("cascade", "false") @@ -549,7 +547,7 @@ func TestDeleteMultipleObjectContinueOnMissing(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -566,7 +564,7 @@ func TestDeleteMultipleObjectContinueOnMissing(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() options := &DeleteOptions{ FilenameOptions: resource.FilenameOptions{ @@ -575,8 +573,9 @@ func TestDeleteMultipleObjectContinueOnMissing(t *testing.T) { GracePeriod: -1, Cascade: false, Output: "name", + IOStreams: streams, } - err := options.Complete(tf, buf, errBuf, []string{}, fakecmd()) + err := options.Complete(tf, []string{}, fakecmd()) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -596,7 +595,7 @@ func TestDeleteMultipleResourcesWithTheSameName(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -618,9 +617,9 @@ func TestDeleteMultipleResourcesWithTheSameName(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdDelete(tf, buf, errBuf) + cmd := NewCmdDelete(tf, streams) cmd.Flags().Set("namespace", "test") cmd.Flags().Set("cascade", "false") cmd.Flags().Set("output", "name") @@ -637,7 +636,7 @@ func TestDeleteDirectory(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -652,9 +651,9 @@ func TestDeleteDirectory(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdDelete(tf, buf, errBuf) + cmd := NewCmdDelete(tf, streams) cmd.Flags().Set("filename", "../../../test/e2e/testing-manifests/guestbook/legacy") cmd.Flags().Set("cascade", "false") cmd.Flags().Set("output", "name") @@ -672,7 +671,7 @@ func TestDeleteMultipleSelector(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -699,9 +698,9 @@ func TestDeleteMultipleSelector(t *testing.T) { }), } tf.Namespace = "test" - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdDelete(tf, buf, errBuf) + cmd := NewCmdDelete(tf, streams) cmd.Flags().Set("selector", "a=b") cmd.Flags().Set("cascade", "false") cmd.Flags().Set("output", "name") @@ -744,15 +743,15 @@ func TestResourceErrors(t *testing.T) { tf.Namespace = "test" tf.ClientConfigVal = defaultClientConfig() - buf, errBuf := bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{}) - + streams, _, buf, _ := genericclioptions.NewTestIOStreams() options := &DeleteOptions{ FilenameOptions: resource.FilenameOptions{}, GracePeriod: -1, Cascade: false, Output: "name", + IOStreams: streams, } - err := options.Complete(tf, buf, errBuf, testCase.args, fakecmd()) + err := options.Complete(tf, testCase.args, fakecmd()) if !testCase.errFn(err) { t.Errorf("%s: unexpected error: %v", k, err) return diff --git a/pkg/kubectl/cmd/describe.go b/pkg/kubectl/cmd/describe.go index e783f8ec166..0e501383176 100644 --- a/pkg/kubectl/cmd/describe.go +++ b/pkg/kubectl/cmd/describe.go @@ -29,7 +29,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) diff --git a/pkg/kubectl/cmd/describe_test.go b/pkg/kubectl/cmd/describe_test.go index e89845fa2a3..6ecc1eb909d 100644 --- a/pkg/kubectl/cmd/describe_test.go +++ b/pkg/kubectl/cmd/describe_test.go @@ -88,7 +88,7 @@ func TestDescribeObject(t *testing.T) { _, _, rc := testData() tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) d := &testDescriber{Output: "test output"} tf.DescriberVal = d @@ -125,7 +125,7 @@ func TestDescribeListObjects(t *testing.T) { pods, _, _ := testData() tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) d := &testDescriber{Output: "test output"} tf.DescriberVal = d @@ -148,7 +148,7 @@ func TestDescribeObjectShowEvents(t *testing.T) { pods, _, _ := testData() tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) d := &testDescriber{Output: "test output"} tf.DescriberVal = d @@ -170,7 +170,7 @@ func TestDescribeObjectSkipEvents(t *testing.T) { pods, _, _ := testData() tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) d := &testDescriber{Output: "test output"} tf.DescriberVal = d diff --git a/pkg/kubectl/cmd/diff.go b/pkg/kubectl/cmd/diff.go index 43440f54907..7f42034b22e 100644 --- a/pkg/kubectl/cmd/diff.go +++ b/pkg/kubectl/cmd/diff.go @@ -34,7 +34,8 @@ import ( "k8s.io/kubernetes/pkg/kubectl/apply/strategy" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/utils/exec" ) @@ -101,12 +102,11 @@ func parseDiffArguments(args []string) (string, string, error) { return from, to, nil } -func NewCmdDiff(f cmdutil.Factory, stdout, stderr io.Writer) *cobra.Command { +func NewCmdDiff(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { var options DiffOptions diff := DiffProgram{ - Exec: exec.New(), - Stdout: stdout, - Stderr: stderr, + Exec: exec.New(), + IOStreams: streams, } cmd := &cobra.Command{ Use: "diff -f FILENAME", @@ -132,9 +132,8 @@ func NewCmdDiff(f cmdutil.Factory, stdout, stderr io.Writer) *cobra.Command { // KUBERNETES_EXTERNAL_DIFF environment variable will be used a diff // program. By default, `diff(1)` will be used. type DiffProgram struct { - Exec exec.Interface - Stdout io.Writer - Stderr io.Writer + Exec exec.Interface + genericclioptions.IOStreams } func (d *DiffProgram) getCommand(args ...string) exec.Cmd { @@ -147,8 +146,8 @@ func (d *DiffProgram) getCommand(args ...string) exec.Cmd { } cmd := d.Exec.Command(diff, args...) - cmd.SetStdout(d.Stdout) - cmd.SetStderr(d.Stderr) + cmd.SetStdout(d.Out) + cmd.SetStderr(d.ErrOut) return cmd } diff --git a/pkg/kubectl/cmd/diff_test.go b/pkg/kubectl/cmd/diff_test.go index 4400023ff3e..302e69e5a99 100644 --- a/pkg/kubectl/cmd/diff_test.go +++ b/pkg/kubectl/cmd/diff_test.go @@ -25,6 +25,7 @@ import ( "strings" "testing" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/utils/exec" ) @@ -137,11 +138,10 @@ func TestArguments(t *testing.T) { func TestDiffProgram(t *testing.T) { os.Setenv("KUBERNETES_EXTERNAL_DIFF", "echo") - stdout := bytes.Buffer{} + streams, _, stdout, _ := genericclioptions.NewTestIOStreams() diff := DiffProgram{ - Stdout: &stdout, - Stderr: &bytes.Buffer{}, - Exec: exec.New(), + IOStreams: streams, + Exec: exec.New(), } err := diff.Run("one", "two") if err != nil { diff --git a/pkg/kubectl/cmd/drain.go b/pkg/kubectl/cmd/drain.go index d4ceeab0f5c..3ae1bd0adbb 100644 --- a/pkg/kubectl/cmd/drain.go +++ b/pkg/kubectl/cmd/drain.go @@ -47,7 +47,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) @@ -107,7 +107,7 @@ var ( func NewCmdCordon(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { options := &DrainOptions{ - PrintFlags: printers.NewPrintFlags("cordoned"), + PrintFlags: printers.NewPrintFlags("cordoned", legacyscheme.Scheme), IOStreams: ioStreams, } @@ -139,7 +139,7 @@ var ( func NewCmdUncordon(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { options := &DrainOptions{ - PrintFlags: printers.NewPrintFlags("uncordoned"), + PrintFlags: printers.NewPrintFlags("uncordoned", legacyscheme.Scheme), IOStreams: ioStreams, } @@ -195,7 +195,7 @@ var ( func NewDrainOptions(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *DrainOptions { return &DrainOptions{ - PrintFlags: printers.NewPrintFlags("drained"), + PrintFlags: printers.NewPrintFlags("drained", legacyscheme.Scheme), IOStreams: ioStreams, backOff: clockwork.NewRealClock(), diff --git a/pkg/kubectl/cmd/drain_test.go b/pkg/kubectl/cmd/drain_test.go index 99dead5f9ed..55713f12abc 100644 --- a/pkg/kubectl/cmd/drain_test.go +++ b/pkg/kubectl/cmd/drain_test.go @@ -152,7 +152,7 @@ func TestCordon(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs new_node := &corev1.Node{} @@ -606,7 +606,7 @@ func TestDrain(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -834,7 +834,7 @@ func TestDeletePods(t *testing.T) { defer tf.Cleanup() o := DrainOptions{ - PrintFlags: printers.NewPrintFlags("drained"), + PrintFlags: printers.NewPrintFlags("drained", legacyscheme.Scheme), } o.Out = os.Stdout diff --git a/pkg/kubectl/cmd/edit_test.go b/pkg/kubectl/cmd/edit_test.go index e0445d279e1..fe5fd28ae44 100644 --- a/pkg/kubectl/cmd/edit_test.go +++ b/pkg/kubectl/cmd/edit_test.go @@ -40,7 +40,7 @@ import ( cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" ) type EditTestCase struct { diff --git a/pkg/kubectl/cmd/exec.go b/pkg/kubectl/cmd/exec.go index f9ceabd3e3d..b5c04dd83ca 100644 --- a/pkg/kubectl/cmd/exec.go +++ b/pkg/kubectl/cmd/exec.go @@ -32,6 +32,7 @@ import ( coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/kubectl/util/term" "k8s.io/kubernetes/pkg/util/interrupt" @@ -62,12 +63,10 @@ const ( execUsageStr = "expected 'exec POD_NAME COMMAND [ARG1] [ARG2] ... [ARGN]'.\nPOD_NAME and COMMAND are required arguments for the exec command" ) -func NewCmdExec(f cmdutil.Factory, cmdIn io.Reader, cmdOut, cmdErr io.Writer) *cobra.Command { +func NewCmdExec(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { options := &ExecOptions{ StreamOptions: StreamOptions{ - In: cmdIn, - Out: cmdOut, - Err: cmdErr, + IOStreams: streams, }, Executor: &DefaultRemoteExecutor{}, @@ -125,9 +124,8 @@ type StreamOptions struct { Quiet bool // InterruptParent, if set, is used to handle interrupts while attached InterruptParent *interrupt.Handler - In io.Reader - Out io.Writer - Err io.Writer + + genericclioptions.IOStreams // for testing overrideStreams func() (io.ReadCloser, io.Writer, io.Writer) @@ -155,7 +153,7 @@ func (p *ExecOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, argsIn []s return cmdutil.UsageErrorf(cmd, execUsageStr) } if len(p.PodName) != 0 { - printDeprecationWarning(p.Err, "exec POD_NAME", "-p POD_NAME") + printDeprecationWarning(p.ErrOut, "exec POD_NAME", "-p POD_NAME") if len(argsIn) < 1 { return cmdutil.UsageErrorf(cmd, execUsageStr) } @@ -205,7 +203,7 @@ func (p *ExecOptions) Validate() error { if len(p.Command) == 0 { return fmt.Errorf("you must specify at least one command for the container") } - if p.Out == nil || p.Err == nil { + if p.Out == nil || p.ErrOut == nil { return fmt.Errorf("both output and error output must be provided") } if p.Executor == nil || p.PodClient == nil || p.Config == nil { @@ -240,8 +238,8 @@ func (o *StreamOptions) setupTTY() term.TTY { if !o.isTerminalIn(t) { o.TTY = false - if o.Err != nil { - fmt.Fprintln(o.Err, "Unable to use a TTY - input is not a terminal or the right kind of file") + if o.ErrOut != nil { + fmt.Fprintln(o.ErrOut, "Unable to use a TTY - input is not a terminal or the right kind of file") } return t @@ -284,7 +282,7 @@ func (p *ExecOptions) Run() error { if len(p.SuggestedCmdUsage) > 0 { usageString = fmt.Sprintf("%s\n%s", usageString, p.SuggestedCmdUsage) } - fmt.Fprintf(p.Err, "%s\n", usageString) + fmt.Fprintf(p.ErrOut, "%s\n", usageString) } containerName = pod.Spec.Containers[0].Name } @@ -299,7 +297,7 @@ func (p *ExecOptions) Run() error { // unset p.Err if it was previously set because both stdout and stderr go over p.Out when tty is // true - p.Err = nil + p.ErrOut = nil } fn := func() error { @@ -320,11 +318,11 @@ func (p *ExecOptions) Run() error { Command: p.Command, Stdin: p.Stdin, Stdout: p.Out != nil, - Stderr: p.Err != nil, + Stderr: p.ErrOut != nil, TTY: t.Raw, }, legacyscheme.ParameterCodec) - return p.Executor.Execute("POST", req.URL(), p.Config, p.In, p.Out, p.Err, t.Raw, sizeQueue) + return p.Executor.Execute("POST", req.URL(), p.Config, p.In, p.Out, p.ErrOut, t.Raw, sizeQueue) } if err := t.Safe(fn); err != nil { diff --git a/pkg/kubectl/cmd/exec_test.go b/pkg/kubectl/cmd/exec_test.go index 228da38d724..90c0e4be2ee 100644 --- a/pkg/kubectl/cmd/exec_test.go +++ b/pkg/kubectl/cmd/exec_test.go @@ -36,6 +36,7 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/scheme" "k8s.io/kubernetes/pkg/kubectl/util/term" ) @@ -145,7 +146,7 @@ func TestPodAndContainer(t *testing.T) { cmd := &cobra.Command{} options := test.p - options.Err = bytes.NewBuffer([]byte{}) + options.ErrOut = bytes.NewBuffer([]byte{}) err := options.Complete(tf, cmd, test.args, test.argsLenAtDash) if test.expectError && err == nil { t.Errorf("%s: unexpected non-error", test.name) @@ -195,7 +196,7 @@ func TestExec(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -213,9 +214,6 @@ func TestExec(t *testing.T) { } tf.Namespace = "test" tf.ClientConfigVal = defaultClientConfig() - bufOut := bytes.NewBuffer([]byte{}) - bufErr := bytes.NewBuffer([]byte{}) - bufIn := bytes.NewBuffer([]byte{}) ex := &fakeRemoteExecutor{} if test.execErr { ex.execErr = fmt.Errorf("exec error") @@ -224,9 +222,7 @@ func TestExec(t *testing.T) { StreamOptions: StreamOptions{ PodName: "foo", ContainerName: "bar", - In: bufIn, - Out: bufOut, - Err: bufErr, + IOStreams: genericclioptions.NewTestIOStreamsDiscard(), }, Executor: ex, } @@ -277,16 +273,14 @@ func execPod() *api.Pod { } func TestSetupTTY(t *testing.T) { - stderr := &bytes.Buffer{} + streams, _, _, stderr := genericclioptions.NewTestIOStreams() // test 1 - don't attach stdin o := &StreamOptions{ // InterruptParent: , - Stdin: false, - In: &bytes.Buffer{}, - Out: &bytes.Buffer{}, - Err: stderr, - TTY: true, + Stdin: false, + IOStreams: streams, + TTY: true, } tty := o.setupTTY() @@ -334,7 +328,7 @@ func TestSetupTTY(t *testing.T) { // test 3 - request a TTY, but stdin is not a terminal o.Stdin = true o.In = &bytes.Buffer{} - o.Err = stderr + o.ErrOut = stderr o.TTY = true tty = o.setupTTY() diff --git a/pkg/kubectl/cmd/expose.go b/pkg/kubectl/cmd/expose.go index 59f7bb70abd..18ae1589fbf 100644 --- a/pkg/kubectl/cmd/expose.go +++ b/pkg/kubectl/cmd/expose.go @@ -35,7 +35,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) @@ -110,7 +110,7 @@ type ExposeServiceOptions struct { func NewExposeServiceOptions(ioStreams genericclioptions.IOStreams) *ExposeServiceOptions { return &ExposeServiceOptions{ RecordFlags: genericclioptions.NewRecordFlags(), - PrintFlags: printers.NewPrintFlags("exposed"), + PrintFlags: printers.NewPrintFlags("exposed", legacyscheme.Scheme), Recorder: genericclioptions.NoopRecorder{}, IOStreams: ioStreams, @@ -136,8 +136,7 @@ func NewCmdExposeService(f cmdutil.Factory, streams genericclioptions.IOStreams) cmdutil.CheckErr(o.Complete(f, cmd)) cmdutil.CheckErr(o.RunExpose(cmd, args)) }, - ValidArgs: validArgs, - ArgAliases: kubectl.ResourceAliases(validArgs), + ValidArgs: validArgs, } o.RecordFlags.AddFlags(cmd) diff --git a/pkg/kubectl/cmd/expose_test.go b/pkg/kubectl/cmd/expose_test.go index 7b1b9d4af88..548a44ac5cb 100644 --- a/pkg/kubectl/cmd/expose_test.go +++ b/pkg/kubectl/cmd/expose_test.go @@ -470,7 +470,7 @@ func TestRunExposeService(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ diff --git a/pkg/kubectl/cmd/get/BUILD b/pkg/kubectl/cmd/get/BUILD index 010f663e883..2f0fcd3c506 100644 --- a/pkg/kubectl/cmd/get/BUILD +++ b/pkg/kubectl/cmd/get/BUILD @@ -31,7 +31,7 @@ go_library( "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/cmd/util/openapi:go_default_library", "//pkg/kubectl/genericclioptions:go_default_library", - "//pkg/kubectl/resource:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/scheme:go_default_library", "//pkg/kubectl/util/i18n:go_default_library", "//pkg/printers:go_default_library", diff --git a/pkg/kubectl/cmd/get/get.go b/pkg/kubectl/cmd/get/get.go index d84c818b04c..8ecbe7fcb7b 100644 --- a/pkg/kubectl/cmd/get/get.go +++ b/pkg/kubectl/cmd/get/get.go @@ -45,7 +45,7 @@ import ( cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" "k8s.io/kubernetes/pkg/util/interrupt" @@ -136,7 +136,7 @@ const ( // NewGetOptions returns a GetOptions with default chunk size 500. func NewGetOptions(parent string, streams genericclioptions.IOStreams) *GetOptions { return &GetOptions{ - PrintFlags: NewGetPrintFlags(), + PrintFlags: NewGetPrintFlags(legacyscheme.Scheme), CmdParent: parent, IOStreams: streams, diff --git a/pkg/kubectl/cmd/get/get_flags.go b/pkg/kubectl/cmd/get/get_flags.go index 4349e79d069..dedb1792d65 100644 --- a/pkg/kubectl/cmd/get/get_flags.go +++ b/pkg/kubectl/cmd/get/get_flags.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi" "k8s.io/kubernetes/pkg/printers" @@ -152,7 +153,7 @@ func (f *PrintFlags) AddFlags(cmd *cobra.Command) { // NewGetPrintFlags returns flags associated with humanreadable, // template, and "name" printing, with default values set. -func NewGetPrintFlags() *PrintFlags { +func NewGetPrintFlags(scheme runtime.ObjectConvertor) *PrintFlags { outputFormat := "" noHeaders := false @@ -160,8 +161,8 @@ func NewGetPrintFlags() *PrintFlags { OutputFormat: &outputFormat, NoHeaders: &noHeaders, - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(scheme), + NamePrintFlags: printers.NewNamePrintFlags("", scheme), TemplateFlags: printers.NewKubeTemplatePrintFlags(), HumanReadableFlags: NewHumanPrintFlags(), CustomColumnsFlags: printers.NewCustomColumnsPrintFlags(), diff --git a/pkg/kubectl/cmd/get/get_test.go b/pkg/kubectl/cmd/get/get_test.go index 28837df82be..d63b8bf03cd 100644 --- a/pkg/kubectl/cmd/get/get_test.go +++ b/pkg/kubectl/cmd/get/get_test.go @@ -75,7 +75,7 @@ func defaultClientConfig() *restclient.Config { ContentConfig: restclient.ContentConfig{ NegotiatedSerializer: scheme.Codecs, ContentType: runtime.ContentTypeJSON, - GroupVersion: &scheme.Registry.GroupOrDie(api.GroupName).GroupVersions[0], + GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}, }, } } @@ -251,7 +251,7 @@ func TestGetObjectsWithOpenAPIOutputFormatPresent(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) // overide the openAPISchema function to return custom output // for Pod type. @@ -308,7 +308,7 @@ func TestGetObjects(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -346,7 +346,7 @@ func TestGetObjectIgnoreNotFound(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -399,7 +399,7 @@ func TestGetSortedObjects(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -431,7 +431,7 @@ func TestGetObjectsIdentifiedByFile(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -458,7 +458,7 @@ func TestGetListObjects(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -485,7 +485,7 @@ func TestGetAllListObjects(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -512,7 +512,7 @@ func TestGetListComponentStatus(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -553,7 +553,7 @@ func TestGetMixedGenericObjects(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -603,7 +603,7 @@ func TestGetMultipleTypeObjects(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -642,7 +642,7 @@ func TestGetMultipleTypeObjectsAsList(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -742,7 +742,7 @@ func TestGetMultipleTypeObjectsWithLabelSelector(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -786,7 +786,7 @@ func TestGetMultipleTypeObjectsWithFieldSelector(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -835,7 +835,7 @@ func TestGetMultipleTypeObjectsWithDirectReference(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -944,7 +944,7 @@ func TestWatchLabelSelector(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) podList := &api.PodList{ Items: pods, @@ -996,7 +996,7 @@ func TestWatchFieldSelector(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) podList := &api.PodList{ Items: pods, @@ -1048,7 +1048,7 @@ func TestWatchResource(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -1092,7 +1092,7 @@ func TestWatchResourceIdentifiedByFile(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -1137,7 +1137,7 @@ func TestWatchOnlyResource(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -1180,7 +1180,7 @@ func TestWatchOnlyList(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) podList := &api.PodList{ Items: pods, diff --git a/pkg/kubectl/cmd/label.go b/pkg/kubectl/cmd/label.go index 7e35218581b..d1abca47783 100644 --- a/pkg/kubectl/cmd/label.go +++ b/pkg/kubectl/cmd/label.go @@ -32,12 +32,13 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) @@ -114,7 +115,7 @@ func NewLabelOptions(ioStreams genericclioptions.IOStreams) *LabelOptions { RecordFlags: genericclioptions.NewRecordFlags(), Recorder: genericclioptions.NoopRecorder{}, - PrintFlags: printers.NewPrintFlags("labeled"), + PrintFlags: printers.NewPrintFlags("labeled", legacyscheme.Scheme), IOStreams: ioStreams, } diff --git a/pkg/kubectl/cmd/label_test.go b/pkg/kubectl/cmd/label_test.go index f609ddb7e1f..4bb226fa81d 100644 --- a/pkg/kubectl/cmd/label_test.go +++ b/pkg/kubectl/cmd/label_test.go @@ -360,7 +360,7 @@ func TestLabelForResourceFromFile(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -449,7 +449,7 @@ func TestLabelMultipleObjects(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, diff --git a/pkg/kubectl/cmd/logs.go b/pkg/kubectl/cmd/logs.go index 7b07806b04f..7faf0cd3ebd 100644 --- a/pkg/kubectl/cmd/logs.go +++ b/pkg/kubectl/cmd/logs.go @@ -35,6 +35,7 @@ import ( "k8s.io/kubernetes/pkg/apis/core/validation" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/util" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -89,12 +90,19 @@ type LogsOptions struct { GetPodTimeout time.Duration LogsForObject func(object, options runtime.Object, timeout time.Duration) (*restclient.Request, error) - Out io.Writer + genericclioptions.IOStreams +} + +func NewLogsOptions(streams genericclioptions.IOStreams) *LogsOptions { + return &LogsOptions{ + IOStreams: streams, + } } // NewCmdLogs creates a new pod logs command -func NewCmdLogs(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { - o := &LogsOptions{} +func NewCmdLogs(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewLogsOptions(streams) + cmd := &cobra.Command{ Use: "logs [-f] [-p] (POD | TYPE/NAME) [-c CONTAINER]", DisableFlagsInUseLine: true, @@ -103,11 +111,11 @@ func NewCmdLogs(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { Example: logsExample, PreRun: func(cmd *cobra.Command, args []string) { if len(os.Args) > 1 && os.Args[1] == "log" { - printDeprecationWarning(errOut, "logs", "log") + printDeprecationWarning(o.ErrOut, "logs", "log") } }, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(o.Complete(f, out, cmd, args)) + cmdutil.CheckErr(o.Complete(f, cmd, args)) cmdutil.CheckErr(o.Validate()) cmdutil.CheckErr(o.RunLogs()) }, @@ -129,7 +137,7 @@ func NewCmdLogs(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { return cmd } -func (o *LogsOptions) Complete(f cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string) error { +func (o *LogsOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { containerName := cmdutil.GetFlagString(cmd, "container") selector := cmdutil.GetFlagString(cmd, "selector") o.AllContainers = cmdutil.GetFlagBool(cmd, "all-containers") @@ -189,7 +197,6 @@ func (o *LogsOptions) Complete(f cmdutil.Factory, out io.Writer, cmd *cobra.Comm } o.Options = logOptions o.LogsForObject = f.LogsForObject - o.Out = out if len(selector) != 0 { if logOptions.Follow { diff --git a/pkg/kubectl/cmd/logs_test.go b/pkg/kubectl/cmd/logs_test.go index 1a7e5e359e1..09a89cca1ee 100644 --- a/pkg/kubectl/cmd/logs_test.go +++ b/pkg/kubectl/cmd/logs_test.go @@ -20,7 +20,6 @@ import ( "bytes" "io/ioutil" "net/http" - "os" "strings" "testing" @@ -31,6 +30,7 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/scheme" ) @@ -53,7 +53,7 @@ func TestLog(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -74,9 +74,9 @@ func TestLog(t *testing.T) { } tf.Namespace = "test" tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdLogs(tf, buf, buf) + cmd := NewCmdLogs(tf, streams) cmd.Flags().Set("namespace", "test") cmd.Run(cmd, []string{"foo"}) @@ -144,17 +144,17 @@ func TestValidateLogFlags(t *testing.T) { }, } for _, test := range tests { - buf := bytes.NewBuffer([]byte{}) - cmd := NewCmdLogs(f, buf, buf) + streams := genericclioptions.NewTestIOStreamsDiscard() + cmd := NewCmdLogs(f, streams) out := "" for flag, value := range test.flags { cmd.Flags().Set(flag, value) } // checkErr breaks tests in case of errors, plus we just // need to check errors returned by the command validation - o := &LogsOptions{} + o := NewLogsOptions(streams) cmd.Run = func(cmd *cobra.Command, args []string) { - o.Complete(f, os.Stdout, cmd, args) + o.Complete(f, cmd, args) out = o.Validate().Error() } cmd.Run(cmd, test.args) @@ -205,8 +205,7 @@ func TestLogComplete(t *testing.T) { }, } for _, test := range tests { - buf := bytes.NewBuffer([]byte{}) - cmd := NewCmdLogs(f, buf, buf) + cmd := NewCmdLogs(f, genericclioptions.NewTestIOStreamsDiscard()) var err error out := "" for flag, value := range test.flags { @@ -214,8 +213,8 @@ func TestLogComplete(t *testing.T) { } // checkErr breaks tests in case of errors, plus we just // need to check errors returned by the command validation - o := &LogsOptions{} - err = o.Complete(f, os.Stdout, cmd, test.args) + o := NewLogsOptions(genericclioptions.NewTestIOStreamsDiscard()) + err = o.Complete(f, cmd, test.args) out = err.Error() if !strings.Contains(out, test.expected) { t.Errorf("%s: expected to find:\n\t%s\nfound:\n\t%s\n", test.name, test.expected, out) diff --git a/pkg/kubectl/cmd/patch.go b/pkg/kubectl/cmd/patch.go index 19a06a747e9..47f0a9127b2 100644 --- a/pkg/kubectl/cmd/patch.go +++ b/pkg/kubectl/cmd/patch.go @@ -33,10 +33,11 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" @@ -98,7 +99,7 @@ func NewPatchOptions(ioStreams genericclioptions.IOStreams) *PatchOptions { return &PatchOptions{ RecordFlags: genericclioptions.NewRecordFlags(), Recorder: genericclioptions.NoopRecorder{}, - PrintFlags: printers.NewPrintFlags("patched"), + PrintFlags: printers.NewPrintFlags("patched", legacyscheme.Scheme), IOStreams: ioStreams, } } diff --git a/pkg/kubectl/cmd/patch_test.go b/pkg/kubectl/cmd/patch_test.go index 0ed892c0c6d..1311f188439 100644 --- a/pkg/kubectl/cmd/patch_test.go +++ b/pkg/kubectl/cmd/patch_test.go @@ -34,7 +34,7 @@ func TestPatchObject(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -76,7 +76,7 @@ func TestPatchObjectFromFile(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -114,7 +114,7 @@ func TestPatchNoop(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -162,7 +162,7 @@ func TestPatchObjectFromFileOutput(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, diff --git a/pkg/kubectl/cmd/plugin.go b/pkg/kubectl/cmd/plugin.go index d34e78733a4..12d621ff3ef 100644 --- a/pkg/kubectl/cmd/plugin.go +++ b/pkg/kubectl/cmd/plugin.go @@ -18,7 +18,6 @@ package cmd import ( "fmt" - "io" "os" "os/exec" "syscall" @@ -28,6 +27,7 @@ import ( "github.com/spf13/pflag" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/plugins" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -42,7 +42,7 @@ var ( ) // NewCmdPlugin creates the command that is the top-level for plugin commands. -func NewCmdPlugin(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cobra.Command { +func NewCmdPlugin(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { // Loads plugins and create commands for each plugin identified loadedPlugins, loadErr := f.PluginLoader().Load() if loadErr != nil { @@ -58,14 +58,14 @@ func NewCmdPlugin(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cobra.Co if len(loadedPlugins) == 0 { cmdutil.CheckErr(fmt.Errorf("no plugins installed.")) } - cmdutil.DefaultSubCommandRun(err)(cmd, args) + cmdutil.DefaultSubCommandRun(streams.ErrOut)(cmd, args) }, } if len(loadedPlugins) > 0 { pluginRunner := f.PluginRunner() for _, p := range loadedPlugins { - cmd.AddCommand(NewCmdForPlugin(f, p, pluginRunner, in, out, err)) + cmd.AddCommand(NewCmdForPlugin(f, p, pluginRunner, streams)) } } @@ -73,7 +73,7 @@ func NewCmdPlugin(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cobra.Co } // NewCmdForPlugin creates a command capable of running the provided plugin. -func NewCmdForPlugin(f cmdutil.Factory, plugin *plugins.Plugin, runner plugins.PluginRunner, in io.Reader, out, errout io.Writer) *cobra.Command { +func NewCmdForPlugin(f cmdutil.Factory, plugin *plugins.Plugin, runner plugins.PluginRunner, streams genericclioptions.IOStreams) *cobra.Command { if !plugin.IsValid() { return nil } @@ -85,7 +85,7 @@ func NewCmdForPlugin(f cmdutil.Factory, plugin *plugins.Plugin, runner plugins.P Example: templates.Examples(plugin.Example), Run: func(cmd *cobra.Command, args []string) { if len(plugin.Command) == 0 { - cmdutil.DefaultSubCommandRun(errout)(cmd, args) + cmdutil.DefaultSubCommandRun(streams.ErrOut)(cmd, args) return } @@ -104,9 +104,7 @@ func NewCmdForPlugin(f cmdutil.Factory, plugin *plugins.Plugin, runner plugins.P } runningContext := plugins.RunningContext{ - In: in, - Out: out, - ErrOut: errout, + IOStreams: streams, Args: args, EnvProvider: envProvider, WorkingDir: plugin.Dir, @@ -117,7 +115,7 @@ func NewCmdForPlugin(f cmdutil.Factory, plugin *plugins.Plugin, runner plugins.P // check for (and exit with) the correct exit code // from a failed plugin command execution if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { - fmt.Fprintf(errout, "error: %v\n", err) + fmt.Fprintf(streams.ErrOut, "error: %v\n", err) os.Exit(status.ExitStatus()) } } @@ -132,7 +130,7 @@ func NewCmdForPlugin(f cmdutil.Factory, plugin *plugins.Plugin, runner plugins.P } for _, childPlugin := range plugin.Tree { - cmd.AddCommand(NewCmdForPlugin(f, childPlugin, runner, in, out, errout)) + cmd.AddCommand(NewCmdForPlugin(f, childPlugin, runner, streams)) } return cmd diff --git a/pkg/kubectl/cmd/plugin_test.go b/pkg/kubectl/cmd/plugin_test.go index 205b266743f..c47646dd8b4 100644 --- a/pkg/kubectl/cmd/plugin_test.go +++ b/pkg/kubectl/cmd/plugin_test.go @@ -17,12 +17,12 @@ limitations under the License. package cmd import ( - "bytes" "fmt" "testing" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/plugins" ) @@ -81,9 +81,7 @@ func TestPluginCmd(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - inBuf := bytes.NewBuffer([]byte{}) - outBuf := bytes.NewBuffer([]byte{}) - errBuf := bytes.NewBuffer([]byte{}) + streams, _, outBuf, errBuf := genericclioptions.NewTestIOStreams() cmdutil.BehaviorOnFatal(func(str string, code int) { errBuf.Write([]byte(str)) @@ -96,7 +94,7 @@ func TestPluginCmd(t *testing.T) { f := cmdtesting.NewTestFactory() defer f.Cleanup() - cmd := NewCmdForPlugin(f, test.plugin, runner, inBuf, outBuf, errBuf) + cmd := NewCmdForPlugin(f, test.plugin, runner, streams) if cmd == nil { if !test.expectedNilCmd { t.Fatalf("%s: command was unexpectedly not registered", test.name) diff --git a/pkg/kubectl/cmd/portforward.go b/pkg/kubectl/cmd/portforward.go index 5f0043104a4..59f0af076f8 100644 --- a/pkg/kubectl/cmd/portforward.go +++ b/pkg/kubectl/cmd/portforward.go @@ -18,7 +18,6 @@ package cmd import ( "fmt" - "io" "net/http" "net/url" "os" @@ -38,6 +37,7 @@ import ( coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/util" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -84,11 +84,10 @@ const ( defaultPodPortForwardWaitTimeout = 60 * time.Second ) -func NewCmdPortForward(f cmdutil.Factory, cmdOut, cmdErr io.Writer) *cobra.Command { +func NewCmdPortForward(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { opts := &PortForwardOptions{ PortForwarder: &defaultPortForwarder{ - cmdOut: cmdOut, - cmdErr: cmdErr, + IOStreams: streams, }, } cmd := &cobra.Command{ @@ -119,7 +118,7 @@ type portForwarder interface { } type defaultPortForwarder struct { - cmdOut, cmdErr io.Writer + genericclioptions.IOStreams } func (f *defaultPortForwarder) ForwardPorts(method string, url *url.URL, opts PortForwardOptions) error { @@ -128,7 +127,7 @@ func (f *defaultPortForwarder) ForwardPorts(method string, url *url.URL, opts Po return err } dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, method, url) - fw, err := portforward.New(dialer, opts.Ports, opts.StopChannel, opts.ReadyChannel, f.cmdOut, f.cmdErr) + fw, err := portforward.New(dialer, opts.Ports, opts.StopChannel, opts.ReadyChannel, f.Out, f.ErrOut) if err != nil { return err } diff --git a/pkg/kubectl/cmd/portforward_test.go b/pkg/kubectl/cmd/portforward_test.go index 24071ca253f..87642e1c853 100644 --- a/pkg/kubectl/cmd/portforward_test.go +++ b/pkg/kubectl/cmd/portforward_test.go @@ -20,7 +20,6 @@ import ( "fmt" "net/http" "net/url" - "os" "reflect" "testing" @@ -32,6 +31,7 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/scheme" ) @@ -76,7 +76,7 @@ func testPortForward(t *testing.T, flags map[string]string, args []string) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -102,7 +102,7 @@ func testPortForward(t *testing.T, flags map[string]string, args []string) { } opts := &PortForwardOptions{} - cmd := NewCmdPortForward(tf, os.Stdout, os.Stderr) + cmd := NewCmdPortForward(tf, genericclioptions.NewTestIOStreamsDiscard()) cmd.Run = func(cmd *cobra.Command, args []string) { if err = opts.Complete(tf, cmd, args); err != nil { return diff --git a/pkg/kubectl/cmd/proxy.go b/pkg/kubectl/cmd/proxy.go index 027dfbf1ee8..440c8c867f4 100644 --- a/pkg/kubectl/cmd/proxy.go +++ b/pkg/kubectl/cmd/proxy.go @@ -28,6 +28,7 @@ import ( "github.com/spf13/cobra" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/proxy" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -69,7 +70,7 @@ var ( kubectl proxy --api-prefix=/k8s-api`)) ) -func NewCmdProxy(f cmdutil.Factory, out io.Writer) *cobra.Command { +func NewCmdProxy(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { cmd := &cobra.Command{ Use: "proxy [--port=PORT] [--www=static-dir] [--www-prefix=prefix] [--api-prefix=prefix]", DisableFlagsInUseLine: true, @@ -77,7 +78,7 @@ func NewCmdProxy(f cmdutil.Factory, out io.Writer) *cobra.Command { Long: proxyLong, Example: proxyExample, Run: func(cmd *cobra.Command, args []string) { - err := RunProxy(f, out, cmd) + err := RunProxy(f, streams.Out, cmd) cmdutil.CheckErr(err) }, } diff --git a/pkg/kubectl/cmd/replace.go b/pkg/kubectl/cmd/replace.go index 8e4e4aa53f9..86390351352 100644 --- a/pkg/kubectl/cmd/replace.go +++ b/pkg/kubectl/cmd/replace.go @@ -18,7 +18,6 @@ package cmd import ( "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -30,11 +29,12 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/kubectl/validation" "k8s.io/kubernetes/pkg/printers" @@ -86,22 +86,27 @@ type ReplaceOptions struct { Recorder genericclioptions.Recorder - Out io.Writer - ErrOut io.Writer + genericclioptions.IOStreams } -func NewReplaceOptions(out, errOut io.Writer) *ReplaceOptions { +func NewReplaceOptions(streams genericclioptions.IOStreams) *ReplaceOptions { + outputFormat := "" + return &ReplaceOptions{ - PrintFlags: printers.NewPrintFlags("replaced"), + // TODO(juanvallejo): figure out why we only support the "name" outputFormat in this command + // we only support "-o name" for this command, so only register the name printer + PrintFlags: &printers.PrintFlags{ + OutputFormat: &outputFormat, + NamePrintFlags: printers.NewNamePrintFlags("replaced", legacyscheme.Scheme), + }, DeleteFlags: NewDeleteFlags("to use to replace the resource."), - Out: out, - ErrOut: errOut, + IOStreams: streams, } } -func NewCmdReplace(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { - o := NewReplaceOptions(out, errOut) +func NewCmdReplace(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewReplaceOptions(streams) cmd := &cobra.Command{ Use: "replace -f FILENAME", @@ -110,7 +115,6 @@ func NewCmdReplace(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { Long: replaceLong, Example: replaceExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(cmdutil.ValidateOutputArgs(cmd)) cmdutil.CheckErr(o.Complete(f, cmd, args)) cmdutil.CheckErr(o.Validate(cmd)) cmdutil.CheckErr(o.Run()) @@ -148,7 +152,7 @@ func (o *ReplaceOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args [] return printer.PrintObj(obj, o.Out) } - deleteOpts := o.DeleteFlags.ToOptions(o.Out, o.ErrOut) + deleteOpts := o.DeleteFlags.ToOptions(o.IOStreams) //Replace will create a resource if it doesn't exist already, so ignore not found error deleteOpts.IgnoreNotFound = true diff --git a/pkg/kubectl/cmd/replace_test.go b/pkg/kubectl/cmd/replace_test.go index 2789967784e..e40bde5148c 100644 --- a/pkg/kubectl/cmd/replace_test.go +++ b/pkg/kubectl/cmd/replace_test.go @@ -17,7 +17,6 @@ limitations under the License. package cmd import ( - "bytes" "net/http" "strings" "testing" @@ -26,6 +25,7 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/scheme" ) @@ -34,7 +34,7 @@ func TestReplaceObject(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) deleted := false tf.UnstructuredClient = &fake.RESTClient{ @@ -63,9 +63,9 @@ func TestReplaceObject(t *testing.T) { }), } tf.Namespace = "test" - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdReplace(tf, buf, buf) + cmd := NewCmdReplace(tf, streams) cmd.Flags().Set("filename", "../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml") cmd.Flags().Set("output", "name") cmd.Run(cmd, []string{}) @@ -91,7 +91,7 @@ func TestReplaceMultipleObject(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) redisMasterDeleted := false frontendDeleted := false @@ -134,9 +134,9 @@ func TestReplaceMultipleObject(t *testing.T) { }), } tf.Namespace = "test" - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdReplace(tf, buf, buf) + cmd := NewCmdReplace(tf, streams) cmd.Flags().Set("filename", "../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml") cmd.Flags().Set("filename", "../../../test/e2e/testing-manifests/guestbook/frontend-service.yaml") cmd.Flags().Set("output", "name") @@ -162,7 +162,7 @@ func TestReplaceDirectory(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) created := map[string]bool{} tf.UnstructuredClient = &fake.RESTClient{ @@ -192,9 +192,9 @@ func TestReplaceDirectory(t *testing.T) { }), } tf.Namespace = "test" - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdReplace(tf, buf, buf) + cmd := NewCmdReplace(tf, streams) cmd.Flags().Set("filename", "../../../test/e2e/testing-manifests/guestbook/legacy") cmd.Flags().Set("namespace", "test") cmd.Flags().Set("output", "name") @@ -220,7 +220,7 @@ func TestForceReplaceObjectNotFound(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) tf.UnstructuredClient = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, @@ -239,9 +239,9 @@ func TestForceReplaceObjectNotFound(t *testing.T) { }), } tf.Namespace = "test" - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdReplace(tf, buf, buf) + cmd := NewCmdReplace(tf, streams) cmd.Flags().Set("filename", "../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml") cmd.Flags().Set("force", "true") cmd.Flags().Set("cascade", "false") diff --git a/pkg/kubectl/cmd/rollingupdate.go b/pkg/kubectl/cmd/rollingupdate.go index 99b98fc0199..462b3ce6c3f 100644 --- a/pkg/kubectl/cmd/rollingupdate.go +++ b/pkg/kubectl/cmd/rollingupdate.go @@ -37,7 +37,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/kubectl/validation" @@ -114,7 +114,7 @@ type RollingUpdateOptions struct { func NewRollingUpdateOptions(streams genericclioptions.IOStreams) *RollingUpdateOptions { return &RollingUpdateOptions{ - PrintFlags: printers.NewPrintFlags("rolling updated"), + PrintFlags: printers.NewPrintFlags("rolling updated", legacyscheme.Scheme), FilenameOptions: &resource.FilenameOptions{}, DeploymentKey: "deployment", Timeout: timeout, diff --git a/pkg/kubectl/cmd/rollout/BUILD b/pkg/kubectl/cmd/rollout/BUILD index 6a2e546f642..a49016ade42 100644 --- a/pkg/kubectl/cmd/rollout/BUILD +++ b/pkg/kubectl/cmd/rollout/BUILD @@ -25,7 +25,7 @@ go_library( "//pkg/kubectl/cmd/templates:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/genericclioptions:go_default_library", - "//pkg/kubectl/resource:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/util/i18n:go_default_library", "//pkg/printers:go_default_library", "//pkg/util/interrupt:go_default_library", diff --git a/pkg/kubectl/cmd/rollout/rollout_history.go b/pkg/kubectl/cmd/rollout/rollout_history.go index e42f3b8a4fb..7e381ef583c 100644 --- a/pkg/kubectl/cmd/rollout/rollout_history.go +++ b/pkg/kubectl/cmd/rollout/rollout_history.go @@ -21,10 +21,9 @@ import ( "io" "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "github.com/spf13/cobra" @@ -46,7 +45,6 @@ func NewCmdRolloutHistory(f cmdutil.Factory, out io.Writer) *cobra.Command { options := &resource.FilenameOptions{} validArgs := []string{"deployment", "daemonset", "statefulset"} - argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ Use: "history (TYPE NAME | TYPE/NAME) [flags]", @@ -57,8 +55,7 @@ func NewCmdRolloutHistory(f cmdutil.Factory, out io.Writer) *cobra.Command { Run: func(cmd *cobra.Command, args []string) { cmdutil.CheckErr(RunHistory(f, cmd, out, args, options)) }, - ValidArgs: validArgs, - ArgAliases: argAliases, + ValidArgs: validArgs, } cmd.Flags().Int64("revision", 0, "See the details, including podTemplate of the revision specified") diff --git a/pkg/kubectl/cmd/rollout/rollout_pause.go b/pkg/kubectl/cmd/rollout/rollout_pause.go index eb4a4b1662e..d288598bb40 100644 --- a/pkg/kubectl/cmd/rollout/rollout_pause.go +++ b/pkg/kubectl/cmd/rollout/rollout_pause.go @@ -24,12 +24,11 @@ import ( "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/set" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) @@ -64,12 +63,11 @@ var ( func NewCmdRolloutPause(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &PauseConfig{ - PrintFlags: printers.NewPrintFlags("paused"), + PrintFlags: printers.NewPrintFlags("paused", legacyscheme.Scheme), IOStreams: streams, } validArgs := []string{"deployment"} - argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ Use: "pause RESOURCE", @@ -89,8 +87,7 @@ func NewCmdRolloutPause(f cmdutil.Factory, streams genericclioptions.IOStreams) } cmdutil.CheckErr(utilerrors.Flatten(utilerrors.NewAggregate(allErrs))) }, - ValidArgs: validArgs, - ArgAliases: argAliases, + ValidArgs: validArgs, } usage := "identifying the resource to get from a server." diff --git a/pkg/kubectl/cmd/rollout/rollout_resume.go b/pkg/kubectl/cmd/rollout/rollout_resume.go index ee28ef9db93..4e512dfabab 100644 --- a/pkg/kubectl/cmd/rollout/rollout_resume.go +++ b/pkg/kubectl/cmd/rollout/rollout_resume.go @@ -24,12 +24,11 @@ import ( "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/set" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) @@ -62,12 +61,11 @@ var ( func NewCmdRolloutResume(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &ResumeConfig{ - PrintFlags: printers.NewPrintFlags("resumed"), + PrintFlags: printers.NewPrintFlags("resumed", legacyscheme.Scheme), IOStreams: streams, } validArgs := []string{"deployment"} - argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ Use: "resume RESOURCE", @@ -87,8 +85,7 @@ func NewCmdRolloutResume(f cmdutil.Factory, streams genericclioptions.IOStreams) } cmdutil.CheckErr(utilerrors.Flatten(utilerrors.NewAggregate(allErrs))) }, - ValidArgs: validArgs, - ArgAliases: argAliases, + ValidArgs: validArgs, } usage := "identifying the resource to get from a server." diff --git a/pkg/kubectl/cmd/rollout/rollout_status.go b/pkg/kubectl/cmd/rollout/rollout_status.go index e6e46a44220..930c85a587e 100644 --- a/pkg/kubectl/cmd/rollout/rollout_status.go +++ b/pkg/kubectl/cmd/rollout/rollout_status.go @@ -26,7 +26,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/util/interrupt" @@ -77,7 +77,6 @@ func NewCmdRolloutStatus(f cmdutil.Factory, streams genericclioptions.IOStreams) o := NewRolloutStatusOptions(streams) validArgs := []string{"deployment", "daemonset", "statefulset"} - argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ Use: "status (TYPE NAME | TYPE/NAME) [flags]", @@ -90,8 +89,7 @@ func NewCmdRolloutStatus(f cmdutil.Factory, streams genericclioptions.IOStreams) cmdutil.CheckErr(o.Validate(cmd, args)) cmdutil.CheckErr(o.Run()) }, - ValidArgs: validArgs, - ArgAliases: argAliases, + ValidArgs: validArgs, } usage := "identifying the resource to get from a server." diff --git a/pkg/kubectl/cmd/rollout/rollout_undo.go b/pkg/kubectl/cmd/rollout/rollout_undo.go index b8c07afc4b1..2a0d6c9fc71 100644 --- a/pkg/kubectl/cmd/rollout/rollout_undo.go +++ b/pkg/kubectl/cmd/rollout/rollout_undo.go @@ -19,16 +19,16 @@ package rollout import ( "io" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" - - "github.com/spf13/cobra" ) // UndoOptions is the start of the data required to perform the operation. As new fields are added, add them here instead of @@ -64,11 +64,10 @@ var ( func NewCmdRolloutUndo(f cmdutil.Factory, out io.Writer) *cobra.Command { o := &UndoOptions{ - PrintFlags: printers.NewPrintFlags(""), + PrintFlags: printers.NewPrintFlags("", legacyscheme.Scheme), } validArgs := []string{"deployment", "daemonset", "statefulset"} - argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ Use: "undo (TYPE NAME | TYPE/NAME) [flags]", @@ -88,8 +87,7 @@ func NewCmdRolloutUndo(f cmdutil.Factory, out io.Writer) *cobra.Command { } cmdutil.CheckErr(utilerrors.Flatten(utilerrors.NewAggregate(allErrs))) }, - ValidArgs: validArgs, - ArgAliases: argAliases, + ValidArgs: validArgs, } cmd.Flags().Int64("to-revision", 0, "The revision to rollback to. Default to 0 (last revision).") diff --git a/pkg/kubectl/cmd/run.go b/pkg/kubectl/cmd/run.go index 92eeabe89d5..8e512d6cadb 100644 --- a/pkg/kubectl/cmd/run.go +++ b/pkg/kubectl/cmd/run.go @@ -40,7 +40,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/util/interrupt" @@ -126,7 +126,7 @@ type RunOptions struct { func NewRunOptions(streams genericclioptions.IOStreams) *RunOptions { return &RunOptions{ - PrintFlags: printers.NewPrintFlags("created"), + PrintFlags: printers.NewPrintFlags("created", legacyscheme.Scheme), DeleteFlags: NewDeleteFlags("to use to replace the resource."), RecordFlags: genericclioptions.NewRecordFlags(), @@ -233,7 +233,7 @@ func (o *RunOptions) Complete(f cmdutil.Factory, cmd *cobra.Command) error { return printer.PrintObj(obj, o.Out) } - deleteOpts := o.DeleteFlags.ToOptions(o.Out, o.ErrOut) + deleteOpts := o.DeleteFlags.ToOptions(o.IOStreams) deleteOpts.IgnoreNotFound = true deleteOpts.WaitForDeletion = false deleteOpts.GracePeriod = -1 @@ -374,12 +374,10 @@ func (o *RunOptions) Run(f cmdutil.Factory, cmd *cobra.Command, args []string) e opts := &AttachOptions{ StreamOptions: StreamOptions{ - In: o.In, - Out: o.Out, - Err: o.ErrOut, - Stdin: o.Interactive, - TTY: o.TTY, - Quiet: o.Quiet, + IOStreams: o.IOStreams, + Stdin: o.Interactive, + TTY: o.TTY, + Quiet: o.Quiet, }, GetPodTimeout: timeout, CommandName: cmd.Parent().CommandPath() + " attach", @@ -528,7 +526,7 @@ func handleAttachPod(f cmdutil.Factory, podClient coreclient.PodsGetter, ns, nam opts.Namespace = ns // TODO: opts.Run sets opts.Err to nil, we need to find a better way - stderr := opts.Err + stderr := opts.ErrOut if err := opts.Run(); err != nil { fmt.Fprintf(stderr, "Error attaching, falling back to logs: %v\n", err) return logOpts(f, pod, opts) @@ -665,7 +663,7 @@ func (o *RunOptions) createGeneratedObject(f cmdutil.Factory, cmd *cobra.Command } if len(overrides) > 0 { - codec := runtime.NewCodec(scheme.DefaultJSONEncoder(), scheme.Codecs.UniversalDecoder(scheme.Registry.RegisteredGroupVersions()...)) + codec := runtime.NewCodec(scheme.DefaultJSONEncoder(), scheme.Codecs.UniversalDecoder(scheme.Scheme.PrioritizedVersionsAllGroups()...)) obj, err = cmdutil.Merge(codec, obj, overrides) if err != nil { return nil, err diff --git a/pkg/kubectl/cmd/run_test.go b/pkg/kubectl/cmd/run_test.go index 6d65d0d5d07..77bd7a11f34 100644 --- a/pkg/kubectl/cmd/run_test.go +++ b/pkg/kubectl/cmd/run_test.go @@ -174,7 +174,7 @@ func TestRunArgsFollowDashRules(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -198,7 +198,7 @@ func TestRunArgsFollowDashRules(t *testing.T) { cmd.Flags().Set("image", "nginx") cmd.Flags().Set("generator", "run/v1") - printFlags := printers.NewPrintFlags("created") + printFlags := printers.NewPrintFlags("created", legacyscheme.Scheme) printer, err := printFlags.ToPrinter() if err != nil { t.Errorf("unexpected error: %v", err) @@ -208,7 +208,7 @@ func TestRunArgsFollowDashRules(t *testing.T) { deleteFlags := NewDeleteFlags("to use to replace the resource.") opts := &RunOptions{ PrintFlags: printFlags, - DeleteOptions: deleteFlags.ToOptions(os.Stdout, os.Stderr), + DeleteOptions: deleteFlags.ToOptions(genericclioptions.NewTestIOStreamsDiscard()), IOStreams: genericclioptions.NewTestIOStreamsDiscard(), @@ -331,7 +331,7 @@ func TestGenerateService(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.ClientConfigVal = defaultClientConfig() @@ -366,7 +366,7 @@ func TestGenerateService(t *testing.T) { }), } - printFlags := printers.NewPrintFlags("created") + printFlags := printers.NewPrintFlags("created", legacyscheme.Scheme) printer, err := printFlags.ToPrinter() if err != nil { t.Errorf("unexpected error: %v", err) @@ -377,7 +377,7 @@ func TestGenerateService(t *testing.T) { deleteFlags := NewDeleteFlags("to use to replace the resource.") opts := &RunOptions{ PrintFlags: printFlags, - DeleteOptions: deleteFlags.ToOptions(os.Stdout, os.Stderr), + DeleteOptions: deleteFlags.ToOptions(genericclioptions.NewTestIOStreamsDiscard()), IOStreams: ioStreams, diff --git a/pkg/kubectl/cmd/scale.go b/pkg/kubectl/cmd/scale.go index b1e710cd240..0823f273399 100644 --- a/pkg/kubectl/cmd/scale.go +++ b/pkg/kubectl/cmd/scale.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" batchclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/batch/internalversion" "k8s.io/kubernetes/pkg/kubectl" @@ -33,7 +34,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" ) @@ -93,9 +94,16 @@ type ScaleOptions struct { } func NewScaleOptions(ioStreams genericclioptions.IOStreams) *ScaleOptions { + outputFormat := "" + return &ScaleOptions{ + // TODO(juanvallejo): figure out why we only support the "name" outputFormat in this command + // we only support "-o name" for this command, so only register the name printer + PrintFlags: &printers.PrintFlags{ + OutputFormat: &outputFormat, + NamePrintFlags: printers.NewNamePrintFlags("scaled", legacyscheme.Scheme), + }, RecordFlags: genericclioptions.NewRecordFlags(), - PrintFlags: printers.NewPrintFlags("scaled"), CurrentReplicas: -1, Recorder: genericclioptions.NoopRecorder{}, IOStreams: ioStreams, @@ -107,7 +115,6 @@ func NewCmdScale(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobr o := NewScaleOptions(ioStreams) validArgs := []string{"deployment", "replicaset", "replicationcontroller", "statefulset"} - argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ Use: "scale [--resource-version=version] [--current-replicas=count] --replicas=COUNT (-f FILENAME | TYPE NAME)", @@ -120,8 +127,7 @@ func NewCmdScale(f cmdutil.Factory, ioStreams genericclioptions.IOStreams) *cobr cmdutil.CheckErr(o.Validate(cmd)) cmdutil.CheckErr(o.RunScale()) }, - ValidArgs: validArgs, - ArgAliases: argAliases, + ValidArgs: validArgs, } o.RecordFlags.AddFlags(cmd) @@ -173,9 +179,6 @@ func (o *ScaleOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []st } func (o *ScaleOptions) Validate(cmd *cobra.Command) error { - if err := cmdutil.ValidateOutputArgs(cmd); err != nil { - return err - } if o.Replicas < 0 { return fmt.Errorf("The --replicas=COUNT flag is required, and COUNT must be greater than or equal to 0") } diff --git a/pkg/kubectl/cmd/set/BUILD b/pkg/kubectl/cmd/set/BUILD index e07ab919d7f..2c8b4f59859 100644 --- a/pkg/kubectl/cmd/set/BUILD +++ b/pkg/kubectl/cmd/set/BUILD @@ -26,7 +26,7 @@ go_library( "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/cmd/util/env:go_default_library", "//pkg/kubectl/genericclioptions:go_default_library", - "//pkg/kubectl/resource:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/scheme:go_default_library", "//pkg/kubectl/util/i18n:go_default_library", "//pkg/printers:go_default_library", @@ -69,7 +69,7 @@ go_test( "//pkg/kubectl/cmd/testing:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/genericclioptions:go_default_library", - "//pkg/kubectl/resource:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/scheme:go_default_library", "//pkg/printers:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", diff --git a/pkg/kubectl/cmd/set/helper.go b/pkg/kubectl/cmd/set/helper.go index 5d6bed200d1..4d267c28da2 100644 --- a/pkg/kubectl/cmd/set/helper.go +++ b/pkg/kubectl/cmd/set/helper.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/strategicpatch" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" ) // selectContainers allows one or more containers to be matched against a string or wildcard diff --git a/pkg/kubectl/cmd/set/set_env.go b/pkg/kubectl/cmd/set/set_env.go index 3d94eb5df4c..75a2b73d298 100644 --- a/pkg/kubectl/cmd/set/set_env.go +++ b/pkg/kubectl/cmd/set/set_env.go @@ -30,11 +30,12 @@ import ( "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" envutil "k8s.io/kubernetes/pkg/kubectl/cmd/util/env" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" "k8s.io/kubernetes/pkg/printers" ) @@ -126,7 +127,7 @@ type EnvOptions struct { // pod templates are selected by default and allowing environment to be overwritten func NewEnvOptions(streams genericclioptions.IOStreams) *EnvOptions { return &EnvOptions{ - PrintFlags: printers.NewPrintFlags("env updated"), + PrintFlags: printers.NewPrintFlags("env updated", legacyscheme.Scheme), ContainerSelector: "*", Overwrite: true, @@ -241,7 +242,7 @@ func (o *EnvOptions) RunEnv() error { if len(o.From) != 0 { b := o.builder(). - WithScheme(scheme.Scheme, scheme.Registry.RegisteredGroupVersions()...). + WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...). LocalParam(o.Local). ContinueOnError(). NamespaceParam(o.namespace).DefaultNamespace(). @@ -305,7 +306,7 @@ func (o *EnvOptions) RunEnv() error { } b := o.builder(). - WithScheme(scheme.Scheme, scheme.Registry.RegisteredGroupVersions()...). + WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...). LocalParam(o.Local). ContinueOnError(). NamespaceParam(o.namespace).DefaultNamespace(). diff --git a/pkg/kubectl/cmd/set/set_env_test.go b/pkg/kubectl/cmd/set/set_env_test.go index d2813387890..f3431a8dd10 100644 --- a/pkg/kubectl/cmd/set/set_env_test.go +++ b/pkg/kubectl/cmd/set/set_env_test.go @@ -24,8 +24,6 @@ import ( "strings" "testing" - "k8s.io/kubernetes/pkg/printers" - "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" appsv1beta1 "k8s.io/api/apps/v1beta1" @@ -39,11 +37,13 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" restclient "k8s.io/client-go/rest" "k8s.io/client-go/rest/fake" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/api/testapi" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" + "k8s.io/kubernetes/pkg/printers" ) func TestSetEnvLocal(t *testing.T) { @@ -66,8 +66,8 @@ func TestSetEnvLocal(t *testing.T) { streams, _, buf, bufErr := genericclioptions.NewTestIOStreams() opts := NewEnvOptions(streams) opts.PrintFlags = &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, } opts.FilenameOptions = resource.FilenameOptions{ @@ -109,8 +109,8 @@ func TestSetMultiResourcesEnvLocal(t *testing.T) { streams, _, buf, bufErr := genericclioptions.NewTestIOStreams() opts := NewEnvOptions(streams) opts.PrintFlags = &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, } opts.FilenameOptions = resource.FilenameOptions{ @@ -497,8 +497,8 @@ func TestSetEnvRemote(t *testing.T) { streams := genericclioptions.NewTestIOStreamsDiscard() opts := NewEnvOptions(streams) opts.PrintFlags = &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, } opts.Local = false diff --git a/pkg/kubectl/cmd/set/set_image.go b/pkg/kubectl/cmd/set/set_image.go index 3fe294d981d..3f13167d87d 100644 --- a/pkg/kubectl/cmd/set/set_image.go +++ b/pkg/kubectl/cmd/set/set_image.go @@ -26,10 +26,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" @@ -87,7 +88,7 @@ var ( func NewImageOptions(streams genericclioptions.IOStreams) *SetImageOptions { return &SetImageOptions{ - PrintFlags: printers.NewPrintFlags("image updated"), + PrintFlags: printers.NewPrintFlags("image updated", legacyscheme.Scheme), RecordFlags: genericclioptions.NewRecordFlags(), Recorder: genericclioptions.NoopRecorder{}, @@ -161,7 +162,7 @@ func (o *SetImageOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args [ includeUninitialized := cmdutil.ShouldIncludeUninitialized(cmd, false) builder := f.NewBuilder(). - WithScheme(scheme.Scheme, scheme.Registry.RegisteredGroupVersions()...). + WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...). LocalParam(o.Local). ContinueOnError(). NamespaceParam(cmdNamespace).DefaultNamespace(). diff --git a/pkg/kubectl/cmd/set/set_image_test.go b/pkg/kubectl/cmd/set/set_image_test.go index 0bb79a5b3de..d6cb422de00 100644 --- a/pkg/kubectl/cmd/set/set_image_test.go +++ b/pkg/kubectl/cmd/set/set_image_test.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "k8s.io/kubernetes/pkg/api/legacyscheme" appsv1 "k8s.io/api/apps/v1" appsv1beta1 "k8s.io/api/apps/v1beta1" @@ -41,7 +42,7 @@ import ( "k8s.io/kubernetes/pkg/api/testapi" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" "k8s.io/kubernetes/pkg/printers" ) @@ -73,8 +74,8 @@ func TestImageLocal(t *testing.T) { opts := SetImageOptions{ PrintFlags: &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, }, @@ -100,8 +101,8 @@ func TestImageLocal(t *testing.T) { func TestSetImageValidation(t *testing.T) { printFlags := &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), } testCases := []struct { @@ -196,8 +197,8 @@ func TestSetMultiResourcesImageLocal(t *testing.T) { opts := SetImageOptions{ PrintFlags: &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, }, @@ -590,8 +591,8 @@ func TestSetImageRemote(t *testing.T) { cmd.Flags().Set("output", outputFormat) opts := SetImageOptions{ PrintFlags: &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, }, diff --git a/pkg/kubectl/cmd/set/set_resources.go b/pkg/kubectl/cmd/set/set_resources.go index a15b32e30fe..686001f352a 100644 --- a/pkg/kubectl/cmd/set/set_resources.go +++ b/pkg/kubectl/cmd/set/set_resources.go @@ -34,7 +34,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -94,7 +94,7 @@ type SetResourcesOptions struct { // pod templates are selected by default. func NewResourcesOptions(streams genericclioptions.IOStreams) *SetResourcesOptions { return &SetResourcesOptions{ - PrintFlags: printers.NewPrintFlags("resource requirements updated"), + PrintFlags: printers.NewPrintFlags("resource requirements updated", legacyscheme.Scheme), RecordFlags: genericclioptions.NewRecordFlags(), Recorder: genericclioptions.NoopRecorder{}, diff --git a/pkg/kubectl/cmd/set/set_resources_test.go b/pkg/kubectl/cmd/set/set_resources_test.go index 1e5ec98c273..8575f9528c7 100644 --- a/pkg/kubectl/cmd/set/set_resources_test.go +++ b/pkg/kubectl/cmd/set/set_resources_test.go @@ -42,7 +42,7 @@ import ( "k8s.io/kubernetes/pkg/api/testapi" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" ) @@ -72,8 +72,8 @@ func TestResourcesLocal(t *testing.T) { opts := SetResourcesOptions{ PrintFlags: &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, }, @@ -127,8 +127,8 @@ func TestSetMultiResourcesLimitsLocal(t *testing.T) { opts := SetResourcesOptions{ PrintFlags: &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, }, @@ -508,8 +508,8 @@ func TestSetResourcesRemote(t *testing.T) { cmd.Flags().Set("output", outputFormat) opts := SetResourcesOptions{ PrintFlags: &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, }, diff --git a/pkg/kubectl/cmd/set/set_selector.go b/pkg/kubectl/cmd/set/set_selector.go index f7c75ae2c6e..f22bee3891a 100644 --- a/pkg/kubectl/cmd/set/set_selector.go +++ b/pkg/kubectl/cmd/set/set_selector.go @@ -33,7 +33,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -79,7 +79,7 @@ var ( func NewSelectorOptions(streams genericclioptions.IOStreams) *SetSelectorOptions { return &SetSelectorOptions{ - PrintFlags: printers.NewPrintFlags("selector updated"), + PrintFlags: printers.NewPrintFlags("selector updated", legacyscheme.Scheme), RecordFlags: genericclioptions.NewRecordFlags(), Recorder: genericclioptions.NoopRecorder{}, diff --git a/pkg/kubectl/cmd/set/set_serviceaccount.go b/pkg/kubectl/cmd/set/set_serviceaccount.go index 781f3f17bdd..c38381006ee 100644 --- a/pkg/kubectl/cmd/set/set_serviceaccount.go +++ b/pkg/kubectl/cmd/set/set_serviceaccount.go @@ -33,7 +33,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -79,7 +79,7 @@ type SetServiceAccountOptions struct { func NewSetServiceAccountOptions(streams genericclioptions.IOStreams) *SetServiceAccountOptions { return &SetServiceAccountOptions{ - PrintFlags: printers.NewPrintFlags("serviceaccount updated"), + PrintFlags: printers.NewPrintFlags("serviceaccount updated", legacyscheme.Scheme), RecordFlags: genericclioptions.NewRecordFlags(), Recorder: genericclioptions.NoopRecorder{}, diff --git a/pkg/kubectl/cmd/set/set_serviceaccount_test.go b/pkg/kubectl/cmd/set/set_serviceaccount_test.go index 2fa9b7784e5..21d94d58fc9 100644 --- a/pkg/kubectl/cmd/set/set_serviceaccount_test.go +++ b/pkg/kubectl/cmd/set/set_serviceaccount_test.go @@ -43,7 +43,7 @@ import ( "k8s.io/kubernetes/pkg/api/testapi" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" ) @@ -91,8 +91,8 @@ func TestSetServiceAccountLocal(t *testing.T) { testapi.Default = testapi.Groups[input.apiGroup] saConfig := SetServiceAccountOptions{ PrintFlags: &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, }, @@ -135,8 +135,8 @@ func TestSetServiceAccountMultiLocal(t *testing.T) { cmd.Flags().Set("local", "true") opts := SetServiceAccountOptions{ PrintFlags: &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, }, @@ -379,8 +379,8 @@ func TestSetServiceAccountRemote(t *testing.T) { cmd.Flags().Set("output", outputFormat) saConfig := SetServiceAccountOptions{ PrintFlags: &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, }, @@ -426,8 +426,8 @@ func TestServiceAccountValidation(t *testing.T) { saConfig := &SetServiceAccountOptions{ PrintFlags: &printers.PrintFlags{ - JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(), - NamePrintFlags: printers.NewNamePrintFlags(""), + JSONYamlPrintFlags: printers.NewJSONYamlPrintFlags(legacyscheme.Scheme), + NamePrintFlags: printers.NewNamePrintFlags("", legacyscheme.Scheme), OutputFormat: &outputFormat, }, diff --git a/pkg/kubectl/cmd/set/set_subject.go b/pkg/kubectl/cmd/set/set_subject.go index e9ee174dfee..081a769426c 100644 --- a/pkg/kubectl/cmd/set/set_subject.go +++ b/pkg/kubectl/cmd/set/set_subject.go @@ -33,7 +33,7 @@ import ( "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" ) @@ -82,7 +82,7 @@ type SubjectOptions struct { func NewSubjectOptions(streams genericclioptions.IOStreams) *SubjectOptions { return &SubjectOptions{ - PrintFlags: printers.NewPrintFlags("subjects updated"), + PrintFlags: printers.NewPrintFlags("subjects updated", legacyscheme.Scheme), IOStreams: streams, } diff --git a/pkg/kubectl/cmd/set/set_subject_test.go b/pkg/kubectl/cmd/set/set_subject_test.go index 8b205095e72..03ee0ee8965 100644 --- a/pkg/kubectl/cmd/set/set_subject_test.go +++ b/pkg/kubectl/cmd/set/set_subject_test.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/pkg/apis/rbac" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" ) func TestValidate(t *testing.T) { diff --git a/pkg/kubectl/cmd/set/set_test.go b/pkg/kubectl/cmd/set/set_test.go index 7432a49bf76..fa985fa65ea 100644 --- a/pkg/kubectl/cmd/set/set_test.go +++ b/pkg/kubectl/cmd/set/set_test.go @@ -26,7 +26,7 @@ import ( ) func TestLocalAndDryRunFlags(t *testing.T) { - f := clientcmdutil.NewFactory(nil) + f := clientcmdutil.NewFactory(clientcmdutil.NewTestConfigFlags()) setCmd := NewCmdSet(f, genericclioptions.NewTestIOStreamsDiscard()) ensureLocalAndDryRunFlagsOnChildren(t, setCmd, "") } diff --git a/pkg/kubectl/cmd/taint.go b/pkg/kubectl/cmd/taint.go index 84b381aa805..5437dd8ce42 100644 --- a/pkg/kubectl/cmd/taint.go +++ b/pkg/kubectl/cmd/taint.go @@ -17,11 +17,10 @@ limitations under the License. package cmd import ( + "encoding/json" "fmt" "strings" - "encoding/json" - "github.com/golang/glog" "github.com/spf13/cobra" @@ -32,11 +31,10 @@ import ( "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/util/i18n" "k8s.io/kubernetes/pkg/printers" taintutils "k8s.io/kubernetes/pkg/util/taints" @@ -88,12 +86,11 @@ var ( func NewCmdTaint(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { options := &TaintOptions{ - PrintFlags: printers.NewPrintFlags("tainted"), + PrintFlags: printers.NewPrintFlags("tainted", legacyscheme.Scheme), IOStreams: streams, } validArgs := []string{"node"} - argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ Use: "taint NODE NAME KEY_1=VAL_1:TAINT_EFFECT_1 ... KEY_N=VAL_N:TAINT_EFFECT_N", @@ -112,8 +109,7 @@ func NewCmdTaint(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra. cmdutil.CheckErr(err) } }, - ValidArgs: validArgs, - ArgAliases: argAliases, + ValidArgs: validArgs, } options.PrintFlags.AddFlags(cmd) @@ -212,7 +208,7 @@ func (o TaintOptions) validateFlags() error { // Validate checks to the TaintOptions to see if there is sufficient information run the command. func (o TaintOptions) Validate() error { resourceType := strings.ToLower(o.resources[0]) - validResources, isValidResource := append(kubectl.ResourceAliases([]string{"node"}), "node"), false + validResources, isValidResource := []string{"node", "nodes"}, false for _, validResource := range validResources { if resourceType == validResource { isValidResource = true diff --git a/pkg/kubectl/cmd/taint_test.go b/pkg/kubectl/cmd/taint_test.go index 22fcee6b917..bf5ec3b7777 100644 --- a/pkg/kubectl/cmd/taint_test.go +++ b/pkg/kubectl/cmd/taint_test.go @@ -242,7 +242,7 @@ func TestTaint(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ diff --git a/pkg/kubectl/cmd/testing/BUILD b/pkg/kubectl/cmd/testing/BUILD index 4140652d799..0e73194b3a5 100644 --- a/pkg/kubectl/cmd/testing/BUILD +++ b/pkg/kubectl/cmd/testing/BUILD @@ -13,11 +13,10 @@ go_library( "//pkg/apis/core:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/kubectl:go_default_library", - "//pkg/kubectl/categories:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/cmd/util/openapi:go_default_library", "//pkg/kubectl/cmd/util/openapi/testing:go_default_library", - "//pkg/kubectl/resource:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/validation:go_default_library", "//pkg/printers:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", @@ -33,6 +32,7 @@ go_library( "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", "//vendor/k8s.io/client-go/rest/fake:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/scale:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", diff --git a/pkg/kubectl/cmd/testing/fake.go b/pkg/kubectl/cmd/testing/fake.go index 461e2a17547..9dc6b0b153d 100644 --- a/pkg/kubectl/cmd/testing/fake.go +++ b/pkg/kubectl/cmd/testing/fake.go @@ -39,6 +39,7 @@ import ( "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/rest/fake" + "k8s.io/client-go/restmapper" scaleclient "k8s.io/client-go/scale" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -46,11 +47,10 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/kubectl" - "k8s.io/kubernetes/pkg/kubectl/categories" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi" openapitesting "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi/testing" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/validation" "k8s.io/kubernetes/pkg/printers" ) @@ -253,11 +253,28 @@ type TestFactory struct { func NewTestFactory() *TestFactory { // specify an optionalClientConfig to explicitly use in testing // to avoid polluting an existing user config. - config, configFile := defaultFakeClientConfig() + tmpFile, err := ioutil.TempFile("", "cmdtests_temp") + if err != nil { + panic(fmt.Sprintf("unable to create a fake client config: %v", err)) + } + + loadingRules := &clientcmd.ClientConfigLoadingRules{ + Precedence: []string{tmpFile.Name()}, + MigrationRules: map[string]string{}, + } + + overrides := &clientcmd.ConfigOverrides{ClusterDefaults: clientcmdapi.Cluster{Server: "http://localhost:8080"}} + fallbackReader := bytes.NewBuffer([]byte{}) + clientConfig := clientcmd.NewInteractiveDeferredLoadingClientConfig(loadingRules, overrides, fallbackReader) + + configFlags := cmdutil.NewTestConfigFlags(). + WithClientConfig(clientConfig). + WithRESTMapper(testRESTMapper()) + return &TestFactory{ - Factory: cmdutil.NewFactory(config), + Factory: cmdutil.NewFactory(configFlags), FakeDynamicClient: fakedynamic.NewSimpleDynamicClient(legacyscheme.Scheme), - tempConfigFile: configFile, + tempConfigFile: tmpFile, } } @@ -269,33 +286,8 @@ func (f *TestFactory) Cleanup() { os.Remove(f.tempConfigFile.Name()) } -func defaultFakeClientConfig() (clientcmd.ClientConfig, *os.File) { - loadingRules, tmpFile, err := newDefaultFakeClientConfigLoadingRules() - if err != nil { - panic(fmt.Sprintf("unable to create a fake client config: %v", err)) - } - - overrides := &clientcmd.ConfigOverrides{ClusterDefaults: clientcmdapi.Cluster{Server: "http://localhost:8080"}} - fallbackReader := bytes.NewBuffer([]byte{}) - - clientConfig := clientcmd.NewInteractiveDeferredLoadingClientConfig(loadingRules, overrides, fallbackReader) - return clientConfig, tmpFile -} - -func newDefaultFakeClientConfigLoadingRules() (*clientcmd.ClientConfigLoadingRules, *os.File, error) { - tmpFile, err := ioutil.TempFile("", "cmdtests_temp") - if err != nil { - return nil, nil, err - } - - return &clientcmd.ClientConfigLoadingRules{ - Precedence: []string{tmpFile.Name()}, - MigrationRules: map[string]string{}, - }, tmpFile, nil -} - -func (f *TestFactory) CategoryExpander() categories.CategoryExpander { - return categories.LegacyCategoryExpander +func (f *TestFactory) CategoryExpander() (restmapper.CategoryExpander, error) { + return resource.FakeCategoryExpander, nil } func (f *TestFactory) ClientConfig() (*restclient.Config, error) { @@ -342,6 +334,7 @@ func (f *TestFactory) Command(*cobra.Command, bool) string { func (f *TestFactory) NewBuilder() *resource.Builder { mapper, err := f.RESTMapper() + categoryExpander, err2 := f.CategoryExpander() return resource.NewFakeBuilder( func(version schema.GroupVersion) (resource.RESTClient, error) { @@ -354,8 +347,8 @@ func (f *TestFactory) NewBuilder() *resource.Builder { return f.Client, nil }, mapper, - f.CategoryExpander(), - ).AddError(err) + categoryExpander, + ).AddError(err).AddError(err2) } func (f *TestFactory) KubernetesClientSet() (*kubernetes.Clientset, error) { @@ -436,22 +429,21 @@ func (f *TestFactory) ClientSetForVersion(requiredVersion *schema.GroupVersion) return f.ClientSet() } -func (f *TestFactory) RESTMapper() (meta.RESTMapper, error) { +func testRESTMapper() meta.RESTMapper { groupResources := testDynamicResources() - mapper := discovery.NewRESTMapper(groupResources) + mapper := restmapper.NewDiscoveryRESTMapper(groupResources) // for backwards compatibility with existing tests, allow rest mappings from the scheme to show up // TODO: make this opt-in? mapper = meta.FirstHitRESTMapper{ MultiRESTMapper: meta.MultiRESTMapper{ mapper, - testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme), + testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme), }, } - // TODO: should probably be the external scheme fakeDs := &fakeCachedDiscoveryClient{} - expander := cmdutil.NewShortcutExpander(mapper, fakeDs) - return expander, nil + expander := restmapper.NewShortcutExpander(mapper, fakeDs) + return expander } func (f *TestFactory) LogsForObject(object, options runtime.Object, timeout time.Duration) (*restclient.Request, error) { @@ -476,8 +468,8 @@ func (f *TestFactory) ScaleClient() (scaleclient.ScalesGetter, error) { return f.ScaleGetter, nil } -func testDynamicResources() []*discovery.APIGroupResources { - return []*discovery.APIGroupResources{ +func testDynamicResources() []*restmapper.APIGroupResources { + return []*restmapper.APIGroupResources{ { Group: metav1.APIGroup{ Versions: []metav1.GroupVersionForDiscovery{ diff --git a/pkg/kubectl/cmd/top.go b/pkg/kubectl/cmd/top.go index 608c0ae7c52..c8abeee226f 100644 --- a/pkg/kubectl/cmd/top.go +++ b/pkg/kubectl/cmd/top.go @@ -17,8 +17,6 @@ limitations under the License. package cmd import ( - "io" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/util/i18n" @@ -26,6 +24,7 @@ import ( "github.com/spf13/cobra" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" ) var ( @@ -40,17 +39,17 @@ var ( This command requires Heapster to be correctly configured and working on the server. `)) ) -func NewCmdTop(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { +func NewCmdTop(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { cmd := &cobra.Command{ Use: "top", Short: i18n.T("Display Resource (CPU/Memory/Storage) usage."), Long: topLong, - Run: cmdutil.DefaultSubCommandRun(errOut), + Run: cmdutil.DefaultSubCommandRun(streams.ErrOut), } // create subcommands - cmd.AddCommand(NewCmdTopNode(f, nil, out)) - cmd.AddCommand(NewCmdTopPod(f, nil, out)) + cmd.AddCommand(NewCmdTopNode(f, nil, streams)) + cmd.AddCommand(NewCmdTopPod(f, nil, streams)) return cmd } diff --git a/pkg/kubectl/cmd/top_node.go b/pkg/kubectl/cmd/top_node.go index 624be6633aa..fb2bbb0f95f 100644 --- a/pkg/kubectl/cmd/top_node.go +++ b/pkg/kubectl/cmd/top_node.go @@ -18,7 +18,6 @@ package cmd import ( "errors" - "io" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -29,6 +28,7 @@ import ( corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/metricsutil" "k8s.io/kubernetes/pkg/kubectl/util/i18n" metricsapi "k8s.io/metrics/pkg/apis/metrics" @@ -46,6 +46,8 @@ type TopNodeOptions struct { Printer *metricsutil.TopCmdPrinter DiscoveryClient discovery.DiscoveryInterface MetricsClient metricsclientset.Interface + + genericclioptions.IOStreams } type HeapsterTopOptions struct { @@ -89,9 +91,11 @@ var ( kubectl top node NODE_NAME`)) ) -func NewCmdTopNode(f cmdutil.Factory, options *TopNodeOptions, out io.Writer) *cobra.Command { - if options == nil { - options = &TopNodeOptions{} +func NewCmdTopNode(f cmdutil.Factory, o *TopNodeOptions, streams genericclioptions.IOStreams) *cobra.Command { + if o == nil { + o = &TopNodeOptions{ + IOStreams: streams, + } } cmd := &cobra.Command{ @@ -101,24 +105,24 @@ func NewCmdTopNode(f cmdutil.Factory, options *TopNodeOptions, out io.Writer) *c Long: topNodeLong, Example: topNodeExample, Run: func(cmd *cobra.Command, args []string) { - if err := options.Complete(f, cmd, args, out); err != nil { + if err := o.Complete(f, cmd, args); err != nil { cmdutil.CheckErr(err) } - if err := options.Validate(); err != nil { + if err := o.Validate(); err != nil { cmdutil.CheckErr(cmdutil.UsageErrorf(cmd, "%v", err)) } - if err := options.RunTopNode(); err != nil { + if err := o.RunTopNode(); err != nil { cmdutil.CheckErr(err) } }, Aliases: []string{"nodes", "no"}, } - cmd.Flags().StringVarP(&options.Selector, "selector", "l", options.Selector, "Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2)") - options.HeapsterOptions.Bind(cmd.Flags()) + cmd.Flags().StringVarP(&o.Selector, "selector", "l", o.Selector, "Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2)") + o.HeapsterOptions.Bind(cmd.Flags()) return cmd } -func (o *TopNodeOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string, out io.Writer) error { +func (o *TopNodeOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { if len(args) == 1 { o.ResourceName = args[0] } else if len(args) > 1 { @@ -144,7 +148,7 @@ func (o *TopNodeOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args [] o.NodeClient = clientset.CoreV1() o.Client = metricsutil.NewHeapsterMetricsClient(clientset.CoreV1(), o.HeapsterOptions.Namespace, o.HeapsterOptions.Scheme, o.HeapsterOptions.Service, o.HeapsterOptions.Port) - o.Printer = metricsutil.NewTopCmdPrinter(out) + o.Printer = metricsutil.NewTopCmdPrinter(o.Out) return nil } diff --git a/pkg/kubectl/cmd/top_node_test.go b/pkg/kubectl/cmd/top_node_test.go index 13c7af7f871..6857355fb1a 100644 --- a/pkg/kubectl/cmd/top_node_test.go +++ b/pkg/kubectl/cmd/top_node_test.go @@ -32,6 +32,7 @@ import ( core "k8s.io/client-go/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" "k8s.io/kubernetes/pkg/kubectl/scheme" metricsv1alpha1api "k8s.io/metrics/pkg/apis/metrics/v1alpha1" metricsv1beta1api "k8s.io/metrics/pkg/apis/metrics/v1beta1" @@ -52,7 +53,7 @@ func TestTopNodeAllMetrics(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -79,9 +80,9 @@ func TestTopNodeAllMetrics(t *testing.T) { } tf.Namespace = "test" tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdTopNode(tf, nil, buf) + cmd := NewCmdTopNode(tf, nil, streams) cmd.Run(cmd, []string{}) // Check the presence of node names in the output. @@ -105,7 +106,7 @@ func TestTopNodeAllMetricsCustomDefaults(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -132,7 +133,7 @@ func TestTopNodeAllMetricsCustomDefaults(t *testing.T) { } tf.Namespace = "test" tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() opts := &TopNodeOptions{ HeapsterOptions: HeapsterTopOptions{ @@ -140,8 +141,9 @@ func TestTopNodeAllMetricsCustomDefaults(t *testing.T) { Scheme: "https", Service: "custom-heapster-service", }, + IOStreams: streams, } - cmd := NewCmdTopNode(tf, opts, buf) + cmd := NewCmdTopNode(tf, opts, streams) cmd.Run(cmd, []string{}) // Check the presence of node names in the output. @@ -168,7 +170,7 @@ func TestTopNodeWithNameMetrics(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -195,9 +197,9 @@ func TestTopNodeWithNameMetrics(t *testing.T) { } tf.Namespace = "test" tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdTopNode(tf, nil, buf) + cmd := NewCmdTopNode(tf, nil, streams) cmd.Run(cmd, []string{expectedMetrics.Name}) // Check the presence of node names in the output. @@ -235,7 +237,7 @@ func TestTopNodeWithLabelSelectorMetrics(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -262,9 +264,9 @@ func TestTopNodeWithLabelSelectorMetrics(t *testing.T) { } tf.Namespace = "test" tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdTopNode(tf, nil, buf) + cmd := NewCmdTopNode(tf, nil, streams) cmd.Flags().Set("selector", label) cmd.Run(cmd, []string{}) @@ -290,7 +292,7 @@ func TestTopNodeAllMetricsFromMetricsServer(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -315,14 +317,16 @@ func TestTopNodeAllMetricsFromMetricsServer(t *testing.T) { }) tf.Namespace = "test" tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdTopNode(tf, nil, buf) + cmd := NewCmdTopNode(tf, nil, streams) // TODO in the long run, we want to test most of our commands like this. Wire the options struct with specific mocks // TODO then check the particular Run functionality and harvest results from fake clients - cmdOptions := &TopNodeOptions{} - if err := cmdOptions.Complete(tf, cmd, []string{}, buf); err != nil { + cmdOptions := &TopNodeOptions{ + IOStreams: streams, + } + if err := cmdOptions.Complete(tf, cmd, []string{}); err != nil { t.Fatal(err) } cmdOptions.MetricsClient = fakemetricsClientset @@ -356,7 +360,7 @@ func TestTopNodeWithNameMetricsFromMetricsServer(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -381,14 +385,16 @@ func TestTopNodeWithNameMetricsFromMetricsServer(t *testing.T) { }) tf.Namespace = "test" tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdTopNode(tf, nil, buf) + cmd := NewCmdTopNode(tf, nil, streams) // TODO in the long run, we want to test most of our commands like this. Wire the options struct with specific mocks // TODO then check the particular Run functionality and harvest results from fake clients - cmdOptions := &TopNodeOptions{} - if err := cmdOptions.Complete(tf, cmd, []string{expectedMetrics.Name}, buf); err != nil { + cmdOptions := &TopNodeOptions{ + IOStreams: streams, + } + if err := cmdOptions.Complete(tf, cmd, []string{expectedMetrics.Name}); err != nil { t.Fatal(err) } cmdOptions.MetricsClient = fakemetricsClientset @@ -432,7 +438,7 @@ func TestTopNodeWithLabelSelectorMetricsFromMetricsServer(t *testing.T) { tf := cmdtesting.NewTestFactory() defer tf.Cleanup() - codec := legacyscheme.Codecs.LegacyCodec(scheme.Versions...) + codec := legacyscheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) ns := legacyscheme.Codecs tf.Client = &fake.RESTClient{ @@ -458,15 +464,17 @@ func TestTopNodeWithLabelSelectorMetricsFromMetricsServer(t *testing.T) { }) tf.Namespace = "test" tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdTopNode(tf, nil, buf) + cmd := NewCmdTopNode(tf, nil, streams) cmd.Flags().Set("selector", label) // TODO in the long run, we want to test most of our commands like this. Wire the options struct with specific mocks // TODO then check the particular Run functionality and harvest results from fake clients - cmdOptions := &TopNodeOptions{} - if err := cmdOptions.Complete(tf, cmd, []string{}, buf); err != nil { + cmdOptions := &TopNodeOptions{ + IOStreams: streams, + } + if err := cmdOptions.Complete(tf, cmd, []string{}); err != nil { t.Fatal(err) } cmdOptions.MetricsClient = fakemetricsClientset diff --git a/pkg/kubectl/cmd/top_pod.go b/pkg/kubectl/cmd/top_pod.go index 71bd8df2cb0..e073099d413 100644 --- a/pkg/kubectl/cmd/top_pod.go +++ b/pkg/kubectl/cmd/top_pod.go @@ -19,7 +19,6 @@ package cmd import ( "errors" "fmt" - "io" "time" "k8s.io/api/core/v1" @@ -37,6 +36,7 @@ import ( "github.com/golang/glog" "github.com/spf13/cobra" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" ) type TopPodOptions struct { @@ -51,6 +51,8 @@ type TopPodOptions struct { Printer *metricsutil.TopCmdPrinter DiscoveryClient discovery.DiscoveryInterface MetricsClient metricsclientset.Interface + + genericclioptions.IOStreams } const metricsCreationDelay = 2 * time.Minute @@ -78,9 +80,11 @@ var ( kubectl top pod -l name=myLabel`)) ) -func NewCmdTopPod(f cmdutil.Factory, options *TopPodOptions, out io.Writer) *cobra.Command { - if options == nil { - options = &TopPodOptions{} +func NewCmdTopPod(f cmdutil.Factory, o *TopPodOptions, streams genericclioptions.IOStreams) *cobra.Command { + if o == nil { + o = &TopPodOptions{ + IOStreams: streams, + } } cmd := &cobra.Command{ @@ -90,26 +94,26 @@ func NewCmdTopPod(f cmdutil.Factory, options *TopPodOptions, out io.Writer) *cob Long: topPodLong, Example: topPodExample, Run: func(cmd *cobra.Command, args []string) { - if err := options.Complete(f, cmd, args, out); err != nil { + if err := o.Complete(f, cmd, args); err != nil { cmdutil.CheckErr(err) } - if err := options.Validate(); err != nil { + if err := o.Validate(); err != nil { cmdutil.CheckErr(cmdutil.UsageErrorf(cmd, "%v", err)) } - if err := options.RunTopPod(); err != nil { + if err := o.RunTopPod(); err != nil { cmdutil.CheckErr(err) } }, Aliases: []string{"pods", "po"}, } - cmd.Flags().StringVarP(&options.Selector, "selector", "l", options.Selector, "Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2)") - cmd.Flags().BoolVar(&options.PrintContainers, "containers", options.PrintContainers, "If present, print usage of containers within a pod.") - cmd.Flags().BoolVar(&options.AllNamespaces, "all-namespaces", options.AllNamespaces, "If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.") - options.HeapsterOptions.Bind(cmd.Flags()) + cmd.Flags().StringVarP(&o.Selector, "selector", "l", o.Selector, "Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2)") + cmd.Flags().BoolVar(&o.PrintContainers, "containers", o.PrintContainers, "If present, print usage of containers within a pod.") + cmd.Flags().BoolVar(&o.AllNamespaces, "all-namespaces", o.AllNamespaces, "If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.") + o.HeapsterOptions.Bind(cmd.Flags()) return cmd } -func (o *TopPodOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string, out io.Writer) error { +func (o *TopPodOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { var err error if len(args) == 1 { o.ResourceName = args[0] @@ -139,7 +143,7 @@ func (o *TopPodOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []s o.PodClient = clientset.CoreV1() o.Client = metricsutil.NewHeapsterMetricsClient(clientset.CoreV1(), o.HeapsterOptions.Namespace, o.HeapsterOptions.Scheme, o.HeapsterOptions.Service, o.HeapsterOptions.Port) - o.Printer = metricsutil.NewTopCmdPrinter(out) + o.Printer = metricsutil.NewTopCmdPrinter(o.Out) return nil } diff --git a/pkg/kubectl/cmd/top_pod_test.go b/pkg/kubectl/cmd/top_pod_test.go index ac5f78aa3d9..494e08efa6c 100644 --- a/pkg/kubectl/cmd/top_pod_test.go +++ b/pkg/kubectl/cmd/top_pod_test.go @@ -37,6 +37,7 @@ import ( core "k8s.io/client-go/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" metricsv1alpha1api "k8s.io/metrics/pkg/apis/metrics/v1alpha1" metricsv1beta1api "k8s.io/metrics/pkg/apis/metrics/v1beta1" metricsfake "k8s.io/metrics/pkg/client/clientset_generated/clientset/fake" @@ -191,9 +192,9 @@ func TestTopPod(t *testing.T) { } tf.Namespace = testNS tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdTopPod(tf, nil, buf) + cmd := NewCmdTopPod(tf, nil, streams) for name, value := range testCase.flags { cmd.Flags().Set(name, value) } @@ -330,19 +331,20 @@ func TestTopPodWithMetricsServer(t *testing.T) { } tf.Namespace = testNS tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() - cmd := NewCmdTopPod(tf, nil, buf) + cmd := NewCmdTopPod(tf, nil, streams) var cmdOptions *TopPodOptions if testCase.options != nil { cmdOptions = testCase.options } else { cmdOptions = &TopPodOptions{} } + cmdOptions.IOStreams = streams // TODO in the long run, we want to test most of our commands like this. Wire the options struct with specific mocks // TODO then check the particular Run functionality and harvest results from fake clients. We probably end up skipping the factory altogether. - if err := cmdOptions.Complete(tf, cmd, testCase.args, buf); err != nil { + if err := cmdOptions.Complete(tf, cmd, testCase.args); err != nil { t.Fatal(err) } cmdOptions.MetricsClient = fakemetricsClientset @@ -534,7 +536,7 @@ func TestTopPodCustomDefaults(t *testing.T) { } tf.Namespace = testNS tf.ClientConfigVal = defaultClientConfig() - buf := bytes.NewBuffer([]byte{}) + streams, _, buf, _ := genericclioptions.NewTestIOStreams() opts := &TopPodOptions{ HeapsterOptions: HeapsterTopOptions{ @@ -543,8 +545,9 @@ func TestTopPodCustomDefaults(t *testing.T) { Service: "custom-heapster-service", }, DiscoveryClient: &fakeDiscovery{}, + IOStreams: streams, } - cmd := NewCmdTopPod(tf, opts, buf) + cmd := NewCmdTopPod(tf, opts, streams) for name, value := range testCase.flags { cmd.Flags().Set(name, value) } diff --git a/pkg/kubectl/cmd/top_test.go b/pkg/kubectl/cmd/top_test.go index 0090782fcd8..2ad63d06bdb 100644 --- a/pkg/kubectl/cmd/top_test.go +++ b/pkg/kubectl/cmd/top_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" metricsv1alpha1api "k8s.io/metrics/pkg/apis/metrics/v1alpha1" metricsv1beta1api "k8s.io/metrics/pkg/apis/metrics/v1beta1" ) @@ -46,9 +47,7 @@ func TestTopSubcommandsExist(t *testing.T) { f := cmdtesting.NewTestFactory() defer f.Cleanup() - buf := bytes.NewBuffer([]byte{}) - - cmd := NewCmdTop(f, buf, buf) + cmd := NewCmdTop(f, genericclioptions.NewTestIOStreamsDiscard()) if !cmd.HasSubCommands() { t.Error("top command should have subcommands") } diff --git a/pkg/kubectl/cmd/util/BUILD b/pkg/kubectl/cmd/util/BUILD index c3ce48d37c9..d344db58e64 100644 --- a/pkg/kubectl/cmd/util/BUILD +++ b/pkg/kubectl/cmd/util/BUILD @@ -4,14 +4,15 @@ go_library( name = "go_default_library", srcs = [ "cached_discovery.go", + "config_flags.go", "conversion.go", "factory.go", "factory_builder.go", "factory_client_access.go", "factory_object_mapping.go", "helpers.go", + "kubectl_match_version.go", "printing.go", - "shortcut_restmapper.go", ], importpath = "k8s.io/kubernetes/pkg/kubectl/cmd/util", visibility = ["//build/visible_to:pkg_kubectl_cmd_util_CONSUMERS"], @@ -26,14 +27,13 @@ go_library( "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", "//pkg/controller:go_default_library", "//pkg/kubectl:go_default_library", - "//pkg/kubectl/categories:go_default_library", "//pkg/kubectl/cmd/templates:go_default_library", "//pkg/kubectl/cmd/util/openapi:go_default_library", "//pkg/kubectl/cmd/util/openapi/validation:go_default_library", + "//pkg/kubectl/cmd/util/transport:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/plugins:go_default_library", - "//pkg/kubectl/resource:go_default_library", "//pkg/kubectl/scheme:go_default_library", - "//pkg/kubectl/util/transport:go_default_library", "//pkg/kubectl/validation:go_default_library", "//pkg/printers:go_default_library", "//pkg/printers/internalversion:go_default_library", @@ -68,6 +68,7 @@ go_library( "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/scale:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/util/homedir:go_default_library", @@ -82,7 +83,6 @@ go_test( "factory_object_mapping_test.go", "factory_test.go", "helpers_test.go", - "shortcut_restmapper_test.go", ], embed = [":go_default_library"], deps = [ @@ -97,8 +97,7 @@ go_test( "//pkg/client/clientset_generated/internalclientset/fake:go_default_library", "//pkg/controller:go_default_library", "//pkg/kubectl:go_default_library", - "//pkg/kubectl/categories:go_default_library", - "//pkg/kubectl/resource:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", @@ -118,6 +117,7 @@ go_test( "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", "//vendor/k8s.io/client-go/rest/fake:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/testing:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], @@ -139,6 +139,7 @@ filegroup( "//pkg/kubectl/cmd/util/jsonmerge:all-srcs", "//pkg/kubectl/cmd/util/openapi:all-srcs", "//pkg/kubectl/cmd/util/sanity:all-srcs", + "//pkg/kubectl/cmd/util/transport:all-srcs", ], tags = ["automanaged"], visibility = ["//build/visible_to:pkg_kubectl_cmd_util_CONSUMERS"], diff --git a/pkg/kubectl/cmd/util/cached_discovery.go b/pkg/kubectl/cmd/util/cached_discovery.go index d053102bb61..60b3ed9bb48 100644 --- a/pkg/kubectl/cmd/util/cached_discovery.go +++ b/pkg/kubectl/cmd/util/cached_discovery.go @@ -91,20 +91,7 @@ func (d *CachedDiscoveryClient) ServerResourcesForGroupVersion(groupVersion stri // ServerResources returns the supported resources for all groups and versions. func (d *CachedDiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) { - apiGroups, err := d.ServerGroups() - if err != nil { - return nil, err - } - groupVersions := metav1.ExtractGroupVersions(apiGroups) - result := []*metav1.APIResourceList{} - for _, groupVersion := range groupVersions { - resources, err := d.ServerResourcesForGroupVersion(groupVersion) - if err != nil { - return nil, err - } - result = append(result, resources) - } - return result, nil + return discovery.ServerResources(d) } func (d *CachedDiscoveryClient) ServerGroups() (*metav1.APIGroupList, error) { diff --git a/pkg/kubectl/cmd/util/config_flags.go b/pkg/kubectl/cmd/util/config_flags.go new file mode 100644 index 00000000000..db6d1a4655f --- /dev/null +++ b/pkg/kubectl/cmd/util/config_flags.go @@ -0,0 +1,367 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "net/http" + "os" + "path/filepath" + "time" + + "github.com/spf13/pflag" + + "k8s.io/apimachinery/pkg/api/meta" + utilflag "k8s.io/apiserver/pkg/util/flag" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" + + "fmt" + + "k8s.io/kubernetes/pkg/kubectl/cmd/util/transport" +) + +const ( + flagClusterName = "cluster" + flagAuthInfoName = "user" + flagContext = "context" + flagNamespace = "namespace" + flagAPIServer = "server" + flagInsecure = "insecure-skip-tls-verify" + flagCertFile = "client-certificate" + flagKeyFile = "client-key" + flagCAFile = "certificate-authority" + flagBearerToken = "token" + flagImpersonate = "as" + flagImpersonateGroup = "as-group" + flagUsername = "username" + flagPassword = "password" + flagTimeout = "request-timeout" + flagHTTPCacheDir = "cache-dir" +) + +// TODO(juanvallejo): move to pkg/kubectl/genericclioptions once +// the dependency on cmdutil is broken here. +// ConfigFlags composes the set of values necessary +// for obtaining a REST client config +type ConfigFlags struct { + CacheDir *string + KubeConfig *string + + // config flags + ClusterName *string + AuthInfoName *string + Context *string + Namespace *string + APIServer *string + Insecure *bool + CertFile *string + KeyFile *string + CAFile *string + BearerToken *string + Impersonate *string + ImpersonateGroup *[]string + Username *string + Password *string + Timeout *string +} + +// ToRESTConfig implements RESTClientGetter. +// Returns a REST client configuration based on a provided path +// to a .kubeconfig file, loading rules, and config flag overrides. +// Expects the AddFlags method to have been called. +func (f *ConfigFlags) ToRESTConfig() (*rest.Config, error) { + return f.ToRawKubeConfigLoader().ClientConfig() +} + +func (f *ConfigFlags) ToRawKubeConfigLoader() clientcmd.ClientConfig { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + // use the standard defaults for this client command + // DEPRECATED: remove and replace with something more accurate + loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig + + if f.KubeConfig != nil { + loadingRules.ExplicitPath = *f.KubeConfig + } + + overrides := &clientcmd.ConfigOverrides{ClusterDefaults: clientcmd.ClusterDefaults} + + // bind auth info flag values to overrides + if f.CertFile != nil { + overrides.AuthInfo.ClientCertificate = *f.CertFile + } + if f.KeyFile != nil { + overrides.AuthInfo.ClientKey = *f.KeyFile + } + if f.BearerToken != nil { + overrides.AuthInfo.Token = *f.BearerToken + } + if f.Impersonate != nil { + overrides.AuthInfo.Impersonate = *f.Impersonate + } + if f.ImpersonateGroup != nil { + overrides.AuthInfo.ImpersonateGroups = *f.ImpersonateGroup + } + if f.Username != nil { + overrides.AuthInfo.Username = *f.Username + } + if f.Password != nil { + overrides.AuthInfo.Password = *f.Password + } + + // bind cluster flags + if f.APIServer != nil { + overrides.ClusterInfo.Server = *f.APIServer + } + if f.CAFile != nil { + overrides.ClusterInfo.CertificateAuthority = *f.CAFile + } + if f.Insecure != nil { + overrides.ClusterInfo.InsecureSkipTLSVerify = *f.Insecure + } + + // bind context flags + if f.Context != nil { + overrides.CurrentContext = *f.Context + } + if f.ClusterName != nil { + overrides.Context.Cluster = *f.ClusterName + } + if f.AuthInfoName != nil { + overrides.Context.AuthInfo = *f.AuthInfoName + } + if f.Namespace != nil { + overrides.Context.Namespace = *f.Namespace + } + + if f.Timeout != nil { + overrides.Timeout = *f.Timeout + } + + var clientConfig clientcmd.ClientConfig + + // we only have an interactive prompt when a password is allowed + if f.Password == nil { + clientConfig = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, overrides) + } else { + clientConfig = clientcmd.NewInteractiveDeferredLoadingClientConfig(loadingRules, overrides, os.Stdin) + } + + return clientConfig +} + +// ToDiscoveryClient implements RESTClientGetter. +// Expects the AddFlags method to have been called. +// Returns a CachedDiscoveryInterface using a computed RESTConfig. +func (f *ConfigFlags) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { + config, err := f.ToRESTConfig() + if err != nil { + return nil, err + } + + // The more groups you have, the more discovery requests you need to make. + // given 25 groups (our groups + a few custom resources) with one-ish version each, discovery needs to make 50 requests + // double it just so we don't end up here again for a while. This config is only used for discovery. + config.Burst = 100 + + cacheDir := filepath.Join(homedir.HomeDir(), ".kube", "http-cache") + if f.CacheDir != nil { + cacheDir = *f.CacheDir + } + + if len(cacheDir) > 0 { + wt := config.WrapTransport + config.WrapTransport = func(rt http.RoundTripper) http.RoundTripper { + if wt != nil { + rt = wt(rt) + } + return transport.NewCacheRoundTripper(cacheDir, rt) + } + } + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + return nil, err + } + cacheDir = computeDiscoverCacheDir(filepath.Join(homedir.HomeDir(), ".kube", "cache", "discovery"), config.Host) + return NewCachedDiscoveryClient(discoveryClient, cacheDir, time.Duration(10*time.Minute)), nil +} + +// RESTMapper returns a mapper. +func (f *ConfigFlags) ToRESTMapper() (meta.RESTMapper, error) { + discoveryClient, err := f.ToDiscoveryClient() + if err != nil { + return nil, err + } + + mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) + expander := restmapper.NewShortcutExpander(mapper, discoveryClient) + return expander, nil +} + +func (f *ConfigFlags) AddFlags(flags *pflag.FlagSet) { + flags.SetNormalizeFunc(utilflag.WarnWordSepNormalizeFunc) // Warn for "_" flags + + // Normalize all flags that are coming from other packages or pre-configurations + // a.k.a. change all "_" to "-". e.g. glog package + flags.SetNormalizeFunc(utilflag.WordSepNormalizeFunc) + + if f.KubeConfig != nil { + flags.StringVar(f.KubeConfig, "kubeconfig", *f.KubeConfig, "Path to the kubeconfig file to use for CLI requests.") + } + if f.CacheDir != nil { + flags.StringVar(f.CacheDir, flagHTTPCacheDir, *f.CacheDir, "Default HTTP cache directory") + } + + // add config options + if f.CertFile != nil { + flags.StringVar(f.CertFile, flagCertFile, *f.CertFile, "Path to a client certificate file for TLS") + } + if f.KeyFile != nil { + flags.StringVar(f.KeyFile, flagKeyFile, *f.KeyFile, "Path to a client key file for TLS") + } + if f.BearerToken != nil { + flags.StringVar(f.BearerToken, flagBearerToken, *f.BearerToken, "Bearer token for authentication to the API server") + } + if f.Impersonate != nil { + flags.StringVar(f.Impersonate, flagImpersonate, *f.Impersonate, "Username to impersonate for the operation") + } + if f.ImpersonateGroup != nil { + flags.StringArrayVar(f.ImpersonateGroup, flagImpersonateGroup, *f.ImpersonateGroup, "Group to impersonate for the operation, this flag can be repeated to specify multiple groups.") + } + if f.Username != nil { + flags.StringVar(f.Username, flagUsername, *f.Username, "Username for basic authentication to the API server") + } + if f.Password != nil { + flags.StringVar(f.Password, flagPassword, *f.Password, "Password for basic authentication to the API server") + } + if f.ClusterName != nil { + flags.StringVar(f.ClusterName, flagClusterName, *f.ClusterName, "The name of the kubeconfig cluster to use") + } + if f.AuthInfoName != nil { + flags.StringVar(f.AuthInfoName, flagAuthInfoName, *f.AuthInfoName, "The name of the kubeconfig user to use") + } + if f.Namespace != nil { + flags.StringVarP(f.Namespace, flagNamespace, "n", *f.Namespace, "If present, the namespace scope for this CLI request") + } + if f.Context != nil { + flags.StringVar(f.Context, flagContext, *f.Context, "The name of the kubeconfig context to use") + } + + if f.APIServer != nil { + flags.StringVarP(f.APIServer, flagAPIServer, "s", *f.APIServer, "The address and port of the Kubernetes API server") + } + if f.Insecure != nil { + flags.BoolVar(f.Insecure, flagInsecure, *f.Insecure, "If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure") + } + if f.CAFile != nil { + flags.StringVar(f.CAFile, flagCAFile, *f.CAFile, "Path to a cert file for the certificate authority") + } + if f.Timeout != nil { + flags.StringVar(f.Timeout, flagTimeout, *f.Timeout, "The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.") + } + +} + +func (f *ConfigFlags) WithDeprecatedPasswordFlag() *ConfigFlags { + f.Username = stringptr("") + f.Password = stringptr("") + return f +} + +func NewConfigFlags() *ConfigFlags { + impersonateGroup := []string{} + insecure := false + + return &ConfigFlags{ + Insecure: &insecure, + Timeout: stringptr("0"), + KubeConfig: stringptr(""), + + ClusterName: stringptr(""), + AuthInfoName: stringptr(""), + Context: stringptr(""), + Namespace: stringptr(""), + APIServer: stringptr(""), + CertFile: stringptr(""), + KeyFile: stringptr(""), + CAFile: stringptr(""), + BearerToken: stringptr(""), + Impersonate: stringptr(""), + ImpersonateGroup: &impersonateGroup, + } +} + +func stringptr(val string) *string { + return &val +} + +// TODO(juanvallejo): move to separate file when config_flags are moved +// to pkg/kubectl/genericclioptions +type TestConfigFlags struct { + clientConfig clientcmd.ClientConfig + discoveryClient discovery.CachedDiscoveryInterface + restMapper meta.RESTMapper +} + +func (f *TestConfigFlags) ToRawKubeConfigLoader() clientcmd.ClientConfig { + if f.clientConfig == nil { + panic("attempt to obtain a test RawKubeConfigLoader with no clientConfig specified") + } + return f.clientConfig +} + +func (f *TestConfigFlags) ToRESTConfig() (*rest.Config, error) { + return f.ToRawKubeConfigLoader().ClientConfig() +} + +func (f *TestConfigFlags) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { + return f.discoveryClient, nil +} + +func (f *TestConfigFlags) ToRESTMapper() (meta.RESTMapper, error) { + if f.restMapper != nil { + return f.restMapper, nil + } + if f.discoveryClient != nil { + mapper := restmapper.NewDeferredDiscoveryRESTMapper(f.discoveryClient) + expander := restmapper.NewShortcutExpander(mapper, f.discoveryClient) + return expander, nil + } + return nil, fmt.Errorf("no restmapper") +} + +func (f *TestConfigFlags) WithClientConfig(clientConfig clientcmd.ClientConfig) *TestConfigFlags { + f.clientConfig = clientConfig + return f +} + +func (f *TestConfigFlags) WithRESTMapper(mapper meta.RESTMapper) *TestConfigFlags { + f.restMapper = mapper + return f +} + +func (f *TestConfigFlags) WithDiscoveryClient(c discovery.CachedDiscoveryInterface) *TestConfigFlags { + f.discoveryClient = c + return f +} + +func NewTestConfigFlags() *TestConfigFlags { + return &TestConfigFlags{} +} diff --git a/pkg/kubectl/cmd/util/conversion.go b/pkg/kubectl/cmd/util/conversion.go index 960b3e08958..98b02ee7127 100644 --- a/pkg/kubectl/cmd/util/conversion.go +++ b/pkg/kubectl/cmd/util/conversion.go @@ -28,7 +28,7 @@ import ( // TODO update call sites to specify the scheme they want on their builder. func AsDefaultVersionedOrOriginal(obj runtime.Object, mapping *meta.RESTMapping) runtime.Object { converter := runtime.ObjectConvertor(legacyscheme.Scheme) - groupVersioner := runtime.GroupVersioner(schema.GroupVersions(legacyscheme.Registry.RegisteredGroupVersions())) + groupVersioner := runtime.GroupVersioner(schema.GroupVersions(legacyscheme.Scheme.PrioritizedVersionsAllGroups())) if mapping != nil { groupVersioner = mapping.GroupVersionKind.GroupVersion() } diff --git a/pkg/kubectl/cmd/util/editor/BUILD b/pkg/kubectl/cmd/util/editor/BUILD index c0fc7acbc08..c22f03b8b66 100644 --- a/pkg/kubectl/cmd/util/editor/BUILD +++ b/pkg/kubectl/cmd/util/editor/BUILD @@ -15,11 +15,12 @@ go_library( "//build/visible_to:pkg_kubectl_cmd_util_editor_CONSUMERS", ], deps = [ + "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/kubectl:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", "//pkg/kubectl/genericclioptions:go_default_library", - "//pkg/kubectl/resource:go_default_library", + "//pkg/kubectl/genericclioptions/resource:go_default_library", "//pkg/kubectl/scheme:go_default_library", "//pkg/kubectl/util/crlf:go_default_library", "//pkg/kubectl/util/term:go_default_library", diff --git a/pkg/kubectl/cmd/util/editor/editoptions.go b/pkg/kubectl/cmd/util/editor/editoptions.go index 9a0f0cd11b0..47308d4dd23 100644 --- a/pkg/kubectl/cmd/util/editor/editoptions.go +++ b/pkg/kubectl/cmd/util/editor/editoptions.go @@ -42,11 +42,12 @@ import ( "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/kubectl" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/genericclioptions" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/scheme" "k8s.io/kubernetes/pkg/kubectl/util/crlf" "k8s.io/kubernetes/pkg/printers" @@ -88,7 +89,7 @@ func NewEditOptions(editMode EditMode, ioStreams genericclioptions.IOStreams) *E EditMode: editMode, - PrintFlags: printers.NewPrintFlags("edited"), + PrintFlags: printers.NewPrintFlags("edited", legacyscheme.Scheme), WindowsLineEndings: goruntime.GOOS == "windows", diff --git a/pkg/kubectl/cmd/util/factory.go b/pkg/kubectl/cmd/util/factory.go index d4a2356f17a..19880c5fb92 100644 --- a/pkg/kubectl/cmd/util/factory.go +++ b/pkg/kubectl/cmd/util/factory.go @@ -36,29 +36,20 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" scaleclient "k8s.io/client-go/scale" - "k8s.io/client-go/tools/clientcmd" api "k8s.io/kubernetes/pkg/apis/core" apiv1 "k8s.io/kubernetes/pkg/apis/core/v1" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" "k8s.io/kubernetes/pkg/kubectl" - "k8s.io/kubernetes/pkg/kubectl/categories" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/plugins" - "k8s.io/kubernetes/pkg/kubectl/resource" "k8s.io/kubernetes/pkg/kubectl/validation" "k8s.io/kubernetes/pkg/printers" ) -const ( - FlagMatchBinaryVersion = "match-server-version" -) - -var ( - FlagHTTPCacheDir = "cache-dir" -) - // Factory provides abstractions that allow the Kubectl command to be extended across multiple types // of resources and different API sets. // The rings are here for a reason. In order for composers to be able to provide alternative factory implementations @@ -99,6 +90,9 @@ type ClientAccessFactory interface { // KubernetesClientSet gives you back an external clientset KubernetesClientSet() (*kubernetes.Clientset, error) + // Returns interfaces for dealing with arbitrary runtime.Objects. + RESTMapper() (meta.RESTMapper, error) + // Returns a RESTClient for accessing Kubernetes resources or an error. RESTClient() (*restclient.RESTClient, error) // Returns a client.Config for accessing the Kubernetes server. @@ -107,6 +101,10 @@ type ClientAccessFactory interface { // just directions to the server. People use this to build RESTMappers on top of BareClientConfig() (*restclient.Config, error) + // NewBuilder returns an object that assists in loading objects from both disk and the server + // and which implements the common patterns for CLI interactions with generic resources. + NewBuilder() *resource.Builder + // UpdatePodSpecForObject will call the provided function on the pod spec this object supports, // return false if no pod spec is supported, or return an error. UpdatePodSpecForObject(obj runtime.Object, fn func(*v1.PodSpec) error) (bool, error) @@ -125,10 +123,6 @@ type ClientAccessFactory interface { // Command will stringify and return all environment arguments ie. a command run by a client // using the factory. Command(cmd *cobra.Command, showSecrets bool) string - // BindFlags adds any flags that are common to all kubectl sub commands. - BindFlags(flags *pflag.FlagSet) - // BindExternalFlags adds any flags defined by external projects (not part of pflags) - BindExternalFlags(flags *pflag.FlagSet) // SuggestedPodTemplateResources returns a list of resource types that declare a pod template SuggestedPodTemplateResources() []schema.GroupResource @@ -167,10 +161,8 @@ type ClientAccessFactory interface { // ObjectMappingFactory holds the second level of factory methods. These functions depend upon ClientAccessFactory methods. // Generally they provide object typing and functions that build requests based on the negotiated clients. type ObjectMappingFactory interface { - // Returns interfaces for dealing with arbitrary runtime.Objects. - RESTMapper() (meta.RESTMapper, error) // Returns interface for expanding categories like `all`. - CategoryExpander() categories.CategoryExpander + CategoryExpander() (restmapper.CategoryExpander, error) // Returns a RESTClient for working with the specified RESTMapping or an error. This is intended // for working with arbitrary resources and is not guaranteed to point to a Kubernetes APIServer. ClientForMapping(mapping *meta.RESTMapping) (resource.RESTClient, error) @@ -205,9 +197,6 @@ type ObjectMappingFactory interface { // BuilderFactory holds the third level of factory methods. These functions depend upon ObjectMappingFactory and ClientAccessFactory methods. // Generally they depend upon client mapper functions type BuilderFactory interface { - // NewBuilder returns an object that assists in loading objects from both disk and the server - // and which implements the common patterns for CLI interactions with generic resources. - NewBuilder() *resource.Builder // PluginLoader provides the implementation to be used to load cli plugins. PluginLoader() plugins.PluginLoader // PluginRunner provides the implementation to be used to run cli plugins. @@ -227,10 +216,9 @@ type factory struct { } // NewFactory creates a factory with the default Kubernetes resources defined -// if optionalClientConfig is nil, then flags will be bound to a new clientcmd.ClientConfig. -// if optionalClientConfig is not nil, then this factory will make use of it. -func NewFactory(optionalClientConfig clientcmd.ClientConfig) Factory { - clientAccessFactory := NewClientAccessFactory(optionalClientConfig) +// Receives a clientGetter capable of providing a discovery client and a REST client configuration. +func NewFactory(clientGetter RESTClientGetter) Factory { + clientAccessFactory := NewClientAccessFactory(clientGetter) objectMappingFactory := NewObjectMappingFactory(clientAccessFactory) builderFactory := NewBuilderFactory(clientAccessFactory, objectMappingFactory) diff --git a/pkg/kubectl/cmd/util/factory_builder.go b/pkg/kubectl/cmd/util/factory_builder.go index 166659470db..1d9cbead9e1 100644 --- a/pkg/kubectl/cmd/util/factory_builder.go +++ b/pkg/kubectl/cmd/util/factory_builder.go @@ -26,7 +26,6 @@ import ( scaleclient "k8s.io/client-go/scale" "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/plugins" - "k8s.io/kubernetes/pkg/kubectl/resource" ) type ring2Factory struct { @@ -43,18 +42,6 @@ func NewBuilderFactory(clientAccessFactory ClientAccessFactory, objectMappingFac return f } -// NewBuilder returns a new resource builder for structured api objects. -func (f *ring2Factory) NewBuilder() *resource.Builder { - mapper, mapperErr := f.objectMappingFactory.RESTMapper() - - categoryExpander := f.objectMappingFactory.CategoryExpander() - return resource.NewBuilder( - f.clientAccessFactory.ClientConfig, - mapper, - categoryExpander, - ).AddError(mapperErr) -} - // PluginLoader loads plugins from a path set by the KUBECTL_PLUGINS_PATH env var. // If this env var is not set, it defaults to // "~/.kube/plugins", plus @@ -84,7 +71,7 @@ func (f *ring2Factory) ScaleClient() (scaleclient.ScalesGetter, error) { return nil, err } resolver := scaleclient.NewDiscoveryScaleKindResolver(discoClient) - mapper, err := f.objectMappingFactory.RESTMapper() + mapper, err := f.clientAccessFactory.RESTMapper() if err != nil { return nil, err } diff --git a/pkg/kubectl/cmd/util/factory_client_access.go b/pkg/kubectl/cmd/util/factory_client_access.go index a5a4dcd32dc..cc281568d88 100644 --- a/pkg/kubectl/cmd/util/factory_client_access.go +++ b/pkg/kubectl/cmd/util/factory_client_access.go @@ -20,23 +20,18 @@ package util import ( "errors" - "flag" "fmt" "io" - "net/http" "os" "path/filepath" "regexp" "strings" - "time" "k8s.io/api/core/v1" "github.com/spf13/cobra" "github.com/spf13/pflag" - "sync" - appsv1 "k8s.io/api/apps/v1" appsv1beta1 "k8s.io/api/apps/v1beta1" appsv1beta2 "k8s.io/api/apps/v1beta2" @@ -50,138 +45,57 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - utilflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/apps" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/kubectl" - "k8s.io/kubernetes/pkg/kubectl/resource" - "k8s.io/kubernetes/pkg/kubectl/util/transport" - "k8s.io/kubernetes/pkg/version" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" ) -type ring0Factory struct { - flags *pflag.FlagSet - clientConfig clientcmd.ClientConfig - discoveryCacheDir string - - requireMatchedServerVersion bool - checkServerVersion sync.Once - matchesServerVersionErr error +type RESTClientGetter interface { + ToRESTConfig() (*restclient.Config, error) + ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) + ToRESTMapper() (meta.RESTMapper, error) + ToRawKubeConfigLoader() clientcmd.ClientConfig } -func NewClientAccessFactory(optionalClientConfig clientcmd.ClientConfig) ClientAccessFactory { - flags := pflag.NewFlagSet("", pflag.ContinueOnError) +type ring0Factory struct { + clientGetter RESTClientGetter +} - clientConfig := optionalClientConfig - if optionalClientConfig == nil { - clientConfig = DefaultClientConfig(flags) +func NewClientAccessFactory(clientGetter RESTClientGetter) ClientAccessFactory { + if clientGetter == nil { + panic("attempt to instantiate client_access_factory with nil clientGetter") } - flags.SetNormalizeFunc(utilflag.WarnWordSepNormalizeFunc) // Warn for "_" flags - f := &ring0Factory{ - flags: flags, - clientConfig: clientConfig, + clientGetter: clientGetter, } return f } -// DefaultClientConfig creates a clientcmd.ClientConfig with the following hierarchy: -// 1. Use the kubeconfig builder. The number of merges and overrides here gets a little crazy. Stay with me. -// 1. Merge the kubeconfig itself. This is done with the following hierarchy rules: -// 1. CommandLineLocation - this parsed from the command line, so it must be late bound. If you specify this, -// then no other kubeconfig files are merged. This file must exist. -// 2. If $KUBECONFIG is set, then it is treated as a list of files that should be merged. -// 3. HomeDirectoryLocation -// Empty filenames are ignored. Files with non-deserializable content produced errors. -// The first file to set a particular value or map key wins and the value or map key is never changed. -// This means that the first file to set CurrentContext will have its context preserved. It also means -// that if two files specify a "red-user", only values from the first file's red-user are used. Even -// non-conflicting entries from the second file's "red-user" are discarded. -// 2. Determine the context to use based on the first hit in this chain -// 1. command line argument - again, parsed from the command line, so it must be late bound -// 2. CurrentContext from the merged kubeconfig file -// 3. Empty is allowed at this stage -// 3. Determine the cluster info and auth info to use. At this point, we may or may not have a context. They -// are built based on the first hit in this chain. (run it twice, once for auth, once for cluster) -// 1. command line argument -// 2. If context is present, then use the context value -// 3. Empty is allowed -// 4. Determine the actual cluster info to use. At this point, we may or may not have a cluster info. Build -// each piece of the cluster info based on the chain: -// 1. command line argument -// 2. If cluster info is present and a value for the attribute is present, use it. -// 3. If you don't have a server location, bail. -// 5. Auth info is build using the same rules as cluster info, EXCEPT that you can only have one authentication -// technique per auth info. The following conditions result in an error: -// 1. If there are two conflicting techniques specified from the command line, fail. -// 2. If the command line does not specify one, and the auth info has conflicting techniques, fail. -// 3. If the command line specifies one and the auth info specifies another, honor the command line technique. -// 2. Use default values and potentially prompt for auth information -// -// However, if it appears that we're running in a kubernetes cluster -// container environment, then run with the auth info kubernetes mounted for -// us. Specifically: -// The env vars KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT are -// set, and the file /var/run/secrets/kubernetes.io/serviceaccount/token -// exists and is not a directory. -func DefaultClientConfig(flags *pflag.FlagSet) clientcmd.ClientConfig { - loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() - // use the standard defaults for this client command - // DEPRECATED: remove and replace with something more accurate - loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig +func (f *ring0Factory) ClientConfig() (*restclient.Config, error) { + return f.clientGetter.ToRESTConfig() +} - flags.StringVar(&loadingRules.ExplicitPath, "kubeconfig", "", "Path to the kubeconfig file to use for CLI requests.") +func (f *ring0Factory) RESTMapper() (meta.RESTMapper, error) { + return f.clientGetter.ToRESTMapper() +} - overrides := &clientcmd.ConfigOverrides{ClusterDefaults: clientcmd.ClusterDefaults} - - flagNames := clientcmd.RecommendedConfigOverrideFlags("") - // short flagnames are disabled by default. These are here for compatibility with existing scripts - flagNames.ClusterOverrideFlags.APIServer.ShortName = "s" - - clientcmd.BindOverrideFlags(overrides, flags, flagNames) - clientConfig := clientcmd.NewInteractiveDeferredLoadingClientConfig(loadingRules, overrides, os.Stdin) - - return clientConfig +func (f *ring0Factory) BareClientConfig() (*restclient.Config, error) { + return f.clientGetter.ToRESTConfig() } func (f *ring0Factory) DiscoveryClient() (discovery.CachedDiscoveryInterface, error) { - cfg, err := f.clientConfig.ClientConfig() - if err != nil { - return nil, err - } - - // The more groups you have, the more discovery requests you need to make. - // given 25 groups (our groups + a few custom resources) with one-ish version each, discovery needs to make 50 requests - // double it just so we don't end up here again for a while. This config is only used for discovery. - cfg.Burst = 100 - - if f.discoveryCacheDir != "" { - wt := cfg.WrapTransport - cfg.WrapTransport = func(rt http.RoundTripper) http.RoundTripper { - if wt != nil { - rt = wt(rt) - } - return transport.NewCacheRoundTripper(f.discoveryCacheDir, rt) - } - } - - discoveryClient, err := discovery.NewDiscoveryClientForConfig(cfg) - if err != nil { - return nil, err - } - cacheDir := computeDiscoverCacheDir(filepath.Join(homedir.HomeDir(), ".kube", "cache", "discovery"), cfg.Host) - return NewCachedDiscoveryClient(discoveryClient, cacheDir, time.Duration(10*time.Minute)), nil + return f.clientGetter.ToDiscoveryClient() } func (f *ring0Factory) KubernetesClientSet() (*kubernetes.Clientset, error) { @@ -207,35 +121,10 @@ func (f *ring0Factory) DynamicClient() (dynamic.DynamicInterface, error) { } return dynamic.NewForConfig(clientConfig) } -func (f *ring0Factory) checkMatchingServerVersion() error { - f.checkServerVersion.Do(func() { - if !f.requireMatchedServerVersion { - return - } - discoveryClient, err := f.DiscoveryClient() - if err != nil { - f.matchesServerVersionErr = err - return - } - f.matchesServerVersionErr = discovery.MatchesServerVersion(version.Get(), discoveryClient) - }) - return f.matchesServerVersionErr -} - -func (f *ring0Factory) ClientConfig() (*restclient.Config, error) { - if err := f.checkMatchingServerVersion(); err != nil { - return nil, err - } - clientConfig, err := f.clientConfig.ClientConfig() - if err != nil { - return nil, err - } - setKubernetesDefaults(clientConfig) - return clientConfig, nil -} -func (f *ring0Factory) BareClientConfig() (*restclient.Config, error) { - return f.clientConfig.ClientConfig() +// NewBuilder returns a new resource builder for structured api objects. +func (f *ring0Factory) NewBuilder() *resource.Builder { + return resource.NewBuilder(f.clientGetter) } func (f *ring0Factory) RESTClient() (*restclient.RESTClient, error) { @@ -243,6 +132,7 @@ func (f *ring0Factory) RESTClient() (*restclient.RESTClient, error) { if err != nil { return nil, err } + setKubernetesDefaults(clientConfig) return restclient.RESTClientFor(clientConfig) } @@ -412,29 +302,6 @@ func (f *ring0Factory) Command(cmd *cobra.Command, showSecrets bool) string { return base + args + flags } -func (f *ring0Factory) BindFlags(flags *pflag.FlagSet) { - // Merge factory's flags - flags.AddFlagSet(f.flags) - - // Globally persistent flags across all subcommands. - // TODO Change flag names to consts to allow safer lookup from subcommands. - // TODO Add a verbose flag that turns on glog logging. Probably need a way - // to do that automatically for every subcommand. - flags.BoolVar(&f.requireMatchedServerVersion, FlagMatchBinaryVersion, false, "Require server version to match client version") - - defaultCacheDir := filepath.Join(homedir.HomeDir(), ".kube", "http-cache") - flags.StringVar(&f.discoveryCacheDir, FlagHTTPCacheDir, defaultCacheDir, "Default HTTP cache directory") - - // Normalize all flags that are coming from other packages or pre-configurations - // a.k.a. change all "_" to "-". e.g. glog package - flags.SetNormalizeFunc(utilflag.WordSepNormalizeFunc) -} - -func (f *ring0Factory) BindExternalFlags(flags *pflag.FlagSet) { - // any flags defined by external projects (not part of pflags) - flags.AddGoFlagSet(flag.CommandLine) -} - func (f *ring0Factory) SuggestedPodTemplateResources() []schema.GroupResource { return []schema.GroupResource{ {Resource: "replicationcontroller"}, @@ -476,7 +343,7 @@ func (f *ring0Factory) Resumer(info *resource.Info) ([]byte, error) { } func (f *ring0Factory) DefaultNamespace() (string, bool, error) { - return f.clientConfig.Namespace() + return f.clientGetter.ToRawKubeConfigLoader().Namespace() } const ( @@ -734,22 +601,6 @@ func InternalVersionDecoder() runtime.Decoder { } func InternalVersionJSONEncoder() runtime.Encoder { - encoder := legacyscheme.Codecs.LegacyCodec(legacyscheme.Registry.RegisteredGroupVersions()...) + encoder := legacyscheme.Codecs.LegacyCodec(legacyscheme.Scheme.PrioritizedVersionsAllGroups()...) return unstructured.JSONFallbackEncoder{Encoder: encoder} } - -// setKubernetesDefaults sets default values on the provided client config for accessing the -// Kubernetes API or returns an error if any of the defaults are impossible or invalid. -// TODO this isn't what we want. Each clientset should be setting defaults as it sees fit. -func setKubernetesDefaults(config *restclient.Config) error { - // TODO remove this hack. This is allowing the GetOptions to be serialized. - config.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"} - - if config.APIPath == "" { - config.APIPath = "/api" - } - if config.NegotiatedSerializer == nil { - config.NegotiatedSerializer = legacyscheme.Codecs - } - return restclient.SetKubernetesDefaults(config) -} diff --git a/pkg/kubectl/cmd/util/factory_object_mapping.go b/pkg/kubectl/cmd/util/factory_object_mapping.go index 14876e25d1c..2e7bc0aa8fb 100644 --- a/pkg/kubectl/cmd/util/factory_object_mapping.go +++ b/pkg/kubectl/cmd/util/factory_object_mapping.go @@ -38,9 +38,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/batch" api "k8s.io/kubernetes/pkg/apis/core" @@ -48,10 +48,9 @@ import ( "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/kubectl" - "k8s.io/kubernetes/pkg/kubectl/categories" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi" openapivalidation "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi/validation" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" "k8s.io/kubernetes/pkg/kubectl/validation" "k8s.io/kubernetes/pkg/printers" printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" @@ -76,37 +75,13 @@ func NewObjectMappingFactory(clientAccessFactory ClientAccessFactory) ObjectMapp return f } -// RESTMapper returns a mapper. -func (f *ring1Factory) RESTMapper() (meta.RESTMapper, error) { +func (f *ring1Factory) CategoryExpander() (restmapper.CategoryExpander, error) { discoveryClient, err := f.clientAccessFactory.DiscoveryClient() if err != nil { return nil, err } - // allow conversion between typed and unstructured objects - mapper := discovery.NewDeferredDiscoveryRESTMapper(discoveryClient) - // TODO: should this also indicate it recognizes typed objects? - expander := NewShortcutExpander(mapper, discoveryClient) - return expander, nil -} - -func (f *ring1Factory) CategoryExpander() categories.CategoryExpander { - legacyExpander := categories.LegacyCategoryExpander - - discoveryClient, err := f.clientAccessFactory.DiscoveryClient() - if err == nil { - // fallback is the legacy expander wrapped with discovery based filtering - fallbackExpander, err := categories.NewDiscoveryFilteredExpander(legacyExpander, discoveryClient) - CheckErr(err) - - // by default use the expander that discovers based on "categories" field from the API - discoveryCategoryExpander, err := categories.NewDiscoveryCategoryExpander(fallbackExpander, discoveryClient) - CheckErr(err) - - return discoveryCategoryExpander - } - - return legacyExpander + return restmapper.NewDiscoveryCategoryExpander(discoveryClient), nil } func (f *ring1Factory) ClientForMapping(mapping *meta.RESTMapping) (resource.RESTClient, error) { diff --git a/pkg/kubectl/cmd/util/factory_test.go b/pkg/kubectl/cmd/util/factory_test.go index e936a2d9800..a11473cf90f 100644 --- a/pkg/kubectl/cmd/util/factory_test.go +++ b/pkg/kubectl/cmd/util/factory_test.go @@ -33,18 +33,18 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" manualfake "k8s.io/client-go/rest/fake" + "k8s.io/client-go/restmapper" testcore "k8s.io/client-go/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/kubectl" - "k8s.io/kubernetes/pkg/kubectl/categories" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" ) func TestPortsForObject(t *testing.T) { - f := NewFactory(nil) + f := NewFactory(NewTestConfigFlags()) pod := &api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "baz", Namespace: "test", ResourceVersion: "12"}, @@ -75,7 +75,7 @@ func TestPortsForObject(t *testing.T) { } func TestProtocolsForObject(t *testing.T) { - f := NewFactory(nil) + f := NewFactory(NewTestConfigFlags()) pod := &api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "baz", Namespace: "test", ResourceVersion: "12"}, @@ -113,7 +113,7 @@ func TestProtocolsForObject(t *testing.T) { } func TestLabelsForObject(t *testing.T) { - f := NewFactory(nil) + f := NewFactory(NewTestConfigFlags()) tests := []struct { name string @@ -164,7 +164,7 @@ func TestLabelsForObject(t *testing.T) { } func TestCanBeExposed(t *testing.T) { - factory := NewFactory(nil) + factory := NewFactory(NewTestConfigFlags()) tests := []struct { kind schema.GroupKind expectErr bool @@ -470,8 +470,8 @@ func TestDiscoveryReplaceAliases(t *testing.T) { } ds := &fakeDiscoveryClient{} - mapper := NewShortcutExpander(testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme), ds) - b := resource.NewFakeBuilder(fakeClient(), mapper, categories.LegacyCategoryExpander) + mapper := restmapper.NewShortcutExpander(testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme), ds) + b := resource.NewFakeBuilder(fakeClient(), mapper, resource.FakeCategoryExpander) for _, test := range tests { replaced := b.ReplaceAliases(test.arg) diff --git a/pkg/kubectl/cmd/util/helpers.go b/pkg/kubectl/cmd/util/helpers.go index edff737ac79..1e5aa7dc6a1 100644 --- a/pkg/kubectl/cmd/util/helpers.go +++ b/pkg/kubectl/cmd/util/helpers.go @@ -39,7 +39,7 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/tools/clientcmd" "k8s.io/kubernetes/pkg/kubectl" - "k8s.io/kubernetes/pkg/kubectl/resource" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource" utilexec "k8s.io/utils/exec" ) diff --git a/pkg/kubectl/cmd/util/kubectl_match_version.go b/pkg/kubectl/cmd/util/kubectl_match_version.go new file mode 100644 index 00000000000..954216dc4f9 --- /dev/null +++ b/pkg/kubectl/cmd/util/kubectl_match_version.go @@ -0,0 +1,125 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "sync" + + "github.com/spf13/pflag" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/kubernetes/pkg/version" +) + +const ( + flagMatchBinaryVersion = "match-server-version" +) + +// MatchVersionFlags is for setting the "match server version" function. +type MatchVersionFlags struct { + Delegate RESTClientGetter + + RequireMatchedServerVersion bool + checkServerVersion sync.Once + matchesServerVersionErr error +} + +var _ RESTClientGetter = &MatchVersionFlags{} + +func (f *MatchVersionFlags) checkMatchingServerVersion() error { + f.checkServerVersion.Do(func() { + if !f.RequireMatchedServerVersion { + return + } + discoveryClient, err := f.Delegate.ToDiscoveryClient() + if err != nil { + f.matchesServerVersionErr = err + return + } + f.matchesServerVersionErr = discovery.MatchesServerVersion(version.Get(), discoveryClient) + }) + + return f.matchesServerVersionErr +} + +// ToRESTConfig implements RESTClientGetter. +// Returns a REST client configuration based on a provided path +// to a .kubeconfig file, loading rules, and config flag overrides. +// Expects the AddFlags method to have been called. +func (f *MatchVersionFlags) ToRESTConfig() (*rest.Config, error) { + if err := f.checkMatchingServerVersion(); err != nil { + return nil, err + } + clientConfig, err := f.Delegate.ToRESTConfig() + if err != nil { + return nil, err + } + // TODO we should not have to do this. It smacks of something going wrong. + setKubernetesDefaults(clientConfig) + return clientConfig, nil +} + +func (f *MatchVersionFlags) ToRawKubeConfigLoader() clientcmd.ClientConfig { + return f.Delegate.ToRawKubeConfigLoader() +} + +func (f *MatchVersionFlags) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { + if err := f.checkMatchingServerVersion(); err != nil { + return nil, err + } + return f.Delegate.ToDiscoveryClient() +} + +// RESTMapper returns a mapper. +func (f *MatchVersionFlags) ToRESTMapper() (meta.RESTMapper, error) { + if err := f.checkMatchingServerVersion(); err != nil { + return nil, err + } + return f.Delegate.ToRESTMapper() +} + +func (f *MatchVersionFlags) AddFlags(flags *pflag.FlagSet) { + flags.BoolVar(&f.RequireMatchedServerVersion, flagMatchBinaryVersion, f.RequireMatchedServerVersion, "Require server version to match client version") +} + +func NewMatchVersionFlags(delegate RESTClientGetter) *MatchVersionFlags { + return &MatchVersionFlags{ + Delegate: delegate, + } +} + +// setKubernetesDefaults sets default values on the provided client config for accessing the +// Kubernetes API or returns an error if any of the defaults are impossible or invalid. +// TODO this isn't what we want. Each clientset should be setting defaults as it sees fit. +func setKubernetesDefaults(config *rest.Config) error { + // TODO remove this hack. This is allowing the GetOptions to be serialized. + config.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"} + + if config.APIPath == "" { + config.APIPath = "/api" + } + if config.NegotiatedSerializer == nil { + config.NegotiatedSerializer = legacyscheme.Codecs + } + return rest.SetKubernetesDefaults(config) +} diff --git a/pkg/kubectl/cmd/util/printing.go b/pkg/kubectl/cmd/util/printing.go index ef3a0eda50e..dab4792c1ee 100644 --- a/pkg/kubectl/cmd/util/printing.go +++ b/pkg/kubectl/cmd/util/printing.go @@ -18,222 +18,10 @@ package util import ( "fmt" - "io" - "strings" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" - kubectlscheme "k8s.io/kubernetes/pkg/kubectl/scheme" - "k8s.io/kubernetes/pkg/printers" - printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" - - "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/kubernetes/pkg/api/legacyscheme" ) -// AddPrinterFlags adds printing related flags to a command (e.g. output format, no headers, template path) -func AddPrinterFlags(cmd *cobra.Command) { - AddNonDeprecatedPrinterFlags(cmd) - - cmd.Flags().String("output-version", "", "DEPRECATED: To use a specific API version, fully-qualify the resource, version, and group (for example: 'jobs.v1.batch/myjob').") - cmd.Flags().MarkDeprecated("output-version", "The resource is used exactly as fetched from the API. To get a specific API version, fully-qualify the resource, version, and group (for example: 'jobs.v1.batch/myjob').") - cmd.Flags().MarkHidden("output-version") -} - -// AddNonDeprecatedPrinterFlags supports the conversion case which must logically have output-version. Once output-version -// is completely removed, this function can go away. -func AddNonDeprecatedPrinterFlags(cmd *cobra.Command) { - AddOutputFlags(cmd) - AddNoHeadersFlags(cmd) - cmd.Flags().Bool("show-labels", false, "When printing, show all labels as the last column (default hide labels column)") - cmd.Flags().String("template", "", "Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].") - cmd.MarkFlagFilename("template") - cmd.Flags().String("sort-by", "", "If non-empty, sort list types using this field specification. The field specification is expressed as a JSONPath expression (e.g. '{.metadata.name}'). The field in the API resource specified by this JSONPath expression must be an integer or a string.") - cmd.Flags().BoolP("show-all", "a", true, "When printing, show all resources (default show all pods including terminated one.)") - cmd.Flags().MarkDeprecated("show-all", "will be removed in an upcoming release") -} - -// AddOutputFlagsForMutation adds output related flags to a command. Used by mutations only. -func AddOutputFlagsForMutation(cmd *cobra.Command) { - cmd.Flags().StringP("output", "o", "", "Output mode. Use \"-o name\" for shorter output (resource/name).") -} - -// AddOutputVarFlagsForMutation adds output related flags to a command. Used by mutations only. -func AddOutputVarFlagsForMutation(cmd *cobra.Command, output *string) { - cmd.Flags().StringVarP(output, "output", "o", *output, "Output mode. Use \"-o name\" for shorter output (resource/name).") -} - -// AddOutputFlags adds output related flags to a command. -func AddOutputFlags(cmd *cobra.Command) { - cmd.Flags().StringP("output", "o", "", "Output format. One of: json|yaml|wide|name|custom-columns=...|custom-columns-file=...|go-template=...|go-template-file=...|jsonpath=...|jsonpath-file=... See custom columns [http://kubernetes.io/docs/user-guide/kubectl-overview/#custom-columns], golang template [http://golang.org/pkg/text/template/#pkg-overview] and jsonpath template [http://kubernetes.io/docs/user-guide/jsonpath].") - cmd.Flags().Bool("allow-missing-template-keys", true, "If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.") -} - -// AddNoHeadersFlags adds no-headers flags to a command. -func AddNoHeadersFlags(cmd *cobra.Command) { - cmd.Flags().Bool("no-headers", false, "When using the default or custom-column output format, don't print headers (default print headers).") -} - -// ValidateOutputArgs validates -o flag args for mutations -func ValidateOutputArgs(cmd *cobra.Command) error { - outputMode := GetFlagString(cmd, "output") - if outputMode != "" && outputMode != "name" { - return UsageErrorf(cmd, "Unexpected -o output mode: %v. We only support '-o name'.", outputMode) - } - return nil -} - -// PrintSuccess prints a success message and can do a "-o name" as "shortOutput" -// TODO this should really just be a printer. It's got just about the exact same signature. -func PrintSuccess(shortOutput bool, out io.Writer, obj runtime.Object, dryRun bool, operation string) { - dryRunMsg := "" - if dryRun { - dryRunMsg = " (dry run)" - } - - // match name printer format - name := "" - if acc, err := meta.Accessor(obj); err == nil { - if n := acc.GetName(); len(n) > 0 { - name = n - } - } - - // legacy scheme to be sure we work ok with internal types. - // TODO internal types aren't supposed to exist here - groupKind := printers.GetObjectGroupKind(obj, legacyscheme.Scheme) - kindString := fmt.Sprintf("%s.%s", strings.ToLower(groupKind.Kind), groupKind.Group) - if len(groupKind.Group) == 0 { - kindString = strings.ToLower(groupKind.Kind) - } - - if shortOutput { - // -o name: prints resource/name - fmt.Fprintf(out, "%s/%s\n", kindString, name) - return - } - - // understandable output by default - fmt.Fprintf(out, "%s \"%s\" %s%s\n", kindString, name, operation, dryRunMsg) -} - -// PrintObject prints a single object based on the default command options -// TODO this should go away once commands can embed the PrintOptions instead -func PrintObject(cmd *cobra.Command, obj runtime.Object, out io.Writer) error { - printer, err := PrinterForOptions(ExtractCmdPrintOptions(cmd, false)) - if err != nil { - return err - } - return printer.PrintObj(obj, out) -} - -// PrinterForOptions returns the printer for the outputOptions (if given) or -// returns the default printer for the command. -// TODO this should become a function on the PrintOptions struct -func PrinterForOptions(options *printers.PrintOptions) (printers.ResourcePrinter, error) { - // TODO: used by the custom column implementation and the name implementation, break this dependency - decoders := []runtime.Decoder{kubectlscheme.Codecs.UniversalDecoder(), unstructured.UnstructuredJSONScheme} - encoder := kubectlscheme.Codecs.LegacyCodec(kubectlscheme.Registry.RegisteredGroupVersions()...) - - printer, err := printers.GetStandardPrinter(kubectlscheme.Scheme, encoder, decoders, *options) - if err != nil { - return nil, err - } - - // we try to convert to HumanReadablePrinter, if return ok, it must be no generic - // we execute AddHandlers() here before maybeWrapSortingPrinter so that we don't - // need to convert to delegatePrinter again then invoke AddHandlers() - // TODO this looks highly questionable. human readable printers are baked into code. This can just live in the definition of the handler itself - // TODO or be registered there - if humanReadablePrinter, ok := printer.(printers.PrintHandler); ok { - printersinternal.AddHandlers(humanReadablePrinter) - } - - printer = maybeWrapSortingPrinter(printer, *options) - - // wrap the printer in a versioning printer that understands when to convert and when not to convert - printer = printers.NewVersionedPrinter(printer, legacyscheme.Scheme, legacyscheme.Scheme, kubectlscheme.Versions...) - - return printer, nil -} - -// ExtractCmdPrintOptions parses printer specific commandline args and -// returns a PrintOptions object. -// Requires that printer flags have been added to cmd (see AddPrinterFlags) -func ExtractCmdPrintOptions(cmd *cobra.Command, withNamespace bool) *printers.PrintOptions { - flags := cmd.Flags() - - columnLabel, err := flags.GetStringSlice("label-columns") - if err != nil { - columnLabel = []string{} - } - - options := &printers.PrintOptions{ - NoHeaders: GetFlagBool(cmd, "no-headers"), - Wide: GetWideFlag(cmd), - ShowAll: GetFlagBool(cmd, "show-all"), - ShowLabels: GetFlagBool(cmd, "show-labels"), - AbsoluteTimestamps: isWatch(cmd), - ColumnLabels: columnLabel, - WithNamespace: withNamespace, - } - - var outputFormat string - if flags.Lookup("output") != nil { - outputFormat = GetFlagString(cmd, "output") - } - - if flags.Lookup("sort-by") != nil { - options.SortBy = GetFlagString(cmd, "sort-by") - } - - // templates are logically optional for specifying a format. - // TODO once https://github.com/kubernetes/kubernetes/issues/12668 is fixed, this should fall back to GetFlagString - var templateFile string - if flag := flags.Lookup("template"); flag != nil { - if flag.Value.Type() == "string" { - templateFile = GetFlagString(cmd, "template") - } - } - if len(outputFormat) == 0 && len(templateFile) != 0 { - outputFormat = "template" - } - - templateFormats := []string{ - "go-template=", "go-template-file=", "jsonpath=", "jsonpath-file=", "custom-columns=", "custom-columns-file=", - } - for _, format := range templateFormats { - if strings.HasPrefix(outputFormat, format) { - templateFile = outputFormat[len(format):] - outputFormat = format[:len(format)-1] - } - } - - // this function may be invoked by a command that did not call AddPrinterFlags first, so we need - // to be safe about how we access the allow-missing-template-keys flag - if flags.Lookup("allow-missing-template-keys") != nil { - options.AllowMissingKeys = GetFlagBool(cmd, "allow-missing-template-keys") - } - - options.OutputFormatType = outputFormat - options.OutputFormatArgument = templateFile - - return options -} - -func maybeWrapSortingPrinter(printer printers.ResourcePrinter, printOpts printers.PrintOptions) printers.ResourcePrinter { - if len(printOpts.SortBy) != 0 { - return &kubectl.SortingPrinter{ - Delegate: printer, - SortField: fmt.Sprintf("{%s}", printOpts.SortBy), - } - } - return printer -} - // SuggestApiResources returns a suggestion to use the "api-resources" command // to retrieve a supported list of resources func SuggestApiResources(parent string) string { diff --git a/pkg/kubectl/util/transport/BUILD b/pkg/kubectl/cmd/util/transport/BUILD similarity index 92% rename from pkg/kubectl/util/transport/BUILD rename to pkg/kubectl/cmd/util/transport/BUILD index c077a903c5b..0e968c1fb2a 100644 --- a/pkg/kubectl/util/transport/BUILD +++ b/pkg/kubectl/cmd/util/transport/BUILD @@ -3,7 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = ["round_tripper.go"], - importpath = "k8s.io/kubernetes/pkg/kubectl/util/transport", + importpath = "k8s.io/kubernetes/pkg/kubectl/cmd/util/transport", visibility = ["//visibility:public"], deps = [ "//vendor/github.com/gregjones/httpcache:go_default_library", diff --git a/pkg/kubectl/util/transport/round_tripper.go b/pkg/kubectl/cmd/util/transport/round_tripper.go similarity index 100% rename from pkg/kubectl/util/transport/round_tripper.go rename to pkg/kubectl/cmd/util/transport/round_tripper.go diff --git a/pkg/kubectl/util/transport/round_tripper_test.go b/pkg/kubectl/cmd/util/transport/round_tripper_test.go similarity index 100% rename from pkg/kubectl/util/transport/round_tripper_test.go rename to pkg/kubectl/cmd/util/transport/round_tripper_test.go diff --git a/pkg/kubectl/explain/explain_test.go b/pkg/kubectl/explain/explain_test.go index 04ec14621ad..2c8c3588847 100644 --- a/pkg/kubectl/explain/explain_test.go +++ b/pkg/kubectl/explain/explain_test.go @@ -57,7 +57,7 @@ func TestSplitAndParseResourceRequest(t *testing.T) { }, } - mapper := testrestmapper.TestOnlyStaticRESTMapper(scheme.Registry, scheme.Scheme, scheme.Versions...) + mapper := testrestmapper.TestOnlyStaticRESTMapper(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...) for _, test := range tests { gotInResource, gotFieldsPath, err := SplitAndParseResourceRequest(test.inresource, mapper) if err != nil { diff --git a/pkg/kubectl/genericclioptions/BUILD b/pkg/kubectl/genericclioptions/BUILD index 0dabfc5309a..5d1b8e69398 100644 --- a/pkg/kubectl/genericclioptions/BUILD +++ b/pkg/kubectl/genericclioptions/BUILD @@ -27,7 +27,10 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//pkg/kubectl/genericclioptions/resource:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/pkg/kubectl/resource/BUILD b/pkg/kubectl/genericclioptions/resource/BUILD similarity index 86% rename from pkg/kubectl/resource/BUILD rename to pkg/kubectl/genericclioptions/resource/BUILD index a2df4dc864f..a8eda3b99a3 100644 --- a/pkg/kubectl/resource/BUILD +++ b/pkg/kubectl/genericclioptions/resource/BUILD @@ -1,8 +1,4 @@ -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -10,6 +6,7 @@ go_library( "builder.go", "client.go", "doc.go", + "fake.go", "helper.go", "interfaces.go", "mapper.go", @@ -17,13 +14,9 @@ go_library( "selector.go", "visitor.go", ], - importpath = "k8s.io/kubernetes/pkg/kubectl/resource", - visibility = [ - "//build/visible_to:pkg_kubectl_resource_CONSUMERS", - ], + importpath = "k8s.io/kubernetes/pkg/kubectl/genericclioptions/resource", + visibility = ["//visibility:public"], deps = [ - "//pkg/kubectl/categories:go_default_library", - "//pkg/kubectl/validation:go_default_library", "//vendor/golang.org/x/text/encoding/unicode:go_default_library", "//vendor/golang.org/x/text/transform:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", @@ -42,8 +35,10 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", + "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", ], ) @@ -60,9 +55,6 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//pkg/apis/core/install:go_default_library", - "//pkg/kubectl/categories:go_default_library", - "//pkg/kubectl/scheme:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", @@ -91,13 +83,12 @@ filegroup( name = "package-srcs", srcs = glob(["**"]), tags = ["automanaged"], + visibility = ["//visibility:private"], ) filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], - visibility = [ - "//build/visible_to:pkg_kubectl_resource_CONSUMERS", - ], + visibility = ["//visibility:public"], ) diff --git a/pkg/kubectl/resource/builder.go b/pkg/kubectl/genericclioptions/resource/builder.go similarity index 97% rename from pkg/kubectl/resource/builder.go rename to pkg/kubectl/genericclioptions/resource/builder.go index f41df2f7b10..e92b2b48936 100644 --- a/pkg/kubectl/resource/builder.go +++ b/pkg/kubectl/genericclioptions/resource/builder.go @@ -34,8 +34,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/kubernetes/pkg/kubectl/categories" - "k8s.io/kubernetes/pkg/kubectl/validation" + "k8s.io/client-go/restmapper" ) var FileExtensions = []string{".json", ".yaml", ".yml"} @@ -47,7 +46,7 @@ const defaultHttpGetAttempts int = 3 // from the command line and converting them to a list of resources to iterate // over using the Visitor interface. type Builder struct { - categoryExpander categories.CategoryExpander + categoryExpander restmapper.CategoryExpander // mapper is set explicitly by resource builders mapper *mapper @@ -105,7 +104,7 @@ type Builder struct { export bool - schema validation.Schema + schema ContentValidator // fakeClientFn is used for testing fakeClientFn FakeClientFunc @@ -143,8 +142,8 @@ type resourceTuple struct { type FakeClientFunc func(version schema.GroupVersion) (RESTClient, error) -func NewFakeBuilder(fakeClientFn FakeClientFunc, restMapper meta.RESTMapper, categoryExpander categories.CategoryExpander) *Builder { - ret := NewBuilder(nil, restMapper, categoryExpander) +func NewFakeBuilder(fakeClientFn FakeClientFunc, restMapper meta.RESTMapper, categoryExpander restmapper.CategoryExpander) *Builder { + ret := newBuilder(nil, restMapper, categoryExpander) ret.fakeClientFn = fakeClientFn return ret } @@ -153,7 +152,7 @@ func NewFakeBuilder(fakeClientFn FakeClientFunc, restMapper meta.RESTMapper, cat // internal or unstructured must be specified. // TODO: Add versioned client (although versioned is still lossy) // TODO remove internal and unstructured mapper and instead have them set the negotiated serializer for use in the client -func NewBuilder(clientConfigFn ClientConfigFunc, restMapper meta.RESTMapper, categoryExpander categories.CategoryExpander) *Builder { +func newBuilder(clientConfigFn ClientConfigFunc, restMapper meta.RESTMapper, categoryExpander restmapper.CategoryExpander) *Builder { return &Builder{ clientConfigFn: clientConfigFn, restMapper: restMapper, @@ -162,7 +161,22 @@ func NewBuilder(clientConfigFn ClientConfigFunc, restMapper meta.RESTMapper, cat } } -func (b *Builder) Schema(schema validation.Schema) *Builder { +func NewBuilder(restClientGetter RESTClientGetter) *Builder { + restMapper, mapperErr := restClientGetter.ToRESTMapper() + discoveryClient, discoveryErr := restClientGetter.ToDiscoveryClient() + var categoryExpander restmapper.CategoryExpander + if discoveryErr == nil { + categoryExpander = restmapper.NewDiscoveryCategoryExpander(discoveryClient) + } + + return newBuilder( + restClientGetter.ToRESTConfig, + restMapper, + categoryExpander, + ).AddError(mapperErr).AddError(discoveryErr) +} + +func (b *Builder) Schema(schema ContentValidator) *Builder { b.schema = schema return b } @@ -543,6 +557,9 @@ func (b *Builder) ResourceTypeOrNameArgs(allowEmptySelector bool, args ...string func (b *Builder) ReplaceAliases(input string) string { replaced := []string{} for _, arg := range strings.Split(input, ",") { + if b.categoryExpander == nil { + continue + } if resources, ok := b.categoryExpander.Expand(arg); ok { asStrings := []string{} for _, resource := range resources { diff --git a/pkg/kubectl/resource/builder_test.go b/pkg/kubectl/genericclioptions/resource/builder_test.go similarity index 96% rename from pkg/kubectl/resource/builder_test.go rename to pkg/kubectl/genericclioptions/resource/builder_test.go index 0d115f4d5ab..517b0d8e61d 100644 --- a/pkg/kubectl/resource/builder_test.go +++ b/pkg/kubectl/genericclioptions/resource/builder_test.go @@ -29,9 +29,9 @@ import ( "strings" "testing" + "github.com/davecgh/go-spew/spew" "github.com/ghodss/yaml" - "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/meta/testrestmapper" @@ -47,19 +47,16 @@ import ( "k8s.io/client-go/rest/fake" restclientwatch "k8s.io/client-go/rest/watch" utiltesting "k8s.io/client-go/util/testing" - "k8s.io/kubernetes/pkg/kubectl/categories" - "k8s.io/kubernetes/pkg/kubectl/scheme" - // install the pod scheme into the legacy scheme for test typer resolution - "github.com/davecgh/go-spew/spew" - _ "k8s.io/kubernetes/pkg/apis/core/install" + // TODO we need to remove this linkage and create our own scheme + "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" ) var ( corev1GV = schema.GroupVersion{Version: "v1"} corev1Codec = scheme.Codecs.CodecForVersions(scheme.Codecs.LegacyCodec(corev1GV), scheme.Codecs.UniversalDecoder(corev1GV), corev1GV, corev1GV) metaAccessor = meta.NewAccessor() - restmapper = testrestmapper.TestOnlyStaticRESTMapper(scheme.Registry, scheme.Scheme) ) func stringBody(body string) io.ReadCloser { @@ -273,13 +270,13 @@ func newDefaultBuilder() *Builder { } func newDefaultBuilderWith(fakeClientFn FakeClientFunc) *Builder { - return NewFakeBuilder(fakeClientFn, restmapper, categories.LegacyCategoryExpander). - WithScheme(scheme.Scheme, scheme.Registry.RegisteredGroupVersions()...) + return NewFakeBuilder(fakeClientFn, testrestmapper.TestOnlyStaticRESTMapper(scheme.Scheme), FakeCategoryExpander). + WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...) } func TestPathBuilderAndVersionedObjectNotDefaulted(t *testing.T) { b := newDefaultBuilder(). - FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../test/fixtures/pkg/kubectl/builder/kitten-rc.yaml"}}) + FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../../test/fixtures/pkg/kubectl/builder/kitten-rc.yaml"}}) test := &testVisitor{} singleItemImplied := false @@ -373,9 +370,9 @@ func TestPathBuilderWithMultiple(t *testing.T) { directory string expectedNames []string }{ - {"pod", &v1.Pod{}, false, "../../../test/e2e/testing-manifests/pod", []string{"nginx"}}, + {"pod", &v1.Pod{}, false, "../../../../test/e2e/testing-manifests/pod", []string{"nginx"}}, {"recursive-pod", &v1.Pod{}, true, fmt.Sprintf("%s/recursive/pod", tmpDir), []string{"busybox0", "busybox1"}}, - {"rc", &v1.ReplicationController{}, false, "../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml", []string{"redis-master"}}, + {"rc", &v1.ReplicationController{}, false, "../../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml", []string{"redis-master"}}, {"recursive-rc", &v1.ReplicationController{}, true, fmt.Sprintf("%s/recursive/rc", tmpDir), []string{"busybox0", "busybox1"}}, {"hardlink", &v1.Pod{}, false, fmt.Sprintf("%s/inode/hardlink/busybox-link.json", tmpDir), []string{"busybox0"}}, {"hardlink", &v1.Pod{}, true, fmt.Sprintf("%s/inode/hardlink/busybox-link.json", tmpDir), []string{"busybox0"}}, @@ -457,7 +454,7 @@ func TestPathBuilderWithMultipleInvalid(t *testing.T) { func TestDirectoryBuilder(t *testing.T) { b := newDefaultBuilder(). - FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../test/e2e/testing-manifests/guestbook/legacy"}}). + FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../../test/e2e/testing-manifests/guestbook/legacy"}}). NamespaceParam("test").DefaultNamespace() test := &testVisitor{} @@ -1027,7 +1024,7 @@ func TestContinueOnErrorVisitor(t *testing.T) { func TestSingleItemImpliedObject(t *testing.T) { obj, err := newDefaultBuilder(). NamespaceParam("test").DefaultNamespace(). - FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml"}}). + FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml"}}). Flatten(). Do().Object() @@ -1047,7 +1044,7 @@ func TestSingleItemImpliedObject(t *testing.T) { func TestSingleItemImpliedObjectNoExtension(t *testing.T) { obj, err := newDefaultBuilder(). NamespaceParam("test").DefaultNamespace(). - FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../test/e2e/testing-manifests/pod"}}). + FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../../test/e2e/testing-manifests/pod"}}). Flatten(). Do().Object() @@ -1158,7 +1155,7 @@ func TestWatch(t *testing.T) { }), })). NamespaceParam("test").DefaultNamespace(). - FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../test/e2e/testing-manifests/guestbook/redis-master-service.yaml"}}).Flatten(). + FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../../test/e2e/testing-manifests/guestbook/redis-master-service.yaml"}}).Flatten(). Do().Watch("12") if err != nil { @@ -1185,8 +1182,8 @@ func TestWatch(t *testing.T) { func TestWatchMultipleError(t *testing.T) { _, err := newDefaultBuilder(). NamespaceParam("test").DefaultNamespace(). - FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml"}}).Flatten(). - FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml"}}).Flatten(). + FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml"}}).Flatten(). + FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../../test/e2e/testing-manifests/guestbook/legacy/redis-master-controller.yaml"}}).Flatten(). Do().Watch("") if err == nil { diff --git a/pkg/kubectl/resource/client.go b/pkg/kubectl/genericclioptions/resource/client.go similarity index 100% rename from pkg/kubectl/resource/client.go rename to pkg/kubectl/genericclioptions/resource/client.go diff --git a/pkg/kubectl/resource/doc.go b/pkg/kubectl/genericclioptions/resource/doc.go similarity index 94% rename from pkg/kubectl/resource/doc.go rename to pkg/kubectl/genericclioptions/resource/doc.go index 94944dfe397..a0e22e7cf78 100644 --- a/pkg/kubectl/resource/doc.go +++ b/pkg/kubectl/genericclioptions/resource/doc.go @@ -21,4 +21,4 @@ limitations under the License. // standard command line arguments and parameters into a Visitor that can iterate // over all of the identified resources, whether on the server or on the local // filesystem. -package resource // import "k8s.io/kubernetes/pkg/kubectl/resource" +package resource diff --git a/pkg/kubectl/genericclioptions/resource/fake.go b/pkg/kubectl/genericclioptions/resource/fake.go new file mode 100644 index 00000000000..276c343e213 --- /dev/null +++ b/pkg/kubectl/genericclioptions/resource/fake.go @@ -0,0 +1,40 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/restmapper" +) + +// FakeCategoryExpander is for testing only +var FakeCategoryExpander restmapper.CategoryExpander = restmapper.SimpleCategoryExpander{ + Expansions: map[string][]schema.GroupResource{ + "all": { + {Group: "", Resource: "pods"}, + {Group: "", Resource: "replicationcontrollers"}, + {Group: "", Resource: "services"}, + {Group: "apps", Resource: "statefulsets"}, + {Group: "autoscaling", Resource: "horizontalpodautoscalers"}, + {Group: "batch", Resource: "jobs"}, + {Group: "batch", Resource: "cronjobs"}, + {Group: "extensions", Resource: "daemonsets"}, + {Group: "extensions", Resource: "deployments"}, + {Group: "extensions", Resource: "replicasets"}, + }, + }, +} diff --git a/pkg/kubectl/resource/helper.go b/pkg/kubectl/genericclioptions/resource/helper.go similarity index 100% rename from pkg/kubectl/resource/helper.go rename to pkg/kubectl/genericclioptions/resource/helper.go diff --git a/pkg/kubectl/resource/helper_test.go b/pkg/kubectl/genericclioptions/resource/helper_test.go similarity index 99% rename from pkg/kubectl/resource/helper_test.go rename to pkg/kubectl/genericclioptions/resource/helper_test.go index 95603ae392f..e9cf7b1c968 100644 --- a/pkg/kubectl/resource/helper_test.go +++ b/pkg/kubectl/genericclioptions/resource/helper_test.go @@ -26,13 +26,15 @@ import ( "strings" "testing" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest/fake" + + // TODO we need to remove this linkage and create our own scheme + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" ) func objBody(obj runtime.Object) io.ReadCloser { diff --git a/pkg/kubectl/resource/interfaces.go b/pkg/kubectl/genericclioptions/resource/interfaces.go similarity index 82% rename from pkg/kubectl/resource/interfaces.go rename to pkg/kubectl/genericclioptions/resource/interfaces.go index 359a014846c..1a26ec9cc84 100644 --- a/pkg/kubectl/resource/interfaces.go +++ b/pkg/kubectl/genericclioptions/resource/interfaces.go @@ -17,10 +17,18 @@ limitations under the License. package resource import ( + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" "k8s.io/client-go/rest" ) +type RESTClientGetter interface { + ToRESTConfig() (*rest.Config, error) + ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) + ToRESTMapper() (meta.RESTMapper, error) +} + type ClientConfigFunc func() (*rest.Config, error) // RESTClient is a client helper for dealing with RESTful resources @@ -70,3 +78,8 @@ func (c *clientOptions) Delete() *rest.Request { func (c *clientOptions) Put() *rest.Request { return c.modify(c.c.Put()) } + +// ContentValidator is an interface that knows how to validate an API object serialized to a byte array. +type ContentValidator interface { + ValidateBytes(data []byte) error +} diff --git a/pkg/kubectl/resource/mapper.go b/pkg/kubectl/genericclioptions/resource/mapper.go similarity index 100% rename from pkg/kubectl/resource/mapper.go rename to pkg/kubectl/genericclioptions/resource/mapper.go diff --git a/pkg/kubectl/resource/result.go b/pkg/kubectl/genericclioptions/resource/result.go similarity index 100% rename from pkg/kubectl/resource/result.go rename to pkg/kubectl/genericclioptions/resource/result.go diff --git a/pkg/kubectl/resource/selector.go b/pkg/kubectl/genericclioptions/resource/selector.go similarity index 100% rename from pkg/kubectl/resource/selector.go rename to pkg/kubectl/genericclioptions/resource/selector.go diff --git a/pkg/kubectl/resource/visitor.go b/pkg/kubectl/genericclioptions/resource/visitor.go similarity index 98% rename from pkg/kubectl/resource/visitor.go rename to pkg/kubectl/genericclioptions/resource/visitor.go index 20550940ebf..55031a470bd 100644 --- a/pkg/kubectl/resource/visitor.go +++ b/pkg/kubectl/genericclioptions/resource/visitor.go @@ -38,7 +38,6 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/apimachinery/pkg/watch" - "k8s.io/kubernetes/pkg/kubectl/validation" ) const ( @@ -219,7 +218,7 @@ func (l EagerVisitorList) Visit(fn VisitorFunc) error { return utilerrors.NewAggregate(errs) } -func ValidateSchema(data []byte, schema validation.Schema) error { +func ValidateSchema(data []byte, schema ContentValidator) error { if schema == nil { return nil } @@ -433,7 +432,7 @@ func ignoreFile(path string, extensions []string) bool { } // FileVisitorForSTDIN return a special FileVisitor just for STDIN -func FileVisitorForSTDIN(mapper *mapper, schema validation.Schema) Visitor { +func FileVisitorForSTDIN(mapper *mapper, schema ContentValidator) Visitor { return &FileVisitor{ Path: constSTDINstr, StreamVisitor: NewStreamVisitor(nil, mapper, constSTDINstr, schema), @@ -443,7 +442,7 @@ func FileVisitorForSTDIN(mapper *mapper, schema validation.Schema) Visitor { // ExpandPathsToFileVisitors will return a slice of FileVisitors that will handle files from the provided path. // After FileVisitors open the files, they will pass an io.Reader to a StreamVisitor to do the reading. (stdin // is also taken care of). Paths argument also accepts a single file, and will return a single visitor -func ExpandPathsToFileVisitors(mapper *mapper, paths string, recursive bool, extensions []string, schema validation.Schema) ([]Visitor, error) { +func ExpandPathsToFileVisitors(mapper *mapper, paths string, recursive bool, extensions []string, schema ContentValidator) ([]Visitor, error) { var visitors []Visitor err := filepath.Walk(paths, func(path string, fi os.FileInfo, err error) error { if err != nil { @@ -513,11 +512,11 @@ type StreamVisitor struct { *mapper Source string - Schema validation.Schema + Schema ContentValidator } // NewStreamVisitor is a helper function that is useful when we want to change the fields of the struct but keep calls the same. -func NewStreamVisitor(r io.Reader, mapper *mapper, source string, schema validation.Schema) *StreamVisitor { +func NewStreamVisitor(r io.Reader, mapper *mapper, source string, schema ContentValidator) *StreamVisitor { return &StreamVisitor{ Reader: r, mapper: mapper, diff --git a/pkg/kubectl/resource/visitor_test.go b/pkg/kubectl/genericclioptions/resource/visitor_test.go similarity index 100% rename from pkg/kubectl/resource/visitor_test.go rename to pkg/kubectl/genericclioptions/resource/visitor_test.go diff --git a/pkg/kubectl/kubectl.go b/pkg/kubectl/kubectl.go deleted file mode 100644 index 9b29eaaf80d..00000000000 --- a/pkg/kubectl/kubectl.go +++ /dev/null @@ -1,184 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// A set of common functions needed by cmd/kubectl and pkg/kubectl packages. -package kubectl - -import ( - "strings" - - "k8s.io/apimachinery/pkg/runtime/schema" -) - -type NamespaceInfo struct { - Namespace string -} - -// ResourceShortcuts represents a structure that holds the information how to -// transition from resource's shortcut to its full name. -type ResourceShortcuts struct { - ShortForm schema.GroupResource - LongForm schema.GroupResource -} - -// ResourcesShortcutStatic is the list of short names to their expanded names. -// Note that the list is ordered by group. -var ResourcesShortcutStatic = []ResourceShortcuts{ - // If you add an entry here, please also take a look at pkg/kubectl/cmd/cmd.go - // and add an entry to valid_resources when appropriate. - { - ShortForm: schema.GroupResource{Resource: "cm"}, - LongForm: schema.GroupResource{Resource: "configmaps"}, - }, - { - ShortForm: schema.GroupResource{Resource: "cs"}, - LongForm: schema.GroupResource{Resource: "componentstatuses"}, - }, - { - ShortForm: schema.GroupResource{Resource: "ep"}, - LongForm: schema.GroupResource{Resource: "endpoints"}, - }, - { - ShortForm: schema.GroupResource{Resource: "ev"}, - LongForm: schema.GroupResource{Resource: "events"}, - }, - { - ShortForm: schema.GroupResource{Resource: "limits"}, - LongForm: schema.GroupResource{Resource: "limitranges"}, - }, - { - ShortForm: schema.GroupResource{Resource: "no"}, - LongForm: schema.GroupResource{Resource: "nodes"}, - }, - { - ShortForm: schema.GroupResource{Resource: "ns"}, - LongForm: schema.GroupResource{Resource: "namespaces"}, - }, - { - ShortForm: schema.GroupResource{Resource: "po"}, - LongForm: schema.GroupResource{Resource: "pods"}, - }, - { - ShortForm: schema.GroupResource{Resource: "pvc"}, - LongForm: schema.GroupResource{Resource: "persistentvolumeclaims"}, - }, - { - ShortForm: schema.GroupResource{Resource: "pv"}, - LongForm: schema.GroupResource{Resource: "persistentvolumes"}, - }, - { - ShortForm: schema.GroupResource{Resource: "quota"}, - LongForm: schema.GroupResource{Resource: "resourcequotas"}, - }, - { - ShortForm: schema.GroupResource{Resource: "rc"}, - LongForm: schema.GroupResource{Resource: "replicationcontrollers"}, - }, - { - ShortForm: schema.GroupResource{Resource: "rs"}, - LongForm: schema.GroupResource{Resource: "replicasets"}, - }, - { - ShortForm: schema.GroupResource{Resource: "sa"}, - LongForm: schema.GroupResource{Resource: "serviceaccounts"}, - }, - { - ShortForm: schema.GroupResource{Resource: "svc"}, - LongForm: schema.GroupResource{Resource: "services"}, - }, - { - ShortForm: schema.GroupResource{Group: "autoscaling", Resource: "hpa"}, - LongForm: schema.GroupResource{Group: "autoscaling", Resource: "horizontalpodautoscalers"}, - }, - { - ShortForm: schema.GroupResource{Group: "certificates.k8s.io", Resource: "csr"}, - LongForm: schema.GroupResource{Group: "certificates.k8s.io", Resource: "certificatesigningrequests"}, - }, - { - ShortForm: schema.GroupResource{Group: "policy", Resource: "pdb"}, - LongForm: schema.GroupResource{Group: "policy", Resource: "poddisruptionbudgets"}, - }, - { - ShortForm: schema.GroupResource{Group: "extensions", Resource: "deploy"}, - LongForm: schema.GroupResource{Group: "extensions", Resource: "deployments"}, - }, - { - ShortForm: schema.GroupResource{Group: "extensions", Resource: "ds"}, - LongForm: schema.GroupResource{Group: "extensions", Resource: "daemonsets"}, - }, - { - ShortForm: schema.GroupResource{Group: "extensions", Resource: "hpa"}, - LongForm: schema.GroupResource{Group: "extensions", Resource: "horizontalpodautoscalers"}, - }, - { - ShortForm: schema.GroupResource{Group: "extensions", Resource: "ing"}, - LongForm: schema.GroupResource{Group: "extensions", Resource: "ingresses"}, - }, - { - ShortForm: schema.GroupResource{Group: "extensions", Resource: "netpol"}, - LongForm: schema.GroupResource{Group: "extensions", Resource: "networkpolicies"}, - }, - { - ShortForm: schema.GroupResource{Group: "extensions", Resource: "psp"}, - LongForm: schema.GroupResource{Group: "extensions", Resource: "podSecurityPolicies"}, - }, -} - -// ResourceShortFormFor looks up for a short form of resource names. -// TODO: Change the signature of this function so that it can -// make use of ResourceShortcuts. -func ResourceShortFormFor(resource string) (string, bool) { - var alias string - exists := false - for _, item := range ResourcesShortcutStatic { - if item.LongForm.Resource == resource { - alias = item.ShortForm.Resource - exists = true - break - } - } - return alias, exists -} - -// ResourceAliases returns the resource shortcuts and plural forms for the given resources. -func ResourceAliases(rs []string) []string { - as := make([]string, 0, len(rs)) - plurals := make(map[string]struct{}, len(rs)) - for _, r := range rs { - var plural string - switch { - case r == "endpoints": - // Endpoints type itself is plural, unlike every other resource. - plural = r - case strings.HasSuffix(r, "y"): - plural = r[0:len(r)-1] + "ies" - case strings.HasSuffix(r, "s"): - plural = r + "es" - default: - plural = r + "s" - } - as = append(as, plural) - - plurals[plural] = struct{}{} - } - - for _, item := range ResourcesShortcutStatic { - if _, found := plurals[item.LongForm.Resource]; found { - as = append(as, item.ShortForm.Resource) - } - } - return as -} diff --git a/pkg/kubectl/plugins/BUILD b/pkg/kubectl/plugins/BUILD index e2f32b7d607..ac8b4f73ff6 100644 --- a/pkg/kubectl/plugins/BUILD +++ b/pkg/kubectl/plugins/BUILD @@ -16,6 +16,7 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/kubectl/plugins", deps = [ + "//pkg/kubectl/genericclioptions:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", @@ -45,5 +46,8 @@ go_test( "runner_test.go", ], embed = [":go_default_library"], - deps = ["//vendor/github.com/spf13/pflag:go_default_library"], + deps = [ + "//pkg/kubectl/genericclioptions:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + ], ) diff --git a/pkg/kubectl/plugins/runner.go b/pkg/kubectl/plugins/runner.go index b30472193bc..eff195564b6 100644 --- a/pkg/kubectl/plugins/runner.go +++ b/pkg/kubectl/plugins/runner.go @@ -17,12 +17,12 @@ limitations under the License. package plugins import ( - "io" "os" "os/exec" "strings" "github.com/golang/glog" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" ) // PluginRunner is capable of running a plugin in a given running context. @@ -34,9 +34,7 @@ type PluginRunner interface { // in, out, and err streams, arguments and environment passed to it, and the // working directory. type RunningContext struct { - In io.Reader - Out io.Writer - ErrOut io.Writer + genericclioptions.IOStreams Args []string EnvProvider EnvProvider WorkingDir string diff --git a/pkg/kubectl/plugins/runner_test.go b/pkg/kubectl/plugins/runner_test.go index c91ad2fb13d..ae378148767 100644 --- a/pkg/kubectl/plugins/runner_test.go +++ b/pkg/kubectl/plugins/runner_test.go @@ -17,9 +17,10 @@ limitations under the License. package plugins import ( - "bytes" "os" "testing" + + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" ) func TestExecRunner(t *testing.T) { @@ -50,7 +51,7 @@ func TestExecRunner(t *testing.T) { defer os.Unsetenv("KUBECTL_PLUGINS_TEST") for _, test := range tests { - outBuf := bytes.NewBuffer([]byte{}) + streams, _, outBuf, _ := genericclioptions.NewTestIOStreams() plugin := &Plugin{ Description: Description{ @@ -61,7 +62,7 @@ func TestExecRunner(t *testing.T) { } ctx := RunningContext{ - Out: outBuf, + IOStreams: streams, WorkingDir: ".", EnvProvider: &EmptyEnvProvider{}, } diff --git a/pkg/kubectl/scheme/BUILD b/pkg/kubectl/scheme/BUILD index 6a35d9edb54..8d1f602f415 100644 --- a/pkg/kubectl/scheme/BUILD +++ b/pkg/kubectl/scheme/BUILD @@ -11,6 +11,7 @@ go_library( deps = [ "//vendor/k8s.io/api/admission/v1beta1:go_default_library", "//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library", + "//vendor/k8s.io/api/admissionregistration/v1beta1:go_default_library", "//vendor/k8s.io/api/apps/v1:go_default_library", "//vendor/k8s.io/api/apps/v1beta1:go_default_library", "//vendor/k8s.io/api/apps/v1beta2:go_default_library", @@ -36,13 +37,12 @@ go_library( "//vendor/k8s.io/api/settings/v1alpha1:go_default_library", "//vendor/k8s.io/api/storage/v1:go_default_library", "//vendor/k8s.io/api/storage/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library", ], ) diff --git a/pkg/kubectl/scheme/install.go b/pkg/kubectl/scheme/install.go index bb6ae520af6..b756689b3af 100644 --- a/pkg/kubectl/scheme/install.go +++ b/pkg/kubectl/scheme/install.go @@ -19,6 +19,7 @@ package scheme import ( admissionv1alpha1 "k8s.io/api/admission/v1beta1" admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" appsv1 "k8s.io/api/apps/v1" appsv1beta1 "k8s.io/api/apps/v1beta1" appsv1beta2 "k8s.io/api/apps/v1beta2" @@ -44,9 +45,9 @@ import ( settingsv1alpha1 "k8s.io/api/settings/v1alpha1" storagev1 "k8s.io/api/storage/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes/scheme" ) @@ -55,238 +56,24 @@ import ( // but does NOT register the internal types. func init() { // Register external types for Scheme - v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) scheme.AddToScheme(Scheme) - // Register external types for Registry - Versions = append(Versions, corev1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: corev1.GroupName, - VersionPreferenceOrder: []string{corev1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - corev1.SchemeGroupVersion.Version: corev1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, admissionv1alpha1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: admissionv1alpha1.GroupName, - VersionPreferenceOrder: []string{admissionv1alpha1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - admissionv1alpha1.SchemeGroupVersion.Version: admissionv1alpha1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, admissionregistrationv1alpha1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: admissionregistrationv1alpha1.GroupName, - VersionPreferenceOrder: []string{admissionregistrationv1alpha1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - admissionregistrationv1alpha1.SchemeGroupVersion.Version: admissionregistrationv1alpha1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, appsv1.SchemeGroupVersion, appsv1beta2.SchemeGroupVersion, appsv1beta1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: appsv1.GroupName, - VersionPreferenceOrder: []string{appsv1beta1.SchemeGroupVersion.Version, appsv1beta2.SchemeGroupVersion.Version, appsv1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - appsv1beta1.SchemeGroupVersion.Version: appsv1beta1.AddToScheme, - appsv1beta2.SchemeGroupVersion.Version: appsv1beta2.AddToScheme, - appsv1.SchemeGroupVersion.Version: appsv1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, authenticationv1.SchemeGroupVersion, authenticationv1beta1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: authenticationv1beta1.GroupName, - VersionPreferenceOrder: []string{authenticationv1.SchemeGroupVersion.Version, authenticationv1beta1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - authenticationv1beta1.SchemeGroupVersion.Version: authenticationv1beta1.AddToScheme, - authenticationv1.SchemeGroupVersion.Version: authenticationv1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, authorizationv1.SchemeGroupVersion, authorizationv1beta1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: authorizationv1.GroupName, - VersionPreferenceOrder: []string{authorizationv1.SchemeGroupVersion.Version, authorizationv1beta1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - authorizationv1beta1.SchemeGroupVersion.Version: authorizationv1beta1.AddToScheme, - authorizationv1.SchemeGroupVersion.Version: authorizationv1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, autoscalingv1.SchemeGroupVersion, autoscalingv2beta1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: autoscalingv1.GroupName, - VersionPreferenceOrder: []string{autoscalingv1.SchemeGroupVersion.Version, autoscalingv2beta1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - autoscalingv1.SchemeGroupVersion.Version: autoscalingv1.AddToScheme, - autoscalingv2beta1.SchemeGroupVersion.Version: autoscalingv2beta1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, batchv1.SchemeGroupVersion, batchv1beta1.SchemeGroupVersion, batchv2alpha1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: batchv1.GroupName, - VersionPreferenceOrder: []string{batchv1.SchemeGroupVersion.Version, batchv1beta1.SchemeGroupVersion.Version, batchv2alpha1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - batchv1.SchemeGroupVersion.Version: batchv1.AddToScheme, - batchv1beta1.SchemeGroupVersion.Version: batchv1beta1.AddToScheme, - batchv2alpha1.SchemeGroupVersion.Version: batchv2alpha1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, certificatesv1beta1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: certificatesv1beta1.GroupName, - VersionPreferenceOrder: []string{certificatesv1beta1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - certificatesv1beta1.SchemeGroupVersion.Version: certificatesv1beta1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, extensionsv1beta1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: extensionsv1beta1.GroupName, - VersionPreferenceOrder: []string{extensionsv1beta1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - extensionsv1beta1.SchemeGroupVersion.Version: extensionsv1beta1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, imagepolicyv1alpha1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: imagepolicyv1alpha1.GroupName, - VersionPreferenceOrder: []string{imagepolicyv1alpha1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - imagepolicyv1alpha1.SchemeGroupVersion.Version: imagepolicyv1alpha1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, networkingv1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: networkingv1.GroupName, - VersionPreferenceOrder: []string{networkingv1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - networkingv1.SchemeGroupVersion.Version: networkingv1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, policyv1beta1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: policyv1beta1.GroupName, - VersionPreferenceOrder: []string{policyv1beta1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - policyv1beta1.SchemeGroupVersion.Version: policyv1beta1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, rbacv1.SchemeGroupVersion, rbacv1beta1.SchemeGroupVersion, rbacv1alpha1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: rbacv1.GroupName, - VersionPreferenceOrder: []string{rbacv1.SchemeGroupVersion.Version, rbacv1beta1.SchemeGroupVersion.Version, rbacv1alpha1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - rbacv1.SchemeGroupVersion.Version: rbacv1.AddToScheme, - rbacv1beta1.SchemeGroupVersion.Version: rbacv1beta1.AddToScheme, - rbacv1alpha1.SchemeGroupVersion.Version: rbacv1alpha1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, schedulingv1alpha1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: schedulingv1alpha1.GroupName, - VersionPreferenceOrder: []string{schedulingv1alpha1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - schedulingv1alpha1.SchemeGroupVersion.Version: schedulingv1alpha1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, settingsv1alpha1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: settingsv1alpha1.GroupName, - VersionPreferenceOrder: []string{settingsv1alpha1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - settingsv1alpha1.SchemeGroupVersion.Version: settingsv1alpha1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } - - Versions = append(Versions, storagev1.SchemeGroupVersion, storagev1beta1.SchemeGroupVersion) - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: storagev1.GroupName, - VersionPreferenceOrder: []string{storagev1.SchemeGroupVersion.Version, storagev1beta1.SchemeGroupVersion.Version}, - }, - announced.VersionToSchemeFunc{ - storagev1.SchemeGroupVersion.Version: storagev1.AddToScheme, - storagev1beta1.SchemeGroupVersion.Version: storagev1beta1.AddToScheme, - }, - ).Register(Registry, Scheme); err != nil { - panic(err) - } + utilruntime.Must(Scheme.SetVersionPriority(corev1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(admissionv1alpha1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(admissionregistrationv1beta1.SchemeGroupVersion, admissionregistrationv1alpha1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(appsv1beta1.SchemeGroupVersion, appsv1beta2.SchemeGroupVersion, appsv1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(authenticationv1.SchemeGroupVersion, authenticationv1beta1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(authorizationv1.SchemeGroupVersion, authorizationv1beta1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(autoscalingv1.SchemeGroupVersion, autoscalingv2beta1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(batchv1.SchemeGroupVersion, batchv1beta1.SchemeGroupVersion, batchv2alpha1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(certificatesv1beta1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(extensionsv1beta1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(imagepolicyv1alpha1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(networkingv1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(policyv1beta1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(rbacv1.SchemeGroupVersion, rbacv1beta1.SchemeGroupVersion, rbacv1alpha1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(schedulingv1alpha1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(settingsv1alpha1.SchemeGroupVersion)) + utilruntime.Must(Scheme.SetVersionPriority(storagev1.SchemeGroupVersion, storagev1beta1.SchemeGroupVersion)) } diff --git a/pkg/kubectl/scheme/scheme.go b/pkg/kubectl/scheme/scheme.go index efd77d7673b..880b115b178 100644 --- a/pkg/kubectl/scheme/scheme.go +++ b/pkg/kubectl/scheme/scheme.go @@ -17,33 +17,20 @@ limitations under the License. package scheme import ( - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" ) // All kubectl code should eventually switch to use this Registry and Scheme instead of the global ones. -// Registry is an instance of an API registry. -var Registry = registered.NewAPIRegistrationManager() - // Scheme is the default instance of runtime.Scheme to which types in the Kubernetes API are already registered. var Scheme = runtime.NewScheme() // Codecs provides access to encoding and decoding for the scheme var Codecs = serializer.NewCodecFactory(Scheme) -// ParameterCodec handles versioning of objects that are converted to query parameters. -var ParameterCodec = runtime.NewParameterCodec(Scheme) - -// Versions is a list of group versions in order of preferred serialization. This used to be discovered dynamically, -// from the server for use in the client, but that gives conflicting lists of non-existent versions. This only needs to -// live until we stop attempting to perform any conversion client-side and is only valid for items existent in our scheme. -var Versions = []schema.GroupVersion{} - // DefaultJSONEncoder returns a default encoder for our scheme func DefaultJSONEncoder() runtime.Encoder { - return unstructured.JSONFallbackEncoder{Encoder: Codecs.LegacyCodec(Versions...)} + return unstructured.JSONFallbackEncoder{Encoder: Codecs.LegacyCodec(Scheme.PrioritizedVersionsAllGroups()...)} } diff --git a/pkg/kubectl/scheme/sparse_test.go b/pkg/kubectl/scheme/sparse_test.go index 14c26f3d8b6..b0d4201ec7f 100644 --- a/pkg/kubectl/scheme/sparse_test.go +++ b/pkg/kubectl/scheme/sparse_test.go @@ -36,9 +36,9 @@ func TestCronJob(t *testing.T) { t.Fatal(err) } t.Log(string(cronjobBytes)) - t.Log(Registry.RegisteredGroupVersions()) + t.Log(Scheme.PrioritizedVersionsAllGroups()) - decoder := Codecs.UniversalDecoder(Registry.RegisteredGroupVersions()...) + decoder := Codecs.UniversalDecoder(Scheme.PrioritizedVersionsAllGroups()...) uncastDst, err := runtime.Decode(decoder, cronjobBytes) if err != nil { diff --git a/pkg/kubectl/util/BUILD b/pkg/kubectl/util/BUILD index 16c098d196c..ad3c1b73130 100644 --- a/pkg/kubectl/util/BUILD +++ b/pkg/kubectl/util/BUILD @@ -104,7 +104,6 @@ filegroup( "//pkg/kubectl/util/logs:all-srcs", "//pkg/kubectl/util/slice:all-srcs", "//pkg/kubectl/util/term:all-srcs", - "//pkg/kubectl/util/transport:all-srcs", ], tags = ["automanaged"], visibility = ["//build/visible_to:pkg_kubectl_util_CONSUMERS"], diff --git a/pkg/kubelet/apis/kubeletconfig/BUILD b/pkg/kubelet/apis/kubeletconfig/BUILD index 0aa5579dde4..969efbc99cc 100644 --- a/pkg/kubelet/apis/kubeletconfig/BUILD +++ b/pkg/kubelet/apis/kubeletconfig/BUILD @@ -17,6 +17,7 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig", deps = [ + "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/pkg/kubelet/apis/kubeletconfig/register.go b/pkg/kubelet/apis/kubeletconfig/register.go index fd5d5de5f73..886f39dec4b 100644 --- a/pkg/kubelet/apis/kubeletconfig/register.go +++ b/pkg/kubelet/apis/kubeletconfig/register.go @@ -46,6 +46,7 @@ func addKnownTypes(scheme *runtime.Scheme) error { // TODO this will get cleaned up with the scheme types are fixed scheme.AddKnownTypes(SchemeGroupVersion, &KubeletConfiguration{}, + &SerializedNodeConfigSource{}, ) return nil } diff --git a/pkg/kubelet/apis/kubeletconfig/types.go b/pkg/kubelet/apis/kubeletconfig/types.go index c61278f587e..e0337e5aaf5 100644 --- a/pkg/kubelet/apis/kubeletconfig/types.go +++ b/pkg/kubelet/apis/kubeletconfig/types.go @@ -17,6 +17,7 @@ limitations under the License. package kubeletconfig import ( + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -333,3 +334,15 @@ type KubeletAnonymousAuthentication struct { // Anonymous requests have a username of system:anonymous, and a group name of system:unauthenticated. Enabled bool } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// SerializedNodeConfigSource allows us to serialize NodeConfigSource +// This type is used internally by the Kubelet for tracking checkpointed dynamic configs. +// It exists in the kubeletconfig API group because it is classified as a versioned input to the Kubelet. +type SerializedNodeConfigSource struct { + metav1.TypeMeta + // Source is the source that we are serializing + // +optional + Source v1.NodeConfigSource +} diff --git a/pkg/kubelet/apis/kubeletconfig/v1beta1/BUILD b/pkg/kubelet/apis/kubeletconfig/v1beta1/BUILD index e54e8b0a34a..c45da80cf97 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1beta1/BUILD +++ b/pkg/kubelet/apis/kubeletconfig/v1beta1/BUILD @@ -23,6 +23,7 @@ go_library( "//pkg/kubelet/types:go_default_library", "//pkg/master/ports:go_default_library", "//pkg/util/pointer:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/pkg/kubelet/apis/kubeletconfig/v1beta1/register.go b/pkg/kubelet/apis/kubeletconfig/v1beta1/register.go index 306fc55e09b..4c6b2931b83 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1beta1/register.go +++ b/pkg/kubelet/apis/kubeletconfig/v1beta1/register.go @@ -45,6 +45,7 @@ func init() { func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &KubeletConfiguration{}, + &SerializedNodeConfigSource{}, ) return nil } diff --git a/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go b/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go index 60d314bace6..984b61b3047 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go +++ b/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta1 import ( + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -512,3 +513,15 @@ type KubeletAnonymousAuthentication struct { // +optional Enabled *bool `json:"enabled,omitempty"` } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// SerializedNodeConfigSource allows us to serialize v1.NodeConfigSource. +// This type is used internally by the Kubelet for tracking checkpointed dynamic configs. +// It exists in the kubeletconfig API group because it is classified as a versioned input to the Kubelet. +type SerializedNodeConfigSource struct { + metav1.TypeMeta `json:",inline"` + // Source is the source that we are serializing + // +optional + Source v1.NodeConfigSource `json:"source,omitempty" protobuf:"bytes,1,opt,name=source"` +} diff --git a/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.conversion.go index 51074bc126c..67234da275c 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.conversion.go @@ -51,6 +51,8 @@ func RegisterConversions(scheme *runtime.Scheme) error { Convert_kubeletconfig_KubeletWebhookAuthorization_To_v1beta1_KubeletWebhookAuthorization, Convert_v1beta1_KubeletX509Authentication_To_kubeletconfig_KubeletX509Authentication, Convert_kubeletconfig_KubeletX509Authentication_To_v1beta1_KubeletX509Authentication, + Convert_v1beta1_SerializedNodeConfigSource_To_kubeletconfig_SerializedNodeConfigSource, + Convert_kubeletconfig_SerializedNodeConfigSource_To_v1beta1_SerializedNodeConfigSource, ) } @@ -453,3 +455,23 @@ func autoConvert_kubeletconfig_KubeletX509Authentication_To_v1beta1_KubeletX509A func Convert_kubeletconfig_KubeletX509Authentication_To_v1beta1_KubeletX509Authentication(in *kubeletconfig.KubeletX509Authentication, out *KubeletX509Authentication, s conversion.Scope) error { return autoConvert_kubeletconfig_KubeletX509Authentication_To_v1beta1_KubeletX509Authentication(in, out, s) } + +func autoConvert_v1beta1_SerializedNodeConfigSource_To_kubeletconfig_SerializedNodeConfigSource(in *SerializedNodeConfigSource, out *kubeletconfig.SerializedNodeConfigSource, s conversion.Scope) error { + out.Source = in.Source + return nil +} + +// Convert_v1beta1_SerializedNodeConfigSource_To_kubeletconfig_SerializedNodeConfigSource is an autogenerated conversion function. +func Convert_v1beta1_SerializedNodeConfigSource_To_kubeletconfig_SerializedNodeConfigSource(in *SerializedNodeConfigSource, out *kubeletconfig.SerializedNodeConfigSource, s conversion.Scope) error { + return autoConvert_v1beta1_SerializedNodeConfigSource_To_kubeletconfig_SerializedNodeConfigSource(in, out, s) +} + +func autoConvert_kubeletconfig_SerializedNodeConfigSource_To_v1beta1_SerializedNodeConfigSource(in *kubeletconfig.SerializedNodeConfigSource, out *SerializedNodeConfigSource, s conversion.Scope) error { + out.Source = in.Source + return nil +} + +// Convert_kubeletconfig_SerializedNodeConfigSource_To_v1beta1_SerializedNodeConfigSource is an autogenerated conversion function. +func Convert_kubeletconfig_SerializedNodeConfigSource_To_v1beta1_SerializedNodeConfigSource(in *kubeletconfig.SerializedNodeConfigSource, out *SerializedNodeConfigSource, s conversion.Scope) error { + return autoConvert_kubeletconfig_SerializedNodeConfigSource_To_v1beta1_SerializedNodeConfigSource(in, out, s) +} diff --git a/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.deepcopy.go b/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.deepcopy.go index 456f551a64e..f7fc0a79cc0 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/kubeletconfig/v1beta1/zz_generated.deepcopy.go @@ -426,3 +426,29 @@ func (in *KubeletX509Authentication) DeepCopy() *KubeletX509Authentication { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SerializedNodeConfigSource) DeepCopyInto(out *SerializedNodeConfigSource) { + *out = *in + out.TypeMeta = in.TypeMeta + in.Source.DeepCopyInto(&out.Source) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SerializedNodeConfigSource. +func (in *SerializedNodeConfigSource) DeepCopy() *SerializedNodeConfigSource { + if in == nil { + return nil + } + out := new(SerializedNodeConfigSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SerializedNodeConfigSource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go b/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go index e1ec3346672..2a312ad8447 100644 --- a/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go @@ -246,3 +246,29 @@ func (in *KubeletX509Authentication) DeepCopy() *KubeletX509Authentication { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SerializedNodeConfigSource) DeepCopyInto(out *SerializedNodeConfigSource) { + *out = *in + out.TypeMeta = in.TypeMeta + in.Source.DeepCopyInto(&out.Source) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SerializedNodeConfigSource. +func (in *SerializedNodeConfigSource) DeepCopy() *SerializedNodeConfigSource { + if in == nil { + return nil + } + out := new(SerializedNodeConfigSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SerializedNodeConfigSource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/pkg/kubelet/certificate/bootstrap/bootstrap.go b/pkg/kubelet/certificate/bootstrap/bootstrap.go index 72c085757d1..9b7b9b3bff5 100644 --- a/pkg/kubelet/certificate/bootstrap/bootstrap.go +++ b/pkg/kubelet/certificate/bootstrap/bootstrap.go @@ -123,10 +123,7 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, } // Marshal to disk - if err := clientcmd.WriteToFile(kubeconfigData, kubeconfigPath); err != nil { - return err - } - return nil + return clientcmd.WriteToFile(kubeconfigData, kubeconfigPath) } func loadRESTClientConfig(kubeconfig string) (*restclient.Config, error) { diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index 6ecc6527909..d932de56460 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -188,6 +188,7 @@ go_test( "container_manager_linux_test.go", "helpers_linux_test.go", "node_container_manager_test.go", + "pod_container_manager_linux_test.go", ], "//conditions:default": [], }), @@ -200,6 +201,7 @@ go_test( "//vendor/github.com/stretchr/testify/require:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", ], "//conditions:default": [], }), diff --git a/pkg/kubelet/cm/pod_container_manager_linux.go b/pkg/kubelet/cm/pod_container_manager_linux.go index a749c9087eb..1838d1ab2c4 100644 --- a/pkg/kubelet/cm/pod_container_manager_linux.go +++ b/pkg/kubelet/cm/pod_container_manager_linux.go @@ -184,6 +184,31 @@ func (m *podContainerManagerImpl) ReduceCPULimits(podCgroup CgroupName) error { return m.cgroupManager.ReduceCPULimits(podCgroup) } +// IsPodCgroup returns true if the literal cgroupfs name corresponds to a pod +func (m *podContainerManagerImpl) IsPodCgroup(cgroupfs string) (bool, types.UID) { + // convert the literal cgroupfs form to the driver specific value + cgroupName := m.cgroupManager.CgroupName(cgroupfs) + qosContainersList := [3]CgroupName{m.qosContainersInfo.BestEffort, m.qosContainersInfo.Burstable, m.qosContainersInfo.Guaranteed} + basePath := "" + for _, qosContainerName := range qosContainersList { + // a pod cgroup is a direct child of a qos node, so check if its a match + if len(cgroupName) == len(qosContainerName)+1 { + basePath = cgroupName[len(qosContainerName)] + } + } + if basePath == "" { + return false, types.UID("") + } + if !strings.HasPrefix(basePath, podCgroupNamePrefix) { + return false, types.UID("") + } + parts := strings.Split(basePath, podCgroupNamePrefix) + if len(parts) != 2 { + return false, types.UID("") + } + return true, types.UID(parts[1]) +} + // GetAllPodsFromCgroups scans through all the subsystems of pod cgroups // Get list of pods whose cgroup still exist on the cgroup mounts func (m *podContainerManagerImpl) GetAllPodsFromCgroups() (map[types.UID]CgroupName, error) { @@ -278,3 +303,7 @@ func (m *podContainerManagerNoop) ReduceCPULimits(_ CgroupName) error { func (m *podContainerManagerNoop) GetAllPodsFromCgroups() (map[types.UID]CgroupName, error) { return nil, nil } + +func (m *podContainerManagerNoop) IsPodCgroup(cgroupfs string) (bool, types.UID) { + return false, types.UID("") +} diff --git a/pkg/kubelet/cm/pod_container_manager_linux_test.go b/pkg/kubelet/cm/pod_container_manager_linux_test.go new file mode 100644 index 00000000000..62c9f203a00 --- /dev/null +++ b/pkg/kubelet/cm/pod_container_manager_linux_test.go @@ -0,0 +1,125 @@ +// +build linux + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cm + +import ( + "strings" + "testing" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +func TestIsCgroupPod(t *testing.T) { + qosContainersInfo := QOSContainersInfo{ + Guaranteed: RootCgroupName, + Burstable: NewCgroupName(RootCgroupName, strings.ToLower(string(v1.PodQOSBurstable))), + BestEffort: NewCgroupName(RootCgroupName, strings.ToLower(string(v1.PodQOSBestEffort))), + } + podUID := types.UID("123") + testCases := []struct { + input CgroupName + expectedResult bool + expectedUID types.UID + }{ + { + input: RootCgroupName, + expectedResult: false, + expectedUID: types.UID(""), + }, + { + input: NewCgroupName(qosContainersInfo.Guaranteed), + expectedResult: false, + expectedUID: types.UID(""), + }, + { + input: NewCgroupName(qosContainersInfo.Guaranteed, GetPodCgroupNameSuffix(podUID)), + expectedResult: true, + expectedUID: podUID, + }, + { + input: NewCgroupName(qosContainersInfo.Guaranteed, GetPodCgroupNameSuffix(podUID), "container.scope"), + expectedResult: false, + expectedUID: types.UID(""), + }, + { + input: NewCgroupName(qosContainersInfo.Burstable), + expectedResult: false, + expectedUID: types.UID(""), + }, + { + input: NewCgroupName(qosContainersInfo.Burstable, GetPodCgroupNameSuffix(podUID)), + expectedResult: true, + expectedUID: podUID, + }, + { + input: NewCgroupName(qosContainersInfo.Burstable, GetPodCgroupNameSuffix(podUID), "container.scope"), + expectedResult: false, + expectedUID: types.UID(""), + }, + { + input: NewCgroupName(qosContainersInfo.BestEffort), + expectedResult: false, + expectedUID: types.UID(""), + }, + { + input: NewCgroupName(qosContainersInfo.BestEffort, GetPodCgroupNameSuffix(podUID)), + expectedResult: true, + expectedUID: podUID, + }, + { + input: NewCgroupName(qosContainersInfo.BestEffort, GetPodCgroupNameSuffix(podUID), "container.scope"), + expectedResult: false, + expectedUID: types.UID(""), + }, + { + input: NewCgroupName(RootCgroupName, "system"), + expectedResult: false, + expectedUID: types.UID(""), + }, + { + input: NewCgroupName(RootCgroupName, "system", "kubelet"), + expectedResult: false, + expectedUID: types.UID(""), + }, + } + for _, cgroupDriver := range []string{"cgroupfs", "systemd"} { + pcm := &podContainerManagerImpl{ + cgroupManager: NewCgroupManager(nil, cgroupDriver), + enforceCPULimits: true, + qosContainersInfo: qosContainersInfo, + } + for _, testCase := range testCases { + // give the right cgroup structure based on driver + cgroupfs := testCase.input.ToCgroupfs() + if cgroupDriver == "systemd" { + cgroupfs = testCase.input.ToSystemd() + } + // check if this is a pod or not with the literal cgroupfs input + result, resultUID := pcm.IsPodCgroup(cgroupfs) + if result != testCase.expectedResult { + t.Errorf("Unexpected result for driver: %v, input: %v, expected: %v, actual: %v", cgroupDriver, testCase.input, testCase.expectedResult, result) + } + if resultUID != testCase.expectedUID { + t.Errorf("Unexpected result for driver: %v, input: %v, expected: %v, actual: %v", cgroupDriver, testCase.input, testCase.expectedUID, resultUID) + } + + } + } +} diff --git a/pkg/kubelet/cm/pod_container_manager_stub.go b/pkg/kubelet/cm/pod_container_manager_stub.go index c4cb71156b0..26c56ec7910 100644 --- a/pkg/kubelet/cm/pod_container_manager_stub.go +++ b/pkg/kubelet/cm/pod_container_manager_stub.go @@ -49,3 +49,7 @@ func (m *podContainerManagerStub) ReduceCPULimits(_ CgroupName) error { func (m *podContainerManagerStub) GetAllPodsFromCgroups() (map[types.UID]CgroupName, error) { return nil, nil } + +func (m *podContainerManagerStub) IsPodCgroup(cgroupfs string) (bool, types.UID) { + return false, types.UID("") +} diff --git a/pkg/kubelet/cm/types.go b/pkg/kubelet/cm/types.go index e95f44340b7..2e60d8a8fde 100644 --- a/pkg/kubelet/cm/types.go +++ b/pkg/kubelet/cm/types.go @@ -124,4 +124,7 @@ type PodContainerManager interface { // GetAllPodsFromCgroups enumerates the set of pod uids to their associated cgroup based on state of cgroupfs system. GetAllPodsFromCgroups() (map[types.UID]CgroupName, error) + + // IsPodCgroup returns true if the literal cgroupfs name corresponds to a pod + IsPodCgroup(cgroupfs string) (bool, types.UID) } diff --git a/pkg/kubelet/config/common_test.go b/pkg/kubelet/config/common_test.go index 55cc54c788c..5aa1447983a 100644 --- a/pkg/kubelet/config/common_test.go +++ b/pkg/kubelet/config/common_test.go @@ -78,7 +78,7 @@ func TestDecodeSinglePod(t *testing.T) { t.Errorf("expected:\n%#v\ngot:\n%#v\n%s", pod, podOut, string(json)) } - for _, gv := range legacyscheme.Registry.RegisteredVersionsForGroup(v1.GroupName) { + for _, gv := range legacyscheme.Scheme.PrioritizedVersionsForGroup(v1.GroupName) { info, _ := runtime.SerializerInfoForMediaType(legacyscheme.Codecs.SupportedMediaTypes(), "application/yaml") encoder := legacyscheme.Codecs.EncoderForVersion(info.Serializer, gv) yaml, err := runtime.Encode(encoder, pod) @@ -144,7 +144,7 @@ func TestDecodePodList(t *testing.T) { t.Errorf("expected:\n%#v\ngot:\n%#v\n%s", podList, &podListOut, string(json)) } - for _, gv := range legacyscheme.Registry.RegisteredVersionsForGroup(v1.GroupName) { + for _, gv := range legacyscheme.Scheme.PrioritizedVersionsForGroup(v1.GroupName) { info, _ := runtime.SerializerInfoForMediaType(legacyscheme.Codecs.SupportedMediaTypes(), "application/yaml") encoder := legacyscheme.Codecs.EncoderForVersion(info.Serializer, gv) yaml, err := runtime.Encode(encoder, podList) diff --git a/pkg/kubelet/eviction/BUILD b/pkg/kubelet/eviction/BUILD index 03d662a5d05..08c744ba87d 100644 --- a/pkg/kubelet/eviction/BUILD +++ b/pkg/kubelet/eviction/BUILD @@ -20,7 +20,6 @@ go_test( "//pkg/kubelet/eviction/api:go_default_library", "//pkg/kubelet/lifecycle:go_default_library", "//pkg/kubelet/types:go_default_library", - "//pkg/quota:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -77,6 +76,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/kubelet/eviction", deps = [ "//pkg/api/v1/resource:go_default_library", + "//pkg/apis/core/v1/helper:go_default_library", "//pkg/apis/core/v1/helper/qos:go_default_library", "//pkg/features:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", @@ -88,6 +88,7 @@ go_library( "//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/format:go_default_library", + "//pkg/scheduler/algorithm:go_default_library", "//pkg/scheduler/util:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/kubelet/eviction/api/types.go b/pkg/kubelet/eviction/api/types.go index 3e16d249c97..d32ba9b30f0 100644 --- a/pkg/kubelet/eviction/api/types.go +++ b/pkg/kubelet/eviction/api/types.go @@ -38,8 +38,6 @@ const ( SignalImageFsInodesFree Signal = "imagefs.inodesFree" // SignalAllocatableMemoryAvailable is amount of memory available for pod allocation (i.e. allocatable - workingSet (of pods), in bytes. SignalAllocatableMemoryAvailable Signal = "allocatableMemory.available" - // SignalAllocatableNodeFsAvailable is amount of local storage available for pod allocation - SignalAllocatableNodeFsAvailable Signal = "allocatableNodeFs.available" // SignalPIDAvailable is amount of PID available for pod allocation SignalPIDAvailable Signal = "pid.available" ) @@ -60,13 +58,11 @@ const ( // from either above or below, never both). There is thus no reason to expose the // operator in the Kubelet's public API. Instead, we internally map signal types to operators. var OpForSignal = map[Signal]ThresholdOperator{ - SignalMemoryAvailable: OpLessThan, - SignalNodeFsAvailable: OpLessThan, - SignalNodeFsInodesFree: OpLessThan, - SignalImageFsAvailable: OpLessThan, - SignalImageFsInodesFree: OpLessThan, - SignalAllocatableMemoryAvailable: OpLessThan, - SignalAllocatableNodeFsAvailable: OpLessThan, + SignalMemoryAvailable: OpLessThan, + SignalNodeFsAvailable: OpLessThan, + SignalNodeFsInodesFree: OpLessThan, + SignalImageFsAvailable: OpLessThan, + SignalImageFsInodesFree: OpLessThan, } // ThresholdValue is a value holder that abstracts literal versus percentage based quantity diff --git a/pkg/kubelet/eviction/eviction_manager.go b/pkg/kubelet/eviction/eviction_manager.go index f580e2951b6..a101d865d1a 100644 --- a/pkg/kubelet/eviction/eviction_manager.go +++ b/pkg/kubelet/eviction/eviction_manager.go @@ -30,6 +30,7 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" apiv1resource "k8s.io/kubernetes/pkg/api/v1/resource" + v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" "k8s.io/kubernetes/pkg/features" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" @@ -41,6 +42,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/server/stats" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/format" + "k8s.io/kubernetes/pkg/scheduler/algorithm" ) const ( @@ -76,10 +78,10 @@ type managerImpl struct { thresholdsFirstObservedAt thresholdsObservedAt // records the set of thresholds that have been met (including graceperiod) but not yet resolved thresholdsMet []evictionapi.Threshold - // resourceToRankFunc maps a resource to ranking function for that resource. - resourceToRankFunc map[v1.ResourceName]rankFunc - // resourceToNodeReclaimFuncs maps a resource to an ordered list of functions that know how to reclaim that resource. - resourceToNodeReclaimFuncs map[v1.ResourceName]nodeReclaimFuncs + // signalToRankFunc maps a resource to ranking function for that resource. + signalToRankFunc map[evictionapi.Signal]rankFunc + // signalToNodeReclaimFuncs maps a resource to an ordered list of functions that know how to reclaim that resource. + signalToNodeReclaimFuncs map[evictionapi.Signal]nodeReclaimFuncs // last observations from synchronize lastObservations signalObservations // notifierStopCh is a channel used to stop all thresholdNotifiers @@ -137,6 +139,16 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd if notBestEffort { return lifecycle.PodAdmitResult{Admit: true} } + + // When node has memory pressure and TaintNodesByCondition is enabled, check BestEffort Pod's toleration: + // admit it if tolerates memory pressure taint, fail for other tolerations, e.g. OutOfDisk. + if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) && + v1helper.TolerationsTolerateTaint(attrs.Pod.Spec.Tolerations, &v1.Taint{ + Key: algorithm.TaintNodeMemoryPressure, + Effect: v1.TaintEffectNoSchedule, + }) { + return lifecycle.PodAdmitResult{Admit: true} + } } // reject pods when under memory pressure (if pod is best effort), or if under disk pressure. @@ -239,8 +251,8 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act return nil } m.dedicatedImageFs = &hasImageFs - m.resourceToRankFunc = buildResourceToRankFunc(hasImageFs) - m.resourceToNodeReclaimFuncs = buildResourceToNodeReclaimFuncs(m.imageGC, m.containerGC, hasImageFs) + m.signalToRankFunc = buildSignalToRankFunc(hasImageFs) + m.signalToNodeReclaimFuncs = buildSignalToNodeReclaimFuncs(m.imageGC, m.containerGC, hasImageFs) } activePods := podFunc() @@ -333,26 +345,26 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act } } - // determine the set of resources under starvation - starvedResources := getStarvedResources(thresholds) - if len(starvedResources) == 0 { + if len(thresholds) == 0 { glog.V(3).Infof("eviction manager: no resources are starved") return nil } - // rank the resources to reclaim by eviction priority - sort.Sort(byEvictionPriority(starvedResources)) - resourceToReclaim := starvedResources[0] + // rank the thresholds by eviction priority + sort.Sort(byEvictionPriority(thresholds)) + thresholdToReclaim := thresholds[0] + resourceToReclaim, found := signalToResource[thresholdToReclaim.Signal] + if !found { + glog.V(3).Infof("eviction manager: threshold %s was crossed, but reclaim is not implemented for this threshold.", thresholdToReclaim.Signal) + return nil + } glog.Warningf("eviction manager: attempting to reclaim %v", resourceToReclaim) - // determine if this is a soft or hard eviction associated with the resource - softEviction := isSoftEvictionThresholds(thresholds, resourceToReclaim) - // record an event about the resources we are now attempting to reclaim via eviction m.recorder.Eventf(m.nodeRef, v1.EventTypeWarning, "EvictionThresholdMet", "Attempting to reclaim %s", resourceToReclaim) // check if there are node-level resources we can reclaim to reduce pressure before evicting end-user pods. - if m.reclaimNodeLevelResources(resourceToReclaim) { + if m.reclaimNodeLevelResources(thresholdToReclaim.Signal, resourceToReclaim) { glog.Infof("eviction manager: able to reduce %v pressure without evicting pods.", resourceToReclaim) return nil } @@ -360,9 +372,9 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act glog.Infof("eviction manager: must evict pod(s) to reclaim %v", resourceToReclaim) // rank the pods for eviction - rank, ok := m.resourceToRankFunc[resourceToReclaim] + rank, ok := m.signalToRankFunc[thresholdToReclaim.Signal] if !ok { - glog.Errorf("eviction manager: no ranking function for resource %s", resourceToReclaim) + glog.Errorf("eviction manager: no ranking function for signal %s", thresholdToReclaim.Signal) return nil } @@ -388,30 +400,13 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act // we kill at most a single pod during each eviction interval for i := range activePods { pod := activePods[i] - // If the pod is marked as critical and static, and support for critical pod annotations is enabled, - // do not evict such pods. Static pods are not re-admitted after evictions. - // https://github.com/kubernetes/kubernetes/issues/40573 has more details. - if utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) && - kubelettypes.IsCriticalPod(pod) && kubepod.IsStaticPod(pod) { - continue - } - status := v1.PodStatus{ - Phase: v1.PodFailed, - Message: fmt.Sprintf(message, resourceToReclaim), - Reason: reason, - } - // record that we are evicting the pod - m.recorder.Eventf(pod, v1.EventTypeWarning, reason, fmt.Sprintf(message, resourceToReclaim)) gracePeriodOverride := int64(0) - if softEviction { + if !isHardEvictionThreshold(thresholdToReclaim) { gracePeriodOverride = m.config.MaxPodGracePeriodSeconds } - // this is a blocking call and should only return when the pod and its containers are killed. - err := m.killPodFunc(pod, status, &gracePeriodOverride) - if err != nil { - glog.Warningf("eviction manager: error while evicting pod %s: %v", format.Pod(pod), err) + if m.evictPod(pod, gracePeriodOverride, evictionMessage(resourceToReclaim, pod, statsFunc)) { + return []*v1.Pod{pod} } - return []*v1.Pod{pod} } glog.Infof("eviction manager: unable to evict any pods from the node") return nil @@ -419,13 +414,15 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act func (m *managerImpl) waitForPodsCleanup(podCleanedUpFunc PodCleanedUpFunc, pods []*v1.Pod) { timeout := m.clock.NewTimer(podCleanupTimeout) - tick := m.clock.Tick(podCleanupPollFreq) + defer timeout.Stop() + ticker := m.clock.NewTicker(podCleanupPollFreq) + defer ticker.Stop() for { select { case <-timeout.C(): glog.Warningf("eviction manager: timed out waiting for pods %s to be cleaned up", format.Pods(pods)) return - case <-tick: + case <-ticker.C(): for i, pod := range pods { if !podCleanedUpFunc(pod) { break @@ -440,8 +437,8 @@ func (m *managerImpl) waitForPodsCleanup(podCleanedUpFunc PodCleanedUpFunc, pods } // reclaimNodeLevelResources attempts to reclaim node level resources. returns true if thresholds were satisfied and no pod eviction is required. -func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceName) bool { - nodeReclaimFuncs := m.resourceToNodeReclaimFuncs[resourceToReclaim] +func (m *managerImpl) reclaimNodeLevelResources(signalToReclaim evictionapi.Signal, resourceToReclaim v1.ResourceName) bool { + nodeReclaimFuncs := m.signalToNodeReclaimFuncs[signalToReclaim] for _, nodeReclaimFunc := range nodeReclaimFuncs { // attempt to reclaim the pressured resource. if err := nodeReclaimFunc(); err != nil { @@ -512,7 +509,7 @@ func (m *managerImpl) emptyDirLimitEviction(podStats statsapi.PodStats, pod *v1. used := podVolumeUsed[pod.Spec.Volumes[i].Name] if used != nil && size != nil && size.Sign() == 1 && used.Cmp(*size) > 0 { // the emptyDir usage exceeds the size limit, evict the pod - return m.evictPod(pod, v1.ResourceName("EmptyDir"), fmt.Sprintf("emptyDir usage exceeds the limit %q", size.String())) + return m.evictPod(pod, 0, fmt.Sprintf(emptyDirMessage, pod.Spec.Volumes[i].Name, size.String())) } } } @@ -540,10 +537,11 @@ func (m *managerImpl) podEphemeralStorageLimitEviction(podStats statsapi.PodStat return false } - podEphemeralStorageTotalUsage.Add(podEphemeralUsage[resourceDisk]) - if podEphemeralStorageTotalUsage.Cmp(podLimits[v1.ResourceEphemeralStorage]) > 0 { + podEphemeralStorageTotalUsage.Add(podEphemeralUsage[v1.ResourceEphemeralStorage]) + podEphemeralStorageLimit := podLimits[v1.ResourceEphemeralStorage] + if podEphemeralStorageTotalUsage.Cmp(podEphemeralStorageLimit) > 0 { // the total usage of pod exceeds the total size limit of containers, evict the pod - return m.evictPod(pod, v1.ResourceEphemeralStorage, fmt.Sprintf("pod ephemeral local storage usage exceeds the total limit of containers %v", podLimits[v1.ResourceEphemeralStorage])) + return m.evictPod(pod, 0, fmt.Sprintf(podEphemeralStorageMessage, podEphemeralStorageLimit.String())) } return false } @@ -565,7 +563,7 @@ func (m *managerImpl) containerEphemeralStorageLimitEviction(podStats statsapi.P if ephemeralStorageThreshold, ok := thresholdsMap[containerStat.Name]; ok { if ephemeralStorageThreshold.Cmp(*containerUsed) < 0 { - return m.evictPod(pod, v1.ResourceEphemeralStorage, fmt.Sprintf("container's ephemeral local storage usage exceeds the limit %q", ephemeralStorageThreshold.String())) + return m.evictPod(pod, 0, fmt.Sprintf(containerEphemeralStorageMessage, containerStat.Name, ephemeralStorageThreshold.String())) } } @@ -573,21 +571,24 @@ func (m *managerImpl) containerEphemeralStorageLimitEviction(podStats statsapi.P return false } -func (m *managerImpl) evictPod(pod *v1.Pod, resourceName v1.ResourceName, evictMsg string) bool { +func (m *managerImpl) evictPod(pod *v1.Pod, gracePeriodOverride int64, evictMsg string) bool { + // If the pod is marked as critical and static, and support for critical pod annotations is enabled, + // do not evict such pods. Static pods are not re-admitted after evictions. + // https://github.com/kubernetes/kubernetes/issues/40573 has more details. if utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) && kubelettypes.IsCriticalPod(pod) && kubepod.IsStaticPod(pod) { - glog.Errorf("eviction manager: cannot evict a critical pod %s", format.Pod(pod)) + glog.Errorf("eviction manager: cannot evict a critical static pod %s", format.Pod(pod)) return false } status := v1.PodStatus{ Phase: v1.PodFailed, - Message: fmt.Sprintf(message, resourceName), + Message: evictMsg, Reason: reason, } // record that we are evicting the pod m.recorder.Eventf(pod, v1.EventTypeWarning, reason, evictMsg) - gracePeriod := int64(0) - err := m.killPodFunc(pod, status, &gracePeriod) + // this is a blocking call and should only return when the pod and its containers are killed. + err := m.killPodFunc(pod, status, &gracePeriodOverride) if err != nil { glog.Errorf("eviction manager: pod %s failed to evict %v", format.Pod(pod), err) } else { diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index 444dbbd1729..6f75a84244d 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -41,19 +41,17 @@ const ( // the reason reported back in status. reason = "Evicted" // the message associated with the reason. - message = "The node was low on resource: %v." - // disk, in bytes. internal to this module, used to account for local disk usage. - resourceDisk v1.ResourceName = "disk" + message = "The node was low on resource: %v. " + // additional information for containers exceeding requests + containerMessage = "Container %s was using %s, which exceeds its request of %s. " + // additional information for containers which have exceeded their ES limit + containerEphemeralStorageMessage = "Container %s exceeded its local ephemeral storage limit %q. " + // additional information for pods which have exceeded their ES limit + podEphemeralStorageMessage = "Pod ephemeral local storage usage exceeds the total limit of containers %s. " + // additional information for empty-dir volumes which have exceeded their size limit + emptyDirMessage = "Usage of EmptyDir volume %q exceeds the limit %q. " // inodes, number. internal to this module, used to account for local disk inode consumption. resourceInodes v1.ResourceName = "inodes" - // imagefs, in bytes. internal to this module, used to account for local image filesystem usage. - resourceImageFs v1.ResourceName = "imagefs" - // imagefs inodes, number. internal to this module, used to account for local image filesystem inodes. - resourceImageFsInodes v1.ResourceName = "imagefsInodes" - // nodefs, in bytes. internal to this module, used to account for local node root filesystem usage. - resourceNodeFs v1.ResourceName = "nodefs" - // nodefs inodes, number. internal to this module, used to account for local node root filesystem inodes. - resourceNodeFsInodes v1.ResourceName = "nodefsInodes" // this prevents constantly updating the memcg notifier if synchronize // is run frequently. notifierRefreshInterval = 10 * time.Second @@ -64,8 +62,6 @@ var ( signalToNodeCondition map[evictionapi.Signal]v1.NodeConditionType // signalToResource maps a Signal to its associated Resource. signalToResource map[evictionapi.Signal]v1.ResourceName - // resourceClaimToSignal maps a Resource that can be reclaimed to its associated Signal - resourceClaimToSignal map[v1.ResourceName][]evictionapi.Signal ) func init() { @@ -83,17 +79,10 @@ func init() { signalToResource = map[evictionapi.Signal]v1.ResourceName{} signalToResource[evictionapi.SignalMemoryAvailable] = v1.ResourceMemory signalToResource[evictionapi.SignalAllocatableMemoryAvailable] = v1.ResourceMemory - signalToResource[evictionapi.SignalImageFsAvailable] = resourceImageFs - signalToResource[evictionapi.SignalImageFsInodesFree] = resourceImageFsInodes - signalToResource[evictionapi.SignalNodeFsAvailable] = resourceNodeFs - signalToResource[evictionapi.SignalNodeFsInodesFree] = resourceNodeFsInodes - - // maps resource to signals (the following resource could be reclaimed) - resourceClaimToSignal = map[v1.ResourceName][]evictionapi.Signal{} - resourceClaimToSignal[resourceNodeFs] = []evictionapi.Signal{evictionapi.SignalNodeFsAvailable} - resourceClaimToSignal[resourceImageFs] = []evictionapi.Signal{evictionapi.SignalImageFsAvailable} - resourceClaimToSignal[resourceNodeFsInodes] = []evictionapi.Signal{evictionapi.SignalNodeFsInodesFree} - resourceClaimToSignal[resourceImageFsInodes] = []evictionapi.Signal{evictionapi.SignalImageFsInodesFree} + signalToResource[evictionapi.SignalImageFsAvailable] = v1.ResourceEphemeralStorage + signalToResource[evictionapi.SignalImageFsInodesFree] = resourceInodes + signalToResource[evictionapi.SignalNodeFsAvailable] = v1.ResourceEphemeralStorage + signalToResource[evictionapi.SignalNodeFsInodesFree] = resourceInodes } // validSignal returns true if the signal is supported. @@ -310,10 +299,10 @@ func diskUsage(fsStats *statsapi.FsStats) *resource.Quantity { // inodeUsage converts inodes consumed into a resource quantity. func inodeUsage(fsStats *statsapi.FsStats) *resource.Quantity { if fsStats == nil || fsStats.InodesUsed == nil { - return &resource.Quantity{Format: resource.BinarySI} + return &resource.Quantity{Format: resource.DecimalSI} } usage := int64(*fsStats.InodesUsed) - return resource.NewQuantity(usage, resource.BinarySI) + return resource.NewQuantity(usage, resource.DecimalSI) } // memoryUsage converts working set into a resource quantity. @@ -343,7 +332,7 @@ func localVolumeNames(pod *v1.Pod) []string { // containerUsage aggregates container disk usage and inode consumption for the specified stats to measure. func containerUsage(podStats statsapi.PodStats, statsToMeasure []fsStatsType) v1.ResourceList { disk := resource.Quantity{Format: resource.BinarySI} - inodes := resource.Quantity{Format: resource.BinarySI} + inodes := resource.Quantity{Format: resource.DecimalSI} for _, container := range podStats.Containers { if hasFsStatsType(statsToMeasure, fsStatsRoot) { disk.Add(*diskUsage(container.Rootfs)) @@ -355,15 +344,15 @@ func containerUsage(podStats statsapi.PodStats, statsToMeasure []fsStatsType) v1 } } return v1.ResourceList{ - resourceDisk: disk, - resourceInodes: inodes, + v1.ResourceEphemeralStorage: disk, + resourceInodes: inodes, } } // podLocalVolumeUsage aggregates pod local volumes disk usage and inode consumption for the specified stats to measure. func podLocalVolumeUsage(volumeNames []string, podStats statsapi.PodStats) v1.ResourceList { disk := resource.Quantity{Format: resource.BinarySI} - inodes := resource.Quantity{Format: resource.BinarySI} + inodes := resource.Quantity{Format: resource.DecimalSI} for _, volumeName := range volumeNames { for _, volumeStats := range podStats.VolumeStats { if volumeStats.Name == volumeName { @@ -374,29 +363,29 @@ func podLocalVolumeUsage(volumeNames []string, podStats statsapi.PodStats) v1.Re } } return v1.ResourceList{ - resourceDisk: disk, - resourceInodes: inodes, + v1.ResourceEphemeralStorage: disk, + resourceInodes: inodes, } } // podDiskUsage aggregates pod disk usage and inode consumption for the specified stats to measure. func podDiskUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsStatsType) (v1.ResourceList, error) { disk := resource.Quantity{Format: resource.BinarySI} - inodes := resource.Quantity{Format: resource.BinarySI} + inodes := resource.Quantity{Format: resource.DecimalSI} containerUsageList := containerUsage(podStats, statsToMeasure) - disk.Add(containerUsageList[resourceDisk]) + disk.Add(containerUsageList[v1.ResourceEphemeralStorage]) inodes.Add(containerUsageList[resourceInodes]) if hasFsStatsType(statsToMeasure, fsStatsLocalVolumeSource) { volumeNames := localVolumeNames(pod) podLocalVolumeUsageList := podLocalVolumeUsage(volumeNames, podStats) - disk.Add(podLocalVolumeUsageList[resourceDisk]) + disk.Add(podLocalVolumeUsageList[v1.ResourceEphemeralStorage]) inodes.Add(podLocalVolumeUsageList[resourceInodes]) } return v1.ResourceList{ - resourceDisk: disk, - resourceInodes: inodes, + v1.ResourceEphemeralStorage: disk, + resourceInodes: inodes, }, nil } @@ -426,21 +415,21 @@ func localEphemeralVolumeNames(pod *v1.Pod) []string { // podLocalEphemeralStorageUsage aggregates pod local ephemeral storage usage and inode consumption for the specified stats to measure. func podLocalEphemeralStorageUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsStatsType) (v1.ResourceList, error) { disk := resource.Quantity{Format: resource.BinarySI} - inodes := resource.Quantity{Format: resource.BinarySI} + inodes := resource.Quantity{Format: resource.DecimalSI} containerUsageList := containerUsage(podStats, statsToMeasure) - disk.Add(containerUsageList[resourceDisk]) + disk.Add(containerUsageList[v1.ResourceEphemeralStorage]) inodes.Add(containerUsageList[resourceInodes]) if hasFsStatsType(statsToMeasure, fsStatsLocalVolumeSource) { volumeNames := localEphemeralVolumeNames(pod) podLocalVolumeUsageList := podLocalVolumeUsage(volumeNames, podStats) - disk.Add(podLocalVolumeUsageList[resourceDisk]) + disk.Add(podLocalVolumeUsageList[v1.ResourceEphemeralStorage]) inodes.Add(podLocalVolumeUsageList[resourceInodes]) } return v1.ResourceList{ - resourceDisk: disk, - resourceInodes: inodes, + v1.ResourceEphemeralStorage: disk, + resourceInodes: inodes, }, nil } @@ -605,7 +594,7 @@ func memory(stats statsFunc) cmpFunc { // max(max of init container requests, sum of container requests) func podRequest(pod *v1.Pod, resourceName v1.ResourceName) resource.Quantity { containerValue := resource.Quantity{Format: resource.BinarySI} - if resourceName == resourceDisk && !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) { + if resourceName == v1.ResourceEphemeralStorage && !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) { // if the local storage capacity isolation feature gate is disabled, pods request 0 disk return containerValue } @@ -613,7 +602,7 @@ func podRequest(pod *v1.Pod, resourceName v1.ResourceName) resource.Quantity { switch resourceName { case v1.ResourceMemory: containerValue.Add(*pod.Spec.Containers[i].Resources.Requests.Memory()) - case resourceDisk: + case v1.ResourceEphemeralStorage: containerValue.Add(*pod.Spec.Containers[i].Resources.Requests.StorageEphemeral()) } } @@ -624,7 +613,7 @@ func podRequest(pod *v1.Pod, resourceName v1.ResourceName) resource.Quantity { if initValue.Cmp(*pod.Spec.InitContainers[i].Resources.Requests.Memory()) < 0 { initValue = *pod.Spec.InitContainers[i].Resources.Requests.Memory() } - case resourceDisk: + case v1.ResourceEphemeralStorage: if initValue.Cmp(*pod.Spec.InitContainers[i].Resources.Requests.StorageEphemeral()) < 0 { initValue = *pod.Spec.InitContainers[i].Resources.Requests.StorageEphemeral() } @@ -681,9 +670,9 @@ func disk(stats statsFunc, fsStatsToMeasure []fsStatsType, diskResource v1.Resou // adjust p1, p2 usage relative to the request (if any) p1Disk := p1Usage[diskResource] p2Disk := p2Usage[diskResource] - p1Request := podRequest(p1, resourceDisk) + p1Request := podRequest(p1, v1.ResourceEphemeralStorage) p1Disk.Sub(p1Request) - p2Request := podRequest(p2, resourceDisk) + p2Request := podRequest(p2, v1.ResourceEphemeralStorage) p2Disk.Sub(p2Request) // prioritize evicting the pod which has the larger consumption of disk return p2Disk.Cmp(p1Disk) @@ -716,14 +705,15 @@ func rankDiskPressureFunc(fsStatsToMeasure []fsStatsType, diskResource v1.Resour } // byEvictionPriority implements sort.Interface for []v1.ResourceName. -type byEvictionPriority []v1.ResourceName +type byEvictionPriority []evictionapi.Threshold func (a byEvictionPriority) Len() int { return len(a) } func (a byEvictionPriority) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -// Less ranks memory before all other resources. +// Less ranks memory before all other resources, and ranks thresholds with no resource to reclaim last func (a byEvictionPriority) Less(i, j int) bool { - return a[i] == v1.ResourceMemory + _, jSignalHasResource := signalToResource[a[j].Signal] + return a[i].Signal == evictionapi.SignalMemoryAvailable || a[i].Signal == evictionapi.SignalAllocatableMemoryAvailable || !jSignalHasResource } // makeSignalObservations derives observations using the specified summary provider. @@ -761,8 +751,8 @@ func makeSignalObservations(summary *statsapi.Summary) (signalObservations, stat } if nodeFs.InodesFree != nil && nodeFs.Inodes != nil { result[evictionapi.SignalNodeFsInodesFree] = signalObservation{ - available: resource.NewQuantity(int64(*nodeFs.InodesFree), resource.BinarySI), - capacity: resource.NewQuantity(int64(*nodeFs.Inodes), resource.BinarySI), + available: resource.NewQuantity(int64(*nodeFs.InodesFree), resource.DecimalSI), + capacity: resource.NewQuantity(int64(*nodeFs.Inodes), resource.DecimalSI), time: nodeFs.Time, } } @@ -777,8 +767,8 @@ func makeSignalObservations(summary *statsapi.Summary) (signalObservations, stat } if imageFs.InodesFree != nil && imageFs.Inodes != nil { result[evictionapi.SignalImageFsInodesFree] = signalObservation{ - available: resource.NewQuantity(int64(*imageFs.InodesFree), resource.BinarySI), - capacity: resource.NewQuantity(int64(*imageFs.Inodes), resource.BinarySI), + available: resource.NewQuantity(int64(*imageFs.InodesFree), resource.DecimalSI), + capacity: resource.NewQuantity(int64(*imageFs.Inodes), resource.DecimalSI), time: imageFs.Time, } } @@ -1006,57 +996,34 @@ func compareThresholdValue(a evictionapi.ThresholdValue, b evictionapi.Threshold return a.Percentage == b.Percentage } -// getStarvedResources returns the set of resources that are starved based on thresholds met. -func getStarvedResources(thresholds []evictionapi.Threshold) []v1.ResourceName { - results := []v1.ResourceName{} - for _, threshold := range thresholds { - if starvedResource, found := signalToResource[threshold.Signal]; found { - results = append(results, starvedResource) - } - } - return results -} - -// isSoftEviction returns true if the thresholds met for the starved resource are only soft thresholds -func isSoftEvictionThresholds(thresholds []evictionapi.Threshold, starvedResource v1.ResourceName) bool { - for _, threshold := range thresholds { - if resourceToCheck := signalToResource[threshold.Signal]; resourceToCheck != starvedResource { - continue - } - if isHardEvictionThreshold(threshold) { - return false - } - } - return true -} - // isHardEvictionThreshold returns true if eviction should immediately occur func isHardEvictionThreshold(threshold evictionapi.Threshold) bool { return threshold.GracePeriod == time.Duration(0) } -// buildResourceToRankFunc returns ranking functions associated with resources -func buildResourceToRankFunc(withImageFs bool) map[v1.ResourceName]rankFunc { - resourceToRankFunc := map[v1.ResourceName]rankFunc{ - v1.ResourceMemory: rankMemoryPressure, +// buildSignalToRankFunc returns ranking functions associated with resources +func buildSignalToRankFunc(withImageFs bool) map[evictionapi.Signal]rankFunc { + signalToRankFunc := map[evictionapi.Signal]rankFunc{ + evictionapi.SignalMemoryAvailable: rankMemoryPressure, + evictionapi.SignalAllocatableMemoryAvailable: rankMemoryPressure, } // usage of an imagefs is optional if withImageFs { // with an imagefs, nodefs pod rank func for eviction only includes logs and local volumes - resourceToRankFunc[resourceNodeFs] = rankDiskPressureFunc([]fsStatsType{fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk) - resourceToRankFunc[resourceNodeFsInodes] = rankDiskPressureFunc([]fsStatsType{fsStatsLogs, fsStatsLocalVolumeSource}, resourceInodes) + signalToRankFunc[evictionapi.SignalNodeFsAvailable] = rankDiskPressureFunc([]fsStatsType{fsStatsLogs, fsStatsLocalVolumeSource}, v1.ResourceEphemeralStorage) + signalToRankFunc[evictionapi.SignalNodeFsInodesFree] = rankDiskPressureFunc([]fsStatsType{fsStatsLogs, fsStatsLocalVolumeSource}, resourceInodes) // with an imagefs, imagefs pod rank func for eviction only includes rootfs - resourceToRankFunc[resourceImageFs] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot}, resourceDisk) - resourceToRankFunc[resourceImageFsInodes] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot}, resourceInodes) + signalToRankFunc[evictionapi.SignalImageFsAvailable] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot}, v1.ResourceEphemeralStorage) + signalToRankFunc[evictionapi.SignalImageFsInodesFree] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot}, resourceInodes) } else { // without an imagefs, nodefs pod rank func for eviction looks at all fs stats. // since imagefs and nodefs share a common device, they share common ranking functions. - resourceToRankFunc[resourceNodeFs] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk) - resourceToRankFunc[resourceNodeFsInodes] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceInodes) - resourceToRankFunc[resourceImageFs] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk) - resourceToRankFunc[resourceImageFsInodes] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceInodes) + signalToRankFunc[evictionapi.SignalNodeFsAvailable] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, v1.ResourceEphemeralStorage) + signalToRankFunc[evictionapi.SignalNodeFsInodesFree] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceInodes) + signalToRankFunc[evictionapi.SignalImageFsAvailable] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, v1.ResourceEphemeralStorage) + signalToRankFunc[evictionapi.SignalImageFsInodesFree] = rankDiskPressureFunc([]fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceInodes) } - return resourceToRankFunc + return signalToRankFunc } // PodIsEvicted returns true if the reported pod status is due to an eviction. @@ -1064,26 +1031,57 @@ func PodIsEvicted(podStatus v1.PodStatus) bool { return podStatus.Phase == v1.PodFailed && podStatus.Reason == reason } -// buildResourceToNodeReclaimFuncs returns reclaim functions associated with resources. -func buildResourceToNodeReclaimFuncs(imageGC ImageGC, containerGC ContainerGC, withImageFs bool) map[v1.ResourceName]nodeReclaimFuncs { - resourceToReclaimFunc := map[v1.ResourceName]nodeReclaimFuncs{} +// buildSignalToNodeReclaimFuncs returns reclaim functions associated with resources. +func buildSignalToNodeReclaimFuncs(imageGC ImageGC, containerGC ContainerGC, withImageFs bool) map[evictionapi.Signal]nodeReclaimFuncs { + signalToReclaimFunc := map[evictionapi.Signal]nodeReclaimFuncs{} // usage of an imagefs is optional if withImageFs { // with an imagefs, nodefs pressure should just delete logs - resourceToReclaimFunc[resourceNodeFs] = nodeReclaimFuncs{} - resourceToReclaimFunc[resourceNodeFsInodes] = nodeReclaimFuncs{} + signalToReclaimFunc[evictionapi.SignalNodeFsAvailable] = nodeReclaimFuncs{} + signalToReclaimFunc[evictionapi.SignalNodeFsInodesFree] = nodeReclaimFuncs{} // with an imagefs, imagefs pressure should delete unused images - resourceToReclaimFunc[resourceImageFs] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} - resourceToReclaimFunc[resourceImageFsInodes] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} + signalToReclaimFunc[evictionapi.SignalImageFsAvailable] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} + signalToReclaimFunc[evictionapi.SignalImageFsInodesFree] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} } else { // without an imagefs, nodefs pressure should delete logs, and unused images // since imagefs and nodefs share a common device, they share common reclaim functions - resourceToReclaimFunc[resourceNodeFs] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} - resourceToReclaimFunc[resourceNodeFsInodes] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} - resourceToReclaimFunc[resourceImageFs] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} - resourceToReclaimFunc[resourceImageFsInodes] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} + signalToReclaimFunc[evictionapi.SignalNodeFsAvailable] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} + signalToReclaimFunc[evictionapi.SignalNodeFsInodesFree] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} + signalToReclaimFunc[evictionapi.SignalImageFsAvailable] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} + signalToReclaimFunc[evictionapi.SignalImageFsInodesFree] = nodeReclaimFuncs{containerGC.DeleteAllUnusedContainers, imageGC.DeleteUnusedImages} } - return resourceToReclaimFunc + return signalToReclaimFunc +} + +// evictionMessage constructs a useful message about why an eviction occurred +func evictionMessage(resourceToReclaim v1.ResourceName, pod *v1.Pod, stats statsFunc) string { + message := fmt.Sprintf(message, resourceToReclaim) + podStats, ok := stats(pod) + if !ok { + return message + } + for _, containerStats := range podStats.Containers { + for _, container := range pod.Spec.Containers { + if container.Name == containerStats.Name { + requests := container.Resources.Requests[resourceToReclaim] + var usage *resource.Quantity + switch resourceToReclaim { + case v1.ResourceEphemeralStorage: + if containerStats.Rootfs != nil && containerStats.Rootfs.UsedBytes != nil && containerStats.Logs != nil && containerStats.Logs.UsedBytes != nil { + usage = resource.NewQuantity(int64(*containerStats.Rootfs.UsedBytes+*containerStats.Logs.UsedBytes), resource.BinarySI) + } + case v1.ResourceMemory: + if containerStats.Memory != nil && containerStats.Memory.WorkingSetBytes != nil { + usage = resource.NewQuantity(int64(*containerStats.Memory.WorkingSetBytes), resource.BinarySI) + } + } + if usage != nil && usage.Cmp(requests) > 0 { + message += fmt.Sprintf(containerMessage, container.Name, usage.String(), requests.String()) + } + } + } + } + return message } // thresholdStopCh is a ThresholdStopCh which can only be closed after notifierRefreshInterval time has passed diff --git a/pkg/kubelet/eviction/helpers_test.go b/pkg/kubelet/eviction/helpers_test.go index 094940bea98..d7ce3c8586f 100644 --- a/pkg/kubelet/eviction/helpers_test.go +++ b/pkg/kubelet/eviction/helpers_test.go @@ -19,6 +19,7 @@ package eviction import ( "fmt" "reflect" + "sort" "sync" "testing" "time" @@ -29,12 +30,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" utilfeature "k8s.io/apiserver/pkg/util/feature" - api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/features" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - "k8s.io/kubernetes/pkg/quota" ) func quantityMustParse(value string) *resource.Quantity { @@ -470,7 +469,7 @@ func TestOrderedByExceedsRequestDisk(t *testing.T) { return result, found } pods := []*v1.Pod{below, exceeds} - orderedBy(exceedDiskRequests(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods) + orderedBy(exceedDiskRequests(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, v1.ResourceEphemeralStorage)).Sort(pods) expected := []*v1.Pod{exceeds, below} for i := range expected { @@ -584,7 +583,7 @@ func TestOrderedbyDisk(t *testing.T) { return result, found } pods := []*v1.Pod{pod1, pod2, pod3, pod4, pod5, pod6} - orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods) + orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, v1.ResourceEphemeralStorage)).Sort(pods) expected := []*v1.Pod{pod1, pod3, pod2, pod4, pod5, pod6} for i := range expected { if pods[i] != expected[i] { @@ -651,7 +650,7 @@ func TestOrderedbyDiskDisableLocalStorage(t *testing.T) { return result, found } pods := []*v1.Pod{pod1, pod3, pod2, pod4, pod5, pod6} - orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods) + orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, v1.ResourceEphemeralStorage)).Sort(pods) expected := []*v1.Pod{pod5, pod3, pod1, pod6, pod4, pod2} for i := range expected { if pods[i] != expected[i] { @@ -780,7 +779,7 @@ func TestOrderedByPriorityDisk(t *testing.T) { pods := []*v1.Pod{pod8, pod7, pod6, pod5, pod4, pod3, pod2, pod1} expected := []*v1.Pod{pod1, pod2, pod3, pod4, pod5, pod6, pod7, pod8} fsStatsToMeasure := []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource} - orderedBy(exceedDiskRequests(statsFn, fsStatsToMeasure, resourceDisk), priority, disk(statsFn, fsStatsToMeasure, resourceDisk)).Sort(pods) + orderedBy(exceedDiskRequests(statsFn, fsStatsToMeasure, v1.ResourceEphemeralStorage), priority, disk(statsFn, fsStatsToMeasure, v1.ResourceEphemeralStorage)).Sort(pods) for i := range expected { if pods[i] != expected[i] { t.Errorf("Expected pod[%d]: %s, but got: %s", i, expected[i].Name, pods[i].Name) @@ -932,6 +931,80 @@ func TestOrderedByPriorityMemory(t *testing.T) { } } +func TestSortByEvictionPriority(t *testing.T) { + for _, tc := range []struct { + name string + thresholds []evictionapi.Threshold + expected []evictionapi.Threshold + }{ + { + name: "empty threshold list", + thresholds: []evictionapi.Threshold{}, + expected: []evictionapi.Threshold{}, + }, + { + name: "memory first, PID last", + thresholds: []evictionapi.Threshold{ + { + Signal: evictionapi.SignalPIDAvailable, + }, + { + Signal: evictionapi.SignalNodeFsAvailable, + }, + { + Signal: evictionapi.SignalMemoryAvailable, + }, + }, + expected: []evictionapi.Threshold{ + { + Signal: evictionapi.SignalMemoryAvailable, + }, + { + Signal: evictionapi.SignalNodeFsAvailable, + }, + { + Signal: evictionapi.SignalPIDAvailable, + }, + }, + }, + { + name: "allocatable memory first, PID last", + thresholds: []evictionapi.Threshold{ + { + Signal: evictionapi.SignalPIDAvailable, + }, + { + Signal: evictionapi.SignalNodeFsAvailable, + }, + { + Signal: evictionapi.SignalAllocatableMemoryAvailable, + }, + }, + expected: []evictionapi.Threshold{ + { + Signal: evictionapi.SignalAllocatableMemoryAvailable, + }, + { + Signal: evictionapi.SignalNodeFsAvailable, + }, + { + Signal: evictionapi.SignalPIDAvailable, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + sort.Sort(byEvictionPriority(tc.thresholds)) + for i := range tc.expected { + if tc.thresholds[i].Signal != tc.expected[i].Signal { + t.Errorf("At index %d, expected threshold with signal %s, but got %s", i, tc.expected[i].Signal, tc.thresholds[i].Signal) + } + } + + }) + } +} + type fakeSummaryProvider struct { result *statsapi.Summary } @@ -1622,47 +1695,6 @@ func TestHasNodeConditions(t *testing.T) { } } -func TestGetStarvedResources(t *testing.T) { - testCases := map[string]struct { - inputs []evictionapi.Threshold - result []v1.ResourceName - }{ - "memory.available": { - inputs: []evictionapi.Threshold{ - {Signal: evictionapi.SignalMemoryAvailable}, - }, - result: []v1.ResourceName{v1.ResourceMemory}, - }, - "imagefs.available": { - inputs: []evictionapi.Threshold{ - {Signal: evictionapi.SignalImageFsAvailable}, - }, - result: []v1.ResourceName{resourceImageFs}, - }, - "nodefs.available": { - inputs: []evictionapi.Threshold{ - {Signal: evictionapi.SignalNodeFsAvailable}, - }, - result: []v1.ResourceName{resourceNodeFs}, - }, - } - var internalResourceNames = func(in []v1.ResourceName) []api.ResourceName { - var out []api.ResourceName - for _, name := range in { - out = append(out, api.ResourceName(name)) - } - return out - } - for testName, testCase := range testCases { - actual := getStarvedResources(testCase.inputs) - actualSet := quota.ToSet(internalResourceNames(actual)) - expectedSet := quota.ToSet(internalResourceNames(testCase.result)) - if !actualSet.Equal(expectedSet) { - t.Errorf("Test case: %s, expected: %v, actual: %v", testName, expectedSet, actualSet) - } - } -} - func TestParsePercentage(t *testing.T) { testCases := map[string]struct { hasError bool diff --git a/pkg/kubelet/kubelet_getters.go b/pkg/kubelet/kubelet_getters.go index 04284f6e3ea..89077eb7f73 100644 --- a/pkg/kubelet/kubelet_getters.go +++ b/pkg/kubelet/kubelet_getters.go @@ -174,6 +174,16 @@ func (kl *Kubelet) GetPodByName(namespace, name string) (*v1.Pod, bool) { return kl.podManager.GetPodByName(namespace, name) } +// GetPodByCgroupfs provides the pod that maps to the specified cgroup, as well +// as whether the pod was found. +func (kl *Kubelet) GetPodByCgroupfs(cgroupfs string) (*v1.Pod, bool) { + pcm := kl.containerManager.NewPodContainerManager() + if result, podUID := pcm.IsPodCgroup(cgroupfs); result { + return kl.podManager.GetPodByUID(podUID) + } + return nil, false +} + // GetHostname Returns the hostname as the kubelet sees it. func (kl *Kubelet) GetHostname() string { return kl.hostname diff --git a/pkg/kubelet/kubelet_node_status_test.go b/pkg/kubelet/kubelet_node_status_test.go index 20ab8c8c74e..10c3f7f8565 100644 --- a/pkg/kubelet/kubelet_node_status_test.go +++ b/pkg/kubelet/kubelet_node_status_test.go @@ -642,8 +642,8 @@ func TestUpdateExistingNodeStatusTimeout(t *testing.T) { assert.Error(t, kubelet.updateNodeStatus()) // should have attempted multiple times - if actualAttempts := atomic.LoadInt64(&attempts); actualAttempts != nodeStatusUpdateRetry { - t.Errorf("Expected %d attempts, got %d", nodeStatusUpdateRetry, actualAttempts) + if actualAttempts := atomic.LoadInt64(&attempts); actualAttempts < nodeStatusUpdateRetry { + t.Errorf("Expected at least %d attempts, got %d", nodeStatusUpdateRetry, actualAttempts) } } diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index d4387a34afc..46a17a01a58 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -207,6 +207,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h VolumePath: volumePath, PodDir: podDir, ContainerName: container.Name, + ReadOnly: mount.ReadOnly || vol.Mounter.GetAttributes().ReadOnly, }) if err != nil { // Don't pass detailed error back to the user because it could give information about host filesystem diff --git a/pkg/kubelet/kubeletconfig/BUILD b/pkg/kubelet/kubeletconfig/BUILD index ae3e99f674e..c409bd2a3ff 100644 --- a/pkg/kubelet/kubeletconfig/BUILD +++ b/pkg/kubelet/kubeletconfig/BUILD @@ -19,12 +19,12 @@ go_library( "//pkg/kubelet/kubeletconfig/checkpoint:go_default_library", "//pkg/kubelet/kubeletconfig/checkpoint/store:go_default_library", "//pkg/kubelet/kubeletconfig/status:go_default_library", - "//pkg/kubelet/kubeletconfig/util/equal:go_default_library", "//pkg/kubelet/kubeletconfig/util/log:go_default_library", "//pkg/kubelet/kubeletconfig/util/panic:go_default_library", "//pkg/util/filesystem:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/pkg/kubelet/kubeletconfig/checkpoint/BUILD b/pkg/kubelet/kubeletconfig/checkpoint/BUILD index 94844991563..cb21adad996 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/BUILD +++ b/pkg/kubelet/kubeletconfig/checkpoint/BUILD @@ -31,7 +31,9 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint", deps = [ - "//pkg/api/legacyscheme:go_default_library", + "//pkg/kubelet/apis/kubeletconfig:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library", "//pkg/kubelet/kubeletconfig/status:go_default_library", "//pkg/kubelet/kubeletconfig/util/codec:go_default_library", "//pkg/kubelet/kubeletconfig/util/log:go_default_library", diff --git a/pkg/kubelet/kubeletconfig/checkpoint/download.go b/pkg/kubelet/kubeletconfig/checkpoint/download.go index bbc624adb76..0e45cba0505 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/download.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/download.go @@ -24,7 +24,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/pkg/api/legacyscheme" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" + "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" + kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/status" utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec" utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log" @@ -46,6 +48,8 @@ type Payload interface { type RemoteConfigSource interface { // UID returns a globally unique identifier of the source described by the remote config source object UID() string + // KubeletFilename returns the name of the Kubelet config file as it should appear in the keys of Payload.Files() + KubeletFilename() string // APIPath returns the API path to the remote resource, e.g. its SelfLink APIPath() string // Download downloads the remote config source object returns a Payload backed by the object, @@ -60,25 +64,18 @@ type RemoteConfigSource interface { object() interface{} } -// NewRemoteConfigSource constructs a RemoteConfigSource from a v1/NodeConfigSource object, or returns -// a sanitized failure reason and an error if the `source` is blatantly invalid. +// NewRemoteConfigSource constructs a RemoteConfigSource from a v1/NodeConfigSource object // You should only call this with a non-nil config source. +// Note that the API server validates Node.Spec.ConfigSource. func NewRemoteConfigSource(source *apiv1.NodeConfigSource) (RemoteConfigSource, string, error) { - // exactly one subfield of the config source must be non-nil, toady ConfigMapRef is the only reference - if source.ConfigMapRef == nil { + // NOTE: Even though the API server validates the config, we check whether all *known* fields are + // nil here, so that if a new API server allows a new config source type, old clients can send + // an error message rather than crashing due to a nil pointer dereference. + + // exactly one reference subfield of the config source must be non-nil, today ConfigMap is the only reference subfield + if source.ConfigMap == nil { return nil, status.FailSyncReasonAllNilSubfields, fmt.Errorf("%s, NodeConfigSource was: %#v", status.FailSyncReasonAllNilSubfields, source) } - - // validate the NodeConfigSource: - - // at this point we know we're using the ConfigMapRef subfield - ref := source.ConfigMapRef - - // name, namespace, and UID must all be non-empty for ConfigMapRef - if ref.Name == "" || ref.Namespace == "" || string(ref.UID) == "" { - return nil, status.FailSyncReasonPartialObjectReference, fmt.Errorf("%s, ObjectReference was: %#v", status.FailSyncReasonPartialObjectReference, ref) - } - return &remoteConfigMap{source}, "", nil } @@ -86,21 +83,25 @@ func NewRemoteConfigSource(source *apiv1.NodeConfigSource) (RemoteConfigSource, // e.g. the metadata stored by checkpoint/store/fsstore.go func DecodeRemoteConfigSource(data []byte) (RemoteConfigSource, error) { // decode the remote config source - obj, err := runtime.Decode(legacyscheme.Codecs.UniversalDecoder(), data) + _, codecs, err := scheme.NewSchemeAndCodecs() + if err != nil { + return nil, err + } + + obj, err := runtime.Decode(codecs.UniversalDecoder(), data) if err != nil { return nil, fmt.Errorf("failed to decode, error: %v", err) } - // for now we assume we are trying to load an apiv1.NodeConfigSource, + // for now we assume we are trying to load an kubeletconfigv1beta1.SerializedNodeConfigSource, // this may need to be extended if e.g. a new version of the api is born - - // convert it to the external NodeConfigSource type, so we're consistently working with the external type outside of the on-disk representation - cs := &apiv1.NodeConfigSource{} - err = legacyscheme.Scheme.Convert(obj, cs, nil) - if err != nil { - return nil, fmt.Errorf("failed to convert decoded object into a v1 NodeConfigSource, error: %v", err) + cs, ok := obj.(*kubeletconfiginternal.SerializedNodeConfigSource) + if !ok { + return nil, fmt.Errorf("failed to cast decoded remote config source to *k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig.SerializedNodeConfigSource") } - source, _, err := NewRemoteConfigSource(cs) + + // we use the v1.NodeConfigSource type on internal and external, so no need to convert to external here + source, _, err := NewRemoteConfigSource(&cs.Source) return source, err } @@ -121,32 +122,36 @@ type remoteConfigMap struct { var _ RemoteConfigSource = (*remoteConfigMap)(nil) func (r *remoteConfigMap) UID() string { - return string(r.source.ConfigMapRef.UID) + return string(r.source.ConfigMap.UID) +} + +func (r *remoteConfigMap) KubeletFilename() string { + return r.source.ConfigMap.KubeletConfigKey } const configMapAPIPathFmt = "/api/v1/namespaces/%s/configmaps/%s" func (r *remoteConfigMap) APIPath() string { - ref := r.source.ConfigMapRef + ref := r.source.ConfigMap return fmt.Sprintf(configMapAPIPathFmt, ref.Namespace, ref.Name) } func (r *remoteConfigMap) Download(client clientset.Interface) (Payload, string, error) { var reason string - uid := string(r.source.ConfigMapRef.UID) + uid := string(r.source.ConfigMap.UID) utillog.Infof("attempting to download ConfigMap with UID %q", uid) // get the ConfigMap via namespace/name, there doesn't seem to be a way to get it by UID - cm, err := client.CoreV1().ConfigMaps(r.source.ConfigMapRef.Namespace).Get(r.source.ConfigMapRef.Name, metav1.GetOptions{}) + cm, err := client.CoreV1().ConfigMaps(r.source.ConfigMap.Namespace).Get(r.source.ConfigMap.Name, metav1.GetOptions{}) if err != nil { reason = fmt.Sprintf(status.FailSyncReasonDownloadFmt, r.APIPath()) return nil, reason, fmt.Errorf("%s, error: %v", reason, err) } - // ensure that UID matches the UID on the reference, the ObjectReference must be unambiguous - if r.source.ConfigMapRef.UID != cm.UID { - reason = fmt.Sprintf(status.FailSyncReasonUIDMismatchFmt, r.source.ConfigMapRef.UID, r.APIPath(), cm.UID) + // ensure that UID matches the UID on the source + if r.source.ConfigMap.UID != cm.UID { + reason = fmt.Sprintf(status.FailSyncReasonUIDMismatchFmt, r.source.ConfigMap.UID, r.APIPath(), cm.UID) return nil, reason, fmt.Errorf(reason) } @@ -161,11 +166,12 @@ func (r *remoteConfigMap) Download(client clientset.Interface) (Payload, string, } func (r *remoteConfigMap) Encode() ([]byte, error) { - encoder, err := utilcodec.NewYAMLEncoder(apiv1.GroupName) + encoder, err := utilcodec.NewKubeletconfigYAMLEncoder(kubeletconfigv1beta1.SchemeGroupVersion) if err != nil { return nil, err } - data, err := runtime.Encode(encoder, r.source) + + data, err := runtime.Encode(encoder, &kubeletconfigv1beta1.SerializedNodeConfigSource{Source: *r.source}) if err != nil { return nil, err } diff --git a/pkg/kubelet/kubeletconfig/checkpoint/download_test.go b/pkg/kubelet/kubeletconfig/checkpoint/download_test.go index 0f9b3a362cd..caa5aa3f3c7 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/download_test.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/download_test.go @@ -36,33 +36,30 @@ func TestNewRemoteConfigSource(t *testing.T) { expect RemoteConfigSource err string }{ - // all NodeConfigSource subfields nil - {"all NodeConfigSource subfields nil", - &apiv1.NodeConfigSource{}, nil, "exactly one subfield must be non-nil"}, - {"ConfigMapRef: empty name, namespace, and UID", - &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{}}, nil, "invalid ObjectReference"}, - // ConfigMapRef: empty name and namespace - {"ConfigMapRef: empty name and namespace", - &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{UID: "uid"}}, nil, "invalid ObjectReference"}, - // ConfigMapRef: empty name and UID - {"ConfigMapRef: empty name and UID", - &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Namespace: "namespace"}}, nil, "invalid ObjectReference"}, - // ConfigMapRef: empty namespace and UID - {"ConfigMapRef: empty namespace and UID", - &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name"}}, nil, "invalid ObjectReference"}, - // ConfigMapRef: empty UID - {"ConfigMapRef: empty namespace and UID", - &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace"}}, nil, "invalid ObjectReference"}, - // ConfigMapRef: empty namespace - {"ConfigMapRef: empty namespace and UID", - &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", UID: "uid"}}, nil, "invalid ObjectReference"}, - // ConfigMapRef: empty name - {"ConfigMapRef: empty namespace and UID", - &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Namespace: "namespace", UID: "uid"}}, nil, "invalid ObjectReference"}, - // ConfigMapRef: valid reference - {"ConfigMapRef: valid reference", - &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}, - &remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}}, ""}, + { + desc: "all NodeConfigSource subfields nil", + source: &apiv1.NodeConfigSource{}, + expect: nil, + err: "exactly one subfield must be non-nil", + }, + { + desc: "ConfigMap: valid reference", + source: &apiv1.NodeConfigSource{ + ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: "uid", + KubeletConfigKey: "kubelet", + }}, + expect: &remoteConfigMap{&apiv1.NodeConfigSource{ + ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: "uid", + KubeletConfigKey: "kubelet", + }}}, + err: "", + }, } for _, c := range cases { @@ -82,7 +79,12 @@ func TestNewRemoteConfigSource(t *testing.T) { func TestRemoteConfigMapUID(t *testing.T) { const expect = "uid" - source, _, err := NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: expect}}) + source, _, err := NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: expect, + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("error constructing remote config source: %v", err) } @@ -93,14 +95,22 @@ func TestRemoteConfigMapUID(t *testing.T) { } func TestRemoteConfigMapAPIPath(t *testing.T) { - const namespace = "namespace" - const name = "name" - source, _, err := NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: name, Namespace: namespace, UID: "uid"}}) + const ( + name = "name" + namespace = "namespace" + ) + source, _, err := NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: name, + Namespace: namespace, + UID: "uid", + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("error constructing remote config source: %v", err) } expect := fmt.Sprintf(configMapAPIPathFmt, namespace, name) path := source.APIPath() + if expect != path { t.Errorf("expect %q, but got %q", expect, path) } @@ -133,18 +143,39 @@ func TestRemoteConfigMapDownload(t *testing.T) { expect Payload err string }{ - // object doesn't exist - {"object doesn't exist", - makeSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "bogus", Namespace: "namespace", UID: "bogus"}}), - nil, "not found"}, - // UID of downloaded object doesn't match UID of referent found via namespace/name - {"UID is incorrect for namespace/name", - makeSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "bogus"}}), - nil, "does not match"}, - // successful download - {"object exists and reference is correct", - makeSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}), - payload, ""}, + { + desc: "object doesn't exist", + source: makeSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "bogus", + Namespace: "namespace", + UID: "bogus", + KubeletConfigKey: "kubelet", + }}), + expect: nil, + err: "not found", + }, + { + desc: "UID is incorrect for namespace/name", + source: makeSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: "bogus", + KubeletConfigKey: "kubelet", + }}), + expect: nil, + err: "does not match", + }, + { + desc: "object exists and reference is correct", + source: makeSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: "uid", + KubeletConfigKey: "kubelet", + }}), + expect: payload, + err: "", + }, } for _, c := range cases { @@ -173,10 +204,12 @@ func TestEqualRemoteConfigSources(t *testing.T) { {"a nil", nil, &remoteConfigMap{}, false}, {"b nil", &remoteConfigMap{}, nil, false}, {"neither nil, equal", &remoteConfigMap{}, &remoteConfigMap{}, true}, - {"neither nil, not equal", - &remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "a"}}}, - &remoteConfigMap{}, - false}, + { + desc: "neither nil, not equal", + a: &remoteConfigMap{&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{Name: "a"}}}, + b: &remoteConfigMap{&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{KubeletConfigKey: "kubelet"}}}, + expect: false, + }, } for _, c := range cases { diff --git a/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore.go b/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore.go index 3ce1744fabc..309e9d908d5 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore.go @@ -35,7 +35,6 @@ const ( lastKnownGoodFile = "last-known-good" checkpointsDir = "checkpoints" - kubeletKey = "kubelet" // TODO(mtaufen): eventually the API will have a way to parameterize the kubelet file name, and then we can remove this ) // fsStore is for tracking checkpoints in the local filesystem, implements Store @@ -101,7 +100,7 @@ func (s *fsStore) Load(source checkpoint.RemoteConfigSource) (*kubeletconfig.Kub } // load the kubelet config file utillog.Infof("loading kubelet configuration checkpoint for source %s", sourceFmt) - loader, err := configfiles.NewFsLoader(s.fs, filepath.Join(s.checkpointPath(source.UID()), kubeletKey)) + loader, err := configfiles.NewFsLoader(s.fs, filepath.Join(s.checkpointPath(source.UID()), source.KubeletFilename())) if err != nil { return nil, err } diff --git a/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore_test.go b/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore_test.go index ee1c8b57e63..0aa656b76f0 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore_test.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore_test.go @@ -125,7 +125,12 @@ func TestFsStoreExists(t *testing.T) { for _, c := range cases { t.Run(c.desc, func(t *testing.T) { source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ - ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: c.uid}}) + ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: c.uid, + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("error constructing remote config source: %v", err) } @@ -214,7 +219,10 @@ func TestFsStoreLoad(t *testing.T) { t.Fatalf("error encoding KubeletConfiguration: %v", err) } // construct a payload that contains the kubeletconfig - const uid = "uid" + const ( + uid = "uid" + kubeletKey = "kubelet" + ) p, err := checkpoint.NewConfigMapPayload(&apiv1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{UID: types.UID(uid)}, Data: map[string]string{ @@ -241,7 +249,12 @@ func TestFsStoreLoad(t *testing.T) { for _, c := range cases { t.Run(c.desc, func(t *testing.T) { source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ - ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: c.uid}}) + ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: c.uid, + KubeletConfigKey: kubeletKey, + }}) if err != nil { t.Fatalf("error constructing remote config source: %v", err) } @@ -291,7 +304,12 @@ func TestFsStoreCurrent(t *testing.T) { } source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ - ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}) + ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: "uid", + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -329,7 +347,12 @@ func TestFsStoreLastKnownGood(t *testing.T) { } source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ - ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}) + ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: "uid", + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -367,15 +390,21 @@ func TestFsStoreSetCurrent(t *testing.T) { } const uid = "uid" - expect := fmt.Sprintf(`apiVersion: v1 -configMapRef: - name: name - namespace: namespace - uid: %s -kind: NodeConfigSource + expect := fmt.Sprintf(`apiVersion: kubelet.config.k8s.io/v1beta1 +kind: SerializedNodeConfigSource +source: + configMap: + kubeletConfigKey: kubelet + name: name + namespace: namespace + uid: %s `, uid) - source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - Name: "name", Namespace: "namespace", UID: types.UID(uid)}}) + source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: types.UID(uid), + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -399,15 +428,21 @@ func TestFsStoreSetLastKnownGood(t *testing.T) { } const uid = "uid" - expect := fmt.Sprintf(`apiVersion: v1 -configMapRef: - name: name - namespace: namespace - uid: %s -kind: NodeConfigSource + expect := fmt.Sprintf(`apiVersion: kubelet.config.k8s.io/v1beta1 +kind: SerializedNodeConfigSource +source: + configMap: + kubeletConfigKey: kubelet + name: name + namespace: namespace + uid: %s `, uid) - source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - Name: "name", Namespace: "namespace", UID: types.UID(uid)}}) + source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: types.UID(uid), + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -430,11 +465,21 @@ func TestFsStoreReset(t *testing.T) { t.Fatalf("error constructing store: %v", err) } - source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}) + source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: "uid", + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("unexpected error: %v", err) } - otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "other-name", Namespace: "namespace", UID: "other-uid"}}) + otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "other-name", + Namespace: "namespace", + UID: "other-uid", + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -498,7 +543,12 @@ func TestFsStoreReadRemoteConfigSource(t *testing.T) { } source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ - ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}) + ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: "uid", + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -534,7 +584,12 @@ func TestFsStoreWriteRemoteConfigSource(t *testing.T) { t.Fatalf("error constructing store: %v", err) } - source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}) + source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: "uid", + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/kubelet/kubeletconfig/checkpoint/store/store_test.go b/pkg/kubelet/kubeletconfig/checkpoint/store/store_test.go index 4d4acb29349..0ef91b236b1 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/store/store_test.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/store/store_test.go @@ -26,11 +26,21 @@ import ( ) func TestReset(t *testing.T) { - source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}) + source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "name", + Namespace: "namespace", + UID: "uid", + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("unexpected error: %v", err) } - otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "other-name", Namespace: "namespace", UID: "other-uid"}}) + otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Name: "other-name", + Namespace: "namespace", + UID: "other-uid", + KubeletConfigKey: "kubelet", + }}) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/kubelet/kubeletconfig/status/status.go b/pkg/kubelet/kubeletconfig/status/status.go index 014b7981d60..c53cd205ff7 100644 --- a/pkg/kubelet/kubeletconfig/status/status.go +++ b/pkg/kubelet/kubeletconfig/status/status.go @@ -65,11 +65,11 @@ const ( FailSyncReasonFmt = "failed to sync, reason: %s" // FailSyncReasonAllNilSubfields is used when no subfields are set FailSyncReasonAllNilSubfields = "invalid NodeConfigSource, exactly one subfield must be non-nil, but all were nil" - // FailSyncReasonPartialObjectReference is used when some required subfields remain unset - FailSyncReasonPartialObjectReference = "invalid ObjectReference, all of UID, Name, and Namespace must be specified" + // FailSyncReasonPartialConfigMapSource is used when some required subfields remain unset + FailSyncReasonPartialConfigMapSource = "invalid ConfigSource.ConfigMap, all of UID, Name, Namespace, and KubeletConfigKey must be specified" // FailSyncReasonUIDMismatchFmt is used when there is a UID mismatch between the referenced and downloaded ConfigMaps, // this can happen because objects must be downloaded by namespace/name, rather than by UID - FailSyncReasonUIDMismatchFmt = "invalid ConfigSource.ConfigMapRef.UID: %s does not match %s.UID: %s" + FailSyncReasonUIDMismatchFmt = "invalid ConfigSource.ConfigMap.UID: %s does not match %s.UID: %s" // FailSyncReasonDownloadFmt is used when the download fails, e.g. due to network issues FailSyncReasonDownloadFmt = "failed to download: %s" // FailSyncReasonInformer is used when the informer fails to report the Node object @@ -256,7 +256,7 @@ func (c *configOkCondition) Sync(client clientset.Interface, nodeName string) { err = fmt.Errorf("unsupported media type %q", mediaType) return } - versions := legacyscheme.Registry.RegisteredVersionsForGroup(api.GroupName) + versions := legacyscheme.Scheme.PrioritizedVersionsForGroup(api.GroupName) if len(versions) == 0 { err = fmt.Errorf("no enabled versions for group %q", api.GroupName) return diff --git a/pkg/kubelet/kubeletconfig/util/codec/codec.go b/pkg/kubelet/kubeletconfig/util/codec/codec.go index c499ac8c0c1..b8356fabd0e 100644 --- a/pkg/kubelet/kubeletconfig/util/codec/codec.go +++ b/pkg/kubelet/kubeletconfig/util/codec/codec.go @@ -32,7 +32,7 @@ import ( // EncodeKubeletConfig encodes an internal KubeletConfiguration to an external YAML representation func EncodeKubeletConfig(internal *kubeletconfig.KubeletConfiguration, targetVersion schema.GroupVersion) ([]byte, error) { - encoder, err := newKubeletConfigYAMLEncoder(targetVersion) + encoder, err := NewKubeletconfigYAMLEncoder(targetVersion) if err != nil { return nil, err } @@ -44,8 +44,8 @@ func EncodeKubeletConfig(internal *kubeletconfig.KubeletConfiguration, targetVer return data, nil } -// newKubeletConfigYAMLEncoder returns an encoder that can write a KubeletConfig to YAML -func newKubeletConfigYAMLEncoder(targetVersion schema.GroupVersion) (runtime.Encoder, error) { +// NewKubeletconfigYAMLEncoder returns an encoder that can write objects in the kubeletconfig API group to YAML +func NewKubeletconfigYAMLEncoder(targetVersion schema.GroupVersion) (runtime.Encoder, error) { _, codecs, err := scheme.NewSchemeAndCodecs() if err != nil { return nil, err @@ -67,7 +67,7 @@ func NewYAMLEncoder(groupName string) (runtime.Encoder, error) { return nil, fmt.Errorf("unsupported media type %q", mediaType) } - versions := legacyscheme.Registry.RegisteredVersionsForGroup(groupName) + versions := legacyscheme.Scheme.PrioritizedVersionsForGroup(groupName) if len(versions) == 0 { return nil, fmt.Errorf("no enabled versions for group %q", groupName) } diff --git a/pkg/kubelet/kubeletconfig/util/equal/equal.go b/pkg/kubelet/kubeletconfig/util/equal/equal.go index 5a59edb94b1..b943e019f92 100644 --- a/pkg/kubelet/kubeletconfig/util/equal/equal.go +++ b/pkg/kubelet/kubeletconfig/util/equal/equal.go @@ -18,33 +18,6 @@ package equal import apiv1 "k8s.io/api/core/v1" -// ConfigSourceEq returns true if the two config sources are semantically equivalent in the context of dynamic config -func ConfigSourceEq(a, b *apiv1.NodeConfigSource) bool { - if a == b { - return true - } else if a == nil || b == nil { - // not equal, and one is nil - return false - } - // check equality of config source subifelds - if a.ConfigMapRef != b.ConfigMapRef { - return ObjectRefEq(a.ConfigMapRef, b.ConfigMapRef) - } - // all internal subfields of the config source are equal - return true -} - -// ObjectRefEq returns true if the two object references are semantically equivalent in the context of dynamic config -func ObjectRefEq(a, b *apiv1.ObjectReference) bool { - if a == b { - return true - } else if a == nil || b == nil { - // not equal, and one is nil - return false - } - return a.UID == b.UID && a.Namespace == b.Namespace && a.Name == b.Name -} - // KubeletConfigOkEq returns true if the two conditions are semantically equivalent in the context of dynamic config func KubeletConfigOkEq(a, b *apiv1.NodeCondition) bool { return a.Message == b.Message && a.Reason == b.Reason && a.Status == b.Status diff --git a/pkg/kubelet/kubeletconfig/util/files/files.go b/pkg/kubelet/kubeletconfig/util/files/files.go index 8fd19ce860b..fc42151a729 100644 --- a/pkg/kubelet/kubeletconfig/util/files/files.go +++ b/pkg/kubelet/kubeletconfig/util/files/files.go @@ -67,6 +67,7 @@ func EnsureFile(fs utilfs.Filesystem, path string) error { } // WriteTmpFile creates a temporary file at `path`, writes `data` into it, and fsyncs the file +// Expects the parent directory to exist. func WriteTmpFile(fs utilfs.Filesystem, path string, data []byte) (tmpPath string, retErr error) { dir := filepath.Dir(path) prefix := tmptag + filepath.Base(path) @@ -103,7 +104,8 @@ func WriteTmpFile(fs utilfs.Filesystem, path string, data []byte) (tmpPath strin } // ReplaceFile replaces the contents of the file at `path` with `data` by writing to a tmp file in the same -// dir as `path` and renaming the tmp file over `path`. The file does not have to exist to use ReplaceFile. +// dir as `path` and renaming the tmp file over `path`. The file does not have to exist to use ReplaceFile, +// but the parent directory must exist. // Note ReplaceFile calls fsync. func ReplaceFile(fs utilfs.Filesystem, path string, data []byte) error { // write data to a temporary file @@ -121,7 +123,7 @@ func DirExists(fs utilfs.Filesystem, path string) (bool, error) { if info.IsDir() { return true, nil } - return false, fmt.Errorf("expected dir at %q, but mode is is %q", path, info.Mode().String()) + return false, fmt.Errorf("expected dir at %q, but mode is %q", path, info.Mode().String()) } else if os.IsNotExist(err) { return false, nil } else { diff --git a/pkg/kubelet/kubeletconfig/util/files/files_test.go b/pkg/kubelet/kubeletconfig/util/files/files_test.go index ba81b82ec3e..4917679abbc 100644 --- a/pkg/kubelet/kubeletconfig/util/files/files_test.go +++ b/pkg/kubelet/kubeletconfig/util/files/files_test.go @@ -209,6 +209,189 @@ func TestHelpers(t *testing.T) { } } +func TestFileExists(t *testing.T) { + fn := func(fs utilfs.Filesystem, dir string, c *test) []error { + ok, err := FileExists(fs, filepath.Join(dir, "foo")) + if err != nil { + return []error{err} + } + if !ok { + return []error{fmt.Errorf("does not exist (test)")} + } + return nil + } + cases := []test{ + { + fn: fn, + desc: "file exists", + writes: []file{{name: "foo"}}, + }, + { + fn: fn, + desc: "file does not exist", + err: "does not exist (test)", + }, + { + fn: fn, + desc: "object has non-file mode", + writes: []file{{name: "foo", mode: os.ModeDir}}, + err: "expected regular file", + }, + } + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + c.run(t, utilfs.DefaultFs{}) + }) + } +} + +func TestEnsureFile(t *testing.T) { + fn := func(fs utilfs.Filesystem, dir string, c *test) []error { + var errs []error + for _, f := range c.expects { + if err := EnsureFile(fs, filepath.Join(dir, f.name)); err != nil { + errs = append(errs, err) + } + } + return errs + } + cases := []test{ + { + fn: fn, + desc: "file exists", + writes: []file{{name: "foo"}}, + expects: []file{{name: "foo"}}, + }, + { + fn: fn, + desc: "file does not exist", + expects: []file{{name: "bar"}}, + }, + { + fn: fn, + desc: "neither parent nor file exists", + expects: []file{{name: "baz/quux"}}, + }, + } + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + c.run(t, utilfs.DefaultFs{}) + }) + } +} + +// Note: This transitively tests WriteTmpFile +func TestReplaceFile(t *testing.T) { + fn := func(fs utilfs.Filesystem, dir string, c *test) []error { + var errs []error + for _, f := range c.expects { + if err := ReplaceFile(fs, filepath.Join(dir, f.name), []byte(f.data)); err != nil { + errs = append(errs, err) + } + } + return errs + } + cases := []test{ + { + fn: fn, + desc: "file exists", + writes: []file{{name: "foo"}}, + expects: []file{{name: "foo", data: "bar"}}, + }, + { + fn: fn, + desc: "file does not exist", + expects: []file{{name: "foo", data: "bar"}}, + }, + { + fn: func(fs utilfs.Filesystem, dir string, c *test) []error { + if err := ReplaceFile(fs, filepath.Join(dir, "foo/bar"), []byte("")); err != nil { + return []error{err} + } + return nil + }, + desc: "neither parent nor file exists", + err: "no such file or directory", + }, + } + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + c.run(t, utilfs.DefaultFs{}) + }) + } +} + +func TestDirExists(t *testing.T) { + fn := func(fs utilfs.Filesystem, dir string, c *test) []error { + ok, err := DirExists(fs, filepath.Join(dir, "foo")) + if err != nil { + return []error{err} + } + if !ok { + return []error{fmt.Errorf("does not exist (test)")} + } + return nil + } + cases := []test{ + { + fn: fn, + desc: "dir exists", + writes: []file{{name: "foo", mode: os.ModeDir}}, + }, + { + fn: fn, + desc: "dir does not exist", + err: "does not exist (test)", + }, + { + fn: fn, + desc: "object has non-dir mode", + writes: []file{{name: "foo"}}, + err: "expected dir", + }, + } + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + c.run(t, utilfs.DefaultFs{}) + }) + } +} + +func TestEnsureDir(t *testing.T) { + fn := func(fs utilfs.Filesystem, dir string, c *test) []error { + var errs []error + for _, f := range c.expects { + if err := EnsureDir(fs, filepath.Join(dir, f.name)); err != nil { + errs = append(errs, err) + } + } + return errs + } + cases := []test{ + { + fn: fn, + desc: "dir exists", + writes: []file{{name: "foo", mode: os.ModeDir}}, + expects: []file{{name: "foo", mode: os.ModeDir}}, + }, + { + fn: fn, + desc: "dir does not exist", + expects: []file{{name: "bar", mode: os.ModeDir}}, + }, + { + fn: fn, + desc: "neither parent nor dir exists", + expects: []file{{name: "baz/quux", mode: os.ModeDir}}, + }, + } + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + c.run(t, utilfs.DefaultFs{}) + }) + } +} + func TestWriteTempDir(t *testing.T) { // writing a tmp dir is covered by TestReplaceDir, but we additionally test filename validation here c := test{ diff --git a/pkg/kubelet/kubeletconfig/watch.go b/pkg/kubelet/kubeletconfig/watch.go index 8c02e3c9a82..09b22233e07 100644 --- a/pkg/kubelet/kubeletconfig/watch.go +++ b/pkg/kubelet/kubeletconfig/watch.go @@ -21,13 +21,13 @@ import ( "time" apiv1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" kuberuntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - utilequal "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/equal" utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log" ) @@ -94,7 +94,7 @@ func (cc *Controller) onUpdateNodeEvent(oldObj interface{}, newObj interface{}) utillog.Errorf("failed to cast old object to Node, couldn't handle event") return } - if !utilequal.ConfigSourceEq(oldNode.Spec.ConfigSource, newNode.Spec.ConfigSource) { + if !apiequality.Semantic.DeepEqual(oldNode.Spec.ConfigSource, newNode.Spec.ConfigSource) { cc.pokeConfigSourceWorker() } } diff --git a/pkg/kubelet/secret/BUILD b/pkg/kubelet/secret/BUILD index 1fdb5850c27..84632b98b47 100644 --- a/pkg/kubelet/secret/BUILD +++ b/pkg/kubelet/secret/BUILD @@ -8,7 +8,7 @@ load( go_test( name = "go_default_test", - srcs = ["secret_manager_test.go"], + srcs = ["caching_secret_manager_test.go"], embed = [":go_default_library"], deps = [ "//vendor/github.com/stretchr/testify/assert:go_default_library", @@ -25,6 +25,7 @@ go_test( go_library( name = "go_default_library", srcs = [ + "caching_secret_manager.go", "fake_manager.go", "secret_manager.go", ], diff --git a/pkg/kubelet/secret/caching_secret_manager.go b/pkg/kubelet/secret/caching_secret_manager.go new file mode 100644 index 00000000000..45756ad2ff0 --- /dev/null +++ b/pkg/kubelet/secret/caching_secret_manager.go @@ -0,0 +1,206 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secret + +import ( + "fmt" + "strconv" + "sync" + "time" + + "k8s.io/api/core/v1" + storageetcd "k8s.io/apiserver/pkg/storage/etcd" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/kubelet/util" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/clock" +) + +const ( + defaultTTL = time.Minute +) + +type GetObjectTTLFunc func() (time.Duration, bool) + +// secretStoreItems is a single item stored in secretStore. +type secretStoreItem struct { + refCount int + secret *secretData +} + +type secretData struct { + sync.Mutex + + secret *v1.Secret + err error + lastUpdateTime time.Time +} + +// secretStore is a local cache of secrets. +type secretStore struct { + kubeClient clientset.Interface + clock clock.Clock + + lock sync.Mutex + items map[objectKey]*secretStoreItem + + defaultTTL time.Duration + getTTL GetObjectTTLFunc +} + +func newSecretStore(kubeClient clientset.Interface, clock clock.Clock, getTTL GetObjectTTLFunc, ttl time.Duration) *secretStore { + return &secretStore{ + kubeClient: kubeClient, + clock: clock, + items: make(map[objectKey]*secretStoreItem), + defaultTTL: ttl, + getTTL: getTTL, + } +} + +func isSecretOlder(newSecret, oldSecret *v1.Secret) bool { + if newSecret == nil || oldSecret == nil { + return false + } + newVersion, _ := storageetcd.Versioner.ObjectResourceVersion(newSecret) + oldVersion, _ := storageetcd.Versioner.ObjectResourceVersion(oldSecret) + return newVersion < oldVersion +} + +func (s *secretStore) AddReference(namespace, name string) { + key := objectKey{namespace: namespace, name: name} + + // AddReference is called from RegisterPod, thus it needs to be efficient. + // Thus Add() is only increasing refCount and generation of a given secret. + // Then Get() is responsible for fetching if needed. + s.lock.Lock() + defer s.lock.Unlock() + item, exists := s.items[key] + if !exists { + item = &secretStoreItem{ + refCount: 0, + secret: &secretData{}, + } + s.items[key] = item + } + + item.refCount++ + // This will trigger fetch on the next Get() operation. + item.secret = nil +} + +func (s *secretStore) DeleteReference(namespace, name string) { + key := objectKey{namespace: namespace, name: name} + + s.lock.Lock() + defer s.lock.Unlock() + if item, ok := s.items[key]; ok { + item.refCount-- + if item.refCount == 0 { + delete(s.items, key) + } + } +} + +func GetObjectTTLFromNodeFunc(getNode func() (*v1.Node, error)) GetObjectTTLFunc { + return func() (time.Duration, bool) { + node, err := getNode() + if err != nil { + return time.Duration(0), false + } + if node != nil && node.Annotations != nil { + if value, ok := node.Annotations[v1.ObjectTTLAnnotationKey]; ok { + if intValue, err := strconv.Atoi(value); err == nil { + return time.Duration(intValue) * time.Second, true + } + } + } + return time.Duration(0), false + } +} + +func (s *secretStore) isSecretFresh(data *secretData) bool { + secretTTL := s.defaultTTL + if ttl, ok := s.getTTL(); ok { + secretTTL = ttl + } + return s.clock.Now().Before(data.lastUpdateTime.Add(secretTTL)) +} + +func (s *secretStore) Get(namespace, name string) (*v1.Secret, error) { + key := objectKey{namespace: namespace, name: name} + + data := func() *secretData { + s.lock.Lock() + defer s.lock.Unlock() + item, exists := s.items[key] + if !exists { + return nil + } + if item.secret == nil { + item.secret = &secretData{} + } + return item.secret + }() + if data == nil { + return nil, fmt.Errorf("secret %q/%q not registered", namespace, name) + } + + // After updating data in secretStore, lock the data, fetch secret if + // needed and return data. + data.Lock() + defer data.Unlock() + if data.err != nil || !s.isSecretFresh(data) { + opts := metav1.GetOptions{} + if data.secret != nil && data.err == nil { + // This is just a periodic refresh of a secret we successfully fetched previously. + // In this case, server data from apiserver cache to reduce the load on both + // etcd and apiserver (the cache is eventually consistent). + util.FromApiserverCache(&opts) + } + secret, err := s.kubeClient.CoreV1().Secrets(namespace).Get(name, opts) + if err != nil && !apierrors.IsNotFound(err) && data.secret == nil && data.err == nil { + // Couldn't fetch the latest secret, but there is no cached data to return. + // Return the fetch result instead. + return secret, err + } + if (err == nil && !isSecretOlder(secret, data.secret)) || apierrors.IsNotFound(err) { + // If the fetch succeeded with a newer version of the secret, or if the + // secret could not be found in the apiserver, update the cached data to + // reflect the current status. + data.secret = secret + data.err = err + data.lastUpdateTime = s.clock.Now() + } + } + return data.secret, data.err +} + +// NewCachingSecretManager creates a manager that keeps a cache of all secrets +// necessary for registered pods. +// It implements the following logic: +// - whenever a pod is created or updated, the cached versions of all its secrets +// are invalidated +// - every GetSecret() call tries to fetch the value from local cache; if it is +// not there, invalidated or too old, we fetch it from apiserver and refresh the +// value in cache; otherwise it is just fetched from cache +func NewCachingSecretManager(kubeClient clientset.Interface, getTTL GetObjectTTLFunc) Manager { + secretStore := newSecretStore(kubeClient, clock.RealClock{}, getTTL, defaultTTL) + return newCacheBasedSecretManager(secretStore) +} diff --git a/pkg/kubelet/secret/secret_manager_test.go b/pkg/kubelet/secret/caching_secret_manager_test.go similarity index 95% rename from pkg/kubelet/secret/secret_manager_test.go rename to pkg/kubelet/secret/caching_secret_manager_test.go index 36380e514cc..6d53043e29c 100644 --- a/pkg/kubelet/secret/secret_manager_test.go +++ b/pkg/kubelet/secret/caching_secret_manager_test.go @@ -53,13 +53,13 @@ func noObjectTTL() (time.Duration, bool) { func TestSecretStore(t *testing.T) { fakeClient := &fake.Clientset{} store := newSecretStore(fakeClient, clock.RealClock{}, noObjectTTL, 0) - store.Add("ns1", "name1") - store.Add("ns2", "name2") - store.Add("ns1", "name1") - store.Add("ns1", "name1") - store.Delete("ns1", "name1") - store.Delete("ns2", "name2") - store.Add("ns3", "name3") + store.AddReference("ns1", "name1") + store.AddReference("ns2", "name2") + store.AddReference("ns1", "name1") + store.AddReference("ns1", "name1") + store.DeleteReference("ns1", "name1") + store.DeleteReference("ns2", "name2") + store.AddReference("ns3", "name3") // Adds don't issue Get requests. actions := fakeClient.Actions() @@ -87,7 +87,7 @@ func TestSecretStore(t *testing.T) { func TestSecretStoreDeletingSecret(t *testing.T) { fakeClient := &fake.Clientset{} store := newSecretStore(fakeClient, clock.RealClock{}, noObjectTTL, 0) - store.Add("ns", "name") + store.AddReference("ns", "name") result := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "name", ResourceVersion: "10"}} fakeClient.AddReactor("get", "secrets", func(action core.Action) (bool, runtime.Object, error) { @@ -119,7 +119,7 @@ func TestSecretStoreGetAlwaysRefresh(t *testing.T) { store := newSecretStore(fakeClient, fakeClock, noObjectTTL, 0) for i := 0; i < 10; i++ { - store.Add(fmt.Sprintf("ns-%d", i), fmt.Sprintf("name-%d", i)) + store.AddReference(fmt.Sprintf("ns-%d", i), fmt.Sprintf("name-%d", i)) } fakeClient.ClearActions() @@ -146,7 +146,7 @@ func TestSecretStoreGetNeverRefresh(t *testing.T) { store := newSecretStore(fakeClient, fakeClock, noObjectTTL, time.Minute) for i := 0; i < 10; i++ { - store.Add(fmt.Sprintf("ns-%d", i), fmt.Sprintf("name-%d", i)) + store.AddReference(fmt.Sprintf("ns-%d", i), fmt.Sprintf("name-%d", i)) } fakeClient.ClearActions() @@ -175,7 +175,7 @@ func TestCustomTTL(t *testing.T) { fakeClock := clock.NewFakeClock(time.Time{}) store := newSecretStore(fakeClient, fakeClock, customTTL, time.Minute) - store.Add("ns", "name") + store.AddReference("ns", "name") store.Get("ns", "name") fakeClient.ClearActions() @@ -345,10 +345,7 @@ func TestCacheInvalidation(t *testing.T) { fakeClient := &fake.Clientset{} fakeClock := clock.NewFakeClock(time.Now()) store := newSecretStore(fakeClient, fakeClock, noObjectTTL, time.Minute) - manager := &cachingSecretManager{ - secretStore: store, - registeredPods: make(map[objectKey]*v1.Pod), - } + manager := newCacheBasedSecretManager(store) // Create a pod with some secrets. s1 := secretsToAttach{ @@ -403,10 +400,7 @@ func TestCacheRefcounts(t *testing.T) { fakeClient := &fake.Clientset{} fakeClock := clock.NewFakeClock(time.Now()) store := newSecretStore(fakeClient, fakeClock, noObjectTTL, time.Minute) - manager := &cachingSecretManager{ - secretStore: store, - registeredPods: make(map[objectKey]*v1.Pod), - } + manager := newCacheBasedSecretManager(store) s1 := secretsToAttach{ imagePullSecretNames: []string{"s1"}, @@ -490,10 +484,7 @@ func TestCacheRefcounts(t *testing.T) { func TestCachingSecretManager(t *testing.T) { fakeClient := &fake.Clientset{} secretStore := newSecretStore(fakeClient, clock.RealClock{}, noObjectTTL, 0) - manager := &cachingSecretManager{ - secretStore: secretStore, - registeredPods: make(map[objectKey]*v1.Pod), - } + manager := newCacheBasedSecretManager(secretStore) // Create a pod with some secrets. s1 := secretsToAttach{ diff --git a/pkg/kubelet/secret/secret_manager.go b/pkg/kubelet/secret/secret_manager.go index 3de230a9032..01def3296ea 100644 --- a/pkg/kubelet/secret/secret_manager.go +++ b/pkg/kubelet/secret/secret_manager.go @@ -17,27 +17,16 @@ limitations under the License. package secret import ( - "fmt" - "strconv" "sync" - "time" "k8s.io/api/core/v1" - storageetcd "k8s.io/apiserver/pkg/storage/etcd" clientset "k8s.io/client-go/kubernetes" podutil "k8s.io/kubernetes/pkg/api/v1/pod" - "k8s.io/kubernetes/pkg/kubelet/util" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/sets" ) -const ( - defaultTTL = time.Minute -) - type Manager interface { // Get secret by secret namespace and name. GetSecret(namespace, name string) (*v1.Secret, error) @@ -53,6 +42,11 @@ type Manager interface { UnregisterPod(pod *v1.Pod) } +type objectKey struct { + namespace string + name string +} + // simpleSecretManager implements SecretManager interfaces with // simple operations to apiserver. type simpleSecretManager struct { @@ -73,190 +67,41 @@ func (s *simpleSecretManager) RegisterPod(pod *v1.Pod) { func (s *simpleSecretManager) UnregisterPod(pod *v1.Pod) { } -type GetObjectTTLFunc func() (time.Duration, bool) - -type objectKey struct { - namespace string - name string +// store is the interface for a secrets cache that +// can be used by cacheBasedSecretManager. +type store interface { + // AddReference adds a reference to the secret to the store. + // Note that multiple additions to the store has to be allowed + // in the implementations and effectively treated as refcounted. + AddReference(namespace, name string) + // DeleteReference deletes reference to the secret from the store. + // Note that secret should be deleted only when there was a + // corresponding Delete call for each of Add calls (effectively + // when refcount was reduced to zero). + DeleteReference(namespace, name string) + // Get a secret from a store. + Get(namespace, name string) (*v1.Secret, error) } -// secretStoreItems is a single item stored in secretStore. -type secretStoreItem struct { - refCount int - secret *secretData -} - -type secretData struct { - sync.Mutex - - secret *v1.Secret - err error - lastUpdateTime time.Time -} - -// secretStore is a local cache of secrets. -type secretStore struct { - kubeClient clientset.Interface - clock clock.Clock - - lock sync.Mutex - items map[objectKey]*secretStoreItem - - defaultTTL time.Duration - getTTL GetObjectTTLFunc -} - -func newSecretStore(kubeClient clientset.Interface, clock clock.Clock, getTTL GetObjectTTLFunc, ttl time.Duration) *secretStore { - return &secretStore{ - kubeClient: kubeClient, - clock: clock, - items: make(map[objectKey]*secretStoreItem), - defaultTTL: ttl, - getTTL: getTTL, - } -} - -func isSecretOlder(newSecret, oldSecret *v1.Secret) bool { - if newSecret == nil || oldSecret == nil { - return false - } - newVersion, _ := storageetcd.Versioner.ObjectResourceVersion(newSecret) - oldVersion, _ := storageetcd.Versioner.ObjectResourceVersion(oldSecret) - return newVersion < oldVersion -} - -func (s *secretStore) Add(namespace, name string) { - key := objectKey{namespace: namespace, name: name} - - // Add is called from RegisterPod, thus it needs to be efficient. - // Thus Add() is only increasing refCount and generation of a given secret. - // Then Get() is responsible for fetching if needed. - s.lock.Lock() - defer s.lock.Unlock() - item, exists := s.items[key] - if !exists { - item = &secretStoreItem{ - refCount: 0, - secret: &secretData{}, - } - s.items[key] = item - } - - item.refCount++ - // This will trigger fetch on the next Get() operation. - item.secret = nil -} - -func (s *secretStore) Delete(namespace, name string) { - key := objectKey{namespace: namespace, name: name} - - s.lock.Lock() - defer s.lock.Unlock() - if item, ok := s.items[key]; ok { - item.refCount-- - if item.refCount == 0 { - delete(s.items, key) - } - } -} - -func GetObjectTTLFromNodeFunc(getNode func() (*v1.Node, error)) GetObjectTTLFunc { - return func() (time.Duration, bool) { - node, err := getNode() - if err != nil { - return time.Duration(0), false - } - if node != nil && node.Annotations != nil { - if value, ok := node.Annotations[v1.ObjectTTLAnnotationKey]; ok { - if intValue, err := strconv.Atoi(value); err == nil { - return time.Duration(intValue) * time.Second, true - } - } - } - return time.Duration(0), false - } -} - -func (s *secretStore) isSecretFresh(data *secretData) bool { - secretTTL := s.defaultTTL - if ttl, ok := s.getTTL(); ok { - secretTTL = ttl - } - return s.clock.Now().Before(data.lastUpdateTime.Add(secretTTL)) -} - -func (s *secretStore) Get(namespace, name string) (*v1.Secret, error) { - key := objectKey{namespace: namespace, name: name} - - data := func() *secretData { - s.lock.Lock() - defer s.lock.Unlock() - item, exists := s.items[key] - if !exists { - return nil - } - if item.secret == nil { - item.secret = &secretData{} - } - return item.secret - }() - if data == nil { - return nil, fmt.Errorf("secret %q/%q not registered", namespace, name) - } - - // After updating data in secretStore, lock the data, fetch secret if - // needed and return data. - data.Lock() - defer data.Unlock() - if data.err != nil || !s.isSecretFresh(data) { - opts := metav1.GetOptions{} - if data.secret != nil && data.err == nil { - // This is just a periodic refresh of a secret we successfully fetched previously. - // In this case, server data from apiserver cache to reduce the load on both - // etcd and apiserver (the cache is eventually consistent). - util.FromApiserverCache(&opts) - } - secret, err := s.kubeClient.CoreV1().Secrets(namespace).Get(name, opts) - if err != nil && !apierrors.IsNotFound(err) && data.secret == nil && data.err == nil { - // Couldn't fetch the latest secret, but there is no cached data to return. - // Return the fetch result instead. - return secret, err - } - if (err == nil && !isSecretOlder(secret, data.secret)) || apierrors.IsNotFound(err) { - // If the fetch succeeded with a newer version of the secret, or if the - // secret could not be found in the apiserver, update the cached data to - // reflect the current status. - data.secret = secret - data.err = err - data.lastUpdateTime = s.clock.Now() - } - } - return data.secret, data.err -} - -// cachingSecretManager keeps a cache of all secrets necessary for registered pods. -// It implements the following logic: -// - whenever a pod is created or updated, the cached versions of all its secrets -// are invalidated -// - every GetSecret() call tries to fetch the value from local cache; if it is -// not there, invalidated or too old, we fetch it from apiserver and refresh the -// value in cache; otherwise it is just fetched from cache -type cachingSecretManager struct { - secretStore *secretStore +// cachingBasedSecretManager keeps a store with secrets necessary +// for registered pods. Different implementations of the store +// may result in different semantics for freshness of secrets +// (e.g. ttl-based implementation vs watch-based implementation). +type cacheBasedSecretManager struct { + secretStore store lock sync.Mutex registeredPods map[objectKey]*v1.Pod } -func NewCachingSecretManager(kubeClient clientset.Interface, getTTL GetObjectTTLFunc) Manager { - csm := &cachingSecretManager{ - secretStore: newSecretStore(kubeClient, clock.RealClock{}, getTTL, defaultTTL), +func newCacheBasedSecretManager(secretStore store) Manager { + return &cacheBasedSecretManager{ + secretStore: secretStore, registeredPods: make(map[objectKey]*v1.Pod), } - return csm } -func (c *cachingSecretManager) GetSecret(namespace, name string) (*v1.Secret, error) { +func (c *cacheBasedSecretManager) GetSecret(namespace, name string) (*v1.Secret, error) { return c.secretStore.Get(namespace, name) } @@ -269,12 +114,12 @@ func getSecretNames(pod *v1.Pod) sets.String { return result } -func (c *cachingSecretManager) RegisterPod(pod *v1.Pod) { +func (c *cacheBasedSecretManager) RegisterPod(pod *v1.Pod) { names := getSecretNames(pod) c.lock.Lock() defer c.lock.Unlock() for name := range names { - c.secretStore.Add(pod.Namespace, name) + c.secretStore.AddReference(pod.Namespace, name) } var prev *v1.Pod key := objectKey{namespace: pod.Namespace, name: pod.Name} @@ -287,12 +132,12 @@ func (c *cachingSecretManager) RegisterPod(pod *v1.Pod) { // names and prev need to have their ref counts decremented. Any that // are only in prev need to be completely removed. This unconditional // call takes care of both cases. - c.secretStore.Delete(prev.Namespace, name) + c.secretStore.DeleteReference(prev.Namespace, name) } } } -func (c *cachingSecretManager) UnregisterPod(pod *v1.Pod) { +func (c *cacheBasedSecretManager) UnregisterPod(pod *v1.Pod) { var prev *v1.Pod key := objectKey{namespace: pod.Namespace, name: pod.Name} c.lock.Lock() @@ -301,7 +146,7 @@ func (c *cachingSecretManager) UnregisterPod(pod *v1.Pod) { delete(c.registeredPods, key) if prev != nil { for name := range getSecretNames(prev) { - c.secretStore.Delete(prev.Namespace, name) + c.secretStore.DeleteReference(prev.Namespace, name) } } } diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index a57033f2044..b472015e148 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -278,7 +278,7 @@ func (s *Server) InstallDefaultHandlers() { // cAdvisor metrics are exposed under the secured handler as well r := prometheus.NewRegistry() - r.MustRegister(metrics.NewPrometheusCollector(prometheusHostAdapter{s.host}, containerPrometheusLabels)) + r.MustRegister(metrics.NewPrometheusCollector(prometheusHostAdapter{s.host}, containerPrometheusLabelsFunc(s.host))) s.restfulCont.Handle(cadvisorMetricsPath, promhttp.HandlerFor(r, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}), ) @@ -825,31 +825,40 @@ func (a prometheusHostAdapter) GetMachineInfo() (*cadvisorapi.MachineInfo, error return a.host.GetCachedMachineInfo() } -// containerPrometheusLabels maps cAdvisor labels to prometheus labels. -func containerPrometheusLabels(c *cadvisorapi.ContainerInfo) map[string]string { - // Prometheus requires that all metrics in the same family have the same labels, - // so we arrange to supply blank strings for missing labels - var name, image, podName, namespace, containerName string - if len(c.Aliases) > 0 { - name = c.Aliases[0] +func containerPrometheusLabelsFunc(s stats.StatsProvider) metrics.ContainerLabelsFunc { + // containerPrometheusLabels maps cAdvisor labels to prometheus labels. + return func(c *cadvisorapi.ContainerInfo) map[string]string { + // Prometheus requires that all metrics in the same family have the same labels, + // so we arrange to supply blank strings for missing labels + var name, image, podName, namespace, containerName string + if len(c.Aliases) > 0 { + name = c.Aliases[0] + } + image = c.Spec.Image + if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNameLabel]; ok { + podName = v + } + if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNamespaceLabel]; ok { + namespace = v + } + if v, ok := c.Spec.Labels[kubelettypes.KubernetesContainerNameLabel]; ok { + containerName = v + } + // Associate pod cgroup with pod so we have an accurate accounting of sandbox + if podName == "" && namespace == "" { + if pod, found := s.GetPodByCgroupfs(c.Name); found { + podName = pod.Name + namespace = pod.Namespace + } + } + set := map[string]string{ + metrics.LabelID: c.Name, + metrics.LabelName: name, + metrics.LabelImage: image, + "pod_name": podName, + "namespace": namespace, + "container_name": containerName, + } + return set } - image = c.Spec.Image - if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNameLabel]; ok { - podName = v - } - if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNamespaceLabel]; ok { - namespace = v - } - if v, ok := c.Spec.Labels[kubelettypes.KubernetesContainerNameLabel]; ok { - containerName = v - } - set := map[string]string{ - metrics.LabelID: c.Name, - metrics.LabelName: name, - metrics.LabelImage: image, - "pod_name": podName, - "namespace": namespace, - "container_name": containerName, - } - return set } diff --git a/pkg/kubelet/server/server_test.go b/pkg/kubelet/server/server_test.go index a4299401814..b5169de1a7c 100644 --- a/pkg/kubelet/server/server_test.go +++ b/pkg/kubelet/server/server_test.go @@ -166,10 +166,10 @@ func (fk *fakeKubelet) StreamingConnectionIdleTimeout() time.Duration { } // Unused functions -func (_ *fakeKubelet) GetNode() (*v1.Node, error) { return nil, nil } -func (_ *fakeKubelet) GetNodeConfig() cm.NodeConfig { return cm.NodeConfig{} } -func (_ *fakeKubelet) GetPodCgroupRoot() string { return "" } - +func (_ *fakeKubelet) GetNode() (*v1.Node, error) { return nil, nil } +func (_ *fakeKubelet) GetNodeConfig() cm.NodeConfig { return cm.NodeConfig{} } +func (_ *fakeKubelet) GetPodCgroupRoot() string { return "" } +func (_ *fakeKubelet) GetPodByCgroupfs(cgroupfs string) (*v1.Pod, bool) { return nil, false } func (fk *fakeKubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) { return map[string]volume.Volume{}, true } diff --git a/pkg/kubelet/server/stats/handler.go b/pkg/kubelet/server/stats/handler.go index ba2033c953d..f069cd898cb 100644 --- a/pkg/kubelet/server/stats/handler.go +++ b/pkg/kubelet/server/stats/handler.go @@ -84,6 +84,10 @@ type StatsProvider interface { // GetPodCgroupRoot returns the literal cgroupfs value for the cgroup containing all pods GetPodCgroupRoot() string + + // GetPodByCgroupfs provides the pod that maps to the specified cgroup literal, as well + // as whether the pod was found. + GetPodByCgroupfs(cgroupfs string) (*v1.Pod, bool) } type handler struct { diff --git a/pkg/kubelet/server/stats/testing/mock_stats_provider.go b/pkg/kubelet/server/stats/testing/mock_stats_provider.go index c49ef5979f1..a50ad43b375 100644 --- a/pkg/kubelet/server/stats/testing/mock_stats_provider.go +++ b/pkg/kubelet/server/stats/testing/mock_stats_provider.go @@ -64,6 +64,12 @@ func (_m *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*v return r0, r1, r2 } +// GetPodByCgroupfs provides the pod that maps to the specified cgroup, as well +// as whether the pod was found. +func (_m *StatsProvider) GetPodByCgroupfs(cgroupfs string) (*corev1.Pod, bool) { + return nil, false +} + // GetContainerInfo provides a mock function with given fields: podFullName, uid, containerName, req func (_m *StatsProvider) GetContainerInfo(podFullName string, uid types.UID, containerName string, req *v1.ContainerInfoRequest) (*v1.ContainerInfo, error) { ret := _m.Called(podFullName, uid, containerName, req) diff --git a/pkg/master/controller.go b/pkg/master/controller.go index e83f0cacb3d..ac5a86db1b6 100644 --- a/pkg/master/controller.go +++ b/pkg/master/controller.go @@ -272,7 +272,7 @@ func (c *Controller) CreateOrUpdateMasterServiceIfNeeded(serviceName string, ser // maintained by this code, not by the pod selector Selector: nil, ClusterIP: serviceIP.String(), - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: serviceType, }, } diff --git a/pkg/master/controller_test.go b/pkg/master/controller_test.go index ec7d63158e8..a20d82cd698 100644 --- a/pkg/master/controller_test.go +++ b/pkg/master/controller_test.go @@ -570,7 +570,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -625,7 +625,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -637,7 +637,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -658,7 +658,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -671,7 +671,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -691,7 +691,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -703,7 +703,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -723,7 +723,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -735,7 +735,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -755,7 +755,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -767,7 +767,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -787,7 +787,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -799,7 +799,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -819,7 +819,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeNodePort, }, }, @@ -831,7 +831,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -851,7 +851,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, @@ -910,7 +910,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", - SessionAffinity: api.ServiceAffinityClientIP, + SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, }, }, diff --git a/pkg/master/import_known_versions_test.go b/pkg/master/import_known_versions_test.go index e6a69e7d6f6..c2aba14015f 100644 --- a/pkg/master/import_known_versions_test.go +++ b/pkg/master/import_known_versions_test.go @@ -49,7 +49,7 @@ func TestGroupVersions(t *testing.T) { t.Errorf("No additional unnamespaced groups should be created") } - for _, gv := range legacyscheme.Registry.RegisteredGroupVersions() { + for _, gv := range legacyscheme.Scheme.PrioritizedVersionsAllGroups() { if !strings.HasSuffix(gv.Group, ".k8s.io") && !legacyUnsuffixedGroups.Has(gv.Group) { t.Errorf("Group %s does not have the standard kubernetes API group suffix of .k8s.io", gv.Group) } diff --git a/pkg/master/master_test.go b/pkg/master/master_test.go index 9da1857326b..0e39a37aaa0 100644 --- a/pkg/master/master_test.go +++ b/pkg/master/master_test.go @@ -77,7 +77,7 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, informers.SharedI }, } - resourceEncoding := serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Registry) + resourceEncoding := serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme) resourceEncoding.SetVersionEncoding(api.GroupName, schema.GroupVersion{Group: "", Version: "v1"}, schema.GroupVersion{Group: api.GroupName, Version: runtime.APIVersionInternal}) resourceEncoding.SetVersionEncoding(autoscaling.GroupName, schema.GroupVersion{Group: "autoscaling", Version: "v1"}, schema.GroupVersion{Group: autoscaling.GroupName, Version: runtime.APIVersionInternal}) resourceEncoding.SetVersionEncoding(batch.GroupName, schema.GroupVersion{Group: "batch", Version: "v1"}, schema.GroupVersion{Group: batch.GroupName, Version: runtime.APIVersionInternal}) diff --git a/pkg/printers/BUILD b/pkg/printers/BUILD index 578a62d0b19..32e5629c577 100644 --- a/pkg/printers/BUILD +++ b/pkg/printers/BUILD @@ -21,7 +21,6 @@ go_library( "kube_template_flags.go", "name.go", "name_flags.go", - "printers.go", "tabwriter.go", "template.go", "template_flags.go", diff --git a/pkg/printers/customcolumn_flags.go b/pkg/printers/customcolumn_flags.go index b5cc54f70bf..13843f07806 100644 --- a/pkg/printers/customcolumn_flags.go +++ b/pkg/printers/customcolumn_flags.go @@ -84,7 +84,7 @@ func (f *CustomColumnsPrintFlags) ToPrinter(templateFormat string) (ResourcePrin } p, err := NewCustomColumnsPrinterFromSpec(templateValue, decoder, f.NoHeaders) - return NewVersionedPrinter(p, legacyscheme.Scheme, legacyscheme.Scheme, scheme.Versions...), err + return NewVersionedPrinter(p, legacyscheme.Scheme, legacyscheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...), err } // AddFlags receives a *cobra.Command reference and binds diff --git a/pkg/printers/flags.go b/pkg/printers/flags.go index 0ab5df8be8e..97089e7cb16 100644 --- a/pkg/printers/flags.go +++ b/pkg/printers/flags.go @@ -20,6 +20,8 @@ import ( "fmt" "github.com/spf13/cobra" + + "k8s.io/apimachinery/pkg/runtime" ) type NoCompatiblePrinterError struct { @@ -53,6 +55,8 @@ type PrintFlags struct { NamePrintFlags *NamePrintFlags OutputFormat *string + + Scheme runtime.ObjectConvertor } func (f *PrintFlags) Complete(successTemplate string) error { @@ -65,12 +69,16 @@ func (f *PrintFlags) ToPrinter() (ResourcePrinter, error) { outputFormat = *f.OutputFormat } - if p, err := f.JSONYamlPrintFlags.ToPrinter(outputFormat); !IsNoCompatiblePrinterError(err) { - return p, err + if f.JSONYamlPrintFlags != nil { + if p, err := f.JSONYamlPrintFlags.ToPrinter(outputFormat); !IsNoCompatiblePrinterError(err) { + return p, err + } } - if p, err := f.NamePrintFlags.ToPrinter(outputFormat); !IsNoCompatiblePrinterError(err) { - return p, err + if f.NamePrintFlags != nil { + if p, err := f.NamePrintFlags.ToPrinter(outputFormat); !IsNoCompatiblePrinterError(err) { + return p, err + } } return nil, NoCompatiblePrinterError{Options: f, OutputFormat: f.OutputFormat} @@ -87,25 +95,19 @@ func (f *PrintFlags) AddFlags(cmd *cobra.Command) { // WithDefaultOutput sets a default output format if one is not provided through a flag value func (f *PrintFlags) WithDefaultOutput(output string) *PrintFlags { - existingFormat := "" - if f.OutputFormat != nil { - existingFormat = *f.OutputFormat - } - if len(existingFormat) == 0 { - existingFormat = output - } - f.OutputFormat = &existingFormat - + f.OutputFormat = &output return f } -func NewPrintFlags(operation string) *PrintFlags { +func NewPrintFlags(operation string, scheme runtime.ObjectConvertor) *PrintFlags { outputFormat := "" return &PrintFlags{ OutputFormat: &outputFormat, - JSONYamlPrintFlags: NewJSONYamlPrintFlags(), - NamePrintFlags: NewNamePrintFlags(operation), + Scheme: scheme, + + JSONYamlPrintFlags: NewJSONYamlPrintFlags(scheme), + NamePrintFlags: NewNamePrintFlags(operation, scheme), } } diff --git a/pkg/printers/internalversion/printers_test.go b/pkg/printers/internalversion/printers_test.go index 47199b5a10f..268c35e276b 100644 --- a/pkg/printers/internalversion/printers_test.go +++ b/pkg/printers/internalversion/printers_test.go @@ -229,78 +229,6 @@ func (obj *TestUnknownType) DeepCopyObject() runtime.Object { return &clone } -func TestPrinter(t *testing.T) { - //test inputs - simpleTest := &TestPrintType{"foo"} - podTest := &api.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} - podListTest := &api.PodList{ - Items: []api.Pod{ - {ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, - {ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, - }, - } - emptyListTest := &api.PodList{} - testapi, err := legacyscheme.Scheme.ConvertToVersion(podTest, schema.GroupVersion{Group: "", Version: "v1"}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - printerTests := []struct { - Name string - PrintOpts *printers.PrintOptions - Input runtime.Object - OutputVersions []schema.GroupVersion - Expect string - }{ - {"test json", &printers.PrintOptions{OutputFormatType: "json", AllowMissingKeys: true}, simpleTest, nil, "{\n \"Data\": \"foo\"\n}\n"}, - {"test yaml", &printers.PrintOptions{OutputFormatType: "yaml", AllowMissingKeys: true}, simpleTest, nil, "Data: foo\n"}, - {"test template", &printers.PrintOptions{OutputFormatType: "template", OutputFormatArgument: "{{if .id}}{{.id}}{{end}}{{if .metadata.name}}{{.metadata.name}}{{end}}", AllowMissingKeys: true}, - podTest, []schema.GroupVersion{v1.SchemeGroupVersion}, "foo"}, - {"test jsonpath", &printers.PrintOptions{OutputFormatType: "jsonpath", OutputFormatArgument: "{.metadata.name}", AllowMissingKeys: true}, podTest, []schema.GroupVersion{v1.SchemeGroupVersion}, "foo"}, - {"test jsonpath list", &printers.PrintOptions{OutputFormatType: "jsonpath", OutputFormatArgument: "{.items[*].metadata.name}", AllowMissingKeys: true}, podListTest, []schema.GroupVersion{v1.SchemeGroupVersion}, "foo bar"}, - {"test jsonpath empty list", &printers.PrintOptions{OutputFormatType: "jsonpath", OutputFormatArgument: "{.items[*].metadata.name}", AllowMissingKeys: true}, emptyListTest, []schema.GroupVersion{v1.SchemeGroupVersion}, ""}, - {"test name", &printers.PrintOptions{OutputFormatType: "name", AllowMissingKeys: true}, podTest, []schema.GroupVersion{v1.SchemeGroupVersion}, "pod/foo\n"}, - {"emits versioned objects", &printers.PrintOptions{OutputFormatType: "template", OutputFormatArgument: "{{.kind}}", AllowMissingKeys: true}, testapi, []schema.GroupVersion{v1.SchemeGroupVersion}, "Pod"}, - } - for _, test := range printerTests { - buf := bytes.NewBuffer([]byte{}) - printer, err := printers.GetStandardPrinter(legacyscheme.Scheme, legacyscheme.Codecs.LegacyCodec(legacyscheme.Registry.RegisteredGroupVersions()...), []runtime.Decoder{legacyscheme.Codecs.UniversalDecoder(), unstructured.UnstructuredJSONScheme}, *test.PrintOpts) - if err != nil { - t.Errorf("in %s, unexpected error: %#v", test.Name, err) - } - if len(test.OutputVersions) > 0 { - printer = printers.NewVersionedPrinter(printer, legacyscheme.Scheme, legacyscheme.Scheme, test.OutputVersions...) - } - if err := printer.PrintObj(test.Input, buf); err != nil { - t.Errorf("in %s, unexpected error: %#v", test.Name, err) - } - if buf.String() != test.Expect { - t.Errorf("in %s, expect %q, got %q", test.Name, test.Expect, buf.String()) - } - } - -} - -func TestBadPrinter(t *testing.T) { - badPrinterTests := []struct { - Name string - PrintOpts *printers.PrintOptions - Error error - }{ - {"empty template", &printers.PrintOptions{OutputFormatType: "template", AllowMissingKeys: false}, fmt.Errorf("template format specified but no template given")}, - {"bad template", &printers.PrintOptions{OutputFormatType: "template", OutputFormatArgument: "{{ .Name", AllowMissingKeys: false}, fmt.Errorf("error parsing template {{ .Name, template: output:1: unclosed action\n")}, - {"bad templatefile", &printers.PrintOptions{OutputFormatType: "templatefile", AllowMissingKeys: false}, fmt.Errorf("template format specified but no template given")}, - {"bad jsonpath", &printers.PrintOptions{OutputFormatType: "jsonpath", OutputFormatArgument: "{.Name", AllowMissingKeys: false}, fmt.Errorf("error parsing jsonpath {.Name, unclosed action\n")}, - {"unknown format", &printers.PrintOptions{OutputFormatType: "anUnknownFormat", OutputFormatArgument: "", AllowMissingKeys: false}, fmt.Errorf("output format \"anUnknownFormat\" not recognized")}, - } - for _, test := range badPrinterTests { - _, err := printers.GetStandardPrinter(legacyscheme.Scheme, legacyscheme.Codecs.LegacyCodec(legacyscheme.Registry.RegisteredGroupVersions()...), []runtime.Decoder{legacyscheme.Codecs.UniversalDecoder(), unstructured.UnstructuredJSONScheme}, *test.PrintOpts) - if err == nil || err.Error() != test.Error.Error() { - t.Errorf("in %s, expect %s, got %s", test.Name, test.Error, err) - } - } -} - func testPrinter(t *testing.T, printer printers.ResourcePrinter, unmarshalFunc func(data []byte, v interface{}) error) { buf := bytes.NewBuffer([]byte{}) @@ -471,8 +399,13 @@ func TestNamePrinter(t *testing.T) { }, "pod/foo\npod/bar\n"}, } - printOpts := &printers.PrintOptions{OutputFormatType: "name", AllowMissingKeys: false} - printer, _ := printers.GetStandardPrinter(legacyscheme.Scheme, legacyscheme.Codecs.LegacyCodec(legacyscheme.Registry.RegisteredGroupVersions()...), []runtime.Decoder{legacyscheme.Codecs.UniversalDecoder(), unstructured.UnstructuredJSONScheme}, *printOpts) + + printFlags := printers.NewPrintFlags("", legacyscheme.Scheme).WithDefaultOutput("name") + printer, err := printFlags.ToPrinter() + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + for name, item := range tests { buff := &bytes.Buffer{} err := printer.PrintObj(item.obj, buff) @@ -2987,44 +2920,6 @@ func TestPrintPodDisruptionBudget(t *testing.T) { } } -func TestAllowMissingKeys(t *testing.T) { - tests := []struct { - Name string - PrintOpts *printers.PrintOptions - Input runtime.Object - Expect string - Error string - }{ - {"test template, allow missing keys", &printers.PrintOptions{OutputFormatType: "template", OutputFormatArgument: "{{.blarg}}", AllowMissingKeys: true}, &api.Pod{}, "", ""}, - {"test template, strict", &printers.PrintOptions{OutputFormatType: "template", OutputFormatArgument: "{{.blarg}}", AllowMissingKeys: false}, &api.Pod{}, "", `error executing template "{{.blarg}}": template: output:1:2: executing "output" at <.blarg>: map has no entry for key "blarg"`}, - {"test jsonpath, allow missing keys", &printers.PrintOptions{OutputFormatType: "jsonpath", OutputFormatArgument: "{.blarg}", AllowMissingKeys: true}, &api.Pod{}, "", ""}, - {"test jsonpath, strict", &printers.PrintOptions{OutputFormatType: "jsonpath", OutputFormatArgument: "{.blarg}", AllowMissingKeys: false}, &api.Pod{}, "", "error executing jsonpath \"{.blarg}\": blarg is not found\n"}, - } - for _, test := range tests { - buf := bytes.NewBuffer([]byte{}) - printer, err := printers.GetStandardPrinter(legacyscheme.Scheme, legacyscheme.Codecs.LegacyCodec(legacyscheme.Registry.RegisteredGroupVersions()...), []runtime.Decoder{legacyscheme.Codecs.UniversalDecoder(), unstructured.UnstructuredJSONScheme}, *test.PrintOpts) - if err != nil { - t.Errorf("in %s, unexpected error: %#v", test.Name, err) - } - err = printer.PrintObj(test.Input, buf) - if len(test.Error) == 0 && err != nil { - t.Errorf("in %s, unexpected error: %v", test.Name, err) - continue - } - if len(test.Error) > 0 { - if err == nil { - t.Errorf("in %s, expected to get error: %v", test.Name, test.Error) - } else if e, a := test.Error, err.Error(); e != a { - t.Errorf("in %s, expected error %q, got %q", test.Name, e, a) - } - continue - } - if buf.String() != test.Expect { - t.Errorf("in %s, expect %q, got %q", test.Name, test.Expect, buf.String()) - } - } -} - func TestPrintControllerRevision(t *testing.T) { tests := []struct { history apps.ControllerRevision diff --git a/pkg/printers/json_yaml_flags.go b/pkg/printers/json_yaml_flags.go index 6cb6a4b5d50..158cd20e669 100644 --- a/pkg/printers/json_yaml_flags.go +++ b/pkg/printers/json_yaml_flags.go @@ -21,14 +21,16 @@ import ( "github.com/spf13/cobra" - "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/apimachinery/pkg/runtime" kubectlscheme "k8s.io/kubernetes/pkg/kubectl/scheme" ) // JSONYamlPrintFlags provides default flags necessary for json/yaml printing. // Given the following flag values, a printer can be requested that knows // how to handle printing based on these values. -type JSONYamlPrintFlags struct{} +type JSONYamlPrintFlags struct { + Scheme runtime.ObjectConvertor +} // ToPrinter receives an outputFormat and returns a printer capable of // handling --output=(yaml|json) printing. @@ -48,7 +50,7 @@ func (f *JSONYamlPrintFlags) ToPrinter(outputFormat string) (ResourcePrinter, er } // wrap the printer in a versioning printer that understands when to convert and when not to convert - return NewVersionedPrinter(printer, legacyscheme.Scheme, legacyscheme.Scheme, kubectlscheme.Versions...), nil + return NewVersionedPrinter(printer, f.Scheme, f.Scheme.(runtime.ObjectTyper), kubectlscheme.Scheme.PrioritizedVersionsAllGroups()...), nil } @@ -58,6 +60,6 @@ func (f *JSONYamlPrintFlags) AddFlags(c *cobra.Command) {} // NewJSONYamlPrintFlags returns flags associated with // yaml or json printing, with default values set. -func NewJSONYamlPrintFlags() *JSONYamlPrintFlags { - return &JSONYamlPrintFlags{} +func NewJSONYamlPrintFlags(scheme runtime.ObjectConvertor) *JSONYamlPrintFlags { + return &JSONYamlPrintFlags{Scheme: scheme} } diff --git a/pkg/printers/json_yaml_flags_test.go b/pkg/printers/json_yaml_flags_test.go index a2de7b25471..3fe8aee4100 100644 --- a/pkg/printers/json_yaml_flags_test.go +++ b/pkg/printers/json_yaml_flags_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/printers" ) @@ -59,7 +60,7 @@ func TestPrinterSupportsExpectedJSONYamlFormats(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - printFlags := printers.JSONYamlPrintFlags{} + printFlags := printers.JSONYamlPrintFlags{Scheme: legacyscheme.Scheme} p, err := printFlags.ToPrinter(tc.outputFormat) if tc.expectNoMatch { diff --git a/pkg/printers/name_flags.go b/pkg/printers/name_flags.go index 193b4e5947f..01501703ea3 100644 --- a/pkg/printers/name_flags.go +++ b/pkg/printers/name_flags.go @@ -32,6 +32,8 @@ import ( // a resource's fully-qualified Kind.group/name, or a successful // message about that resource if an Operation is provided. type NamePrintFlags struct { + Scheme runtime.ObjectConvertor + // Operation describes the name of the action that // took place on an object, to be included in the // finalized "successful" message. @@ -73,7 +75,7 @@ func (f *NamePrintFlags) AddFlags(c *cobra.Command) {} // NewNamePrintFlags returns flags associated with // --name printing, with default values set. -func NewNamePrintFlags(operation string) *NamePrintFlags { +func NewNamePrintFlags(operation string, scheme runtime.ObjectConvertor) *NamePrintFlags { return &NamePrintFlags{ Operation: operation, } diff --git a/pkg/printers/printers.go b/pkg/printers/printers.go deleted file mode 100644 index 9daee38807d..00000000000 --- a/pkg/printers/printers.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package printers - -import ( - "fmt" - - "k8s.io/apimachinery/pkg/runtime" -) - -// GetStandardPrinter takes a format type, an optional format argument. It will return -// a printer or an error. The printer is agnostic to schema versions, so you must -// send arguments to PrintObj in the version you wish them to be shown using a -// VersionedPrinter (typically when generic is true). -func GetStandardPrinter(typer runtime.ObjectTyper, encoder runtime.Encoder, decoders []runtime.Decoder, options PrintOptions) (ResourcePrinter, error) { - format, formatArgument, allowMissingTemplateKeys := options.OutputFormatType, options.OutputFormatArgument, options.AllowMissingKeys - - var printer ResourcePrinter - switch format { - - case "json", "yaml": - jsonYamlFlags := NewJSONYamlPrintFlags() - p, err := jsonYamlFlags.ToPrinter(format) - if err != nil { - return nil, err - } - - printer = p - - case "name": - nameFlags := NewNamePrintFlags("") - namePrinter, err := nameFlags.ToPrinter(format) - if err != nil { - return nil, err - } - - printer = namePrinter - - case "template", "go-template", "jsonpath", "templatefile", "go-template-file", "jsonpath-file": - // TODO: construct and bind this separately (at the command level) - kubeTemplateFlags := KubeTemplatePrintFlags{ - GoTemplatePrintFlags: &GoTemplatePrintFlags{ - AllowMissingKeys: &allowMissingTemplateKeys, - TemplateArgument: &formatArgument, - }, - JSONPathPrintFlags: &JSONPathPrintFlags{ - AllowMissingKeys: &allowMissingTemplateKeys, - TemplateArgument: &formatArgument, - }, - } - - kubeTemplatePrinter, err := kubeTemplateFlags.ToPrinter(format) - if err != nil { - return nil, err - } - - printer = kubeTemplatePrinter - - case "custom-columns", "custom-columns-file": - customColumnsFlags := &CustomColumnsPrintFlags{ - NoHeaders: options.NoHeaders, - TemplateArgument: formatArgument, - } - customColumnsPrinter, err := customColumnsFlags.ToPrinter(format) - if err != nil { - return nil, err - } - - printer = customColumnsPrinter - - default: - return nil, fmt.Errorf("output format %q not recognized", format) - } - return printer, nil -} diff --git a/pkg/proxy/ipvs/ipset.go b/pkg/proxy/ipvs/ipset.go index 037df0ada85..54634e342d9 100644 --- a/pkg/proxy/ipvs/ipset.go +++ b/pkg/proxy/ipvs/ipset.go @@ -40,8 +40,11 @@ const ( // KubeLoadBalancerSet is used to store service load balancer ingress ip + port, it is the service lb portal. KubeLoadBalancerSet = "KUBE-LOAD-BALANCER" - // KubeLoadBalancerIngressLocalSet is used to store service load balancer ingress ip + port with externalTrafficPolicy=local. - KubeLoadBalancerIngressLocalSet = "KUBE-LB-INGRESS-LOCAL" + // KubeLoadBalancerLocalSet is used to store service load balancer ingress ip + port with externalTrafficPolicy=local. + KubeLoadBalancerLocalSet = "KUBE-LOAD-BALANCER-LOCAL" + + // KubeLoadbalancerFWSet is used to store service load balancer ingress ip + port for load balancer with sourceRange. + KubeLoadbalancerFWSet = "KUBE-LOAD-BALANCER-FW" // KubeLoadBalancerSourceIPSet is used to store service load balancer ingress ip + port + source IP for packet filter purpose. KubeLoadBalancerSourceIPSet = "KUBE-LOAD-BALANCER-SOURCE-IP" diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 9de9aa7204a..38df2bb4d22 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -57,7 +57,7 @@ const ( kubeServicesChain utiliptables.Chain = "KUBE-SERVICES" // KubeFireWallChain is the kubernetes firewall chain. - KubeFireWallChain utiliptables.Chain = "KUBE-FIRE-WALL" + KubeFireWallChain utiliptables.Chain = "KUBE-FIREWALL" // kubePostroutingChain is the kubernetes postrouting chain kubePostroutingChain utiliptables.Chain = "KUBE-POSTROUTING" @@ -74,6 +74,9 @@ const ( // KubeForwardChain is the kubernetes forward chain KubeForwardChain utiliptables.Chain = "KUBE-FORWARD" + // KubeLoadBalancerChain is the kubernetes chain for loadbalancer type service + KubeLoadBalancerChain utiliptables.Chain = "KUBE-LOAD-BALANCER" + // DefaultScheduler is the default ipvs scheduler algorithm - round robin. DefaultScheduler = "rr" @@ -164,16 +167,18 @@ type Proxier struct { nodePortSetTCP *IPSet // nodePortSetTCP is the bitmap:port type ipset which stores all UDP node port nodePortSetUDP *IPSet - // lbIngressLocalSet is the hash:ip type ipset which stores all service ip's with externaltrafficPolicy=local - lbIngressLocalSet *IPSet // nodePortLocalSetTCP is the bitmap:port type ipset which stores all TCP nodeport's with externaltrafficPolicy=local nodePortLocalSetTCP *IPSet // nodePortLocalSetUDP is the bitmap:port type ipset which stores all UDP nodeport's with externaltrafficPolicy=local nodePortLocalSetUDP *IPSet // externalIPSet is the hash:ip,port type ipset which stores all service ExternalIP:Port externalIPSet *IPSet - // lbIngressSet is the hash:ip,port type ipset which stores all service load balancer ingress IP:Port. - lbIngressSet *IPSet + // lbSet is the hash:ip,port type ipset which stores all service load balancer IP:Port. + lbSet *IPSet + // lbLocalSet is the hash:ip type ipset which stores all service ip's with externaltrafficPolicy=local + lbLocalSet *IPSet + // lbFWSet is the hash:ip,port type ipset which stores all service load balancer ingress IP:Port for load balancer with sourceRange. + lbFWSet *IPSet // lbWhiteListIPSet is the hash:ip,port,ip type ipset which stores all service load balancer ingress IP:Port,sourceIP pair, any packets // with the source IP visit ingress IP:Port can pass through. lbWhiteListIPSet *IPSet @@ -351,8 +356,9 @@ func NewProxier(ipt utiliptables.Interface, loopbackSet: NewIPSet(ipset, KubeLoopBackIPSet, utilipset.HashIPPortIP, isIPv6), clusterIPSet: NewIPSet(ipset, KubeClusterIPSet, utilipset.HashIPPort, isIPv6), externalIPSet: NewIPSet(ipset, KubeExternalIPSet, utilipset.HashIPPort, isIPv6), - lbIngressSet: NewIPSet(ipset, KubeLoadBalancerSet, utilipset.HashIPPort, isIPv6), - lbIngressLocalSet: NewIPSet(ipset, KubeLoadBalancerIngressLocalSet, utilipset.HashIPPort, isIPv6), + lbSet: NewIPSet(ipset, KubeLoadBalancerSet, utilipset.HashIPPort, isIPv6), + lbFWSet: NewIPSet(ipset, KubeLoadbalancerFWSet, utilipset.HashIPPort, isIPv6), + lbLocalSet: NewIPSet(ipset, KubeLoadBalancerLocalSet, utilipset.HashIPPort, isIPv6), lbWhiteListIPSet: NewIPSet(ipset, KubeLoadBalancerSourceIPSet, utilipset.HashIPPortIP, isIPv6), lbWhiteListCIDRSet: NewIPSet(ipset, KubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, isIPv6), nodePortSetTCP: NewIPSet(ipset, KubeNodePortSetTCP, utilipset.BitmapPort, false), @@ -496,7 +502,7 @@ func cleanupIptablesLeftovers(ipt utiliptables.Interface) (encounteredError bool } // Flush and remove all of our "-t nat" chains. - for _, chain := range []utiliptables.Chain{kubeServicesChain, kubePostroutingChain} { + for _, chain := range []utiliptables.Chain{kubeServicesChain, kubePostroutingChain, KubeNodePortChain, KubeLoadBalancerChain, KubeFireWallChain} { if err := ipt.FlushChain(utiliptables.TableNAT, chain); err != nil { if !utiliptables.IsNotFoundError(err) { glog.Errorf("Error removing iptables rules in ipvs proxier: %v", err) @@ -553,8 +559,8 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset // Destroy ip sets created by ipvs Proxier. We should call it after cleaning up // iptables since we can NOT delete ip set which is still referenced by iptables. ipSetsToDestroy := []string{KubeLoopBackIPSet, KubeClusterIPSet, KubeLoadBalancerSet, KubeNodePortSetTCP, KubeNodePortSetUDP, - KubeExternalIPSet, KubeLoadBalancerSourceIPSet, KubeLoadBalancerSourceCIDRSet, - KubeLoadBalancerIngressLocalSet, KubeNodePortLocalSetUDP, KubeNodePortLocalSetTCP} + KubeExternalIPSet, KubeLoadbalancerFWSet, KubeLoadBalancerSourceIPSet, KubeLoadBalancerSourceCIDRSet, + KubeLoadBalancerLocalSet, KubeNodePortLocalSetUDP, KubeNodePortLocalSetTCP} for _, set := range ipSetsToDestroy { err = ipset.DestroySet(set) if err != nil { @@ -755,7 +761,7 @@ func (proxier *Proxier) syncProxyRules() { // make sure ip sets exists in the system. ipSets := []*IPSet{proxier.loopbackSet, proxier.clusterIPSet, proxier.externalIPSet, proxier.nodePortSetUDP, proxier.nodePortSetTCP, - proxier.lbIngressSet, proxier.lbWhiteListCIDRSet, proxier.lbWhiteListIPSet, proxier.lbIngressLocalSet, + proxier.lbSet, proxier.lbFWSet, proxier.lbWhiteListCIDRSet, proxier.lbWhiteListIPSet, proxier.lbLocalSet, proxier.nodePortLocalSetTCP, proxier.nodePortLocalSetUDP} if err := ensureIPSets(ipSets...); err != nil { return @@ -788,9 +794,19 @@ func (proxier *Proxier) syncProxyRules() { glog.Errorf("Failed to link KUBE-SERVICES chain: %v", err) return } - // `iptables -t nat -N KUBE-FIRE-WALL` + // `iptables -t nat -N KUBE-FIREWALL` if err := proxier.createKubeChain(existingNATChains, KubeFireWallChain); err != nil { - glog.Errorf("Failed to create KUBE-FIRE-WALL chain: %v", err) + glog.Errorf("Failed to create KUBE-FIREWALL chain: %v", err) + return + } + // `iptables -t nat -N KUBE-NODE-PORT` + if err := proxier.createKubeChain(existingNATChains, KubeNodePortChain); err != nil { + glog.Errorf("Failed to create KUBE-NODE-PORT chain: %v", err) + return + } + // `iptables -t nat -N KUBE-LOAD-BALANCER` + if err := proxier.createKubeChain(existingNATChains, KubeLoadBalancerChain); err != nil { + glog.Errorf("Failed to create KUBE-LOAD-BALANCER chain: %v", err) return } // Kube forward @@ -799,12 +815,6 @@ func (proxier *Proxier) syncProxyRules() { return } - // `iptables -t nat -N KUBE-NODE-PORT` - if err := proxier.createKubeChain(existingNATChains, KubeNodePortChain); err != nil { - glog.Errorf("Failed to create KUBE-NODE-PORT chain: %v", err) - return - } - // Build IPVS rules for each service. for svcName, svc := range proxier.serviceMap { svcInfo, ok := svc.(*serviceInfo) @@ -968,23 +978,28 @@ func (proxier *Proxier) syncProxyRules() { // proxier.kubeServiceAccessSet.activeEntries.Insert(entry.String()) // If we are proxying globally, we need to masquerade in case we cross nodes. // If we are proxying only locally, we can retain the source IP. - if valid := proxier.lbIngressSet.validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.lbIngressSet.Name)) + if valid := proxier.lbSet.validateEntry(entry); !valid { + glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.lbSet.Name)) continue } - proxier.lbIngressSet.activeEntries.Insert(entry.String()) + proxier.lbSet.activeEntries.Insert(entry.String()) // insert loadbalancer entry to lbIngressLocalSet if service externaltrafficpolicy=local if svcInfo.OnlyNodeLocalEndpoints { - if valid := proxier.lbIngressLocalSet.validateEntry(entry); !valid { - glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.lbIngressSet.Name)) + if valid := proxier.lbLocalSet.validateEntry(entry); !valid { + glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.lbLocalSet.Name)) continue } - proxier.lbIngressLocalSet.activeEntries.Insert(entry.String()) + proxier.lbLocalSet.activeEntries.Insert(entry.String()) } if len(svcInfo.LoadBalancerSourceRanges) != 0 { // The service firewall rules are created based on ServiceSpec.loadBalancerSourceRanges field. // This currently works for loadbalancers that preserves source ips. // For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply. + if valid := proxier.lbFWSet.validateEntry(entry); !valid { + glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.lbFWSet.Name)) + continue + } + proxier.lbFWSet.activeEntries.Insert(entry.String()) allowFromNode := false for _, src := range svcInfo.LoadBalancerSourceRanges { // ipset call @@ -1164,9 +1179,9 @@ func (proxier *Proxier) syncProxyRules() { } // sync ipset entries - ipsetsToSync := []*IPSet{proxier.loopbackSet, proxier.clusterIPSet, proxier.lbIngressSet, proxier.nodePortSetTCP, - proxier.nodePortSetUDP, proxier.externalIPSet, proxier.lbWhiteListIPSet, proxier.lbWhiteListCIDRSet, proxier.lbIngressLocalSet, - proxier.nodePortLocalSetTCP, proxier.nodePortLocalSetUDP} + ipsetsToSync := []*IPSet{proxier.loopbackSet, proxier.clusterIPSet, proxier.lbSet, proxier.nodePortSetTCP, + proxier.lbFWSet, proxier.nodePortSetUDP, proxier.externalIPSet, proxier.lbWhiteListIPSet, + proxier.lbWhiteListCIDRSet, proxier.lbLocalSet, proxier.nodePortLocalSetTCP, proxier.nodePortLocalSetUDP} for i := range ipsetsToSync { ipsetsToSync[i].syncIPSetEntries() } @@ -1219,39 +1234,31 @@ func (proxier *Proxier) syncProxyRules() { // This covers cases like GCE load-balancers which get added to the local routing table. writeLine(proxier.natRules, append(dstLocalOnlyArgs, "-j", "ACCEPT")...) } - if !proxier.lbIngressSet.isEmpty() { + if !proxier.lbSet.isEmpty() { // Build masquerade rules for packets which cross node visit load balancer ingress IPs. args = append(args[:0], "-A", string(kubeServicesChain), - "-m", "set", "--match-set", proxier.lbIngressSet.Name, + "-m", "set", "--match-set", proxier.lbSet.Name, "dst,dst", ) - writeLine(proxier.natRules, append(args, "-j", string(KubeFireWallChain))...) - // Don't masq for service with externaltrafficpolicy =local - if !proxier.lbIngressLocalSet.isEmpty() { - args = append(args[:0], - "-A", string(KubeFireWallChain), - "-m", "set", "--match-set", proxier.lbIngressLocalSet.Name, - "dst,dst", - ) - writeLine(proxier.natRules, append(args, "-j", "ACCEPT")...) - } - // mark masq for others - args = append(args[:0], - "-A", string(KubeFireWallChain), - "-m", "comment", "--comment", - fmt.Sprintf(`"mark MASQ for external traffic policy not local"`), - ) - writeLine(proxier.natRules, append(args, "-j", string(KubeMarkMasqChain))...) + writeLine(proxier.natRules, append(args, "-j", string(KubeLoadBalancerChain))...) // if have whitelist, accept or drop. if !proxier.lbWhiteListCIDRSet.isEmpty() || !proxier.lbWhiteListIPSet.isEmpty() { + if !proxier.lbFWSet.isEmpty() { + args = append(args[:0], + "-A", string(KubeLoadBalancerChain), + "-m", "set", "--match-set", proxier.lbFWSet.Name, + "dst,dst", + ) + writeLine(proxier.natRules, append(args, "-j", string(KubeFireWallChain))...) + } if !proxier.lbWhiteListCIDRSet.isEmpty() { args = append(args[:0], "-A", string(KubeFireWallChain), "-m", "set", "--match-set", proxier.lbWhiteListCIDRSet.Name, "dst,dst,src", ) - writeLine(proxier.natRules, append(args, "-j", "ACCEPT")...) + writeLine(proxier.natRules, append(args, "-j", "RETURN")...) } if !proxier.lbWhiteListIPSet.isEmpty() { args = append(args[:0], @@ -1259,7 +1266,7 @@ func (proxier *Proxier) syncProxyRules() { "-m", "set", "--match-set", proxier.lbWhiteListIPSet.Name, "dst,dst,src", ) - writeLine(proxier.natRules, append(args, "-j", "ACCEPT")...) + writeLine(proxier.natRules, append(args, "-j", "RETURN")...) } args = append(args[:0], "-A", string(KubeFireWallChain), @@ -1268,6 +1275,22 @@ func (proxier *Proxier) syncProxyRules() { // It means the packet cannot go thru the firewall, then mark it for DROP writeLine(proxier.natRules, append(args, "-j", string(KubeMarkDropChain))...) } + // Don't masq for service with externaltrafficpolicy =local + if !proxier.lbLocalSet.isEmpty() { + args = append(args[:0], + "-A", string(KubeLoadBalancerChain), + "-m", "set", "--match-set", proxier.lbLocalSet.Name, + "dst,dst", + ) + writeLine(proxier.natRules, append(args, "-j", "RETURN")...) + } + // mark masq for others + args = append(args[:0], + "-A", string(KubeLoadBalancerChain), + "-m", "comment", "--comment", + fmt.Sprintf(`"mark MASQ for external traffic policy not local"`), + ) + writeLine(proxier.natRules, append(args, "-j", string(KubeMarkMasqChain))...) } if !proxier.nodePortSetTCP.isEmpty() { // Build masquerade rules for packets which cross node visit nodeport. @@ -1424,7 +1447,7 @@ func (proxier *Proxier) syncProxyRules() { } func (proxier *Proxier) acceptIPVSTraffic() { - sets := []*IPSet{proxier.clusterIPSet, proxier.externalIPSet, proxier.lbIngressSet} + sets := []*IPSet{proxier.clusterIPSet, proxier.externalIPSet, proxier.lbSet} for _, set := range sets { var matchType string if !set.isEmpty() { diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index e540c68df68..bcc506ddb2a 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -145,8 +145,9 @@ func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset u loopbackSet: NewIPSet(ipset, KubeLoopBackIPSet, utilipset.HashIPPortIP, false), clusterIPSet: NewIPSet(ipset, KubeClusterIPSet, utilipset.HashIPPort, false), externalIPSet: NewIPSet(ipset, KubeExternalIPSet, utilipset.HashIPPort, false), - lbIngressSet: NewIPSet(ipset, KubeLoadBalancerSet, utilipset.HashIPPort, false), - lbIngressLocalSet: NewIPSet(ipset, KubeLoadBalancerIngressLocalSet, utilipset.HashIPPort, false), + lbSet: NewIPSet(ipset, KubeLoadBalancerSet, utilipset.HashIPPort, false), + lbFWSet: NewIPSet(ipset, KubeLoadbalancerFWSet, utilipset.HashIPPort, false), + lbLocalSet: NewIPSet(ipset, KubeLoadBalancerLocalSet, utilipset.HashIPPort, false), lbWhiteListIPSet: NewIPSet(ipset, KubeLoadBalancerSourceIPSet, utilipset.HashIPPortIP, false), lbWhiteListCIDRSet: NewIPSet(ipset, KubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, false), nodePortSetTCP: NewIPSet(ipset, KubeNodePortSetTCP, utilipset.BitmapPort, false), @@ -957,6 +958,12 @@ func TestLoadBalanceSourceRanges(t *testing.T) { Protocol: strings.ToLower(string(api.ProtocolTCP)), SetType: utilipset.HashIPPort, }}, + KubeLoadbalancerFWSet: {{ + IP: svcLBIP, + Port: svcPort, + Protocol: strings.ToLower(string(api.ProtocolTCP)), + SetType: utilipset.HashIPPort, + }}, KubeLoadBalancerSourceCIDRSet: {{ IP: svcLBIP, Port: svcPort, @@ -970,10 +977,15 @@ func TestLoadBalanceSourceRanges(t *testing.T) { // Check iptables chain and rules epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): {{ - JumpChain: string(KubeFireWallChain), MatchSet: KubeLoadBalancerSet, + JumpChain: string(KubeLoadBalancerChain), MatchSet: KubeLoadBalancerSet, + }}, + string(KubeLoadBalancerChain): {{ + JumpChain: string(KubeFireWallChain), MatchSet: KubeLoadbalancerFWSet, + }, { + JumpChain: string(KubeMarkMasqChain), MatchSet: "", }}, string(KubeFireWallChain): {{ - JumpChain: "ACCEPT", MatchSet: KubeLoadBalancerSourceCIDRSet, + JumpChain: "RETURN", MatchSet: KubeLoadBalancerSourceCIDRSet, }, { JumpChain: string(KubeMarkDropChain), MatchSet: "", }}, @@ -1109,7 +1121,7 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { Protocol: strings.ToLower(string(api.ProtocolTCP)), SetType: utilipset.HashIPPort, }}, - KubeLoadBalancerIngressLocalSet: {{ + KubeLoadBalancerLocalSet: {{ IP: svcLBIP, Port: svcPort, Protocol: strings.ToLower(string(api.ProtocolTCP)), @@ -1121,10 +1133,10 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { // Check iptables chain and rules epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): {{ - JumpChain: string(KubeFireWallChain), MatchSet: KubeLoadBalancerSet, + JumpChain: string(KubeLoadBalancerChain), MatchSet: KubeLoadBalancerSet, }}, - string(KubeFireWallChain): {{ - JumpChain: "ACCEPT", MatchSet: KubeLoadBalancerIngressLocalSet, + string(KubeLoadBalancerChain): {{ + JumpChain: "RETURN", MatchSet: KubeLoadBalancerLocalSet, }, { JumpChain: string(KubeMarkMasqChain), MatchSet: "", }}, @@ -2580,7 +2592,7 @@ func checkIPSet(t *testing.T, fp *Proxier, ipSet netlinktest.ExpectedIPSet) { for set, entries := range ipSet { ents, err := fp.ipset.ListEntries(set) if err != nil || len(ents) != len(entries) { - t.Errorf("Check ipset entries failed for ipset: %q", set) + t.Errorf("Check ipset entries failed for ipset: %q, expect %d, got %d", set, len(entries), len(ents)) continue } if len(entries) == 1 { diff --git a/pkg/registry/admissionregistration/rest/storage_apiserver.go b/pkg/registry/admissionregistration/rest/storage_apiserver.go index 1905223f4e0..8bfcb35eb75 100644 --- a/pkg/registry/admissionregistration/rest/storage_apiserver.go +++ b/pkg/registry/admissionregistration/rest/storage_apiserver.go @@ -33,7 +33,7 @@ import ( type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(admissionregistration.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(admissionregistration.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/apps/rest/storage_apps.go b/pkg/registry/apps/rest/storage_apps.go index 03e6124a662..b8e683ee9a1 100644 --- a/pkg/registry/apps/rest/storage_apps.go +++ b/pkg/registry/apps/rest/storage_apps.go @@ -36,7 +36,7 @@ import ( type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apps.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apps.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/authentication/rest/storage_authentication.go b/pkg/registry/authentication/rest/storage_authentication.go index 9fbb74c0190..5dec4aeb077 100644 --- a/pkg/registry/authentication/rest/storage_authentication.go +++ b/pkg/registry/authentication/rest/storage_authentication.go @@ -39,7 +39,7 @@ func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorag // return genericapiserver.APIGroupInfo{}, false // } - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(authentication.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(authentication.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/authorization/rest/storage_authorization.go b/pkg/registry/authorization/rest/storage_authorization.go index 74d8683f9a0..af79bf44134 100644 --- a/pkg/registry/authorization/rest/storage_authorization.go +++ b/pkg/registry/authorization/rest/storage_authorization.go @@ -42,7 +42,7 @@ func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorag return genericapiserver.APIGroupInfo{}, false } - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(authorization.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(authorization.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/autoscaling/rest/storage_autoscaling.go b/pkg/registry/autoscaling/rest/storage_autoscaling.go index 6ce7893cb54..021beb07042 100644 --- a/pkg/registry/autoscaling/rest/storage_autoscaling.go +++ b/pkg/registry/autoscaling/rest/storage_autoscaling.go @@ -31,7 +31,7 @@ import ( type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(autoscaling.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(autoscaling.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/batch/rest/storage_batch.go b/pkg/registry/batch/rest/storage_batch.go index 8c422c94d35..4568e8bd929 100644 --- a/pkg/registry/batch/rest/storage_batch.go +++ b/pkg/registry/batch/rest/storage_batch.go @@ -33,7 +33,7 @@ import ( type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(batch.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(batch.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/certificates/rest/storage_certificates.go b/pkg/registry/certificates/rest/storage_certificates.go index 0ee412562cd..aa1fbc1ae37 100644 --- a/pkg/registry/certificates/rest/storage_certificates.go +++ b/pkg/registry/certificates/rest/storage_certificates.go @@ -30,7 +30,7 @@ import ( type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(certificates.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(certificates.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index 26fc0304f30..41e21f995f1 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -95,7 +95,7 @@ type LegacyRESTStorage struct { func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generic.RESTOptionsGetter) (LegacyRESTStorage, genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.APIGroupInfo{ - GroupMeta: *legacyscheme.Registry.GroupOrDie(api.GroupName), + PrioritizedVersions: legacyscheme.Scheme.PrioritizedVersionsForGroup(""), VersionedResourcesStorageMap: map[string]map[string]rest.Storage{}, Scheme: legacyscheme.Scheme, ParameterCodec: legacyscheme.ParameterCodec, @@ -103,7 +103,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi } var podDisruptionClient policyclient.PodDisruptionBudgetsGetter - if policyGroupVersion := (schema.GroupVersion{Group: "policy", Version: "v1beta1"}); legacyscheme.Registry.IsRegisteredVersion(policyGroupVersion) { + if policyGroupVersion := (schema.GroupVersion{Group: "policy", Version: "v1beta1"}); legacyscheme.Scheme.IsVersionRegistered(policyGroupVersion) { var err error podDisruptionClient, err = policyclient.NewForConfig(c.LoopbackClientConfig) if err != nil { @@ -226,10 +226,10 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi "componentStatuses": componentstatus.NewStorage(componentStatusStorage{c.StorageFactory}.serversToValidate), } - if legacyscheme.Registry.IsRegisteredVersion(schema.GroupVersion{Group: "autoscaling", Version: "v1"}) { + if legacyscheme.Scheme.IsVersionRegistered(schema.GroupVersion{Group: "autoscaling", Version: "v1"}) { restStorageMap["replicationControllers/scale"] = controllerStorage.Scale } - if legacyscheme.Registry.IsRegisteredVersion(schema.GroupVersion{Group: "policy", Version: "v1beta1"}) { + if legacyscheme.Scheme.IsVersionRegistered(schema.GroupVersion{Group: "policy", Version: "v1beta1"}) { restStorageMap["pods/eviction"] = podStorage.Eviction } if serviceAccountStorage.Token != nil { diff --git a/pkg/registry/events/rest/storage_events.go b/pkg/registry/events/rest/storage_events.go index 681dd342730..de4caba3839 100644 --- a/pkg/registry/events/rest/storage_events.go +++ b/pkg/registry/events/rest/storage_events.go @@ -34,7 +34,7 @@ type RESTStorageProvider struct { } func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(events.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(events.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/extensions/rest/storage_extensions.go b/pkg/registry/extensions/rest/storage_extensions.go index 77478b7c7d9..2b21e502f07 100644 --- a/pkg/registry/extensions/rest/storage_extensions.go +++ b/pkg/registry/extensions/rest/storage_extensions.go @@ -36,7 +36,7 @@ import ( type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(extensions.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(extensions.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/networking/rest/storage_settings.go b/pkg/registry/networking/rest/storage_settings.go index fc0bd37327b..40cf8c525ec 100644 --- a/pkg/registry/networking/rest/storage_settings.go +++ b/pkg/registry/networking/rest/storage_settings.go @@ -30,7 +30,7 @@ import ( type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(networking.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(networking.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/policy/rest/storage_policy.go b/pkg/registry/policy/rest/storage_policy.go index 6935508243c..a4ddb9591d9 100644 --- a/pkg/registry/policy/rest/storage_policy.go +++ b/pkg/registry/policy/rest/storage_policy.go @@ -31,7 +31,7 @@ import ( type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(policy.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(policy.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/rbac/rest/storage_rbac.go b/pkg/registry/rbac/rest/storage_rbac.go index 8708dfe8fe2..b2bdb4bbb7b 100644 --- a/pkg/registry/rbac/rest/storage_rbac.go +++ b/pkg/registry/rbac/rest/storage_rbac.go @@ -66,7 +66,7 @@ type RESTStorageProvider struct { var _ genericapiserver.PostStartHookProvider = RESTStorageProvider{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(rbac.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(rbac.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/scheduling/rest/storage_scheduling.go b/pkg/registry/scheduling/rest/storage_scheduling.go index 32df45618eb..4d6b2e6c982 100644 --- a/pkg/registry/scheduling/rest/storage_scheduling.go +++ b/pkg/registry/scheduling/rest/storage_scheduling.go @@ -44,7 +44,7 @@ type RESTStorageProvider struct{} var _ genericapiserver.PostStartHookProvider = RESTStorageProvider{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(scheduling.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(scheduling.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) if apiResourceConfigSource.VersionEnabled(schedulingapiv1alpha1.SchemeGroupVersion) { apiGroupInfo.VersionedResourcesStorageMap[schedulingapiv1alpha1.SchemeGroupVersion.Version] = p.v1alpha1Storage(apiResourceConfigSource, restOptionsGetter) diff --git a/pkg/registry/settings/rest/storage_settings.go b/pkg/registry/settings/rest/storage_settings.go index a1e3dd22d96..3f50a54f924 100644 --- a/pkg/registry/settings/rest/storage_settings.go +++ b/pkg/registry/settings/rest/storage_settings.go @@ -30,7 +30,7 @@ import ( type RESTStorageProvider struct{} func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(settings.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(settings.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/registry/storage/rest/storage_storage.go b/pkg/registry/storage/rest/storage_storage.go index 3fe7eaa14dc..5e8b2529a5f 100644 --- a/pkg/registry/storage/rest/storage_storage.go +++ b/pkg/registry/storage/rest/storage_storage.go @@ -34,7 +34,7 @@ type RESTStorageProvider struct { } func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(storageapi.GroupName, legacyscheme.Registry, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(storageapi.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs) // If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities. // TODO refactor the plumbing to provide the information in the APIGroupInfo diff --git a/pkg/scheduler/algorithm/BUILD b/pkg/scheduler/algorithm/BUILD index f2230c48c36..9d2f4b7021e 100644 --- a/pkg/scheduler/algorithm/BUILD +++ b/pkg/scheduler/algorithm/BUILD @@ -16,6 +16,7 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/scheduler/algorithm", deps = [ + "//pkg/apis/core:go_default_library", "//pkg/scheduler/api:go_default_library", "//pkg/scheduler/schedulercache:go_default_library", "//vendor/k8s.io/api/apps/v1beta1:go_default_library", diff --git a/pkg/scheduler/algorithm/predicates/BUILD b/pkg/scheduler/algorithm/predicates/BUILD index d85c822872c..069dac60931 100644 --- a/pkg/scheduler/algorithm/predicates/BUILD +++ b/pkg/scheduler/algorithm/predicates/BUILD @@ -32,6 +32,7 @@ go_library( "//vendor/k8s.io/api/storage/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index abca33adbee..f1beb2394fc 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -24,10 +24,12 @@ import ( "sync" "github.com/golang/glog" + "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" @@ -670,29 +672,7 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource { // take max_resource(sum_pod, any_init_container) for _, container := range pod.Spec.InitContainers { - for rName, rQuantity := range container.Resources.Requests { - switch rName { - case v1.ResourceMemory: - if mem := rQuantity.Value(); mem > result.Memory { - result.Memory = mem - } - case v1.ResourceEphemeralStorage: - if ephemeralStorage := rQuantity.Value(); ephemeralStorage > result.EphemeralStorage { - result.EphemeralStorage = ephemeralStorage - } - case v1.ResourceCPU: - if cpu := rQuantity.MilliValue(); cpu > result.MilliCPU { - result.MilliCPU = cpu - } - default: - if v1helper.IsScalarResourceName(rName) { - value := rQuantity.Value() - if value > result.ScalarResources[rName] { - result.SetScalar(rName, value) - } - } - } - } + result.SetMaxResource(container.Resources.Requests) } return result @@ -775,21 +755,16 @@ func PodFitsResources(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *s // nodeMatchesNodeSelectorTerms checks if a node's labels satisfy a list of node selector terms, // terms are ORed, and an empty list of terms will match nothing. func nodeMatchesNodeSelectorTerms(node *v1.Node, nodeSelectorTerms []v1.NodeSelectorTerm) bool { - for _, req := range nodeSelectorTerms { - nodeSelector, err := v1helper.NodeSelectorRequirementsAsSelector(req.MatchExpressions) - if err != nil { - glog.V(10).Infof("Failed to parse MatchExpressions: %+v, regarding as not match.", req.MatchExpressions) - return false - } - if nodeSelector.Matches(labels.Set(node.Labels)) { - return true - } + nodeFields := map[string]string{} + for k, f := range algorithm.NodeFieldSelectorKeys { + nodeFields[k] = f(node) } - return false + return v1helper.MatchNodeSelectorTerms(nodeSelectorTerms, labels.Set(node.Labels), fields.Set(nodeFields)) } -// The pod can only schedule onto nodes that satisfy requirements in both NodeAffinity and nodeSelector. -func podMatchesNodeLabels(pod *v1.Pod, node *v1.Node) bool { +// podMatchesNodeSelectorAndAffinityTerms checks whether the pod is schedulable onto nodes according to +// the requirements in both NodeAffinity and nodeSelector. +func podMatchesNodeSelectorAndAffinityTerms(pod *v1.Pod, node *v1.Node) bool { // Check if node.Labels match pod.Spec.NodeSelector. if len(pod.Spec.NodeSelector) > 0 { selector := labels.SelectorFromSet(pod.Spec.NodeSelector) @@ -840,7 +815,7 @@ func PodMatchNodeSelector(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInf if node == nil { return false, nil, fmt.Errorf("node not found") } - if podMatchesNodeLabels(pod, node) { + if podMatchesNodeSelectorAndAffinityTerms(pod, node) { return true, nil, nil } return false, []algorithm.PredicateFailureReason{ErrNodeSelectorNotMatch}, nil diff --git a/pkg/scheduler/algorithm/predicates/predicates_test.go b/pkg/scheduler/algorithm/predicates/predicates_test.go index b34ccdb0d36..ab757640b0a 100644 --- a/pkg/scheduler/algorithm/predicates/predicates_test.go +++ b/pkg/scheduler/algorithm/predicates/predicates_test.go @@ -911,10 +911,11 @@ func TestISCSIDiskConflicts(t *testing.T) { // TODO: Add test case for RequiredDuringSchedulingRequiredDuringExecution after it's implemented. func TestPodFitsSelector(t *testing.T) { tests := []struct { - pod *v1.Pod - labels map[string]string - fits bool - test string + pod *v1.Pod + labels map[string]string + nodeName string + fits bool + test string }{ { pod: &v1.Pod{}, @@ -1387,11 +1388,206 @@ func TestPodFitsSelector(t *testing.T) { fits: false, test: "Pod with an invalid value in Affinity term won't be scheduled onto the node", }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: algorithm.NodeFieldSelectorKeyNodeName, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_1", + fits: true, + test: "Pod with matchFields using In operator that matches the existing node", + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: algorithm.NodeFieldSelectorKeyNodeName, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_2", + fits: false, + test: "Pod with matchFields using In operator that does not match the existing node", + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: algorithm.NodeFieldSelectorKeyNodeName, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + }, + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_2", + labels: map[string]string{"foo": "bar"}, + fits: true, + test: "Pod with two terms: matchFields does not match, but matchExpressions matches", + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: algorithm.NodeFieldSelectorKeyNodeName, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_2", + labels: map[string]string{"foo": "bar"}, + fits: false, + test: "Pod with one term: matchFields does not match, but matchExpressions matches", + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: algorithm.NodeFieldSelectorKeyNodeName, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_1", + labels: map[string]string{"foo": "bar"}, + fits: true, + test: "Pod with one term: both matchFields and matchExpressions match", + }, + { + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{ + { + Key: algorithm.NodeFieldSelectorKeyNodeName, + Operator: v1.NodeSelectorOpIn, + Values: []string{"node_1"}, + }, + }, + }, + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"not-match-to-bar"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + nodeName: "node_2", + labels: map[string]string{"foo": "bar"}, + fits: false, + test: "Pod with two terms: both matchFields and matchExpressions do not match", + }, } expectedFailureReasons := []algorithm.PredicateFailureReason{ErrNodeSelectorNotMatch} for _, test := range tests { - node := v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: test.labels}} + node := v1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: test.nodeName, + Labels: test.labels, + }} nodeInfo := schedulercache.NewNodeInfo() nodeInfo.SetNode(&node) diff --git a/pkg/scheduler/algorithm/priorities/image_locality.go b/pkg/scheduler/algorithm/priorities/image_locality.go index 5df5d35308d..e7128331029 100644 --- a/pkg/scheduler/algorithm/priorities/image_locality.go +++ b/pkg/scheduler/algorithm/priorities/image_locality.go @@ -42,7 +42,7 @@ func ImageLocalityPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo *scheduler return schedulerapi.HostPriority{}, fmt.Errorf("node not found") } - sumSize := totalImageSize(node, pod.Spec.Containers) + sumSize := totalImageSize(nodeInfo, pod.Spec.Containers) return schedulerapi.HostPriority{ Host: node.Name, @@ -69,15 +69,10 @@ func calculateScoreFromSize(sumSize int64) int { } // totalImageSize returns the total image size of all the containers that are already on the node. -func totalImageSize(node *v1.Node, containers []v1.Container) int64 { - imageSizes := make(map[string]int64) - for _, image := range node.Status.Images { - for _, name := range image.Names { - imageSizes[name] = image.SizeBytes - } - } - +func totalImageSize(nodeInfo *schedulercache.NodeInfo, containers []v1.Container) int64 { var total int64 + + imageSizes := nodeInfo.Images() for _, container := range containers { if size, ok := imageSizes[container.Image]; ok { total += size diff --git a/pkg/scheduler/algorithm/priorities/resource_limits.go b/pkg/scheduler/algorithm/priorities/resource_limits.go index 6f440d8f1dd..5a02ff8dccf 100644 --- a/pkg/scheduler/algorithm/priorities/resource_limits.go +++ b/pkg/scheduler/algorithm/priorities/resource_limits.go @@ -20,7 +20,6 @@ import ( "fmt" "k8s.io/api/core/v1" - v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" "k8s.io/kubernetes/pkg/scheduler/schedulercache" @@ -93,31 +92,7 @@ func getResourceLimits(pod *v1.Pod) *schedulercache.Resource { // take max_resource(sum_pod, any_init_container) for _, container := range pod.Spec.InitContainers { - for rName, rQuantity := range container.Resources.Limits { - switch rName { - case v1.ResourceMemory: - if mem := rQuantity.Value(); mem > result.Memory { - result.Memory = mem - } - case v1.ResourceCPU: - if cpu := rQuantity.MilliValue(); cpu > result.MilliCPU { - result.MilliCPU = cpu - } - // keeping these resources though score computation in other priority functions and in this - // are only computed based on cpu and memory only. - case v1.ResourceEphemeralStorage: - if ephemeralStorage := rQuantity.Value(); ephemeralStorage > result.EphemeralStorage { - result.EphemeralStorage = ephemeralStorage - } - default: - if v1helper.IsScalarResourceName(rName) { - value := rQuantity.Value() - if value > result.ScalarResources[rName] { - result.SetScalar(rName, value) - } - } - } - } + result.SetMaxResource(container.Resources.Limits) } return result diff --git a/pkg/scheduler/algorithm/priorities/util/non_zero.go b/pkg/scheduler/algorithm/priorities/util/non_zero.go index bfbe86d6639..b671945f338 100644 --- a/pkg/scheduler/algorithm/priorities/util/non_zero.go +++ b/pkg/scheduler/algorithm/priorities/util/non_zero.go @@ -32,9 +32,8 @@ const DefaultMilliCPURequest int64 = 100 // 0.1 core // DefaultMemoryRequest defines default memory request size. const DefaultMemoryRequest int64 = 200 * 1024 * 1024 // 200 MB -// GetNonzeroRequests returns the default resource request if none is found or what is provided on the request -// TODO: Consider setting default as a fixed fraction of machine capacity (take "capacity v1.ResourceList" -// as an additional argument here) rather than using constants +// GetNonzeroRequests returns the default resource request if none is found or +// what is provided on the request. func GetNonzeroRequests(requests *v1.ResourceList) (int64, int64) { var outMilliCPU, outMemory int64 // Override if un-set, but not if explicitly set to zero diff --git a/pkg/scheduler/algorithm/types.go b/pkg/scheduler/algorithm/types.go index 1ebf50d5c9b..e0109d089a8 100644 --- a/pkg/scheduler/algorithm/types.go +++ b/pkg/scheduler/algorithm/types.go @@ -25,6 +25,12 @@ import ( "k8s.io/kubernetes/pkg/scheduler/schedulercache" ) +// NodeFieldSelectorKeys is a map that: the key are node field selector keys; the values are +// the functions to get the value of the node field. +var NodeFieldSelectorKeys = map[string]func(*v1.Node) string{ + NodeFieldSelectorKeyNodeName: func(n *v1.Node) string { return n.Name }, +} + // FitPredicate is a function that indicates if a pod fits into an existing node. // The failure information is given by the error. type FitPredicate func(pod *v1.Pod, meta PredicateMetadata, nodeInfo *schedulercache.NodeInfo) (bool, []PredicateFailureReason, error) diff --git a/pkg/scheduler/algorithm/well_known_labels.go b/pkg/scheduler/algorithm/well_known_labels.go index 887b5b862fc..3482649b416 100644 --- a/pkg/scheduler/algorithm/well_known_labels.go +++ b/pkg/scheduler/algorithm/well_known_labels.go @@ -16,6 +16,10 @@ limitations under the License. package algorithm +import ( + api "k8s.io/kubernetes/pkg/apis/core" +) + const ( // TaintNodeNotReady will be added when node is not ready // and feature-gate for TaintBasedEvictions flag is enabled, @@ -74,4 +78,8 @@ const ( // TaintNodeShutdown when node is shutdown in external cloud provider TaintNodeShutdown = "node.cloudprovider.kubernetes.io/shutdown" + + // NodeFieldSelectorKeyNodeName ('metadata.name') uses this as node field selector key + // when selecting node by node's name. + NodeFieldSelectorKeyNodeName = api.ObjectNameField ) diff --git a/pkg/scheduler/schedulercache/cache_test.go b/pkg/scheduler/schedulercache/cache_test.go index 69cd53a8b64..748691ffefb 100644 --- a/pkg/scheduler/schedulercache/cache_test.go +++ b/pkg/scheduler/schedulercache/cache_test.go @@ -108,6 +108,7 @@ func TestAssumePodScheduled(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[0]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 80).build(), + imageSizes: map[string]int64{}, }, }, { pods: []*v1.Pod{testPods[1], testPods[2]}, @@ -124,6 +125,7 @@ func TestAssumePodScheduled(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[1], testPods[2]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 80).add("TCP", "127.0.0.1", 8080).build(), + imageSizes: map[string]int64{}, }, }, { // test non-zero request pods: []*v1.Pod{testPods[3]}, @@ -140,6 +142,7 @@ func TestAssumePodScheduled(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[3]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 80).build(), + imageSizes: map[string]int64{}, }, }, { pods: []*v1.Pod{testPods[4]}, @@ -157,6 +160,7 @@ func TestAssumePodScheduled(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[4]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 80).build(), + imageSizes: map[string]int64{}, }, }, { pods: []*v1.Pod{testPods[4], testPods[5]}, @@ -174,6 +178,7 @@ func TestAssumePodScheduled(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[4], testPods[5]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 80).add("TCP", "127.0.0.1", 8080).build(), + imageSizes: map[string]int64{}, }, }, { pods: []*v1.Pod{testPods[6]}, @@ -190,6 +195,7 @@ func TestAssumePodScheduled(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[6]}, usedPorts: newHostPortInfoBuilder().build(), + imageSizes: map[string]int64{}, }, }, } @@ -269,6 +275,7 @@ func TestExpirePod(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[1]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 8080).build(), + imageSizes: map[string]int64{}, }, }} @@ -321,6 +328,7 @@ func TestAddPodWillConfirm(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[0]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 80).build(), + imageSizes: map[string]int64{}, }, }} @@ -417,6 +425,7 @@ func TestAddPodWillReplaceAssumed(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{updatedPod.DeepCopy()}, usedPorts: newHostPortInfoBuilder().add("TCP", "0.0.0.0", 90).build(), + imageSizes: map[string]int64{}, }, }, }} @@ -472,6 +481,7 @@ func TestAddPodAfterExpiration(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{basePod}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 80).build(), + imageSizes: map[string]int64{}, }, }} @@ -528,6 +538,7 @@ func TestUpdatePod(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[1]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 8080).build(), + imageSizes: map[string]int64{}, }, { requestedResource: &Resource{ MilliCPU: 100, @@ -541,6 +552,7 @@ func TestUpdatePod(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[0]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 80).build(), + imageSizes: map[string]int64{}, }}, }} @@ -599,6 +611,7 @@ func TestExpireAddUpdatePod(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[1]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 8080).build(), + imageSizes: map[string]int64{}, }, { requestedResource: &Resource{ MilliCPU: 100, @@ -612,6 +625,7 @@ func TestExpireAddUpdatePod(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{testPods[0]}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 80).build(), + imageSizes: map[string]int64{}, }}, }} @@ -689,6 +703,7 @@ func TestEphemeralStorageResource(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{podE}, usedPorts: schedutil.HostPortInfo{}, + imageSizes: map[string]int64{}, }, }, } @@ -735,6 +750,7 @@ func TestRemovePod(t *testing.T) { allocatableResource: &Resource{}, pods: []*v1.Pod{basePod}, usedPorts: newHostPortInfoBuilder().add("TCP", "127.0.0.1", 80).build(), + imageSizes: map[string]int64{}, }, }} @@ -992,6 +1008,8 @@ func TestNodeOperators(t *testing.T) { t.Errorf("Failed to find node %v in schedulercache.", node.Name) } + // Generations are globally unique. We check in our unit tests that they are incremented correctly. + expected.generation = got.generation if !reflect.DeepEqual(got, expected) { t.Errorf("Failed to add node into schedulercache:\n got: %+v \nexpected: %+v", got, expected) } @@ -1003,6 +1021,7 @@ func TestNodeOperators(t *testing.T) { if !found || len(cachedNodes) != 1 { t.Errorf("failed to dump cached nodes:\n got: %v \nexpected: %v", cachedNodes, cache.nodes) } + expected.generation = newNode.generation if !reflect.DeepEqual(newNode, expected) { t.Errorf("Failed to clone node:\n got: %+v, \n expected: %+v", newNode, expected) } @@ -1010,12 +1029,15 @@ func TestNodeOperators(t *testing.T) { // Case 3: update node attribute successfully. node.Status.Allocatable[v1.ResourceMemory] = mem50m expected.allocatableResource.Memory = mem50m.Value() - expected.generation++ cache.UpdateNode(nil, node) got, found = cache.nodes[node.Name] if !found { t.Errorf("Failed to find node %v in schedulercache after UpdateNode.", node.Name) } + if got.generation <= expected.generation { + t.Errorf("generation is not incremented. got: %v, expected: %v", got.generation, expected.generation) + } + expected.generation = got.generation if !reflect.DeepEqual(got, expected) { t.Errorf("Failed to update node in schedulercache:\n got: %+v \nexpected: %+v", got, expected) diff --git a/pkg/scheduler/schedulercache/node_info.go b/pkg/scheduler/schedulercache/node_info.go index 03907b334fe..2773437327f 100644 --- a/pkg/scheduler/schedulercache/node_info.go +++ b/pkg/scheduler/schedulercache/node_info.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "github.com/golang/glog" @@ -30,7 +31,10 @@ import ( "k8s.io/kubernetes/pkg/scheduler/util" ) -var emptyResource = Resource{} +var ( + emptyResource = Resource{} + generation int64 +) // NodeInfo is node level aggregated information. type NodeInfo struct { @@ -54,6 +58,10 @@ type NodeInfo struct { taints []v1.Taint taintsErr error + // This is a map from image name to image size, also for checking image existence on the node + // Cache it here to avoid rebuilding the map during scheduling, e.g., in image_locality.go + imageSizes map[string]int64 + // TransientInfo holds the information pertaining to a scheduling cycle. This will be destructed at the end of // scheduling cycle. // TODO: @ravig. Remove this once we have a clear approach for message passing across predicates and priorities. @@ -74,6 +82,14 @@ func initializeNodeTransientInfo() nodeTransientInfo { return nodeTransientInfo{AllocatableVolumesCount: 0, RequestedVolumes: 0} } +// nextGeneration: Let's make sure history never forgets the name... +// Increments the generation number monotonically ensuring that generation numbers never collide. +// Collision of the generation numbers would be particularly problematic if a node was deleted and +// added back with the same name. See issue#63262. +func nextGeneration() int64 { + return atomic.AddInt64(&generation, 1) +} + // nodeTransientInfo contains transient node information while scheduling. type nodeTransientInfo struct { // AllocatableVolumesCount contains number of volumes that could be attached to node. @@ -203,6 +219,37 @@ func (r *Resource) SetScalar(name v1.ResourceName, quantity int64) { r.ScalarResources[name] = quantity } +// SetMaxResource compares with ResourceList and takes max value for each Resource. +func (r *Resource) SetMaxResource(rl v1.ResourceList) { + if r == nil { + return + } + + for rName, rQuantity := range rl { + switch rName { + case v1.ResourceMemory: + if mem := rQuantity.Value(); mem > r.Memory { + r.Memory = mem + } + case v1.ResourceCPU: + if cpu := rQuantity.MilliValue(); cpu > r.MilliCPU { + r.MilliCPU = cpu + } + case v1.ResourceEphemeralStorage: + if ephemeralStorage := rQuantity.Value(); ephemeralStorage > r.EphemeralStorage { + r.EphemeralStorage = ephemeralStorage + } + default: + if v1helper.IsScalarResourceName(rName) { + value := rQuantity.Value() + if value > r.ScalarResources[rName] { + r.SetScalar(rName, value) + } + } + } + } +} + // NewNodeInfo returns a ready to use empty NodeInfo object. // If any pods are given in arguments, their information will be aggregated in // the returned object. @@ -212,8 +259,9 @@ func NewNodeInfo(pods ...*v1.Pod) *NodeInfo { nonzeroRequest: &Resource{}, allocatableResource: &Resource{}, TransientInfo: newTransientSchedulerInfo(), - generation: 0, + generation: nextGeneration(), usedPorts: make(util.HostPortInfo), + imageSizes: make(map[string]int64), } for _, pod := range pods { ni.AddPod(pod) @@ -245,6 +293,14 @@ func (n *NodeInfo) UsedPorts() util.HostPortInfo { return n.usedPorts } +// Images returns the image size information on this node. +func (n *NodeInfo) Images() map[string]int64 { + if n == nil { + return nil + } + return n.imageSizes +} + // PodsWithAffinity return all pods with (anti)affinity constraints on this node. func (n *NodeInfo) PodsWithAffinity() []*v1.Pod { if n == nil { @@ -320,6 +376,7 @@ func (n *NodeInfo) AllocatableResource() Resource { // SetAllocatableResource sets the allocatableResource information of given node. func (n *NodeInfo) SetAllocatableResource(allocatableResource *Resource) { n.allocatableResource = allocatableResource + n.generation = nextGeneration() } // Clone returns a copy of this node. @@ -335,6 +392,7 @@ func (n *NodeInfo) Clone() *NodeInfo { diskPressureCondition: n.diskPressureCondition, pidPressureCondition: n.pidPressureCondition, usedPorts: make(util.HostPortInfo), + imageSizes: n.imageSizes, generation: n.generation, } if len(n.pods) > 0 { @@ -391,7 +449,7 @@ func (n *NodeInfo) AddPod(pod *v1.Pod) { // Consume ports when pods added. n.updateUsedPorts(pod, true) - n.generation++ + n.generation = nextGeneration() } // RemovePod subtracts pod information from this NodeInfo. @@ -442,7 +500,7 @@ func (n *NodeInfo) RemovePod(pod *v1.Pod) error { // Release ports when remove Pods. n.updateUsedPorts(pod, false) - n.generation++ + n.generation = nextGeneration() return nil } @@ -478,6 +536,17 @@ func (n *NodeInfo) updateUsedPorts(pod *v1.Pod, add bool) { } } +func (n *NodeInfo) updateImageSizes() { + node := n.Node() + imageSizes := make(map[string]int64) + for _, image := range node.Status.Images { + for _, name := range image.Names { + imageSizes[name] = image.SizeBytes + } + } + n.imageSizes = imageSizes +} + // SetNode sets the overall node information. func (n *NodeInfo) SetNode(node *v1.Node) error { n.node = node @@ -499,7 +568,8 @@ func (n *NodeInfo) SetNode(node *v1.Node) error { } } n.TransientInfo = newTransientSchedulerInfo() - n.generation++ + n.updateImageSizes() + n.generation = nextGeneration() return nil } @@ -515,7 +585,7 @@ func (n *NodeInfo) RemoveNode(node *v1.Node) error { n.memoryPressureCondition = v1.ConditionUnknown n.diskPressureCondition = v1.ConditionUnknown n.pidPressureCondition = v1.ConditionUnknown - n.generation++ + n.generation = nextGeneration() return nil } diff --git a/pkg/scheduler/schedulercache/node_info_test.go b/pkg/scheduler/schedulercache/node_info_test.go index 40a9e5afbac..2444d6ce61f 100644 --- a/pkg/scheduler/schedulercache/node_info_test.go +++ b/pkg/scheduler/schedulercache/node_info_test.go @@ -185,6 +185,56 @@ func TestResourceAddScalar(t *testing.T) { } } +func TestSetMaxResource(t *testing.T) { + tests := []struct { + resource *Resource + resourceList v1.ResourceList + expected *Resource + }{ + { + resource: &Resource{}, + resourceList: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: *resource.NewScaledQuantity(4, -3), + v1.ResourceMemory: *resource.NewQuantity(2000, resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(5000, resource.BinarySI), + }, + expected: &Resource{ + MilliCPU: 4, + Memory: 2000, + EphemeralStorage: 5000, + }, + }, + { + resource: &Resource{ + MilliCPU: 4, + Memory: 4000, + EphemeralStorage: 5000, + ScalarResources: map[v1.ResourceName]int64{"scalar.test/scalar1": 1, "hugepages-test": 2}, + }, + resourceList: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: *resource.NewScaledQuantity(4, -3), + v1.ResourceMemory: *resource.NewQuantity(2000, resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(7000, resource.BinarySI), + "scalar.test/scalar1": *resource.NewQuantity(4, resource.DecimalSI), + v1.ResourceHugePagesPrefix + "test": *resource.NewQuantity(5, resource.BinarySI), + }, + expected: &Resource{ + MilliCPU: 4, + Memory: 4000, + EphemeralStorage: 7000, + ScalarResources: map[v1.ResourceName]int64{"scalar.test/scalar1": 4, "hugepages-test": 5}, + }, + }, + } + + for _, test := range tests { + test.resource.SetMaxResource(test.resourceList) + if !reflect.DeepEqual(test.expected, test.resource) { + t.Errorf("expected: %#v, got: %#v", test.expected, test.resource) + } + } +} + func TestNewNodeInfo(t *testing.T) { nodeName := "test-node" pods := []*v1.Pod{ @@ -216,6 +266,7 @@ func TestNewNodeInfo(t *testing.T) { {Protocol: "TCP", Port: 8080}: {}, }, }, + imageSizes: map[string]int64{}, pods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ @@ -274,7 +325,12 @@ func TestNewNodeInfo(t *testing.T) { }, } + gen := generation ni := NewNodeInfo(pods...) + if ni.generation <= gen { + t.Errorf("generation is not incremented. previous: %v, current: %v", gen, ni.generation) + } + expected.generation = ni.generation if !reflect.DeepEqual(expected, ni) { t.Errorf("expected: %#v, got: %#v", expected, ni) } @@ -299,6 +355,9 @@ func TestNodeInfoClone(t *testing.T) { {Protocol: "TCP", Port: 8080}: {}, }, }, + imageSizes: map[string]int64{ + "gcr.io/10": 10 * 1024 * 1024, + }, pods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ @@ -368,6 +427,9 @@ func TestNodeInfoClone(t *testing.T) { {Protocol: "TCP", Port: 8080}: {}, }, }, + imageSizes: map[string]int64{ + "gcr.io/10": 10 * 1024 * 1024, + }, pods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ @@ -525,6 +587,7 @@ func TestNodeInfoAddPod(t *testing.T) { {Protocol: "TCP", Port: 8080}: {}, }, }, + imageSizes: map[string]int64{}, pods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ @@ -584,10 +647,16 @@ func TestNodeInfoAddPod(t *testing.T) { } ni := fakeNodeInfo() + gen := ni.generation for _, pod := range pods { ni.AddPod(pod) + if ni.generation <= gen { + t.Errorf("generation is not incremented. Prev: %v, current: %v", gen, ni.generation) + } + gen = ni.generation } + expected.generation = ni.generation if !reflect.DeepEqual(expected, ni) { t.Errorf("expected: %#v, got: %#v", expected, ni) } @@ -637,6 +706,7 @@ func TestNodeInfoRemovePod(t *testing.T) { {Protocol: "TCP", Port: 8080}: {}, }, }, + imageSizes: map[string]int64{}, pods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ @@ -752,6 +822,7 @@ func TestNodeInfoRemovePod(t *testing.T) { {Protocol: "TCP", Port: 8080}: {}, }, }, + imageSizes: map[string]int64{}, pods: []*v1.Pod{ { ObjectMeta: metav1.ObjectMeta{ @@ -788,6 +859,7 @@ func TestNodeInfoRemovePod(t *testing.T) { for _, test := range tests { ni := fakeNodeInfo(pods...) + gen := ni.generation err := ni.RemovePod(test.pod) if err != nil { if test.errExpected { @@ -798,8 +870,13 @@ func TestNodeInfoRemovePod(t *testing.T) { } else { t.Errorf("expected no error, got: %v", err) } + } else { + if ni.generation <= gen { + t.Errorf("generation is not incremented. Prev: %v, current: %v", gen, ni.generation) + } } + test.expectedNodeInfo.generation = ni.generation if !reflect.DeepEqual(test.expectedNodeInfo, ni) { t.Errorf("expected: %#v, got: %#v", test.expectedNodeInfo, ni) } diff --git a/pkg/util/configz/configz.go b/pkg/util/configz/configz.go index cc2aa3a9fe3..99413d5e7fd 100644 --- a/pkg/util/configz/configz.go +++ b/pkg/util/configz/configz.go @@ -19,7 +19,6 @@ package configz import ( "encoding/json" "fmt" - "io" "net/http" "sync" ) @@ -74,7 +73,7 @@ func handle(w http.ResponseWriter, r *http.Request) { } } -func write(w io.Writer) error { +func write(w http.ResponseWriter) error { var b []byte var err error func() { @@ -85,6 +84,7 @@ func write(w io.Writer) error { if err != nil { return fmt.Errorf("error marshaling json: %v", err) } + w.Header().Set("Content-Type", "application/json") _, err = w.Write(b) return err } diff --git a/pkg/util/configz/configz_test.go b/pkg/util/configz/configz_test.go index 78f22593d16..850739df2e0 100644 --- a/pkg/util/configz/configz_test.go +++ b/pkg/util/configz/configz_test.go @@ -44,7 +44,7 @@ func TestConfigz(t *testing.T) { t.Fatalf("err: %v", err) } if string(body) != `{"testing":"blah"}` { - t.Fatalf("unexpected output: %v", err) + t.Fatalf("unexpected output: %s", body) } v.Set("bing") @@ -58,7 +58,7 @@ func TestConfigz(t *testing.T) { t.Fatalf("err: %v", err) } if string(body) != `{"testing":"bing"}` { - t.Fatalf("unexpected output: %v", err) + t.Fatalf("unexpected output: %s", body) } Delete("testing") @@ -72,6 +72,9 @@ func TestConfigz(t *testing.T) { t.Fatalf("err: %v", err) } if string(body) != `{}` { - t.Fatalf("unexpected output: %v", err) + t.Fatalf("unexpected output: %s", body) + } + if resp.Header.Get("Content-Type") != "application/json" { + t.Fatalf("unexpected Content-Type: %s", resp.Header.Get("Content-Type")) } } diff --git a/pkg/util/mount/exec_mount_test.go b/pkg/util/mount/exec_mount_test.go index 6adf44cbef6..c15a9c16d94 100644 --- a/pkg/util/mount/exec_mount_test.go +++ b/pkg/util/mount/exec_mount_test.go @@ -67,7 +67,7 @@ func TestBindMount(t *testing.T) { expectedArgs = []string{"-t", fsType, "-o", "bind", sourcePath, destinationPath} case 2: // mount -t fstype -o "remount,opts" source target - expectedArgs = []string{"-t", fsType, "-o", "remount," + strings.Join(mountOptions, ","), sourcePath, destinationPath} + expectedArgs = []string{"-t", fsType, "-o", "bind,remount," + strings.Join(mountOptions, ","), sourcePath, destinationPath} } if !reflect.DeepEqual(expectedArgs, args) { t.Errorf("expected arguments %q, got %q", strings.Join(expectedArgs, " "), strings.Join(args, " ")) diff --git a/pkg/util/mount/fake.go b/pkg/util/mount/fake.go index dd4e9e64042..ef79e731954 100644 --- a/pkg/util/mount/fake.go +++ b/pkg/util/mount/fake.go @@ -59,8 +59,10 @@ func (f *FakeMounter) Mount(source string, target string, fstype string, options f.mutex.Lock() defer f.mutex.Unlock() - // find 'bind' option + opts := []string{} + for _, option := range options { + // find 'bind' option if option == "bind" { // This is a bind-mount. In order to mimic linux behaviour, we must // use the original device of the bind-mount as the real source. @@ -79,7 +81,11 @@ func (f *FakeMounter) Mount(source string, target string, fstype string, options break } } - break + } + // find 'ro' option + if option == "ro" { + // reuse MountPoint.Opts field to mark mount as readonly + opts = append(opts, "ro") } } @@ -89,7 +95,7 @@ func (f *FakeMounter) Mount(source string, target string, fstype string, options absTarget = target } - f.MountPoints = append(f.MountPoints, MountPoint{Device: source, Path: absTarget, Type: fstype}) + f.MountPoints = append(f.MountPoints, MountPoint{Device: source, Path: absTarget, Type: fstype, Opts: opts}) glog.V(5).Infof("Fake mounter: mounted %s to %s", source, absTarget) f.Log = append(f.Log, FakeAction{Action: FakeActionMount, Target: absTarget, Source: source, FSType: fstype}) return nil diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index 42c1d6ba668..a7a08fa82f4 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -129,6 +129,8 @@ type Subpath struct { PodDir string // Name of the container ContainerName string + // True if the mount needs to be readonly + ReadOnly bool } // Exec executes command where mount utilities are. This can be either the host, @@ -277,7 +279,13 @@ func IsNotMountPoint(mounter Interface, file string) (bool, error) { // The list equals: // options - 'bind' + 'remount' (no duplicate) func isBind(options []string) (bool, []string) { - bindRemountOpts := []string{"remount"} + // Because we have an FD opened on the subpath bind mount, the "bind" option + // needs to be included, otherwise the mount target will error as busy if you + // remount as readonly. + // + // As a consequence, all read only bind mounts will no longer change the underlying + // volume mount to be read only. + bindRemountOpts := []string{"bind", "remount"} bind := false if len(options) != 0 { diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index cc8434dd258..05bf54dae8e 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -777,8 +777,13 @@ func doBindSubPath(mounter Interface, subpath Subpath, kubeletPid int) (hostPath mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd) // Do the bind mount + options := []string{"bind"} + if subpath.ReadOnly { + options = append(options, "ro") + } + glog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget) - if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil { + if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil { return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err) } diff --git a/pkg/util/mount/mount_linux_test.go b/pkg/util/mount/mount_linux_test.go index ffba769bc9e..7bec4e447ac 100644 --- a/pkg/util/mount/mount_linux_test.go +++ b/pkg/util/mount/mount_linux_test.go @@ -1009,6 +1009,7 @@ func getTestPaths(base string) (string, string) { func TestBindSubPath(t *testing.T) { defaultPerm := os.FileMode(0750) + readOnlyPerm := os.FileMode(0444) tests := []struct { name string @@ -1016,6 +1017,7 @@ func TestBindSubPath(t *testing.T) { // base. prepare func(base string) ([]string, string, string, error) expectError bool + readOnly bool }{ { name: "subpath-dir", @@ -1220,6 +1222,55 @@ func TestBindSubPath(t *testing.T) { }, expectError: false, }, + { + name: "subpath-dir-readonly", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "dir0") + return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm) + }, + expectError: false, + readOnly: true, + }, + { + name: "subpath-file-readonly", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "file0") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + return nil, volpath, subpath, ioutil.WriteFile(subpath, []byte{}, defaultPerm) + }, + expectError: false, + readOnly: true, + }, + { + name: "subpath-dir-and-volume-readonly", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "dir0") + if err := os.MkdirAll(subpath, defaultPerm); err != nil { + return nil, "", "", err + } + return nil, volpath, subpath, os.Chmod(subpath, readOnlyPerm) + }, + expectError: false, + readOnly: true, + }, + { + name: "subpath-file-and-vol-readonly", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "file0") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + return nil, volpath, subpath, ioutil.WriteFile(subpath, []byte{}, readOnlyPerm) + }, + expectError: false, + readOnly: true, + }, } for _, test := range tests { @@ -1244,6 +1295,7 @@ func TestBindSubPath(t *testing.T) { VolumePath: volPath, PodDir: filepath.Join(base, "pod0"), ContainerName: testContainer, + ReadOnly: test.readOnly, } _, subpathMount := getTestPaths(base) @@ -1269,12 +1321,39 @@ func TestBindSubPath(t *testing.T) { if err = validateFileExists(subpathMount); err != nil { t.Errorf("test %q failed: %v", test.name, err) } + if err = validateReadOnlyMount(test.readOnly, bindPathTarget, fm); err != nil { + t.Errorf("test %q failed: %v", test.name, err) + } } os.RemoveAll(base) } } +func validateReadOnlyMount(expectedReadOnly bool, bindPathTarget string, mounter *FakeMounter) error { + mps, err := mounter.List() + if err != nil { + return fmt.Errorf("fakeMounter.List() returned error: %v", err) + } + for _, mp := range mps { + if mp.Path == bindPathTarget { + foundReadOnly := false + for _, opts := range mp.Opts { + if opts == "ro" { + foundReadOnly = true + break + } + } + if expectedReadOnly != foundReadOnly { + return fmt.Errorf("expected readOnly %v, got %v for mount point %v", expectedReadOnly, foundReadOnly, bindPathTarget) + } else { + return nil + } + } + } + return fmt.Errorf("failed to find mountPoint %v", bindPathTarget) +} + func TestParseMountInfo(t *testing.T) { info := `62 0 253:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index f4f211b4dd8..615b89554ab 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -232,6 +232,10 @@ func (plugin *rbdPlugin) createMounterFromVolumeSpecAndPod(spec *volume.Spec, po if err != nil { return nil, err } + ams, err := getVolumeAccessModes(spec) + if err != nil { + return nil, err + } secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace) if err != nil { @@ -255,12 +259,13 @@ func (plugin *rbdPlugin) createMounterFromVolumeSpecAndPod(spec *volume.Spec, po } return &rbdMounter{ - rbd: newRBD("", spec.Name(), img, pool, ro, plugin, &RBDUtil{}), - Mon: mon, - Id: id, - Keyring: keyring, - Secret: secret, - fsType: fstype, + rbd: newRBD("", spec.Name(), img, pool, ro, plugin, &RBDUtil{}), + Mon: mon, + Id: id, + Keyring: keyring, + Secret: secret, + fsType: fstype, + accessModes: ams, }, nil } @@ -319,6 +324,10 @@ func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, if err != nil { return nil, err } + ams, err := getVolumeAccessModes(spec) + if err != nil { + return nil, err + } return &rbdMounter{ rbd: newRBD(podUID, spec.Name(), img, pool, ro, plugin, manager), @@ -328,6 +337,7 @@ func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, Secret: secret, fsType: fstype, mountOptions: volutil.MountOptionFromSpec(spec), + accessModes: ams, }, nil } @@ -764,6 +774,7 @@ type rbdMounter struct { mountOptions []string imageFormat string imageFeatures []string + accessModes []v1.PersistentVolumeAccessMode } var _ volume.Mounter = &rbdMounter{} @@ -936,14 +947,20 @@ func (rbd *rbdDiskUnmapper) TearDownDevice(mapPath, _ string) error { blkUtil := volumepathhandler.NewBlockVolumePathHandler() loop, err := volumepathhandler.BlockVolumePathHandler.GetLoopDevice(blkUtil, device) if err != nil { - return fmt.Errorf("rbd: failed to get loopback for device: %v, err: %v", device, err) + if err.Error() != volumepathhandler.ErrDeviceNotFound { + return fmt.Errorf("rbd: failed to get loopback for device: %v, err: %v", device, err) + } + glog.Warning("rbd: loopback for device: % not found", device) + } else { + if len(loop) != 0 { + // Remove loop device before detaching volume since volume detach operation gets busy if volume is opened by loopback. + err = volumepathhandler.BlockVolumePathHandler.RemoveLoopDevice(blkUtil, loop) + if err != nil { + return fmt.Errorf("rbd: failed to remove loopback :%v, err: %v", loop, err) + } + glog.V(4).Infof("rbd: successfully removed loop device: %s", loop) + } } - // Remove loop device before detaching volume since volume detach operation gets busy if volume is opened by loopback. - err = volumepathhandler.BlockVolumePathHandler.RemoveLoopDevice(blkUtil, loop) - if err != nil { - return fmt.Errorf("rbd: failed to remove loopback :%v, err: %v", loop, err) - } - glog.V(4).Infof("rbd: successfully removed loop device: %s", loop) err = rbd.manager.DetachBlockDisk(*rbd, mapPath) if err != nil { @@ -1039,6 +1056,19 @@ func getVolumeSourceReadOnly(spec *volume.Spec) (bool, error) { return false, fmt.Errorf("Spec does not reference a RBD volume type") } +func getVolumeAccessModes(spec *volume.Spec) ([]v1.PersistentVolumeAccessMode, error) { + // Only PersistentVolumeSpec has AccessModes + if spec.PersistentVolume != nil { + if spec.PersistentVolume.Spec.RBD != nil { + return spec.PersistentVolume.Spec.AccessModes, nil + } else { + return nil, fmt.Errorf("Spec does not reference a RBD volume type") + } + } + + return nil, nil +} + func parsePodSecret(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (string, error) { secret, err := volutil.GetSecretForPod(pod, secretName, kubeClient) if err != nil { diff --git a/pkg/volume/rbd/rbd_test.go b/pkg/volume/rbd/rbd_test.go index 73a5aea9d98..4ab32f6b5cf 100644 --- a/pkg/volume/rbd/rbd_test.go +++ b/pkg/volume/rbd/rbd_test.go @@ -358,6 +358,7 @@ func TestPlugin(t *testing.T) { FSType: "ext4", }, }, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}, }, }, false), root: tmpDir, diff --git a/pkg/volume/rbd/rbd_util.go b/pkg/volume/rbd/rbd_util.go index c64be23e56a..1860886777c 100644 --- a/pkg/volume/rbd/rbd_util.go +++ b/pkg/volume/rbd/rbd_util.go @@ -390,12 +390,22 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) (string, error) { Factor: rbdImageWatcherFactor, Steps: rbdImageWatcherSteps, } + needValidUsed := true + // If accessModes contain ReadOnlyMany, we don't need check rbd status of being used. + if b.accessModes != nil { + for _, v := range b.accessModes { + if v != v1.ReadWriteOnce { + needValidUsed = false + break + } + } + } err := wait.ExponentialBackoff(backoff, func() (bool, error) { used, rbdOutput, err := util.rbdStatus(&b) if err != nil { return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) } - return !used, nil + return !needValidUsed || !used, nil }) // Return error if rbd image has not become available for the specified timeout. if err == wait.ErrWaitTimeout { diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index b897bdb9ba0..a9312fe7fcf 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -1068,11 +1068,18 @@ func (og *operationGenerator) GenerateUnmapDeviceFunc( glog.V(4).Infof("UnmapDevice: deviceToDetach.DevicePath: %v", deviceToDetach.DevicePath) loopPath, err := og.blkUtil.GetLoopDevice(deviceToDetach.DevicePath) if err != nil { - glog.Warningf(deviceToDetach.GenerateMsgDetailed("UnmapDevice: Couldn't find loopback device which takes file descriptor lock", fmt.Sprintf("device path: %q", deviceToDetach.DevicePath))) + if err.Error() == volumepathhandler.ErrDeviceNotFound { + glog.Warningf(deviceToDetach.GenerateMsgDetailed("UnmapDevice: Couldn't find loopback device which takes file descriptor lock", fmt.Sprintf("device path: %q", deviceToDetach.DevicePath))) + } else { + errInfo := "UnmapDevice.GetLoopDevice failed to get loopback device, " + fmt.Sprintf("device path: %q", deviceToDetach.DevicePath) + return deviceToDetach.GenerateError(errInfo, err) + } } else { - err = og.blkUtil.RemoveLoopDevice(loopPath) - if err != nil { - return deviceToDetach.GenerateError("UnmapDevice.AttachFileDevice failed", err) + if len(loopPath) != 0 { + err = og.blkUtil.RemoveLoopDevice(loopPath) + if err != nil { + return deviceToDetach.GenerateError("UnmapDevice.RemoveLoopDevice failed", err) + } } } diff --git a/plugin/pkg/admission/eventratelimit/BUILD b/plugin/pkg/admission/eventratelimit/BUILD index edb86ee59b1..9dfc093b7b4 100644 --- a/plugin/pkg/admission/eventratelimit/BUILD +++ b/plugin/pkg/admission/eventratelimit/BUILD @@ -45,7 +45,6 @@ go_library( "//plugin/pkg/admission/eventratelimit/apis/eventratelimit/validation:go_default_library", "//vendor/github.com/hashicorp/golang-lru:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", diff --git a/plugin/pkg/admission/eventratelimit/apis/eventratelimit/install/BUILD b/plugin/pkg/admission/eventratelimit/apis/eventratelimit/install/BUILD index 6e2a32ff65b..36d008a6497 100644 --- a/plugin/pkg/admission/eventratelimit/apis/eventratelimit/install/BUILD +++ b/plugin/pkg/admission/eventratelimit/apis/eventratelimit/install/BUILD @@ -12,9 +12,8 @@ go_library( deps = [ "//plugin/pkg/admission/eventratelimit/apis/eventratelimit:go_default_library", "//plugin/pkg/admission/eventratelimit/apis/eventratelimit/v1alpha1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/plugin/pkg/admission/eventratelimit/apis/eventratelimit/install/install.go b/plugin/pkg/admission/eventratelimit/apis/eventratelimit/install/install.go index 6204bae968e..413cd56b8da 100644 --- a/plugin/pkg/admission/eventratelimit/apis/eventratelimit/install/install.go +++ b/plugin/pkg/admission/eventratelimit/apis/eventratelimit/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" internalapi "k8s.io/kubernetes/plugin/pkg/admission/eventratelimit/apis/eventratelimit" versionedapi "k8s.io/kubernetes/plugin/pkg/admission/eventratelimit/apis/eventratelimit/v1alpha1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: internalapi.GroupName, - VersionPreferenceOrder: []string{versionedapi.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: internalapi.AddToScheme, - }, - announced.VersionToSchemeFunc{ - versionedapi.SchemeGroupVersion.Version: versionedapi.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(internalapi.AddToScheme(scheme)) + utilruntime.Must(versionedapi.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(versionedapi.SchemeGroupVersion)) } diff --git a/plugin/pkg/admission/eventratelimit/config.go b/plugin/pkg/admission/eventratelimit/config.go index b486b7723ee..34610391fac 100644 --- a/plugin/pkg/admission/eventratelimit/config.go +++ b/plugin/pkg/admission/eventratelimit/config.go @@ -21,7 +21,6 @@ import ( "io" "io/ioutil" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" eventratelimitapi "k8s.io/kubernetes/plugin/pkg/admission/eventratelimit/apis/eventratelimit" @@ -30,13 +29,12 @@ import ( ) var ( - registry = registered.NewAPIRegistrationManager() - scheme = runtime.NewScheme() - codecs = serializer.NewCodecFactory(scheme) + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme) ) func init() { - install.Install(registry, scheme) + install.Install(scheme) } // LoadConfiguration loads the provided configuration. diff --git a/plugin/pkg/admission/gc/gc_admission_test.go b/plugin/pkg/admission/gc/gc_admission_test.go index ccc1dd977fa..d5eae446541 100644 --- a/plugin/pkg/admission/gc/gc_admission_test.go +++ b/plugin/pkg/admission/gc/gc_admission_test.go @@ -88,7 +88,7 @@ func newGCPermissionsEnforcement() (*gcPermissionsEnforcement, error) { } genericPluginInitializer := initializer.New(nil, nil, fakeAuthorizer{}, nil) - pluginInitializer := kubeadmission.NewPluginInitializer(nil, nil, nil, testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme), nil) + pluginInitializer := kubeadmission.NewPluginInitializer(nil, nil, nil, testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Scheme), nil) initializersChain := admission.PluginInitializers{} initializersChain = append(initializersChain, genericPluginInitializer) initializersChain = append(initializersChain, pluginInitializer) diff --git a/plugin/pkg/admission/imagepolicy/admission.go b/plugin/pkg/admission/imagepolicy/admission.go index 3a3701cf4e1..76b53c8bb57 100644 --- a/plugin/pkg/admission/imagepolicy/admission.go +++ b/plugin/pkg/admission/imagepolicy/admission.go @@ -239,7 +239,7 @@ func NewImagePolicyWebhook(configFile io.Reader) (*Plugin, error) { return nil, err } - gw, err := webhook.NewGenericWebhook(legacyscheme.Registry, legacyscheme.Codecs, whConfig.KubeConfigFile, groupVersions, whConfig.RetryBackoff) + gw, err := webhook.NewGenericWebhook(legacyscheme.Scheme, legacyscheme.Codecs, whConfig.KubeConfigFile, groupVersions, whConfig.RetryBackoff) if err != nil { return nil, err } diff --git a/plugin/pkg/admission/noderestriction/BUILD b/plugin/pkg/admission/noderestriction/BUILD index 4e83ee51578..25fe1995350 100644 --- a/plugin/pkg/admission/noderestriction/BUILD +++ b/plugin/pkg/admission/noderestriction/BUILD @@ -16,13 +16,12 @@ go_library( "//pkg/apis/core:go_default_library", "//pkg/apis/policy:go_default_library", "//pkg/auth/nodeidentifier:go_default_library", - "//pkg/client/clientset_generated/internalclientset:go_default_library", - "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", + "//pkg/client/informers/informers_generated/internalversion:go_default_library", + "//pkg/client/listers/core/internalversion:go_default_library", "//pkg/features:go_default_library", "//pkg/kubeapiserver/admission:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", @@ -38,14 +37,14 @@ go_test( "//pkg/apis/core:go_default_library", "//pkg/apis/policy:go_default_library", "//pkg/auth/nodeidentifier:go_default_library", - "//pkg/client/clientset_generated/internalclientset/fake:go_default_library", - "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", + "//pkg/client/listers/core/internalversion:go_default_library", "//pkg/features:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/client-go/tools/cache:go_default_library", ], ) diff --git a/plugin/pkg/admission/noderestriction/admission.go b/plugin/pkg/admission/noderestriction/admission.go index 333506e6657..3b352d11734 100644 --- a/plugin/pkg/admission/noderestriction/admission.go +++ b/plugin/pkg/admission/noderestriction/admission.go @@ -22,7 +22,6 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apiserver/pkg/admission" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -31,8 +30,8 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/policy" "k8s.io/kubernetes/pkg/auth/nodeidentifier" - "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - coreinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" + informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" + internalversion "k8s.io/kubernetes/pkg/client/listers/core/internalversion" "k8s.io/kubernetes/pkg/features" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" ) @@ -62,18 +61,18 @@ func NewPlugin(nodeIdentifier nodeidentifier.NodeIdentifier) *nodePlugin { type nodePlugin struct { *admission.Handler nodeIdentifier nodeidentifier.NodeIdentifier - podsGetter coreinternalversion.PodsGetter + podsGetter internalversion.PodLister // allows overriding for testing features utilfeature.FeatureGate } var ( _ = admission.Interface(&nodePlugin{}) - _ = kubeapiserveradmission.WantsInternalKubeClientSet(&nodePlugin{}) + _ = kubeapiserveradmission.WantsInternalKubeInformerFactory(&nodePlugin{}) ) -func (p *nodePlugin) SetInternalKubeClientSet(f internalclientset.Interface) { - p.podsGetter = f.Core() +func (p *nodePlugin) SetInternalKubeInformerFactory(f informers.SharedInformerFactory) { + p.podsGetter = f.Core().InternalVersion().Pods().Lister() } func (p *nodePlugin) ValidateInitialization() error { @@ -183,14 +182,10 @@ func (c *nodePlugin) admitPod(nodeName string, a admission.Attributes) error { return nil case admission.Delete: - // get the existing pod from the server cache - existingPod, err := c.podsGetter.Pods(a.GetNamespace()).Get(a.GetName(), v1.GetOptions{ResourceVersion: "0"}) + // get the existing pod + existingPod, err := c.podsGetter.Pods(a.GetNamespace()).Get(a.GetName()) if errors.IsNotFound(err) { - // wasn't found in the server cache, do a live lookup before forbidding - existingPod, err = c.podsGetter.Pods(a.GetNamespace()).Get(a.GetName(), v1.GetOptions{}) - if errors.IsNotFound(err) { - return err - } + return err } if err != nil { return admission.NewForbidden(a, err) @@ -241,14 +236,10 @@ func (c *nodePlugin) admitPodEviction(nodeName string, a admission.Attributes) e } podName = eviction.Name } - // get the existing pod from the server cache - existingPod, err := c.podsGetter.Pods(a.GetNamespace()).Get(podName, v1.GetOptions{ResourceVersion: "0"}) + // get the existing pod + existingPod, err := c.podsGetter.Pods(a.GetNamespace()).Get(podName) if errors.IsNotFound(err) { - // wasn't found in the server cache, do a live lookup before forbidding - existingPod, err = c.podsGetter.Pods(a.GetNamespace()).Get(podName, v1.GetOptions{}) - if errors.IsNotFound(err) { - return err - } + return err } if err != nil { return admission.NewForbidden(a, err) @@ -376,7 +367,7 @@ func (c *nodePlugin) admitServiceAccount(nodeName string, a admission.Attributes if ref.UID == "" { return admission.NewForbidden(a, fmt.Errorf("node requested token with a pod binding without a uid")) } - pod, err := c.podsGetter.Pods(a.GetNamespace()).Get(ref.Name, v1.GetOptions{}) + pod, err := c.podsGetter.Pods(a.GetNamespace()).Get(ref.Name) if errors.IsNotFound(err) { return err } diff --git a/plugin/pkg/admission/noderestriction/admission_test.go b/plugin/pkg/admission/noderestriction/admission_test.go index 93dec714618..68465357945 100644 --- a/plugin/pkg/admission/noderestriction/admission_test.go +++ b/plugin/pkg/admission/noderestriction/admission_test.go @@ -25,12 +25,12 @@ import ( "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authentication/user" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/tools/cache" authenticationapi "k8s.io/kubernetes/pkg/apis/authentication" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/policy" "k8s.io/kubernetes/pkg/auth/nodeidentifier" - "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" - coreinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" + "k8s.io/kubernetes/pkg/client/listers/core/internalversion" "k8s.io/kubernetes/pkg/features" ) @@ -63,6 +63,7 @@ func makeTestPod(namespace, name, node string, mirror bool) *api.Pod { func makeTestPodEviction(name string) *policy.Eviction { eviction := &policy.Eviction{} eviction.Name = name + eviction.Namespace = "ns" return eviction } @@ -91,9 +92,19 @@ func Test_nodePlugin_Admit(t *testing.T) { mynodeObjMeta = metav1.ObjectMeta{Name: "mynode"} mynodeObj = &api.Node{ObjectMeta: mynodeObjMeta} mynodeObjConfigA = &api.Node{ObjectMeta: mynodeObjMeta, Spec: api.NodeSpec{ConfigSource: &api.NodeConfigSource{ - ConfigMapRef: &api.ObjectReference{Name: "foo", Namespace: "bar", UID: "fooUID"}}}} + ConfigMap: &api.ConfigMapNodeConfigSource{ + Name: "foo", + Namespace: "bar", + UID: "fooUID", + KubeletConfigKey: "kubelet", + }}}} mynodeObjConfigB = &api.Node{ObjectMeta: mynodeObjMeta, Spec: api.NodeSpec{ConfigSource: &api.NodeConfigSource{ - ConfigMapRef: &api.ObjectReference{Name: "qux", Namespace: "bar", UID: "quxUID"}}}} + ConfigMap: &api.ConfigMapNodeConfigSource{ + Name: "qux", + Namespace: "bar", + UID: "quxUID", + KubeletConfigKey: "kubelet", + }}}} othernodeObj = &api.Node{ObjectMeta: metav1.ObjectMeta{Name: "othernode"}} mymirrorpod = makeTestPod("ns", "mymirrorpod", "mynode", true) @@ -125,10 +136,20 @@ func Test_nodePlugin_Admit(t *testing.T) { svcacctResource = api.Resource("serviceaccounts").WithVersion("v1") tokenrequestKind = api.Kind("TokenRequest").WithVersion("v1") - noExistingPods = fake.NewSimpleClientset().Core() - existingPods = fake.NewSimpleClientset(mymirrorpod, othermirrorpod, unboundmirrorpod, mypod, otherpod, unboundpod).Core() + noExistingPodsIndex = cache.NewIndexer(cache.MetaNamespaceKeyFunc, nil) + noExistingPods = internalversion.NewPodLister(noExistingPodsIndex) + + existingPodsIndex = cache.NewIndexer(cache.MetaNamespaceKeyFunc, nil) + existingPods = internalversion.NewPodLister(existingPodsIndex) ) + existingPodsIndex.Add(mymirrorpod) + existingPodsIndex.Add(othermirrorpod) + existingPodsIndex.Add(unboundmirrorpod) + existingPodsIndex.Add(mypod) + existingPodsIndex.Add(otherpod) + existingPodsIndex.Add(unboundpod) + sapod := makeTestPod("ns", "mysapod", "mynode", true) sapod.Spec.ServiceAccountName = "foo" @@ -143,7 +164,7 @@ func Test_nodePlugin_Admit(t *testing.T) { tests := []struct { name string - podsGetter coreinternalversion.PodsGetter + podsGetter internalversion.PodLister attributes admission.Attributes features utilfeature.FeatureGate err string @@ -446,7 +467,7 @@ func Test_nodePlugin_Admit(t *testing.T) { err: "forbidden: unexpected operation", }, { - name: "forbid create of eviction for normal pod bound to another", + name: "forbid create of unnamed eviction for normal pod bound to another", podsGetter: existingPods, attributes: admission.NewAttributesRecord(unnamedEviction, nil, evictionKind, otherpod.Namespace, otherpod.Name, podResource, "eviction", admission.Create, mynode), err: "spec.nodeName set to itself", diff --git a/plugin/pkg/admission/podtolerationrestriction/BUILD b/plugin/pkg/admission/podtolerationrestriction/BUILD index abcb37e9c00..6421a156260 100644 --- a/plugin/pkg/admission/podtolerationrestriction/BUILD +++ b/plugin/pkg/admission/podtolerationrestriction/BUILD @@ -51,7 +51,6 @@ go_library( "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", diff --git a/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/install/BUILD b/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/install/BUILD index 6b1210fe035..97dabce249c 100644 --- a/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/install/BUILD +++ b/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/install/BUILD @@ -12,9 +12,8 @@ go_library( deps = [ "//plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction:go_default_library", "//plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/v1alpha1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/install/install.go b/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/install/install.go index e81da6e64f7..3bd5deb1582 100644 --- a/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/install/install.go +++ b/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" internalapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction" versionedapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/v1alpha1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: internalapi.GroupName, - VersionPreferenceOrder: []string{versionedapi.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: internalapi.AddToScheme, - }, - announced.VersionToSchemeFunc{ - versionedapi.SchemeGroupVersion.Version: versionedapi.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(internalapi.AddToScheme(scheme)) + utilruntime.Must(versionedapi.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(versionedapi.SchemeGroupVersion)) } diff --git a/plugin/pkg/admission/podtolerationrestriction/config.go b/plugin/pkg/admission/podtolerationrestriction/config.go index 8002cf6457f..d5d954a50b4 100644 --- a/plugin/pkg/admission/podtolerationrestriction/config.go +++ b/plugin/pkg/admission/podtolerationrestriction/config.go @@ -21,7 +21,6 @@ import ( "io" "io/ioutil" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" internalapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction" @@ -31,13 +30,12 @@ import ( ) var ( - registry = registered.NewAPIRegistrationManager() - scheme = runtime.NewScheme() - codecs = serializer.NewCodecFactory(scheme) + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme) ) func init() { - install.Install(registry, scheme) + install.Install(scheme) } // LoadConfiguration loads the provided configuration. diff --git a/plugin/pkg/admission/resourcequota/BUILD b/plugin/pkg/admission/resourcequota/BUILD index 4d2651cdc79..fea03de5446 100644 --- a/plugin/pkg/admission/resourcequota/BUILD +++ b/plugin/pkg/admission/resourcequota/BUILD @@ -33,7 +33,6 @@ go_library( "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/hashicorp/golang-lru:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/plugin/pkg/admission/resourcequota/apis/resourcequota/install/BUILD b/plugin/pkg/admission/resourcequota/apis/resourcequota/install/BUILD index 61d6cc71468..dfc40ecfbae 100644 --- a/plugin/pkg/admission/resourcequota/apis/resourcequota/install/BUILD +++ b/plugin/pkg/admission/resourcequota/apis/resourcequota/install/BUILD @@ -12,9 +12,8 @@ go_library( deps = [ "//plugin/pkg/admission/resourcequota/apis/resourcequota:go_default_library", "//plugin/pkg/admission/resourcequota/apis/resourcequota/v1alpha1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/plugin/pkg/admission/resourcequota/apis/resourcequota/install/install.go b/plugin/pkg/admission/resourcequota/apis/resourcequota/install/install.go index 3958e4604e7..87c9d6f9a7c 100644 --- a/plugin/pkg/admission/resourcequota/apis/resourcequota/install/install.go +++ b/plugin/pkg/admission/resourcequota/apis/resourcequota/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" resourcequotaapi "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota" resourcequotav1alpha1 "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota/v1alpha1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: resourcequotaapi.GroupName, - VersionPreferenceOrder: []string{resourcequotav1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: resourcequotaapi.AddToScheme, - }, - announced.VersionToSchemeFunc{ - resourcequotav1alpha1.SchemeGroupVersion.Version: resourcequotav1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(resourcequotaapi.AddToScheme(scheme)) + utilruntime.Must(resourcequotav1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(resourcequotav1alpha1.SchemeGroupVersion)) } diff --git a/plugin/pkg/admission/resourcequota/config.go b/plugin/pkg/admission/resourcequota/config.go index a7185be91bf..8bb5db13c80 100644 --- a/plugin/pkg/admission/resourcequota/config.go +++ b/plugin/pkg/admission/resourcequota/config.go @@ -21,7 +21,6 @@ import ( "io" "io/ioutil" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" resourcequotaapi "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota" @@ -30,13 +29,12 @@ import ( ) var ( - registry = registered.NewAPIRegistrationManager() - scheme = runtime.NewScheme() - codecs = serializer.NewCodecFactory(scheme) + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme) ) func init() { - install.Install(registry, scheme) + install.Install(scheme) } // LoadConfiguration loads the provided configuration. diff --git a/plugin/pkg/auth/authorizer/node/graph.go b/plugin/pkg/auth/authorizer/node/graph.go index bf545790382..5781b56b01d 100644 --- a/plugin/pkg/auth/authorizer/node/graph.go +++ b/plugin/pkg/auth/authorizer/node/graph.go @@ -408,7 +408,7 @@ func (g *Graph) DeleteVolumeAttachment(name string) { g.deleteVertex_locked(vaVertexType, "", name) } -// SetNodeConfigMap sets up edges for the Node.Spec.ConfigSource.ConfigMapRef relationship: +// SetNodeConfigMap sets up edges for the Node.Spec.ConfigSource.ConfigMap relationship: // // configmap -> node func (g *Graph) SetNodeConfigMap(nodeName, configMapName, configMapNamespace string) { diff --git a/plugin/pkg/auth/authorizer/node/graph_populator.go b/plugin/pkg/auth/authorizer/node/graph_populator.go index bf8e0e324f0..3d8f9baa6a7 100644 --- a/plugin/pkg/auth/authorizer/node/graph_populator.go +++ b/plugin/pkg/auth/authorizer/node/graph_populator.go @@ -84,19 +84,19 @@ func (g *graphPopulator) updateNode(oldObj, obj interface{}) { oldNode = oldObj.(*api.Node) } - // we only set up rules for ConfigMapRef today, because that is the only reference type + // we only set up rules for ConfigMap today, because that is the only reference type var name, namespace string - if source := node.Spec.ConfigSource; source != nil && source.ConfigMapRef != nil { - name = source.ConfigMapRef.Name - namespace = source.ConfigMapRef.Namespace + if source := node.Spec.ConfigSource; source != nil && source.ConfigMap != nil { + name = source.ConfigMap.Name + namespace = source.ConfigMap.Namespace } var oldName, oldNamespace string if oldNode != nil { - if oldSource := oldNode.Spec.ConfigSource; oldSource != nil && oldSource.ConfigMapRef != nil { - oldName = oldSource.ConfigMapRef.Name - oldNamespace = oldSource.ConfigMapRef.Namespace + if oldSource := oldNode.Spec.ConfigSource; oldSource != nil && oldSource.ConfigMap != nil { + oldName = oldSource.ConfigMap.Name + oldNamespace = oldSource.ConfigMap.Namespace } } diff --git a/plugin/pkg/auth/authorizer/node/node_authorizer_test.go b/plugin/pkg/auth/authorizer/node/node_authorizer_test.go index 3e27165d27b..ecc7bc3fe3d 100644 --- a/plugin/pkg/auth/authorizer/node/node_authorizer_test.go +++ b/plugin/pkg/auth/authorizer/node/node_authorizer_test.go @@ -753,10 +753,11 @@ func generate(opts sampleDataOpts) ([]*api.Node, []*api.Pod, []*api.PersistentVo ObjectMeta: metav1.ObjectMeta{Name: nodeName}, Spec: api.NodeSpec{ ConfigSource: &api.NodeConfigSource{ - ConfigMapRef: &api.ObjectReference{ - Name: name, - Namespace: "ns0", - UID: types.UID(fmt.Sprintf("ns0-%s", name)), + ConfigMap: &api.ConfigMapNodeConfigSource{ + Name: name, + Namespace: "ns0", + UID: types.UID(fmt.Sprintf("ns0-%s", name)), + KubeletConfigKey: "kubelet", }, }, }, diff --git a/staging/BUILD b/staging/BUILD index ee79c600b82..15b7d66c27d 100644 --- a/staging/BUILD +++ b/staging/BUILD @@ -19,7 +19,6 @@ filegroup( "//staging/src/k8s.io/apimachinery/pkg/api/resource:all-srcs", "//staging/src/k8s.io/apimachinery/pkg/api/testing:all-srcs", "//staging/src/k8s.io/apimachinery/pkg/api/validation:all-srcs", - "//staging/src/k8s.io/apimachinery/pkg/apimachinery:all-srcs", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/fuzzer:all-srcs", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/internalversion:all-srcs", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:all-srcs", @@ -141,6 +140,7 @@ filegroup( "//staging/src/k8s.io/client-go/pkg/version:all-srcs", "//staging/src/k8s.io/client-go/plugin/pkg/client/auth:all-srcs", "//staging/src/k8s.io/client-go/rest:all-srcs", + "//staging/src/k8s.io/client-go/restmapper:all-srcs", "//staging/src/k8s.io/client-go/scale:all-srcs", "//staging/src/k8s.io/client-go/testing:all-srcs", "//staging/src/k8s.io/client-go/third_party/forked/golang/template:all-srcs", diff --git a/staging/src/k8s.io/api/Godeps/Godeps.json b/staging/src/k8s.io/api/Godeps/Godeps.json index 0c441ed7cfc..bb2cd68b30d 100644 --- a/staging/src/k8s.io/api/Godeps/Godeps.json +++ b/staging/src/k8s.io/api/Godeps/Godeps.json @@ -110,14 +110,6 @@ "ImportPath": "k8s.io/apimachinery/pkg/api/testing/fuzzer", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apimachinery/pkg/apis/meta/fuzzer", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/staging/src/k8s.io/api/core/v1/generated.pb.go b/staging/src/k8s.io/api/core/v1/generated.pb.go index 5406c1283c8..b126dd07438 100644 --- a/staging/src/k8s.io/api/core/v1/generated.pb.go +++ b/staging/src/k8s.io/api/core/v1/generated.pb.go @@ -46,6 +46,7 @@ limitations under the License. ConfigMapEnvSource ConfigMapKeySelector ConfigMapList + ConfigMapNodeConfigSource ConfigMapProjection ConfigMapVolumeSource Container @@ -333,708 +334,714 @@ func (m *ConfigMapList) Reset() { *m = ConfigMapList{} } func (*ConfigMapList) ProtoMessage() {} func (*ConfigMapList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{20} } +func (m *ConfigMapNodeConfigSource) Reset() { *m = ConfigMapNodeConfigSource{} } +func (*ConfigMapNodeConfigSource) ProtoMessage() {} +func (*ConfigMapNodeConfigSource) Descriptor() ([]byte, []int) { + return fileDescriptorGenerated, []int{21} +} + func (m *ConfigMapProjection) Reset() { *m = ConfigMapProjection{} } func (*ConfigMapProjection) ProtoMessage() {} -func (*ConfigMapProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{21} } +func (*ConfigMapProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{22} } func (m *ConfigMapVolumeSource) Reset() { *m = ConfigMapVolumeSource{} } func (*ConfigMapVolumeSource) ProtoMessage() {} -func (*ConfigMapVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{22} } +func (*ConfigMapVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{23} } func (m *Container) Reset() { *m = Container{} } func (*Container) ProtoMessage() {} -func (*Container) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{23} } +func (*Container) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{24} } func (m *ContainerImage) Reset() { *m = ContainerImage{} } func (*ContainerImage) ProtoMessage() {} -func (*ContainerImage) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{24} } +func (*ContainerImage) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{25} } func (m *ContainerPort) Reset() { *m = ContainerPort{} } func (*ContainerPort) ProtoMessage() {} -func (*ContainerPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{25} } +func (*ContainerPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{26} } func (m *ContainerState) Reset() { *m = ContainerState{} } func (*ContainerState) ProtoMessage() {} -func (*ContainerState) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{26} } +func (*ContainerState) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{27} } func (m *ContainerStateRunning) Reset() { *m = ContainerStateRunning{} } func (*ContainerStateRunning) ProtoMessage() {} -func (*ContainerStateRunning) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{27} } +func (*ContainerStateRunning) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{28} } func (m *ContainerStateTerminated) Reset() { *m = ContainerStateTerminated{} } func (*ContainerStateTerminated) ProtoMessage() {} func (*ContainerStateTerminated) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{28} + return fileDescriptorGenerated, []int{29} } func (m *ContainerStateWaiting) Reset() { *m = ContainerStateWaiting{} } func (*ContainerStateWaiting) ProtoMessage() {} -func (*ContainerStateWaiting) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{29} } +func (*ContainerStateWaiting) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{30} } func (m *ContainerStatus) Reset() { *m = ContainerStatus{} } func (*ContainerStatus) ProtoMessage() {} -func (*ContainerStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{30} } +func (*ContainerStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{31} } func (m *DaemonEndpoint) Reset() { *m = DaemonEndpoint{} } func (*DaemonEndpoint) ProtoMessage() {} -func (*DaemonEndpoint) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{31} } +func (*DaemonEndpoint) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{32} } func (m *DownwardAPIProjection) Reset() { *m = DownwardAPIProjection{} } func (*DownwardAPIProjection) ProtoMessage() {} -func (*DownwardAPIProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{32} } +func (*DownwardAPIProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{33} } func (m *DownwardAPIVolumeFile) Reset() { *m = DownwardAPIVolumeFile{} } func (*DownwardAPIVolumeFile) ProtoMessage() {} -func (*DownwardAPIVolumeFile) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{33} } +func (*DownwardAPIVolumeFile) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{34} } func (m *DownwardAPIVolumeSource) Reset() { *m = DownwardAPIVolumeSource{} } func (*DownwardAPIVolumeSource) ProtoMessage() {} func (*DownwardAPIVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{34} + return fileDescriptorGenerated, []int{35} } func (m *EmptyDirVolumeSource) Reset() { *m = EmptyDirVolumeSource{} } func (*EmptyDirVolumeSource) ProtoMessage() {} -func (*EmptyDirVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{35} } +func (*EmptyDirVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{36} } func (m *EndpointAddress) Reset() { *m = EndpointAddress{} } func (*EndpointAddress) ProtoMessage() {} -func (*EndpointAddress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{36} } +func (*EndpointAddress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{37} } func (m *EndpointPort) Reset() { *m = EndpointPort{} } func (*EndpointPort) ProtoMessage() {} -func (*EndpointPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{37} } +func (*EndpointPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{38} } func (m *EndpointSubset) Reset() { *m = EndpointSubset{} } func (*EndpointSubset) ProtoMessage() {} -func (*EndpointSubset) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{38} } +func (*EndpointSubset) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{39} } func (m *Endpoints) Reset() { *m = Endpoints{} } func (*Endpoints) ProtoMessage() {} -func (*Endpoints) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{39} } +func (*Endpoints) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{40} } func (m *EndpointsList) Reset() { *m = EndpointsList{} } func (*EndpointsList) ProtoMessage() {} -func (*EndpointsList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{40} } +func (*EndpointsList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{41} } func (m *EnvFromSource) Reset() { *m = EnvFromSource{} } func (*EnvFromSource) ProtoMessage() {} -func (*EnvFromSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{41} } +func (*EnvFromSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{42} } func (m *EnvVar) Reset() { *m = EnvVar{} } func (*EnvVar) ProtoMessage() {} -func (*EnvVar) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{42} } +func (*EnvVar) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{43} } func (m *EnvVarSource) Reset() { *m = EnvVarSource{} } func (*EnvVarSource) ProtoMessage() {} -func (*EnvVarSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{43} } +func (*EnvVarSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{44} } func (m *Event) Reset() { *m = Event{} } func (*Event) ProtoMessage() {} -func (*Event) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{44} } +func (*Event) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{45} } func (m *EventList) Reset() { *m = EventList{} } func (*EventList) ProtoMessage() {} -func (*EventList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{45} } +func (*EventList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{46} } func (m *EventSeries) Reset() { *m = EventSeries{} } func (*EventSeries) ProtoMessage() {} -func (*EventSeries) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{46} } +func (*EventSeries) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{47} } func (m *EventSource) Reset() { *m = EventSource{} } func (*EventSource) ProtoMessage() {} -func (*EventSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{47} } +func (*EventSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{48} } func (m *ExecAction) Reset() { *m = ExecAction{} } func (*ExecAction) ProtoMessage() {} -func (*ExecAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{48} } +func (*ExecAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{49} } func (m *FCVolumeSource) Reset() { *m = FCVolumeSource{} } func (*FCVolumeSource) ProtoMessage() {} -func (*FCVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{49} } +func (*FCVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{50} } func (m *FlexPersistentVolumeSource) Reset() { *m = FlexPersistentVolumeSource{} } func (*FlexPersistentVolumeSource) ProtoMessage() {} func (*FlexPersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{50} + return fileDescriptorGenerated, []int{51} } func (m *FlexVolumeSource) Reset() { *m = FlexVolumeSource{} } func (*FlexVolumeSource) ProtoMessage() {} -func (*FlexVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{51} } +func (*FlexVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{52} } func (m *FlockerVolumeSource) Reset() { *m = FlockerVolumeSource{} } func (*FlockerVolumeSource) ProtoMessage() {} -func (*FlockerVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{52} } +func (*FlockerVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{53} } func (m *GCEPersistentDiskVolumeSource) Reset() { *m = GCEPersistentDiskVolumeSource{} } func (*GCEPersistentDiskVolumeSource) ProtoMessage() {} func (*GCEPersistentDiskVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{53} + return fileDescriptorGenerated, []int{54} } func (m *GitRepoVolumeSource) Reset() { *m = GitRepoVolumeSource{} } func (*GitRepoVolumeSource) ProtoMessage() {} -func (*GitRepoVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{54} } +func (*GitRepoVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{55} } func (m *GlusterfsVolumeSource) Reset() { *m = GlusterfsVolumeSource{} } func (*GlusterfsVolumeSource) ProtoMessage() {} -func (*GlusterfsVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{55} } +func (*GlusterfsVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{56} } func (m *HTTPGetAction) Reset() { *m = HTTPGetAction{} } func (*HTTPGetAction) ProtoMessage() {} -func (*HTTPGetAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{56} } +func (*HTTPGetAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{57} } func (m *HTTPHeader) Reset() { *m = HTTPHeader{} } func (*HTTPHeader) ProtoMessage() {} -func (*HTTPHeader) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{57} } +func (*HTTPHeader) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{58} } func (m *Handler) Reset() { *m = Handler{} } func (*Handler) ProtoMessage() {} -func (*Handler) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{58} } +func (*Handler) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{59} } func (m *HostAlias) Reset() { *m = HostAlias{} } func (*HostAlias) ProtoMessage() {} -func (*HostAlias) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{59} } +func (*HostAlias) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{60} } func (m *HostPathVolumeSource) Reset() { *m = HostPathVolumeSource{} } func (*HostPathVolumeSource) ProtoMessage() {} -func (*HostPathVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{60} } +func (*HostPathVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{61} } func (m *ISCSIPersistentVolumeSource) Reset() { *m = ISCSIPersistentVolumeSource{} } func (*ISCSIPersistentVolumeSource) ProtoMessage() {} func (*ISCSIPersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{61} + return fileDescriptorGenerated, []int{62} } func (m *ISCSIVolumeSource) Reset() { *m = ISCSIVolumeSource{} } func (*ISCSIVolumeSource) ProtoMessage() {} -func (*ISCSIVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{62} } +func (*ISCSIVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{63} } func (m *KeyToPath) Reset() { *m = KeyToPath{} } func (*KeyToPath) ProtoMessage() {} -func (*KeyToPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{63} } +func (*KeyToPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{64} } func (m *Lifecycle) Reset() { *m = Lifecycle{} } func (*Lifecycle) ProtoMessage() {} -func (*Lifecycle) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{64} } +func (*Lifecycle) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{65} } func (m *LimitRange) Reset() { *m = LimitRange{} } func (*LimitRange) ProtoMessage() {} -func (*LimitRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{65} } +func (*LimitRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{66} } func (m *LimitRangeItem) Reset() { *m = LimitRangeItem{} } func (*LimitRangeItem) ProtoMessage() {} -func (*LimitRangeItem) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{66} } +func (*LimitRangeItem) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{67} } func (m *LimitRangeList) Reset() { *m = LimitRangeList{} } func (*LimitRangeList) ProtoMessage() {} -func (*LimitRangeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{67} } +func (*LimitRangeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{68} } func (m *LimitRangeSpec) Reset() { *m = LimitRangeSpec{} } func (*LimitRangeSpec) ProtoMessage() {} -func (*LimitRangeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{68} } +func (*LimitRangeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{69} } func (m *List) Reset() { *m = List{} } func (*List) ProtoMessage() {} -func (*List) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{69} } +func (*List) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{70} } func (m *LoadBalancerIngress) Reset() { *m = LoadBalancerIngress{} } func (*LoadBalancerIngress) ProtoMessage() {} -func (*LoadBalancerIngress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{70} } +func (*LoadBalancerIngress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{71} } func (m *LoadBalancerStatus) Reset() { *m = LoadBalancerStatus{} } func (*LoadBalancerStatus) ProtoMessage() {} -func (*LoadBalancerStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{71} } +func (*LoadBalancerStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{72} } func (m *LocalObjectReference) Reset() { *m = LocalObjectReference{} } func (*LocalObjectReference) ProtoMessage() {} -func (*LocalObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{72} } +func (*LocalObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{73} } func (m *LocalVolumeSource) Reset() { *m = LocalVolumeSource{} } func (*LocalVolumeSource) ProtoMessage() {} -func (*LocalVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{73} } +func (*LocalVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{74} } func (m *NFSVolumeSource) Reset() { *m = NFSVolumeSource{} } func (*NFSVolumeSource) ProtoMessage() {} -func (*NFSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{74} } +func (*NFSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{75} } func (m *Namespace) Reset() { *m = Namespace{} } func (*Namespace) ProtoMessage() {} -func (*Namespace) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{75} } +func (*Namespace) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{76} } func (m *NamespaceList) Reset() { *m = NamespaceList{} } func (*NamespaceList) ProtoMessage() {} -func (*NamespaceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{76} } +func (*NamespaceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{77} } func (m *NamespaceSpec) Reset() { *m = NamespaceSpec{} } func (*NamespaceSpec) ProtoMessage() {} -func (*NamespaceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{77} } +func (*NamespaceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{78} } func (m *NamespaceStatus) Reset() { *m = NamespaceStatus{} } func (*NamespaceStatus) ProtoMessage() {} -func (*NamespaceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{78} } +func (*NamespaceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{79} } func (m *Node) Reset() { *m = Node{} } func (*Node) ProtoMessage() {} -func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{79} } +func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{80} } func (m *NodeAddress) Reset() { *m = NodeAddress{} } func (*NodeAddress) ProtoMessage() {} -func (*NodeAddress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{80} } +func (*NodeAddress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{81} } func (m *NodeAffinity) Reset() { *m = NodeAffinity{} } func (*NodeAffinity) ProtoMessage() {} -func (*NodeAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{81} } +func (*NodeAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{82} } func (m *NodeCondition) Reset() { *m = NodeCondition{} } func (*NodeCondition) ProtoMessage() {} -func (*NodeCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{82} } +func (*NodeCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{83} } func (m *NodeConfigSource) Reset() { *m = NodeConfigSource{} } func (*NodeConfigSource) ProtoMessage() {} -func (*NodeConfigSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{83} } +func (*NodeConfigSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{84} } func (m *NodeDaemonEndpoints) Reset() { *m = NodeDaemonEndpoints{} } func (*NodeDaemonEndpoints) ProtoMessage() {} -func (*NodeDaemonEndpoints) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{84} } +func (*NodeDaemonEndpoints) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{85} } func (m *NodeList) Reset() { *m = NodeList{} } func (*NodeList) ProtoMessage() {} -func (*NodeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{85} } +func (*NodeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{86} } func (m *NodeProxyOptions) Reset() { *m = NodeProxyOptions{} } func (*NodeProxyOptions) ProtoMessage() {} -func (*NodeProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{86} } +func (*NodeProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{87} } func (m *NodeResources) Reset() { *m = NodeResources{} } func (*NodeResources) ProtoMessage() {} -func (*NodeResources) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{87} } +func (*NodeResources) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{88} } func (m *NodeSelector) Reset() { *m = NodeSelector{} } func (*NodeSelector) ProtoMessage() {} -func (*NodeSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{88} } +func (*NodeSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{89} } func (m *NodeSelectorRequirement) Reset() { *m = NodeSelectorRequirement{} } func (*NodeSelectorRequirement) ProtoMessage() {} func (*NodeSelectorRequirement) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{89} + return fileDescriptorGenerated, []int{90} } func (m *NodeSelectorTerm) Reset() { *m = NodeSelectorTerm{} } func (*NodeSelectorTerm) ProtoMessage() {} -func (*NodeSelectorTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{90} } +func (*NodeSelectorTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{91} } func (m *NodeSpec) Reset() { *m = NodeSpec{} } func (*NodeSpec) ProtoMessage() {} -func (*NodeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{91} } +func (*NodeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{92} } func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{92} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{93} } func (m *NodeSystemInfo) Reset() { *m = NodeSystemInfo{} } func (*NodeSystemInfo) ProtoMessage() {} -func (*NodeSystemInfo) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{93} } +func (*NodeSystemInfo) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{94} } func (m *ObjectFieldSelector) Reset() { *m = ObjectFieldSelector{} } func (*ObjectFieldSelector) ProtoMessage() {} -func (*ObjectFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{94} } +func (*ObjectFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{95} } func (m *ObjectReference) Reset() { *m = ObjectReference{} } func (*ObjectReference) ProtoMessage() {} -func (*ObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{95} } +func (*ObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{96} } func (m *PersistentVolume) Reset() { *m = PersistentVolume{} } func (*PersistentVolume) ProtoMessage() {} -func (*PersistentVolume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{96} } +func (*PersistentVolume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{97} } func (m *PersistentVolumeClaim) Reset() { *m = PersistentVolumeClaim{} } func (*PersistentVolumeClaim) ProtoMessage() {} -func (*PersistentVolumeClaim) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{97} } +func (*PersistentVolumeClaim) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{98} } func (m *PersistentVolumeClaimCondition) Reset() { *m = PersistentVolumeClaimCondition{} } func (*PersistentVolumeClaimCondition) ProtoMessage() {} func (*PersistentVolumeClaimCondition) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{98} + return fileDescriptorGenerated, []int{99} } func (m *PersistentVolumeClaimList) Reset() { *m = PersistentVolumeClaimList{} } func (*PersistentVolumeClaimList) ProtoMessage() {} func (*PersistentVolumeClaimList) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{99} + return fileDescriptorGenerated, []int{100} } func (m *PersistentVolumeClaimSpec) Reset() { *m = PersistentVolumeClaimSpec{} } func (*PersistentVolumeClaimSpec) ProtoMessage() {} func (*PersistentVolumeClaimSpec) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{100} + return fileDescriptorGenerated, []int{101} } func (m *PersistentVolumeClaimStatus) Reset() { *m = PersistentVolumeClaimStatus{} } func (*PersistentVolumeClaimStatus) ProtoMessage() {} func (*PersistentVolumeClaimStatus) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{101} + return fileDescriptorGenerated, []int{102} } func (m *PersistentVolumeClaimVolumeSource) Reset() { *m = PersistentVolumeClaimVolumeSource{} } func (*PersistentVolumeClaimVolumeSource) ProtoMessage() {} func (*PersistentVolumeClaimVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{102} + return fileDescriptorGenerated, []int{103} } func (m *PersistentVolumeList) Reset() { *m = PersistentVolumeList{} } func (*PersistentVolumeList) ProtoMessage() {} -func (*PersistentVolumeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{103} } +func (*PersistentVolumeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{104} } func (m *PersistentVolumeSource) Reset() { *m = PersistentVolumeSource{} } func (*PersistentVolumeSource) ProtoMessage() {} func (*PersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{104} + return fileDescriptorGenerated, []int{105} } func (m *PersistentVolumeSpec) Reset() { *m = PersistentVolumeSpec{} } func (*PersistentVolumeSpec) ProtoMessage() {} -func (*PersistentVolumeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{105} } +func (*PersistentVolumeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{106} } func (m *PersistentVolumeStatus) Reset() { *m = PersistentVolumeStatus{} } func (*PersistentVolumeStatus) ProtoMessage() {} func (*PersistentVolumeStatus) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{106} + return fileDescriptorGenerated, []int{107} } func (m *PhotonPersistentDiskVolumeSource) Reset() { *m = PhotonPersistentDiskVolumeSource{} } func (*PhotonPersistentDiskVolumeSource) ProtoMessage() {} func (*PhotonPersistentDiskVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{107} + return fileDescriptorGenerated, []int{108} } func (m *Pod) Reset() { *m = Pod{} } func (*Pod) ProtoMessage() {} -func (*Pod) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{108} } +func (*Pod) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{109} } func (m *PodAffinity) Reset() { *m = PodAffinity{} } func (*PodAffinity) ProtoMessage() {} -func (*PodAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{109} } +func (*PodAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{110} } func (m *PodAffinityTerm) Reset() { *m = PodAffinityTerm{} } func (*PodAffinityTerm) ProtoMessage() {} -func (*PodAffinityTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{110} } +func (*PodAffinityTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{111} } func (m *PodAntiAffinity) Reset() { *m = PodAntiAffinity{} } func (*PodAntiAffinity) ProtoMessage() {} -func (*PodAntiAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{111} } +func (*PodAntiAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{112} } func (m *PodAttachOptions) Reset() { *m = PodAttachOptions{} } func (*PodAttachOptions) ProtoMessage() {} -func (*PodAttachOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{112} } +func (*PodAttachOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{113} } func (m *PodCondition) Reset() { *m = PodCondition{} } func (*PodCondition) ProtoMessage() {} -func (*PodCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{113} } +func (*PodCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{114} } func (m *PodDNSConfig) Reset() { *m = PodDNSConfig{} } func (*PodDNSConfig) ProtoMessage() {} -func (*PodDNSConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{114} } +func (*PodDNSConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{115} } func (m *PodDNSConfigOption) Reset() { *m = PodDNSConfigOption{} } func (*PodDNSConfigOption) ProtoMessage() {} -func (*PodDNSConfigOption) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{115} } +func (*PodDNSConfigOption) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{116} } func (m *PodExecOptions) Reset() { *m = PodExecOptions{} } func (*PodExecOptions) ProtoMessage() {} -func (*PodExecOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{116} } +func (*PodExecOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{117} } func (m *PodList) Reset() { *m = PodList{} } func (*PodList) ProtoMessage() {} -func (*PodList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{117} } +func (*PodList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{118} } func (m *PodLogOptions) Reset() { *m = PodLogOptions{} } func (*PodLogOptions) ProtoMessage() {} -func (*PodLogOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{118} } +func (*PodLogOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{119} } func (m *PodPortForwardOptions) Reset() { *m = PodPortForwardOptions{} } func (*PodPortForwardOptions) ProtoMessage() {} -func (*PodPortForwardOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{119} } +func (*PodPortForwardOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{120} } func (m *PodProxyOptions) Reset() { *m = PodProxyOptions{} } func (*PodProxyOptions) ProtoMessage() {} -func (*PodProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{120} } +func (*PodProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{121} } func (m *PodSecurityContext) Reset() { *m = PodSecurityContext{} } func (*PodSecurityContext) ProtoMessage() {} -func (*PodSecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{121} } +func (*PodSecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{122} } func (m *PodSignature) Reset() { *m = PodSignature{} } func (*PodSignature) ProtoMessage() {} -func (*PodSignature) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{122} } +func (*PodSignature) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{123} } func (m *PodSpec) Reset() { *m = PodSpec{} } func (*PodSpec) ProtoMessage() {} -func (*PodSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{123} } +func (*PodSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{124} } func (m *PodStatus) Reset() { *m = PodStatus{} } func (*PodStatus) ProtoMessage() {} -func (*PodStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{124} } +func (*PodStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{125} } func (m *PodStatusResult) Reset() { *m = PodStatusResult{} } func (*PodStatusResult) ProtoMessage() {} -func (*PodStatusResult) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{125} } +func (*PodStatusResult) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{126} } func (m *PodTemplate) Reset() { *m = PodTemplate{} } func (*PodTemplate) ProtoMessage() {} -func (*PodTemplate) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{126} } +func (*PodTemplate) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{127} } func (m *PodTemplateList) Reset() { *m = PodTemplateList{} } func (*PodTemplateList) ProtoMessage() {} -func (*PodTemplateList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{127} } +func (*PodTemplateList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{128} } func (m *PodTemplateSpec) Reset() { *m = PodTemplateSpec{} } func (*PodTemplateSpec) ProtoMessage() {} -func (*PodTemplateSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{128} } +func (*PodTemplateSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{129} } func (m *PortworxVolumeSource) Reset() { *m = PortworxVolumeSource{} } func (*PortworxVolumeSource) ProtoMessage() {} -func (*PortworxVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{129} } +func (*PortworxVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{130} } func (m *Preconditions) Reset() { *m = Preconditions{} } func (*Preconditions) ProtoMessage() {} -func (*Preconditions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{130} } +func (*Preconditions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{131} } func (m *PreferAvoidPodsEntry) Reset() { *m = PreferAvoidPodsEntry{} } func (*PreferAvoidPodsEntry) ProtoMessage() {} -func (*PreferAvoidPodsEntry) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{131} } +func (*PreferAvoidPodsEntry) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{132} } func (m *PreferredSchedulingTerm) Reset() { *m = PreferredSchedulingTerm{} } func (*PreferredSchedulingTerm) ProtoMessage() {} func (*PreferredSchedulingTerm) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{132} + return fileDescriptorGenerated, []int{133} } func (m *Probe) Reset() { *m = Probe{} } func (*Probe) ProtoMessage() {} -func (*Probe) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{133} } +func (*Probe) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{134} } func (m *ProjectedVolumeSource) Reset() { *m = ProjectedVolumeSource{} } func (*ProjectedVolumeSource) ProtoMessage() {} -func (*ProjectedVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{134} } +func (*ProjectedVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{135} } func (m *QuobyteVolumeSource) Reset() { *m = QuobyteVolumeSource{} } func (*QuobyteVolumeSource) ProtoMessage() {} -func (*QuobyteVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{135} } +func (*QuobyteVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{136} } func (m *RBDPersistentVolumeSource) Reset() { *m = RBDPersistentVolumeSource{} } func (*RBDPersistentVolumeSource) ProtoMessage() {} func (*RBDPersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{136} + return fileDescriptorGenerated, []int{137} } func (m *RBDVolumeSource) Reset() { *m = RBDVolumeSource{} } func (*RBDVolumeSource) ProtoMessage() {} -func (*RBDVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{137} } +func (*RBDVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{138} } func (m *RangeAllocation) Reset() { *m = RangeAllocation{} } func (*RangeAllocation) ProtoMessage() {} -func (*RangeAllocation) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{138} } +func (*RangeAllocation) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{139} } func (m *ReplicationController) Reset() { *m = ReplicationController{} } func (*ReplicationController) ProtoMessage() {} -func (*ReplicationController) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{139} } +func (*ReplicationController) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{140} } func (m *ReplicationControllerCondition) Reset() { *m = ReplicationControllerCondition{} } func (*ReplicationControllerCondition) ProtoMessage() {} func (*ReplicationControllerCondition) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{140} + return fileDescriptorGenerated, []int{141} } func (m *ReplicationControllerList) Reset() { *m = ReplicationControllerList{} } func (*ReplicationControllerList) ProtoMessage() {} func (*ReplicationControllerList) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{141} + return fileDescriptorGenerated, []int{142} } func (m *ReplicationControllerSpec) Reset() { *m = ReplicationControllerSpec{} } func (*ReplicationControllerSpec) ProtoMessage() {} func (*ReplicationControllerSpec) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{142} + return fileDescriptorGenerated, []int{143} } func (m *ReplicationControllerStatus) Reset() { *m = ReplicationControllerStatus{} } func (*ReplicationControllerStatus) ProtoMessage() {} func (*ReplicationControllerStatus) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{143} + return fileDescriptorGenerated, []int{144} } func (m *ResourceFieldSelector) Reset() { *m = ResourceFieldSelector{} } func (*ResourceFieldSelector) ProtoMessage() {} -func (*ResourceFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{144} } +func (*ResourceFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{145} } func (m *ResourceQuota) Reset() { *m = ResourceQuota{} } func (*ResourceQuota) ProtoMessage() {} -func (*ResourceQuota) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{145} } +func (*ResourceQuota) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{146} } func (m *ResourceQuotaList) Reset() { *m = ResourceQuotaList{} } func (*ResourceQuotaList) ProtoMessage() {} -func (*ResourceQuotaList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{146} } +func (*ResourceQuotaList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{147} } func (m *ResourceQuotaSpec) Reset() { *m = ResourceQuotaSpec{} } func (*ResourceQuotaSpec) ProtoMessage() {} -func (*ResourceQuotaSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{147} } +func (*ResourceQuotaSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{148} } func (m *ResourceQuotaStatus) Reset() { *m = ResourceQuotaStatus{} } func (*ResourceQuotaStatus) ProtoMessage() {} -func (*ResourceQuotaStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{148} } +func (*ResourceQuotaStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{149} } func (m *ResourceRequirements) Reset() { *m = ResourceRequirements{} } func (*ResourceRequirements) ProtoMessage() {} -func (*ResourceRequirements) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{149} } +func (*ResourceRequirements) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{150} } func (m *SELinuxOptions) Reset() { *m = SELinuxOptions{} } func (*SELinuxOptions) ProtoMessage() {} -func (*SELinuxOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{150} } +func (*SELinuxOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{151} } func (m *ScaleIOPersistentVolumeSource) Reset() { *m = ScaleIOPersistentVolumeSource{} } func (*ScaleIOPersistentVolumeSource) ProtoMessage() {} func (*ScaleIOPersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{151} + return fileDescriptorGenerated, []int{152} } func (m *ScaleIOVolumeSource) Reset() { *m = ScaleIOVolumeSource{} } func (*ScaleIOVolumeSource) ProtoMessage() {} -func (*ScaleIOVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{152} } +func (*ScaleIOVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{153} } func (m *Secret) Reset() { *m = Secret{} } func (*Secret) ProtoMessage() {} -func (*Secret) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{153} } +func (*Secret) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{154} } func (m *SecretEnvSource) Reset() { *m = SecretEnvSource{} } func (*SecretEnvSource) ProtoMessage() {} -func (*SecretEnvSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{154} } +func (*SecretEnvSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{155} } func (m *SecretKeySelector) Reset() { *m = SecretKeySelector{} } func (*SecretKeySelector) ProtoMessage() {} -func (*SecretKeySelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{155} } +func (*SecretKeySelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{156} } func (m *SecretList) Reset() { *m = SecretList{} } func (*SecretList) ProtoMessage() {} -func (*SecretList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{156} } +func (*SecretList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{157} } func (m *SecretProjection) Reset() { *m = SecretProjection{} } func (*SecretProjection) ProtoMessage() {} -func (*SecretProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{157} } +func (*SecretProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{158} } func (m *SecretReference) Reset() { *m = SecretReference{} } func (*SecretReference) ProtoMessage() {} -func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{158} } +func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{159} } func (m *SecretVolumeSource) Reset() { *m = SecretVolumeSource{} } func (*SecretVolumeSource) ProtoMessage() {} -func (*SecretVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{159} } +func (*SecretVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{160} } func (m *SecurityContext) Reset() { *m = SecurityContext{} } func (*SecurityContext) ProtoMessage() {} -func (*SecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{160} } +func (*SecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{161} } func (m *SerializedReference) Reset() { *m = SerializedReference{} } func (*SerializedReference) ProtoMessage() {} -func (*SerializedReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{161} } +func (*SerializedReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{162} } func (m *Service) Reset() { *m = Service{} } func (*Service) ProtoMessage() {} -func (*Service) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{162} } +func (*Service) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{163} } func (m *ServiceAccount) Reset() { *m = ServiceAccount{} } func (*ServiceAccount) ProtoMessage() {} -func (*ServiceAccount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{163} } +func (*ServiceAccount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{164} } func (m *ServiceAccountList) Reset() { *m = ServiceAccountList{} } func (*ServiceAccountList) ProtoMessage() {} -func (*ServiceAccountList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{164} } +func (*ServiceAccountList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{165} } func (m *ServiceList) Reset() { *m = ServiceList{} } func (*ServiceList) ProtoMessage() {} -func (*ServiceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{165} } +func (*ServiceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{166} } func (m *ServicePort) Reset() { *m = ServicePort{} } func (*ServicePort) ProtoMessage() {} -func (*ServicePort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{166} } +func (*ServicePort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{167} } func (m *ServiceProxyOptions) Reset() { *m = ServiceProxyOptions{} } func (*ServiceProxyOptions) ProtoMessage() {} -func (*ServiceProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{167} } +func (*ServiceProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{168} } func (m *ServiceSpec) Reset() { *m = ServiceSpec{} } func (*ServiceSpec) ProtoMessage() {} -func (*ServiceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{168} } +func (*ServiceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{169} } func (m *ServiceStatus) Reset() { *m = ServiceStatus{} } func (*ServiceStatus) ProtoMessage() {} -func (*ServiceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{169} } +func (*ServiceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{170} } func (m *SessionAffinityConfig) Reset() { *m = SessionAffinityConfig{} } func (*SessionAffinityConfig) ProtoMessage() {} -func (*SessionAffinityConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{170} } +func (*SessionAffinityConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{171} } func (m *StorageOSPersistentVolumeSource) Reset() { *m = StorageOSPersistentVolumeSource{} } func (*StorageOSPersistentVolumeSource) ProtoMessage() {} func (*StorageOSPersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{171} + return fileDescriptorGenerated, []int{172} } func (m *StorageOSVolumeSource) Reset() { *m = StorageOSVolumeSource{} } func (*StorageOSVolumeSource) ProtoMessage() {} -func (*StorageOSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{172} } +func (*StorageOSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{173} } func (m *Sysctl) Reset() { *m = Sysctl{} } func (*Sysctl) ProtoMessage() {} -func (*Sysctl) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{173} } +func (*Sysctl) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{174} } func (m *TCPSocketAction) Reset() { *m = TCPSocketAction{} } func (*TCPSocketAction) ProtoMessage() {} -func (*TCPSocketAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{174} } +func (*TCPSocketAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{175} } func (m *Taint) Reset() { *m = Taint{} } func (*Taint) ProtoMessage() {} -func (*Taint) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{175} } +func (*Taint) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{176} } func (m *Toleration) Reset() { *m = Toleration{} } func (*Toleration) ProtoMessage() {} -func (*Toleration) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{176} } +func (*Toleration) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{177} } func (m *Volume) Reset() { *m = Volume{} } func (*Volume) ProtoMessage() {} -func (*Volume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{177} } +func (*Volume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{178} } func (m *VolumeDevice) Reset() { *m = VolumeDevice{} } func (*VolumeDevice) ProtoMessage() {} -func (*VolumeDevice) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{178} } +func (*VolumeDevice) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{179} } func (m *VolumeMount) Reset() { *m = VolumeMount{} } func (*VolumeMount) ProtoMessage() {} -func (*VolumeMount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{179} } +func (*VolumeMount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{180} } func (m *VolumeNodeAffinity) Reset() { *m = VolumeNodeAffinity{} } func (*VolumeNodeAffinity) ProtoMessage() {} -func (*VolumeNodeAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{180} } +func (*VolumeNodeAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{181} } func (m *VolumeProjection) Reset() { *m = VolumeProjection{} } func (*VolumeProjection) ProtoMessage() {} -func (*VolumeProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{181} } +func (*VolumeProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{182} } func (m *VolumeSource) Reset() { *m = VolumeSource{} } func (*VolumeSource) ProtoMessage() {} -func (*VolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{182} } +func (*VolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{183} } func (m *VsphereVirtualDiskVolumeSource) Reset() { *m = VsphereVirtualDiskVolumeSource{} } func (*VsphereVirtualDiskVolumeSource) ProtoMessage() {} func (*VsphereVirtualDiskVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{183} + return fileDescriptorGenerated, []int{184} } func (m *WeightedPodAffinityTerm) Reset() { *m = WeightedPodAffinityTerm{} } func (*WeightedPodAffinityTerm) ProtoMessage() {} func (*WeightedPodAffinityTerm) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{184} + return fileDescriptorGenerated, []int{185} } func init() { @@ -1059,6 +1066,7 @@ func init() { proto.RegisterType((*ConfigMapEnvSource)(nil), "k8s.io.api.core.v1.ConfigMapEnvSource") proto.RegisterType((*ConfigMapKeySelector)(nil), "k8s.io.api.core.v1.ConfigMapKeySelector") proto.RegisterType((*ConfigMapList)(nil), "k8s.io.api.core.v1.ConfigMapList") + proto.RegisterType((*ConfigMapNodeConfigSource)(nil), "k8s.io.api.core.v1.ConfigMapNodeConfigSource") proto.RegisterType((*ConfigMapProjection)(nil), "k8s.io.api.core.v1.ConfigMapProjection") proto.RegisterType((*ConfigMapVolumeSource)(nil), "k8s.io.api.core.v1.ConfigMapVolumeSource") proto.RegisterType((*Container)(nil), "k8s.io.api.core.v1.Container") @@ -2148,6 +2156,44 @@ func (m *ConfigMapList) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *ConfigMapNodeConfigSource) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ConfigMapNodeConfigSource) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Namespace))) + i += copy(dAtA[i:], m.Namespace) + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + dAtA[i] = 0x1a + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.UID))) + i += copy(dAtA[i:], m.UID) + dAtA[i] = 0x22 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.ResourceVersion))) + i += copy(dAtA[i:], m.ResourceVersion) + dAtA[i] = 0x2a + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.KubeletConfigKey))) + i += copy(dAtA[i:], m.KubeletConfigKey) + return i, nil +} + func (m *ConfigMapProjection) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -5020,11 +5066,11 @@ func (m *NodeConfigSource) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.ConfigMapRef != nil { - dAtA[i] = 0xa + if m.ConfigMap != nil { + dAtA[i] = 0x12 i++ - i = encodeVarintGenerated(dAtA, i, uint64(m.ConfigMapRef.Size())) - n84, err := m.ConfigMapRef.MarshalTo(dAtA[i:]) + i = encodeVarintGenerated(dAtA, i, uint64(m.ConfigMap.Size())) + n84, err := m.ConfigMap.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -10786,6 +10832,22 @@ func (m *ConfigMapList) Size() (n int) { return n } +func (m *ConfigMapNodeConfigSource) Size() (n int) { + var l int + _ = l + l = len(m.Namespace) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.Name) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.UID) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.ResourceVersion) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.KubeletConfigKey) + n += 1 + l + sovGenerated(uint64(l)) + return n +} + func (m *ConfigMapProjection) Size() (n int) { var l int _ = l @@ -11826,8 +11888,8 @@ func (m *NodeCondition) Size() (n int) { func (m *NodeConfigSource) Size() (n int) { var l int _ = l - if m.ConfigMapRef != nil { - l = m.ConfigMapRef.Size() + if m.ConfigMap != nil { + l = m.ConfigMap.Size() n += 1 + l + sovGenerated(uint64(l)) } return n @@ -14095,6 +14157,20 @@ func (this *ConfigMapList) String() string { }, "") return s } +func (this *ConfigMapNodeConfigSource) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ConfigMapNodeConfigSource{`, + `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `UID:` + fmt.Sprintf("%v", this.UID) + `,`, + `ResourceVersion:` + fmt.Sprintf("%v", this.ResourceVersion) + `,`, + `KubeletConfigKey:` + fmt.Sprintf("%v", this.KubeletConfigKey) + `,`, + `}`, + }, "") + return s +} func (this *ConfigMapProjection) String() string { if this == nil { return "nil" @@ -14950,7 +15026,7 @@ func (this *NodeConfigSource) String() string { return "nil" } s := strings.Join([]string{`&NodeConfigSource{`, - `ConfigMapRef:` + strings.Replace(fmt.Sprintf("%v", this.ConfigMapRef), "ObjectReference", "ObjectReference", 1) + `,`, + `ConfigMap:` + strings.Replace(fmt.Sprintf("%v", this.ConfigMap), "ConfigMapNodeConfigSource", "ConfigMapNodeConfigSource", 1) + `,`, `}`, }, "") return s @@ -19718,6 +19794,201 @@ func (m *ConfigMapList) Unmarshal(dAtA []byte) error { } return nil } +func (m *ConfigMapNodeConfigSource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ConfigMapNodeConfigSource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ConfigMapNodeConfigSource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Namespace = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UID = k8s_io_apimachinery_pkg_types.UID(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ResourceVersion", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ResourceVersion = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field KubeletConfigKey", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.KubeletConfigKey = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *ConfigMapProjection) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -29784,9 +30055,9 @@ func (m *NodeConfigSource) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: NodeConfigSource: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ConfigMapRef", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ConfigMap", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -29810,10 +30081,10 @@ func (m *NodeConfigSource) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.ConfigMapRef == nil { - m.ConfigMapRef = &ObjectReference{} + if m.ConfigMap == nil { + m.ConfigMap = &ConfigMapNodeConfigSource{} } - if err := m.ConfigMapRef.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.ConfigMap.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -48907,769 +49178,772 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 12209 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x6f, 0x70, 0x24, 0x49, - 0x56, 0x18, 0x7e, 0xd5, 0xdd, 0xfa, 0xd3, 0x4f, 0xff, 0x73, 0x34, 0xb3, 0x1a, 0xed, 0xce, 0xf4, - 0x6c, 0xed, 0xdd, 0xec, 0xec, 0xed, 0xae, 0xc4, 0xce, 0xee, 0xde, 0x0e, 0xb7, 0x77, 0x0b, 0x92, - 0x5a, 0x9a, 0xd1, 0xce, 0x48, 0xd3, 0x9b, 0xad, 0x99, 0xb9, 0x3b, 0x96, 0xe3, 0x4a, 0xdd, 0x29, - 0xa9, 0x56, 0xa5, 0xaa, 0xde, 0xaa, 0x6a, 0xcd, 0x68, 0x03, 0x22, 0x7e, 0xbf, 0x03, 0x63, 0x63, - 0xf8, 0x70, 0x61, 0x08, 0x1b, 0x03, 0x81, 0x23, 0x6c, 0x1c, 0x70, 0xc6, 0x76, 0x04, 0x06, 0x03, - 0x3e, 0xb0, 0x8d, 0xb1, 0x3f, 0xe0, 0x2f, 0x18, 0xfb, 0xcb, 0x11, 0x41, 0x58, 0x06, 0x41, 0xd8, - 0xc1, 0x07, 0x3b, 0x1c, 0x26, 0xc2, 0x11, 0xc8, 0x84, 0x71, 0xe4, 0xdf, 0xca, 0xac, 0xae, 0xea, - 0x6e, 0xcd, 0x6a, 0xb4, 0xcb, 0xc5, 0x7d, 0xeb, 0xce, 0xf7, 0xf2, 0x65, 0x56, 0xfe, 0x79, 0xf9, - 0xf2, 0xe5, 0xfb, 0x03, 0x6f, 0xee, 0xde, 0x88, 0xe6, 0xdc, 0x60, 0x7e, 0xb7, 0xbd, 0x49, 0x42, - 0x9f, 0xc4, 0x24, 0x9a, 0xdf, 0x27, 0x7e, 0x33, 0x08, 0xe7, 0x05, 0xc0, 0x69, 0xb9, 0xf3, 0x8d, - 0x20, 0x24, 0xf3, 0xfb, 0xaf, 0xcc, 0x6f, 0x13, 0x9f, 0x84, 0x4e, 0x4c, 0x9a, 0x73, 0xad, 0x30, - 0x88, 0x03, 0x84, 0x38, 0xce, 0x9c, 0xd3, 0x72, 0xe7, 0x28, 0xce, 0xdc, 0xfe, 0x2b, 0xb3, 0x2f, - 0x6f, 0xbb, 0xf1, 0x4e, 0x7b, 0x73, 0xae, 0x11, 0xec, 0xcd, 0x6f, 0x07, 0xdb, 0xc1, 0x3c, 0x43, - 0xdd, 0x6c, 0x6f, 0xb1, 0x7f, 0xec, 0x0f, 0xfb, 0xc5, 0x49, 0xcc, 0xae, 0x25, 0xcd, 0x90, 0x47, - 0x31, 0xf1, 0x23, 0x37, 0xf0, 0xa3, 0x97, 0x9d, 0x96, 0x1b, 0x91, 0x70, 0x9f, 0x84, 0xf3, 0xad, - 0xdd, 0x6d, 0x0a, 0x8b, 0x4c, 0x84, 0xf9, 0xfd, 0x57, 0x36, 0x49, 0xec, 0x74, 0xf4, 0x68, 0xf6, - 0xb5, 0x84, 0xdc, 0x9e, 0xd3, 0xd8, 0x71, 0x7d, 0x12, 0x1e, 0x48, 0x1a, 0xf3, 0x21, 0x89, 0x82, - 0x76, 0xd8, 0x20, 0x27, 0xaa, 0x15, 0xcd, 0xef, 0x91, 0xd8, 0xc9, 0xf8, 0xfa, 0xd9, 0xf9, 0xbc, - 0x5a, 0x61, 0xdb, 0x8f, 0xdd, 0xbd, 0xce, 0x66, 0x3e, 0xd3, 0xab, 0x42, 0xd4, 0xd8, 0x21, 0x7b, - 0x4e, 0x47, 0xbd, 0x57, 0xf3, 0xea, 0xb5, 0x63, 0xd7, 0x9b, 0x77, 0xfd, 0x38, 0x8a, 0xc3, 0x74, - 0x25, 0xfb, 0x9b, 0x16, 0x5c, 0x59, 0x78, 0x50, 0x5f, 0xf6, 0x9c, 0x28, 0x76, 0x1b, 0x8b, 0x5e, - 0xd0, 0xd8, 0xad, 0xc7, 0x41, 0x48, 0xee, 0x07, 0x5e, 0x7b, 0x8f, 0xd4, 0xd9, 0x40, 0xa0, 0x97, - 0x60, 0x78, 0x9f, 0xfd, 0x5f, 0xad, 0xce, 0x58, 0x57, 0xac, 0x6b, 0xe5, 0xc5, 0xc9, 0xdf, 0x39, - 0xac, 0x7c, 0xe2, 0xe8, 0xb0, 0x32, 0x7c, 0x5f, 0x94, 0x63, 0x85, 0x81, 0xae, 0xc2, 0xe0, 0x56, - 0xb4, 0x71, 0xd0, 0x22, 0x33, 0x05, 0x86, 0x3b, 0x2e, 0x70, 0x07, 0x57, 0xea, 0xb4, 0x14, 0x0b, - 0x28, 0x9a, 0x87, 0x72, 0xcb, 0x09, 0x63, 0x37, 0x76, 0x03, 0x7f, 0xa6, 0x78, 0xc5, 0xba, 0x36, - 0xb0, 0x38, 0x25, 0x50, 0xcb, 0x35, 0x09, 0xc0, 0x09, 0x0e, 0xed, 0x46, 0x48, 0x9c, 0xe6, 0x5d, - 0xdf, 0x3b, 0x98, 0x29, 0x5d, 0xb1, 0xae, 0x0d, 0x27, 0xdd, 0xc0, 0xa2, 0x1c, 0x2b, 0x0c, 0xfb, - 0xa7, 0x0a, 0x30, 0xbc, 0xb0, 0xb5, 0xe5, 0xfa, 0x6e, 0x7c, 0x80, 0xee, 0xc3, 0xa8, 0x1f, 0x34, - 0x89, 0xfc, 0xcf, 0xbe, 0x62, 0xe4, 0xfa, 0x95, 0xb9, 0xce, 0x95, 0x39, 0xb7, 0xae, 0xe1, 0x2d, - 0x4e, 0x1e, 0x1d, 0x56, 0x46, 0xf5, 0x12, 0x6c, 0xd0, 0x41, 0x18, 0x46, 0x5a, 0x41, 0x53, 0x91, - 0x2d, 0x30, 0xb2, 0x95, 0x2c, 0xb2, 0xb5, 0x04, 0x6d, 0x71, 0xe2, 0xe8, 0xb0, 0x32, 0xa2, 0x15, - 0x60, 0x9d, 0x08, 0xda, 0x84, 0x09, 0xfa, 0xd7, 0x8f, 0x5d, 0x45, 0xb7, 0xc8, 0xe8, 0x3e, 0x97, - 0x47, 0x57, 0x43, 0x5d, 0x3c, 0x77, 0x74, 0x58, 0x99, 0x48, 0x15, 0xe2, 0x34, 0x41, 0xfb, 0x03, - 0x18, 0x5f, 0x88, 0x63, 0xa7, 0xb1, 0x43, 0x9a, 0x7c, 0x06, 0xd1, 0x6b, 0x50, 0xf2, 0x9d, 0x3d, - 0x22, 0xe6, 0xf7, 0x8a, 0x18, 0xd8, 0xd2, 0xba, 0xb3, 0x47, 0x8e, 0x0f, 0x2b, 0x93, 0xf7, 0x7c, - 0xf7, 0xfd, 0xb6, 0x58, 0x15, 0xb4, 0x0c, 0x33, 0x6c, 0x74, 0x1d, 0xa0, 0x49, 0xf6, 0xdd, 0x06, - 0xa9, 0x39, 0xf1, 0x8e, 0x98, 0x6f, 0x24, 0xea, 0x42, 0x55, 0x41, 0xb0, 0x86, 0x65, 0x3f, 0x82, - 0xf2, 0xc2, 0x7e, 0xe0, 0x36, 0x6b, 0x41, 0x33, 0x42, 0xbb, 0x30, 0xd1, 0x0a, 0xc9, 0x16, 0x09, - 0x55, 0xd1, 0x8c, 0x75, 0xa5, 0x78, 0x6d, 0xe4, 0xfa, 0xb5, 0xcc, 0x8f, 0x35, 0x51, 0x97, 0xfd, - 0x38, 0x3c, 0x58, 0x7c, 0x4a, 0xb4, 0x37, 0x91, 0x82, 0xe2, 0x34, 0x65, 0xfb, 0xdf, 0x15, 0xe0, - 0xfc, 0xc2, 0x07, 0xed, 0x90, 0x54, 0xdd, 0x68, 0x37, 0xbd, 0xc2, 0x9b, 0x6e, 0xb4, 0xbb, 0x9e, - 0x8c, 0x80, 0x5a, 0x5a, 0x55, 0x51, 0x8e, 0x15, 0x06, 0x7a, 0x19, 0x86, 0xe8, 0xef, 0x7b, 0x78, - 0x55, 0x7c, 0xf2, 0x39, 0x81, 0x3c, 0x52, 0x75, 0x62, 0xa7, 0xca, 0x41, 0x58, 0xe2, 0xa0, 0x35, - 0x18, 0x69, 0xb0, 0x0d, 0xb9, 0xbd, 0x16, 0x34, 0x09, 0x9b, 0xcc, 0xf2, 0xe2, 0x8b, 0x14, 0x7d, - 0x29, 0x29, 0x3e, 0x3e, 0xac, 0xcc, 0xf0, 0xbe, 0x09, 0x12, 0x1a, 0x0c, 0xeb, 0xf5, 0x91, 0xad, - 0xf6, 0x57, 0x89, 0x51, 0x82, 0x8c, 0xbd, 0x75, 0x4d, 0xdb, 0x2a, 0x03, 0x6c, 0xab, 0x8c, 0x66, - 0x6f, 0x13, 0xf4, 0x0a, 0x94, 0x76, 0x5d, 0xbf, 0x39, 0x33, 0xc8, 0x68, 0x5d, 0xa2, 0x73, 0x7e, - 0xdb, 0xf5, 0x9b, 0xc7, 0x87, 0x95, 0x29, 0xa3, 0x3b, 0xb4, 0x10, 0x33, 0x54, 0xfb, 0xcf, 0x2c, - 0xa8, 0x30, 0xd8, 0x8a, 0xeb, 0x91, 0x1a, 0x09, 0x23, 0x37, 0x8a, 0x89, 0x1f, 0x1b, 0x03, 0x7a, - 0x1d, 0x20, 0x22, 0x8d, 0x90, 0xc4, 0xda, 0x90, 0xaa, 0x85, 0x51, 0x57, 0x10, 0xac, 0x61, 0x51, - 0x86, 0x10, 0xed, 0x38, 0x21, 0x5b, 0x5f, 0x62, 0x60, 0x15, 0x43, 0xa8, 0x4b, 0x00, 0x4e, 0x70, - 0x0c, 0x86, 0x50, 0xec, 0xc5, 0x10, 0xd0, 0xe7, 0x61, 0x22, 0x69, 0x2c, 0x6a, 0x39, 0x0d, 0x39, - 0x80, 0x6c, 0xcb, 0xd4, 0x4d, 0x10, 0x4e, 0xe3, 0xda, 0xff, 0xc8, 0x12, 0x8b, 0x87, 0x7e, 0xf5, - 0xc7, 0xfc, 0x5b, 0xed, 0x5f, 0xb7, 0x60, 0x68, 0xd1, 0xf5, 0x9b, 0xae, 0xbf, 0x8d, 0xbe, 0x02, - 0xc3, 0xf4, 0x6c, 0x6a, 0x3a, 0xb1, 0x23, 0xf8, 0xde, 0x77, 0x68, 0x7b, 0x4b, 0x1d, 0x15, 0x73, - 0xad, 0xdd, 0x6d, 0x5a, 0x10, 0xcd, 0x51, 0x6c, 0xba, 0xdb, 0xee, 0x6e, 0xbe, 0x47, 0x1a, 0xf1, - 0x1a, 0x89, 0x9d, 0xe4, 0x73, 0x92, 0x32, 0xac, 0xa8, 0xa2, 0xdb, 0x30, 0x18, 0x3b, 0xe1, 0x36, - 0x89, 0x05, 0x03, 0xcc, 0x64, 0x54, 0xbc, 0x26, 0xa6, 0x3b, 0x92, 0xf8, 0x0d, 0x92, 0x1c, 0x0b, - 0x1b, 0xac, 0x2a, 0x16, 0x24, 0xec, 0xbf, 0x39, 0x08, 0x17, 0x97, 0xea, 0xab, 0x39, 0xeb, 0xea, - 0x2a, 0x0c, 0x36, 0x43, 0x77, 0x9f, 0x84, 0x62, 0x9c, 0x15, 0x95, 0x2a, 0x2b, 0xc5, 0x02, 0x8a, - 0x6e, 0xc0, 0x28, 0x3f, 0x90, 0x6e, 0x39, 0x7e, 0xd3, 0x93, 0x43, 0x3c, 0x2d, 0xb0, 0x47, 0xef, - 0x6b, 0x30, 0x6c, 0x60, 0x9e, 0x70, 0x51, 0x5d, 0x4d, 0x6d, 0xc6, 0xbc, 0xc3, 0xee, 0x47, 0x2c, - 0x98, 0xe4, 0xcd, 0x2c, 0xc4, 0x71, 0xe8, 0x6e, 0xb6, 0x63, 0x12, 0xcd, 0x0c, 0x30, 0x4e, 0xb7, - 0x94, 0x35, 0x5a, 0xb9, 0x23, 0x30, 0x77, 0x3f, 0x45, 0x85, 0x33, 0xc1, 0x19, 0xd1, 0xee, 0x64, - 0x1a, 0x8c, 0x3b, 0x9a, 0x45, 0x3f, 0x68, 0xc1, 0x6c, 0x23, 0xf0, 0xe3, 0x30, 0xf0, 0x3c, 0x12, - 0xd6, 0xda, 0x9b, 0x9e, 0x1b, 0xed, 0xf0, 0x75, 0x8a, 0xc9, 0x16, 0xe3, 0x04, 0x39, 0x73, 0xa8, - 0x90, 0xc4, 0x1c, 0x5e, 0x3e, 0x3a, 0xac, 0xcc, 0x2e, 0xe5, 0x92, 0xc2, 0x5d, 0x9a, 0x41, 0xbb, - 0x80, 0xe8, 0x51, 0x5a, 0x8f, 0x9d, 0x6d, 0x92, 0x34, 0x3e, 0xd4, 0x7f, 0xe3, 0x17, 0x8e, 0x0e, - 0x2b, 0x68, 0xbd, 0x83, 0x04, 0xce, 0x20, 0x8b, 0xde, 0x87, 0x69, 0x5a, 0xda, 0xf1, 0xad, 0xc3, - 0xfd, 0x37, 0x37, 0x73, 0x74, 0x58, 0x99, 0x5e, 0xcf, 0x20, 0x82, 0x33, 0x49, 0xcf, 0x2e, 0xc1, - 0xf9, 0xcc, 0xa9, 0x42, 0x93, 0x50, 0xdc, 0x25, 0x5c, 0x04, 0x29, 0x63, 0xfa, 0x13, 0x4d, 0xc3, - 0xc0, 0xbe, 0xe3, 0xb5, 0xc5, 0x2a, 0xc5, 0xfc, 0xcf, 0x67, 0x0b, 0x37, 0x2c, 0xbb, 0x01, 0xa3, - 0x4b, 0x4e, 0xcb, 0xd9, 0x74, 0x3d, 0x37, 0x76, 0x49, 0x84, 0x9e, 0x87, 0xa2, 0xd3, 0x6c, 0xb2, - 0x23, 0xb2, 0xbc, 0x78, 0xfe, 0xe8, 0xb0, 0x52, 0x5c, 0x68, 0x52, 0x5e, 0x0d, 0x0a, 0xeb, 0x00, - 0x53, 0x0c, 0xf4, 0x69, 0x28, 0x35, 0xc3, 0xa0, 0x35, 0x53, 0x60, 0x98, 0x74, 0xa8, 0x4a, 0xd5, - 0x30, 0x68, 0xa5, 0x50, 0x19, 0x8e, 0xfd, 0x5b, 0x05, 0x78, 0x66, 0x89, 0xb4, 0x76, 0x56, 0xea, - 0x39, 0x9b, 0xee, 0x1a, 0x0c, 0xef, 0x05, 0xbe, 0x1b, 0x07, 0x61, 0x24, 0x9a, 0x66, 0xa7, 0xc9, - 0x9a, 0x28, 0xc3, 0x0a, 0x8a, 0xae, 0x40, 0xa9, 0x95, 0x48, 0x02, 0xa3, 0x52, 0x8a, 0x60, 0x32, - 0x00, 0x83, 0x50, 0x8c, 0x76, 0x44, 0x42, 0x71, 0x0a, 0x2a, 0x8c, 0x7b, 0x11, 0x09, 0x31, 0x83, - 0x24, 0xec, 0x94, 0x32, 0x5a, 0xb1, 0xad, 0x52, 0xec, 0x94, 0x42, 0xb0, 0x86, 0x85, 0x6a, 0x50, - 0x8e, 0xd4, 0xa4, 0x0e, 0xf4, 0x3f, 0xa9, 0x63, 0x8c, 0xdf, 0xaa, 0x99, 0x4c, 0x88, 0x18, 0x6c, - 0x60, 0xb0, 0x27, 0xbf, 0xfd, 0x8d, 0x02, 0x20, 0x3e, 0x84, 0x7f, 0xc5, 0x06, 0xee, 0x5e, 0xe7, - 0xc0, 0x65, 0x4a, 0x5e, 0x77, 0x82, 0x86, 0xe3, 0xa5, 0x59, 0xf8, 0x69, 0x8d, 0xde, 0x4f, 0x5a, - 0x80, 0x96, 0x5c, 0xbf, 0x49, 0xc2, 0x33, 0xb8, 0x76, 0x9c, 0xec, 0x20, 0xbd, 0x03, 0xe3, 0x4b, - 0x9e, 0x4b, 0xfc, 0x78, 0xb5, 0xb6, 0x14, 0xf8, 0x5b, 0xee, 0x36, 0xfa, 0x2c, 0x8c, 0xd3, 0x5b, - 0x58, 0xd0, 0x8e, 0xeb, 0xa4, 0x11, 0xf8, 0x4c, 0x60, 0xa5, 0x77, 0x17, 0x74, 0x74, 0x58, 0x19, - 0xdf, 0x30, 0x20, 0x38, 0x85, 0x69, 0xff, 0x01, 0xfd, 0xd0, 0x60, 0xaf, 0x15, 0xf8, 0xc4, 0x8f, - 0x97, 0x02, 0xbf, 0xc9, 0x2f, 0x36, 0x9f, 0x85, 0x52, 0x4c, 0x3b, 0xce, 0x3f, 0xf2, 0xaa, 0x9c, - 0x5a, 0xda, 0xdd, 0xe3, 0xc3, 0xca, 0x85, 0xce, 0x1a, 0xec, 0x83, 0x58, 0x1d, 0xf4, 0x9d, 0x30, - 0x18, 0xc5, 0x4e, 0xdc, 0x8e, 0xc4, 0x67, 0x3f, 0x2b, 0x3f, 0xbb, 0xce, 0x4a, 0x8f, 0x0f, 0x2b, - 0x13, 0xaa, 0x1a, 0x2f, 0xc2, 0xa2, 0x02, 0x7a, 0x01, 0x86, 0xf6, 0x48, 0x14, 0x39, 0xdb, 0x52, - 0x26, 0x9d, 0x10, 0x75, 0x87, 0xd6, 0x78, 0x31, 0x96, 0x70, 0xf4, 0x1c, 0x0c, 0x90, 0x30, 0x0c, - 0x42, 0xb1, 0xaa, 0xc6, 0x04, 0xe2, 0xc0, 0x32, 0x2d, 0xc4, 0x1c, 0x66, 0xff, 0x07, 0x0b, 0x26, - 0x54, 0x5f, 0x79, 0x5b, 0x67, 0x20, 0x7c, 0x7c, 0x09, 0xa0, 0x21, 0x3f, 0x30, 0x62, 0xfc, 0x6e, - 0xe4, 0xfa, 0xd5, 0xcc, 0x23, 0xb5, 0x63, 0x18, 0x13, 0xca, 0xaa, 0x28, 0xc2, 0x1a, 0x35, 0xfb, - 0x5f, 0x5a, 0x70, 0x2e, 0xf5, 0x45, 0x77, 0xdc, 0x28, 0x46, 0xef, 0x76, 0x7c, 0xd5, 0x5c, 0x7f, - 0x5f, 0x45, 0x6b, 0xb3, 0x6f, 0x52, 0x6b, 0x4e, 0x96, 0x68, 0x5f, 0x74, 0x0b, 0x06, 0xdc, 0x98, - 0xec, 0xc9, 0x8f, 0x79, 0xae, 0xeb, 0xc7, 0xf0, 0x5e, 0x25, 0x33, 0xb2, 0x4a, 0x6b, 0x62, 0x4e, - 0xc0, 0xfe, 0xf1, 0x22, 0x94, 0xf9, 0xb2, 0x5d, 0x73, 0x5a, 0x67, 0x30, 0x17, 0xab, 0x50, 0x62, - 0xd4, 0x79, 0xc7, 0x9f, 0xcf, 0xee, 0xb8, 0xe8, 0xce, 0x1c, 0xbd, 0x59, 0x70, 0xe1, 0x45, 0x31, - 0x33, 0x5a, 0x84, 0x19, 0x09, 0xe4, 0x00, 0x6c, 0xba, 0xbe, 0x13, 0x1e, 0xd0, 0xb2, 0x99, 0x22, - 0x23, 0xf8, 0x72, 0x77, 0x82, 0x8b, 0x0a, 0x9f, 0x93, 0x55, 0x7d, 0x4d, 0x00, 0x58, 0x23, 0x3a, - 0xfb, 0x06, 0x94, 0x15, 0xf2, 0x49, 0x4e, 0xe5, 0xd9, 0xcf, 0xc3, 0x44, 0xaa, 0xad, 0x5e, 0xd5, - 0x47, 0xf5, 0x43, 0xfd, 0x1b, 0x8c, 0x0b, 0x88, 0x5e, 0x2f, 0xfb, 0xfb, 0x82, 0xdd, 0x7d, 0x00, - 0xd3, 0x5e, 0x06, 0x97, 0x15, 0x53, 0xd5, 0x3f, 0x57, 0x7e, 0x46, 0x7c, 0xf6, 0x74, 0x16, 0x14, - 0x67, 0xb6, 0x41, 0x0f, 0xaa, 0xa0, 0x45, 0xd7, 0xbc, 0xe3, 0xb1, 0xfe, 0x8a, 0xfb, 0xe2, 0x5d, - 0x51, 0x86, 0x15, 0x94, 0xb2, 0xb0, 0x69, 0xd5, 0xf9, 0xdb, 0xe4, 0xa0, 0x4e, 0x3c, 0xd2, 0x88, - 0x83, 0xf0, 0x23, 0xed, 0xfe, 0x25, 0x3e, 0xfa, 0x9c, 0x03, 0x8e, 0x08, 0x02, 0xc5, 0xdb, 0xe4, - 0x80, 0x4f, 0x85, 0xfe, 0x75, 0xc5, 0xae, 0x5f, 0xf7, 0x4b, 0x16, 0x8c, 0xa9, 0xaf, 0x3b, 0x83, - 0xad, 0xbe, 0x68, 0x6e, 0xf5, 0x4b, 0x5d, 0x17, 0x78, 0xce, 0x26, 0xff, 0x4b, 0xc6, 0xa4, 0x04, - 0x4e, 0x2d, 0x0c, 0xe8, 0xd0, 0xd0, 0x53, 0xe5, 0xa3, 0x9c, 0x90, 0x7e, 0xbe, 0xeb, 0x36, 0x39, - 0xd8, 0x08, 0xa8, 0x80, 0x93, 0xfd, 0x5d, 0xc6, 0xac, 0x95, 0xba, 0xce, 0xda, 0xaf, 0x14, 0xe0, - 0xbc, 0x1a, 0x01, 0x43, 0x84, 0xf8, 0xab, 0x3e, 0x06, 0xaf, 0xc0, 0x48, 0x93, 0x6c, 0x39, 0x6d, - 0x2f, 0x56, 0xaa, 0xa3, 0x01, 0xae, 0x3e, 0xac, 0x26, 0xc5, 0x58, 0xc7, 0x39, 0xc1, 0xb0, 0xfd, - 0xdc, 0x08, 0x3b, 0x1d, 0x62, 0x87, 0xae, 0x60, 0x2a, 0x5f, 0x6a, 0x0a, 0xc0, 0x51, 0x5d, 0x01, - 0x28, 0x94, 0x7d, 0xcf, 0xc1, 0x80, 0xbb, 0x47, 0xa5, 0x85, 0x82, 0x29, 0x04, 0xac, 0xd2, 0x42, - 0xcc, 0x61, 0xe8, 0x53, 0x30, 0xd4, 0x08, 0xf6, 0xf6, 0x1c, 0xbf, 0xc9, 0x98, 0x76, 0x79, 0x71, - 0x84, 0x0a, 0x14, 0x4b, 0xbc, 0x08, 0x4b, 0x18, 0x7a, 0x06, 0x4a, 0x4e, 0xb8, 0x1d, 0xcd, 0x94, - 0x18, 0xce, 0x30, 0x6d, 0x69, 0x21, 0xdc, 0x8e, 0x30, 0x2b, 0xa5, 0x92, 0xec, 0xc3, 0x20, 0xdc, - 0x75, 0xfd, 0xed, 0xaa, 0x1b, 0x32, 0xb1, 0x54, 0x93, 0x64, 0x1f, 0x28, 0x08, 0xd6, 0xb0, 0xd0, - 0x0a, 0x0c, 0xb4, 0x82, 0x30, 0x8e, 0x66, 0x06, 0xd9, 0x70, 0x3f, 0x9b, 0xb3, 0x95, 0xf8, 0xd7, - 0xd6, 0x82, 0x30, 0x4e, 0x3e, 0x80, 0xfe, 0x8b, 0x30, 0xaf, 0x8e, 0xbe, 0x13, 0x8a, 0xc4, 0xdf, - 0x9f, 0x19, 0x62, 0x54, 0x66, 0xb3, 0xa8, 0x2c, 0xfb, 0xfb, 0xf7, 0x9d, 0x30, 0xe1, 0x33, 0xcb, - 0xfe, 0x3e, 0xa6, 0x75, 0xd0, 0x17, 0xa1, 0x2c, 0x1f, 0x0f, 0x22, 0x71, 0xb5, 0xcc, 0x5c, 0x62, - 0x58, 0x20, 0x61, 0xf2, 0x7e, 0xdb, 0x0d, 0xc9, 0x1e, 0xf1, 0xe3, 0x28, 0x51, 0xff, 0x48, 0x68, - 0x84, 0x13, 0x6a, 0xe8, 0x8b, 0x52, 0x9f, 0xb1, 0x16, 0xb4, 0xfd, 0x38, 0x9a, 0x29, 0xb3, 0xee, - 0x65, 0x6a, 0x9a, 0xef, 0x27, 0x78, 0x69, 0x85, 0x07, 0xaf, 0x8c, 0x0d, 0x52, 0x08, 0xc3, 0x98, - 0xe7, 0xee, 0x13, 0x9f, 0x44, 0x51, 0x2d, 0x0c, 0x36, 0xc9, 0x0c, 0xb0, 0x9e, 0x5f, 0xcc, 0x56, - 0xc0, 0x06, 0x9b, 0x64, 0x71, 0xea, 0xe8, 0xb0, 0x32, 0x76, 0x47, 0xaf, 0x83, 0x4d, 0x12, 0xe8, - 0x1e, 0x8c, 0x53, 0x11, 0xda, 0x4d, 0x88, 0x8e, 0xf4, 0x22, 0xca, 0xe4, 0x67, 0x6c, 0x54, 0xc2, - 0x29, 0x22, 0xe8, 0x6d, 0x28, 0x7b, 0xee, 0x16, 0x69, 0x1c, 0x34, 0x3c, 0x32, 0x33, 0xca, 0x28, - 0x66, 0x6e, 0xab, 0x3b, 0x12, 0x89, 0x5f, 0x51, 0xd4, 0x5f, 0x9c, 0x54, 0x47, 0xf7, 0xe1, 0x42, - 0x4c, 0xc2, 0x3d, 0xd7, 0x77, 0xe8, 0x76, 0x10, 0x12, 0x2f, 0x53, 0x63, 0x8f, 0xb1, 0xf5, 0x76, - 0x59, 0x0c, 0xdd, 0x85, 0x8d, 0x4c, 0x2c, 0x9c, 0x53, 0x1b, 0xdd, 0x85, 0x09, 0xb6, 0x13, 0x6a, - 0x6d, 0xcf, 0xab, 0x05, 0x9e, 0xdb, 0x38, 0x98, 0x19, 0x67, 0x04, 0x3f, 0x25, 0xf5, 0xd4, 0xab, - 0x26, 0x98, 0xde, 0xc9, 0x93, 0x7f, 0x38, 0x5d, 0x1b, 0x6d, 0x32, 0xbd, 0x65, 0x3b, 0x74, 0xe3, - 0x03, 0xba, 0x7e, 0xc9, 0xa3, 0x78, 0x66, 0xa2, 0xeb, 0x0d, 0x57, 0x47, 0x55, 0xca, 0x4d, 0xbd, - 0x10, 0xa7, 0x09, 0xd2, 0xad, 0x1d, 0xc5, 0x4d, 0xd7, 0x9f, 0x99, 0x64, 0x1c, 0x43, 0xed, 0x8c, - 0x3a, 0x2d, 0xc4, 0x1c, 0xc6, 0x74, 0x96, 0xf4, 0xc7, 0x5d, 0xca, 0x41, 0xa7, 0x18, 0x62, 0xa2, - 0xb3, 0x94, 0x00, 0x9c, 0xe0, 0xd0, 0x63, 0x39, 0x8e, 0x0f, 0x66, 0x10, 0x43, 0x55, 0xdb, 0x65, - 0x63, 0xe3, 0x8b, 0x98, 0x96, 0xa3, 0x3b, 0x30, 0x44, 0xfc, 0xfd, 0x95, 0x30, 0xd8, 0x9b, 0x39, - 0x97, 0xbf, 0x67, 0x97, 0x39, 0x0a, 0x67, 0xe8, 0xc9, 0x15, 0x45, 0x14, 0x63, 0x49, 0x02, 0x3d, - 0x82, 0x99, 0x8c, 0x19, 0xe1, 0x13, 0x30, 0xcd, 0x26, 0xe0, 0x73, 0xa2, 0xee, 0xcc, 0x46, 0x0e, - 0xde, 0x71, 0x17, 0x18, 0xce, 0xa5, 0x8e, 0xbe, 0x17, 0xc6, 0xf8, 0x86, 0xe2, 0x0f, 0x1e, 0xd1, - 0xcc, 0x79, 0xf6, 0x35, 0x57, 0xf2, 0x37, 0x27, 0x47, 0x5c, 0x3c, 0x2f, 0x3a, 0x34, 0xa6, 0x97, - 0x46, 0xd8, 0xa4, 0x66, 0x6f, 0xc2, 0xb8, 0xe2, 0x5b, 0x6c, 0xe9, 0xa0, 0x0a, 0x0c, 0x50, 0x86, - 0x2c, 0x75, 0x0a, 0x65, 0x3a, 0x53, 0x4c, 0x51, 0x8d, 0x79, 0x39, 0x9b, 0x29, 0xf7, 0x03, 0xb2, - 0x78, 0x10, 0x13, 0x7e, 0x2f, 0x2c, 0x6a, 0x33, 0x25, 0x01, 0x38, 0xc1, 0xb1, 0xff, 0x2f, 0x97, - 0x7b, 0x12, 0xe6, 0xd8, 0xc7, 0x71, 0xf0, 0x12, 0x0c, 0xef, 0x04, 0x51, 0x4c, 0xb1, 0x59, 0x1b, - 0x03, 0x89, 0xa4, 0x73, 0x4b, 0x94, 0x63, 0x85, 0x81, 0xde, 0x84, 0xb1, 0x86, 0xde, 0x80, 0x38, - 0xcb, 0xd4, 0x10, 0x18, 0xad, 0x63, 0x13, 0x17, 0xdd, 0x80, 0x61, 0xf6, 0x5c, 0xd9, 0x08, 0x3c, - 0x71, 0x03, 0x95, 0x07, 0xf2, 0x70, 0x4d, 0x94, 0x1f, 0x6b, 0xbf, 0xb1, 0xc2, 0x46, 0x57, 0x61, - 0x90, 0x76, 0x61, 0xb5, 0x26, 0x4e, 0x11, 0xa5, 0x15, 0xb8, 0xc5, 0x4a, 0xb1, 0x80, 0xda, 0x7f, - 0xab, 0xa0, 0x8d, 0x32, 0xbd, 0x53, 0x11, 0x54, 0x83, 0xa1, 0x87, 0x8e, 0x1b, 0xbb, 0xfe, 0xb6, - 0x10, 0x17, 0x5e, 0xe8, 0x7a, 0xa4, 0xb0, 0x4a, 0x0f, 0x78, 0x05, 0x7e, 0xe8, 0x89, 0x3f, 0x58, - 0x92, 0xa1, 0x14, 0xc3, 0xb6, 0xef, 0x53, 0x8a, 0x85, 0x7e, 0x29, 0x62, 0x5e, 0x81, 0x53, 0x14, - 0x7f, 0xb0, 0x24, 0x83, 0xde, 0x05, 0x90, 0xcb, 0x92, 0x34, 0xc5, 0x33, 0xe1, 0x4b, 0xbd, 0x89, - 0x6e, 0xa8, 0x3a, 0x8b, 0xe3, 0xf4, 0x48, 0x4d, 0xfe, 0x63, 0x8d, 0x9e, 0x1d, 0x33, 0xb1, 0xaa, - 0xb3, 0x33, 0xe8, 0x7b, 0x28, 0x27, 0x70, 0xc2, 0x98, 0x34, 0x17, 0x62, 0x31, 0x38, 0x9f, 0xee, - 0x4f, 0x2a, 0xde, 0x70, 0xf7, 0x88, 0xce, 0x35, 0x04, 0x11, 0x9c, 0xd0, 0xb3, 0x7f, 0xad, 0x08, - 0x33, 0x79, 0xdd, 0xa5, 0x8b, 0x8e, 0x3c, 0x72, 0xe3, 0x25, 0x2a, 0x0d, 0x59, 0xe6, 0xa2, 0x5b, - 0x16, 0xe5, 0x58, 0x61, 0xd0, 0xd9, 0x8f, 0xdc, 0x6d, 0x79, 0xa9, 0x19, 0x48, 0x66, 0xbf, 0xce, - 0x4a, 0xb1, 0x80, 0x52, 0xbc, 0x90, 0x38, 0x91, 0x78, 0x87, 0xd6, 0x56, 0x09, 0x66, 0xa5, 0x58, - 0x40, 0x75, 0x8d, 0x49, 0xa9, 0x87, 0xc6, 0xc4, 0x18, 0xa2, 0x81, 0xd3, 0x1d, 0x22, 0xf4, 0x65, - 0x80, 0x2d, 0xd7, 0x77, 0xa3, 0x1d, 0x46, 0x7d, 0xf0, 0xc4, 0xd4, 0x95, 0x2c, 0xb5, 0xa2, 0xa8, - 0x60, 0x8d, 0x22, 0x7a, 0x1d, 0x46, 0xd4, 0x06, 0x5c, 0xad, 0x32, 0xa5, 0xbc, 0xf6, 0xc8, 0x99, - 0x70, 0xa3, 0x2a, 0xd6, 0xf1, 0xec, 0xf7, 0xd2, 0xeb, 0x45, 0xec, 0x00, 0x6d, 0x7c, 0xad, 0x7e, - 0xc7, 0xb7, 0xd0, 0x7d, 0x7c, 0xed, 0xdf, 0x2e, 0xc2, 0x84, 0xd1, 0x58, 0x3b, 0xea, 0x83, 0x67, - 0xdd, 0xa4, 0xe7, 0x9c, 0x13, 0x13, 0xb1, 0xff, 0xec, 0xde, 0x5b, 0x45, 0x3f, 0x0b, 0xe9, 0x0e, - 0xe0, 0xf5, 0xd1, 0x97, 0xa1, 0xec, 0x39, 0x11, 0xd3, 0xbe, 0x10, 0xb1, 0xef, 0xfa, 0x21, 0x96, - 0xdc, 0x23, 0x9c, 0x28, 0xd6, 0x8e, 0x1a, 0x4e, 0x3b, 0x21, 0x49, 0x0f, 0x64, 0x2a, 0xfb, 0x48, - 0x43, 0x07, 0xd5, 0x09, 0x2a, 0x20, 0x1d, 0x60, 0x0e, 0x43, 0x37, 0x60, 0x34, 0x24, 0x6c, 0x55, - 0x2c, 0x51, 0x51, 0x8e, 0x2d, 0xb3, 0x81, 0x44, 0xe6, 0xc3, 0x1a, 0x0c, 0x1b, 0x98, 0x89, 0x28, - 0x3f, 0xd8, 0x45, 0x94, 0x7f, 0x01, 0x86, 0xd8, 0x0f, 0xb5, 0x02, 0xd4, 0x6c, 0xac, 0xf2, 0x62, - 0x2c, 0xe1, 0xe9, 0x05, 0x33, 0xdc, 0xe7, 0x82, 0xf9, 0x34, 0x8c, 0x57, 0x1d, 0xb2, 0x17, 0xf8, - 0xcb, 0x7e, 0xb3, 0x15, 0xb8, 0x7e, 0x8c, 0x66, 0xa0, 0xc4, 0x4e, 0x07, 0xbe, 0xb7, 0x4b, 0x94, - 0x02, 0x2e, 0x51, 0xc1, 0xdc, 0xde, 0x86, 0xf3, 0xd5, 0xe0, 0xa1, 0xff, 0xd0, 0x09, 0x9b, 0x0b, - 0xb5, 0x55, 0xed, 0x9e, 0xbb, 0x2e, 0xef, 0x59, 0xdc, 0x70, 0x20, 0x93, 0xa7, 0x6a, 0x35, 0xf9, - 0x59, 0xbb, 0xe2, 0x7a, 0x24, 0xe7, 0x3e, 0xfd, 0x77, 0x0a, 0x46, 0x4b, 0x09, 0xbe, 0x52, 0xd2, - 0x5b, 0xb9, 0x4a, 0xfa, 0x77, 0x60, 0x78, 0xcb, 0x25, 0x5e, 0x13, 0x93, 0x2d, 0xb1, 0xc4, 0x9e, - 0xcf, 0x7f, 0x0b, 0x5d, 0xa1, 0x98, 0x52, 0x7f, 0xc2, 0x6f, 0x69, 0x2b, 0xa2, 0x32, 0x56, 0x64, - 0xd0, 0x2e, 0x4c, 0xca, 0x6b, 0x80, 0x84, 0x8a, 0x05, 0xf7, 0x42, 0xb7, 0xbb, 0x85, 0x49, 0x7c, - 0xfa, 0xe8, 0xb0, 0x32, 0x89, 0x53, 0x64, 0x70, 0x07, 0x61, 0x7a, 0x2d, 0xdb, 0xa3, 0xac, 0xb5, - 0xc4, 0x86, 0x9f, 0x5d, 0xcb, 0xd8, 0x0d, 0x93, 0x95, 0xda, 0x3f, 0x63, 0xc1, 0x53, 0x1d, 0x23, - 0x23, 0x6e, 0xda, 0xa7, 0x3c, 0x0b, 0xe9, 0x9b, 0x6f, 0xa1, 0xf7, 0xcd, 0xd7, 0xfe, 0xc7, 0x16, - 0x4c, 0x2f, 0xef, 0xb5, 0xe2, 0x83, 0xaa, 0x6b, 0x3e, 0x24, 0xbc, 0x01, 0x83, 0x7b, 0xa4, 0xe9, - 0xb6, 0xf7, 0xc4, 0xcc, 0x55, 0x24, 0xfb, 0x59, 0x63, 0xa5, 0xc7, 0x87, 0x95, 0xb1, 0x7a, 0x1c, - 0x84, 0xce, 0x36, 0xe1, 0x05, 0x58, 0xa0, 0x33, 0x26, 0xee, 0x7e, 0x40, 0xee, 0xb8, 0x7b, 0xae, - 0x7c, 0xdb, 0xee, 0xaa, 0xfd, 0x99, 0x93, 0x03, 0x3a, 0xf7, 0x4e, 0xdb, 0xf1, 0x63, 0x37, 0x3e, - 0x10, 0x6f, 0x24, 0x92, 0x08, 0x4e, 0xe8, 0xd9, 0xdf, 0xb4, 0x60, 0x42, 0xae, 0xfb, 0x85, 0x66, - 0x33, 0x24, 0x51, 0x84, 0x66, 0xa1, 0xe0, 0xb6, 0x44, 0x2f, 0x41, 0xf4, 0xb2, 0xb0, 0x5a, 0xc3, - 0x05, 0xb7, 0x85, 0x6a, 0x50, 0xe6, 0x4f, 0xe4, 0xc9, 0xe2, 0xea, 0xeb, 0xa1, 0x9d, 0xf5, 0x60, - 0x43, 0xd6, 0xc4, 0x09, 0x11, 0x29, 0xc1, 0x31, 0x9e, 0x59, 0x34, 0x1f, 0x58, 0x6e, 0x89, 0x72, - 0xac, 0x30, 0xd0, 0x35, 0x18, 0xf6, 0x83, 0x26, 0xb7, 0x58, 0xe0, 0xa7, 0x1f, 0x5b, 0xb2, 0xeb, - 0xa2, 0x0c, 0x2b, 0xa8, 0xfd, 0x63, 0x16, 0x8c, 0xca, 0x2f, 0xeb, 0x53, 0x98, 0xa4, 0x5b, 0x2b, - 0x11, 0x24, 0x93, 0xad, 0x45, 0x85, 0x41, 0x06, 0x31, 0x64, 0xc0, 0xe2, 0x49, 0x64, 0x40, 0xfb, - 0xa7, 0x0b, 0x30, 0x2e, 0xbb, 0x53, 0x6f, 0x6f, 0x46, 0x24, 0x46, 0x1b, 0x50, 0x76, 0xf8, 0x90, - 0x13, 0xb9, 0x62, 0x9f, 0xcb, 0xbe, 0x7c, 0x18, 0xf3, 0x93, 0x1c, 0xcb, 0x0b, 0xb2, 0x36, 0x4e, - 0x08, 0x21, 0x0f, 0xa6, 0xfc, 0x20, 0x66, 0x2c, 0x5a, 0xc1, 0xbb, 0x29, 0xf1, 0xd3, 0xd4, 0x2f, - 0x0a, 0xea, 0x53, 0xeb, 0x69, 0x2a, 0xb8, 0x93, 0x30, 0x5a, 0x96, 0x0a, 0x8f, 0x62, 0xfe, 0x75, - 0x43, 0x9f, 0x85, 0x6c, 0x7d, 0x87, 0xfd, 0x9b, 0x16, 0x94, 0x25, 0xda, 0x59, 0xbc, 0xd7, 0xac, - 0xc1, 0x50, 0xc4, 0x26, 0x41, 0x0e, 0x8d, 0xdd, 0xad, 0xe3, 0x7c, 0xbe, 0x92, 0x93, 0x87, 0xff, - 0x8f, 0xb0, 0xa4, 0xc1, 0x34, 0xb6, 0xaa, 0xfb, 0x1f, 0x13, 0x8d, 0xad, 0xea, 0x4f, 0xce, 0x09, - 0xf3, 0xdf, 0x58, 0x9f, 0xb5, 0x6b, 0x2d, 0x15, 0x90, 0x5a, 0x21, 0xd9, 0x72, 0x1f, 0xa5, 0x05, - 0xa4, 0x1a, 0x2b, 0xc5, 0x02, 0x8a, 0xde, 0x85, 0xd1, 0x86, 0x54, 0x74, 0x26, 0x6c, 0xe0, 0x6a, - 0x57, 0xb5, 0xb1, 0x7a, 0x61, 0xe0, 0xd6, 0x8c, 0x4b, 0x5a, 0x7d, 0x6c, 0x50, 0x33, 0x5f, 0xd1, - 0x8b, 0xbd, 0x5e, 0xd1, 0x13, 0xba, 0xb9, 0xef, 0xc0, 0xf6, 0xcf, 0x5a, 0x30, 0xc8, 0xd5, 0x65, - 0xfd, 0xe9, 0x17, 0xb5, 0x07, 0x97, 0x64, 0xec, 0xee, 0xd3, 0x42, 0xf1, 0x80, 0x82, 0xd6, 0xa0, - 0xcc, 0x7e, 0x30, 0xb5, 0x41, 0x31, 0xdf, 0x8c, 0x93, 0xb7, 0xaa, 0x77, 0xf0, 0xbe, 0xac, 0x86, - 0x13, 0x0a, 0xf6, 0x4f, 0x14, 0x29, 0xab, 0x4a, 0x50, 0x8d, 0x13, 0xdc, 0x7a, 0x72, 0x27, 0x78, - 0xe1, 0x49, 0x9d, 0xe0, 0xdb, 0x30, 0xd1, 0xd0, 0x9e, 0x67, 0x92, 0x99, 0xbc, 0xd6, 0x75, 0x91, - 0x68, 0x2f, 0x39, 0x5c, 0x65, 0xb4, 0x64, 0x12, 0xc1, 0x69, 0xaa, 0xe8, 0x7b, 0x60, 0x94, 0xcf, - 0xb3, 0x68, 0xa5, 0xc4, 0x5a, 0xf9, 0x54, 0xfe, 0x7a, 0xd1, 0x9b, 0x60, 0x2b, 0xb1, 0xae, 0x55, - 0xc7, 0x06, 0x31, 0xfb, 0xd7, 0x86, 0x61, 0x60, 0x79, 0x9f, 0xf8, 0xf1, 0x19, 0x30, 0xa4, 0x06, - 0x8c, 0xbb, 0xfe, 0x7e, 0xe0, 0xed, 0x93, 0x26, 0x87, 0x9f, 0xe4, 0x70, 0xbd, 0x20, 0x48, 0x8f, - 0xaf, 0x1a, 0x24, 0x70, 0x8a, 0xe4, 0x93, 0xb8, 0x61, 0xde, 0x84, 0x41, 0x3e, 0xf7, 0xe2, 0x7a, - 0x99, 0xa9, 0x0c, 0x66, 0x83, 0x28, 0x76, 0x41, 0x72, 0xfb, 0xe5, 0xda, 0x67, 0x51, 0x1d, 0xbd, - 0x07, 0xe3, 0x5b, 0x6e, 0x18, 0xc5, 0xf4, 0x6a, 0x18, 0xc5, 0xce, 0x5e, 0xeb, 0x31, 0x6e, 0x94, - 0x6a, 0x1c, 0x56, 0x0c, 0x4a, 0x38, 0x45, 0x19, 0x6d, 0xc3, 0x18, 0xbd, 0xe4, 0x24, 0x4d, 0x0d, - 0x9d, 0xb8, 0x29, 0xa5, 0x32, 0xba, 0xa3, 0x13, 0xc2, 0x26, 0x5d, 0xca, 0x4c, 0x1a, 0xec, 0x52, - 0x34, 0xcc, 0x24, 0x0a, 0xc5, 0x4c, 0xf8, 0x6d, 0x88, 0xc3, 0x28, 0x4f, 0x62, 0x86, 0x17, 0x65, - 0x93, 0x27, 0x69, 0xe6, 0x15, 0x5f, 0x81, 0x32, 0xa1, 0x43, 0x48, 0x09, 0x0b, 0xc5, 0xf8, 0x7c, - 0x7f, 0x7d, 0x5d, 0x73, 0x1b, 0x61, 0x60, 0xde, 0xe5, 0x97, 0x25, 0x25, 0x9c, 0x10, 0x45, 0x4b, - 0x30, 0x18, 0x91, 0xd0, 0x25, 0x91, 0x50, 0x91, 0x77, 0x99, 0x46, 0x86, 0xc6, 0xed, 0x7d, 0xf9, - 0x6f, 0x2c, 0xaa, 0xd2, 0xe5, 0xe5, 0xb0, 0xdb, 0x10, 0xd3, 0x8a, 0x6b, 0xcb, 0x6b, 0x81, 0x95, - 0x62, 0x01, 0x45, 0x6f, 0xc3, 0x50, 0x48, 0x3c, 0xa6, 0x2c, 0x1a, 0xeb, 0x7f, 0x91, 0x73, 0xdd, - 0x13, 0xaf, 0x87, 0x25, 0x01, 0x74, 0x1b, 0x50, 0x48, 0xa8, 0x0c, 0xe1, 0xfa, 0xdb, 0xca, 0x1c, - 0x41, 0xe8, 0xba, 0x9f, 0x16, 0xed, 0x9f, 0xc3, 0x09, 0x86, 0xb4, 0x04, 0xc4, 0x19, 0xd5, 0xd0, - 0x4d, 0x98, 0x52, 0xa5, 0xab, 0x7e, 0x14, 0x3b, 0x7e, 0x83, 0x30, 0x35, 0x77, 0x39, 0x91, 0x8a, - 0x70, 0x1a, 0x01, 0x77, 0xd6, 0xb1, 0xbf, 0x4e, 0xc5, 0x19, 0x3a, 0x5a, 0x67, 0x20, 0x0b, 0xbc, - 0x65, 0xca, 0x02, 0x17, 0x73, 0x67, 0x2e, 0x47, 0x0e, 0x38, 0xb2, 0x60, 0x44, 0x9b, 0xd9, 0x64, - 0xcd, 0x5a, 0x5d, 0xd6, 0x6c, 0x1b, 0x26, 0xe9, 0x4a, 0xbf, 0xbb, 0xc9, 0x5c, 0x5f, 0x9a, 0x6c, - 0x61, 0x16, 0x1e, 0x6f, 0x61, 0x2a, 0xa3, 0xd1, 0x3b, 0x29, 0x82, 0xb8, 0xa3, 0x09, 0xf4, 0x86, - 0xd4, 0x9c, 0x14, 0x0d, 0x33, 0x23, 0xae, 0x15, 0x39, 0x3e, 0xac, 0x4c, 0x6a, 0x1f, 0xa2, 0x6b, - 0x4a, 0xec, 0xaf, 0xc8, 0x6f, 0xe4, 0xcc, 0x66, 0x1e, 0xca, 0x0d, 0xb5, 0x58, 0x2c, 0xd3, 0xf0, - 0x59, 0x2d, 0x07, 0x9c, 0xe0, 0xd0, 0x3d, 0x4a, 0xaf, 0x20, 0x69, 0xcb, 0x38, 0x7a, 0x41, 0xc1, - 0x0c, 0x62, 0xbf, 0x0a, 0xb0, 0xfc, 0x88, 0x34, 0xf8, 0x52, 0xd7, 0x1f, 0x20, 0xad, 0xfc, 0x07, - 0x48, 0xfb, 0x3f, 0x59, 0x30, 0xbe, 0xb2, 0x64, 0x5c, 0x13, 0xe7, 0x00, 0xf8, 0xdd, 0xe8, 0xc1, - 0x83, 0x75, 0xa9, 0x5b, 0xe7, 0xea, 0x51, 0x55, 0x8a, 0x35, 0x0c, 0x74, 0x11, 0x8a, 0x5e, 0xdb, - 0x17, 0x57, 0x96, 0xa1, 0xa3, 0xc3, 0x4a, 0xf1, 0x4e, 0xdb, 0xc7, 0xb4, 0x4c, 0x33, 0x46, 0x2b, - 0xf6, 0x6d, 0x8c, 0xd6, 0xd3, 0xa5, 0x05, 0x55, 0x60, 0xe0, 0xe1, 0x43, 0xb7, 0xc9, 0x0d, 0x87, - 0x85, 0xde, 0xff, 0xc1, 0x83, 0xd5, 0x6a, 0x84, 0x79, 0xb9, 0xfd, 0xb5, 0x22, 0xcc, 0xae, 0x78, - 0xe4, 0xd1, 0x87, 0x34, 0x9e, 0xee, 0xd7, 0x94, 0xee, 0x64, 0xf2, 0xe2, 0x49, 0xed, 0x06, 0x7b, - 0x8f, 0xc7, 0x16, 0x0c, 0xf1, 0xc7, 0x6c, 0x69, 0x4a, 0xfd, 0x66, 0x56, 0xeb, 0xf9, 0x03, 0x32, - 0xc7, 0x1f, 0xc5, 0x85, 0x09, 0xb5, 0x3a, 0x69, 0x45, 0x29, 0x96, 0xc4, 0x67, 0x3f, 0x0b, 0xa3, - 0x3a, 0xe6, 0x89, 0x2c, 0x78, 0xff, 0xff, 0x22, 0x4c, 0xd2, 0x1e, 0x3c, 0xd1, 0x89, 0xb8, 0xd7, - 0x39, 0x11, 0xa7, 0x6d, 0xc5, 0xd9, 0x7b, 0x36, 0xde, 0x4d, 0xcf, 0xc6, 0x2b, 0x79, 0xb3, 0x71, - 0xd6, 0x73, 0xf0, 0x83, 0x16, 0x9c, 0x5b, 0xf1, 0x82, 0xc6, 0x6e, 0xca, 0xc0, 0xf4, 0x75, 0x18, - 0xa1, 0x7c, 0x3c, 0x32, 0x3c, 0x37, 0x0c, 0x5f, 0x1e, 0x01, 0xc2, 0x3a, 0x9e, 0x56, 0xed, 0xde, - 0xbd, 0xd5, 0x6a, 0x96, 0x0b, 0x90, 0x00, 0x61, 0x1d, 0xcf, 0xfe, 0x5d, 0x0b, 0x2e, 0xdd, 0x5c, - 0x5a, 0x4e, 0x96, 0x62, 0x87, 0x17, 0x12, 0xbd, 0x05, 0x36, 0xb5, 0xae, 0x24, 0xb7, 0xc0, 0x2a, - 0xeb, 0x85, 0x80, 0x7e, 0x5c, 0x3c, 0xec, 0x7e, 0xc1, 0x82, 0x73, 0x37, 0xdd, 0x98, 0x1e, 0xcb, - 0x69, 0x7f, 0x18, 0x7a, 0x2e, 0x47, 0x6e, 0x1c, 0x84, 0x07, 0x69, 0x7f, 0x18, 0xac, 0x20, 0x58, - 0xc3, 0xe2, 0x2d, 0xef, 0xbb, 0x11, 0xed, 0x69, 0xc1, 0x54, 0x45, 0x61, 0x51, 0x8e, 0x15, 0x06, - 0xfd, 0xb0, 0xa6, 0x1b, 0xb2, 0xab, 0xc4, 0x81, 0xe0, 0xb0, 0xea, 0xc3, 0xaa, 0x12, 0x80, 0x13, - 0x1c, 0xfb, 0x67, 0x2c, 0x38, 0x7f, 0xd3, 0x6b, 0x47, 0x31, 0x09, 0xb7, 0x22, 0xa3, 0xb3, 0xaf, - 0x42, 0x99, 0xc8, 0xeb, 0xba, 0xe8, 0xab, 0x12, 0x30, 0xd5, 0x3d, 0x9e, 0x3b, 0xe3, 0x28, 0xbc, - 0x3e, 0xac, 0xb5, 0x4f, 0x66, 0x65, 0xfc, 0xcb, 0x05, 0x18, 0xbb, 0xb5, 0xb1, 0x51, 0xbb, 0x49, - 0x62, 0x71, 0x8a, 0xf5, 0x56, 0x35, 0x63, 0x4d, 0x63, 0xd6, 0xed, 0x52, 0xd4, 0x8e, 0x5d, 0x6f, - 0x8e, 0x7b, 0x7f, 0xce, 0xad, 0xfa, 0xf1, 0xdd, 0xb0, 0x1e, 0x87, 0xae, 0xbf, 0x9d, 0xa9, 0x63, - 0x93, 0x67, 0x6d, 0x31, 0xef, 0xac, 0x45, 0xaf, 0xc2, 0x20, 0x73, 0x3f, 0x95, 0xd7, 0x93, 0xa7, - 0xd5, 0x9d, 0x82, 0x95, 0x1e, 0x1f, 0x56, 0xca, 0xf7, 0xf0, 0x2a, 0xff, 0x83, 0x05, 0x2a, 0xba, - 0x07, 0x23, 0x3b, 0x71, 0xdc, 0xba, 0x45, 0x9c, 0x26, 0x09, 0x25, 0x77, 0xb8, 0x9c, 0xc5, 0x1d, - 0xe8, 0x20, 0x70, 0xb4, 0x64, 0x43, 0x25, 0x65, 0x11, 0xd6, 0xe9, 0xd8, 0x75, 0x80, 0x04, 0x76, - 0x4a, 0xfa, 0x05, 0xfb, 0x8f, 0x2d, 0x18, 0xe2, 0x9e, 0x40, 0x21, 0xfa, 0x1c, 0x94, 0xc8, 0x23, - 0xd2, 0x10, 0x92, 0x63, 0x66, 0x87, 0x13, 0xc1, 0x83, 0x6b, 0xcb, 0xe9, 0x7f, 0xcc, 0x6a, 0xa1, - 0x5b, 0x30, 0x44, 0x7b, 0x7b, 0x53, 0xb9, 0x45, 0x3d, 0x9b, 0xf7, 0xc5, 0x6a, 0xda, 0xb9, 0xac, - 0x22, 0x8a, 0xb0, 0xac, 0xce, 0x34, 0xbf, 0x8d, 0x56, 0x9d, 0x32, 0xb0, 0xb8, 0xdb, 0x39, 0xbb, - 0xb1, 0x54, 0xe3, 0x48, 0x82, 0x1a, 0xd7, 0xfc, 0xca, 0x42, 0x9c, 0x10, 0xb1, 0x37, 0xa0, 0x4c, - 0x27, 0x75, 0xc1, 0x73, 0x9d, 0xee, 0x4a, 0xe7, 0x17, 0xa1, 0x2c, 0x15, 0xc0, 0x91, 0x70, 0x26, - 0x61, 0x54, 0xa5, 0x7e, 0x38, 0xc2, 0x09, 0xdc, 0xde, 0x82, 0x69, 0xf6, 0xf2, 0xef, 0xc4, 0x3b, - 0xc6, 0x1e, 0xeb, 0xbd, 0x98, 0x5f, 0x12, 0x17, 0x31, 0x3e, 0x33, 0x33, 0x9a, 0xf5, 0xfb, 0xa8, - 0xa4, 0x98, 0x5c, 0xca, 0xec, 0x3f, 0x2d, 0xc1, 0xd3, 0xab, 0xf5, 0x7c, 0x27, 0xb1, 0x1b, 0x30, - 0xca, 0xc5, 0x34, 0xba, 0xb4, 0x1d, 0x4f, 0xb4, 0xab, 0xde, 0xc5, 0x36, 0x34, 0x18, 0x36, 0x30, - 0xd1, 0x25, 0x28, 0xba, 0xef, 0xfb, 0x69, 0x43, 0xd2, 0xd5, 0x77, 0xd6, 0x31, 0x2d, 0xa7, 0x60, - 0x2a, 0xf1, 0x71, 0x56, 0xaa, 0xc0, 0x4a, 0xea, 0x7b, 0x0b, 0xc6, 0xdd, 0xa8, 0x11, 0xb9, 0xab, - 0x3e, 0xe5, 0x33, 0x89, 0x83, 0x61, 0xa2, 0x24, 0xa0, 0x9d, 0x56, 0x50, 0x9c, 0xc2, 0xd6, 0xf8, - 0xfa, 0x40, 0xdf, 0x52, 0x63, 0x4f, 0xef, 0x0a, 0x2a, 0x10, 0xb7, 0xd8, 0xd7, 0x45, 0xcc, 0xa8, - 0x4d, 0x08, 0xc4, 0xfc, 0x83, 0x23, 0x2c, 0x61, 0xf4, 0x06, 0xd6, 0xd8, 0x71, 0x5a, 0x0b, 0xed, - 0x78, 0xa7, 0xea, 0x46, 0x8d, 0x60, 0x9f, 0x84, 0x07, 0xec, 0xf2, 0x3c, 0x9c, 0xdc, 0xc0, 0x14, - 0x60, 0xe9, 0xd6, 0x42, 0x8d, 0x62, 0xe2, 0xce, 0x3a, 0xa6, 0x54, 0x08, 0xa7, 0x21, 0x15, 0x2e, - 0xc0, 0x84, 0x6c, 0xa6, 0x4e, 0x22, 0x76, 0x46, 0x8c, 0xb0, 0x8e, 0x29, 0xd7, 0x5f, 0x51, 0xac, - 0xba, 0x95, 0xc6, 0x47, 0x6f, 0xc0, 0x98, 0xeb, 0xbb, 0xb1, 0xeb, 0xc4, 0x41, 0xc8, 0x4e, 0x58, - 0x7e, 0x4f, 0x66, 0x96, 0x6c, 0xab, 0x3a, 0x00, 0x9b, 0x78, 0xf6, 0x9f, 0x94, 0x60, 0x8a, 0x4d, - 0xdb, 0xb7, 0x57, 0xd8, 0xc7, 0x66, 0x85, 0xdd, 0xeb, 0x5c, 0x61, 0xa7, 0x21, 0xee, 0x7e, 0x94, - 0xcb, 0xec, 0x3d, 0x28, 0x2b, 0x5b, 0x60, 0x69, 0xce, 0x6e, 0xe5, 0x98, 0xb3, 0xf7, 0x96, 0x3e, - 0xe4, 0x33, 0x6e, 0x31, 0xf3, 0x19, 0xf7, 0xef, 0x5a, 0x90, 0x98, 0x44, 0xa2, 0x5b, 0x50, 0x6e, - 0x05, 0xcc, 0xec, 0x20, 0x94, 0xb6, 0x3c, 0x4f, 0x67, 0x1e, 0x54, 0xfc, 0x50, 0xe4, 0xe3, 0x57, - 0x93, 0x35, 0x70, 0x52, 0x19, 0x2d, 0xc2, 0x50, 0x2b, 0x24, 0xf5, 0x98, 0xb9, 0x1d, 0xf6, 0xa4, - 0xc3, 0xd7, 0x08, 0xc7, 0xc7, 0xb2, 0xa2, 0xfd, 0x2b, 0x16, 0x00, 0x7f, 0x29, 0x75, 0xfc, 0x6d, - 0x72, 0x06, 0xda, 0xdf, 0x2a, 0x94, 0xa2, 0x16, 0x69, 0x74, 0x33, 0x08, 0x49, 0xfa, 0x53, 0x6f, - 0x91, 0x46, 0x32, 0xe0, 0xf4, 0x1f, 0x66, 0xb5, 0xed, 0xbf, 0x06, 0x30, 0x9e, 0xa0, 0xad, 0xc6, - 0x64, 0x0f, 0xbd, 0x6c, 0x38, 0x75, 0x5d, 0x4c, 0x39, 0x75, 0x95, 0x19, 0xb6, 0xa6, 0x68, 0x7c, - 0x0f, 0x8a, 0x7b, 0xce, 0x23, 0xa1, 0x49, 0x7a, 0xb1, 0x7b, 0x37, 0x28, 0xfd, 0xb9, 0x35, 0xe7, - 0x11, 0xbf, 0x33, 0xbd, 0x28, 0x17, 0xc8, 0x9a, 0xf3, 0xe8, 0x98, 0x9b, 0x7d, 0x30, 0x26, 0x75, - 0xc7, 0x8d, 0xe2, 0xaf, 0xfe, 0x97, 0xe4, 0x3f, 0x5b, 0x76, 0xb4, 0x11, 0xd6, 0x96, 0xeb, 0x8b, - 0x77, 0xc3, 0xbe, 0xda, 0x72, 0xfd, 0x74, 0x5b, 0xae, 0xdf, 0x47, 0x5b, 0xae, 0x8f, 0x3e, 0x80, - 0x21, 0xf1, 0x46, 0xcf, 0x6c, 0xbd, 0x4d, 0x2d, 0x55, 0x5e, 0x7b, 0xe2, 0x89, 0x9f, 0xb7, 0x39, - 0x2f, 0xef, 0x84, 0xa2, 0xb4, 0x67, 0xbb, 0xb2, 0x41, 0xf4, 0xb7, 0x2d, 0x18, 0x17, 0xbf, 0x31, - 0x79, 0xbf, 0x4d, 0xa2, 0x58, 0xc8, 0x9e, 0x9f, 0xe9, 0xbf, 0x0f, 0xa2, 0x22, 0xef, 0xca, 0x67, - 0x24, 0x9b, 0x35, 0x81, 0x3d, 0x7b, 0x94, 0xea, 0x05, 0xfa, 0xa7, 0x16, 0x4c, 0xef, 0x39, 0x8f, - 0x78, 0x8b, 0xbc, 0x0c, 0x3b, 0xb1, 0x1b, 0x08, 0xdb, 0xf5, 0xcf, 0xf5, 0x37, 0xfd, 0x1d, 0xd5, - 0x79, 0x27, 0xa5, 0x99, 0xeb, 0x74, 0x16, 0x4a, 0xcf, 0xae, 0x66, 0xf6, 0x6b, 0x76, 0x0b, 0x86, - 0xe5, 0x7a, 0xcb, 0xb8, 0x79, 0x57, 0x75, 0xc1, 0xfa, 0xc4, 0x26, 0x12, 0xba, 0x67, 0x15, 0x6d, - 0x47, 0xac, 0xb5, 0x27, 0xda, 0xce, 0x7b, 0x30, 0xaa, 0xaf, 0xb1, 0x27, 0xda, 0xd6, 0xfb, 0x70, - 0x2e, 0x63, 0x2d, 0x3d, 0xd1, 0x26, 0x1f, 0xc2, 0xc5, 0xdc, 0xf5, 0xf1, 0x24, 0x1b, 0xb6, 0x7f, - 0xd9, 0xd2, 0xf9, 0xe0, 0x19, 0xa8, 0xe0, 0x97, 0x4c, 0x15, 0xfc, 0xe5, 0xee, 0x3b, 0x27, 0x47, - 0x0f, 0xff, 0xae, 0xde, 0x69, 0xca, 0xd5, 0xd1, 0xdb, 0x30, 0xe8, 0xd1, 0x12, 0x69, 0x1c, 0x62, - 0xf7, 0xde, 0x91, 0x89, 0x2c, 0xc5, 0xca, 0x23, 0x2c, 0x28, 0xd8, 0xbf, 0x6e, 0x41, 0xe9, 0x0c, - 0x46, 0x02, 0x9b, 0x23, 0xf1, 0x72, 0x2e, 0x69, 0x11, 0x46, 0x6a, 0x0e, 0x3b, 0x0f, 0x97, 0x65, - 0xa8, 0xac, 0x9c, 0x81, 0xf9, 0x3e, 0x38, 0x77, 0x27, 0x70, 0x9a, 0x8b, 0x8e, 0xe7, 0xf8, 0x0d, - 0x12, 0xae, 0xfa, 0xdb, 0x3d, 0xad, 0x94, 0x74, 0x9b, 0xa2, 0x42, 0x2f, 0x9b, 0x22, 0x7b, 0x07, - 0x90, 0xde, 0x80, 0xb0, 0xe3, 0xc4, 0x30, 0xe4, 0xf2, 0xa6, 0xc4, 0xf0, 0x3f, 0x9f, 0x2d, 0xdd, - 0x75, 0xf4, 0x4c, 0xb3, 0x50, 0xe4, 0x05, 0x58, 0x12, 0xb2, 0x6f, 0x40, 0xa6, 0xef, 0x56, 0x6f, - 0xb5, 0x81, 0xfd, 0x3a, 0x4c, 0xb1, 0x9a, 0x27, 0xbb, 0xd2, 0xda, 0x3f, 0x62, 0xc1, 0xc4, 0x7a, - 0x2a, 0x1e, 0xc0, 0x55, 0xf6, 0xd6, 0x97, 0xa1, 0xf7, 0xad, 0xb3, 0x52, 0x2c, 0xa0, 0xa7, 0xae, - 0x5f, 0xfa, 0x4b, 0x0b, 0xca, 0x2a, 0x92, 0xcd, 0x19, 0x08, 0x55, 0x4b, 0x86, 0x50, 0x95, 0xa9, - 0xf7, 0x50, 0xdd, 0xc9, 0x93, 0xa9, 0xd0, 0x6d, 0xe5, 0xd9, 0xde, 0x45, 0xe5, 0x91, 0x90, 0xe1, - 0x7e, 0xd0, 0xe3, 0xa6, 0xfb, 0xbb, 0xf4, 0x75, 0x67, 0x66, 0x42, 0x0a, 0xf7, 0x63, 0x62, 0x26, - 0xa4, 0xfa, 0x93, 0xb3, 0xfb, 0x6a, 0x5a, 0x97, 0x19, 0x57, 0xfa, 0x2e, 0x66, 0xf6, 0xed, 0x78, - 0xee, 0x07, 0x44, 0x05, 0x94, 0xa8, 0x08, 0x33, 0x6e, 0x51, 0x7a, 0x7c, 0x58, 0x19, 0x53, 0xff, - 0x78, 0xd4, 0xa1, 0xa4, 0x8a, 0x7d, 0x0b, 0x26, 0x52, 0x03, 0x86, 0x5e, 0x87, 0x81, 0xd6, 0x8e, - 0x13, 0x91, 0x94, 0x69, 0xe4, 0x40, 0x8d, 0x16, 0x1e, 0x1f, 0x56, 0xc6, 0x55, 0x05, 0x56, 0x82, - 0x39, 0xb6, 0xfd, 0x3f, 0x2d, 0x28, 0xad, 0x07, 0xcd, 0xb3, 0x58, 0x4c, 0x6f, 0x19, 0x8b, 0xe9, - 0x99, 0xbc, 0x98, 0x6d, 0xb9, 0xeb, 0x68, 0x25, 0xb5, 0x8e, 0x2e, 0xe7, 0x52, 0xe8, 0xbe, 0x84, - 0xf6, 0x60, 0x84, 0x45, 0x82, 0x13, 0xa6, 0x9a, 0xaf, 0x1a, 0xf2, 0x7d, 0x25, 0x25, 0xdf, 0x4f, - 0x68, 0xa8, 0x9a, 0x94, 0xff, 0x02, 0x0c, 0x09, 0x73, 0xc1, 0xb4, 0x81, 0xbb, 0xc0, 0xc5, 0x12, - 0x6e, 0xff, 0x6c, 0x11, 0x8c, 0xc8, 0x73, 0xe8, 0x37, 0x2d, 0x98, 0x0b, 0xb9, 0xc7, 0x60, 0xb3, - 0xda, 0x0e, 0x5d, 0x7f, 0xbb, 0xde, 0xd8, 0x21, 0xcd, 0xb6, 0xe7, 0xfa, 0xdb, 0xab, 0xdb, 0x7e, - 0xa0, 0x8a, 0x97, 0x1f, 0x91, 0x46, 0x9b, 0xe9, 0xfc, 0x7b, 0x84, 0xb9, 0x53, 0xe6, 0x38, 0xd7, - 0x8f, 0x0e, 0x2b, 0x73, 0xf8, 0x44, 0xb4, 0xf1, 0x09, 0xfb, 0x82, 0x7e, 0xd7, 0x82, 0x79, 0x1e, - 0x90, 0xad, 0xff, 0xfe, 0x77, 0xb9, 0x0d, 0xd5, 0x24, 0xa9, 0x84, 0xc8, 0x06, 0x09, 0xf7, 0x16, - 0xdf, 0x10, 0x03, 0x3a, 0x5f, 0x3b, 0x59, 0x5b, 0xf8, 0xa4, 0x9d, 0xb3, 0xff, 0x4d, 0x11, 0xc6, - 0xe8, 0x28, 0x26, 0x71, 0x3c, 0x5e, 0x37, 0x96, 0xc4, 0xb3, 0xa9, 0x25, 0x31, 0x65, 0x20, 0x9f, - 0x4e, 0x08, 0x8f, 0x08, 0xa6, 0x3c, 0x27, 0x8a, 0x6f, 0x11, 0x27, 0x8c, 0x37, 0x89, 0xc3, 0xcd, - 0x54, 0x8a, 0x27, 0x36, 0xa9, 0x51, 0xea, 0x97, 0x3b, 0x69, 0x62, 0xb8, 0x93, 0x3e, 0xda, 0x07, - 0xc4, 0x6c, 0x6d, 0x42, 0xc7, 0x8f, 0xf8, 0xb7, 0xb8, 0xe2, 0x3d, 0xe0, 0x64, 0xad, 0xce, 0x8a, - 0x56, 0xd1, 0x9d, 0x0e, 0x6a, 0x38, 0xa3, 0x05, 0xcd, 0x86, 0x6a, 0xa0, 0x5f, 0x1b, 0xaa, 0xc1, - 0x1e, 0x5e, 0x24, 0x7b, 0x30, 0x29, 0x66, 0x65, 0xcb, 0xdd, 0x16, 0x87, 0xf4, 0x17, 0x53, 0x36, - 0x96, 0x56, 0xff, 0x86, 0x32, 0x3d, 0x0c, 0x2c, 0xed, 0xef, 0x87, 0x73, 0xb4, 0x39, 0xd3, 0xe7, - 0x21, 0x42, 0x04, 0x26, 0x76, 0xdb, 0x9b, 0xc4, 0x23, 0xb1, 0x2c, 0x13, 0x8d, 0x66, 0x8a, 0x9d, - 0x66, 0xed, 0x44, 0x39, 0x75, 0xdb, 0x24, 0x81, 0xd3, 0x34, 0xed, 0x9f, 0xb7, 0x80, 0x59, 0x6b, - 0x9f, 0xc1, 0xf1, 0xf7, 0x79, 0xf3, 0xf8, 0x9b, 0xc9, 0xe3, 0x40, 0x39, 0x27, 0xdf, 0x6b, 0x7c, - 0x5a, 0x6a, 0x61, 0xf0, 0xe8, 0x40, 0xbc, 0xfa, 0xf6, 0x21, 0x71, 0xfd, 0x1f, 0x8b, 0x6f, 0x48, - 0xe5, 0x40, 0x8d, 0x7e, 0x00, 0x86, 0x1b, 0x4e, 0xcb, 0x69, 0xf0, 0x90, 0x9f, 0xb9, 0xda, 0x07, - 0xa3, 0xd2, 0xdc, 0x92, 0xa8, 0xc1, 0x6f, 0xd3, 0xdf, 0x21, 0xbf, 0x52, 0x16, 0xf7, 0xbc, 0x41, - 0xab, 0x26, 0x67, 0x77, 0x61, 0xcc, 0x20, 0xf6, 0x44, 0xaf, 0x5e, 0x3f, 0xc0, 0x8f, 0x0b, 0x15, - 0x8f, 0x63, 0x0f, 0xa6, 0x7c, 0xed, 0x3f, 0x65, 0x8e, 0x52, 0x9c, 0xfe, 0x64, 0xaf, 0x03, 0x81, - 0x71, 0x52, 0xcd, 0x1a, 0x3d, 0x45, 0x06, 0x77, 0x52, 0xb6, 0xff, 0xbe, 0x05, 0x4f, 0xe9, 0x88, - 0x9a, 0x6f, 0x7b, 0x2f, 0x7d, 0x66, 0x15, 0x86, 0x83, 0x16, 0x09, 0x9d, 0x38, 0x08, 0x05, 0x07, - 0xbc, 0x26, 0x07, 0xfd, 0xae, 0x28, 0x3f, 0x16, 0xb1, 0xd7, 0x24, 0x75, 0x59, 0x8e, 0x55, 0x4d, - 0x64, 0xc3, 0x20, 0x1b, 0x8c, 0x48, 0xc4, 0x1d, 0x60, 0x66, 0x72, 0xec, 0x69, 0x2f, 0xc2, 0x02, - 0x62, 0xff, 0xa9, 0xc5, 0x17, 0x96, 0xde, 0x75, 0xf4, 0x3e, 0x4c, 0xee, 0x39, 0x71, 0x63, 0x67, - 0xf9, 0x51, 0x2b, 0xe4, 0x6a, 0x5c, 0x39, 0x4e, 0x2f, 0xf6, 0x1a, 0x27, 0xed, 0x23, 0x13, 0x63, - 0xaa, 0xb5, 0x14, 0x31, 0xdc, 0x41, 0x1e, 0x6d, 0xc2, 0x08, 0x2b, 0x63, 0x56, 0xbb, 0x51, 0xb7, - 0x63, 0x2e, 0xaf, 0x35, 0xf5, 0x3a, 0xba, 0x96, 0xd0, 0xc1, 0x3a, 0x51, 0xfb, 0xab, 0x45, 0xbe, - 0xdb, 0x99, 0xe4, 0xf8, 0x02, 0x0c, 0xb5, 0x82, 0xe6, 0xd2, 0x6a, 0x15, 0x8b, 0x59, 0x50, 0x2c, - 0xb1, 0xc6, 0x8b, 0xb1, 0x84, 0xa3, 0x37, 0x01, 0xc8, 0xa3, 0x98, 0x84, 0xbe, 0xe3, 0x29, 0xe3, - 0x06, 0x65, 0xce, 0x57, 0x0d, 0xd6, 0x83, 0xf8, 0x5e, 0x44, 0xbe, 0x6f, 0x59, 0xa1, 0x60, 0x0d, - 0x1d, 0x5d, 0x07, 0x68, 0x85, 0xc1, 0xbe, 0xdb, 0x64, 0x6e, 0x60, 0x45, 0xf3, 0xe9, 0xbf, 0xa6, - 0x20, 0x58, 0xc3, 0x42, 0x6f, 0xc2, 0x58, 0xdb, 0x8f, 0xf8, 0x69, 0xeb, 0x6c, 0x8a, 0xc8, 0x65, - 0xc3, 0xc9, 0x2b, 0xfc, 0x3d, 0x1d, 0x88, 0x4d, 0x5c, 0xb4, 0x00, 0x83, 0xb1, 0xc3, 0xde, 0xee, - 0x07, 0xf2, 0x6d, 0xf0, 0x36, 0x28, 0x86, 0x1e, 0x70, 0x92, 0x56, 0xc0, 0xa2, 0x22, 0xfa, 0x92, - 0xe4, 0xf7, 0x9c, 0xff, 0x0b, 0xe3, 0xd7, 0xdc, 0x3d, 0xa2, 0x9f, 0x15, 0x3a, 0xc3, 0x17, 0x46, - 0xb5, 0x06, 0x2d, 0xfb, 0x87, 0xca, 0x00, 0x89, 0x68, 0x89, 0x3e, 0xe8, 0xe0, 0x47, 0x2f, 0x75, - 0x17, 0x46, 0x4f, 0x8f, 0x19, 0xa1, 0x1f, 0xb6, 0x60, 0xc4, 0xf1, 0xbc, 0xa0, 0xe1, 0xc4, 0x6c, - 0x94, 0x0b, 0xdd, 0xf9, 0xa1, 0x68, 0x7f, 0x21, 0xa9, 0xc1, 0xbb, 0xf0, 0xaa, 0x5c, 0x78, 0x1a, - 0xa4, 0x67, 0x2f, 0xf4, 0x86, 0xd1, 0x77, 0xc8, 0x1b, 0x07, 0x5f, 0x1e, 0xb3, 0xe9, 0x1b, 0x47, - 0x99, 0xb1, 0x7e, 0xed, 0xb2, 0x81, 0xee, 0x19, 0x21, 0xbe, 0x4a, 0xf9, 0xb1, 0x02, 0x0c, 0x09, - 0xab, 0x57, 0x74, 0x2f, 0x54, 0xd3, 0x9d, 0x80, 0x06, 0xf2, 0x03, 0x6a, 0x68, 0xa2, 0x7c, 0x0f, - 0x07, 0xa0, 0xf7, 0x60, 0xa2, 0x69, 0x9e, 0xed, 0x62, 0x35, 0x3d, 0x9f, 0x47, 0x37, 0x25, 0x0a, - 0x24, 0xa7, 0x79, 0x0a, 0x80, 0xd3, 0x84, 0x51, 0x8d, 0xbb, 0x63, 0xad, 0xfa, 0x5b, 0x81, 0x30, - 0xa2, 0xb6, 0x73, 0xe7, 0xf2, 0x20, 0x8a, 0xc9, 0x1e, 0xc5, 0x4c, 0x0e, 0xed, 0x75, 0x51, 0x17, - 0x2b, 0x2a, 0xe8, 0x6d, 0x18, 0x64, 0xfe, 0x9c, 0xd1, 0xcc, 0x70, 0xbe, 0xd2, 0xcb, 0x0c, 0x45, - 0x90, 0x6c, 0x2a, 0xf6, 0x37, 0xc2, 0x82, 0x02, 0xba, 0x25, 0xe3, 0x95, 0x44, 0xab, 0xfe, 0xbd, - 0x88, 0xb0, 0x78, 0x25, 0xe5, 0xc5, 0x4f, 0x26, 0xa1, 0x48, 0x78, 0x79, 0x66, 0x68, 0x69, 0xa3, - 0x26, 0x15, 0x8e, 0xc4, 0x7f, 0x19, 0xb1, 0x7a, 0x06, 0xf2, 0xbb, 0x67, 0x46, 0xb5, 0x4e, 0x86, - 0xf3, 0xbe, 0x49, 0x02, 0xa7, 0x69, 0x9e, 0xe9, 0x59, 0x3d, 0xeb, 0xc3, 0x64, 0x7a, 0x63, 0x3d, - 0x51, 0xd9, 0xe0, 0x8f, 0x4b, 0x30, 0x6e, 0x2e, 0x04, 0x34, 0x0f, 0x65, 0x41, 0x44, 0x45, 0x57, - 0x54, 0x6b, 0x7b, 0x4d, 0x02, 0x70, 0x82, 0xc3, 0xa2, 0x4b, 0xb2, 0xea, 0x9a, 0xd1, 0x5b, 0x12, - 0x5d, 0x52, 0x41, 0xb0, 0x86, 0x45, 0x25, 0xf6, 0xcd, 0x20, 0x88, 0xd5, 0x51, 0xa0, 0x56, 0xcb, - 0x22, 0x2b, 0xc5, 0x02, 0x4a, 0x8f, 0x80, 0x5d, 0x12, 0xfa, 0xc4, 0xbb, 0x4f, 0x42, 0xf6, 0xee, - 0x5a, 0x32, 0x0d, 0xb1, 0x6e, 0xeb, 0x40, 0x6c, 0xe2, 0xd2, 0xb3, 0x2d, 0x88, 0xd8, 0xf2, 0x13, - 0xf7, 0x82, 0xc4, 0x88, 0xb0, 0xce, 0xfd, 0x99, 0x25, 0x1c, 0x7d, 0x11, 0x9e, 0x52, 0xee, 0xc7, - 0x98, 0xab, 0x41, 0x65, 0x8b, 0x83, 0xc6, 0x35, 0xfe, 0xa9, 0xa5, 0x6c, 0x34, 0x9c, 0x57, 0x1f, - 0xbd, 0x05, 0xe3, 0x42, 0xde, 0x96, 0x14, 0x87, 0xcc, 0x97, 0xf9, 0xdb, 0x06, 0x14, 0xa7, 0xb0, - 0x51, 0x15, 0x26, 0x69, 0x09, 0x13, 0x79, 0x25, 0x05, 0xee, 0x46, 0xad, 0x04, 0x8b, 0xdb, 0x29, - 0x38, 0xee, 0xa8, 0x81, 0x16, 0x60, 0x82, 0x0b, 0x44, 0xf4, 0x02, 0xcb, 0xe6, 0x41, 0xf8, 0x36, - 0xa8, 0x8d, 0x70, 0xd7, 0x04, 0xe3, 0x34, 0x3e, 0xba, 0x01, 0xa3, 0x4e, 0xd8, 0xd8, 0x71, 0x63, - 0xd2, 0x88, 0xdb, 0x21, 0x77, 0x7a, 0xd0, 0x4c, 0x1b, 0x16, 0x34, 0x18, 0x36, 0x30, 0xed, 0x0f, - 0xe0, 0x5c, 0x86, 0x5b, 0x14, 0x5d, 0x38, 0x4e, 0xcb, 0x95, 0xdf, 0x94, 0x32, 0x07, 0x5c, 0xa8, - 0xad, 0xca, 0xaf, 0xd1, 0xb0, 0xe8, 0xea, 0x64, 0xee, 0x53, 0x5a, 0x58, 0x79, 0xb5, 0x3a, 0x57, - 0x24, 0x00, 0x27, 0x38, 0xf6, 0xff, 0x2a, 0xc0, 0x44, 0x86, 0x6a, 0x97, 0x85, 0x36, 0x4f, 0xdd, - 0x18, 0x92, 0x48, 0xe6, 0xb4, 0x19, 0x5f, 0x05, 0x03, 0x4f, 0x35, 0x93, 0x84, 0x02, 0x4f, 0x70, - 0x94, 0xb6, 0xb8, 0x98, 0x6b, 0x64, 0xb6, 0x08, 0xc5, 0xb6, 0xdb, 0x14, 0x0b, 0x58, 0x9e, 0xd7, - 0xc5, 0x7b, 0xab, 0xd5, 0xe3, 0xc3, 0xca, 0xb3, 0x79, 0x19, 0x1a, 0xe2, 0x83, 0x16, 0x89, 0xe6, - 0xe8, 0xee, 0xa1, 0x95, 0x53, 0x23, 0x36, 0xd0, 0xd7, 0x88, 0x2d, 0xc0, 0x84, 0x64, 0x01, 0xe6, - 0x92, 0x56, 0x33, 0x8f, 0x4d, 0x30, 0x4e, 0xe3, 0x9b, 0x83, 0x3e, 0xd4, 0xc7, 0xa0, 0xff, 0x44, - 0x01, 0x26, 0xd3, 0x26, 0x58, 0x67, 0xa0, 0x10, 0x7c, 0xdb, 0x50, 0x08, 0x66, 0x27, 0x0a, 0x48, - 0x1b, 0x86, 0xe5, 0x29, 0x07, 0x71, 0x4a, 0x39, 0xf8, 0xe9, 0xbe, 0xa8, 0x75, 0x57, 0x14, 0xfe, - 0x83, 0x02, 0x9c, 0x4f, 0x57, 0x59, 0xf2, 0x1c, 0x77, 0xef, 0x0c, 0xc6, 0xe6, 0xae, 0x31, 0x36, - 0x2f, 0xf7, 0xf3, 0x35, 0xac, 0x6b, 0xb9, 0x03, 0xf4, 0x20, 0x35, 0x40, 0xf3, 0xfd, 0x93, 0xec, - 0x3e, 0x4a, 0xdf, 0x2c, 0xc2, 0xe5, 0xcc, 0x7a, 0x89, 0x3e, 0x6d, 0xc5, 0xd0, 0xa7, 0x5d, 0x4f, - 0xe9, 0xd3, 0xec, 0xee, 0xb5, 0x4f, 0x47, 0xc1, 0x26, 0xfc, 0xd5, 0x58, 0xf8, 0xb1, 0xc7, 0x54, - 0xae, 0x19, 0xfe, 0x6a, 0x8a, 0x10, 0x36, 0xe9, 0x7e, 0x2b, 0x29, 0xd5, 0xfe, 0xbd, 0x05, 0x17, - 0x33, 0xe7, 0xe6, 0x0c, 0x14, 0x4f, 0xeb, 0xa6, 0xe2, 0xe9, 0x85, 0xbe, 0x57, 0x6b, 0x8e, 0x26, - 0xea, 0x4f, 0x8a, 0x39, 0xdf, 0xc2, 0xae, 0xd5, 0x77, 0x61, 0xc4, 0x69, 0x34, 0x48, 0x14, 0xad, - 0x05, 0x4d, 0x15, 0x8e, 0xeb, 0x65, 0x76, 0x3b, 0x4a, 0x8a, 0x8f, 0x0f, 0x2b, 0xb3, 0x69, 0x12, - 0x09, 0x18, 0xeb, 0x14, 0xcc, 0x08, 0x82, 0x85, 0x53, 0x8d, 0x20, 0x78, 0x1d, 0x60, 0x5f, 0xc9, - 0xd8, 0xe9, 0xab, 0xb9, 0x26, 0x7d, 0x6b, 0x58, 0xe8, 0x7b, 0x61, 0x38, 0x12, 0xc7, 0xb8, 0x58, - 0x8a, 0xaf, 0xf6, 0x39, 0x57, 0xce, 0x26, 0xf1, 0x4c, 0xc7, 0x68, 0xa5, 0xc5, 0x50, 0x24, 0xd1, - 0x77, 0xc3, 0x64, 0xc4, 0xe3, 0x6e, 0x2c, 0x79, 0x4e, 0xc4, 0xac, 0xec, 0xc5, 0x2a, 0x64, 0xde, - 0xce, 0xf5, 0x14, 0x0c, 0x77, 0x60, 0xa3, 0x15, 0xf9, 0x51, 0x2c, 0x48, 0x08, 0x5f, 0x98, 0x57, - 0x93, 0x0f, 0x12, 0x89, 0x55, 0xa6, 0xd3, 0xc3, 0xcf, 0x06, 0x5e, 0xab, 0x69, 0xff, 0x44, 0x09, - 0x9e, 0xee, 0xc2, 0xc4, 0xd0, 0x82, 0xf9, 0x4a, 0xf6, 0x62, 0xfa, 0xce, 0x3a, 0x9b, 0x59, 0xd9, - 0xb8, 0xc4, 0xa6, 0xd6, 0x4a, 0xe1, 0x43, 0xaf, 0x95, 0x1f, 0xb5, 0x34, 0x6d, 0x02, 0xb7, 0xe5, - 0xfa, 0xfc, 0x09, 0x99, 0xf3, 0x29, 0xaa, 0x17, 0xb6, 0x32, 0xee, 0xe8, 0xd7, 0xfb, 0xee, 0x4e, - 0xdf, 0x97, 0xf6, 0xb3, 0xd5, 0xa9, 0x7e, 0xd5, 0x82, 0x67, 0x33, 0xfb, 0x6b, 0xbc, 0xea, 0xcf, - 0x43, 0xb9, 0x41, 0x0b, 0x35, 0xcf, 0x9d, 0xc4, 0xa5, 0x51, 0x02, 0x70, 0x82, 0x63, 0x3c, 0xde, - 0x17, 0x7a, 0x3e, 0xde, 0xff, 0x6b, 0x0b, 0x3a, 0x16, 0xf0, 0x19, 0x70, 0xd2, 0x55, 0x93, 0x93, - 0x7e, 0xb2, 0x9f, 0xb9, 0xcc, 0x61, 0xa2, 0x7f, 0x30, 0x01, 0x17, 0x72, 0x4c, 0xf5, 0xf7, 0x61, - 0x6a, 0xbb, 0x41, 0x4c, 0x9f, 0x28, 0xf1, 0x31, 0x99, 0xee, 0x63, 0x5d, 0x1d, 0xa8, 0x58, 0x46, - 0x8c, 0xa9, 0x0e, 0x14, 0xdc, 0xd9, 0x04, 0xfa, 0xaa, 0x05, 0xd3, 0xce, 0xc3, 0xa8, 0x23, 0xef, - 0x99, 0x58, 0x33, 0xaf, 0x65, 0xea, 0x16, 0x7a, 0xe4, 0x49, 0xe3, 0x29, 0x42, 0xb2, 0xb0, 0x70, - 0x66, 0x5b, 0x08, 0x8b, 0x08, 0x8a, 0x54, 0xde, 0xee, 0xe2, 0xb5, 0x97, 0xe5, 0x53, 0xc1, 0x79, - 0xaa, 0x84, 0x60, 0x45, 0x07, 0xdd, 0x87, 0xf2, 0xb6, 0x74, 0x74, 0x12, 0x3c, 0x3b, 0xf3, 0x10, - 0xcc, 0xf4, 0x86, 0xe2, 0xc6, 0xbd, 0x0a, 0x84, 0x13, 0x52, 0xe8, 0x2d, 0x28, 0xfa, 0x5b, 0x51, - 0xb7, 0xdc, 0x1a, 0x29, 0x63, 0x17, 0xee, 0x11, 0xbb, 0xbe, 0x52, 0xc7, 0xb4, 0x22, 0xba, 0x05, - 0xc5, 0x70, 0xb3, 0x29, 0xd4, 0x61, 0x99, 0x72, 0x29, 0x5e, 0xac, 0x66, 0x2f, 0x12, 0x4e, 0x09, - 0x2f, 0x56, 0x31, 0x25, 0x81, 0x6a, 0x30, 0xc0, 0xac, 0xda, 0x85, 0xd6, 0x2b, 0x53, 0x20, 0xed, - 0xe2, 0x1d, 0xc2, 0xdd, 0x66, 0x19, 0x02, 0xe6, 0x84, 0xd0, 0xdb, 0x30, 0xd8, 0x60, 0xe9, 0x27, - 0x44, 0xd0, 0xde, 0xec, 0x78, 0x2a, 0x1d, 0x09, 0x2a, 0xf8, 0x13, 0x03, 0x2f, 0xc7, 0x82, 0x02, - 0xda, 0x80, 0xc1, 0x06, 0x69, 0xed, 0x6c, 0x45, 0xec, 0xe2, 0x6d, 0x0a, 0xf8, 0x09, 0xad, 0x2e, - 0xd9, 0x56, 0x04, 0x55, 0x86, 0x81, 0x05, 0x2d, 0xf4, 0x59, 0x28, 0x6c, 0x35, 0x84, 0xa9, 0x7b, - 0xa6, 0xde, 0xcb, 0x74, 0x65, 0x5e, 0x1c, 0x3c, 0x3a, 0xac, 0x14, 0x56, 0x96, 0x70, 0x61, 0xab, - 0x81, 0xd6, 0x61, 0x68, 0x8b, 0x3b, 0x3f, 0x8a, 0x08, 0x03, 0xcf, 0x67, 0xfb, 0x65, 0x76, 0xf8, - 0x47, 0x72, 0x13, 0x6d, 0x01, 0xc0, 0x92, 0x08, 0x0b, 0x3e, 0xa8, 0x9c, 0x38, 0x45, 0x14, 0xde, - 0xb9, 0x93, 0x39, 0xde, 0x72, 0xb7, 0xea, 0xc4, 0x15, 0x14, 0x6b, 0x14, 0xd1, 0x57, 0xa0, 0xec, - 0xc8, 0x3c, 0x5b, 0x22, 0x4a, 0xc1, 0xab, 0x99, 0xdb, 0xb1, 0x7b, 0x0a, 0x32, 0xbe, 0x96, 0x15, - 0x12, 0x4e, 0x88, 0xa2, 0x5d, 0x18, 0xdb, 0x8f, 0x5a, 0x3b, 0x44, 0x6e, 0x5f, 0x16, 0xb4, 0x20, - 0xe7, 0xb8, 0xba, 0x2f, 0x10, 0xdd, 0x30, 0x6e, 0x3b, 0x5e, 0x07, 0xc7, 0x61, 0x96, 0xfd, 0xf7, - 0x75, 0x62, 0xd8, 0xa4, 0x4d, 0x87, 0xff, 0xfd, 0x76, 0xb0, 0x79, 0x10, 0x13, 0x11, 0xb6, 0x37, - 0x73, 0xf8, 0xdf, 0xe1, 0x28, 0x9d, 0xc3, 0x2f, 0x00, 0x58, 0x12, 0xa1, 0x1b, 0xdc, 0x91, 0x39, - 0xec, 0x58, 0xb8, 0xde, 0x9c, 0x0d, 0x9e, 0x99, 0xe8, 0x4e, 0x1b, 0x14, 0xc6, 0x19, 0x13, 0x52, - 0x8c, 0x23, 0xb6, 0x76, 0x82, 0x38, 0xf0, 0x53, 0xdc, 0x78, 0x2a, 0x9f, 0x23, 0xd6, 0x32, 0xf0, - 0x3b, 0x39, 0x62, 0x16, 0x16, 0xce, 0x6c, 0x0b, 0x35, 0x61, 0xbc, 0x15, 0x84, 0xf1, 0xc3, 0x20, - 0x94, 0xeb, 0x0b, 0x75, 0xb9, 0xe4, 0x1b, 0x98, 0xa2, 0x45, 0x16, 0x46, 0xda, 0x84, 0xe0, 0x14, - 0x4d, 0xf4, 0x05, 0x18, 0x8a, 0x1a, 0x8e, 0x47, 0x56, 0xef, 0xce, 0x9c, 0xcb, 0x3f, 0x6a, 0xea, - 0x1c, 0x25, 0x67, 0x75, 0xb1, 0xc9, 0x11, 0x28, 0x58, 0x92, 0x43, 0x2b, 0x30, 0xc0, 0x62, 0xc1, - 0xb3, 0x88, 0xc3, 0x39, 0xd1, 0x70, 0x3a, 0x8c, 0x09, 0x39, 0x47, 0x62, 0xc5, 0x98, 0x57, 0xa7, - 0x7b, 0x40, 0xc8, 0xba, 0x41, 0x34, 0x73, 0x3e, 0x7f, 0x0f, 0x08, 0x11, 0xf9, 0x6e, 0xbd, 0xdb, - 0x1e, 0x50, 0x48, 0x38, 0x21, 0x4a, 0xf9, 0x31, 0xe5, 0xa1, 0x17, 0xf2, 0xf9, 0x71, 0x3e, 0x07, - 0x65, 0xfc, 0x98, 0xf2, 0x4f, 0x4a, 0xc2, 0xfe, 0xc3, 0xa1, 0x4e, 0xf9, 0x84, 0xdd, 0x8e, 0x7e, - 0xc8, 0xea, 0x78, 0xee, 0xfa, 0x4c, 0xbf, 0xca, 0x9a, 0x53, 0x94, 0x4c, 0xbf, 0x6a, 0xc1, 0x85, - 0x56, 0xe6, 0x87, 0x88, 0xc3, 0xbe, 0x3f, 0x9d, 0x0f, 0xff, 0x74, 0x15, 0x15, 0x3c, 0x1b, 0x8e, - 0x73, 0x5a, 0x4a, 0x4b, 0xff, 0xc5, 0x0f, 0x2d, 0xfd, 0xaf, 0xc1, 0x30, 0x13, 0x28, 0x93, 0xd0, - 0x4b, 0x7d, 0x59, 0xa8, 0x30, 0xb1, 0x61, 0x49, 0x54, 0xc4, 0x8a, 0x04, 0xfa, 0x31, 0x0b, 0x2e, - 0xa5, 0xbb, 0x8e, 0x09, 0x03, 0x8b, 0x18, 0xda, 0xfc, 0x62, 0xb6, 0x22, 0xbe, 0xff, 0x52, 0xad, - 0x1b, 0xf2, 0x71, 0x2f, 0x04, 0xdc, 0xbd, 0x31, 0x54, 0xcd, 0xb8, 0x19, 0x0e, 0x9a, 0xda, 0xf0, - 0x3e, 0x6e, 0x87, 0xaf, 0xc1, 0xe8, 0x5e, 0xd0, 0xf6, 0x63, 0x61, 0x42, 0x22, 0x9c, 0xd3, 0xd8, - 0x9b, 0xed, 0x9a, 0x56, 0x8e, 0x0d, 0xac, 0xd4, 0x9d, 0x72, 0xf8, 0x71, 0xef, 0x94, 0xe8, 0xdd, - 0x54, 0xce, 0xd9, 0x72, 0xbe, 0x6c, 0x21, 0xae, 0xdf, 0x27, 0xc8, 0x3c, 0x7b, 0xb6, 0xf7, 0xa0, - 0xaf, 0x5b, 0x19, 0x02, 0x3c, 0xbf, 0x19, 0x7f, 0xce, 0xbc, 0x19, 0x5f, 0x4d, 0xdf, 0x8c, 0x3b, - 0x34, 0xa1, 0xc6, 0xa5, 0xb8, 0xff, 0x80, 0xbf, 0xfd, 0x46, 0xd0, 0xb2, 0x3d, 0xb8, 0xd2, 0xeb, - 0x58, 0x62, 0xb6, 0x44, 0x4d, 0xf5, 0xee, 0x95, 0xd8, 0x12, 0x35, 0x57, 0xab, 0x98, 0x41, 0xfa, - 0x0d, 0xb1, 0x60, 0xff, 0x77, 0x0b, 0x8a, 0xb5, 0xa0, 0x79, 0x06, 0x9a, 0xdd, 0xcf, 0x1b, 0x9a, - 0xdd, 0xa7, 0x73, 0x72, 0x01, 0xe7, 0xea, 0x71, 0x97, 0x53, 0x7a, 0xdc, 0x4b, 0x79, 0x04, 0xba, - 0x6b, 0x6d, 0x7f, 0xae, 0x08, 0x7a, 0xe6, 0x62, 0xf4, 0x6f, 0x1f, 0xc7, 0x28, 0xb5, 0xd8, 0x2d, - 0x99, 0xb1, 0xa0, 0xcc, 0x4c, 0x90, 0xa4, 0xbf, 0xd5, 0x5f, 0x31, 0xdb, 0xd4, 0x07, 0xc4, 0xdd, - 0xde, 0x89, 0x49, 0x33, 0xfd, 0x39, 0x67, 0x67, 0x9b, 0xfa, 0x5f, 0x2d, 0x98, 0x48, 0xb5, 0x8e, - 0x3c, 0x18, 0xf3, 0x74, 0xb5, 0x9c, 0x58, 0xa7, 0x8f, 0xa5, 0xd1, 0xe3, 0x19, 0x40, 0xf4, 0x22, - 0x6c, 0x12, 0x47, 0x73, 0x00, 0xea, 0xd9, 0x4c, 0x6a, 0xbb, 0x98, 0xd4, 0xaf, 0xde, 0xd5, 0x22, - 0xac, 0x61, 0xa0, 0xd7, 0x61, 0x24, 0x0e, 0x5a, 0x81, 0x17, 0x6c, 0x1f, 0xdc, 0x26, 0x32, 0xa8, - 0x87, 0xb2, 0x72, 0xda, 0x48, 0x40, 0x58, 0xc7, 0xb3, 0x7f, 0xa1, 0x08, 0xe9, 0x6c, 0xd7, 0xdf, - 0x5e, 0x93, 0x1f, 0xcf, 0x35, 0xf9, 0x4d, 0x0b, 0x26, 0x69, 0xeb, 0xcc, 0xe2, 0x42, 0x1e, 0xb6, - 0x2a, 0xf1, 0x88, 0xd5, 0x25, 0xf1, 0xc8, 0x55, 0xca, 0xbb, 0x9a, 0x41, 0x3b, 0x16, 0xda, 0x32, - 0x8d, 0x39, 0xd1, 0x52, 0x2c, 0xa0, 0x02, 0x8f, 0x84, 0xa1, 0x70, 0x89, 0xd1, 0xf1, 0x48, 0x18, - 0x62, 0x01, 0x95, 0x79, 0x49, 0x4a, 0x39, 0x79, 0x49, 0x58, 0x88, 0x32, 0xf1, 0xca, 0x2f, 0xc4, - 0x1e, 0x2d, 0x44, 0x99, 0x7c, 0xfe, 0x4f, 0x70, 0xec, 0x5f, 0x2e, 0xc2, 0x68, 0x2d, 0x68, 0x26, - 0x0f, 0x57, 0xaf, 0x19, 0x0f, 0x57, 0x57, 0x52, 0x0f, 0x57, 0x93, 0x3a, 0xee, 0xb7, 0x9f, 0xa9, - 0x3e, 0xaa, 0x67, 0xaa, 0x7f, 0x65, 0xb1, 0x59, 0xab, 0xae, 0xd7, 0x45, 0x66, 0xcf, 0x57, 0x60, - 0x84, 0x31, 0x24, 0xe6, 0x83, 0x25, 0x5f, 0x73, 0x58, 0xc8, 0xf1, 0xf5, 0xa4, 0x18, 0xeb, 0x38, - 0xe8, 0x1a, 0x0c, 0x47, 0xc4, 0x09, 0x1b, 0x3b, 0x8a, 0xc7, 0x89, 0xb7, 0x0e, 0x5e, 0x86, 0x15, - 0x14, 0xbd, 0x93, 0x44, 0xc7, 0x2a, 0xe6, 0xe7, 0xa8, 0xd4, 0xfb, 0xc3, 0xb7, 0x48, 0x7e, 0x48, - 0x2c, 0xfb, 0x01, 0xa0, 0x4e, 0xfc, 0x3e, 0xe2, 0xe0, 0x54, 0xcc, 0x38, 0x38, 0xe5, 0x8e, 0x18, - 0x38, 0x7f, 0x61, 0xc1, 0x78, 0x2d, 0x68, 0xd2, 0xad, 0xfb, 0xad, 0xb4, 0x4f, 0xf5, 0xd0, 0x80, - 0x83, 0x5d, 0x42, 0x03, 0xfe, 0x43, 0x0b, 0x86, 0x6a, 0x41, 0xf3, 0x0c, 0x74, 0xec, 0x9f, 0x33, - 0x75, 0xec, 0x4f, 0xe5, 0x2c, 0x89, 0x1c, 0xb5, 0xfa, 0xaf, 0x16, 0x61, 0x8c, 0xf6, 0x33, 0xd8, - 0x96, 0xb3, 0x64, 0x8c, 0x88, 0xd5, 0xc7, 0x88, 0x50, 0x31, 0x37, 0xf0, 0xbc, 0xe0, 0x61, 0x7a, - 0xc6, 0x56, 0x58, 0x29, 0x16, 0x50, 0xf4, 0x12, 0x0c, 0xb7, 0x42, 0xb2, 0xef, 0x06, 0x42, 0x7e, - 0xd4, 0x5e, 0x2c, 0x6a, 0xa2, 0x1c, 0x2b, 0x0c, 0x7a, 0xef, 0x8a, 0x5c, 0xbf, 0x41, 0x64, 0x82, - 0xdc, 0x12, 0xcb, 0x40, 0xc4, 0x63, 0xfe, 0x6a, 0xe5, 0xd8, 0xc0, 0x42, 0x0f, 0xa0, 0xcc, 0xfe, - 0x33, 0x8e, 0x72, 0xf2, 0x8c, 0x29, 0x22, 0xd0, 0xbe, 0x20, 0x80, 0x13, 0x5a, 0xe8, 0x3a, 0x40, - 0x2c, 0xe3, 0xc2, 0x46, 0x22, 0x9c, 0x89, 0x92, 0xb5, 0x55, 0xc4, 0xd8, 0x08, 0x6b, 0x58, 0xe8, - 0x45, 0x28, 0xc7, 0x8e, 0xeb, 0xdd, 0x71, 0x7d, 0x12, 0x31, 0x45, 0x73, 0x51, 0xc6, 0xd1, 0x17, - 0x85, 0x38, 0x81, 0x53, 0x59, 0x87, 0xf9, 0xfa, 0xf2, 0x7c, 0x4b, 0xc3, 0x0c, 0x9b, 0xc9, 0x3a, - 0x77, 0x54, 0x29, 0xd6, 0x30, 0xec, 0x1b, 0x70, 0xbe, 0x16, 0x34, 0x6b, 0x41, 0x18, 0xaf, 0x04, - 0xe1, 0x43, 0x27, 0x6c, 0xca, 0xf9, 0xab, 0xc8, 0x90, 0xee, 0x94, 0xf7, 0x0c, 0xf0, 0x9d, 0x69, - 0x04, 0x6b, 0x7f, 0x95, 0x49, 0x3b, 0x27, 0xf4, 0x8b, 0xf8, 0xdf, 0x05, 0xc6, 0x28, 0x52, 0x49, - 0xc0, 0xd0, 0x97, 0x61, 0x3c, 0x22, 0x77, 0x5c, 0xbf, 0xfd, 0x48, 0xde, 0x8f, 0xbb, 0x38, 0x9d, - 0xd4, 0x97, 0x75, 0x4c, 0xae, 0x65, 0x33, 0xcb, 0x70, 0x8a, 0x1a, 0x1d, 0xc2, 0xb0, 0xed, 0x2f, - 0x44, 0xf7, 0x22, 0x12, 0x8a, 0x24, 0x54, 0x6c, 0x08, 0xb1, 0x2c, 0xc4, 0x09, 0x9c, 0x2e, 0x19, - 0xf6, 0x67, 0x3d, 0xf0, 0x71, 0x10, 0xc4, 0x72, 0x91, 0xb1, 0x34, 0x26, 0x5a, 0x39, 0x36, 0xb0, - 0xd0, 0x0a, 0xa0, 0xa8, 0xdd, 0x6a, 0x79, 0xec, 0x01, 0xdc, 0xf1, 0x6e, 0x86, 0x41, 0xbb, 0xc5, - 0x1f, 0x1f, 0x8b, 0x3c, 0x3d, 0x7c, 0xbd, 0x03, 0x8a, 0x33, 0x6a, 0x50, 0xc6, 0xb0, 0x15, 0xb1, - 0xdf, 0x6c, 0xe1, 0x15, 0x85, 0xe6, 0xbb, 0xce, 0x8a, 0xb0, 0x84, 0xd1, 0x79, 0x66, 0xcd, 0x73, - 0xcc, 0xc1, 0x64, 0x9e, 0xb1, 0x2a, 0xc5, 0x1a, 0x86, 0xfd, 0x03, 0xec, 0x80, 0x61, 0xb9, 0x86, - 0xe2, 0x76, 0x48, 0xd0, 0x1e, 0x8c, 0xb5, 0xd8, 0xd1, 0x2f, 0x22, 0xe1, 0x8a, 0x01, 0x7f, 0xad, - 0xcf, 0x9b, 0xe2, 0x43, 0xba, 0x79, 0x95, 0x26, 0x87, 0x89, 0xe0, 0x35, 0x9d, 0x1c, 0x36, 0xa9, - 0xdb, 0x3f, 0x37, 0xc9, 0xf8, 0x58, 0x9d, 0x5f, 0xff, 0x86, 0x84, 0xc5, 0xab, 0x90, 0x75, 0x67, - 0xf3, 0xf5, 0x10, 0xc9, 0x91, 0x23, 0xac, 0x66, 0xb1, 0xac, 0x8b, 0xde, 0x61, 0xaf, 0xbc, 0x9c, - 0x79, 0xf4, 0x4a, 0x5a, 0xca, 0xb1, 0x8c, 0x07, 0x5d, 0x51, 0x11, 0x6b, 0x44, 0xd0, 0x1d, 0x18, - 0x13, 0xa9, 0x69, 0x84, 0xa2, 0xa9, 0x68, 0x28, 0x12, 0xc6, 0xb0, 0x0e, 0x3c, 0x4e, 0x17, 0x60, - 0xb3, 0x32, 0xda, 0x86, 0x4b, 0x5a, 0x9e, 0xb6, 0x9b, 0xa1, 0xc3, 0x5e, 0xfe, 0x5c, 0xb6, 0xfa, - 0x35, 0x5e, 0xf4, 0xec, 0xd1, 0x61, 0xe5, 0xd2, 0x46, 0x37, 0x44, 0xdc, 0x9d, 0x0e, 0xba, 0x0b, - 0xe7, 0x9d, 0x46, 0xec, 0xee, 0x93, 0x2a, 0x71, 0x9a, 0x9e, 0xeb, 0x2b, 0x66, 0xc7, 0x17, 0xd0, - 0xc5, 0xa3, 0xc3, 0xca, 0xf9, 0x85, 0x2c, 0x04, 0x9c, 0x5d, 0x0f, 0x7d, 0x0e, 0xca, 0x4d, 0x3f, - 0x12, 0x63, 0x30, 0x68, 0xa4, 0x20, 0x2c, 0x57, 0xd7, 0xeb, 0xea, 0xfb, 0x93, 0x3f, 0x38, 0xa9, - 0x80, 0xb6, 0xb9, 0xb2, 0x49, 0xdd, 0xed, 0x86, 0xf2, 0x13, 0x26, 0x8b, 0x25, 0x61, 0xb8, 0x96, - 0x70, 0x2d, 0xab, 0x32, 0xf2, 0x34, 0xbc, 0x4e, 0x0c, 0xc2, 0xe8, 0x6d, 0x40, 0x54, 0xf8, 0x71, - 0x1b, 0x64, 0xa1, 0xc1, 0x02, 0x12, 0x33, 0xdd, 0xdc, 0xb0, 0x61, 0xca, 0x8f, 0xea, 0x1d, 0x18, - 0x38, 0xa3, 0x16, 0xba, 0x45, 0x39, 0x90, 0x5e, 0x2a, 0x8c, 0x55, 0xa5, 0xc0, 0x3c, 0x53, 0x25, - 0xad, 0x90, 0x34, 0x9c, 0x98, 0x34, 0x4d, 0x8a, 0x38, 0x55, 0x8f, 0x9e, 0x4f, 0x2a, 0x37, 0x09, - 0x98, 0x51, 0x07, 0x3a, 0xf3, 0x93, 0xd0, 0xbb, 0xe6, 0x4e, 0x10, 0xc5, 0xeb, 0x24, 0x7e, 0x18, - 0x84, 0xbb, 0x22, 0xc8, 0x53, 0x12, 0x6f, 0x30, 0x01, 0x61, 0x1d, 0x8f, 0xca, 0x96, 0xec, 0x99, - 0x75, 0xb5, 0xca, 0x5e, 0xbd, 0x86, 0x93, 0x7d, 0x72, 0x8b, 0x17, 0x63, 0x09, 0x97, 0xa8, 0xab, - 0xb5, 0x25, 0xf6, 0x82, 0x95, 0x42, 0x5d, 0xad, 0x2d, 0x61, 0x09, 0x47, 0xa4, 0x33, 0xbd, 0xe3, - 0x78, 0xbe, 0xa6, 0xb0, 0x93, 0x8f, 0xf7, 0x99, 0xe1, 0xd1, 0x87, 0x49, 0x95, 0x58, 0x92, 0x47, - 0xbf, 0x8a, 0x66, 0x26, 0xd8, 0x22, 0xe9, 0x3f, 0x74, 0x96, 0xd2, 0xbd, 0xae, 0xa6, 0x28, 0xe1, - 0x0e, 0xda, 0x46, 0x1c, 0x88, 0xc9, 0x9e, 0xb9, 0x65, 0xe6, 0xa1, 0x1c, 0xb5, 0x37, 0x9b, 0xc1, - 0x9e, 0xe3, 0xfa, 0xec, 0xc1, 0x49, 0x13, 0x5c, 0xea, 0x12, 0x80, 0x13, 0x1c, 0xb4, 0x02, 0xc3, - 0x8e, 0x54, 0xac, 0xa2, 0x7c, 0xc7, 0x70, 0xa5, 0x4e, 0x65, 0x62, 0xb9, 0x52, 0xa5, 0xaa, 0xba, - 0xe8, 0x4d, 0x18, 0x13, 0xee, 0x44, 0xdc, 0x5d, 0x9e, 0x3d, 0x08, 0x69, 0x96, 0xe7, 0x75, 0x1d, - 0x88, 0x4d, 0x5c, 0xf4, 0xbd, 0x30, 0x4e, 0xa9, 0x24, 0x8c, 0x6d, 0x66, 0xba, 0x1f, 0x8e, 0xa8, - 0xe5, 0x0c, 0xd0, 0x2b, 0xe3, 0x14, 0x31, 0xd4, 0x84, 0x67, 0x9c, 0x76, 0x1c, 0x30, 0xe5, 0xb4, - 0xb9, 0xfe, 0x37, 0x82, 0x5d, 0xe2, 0xb3, 0x77, 0xa1, 0xe1, 0xc5, 0x2b, 0x47, 0x87, 0x95, 0x67, - 0x16, 0xba, 0xe0, 0xe1, 0xae, 0x54, 0xd0, 0x3d, 0x18, 0x89, 0x03, 0x8f, 0xd9, 0x80, 0x53, 0x19, - 0xe0, 0x42, 0x7e, 0x1c, 0x95, 0x0d, 0x85, 0xa6, 0x2b, 0x66, 0x54, 0x55, 0xac, 0xd3, 0x41, 0x1b, - 0x7c, 0x8f, 0xb1, 0x08, 0x93, 0x24, 0x9a, 0x79, 0x2a, 0x7f, 0x60, 0x54, 0x20, 0x4a, 0x73, 0x0b, - 0x8a, 0x9a, 0x58, 0x27, 0x83, 0x6e, 0xc2, 0x54, 0x2b, 0x74, 0x03, 0xb6, 0xb0, 0xd5, 0xc3, 0xc0, - 0x8c, 0x19, 0x26, 0xbe, 0x96, 0x46, 0xc0, 0x9d, 0x75, 0xe8, 0xc5, 0x4d, 0x16, 0xce, 0x5c, 0xe4, - 0x39, 0x87, 0xb8, 0x30, 0xcb, 0xcb, 0xb0, 0x82, 0xa2, 0x35, 0xc6, 0x97, 0xf9, 0x15, 0x6b, 0x66, - 0x36, 0xdf, 0xa1, 0x5e, 0xbf, 0x8a, 0x71, 0x41, 0x47, 0xfd, 0xc5, 0x09, 0x05, 0x7a, 0x6e, 0x44, - 0x3b, 0x4e, 0x48, 0x6a, 0x61, 0xd0, 0x20, 0xbc, 0x33, 0xdc, 0xfc, 0xfc, 0x69, 0x1e, 0x08, 0x8f, - 0x9e, 0x1b, 0xf5, 0x2c, 0x04, 0x9c, 0x5d, 0x6f, 0xf6, 0xbb, 0x60, 0xaa, 0x83, 0x93, 0x9f, 0x28, - 0x3a, 0xf2, 0x9f, 0x0f, 0x40, 0x59, 0x29, 0x7e, 0xd1, 0xbc, 0xa9, 0xcf, 0xbf, 0x98, 0xd6, 0xe7, - 0x0f, 0x53, 0xf9, 0x53, 0x57, 0xe1, 0x6f, 0x18, 0x86, 0x5f, 0x85, 0xfc, 0x5c, 0x44, 0xba, 0xd6, - 0xa3, 0xa7, 0x6f, 0x96, 0x76, 0x8f, 0x2f, 0xf6, 0xfd, 0x30, 0x50, 0xea, 0xaa, 0x1a, 0xe8, 0x33, - 0x15, 0x28, 0xbd, 0xea, 0xb6, 0x82, 0xe6, 0x6a, 0x2d, 0x9d, 0x1b, 0xaf, 0x46, 0x0b, 0x31, 0x87, - 0xb1, 0xcb, 0x0a, 0x15, 0x3b, 0xd8, 0x65, 0x65, 0xe8, 0x31, 0x2f, 0x2b, 0x92, 0x00, 0x4e, 0x68, - 0x21, 0x0f, 0xa6, 0x1a, 0x66, 0x5a, 0x43, 0xe5, 0x8f, 0xf5, 0x5c, 0xcf, 0x04, 0x83, 0x6d, 0x2d, - 0x87, 0xd4, 0x52, 0x9a, 0x0a, 0xee, 0x24, 0x8c, 0xde, 0x84, 0xe1, 0xf7, 0x83, 0x88, 0x6d, 0x0b, - 0x71, 0xf6, 0x4a, 0x0f, 0x98, 0xe1, 0x77, 0xee, 0xd6, 0x59, 0xf9, 0xf1, 0x61, 0x65, 0xa4, 0x16, - 0x34, 0xe5, 0x5f, 0xac, 0x2a, 0xa0, 0x47, 0x70, 0xde, 0xe0, 0x58, 0xaa, 0xbb, 0xd0, 0x7f, 0x77, - 0x2f, 0x89, 0xe6, 0xce, 0xaf, 0x66, 0x51, 0xc2, 0xd9, 0x0d, 0x50, 0x36, 0xe0, 0x07, 0x22, 0x25, - 0xa8, 0x3c, 0xdf, 0xd9, 0x31, 0x5e, 0xd6, 0xbd, 0x96, 0x53, 0x08, 0xb8, 0xb3, 0x8e, 0xfd, 0x0d, - 0xae, 0x27, 0x17, 0xda, 0x34, 0x12, 0xb5, 0xbd, 0xb3, 0xc8, 0x38, 0xb3, 0x6c, 0x28, 0xfa, 0x1e, - 0xfb, 0x2d, 0xe6, 0xb7, 0x2d, 0xf6, 0x16, 0xb3, 0x41, 0xf6, 0x5a, 0x9e, 0x13, 0x9f, 0x85, 0xe7, - 0xc5, 0x3b, 0x30, 0x1c, 0x8b, 0xd6, 0xba, 0x25, 0xc9, 0xd1, 0x3a, 0xc5, 0xde, 0xa3, 0xd4, 0xc9, - 0x2f, 0x4b, 0xb1, 0x22, 0x63, 0xff, 0x73, 0x3e, 0x03, 0x12, 0x72, 0x06, 0x4a, 0x97, 0xaa, 0xa9, - 0x74, 0xa9, 0xf4, 0xf8, 0x82, 0x1c, 0xe5, 0xcb, 0x3f, 0x33, 0xfb, 0xcd, 0x2e, 0x59, 0x1f, 0xf7, - 0x47, 0x40, 0xfb, 0xa7, 0x2c, 0x98, 0xce, 0xb2, 0x9a, 0xa1, 0xd2, 0x1a, 0xbf, 0xe2, 0xa9, 0x47, - 0x51, 0x35, 0x82, 0xf7, 0x45, 0x39, 0x56, 0x18, 0x7d, 0xc7, 0x9f, 0x3f, 0x59, 0x90, 0xaa, 0xbb, - 0x30, 0x56, 0x0b, 0x89, 0x76, 0x06, 0xbc, 0xc5, 0x5d, 0xa9, 0x78, 0x7f, 0x5e, 0x3a, 0xb1, 0x1b, - 0x95, 0xfd, 0x8b, 0x05, 0x98, 0xe6, 0xaf, 0x1a, 0x0b, 0xfb, 0x81, 0xdb, 0xac, 0x05, 0x4d, 0x91, - 0x3b, 0xe0, 0x4b, 0x30, 0xda, 0xd2, 0xee, 0xe5, 0xdd, 0xc2, 0xe4, 0xe8, 0xf7, 0xf7, 0xe4, 0x7e, - 0xa4, 0x97, 0x62, 0x83, 0x16, 0x6a, 0xc2, 0x28, 0xd9, 0x77, 0x1b, 0x4a, 0x35, 0x5e, 0x38, 0xf1, - 0xd9, 0xa0, 0x5a, 0x59, 0xd6, 0xe8, 0x60, 0x83, 0xea, 0x13, 0x48, 0x27, 0x65, 0xff, 0xb4, 0x05, - 0x4f, 0xe5, 0x04, 0xd5, 0xa1, 0xcd, 0x3d, 0x64, 0xef, 0x47, 0x22, 0x33, 0x8d, 0x6a, 0x8e, 0xbf, - 0x2a, 0x61, 0x01, 0x45, 0x5f, 0x00, 0xe0, 0xaf, 0x42, 0xf4, 0xba, 0x20, 0x3e, 0xbd, 0xbf, 0x60, - 0x13, 0x5a, 0x90, 0x00, 0x59, 0x1f, 0x6b, 0xb4, 0xec, 0x9f, 0x2f, 0xc2, 0x00, 0x7b, 0x85, 0x40, - 0x2b, 0x30, 0xb4, 0xc3, 0x43, 0xc8, 0xf6, 0x13, 0xad, 0x36, 0xb9, 0x77, 0xf1, 0x02, 0x2c, 0x2b, - 0xa3, 0x35, 0x38, 0xc7, 0x43, 0xf0, 0x7a, 0x55, 0xe2, 0x39, 0x07, 0xf2, 0xfa, 0xce, 0xb3, 0xb9, - 0xa8, 0x80, 0x07, 0xab, 0x9d, 0x28, 0x38, 0xab, 0x1e, 0x7a, 0x0b, 0xc6, 0x63, 0x77, 0x8f, 0x04, - 0xed, 0x58, 0x52, 0xe2, 0xc1, 0x77, 0x95, 0xb0, 0xbf, 0x61, 0x40, 0x71, 0x0a, 0x9b, 0x5e, 0x44, - 0x5a, 0x1d, 0x8a, 0x0a, 0x2d, 0x3f, 0xba, 0xa9, 0x9c, 0x30, 0x71, 0x99, 0xb9, 0x4c, 0x9b, 0x19, - 0x07, 0x6d, 0xec, 0x84, 0x24, 0xda, 0x09, 0xbc, 0xa6, 0x48, 0x06, 0x9c, 0x98, 0xcb, 0xa4, 0xe0, - 0xb8, 0xa3, 0x06, 0xa5, 0xb2, 0xe5, 0xb8, 0x5e, 0x3b, 0x24, 0x09, 0x95, 0x41, 0x93, 0xca, 0x4a, - 0x0a, 0x8e, 0x3b, 0x6a, 0xd0, 0x75, 0x74, 0x5e, 0x64, 0xe7, 0x95, 0x6e, 0xd8, 0xca, 0x06, 0x6a, - 0x48, 0xba, 0xb6, 0x74, 0x89, 0x43, 0x22, 0xac, 0x44, 0x54, 0x7e, 0x5f, 0x2d, 0xf7, 0xa3, 0x70, - 0x6a, 0x91, 0x54, 0x1e, 0x27, 0x47, 0xec, 0x1f, 0x5a, 0x70, 0x2e, 0xc3, 0xd6, 0x92, 0xb3, 0xaa, - 0x6d, 0x37, 0x8a, 0x55, 0xc6, 0x0a, 0x8d, 0x55, 0xf1, 0x72, 0xac, 0x30, 0xe8, 0x7e, 0xe0, 0xcc, - 0x30, 0xcd, 0x00, 0x85, 0x2d, 0x93, 0x80, 0x9e, 0x8c, 0x01, 0xa2, 0x2b, 0x50, 0x6a, 0x47, 0x24, - 0x94, 0xc9, 0x55, 0x25, 0xff, 0x66, 0xaa, 0x52, 0x06, 0xa1, 0xa2, 0xe9, 0xb6, 0xd2, 0x52, 0x6a, - 0xa2, 0x29, 0x57, 0x3d, 0x72, 0x98, 0xfd, 0xb5, 0x22, 0x5c, 0xcc, 0xb5, 0xa5, 0xa6, 0x5d, 0xda, - 0x0b, 0x7c, 0x37, 0x0e, 0xd4, 0x0b, 0x17, 0x8f, 0x61, 0x41, 0x5a, 0x3b, 0x6b, 0xa2, 0x1c, 0x2b, - 0x0c, 0x74, 0x55, 0xe6, 0x89, 0x4e, 0xe7, 0xe4, 0x58, 0xac, 0x1a, 0xa9, 0xa2, 0xfb, 0xcd, 0x77, - 0xf4, 0x1c, 0x94, 0x5a, 0x81, 0x4a, 0xe2, 0xaf, 0x66, 0x96, 0x76, 0x37, 0x08, 0x3c, 0xcc, 0x80, - 0xe8, 0x53, 0x62, 0x1c, 0x52, 0x4f, 0x3a, 0xd8, 0x69, 0x06, 0x91, 0x36, 0x18, 0x2f, 0xc0, 0xd0, - 0x2e, 0x39, 0x08, 0x5d, 0x7f, 0x3b, 0xfd, 0xd4, 0x77, 0x9b, 0x17, 0x63, 0x09, 0x37, 0x43, 0xd2, - 0x0f, 0x9d, 0x76, 0xa2, 0xa2, 0xe1, 0x9e, 0x47, 0xdb, 0x8f, 0x16, 0x61, 0x02, 0x2f, 0x56, 0xbf, - 0x3d, 0x11, 0xf7, 0x3a, 0x27, 0xe2, 0xb4, 0x13, 0x15, 0xf5, 0x9e, 0x8d, 0x5f, 0xb5, 0x60, 0x82, - 0x85, 0x6d, 0x15, 0x31, 0x18, 0xdc, 0xc0, 0x3f, 0x03, 0xd1, 0xed, 0x39, 0x18, 0x08, 0x69, 0xa3, - 0xe9, 0xec, 0x23, 0xac, 0x27, 0x98, 0xc3, 0xd0, 0x33, 0x50, 0x62, 0x5d, 0xa0, 0x93, 0x37, 0xca, - 0x03, 0xb7, 0x57, 0x9d, 0xd8, 0xc1, 0xac, 0x94, 0x39, 0x16, 0x63, 0xd2, 0xf2, 0x5c, 0xde, 0xe9, - 0x44, 0xd5, 0xff, 0xf1, 0x70, 0x2c, 0xce, 0xec, 0xda, 0x87, 0x73, 0x2c, 0xce, 0x26, 0xd9, 0xfd, - 0x5a, 0xf4, 0x3f, 0x0a, 0x70, 0x39, 0xb3, 0x5e, 0xdf, 0x8e, 0xc5, 0xdd, 0x6b, 0x9f, 0x8e, 0xc5, - 0x46, 0xb6, 0x21, 0x45, 0xf1, 0x0c, 0x0d, 0x29, 0x4a, 0xfd, 0x4a, 0x8e, 0x03, 0x7d, 0xf8, 0xfb, - 0x66, 0x0e, 0xd9, 0xc7, 0xc4, 0xdf, 0x37, 0xb3, 0x6f, 0x39, 0xd7, 0xba, 0xbf, 0x2c, 0xe4, 0x7c, - 0x0b, 0xbb, 0xe0, 0x5d, 0xa3, 0x7c, 0x86, 0x01, 0x23, 0x21, 0x09, 0x8f, 0x72, 0x1e, 0xc3, 0xcb, - 0xb0, 0x82, 0x22, 0x57, 0xf3, 0x9c, 0x2d, 0xe4, 0xe7, 0xa6, 0xcb, 0x6d, 0x6a, 0xce, 0x7c, 0x99, - 0x51, 0x43, 0x90, 0xe1, 0x45, 0xbb, 0xa6, 0x5d, 0xca, 0x8b, 0xfd, 0x5f, 0xca, 0x47, 0xb3, 0x2f, - 0xe4, 0x68, 0x01, 0x26, 0xf6, 0x5c, 0x9f, 0xe5, 0x1a, 0x37, 0x45, 0x51, 0x15, 0x48, 0x62, 0xcd, - 0x04, 0xe3, 0x34, 0xfe, 0xec, 0x9b, 0x30, 0xf6, 0xf8, 0xea, 0xc8, 0x6f, 0x16, 0xe1, 0xe9, 0x2e, - 0xdb, 0x9e, 0xf3, 0x7a, 0x63, 0x0e, 0x34, 0x5e, 0xdf, 0x31, 0x0f, 0x35, 0x98, 0xde, 0x6a, 0x7b, - 0xde, 0x01, 0xb3, 0x55, 0x24, 0x4d, 0x89, 0x21, 0x64, 0x45, 0x99, 0x41, 0x7e, 0x7a, 0x25, 0x03, - 0x07, 0x67, 0xd6, 0x44, 0x6f, 0x03, 0x0a, 0x44, 0x62, 0xcc, 0x9b, 0xc4, 0x17, 0xfa, 0x6e, 0x36, - 0xf0, 0xc5, 0x64, 0x33, 0xde, 0xed, 0xc0, 0xc0, 0x19, 0xb5, 0xa8, 0xd0, 0x4f, 0x4f, 0xa5, 0x03, - 0xd5, 0xad, 0x94, 0xd0, 0x8f, 0x75, 0x20, 0x36, 0x71, 0xd1, 0x4d, 0x98, 0x72, 0xf6, 0x1d, 0x97, - 0x87, 0x45, 0x93, 0x04, 0xb8, 0xd4, 0xaf, 0x94, 0x60, 0x0b, 0x69, 0x04, 0xdc, 0x59, 0x27, 0xe5, - 0xba, 0x3b, 0x98, 0xef, 0xba, 0xdb, 0x9d, 0x2f, 0xf6, 0xd2, 0xe9, 0xda, 0xff, 0xd9, 0xa2, 0xc7, - 0x57, 0x46, 0x72, 0x6b, 0x3a, 0x0e, 0x4a, 0x37, 0xa9, 0x79, 0xd1, 0xaa, 0x71, 0x58, 0xd2, 0x81, - 0xd8, 0xc4, 0xe5, 0x0b, 0x22, 0x4a, 0x1c, 0x3a, 0x0c, 0xd1, 0x5d, 0xb8, 0xc9, 0x2b, 0x0c, 0xf4, - 0x45, 0x18, 0x6a, 0xba, 0xfb, 0x6e, 0x14, 0x84, 0x62, 0xb3, 0x9c, 0xd0, 0x2c, 0x3e, 0xe1, 0x83, - 0x55, 0x4e, 0x06, 0x4b, 0x7a, 0xf6, 0x8f, 0x16, 0x60, 0x4c, 0xb6, 0xf8, 0x4e, 0x3b, 0x88, 0x9d, - 0x33, 0x38, 0x96, 0x6f, 0x1a, 0xc7, 0xf2, 0xa7, 0xba, 0xc5, 0x0a, 0x60, 0x5d, 0xca, 0x3d, 0x8e, - 0xef, 0xa6, 0x8e, 0xe3, 0xe7, 0x7b, 0x93, 0xea, 0x7e, 0x0c, 0xff, 0x0b, 0x0b, 0xa6, 0x0c, 0xfc, - 0x33, 0x38, 0x0d, 0x56, 0xcc, 0xd3, 0xe0, 0xd9, 0x9e, 0xdf, 0x90, 0x73, 0x0a, 0x7c, 0xbd, 0x90, - 0xea, 0x3b, 0xe3, 0xfe, 0xef, 0x43, 0x69, 0xc7, 0x09, 0x9b, 0xdd, 0x22, 0x89, 0x76, 0x54, 0x9a, - 0xbb, 0xe5, 0x84, 0x4d, 0xce, 0xc3, 0x5f, 0x52, 0xe9, 0xf4, 0x9c, 0xb0, 0xd9, 0xd3, 0x7f, 0x89, - 0x35, 0x85, 0x6e, 0xc0, 0x60, 0xd4, 0x08, 0x5a, 0xca, 0xba, 0xf0, 0x0a, 0x4f, 0xb5, 0x47, 0x4b, - 0x8e, 0x0f, 0x2b, 0xc8, 0x6c, 0x8e, 0x16, 0x63, 0x81, 0x3f, 0xbb, 0x0d, 0x65, 0xd5, 0xf4, 0x13, - 0xf5, 0x0d, 0xf9, 0x8f, 0x45, 0x38, 0x97, 0xb1, 0x2e, 0x50, 0x64, 0x8c, 0xd6, 0x2b, 0x7d, 0x2e, - 0xa7, 0x0f, 0x39, 0x5e, 0x11, 0xbb, 0xb1, 0x34, 0xc5, 0xfc, 0xf7, 0xdd, 0xe8, 0xbd, 0x88, 0xa4, - 0x1b, 0xa5, 0x45, 0xbd, 0x1b, 0xa5, 0x8d, 0x9d, 0xd9, 0x50, 0xd3, 0x86, 0x54, 0x4f, 0x9f, 0xe8, - 0x9c, 0xfe, 0x59, 0x11, 0xa6, 0xb3, 0x42, 0x8c, 0xa0, 0xef, 0x4f, 0xe5, 0xc5, 0x78, 0xad, 0xdf, - 0xe0, 0x24, 0x3c, 0x59, 0x86, 0xc8, 0xf2, 0x3a, 0x67, 0x66, 0xca, 0xe8, 0x39, 0xcc, 0xa2, 0x4d, - 0xe6, 0x50, 0x18, 0xf2, 0x7c, 0x26, 0x72, 0x8b, 0x7f, 0xa6, 0xef, 0x0e, 0x88, 0x44, 0x28, 0x51, - 0xca, 0xa1, 0x50, 0x16, 0xf7, 0x76, 0x28, 0x94, 0x2d, 0xcf, 0xba, 0x30, 0xa2, 0x7d, 0xcd, 0x13, - 0x9d, 0xf1, 0x5d, 0x7a, 0xa2, 0x68, 0xfd, 0x7e, 0xa2, 0xb3, 0xfe, 0xd3, 0x16, 0xa4, 0x6c, 0xfa, - 0x94, 0x4a, 0xca, 0xca, 0x55, 0x49, 0x5d, 0x81, 0x52, 0x18, 0x78, 0x24, 0x9d, 0xaa, 0x02, 0x07, - 0x1e, 0xc1, 0x0c, 0xa2, 0x92, 0xec, 0x17, 0x73, 0x93, 0xec, 0x3f, 0x07, 0x03, 0x1e, 0xd9, 0x27, - 0x52, 0x1b, 0xa1, 0x78, 0xf2, 0x1d, 0x5a, 0x88, 0x39, 0xcc, 0xfe, 0xd5, 0x12, 0x5c, 0xea, 0xea, - 0x92, 0x4b, 0xaf, 0x2c, 0xdb, 0x4e, 0x4c, 0x1e, 0x3a, 0x07, 0xe9, 0x20, 0xb7, 0x37, 0x79, 0x31, - 0x96, 0x70, 0x66, 0x81, 0xcc, 0xc3, 0xe3, 0xa5, 0x14, 0x78, 0x22, 0x2a, 0x9e, 0x80, 0x3e, 0x81, - 0x0c, 0xd7, 0xd7, 0x01, 0xa2, 0xc8, 0x5b, 0xf6, 0xa9, 0x04, 0xd6, 0x14, 0xa6, 0xcd, 0x49, 0x18, - 0xc5, 0xfa, 0x1d, 0x01, 0xc1, 0x1a, 0x16, 0xaa, 0xc2, 0x64, 0x2b, 0x0c, 0x62, 0xae, 0x0f, 0xad, - 0x72, 0x23, 0x99, 0x01, 0xd3, 0x1b, 0xb2, 0x96, 0x82, 0xe3, 0x8e, 0x1a, 0xe8, 0x75, 0x18, 0x11, - 0x1e, 0x92, 0xb5, 0x20, 0xf0, 0x84, 0xaa, 0x46, 0x99, 0x5c, 0xd4, 0x13, 0x10, 0xd6, 0xf1, 0xb4, - 0x6a, 0x4c, 0xc9, 0x3a, 0x94, 0x59, 0x8d, 0x2b, 0x5a, 0x35, 0xbc, 0x54, 0xb8, 0xa1, 0xe1, 0xbe, - 0xc2, 0x0d, 0x25, 0xca, 0xab, 0x72, 0xdf, 0xef, 0x4a, 0xd0, 0x53, 0xdd, 0xf3, 0x4b, 0x25, 0x38, - 0x27, 0x16, 0xce, 0x93, 0x5e, 0x2e, 0x4f, 0x28, 0x0f, 0xf7, 0xb7, 0xd7, 0xcc, 0x59, 0xaf, 0x99, - 0x6f, 0x14, 0x61, 0x90, 0x4f, 0xc5, 0x19, 0xc8, 0xf0, 0x2b, 0x42, 0xe9, 0xd7, 0x25, 0xd0, 0x0e, - 0xef, 0xcb, 0x5c, 0xd5, 0x89, 0x1d, 0x7e, 0x7e, 0x29, 0x36, 0x9a, 0xa8, 0x07, 0xd1, 0x9c, 0xc1, - 0x68, 0x67, 0x53, 0x5a, 0x2d, 0xe0, 0x34, 0x34, 0xb6, 0xfb, 0x65, 0x80, 0x88, 0xe5, 0x82, 0xa6, - 0x34, 0x44, 0xc8, 0xa6, 0x4f, 0x77, 0x69, 0xbd, 0xae, 0x90, 0x79, 0x1f, 0x92, 0x25, 0xa8, 0x00, - 0x58, 0xa3, 0x38, 0xfb, 0x06, 0x94, 0x15, 0x72, 0x2f, 0x15, 0xc0, 0xa8, 0x7e, 0xea, 0x7d, 0x1e, - 0x26, 0x52, 0x6d, 0x9d, 0x48, 0x83, 0xf0, 0x6b, 0x16, 0x4c, 0xf0, 0x2e, 0x2f, 0xfb, 0xfb, 0x62, - 0xb3, 0x7f, 0x00, 0xd3, 0x5e, 0xc6, 0xa6, 0x13, 0x33, 0xda, 0xff, 0x26, 0x55, 0x1a, 0x83, 0x2c, - 0x28, 0xce, 0x6c, 0x03, 0x5d, 0x83, 0x61, 0xee, 0xb2, 0xe3, 0x78, 0xc2, 0xcd, 0x62, 0x94, 0xc7, - 0xb6, 0xe7, 0x65, 0x58, 0x41, 0xed, 0xdf, 0xb7, 0x60, 0x8a, 0xf7, 0xfc, 0x36, 0x39, 0x50, 0xb7, - 0xe3, 0x8f, 0xb2, 0xef, 0x22, 0x74, 0x7f, 0x21, 0x27, 0x74, 0xbf, 0xfe, 0x69, 0xc5, 0xae, 0x9f, - 0xf6, 0x8b, 0x16, 0x88, 0x15, 0x78, 0x06, 0xf7, 0xc0, 0xef, 0x32, 0xef, 0x81, 0xb3, 0xf9, 0x8b, - 0x3a, 0xe7, 0x02, 0xf8, 0x17, 0x16, 0x4c, 0x72, 0x84, 0xe4, 0x21, 0xf2, 0x23, 0x9d, 0x87, 0x7e, - 0xf2, 0x49, 0xa9, 0x04, 0xb2, 0xd9, 0x1f, 0x65, 0x4c, 0x56, 0xa9, 0xeb, 0x64, 0x35, 0xe5, 0x06, - 0x3a, 0x41, 0x9e, 0xb4, 0x13, 0x07, 0xd3, 0xb5, 0xff, 0xd4, 0x02, 0xc4, 0x9b, 0x31, 0xce, 0x65, - 0x7a, 0xda, 0xb1, 0x52, 0x4d, 0x13, 0x94, 0xb0, 0x1a, 0x05, 0xc1, 0x1a, 0xd6, 0xa9, 0x0c, 0x4f, - 0xea, 0x35, 0xb9, 0xd8, 0xfb, 0x35, 0xf9, 0x04, 0x23, 0xfa, 0x8d, 0x12, 0xa4, 0x6d, 0xb4, 0xd1, - 0x7d, 0x18, 0x6d, 0x38, 0x2d, 0x67, 0xd3, 0xf5, 0xdc, 0xd8, 0x25, 0x51, 0x37, 0x33, 0x94, 0x25, - 0x0d, 0x4f, 0xbc, 0x13, 0x6a, 0x25, 0xd8, 0xa0, 0x83, 0xe6, 0x00, 0x5a, 0xa1, 0xbb, 0xef, 0x7a, - 0x64, 0x9b, 0x5d, 0x85, 0x99, 0x63, 0x17, 0xb7, 0xad, 0x90, 0xa5, 0x58, 0xc3, 0xc8, 0x70, 0x04, - 0x2a, 0x3e, 0x39, 0x47, 0xa0, 0xd2, 0x09, 0x1d, 0x81, 0x06, 0xfa, 0x72, 0x04, 0xc2, 0x70, 0x41, - 0x9e, 0xdd, 0xf4, 0xff, 0x8a, 0xeb, 0x11, 0x21, 0xb0, 0x71, 0x77, 0xaf, 0xd9, 0xa3, 0xc3, 0xca, - 0x05, 0x9c, 0x89, 0x81, 0x73, 0x6a, 0xa2, 0x2f, 0xc0, 0x8c, 0xe3, 0x79, 0xc1, 0x43, 0x35, 0x6a, - 0xcb, 0x51, 0xc3, 0xf1, 0xb8, 0xba, 0x77, 0x88, 0x51, 0x7d, 0xe6, 0xe8, 0xb0, 0x32, 0xb3, 0x90, - 0x83, 0x83, 0x73, 0x6b, 0xa7, 0xfc, 0x88, 0x86, 0x7b, 0xfa, 0x11, 0xed, 0xc2, 0xb9, 0x3a, 0x09, - 0x5d, 0x96, 0xc5, 0xad, 0x99, 0x6c, 0xc9, 0x0d, 0x28, 0x87, 0x29, 0x26, 0xd4, 0x57, 0x0c, 0x18, - 0x2d, 0x4e, 0xa8, 0x64, 0x3a, 0x09, 0x21, 0xfb, 0xcf, 0x2d, 0x18, 0x12, 0x76, 0xe2, 0x67, 0x20, - 0xfb, 0x2c, 0x18, 0xfa, 0xcb, 0x4a, 0x36, 0xa3, 0x66, 0x9d, 0xc9, 0xd5, 0x5c, 0xae, 0xa6, 0x34, - 0x97, 0xcf, 0x76, 0x23, 0xd2, 0x5d, 0x67, 0xf9, 0x93, 0x45, 0x18, 0x37, 0x6d, 0xe4, 0xcf, 0x60, - 0x08, 0xd6, 0x61, 0x28, 0x12, 0x0e, 0x19, 0x85, 0x7c, 0xc3, 0xd9, 0xf4, 0x24, 0x26, 0x56, 0x31, - 0xc2, 0x05, 0x43, 0x12, 0xc9, 0xf4, 0xf4, 0x28, 0x3e, 0x41, 0x4f, 0x8f, 0x5e, 0x6e, 0x0a, 0xa5, - 0xd3, 0x70, 0x53, 0xb0, 0x7f, 0x83, 0x1d, 0x16, 0x7a, 0xf9, 0x19, 0xc8, 0x11, 0x37, 0xcd, 0x63, - 0xc5, 0xee, 0xb2, 0xb2, 0x44, 0xa7, 0x72, 0xe4, 0x89, 0x7f, 0x62, 0xc1, 0x88, 0x40, 0x3c, 0x83, - 0x6e, 0x7f, 0xb7, 0xd9, 0xed, 0xa7, 0xbb, 0x74, 0x3b, 0xa7, 0xbf, 0x7f, 0xaf, 0xa0, 0xfa, 0x5b, - 0x0b, 0xc2, 0xb8, 0x8f, 0xd3, 0xff, 0x06, 0x0c, 0xd3, 0xdb, 0x63, 0xd0, 0x08, 0x3c, 0x71, 0xf8, - 0x3f, 0x93, 0x78, 0x08, 0xf3, 0xf2, 0x63, 0xed, 0x37, 0x56, 0xd8, 0xcc, 0x81, 0x35, 0x08, 0x63, - 0x71, 0xe0, 0x26, 0x0e, 0xac, 0x41, 0x18, 0x63, 0x06, 0x41, 0x4d, 0x80, 0xd8, 0x09, 0xb7, 0x49, - 0x4c, 0xcb, 0x44, 0xb0, 0x81, 0xfc, 0x5d, 0xd8, 0x8e, 0x5d, 0x6f, 0xce, 0xf5, 0xe3, 0x28, 0x0e, - 0xe7, 0x56, 0xfd, 0xf8, 0x6e, 0xc8, 0xef, 0x12, 0x9a, 0xcb, 0xaf, 0xa2, 0x85, 0x35, 0xba, 0xd2, - 0x87, 0x8c, 0xb5, 0x31, 0x60, 0x3e, 0x2c, 0xae, 0x8b, 0x72, 0xac, 0x30, 0xec, 0x37, 0x18, 0x4f, - 0x66, 0x03, 0x74, 0x32, 0x6f, 0xdc, 0xdf, 0x1d, 0x56, 0x43, 0xcb, 0x5e, 0x15, 0xaa, 0xba, 0xcf, - 0x6f, 0x77, 0x16, 0x48, 0x1b, 0xd6, 0xfd, 0x11, 0x12, 0xc7, 0x60, 0xf4, 0x3d, 0x1d, 0xef, 0xcd, - 0x2f, 0xf7, 0xe0, 0xa5, 0x27, 0x78, 0x61, 0x66, 0x01, 0x6e, 0x59, 0x20, 0xd0, 0xd5, 0x9a, 0xb8, - 0x5d, 0x6a, 0x01, 0x6e, 0x05, 0x00, 0x27, 0x38, 0x68, 0x5e, 0xdc, 0x44, 0x4b, 0x46, 0xf6, 0x28, - 0x79, 0x13, 0x95, 0x9f, 0xaf, 0x5d, 0x45, 0x5f, 0x81, 0x11, 0x95, 0x45, 0xaa, 0xc6, 0x93, 0xf1, - 0x88, 0xd0, 0x0b, 0xcb, 0x49, 0x31, 0xd6, 0x71, 0xd0, 0x06, 0x4c, 0x44, 0x3c, 0x9f, 0x96, 0x8a, - 0xb0, 0xc5, 0xf5, 0x0c, 0x9f, 0x96, 0xef, 0xd4, 0x75, 0x13, 0x7c, 0xcc, 0x8a, 0xf8, 0x66, 0x95, - 0x8e, 0x60, 0x69, 0x12, 0xe8, 0x2d, 0x18, 0xf7, 0xf4, 0xbc, 0xc2, 0x35, 0xa1, 0x86, 0x50, 0x66, - 0x9c, 0x46, 0xd6, 0xe1, 0x1a, 0x4e, 0x61, 0x53, 0xa1, 0x41, 0x2f, 0x11, 0x51, 0xe1, 0x1c, 0x7f, - 0x9b, 0x44, 0x22, 0x07, 0x0e, 0x13, 0x1a, 0xee, 0xe4, 0xe0, 0xe0, 0xdc, 0xda, 0xe8, 0x06, 0x8c, - 0xca, 0xcf, 0xd7, 0xdc, 0x1c, 0x13, 0x63, 0x61, 0x0d, 0x86, 0x0d, 0x4c, 0xf4, 0x10, 0xce, 0xcb, - 0xff, 0x1b, 0xa1, 0xb3, 0xb5, 0xe5, 0x36, 0x84, 0x97, 0x29, 0xf7, 0x98, 0x58, 0x90, 0x2e, 0x18, - 0xcb, 0x59, 0x48, 0xc7, 0x87, 0x95, 0x2b, 0x62, 0xd4, 0x32, 0xe1, 0x6c, 0x12, 0xb3, 0xe9, 0xa3, - 0x35, 0x38, 0xb7, 0x43, 0x1c, 0x2f, 0xde, 0x59, 0xda, 0x21, 0x8d, 0x5d, 0xb9, 0x89, 0x98, 0xf3, - 0xa4, 0x66, 0x62, 0x7b, 0xab, 0x13, 0x05, 0x67, 0xd5, 0x43, 0xef, 0xc2, 0x4c, 0xab, 0xbd, 0xe9, - 0xb9, 0xd1, 0xce, 0x7a, 0x10, 0xb3, 0xa7, 0x71, 0x95, 0x84, 0x49, 0x78, 0x59, 0x2a, 0xc7, 0xd1, - 0x5a, 0x0e, 0x1e, 0xce, 0xa5, 0x80, 0x3e, 0x80, 0xf3, 0xa9, 0xc5, 0x20, 0x7c, 0xbe, 0xc6, 0xf3, - 0x63, 0x6c, 0xd6, 0xb3, 0x2a, 0x08, 0x1f, 0xae, 0x2c, 0x10, 0xce, 0x6e, 0xe2, 0xc3, 0x19, 0x4c, - 0xbc, 0x4f, 0x2b, 0x6b, 0xd2, 0x0d, 0xfa, 0x0a, 0x8c, 0xea, 0xab, 0x48, 0x1c, 0x30, 0x57, 0x7b, - 0xe5, 0xd0, 0x16, 0xb2, 0x91, 0x5a, 0x51, 0x3a, 0x0c, 0x1b, 0x14, 0x6d, 0x02, 0xd9, 0xdf, 0x87, - 0xee, 0xc0, 0x70, 0xc3, 0x73, 0x89, 0x1f, 0xaf, 0xd6, 0xba, 0x45, 0x13, 0x58, 0x12, 0x38, 0x62, - 0xc0, 0x44, 0x50, 0x42, 0x5e, 0x86, 0x15, 0x05, 0xfb, 0xb7, 0x0a, 0x50, 0xe9, 0x11, 0xe1, 0x32, - 0xa5, 0x33, 0xb4, 0xfa, 0xd2, 0x19, 0x2e, 0xc8, 0x94, 0x52, 0xeb, 0xa9, 0xfb, 0x6a, 0x2a, 0x5d, - 0x54, 0x72, 0x6b, 0x4d, 0xe3, 0xf7, 0x6d, 0x67, 0xa9, 0xab, 0x1d, 0x4b, 0x3d, 0x2d, 0x80, 0x8d, - 0xe7, 0x86, 0x81, 0xfe, 0x25, 0xfa, 0x5c, 0xd5, 0xb1, 0xfd, 0x1b, 0x05, 0x38, 0xaf, 0x86, 0xf0, - 0x5b, 0x77, 0xe0, 0xee, 0x75, 0x0e, 0xdc, 0x29, 0x28, 0xde, 0xed, 0xbb, 0x30, 0x58, 0x3f, 0x88, - 0x1a, 0xb1, 0xd7, 0x87, 0x00, 0xf4, 0x9c, 0x19, 0x55, 0x47, 0x1d, 0xd3, 0x46, 0x64, 0x9d, 0xbf, - 0x6e, 0xc1, 0xc4, 0xc6, 0x52, 0xad, 0x1e, 0x34, 0x76, 0x49, 0xbc, 0xc0, 0xd5, 0x4a, 0x58, 0xc8, - 0x3f, 0xd6, 0x63, 0xca, 0x35, 0x59, 0x12, 0xd3, 0x15, 0x28, 0xed, 0x04, 0x51, 0x9c, 0x7e, 0x95, - 0xbb, 0x15, 0x44, 0x31, 0x66, 0x10, 0xfb, 0x0f, 0x2c, 0x18, 0x60, 0x89, 0x10, 0x7b, 0x65, 0xe7, - 0xec, 0xe7, 0xbb, 0xd0, 0xeb, 0x30, 0x48, 0xb6, 0xb6, 0x48, 0x23, 0x16, 0xb3, 0x2a, 0xdd, 0xfa, - 0x06, 0x97, 0x59, 0x29, 0x3d, 0xf4, 0x59, 0x63, 0xfc, 0x2f, 0x16, 0xc8, 0xe8, 0x01, 0x94, 0x63, - 0x77, 0x8f, 0x2c, 0x34, 0x9b, 0xe2, 0x5d, 0xe3, 0x31, 0xbc, 0x28, 0x37, 0x24, 0x01, 0x9c, 0xd0, - 0xb2, 0xbf, 0x56, 0x00, 0x48, 0x5c, 0x93, 0x7b, 0x7d, 0xe2, 0x62, 0x47, 0x02, 0xd2, 0xab, 0x19, - 0x09, 0x48, 0x51, 0x42, 0x30, 0x23, 0xfd, 0xa8, 0x1a, 0xa6, 0x62, 0x5f, 0xc3, 0x54, 0x3a, 0xc9, - 0x30, 0x2d, 0xc1, 0x54, 0xe2, 0x5a, 0x6d, 0xc6, 0x99, 0x60, 0x11, 0xef, 0x37, 0xd2, 0x40, 0xdc, - 0x89, 0x6f, 0xff, 0xb0, 0x05, 0xc2, 0x3d, 0xa1, 0x8f, 0xc5, 0xfc, 0x25, 0x99, 0xbe, 0xcf, 0x08, - 0x94, 0x7b, 0x25, 0xdf, 0x5f, 0x43, 0x84, 0xc7, 0x55, 0x87, 0x87, 0x11, 0x14, 0xd7, 0xa0, 0x65, - 0x37, 0x41, 0x40, 0xab, 0x84, 0x29, 0x19, 0x7a, 0xf7, 0xe6, 0x3a, 0x40, 0x93, 0xe1, 0x6a, 0xe9, - 0xc0, 0x14, 0xab, 0xaa, 0x2a, 0x08, 0xd6, 0xb0, 0xec, 0x1f, 0x2f, 0xc0, 0x88, 0x0c, 0xcc, 0x4a, - 0xef, 0xf1, 0xbd, 0x5b, 0x39, 0x51, 0x56, 0x06, 0x96, 0x3f, 0x8f, 0x12, 0x56, 0xc1, 0xfb, 0xf5, - 0xfc, 0x79, 0x12, 0x80, 0x13, 0x1c, 0xf4, 0x02, 0x0c, 0x45, 0xed, 0x4d, 0x86, 0x9e, 0x32, 0xba, - 0xaf, 0xf3, 0x62, 0x2c, 0xe1, 0xe8, 0x0b, 0x30, 0xc9, 0xeb, 0x85, 0x41, 0xcb, 0xd9, 0xe6, 0x1a, - 0xa7, 0x01, 0xe5, 0x05, 0x37, 0xb9, 0x96, 0x82, 0x1d, 0x1f, 0x56, 0xa6, 0xd3, 0x65, 0x4c, 0x57, - 0xd9, 0x41, 0xc5, 0xfe, 0x0a, 0xa0, 0xce, 0x58, 0xb3, 0xe8, 0x6d, 0x6e, 0x56, 0xe1, 0x86, 0xa4, - 0xd9, 0x77, 0xca, 0xf8, 0x51, 0x69, 0x3c, 0x41, 0x6b, 0x61, 0x55, 0x9f, 0xee, 0xbc, 0xc9, 0xb4, - 0x0b, 0x0f, 0xba, 0x05, 0x83, 0x9c, 0xa9, 0x0a, 0xf2, 0x5d, 0xde, 0xb8, 0x34, 0xc7, 0x1f, 0x16, - 0x08, 0x5f, 0xf0, 0x65, 0x51, 0x1f, 0xbd, 0x0b, 0x23, 0xcd, 0xe0, 0xa1, 0xff, 0xd0, 0x09, 0x9b, - 0x0b, 0xb5, 0x55, 0xb1, 0x2e, 0x33, 0x65, 0xb3, 0x6a, 0x82, 0xa6, 0x3b, 0x13, 0x31, 0x7d, 0x6e, - 0x02, 0xc2, 0x3a, 0x39, 0xb4, 0xc1, 0xe2, 0x67, 0xf1, 0x7c, 0xdd, 0xdd, 0xec, 0xe0, 0x54, 0x8a, - 0x6f, 0x8d, 0xf2, 0x98, 0x08, 0xb2, 0x25, 0xb2, 0x7d, 0x27, 0x84, 0xec, 0xaf, 0x9e, 0x03, 0x63, - 0x3f, 0x18, 0x79, 0x1f, 0xac, 0x53, 0xca, 0xfb, 0x80, 0x61, 0x98, 0xec, 0xb5, 0xe2, 0x83, 0xaa, - 0x1b, 0x76, 0x4b, 0x1c, 0xb4, 0x2c, 0x70, 0x3a, 0x69, 0x4a, 0x08, 0x56, 0x74, 0xb2, 0x93, 0x73, - 0x14, 0x3f, 0xc2, 0xe4, 0x1c, 0xa5, 0x33, 0x4c, 0xce, 0xb1, 0x0e, 0x43, 0xdb, 0x6e, 0x8c, 0x49, - 0x2b, 0x10, 0x02, 0x45, 0xe6, 0x4a, 0xb8, 0xc9, 0x51, 0x3a, 0x43, 0xc3, 0x0b, 0x00, 0x96, 0x44, - 0xd0, 0xdb, 0x6a, 0x0f, 0x0c, 0xe6, 0xcb, 0xe3, 0x9d, 0xcf, 0x21, 0x99, 0xbb, 0x40, 0x24, 0xe3, - 0x18, 0x7a, 0xdc, 0x64, 0x1c, 0x2b, 0x32, 0x85, 0xc6, 0x70, 0xbe, 0xd9, 0x28, 0xcb, 0x90, 0xd1, - 0x23, 0x71, 0x86, 0x91, 0x6c, 0xa4, 0x7c, 0x7a, 0xc9, 0x46, 0x7e, 0xd8, 0x82, 0xf3, 0xad, 0xac, - 0xbc, 0x3b, 0x22, 0x05, 0xc6, 0xeb, 0x7d, 0x27, 0x16, 0x32, 0x1a, 0x64, 0x17, 0xb3, 0x4c, 0x34, - 0x9c, 0xdd, 0x1c, 0x1d, 0xe8, 0x70, 0xb3, 0x29, 0xf2, 0x66, 0x3c, 0x97, 0x93, 0xb5, 0xa4, 0x4b, - 0xae, 0x92, 0x8d, 0x8c, 0x5c, 0x19, 0x9f, 0xcc, 0xcb, 0x95, 0xd1, 0x77, 0x86, 0x8c, 0x24, 0x5f, - 0xc9, 0xd8, 0x87, 0xce, 0x57, 0xf2, 0xb6, 0xca, 0x57, 0xd2, 0x25, 0xea, 0x10, 0xcf, 0x46, 0xd2, - 0x33, 0x4b, 0x89, 0x96, 0x69, 0x64, 0xe2, 0x74, 0x32, 0x8d, 0x18, 0xcc, 0x9e, 0x27, 0xbb, 0x78, - 0xb1, 0x07, 0xb3, 0x37, 0xe8, 0x76, 0x67, 0xf7, 0x3c, 0xab, 0xca, 0xd4, 0x63, 0x65, 0x55, 0xb9, - 0xaf, 0x67, 0x29, 0x41, 0x3d, 0xd2, 0x70, 0x50, 0xa4, 0x3e, 0x73, 0x93, 0xdc, 0xd7, 0x8f, 0xa0, - 0x73, 0xf9, 0x74, 0xd5, 0x49, 0xd3, 0x49, 0x37, 0xeb, 0x10, 0xea, 0xcc, 0x79, 0x32, 0x7d, 0x36, - 0x39, 0x4f, 0xce, 0x9f, 0x7a, 0xce, 0x93, 0x0b, 0x67, 0x90, 0xf3, 0xe4, 0xa9, 0x8f, 0x34, 0xe7, - 0xc9, 0xcc, 0x13, 0xc8, 0x79, 0xb2, 0x9e, 0xe4, 0x3c, 0xb9, 0x98, 0x3f, 0x25, 0x19, 0x76, 0x72, - 0x39, 0x99, 0x4e, 0xee, 0x43, 0xb9, 0x25, 0xbd, 0xbc, 0x45, 0x58, 0xa4, 0xec, 0x64, 0x8b, 0x59, - 0xae, 0xe0, 0x7c, 0x4a, 0x14, 0x08, 0x27, 0xa4, 0x28, 0xdd, 0x24, 0xf3, 0xc9, 0xd3, 0x5d, 0x54, - 0x6f, 0x59, 0x4a, 0x8d, 0xfc, 0x7c, 0x27, 0xf6, 0xdf, 0x28, 0xc0, 0xe5, 0xee, 0xeb, 0x3a, 0xd1, - 0x88, 0xd4, 0x12, 0x0d, 0x7e, 0x4a, 0x23, 0xc2, 0xaf, 0x19, 0x09, 0x56, 0xdf, 0xa1, 0x30, 0x6e, - 0xc2, 0x94, 0x32, 0x90, 0xf3, 0xdc, 0xc6, 0x81, 0x96, 0x84, 0x51, 0x39, 0xeb, 0xd4, 0xd3, 0x08, - 0xb8, 0xb3, 0x0e, 0x5a, 0x80, 0x09, 0xa3, 0x70, 0xb5, 0x2a, 0xae, 0x13, 0x4a, 0x05, 0x53, 0x37, - 0xc1, 0x38, 0x8d, 0x6f, 0x7f, 0xdd, 0x82, 0xa7, 0x72, 0xc2, 0x81, 0xf7, 0x1d, 0xe9, 0x61, 0x0b, - 0x26, 0x5a, 0x66, 0xd5, 0x1e, 0x01, 0x61, 0x8c, 0xa0, 0xe3, 0xaa, 0xaf, 0x29, 0x00, 0x4e, 0x13, - 0x5d, 0xbc, 0xf6, 0x3b, 0x7f, 0x74, 0xf9, 0x13, 0xbf, 0xf7, 0x47, 0x97, 0x3f, 0xf1, 0xfb, 0x7f, - 0x74, 0xf9, 0x13, 0xff, 0xdf, 0xd1, 0x65, 0xeb, 0x77, 0x8e, 0x2e, 0x5b, 0xbf, 0x77, 0x74, 0xd9, - 0xfa, 0xfd, 0xa3, 0xcb, 0xd6, 0x1f, 0x1e, 0x5d, 0xb6, 0xbe, 0xf6, 0xc7, 0x97, 0x3f, 0xf1, 0xa5, - 0xc2, 0xfe, 0x2b, 0xff, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x64, 0x10, 0x79, 0xf1, 0xa3, 0xdc, 0x00, - 0x00, + // 12262 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x5d, 0x6c, 0x24, 0x57, + 0x76, 0x18, 0xbc, 0xd5, 0xdd, 0xfc, 0xe9, 0xc3, 0xff, 0x3b, 0x1c, 0x89, 0x43, 0x49, 0xd3, 0xa3, + 0xd2, 0xee, 0x68, 0xb4, 0x92, 0x48, 0x6b, 0x24, 0xad, 0xe4, 0xd5, 0xae, 0x6c, 0x92, 0x4d, 0xce, + 0xb4, 0x66, 0xc8, 0x69, 0xdd, 0xe6, 0x8c, 0x76, 0xd7, 0xf2, 0x7a, 0x8b, 0xdd, 0x97, 0x64, 0x89, + 0xc5, 0xaa, 0x56, 0x55, 0x35, 0x67, 0x28, 0xd8, 0xc0, 0xf7, 0xad, 0x1d, 0x27, 0x8e, 0xfd, 0xb0, + 0x88, 0x8d, 0xc4, 0xb1, 0x0d, 0x07, 0x48, 0x1c, 0xd8, 0x1b, 0x27, 0x01, 0x1c, 0x3b, 0xb6, 0xb3, + 0xeb, 0x24, 0x8e, 0x93, 0x07, 0xe7, 0xc5, 0x71, 0xf2, 0xb2, 0x06, 0x8c, 0x30, 0x36, 0x6d, 0x24, + 0xf0, 0x43, 0x82, 0x20, 0x06, 0x02, 0x98, 0x31, 0xe2, 0xe0, 0xfe, 0xd6, 0xbd, 0xd5, 0x55, 0xdd, + 0xcd, 0x11, 0x87, 0x92, 0x8d, 0x7d, 0xeb, 0xbe, 0xe7, 0xdc, 0x73, 0x6f, 0xdd, 0xdf, 0x73, 0xce, + 0x3d, 0x3f, 0xf0, 0xc6, 0xde, 0xeb, 0xd1, 0x82, 0x1b, 0x2c, 0xee, 0x75, 0xb6, 0x48, 0xe8, 0x93, + 0x98, 0x44, 0x8b, 0x07, 0xc4, 0x6f, 0x05, 0xe1, 0xa2, 0x00, 0x38, 0x6d, 0x77, 0xb1, 0x19, 0x84, + 0x64, 0xf1, 0xe0, 0xa5, 0xc5, 0x1d, 0xe2, 0x93, 0xd0, 0x89, 0x49, 0x6b, 0xa1, 0x1d, 0x06, 0x71, + 0x80, 0x10, 0xc7, 0x59, 0x70, 0xda, 0xee, 0x02, 0xc5, 0x59, 0x38, 0x78, 0x69, 0xfe, 0xc5, 0x1d, + 0x37, 0xde, 0xed, 0x6c, 0x2d, 0x34, 0x83, 0xfd, 0xc5, 0x9d, 0x60, 0x27, 0x58, 0x64, 0xa8, 0x5b, + 0x9d, 0x6d, 0xf6, 0x8f, 0xfd, 0x61, 0xbf, 0x38, 0x89, 0xf9, 0xf5, 0xa4, 0x19, 0xf2, 0x20, 0x26, + 0x7e, 0xe4, 0x06, 0x7e, 0xf4, 0xa2, 0xd3, 0x76, 0x23, 0x12, 0x1e, 0x90, 0x70, 0xb1, 0xbd, 0xb7, + 0x43, 0x61, 0x91, 0x89, 0xb0, 0x78, 0xf0, 0xd2, 0x16, 0x89, 0x9d, 0xae, 0x1e, 0xcd, 0xbf, 0x92, + 0x90, 0xdb, 0x77, 0x9a, 0xbb, 0xae, 0x4f, 0xc2, 0x43, 0x49, 0x63, 0x31, 0x24, 0x51, 0xd0, 0x09, + 0x9b, 0xe4, 0x54, 0xb5, 0xa2, 0xc5, 0x7d, 0x12, 0x3b, 0x19, 0x5f, 0x3f, 0xbf, 0x98, 0x57, 0x2b, + 0xec, 0xf8, 0xb1, 0xbb, 0xdf, 0xdd, 0xcc, 0x67, 0xfa, 0x55, 0x88, 0x9a, 0xbb, 0x64, 0xdf, 0xe9, + 0xaa, 0xf7, 0x72, 0x5e, 0xbd, 0x4e, 0xec, 0x7a, 0x8b, 0xae, 0x1f, 0x47, 0x71, 0x98, 0xae, 0x64, + 0x7f, 0xcb, 0x82, 0x2b, 0x4b, 0xef, 0x34, 0x56, 0x3d, 0x27, 0x8a, 0xdd, 0xe6, 0xb2, 0x17, 0x34, + 0xf7, 0x1a, 0x71, 0x10, 0x92, 0x7b, 0x81, 0xd7, 0xd9, 0x27, 0x0d, 0x36, 0x10, 0xe8, 0x05, 0x18, + 0x3d, 0x60, 0xff, 0x6b, 0xd5, 0x39, 0xeb, 0x8a, 0x75, 0xad, 0xbc, 0x3c, 0xfd, 0xdb, 0x47, 0x95, + 0x4f, 0x1c, 0x1f, 0x55, 0x46, 0xef, 0x89, 0x72, 0xac, 0x30, 0xd0, 0x55, 0x18, 0xde, 0x8e, 0x36, + 0x0f, 0xdb, 0x64, 0xae, 0xc0, 0x70, 0x27, 0x05, 0xee, 0xf0, 0x5a, 0x83, 0x96, 0x62, 0x01, 0x45, + 0x8b, 0x50, 0x6e, 0x3b, 0x61, 0xec, 0xc6, 0x6e, 0xe0, 0xcf, 0x15, 0xaf, 0x58, 0xd7, 0x86, 0x96, + 0x67, 0x04, 0x6a, 0xb9, 0x2e, 0x01, 0x38, 0xc1, 0xa1, 0xdd, 0x08, 0x89, 0xd3, 0xba, 0xe3, 0x7b, + 0x87, 0x73, 0xa5, 0x2b, 0xd6, 0xb5, 0xd1, 0xa4, 0x1b, 0x58, 0x94, 0x63, 0x85, 0x61, 0xff, 0x54, + 0x01, 0x46, 0x97, 0xb6, 0xb7, 0x5d, 0xdf, 0x8d, 0x0f, 0xd1, 0x3d, 0x18, 0xf7, 0x83, 0x16, 0x91, + 0xff, 0xd9, 0x57, 0x8c, 0x5d, 0xbf, 0xb2, 0xd0, 0xbd, 0x32, 0x17, 0x36, 0x34, 0xbc, 0xe5, 0xe9, + 0xe3, 0xa3, 0xca, 0xb8, 0x5e, 0x82, 0x0d, 0x3a, 0x08, 0xc3, 0x58, 0x3b, 0x68, 0x29, 0xb2, 0x05, + 0x46, 0xb6, 0x92, 0x45, 0xb6, 0x9e, 0xa0, 0x2d, 0x4f, 0x1d, 0x1f, 0x55, 0xc6, 0xb4, 0x02, 0xac, + 0x13, 0x41, 0x5b, 0x30, 0x45, 0xff, 0xfa, 0xb1, 0xab, 0xe8, 0x16, 0x19, 0xdd, 0x67, 0xf2, 0xe8, + 0x6a, 0xa8, 0xcb, 0x17, 0x8e, 0x8f, 0x2a, 0x53, 0xa9, 0x42, 0x9c, 0x26, 0x68, 0x7f, 0x00, 0x93, + 0x4b, 0x71, 0xec, 0x34, 0x77, 0x49, 0x8b, 0xcf, 0x20, 0x7a, 0x05, 0x4a, 0xbe, 0xb3, 0x4f, 0xc4, + 0xfc, 0x5e, 0x11, 0x03, 0x5b, 0xda, 0x70, 0xf6, 0xc9, 0xc9, 0x51, 0x65, 0xfa, 0xae, 0xef, 0xbe, + 0xdf, 0x11, 0xab, 0x82, 0x96, 0x61, 0x86, 0x8d, 0xae, 0x03, 0xb4, 0xc8, 0x81, 0xdb, 0x24, 0x75, + 0x27, 0xde, 0x15, 0xf3, 0x8d, 0x44, 0x5d, 0xa8, 0x2a, 0x08, 0xd6, 0xb0, 0xec, 0x07, 0x50, 0x5e, + 0x3a, 0x08, 0xdc, 0x56, 0x3d, 0x68, 0x45, 0x68, 0x0f, 0xa6, 0xda, 0x21, 0xd9, 0x26, 0xa1, 0x2a, + 0x9a, 0xb3, 0xae, 0x14, 0xaf, 0x8d, 0x5d, 0xbf, 0x96, 0xf9, 0xb1, 0x26, 0xea, 0xaa, 0x1f, 0x87, + 0x87, 0xcb, 0x8f, 0x8b, 0xf6, 0xa6, 0x52, 0x50, 0x9c, 0xa6, 0x6c, 0xff, 0xbb, 0x02, 0x5c, 0x5c, + 0xfa, 0xa0, 0x13, 0x92, 0xaa, 0x1b, 0xed, 0xa5, 0x57, 0x78, 0xcb, 0x8d, 0xf6, 0x36, 0x92, 0x11, + 0x50, 0x4b, 0xab, 0x2a, 0xca, 0xb1, 0xc2, 0x40, 0x2f, 0xc2, 0x08, 0xfd, 0x7d, 0x17, 0xd7, 0xc4, + 0x27, 0x5f, 0x10, 0xc8, 0x63, 0x55, 0x27, 0x76, 0xaa, 0x1c, 0x84, 0x25, 0x0e, 0x5a, 0x87, 0xb1, + 0x26, 0xdb, 0x90, 0x3b, 0xeb, 0x41, 0x8b, 0xb0, 0xc9, 0x2c, 0x2f, 0x3f, 0x4f, 0xd1, 0x57, 0x92, + 0xe2, 0x93, 0xa3, 0xca, 0x1c, 0xef, 0x9b, 0x20, 0xa1, 0xc1, 0xb0, 0x5e, 0x1f, 0xd9, 0x6a, 0x7f, + 0x95, 0x18, 0x25, 0xc8, 0xd8, 0x5b, 0xd7, 0xb4, 0xad, 0x32, 0xc4, 0xb6, 0xca, 0x78, 0xf6, 0x36, + 0x41, 0x2f, 0x41, 0x69, 0xcf, 0xf5, 0x5b, 0x73, 0xc3, 0x8c, 0xd6, 0x53, 0x74, 0xce, 0x6f, 0xb9, + 0x7e, 0xeb, 0xe4, 0xa8, 0x32, 0x63, 0x74, 0x87, 0x16, 0x62, 0x86, 0x6a, 0xff, 0xa9, 0x05, 0x15, + 0x06, 0x5b, 0x73, 0x3d, 0x52, 0x27, 0x61, 0xe4, 0x46, 0x31, 0xf1, 0x63, 0x63, 0x40, 0xaf, 0x03, + 0x44, 0xa4, 0x19, 0x92, 0x58, 0x1b, 0x52, 0xb5, 0x30, 0x1a, 0x0a, 0x82, 0x35, 0x2c, 0x7a, 0x20, + 0x44, 0xbb, 0x4e, 0xc8, 0xd6, 0x97, 0x18, 0x58, 0x75, 0x20, 0x34, 0x24, 0x00, 0x27, 0x38, 0xc6, + 0x81, 0x50, 0xec, 0x77, 0x20, 0xa0, 0xcf, 0xc3, 0x54, 0xd2, 0x58, 0xd4, 0x76, 0x9a, 0x72, 0x00, + 0xd9, 0x96, 0x69, 0x98, 0x20, 0x9c, 0xc6, 0xb5, 0xff, 0x91, 0x25, 0x16, 0x0f, 0xfd, 0xea, 0x8f, + 0xf9, 0xb7, 0xda, 0xbf, 0x6e, 0xc1, 0xc8, 0xb2, 0xeb, 0xb7, 0x5c, 0x7f, 0x07, 0x7d, 0x05, 0x46, + 0xe9, 0xdd, 0xd4, 0x72, 0x62, 0x47, 0x9c, 0x7b, 0xdf, 0xa1, 0xed, 0x2d, 0x75, 0x55, 0x2c, 0xb4, + 0xf7, 0x76, 0x68, 0x41, 0xb4, 0x40, 0xb1, 0xe9, 0x6e, 0xbb, 0xb3, 0xf5, 0x1e, 0x69, 0xc6, 0xeb, + 0x24, 0x76, 0x92, 0xcf, 0x49, 0xca, 0xb0, 0xa2, 0x8a, 0x6e, 0xc1, 0x70, 0xec, 0x84, 0x3b, 0x24, + 0x16, 0x07, 0x60, 0xe6, 0x41, 0xc5, 0x6b, 0x62, 0xba, 0x23, 0x89, 0xdf, 0x24, 0xc9, 0xb5, 0xb0, + 0xc9, 0xaa, 0x62, 0x41, 0xc2, 0xfe, 0x9b, 0xc3, 0x70, 0x69, 0xa5, 0x51, 0xcb, 0x59, 0x57, 0x57, + 0x61, 0xb8, 0x15, 0xba, 0x07, 0x24, 0x14, 0xe3, 0xac, 0xa8, 0x54, 0x59, 0x29, 0x16, 0x50, 0xf4, + 0x3a, 0x8c, 0xf3, 0x0b, 0xe9, 0xa6, 0xe3, 0xb7, 0x3c, 0x39, 0xc4, 0xb3, 0x02, 0x7b, 0xfc, 0x9e, + 0x06, 0xc3, 0x06, 0xe6, 0x29, 0x17, 0xd5, 0xd5, 0xd4, 0x66, 0xcc, 0xbb, 0xec, 0x7e, 0xc4, 0x82, + 0x69, 0xde, 0xcc, 0x52, 0x1c, 0x87, 0xee, 0x56, 0x27, 0x26, 0xd1, 0xdc, 0x10, 0x3b, 0xe9, 0x56, + 0xb2, 0x46, 0x2b, 0x77, 0x04, 0x16, 0xee, 0xa5, 0xa8, 0xf0, 0x43, 0x70, 0x4e, 0xb4, 0x3b, 0x9d, + 0x06, 0xe3, 0xae, 0x66, 0xd1, 0x0f, 0x5a, 0x30, 0xdf, 0x0c, 0xfc, 0x38, 0x0c, 0x3c, 0x8f, 0x84, + 0xf5, 0xce, 0x96, 0xe7, 0x46, 0xbb, 0x7c, 0x9d, 0x62, 0xb2, 0xcd, 0x4e, 0x82, 0x9c, 0x39, 0x54, + 0x48, 0x62, 0x0e, 0x2f, 0x1f, 0x1f, 0x55, 0xe6, 0x57, 0x72, 0x49, 0xe1, 0x1e, 0xcd, 0xa0, 0x3d, + 0x40, 0xf4, 0x2a, 0x6d, 0xc4, 0xce, 0x0e, 0x49, 0x1a, 0x1f, 0x19, 0xbc, 0xf1, 0xc7, 0x8e, 0x8f, + 0x2a, 0x68, 0xa3, 0x8b, 0x04, 0xce, 0x20, 0x8b, 0xde, 0x87, 0x59, 0x5a, 0xda, 0xf5, 0xad, 0xa3, + 0x83, 0x37, 0x37, 0x77, 0x7c, 0x54, 0x99, 0xdd, 0xc8, 0x20, 0x82, 0x33, 0x49, 0xcf, 0xaf, 0xc0, + 0xc5, 0xcc, 0xa9, 0x42, 0xd3, 0x50, 0xdc, 0x23, 0x9c, 0x05, 0x29, 0x63, 0xfa, 0x13, 0xcd, 0xc2, + 0xd0, 0x81, 0xe3, 0x75, 0xc4, 0x2a, 0xc5, 0xfc, 0xcf, 0x67, 0x0b, 0xaf, 0x5b, 0x76, 0x13, 0xc6, + 0x57, 0x9c, 0xb6, 0xb3, 0xe5, 0x7a, 0x6e, 0xec, 0x92, 0x08, 0x3d, 0x0b, 0x45, 0xa7, 0xd5, 0x62, + 0x57, 0x64, 0x79, 0xf9, 0xe2, 0xf1, 0x51, 0xa5, 0xb8, 0xd4, 0xa2, 0x67, 0x35, 0x28, 0xac, 0x43, + 0x4c, 0x31, 0xd0, 0xa7, 0xa1, 0xd4, 0x0a, 0x83, 0xf6, 0x5c, 0x81, 0x61, 0xd2, 0xa1, 0x2a, 0x55, + 0xc3, 0xa0, 0x9d, 0x42, 0x65, 0x38, 0xf6, 0x6f, 0x16, 0xe0, 0xc9, 0x15, 0xd2, 0xde, 0x5d, 0x6b, + 0xe4, 0x6c, 0xba, 0x6b, 0x30, 0xba, 0x1f, 0xf8, 0x6e, 0x1c, 0x84, 0x91, 0x68, 0x9a, 0xdd, 0x26, + 0xeb, 0xa2, 0x0c, 0x2b, 0x28, 0xba, 0x02, 0xa5, 0x76, 0xc2, 0x09, 0x8c, 0x4b, 0x2e, 0x82, 0xf1, + 0x00, 0x0c, 0x42, 0x31, 0x3a, 0x11, 0x09, 0xc5, 0x2d, 0xa8, 0x30, 0xee, 0x46, 0x24, 0xc4, 0x0c, + 0x92, 0x1c, 0xa7, 0xf4, 0xa0, 0x15, 0xdb, 0x2a, 0x75, 0x9c, 0x52, 0x08, 0xd6, 0xb0, 0x50, 0x1d, + 0xca, 0x91, 0x9a, 0xd4, 0xa1, 0xc1, 0x27, 0x75, 0x82, 0x9d, 0xb7, 0x6a, 0x26, 0x13, 0x22, 0xc6, + 0x31, 0x30, 0xdc, 0xf7, 0xbc, 0xfd, 0x66, 0x01, 0x10, 0x1f, 0xc2, 0xbf, 0x64, 0x03, 0x77, 0xb7, + 0x7b, 0xe0, 0x32, 0x39, 0xaf, 0xdb, 0x41, 0xd3, 0xf1, 0xd2, 0x47, 0xf8, 0x59, 0x8d, 0xde, 0x4f, + 0x5a, 0x80, 0x56, 0x5c, 0xbf, 0x45, 0xc2, 0x73, 0x10, 0x3b, 0x4e, 0x77, 0x91, 0xde, 0x86, 0xc9, + 0x15, 0xcf, 0x25, 0x7e, 0x5c, 0xab, 0xaf, 0x04, 0xfe, 0xb6, 0xbb, 0x83, 0x3e, 0x0b, 0x93, 0x54, + 0x0a, 0x0b, 0x3a, 0x71, 0x83, 0x34, 0x03, 0x9f, 0x31, 0xac, 0x54, 0x76, 0x41, 0xc7, 0x47, 0x95, + 0xc9, 0x4d, 0x03, 0x82, 0x53, 0x98, 0xf6, 0xef, 0xd3, 0x0f, 0x0d, 0xf6, 0xdb, 0x81, 0x4f, 0xfc, + 0x78, 0x25, 0xf0, 0x5b, 0x5c, 0xb0, 0xf9, 0x2c, 0x94, 0x62, 0xda, 0x71, 0xfe, 0x91, 0x57, 0xe5, + 0xd4, 0xd2, 0xee, 0x9e, 0x1c, 0x55, 0x1e, 0xeb, 0xae, 0xc1, 0x3e, 0x88, 0xd5, 0x41, 0xdf, 0x09, + 0xc3, 0x51, 0xec, 0xc4, 0x9d, 0x48, 0x7c, 0xf6, 0xd3, 0xf2, 0xb3, 0x1b, 0xac, 0xf4, 0xe4, 0xa8, + 0x32, 0xa5, 0xaa, 0xf1, 0x22, 0x2c, 0x2a, 0xa0, 0xe7, 0x60, 0x64, 0x9f, 0x44, 0x91, 0xb3, 0x23, + 0x79, 0xd2, 0x29, 0x51, 0x77, 0x64, 0x9d, 0x17, 0x63, 0x09, 0x47, 0xcf, 0xc0, 0x10, 0x09, 0xc3, + 0x20, 0x14, 0xab, 0x6a, 0x42, 0x20, 0x0e, 0xad, 0xd2, 0x42, 0xcc, 0x61, 0xf6, 0x7f, 0xb0, 0x60, + 0x4a, 0xf5, 0x95, 0xb7, 0x75, 0x0e, 0xcc, 0xc7, 0x97, 0x00, 0x9a, 0xf2, 0x03, 0x23, 0x76, 0xde, + 0x8d, 0x5d, 0xbf, 0x9a, 0x79, 0xa5, 0x76, 0x0d, 0x63, 0x42, 0x59, 0x15, 0x45, 0x58, 0xa3, 0x66, + 0xff, 0x4b, 0x0b, 0x2e, 0xa4, 0xbe, 0xe8, 0xb6, 0x1b, 0xc5, 0xe8, 0xdd, 0xae, 0xaf, 0x5a, 0x18, + 0xec, 0xab, 0x68, 0x6d, 0xf6, 0x4d, 0x6a, 0xcd, 0xc9, 0x12, 0xed, 0x8b, 0x6e, 0xc2, 0x90, 0x1b, + 0x93, 0x7d, 0xf9, 0x31, 0xcf, 0xf4, 0xfc, 0x18, 0xde, 0xab, 0x64, 0x46, 0x6a, 0xb4, 0x26, 0xe6, + 0x04, 0xec, 0x1f, 0x2f, 0x42, 0x99, 0x2f, 0xdb, 0x75, 0xa7, 0x7d, 0x0e, 0x73, 0x51, 0x83, 0x12, + 0xa3, 0xce, 0x3b, 0xfe, 0x6c, 0x76, 0xc7, 0x45, 0x77, 0x16, 0xa8, 0x64, 0xc1, 0x99, 0x17, 0x75, + 0x98, 0xd1, 0x22, 0xcc, 0x48, 0x20, 0x07, 0x60, 0xcb, 0xf5, 0x9d, 0xf0, 0x90, 0x96, 0xcd, 0x15, + 0x19, 0xc1, 0x17, 0x7b, 0x13, 0x5c, 0x56, 0xf8, 0x9c, 0xac, 0xea, 0x6b, 0x02, 0xc0, 0x1a, 0xd1, + 0xf9, 0xd7, 0xa0, 0xac, 0x90, 0x4f, 0x73, 0x2b, 0xcf, 0x7f, 0x1e, 0xa6, 0x52, 0x6d, 0xf5, 0xab, + 0x3e, 0xae, 0x5f, 0xea, 0xdf, 0x60, 0xa7, 0x80, 0xe8, 0xf5, 0xaa, 0x7f, 0x20, 0x8e, 0xbb, 0x0f, + 0x60, 0xd6, 0xcb, 0x38, 0x65, 0xc5, 0x54, 0x0d, 0x7e, 0x2a, 0x3f, 0x29, 0x3e, 0x7b, 0x36, 0x0b, + 0x8a, 0x33, 0xdb, 0xa0, 0x17, 0x55, 0xd0, 0xa6, 0x6b, 0xde, 0xf1, 0x58, 0x7f, 0x85, 0xbc, 0x78, + 0x47, 0x94, 0x61, 0x05, 0xa5, 0x47, 0xd8, 0xac, 0xea, 0xfc, 0x2d, 0x72, 0xd8, 0x20, 0x1e, 0x69, + 0xc6, 0x41, 0xf8, 0x91, 0x76, 0xff, 0x29, 0x3e, 0xfa, 0xfc, 0x04, 0x1c, 0x13, 0x04, 0x8a, 0xb7, + 0xc8, 0x21, 0x9f, 0x0a, 0xfd, 0xeb, 0x8a, 0x3d, 0xbf, 0xee, 0x97, 0x2c, 0x98, 0x50, 0x5f, 0x77, + 0x0e, 0x5b, 0x7d, 0xd9, 0xdc, 0xea, 0x4f, 0xf5, 0x5c, 0xe0, 0x39, 0x9b, 0xfc, 0x9b, 0x05, 0xb8, + 0xa4, 0x70, 0x28, 0x83, 0xca, 0xff, 0x88, 0x55, 0xb5, 0x08, 0x65, 0x5f, 0xc9, 0xbb, 0x96, 0x29, + 0x68, 0x26, 0xd2, 0x6e, 0x82, 0x43, 0xf9, 0x0c, 0x3f, 0x11, 0x4a, 0xc7, 0x75, 0x45, 0x90, 0x50, + 0xfa, 0x2c, 0x43, 0xb1, 0xe3, 0xb6, 0xc4, 0x9d, 0xf1, 0x1d, 0x72, 0xb4, 0xef, 0xd6, 0xaa, 0x27, + 0x47, 0x95, 0xa7, 0xf3, 0x94, 0x90, 0xf4, 0xb2, 0x8a, 0x16, 0xee, 0xd6, 0xaa, 0x98, 0x56, 0x46, + 0x4b, 0x30, 0x25, 0xf5, 0xac, 0xf7, 0x28, 0xd3, 0x19, 0xf8, 0xe2, 0x6a, 0x51, 0xda, 0x1c, 0x6c, + 0x82, 0x71, 0x1a, 0x1f, 0x55, 0x61, 0x7a, 0xaf, 0xb3, 0x45, 0x3c, 0x12, 0xf3, 0x0f, 0xbe, 0x45, + 0xb8, 0xae, 0xa3, 0x9c, 0x08, 0x43, 0xb7, 0x52, 0x70, 0xdc, 0x55, 0xc3, 0xfe, 0x0b, 0x76, 0xc4, + 0x8b, 0xd1, 0xab, 0x87, 0x01, 0x5d, 0x58, 0x94, 0xfa, 0x47, 0xb9, 0x9c, 0x07, 0x59, 0x15, 0xb7, + 0xc8, 0xe1, 0x66, 0x40, 0xd9, 0xc3, 0xec, 0x55, 0x61, 0xac, 0xf9, 0x52, 0xcf, 0x35, 0xff, 0x2b, + 0x05, 0xb8, 0xa8, 0x46, 0xc0, 0x60, 0xc0, 0xfe, 0xb2, 0x8f, 0xc1, 0x4b, 0x30, 0xd6, 0x22, 0xdb, + 0x4e, 0xc7, 0x8b, 0x95, 0xe2, 0x6d, 0x88, 0x2b, 0x5f, 0xab, 0x49, 0x31, 0xd6, 0x71, 0x4e, 0x31, + 0x6c, 0x3f, 0x37, 0xc6, 0xee, 0xd6, 0xd8, 0xa1, 0x6b, 0x5c, 0xed, 0x1a, 0x2b, 0x77, 0xd7, 0x3c, + 0x03, 0x43, 0xee, 0x3e, 0xe5, 0xb5, 0x0a, 0x26, 0x0b, 0x55, 0xa3, 0x85, 0x98, 0xc3, 0xd0, 0xa7, + 0x60, 0xa4, 0x19, 0xec, 0xef, 0x3b, 0x7e, 0x8b, 0x5d, 0x79, 0xe5, 0xe5, 0x31, 0xca, 0x8e, 0xad, + 0xf0, 0x22, 0x2c, 0x61, 0xe8, 0x49, 0x28, 0x39, 0xe1, 0x4e, 0x34, 0x57, 0x62, 0x38, 0xa3, 0xb4, + 0xa5, 0xa5, 0x70, 0x27, 0xc2, 0xac, 0x94, 0xca, 0x01, 0xf7, 0x83, 0x70, 0xcf, 0xf5, 0x77, 0xaa, + 0x6e, 0x28, 0xb6, 0x84, 0xba, 0x0b, 0xdf, 0x51, 0x10, 0xac, 0x61, 0xa1, 0x35, 0x18, 0x6a, 0x07, + 0x61, 0x1c, 0xcd, 0x0d, 0xb3, 0xe1, 0x7e, 0x3a, 0xe7, 0x20, 0xe2, 0x5f, 0x5b, 0x0f, 0xc2, 0x38, + 0xf9, 0x00, 0xfa, 0x2f, 0xc2, 0xbc, 0x3a, 0xfa, 0x4e, 0x28, 0x12, 0xff, 0x60, 0x6e, 0x84, 0x51, + 0x99, 0xcf, 0xa2, 0xb2, 0xea, 0x1f, 0xdc, 0x73, 0xc2, 0xe4, 0x94, 0x5e, 0xf5, 0x0f, 0x30, 0xad, + 0x83, 0xbe, 0x08, 0x65, 0xb9, 0xc5, 0x23, 0x21, 0x98, 0x67, 0x2e, 0x31, 0x79, 0x30, 0x60, 0xf2, + 0x7e, 0xc7, 0x0d, 0xc9, 0x3e, 0xf1, 0xe3, 0x28, 0x39, 0xd3, 0x24, 0x34, 0xc2, 0x09, 0x35, 0xf4, + 0x45, 0xa9, 0x0d, 0x5a, 0x0f, 0x3a, 0x7e, 0x1c, 0xcd, 0x95, 0x59, 0xf7, 0x32, 0xf5, 0xf4, 0xf7, + 0x12, 0xbc, 0xb4, 0xba, 0x88, 0x57, 0xc6, 0x06, 0x29, 0x84, 0x61, 0xc2, 0x73, 0x0f, 0x88, 0x4f, + 0xa2, 0xa8, 0x1e, 0x06, 0x5b, 0x64, 0x0e, 0x58, 0xcf, 0x2f, 0x65, 0xab, 0xaf, 0x83, 0x2d, 0xb2, + 0x3c, 0x73, 0x7c, 0x54, 0x99, 0xb8, 0xad, 0xd7, 0xc1, 0x26, 0x09, 0x74, 0x17, 0x26, 0xa9, 0x00, + 0xe2, 0x26, 0x44, 0xc7, 0xfa, 0x11, 0x65, 0xd2, 0x07, 0x36, 0x2a, 0xe1, 0x14, 0x11, 0xf4, 0x16, + 0x94, 0x3d, 0x77, 0x9b, 0x34, 0x0f, 0x9b, 0x1e, 0x99, 0x1b, 0x67, 0x14, 0x33, 0xb7, 0xd5, 0x6d, + 0x89, 0xc4, 0x05, 0x3c, 0xf5, 0x17, 0x27, 0xd5, 0xd1, 0x3d, 0x78, 0x2c, 0x26, 0xe1, 0xbe, 0xeb, + 0x3b, 0x74, 0x3b, 0x08, 0x79, 0x81, 0x3d, 0x02, 0x4c, 0xb0, 0xf5, 0x76, 0x59, 0x0c, 0xdd, 0x63, + 0x9b, 0x99, 0x58, 0x38, 0xa7, 0x36, 0xba, 0x03, 0x53, 0x6c, 0x27, 0xd4, 0x3b, 0x9e, 0x57, 0x0f, + 0x3c, 0xb7, 0x79, 0x38, 0x37, 0xc9, 0x08, 0x7e, 0x4a, 0xde, 0x0b, 0x35, 0x13, 0x7c, 0x72, 0x54, + 0x81, 0xe4, 0x1f, 0x4e, 0xd7, 0x46, 0x5b, 0x4c, 0xeb, 0xdb, 0x09, 0xdd, 0xf8, 0x90, 0xae, 0x5f, + 0xf2, 0x20, 0x9e, 0x9b, 0xea, 0xa9, 0x1f, 0xd0, 0x51, 0x95, 0x6a, 0x58, 0x2f, 0xc4, 0x69, 0x82, + 0x74, 0x6b, 0x47, 0x71, 0xcb, 0xf5, 0xe7, 0xa6, 0xd9, 0x89, 0xa1, 0x76, 0x46, 0x83, 0x16, 0x62, + 0x0e, 0x63, 0x1a, 0x5f, 0xfa, 0xe3, 0x0e, 0x3d, 0x41, 0x67, 0x18, 0x62, 0xa2, 0xf1, 0x95, 0x00, + 0x9c, 0xe0, 0x50, 0xa6, 0x26, 0x8e, 0x0f, 0xe7, 0x10, 0x43, 0x55, 0xdb, 0x65, 0x73, 0xf3, 0x8b, + 0x98, 0x96, 0xa3, 0xdb, 0x30, 0x42, 0xfc, 0x83, 0xb5, 0x30, 0xd8, 0x9f, 0xbb, 0x90, 0xbf, 0x67, + 0x57, 0x39, 0x0a, 0x3f, 0xd0, 0x13, 0x01, 0x4f, 0x14, 0x63, 0x49, 0x02, 0x3d, 0x80, 0xb9, 0x8c, + 0x19, 0xe1, 0x13, 0x30, 0xcb, 0x26, 0xe0, 0x73, 0xa2, 0xee, 0xdc, 0x66, 0x0e, 0xde, 0x49, 0x0f, + 0x18, 0xce, 0xa5, 0x8e, 0xbe, 0x17, 0x26, 0xf8, 0x86, 0xe2, 0xcf, 0x45, 0xd1, 0xdc, 0x45, 0xf6, + 0x35, 0x57, 0xf2, 0x37, 0x27, 0x47, 0x5c, 0xbe, 0x28, 0x3a, 0x34, 0xa1, 0x97, 0x46, 0xd8, 0xa4, + 0x66, 0x6f, 0xc1, 0xa4, 0x3a, 0xb7, 0xd8, 0xd2, 0x41, 0x15, 0x18, 0x62, 0xdc, 0x8e, 0xd0, 0xc8, + 0x94, 0xe9, 0x4c, 0x31, 0x4e, 0x08, 0xf3, 0x72, 0x36, 0x53, 0xee, 0x07, 0x64, 0xf9, 0x30, 0x26, + 0x5c, 0xaa, 0x2e, 0x6a, 0x33, 0x25, 0x01, 0x38, 0xc1, 0xb1, 0xff, 0x2f, 0xe7, 0x1a, 0x93, 0xc3, + 0x71, 0x80, 0xeb, 0xe0, 0x05, 0x18, 0xdd, 0x0d, 0xa2, 0x98, 0x62, 0xb3, 0x36, 0x86, 0x12, 0x3e, + 0xf1, 0xa6, 0x28, 0xc7, 0x0a, 0x03, 0xbd, 0x01, 0x13, 0x4d, 0xbd, 0x01, 0x71, 0x97, 0xa9, 0x21, + 0x30, 0x5a, 0xc7, 0x26, 0x2e, 0x7a, 0x1d, 0x46, 0xd9, 0x63, 0x6f, 0x33, 0xf0, 0x04, 0x93, 0x25, + 0x2f, 0xe4, 0xd1, 0xba, 0x28, 0x3f, 0xd1, 0x7e, 0x63, 0x85, 0x8d, 0xae, 0xc2, 0x30, 0xed, 0x42, + 0xad, 0x2e, 0x6e, 0x11, 0xa5, 0x53, 0xb9, 0xc9, 0x4a, 0xb1, 0x80, 0xda, 0x7f, 0xab, 0xa0, 0x8d, + 0x32, 0x95, 0x48, 0x09, 0xaa, 0xc3, 0xc8, 0x7d, 0xc7, 0x8d, 0x5d, 0x7f, 0x47, 0xb0, 0x0b, 0xcf, + 0xf5, 0xbc, 0x52, 0x58, 0xa5, 0x77, 0x78, 0x05, 0x7e, 0xe9, 0x89, 0x3f, 0x58, 0x92, 0xa1, 0x14, + 0xc3, 0x8e, 0xef, 0x53, 0x8a, 0x85, 0x41, 0x29, 0x62, 0x5e, 0x81, 0x53, 0x14, 0x7f, 0xb0, 0x24, + 0x83, 0xde, 0x05, 0x90, 0xcb, 0x92, 0xb4, 0xc4, 0x23, 0xeb, 0x0b, 0xfd, 0x89, 0x6e, 0xaa, 0x3a, + 0xcb, 0x93, 0xf4, 0x4a, 0x4d, 0xfe, 0x63, 0x8d, 0x9e, 0x1d, 0x33, 0xb6, 0xaa, 0xbb, 0x33, 0xe8, + 0x7b, 0xe8, 0x49, 0xe0, 0x84, 0x31, 0x69, 0x2d, 0xc5, 0x62, 0x70, 0x3e, 0x3d, 0x98, 0x4c, 0xb1, + 0xe9, 0xee, 0x13, 0xfd, 0xd4, 0x10, 0x44, 0x70, 0x42, 0xcf, 0xfe, 0xb5, 0x22, 0xcc, 0xe5, 0x75, + 0x97, 0x2e, 0x3a, 0xf2, 0xc0, 0x8d, 0x57, 0x28, 0x37, 0x64, 0x99, 0x8b, 0x6e, 0x55, 0x94, 0x63, + 0x85, 0x41, 0x67, 0x3f, 0x72, 0x77, 0xa4, 0x48, 0x38, 0x94, 0xcc, 0x7e, 0x83, 0x95, 0x62, 0x01, + 0xa5, 0x78, 0x21, 0x71, 0x22, 0xf1, 0x8a, 0xaf, 0xad, 0x12, 0xcc, 0x4a, 0xb1, 0x80, 0xea, 0xfa, + 0xa6, 0x52, 0x1f, 0x7d, 0x93, 0x31, 0x44, 0x43, 0x67, 0x3b, 0x44, 0xe8, 0xcb, 0x00, 0xdb, 0xae, + 0xef, 0x46, 0xbb, 0x8c, 0xfa, 0xf0, 0xa9, 0xa9, 0x2b, 0x5e, 0x6a, 0x4d, 0x51, 0xc1, 0x1a, 0x45, + 0xf4, 0x2a, 0x8c, 0xa9, 0x0d, 0x58, 0xab, 0xb2, 0x27, 0x0d, 0xed, 0x89, 0x38, 0x39, 0x8d, 0xaa, + 0x58, 0xc7, 0xb3, 0xdf, 0x4b, 0xaf, 0x17, 0xb1, 0x03, 0xb4, 0xf1, 0xb5, 0x06, 0x1d, 0xdf, 0x42, + 0xef, 0xf1, 0xb5, 0x7f, 0xab, 0x08, 0x53, 0x46, 0x63, 0x9d, 0x68, 0x80, 0x33, 0xeb, 0x06, 0xbd, + 0xe7, 0x9c, 0x98, 0x88, 0xfd, 0x67, 0xf7, 0xdf, 0x2a, 0xfa, 0x5d, 0x48, 0x77, 0x00, 0xaf, 0x8f, + 0xbe, 0x0c, 0x65, 0xcf, 0x89, 0x98, 0xee, 0x8a, 0x88, 0x7d, 0x37, 0x08, 0xb1, 0x44, 0x8e, 0x70, + 0xa2, 0x58, 0xbb, 0x6a, 0x38, 0xed, 0x84, 0x24, 0xbd, 0x90, 0x29, 0xef, 0x23, 0xcd, 0x44, 0x54, + 0x27, 0x28, 0x83, 0x74, 0x88, 0x39, 0x0c, 0xbd, 0x0e, 0xe3, 0x21, 0x61, 0xab, 0x62, 0x85, 0xb2, + 0x72, 0x6c, 0x99, 0x0d, 0x25, 0x3c, 0x1f, 0xd6, 0x60, 0xd8, 0xc0, 0x4c, 0x58, 0xf9, 0xe1, 0x1e, + 0xac, 0xfc, 0x73, 0x30, 0xc2, 0x7e, 0xa8, 0x15, 0xa0, 0x66, 0xa3, 0xc6, 0x8b, 0xb1, 0x84, 0xa7, + 0x17, 0xcc, 0xe8, 0x80, 0x0b, 0xe6, 0xd3, 0x30, 0x59, 0x75, 0xc8, 0x7e, 0xe0, 0xaf, 0xfa, 0xad, + 0x76, 0xe0, 0xfa, 0x31, 0x9a, 0x83, 0x12, 0xbb, 0x1d, 0xf8, 0xde, 0x2e, 0x51, 0x0a, 0xb8, 0x44, + 0x19, 0x73, 0x7b, 0x07, 0x2e, 0x56, 0x83, 0xfb, 0xfe, 0x7d, 0x27, 0x6c, 0x2d, 0xd5, 0x6b, 0x9a, + 0x9c, 0xbb, 0x21, 0xe5, 0x2c, 0x6e, 0x76, 0x91, 0x79, 0xa6, 0x6a, 0x35, 0xf9, 0x5d, 0xbb, 0xe6, + 0x7a, 0x24, 0x47, 0x1b, 0xf1, 0x77, 0x0a, 0x46, 0x4b, 0x09, 0xbe, 0x7a, 0xe2, 0xb0, 0x72, 0x9f, + 0x38, 0xde, 0x86, 0xd1, 0x6d, 0x97, 0x78, 0x2d, 0x4c, 0xb6, 0xc5, 0x12, 0x7b, 0x36, 0xff, 0x25, + 0x79, 0x8d, 0x62, 0x4a, 0xed, 0x13, 0x97, 0xd2, 0xd6, 0x44, 0x65, 0xac, 0xc8, 0xa0, 0x3d, 0x98, + 0x96, 0x62, 0x80, 0x84, 0x8a, 0x05, 0xf7, 0x5c, 0x2f, 0xd9, 0xc2, 0x24, 0x3e, 0x7b, 0x7c, 0x54, + 0x99, 0xc6, 0x29, 0x32, 0xb8, 0x8b, 0x30, 0x15, 0xcb, 0xf6, 0xe9, 0xd1, 0x5a, 0x62, 0xc3, 0xcf, + 0xc4, 0x32, 0x26, 0x61, 0xb2, 0x52, 0xfb, 0x67, 0x2c, 0x78, 0xbc, 0x6b, 0x64, 0x84, 0xa4, 0x7d, + 0xc6, 0xb3, 0x90, 0x96, 0x7c, 0x0b, 0xfd, 0x25, 0x5f, 0xfb, 0x1f, 0x5b, 0x30, 0xbb, 0xba, 0xdf, + 0x8e, 0x0f, 0xab, 0xae, 0xf9, 0x0c, 0xf3, 0x1a, 0x0c, 0xef, 0x93, 0x96, 0xdb, 0xd9, 0x17, 0x33, + 0x57, 0x91, 0xc7, 0xcf, 0x3a, 0x2b, 0x3d, 0x39, 0xaa, 0x4c, 0x34, 0xe2, 0x20, 0x74, 0x76, 0x08, + 0x2f, 0xc0, 0x02, 0x9d, 0x1d, 0xe2, 0xee, 0x07, 0xe4, 0xb6, 0xbb, 0xef, 0x4a, 0xcb, 0x80, 0x9e, + 0xba, 0xb3, 0x05, 0x39, 0xa0, 0x0b, 0x6f, 0x77, 0x1c, 0x3f, 0x76, 0xe3, 0x43, 0xf1, 0xc2, 0x24, + 0x89, 0xe0, 0x84, 0x9e, 0xfd, 0x2d, 0x0b, 0xa6, 0xe4, 0xba, 0x5f, 0x6a, 0xb5, 0x42, 0x12, 0x45, + 0x68, 0x1e, 0x0a, 0x6e, 0x5b, 0xf4, 0x12, 0x44, 0x2f, 0x0b, 0xb5, 0x3a, 0x2e, 0xb8, 0x6d, 0x54, + 0x87, 0x32, 0x37, 0x30, 0x48, 0x16, 0xd7, 0x40, 0x66, 0x0a, 0xac, 0x07, 0x9b, 0xb2, 0x26, 0x4e, + 0x88, 0x48, 0x0e, 0x8e, 0x9d, 0x99, 0x45, 0xf3, 0x79, 0xea, 0xa6, 0x28, 0xc7, 0x0a, 0x03, 0x5d, + 0x83, 0x51, 0x3f, 0x68, 0x71, 0x7b, 0x0f, 0x7e, 0xfb, 0xb1, 0x25, 0xbb, 0x21, 0xca, 0xb0, 0x82, + 0xda, 0x3f, 0x66, 0xc1, 0xb8, 0xfc, 0xb2, 0x01, 0x99, 0x49, 0xba, 0xb5, 0x12, 0x46, 0x32, 0xd9, + 0x5a, 0x94, 0x19, 0x64, 0x10, 0x83, 0x07, 0x2c, 0x9e, 0x86, 0x07, 0xb4, 0x7f, 0xba, 0x00, 0x93, + 0xb2, 0x3b, 0x8d, 0xce, 0x56, 0x44, 0x62, 0xb4, 0x09, 0x65, 0x87, 0x0f, 0x39, 0x91, 0x2b, 0xf6, + 0x99, 0x6c, 0xe1, 0xc3, 0x98, 0x9f, 0xe4, 0x5a, 0x5e, 0x92, 0xb5, 0x71, 0x42, 0x08, 0x79, 0x30, + 0xe3, 0x07, 0x31, 0x3b, 0xa2, 0x15, 0xbc, 0xd7, 0x13, 0x48, 0x9a, 0xfa, 0x25, 0x41, 0x7d, 0x66, + 0x23, 0x4d, 0x05, 0x77, 0x13, 0x46, 0xab, 0x52, 0xe1, 0x51, 0xcc, 0x17, 0x37, 0xf4, 0x59, 0xc8, + 0xd6, 0x77, 0xd8, 0xbf, 0x61, 0x41, 0x59, 0xa2, 0x9d, 0xc7, 0x6b, 0xd7, 0x3a, 0x8c, 0x44, 0x6c, + 0x12, 0xe4, 0xd0, 0xd8, 0xbd, 0x3a, 0xce, 0xe7, 0x2b, 0xb9, 0x79, 0xf8, 0xff, 0x08, 0x4b, 0x1a, + 0x4c, 0xdf, 0xad, 0xba, 0xff, 0x31, 0xd1, 0x77, 0xab, 0xfe, 0xe4, 0xdc, 0x30, 0xff, 0x8d, 0xf5, + 0x59, 0x13, 0x6b, 0x29, 0x83, 0xd4, 0x0e, 0xc9, 0xb6, 0xfb, 0x20, 0xcd, 0x20, 0xd5, 0x59, 0x29, + 0x16, 0x50, 0xf4, 0x2e, 0x8c, 0x37, 0xa5, 0xa2, 0x33, 0x39, 0x06, 0xae, 0xf6, 0x54, 0xba, 0xab, + 0xf7, 0x19, 0x6e, 0x0b, 0xba, 0xa2, 0xd5, 0xc7, 0x06, 0x35, 0xd3, 0x06, 0xa1, 0xd8, 0xcf, 0x06, + 0x21, 0xa1, 0x9b, 0xfb, 0x8a, 0x6e, 0xff, 0xac, 0x05, 0xc3, 0x5c, 0x5d, 0x36, 0x98, 0x7e, 0x51, + 0x7b, 0xae, 0x4a, 0xc6, 0xee, 0x1e, 0x2d, 0x14, 0xcf, 0x4f, 0x68, 0x1d, 0xca, 0xec, 0x07, 0x53, + 0x1b, 0x14, 0xf3, 0x8d, 0x60, 0x79, 0xab, 0x7a, 0x07, 0xef, 0xc9, 0x6a, 0x38, 0xa1, 0x60, 0xff, + 0x44, 0x91, 0x1e, 0x55, 0x09, 0xaa, 0x71, 0x83, 0x5b, 0x8f, 0xee, 0x06, 0x2f, 0x3c, 0xaa, 0x1b, + 0x7c, 0x07, 0xa6, 0x9a, 0xda, 0xe3, 0x56, 0x32, 0x93, 0xd7, 0x7a, 0x2e, 0x12, 0xed, 0x1d, 0x8c, + 0xab, 0x8c, 0x56, 0x4c, 0x22, 0x38, 0x4d, 0x15, 0x7d, 0x0f, 0x8c, 0xf3, 0x79, 0x16, 0xad, 0x94, + 0x58, 0x2b, 0x9f, 0xca, 0x5f, 0x2f, 0x7a, 0x13, 0x6c, 0x25, 0x36, 0xb4, 0xea, 0xd8, 0x20, 0x66, + 0xff, 0xda, 0x28, 0x0c, 0xad, 0x1e, 0x10, 0x3f, 0x3e, 0x87, 0x03, 0xa9, 0x09, 0x93, 0xae, 0x7f, + 0x10, 0x78, 0x07, 0xa4, 0xc5, 0xe1, 0xa7, 0xb9, 0x5c, 0x1f, 0x13, 0xa4, 0x27, 0x6b, 0x06, 0x09, + 0x9c, 0x22, 0xf9, 0x28, 0x24, 0xcc, 0x1b, 0x30, 0xcc, 0xe7, 0x5e, 0x88, 0x97, 0x99, 0xca, 0x60, + 0x36, 0x88, 0x62, 0x17, 0x24, 0xd2, 0x2f, 0xd7, 0x3e, 0x8b, 0xea, 0xe8, 0x3d, 0x98, 0xdc, 0x76, + 0xc3, 0x28, 0xa6, 0xa2, 0x61, 0x14, 0x3b, 0xfb, 0xed, 0x87, 0x90, 0x28, 0xd5, 0x38, 0xac, 0x19, + 0x94, 0x70, 0x8a, 0x32, 0xda, 0x81, 0x09, 0x2a, 0xe4, 0x24, 0x4d, 0x8d, 0x9c, 0xba, 0x29, 0xa5, + 0x32, 0xba, 0xad, 0x13, 0xc2, 0x26, 0x5d, 0x7a, 0x98, 0x34, 0x99, 0x50, 0x34, 0xca, 0x38, 0x0a, + 0x75, 0x98, 0x70, 0x69, 0x88, 0xc3, 0xe8, 0x99, 0xc4, 0xcc, 0x56, 0xca, 0xe6, 0x99, 0xa4, 0x19, + 0xa7, 0x7c, 0x05, 0xca, 0x84, 0x0e, 0x21, 0x25, 0x2c, 0x14, 0xe3, 0x8b, 0x83, 0xf5, 0x75, 0xdd, + 0x6d, 0x86, 0x81, 0x29, 0xcb, 0xaf, 0x4a, 0x4a, 0x38, 0x21, 0x8a, 0x56, 0x60, 0x38, 0x22, 0xa1, + 0x4b, 0x22, 0xa1, 0x22, 0xef, 0x31, 0x8d, 0x0c, 0x8d, 0x5b, 0x4b, 0xf3, 0xdf, 0x58, 0x54, 0xa5, + 0xcb, 0xcb, 0x61, 0xd2, 0x10, 0xd3, 0x8a, 0x6b, 0xcb, 0x6b, 0x89, 0x95, 0x62, 0x01, 0x45, 0x6f, + 0xc1, 0x48, 0x48, 0x3c, 0xa6, 0x2c, 0x9a, 0x18, 0x7c, 0x91, 0x73, 0xdd, 0x13, 0xaf, 0x87, 0x25, + 0x01, 0x74, 0x0b, 0x50, 0x48, 0x28, 0x0f, 0xe1, 0xfa, 0x3b, 0xca, 0x98, 0x43, 0xe8, 0xba, 0x9f, + 0x10, 0xed, 0x5f, 0xc0, 0x09, 0x86, 0xb4, 0xa3, 0xc4, 0x19, 0xd5, 0xd0, 0x0d, 0x98, 0x51, 0xa5, + 0x35, 0x3f, 0x8a, 0x1d, 0xbf, 0x49, 0x98, 0x9a, 0xbb, 0x9c, 0x70, 0x45, 0x38, 0x8d, 0x80, 0xbb, + 0xeb, 0xd8, 0x5f, 0xa7, 0xec, 0x0c, 0x1d, 0xad, 0x73, 0xe0, 0x05, 0xde, 0x34, 0x79, 0x81, 0x4b, + 0xb9, 0x33, 0x97, 0xc3, 0x07, 0x1c, 0x5b, 0x30, 0xa6, 0xcd, 0x6c, 0xb2, 0x66, 0xad, 0x1e, 0x6b, + 0xb6, 0x03, 0xd3, 0x74, 0xa5, 0xdf, 0xd9, 0x62, 0x8e, 0x43, 0x2d, 0xb6, 0x30, 0x0b, 0x0f, 0xb7, + 0x30, 0xd5, 0x2b, 0xf3, 0xed, 0x14, 0x41, 0xdc, 0xd5, 0x04, 0x7a, 0x4d, 0x6a, 0x4e, 0x8a, 0x86, + 0x91, 0x16, 0xd7, 0x8a, 0x9c, 0x1c, 0x55, 0xa6, 0xb5, 0x0f, 0xd1, 0x35, 0x25, 0xf6, 0x57, 0xe4, + 0x37, 0xaa, 0xd7, 0xfc, 0xa6, 0x5a, 0x2c, 0xa9, 0xd7, 0x7c, 0xb5, 0x1c, 0x70, 0x82, 0x43, 0xf7, + 0x28, 0x15, 0x41, 0xd2, 0xaf, 0xf9, 0x54, 0x40, 0xc1, 0x0c, 0x62, 0xbf, 0x0c, 0xb0, 0xfa, 0x80, + 0x34, 0xf9, 0x52, 0xd7, 0x1f, 0x20, 0xad, 0xfc, 0x07, 0x48, 0xfb, 0x3f, 0x59, 0x30, 0xb9, 0xb6, + 0x62, 0x88, 0x89, 0x0b, 0x00, 0x5c, 0x36, 0x7a, 0xe7, 0x9d, 0x0d, 0xa9, 0x5b, 0xe7, 0xea, 0x51, + 0x55, 0x8a, 0x35, 0x0c, 0x74, 0x09, 0x8a, 0x5e, 0xc7, 0x17, 0x22, 0xcb, 0xc8, 0xf1, 0x51, 0xa5, + 0x78, 0xbb, 0xe3, 0x63, 0x5a, 0xa6, 0x99, 0xf2, 0x15, 0x07, 0x36, 0xe5, 0xeb, 0xeb, 0x10, 0x84, + 0x2a, 0x30, 0x74, 0xff, 0xbe, 0xdb, 0xe2, 0x66, 0xd7, 0x42, 0xef, 0xff, 0xce, 0x3b, 0xb5, 0x6a, + 0x84, 0x79, 0xb9, 0xfd, 0xb5, 0x22, 0xcc, 0xaf, 0x79, 0xe4, 0xc1, 0x87, 0x34, 0x3d, 0x1f, 0xd4, + 0x10, 0xf1, 0x74, 0xfc, 0xe2, 0x69, 0xad, 0x2e, 0xfb, 0x8f, 0xc7, 0x36, 0x8c, 0xf0, 0xc7, 0x6c, + 0x69, 0x88, 0xfe, 0x46, 0x56, 0xeb, 0xf9, 0x03, 0xb2, 0xc0, 0x1f, 0xc5, 0x85, 0x01, 0xba, 0xba, + 0x69, 0x45, 0x29, 0x96, 0xc4, 0xe7, 0x3f, 0x0b, 0xe3, 0x3a, 0xe6, 0xa9, 0xec, 0x9f, 0xff, 0xff, + 0x22, 0x4c, 0xd3, 0x1e, 0x3c, 0xd2, 0x89, 0xb8, 0xdb, 0x3d, 0x11, 0x67, 0x6d, 0x03, 0xdb, 0x7f, + 0x36, 0xde, 0x4d, 0xcf, 0xc6, 0x4b, 0x79, 0xb3, 0x71, 0xde, 0x73, 0xf0, 0x83, 0x16, 0x5c, 0x58, + 0xf3, 0x82, 0xe6, 0x5e, 0xca, 0x3c, 0xf7, 0x55, 0x18, 0xa3, 0xe7, 0x78, 0x64, 0xf8, 0xbd, 0x18, + 0x9e, 0x50, 0x02, 0x84, 0x75, 0x3c, 0xad, 0xda, 0xdd, 0xbb, 0xb5, 0x6a, 0x96, 0x03, 0x95, 0x00, + 0x61, 0x1d, 0xcf, 0xfe, 0x1d, 0x0b, 0x9e, 0xba, 0xb1, 0xb2, 0x9a, 0x2c, 0xc5, 0x2e, 0x1f, 0x2e, + 0x2a, 0x05, 0xb6, 0xb4, 0xae, 0x24, 0x52, 0x60, 0x95, 0xf5, 0x42, 0x40, 0x3f, 0x2e, 0xfe, 0x89, + 0xbf, 0x60, 0xc1, 0x85, 0x1b, 0x6e, 0x4c, 0xaf, 0xe5, 0xb4, 0x37, 0x11, 0xbd, 0x97, 0x23, 0x37, + 0x0e, 0xc2, 0xc3, 0xb4, 0x37, 0x11, 0x56, 0x10, 0xac, 0x61, 0xf1, 0x96, 0x0f, 0x5c, 0x66, 0x46, + 0x55, 0x30, 0x55, 0x51, 0x58, 0x94, 0x63, 0x85, 0x41, 0x3f, 0xac, 0xe5, 0x86, 0x4c, 0x94, 0x38, + 0x14, 0x27, 0xac, 0xfa, 0xb0, 0xaa, 0x04, 0xe0, 0x04, 0xc7, 0xfe, 0x19, 0x0b, 0x2e, 0xde, 0xf0, + 0x3a, 0x51, 0x4c, 0xc2, 0xed, 0xc8, 0xe8, 0xec, 0xcb, 0x50, 0x26, 0x52, 0x5c, 0x17, 0x7d, 0x55, + 0x0c, 0xa6, 0x92, 0xe3, 0xb9, 0x2b, 0x93, 0xc2, 0x1b, 0xc0, 0xd6, 0xfd, 0x74, 0x36, 0xda, 0xbf, + 0x5c, 0x80, 0x89, 0x9b, 0x9b, 0x9b, 0xf5, 0x1b, 0x24, 0x16, 0xb7, 0x58, 0x7f, 0x55, 0x33, 0xd6, + 0x34, 0x66, 0xbd, 0x84, 0xa2, 0x4e, 0xec, 0x7a, 0x0b, 0xdc, 0x77, 0x76, 0xa1, 0xe6, 0xc7, 0x77, + 0xc2, 0x46, 0x1c, 0xba, 0xfe, 0x4e, 0xa6, 0x8e, 0x4d, 0xde, 0xb5, 0xc5, 0xbc, 0xbb, 0x16, 0xbd, + 0x0c, 0xc3, 0xcc, 0x79, 0x57, 0x8a, 0x27, 0x4f, 0x28, 0x99, 0x82, 0x95, 0x9e, 0x1c, 0x55, 0xca, + 0x77, 0x71, 0x8d, 0xff, 0xc1, 0x02, 0x15, 0xdd, 0x85, 0xb1, 0xdd, 0x38, 0x6e, 0xdf, 0x24, 0x4e, + 0x8b, 0x84, 0xf2, 0x74, 0xb8, 0x9c, 0x75, 0x3a, 0xd0, 0x41, 0xe0, 0x68, 0xc9, 0x86, 0x4a, 0xca, + 0x22, 0xac, 0xd3, 0xb1, 0x1b, 0x00, 0x09, 0xec, 0x8c, 0xf4, 0x0b, 0xf6, 0x1f, 0x59, 0x30, 0xc2, + 0xfd, 0xa8, 0x42, 0xf4, 0x39, 0x28, 0x91, 0x07, 0xa4, 0x29, 0x38, 0xc7, 0xcc, 0x0e, 0x27, 0x8c, + 0x07, 0xd7, 0x96, 0xd3, 0xff, 0x98, 0xd5, 0x42, 0x37, 0x61, 0x84, 0xf6, 0xf6, 0x86, 0x72, 0x2a, + 0x7b, 0x3a, 0xef, 0x8b, 0xd5, 0xb4, 0x73, 0x5e, 0x45, 0x14, 0x61, 0x59, 0x9d, 0x69, 0x7e, 0x9b, + 0xed, 0x06, 0x3d, 0xc0, 0xe2, 0x5e, 0xf7, 0xec, 0xe6, 0x4a, 0x9d, 0x23, 0x09, 0x6a, 0x5c, 0xf3, + 0x2b, 0x0b, 0x71, 0x42, 0xc4, 0xde, 0x84, 0x32, 0x9d, 0xd4, 0x25, 0xcf, 0x75, 0x7a, 0x2b, 0x9d, + 0x9f, 0x87, 0xb2, 0x54, 0x00, 0x47, 0xc2, 0x15, 0x87, 0x51, 0x95, 0xfa, 0xe1, 0x08, 0x27, 0x70, + 0x7b, 0x1b, 0x66, 0xd9, 0xcb, 0xbf, 0x13, 0xef, 0x1a, 0x7b, 0xac, 0xff, 0x62, 0x7e, 0x41, 0x08, + 0x62, 0x7c, 0x66, 0xe6, 0x34, 0xdf, 0x81, 0x71, 0x49, 0x31, 0x11, 0xca, 0xec, 0x3f, 0x29, 0xc1, + 0x13, 0xb5, 0x46, 0xbe, 0x8b, 0xdd, 0xeb, 0x30, 0xce, 0xd9, 0x34, 0xba, 0xb4, 0x1d, 0x4f, 0xb4, + 0xab, 0xde, 0xc5, 0x36, 0x35, 0x18, 0x36, 0x30, 0xd1, 0x53, 0x50, 0x74, 0xdf, 0xf7, 0xd3, 0x66, + 0xb8, 0xb5, 0xb7, 0x37, 0x30, 0x2d, 0xa7, 0x60, 0xca, 0xf1, 0xf1, 0xa3, 0x54, 0x81, 0x15, 0xd7, + 0xf7, 0x26, 0x4c, 0xba, 0x51, 0x33, 0x72, 0x6b, 0x3e, 0x3d, 0x67, 0x12, 0xf7, 0xcc, 0x44, 0x49, + 0x40, 0x3b, 0xad, 0xa0, 0x38, 0x85, 0xad, 0x9d, 0xeb, 0x43, 0x03, 0x73, 0x8d, 0x7d, 0x7d, 0x53, + 0x28, 0x43, 0xdc, 0x66, 0x5f, 0x17, 0x31, 0xa3, 0x36, 0xc1, 0x10, 0xf3, 0x0f, 0x8e, 0xb0, 0x84, + 0x51, 0x09, 0xac, 0xb9, 0xeb, 0xb4, 0x97, 0x3a, 0xf1, 0x6e, 0xd5, 0x8d, 0x9a, 0xc1, 0x01, 0x09, + 0x0f, 0x99, 0xf0, 0x3c, 0x9a, 0x48, 0x60, 0x0a, 0xb0, 0x72, 0x73, 0xa9, 0x4e, 0x31, 0x71, 0x77, + 0x1d, 0x93, 0x2b, 0x84, 0xb3, 0xe0, 0x0a, 0x97, 0x60, 0x4a, 0x36, 0xd3, 0x20, 0x11, 0xbb, 0x23, + 0xc6, 0x58, 0xc7, 0x94, 0xa9, 0xad, 0x28, 0x56, 0xdd, 0x4a, 0xe3, 0xa3, 0xd7, 0x60, 0xc2, 0xf5, + 0xdd, 0xd8, 0x75, 0xe2, 0x20, 0x64, 0x37, 0x2c, 0x97, 0x93, 0x99, 0x25, 0x5b, 0x4d, 0x07, 0x60, + 0x13, 0xcf, 0xfe, 0xe3, 0x12, 0xcc, 0xb0, 0x69, 0xfb, 0xf6, 0x0a, 0xfb, 0xd8, 0xac, 0xb0, 0xbb, + 0xdd, 0x2b, 0xec, 0x2c, 0xd8, 0xdd, 0x8f, 0x72, 0x99, 0xbd, 0x07, 0x65, 0x65, 0x0b, 0x2c, 0x9d, + 0x01, 0xac, 0x1c, 0x67, 0x80, 0xfe, 0xdc, 0x87, 0x7c, 0xc6, 0x2d, 0x66, 0x3e, 0xe3, 0xfe, 0x5d, + 0x0b, 0x12, 0x93, 0x48, 0x74, 0x13, 0xca, 0xed, 0x80, 0x99, 0x1d, 0x84, 0xd2, 0x96, 0xe7, 0x89, + 0xcc, 0x8b, 0x8a, 0x5f, 0x8a, 0x7c, 0xfc, 0xea, 0xb2, 0x06, 0x4e, 0x2a, 0xa3, 0x65, 0x18, 0x69, + 0x87, 0xa4, 0x11, 0x33, 0xa7, 0xcd, 0xbe, 0x74, 0xf8, 0x1a, 0xe1, 0xf8, 0x58, 0x56, 0xb4, 0x7f, + 0xc5, 0x02, 0xe0, 0x2f, 0xa5, 0x8e, 0xbf, 0x43, 0xce, 0x41, 0xfb, 0x5b, 0x85, 0x52, 0xd4, 0x26, + 0xcd, 0x5e, 0x06, 0x21, 0x49, 0x7f, 0x1a, 0x6d, 0xd2, 0x4c, 0x06, 0x9c, 0xfe, 0xc3, 0xac, 0xb6, + 0xfd, 0xd7, 0x00, 0x26, 0x13, 0xb4, 0x5a, 0x4c, 0xf6, 0xd1, 0x8b, 0x86, 0x4b, 0xdc, 0xa5, 0x94, + 0x4b, 0x5c, 0x99, 0x61, 0x6b, 0x8a, 0xc6, 0xf7, 0xa0, 0xb8, 0xef, 0x3c, 0x10, 0x9a, 0xa4, 0xe7, + 0x7b, 0x77, 0x83, 0xd2, 0x5f, 0x58, 0x77, 0x1e, 0x70, 0x99, 0xe9, 0x79, 0xb9, 0x40, 0xd6, 0x9d, + 0x07, 0x27, 0xdc, 0xec, 0x83, 0x1d, 0x52, 0xb7, 0xdd, 0x28, 0xfe, 0xea, 0x7f, 0x49, 0xfe, 0xb3, + 0x65, 0x47, 0x1b, 0x61, 0x6d, 0xb9, 0xbe, 0x78, 0x37, 0x1c, 0xa8, 0x2d, 0xd7, 0x4f, 0xb7, 0xe5, + 0xfa, 0x03, 0xb4, 0xe5, 0xfa, 0xe8, 0x03, 0x18, 0x11, 0x6f, 0xf4, 0xcc, 0xd6, 0xdb, 0xd4, 0x52, + 0xe5, 0xb5, 0x27, 0x9e, 0xf8, 0x79, 0x9b, 0x8b, 0x52, 0x26, 0x14, 0xa5, 0x7d, 0xdb, 0x95, 0x0d, + 0xa2, 0xbf, 0x6d, 0xc1, 0xa4, 0xf8, 0x8d, 0xc9, 0xfb, 0x1d, 0x12, 0xc5, 0x82, 0xf7, 0xfc, 0xcc, + 0xe0, 0x7d, 0x10, 0x15, 0x79, 0x57, 0x3e, 0x23, 0x8f, 0x59, 0x13, 0xd8, 0xb7, 0x47, 0xa9, 0x5e, + 0xa0, 0x7f, 0x6a, 0xc1, 0xec, 0xbe, 0xf3, 0x80, 0xb7, 0xc8, 0xcb, 0xb0, 0x13, 0xbb, 0x81, 0xb0, + 0x5d, 0xff, 0xdc, 0x60, 0xd3, 0xdf, 0x55, 0x9d, 0x77, 0x52, 0x9a, 0xb9, 0xce, 0x66, 0xa1, 0xf4, + 0xed, 0x6a, 0x66, 0xbf, 0xe6, 0xb7, 0x61, 0x54, 0xae, 0xb7, 0x0c, 0xc9, 0xbb, 0xaa, 0x33, 0xd6, + 0xa7, 0x36, 0x91, 0xd0, 0xfd, 0xd2, 0x68, 0x3b, 0x62, 0xad, 0x3d, 0xd2, 0x76, 0xde, 0x83, 0x71, + 0x7d, 0x8d, 0x3d, 0xd2, 0xb6, 0xde, 0x87, 0x0b, 0x19, 0x6b, 0xe9, 0x91, 0x36, 0x79, 0x1f, 0x2e, + 0xe5, 0xae, 0x8f, 0x47, 0xd9, 0xb0, 0xfd, 0xcb, 0x96, 0x7e, 0x0e, 0x9e, 0x83, 0x0a, 0x7e, 0xc5, + 0x54, 0xc1, 0x5f, 0xee, 0xbd, 0x73, 0x72, 0xf4, 0xf0, 0xef, 0xea, 0x9d, 0xa6, 0xa7, 0x3a, 0x7a, + 0x0b, 0x86, 0x3d, 0x5a, 0x22, 0x8d, 0x43, 0xec, 0xfe, 0x3b, 0x32, 0xe1, 0xa5, 0x58, 0x79, 0x84, + 0x05, 0x05, 0xfb, 0xd7, 0x2d, 0x28, 0x9d, 0xc3, 0x48, 0x60, 0x73, 0x24, 0x5e, 0xcc, 0x25, 0x2d, + 0x82, 0x70, 0x2d, 0x60, 0xe7, 0xfe, 0xaa, 0x0c, 0x34, 0x96, 0x33, 0x30, 0xdf, 0x07, 0x17, 0x6e, + 0x07, 0x4e, 0x6b, 0xd9, 0xf1, 0x1c, 0xbf, 0x49, 0xc2, 0x9a, 0xbf, 0xd3, 0xd7, 0x4a, 0x49, 0xb7, + 0x29, 0x2a, 0xf4, 0xb3, 0x29, 0xb2, 0x77, 0x01, 0xe9, 0x0d, 0x08, 0x3b, 0x4e, 0x0c, 0x23, 0x2e, + 0x6f, 0x4a, 0x0c, 0xff, 0xb3, 0xd9, 0xdc, 0x5d, 0x57, 0xcf, 0x34, 0x0b, 0x45, 0x5e, 0x80, 0x25, + 0x21, 0xfb, 0x75, 0xc8, 0xf4, 0xdd, 0xea, 0xaf, 0x36, 0xb0, 0x5f, 0x85, 0x19, 0x56, 0xf3, 0x74, + 0x22, 0xad, 0xfd, 0x23, 0x16, 0x4c, 0x6d, 0xa4, 0xa2, 0x29, 0x5c, 0x65, 0x6f, 0x7d, 0x19, 0x7a, + 0xdf, 0x06, 0x2b, 0xc5, 0x02, 0x7a, 0xe6, 0xfa, 0xa5, 0xbf, 0xb0, 0x20, 0x71, 0x95, 0x3c, 0x07, + 0xa6, 0x6a, 0xc5, 0x60, 0xaa, 0x32, 0xf5, 0x1e, 0xaa, 0x3b, 0x79, 0x3c, 0x15, 0xba, 0xa5, 0xe2, + 0x02, 0xf4, 0x50, 0x79, 0x24, 0x64, 0xb8, 0x17, 0xf9, 0xa4, 0x19, 0x3c, 0x40, 0x46, 0x0a, 0x60, + 0x66, 0x42, 0x0a, 0xf7, 0x63, 0x62, 0x26, 0xa4, 0xfa, 0x93, 0xb3, 0xfb, 0xea, 0x5a, 0x97, 0xd9, + 0xa9, 0xf4, 0x5d, 0xcc, 0xec, 0xdb, 0xf1, 0xdc, 0x0f, 0x88, 0x0a, 0xc7, 0x51, 0x11, 0x66, 0xdc, + 0xa2, 0xf4, 0xe4, 0xa8, 0x32, 0xa1, 0xfe, 0xf1, 0x98, 0x4d, 0x49, 0x15, 0xfb, 0x26, 0x4c, 0xa5, + 0x06, 0x0c, 0xbd, 0x0a, 0x43, 0xed, 0x5d, 0x27, 0x22, 0x29, 0xd3, 0xc8, 0xa1, 0x3a, 0x2d, 0x3c, + 0x39, 0xaa, 0x4c, 0xaa, 0x0a, 0xac, 0x04, 0x73, 0x6c, 0xfb, 0x7f, 0x5a, 0x50, 0xda, 0x08, 0x5a, + 0xe7, 0xb1, 0x98, 0xde, 0x34, 0x16, 0xd3, 0x93, 0x79, 0x11, 0xef, 0x72, 0xd7, 0xd1, 0x5a, 0x6a, + 0x1d, 0x5d, 0xce, 0xa5, 0xd0, 0x7b, 0x09, 0xed, 0xc3, 0x18, 0x8b, 0xa3, 0x27, 0x4c, 0x35, 0x5f, + 0x36, 0xf8, 0xfb, 0x4a, 0x8a, 0xbf, 0x9f, 0xd2, 0x50, 0x35, 0x2e, 0xff, 0x39, 0x18, 0x11, 0xe6, + 0x82, 0x69, 0x03, 0x77, 0x81, 0x8b, 0x25, 0xdc, 0xfe, 0xd9, 0x22, 0x18, 0x71, 0xfb, 0xd0, 0x6f, + 0x58, 0xb0, 0x10, 0x72, 0x8f, 0xc1, 0x56, 0xb5, 0x13, 0xba, 0xfe, 0x4e, 0xa3, 0xb9, 0x4b, 0x5a, + 0x1d, 0xcf, 0xf5, 0x77, 0x6a, 0x3b, 0x7e, 0xa0, 0x8a, 0x57, 0x1f, 0x90, 0x66, 0x87, 0xe9, 0xfc, + 0xfb, 0x04, 0x09, 0x54, 0xe6, 0x38, 0xd7, 0x8f, 0x8f, 0x2a, 0x0b, 0xf8, 0x54, 0xb4, 0xf1, 0x29, + 0xfb, 0x82, 0x7e, 0xc7, 0x82, 0x45, 0x1e, 0xce, 0x6e, 0xf0, 0xfe, 0xf7, 0x90, 0x86, 0xea, 0x92, + 0x54, 0x42, 0x64, 0x93, 0x84, 0xfb, 0xcb, 0xaf, 0x89, 0x01, 0x5d, 0xac, 0x9f, 0xae, 0x2d, 0x7c, + 0xda, 0xce, 0xd9, 0xff, 0xa6, 0x08, 0x13, 0xc2, 0x59, 0x5d, 0x44, 0x41, 0x79, 0xd5, 0x58, 0x12, + 0x4f, 0xa7, 0x96, 0xc4, 0x8c, 0x81, 0x7c, 0x36, 0x01, 0x50, 0x22, 0x98, 0xf1, 0x9c, 0x28, 0xbe, + 0x49, 0x9c, 0x30, 0xde, 0x22, 0x0e, 0x37, 0x53, 0x29, 0x9e, 0xda, 0xa4, 0x46, 0xa9, 0x5f, 0x6e, + 0xa7, 0x89, 0xe1, 0x6e, 0xfa, 0xe8, 0x00, 0x10, 0xb3, 0xb5, 0x09, 0x1d, 0x3f, 0xe2, 0xdf, 0xe2, + 0x8a, 0xf7, 0x80, 0xd3, 0xb5, 0x3a, 0x2f, 0x5a, 0x45, 0xb7, 0xbb, 0xa8, 0xe1, 0x8c, 0x16, 0x34, + 0x1b, 0xaa, 0xa1, 0x41, 0x6d, 0xa8, 0x86, 0xfb, 0x78, 0x91, 0xf8, 0x30, 0xdd, 0x15, 0x6f, 0xe0, + 0x4b, 0x50, 0x56, 0xb6, 0x6e, 0xe2, 0xd0, 0xe9, 0x1d, 0xb6, 0x23, 0x4d, 0x81, 0xab, 0x48, 0x12, + 0x3b, 0xcb, 0x84, 0x9c, 0xfd, 0xfd, 0x70, 0x81, 0x62, 0x9b, 0x4e, 0x0f, 0x11, 0x22, 0x30, 0x25, + 0xdc, 0xfa, 0x65, 0x99, 0xd8, 0xba, 0x99, 0x7c, 0xa7, 0x59, 0x3b, 0xd1, 0x4e, 0xdd, 0x32, 0x49, + 0xe0, 0x34, 0x4d, 0xfb, 0xe7, 0x2d, 0x60, 0xe6, 0xda, 0xe7, 0x70, 0xff, 0x7d, 0xde, 0xbc, 0xff, + 0xe6, 0xf2, 0x8e, 0xa0, 0x9c, 0xab, 0xef, 0x15, 0x3e, 0x2f, 0xf5, 0x30, 0x78, 0x70, 0x28, 0x9e, + 0x7d, 0x07, 0x60, 0xb9, 0xfe, 0x8f, 0xc5, 0x77, 0xa4, 0xf2, 0xa0, 0x46, 0x3f, 0x00, 0xa3, 0x4d, + 0xa7, 0xed, 0x34, 0x79, 0xc4, 0xd4, 0x5c, 0xf5, 0x83, 0x51, 0x69, 0x61, 0x45, 0xd4, 0xe0, 0xe2, + 0xb4, 0x0c, 0x0f, 0x31, 0x2a, 0x8b, 0xfb, 0x8a, 0xd0, 0xaa, 0xc9, 0xf9, 0x3d, 0x98, 0x30, 0x88, + 0x3d, 0x52, 0xd9, 0xeb, 0x07, 0xf8, 0x7d, 0xa1, 0xc2, 0x99, 0xec, 0xc3, 0x8c, 0xaf, 0xfd, 0xa7, + 0xa7, 0xa3, 0xe4, 0xa7, 0x3f, 0xd9, 0xef, 0x46, 0x60, 0x47, 0xa9, 0x66, 0x8e, 0x9e, 0x22, 0x83, + 0xbb, 0x29, 0xdb, 0x7f, 0xdf, 0x82, 0xc7, 0x75, 0x44, 0xcd, 0xb9, 0xbd, 0x9f, 0x42, 0xb3, 0x0a, + 0xa3, 0x41, 0x9b, 0x84, 0x4e, 0x1c, 0x84, 0xe2, 0x08, 0xbc, 0x26, 0x07, 0xfd, 0x8e, 0x28, 0x3f, + 0x11, 0xa1, 0xeb, 0x24, 0x75, 0x59, 0x8e, 0x55, 0x4d, 0x64, 0xc3, 0x30, 0x1b, 0x8c, 0x48, 0x04, + 0x1e, 0x60, 0x76, 0x72, 0xec, 0x6d, 0x2f, 0xc2, 0x02, 0x62, 0xff, 0x89, 0xc5, 0x17, 0x96, 0xde, + 0x75, 0xf4, 0x3e, 0x4c, 0xef, 0x3b, 0x71, 0x73, 0x77, 0xf5, 0x41, 0x3b, 0xe4, 0x7a, 0x5c, 0x39, + 0x4e, 0xcf, 0xf7, 0x1b, 0x27, 0xed, 0x23, 0x13, 0x6b, 0xaa, 0xf5, 0x14, 0x31, 0xdc, 0x45, 0x1e, + 0x6d, 0xc1, 0x18, 0x2b, 0x63, 0x66, 0xbb, 0x51, 0xaf, 0x7b, 0x2e, 0xaf, 0x35, 0xf5, 0x3c, 0xba, + 0x9e, 0xd0, 0xc1, 0x3a, 0x51, 0xfb, 0xab, 0x45, 0xbe, 0xdb, 0x19, 0xeb, 0xf8, 0x1c, 0x8c, 0xb4, + 0x83, 0xd6, 0x4a, 0xad, 0x8a, 0xc5, 0x2c, 0xa8, 0x33, 0xb1, 0xce, 0x8b, 0xb1, 0x84, 0xa3, 0x37, + 0x00, 0xc8, 0x83, 0x98, 0x84, 0xbe, 0xe3, 0x29, 0xeb, 0x06, 0x65, 0xcf, 0x57, 0x0d, 0x36, 0x82, + 0xf8, 0x6e, 0x44, 0xbe, 0x6f, 0x55, 0xa1, 0x60, 0x0d, 0x1d, 0x5d, 0x07, 0x68, 0x87, 0xc1, 0x81, + 0xdb, 0x62, 0x7e, 0x60, 0x45, 0xf3, 0xed, 0xbf, 0xae, 0x20, 0x58, 0xc3, 0x42, 0x6f, 0xc0, 0x44, + 0xc7, 0x8f, 0xf8, 0x75, 0xeb, 0x6c, 0x89, 0xc0, 0x6f, 0xa3, 0xc9, 0x33, 0xfc, 0x5d, 0x1d, 0x88, + 0x4d, 0x5c, 0xb4, 0x04, 0xc3, 0xb1, 0xc3, 0x1e, 0xef, 0x87, 0xf2, 0x8d, 0xf0, 0x36, 0x29, 0x86, + 0x1e, 0xaf, 0x93, 0x56, 0xc0, 0xa2, 0x22, 0xfa, 0x92, 0x34, 0xaa, 0xe7, 0xc7, 0xb7, 0xb0, 0x7e, + 0xcd, 0xdd, 0x23, 0xc6, 0x51, 0xaf, 0x99, 0xd4, 0x0b, 0xab, 0x5a, 0x83, 0x96, 0xfd, 0x43, 0x65, + 0x80, 0x84, 0xb7, 0x44, 0x1f, 0x74, 0x9d, 0x47, 0x2f, 0xf4, 0xe6, 0x46, 0xcf, 0xee, 0x30, 0x42, + 0x3f, 0x6c, 0xc1, 0x98, 0xe3, 0x79, 0x41, 0xd3, 0x89, 0xd9, 0x28, 0x17, 0x7a, 0x9f, 0x87, 0xa2, + 0xfd, 0xa5, 0xa4, 0x06, 0xef, 0xc2, 0xcb, 0x72, 0xe1, 0x69, 0x90, 0xbe, 0xbd, 0xd0, 0x1b, 0x46, + 0xdf, 0x21, 0x45, 0x0e, 0xbe, 0x3c, 0xe6, 0xd3, 0x22, 0x47, 0x99, 0x1d, 0xfd, 0x9a, 0xb4, 0x81, + 0xee, 0x1a, 0x11, 0xd2, 0x4a, 0xf9, 0xc1, 0x02, 0x0c, 0x16, 0xab, 0x5f, 0x70, 0x34, 0x54, 0xd7, + 0xbd, 0x80, 0x86, 0xf2, 0x23, 0x6a, 0x68, 0xbc, 0x7c, 0x1f, 0x0f, 0xa0, 0xf7, 0x60, 0xaa, 0x65, + 0xde, 0xed, 0x62, 0x35, 0x3d, 0x9b, 0x47, 0x37, 0xc5, 0x0a, 0x24, 0xb7, 0x79, 0x0a, 0x80, 0xd3, + 0x84, 0x51, 0x9d, 0xfb, 0x63, 0xd5, 0xfc, 0xed, 0x40, 0x58, 0x51, 0xdb, 0xb9, 0x73, 0x79, 0x18, + 0xc5, 0x64, 0x9f, 0x62, 0x26, 0x97, 0xf6, 0x86, 0xa8, 0x8b, 0x15, 0x15, 0xf4, 0x16, 0x0c, 0x33, + 0x87, 0xce, 0x68, 0x6e, 0x34, 0x5f, 0xeb, 0x65, 0xc6, 0x22, 0x48, 0x36, 0x15, 0xfb, 0x1b, 0x61, + 0x41, 0x01, 0xdd, 0x94, 0x01, 0x4b, 0xa2, 0x9a, 0x7f, 0x37, 0x22, 0x2c, 0x60, 0x49, 0x79, 0xf9, + 0x93, 0x49, 0x2c, 0x12, 0x5e, 0x9e, 0x19, 0x99, 0xdb, 0xa8, 0x49, 0x99, 0x23, 0xf1, 0x5f, 0x06, + 0xfc, 0x9e, 0x83, 0xfc, 0xee, 0x99, 0x41, 0xc1, 0x93, 0xe1, 0xbc, 0x67, 0x92, 0xc0, 0x69, 0x9a, + 0xe7, 0x7a, 0x57, 0xcf, 0xfb, 0x30, 0x9d, 0xde, 0x58, 0x8f, 0x94, 0x37, 0xf8, 0xa3, 0x12, 0x4c, + 0x9a, 0x0b, 0x01, 0x2d, 0x42, 0x59, 0x10, 0x51, 0xc1, 0x29, 0xd5, 0xda, 0x5e, 0x97, 0x00, 0x9c, + 0xe0, 0xb0, 0xe0, 0x9c, 0xac, 0xba, 0x66, 0xf5, 0x96, 0x04, 0xe7, 0x54, 0x10, 0xac, 0x61, 0x51, + 0x96, 0x7d, 0x2b, 0x08, 0x62, 0x75, 0x15, 0xa8, 0xd5, 0xb2, 0xcc, 0x4a, 0xb1, 0x80, 0xd2, 0x2b, + 0x60, 0x8f, 0x84, 0x3e, 0xf1, 0xcc, 0x50, 0x5a, 0xea, 0x0a, 0xb8, 0xa5, 0x03, 0xb1, 0x89, 0x4b, + 0xef, 0xb6, 0x20, 0x62, 0xcb, 0x4f, 0x08, 0x06, 0x89, 0x15, 0x61, 0x83, 0x3b, 0x34, 0x4b, 0x38, + 0xfa, 0x22, 0x3c, 0xae, 0xfc, 0x8f, 0x31, 0xd7, 0x83, 0xca, 0x16, 0x87, 0x0d, 0x39, 0xfe, 0xf1, + 0x95, 0x6c, 0x34, 0x9c, 0x57, 0x1f, 0xbd, 0x09, 0x93, 0x82, 0xdf, 0x96, 0x14, 0x47, 0xcc, 0xa7, + 0xf9, 0x5b, 0x06, 0x14, 0xa7, 0xb0, 0x65, 0x30, 0x30, 0xc6, 0xf2, 0x4a, 0x0a, 0xa3, 0xdd, 0xc1, + 0xc0, 0x74, 0x38, 0xee, 0xaa, 0x81, 0x96, 0x60, 0x8a, 0x33, 0x44, 0x54, 0x82, 0x65, 0xf3, 0x20, + 0x9c, 0x1b, 0xd4, 0x46, 0xb8, 0x63, 0x82, 0x71, 0x1a, 0x1f, 0xbd, 0x0e, 0xe3, 0x4e, 0xd8, 0xdc, + 0x75, 0x63, 0xd2, 0x8c, 0x3b, 0x21, 0xf7, 0x7a, 0xd0, 0x6c, 0x1b, 0x96, 0x34, 0x18, 0x36, 0x30, + 0xed, 0x0f, 0xe0, 0x42, 0x86, 0x5f, 0x14, 0x5d, 0x38, 0x4e, 0xdb, 0x95, 0xdf, 0x94, 0xb2, 0x07, + 0x5c, 0xaa, 0xd7, 0xe4, 0xd7, 0x68, 0x58, 0x74, 0x75, 0x32, 0xff, 0x29, 0x2d, 0x2a, 0xbf, 0x5a, + 0x9d, 0x6b, 0x12, 0x80, 0x13, 0x1c, 0xfb, 0x7f, 0x15, 0x60, 0x2a, 0x43, 0xb7, 0xcb, 0x22, 0xc3, + 0xa7, 0x24, 0x86, 0x24, 0x10, 0xbc, 0x19, 0x5b, 0xae, 0x70, 0x8a, 0xd8, 0x72, 0xc5, 0x7e, 0xb1, + 0xe5, 0x4a, 0x1f, 0x26, 0xb6, 0x9c, 0x39, 0x62, 0x43, 0x03, 0x8d, 0x58, 0x46, 0x3c, 0xba, 0xe1, + 0x53, 0xc6, 0xa3, 0x33, 0x06, 0x7d, 0x64, 0x80, 0x41, 0xff, 0x89, 0x02, 0x4c, 0xa7, 0x6d, 0xb0, + 0xce, 0x41, 0x23, 0xf8, 0x96, 0xa1, 0x11, 0xcc, 0xce, 0xb3, 0x90, 0xb6, 0x0c, 0xcb, 0xd3, 0x0e, + 0xe2, 0x94, 0x76, 0xf0, 0xd3, 0x03, 0x51, 0xeb, 0xad, 0x29, 0xfc, 0x07, 0x05, 0xb8, 0x98, 0xae, + 0xb2, 0xe2, 0x39, 0xee, 0xfe, 0x39, 0x8c, 0xcd, 0x1d, 0x63, 0x6c, 0x5e, 0x1c, 0xe4, 0x6b, 0x58, + 0xd7, 0x72, 0x07, 0xe8, 0x9d, 0xd4, 0x00, 0x2d, 0x0e, 0x4e, 0xb2, 0xf7, 0x28, 0x7d, 0xab, 0x08, + 0x97, 0x33, 0xeb, 0x25, 0x0a, 0xb5, 0x35, 0x43, 0xa1, 0x76, 0x3d, 0xa5, 0x50, 0xb3, 0x7b, 0xd7, + 0x3e, 0x1b, 0x0d, 0x9b, 0x70, 0x58, 0x63, 0xf1, 0xc7, 0x1e, 0x52, 0xbb, 0x66, 0x38, 0xac, 0x29, + 0x42, 0xd8, 0xa4, 0xfb, 0x57, 0x49, 0xab, 0xf6, 0xef, 0x2d, 0xb8, 0x94, 0x39, 0x37, 0xe7, 0xa0, + 0x78, 0xda, 0x30, 0x15, 0x4f, 0xcf, 0x0d, 0xbc, 0x5a, 0x73, 0x34, 0x51, 0x7f, 0x5c, 0xcc, 0xf9, + 0x16, 0x26, 0x56, 0xdf, 0x81, 0x31, 0xa7, 0xd9, 0x24, 0x51, 0xb4, 0x1e, 0xb4, 0x54, 0x3c, 0xae, + 0x17, 0x99, 0x74, 0x94, 0x14, 0x9f, 0x1c, 0x55, 0xe6, 0xd3, 0x24, 0x12, 0x30, 0xd6, 0x29, 0x98, + 0x21, 0x04, 0x0b, 0x67, 0x1a, 0x42, 0xf0, 0x3a, 0xc0, 0x81, 0xe2, 0xb1, 0xd3, 0xa2, 0xb9, 0xc6, + 0x7d, 0x6b, 0x58, 0xe8, 0x7b, 0x61, 0x34, 0x12, 0xd7, 0xb8, 0x58, 0x8a, 0x2f, 0x0f, 0x38, 0x57, + 0xce, 0x16, 0xf1, 0x4c, 0xcf, 0x68, 0xa5, 0xc5, 0x50, 0x24, 0xd1, 0x77, 0xc3, 0x74, 0xc4, 0x03, + 0x6f, 0xac, 0x78, 0x4e, 0xc4, 0xcc, 0xec, 0xc5, 0x2a, 0x64, 0xee, 0xce, 0x8d, 0x14, 0x0c, 0x77, + 0x61, 0xa3, 0x35, 0xf9, 0x51, 0x2c, 0x4a, 0x08, 0x5f, 0x98, 0x57, 0x93, 0x0f, 0x12, 0x79, 0x69, + 0x66, 0xd3, 0xc3, 0xcf, 0x06, 0x5e, 0xab, 0x69, 0xff, 0x44, 0x09, 0x9e, 0xe8, 0x71, 0x88, 0xa1, + 0x25, 0xf3, 0x99, 0xec, 0xf9, 0xb4, 0xcc, 0x3a, 0x9f, 0x59, 0xd9, 0x10, 0x62, 0x53, 0x6b, 0xa5, + 0xf0, 0xa1, 0xd7, 0xca, 0x8f, 0x5a, 0x9a, 0x36, 0x81, 0x1b, 0x73, 0x7d, 0xfe, 0x94, 0x87, 0xf3, + 0x19, 0xaa, 0x17, 0xb6, 0x33, 0x64, 0xf4, 0xeb, 0x03, 0x77, 0x67, 0x60, 0xa1, 0xfd, 0x7c, 0x75, + 0xaa, 0x5f, 0xb5, 0xe0, 0xe9, 0xcc, 0xfe, 0x1a, 0xcf, 0xfa, 0x8b, 0x50, 0x6e, 0xd2, 0x42, 0xcd, + 0x75, 0x27, 0xf1, 0x69, 0x94, 0x00, 0x9c, 0xe0, 0x18, 0xaf, 0xf7, 0x85, 0xbe, 0xaf, 0xf7, 0xff, + 0xda, 0x82, 0xae, 0x05, 0x7c, 0x0e, 0x27, 0x69, 0xcd, 0x3c, 0x49, 0x3f, 0x39, 0xc8, 0x5c, 0xe6, + 0x1c, 0xa2, 0xbf, 0x3f, 0x05, 0x8f, 0xe5, 0xd8, 0xea, 0x1f, 0xc0, 0xcc, 0x4e, 0x93, 0x98, 0x4e, + 0x51, 0xe2, 0x63, 0x32, 0xfd, 0xc7, 0x7a, 0x7a, 0x50, 0xb1, 0x84, 0x22, 0x33, 0x5d, 0x28, 0xb8, + 0xbb, 0x09, 0xf4, 0x55, 0x0b, 0x66, 0x9d, 0xfb, 0x51, 0x57, 0xda, 0x38, 0xb1, 0x66, 0x5e, 0xc9, + 0xd4, 0x2d, 0xf4, 0x49, 0x33, 0xc7, 0x33, 0xac, 0x64, 0x61, 0xe1, 0xcc, 0xb6, 0x10, 0x16, 0x21, + 0x14, 0x29, 0xbf, 0xdd, 0xc3, 0x6d, 0x2f, 0xcb, 0xa9, 0x82, 0x9f, 0xa9, 0x12, 0x82, 0x15, 0x1d, + 0x74, 0x0f, 0xca, 0x3b, 0xd2, 0xd3, 0x49, 0x9c, 0xd9, 0x99, 0x97, 0x60, 0xa6, 0x3b, 0x14, 0x7f, + 0xba, 0x52, 0x20, 0x9c, 0x90, 0x42, 0x6f, 0x42, 0xd1, 0xdf, 0x8e, 0x7a, 0xa5, 0x26, 0x49, 0x59, + 0xbb, 0x70, 0x97, 0xd8, 0x8d, 0xb5, 0x06, 0xa6, 0x15, 0xd1, 0x4d, 0x28, 0x86, 0x5b, 0x2d, 0xa1, + 0x0e, 0xcb, 0xe4, 0x4b, 0xf1, 0x72, 0x35, 0x7b, 0x91, 0x70, 0x4a, 0x78, 0xb9, 0x8a, 0x29, 0x09, + 0x54, 0x87, 0x21, 0x66, 0xd6, 0x2e, 0xb4, 0x5e, 0x99, 0x0c, 0x69, 0x0f, 0xf7, 0x10, 0xee, 0x37, + 0xcb, 0x10, 0x30, 0x27, 0x84, 0xde, 0x82, 0xe1, 0x26, 0xcb, 0xde, 0x21, 0xa2, 0xf6, 0x66, 0x07, + 0x54, 0xe9, 0xca, 0xef, 0xc1, 0x9f, 0x18, 0x78, 0x39, 0x16, 0x14, 0xd0, 0x26, 0x0c, 0x37, 0x49, + 0x7b, 0x77, 0x3b, 0x62, 0x82, 0xb7, 0xc9, 0xe0, 0x27, 0xb4, 0x7a, 0x24, 0xab, 0x11, 0x54, 0x19, + 0x06, 0x16, 0xb4, 0xd0, 0x67, 0xa1, 0xb0, 0xdd, 0x14, 0xb6, 0xee, 0x99, 0x7a, 0x2f, 0xd3, 0x97, + 0x79, 0x79, 0xf8, 0xf8, 0xa8, 0x52, 0x58, 0x5b, 0xc1, 0x85, 0xed, 0x26, 0xda, 0x80, 0x91, 0x6d, + 0xee, 0xfd, 0x28, 0x42, 0x0c, 0x3c, 0x9b, 0xed, 0x98, 0xd9, 0xe5, 0x20, 0xc9, 0x6d, 0xb4, 0x05, + 0x00, 0x4b, 0x22, 0x2c, 0xfa, 0xa0, 0xf2, 0xe2, 0x14, 0x61, 0x78, 0x17, 0x4e, 0xe7, 0x79, 0xcb, + 0xfd, 0xaa, 0x13, 0x5f, 0x50, 0xac, 0x51, 0x44, 0x5f, 0x81, 0xb2, 0x23, 0xd3, 0x94, 0x89, 0x30, + 0x05, 0x2f, 0x67, 0x6e, 0xc7, 0xde, 0x19, 0xdc, 0xf8, 0x5a, 0x56, 0x48, 0x38, 0x21, 0x8a, 0xf6, + 0x60, 0xe2, 0x20, 0x6a, 0xef, 0x12, 0xb9, 0x7d, 0x59, 0xd4, 0x82, 0x9c, 0xeb, 0xea, 0x9e, 0x40, + 0x74, 0xc3, 0xb8, 0xe3, 0x78, 0x5d, 0x27, 0x0e, 0x33, 0xed, 0xbf, 0xa7, 0x13, 0xc3, 0x26, 0x6d, + 0x3a, 0xfc, 0xef, 0x77, 0x82, 0xad, 0xc3, 0x98, 0x88, 0xb8, 0xbd, 0x99, 0xc3, 0xff, 0x36, 0x47, + 0xe9, 0x1e, 0x7e, 0x01, 0xc0, 0x92, 0x08, 0xdd, 0xe0, 0x8e, 0x4c, 0x01, 0xc8, 0xe2, 0xf5, 0xe6, + 0x6c, 0xf0, 0xcc, 0x3c, 0x81, 0xda, 0xa0, 0xb0, 0x93, 0x31, 0x21, 0xc5, 0x4e, 0xc4, 0xf6, 0x6e, + 0x10, 0x07, 0x7e, 0xea, 0x34, 0x9e, 0xc9, 0x3f, 0x11, 0xeb, 0x19, 0xf8, 0xdd, 0x27, 0x62, 0x16, + 0x16, 0xce, 0x6c, 0x0b, 0xb5, 0x60, 0xb2, 0x1d, 0x84, 0xf1, 0xfd, 0x20, 0x94, 0xeb, 0x0b, 0xf5, + 0x10, 0xf2, 0x0d, 0x4c, 0xd1, 0x22, 0x8b, 0x23, 0x6d, 0x42, 0x70, 0x8a, 0x26, 0xfa, 0x02, 0x8c, + 0x44, 0x4d, 0xc7, 0x23, 0xb5, 0x3b, 0x73, 0x17, 0xf2, 0xaf, 0x9a, 0x06, 0x47, 0xc9, 0x59, 0x5d, + 0x6c, 0x72, 0x04, 0x0a, 0x96, 0xe4, 0xd0, 0x1a, 0x0c, 0xb1, 0x60, 0xf0, 0x2c, 0xe4, 0x70, 0x4e, + 0x38, 0x9c, 0x2e, 0x6b, 0x42, 0x7e, 0x22, 0xb1, 0x62, 0xcc, 0xab, 0xd3, 0x3d, 0x20, 0x78, 0xdd, + 0x20, 0x9a, 0xbb, 0x98, 0xbf, 0x07, 0x04, 0x8b, 0x7c, 0xa7, 0xd1, 0x6b, 0x0f, 0x28, 0x24, 0x9c, + 0x10, 0xa5, 0xe7, 0x31, 0x3d, 0x43, 0x1f, 0xeb, 0x61, 0xe0, 0x90, 0x7b, 0x82, 0xb2, 0xf3, 0x98, + 0x9e, 0x9f, 0x94, 0x84, 0xfd, 0x07, 0x23, 0xdd, 0xfc, 0x09, 0x93, 0x8e, 0x7e, 0xc8, 0xea, 0x7a, + 0xee, 0xfa, 0xcc, 0xa0, 0xca, 0x9a, 0x33, 0xe4, 0x4c, 0xbf, 0x6a, 0xc1, 0x63, 0xed, 0xcc, 0x0f, + 0x11, 0x97, 0xfd, 0x60, 0x3a, 0x1f, 0xfe, 0xe9, 0x2a, 0x2c, 0x78, 0x36, 0x1c, 0xe7, 0xb4, 0x94, + 0xe6, 0xfe, 0x8b, 0x1f, 0x9a, 0xfb, 0x5f, 0x87, 0x51, 0xc6, 0x50, 0x26, 0xb1, 0x97, 0x06, 0x8a, + 0xe5, 0xc2, 0xd8, 0x86, 0x15, 0x51, 0x11, 0x2b, 0x12, 0xe8, 0xc7, 0x2c, 0x78, 0x2a, 0xdd, 0x75, + 0x4c, 0x18, 0x58, 0x04, 0xd1, 0xe6, 0x82, 0xd9, 0x9a, 0xf8, 0xfe, 0xa7, 0xea, 0xbd, 0x90, 0x4f, + 0xfa, 0x21, 0xe0, 0xde, 0x8d, 0xa1, 0x6a, 0x86, 0x64, 0x38, 0x6c, 0x6a, 0xc3, 0x07, 0x90, 0x0e, + 0x5f, 0x81, 0xf1, 0xfd, 0xa0, 0xe3, 0xc7, 0xc2, 0x84, 0x44, 0x78, 0xa7, 0xb1, 0x37, 0xdb, 0x75, + 0xad, 0x1c, 0x1b, 0x58, 0x29, 0x99, 0x72, 0xf4, 0x61, 0x65, 0x4a, 0xf4, 0x6e, 0x2a, 0x65, 0x6f, + 0x39, 0x9f, 0xb7, 0x10, 0xe2, 0xf7, 0x29, 0x12, 0xf7, 0x9e, 0xaf, 0x1c, 0xf4, 0x75, 0x2b, 0x83, + 0x81, 0xe7, 0x92, 0xf1, 0xe7, 0x4c, 0xc9, 0xf8, 0x6a, 0x5a, 0x32, 0xee, 0xd2, 0x84, 0x1a, 0x42, + 0xf1, 0xe0, 0x11, 0x7f, 0x07, 0x0d, 0xa1, 0x65, 0x7b, 0x70, 0xa5, 0xdf, 0xb5, 0xc4, 0x6c, 0x89, + 0x5a, 0xea, 0xdd, 0x2b, 0xb1, 0x25, 0x6a, 0xd5, 0xaa, 0x98, 0x41, 0x06, 0x8d, 0xb1, 0x60, 0xff, + 0x77, 0x0b, 0x8a, 0xf5, 0xa0, 0x75, 0x0e, 0x9a, 0xdd, 0xcf, 0x1b, 0x9a, 0xdd, 0x27, 0x72, 0x52, + 0x29, 0xe7, 0xea, 0x71, 0x57, 0x53, 0x7a, 0xdc, 0xa7, 0xf2, 0x08, 0xf4, 0xd6, 0xda, 0xfe, 0x5c, + 0x11, 0xf4, 0xc4, 0xcf, 0xe8, 0xdf, 0x3e, 0x8c, 0x55, 0x6a, 0xb1, 0x57, 0x2e, 0x68, 0x41, 0x99, + 0x99, 0x20, 0x49, 0x87, 0xab, 0xbf, 0x64, 0xc6, 0xa9, 0xef, 0x10, 0x77, 0x67, 0x37, 0x26, 0xad, + 0xf4, 0xe7, 0x9c, 0x9f, 0x71, 0xea, 0x7f, 0xb5, 0x60, 0x2a, 0xd5, 0x3a, 0xf2, 0x60, 0xc2, 0xd3, + 0xd5, 0x72, 0x62, 0x9d, 0x3e, 0x94, 0x46, 0x8f, 0xa7, 0x00, 0xd1, 0x8b, 0xb0, 0x49, 0x1c, 0x2d, + 0x00, 0xa8, 0x67, 0x33, 0xa9, 0xed, 0x62, 0x5c, 0xbf, 0x7a, 0x57, 0x8b, 0xb0, 0x86, 0x81, 0x5e, + 0x85, 0xb1, 0x38, 0x68, 0x07, 0x5e, 0xb0, 0x73, 0x78, 0x8b, 0xc8, 0xa8, 0x1e, 0xca, 0xca, 0x69, + 0x33, 0x01, 0x61, 0x1d, 0xcf, 0xfe, 0x85, 0x22, 0xa4, 0x93, 0x85, 0x7f, 0x7b, 0x4d, 0x7e, 0x3c, + 0xd7, 0xe4, 0xb7, 0x2c, 0x98, 0xa6, 0xad, 0x33, 0x8b, 0x0b, 0x79, 0xd9, 0xaa, 0xcc, 0x23, 0x56, + 0x8f, 0xcc, 0x23, 0x57, 0xe9, 0xd9, 0xd5, 0x0a, 0x3a, 0xb1, 0xd0, 0x96, 0x69, 0x87, 0x13, 0x2d, + 0xc5, 0x02, 0x2a, 0xf0, 0x48, 0x18, 0x0a, 0x9f, 0x18, 0x1d, 0x8f, 0x84, 0x21, 0x16, 0x50, 0x99, + 0x98, 0xa4, 0x94, 0x93, 0x98, 0x84, 0xc5, 0x28, 0x13, 0xaf, 0xfc, 0x82, 0xed, 0xd1, 0x62, 0x94, + 0xc9, 0xe7, 0xff, 0x04, 0xc7, 0xfe, 0xe5, 0x22, 0x8c, 0xd7, 0x83, 0x56, 0xf2, 0x70, 0xf5, 0x8a, + 0xf1, 0x70, 0x75, 0x25, 0xf5, 0x70, 0x35, 0xad, 0xe3, 0x7e, 0xfb, 0x99, 0xea, 0xa3, 0x7a, 0xa6, + 0xfa, 0x57, 0x16, 0x9b, 0xb5, 0xea, 0x46, 0x43, 0x24, 0x46, 0x7d, 0x09, 0xc6, 0xd8, 0x81, 0xc4, + 0x9c, 0xb0, 0xe4, 0x6b, 0x0e, 0x8b, 0x39, 0xbe, 0x91, 0x14, 0x63, 0x1d, 0x07, 0x5d, 0x83, 0xd1, + 0x88, 0x38, 0x61, 0x73, 0x57, 0x9d, 0x71, 0xe2, 0xad, 0x83, 0x97, 0x61, 0x05, 0x45, 0x6f, 0x27, + 0xe1, 0xb1, 0x8a, 0xf9, 0x29, 0x3e, 0xf5, 0xfe, 0xf0, 0x2d, 0x92, 0x1f, 0x13, 0xcb, 0x7e, 0x07, + 0x50, 0x37, 0xfe, 0x00, 0x81, 0x70, 0x2a, 0x66, 0x20, 0x9c, 0x72, 0x57, 0x10, 0x9c, 0x3f, 0xb7, + 0x60, 0xb2, 0x1e, 0xb4, 0xe8, 0xd6, 0xfd, 0xab, 0xb4, 0x4f, 0xf5, 0xd8, 0x80, 0xc3, 0x3d, 0x62, + 0x03, 0xfe, 0x43, 0x0b, 0x46, 0xea, 0x41, 0xeb, 0x1c, 0x74, 0xec, 0x9f, 0x33, 0x75, 0xec, 0x8f, + 0xe7, 0x2c, 0x89, 0x1c, 0xb5, 0xfa, 0xaf, 0x16, 0x61, 0x82, 0xf6, 0x33, 0xd8, 0x91, 0xb3, 0x64, + 0x8c, 0x88, 0x35, 0xc0, 0x88, 0x50, 0x36, 0x37, 0xf0, 0xbc, 0xe0, 0x7e, 0x7a, 0xc6, 0xd6, 0x58, + 0x29, 0x16, 0x50, 0xf4, 0x02, 0x8c, 0xb6, 0x43, 0x72, 0xe0, 0x06, 0x82, 0x7f, 0xd4, 0x5e, 0x2c, + 0xea, 0xa2, 0x1c, 0x2b, 0x0c, 0x2a, 0x77, 0x45, 0xae, 0xdf, 0x24, 0x32, 0xbf, 0x70, 0x89, 0xa5, + 0x20, 0xe2, 0x41, 0x7f, 0xb5, 0x72, 0x6c, 0x60, 0xa1, 0x77, 0xa0, 0xcc, 0xfe, 0xb3, 0x13, 0xe5, + 0xf4, 0x29, 0x53, 0x44, 0xa4, 0x7d, 0x41, 0x00, 0x27, 0xb4, 0xd0, 0x75, 0x80, 0x58, 0x06, 0x86, + 0x8d, 0x44, 0x3c, 0x13, 0xc5, 0x6b, 0xab, 0x90, 0xb1, 0x11, 0xd6, 0xb0, 0xd0, 0xf3, 0x50, 0x8e, + 0x1d, 0xd7, 0xbb, 0xed, 0xfa, 0x24, 0x62, 0x8a, 0xe6, 0xa2, 0x0c, 0xa4, 0x2f, 0x0a, 0x71, 0x02, + 0xa7, 0xbc, 0x0e, 0x73, 0xf6, 0xe5, 0x09, 0x97, 0x46, 0x19, 0x36, 0xe3, 0x75, 0x6e, 0xab, 0x52, + 0xac, 0x61, 0xd8, 0xaf, 0xc3, 0xc5, 0x7a, 0xd0, 0xaa, 0x07, 0x61, 0xbc, 0x16, 0x84, 0xf7, 0x9d, + 0xb0, 0x25, 0xe7, 0xaf, 0x22, 0x63, 0xba, 0xd3, 0xb3, 0x67, 0x88, 0xef, 0x4c, 0x23, 0x5a, 0xfb, + 0xcb, 0x8c, 0xdb, 0x39, 0xa5, 0x5f, 0xc4, 0xff, 0x2e, 0xb0, 0x83, 0x22, 0x95, 0x05, 0x0c, 0x7d, + 0x19, 0x26, 0x23, 0x72, 0xdb, 0xf5, 0x3b, 0x0f, 0xa4, 0x7c, 0xdc, 0xc3, 0xe9, 0xa4, 0xb1, 0xaa, + 0x63, 0x72, 0x2d, 0x9b, 0x59, 0x86, 0x53, 0xd4, 0xe8, 0x10, 0x86, 0x1d, 0x7f, 0x29, 0xba, 0x1b, + 0x91, 0x50, 0x64, 0xa1, 0x62, 0x43, 0x88, 0x65, 0x21, 0x4e, 0xe0, 0x74, 0xc9, 0xb0, 0x3f, 0x1b, + 0x81, 0x8f, 0x83, 0x20, 0x96, 0x8b, 0x8c, 0xe5, 0x31, 0xd1, 0xca, 0xb1, 0x81, 0x85, 0xd6, 0x00, + 0x45, 0x9d, 0x76, 0xdb, 0x63, 0x0f, 0xe0, 0x8e, 0x77, 0x23, 0x0c, 0x3a, 0x6d, 0xfe, 0xf8, 0x58, + 0xe4, 0xd9, 0xf5, 0x1b, 0x5d, 0x50, 0x9c, 0x51, 0x83, 0x1e, 0x0c, 0xdb, 0x11, 0xfb, 0xcd, 0x16, + 0x5e, 0x51, 0x68, 0xbe, 0x1b, 0xac, 0x08, 0x4b, 0x18, 0x9d, 0x67, 0xd6, 0x3c, 0xc7, 0x1c, 0x4e, + 0xe6, 0x19, 0xab, 0x52, 0xac, 0x61, 0xd8, 0x3f, 0xc0, 0x2e, 0x18, 0x96, 0x6c, 0x28, 0xee, 0x84, + 0x04, 0xed, 0xc3, 0x44, 0x9b, 0x5d, 0xfd, 0x22, 0x14, 0xae, 0x18, 0xf0, 0x57, 0x06, 0x94, 0x14, + 0xef, 0xd3, 0xcd, 0xab, 0x34, 0x39, 0x8c, 0x05, 0xaf, 0xeb, 0xe4, 0xb0, 0x49, 0xdd, 0xfe, 0xb9, + 0x69, 0x76, 0x8e, 0x35, 0xb8, 0xf8, 0x37, 0x22, 0x2c, 0x5e, 0x05, 0xaf, 0x3b, 0x9f, 0xaf, 0x87, + 0x48, 0xae, 0x1c, 0x61, 0x35, 0x8b, 0x65, 0x5d, 0xf4, 0x36, 0x7b, 0xe5, 0xe5, 0x87, 0x47, 0xbf, + 0x9c, 0xaf, 0x1c, 0xcb, 0x78, 0xd0, 0x15, 0x15, 0xb1, 0x46, 0x04, 0xdd, 0x86, 0x09, 0x91, 0x9b, + 0x46, 0x28, 0x9a, 0x8a, 0x86, 0x22, 0x61, 0x02, 0xeb, 0xc0, 0x93, 0x74, 0x01, 0x36, 0x2b, 0xa3, + 0x1d, 0x78, 0x4a, 0x4b, 0xd4, 0x76, 0x23, 0x74, 0xd8, 0xcb, 0x9f, 0xcb, 0x56, 0xbf, 0x76, 0x16, + 0x3d, 0x7d, 0x7c, 0x54, 0x79, 0x6a, 0xb3, 0x17, 0x22, 0xee, 0x4d, 0x07, 0xdd, 0x81, 0x8b, 0x4e, + 0x33, 0x76, 0x0f, 0x48, 0x95, 0x38, 0x2d, 0xcf, 0xf5, 0xd5, 0x61, 0xc7, 0x17, 0xd0, 0xa5, 0xe3, + 0xa3, 0xca, 0xc5, 0xa5, 0x2c, 0x04, 0x9c, 0x5d, 0x0f, 0x7d, 0x0e, 0xca, 0x2d, 0x3f, 0x12, 0x63, + 0x30, 0x6c, 0xe4, 0x20, 0x2c, 0x57, 0x37, 0x1a, 0xea, 0xfb, 0x93, 0x3f, 0x38, 0xa9, 0x80, 0x76, + 0xb8, 0xb2, 0x49, 0xc9, 0x76, 0x23, 0xf9, 0xf9, 0xa6, 0xc5, 0x92, 0x30, 0x5c, 0x4b, 0xb8, 0x96, + 0x55, 0x19, 0x79, 0x1a, 0x5e, 0x27, 0x06, 0x61, 0xf4, 0x16, 0x20, 0xca, 0xfc, 0xb8, 0x4d, 0xb2, + 0xd4, 0x64, 0x11, 0x89, 0x99, 0x6e, 0x6e, 0xd4, 0x30, 0xe5, 0x47, 0x8d, 0x2e, 0x0c, 0x9c, 0x51, + 0x0b, 0xdd, 0xa4, 0x27, 0x90, 0x5e, 0x2a, 0x8c, 0x55, 0x25, 0xc3, 0x3c, 0x57, 0x25, 0xed, 0x90, + 0x34, 0x9d, 0x98, 0xb4, 0x4c, 0x8a, 0x38, 0x55, 0x8f, 0xde, 0x4f, 0x2a, 0x39, 0x09, 0x98, 0x61, + 0x07, 0xba, 0x13, 0x94, 0x50, 0x59, 0x73, 0x37, 0x88, 0xe2, 0x0d, 0x12, 0xdf, 0x0f, 0xc2, 0x3d, + 0x11, 0xe5, 0x29, 0x09, 0x38, 0x98, 0x80, 0xb0, 0x8e, 0x47, 0x79, 0x4b, 0xf6, 0xcc, 0x5a, 0xab, + 0xb2, 0x57, 0xaf, 0xd1, 0x64, 0x9f, 0xdc, 0xe4, 0xc5, 0x58, 0xc2, 0x25, 0x6a, 0xad, 0xbe, 0xc2, + 0x5e, 0xb0, 0x52, 0xa8, 0xb5, 0xfa, 0x0a, 0x96, 0x70, 0x44, 0xba, 0xf3, 0x3b, 0x4e, 0xe6, 0x6b, + 0x0a, 0xbb, 0xcf, 0xf1, 0x01, 0x53, 0x3c, 0xfa, 0x30, 0xad, 0x32, 0x4b, 0xf2, 0xf0, 0x57, 0xd1, + 0xdc, 0x14, 0x5b, 0x24, 0x83, 0xc7, 0xce, 0x52, 0xba, 0xd7, 0x5a, 0x8a, 0x12, 0xee, 0xa2, 0x6d, + 0x04, 0x82, 0x98, 0xee, 0x9b, 0x5c, 0x66, 0x11, 0xca, 0x51, 0x67, 0xab, 0x15, 0xec, 0x3b, 0xae, + 0xcf, 0x1e, 0x9c, 0x34, 0xc6, 0xa5, 0x21, 0x01, 0x38, 0xc1, 0x41, 0x6b, 0x30, 0xea, 0x48, 0xc5, + 0x2a, 0xca, 0xf7, 0x0c, 0x57, 0xea, 0x54, 0xc6, 0x96, 0x2b, 0x55, 0xaa, 0xaa, 0x8b, 0xde, 0x80, + 0x09, 0xe1, 0x4e, 0xc4, 0xfd, 0xe5, 0xd9, 0x83, 0x90, 0x66, 0x79, 0xde, 0xd0, 0x81, 0xd8, 0xc4, + 0x45, 0xdf, 0x0b, 0x93, 0x94, 0x4a, 0x72, 0xb0, 0xcd, 0xcd, 0x0e, 0x72, 0x22, 0x6a, 0x49, 0x03, + 0xf4, 0xca, 0x38, 0x45, 0x0c, 0xb5, 0xe0, 0x49, 0xa7, 0x13, 0x07, 0x4c, 0x39, 0x6d, 0xae, 0xff, + 0xcd, 0x60, 0x8f, 0xf8, 0xec, 0x5d, 0x68, 0x74, 0xf9, 0xca, 0xf1, 0x51, 0xe5, 0xc9, 0xa5, 0x1e, + 0x78, 0xb8, 0x27, 0x15, 0x74, 0x17, 0xc6, 0xe2, 0xc0, 0x63, 0x36, 0xe0, 0x94, 0x07, 0x78, 0x2c, + 0x3f, 0x90, 0xca, 0xa6, 0x42, 0xd3, 0x15, 0x33, 0xaa, 0x2a, 0xd6, 0xe9, 0xa0, 0x4d, 0xbe, 0xc7, + 0x58, 0x88, 0x49, 0x12, 0xcd, 0x3d, 0x9e, 0x3f, 0x30, 0x2a, 0x12, 0xa5, 0xb9, 0x05, 0x45, 0x4d, + 0xac, 0x93, 0x41, 0x37, 0x60, 0xa6, 0x1d, 0xba, 0x01, 0x5b, 0xd8, 0xea, 0x61, 0x60, 0xce, 0x8c, + 0x13, 0x5f, 0x4f, 0x23, 0xe0, 0xee, 0x3a, 0x54, 0x70, 0x93, 0x85, 0x73, 0x97, 0x78, 0xd2, 0x21, + 0xce, 0xcc, 0xf2, 0x32, 0xac, 0xa0, 0x68, 0x9d, 0x9d, 0xcb, 0x5c, 0xc4, 0x9a, 0x9b, 0xcf, 0xf7, + 0xa8, 0xd7, 0x45, 0x31, 0xce, 0xe8, 0xa8, 0xbf, 0x38, 0xa1, 0x40, 0xef, 0x8d, 0x68, 0xd7, 0x09, + 0x49, 0x3d, 0x0c, 0x9a, 0x84, 0x77, 0x86, 0x9b, 0x9f, 0x3f, 0xc1, 0x23, 0xe1, 0xd1, 0x7b, 0xa3, + 0x91, 0x85, 0x80, 0xb3, 0xeb, 0xcd, 0x7f, 0x17, 0xcc, 0x74, 0x9d, 0xe4, 0xa7, 0x0a, 0x8f, 0xfc, + 0x67, 0x43, 0x50, 0x56, 0x8a, 0x5f, 0xb4, 0x68, 0xea, 0xf3, 0x2f, 0xa5, 0xf5, 0xf9, 0xa3, 0x94, + 0xff, 0xd4, 0x55, 0xf8, 0x9b, 0x86, 0xe1, 0x57, 0x21, 0x3f, 0x19, 0x91, 0xae, 0xf5, 0xe8, 0xeb, + 0x9b, 0xa5, 0xc9, 0xf1, 0xc5, 0x81, 0x1f, 0x06, 0x4a, 0x3d, 0x55, 0x03, 0x03, 0xe6, 0x02, 0xa5, + 0xa2, 0x6e, 0x3b, 0x68, 0xd5, 0xea, 0xe9, 0xe4, 0x78, 0x75, 0x5a, 0x88, 0x39, 0x8c, 0x09, 0x2b, + 0x94, 0xed, 0x60, 0xc2, 0xca, 0xc8, 0x43, 0x0a, 0x2b, 0x92, 0x00, 0x4e, 0x68, 0x21, 0x0f, 0x66, + 0x9a, 0x66, 0x5e, 0x43, 0xe5, 0x8f, 0xf5, 0x4c, 0xdf, 0x0c, 0x83, 0x1d, 0x2d, 0x89, 0xd4, 0x4a, + 0x9a, 0x0a, 0xee, 0x26, 0x8c, 0xde, 0x80, 0xd1, 0xf7, 0x83, 0x88, 0x6d, 0x0b, 0x71, 0xf7, 0x4a, + 0x0f, 0x98, 0xd1, 0xb7, 0xef, 0x34, 0x58, 0xf9, 0xc9, 0x51, 0x65, 0xac, 0x1e, 0xb4, 0xe4, 0x5f, + 0xac, 0x2a, 0xa0, 0x07, 0x70, 0xd1, 0x38, 0xb1, 0x54, 0x77, 0x61, 0xf0, 0xee, 0x3e, 0x25, 0x9a, + 0xbb, 0x58, 0xcb, 0xa2, 0x84, 0xb3, 0x1b, 0xa0, 0xc7, 0x80, 0x1f, 0x88, 0x9c, 0xa0, 0xf2, 0x7e, + 0x67, 0xd7, 0x78, 0x59, 0xf7, 0x5a, 0x4e, 0x21, 0xe0, 0xee, 0x3a, 0xf6, 0x37, 0xb8, 0x9e, 0x5c, + 0x68, 0xd3, 0x48, 0xd4, 0xf1, 0xce, 0x23, 0xe5, 0xcc, 0xaa, 0xa1, 0xe8, 0x7b, 0xe8, 0xb7, 0x98, + 0xdf, 0xb2, 0xd8, 0x5b, 0xcc, 0x26, 0xd9, 0x6f, 0x7b, 0x4e, 0x7c, 0x1e, 0x9e, 0x17, 0x6f, 0xc3, + 0x68, 0x2c, 0x5a, 0xeb, 0x95, 0x25, 0x47, 0xeb, 0x14, 0x7b, 0x8f, 0x52, 0x37, 0xbf, 0x2c, 0xc5, + 0x8a, 0x8c, 0xfd, 0xcf, 0xf9, 0x0c, 0x48, 0xc8, 0x39, 0x28, 0x5d, 0xaa, 0xa6, 0xd2, 0xa5, 0xd2, + 0xe7, 0x0b, 0x72, 0x94, 0x2f, 0xff, 0xcc, 0xec, 0x37, 0x13, 0xb2, 0x3e, 0xee, 0x8f, 0x80, 0xf6, + 0x4f, 0x59, 0x30, 0x9b, 0x65, 0x35, 0x43, 0xb9, 0x35, 0x2e, 0xe2, 0xa9, 0x47, 0x51, 0x35, 0x82, + 0xf7, 0x44, 0x39, 0x56, 0x18, 0x03, 0x07, 0xa0, 0x3f, 0x5d, 0x94, 0xaa, 0x3b, 0x30, 0x51, 0x0f, + 0x89, 0x76, 0x07, 0xbc, 0xc9, 0x5d, 0xa9, 0x78, 0x7f, 0x5e, 0x38, 0xb5, 0x1b, 0x95, 0xfd, 0x8b, + 0x05, 0x98, 0xe5, 0xaf, 0x1a, 0x4b, 0x07, 0x81, 0xdb, 0xaa, 0x07, 0x2d, 0x91, 0x3c, 0xe0, 0x4b, + 0x30, 0xde, 0xd6, 0xe4, 0xf2, 0x5e, 0x71, 0x72, 0x74, 0xf9, 0x3d, 0x91, 0x8f, 0xf4, 0x52, 0x6c, + 0xd0, 0x42, 0x2d, 0x18, 0x27, 0x07, 0x6e, 0x53, 0xa9, 0xc6, 0x0b, 0xa7, 0xbe, 0x1b, 0x54, 0x2b, + 0xab, 0x1a, 0x1d, 0x6c, 0x50, 0x7d, 0x04, 0xf9, 0xa4, 0xec, 0x9f, 0xb6, 0xe0, 0xf1, 0x9c, 0xa8, + 0x3a, 0xb4, 0xb9, 0xfb, 0xec, 0xfd, 0x48, 0xa4, 0xa6, 0x51, 0xcd, 0xf1, 0x57, 0x25, 0x2c, 0xa0, + 0xe8, 0x0b, 0x00, 0xfc, 0x55, 0x88, 0x8a, 0x0b, 0xe2, 0xd3, 0x07, 0x0b, 0x36, 0xa1, 0x05, 0x09, + 0x90, 0xf5, 0xb1, 0x46, 0xcb, 0xfe, 0xf9, 0x22, 0x0c, 0xb1, 0x57, 0x08, 0xb4, 0x06, 0x23, 0xbb, + 0x3c, 0x86, 0xec, 0x20, 0xe1, 0x6a, 0x13, 0xb9, 0x8b, 0x17, 0x60, 0x59, 0x19, 0xad, 0xc3, 0x05, + 0x1e, 0x83, 0xd7, 0xab, 0x12, 0xcf, 0x39, 0x94, 0xe2, 0x3b, 0x4f, 0xe7, 0xa2, 0x02, 0x1e, 0xd4, + 0xba, 0x51, 0x70, 0x56, 0x3d, 0xf4, 0x26, 0x4c, 0xc6, 0xee, 0x3e, 0x09, 0x3a, 0xb1, 0xa4, 0xc4, + 0xa3, 0xef, 0x2a, 0x66, 0x7f, 0xd3, 0x80, 0xe2, 0x14, 0x36, 0x15, 0x44, 0xda, 0x5d, 0x8a, 0x0a, + 0x2d, 0x41, 0xba, 0xa9, 0x9c, 0x30, 0x71, 0x99, 0xb9, 0x4c, 0x87, 0x19, 0x07, 0x6d, 0xee, 0x86, + 0x24, 0xda, 0x0d, 0xbc, 0x96, 0xc8, 0x06, 0x9c, 0x98, 0xcb, 0xa4, 0xe0, 0xb8, 0xab, 0x06, 0xa5, + 0xb2, 0xed, 0xb8, 0x5e, 0x27, 0x24, 0x09, 0x95, 0x61, 0x93, 0xca, 0x5a, 0x0a, 0x8e, 0xbb, 0x6a, + 0xd0, 0x75, 0x74, 0x51, 0xa4, 0xe7, 0x95, 0x6e, 0xd8, 0xca, 0x06, 0x6a, 0x44, 0xba, 0xb6, 0xf4, + 0x88, 0x43, 0x22, 0xac, 0x44, 0x54, 0x82, 0x5f, 0x2d, 0xf9, 0xa3, 0x70, 0x6a, 0x91, 0x54, 0x1e, + 0x26, 0x49, 0xec, 0x1f, 0x58, 0x70, 0x21, 0xc3, 0xd6, 0x92, 0x1f, 0x55, 0x3b, 0x6e, 0x14, 0xab, + 0x94, 0x15, 0xda, 0x51, 0xc5, 0xcb, 0xb1, 0xc2, 0xa0, 0xfb, 0x81, 0x1f, 0x86, 0xe9, 0x03, 0x50, + 0xd8, 0x32, 0x09, 0xe8, 0xe9, 0x0e, 0x40, 0x74, 0x05, 0x4a, 0x9d, 0x88, 0x84, 0x32, 0xbb, 0xaa, + 0x3c, 0xbf, 0x99, 0xaa, 0x94, 0x41, 0x28, 0x6b, 0xba, 0xa3, 0xb4, 0x94, 0x1a, 0x6b, 0xca, 0x55, + 0x8f, 0x1c, 0x66, 0x7f, 0xad, 0x08, 0x97, 0x72, 0x6d, 0xa9, 0x69, 0x97, 0xf6, 0x03, 0xdf, 0x8d, + 0x03, 0xf5, 0xc2, 0xc5, 0x63, 0x58, 0x90, 0xf6, 0xee, 0xba, 0x28, 0xc7, 0x0a, 0x03, 0x5d, 0x95, + 0x89, 0xa2, 0xd3, 0x49, 0x39, 0x96, 0xab, 0x46, 0xae, 0xe8, 0x41, 0x13, 0x1e, 0x3d, 0x03, 0xa5, + 0x76, 0xa0, 0xb2, 0xf8, 0xab, 0x99, 0xa5, 0xdd, 0x0d, 0x02, 0x0f, 0x33, 0x20, 0xfa, 0x94, 0x18, + 0x87, 0xd4, 0x93, 0x0e, 0x76, 0x5a, 0x41, 0xa4, 0x0d, 0xc6, 0x73, 0x30, 0xb2, 0x47, 0x0e, 0x43, + 0xd7, 0xdf, 0x49, 0x3f, 0xf5, 0xdd, 0xe2, 0xc5, 0x58, 0xc2, 0xcd, 0x98, 0xf4, 0x23, 0x67, 0x9d, + 0xa9, 0x68, 0xb4, 0xef, 0xd5, 0xf6, 0xa3, 0x45, 0x98, 0xc2, 0xcb, 0xd5, 0x6f, 0x4f, 0xc4, 0xdd, + 0xee, 0x89, 0x38, 0xeb, 0x4c, 0x45, 0xfd, 0x67, 0xe3, 0x57, 0x2d, 0x98, 0x62, 0x71, 0x5b, 0x45, + 0x0c, 0x06, 0x37, 0xf0, 0xcf, 0x81, 0x75, 0x7b, 0x06, 0x86, 0x42, 0xda, 0x68, 0x3a, 0xfd, 0x08, + 0xeb, 0x09, 0xe6, 0x30, 0xf4, 0x24, 0x94, 0x58, 0x17, 0xe8, 0xe4, 0x8d, 0xf3, 0xc8, 0xed, 0x55, + 0x27, 0x76, 0x30, 0x2b, 0x65, 0x8e, 0xc5, 0x98, 0xb4, 0x3d, 0x97, 0x77, 0x3a, 0x51, 0xf5, 0x7f, + 0x3c, 0x1c, 0x8b, 0x33, 0xbb, 0xf6, 0xe1, 0x1c, 0x8b, 0xb3, 0x49, 0xf6, 0x16, 0x8b, 0xfe, 0x47, + 0x01, 0x2e, 0x67, 0xd6, 0x1b, 0xd8, 0xb1, 0xb8, 0x77, 0xed, 0xb3, 0xb1, 0xd8, 0xc8, 0x36, 0xa4, + 0x28, 0x9e, 0xa3, 0x21, 0x45, 0x69, 0x50, 0xce, 0x71, 0x68, 0x00, 0x7f, 0xdf, 0xcc, 0x21, 0xfb, + 0x98, 0xf8, 0xfb, 0x66, 0xf6, 0x2d, 0x47, 0xac, 0xfb, 0x8b, 0x42, 0xce, 0xb7, 0x30, 0x01, 0xef, + 0x1a, 0x3d, 0x67, 0x18, 0x30, 0x12, 0x9c, 0xf0, 0x38, 0x3f, 0x63, 0x78, 0x19, 0x56, 0x50, 0xe4, + 0x6a, 0x9e, 0xb3, 0x85, 0xfc, 0xe4, 0x74, 0xb9, 0x4d, 0x2d, 0x98, 0x2f, 0x33, 0x6a, 0x08, 0x32, + 0xbc, 0x68, 0xd7, 0x35, 0xa1, 0xbc, 0x38, 0xb8, 0x50, 0x3e, 0x9e, 0x2d, 0x90, 0xa3, 0x25, 0x98, + 0xda, 0x77, 0x7d, 0x96, 0x6c, 0xdc, 0x64, 0x45, 0x55, 0x20, 0x89, 0x75, 0x13, 0x8c, 0xd3, 0xf8, + 0xf3, 0x6f, 0xc0, 0xc4, 0xc3, 0xab, 0x23, 0xbf, 0x55, 0x84, 0x27, 0x7a, 0x6c, 0x7b, 0x7e, 0xd6, + 0x1b, 0x73, 0xa0, 0x9d, 0xf5, 0x5d, 0xf3, 0x50, 0x87, 0xd9, 0xed, 0x8e, 0xe7, 0x1d, 0x32, 0x5b, + 0x45, 0xd2, 0x92, 0x18, 0x82, 0x57, 0x94, 0x29, 0xe4, 0x67, 0xd7, 0x32, 0x70, 0x70, 0x66, 0x4d, + 0xf4, 0x16, 0xa0, 0x40, 0x64, 0xc6, 0xbc, 0x41, 0x7c, 0xa1, 0xef, 0x66, 0x03, 0x5f, 0x4c, 0x36, + 0xe3, 0x9d, 0x2e, 0x0c, 0x9c, 0x51, 0x8b, 0x32, 0xfd, 0xf4, 0x56, 0x3a, 0x54, 0xdd, 0x4a, 0x31, + 0xfd, 0x58, 0x07, 0x62, 0x13, 0x17, 0xdd, 0x80, 0x19, 0xe7, 0xc0, 0x71, 0x79, 0x58, 0x34, 0x49, + 0x80, 0x73, 0xfd, 0x4a, 0x09, 0xb6, 0x94, 0x46, 0xc0, 0xdd, 0x75, 0x52, 0xae, 0xbb, 0xc3, 0xf9, + 0xae, 0xbb, 0xbd, 0xcf, 0xc5, 0x7e, 0x3a, 0x5d, 0xfb, 0x3f, 0x5b, 0xf4, 0xfa, 0xca, 0xc8, 0x6e, + 0x4d, 0xc7, 0x41, 0xe9, 0x26, 0x35, 0x2f, 0x5a, 0x35, 0x0e, 0x2b, 0x3a, 0x10, 0x9b, 0xb8, 0x7c, + 0x41, 0x44, 0x89, 0x43, 0x87, 0xc1, 0xba, 0x0b, 0x37, 0x79, 0x85, 0x81, 0xbe, 0x08, 0x23, 0x2d, + 0xf7, 0xc0, 0x8d, 0x82, 0x50, 0x6c, 0x96, 0x53, 0x9a, 0xc5, 0x27, 0xe7, 0x60, 0x95, 0x93, 0xc1, + 0x92, 0x9e, 0xfd, 0xa3, 0x05, 0x98, 0x90, 0x2d, 0xbe, 0xdd, 0x09, 0x62, 0xe7, 0x1c, 0xae, 0xe5, + 0x1b, 0xc6, 0xb5, 0xfc, 0xa9, 0x5e, 0xb1, 0x02, 0x58, 0x97, 0x72, 0xaf, 0xe3, 0x3b, 0xa9, 0xeb, + 0xf8, 0xd9, 0xfe, 0xa4, 0x7a, 0x5f, 0xc3, 0xff, 0xc2, 0x82, 0x19, 0x03, 0xff, 0x1c, 0x6e, 0x83, + 0x35, 0xf3, 0x36, 0x78, 0xba, 0xef, 0x37, 0xe4, 0xdc, 0x02, 0x5f, 0x2f, 0xa4, 0xfa, 0xce, 0x4e, + 0xff, 0xf7, 0xa1, 0xb4, 0xeb, 0x84, 0xad, 0x5e, 0x91, 0x44, 0xbb, 0x2a, 0x2d, 0xdc, 0x74, 0xc2, + 0x16, 0x3f, 0xc3, 0x5f, 0x50, 0xf9, 0xf4, 0x9c, 0xb0, 0xd5, 0xd7, 0x7f, 0x89, 0x35, 0x85, 0x5e, + 0x87, 0xe1, 0xa8, 0x19, 0xb4, 0x95, 0x75, 0xe1, 0x15, 0x9e, 0x6b, 0x8f, 0x96, 0x9c, 0x1c, 0x55, + 0x90, 0xd9, 0x1c, 0x2d, 0xc6, 0x02, 0x7f, 0x7e, 0x07, 0xca, 0xaa, 0xe9, 0x47, 0xea, 0x1b, 0xf2, + 0x1f, 0x8b, 0x70, 0x21, 0x63, 0x5d, 0xa0, 0xc8, 0x18, 0xad, 0x97, 0x06, 0x5c, 0x4e, 0x1f, 0x72, + 0xbc, 0x22, 0x26, 0xb1, 0xb4, 0xc4, 0xfc, 0x0f, 0xdc, 0xe8, 0xdd, 0x88, 0xa4, 0x1b, 0xa5, 0x45, + 0xfd, 0x1b, 0xa5, 0x8d, 0x9d, 0xdb, 0x50, 0xd3, 0x86, 0x54, 0x4f, 0x1f, 0xe9, 0x9c, 0xfe, 0x69, + 0x11, 0x66, 0xb3, 0x42, 0x8c, 0xa0, 0xef, 0x4f, 0x25, 0xc6, 0x78, 0x65, 0xd0, 0xe0, 0x24, 0x3c, + 0x5b, 0x86, 0x48, 0xf3, 0xba, 0x60, 0xa6, 0xca, 0xe8, 0x3b, 0xcc, 0xa2, 0x4d, 0xe6, 0x50, 0x18, + 0xf2, 0x84, 0x26, 0x72, 0x8b, 0x7f, 0x66, 0xe0, 0x0e, 0x88, 0x4c, 0x28, 0x51, 0xca, 0xa1, 0x50, + 0x16, 0xf7, 0x77, 0x28, 0x94, 0x2d, 0xcf, 0xbb, 0x30, 0xa6, 0x7d, 0xcd, 0x23, 0x9d, 0xf1, 0x3d, + 0x7a, 0xa3, 0x68, 0xfd, 0x7e, 0xa4, 0xb3, 0xfe, 0xd3, 0x16, 0xa4, 0x6c, 0xfa, 0x94, 0x4a, 0xca, + 0xca, 0x55, 0x49, 0x5d, 0x81, 0x52, 0x18, 0x78, 0x24, 0x9d, 0xab, 0x02, 0x07, 0x1e, 0xc1, 0x0c, + 0xa2, 0xb2, 0xec, 0x17, 0x73, 0xb3, 0xec, 0x3f, 0x03, 0x43, 0x1e, 0x39, 0x20, 0x52, 0x1b, 0xa1, + 0xce, 0xe4, 0xdb, 0xb4, 0x10, 0x73, 0x98, 0xfd, 0xab, 0x25, 0x78, 0xaa, 0xa7, 0x4b, 0x2e, 0x15, + 0x59, 0x76, 0x9c, 0x98, 0xdc, 0x77, 0x0e, 0xd3, 0x41, 0x6e, 0x6f, 0xf0, 0x62, 0x2c, 0xe1, 0xcc, + 0x02, 0x99, 0x87, 0xc7, 0x4b, 0x29, 0xf0, 0x44, 0x54, 0x3c, 0x01, 0x7d, 0x04, 0x29, 0xae, 0xaf, + 0x03, 0x44, 0x91, 0xb7, 0xea, 0x53, 0x0e, 0xac, 0x25, 0x4c, 0x9b, 0x93, 0x30, 0x8a, 0x8d, 0xdb, + 0x02, 0x82, 0x35, 0x2c, 0x54, 0x85, 0xe9, 0x76, 0x18, 0xc4, 0x5c, 0x1f, 0x5a, 0xe5, 0x46, 0x32, + 0x43, 0xa6, 0x37, 0x64, 0x3d, 0x05, 0xc7, 0x5d, 0x35, 0xd0, 0xab, 0x30, 0x26, 0x3c, 0x24, 0xeb, + 0x41, 0xe0, 0x09, 0x55, 0x8d, 0x32, 0xb9, 0x68, 0x24, 0x20, 0xac, 0xe3, 0x69, 0xd5, 0x98, 0x92, + 0x75, 0x24, 0xb3, 0x1a, 0x57, 0xb4, 0x6a, 0x78, 0xa9, 0x70, 0x43, 0xa3, 0x03, 0x85, 0x1b, 0x4a, + 0x94, 0x57, 0xe5, 0x81, 0xdf, 0x95, 0xa0, 0xaf, 0xba, 0xe7, 0x97, 0x4a, 0x70, 0x41, 0x2c, 0x9c, + 0x47, 0xbd, 0x5c, 0x1e, 0x51, 0x22, 0xee, 0x6f, 0xaf, 0x99, 0xf3, 0x5e, 0x33, 0xdf, 0x28, 0xc2, + 0x30, 0x9f, 0x8a, 0x73, 0xe0, 0xe1, 0xd7, 0x84, 0xd2, 0xaf, 0x47, 0xa0, 0x1d, 0xde, 0x97, 0x85, + 0xaa, 0x13, 0x3b, 0xfc, 0xfe, 0x52, 0xc7, 0x68, 0xa2, 0x1e, 0x44, 0x0b, 0xc6, 0x41, 0x3b, 0x9f, + 0xd2, 0x6a, 0x01, 0xa7, 0xa1, 0x1d, 0xbb, 0x5f, 0x06, 0x88, 0x58, 0x32, 0x68, 0x4a, 0x43, 0x84, + 0x6c, 0xfa, 0x74, 0x8f, 0xd6, 0x1b, 0x0a, 0x99, 0xf7, 0x21, 0x59, 0x82, 0x0a, 0x80, 0x35, 0x8a, + 0xf3, 0xaf, 0x41, 0x59, 0x21, 0xf7, 0x53, 0x01, 0x8c, 0xeb, 0xb7, 0xde, 0xe7, 0x61, 0x2a, 0xd5, + 0xd6, 0xa9, 0x34, 0x08, 0xbf, 0x66, 0xc1, 0x14, 0xef, 0xf2, 0xaa, 0x7f, 0x20, 0x36, 0xfb, 0x07, + 0x30, 0xeb, 0x65, 0x6c, 0x3a, 0x31, 0xa3, 0x83, 0x6f, 0x52, 0xa5, 0x31, 0xc8, 0x82, 0xe2, 0xcc, + 0x36, 0xd0, 0x35, 0x18, 0xe5, 0x2e, 0x3b, 0x8e, 0x27, 0xdc, 0x2c, 0xc6, 0x79, 0x6c, 0x7b, 0x5e, + 0x86, 0x15, 0xd4, 0xfe, 0x3d, 0x0b, 0x66, 0x78, 0xcf, 0x6f, 0x91, 0x43, 0x25, 0x1d, 0x7f, 0x94, + 0x7d, 0x17, 0xa1, 0xfb, 0x0b, 0x39, 0xa1, 0xfb, 0xf5, 0x4f, 0x2b, 0xf6, 0xfc, 0xb4, 0x5f, 0xb4, + 0x40, 0xac, 0xc0, 0x73, 0x90, 0x03, 0xbf, 0xcb, 0x94, 0x03, 0xe7, 0xf3, 0x17, 0x75, 0x8e, 0x00, + 0xf8, 0xe7, 0x16, 0x4c, 0x73, 0x84, 0xe4, 0x21, 0xf2, 0x23, 0x9d, 0x87, 0x41, 0x12, 0x4a, 0xa9, + 0x0c, 0xb2, 0xd9, 0x1f, 0x65, 0x4c, 0x56, 0xa9, 0xe7, 0x64, 0xb5, 0xe4, 0x06, 0x3a, 0x45, 0xa2, + 0xb4, 0x53, 0x07, 0xd3, 0xb5, 0xff, 0xc4, 0x02, 0xc4, 0x9b, 0x31, 0xee, 0x65, 0x7a, 0xdb, 0xb1, + 0x52, 0x4d, 0x13, 0x94, 0x1c, 0x35, 0x0a, 0x82, 0x35, 0xac, 0x33, 0x19, 0x9e, 0xd4, 0x6b, 0x72, + 0xb1, 0xff, 0x6b, 0xf2, 0x29, 0x46, 0xf4, 0x1b, 0x25, 0x48, 0xdb, 0x68, 0xa3, 0x7b, 0x30, 0xde, + 0x74, 0xda, 0xce, 0x96, 0xeb, 0xb9, 0xb1, 0x4b, 0xa2, 0x5e, 0x66, 0x28, 0x2b, 0x1a, 0x9e, 0x78, + 0x27, 0xd4, 0x4a, 0xb0, 0x41, 0x07, 0x2d, 0x00, 0xb4, 0x43, 0xf7, 0xc0, 0xf5, 0xc8, 0x0e, 0x13, + 0x85, 0x99, 0x63, 0x17, 0xb7, 0xad, 0x90, 0xa5, 0x58, 0xc3, 0xc8, 0x70, 0x04, 0x2a, 0x3e, 0x3a, + 0x47, 0xa0, 0xd2, 0x29, 0x1d, 0x81, 0x86, 0x06, 0x72, 0x04, 0xc2, 0xf0, 0x98, 0xbc, 0xbb, 0xe9, + 0xff, 0x35, 0xd7, 0x23, 0x82, 0x61, 0xe3, 0xee, 0x5e, 0xf3, 0xc7, 0x47, 0x95, 0xc7, 0x70, 0x26, + 0x06, 0xce, 0xa9, 0x89, 0xbe, 0x00, 0x73, 0x8e, 0xe7, 0x05, 0xf7, 0xd5, 0xa8, 0xad, 0x46, 0x4d, + 0xc7, 0xe3, 0xea, 0xde, 0x11, 0x46, 0xf5, 0xc9, 0xe3, 0xa3, 0xca, 0xdc, 0x52, 0x0e, 0x0e, 0xce, + 0xad, 0x9d, 0xf2, 0x23, 0x1a, 0xed, 0xeb, 0x47, 0xb4, 0x07, 0x17, 0x1a, 0x24, 0x74, 0x59, 0x1a, + 0xb7, 0x56, 0xb2, 0x25, 0x37, 0xa1, 0x1c, 0xa6, 0x0e, 0xa1, 0x81, 0x62, 0xc0, 0x68, 0x71, 0x42, + 0xe5, 0xa1, 0x93, 0x10, 0xb2, 0xff, 0xcc, 0x82, 0x11, 0x61, 0x27, 0x7e, 0x0e, 0xbc, 0xcf, 0x92, + 0xa1, 0xbf, 0xac, 0x64, 0x1f, 0xd4, 0xac, 0x33, 0xb9, 0x9a, 0xcb, 0x5a, 0x4a, 0x73, 0xf9, 0x74, + 0x2f, 0x22, 0xbd, 0x75, 0x96, 0x3f, 0x59, 0x84, 0x49, 0xd3, 0x46, 0xfe, 0x1c, 0x86, 0x60, 0x03, + 0x46, 0x22, 0xe1, 0x90, 0x51, 0xc8, 0x37, 0x9c, 0x4d, 0x4f, 0x62, 0x62, 0x15, 0x23, 0x5c, 0x30, + 0x24, 0x91, 0x4c, 0x4f, 0x8f, 0xe2, 0x23, 0xf4, 0xf4, 0xe8, 0xe7, 0xa6, 0x50, 0x3a, 0x0b, 0x37, + 0x05, 0xfb, 0x9b, 0xec, 0xb2, 0xd0, 0xcb, 0xcf, 0x81, 0x8f, 0xb8, 0x61, 0x5e, 0x2b, 0x76, 0x8f, + 0x95, 0x25, 0x3a, 0x95, 0xc3, 0x4f, 0xfc, 0x13, 0x0b, 0xc6, 0x04, 0xe2, 0x39, 0x74, 0xfb, 0xbb, + 0xcd, 0x6e, 0x3f, 0xd1, 0xa3, 0xdb, 0x39, 0xfd, 0xfd, 0x7b, 0x05, 0xd5, 0xdf, 0x7a, 0x10, 0xc6, + 0x03, 0xdc, 0xfe, 0xaf, 0xc3, 0x28, 0x95, 0x1e, 0x83, 0x66, 0xe0, 0x89, 0xcb, 0xff, 0xc9, 0xc4, + 0x43, 0x98, 0x97, 0x9f, 0x68, 0xbf, 0xb1, 0xc2, 0x66, 0x0e, 0xac, 0x41, 0x18, 0x8b, 0x0b, 0x37, + 0x71, 0x60, 0x0d, 0xc2, 0x18, 0x33, 0x08, 0x6a, 0x01, 0xc4, 0x4e, 0xb8, 0x43, 0x62, 0x5a, 0x26, + 0x82, 0x0d, 0xe4, 0xef, 0xc2, 0x4e, 0xec, 0x7a, 0x0b, 0xae, 0x1f, 0x47, 0x71, 0xb8, 0x50, 0xf3, + 0xe3, 0x3b, 0x21, 0x97, 0x25, 0x34, 0x97, 0x5f, 0x45, 0x0b, 0x6b, 0x74, 0xa5, 0x0f, 0x19, 0x6b, + 0x63, 0xc8, 0x7c, 0x58, 0xdc, 0x10, 0xe5, 0x58, 0x61, 0xd8, 0xaf, 0xb1, 0x33, 0x99, 0x0d, 0xd0, + 0xe9, 0xbc, 0x71, 0x7f, 0x67, 0x54, 0x0d, 0x2d, 0x7b, 0x55, 0xa8, 0xea, 0x3e, 0xbf, 0xbd, 0x8f, + 0x40, 0xda, 0xb0, 0xee, 0x8f, 0x90, 0x38, 0x06, 0xa3, 0xef, 0xe9, 0x7a, 0x6f, 0x7e, 0xb1, 0xcf, + 0x59, 0x7a, 0x8a, 0x17, 0x66, 0x16, 0xe0, 0x96, 0x05, 0x02, 0xad, 0xd5, 0x85, 0x74, 0xa9, 0x05, + 0xb8, 0x15, 0x00, 0x9c, 0xe0, 0xa0, 0x45, 0x21, 0x89, 0x96, 0x8c, 0xec, 0x51, 0x52, 0x12, 0x95, + 0x9f, 0xaf, 0x89, 0xa2, 0x2f, 0xc1, 0x98, 0xca, 0x22, 0x55, 0xe7, 0xc9, 0x78, 0x44, 0xe8, 0x85, + 0xd5, 0xa4, 0x18, 0xeb, 0x38, 0x68, 0x13, 0xa6, 0x22, 0x9e, 0x4f, 0x4b, 0x45, 0xd8, 0xe2, 0x7a, + 0x86, 0x4f, 0xcb, 0x77, 0xea, 0x86, 0x09, 0x3e, 0x61, 0x45, 0x7c, 0xb3, 0x4a, 0x47, 0xb0, 0x34, + 0x09, 0xf4, 0x26, 0x4c, 0x7a, 0x7a, 0x62, 0xe1, 0xba, 0x50, 0x43, 0x28, 0x33, 0x4e, 0x23, 0xed, + 0x70, 0x1d, 0xa7, 0xb0, 0x29, 0xd3, 0xa0, 0x97, 0x88, 0xa8, 0x70, 0x8e, 0xbf, 0x43, 0x22, 0x91, + 0x03, 0x87, 0x31, 0x0d, 0xb7, 0x73, 0x70, 0x70, 0x6e, 0x6d, 0xf4, 0x3a, 0x8c, 0xcb, 0xcf, 0xd7, + 0xdc, 0x1c, 0x13, 0x63, 0x61, 0x0d, 0x86, 0x0d, 0x4c, 0x74, 0x1f, 0x2e, 0xca, 0xff, 0x9b, 0xa1, + 0xb3, 0xbd, 0xed, 0x36, 0x85, 0x97, 0x29, 0xf7, 0x98, 0x58, 0x92, 0x2e, 0x18, 0xab, 0x59, 0x48, + 0x27, 0x47, 0x95, 0x2b, 0x62, 0xd4, 0x32, 0xe1, 0x6c, 0x12, 0xb3, 0xe9, 0xa3, 0x75, 0xb8, 0xb0, + 0x4b, 0x1c, 0x2f, 0xde, 0x5d, 0xd9, 0x25, 0xcd, 0x3d, 0xb9, 0x89, 0x98, 0xf3, 0xa4, 0x66, 0x62, + 0x7b, 0xb3, 0x1b, 0x05, 0x67, 0xd5, 0x43, 0xef, 0xc2, 0x5c, 0xbb, 0xb3, 0xe5, 0xb9, 0xd1, 0xee, + 0x46, 0x10, 0xb3, 0xa7, 0x71, 0x95, 0x84, 0x49, 0x78, 0x59, 0x2a, 0xc7, 0xd1, 0x7a, 0x0e, 0x1e, + 0xce, 0xa5, 0x80, 0x3e, 0x80, 0x8b, 0xa9, 0xc5, 0x20, 0x7c, 0xbe, 0x26, 0xf3, 0x63, 0x6c, 0x36, + 0xb2, 0x2a, 0x08, 0x1f, 0xae, 0x2c, 0x10, 0xce, 0x6e, 0xe2, 0xc3, 0x19, 0x4c, 0xbc, 0x4f, 0x2b, + 0x6b, 0xdc, 0x0d, 0xfa, 0x0a, 0x8c, 0xeb, 0xab, 0x48, 0x5c, 0x30, 0x57, 0xfb, 0x25, 0xd1, 0x16, + 0xbc, 0x91, 0x5a, 0x51, 0x3a, 0x0c, 0x1b, 0x14, 0x6d, 0x02, 0xd9, 0xdf, 0x87, 0x6e, 0xc3, 0x68, + 0xd3, 0x73, 0x89, 0x1f, 0xd7, 0xea, 0xbd, 0xa2, 0x09, 0xac, 0x08, 0x1c, 0x31, 0x60, 0x22, 0x28, + 0x21, 0x2f, 0xc3, 0x8a, 0x82, 0xfd, 0x9b, 0x05, 0xa8, 0xf4, 0x89, 0x70, 0x99, 0xd2, 0x19, 0x5a, + 0x03, 0xe9, 0x0c, 0x97, 0x64, 0x4a, 0xa9, 0x8d, 0x94, 0xbc, 0x9a, 0x4a, 0x17, 0x95, 0x48, 0xad, + 0x69, 0xfc, 0x81, 0xed, 0x2c, 0x75, 0xb5, 0x63, 0xa9, 0xaf, 0x05, 0xb0, 0xf1, 0xdc, 0x30, 0x34, + 0x38, 0x47, 0x9f, 0xab, 0x3a, 0xb6, 0xbf, 0x59, 0x80, 0x8b, 0x6a, 0x08, 0xff, 0xea, 0x0e, 0xdc, + 0xdd, 0xee, 0x81, 0x3b, 0x03, 0xc5, 0xbb, 0x7d, 0x07, 0x86, 0x1b, 0x87, 0x51, 0x33, 0xf6, 0x06, + 0x60, 0x80, 0x9e, 0x31, 0xa3, 0xea, 0xa8, 0x6b, 0xda, 0x88, 0xac, 0xf3, 0xd7, 0x2d, 0x98, 0xda, + 0x5c, 0xa9, 0x37, 0x82, 0xe6, 0x1e, 0x89, 0x97, 0xb8, 0x5a, 0x09, 0x0b, 0xfe, 0xc7, 0x7a, 0x48, + 0xbe, 0x26, 0x8b, 0x63, 0xba, 0x02, 0xa5, 0xdd, 0x20, 0x8a, 0xd3, 0xaf, 0x72, 0x37, 0x83, 0x28, + 0xc6, 0x0c, 0x62, 0xff, 0xbe, 0x05, 0x43, 0x2c, 0x11, 0x62, 0xbf, 0xec, 0x9c, 0x83, 0x7c, 0x17, + 0x7a, 0x15, 0x86, 0xc9, 0xf6, 0x36, 0x69, 0xc6, 0x62, 0x56, 0xa5, 0x5b, 0xdf, 0xf0, 0x2a, 0x2b, + 0xa5, 0x97, 0x3e, 0x6b, 0x8c, 0xff, 0xc5, 0x02, 0x19, 0xbd, 0x03, 0xe5, 0xd8, 0xdd, 0x27, 0x4b, + 0xad, 0x96, 0x78, 0xd7, 0x78, 0x08, 0x2f, 0xca, 0x4d, 0x49, 0x00, 0x27, 0xb4, 0xec, 0xaf, 0x15, + 0x00, 0x12, 0xd7, 0xe4, 0x7e, 0x9f, 0xb8, 0xdc, 0x95, 0x80, 0xf4, 0x6a, 0x46, 0x02, 0x52, 0x94, + 0x10, 0xcc, 0x48, 0x3f, 0xaa, 0x86, 0xa9, 0x38, 0xd0, 0x30, 0x95, 0x4e, 0x33, 0x4c, 0x2b, 0x30, + 0x93, 0xb8, 0x56, 0x9b, 0x71, 0x26, 0x58, 0xc4, 0xfb, 0xcd, 0x34, 0x10, 0x77, 0xe3, 0xdb, 0x3f, + 0x6c, 0x81, 0x70, 0x4f, 0x18, 0x60, 0x31, 0x7f, 0x49, 0xa6, 0xef, 0x33, 0x02, 0xe5, 0x5e, 0xc9, + 0xf7, 0xd7, 0x10, 0xe1, 0x71, 0xd5, 0xe5, 0x61, 0x04, 0xc5, 0x35, 0x68, 0xd9, 0x2d, 0x10, 0xd0, + 0x2a, 0x61, 0x4a, 0x86, 0xfe, 0xbd, 0xb9, 0x0e, 0xd0, 0x62, 0xb8, 0x5a, 0x3a, 0x30, 0x75, 0x54, + 0x55, 0x15, 0x04, 0x6b, 0x58, 0xf6, 0x8f, 0x17, 0x60, 0x4c, 0x06, 0x66, 0xa5, 0x72, 0x7c, 0xff, + 0x56, 0x4e, 0x95, 0x95, 0x81, 0xe5, 0xcf, 0xa3, 0x84, 0x55, 0xf0, 0x7e, 0x3d, 0x7f, 0x9e, 0x04, + 0xe0, 0x04, 0x07, 0x3d, 0x07, 0x23, 0x51, 0x67, 0x8b, 0xa1, 0xa7, 0x8c, 0xee, 0x1b, 0xbc, 0x18, + 0x4b, 0x38, 0xfa, 0x02, 0x4c, 0xf3, 0x7a, 0x61, 0xd0, 0x76, 0x76, 0xb8, 0xc6, 0x69, 0x48, 0x79, + 0xc1, 0x4d, 0xaf, 0xa7, 0x60, 0x27, 0x47, 0x95, 0xd9, 0x74, 0x19, 0xd3, 0x55, 0x76, 0x51, 0xb1, + 0xbf, 0x02, 0xa8, 0x3b, 0xd6, 0x2c, 0x7a, 0x8b, 0x9b, 0x55, 0xb8, 0x21, 0x69, 0x0d, 0x9c, 0x33, + 0x7e, 0x5c, 0x1a, 0x4f, 0xd0, 0x5a, 0x58, 0xd5, 0xa7, 0x3b, 0x6f, 0x3a, 0xed, 0xc2, 0x83, 0x6e, + 0xc2, 0x30, 0x3f, 0x54, 0x05, 0xf9, 0x1e, 0x6f, 0x5c, 0x9a, 0xe3, 0x0f, 0x0b, 0x84, 0x2f, 0xce, + 0x65, 0x51, 0x1f, 0xbd, 0x0b, 0x63, 0xad, 0xe0, 0xbe, 0x7f, 0xdf, 0x09, 0x5b, 0x4b, 0xf5, 0x9a, + 0x58, 0x97, 0x99, 0xbc, 0x59, 0x35, 0x41, 0xd3, 0x9d, 0x89, 0x98, 0x3e, 0x37, 0x01, 0x61, 0x9d, + 0x1c, 0xda, 0xd4, 0x73, 0x7f, 0xf7, 0xb0, 0x83, 0x53, 0xc9, 0xbd, 0x35, 0xca, 0xf9, 0x59, 0xbf, + 0xbf, 0x7a, 0x01, 0x8c, 0xfd, 0x60, 0xe4, 0x7d, 0xb0, 0xce, 0x28, 0xef, 0x03, 0x86, 0x51, 0xb2, + 0xdf, 0x8e, 0x0f, 0xab, 0x6e, 0xd8, 0x2b, 0x71, 0xd0, 0xaa, 0xc0, 0xe9, 0xa6, 0x29, 0x21, 0x58, + 0xd1, 0xc9, 0x4e, 0xce, 0x51, 0xfc, 0x08, 0x93, 0x73, 0x94, 0xce, 0x31, 0x39, 0xc7, 0x06, 0x8c, + 0xec, 0xb8, 0x31, 0x26, 0xed, 0x40, 0x30, 0x14, 0x99, 0x2b, 0xe1, 0x06, 0x47, 0xe9, 0x0e, 0x0d, + 0x2f, 0x00, 0x58, 0x12, 0x41, 0x6f, 0xa9, 0x3d, 0x30, 0x9c, 0xcf, 0x8f, 0x77, 0x3f, 0x87, 0x64, + 0xee, 0x02, 0x91, 0x8c, 0x63, 0xe4, 0x61, 0x93, 0x71, 0xac, 0xc9, 0x14, 0x1a, 0xa3, 0xf9, 0x66, + 0xa3, 0x2c, 0x43, 0x46, 0x9f, 0xc4, 0x19, 0x46, 0xb2, 0x91, 0xf2, 0xd9, 0x25, 0x1b, 0xf9, 0x61, + 0x0b, 0x2e, 0xb6, 0xb3, 0xf2, 0xee, 0x88, 0x14, 0x18, 0xaf, 0x0e, 0x9c, 0x58, 0xc8, 0x68, 0x90, + 0x09, 0x66, 0x99, 0x68, 0x38, 0xbb, 0x39, 0x3a, 0xd0, 0xe1, 0x56, 0x4b, 0xe4, 0xcd, 0x78, 0x26, + 0x27, 0x6b, 0x49, 0x8f, 0x5c, 0x25, 0x9b, 0x19, 0xb9, 0x32, 0x3e, 0x99, 0x97, 0x2b, 0x63, 0xe0, + 0x0c, 0x19, 0x49, 0xbe, 0x92, 0x89, 0x0f, 0x9d, 0xaf, 0xe4, 0x2d, 0x95, 0xaf, 0xa4, 0x47, 0xd4, + 0x21, 0x9e, 0x8d, 0xa4, 0x6f, 0x96, 0x12, 0x2d, 0xd3, 0xc8, 0xd4, 0xd9, 0x64, 0x1a, 0x31, 0x0e, + 0x7b, 0x9e, 0xec, 0xe2, 0xf9, 0x3e, 0x87, 0xbd, 0x41, 0xb7, 0xf7, 0x71, 0xcf, 0xb3, 0xaa, 0xcc, + 0x3c, 0x54, 0x56, 0x95, 0x7b, 0x7a, 0x96, 0x12, 0xd4, 0x27, 0x0d, 0x07, 0x45, 0x1a, 0x30, 0x37, + 0xc9, 0x3d, 0xfd, 0x0a, 0xba, 0x90, 0x4f, 0x57, 0xdd, 0x34, 0xdd, 0x74, 0xb3, 0x2e, 0xa1, 0xee, + 0x9c, 0x27, 0xb3, 0xe7, 0x93, 0xf3, 0xe4, 0xe2, 0x99, 0xe7, 0x3c, 0x79, 0xec, 0x1c, 0x72, 0x9e, + 0x3c, 0xfe, 0x91, 0xe6, 0x3c, 0x99, 0x7b, 0x04, 0x39, 0x4f, 0x36, 0x92, 0x9c, 0x27, 0x97, 0xf2, + 0xa7, 0x24, 0xc3, 0x4e, 0x2e, 0x27, 0xd3, 0xc9, 0x3d, 0x28, 0xb7, 0xa5, 0x97, 0xb7, 0x08, 0x8b, + 0x94, 0x9d, 0x6c, 0x31, 0xcb, 0x15, 0x9c, 0x4f, 0x89, 0x02, 0xe1, 0x84, 0x14, 0xa5, 0x9b, 0x64, + 0x3e, 0x79, 0xa2, 0x87, 0xea, 0x2d, 0x4b, 0xa9, 0x91, 0x9f, 0xef, 0xc4, 0xfe, 0x1b, 0x05, 0xb8, + 0xdc, 0x7b, 0x5d, 0x27, 0x1a, 0x91, 0x7a, 0xa2, 0xc1, 0x4f, 0x69, 0x44, 0xb8, 0x98, 0x91, 0x60, + 0x0d, 0x1c, 0x0a, 0xe3, 0x06, 0xcc, 0x28, 0x03, 0x39, 0xcf, 0x6d, 0x1e, 0x6a, 0x49, 0x18, 0x95, + 0xb3, 0x4e, 0x23, 0x8d, 0x80, 0xbb, 0xeb, 0xa0, 0x25, 0x98, 0x32, 0x0a, 0x6b, 0x55, 0x21, 0x4e, + 0x28, 0x15, 0x4c, 0xc3, 0x04, 0xe3, 0x34, 0xbe, 0xfd, 0x75, 0x0b, 0x1e, 0xcf, 0x09, 0x07, 0x3e, + 0x70, 0xa4, 0x87, 0x6d, 0x98, 0x6a, 0x9b, 0x55, 0xfb, 0x04, 0x84, 0x31, 0x82, 0x8e, 0xab, 0xbe, + 0xa6, 0x00, 0x38, 0x4d, 0x74, 0xf9, 0xda, 0x6f, 0xff, 0xe1, 0xe5, 0x4f, 0xfc, 0xee, 0x1f, 0x5e, + 0xfe, 0xc4, 0xef, 0xfd, 0xe1, 0xe5, 0x4f, 0xfc, 0x7f, 0xc7, 0x97, 0xad, 0xdf, 0x3e, 0xbe, 0x6c, + 0xfd, 0xee, 0xf1, 0x65, 0xeb, 0xf7, 0x8e, 0x2f, 0x5b, 0x7f, 0x70, 0x7c, 0xd9, 0xfa, 0xda, 0x1f, + 0x5d, 0xfe, 0xc4, 0x97, 0x0a, 0x07, 0x2f, 0xfd, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x72, 0x3b, + 0x4e, 0x60, 0xe2, 0xdd, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/core/v1/generated.proto b/staging/src/k8s.io/api/core/v1/generated.proto index 643d256d381..2ee9b7c4d91 100644 --- a/staging/src/k8s.io/api/core/v1/generated.proto +++ b/staging/src/k8s.io/api/core/v1/generated.proto @@ -439,6 +439,34 @@ message ConfigMapList { repeated ConfigMap items = 2; } +// ConfigMapNodeConfigSource contains the information to reference a ConfigMap as a config source for the Node. +message ConfigMapNodeConfigSource { + // Namespace is the metadata.namespace of the referenced ConfigMap. + // This field is required in all cases. + optional string namespace = 1; + + // Name is the metadata.name of the referenced ConfigMap. + // This field is required in all cases. + optional string name = 2; + + // UID is the metadata.UID of the referenced ConfigMap. + // This field is currently reqired in Node.Spec. + // TODO(#61643): This field will be forbidden in Node.Spec when #61643 is resolved. + // TODO(#56896): This field will be required in Node.Status when #56896 is resolved. + // +optional + optional string uid = 3; + + // ResourceVersion is the metadata.ResourceVersion of the referenced ConfigMap. + // This field is forbidden in Node.Spec. + // TODO(#56896): This field will be required in Node.Status when #56896 is resolved. + // +optional + optional string resourceVersion = 4; + + // KubeletConfigKey declares which key of the referenced ConfigMap corresponds to the KubeletConfiguration structure + // This field is required in all cases. + optional string kubeletConfigKey = 5; +} + // Adapts a ConfigMap into a projected volume. // // The contents of the target ConfigMap's Data field will be presented in a @@ -1815,7 +1843,8 @@ message NodeCondition { // NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil. message NodeConfigSource { - optional ObjectReference configMapRef = 1; + // ConfigMap is a reference to a Node's ConfigMap + optional ConfigMapNodeConfigSource configMap = 2; } // NodeDaemonEndpoints lists ports opened by daemons running on the Node. diff --git a/staging/src/k8s.io/api/core/v1/register.go b/staging/src/k8s.io/api/core/v1/register.go index 526e1320ade..1aac0cb41e0 100644 --- a/staging/src/k8s.io/api/core/v1/register.go +++ b/staging/src/k8s.io/api/core/v1/register.go @@ -57,7 +57,6 @@ func addKnownTypes(scheme *runtime.Scheme) error { &Endpoints{}, &EndpointsList{}, &Node{}, - &NodeConfigSource{}, &NodeList{}, &NodeProxyOptions{}, &Binding{}, diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 7ccae7683bb..d4c9260c55d 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -3640,12 +3640,49 @@ type NodeSpec struct { DoNotUse_ExternalID string `json:"externalID,omitempty" protobuf:"bytes,2,opt,name=externalID"` } -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - // NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil. type NodeConfigSource struct { - metav1.TypeMeta `json:",inline"` - ConfigMapRef *ObjectReference `json:"configMapRef,omitempty" protobuf:"bytes,1,opt,name=configMapRef"` + // For historical context, regarding the below kind, apiVersion, and configMapRef deprecation tags: + // 1. kind/apiVersion were used by the kubelet to persist this struct to disk (they had no protobuf tags) + // 2. configMapRef and proto tag 1 were used by the API to refer to a configmap, + // but used a generic ObjectReference type that didn't really have the fields we needed + // All uses/persistence of the NodeConfigSource struct prior to 1.11 were gated by alpha feature flags, + // so there was no persisted data for these fields that needed to be migrated/handled. + + // +k8s:deprecated=kind + // +k8s:deprecated=apiVersion + // +k8s:deprecated=configMapRef,protobuf=1 + + // ConfigMap is a reference to a Node's ConfigMap + ConfigMap *ConfigMapNodeConfigSource `json:"configMap,omitempty" protobuf:"bytes,2,opt,name=configMap"` +} + +// ConfigMapNodeConfigSource contains the information to reference a ConfigMap as a config source for the Node. +type ConfigMapNodeConfigSource struct { + // Namespace is the metadata.namespace of the referenced ConfigMap. + // This field is required in all cases. + Namespace string `json:"namespace" protobuf:"bytes,1,opt,name=namespace"` + + // Name is the metadata.name of the referenced ConfigMap. + // This field is required in all cases. + Name string `json:"name" protobuf:"bytes,2,opt,name=name"` + + // UID is the metadata.UID of the referenced ConfigMap. + // This field is currently reqired in Node.Spec. + // TODO(#61643): This field will be forbidden in Node.Spec when #61643 is resolved. + // TODO(#56896): This field will be required in Node.Status when #56896 is resolved. + // +optional + UID types.UID `json:"uid,omitempty" protobuf:"bytes,3,opt,name=uid"` + + // ResourceVersion is the metadata.ResourceVersion of the referenced ConfigMap. + // This field is forbidden in Node.Spec. + // TODO(#56896): This field will be required in Node.Status when #56896 is resolved. + // +optional + ResourceVersion string `json:"resourceVersion,omitempty" protobuf:"bytes,4,opt,name=resourceVersion"` + + // KubeletConfigKey declares which key of the referenced ConfigMap corresponds to the KubeletConfiguration structure + // This field is required in all cases. + KubeletConfigKey string `json:"kubeletConfigKey" protobuf:"bytes,5,opt,name=kubeletConfigKey"` } // DaemonEndpoint contains information about a single Daemon endpoint. diff --git a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go index 4177e499d99..2aa49d5b698 100644 --- a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go @@ -262,6 +262,19 @@ func (ConfigMapList) SwaggerDoc() map[string]string { return map_ConfigMapList } +var map_ConfigMapNodeConfigSource = map[string]string{ + "": "ConfigMapNodeConfigSource contains the information to reference a ConfigMap as a config source for the Node.", + "namespace": "Namespace is the metadata.namespace of the referenced ConfigMap. This field is required in all cases.", + "name": "Name is the metadata.name of the referenced ConfigMap. This field is required in all cases.", + "uid": "UID is the metadata.UID of the referenced ConfigMap. This field is currently reqired in Node.Spec.", + "resourceVersion": "ResourceVersion is the metadata.ResourceVersion of the referenced ConfigMap. This field is forbidden in Node.Spec.", + "kubeletConfigKey": "KubeletConfigKey declares which key of the referenced ConfigMap corresponds to the KubeletConfiguration structure This field is required in all cases.", +} + +func (ConfigMapNodeConfigSource) SwaggerDoc() map[string]string { + return map_ConfigMapNodeConfigSource +} + var map_ConfigMapProjection = map[string]string{ "": "Adapts a ConfigMap into a projected volume.\n\nThe contents of the target ConfigMap's Data field will be presented in a projected volume as files using the keys in the Data field as the file names, unless the items element is populated with specific mappings of keys to paths. Note that this is identical to a configmap volume source without the default mode.", "items": "If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", @@ -969,7 +982,8 @@ func (NodeCondition) SwaggerDoc() map[string]string { } var map_NodeConfigSource = map[string]string{ - "": "NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil.", + "": "NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil.", + "configMap": "ConfigMap is a reference to a Node's ConfigMap", } func (NodeConfigSource) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go index 47b62205239..d2d3978b822 100644 --- a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go @@ -631,6 +631,22 @@ func (in *ConfigMapList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigMapNodeConfigSource) DeepCopyInto(out *ConfigMapNodeConfigSource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigMapNodeConfigSource. +func (in *ConfigMapNodeConfigSource) DeepCopy() *ConfigMapNodeConfigSource { + if in == nil { + return nil + } + out := new(ConfigMapNodeConfigSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConfigMapProjection) DeepCopyInto(out *ConfigMapProjection) { *out = *in @@ -2360,13 +2376,12 @@ func (in *NodeCondition) DeepCopy() *NodeCondition { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeConfigSource) DeepCopyInto(out *NodeConfigSource) { *out = *in - out.TypeMeta = in.TypeMeta - if in.ConfigMapRef != nil { - in, out := &in.ConfigMapRef, &out.ConfigMapRef + if in.ConfigMap != nil { + in, out := &in.ConfigMap, &out.ConfigMap if *in == nil { *out = nil } else { - *out = new(ObjectReference) + *out = new(ConfigMapNodeConfigSource) **out = **in } } @@ -2383,14 +2398,6 @@ func (in *NodeConfigSource) DeepCopy() *NodeConfigSource { return out } -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *NodeConfigSource) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeDaemonEndpoints) DeepCopyInto(out *NodeDaemonEndpoints) { *out = *in diff --git a/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json b/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json index 098a0454ec4..0799ec6fc2a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json @@ -950,18 +950,6 @@ "ImportPath": "k8s.io/apimachinery/pkg/api/validation/path", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/announced", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apimachinery/pkg/apis/meta/fuzzer", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2008,23 +1996,23 @@ }, { "ImportPath": "k8s.io/kube-openapi/pkg/builder", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/handler", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/apimachinery/pkg/api/equality", @@ -2058,14 +2046,6 @@ "ImportPath": "k8s.io/apimachinery/pkg/api/validation", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/announced", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apimachinery/pkg/apis/meta/internalversion", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2262,6 +2242,10 @@ "ImportPath": "k8s.io/client-go/rest", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/client-go/restmapper", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/client-go/scale", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/staging/src/k8s.io/apiextensions-apiserver/main.go b/staging/src/k8s.io/apiextensions-apiserver/main.go index 7723e651356..09143ab6e1b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/main.go +++ b/staging/src/k8s.io/apiextensions-apiserver/main.go @@ -19,7 +19,6 @@ package main import ( "flag" "os" - "runtime" "github.com/golang/glog" @@ -32,10 +31,6 @@ func main() { logs.InitLogs() defer logs.FlushLogs() - if len(os.Getenv("GOMAXPROCS")) == 0 { - runtime.GOMAXPROCS(runtime.NumCPU()) - } - stopCh := genericapiserver.SetupSignalHandler() cmd := server.NewCommandStartCustomResourceDefinitionsServer(os.Stdout, os.Stderr, stopCh) cmd.Flags().AddGoFlagSet(flag.CommandLine) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/deepcopy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/deepcopy.go index dd9680c36f5..37b4d1df9fe 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/deepcopy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/deepcopy.go @@ -16,6 +16,8 @@ limitations under the License. package apiextensions +import "k8s.io/apimachinery/pkg/runtime" + // TODO: Update this after a tag is created for interface fields in DeepCopy func (in *JSONSchemaProps) DeepCopy() *JSONSchemaProps { if in == nil { @@ -26,14 +28,14 @@ func (in *JSONSchemaProps) DeepCopy() *JSONSchemaProps { *out = *in if in.Default != nil { - defaultJSON := JSON(deepCopyJSON(*(in.Default))) + defaultJSON := JSON(runtime.DeepCopyJSONValue(*(in.Default))) out.Default = &(defaultJSON) } else { out.Default = nil } if in.Example != nil { - exampleJSON := JSON(deepCopyJSON(*(in.Example))) + exampleJSON := JSON(runtime.DeepCopyJSONValue(*(in.Example))) out.Example = &(exampleJSON) } else { out.Example = nil @@ -121,7 +123,7 @@ func (in *JSONSchemaProps) DeepCopy() *JSONSchemaProps { if in.Enum != nil { out.Enum = make([]JSON, len(in.Enum)) for i := range in.Enum { - out.Enum[i] = deepCopyJSON(in.Enum[i]) + out.Enum[i] = runtime.DeepCopyJSONValue(in.Enum[i]) } } @@ -258,22 +260,3 @@ func (in *JSONSchemaProps) DeepCopy() *JSONSchemaProps { return out } - -func deepCopyJSON(x interface{}) interface{} { - switch x := x.(type) { - case map[string]interface{}: - clone := make(map[string]interface{}, len(x)) - for k, v := range x { - clone[k] = deepCopyJSON(v) - } - return clone - case []interface{}: - clone := make([]interface{}, len(x)) - for i := range x { - clone[i] = deepCopyJSON(x[i]) - } - return clone - default: - return x - } -} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install/BUILD index 95d9f16e43e..26acfd2c279 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install/BUILD @@ -23,9 +23,8 @@ go_library( deps = [ "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install/install.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install/install.go index a574ee45333..2fd04b77d36 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install/install.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install/install.go @@ -19,23 +19,13 @@ package install import ( "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: apiextensions.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: apiextensions.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(apiextensions.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/deepcopy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/deepcopy.go index 903773ae213..f6a114e2b39 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/deepcopy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/deepcopy.go @@ -236,22 +236,3 @@ func (in *JSONSchemaProps) DeepCopy() *JSONSchemaProps { return out } - -func deepCopyJSON(x interface{}) interface{} { - switch x := x.(type) { - case map[string]interface{}: - clone := make(map[string]interface{}, len(x)) - for k, v := range x { - clone[k] = deepCopyJSON(v) - } - return clone - case []interface{}: - clone := make([]interface{}, len(x)) - for i := range x { - clone[i] = deepCopyJSON(x[i]) - } - return clone - default: - return x - } -} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD index b641b609370..b9f40d1c1d6 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD @@ -27,8 +27,11 @@ go_test( embed = [":go_default_library"], deps = [ "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", + "//vendor/k8s.io/apiextensions-apiserver/pkg/features:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go index 644557837c6..390f75f4426 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go @@ -106,7 +106,11 @@ func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi allErrs = append(allErrs, ValidateCustomResourceDefinitionNames(&spec.Names, fldPath.Child("names"))...) if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceValidation) { - allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(spec.Validation, fldPath.Child("validation"))...) + statusEnabled := false + if spec.Subresources != nil && spec.Subresources.Status != nil { + statusEnabled = true + } + allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(spec.Validation, statusEnabled, fldPath.Child("validation"))...) } else if spec.Validation != nil { allErrs = append(allErrs, field.Forbidden(fldPath.Child("validation"), "disabled by feature-gate CustomResourceValidation")) } @@ -187,7 +191,7 @@ type specStandardValidator interface { } // ValidateCustomResourceDefinitionValidation statically validates -func ValidateCustomResourceDefinitionValidation(customResourceValidation *apiextensions.CustomResourceValidation, fldPath *field.Path) field.ErrorList { +func ValidateCustomResourceDefinitionValidation(customResourceValidation *apiextensions.CustomResourceValidation, statusSubresourceEnabled bool, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if customResourceValidation == nil { @@ -195,21 +199,19 @@ func ValidateCustomResourceDefinitionValidation(customResourceValidation *apiext } if schema := customResourceValidation.OpenAPIV3Schema; schema != nil { - // if subresources are enabled, only properties is allowed inside the root schema - if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) { + // if subresources are enabled, only "properties" and "required" is allowed inside the root schema + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && statusSubresourceEnabled { v := reflect.ValueOf(schema).Elem() - fieldsPresent := 0 - for i := 0; i < v.NumField(); i++ { - field := v.Field(i).Interface() - if !reflect.DeepEqual(field, reflect.Zero(reflect.TypeOf(field)).Interface()) { - fieldsPresent++ + // skip zero values + if value := v.Field(i).Interface(); reflect.DeepEqual(value, reflect.Zero(reflect.TypeOf(value)).Interface()) { + continue } - } - if fieldsPresent > 1 || (fieldsPresent == 1 && v.FieldByName("Properties").IsNil()) { - allErrs = append(allErrs, field.Invalid(fldPath.Child("openAPIV3Schema"), *schema, fmt.Sprintf("if subresources for custom resources are enabled, only properties can be used at the root of the schema"))) - return allErrs + if name := v.Type().Field(i).Name; name != "Properties" && name != "Required" { + allErrs = append(allErrs, field.Invalid(fldPath.Child("openAPIV3Schema"), *schema, fmt.Sprintf(`must only have "properties" or "required" at the root if the status subresource is enabled`))) + break + } } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go index c7c9b6d494b..687add9d7b0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go @@ -19,9 +19,13 @@ package validation import ( "testing" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/apiextensions-apiserver/pkg/features" ) type validationMatch struct { @@ -517,3 +521,70 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { } } } + +func TestValidateCustomResourceDefinitionValidation(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CustomResourceSubresources, true)() + + tests := []struct { + name string + input apiextensions.CustomResourceValidation + statusEnabled bool + wantError bool + }{ + { + name: "empty", + input: apiextensions.CustomResourceValidation{}, + wantError: false, + }, + { + name: "empty with status", + input: apiextensions.CustomResourceValidation{}, + statusEnabled: true, + wantError: false, + }, + { + name: "root type without status", + input: apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + }, + }, + statusEnabled: false, + wantError: false, + }, + { + name: "root type with status", + input: apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + }, + }, + statusEnabled: true, + wantError: true, + }, + { + name: "properties and required with status", + input: apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Properties: map[string]apiextensions.JSONSchemaProps{ + "spec": {}, + "status": {}, + }, + Required: []string{"spec", "status"}, + }, + }, + statusEnabled: true, + wantError: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ValidateCustomResourceDefinitionValidation(&tt.input, tt.statusEnabled, field.NewPath("spec", "validation")) + if !tt.wantError && len(got) > 0 { + t.Errorf("Expected no error, but got: %v", got) + } else if tt.wantError && len(got) == 0 { + t.Error("Expected error, but got none") + } + }) + } +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD index 8bd53039ddd..d4f9a30c849 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD @@ -40,7 +40,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index 6803b5c7a15..f2c73601135 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -21,7 +21,6 @@ import ( "net/http" "time" - "k8s.io/apimachinery/pkg/apimachinery/registered" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -49,9 +48,8 @@ import ( ) var ( - Registry = registered.NewAPIRegistrationManager() - Scheme = runtime.NewScheme() - Codecs = serializer.NewCodecFactory(Scheme) + Scheme = runtime.NewScheme() + Codecs = serializer.NewCodecFactory(Scheme) // if you modify this, make sure you update the crEncoder unversionedVersion = schema.GroupVersion{Group: "", Version: "v1"} @@ -66,7 +64,7 @@ var ( ) func init() { - install.Install(Registry, Scheme) + install.Install(Scheme) // we need to add the options to empty v1 metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Group: "", Version: "v1"}) @@ -128,7 +126,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) } apiResourceConfig := c.GenericConfig.MergedResourceConfig - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiextensions.GroupName, Registry, Scheme, metav1.ParameterCodec, Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiextensions.GroupName, Scheme, metav1.ParameterCodec, Codecs) if apiResourceConfig.VersionEnabled(v1beta1.SchemeGroupVersion) { storage := map[string]rest.Storage{} // customresourcedefinitions diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 4c0c01ba20d..cc24f1974f4 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -385,7 +385,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource kind := schema.GroupVersionKind{Group: crd.Spec.Group, Version: crd.Spec.Version, Kind: crd.Status.AcceptedNames.Kind} typer := UnstructuredObjectTyper{ Delegate: parameterScheme, - UnstructuredTyper: discovery.NewUnstructuredObjectTyper(nil), + UnstructuredTyper: discovery.NewUnstructuredObjectTyper(), } creator := unstructuredCreator{} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/scheme/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/scheme/BUILD index cb69237265e..aca5e11470e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/scheme/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/scheme/BUILD @@ -14,7 +14,6 @@ go_library( importpath = "k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/scheme", deps = [ "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/scheme/register.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/scheme/register.go index 1e0f55f0b77..01df98a6bed 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/scheme/register.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/scheme/register.go @@ -20,7 +20,6 @@ package scheme import ( apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" - registered "k8s.io/apimachinery/pkg/apimachinery/registered" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -31,14 +30,12 @@ var Scheme = runtime.NewScheme() var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) -var Registry = registered.NewAPIRegistrationManager() - func init() { v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - Install(Registry, Scheme) + Install(Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - apiextensions.Install(registry, scheme) +func Install(scheme *runtime.Scheme) { + apiextensions.Install(scheme) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion/apiextensions_client.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion/apiextensions_client.go index d2e46ca92ea..31b34404c4e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion/apiextensions_client.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion/apiextensions_client.go @@ -66,17 +66,12 @@ func New(c rest.Interface) *ApiextensionsClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("apiextensions.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("apiextensions.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("apiextensions.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go index adc3ff98d3e..e102e213d93 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go @@ -86,7 +86,7 @@ func NewCommandStartCustomResourceDefinitionsServer(out, errOut io.Writer, stopC func (o CustomResourceDefinitionsServerOptions) Validate(args []string) error { errors := []error{} errors = append(errors, o.RecommendedOptions.Validate()...) - errors = append(errors, o.APIEnablement.Validate(apiserver.Registry)...) + errors = append(errors, o.APIEnablement.Validate(apiserver.Scheme)...) return utilerrors.NewAggregate(errors) } @@ -104,7 +104,7 @@ func (o CustomResourceDefinitionsServerOptions) Config() (*apiserver.Config, err if err := o.RecommendedOptions.ApplyTo(serverConfig, apiserver.Scheme); err != nil { return nil, err } - if err := o.APIEnablement.ApplyTo(&serverConfig.Config, apiserver.DefaultAPIResourceConfigSource(), apiserver.Registry); err != nil { + if err := o.APIEnablement.ApplyTo(&serverConfig.Config, apiserver.DefaultAPIResourceConfigSource(), apiserver.Scheme); err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/BUILD index 98184ae6b78..e993d43ba0d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/BUILD @@ -87,3 +87,10 @@ go_test( "//vendor/k8s.io/client-go/discovery:go_default_library", ], ) + +go_test( + name = "go_default_test", + srcs = ["status_strategy_test.go"], + embed = [":go_default_library"], + deps = ["//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library"], +) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd_test.go index 79fc7e83d0c..4487a3554f2 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd_test.go @@ -61,7 +61,7 @@ func newStorage(t *testing.T) (customresource.CustomResourceStorage, *etcdtestin typer := apiserver.UnstructuredObjectTyper{ Delegate: parameterScheme, - UnstructuredTyper: discovery.NewUnstructuredObjectTyper(nil), + UnstructuredTyper: discovery.NewUnstructuredObjectTyper(), } kind := schema.GroupVersionKind{Group: "mygroup.example.com", Version: "v1beta1", Kind: "Noxu"} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/status_strategy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/status_strategy.go index c98b06c4b32..1710eb2e15a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/status_strategy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/status_strategy.go @@ -33,28 +33,24 @@ func NewStatusStrategy(strategy customResourceStrategy) statusStrategy { } func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { + // update is only allowed to set status newCustomResourceObject := obj.(*unstructured.Unstructured) - oldCustomResourceObject := old.(*unstructured.Unstructured) - newCustomResource := newCustomResourceObject.UnstructuredContent() - oldCustomResource := oldCustomResourceObject.UnstructuredContent() + status, ok := newCustomResource["status"] - // update is not allowed to set spec and metadata - _, ok1 := newCustomResource["spec"] - _, ok2 := oldCustomResource["spec"] - switch { - case ok2: - newCustomResource["spec"] = oldCustomResource["spec"] - case ok1: - delete(newCustomResource, "spec") + // copy old object into new object + oldCustomResourceObject := old.(*unstructured.Unstructured) + // overridding the resourceVersion in metadata is safe here, we have already checked that + // new object and old object have the same resourceVersion. + *newCustomResourceObject = *oldCustomResourceObject.DeepCopy() + + // set status + newCustomResource = newCustomResourceObject.UnstructuredContent() + if ok { + newCustomResource["status"] = status + } else { + delete(newCustomResource, "status") } - - newCustomResourceObject.SetAnnotations(oldCustomResourceObject.GetAnnotations()) - newCustomResourceObject.SetFinalizers(oldCustomResourceObject.GetFinalizers()) - newCustomResourceObject.SetGeneration(oldCustomResourceObject.GetGeneration()) - newCustomResourceObject.SetLabels(oldCustomResourceObject.GetLabels()) - newCustomResourceObject.SetOwnerReferences(oldCustomResourceObject.GetOwnerReferences()) - newCustomResourceObject.SetSelfLink(oldCustomResourceObject.GetSelfLink()) } // ValidateUpdate is the default update validation for an end user updating status. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/status_strategy_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/status_strategy_test.go new file mode 100644 index 00000000000..8a651d12bb0 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/status_strategy_test.go @@ -0,0 +1,138 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresource + +import ( + "context" + "reflect" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestPrepareForUpdate(t *testing.T) { + strategy := statusStrategy{} + tcs := []struct { + old *unstructured.Unstructured + obj *unstructured.Unstructured + expected *unstructured.Unstructured + }{ + { + // changes to spec are ignored + old: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "new", + }, + }, + expected: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + }, + }, + }, + { + // changes to other places are also ignored + old: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "new": "new", + }, + }, + expected: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + }, + }, + }, + { + // delete status + old: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + "status": "old", + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + }, + }, + expected: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + }, + }, + }, + { + // update status + old: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + "status": "old", + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "status": "new", + }, + }, + expected: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + "status": "new", + }, + }, + }, + { + // update status and other parts + old: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + "status": "old", + }, + }, + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "new", + "new": "new", + "status": "new", + }, + }, + expected: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": "old", + "status": "new", + }, + }, + }, + } + for index, tc := range tcs { + strategy.PrepareForUpdate(context.TODO(), tc.obj, tc.old) + if !reflect.DeepEqual(tc.obj, tc.expected) { + t.Errorf("test %d failed: expected: %v, got %v", index, tc.expected, tc.obj) + } + } +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor/tableconvertor.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor/tableconvertor.go index d4ea111fa59..8dbc0e77265 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor/tableconvertor.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor/tableconvertor.go @@ -42,7 +42,7 @@ var swaggerMetadataDescriptions = metav1.ObjectMeta{}.SwaggerDoc() func New(extensions spec.Extensions) (rest.TableConvertor, error) { headers := []metav1beta1.TableColumnDefinition{ {Name: "Name", Type: "string", Format: "name", Description: swaggerMetadataDescriptions["name"]}, - {Name: "Created At", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"]}, + {Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"]}, } c := &convertor{ headers: headers, diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go index 763c0f9abf3..785ac4da531 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go @@ -22,7 +22,6 @@ import ( "sort" "strings" "testing" - "time" autoscaling "k8s.io/api/autoscaling/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -30,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/dynamic" @@ -355,9 +353,12 @@ func TestValidationSchema(t *testing.T) { // fields other than properties in root schema are not allowed noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped) + noxuDefinition.Spec.Subresources = &apiextensionsv1beta1.CustomResourceSubresources{ + Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{}, + } err = testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) if err == nil { - t.Fatalf("unexpected non-error: if subresources for custom resources are enabled, only properties can be used at the root of the schema") + t.Fatalf(`unexpected non-error, expected: must only have "properties" or "required" at the root if the status subresource is enabled`) } // make sure we are not restricting fields to properties even in subschemas @@ -372,6 +373,7 @@ func TestValidationSchema(t *testing.T) { }, }, }, + Required: []string{"spec"}, } err = testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) if err != nil { @@ -392,9 +394,8 @@ func TestValidateOnlyStatus(t *testing.T) { // UpdateStatus should validate only status // 1. create a crd with max value of .spec.num = 10 and .status.num = 10 // 2. create a cr with .spec.num = 10 and .status.num = 10 (valid) - // 3. update the crd so that max value of .spec.num = 5 and .status.num = 10 - // 4. update the status of the cr with .status.num = 5 (spec is invalid) - // validation passes becauses spec is not validated + // 3. update the spec of the cr with .spec.num = 15 (spec is invalid), expect no error + // 4. update the spec of the cr with .spec.num = 15 (spec is invalid), expect error // max value of spec.num = 10 and status.num = 10 schema := &apiextensionsv1beta1.JSONSchemaProps{ @@ -443,58 +444,31 @@ func TestValidateOnlyStatus(t *testing.T) { t.Fatalf("unable to create noxu instance: %v", err) } - gottenCRD, err := apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get("noxus.mygroup.example.com", metav1.GetOptions{}) + // update the spec with .spec.num = 15, expecting no error + err = unstructured.SetNestedField(createdNoxuInstance.Object, int64(15), "spec", "num") if err != nil { - t.Fatal(err) + t.Fatalf("unexpected error setting .spec.num: %v", err) } - - // update the crd so that max value of spec.num = 5 and status.num = 10 - gottenCRD.Spec.Validation.OpenAPIV3Schema = &apiextensionsv1beta1.JSONSchemaProps{ - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "spec": { - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "num": { - Type: "integer", - Maximum: float64Ptr(5), - }, - }, - }, - "status": { - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "num": { - Type: "integer", - Maximum: float64Ptr(10), - }, - }, - }, - }, - } - - if _, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(gottenCRD); err != nil { - t.Fatal(err) - } - - // update the status with .status.num = 5 - err = unstructured.SetNestedField(createdNoxuInstance.Object, int64(5), "status", "num") + createdNoxuInstance, err = noxuStatusResourceClient.Update(createdNoxuInstance) if err != nil { - t.Fatalf("unexpected error: %v", err) + t.Errorf("unexpected error: %v", err) } - // cr is updated even though spec is invalid - err = wait.Poll(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { - _, err := noxuStatusResourceClient.Update(createdNoxuInstance) - if statusError, isStatus := err.(*apierrors.StatusError); isStatus { - if strings.Contains(statusError.Error(), "is invalid") { - return false, nil - } - } - if err != nil { - return false, err - } - return true, nil - }) + // update with .status.num = 15, expecting an error + err = unstructured.SetNestedField(createdNoxuInstance.Object, int64(15), "status", "num") if err != nil { - t.Fatal(err) + t.Fatalf("unexpected error setting .status.num: %v", err) + } + createdNoxuInstance, err = noxuStatusResourceClient.Update(createdNoxuInstance) + if err == nil { + t.Fatal("expected error, but got none") + } + statusError, isStatus := err.(*apierrors.StatusError) + if !isStatus || statusError == nil { + t.Fatalf("expected status error, got %T: %v", err, err) + } + if !strings.Contains(statusError.Error(), "Invalid value") { + t.Fatalf("expected 'Invalid value' in error, got: %v", err) } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/BUILD index 8c12d58d30d..6ea1e1ec92b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/BUILD @@ -31,6 +31,7 @@ go_library( "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/scale:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/resources.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/resources.go index bb7060e38f8..1f4a952b332 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/resources.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/resources.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" "k8s.io/client-go/scale" ) @@ -245,6 +246,7 @@ func checkForWatchCachePrimed(crd *apiextensionsv1beta1.CustomResourceDefinition "gamma": "bar", "delta": "hello", "epsilon": "foobar", + "spec": map[string]interface{}{}, }, } if _, err := resourceClient.Create(instance); err != nil { @@ -273,6 +275,25 @@ func checkForWatchCachePrimed(crd *apiextensionsv1beta1.CustomResourceDefinition } } +// UpdateCustomResourceDefinition updates a CRD, retrying up to 5 times on version conflict errors. +func UpdateCustomResourceDefinition(client clientset.Interface, name string, update func(*apiextensionsv1beta1.CustomResourceDefinition)) (*apiextensionsv1beta1.CustomResourceDefinition, error) { + for i := 0; i < 5; i++ { + crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get CustomResourceDefinition %q: %v", name, err) + } + update(crd) + crd, err = client.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd) + if err == nil { + return crd, nil + } + if !errors.IsConflict(err) { + return nil, fmt.Errorf("failed to update CustomResourceDefinition %q: %v", name, err) + } + } + return nil, fmt.Errorf("too many retries after conflicts updating CustomResourceDefinition %q", name) +} + func DeleteCustomResourceDefinition(crd *apiextensionsv1beta1.CustomResourceDefinition, apiExtensionsClient clientset.Interface) error { if err := apiExtensionsClient.Apiextensions().CustomResourceDefinitions().Delete(crd.Name, nil); err != nil { return err @@ -310,7 +331,7 @@ func CreateNewScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, co return nil, err } - resources := []*discovery.APIGroupResources{ + resources := []*restmapper.APIGroupResources{ { Group: metav1.APIGroup{ Name: crd.Spec.Group, @@ -325,7 +346,7 @@ func CreateNewScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, co }, } - restMapper := discovery.NewRESTMapper(resources) + restMapper := restmapper.NewDiscoveryRESTMapper(resources) resolver := scale.NewDiscoveryScaleKindResolver(discoveryClient) return scale.NewForConfig(config, restMapper, dynamic.LegacyAPIPathResolverFunc, resolver) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go index d8711e0031f..cac4a9d066d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/testserver/start.go @@ -64,7 +64,7 @@ func DefaultServerConfig() (*extensionsapiserver.Config, error) { if err := options.RecommendedOptions.ApplyTo(genericConfig, nil); err != nil { return nil, err } - if err := options.APIEnablement.ApplyTo(&genericConfig.Config, extensionsapiserver.DefaultAPIResourceConfigSource(), extensionsapiserver.Registry); err != nil { + if err := options.APIEnablement.ApplyTo(&genericConfig.Config, extensionsapiserver.DefaultAPIResourceConfigSource(), extensionsapiserver.Scheme); err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go index aa5984aa2a3..bf85a41b98d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go @@ -348,14 +348,11 @@ func TestCRValidationOnCRDUpdate(t *testing.T) { t.Fatalf("unexpected non-error: CR should be rejected") } - gottenCRD, err := apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get("noxus.mygroup.example.com", metav1.GetOptions{}) - if err != nil { - t.Fatal(err) - } - // update the CRD to a less stricter schema - gottenCRD.Spec.Validation.OpenAPIV3Schema.Required = []string{"alpha", "beta"} - if _, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(gottenCRD); err != nil { + _, err = testserver.UpdateCustomResourceDefinition(apiExtensionClient, "noxus.mygroup.example.com", func(crd *apiextensionsv1beta1.CustomResourceDefinition) { + crd.Spec.Validation.OpenAPIV3Schema.Required = []string{"alpha", "beta"} + }) + if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apimachinery/Godeps/Godeps.json b/staging/src/k8s.io/apimachinery/Godeps/Godeps.json index 404a6cb6e26..0607ae6e038 100644 --- a/staging/src/k8s.io/apimachinery/Godeps/Godeps.json +++ b/staging/src/k8s.io/apimachinery/Godeps/Godeps.json @@ -180,7 +180,7 @@ }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" } ] } diff --git a/staging/src/k8s.io/apimachinery/pkg/api/meta/testrestmapper/BUILD b/staging/src/k8s.io/apimachinery/pkg/api/meta/testrestmapper/BUILD index 094b209b68e..5380d155b10 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/meta/testrestmapper/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/api/meta/testrestmapper/BUILD @@ -7,8 +7,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/staging/src/k8s.io/apimachinery/pkg/api/meta/testrestmapper/test_restmapper.go b/staging/src/k8s.io/apimachinery/pkg/api/meta/testrestmapper/test_restmapper.go index 5c764528fce..a08b42b826e 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/meta/testrestmapper/test_restmapper.go +++ b/staging/src/k8s.io/apimachinery/pkg/api/meta/testrestmapper/test_restmapper.go @@ -18,8 +18,6 @@ package testrestmapper import ( "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/apimachinery" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" @@ -31,16 +29,13 @@ import ( // all other groups alphabetical. // TODO callers of this method should be updated to build their own specific restmapper based on their scheme for their tests // TODO the things being tested are related to whether various cases are handled, not tied to the particular types being checked. -func TestOnlyStaticRESTMapper(m *registered.APIRegistrationManager, scheme *runtime.Scheme, versionPatterns ...schema.GroupVersion) meta.RESTMapper { +func TestOnlyStaticRESTMapper(scheme *runtime.Scheme, versionPatterns ...schema.GroupVersion) meta.RESTMapper { unionMapper := meta.MultiRESTMapper{} unionedGroups := sets.NewString() - for _, enabledVersion := range m.RegisteredGroupVersions() { + for _, enabledVersion := range scheme.PrioritizedVersionsAllGroups() { if !unionedGroups.Has(enabledVersion.Group) { unionedGroups.Insert(enabledVersion.Group) - groupMeta := m.GroupOrDie(enabledVersion.Group) - if groupMeta != nil { - unionMapper = append(unionMapper, newRESTMapper(scheme, groupMeta)) - } + unionMapper = append(unionMapper, newRESTMapper(enabledVersion.Group, scheme)) } } @@ -56,17 +51,17 @@ func TestOnlyStaticRESTMapper(m *registered.APIRegistrationManager, scheme *runt } prioritizedGroups := []string{"", "extensions", "metrics"} - resourcePriority, kindPriority := prioritiesForGroups(m, prioritizedGroups...) + resourcePriority, kindPriority := prioritiesForGroups(scheme, prioritizedGroups...) prioritizedGroupsSet := sets.NewString(prioritizedGroups...) remainingGroups := sets.String{} - for _, enabledVersion := range m.RegisteredGroupVersions() { + for _, enabledVersion := range scheme.PrioritizedVersionsAllGroups() { if !prioritizedGroupsSet.Has(enabledVersion.Group) { remainingGroups.Insert(enabledVersion.Group) } } - remainingResourcePriority, remainingKindPriority := prioritiesForGroups(m, remainingGroups.List()...) + remainingResourcePriority, remainingKindPriority := prioritiesForGroups(scheme, remainingGroups.List()...) resourcePriority = append(resourcePriority, remainingResourcePriority...) kindPriority = append(kindPriority, remainingKindPriority...) @@ -75,12 +70,12 @@ func TestOnlyStaticRESTMapper(m *registered.APIRegistrationManager, scheme *runt // prioritiesForGroups returns the resource and kind priorities for a PriorityRESTMapper, preferring the preferred version of each group first, // then any non-preferred version of the group second. -func prioritiesForGroups(m *registered.APIRegistrationManager, groups ...string) ([]schema.GroupVersionResource, []schema.GroupVersionKind) { +func prioritiesForGroups(scheme *runtime.Scheme, groups ...string) ([]schema.GroupVersionResource, []schema.GroupVersionKind) { resourcePriority := []schema.GroupVersionResource{} kindPriority := []schema.GroupVersionKind{} for _, group := range groups { - availableVersions := m.RegisteredVersionsForGroup(group) + availableVersions := scheme.PrioritizedVersionsForGroup(group) if len(availableVersions) > 0 { resourcePriority = append(resourcePriority, availableVersions[0].WithResource(meta.AnyResource)) kindPriority = append(kindPriority, availableVersions[0].WithKind(meta.AnyKind)) @@ -94,9 +89,9 @@ func prioritiesForGroups(m *registered.APIRegistrationManager, groups ...string) return resourcePriority, kindPriority } -func newRESTMapper(scheme *runtime.Scheme, groupMeta *apimachinery.GroupMeta) meta.RESTMapper { - mapper := meta.NewDefaultRESTMapper(groupMeta.GroupVersions) - for _, gv := range groupMeta.GroupVersions { +func newRESTMapper(group string, scheme *runtime.Scheme) meta.RESTMapper { + mapper := meta.NewDefaultRESTMapper(scheme.PrioritizedVersionsForGroup(group)) + for _, gv := range scheme.PrioritizedVersionsForGroup(group) { for kind := range scheme.KnownTypes(gv) { if ignoredKinds.Has(kind) { continue diff --git a/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/BUILD b/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/BUILD index 3c0552153c6..9c201c91fd2 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/BUILD @@ -18,7 +18,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/testing:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/testing/fuzzer:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/fuzzer:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/roundtrip.go b/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/roundtrip.go index 13ffd60c300..f8b12aa859a 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/roundtrip.go +++ b/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/roundtrip.go @@ -33,7 +33,6 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" apitesting "k8s.io/apimachinery/pkg/api/testing" "k8s.io/apimachinery/pkg/api/testing/fuzzer" - "k8s.io/apimachinery/pkg/apimachinery/registered" metafuzzer "k8s.io/apimachinery/pkg/apis/meta/fuzzer" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -44,14 +43,13 @@ import ( "k8s.io/apimachinery/pkg/util/sets" ) -type InstallFunc func(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) +type InstallFunc func(scheme *runtime.Scheme) // RoundTripTestForAPIGroup is convenient to call from your install package to make sure that a "bare" install of your group provides // enough information to round trip func RoundTripTestForAPIGroup(t *testing.T, installFn InstallFunc, fuzzingFuncs fuzzer.FuzzerFuncs) { - registry := registered.NewAPIRegistrationManager() scheme := runtime.NewScheme() - installFn(registry, scheme) + installFn(scheme) RoundTripTestForScheme(t, scheme, fuzzingFuncs) } @@ -70,9 +68,8 @@ func RoundTripTestForScheme(t *testing.T, scheme *runtime.Scheme, fuzzingFuncs f // RoundTripProtobufTestForAPIGroup is convenient to call from your install package to make sure that a "bare" install of your group provides // enough information to round trip func RoundTripProtobufTestForAPIGroup(t *testing.T, installFn InstallFunc, fuzzingFuncs fuzzer.FuzzerFuncs) { - registry := registered.NewAPIRegistrationManager() scheme := runtime.NewScheme() - installFn(registry, scheme) + installFn(scheme) RoundTripProtobufTestForScheme(t, scheme, fuzzingFuncs) } diff --git a/staging/src/k8s.io/apimachinery/pkg/apimachinery/BUILD b/staging/src/k8s.io/apimachinery/pkg/apimachinery/BUILD deleted file mode 100644 index 885d7d5f87b..00000000000 --- a/staging/src/k8s.io/apimachinery/pkg/apimachinery/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = [ - "doc.go", - "types.go", - ], - importpath = "k8s.io/apimachinery/pkg/apimachinery", - deps = [ - "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//staging/src/k8s.io/apimachinery/pkg/apimachinery/announced:all-srcs", - "//staging/src/k8s.io/apimachinery/pkg/apimachinery/registered:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/staging/src/k8s.io/apimachinery/pkg/apimachinery/announced/BUILD b/staging/src/k8s.io/apimachinery/pkg/apimachinery/announced/BUILD deleted file mode 100644 index 3c82717283a..00000000000 --- a/staging/src/k8s.io/apimachinery/pkg/apimachinery/announced/BUILD +++ /dev/null @@ -1,33 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = ["group_factory.go"], - importpath = "k8s.io/apimachinery/pkg/apimachinery/announced", - deps = [ - "//vendor/github.com/golang/glog:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/staging/src/k8s.io/apimachinery/pkg/apimachinery/announced/group_factory.go b/staging/src/k8s.io/apimachinery/pkg/apimachinery/announced/group_factory.go deleted file mode 100644 index bffb10d3c6a..00000000000 --- a/staging/src/k8s.io/apimachinery/pkg/apimachinery/announced/group_factory.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package announced - -import ( - "fmt" - - "github.com/golang/glog" - - "k8s.io/apimachinery/pkg/apimachinery" - "k8s.io/apimachinery/pkg/apimachinery/registered" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/sets" -) - -type SchemeFunc func(*runtime.Scheme) error -type VersionToSchemeFunc map[string]SchemeFunc - -// GroupVersionFactoryArgs contains all the per-version parts of a GroupMetaFactory. -type GroupVersionFactoryArgs struct { - GroupName string - VersionName string - - AddToScheme SchemeFunc -} - -// GroupMetaFactoryArgs contains the group-level args of a GroupMetaFactory. -type GroupMetaFactoryArgs struct { - // GroupName is the name of the API-Group - // - // example: 'servicecatalog.k8s.io' - GroupName string - VersionPreferenceOrder []string - - // May be nil if there are no internal objects. - AddInternalObjectsToScheme SchemeFunc -} - -// NewGroupMetaFactory builds the args for you. This is for if you're -// constructing a factory all at once and not using the registry. -func NewGroupMetaFactory(groupArgs *GroupMetaFactoryArgs, versions VersionToSchemeFunc) *GroupMetaFactory { - gmf := &GroupMetaFactory{ - GroupArgs: groupArgs, - VersionArgs: map[string]*GroupVersionFactoryArgs{}, - } - for v, f := range versions { - gmf.VersionArgs[v] = &GroupVersionFactoryArgs{ - GroupName: groupArgs.GroupName, - VersionName: v, - AddToScheme: f, - } - } - return gmf -} - -// GroupMetaFactory has the logic for actually assembling and registering a group. -// -// Note that GroupMetaFactory actually does construct GroupMeta objects, but -// currently it does so in a way that's very entangled with an -// APIRegistrationManager. It's a TODO item to cleanly separate that interface. -type GroupMetaFactory struct { - GroupArgs *GroupMetaFactoryArgs - // map of version name to version factory - VersionArgs map[string]*GroupVersionFactoryArgs - - // assembled by Register() - prioritizedVersionList []schema.GroupVersion -} - -// Register constructs the finalized prioritized version list and sanity checks -// the registered group & versions. Then it calls register. -func (gmf *GroupMetaFactory) Register(m *registered.APIRegistrationManager, scheme *runtime.Scheme) error { - if gmf.GroupArgs == nil { - return fmt.Errorf("partially registered groups are not allowed, only got versions: %#v", gmf.VersionArgs) - } - if len(gmf.VersionArgs) == 0 { - return fmt.Errorf("group %v registered but no versions registered", gmf.GroupArgs.GroupName) - } - if m.IsRegistered(gmf.GroupArgs.GroupName) { - return fmt.Errorf("the group %q has already been registered.", gmf.GroupArgs.GroupName) - } - - pvSet := sets.NewString(gmf.GroupArgs.VersionPreferenceOrder...) - if pvSet.Len() != len(gmf.GroupArgs.VersionPreferenceOrder) { - return fmt.Errorf("preference order for group %v has duplicates: %v", gmf.GroupArgs.GroupName, gmf.GroupArgs.VersionPreferenceOrder) - } - prioritizedVersions := []schema.GroupVersion{} - for _, v := range gmf.GroupArgs.VersionPreferenceOrder { - prioritizedVersions = append( - prioritizedVersions, - schema.GroupVersion{ - Group: gmf.GroupArgs.GroupName, - Version: v, - }, - ) - } - - // Go through versions that weren't explicitly prioritized. - unprioritizedVersions := []schema.GroupVersion{} - for _, v := range gmf.VersionArgs { - if v.GroupName != gmf.GroupArgs.GroupName { - return fmt.Errorf("found %v/%v in group %v?", v.GroupName, v.VersionName, gmf.GroupArgs.GroupName) - } - if pvSet.Has(v.VersionName) { - pvSet.Delete(v.VersionName) - continue - } - unprioritizedVersions = append(unprioritizedVersions, schema.GroupVersion{Group: v.GroupName, Version: v.VersionName}) - } - if len(unprioritizedVersions) > 1 { - glog.Warningf("group %v has multiple unprioritized versions: %#v. They will have an arbitrary preference order!", gmf.GroupArgs.GroupName, unprioritizedVersions) - } - if pvSet.Len() != 0 { - return fmt.Errorf("group %v has versions in the priority list that were never registered: %s", gmf.GroupArgs.GroupName, pvSet) - } - prioritizedVersions = append(prioritizedVersions, unprioritizedVersions...) - m.RegisterVersions(prioritizedVersions) - gmf.prioritizedVersionList = prioritizedVersions - - externalVersions := []schema.GroupVersion{} - for _, v := range gmf.prioritizedVersionList { - externalVersions = append(externalVersions, v) - gmf.VersionArgs[v.Version].AddToScheme(scheme) - } - if len(externalVersions) == 0 { - glog.V(4).Infof("No version is registered for group %v", gmf.GroupArgs.GroupName) - return nil - } - - if gmf.GroupArgs.AddInternalObjectsToScheme != nil { - gmf.GroupArgs.AddInternalObjectsToScheme(scheme) - } - - groupMeta := &apimachinery.GroupMeta{ - GroupVersions: externalVersions, - } - - if err := m.RegisterGroup(*groupMeta); err != nil { - return err - } - return nil -} diff --git a/staging/src/k8s.io/apimachinery/pkg/apimachinery/doc.go b/staging/src/k8s.io/apimachinery/pkg/apimachinery/doc.go deleted file mode 100644 index b238454b261..00000000000 --- a/staging/src/k8s.io/apimachinery/pkg/apimachinery/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package apimachinery contains the generic API machinery code that -// is common to both server and clients. -// This package should never import specific API objects. -package apimachinery // import "k8s.io/apimachinery/pkg/apimachinery" diff --git a/staging/src/k8s.io/apimachinery/pkg/apimachinery/registered/BUILD b/staging/src/k8s.io/apimachinery/pkg/apimachinery/registered/BUILD deleted file mode 100644 index 93cd587da8e..00000000000 --- a/staging/src/k8s.io/apimachinery/pkg/apimachinery/registered/BUILD +++ /dev/null @@ -1,40 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["registered_test.go"], - embed = [":go_default_library"], - deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = ["registered.go"], - importpath = "k8s.io/apimachinery/pkg/apimachinery/registered", - deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/staging/src/k8s.io/apimachinery/pkg/apimachinery/registered/registered.go b/staging/src/k8s.io/apimachinery/pkg/apimachinery/registered/registered.go deleted file mode 100644 index dd6574acd48..00000000000 --- a/staging/src/k8s.io/apimachinery/pkg/apimachinery/registered/registered.go +++ /dev/null @@ -1,153 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package to keep track of API Versions that can be registered and are enabled in a Scheme. -package registered - -import ( - "fmt" - "sort" - "strings" - - "k8s.io/apimachinery/pkg/apimachinery" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// APIRegistrationManager provides the concept of what API groups are enabled. -// -// TODO: currently, it also provides a "registered" concept. But it's wrong to -// have both concepts in the same object. Therefore the "announced" package is -// going to take over the registered concept. After all the install packages -// are switched to using the announce package instead of this package, then we -// can combine the registered/enabled concepts in this object. Simplifying this -// isn't easy right now because there are so many callers of this package. -type APIRegistrationManager struct { - // registeredGroupVersions stores all API group versions for which RegisterGroup is called. - registeredVersions map[schema.GroupVersion]struct{} - - // map of group meta for all groups. - groupMetaMap map[string]*apimachinery.GroupMeta -} - -// NewAPIRegistrationManager constructs a new manager. -func NewAPIRegistrationManager() *APIRegistrationManager { - m := &APIRegistrationManager{ - registeredVersions: map[schema.GroupVersion]struct{}{}, - groupMetaMap: map[string]*apimachinery.GroupMeta{}, - } - - return m -} - -// RegisterVersions adds the given group versions to the list of registered group versions. -func (m *APIRegistrationManager) RegisterVersions(availableVersions []schema.GroupVersion) { - for _, v := range availableVersions { - m.registeredVersions[v] = struct{}{} - } -} - -// RegisterGroup adds the given group to the list of registered groups. -func (m *APIRegistrationManager) RegisterGroup(groupMeta apimachinery.GroupMeta) error { - groupName := groupMeta.GroupVersions[0].Group - if _, found := m.groupMetaMap[groupName]; found { - return fmt.Errorf("group %q is already registered in groupsMap: %v", groupName, m.groupMetaMap) - } - m.groupMetaMap[groupName] = &groupMeta - return nil -} - -// Group returns the metadata of a group if the group is registered, otherwise -// an error is returned. -func (m *APIRegistrationManager) Group(group string) (*apimachinery.GroupMeta, error) { - groupMeta, found := m.groupMetaMap[group] - if !found { - return nil, fmt.Errorf("group %v has not been registered", group) - } - groupMetaCopy := *groupMeta - return &groupMetaCopy, nil -} - -// IsRegistered takes a string and determines if it's one of the registered groups -func (m *APIRegistrationManager) IsRegistered(group string) bool { - _, found := m.groupMetaMap[group] - return found -} - -// IsRegisteredVersion returns if a version is registered. -func (m *APIRegistrationManager) IsRegisteredVersion(v schema.GroupVersion) bool { - _, found := m.registeredVersions[v] - return found -} - -// RegisteredGroupVersions returns all registered group versions. Groups are randomly ordered, but versions within groups -// are priority order from best to worst -func (m *APIRegistrationManager) RegisteredGroupVersions() []schema.GroupVersion { - ret := []schema.GroupVersion{} - for _, groupMeta := range m.groupMetaMap { - for _, version := range groupMeta.GroupVersions { - if m.IsRegisteredVersion(version) { - ret = append(ret, version) - } - } - } - return ret -} - -// RegisteredVersionsForGroup returns all enabled versions for a group in order of best to worst -func (m *APIRegistrationManager) RegisteredVersionsForGroup(group string) []schema.GroupVersion { - groupMeta, ok := m.groupMetaMap[group] - if !ok { - return []schema.GroupVersion{} - } - - ret := []schema.GroupVersion{} - for _, version := range groupMeta.GroupVersions { - if m.IsRegisteredVersion(version) { - ret = append(ret, version) - } - } - return ret -} - -// TODO: This is an expedient function, because we don't check if a Group is -// supported throughout the code base. We will abandon this function and -// checking the error returned by the Group() function. -func (m *APIRegistrationManager) GroupOrDie(group string) *apimachinery.GroupMeta { - groupMeta, found := m.groupMetaMap[group] - if !found { - if group == "" { - panic("The legacy v1 API is not registered.") - } else { - panic(fmt.Sprintf("Group %s is not registered.", group)) - } - } - groupMetaCopy := *groupMeta - return &groupMetaCopy -} - -// AllPreferredGroupVersions returns the preferred versions of all registered -// groups in the form of "group1/version1,group2/version2,..." -func (m *APIRegistrationManager) AllPreferredGroupVersions() string { - if len(m.groupMetaMap) == 0 { - return "" - } - var defaults []string - for _, groupMeta := range m.groupMetaMap { - defaults = append(defaults, groupMeta.GroupVersions[0].String()) - } - sort.Strings(defaults) - return strings.Join(defaults, ",") -} diff --git a/staging/src/k8s.io/apimachinery/pkg/apimachinery/registered/registered_test.go b/staging/src/k8s.io/apimachinery/pkg/apimachinery/registered/registered_test.go deleted file mode 100644 index a986b2c04a3..00000000000 --- a/staging/src/k8s.io/apimachinery/pkg/apimachinery/registered/registered_test.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package registered - -import ( - "testing" - - "k8s.io/apimachinery/pkg/apimachinery" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -func TestAllPreferredGroupVersions(t *testing.T) { - testCases := []struct { - groupMetas []apimachinery.GroupMeta - expect string - }{ - { - groupMetas: []apimachinery.GroupMeta{ - { - GroupVersions: []schema.GroupVersion{{Group: "group1", Version: "v1"}}, - }, - { - GroupVersions: []schema.GroupVersion{{Group: "group2", Version: "v2"}}, - }, - { - GroupVersions: []schema.GroupVersion{{Group: "", Version: "v1"}}, - }, - }, - expect: "group1/v1,group2/v2,v1", - }, - { - groupMetas: []apimachinery.GroupMeta{ - { - GroupVersions: []schema.GroupVersion{{Group: "", Version: "v1"}}, - }, - }, - expect: "v1", - }, - { - groupMetas: []apimachinery.GroupMeta{}, - expect: "", - }, - } - for _, testCase := range testCases { - m := NewAPIRegistrationManager() - for _, groupMeta := range testCase.groupMetas { - m.RegisterGroup(groupMeta) - } - output := m.AllPreferredGroupVersions() - if testCase.expect != output { - t.Errorf("Error. expect: %s, got: %s", testCase.expect, output) - } - } -} diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/fuzzer/fuzzer.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/fuzzer/fuzzer.go index 54fab9c02ea..3d49a5d5cf4 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/fuzzer/fuzzer.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/fuzzer/fuzzer.go @@ -268,7 +268,7 @@ func v1alpha1FuzzerFuncs(codecs runtimeserializer.CodecFactory) []interface{} { case 0: r.Cells[i] = c.RandString() case 1: - r.Cells[i] = c.Uint64() + r.Cells[i] = c.Int63() case 2: r.Cells[i] = c.RandBool() case 3: @@ -280,7 +280,7 @@ func v1alpha1FuzzerFuncs(codecs runtimeserializer.CodecFactory) []interface{} { case 4: x := make([]interface{}, c.Intn(10)) for i := range x { - x[i] = c.Uint64() + x[i] = c.Int63() } r.Cells[i] = x default: diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1/deepcopy.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1/deepcopy.go index 2dd440bb729..3b2bedd9233 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1/deepcopy.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1/deepcopy.go @@ -16,6 +16,8 @@ limitations under the License. package v1beta1 +import "k8s.io/apimachinery/pkg/runtime" + func (in *TableRow) DeepCopy() *TableRow { if in == nil { return nil @@ -26,7 +28,7 @@ func (in *TableRow) DeepCopy() *TableRow { if in.Cells != nil { out.Cells = make([]interface{}, len(in.Cells)) for i := range in.Cells { - out.Cells[i] = deepCopyJSON(in.Cells[i]) + out.Cells[i] = runtime.DeepCopyJSONValue(in.Cells[i]) } } @@ -40,22 +42,3 @@ func (in *TableRow) DeepCopy() *TableRow { in.Object.DeepCopyInto(&out.Object) return out } - -func deepCopyJSON(x interface{}) interface{} { - switch x := x.(type) { - case map[string]interface{}: - clone := make(map[string]interface{}, len(x)) - for k, v := range x { - clone[k] = deepCopyJSON(v) - } - return clone - case []interface{}: - clone := make([]interface{}, len(x)) - for i := range x { - clone[i] = deepCopyJSON(x[i]) - } - return clone - default: - return x - } -} diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/install/BUILD b/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/install/BUILD index 7e3e76d5d91..c5d6e0d78cc 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/install/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/install/BUILD @@ -11,11 +11,10 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/apimachinery/pkg/apis/testapigroup/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/testapigroup:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/testapigroup/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/install/install.go b/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/install/install.go index ea8d23ace6a..6fc079f51bd 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/install/install.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/apis/testapigroup" "k8s.io/apimachinery/pkg/apis/testapigroup/v1" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: testapigroup.GroupName, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: testapigroup.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1.SchemeGroupVersion.Version: v1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(testapigroup.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/BUILD b/staging/src/k8s.io/apimachinery/pkg/runtime/BUILD index 7a53fbc41d1..a2c2eb308ab 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/BUILD @@ -8,8 +8,15 @@ load( go_test( name = "go_default_test", - srcs = ["swagger_doc_generator_test.go"], + srcs = [ + "local_scheme_test.go", + "swagger_doc_generator_test.go", + ], embed = [":go_default_library"], + deps = [ + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", + ], ) go_library( @@ -44,6 +51,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/json:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go b/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go index 91920535d36..291d7a4e888 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go @@ -73,7 +73,6 @@ var ( mapStringInterfaceType = reflect.TypeOf(map[string]interface{}{}) stringType = reflect.TypeOf(string("")) int64Type = reflect.TypeOf(int64(0)) - uint64Type = reflect.TypeOf(uint64(0)) float64Type = reflect.TypeOf(float64(0)) boolType = reflect.TypeOf(bool(false)) fieldCache = newFieldsCache() @@ -438,13 +437,15 @@ func (c *unstructuredConverter) ToUnstructured(obj interface{}) (map[string]inte } // DeepCopyJSON deep copies the passed value, assuming it is a valid JSON representation i.e. only contains -// types produced by json.Unmarshal(). +// types produced by json.Unmarshal() and also int64. +// bool, int64, float64, string, []interface{}, map[string]interface{}, json.Number and nil func DeepCopyJSON(x map[string]interface{}) map[string]interface{} { return DeepCopyJSONValue(x).(map[string]interface{}) } // DeepCopyJSONValue deep copies the passed value, assuming it is a valid JSON representation i.e. only contains -// types produced by json.Unmarshal(). +// types produced by json.Unmarshal() and also int64. +// bool, int64, float64, string, []interface{}, map[string]interface{}, json.Number and nil func DeepCopyJSONValue(x interface{}) interface{} { switch x := x.(type) { case map[string]interface{}: @@ -591,10 +592,14 @@ func toUnstructured(sv, dv reflect.Value) error { dv.Set(reflect.ValueOf(sv.Int())) return nil case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - if dt.Kind() == reflect.Interface && dv.NumMethod() == 0 { - dv.Set(reflect.New(uint64Type)) + uVal := sv.Uint() + if uVal > math.MaxInt64 { + return fmt.Errorf("unsigned value %d does not fit into int64 (overflow)", uVal) } - dv.Set(reflect.ValueOf(sv.Uint())) + if dt.Kind() == reflect.Interface && dv.NumMethod() == 0 { + dv.Set(reflect.New(int64Type)) + } + dv.Set(reflect.ValueOf(int64(uVal))) return nil case reflect.Float32, reflect.Float64: if dt.Kind() == reflect.Interface && dv.NumMethod() == 0 { diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/local_scheme_test.go b/staging/src/k8s.io/apimachinery/pkg/runtime/local_scheme_test.go new file mode 100644 index 00000000000..45a0dde5e70 --- /dev/null +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/local_scheme_test.go @@ -0,0 +1,150 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtime + +import ( + "testing" + + "reflect" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/diff" +) + +func TestPreferredVersionsAllGroups(t *testing.T) { + tests := []struct { + name string + versionPriority map[string][]string + observedVersions []schema.GroupVersion + expectedPrioritized map[string][]schema.GroupVersion + expectedPreferred map[schema.GroupVersion]bool + }{ + { + name: "observedOnly", + observedVersions: []schema.GroupVersion{ + {Group: "", Version: "v3"}, + {Group: "foo", Version: "v1"}, + {Group: "foo", Version: "v2"}, + {Group: "", Version: "v1"}, + }, + expectedPrioritized: map[string][]schema.GroupVersion{ + "": { + {Group: "", Version: "v3"}, + {Group: "", Version: "v1"}, + }, + "foo": { + {Group: "foo", Version: "v1"}, + {Group: "foo", Version: "v2"}, + }, + }, + expectedPreferred: map[schema.GroupVersion]bool{ + {Group: "", Version: "v3"}: true, + {Group: "foo", Version: "v1"}: true, + }, + }, + { + name: "specifiedOnly", + versionPriority: map[string][]string{ + "": {"v3", "v1"}, + "foo": {"v1", "v2"}, + }, + expectedPrioritized: map[string][]schema.GroupVersion{ + "": { + {Group: "", Version: "v3"}, + {Group: "", Version: "v1"}, + }, + "foo": { + {Group: "foo", Version: "v1"}, + {Group: "foo", Version: "v2"}, + }, + }, + expectedPreferred: map[schema.GroupVersion]bool{ + {Group: "", Version: "v3"}: true, + {Group: "foo", Version: "v1"}: true, + }, + }, + { + name: "both", + versionPriority: map[string][]string{ + "": {"v3", "v1"}, + "foo": {"v1", "v2"}, + }, + observedVersions: []schema.GroupVersion{ + {Group: "", Version: "v1"}, + {Group: "", Version: "v3"}, + {Group: "", Version: "v4"}, + {Group: "", Version: "v5"}, + {Group: "bar", Version: "v1"}, + {Group: "bar", Version: "v2"}, + }, + expectedPrioritized: map[string][]schema.GroupVersion{ + "": { + {Group: "", Version: "v3"}, + {Group: "", Version: "v1"}, + {Group: "", Version: "v4"}, + {Group: "", Version: "v5"}, + }, + "foo": { + {Group: "foo", Version: "v1"}, + {Group: "foo", Version: "v2"}, + }, + "bar": { + {Group: "bar", Version: "v1"}, + {Group: "bar", Version: "v2"}, + }, + }, + expectedPreferred: map[schema.GroupVersion]bool{ + {Group: "", Version: "v3"}: true, + {Group: "foo", Version: "v1"}: true, + {Group: "bar", Version: "v1"}: true, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + scheme := NewScheme() + scheme.versionPriority = test.versionPriority + scheme.observedVersions = test.observedVersions + + for group, expected := range test.expectedPrioritized { + actual := scheme.PrioritizedVersionsForGroup(group) + if !reflect.DeepEqual(expected, actual) { + t.Error(diff.ObjectDiff(expected, actual)) + } + } + + prioritizedAll := scheme.PrioritizedVersionsAllGroups() + actualPrioritizedAll := map[string][]schema.GroupVersion{} + for _, actual := range prioritizedAll { + actualPrioritizedAll[actual.Group] = append(actualPrioritizedAll[actual.Group], actual) + } + if !reflect.DeepEqual(test.expectedPrioritized, actualPrioritizedAll) { + t.Error(diff.ObjectDiff(test.expectedPrioritized, actualPrioritizedAll)) + } + + preferredAll := scheme.PreferredVersionAllGroups() + actualPreferredAll := map[schema.GroupVersion]bool{} + for _, actual := range preferredAll { + actualPreferredAll[actual] = true + } + if !reflect.DeepEqual(test.expectedPreferred, actualPreferredAll) { + t.Error(diff.ObjectDiff(test.expectedPreferred, actualPreferredAll)) + } + }) + } +} diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/scheme.go b/staging/src/k8s.io/apimachinery/pkg/runtime/scheme.go index 3d94a3041d1..450011f15f5 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/scheme.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/scheme.go @@ -21,8 +21,11 @@ import ( "net/url" "reflect" + "strings" + "k8s.io/apimachinery/pkg/conversion" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" ) // Scheme defines methods for serializing and deserializing API objects, a type @@ -68,6 +71,13 @@ type Scheme struct { // converter stores all registered conversion functions. It also has // default coverting behavior. converter *conversion.Converter + + // versionPriority is a map of groups to ordered lists of versions for those groups indicating the + // default priorities of these versions as registered in the scheme + versionPriority map[string][]string + + // observedVersions keeps track of the order we've seen versions during type registration + observedVersions []schema.GroupVersion } // Function to convert a field selector to internal representation. @@ -82,6 +92,7 @@ func NewScheme() *Scheme { unversionedKinds: map[string]reflect.Type{}, fieldLabelConversionFuncs: map[string]map[string]FieldLabelConversionFunc{}, defaulterFuncs: map[reflect.Type]func(interface{}){}, + versionPriority: map[string][]string{}, } s.converter = conversion.NewConverter(s.nameFunc) @@ -141,6 +152,7 @@ func (s *Scheme) Converter() *conversion.Converter { // TODO: there is discussion about removing unversioned and replacing it with objects that are manifest into // every version with particular schemas. Resolve this method at that point. func (s *Scheme) AddUnversionedTypes(version schema.GroupVersion, types ...Object) { + s.addObservedVersion(version) s.AddKnownTypes(version, types...) for _, obj := range types { t := reflect.TypeOf(obj).Elem() @@ -158,6 +170,7 @@ func (s *Scheme) AddUnversionedTypes(version schema.GroupVersion, types ...Objec // the struct becomes the "kind" field when encoding. Version may not be empty - use the // APIVersionInternal constant if you have a type that does not have a formal version. func (s *Scheme) AddKnownTypes(gv schema.GroupVersion, types ...Object) { + s.addObservedVersion(gv) for _, obj := range types { t := reflect.TypeOf(obj) if t.Kind() != reflect.Ptr { @@ -173,6 +186,7 @@ func (s *Scheme) AddKnownTypes(gv schema.GroupVersion, types ...Object) { // your structs. Version may not be empty - use the APIVersionInternal constant if you have a // type that does not have a formal version. func (s *Scheme) AddKnownTypeWithName(gvk schema.GroupVersionKind, obj Object) { + s.addObservedVersion(gvk.GroupVersion()) t := reflect.TypeOf(obj) if len(gvk.Version) == 0 { panic(fmt.Sprintf("version is required on all types: %s %v", gvk, t)) @@ -620,3 +634,133 @@ func setTargetKind(obj Object, kind schema.GroupVersionKind) { } obj.GetObjectKind().SetGroupVersionKind(kind) } + +// SetVersionPriority allows specifying a precise order of priority. All specified versions must be in the same group, +// and the specified order overwrites any previously specified order for this group +func (s *Scheme) SetVersionPriority(versions ...schema.GroupVersion) error { + groups := sets.String{} + order := []string{} + for _, version := range versions { + if len(version.Version) == 0 || version.Version == APIVersionInternal { + return fmt.Errorf("internal versions cannot be prioritized: %v", version) + } + + groups.Insert(version.Group) + order = append(order, version.Version) + } + if len(groups) != 1 { + return fmt.Errorf("must register versions for exactly one group: %v", strings.Join(groups.List(), ", ")) + } + + s.versionPriority[groups.List()[0]] = order + return nil +} + +// PrioritizedVersionsForGroup returns versions for a single group in priority order +func (s *Scheme) PrioritizedVersionsForGroup(group string) []schema.GroupVersion { + ret := []schema.GroupVersion{} + for _, version := range s.versionPriority[group] { + ret = append(ret, schema.GroupVersion{Group: group, Version: version}) + } + for _, observedVersion := range s.observedVersions { + if observedVersion.Group != group { + continue + } + found := false + for _, existing := range ret { + if existing == observedVersion { + found = true + break + } + } + if !found { + ret = append(ret, observedVersion) + } + } + + return ret +} + +// PrioritizedVersionsAllGroups returns all known versions in their priority order. Groups are random, but +// versions for a single group are prioritized +func (s *Scheme) PrioritizedVersionsAllGroups() []schema.GroupVersion { + ret := []schema.GroupVersion{} + for group, versions := range s.versionPriority { + for _, version := range versions { + ret = append(ret, schema.GroupVersion{Group: group, Version: version}) + } + } + for _, observedVersion := range s.observedVersions { + found := false + for _, existing := range ret { + if existing == observedVersion { + found = true + break + } + } + if !found { + ret = append(ret, observedVersion) + } + } + return ret +} + +// PreferredVersionAllGroups returns the most preferred version for every group. +// group ordering is random. +func (s *Scheme) PreferredVersionAllGroups() []schema.GroupVersion { + ret := []schema.GroupVersion{} + for group, versions := range s.versionPriority { + for _, version := range versions { + ret = append(ret, schema.GroupVersion{Group: group, Version: version}) + break + } + } + for _, observedVersion := range s.observedVersions { + found := false + for _, existing := range ret { + if existing.Group == observedVersion.Group { + found = true + break + } + } + if !found { + ret = append(ret, observedVersion) + } + } + + return ret +} + +// IsGroupRegistered returns true if types for the group have been registered with the scheme +func (s *Scheme) IsGroupRegistered(group string) bool { + for _, observedVersion := range s.observedVersions { + if observedVersion.Group == group { + return true + } + } + return false +} + +// IsVersionRegistered returns true if types for the version have been registered with the scheme +func (s *Scheme) IsVersionRegistered(version schema.GroupVersion) bool { + for _, observedVersion := range s.observedVersions { + if observedVersion == version { + return true + } + } + + return false +} + +func (s *Scheme) addObservedVersion(version schema.GroupVersion) { + if len(version.Version) == 0 || version.Version == APIVersionInternal { + return + } + for _, observedVersion := range s.observedVersions { + if observedVersion == version { + return + } + } + + s.observedVersions = append(s.observedVersions, version) +} diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go index 2b795b5b84f..f7738d116cd 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go @@ -75,11 +75,6 @@ func init() { case jsoniter.NumberValue: var number json.Number iter.ReadVal(&number) - u64, err := strconv.ParseUint(string(number), 10, 64) - if err == nil { - *(*interface{})(ptr) = u64 - return - } i64, err := strconv.ParseInt(string(number), 10, 64) if err == nil { *(*interface{})(ptr) = i64 diff --git a/staging/src/k8s.io/apimachinery/pkg/util/clock/clock.go b/staging/src/k8s.io/apimachinery/pkg/util/clock/clock.go index c303a212a04..9567f90060d 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/clock/clock.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/clock/clock.go @@ -26,18 +26,12 @@ import ( type Clock interface { Now() time.Time Since(time.Time) time.Duration - After(d time.Duration) <-chan time.Time - NewTimer(d time.Duration) Timer - Sleep(d time.Duration) - Tick(d time.Duration) <-chan time.Time + After(time.Duration) <-chan time.Time + NewTimer(time.Duration) Timer + Sleep(time.Duration) + NewTicker(time.Duration) Ticker } -var ( - _ = Clock(RealClock{}) - _ = Clock(&FakeClock{}) - _ = Clock(&IntervalClock{}) -) - // RealClock really calls time.Now() type RealClock struct{} @@ -62,8 +56,10 @@ func (RealClock) NewTimer(d time.Duration) Timer { } } -func (RealClock) Tick(d time.Duration) <-chan time.Time { - return time.Tick(d) +func (RealClock) NewTicker(d time.Duration) Ticker { + return &realTicker{ + ticker: time.NewTicker(d), + } } func (RealClock) Sleep(d time.Duration) { @@ -137,7 +133,7 @@ func (f *FakeClock) NewTimer(d time.Duration) Timer { return timer } -func (f *FakeClock) Tick(d time.Duration) <-chan time.Time { +func (f *FakeClock) NewTicker(d time.Duration) Ticker { f.lock.Lock() defer f.lock.Unlock() tickTime := f.time.Add(d) @@ -149,7 +145,9 @@ func (f *FakeClock) Tick(d time.Duration) <-chan time.Time { destChan: ch, }) - return ch + return &fakeTicker{ + c: ch, + } } // Move clock by Duration, notify anyone that's called After, Tick, or NewTimer @@ -242,8 +240,8 @@ func (*IntervalClock) NewTimer(d time.Duration) Timer { // Unimplemented, will panic. // TODO: make interval clock use FakeClock so this can be implemented. -func (*IntervalClock) Tick(d time.Duration) <-chan time.Time { - panic("IntervalClock doesn't implement Tick") +func (*IntervalClock) NewTicker(d time.Duration) Ticker { + panic("IntervalClock doesn't implement NewTicker") } func (*IntervalClock) Sleep(d time.Duration) { @@ -258,11 +256,6 @@ type Timer interface { Reset(d time.Duration) bool } -var ( - _ = Timer(&realTimer{}) - _ = Timer(&fakeTimer{}) -) - // realTimer is backed by an actual time.Timer. type realTimer struct { timer *time.Timer @@ -325,3 +318,31 @@ func (f *fakeTimer) Reset(d time.Duration) bool { return active } + +type Ticker interface { + C() <-chan time.Time + Stop() +} + +type realTicker struct { + ticker *time.Ticker +} + +func (t *realTicker) C() <-chan time.Time { + return t.ticker.C +} + +func (t *realTicker) Stop() { + t.ticker.Stop() +} + +type fakeTicker struct { + c <-chan time.Time +} + +func (t *fakeTicker) C() <-chan time.Time { + return t.c +} + +func (t *fakeTicker) Stop() { +} diff --git a/staging/src/k8s.io/apimachinery/pkg/util/clock/clock_test.go b/staging/src/k8s.io/apimachinery/pkg/util/clock/clock_test.go index 27d34605f50..c7b371fc6d1 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/clock/clock_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/clock/clock_test.go @@ -21,6 +21,18 @@ import ( "time" ) +var ( + _ = Clock(RealClock{}) + _ = Clock(&FakeClock{}) + _ = Clock(&IntervalClock{}) + + _ = Timer(&realTimer{}) + _ = Timer(&fakeTimer{}) + + _ = Ticker(&realTicker{}) + _ = Ticker(&fakeTicker{}) +) + func TestFakeClock(t *testing.T) { startTime := time.Now() tc := NewFakeClock(startTime) @@ -110,13 +122,13 @@ func TestFakeTick(t *testing.T) { if tc.HasWaiters() { t.Errorf("unexpected waiter?") } - oneSec := tc.Tick(time.Second) + oneSec := tc.NewTicker(time.Second).C() if !tc.HasWaiters() { t.Errorf("unexpected lack of waiter?") } - oneOhOneSec := tc.Tick(time.Second + time.Millisecond) - twoSec := tc.Tick(2 * time.Second) + oneOhOneSec := tc.NewTicker(time.Second + time.Millisecond).C() + twoSec := tc.NewTicker(2 * time.Second).C() select { case <-oneSec: t.Errorf("unexpected channel read") diff --git a/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch.go b/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch.go index e81e4f23a80..82e4b4b5704 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch.go @@ -116,10 +116,26 @@ func keepOrDeleteNullInObj(m map[string]interface{}, keepNull bool) (map[string] case val != nil: switch typedVal := val.(type) { case map[string]interface{}: - filteredMap[key], err = keepOrDeleteNullInObj(typedVal, keepNull) + // Explicitly-set empty maps are treated as values instead of empty patches + if len(typedVal) == 0 { + if !keepNull { + filteredMap[key] = typedVal + } + continue + } + + var filteredSubMap map[string]interface{} + filteredSubMap, err = keepOrDeleteNullInObj(typedVal, keepNull) if err != nil { return nil, err } + + // If the returned filtered submap was empty, this is an empty patch for the entire subdict, so the key + // should not be set + if len(filteredSubMap) != 0 { + filteredMap[key] = filteredSubMap + } + case []interface{}, string, float64, bool, int, int64, nil: // Lists are always replaced in Json, no need to check each entry in the list. if !keepNull { diff --git a/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch_test.go b/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch_test.go index f462bf915f6..9672deaad61 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/jsonmergepatch/patch_test.go @@ -62,12 +62,12 @@ testCases: expectedWithoutNull: {} - description: simple map with all non-nil values originalObj: - nonNilKey: foo - nonNilKey: bar + nonNilKey1: foo + nonNilKey2: bar expectedWithNull: {} expectedWithoutNull: - nonNilKey: foo - nonNilKey: bar + nonNilKey1: foo + nonNilKey2: bar - description: nested map originalObj: mapKey: @@ -88,19 +88,52 @@ testCases: mapKey: nilKey1: null nilKey2: null - expectedWithoutNull: - mapKey: {} + expectedWithoutNull: {} - description: nested map that all subkeys are non-nil originalObj: mapKey: - nonNilKey: foo - nonNilKey: bar - expectedWithNull: - mapKey: {} + nonNilKey1: foo + nonNilKey2: bar + expectedWithNull: {} expectedWithoutNull: mapKey: - nonNilKey: foo - nonNilKey: bar + nonNilKey1: foo + nonNilKey2: bar + - description: explicitly empty map as value + originalObj: + mapKey: {} + expectedWithNull: {} + expectedWithoutNull: + mapKey: {} + - description: explicitly empty nested map + originalObj: + mapKey: + nonNilKey: {} + expectedWithNull: {} + expectedWithoutNull: + mapKey: + nonNilKey: {} + - description: multiple expliclty empty nested maps + originalObj: + mapKey: + nonNilKey1: {} + nonNilKey2: {} + expectedWithNull: {} + expectedWithoutNull: + mapKey: + nonNilKey1: {} + nonNilKey2: {} + - description: nested map with non-null value as empty map + originalObj: + mapKey: + nonNilKey: {} + nilKey: null + expectedWithNull: + mapKey: + nilKey: null + expectedWithoutNull: + mapKey: + nonNilKey: {} - description: empty list originalObj: listKey: [] diff --git a/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go b/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go index c3cb9c03798..da32fe12f33 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/runtime/runtime.go @@ -160,3 +160,10 @@ func RecoverFromPanic(err *error) { callers) } } + +// Must panics on non-nil errors. Useful to handling programmer level errors. +func Must(err error) { + if err != nil { + panic(err) + } +} diff --git a/staging/src/k8s.io/apiserver/Godeps/Godeps.json b/staging/src/k8s.io/apiserver/Godeps/Godeps.json index 0b581a1f2b8..96673818710 100644 --- a/staging/src/k8s.io/apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/apiserver/Godeps/Godeps.json @@ -970,18 +970,6 @@ "ImportPath": "k8s.io/apimachinery/pkg/api/validation/path", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/announced", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apimachinery/pkg/apis/meta/fuzzer", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1736,23 +1724,23 @@ }, { "ImportPath": "k8s.io/kube-openapi/pkg/builder", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/handler", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/client-go/discovery", diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/install/BUILD b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/install/BUILD index 867b0a9aa91..c6a9e7c1804 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/install/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/install/BUILD @@ -6,9 +6,8 @@ go_library( importpath = "k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/install", visibility = ["//visibility:public"], deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/v1alpha1:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/install/install.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/install/install.go index e67059ea997..b08fe72ea81 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/install/install.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission" "k8s.io/apiserver/pkg/admission/plugin/webhook/config/apis/webhookadmission/v1alpha1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: webhookadmission.GroupName, - VersionPreferenceOrder: []string{v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: webhookadmission.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(webhookadmission.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/install/BUILD b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/install/BUILD index f3b80e9de0c..2084b46151a 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/install/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/install/BUILD @@ -10,9 +10,8 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/apiserver/pkg/apis/apiserver/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/install/install.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/install/install.go index 805d2e08f04..4b58a9710be 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/install/install.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/install/install.go @@ -17,25 +17,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/apis/apiserver" "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: apiserver.GroupName, - VersionPreferenceOrder: []string{v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: apiserver.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(apiserver.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/audit/install/BUILD b/staging/src/k8s.io/apiserver/pkg/apis/audit/install/BUILD index f25264d665f..796866c5969 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/audit/install/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/apis/audit/install/BUILD @@ -11,9 +11,8 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/apiserver/pkg/apis/audit/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/audit/v1alpha1:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/apis/audit/install/install.go b/staging/src/k8s.io/apiserver/pkg/apis/audit/install/install.go index 5faddcf48db..026f822254c 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/audit/install/install.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/audit/install/install.go @@ -19,27 +19,17 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/apis/audit" "k8s.io/apiserver/pkg/apis/audit/v1alpha1" "k8s.io/apiserver/pkg/apis/audit/v1beta1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: audit.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version, v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: audit.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(audit.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion, v1alpha1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/example/install/BUILD b/staging/src/k8s.io/apiserver/pkg/apis/example/install/BUILD index 011bc1dffff..a30a0ec941b 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/example/install/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/apis/example/install/BUILD @@ -11,9 +11,8 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/apiserver/pkg/apis/example/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/example:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/example/v1:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/apis/example/install/install.go b/staging/src/k8s.io/apiserver/pkg/apis/example/install/install.go index bce46107f6f..77cceff36f2 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/example/install/install.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/example/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/apis/example" examplev1 "k8s.io/apiserver/pkg/apis/example/v1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: example.GroupName, - VersionPreferenceOrder: []string{examplev1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: example.AddToScheme, - }, - announced.VersionToSchemeFunc{ - examplev1.SchemeGroupVersion.Version: examplev1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(example.AddToScheme(scheme)) + utilruntime.Must(examplev1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(examplev1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/example2/install/BUILD b/staging/src/k8s.io/apiserver/pkg/apis/example2/install/BUILD index 2bcdf82d944..b5e80a03443 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/example2/install/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/apis/example2/install/BUILD @@ -11,11 +11,9 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/apiserver/pkg/apis/example2/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/example:go_default_library", - "//vendor/k8s.io/apiserver/pkg/apis/example2:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/example2/v1:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/apis/example2/install/install.go b/staging/src/k8s.io/apiserver/pkg/apis/example2/install/install.go index 233e37ad549..473895998f1 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/example2/install/install.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/example2/install/install.go @@ -19,26 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/apis/example" - "k8s.io/apiserver/pkg/apis/example2" example2v1 "k8s.io/apiserver/pkg/apis/example2/v1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: example2.GroupName, - VersionPreferenceOrder: []string{example2v1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: example.AddToScheme, - }, - announced.VersionToSchemeFunc{ - example2v1.SchemeGroupVersion.Version: example2v1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(example.AddToScheme(scheme)) + utilruntime.Must(example2v1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(example2v1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/audit_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/audit_test.go index 728e93b2a6e..78bbe4464e4 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/audit_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/audit_test.go @@ -70,7 +70,7 @@ func TestAudit(t *testing.T) { if events[i].RequestObject == nil { return nil } - return fmt.Errorf("expected RequestBody to be nil, got non-nill '%s'", events[i].RequestObject.Raw) + return fmt.Errorf("expected RequestBody to be nil, got non-nil '%s'", events[i].RequestObject.Raw) } } requestBodyIs := func(i int, text string) eventCheck { @@ -100,7 +100,7 @@ func TestAudit(t *testing.T) { if events[i].ResponseObject == nil { return nil } - return fmt.Errorf("expected ResponseBody to be nil, got non-nill '%s'", events[i].ResponseObject.Raw) + return fmt.Errorf("expected ResponseBody to be nil, got non-nil '%s'", events[i].ResponseObject.Raw) } } responseBodyMatches := func(i int, pattern string) eventCheck { diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD index bcc79397828..421083cceab 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD @@ -15,7 +15,6 @@ go_test( embed = [":go_default_library"], deps = [ "//vendor/github.com/stretchr/testify/assert:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/root_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/root_test.go index 851215ddaf7..3e7c3c906b1 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/root_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/root_test.go @@ -26,7 +26,6 @@ import ( "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/apimachinery/registered" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -37,9 +36,8 @@ import ( ) var ( - registry = registered.NewAPIRegistrationManager() - scheme = runtime.NewScheme() - codecs = serializer.NewCodecFactory(scheme) + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme) ) func init() { diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go index becfb405bfb..aba84a0a7ed 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go @@ -499,38 +499,17 @@ func TestPatchResourceNumberConversion(t *testing.T) { expectedPod: &example.Pod{}, } - tc.startingPod.Name = name - tc.startingPod.Namespace = namespace - tc.startingPod.UID = uid - tc.startingPod.ResourceVersion = "1" - tc.startingPod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.startingPod.Spec.ActiveDeadlineSeconds = &fifteen + setTcPod(tc.startingPod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), &fifteen, "") // Patch tries to change to 30. - tc.changedPod.Name = name - tc.changedPod.Namespace = namespace - tc.changedPod.UID = uid - tc.changedPod.ResourceVersion = "1" - tc.changedPod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.changedPod.Spec.ActiveDeadlineSeconds = &thirty + setTcPod(tc.changedPod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), &thirty, "") // Someone else already changed it to 30. // This should be fine since it's not a "meaningful conflict". // Previously this was detected as a meaningful conflict because int64(30) != float64(30). - tc.updatePod.Name = name - tc.updatePod.Namespace = namespace - tc.updatePod.UID = uid - tc.updatePod.ResourceVersion = "2" - tc.updatePod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.updatePod.Spec.ActiveDeadlineSeconds = &thirty - tc.updatePod.Spec.NodeName = "anywhere" + setTcPod(tc.updatePod, name, namespace, uid, "2", examplev1.SchemeGroupVersion.String(), &thirty, "anywhere") - tc.expectedPod.Name = name - tc.expectedPod.Namespace = namespace - tc.expectedPod.UID = uid - tc.expectedPod.ResourceVersion = "2" - tc.expectedPod.Spec.ActiveDeadlineSeconds = &thirty - tc.expectedPod.Spec.NodeName = "anywhere" + setTcPod(tc.expectedPod, name, namespace, uid, "2", "", &thirty, "anywhere") tc.Run(t) } @@ -552,34 +531,13 @@ func TestPatchResourceWithVersionConflict(t *testing.T) { expectedPod: &example.Pod{}, } - tc.startingPod.Name = name - tc.startingPod.Namespace = namespace - tc.startingPod.UID = uid - tc.startingPod.ResourceVersion = "1" - tc.startingPod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.startingPod.Spec.ActiveDeadlineSeconds = &fifteen + setTcPod(tc.startingPod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), &fifteen, "") - tc.changedPod.Name = name - tc.changedPod.Namespace = namespace - tc.changedPod.UID = uid - tc.changedPod.ResourceVersion = "1" - tc.changedPod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.changedPod.Spec.ActiveDeadlineSeconds = &thirty + setTcPod(tc.changedPod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), &thirty, "") - tc.updatePod.Name = name - tc.updatePod.Namespace = namespace - tc.updatePod.UID = uid - tc.updatePod.ResourceVersion = "2" - tc.updatePod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.updatePod.Spec.ActiveDeadlineSeconds = &fifteen - tc.updatePod.Spec.NodeName = "anywhere" + setTcPod(tc.updatePod, name, namespace, uid, "2", examplev1.SchemeGroupVersion.String(), &fifteen, "anywhere") - tc.expectedPod.Name = name - tc.expectedPod.Namespace = namespace - tc.expectedPod.UID = uid - tc.expectedPod.ResourceVersion = "2" - tc.expectedPod.Spec.ActiveDeadlineSeconds = &thirty - tc.expectedPod.Spec.NodeName = "anywhere" + setTcPod(tc.expectedPod, name, namespace, uid, "2", "", &thirty, "anywhere") tc.Run(t) } @@ -667,26 +625,11 @@ func TestPatchResourceWithConflict(t *testing.T) { // See issue #63104 for discussion of how much sense this makes. - tc.startingPod.Name = name - tc.startingPod.Namespace = namespace - tc.startingPod.UID = uid - tc.startingPod.ResourceVersion = "1" - tc.startingPod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.startingPod.Spec.NodeName = "here" + setTcPod(tc.startingPod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), nil, "here") - tc.changedPod.Name = name - tc.changedPod.Namespace = namespace - tc.changedPod.UID = uid - tc.changedPod.ResourceVersion = "1" - tc.changedPod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.changedPod.Spec.NodeName = "there" + setTcPod(tc.changedPod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), nil, "there") - tc.updatePod.Name = name - tc.updatePod.Namespace = namespace - tc.updatePod.UID = uid - tc.updatePod.ResourceVersion = "2" - tc.updatePod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.updatePod.Spec.NodeName = "anywhere" + setTcPod(tc.updatePod, name, namespace, uid, "2", examplev1.SchemeGroupVersion.String(), nil, "anywhere") tc.expectedPod.Name = name tc.expectedPod.Namespace = namespace @@ -752,26 +695,11 @@ func TestPatchWithAdmissionRejection(t *testing.T) { expectedError: test.expectedError, } - tc.startingPod.Name = name - tc.startingPod.Namespace = namespace - tc.startingPod.UID = uid - tc.startingPod.ResourceVersion = "1" - tc.startingPod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.startingPod.Spec.ActiveDeadlineSeconds = &fifteen + setTcPod(tc.startingPod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), &fifteen, "") - tc.changedPod.Name = name - tc.changedPod.Namespace = namespace - tc.changedPod.UID = uid - tc.changedPod.ResourceVersion = "1" - tc.changedPod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.changedPod.Spec.ActiveDeadlineSeconds = &thirty + setTcPod(tc.changedPod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), &thirty, "") - tc.updatePod.Name = name - tc.updatePod.Namespace = namespace - tc.updatePod.UID = uid - tc.updatePod.ResourceVersion = "1" - tc.updatePod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.updatePod.Spec.ActiveDeadlineSeconds = &fifteen + setTcPod(tc.updatePod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), &fifteen, "") tc.Run(t) } @@ -804,27 +732,11 @@ func TestPatchWithVersionConflictThenAdmissionFailure(t *testing.T) { expectedError: "admission failure", } - tc.startingPod.Name = name - tc.startingPod.Namespace = namespace - tc.startingPod.UID = uid - tc.startingPod.ResourceVersion = "1" - tc.startingPod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.startingPod.Spec.ActiveDeadlineSeconds = &fifteen + setTcPod(tc.startingPod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), &fifteen, "") - tc.changedPod.Name = name - tc.changedPod.Namespace = namespace - tc.changedPod.UID = uid - tc.changedPod.ResourceVersion = "1" - tc.changedPod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.changedPod.Spec.ActiveDeadlineSeconds = &thirty + setTcPod(tc.changedPod, name, namespace, uid, "1", examplev1.SchemeGroupVersion.String(), &thirty, "") - tc.updatePod.Name = name - tc.updatePod.Namespace = namespace - tc.updatePod.UID = uid - tc.updatePod.ResourceVersion = "2" - tc.updatePod.APIVersion = examplev1.SchemeGroupVersion.String() - tc.updatePod.Spec.ActiveDeadlineSeconds = &fifteen - tc.updatePod.Spec.NodeName = "anywhere" + setTcPod(tc.updatePod, name, namespace, uid, "2", examplev1.SchemeGroupVersion.String(), &fifteen, "anywhere") tc.Run(t) } @@ -922,3 +834,19 @@ func TestFinishRequest(t *testing.T) { } } } + +func setTcPod(tcPod *example.Pod, name string, namespace string, uid types.UID, resourceVersion string, apiVersion string, activeDeadlineSeconds *int64, nodeName string) { + tcPod.Name = name + tcPod.Namespace = namespace + tcPod.UID = uid + tcPod.ResourceVersion = resourceVersion + if len(apiVersion) != 0 { + tcPod.APIVersion = apiVersion + } + if activeDeadlineSeconds != nil { + tcPod.Spec.ActiveDeadlineSeconds = activeDeadlineSeconds + } + if len(nodeName) != 0 { + tcPod.Spec.NodeName = nodeName + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/BUILD b/staging/src/k8s.io/apiserver/pkg/server/BUILD index 31c73d191c4..fc925bfe3b0 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/BUILD @@ -16,7 +16,6 @@ go_test( embed = [":go_default_library"], deps = [ "//vendor/github.com/stretchr/testify/assert:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", @@ -97,8 +96,6 @@ go_library( "//vendor/golang.org/x/net/http2:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD b/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD index 6404a3953c7..6822469d451 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/BUILD @@ -49,6 +49,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/waitgroup:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library", + "//vendor/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters:go_default_library", "//vendor/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library", "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/httplog:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/filters/waitgroup.go b/staging/src/k8s.io/apiserver/pkg/server/filters/waitgroup.go index 12da97e76d6..b40a4227296 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/filters/waitgroup.go +++ b/staging/src/k8s.io/apiserver/pkg/server/filters/waitgroup.go @@ -17,9 +17,11 @@ limitations under the License. package filters import ( + "errors" "net/http" utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" + "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" apirequest "k8s.io/apiserver/pkg/endpoints/request" ) @@ -30,7 +32,7 @@ func WithWaitGroup(handler http.Handler, longRunning apirequest.LongRunningReque requestInfo, ok := apirequest.RequestInfoFrom(ctx) if !ok { // if this happens, the handler chain isn't setup correctly because there is no request info - handler.ServeHTTP(w, req) + responsewriters.InternalError(w, req, errors.New("no RequestInfo found in the context")) return } diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index d6c2c9a949a..10a8ddff6e5 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -28,8 +28,6 @@ import ( "github.com/golang/glog" "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/apimachinery" - "k8s.io/apimachinery/pkg/apimachinery/registered" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -49,7 +47,7 @@ import ( // Info about an API group. type APIGroupInfo struct { - GroupMeta apimachinery.GroupMeta + PrioritizedVersions []schema.GroupVersion // Info about the resources in this group. It's a map from version to resource to the storage. VersionedResourcesStorageMap map[string]map[string]rest.Storage // OptionsExternalVersion controls the APIVersion used for common objects in the @@ -316,7 +314,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error { // installAPIResources is a private method for installing the REST storage backing each api groupversionresource func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo) error { - for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions { + for _, groupVersion := range apiGroupInfo.PrioritizedVersions { if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 { glog.Warningf("Skipping API %v because it has no resources.", groupVersion) continue @@ -345,7 +343,7 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo // setup discovery apiVersions := []string{} - for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions { + for _, groupVersion := range apiGroupInfo.PrioritizedVersions { apiVersions = append(apiVersions, groupVersion.Version) } // Install the version handler. @@ -359,10 +357,10 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo func (s *GenericAPIServer) InstallAPIGroup(apiGroupInfo *APIGroupInfo) error { // Do not register empty group or empty version. Doing so claims /apis/ for the wrong entity to be returned. // Catching these here places the error much closer to its origin - if len(apiGroupInfo.GroupMeta.GroupVersions[0].Group) == 0 { + if len(apiGroupInfo.PrioritizedVersions[0].Group) == 0 { return fmt.Errorf("cannot register handler with an empty group for %#v", *apiGroupInfo) } - if len(apiGroupInfo.GroupMeta.GroupVersions[0].Version) == 0 { + if len(apiGroupInfo.PrioritizedVersions[0].Version) == 0 { return fmt.Errorf("cannot register handler with an empty version for %#v", *apiGroupInfo) } @@ -374,7 +372,7 @@ func (s *GenericAPIServer) InstallAPIGroup(apiGroupInfo *APIGroupInfo) error { // Install the version handler. // Add a handler at /apis/ to enumerate all versions supported by this group. apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{} - for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions { + for _, groupVersion := range apiGroupInfo.PrioritizedVersions { // Check the config to make sure that we elide versions that don't have any resources if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 { continue @@ -385,11 +383,11 @@ func (s *GenericAPIServer) InstallAPIGroup(apiGroupInfo *APIGroupInfo) error { }) } preferredVersionForDiscovery := metav1.GroupVersionForDiscovery{ - GroupVersion: apiGroupInfo.GroupMeta.GroupVersions[0].String(), - Version: apiGroupInfo.GroupMeta.GroupVersions[0].Version, + GroupVersion: apiGroupInfo.PrioritizedVersions[0].String(), + Version: apiGroupInfo.PrioritizedVersions[0].Version, } apiGroup := metav1.APIGroup{ - Name: apiGroupInfo.GroupMeta.GroupVersions[0].Group, + Name: apiGroupInfo.PrioritizedVersions[0].Group, Versions: apiVersionsForDiscovery, PreferredVersion: preferredVersionForDiscovery, } @@ -433,11 +431,9 @@ func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupV // NewDefaultAPIGroupInfo returns an APIGroupInfo stubbed with "normal" values // exposed for easier composition from other packages -func NewDefaultAPIGroupInfo(group string, registry *registered.APIRegistrationManager, scheme *runtime.Scheme, parameterCodec runtime.ParameterCodec, codecs serializer.CodecFactory) APIGroupInfo { - groupMeta := registry.GroupOrDie(group) - +func NewDefaultAPIGroupInfo(group string, scheme *runtime.Scheme, parameterCodec runtime.ParameterCodec, codecs serializer.CodecFactory) APIGroupInfo { return APIGroupInfo{ - GroupMeta: *groupMeta, + PrioritizedVersions: scheme.PrioritizedVersionsForGroup(group), VersionedResourcesStorageMap: map[string]map[string]rest.Storage{}, // TODO unhardcode this. It was hardcoded before, but we need to re-evaluate OptionsExternalVersion: &schema.GroupVersion{Version: "v1"}, diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go index 7a8fb645691..b68b30982e7 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go @@ -34,7 +34,6 @@ import ( // "github.com/go-openapi/spec" "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/apimachinery" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -151,12 +150,8 @@ func TestInstallAPIGroups(t *testing.T) { scheme.AddKnownTypes(v1GroupVersion, &metav1.Status{}) metav1.AddToGroupVersion(scheme, v1GroupVersion) - groupMeta := apimachinery.GroupMeta{ - GroupVersions: []schema.GroupVersion{gv}, - } - return APIGroupInfo{ - GroupMeta: groupMeta, + PrioritizedVersions: []schema.GroupVersion{gv}, VersionedResourcesStorageMap: map[string]map[string]rest.Storage{ gv.Version: { "getter": &testGetterStorage{Version: gv.Version}, @@ -184,7 +179,7 @@ func TestInstallAPIGroups(t *testing.T) { for _, api := range apis[1:] { err = s.InstallAPIGroup(&api) assert.NoError(err) - groupPaths = append(groupPaths, APIGroupPrefix+"/"+api.GroupMeta.GroupVersions[0].Group) // /apis/ + groupPaths = append(groupPaths, APIGroupPrefix+"/"+api.PrioritizedVersions[0].Group) // /apis/ } server := httptest.NewServer(s.Handler) @@ -225,19 +220,19 @@ func TestInstallAPIGroups(t *testing.T) { continue } - if got, expected := group.Name, info.GroupMeta.GroupVersions[0].Group; got != expected { + if got, expected := group.Name, info.PrioritizedVersions[0].Group; got != expected { t.Errorf("[%d] unexpected group name at path %q: got=%q expected=%q", i, path, got, expected) continue } - if got, expected := group.PreferredVersion.Version, info.GroupMeta.GroupVersions[0].Version; got != expected { + if got, expected := group.PreferredVersion.Version, info.PrioritizedVersions[0].Version; got != expected { t.Errorf("[%d] unexpected group version at path %q: got=%q expected=%q", i, path, got, expected) continue } } // should serve APIResourceList at group path + / - path = path + "/" + info.GroupMeta.GroupVersions[0].Version + path = path + "/" + info.PrioritizedVersions[0].Version resp, err = http.Get(server.URL + path) if err != nil { t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err) @@ -259,7 +254,7 @@ func TestInstallAPIGroups(t *testing.T) { continue } - if got, expected := resources.GroupVersion, info.GroupMeta.GroupVersions[0].String(); got != expected { + if got, expected := resources.GroupVersion, info.PrioritizedVersions[0].String(); got != expected { t.Errorf("[%d] unexpected groupVersion at path %q: got=%q expected=%q", i, path, got, expected) continue } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/admission.go b/staging/src/k8s.io/apiserver/pkg/server/options/admission.go index 80c2204f8bf..37d0457d7a6 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/admission.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/admission.go @@ -87,6 +87,10 @@ func NewAdmissionOptions() *AdmissionOptions { // AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) { + if a == nil { + return + } + fs.StringSliceVar(&a.EnablePlugins, "enable-admission-plugins", a.EnablePlugins, ""+ "admission plugins that should be enabled in addition to default enabled ones. "+ "Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+ diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement.go b/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement.go index 3901511f677..8c64beed80c 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement.go @@ -97,7 +97,7 @@ func (s *APIEnablementOptions) ApplyTo(c *server.Config, defaultResourceConfig * func unknownGroups(groups []string, registry GroupRegisty) []string { unknownGroups := []string{} for _, group := range groups { - if !registry.IsRegistered(group) { + if !registry.IsGroupRegistered(group) { unknownGroups = append(unknownGroups, group) } } @@ -107,5 +107,5 @@ func unknownGroups(groups []string, registry GroupRegisty) []string { // GroupRegisty provides a method to check whether given group is registered. type GroupRegisty interface { // IsRegistered returns true if given group is registered. - IsRegistered(group string) bool + IsGroupRegistered(group string) bool } diff --git a/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/BUILD b/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/BUILD index 5036ae75bc1..c28bc6c0b1f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/BUILD @@ -23,9 +23,7 @@ go_test( deps = [ "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go b/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go index ee0f08e5f79..cb1c54e3f0b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go +++ b/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go @@ -29,12 +29,12 @@ import ( // GroupVersionRegistry provides access to registered group versions. type GroupVersionRegistry interface { - // IsRegistered returns true if given group is registered. - IsRegistered(group string) bool - // IsRegisteredVersion returns true if given version is registered. - IsRegisteredVersion(v schema.GroupVersion) bool - // RegisteredGroupVersions returns all registered group versions. - RegisteredGroupVersions() []schema.GroupVersion + // IsGroupRegistered returns true if given group is registered. + IsGroupRegistered(group string) bool + // IsVersionRegistered returns true if given version is registered. + IsVersionRegistered(v schema.GroupVersion) bool + // PrioritizedVersionsAllGroups returns all registered group versions. + PrioritizedVersionsAllGroups() []schema.GroupVersion } // MergeResourceEncodingConfigs merges the given defaultResourceConfig with specific GroupVersionResource overrides. @@ -104,12 +104,12 @@ func MergeAPIResourceConfigs( } // Exclude group not registered into the registry. - if !registry.IsRegistered(groupVersion.Group) { + if !registry.IsGroupRegistered(groupVersion.Group) { continue } // Verify that the groupVersion is registered into registry. - if !registry.IsRegisteredVersion(groupVersion) { + if !registry.IsVersionRegistered(groupVersion) { return nil, fmt.Errorf("group version %s that has not been registered", groupVersion.String()) } enabled, err := getRuntimeConfigValue(overrides, key, false) diff --git a/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers_test.go b/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers_test.go index 76a1086e61a..dd5d04c69b7 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers_test.go @@ -22,14 +22,12 @@ import ( apiv1 "k8s.io/api/core/v1" extensionsapiv1beta1 "k8s.io/api/extensions/v1beta1" - "k8s.io/apimachinery/pkg/apimachinery" - "k8s.io/apimachinery/pkg/apimachinery/registered" - "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime" serverstore "k8s.io/apiserver/pkg/server/storage" ) func TestParseRuntimeConfig(t *testing.T) { - registry := newFakeRegistry() + scheme := newFakeScheme() apiv1GroupVersion := apiv1.SchemeGroupVersion testCases := []struct { runtimeConfig map[string]string @@ -116,7 +114,7 @@ func TestParseRuntimeConfig(t *testing.T) { }, expectedAPIConfig: func() *serverstore.ResourceConfig { config := newFakeAPIResourceConfigSource() - config.EnableVersions(registry.RegisteredGroupVersions()...) + config.EnableVersions(scheme.PrioritizedVersionsAllGroups()...) return config }, err: false, @@ -139,8 +137,8 @@ func TestParseRuntimeConfig(t *testing.T) { }, } for index, test := range testCases { - t.Log(registry.RegisteredGroupVersions()) - actualDisablers, err := MergeAPIResourceConfigs(test.defaultResourceConfig(), test.runtimeConfig, registry) + t.Log(scheme.PrioritizedVersionsAllGroups()) + actualDisablers, err := MergeAPIResourceConfigs(test.defaultResourceConfig(), test.runtimeConfig, scheme) if err == nil && test.err { t.Fatalf("expected error for test case: %v", index) } else if err != nil && !test.err { @@ -165,15 +163,13 @@ func newFakeAPIResourceConfigSource() *serverstore.ResourceConfig { return ret } -func newFakeRegistry() *registered.APIRegistrationManager { - registry := registered.NewAPIRegistrationManager() +func newFakeScheme() *runtime.Scheme { + ret := runtime.NewScheme() + apiv1.AddToScheme(ret) + extensionsapiv1beta1.AddToScheme(ret) - registry.RegisterGroup(apimachinery.GroupMeta{ - GroupVersions: []schema.GroupVersion{apiv1.SchemeGroupVersion}, - }) - registry.RegisterGroup(apimachinery.GroupMeta{ - GroupVersions: []schema.GroupVersion{extensionsapiv1beta1.SchemeGroupVersion}, - }) - registry.RegisterVersions([]schema.GroupVersion{apiv1.SchemeGroupVersion, extensionsapiv1beta1.SchemeGroupVersion}) - return registry + ret.SetVersionPriority(apiv1.SchemeGroupVersion) + ret.SetVersionPriority(extensionsapiv1beta1.SchemeGroupVersion) + + return ret } diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/BUILD b/staging/src/k8s.io/apiserver/pkg/server/storage/BUILD index 3a2a57e5f1c..1fe92504a65 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/BUILD @@ -14,7 +14,6 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", @@ -38,7 +37,6 @@ go_library( importpath = "k8s.io/apiserver/pkg/server/storage", deps = [ "//vendor/github.com/golang/glog:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/recognizer:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go index 554cb6c35c1..eff1fe8a417 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go @@ -17,7 +17,8 @@ limitations under the License. package storage import ( - "k8s.io/apimachinery/pkg/apimachinery/registered" + "fmt" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -33,8 +34,8 @@ type ResourceEncodingConfig interface { } type DefaultResourceEncodingConfig struct { - groups map[string]*GroupResourceEncodingConfig - registry *registered.APIRegistrationManager + groups map[string]*GroupResourceEncodingConfig + scheme *runtime.Scheme } type GroupResourceEncodingConfig struct { @@ -47,8 +48,8 @@ type GroupResourceEncodingConfig struct { var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{} -func NewDefaultResourceEncodingConfig(registry *registered.APIRegistrationManager) *DefaultResourceEncodingConfig { - return &DefaultResourceEncodingConfig{groups: map[string]*GroupResourceEncodingConfig{}, registry: registry} +func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig { + return &DefaultResourceEncodingConfig{groups: map[string]*GroupResourceEncodingConfig{}, scheme: scheme} } func newGroupResourceEncodingConfig(defaultEncoding, defaultInternalVersion schema.GroupVersion) *GroupResourceEncodingConfig { @@ -80,16 +81,15 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored } func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) { - groupMeta, err := o.registry.Group(resource.Group) - if err != nil { - return schema.GroupVersion{}, err + if !o.scheme.IsGroupRegistered(resource.Group) { + return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) } groupEncoding, groupExists := o.groups[resource.Group] if !groupExists { // return the most preferred external version for the group - return groupMeta.GroupVersions[0], nil + return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil } resourceOverride, resourceExists := groupEncoding.ExternalResourceEncodings[resource.Resource] @@ -101,8 +101,8 @@ func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.Group } func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) { - if _, err := o.registry.Group(resource.Group); err != nil { - return schema.GroupVersion{}, err + if !o.scheme.IsGroupRegistered(resource.Group) { + return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) } groupEncoding, groupExists := o.groups[resource.Group] diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go index 0f9b455a3be..197ff6b795e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go @@ -20,7 +20,6 @@ import ( "reflect" "testing" - "k8s.io/apimachinery/pkg/apimachinery/registered" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -34,7 +33,6 @@ import ( var ( v1GroupVersion = schema.GroupVersion{Group: "", Version: "v1"} - registry = registered.NewAPIRegistrationManager() scheme = runtime.NewScheme() codecs = serializer.NewCodecFactory(scheme) parameterCodec = runtime.NewParameterCodec(scheme) @@ -50,7 +48,7 @@ func init() { &metav1.APIResourceList{}, ) - exampleinstall.Install(registry, scheme) + exampleinstall.Install(scheme) } type fakeNegotiater struct { @@ -91,7 +89,7 @@ func (n *fakeNegotiater) DecoderToVersion(serializer runtime.Decoder, gv runtime func TestConfigurableStorageFactory(t *testing.T) { ns := &fakeNegotiater{types: []string{"test/test"}} - f := NewDefaultStorageFactory(storagebackend.Config{}, "test/test", ns, NewDefaultResourceEncodingConfig(registry), NewResourceConfig(), nil) + f := NewDefaultStorageFactory(storagebackend.Config{}, "test/test", ns, NewDefaultResourceEncodingConfig(scheme), NewResourceConfig(), nil) f.AddCohabitatingResources(example.Resource("test"), schema.GroupResource{Resource: "test2", Group: "2"}) called := false testEncoderChain := func(e runtime.Encoder) runtime.Encoder { @@ -115,8 +113,7 @@ func TestConfigurableStorageFactory(t *testing.T) { } func TestUpdateEtcdOverrides(t *testing.T) { - registry := registered.NewAPIRegistrationManager() - exampleinstall.Install(registry, scheme) + exampleinstall.Install(scheme) testCases := []struct { resource schema.GroupResource @@ -142,7 +139,7 @@ func TestUpdateEtcdOverrides(t *testing.T) { Prefix: "/registry", ServerList: defaultEtcdLocation, } - storageFactory := NewDefaultStorageFactory(defaultConfig, "", codecs, NewDefaultResourceEncodingConfig(registry), NewResourceConfig(), nil) + storageFactory := NewDefaultStorageFactory(defaultConfig, "", codecs, NewDefaultResourceEncodingConfig(scheme), NewResourceConfig(), nil) storageFactory.SetEtcdLocation(test.resource, test.servers) var err error diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go index 90f1dca42b2..e3038c9d9ac 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go @@ -77,7 +77,7 @@ func (t *envelopeTransformer) TransformFromStorage(data []byte, context value.Co // length cannot fit in 8 bits (1 byte). Thus, we use 16 bits (2 bytes) to store the length. keyLen := int(binary.BigEndian.Uint16(data[:2])) if keyLen+2 > len(data) { - return nil, false, fmt.Errorf("invalid data encountered by genvelope transformer, length longer than available bytes: %q", data) + return nil, false, fmt.Errorf("invalid data encountered by envelope transformer, length longer than available bytes: %q", data) } encKey := data[2 : keyLen+2] encData := data[2+keyLen:] diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD b/staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD index 4cacaf511a1..c96af1f9c17 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD @@ -12,7 +12,6 @@ go_library( importpath = "k8s.io/apiserver/pkg/util/webhook", deps = [ "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", @@ -32,7 +31,7 @@ go_test( embed = [":go_default_library"], deps = [ "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go index 4712f079976..3b03fd3fd5e 100755 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go @@ -22,7 +22,6 @@ import ( "time" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -42,13 +41,13 @@ type GenericWebhook struct { } // NewGenericWebhook creates a new GenericWebhook from the provided kubeconfig file. -func NewGenericWebhook(registry *registered.APIRegistrationManager, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff time.Duration) (*GenericWebhook, error) { - return newGenericWebhook(registry, codecFactory, kubeConfigFile, groupVersions, initialBackoff, defaultRequestTimeout) +func NewGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff time.Duration) (*GenericWebhook, error) { + return newGenericWebhook(scheme, codecFactory, kubeConfigFile, groupVersions, initialBackoff, defaultRequestTimeout) } -func newGenericWebhook(registry *registered.APIRegistrationManager, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff, requestTimeout time.Duration) (*GenericWebhook, error) { +func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff, requestTimeout time.Duration) (*GenericWebhook, error) { for _, groupVersion := range groupVersions { - if !registry.IsRegisteredVersion(groupVersion) { + if !scheme.IsVersionRegistered(groupVersion) { return nil, fmt.Errorf("webhook plugin requires enabling extension resource: %s", groupVersion) } } diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go index 83756188440..0399ed8435e 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go @@ -32,7 +32,7 @@ import ( "time" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apimachinery/registered" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -258,7 +258,7 @@ func TestKubeConfigFile(t *testing.T) { if err == nil { defer os.Remove(kubeConfigFile) - _, err = NewGenericWebhook(registered.NewAPIRegistrationManager(), scheme.Codecs, kubeConfigFile, groupVersions, retryBackoff) + _, err = NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, kubeConfigFile, groupVersions, retryBackoff) } return err @@ -281,7 +281,7 @@ func TestKubeConfigFile(t *testing.T) { // TestMissingKubeConfigFile ensures that a kube config path to a missing file is handled properly func TestMissingKubeConfigFile(t *testing.T) { kubeConfigPath := "/some/missing/path" - _, err := NewGenericWebhook(registered.NewAPIRegistrationManager(), scheme.Codecs, kubeConfigPath, groupVersions, retryBackoff) + _, err := NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, kubeConfigPath, groupVersions, retryBackoff) if err == nil { t.Errorf("creating the webhook should had failed") @@ -393,7 +393,7 @@ func TestTLSConfig(t *testing.T) { defer os.Remove(configFile) - wh, err := NewGenericWebhook(registered.NewAPIRegistrationManager(), scheme.Codecs, configFile, groupVersions, retryBackoff) + wh, err := NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, configFile, groupVersions, retryBackoff) if err == nil { err = wh.RestClient.Get().Do().Error() @@ -458,7 +458,7 @@ func TestRequestTimeout(t *testing.T) { var requestTimeout = 10 * time.Millisecond - wh, err := newGenericWebhook(registered.NewAPIRegistrationManager(), scheme.Codecs, configFile, groupVersions, retryBackoff, requestTimeout) + wh, err := newGenericWebhook(runtime.NewScheme(), scheme.Codecs, configFile, groupVersions, retryBackoff, requestTimeout) if err != nil { t.Fatalf("failed to create the webhook: %v", err) } @@ -544,7 +544,7 @@ func TestWithExponentialBackoff(t *testing.T) { defer os.Remove(configFile) - wh, err := NewGenericWebhook(registered.NewAPIRegistrationManager(), scheme.Codecs, configFile, groupVersions, retryBackoff) + wh, err := NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, configFile, groupVersions, retryBackoff) if err != nil { t.Fatalf("failed to create the webhook: %v", err) diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/log/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/audit/log/BUILD index e0726299189..9a43f94baf2 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/audit/log/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/log/BUILD @@ -37,10 +37,8 @@ go_test( embed = [":go_default_library"], deps = [ "//vendor/github.com/pborman/uuid:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/audit/install:go_default_library", diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/log/backend_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/log/backend_test.go index a58e2d31bdc..68932d9319d 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/audit/log/backend_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/log/backend_test.go @@ -25,10 +25,8 @@ import ( "github.com/pborman/uuid" - "k8s.io/apimachinery/pkg/apimachinery/registered" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" auditinternal "k8s.io/apiserver/pkg/apis/audit" "k8s.io/apiserver/pkg/apis/audit/install" @@ -36,15 +34,8 @@ import ( "k8s.io/apiserver/pkg/audit" ) -// NOTE: Copied from webhook backend to register auditv1beta1 to scheme -var ( - registry = registered.NewAPIRegistrationManager() -) - func init() { - allGVs := []schema.GroupVersion{auditv1beta1.SchemeGroupVersion} - registry.RegisterVersions(allGVs) - install.Install(registry, audit.Scheme) + install.Install(audit.Scheme) } func TestLogEventsLegacy(t *testing.T) { diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/BUILD index 69eb2fbf339..62faa456f0b 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/BUILD @@ -28,12 +28,9 @@ go_library( srcs = ["webhook.go"], importpath = "k8s.io/apiserver/plugin/pkg/audit/webhook", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/audit/install:go_default_library", - "//vendor/k8s.io/apiserver/pkg/apis/audit/v1alpha1:go_default_library", - "//vendor/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library", "//vendor/k8s.io/apiserver/pkg/audit:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/webhook:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go index b36f45ef377..80b4842fd29 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook.go @@ -20,12 +20,9 @@ package webhook import ( "time" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime/schema" auditinternal "k8s.io/apiserver/pkg/apis/audit" "k8s.io/apiserver/pkg/apis/audit/install" - auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1" - auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/util/webhook" "k8s.io/client-go/rest" @@ -40,22 +37,12 @@ const ( DefaultInitialBackoff = 10 * time.Second ) -var ( - // NOTE: Copied from other webhook implementations - // - // Can we make these passable to NewGenericWebhook? - // TODO(audit): figure out a general way to let the client choose their preferred version - registry = registered.NewAPIRegistrationManager() -) - func init() { - allGVs := []schema.GroupVersion{auditv1alpha1.SchemeGroupVersion, auditv1beta1.SchemeGroupVersion} - registry.RegisterVersions(allGVs) - install.Install(registry, audit.Scheme) + install.Install(audit.Scheme) } func loadWebhook(configFile string, groupVersion schema.GroupVersion, initialBackoff time.Duration) (*webhook.GenericWebhook, error) { - return webhook.NewGenericWebhook(registry, audit.Codecs, configFile, + return webhook.NewGenericWebhook(audit.Scheme, audit.Codecs, configFile, []schema.GroupVersion{groupVersion}, initialBackoff) } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD index 9739495ef0f..4cd60e841a7 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/BUILD @@ -28,9 +28,10 @@ go_library( deps = [ "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/authentication/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/cache:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/webhook:go_default_library", diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go index 77cd097f309..7aa1c0c0ab6 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook/webhook.go @@ -23,9 +23,10 @@ import ( "github.com/golang/glog" authentication "k8s.io/api/authentication/v1beta1" - "k8s.io/apimachinery/pkg/apimachinery/registered" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/cache" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/util/webhook" @@ -112,21 +113,15 @@ func (w *WebhookTokenAuthenticator) AuthenticateToken(token string) (user.Info, }, true, nil } -// NOTE: client-go doesn't provide a registry. client-go does registers the -// authentication/v1beta1. We construct a registry that acknowledges -// authentication/v1beta1 as an enabled version to pass a check enforced in -// NewGenericWebhook. -var registry = registered.NewAPIRegistrationManager() - -func init() { - registry.RegisterVersions(groupVersions) -} - // tokenReviewInterfaceFromKubeconfig builds a client from the specified kubeconfig file, // and returns a TokenReviewInterface that uses that client. Note that the client submits TokenReview // requests to the exact path specified in the kubeconfig file, so arbitrary non-API servers can be targeted. func tokenReviewInterfaceFromKubeconfig(kubeConfigFile string) (authenticationclient.TokenReviewInterface, error) { - gw, err := webhook.NewGenericWebhook(registry, scheme.Codecs, kubeConfigFile, groupVersions, 0) + localScheme := runtime.NewScheme() + scheme.AddToScheme(localScheme) + utilruntime.Must(localScheme.SetVersionPriority(groupVersions...)) + + gw, err := webhook.NewGenericWebhook(localScheme, scheme.Codecs, kubeConfigFile, groupVersions, 0) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD index 08acf7b8480..a1099d8e40c 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/BUILD @@ -30,7 +30,7 @@ go_library( deps = [ "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/authorization/v1beta1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/cache:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library", diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go index 7f1a3d99f9e..83157c48c86 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go @@ -25,7 +25,7 @@ import ( "github.com/golang/glog" authorization "k8s.io/api/authorization/v1beta1" - "k8s.io/apimachinery/pkg/apimachinery/registered" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/cache" "k8s.io/apiserver/pkg/authentication/user" @@ -234,21 +234,15 @@ func convertToSARExtra(extra map[string][]string) map[string]authorization.Extra return ret } -// NOTE: client-go doesn't provide a registry. client-go does registers the -// authorization/v1beta1. We construct a registry that acknowledges -// authorization/v1beta1 as an enabled version to pass a check enforced in -// NewGenericWebhook. -var registry = registered.NewAPIRegistrationManager() - -func init() { - registry.RegisterVersions(groupVersions) -} - // subjectAccessReviewInterfaceFromKubeconfig builds a client from the specified kubeconfig file, // and returns a SubjectAccessReviewInterface that uses that client. Note that the client submits SubjectAccessReview // requests to the exact path specified in the kubeconfig file, so arbitrary non-API servers can be targeted. func subjectAccessReviewInterfaceFromKubeconfig(kubeConfigFile string) (authorizationclient.SubjectAccessReviewInterface, error) { - gw, err := webhook.NewGenericWebhook(registry, scheme.Codecs, kubeConfigFile, groupVersions, 0) + localScheme := runtime.NewScheme() + scheme.AddToScheme(localScheme) + localScheme.SetVersionPriority(groupVersions...) + + gw, err := webhook.NewGenericWebhook(localScheme, scheme.Codecs, kubeConfigFile, groupVersions, 0) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/client-go/Godeps/Godeps.json b/staging/src/k8s.io/client-go/Godeps/Godeps.json index 6e33ada2b22..cb7b051b5b2 100644 --- a/staging/src/k8s.io/client-go/Godeps/Godeps.json +++ b/staging/src/k8s.io/client-go/Godeps/Godeps.json @@ -394,18 +394,6 @@ "ImportPath": "k8s.io/apimachinery/pkg/api/testing/roundtrip", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/announced", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apimachinery/pkg/apis/meta/fuzzer", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -580,7 +568,7 @@ }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" } ] } diff --git a/staging/src/k8s.io/client-go/discovery/BUILD b/staging/src/k8s.io/client-go/discovery/BUILD index 82936005b12..397a390a57c 100644 --- a/staging/src/k8s.io/client-go/discovery/BUILD +++ b/staging/src/k8s.io/client-go/discovery/BUILD @@ -11,20 +11,18 @@ go_library( srcs = [ "discovery_client.go", "helper.go", - "restmapper.go", "unstructured.go", ], importpath = "k8s.io/client-go/discovery", deps = [ - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/protobuf/proto:go_default_library", "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/version:go_default_library", "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library", @@ -34,20 +32,13 @@ go_library( go_test( name = "go_default_xtest", - srcs = [ - "helper_blackbox_test.go", - "restmapper_test.go", - ], + srcs = ["helper_blackbox_test.go"], deps = [ - "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library", - "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/version:go_default_library", "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", @@ -81,6 +72,7 @@ go_test( "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/version:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", diff --git a/staging/src/k8s.io/client-go/discovery/discovery_client.go b/staging/src/k8s.io/client-go/discovery/discovery_client.go index d59e8a02ac1..cef4d40152a 100644 --- a/staging/src/k8s.io/client-go/discovery/discovery_client.go +++ b/staging/src/k8s.io/client-go/discovery/discovery_client.go @@ -22,6 +22,7 @@ import ( "net/url" "sort" "strings" + "sync" "time" "github.com/golang/protobuf/proto" @@ -32,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/kubernetes/scheme" restclient "k8s.io/client-go/rest" @@ -191,33 +193,7 @@ func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (r // serverResources returns the supported resources for all groups and versions. func (d *DiscoveryClient) serverResources() ([]*metav1.APIResourceList, error) { - apiGroups, err := d.ServerGroups() - if err != nil { - return nil, err - } - - result := []*metav1.APIResourceList{} - failedGroups := make(map[schema.GroupVersion]error) - - for _, apiGroup := range apiGroups.Groups { - for _, version := range apiGroup.Versions { - gv := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} - resources, err := d.ServerResourcesForGroupVersion(version.GroupVersion) - if err != nil { - // TODO: maybe restrict this to NotFound errors - failedGroups[gv] = err - continue - } - - result = append(result, resources) - } - } - - if len(failedGroups) == 0 { - return result, nil - } - - return result, &ErrGroupDiscoveryFailed{Groups: failedGroups} + return ServerResources(d) } // ServerResources returns the supported resources for all groups and versions. @@ -253,6 +229,33 @@ func (d *DiscoveryClient) serverPreferredResources() ([]*metav1.APIResourceList, return ServerPreferredResources(d) } +// ServerResources uses the provided discovery interface to look up supported resources for all groups and versions. +func ServerResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { + apiGroups, err := d.ServerGroups() + if err != nil { + return nil, err + } + + groupVersionResources, failedGroups := fetchGroupVersionResources(d, apiGroups) + + // order results by group/version discovery order + result := []*metav1.APIResourceList{} + for _, apiGroup := range apiGroups.Groups { + for _, version := range apiGroup.Versions { + gv := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} + if resources, ok := groupVersionResources[gv]; ok { + result = append(result, resources) + } + } + } + + if len(failedGroups) == 0 { + return result, nil + } + + return result, &ErrGroupDiscoveryFailed{Groups: failedGroups} +} + // ServerPreferredResources uses the provided discovery interface to look up preferred resources func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { serverGroupList, err := d.ServerGroups() @@ -260,9 +263,9 @@ func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList, return nil, err } - result := []*metav1.APIResourceList{} - failedGroups := make(map[schema.GroupVersion]error) + groupVersionResources, failedGroups := fetchGroupVersionResources(d, serverGroupList) + result := []*metav1.APIResourceList{} grVersions := map[schema.GroupResource]string{} // selected version of a GroupResource grApiResources := map[schema.GroupResource]*metav1.APIResource{} // selected APIResource for a GroupResource gvApiResourceLists := map[schema.GroupVersion]*metav1.APIResourceList{} // blueprint for a APIResourceList for later grouping @@ -270,10 +273,9 @@ func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList, for _, apiGroup := range serverGroupList.Groups { for _, version := range apiGroup.Versions { groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} - apiResourceList, err := d.ServerResourcesForGroupVersion(version.GroupVersion) - if err != nil { - // TODO: maybe restrict this to NotFound errors - failedGroups[groupVersion] = err + + apiResourceList, ok := groupVersionResources[groupVersion] + if !ok { continue } @@ -315,6 +317,41 @@ func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList, return result, &ErrGroupDiscoveryFailed{Groups: failedGroups} } +// fetchServerResourcesForGroupVersions uses the discovery client to fetch the resources for the specified groups in parallel +func fetchGroupVersionResources(d DiscoveryInterface, apiGroups *metav1.APIGroupList) (map[schema.GroupVersion]*metav1.APIResourceList, map[schema.GroupVersion]error) { + groupVersionResources := make(map[schema.GroupVersion]*metav1.APIResourceList) + failedGroups := make(map[schema.GroupVersion]error) + + wg := &sync.WaitGroup{} + resultLock := &sync.Mutex{} + for _, apiGroup := range apiGroups.Groups { + for _, version := range apiGroup.Versions { + groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} + wg.Add(1) + go func() { + defer wg.Done() + defer utilruntime.HandleCrash() + + apiResourceList, err := d.ServerResourcesForGroupVersion(groupVersion.String()) + + // lock to record results + resultLock.Lock() + defer resultLock.Unlock() + + if err != nil { + // TODO: maybe restrict this to NotFound errors + failedGroups[groupVersion] = err + } else { + groupVersionResources[groupVersion] = apiResourceList + } + }() + } + } + wg.Wait() + + return groupVersionResources, failedGroups +} + // ServerPreferredResources returns the supported resources with the version preferred by the // server. func (d *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) { diff --git a/staging/src/k8s.io/client-go/discovery/discovery_client_test.go b/staging/src/k8s.io/client-go/discovery/discovery_client_test.go index 1af25be747a..15e0a9e891d 100644 --- a/staging/src/k8s.io/client-go/discovery/discovery_client_test.go +++ b/staging/src/k8s.io/client-go/discovery/discovery_client_test.go @@ -32,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/version" restclient "k8s.io/client-go/rest" @@ -204,6 +205,14 @@ func TestGetServerResources(t *testing.T) { {Name: "jobs", Namespaced: true, Kind: "Job"}, }, } + beta2 := metav1.APIResourceList{ + GroupVersion: "extensions/v1beta2", + APIResources: []metav1.APIResource{ + {Name: "deployments", Namespaced: true, Kind: "Deployment"}, + {Name: "ingresses", Namespaced: true, Kind: "Ingress"}, + {Name: "jobs", Namespaced: true, Kind: "Job"}, + }, + } tests := []struct { resourcesList *metav1.APIResourceList path string @@ -236,6 +245,8 @@ func TestGetServerResources(t *testing.T) { list = &stable case "/apis/extensions/v1beta1": list = &beta + case "/apis/extensions/v1beta2": + list = &beta2 case "/api": list = &metav1.APIVersions{ Versions: []string{ @@ -246,8 +257,10 @@ func TestGetServerResources(t *testing.T) { list = &metav1.APIGroupList{ Groups: []metav1.APIGroup{ { + Name: "extensions", Versions: []metav1.GroupVersionForDiscovery{ - {GroupVersion: "extensions/v1beta1"}, + {GroupVersion: "extensions/v1beta1", Version: "v1beta1"}, + {GroupVersion: "extensions/v1beta2", Version: "v1beta2"}, }, }, }, @@ -289,11 +302,10 @@ func TestGetServerResources(t *testing.T) { if err != nil { t.Errorf("unexpected error: %v", err) } - serverGroupVersions := sets.NewString(groupVersions(serverResources)...) - for _, api := range []string{"v1", "extensions/v1beta1"} { - if !serverGroupVersions.Has(api) { - t.Errorf("missing expected api %q in %v", api, serverResources) - } + serverGroupVersions := groupVersions(serverResources) + expectedGroupVersions := []string{"v1", "extensions/v1beta1", "extensions/v1beta2"} + if !reflect.DeepEqual(expectedGroupVersions, serverGroupVersions) { + t.Errorf("unexpected group versions: %v", diff.ObjectReflectDiff(expectedGroupVersions, serverGroupVersions)) } } @@ -639,7 +651,7 @@ func TestServerPreferredResourcesRetries(t *testing.T) { { Name: "extensions", Versions: []metav1.GroupVersionForDiscovery{ - {GroupVersion: "extensions/v1beta1"}, + {GroupVersion: "extensions/v1beta1", Version: "v1beta1"}, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: "extensions/v1beta1", diff --git a/staging/src/k8s.io/client-go/discovery/unstructured.go b/staging/src/k8s.io/client-go/discovery/unstructured.go index fa7f2ec061b..81913a41423 100644 --- a/staging/src/k8s.io/client-go/discovery/unstructured.go +++ b/staging/src/k8s.io/client-go/discovery/unstructured.go @@ -26,31 +26,17 @@ import ( // UnstructuredObjectTyper provides a runtime.ObjectTyper implementation for // runtime.Unstructured object based on discovery information. type UnstructuredObjectTyper struct { - registered map[schema.GroupVersionKind]bool - typers []runtime.ObjectTyper + typers []runtime.ObjectTyper } // NewUnstructuredObjectTyper returns a runtime.ObjectTyper for // unstructured objects based on discovery information. It accepts a list of fallback typers // for handling objects that are not runtime.Unstructured. It does not delegate the Recognizes // check, only ObjectKinds. -func NewUnstructuredObjectTyper(groupResources []*APIGroupResources, typers ...runtime.ObjectTyper) *UnstructuredObjectTyper { +// TODO this only works for the apiextensions server and doesn't recognize any types. Move to point of use. +func NewUnstructuredObjectTyper(typers ...runtime.ObjectTyper) *UnstructuredObjectTyper { dot := &UnstructuredObjectTyper{ - registered: make(map[schema.GroupVersionKind]bool), - typers: typers, - } - for _, group := range groupResources { - for _, discoveryVersion := range group.Group.Versions { - resources, ok := group.VersionedResources[discoveryVersion.Version] - if !ok { - continue - } - - gv := schema.GroupVersion{Group: group.Group.Name, Version: discoveryVersion.Version} - for _, resource := range resources { - dot.registered[gv.WithKind(resource.Kind)] = true - } - } + typers: typers, } return dot } @@ -89,7 +75,7 @@ func (d *UnstructuredObjectTyper) ObjectKinds(obj runtime.Object) (gvks []schema // Recognizes returns true if the provided group,version,kind was in the // discovery information. func (d *UnstructuredObjectTyper) Recognizes(gvk schema.GroupVersionKind) bool { - return d.registered[gvk] + return false } var _ runtime.ObjectTyper = &UnstructuredObjectTyper{} diff --git a/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/install/BUILD b/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/install/BUILD index de371ab22a8..949935bc416 100644 --- a/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/install/BUILD +++ b/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/install/BUILD @@ -10,9 +10,8 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/client-go/pkg/apis/clientauthentication/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/client-go/pkg/apis/clientauthentication:go_default_library", "//vendor/k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1:go_default_library", ], diff --git a/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/install/install.go b/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/install/install.go index f1b5ddebb40..1b7b5f94900 100644 --- a/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/install/install.go +++ b/staging/src/k8s.io/client-go/pkg/apis/clientauthentication/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/pkg/apis/clientauthentication" "k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: clientauthentication.GroupName, - VersionPreferenceOrder: []string{v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: clientauthentication.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(clientauthentication.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/BUILD b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/BUILD index 750ffd6f537..0d73975834c 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/BUILD +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/BUILD @@ -18,6 +18,7 @@ go_library( importpath = "k8s.io/client-go/plugin/pkg/client/auth/openstack", deps = [ "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/gophercloud/gophercloud:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack.go index 490eff12817..7f524a64971 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack.go @@ -23,6 +23,7 @@ import ( "time" "github.com/golang/glog" + "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack" "k8s.io/apimachinery/pkg/util/net" @@ -42,8 +43,7 @@ const DefaultTTLDuration = 10 * time.Minute // the environment variables to determine the client identity, and generates a // token which will be inserted into the request header later. type openstackAuthProvider struct { - ttl time.Duration - + ttl time.Duration tokenGetter TokenGetter } @@ -52,13 +52,23 @@ type TokenGetter interface { Token() (string, error) } -type tokenGetter struct{} +type tokenGetter struct { + authOpt *gophercloud.AuthOptions +} // Token creates a token by authenticate with keystone. -func (*tokenGetter) Token() (string, error) { - options, err := openstack.AuthOptionsFromEnv() - if err != nil { - return "", fmt.Errorf("failed to read openstack env vars: %s", err) +func (t *tokenGetter) Token() (string, error) { + var options gophercloud.AuthOptions + var err error + if t.authOpt == nil { + // reads the config from the environment + glog.V(4).Info("reading openstack config from the environment variables") + options, err = openstack.AuthOptionsFromEnv() + if err != nil { + return "", fmt.Errorf("failed to read openstack env vars: %s", err) + } + } else { + options = *t.authOpt } client, err := openstack.AuthenticatedClient(options) if err != nil { @@ -126,7 +136,7 @@ func (t *tokenRoundTripper) WrappedRoundTripper() http.RoundTripper { return t.R // newOpenstackAuthProvider creates an auth provider which works with openstack // environment. -func newOpenstackAuthProvider(clusterAddress string, config map[string]string, persister restclient.AuthProviderConfigPersister) (restclient.AuthProvider, error) { +func newOpenstackAuthProvider(_ string, config map[string]string, persister restclient.AuthProviderConfigPersister) (restclient.AuthProvider, error) { var ttlDuration time.Duration var err error @@ -145,11 +155,27 @@ func newOpenstackAuthProvider(clusterAddress string, config map[string]string, p } } - // TODO: read/persist client configuration(OS_XXX env vars) in config + authOpt := gophercloud.AuthOptions{ + IdentityEndpoint: config["identityEndpoint"], + Username: config["username"], + Password: config["password"], + DomainName: config["name"], + TenantID: config["tenantId"], + TenantName: config["tenantName"], + } + + getter := tokenGetter{} + // not empty + if (authOpt != gophercloud.AuthOptions{}) { + if len(authOpt.IdentityEndpoint) == 0 { + return nil, fmt.Errorf("empty %q in the config for openstack auth provider", "identityEndpoint") + } + getter.authOpt = &authOpt + } return &openstackAuthProvider{ ttl: ttlDuration, - tokenGetter: &tokenGetter{}, + tokenGetter: &getter, }, nil } diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack_test.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack_test.go index 411bec70f7f..24d55b9fcaf 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack_test.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/openstack/openstack_test.go @@ -114,3 +114,60 @@ func TestOpenstackAuthProvider(t *testing.T) { } } + +type fakePersister struct{} + +func (i *fakePersister) Persist(map[string]string) error { + return nil +} + +func TestNewOpenstackAuthProvider(t *testing.T) { + tests := []struct { + name string + config map[string]string + expectError bool + }{ + { + name: "normal config without openstack configurations", + config: map[string]string{ + "ttl": "1s", + "foo": "bar", + }, + }, + { + name: "openstack auth provider: missing identityEndpoint", + config: map[string]string{ + "ttl": "1s", + "foo": "bar", + "username": "xyz", + "password": "123", + "tenantName": "admin", + }, + expectError: true, + }, + { + name: "openstack auth provider", + config: map[string]string{ + "ttl": "1s", + "foo": "bar", + "identityEndpoint": "http://controller:35357/v3", + "username": "xyz", + "password": "123", + "tenantName": "admin", + }, + }, + } + + for _, test := range tests { + _, err := newOpenstackAuthProvider("test", test.config, &fakePersister{}) + if err != nil { + if !test.expectError { + t.Errorf("unexpected error: %v", err) + } + } else { + if test.expectError { + t.Error("expect error, but nil") + } + } + } +} diff --git a/pkg/kubectl/categories/BUILD b/staging/src/k8s.io/client-go/restmapper/BUILD similarity index 63% rename from pkg/kubectl/categories/BUILD rename to staging/src/k8s.io/client-go/restmapper/BUILD index 2a7d7588ec3..afd6762fd74 100644 --- a/pkg/kubectl/categories/BUILD +++ b/staging/src/k8s.io/client-go/restmapper/BUILD @@ -2,10 +2,17 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["categories.go"], - importpath = "k8s.io/kubernetes/pkg/kubectl/categories", + srcs = [ + "category_expansion.go", + "discovery.go", + "shortcut.go", + ], + importpath = "k8s.io/client-go/restmapper", visibility = ["//visibility:public"], deps = [ + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/client-go/discovery:go_default_library", ], @@ -13,10 +20,17 @@ go_library( go_test( name = "go_default_test", - srcs = ["categories_test.go"], + srcs = [ + "category_expansion_test.go", + "discovery_test.go", + "shortcut_test.go", + ], embed = [":go_default_library"], deps = [ "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/version:go_default_library", diff --git a/pkg/kubectl/categories/categories.go b/staging/src/k8s.io/client-go/restmapper/category_expansion.go similarity index 53% rename from pkg/kubectl/categories/categories.go rename to staging/src/k8s.io/client-go/restmapper/category_expansion.go index 29bcc5d3948..1620bbcf810 100644 --- a/pkg/kubectl/categories/categories.go +++ b/staging/src/k8s.io/client-go/restmapper/category_expansion.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package categories +package restmapper import ( "k8s.io/apimachinery/pkg/runtime/schema" @@ -33,6 +33,7 @@ type SimpleCategoryExpander struct { Expansions map[string][]schema.GroupResource } +// Expand fulfills CategoryExpander func (e SimpleCategoryExpander) Expand(category string) ([]schema.GroupResource, bool) { ret, ok := e.Expansions[category] return ret, ok @@ -41,33 +42,32 @@ func (e SimpleCategoryExpander) Expand(category string) ([]schema.GroupResource, // discoveryCategoryExpander struct lets a REST Client wrapper (discoveryClient) to retrieve list of APIResourceList, // and then convert to fallbackExpander type discoveryCategoryExpander struct { - fallbackExpander CategoryExpander - discoveryClient discovery.DiscoveryInterface + discoveryClient discovery.DiscoveryInterface } // NewDiscoveryCategoryExpander returns a category expander that makes use of the "categories" fields from // the API, found through the discovery client. In case of any error or no category found (which likely // means we're at a cluster prior to categories support, fallback to the expander provided. -func NewDiscoveryCategoryExpander(fallbackExpander CategoryExpander, client discovery.DiscoveryInterface) (discoveryCategoryExpander, error) { +func NewDiscoveryCategoryExpander(client discovery.DiscoveryInterface) CategoryExpander { if client == nil { panic("Please provide discovery client to shortcut expander") } - return discoveryCategoryExpander{fallbackExpander: fallbackExpander, discoveryClient: client}, nil + return discoveryCategoryExpander{discoveryClient: client} } +// Expand fulfills CategoryExpander func (e discoveryCategoryExpander) Expand(category string) ([]schema.GroupResource, bool) { // Get all supported resources for groups and versions from server, if no resource found, fallback anyway. apiResourceLists, _ := e.discoveryClient.ServerResources() if len(apiResourceLists) == 0 { - return e.fallbackExpander.Expand(category) + return nil, false } discoveredExpansions := map[string][]schema.GroupResource{} - for _, apiResourceList := range apiResourceLists { gv, err := schema.ParseGroupVersion(apiResourceList.GroupVersion) if err != nil { - return e.fallbackExpander.Expand(category) + continue } // Collect GroupVersions by categories for _, apiResource := range apiResourceList.APIResources { @@ -83,64 +83,15 @@ func (e discoveryCategoryExpander) Expand(category string) ([]schema.GroupResour } } - if len(discoveredExpansions) == 0 { - // We don't know if the server really don't have any resource with categories, - // or we're on a cluster version prior to categories support. Anyways, fallback. - return e.fallbackExpander.Expand(category) - } - ret, ok := discoveredExpansions[category] return ret, ok } -// discoveryFilteredExpander expands the given CategoryExpander (delegate) to filter group and resource returned from server -type discoveryFilteredExpander struct { - delegate CategoryExpander - - discoveryClient discovery.DiscoveryInterface -} - -// NewDiscoveryFilteredExpander returns a category expander that filters the returned groupresources -// by what the server has available -func NewDiscoveryFilteredExpander(delegate CategoryExpander, client discovery.DiscoveryInterface) (discoveryFilteredExpander, error) { - if client == nil { - panic("Please provide discovery client to shortcut expander") - } - return discoveryFilteredExpander{delegate: delegate, discoveryClient: client}, nil -} - -func (e discoveryFilteredExpander) Expand(category string) ([]schema.GroupResource, bool) { - delegateExpansion, ok := e.delegate.Expand(category) - - // Check if we have access to server resources - apiResources, err := e.discoveryClient.ServerResources() - if err != nil { - return delegateExpansion, ok - } - - availableResources, err := discovery.GroupVersionResources(apiResources) - if err != nil { - return delegateExpansion, ok - } - - available := []schema.GroupResource{} - for _, requestedResource := range delegateExpansion { - for availableResource := range availableResources { - if requestedResource.Group == availableResource.Group && - requestedResource.Resource == availableResource.Resource { - available = append(available, requestedResource) - break - } - } - } - - return available, ok -} - // UnionCategoryExpander implements CategoryExpander interface. // It maps given category string to union of expansions returned by all the CategoryExpanders in the list. type UnionCategoryExpander []CategoryExpander +// Expand fulfills CategoryExpander func (u UnionCategoryExpander) Expand(category string) ([]schema.GroupResource, bool) { ret := []schema.GroupResource{} ok := false @@ -166,27 +117,3 @@ func (u UnionCategoryExpander) Expand(category string) ([]schema.GroupResource, return ret, ok } - -// legacyUserResources are the resource names that apply to the primary, user facing resources used by -// client tools. They are in deletion-first order - dependent resources should be last. -// Should remain exported in order to expose a current list of resources to downstream -// composition that wants to build on the concept of 'all' for their CLIs. -var legacyUserResources = []schema.GroupResource{ - {Group: "", Resource: "pods"}, - {Group: "", Resource: "replicationcontrollers"}, - {Group: "", Resource: "services"}, - {Group: "apps", Resource: "statefulsets"}, - {Group: "autoscaling", Resource: "horizontalpodautoscalers"}, - {Group: "batch", Resource: "jobs"}, - {Group: "batch", Resource: "cronjobs"}, - {Group: "extensions", Resource: "daemonsets"}, - {Group: "extensions", Resource: "deployments"}, - {Group: "extensions", Resource: "replicasets"}, -} - -// LegacyCategoryExpander is the old hardcoded expansion -var LegacyCategoryExpander CategoryExpander = SimpleCategoryExpander{ - Expansions: map[string][]schema.GroupResource{ - "all": legacyUserResources, - }, -} diff --git a/pkg/kubectl/categories/categories_test.go b/staging/src/k8s.io/client-go/restmapper/category_expansion_test.go similarity index 59% rename from pkg/kubectl/categories/categories_test.go rename to staging/src/k8s.io/client-go/restmapper/category_expansion_test.go index 5344bad2242..8537a6b49b9 100644 --- a/pkg/kubectl/categories/categories_test.go +++ b/staging/src/k8s.io/client-go/restmapper/category_expansion_test.go @@ -14,19 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package categories +package restmapper import ( "reflect" "testing" - "github.com/googleapis/gnostic/OpenAPIv2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/version" - "k8s.io/client-go/discovery" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/rest/fake" ) func TestCategoryExpansion(t *testing.T) { @@ -46,23 +41,28 @@ func TestCategoryExpansion(t *testing.T) { name: "all-replacement", arg: "all", expected: []schema.GroupResource{ - {Resource: "pods"}, - {Resource: "replicationcontrollers"}, - {Resource: "services"}, - {Resource: "statefulsets", Group: "apps"}, - {Resource: "horizontalpodautoscalers", Group: "autoscaling"}, - {Resource: "jobs", Group: "batch"}, - {Resource: "cronjobs", Group: "batch"}, - {Resource: "daemonsets", Group: "extensions"}, - {Resource: "deployments", Group: "extensions"}, - {Resource: "replicasets", Group: "extensions"}, + {Resource: "one"}, + {Resource: "two"}, + {Resource: "three", Group: "alpha"}, + {Resource: "four", Group: "bravo"}, }, expectedOk: true, }, } for _, test := range tests { - actual, actualOk := LegacyCategoryExpander.Expand(test.arg) + simpleCategoryExpander := SimpleCategoryExpander{ + Expansions: map[string][]schema.GroupResource{ + "all": { + {Group: "", Resource: "one"}, + {Group: "", Resource: "two"}, + {Group: "alpha", Resource: "three"}, + {Group: "bravo", Resource: "four"}, + }, + }, + } + + actual, actualOk := simpleCategoryExpander.Expand(test.arg) if e, a := test.expected, actual; !reflect.DeepEqual(e, a) { t.Errorf("%s: expected %s, got %s", test.name, e, a) } @@ -135,10 +135,7 @@ func TestDiscoveryCategoryExpander(t *testing.T) { dc.serverResourcesHandler = func() ([]*metav1.APIResourceList, error) { return test.serverResponse, nil } - expander, err := NewDiscoveryCategoryExpander(SimpleCategoryExpander{}, dc) - if err != nil { - t.Fatalf("unexpected error %v", err) - } + expander := NewDiscoveryCategoryExpander(dc) expanded, _ := expander.Expand(test.category) if !reflect.DeepEqual(expanded, test.expected) { t.Errorf("expected %v, got %v", test.expected, expanded) @@ -146,41 +143,3 @@ func TestDiscoveryCategoryExpander(t *testing.T) { } } - -type fakeDiscoveryClient struct { - serverResourcesHandler func() ([]*metav1.APIResourceList, error) -} - -var _ discovery.DiscoveryInterface = &fakeDiscoveryClient{} - -func (c *fakeDiscoveryClient) RESTClient() restclient.Interface { - return &fake.RESTClient{} -} - -func (c *fakeDiscoveryClient) ServerGroups() (*metav1.APIGroupList, error) { - return nil, nil -} - -func (c *fakeDiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) { - return &metav1.APIResourceList{}, nil -} - -func (c *fakeDiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) { - return c.serverResourcesHandler() -} - -func (c *fakeDiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) { - return nil, nil -} - -func (c *fakeDiscoveryClient) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) { - return nil, nil -} - -func (c *fakeDiscoveryClient) ServerVersion() (*version.Info, error) { - return &version.Info{}, nil -} - -func (c *fakeDiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) { - return &openapi_v2.Document{}, nil -} diff --git a/staging/src/k8s.io/client-go/discovery/restmapper.go b/staging/src/k8s.io/client-go/restmapper/discovery.go similarity index 95% rename from staging/src/k8s.io/client-go/discovery/restmapper.go rename to staging/src/k8s.io/client-go/restmapper/discovery.go index 17ddc6c5c22..58887cd89ca 100644 --- a/staging/src/k8s.io/client-go/discovery/restmapper.go +++ b/staging/src/k8s.io/client-go/restmapper/discovery.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package discovery +package restmapper import ( "fmt" @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" "github.com/golang/glog" ) @@ -37,9 +38,9 @@ type APIGroupResources struct { VersionedResources map[string][]metav1.APIResource } -// NewRESTMapper returns a PriorityRESTMapper based on the discovered +// NewDiscoveryRESTMapper returns a PriorityRESTMapper based on the discovered // groups and resources passed in. -func NewRESTMapper(groupResources []*APIGroupResources) meta.RESTMapper { +func NewDiscoveryRESTMapper(groupResources []*APIGroupResources) meta.RESTMapper { unionMapper := meta.MultiRESTMapper{} var groupPriority []string @@ -141,7 +142,7 @@ func NewRESTMapper(groupResources []*APIGroupResources) meta.RESTMapper { // GetAPIGroupResources uses the provided discovery client to gather // discovery information and populate a slice of APIGroupResources. -func GetAPIGroupResources(cl DiscoveryInterface) ([]*APIGroupResources, error) { +func GetAPIGroupResources(cl discovery.DiscoveryInterface) ([]*APIGroupResources, error) { apiGroups, err := cl.ServerGroups() if err != nil { if apiGroups == nil || len(apiGroups.Groups) == 0 { @@ -177,13 +178,13 @@ func GetAPIGroupResources(cl DiscoveryInterface) ([]*APIGroupResources, error) { type DeferredDiscoveryRESTMapper struct { initMu sync.Mutex delegate meta.RESTMapper - cl CachedDiscoveryInterface + cl discovery.CachedDiscoveryInterface } // NewDeferredDiscoveryRESTMapper returns a // DeferredDiscoveryRESTMapper that will lazily query the provided // client for discovery information to do REST mappings. -func NewDeferredDiscoveryRESTMapper(cl CachedDiscoveryInterface) *DeferredDiscoveryRESTMapper { +func NewDeferredDiscoveryRESTMapper(cl discovery.CachedDiscoveryInterface) *DeferredDiscoveryRESTMapper { return &DeferredDiscoveryRESTMapper{ cl: cl, } @@ -202,7 +203,7 @@ func (d *DeferredDiscoveryRESTMapper) getDelegate() (meta.RESTMapper, error) { return nil, err } - d.delegate = NewRESTMapper(groupResources) + d.delegate = NewDiscoveryRESTMapper(groupResources) return d.delegate, err } diff --git a/staging/src/k8s.io/client-go/discovery/restmapper_test.go b/staging/src/k8s.io/client-go/restmapper/discovery_test.go similarity index 99% rename from staging/src/k8s.io/client-go/discovery/restmapper_test.go rename to staging/src/k8s.io/client-go/restmapper/discovery_test.go index 4489bdb5fb6..1b8e7c4f056 100644 --- a/staging/src/k8s.io/client-go/discovery/restmapper_test.go +++ b/staging/src/k8s.io/client-go/restmapper/discovery_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package discovery_test +package restmapper import ( "reflect" @@ -94,7 +94,7 @@ func TestRESTMapper(t *testing.T) { }, } - restMapper := NewRESTMapper(resources) + restMapper := NewDiscoveryRESTMapper(resources) kindTCs := []struct { input schema.GroupVersionResource diff --git a/pkg/kubectl/cmd/util/shortcut_restmapper.go b/staging/src/k8s.io/client-go/restmapper/shortcut.go similarity index 87% rename from pkg/kubectl/cmd/util/shortcut_restmapper.go rename to staging/src/k8s.io/client-go/restmapper/shortcut.go index 2ef8ce02bd6..d9f4be0b6b1 100644 --- a/pkg/kubectl/cmd/util/shortcut_restmapper.go +++ b/staging/src/k8s.io/client-go/restmapper/shortcut.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package restmapper import ( "strings" @@ -25,7 +25,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" - "k8s.io/kubernetes/pkg/kubectl" ) // shortcutExpander is a RESTMapper that can be used for Kubernetes resources. It expands the resource first, then invokes the wrapped @@ -37,34 +36,42 @@ type shortcutExpander struct { var _ meta.RESTMapper = &shortcutExpander{} -func NewShortcutExpander(delegate meta.RESTMapper, client discovery.DiscoveryInterface) shortcutExpander { +// NewShortcutExpander wraps a restmapper in a layer that expands shortcuts found via discovery +func NewShortcutExpander(delegate meta.RESTMapper, client discovery.DiscoveryInterface) meta.RESTMapper { return shortcutExpander{RESTMapper: delegate, discoveryClient: client} } +// KindFor fulfills meta.RESTMapper func (e shortcutExpander) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) { return e.RESTMapper.KindFor(e.expandResourceShortcut(resource)) } +// KindsFor fulfills meta.RESTMapper func (e shortcutExpander) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) { return e.RESTMapper.KindsFor(e.expandResourceShortcut(resource)) } +// ResourcesFor fulfills meta.RESTMapper func (e shortcutExpander) ResourcesFor(resource schema.GroupVersionResource) ([]schema.GroupVersionResource, error) { return e.RESTMapper.ResourcesFor(e.expandResourceShortcut(resource)) } +// ResourceFor fulfills meta.RESTMapper func (e shortcutExpander) ResourceFor(resource schema.GroupVersionResource) (schema.GroupVersionResource, error) { return e.RESTMapper.ResourceFor(e.expandResourceShortcut(resource)) } +// ResourceSingularizer fulfills meta.RESTMapper func (e shortcutExpander) ResourceSingularizer(resource string) (string, error) { return e.RESTMapper.ResourceSingularizer(e.expandResourceShortcut(schema.GroupVersionResource{Resource: resource}).Resource) } +// RESTMapping fulfills meta.RESTMapper func (e shortcutExpander) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { return e.RESTMapper.RESTMapping(gk, versions...) } +// RESTMappings fulfills meta.RESTMapper func (e shortcutExpander) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) { return e.RESTMapper.RESTMappings(gk, versions...) } @@ -73,8 +80,8 @@ func (e shortcutExpander) RESTMappings(gk schema.GroupKind, versions ...string) // First the list of potential resources will be taken from the API server. // Next we will append the hardcoded list of resources - to be backward compatible with old servers. // NOTE that the list is ordered by group priority. -func (e shortcutExpander) getShortcutMappings() ([]*metav1.APIResourceList, []kubectl.ResourceShortcuts, error) { - res := []kubectl.ResourceShortcuts{} +func (e shortcutExpander) getShortcutMappings() ([]*metav1.APIResourceList, []resourceShortcuts, error) { + res := []resourceShortcuts{} // get server resources // This can return an error *and* the results it was able to find. We don't need to fail on the error. apiResList, err := e.discoveryClient.ServerResources() @@ -89,7 +96,7 @@ func (e shortcutExpander) getShortcutMappings() ([]*metav1.APIResourceList, []ku } for _, apiRes := range apiResources.APIResources { for _, shortName := range apiRes.ShortNames { - rs := kubectl.ResourceShortcuts{ + rs := resourceShortcuts{ ShortForm: schema.GroupResource{Group: gv.Group, Resource: shortName}, LongForm: schema.GroupResource{Group: gv.Group, Resource: apiRes.Name}, } @@ -98,8 +105,6 @@ func (e shortcutExpander) getShortcutMappings() ([]*metav1.APIResourceList, []ku } } - // append hardcoded short forms at the end of the list - res = append(res, kubectl.ResourcesShortcutStatic...) return apiResList, res, nil } @@ -158,3 +163,10 @@ func (e shortcutExpander) expandResourceShortcut(resource schema.GroupVersionRes return resource } + +// ResourceShortcuts represents a structure that holds the information how to +// transition from resource's shortcut to its full name. +type resourceShortcuts struct { + ShortForm schema.GroupResource + LongForm schema.GroupResource +} diff --git a/pkg/kubectl/cmd/util/shortcut_restmapper_test.go b/staging/src/k8s.io/client-go/restmapper/shortcut_test.go similarity index 56% rename from pkg/kubectl/cmd/util/shortcut_restmapper_test.go rename to staging/src/k8s.io/client-go/restmapper/shortcut_test.go index 34353020ff1..dbc288fadff 100644 --- a/pkg/kubectl/cmd/util/shortcut_restmapper_test.go +++ b/staging/src/k8s.io/client-go/restmapper/shortcut_test.go @@ -14,15 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package restmapper import ( "testing" - "k8s.io/apimachinery/pkg/api/meta/testrestmapper" + "github.com/googleapis/gnostic/OpenAPIv2" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/apimachinery/pkg/version" + "k8s.io/client-go/discovery" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/rest/fake" ) func TestReplaceAliases(t *testing.T) { @@ -32,12 +38,6 @@ func TestReplaceAliases(t *testing.T) { expected schema.GroupVersionResource srvRes []*metav1.APIResourceList }{ - { - name: "rc-resolves-to-replicationcontrollers", - arg: "rc", - expected: schema.GroupVersionResource{Resource: "replicationcontrollers"}, - srvRes: []*metav1.APIResourceList{}, - }, { name: "storageclasses-no-replacement", arg: "storageclasses", @@ -126,13 +126,13 @@ func TestReplaceAliases(t *testing.T) { }, } - ds := &fakeDiscoveryClient{} - mapper := NewShortcutExpander(testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme), ds) - for _, test := range tests { + ds := &fakeDiscoveryClient{} ds.serverResourcesHandler = func() ([]*metav1.APIResourceList, error) { return test.srvRes, nil } + mapper := NewShortcutExpander(&fakeRESTMapper{}, ds).(shortcutExpander) + actual := mapper.expandResourceShortcut(schema.GroupVersionResource{Resource: test.arg}) if actual != test.expected { t.Errorf("%s: unexpected argument: expected %s, got %s", test.name, test.expected, actual) @@ -143,12 +143,12 @@ func TestReplaceAliases(t *testing.T) { func TestKindFor(t *testing.T) { tests := []struct { in schema.GroupVersionResource - expected schema.GroupVersionKind + expected schema.GroupVersionResource srvRes []*metav1.APIResourceList }{ { in: schema.GroupVersionResource{Group: "storage.k8s.io", Version: "", Resource: "sc"}, - expected: schema.GroupVersionKind{Group: "storage.k8s.io", Version: "v1", Kind: "StorageClass"}, + expected: schema.GroupVersionResource{Group: "storage.k8s.io", Version: "", Resource: "storageclasses"}, srvRes: []*metav1.APIResourceList{ { GroupVersion: "storage.k8s.io/v1", @@ -163,7 +163,7 @@ func TestKindFor(t *testing.T) { }, { in: schema.GroupVersionResource{Group: "", Version: "", Resource: "sc"}, - expected: schema.GroupVersionKind{Group: "storage.k8s.io", Version: "v1", Kind: "StorageClass"}, + expected: schema.GroupVersionResource{Group: "storage.k8s.io", Version: "", Resource: "storageclasses"}, srvRes: []*metav1.APIResourceList{ { GroupVersion: "storage.k8s.io/v1", @@ -178,19 +178,112 @@ func TestKindFor(t *testing.T) { }, } - ds := &fakeDiscoveryClient{} - mapper := NewShortcutExpander(testrestmapper.TestOnlyStaticRESTMapper(legacyscheme.Registry, legacyscheme.Scheme), ds) - for i, test := range tests { + ds := &fakeDiscoveryClient{} ds.serverResourcesHandler = func() ([]*metav1.APIResourceList, error) { return test.srvRes, nil } - ret, err := mapper.KindFor(test.in) - if err != nil { - t.Errorf("%d: unexpected error returned %s", i, err.Error()) - } - if ret != test.expected { - t.Errorf("%d: unexpected data returned %#v, expected %#v", i, ret, test.expected) + + delegate := &fakeRESTMapper{} + mapper := NewShortcutExpander(delegate, ds) + + mapper.KindFor(test.in) + if delegate.kindForInput != test.expected { + t.Errorf("%d: unexpected data returned %#v, expected %#v", i, delegate.kindForInput, test.expected) } } } + +type fakeRESTMapper struct { + kindForInput schema.GroupVersionResource +} + +func (f *fakeRESTMapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) { + f.kindForInput = resource + return schema.GroupVersionKind{}, nil +} + +func (f *fakeRESTMapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) { + return nil, nil +} + +func (f *fakeRESTMapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) { + return schema.GroupVersionResource{}, nil +} + +func (f *fakeRESTMapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) { + return nil, nil +} + +func (f *fakeRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { + return nil, nil +} + +func (f *fakeRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) { + return nil, nil +} + +func (f *fakeRESTMapper) ResourceSingularizer(resource string) (singular string, err error) { + return "", nil +} + +type fakeDiscoveryClient struct { + serverResourcesHandler func() ([]*metav1.APIResourceList, error) +} + +var _ discovery.DiscoveryInterface = &fakeDiscoveryClient{} + +func (c *fakeDiscoveryClient) RESTClient() restclient.Interface { + return &fake.RESTClient{} +} + +func (c *fakeDiscoveryClient) ServerGroups() (*metav1.APIGroupList, error) { + return &metav1.APIGroupList{ + Groups: []metav1.APIGroup{ + { + Name: "a", + Versions: []metav1.GroupVersionForDiscovery{ + { + GroupVersion: "a/v1", + Version: "v1", + }, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "a/v1", + Version: "v1", + }, + }, + }, + }, nil +} + +func (c *fakeDiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) { + if groupVersion == "a/v1" { + return &metav1.APIResourceList{APIResources: []metav1.APIResource{{Name: "widgets", Kind: "Widget"}}}, nil + } + + return nil, errors.NewNotFound(schema.GroupResource{}, "") +} + +func (c *fakeDiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) { + if c.serverResourcesHandler != nil { + return c.serverResourcesHandler() + } + return []*metav1.APIResourceList{}, nil +} + +func (c *fakeDiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) { + return nil, nil +} + +func (c *fakeDiscoveryClient) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) { + return nil, nil +} + +func (c *fakeDiscoveryClient) ServerVersion() (*version.Info, error) { + return &version.Info{}, nil +} + +func (c *fakeDiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) { + return &openapi_v2.Document{}, nil +} diff --git a/staging/src/k8s.io/client-go/scale/BUILD b/staging/src/k8s.io/client-go/scale/BUILD index aa6e04ee4aa..fea5cedc7c5 100644 --- a/staging/src/k8s.io/client-go/scale/BUILD +++ b/staging/src/k8s.io/client-go/scale/BUILD @@ -48,10 +48,10 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/discovery/fake:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/rest/fake:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/testing:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/scale/client_test.go b/staging/src/k8s.io/client-go/scale/client_test.go index d7699453405..a3a8c037dd7 100644 --- a/staging/src/k8s.io/client-go/scale/client_test.go +++ b/staging/src/k8s.io/client-go/scale/client_test.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/client-go/discovery" fakedisco "k8s.io/client-go/discovery/fake" "k8s.io/client-go/dynamic" fakerest "k8s.io/client-go/rest/fake" @@ -40,6 +39,7 @@ import ( autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" extv1beta1 "k8s.io/api/extensions/v1beta1" + "k8s.io/client-go/restmapper" coretesting "k8s.io/client-go/testing" ) @@ -96,11 +96,11 @@ func fakeScaleClient(t *testing.T) (ScalesGetter, []schema.GroupResource) { }, } - restMapperRes, err := discovery.GetAPIGroupResources(fakeDiscoveryClient) + restMapperRes, err := restmapper.GetAPIGroupResources(fakeDiscoveryClient) if err != nil { t.Fatalf("unexpected error while constructing resource list from fake discovery client: %v", err) } - restMapper := discovery.NewRESTMapper(restMapperRes) + restMapper := restmapper.NewDiscoveryRESTMapper(restMapperRes) autoscalingScale := &autoscalingv1.Scale{ TypeMeta: metav1.TypeMeta{ diff --git a/staging/src/k8s.io/client-go/testing/actions.go b/staging/src/k8s.io/client-go/testing/actions.go index 0210b67845c..232cf21e059 100644 --- a/staging/src/k8s.io/client-go/testing/actions.go +++ b/staging/src/k8s.io/client-go/testing/actions.go @@ -58,6 +58,16 @@ func NewGetSubresourceAction(resource schema.GroupVersionResource, namespace, su return action } +func NewRootGetSubresourceAction(resource schema.GroupVersionResource, subresource, name string) GetActionImpl { + action := GetActionImpl{} + action.Verb = "get" + action.Resource = resource + action.Subresource = subresource + action.Name = name + + return action +} + func NewRootListAction(resource schema.GroupVersionResource, kind schema.GroupVersionKind, opts interface{}) ListActionImpl { action := ListActionImpl{} action.Verb = "list" @@ -81,20 +91,6 @@ func NewListAction(resource schema.GroupVersionResource, kind schema.GroupVersio return action } -func NewListSubresourceAction(resource schema.GroupVersionResource, name, subresource string, kind schema.GroupVersionKind, namespace string, opts interface{}) ListActionImpl { - action := ListActionImpl{} - action.Verb = "list" - action.Resource = resource - action.Subresource = subresource - action.Kind = kind - action.Namespace = namespace - action.Name = name - labelSelector, fieldSelector, _ := ExtractFromListOptions(opts) - action.ListRestrictions = ListRestrictions{labelSelector, fieldSelector} - - return action -} - func NewRootCreateAction(resource schema.GroupVersionResource, object runtime.Object) CreateActionImpl { action := CreateActionImpl{} action.Verb = "create" @@ -114,12 +110,23 @@ func NewCreateAction(resource schema.GroupVersionResource, namespace string, obj return action } -func NewCreateSubresourceAction(resource schema.GroupVersionResource, name, subresource string, namespace string, object runtime.Object) CreateActionImpl { +func NewRootCreateSubresourceAction(resource schema.GroupVersionResource, name, subresource string, object runtime.Object) CreateActionImpl { action := CreateActionImpl{} action.Verb = "create" action.Resource = resource action.Subresource = subresource + action.Name = name + action.Object = object + + return action +} + +func NewCreateSubresourceAction(resource schema.GroupVersionResource, name, subresource, namespace string, object runtime.Object) CreateActionImpl { + action := CreateActionImpl{} + action.Verb = "create" + action.Resource = resource action.Namespace = namespace + action.Subresource = subresource action.Name = name action.Object = object diff --git a/staging/src/k8s.io/client-go/testing/fixture_test.go b/staging/src/k8s.io/client-go/testing/fixture_test.go index 5c912c2b291..405fe1a7e23 100644 --- a/staging/src/k8s.io/client-go/testing/fixture_test.go +++ b/staging/src/k8s.io/client-go/testing/fixture_test.go @@ -63,6 +63,9 @@ func TestWatchCallNonNamespace(t *testing.T) { codecs := serializer.NewCodecFactory(scheme) o := NewObjectTracker(scheme, codecs.UniversalDecoder()) watch, err := o.Watch(testResource, ns) + if err != nil { + t.Fatalf("test resource watch failed in %s: %v ", ns, err) + } go func() { err := o.Create(testResource, testObj, ns) if err != nil { @@ -85,7 +88,13 @@ func TestWatchCallAllNamespace(t *testing.T) { codecs := serializer.NewCodecFactory(scheme) o := NewObjectTracker(scheme, codecs.UniversalDecoder()) w, err := o.Watch(testResource, "test_namespace") + if err != nil { + t.Fatalf("test resource watch failed in test_namespace: %v", err) + } wAll, err := o.Watch(testResource, "") + if err != nil { + t.Fatalf("test resource watch failed in all namespaces: %v", err) + } go func() { err := o.Create(testResource, testObj, ns) assert.NoError(t, err, "test resource creation failed") @@ -161,6 +170,9 @@ func TestWatchCallMultipleInvocation(t *testing.T) { for idx, watchNamespace := range watchNamespaces { i := idx w, err := o.Watch(testResource, watchNamespace) + if err != nil { + t.Fatalf("test resource watch failed in %s: %v", watchNamespace, err) + } go func() { assert.NoError(t, err, "watch invocation failed") for _, c := range cases { diff --git a/staging/src/k8s.io/client-go/util/workqueue/delaying_queue.go b/staging/src/k8s.io/client-go/util/workqueue/delaying_queue.go index c62ed32efa7..a37177425d7 100644 --- a/staging/src/k8s.io/client-go/util/workqueue/delaying_queue.go +++ b/staging/src/k8s.io/client-go/util/workqueue/delaying_queue.go @@ -45,7 +45,7 @@ func newDelayingQueue(clock clock.Clock, name string) DelayingInterface { ret := &delayingType{ Interface: NewNamed(name), clock: clock, - heartbeat: clock.Tick(maxWait), + heartbeat: clock.NewTicker(maxWait), stopCh: make(chan struct{}), waitingForAddCh: make(chan *waitFor, 1000), metrics: newRetryMetrics(name), @@ -67,10 +67,7 @@ type delayingType struct { stopCh chan struct{} // heartbeat ensures we wait no more than maxWait before firing - // - // TODO: replace with Ticker (and add to clock) so this can be cleaned up. - // clock.Tick will leak. - heartbeat <-chan time.Time + heartbeat clock.Ticker // waitingForAddCh is a buffered channel that feeds waitingForAdd waitingForAddCh chan *waitFor @@ -138,6 +135,7 @@ func (pq waitForPriorityQueue) Peek() interface{} { func (q *delayingType) ShutDown() { q.Interface.ShutDown() close(q.stopCh) + q.heartbeat.Stop() } // AddAfter adds the given item to the work queue after the given delay @@ -209,7 +207,7 @@ func (q *delayingType) waitingLoop() { case <-q.stopCh: return - case <-q.heartbeat: + case <-q.heartbeat.C(): // continue the loop, which will add ready items case <-nextReadyAt: diff --git a/staging/src/k8s.io/client-go/util/workqueue/rate_limitting_queue_test.go b/staging/src/k8s.io/client-go/util/workqueue/rate_limitting_queue_test.go index 32d7fc90683..3fbe07d0d8b 100644 --- a/staging/src/k8s.io/client-go/util/workqueue/rate_limitting_queue_test.go +++ b/staging/src/k8s.io/client-go/util/workqueue/rate_limitting_queue_test.go @@ -30,7 +30,7 @@ func TestRateLimitingQueue(t *testing.T) { delayingQueue := &delayingType{ Interface: New(), clock: fakeClock, - heartbeat: fakeClock.Tick(maxWait), + heartbeat: fakeClock.NewTicker(maxWait), stopCh: make(chan struct{}), waitingForAddCh: make(chan *waitFor, 1000), metrics: newRetryMetrics(""), diff --git a/staging/src/k8s.io/code-generator/Godeps/Godeps.json b/staging/src/k8s.io/code-generator/Godeps/Godeps.json index 4c96df28aa1..ad8da9bbe1e 100644 --- a/staging/src/k8s.io/code-generator/Godeps/Godeps.json +++ b/staging/src/k8s.io/code-generator/Godeps/Godeps.json @@ -260,11 +260,11 @@ }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/generators", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" } ] } diff --git a/staging/src/k8s.io/code-generator/_examples/apiserver/apis/example/install/install.go b/staging/src/k8s.io/code-generator/_examples/apiserver/apis/example/install/install.go index 307cab45bc6..d056c5b7881 100644 --- a/staging/src/k8s.io/code-generator/_examples/apiserver/apis/example/install/install.go +++ b/staging/src/k8s.io/code-generator/_examples/apiserver/apis/example/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/code-generator/_examples/apiserver/apis/example" "k8s.io/code-generator/_examples/apiserver/apis/example/v1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: example.SchemeGroupVersion.Group, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: example.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1.SchemeGroupVersion.Version: v1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(example.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/code-generator/_examples/apiserver/apis/example2/install/install.go b/staging/src/k8s.io/code-generator/_examples/apiserver/apis/example2/install/install.go index 9fc20da1d00..1e786f5c530 100644 --- a/staging/src/k8s.io/code-generator/_examples/apiserver/apis/example2/install/install.go +++ b/staging/src/k8s.io/code-generator/_examples/apiserver/apis/example2/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/code-generator/_examples/apiserver/apis/example2" "k8s.io/code-generator/_examples/apiserver/apis/example2/v1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: example2.SchemeGroupVersion.Group, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: example2.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1.SchemeGroupVersion.Version: v1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(example2.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/scheme/register.go b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/scheme/register.go index da5d12676a7..97b03068e32 100644 --- a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/scheme/register.go +++ b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/scheme/register.go @@ -19,7 +19,6 @@ limitations under the License. package scheme import ( - registered "k8s.io/apimachinery/pkg/apimachinery/registered" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -32,15 +31,13 @@ var Scheme = runtime.NewScheme() var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) -var Registry = registered.NewAPIRegistrationManager() - func init() { v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - Install(Registry, Scheme) + Install(Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - example.Install(registry, scheme) - secondexample.Install(registry, scheme) +func Install(scheme *runtime.Scheme) { + example.Install(scheme) + secondexample.Install(scheme) } diff --git a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example/internalversion/example_client.go b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example/internalversion/example_client.go index 36bf92cb5ef..61d99cad021 100644 --- a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example/internalversion/example_client.go +++ b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example/internalversion/example_client.go @@ -66,17 +66,12 @@ func New(c rest.Interface) *ExampleClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("example.apiserver.code-generator.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("example.apiserver.code-generator.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("example.apiserver.code-generator.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example2/internalversion/example2_client.go b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example2/internalversion/example2_client.go index 682148d918b..0fcc7baa42e 100644 --- a/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example2/internalversion/example2_client.go +++ b/staging/src/k8s.io/code-generator/_examples/apiserver/clientset/internalversion/typed/example2/internalversion/example2_client.go @@ -66,17 +66,12 @@ func New(c rest.Interface) *SecondExampleClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("example.test.apiserver.code-generator.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("example.test.apiserver.code-generator.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("example.test.apiserver.code-generator.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/staging/src/k8s.io/code-generator/_examples/crd/apis/example/v1/types.go b/staging/src/k8s.io/code-generator/_examples/crd/apis/example/v1/types.go index 5c2ebc4d6f4..d79ea38b720 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/apis/example/v1/types.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/apis/example/v1/types.go @@ -45,3 +45,30 @@ type TestTypeList struct { type TestTypeStatus struct { Blah string } + +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type ClusterTestTypeList struct { + metav1.TypeMeta + metav1.ListMeta + Items []ClusterTestType +} + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/kubernetes/pkg/apis/autoscaling.Scale +// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/kubernetes/pkg/apis/autoscaling.Scale,result=k8s.io/kubernetes/pkg/apis/autoscaling.Scale + +type ClusterTestType struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + // +optional + Status ClusterTestTypeStatus `json:"status,omitempty"` +} + +type ClusterTestTypeStatus struct { + Blah string +} diff --git a/staging/src/k8s.io/code-generator/_examples/crd/apis/example/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/code-generator/_examples/crd/apis/example/v1/zz_generated.deepcopy.go index dae52ff1287..a3b4bfa9c21 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/apis/example/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/apis/example/v1/zz_generated.deepcopy.go @@ -24,6 +24,82 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterTestType) DeepCopyInto(out *ClusterTestType) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Status = in.Status + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterTestType. +func (in *ClusterTestType) DeepCopy() *ClusterTestType { + if in == nil { + return nil + } + out := new(ClusterTestType) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterTestType) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterTestTypeList) DeepCopyInto(out *ClusterTestTypeList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterTestType, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterTestTypeList. +func (in *ClusterTestTypeList) DeepCopy() *ClusterTestTypeList { + if in == nil { + return nil + } + out := new(ClusterTestTypeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterTestTypeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterTestTypeStatus) DeepCopyInto(out *ClusterTestTypeStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterTestTypeStatus. +func (in *ClusterTestTypeStatus) DeepCopy() *ClusterTestTypeStatus { + if in == nil { + return nil + } + out := new(ClusterTestTypeStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestType) DeepCopyInto(out *TestType) { *out = *in diff --git a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/clustertesttype.go b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/clustertesttype.go new file mode 100644 index 00000000000..a55b859e1f0 --- /dev/null +++ b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/clustertesttype.go @@ -0,0 +1,193 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" + v1 "k8s.io/code-generator/_examples/crd/apis/example/v1" + scheme "k8s.io/code-generator/_examples/crd/clientset/versioned/scheme" + autoscaling "k8s.io/kubernetes/pkg/apis/autoscaling" +) + +// ClusterTestTypesGetter has a method to return a ClusterTestTypeInterface. +// A group's client should implement this interface. +type ClusterTestTypesGetter interface { + ClusterTestTypes() ClusterTestTypeInterface +} + +// ClusterTestTypeInterface has methods to work with ClusterTestType resources. +type ClusterTestTypeInterface interface { + Create(*v1.ClusterTestType) (*v1.ClusterTestType, error) + Update(*v1.ClusterTestType) (*v1.ClusterTestType, error) + UpdateStatus(*v1.ClusterTestType) (*v1.ClusterTestType, error) + Delete(name string, options *meta_v1.DeleteOptions) error + DeleteCollection(options *meta_v1.DeleteOptions, listOptions meta_v1.ListOptions) error + Get(name string, options meta_v1.GetOptions) (*v1.ClusterTestType, error) + List(opts meta_v1.ListOptions) (*v1.ClusterTestTypeList, error) + Watch(opts meta_v1.ListOptions) (watch.Interface, error) + Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.ClusterTestType, err error) + GetScale(clusterTestTypeName string, options meta_v1.GetOptions) (*autoscaling.Scale, error) + UpdateScale(clusterTestTypeName string, scale *autoscaling.Scale) (*autoscaling.Scale, error) + + ClusterTestTypeExpansion +} + +// clusterTestTypes implements ClusterTestTypeInterface +type clusterTestTypes struct { + client rest.Interface +} + +// newClusterTestTypes returns a ClusterTestTypes +func newClusterTestTypes(c *ExampleV1Client) *clusterTestTypes { + return &clusterTestTypes{ + client: c.RESTClient(), + } +} + +// Get takes name of the clusterTestType, and returns the corresponding clusterTestType object, and an error if there is any. +func (c *clusterTestTypes) Get(name string, options meta_v1.GetOptions) (result *v1.ClusterTestType, err error) { + result = &v1.ClusterTestType{} + err = c.client.Get(). + Resource("clustertesttypes"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of ClusterTestTypes that match those selectors. +func (c *clusterTestTypes) List(opts meta_v1.ListOptions) (result *v1.ClusterTestTypeList, err error) { + result = &v1.ClusterTestTypeList{} + err = c.client.Get(). + Resource("clustertesttypes"). + VersionedParams(&opts, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested clusterTestTypes. +func (c *clusterTestTypes) Watch(opts meta_v1.ListOptions) (watch.Interface, error) { + opts.Watch = true + return c.client.Get(). + Resource("clustertesttypes"). + VersionedParams(&opts, scheme.ParameterCodec). + Watch() +} + +// Create takes the representation of a clusterTestType and creates it. Returns the server's representation of the clusterTestType, and an error, if there is any. +func (c *clusterTestTypes) Create(clusterTestType *v1.ClusterTestType) (result *v1.ClusterTestType, err error) { + result = &v1.ClusterTestType{} + err = c.client.Post(). + Resource("clustertesttypes"). + Body(clusterTestType). + Do(). + Into(result) + return +} + +// Update takes the representation of a clusterTestType and updates it. Returns the server's representation of the clusterTestType, and an error, if there is any. +func (c *clusterTestTypes) Update(clusterTestType *v1.ClusterTestType) (result *v1.ClusterTestType, err error) { + result = &v1.ClusterTestType{} + err = c.client.Put(). + Resource("clustertesttypes"). + Name(clusterTestType.Name). + Body(clusterTestType). + Do(). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + +func (c *clusterTestTypes) UpdateStatus(clusterTestType *v1.ClusterTestType) (result *v1.ClusterTestType, err error) { + result = &v1.ClusterTestType{} + err = c.client.Put(). + Resource("clustertesttypes"). + Name(clusterTestType.Name). + SubResource("status"). + Body(clusterTestType). + Do(). + Into(result) + return +} + +// Delete takes name of the clusterTestType and deletes it. Returns an error if one occurs. +func (c *clusterTestTypes) Delete(name string, options *meta_v1.DeleteOptions) error { + return c.client.Delete(). + Resource("clustertesttypes"). + Name(name). + Body(options). + Do(). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *clusterTestTypes) DeleteCollection(options *meta_v1.DeleteOptions, listOptions meta_v1.ListOptions) error { + return c.client.Delete(). + Resource("clustertesttypes"). + VersionedParams(&listOptions, scheme.ParameterCodec). + Body(options). + Do(). + Error() +} + +// Patch applies the patch and returns the patched clusterTestType. +func (c *clusterTestTypes) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.ClusterTestType, err error) { + result = &v1.ClusterTestType{} + err = c.client.Patch(pt). + Resource("clustertesttypes"). + SubResource(subresources...). + Name(name). + Body(data). + Do(). + Into(result) + return +} + +// GetScale takes name of the clusterTestType, and returns the corresponding autoscaling.Scale object, and an error if there is any. +func (c *clusterTestTypes) GetScale(clusterTestTypeName string, options meta_v1.GetOptions) (result *autoscaling.Scale, err error) { + result = &autoscaling.Scale{} + err = c.client.Get(). + Resource("clustertesttypes"). + Name(clusterTestTypeName). + SubResource("scale"). + VersionedParams(&options, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// UpdateScale takes the top resource name and the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *clusterTestTypes) UpdateScale(clusterTestTypeName string, scale *autoscaling.Scale) (result *autoscaling.Scale, err error) { + result = &autoscaling.Scale{} + err = c.client.Put(). + Resource("clustertesttypes"). + Name(clusterTestTypeName). + SubResource("scale"). + Body(scale). + Do(). + Into(result) + return +} diff --git a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/example_client.go b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/example_client.go index ea644545c80..25d80728c9f 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/example_client.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/example_client.go @@ -27,6 +27,7 @@ import ( type ExampleV1Interface interface { RESTClient() rest.Interface + ClusterTestTypesGetter TestTypesGetter } @@ -35,6 +36,10 @@ type ExampleV1Client struct { restClient rest.Interface } +func (c *ExampleV1Client) ClusterTestTypes() ClusterTestTypeInterface { + return newClusterTestTypes(c) +} + func (c *ExampleV1Client) TestTypes(namespace string) TestTypeInterface { return newTestTypes(c, namespace) } diff --git a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/fake/fake_clustertesttype.go b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/fake/fake_clustertesttype.go new file mode 100644 index 00000000000..ba468d7175e --- /dev/null +++ b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/fake/fake_clustertesttype.go @@ -0,0 +1,152 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + example_v1 "k8s.io/code-generator/_examples/crd/apis/example/v1" + autoscaling "k8s.io/kubernetes/pkg/apis/autoscaling" +) + +// FakeClusterTestTypes implements ClusterTestTypeInterface +type FakeClusterTestTypes struct { + Fake *FakeExampleV1 +} + +var clustertesttypesResource = schema.GroupVersionResource{Group: "example.crd.code-generator.k8s.io", Version: "v1", Resource: "clustertesttypes"} + +var clustertesttypesKind = schema.GroupVersionKind{Group: "example.crd.code-generator.k8s.io", Version: "v1", Kind: "ClusterTestType"} + +// Get takes name of the clusterTestType, and returns the corresponding clusterTestType object, and an error if there is any. +func (c *FakeClusterTestTypes) Get(name string, options v1.GetOptions) (result *example_v1.ClusterTestType, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(clustertesttypesResource, name), &example_v1.ClusterTestType{}) + if obj == nil { + return nil, err + } + return obj.(*example_v1.ClusterTestType), err +} + +// List takes label and field selectors, and returns the list of ClusterTestTypes that match those selectors. +func (c *FakeClusterTestTypes) List(opts v1.ListOptions) (result *example_v1.ClusterTestTypeList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(clustertesttypesResource, clustertesttypesKind, opts), &example_v1.ClusterTestTypeList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &example_v1.ClusterTestTypeList{} + for _, item := range obj.(*example_v1.ClusterTestTypeList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested clusterTestTypes. +func (c *FakeClusterTestTypes) Watch(opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(clustertesttypesResource, opts)) +} + +// Create takes the representation of a clusterTestType and creates it. Returns the server's representation of the clusterTestType, and an error, if there is any. +func (c *FakeClusterTestTypes) Create(clusterTestType *example_v1.ClusterTestType) (result *example_v1.ClusterTestType, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(clustertesttypesResource, clusterTestType), &example_v1.ClusterTestType{}) + if obj == nil { + return nil, err + } + return obj.(*example_v1.ClusterTestType), err +} + +// Update takes the representation of a clusterTestType and updates it. Returns the server's representation of the clusterTestType, and an error, if there is any. +func (c *FakeClusterTestTypes) Update(clusterTestType *example_v1.ClusterTestType) (result *example_v1.ClusterTestType, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(clustertesttypesResource, clusterTestType), &example_v1.ClusterTestType{}) + if obj == nil { + return nil, err + } + return obj.(*example_v1.ClusterTestType), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeClusterTestTypes) UpdateStatus(clusterTestType *example_v1.ClusterTestType) (*example_v1.ClusterTestType, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(clustertesttypesResource, "status", clusterTestType), &example_v1.ClusterTestType{}) + if obj == nil { + return nil, err + } + return obj.(*example_v1.ClusterTestType), err +} + +// Delete takes name of the clusterTestType and deletes it. Returns an error if one occurs. +func (c *FakeClusterTestTypes) Delete(name string, options *v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteAction(clustertesttypesResource, name), &example_v1.ClusterTestType{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeClusterTestTypes) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(clustertesttypesResource, listOptions) + + _, err := c.Fake.Invokes(action, &example_v1.ClusterTestTypeList{}) + return err +} + +// Patch applies the patch and returns the patched clusterTestType. +func (c *FakeClusterTestTypes) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *example_v1.ClusterTestType, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(clustertesttypesResource, name, data, subresources...), &example_v1.ClusterTestType{}) + if obj == nil { + return nil, err + } + return obj.(*example_v1.ClusterTestType), err +} + +// GetScale takes name of the clusterTestType, and returns the corresponding scale object, and an error if there is any. +func (c *FakeClusterTestTypes) GetScale(clusterTestTypeName string, options v1.GetOptions) (result *autoscaling.Scale, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetSubresourceAction(clustertesttypesResource, clusterTestTypeName), &autoscaling.Scale{}) + if obj == nil { + return nil, err + } + return obj.(*autoscaling.Scale), err +} + +// UpdateScale takes the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *FakeClusterTestTypes) UpdateScale(clusterTestTypeName string, scale *autoscaling.Scale) (result *autoscaling.Scale, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(clustertesttypesResource, "scale", scale), &autoscaling.Scale{}) + if obj == nil { + return nil, err + } + return obj.(*autoscaling.Scale), err +} diff --git a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/fake/fake_example_client.go b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/fake/fake_example_client.go index 647f5e1223c..e6aa99993f8 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/fake/fake_example_client.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/fake/fake_example_client.go @@ -28,6 +28,10 @@ type FakeExampleV1 struct { *testing.Fake } +func (c *FakeExampleV1) ClusterTestTypes() v1.ClusterTestTypeInterface { + return &FakeClusterTestTypes{c} +} + func (c *FakeExampleV1) TestTypes(namespace string) v1.TestTypeInterface { return &FakeTestTypes{c, namespace} } diff --git a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/generated_expansion.go b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/generated_expansion.go index d513810d0d7..3059734a9ea 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/generated_expansion.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/clientset/versioned/typed/example/v1/generated_expansion.go @@ -18,4 +18,6 @@ limitations under the License. package v1 +type ClusterTestTypeExpansion interface{} + type TestTypeExpansion interface{} diff --git a/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/example/v1/clustertesttype.go b/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/example/v1/clustertesttype.go new file mode 100644 index 00000000000..cef599f485f --- /dev/null +++ b/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/example/v1/clustertesttype.go @@ -0,0 +1,88 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + time "time" + + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" + example_v1 "k8s.io/code-generator/_examples/crd/apis/example/v1" + versioned "k8s.io/code-generator/_examples/crd/clientset/versioned" + internalinterfaces "k8s.io/code-generator/_examples/crd/informers/externalversions/internalinterfaces" + v1 "k8s.io/code-generator/_examples/crd/listers/example/v1" +) + +// ClusterTestTypeInformer provides access to a shared informer and lister for +// ClusterTestTypes. +type ClusterTestTypeInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.ClusterTestTypeLister +} + +type clusterTestTypeInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewClusterTestTypeInformer constructs a new informer for ClusterTestType type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewClusterTestTypeInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredClusterTestTypeInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredClusterTestTypeInformer constructs a new informer for ClusterTestType type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredClusterTestTypeInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ExampleV1().ClusterTestTypes().List(options) + }, + WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ExampleV1().ClusterTestTypes().Watch(options) + }, + }, + &example_v1.ClusterTestType{}, + resyncPeriod, + indexers, + ) +} + +func (f *clusterTestTypeInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredClusterTestTypeInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *clusterTestTypeInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&example_v1.ClusterTestType{}, f.defaultInformer) +} + +func (f *clusterTestTypeInformer) Lister() v1.ClusterTestTypeLister { + return v1.NewClusterTestTypeLister(f.Informer().GetIndexer()) +} diff --git a/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/example/v1/interface.go b/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/example/v1/interface.go index 024352284e4..1e17b13255f 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/example/v1/interface.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/example/v1/interface.go @@ -24,6 +24,8 @@ import ( // Interface provides access to all the informers in this group version. type Interface interface { + // ClusterTestTypes returns a ClusterTestTypeInformer. + ClusterTestTypes() ClusterTestTypeInformer // TestTypes returns a TestTypeInformer. TestTypes() TestTypeInformer } @@ -39,6 +41,11 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } +// ClusterTestTypes returns a ClusterTestTypeInformer. +func (v *version) ClusterTestTypes() ClusterTestTypeInformer { + return &clusterTestTypeInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} + // TestTypes returns a TestTypeInformer. func (v *version) TestTypes() TestTypeInformer { return &testTypeInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/generic.go b/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/generic.go index df63a41ec66..d89bd2a76fe 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/generic.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/informers/externalversions/generic.go @@ -54,6 +54,8 @@ func (f *genericInformer) Lister() cache.GenericLister { func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { switch resource { // Group=example.crd.code-generator.k8s.io, Version=v1 + case v1.SchemeGroupVersion.WithResource("clustertesttypes"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Example().V1().ClusterTestTypes().Informer()}, nil case v1.SchemeGroupVersion.WithResource("testtypes"): return &genericInformer{resource: resource.GroupResource(), informer: f.Example().V1().TestTypes().Informer()}, nil diff --git a/staging/src/k8s.io/code-generator/_examples/crd/listers/example/v1/clustertesttype.go b/staging/src/k8s.io/code-generator/_examples/crd/listers/example/v1/clustertesttype.go new file mode 100644 index 00000000000..584b3b27352 --- /dev/null +++ b/staging/src/k8s.io/code-generator/_examples/crd/listers/example/v1/clustertesttype.go @@ -0,0 +1,65 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" + v1 "k8s.io/code-generator/_examples/crd/apis/example/v1" +) + +// ClusterTestTypeLister helps list ClusterTestTypes. +type ClusterTestTypeLister interface { + // List lists all ClusterTestTypes in the indexer. + List(selector labels.Selector) (ret []*v1.ClusterTestType, err error) + // Get retrieves the ClusterTestType from the index for a given name. + Get(name string) (*v1.ClusterTestType, error) + ClusterTestTypeListerExpansion +} + +// clusterTestTypeLister implements the ClusterTestTypeLister interface. +type clusterTestTypeLister struct { + indexer cache.Indexer +} + +// NewClusterTestTypeLister returns a new ClusterTestTypeLister. +func NewClusterTestTypeLister(indexer cache.Indexer) ClusterTestTypeLister { + return &clusterTestTypeLister{indexer: indexer} +} + +// List lists all ClusterTestTypes in the indexer. +func (s *clusterTestTypeLister) List(selector labels.Selector) (ret []*v1.ClusterTestType, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1.ClusterTestType)) + }) + return ret, err +} + +// Get retrieves the ClusterTestType from the index for a given name. +func (s *clusterTestTypeLister) Get(name string) (*v1.ClusterTestType, error) { + obj, exists, err := s.indexer.GetByKey(name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1.Resource("clustertesttype"), name) + } + return obj.(*v1.ClusterTestType), nil +} diff --git a/staging/src/k8s.io/code-generator/_examples/crd/listers/example/v1/expansion_generated.go b/staging/src/k8s.io/code-generator/_examples/crd/listers/example/v1/expansion_generated.go index 0192e05f0d1..2681a29f472 100644 --- a/staging/src/k8s.io/code-generator/_examples/crd/listers/example/v1/expansion_generated.go +++ b/staging/src/k8s.io/code-generator/_examples/crd/listers/example/v1/expansion_generated.go @@ -18,6 +18,10 @@ limitations under the License. package v1 +// ClusterTestTypeListerExpansion allows custom methods to be added to +// ClusterTestTypeLister. +type ClusterTestTypeListerExpansion interface{} + // TestTypeListerExpansion allows custom methods to be added to // TestTypeLister. type TestTypeListerExpansion interface{} diff --git a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/fake/generator_fake_for_type.go b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/fake/generator_fake_for_type.go index c129a5f4b96..3cbade874cc 100644 --- a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/fake/generator_fake_for_type.go +++ b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/fake/generator_fake_for_type.go @@ -146,9 +146,10 @@ func (g *genFakeForType) GenerateType(c *generator.Context, t *types.Type, w io. "NewRootWatchAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootWatchAction"}), "NewWatchAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewWatchAction"}), "NewCreateSubresourceAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewCreateSubresourceAction"}), + "NewRootCreateSubresourceAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootCreateSubresourceAction"}), "NewUpdateSubresourceAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewUpdateSubresourceAction"}), "NewGetSubresourceAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewGetSubresourceAction"}), - "NewListSubresourceAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewListSubresourceAction"}), + "NewRootGetSubresourceAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootGetSubresourceAction"}), "NewRootUpdateSubresourceAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootUpdateSubresourceAction"}), "NewRootPatchAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootPatchAction"}), "NewPatchAction": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewPatchAction"}), @@ -235,11 +236,8 @@ func (g *genFakeForType) GenerateType(c *generator.Context, t *types.Type, w io. } if e.HasVerb("list") { - if e.IsSubresource() { - sw.Do(adjustTemplate(e.VerbName, e.VerbType, listSubresourceTemplate), m) - } else { - sw.Do(adjustTemplate(e.VerbName, e.VerbType, listTemplate), m) - } + + sw.Do(adjustTemplate(e.VerbName, e.VerbType, listTemplate), m) } // TODO: Figure out schemantic for watching a sub-resource. @@ -322,19 +320,6 @@ func (c *Fake$.type|publicPlural$) List(opts $.ListOptions|raw$) (result *$.type } ` -var listSubresourceTemplate = ` -// List takes label and field selectors, and returns the list of $.resultType|publicPlural$ that match those selectors. -func (c *Fake$.type|publicPlural$) List($.type|private$Name string, opts $.ListOptions|raw$) (result *$.resultType|raw$List, err error) { - obj, err := c.Fake. - $if .namespaced$Invokes($.NewListSubresourceAction|raw$($.type|allLowercasePlural$Resource, $.type|private$Name, "$.subresourcePath$", $.type|allLowercasePlural$Kind, c.ns, opts), &$.resultType|raw$List{}) - $else$Invokes($.NewRootListAction|raw$($.type|allLowercasePlural$Resource, $.type|allLowercasePlural$Kind, opts), &$.resultType|raw$List{})$end$ - if obj == nil { - return nil, err - } - return obj.(*$.resultType|raw$List), err -} -` - var listUsingOptionsTemplate = ` // List takes label and field selectors, and returns the list of $.type|publicPlural$ that match those selectors. func (c *Fake$.type|publicPlural$) List(opts $.ListOptions|raw$) (result *$.type|raw$List, err error) { @@ -377,7 +362,7 @@ var getSubresourceTemplate = ` func (c *Fake$.type|publicPlural$) Get($.type|private$Name string, options $.GetOptions|raw$) (result *$.resultType|raw$, err error) { obj, err := c.Fake. $if .namespaced$Invokes($.NewGetSubresourceAction|raw$($.type|allLowercasePlural$Resource, c.ns, "$.subresourcePath$", $.type|private$Name), &$.resultType|raw${}) - $else$Invokes($.NewRootGetAction|raw$($.type|allLowercasePlural$Resource, $.type|private$Name), &$.resultType|raw${})$end$ + $else$Invokes($.NewRootGetSubresourceAction|raw$($.type|allLowercasePlural$Resource, $.type|private$Name), &$.resultType|raw${})$end$ if obj == nil { return nil, err } @@ -423,7 +408,7 @@ var createSubresourceTemplate = ` func (c *Fake$.type|publicPlural$) Create($.type|private$Name string, $.inputType|private$ *$.inputType|raw$) (result *$.resultType|raw$, err error) { obj, err := c.Fake. $if .namespaced$Invokes($.NewCreateSubresourceAction|raw$($.type|allLowercasePlural$Resource, $.type|private$Name, "$.subresourcePath$", c.ns, $.inputType|private$), &$.resultType|raw${}) - $else$Invokes($.NewRootCreateAction|raw$($.inputType|allLowercasePlural$Resource, $.inputType|private$), &$.resultType|raw${})$end$ + $else$Invokes($.NewRootCreateSubresourceAction|raw$($.type|allLowercasePlural$Resource, "$.subresourcePath$", $.inputType|private$), &$.resultType|raw${})$end$ if obj == nil { return nil, err } @@ -449,7 +434,7 @@ var updateSubresourceTemplate = ` func (c *Fake$.type|publicPlural$) Update($.type|private$Name string, $.inputType|private$ *$.inputType|raw$) (result *$.resultType|raw$, err error) { obj, err := c.Fake. $if .namespaced$Invokes($.NewUpdateSubresourceAction|raw$($.type|allLowercasePlural$Resource, "$.subresourcePath$", c.ns, $.inputType|private$), &$.inputType|raw${}) - $else$Invokes($.NewRootUpdateAction|raw$($.type|allLowercasePlural$Resource, $.type|private$), &$.type|raw${})$end$ + $else$Invokes($.NewRootUpdateSubresourceAction|raw$($.type|allLowercasePlural$Resource, "$.subresourcePath$", $.inputType|private$), &$.resultType|raw${})$end$ if obj == nil { return nil, err } diff --git a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/generator_for_group.go b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/generator_for_group.go index 6c4cd9bf4ef..fd59715c42b 100644 --- a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/generator_for_group.go +++ b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/generator_for_group.go @@ -210,17 +210,12 @@ func New(c $.restRESTClientInterface|raw$) *$.GroupGoName$$.Version$Client { var setInternalVersionClientDefaultsTemplate = ` func setConfigDefaults(config *$.restConfig|raw$) error { - g, err := scheme.Registry.Group("$.groupName$") - if err != nil { - return err - } - config.APIPath = $.apiPath$ if config.UserAgent == "" { config.UserAgent = $.restDefaultKubernetesUserAgent|raw$() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("$.groupName$")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("$.groupName$")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/scheme/generator_for_scheme.go b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/scheme/generator_for_scheme.go index a7e33553899..dd9afadbf3f 100644 --- a/staging/src/k8s.io/code-generator/cmd/client-gen/generators/scheme/generator_for_scheme.go +++ b/staging/src/k8s.io/code-generator/cmd/client-gen/generators/scheme/generator_for_scheme.go @@ -86,17 +86,15 @@ func (g *GenScheme) GenerateType(c *generator.Context, t *types.Type, w io.Write allInstallGroups := clientgentypes.ToGroupInstallPackages(g.Groups, g.GroupGoNames) m := map[string]interface{}{ - "allGroupVersions": allGroupVersions, - "allInstallGroups": allInstallGroups, - "customRegister": false, - "runtimeNewParameterCodec": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "NewParameterCodec"}), - "runtimeNewScheme": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "NewScheme"}), - "serializerNewCodecFactory": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/runtime/serializer", Name: "NewCodecFactory"}), - "runtimeScheme": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "Scheme"}), - "schemaGroupVersion": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/runtime/schema", Name: "GroupVersion"}), - "metav1AddToGroupVersion": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "AddToGroupVersion"}), - "registeredNew": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/apimachinery/registered", Name: "NewAPIRegistrationManager"}), - "registeredAPIRegistrationManager": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apimachinery/registered", Name: "APIRegistrationManager"}), + "allGroupVersions": allGroupVersions, + "allInstallGroups": allInstallGroups, + "customRegister": false, + "runtimeNewParameterCodec": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "NewParameterCodec"}), + "runtimeNewScheme": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "NewScheme"}), + "serializerNewCodecFactory": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/runtime/serializer", Name: "NewCodecFactory"}), + "runtimeScheme": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "Scheme"}), + "schemaGroupVersion": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/runtime/schema", Name: "GroupVersion"}), + "metav1AddToGroupVersion": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "AddToGroupVersion"}), } globals := map[string]string{ "Scheme": "Scheme", @@ -136,20 +134,18 @@ var $.ParameterCodec$ = $.runtimeNewParameterCodec|raw$($.Scheme$) ` var registryRegistration = ` -var $.Registry$ = $.registeredNew|raw$() - func init() { $.metav1AddToGroupVersion|raw$($.Scheme$, $.schemaGroupVersion|raw${Version: "v1"}) - Install($.Registry$, $.Scheme$) + Install($.Scheme$) } // Install registers the API group and adds types to a scheme -func Install(registry *$.registeredAPIRegistrationManager|raw$, scheme *$.runtimeScheme|raw$) { +func Install(scheme *$.runtimeScheme|raw$) { $- range .allInstallGroups$ - $.InstallPackageAlias$.Install(registry, scheme) + $.InstallPackageAlias$.Install(scheme) $- end$ $if .customRegister$ - ExtraInstall(registry, scheme) + ExtraInstall(scheme) $end -$ } ` diff --git a/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json b/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json index ee3fe313481..22bfaae89d0 100644 --- a/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json +++ b/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json @@ -618,18 +618,6 @@ "ImportPath": "k8s.io/apimachinery/pkg/api/validation/path", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/announced", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apimachinery/pkg/apis/meta/internalversion", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1652,27 +1640,27 @@ }, { "ImportPath": "k8s.io/kube-openapi/pkg/aggregator", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/builder", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/handler", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" } ] } diff --git a/staging/src/k8s.io/kube-aggregator/main.go b/staging/src/k8s.io/kube-aggregator/main.go index 3be730e47a6..b0d76a545f8 100644 --- a/staging/src/k8s.io/kube-aggregator/main.go +++ b/staging/src/k8s.io/kube-aggregator/main.go @@ -19,7 +19,6 @@ package main import ( "flag" "os" - "runtime" "github.com/golang/glog" @@ -39,10 +38,6 @@ func main() { logs.InitLogs() defer logs.FlushLogs() - if len(os.Getenv("GOMAXPROCS")) == 0 { - runtime.GOMAXPROCS(runtime.NumCPU()) - } - stopCh := genericapiserver.SetupSignalHandler() options := server.NewDefaultOptions(os.Stdout, os.Stderr) cmd := server.NewCommandStartAggregator(options, stopCh) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/install/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/install/BUILD index d57ee90555d..8eed75f0d1d 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/install/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/install/BUILD @@ -10,9 +10,8 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/kube-aggregator/pkg/apis/apiregistration/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library", "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1:go_default_library", diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/install/install.go b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/install/install.go index 75c976a8152..75dd8b5d492 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/install/install.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/install/install.go @@ -17,27 +17,17 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/kube-aggregator/pkg/apis/apiregistration" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: apiregistration.GroupName, - VersionPreferenceOrder: []string{v1.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: apiregistration.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1.SchemeGroupVersion.Version: v1.AddToScheme, - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(apiregistration.AddToScheme(scheme)) + utilruntime.Must(v1.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme/BUILD index 49897ee89d4..a610c570378 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme/BUILD @@ -6,7 +6,6 @@ go_library( importpath = "k8s.io/kube-aggregator/pkg/apiserver/scheme", visibility = ["//visibility:public"], deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library", diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme/scheme.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme/scheme.go index 233285ef1cd..4537edb1655 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme/scheme.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme/scheme.go @@ -17,7 +17,6 @@ limitations under the License. package scheme import ( - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -33,14 +32,11 @@ var ( // Codecs provides methods for retrieving codecs and serializers for specific // versions and content types. Codecs = serializer.NewCodecFactory(Scheme) - // Registry is an instance of an API registry. This is an interim step to start removing the idea of a global - // API registry. - Registry = registered.NewAPIRegistrationManager() ) func init() { AddToScheme(Scheme) - install.Install(Registry, Scheme) + install.Install(Scheme) } // AddToScheme adds the types of this group into the given scheme. diff --git a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/scheme/BUILD b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/scheme/BUILD index fa6aa4cd13b..59ebb38a257 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/scheme/BUILD +++ b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/scheme/BUILD @@ -13,7 +13,6 @@ go_library( ], importpath = "k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/scheme", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/scheme/register.go b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/scheme/register.go index 96d51bfcde0..5b4631eeecd 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/scheme/register.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/scheme/register.go @@ -19,7 +19,6 @@ limitations under the License. package scheme import ( - registered "k8s.io/apimachinery/pkg/apimachinery/registered" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -31,14 +30,12 @@ var Scheme = runtime.NewScheme() var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) -var Registry = registered.NewAPIRegistrationManager() - func init() { v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - Install(Registry, Scheme) + Install(Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - apiregistration.Install(registry, scheme) +func Install(scheme *runtime.Scheme) { + apiregistration.Install(scheme) } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion/apiregistration_client.go b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion/apiregistration_client.go index 3f42fcae35c..7724857668a 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion/apiregistration_client.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion/apiregistration_client.go @@ -66,17 +66,12 @@ func New(c rest.Interface) *ApiregistrationClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("apiregistration.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("apiregistration.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("apiregistration.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index 0ce5cded59e..40c794972d9 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -98,7 +98,7 @@ func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { func (o AggregatorOptions) Validate(args []string) error { errors := []error{} errors = append(errors, o.RecommendedOptions.Validate()...) - errors = append(errors, o.APIEnablement.Validate(aggregatorscheme.Registry)...) + errors = append(errors, o.APIEnablement.Validate(aggregatorscheme.Scheme)...) return utilerrors.NewAggregate(errors) } @@ -117,7 +117,7 @@ func (o AggregatorOptions) RunAggregator(stopCh <-chan struct{}) error { if err := o.RecommendedOptions.ApplyTo(serverConfig, aggregatorscheme.Scheme); err != nil { return err } - if err := o.APIEnablement.ApplyTo(&serverConfig.Config, apiserver.DefaultAPIResourceConfigSource(), aggregatorscheme.Registry); err != nil { + if err := o.APIEnablement.ApplyTo(&serverConfig.Config, apiserver.DefaultAPIResourceConfigSource(), aggregatorscheme.Scheme); err != nil { return err } serverConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck( diff --git a/staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice/rest/storage_apiservice.go b/staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice/rest/storage_apiservice.go index 47bac1b6720..c10b5b233f4 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice/rest/storage_apiservice.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice/rest/storage_apiservice.go @@ -32,7 +32,7 @@ import ( // NewRESTStorage returns an APIGroupInfo object that will work against apiservice. func NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) genericapiserver.APIGroupInfo { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiregistration.GroupName, aggregatorscheme.Registry, aggregatorscheme.Scheme, metav1.ParameterCodec, aggregatorscheme.Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiregistration.GroupName, aggregatorscheme.Scheme, metav1.ParameterCodec, aggregatorscheme.Codecs) if apiResourceConfigSource.VersionEnabled(v1beta1.SchemeGroupVersion) { storage := map[string]rest.Storage{} diff --git a/staging/src/k8s.io/metrics/Godeps/Godeps.json b/staging/src/k8s.io/metrics/Godeps/Godeps.json index 2a31c77da94..0c97cf6f228 100644 --- a/staging/src/k8s.io/metrics/Godeps/Godeps.json +++ b/staging/src/k8s.io/metrics/Godeps/Godeps.json @@ -258,18 +258,6 @@ "ImportPath": "k8s.io/apimachinery/pkg/api/resource", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/announced", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install/BUILD b/staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install/BUILD index 34aef6a4060..4cf59f0f39d 100644 --- a/staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install/BUILD +++ b/staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install/BUILD @@ -10,9 +10,8 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/metrics/pkg/apis/custom_metrics/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/metrics/pkg/apis/custom_metrics:go_default_library", "//vendor/k8s.io/metrics/pkg/apis/custom_metrics/v1beta1:go_default_library", ], diff --git a/staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install/install.go b/staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install/install.go index dc1196bd1cf..a20d6e4a5af 100644 --- a/staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install/install.go +++ b/staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/metrics/pkg/apis/custom_metrics" "k8s.io/metrics/pkg/apis/custom_metrics/v1beta1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: custom_metrics.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: custom_metrics.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(custom_metrics.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/metrics/pkg/apis/external_metrics/install/BUILD b/staging/src/k8s.io/metrics/pkg/apis/external_metrics/install/BUILD index df63ba222fb..9e075185e22 100644 --- a/staging/src/k8s.io/metrics/pkg/apis/external_metrics/install/BUILD +++ b/staging/src/k8s.io/metrics/pkg/apis/external_metrics/install/BUILD @@ -10,9 +10,8 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/metrics/pkg/apis/external_metrics/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/metrics/pkg/apis/external_metrics:go_default_library", "//vendor/k8s.io/metrics/pkg/apis/external_metrics/v1beta1:go_default_library", ], diff --git a/staging/src/k8s.io/metrics/pkg/apis/external_metrics/install/install.go b/staging/src/k8s.io/metrics/pkg/apis/external_metrics/install/install.go index a036007399f..5e110566d9f 100644 --- a/staging/src/k8s.io/metrics/pkg/apis/external_metrics/install/install.go +++ b/staging/src/k8s.io/metrics/pkg/apis/external_metrics/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/metrics/pkg/apis/external_metrics" "k8s.io/metrics/pkg/apis/external_metrics/v1beta1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: external_metrics.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: external_metrics.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(external_metrics.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/metrics/pkg/apis/metrics/install/BUILD b/staging/src/k8s.io/metrics/pkg/apis/metrics/install/BUILD index d42c4ad1ad9..81fee72db56 100644 --- a/staging/src/k8s.io/metrics/pkg/apis/metrics/install/BUILD +++ b/staging/src/k8s.io/metrics/pkg/apis/metrics/install/BUILD @@ -10,9 +10,8 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/metrics/pkg/apis/metrics/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/metrics/pkg/apis/metrics:go_default_library", "//vendor/k8s.io/metrics/pkg/apis/metrics/v1beta1:go_default_library", ], diff --git a/staging/src/k8s.io/metrics/pkg/apis/metrics/install/install.go b/staging/src/k8s.io/metrics/pkg/apis/metrics/install/install.go index c1cf503c9b8..c5f719dcb8d 100644 --- a/staging/src/k8s.io/metrics/pkg/apis/metrics/install/install.go +++ b/staging/src/k8s.io/metrics/pkg/apis/metrics/install/install.go @@ -19,25 +19,15 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/metrics/pkg/apis/metrics" "k8s.io/metrics/pkg/apis/metrics/v1beta1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: metrics.GroupName, - VersionPreferenceOrder: []string{v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: metrics.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(metrics.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json b/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json index 51aabfc8622..1e9bd8764d4 100644 --- a/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json @@ -602,18 +602,6 @@ "ImportPath": "k8s.io/apimachinery/pkg/api/validation/path", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/announced", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apimachinery/pkg/apis/meta/fuzzer", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -1616,23 +1604,23 @@ }, { "ImportPath": "k8s.io/kube-openapi/pkg/builder", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/common", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/handler", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" } ] } diff --git a/staging/src/k8s.io/sample-apiserver/README.md b/staging/src/k8s.io/sample-apiserver/README.md index c5cbc41175a..6f4bf686d85 100644 --- a/staging/src/k8s.io/sample-apiserver/README.md +++ b/staging/src/k8s.io/sample-apiserver/README.md @@ -23,3 +23,64 @@ HEAD of this repo will match HEAD of k8s.io/apiserver, k8s.io/apimachinery, and `sample-apiserver` is synced from https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/sample-apiserver. Code changes are made in that location, merged into `k8s.io/kubernetes` and later synced here. +## Running it stand-alone + +During development it is helpful to run sample-apiserver stand-alone, i.e. without +a Kubernetes API server for authn/authz and without aggregation. This is possible, but needs +a couple of flags, keys and certs as described below. You will still need some kubeconfig, +e.g. `~/.kube/config`, but the Kubernetes cluster is not used for authn/z. A minikube or +hack/local-up-cluster.sh cluster will work. + +Instead of trusting the aggregator inside kube-apiserver, the described setup uses local +client certificate based X.509 authentication and authorization. This means that the client +certificate is trusted by a CA and the passed certificate contains the group membership +to the `system:masters` group. As we disable delegated authorization with `--authorization-skip-lookup`, +only this superuser group is authorized. + +1. First we need a CA to later sign the client certificate: + +``` shell +openssl req -nodes -new -x509 -keyout ca.key -out ca.crt +``` + +2. Then we create a client cert signed by this CA for the user `development` in the superuser group + `system:masters`: + +``` shell +openssl req -out client.csr -new -newkey rsa:4096 -nodes -keyout client.key -subj "/CN=development/O=system:masters" +openssl x509 -req -days 365 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out client.crt +``` + +3. As curl requires client certificates in p12 format with password, do the conversion: + +``` shell +openssl pkcs12 -export -in ./client.crt -inkey ./client.key -out client.p12 -passout pass:password +``` + +4. With these keys and certs in-place, we start the server: + +``` shell +etcd & +sample-apiserver --secure-port 8443 --etcd-servers http://127.0.0.1:2379 --v=7 \ + --client-ca-file ca.crt \ + --kubeconfig ~/.kube/config \ + --authentication-kubeconfig ~/.kube/config \ + --authorization-kubeconfig ~/.kube/config +``` + +The first kubeconfig is used for the shared informers to access Kubernetes resources. The second kubeconfig passed to `--authentication-kubeconfig` is used to satisfy the delegated authenticator. The third kubeconfig passed to `--authorized-kubeconfig` is used to satisfy the delegated authorizer. Neither the authenticator, nor the authorizer will actually be used: due to `--client-ca-file`, our development X.509 certificate is accepted and authenticates us as `system:masters` member. `system:masters` is the superuser group +such that delegated authorization is skipped. + +5. Use curl to access the server using the client certificate in p12 format for authentication: + +``` shell +curl -fv -k --cert client.p12:password \ + https://localhost:8443/apis/wardle.k8s.io/v1alpha1/namespaces/default/flunders +``` + + Note: Recent OSX versions broke client certs with curl. On Mac try `brew install httpie` and then: + +``` shell +http --verify=no --cert client.crt --cert-key client.key \ + https://localhost:8443/apis/wardle.k8s.io/v1alpha1/namespaces/default/flunders +``` diff --git a/staging/src/k8s.io/sample-apiserver/main.go b/staging/src/k8s.io/sample-apiserver/main.go index f05327d980a..8e1769a36ff 100644 --- a/staging/src/k8s.io/sample-apiserver/main.go +++ b/staging/src/k8s.io/sample-apiserver/main.go @@ -19,7 +19,6 @@ package main import ( "flag" "os" - "runtime" "github.com/golang/glog" @@ -32,10 +31,6 @@ func main() { logs.InitLogs() defer logs.FlushLogs() - if len(os.Getenv("GOMAXPROCS")) == 0 { - runtime.GOMAXPROCS(runtime.NumCPU()) - } - stopCh := genericapiserver.SetupSignalHandler() options := server.NewWardleServerOptions(os.Stdout, os.Stderr) cmd := server.NewCommandStartWardleServer(options, stopCh) diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install/BUILD b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install/BUILD index 600056cab16..02042ba5f94 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install/BUILD +++ b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install/BUILD @@ -21,9 +21,8 @@ go_library( srcs = ["install.go"], importpath = "k8s.io/sample-apiserver/pkg/apis/wardle/install", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/sample-apiserver/pkg/apis/wardle:go_default_library", "//vendor/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1:go_default_library", "//vendor/k8s.io/sample-apiserver/pkg/apis/wardle/v1beta1:go_default_library", diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install/install.go b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install/install.go index 4995e68e45a..98bde92f416 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install/install.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install/install.go @@ -17,27 +17,17 @@ limitations under the License. package install import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/sample-apiserver/pkg/apis/wardle" "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" "k8s.io/sample-apiserver/pkg/apis/wardle/v1beta1" ) // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: wardle.GroupName, - VersionPreferenceOrder: []string{v1alpha1.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: wardle.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme, - }, - ).Register(registry, scheme); err != nil { - panic(err) - } +func Install(scheme *runtime.Scheme) { + utilruntime.Must(wardle.AddToScheme(scheme)) + utilruntime.Must(v1beta1.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion, v1alpha1.SchemeGroupVersion)) } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/BUILD b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/BUILD index 1fc41d041b6..519b6eef0bd 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/BUILD +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/BUILD @@ -21,7 +21,6 @@ go_library( srcs = ["apiserver.go"], importpath = "k8s.io/sample-apiserver/pkg/apiserver", deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go index 7f0045bfa30..e93f1fa213a 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go @@ -17,7 +17,6 @@ limitations under the License. package apiserver import ( - "k8s.io/apimachinery/pkg/apimachinery/registered" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -34,13 +33,12 @@ import ( ) var ( - registry = registered.NewAPIRegistrationManager() - Scheme = runtime.NewScheme() - Codecs = serializer.NewCodecFactory(Scheme) + Scheme = runtime.NewScheme() + Codecs = serializer.NewCodecFactory(Scheme) ) func init() { - install.Install(registry, Scheme) + install.Install(Scheme) // we need to add the options to empty v1 // TODO fix the server code to avoid this @@ -107,7 +105,7 @@ func (c completedConfig) New() (*WardleServer, error) { GenericAPIServer: genericServer, } - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(wardle.GroupName, registry, Scheme, metav1.ParameterCodec, Codecs) + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(wardle.GroupName, Scheme, metav1.ParameterCodec, Codecs) v1alpha1storage := map[string]rest.Storage{} v1alpha1storage["flunders"] = wardleregistry.RESTInPeace(flunderstorage.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter)) diff --git a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/scheme/BUILD b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/scheme/BUILD index 715cbbad282..56e9b2fc99c 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/scheme/BUILD +++ b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/scheme/BUILD @@ -9,7 +9,6 @@ go_library( importpath = "k8s.io/sample-apiserver/pkg/client/clientset/internalversion/scheme", visibility = ["//visibility:public"], deps = [ - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/scheme/register.go b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/scheme/register.go index 4183cc85d32..7a519381928 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/scheme/register.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/scheme/register.go @@ -19,7 +19,6 @@ limitations under the License. package scheme import ( - registered "k8s.io/apimachinery/pkg/apimachinery/registered" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -31,14 +30,12 @@ var Scheme = runtime.NewScheme() var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) -var Registry = registered.NewAPIRegistrationManager() - func init() { v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - Install(Registry, Scheme) + Install(Scheme) } // Install registers the API group and adds types to a scheme -func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - wardle.Install(registry, scheme) +func Install(scheme *runtime.Scheme) { + wardle.Install(scheme) } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/wardle_client.go b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/wardle_client.go index 44d2a9f0894..12e618e9699 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/wardle_client.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/client/clientset/internalversion/typed/wardle/internalversion/wardle_client.go @@ -71,17 +71,12 @@ func New(c rest.Interface) *WardleClient { } func setConfigDefaults(config *rest.Config) error { - g, err := scheme.Registry.Group("wardle.k8s.io") - if err != nil { - return err - } - config.APIPath = "/apis" if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - if config.GroupVersion == nil || config.GroupVersion.Group != g.GroupVersions[0].Group { - gv := g.GroupVersions[0] + if config.GroupVersion == nil || config.GroupVersion.Group != scheme.Scheme.PrioritizedVersionsForGroup("wardle.k8s.io")[0].Group { + gv := scheme.Scheme.PrioritizedVersionsForGroup("wardle.k8s.io")[0] config.GroupVersion = &gv } config.NegotiatedSerializer = scheme.Codecs diff --git a/staging/src/k8s.io/sample-controller/Godeps/Godeps.json b/staging/src/k8s.io/sample-controller/Godeps/Godeps.json index 16888629a82..b767271e22c 100644 --- a/staging/src/k8s.io/sample-controller/Godeps/Godeps.json +++ b/staging/src/k8s.io/sample-controller/Godeps/Godeps.json @@ -916,7 +916,7 @@ }, { "ImportPath": "k8s.io/kube-openapi/pkg/util/proto", - "Rev": "f08db293d3ef80052d6513ece19792642a289fea" + "Rev": "61db125d227fc9d4e373819a059516f32f7f23c7" } ] } diff --git a/test/BUILD b/test/BUILD index d8087c512ea..f55d259350f 100644 --- a/test/BUILD +++ b/test/BUILD @@ -13,6 +13,7 @@ filegroup( ":package-srcs", "//test/conformance:all-srcs", "//test/e2e:all-srcs", + "//test/e2e_kubeadm:all-srcs", "//test/e2e_node:all-srcs", "//test/fixtures:all-srcs", "//test/images:all-srcs", diff --git a/test/e2e/apimachinery/aggregator.go b/test/e2e/apimachinery/aggregator.go index 0aa64361680..d7924800bed 100644 --- a/test/e2e/apimachinery/aggregator.go +++ b/test/e2e/apimachinery/aggregator.go @@ -382,14 +382,13 @@ func TestSampleAPIServer(f *framework.Framework, image string) { flunderName = generateFlunderName("dynamic-flunder") // Rerun the Create/List/Delete tests using the Dynamic client. - resources, err := client.Discovery().ServerPreferredNamespacedResources() - framework.ExpectNoError(err, "getting server preferred namespaces resources for dynamic client") + resources, discoveryErr := client.Discovery().ServerPreferredNamespacedResources() groupVersionResources, err := discovery.GroupVersionResources(resources) framework.ExpectNoError(err, "getting group version resources for dynamic client") gvr := schema.GroupVersionResource{Group: "wardle.k8s.io", Version: "v1alpha1", Resource: "flunders"} _, ok := groupVersionResources[gvr] if !ok { - framework.Failf("could not find group version resource for dynamic client and wardle/flunders.") + framework.Failf("could not find group version resource for dynamic client and wardle/flunders (discovery error: %v, discovery results: %#v)", discoveryErr, groupVersionResources) } dynamicClient := f.DynamicClient.Resource(gvr).Namespace(namespace) diff --git a/test/e2e/apimachinery/watch.go b/test/e2e/apimachinery/watch.go index e67dbf184c6..eadbe3180e2 100644 --- a/test/e2e/apimachinery/watch.go +++ b/test/e2e/apimachinery/watch.go @@ -64,7 +64,6 @@ var _ = SIGDescribe("Watchers", func() { }, }, Spec: v1.PodSpec{ - ActiveDeadlineSeconds: int64ptr(20), Containers: []v1.Container{ { Name: "example", @@ -81,7 +80,6 @@ var _ = SIGDescribe("Watchers", func() { }, }, Spec: v1.PodSpec{ - ActiveDeadlineSeconds: int64ptr(20), Containers: []v1.Container{ { Name: "example", @@ -97,7 +95,7 @@ var _ = SIGDescribe("Watchers", func() { expectNoEvent(watchB, watch.Added, testPodA) testPodA, err = updatePod(f, testPodA.GetName(), func(p *v1.Pod) { - p.Spec.ActiveDeadlineSeconds = int64ptr(10) + p.ObjectMeta.Labels["mutation"] = "1" }) Expect(err).NotTo(HaveOccurred()) expectEvent(watchA, watch.Modified, testPodA) @@ -105,7 +103,7 @@ var _ = SIGDescribe("Watchers", func() { expectNoEvent(watchB, watch.Modified, testPodA) testPodA, err = updatePod(f, testPodA.GetName(), func(p *v1.Pod) { - p.Spec.ActiveDeadlineSeconds = int64ptr(5) + p.ObjectMeta.Labels["mutation"] = "2" }) Expect(err).NotTo(HaveOccurred()) expectEvent(watchA, watch.Modified, testPodA) diff --git a/test/e2e/apps/daemon_set.go b/test/e2e/apps/daemon_set.go index 0a1692f489b..b77cc0ddcdf 100644 --- a/test/e2e/apps/daemon_set.go +++ b/test/e2e/apps/daemon_set.go @@ -79,12 +79,12 @@ var _ = SIGDescribe("Daemon set [Serial]", func() { } } if daemonsets, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).List(metav1.ListOptions{}); err == nil { - framework.Logf("daemonset: %s", runtime.EncodeOrDie(legacyscheme.Codecs.LegacyCodec(legacyscheme.Registry.RegisteredGroupVersions()...), daemonsets)) + framework.Logf("daemonset: %s", runtime.EncodeOrDie(legacyscheme.Codecs.LegacyCodec(legacyscheme.Scheme.PrioritizedVersionsAllGroups()...), daemonsets)) } else { framework.Logf("unable to dump daemonsets: %v", err) } if pods, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(metav1.ListOptions{}); err == nil { - framework.Logf("pods: %s", runtime.EncodeOrDie(legacyscheme.Codecs.LegacyCodec(legacyscheme.Registry.RegisteredGroupVersions()...), pods)) + framework.Logf("pods: %s", runtime.EncodeOrDie(legacyscheme.Codecs.LegacyCodec(legacyscheme.Scheme.PrioritizedVersionsAllGroups()...), pods)) } else { framework.Logf("unable to dump pods: %v", err) } diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index b387d0fb4d3..f0b7207558f 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -146,6 +146,7 @@ go_library( "//vendor/k8s.io/client-go/kubernetes/typed/extensions/v1beta1:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/rbac/v1beta1:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/scale:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index b1930ef4319..dcf04ec2a8c 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -39,6 +39,7 @@ import ( "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" scaleclient "k8s.io/client-go/scale" "k8s.io/client-go/tools/clientcmd" aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" @@ -182,7 +183,7 @@ func (f *Framework) BeforeEach() { discoClient, err := discovery.NewDiscoveryClientForConfig(config) Expect(err).NotTo(HaveOccurred()) cachedDiscoClient := cacheddiscovery.NewMemCacheClient(discoClient) - restMapper := discovery.NewDeferredDiscoveryRESTMapper(cachedDiscoClient) + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoClient) restMapper.Reset() resolver := scaleclient.NewDiscoveryScaleKindResolver(cachedDiscoClient) f.ScalesGetter = scaleclient.New(restClient, restMapper, dynamic.LegacyAPIPathResolverFunc, resolver) diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 91e5ba50153..dc7aee01e6c 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -1221,6 +1221,8 @@ func isDynamicDiscoveryError(err error) bool { // garbage_collector case "wardle.k8s.io": // aggregator + case "metrics.k8s.io": + // aggregated metrics server add-on, no persisted resources default: Logf("discovery error for unexpected group: %#v", gv) return false diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index 0dbbc7c800c..dcb38f315f4 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -2024,6 +2024,9 @@ func execAffinityTestForLBService(f *framework.Framework, cs clientset.Interface jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer) defer func() { framework.StopServeHostnameService(cs, f.InternalClientset, f.ScalesGetter, ns, serviceName) + lb := cloudprovider.GetLoadBalancerName(svc) + framework.Logf("cleaning gce resource for %s", lb) + framework.CleanupServiceGCEResources(cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone) }() ingressIP := framework.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0]) port := int(svc.Spec.Ports[0].Port) diff --git a/test/e2e/scalability/BUILD b/test/e2e/scalability/BUILD index e8aafde8168..11238a2f260 100644 --- a/test/e2e/scalability/BUILD +++ b/test/e2e/scalability/BUILD @@ -39,6 +39,7 @@ go_library( "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/scale:go_default_library", "//vendor/k8s.io/client-go/tools/cache:go_default_library", "//vendor/k8s.io/client-go/transport:go_default_library", diff --git a/test/e2e/scalability/load.go b/test/e2e/scalability/load.go index cdfc4b9b40e..99a808f606f 100644 --- a/test/e2e/scalability/load.go +++ b/test/e2e/scalability/load.go @@ -53,6 +53,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/client-go/dynamic" + "k8s.io/client-go/restmapper" "k8s.io/kubernetes/pkg/api/legacyscheme" ) @@ -409,7 +410,7 @@ func createClients(numberOfClients int) ([]clientset.Interface, []internalclient return nil, nil, nil, err } cachedDiscoClient := cacheddiscovery.NewMemCacheClient(discoClient) - restMapper := discovery.NewDeferredDiscoveryRESTMapper(cachedDiscoClient) + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoClient) restMapper.Reset() resolver := scaleclient.NewDiscoveryScaleKindResolver(cachedDiscoClient) scalesClients[i] = scaleclient.New(restClient, restMapper, dynamic.LegacyAPIPathResolverFunc, resolver) diff --git a/test/e2e/storage/subpath.go b/test/e2e/storage/subpath.go index f32a3ae51e5..4b7394fa3db 100644 --- a/test/e2e/storage/subpath.go +++ b/test/e2e/storage/subpath.go @@ -56,13 +56,14 @@ type volInfo struct { type volSource interface { createVolume(f *framework.Framework) volInfo cleanupVolume(f *framework.Framework) + getReadOnlyVolumeSpec() *v1.VolumeSource } var initVolSources = map[string]func() volSource{ "hostPath": initHostpath, "hostPathSymlink": initHostpathSymlink, "emptyDir": initEmptydir, - "gcePDPVC": initGCEPD, + "gcePDPVC": initGCEPDPVC, "gcePDPartitioned": initGCEPDPartition, "nfs": initNFS, "nfsPVC": initNFSPVC, @@ -271,6 +272,46 @@ var _ = utils.SIGDescribe("Subpath", func() { } testSubpathReconstruction(f, pod, true) }) + + It("should support readOnly directory specified in the volumeMount", func() { + // Create the directory + setInitCommand(pod, fmt.Sprintf("mkdir -p %s", subPathDir)) + + // Write the file in the volume from container 1 + setWriteCommand(filePathInVolume, &pod.Spec.Containers[1]) + + // Read it from inside the subPath from container 0 + pod.Spec.Containers[0].VolumeMounts[0].ReadOnly = true + testReadFile(f, filePathInSubpath, pod, 0) + }) + + It("should support readOnly file specified in the volumeMount", func() { + // Create the file + setInitCommand(pod, fmt.Sprintf("touch %s", subPathDir)) + + // Write the file in the volume from container 1 + setWriteCommand(subPathDir, &pod.Spec.Containers[1]) + + // Read it from inside the subPath from container 0 + pod.Spec.Containers[0].VolumeMounts[0].ReadOnly = true + testReadFile(f, volumePath, pod, 0) + }) + + It("should support readOnly directory specified in the volumeSource", func() { + roVol := vol.getReadOnlyVolumeSpec() + if roVol == nil { + framework.Skipf("Volume type %v doesn't support readOnly source", curVolType) + } + + // Initialize content in the volume while it's writable + initVolumeContent(f, pod, filePathInVolume, filePathInSubpath) + + // Set volume source to read only + pod.Spec.Volumes[0].VolumeSource = *roVol + + // Read it from inside the subPath from container 0 + testReadFile(f, filePathInSubpath, pod, 0) + }) }) } @@ -375,6 +416,12 @@ func testPodSubpath(f *framework.Framework, subpath, volumeType string, source * } } +func clearSubpathPodCommands(pod *v1.Pod) { + pod.Spec.InitContainers[0].Command = nil + pod.Spec.Containers[0].Args = nil + pod.Spec.Containers[1].Args = nil +} + func setInitCommand(pod *v1.Pod, command string) { pod.Spec.InitContainers[0].Command = []string{"/bin/sh", "-ec", command} } @@ -570,6 +617,23 @@ func testSubpathReconstruction(f *framework.Framework, pod *v1.Pod, forceDelete utils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true) } +func initVolumeContent(f *framework.Framework, pod *v1.Pod, volumeFilepath, subpathFilepath string) { + setWriteCommand(volumeFilepath, &pod.Spec.Containers[1]) + setReadCommand(subpathFilepath, &pod.Spec.Containers[0]) + + By(fmt.Sprintf("Creating pod to write volume content %s", pod.Name)) + f.TestContainerOutput("subpath", pod, 0, []string{ + "content of file \"" + subpathFilepath + "\": mount-tester new file", + }) + + By(fmt.Sprintf("Deleting pod %s", pod.Name)) + err := framework.DeletePodWithWait(f, f.ClientSet, pod) + Expect(err).NotTo(HaveOccurred(), "while deleting pod") + + // This pod spec is going to be reused; reset all the commands + clearSubpathPodCommands(pod) +} + func podContainerExec(pod *v1.Pod, containerIndex int, bashExec string) (string, error) { return framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--container", pod.Spec.Containers[containerIndex].Name, "--", "/bin/sh", "-c", bashExec) } @@ -591,6 +655,10 @@ func (s *hostpathSource) createVolume(f *framework.Framework) volInfo { } } +func (s *hostpathSource) getReadOnlyVolumeSpec() *v1.VolumeSource { + return nil +} + func (s *hostpathSource) cleanupVolume(f *framework.Framework) { } @@ -666,6 +734,10 @@ func (s *hostpathSymlinkSource) createVolume(f *framework.Framework) volInfo { } } +func (s *hostpathSymlinkSource) getReadOnlyVolumeSpec() *v1.VolumeSource { + return nil +} + func (s *hostpathSymlinkSource) cleanupVolume(f *framework.Framework) { } @@ -684,19 +756,23 @@ func (s *emptydirSource) createVolume(f *framework.Framework) volInfo { } } +func (s *emptydirSource) getReadOnlyVolumeSpec() *v1.VolumeSource { + return nil +} + func (s *emptydirSource) cleanupVolume(f *framework.Framework) { } -type gcepdSource struct { +type gcepdPVCSource struct { pvc *v1.PersistentVolumeClaim } -func initGCEPD() volSource { +func initGCEPDPVC() volSource { framework.SkipUnlessProviderIs("gce", "gke") - return &gcepdSource{} + return &gcepdPVCSource{} } -func (s *gcepdSource) createVolume(f *framework.Framework) volInfo { +func (s *gcepdPVCSource) createVolume(f *framework.Framework) volInfo { var err error framework.Logf("Creating GCE PD volume via dynamic provisioning") @@ -718,7 +794,16 @@ func (s *gcepdSource) createVolume(f *framework.Framework) volInfo { } } -func (s *gcepdSource) cleanupVolume(f *framework.Framework) { +func (s *gcepdPVCSource) getReadOnlyVolumeSpec() *v1.VolumeSource { + return &v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: s.pvc.Name, + ReadOnly: true, + }, + } +} + +func (s *gcepdPVCSource) cleanupVolume(f *framework.Framework) { if s.pvc != nil { err := f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Delete(s.pvc.Name, nil) framework.ExpectNoError(err, "Error deleting PVC") @@ -756,6 +841,10 @@ func (s *gcepdPartitionSource) createVolume(f *framework.Framework) volInfo { } } +func (s *gcepdPartitionSource) getReadOnlyVolumeSpec() *v1.VolumeSource { + return nil +} + func (s *gcepdPartitionSource) cleanupVolume(f *framework.Framework) { if s.diskName != "" { // err := framework.DeletePDWithRetry(s.diskName) @@ -765,6 +854,7 @@ func (s *gcepdPartitionSource) cleanupVolume(f *framework.Framework) { type nfsSource struct { serverPod *v1.Pod + serverIP string } func initNFS() volSource { @@ -772,21 +862,29 @@ func initNFS() volSource { } func (s *nfsSource) createVolume(f *framework.Framework) volInfo { - var serverIP string - framework.Logf("Creating NFS server") - _, s.serverPod, serverIP = framework.NewNFSServer(f.ClientSet, f.Namespace.Name, []string{"-G", "777", "/exports"}) + _, s.serverPod, s.serverIP = framework.NewNFSServer(f.ClientSet, f.Namespace.Name, []string{"-G", "777", "/exports"}) return volInfo{ source: &v1.VolumeSource{ NFS: &v1.NFSVolumeSource{ - Server: serverIP, + Server: s.serverIP, Path: "/exports", }, }, } } +func (s *nfsSource) getReadOnlyVolumeSpec() *v1.VolumeSource { + return &v1.VolumeSource{ + NFS: &v1.NFSVolumeSource{ + Server: s.serverIP, + Path: "/exports", + ReadOnly: true, + }, + } +} + func (s *nfsSource) cleanupVolume(f *framework.Framework) { if s.serverPod != nil { framework.DeletePodWithWait(f, f.ClientSet, s.serverPod) @@ -816,6 +914,16 @@ func (s *glusterSource) createVolume(f *framework.Framework) volInfo { } } +func (s *glusterSource) getReadOnlyVolumeSpec() *v1.VolumeSource { + return &v1.VolumeSource{ + Glusterfs: &v1.GlusterfsVolumeSource{ + EndpointsName: "gluster-server", + Path: "test_vol", + ReadOnly: true, + }, + } +} + func (s *glusterSource) cleanupVolume(f *framework.Framework) { if s.serverPod != nil { framework.DeletePodWithWait(f, f.ClientSet, s.serverPod) @@ -875,6 +983,15 @@ func (s *nfsPVCSource) createVolume(f *framework.Framework) volInfo { } } +func (s *nfsPVCSource) getReadOnlyVolumeSpec() *v1.VolumeSource { + return &v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: s.pvc.Name, + ReadOnly: true, + }, + } +} + func (s *nfsPVCSource) cleanupVolume(f *framework.Framework) { if s.pvc != nil || s.pv != nil { if errs := framework.PVPVCCleanup(f.ClientSet, f.Namespace.Name, s.pv, s.pvc); len(errs) != 0 { diff --git a/test/e2e/storage/volume_provisioning.go b/test/e2e/storage/volume_provisioning.go index 6454d936a42..8dd4a4cffcd 100644 --- a/test/e2e/storage/volume_provisioning.go +++ b/test/e2e/storage/volume_provisioning.go @@ -99,14 +99,10 @@ func testDynamicProvisioning(t storageClassTest, client clientset.Interface, cla pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - if class.Provisioner == "kubernetes.io/glusterfs" && framework.ProviderIs("gke", "gce") { - framework.Logf("Skipping glusterfs dynamic test for cloud provider %v", "GCE/GKE") - } else { - // Check sizes - expectedCapacity := resource.MustParse(t.expectedSize) - pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)] - Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity") - } + // Check sizes + expectedCapacity := resource.MustParse(t.expectedSize) + pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)] + Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity") requestedCapacity := resource.MustParse(t.claimSize) claimCapacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] @@ -802,6 +798,17 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { parameters: map[string]string{"resturl": serverUrl}, attach: false, } + + // GCE/GKE + if getDefaultPluginName() == "kubernetes.io/gce-pd" { + // Keeping an extra condition here based on below facts: + //*) gce-pd rounds up to the next gb. + //*) GlusterFS provisioner rounduptoGiB() and send it to backend, + // which does 'size/number' from provisioner*1024*1024*1024 + test.claimSize = "2Gi" + test.expectedSize = "3G" + } + suffix := fmt.Sprintf("glusterdptest") class := newStorageClass(test, ns, suffix) diff --git a/test/e2e/upgrades/ingress.go b/test/e2e/upgrades/ingress.go index 851c08304e9..09832ee939d 100644 --- a/test/e2e/upgrades/ingress.go +++ b/test/e2e/upgrades/ingress.go @@ -177,6 +177,14 @@ func (t *IngressUpgradeTest) verify(f *framework.Framework, done <-chan struct{} postUpgradeResourceStore := &GCPResourceStore{} t.populateGCPResourceStore(postUpgradeResourceStore) + // Stub out the number of instances as that is out of Ingress controller's control. + for _, ig := range t.resourceStore.IgList { + ig.Size = 0 + } + for _, ig := range postUpgradeResourceStore.IgList { + ig.Size = 0 + } + framework.ExpectNoError(compareGCPResourceStores(t.resourceStore, postUpgradeResourceStore, func(v1 reflect.Value, v2 reflect.Value) error { i1 := v1.Interface() i2 := v2.Interface() diff --git a/test/e2e_kubeadm/BUILD b/test/e2e_kubeadm/BUILD new file mode 100644 index 00000000000..ead6cc65d8d --- /dev/null +++ b/test/e2e_kubeadm/BUILD @@ -0,0 +1,65 @@ +package(default_visibility = ["//visibility:public"]) + +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_test( + name = "go_default_test", + srcs = [ + "e2e_kubeadm_suite_test.go", + "kubeadm_test.go", + ], + embed = [":go_default_library"], + tags = ["e2e"], + deps = [ + "//test/e2e/framework:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", + "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/api/rbac/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/tools/bootstrap/token/api:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +genrule( + name = "gen_e2e_kubeadm.test", + testonly = 1, + srcs = [":go_default_test"], + outs = ["e2e_kubeadm.test"], + cmd = "srcs=($(SRCS)); cp $$(dirname $${srcs[0]})/go_default_test $@;", + output_to_bindir = 1, +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//test/e2e_kubeadm/runner/local:all-srcs", + "//test/e2e_kubeadm/tests:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_library( + name = "go_default_library", + srcs = ["matchers.go"], + importpath = "k8s.io/kubernetes/test/e2e_kubeadm", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/onsi/gomega/gstruct:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + ], +) diff --git a/test/e2e_kubeadm/e2e_kubeadm_suite_test.go b/test/e2e_kubeadm/e2e_kubeadm_suite_test.go new file mode 100644 index 00000000000..8fc3114b88e --- /dev/null +++ b/test/e2e_kubeadm/e2e_kubeadm_suite_test.go @@ -0,0 +1,48 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + "flag" + "os" + "testing" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + "github.com/spf13/pflag" + + "k8s.io/kubernetes/test/e2e/framework" +) + +func init() { + framework.RegisterCommonFlags() + framework.RegisterClusterFlags() + + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) +} + +func TestMain(m *testing.M) { + pflag.Parse() + framework.AfterReadingAllFlags(&framework.TestContext) + os.Exit(m.Run()) +} + +func TestE2E(t *testing.T) { + reporters := []ginkgo.Reporter{} + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "E2EKubadm suite", reporters) +} diff --git a/test/e2e_kubeadm/kubeadm_test.go b/test/e2e_kubeadm/kubeadm_test.go new file mode 100644 index 00000000000..b8dc18c3c06 --- /dev/null +++ b/test/e2e_kubeadm/kubeadm_test.go @@ -0,0 +1,136 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + masterTaint = "node-role.kubernetes.io/master" + kubeadmConfigNamespace = "kube-system" + kubeadmConfigName = "kubeadm-config" + clusterInfoNamespace = "kube-public" + clusterInfoName = "cluster-info" + bootstrapSignerRoleNamespace = "kube-system" + bootstrapSignerRoleName = "system:controller:bootstrap-signer" +) + +var _ = framework.KubeDescribe("Kubeadm [Feature:Kubeadm]", func() { + f := framework.NewDefaultFramework("kubeadm") + + Describe("kubeadm master", func() { + It("should be labelled and tainted", func() { + selector := labels.Set{masterTaint: ""}.AsSelector() + master, err := f.ClientSet.CoreV1().Nodes(). + List(metav1.ListOptions{LabelSelector: selector.String()}) + framework.ExpectNoError(err, "couldn't find a master node") + Expect(master.Items).NotTo(BeEmpty()) + for _, master := range master.Items { + Expect(master.Spec.Taints).To( + ContainElement(taint(masterTaint, corev1.TaintEffectNoSchedule)), + ) + } + }) + }) + + Describe("kubeadm-config config map", func() { + It("should exist", func() { + _, err := f.ClientSet.CoreV1(). + ConfigMaps(kubeadmConfigNamespace). + Get(kubeadmConfigName, metav1.GetOptions{}) + framework.ExpectNoError(err) + }) + }) + + Describe("cluster-info", func() { + It("should have expected keys", func() { + clientInfo, err := f.ClientSet.CoreV1(). + ConfigMaps(clusterInfoNamespace). + Get(clusterInfoName, metav1.GetOptions{}) + framework.ExpectNoError(err, "couldn't find config map") + + Expect(clientInfo.Data).To(HaveKey(HavePrefix(bootstrapapi.JWSSignatureKeyPrefix))) + Expect(clientInfo.Data).To(HaveKey(bootstrapapi.KubeConfigKey)) + }) + + It("should be public", func() { + cfg, err := framework.LoadConfig() + framework.ExpectNoError(err, "couldn't get config") + cfg = rest.AnonymousClientConfig(cfg) + client, err := kubernetes.NewForConfig(cfg) + framework.ExpectNoError(err, "couldn't create client") + + _, err = client.CoreV1().ConfigMaps(clusterInfoNamespace). + Get(clusterInfoName, metav1.GetOptions{}) + framework.ExpectNoError(err, "couldn't anonymously access config") + }) + }) + + Describe("bootstrap signer RBAC role", func() { + It("should exist", func() { + _, err := f.ClientSet.RbacV1(). + Roles(bootstrapSignerRoleNamespace). + Get(bootstrapSignerRoleName, metav1.GetOptions{}) + framework.ExpectNoError(err, "doesn't exist") + }) + }) + + Describe("kubeadm:kubelet-bootstrap cluster role binding", func() { + It("should exist", func() { + binding, err := f.ClientSet.RbacV1(). + ClusterRoleBindings(). + Get("kubeadm:kubelet-bootstrap", metav1.GetOptions{}) + framework.ExpectNoError(err, "couldn't get clusterrolebinding") + Expect(binding.Subjects).To( + ContainElement(subject( + "system:bootstrappers:kubeadm:default-node-token", + rbacv1.GroupKind, + )), + ) + Expect(binding.RoleRef.Name).To(Equal("system:node-bootstrapper")) + }) + }) + + Describe("autoapproval for new bootstrap token", func() { + It("should create a clusterrolebinding", func() { + binding, err := f.ClientSet.RbacV1(). + ClusterRoleBindings(). + Get("kubeadm:node-autoapprove-bootstrap", metav1.GetOptions{}) + framework.ExpectNoError(err, "couldn't get clusterrolebinding") + Expect(binding.Subjects).To( + ContainElement(subject( + "system:bootstrappers:kubeadm:default-node-token", + rbacv1.GroupKind, + )), + ) + Expect(binding.RoleRef.Name).To( + Equal("system:certificates.k8s.io:certificatesigningrequests:nodeclient"), + ) + }) + }) +}) diff --git a/staging/src/k8s.io/apimachinery/pkg/apimachinery/types.go b/test/e2e_kubeadm/matchers.go similarity index 50% rename from staging/src/k8s.io/apimachinery/pkg/apimachinery/types.go rename to test/e2e_kubeadm/matchers.go index 4b2d113d248..8af786616b7 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apimachinery/types.go +++ b/test/e2e_kubeadm/matchers.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,17 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ -package apimachinery +package e2e_kubeadm import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/sets" + "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + corev1 "k8s.io/api/core/v1" ) -// GroupMeta stores the metadata of a group. -type GroupMeta struct { - // GroupVersions is Group + all versions in that group. - GroupVersions []schema.GroupVersion - - RootScopedKinds sets.String +func subject(name, kind string) gomega.OmegaMatcher { + return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Name": gomega.Equal(name), + "Kind": gomega.Equal(kind), + }) +} + +func taint(key string, effect corev1.TaintEffect) gomega.OmegaMatcher { + return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Key": gomega.Equal(key), + "Effect": gomega.Equal(effect), + }) } diff --git a/test/e2e_kubeadm/runner/local/BUILD b/test/e2e_kubeadm/runner/local/BUILD new file mode 100644 index 00000000000..412adbe48e3 --- /dev/null +++ b/test/e2e_kubeadm/runner/local/BUILD @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["run_local.go"], + importpath = "k8s.io/kubernetes/test/e2e_kubeadm/runner/local", + visibility = ["//visibility:private"], + deps = [ + "//test/utils:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + ], +) + +go_binary( + name = "local", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e_kubeadm/runner/local/run_local.go b/test/e2e_kubeadm/runner/local/run_local.go new file mode 100644 index 00000000000..9ccdfcb7309 --- /dev/null +++ b/test/e2e_kubeadm/runner/local/run_local.go @@ -0,0 +1,106 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/golang/glog" + "k8s.io/kubernetes/test/utils" +) + +func bazelBuild() error { + targets := []string{ + "//vendor/github.com/onsi/ginkgo/ginkgo", + "//test/e2e_kubeadm:e2e_kubeadm.test", + } + + args := append([]string{"build"}, targets...) + + return execCommand("bazel", args...) +} + +var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.") +var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.") +var build = flag.Bool("build", false, "use Bazel to build binaries before testing") + +func main() { + flag.Parse() + + if *build { + if err := bazelBuild(); err != nil { + glog.Exitf("couldn't build with bazel: %v", err) + } + } + + ginkgo, err := getBazelGinkgo() + if err != nil { + glog.Fatalf("Failed to get ginkgo binary: %v", err) + } + + test, err := getBazelTestBin() + if err != nil { + glog.Fatalf("Failed to get test file: %v", err) + } + + args := append(strings.Split(*ginkgoFlags, " "), test, "--") + args = append(args, strings.Split(*testFlags, " ")...) + + if execCommand(ginkgo, args...); err != nil { + glog.Exitf("Test failed: %v", err) + } + +} + +func getBazelTestBin() (string, error) { + k8sRoot, err := utils.GetK8sRootDir() + if err != nil { + return "", err + } + buildFile := filepath.Join(k8sRoot, "bazel-bin/test/e2e_kubeadm/e2e_kubeadm.test") + if _, err := os.Stat(buildFile); err != nil { + return "", err + } + return buildFile, nil + +} + +func getBazelGinkgo() (string, error) { + k8sRoot, err := utils.GetK8sRootDir() + if err != nil { + return "", err + } + buildOutputDir := filepath.Join(k8sRoot, "bazel-bin", "vendor/github.com/onsi/ginkgo/ginkgo", fmt.Sprintf("%s_%s_stripped", runtime.GOOS, runtime.GOARCH), "ginkgo") + if _, err := os.Stat(buildOutputDir); err != nil { + return "", err + } + return buildOutputDir, nil +} + +func execCommand(binary string, args ...string) error { + fmt.Printf("Running command: %v %v\n", binary, strings.Join(args, " ")) + cmd := exec.Command(binary, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/test/e2e_kubeadm/tests/BUILD b/test/e2e_kubeadm/tests/BUILD new file mode 100644 index 00000000000..6df04e38cd7 --- /dev/null +++ b/test/e2e_kubeadm/tests/BUILD @@ -0,0 +1,13 @@ +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e_node/dynamic_kubelet_config_test.go b/test/e2e_node/dynamic_kubelet_config_test.go index b3af81a665a..fe1b073dd8a 100644 --- a/test/e2e_node/dynamic_kubelet_config_test.go +++ b/test/e2e_node/dynamic_kubelet_config_test.go @@ -84,10 +84,12 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube // were initially set via the locally provisioned configuration. // This is the same strategy several other e2e node tests use. setAndTestKubeletConfigState(f, &configState{desc: "reset to original values", - configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - UID: originalConfigMap.UID, - Namespace: originalConfigMap.Namespace, - Name: originalConfigMap.Name}}, + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: originalConfigMap.UID, + Namespace: originalConfigMap.Namespace, + Name: originalConfigMap.Name, + KubeletConfigKey: "kubelet", + }}, expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionTrue, Message: fmt.Sprintf(status.CurRemoteMessageFmt, configMapAPIPath(originalConfigMap)), Reason: status.CurRemoteOkayReason}, @@ -123,8 +125,8 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube framework.ExpectNoError(err) states := []configState{ - // Node.Spec.ConfigSource is nil - {desc: "Node.Spec.ConfigSource is nil", + { + desc: "Node.Spec.ConfigSource is nil", configSource: nil, expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionTrue, Message: status.CurLocalMessage, @@ -132,65 +134,107 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube expectConfig: nil, event: true, }, - // Node.Spec.ConfigSource has all nil subfields {desc: "Node.Spec.ConfigSource has all nil subfields", configSource: &apiv1.NodeConfigSource{}, apierr: "exactly one reference subfield must be non-nil", }, - - // Node.Spec.ConfigSource.ConfigMapRef is partial - {desc: "Node.Spec.ConfigSource.ConfigMapRef is partial", - configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - UID: "foo", - Name: "bar"}}, // missing Namespace - apierr: "name, namespace, and UID must all be non-empty", + // Node.Spec.ConfigSource.ConfigMap is partial + {desc: "Node.Spec.ConfigSource.ConfigMap is partial", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: "foo", + Name: "bar", + KubeletConfigKey: "kubelet", + }}, // missing Namespace + apierr: "spec.configSource.configMap.namespace: Required value: namespace must be set in spec", }, - - // Node.Spec.ConfigSource's UID does not align with namespace/name - {desc: "Node.Spec.ConfigSource.ConfigMapRef.UID does not align with Namespace/Name", - configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{UID: "foo", - Namespace: correctConfigMap.Namespace, - Name: correctConfigMap.Name}}, + {desc: "Node.Spec.ConfigSource.ConfigMap.ResourceVersion is illegally specified", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: "foo", + Name: "bar", + Namespace: "baz", + ResourceVersion: "1", + KubeletConfigKey: "kubelet", + }}, + apierr: "spec.configSource.configMap.resourceVersion: Forbidden: resourceVersion must not be set in spec", + }, + {desc: "Node.Spec.ConfigSource.ConfigMap has invalid namespace", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: "foo", + Name: "bar", + Namespace: "../baz", + KubeletConfigKey: "kubelet", + }}, + apierr: "spec.configSource.configMap.namespace: Invalid value", + }, + {desc: "Node.Spec.ConfigSource.ConfigMap has invalid name", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: "foo", + Name: "../bar", + Namespace: "baz", + KubeletConfigKey: "kubelet", + }}, + apierr: "spec.configSource.configMap.name: Invalid value", + }, + {desc: "Node.Spec.ConfigSource.ConfigMap has invalid kubeletConfigKey", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: "foo", + Name: "bar", + Namespace: "baz", + KubeletConfigKey: "../qux", + }}, + apierr: "spec.configSource.configMap.kubeletConfigKey: Invalid value", + }, + { + desc: "Node.Spec.ConfigSource.ConfigMap.UID does not align with Namespace/Name", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: "foo", + Namespace: correctConfigMap.Namespace, + Name: correctConfigMap.Name, + KubeletConfigKey: "kubelet", + }}, expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionFalse, Message: "", Reason: fmt.Sprintf(status.FailSyncReasonFmt, fmt.Sprintf(status.FailSyncReasonUIDMismatchFmt, "foo", configMapAPIPath(correctConfigMap), correctConfigMap.UID))}, expectConfig: nil, event: false, }, - - // correct - {desc: "correct", - configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - UID: correctConfigMap.UID, - Namespace: correctConfigMap.Namespace, - Name: correctConfigMap.Name}}, + { + desc: "correct", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: correctConfigMap.UID, + Namespace: correctConfigMap.Namespace, + Name: correctConfigMap.Name, + KubeletConfigKey: "kubelet", + }}, expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionTrue, Message: fmt.Sprintf(status.CurRemoteMessageFmt, configMapAPIPath(correctConfigMap)), Reason: status.CurRemoteOkayReason}, expectConfig: correctKC, event: true, }, - - // fail-parse - {desc: "fail-parse", - configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - UID: failParseConfigMap.UID, - Namespace: failParseConfigMap.Namespace, - Name: failParseConfigMap.Name}}, + { + desc: "fail-parse", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: failParseConfigMap.UID, + Namespace: failParseConfigMap.Namespace, + Name: failParseConfigMap.Name, + KubeletConfigKey: "kubelet", + }}, expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionFalse, Message: status.LkgLocalMessage, Reason: fmt.Sprintf(status.CurFailLoadReasonFmt, configMapAPIPath(failParseConfigMap))}, expectConfig: nil, event: true, }, - - // fail-validate - {desc: "fail-validate", - configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - UID: failValidateConfigMap.UID, - Namespace: failValidateConfigMap.Namespace, - Name: failValidateConfigMap.Name}}, + { + desc: "fail-validate", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: failValidateConfigMap.UID, + Namespace: failValidateConfigMap.Namespace, + Name: failValidateConfigMap.Name, + KubeletConfigKey: "kubelet", + }}, expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionFalse, Message: status.LkgLocalMessage, Reason: fmt.Sprintf(status.CurFailValidateReasonFmt, configMapAPIPath(failValidateConfigMap))}, @@ -229,10 +273,12 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube states := []configState{ // intended lkg {desc: "intended last-known-good", - configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - UID: lkgConfigMap.UID, - Namespace: lkgConfigMap.Namespace, - Name: lkgConfigMap.Name}}, + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: lkgConfigMap.UID, + Namespace: lkgConfigMap.Namespace, + Name: lkgConfigMap.Name, + KubeletConfigKey: "kubelet", + }}, expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionTrue, Message: fmt.Sprintf(status.CurRemoteMessageFmt, configMapAPIPath(lkgConfigMap)), Reason: status.CurRemoteOkayReason}, @@ -242,10 +288,12 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube // bad config {desc: "bad config", - configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - UID: badConfigMap.UID, - Namespace: badConfigMap.Namespace, - Name: badConfigMap.Name}}, + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: badConfigMap.UID, + Namespace: badConfigMap.Namespace, + Name: badConfigMap.Name, + KubeletConfigKey: "kubelet", + }}, expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionFalse, Message: fmt.Sprintf(status.LkgRemoteMessageFmt, configMapAPIPath(lkgConfigMap)), Reason: fmt.Sprintf(status.CurFailLoadReasonFmt, configMapAPIPath(badConfigMap))}, @@ -259,6 +307,55 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube }) }) + Context("When a remote config becomes the new last-known-good, and then Node.ConfigSource.ConfigMap.KubeletConfigKey is updated to use a new, bad config", func() { + It("the Kubelet should report a status and configz indicating that it rolled back to the new last-known-good", func() { + const badConfigKey = "bad" + var err error + // we base the "lkg" configmap off of the current configuration + lkgKC := originalKC.DeepCopy() + combinedConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-combined", lkgKC) + combinedConfigMap.Data[badConfigKey] = "{0xdeadbeef}" + combinedConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(combinedConfigMap) + framework.ExpectNoError(err) + + states := []configState{ + // intended lkg + {desc: "intended last-known-good", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: combinedConfigMap.UID, + Namespace: combinedConfigMap.Namespace, + Name: combinedConfigMap.Name, + KubeletConfigKey: "kubelet", + }}, + expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionTrue, + Message: fmt.Sprintf(status.CurRemoteMessageFmt, configMapAPIPath(combinedConfigMap)), + Reason: status.CurRemoteOkayReason}, + expectConfig: lkgKC, + event: true, + }, + + // bad config + {desc: "bad config", + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: combinedConfigMap.UID, + Namespace: combinedConfigMap.Namespace, + Name: combinedConfigMap.Name, + KubeletConfigKey: badConfigKey, + }}, + expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionFalse, + // TODO(mtaufen): status should be more informative, and report the key being used + Message: fmt.Sprintf(status.LkgRemoteMessageFmt, configMapAPIPath(combinedConfigMap)), + Reason: fmt.Sprintf(status.CurFailLoadReasonFmt, configMapAPIPath(combinedConfigMap))}, + expectConfig: lkgKC, + event: true, + }, + } + + // wait 12 minutes after setting the first config to ensure it has time to pass the trial duration + testBothDirections(f, &states[0], states[1:], 12*time.Minute) + }) + }) + // This stress test will help turn up resource leaks across kubelet restarts that can, over time, // break our ability to dynamically update kubelet config Context("When changing the configuration 100 times", func() { @@ -280,10 +377,12 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube states := []configState{ {desc: "cm1", - configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - UID: cm1.UID, - Namespace: cm1.Namespace, - Name: cm1.Name}}, + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: cm1.UID, + Namespace: cm1.Namespace, + Name: cm1.Name, + KubeletConfigKey: "kubelet", + }}, expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionTrue, Message: fmt.Sprintf(status.CurRemoteMessageFmt, configMapAPIPath(cm1)), Reason: status.CurRemoteOkayReason}, @@ -292,10 +391,12 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube }, {desc: "cm2", - configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{ - UID: cm2.UID, - Namespace: cm2.Namespace, - Name: cm2.Name}}, + configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + UID: cm2.UID, + Namespace: cm2.Namespace, + Name: cm2.Name, + KubeletConfigKey: "kubelet", + }}, expectConfigOk: &apiv1.NodeCondition{Type: apiv1.NodeKubeletConfigOk, Status: apiv1.ConditionTrue, Message: fmt.Sprintf(status.CurRemoteMessageFmt, configMapAPIPath(cm2)), Reason: status.CurRemoteOkayReason}, @@ -335,7 +436,7 @@ func testBothDirections(f *framework.Framework, first *configState, states []con } } -// setAndTestKubeletConfigState tests that after setting the config source, the ConfigOk condition +// setAndTestKubeletConfigState tests that after setting the config source, the KubeletConfigOk condition // and (if appropriate) configuration exposed via conifgz are as expected. // The configuration will be converted to the internal type prior to comparison. func setAndTestKubeletConfigState(f *framework.Framework, state *configState, expectEvent bool) { @@ -479,8 +580,8 @@ func checkEvent(f *framework.Framework, desc string, expect *apiv1.NodeConfigSou // ensure the message is what we expect (including the resource path) expectMessage := fmt.Sprintf(controller.EventMessageFmt, controller.LocalConfigMessage) if expect != nil { - if expect.ConfigMapRef != nil { - expectMessage = fmt.Sprintf(controller.EventMessageFmt, fmt.Sprintf("/api/v1/namespaces/%s/configmaps/%s", expect.ConfigMapRef.Namespace, expect.ConfigMapRef.Name)) + if expect.ConfigMap != nil { + expectMessage = fmt.Sprintf(controller.EventMessageFmt, fmt.Sprintf("/api/v1/namespaces/%s/configmaps/%s", expect.ConfigMap.Namespace, expect.ConfigMap.Name)) } } if expectMessage != recent.Message { diff --git a/test/e2e_node/node_container_manager_test.go b/test/e2e_node/node_container_manager_test.go index c1bf7e2dce0..09e6dfad41d 100644 --- a/test/e2e_node/node_container_manager_test.go +++ b/test/e2e_node/node_container_manager_test.go @@ -38,7 +38,7 @@ import ( ) func setDesiredConfiguration(initialConfig *kubeletconfig.KubeletConfiguration) { - initialConfig.EnforceNodeAllocatable = []string{"pods", "kube-reserved", "system-reserved"} + initialConfig.EnforceNodeAllocatable = []string{"pods", kubeReservedCgroup, systemReservedCgroup} initialConfig.SystemReserved = map[string]string{ string(v1.ResourceCPU): "100m", string(v1.ResourceMemory): "100Mi", @@ -98,8 +98,8 @@ func getAllocatableLimits(cpu, memory string, capacity v1.ResourceList) (*resour } const ( - kubeReservedCgroup = "kube_reserved" - systemReservedCgroup = "system_reserved" + kubeReservedCgroup = "kube-reserved" + systemReservedCgroup = "system-reserved" ) func createIfNotExists(cm cm.CgroupManager, cgroupConfig *cm.CgroupConfig) error { diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index d86d11d3a40..4df85bcdebb 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -57,8 +57,10 @@ var startServices = flag.Bool("start-services", true, "If true, start local node var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests") var busyboxImage = "busybox" -// Kubelet internal cgroup name for node allocatable cgroup. -const defaultNodeAllocatableCgroup = "kubepods" +const ( + // Kubelet internal cgroup name for node allocatable cgroup. + defaultNodeAllocatableCgroup = "kubepods" +) func getNodeSummary() (*stats.Summary, error) { req, err := http.NewRequest("GET", *kubeletAddress+"/stats/summary", nil) @@ -167,10 +169,11 @@ func setKubeletConfiguration(f *framework.Framework, kubeCfg *kubeletconfig.Kube // create the reference and set Node.Spec.ConfigSource src := &apiv1.NodeConfigSource{ - ConfigMapRef: &apiv1.ObjectReference{ - Namespace: "kube-system", - Name: cm.Name, - UID: cm.UID, + ConfigMap: &apiv1.ConfigMapNodeConfigSource{ + Namespace: "kube-system", + Name: cm.Name, + UID: cm.UID, + KubeletConfigKey: "kubelet", }, } diff --git a/test/images/logs-generator/logs_generator.go b/test/images/logs-generator/logs_generator.go index 87189172f16..d11a0025e54 100644 --- a/test/images/logs-generator/logs_generator.go +++ b/test/images/logs-generator/logs_generator.go @@ -62,10 +62,11 @@ func generateLogs(linesTotal int, duration time.Duration) { delay := duration / time.Duration(linesTotal) rand.Seed(time.Now().UnixNano()) - tick := time.Tick(delay) + ticker := time.NewTicker(delay) + defer ticker.Stop() for id := 0; id < linesTotal; id++ { glog.Info(generateLogLine(id)) - <-tick + <-ticker.C } } diff --git a/test/integration/apiserver/BUILD b/test/integration/apiserver/BUILD index cc1d05cc73d..238349f9498 100644 --- a/test/integration/apiserver/BUILD +++ b/test/integration/apiserver/BUILD @@ -45,6 +45,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apiserver/pkg/features:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", diff --git a/test/integration/apiserver/print_test.go b/test/integration/apiserver/print_test.go index 511e05a7690..37ea8f8657b 100644 --- a/test/integration/apiserver/print_test.go +++ b/test/integration/apiserver/print_test.go @@ -19,9 +19,12 @@ package apiserver import ( "encoding/json" "fmt" + "io/ioutil" + "os" "reflect" "strings" "testing" + "time" batchv2alpha1 "k8s.io/api/batch/v2alpha1" rbacv1alpha1 "k8s.io/api/rbac/v1alpha1" @@ -31,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/discovery" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/gengo/examples/set-gen/sets" @@ -50,7 +54,6 @@ var kindWhiteList = sets.NewString( "ExportOptions", "GetOptions", "ListOptions", - "NodeConfigSource", "NodeProxyOptions", "PodAttachOptions", "PodExecOptions", @@ -146,7 +149,29 @@ func TestServerSidePrint(t *testing.T) { tableParam := fmt.Sprintf("application/json;as=Table;g=%s;v=%s, application/json", metav1beta1.GroupName, metav1beta1.SchemeGroupVersion.Version) printer := newFakePrinter(printersinternal.AddHandlers) - factory := util.NewFactory(clientcmd.NewDefaultClientConfig(*createKubeConfig(s.URL), &clientcmd.ConfigOverrides{})) + configFlags := util.NewTestConfigFlags(). + WithClientConfig(clientcmd.NewDefaultClientConfig(*createKubeConfig(s.URL), &clientcmd.ConfigOverrides{})) + + restConfig, err := configFlags.ToRESTConfig() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + cacheDir, err := ioutil.TempDir(os.TempDir(), "test-integration-apiserver-print") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + defer func() { + os.Remove(cacheDir) + }() + + configFlags.WithDiscoveryClient(util.NewCachedDiscoveryClient(discoveryClient, cacheDir, time.Duration(10*time.Minute))) + + factory := util.NewFactory(configFlags) mapper, err := factory.RESTMapper() if err != nil { t.Errorf("unexpected error getting mapper: %v", err) diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index 92efd530e7e..1840b058799 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -97,7 +97,7 @@ func TestNodeAuthorizer(t *testing.T) { // Set up NodeRestriction admission nodeRestrictionAdmission := noderestriction.NewPlugin(nodeidentifier.NewDefaultNodeIdentifier()) - nodeRestrictionAdmission.SetInternalKubeClientSet(superuserClient) + nodeRestrictionAdmission.SetInternalKubeInformerFactory(informerFactory) if err := nodeRestrictionAdmission.ValidateInitialization(); err != nil { t.Fatal(err) } @@ -286,12 +286,13 @@ func TestNodeAuthorizer(t *testing.T) { return err } node2.Spec.ConfigSource = &api.NodeConfigSource{ - ConfigMapRef: &api.ObjectReference{ + ConfigMap: &api.ConfigMapNodeConfigSource{ Namespace: "ns", Name: "myconfigmapconfigsource", // validation just requires UID to be non-empty and it isn't necessary for GET, // so we just use a bogus one for the test - UID: "uid", + UID: "uid", + KubeletConfigKey: "kubelet", }, } _, err = client.Core().Nodes().Update(node2) diff --git a/test/integration/daemonset/BUILD b/test/integration/daemonset/BUILD index f091f92c768..138dddfb5bf 100644 --- a/test/integration/daemonset/BUILD +++ b/test/integration/daemonset/BUILD @@ -23,6 +23,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", diff --git a/test/integration/daemonset/daemonset_test.go b/test/integration/daemonset/daemonset_test.go index e94ec026710..32f4a95ac5f 100644 --- a/test/integration/daemonset/daemonset_test.go +++ b/test/integration/daemonset/daemonset_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" @@ -40,6 +41,8 @@ import ( "k8s.io/kubernetes/test/integration/framework" ) +var zero = int64(0) + func setup(t *testing.T) (*httptest.Server, framework.CloseFunc, *daemon.DaemonSetsController, informers.SharedInformerFactory, clientset.Interface) { masterConfig := framework.NewIntegrationTestMasterConfig() _, server, closeFn := framework.RunAMaster(masterConfig) @@ -92,13 +95,55 @@ func newDaemonSet(name, namespace string) *apps.DaemonSet { Labels: testLabels(), }, Spec: v1.PodSpec{ - Containers: []v1.Container{{Name: "foo", Image: "bar"}}, + Containers: []v1.Container{{Name: "foo", Image: "bar"}}, + TerminationGracePeriodSeconds: &zero, }, }, }, } } +func cleanupDaemonSets(t *testing.T, cs clientset.Interface, ds *apps.DaemonSet) { + ds, err := cs.AppsV1().DaemonSets(ds.Namespace).Get(ds.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Failed to get DaemonSet %s/%s: %v", ds.Namespace, ds.Name, err) + return + } + + // We set the nodeSelector to a random label. This label is nearly guaranteed + // to not be set on any node so the DameonSetController will start deleting + // daemon pods. Once it's done deleting the daemon pods, it's safe to delete + // the DaemonSet. + ds.Spec.Template.Spec.NodeSelector = map[string]string{ + string(uuid.NewUUID()): string(uuid.NewUUID()), + } + // force update to avoid version conflict + ds.ResourceVersion = "" + + if ds, err = cs.AppsV1().DaemonSets(ds.Namespace).Update(ds); err != nil { + t.Errorf("Failed to update DaemonSet %s/%s: %v", ds.Namespace, ds.Name, err) + return + } + + // Wait for the daemon set controller to kill all the daemon pods. + if err := wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { + updatedDS, err := cs.AppsV1().DaemonSets(ds.Namespace).Get(ds.Name, metav1.GetOptions{}) + if err != nil { + return false, nil + } + return updatedDS.Status.CurrentNumberScheduled+updatedDS.Status.NumberMisscheduled == 0, nil + }); err != nil { + t.Errorf("Failed to kill the pods of DaemonSet %s/%s: %v", ds.Namespace, ds.Name, err) + return + } + + falseVar := false + deleteOptions := &metav1.DeleteOptions{OrphanDependents: &falseVar} + if err := cs.AppsV1().DaemonSets(ds.Namespace).Delete(ds.Name, deleteOptions); err != nil { + t.Errorf("Failed to delete DaemonSet %s/%s: %v", ds.Namespace, ds.Name, err) + } +} + func newRollbackStrategy() *apps.DaemonSetUpdateStrategy { one := intstr.FromInt(1) return &apps.DaemonSetUpdateStrategy{ @@ -140,6 +185,7 @@ func resourcePodSpec(nodeName, memory, cpu string) v1.PodSpec { }, }, }, + TerminationGracePeriodSeconds: &zero, } } @@ -257,8 +303,15 @@ func validateFailedPlacementEvent(eventClient corev1typed.EventInterface, t *tes } } -func TestOneNodeDaemonLaunchesPod(t *testing.T) { +func forEachStrategy(t *testing.T, tf func(t *testing.T, strategy *apps.DaemonSetUpdateStrategy)) { for _, strategy := range updateStrategies() { + t.Run(fmt.Sprintf("%s (%v)", t.Name(), strategy), + func(tt *testing.T) { tf(tt, strategy) }) + } +} + +func TestOneNodeDaemonLaunchesPod(t *testing.T) { + forEachStrategy(t, func(t *testing.T, strategy *apps.DaemonSetUpdateStrategy) { server, closeFn, dc, informers, clientset := setup(t) defer closeFn() ns := framework.CreateTestingNamespace("one-node-daemonset-test", server, t) @@ -268,9 +321,11 @@ func TestOneNodeDaemonLaunchesPod(t *testing.T) { podClient := clientset.CoreV1().Pods(ns.Name) nodeClient := clientset.CoreV1().Nodes() podInformer := informers.Core().V1().Pods().Informer() + stopCh := make(chan struct{}) informers.Start(stopCh) go dc.Run(5, stopCh) + defer close(stopCh) ds := newDaemonSet("foo", ns.Name) ds.Spec.UpdateStrategy = *strategy @@ -278,6 +333,8 @@ func TestOneNodeDaemonLaunchesPod(t *testing.T) { if err != nil { t.Fatalf("Failed to create DaemonSet: %v", err) } + defer cleanupDaemonSets(t, clientset, ds) + _, err = nodeClient.Create(newNode("single-node", nil)) if err != nil { t.Fatalf("Failed to create node: %v", err) @@ -285,13 +342,11 @@ func TestOneNodeDaemonLaunchesPod(t *testing.T) { validateDaemonSetPodsAndMarkReady(podClient, podInformer, 1, t) validateDaemonSetStatus(dsClient, ds.Name, ds.Namespace, 1, t) - - close(stopCh) - } + }) } func TestSimpleDaemonSetLaunchesPods(t *testing.T) { - for _, strategy := range updateStrategies() { + forEachStrategy(t, func(t *testing.T, strategy *apps.DaemonSetUpdateStrategy) { server, closeFn, dc, informers, clientset := setup(t) defer closeFn() ns := framework.CreateTestingNamespace("simple-daemonset-test", server, t) @@ -301,9 +356,11 @@ func TestSimpleDaemonSetLaunchesPods(t *testing.T) { podClient := clientset.CoreV1().Pods(ns.Name) nodeClient := clientset.CoreV1().Nodes() podInformer := informers.Core().V1().Pods().Informer() + stopCh := make(chan struct{}) informers.Start(stopCh) go dc.Run(5, stopCh) + defer close(stopCh) ds := newDaemonSet("foo", ns.Name) ds.Spec.UpdateStrategy = *strategy @@ -311,17 +368,17 @@ func TestSimpleDaemonSetLaunchesPods(t *testing.T) { if err != nil { t.Fatalf("Failed to create DaemonSet: %v", err) } + defer cleanupDaemonSets(t, clientset, ds) + addNodes(nodeClient, 0, 5, nil, t) validateDaemonSetPodsAndMarkReady(podClient, podInformer, 5, t) validateDaemonSetStatus(dsClient, ds.Name, ds.Namespace, 5, t) - - close(stopCh) - } + }) } func TestNotReadyNodeDaemonDoesLaunchPod(t *testing.T) { - for _, strategy := range updateStrategies() { + forEachStrategy(t, func(t *testing.T, strategy *apps.DaemonSetUpdateStrategy) { server, closeFn, dc, informers, clientset := setup(t) defer closeFn() ns := framework.CreateTestingNamespace("simple-daemonset-test", server, t) @@ -331,9 +388,11 @@ func TestNotReadyNodeDaemonDoesLaunchPod(t *testing.T) { podClient := clientset.CoreV1().Pods(ns.Name) nodeClient := clientset.CoreV1().Nodes() podInformer := informers.Core().V1().Pods().Informer() + stopCh := make(chan struct{}) informers.Start(stopCh) go dc.Run(5, stopCh) + defer close(stopCh) ds := newDaemonSet("foo", ns.Name) ds.Spec.UpdateStrategy = *strategy @@ -341,6 +400,8 @@ func TestNotReadyNodeDaemonDoesLaunchPod(t *testing.T) { if err != nil { t.Fatalf("Failed to create DaemonSet: %v", err) } + defer cleanupDaemonSets(t, clientset, ds) + node := newNode("single-node", nil) node.Status.Conditions = []v1.NodeCondition{ {Type: v1.NodeReady, Status: v1.ConditionFalse}, @@ -352,13 +413,11 @@ func TestNotReadyNodeDaemonDoesLaunchPod(t *testing.T) { validateDaemonSetPodsAndMarkReady(podClient, podInformer, 1, t) validateDaemonSetStatus(dsClient, ds.Name, ds.Namespace, 1, t) - - close(stopCh) - } + }) } func TestInsufficientCapacityNodeDaemonDoesNotLaunchPod(t *testing.T) { - for _, strategy := range updateStrategies() { + forEachStrategy(t, func(t *testing.T, strategy *apps.DaemonSetUpdateStrategy) { server, closeFn, dc, informers, clientset := setup(t) defer closeFn() ns := framework.CreateTestingNamespace("insufficient-capacity", server, t) @@ -367,9 +426,11 @@ func TestInsufficientCapacityNodeDaemonDoesNotLaunchPod(t *testing.T) { dsClient := clientset.AppsV1().DaemonSets(ns.Name) nodeClient := clientset.CoreV1().Nodes() eventClient := clientset.CoreV1().Events(ns.Namespace) + stopCh := make(chan struct{}) informers.Start(stopCh) go dc.Run(5, stopCh) + defer close(stopCh) ds := newDaemonSet("foo", ns.Name) ds.Spec.Template.Spec = resourcePodSpec("node-with-limited-memory", "120M", "75m") @@ -378,6 +439,8 @@ func TestInsufficientCapacityNodeDaemonDoesNotLaunchPod(t *testing.T) { if err != nil { t.Fatalf("Failed to create DaemonSet: %v", err) } + defer cleanupDaemonSets(t, clientset, ds) + node := newNode("node-with-limited-memory", nil) node.Status.Allocatable = allocatableResources("100M", "200m") _, err = nodeClient.Create(node) @@ -386,7 +449,5 @@ func TestInsufficientCapacityNodeDaemonDoesNotLaunchPod(t *testing.T) { } validateFailedPlacementEvent(eventClient, t) - - close(stopCh) - } + }) } diff --git a/test/integration/etcd/BUILD b/test/integration/etcd/BUILD index 7bdff725faf..4198e15cc06 100644 --- a/test/integration/etcd/BUILD +++ b/test/integration/etcd/BUILD @@ -37,10 +37,10 @@ go_test( "//vendor/k8s.io/apiserver/pkg/server:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", - "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/discovery/cached:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", ], ) diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index d91cb0bbdfd..262e50f85b5 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -43,7 +43,6 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/storagebackend" - "k8s.io/client-go/discovery" cacheddiscovery "k8s.io/client-go/discovery/cached" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" @@ -59,6 +58,7 @@ import ( _ "k8s.io/kubernetes/pkg/master" // TODO what else is needed "github.com/coreos/etcd/clientv3" + "k8s.io/client-go/restmapper" ) // Etcd data for all persisted objects. @@ -437,7 +437,6 @@ var ephemeralWhiteList = createEphemeralWhiteList( gvk("", "v1", "RangeAllocation"), // stored in various places in etcd but cannot be directly created gvk("", "v1", "ComponentStatus"), // status info not stored in etcd gvk("", "v1", "SerializedReference"), // used for serilization, not stored in etcd - gvk("", "v1", "NodeConfigSource"), // subfield of node.spec, but shouldn't be directly created gvk("", "v1", "PodStatusResult"), // wrapper object not stored in etcd // -- @@ -802,12 +801,10 @@ func startRealMasterOrDie(t *testing.T, certDir string) (*allClient, clientv3.KV } discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery()) - restMapper := discovery.NewDeferredDiscoveryRESTMapper(discoveryClient) + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) restMapper.Reset() - // allow conversion between typed and unstructured objects - mapper := discovery.NewDeferredDiscoveryRESTMapper(discoveryClient) - return client, kvClient, mapper + return client, kvClient, restMapper } func dumpEtcdKVOnFailure(t *testing.T, kvClient clientv3.KV) { diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 68cf9cff041..8b0d00a9633 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -429,9 +429,9 @@ func testAPIGroupList(t *testing.T, client rest.Interface) { Version: wardlev1beta1.SchemeGroupVersion.Version, } - assert.Equal(t, v1alpha1, apiGroupList.Groups[0].Versions[0]) - assert.Equal(t, v1beta1, apiGroupList.Groups[0].Versions[1]) - assert.Equal(t, v1alpha1, apiGroupList.Groups[0].PreferredVersion) + assert.Equal(t, v1beta1, apiGroupList.Groups[0].Versions[0]) + assert.Equal(t, v1alpha1, apiGroupList.Groups[0].Versions[1]) + assert.Equal(t, v1beta1, apiGroupList.Groups[0].PreferredVersion) } func testAPIGroup(t *testing.T, client rest.Interface) { @@ -447,8 +447,8 @@ func testAPIGroup(t *testing.T, client rest.Interface) { } assert.Equal(t, wardlev1alpha1.SchemeGroupVersion.Group, apiGroup.Name) assert.Equal(t, 2, len(apiGroup.Versions)) - assert.Equal(t, wardlev1alpha1.SchemeGroupVersion.String(), apiGroup.Versions[0].GroupVersion) - assert.Equal(t, wardlev1alpha1.SchemeGroupVersion.Version, apiGroup.Versions[0].Version) + assert.Equal(t, wardlev1alpha1.SchemeGroupVersion.String(), apiGroup.Versions[1].GroupVersion) + assert.Equal(t, wardlev1alpha1.SchemeGroupVersion.Version, apiGroup.Versions[1].Version) assert.Equal(t, apiGroup.PreferredVersion, apiGroup.Versions[0]) } diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index e5e15c0f219..387d3bb4d47 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -235,7 +235,7 @@ func NewMasterConfig() *master.Config { info, _ := runtime.SerializerInfoForMediaType(legacyscheme.Codecs.SupportedMediaTypes(), runtime.ContentTypeJSON) ns := NewSingleContentTypeSerializer(legacyscheme.Scheme, info) - resourceEncoding := serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Registry) + resourceEncoding := serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme) // FIXME (soltysh): this GroupVersionResource override should be configurable // we need to set both for the whole group and for cronjobs, separately resourceEncoding.SetVersionEncoding(batch.GroupName, *testapi.Batch.GroupVersion(), schema.GroupVersion{Group: batch.GroupName, Version: runtime.APIVersionInternal}) diff --git a/test/integration/garbagecollector/BUILD b/test/integration/garbagecollector/BUILD index 8bc8f3eeb83..9229bdb7dc6 100644 --- a/test/integration/garbagecollector/BUILD +++ b/test/integration/garbagecollector/BUILD @@ -27,11 +27,11 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/names:go_default_library", - "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/discovery/cached:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/client-go/restmapper:go_default_library", "//vendor/k8s.io/client-go/tools/cache:go_default_library", ], ) diff --git a/test/integration/garbagecollector/garbage_collector_test.go b/test/integration/garbagecollector/garbage_collector_test.go index 2e20d637513..83fc2604865 100644 --- a/test/integration/garbagecollector/garbage_collector_test.go +++ b/test/integration/garbagecollector/garbage_collector_test.go @@ -36,11 +36,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/storage/names" - "k8s.io/client-go/discovery" cacheddiscovery "k8s.io/client-go/discovery/cached" "k8s.io/client-go/dynamic" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/cache" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/controller/garbagecollector" @@ -220,13 +220,10 @@ func setupWithServer(t *testing.T, result *kubeapiservertesting.TestServer, work createNamespaceOrDie("aval", clientSet, t) discoveryClient := cacheddiscovery.NewMemCacheClient(clientSet.Discovery()) - restMapper := discovery.NewDeferredDiscoveryRESTMapper(discoveryClient) + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) restMapper.Reset() deletableResources := garbagecollector.GetDeletableResources(discoveryClient) config := *result.ClientConfig - config.ContentConfig = dynamic.ContentConfig() - metaOnlyClientPool := dynamic.NewClientPool(&config, restMapper, dynamic.LegacyAPIPathResolverFunc) - clientPool := dynamic.NewClientPool(&config, restMapper, dynamic.LegacyAPIPathResolverFunc) dynamicClient, err := dynamic.NewForConfig(&config) if err != nil { t.Fatalf("failed to create dynamicClient: %v", err) @@ -235,8 +232,7 @@ func setupWithServer(t *testing.T, result *kubeapiservertesting.TestServer, work alwaysStarted := make(chan struct{}) close(alwaysStarted) gc, err := garbagecollector.NewGarbageCollector( - metaOnlyClientPool, - clientPool, + dynamicClient, restMapper, deletableResources, garbagecollector.DefaultIgnoredResources(), diff --git a/test/integration/scheduler/preemption_test.go b/test/integration/scheduler/preemption_test.go index af88cd84d6c..8e2e4976b79 100644 --- a/test/integration/scheduler/preemption_test.go +++ b/test/integration/scheduler/preemption_test.go @@ -506,7 +506,11 @@ func TestNominatedNodeCleanUp(t *testing.T) { // Initialize scheduler. context := initTest(t, "preemption") defer cleanupTest(t, context) + cs := context.clientSet + + defer cleanupPodsInNamespace(cs, t, context.ns.Name) + // Create a node with some resources and a label. nodeRes := &v1.ResourceList{ v1.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), diff --git a/test/integration/scheduler/util.go b/test/integration/scheduler/util.go index 6074913556b..d42803c1786 100644 --- a/test/integration/scheduler/util.go +++ b/test/integration/scheduler/util.go @@ -534,3 +534,29 @@ func cleanupPods(cs clientset.Interface, t *testing.T, pods []*v1.Pod) { } } } + +// noPodsInNamespace returns true if no pods in the given namespace. +func noPodsInNamespace(c clientset.Interface, podNamespace string) wait.ConditionFunc { + return func() (bool, error) { + pods, err := c.CoreV1().Pods(podNamespace).List(metav1.ListOptions{}) + if err != nil { + return false, err + } + + return len(pods.Items) == 0, nil + } +} + +// cleanupPodsInNamespace deletes the pods in the given namespace and waits for them to +// be actually deleted. +func cleanupPodsInNamespace(cs clientset.Interface, t *testing.T, ns string) { + if err := cs.CoreV1().Pods(ns).DeleteCollection(nil, metav1.ListOptions{}); err != nil { + t.Errorf("error while listing pod in namespace %v: %v", ns, err) + return + } + + if err := wait.Poll(time.Second, wait.ForeverTestTimeout, + noPodsInNamespace(cs, ns)); err != nil { + t.Errorf("error while waiting for pods in namespace %v: %v", ns, err) + } +} diff --git a/vendor/k8s.io/kube-openapi/pkg/aggregator/aggregator.go b/vendor/k8s.io/kube-openapi/pkg/aggregator/aggregator.go index e453a27bdbd..f05b5017252 100644 --- a/vendor/k8s.io/kube-openapi/pkg/aggregator/aggregator.go +++ b/vendor/k8s.io/kube-openapi/pkg/aggregator/aggregator.go @@ -61,6 +61,10 @@ func (s *referenceWalker) walkRef(ref spec.Ref) spec.Ref { k := refStr[len(definitionPrefix):] def := s.root.Definitions[k] s.walkSchema(&def) + // Make sure we don't assign to nil map + if s.root.Definitions == nil { + s.root.Definitions = spec.Definitions{} + } s.root.Definitions[k] = def } return s.walkRefCallback(ref) @@ -147,6 +151,9 @@ func (s *referenceWalker) walkOperation(op *spec.Operation) { } func (s *referenceWalker) Start() { + if s.root.Paths == nil { + return + } for _, pathItem := range s.root.Paths.Paths { s.walkParams(pathItem.Parameters) s.walkOperation(pathItem.Delete) @@ -220,6 +227,10 @@ func renameDefinition(s *spec.Swagger, old, new string) { } return ref }, s) + // Make sure we don't assign to nil map + if s.Definitions == nil { + s.Definitions = spec.Definitions{} + } s.Definitions[new] = s.Definitions[old] delete(s.Definitions, old) } @@ -244,6 +255,13 @@ func MergeSpecs(dest, source *spec.Swagger) error { func mergeSpecs(dest, source *spec.Swagger, renameModelConflicts, ignorePathConflicts bool) (err error) { specCloned := false + // Paths may be empty, due to [ACL constraints](http://goo.gl/8us55a#securityFiltering). + if source.Paths == nil { + source.Paths = &spec.Paths{} + } + if dest.Paths == nil { + dest.Paths = &spec.Paths{} + } if ignorePathConflicts { keepPaths := []string{} hasConflictingPath := false @@ -346,6 +364,10 @@ func mergeSpecs(dest, source *spec.Swagger, renameModelConflicts, ignorePathConf if _, found := dest.Paths.Paths[k]; found { return fmt.Errorf("unable to merge: duplicated path %s", k) } + // PathItem may be empty, due to [ACL constraints](http://goo.gl/8us55a#securityFiltering). + if dest.Paths.Paths == nil { + dest.Paths.Paths = map[string]spec.PathItem{} + } dest.Paths.Paths[k] = v } return nil