diff --git a/hack/make-rules/test-cmd-util.sh b/hack/make-rules/test-cmd-util.sh index 7a32851dfa4..835a5a229c1 100755 --- a/hack/make-rules/test-cmd-util.sh +++ b/hack/make-rules/test-cmd-util.sh @@ -41,9 +41,11 @@ IMAGE_NGINX="gcr.io/google-containers/nginx:1.7.9" IMAGE_DEPLOYMENT_R1="gcr.io/google-containers/nginx:test-cmd" # deployment-revision1.yaml IMAGE_DEPLOYMENT_R2="$IMAGE_NGINX" # deployment-revision2.yaml IMAGE_PERL="gcr.io/google-containers/perl" -IMAGE_DAEMONSET_R1="gcr.io/google-containers/pause:2.0" +IMAGE_PAUSE_V2="gcr.io/google-containers/pause:2.0" IMAGE_DAEMONSET_R2="gcr.io/google-containers/pause:latest" IMAGE_DAEMONSET_R2_2="gcr.io/google-containers/nginx:test-cmd" # rollingupdate-daemonset-rv2.yaml +IMAGE_STATEFULSET_R1="gcr.io/google_containers/nginx-slim:0.7" +IMAGE_STATEFULSET_R2="gcr.io/google_containers/nginx-slim:0.8" # Expose kubectl directly for readability PATH="${KUBE_OUTPUT_HOSTBIN}":$PATH @@ -1859,7 +1861,7 @@ run_recursive_resources_tests() { # Create a deployment (revision 1) kubectl create -f hack/testdata/deployment-revision1.yaml "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx:' - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Command output_message=$(kubectl convert --local -f hack/testdata/deployment-revision1.yaml --output-version=apps/v1beta1 -o go-template='{{ .apiVersion }}' "${kube_flags[@]}") echo $output_message @@ -1982,11 +1984,11 @@ run_recursive_resources_tests() { # Create deployments (revision 1) recursively from directory of YAML files ! kubectl create -f hack/testdata/recursive/deployment --recursive "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx0-deployment:nginx1-deployment:' - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_NGINX}:${IMAGE_NGINX}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_NGINX}:${IMAGE_NGINX}:" ## Rollback the deployments to revision 1 recursively output_message=$(! kubectl rollout undo -f hack/testdata/recursive/deployment --recursive --to-revision=1 2>&1 "${kube_flags[@]}") # Post-condition: nginx0 & nginx1 should be a no-op, and since nginx2 is malformed, it should error - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_NGINX}:${IMAGE_NGINX}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_NGINX}:${IMAGE_NGINX}:" kube::test::if_has_string "${output_message}" "Object 'Kind' is missing" ## Pause the deployments recursively PRESERVE_ERR_FILE=true @@ -2621,8 +2623,8 @@ run_rc_tests() { # Create a deployment kubectl create -f hack/testdata/deployment-multicontainer-resources.yaml "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx-deployment-resources:' - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set the deployment's cpu limits kubectl set resources deployment nginx-deployment-resources --limits=cpu=100m "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{(index .spec.template.spec.containers 0).resources.limits.cpu}}:{{end}}" "100m:" @@ -2753,27 +2755,27 @@ run_deployment_tests() { # Create a deployment (revision 1) kubectl create -f hack/testdata/deployment-revision1.yaml "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx:' - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Rollback to revision 1 - should be no-op kubectl rollout undo deployment nginx --to-revision=1 "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Update the deployment (revision 2) kubectl apply -f hack/testdata/deployment-revision2.yaml "${kube_flags[@]}" - kube::test::get_object_assert deployment.extensions "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment.extensions "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" # Rollback to revision 1 with dry-run - should be no-op kubectl rollout undo deployment nginx --dry-run=true "${kube_flags[@]}" | grep "test-cmd" - kube::test::get_object_assert deployment.extensions "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment.extensions "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" # Rollback to revision 1 kubectl rollout undo deployment nginx --to-revision=1 "${kube_flags[@]}" sleep 1 - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Rollback to revision 1000000 - should be no-op kubectl rollout undo deployment nginx --to-revision=1000000 "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Rollback to last revision kubectl rollout undo deployment nginx "${kube_flags[@]}" sleep 1 - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" # Pause the deployment kubectl-with-retry rollout pause deployment nginx "${kube_flags[@]}" # A paused deployment cannot be rolled back @@ -2799,34 +2801,35 @@ run_deployment_tests() { # Create a deployment kubectl create -f hack/testdata/deployment-multicontainer.yaml "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx-deployment:' - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set the deployment's image kubectl set image deployment nginx-deployment nginx="${IMAGE_DEPLOYMENT_R2}" "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set non-existing container should fail ! kubectl set image deployment nginx-deployment redis=redis "${kube_flags[@]}" # Set image of deployments without specifying name kubectl set image deployments --all nginx="${IMAGE_DEPLOYMENT_R1}" "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set image of a deployment specified by file kubectl set image -f hack/testdata/deployment-multicontainer.yaml nginx="${IMAGE_DEPLOYMENT_R2}" "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set image of a local file without talking to the server kubectl set image -f hack/testdata/deployment-multicontainer.yaml nginx="${IMAGE_DEPLOYMENT_R1}" "${kube_flags[@]}" --local -o yaml - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set image of all containers of the deployment kubectl set image deployment nginx-deployment "*"="${IMAGE_DEPLOYMENT_R1}" "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Set image of all containners of the deployment again when image not change kubectl set image deployment nginx-deployment "*"="${IMAGE_DEPLOYMENT_R1}" "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + # Clean up kubectl delete deployment nginx-deployment "${kube_flags[@]}" @@ -3004,35 +3007,35 @@ run_daemonset_history_tests() { # Command # Create a DaemonSet (revision 1) kubectl apply -f hack/testdata/rollingupdate-daemonset.yaml --record "${kube_flags[@]}" - kube::test::get_object_assert controllerrevisions "{{range.items}}{{$annotations_field}}:{{end}}" ".*rollingupdate-daemonset.yaml --record.*" + kube::test::wait_object_assert controllerrevisions "{{range.items}}{{$annotations_field}}:{{end}}" ".*rollingupdate-daemonset.yaml --record.*" # Rollback to revision 1 - should be no-op kubectl rollout undo daemonset --to-revision=1 "${kube_flags[@]}" - kube::test::get_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R1}:" + kube::test::get_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_PAUSE_V2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "1" # Update the DaemonSet (revision 2) kubectl apply -f hack/testdata/rollingupdate-daemonset-rv2.yaml --record "${kube_flags[@]}" - kube::test::wait_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" - kube::test::wait_object_assert daemonset "{{range.items}}{{$daemonset_image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" + kube::test::wait_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" + kube::test::wait_object_assert daemonset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "2" kube::test::wait_object_assert controllerrevisions "{{range.items}}{{$annotations_field}}:{{end}}" ".*rollingupdate-daemonset-rv2.yaml --record.*" # Rollback to revision 1 with dry-run - should be no-op kubectl rollout undo daemonset --dry-run=true "${kube_flags[@]}" - kube::test::get_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" - kube::test::get_object_assert daemonset "{{range.items}}{{$daemonset_image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" + kube::test::get_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" + kube::test::get_object_assert daemonset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "2" # Rollback to revision 1 kubectl rollout undo daemonset --to-revision=1 "${kube_flags[@]}" - kube::test::wait_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R1}:" + kube::test::wait_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_PAUSE_V2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "1" # Rollback to revision 1000000 - should fail output_message=$(! kubectl rollout undo daemonset --to-revision=1000000 "${kube_flags[@]}" 2>&1) kube::test::if_has_string "${output_message}" "unable to find specified revision" - kube::test::get_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R1}:" + kube::test::get_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_PAUSE_V2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "1" # Rollback to last revision kubectl rollout undo daemonset "${kube_flags[@]}" - kube::test::wait_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" - kube::test::wait_object_assert daemonset "{{range.items}}{{$daemonset_image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" + kube::test::wait_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" + kube::test::wait_object_assert daemonset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "2" # Clean up kubectl delete -f hack/testdata/rollingupdate-daemonset.yaml "${kube_flags[@]}" @@ -3041,6 +3044,58 @@ run_daemonset_history_tests() { set +o errexit } +run_statefulset_history_tests() { + set -o nounset + set -o errexit + + create_and_use_new_namespace + kube::log::status "Testing kubectl(v1:statefulsets, v1:controllerrevisions)" + + ### Test rolling back a StatefulSet + # Pre-condition: no statefulset or its pods exists + kube::test::get_object_assert statefulset "{{range.items}}{{$id_field}}:{{end}}" '' + # Command + # Create a StatefulSet (revision 1) + kubectl apply -f hack/testdata/rollingupdate-statefulset.yaml --record "${kube_flags[@]}" + kube::test::wait_object_assert controllerrevisions "{{range.items}}{{$annotations_field}}:{{end}}" ".*rollingupdate-statefulset.yaml --record.*" + # Rollback to revision 1 - should be no-op + kubectl rollout undo statefulset --to-revision=1 "${kube_flags[@]}" + kube::test::get_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R1}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "1" + # Update the statefulset (revision 2) + kubectl apply -f hack/testdata/rollingupdate-statefulset-rv2.yaml --record "${kube_flags[@]}" + kube::test::wait_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R2}:" + kube::test::wait_object_assert statefulset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PAUSE_V2}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "2" + kube::test::wait_object_assert controllerrevisions "{{range.items}}{{$annotations_field}}:{{end}}" ".*rollingupdate-statefulset-rv2.yaml --record.*" + # Rollback to revision 1 with dry-run - should be no-op + kubectl rollout undo statefulset --dry-run=true "${kube_flags[@]}" + kube::test::get_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R2}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PAUSE_V2}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "2" + # Rollback to revision 1 + kubectl rollout undo statefulset --to-revision=1 "${kube_flags[@]}" + kube::test::wait_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R1}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "1" + # Rollback to revision 1000000 - should fail + output_message=$(! kubectl rollout undo statefulset --to-revision=1000000 "${kube_flags[@]}" 2>&1) + kube::test::if_has_string "${output_message}" "unable to find specified revision" + kube::test::get_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R1}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "1" + # Rollback to last revision + kubectl rollout undo statefulset "${kube_flags[@]}" + kube::test::wait_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R2}:" + kube::test::wait_object_assert statefulset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PAUSE_V2}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "2" + # Clean up - delete newest configuration + kubectl delete -f hack/testdata/rollingupdate-statefulset-rv2.yaml "${kube_flags[@]}" + # Post-condition: no pods from statefulset controller + wait-for-pods-with-label "app=nginx-statefulset" "" + + set +o nounset + set +o errexit +} + run_multi_resources_tests() { set -o nounset set -o errexit @@ -3618,7 +3673,7 @@ run_stateful_set_tests() { # Pre-condition: no statefulset exists kube::test::get_object_assert statefulset "{{range.items}}{{$id_field}}:{{end}}" '' # Command: create statefulset - kubectl create -f hack/testdata/nginx-statefulset.yaml "${kube_flags[@]}" + kubectl create -f hack/testdata/rollingupdate-statefulset.yaml "${kube_flags[@]}" ### Scale statefulset test with current-replicas and replicas # Pre-condition: 0 replicas @@ -3635,7 +3690,7 @@ run_stateful_set_tests() { wait-for-pods-with-label "app=nginx-statefulset" "nginx-0" ### Clean up - kubectl delete -f hack/testdata/nginx-statefulset.yaml "${kube_flags[@]}" + kubectl delete -f hack/testdata/rollingupdate-statefulset.yaml "${kube_flags[@]}" # Post-condition: no pods from statefulset controller wait-for-pods-with-label "app=nginx-statefulset" "" @@ -4220,15 +4275,13 @@ runTests() { deployment_replicas=".spec.replicas" secret_data=".data" secret_type=".type" - deployment_image_field="(index .spec.template.spec.containers 0).image" - deployment_second_image_field="(index .spec.template.spec.containers 1).image" change_cause_annotation='.*kubernetes.io/change-cause.*' pdb_min_available=".spec.minAvailable" pdb_max_unavailable=".spec.maxUnavailable" template_generation_field=".spec.templateGeneration" container_len="(len .spec.template.spec.containers)" - daemonset_image_field0="(index .spec.template.spec.containers 0).image" - daemonset_image_field1="(index .spec.template.spec.containers 1).image" + image_field0="(index .spec.template.spec.containers 0).image" + image_field1="(index .spec.template.spec.containers 1).image" # Make sure "default" namespace exists. if kube::test::if_supports_resource "${namespaces}" ; then @@ -4434,7 +4487,6 @@ runTests() { record_command run_service_tests fi - ################## # DaemonSets # ################## @@ -4472,16 +4524,17 @@ runTests() { record_command run_rs_tests fi - ################# # Stateful Sets # ################# if kube::test::if_supports_resource "${statefulsets}" ; then record_command run_stateful_set_tests + if kube::test::if_supports_resource "${controllerrevisions}"; then + record_command run_statefulset_history_tests + fi fi - ###################### # Lists # ###################### diff --git a/hack/testdata/rollingupdate-statefulset-rv2.yaml b/hack/testdata/rollingupdate-statefulset-rv2.yaml new file mode 100644 index 00000000000..fec5493ab69 --- /dev/null +++ b/hack/testdata/rollingupdate-statefulset-rv2.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1beta2 +kind: StatefulSet +metadata: + name: nginx +spec: + selector: + matchLabels: + app: nginx-statefulset + updateStrategy: + type: RollingUpdate + serviceName: "nginx" + replicas: 0 + template: + metadata: + labels: + app: nginx-statefulset + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: nginx + image: gcr.io/google_containers/nginx-slim:0.8 + ports: + - containerPort: 80 + name: web + command: + - sh + - -c + - 'while true; do sleep 1; done' + - name: pause + image: gcr.io/google-containers/pause:2.0 + ports: + - containerPort: 81 + name: web-2 diff --git a/hack/testdata/nginx-statefulset.yaml b/hack/testdata/rollingupdate-statefulset.yaml similarity index 50% rename from hack/testdata/nginx-statefulset.yaml rename to hack/testdata/rollingupdate-statefulset.yaml index 299e407dfa3..2acbf0f322b 100644 --- a/hack/testdata/nginx-statefulset.yaml +++ b/hack/testdata/rollingupdate-statefulset.yaml @@ -1,26 +1,13 @@ -# A headless service to create DNS records -apiVersion: v1 -kind: Service -metadata: - annotations: - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" - name: nginx - labels: - app: nginx-statefulset -spec: - ports: - - port: 80 - name: web - # *.nginx.default.svc.cluster.local - clusterIP: None - selector: - app: nginx-statefulset ---- -apiVersion: apps/v1beta1 +apiVersion: apps/v1beta2 kind: StatefulSet metadata: name: nginx spec: + selector: + matchLabels: + app: nginx-statefulset + updateStrategy: + type: RollingUpdate serviceName: "nginx" replicas: 0 template: @@ -28,7 +15,7 @@ spec: labels: app: nginx-statefulset spec: - terminationGracePeriodSeconds: 0 + terminationGracePeriodSeconds: 5 containers: - name: nginx image: gcr.io/google_containers/nginx-slim:0.7 diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index 0fe3ff9708c..dd264790d9e 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -256,11 +256,11 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( collisionCount int32, pods []*v1.Pod) (*apps.StatefulSetStatus, error) { // get the current and update revisions of the set. - currentSet, err := applyRevision(set, currentRevision) + currentSet, err := ApplyRevision(set, currentRevision) if err != nil { return nil, err } - updateSet, err := applyRevision(set, updateRevision) + updateSet, err := ApplyRevision(set, updateRevision) if err != nil { return nil, err } diff --git a/pkg/controller/statefulset/stateful_set_control_test.go b/pkg/controller/statefulset/stateful_set_control_test.go index 548180d34d7..bcb38badf45 100644 --- a/pkg/controller/statefulset/stateful_set_control_test.go +++ b/pkg/controller/statefulset/stateful_set_control_test.go @@ -1281,7 +1281,7 @@ func TestStatefulSetControlRollback(t *testing.T) { t.Fatalf("%s: %s", test.name, err) } history.SortControllerRevisions(revisions) - set, err = applyRevision(set, revisions[0]) + set, err = ApplyRevision(set, revisions[0]) if err != nil { t.Fatalf("%s: %s", test.name, err) } diff --git a/pkg/controller/statefulset/stateful_set_utils.go b/pkg/controller/statefulset/stateful_set_utils.go index 0dc861a7212..fb80888a956 100644 --- a/pkg/controller/statefulset/stateful_set_utils.go +++ b/pkg/controller/statefulset/stateful_set_utils.go @@ -17,6 +17,7 @@ limitations under the License. package statefulset import ( + "bytes" "encoding/json" "fmt" "regexp" @@ -266,6 +267,15 @@ func newVersionedStatefulSetPod(currentSet, updateSet *apps.StatefulSet, current return pod } +// Match check if the given StatefulSet's template matches the template stored in the given history. +func Match(ss *apps.StatefulSet, history *apps.ControllerRevision) (bool, error) { + patch, err := getPatch(ss) + if err != nil { + return false, err + } + return bytes.Equal(patch, history.Data.Raw), nil +} + // getPatch returns a strategic merge patch that can be applied to restore a StatefulSet to a // previous version. If the returned error is nil the patch is valid. The current state that we save is just the // PodSpecTemplate. We can modify this later to encompass more state (or less) and remain compatible with previously @@ -319,9 +329,9 @@ func newRevision(set *apps.StatefulSet, revision int64, collisionCount *int32) ( return cr, nil } -// applyRevision returns a new StatefulSet constructed by restoring the state in revision to set. If the returned error +// ApplyRevision returns a new StatefulSet constructed by restoring the state in revision to set. If the returned error // is nil, the returned StatefulSet is valid. -func applyRevision(set *apps.StatefulSet, revision *apps.ControllerRevision) (*apps.StatefulSet, error) { +func ApplyRevision(set *apps.StatefulSet, revision *apps.ControllerRevision) (*apps.StatefulSet, error) { obj, err := scheme.Scheme.DeepCopy(set) if err != nil { return nil, err diff --git a/pkg/controller/statefulset/stateful_set_utils_test.go b/pkg/controller/statefulset/stateful_set_utils_test.go index 85fb01ae8c6..3b5c10c99fc 100644 --- a/pkg/controller/statefulset/stateful_set_utils_test.go +++ b/pkg/controller/statefulset/stateful_set_utils_test.go @@ -258,7 +258,7 @@ func TestCreateApplyRevision(t *testing.T) { key := "foo" expectedValue := "bar" set.Annotations[key] = expectedValue - restoredSet, err := applyRevision(set, revision) + restoredSet, err := ApplyRevision(set, revision) if err != nil { t.Fatal(err) } diff --git a/pkg/kubectl/BUILD b/pkg/kubectl/BUILD index 35f8eff04c8..eda5d6de078 100644 --- a/pkg/kubectl/BUILD +++ b/pkg/kubectl/BUILD @@ -129,6 +129,7 @@ go_library( "//pkg/client/unversioned:go_default_library", "//pkg/controller/daemon:go_default_library", "//pkg/controller/deployment/util:go_default_library", + "//pkg/controller/statefulset:go_default_library", "//pkg/credentialprovider:go_default_library", "//pkg/kubectl/resource:go_default_library", "//pkg/kubectl/util:go_default_library", diff --git a/pkg/kubectl/cmd/rollout/rollout.go b/pkg/kubectl/cmd/rollout/rollout.go index fb816c643d8..a65ed4d8df5 100644 --- a/pkg/kubectl/cmd/rollout/rollout.go +++ b/pkg/kubectl/cmd/rollout/rollout.go @@ -42,6 +42,7 @@ var ( * deployments * daemonsets + * statefulsets `) ) diff --git a/pkg/kubectl/cmd/rollout/rollout_history.go b/pkg/kubectl/cmd/rollout/rollout_history.go index 986a51c734b..22f31e8e7c2 100644 --- a/pkg/kubectl/cmd/rollout/rollout_history.go +++ b/pkg/kubectl/cmd/rollout/rollout_history.go @@ -44,7 +44,7 @@ var ( func NewCmdRolloutHistory(f cmdutil.Factory, out io.Writer) *cobra.Command { options := &resource.FilenameOptions{} - validArgs := []string{"deployment", "daemonset"} + validArgs := []string{"deployment", "daemonset", "statefulset"} argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ diff --git a/pkg/kubectl/cmd/rollout/rollout_status.go b/pkg/kubectl/cmd/rollout/rollout_status.go index be9c7f06924..e54f22030f8 100644 --- a/pkg/kubectl/cmd/rollout/rollout_status.go +++ b/pkg/kubectl/cmd/rollout/rollout_status.go @@ -50,7 +50,7 @@ var ( func NewCmdRolloutStatus(f cmdutil.Factory, out io.Writer) *cobra.Command { options := &resource.FilenameOptions{} - validArgs := []string{"deployment", "daemonset"} + validArgs := []string{"deployment", "daemonset", "statefulset"} argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ diff --git a/pkg/kubectl/cmd/rollout/rollout_undo.go b/pkg/kubectl/cmd/rollout/rollout_undo.go index 2db00342315..c703924c42f 100644 --- a/pkg/kubectl/cmd/rollout/rollout_undo.go +++ b/pkg/kubectl/cmd/rollout/rollout_undo.go @@ -64,7 +64,7 @@ var ( func NewCmdRolloutUndo(f cmdutil.Factory, out io.Writer) *cobra.Command { options := &UndoOptions{} - validArgs := []string{"deployment", "daemonset"} + validArgs := []string{"deployment", "daemonset", "statefulset"} argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ diff --git a/pkg/kubectl/history.go b/pkg/kubectl/history.go index 5277abfbc89..7bf4eb5b88c 100644 --- a/pkg/kubectl/history.go +++ b/pkg/kubectl/history.go @@ -153,9 +153,9 @@ type DaemonSetHistoryViewer struct { // ViewHistory returns a revision-to-history map as the revision history of a deployment // TODO: this should be a describer func (h *DaemonSetHistoryViewer) ViewHistory(namespace, name string, revision int64) (string, error) { - versionedExtensionsClient := versionedExtensionsClientV1beta1(h.c) versionedAppsClient := versionedAppsClientV1beta1(h.c) - ds, allHistory, err := controlledHistories(versionedExtensionsClient, versionedAppsClient, namespace, name) + versionedExtensionsClient := versionedExtensionsClientV1beta1(h.c) + versionedObj, allHistory, err := controlledHistories(versionedAppsClient, versionedExtensionsClient, namespace, name, "DaemonSet") if err != nil { return "", fmt.Errorf("unable to find history controlled by DaemonSet %s: %v", name, err) } @@ -175,7 +175,13 @@ func (h *DaemonSetHistoryViewer) ViewHistory(namespace, name string, revision in if !ok { return "", fmt.Errorf("unable to find the specified revision") } - dsOfHistory, err := applyHistory(ds, history) + + versionedDS, ok := versionedObj.(*extensionsv1beta1.DaemonSet) + if !ok { + return "", fmt.Errorf("unexpected non-DaemonSet object returned: %v", versionedDS) + } + + dsOfHistory, err := applyHistory(versionedDS, history) if err != nil { return "", fmt.Errorf("unable to parse history %s", history.Name) } @@ -256,29 +262,51 @@ func (h *StatefulSetHistoryViewer) ViewHistory(namespace, name string, revision }) } -// controlledHistories returns all ControllerRevisions controlled by the given DaemonSet -func controlledHistories(extensions clientextensionsv1beta1.ExtensionsV1beta1Interface, apps clientappsv1beta1.AppsV1beta1Interface, namespace, name string) (*extensionsv1beta1.DaemonSet, []*appsv1beta1.ControllerRevision, error) { - ds, err := extensions.DaemonSets(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return nil, nil, fmt.Errorf("failed to retrieve DaemonSet %s: %v", name, err) +// controlledHistories returns all ControllerRevisions controlled by the given API object +func controlledHistories(apps clientappsv1beta1.AppsV1beta1Interface, extensions clientextensionsv1beta1.ExtensionsV1beta1Interface, namespace, name, kind string) (runtime.Object, []*appsv1beta1.ControllerRevision, error) { + var obj runtime.Object + var labelSelector *metav1.LabelSelector + + switch kind { + case "DaemonSet": + ds, err := extensions.DaemonSets(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to retrieve DaemonSet %s: %v", name, err) + } + labelSelector = ds.Spec.Selector + obj = ds + case "StatefulSet": + ss, err := apps.StatefulSets(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to retrieve StatefulSet %s: %v", name, err) + } + labelSelector = ss.Spec.Selector + obj = ss + default: + return nil, nil, fmt.Errorf("unsupported API object kind: %s", kind) } + var result []*appsv1beta1.ControllerRevision - selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector) + selector, err := metav1.LabelSelectorAsSelector(labelSelector) if err != nil { return nil, nil, err } - historyList, err := apps.ControllerRevisions(ds.Namespace).List(metav1.ListOptions{LabelSelector: selector.String()}) + historyList, err := apps.ControllerRevisions(namespace).List(metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { return nil, nil, err } + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, nil, fmt.Errorf("failed to obtain accessor for %s named %s: %v", kind, name, err) + } for i := range historyList.Items { history := historyList.Items[i] - // Only add history that belongs to the DaemonSet - if metav1.IsControlledBy(&history, ds) { + // Only add history that belongs to the API object + if metav1.IsControlledBy(&history, accessor) { result = append(result, &history) } } - return ds, result, nil + return obj, result, nil } // applyHistory returns a specific revision of DaemonSet by applying the given history to a copy of the given DaemonSet diff --git a/pkg/kubectl/rollback.go b/pkg/kubectl/rollback.go index 957cbf111e4..31f74d81c06 100644 --- a/pkg/kubectl/rollback.go +++ b/pkg/kubectl/rollback.go @@ -39,6 +39,7 @@ import ( clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/controller/daemon" deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util" + "k8s.io/kubernetes/pkg/controller/statefulset" sliceutil "k8s.io/kubernetes/pkg/kubectl/util/slice" printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" ) @@ -59,6 +60,8 @@ func RollbackerFor(kind schema.GroupKind, c clientset.Interface) (Rollbacker, er return &DeploymentRollbacker{c}, nil case extensions.Kind("DaemonSet"), apps.Kind("DaemonSet"): return &DaemonSetRollbacker{c}, nil + case apps.Kind("StatefulSet"): + return &StatefulSetRollbacker{c}, nil } return nil, fmt.Errorf("no rollbacker has been implemented for %q", kind) } @@ -221,32 +224,22 @@ func (r *DaemonSetRollbacker) Rollback(obj runtime.Object, updatedAnnotations ma if !ok { return "", fmt.Errorf("passed object is not a DaemonSet: %#v", obj) } - versionedExtensionsClient := versionedExtensionsClientV1beta1(r.c) versionedAppsClient := versionedAppsClientV1beta1(r.c) - versionedDS, allHistory, err := controlledHistories(versionedExtensionsClient, versionedAppsClient, ds.Namespace, ds.Name) + versionedExtensionsClient := versionedExtensionsClientV1beta1(r.c) + versionedObj, allHistory, err := controlledHistories(versionedAppsClient, versionedExtensionsClient, ds.Namespace, ds.Name, "DaemonSet") if err != nil { return "", fmt.Errorf("unable to find history controlled by DaemonSet %s: %v", ds.Name, err) } + versionedDS, ok := versionedObj.(*externalextensions.DaemonSet) + if !ok { + return "", fmt.Errorf("unexpected non-DaemonSet object returned: %v", versionedDS) + } if toRevision == 0 && len(allHistory) <= 1 { return "", fmt.Errorf("no last revision to roll back to") } - // Find the history to rollback to - var toHistory *appsv1beta1.ControllerRevision - if toRevision == 0 { - // If toRevision == 0, find the latest revision (2nd max) - sort.Sort(historiesByRevision(allHistory)) - toHistory = allHistory[len(allHistory)-2] - } else { - for _, h := range allHistory { - if h.Revision == toRevision { - // If toRevision != 0, find the history with matching revision - toHistory = h - break - } - } - } + toHistory := findHistory(toRevision, allHistory) if toHistory == nil { return "", revisionNotFoundErr(toRevision) } @@ -256,14 +249,7 @@ func (r *DaemonSetRollbacker) Rollback(obj runtime.Object, updatedAnnotations ma if err != nil { return "", err } - content := bytes.NewBuffer([]byte{}) - w := printersinternal.NewPrefixWriter(content) - internalTemplate := &api.PodTemplateSpec{} - if err := apiv1.Convert_v1_PodTemplateSpec_To_api_PodTemplateSpec(&appliedDS.Spec.Template, internalTemplate, nil); err != nil { - return "", fmt.Errorf("failed to convert podtemplate while printing: %v", err) - } - printersinternal.DescribePodTemplate(internalTemplate, w) - return fmt.Sprintf("will roll back to %s", content.String()), nil + return printPodTemplate(&appliedDS.Spec.Template) } // Skip if the revision already matches current DaemonSet @@ -283,6 +269,104 @@ func (r *DaemonSetRollbacker) Rollback(obj runtime.Object, updatedAnnotations ma return rollbackSuccess, nil } +type StatefulSetRollbacker struct { + c clientset.Interface +} + +// toRevision is a non-negative integer, with 0 being reserved to indicate rolling back to previous configuration +func (r *StatefulSetRollbacker) Rollback(obj runtime.Object, updatedAnnotations map[string]string, toRevision int64, dryRun bool) (string, error) { + if toRevision < 0 { + return "", revisionNotFoundErr(toRevision) + } + + ss, ok := obj.(*apps.StatefulSet) + if !ok { + return "", fmt.Errorf("passed object is not a StatefulSet: %#v", obj) + } + versionedAppsClient := versionedAppsClientV1beta1(r.c) + versionedExtensionsClient := versionedExtensionsClientV1beta1(r.c) + versionedObj, allHistory, err := controlledHistories(versionedAppsClient, versionedExtensionsClient, ss.Namespace, ss.Name, "StatefulSet") + if err != nil { + return "", fmt.Errorf("unable to find history controlled by StatefulSet %s: %v", ss.Name, err) + } + + versionedSS, ok := versionedObj.(*appsv1beta1.StatefulSet) + if !ok { + return "", fmt.Errorf("unexpected non-StatefulSet object returned: %v", versionedSS) + } + + if toRevision == 0 && len(allHistory) <= 1 { + return "", fmt.Errorf("no last revision to roll back to") + } + + toHistory := findHistory(toRevision, allHistory) + if toHistory == nil { + return "", revisionNotFoundErr(toRevision) + } + + if dryRun { + appliedSS, err := statefulset.ApplyRevision(versionedSS, toHistory) + if err != nil { + return "", err + } + return printPodTemplate(&appliedSS.Spec.Template) + } + + // Skip if the revision already matches current StatefulSet + done, err := statefulset.Match(versionedSS, toHistory) + if err != nil { + return "", err + } + if done { + return fmt.Sprintf("%s (current template already matches revision %d)", rollbackSkipped, toRevision), nil + } + + // Restore revision + if _, err = versionedAppsClient.StatefulSets(ss.Namespace).Patch(ss.Name, types.StrategicMergePatchType, toHistory.Data.Raw); err != nil { + return "", fmt.Errorf("failed restoring revision %d: %v", toRevision, err) + } + + return rollbackSuccess, nil +} + +// findHistory returns a controllerrevision of a specific revision from the given controllerrevisions. +// It returns nil if no such controllerrevision exists. +// If toRevision is 0, the last previously used history is returned. +func findHistory(toRevision int64, allHistory []*appsv1beta1.ControllerRevision) *appsv1beta1.ControllerRevision { + if toRevision == 0 && len(allHistory) <= 1 { + return nil + } + + // Find the history to rollback to + var toHistory *appsv1beta1.ControllerRevision + if toRevision == 0 { + // If toRevision == 0, find the latest revision (2nd max) + sort.Sort(historiesByRevision(allHistory)) + toHistory = allHistory[len(allHistory)-2] + } else { + for _, h := range allHistory { + if h.Revision == toRevision { + // If toRevision != 0, find the history with matching revision + return h + } + } + } + + return toHistory +} + +// printPodTemplate converts a given pod template into a human-readable string. +func printPodTemplate(specTemplate *v1.PodTemplateSpec) (string, error) { + content := bytes.NewBuffer([]byte{}) + w := printersinternal.NewPrefixWriter(content) + internalTemplate := &api.PodTemplateSpec{} + if err := apiv1.Convert_v1_PodTemplateSpec_To_api_PodTemplateSpec(specTemplate, internalTemplate, nil); err != nil { + return "", fmt.Errorf("failed to convert podtemplate while printing: %v", err) + } + printersinternal.DescribePodTemplate(internalTemplate, w) + return fmt.Sprintf("will roll back to %s", content.String()), nil +} + func revisionNotFoundErr(r int64) error { return fmt.Errorf("unable to find specified revision %v in history", r) }