github/client-go
1
0
Fork 0
mirror of https://github.com/kubernetes/client-go.git synced 2025-07-17 00:31:27 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #65594 from liggitt/node-csr-addresses-2

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 65052, 65594). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Derive kubelet serving certificate CSR template from node status addresses

xref https://github.com/kubernetes/features/issues/267
fixes #55633

Builds on https://github.com/kubernetes/kubernetes/pull/65587

* Makes the cloud provider authoritative when recording node status addresses
* Makes the node status addresses authoritative for the kube-apiserver determining how to speak to a kubelet (stops paying attention to the hostname label when determining how to reach a kubelet, which was only done to support kubelets < 1.5)
* Updates kubelet certificate rotation to be driven from node status
  * Avoids needing to compute node addresses a second time, and differently, in order to request serving certificates.
  * Allows the kubelet to react to changes in its status addresses by updating its serving certificate
  * Allows the kubelet to be driven by external cloud providers recording node addresses on the node status

test procedure:
```sh
# setup
export FEATURE_GATES=RotateKubeletServerCertificate=true
export KUBELET_FLAGS="--rotate-server-certificates=true --cloud-provider=external"

# cleanup from previous runs
sudo rm -fr /var/lib/kubelet/pki/

# startup
hack/local-up-cluster.sh

# wait for a node to register, verify it didn't set addresses
kubectl get nodes
kubectl get node/127.0.0.1 -o jsonpath={.status.addresses}

# verify the kubelet server isn't available, and that it didn't populate a serving certificate
curl --cacert _output/certs/server-ca.crt -v https://localhost:10250/pods
ls -la /var/lib/kubelet/pki

# set an address on the node
curl -X PATCH http://localhost:8080/api/v1/nodes/127.0.0.1/status \
  -H "Content-Type: application/merge-patch+json" \
  --data '{"status":{"addresses":[{"type":"Hostname","address":"localhost"}]}}'

# verify a csr was submitted with the right SAN, and approve it
kubectl describe csr
kubectl certificate approve csr-...

# verify the kubelet connection uses a cert that is properly signed and valid for the specified hostname, but NOT the IP
curl --cacert _output/certs/server-ca.crt -v https://localhost:10250/pods
curl --cacert _output/certs/server-ca.crt -v https://127.0.0.1:10250/pods
ls -la /var/lib/kubelet/pki

# set an hostname and IP address on the node
curl -X PATCH http://localhost:8080/api/v1/nodes/127.0.0.1/status \
  -H "Content-Type: application/merge-patch+json" \
  --data '{"status":{"addresses":[{"type":"Hostname","address":"localhost"},{"type":"InternalIP","address":"127.0.0.1"}]}}'

# verify a csr was submitted with the right SAN, and approve it
kubectl describe csr
kubectl certificate approve csr-...

# verify the kubelet connection uses a cert that is properly signed and valid for the specified hostname AND IP
curl --cacert _output/certs/server-ca.crt -v https://localhost:10250/pods
curl --cacert _output/certs/server-ca.crt -v https://127.0.0.1:10250/pods
ls -la /var/lib/kubelet/pki
```

```release-note
* kubelets that specify `--cloud-provider` now only report addresses in Node status as determined by the cloud provider
* kubelet serving certificate rotation now reacts to changes in reported node addresses, and will request certificates for addresses set by an external cloud provider
```

Kubernetes-commit: 337dfe0a9cde3894eb6a26f9184df659d54007c6
This commit is contained in:
Kubernetes Publisher 2018-07-11 22:25:07 -07:00
commit f1995823ac
3 changed files with 197 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
Godeps/Godeps.json generated
View File

@ -268,331 +268,331 @@
},
{
"ImportPath": "k8s.io/api/admissionregistration/v1alpha1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/admissionregistration/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/apps/v1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/apps/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/apps/v1beta2",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/authentication/v1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/authentication/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/authorization/v1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/authorization/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/autoscaling/v1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/autoscaling/v2beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/batch/v1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/batch/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/batch/v2alpha1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/certificates/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/coordination/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/core/v1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/events/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/extensions/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/imagepolicy/v1alpha1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/networking/v1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/policy/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/rbac/v1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/rbac/v1alpha1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/rbac/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/scheduling/v1alpha1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/scheduling/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/settings/v1alpha1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/storage/v1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/storage/v1alpha1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/api/storage/v1beta1",
"Rev": "8be2a0b24ed0dac9cfc1ac2d987ea16cfcdbecb6"
"Rev": "183f3326a9353bd6d41430fc80f96259331d029c"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/equality",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/errors",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/meta",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/resource",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/testing",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/testing/fuzzer",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/testing/roundtrip",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/fuzzer",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/internalversion",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1beta1",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/conversion",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/conversion/queryparams",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/fields",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/labels",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/schema",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/json",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/protobuf",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/recognizer",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/streaming",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/versioning",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/selection",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/types",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/cache",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/clock",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/diff",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/errors",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/framer",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/httpstream",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/httpstream/spdy",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/intstr",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/json",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/mergepatch",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/naming",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/net",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/remotecommand",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/runtime",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/sets",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/strategicpatch",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/validation",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/validation/field",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/wait",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/yaml",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/version",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/watch",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/json",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/netutil",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect",
"Rev": "bce280dade67588308614434551c0b865433e44d"
"Rev": "8dd74a0baf0dd7bdb3726e3fc9db6c59e03b0c45"
},
{
"ImportPath": "k8s.io/kube-openapi/pkg/util/proto",

122
util/certificate/certificate_manager.go
View File

@ -24,6 +24,7 @@ import (
"crypto/x509"
"encoding/pem"
"fmt"
"reflect"
"sync"
"time"
@ -32,6 +33,7 @@ import (
certificates "k8s.io/api/certificates/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
"k8s.io/client-go/util/cert"
@ -75,6 +77,13 @@ type Config struct {
// part of rotation. It follows the same rules as the template parameter of
// crypto.x509.CreateCertificateRequest in the Go standard libraries.
Template *x509.CertificateRequest
// GetTemplate returns the CertificateRequest that will be used as a template for
// generating certificate signing requests for all new keys generated as
// part of rotation. It follows the same rules as the template parameter of
// crypto.x509.CreateCertificateRequest in the Go standard libraries.
// If no template is available, nil may be returned, and no certificate will be requested.
// If specified, takes precedence over Template.
GetTemplate func() *x509.CertificateRequest
// Usages is the types of usages that certificates generated by the manager
// can be used for.
Usages []certificates.KeyUsage
@ -136,7 +145,10 @@ func (e *NoCertKeyError) Error() string { return string(*e) }
type manager struct {
certSigningRequestClient certificatesclient.CertificateSigningRequestInterface
template *x509.CertificateRequest
getTemplate func() *x509.CertificateRequest
lastRequestLock sync.Mutex
lastRequest *x509.CertificateRequest
dynamicTemplate bool
usages []certificates.KeyUsage
certStore Store
certAccessLock sync.RWMutex
@ -158,9 +170,15 @@ func NewManager(config *Config) (Manager, error) {
return nil, err
}
getTemplate := config.GetTemplate
if getTemplate == nil {
getTemplate = func() *x509.CertificateRequest { return config.Template }
}
m := manager{
certSigningRequestClient: config.CertificateSigningRequestClient,
template: config.Template,
getTemplate: getTemplate,
dynamicTemplate: config.GetTemplate != nil,
usages: config.Usages,
certStore: config.CertificateStore,
cert: cert,
@ -215,12 +233,32 @@ func (m *manager) Start() {
glog.V(2).Infof("Certificate rotation is enabled.")
templateChanged := make(chan struct{})
go wait.Forever(func() {
deadline := m.nextRotationDeadline()
if sleepInterval := deadline.Sub(time.Now()); sleepInterval > 0 {
glog.V(2).Infof("Waiting %v for next certificate rotation", sleepInterval)
time.Sleep(sleepInterval)
timer := time.NewTimer(sleepInterval)
defer timer.Stop()
select {
case <-timer.C:
// unblock when deadline expires
case <-templateChanged:
if reflect.DeepEqual(m.getLastRequest(), m.getTemplate()) {
// if the template now matches what we last requested, restart the rotation deadline loop
return
}
glog.V(2).Infof("Certificate template changed, rotating")
}
}
// Don't enter rotateCerts and trigger backoff if we don't even have a template to request yet
if m.getTemplate() == nil {
return
}
backoff := wait.Backoff{
Duration: 2 * time.Second,
Factor: 2,
@ -231,7 +269,18 @@ func (m *manager) Start() {
utilruntime.HandleError(fmt.Errorf("Reached backoff limit, still unable to rotate certs: %v", err))
wait.PollInfinite(32*time.Second, m.rotateCerts)
}
}, 0)
}, time.Second)
if m.dynamicTemplate {
go wait.Forever(func() {
// check if the current template matches what we last requested
if !reflect.DeepEqual(m.getLastRequest(), m.getTemplate()) {
// if the template is different, queue up an interrupt of the rotation deadline loop.
// if we've requested a CSR that matches the new template by the time the interrupt is handled, the interrupt is disregarded.
templateChanged <- struct{}{}
}
}, time.Second)
}
}
func getCurrentCertificateOrBootstrap(
@ -286,7 +335,7 @@ func getCurrentCertificateOrBootstrap(
func (m *manager) rotateCerts() (bool, error) {
glog.V(2).Infof("Rotating certificates")
csrPEM, keyPEM, privateKey, err := m.generateCSR()
template, csrPEM, keyPEM, privateKey, err := m.generateCSR()
if err != nil {
utilruntime.HandleError(fmt.Errorf("Unable to generate a certificate signing request: %v", err))
return false, nil
@ -300,6 +349,9 @@ func (m *manager) rotateCerts() (bool, error) {
return false, m.updateServerError(err)
}
// Once we've successfully submitted a CSR for this template, record that we did so
m.setLastRequest(template)
// Wait for the certificate to be signed. Instead of one long watch, we retry with slightly longer
// intervals each time in order to tolerate failures from the server AND to preserve the liveliness
// of the cert manager loop. This creates slightly more traffic against the API server in return
@ -353,6 +405,36 @@ func (m *manager) nextRotationDeadline() time.Time {
return time.Now()
}
// Ensure the currently held certificate satisfies the requested subject CN and SANs
if template := m.getTemplate(); template != nil {
if template.Subject.CommonName != m.cert.Leaf.Subject.CommonName {
glog.V(2).Infof("Current certificate CN (%s) does not match requested CN (%s), rotating now", m.cert.Leaf.Subject.CommonName, template.Subject.CommonName)
return time.Now()
}
currentDNSNames := sets.NewString(m.cert.Leaf.DNSNames...)
desiredDNSNames := sets.NewString(template.DNSNames...)
missingDNSNames := desiredDNSNames.Difference(currentDNSNames)
if len(missingDNSNames) > 0 {
glog.V(2).Infof("Current certificate is missing requested DNS names %v, rotating now", missingDNSNames.List())
return time.Now()
}
currentIPs := sets.NewString()
for _, ip := range m.cert.Leaf.IPAddresses {
currentIPs.Insert(ip.String())
}
desiredIPs := sets.NewString()
for _, ip := range template.IPAddresses {
desiredIPs.Insert(ip.String())
}
missingIPs := desiredIPs.Difference(currentIPs)
if len(missingIPs) > 0 {
glog.V(2).Infof("Current certificate is missing requested IP addresses %v, rotating now", missingIPs.List())
return time.Now()
}
}
notAfter := m.cert.Leaf.NotAfter
totalDuration := float64(notAfter.Sub(m.cert.Leaf.NotBefore))
deadline := m.cert.Leaf.NotBefore.Add(jitteryDuration(totalDuration))
@ -408,22 +490,38 @@ func (m *manager) updateServerError(err error) error {
return nil
}
func (m *manager) generateCSR() (csrPEM []byte, keyPEM []byte, key interface{}, err error) {
func (m *manager) generateCSR() (template *x509.CertificateRequest, csrPEM []byte, keyPEM []byte, key interface{}, err error) {
// Generate a new private key.
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
if err != nil {
return nil, nil, nil, fmt.Errorf("unable to generate a new private key: %v", err)
return nil, nil, nil, nil, fmt.Errorf("unable to generate a new private key: %v", err)
}
der, err := x509.MarshalECPrivateKey(privateKey)
if err != nil {
return nil, nil, nil, fmt.Errorf("unable to marshal the new key to DER: %v", err)
return nil, nil, nil, nil, fmt.Errorf("unable to marshal the new key to DER: %v", err)
}
keyPEM = pem.EncodeToMemory(&pem.Block{Type: cert.ECPrivateKeyBlockType, Bytes: der})
csrPEM, err = cert.MakeCSRFromTemplate(privateKey, m.template)
if err != nil {
return nil, nil, nil, fmt.Errorf("unable to create a csr from the private key: %v", err)
template = m.getTemplate()
if template == nil {
return nil, nil, nil, nil, fmt.Errorf("unable to create a csr, no template available")
}
return csrPEM, keyPEM, privateKey, nil
csrPEM, err = cert.MakeCSRFromTemplate(privateKey, template)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("unable to create a csr from the private key: %v", err)
}
return template, csrPEM, keyPEM, privateKey, nil
}
func (m *manager) getLastRequest() *x509.CertificateRequest {
m.lastRequestLock.Lock()
defer m.lastRequestLock.Unlock()
return m.lastRequest
}
func (m *manager) setLastRequest(r *x509.CertificateRequest) {
m.lastRequestLock.Lock()
defer m.lastRequestLock.Unlock()
m.lastRequest = r
}

10
util/certificate/certificate_manager_test.go
View File

@ -186,7 +186,7 @@ func TestSetRotationDeadline(t *testing.T) {
NotAfter: tc.notAfter,
},
},
template: &x509.CertificateRequest{},
getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
usages: []certificates.KeyUsage{},
certificateExpiration: &g,
}
@ -221,8 +221,8 @@ func TestRotateCertCreateCSRError(t *testing.T) {
NotAfter: now.Add(-1 * time.Hour),
},
},
template: &x509.CertificateRequest{},
usages: []certificates.KeyUsage{},
getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
usages: []certificates.KeyUsage{},
certSigningRequestClient: fakeClient{
failureType: createError,
},
@ -244,8 +244,8 @@ func TestRotateCertWaitingForResultError(t *testing.T) {
NotAfter: now.Add(-1 * time.Hour),
},
},
template: &x509.CertificateRequest{},
usages: []certificates.KeyUsage{},
getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
usages: []certificates.KeyUsage{},
certSigningRequestClient: fakeClient{
failureType: watchError,
},