From 9d6a19efff05a40081337a2fcb3dce6331b04022 Mon Sep 17 00:00:00 2001 From: Stanislav Laznicka Date: Tue, 6 Oct 2020 13:02:52 +0200 Subject: [PATCH 1/2] webhooks,aggregation: add metrics to count certs missing SAN Adds counters to generic webhook code and to the kube aggregator so that it is possible to effectively measure the impact of Golang 1.15's deprecation of x509 cert CN hostname verification. --- .../apiserver/pkg/util/webhook/certs_test.go | 357 +++++++++--------- .../apiserver/pkg/util/webhook/client.go | 6 + .../apiserver/pkg/util/webhook/gencerts.sh | 18 +- .../apiserver/pkg/util/webhook/metrics.go | 39 ++ .../apiserver/pkg/util/webhook/webhook.go | 2 + .../pkg/util/webhook/webhook_test.go | 59 ++- .../pkg/util/x509metrics/missing_san.go | 92 +++++ .../pkg/util/x509metrics/missing_san_test.go | 301 +++++++++++++++ .../pkg/apiserver/handler_proxy.go | 24 +- .../pkg/apiserver/handler_proxy_test.go | 255 +++++++++---- .../kube-aggregator/pkg/apiserver/metrics.go | 39 ++ 11 files changed, 939 insertions(+), 253 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/util/webhook/metrics.go create mode 100644 staging/src/k8s.io/apiserver/pkg/util/x509metrics/missing_san.go create mode 100644 staging/src/k8s.io/apiserver/pkg/util/x509metrics/missing_san_test.go create mode 100644 staging/src/k8s.io/kube-aggregator/pkg/apiserver/metrics.go diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/certs_test.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/certs_test.go index e6b63e2b253..989c43bba60 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/certs_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/certs_test.go @@ -20,195 +20,212 @@ limitations under the License. package webhook var caKey = []byte(`-----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAwmdGuDsPPyirvNqlqoDmwf/bmF3zTJBFYQRsJK0vKBAdrfMY -MVwoPEpM5ZZ+VCHEB6vSuGYbC0PyO97H/kIV22FsMwN7zifhJP2T2Hb+B7Nc6s8W -tAQn1J4xUY31xOXOEcXe2nuGezrlKfX3DpA1FVXp6/Cf8vhyUQ0fJXwuZE/Pbhvp -bBUciQLqfPSH6EnShZvzjJBP5Bs13UqzaRobhUf9A3pyk2Mb3PXkTetET7hLc4J2 -uIp5BxOoNZSvgQCvCjRyV5s1his7BNRALKG3qz/48i6JRyO7FvNoxDkWC09zIqD8 -1bw9I1d/+EwWdqAPZLa8iLpSsd0gD03gDSFbOwIDAQABAoIBAQC0JPO5oLDePBf4 -pzxBJbWwLCIXrWfZmQ9RecGksv8xxs1Z9hyDEP0P8WIUlkJ2P9vhp+1ahvOkmtAL -fsQg7qhGZJ7ZHu9I+Fd/6aNpQcrg4+rEhCZrpjYqpnTZOA146eLtQUjjePgDlW3q -Vk0cJ7GpFbXwt0fg5S05wkkMeWib9mKvme3vooNbog52164U1wo/p4WBTpbAMoYA -XlJSqXeoxBpsLWBZHlRG+AYfYpk7BXk8MkIslcKh97RmLsZt52Fh3SbsFJeIEmD5 -2hQDvn/PJojAnM6SMkUqfvv87SdkryvqQYJ80b2D6qd+y8o7gUFr8WkEqVRCqVLh -GaD2C06hAoGBAO9JOe+typoQUPi24aj5BoqWcpyrHQkCdjLlxS805oLsPfmb+EqF -1HwnA8UHNMlrdiczJ8f2M7Y4cSUIEXv6LSE5r4teSiYWASidDLREi0q8scw21CGH -BnCc7PUhUnBngXJ3B1MtCj+r3TFfpOEEi1J1HtMK1AxAaq7zEFzdOrtjAoGBAM/7 -fC89Awvd7yJsgTVumKVx/bA+Q54YJOFMkdba3JbcLsQyn4TBaFv0pwqqXqmkTLZz -WHjkNscomRf9VY34D4q07nO4YGXCBNqm3MaV3mE0xhIyBsATRZnf03O2a/pnRPu/ -yTE1EyuIqK/l4+5iv2O5mWzxorC4qdV34Wf5WCRJAoGBANfmfjvf1zoDFswSVrGb -X2eUL31kdyI18mgiITRiysm+VnztWa4D6qDKowAXbG2AZG8iHPazEh2L96quCPiP -1kBwSA+717Ndj1YRvfC5F+UrNFFJ90T5C7p4HOVgV33MJmQdOaK2tNSWQVHXNnFB -JGQWAOXykzkqthd8gHsJsYB5AoGAd7BfKAQxg5vFqYbN2MT7vYJbHxjF6u40Ex/w -cbfj6EFv/GKxoEF5YCnsE1w2O+QcbYb1rCSRTY2UhNS6bogJ0aYL77Z0azr7diU+ -ul226zPmpMP7VIACtumzE00w2JqjfUlCbDoB/TSY9xkSUbasM6S0oZhxKsgqnHlv -01kQG1kCgYBPZfZiqKwnnsOSRy8UBN4Oo5zg9QrbMki472/s4FhHaunXF0pFmIUG -QU/9kYteJ8DlCppvvtU5C3qmEkW6c2u8KAfJXRmA51uS6v36kEx/8313ZJ5afwLU -i2ZMmS8OabHjIhdnCSA2N7geqaAZa7BCLqt8735Doys1p0KB0y+ZNw== +MIIEogIBAAKCAQEAzLyjWxFJ7y+K5LGHxrs67ye/IMnF4kD2yif5yVl1LBrViluF +Kxg+vay9MYtPhnFfcmS8dkYNoH4bpCtk0hY6ywnj5+O8SxVKeT2rtivzBfxB1KGS +ydI/Sp0HXEUqWrBMVxnJAeqOXJy1X3f8sSiwz0NFnOA+lcm4LrAPup0tFI/FDC56 +BlhwwZPZjj/wJ8GCIRSRk2sZgmIMWtuUjQ32NfaNR4/+DzFYkgotMF51VsDOPTZj +rpZ015xhzJMxE51yU8HUhsXgAFsIbKec/R9uJp/T7i/Skt52rAJhJYP+6NqPIO1i +6EjxT46QQ3CRn+HmPD/ppart6wxkNsQBPpQ+AQIDAQABAoIBACNVEqPuseGzRLb+ +5D2pphwLsfJIn/vR1kVA7NyQsaXw45VDAhJT1tAI9YWNUyYSfrRRKi0HHebqnH6S +do3DBFZx2ID2TlJu1DFkMTP5FbGcvQei8qAdxopr2w9TaGHKNwJ5ErVf8z2MVcip +/lPcf9bk9yiBkotrCnwUKkK8SUFY11WdXfVovzKIoTsceu0LbN87kyaXrsFNOsj5 +BW8AAo9boAYWTpKZ91OkS5yJaxZYd1o9MDbR3hhpXR08Wk5DV2ko2lQQg4eC+QAd +9enndtDvrmjGfJzbbN5wb8i9raW020//29YOBwUbr9RPfdS/1TqgVvZoJLz1YUKO +/rOLFxkCgYEA5gJdgWDBzclL8cJLScLv4EtLmJCbL/Bmc6pLVtK4bHZiYfEL8jf/ +/g14srtttqrltyhXlxnFnyJiZ3gW3FHsEQ97poOn61cKjfk0XbZRgr38VqFyhyEW +BDU6T4yAJf6MGv0sOgKX/EXtxH0lUgusvqjmM3cinHmGdNAiuzDceiMCgYEA498z +n6ELpR0iYdl/cinTFSrWLsmeIWkmX9OPsE4PyZfIVX2Td+D1dqOhvz90IDuY3hIv +6YIpra7mpamDoT+YiMaZfM5QQm8hqGuYxmjceueQQ8BtreAmTB+nMqnG59KqDnHR +deckmMwuYNkBdKEpkOhga73UzEfIKOUP+Ykgr4sCgYAZZFt0Q/1BPZ6/ssZi/z1C +Eq3mytgHA9AbpsoobJzlhHA6BcVe2SXOoygW+zASgW1YugBpcvPpLH+WkqBbdEPO +YloxOMmZ5oMIx2Erk0wOVfD8k1g0aMeocUPdslIXX7GR7S8rvEuRExs7nOE43IcW +iUjYl6dfhN7+GFgtOCiu5wKBgG3qMACDshZG41rHis4KlqLTMtUGs5vyGGyIo/qq +7LqU2DFEjWl5vW0oqNCTTvRtSuNzamD8RBZfEyo9hhy1jGINSeQFsHkbGeUfYWXz +FsbCA7hqjX0dbduyOPgbJLKVzymAhUSV5fU+J/DXO/iB9IbNUBUoUAjpZwcy1m3L +U1TZAoGAPGYKIawt7KN4n6AB5zjWbUZnqUjotcaJNRHRqfiRGatHJh4ZayIy2iVy +g6xVUnLrCmgKsVFiTY+3QhN+p6qhntbkQ+nxKwAniBnKHyK8e+HTxChYRZQDzlFj +dp+L9OwuSvcReU4j0c9QH4wJGGgobOsJ5I0mfGgNN2+IW8GqOFw= -----END RSA PRIVATE KEY-----`) var caCert = []byte(`-----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJAM1Z9H27fWCyMA0GCSqGSIb3DQEBBQUAMBsxGTAXBgNV -BAMUEHdlYmhvb2tfdGVzdHNfY2EwIBcNMTcwMjExMDAyMjUzWhgPMjI5MDExMjcw -MDIyNTNaMBsxGTAXBgNVBAMUEHdlYmhvb2tfdGVzdHNfY2EwggEiMA0GCSqGSIb3 -DQEBAQUAA4IBDwAwggEKAoIBAQDCZ0a4Ow8/KKu82qWqgObB/9uYXfNMkEVhBGwk -rS8oEB2t8xgxXCg8Skzlln5UIcQHq9K4ZhsLQ/I73sf+QhXbYWwzA3vOJ+Ek/ZPY -dv4Hs1zqzxa0BCfUnjFRjfXE5c4Rxd7ae4Z7OuUp9fcOkDUVVenr8J/y+HJRDR8l -fC5kT89uG+lsFRyJAup89IfoSdKFm/OMkE/kGzXdSrNpGhuFR/0DenKTYxvc9eRN -60RPuEtzgna4inkHE6g1lK+BAK8KNHJXmzWGKzsE1EAsoberP/jyLolHI7sW82jE -ORYLT3MioPzVvD0jV3/4TBZ2oA9ktryIulKx3SAPTeANIVs7AgMBAAGjfDB6MB0G -A1UdDgQWBBS0/gwwXmdxIe8o+a7WKbdiTYPy+TBLBgNVHSMERDBCgBS0/gwwXmdx -Ie8o+a7WKbdiTYPy+aEfpB0wGzEZMBcGA1UEAxQQd2ViaG9va190ZXN0c19jYYIJ -AM1Z9H27fWCyMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAHGUdTCN -rm0Mx6V5SmSIGbOB/+3QMqE1ZBIWwPsVFKHhROLaouELZFO+QysfLufB8SS54aM6 -ewufjfSz4KL26DnoSwOFirPxpG0+Sdry55lCjmZ50KtENZDu6g288Qx9GBzqgVHz -kGi/eciV4fZ4HYIhZY+oR29n3YYQOID4UqbQ86lSoN781dmsEQLL+TEK4mJJFcNg -SKHM526WdwJ15zqpKNlcqXtTyx3UfBFlNwvrxHNFbth1vOfdTW8zAs9Xzcr5vSm2 -G8nJ3FF/UF4dYpjDzggO3ALZZqUJHnl/XusETo5kYY3Ozp0xQYg2beR8irElqP8f -oNcE4Ycfe10Hmec= +MIIDGTCCAgGgAwIBAgIUOS2MkobR2t4rguefcC78gLuXkc0wDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxMjMxNDFa +GA8yMjk0MDcyMzEyMzE0MVowGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMy8o1sRSe8viuSxh8a7Ou8n +vyDJxeJA9son+clZdSwa1YpbhSsYPr2svTGLT4ZxX3JkvHZGDaB+G6QrZNIWOssJ +4+fjvEsVSnk9q7Yr8wX8QdShksnSP0qdB1xFKlqwTFcZyQHqjlyctV93/LEosM9D +RZzgPpXJuC6wD7qdLRSPxQwuegZYcMGT2Y4/8CfBgiEUkZNrGYJiDFrblI0N9jX2 +jUeP/g8xWJIKLTBedVbAzj02Y66WdNecYcyTMROdclPB1IbF4ABbCGynnP0fbiaf +0+4v0pLedqwCYSWD/ujajyDtYuhI8U+OkENwkZ/h5jw/6aWq7esMZDbEAT6UPgEC +AwEAAaNTMFEwHQYDVR0OBBYEFBgvyZWkRJCRjxclYA0mMlnzc/GmMB8GA1UdIwQY +MBaAFBgvyZWkRJCRjxclYA0mMlnzc/GmMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAHigaFJ8DqYFMGXNUT5lYM8oLLWYcbivGxR7ofDm8rXWGLdQ +tKsWeusSlgpzeO1PKCzwSrYQhlFIZI6AH+ch7EAWt84MfAaMz/1DiXF6Q8fAcR0h +QoyFIAKTiVkcgOQjSQOIM2SS5cyGeDRaHGVWfaJOwdIYo6ctFzI2asPJ4yU0QsA7 +0WTD2+sBG6AXGhfafGUHEmou8sGQ+QT8rgi4hs1bfyHuT5dPgC4TbwknD1G8pMqm +ID3CIiCF8hhF5C2hjrW0LTJ6zTlg1c1K0NmmUL1ucsfzEk//c7GsU8dk+FYmtW9A +VzryJj1NmnSqA3zd3jBMuK37Ei3pRvVbO7Uxf14= -----END CERTIFICATE-----`) var badCAKey = []byte(`-----BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEAxrCZsZHIeKuYSUYw4qKB8P+HeU81VP7F0tVbLOffj587sfDu -N2QsOsnmiWKS9HiNVUs5ap/Zg3rekbnZV5eBeS5qOPC95c8Oyac7A2HNYSRp08kQ -dUUo3ueDqwmZ5dNiDp0SriQCWfPYxiWHdWSacqDxxzXPFWCs6z3vwFYV35VxDFhr -M8gPWgSqjWdqj+p5cHrL88me7vWWsj6ceL/i2KWo8lmuHuhUzggn0ekU/aKsavHi -X/MNVd5tPzGIR+B49aywQZ9KyrJ7V8SdFqKYEGfH6RaiDhaNNUOv9E9668PdupnG -qei0260zB8SDY4j6VKPGOS90YBA66qOP6J11rwIDAQABAoIBAQCsgs0PNep/i01v -4Xe0bzCvVM4Fb9Z4c7TDN+gv9ytOggzMlMngYiNc78wwYNwDU2AzPFsfzqaG1/nD -QUAKI0uRMdGcmrnmfH70azR73UD7JSiVb6/QgjnYP988c9uhhoVO9uYvOKip/WSr -tg4EyVKoUEFcm8WvY/7/SQmPT68yLf5VpbtuCysAkSLPUjcBer4A6eWwlZ9PWrtj -rLjUCGXXDKRgMmQRAwL+tpBgMwb1+euriv4+M6ddZCkcyaohW076qA3aaq3+XtAB -RTGQubWshuri3N1WRQcn1ZvGURLCAhI8q+9i/wXADKrAlL6imDuYzTW+LMQdZLuH -bwHvq/yRAoGBAP3beuc2R/jjkDttFsSce2dMx6F8AKPpmdbzVw7/DhflzNRDM/Yo -dfVOabRLqcAyfhNm2L6CdUaIJuHRKyRJT3X5wgxUapAjXFUE0kH+qnaq3BxZCCjU -fwDUZ4SUVDAuyaMo5OfVbqkI/L3rvSSgklNOnSkXMPtftDkz8pVljLo9AoGBAMhd -6uiddCt3Dpt75C1BDRX0xGKc4KwtPK0CnQeQmQNXUx192m6IhfPW7YUoKvIZibWB -f9NNJ/KCxDGG+QP7X+0sWQZMfdp5f1l6EsM6HFPLAOgjQ4PyBVWqxknJyxy6GCnt -vI3s6cwMxN7B7QJ/87ffO23elEu7bCdg0lrOAmpbAoGBALN6fI+B6irGwU+ylflV -5U2olC/Q2ycIXtMBYpjgrRcqSsH77X3pJ1TTJprpL9AKIucWvMEcvUursUnQt97E -0iBH//D1sg3MYlhdu0Ybhmu16z9Dlyg+7LgqdDHhKRCT082+ePCMDtwF1aN1S1nd -CPdLSoQluGTRSjtzRdxoWrHFAoGAJqNlz2G9qzwURwuHHuryeQ9wZ4vVD57RmpNs -cK8Dss8+KevBGZueKT2DJDBwx6sBEU1dtwOj9nIdH2fl0UzCXNw2dq59fon7cufF -gnxMRiRZkmpqdKFRQgninwwY7Ps9+afsunm7RCwaMtK2v8qo1wZnUXKgqlIEMzvK -lNQxRw0CgYEA0uT5TkrZ3S+GAbyQtwb6FtXiLoocL+8QyBw6d1+5Fy7D0pYYQDLw -TMeR2NOEqQGLgsqCnbuKzKBIuIY8wm0VIzOqRGmk4fWiOGxrtEBrfPl+A21bexyC -qv5UBMLcEinZEM3x1e/rloDwKi0IGfyKiRfVpxdVKebs7dJfFYkhmw0= +MIIEowIBAAKCAQEAsmGBQnmBK88DdHezRXed0pX0vtbxMP6WEwaxW1d6x7jOafn2 +HbfxuxC/B1dpewpKswBlQTi4/0LqjUi2/YJIyVJZXGdFRaR5zqYVK4LYNHSBvY8y +9wyupUOKGiP6fx9bMAXOzafdK0LR7C8P5q9+zxRcjU8G5sK9mnd+oNLKMZJ1gaMU +3KMYi3gnxi5ebNxxP0LAaPqY+yfpjSm2OZkS/rcMWMXsq42ERPvbclf0rY5o+uau +XXaNilkjJhUUoEk0bOlqflsh2ol+WVwhS3XIO9MORwspP58drn574JcwhMFlM7aU +kT4/hVoqnSaOmSdLR3Op3OmQSq+n+J9cbJiVfQIDAQABAoIBAQCZd8n9pwu65R/T +1CgoXAEsbFdk2QgpXt8+/0MXkuvPaPAtvSBB8T/H8WBosIvPj8s0teJneqWu96NU +ansFIFH+4xp+pVqz0A37/Ge6R5g7iQEWVV1Dr2WSSclHNC0PsaqCZnzF8uYVkieJ +S/QiRFqVTq9R4+vMHT+C5cvMEY5jlm5LA35UKOZ0SQRHJ717WJZK+p+XszljaI9g +9sCZX0uR25nECalhwZVbaK6IB4u1nBM6+67j2sOUV/7udyPbDlAfRE15dQxBR+rW +Uu30jQz6+GMsPh1bHlfDHy4L/FFPCff/ZI3fbbfuvfUMJHH3G/eJvi+QPtlQN6Xp +H9xl/NgBAoGBANdU96dAl4N0LlkEagzGXXSQMgxDbuMkOHlDGH1Vm0P+Nq7tZKwF +EQKvVa6QPaSiAzgeuUwPkiVlIJZ6lMzzutRMKu3fSYnM57X45dnKWfs9AsskDSMp +Pj2LQ1RBo8Pgd8H/aFDVmCjBdn6w/6xrAPn4TvpYM6mJyQ+KbcMINrXZAoGBANQS +ADwajz00IXNxE6A86mvQZ68fphNNoMByI6q5bA+T8AorlQ9YaofYfb1ezmHIh6nt +QPQIHATYF6BmY23ypgHF/aE+lcgK7Nej4u+3Lc9ImJ/0o5wzvxUpX9XFsjOjAGEz +TentETTbLnp6a3f+bnXAOaM/ofQNPvnqiQnR0OJFAoGAC7Jp4YP4twNQoTVELX15 +BiPvFAt1spD9IFkss2I7FO5yOf5bQZzk16h+lwTu1EqYsiu5FRCjd7SOmJ4AB0IW +HAInMtS2Qe4HiDMFCVecm7EsvawvqoFLCDzQY3tNUg6XcspU+E8h/NTFgwxKVytY +2jtKzv6Lj+IUMevrGnUPw8ECgYAGnuE+/x1FreD1d6xDLmOrJgB2qShIJf5Ew8t1 +QwCqo9W0m5O1vO7mes3CIbmTt+z0UyHZ/H7Tb+Oc8FVeU1r3ZzT52bhXXG/0c3tc +PH3DoOKS69JHyB3JDVeeluNvVUFnx3BBQ1NsMQOMc1Hzlw/fwTaLcCsgMWGr77SD +h/dbeQKBgD9wanYnE7uQAP9N49FDtCiiMwFyYc8QwULU2jD6umfByjmvo4ocqDE/ +sMwQmu62qrQlhHPMXh28zsobyC3DpkgutlTB9U1xyMoElFGeEBbpd3JhTZFKs7Hd +RggKDB1828evoGHyfA+Tu1+uh0viEuP0yfIMjRABbJaLjTYAx1YG -----END RSA PRIVATE KEY-----`) var badCACert = []byte(`-----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJAPbb5w6p8Cw8MA0GCSqGSIb3DQEBBQUAMBsxGTAXBgNV -BAMUEHdlYmhvb2tfdGVzdHNfY2EwIBcNMTcwMjExMDAyMjUzWhgPMjI5MDExMjcw -MDIyNTNaMBsxGTAXBgNVBAMUEHdlYmhvb2tfdGVzdHNfY2EwggEiMA0GCSqGSIb3 -DQEBAQUAA4IBDwAwggEKAoIBAQDGsJmxkch4q5hJRjDiooHw/4d5TzVU/sXS1Vss -59+Pnzux8O43ZCw6yeaJYpL0eI1VSzlqn9mDet6RudlXl4F5Lmo48L3lzw7JpzsD -Yc1hJGnTyRB1RSje54OrCZnl02IOnRKuJAJZ89jGJYd1ZJpyoPHHNc8VYKzrPe/A -VhXflXEMWGszyA9aBKqNZ2qP6nlwesvzyZ7u9ZayPpx4v+LYpajyWa4e6FTOCCfR -6RT9oqxq8eJf8w1V3m0/MYhH4Hj1rLBBn0rKsntXxJ0WopgQZ8fpFqIOFo01Q6/0 -T3rrw926mcap6LTbrTMHxINjiPpUo8Y5L3RgEDrqo4/onXWvAgMBAAGjfDB6MB0G -A1UdDgQWBBTTHlbuK0loVSNNa+TCM0Bt7dLEcTBLBgNVHSMERDBCgBTTHlbuK0lo -VSNNa+TCM0Bt7dLEcaEfpB0wGzEZMBcGA1UEAxQQd2ViaG9va190ZXN0c19jYYIJ -APbb5w6p8Cw8MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBACrsdqfQ -7be0HI4+/cz/VwzvbQwjr9O6Ybxqs/eKOprh8RWRtjYeuagvTc6f5jH39W9kZdjs -E3ktCG3RJJ/SyooeJlzNhgaAaATnMqEf7GyiQv3ch0B/Mc4TOPJeQ2E/pzFj3snq -Edm8Xu9+WLwTTF4j/WlSY1sgVSnFk3Yzl5cn0ip00DVCOsL2sP3JlX9HRG4IrdiX -jYXb+nGUPYFultSGSUw+5SiL2yM1ZyHfOBaO1RH8QIRA1/aFTfE+1QRuja5YtCwl -ahpWVRhii7GVR3zKEgKFTxjELHm8x3vBC/HAhj5J3433nlRrgvwZXsZYplqp8422 -IpexMtsutA+y9aE= +MIIDGTCCAgGgAwIBAgIUINBaI0NGgSo4UqKHuPd/HSBO38EwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxMjMxNDFa +GA8yMjk0MDcyMzEyMzE0MVowGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJhgUJ5gSvPA3R3s0V3ndKV +9L7W8TD+lhMGsVtXese4zmn59h238bsQvwdXaXsKSrMAZUE4uP9C6o1Itv2CSMlS +WVxnRUWkec6mFSuC2DR0gb2PMvcMrqVDihoj+n8fWzAFzs2n3StC0ewvD+avfs8U +XI1PBubCvZp3fqDSyjGSdYGjFNyjGIt4J8YuXmzccT9CwGj6mPsn6Y0ptjmZEv63 +DFjF7KuNhET723JX9K2OaPrmrl12jYpZIyYVFKBJNGzpan5bIdqJfllcIUt1yDvT +DkcLKT+fHa5+e+CXMITBZTO2lJE+P4VaKp0mjpknS0dzqdzpkEqvp/ifXGyYlX0C +AwEAAaNTMFEwHQYDVR0OBBYEFLD1yOtonJPAufGLqZ3QSRZDAeoiMB8GA1UdIwQY +MBaAFLD1yOtonJPAufGLqZ3QSRZDAeoiMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBACpuDaS4SPpiA3cU6/tadt8F1vqDs0T4sHbPfiCLdb4P+3N/ +2R8q3SvGzpxJzPflkR8QFwjK0fQuIb0FLyXbBCAXz1KyDb7m5OAs50Cn8OVqsjci +LWUGR/QPhAlcNba5jLhbbq7P4qh28aNx8u6XT/l/PSxgcfWm5vOc/2dN4I948R3P +U1iyLSqHTDFzsawHgXIyTUq9TOZXlBjWyWHBFsDI1AnQdeT3ELdeDzGyD6paGSB5 +IJ4LFyvJkslvKFGWgOc6EhB9zvxrKWBEMTDUTOfRe7gNQ1esr6XcsAqtJ7/QRTUF +36VxjlPiERA2m5tmNNy4hZ4zUXKlsJgeY+bAdVM= -----END CERTIFICATE-----`) var serverKey = []byte(`-----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA2lF1EXRYzsUIwM1vrMM/OIinPvOzqSQSQ0euiUYgcNTw2Y87 -2fNWnQXOP3JDHgzZw231kJbdRxeWgRdeTnvCFUaBzA63GXMMCljs1sgAVLXnAmWR -sE/TzG/OwsZUnyYzQMdG49PAEpw8GlX9t0eojmXPy7C/M0GnyAMx/UyMrq40lht6 -gdUI2gYRUofRRtHFs+a0EX8/q+49ciwgQx5inj2BX3Jc2cvc35Y3bSY86CYsAZic -PZP84wP5iWkYmvFhJUoS/JY2FMC6CvRFPcTi8Dnp28kHEaqrocmwajSfyiJe/1qJ -dMlHAInvTxp9E53cOzfCP6nmHbSKQPxm5u8hSQIDAQABAoIBAQCDhfNbEpa16qn9 -TUZr9CxQKLNpD3Q6/8oo0jRp6t98WizHRK0v/pM9gdPhETsyDVfbjpEUDG8+dw1q -s+NSsOgZ3SIxBuRz5oVobm4wbskUP4nuPbZpW44jaXBMkyNDxcW2ztb8RgM+svTa -gNea5Qa80sU+1zo47OLhcltZWBag3KCU/JQT+3LThVZDHt3GRx4QCASTJx3v/vBB -o9M5wCYZp6sP7wmFUZfwEpkTfJ5M7sG1h7ibD/8kjIvpnQj+OFpcoylDxTINvqsN -ADAe1NPK00Rx6vE9GNQ8ZA/lg0pih+EpK4PpE5cDDkYs3VchUlYHBSrsc7+K6kUk -mMTdmVvpAoGBAP7sHhKMEpmPIUqxb5M95l+PX5uOS0x08HM40Vr8mxgx4z849CpW -1vcQlZwcXwkxUfJyXZCPx9CK0Sw877Afpac1OL4RiEKZ3qmwLeI9RnRTeKC5qXJ9 -u31l+dgoSbRZDUdcM1ZwFs9+V8+zId58SifDaBjm3466VCMnD7KQUz4jAoGBANs9 -udy4Os+SvCYVUaiVvtoYMdcyym+VQzf3ycVAk91dO8Qha/5uFYD/f7ykgEgg7QCd -jQp+ZVYPD7Hbh8XNwAt/6T+bF1qe8TSM3K8uk2Wt/tlk1ZqRnNNYsIZ8BO8c4T+f -pbu/mCDdmTKWQWVEwCj2kKNBHptmlLO5Ie2nebujAoGBAIqoZccS138dAi+9iYHe -VnM96fQTltN0e+FAU2eZJMcpQ4D8+poY9/4U0DvEltDKOdeU612ZR0cgapwUXQ9A -d3sWkNGZebM4PIux35NCXxMg3+kUc51p1FRl5lrztvtYwMdC2E241D9yalL4DYEV -u8QbHoEE+y6IHQGt2nT22cBfAoGAWmZuT+uLHHIFsLJTtG7ifi1Bx9lCjZX/XIGI -qhQBpGJANZQOYp/jsAgqFI/D8XnaH8nXET+i60RUlWLO7inziQpaFAcQLyagkKmQ -iY9r6Z5AGkWwqgZmouLMDvfuVOYUntZmUS8kPFEDTU+VcXtSvNFGPHqqcytuH1kz -+zl2QX8CgYB9nFMpqARHl0kJp1UXtq9prUei+Lyu2gvrl3a74w96gqg3yx9+fU/n -FzGF2VXnC5n1KvCvQi3xjLNzCOXKM3igu7u50CiaA/HEYmyWyOJw2Nt2+ICvxcCH -rnsA8P8I/R5Esl0rvv2BbA1Q1O6SLC+Dfnhf7KulWmNgqVXKllj+Ng== +MIIEowIBAAKCAQEA0Gi/3oXPU0oRP38xNAQ1js4Py1fVqy5OW+3rcaZONdUiiHJ4 +6K1mIMKuEH2nrFAq2cFiXIm9prrSN8hyLY9lUguyAlecdHe6he/7/PAoX7NM8NX1 +LzOfUotcO+iZtrnMVCut2vlqjsGsnxppLKnFnp17ycOy7yKCuQHowPln5xp6JkhR +F0bxDXIFFmve4uHHlRsWcEoNQRJZTaOMrfrFYfqbc5iBitI7/5xgMPZNq282TQDc +GeJmbFHEu/L1qO6uTg1LkbNbZKNfXeAU46K+eecGCwPyTMlDYuREdoVpH9GYW7Cx +OIRTYgCLWVok96yk2jeVCq21BbxZ8+NBJ2rhqQIDAQABAoIBAB8bY3gdVOLDrWti +2r8+2ZelHiplw9i3Iq8KBhiCkC3s0Ci5nV5tc070f/KqLrrDhIHYIYxaatpWDEaT +PqeaPa9PW5SJ6ypfLJINTfllB0Gxi4xvAxe2htNVRcETaM4jUWJG2r5SeBsywUdG +M+ictoiETRPCiBS1e/mNVWZoU5/kyMApP+5U9+THKhV4V2Pi2OHon/AeTP1Yc/1A +lTB9giQIysDK11j9zbpL2ICW9HSbbeLJlsw8/wCOdnPHFn5kun6EuesOF6MingvM +vL9ZHsh6N7oOHupqRvDPImM3QTYSuBe1hTJSvhi7hmnl9dolsBKKxJz0wjCO1N+V +wdPzrwECgYEA9PH+5RSWJtUb2riDpM0QB/Cci6UEFeTJbaCSluog/EH1/E+mOo2J +VbLCMhWwF8kJYToG0sTFGbJ1J1SiaYan7pxH2YIZkgXePC8Vj5WJnLhv05kKRanq +kOE1hVnaaCeFJFiLjDW2WeEaNLo6Ap1Qnb6ObzwV+0QvWCMUnVQwfjECgYEA2dCh +JKDXdsuLUklTc39VKCXXhqO/5FFOzNrZhPEo6KV2mY0BZnNjau4Ff2F6UJb6NMza +fFSLScEZTdbCv5lElGAxJueqC+p2LR3dS/1JfdWJ0zrP1BZa+MWr8O510Go/NOC2 +/s5cR2aVBdJ2WK4d/3XShOr6W8T6hPisr5wFZPkCgYBptUIWpNLEAXZa5wRRC/pe +ItW8YkOoGytet0xr+rCvjNvWvpzzaf+Zz2KFcNyk9yqoHf2x2h9hnqV2iszok6dH +j4RmdwIIBaZJ/NvmMlfIHcSM4eAP/mtviPGrEgLyrhOEgv3+TXPbyAyiMrg0RqXy +3bjkgl7OKDfyZnlQCHRBEQKBgCfmLK6N/AoZzQKcxfmhOJMrI2jZdBw5vKqP6EqO +9oRvUuNbzgbbWjnLMhycWZCLp3emkts1jXJMOftlPLVmOQbI/Bf5Vc/q+gzXrKLv +2deAF0gnPMzH75AkfZObyt8Lp1pjU4IngQXfR6sSW3VxJ7OU/KQ2evf2hEF5YACn +HuHZAoGBAI+i6KI0WiWadenDctkuzLgxEPXaQ3oLBJjhE9CjpcdF6mRMWaU8lPgj +D1bo9L8wxvqIW5qIrX9dKnGgYAxnomhBNQn3C+5XDgtq6UiANalilqs3AoaWlQiF +WKaPuWf2T2ypFVzdzPl/0fsFUo8Rw5D4VO4nHeglOrkGQx+OdXA6 -----END RSA PRIVATE KEY-----`) var serverCert = []byte(`-----BEGIN CERTIFICATE----- -MIIDCTCCAfGgAwIBAgIJAPJbY53f15/vMA0GCSqGSIb3DQEBBQUAMBsxGTAXBgNV -BAMUEHdlYmhvb2tfdGVzdHNfY2EwIBcNMTcwMjExMDAyMjU0WhgPMjI5MDExMjcw -MDIyNTRaMB8xHTAbBgNVBAMMFHdlYmhvb2tfdGVzdHNfc2VydmVyMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2lF1EXRYzsUIwM1vrMM/OIinPvOzqSQS -Q0euiUYgcNTw2Y872fNWnQXOP3JDHgzZw231kJbdRxeWgRdeTnvCFUaBzA63GXMM -Cljs1sgAVLXnAmWRsE/TzG/OwsZUnyYzQMdG49PAEpw8GlX9t0eojmXPy7C/M0Gn -yAMx/UyMrq40lht6gdUI2gYRUofRRtHFs+a0EX8/q+49ciwgQx5inj2BX3Jc2cvc -35Y3bSY86CYsAZicPZP84wP5iWkYmvFhJUoS/JY2FMC6CvRFPcTi8Dnp28kHEaqr -ocmwajSfyiJe/1qJdMlHAInvTxp9E53cOzfCP6nmHbSKQPxm5u8hSQIDAQABo0ow -SDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEFBQcDAgYI -KwYBBQUHAwEwDwYDVR0RBAgwBocEfwAAATANBgkqhkiG9w0BAQUFAAOCAQEAX/CG -3C1nwCWabpBw6h0k7UlDI55nnTH6xHdSX9EFmHz49NmAym9gUwXK5xDPVDNYURjb -TD3R2e76Cov7wXRzw99BMzKOhNrMgjiOrc0WT4Ck5MOaKgjzZEIXRSSBllsrF9ut -hnnuSaaKwUVn4D/9vPMp/TuZoK7yZaW3Pyv0ScQfpkECDLKYIkXOlyhC/I5Tfbof -+zReStbTsc0EWMVLLIAbP7uPf1VcH5HnElh1ignxRAPBsXwF8jQzjUBTWcZ5dEi9 -ofIrWo+AVKvcoRlyZZyLjOKPzhA5+pwG4yBkWJB5Cshq2trOYVf3+uUN8lz6i57M -wqxS1Q1MmtLhyhy79Q== +MIIDHzCCAgegAwIBAgIUZIt+6zrmR41Be/CrcPHLsj3pCAQwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxMjMxNDFa +GA8yMjk0MDcyMzEyMzE0MVowHzEdMBsGA1UEAwwUd2ViaG9va190ZXN0c19zZXJ2 +ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQaL/ehc9TShE/fzE0 +BDWOzg/LV9WrLk5b7etxpk411SKIcnjorWYgwq4QfaesUCrZwWJcib2mutI3yHIt +j2VSC7ICV5x0d7qF7/v88Chfs0zw1fUvM59Si1w76Jm2ucxUK63a+WqOwayfGmks +qcWenXvJw7LvIoK5AejA+WfnGnomSFEXRvENcgUWa97i4ceVGxZwSg1BEllNo4yt ++sVh+ptzmIGK0jv/nGAw9k2rbzZNANwZ4mZsUcS78vWo7q5ODUuRs1tko19d4BTj +or555wYLA/JMyUNi5ER2hWkf0ZhbsLE4hFNiAItZWiT3rKTaN5UKrbUFvFnz40En +auGpAgMBAAGjVTBTMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgXgMB0GA1UdJQQWMBQG +CCsGAQUFBwMCBggrBgEFBQcDATAaBgNVHREEEzARhwR/AAABgglsb2NhbGhvc3Qw +DQYJKoZIhvcNAQELBQADggEBAFXNoW48IHJAcO84Smb+/k8DvJBwOorOPspJ/6DY +pYITCANq9kQ54bv3LgPuel5s2PytVL1dVQmAya1cT9kG3nIjNaulR2j5Sgt0Ilyd +Dk/HOE/zBi6KyifV3dgQSbzua8AI9VboR3o3FhmA9C9jDDxAS+q9+NQjB40/aG8m +TBx+oKgeYHee5llKNTsY1Jqh6TT47om70+sjvmgZ4blAV7ft+WG/h3ZVtAZJuFee +tchgUEpGR8ZGyK0r/vWBIKHNSqtG5gdOS9swQLdUFG90OivhddKUU8Zt52uUXbc/ +/ggEd4dM4X6B21xKJQY6vCnTvHFXcVJezV3g1xaNN0yR0DA= +-----END CERTIFICATE-----`) + +var serverCertNoSAN = []byte(`-----BEGIN CERTIFICATE----- +MIIC+DCCAeCgAwIBAgIUZIt+6zrmR41Be/CrcPHLsj3pCAUwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxMjMxNDFa +GA8yMjk0MDcyMzEyMzE0MVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0Gi/3oXPU0oRP38xNAQ1js4Py1fVqy5O +W+3rcaZONdUiiHJ46K1mIMKuEH2nrFAq2cFiXIm9prrSN8hyLY9lUguyAlecdHe6 +he/7/PAoX7NM8NX1LzOfUotcO+iZtrnMVCut2vlqjsGsnxppLKnFnp17ycOy7yKC +uQHowPln5xp6JkhRF0bxDXIFFmve4uHHlRsWcEoNQRJZTaOMrfrFYfqbc5iBitI7 +/5xgMPZNq282TQDcGeJmbFHEu/L1qO6uTg1LkbNbZKNfXeAU46K+eecGCwPyTMlD +YuREdoVpH9GYW7CxOIRTYgCLWVok96yk2jeVCq21BbxZ8+NBJ2rhqQIDAQABozkw +NzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEFBQcDAgYI +KwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggEBAGgrpuQ4n0W/TaaXhdbfFELziXoN +eT89eFYOqgtx/o97sj8B/y+n8tm+lBopfmdnifta3e8iNVULAd6JKjBhkFL1NY3Q +tR+VqT8uEZthDM69cyuGTv1GybnUCY9mtW9dSgHHkcJNsZGn9oMTCYDcBrgoA4s3 +vZc4wuPj8wyiSJBDYMNZfLWHCvLTOa0XUfh0Pf0/d26KuAUTNQkJZLZ5cbNXZuu3 +fVN5brtOw+Md5nUa60z+Xp0ESaGvOLUjnmd2SWUfAzcbFbbV4fAyYZF/93zCVTJ1 +ig9gRWmPqLcDDLf9LPyDR5yHRTSF4USH2ykun4PiPfstjfv0xwddWgG2+S8= -----END CERTIFICATE-----`) var clientKey = []byte(`-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAt6auF7X7jl94ilPpJid6k15/Tnt/hFWMRv2eGeuSBT1YMjDP -X5pIaseeIVdXl0ePKzPeyYJma4t7EpGhUO7TX12Cr1e5S8IMExNDpYGti3ncLf1O -HP+faJNAlbKMTgf2xr0HPg8Q2lsDKfam/OEn+7jqv3FpZ5guwQQy7beQhWV38YuR -vf2ChNJVcalN0h+MRNkT+hsGKGM9XKgKFGknDuCP8N0H7HrP7LLf/tLOMNq/PeMz -I6MXMlXmB4VRPMlf1zJGfvE6i0sSbNM0p2ZjArpjhjdveLuvBwan/Guk9vW970ON -sNn9gdLiwCSLqzhRy0cTlIJsSnlkhbuOQZsqKwIDAQABAoIBAE2gCVQCWsrJ9dRa -NWEoLLpfpeXRc4vG8R0MlCgWl0jZrg7A7NZXCyb/KwqitWY/G/fB2/hGwu3QLfwi -TBI+cF+N0fA1Xx/zbFEfwmcRkf4zSuqxd7PwJDv6icD8kCtnWFqWiZokmhYBhCvX -kquuq8zNU4QJ9uiPvateD/zEqzSGgMeL+j7RGJsRmh2TnSBgKXLwadRhYLeHiFu/ -AwoWljlhLNXrCzCLx2kJPIA9CNYYtShhQncfZfkfC0I02vPWX9hu8lMpKQp2MmD9 -b3DvVW3H6cjAtm/nsjGghYNCngep8uPX2twcrLOZfzJgsZJf+yn/KLWb/yhGBXjd -TERHRCECgYEA2i5OfkIrBIPQcjhQCtkjYBgKUKVS54KTTPQQ0zoiGRPkXb6kpqrt -kaCKGYXT4oqvQQapNZQykrLUQ/xzbdAAzdIwZ8hTWS5K5cxHOnkmOcPiKu2+jM4I -zT7sdAYn0aSbrh1pNRQDV0tQZcI1Urp/OcEuniaEblWhq5/VRCmpCBECgYEA13wg -jKRobq4QBoQM8pu1Ha7waeJZ26NcZwCxno0TwH2JZ6X9e4iXfhywUOgVW7hXzcs5 -2nBciVX5h31u1EDPJz6aPyzzQHi0YspDy/zuO0GWEJxLKm5QMyjh5vakhF5bVP6f -Dh3rXts/ZYKk3+p4ezXs2b+uTelowuq8Kk55qnsCgYAy/tvN2v1fAsg3yj27K2F/ -Vl8i1mF4RybSt8Eu/cl2fxXDa4nkgtMgVJuyt3r82ll4I2xtX4Qqka3Xbiw0oIdv -lA9IUqRYld9fss17N1Hd8pDsY8FD++xGvMxbmgy4jXbtzWYHx/O39ZyHDEuWWIzg -HO0effY6K72r9aHNWsdtYQKBgQDNXUw8Hbg1u4gkXZdlZEYxevc/Qmz3KXK36+5b -uAJaEopwkL7LC/utQjQ7d2RbnI154TRK3Ykjjh+ZJE8K1JVYxo4EpZdTG3Z3LGN+ -tphpOvGE9R+h2a5vg4gAMZHLYY3TrDL0JkmahoOd/+uYR4L5kgQf5lF9iXTBRyt7 -enzznwKBgBr9rHUsdd8whEo/UntKaGQ6sqevd/ESxDErUqkTkiTtDAT2vuLQwgA0 -JnzYUv1/wmoLCz6Nbe6Fg9shAgRY/tz7GI75NBi4HBjp286df6Uu6qEJ4LAELc0+ -Yh/uRF/H4WkpmBckEXobnRxoX/0HWFY5SETLoG/nwaxJG9YL/Acr +MIIEowIBAAKCAQEA4JpoQtghwWWUQWMrjAdbwoxvXk4WVVBR/SJPeRxvDDhyQzUh +3PJDmarSck1ZbWdhaa81p2hezt7boWyOe9Y8X+yASYOPJ+e+JYEioDttV1DW5Kz1 +f4VjRXgGgEPZMzjm5VttXGGddwIRV7mH7M2gjyVfnKWYtfrwqf55evZm+vYvydZ+ +bz054vFusWiAUWRU4W3cHcrmKw4TC/f/I0BpWbC02E0yFfzVkF5qDNtX1NBVVG6S +sRooegkU6LIok0+XcZ7gyuHfb0DT1AhUFz3aweMqJRMcXi9zQoB2k9z+GBiliJU/ +Q6UFGoDpi708omY6ZJOVDoSS1u27k/rrJLtjBwIDAQABAoIBAD1XjoArHVT/lDUj +i6Ir5FDHICeGdi0Iycuz0jbGoEGbLQHUAhKhC/ttHxreqzzpDj7Z/nJhW/tt/Fgj +GOsTgNvF8OejCbchYNpHYZiXgm5lgrtjzYztdkZj2IqxfSGzOcSMatrfQi1doDTw +VAJFysMttuZGh+WzyoTzHmSJ5WhGz5W1QDCNe/YzMSYNkY86QRq2sekloVH03bd3 +vSV17TfRn3JoLhSp2xQdWGZ5x6BosUTG0H6yOMYC1E2qKRb6N3h1LRwkTOX6o83z +wE31lFOjN4zo9qoksKx3In+qJb9UWQ6m6kPjVCH6UMZ+usExtMhYpY8EHEN6LXMV +iiarwWkCgYEA9P6bieG1GOAsRdyenC+fsYNHDiV+eJnAev0Q+etr2ATt7V6Fo3ed +XEdTP/ye6uK462paWrTjUpJGjQVOEHKMjrdz8xjss8zJ5qk2kcCiljwhEVhlzJWU +1ZwZyhpKwFCJJ8sKdoyBi8xMJ5wsn1tBiS1ajE2HqwXlYjCKXwF4YJMCgYEA6rFN +YNqgcznYxwqri/0ZZ3gvWTiYQMc02vmHK7QEh0tdzMgP8wKMYyGXYN+BQw0PtZUB +tRiQ1ZZbGukeoXGuTat4RKZPQtucC2FCzV6hwcQjR41wO1uvgANXXt9d9bssV0RJ +QFRtvw3lT5NveNJttOtXjpnrFKV5qzHhuDNeID0CgYB3f1CiWkMY1Q8DLG74w4ni +7aNU1Rjw/h2SX6lOHXRGs3DR3M7WQtuwTOHgTfm9m8SSnxdfMufGOK7Mzg+bmiPQ +4M5ffoLF2FeKeHKzeQBUwxNmAHX26ebkofU6TMS1NRMRI8oAtNvjMkza7SQevkYj +AMovsQlBc+jEkeSMHo12nQKBgQDdBvghxCSiABO5eMIQx9/aA2VMOLGjKDkOFTyJ +GzO8m20OhEgJDBhc7/DaCgR5ul5WG6wM+sLEdKL+mBgB4cdPXLKarKovBBZU3VeF +hJ8AlpDHPVw6euGwxPTO47xgxdM/qI74J8tnyFVy+R5dgohRf+bo5Opxelo4sXSr +/qyh3QKBgGbKrhvrShZG9GbWWLYKy7ZxW6n1fUuU3clr1pkWI6Xe6fOPVpYGcsKi +T+jfxMsj8NBtNh20sDohWj0wXiCvfNVwmC5B8wa7kXjvCJtxvLW6FgCQSFAI5z/Y +cZW3W/GCCrvIQQZ+W4/MetTnK7WawHjZ5JxjWGCl6FrG+pKpKiqQ -----END RSA PRIVATE KEY-----`) var clientCert = []byte(`-----BEGIN CERTIFICATE----- -MIIDCTCCAfGgAwIBAgIJAPJbY53f15/wMA0GCSqGSIb3DQEBBQUAMBsxGTAXBgNV -BAMUEHdlYmhvb2tfdGVzdHNfY2EwIBcNMTcwMjExMDAyMjU0WhgPMjI5MDExMjcw -MDIyNTRaMB8xHTAbBgNVBAMMFHdlYmhvb2tfdGVzdHNfY2xpZW50MIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt6auF7X7jl94ilPpJid6k15/Tnt/hFWM -Rv2eGeuSBT1YMjDPX5pIaseeIVdXl0ePKzPeyYJma4t7EpGhUO7TX12Cr1e5S8IM -ExNDpYGti3ncLf1OHP+faJNAlbKMTgf2xr0HPg8Q2lsDKfam/OEn+7jqv3FpZ5gu -wQQy7beQhWV38YuRvf2ChNJVcalN0h+MRNkT+hsGKGM9XKgKFGknDuCP8N0H7HrP -7LLf/tLOMNq/PeMzI6MXMlXmB4VRPMlf1zJGfvE6i0sSbNM0p2ZjArpjhjdveLuv -Bwan/Guk9vW970ONsNn9gdLiwCSLqzhRy0cTlIJsSnlkhbuOQZsqKwIDAQABo0ow -SDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEFBQcDAgYI -KwYBBQUHAwEwDwYDVR0RBAgwBocEfwAAATANBgkqhkiG9w0BAQUFAAOCAQEAD8aZ -E0nof9Rh7s+uwtF70KPgcz71ft0c1+vSmeLm4IkN0f+amcvgaT8xZLwNv1b77NZo -uMWXvit24eIuiqzq7umKiHP/UrFv+Rl+9ue+lA3N0e3WikRoJsh3aoIn8BQUBbnX -Nr9R69SeRYYRpMrs19N5Wn4gN7Nfie+1FKWsL3myJYDFsg+8GMEcOJ0YdOMALMy0 -tIJdYji28mTQ++lpGbekjhf7p9wazQ/6CVd8WNpIbGO84QbGCcpCaVM2XxOSiV/F -hIGO1Z30SBq8rQw51XbhdRX+uvRM1ya4RuBMCSX/hpsMu9lVRqCzbkU4PvuUTqLA -CebKCgjYbM0CWrP9kw== +MIIDFDCCAfygAwIBAgIUZIt+6zrmR41Be/CrcPHLsj3pCAYwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxMjMxNDFa +GA8yMjk0MDcyMzEyMzE0MVowHzEdMBsGA1UEAwwUd2ViaG9va190ZXN0c19jbGll +bnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDgmmhC2CHBZZRBYyuM +B1vCjG9eThZVUFH9Ik95HG8MOHJDNSHc8kOZqtJyTVltZ2FprzWnaF7O3tuhbI57 +1jxf7IBJg48n574lgSKgO21XUNbkrPV/hWNFeAaAQ9kzOOblW21cYZ13AhFXuYfs +zaCPJV+cpZi1+vCp/nl69mb69i/J1n5vPTni8W6xaIBRZFThbdwdyuYrDhML9/8j +QGlZsLTYTTIV/NWQXmoM21fU0FVUbpKxGih6CRTosiiTT5dxnuDK4d9vQNPUCFQX +PdrB4yolExxeL3NCgHaT3P4YGKWIlT9DpQUagOmLvTyiZjpkk5UOhJLW7buT+usk +u2MHAgMBAAGjSjBIMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgXgMB0GA1UdJQQWMBQG +CCsGAQUFBwMCBggrBgEFBQcDATAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEB +CwUAA4IBAQBUockuDBViT8F/1plpUT5fccMkePO8h/wsk/K8IxpXAv8XerBL0H/j +k235/cGvksiYH1nKSS8VOZfEYY+b0lUDRJFf6kppU7TFiEaH3idGAcmlQ1dSBu1f +iVP63tF0KOfdPlCzdXY4jzyXSnUsxZejALmecCZt1LCOMai2w5zzqy0isjWnDPFF ++Ssmf5HxtfrPvsKDJGD+79HGtTJ3dXmifuvRuuXsYSv2uQPtJsbtXhQWaQHs/6Bg +OtuFbTvFOyxYFJmAl/YM6hRbvKB8Itq38aayi47TA/b11QdAXCB9DMvivI6a89F7 +uL8+zNieoQkyj84eXfkyiSs03TrqOsj9 -----END CERTIFICATE-----`) diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/client.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/client.go index d750c3f7c03..56600f155bd 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/client.go +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/client.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apiserver/pkg/util/x509metrics" "k8s.io/client-go/rest" ) @@ -148,6 +149,11 @@ func (cm *ClientManager) HookClient(cc ClientConfig) (*rest.RESTClient, error) { cfg.ContentConfig.NegotiatedSerializer = cm.negotiatedSerializer cfg.ContentConfig.ContentType = runtime.ContentTypeJSON + + // Add a transport wrapper that allows detection of TLS connections to + // servers without SAN extension in their serving certificates + cfg.Wrap(x509metrics.NewMissingSANRoundTripperWrapperConstructor(x509MissingSANCounter)) + client, err := rest.UnversionedRESTClientFor(cfg) if err == nil { cm.cache.Add(string(cacheKey), client) diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/gencerts.sh b/staging/src/k8s.io/apiserver/pkg/util/webhook/gencerts.sh index de407730117..f99895e298a 100755 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/gencerts.sh +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/gencerts.sh @@ -35,6 +35,18 @@ extendedKeyUsage = clientAuth, serverAuth subjectAltName = @alt_names [alt_names] IP.1 = 127.0.0.1 +DNS.1 = localhost +EOF + +cat > server_no_san.conf << EOF +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth EOF cat > client.conf << EOF @@ -64,6 +76,10 @@ openssl genrsa -out serverKey.pem 2048 openssl req -new -key serverKey.pem -out server.csr -subj "/CN=${CN_BASE}_server" -config server.conf openssl x509 -req -in server.csr -CA caCert.pem -CAkey caKey.pem -CAcreateserial -out serverCert.pem -days 100000 -extensions v3_req -extfile server.conf +# Create a server certiticate w/o SAN +openssl req -new -key serverKey.pem -out serverNoSAN.csr -subj "/CN=localhost" -config server_no_san.conf +openssl x509 -req -in serverNoSAN.csr -CA caCert.pem -CAkey caKey.pem -CAcreateserial -out serverCertNoSAN.pem -days 100000 -extensions v3_req -extfile server_no_san.conf + # Create a client certiticate openssl genrsa -out clientKey.pem 2048 openssl req -new -key clientKey.pem -out client.csr -subj "/CN=${CN_BASE}_client" -config client.conf @@ -94,7 +110,7 @@ limitations under the License. package webhook EOF -for file in caKey caCert badCAKey badCACert serverKey serverCert clientKey clientCert; do +for file in caKey caCert badCAKey badCACert serverKey serverCert serverCertNoSAN clientKey clientCert; do data=$(cat ${file}.pem) echo "" >> $outfile echo "var $file = []byte(\`$data\`)" >> $outfile diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/metrics.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/metrics.go new file mode 100644 index 00000000000..c3085226cfe --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/metrics.go @@ -0,0 +1,39 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +var x509MissingSANCounter = metrics.NewCounter( + &metrics.CounterOpts{ + Subsystem: "webhooks", + Namespace: "apiserver", + Name: "x509_missing_san_total", + Help: "Counts the number of requests to servers missing SAN extension " + + "in their serving certificate OR the number of connection failures " + + "due to the lack of x509 certificate SAN extension missing " + + "(either/or, based on the runtime environment)", + StabilityLevel: metrics.ALPHA, + }, +) + +func init() { + legacyregistry.MustRegister(x509MissingSANCounter) +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go index abaade352b0..a17ab8f48f7 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/util/x509metrics" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -107,6 +108,7 @@ func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFact clientConfig.ContentConfig.NegotiatedSerializer = serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{Serializer: codec}) clientConfig.Dial = customDial + clientConfig.Wrap(x509metrics.NewMissingSANRoundTripperWrapperConstructor(x509MissingSANCounter)) restClient, err := rest.UnversionedRESTClientFor(clientConfig) if err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go index 0c4f208cfb3..a61c40bab8f 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook_test.go @@ -24,8 +24,10 @@ import ( "errors" "fmt" "io/ioutil" + "net" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "regexp" @@ -40,6 +42,8 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" v1 "k8s.io/client-go/tools/clientcmd/api/v1" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) const ( @@ -301,6 +305,7 @@ func TestTLSConfig(t *testing.T) { clientCert, clientKey, clientCA []byte serverCert, serverKey, serverCA []byte errRegex string + increaseSANWarnCounter bool }{ { test: "invalid server CA", @@ -351,9 +356,19 @@ func TestTLSConfig(t *testing.T) { errRegex: "", }, { - test: "webhook does not support insecure servers", + test: "webhook does not support insecure servers", + serverCert: serverCert, serverKey: serverKey, errRegex: errSignedByUnknownCA, }, + { + // this will fail when GODEBUG is set to x509ignoreCN=0 with + // expected err, but the SAN counter gets increased + test: "server cert does not have SAN extension", + clientCA: caCert, + serverCert: serverCertNoSAN, serverKey: serverKey, + errRegex: "x509: certificate relies on legacy Common Name field", + increaseSANWarnCounter: true, + }, } for _, tt := range tests { @@ -361,12 +376,18 @@ func TestTLSConfig(t *testing.T) { func() { // Create and start a simple HTTPS server server, err := newTestServer(tt.serverCert, tt.serverKey, tt.serverCA, nil) - if err != nil { t.Errorf("%s: failed to create server: %v", tt.test, err) return } + serverURL, err := url.Parse(server.URL) + if err != nil { + t.Errorf("%s: failed to parse the testserver URL: %v", tt.test, err) + return + } + serverURL.Host = net.JoinHostPort("localhost", serverURL.Port()) + defer server.Close() // Create a Kubernetes client configuration file @@ -374,7 +395,7 @@ func TestTLSConfig(t *testing.T) { Clusters: []v1.NamedCluster{ { Cluster: v1.Cluster{ - Server: server.URL, + Server: serverURL.String(), CertificateAuthorityData: tt.clientCA, }, }, @@ -413,6 +434,17 @@ func TestTLSConfig(t *testing.T) { t.Errorf("%s: unexpected error message mismatch:\n Expected: %s\n Actual: %s", tt.test, tt.errRegex, err.Error()) } } + + if tt.increaseSANWarnCounter { + errorCounter := getSingleCounterValueFromRegistry(t, legacyregistry.DefaultGatherer, "apiserver_webhooks_x509_missing_san_total") + + if errorCounter == -1 { + t.Errorf("failed to get the x509_common_name_error_count metrics: %v", err) + } + if int(errorCounter) != 1 { + t.Errorf("expected the x509_common_name_error_count to be 1, but it's %d", errorCounter) + } + } }() } } @@ -790,3 +822,24 @@ func TestGenericWebhookWithExponentialBackoff(t *testing.T) { t.Errorf("expected a total of %d webhook attempts but got: %d", totalAttemptsExpected, attemptsGot) } } + +func getSingleCounterValueFromRegistry(t *testing.T, r metrics.Gatherer, name string) int { + mfs, err := r.Gather() + if err != nil { + t.Logf("failed to gather local registry metrics: %v", err) + return -1 + } + + for _, mf := range mfs { + if mf.Name != nil && *mf.Name == name { + mfMetric := mf.GetMetric() + for _, m := range mfMetric { + if m.GetCounter() != nil { + return int(m.GetCounter().GetValue()) + } + } + } + } + + return -1 +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/x509metrics/missing_san.go b/staging/src/k8s.io/apiserver/pkg/util/x509metrics/missing_san.go new file mode 100644 index 00000000000..d16daec2511 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/x509metrics/missing_san.go @@ -0,0 +1,92 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package x509metrics + +import ( + "crypto/x509" + "errors" + "net/http" + "strings" + + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/component-base/metrics" +) + +var _ utilnet.RoundTripperWrapper = &x509MissingSANErrorMetricsRTWrapper{} + +type x509MissingSANErrorMetricsRTWrapper struct { + rt http.RoundTripper + + counter *metrics.Counter +} + +// NewMissingSANRoundTripperWrapperConstructor returns a RoundTripper wrapper that's usable +// within ClientConfig.Wrap that increases the `metricCounter` whenever: +// 1. we get a x509.HostnameError with string `x509: certificate relies on legacy Common Name field` +// which indicates an error caused by the deprecation of Common Name field when veryfing remote +// hostname +// 2. the server certificate in response contains no SAN. This indicates that this binary run +// with the GODEBUG=x509ignoreCN=0 in env +func NewMissingSANRoundTripperWrapperConstructor(metricCounter *metrics.Counter) func(rt http.RoundTripper) http.RoundTripper { + return func(rt http.RoundTripper) http.RoundTripper { + return &x509MissingSANErrorMetricsRTWrapper{ + rt: rt, + counter: metricCounter, + } + } +} + +func (w *x509MissingSANErrorMetricsRTWrapper) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := w.rt.RoundTrip(req) + checkForHostnameError(err, w.counter) + checkRespForNoSAN(resp, w.counter) + return resp, err +} + +func (w *x509MissingSANErrorMetricsRTWrapper) WrappedRoundTripper() http.RoundTripper { + return w.rt +} + +// checkForHostnameError increases the metricCounter when we're running w/o GODEBUG=x509ignoreCN=0 +// and the client reports a HostnameError about the legacy CN fields +func checkForHostnameError(err error, metricCounter *metrics.Counter) { + if err != nil && errors.As(err, &x509.HostnameError{}) && strings.Contains(err.Error(), "x509: certificate relies on legacy Common Name field") { + // increase the count of registered failures due to Go 1.15 x509 cert Common Name deprecation + metricCounter.Inc() + } +} + +// checkRespForNoSAN increases the metricCounter when the server response contains +// a leaf certificate w/o the SAN extension +func checkRespForNoSAN(resp *http.Response, metricCounter *metrics.Counter) { + if resp != nil && resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 { + if serverCert := resp.TLS.PeerCertificates[0]; !hasSAN(serverCert) { + metricCounter.Inc() + } + } +} + +func hasSAN(c *x509.Certificate) bool { + sanOID := []int{2, 5, 29, 17} + + for _, e := range c.Extensions { + if e.Id.Equal(sanOID) { + return true + } + } + return false +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/x509metrics/missing_san_test.go b/staging/src/k8s.io/apiserver/pkg/util/x509metrics/missing_san_test.go new file mode 100644 index 00000000000..227cc1924b4 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/x509metrics/missing_san_test.go @@ -0,0 +1,301 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package x509metrics + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/testutil" +) + +// taken from pkg/util/webhook/certs_test.go +var caCert = []byte(`-----BEGIN CERTIFICATE----- +MIIDGTCCAgGgAwIBAgIUOS2MkobR2t4rguefcC78gLuXkc0wDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxMjMxNDFa +GA8yMjk0MDcyMzEyMzE0MVowGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMy8o1sRSe8viuSxh8a7Ou8n +vyDJxeJA9son+clZdSwa1YpbhSsYPr2svTGLT4ZxX3JkvHZGDaB+G6QrZNIWOssJ +4+fjvEsVSnk9q7Yr8wX8QdShksnSP0qdB1xFKlqwTFcZyQHqjlyctV93/LEosM9D +RZzgPpXJuC6wD7qdLRSPxQwuegZYcMGT2Y4/8CfBgiEUkZNrGYJiDFrblI0N9jX2 +jUeP/g8xWJIKLTBedVbAzj02Y66WdNecYcyTMROdclPB1IbF4ABbCGynnP0fbiaf +0+4v0pLedqwCYSWD/ujajyDtYuhI8U+OkENwkZ/h5jw/6aWq7esMZDbEAT6UPgEC +AwEAAaNTMFEwHQYDVR0OBBYEFBgvyZWkRJCRjxclYA0mMlnzc/GmMB8GA1UdIwQY +MBaAFBgvyZWkRJCRjxclYA0mMlnzc/GmMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAHigaFJ8DqYFMGXNUT5lYM8oLLWYcbivGxR7ofDm8rXWGLdQ +tKsWeusSlgpzeO1PKCzwSrYQhlFIZI6AH+ch7EAWt84MfAaMz/1DiXF6Q8fAcR0h +QoyFIAKTiVkcgOQjSQOIM2SS5cyGeDRaHGVWfaJOwdIYo6ctFzI2asPJ4yU0QsA7 +0WTD2+sBG6AXGhfafGUHEmou8sGQ+QT8rgi4hs1bfyHuT5dPgC4TbwknD1G8pMqm +ID3CIiCF8hhF5C2hjrW0LTJ6zTlg1c1K0NmmUL1ucsfzEk//c7GsU8dk+FYmtW9A +VzryJj1NmnSqA3zd3jBMuK37Ei3pRvVbO7Uxf14= +-----END CERTIFICATE-----`) + +var serverKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0Gi/3oXPU0oRP38xNAQ1js4Py1fVqy5OW+3rcaZONdUiiHJ4 +6K1mIMKuEH2nrFAq2cFiXIm9prrSN8hyLY9lUguyAlecdHe6he/7/PAoX7NM8NX1 +LzOfUotcO+iZtrnMVCut2vlqjsGsnxppLKnFnp17ycOy7yKCuQHowPln5xp6JkhR +F0bxDXIFFmve4uHHlRsWcEoNQRJZTaOMrfrFYfqbc5iBitI7/5xgMPZNq282TQDc +GeJmbFHEu/L1qO6uTg1LkbNbZKNfXeAU46K+eecGCwPyTMlDYuREdoVpH9GYW7Cx +OIRTYgCLWVok96yk2jeVCq21BbxZ8+NBJ2rhqQIDAQABAoIBAB8bY3gdVOLDrWti +2r8+2ZelHiplw9i3Iq8KBhiCkC3s0Ci5nV5tc070f/KqLrrDhIHYIYxaatpWDEaT +PqeaPa9PW5SJ6ypfLJINTfllB0Gxi4xvAxe2htNVRcETaM4jUWJG2r5SeBsywUdG +M+ictoiETRPCiBS1e/mNVWZoU5/kyMApP+5U9+THKhV4V2Pi2OHon/AeTP1Yc/1A +lTB9giQIysDK11j9zbpL2ICW9HSbbeLJlsw8/wCOdnPHFn5kun6EuesOF6MingvM +vL9ZHsh6N7oOHupqRvDPImM3QTYSuBe1hTJSvhi7hmnl9dolsBKKxJz0wjCO1N+V +wdPzrwECgYEA9PH+5RSWJtUb2riDpM0QB/Cci6UEFeTJbaCSluog/EH1/E+mOo2J +VbLCMhWwF8kJYToG0sTFGbJ1J1SiaYan7pxH2YIZkgXePC8Vj5WJnLhv05kKRanq +kOE1hVnaaCeFJFiLjDW2WeEaNLo6Ap1Qnb6ObzwV+0QvWCMUnVQwfjECgYEA2dCh +JKDXdsuLUklTc39VKCXXhqO/5FFOzNrZhPEo6KV2mY0BZnNjau4Ff2F6UJb6NMza +fFSLScEZTdbCv5lElGAxJueqC+p2LR3dS/1JfdWJ0zrP1BZa+MWr8O510Go/NOC2 +/s5cR2aVBdJ2WK4d/3XShOr6W8T6hPisr5wFZPkCgYBptUIWpNLEAXZa5wRRC/pe +ItW8YkOoGytet0xr+rCvjNvWvpzzaf+Zz2KFcNyk9yqoHf2x2h9hnqV2iszok6dH +j4RmdwIIBaZJ/NvmMlfIHcSM4eAP/mtviPGrEgLyrhOEgv3+TXPbyAyiMrg0RqXy +3bjkgl7OKDfyZnlQCHRBEQKBgCfmLK6N/AoZzQKcxfmhOJMrI2jZdBw5vKqP6EqO +9oRvUuNbzgbbWjnLMhycWZCLp3emkts1jXJMOftlPLVmOQbI/Bf5Vc/q+gzXrKLv +2deAF0gnPMzH75AkfZObyt8Lp1pjU4IngQXfR6sSW3VxJ7OU/KQ2evf2hEF5YACn +HuHZAoGBAI+i6KI0WiWadenDctkuzLgxEPXaQ3oLBJjhE9CjpcdF6mRMWaU8lPgj +D1bo9L8wxvqIW5qIrX9dKnGgYAxnomhBNQn3C+5XDgtq6UiANalilqs3AoaWlQiF +WKaPuWf2T2ypFVzdzPl/0fsFUo8Rw5D4VO4nHeglOrkGQx+OdXA6 +-----END RSA PRIVATE KEY-----`) + +var serverCert = []byte(`-----BEGIN CERTIFICATE----- +MIIDHzCCAgegAwIBAgIUZIt+6zrmR41Be/CrcPHLsj3pCAQwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxMjMxNDFa +GA8yMjk0MDcyMzEyMzE0MVowHzEdMBsGA1UEAwwUd2ViaG9va190ZXN0c19zZXJ2 +ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQaL/ehc9TShE/fzE0 +BDWOzg/LV9WrLk5b7etxpk411SKIcnjorWYgwq4QfaesUCrZwWJcib2mutI3yHIt +j2VSC7ICV5x0d7qF7/v88Chfs0zw1fUvM59Si1w76Jm2ucxUK63a+WqOwayfGmks +qcWenXvJw7LvIoK5AejA+WfnGnomSFEXRvENcgUWa97i4ceVGxZwSg1BEllNo4yt ++sVh+ptzmIGK0jv/nGAw9k2rbzZNANwZ4mZsUcS78vWo7q5ODUuRs1tko19d4BTj +or555wYLA/JMyUNi5ER2hWkf0ZhbsLE4hFNiAItZWiT3rKTaN5UKrbUFvFnz40En +auGpAgMBAAGjVTBTMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgXgMB0GA1UdJQQWMBQG +CCsGAQUFBwMCBggrBgEFBQcDATAaBgNVHREEEzARhwR/AAABgglsb2NhbGhvc3Qw +DQYJKoZIhvcNAQELBQADggEBAFXNoW48IHJAcO84Smb+/k8DvJBwOorOPspJ/6DY +pYITCANq9kQ54bv3LgPuel5s2PytVL1dVQmAya1cT9kG3nIjNaulR2j5Sgt0Ilyd +Dk/HOE/zBi6KyifV3dgQSbzua8AI9VboR3o3FhmA9C9jDDxAS+q9+NQjB40/aG8m +TBx+oKgeYHee5llKNTsY1Jqh6TT47om70+sjvmgZ4blAV7ft+WG/h3ZVtAZJuFee +tchgUEpGR8ZGyK0r/vWBIKHNSqtG5gdOS9swQLdUFG90OivhddKUU8Zt52uUXbc/ +/ggEd4dM4X6B21xKJQY6vCnTvHFXcVJezV3g1xaNN0yR0DA= +-----END CERTIFICATE-----`) + +var serverCertNoSAN = []byte(`-----BEGIN CERTIFICATE----- +MIIC+DCCAeCgAwIBAgIUZIt+6zrmR41Be/CrcPHLsj3pCAUwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxMjMxNDFa +GA8yMjk0MDcyMzEyMzE0MVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0Gi/3oXPU0oRP38xNAQ1js4Py1fVqy5O +W+3rcaZONdUiiHJ46K1mIMKuEH2nrFAq2cFiXIm9prrSN8hyLY9lUguyAlecdHe6 +he/7/PAoX7NM8NX1LzOfUotcO+iZtrnMVCut2vlqjsGsnxppLKnFnp17ycOy7yKC +uQHowPln5xp6JkhRF0bxDXIFFmve4uHHlRsWcEoNQRJZTaOMrfrFYfqbc5iBitI7 +/5xgMPZNq282TQDcGeJmbFHEu/L1qO6uTg1LkbNbZKNfXeAU46K+eecGCwPyTMlD +YuREdoVpH9GYW7CxOIRTYgCLWVok96yk2jeVCq21BbxZ8+NBJ2rhqQIDAQABozkw +NzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEFBQcDAgYI +KwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggEBAGgrpuQ4n0W/TaaXhdbfFELziXoN +eT89eFYOqgtx/o97sj8B/y+n8tm+lBopfmdnifta3e8iNVULAd6JKjBhkFL1NY3Q +tR+VqT8uEZthDM69cyuGTv1GybnUCY9mtW9dSgHHkcJNsZGn9oMTCYDcBrgoA4s3 +vZc4wuPj8wyiSJBDYMNZfLWHCvLTOa0XUfh0Pf0/d26KuAUTNQkJZLZ5cbNXZuu3 +fVN5brtOw+Md5nUa60z+Xp0ESaGvOLUjnmd2SWUfAzcbFbbV4fAyYZF/93zCVTJ1 +ig9gRWmPqLcDDLf9LPyDR5yHRTSF4USH2ykun4PiPfstjfv0xwddWgG2+S8= +-----END CERTIFICATE-----`) + +// Test_checkForHostnameError tests that the upstream message for remote server +// certificate's hostname hasn't changed when no SAN extension is present and that +// the metrics counter increases properly when such an error is encountered +// +// Requires GODEBUG=x509ignoreCN=0 to not be set in the environment +func TestCheckForHostnameError(t *testing.T) { + tests := []struct { + name string + serverCert []byte + counterIncrease bool + }{ + { + name: "no SAN", + serverCert: serverCertNoSAN, + counterIncrease: true, + }, + { + name: "with SAN", + serverCert: serverCert, + }, + } + + // register the test metrics + x509MissingSANCounter := metrics.NewCounter(&metrics.CounterOpts{Name: "Test_checkForHostnameError"}) + registry := testutil.NewFakeKubeRegistry("0.0.0") + registry.MustRegister(x509MissingSANCounter) + + var lastCounterVal int + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tlsServer, serverURL := testServer(t, tt.serverCert) + defer tlsServer.Close() + + client := tlsServer.Client() + req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil) + if err != nil { + t.Fatalf("failed to create an http request: %v", err) + } + + _, err = client.Transport.RoundTrip(req) + + checkForHostnameError(err, x509MissingSANCounter) + + errorCounterVal := getSingleCounterValueFromRegistry(t, registry, "Test_checkForHostnameError") + if errorCounterVal == -1 { + t.Fatalf("failed to get the error counter from the registry") + } + + if tt.counterIncrease && errorCounterVal != lastCounterVal+1 { + t.Errorf("expected the Test_checkForHostnameError metrics to increase by 1 from %d, but it is %d", lastCounterVal, errorCounterVal) + } + + if !tt.counterIncrease && errorCounterVal != lastCounterVal { + t.Errorf("expected the Test_checkForHostnameError metrics to stay the same (%d), but it is %d", lastCounterVal, errorCounterVal) + } + + lastCounterVal = errorCounterVal + }) + } +} + +func TestCheckRespForNoSAN(t *testing.T) { + tests := []struct { + name string + serverCert []byte + counterIncrease bool + }{ + { + name: "no certs", + }, + { + name: "no SAN", + serverCert: serverCertNoSAN, + counterIncrease: true, + }, + { + name: "with SAN", + serverCert: serverCert, + }, + } + + // register the test metrics + x509MissingSANCounter := metrics.NewCounter(&metrics.CounterOpts{Name: "Test_checkRespForNoSAN"}) + registry := testutil.NewFakeKubeRegistry("0.0.0") + registry.MustRegister(x509MissingSANCounter) + + var lastCounterVal int + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var tlsConnectionState *tls.ConnectionState + if tt.serverCert != nil { + block, _ := pem.Decode([]byte(tt.serverCert)) + if block == nil { + t.Fatal("failed to parse certificate PEM") + } + + serverCert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("failed to parse certificate: %v", err) + } + + tlsConnectionState = &tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{serverCert}, + } + } + + resp := &http.Response{ + TLS: tlsConnectionState, + } + + checkRespForNoSAN(resp, x509MissingSANCounter) + + errorCounterVal := getSingleCounterValueFromRegistry(t, registry, "Test_checkRespForNoSAN") + if errorCounterVal == -1 { + t.Fatalf("failed to get the error counter from the registry") + } + + if tt.counterIncrease && errorCounterVal != lastCounterVal+1 { + t.Errorf("expected the Test_checkRespForNoSAN metrics to increase by 1 from %d, but it is %d", lastCounterVal, errorCounterVal) + } + + if !tt.counterIncrease && errorCounterVal != lastCounterVal { + t.Errorf("expected the Test_checkRespForNoSAN metrics to stay the same (%d), but it is %d", lastCounterVal, errorCounterVal) + } + + lastCounterVal = errorCounterVal + }) + } +} + +func testServer(t *testing.T, serverCert []byte) (*httptest.Server, *url.URL) { + rootCAs := x509.NewCertPool() + rootCAs.AppendCertsFromPEM(caCert) + + cert, err := tls.X509KeyPair(serverCert, serverKey) + if err != nil { + t.Fatalf("failed to init x509 cert/key pair: %v", err) + } + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: rootCAs, + } + + tlsServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(200) + w.Write([]byte("ok")) + })) + + tlsServer.TLS = tlsConfig + tlsServer.StartTLS() + + serverURL, err := url.Parse(tlsServer.URL) + if err != nil { + tlsServer.Close() + t.Fatalf("failed to parse the testserver URL: %v", err) + } + serverURL.Host = net.JoinHostPort("localhost", serverURL.Port()) + + return tlsServer, serverURL +} + +func getSingleCounterValueFromRegistry(t *testing.T, r metrics.Gatherer, name string) int { + mfs, err := r.Gather() + if err != nil { + t.Logf("failed to gather local registry metrics: %v", err) + return -1 + } + + for _, mf := range mfs { + if mf.Name != nil && *mf.Name == name { + mfMetric := mf.GetMetric() + for _, m := range mfMetric { + if m.GetCounter() != nil { + return int(m.GetCounter().GetValue()) + } + } + } + } + + return -1 +} diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go index e09f0d1a238..2f19981d3bb 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go @@ -35,6 +35,7 @@ import ( genericfeatures "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server/egressselector" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/apiserver/pkg/util/x509metrics" restclient "k8s.io/client-go/rest" "k8s.io/client-go/transport" "k8s.io/klog/v2" @@ -250,17 +251,20 @@ func (r *proxyHandler) updateAPIService(apiService *apiregistrationv1api.APIServ proxyClientCert, proxyClientKey := r.proxyCurrentCertKeyContent() - newInfo := proxyHandlingInfo{ - name: apiService.Name, - restConfig: &restclient.Config{ - TLSClientConfig: restclient.TLSClientConfig{ - Insecure: apiService.Spec.InsecureSkipTLSVerify, - ServerName: apiService.Spec.Service.Name + "." + apiService.Spec.Service.Namespace + ".svc", - CertData: proxyClientCert, - KeyData: proxyClientKey, - CAData: apiService.Spec.CABundle, - }, + clientConfig := &restclient.Config{ + TLSClientConfig: restclient.TLSClientConfig{ + Insecure: apiService.Spec.InsecureSkipTLSVerify, + ServerName: apiService.Spec.Service.Name + "." + apiService.Spec.Service.Namespace + ".svc", + CertData: proxyClientCert, + KeyData: proxyClientKey, + CAData: apiService.Spec.CABundle, }, + } + clientConfig.Wrap(x509metrics.NewMissingSANRoundTripperWrapperConstructor(x509MissingSANCounter)) + + newInfo := proxyHandlingInfo{ + name: apiService.Name, + restConfig: clientConfig, serviceName: apiService.Spec.Service.Name, serviceNamespace: apiService.Spec.Service.Namespace, servicePort: *apiService.Spec.Service.Port, diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy_test.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy_test.go index 95282152b85..68a88a8696d 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy_test.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy_test.go @@ -21,7 +21,6 @@ import ( "crypto/x509" "fmt" "io/ioutil" - "k8s.io/apiserver/pkg/server/dynamiccertificates" "net/http" "net/http/httptest" "net/http/httputil" @@ -33,6 +32,8 @@ import ( "sync/atomic" "testing" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "golang.org/x/net/websocket" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,6 +41,8 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/user" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" apiregistration "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "k8s.io/utils/pointer" ) @@ -99,22 +102,14 @@ func emptyCert() []byte { } func TestProxyHandler(t *testing.T) { - target := &targetHTTPHandler{} - targetServer := httptest.NewUnstartedServer(target) - if cert, err := tls.X509KeyPair(svcCrt, svcKey); err != nil { - t.Fatal(err) - } else { - targetServer.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} - } - targetServer.StartTLS() - defer targetServer.Close() - tests := map[string]struct { user user.Info path string apiService *apiregistration.APIService - serviceResolver ServiceResolver + serviceResolver ServiceResolver + serviceCertOverride []byte + increaseSANWarnCounter bool expectedStatusCode int expectedBody string @@ -271,12 +266,50 @@ func TestProxyHandler(t *testing.T) { }, expectedStatusCode: http.StatusServiceUnavailable, }, + "fail on bad serving cert w/o SAN and increase SAN error counter metrics": { + user: &user.DefaultInfo{ + Name: "username", + Groups: []string{"one", "two"}, + }, + path: "/request/path", + apiService: &apiregistration.APIService{ + ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"}, + Spec: apiregistration.APIServiceSpec{ + Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: pointer.Int32Ptr(443)}, + Group: "foo", + Version: "v1", + CABundle: testCACrt, + }, + Status: apiregistration.APIServiceStatus{ + Conditions: []apiregistration.APIServiceCondition{ + {Type: apiregistration.Available, Status: apiregistration.ConditionTrue}, + }, + }, + }, + serviceCertOverride: svcCrtNoSAN, + increaseSANWarnCounter: true, + expectedStatusCode: http.StatusServiceUnavailable, + }, } + target := &targetHTTPHandler{} for name, tc := range tests { target.Reset() func() { + targetServer := httptest.NewUnstartedServer(target) + serviceCert := tc.serviceCertOverride + if serviceCert == nil { + serviceCert = svcCrt + } + if cert, err := tls.X509KeyPair(serviceCert, svcKey); err != nil { + t.Fatal(err) + } else { + targetServer.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + } + targetServer.StartTLS() + defer targetServer.Close() + serviceResolver := tc.serviceResolver if serviceResolver == nil { serviceResolver = &mockedRouter{destinationHost: targetServer.Listener.Addr().String()} @@ -331,6 +364,16 @@ func TestProxyHandler(t *testing.T) { t.Errorf("%s: expected %v, got %v", name, e, a) return } + + if tc.increaseSANWarnCounter { + errorCounter := getSingleCounterValueFromRegistry(t, legacyregistry.DefaultGatherer, "apiserver_kube_aggregator_x509_missing_san_total") + if errorCounter == -1 { + t.Errorf("failed to get the x509_missing_san_total metrics: %v", err) + } + if int(errorCounter) != 1 { + t.Errorf("expected the x509_missing_san_total to be 1, but it's %d", errorCounter) + } + } }() } } @@ -468,71 +511,124 @@ func TestProxyUpgrade(t *testing.T) { } var testCACrt = []byte(`-----BEGIN CERTIFICATE----- -MIICxDCCAaygAwIBAgIBATANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDEwd0ZXN0 -LWNhMCAXDTE3MDcyMDIxMTc1MloYDzIxMTcwNjI2MjExNzUzWjASMRAwDgYDVQQD -Ewd0ZXN0LWNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuv/sT2xH -VS1/uXVNAEIwvEb2yTMbXwP6FD38LWkc37Ri7YMB9xiXEDBrbr6K1JThsqyitBxU -22QIl53LUm6I7c/vej1tdYtE2rDVuviiiRgy6omR8imVSv9vU024rgDe+nC9zTT1 -3aNKR03olCG6fkygdcZOghzlyQLhyh8LG75XdnLNksnakum2dNxQ5QIFmBKAuev3 -A069oRMNjudot+t/nFP9UDZ8dL80PNTNPF22bPsnxiau7KLZ4I0Lf7gt6yHlNcue -Fd5sqzqsw/LUFJR5Xuo1+0e7NV3SwCH5CymG6hkboM4Rf5S3EDDyXTxPbXzbQHf1 -7ksW6gjAxh4x/wIDAQABoyMwITAOBgNVHQ8BAf8EBAMCAqQwDwYDVR0TAQH/BAUw -AwEB/zANBgkqhkiG9w0BAQsFAAOCAQEATgmDrW1BjFp+Vmw6T+ojVK4lJuIoerGw -TCCqabHs6O1iWkNi5KsY6vV86tofBIEXsf6S3mV2jcBn87+CIbNHlHFKrXwmcydA -WOc0LWVqqoeqIvEcMNoWQskzmOOUDTanX9mXkirm8d8BljC351TH17rSjLGzFuNh -Cy48xyKFM7kPauNZGfCyaZsGbNJP3Keeu35dOLZMDdBJw7ZvYEUqX7MLOO+d7vlO -JGNA5jsU2uBnSo6qsjxfsbGemk2uRO0nLhplWurw+4qzA79D0lKNLtH9yTn12KZb -/kUpsOSCtLomjWwp67lQyA/yFXf897pSKMXbnIfZfIlDg51CI3U2Sw== +MIIDGTCCAgGgAwIBAgIUAlOGbZ9MSBRFDMq483nGW7h4YNIwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxNDI4MDVa +GA8yMjk0MDcyMzE0MjgwNVowGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANQF9aox1wJlB7wrFeEDYlRk +2AIfC28PZYjW3LsW7/gas2ImmRpzdZYq3nNFQwF67sUudeuuNNAvEngb8Q1wojG7 +Uftt52c9e0Hi5LDxElWV3Tw1XyZFJsk5uwVNb377r7CDfTX3WUsX1WlUeUF6xmwE +M4jYQJ9pMPNUOEWpe7G8daTYineTVvrHvGpxVMMSpOWTWy4+oqWaz5tfFSbyvNZT ++eOLNkDo441KfXvb66zWV4AEfB2QDyGGMuPUT/FgsZHNuj/WNjt3bWvyey9ZGlDm +LPnJgbzEP1FnfIdtuSpHhbWox2Jnuht4hCwhTW1lcAi68MSQEs8KqptEhIJoIxkC +AwEAAaNTMFEwHQYDVR0OBBYEFJnGJQd3VkQP5cZLB1n9/FRKyBLPMB8GA1UdIwQY +MBaAFJnGJQd3VkQP5cZLB1n9/FRKyBLPMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBALwqR2oo3v5Ghs9hS1YQIqegQ/IGZqQwiRz2HFTUGzf5+nUY +BpZHQPapLJ6Ki687rY4nkdEAMCeZjefBFc6uawo6rY4O8IiJAQbDprNNK8oerwiM +BWSDDDjoNxMZMCegSAv39YSonecKZsg7+l1K/nmuQNehgHNem71ZroaRCFvJJ59E +WSd3QP+Gh9iKabsDnkBrTk5KFa7X24c43DJ23kPE49NOwBhiM6Fs8q+tdzWzaVSb +56uXONZxYmFH5yDFvnBIqk2Fys5Klsn6IsM1BCgH2snbA6kwh9Kph4pLdAVGyR9i +MxfBxx4eUypOzIBGqa3OmvMcuNElBe8fcUtpqO0= -----END CERTIFICATE-----`) +/* testCAKey +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA1AX1qjHXAmUHvCsV4QNiVGTYAh8Lbw9liNbcuxbv+BqzYiaZ +GnN1lirec0VDAXruxS5166400C8SeBvxDXCiMbtR+23nZz17QeLksPESVZXdPDVf +JkUmyTm7BU1vfvuvsIN9NfdZSxfVaVR5QXrGbAQziNhAn2kw81Q4Ral7sbx1pNiK +d5NW+se8anFUwxKk5ZNbLj6ipZrPm18VJvK81lP544s2QOjjjUp9e9vrrNZXgAR8 +HZAPIYYy49RP8WCxkc26P9Y2O3dta/J7L1kaUOYs+cmBvMQ/UWd8h225KkeFtajH +Yme6G3iELCFNbWVwCLrwxJASzwqqm0SEgmgjGQIDAQABAoIBAGRWua8kzRMWCvYT +EdSeDF/SJaPDW17g03VR8b4cmc45nKEbkSNCduhtOz8kDRTbP7pTRX0WwWmwjTYI +SyjIIAoXEzJBDdz+7KD+pqnSPJICTWPcAj6TRUq/pnFY9yYKKFgJsizi9QAjtFyX +nJbPaq3dwyHE7bhDSOYu+j6FecNfhqrvj1JbRvIhllKaZJC6He3mNCkHeHtW6ZFk +qJJzWQtPFwqT7tsYCJikwUcQs6QqhD+pPnTYlAkBf24z8ByR8ET7vvOBpH53ufG/ ++gv1K1H+JXQhyO8p4ga4/DdWl+qZoQyTDzm0wy/q+lo/w/pzdIVtgUmguVk/qXad +Bgb6ie0CgYEA/StHVneYIIBmHNdz2fPMMbMzv99/ngDN+ed6Swj+aZKeRgf+QljU +QScwRHUlGKvrFE7Dq/TXVEYO9ksC7tEpIQaSKUlHLBJhhCo8YvfLvH6zoF3F1W6d +7a0ZyXhCWEp5NaNhKdRUMNVXt5H5jf5IGcGXgsErDAStJhiBW/+D0mcCgYEA1mTl +qjclhUr1Ef+wu2N2kNhi8NnScapC1fKjqCzlGcT74lB0BZpizY/hsxSlrtXZE0jI +DhrpiYaxKx4G/Ktr6cu4u0V6sYLH36+wbxmSV2XokHhUfPXZYZNO6+s7mHa8P10N +byTvocQzDhRwN3aD0d32/f8FFvPCZrg2MKAB7n8CgYEArLvZqZJhtlNE2IrcHaos ++QAG3/QzE3ADGW4pT4bsZsXFvYx4m3YWI/oEAcFXtTSfaTSwZuPgAzzlun/FmYIW +KNVd5lN7/wLvjAhxOSlO1eYw0ssITy5xDJhdjsvBoJH3j3RQuASKCOOXPMWZWptT +QFeI84quvz11kheIM2fr3iMCgYApKPnGsgusCXX/XJ1rfG744/Iq10bFt7BZLtoo +oWXiiqTpEBUWNkudt2/XV7FvXXLtdt2hh50qYAeHhZ5FyAtRuWDf4zjo93i0AyDW +U4x65v+9LLzbuL9hMkzGkkTAwprld1Hq8qZm4ioDG/1nSIOKORkALoOlomrCGb+d +mjqEtQKBgQCwtZ7yWxDn/dHeO32VBZOR2YZutOc61BtQHdMqoYsk7qR2ixxVG2bb +1jTedAqac+x0HnJ6au5jdbv0Z95cyyX22MMWaW/H/LNMLxL85OaZfiqjVnntzHcK +jHXdYlJHC8Eslr3iUvRUodgRwOB8c4wWF7s5b6mxGqoXgsNsLrOUPw== +-----END RSA PRIVATE KEY----- +*/ + // valid for hostname test-service.test-ns.svc // signed by testCACrt var svcCrt = []byte(`-----BEGIN CERTIFICATE----- -MIIDDDCCAfSgAwIBAgIBBDANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDEwd0ZXN0 -LWNhMCAXDTE3MDcyMDIxMjAzN1oYDzIxMTcwNjI2MjEyMDM4WjAjMSEwHwYDVQQD -Exh0ZXN0LXNlcnZpY2UudGVzdC1ucy5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IB -DwAwggEKAoIBAQDOKgoTmlVeDhImiBLBccxdniKkS+FZSaoAEtoTvJG1wjk0ewzF -vKhjbHolJ+/qEANiQ6CpTz4hU3m/Iad6IrnmKd1jnkh9yKEaU32B2Xbh6VaV7Sca -Hv4cKWTe50sBvufZinTT8hlFcGufFlJIOLXya5t6HH1Ld7Xf2qwNqusHdmFlJko7 -0By8jhTtD7+2OAJsIPQDWfAsXxFa6LeQ/lqS2DCFnp45DirTNetXoIH8ZJvTBjak -bQuAAA3H+61gRm1blIu8/JjHYTDOcUe5pFyrFLFPgA+eIcpIbzTD61UTNhVlusV2 -eRrBr5BlRM13Zj6ZMcWp0Iiw5QI/W9QU7O4jAgMBAAGjWjBYMA4GA1UdDwEB/wQE -AwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMCMGA1UdEQQc -MBqCGHRlc3Qtc2VydmljZS50ZXN0LW5zLnN2YzANBgkqhkiG9w0BAQsFAAOCAQEA -kpULlml6Ct0cjOuHgDKUnTboFTUm2FHJY27p4NXUvXUMoipg/eSxk0r5JynzXrPa -jaJfY2bC45ixLjJv9irp9ER/lGYUcBQ8OHouXy+uKtA5YKHI3wYf8ITZrQwzRpsK -C5v7qDW+iyb9dn4T6qgpZvylKtkH5cH31hQooNgjZd5aEq3JSFnCM12IVqs/5kjL -NnbPXzeBQ9CHbC+mx7Mm6eSQVtGcQOy4yXFrh2/vrIB2t4gNeWaI1b+7l4MaJjV/ -kRrOirhZaJ90ow/PdYrILtEAdpeC/2Varpr3l4rIKhkkop4gfPwaFeWhG38shH3E -eG5PW2waPpxJzEnGBoAWxQ== +MIIDMjCCAhqgAwIBAgIUEBND1EVKxjU7UaJ1ZBw1glkXuaowDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxNDI4MDVa +GA8yMjk0MDcyMzE0MjgwNVowIzEhMB8GA1UEAwwYdGVzdC1zZXJ2aWNlLnRlc3Qt +bnMuc3ZjMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvDXYxvaUdbX/ +MA3+3SdYY4o8Jl2s1PW9MX4Mr/nCNltyOKDgfSABCN4XVsrd+/A+/zQt+EyJEJxM +rd1syhzd/TJAnGzexmZg/dIi0jC3oBe/qyERWimZhqbu0O+0EpFx5qLzQ5eLabLU +9CtBwRSyYQjqsDmPoqplsKxaFF9NIFQrh1zmxBay9vTY7P7sLkfZ8LifP6jgQ5NH +QkjaY9XCMzYbcrzbc2r9vxTm//IR1cWxaifTNE9qo2NL1iiPGTpot65z83BWeu/q +WOU+aGUhY/xcZH0w/rUJ7ffviyd94EY4IN7FUJv53EJgmEp4UOaY1fAFtAFQQbVz +tGjYGpZ22wIDAQABo2QwYjAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DAdBgNVHSUE +FjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwKQYDVR0RBCIwIIcEfwAAAYIYdGVzdC1z +ZXJ2aWNlLnRlc3QtbnMuc3ZjMA0GCSqGSIb3DQEBCwUAA4IBAQCw/EoFXFahLC4g +4iq9VWhnCmAqUv6IuJqOMC+qEH7fSB3UDAjL4A2iuNJaBAxhI2bccoP2wtqZCkHH +0YLyoKOPjgl6VZtByco8Su7T9yOaef6aX1OP4Snm/aeYdVbjSBKVwMywmmb34XFa +azChi6sq4TFPNesUUoEGkKErU+XG/ecp9Obc0DK/3AAVx/Fk8W5104m1i9PWlUZ2 +KlyxQ5F2alBRv9csIpl2syWQ90DMSQ1Y/R8b+kfsBG7RwDbmwGpZLQTwhE8Uga9T +ZDnmwjUmWn7SD3ouyBSnbWkLE1KcbB32mz5jrwfKCPIa5ka+GIFrme1HxRoQziGo +w+KU2RWu +-----END CERTIFICATE-----`) + +var svcCrtNoSAN = []byte(`-----BEGIN CERTIFICATE----- +MIIDBzCCAe+gAwIBAgIUEBND1EVKxjU7UaJ1ZBw1glkXuaswDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQd2ViaG9va190ZXN0c19jYTAgFw0yMDEwMDcxNDI4MDVa +GA8yMjk0MDcyMzE0MjgwNVowIzEhMB8GA1UEAwwYdGVzdC1zZXJ2aWNlLnRlc3Qt +bnMuc3ZjMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvDXYxvaUdbX/ +MA3+3SdYY4o8Jl2s1PW9MX4Mr/nCNltyOKDgfSABCN4XVsrd+/A+/zQt+EyJEJxM +rd1syhzd/TJAnGzexmZg/dIi0jC3oBe/qyERWimZhqbu0O+0EpFx5qLzQ5eLabLU +9CtBwRSyYQjqsDmPoqplsKxaFF9NIFQrh1zmxBay9vTY7P7sLkfZ8LifP6jgQ5NH +QkjaY9XCMzYbcrzbc2r9vxTm//IR1cWxaifTNE9qo2NL1iiPGTpot65z83BWeu/q +WOU+aGUhY/xcZH0w/rUJ7ffviyd94EY4IN7FUJv53EJgmEp4UOaY1fAFtAFQQbVz +tGjYGpZ22wIDAQABozkwNzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DAdBgNVHSUE +FjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggEBAMPhbecq +wJtlKnSe27xQIM1bNkI/+r1aVmuJqYYbtzCaVZFnFRD6ZbCLfEo7QT17gs7ulryI +yfeITEMAWG6Bq8cOhNQfXRIf2YMFHbDsFbfAEREy/jfYGw8G4b6RBVQzcuglCCB/ +Y0++skz8kYIR1KuZnCtC6A0kaM2XrTWCXAc5KB0Q/WO0wqqWbH/xmEYQVZmDqWOH +k+qVFD+I1oT5NOzFpzaUe4T7grzoLs24IE0c+0clcc9pxTDXTfPyoLG9n3zxG0Ma +hPtkUeeEK8p73Zf/F4JHQ4tJv5XY1ytWkTROE79P6qT0BY/XZSpsGmB7TIS7wFCW +RfKAqN95Uso3IBI= -----END CERTIFICATE-----`) var svcKey = []byte(`-----BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEAzioKE5pVXg4SJogSwXHMXZ4ipEvhWUmqABLaE7yRtcI5NHsM -xbyoY2x6JSfv6hADYkOgqU8+IVN5vyGneiK55indY55IfcihGlN9gdl24elWle0n -Gh7+HClk3udLAb7n2Yp00/IZRXBrnxZSSDi18mubehx9S3e139qsDarrB3ZhZSZK -O9AcvI4U7Q+/tjgCbCD0A1nwLF8RWui3kP5aktgwhZ6eOQ4q0zXrV6CB/GSb0wY2 -pG0LgAANx/utYEZtW5SLvPyYx2EwznFHuaRcqxSxT4APniHKSG80w+tVEzYVZbrF -dnkawa+QZUTNd2Y+mTHFqdCIsOUCP1vUFOzuIwIDAQABAoIBABiX9z/DZ2+i6hNi -pCojcyev154V1zoZiYgct5snIZK3Kq/SBgIIsWW66Q9Jplsbseuk+aN46oZ7OMjO -MPZm8ho84EYj+a3XozBKyWwWDxKADW4xLjr1e4bMgVX97Xq11V6kH6+w78bS1GPT -+9jVuw7CO3fjsiawjye3JFM1Enh/NeRLEpT/oaQoWIV8b0IQB0VyqrdxWOO0rQhd -xA5w39tAZPDQ79MbMQyNWtPgBy0FuulP0GB12PrEbE+SXxsFhWViEwdB5Qx6Gqsx -KGn9vB1oaeSuuKIAjyBV0rXszrGektorDchsOY9UQi1mQsPSvvRFTM9T3qqSFIpu -oPNQLvECgYEA3ox3WJGjEve6VI4RMRt0l6ZFswNbNaHcTMPVsayqsl9KfebG+uyn -Z7TyyoCRzZZQa+3Z9jjW3hAGM9e7MG8jkeHbZpJpZv9X7eB3dgq3eZ1Zt5dyoDrU -PTdIPA2efFAf6V1ejyqH9h6RPQMeAb4uFU9nbI4rPagMxRdp5qIveIUCgYEA7Scb -0zWplDit4EUo+Fq80wzItwJZv64my8KIkEPpW3Fu6UPQvY74qyhE2fCSCwHqRpYJ -jVylyE0GIMx42kjwBgOpi4yEg8M3uMTal+Iy9SgrxZ5cPetaFpEF3Wk7/tz6ppr+ -wnZQTO2WH3YLzv7JIWVrOKuBNVfNEbguVFWw4IcCgYB54mp2uoSancySBKDLyWKo -r6raqQrqK7TQ4iyGO6/dMy1EGQF/ad8hgEu8tn+kHh/7jG/kVyruwc3z1MIze5r6 -ib00xxktDMnmgRpMLwBffdsmHq7rrGyS/lT0du0G3ocrszRXqo5+MC2RQcTMZZEt -oKhfHtn10bT0uKcKZmcjVQKBgEls2WWccMOuhM8yOowic+IYTDC1bpo1Tle6BFQ+ -YoroZQGd+IwoLv+3ORINNPppfmKaY5y7+aw5hNM025oiCQajraPCPukY0TDI6jEq -XMKgzGSkMkUNkFf6UMmLooK3Yneg94232gbnbJqTDvbo1dccMoVaPGgKpjh9QQLl -gR0TAoGACFOvhl8txfbkwLeuNeunyOPL7J4nIccthgd2ioFOr3HTou6wzN++vYTa -a3OF9jH5Z7m6X1rrwn6J1+Gw9sBme38/GeGXHigsBI/8WaTvyuppyVIXOVPoTvVf -VYsTwo5YgV1HzDkV+BNmBCw1GYcGXAElhJI+dCsgQuuU6TKzgl8= +MIIEpAIBAAKCAQEAvDXYxvaUdbX/MA3+3SdYY4o8Jl2s1PW9MX4Mr/nCNltyOKDg +fSABCN4XVsrd+/A+/zQt+EyJEJxMrd1syhzd/TJAnGzexmZg/dIi0jC3oBe/qyER +WimZhqbu0O+0EpFx5qLzQ5eLabLU9CtBwRSyYQjqsDmPoqplsKxaFF9NIFQrh1zm +xBay9vTY7P7sLkfZ8LifP6jgQ5NHQkjaY9XCMzYbcrzbc2r9vxTm//IR1cWxaifT +NE9qo2NL1iiPGTpot65z83BWeu/qWOU+aGUhY/xcZH0w/rUJ7ffviyd94EY4IN7F +UJv53EJgmEp4UOaY1fAFtAFQQbVztGjYGpZ22wIDAQABAoIBAD7Wl5buUujuJ9Jq +idJaxZcOW0DP+9lqZo10sVW7xM0TQRKJHAqKue21AQPYXb81GkNor4R8QTMLjEps +aFsewjs8IPhZHRQOsIluNHQLEfPgmfzP4JRC2WBsscWOkoe0idvgQeoqWcCjlZgk +LSMC/v+I05qczUkZLTSMhtLQcta80OxU99kNU8Kfi6NFiAioqVQl4KlczjLJiUbK +3RGOqThtjS0IzXXFr+T+bgxQkmkyAPGmx06OqqM8hdA+6WsRb8LS1XfK7qGWbU0T +7mIehkcMFDRgxlDh4JfCQzWuLTax3Ds8BApJwZCBEQz8T+FbVWJpBwezyhaKBOis +nQmtw8ECgYEA3E+mANY6YNVfFztMpjfh57dY2DLZY9h1yHRK13FM7EK0Z8GgMji6 +kDIubUBta19g3+YI4qIJgvS527ipVEHW0lYUIQ3q+JnafTC7mMxT+2J/j+lrZhrw +aIPxZML29iEm64Wr3mCmUU98iy5z7EUqqKTNwr03f2eSBeO/xn6VtrsCgYEA2rL4 +tOJMoMDfQzAe7KIqEUn2Ob0nYP/MJZ1I8wrrdGMDhp4xofr+m99++uFPqm5u5uI5 +cJ6+xZQ1A6CJSKWtzOALsKN1xx+JJh9Wo2vUliDomKtarFiQO+ONLpnjuSraDMWY +cKx6eXqqgit5hlQeCva2cbUP1De++3RhEpC6DmECgYA8kCiyUjH6LK3XVRXdG7+e +U2i5BkF8kSTP1ig80Yiz6iJt42yGYdHnkePxZKSvv6iB5FrM8n5q4Zu2Ky1hXDgR +2lfuPkU50hGeGKd5ebIciRdIGILNrton4R2a9X2ua66nUDfPCgKul4tFN5/mc50m +fyeRQTLgczhRJiqyBlphwQKBgQDTnjBIH12Ug2zF/688vGHGXvIRxrVvB7XLg9lN +y/gvo4uK3FIccdmijG27Zv+GY9uOL8Ly9biVSKbPvqx4jlCRmQ3WuyTBLAOyzsov +0axgJLHM4KoZcI0IVlSLjj8rMorRpvWtuUe9enO5B0ZNM+HqK/Y4KsKJT/POLzur +Ej3moQKBgQC+RWcly9opx0We4LG0lcdG3V0cawDRP2MmLbxHA/kSuGf5aBMJoCdf +f0vRPPCK7dpPGOX9x8Oz7K7QiOEvFL3Mv1sWBEnl5lSkK8gdBhi6St9RRBGimt2H +S+8g5OWupiWGF6qN+XX5WgYyuipW8mVRaROj8Vyl7JSiwu6KHfZ8RQ== -----END RSA PRIVATE KEY-----`) func TestGetContextForNewRequest(t *testing.T) { @@ -691,6 +787,27 @@ func writeCerts(certFile, keyFile string, certContent, keyContent []byte, t *tes } } +func getSingleCounterValueFromRegistry(t *testing.T, r metrics.Gatherer, name string) int { + mfs, err := r.Gather() + if err != nil { + t.Logf("failed to gather local registry metrics: %v", err) + return -1 + } + + for _, mf := range mfs { + if mf.Name != nil && *mf.Name == name { + mfMetric := mf.GetMetric() + for _, m := range mfMetric { + if m.GetCounter() != nil { + return int(m.GetCounter().GetValue()) + } + } + } + } + + return -1 +} + // cert and ca for client auth var clientCert = []byte(`-----BEGIN CERTIFICATE----- MIIFaDCCA1ACAWUwDQYJKoZIhvcNAQEFBQAwejELMAkGA1UEBhMCVVMxEzARBgNV diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/metrics.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/metrics.go new file mode 100644 index 00000000000..ba25750ec9a --- /dev/null +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/metrics.go @@ -0,0 +1,39 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiserver + +import ( + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +var x509MissingSANCounter = metrics.NewCounter( + &metrics.CounterOpts{ + Subsystem: "kube_aggregator", + Namespace: "apiserver", + Name: "x509_missing_san_total", + Help: "Counts the number of requests to servers missing SAN extension " + + "in their serving certificate OR the number of connection failures " + + "due to the lack of x509 certificate SAN extension missing " + + "(either/or, based on the runtime environment)", + StabilityLevel: metrics.ALPHA, + }, +) + +func init() { + legacyregistry.MustRegister(x509MissingSANCounter) +} From 3ff82908005a949f396ef241633657f2a4e27b1f Mon Sep 17 00:00:00 2001 From: Stanislav Laznicka Date: Mon, 19 Oct 2020 14:07:07 +0200 Subject: [PATCH 2/2] vendor update --- vendor/modules.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/vendor/modules.txt b/vendor/modules.txt index 2749d593e88..864ce684476 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1885,6 +1885,7 @@ k8s.io/apiserver/pkg/util/proxy k8s.io/apiserver/pkg/util/shufflesharding k8s.io/apiserver/pkg/util/webhook k8s.io/apiserver/pkg/util/wsstream +k8s.io/apiserver/pkg/util/x509metrics k8s.io/apiserver/pkg/warning k8s.io/apiserver/plugin/pkg/audit/buffered k8s.io/apiserver/plugin/pkg/audit/log