From f02008338f261e689d625bda469b0617766cf64c Mon Sep 17 00:00:00 2001 From: Haoran Wang Date: Mon, 17 Jul 2017 19:35:27 +0800 Subject: [PATCH] add integration testing for bootstrap token auth --- hack/.golint_failures | 1 + pkg/api/types.go | 4 + test/e2e/BUILD | 3 +- test/e2e/e2e_test.go | 2 +- test/e2e/lifecycle/BUILD | 5 +- test/e2e/{ => lifecycle}/bootstrap/BUILD | 3 +- test/e2e/{ => lifecycle}/bootstrap/OWNERS | 0 .../bootstrap/bootstrap_signer.go | 11 +- .../bootstrap/bootstrap_token_cleaner.go | 17 +- test/e2e/{ => lifecycle}/bootstrap/util.go | 12 +- test/integration/auth/BUILD | 5 + test/integration/auth/bootstraptoken_test.go | 188 ++++++++++++++++++ test/integration/utils.go | 1 + 13 files changed, 228 insertions(+), 24 deletions(-) rename test/e2e/{ => lifecycle}/bootstrap/BUILD (90%) rename test/e2e/{ => lifecycle}/bootstrap/OWNERS (100%) rename test/e2e/{ => lifecycle}/bootstrap/bootstrap_signer.go (94%) rename test/e2e/{ => lifecycle}/bootstrap/bootstrap_token_cleaner.go (84%) rename test/e2e/{ => lifecycle}/bootstrap/util.go (94%) create mode 100644 test/integration/auth/bootstraptoken_test.go diff --git a/hack/.golint_failures b/hack/.golint_failures index 74f1571a0be..34d514c4847 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -811,6 +811,7 @@ test/e2e/instrumentation/logging test/e2e/instrumentation/monitoring test/e2e/kubectl test/e2e/lifecycle +test/e2e/lifecycle/bootstrap test/e2e/metrics test/e2e/scalability test/e2e/scheduling diff --git a/pkg/api/types.go b/pkg/api/types.go index 6a81d43a81c..8dcd21e1580 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -3868,6 +3868,10 @@ const ( TLSCertKey = "tls.crt" // TLSPrivateKeyKey is the key for the private key field in a TLS secret. TLSPrivateKeyKey = "tls.key" + // SecretTypeBootstrapToken is used during the automated bootstrap process (first + // implemented by kubeadm). It stores tokens that are used to sign well known + // ConfigMaps. They are used for authn. + SecretTypeBootstrapToken SecretType = "bootstrap.kubernetes.io/token" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 9e6b6167fa6..7618d46bf26 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -20,12 +20,12 @@ go_test( deps = [ "//test/e2e/apimachinery:go_default_library", "//test/e2e/autoscaling:go_default_library", - "//test/e2e/bootstrap:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/instrumentation/logging:go_default_library", "//test/e2e/instrumentation/monitoring:go_default_library", "//test/e2e/kubectl:go_default_library", "//test/e2e/lifecycle:go_default_library", + "//test/e2e/lifecycle/bootstrap:go_default_library", "//test/e2e/metrics:go_default_library", "//test/e2e/scalability:go_default_library", "//test/e2e/scheduling:go_default_library", @@ -174,7 +174,6 @@ filegroup( ":package-srcs", "//test/e2e/apimachinery:all-srcs", "//test/e2e/autoscaling:all-srcs", - "//test/e2e/bootstrap:all-srcs", "//test/e2e/chaosmonkey:all-srcs", "//test/e2e/common:all-srcs", "//test/e2e/framework:all-srcs", diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 55f2f611c74..ae07686919e 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -21,12 +21,12 @@ import ( _ "k8s.io/kubernetes/test/e2e/apimachinery" _ "k8s.io/kubernetes/test/e2e/autoscaling" - _ "k8s.io/kubernetes/test/e2e/bootstrap" "k8s.io/kubernetes/test/e2e/framework" _ "k8s.io/kubernetes/test/e2e/instrumentation/logging" _ "k8s.io/kubernetes/test/e2e/instrumentation/monitoring" _ "k8s.io/kubernetes/test/e2e/kubectl" _ "k8s.io/kubernetes/test/e2e/lifecycle" + _ "k8s.io/kubernetes/test/e2e/lifecycle/bootstrap" _ "k8s.io/kubernetes/test/e2e/scalability" _ "k8s.io/kubernetes/test/e2e/scheduling" _ "k8s.io/kubernetes/test/e2e/storage" diff --git a/test/e2e/lifecycle/BUILD b/test/e2e/lifecycle/BUILD index d4c74b960b1..717e3f47902 100644 --- a/test/e2e/lifecycle/BUILD +++ b/test/e2e/lifecycle/BUILD @@ -54,6 +54,9 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//test/e2e/lifecycle/bootstrap:all-srcs", + ], tags = ["automanaged"], ) diff --git a/test/e2e/bootstrap/BUILD b/test/e2e/lifecycle/bootstrap/BUILD similarity index 90% rename from test/e2e/bootstrap/BUILD rename to test/e2e/lifecycle/bootstrap/BUILD index b41abbed7af..ca411e4a154 100644 --- a/test/e2e/bootstrap/BUILD +++ b/test/e2e/lifecycle/bootstrap/BUILD @@ -17,14 +17,15 @@ go_library( tags = ["automanaged"], deps = [ "//pkg/bootstrap/api:go_default_library", - "//pkg/client/clientset_generated/clientset:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/lifecycle:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", ], ) diff --git a/test/e2e/bootstrap/OWNERS b/test/e2e/lifecycle/bootstrap/OWNERS similarity index 100% rename from test/e2e/bootstrap/OWNERS rename to test/e2e/lifecycle/bootstrap/OWNERS diff --git a/test/e2e/bootstrap/bootstrap_signer.go b/test/e2e/lifecycle/bootstrap/bootstrap_signer.go similarity index 94% rename from test/e2e/bootstrap/bootstrap_signer.go rename to test/e2e/lifecycle/bootstrap/bootstrap_signer.go index d6d7d51afad..8b2c7fd7411 100644 --- a/test/e2e/bootstrap/bootstrap_signer.go +++ b/test/e2e/lifecycle/bootstrap/bootstrap_signer.go @@ -21,9 +21,10 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" - "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/lifecycle" ) const ( @@ -31,7 +32,7 @@ const ( TokenSecretBytes = 8 ) -var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { +var _ = lifecycle.SIGDescribe("[Feature:BootstrapTokens]", func() { var c clientset.Interface @@ -50,7 +51,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { It("should sign the new added bootstrap tokens", func() { By("create a new bootstrap token secret") - tokenId, err := generateTokenId() + tokenId, err := GenerateTokenId() Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, "tokenSecret") _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) @@ -65,7 +66,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { It("should resign the bootstrap tokens when the clusterInfo ConfigMap updated [Serial][Disruptive]", func() { By("create a new bootstrap token secret") - tokenId, err := generateTokenId() + tokenId, err := GenerateTokenId() Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, "tokenSecret") secret, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) @@ -102,7 +103,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { It("should delete the signed bootstrap tokens from clusterInfo ConfigMap when bootstrap token is deleted", func() { By("create a new bootstrap token secret") - tokenId, err := generateTokenId() + tokenId, err := GenerateTokenId() Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, "tokenSecret") _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) diff --git a/test/e2e/bootstrap/bootstrap_token_cleaner.go b/test/e2e/lifecycle/bootstrap/bootstrap_token_cleaner.go similarity index 84% rename from test/e2e/bootstrap/bootstrap_token_cleaner.go rename to test/e2e/lifecycle/bootstrap/bootstrap_token_cleaner.go index d7c7f4ad360..0acab639b69 100644 --- a/test/e2e/bootstrap/bootstrap_token_cleaner.go +++ b/test/e2e/lifecycle/bootstrap/bootstrap_token_cleaner.go @@ -23,13 +23,14 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" - "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/lifecycle" ) var secretNeedClean string -var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { +var _ = lifecycle.SIGDescribe("[Feature:BootstrapTokens]", func() { var c clientset.Interface @@ -49,13 +50,13 @@ var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { }) It("should delete the token secret when the secret expired", func() { By("create a new expired bootstrap token secret") - tokenId, err := generateTokenId() + tokenId, err := GenerateTokenId() Expect(err).NotTo(HaveOccurred()) - tokenSecret, err := generateTokenSecret() + tokenSecret, err := GenerateTokenSecret() Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, tokenSecret) - addSecretExpiration(secret, timeStringFromNow(-time.Hour)) + addSecretExpiration(secret, TimeStringFromNow(-time.Hour)) _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) Expect(err).NotTo(HaveOccurred()) @@ -67,12 +68,12 @@ var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { It("should not delete the token secret when the secret is not expired", func() { By("create a new expired bootstrap token secret") - tokenId, err := generateTokenId() + tokenId, err := GenerateTokenId() Expect(err).NotTo(HaveOccurred()) - tokenSecret, err := generateTokenSecret() + tokenSecret, err := GenerateTokenSecret() Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, tokenSecret) - addSecretExpiration(secret, timeStringFromNow(time.Hour)) + addSecretExpiration(secret, TimeStringFromNow(time.Hour)) _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/bootstrap/util.go b/test/e2e/lifecycle/bootstrap/util.go similarity index 94% rename from test/e2e/bootstrap/util.go rename to test/e2e/lifecycle/bootstrap/util.go index 019117cd139..5b3e663af2b 100644 --- a/test/e2e/bootstrap/util.go +++ b/test/e2e/lifecycle/bootstrap/util.go @@ -22,12 +22,12 @@ import ( "errors" "time" - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" - "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" ) @@ -46,14 +46,14 @@ func newTokenSecret(tokenID, tokenSecret string) *v1.Secret { } } -func generateTokenId() (string, error) { +func GenerateTokenId() (string, error) { tokenID, err := randBytes(TokenIDBytes) if err != nil { return "", err } return tokenID, nil } -func generateTokenSecret() (string, error) { +func GenerateTokenSecret() (string, error) { tokenSecret, err := randBytes(TokenSecretBytes) if err != nil { return "", err @@ -74,7 +74,7 @@ func addSecretExpiration(s *v1.Secret, expiration string) { s.Data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expiration) } -func timeStringFromNow(delta time.Duration) string { +func TimeStringFromNow(delta time.Duration) string { return time.Now().Add(delta).Format(time.RFC3339) } @@ -141,7 +141,7 @@ func WaitForBootstrapTokenSecretNotDisappear(c clientset.Interface, tokenID stri err := wait.Poll(framework.Poll, t, func() (bool, error) { secret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(bootstrapapi.BootstrapTokenSecretPrefix+tokenID, metav1.GetOptions{}) if apierrs.IsNotFound(err) { - return true, errors.New("secret not exits") + return true, errors.New("secret not exists") } if secret != nil { return false, nil diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index b825f610265..d58486d18d7 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -13,6 +13,7 @@ go_test( srcs = [ "accessreview_test.go", "auth_test.go", + "bootstraptoken_test.go", "main_test.go", "node_test.go", "rbac_test.go", @@ -30,6 +31,7 @@ go_test( "//pkg/apis/rbac:go_default_library", "//pkg/auth/authorizer/abac:go_default_library", "//pkg/auth/nodeidentifier:go_default_library", + "//pkg/bootstrap/api:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/informers/informers_generated/internalversion:go_default_library", "//pkg/kubeapiserver/authorizer:go_default_library", @@ -44,8 +46,10 @@ go_test( "//pkg/registry/rbac/rolebinding/storage:go_default_library", "//plugin/pkg/admission/admit:go_default_library", "//plugin/pkg/admission/noderestriction:go_default_library", + "//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library", "//plugin/pkg/auth/authorizer/rbac:go_default_library", "//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library", + "//test/e2e/lifecycle/bootstrap:go_default_library", "//test/integration:go_default_library", "//test/integration/framework:go_default_library", "//vendor/github.com/golang/glog:go_default_library", @@ -53,6 +57,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", diff --git a/test/integration/auth/bootstraptoken_test.go b/test/integration/auth/bootstraptoken_test.go new file mode 100644 index 00000000000..ab62aff8156 --- /dev/null +++ b/test/integration/auth/bootstraptoken_test.go @@ -0,0 +1,188 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apiserver/pkg/authentication/request/bearertoken" + "k8s.io/kubernetes/pkg/api" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/plugin/pkg/admission/admit" + "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" + bootstraputil "k8s.io/kubernetes/test/e2e/lifecycle/bootstrap" + "k8s.io/kubernetes/test/integration" + "k8s.io/kubernetes/test/integration/framework" +) + +type bootstrapSecrets []*api.Secret + +func (b bootstrapSecrets) List(selector labels.Selector) (ret []*api.Secret, err error) { + return b, nil +} + +func (b bootstrapSecrets) Get(name string) (*api.Secret, error) { + return b[0], nil +} + +// TestBootstrapTokenAuth tests the bootstrap token auth provider +func TestBootstrapTokenAuth(t *testing.T) { + tokenId, err := bootstraputil.GenerateTokenId() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + secret, err := bootstraputil.GenerateTokenSecret() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + var bootstrapSecretValid = &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: bootstrapapi.BootstrapTokenSecretPrefix, + }, + Type: api.SecretTypeBootstrapToken, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenId), + bootstrapapi.BootstrapTokenSecretKey: []byte(secret), + bootstrapapi.BootstrapTokenUsageAuthentication: []byte("true"), + }, + } + var bootstrapSecretInvalid = &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: bootstrapapi.BootstrapTokenSecretPrefix, + }, + Type: api.SecretTypeBootstrapToken, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenId), + bootstrapapi.BootstrapTokenSecretKey: []byte("invalid"), + bootstrapapi.BootstrapTokenUsageAuthentication: []byte("true"), + }, + } + var expiredBootstrapToken = &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: bootstrapapi.BootstrapTokenSecretPrefix, + }, + Type: api.SecretTypeBootstrapToken, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenId), + bootstrapapi.BootstrapTokenSecretKey: []byte("invalid"), + bootstrapapi.BootstrapTokenUsageAuthentication: []byte("true"), + bootstrapapi.BootstrapTokenExpirationKey: []byte(bootstraputil.TimeStringFromNow(-time.Hour)), + }, + } + type request struct { + verb string + URL string + body string + statusCodes map[int]bool // Set of expected resp.StatusCode if all goes well. + } + tests := []struct { + name string + request request + secret *api.Secret + }{ + { + name: "valid token", + request: request{verb: "GET", URL: path("pods", "", ""), body: "", statusCodes: integration.Code200}, + secret: bootstrapSecretValid, + }, + { + name: "invalid token format", + request: request{verb: "GET", URL: path("pods", "", ""), body: "", statusCodes: integration.Code401}, + secret: bootstrapSecretInvalid, + }, + { + name: "invalid token expired", + request: request{verb: "GET", URL: path("pods", "", ""), body: "", statusCodes: integration.Code401}, + secret: expiredBootstrapToken, + }, + } + for _, test := range tests { + + authenticator := bearertoken.New(bootstrap.NewTokenAuthenticator(bootstrapSecrets{test.secret})) + // Set up a master + masterConfig := framework.NewIntegrationTestMasterConfig() + masterConfig.GenericConfig.Authenticator = authenticator + masterConfig.GenericConfig.AdmissionControl = admit.NewAlwaysAdmit() + _, s, closeFn := framework.RunAMaster(masterConfig) + defer closeFn() + + ns := framework.CreateTestingNamespace("auth-bootstrap-token", s, t) + defer framework.DeleteTestingNamespace(ns, s, t) + + previousResourceVersion := make(map[string]float64) + transport := http.DefaultTransport + + token := tokenId + "." + secret + var bodyStr string + if test.request.body != "" { + sub := "" + if test.request.verb == "PUT" { + // For update operations, insert previous resource version + if resVersion := previousResourceVersion[getPreviousResourceVersionKey(test.request.URL, "")]; resVersion != 0 { + sub += fmt.Sprintf(",\r\n\"resourceVersion\": \"%v\"", resVersion) + } + sub += fmt.Sprintf(",\r\n\"namespace\": %q", ns.Name) + } + bodyStr = fmt.Sprintf(test.request.body, sub) + } + test.request.body = bodyStr + bodyBytes := bytes.NewReader([]byte(bodyStr)) + req, err := http.NewRequest(test.request.verb, s.URL+test.request.URL, bodyBytes) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + if test.request.verb == "PATCH" { + req.Header.Set("Content-Type", "application/merge-patch+json") + } + + func() { + resp, err := transport.RoundTrip(req) + defer resp.Body.Close() + if err != nil { + t.Logf("case %v", test.name) + t.Fatalf("unexpected error: %v", err) + } + b, _ := ioutil.ReadAll(resp.Body) + if _, ok := test.request.statusCodes[resp.StatusCode]; !ok { + t.Logf("case %v", test.name) + t.Errorf("Expected status one of %v, but got %v", test.request.statusCodes, resp.StatusCode) + t.Errorf("Body: %v", string(b)) + } else { + if test.request.verb == "POST" { + // For successful create operations, extract resourceVersion + id, currentResourceVersion, err := parseResourceVersion(b) + if err == nil { + key := getPreviousResourceVersionKey(test.request.URL, id) + previousResourceVersion[key] = currentResourceVersion + } + } + } + + }() + } +} diff --git a/test/integration/utils.go b/test/integration/utils.go index f25e39a3403..629a58a2d87 100644 --- a/test/integration/utils.go +++ b/test/integration/utils.go @@ -38,6 +38,7 @@ func DeletePodOrErrorf(t *testing.T, c clientset.Interface, ns, name string) { var Code200 = map[int]bool{200: true} var Code201 = map[int]bool{201: true} var Code400 = map[int]bool{400: true} +var Code401 = map[int]bool{401: true} var Code403 = map[int]bool{403: true} var Code404 = map[int]bool{404: true} var Code405 = map[int]bool{405: true}