mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-09-14 13:45:06 +00:00
Add NodeRestriction admission plugin
This commit is contained in:
@@ -54,6 +54,7 @@ go_library(
|
|||||||
"//plugin/pkg/admission/namespace/autoprovision:go_default_library",
|
"//plugin/pkg/admission/namespace/autoprovision:go_default_library",
|
||||||
"//plugin/pkg/admission/namespace/exists:go_default_library",
|
"//plugin/pkg/admission/namespace/exists:go_default_library",
|
||||||
"//plugin/pkg/admission/namespace/lifecycle:go_default_library",
|
"//plugin/pkg/admission/namespace/lifecycle:go_default_library",
|
||||||
|
"//plugin/pkg/admission/noderestriction:go_default_library",
|
||||||
"//plugin/pkg/admission/persistentvolume/label:go_default_library",
|
"//plugin/pkg/admission/persistentvolume/label:go_default_library",
|
||||||
"//plugin/pkg/admission/podnodeselector:go_default_library",
|
"//plugin/pkg/admission/podnodeselector:go_default_library",
|
||||||
"//plugin/pkg/admission/podpreset:go_default_library",
|
"//plugin/pkg/admission/podpreset:go_default_library",
|
||||||
|
@@ -37,6 +37,7 @@ import (
|
|||||||
_ "k8s.io/kubernetes/plugin/pkg/admission/namespace/autoprovision"
|
_ "k8s.io/kubernetes/plugin/pkg/admission/namespace/autoprovision"
|
||||||
_ "k8s.io/kubernetes/plugin/pkg/admission/namespace/exists"
|
_ "k8s.io/kubernetes/plugin/pkg/admission/namespace/exists"
|
||||||
_ "k8s.io/kubernetes/plugin/pkg/admission/namespace/lifecycle"
|
_ "k8s.io/kubernetes/plugin/pkg/admission/namespace/lifecycle"
|
||||||
|
_ "k8s.io/kubernetes/plugin/pkg/admission/noderestriction"
|
||||||
_ "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/label"
|
_ "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/label"
|
||||||
_ "k8s.io/kubernetes/plugin/pkg/admission/podnodeselector"
|
_ "k8s.io/kubernetes/plugin/pkg/admission/podnodeselector"
|
||||||
_ "k8s.io/kubernetes/plugin/pkg/admission/podpreset"
|
_ "k8s.io/kubernetes/plugin/pkg/admission/podpreset"
|
||||||
|
@@ -388,6 +388,9 @@ function start_apiserver {
|
|||||||
if [[ -n "${PSP_ADMISSION}" ]]; then
|
if [[ -n "${PSP_ADMISSION}" ]]; then
|
||||||
security_admission=",PodSecurityPolicy"
|
security_admission=",PodSecurityPolicy"
|
||||||
fi
|
fi
|
||||||
|
if [[ -n "${NODE_ADMISSION}" ]]; then
|
||||||
|
security_admission=",NodeRestriction"
|
||||||
|
fi
|
||||||
|
|
||||||
# Admission Controllers to invoke prior to persisting objects in cluster
|
# Admission Controllers to invoke prior to persisting objects in cluster
|
||||||
ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,ServiceAccount${security_admission},ResourceQuota,DefaultStorageClass,DefaultTolerationSeconds
|
ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,ServiceAccount${security_admission},ResourceQuota,DefaultStorageClass,DefaultTolerationSeconds
|
||||||
|
@@ -27,6 +27,7 @@ filegroup(
|
|||||||
"//plugin/pkg/admission/namespace/autoprovision:all-srcs",
|
"//plugin/pkg/admission/namespace/autoprovision:all-srcs",
|
||||||
"//plugin/pkg/admission/namespace/exists:all-srcs",
|
"//plugin/pkg/admission/namespace/exists:all-srcs",
|
||||||
"//plugin/pkg/admission/namespace/lifecycle:all-srcs",
|
"//plugin/pkg/admission/namespace/lifecycle:all-srcs",
|
||||||
|
"//plugin/pkg/admission/noderestriction:all-srcs",
|
||||||
"//plugin/pkg/admission/persistentvolume/label:all-srcs",
|
"//plugin/pkg/admission/persistentvolume/label:all-srcs",
|
||||||
"//plugin/pkg/admission/podnodeselector:all-srcs",
|
"//plugin/pkg/admission/podnodeselector:all-srcs",
|
||||||
"//plugin/pkg/admission/podpreset:all-srcs",
|
"//plugin/pkg/admission/podpreset:all-srcs",
|
||||||
|
54
plugin/pkg/admission/noderestriction/BUILD
Normal file
54
plugin/pkg/admission/noderestriction/BUILD
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
package(default_visibility = ["//visibility:public"])
|
||||||
|
|
||||||
|
licenses(["notice"])
|
||||||
|
|
||||||
|
load(
|
||||||
|
"@io_bazel_rules_go//go:def.bzl",
|
||||||
|
"go_library",
|
||||||
|
"go_test",
|
||||||
|
)
|
||||||
|
|
||||||
|
go_library(
|
||||||
|
name = "go_default_library",
|
||||||
|
srcs = ["admission.go"],
|
||||||
|
tags = ["automanaged"],
|
||||||
|
deps = [
|
||||||
|
"//pkg/api:go_default_library",
|
||||||
|
"//pkg/api/pod:go_default_library",
|
||||||
|
"//pkg/auth/nodeidentifier:go_default_library",
|
||||||
|
"//pkg/client/clientset_generated/internalclientset:go_default_library",
|
||||||
|
"//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library",
|
||||||
|
"//pkg/kubeapiserver/admission:go_default_library",
|
||||||
|
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||||
|
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
go_test(
|
||||||
|
name = "go_default_test",
|
||||||
|
srcs = ["admission_test.go"],
|
||||||
|
library = ":go_default_library",
|
||||||
|
tags = ["automanaged"],
|
||||||
|
deps = [
|
||||||
|
"//pkg/api:go_default_library",
|
||||||
|
"//pkg/auth/nodeidentifier:go_default_library",
|
||||||
|
"//pkg/client/clientset_generated/internalclientset/fake:go_default_library",
|
||||||
|
"//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library",
|
||||||
|
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||||
|
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||||
|
"//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
filegroup(
|
||||||
|
name = "package-srcs",
|
||||||
|
srcs = glob(["**"]),
|
||||||
|
tags = ["automanaged"],
|
||||||
|
visibility = ["//visibility:private"],
|
||||||
|
)
|
||||||
|
|
||||||
|
filegroup(
|
||||||
|
name = "all-srcs",
|
||||||
|
srcs = [":package-srcs"],
|
||||||
|
tags = ["automanaged"],
|
||||||
|
)
|
8
plugin/pkg/admission/noderestriction/OWNERS
Normal file
8
plugin/pkg/admission/noderestriction/OWNERS
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
approvers:
|
||||||
|
- deads2k
|
||||||
|
- liggitt
|
||||||
|
- timstclair
|
||||||
|
reviewers:
|
||||||
|
- deads2k
|
||||||
|
- liggitt
|
||||||
|
- timstclair
|
203
plugin/pkg/admission/noderestriction/admission.go
Normal file
203
plugin/pkg/admission/noderestriction/admission.go
Normal file
@@ -0,0 +1,203 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2017 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package node
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apiserver/pkg/admission"
|
||||||
|
"k8s.io/kubernetes/pkg/api"
|
||||||
|
podutil "k8s.io/kubernetes/pkg/api/pod"
|
||||||
|
"k8s.io/kubernetes/pkg/auth/nodeidentifier"
|
||||||
|
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
|
||||||
|
coreinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
|
||||||
|
kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
PluginName = "NodeRestriction"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
kubeapiserveradmission.Plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||||
|
return NewPlugin(nodeidentifier.NewDefaultNodeIdentifier(), false), nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPlugin creates a new NodeRestriction admission plugin.
|
||||||
|
// This plugin identifies requests from nodes
|
||||||
|
func NewPlugin(nodeIdentifier nodeidentifier.NodeIdentifier, strict bool) *nodePlugin {
|
||||||
|
return &nodePlugin{
|
||||||
|
Handler: admission.NewHandler(admission.Create, admission.Update, admission.Delete),
|
||||||
|
nodeIdentifier: nodeIdentifier,
|
||||||
|
strict: strict,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodePlugin holds state for and implements the admission plugin.
|
||||||
|
type nodePlugin struct {
|
||||||
|
*admission.Handler
|
||||||
|
strict bool
|
||||||
|
nodeIdentifier nodeidentifier.NodeIdentifier
|
||||||
|
podsGetter coreinternalversion.PodsGetter
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
_ = admission.Interface(&nodePlugin{})
|
||||||
|
_ = kubeapiserveradmission.WantsInternalKubeClientSet(&nodePlugin{})
|
||||||
|
)
|
||||||
|
|
||||||
|
func (p *nodePlugin) SetInternalKubeClientSet(f internalclientset.Interface) {
|
||||||
|
p.podsGetter = f.Core()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *nodePlugin) Validate() error {
|
||||||
|
if p.nodeIdentifier == nil {
|
||||||
|
return fmt.Errorf("%s requires a node identifier", PluginName)
|
||||||
|
}
|
||||||
|
if p.podsGetter == nil {
|
||||||
|
return fmt.Errorf("%s requires a pod getter", PluginName)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
podResource = api.Resource("pods")
|
||||||
|
nodeResource = api.Resource("nodes")
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *nodePlugin) Admit(a admission.Attributes) error {
|
||||||
|
nodeName, isNode := c.nodeIdentifier.NodeIdentity(a.GetUserInfo())
|
||||||
|
|
||||||
|
// Our job is just to restrict nodes
|
||||||
|
if !isNode {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(nodeName) == 0 {
|
||||||
|
if c.strict {
|
||||||
|
// In strict mode, disallow requests from nodes we cannot match to a particular node
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("could not determine node identity from user"))
|
||||||
|
}
|
||||||
|
// Our job is just to restrict identifiable nodes
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch a.GetResource().GroupResource() {
|
||||||
|
case podResource:
|
||||||
|
switch a.GetSubresource() {
|
||||||
|
case "":
|
||||||
|
return c.admitPod(nodeName, a)
|
||||||
|
case "status":
|
||||||
|
return c.admitPodStatus(nodeName, a)
|
||||||
|
default:
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("unexpected pod subresource %s", a.GetSubresource()))
|
||||||
|
}
|
||||||
|
|
||||||
|
case nodeResource:
|
||||||
|
return c.admitNode(nodeName, a)
|
||||||
|
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *nodePlugin) admitPod(nodeName string, a admission.Attributes) error {
|
||||||
|
switch a.GetOperation() {
|
||||||
|
case admission.Create:
|
||||||
|
// require a pod object
|
||||||
|
pod, ok := a.GetObject().(*api.Pod)
|
||||||
|
if !ok {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetObject()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// only allow nodes to create mirror pods
|
||||||
|
if _, isMirrorPod := pod.Annotations[api.MirrorPodAnnotationKey]; !isMirrorPod {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("pod does not have %q annotation, node %s can only create mirror pods", api.MirrorPodAnnotationKey, nodeName))
|
||||||
|
}
|
||||||
|
|
||||||
|
// only allow nodes to create a pod bound to itself
|
||||||
|
if pod.Spec.NodeName != nodeName {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("node %s can only create pods with spec.nodeName set to itself", nodeName))
|
||||||
|
}
|
||||||
|
|
||||||
|
// don't allow a node to create a pod that references any other API objects
|
||||||
|
if pod.Spec.ServiceAccountName != "" {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("node %s can not create pods that reference a service account", nodeName))
|
||||||
|
}
|
||||||
|
hasSecrets := false
|
||||||
|
podutil.VisitPodSecretNames(pod, func(name string) (shouldContinue bool) { hasSecrets = true; return false })
|
||||||
|
if hasSecrets {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("node %s can not create pods that reference secrets", nodeName))
|
||||||
|
}
|
||||||
|
hasConfigMaps := false
|
||||||
|
podutil.VisitPodConfigmapNames(pod, func(name string) (shouldContinue bool) { hasConfigMaps = true; return false })
|
||||||
|
if hasConfigMaps {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("node %s can not create pods that reference configmaps", nodeName))
|
||||||
|
}
|
||||||
|
for _, v := range pod.Spec.Volumes {
|
||||||
|
if v.PersistentVolumeClaim != nil {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("node %s can not create pods that reference persistentvolumeclaims", nodeName))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
|
||||||
|
case admission.Delete:
|
||||||
|
// get the existing pod
|
||||||
|
existingPod, err := c.podsGetter.Pods(a.GetNamespace()).Get(a.GetName(), v1.GetOptions{ResourceVersion: "0"})
|
||||||
|
if err != nil {
|
||||||
|
return admission.NewForbidden(a, err)
|
||||||
|
}
|
||||||
|
// only allow a node to delete a pod bound to itself
|
||||||
|
if existingPod.Spec.NodeName != nodeName {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("node %s can only delete pods with spec.nodeName set to itself", nodeName))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
|
||||||
|
default:
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("unexpected operation %s", a.GetOperation()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *nodePlugin) admitPodStatus(nodeName string, a admission.Attributes) error {
|
||||||
|
switch a.GetOperation() {
|
||||||
|
case admission.Update:
|
||||||
|
// require an existing pod
|
||||||
|
pod, ok := a.GetOldObject().(*api.Pod)
|
||||||
|
if !ok {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetObject()))
|
||||||
|
}
|
||||||
|
// only allow a node to update status of a pod bound to itself
|
||||||
|
if pod.Spec.NodeName != nodeName {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("node %s can only update pod status for pods with spec.nodeName set to itself", nodeName))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
|
||||||
|
default:
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("unexpected operation %s", a.GetOperation()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *nodePlugin) admitNode(nodeName string, a admission.Attributes) error {
|
||||||
|
if a.GetName() != nodeName {
|
||||||
|
return admission.NewForbidden(a, fmt.Errorf("cannot modify other nodes"))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
476
plugin/pkg/admission/noderestriction/admission_test.go
Normal file
476
plugin/pkg/admission/noderestriction/admission_test.go
Normal file
@@ -0,0 +1,476 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2017 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package node
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apiserver/pkg/admission"
|
||||||
|
"k8s.io/apiserver/pkg/authentication/user"
|
||||||
|
"k8s.io/kubernetes/pkg/api"
|
||||||
|
"k8s.io/kubernetes/pkg/auth/nodeidentifier"
|
||||||
|
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake"
|
||||||
|
coreinternalversion "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
|
||||||
|
)
|
||||||
|
|
||||||
|
func makeTestPod(namespace, name, node string, mirror bool) *api.Pod {
|
||||||
|
pod := &api.Pod{}
|
||||||
|
pod.Namespace = namespace
|
||||||
|
pod.Name = name
|
||||||
|
pod.Spec.NodeName = node
|
||||||
|
if mirror {
|
||||||
|
pod.Annotations = map[string]string{api.MirrorPodAnnotationKey: "true"}
|
||||||
|
}
|
||||||
|
return pod
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_nodePlugin_Admit(t *testing.T) {
|
||||||
|
var (
|
||||||
|
mynode = &user.DefaultInfo{Name: "system:node:mynode", Groups: []string{"system:nodes"}}
|
||||||
|
bob = &user.DefaultInfo{Name: "bob"}
|
||||||
|
|
||||||
|
mynodeObj = &api.Node{ObjectMeta: metav1.ObjectMeta{Name: "mynode"}}
|
||||||
|
othernodeObj = &api.Node{ObjectMeta: metav1.ObjectMeta{Name: "othernode"}}
|
||||||
|
|
||||||
|
mymirrorpod = makeTestPod("ns", "mymirrorpod", "mynode", true)
|
||||||
|
othermirrorpod = makeTestPod("ns", "othermirrorpod", "othernode", true)
|
||||||
|
unboundmirrorpod = makeTestPod("ns", "unboundmirrorpod", "", true)
|
||||||
|
mypod = makeTestPod("ns", "mypod", "mynode", false)
|
||||||
|
otherpod = makeTestPod("ns", "otherpod", "othernode", false)
|
||||||
|
unboundpod = makeTestPod("ns", "unboundpod", "", false)
|
||||||
|
|
||||||
|
configmapResource = api.Resource("configmap").WithVersion("v1")
|
||||||
|
configmapKind = api.Kind("ConfigMap").WithVersion("v1")
|
||||||
|
|
||||||
|
podResource = api.Resource("pods").WithVersion("v1")
|
||||||
|
podKind = api.Kind("Pod").WithVersion("v1")
|
||||||
|
|
||||||
|
nodeResource = api.Resource("nodes").WithVersion("v1")
|
||||||
|
nodeKind = api.Kind("Node").WithVersion("v1")
|
||||||
|
|
||||||
|
noExistingPods = fake.NewSimpleClientset().Core()
|
||||||
|
existingPods = fake.NewSimpleClientset(mymirrorpod, othermirrorpod, unboundmirrorpod, mypod, otherpod, unboundpod).Core()
|
||||||
|
)
|
||||||
|
|
||||||
|
sapod := makeTestPod("ns", "mysapod", "mynode", true)
|
||||||
|
sapod.Spec.ServiceAccountName = "foo"
|
||||||
|
|
||||||
|
secretpod := makeTestPod("ns", "mysecretpod", "mynode", true)
|
||||||
|
secretpod.Spec.Volumes = []api.Volume{{VolumeSource: api.VolumeSource{Secret: &api.SecretVolumeSource{SecretName: "foo"}}}}
|
||||||
|
|
||||||
|
configmappod := makeTestPod("ns", "myconfigmappod", "mynode", true)
|
||||||
|
configmappod.Spec.Volumes = []api.Volume{{VolumeSource: api.VolumeSource{ConfigMap: &api.ConfigMapVolumeSource{LocalObjectReference: api.LocalObjectReference{Name: "foo"}}}}}
|
||||||
|
|
||||||
|
pvcpod := makeTestPod("ns", "mypvcpod", "mynode", true)
|
||||||
|
pvcpod.Spec.Volumes = []api.Volume{{VolumeSource: api.VolumeSource{PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{ClaimName: "foo"}}}}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
strict bool
|
||||||
|
podsGetter coreinternalversion.PodsGetter
|
||||||
|
attributes admission.Attributes
|
||||||
|
err string
|
||||||
|
}{
|
||||||
|
// Mirror pods bound to us
|
||||||
|
{
|
||||||
|
name: "allow creating a mirror pod bound to self",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mymirrorpod, nil, podKind, mymirrorpod.Namespace, mymirrorpod.Name, podResource, "", admission.Create, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of mirror pod bound to self",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mymirrorpod, mymirrorpod, podKind, mymirrorpod.Namespace, mymirrorpod.Name, podResource, "", admission.Update, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow delete of mirror pod bound to self",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, mymirrorpod.Namespace, mymirrorpod.Name, podResource, "", admission.Delete, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid create of mirror pod status bound to self",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mymirrorpod, nil, podKind, mymirrorpod.Namespace, mymirrorpod.Name, podResource, "status", admission.Create, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow update of mirror pod status bound to self",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mymirrorpod, mymirrorpod, podKind, mymirrorpod.Namespace, mymirrorpod.Name, podResource, "status", admission.Update, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of mirror pod status bound to self",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, mymirrorpod.Namespace, mymirrorpod.Name, podResource, "status", admission.Delete, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Mirror pods bound to another node
|
||||||
|
{
|
||||||
|
name: "forbid creating a mirror pod bound to another",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(othermirrorpod, nil, podKind, othermirrorpod.Namespace, othermirrorpod.Name, podResource, "", admission.Create, mynode),
|
||||||
|
err: "spec.nodeName set to itself",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of mirror pod bound to another",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(othermirrorpod, othermirrorpod, podKind, othermirrorpod.Namespace, othermirrorpod.Name, podResource, "", admission.Update, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of mirror pod bound to another",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, othermirrorpod.Namespace, othermirrorpod.Name, podResource, "", admission.Delete, mynode),
|
||||||
|
err: "spec.nodeName set to itself",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid create of mirror pod status bound to another",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(othermirrorpod, nil, podKind, othermirrorpod.Namespace, othermirrorpod.Name, podResource, "status", admission.Create, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of mirror pod status bound to another",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(othermirrorpod, othermirrorpod, podKind, othermirrorpod.Namespace, othermirrorpod.Name, podResource, "status", admission.Update, mynode),
|
||||||
|
err: "spec.nodeName set to itself",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of mirror pod status bound to another",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, othermirrorpod.Namespace, othermirrorpod.Name, podResource, "status", admission.Delete, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Mirror pods not bound to any node
|
||||||
|
{
|
||||||
|
name: "forbid creating a mirror pod unbound",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundmirrorpod, nil, podKind, unboundmirrorpod.Namespace, unboundmirrorpod.Name, podResource, "", admission.Create, mynode),
|
||||||
|
err: "spec.nodeName set to itself",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of mirror pod unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundmirrorpod, unboundmirrorpod, podKind, unboundmirrorpod.Namespace, unboundmirrorpod.Name, podResource, "", admission.Update, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of mirror pod unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, unboundmirrorpod.Namespace, unboundmirrorpod.Name, podResource, "", admission.Delete, mynode),
|
||||||
|
err: "spec.nodeName set to itself",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid create of mirror pod status unbound",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundmirrorpod, nil, podKind, unboundmirrorpod.Namespace, unboundmirrorpod.Name, podResource, "status", admission.Create, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of mirror pod status unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundmirrorpod, unboundmirrorpod, podKind, unboundmirrorpod.Namespace, unboundmirrorpod.Name, podResource, "status", admission.Update, mynode),
|
||||||
|
err: "spec.nodeName set to itself",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of mirror pod status unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, unboundmirrorpod.Namespace, unboundmirrorpod.Name, podResource, "status", admission.Delete, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Normal pods bound to us
|
||||||
|
{
|
||||||
|
name: "forbid creating a normal pod bound to self",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mypod, nil, podKind, mypod.Namespace, mypod.Name, podResource, "", admission.Create, mynode),
|
||||||
|
err: "can only create mirror pods",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of normal pod bound to self",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mypod, mypod, podKind, mypod.Namespace, mypod.Name, podResource, "", admission.Update, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow delete of normal pod bound to self",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, mypod.Namespace, mypod.Name, podResource, "", admission.Delete, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid create of normal pod status bound to self",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mypod, nil, podKind, mypod.Namespace, mypod.Name, podResource, "status", admission.Create, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow update of normal pod status bound to self",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mypod, mypod, podKind, mypod.Namespace, mypod.Name, podResource, "status", admission.Update, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of normal pod status bound to self",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, mypod.Namespace, mypod.Name, podResource, "status", admission.Delete, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Normal pods bound to another
|
||||||
|
{
|
||||||
|
name: "forbid creating a normal pod bound to another",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(otherpod, nil, podKind, otherpod.Namespace, otherpod.Name, podResource, "", admission.Create, mynode),
|
||||||
|
err: "can only create mirror pods",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of normal pod bound to another",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(otherpod, otherpod, podKind, otherpod.Namespace, otherpod.Name, podResource, "", admission.Update, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of normal pod bound to another",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, otherpod.Namespace, otherpod.Name, podResource, "", admission.Delete, mynode),
|
||||||
|
err: "spec.nodeName set to itself",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid create of normal pod status bound to another",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(otherpod, nil, podKind, otherpod.Namespace, otherpod.Name, podResource, "status", admission.Create, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of normal pod status bound to another",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(otherpod, otherpod, podKind, otherpod.Namespace, otherpod.Name, podResource, "status", admission.Update, mynode),
|
||||||
|
err: "spec.nodeName set to itself",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of normal pod status bound to another",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, otherpod.Namespace, otherpod.Name, podResource, "status", admission.Delete, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Normal pods not bound to any node
|
||||||
|
{
|
||||||
|
name: "forbid creating a normal pod unbound",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundpod, nil, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "", admission.Create, mynode),
|
||||||
|
err: "can only create mirror pods",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of normal pod unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundpod, unboundpod, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "", admission.Update, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of normal pod unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "", admission.Delete, mynode),
|
||||||
|
err: "spec.nodeName set to itself",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid create of normal pod status unbound",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundpod, nil, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "status", admission.Create, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of normal pod status unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundpod, unboundpod, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "status", admission.Update, mynode),
|
||||||
|
err: "spec.nodeName set to itself",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of normal pod status unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "status", admission.Delete, mynode),
|
||||||
|
err: "forbidden: unexpected operation",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Missing pod
|
||||||
|
{
|
||||||
|
name: "forbid delete of unknown pod",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "", admission.Delete, mynode),
|
||||||
|
err: "not found",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Resource pods
|
||||||
|
{
|
||||||
|
name: "forbid create of pod referencing service account",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(sapod, nil, podKind, sapod.Namespace, sapod.Name, podResource, "", admission.Create, mynode),
|
||||||
|
err: "reference a service account",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid create of pod referencing secret",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(secretpod, nil, podKind, secretpod.Namespace, secretpod.Name, podResource, "", admission.Create, mynode),
|
||||||
|
err: "reference secrets",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid create of pod referencing configmap",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(configmappod, nil, podKind, configmappod.Namespace, configmappod.Name, podResource, "", admission.Create, mynode),
|
||||||
|
err: "reference configmaps",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid create of pod referencing persistentvolumeclaim",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(pvcpod, nil, podKind, pvcpod.Namespace, pvcpod.Name, podResource, "", admission.Create, mynode),
|
||||||
|
err: "reference persistentvolumeclaims",
|
||||||
|
},
|
||||||
|
|
||||||
|
// My node object
|
||||||
|
{
|
||||||
|
name: "allow create of my node",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mynodeObj, nil, nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Create, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow update of my node",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mynodeObj, mynodeObj, nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Update, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow delete of my node",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "", admission.Delete, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow update of my node status",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(mynodeObj, mynodeObj, nodeKind, mynodeObj.Namespace, mynodeObj.Name, nodeResource, "status", admission.Update, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Other node object
|
||||||
|
{
|
||||||
|
name: "forbid create of other node",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(othernodeObj, nil, nodeKind, othernodeObj.Namespace, othernodeObj.Name, nodeResource, "", admission.Create, mynode),
|
||||||
|
err: "cannot modify other nodes",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of other node",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(othernodeObj, othernodeObj, nodeKind, othernodeObj.Namespace, othernodeObj.Name, nodeResource, "", admission.Update, mynode),
|
||||||
|
err: "cannot modify other nodes",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid delete of other node",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, nodeKind, othernodeObj.Namespace, othernodeObj.Name, nodeResource, "", admission.Delete, mynode),
|
||||||
|
err: "cannot modify other nodes",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "forbid update of other node status",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(othernodeObj, othernodeObj, nodeKind, othernodeObj.Namespace, othernodeObj.Name, nodeResource, "status", admission.Update, mynode),
|
||||||
|
err: "cannot modify other nodes",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Unrelated objects
|
||||||
|
{
|
||||||
|
name: "allow create of unrelated object",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(&api.ConfigMap{}, nil, configmapKind, "myns", "mycm", configmapResource, "", admission.Create, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow update of unrelated object",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(&api.ConfigMap{}, &api.ConfigMap{}, configmapKind, "myns", "mycm", configmapResource, "", admission.Update, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow delete of unrelated object",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, configmapKind, "myns", "mycm", configmapResource, "", admission.Delete, mynode),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
|
||||||
|
// Unrelated user
|
||||||
|
{
|
||||||
|
name: "allow unrelated user creating a normal pod unbound",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundpod, nil, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "", admission.Create, bob),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow unrelated user update of normal pod unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundpod, unboundpod, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "", admission.Update, bob),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow unrelated user delete of normal pod unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "", admission.Delete, bob),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow unrelated user create of normal pod status unbound",
|
||||||
|
podsGetter: noExistingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundpod, nil, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "status", admission.Create, bob),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow unrelated user update of normal pod status unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(unboundpod, unboundpod, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "status", admission.Update, bob),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "allow unrelated user delete of normal pod status unbound",
|
||||||
|
podsGetter: existingPods,
|
||||||
|
attributes: admission.NewAttributesRecord(nil, nil, podKind, unboundpod.Namespace, unboundpod.Name, podResource, "status", admission.Delete, bob),
|
||||||
|
err: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
c := NewPlugin(nodeidentifier.NewDefaultNodeIdentifier(), tt.strict)
|
||||||
|
c.podsGetter = tt.podsGetter
|
||||||
|
err := c.Admit(tt.attributes)
|
||||||
|
if (err == nil) != (len(tt.err) == 0) {
|
||||||
|
t.Errorf("nodePlugin.Admit() error = %v, expected %v", err, tt.err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(tt.err) > 0 && !strings.Contains(err.Error(), tt.err) {
|
||||||
|
t.Errorf("nodePlugin.Admit() error = %v, expected %v", err, tt.err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
Reference in New Issue
Block a user