node authorizer sets up access rules for dynamic config

This PR makes the node authorizer automatically set up access rules for
dynamic Kubelet config.

I also added some validation to the node strategy, which I discovered we
were missing while writing this.
This commit is contained in:
Michael Taufen
2018-02-20 11:28:28 -08:00
parent 90c09c75d6
commit ab8dc12333
11 changed files with 523 additions and 34 deletions

View File

@@ -8,15 +8,20 @@ load(
go_test(
name = "go_default_test",
srcs = ["node_authorizer_test.go"],
srcs = [
"graph_test.go",
"node_authorizer_test.go",
],
embed = [":go_default_library"],
deps = [
"//pkg/apis/core:go_default_library",
"//pkg/auth/nodeidentifier:go_default_library",
"//pkg/features:go_default_library",
"//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/k8s.io/api/storage/v1beta1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",

View File

@@ -198,6 +198,50 @@ func (g *Graph) deleteVertex_locked(vertexType vertexType, namespace, name strin
}
}
// must be called under write lock
// deletes edges from a given vertex type to a specific vertex
// will delete each orphaned "from" vertex, but will never delete the "to" vertex
func (g *Graph) deleteEdges_locked(fromType, toType vertexType, toNamespace, toName string) {
// get the "to" side
toVert, exists := g.getVertex_rlocked(toType, toNamespace, toName)
if !exists {
return
}
// get potential "from" verts that match fromType
namespaces, exists := g.vertices[fromType]
if !exists {
return
}
// delete all edges between vertices of fromType and toVert
removeVerts := []*namedVertex{}
for _, vertexMapping := range namespaces {
for _, fromVert := range vertexMapping {
if g.graph.HasEdgeBetween(fromVert, toVert) {
// remove the edge (no-op if edge doesn't exist)
g.graph.RemoveEdge(newDestinationEdge(fromVert, toVert, nil))
// remember to clean up the fromVert if we orphaned it
if g.graph.Degree(fromVert) == 0 {
removeVerts = append(removeVerts, fromVert)
}
}
}
}
// clean up orphaned verts
for _, v := range removeVerts {
g.graph.RemoveNode(v)
delete(g.vertices[v.vertexType][v.namespace], v.name)
if len(g.vertices[v.vertexType][v.namespace]) == 0 {
delete(g.vertices[v.vertexType], v.namespace)
}
if len(g.vertices[v.vertexType]) == 0 {
delete(g.vertices, v.vertexType)
}
}
}
// AddPod should only be called once spec.NodeName is populated.
// It sets up edges for the following relationships (which are immutable for a pod once bound to a node):
//
@@ -301,3 +345,25 @@ func (g *Graph) DeleteVolumeAttachment(name string) {
defer g.lock.Unlock()
g.deleteVertex_locked(vaVertexType, "", name)
}
// SetNodeConfigMap sets up edges for the Node.Spec.ConfigSource.ConfigMapRef relationship:
//
// configmap -> node
func (g *Graph) SetNodeConfigMap(nodeName, configMapName, configMapNamespace string) {
g.lock.Lock()
defer g.lock.Unlock()
// TODO(mtaufen): ensure len(nodeName) > 0 in all cases (would sure be nice to have a dependently-typed language here...)
// clear edges configmaps -> node where the destination is the current node *only*
// at present, a node can only have one *direct* configmap reference at a time
g.deleteEdges_locked(configMapVertexType, nodeVertexType, "", nodeName)
// establish new edges if we have a real ConfigMap to reference
if len(configMapName) > 0 && len(configMapNamespace) > 0 {
configmapVertex := g.getOrCreateVertex_locked(configMapVertexType, configMapNamespace, configMapName)
nodeVertex := g.getOrCreateVertex_locked(nodeVertexType, "", nodeName)
g.graph.SetEdge(newDestinationEdge(configmapVertex, nodeVertex, nodeVertex))
}
}

View File

@@ -17,6 +17,7 @@ limitations under the License.
package node
import (
"fmt"
"github.com/golang/glog"
storagev1beta1 "k8s.io/api/storage/v1beta1"
@@ -34,6 +35,7 @@ type graphPopulator struct {
func AddGraphEventHandlers(
graph *Graph,
nodes coreinformers.NodeInformer,
pods coreinformers.PodInformer,
pvs coreinformers.PersistentVolumeInformer,
attachments storageinformers.VolumeAttachmentInformer,
@@ -42,6 +44,14 @@ func AddGraphEventHandlers(
graph: graph,
}
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
nodes.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: g.addNode,
UpdateFunc: g.updateNode,
DeleteFunc: g.deleteNode,
})
}
pods.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: g.addPod,
UpdateFunc: g.updatePod,
@@ -63,6 +73,57 @@ func AddGraphEventHandlers(
}
}
func (g *graphPopulator) addNode(obj interface{}) {
g.updateNode(nil, obj)
}
func (g *graphPopulator) updateNode(oldObj, obj interface{}) {
node := obj.(*api.Node)
var oldNode *api.Node
if oldObj != nil {
oldNode = oldObj.(*api.Node)
}
// we only set up rules for ConfigMapRef today, because that is the only reference type
var name, namespace string
if source := node.Spec.ConfigSource; source != nil && source.ConfigMapRef != nil {
name = source.ConfigMapRef.Name
namespace = source.ConfigMapRef.Namespace
}
var oldName, oldNamespace string
if oldNode != nil {
if oldSource := oldNode.Spec.ConfigSource; oldSource != nil && oldSource.ConfigMapRef != nil {
oldName = oldSource.ConfigMapRef.Name
oldNamespace = oldSource.ConfigMapRef.Namespace
}
}
// if Node.Spec.ConfigSource wasn't updated, nothing for us to do
if name == oldName && namespace == oldNamespace {
return
}
path := "nil"
if node.Spec.ConfigSource != nil {
path = fmt.Sprintf("%s/%s", namespace, name)
}
glog.V(4).Infof("updateNode configSource reference to %s for node %s", path, node.Name)
g.graph.SetNodeConfigMap(node.Name, name, namespace)
}
func (g *graphPopulator) deleteNode(obj interface{}) {
node := obj.(*api.Node)
// TODO(mtaufen): ensure len(nodeName) > 0 in all cases (would sure be nice to have a dependently-typed language here...)
// NOTE: We don't remove the node, because if the node is re-created not all pod -> node
// links are re-established (we don't get relevant events because the no mutations need
// to happen in the API; the state is already there).
g.graph.SetNodeConfigMap(node.Name, "", "")
}
func (g *graphPopulator) addPod(obj interface{}) {
g.updatePod(nil, obj)
}

View File

@@ -0,0 +1,176 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package node
import (
"sort"
"testing"
"github.com/stretchr/testify/assert"
)
func TestDeleteEdges_locked(t *testing.T) {
cases := []struct {
desc string
fromType vertexType
toType vertexType
toNamespace string
toName string
start *Graph
expect *Graph
}{
{
// single edge from a configmap to a node, will delete edge and orphaned configmap
desc: "edges and source orphans are deleted, destination orphans are preserved",
fromType: configMapVertexType,
toType: nodeVertexType,
toNamespace: "",
toName: "node1",
start: func() *Graph {
g := NewGraph()
nodeVertex := g.getOrCreateVertex_locked(nodeVertexType, "", "node1")
configmapVertex := g.getOrCreateVertex_locked(configMapVertexType, "namespace1", "configmap1")
g.graph.SetEdge(newDestinationEdge(configmapVertex, nodeVertex, nodeVertex))
return g
}(),
expect: func() *Graph {
g := NewGraph()
g.getOrCreateVertex_locked(nodeVertexType, "", "node1")
return g
}(),
},
{
// two edges from the same configmap to distinct nodes, will delete one of the edges
desc: "edges are deleted, non-orphans and destination orphans are preserved",
fromType: configMapVertexType,
toType: nodeVertexType,
toNamespace: "",
toName: "node2",
start: func() *Graph {
g := NewGraph()
nodeVertex1 := g.getOrCreateVertex_locked(nodeVertexType, "", "node1")
nodeVertex2 := g.getOrCreateVertex_locked(nodeVertexType, "", "node2")
configmapVertex := g.getOrCreateVertex_locked(configMapVertexType, "namespace1", "configmap1")
g.graph.SetEdge(newDestinationEdge(configmapVertex, nodeVertex1, nodeVertex1))
g.graph.SetEdge(newDestinationEdge(configmapVertex, nodeVertex2, nodeVertex2))
return g
}(),
expect: func() *Graph {
g := NewGraph()
nodeVertex1 := g.getOrCreateVertex_locked(nodeVertexType, "", "node1")
g.getOrCreateVertex_locked(nodeVertexType, "", "node2")
configmapVertex := g.getOrCreateVertex_locked(configMapVertexType, "namespace1", "configmap1")
g.graph.SetEdge(newDestinationEdge(configmapVertex, nodeVertex1, nodeVertex1))
return g
}(),
},
{
desc: "no edges to delete",
fromType: configMapVertexType,
toType: nodeVertexType,
toNamespace: "",
toName: "node1",
start: func() *Graph {
g := NewGraph()
g.getOrCreateVertex_locked(nodeVertexType, "", "node1")
g.getOrCreateVertex_locked(configMapVertexType, "namespace1", "configmap1")
return g
}(),
expect: func() *Graph {
g := NewGraph()
g.getOrCreateVertex_locked(nodeVertexType, "", "node1")
g.getOrCreateVertex_locked(configMapVertexType, "namespace1", "configmap1")
return g
}(),
},
{
desc: "destination vertex does not exist",
fromType: configMapVertexType,
toType: nodeVertexType,
toNamespace: "",
toName: "node1",
start: func() *Graph {
g := NewGraph()
g.getOrCreateVertex_locked(configMapVertexType, "namespace1", "configmap1")
return g
}(),
expect: func() *Graph {
g := NewGraph()
g.getOrCreateVertex_locked(configMapVertexType, "namespace1", "configmap1")
return g
}(),
},
{
desc: "source vertex type doesn't exist",
fromType: configMapVertexType,
toType: nodeVertexType,
toNamespace: "",
toName: "node1",
start: func() *Graph {
g := NewGraph()
g.getOrCreateVertex_locked(nodeVertexType, "", "node1")
return g
}(),
expect: func() *Graph {
g := NewGraph()
g.getOrCreateVertex_locked(nodeVertexType, "", "node1")
return g
}(),
},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
c.start.deleteEdges_locked(c.fromType, c.toType, c.toNamespace, c.toName)
// Note: We assert on substructures (graph.Nodes(), graph.Edges()) because the graph tracks
// freed IDs for reuse, which results in an irrelevant inequality between start and expect.
// sort the nodes by ID
// (the slices we get back are from map iteration, where order is not guaranteed)
expectNodes := c.expect.graph.Nodes()
sort.Slice(expectNodes, func(i, j int) bool {
return expectNodes[i].ID() < expectNodes[j].ID()
})
startNodes := c.start.graph.Nodes()
sort.Slice(expectNodes, func(i, j int) bool {
return startNodes[i].ID() < startNodes[j].ID()
})
assert.Equal(t, expectNodes, startNodes)
// sort the edges by from ID, then to ID
// (the slices we get back are from map iteration, where order is not guaranteed)
expectEdges := c.expect.graph.Edges()
sort.Slice(expectEdges, func(i, j int) bool {
if expectEdges[i].From().ID() == expectEdges[j].From().ID() {
return expectEdges[i].To().ID() < expectEdges[j].To().ID()
}
return expectEdges[i].From().ID() < expectEdges[j].From().ID()
})
startEdges := c.start.graph.Edges()
sort.Slice(expectEdges, func(i, j int) bool {
if startEdges[i].From().ID() == startEdges[j].From().ID() {
return startEdges[i].To().ID() < startEdges[j].To().ID()
}
return startEdges[i].From().ID() < startEdges[j].From().ID()
})
assert.Equal(t, expectEdges, startEdges)
// vertices is a recursive map, no need to sort
assert.Equal(t, c.expect.vertices, c.start.vertices)
})
}
}

View File

@@ -38,6 +38,7 @@ import (
// 1. If a request is not from a node (NodeIdentity() returns isNode=false), reject
// 2. If a specific node cannot be identified (NodeIdentity() returns nodeName=""), reject
// 3. If a request is for a secret, configmap, persistent volume or persistent volume claim, reject unless the verb is get, and the requested object is related to the requesting node:
// node <- configmap
// node <- pod
// node <- pod <- secret
// node <- pod <- configmap

View File

@@ -26,6 +26,7 @@ import (
storagev1beta1 "k8s.io/api/storage/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -72,8 +73,8 @@ func TestAuthorizer(t *testing.T) {
sharedPVCsPerPod: 0,
uniquePVCsPerPod: 1,
}
pods, pvs, attachments := generate(opts)
populate(g, pods, pvs, attachments)
nodes, pods, pvs, attachments := generate(opts)
populate(g, nodes, pods, pvs, attachments)
identifier := nodeidentifier.NewDefaultNodeIdentifier()
authz := NewAuthorizer(g, identifier, bootstrappolicy.NodeRules()).(*NodeAuthorizer)
@@ -86,6 +87,11 @@ func TestAuthorizer(t *testing.T) {
expect authorizer.Decision
features utilfeature.FeatureGate
}{
{
name: "allowed node configmap",
attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "node0-configmap", Namespace: "ns0"},
expect: authorizer.DecisionAllow,
},
{
name: "allowed configmap",
attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "configmap0-pod0-node0", Namespace: "ns0"},
@@ -117,6 +123,11 @@ func TestAuthorizer(t *testing.T) {
expect: authorizer.DecisionAllow,
},
{
name: "disallowed node configmap",
attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "node1-configmap", Namespace: "ns0"},
expect: authorizer.DecisionNoOpinion,
},
{
name: "disallowed configmap",
attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "configmap0-pod0-node1", Namespace: "ns0"},
@@ -247,9 +258,14 @@ func TestAuthorizerSharedResources(t *testing.T) {
},
})
g.SetNodeConfigMap("node1", "shared-configmap", "ns1")
g.SetNodeConfigMap("node2", "shared-configmap", "ns1")
g.SetNodeConfigMap("node3", "configmap", "ns1")
testcases := []struct {
User user.Info
Secret string
ConfigMap string
ExpectAllowed bool
}{
{User: node1, ExpectAllowed: true, Secret: "node1-only"},
@@ -263,14 +279,39 @@ func TestAuthorizerSharedResources(t *testing.T) {
{User: node3, ExpectAllowed: false, Secret: "node1-only"},
{User: node3, ExpectAllowed: false, Secret: "node1-node2-only"},
{User: node3, ExpectAllowed: true, Secret: "shared-all"},
{User: node1, ExpectAllowed: true, ConfigMap: "shared-configmap"},
{User: node1, ExpectAllowed: false, ConfigMap: "configmap"},
{User: node2, ExpectAllowed: true, ConfigMap: "shared-configmap"},
{User: node2, ExpectAllowed: false, ConfigMap: "configmap"},
{User: node3, ExpectAllowed: false, ConfigMap: "shared-configmap"},
{User: node3, ExpectAllowed: true, ConfigMap: "configmap"},
}
for i, tc := range testcases {
decision, _, err := authz.Authorize(authorizer.AttributesRecord{User: tc.User, ResourceRequest: true, Verb: "get", Resource: "secrets", Namespace: "ns1", Name: tc.Secret})
if err != nil {
t.Errorf("%d: unexpected error: %v", i, err)
continue
var (
decision authorizer.Decision
err error
)
if len(tc.Secret) > 0 {
decision, _, err = authz.Authorize(authorizer.AttributesRecord{User: tc.User, ResourceRequest: true, Verb: "get", Resource: "secrets", Namespace: "ns1", Name: tc.Secret})
if err != nil {
t.Errorf("%d: unexpected error: %v", i, err)
continue
}
} else if len(tc.ConfigMap) > 0 {
decision, _, err = authz.Authorize(authorizer.AttributesRecord{User: tc.User, ResourceRequest: true, Verb: "get", Resource: "configmaps", Namespace: "ns1", Name: tc.ConfigMap})
if err != nil {
t.Errorf("%d: unexpected error: %v", i, err)
continue
}
} else {
t.Fatalf("test case must include a request for a Secret or ConfigMap")
}
if (decision == authorizer.DecisionAllow) != tc.ExpectAllowed {
t.Errorf("%d: expected %v, got %v", i, tc.ExpectAllowed, decision)
}
@@ -309,12 +350,12 @@ func BenchmarkPopulationAllocation(b *testing.B) {
uniquePVCsPerPod: 1,
}
pods, pvs, attachments := generate(opts)
nodes, pods, pvs, attachments := generate(opts)
b.ResetTimer()
for i := 0; i < b.N; i++ {
g := NewGraph()
populate(g, pods, pvs, attachments)
populate(g, nodes, pods, pvs, attachments)
}
}
@@ -340,14 +381,14 @@ func BenchmarkPopulationRetention(b *testing.B) {
uniquePVCsPerPod: 1,
}
pods, pvs, attachments := generate(opts)
nodes, pods, pvs, attachments := generate(opts)
// Garbage collect before the first iteration
runtime.GC()
b.ResetTimer()
for i := 0; i < b.N; i++ {
g := NewGraph()
populate(g, pods, pvs, attachments)
populate(g, nodes, pods, pvs, attachments)
if i == 0 {
f, _ := os.Create("BenchmarkPopulationRetention.profile")
@@ -375,8 +416,8 @@ func BenchmarkAuthorization(b *testing.B) {
sharedPVCsPerPod: 0,
uniquePVCsPerPod: 1,
}
pods, pvs, attachments := generate(opts)
populate(g, pods, pvs, attachments)
nodes, pods, pvs, attachments := generate(opts)
populate(g, nodes, pods, pvs, attachments)
identifier := nodeidentifier.NewDefaultNodeIdentifier()
authz := NewAuthorizer(g, identifier, bootstrappolicy.NodeRules()).(*NodeAuthorizer)
@@ -389,6 +430,11 @@ func BenchmarkAuthorization(b *testing.B) {
expect authorizer.Decision
features utilfeature.FeatureGate
}{
{
name: "allowed node configmap",
attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "node0-configmap", Namespace: "ns0"},
expect: authorizer.DecisionAllow,
},
{
name: "allowed configmap",
attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "configmap0-pod0-node0", Namespace: "ns0"},
@@ -404,6 +450,12 @@ func BenchmarkAuthorization(b *testing.B) {
attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "secrets", Name: "secret0-shared", Namespace: "ns0"},
expect: authorizer.DecisionAllow,
},
{
name: "disallowed node configmap",
attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "node1-configmap", Namespace: "ns0"},
expect: authorizer.DecisionNoOpinion,
},
{
name: "disallowed configmap",
attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "configmap0-pod0-node1", Namespace: "ns0"},
@@ -467,9 +519,12 @@ func BenchmarkAuthorization(b *testing.B) {
}
}
func populate(graph *Graph, pods []*api.Pod, pvs []*api.PersistentVolume, attachments []*storagev1beta1.VolumeAttachment) {
func populate(graph *Graph, nodes []*api.Node, pods []*api.Pod, pvs []*api.PersistentVolume, attachments []*storagev1beta1.VolumeAttachment) {
p := &graphPopulator{}
p.graph = graph
for _, node := range nodes {
p.addNode(node)
}
for _, pod := range pods {
p.addPod(pod)
}
@@ -485,7 +540,8 @@ func populate(graph *Graph, pods []*api.Pod, pvs []*api.PersistentVolume, attach
// the secret/configmap/pvc/node references in the pod and pv objects are named to indicate the connections between the objects.
// for example, secret0-pod0-node0 is a secret referenced by pod0 which is bound to node0.
// when populated into the graph, the node authorizer should allow node0 to access that secret, but not node1.
func generate(opts sampleDataOpts) ([]*api.Pod, []*api.PersistentVolume, []*storagev1beta1.VolumeAttachment) {
func generate(opts sampleDataOpts) ([]*api.Node, []*api.Pod, []*api.PersistentVolume, []*storagev1beta1.VolumeAttachment) {
nodes := make([]*api.Node, 0, opts.nodes)
pods := make([]*api.Pod, 0, opts.nodes*opts.podsPerNode)
pvs := make([]*api.PersistentVolume, 0, (opts.nodes*opts.podsPerNode*opts.uniquePVCsPerPod)+(opts.sharedPVCsPerPod*opts.namespaces))
attachments := make([]*storagev1beta1.VolumeAttachment, 0, opts.nodes*opts.attachmentsPerNode)
@@ -552,6 +608,20 @@ func generate(opts sampleDataOpts) ([]*api.Pod, []*api.PersistentVolume, []*stor
attachment.Spec.NodeName = nodeName
attachments = append(attachments, attachment)
}
name := fmt.Sprintf("%s-configmap", nodeName)
nodes = append(nodes, &api.Node{
ObjectMeta: metav1.ObjectMeta{Name: nodeName},
Spec: api.NodeSpec{
ConfigSource: &api.NodeConfigSource{
ConfigMapRef: &api.ObjectReference{
Name: name,
Namespace: "ns0",
UID: types.UID(fmt.Sprintf("ns0-%s", name)),
},
},
},
})
}
return pods, pvs, attachments
return nodes, pods, pvs, attachments
}