kubelet/networking: add support for cni ConfigLists, pass hostport parameters

** reason for this change **
CNI has recently introduced a new configuration list feature. This
allows for plugin chaining. It also supports varied plugin versions.
This commit is contained in:
Casey Callendrello 2017-04-10 13:18:32 +02:00
parent 8963dd1b8c
commit e4eaad3d24
14 changed files with 258 additions and 43 deletions

View File

@ -36,9 +36,10 @@ go_test(
"//pkg/kubelet/container/testing:go_default_library",
"//pkg/kubelet/network:go_default_library",
"//pkg/kubelet/network/cni/testing:go_default_library",
"//pkg/kubelet/network/hostport:go_default_library",
"//pkg/kubelet/network/testing:go_default_library",
"//pkg/util/exec:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
"//vendor/github.com/stretchr/testify/mock:go_default_library",
"//vendor/k8s.io/client-go/util/testing:go_default_library",
],

View File

@ -20,6 +20,7 @@ import (
"errors"
"fmt"
"sort"
"strings"
"sync"
"github.com/containernetworking/cni/libcni"
@ -56,10 +57,19 @@ type cniNetworkPlugin struct {
type cniNetwork struct {
name string
NetworkConfig *libcni.NetworkConfig
NetworkConfig *libcni.NetworkConfigList
CNIConfig libcni.CNI
}
// cniPortMapping maps to the standard CNI portmapping Capability
// see: https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md
type cniPortMapping struct {
HostPort int32 `json:"hostPort"`
ContainerPort int32 `json:"containerPort"`
Protocol string `json:"protocol"`
HostIP string `json:"hostIP"`
}
func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin {
if binDir == "" {
binDir = DefaultCNIDir
@ -86,7 +96,7 @@ func getDefaultCNINetwork(pluginDir, binDir, vendorCNIDirPrefix string) (*cniNet
if pluginDir == "" {
pluginDir = DefaultNetDir
}
files, err := libcni.ConfFiles(pluginDir)
files, err := libcni.ConfFiles(pluginDir, []string{".conf", ".conflist", ".json"})
switch {
case err != nil:
return nil, err
@ -96,17 +106,37 @@ func getDefaultCNINetwork(pluginDir, binDir, vendorCNIDirPrefix string) (*cniNet
sort.Strings(files)
for _, confFile := range files {
conf, err := libcni.ConfFromFile(confFile)
if err != nil {
glog.Warningf("Error loading CNI config file %s: %v", confFile, err)
var confList *libcni.NetworkConfigList
if strings.HasSuffix(confFile, ".conflist") {
confList, err = libcni.ConfListFromFile(confFile)
if err != nil {
glog.Warningf("Error loading CNI config list file %s: %v", confFile, err)
continue
}
} else {
conf, err := libcni.ConfFromFile(confFile)
if err != nil {
glog.Warningf("Error loading CNI config file %s: %v", confFile, err)
continue
}
confList, err = libcni.ConfListFromConf(conf)
if err != nil {
glog.Warningf("Error converting CNI config file %s to list: %v", confFile, err)
continue
}
}
if len(confList.Plugins) == 0 {
glog.Warningf("CNI config list %s has no networks, skipping", confFile)
continue
}
confType := confList.Plugins[0].Network.Type
// Search for vendor-specific plugins as well as default plugins in the CNI codebase.
vendorDir := vendorCNIDir(vendorCNIDirPrefix, conf.Network.Type)
vendorDir := vendorCNIDir(vendorCNIDirPrefix, confType)
cninet := &libcni.CNIConfig{
Path: []string{binDir, vendorDir},
}
network := &cniNetwork{name: conf.Network.Name, NetworkConfig: conf, CNIConfig: cninet}
network := &cniNetwork{name: confList.Name, NetworkConfig: confList, CNIConfig: cninet}
return network, nil
}
return nil, fmt.Errorf("No valid networks found in %s", pluginDir)
@ -117,10 +147,12 @@ func vendorCNIDir(prefix, pluginType string) string {
}
func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
loConfig, err := libcni.ConfFromBytes([]byte(`{
"cniVersion": "0.1.0",
loConfig, err := libcni.ConfListFromBytes([]byte(`{
"cniVersion": "0.2.0",
"name": "cni-loopback",
"type": "loopback"
"plugins":[{
"type": "loopback"
}]
}`))
if err != nil {
// The hardcoded config above should always be valid and unit tests will
@ -128,7 +160,7 @@ func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
panic(err)
}
cninet := &libcni.CNIConfig{
Path: []string{vendorCNIDir(vendorDirPrefix, loConfig.Network.Type), binDir},
Path: []string{vendorCNIDir(vendorDirPrefix, "loopback"), binDir},
}
loNetwork := &cniNetwork{
name: "lo",
@ -200,13 +232,13 @@ func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubec
return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
}
_, err = plugin.loNetwork.addToNetwork(name, namespace, id, netnsPath)
_, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath)
if err != nil {
glog.Errorf("Error while adding to cni lo network: %s", err)
return err
}
_, err = plugin.getDefaultNetwork().addToNetwork(name, namespace, id, netnsPath)
_, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
if err != nil {
glog.Errorf("Error while adding to cni network: %s", err)
return err
@ -224,7 +256,7 @@ func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id ku
return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
}
return plugin.getDefaultNetwork().deleteFromNetwork(name, namespace, id, netnsPath)
return plugin.deleteFromNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
}
// TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin.
@ -243,16 +275,16 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin
return &network.PodNetworkStatus{IP: ip}, nil
}
func (network *cniNetwork) addToNetwork(podName string, podNamespace string, podInfraContainerID kubecontainer.ContainerID, podNetnsPath string) (*cnitypes.Result, error) {
rt, err := buildCNIRuntimeConf(podName, podNamespace, podInfraContainerID, podNetnsPath)
func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podInfraContainerID kubecontainer.ContainerID, podNetnsPath string) (cnitypes.Result, error) {
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podInfraContainerID, podNetnsPath)
if err != nil {
glog.Errorf("Error adding network when buliding cni runtime conf: %v", err)
return nil, err
}
netconf, cninet := network.NetworkConfig, network.CNIConfig
glog.V(4).Infof("About to run with conf.Network.Type=%v", netconf.Network.Type)
res, err := cninet.AddNetwork(netconf, rt)
netConf, cniNet := network.NetworkConfig, network.CNIConfig
glog.V(4).Infof("About to add CNI network %v (type=%v)", netConf.Name, netConf.Plugins[0].Network.Type)
res, err := cniNet.AddNetworkList(netConf, rt)
if err != nil {
glog.Errorf("Error adding network: %v", err)
return nil, err
@ -261,16 +293,16 @@ func (network *cniNetwork) addToNetwork(podName string, podNamespace string, pod
return res, nil
}
func (network *cniNetwork) deleteFromNetwork(podName string, podNamespace string, podInfraContainerID kubecontainer.ContainerID, podNetnsPath string) error {
rt, err := buildCNIRuntimeConf(podName, podNamespace, podInfraContainerID, podNetnsPath)
func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName string, podNamespace string, podInfraContainerID kubecontainer.ContainerID, podNetnsPath string) error {
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podInfraContainerID, podNetnsPath)
if err != nil {
glog.Errorf("Error deleting network when buliding cni runtime conf: %v", err)
return err
}
netconf, cninet := network.NetworkConfig, network.CNIConfig
glog.V(4).Infof("About to run with conf.Network.Type=%v", netconf.Network.Type)
err = cninet.DelNetwork(netconf, rt)
netConf, cniNet := network.NetworkConfig, network.CNIConfig
glog.V(4).Infof("About to del CNI network %v (type=%v)", netConf.Name, netConf.Plugins[0].Network.Type)
err = cniNet.DelNetworkList(netConf, rt)
if err != nil {
glog.Errorf("Error deleting network: %v", err)
return err
@ -278,7 +310,7 @@ func (network *cniNetwork) deleteFromNetwork(podName string, podNamespace string
return nil
}
func buildCNIRuntimeConf(podName string, podNs string, podInfraContainerID kubecontainer.ContainerID, podNetnsPath string) (*libcni.RuntimeConf, error) {
func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podInfraContainerID kubecontainer.ContainerID, podNetnsPath string) (*libcni.RuntimeConf, error) {
glog.V(4).Infof("Got netns path %v", podNetnsPath)
glog.V(4).Infof("Using netns path %v", podNs)
@ -294,5 +326,24 @@ func buildCNIRuntimeConf(podName string, podNs string, podInfraContainerID kubec
},
}
// port mappings are a cni capability-based args, rather than parameters
// to a specific plugin
portMappings, err := plugin.host.GetPodPortMappings(podInfraContainerID.ID)
if err != nil {
return nil, fmt.Errorf("could not retrieve port mappings: %v", err)
}
portMappingsParam := make([]cniPortMapping, 0, len(portMappings))
for _, p := range portMappings {
portMappingsParam = append(portMappingsParam, cniPortMapping{
HostPort: p.HostPort,
ContainerPort: p.ContainerPort,
Protocol: strings.ToLower(string(p.Protocol)),
HostIP: p.HostIP,
})
}
rt.CapabilityArgs = map[string]interface{}{
"portMappings": portMappingsParam,
}
return rt, nil
}

View File

@ -20,16 +20,18 @@ package cni
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"net"
"os"
"path"
"reflect"
"testing"
"text/template"
cnitypes "github.com/containernetworking/cni/pkg/types"
types020 "github.com/containernetworking/cni/pkg/types/020"
"github.com/stretchr/testify/mock"
utiltesting "k8s.io/client-go/util/testing"
"k8s.io/kubernetes/pkg/api/v1"
@ -39,6 +41,7 @@ import (
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
"k8s.io/kubernetes/pkg/kubelet/network"
"k8s.io/kubernetes/pkg/kubelet/network/cni/testing"
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
networktest "k8s.io/kubernetes/pkg/kubelet/network/testing"
utilexec "k8s.io/kubernetes/pkg/util/exec"
)
@ -54,7 +57,7 @@ func installPluginUnderTest(t *testing.T, testVendorCNIDirPrefix, testNetworkCon
if err != nil {
t.Fatalf("Failed to install plugin")
}
networkConfig := fmt.Sprintf("{ \"name\": \"%s\", \"type\": \"%s\" }", plugName, vendorName)
networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true} }`, plugName, vendorName)
_, err = f.WriteString(networkConfig)
if err != nil {
@ -71,7 +74,7 @@ func installPluginUnderTest(t *testing.T, testVendorCNIDirPrefix, testNetworkCon
f, err = os.Create(pluginExec)
const execScriptTempl = `#!/bin/bash
read ignore
cat > {{.InputFile}}
env > {{.OutputEnv}}
echo "%@" >> {{.OutputEnv}}
export $(echo ${CNI_ARGS} | sed 's/;/ /g') &> /dev/null
@ -80,6 +83,7 @@ echo -n "$CNI_COMMAND $CNI_NETNS $K8S_POD_NAMESPACE $K8S_POD_NAME $K8S_POD_INFRA
echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }"
`
execTemplateData := &map[string]interface{}{
"InputFile": path.Join(pluginDir, plugName+".in"),
"OutputFile": path.Join(pluginDir, plugName+".out"),
"OutputEnv": path.Join(pluginDir, plugName+".env"),
"OutputDir": pluginDir,
@ -117,10 +121,11 @@ type fakeNetworkHost struct {
runtime kubecontainer.Runtime
}
func NewFakeHost(kubeClient clientset.Interface, pods []*containertest.FakePod) *fakeNetworkHost {
func NewFakeHost(kubeClient clientset.Interface, pods []*containertest.FakePod, ports map[string][]*hostport.PortMapping) *fakeNetworkHost {
host := &fakeNetworkHost{
kubeClient: kubeClient,
runtime: &containertest.FakeRuntime{
networktest.FakePortMappingGetter{PortMaps: ports},
kubeClient,
&containertest.FakeRuntime{
AllPodList: pods,
},
}
@ -207,9 +212,22 @@ func TestCNIPlugin(t *testing.T) {
cniPlugin.execer = fexec
cniPlugin.loNetwork.CNIConfig = mockLoCNI
mockLoCNI.On("AddNetwork", cniPlugin.loNetwork.NetworkConfig, mock.AnythingOfType("*libcni.RuntimeConf")).Return(&cnitypes.Result{IP4: &cnitypes.IPConfig{IP: net.IPNet{IP: []byte{127, 0, 0, 1}}}}, nil)
mockLoCNI.On("AddNetworkList", cniPlugin.loNetwork.NetworkConfig, mock.AnythingOfType("*libcni.RuntimeConf")).Return(&types020.Result{IP4: &types020.IPConfig{IP: net.IPNet{IP: []byte{127, 0, 0, 1}}}}, nil)
plug, err := network.InitNetworkPlugin(plugins, "cni", NewFakeHost(nil, pods), componentconfig.HairpinNone, "10.0.0.0/8", network.UseDefaultMTU)
ports := map[string][]*hostport.PortMapping{
containerID.ID: {
{
Name: "name",
HostPort: 8008,
ContainerPort: 80,
Protocol: "UDP",
HostIP: "0.0.0.0",
},
},
}
fakeHost := NewFakeHost(nil, pods, ports)
plug, err := network.InitNetworkPlugin(plugins, "cni", fakeHost, componentconfig.HairpinNone, "10.0.0.0/8", network.UseDefaultMTU)
if err != nil {
t.Fatalf("Failed to select the desired plugin: %v", err)
}
@ -223,14 +241,35 @@ func TestCNIPlugin(t *testing.T) {
eo, eerr := ioutil.ReadFile(outputEnv)
outputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".out")
output, err := ioutil.ReadFile(outputFile)
if err != nil {
if err != nil || eerr != nil {
t.Errorf("Failed to read output file %s: %v (env %s err %v)", outputFile, err, eo, eerr)
}
expectedOutput := "ADD /proc/12345/ns/net podNamespace podName test_infra_container"
if string(output) != expectedOutput {
t.Errorf("Mismatch in expected output for setup hook. Expected '%s', got '%s'", expectedOutput, string(output))
}
// Verify the correct network configuration was passed
inputConfig := struct {
RuntimeConfig struct {
PortMappings []map[string]interface{} `json:"portMappings"`
} `json:"runtimeConfig"`
}{}
inputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".in")
inputBytes, inerr := ioutil.ReadFile(inputFile)
parseerr := json.Unmarshal(inputBytes, &inputConfig)
if inerr != nil || parseerr != nil {
t.Errorf("failed to parse reported cni input config %s: (%v %v)", inputFile, inerr, parseerr)
}
expectedMappings := []map[string]interface{}{
// hah, golang always unmarshals unstructured json numbers as float64
{"hostPort": 8008.0, "containerPort": 80.0, "protocol": "udp", "hostIP": "0.0.0.0"},
}
if !reflect.DeepEqual(inputConfig.RuntimeConfig.PortMappings, expectedMappings) {
t.Errorf("mismatch in expected port mappings. expected %v got %v", expectedMappings, inputConfig.RuntimeConfig.PortMappings)
}
// Get its IP address
status, err := plug.GetPodNetworkStatus("podNamespace", "podName", containerID)
if err != nil {

View File

@ -28,12 +28,22 @@ type MockCNI struct {
mock.Mock
}
func (m *MockCNI) AddNetwork(net *libcni.NetworkConfig, rt *libcni.RuntimeConf) (*types.Result, error) {
func (m *MockCNI) AddNetwork(net *libcni.NetworkConfig, rt *libcni.RuntimeConf) (types.Result, error) {
args := m.Called(net, rt)
return args.Get(0).(*types.Result), args.Error(1)
return args.Get(0).(types.Result), args.Error(1)
}
func (m *MockCNI) DelNetwork(net *libcni.NetworkConfig, rt *libcni.RuntimeConf) error {
args := m.Called(net, rt)
return args.Error(0)
}
func (m *MockCNI) DelNetworkList(net *libcni.NetworkConfigList, rt *libcni.RuntimeConf) error {
args := m.Called(net, rt)
return args.Error(0)
}
func (m *MockCNI) AddNetworkList(net *libcni.NetworkConfigList, rt *libcni.RuntimeConf) (types.Result, error) {
args := m.Called(net, rt)
return args.Get(0).(types.Result), args.Error(1)
}

View File

@ -29,6 +29,7 @@ go_library(
"//pkg/util/sysctl:go_default_library",
"//vendor/github.com/containernetworking/cni/libcni:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/vishvananda/netlink:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",

View File

@ -31,6 +31,7 @@ import (
"github.com/containernetworking/cni/libcni"
cnitypes "github.com/containernetworking/cni/pkg/types"
cnitypes020 "github.com/containernetworking/cni/pkg/types/020"
"github.com/golang/glog"
"github.com/vishvananda/netlink"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
@ -314,10 +315,15 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube
}
// Hook container up with our bridge
res, err := plugin.addContainerToNetwork(plugin.netConfig, network.DefaultInterfaceName, namespace, name, id)
resT, err := plugin.addContainerToNetwork(plugin.netConfig, network.DefaultInterfaceName, namespace, name, id)
if err != nil {
return err
}
// Coerce the CNI result version
res, err := cnitypes020.GetResult(resT)
if err != nil {
return fmt.Errorf("unable to understand network config: %v", err)
}
if res.IP4 == nil {
return fmt.Errorf("CNI plugin reported no IPv4 address for container %v.", id)
}
@ -743,7 +749,7 @@ func (plugin *kubenetNetworkPlugin) buildCNIRuntimeConf(ifName string, id kubeco
}, nil
}
func (plugin *kubenetNetworkPlugin) addContainerToNetwork(config *libcni.NetworkConfig, ifName, namespace, name string, id kubecontainer.ContainerID) (*cnitypes.Result, error) {
func (plugin *kubenetNetworkPlugin) addContainerToNetwork(config *libcni.NetworkConfig, ifName, namespace, name string, id kubecontainer.ContainerID) (cnitypes.Result, error) {
rt, err := plugin.buildCNIRuntimeConf(ifName, id)
if err != nil {
return nil, fmt.Errorf("Error building CNI config: %v", err)

View File

@ -65,9 +65,9 @@ func (nh *fakeNamespaceGetter) GetNetNS(containerID string) (string, error) {
}
type FakePortMappingGetter struct {
mem map[string][]*hostport.PortMapping
PortMaps map[string][]*hostport.PortMapping
}
func (pm *FakePortMappingGetter) GetPodPortMappings(containerID string) ([]*hostport.PortMapping, error) {
return pm.mem[containerID], nil
return pm.PortMaps[containerID], nil
}

1
vendor/BUILD vendored
View File

@ -63,6 +63,7 @@ filegroup(
"//vendor/github.com/containernetworking/cni/libcni:all-srcs",
"//vendor/github.com/containernetworking/cni/pkg/invoke:all-srcs",
"//vendor/github.com/containernetworking/cni/pkg/types:all-srcs",
"//vendor/github.com/containernetworking/cni/pkg/version:all-srcs",
"//vendor/github.com/coreos/etcd/alarm:all-srcs",
"//vendor/github.com/coreos/etcd/auth:all-srcs",
"//vendor/github.com/coreos/etcd/client:all-srcs",

View File

@ -17,6 +17,7 @@ go_library(
deps = [
"//vendor/github.com/containernetworking/cni/pkg/invoke:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/version:go_default_library",
],
)

View File

@ -14,9 +14,14 @@ go_library(
"delegate.go",
"exec.go",
"find.go",
"os_unix.go",
"raw_exec.go",
],
tags = ["automanaged"],
deps = ["//vendor/github.com/containernetworking/cni/pkg/types:go_default_library"],
deps = [
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/version:go_default_library",
],
)
filegroup(

View File

@ -0,0 +1,28 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["types.go"],
tags = ["automanaged"],
deps = ["//vendor/github.com/containernetworking/cni/pkg/types:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@ -25,6 +25,10 @@ filegroup(
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
srcs = [
":package-srcs",
"//vendor/github.com/containernetworking/cni/pkg/types/020:all-srcs",
"//vendor/github.com/containernetworking/cni/pkg/types/current:all-srcs",
],
tags = ["automanaged"],
)

View File

@ -0,0 +1,31 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["types.go"],
tags = ["automanaged"],
deps = [
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@ -0,0 +1,37 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"conf.go",
"plugin.go",
"reconcile.go",
"version.go",
],
tags = ["automanaged"],
deps = [
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/current:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)