github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-29 06:27:05 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #42895 from shashidharatd/coredns-2

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 42895, 45940)

[Federation] Automate configuring nameserver in cluster-dns for CoreDNS provider

Addresses issue #42894 #42822

**Release note**:
```
[Federation] CoreDNS server will be automatically added to nameserver resolv.conf chain When using CoreDNS as dns provider for federation during federation join.
```
cc @madhusudancs @kubernetes/sig-federation-bugs
This commit is contained in:
Kubernetes Submit Queue 2017-05-17 03:22:49 -07:00 committed by GitHub
commit a45a1ef28f
8 changed files with 154 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
federation/pkg/dnsprovider/providers/coredns/coredns.go
View File

@ -36,8 +36,9 @@ const (
// Config to override defaults
type Config struct {
Global struct {
EtcdEndpoints string `gcfg:"etcd-endpoints"`
DNSZones string `gcfg:"zones"`
EtcdEndpoints string `gcfg:"etcd-endpoints"`
DNSZones string `gcfg:"zones"`
CoreDNSEndpoints string `gcfg:"coredns-endpoints"`
}
}

4
federation/pkg/kubefed/init/BUILD
View File

@ -15,6 +15,7 @@ go_library(
deps = [
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//federation/apis/federation:go_default_library",
"//federation/pkg/dnsprovider/providers/coredns:go_default_library",
"//federation/pkg/kubefed/util:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/v1:go_default_library",
@ -27,6 +28,7 @@ go_library(
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/spf13/cobra:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/gopkg.in/gcfg.v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
@ -46,6 +48,7 @@ go_test(
tags = ["automanaged"],
deps = [
"//federation/apis/federation:go_default_library",
"//federation/pkg/dnsprovider/providers/coredns:go_default_library",
"//federation/pkg/kubefed/testing:go_default_library",
"//federation/pkg/kubefed/util:go_default_library",
"//pkg/api:go_default_library",
@ -57,6 +60,7 @@ go_test(
"//pkg/apis/rbac/v1beta1:go_default_library",
"//pkg/kubectl/cmd/testing:go_default_library",
"//pkg/kubectl/cmd/util:go_default_library",
"//vendor/gopkg.in/gcfg.v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",

33
federation/pkg/kubefed/init/init.go
View File

@ -42,6 +42,7 @@ import (
triple "k8s.io/client-go/util/cert/triple"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
"k8s.io/kubernetes/federation/apis/federation"
"k8s.io/kubernetes/federation/pkg/dnsprovider/providers/coredns"
"k8s.io/kubernetes/federation/pkg/kubefed/util"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
@ -55,6 +56,7 @@ import (
"github.com/golang/glog"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"gopkg.in/gcfg.v1"
)
const (
@ -379,7 +381,7 @@ func (i *initFederation) Run(cmdOut io.Writer, config util.AdminConfig) error {
glog.V(4).Info("Creating federation controller manager deployment")
_, err = createControllerManager(hostClientset, i.commonOptions.FederationSystemNamespace, i.commonOptions.Name, svc.Name, cmName, i.options.image, cmKubeconfigName, i.options.dnsZoneName, i.options.dnsProvider, sa.Name, dnsProviderSecret, i.options.controllerManagerOverrides, i.options.dryRun)
_, err = createControllerManager(hostClientset, i.commonOptions.FederationSystemNamespace, i.commonOptions.Name, svc.Name, cmName, i.options.image, cmKubeconfigName, i.options.dnsZoneName, i.options.dnsProvider, i.options.dnsProviderConfig, sa.Name, dnsProviderSecret, i.options.controllerManagerOverrides, i.options.dryRun)
if err != nil {
return err
}
@ -847,7 +849,7 @@ func createRoleBindings(clientset client.Interface, namespace, saName, federatio
return newRole, newRolebinding, err
}
func createControllerManager(clientset client.Interface, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, dnsProviderSecret *api.Secret, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) {
func createControllerManager(clientset client.Interface, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, dnsProviderConfig, saName string, dnsProviderSecret *api.Secret, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) {
command := []string{
"/hyperkube",
"federation-controller-manager",
@ -935,12 +937,19 @@ func createControllerManager(clientset client.Interface, namespace, name, svcNam
dep.Spec.Template.Spec.ServiceAccountName = saName
}
if dryRun {
return dep, nil
}
if dnsProviderSecret != nil {
dep = addDNSProviderConfig(dep, dnsProviderSecret.Name)
if dnsProvider == util.FedDNSProviderCoreDNS {
var err error
dep, err = addCoreDNSServerAnnotation(dep, dnsZoneName, dnsProviderConfig)
if err != nil {
return nil, err
}
}
}
if dryRun {
return dep, nil
}
return clientset.Extensions().Deployments(namespace).Create(dep)
@ -1154,3 +1163,15 @@ func addDNSProviderConfig(dep *extensions.Deployment, secretName string) *extens
func authFileContents(username, authSecret string) []byte {
return []byte(fmt.Sprintf("%s,%s,%s\n", authSecret, username, uuid.NewUUID()))
}
func addCoreDNSServerAnnotation(deployment *extensions.Deployment, dnsZoneName, dnsProviderConfig string) (*extensions.Deployment, error) {
var cfg coredns.Config
if err := gcfg.ReadFileInto(&cfg, dnsProviderConfig); err != nil {
return nil, err
}
deployment.Annotations[util.FedDNSZoneName] = dnsZoneName
deployment.Annotations[util.FedNameServer] = cfg.Global.CoreDNSEndpoints
deployment.Annotations[util.FedDNSProvider] = util.FedDNSProviderCoreDNS
return deployment, nil
}

32
federation/pkg/kubefed/init/init_test.go
View File

@ -44,6 +44,7 @@ import (
"k8s.io/client-go/rest/fake"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/kubernetes/federation/apis/federation"
"k8s.io/kubernetes/federation/pkg/dnsprovider/providers/coredns"
kubefedtesting "k8s.io/kubernetes/federation/pkg/kubefed/testing"
"k8s.io/kubernetes/federation/pkg/kubefed/util"
"k8s.io/kubernetes/pkg/api"
@ -55,6 +56,8 @@ import (
rbacv1beta1 "k8s.io/kubernetes/pkg/apis/rbac/v1beta1"
cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing"
cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
"gopkg.in/gcfg.v1"
)
const (
@ -74,7 +77,6 @@ const (
func TestInitFederation(t *testing.T) {
cmdErrMsg := ""
dnsProvider := "google-clouddns"
cmdutil.BehaviorOnFatal(func(str string, code int) {
cmdErrMsg = str
})
@ -97,6 +99,7 @@ func TestInitFederation(t *testing.T) {
etcdPVCapacity string
etcdPersistence string
expectedErr string
dnsProvider string
dnsProviderConfig string
dryRun string
apiserverArgOverrides string
@ -116,6 +119,7 @@ func TestInitFederation(t *testing.T) {
etcdPVCapacity: "5Gi",
etcdPersistence: "true",
expectedErr: "",
dnsProvider: util.FedDNSProviderCoreDNS,
dnsProviderConfig: "dns-provider.conf",
dryRun: "",
apiserverArgOverrides: "--client-ca-file=override,--log-dir=override",
@ -210,6 +214,9 @@ func TestInitFederation(t *testing.T) {
tmpDirPath := ""
buf := bytes.NewBuffer([]byte{})
if tc.dnsProvider == "" {
tc.dnsProvider = "google-clouddns"
}
if tc.dnsProviderConfig != "" {
tmpfile, err := ioutil.TempFile("", tc.dnsProviderConfig)
if err != nil {
@ -227,7 +234,7 @@ func TestInitFederation(t *testing.T) {
}
defer os.Remove(tmpDirPath)
hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.dnsProviderConfig, tc.etcdPersistence, tc.etcdPVCapacity, tc.apiserverArgOverrides, tc.cmArgOverrides, tmpDirPath, tc.apiserverEnableHTTPBasicAuth, tc.apiserverEnableTokenAuth, tc.isRBACAPIAvailable)
hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, tc.dnsProvider, tc.dnsProviderConfig, tc.etcdPersistence, tc.etcdPVCapacity, tc.apiserverArgOverrides, tc.cmArgOverrides, tmpDirPath, tc.apiserverEnableHTTPBasicAuth, tc.apiserverEnableTokenAuth, tc.isRBACAPIAvailable)
if err != nil {
t.Fatalf("[%d] unexpected error: %v", i, err)
}
@ -243,7 +250,7 @@ func TestInitFederation(t *testing.T) {
cmd.Flags().Set("host-cluster-context", "substrate")
cmd.Flags().Set("dns-zone-name", tc.dnsZoneName)
cmd.Flags().Set("image", tc.image)
cmd.Flags().Set("dns-provider", dnsProvider)
cmd.Flags().Set("dns-provider", tc.dnsProvider)
cmd.Flags().Set("apiserver-arg-overrides", tc.apiserverArgOverrides)
cmd.Flags().Set("controllermanager-arg-overrides", tc.cmArgOverrides)
@ -1043,6 +1050,12 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
}
if dnsProviderConfig != "" {
cm = addDNSProviderConfigTest(cm, cmDNSProviderSecret.Name)
if dnsProvider == util.FedDNSProviderCoreDNS {
cm, err = addCoreDNSServerAnnotationTest(cm, dnsZoneName, dnsProviderConfig)
if err != nil {
return nil, err
}
}
}
podList := v1.PodList{}
@ -1540,3 +1553,16 @@ func addDNSProviderConfigTest(dep *v1beta1.Deployment, secretName string) *v1bet
return dep
}
// TODO: Reuse the function addCoreDNSServerAnnotation once that function is converted to use versioned objects.
func addCoreDNSServerAnnotationTest(deployment *v1beta1.Deployment, dnsZoneName, dnsProviderConfig string) (*v1beta1.Deployment, error) {
var cfg coredns.Config
if err := gcfg.ReadFileInto(&cfg, dnsProviderConfig); err != nil {
return nil, err
}
deployment.Annotations[util.FedDNSZoneName] = dnsZoneName
deployment.Annotations[util.FedNameServer] = cfg.Global.CoreDNSEndpoints
deployment.Annotations[util.FedDNSProvider] = util.FedDNSProviderCoreDNS
return deployment, nil
}

13
federation/pkg/kubefed/join.go
View File

@ -278,6 +278,7 @@ func createConfigMap(hostClientSet internalclientset.Interface, config util.Admi
util.FedDomainMapKey: domainMap,
},
}
newConfigMap = populateStubDomainsIfRequired(newConfigMap, cmDep.Annotations)
if dryRun {
return newConfigMap, nil
@ -397,3 +398,15 @@ func getFederationName(hostClientSet internalclientset.Interface, fedNamespace s
return name, nil
}
func populateStubDomainsIfRequired(configMap *api.ConfigMap, annotations map[string]string) *api.ConfigMap {
dnsProvider := annotations[util.FedDNSProvider]
dnsZoneName := annotations[util.FedDNSZoneName]
nameServer := annotations[util.FedNameServer]
if dnsProvider != util.FedDNSProviderCoreDNS || dnsZoneName == "" || nameServer == "" {
return configMap
}
configMap.Data[util.KubeDnsStubDomains] = fmt.Sprintf(`{"%s":["%s"]}`, dnsZoneName, nameServer)
return configMap
}

84
federation/pkg/kubefed/join_test.go
View File

@ -42,10 +42,15 @@ import (
cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
)
// testFederationName is a name to use for the federation in tests. Since the federation
// name is recovered from the federation itself, this constant is an appropriate
// functional replica.
const testFederationName = "test-federation"
const (
// testFederationName is a name to use for the federation in tests. Since the federation
// name is recovered from the federation itself, this constant is an appropriate
// functional replica.
testFederationName = "test-federation"
zoneName = "test-dns-zone"
coreDNSServer = "11.22.33.44:53"
)
func TestJoinFederation(t *testing.T) {
cmdErrMsg := ""
@ -69,6 +74,7 @@ func TestJoinFederation(t *testing.T) {
kubeconfigExplicit string
expectedServer string
expectedErr string
dnsProvider string
}{
{
cluster: "syndicate",
@ -79,6 +85,7 @@ func TestJoinFederation(t *testing.T) {
kubeconfigExplicit: "",
expectedServer: "https://10.20.30.40",
expectedErr: "",
dnsProvider: util.FedDNSProviderCoreDNS,
},
{
cluster: "ally",
@ -138,12 +145,12 @@ func TestJoinFederation(t *testing.T) {
f := testJoinFederationFactory(tc.cluster, tc.secret, tc.expectedServer)
buf := bytes.NewBuffer([]byte{})
hostFactory, err := fakeJoinHostFactory(tc.cluster, tc.clusterCtx, tc.secret, tc.server, tc.token)
hostFactory, err := fakeJoinHostFactory(tc.cluster, tc.clusterCtx, tc.secret, tc.server, tc.token, tc.dnsProvider)
if err != nil {
t.Fatalf("[%d] unexpected error: %v", i, err)
}
targetClusterFactory, err := fakeJoinTargetClusterFactory(tc.cluster, tc.clusterCtx)
targetClusterFactory, err := fakeJoinTargetClusterFactory(tc.cluster, tc.clusterCtx, tc.dnsProvider)
if err != nil {
t.Fatalf("[%d] unexpected error: %v", i, err)
}
@ -229,7 +236,7 @@ func testJoinFederationFactory(clusterName, secretName, server string) cmdutil.F
return f
}
func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token string) (cmdutil.Factory, error) {
func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token, dnsProvider string) (cmdutil.Factory, error) {
if clusterCtx == "" {
clusterCtx = clusterName
}
@ -281,28 +288,26 @@ func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token stri
}
cmName := "controller-manager"
deploymentList := v1beta1.DeploymentList{
deployment := v1beta1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "DeploymentList",
Kind: "Deployment",
APIVersion: testapi.Extensions.GroupVersion().String(),
},
Items: []v1beta1.Deployment{
{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: testapi.Extensions.GroupVersion().String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: cmName,
Namespace: util.DefaultFederationSystemNamespace,
Annotations: map[string]string{
util.FedDomainMapKey: fmt.Sprintf("%s=%s", clusterCtx, "test-dns-zone"),
federation.FederationNameAnnotation: testFederationName,
},
},
ObjectMeta: metav1.ObjectMeta{
Name: cmName,
Namespace: util.DefaultFederationSystemNamespace,
Annotations: map[string]string{
util.FedDomainMapKey: fmt.Sprintf("%s=%s", clusterCtx, zoneName),
federation.FederationNameAnnotation: testFederationName,
},
},
}
if dnsProvider == util.FedDNSProviderCoreDNS {
deployment.Annotations[util.FedDNSZoneName] = zoneName
deployment.Annotations[util.FedNameServer] = coreDNSServer
deployment.Annotations[util.FedDNSProvider] = util.FedDNSProviderCoreDNS
}
deploymentList := v1beta1.DeploymentList{Items: []v1beta1.Deployment{deployment}}
f, tf, codec, _ := cmdtesting.NewAPIFactory()
extensionCodec := testapi.Extensions.Codec()
@ -346,12 +351,12 @@ func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token stri
return f, nil
}
func fakeJoinTargetClusterFactory(clusterName, clusterCtx string) (cmdutil.Factory, error) {
func fakeJoinTargetClusterFactory(clusterName, clusterCtx, dnsProvider string) (cmdutil.Factory, error) {
if clusterCtx == "" {
clusterCtx = clusterName
}
configmapObject := v1.ConfigMap{
configmapObject := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: util.KubeDnsConfigmapName,
Namespace: metav1.NamespaceSystem,
@ -361,9 +366,17 @@ func fakeJoinTargetClusterFactory(clusterName, clusterCtx string) (cmdutil.Facto
},
},
Data: map[string]string{
util.FedDomainMapKey: fmt.Sprintf("%s=%s", clusterCtx, "test-dns-zone"),
util.FedDomainMapKey: fmt.Sprintf("%s=%s", clusterCtx, zoneName),
},
}
if dnsProvider == util.FedDNSProviderCoreDNS {
annotations := map[string]string{
util.FedDNSProvider: util.FedDNSProviderCoreDNS,
util.FedDNSZoneName: zoneName,
util.FedNameServer: coreDNSServer,
}
configmapObject = populateStubDomainsIfRequiredTest(configmapObject, annotations)
}
f, tf, codec, _ := cmdtesting.NewAPIFactory()
ns := dynamic.ContentConfig().NegotiatedSerializer
@ -383,10 +396,10 @@ func fakeJoinTargetClusterFactory(clusterName, clusterCtx string) (cmdutil.Facto
if err != nil {
return nil, err
}
if !apiequality.Semantic.DeepEqual(got, configmapObject) {
return nil, fmt.Errorf("Unexpected configmap object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, configmapObject))
if !apiequality.Semantic.DeepEqual(&got, configmapObject) {
return nil, fmt.Errorf("Unexpected configmap object\n\tDiff: %s", diff.ObjectGoPrintDiff(&got, configmapObject))
}
return &http.Response{StatusCode: http.StatusCreated, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(codec, &configmapObject)}, nil
return &http.Response{StatusCode: http.StatusCreated, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(codec, configmapObject)}, nil
default:
return nil, fmt.Errorf("unexpected request: %#v\n%#v", req.URL, req)
}
@ -413,3 +426,16 @@ func fakeCluster(clusterName, secretName, server string) federationapi.Cluster {
},
}
}
// TODO: Reuse the function populateStubDomainsIfRequired once that function is converted to use versioned objects.
func populateStubDomainsIfRequiredTest(configMap *v1.ConfigMap, annotations map[string]string) *v1.ConfigMap {
dnsProvider := annotations[util.FedDNSProvider]
dnsZoneName := annotations[util.FedDNSZoneName]
nameServer := annotations[util.FedNameServer]
if dnsProvider != util.FedDNSProviderCoreDNS || dnsZoneName == "" || nameServer == "" {
return configMap
}
configMap.Data[util.KubeDnsStubDomains] = fmt.Sprintf(`{"%s":["%s"]}`, dnsZoneName, nameServer)
return configMap
}

24
federation/pkg/kubefed/unjoin.go
View File

@ -111,10 +111,10 @@ func (u *unjoinFederation) Run(f cmdutil.Factory, cmdOut, cmdErr io.Writer, conf
// We anyways continue to try and delete the config map but with above warning
}
// We need to ensure deleting the config map created in the deregistered cluster
// This configmap was created when the cluster joined this federation to aid
// We need to ensure updating the config map created in the deregistered cluster
// This configmap was created/updated when the cluster joined this federation to aid
// the kube-dns of that cluster to aid service discovery.
err = deleteConfigMapFromCluster(hostClientset, secret, cluster, u.commonOptions.FederationSystemNamespace)
err = updateConfigMapFromCluster(hostClientset, secret, cluster, u.commonOptions.FederationSystemNamespace)
if err != nil {
fmt.Fprintf(cmdErr, "WARNING: Encountered error in deleting kube-dns configmap, %v", err)
// We anyways continue to print success message but with above warning
@ -162,7 +162,7 @@ func popCluster(f cmdutil.Factory, name string) (*federationapi.Cluster, error)
return cluster, rh.Delete("", name)
}
func deleteConfigMapFromCluster(hostClientset internalclientset.Interface, secret *api.Secret, cluster *federationapi.Cluster, fedSystemNamespace string) error {
func updateConfigMapFromCluster(hostClientset internalclientset.Interface, secret *api.Secret, cluster *federationapi.Cluster, fedSystemNamespace string) error {
clientset, err := getClientsetFromCluster(secret, cluster)
if err != nil {
return err
@ -182,12 +182,20 @@ func deleteConfigMapFromCluster(hostClientset internalclientset.Interface, secre
return err
}
if _, ok := configMap.Data[util.FedDomainMapKey]; !ok {
return nil
needUpdate := false
if _, ok := configMap.Data[util.FedDomainMapKey]; ok {
configMap.Data[util.FedDomainMapKey] = removeConfigMapString(configMap.Data[util.FedDomainMapKey], domainMap)
needUpdate = true
}
configMap.Data[util.FedDomainMapKey] = removeConfigMapString(configMap.Data[util.FedDomainMapKey], domainMap)
_, err = clientset.Core().ConfigMaps(metav1.NamespaceSystem).Update(configMap)
if _, ok := configMap.Data[util.KubeDnsStubDomains]; ok {
delete(configMap.Data, util.KubeDnsStubDomains)
needUpdate = true
}
if needUpdate {
_, err = clientset.Core().ConfigMaps(metav1.NamespaceSystem).Update(configMap)
}
return err
}

9
federation/pkg/kubefed/util/util.go
View File

@ -44,8 +44,13 @@ const (
KubeconfigSecretDataKey = "kubeconfig"
// Used in and to create the kube-dns configmap storing the zone info
FedDomainMapKey = "federations"
KubeDnsConfigmapName = "kube-dns"
FedDomainMapKey = "federations"
KubeDnsConfigmapName = "kube-dns"
FedDNSZoneName = "dns-zone-name"
FedNameServer = "nameserver"
FedDNSProvider = "dns-provider"
FedDNSProviderCoreDNS = "coredns"
KubeDnsStubDomains = "stubDomains"
// DefaultFederationSystemNamespace is the namespace in which
// federation system components are hosted.