Merge pull request #14397 from caesarxuchao/StorageVersion-map

Remove ExpStorageVersion and add StorageVersions to APIServer struct

cmd/kube-apiserver/app/server.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/latest"
 	"k8s.io/kubernetes/pkg/api/meta"
+	apiutil "k8s.io/kubernetes/pkg/api/util"
 	"k8s.io/kubernetes/pkg/apiserver"
 	"k8s.io/kubernetes/pkg/capabilities"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
@@ -73,8 +74,8 @@ type APIServer struct {
 	CertDirectory              string
 	APIPrefix                  string
 	APIGroupPrefix             string
-	StorageVersion             string
+	DeprecatedStorageVersion   string
-	ExpStorageVersion          string
+	StorageVersions            string
 	CloudProvider              string
 	CloudConfigFile            string
 	EventTTL                   time.Duration
@@ -131,6 +132,7 @@ func NewAPIServer() *APIServer {
 		MasterServiceNamespace: api.NamespaceDefault,
 		ClusterName:            "kubernetes",
 		CertDirectory:          "/var/run/kubernetes",
+		StorageVersions:        latest.AllPreferredGroupVersions(),
 
 		RuntimeConfig: make(util.ConfigurationMap),
 		KubeletConfig: client.KubeletConfig{
@@ -181,8 +183,13 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.CertDirectory, "cert-dir", s.CertDirectory, "The directory where the TLS certs are located (by default /var/run/kubernetes). "+
 		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
 	fs.StringVar(&s.APIPrefix, "api-prefix", s.APIPrefix, "The prefix for API requests on the server. Default '/api'.")
-	fs.MarkDeprecated("api-prefix", "--api-prefix is deprecated and will be removed when the v1 API is retired")
+	fs.MarkDeprecated("api-prefix", "--api-prefix is deprecated and will be removed when the v1 API is retired.")
-	fs.StringVar(&s.StorageVersion, "storage-version", s.StorageVersion, "The version to store resources with. Defaults to server preferred")
+	fs.StringVar(&s.DeprecatedStorageVersion, "storage-version", s.DeprecatedStorageVersion, "The version to store the legacy v1 resources with. Defaults to server preferred")
+	fs.MarkDeprecated("storage-version", "--storage-version is deprecated and will be removed when the v1 API is retired. See --storage-versions instead.")
+	fs.StringVar(&s.StorageVersions, "storage-versions", s.StorageVersions, "The versions to store resources with. "+
+		"Different groups may be stored in different versions. Specified in the format \"group1/version1,group2/version2...\". "+
+		"This flag expects a complete list of storage versions of ALL groups registered in the server. "+
+		"It defaults to a list of preferred versions of all registered groups, which is derived from the KUBE_API_VERSIONS environment variable.")
 	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services.  Empty string for no provider.")
 	fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
 	fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL, "Amount of time to retain events. Default 1 hour.")
@@ -272,6 +279,21 @@ func newEtcd(etcdConfigFile string, etcdServerList []string, interfacesFunc meta
 	return etcdStorage, err
 }
 
+// convert to a map between group and groupVersions.
+func generateStorageVersionMap(legacyVersion string, storageVersions string) map[string]string {
+	storageVersionMap := map[string]string{}
+	if legacyVersion != "" {
+		storageVersionMap[""] = legacyVersion
+	}
+	if storageVersions != "" {
+		groupVersions := strings.Split(storageVersions, ",")
+		for _, gv := range groupVersions {
+			storageVersionMap[apiutil.GetGroup(gv)] = gv
+		}
+	}
+	return storageVersionMap
+}
+
 // Run runs the specified APIServer.  This should never exit.
 func (s *APIServer) Run(_ []string) error {
 	s.verifyClusterIPFlags()
@@ -335,41 +357,40 @@ func (s *APIServer) Run(_ []string) error {
 
 	clientConfig := &client.Config{
 		Host:    net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
-		Version: s.StorageVersion,
+		Version: s.DeprecatedStorageVersion,
 	}
 	client, err := client.New(clientConfig)
 	if err != nil {
 		glog.Fatalf("Invalid server address: %v", err)
 	}
 
-	g, err := latest.Group("")
+	legacyV1Group, err := latest.Group("")
 	if err != nil {
 		return err
 	}
-	storageVersions := make(map[string]string)
+
-	if s.StorageVersion == "" {
+	storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions)
-		s.StorageVersion = g.Version
+	if _, found := storageVersions[legacyV1Group.Group]; !found {
+		glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.Group, storageVersions)
 	}
-	etcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, g.InterfacesFor, s.StorageVersion, s.EtcdPathPrefix)
+	etcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, legacyV1Group.InterfacesFor, storageVersions[legacyV1Group.Group], s.EtcdPathPrefix)
-	storageVersions[""] = s.StorageVersion
 	if err != nil {
 		glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
 	}
 
 	var expEtcdStorage storage.Interface
 	if enableExp {
-		g, err := latest.Group("experimental")
+		expGroup, err := latest.Group("experimental")
 		if err != nil {
 			glog.Fatalf("experimental API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
 		}
-		if s.ExpStorageVersion == "" {
+		if _, found := storageVersions[expGroup.Group]; !found {
-			s.ExpStorageVersion = g.GroupVersion
+			glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.Group, storageVersions)
 		}
-		expEtcdStorage, err = newEtcd(s.EtcdConfigFile, s.EtcdServerList, g.InterfacesFor, s.ExpStorageVersion, s.EtcdPathPrefix)
+		expEtcdStorage, err = newEtcd(s.EtcdConfigFile, s.EtcdServerList, expGroup.InterfacesFor, storageVersions[expGroup.Group], s.EtcdPathPrefix)
 		if err != nil {
 			glog.Fatalf("Invalid experimental storage version or misconfigured etcd: %v", err)
 		}
-		storageVersions["experimental"] = s.StorageVersion
 	}
 
 	n := s.ServiceClusterIPRange

cmd/kube-apiserver/app/server_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package app
 
 import (
+	"reflect"
 	"regexp"
 	"testing"
 )
@@ -61,3 +62,39 @@ func TestLongRunningRequestRegexp(t *testing.T) {
 		}
 	}
 }
+
+func TestGenerateStorageVersionMap(t *testing.T) {
+	testCases := []struct {
+		legacyVersion   string
+		storageVersions string
+		expectedMap     map[string]string
+	}{
+		{
+			legacyVersion:   "v1",
+			storageVersions: "v1,experimental/v1alpha1",
+			expectedMap: map[string]string{
+				"":             "v1",
+				"experimental": "experimental/v1alpha1",
+			},
+		},
+		{
+			legacyVersion:   "",
+			storageVersions: "experimental/v1alpha1,v1",
+			expectedMap: map[string]string{
+				"":             "v1",
+				"experimental": "experimental/v1alpha1",
+			},
+		},
+		{
+			legacyVersion:   "",
+			storageVersions: "",
+			expectedMap:     map[string]string{},
+		},
+	}
+	for _, test := range testCases {
+		output := generateStorageVersionMap(test.legacyVersion, test.storageVersions)
+		if !reflect.DeepEqual(test.expectedMap, output) {
+			t.Errorf("unexpected error. expect: %v, got: %v", test.expectedMap, output)
+		}
+	}
+}

hack/verify-flags/known-flags.txt

@@ -268,6 +268,7 @@ ssh-user
 static-pods-config
 stats-port
 storage-version
+storage-versions
 streaming-connection-idle-timeout
 suicide-timeout
 sync-frequency

pkg/api/latest/latest.go

@@ -18,6 +18,8 @@ package latest
 
 import (
 	"fmt"
+	"sort"
+	"strings"
 
 	"k8s.io/kubernetes/pkg/api/meta"
 	"k8s.io/kubernetes/pkg/api/registered"
@@ -32,6 +34,9 @@ var (
 	RegisterGroup = allGroups.RegisterGroup
 	// GroupOrDie is a shortcut to allGroups.GroupOrDie.
 	GroupOrDie = allGroups.GroupOrDie
+	// AllPreferredGroupVersions returns the preferred versions of all
+	// registered groups in the form of "group1/version1,group2/version2,..."
+	AllPreferredGroupVersions = allGroups.AllPreferredGroupVersions
 )
 
 // GroupMetaMap is a map between group names and their metadata.
@@ -76,6 +81,20 @@ func (g GroupMetaMap) GroupOrDie(group string) *GroupMeta {
 	return groupMeta
 }
 
+// AllPreferredGroupVersions returns the preferred versions of all registered
+// groups in the form of "group1/version1,group2/version2,..."
+func (g GroupMetaMap) AllPreferredGroupVersions() string {
+	if len(g) == 0 {
+		return ""
+	}
+	var defaults []string
+	for _, groupMeta := range g {
+		defaults = append(defaults, groupMeta.GroupVersion)
+	}
+	sort.Strings(defaults)
+	return strings.Join(defaults, ",")
+}
+
 // GroupMeta stores the metadata of a group, such as the latest supported version.
 type GroupMeta struct {
 	// GroupVersion represents the current external default version of the group. It

pkg/api/latest/latest_test.go

@@ -0,0 +1,59 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package latest
+
+import "testing"
+
+func TestAllPreferredGroupVersions(t *testing.T) {
+	testCases := []struct {
+		groupMetaMap GroupMetaMap
+		expect       string
+	}{
+		{
+			groupMetaMap: GroupMetaMap{
+				"group1": &GroupMeta{
+					GroupVersion: "group1/v1",
+				},
+				"group2": &GroupMeta{
+					GroupVersion: "group2/v2",
+				},
+				"": &GroupMeta{
+					GroupVersion: "v1",
+				},
+			},
+			expect: "group1/v1,group2/v2,v1",
+		},
+		{
+			groupMetaMap: GroupMetaMap{
+				"": &GroupMeta{
+					GroupVersion: "v1",
+				},
+			},
+			expect: "v1",
+		},
+		{
+			groupMetaMap: GroupMetaMap{},
+			expect:       "",
+		},
+	}
+	for _, testCase := range testCases {
+		output := testCase.groupMetaMap.AllPreferredGroupVersions()
+		if testCase.expect != output {
+			t.Errorf("Error. expect: %s, got: %s", testCase.expect, output)
+		}
+	}
+}