Move rke away from kdm vendor

README.md

@@ -23,6 +23,31 @@ Please use [High Availability (HA) Install](https://rancher.com/docs/rancher/v2.
 
 RKE can be built using the `make` command, and will use the scripts in the `scripts` directory as subcommands. The default subcommand is `ci` and will use `scripts/ci`. Cross compiling can be enabled by setting the environment variable `CROSS=1`. The compiled binaries can be found in the `build/bin` directory. Dependencies are managed by Go modules and can be found in [go.mod](https://github.com/rancher/rke/blob/master/go.mod).
 
+RKE now fetches `data.json` from https://github.com/rancher/kontainer-driver-metadata. To fetch data.json and compile it in rke, run 
+
+```bash
+go generate
+
+# Change RANCHER_METADATA_URL to an external URL instead of using https://releases.rancher.com/kontainer-driver-metadata/dev-v2.4/data.json by default
+RANCHER_METADATA_URL=${URL} go generate
+
+# Or load it from local file
+RANCHER_METATDATA_URL=./local/data.json go generate
+
+# Compile RKE
+make
+```
+
+To specify RANCHER_METADATA_URL in runtime, populate the environment variable when running rke CLI. For example:
+
+```bash
+RANCHER_METADATA_URL=${URL} rke [commands] [options]
+
+RANCHER_METATDATA_URL=${./local/data.json} rke [commands] [options]
+```
+    
+`RANCHER_METADATA_URL` defaults to `https://releases.rancher.com/kontainer-driver-metadata/dev-v2.4/data.json`.
+
 ## License
 
 Copyright (c) 2019 [Rancher Labs, Inc.](http://rancher.com)

cluster/addons.go

@@ -12,7 +12,6 @@ import (
 	"strings"
 	"time"
 
-	rkeData "github.com/rancher/kontainer-driver-metadata/rke/templates"
 	"github.com/rancher/rke/addons"
 	"github.com/rancher/rke/authz"
 	"github.com/rancher/rke/k8s"
@@ -20,7 +19,8 @@ import (
 	"github.com/rancher/rke/services"
 	"github.com/rancher/rke/templates"
 	"github.com/rancher/rke/util"
-	"github.com/rancher/types/apis/management.cattle.io/v3"
+	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
+	"github.com/rancher/types/kdm"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 	appsv1 "k8s.io/api/apps/v1"
@@ -285,7 +285,7 @@ func (c *Cluster) deployKubeDNS(ctx context.Context, data map[string]interface{}
 		return err
 	}
 	KubeDNSConfig.LinearAutoscalerParams = string(linearModeBytes)
-	tmplt, err := templates.GetVersionedTemplates(rkeData.KubeDNS, data, c.Version)
+	tmplt, err := templates.GetVersionedTemplates(kdm.KubeDNS, data, c.Version)
 	if err != nil {
 		return err
 	}
@@ -318,7 +318,7 @@ func (c *Cluster) deployCoreDNS(ctx context.Context, data map[string]interface{}
 		return err
 	}
 	CoreDNSConfig.LinearAutoscalerParams = string(linearModeBytes)
-	tmplt, err := templates.GetVersionedTemplates(rkeData.CoreDNS, data, c.Version)
+	tmplt, err := templates.GetVersionedTemplates(kdm.CoreDNS, data, c.Version)
 	if err != nil {
 		return err
 	}
@@ -364,7 +364,7 @@ func (c *Cluster) deployMetricServer(ctx context.Context, data map[string]interf
 		UpdateStrategy:     c.Monitoring.UpdateStrategy,
 		Replicas:           c.Monitoring.Replicas,
 	}
-	tmplt, err := templates.GetVersionedTemplates(rkeData.MetricsServer, data, c.Version)
+	tmplt, err := templates.GetVersionedTemplates(kdm.MetricsServer, data, c.Version)
 	if err != nil {
 		return err
 	}
@@ -531,7 +531,7 @@ func (c *Cluster) deployIngress(ctx context.Context, data map[string]interface{}
 			ingressConfig.AlpineImage = c.SystemImages.Alpine
 		}
 	}
-	tmplt, err := templates.GetVersionedTemplates(rkeData.NginxIngress, data, c.Version)
+	tmplt, err := templates.GetVersionedTemplates(kdm.NginxIngress, data, c.Version)
 	if err != nil {
 		return err
 	}

cluster/cluster.go

@@ -556,7 +556,9 @@ func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngin
 		},
 	}
 	if metadata.K8sVersionToRKESystemImages == nil {
-		metadata.InitMetadata(ctx)
+		if err := metadata.InitMetadata(ctx); err != nil {
+			return nil, err
+		}
 	}
 	if len(c.ConfigPath) == 0 {
 		c.ConfigPath = pki.ClusterConfig

cluster/encryption.go

@@ -381,9 +381,6 @@ func resolveCustomEncryptionConfig(clusterFile string) (string, *apiserverconfig
 	if err != nil {
 		return clusterFile, nil, fmt.Errorf("error unmarshalling: %v", err)
 	}
-	if err != nil {
-		return "", nil, fmt.Errorf("error unmarshalling encryption custom config: %v", err)
-	}
 	services, ok := r["services"].(map[string]interface{})
 	if services == nil || !ok {
 		return clusterFile, nil, nil

codegen/codegen.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	"github.com/rancher/rke/metadata"
+)
+
+const (
+	defaultURL = "https://releases.rancher.com/kontainer-driver-metadata/dev-v2.4/data.json"
+	dataFile   = "data/data.json"
+)
+
+// Codegen fetch data.json from https://releases.rancher.com/kontainer-driver-metadata/dev-v2.4/data.json and generates bindata
+func main() {
+	u := os.Getenv(metadata.RancherMetadataURLEnv)
+	if u == "" {
+		u = defaultURL
+	}
+	data, err := http.Get(u)
+	if err != nil {
+		panic(fmt.Errorf("failed to fetch data.json from kontainer-driver-metadata repository"))
+	}
+	defer data.Body.Close()
+
+	b, err := ioutil.ReadAll(data.Body)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Println("Writing data")
+	if err := ioutil.WriteFile(dataFile, b, 0755); err != nil {
+		return
+	}
+}

main.go

@@ -1,3 +1,5 @@
+//go:generate go run ./codegen/codegen.go
+//go:generate go run ./vendor/github.com/go-bindata/go-bindata/go-bindata -o ./data/bindata.go -ignore bindata.go -pkg data -modtime 1557785965 -mode 0644 ./data/
 package main
 
 import (
@@ -5,10 +7,9 @@ import (
 	"os"
 	"regexp"
 
-	"github.com/rancher/rke/metadata"
-
 	"github.com/mattn/go-colorable"
 	"github.com/rancher/rke/cmd"
+	"github.com/rancher/rke/metadata"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )

metadata/metadata.go

@@ -2,11 +2,23 @@ package metadata
 
 import (
 	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
 	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
 
 	mVersion "github.com/mcuadros/go-version"
-	"github.com/rancher/kontainer-driver-metadata/rke"
+	"github.com/rancher/rke/data"
-	"github.com/rancher/types/apis/management.cattle.io/v3"
+	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
+	"github.com/rancher/types/kdm"
+)
+
+const (
+	RancherMetadataURLEnv = "RANCHER_METADATA_URL"
 )
 
 var (
@@ -20,34 +32,76 @@ var (
 	K8sBadVersions              = map[string]bool{}
 
 	K8sVersionToWindowsServiceOptions map[string]v3.KubernetesServicesOptions
+
+	c = http.Client{
+		Timeout: time.Second * 30,
+	}
 )
 
 func InitMetadata(ctx context.Context) error {
-	initK8sRKESystemImages()
+	data, err := loadData()
-	initAddonTemplates()
+	if err != nil {
-	initServiceOptions()
+		return fmt.Errorf("failed to load data.json, error: %v", err)
-	initDockerOptions()
+	}
+	initK8sRKESystemImages(data)
+	initAddonTemplates(data)
+	initServiceOptions(data)
+	initDockerOptions(data)
 	return nil
 }
 
+// this method loads metadata from three place, if RANCHER_METADATA_URL is provided then load data from specified location. Otherwise load data from bindata.
+func loadData() (kdm.Data, error) {
+	var b []byte
+	var err error
+	u := os.Getenv(RancherMetadataURLEnv)
+	if u != "" {
+		logrus.Debugf("Loading data.json from %s, timestamp: %s", u, time.Now().Format(time.RFC822))
+		b, err = readFile(u)
+		if err != nil {
+			return kdm.Data{}, err
+		}
+	} else {
+		logrus.Debugf("Logging data.json from local source, timestamp: %s", time.Now().Format(time.RFC822))
+		b, err = data.Asset("data/data.json")
+		if err != nil {
+			return kdm.Data{}, err
+		}
+	}
+	logrus.Debugf("data.json content: %v", string(b))
+	return kdm.FromData(b)
+}
+
+func readFile(file string) ([]byte, error) {
+	if strings.HasPrefix(file, "http") {
+		resp, err := c.Get(file)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		return ioutil.ReadAll(resp.Body)
+	}
+	return ioutil.ReadFile(file)
+}
+
 const RKEVersionDev = "v0.2.3"
 
-func initAddonTemplates() {
+func initAddonTemplates(data kdm.Data) {
-	K8sVersionToTemplates = rke.DriverData.K8sVersionedTemplates
+	K8sVersionToTemplates = data.K8sVersionedTemplates
 }
 
-func initServiceOptions() {
+func initServiceOptions(data kdm.Data) {
-	K8sVersionToServiceOptions = interface{}(rke.DriverData.K8sVersionServiceOptions).(map[string]v3.KubernetesServicesOptions)
+	K8sVersionToServiceOptions = interface{}(data.K8sVersionServiceOptions).(map[string]v3.KubernetesServicesOptions)
-	K8sVersionToWindowsServiceOptions = rke.DriverData.K8sVersionWindowsServiceOptions
+	K8sVersionToWindowsServiceOptions = data.K8sVersionWindowsServiceOptions
 }
 
-func initDockerOptions() {
+func initDockerOptions(data kdm.Data) {
-	K8sVersionToDockerVersions = rke.DriverData.K8sVersionDockerInfo
+	K8sVersionToDockerVersions = data.K8sVersionDockerInfo
 }
 
-func initK8sRKESystemImages() {
+func initK8sRKESystemImages(data kdm.Data) {
 	K8sVersionToRKESystemImages = map[string]v3.RKESystemImages{}
-	rkeData := rke.DriverData
+	rkeData := data
 	// non released versions
 	if RKEVersion == "" {
 		RKEVersion = RKEVersionDev

scripts/validate

@@ -5,7 +5,7 @@ cd $(dirname $0)/..
 
 echo Running validation
 
-PACKAGES=". $(find -name '*.go' | xargs -I{} dirname {} |  cut -f2 -d/ | sort -u | grep -Ev '(^\.$|.git|.trash-cache|vendor|bin)' | sed -e 's!^!./!' -e 's!$!/...!')"
+PACKAGES=". $(find -name '*.go' | xargs -I{} dirname {} |  cut -f2 -d/ | sort -u | grep -Ev '(^\.$|.git|.trash-cache|vendor|bin|data)' | sed -e 's!^!./!' -e 's!$!/...!')"
 
 echo Running: go vet
 go vet ${PACKAGES}

templates/templates.go

@@ -10,9 +10,9 @@ import (
 	"github.com/Masterminds/sprig/v3"
 	"github.com/blang/semver"
 	"github.com/ghodss/yaml"
-	"github.com/rancher/kontainer-driver-metadata/rke/templates"
 	"github.com/rancher/norman/types/convert"
 	"github.com/rancher/rke/metadata"
+	"github.com/rancher/types/kdm"
 	"github.com/sirupsen/logrus"
 )
 
@@ -69,7 +69,7 @@ func getTemplate(templateName, k8sVersion string) (string, error) {
 			continue
 		}
 		if testRange(toMatch) {
-			return metadata.K8sVersionToTemplates[templates.TemplateKeys][versionData[k]], nil
+			return metadata.K8sVersionToTemplates[kdm.TemplateKeys][versionData[k]], nil
 		}
 	}
 	return "", fmt.Errorf("no %s template found for k8sVersion %s", templateName, k8sVersion)

vendor.go

@@ -0,0 +1,9 @@
+// +build vendor
+
+package main
+
+import (
+	_ "github.com/go-bindata/go-bindata/go-bindata"
+)
+
+func main() {}