mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-21 02:41:25 +00:00
Merge pull request #80905 from fabriziopandini/kubeadm-kustomize-core
Kubeadm: kustomize core
This commit is contained in:
commit
c08ee9d51b
@ -144,6 +144,10 @@ func runControlPlaneSubphase(component string) func(c workflow.RunData) error {
|
|||||||
cfg := data.Cfg()
|
cfg := data.Cfg()
|
||||||
|
|
||||||
fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component)
|
fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component)
|
||||||
return controlplane.CreateStaticPodFiles(data.ManifestDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
|
|
||||||
|
// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
|
||||||
|
kustomizeDir := ""
|
||||||
|
|
||||||
|
return controlplane.CreateStaticPodFiles(data.ManifestDir(), kustomizeDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -92,7 +92,11 @@ func runEtcdPhaseLocal() func(c workflow.RunData) error {
|
|||||||
fmt.Printf("[dryrun] Would ensure that %q directory is present\n", cfg.Etcd.Local.DataDir)
|
fmt.Printf("[dryrun] Would ensure that %q directory is present\n", cfg.Etcd.Local.DataDir)
|
||||||
}
|
}
|
||||||
fmt.Printf("[etcd] Creating static Pod manifest for local etcd in %q\n", data.ManifestDir())
|
fmt.Printf("[etcd] Creating static Pod manifest for local etcd in %q\n", data.ManifestDir())
|
||||||
if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
|
|
||||||
|
// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
|
||||||
|
kustomizeDir := ""
|
||||||
|
|
||||||
|
if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), kustomizeDir, cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
|
||||||
return errors.Wrap(err, "error creating local etcd static pod manifest file")
|
return errors.Wrap(err, "error creating local etcd static pod manifest file")
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
@ -184,10 +184,14 @@ func runControlPlanePrepareControlPlaneSubphase(c workflow.RunData) error {
|
|||||||
|
|
||||||
fmt.Printf("[control-plane] Using manifest folder %q\n", kubeadmconstants.GetStaticPodDirectory())
|
fmt.Printf("[control-plane] Using manifest folder %q\n", kubeadmconstants.GetStaticPodDirectory())
|
||||||
|
|
||||||
|
// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
|
||||||
|
kustomizeDir := ""
|
||||||
|
|
||||||
for _, component := range kubeadmconstants.ControlPlaneComponents {
|
for _, component := range kubeadmconstants.ControlPlaneComponents {
|
||||||
fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component)
|
fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component)
|
||||||
err := controlplane.CreateStaticPodFiles(
|
err := controlplane.CreateStaticPodFiles(
|
||||||
kubeadmconstants.GetStaticPodDirectory(),
|
kubeadmconstants.GetStaticPodDirectory(),
|
||||||
|
kustomizeDir,
|
||||||
&cfg.ClusterConfiguration,
|
&cfg.ClusterConfiguration,
|
||||||
&cfg.LocalAPIEndpoint,
|
&cfg.LocalAPIEndpoint,
|
||||||
component,
|
component,
|
||||||
|
@ -17,9 +17,11 @@ go_test(
|
|||||||
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
||||||
"//cmd/kubeadm/app/constants:go_default_library",
|
"//cmd/kubeadm/app/constants:go_default_library",
|
||||||
"//cmd/kubeadm/app/phases/certs:go_default_library",
|
"//cmd/kubeadm/app/phases/certs:go_default_library",
|
||||||
|
"//cmd/kubeadm/app/util/staticpod:go_default_library",
|
||||||
"//cmd/kubeadm/test:go_default_library",
|
"//cmd/kubeadm/test:go_default_library",
|
||||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||||
|
"//vendor/github.com/lithammer/dedent:go_default_library",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -39,9 +39,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane.
|
// CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane.
|
||||||
func CreateInitStaticPodManifestFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration) error {
|
func CreateInitStaticPodManifestFiles(manifestDir, kustomizeDir string, cfg *kubeadmapi.InitConfiguration) error {
|
||||||
klog.V(1).Infoln("[control-plane] creating static Pod files")
|
klog.V(1).Infoln("[control-plane] creating static Pod files")
|
||||||
return CreateStaticPodFiles(manifestDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler)
|
return CreateStaticPodFiles(manifestDir, kustomizeDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current configuration
|
// GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current configuration
|
||||||
@ -103,7 +103,7 @@ func livenessProbe(host string, port int, scheme v1.URIScheme) *v1.Probe {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CreateStaticPodFiles creates all the requested static pod files.
|
// CreateStaticPodFiles creates all the requested static pod files.
|
||||||
func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, componentNames ...string) error {
|
func CreateStaticPodFiles(manifestDir, kustomizeDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, componentNames ...string) error {
|
||||||
// gets the StaticPodSpecs, actualized for the current ClusterConfiguration
|
// gets the StaticPodSpecs, actualized for the current ClusterConfiguration
|
||||||
klog.V(1).Infoln("[control-plane] getting StaticPodSpecs")
|
klog.V(1).Infoln("[control-plane] getting StaticPodSpecs")
|
||||||
specs := GetStaticPodSpecs(cfg, endpoint)
|
specs := GetStaticPodSpecs(cfg, endpoint)
|
||||||
@ -116,6 +116,15 @@ func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.ClusterConfigurati
|
|||||||
return errors.Errorf("couldn't retrieve StaticPodSpec for %q", componentName)
|
return errors.Errorf("couldn't retrieve StaticPodSpec for %q", componentName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// if kustomizeDir is defined, customize the static pod manifest
|
||||||
|
if kustomizeDir != "" {
|
||||||
|
kustomizedSpec, err := staticpodutil.KustomizeStaticPod(&spec, kustomizeDir)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "failed to kustomize static pod manifest file for %q", componentName)
|
||||||
|
}
|
||||||
|
spec = *kustomizedSpec
|
||||||
|
}
|
||||||
|
|
||||||
// writes the StaticPodSpec to disk
|
// writes the StaticPodSpec to disk
|
||||||
if err := staticpodutil.WriteStaticPodToDisk(componentName, manifestDir, spec); err != nil {
|
if err := staticpodutil.WriteStaticPodToDisk(componentName, manifestDir, spec); err != nil {
|
||||||
return errors.Wrapf(err, "failed to create static pod manifest file for %q", componentName)
|
return errors.Wrapf(err, "failed to create static pod manifest file for %q", componentName)
|
||||||
|
@ -18,6 +18,7 @@ package controlplane
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
@ -25,11 +26,13 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/lithammer/dedent"
|
||||||
|
|
||||||
"k8s.io/apimachinery/pkg/util/sets"
|
"k8s.io/apimachinery/pkg/util/sets"
|
||||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||||
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
|
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
|
||||||
|
staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
|
||||||
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
|
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -121,7 +124,7 @@ func TestCreateStaticPodFilesAndWrappers(t *testing.T) {
|
|||||||
|
|
||||||
// Execute createStaticPodFunction
|
// Execute createStaticPodFunction
|
||||||
manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
|
manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
|
||||||
err := CreateStaticPodFiles(manifestPath, cfg, &kubeadmapi.APIEndpoint{}, test.components...)
|
err := CreateStaticPodFiles(manifestPath, "", cfg, &kubeadmapi.APIEndpoint{}, test.components...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Error executing createStaticPodFunction: %v", err)
|
t.Errorf("Error executing createStaticPodFunction: %v", err)
|
||||||
return
|
return
|
||||||
@ -137,6 +140,56 @@ func TestCreateStaticPodFilesAndWrappers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCreateStaticPodFilesKustomize(t *testing.T) {
|
||||||
|
// Create temp folder for the test case
|
||||||
|
tmpdir := testutil.SetupTempDir(t)
|
||||||
|
defer os.RemoveAll(tmpdir)
|
||||||
|
|
||||||
|
// Creates a Cluster Configuration
|
||||||
|
cfg := &kubeadmapi.ClusterConfiguration{
|
||||||
|
KubernetesVersion: "v1.9.0",
|
||||||
|
}
|
||||||
|
|
||||||
|
kustomizePath := filepath.Join(tmpdir, "kustomize")
|
||||||
|
err := os.MkdirAll(kustomizePath, 0777)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Couldn't create %s", kustomizePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
patchString := dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
namespace: kube-system
|
||||||
|
annotations:
|
||||||
|
kustomize: patch for kube-apiserver
|
||||||
|
`)
|
||||||
|
|
||||||
|
err = ioutil.WriteFile(filepath.Join(kustomizePath, "patch.yaml"), []byte(patchString), 0644)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile returned unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute createStaticPodFunction with kustomizations
|
||||||
|
manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
|
||||||
|
err = CreateStaticPodFiles(manifestPath, kustomizePath, cfg, &kubeadmapi.APIEndpoint{}, kubeadmconstants.KubeAPIServer)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Error executing createStaticPodFunction: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, fmt.Sprintf("%s.yaml", kubeadmconstants.KubeAPIServer)))
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Error executing ReadStaticPodFromDisk: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := pod.ObjectMeta.Annotations["kustomize"]; !ok {
|
||||||
|
t.Error("Kustomize did not apply patches corresponding to the resource")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGetAPIServerCommand(t *testing.T) {
|
func TestGetAPIServerCommand(t *testing.T) {
|
||||||
var tests = []struct {
|
var tests = []struct {
|
||||||
name string
|
name string
|
||||||
|
@ -14,7 +14,9 @@ go_test(
|
|||||||
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
||||||
"//cmd/kubeadm/app/constants:go_default_library",
|
"//cmd/kubeadm/app/constants:go_default_library",
|
||||||
"//cmd/kubeadm/app/util/etcd:go_default_library",
|
"//cmd/kubeadm/app/util/etcd:go_default_library",
|
||||||
|
"//cmd/kubeadm/app/util/staticpod:go_default_library",
|
||||||
"//cmd/kubeadm/test:go_default_library",
|
"//cmd/kubeadm/test:go_default_library",
|
||||||
|
"//vendor/github.com/lithammer/dedent:go_default_library",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -25,7 +25,7 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"k8s.io/klog"
|
"k8s.io/klog"
|
||||||
|
|
||||||
"k8s.io/api/core/v1"
|
v1 "k8s.io/api/core/v1"
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||||
@ -45,13 +45,22 @@ const (
|
|||||||
// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
|
// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
|
||||||
// This function is used by init - when the etcd cluster is empty - or by kubeadm
|
// This function is used by init - when the etcd cluster is empty - or by kubeadm
|
||||||
// upgrade - when the etcd cluster is already up and running (and the --initial-cluster flag have no impact)
|
// upgrade - when the etcd cluster is already up and running (and the --initial-cluster flag have no impact)
|
||||||
func CreateLocalEtcdStaticPodManifestFile(manifestDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
|
func CreateLocalEtcdStaticPodManifestFile(manifestDir, kustomizeDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
|
||||||
if cfg.Etcd.External != nil {
|
if cfg.Etcd.External != nil {
|
||||||
return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd")
|
return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd")
|
||||||
}
|
}
|
||||||
// gets etcd StaticPodSpec
|
// gets etcd StaticPodSpec
|
||||||
spec := GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{})
|
spec := GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{})
|
||||||
|
|
||||||
|
// if kustomizeDir is defined, customize the static pod manifest
|
||||||
|
if kustomizeDir != "" {
|
||||||
|
kustomizedSpec, err := staticpodutil.KustomizeStaticPod(&spec, kustomizeDir)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "failed to kustomize static pod manifest file for %q", kubeadmconstants.Etcd)
|
||||||
|
}
|
||||||
|
spec = *kustomizedSpec
|
||||||
|
}
|
||||||
|
|
||||||
// writes etcd StaticPod to disk
|
// writes etcd StaticPod to disk
|
||||||
if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
|
if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -18,15 +18,19 @@ package etcd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"sort"
|
"sort"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/lithammer/dedent"
|
||||||
|
|
||||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||||
etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
|
etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
|
||||||
|
staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
|
||||||
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
|
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -92,7 +96,7 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) {
|
|||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
// Execute createStaticPodFunction
|
// Execute createStaticPodFunction
|
||||||
manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
|
manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
|
||||||
err := CreateLocalEtcdStaticPodManifestFile(manifestPath, "", test.cfg, &kubeadmapi.APIEndpoint{})
|
err := CreateLocalEtcdStaticPodManifestFile(manifestPath, "", "", test.cfg, &kubeadmapi.APIEndpoint{})
|
||||||
|
|
||||||
if !test.expectedError {
|
if !test.expectedError {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -107,6 +111,61 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCreateLocalEtcdStaticPodManifestFileKustomize(t *testing.T) {
|
||||||
|
// Create temp folder for the test case
|
||||||
|
tmpdir := testutil.SetupTempDir(t)
|
||||||
|
defer os.RemoveAll(tmpdir)
|
||||||
|
|
||||||
|
// Creates a Cluster Configuration
|
||||||
|
cfg := &kubeadmapi.ClusterConfiguration{
|
||||||
|
KubernetesVersion: "v1.7.0",
|
||||||
|
Etcd: kubeadmapi.Etcd{
|
||||||
|
Local: &kubeadmapi.LocalEtcd{
|
||||||
|
DataDir: tmpdir + "/etcd",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
kustomizePath := filepath.Join(tmpdir, "kustomize")
|
||||||
|
err := os.MkdirAll(kustomizePath, 0777)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Couldn't create %s", kustomizePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
patchString := dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: etcd
|
||||||
|
namespace: kube-system
|
||||||
|
annotations:
|
||||||
|
kustomize: patch for etcd
|
||||||
|
`)
|
||||||
|
|
||||||
|
err = ioutil.WriteFile(filepath.Join(kustomizePath, "patch.yaml"), []byte(patchString), 0644)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile returned unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute createStaticPodFunction with kustomizations
|
||||||
|
manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
|
||||||
|
err = CreateLocalEtcdStaticPodManifestFile(manifestPath, kustomizePath, "", cfg, &kubeadmapi.APIEndpoint{})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Error executing createStaticPodFunction: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, fmt.Sprintf("%s.yaml", kubeadmconstants.Etcd)))
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Error executing ReadStaticPodFromDisk: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := pod.ObjectMeta.Annotations["kustomize"]; !ok {
|
||||||
|
t.Error("Kustomize did not apply patches corresponding to the resource")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGetEtcdCommand(t *testing.T) {
|
func TestGetEtcdCommand(t *testing.T) {
|
||||||
var tests = []struct {
|
var tests = []struct {
|
||||||
name string
|
name string
|
||||||
|
@ -314,7 +314,11 @@ func performEtcdStaticPodUpgrade(certsRenewMgr *renewal.Manager, client clientse
|
|||||||
|
|
||||||
// Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change
|
// Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change
|
||||||
// has occurred in any aspects.
|
// has occurred in any aspects.
|
||||||
if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
|
|
||||||
|
// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
|
||||||
|
kustomizeDir := ""
|
||||||
|
|
||||||
|
if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), kustomizeDir, cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
|
||||||
return true, errors.Wrap(err, "error creating local etcd static pod manifest file")
|
return true, errors.Wrap(err, "error creating local etcd static pod manifest file")
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -460,7 +464,11 @@ func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter,
|
|||||||
|
|
||||||
// Write the updated static Pod manifests into the temporary directory
|
// Write the updated static Pod manifests into the temporary directory
|
||||||
fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir())
|
fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir())
|
||||||
err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), cfg)
|
|
||||||
|
// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
|
||||||
|
kustomizeDir := ""
|
||||||
|
|
||||||
|
err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), kustomizeDir, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "error creating init static pod manifest files")
|
return errors.Wrap(err, "error creating init static pod manifest files")
|
||||||
}
|
}
|
||||||
@ -612,7 +620,10 @@ func DryRunStaticPodUpgrade(internalcfg *kubeadmapi.InitConfiguration) error {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(dryRunManifestDir)
|
defer os.RemoveAll(dryRunManifestDir)
|
||||||
|
|
||||||
if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, internalcfg); err != nil {
|
// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
|
||||||
|
kustomizeDir := ""
|
||||||
|
|
||||||
|
if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, kustomizeDir, internalcfg); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -512,11 +512,11 @@ func TestStaticPodControlPlane(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method
|
// Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method
|
||||||
err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg)
|
err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), "", oldcfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err)
|
t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err)
|
||||||
}
|
}
|
||||||
err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint)
|
err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), "", oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("couldn't run CreateLocalEtcdStaticPodManifestFile: %v", err)
|
t.Fatalf("couldn't run CreateLocalEtcdStaticPodManifestFile: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -83,6 +83,7 @@ filegroup(
|
|||||||
"//cmd/kubeadm/app/util/etcd:all-srcs",
|
"//cmd/kubeadm/app/util/etcd:all-srcs",
|
||||||
"//cmd/kubeadm/app/util/initsystem:all-srcs",
|
"//cmd/kubeadm/app/util/initsystem:all-srcs",
|
||||||
"//cmd/kubeadm/app/util/kubeconfig:all-srcs",
|
"//cmd/kubeadm/app/util/kubeconfig:all-srcs",
|
||||||
|
"//cmd/kubeadm/app/util/kustomize:all-srcs",
|
||||||
"//cmd/kubeadm/app/util/pkiutil:all-srcs",
|
"//cmd/kubeadm/app/util/pkiutil:all-srcs",
|
||||||
"//cmd/kubeadm/app/util/pubkeypin:all-srcs",
|
"//cmd/kubeadm/app/util/pubkeypin:all-srcs",
|
||||||
"//cmd/kubeadm/app/util/runtime:all-srcs",
|
"//cmd/kubeadm/app/util/runtime:all-srcs",
|
||||||
|
49
cmd/kubeadm/app/util/kustomize/BUILD
Normal file
49
cmd/kubeadm/app/util/kustomize/BUILD
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||||
|
|
||||||
|
go_library(
|
||||||
|
name = "go_default_library",
|
||||||
|
srcs = [
|
||||||
|
"kustomize.go",
|
||||||
|
"unstructured.go",
|
||||||
|
],
|
||||||
|
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/kustomize",
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
deps = [
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
|
||||||
|
"//staging/src/k8s.io/cli-runtime/pkg/kustomize:go_default_library",
|
||||||
|
"//vendor/github.com/pkg/errors:go_default_library",
|
||||||
|
"//vendor/sigs.k8s.io/kustomize/pkg/fs:go_default_library",
|
||||||
|
"//vendor/sigs.k8s.io/kustomize/pkg/ifc:go_default_library",
|
||||||
|
"//vendor/sigs.k8s.io/kustomize/pkg/loader:go_default_library",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
filegroup(
|
||||||
|
name = "package-srcs",
|
||||||
|
srcs = glob(["**"]),
|
||||||
|
tags = ["automanaged"],
|
||||||
|
visibility = ["//visibility:private"],
|
||||||
|
)
|
||||||
|
|
||||||
|
filegroup(
|
||||||
|
name = "all-srcs",
|
||||||
|
srcs = [":package-srcs"],
|
||||||
|
tags = ["automanaged"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
go_test(
|
||||||
|
name = "go_default_test",
|
||||||
|
srcs = [
|
||||||
|
"kustomize_test.go",
|
||||||
|
"unstructured_test.go",
|
||||||
|
],
|
||||||
|
embed = [":go_default_library"],
|
||||||
|
deps = [
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
|
||||||
|
"//vendor/github.com/lithammer/dedent:go_default_library",
|
||||||
|
],
|
||||||
|
)
|
181
cmd/kubeadm/app/util/kustomize/kustomize.go
Normal file
181
cmd/kubeadm/app/util/kustomize/kustomize.go
Normal file
@ -0,0 +1,181 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Package kustomize contains helpers for working with embedded kustomize commands
|
||||||
|
package kustomize
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"k8s.io/cli-runtime/pkg/kustomize"
|
||||||
|
"sigs.k8s.io/kustomize/pkg/fs"
|
||||||
|
"sigs.k8s.io/kustomize/pkg/loader"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Manager define a manager that allow access to kustomize capabilities
|
||||||
|
type Manager struct {
|
||||||
|
kustomizeDir string
|
||||||
|
us UnstructuredSlice
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
lock = &sync.Mutex{}
|
||||||
|
instances = map[string]*Manager{}
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetManager return the KustomizeManager singleton instance
|
||||||
|
func GetManager(kustomizeDir string) (*Manager, error) {
|
||||||
|
lock.Lock()
|
||||||
|
defer lock.Unlock()
|
||||||
|
|
||||||
|
// if the instance does not exists, create it
|
||||||
|
if _, ok := instances[kustomizeDir]; !ok {
|
||||||
|
km := &Manager{
|
||||||
|
kustomizeDir: kustomizeDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
// loads the UnstructuredSlice with all the patches into the Manager
|
||||||
|
// NB. this is done at singleton instance level because kubeadm has a unique pool
|
||||||
|
// of patches that are applied to different content, at different time
|
||||||
|
if err := km.getUnstructuredSlice(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
instances[kustomizeDir] = km
|
||||||
|
}
|
||||||
|
|
||||||
|
return instances[kustomizeDir], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getUnstructuredSlice returns a UnstructuredSlice with all the patches.
|
||||||
|
func (km *Manager) getUnstructuredSlice() error {
|
||||||
|
// kubeadm does not require a kustomization.yaml file listing all the resources/patches, so it is necessary
|
||||||
|
// to rebuild the list of patches manually
|
||||||
|
// TODO: make this git friendly - currently this works only for patches in local folders -
|
||||||
|
files, err := ioutil.ReadDir(km.kustomizeDir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var paths = []string{}
|
||||||
|
for _, file := range files {
|
||||||
|
if file.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
paths = append(paths, file.Name())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a loader that mimics the behavior of kubectl kustomize, including support for reading from
|
||||||
|
// a local git repository like git@github.com:someOrg/someRepo.git or https://github.com/someOrg/someRepo?ref=someHash
|
||||||
|
fSys := fs.MakeRealFS()
|
||||||
|
ldr, err := loader.NewLoader(km.kustomizeDir, fSys)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer ldr.Cleanup()
|
||||||
|
|
||||||
|
// read all the kustomizations and build the UnstructuredSlice
|
||||||
|
us, err := NewUnstructuredSliceFromFiles(ldr, paths)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
km.us = us
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kustomize apply a set of patches to a resource.
|
||||||
|
// Portions of the kustomize logic in this function are taken from the kubernetes-sigs/kind project
|
||||||
|
func (km *Manager) Kustomize(res []byte) ([]byte, error) {
|
||||||
|
// create a loader that mimics the behavior of kubectl kustomize
|
||||||
|
// and converts the resource into a UnstructuredSlice
|
||||||
|
// Nb. in kubeadm we are controlling resource generation, and so we
|
||||||
|
// we are expecting 1 object into each resource, eg. the static pod.
|
||||||
|
// Nevertheless, this code is ready for more than one object per resource
|
||||||
|
resList, err := NewUnstructuredSliceFromBytes(res)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// create a list of resource and corresponding patches
|
||||||
|
var resources, patches UnstructuredSlice
|
||||||
|
for _, r := range resList {
|
||||||
|
resources = append(resources, r)
|
||||||
|
|
||||||
|
resourcePatches := km.us.FilterResource(r.GroupVersionKind(), r.GetNamespace(), r.GetName())
|
||||||
|
patches = append(patches, resourcePatches...)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("[kustomize] Applying %d patches\n", len(patches))
|
||||||
|
|
||||||
|
// if there are no patches, for the target resources, exit
|
||||||
|
if len(patches) == 0 {
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// create an in memory fs to use for the kustomization
|
||||||
|
memFS := fs.MakeFakeFS()
|
||||||
|
|
||||||
|
var kustomization bytes.Buffer
|
||||||
|
fakeDir := "/"
|
||||||
|
// for Windows we need this to be a drive because kustomize uses filepath.Abs()
|
||||||
|
// which will add a drive letter if there is none. which drive letter is
|
||||||
|
// unimportant as the path is on the fake filesystem anyhow
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
fakeDir = `C:\`
|
||||||
|
}
|
||||||
|
|
||||||
|
// write resources and patches to the in memory fs, generate the kustomization.yaml
|
||||||
|
// that ties everything together
|
||||||
|
kustomization.WriteString("resources:\n")
|
||||||
|
for i, r := range resources {
|
||||||
|
b, err := r.MarshalJSON()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
name := fmt.Sprintf("resource-%d.json", i)
|
||||||
|
_ = memFS.WriteFile(filepath.Join(fakeDir, name), b)
|
||||||
|
fmt.Fprintf(&kustomization, " - %s\n", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
kustomization.WriteString("patches:\n")
|
||||||
|
for i, p := range patches {
|
||||||
|
b, err := p.MarshalJSON()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
name := fmt.Sprintf("patch-%d.json", i)
|
||||||
|
_ = memFS.WriteFile(filepath.Join(fakeDir, name), b)
|
||||||
|
fmt.Fprintf(&kustomization, " - %s\n", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
memFS.WriteFile(filepath.Join(fakeDir, "kustomization.yaml"), kustomization.Bytes())
|
||||||
|
|
||||||
|
// Finally customize the target resource
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := kustomize.RunKustomizeBuild(&out, memFS, fakeDir); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return out.Bytes(), nil
|
||||||
|
}
|
88
cmd/kubeadm/app/util/kustomize/kustomize_test.go
Normal file
88
cmd/kubeadm/app/util/kustomize/kustomize_test.go
Normal file
@ -0,0 +1,88 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package kustomize
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/lithammer/dedent"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestKustomize(t *testing.T) {
|
||||||
|
tmpdir, err := ioutil.TempDir("", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Couldn't create tmpdir")
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tmpdir)
|
||||||
|
|
||||||
|
resourceString := dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
`)
|
||||||
|
|
||||||
|
patch1String := dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
annotations:
|
||||||
|
kustomize: patch for kube-apiserver
|
||||||
|
`)
|
||||||
|
|
||||||
|
err = ioutil.WriteFile(filepath.Join(tmpdir, "patch-1.yaml"), []byte(patch1String), 0644)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile returned unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
patch2String := dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-scheduler
|
||||||
|
annotations:
|
||||||
|
kustomize: patch for kube-scheduler
|
||||||
|
`)
|
||||||
|
|
||||||
|
err = ioutil.WriteFile(filepath.Join(tmpdir, "patch-2.yaml"), []byte(patch2String), 0644)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile returned unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
km, err := GetManager(tmpdir)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("GetManager returned unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
kustomized, err := km.Kustomize([]byte(resourceString))
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Kustomize returned unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(string(kustomized), "kustomize: patch for kube-apiserver") {
|
||||||
|
t.Error("Kustomize did not apply patches corresponding to the resource")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(string(kustomized), "kustomize: patch for kube-scheduler") {
|
||||||
|
t.Error("Kustomize did apply patches not corresponding to the resource")
|
||||||
|
}
|
||||||
|
}
|
148
cmd/kubeadm/app/util/kustomize/unstructured.go
Normal file
148
cmd/kubeadm/app/util/kustomize/unstructured.go
Normal file
@ -0,0 +1,148 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package kustomize
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
|
"k8s.io/apimachinery/pkg/util/yaml"
|
||||||
|
"sigs.k8s.io/kustomize/pkg/ifc"
|
||||||
|
)
|
||||||
|
|
||||||
|
// UnstructuredSlice is a slice of Unstructured objects.
|
||||||
|
// Unstructured objects are used to represent both resources and patches of any group/version/kind.
|
||||||
|
type UnstructuredSlice []*unstructured.Unstructured
|
||||||
|
|
||||||
|
// NewUnstructuredSliceFromFiles returns a ResMap given a resource path slice.
|
||||||
|
// This func use a Loader to mimic the behavior of kubectl kustomize, and most specifically support for reading from
|
||||||
|
// a local git repository like git@github.com:someOrg/someRepo.git or https://github.com/someOrg/someRepo?ref=someHash
|
||||||
|
func NewUnstructuredSliceFromFiles(loader ifc.Loader, paths []string) (UnstructuredSlice, error) {
|
||||||
|
var result UnstructuredSlice
|
||||||
|
for _, path := range paths {
|
||||||
|
content, err := loader.Load(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "load from path %q failed", path)
|
||||||
|
}
|
||||||
|
res, err := NewUnstructuredSliceFromBytes(content)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "convert %q to Unstructured failed", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
result = append(result, res...)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewUnstructuredSliceFromBytes returns a slice of Unstructured.
|
||||||
|
// This functions handles all the nuances of Kubernetes yaml (e.g. many yaml
|
||||||
|
// documents in one file, List of objects)
|
||||||
|
func NewUnstructuredSliceFromBytes(in []byte) (UnstructuredSlice, error) {
|
||||||
|
decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(in), 1024)
|
||||||
|
var result UnstructuredSlice
|
||||||
|
var err error
|
||||||
|
// Parse all the yaml documents in the file
|
||||||
|
for err == nil || isEmptyYamlError(err) {
|
||||||
|
var u unstructured.Unstructured
|
||||||
|
err = decoder.Decode(&u)
|
||||||
|
// if the yaml document is a valid unstructured object
|
||||||
|
if err == nil {
|
||||||
|
// it the unstructured object is empty, move to the next
|
||||||
|
if len(u.Object) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// validate the object has kind, metadata.name as required by Kustomize
|
||||||
|
if err := validate(u); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// if the document is a list of objects
|
||||||
|
if strings.HasSuffix(u.GetKind(), "List") {
|
||||||
|
// for each item in the list of objects
|
||||||
|
if err := u.EachListItem(func(item runtime.Object) error {
|
||||||
|
// Marshal the object
|
||||||
|
itemJSON, err := json.Marshal(item)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the UnstructuredSlice for the item
|
||||||
|
itemU, err := NewUnstructuredSliceFromBytes(itemJSON)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// append the UnstructuredSlice for the item to the UnstructuredSlice
|
||||||
|
result = append(result, itemU...)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// append the object to the UnstructuredSlice
|
||||||
|
result = append(result, &u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != io.EOF {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FilterResource returns all the Unstructured items in the UnstructuredSlice corresponding to a given resource
|
||||||
|
func (rs *UnstructuredSlice) FilterResource(gvk schema.GroupVersionKind, namespace, name string) UnstructuredSlice {
|
||||||
|
var result UnstructuredSlice
|
||||||
|
for _, r := range *rs {
|
||||||
|
if r.GroupVersionKind() == gvk &&
|
||||||
|
r.GetNamespace() == namespace &&
|
||||||
|
r.GetName() == name {
|
||||||
|
result = append(result, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// validate validates that u has kind and name
|
||||||
|
// except for kind `List`, which doesn't require a name
|
||||||
|
func validate(u unstructured.Unstructured) error {
|
||||||
|
kind := u.GetKind()
|
||||||
|
if kind == "" {
|
||||||
|
return errors.New("missing kind in object")
|
||||||
|
} else if strings.HasSuffix(kind, "List") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if u.GetName() == "" {
|
||||||
|
return errors.New("missing metadata.name in object")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isEmptyYamlError(err error) bool {
|
||||||
|
return strings.Contains(err.Error(), "is missing in 'null'")
|
||||||
|
}
|
222
cmd/kubeadm/app/util/kustomize/unstructured_test.go
Normal file
222
cmd/kubeadm/app/util/kustomize/unstructured_test.go
Normal file
@ -0,0 +1,222 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package kustomize
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/lithammer/dedent"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewUnstructuredSliceFromBytes(t *testing.T) {
|
||||||
|
var useCases = []struct {
|
||||||
|
name string
|
||||||
|
in string
|
||||||
|
expectedUnctructured int
|
||||||
|
expectedError bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty",
|
||||||
|
in: "",
|
||||||
|
expectedUnctructured: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "single patch",
|
||||||
|
in: dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
`),
|
||||||
|
expectedUnctructured: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "two patches as separated yaml documents",
|
||||||
|
in: dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
`),
|
||||||
|
expectedUnctructured: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "two patches as a k8s list",
|
||||||
|
in: dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: List
|
||||||
|
items:
|
||||||
|
- apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
- apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
`),
|
||||||
|
expectedUnctructured: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested k8s lists",
|
||||||
|
in: dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: List
|
||||||
|
items:
|
||||||
|
- apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
- apiVersion: v1
|
||||||
|
kind: List
|
||||||
|
items:
|
||||||
|
- apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
`),
|
||||||
|
expectedUnctructured: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid yaml",
|
||||||
|
in: "$$$",
|
||||||
|
expectedError: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid patch (missing kind)",
|
||||||
|
in: dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
#kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
`),
|
||||||
|
expectedError: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid patch (missing name)",
|
||||||
|
in: dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
#name: kube-apiserver
|
||||||
|
`),
|
||||||
|
expectedError: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, rt := range useCases {
|
||||||
|
t.Run(rt.name, func(t *testing.T) {
|
||||||
|
r, err := NewUnstructuredSliceFromBytes([]byte(rt.in))
|
||||||
|
if err != nil {
|
||||||
|
if !rt.expectedError {
|
||||||
|
t.Errorf("NewUnstructuredSliceFromBytes returned unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil && rt.expectedError {
|
||||||
|
t.Error("NewUnstructuredSliceFromBytes does not returned expected error")
|
||||||
|
}
|
||||||
|
if len(r) != rt.expectedUnctructured {
|
||||||
|
t.Errorf("Expected %d Unstructured items in the slice, actual %d", rt.expectedUnctructured, len(r))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilterResource(t *testing.T) {
|
||||||
|
in := dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
namespace: kube-system
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-scheduler
|
||||||
|
namespace: kube-system
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-scheduler
|
||||||
|
namespace: kube-system
|
||||||
|
`)
|
||||||
|
u, err := NewUnstructuredSliceFromBytes([]byte(in))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUnstructuredSliceFromBytes returned unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var useCases = []struct {
|
||||||
|
name string
|
||||||
|
rgvk schema.GroupVersionKind
|
||||||
|
rnamespace string
|
||||||
|
rname string
|
||||||
|
expectedUnctructured int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "match 1",
|
||||||
|
rgvk: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"},
|
||||||
|
rnamespace: "kube-system",
|
||||||
|
rname: "kube-apiserver",
|
||||||
|
expectedUnctructured: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "match 2",
|
||||||
|
rgvk: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"},
|
||||||
|
rnamespace: "kube-system",
|
||||||
|
rname: "kube-scheduler",
|
||||||
|
expectedUnctructured: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "match 0 (wrong gvk)",
|
||||||
|
rgvk: schema.GroupVersionKind{Group: "something", Version: "v1", Kind: "Pod"},
|
||||||
|
rnamespace: "kube-system",
|
||||||
|
rname: "kube-scheduler",
|
||||||
|
expectedUnctructured: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "match 0 (wrong namespace)",
|
||||||
|
rgvk: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"},
|
||||||
|
rnamespace: "kube-something",
|
||||||
|
rname: "kube-scheduler",
|
||||||
|
expectedUnctructured: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "match 0 (wrong namr)",
|
||||||
|
rgvk: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"},
|
||||||
|
rnamespace: "kube-system",
|
||||||
|
rname: "kube-something",
|
||||||
|
expectedUnctructured: 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, rt := range useCases {
|
||||||
|
t.Run(rt.name, func(t *testing.T) {
|
||||||
|
r := u.FilterResource(rt.rgvk, rt.rnamespace, rt.rname)
|
||||||
|
|
||||||
|
if len(r) != rt.expectedUnctructured {
|
||||||
|
t.Errorf("Expected %d Unstructured items in the slice, actual %d", rt.expectedUnctructured, len(r))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
@ -15,6 +15,7 @@ go_test(
|
|||||||
"//cmd/kubeadm/test:go_default_library",
|
"//cmd/kubeadm/test:go_default_library",
|
||||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||||
|
"//vendor/github.com/lithammer/dedent:go_default_library",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -26,6 +27,7 @@ go_library(
|
|||||||
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
||||||
"//cmd/kubeadm/app/constants:go_default_library",
|
"//cmd/kubeadm/app/constants:go_default_library",
|
||||||
"//cmd/kubeadm/app/util:go_default_library",
|
"//cmd/kubeadm/app/util:go_default_library",
|
||||||
|
"//cmd/kubeadm/app/util/kustomize:go_default_library",
|
||||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||||
|
@ -28,12 +28,13 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"k8s.io/api/core/v1"
|
v1 "k8s.io/api/core/v1"
|
||||||
"k8s.io/apimachinery/pkg/api/resource"
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||||
"k8s.io/kubernetes/cmd/kubeadm/app/util"
|
"k8s.io/kubernetes/cmd/kubeadm/app/util"
|
||||||
|
"k8s.io/kubernetes/cmd/kubeadm/app/util/kustomize"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -165,6 +166,38 @@ func GetExtraParameters(overrides map[string]string, defaults map[string]string)
|
|||||||
return command
|
return command
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// KustomizeStaticPod applies patches defined in kustomizeDir to a static Pod manifest
|
||||||
|
func KustomizeStaticPod(pod *v1.Pod, kustomizeDir string) (*v1.Pod, error) {
|
||||||
|
// marshal the pod manifest into yaml
|
||||||
|
serialized, err := util.MarshalToYaml(pod, v1.SchemeGroupVersion)
|
||||||
|
if err != nil {
|
||||||
|
return pod, errors.Wrapf(err, "failed to marshal manifest to YAML")
|
||||||
|
}
|
||||||
|
|
||||||
|
km, err := kustomize.GetManager(kustomizeDir)
|
||||||
|
if err != nil {
|
||||||
|
return pod, errors.Wrapf(err, "failed to GetPatches from %q", kustomizeDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
kustomized, err := km.Kustomize(serialized)
|
||||||
|
if err != nil {
|
||||||
|
return pod, errors.Wrap(err, "failed to kustomize static Pod manifest")
|
||||||
|
}
|
||||||
|
|
||||||
|
// unmarshal kustomized yaml back into a pod manifest
|
||||||
|
obj, err := util.UnmarshalFromYaml(kustomized, v1.SchemeGroupVersion)
|
||||||
|
if err != nil {
|
||||||
|
return pod, errors.Wrap(err, "failed to unmarshal kustomize manifest from YAML")
|
||||||
|
}
|
||||||
|
|
||||||
|
pod2, ok := obj.(*v1.Pod)
|
||||||
|
if !ok {
|
||||||
|
return pod, errors.Wrap(err, "kustomized manifest is not a valid Pod object")
|
||||||
|
}
|
||||||
|
|
||||||
|
return pod2, nil
|
||||||
|
}
|
||||||
|
|
||||||
// WriteStaticPodToDisk writes a static pod file to disk
|
// WriteStaticPodToDisk writes a static pod file to disk
|
||||||
func WriteStaticPodToDisk(componentName, manifestDir string, pod v1.Pod) error {
|
func WriteStaticPodToDisk(componentName, manifestDir string, pod v1.Pod) error {
|
||||||
|
|
||||||
|
@ -25,7 +25,9 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"k8s.io/api/core/v1"
|
"github.com/lithammer/dedent"
|
||||||
|
|
||||||
|
v1 "k8s.io/api/core/v1"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||||
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
|
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
|
||||||
@ -604,3 +606,44 @@ func TestManifestFilesAreEqual(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestKustomizeStaticPod(t *testing.T) {
|
||||||
|
// Create temp folder for the test case
|
||||||
|
tmpdir := testutil.SetupTempDir(t)
|
||||||
|
defer os.RemoveAll(tmpdir)
|
||||||
|
|
||||||
|
patchString := dedent.Dedent(`
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-apiserver
|
||||||
|
namespace: kube-system
|
||||||
|
annotations:
|
||||||
|
kustomize: patch for kube-apiserver
|
||||||
|
`)
|
||||||
|
|
||||||
|
err := ioutil.WriteFile(filepath.Join(tmpdir, "patch.yaml"), []byte(patchString), 0644)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile returned unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
pod := &v1.Pod{
|
||||||
|
TypeMeta: metav1.TypeMeta{
|
||||||
|
APIVersion: "v1",
|
||||||
|
Kind: "Pod",
|
||||||
|
},
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "kube-apiserver",
|
||||||
|
Namespace: "kube-system",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
kpod, err := KustomizeStaticPod(pod, tmpdir)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("KustomizeStaticPod returned unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := kpod.ObjectMeta.Annotations["kustomize"]; !ok {
|
||||||
|
t.Error("Kustomize did not apply patches corresponding to the resource")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
1
go.mod
1
go.mod
@ -168,6 +168,7 @@ require (
|
|||||||
k8s.io/repo-infra v0.0.0-20181204233714-00fe14e3d1a3
|
k8s.io/repo-infra v0.0.0-20181204233714-00fe14e3d1a3
|
||||||
k8s.io/sample-apiserver v0.0.0
|
k8s.io/sample-apiserver v0.0.0
|
||||||
k8s.io/utils v0.0.0-20190801114015-581e00157fb1
|
k8s.io/utils v0.0.0-20190801114015-581e00157fb1
|
||||||
|
sigs.k8s.io/kustomize v2.0.3+incompatible
|
||||||
sigs.k8s.io/yaml v1.1.0
|
sigs.k8s.io/yaml v1.1.0
|
||||||
vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc
|
vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc
|
||||||
)
|
)
|
||||||
|
Loading…
Reference in New Issue
Block a user