From 7eb49628c620861ee56f84fcd9d286263227af8a Mon Sep 17 00:00:00 2001 From: Michael Fraenkel Date: Fri, 9 Dec 2016 10:35:23 -0700 Subject: [PATCH 1/2] create configmap from-env-file --- hack/verify-flags/known-flags.txt | 1 + pkg/kubectl/cmd/create_configmap.go | 10 ++- pkg/kubectl/configmap.go | 38 ++++++++++++ pkg/kubectl/configmap_test.go | 96 +++++++++++++++++++++++++++++ pkg/kubectl/env_file.go | 77 +++++++++++++++++++++++ 5 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 pkg/kubectl/env_file.go diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 3927af4e763..9d8de311429 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -271,6 +271,7 @@ forward-services framework-name framework-store-uri framework-weburi +from-env-file from-file from-literal func-dest diff --git a/pkg/kubectl/cmd/create_configmap.go b/pkg/kubectl/cmd/create_configmap.go index f095c00d9d3..c2054f4c33a 100644 --- a/pkg/kubectl/cmd/create_configmap.go +++ b/pkg/kubectl/cmd/create_configmap.go @@ -49,7 +49,13 @@ var ( kubectl create configmap my-config --from-file=key1=/path/to/bar/file1.txt --from-file=key2=/path/to/bar/file2.txt # Create a new configmap named my-config with key1=config1 and key2=config2 - kubectl create configmap my-config --from-literal=key1=config1 --from-literal=key2=config2`) + kubectl create configmap my-config --from-literal=key1=config1 --from-literal=key2=config2 + + # Create a new configmap named my-config from the key=value pairs in the file + kubectl create configmap my-config --from-file=path/to/bar + + # Create a new configmap named my-config from an env file + kubectl create configmap my-config --from-env-file=path/to/bar.env`) ) // ConfigMap is a command to ease creating ConfigMaps. @@ -71,6 +77,7 @@ func NewCmdCreateConfigMap(f cmdutil.Factory, cmdOut io.Writer) *cobra.Command { cmdutil.AddGeneratorFlags(cmd, cmdutil.ConfigMapV1GeneratorName) cmd.Flags().StringSlice("from-file", []string{}, "Key file can be specified using its file path, in which case file basename will be used as configmap key, or optionally with a key and file path, in which case the given key will be used. Specifying a directory will iterate each named file in the directory whose basename is a valid configmap key.") cmd.Flags().StringArray("from-literal", []string{}, "Specify a key and literal value to insert in configmap (i.e. mykey=somevalue)") + cmd.Flags().String("from-env-file", "", "Specify the path to a file to read lines of key=val pairs to create a configmap (i.e. a Docker .env file).") return cmd } @@ -87,6 +94,7 @@ func CreateConfigMap(f cmdutil.Factory, cmdOut io.Writer, cmd *cobra.Command, ar Name: name, FileSources: cmdutil.GetFlagStringSlice(cmd, "from-file"), LiteralSources: cmdutil.GetFlagStringArray(cmd, "from-literal"), + EnvFileSource: cmdutil.GetFlagString(cmd, "from-env-file"), } default: return cmdutil.UsageError(cmd, fmt.Sprintf("Generator: %s not supported.", generatorName)) diff --git a/pkg/kubectl/configmap.go b/pkg/kubectl/configmap.go index 239c5ac7ecc..71e8c0c3b85 100644 --- a/pkg/kubectl/configmap.go +++ b/pkg/kubectl/configmap.go @@ -38,6 +38,8 @@ type ConfigMapGeneratorV1 struct { FileSources []string // LiteralSources to derive the configMap from (optional) LiteralSources []string + // EnvFileSource to derive the configMap from (optional) + EnvFileSource string } // Ensure it supports the generator pattern that uses parameter injection. @@ -79,6 +81,15 @@ func (s ConfigMapGeneratorV1) Generate(genericParams map[string]interface{}) (ru } params[key] = strVal } + fromEnvFileString, found := genericParams["from-env-file"] + if found { + fromEnvFile, isString := fromEnvFileString.(string) + if !isString { + return nil, fmt.Errorf("expected string, found :%v", fromEnvFileString) + } + delegate.EnvFileSource = fromEnvFile + delete(genericParams, "from-env-file") + } delegate.Name = params["name"] delegate.Type = params["type"] return delegate.StructuredGenerate() @@ -91,6 +102,7 @@ func (s ConfigMapGeneratorV1) ParamNames() []GeneratorParam { {"type", false}, {"from-file", false}, {"from-literal", false}, + {"from-env-file", false}, {"force", false}, } } @@ -113,6 +125,11 @@ func (s ConfigMapGeneratorV1) StructuredGenerate() (runtime.Object, error) { return nil, err } } + if len(s.EnvFileSource) > 0 { + if err := handleConfigMapFromEnvFileSource(configMap, s.EnvFileSource); err != nil { + return nil, err + } + } return configMap, nil } @@ -121,6 +138,9 @@ func (s ConfigMapGeneratorV1) validate() error { if len(s.Name) == 0 { return fmt.Errorf("name must be specified") } + if len(s.EnvFileSource) > 0 && (len(s.FileSources) > 0 || len(s.LiteralSources) > 0) { + return fmt.Errorf("from-env-file cannot be combined with from-file or from-literal") + } return nil } @@ -186,6 +206,24 @@ func handleConfigMapFromFileSources(configMap *api.ConfigMap, fileSources []stri return nil } +// handleConfigMapFromEnvFileSource adds the specified env file source information +// into the provided configMap +func handleConfigMapFromEnvFileSource(configMap *api.ConfigMap, envFileSource string) error { + info, err := os.Stat(envFileSource) + if err != nil { + switch err := err.(type) { + case *os.PathError: + return fmt.Errorf("error reading %s: %v", envFileSource, err.Err) + default: + return fmt.Errorf("error reading %s: %v", envFileSource, err) + } + } + if info.IsDir() { + return fmt.Errorf("must be a file") + } + return addFromEnvFileToConfigMap(configMap, envFileSource) +} + // addKeyFromFileToConfigMap adds a key with the given name to a ConfigMap, populating // the value with the content of the given file path, or returns an error. func addKeyFromFileToConfigMap(configMap *api.ConfigMap, keyName, filePath string) error { diff --git a/pkg/kubectl/configmap_test.go b/pkg/kubectl/configmap_test.go index e7ae10a0713..5690c2e0f0a 100644 --- a/pkg/kubectl/configmap_test.go +++ b/pkg/kubectl/configmap_test.go @@ -17,6 +17,8 @@ limitations under the License. package kubectl import ( + "io/ioutil" + "os" "reflect" "testing" @@ -26,6 +28,7 @@ import ( func TestConfigMapGenerate(t *testing.T) { tests := []struct { + setup func(t *testing.T, params map[string]interface{}) func() params map[string]interface{} expected *api.ConfigMap expectErr bool @@ -107,9 +110,84 @@ func TestConfigMapGenerate(t *testing.T) { }, expectErr: false, }, + { + setup: setupEnvFile("key1=value1", "#", "", "key2=value2"), + params: map[string]interface{}{ + "name": "valid_env", + "from-env-file": "file.env", + }, + expected: &api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "valid_env", + }, + Data: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + expectErr: false, + }, + { + setup: func() func(t *testing.T, params map[string]interface{}) func() { + os.Setenv("g_key1", "1") + os.Setenv("g_key2", "2") + return setupEnvFile("g_key1", "g_key2=") + }(), + params: map[string]interface{}{ + "name": "getenv", + "from-env-file": "file.env", + }, + expected: &api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "getenv", + }, + Data: map[string]string{ + "g_key1": "1", + "g_key2": "", + }, + }, + expectErr: false, + }, + { + params: map[string]interface{}{ + "name": "too_many_args", + "from-literal": []string{"key1=value1"}, + "from-env-file": "file.env", + }, + expectErr: true, + }, + { + setup: setupEnvFile("key.1=value1"), + params: map[string]interface{}{ + "name": "invalid_key", + "from-env-file": "file.env", + }, + expectErr: true, + }, + { + setup: setupEnvFile(" key1= value1"), + params: map[string]interface{}{ + "name": "with_spaces", + "from-env-file": "file.env", + }, + expected: &api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "with_spaces", + }, + Data: map[string]string{ + "key1": " value1", + }, + }, + expectErr: false, + }, } generator := ConfigMapGeneratorV1{} for _, test := range tests { + if test.setup != nil { + if teardown := test.setup(t, test.params); teardown != nil { + defer teardown() + } + } obj, err := generator.Generate(test.params) if !test.expectErr && err != nil { t.Errorf("unexpected error: %v", err) @@ -122,3 +200,21 @@ func TestConfigMapGenerate(t *testing.T) { } } } + +func setupEnvFile(lines ...string) func(*testing.T, map[string]interface{}) func() { + return func(t *testing.T, params map[string]interface{}) func() { + f, err := ioutil.TempFile("", "cme") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + for _, l := range lines { + f.WriteString(l) + f.WriteString("\r\n") + } + f.Close() + params["from-env-file"] = f.Name() + return func() { + os.Remove(f.Name()) + } + } +} diff --git a/pkg/kubectl/env_file.go b/pkg/kubectl/env_file.go new file mode 100644 index 00000000000..bbdcdc657ab --- /dev/null +++ b/pkg/kubectl/env_file.go @@ -0,0 +1,77 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubectl + +import ( + "bufio" + "bytes" + "fmt" + "os" + "strings" + "unicode" + "unicode/utf8" + + "k8s.io/apimachinery/pkg/util/validation" +) + +// addFromEnvFile processes an env file allows a generic addTo to handle the +// collection of key value pairs or returns an error. +func addFromEnvFile(filePath string, addTo func(key, value string) error) error { + f, err := os.Open(filePath) + if err != nil { + return err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + currentLine := 0 + utf8bom := []byte{0xEF, 0xBB, 0xBF} + for scanner.Scan() { + scannedBytes := scanner.Bytes() + if !utf8.Valid(scannedBytes) { + return fmt.Errorf("env file %s contains invalid utf8 bytes at line %d: %v", filePath, currentLine+1, scannedBytes) + } + // We trim UTF8 BOM + if currentLine == 0 { + scannedBytes = bytes.TrimPrefix(scannedBytes, utf8bom) + } + // trim the line from all leading whitespace first + line := strings.TrimLeftFunc(string(scannedBytes), unicode.IsSpace) + currentLine++ + // line is not empty, and not starting with '#' + if len(line) > 0 && !strings.HasPrefix(line, "#") { + data := strings.SplitN(line, "=", 2) + key := data[0] + if errs := validation.IsCIdentifier(key); len(errs) != 0 { + return fmt.Errorf("%q is not a valid key name: %s", key, strings.Join(errs, ";")) + } + + value := "" + if len(data) > 1 { + // pass the value through, no trimming + value = data[1] + } else { + // a pass-through variable is given + value = os.Getenv(key) + } + if err = addTo(key, value); err != nil { + return err + } + } + } + return nil +} From f2815156b0854ccebce8062f5dd5c5dd5c1a962f Mon Sep 17 00:00:00 2001 From: Michael Fraenkel Date: Thu, 12 Jan 2017 07:35:46 -0800 Subject: [PATCH 2/2] create secret from-env-file --- pkg/kubectl/BUILD | 1 + pkg/kubectl/cmd/create_secret.go | 7 ++- pkg/kubectl/configmap.go | 5 ++- pkg/kubectl/secret.go | 41 +++++++++++++++++ pkg/kubectl/secret_test.go | 77 ++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 2 deletions(-) diff --git a/pkg/kubectl/BUILD b/pkg/kubectl/BUILD index 98923897872..e1a70141f6c 100644 --- a/pkg/kubectl/BUILD +++ b/pkg/kubectl/BUILD @@ -19,6 +19,7 @@ go_library( "configmap.go", "deployment.go", "doc.go", + "env_file.go", "explain.go", "generate.go", "history.go", diff --git a/pkg/kubectl/cmd/create_secret.go b/pkg/kubectl/cmd/create_secret.go index 176703c9891..e79af190838 100644 --- a/pkg/kubectl/cmd/create_secret.go +++ b/pkg/kubectl/cmd/create_secret.go @@ -64,7 +64,10 @@ var ( kubectl create secret generic my-secret --from-file=ssh-privatekey=~/.ssh/id_rsa --from-file=ssh-publickey=~/.ssh/id_rsa.pub # Create a new secret named my-secret with key1=supersecret and key2=topsecret - kubectl create secret generic my-secret --from-literal=key1=supersecret --from-literal=key2=topsecret`) + kubectl create secret generic my-secret --from-literal=key1=supersecret --from-literal=key2=topsecret + + # Create a new secret named my-secret from an env file + kubectl create secret generic my-secret --from-env-file=path/to/bar.env`) ) // NewCmdCreateSecretGeneric is a command to create generic secrets from files, directories, or literal values @@ -85,6 +88,7 @@ func NewCmdCreateSecretGeneric(f cmdutil.Factory, cmdOut io.Writer) *cobra.Comma cmdutil.AddGeneratorFlags(cmd, cmdutil.SecretV1GeneratorName) cmd.Flags().StringSlice("from-file", []string{}, "Key files can be specified using their file path, in which case a default name will be given to them, or optionally with a name and file path, in which case the given name will be used. Specifying a directory will iterate each named file in the directory that is a valid secret key.") cmd.Flags().StringArray("from-literal", []string{}, "Specify a key and literal value to insert in secret (i.e. mykey=somevalue)") + cmd.Flags().String("from-env-file", "", "Specify the path to a file to read lines of key=val pairs to create a secret (i.e. a Docker .env file).") cmd.Flags().String("type", "", i18n.T("The type of secret to create")) return cmd } @@ -103,6 +107,7 @@ func CreateSecretGeneric(f cmdutil.Factory, cmdOut io.Writer, cmd *cobra.Command Type: cmdutil.GetFlagString(cmd, "type"), FileSources: cmdutil.GetFlagStringSlice(cmd, "from-file"), LiteralSources: cmdutil.GetFlagStringArray(cmd, "from-literal"), + EnvFileSource: cmdutil.GetFlagString(cmd, "from-env-file"), } default: return cmdutil.UsageError(cmd, fmt.Sprintf("Generator: %s not supported.", generatorName)) diff --git a/pkg/kubectl/configmap.go b/pkg/kubectl/configmap.go index 71e8c0c3b85..794903dd8e1 100644 --- a/pkg/kubectl/configmap.go +++ b/pkg/kubectl/configmap.go @@ -221,7 +221,10 @@ func handleConfigMapFromEnvFileSource(configMap *api.ConfigMap, envFileSource st if info.IsDir() { return fmt.Errorf("must be a file") } - return addFromEnvFileToConfigMap(configMap, envFileSource) + + return addFromEnvFile(envFileSource, func(key, value string) error { + return addKeyFromLiteralToConfigMap(configMap, key, value) + }) } // addKeyFromFileToConfigMap adds a key with the given name to a ConfigMap, populating diff --git a/pkg/kubectl/secret.go b/pkg/kubectl/secret.go index 70d436b05bc..1882adf4588 100644 --- a/pkg/kubectl/secret.go +++ b/pkg/kubectl/secret.go @@ -38,6 +38,8 @@ type SecretGeneratorV1 struct { FileSources []string // LiteralSources to derive the secret from (optional) LiteralSources []string + // EnvFileSource to derive the secret from (optional) + EnvFileSource string } // Ensure it supports the generator pattern that uses parameter injection @@ -71,6 +73,15 @@ func (s SecretGeneratorV1) Generate(genericParams map[string]interface{}) (runti delegate.LiteralSources = fromLiteralArray delete(genericParams, "from-literal") } + fromEnvFileString, found := genericParams["from-env-file"] + if found { + fromEnvFile, isString := fromEnvFileString.(string) + if !isString { + return nil, fmt.Errorf("expected string, found :%v", fromEnvFileString) + } + delegate.EnvFileSource = fromEnvFile + delete(genericParams, "from-env-file") + } params := map[string]string{} for key, value := range genericParams { strVal, isString := value.(string) @@ -91,6 +102,7 @@ func (s SecretGeneratorV1) ParamNames() []GeneratorParam { {"type", false}, {"from-file", false}, {"from-literal", false}, + {"from-env-file", false}, {"force", false}, } } @@ -116,6 +128,11 @@ func (s SecretGeneratorV1) StructuredGenerate() (runtime.Object, error) { return nil, err } } + if len(s.EnvFileSource) > 0 { + if err := handleFromEnvFileSource(secret, s.EnvFileSource); err != nil { + return nil, err + } + } return secret, nil } @@ -124,6 +141,9 @@ func (s SecretGeneratorV1) validate() error { if len(s.Name) == 0 { return fmt.Errorf("name must be specified") } + if len(s.EnvFileSource) > 0 && (len(s.FileSources) > 0 || len(s.LiteralSources) > 0) { + return fmt.Errorf("from-env-file cannot be combined with from-file or from-literal") + } return nil } @@ -187,6 +207,27 @@ func handleFromFileSources(secret *api.Secret, fileSources []string) error { return nil } +// handleFromEnvFileSource adds the specified env file source information +// into the provided secret +func handleFromEnvFileSource(secret *api.Secret, envFileSource string) error { + info, err := os.Stat(envFileSource) + if err != nil { + switch err := err.(type) { + case *os.PathError: + return fmt.Errorf("error reading %s: %v", envFileSource, err.Err) + default: + return fmt.Errorf("error reading %s: %v", envFileSource, err) + } + } + if info.IsDir() { + return fmt.Errorf("must be a file") + } + + return addFromEnvFile(envFileSource, func(key, value string) error { + return addKeyFromLiteralToSecret(secret, key, []byte(value)) + }) +} + func addKeyFromFileToSecret(secret *api.Secret, keyName, filePath string) error { data, err := ioutil.ReadFile(filePath) if err != nil { diff --git a/pkg/kubectl/secret_test.go b/pkg/kubectl/secret_test.go index cb9856d72ae..309c7848d65 100644 --- a/pkg/kubectl/secret_test.go +++ b/pkg/kubectl/secret_test.go @@ -17,6 +17,7 @@ limitations under the License. package kubectl import ( + "os" "reflect" "testing" @@ -26,6 +27,7 @@ import ( func TestSecretGenerate(t *testing.T) { tests := []struct { + setup func(t *testing.T, params map[string]interface{}) func() params map[string]interface{} expected *api.Secret expectErr bool @@ -108,9 +110,84 @@ func TestSecretGenerate(t *testing.T) { }, expectErr: false, }, + { + setup: setupEnvFile("key1=value1", "#", "", "key2=value2"), + params: map[string]interface{}{ + "name": "valid_env", + "from-env-file": "file.env", + }, + expected: &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "valid_env", + }, + Data: map[string][]byte{ + "key1": []byte("value1"), + "key2": []byte("value2"), + }, + }, + expectErr: false, + }, + { + setup: func() func(t *testing.T, params map[string]interface{}) func() { + os.Setenv("g_key1", "1") + os.Setenv("g_key2", "2") + return setupEnvFile("g_key1", "g_key2=") + }(), + params: map[string]interface{}{ + "name": "getenv", + "from-env-file": "file.env", + }, + expected: &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "getenv", + }, + Data: map[string][]byte{ + "g_key1": []byte("1"), + "g_key2": []byte(""), + }, + }, + expectErr: false, + }, + { + params: map[string]interface{}{ + "name": "too_many_args", + "from-literal": []string{"key1=value1"}, + "from-env-file": "file.env", + }, + expectErr: true, + }, + { + setup: setupEnvFile("key.1=value1"), + params: map[string]interface{}{ + "name": "invalid_key", + "from-env-file": "file.env", + }, + expectErr: true, + }, + { + setup: setupEnvFile(" key1= value1"), + params: map[string]interface{}{ + "name": "with_spaces", + "from-env-file": "file.env", + }, + expected: &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "with_spaces", + }, + Data: map[string][]byte{ + "key1": []byte(" value1"), + }, + }, + expectErr: false, + }, } generator := SecretGeneratorV1{} for _, test := range tests { + if test.setup != nil { + if teardown := test.setup(t, test.params); teardown != nil { + defer teardown() + } + } obj, err := generator.Generate(test.params) if !test.expectErr && err != nil { t.Errorf("unexpected error: %v", err)