Merge pull request #38882 from fraenkel/configmap_env_file

Automatic merge from submit-queue (batch tested with PRs 41139, 41186, 38882, 37698, 42034)

create configmap from-env-file

Allow ConfigMaps to be created from Docker based env files.

See proposal https://github.com/kubernetes/community/issues/165

**Release-note:**
```release-note
1. create configmap has a new option --from-env-file that populates a configmap from file which follows a key=val format for each line.
2. create secret has a new option --from-env-file that populates a configmap from file which follows a key=val format for each line.
```

hack/verify-flags/known-flags.txt

@@ -271,6 +271,7 @@ forward-services
 framework-name
 framework-store-uri
 framework-weburi
+from-env-file
 from-file
 from-literal
 func-dest

pkg/kubectl/BUILD

@@ -19,6 +19,7 @@ go_library(
         "configmap.go",
         "deployment.go",
         "doc.go",
+        "env_file.go",
         "explain.go",
         "generate.go",
         "history.go",

pkg/kubectl/cmd/create_configmap.go

@@ -49,7 +49,13 @@ var (
 		  kubectl create configmap my-config --from-file=key1=/path/to/bar/file1.txt --from-file=key2=/path/to/bar/file2.txt
 
 		  # Create a new configmap named my-config with key1=config1 and key2=config2
-		  kubectl create configmap my-config --from-literal=key1=config1 --from-literal=key2=config2`)
+		  kubectl create configmap my-config --from-literal=key1=config1 --from-literal=key2=config2
+
+		  # Create a new configmap named my-config from the key=value pairs in the file
+		  kubectl create configmap my-config --from-file=path/to/bar
+
+		  # Create a new configmap named my-config from an env file
+		  kubectl create configmap my-config --from-env-file=path/to/bar.env`)
 )
 
 // ConfigMap is a command to ease creating ConfigMaps.
@@ -71,6 +77,7 @@ func NewCmdCreateConfigMap(f cmdutil.Factory, cmdOut io.Writer) *cobra.Command {
 	cmdutil.AddGeneratorFlags(cmd, cmdutil.ConfigMapV1GeneratorName)
 	cmd.Flags().StringSlice("from-file", []string{}, "Key file can be specified using its file path, in which case file basename will be used as configmap key, or optionally with a key and file path, in which case the given key will be used.  Specifying a directory will iterate each named file in the directory whose basename is a valid configmap key.")
 	cmd.Flags().StringArray("from-literal", []string{}, "Specify a key and literal value to insert in configmap (i.e. mykey=somevalue)")
+	cmd.Flags().String("from-env-file", "", "Specify the path to a file to read lines of key=val pairs to create a configmap (i.e. a Docker .env file).")
 	return cmd
 }
 
@@ -87,6 +94,7 @@ func CreateConfigMap(f cmdutil.Factory, cmdOut io.Writer, cmd *cobra.Command, ar
 			Name:           name,
 			FileSources:    cmdutil.GetFlagStringSlice(cmd, "from-file"),
 			LiteralSources: cmdutil.GetFlagStringArray(cmd, "from-literal"),
+			EnvFileSource:  cmdutil.GetFlagString(cmd, "from-env-file"),
 		}
 	default:
 		return cmdutil.UsageError(cmd, fmt.Sprintf("Generator: %s not supported.", generatorName))

pkg/kubectl/cmd/create_secret.go

@@ -64,7 +64,10 @@ var (
 	  kubectl create secret generic my-secret --from-file=ssh-privatekey=~/.ssh/id_rsa --from-file=ssh-publickey=~/.ssh/id_rsa.pub
 
 	  # Create a new secret named my-secret with key1=supersecret and key2=topsecret
-	  kubectl create secret generic my-secret --from-literal=key1=supersecret --from-literal=key2=topsecret`)
+	  kubectl create secret generic my-secret --from-literal=key1=supersecret --from-literal=key2=topsecret
+
+	  # Create a new secret named my-secret from an env file
+	  kubectl create secret generic my-secret --from-env-file=path/to/bar.env`)
 )
 
 // NewCmdCreateSecretGeneric is a command to create generic secrets from files, directories, or literal values
@@ -85,6 +88,7 @@ func NewCmdCreateSecretGeneric(f cmdutil.Factory, cmdOut io.Writer) *cobra.Comma
 	cmdutil.AddGeneratorFlags(cmd, cmdutil.SecretV1GeneratorName)
 	cmd.Flags().StringSlice("from-file", []string{}, "Key files can be specified using their file path, in which case a default name will be given to them, or optionally with a name and file path, in which case the given name will be used.  Specifying a directory will iterate each named file in the directory that is a valid secret key.")
 	cmd.Flags().StringArray("from-literal", []string{}, "Specify a key and literal value to insert in secret (i.e. mykey=somevalue)")
+	cmd.Flags().String("from-env-file", "", "Specify the path to a file to read lines of key=val pairs to create a secret (i.e. a Docker .env file).")
 	cmd.Flags().String("type", "", i18n.T("The type of secret to create"))
 	return cmd
 }
@@ -103,6 +107,7 @@ func CreateSecretGeneric(f cmdutil.Factory, cmdOut io.Writer, cmd *cobra.Command
 			Type:           cmdutil.GetFlagString(cmd, "type"),
 			FileSources:    cmdutil.GetFlagStringSlice(cmd, "from-file"),
 			LiteralSources: cmdutil.GetFlagStringArray(cmd, "from-literal"),
+			EnvFileSource:  cmdutil.GetFlagString(cmd, "from-env-file"),
 		}
 	default:
 		return cmdutil.UsageError(cmd, fmt.Sprintf("Generator: %s not supported.", generatorName))

pkg/kubectl/configmap.go

@@ -38,6 +38,8 @@ type ConfigMapGeneratorV1 struct {
 	FileSources []string
 	// LiteralSources to derive the configMap from (optional)
 	LiteralSources []string
+	// EnvFileSource to derive the configMap from (optional)
+	EnvFileSource string
 }
 
 // Ensure it supports the generator pattern that uses parameter injection.
@@ -79,6 +81,15 @@ func (s ConfigMapGeneratorV1) Generate(genericParams map[string]interface{}) (ru
 		}
 		params[key] = strVal
 	}
+	fromEnvFileString, found := genericParams["from-env-file"]
+	if found {
+		fromEnvFile, isString := fromEnvFileString.(string)
+		if !isString {
+			return nil, fmt.Errorf("expected string, found :%v", fromEnvFileString)
+		}
+		delegate.EnvFileSource = fromEnvFile
+		delete(genericParams, "from-env-file")
+	}
 	delegate.Name = params["name"]
 	delegate.Type = params["type"]
 	return delegate.StructuredGenerate()
@@ -91,6 +102,7 @@ func (s ConfigMapGeneratorV1) ParamNames() []GeneratorParam {
 		{"type", false},
 		{"from-file", false},
 		{"from-literal", false},
+		{"from-env-file", false},
 		{"force", false},
 	}
 }
@@ -113,6 +125,11 @@ func (s ConfigMapGeneratorV1) StructuredGenerate() (runtime.Object, error) {
 			return nil, err
 		}
 	}
+	if len(s.EnvFileSource) > 0 {
+		if err := handleConfigMapFromEnvFileSource(configMap, s.EnvFileSource); err != nil {
+			return nil, err
+		}
+	}
 	return configMap, nil
 }
 
@@ -121,6 +138,9 @@ func (s ConfigMapGeneratorV1) validate() error {
 	if len(s.Name) == 0 {
 		return fmt.Errorf("name must be specified")
 	}
+	if len(s.EnvFileSource) > 0 && (len(s.FileSources) > 0 || len(s.LiteralSources) > 0) {
+		return fmt.Errorf("from-env-file cannot be combined with from-file or from-literal")
+	}
 	return nil
 }
 
@@ -186,6 +206,27 @@ func handleConfigMapFromFileSources(configMap *api.ConfigMap, fileSources []stri
 	return nil
 }
 
+// handleConfigMapFromEnvFileSource adds the specified env file source information
+// into the provided configMap
+func handleConfigMapFromEnvFileSource(configMap *api.ConfigMap, envFileSource string) error {
+	info, err := os.Stat(envFileSource)
+	if err != nil {
+		switch err := err.(type) {
+		case *os.PathError:
+			return fmt.Errorf("error reading %s: %v", envFileSource, err.Err)
+		default:
+			return fmt.Errorf("error reading %s: %v", envFileSource, err)
+		}
+	}
+	if info.IsDir() {
+		return fmt.Errorf("must be a file")
+	}
+
+	return addFromEnvFile(envFileSource, func(key, value string) error {
+		return addKeyFromLiteralToConfigMap(configMap, key, value)
+	})
+}
+
 // addKeyFromFileToConfigMap adds a key with the given name to a ConfigMap, populating
 // the value with the content of the given file path, or returns an error.
 func addKeyFromFileToConfigMap(configMap *api.ConfigMap, keyName, filePath string) error {

pkg/kubectl/configmap_test.go

@@ -17,6 +17,8 @@ limitations under the License.
 package kubectl
 
 import (
+	"io/ioutil"
+	"os"
 	"reflect"
 	"testing"
 
@@ -26,6 +28,7 @@ import (
 
 func TestConfigMapGenerate(t *testing.T) {
 	tests := []struct {
+		setup     func(t *testing.T, params map[string]interface{}) func()
 		params    map[string]interface{}
 		expected  *api.ConfigMap
 		expectErr bool
@@ -107,9 +110,84 @@ func TestConfigMapGenerate(t *testing.T) {
 			},
 			expectErr: false,
 		},
+		{
+			setup: setupEnvFile("key1=value1", "#", "", "key2=value2"),
+			params: map[string]interface{}{
+				"name":          "valid_env",
+				"from-env-file": "file.env",
+			},
+			expected: &api.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "valid_env",
+				},
+				Data: map[string]string{
+					"key1": "value1",
+					"key2": "value2",
+				},
+			},
+			expectErr: false,
+		},
+		{
+			setup: func() func(t *testing.T, params map[string]interface{}) func() {
+				os.Setenv("g_key1", "1")
+				os.Setenv("g_key2", "2")
+				return setupEnvFile("g_key1", "g_key2=")
+			}(),
+			params: map[string]interface{}{
+				"name":          "getenv",
+				"from-env-file": "file.env",
+			},
+			expected: &api.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "getenv",
+				},
+				Data: map[string]string{
+					"g_key1": "1",
+					"g_key2": "",
+				},
+			},
+			expectErr: false,
+		},
+		{
+			params: map[string]interface{}{
+				"name":          "too_many_args",
+				"from-literal":  []string{"key1=value1"},
+				"from-env-file": "file.env",
+			},
+			expectErr: true,
+		},
+		{
+			setup: setupEnvFile("key.1=value1"),
+			params: map[string]interface{}{
+				"name":          "invalid_key",
+				"from-env-file": "file.env",
+			},
+			expectErr: true,
+		},
+		{
+			setup: setupEnvFile("  key1=  value1"),
+			params: map[string]interface{}{
+				"name":          "with_spaces",
+				"from-env-file": "file.env",
+			},
+			expected: &api.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "with_spaces",
+				},
+				Data: map[string]string{
+					"key1": "  value1",
+				},
+			},
+			expectErr: false,
+		},
 	}
 	generator := ConfigMapGeneratorV1{}
 	for _, test := range tests {
+		if test.setup != nil {
+			if teardown := test.setup(t, test.params); teardown != nil {
+				defer teardown()
+			}
+		}
 		obj, err := generator.Generate(test.params)
 		if !test.expectErr && err != nil {
 			t.Errorf("unexpected error: %v", err)
@@ -122,3 +200,21 @@ func TestConfigMapGenerate(t *testing.T) {
 		}
 	}
 }
+
+func setupEnvFile(lines ...string) func(*testing.T, map[string]interface{}) func() {
+	return func(t *testing.T, params map[string]interface{}) func() {
+		f, err := ioutil.TempFile("", "cme")
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		for _, l := range lines {
+			f.WriteString(l)
+			f.WriteString("\r\n")
+		}
+		f.Close()
+		params["from-env-file"] = f.Name()
+		return func() {
+			os.Remove(f.Name())
+		}
+	}
+}

pkg/kubectl/env_file.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubectl
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"k8s.io/apimachinery/pkg/util/validation"
+)
+
+// addFromEnvFile processes an env file allows a generic addTo to handle the
+// collection of key value pairs or returns an error.
+func addFromEnvFile(filePath string, addTo func(key, value string) error) error {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	currentLine := 0
+	utf8bom := []byte{0xEF, 0xBB, 0xBF}
+	for scanner.Scan() {
+		scannedBytes := scanner.Bytes()
+		if !utf8.Valid(scannedBytes) {
+			return fmt.Errorf("env file %s contains invalid utf8 bytes at line %d: %v", filePath, currentLine+1, scannedBytes)
+		}
+		// We trim UTF8 BOM
+		if currentLine == 0 {
+			scannedBytes = bytes.TrimPrefix(scannedBytes, utf8bom)
+		}
+		// trim the line from all leading whitespace first
+		line := strings.TrimLeftFunc(string(scannedBytes), unicode.IsSpace)
+		currentLine++
+		// line is not empty, and not starting with '#'
+		if len(line) > 0 && !strings.HasPrefix(line, "#") {
+			data := strings.SplitN(line, "=", 2)
+			key := data[0]
+			if errs := validation.IsCIdentifier(key); len(errs) != 0 {
+				return fmt.Errorf("%q is not a valid key name: %s", key, strings.Join(errs, ";"))
+			}
+
+			value := ""
+			if len(data) > 1 {
+				// pass the value through, no trimming
+				value = data[1]
+			} else {
+				// a pass-through variable is given
+				value = os.Getenv(key)
+			}
+			if err = addTo(key, value); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/kubectl/secret.go

@@ -38,6 +38,8 @@ type SecretGeneratorV1 struct {
 	FileSources []string
 	// LiteralSources to derive the secret from (optional)
 	LiteralSources []string
+	// EnvFileSource to derive the secret from (optional)
+	EnvFileSource string
 }
 
 // Ensure it supports the generator pattern that uses parameter injection
@@ -71,6 +73,15 @@ func (s SecretGeneratorV1) Generate(genericParams map[string]interface{}) (runti
 		delegate.LiteralSources = fromLiteralArray
 		delete(genericParams, "from-literal")
 	}
+	fromEnvFileString, found := genericParams["from-env-file"]
+	if found {
+		fromEnvFile, isString := fromEnvFileString.(string)
+		if !isString {
+			return nil, fmt.Errorf("expected string, found :%v", fromEnvFileString)
+		}
+		delegate.EnvFileSource = fromEnvFile
+		delete(genericParams, "from-env-file")
+	}
 	params := map[string]string{}
 	for key, value := range genericParams {
 		strVal, isString := value.(string)
@@ -91,6 +102,7 @@ func (s SecretGeneratorV1) ParamNames() []GeneratorParam {
 		{"type", false},
 		{"from-file", false},
 		{"from-literal", false},
+		{"from-env-file", false},
 		{"force", false},
 	}
 }
@@ -116,6 +128,11 @@ func (s SecretGeneratorV1) StructuredGenerate() (runtime.Object, error) {
 			return nil, err
 		}
 	}
+	if len(s.EnvFileSource) > 0 {
+		if err := handleFromEnvFileSource(secret, s.EnvFileSource); err != nil {
+			return nil, err
+		}
+	}
 	return secret, nil
 }
 
@@ -124,6 +141,9 @@ func (s SecretGeneratorV1) validate() error {
 	if len(s.Name) == 0 {
 		return fmt.Errorf("name must be specified")
 	}
+	if len(s.EnvFileSource) > 0 && (len(s.FileSources) > 0 || len(s.LiteralSources) > 0) {
+		return fmt.Errorf("from-env-file cannot be combined with from-file or from-literal")
+	}
 	return nil
 }
 
@@ -187,6 +207,27 @@ func handleFromFileSources(secret *api.Secret, fileSources []string) error {
 	return nil
 }
 
+// handleFromEnvFileSource adds the specified env file source information
+// into the provided secret
+func handleFromEnvFileSource(secret *api.Secret, envFileSource string) error {
+	info, err := os.Stat(envFileSource)
+	if err != nil {
+		switch err := err.(type) {
+		case *os.PathError:
+			return fmt.Errorf("error reading %s: %v", envFileSource, err.Err)
+		default:
+			return fmt.Errorf("error reading %s: %v", envFileSource, err)
+		}
+	}
+	if info.IsDir() {
+		return fmt.Errorf("must be a file")
+	}
+
+	return addFromEnvFile(envFileSource, func(key, value string) error {
+		return addKeyFromLiteralToSecret(secret, key, []byte(value))
+	})
+}
+
 func addKeyFromFileToSecret(secret *api.Secret, keyName, filePath string) error {
 	data, err := ioutil.ReadFile(filePath)
 	if err != nil {

pkg/kubectl/secret_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package kubectl
 
 import (
+	"os"
 	"reflect"
 	"testing"
 
@@ -26,6 +27,7 @@ import (
 
 func TestSecretGenerate(t *testing.T) {
 	tests := []struct {
+		setup     func(t *testing.T, params map[string]interface{}) func()
 		params    map[string]interface{}
 		expected  *api.Secret
 		expectErr bool
@@ -108,9 +110,84 @@ func TestSecretGenerate(t *testing.T) {
 			},
 			expectErr: false,
 		},
+		{
+			setup: setupEnvFile("key1=value1", "#", "", "key2=value2"),
+			params: map[string]interface{}{
+				"name":          "valid_env",
+				"from-env-file": "file.env",
+			},
+			expected: &api.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "valid_env",
+				},
+				Data: map[string][]byte{
+					"key1": []byte("value1"),
+					"key2": []byte("value2"),
+				},
+			},
+			expectErr: false,
+		},
+		{
+			setup: func() func(t *testing.T, params map[string]interface{}) func() {
+				os.Setenv("g_key1", "1")
+				os.Setenv("g_key2", "2")
+				return setupEnvFile("g_key1", "g_key2=")
+			}(),
+			params: map[string]interface{}{
+				"name":          "getenv",
+				"from-env-file": "file.env",
+			},
+			expected: &api.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "getenv",
+				},
+				Data: map[string][]byte{
+					"g_key1": []byte("1"),
+					"g_key2": []byte(""),
+				},
+			},
+			expectErr: false,
+		},
+		{
+			params: map[string]interface{}{
+				"name":          "too_many_args",
+				"from-literal":  []string{"key1=value1"},
+				"from-env-file": "file.env",
+			},
+			expectErr: true,
+		},
+		{
+			setup: setupEnvFile("key.1=value1"),
+			params: map[string]interface{}{
+				"name":          "invalid_key",
+				"from-env-file": "file.env",
+			},
+			expectErr: true,
+		},
+		{
+			setup: setupEnvFile("  key1=  value1"),
+			params: map[string]interface{}{
+				"name":          "with_spaces",
+				"from-env-file": "file.env",
+			},
+			expected: &api.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "with_spaces",
+				},
+				Data: map[string][]byte{
+					"key1": []byte("  value1"),
+				},
+			},
+			expectErr: false,
+		},
 	}
 	generator := SecretGeneratorV1{}
 	for _, test := range tests {
+		if test.setup != nil {
+			if teardown := test.setup(t, test.params); teardown != nil {
+				defer teardown()
+			}
+		}
 		obj, err := generator.Generate(test.params)
 		if !test.expectErr && err != nil {
 			t.Errorf("unexpected error: %v", err)