Get RancherOS logging to throw debug logs to a remote syslog server when the kernel netconsole is configured

Signed-off-by: Sven Dowideit <SvenDowideit@home.org.au>

config/cmdline/cmdline.go

@@ -0,0 +1,171 @@
+package cmdline
+
+import (
+	"io/ioutil"
+	"strings"
+
+	yaml "github.com/cloudfoundry-incubator/candiedyaml"
+	"github.com/rancher/os/util"
+)
+
+func Read(parseAll bool) (m map[interface{}]interface{}, err error) {
+	cmdLine, err := ioutil.ReadFile("/proc/cmdline")
+	if err != nil {
+		return nil, err
+	}
+
+	if len(cmdLine) == 0 {
+		return nil, nil
+	}
+
+	cmdLineObj := Parse(strings.TrimSpace(util.UnescapeKernelParams(string(cmdLine))), parseAll)
+
+	return cmdLineObj, nil
+}
+
+func GetCmdline(key string) interface{} {
+	parseAll := true
+	if strings.HasPrefix(key, "cc.") || strings.HasPrefix(key, "rancher.") {
+		// the normal case
+		parseAll = false
+	}
+	cmdline, _ := Read(parseAll)
+	v, _ := GetOrSetVal(key, cmdline, nil)
+	return v
+}
+
+func GetOrSetVal(args string, data map[interface{}]interface{}, value interface{}) (interface{}, map[interface{}]interface{}) {
+	parts := strings.Split(args, ".")
+
+	tData := data
+	if value != nil {
+		tData = util.MapCopy(data)
+	}
+	t := tData
+	for i, part := range parts {
+		val, ok := t[part]
+		last := i+1 == len(parts)
+
+		// Reached end, set the value
+		if last && value != nil {
+			if s, ok := value.(string); ok {
+				value = unmarshalOrReturnString(s)
+			}
+
+			t[part] = value
+			return value, tData
+		}
+
+		// Missing intermediate key, create key
+		if !last && value != nil && !ok {
+			newData := map[interface{}]interface{}{}
+			t[part] = newData
+			t = newData
+			continue
+		}
+
+		if !ok {
+			break
+		}
+
+		if last {
+			return val, tData
+		}
+
+		newData, ok := val.(map[interface{}]interface{})
+		if !ok {
+			break
+		}
+
+		t = newData
+	}
+
+	return "", tData
+}
+
+// Replace newlines, colons, and question marks with random strings
+// This is done to avoid YAML treating these as special characters
+var (
+	newlineMagicString      = "9XsJcx6dR5EERYCC"
+	colonMagicString        = "V0Rc21pIVknMm2rr"
+	questionMarkMagicString = "FoPL6JLMAaJqKMJT"
+)
+
+func reverseReplacement(result interface{}) interface{} {
+	switch val := result.(type) {
+	case map[interface{}]interface{}:
+		for k, v := range val {
+			val[k] = reverseReplacement(v)
+		}
+		return val
+	case []interface{}:
+		for i, item := range val {
+			val[i] = reverseReplacement(item)
+		}
+		return val
+	case string:
+		val = strings.Replace(val, newlineMagicString, "\n", -1)
+		val = strings.Replace(val, colonMagicString, ":", -1)
+		val = strings.Replace(val, questionMarkMagicString, "?", -1)
+		return val
+	}
+
+	return result
+}
+
+func unmarshalOrReturnString(value string) (result interface{}) {
+	value = strings.Replace(value, "\n", newlineMagicString, -1)
+	value = strings.Replace(value, ":", colonMagicString, -1)
+	value = strings.Replace(value, "?", questionMarkMagicString, -1)
+	if err := yaml.Unmarshal([]byte(value), &result); err != nil {
+		result = value
+	}
+	result = reverseReplacement(result)
+	return
+}
+
+func Parse(cmdLine string, parseAll bool) map[interface{}]interface{} {
+	result := make(map[interface{}]interface{})
+
+outer:
+	for _, part := range strings.Split(cmdLine, " ") {
+		if strings.HasPrefix(part, "cc.") {
+			part = part[3:]
+		} else if !strings.HasPrefix(part, "rancher.") {
+			if !parseAll {
+				continue
+			}
+		}
+
+		var value string
+		kv := strings.SplitN(part, "=", 2)
+
+		if len(kv) == 1 {
+			value = "true"
+		} else {
+			value = kv[1]
+		}
+
+		current := result
+		keys := strings.Split(kv[0], ".")
+		for i, key := range keys {
+			if i == len(keys)-1 {
+				current[key] = unmarshalOrReturnString(value)
+			} else {
+				if obj, ok := current[key]; ok {
+					if newCurrent, ok := obj.(map[interface{}]interface{}); ok {
+						current = newCurrent
+					} else {
+						continue outer
+					}
+				} else {
+					newCurrent := make(map[interface{}]interface{})
+					current[key] = newCurrent
+					current = newCurrent
+				}
+			}
+		}
+	}
+
+	return result
+}

config/config.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
+	"github.com/rancher/os/config/cmdline"
 	"github.com/rancher/os/util"
 )
 
@@ -41,6 +42,13 @@ func Export(private, full bool) (string, error) {
 	bytes, err := yaml.Marshal(rawCfg)
 	return string(bytes), err
 }
+func filterPrivateKeys(data map[interface{}]interface{}) map[interface{}]interface{} {
+	for _, privateKey := range PrivateKeys {
+		_, data = filterKey(data, strings.Split(privateKey, "."))
+	}
+
+	return data
+}
 
 func Get(key string) (interface{}, error) {
 	cfg := LoadConfig()
@@ -50,23 +58,17 @@ func Get(key string) (interface{}, error) {
 		return nil, err
 	}
 
-	v, _ := getOrSetVal(key, data, nil)
+	v, _ := cmdline.GetOrSetVal(key, data, nil)
 	return v, nil
 }
 
-func GetCmdline(key string) interface{} {
-	cmdline := readCmdline()
-	v, _ := getOrSetVal(key, cmdline, nil)
-	return v
-}
-
 func Set(key string, value interface{}) error {
 	existing, err := readConfigs(nil, false, true, CloudConfigFile)
 	if err != nil {
 		return err
 	}
 
-	_, modified := getOrSetVal(key, existing, value)
+	_, modified := cmdline.GetOrSetVal(key, existing, value)
 
 	c := &CloudConfig{}
 	if err = util.Convert(modified, c); err != nil {

config/config_test.go

@@ -5,6 +5,7 @@ import (
 
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 
+	"github.com/rancher/os/config/cmdline"
 	"github.com/rancher/os/util"
 	"github.com/stretchr/testify/require"
 )
@@ -100,7 +101,7 @@ func TestUnmarshalOrReturnString(t *testing.T) {
 	assert.Equal([]interface{}{false, "a"}, unmarshalOrReturnString("[false,a]"))
 }
 
-func TestParseCmdline(t *testing.T) {
+func TestCmdlineParse(t *testing.T) {
 	assert := require.New(t)
 
 	assert.Equal(map[interface{}]interface{}{
@@ -108,55 +109,55 @@ func TestParseCmdline(t *testing.T) {
 			"key1": "value1",
 			"key2": "value2",
 		},
-	}, parseCmdline("a b rancher.key1=value1 c rancher.key2=value2"))
+	}, cmdline.Parse("a b rancher.key1=value1 c rancher.key2=value2"), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"key": "a,b",
 		},
-	}, parseCmdline("rancher.key=a,b"))
+	}, cmdline.Parse("rancher.key=a,b"), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"key": "a\nb",
 		},
-	}, parseCmdline("rancher.key=a\nb"))
+	}, cmdline.Parse("rancher.key=a\nb"), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"key": "a:b",
 		},
-	}, parseCmdline("rancher.key=a:b"))
+	}, cmdline.Parse("rancher.key=a:b"), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"key": int64(5),
 		},
-	}, parseCmdline("rancher.key=5"))
+	}, cmdline.Parse("rancher.key=5"), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"rescue": true,
 		},
-	}, parseCmdline("rancher.rescue"))
+	}, cmdline.Parse("rancher.rescue"), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"keyArray": []interface{}{int64(1), int64(2)},
 		},
-	}, parseCmdline("rancher.keyArray=[1,2]"))
+	}, cmdline.Parse("rancher.keyArray=[1,2]"), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"strArray": []interface{}{"url:http://192.168.1.100/cloud-config?a=b"},
 		},
-	}, parseCmdline("rancher.strArray=[\"url:http://192.168.1.100/cloud-config?a=b\"]"))
+	}, cmdline.Parse("rancher.strArray=[\"url:http://192.168.1.100/cloud-config?a=b\"]"), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"strArray": []interface{}{"url:http://192.168.1.100/cloud-config?a=b"},
 		},
-	}, parseCmdline("rancher.strArray=[url:http://192.168.1.100/cloud-config?a=b]"))
+	}, cmdline.Parse("rancher.strArray=[url:http://192.168.1.100/cloud-config?a=b]"), false)
 }
 
 func TestGet(t *testing.T) {
@@ -181,7 +182,7 @@ func TestGet(t *testing.T) {
 	}
 
 	for k, v := range tests {
-		val, _ := getOrSetVal(k, data, nil)
+		val, _ := cmdline.GeOrSetVal(k, data, nil)
 		assert.Equal(v, val)
 	}
 }
@@ -225,8 +226,8 @@ func TestSet(t *testing.T) {
 	}
 
 	for k, v := range tests {
-		_, tData := getOrSetVal(k, data, v)
-		val, _ := getOrSetVal(k, tData, nil)
+		_, tData := cmdline.GetOrSetVal(k, data, v)
+		val, _ := cmdline.GetOrSetVal(k, tData, nil)
 		data = tData
 		assert.Equal(v, val)
 	}

config/data_funcs.go

@@ -1,11 +1,7 @@
 package config
 
 import (
-	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 	"github.com/rancher/os/log"
-
-	"strings"
-
 	"github.com/rancher/os/util"
 )
 
@@ -14,6 +10,7 @@ type CfgFuncData struct {
 	Name string
 	Func CfgFunc
 }
+
 type CfgFuncs []CfgFuncData
 
 func ChainCfgFuncs(cfg *CloudConfig, cfgFuncs CfgFuncs) (*CloudConfig, error) {
@@ -71,145 +68,3 @@ func filterKey(data map[interface{}]interface{}, key []string) (filtered, rest m
 
 	return
 }
-
-func filterPrivateKeys(data map[interface{}]interface{}) map[interface{}]interface{} {
-	for _, privateKey := range PrivateKeys {
-		_, data = filterKey(data, strings.Split(privateKey, "."))
-	}
-
-	return data
-}
-
-func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) (interface{}, map[interface{}]interface{}) {
-	parts := strings.Split(args, ".")
-
-	tData := data
-	if value != nil {
-		tData = util.MapCopy(data)
-	}
-	t := tData
-	for i, part := range parts {
-		val, ok := t[part]
-		last := i+1 == len(parts)
-
-		// Reached end, set the value
-		if last && value != nil {
-			if s, ok := value.(string); ok {
-				value = unmarshalOrReturnString(s)
-			}
-
-			t[part] = value
-			return value, tData
-		}
-
-		// Missing intermediate key, create key
-		if !last && value != nil && !ok {
-			newData := map[interface{}]interface{}{}
-			t[part] = newData
-			t = newData
-			continue
-		}
-
-		if !ok {
-			break
-		}
-
-		if last {
-			return val, tData
-		}
-
-		newData, ok := val.(map[interface{}]interface{})
-		if !ok {
-			break
-		}
-
-		t = newData
-	}
-
-	return "", tData
-}
-
-// Replace newlines, colons, and question marks with random strings
-// This is done to avoid YAML treating these as special characters
-var (
-	newlineMagicString      = "9XsJcx6dR5EERYCC"
-	colonMagicString        = "V0Rc21pIVknMm2rr"
-	questionMarkMagicString = "FoPL6JLMAaJqKMJT"
-)
-
-func reverseReplacement(result interface{}) interface{} {
-	switch val := result.(type) {
-	case map[interface{}]interface{}:
-		for k, v := range val {
-			val[k] = reverseReplacement(v)
-		}
-		return val
-	case []interface{}:
-		for i, item := range val {
-			val[i] = reverseReplacement(item)
-		}
-		return val
-	case string:
-		val = strings.Replace(val, newlineMagicString, "\n", -1)
-		val = strings.Replace(val, colonMagicString, ":", -1)
-		val = strings.Replace(val, questionMarkMagicString, "?", -1)
-		return val
-	}
-
-	return result
-}
-
-func unmarshalOrReturnString(value string) (result interface{}) {
-	value = strings.Replace(value, "\n", newlineMagicString, -1)
-	value = strings.Replace(value, ":", colonMagicString, -1)
-	value = strings.Replace(value, "?", questionMarkMagicString, -1)
-	if err := yaml.Unmarshal([]byte(value), &result); err != nil {
-		result = value
-	}
-	result = reverseReplacement(result)
-	return
-}
-
-func parseCmdline(cmdLine string) map[interface{}]interface{} {
-	result := make(map[interface{}]interface{})
-
-outer:
-	for _, part := range strings.Split(cmdLine, " ") {
-		if strings.HasPrefix(part, "cc.") {
-			part = part[3:]
-		} else if !strings.HasPrefix(part, "rancher.") {
-			continue
-		}
-
-		var value string
-		kv := strings.SplitN(part, "=", 2)
-
-		if len(kv) == 1 {
-			value = "true"
-		} else {
-			value = kv[1]
-		}
-
-		current := result
-		keys := strings.Split(kv[0], ".")
-		for i, key := range keys {
-			if i == len(keys)-1 {
-				current[key] = unmarshalOrReturnString(value)
-			} else {
-				if obj, ok := current[key]; ok {
-					if newCurrent, ok := obj.(map[interface{}]interface{}); ok {
-						current = newCurrent
-					} else {
-						continue outer
-					}
-				} else {
-					newCurrent := make(map[interface{}]interface{})
-					current[key] = newCurrent
-					current = newCurrent
-				}
-			}
-		}
-	}
-
-	return result
-}

config/disk.go

@@ -14,6 +14,7 @@ import (
 	composeConfig "github.com/docker/libcompose/config"
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/initialize"
+	"github.com/rancher/os/config/cmdline"
 	"github.com/rancher/os/log"
 	"github.com/rancher/os/util"
 )
@@ -48,7 +49,11 @@ func loadRawDiskConfig(dirPrefix string, full bool) map[interface{}]interface{}
 
 func loadRawConfig(dirPrefix string, full bool) map[interface{}]interface{} {
 	rawCfg := loadRawDiskConfig(dirPrefix, full)
-	rawCfg = util.Merge(rawCfg, readCmdline())
+	procCmdline, err := cmdline.Read(false)
+	if err != nil {
+		log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params")
+	}
+	rawCfg = util.Merge(rawCfg, procCmdline)
 	rawCfg = util.Merge(rawCfg, readElidedCmdline(rawCfg))
 	rawCfg = applyDebugFlags(rawCfg)
 	return mergeMetadata(rawCfg, readMetadata())
@@ -107,7 +112,7 @@ func Insert(m interface{}, args ...interface{}) interface{} {
 }
 
 func SaveInitCmdline(cmdLineArgs string) {
-	elidedCfg := parseCmdline(cmdLineArgs)
+	elidedCfg := cmdline.Parse(cmdLineArgs, false)
 
 	env := Insert(make(map[interface{}]interface{}), interface{}("EXTRA_CMDLINE"), interface{}(cmdLineArgs))
 	rancher := Insert(make(map[interface{}]interface{}), interface{}("environment"), env)
@@ -155,10 +160,10 @@ func applyDebugFlags(rawCfg map[interface{}]interface{}) map[interface{}]interfa
 	}
 
 	log.SetLevel(log.DebugLevel)
-	_, rawCfg = getOrSetVal("rancher.docker.debug", rawCfg, true)
-	_, rawCfg = getOrSetVal("rancher.system_docker.debug", rawCfg, true)
-	_, rawCfg = getOrSetVal("rancher.bootstrap_docker.debug", rawCfg, true)
-	_, rawCfg = getOrSetVal("rancher.log", rawCfg, true)
+	_, rawCfg = cmdline.GetOrSetVal("rancher.docker.debug", rawCfg, true)
+	_, rawCfg = cmdline.GetOrSetVal("rancher.system_docker.debug", rawCfg, true)
+	_, rawCfg = cmdline.GetOrSetVal("rancher.bootstrap_docker.debug", rawCfg, true)
+	_, rawCfg = cmdline.GetOrSetVal("rancher.log", rawCfg, true)
 
 	return rawCfg
 }
@@ -216,7 +221,7 @@ func readElidedCmdline(rawCfg map[interface{}]interface{}) map[interface{}]inter
 	for k, v := range rawCfg {
 		if key, _ := k.(string); key == "EXTRA_CMDLINE" {
 			if val, ok := v.(string); ok {
-				cmdLineObj := parseCmdline(strings.TrimSpace(util.UnescapeKernelParams(string(val))))
+				cmdLineObj := cmdline.Parse(strings.TrimSpace(util.UnescapeKernelParams(string(val))), false)
 
 				return cmdLineObj
 			}
@@ -225,22 +230,6 @@ func readElidedCmdline(rawCfg map[interface{}]interface{}) map[interface{}]inter
 	return nil
 }
 
-func readCmdline() map[interface{}]interface{} {
-	cmdLine, err := ioutil.ReadFile("/proc/cmdline")
-	if err != nil {
-		log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params")
-		return nil
-	}
-
-	if len(cmdLine) == 0 {
-		return nil
-	}
-
-	cmdLineObj := parseCmdline(strings.TrimSpace(util.UnescapeKernelParams(string(cmdLine))))
-
-	return cmdLineObj
-}
-
 func amendNils(c *CloudConfig) *CloudConfig {
 	t := *c
 	if t.Rancher.Environment == nil {