diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 080a3ca3..bc786ad5 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -63,7 +63,7 @@ ARG DOCKER_BUILD_VERSION=1.10.3 ARG DOCKER_BUILD_PATCH_VERSION=v${DOCKER_BUILD_VERSION}-ros1 ARG SELINUX_POLICY_URL=https://github.com/rancher/refpolicy/releases/download/v0.0.3/policy.29 -ARG KERNEL_VERSION_amd64=4.9.40-rancher +ARG KERNEL_VERSION_amd64=4.9.42-rancher ARG KERNEL_URL_amd64=https://github.com/rancher/os-kernel/releases/download/v${KERNEL_VERSION_amd64}/linux-${KERNEL_VERSION_amd64}-x86.tar.gz #ARG KERNEL_URL_arm64=https://github.com/imikushin/os-kernel/releases/download/Estuary-4.4.0-arm64.8/linux-4.4.0-rancher-arm64.tar.gz diff --git a/cmd/cloudinitexecute/cloudinitexecute.go b/cmd/cloudinitexecute/cloudinitexecute.go index aae4d01a..cafebcca 100644 --- a/cmd/cloudinitexecute/cloudinitexecute.go +++ b/cmd/cloudinitexecute/cloudinitexecute.go @@ -108,7 +108,10 @@ func ApplyConsole(cfg *rancherConfig.CloudConfig) { } } - util.RunCommandSequence(cfg.Runcmd) + err := util.RunCommandSequence(cfg.Runcmd) + if err != nil { + log.Error(err) + } } func WriteFiles(cfg *rancherConfig.CloudConfig, container string) { diff --git a/cmd/control/bootstrap.go b/cmd/control/bootstrap.go index b069c8a3..9ccaf118 100644 --- a/cmd/control/bootstrap.go +++ b/cmd/control/bootstrap.go @@ -39,7 +39,10 @@ func bootstrapAction(c *cli.Context) error { } log.Debugf("bootstrapAction: RunCommandSequence(%v)", cfg.Bootcmd) - util.RunCommandSequence(cfg.Bootcmd) + err := util.RunCommandSequence(cfg.Bootcmd) + if err != nil { + log.Error(err) + } if cfg.Rancher.State.Dev != "" && cfg.Rancher.State.Wait { waitForRoot(cfg) diff --git a/cmd/control/console_init.go b/cmd/control/console_init.go index 0a4ea41c..2f73b4a3 100644 --- a/cmd/control/console_init.go +++ b/cmd/control/console_init.go @@ -14,6 +14,7 @@ import ( "github.com/codegangsta/cli" "github.com/rancher/os/cmd/cloudinitexecute" "github.com/rancher/os/config" + "github.com/rancher/os/config/cmdline" "github.com/rancher/os/log" "github.com/rancher/os/util" ) @@ -63,7 +64,7 @@ func consoleInitFunc() error { createHomeDir(rancherHome, 1100, 1100) createHomeDir(dockerHome, 1101, 1101) - password := config.GetCmdline("rancher.password") + password := cmdline.GetCmdline("rancher.password") if password != "" { cmd := exec.Command("chpasswd") cmd.Stdin = strings.NewReader(fmt.Sprint("rancher:", password)) diff --git a/cmd/control/install.go b/cmd/control/install.go index 4af6695c..d0a99614 100755 --- a/cmd/control/install.go +++ b/cmd/control/install.go @@ -368,7 +368,11 @@ func runInstall(image, installType, cloudConfig, device, partition, statedir, ka func mountBootIso() error { deviceName := "/dev/sr0" deviceType := "iso9660" - if d, t := util.Blkid("RancherOS"); d != "" { + d, t, err := util.Blkid("RancherOS") + if err != nil { + log.Errorf("Failed to run blkid: %s", err) + } + if d != "" { deviceName = d deviceType = t } @@ -728,6 +732,66 @@ func formatdevice(device, partition string) error { return nil } +func mountdevice(baseName, bootDir, device, partition string, raw bool) (string, string, error) { + log.Debugf("mountdevice %s, raw %v", partition, raw) + + if partition == "" { + if raw { + log.Debugf("util.Mount (raw) %s, %s", partition, baseName) + + cmd := exec.Command("lsblk", "-no", "pkname", partition) + log.Debugf("Run(%v)", cmd) + cmd.Stderr = os.Stderr + device := "" + // TODO: out can == "" - this is used to "detect software RAID" which is terrible + if out, err := cmd.Output(); err == nil { + device = "/dev/" + strings.TrimSpace(string(out)) + } + + log.Debugf("mountdevice return -> d: %s, p: %s", device, partition) + return device, partition, util.Mount(partition, baseName, "", "") + } + + //rootfs := partition + // Don't use ResolveDevice - it can fail, whereas `blkid -L LABEL` works more often + + cfg := config.LoadConfig() + d, _, err := util.Blkid("RANCHER_BOOT") + if err != nil { + log.Errorf("Failed to run blkid: %s", err) + } + if d != "" { + partition = d + baseName = filepath.Join(baseName, "boot") + } else { + if dev := util.ResolveDevice(cfg.Rancher.State.Dev); dev != "" { + // try the rancher.state.dev setting + partition = dev + } else { + d, _, err := util.Blkid("RANCHER_STATE") + if err != nil { + log.Errorf("Failed to run blkid: %s", err) + } + if d != "" { + partition = d + } + } + } + cmd := exec.Command("lsblk", "-no", "pkname", partition) + log.Debugf("Run(%v)", cmd) + cmd.Stderr = os.Stderr + // TODO: out can == "" - this is used to "detect software RAID" which is terrible + if out, err := cmd.Output(); err == nil { + device = "/dev/" + strings.TrimSpace(string(out)) + } + } + os.MkdirAll(baseName, 0755) + cmd := exec.Command("mount", partition, baseName) + //cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + log.Debugf("mountdevice return2 -> d: %s, p: %s", device, partition) + return device, partition, cmd.Run() +} + func formatAndMount(baseName, device, partition string) (string, string, error) { log.Debugf("formatAndMount") diff --git a/cmd/control/install/install.go b/cmd/control/install/install.go index 0005b9cc..7798a9bc 100644 --- a/cmd/control/install/install.go +++ b/cmd/control/install/install.go @@ -47,7 +47,11 @@ func MountDevice(baseName, device, partition string, raw bool) (string, string, // Don't use ResolveDevice - it can fail, whereas `blkid -L LABEL` works more often cfg := config.LoadConfig() - if d, _ := util.Blkid("RANCHER_BOOT"); d != "" { + d, _, err := util.Blkid("RANCHER_BOOT") + if err != nil { + log.Errorf("Failed to run blkid: %s", err) + } + if d != "" { partition = d baseName = filepath.Join(baseName, BootDir) } else { @@ -55,7 +59,11 @@ func MountDevice(baseName, device, partition string, raw bool) (string, string, // try the rancher.state.dev setting partition = dev } else { - if d, _ := util.Blkid("RANCHER_STATE"); d != "" { + d, _, err := util.Blkid("RANCHER_STATE") + if err != nil { + log.Errorf("Failed to run blkid: %s", err) + } + if d != "" { partition = d } } diff --git a/config/cmdline/cmdline.go b/config/cmdline/cmdline.go new file mode 100755 index 00000000..9a4b1001 --- /dev/null +++ b/config/cmdline/cmdline.go @@ -0,0 +1,171 @@ +package cmdline + +import ( + "io/ioutil" + "strings" + + yaml "github.com/cloudfoundry-incubator/candiedyaml" + "github.com/rancher/os/util" +) + +func Read(parseAll bool) (m map[interface{}]interface{}, err error) { + cmdLine, err := ioutil.ReadFile("/proc/cmdline") + if err != nil { + return nil, err + } + + if len(cmdLine) == 0 { + return nil, nil + } + + cmdLineObj := Parse(strings.TrimSpace(util.UnescapeKernelParams(string(cmdLine))), parseAll) + + return cmdLineObj, nil +} + +func GetCmdline(key string) interface{} { + parseAll := true + if strings.HasPrefix(key, "cc.") || strings.HasPrefix(key, "rancher.") { + // the normal case + parseAll = false + } + cmdline, _ := Read(parseAll) + v, _ := GetOrSetVal(key, cmdline, nil) + return v +} + +func GetOrSetVal(args string, data map[interface{}]interface{}, value interface{}) (interface{}, map[interface{}]interface{}) { + parts := strings.Split(args, ".") + + tData := data + if value != nil { + tData = util.MapCopy(data) + } + t := tData + for i, part := range parts { + val, ok := t[part] + last := i+1 == len(parts) + + // Reached end, set the value + if last && value != nil { + if s, ok := value.(string); ok { + value = unmarshalOrReturnString(s) + } + + t[part] = value + return value, tData + } + + // Missing intermediate key, create key + if !last && value != nil && !ok { + newData := map[interface{}]interface{}{} + t[part] = newData + t = newData + continue + } + + if !ok { + break + } + + if last { + return val, tData + } + + newData, ok := val.(map[interface{}]interface{}) + if !ok { + break + } + + t = newData + } + + return "", tData +} + +// Replace newlines, colons, and question marks with random strings +// This is done to avoid YAML treating these as special characters +var ( + newlineMagicString = "9XsJcx6dR5EERYCC" + colonMagicString = "V0Rc21pIVknMm2rr" + questionMarkMagicString = "FoPL6JLMAaJqKMJT" +) + +func reverseReplacement(result interface{}) interface{} { + switch val := result.(type) { + case map[interface{}]interface{}: + for k, v := range val { + val[k] = reverseReplacement(v) + } + return val + case []interface{}: + for i, item := range val { + val[i] = reverseReplacement(item) + } + return val + case string: + val = strings.Replace(val, newlineMagicString, "\n", -1) + val = strings.Replace(val, colonMagicString, ":", -1) + val = strings.Replace(val, questionMarkMagicString, "?", -1) + return val + } + + return result +} + +func unmarshalOrReturnString(value string) (result interface{}) { + value = strings.Replace(value, "\n", newlineMagicString, -1) + value = strings.Replace(value, ":", colonMagicString, -1) + value = strings.Replace(value, "?", questionMarkMagicString, -1) + if err := yaml.Unmarshal([]byte(value), &result); err != nil { + result = value + } + result = reverseReplacement(result) + return +} + +func Parse(cmdLine string, parseAll bool) map[interface{}]interface{} { + result := make(map[interface{}]interface{}) + +outer: + for _, part := range strings.Split(cmdLine, " ") { + if strings.HasPrefix(part, "cc.") { + part = part[3:] + } else if !strings.HasPrefix(part, "rancher.") { + if !parseAll { + continue + } + } + + var value string + kv := strings.SplitN(part, "=", 2) + + if len(kv) == 1 { + value = "true" + } else { + value = kv[1] + } + + current := result + keys := strings.Split(kv[0], ".") + for i, key := range keys { + if i == len(keys)-1 { + current[key] = unmarshalOrReturnString(value) + } else { + if obj, ok := current[key]; ok { + if newCurrent, ok := obj.(map[interface{}]interface{}); ok { + current = newCurrent + } else { + continue outer + } + } else { + newCurrent := make(map[interface{}]interface{}) + current[key] = newCurrent + current = newCurrent + } + } + } + } + + return result +} diff --git a/config/config.go b/config/config.go index a6a8c05c..2dee2b73 100644 --- a/config/config.go +++ b/config/config.go @@ -5,6 +5,7 @@ import ( "strings" yaml "github.com/cloudfoundry-incubator/candiedyaml" + "github.com/rancher/os/config/cmdline" "github.com/rancher/os/util" ) @@ -41,6 +42,13 @@ func Export(private, full bool) (string, error) { bytes, err := yaml.Marshal(rawCfg) return string(bytes), err } +func filterPrivateKeys(data map[interface{}]interface{}) map[interface{}]interface{} { + for _, privateKey := range PrivateKeys { + _, data = filterKey(data, strings.Split(privateKey, ".")) + } + + return data +} func Get(key string) (interface{}, error) { cfg := LoadConfig() @@ -50,23 +58,17 @@ func Get(key string) (interface{}, error) { return nil, err } - v, _ := getOrSetVal(key, data, nil) + v, _ := cmdline.GetOrSetVal(key, data, nil) return v, nil } -func GetCmdline(key string) interface{} { - cmdline := readCmdline() - v, _ := getOrSetVal(key, cmdline, nil) - return v -} - func Set(key string, value interface{}) error { existing, err := readConfigs(nil, false, true, CloudConfigFile) if err != nil { return err } - _, modified := getOrSetVal(key, existing, value) + _, modified := cmdline.GetOrSetVal(key, existing, value) c := &CloudConfig{} if err = util.Convert(modified, c); err != nil { diff --git a/config/config_test.go b/config/config_test.go index 2d03f693..bedd508c 100755 --- a/config/config_test.go +++ b/config/config_test.go @@ -5,6 +5,7 @@ import ( yaml "github.com/cloudfoundry-incubator/candiedyaml" + "github.com/rancher/os/config/cmdline" "github.com/rancher/os/util" "github.com/stretchr/testify/require" ) @@ -100,7 +101,7 @@ func TestUnmarshalOrReturnString(t *testing.T) { assert.Equal([]interface{}{false, "a"}, unmarshalOrReturnString("[false,a]")) } -func TestParseCmdline(t *testing.T) { +func TestCmdlineParse(t *testing.T) { assert := require.New(t) assert.Equal(map[interface{}]interface{}{ @@ -108,55 +109,55 @@ func TestParseCmdline(t *testing.T) { "key1": "value1", "key2": "value2", }, - }, parseCmdline("a b rancher.key1=value1 c rancher.key2=value2")) + }, cmdline.Parse("a b rancher.key1=value1 c rancher.key2=value2"), false) assert.Equal(map[interface{}]interface{}{ "rancher": map[interface{}]interface{}{ "key": "a,b", }, - }, parseCmdline("rancher.key=a,b")) + }, cmdline.Parse("rancher.key=a,b"), false) assert.Equal(map[interface{}]interface{}{ "rancher": map[interface{}]interface{}{ "key": "a\nb", }, - }, parseCmdline("rancher.key=a\nb")) + }, cmdline.Parse("rancher.key=a\nb"), false) assert.Equal(map[interface{}]interface{}{ "rancher": map[interface{}]interface{}{ "key": "a:b", }, - }, parseCmdline("rancher.key=a:b")) + }, cmdline.Parse("rancher.key=a:b"), false) assert.Equal(map[interface{}]interface{}{ "rancher": map[interface{}]interface{}{ "key": int64(5), }, - }, parseCmdline("rancher.key=5")) + }, cmdline.Parse("rancher.key=5"), false) assert.Equal(map[interface{}]interface{}{ "rancher": map[interface{}]interface{}{ "rescue": true, }, - }, parseCmdline("rancher.rescue")) + }, cmdline.Parse("rancher.rescue"), false) assert.Equal(map[interface{}]interface{}{ "rancher": map[interface{}]interface{}{ "keyArray": []interface{}{int64(1), int64(2)}, }, - }, parseCmdline("rancher.keyArray=[1,2]")) + }, cmdline.Parse("rancher.keyArray=[1,2]"), false) assert.Equal(map[interface{}]interface{}{ "rancher": map[interface{}]interface{}{ "strArray": []interface{}{"url:http://192.168.1.100/cloud-config?a=b"}, }, - }, parseCmdline("rancher.strArray=[\"url:http://192.168.1.100/cloud-config?a=b\"]")) + }, cmdline.Parse("rancher.strArray=[\"url:http://192.168.1.100/cloud-config?a=b\"]"), false) assert.Equal(map[interface{}]interface{}{ "rancher": map[interface{}]interface{}{ "strArray": []interface{}{"url:http://192.168.1.100/cloud-config?a=b"}, }, - }, parseCmdline("rancher.strArray=[url:http://192.168.1.100/cloud-config?a=b]")) + }, cmdline.Parse("rancher.strArray=[url:http://192.168.1.100/cloud-config?a=b]"), false) } func TestGet(t *testing.T) { @@ -181,7 +182,7 @@ func TestGet(t *testing.T) { } for k, v := range tests { - val, _ := getOrSetVal(k, data, nil) + val, _ := cmdline.GeOrSetVal(k, data, nil) assert.Equal(v, val) } } @@ -225,8 +226,8 @@ func TestSet(t *testing.T) { } for k, v := range tests { - _, tData := getOrSetVal(k, data, v) - val, _ := getOrSetVal(k, tData, nil) + _, tData := cmdline.GetOrSetVal(k, data, v) + val, _ := cmdline.GetOrSetVal(k, tData, nil) data = tData assert.Equal(v, val) } diff --git a/config/data_funcs.go b/config/data_funcs.go index 71a98caf..4b27f83a 100755 --- a/config/data_funcs.go +++ b/config/data_funcs.go @@ -1,11 +1,7 @@ package config import ( - yaml "github.com/cloudfoundry-incubator/candiedyaml" "github.com/rancher/os/log" - - "strings" - "github.com/rancher/os/util" ) @@ -14,6 +10,7 @@ type CfgFuncData struct { Name string Func CfgFunc } + type CfgFuncs []CfgFuncData func ChainCfgFuncs(cfg *CloudConfig, cfgFuncs CfgFuncs) (*CloudConfig, error) { @@ -71,145 +68,3 @@ func filterKey(data map[interface{}]interface{}, key []string) (filtered, rest m return } - -func filterPrivateKeys(data map[interface{}]interface{}) map[interface{}]interface{} { - for _, privateKey := range PrivateKeys { - _, data = filterKey(data, strings.Split(privateKey, ".")) - } - - return data -} - -func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) (interface{}, map[interface{}]interface{}) { - parts := strings.Split(args, ".") - - tData := data - if value != nil { - tData = util.MapCopy(data) - } - t := tData - for i, part := range parts { - val, ok := t[part] - last := i+1 == len(parts) - - // Reached end, set the value - if last && value != nil { - if s, ok := value.(string); ok { - value = unmarshalOrReturnString(s) - } - - t[part] = value - return value, tData - } - - // Missing intermediate key, create key - if !last && value != nil && !ok { - newData := map[interface{}]interface{}{} - t[part] = newData - t = newData - continue - } - - if !ok { - break - } - - if last { - return val, tData - } - - newData, ok := val.(map[interface{}]interface{}) - if !ok { - break - } - - t = newData - } - - return "", tData -} - -// Replace newlines, colons, and question marks with random strings -// This is done to avoid YAML treating these as special characters -var ( - newlineMagicString = "9XsJcx6dR5EERYCC" - colonMagicString = "V0Rc21pIVknMm2rr" - questionMarkMagicString = "FoPL6JLMAaJqKMJT" -) - -func reverseReplacement(result interface{}) interface{} { - switch val := result.(type) { - case map[interface{}]interface{}: - for k, v := range val { - val[k] = reverseReplacement(v) - } - return val - case []interface{}: - for i, item := range val { - val[i] = reverseReplacement(item) - } - return val - case string: - val = strings.Replace(val, newlineMagicString, "\n", -1) - val = strings.Replace(val, colonMagicString, ":", -1) - val = strings.Replace(val, questionMarkMagicString, "?", -1) - return val - } - - return result -} - -func unmarshalOrReturnString(value string) (result interface{}) { - value = strings.Replace(value, "\n", newlineMagicString, -1) - value = strings.Replace(value, ":", colonMagicString, -1) - value = strings.Replace(value, "?", questionMarkMagicString, -1) - if err := yaml.Unmarshal([]byte(value), &result); err != nil { - result = value - } - result = reverseReplacement(result) - return -} - -func parseCmdline(cmdLine string) map[interface{}]interface{} { - result := make(map[interface{}]interface{}) - -outer: - for _, part := range strings.Split(cmdLine, " ") { - if strings.HasPrefix(part, "cc.") { - part = part[3:] - } else if !strings.HasPrefix(part, "rancher.") { - continue - } - - var value string - kv := strings.SplitN(part, "=", 2) - - if len(kv) == 1 { - value = "true" - } else { - value = kv[1] - } - - current := result - keys := strings.Split(kv[0], ".") - for i, key := range keys { - if i == len(keys)-1 { - current[key] = unmarshalOrReturnString(value) - } else { - if obj, ok := current[key]; ok { - if newCurrent, ok := obj.(map[interface{}]interface{}); ok { - current = newCurrent - } else { - continue outer - } - } else { - newCurrent := make(map[interface{}]interface{}) - current[key] = newCurrent - current = newCurrent - } - } - } - } - - return result -} diff --git a/config/disk.go b/config/disk.go index 9b351c29..ebe01113 100755 --- a/config/disk.go +++ b/config/disk.go @@ -14,6 +14,7 @@ import ( composeConfig "github.com/docker/libcompose/config" "github.com/rancher/os/config/cloudinit/datasource" "github.com/rancher/os/config/cloudinit/initialize" + "github.com/rancher/os/config/cmdline" "github.com/rancher/os/log" "github.com/rancher/os/util" ) @@ -48,7 +49,11 @@ func loadRawDiskConfig(dirPrefix string, full bool) map[interface{}]interface{} func loadRawConfig(dirPrefix string, full bool) map[interface{}]interface{} { rawCfg := loadRawDiskConfig(dirPrefix, full) - rawCfg = util.Merge(rawCfg, readCmdline()) + procCmdline, err := cmdline.Read(false) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params") + } + rawCfg = util.Merge(rawCfg, procCmdline) rawCfg = util.Merge(rawCfg, readElidedCmdline(rawCfg)) rawCfg = applyDebugFlags(rawCfg) return mergeMetadata(rawCfg, readMetadata()) @@ -107,7 +112,7 @@ func Insert(m interface{}, args ...interface{}) interface{} { } func SaveInitCmdline(cmdLineArgs string) { - elidedCfg := parseCmdline(cmdLineArgs) + elidedCfg := cmdline.Parse(cmdLineArgs, false) env := Insert(make(map[interface{}]interface{}), interface{}("EXTRA_CMDLINE"), interface{}(cmdLineArgs)) rancher := Insert(make(map[interface{}]interface{}), interface{}("environment"), env) @@ -155,10 +160,10 @@ func applyDebugFlags(rawCfg map[interface{}]interface{}) map[interface{}]interfa } log.SetLevel(log.DebugLevel) - _, rawCfg = getOrSetVal("rancher.docker.debug", rawCfg, true) - _, rawCfg = getOrSetVal("rancher.system_docker.debug", rawCfg, true) - _, rawCfg = getOrSetVal("rancher.bootstrap_docker.debug", rawCfg, true) - _, rawCfg = getOrSetVal("rancher.log", rawCfg, true) + _, rawCfg = cmdline.GetOrSetVal("rancher.docker.debug", rawCfg, true) + _, rawCfg = cmdline.GetOrSetVal("rancher.system_docker.debug", rawCfg, true) + _, rawCfg = cmdline.GetOrSetVal("rancher.bootstrap_docker.debug", rawCfg, true) + _, rawCfg = cmdline.GetOrSetVal("rancher.log", rawCfg, true) return rawCfg } @@ -216,7 +221,7 @@ func readElidedCmdline(rawCfg map[interface{}]interface{}) map[interface{}]inter for k, v := range rawCfg { if key, _ := k.(string); key == "EXTRA_CMDLINE" { if val, ok := v.(string); ok { - cmdLineObj := parseCmdline(strings.TrimSpace(util.UnescapeKernelParams(string(val)))) + cmdLineObj := cmdline.Parse(strings.TrimSpace(util.UnescapeKernelParams(string(val))), false) return cmdLineObj } @@ -225,22 +230,6 @@ func readElidedCmdline(rawCfg map[interface{}]interface{}) map[interface{}]inter return nil } -func readCmdline() map[interface{}]interface{} { - cmdLine, err := ioutil.ReadFile("/proc/cmdline") - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params") - return nil - } - - if len(cmdLine) == 0 { - return nil - } - - cmdLineObj := parseCmdline(strings.TrimSpace(util.UnescapeKernelParams(string(cmdLine)))) - - return cmdLineObj -} - func amendNils(c *CloudConfig) *CloudConfig { t := *c if t.Rancher.Environment == nil { diff --git a/docs/_includes/os-sidebar-v1.1.html b/docs/_includes/os-sidebar-v1.1.html index 6a44264a..50e21f33 100644 --- a/docs/_includes/os-sidebar-v1.1.html +++ b/docs/_includes/os-sidebar-v1.1.html @@ -69,6 +69,7 @@
  • Built-in System Services
  • Cloud-init
  • Image Preloading
    • +
  • Logging
  • diff --git a/docs/os/boot-process/logging/index.md b/docs/os/boot-process/logging/index.md new file mode 100644 index 00000000..64306292 --- /dev/null +++ b/docs/os/boot-process/logging/index.md @@ -0,0 +1,44 @@ +--- +title: System logging +layout: os-default + +--- + +## RancherOS system logging + +### System services + +RancherOS uses containers for its system services. This means the logs for `syslog`, `acipd`, `system-cron`, `udev`, `network`, `ntp`, `console` and the user Docker are available using `sudo ros service logs `. + +### Boot logging + +Since v1.1.0, the init process's logs are copied to `/var/log/boot` after the user-space filesystem is made available. These can be used to diagnose initialisation, network, and cloud-init issues. + +### Remote Syslog logging + +The Linux kernel has a `netconsole` logging facility that allows it to send the Kernel level logs to a remote Syslog server. +When you set this kernel boot parameter in RancherOS v1.1.0 and later, the RancherOS debug logs will also be sent to it. + +To set up Linux kernel and RancherOS remote Syslog logging, you need to set both a local, and remote host IP address - even if this address isn't the final IP address of your system. The kernel setting looks like: + +``` + netconsole=[+][src-port]@[src-ip]/[],[tgt-port]@/[tgt-macaddr] + + where + + if present, enable extended console support + src-port source for UDP packets (defaults to 6665) + src-ip source IP to use (interface address) + dev network interface (eth0) + tgt-port port for logging agent (6666) + tgt-ip IP address for logging agent + tgt-macaddr ethernet MAC address for logging agent (broadcast) +``` + +For example, on my current test system, I have set the kernel boot line to: + + +``` +printk.devkmsg=on console=tty1 rancher.autologin=tty1 console=ttyS0 rancher.autologin=ttyS0 rancher.state.dev=LABEL=RANCHER_STATE rancher.state.autoformat=[/dev/sda,/dev/vda] rancher.rm_usr loglevel=8 netconsole=+9999@10.0.2.14/,514@192.168.42.223/ +``` + +The kernel boot parameters can be set during installation using `sudo ros install --append "...."`, or on an installed RancherOS system, by running `sudo ros config syslinx` (which will start vi in a container, editing the `global.cfg` boot config file. diff --git a/init/init.go b/init/init.go index 21b2aaa0..edb82633 100755 --- a/init/init.go +++ b/init/init.go @@ -247,7 +247,7 @@ func RunInit() error { return c, dfs.PrepareFs(&mountConfig) }}, config.CfgFuncData{"save init cmdline", func(c *config.CloudConfig) (*config.CloudConfig, error) { - // will this be passed to cloud-init-save? + // the Kernel Patch added for RancherOS passes `--` (only) elided kernel boot params to the init process cmdLineArgs := strings.Join(os.Args, " ") config.SaveInitCmdline(cmdLineArgs) @@ -327,6 +327,11 @@ func RunInit() error { config.CfgFuncData{"cloud-init", func(cfg *config.CloudConfig) (*config.CloudConfig, error) { cfg.Rancher.CloudInit.Datasources = config.LoadConfigWithPrefix(state).Rancher.CloudInit.Datasources hypervisor = util.GetHypervisor() + if hypervisor == "" { + log.Infof("ros init: No Detected Hypervisor") + } else { + log.Infof("ros init: Detected Hypervisor: %s", hypervisor) + } if hypervisor == "vmware" { // add vmware to the end - we don't want to over-ride an choices the user has made cfg.Rancher.CloudInit.Datasources = append(cfg.Rancher.CloudInit.Datasources, hypervisor) @@ -339,6 +344,8 @@ func RunInit() error { if err := runCloudInitServices(cfg); err != nil { log.Error(err) } + // It'd be nice to push to rsyslog before this, but we don't have network + log.AddRSyslogHook() return config.LoadConfig(), nil }}, diff --git a/log/log.go b/log/log.go index f6b3f944..63e4c268 100755 --- a/log/log.go +++ b/log/log.go @@ -1,11 +1,17 @@ package log import ( + "fmt" "io" + "log/syslog" "os" "path/filepath" + "strings" "github.com/Sirupsen/logrus" + lsyslog "github.com/Sirupsen/logrus/hooks/syslog" + + "github.com/rancher/os/config/cmdline" ) var userHook *ShowuserlogHook @@ -118,6 +124,7 @@ func InitLogger() { if logTheseApps() { innerInit(false) FsReady() + AddRSyslogHook() pwd, err := os.Getwd() if err != nil { @@ -171,6 +178,37 @@ func innerInit(deferedHook bool) { } } +// AddRSyslogHook only needs to be called separately when using the InitDeferedLogger +// init.Main can't read /proc/cmdline at start. +// and then fails due to the network not being up +// TODO: create a "defered SyslogHook that always gets initialised, but if it fails to connect, stores the logs +// and retries connecting every time its triggered.... +func AddRSyslogHook() { + val := cmdline.GetCmdline("netconsole") + netconsole := val.(string) + if netconsole != "" { + // "loglevel=8 netconsole=9999@10.0.2.14/,514@192.168.33.148/" + + // 192.168.33.148:514 + n := strings.Split(netconsole, ",") + if len(n) == 2 { + d := strings.Split(n[1], "@") + if len(d) == 2 { + netconsoleDestination := fmt.Sprintf("%s:%s", strings.TrimRight(d[1], "/"), d[0]) + + hook, err := lsyslog.NewSyslogHook("udp", netconsoleDestination, syslog.LOG_DEBUG, "") + if err == nil { + logrus.StandardLogger().Hooks.Add(hook) + Infof("Sending RancherOS Logs to: %s", netconsoleDestination) + } else { + Errorf("Error creating SyslogHook: %s", err) + } + } + } + } + +} + func FsReady() { filename := "/var/log/boot/" + filepath.Base(os.Args[0]) + ".log" if err := os.MkdirAll(filepath.Dir(filename), os.ModeDir|0755); debugThisLogger && err != nil { diff --git a/scripts/run b/scripts/run index b3edca3d..30663d62 100755 --- a/scripts/run +++ b/scripts/run @@ -60,6 +60,9 @@ while [ "$#" -gt 0 ]; do shift 1 QEMU_APPEND="${QEMU_APPEND} $1" ;; + --rsyslog) + RSYSLOG=1 + ;; --append-init) shift 1 APPEND_INIT="${APPEND_INIT} $1" @@ -172,10 +175,6 @@ fi if [ "$RM_USR" == "1" ]; then KERNEL_ARGS="${KERNEL_ARGS} rancher.rm_usr" fi -if [ "$APPEND_INIT" != "" ]; then - KERNEL_ARGS="${KERNEL_ARGS} -- ${APPEND_INIT}" -fi - if [ "$BOOT_PXE" == "1" ]; then KERNEL_ARGS="console=tty1 rancher.autologin=tty1 ${KERNEL_ARGS}" set -ex @@ -186,6 +185,15 @@ if [ "$BOOT_PXE" == "1" ]; then --cmdline="${KERNEL_ARGS}" return 0 fi +if [ "$RSYSLOG" == "1" ]; then + defaultDev=$(ip route | grep default | cut -f 5 -d " ") + devIP=$(ip a show dev $defaultDev | grep "inet " | cut -d " " -f 6 | cut -d / -f 1) + KERNEL_ARGS="${KERNEL_ARGS} loglevel=8 netconsole=+9999@10.0.2.14/,514@${devIP}/" +fi +# ELIDE_COMMANDLINE - MUST BE LAST +if [ "$APPEND_INIT" != "" ]; then + KERNEL_ARGS="${KERNEL_ARGS} -- ${APPEND_INIT}" +fi if [ "$KVM" == "" ] && [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then KVM=1 diff --git a/util/util.go b/util/util.go index 91c5c1ee..b19992fe 100644 --- a/util/util.go +++ b/util/util.go @@ -11,8 +11,6 @@ import ( "path" "strings" - "github.com/rancher/os/log" - yaml "github.com/cloudfoundry-incubator/candiedyaml" osYaml "github.com/rancher/os/config/yaml" ) @@ -71,7 +69,6 @@ func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error { func Convert(from, to interface{}) error { bytes, err := yaml.Marshal(from) if err != nil { - log.WithFields(log.Fields{"from": from, "err": err}).Warn("Error serializing to YML") return err } @@ -270,7 +267,7 @@ func RunScript(path string) error { return cmd.Run() } -func RunCommandSequence(commandSequence []osYaml.StringandSlice) { +func RunCommandSequence(commandSequence []osYaml.StringandSlice) error { for _, command := range commandSequence { var cmd *exec.Cmd if command.StringValue != "" { @@ -283,7 +280,8 @@ func RunCommandSequence(commandSequence []osYaml.StringandSlice) { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - log.Errorf("Failed to run %s: %v", command, err) + return fmt.Errorf("Failed to run %s: %v", command, err) } } + return nil } diff --git a/util/util_linux.go b/util/util_linux.go index 7525b0d8..fd3ce4d4 100755 --- a/util/util_linux.go +++ b/util/util_linux.go @@ -13,7 +13,6 @@ import ( "github.com/SvenDowideit/cpuid" "github.com/docker/docker/pkg/mount" - "github.com/rancher/os/log" ) func mountProc() error { @@ -94,20 +93,18 @@ func Unmount(target string) error { return mount.Unmount(target) } -func Blkid(label string) (deviceName, deviceType string) { +func Blkid(label string) (deviceName, deviceType string, err error) { // Not all blkid's have `blkid -L label (see busybox/alpine) cmd := exec.Command("blkid") cmd.Stderr = os.Stderr out, err := cmd.Output() if err != nil { - log.Errorf("Failed to run blkid: %s", err) return } r := bytes.NewReader(out) s := bufio.NewScanner(r) for s.Scan() { line := s.Text() - //log.Debugf("blkid: %s", cmd, line) if !strings.Contains(line, `LABEL="`+label+`"`) { continue } @@ -117,7 +114,6 @@ func Blkid(label string) (deviceName, deviceType string) { s1 := strings.Split(line, `TYPE="`) s2 := strings.Split(s1[1], `"`) deviceType = s2[0] - log.Debugf("blkid type of %s: %s", deviceName, deviceType) return } return @@ -125,10 +121,5 @@ func Blkid(label string) (deviceName, deviceType string) { // GetHypervisor tries to detect if we're running in a VM, and returns a string for its type func GetHypervisor() string { - if cpuid.CPU.HypervisorName == "" { - log.Infof("ros init: No Detected Hypervisor") - } else { - log.Infof("ros init: Detected Hypervisor: %s", cpuid.CPU.HypervisorName) - } return cpuid.CPU.HypervisorName }