From a91c4a77b033a795e76c767873bb1bafe71082ef Mon Sep 17 00:00:00 2001 From: Avi Deitcher Date: Tue, 3 Jan 2023 10:47:42 +0200 Subject: [PATCH] Logwrite sync with memlogd (#3890) * sync logwrite with memlogd Signed-off-by: Avi Deitcher * update linuxkit/logwrite and linuxkit/memlogd dependencies Signed-off-by: Avi Deitcher Signed-off-by: Avi Deitcher --- examples/logging.yml | 4 +- pkg/logwrite/logwrite.go | 67 +++++-------------- test/cases/040_packages/011_kmsg/test.yml | 4 +- test/cases/040_packages/012_logwrite/test.yml | 4 +- 4 files changed, 24 insertions(+), 55 deletions(-) diff --git a/examples/logging.yml b/examples/logging.yml index 220279f80..849f16cf0 100644 --- a/examples/logging.yml +++ b/examples/logging.yml @@ -7,7 +7,7 @@ init: - linuxkit/runc:f01b88c7033180d50ae43562d72707c6881904e4 - linuxkit/containerd:de1b18eed76a266baa3092e5c154c84f595e56da - linuxkit/ca-certificates:c1c73ef590dffb6a0138cf758fe4a4305c9864f4 - - linuxkit/memlogd:014f86dce2ea4bb2ec13e92ae5c1e854bcefec40 + - linuxkit/memlogd:36758a7ab239943852c4f849283e28ba975413c6 onboot: - name: sysctl image: linuxkit/sysctl:bdc99eeedc224439ff237990ee06e5b992c8c1ae @@ -25,6 +25,6 @@ services: image: alpine:3.13 command: ["/bin/sh", "-c", "while /bin/true; do echo hello $(date); sleep 1; done" ] - name: write-and-rotate-logs - image: linuxkit/logwrite:4d8aa07d4a7130239fc62b09f33e3401ecf62a38 + image: linuxkit/logwrite:6100d207d36a089f322defb60e86032c6f66078c - name: kmsg image: linuxkit/kmsg:b2f6cd4ce9041120e30a4b5ab36bb8db4f5eb458 diff --git a/pkg/logwrite/logwrite.go b/pkg/logwrite/logwrite.go index b8667b745..b8e04e6f6 100644 --- a/pkg/logwrite/logwrite.go +++ b/pkg/logwrite/logwrite.go @@ -3,8 +3,7 @@ package main // Write logs to files and perform rotation. import ( - "bufio" - "errors" + "encoding/json" "flag" "fmt" "io" @@ -27,36 +26,13 @@ const mb = 1024 * 1024 // LogMessage is a message received from memlogd. type LogMessage struct { - Time time.Time // time message was received by memlogd - Name string // name of the service that wrote the message - Message string // body of the message + Time time.Time `json:"time"` // time message was received by memlogd + Source string `json:"source"` // name of the service that wrote the message + Msg string `json:"msg"` // body of the message } func (m *LogMessage) String() string { - return m.Time.Format(time.RFC3339) + " " + m.Name + " " + m.Message -} - -// ParseLogMessage reconstructs a LogMessage from a line of text which looks like: -// ,; -func ParseLogMessage(line string) (*LogMessage, error) { - bits := strings.SplitN(line, ";", 2) - if len(bits) != 2 { - return nil, errors.New("Failed to parse log message: " + line) - } - bits2 := strings.Split(bits[0], ",") - if len(bits2) < 2 { - // There could be more parameters in future - return nil, errors.New("Failed to parse log message: " + line) - } - Time, err := time.Parse(time.RFC3339, bits2[0]) - if err != nil { - return nil, err - } - return &LogMessage{ - Time: Time, - Name: bits2[1], - Message: bits[1], - }, nil + return m.Time.Format(time.RFC3339) + " " + m.Source + " " + m.Msg } // LogFile is where we write LogMessages to @@ -87,7 +63,7 @@ func NewLogFile(dir, name string) (*LogFile, error) { // Write appends a message to the log file func (l *LogFile) Write(m *LogMessage) error { - s := m.String() + s := m.String() + "\n" _, err := io.WriteString(l.File, s) if err == nil { l.BytesWritten += len(s) @@ -157,41 +133,34 @@ func main() { // map of service name to active log file logs := make(map[string]*LogFile) - r := bufio.NewReader(conn) + var msg LogMessage + decoder := json.NewDecoder(conn) for { - line, err := r.ReadString('\n') - if err == io.EOF { - return - } - if err != nil { - log.Fatalf("Failed to read from memlogd: %v", err) - } - msg, err := ParseLogMessage(line) - if err != nil { + if err := decoder.Decode(&msg); err != nil { log.Println(err) continue } - if strings.HasPrefix(msg.Name, "logwrite") { + if strings.HasPrefix(msg.Source, "logwrite") { // don't log our own output in a loop continue } var logF *LogFile var ok bool - if logF, ok = logs[msg.Name]; !ok { - logF, err = NewLogFile(*logDir, msg.Name) + if logF, ok = logs[msg.Source]; !ok { + logF, err = NewLogFile(*logDir, msg.Source) if err != nil { - log.Printf("Failed to create log file %s: %v", msg.Name, err) + log.Printf("Failed to create log file %s: %v", msg.Source, err) continue } - logs[msg.Name] = logF + logs[msg.Source] = logF } - if err = logF.Write(msg); err != nil { - log.Printf("Failed to write to log file %s: %v", msg.Name, err) + if err = logF.Write(&msg); err != nil { + log.Printf("Failed to write to log file %s: %v", msg.Source, err) if err := logF.Close(); err != nil { - log.Printf("Failed to close log file %s: %v", msg.Name, err) + log.Printf("Failed to close log file %s: %v", msg.Source, err) } - delete(logs, msg.Name) + delete(logs, msg.Source) continue } if logF.BytesWritten > *maxLogSize { diff --git a/test/cases/040_packages/011_kmsg/test.yml b/test/cases/040_packages/011_kmsg/test.yml index cb58fb3f5..d59a7d55e 100644 --- a/test/cases/040_packages/011_kmsg/test.yml +++ b/test/cases/040_packages/011_kmsg/test.yml @@ -6,12 +6,12 @@ init: - linuxkit/runc:f01b88c7033180d50ae43562d72707c6881904e4 - linuxkit/containerd:de1b18eed76a266baa3092e5c154c84f595e56da - linuxkit/ca-certificates:c1c73ef590dffb6a0138cf758fe4a4305c9864f4 - - linuxkit/memlogd:014f86dce2ea4bb2ec13e92ae5c1e854bcefec40 + - linuxkit/memlogd:36758a7ab239943852c4f849283e28ba975413c6 services: - name: kmsg image: linuxkit/kmsg:b2f6cd4ce9041120e30a4b5ab36bb8db4f5eb458 - name: write-and-rotate-logs - image: linuxkit/logwrite:4d8aa07d4a7130239fc62b09f33e3401ecf62a38 + image: linuxkit/logwrite:6100d207d36a089f322defb60e86032c6f66078c - name: check-the-logs image: alpine:3.13 binds: diff --git a/test/cases/040_packages/012_logwrite/test.yml b/test/cases/040_packages/012_logwrite/test.yml index abcfae540..6be144f93 100644 --- a/test/cases/040_packages/012_logwrite/test.yml +++ b/test/cases/040_packages/012_logwrite/test.yml @@ -6,14 +6,14 @@ init: - linuxkit/runc:f01b88c7033180d50ae43562d72707c6881904e4 - linuxkit/containerd:de1b18eed76a266baa3092e5c154c84f595e56da - linuxkit/ca-certificates:c1c73ef590dffb6a0138cf758fe4a4305c9864f4 - - linuxkit/memlogd:014f86dce2ea4bb2ec13e92ae5c1e854bcefec40 + - linuxkit/memlogd:36758a7ab239943852c4f849283e28ba975413c6 services: # A service which generates logs of log messages - name: fill-the-logs image: alpine command: ["/bin/sh", "-c", "while /bin/true; do echo hello $(date); done" ] - name: write-and-rotate-logs - image: linuxkit/logwrite:4d8aa07d4a7130239fc62b09f33e3401ecf62a38 + image: linuxkit/logwrite:6100d207d36a089f322defb60e86032c6f66078c command: ["/usr/bin/logwrite", "-max-log-size", "1024"] - name: check-the-logs image: alpine:3.13