Merge pull request #244 from jodh-intel/backtrace-on-sigusr1

cli: Backtrace on SIGUSR1

cli/config.go

@@ -470,6 +470,7 @@ func loadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 	}
 
 	if tomlConf.Runtime.Debug {
+		debug = true
 		crashOnError = true
 	} else {
 		// If debug is not required, switch back to the original

cli/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -52,6 +53,8 @@ var kataLog *logrus.Entry
 // required.
 var originalLoggerLevel logrus.Level
 
+var debug = false
+
 // if true, coredump when an internal error occurs or a fatal signal is received
 var crashOnError = false
 
@@ -156,18 +159,27 @@ func init() {
 func setupSignalHandler() {
 	sigCh := make(chan os.Signal, 8)
 
-	for _, sig := range fatalSignals() {
+	for _, sig := range handledSignals() {
 		signal.Notify(sigCh, sig)
 	}
 
 	go func() {
-		sig := <-sigCh
+		for {
+			sig := <-sigCh
+
+			nativeSignal, ok := sig.(syscall.Signal)
+			if !ok {
+				err := errors.New("unknown signal")
+				kataLog.WithError(err).WithField("signal", sig.String()).Error()
+				continue
+			}
 
-		nativeSignal, ok := sig.(syscall.Signal)
-		if ok {
 			if fatalSignal(nativeSignal) {
 				kataLog.WithField("signal", sig).Error("received fatal signal")
 				die()
+			} else if debug && nonFatalSignal(nativeSignal) {
+				kataLog.WithField("signal", sig).Debug("handling signal")
+				backtrace()
 			}
 		}
 	}()

cli/signals.go

@@ -14,8 +14,10 @@ import (
 	"syscall"
 )
 
-// List of fatal signals
-var sigFatal = map[syscall.Signal]bool{
+// List of handled signals.
+//
+// The value is true if receiving the signal should be fatal.
+var handledSignalsMap = map[syscall.Signal]bool{
 	syscall.SIGABRT:   true,
 	syscall.SIGBUS:    true,
 	syscall.SIGILL:    true,
@@ -24,6 +26,7 @@ var sigFatal = map[syscall.Signal]bool{
 	syscall.SIGSTKFLT: true,
 	syscall.SIGSYS:    true,
 	syscall.SIGTRAP:   true,
+	syscall.SIGUSR1:   false,
 }
 
 func handlePanic() {
@@ -54,15 +57,28 @@ func backtrace() {
 }
 
 func fatalSignal(sig syscall.Signal) bool {
-	return sigFatal[sig]
+	s, exists := handledSignalsMap[sig]
+	if !exists {
+		return false
+	}
+
+	return s
 }
 
-func fatalSignals() []syscall.Signal {
+func nonFatalSignal(sig syscall.Signal) bool {
+	s, exists := handledSignalsMap[sig]
+	if !exists {
+		return false
+	}
+
+	return !s
+}
+
+func handledSignals() []syscall.Signal {
 	var signals []syscall.Signal
 
-	for sig := range sigFatal {
+	for sig := range handledSignalsMap {
 		signals = append(signals, sig)
-
 	}
 
 	return signals

cli/signals_test.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"bytes"
+	"reflect"
+	goruntime "runtime"
+	"sort"
+	"strings"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSignalFatalSignal(t *testing.T) {
+	assert := assert.New(t)
+
+	for sig, fatal := range handledSignalsMap {
+		result := nonFatalSignal(sig)
+		if fatal {
+			assert.False(result)
+		} else {
+			assert.True(result)
+		}
+	}
+}
+
+func TestSignalHandledSignalsMap(t *testing.T) {
+	assert := assert.New(t)
+
+	for sig, fatal := range handledSignalsMap {
+		result := fatalSignal(sig)
+		if fatal {
+			assert.True(result)
+		} else {
+			assert.False(result)
+		}
+	}
+}
+
+func TestSignalHandledSignals(t *testing.T) {
+	assert := assert.New(t)
+
+	var expected []syscall.Signal
+
+	for sig := range handledSignalsMap {
+		expected = append(expected, sig)
+	}
+
+	got := handledSignals()
+
+	sort.Slice(expected, func(i, j int) bool {
+		return int(expected[i]) < int(expected[j])
+	})
+
+	sort.Slice(got, func(i, j int) bool {
+		return int(got[i]) < int(got[j])
+	})
+
+	assert.True(reflect.DeepEqual(expected, got))
+}
+
+func TestSignalNonFatalSignal(t *testing.T) {
+	assert := assert.New(t)
+
+	for sig, fatal := range handledSignalsMap {
+		result := nonFatalSignal(sig)
+		if fatal {
+			assert.False(result)
+		} else {
+			assert.True(result)
+		}
+	}
+}
+
+func TestSignalFatalSignalInvalidSignal(t *testing.T) {
+	assert := assert.New(t)
+
+	sig := syscall.SIGXCPU
+
+	result := fatalSignal(sig)
+	assert.False(result)
+}
+
+func TestSignalNonFatalSignalInvalidSignal(t *testing.T) {
+	assert := assert.New(t)
+
+	sig := syscall.SIGXCPU
+
+	result := nonFatalSignal(sig)
+	assert.False(result)
+}
+
+func TestSignalBacktrace(t *testing.T) {
+	assert := assert.New(t)
+
+	// create buffer to save logger output
+	buf := &bytes.Buffer{}
+
+	savedOut := kataLog.Logger.Out
+	defer func() {
+		kataLog.Logger.Out = savedOut
+	}()
+
+	// capture output to buffer
+	kataLog.Logger.Out = buf
+
+	// determine name of *this* function
+	pc := make([]uintptr, 1)
+	goruntime.Callers(1, pc)
+	fn := goruntime.FuncForPC(pc[0])
+	name := fn.Name()
+
+	backtrace()
+
+	b := buf.String()
+
+	// very basic tests to check if a backtrace was produced
+	assert.True(strings.Contains(b, "contention:"))
+	assert.True(strings.Contains(b, `"level":"error"`))
+	assert.True(strings.Contains(b, name))
+}