github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-06-21 13:58:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Introducing mizu logs dump & Log prints alignment in API server using rlog (#165)

Browse Source
This commit is contained in:
Igor Gov 2021-08-05 11:01:08 +03:00 committed by GitHub
parent 04579eb03c
commit fa632b49a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 241 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
agent/pkg/api/socket_routes.go
View File

@ -2,9 +2,9 @@ package api
import (
"errors"
"fmt"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared/debounce"
"net/http"
"sync"
@ -50,7 +50,7 @@ func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
conn, err := websocketUpgrader.Upgrade(w, r, nil)
if err != nil {
fmt.Println("Failed to set websocket upgrade: %+v", err)
rlog.Errorf("Failed to set websocket upgrade: %v", err)
return
}
@ -71,7 +71,7 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
for {
_, msg, err := conn.ReadMessage()
if err != nil {
fmt.Printf("Conn err: %v\n", err)
rlog.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
break
}
eventHandlers.WebSocketMessage(socketId, msg)
@ -81,7 +81,7 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
func socketCleanup(socketId int, socketConnection *SocketConnection) {
err := socketConnection.connection.Close()
if err != nil {
fmt.Printf("Error closing socket connection for socket id %d: %v\n", socketId, err)
rlog.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
}
websocketIdsLock.Lock()
@ -92,7 +92,7 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
}
var db = debounce.NewDebouncer(time.Second*5, func() {
fmt.Println("Successfully sent to socket")
rlog.Error("Successfully sent to socket")
})
func SendToSocket(socketId int, message []byte) error {
@ -104,7 +104,7 @@ func SendToSocket(socketId int, message []byte) error {
var sent = false
time.AfterFunc(time.Second*5, func() {
if !sent {
fmt.Println("Socket timed out")
rlog.Error("Socket timed out")
socketCleanup(socketId, socketObj)
}
})

4
agent/pkg/api/socket_server_handlers.go
View File

@ -52,7 +52,7 @@ func BroadcastToBrowserClients(message []byte) {
go func(socketId int) {
err := SendToSocket(socketId, message)
if err != nil {
fmt.Printf("error sending message to socket ID %d: %v", socketId, err)
rlog.Errorf("error sending message to socket ID %d: %v", socketId, err)
}
}(socketId)
}
@ -114,7 +114,7 @@ func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
if err != nil {
rlog.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
} else {
fmt.Printf("Broadcasting outboundlink message %s\n", string(marshaledMessage))
rlog.Errorf("Broadcasting outboundlink message %s", string(marshaledMessage))
BroadcastToBrowserClients(marshaledMessage)
}
}

16
agent/pkg/database/size_enforcer.go
View File

@ -1,8 +1,8 @@
package database
import (
"fmt"
"github.com/fsnotify/fsnotify"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
"github.com/up9inc/mizu/shared/units"
@ -47,7 +47,7 @@ func StartEnforcingDatabaseSize() {
if !ok {
return // closed channel
}
fmt.Printf("filesystem watcher encountered error:%v\n", err)
rlog.Errorf("filesystem watcher encountered error:%v", err)
}
}
}()
@ -72,7 +72,7 @@ func getMaxEntriesDBByteSize() (int64, error) {
func checkFileSize(maxSizeBytes int64) {
fileStat, err := os.Stat(DBPath)
if err != nil {
fmt.Printf("Error checking %s file size: %v\n", DBPath, err)
rlog.Errorf("Error checking %s file size: %v", DBPath, err)
} else {
if fileStat.Size() > maxSizeBytes {
pruneOldEntries(fileStat.Size())
@ -83,13 +83,13 @@ func checkFileSize(maxSizeBytes int64) {
func pruneOldEntries(currentFileSize int64) {
// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
IsDBLocked = true
defer func() {IsDBLocked = false}()
defer func() { IsDBLocked = false }()
amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
if err != nil {
fmt.Printf("Error getting 10000 first db rows: %v\n", err)
rlog.Errorf("Error getting 10000 first db rows: %v", err)
return
}
@ -102,7 +102,7 @@ func pruneOldEntries(currentFileSize int64) {
var entry models.MizuEntry
err = DB.ScanRows(rows, &entry)
if err != nil {
fmt.Printf("Error scanning db row: %v\n", err)
rlog.Errorf("Error scanning db row: %v", err)
continue
}
@ -114,8 +114,8 @@ func pruneOldEntries(currentFileSize int64) {
GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{})
// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
DB.Exec("VACUUM")
fmt.Printf("Removed %d rows and cleared %s\n", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
rlog.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
} else {
fmt.Println("Found no rows to remove when pruning")
rlog.Error("Found no rows to remove when pruning")
}
}

8
agent/pkg/resolver/README.md
View File

@ -32,7 +32,7 @@ Now you will be able to import `github.com/up9inc/mizu/resolver` in any `.go` fi
errOut := make(chan error, 100)
k8sResolver, err := resolver.NewFromOutOfCluster("", errOut)
if err != nil {
fmt.Printf("error creating k8s resolver %s", err)
rlog.Errorf("error creating k8s resolver %s", err)
}
ctx, cancel := context.WithCancel(context.Background())
@ -40,15 +40,15 @@ k8sResolver.Start(ctx)
resolvedName := k8sResolver.Resolve("10.107.251.91") // will always return `nil` in real scenarios as the internal map takes a moment to populate after `Start` is called
if resolvedName != nil {
fmt.Printf("resolved 10.107.251.91=%s", *resolvedName)
rlog.Errorf("resolved 10.107.251.91=%s", *resolvedName)
} else {
fmt.Printf("Could not find a resolved name for 10.107.251.91")
rlog.Error("Could not find a resolved name for 10.107.251.91")
}
for {
select {
case err := <- errOut:
fmt.Printf("name resolving error %s", err)
rlog.Errorf("name resolving error %s", err)
}
}
```

9
agent/pkg/utils/truncating_logger.go
View File

@ -3,6 +3,7 @@ package utils
import (
"context"
"fmt"
"github.com/romana/rlog"
"gorm.io/gorm/logger"
"gorm.io/gorm/utils"
"time"
@ -10,7 +11,7 @@ import (
// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact
type TruncatingLogger struct {
LogLevel logger.LogLevel
LogLevel logger.LogLevel
SlowThreshold time.Duration
}
@ -23,21 +24,21 @@ func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string
if truncatingLogger.LogLevel < logger.Info {
return
}
fmt.Printf("gorm info: %.150s\n", message)
rlog.Errorf("gorm info: %.150s", message)
}
func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) {
if truncatingLogger.LogLevel < logger.Warn {
return
}
fmt.Printf("gorm warning: %.150s\n", message)
rlog.Errorf("gorm warning: %.150s", message)
}
func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) {
if truncatingLogger.LogLevel < logger.Error {
return
}
fmt.Printf("gorm error: %.150s\n", message)
rlog.Errorf("gorm error: %.150s", message)
}
func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {

46
cli/cmd/logs.go Normal file
View File

@ -0,0 +1,46 @@
package cmd
import (
"context"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/logsUtils"
"github.com/up9inc/mizu/cli/mizu"
"os"
"path"
)
var filePath string
var logsCmd = &cobra.Command{
Use: "logs",
Short: "Create a zip file with logs for Github issue or troubleshoot",
RunE: func(cmd *cobra.Command, args []string) error {
kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath)
if err != nil {
return nil
}
ctx, _ := context.WithCancel(context.Background())
if filePath == "" {
pwd, err := os.Getwd()
if err != nil {
mizu.Log.Errorf("Failed to get PWD, %v (try using `mizu logs -f <full path dest zip file>)`", err)
return nil
}
filePath = path.Join(pwd, "mizu_logs.zip")
}
mizu.Log.Debugf("Using file path %s", filePath)
if err := logsUtils.DumpLogs(kubernetesProvider, ctx, filePath); err != nil {
mizu.Log.Errorf("Failed dump logs %v", err)
}
return nil
},
}
func init() {
rootCmd.AddCommand(logsCmd)
logsCmd.Flags().StringVarP(&filePath, "file", "f", "", "Path for zip file (default current <pwd>\\mizu_logs.zip)")
}

24
cli/cmd/tapRunner.go
View File

@ -9,12 +9,14 @@ import (
"net/url"
"os"
"os/signal"
"path"
"regexp"
"syscall"
"time"
"github.com/up9inc/mizu/cli/errormessage"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/logsUtils"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
@ -22,7 +24,6 @@ import (
yaml "gopkg.in/yaml.v3"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/clientcmd"
)
const (
@ -56,14 +57,8 @@ func RunMizuTap() {
kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.Tap.KubeConfigPath)
if err != nil {
if clientcmd.IsEmptyConfig(err) {
mizu.Log.Errorf(uiUtils.Error, "Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
return
}
if clientcmd.IsConfigurationInvalid(err) {
mizu.Log.Errorf(uiUtils.Error, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
return
}
mizu.Log.Error(err)
return
}
defer cleanUpMizuResources(kubernetesProvider)
@ -244,11 +239,20 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
}
func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
mizu.Log.Infof("\nRemoving mizu resources\n")
removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
defer cancel()
if mizu.Config.DumpLogs {
mizuDir := mizu.GetMizuFolderPath()
filePath = path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
if err := logsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil {
mizu.Log.Errorf("Failed dump logs %v", err)
}
}
mizu.Log.Infof("\nRemoving mizu resources\n")
if mizu.Config.IsOwnNamespace() {
if err := kubernetesProvider.RemoveNamespace(removalCtx, mizu.Config.ResourcesNamespace()); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", mizu.Config.ResourcesNamespace(), errormessage.FormatError(err)))

15
cli/cmd/viewRunner.go
View File

@ -3,25 +3,16 @@ package cmd
import (
"context"
"fmt"
"net/http"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"k8s.io/client-go/tools/clientcmd"
"net/http"
)
func runMizuView() {
kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath)
if err != nil {
if clientcmd.IsEmptyConfig(err) {
mizu.Log.Infof("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'")
return
}
if clientcmd.IsConfigurationInvalid(err) {
mizu.Log.Infof(uiUtils.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'")
return
}
mizu.Log.Error(err)
return
}
ctx, cancel := context.WithCancel(context.Background())

46
cli/kubernetes/provider.go
View File

@ -1,6 +1,7 @@
package kubernetes
import (
"bytes"
_ "bytes"
"context"
"encoding/json"
@ -12,13 +13,17 @@ import (
"strconv"
"github.com/up9inc/mizu/cli/mizu"
"io"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/homedir"
"github.com/up9inc/mizu/shared"
core "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
resource "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/watch"
applyconfapp "k8s.io/client-go/applyconfigurations/apps/v1"
@ -30,11 +35,9 @@ import (
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/clientcmd"
_ "k8s.io/client-go/tools/portforward"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/homedir"
)
type Provider struct {
@ -52,7 +55,12 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
kubernetesConfig := loadKubernetesConfiguration(kubeConfigPath)
restClientConfig, err := kubernetesConfig.ClientConfig()
if err != nil {
return nil, err
if clientcmd.IsEmptyConfig(err) {
return nil, fmt.Errorf("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
}
if clientcmd.IsConfigurationInvalid(err) {
return nil, fmt.Errorf("Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
}
}
clientSet := getClientSet(restClientConfig)
@ -551,10 +559,22 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
if _, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}); err != nil {
return err
}
return nil
}
func (provider *Provider) ListPods(ctx context.Context, namespace string) ([]shared.PodInfo, error) {
podInfos := make([]shared.PodInfo, 0)
listOptions := metav1.ListOptions{}
pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, listOptions)
if err != nil {
return podInfos, fmt.Errorf("error getting pods in ns: %s, %w", namespace, err)
}
for _, pod := range pods.Items {
podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
}
return podInfos, nil
}
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool) error {
mizu.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
@ -685,6 +705,22 @@ func (provider *Provider) GetAllRunningPodsMatchingRegex(ctx context.Context, re
return matchingPods, nil
}
func (provider *Provider) GetPodLogs(namespace string, podName string, ctx context.Context) (string, error) {
podLogOpts := core.PodLogOptions{}
req := provider.clientSet.CoreV1().Pods(namespace).GetLogs(podName, &podLogOpts)
podLogs, err := req.Stream(ctx)
if err != nil {
return "", fmt.Errorf("error opening log stream on ns: %s, pod: %s, %w", namespace, podName, err)
}
defer podLogs.Close()
buf := new(bytes.Buffer)
if _, err = io.Copy(buf, podLogs); err != nil {
return "", fmt.Errorf("error copy information from podLogs to buf, ns: %s, pod: %s, %w", namespace, podName, err)
}
str := buf.String()
return str, nil
}
func getClientSet(config *restclient.Config) *kubernetes.Clientset {
clientSet, err := kubernetes.NewForConfig(config)
if err != nil {

104
cli/logsUtils/mizuLogsUtils.go Normal file
View File

@ -0,0 +1,104 @@
package logsUtils
import (
"archive/zip"
"context"
"fmt"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"io"
"os"
"path/filepath"
)
func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath string) error {
pods, err := provider.ListPods(ctx, mizu.Config.ResourcesNamespace())
if err != nil {
return err
}
if len(pods) == 0 {
return fmt.Errorf("no pods found in namespace %s", mizu.Config.ResourcesNamespace())
}
newZipFile, err := os.Create(filePath)
if err != nil {
return err
}
defer newZipFile.Close()
zipWriter := zip.NewWriter(newZipFile)
defer zipWriter.Close()
for _, pod := range pods {
logs, err := provider.GetPodLogs(pod.Namespace, pod.Name, ctx)
if err != nil {
mizu.Log.Errorf("Failed to get logs, %v", err)
continue
} else {
mizu.Log.Debugf("Successfully read log length %d for pod: %s.%s", len(logs), pod.Namespace, pod.Name)
}
if err := addLogsToZip(zipWriter, logs, fmt.Sprintf("%s.%s.log", pod.Namespace, pod.Name)); err != nil {
mizu.Log.Errorf("Failed write logs, %v", err)
} else {
mizu.Log.Infof("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name)
}
}
if err := addFileToZip(zipWriter, mizu.GetConfigFilePath()); err != nil {
mizu.Log.Errorf("Failed write file, %v", err)
} else {
mizu.Log.Infof("Successfully added file %s", mizu.GetConfigFilePath())
}
if err := addFileToZip(zipWriter, mizu.GetLogFilePath()); err != nil {
mizu.Log.Errorf("Failed write file, %v", err)
} else {
mizu.Log.Infof("Successfully added file %s", mizu.GetLogFilePath())
}
mizu.Log.Infof("You can find the zip with all logs in %s\n", filePath)
return nil
}
func addFileToZip(zipWriter *zip.Writer, filename string) error {
fileToZip, err := os.Open(filename)
if err != nil {
return fmt.Errorf("failed to open file %s, %w", filename, err)
}
defer fileToZip.Close()
// Get the file information
info, err := fileToZip.Stat()
if err != nil {
return fmt.Errorf("failed to get file information %s, %w", filename, err)
}
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
// Using FileInfoHeader() above only uses the basename of the file. If we want
// to preserve the folder structure we can overwrite this with the full path.
header.Name = filepath.Base(filename)
// Change to deflate to gain better compression
// see http://golang.org/pkg/archive/zip/#pkg-constants
header.Method = zip.Deflate
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return fmt.Errorf("failed to create header in zip for %s, %w", filename, err)
}
_, err = io.Copy(writer, fileToZip)
return err
}
func addLogsToZip(writer *zip.Writer, logs string, fileName string) error {
if zipFile, err := writer.Create(fileName); err != nil {
return fmt.Errorf("couldn't create a log file inside zip for %s, %w", fileName, err)
} else {
if _, err = zipFile.Write([]byte(logs)); err != nil {
return fmt.Errorf("couldn't write logs to zip file: %s, %w", fileName, err)
}
}
return nil
}

2
cli/mizu/config.go
View File

@ -51,7 +51,7 @@ func GetConfigWithDefaults() (string, error) {
}
func GetConfigFilePath() string {
return path.Join(getMizuFolderPath(), "config.yaml")
return path.Join(GetMizuFolderPath(), "config.yaml")
}
func mergeConfigFile() error {

1
cli/mizu/configStruct.go
View File

@ -14,6 +14,7 @@ type ConfigStruct struct {
MizuImage string `yaml:"mizu-image"`
MizuNamespace string `yaml:"mizu-namespace"`
Telemetry bool `yaml:"telemetry" default:"true"`
DumpLogs bool `yaml:"dump-logs" default:"false"`
}
func (config *ConfigStruct) SetDefaults() {

2
cli/mizu/consts.go
View File

@ -27,7 +27,7 @@ const (
ConfigMapName = "mizu-policy"
)
func getMizuFolderPath() string {
func GetMizuFolderPath() string {
home, homeDirErr := os.UserHomeDir()
if homeDirErr != nil {
return ""

10
cli/mizu/logger.go
View File

@ -13,12 +13,12 @@ var format = logging.MustStringFormatter(
`%{time} %{level:.5s} ▶ %{pid} %{shortfile} %{shortfunc} ▶ %{message}`,
)
func GetLogFilePath() string {
return path.Join(GetMizuFolderPath(), "mizu_cli.log")
}
func InitLogger() {
mizuDirPath := getMizuFolderPath()
if err := os.MkdirAll(mizuDirPath, os.ModePerm); err != nil {
panic(fmt.Sprintf("Failed creating mizu dir: %v, err %v", mizuDirPath, err))
}
logPath := path.Join(mizuDirPath, "log.log")
logPath := GetLogFilePath()
f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
panic(fmt.Sprintf("Failed mizu log file: %v, err %v", logPath, err))