Merge branch 'develop'

Conflicts:
	api/pkg/models/models.go
	cli/cmd/tapRunner.go
	cli/mizu/mizuRunner.go
This commit is contained in:
RamiBerm 2021-05-25 09:47:16 +03:00
commit 1f09a140a4
14 changed files with 252 additions and 64 deletions

View File

@ -64,6 +64,65 @@ func GetEntries(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).JSON(baseEntries)
}
func GetHAR(c *fiber.Ctx) error {
entriesFilter := &models.HarFetchRequestBody{}
order := OrderDesc
if err := c.QueryParser(entriesFilter); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
}
err := validation.Validate(entriesFilter)
if err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
}
var entries []models.MizuEntry
database.GetEntriesTable().
Order(fmt.Sprintf("timestamp %s", order)).
// Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
Limit(1000).
Find(&entries)
if len(entries) > 0 {
// the entries always order from oldest to newest so we should revers
utils.ReverseSlice(entries)
}
harsObject := map[string]*har.HAR{}
for _, entryData := range entries {
harEntryObject := []byte(entryData.Entry)
var harEntry har.Entry
_ = json.Unmarshal(harEntryObject, &harEntry)
sourceOfEntry := *entryData.ResolvedSource
if harOfSource, ok := harsObject[sourceOfEntry]; ok {
harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry)
} else {
var entriesHar []*har.Entry
entriesHar = append(entriesHar, &harEntry)
harsObject[sourceOfEntry] = &har.HAR{
Log: &har.Log{
Version: "1.2",
Creator: &har.Creator{
Name: "mizu",
Version: "0.0.1",
},
Entries: entriesHar,
},
}
}
}
retObj := map[string][]byte{}
for k, v := range harsObject {
bytesData, _ := json.Marshal(v)
retObj[k] = bytesData
}
buffer := utils.ZipData(retObj)
return c.Status(fiber.StatusOK).SendStream(buffer)
}
func GetEntry(c *fiber.Ctx) error {
var entryData models.EntryData
database.GetEntriesTable().

View File

@ -45,11 +45,16 @@ type EntriesFilter struct {
Timestamp int64 `query:"timestamp" validate:"required,min=1"`
}
type HarFetchRequestBody struct {
Limit int `query:"limit" validate:"max=5000"`
}
type WebSocketEntryMessage struct {
*shared.WebSocketMessageMetadata
Data *BaseEntryDetails `json:"data,omitempty"`
}
type WebSocketTappedEntryMessage struct {
*shared.WebSocketMessageMetadata
Data *tap.OutputChannelItem

View File

@ -12,6 +12,7 @@ func EntriesRoutes(fiberApp *fiber.App) {
routeGroup.Get("/entries", controllers.GetEntries) // get entries (base/thin entries)
routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry
routeGroup.Get("/har", controllers.GetHAR)
routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry
routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB

View File

@ -1,6 +1,7 @@
package utils
import (
"encoding/json"
"fmt"
"github.com/gofiber/fiber/v2"
"log"
@ -43,7 +44,6 @@ func ReverseSlice(data interface{}) {
}
}
func CheckErr(e error) {
if e != nil {
log.Printf("%v", e)
@ -51,7 +51,6 @@ func CheckErr(e error) {
}
}
func SetHostname(address, newHostname string) string {
replacedUrl, err := url.Parse(address)
if err != nil{
@ -81,3 +80,8 @@ func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails {
RequestSenderIp: entry.RequestSenderIp,
}
}
func GetBytesFromStruct(v interface{}) []byte{
a, _ := json.Marshal(v)
return a
}

20
api/pkg/utils/zip.go Normal file
View File

@ -0,0 +1,20 @@
package utils
import (
"archive/zip"
"bytes"
)
func ZipData(files map[string][]byte) *bytes.Buffer {
// Create a buffer to write our archive to.
buf := new(bytes.Buffer)
// Create a new zip archive.
zipWriter := zip.NewWriter(buf)
defer func() { _ = zipWriter.Close() }()
for fileName, fileBytes := range files {
zipFile, _ := zipWriter.Create(fileName)
_, _ = zipFile.Write(fileBytes)
}
return buf
}

View File

@ -1,20 +1,28 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
)
type MizuFetchOptions struct {
Limit uint16
Directory string
}
var mizuFetchOptions = MizuFetchOptions{}
var fetchCmd = &cobra.Command{
Use: "fetch",
Short: "Download recorded traffic",
Short: "Download recorded traffic to files",
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Println("Not implemented")
RunMizuFetch(&mizuFetchOptions)
return nil
},
}
func init() {
rootCmd.AddCommand(fetchCmd)
fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.Limit, "limit", "l", 1000, "Provide a custom limit for entries to fetch")
fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries")
}

92
cli/cmd/fetchRunner.go Normal file
View File

@ -0,0 +1,92 @@
package cmd
import (
"archive/zip"
"bytes"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"strings"
)
func RunMizuFetch(fetch *MizuFetchOptions) {
resp, err := http.Get(fmt.Sprintf("http://localhost:8899/api/har?limit=%v", fetch.Limit))
if err != nil {
log.Fatal(err)
}
defer func() { _ = resp.Body.Close() }()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
if err != nil {
log.Fatal(err)
}
_ = Unzip(zipReader, fetch.Directory)
}
func Unzip(reader *zip.Reader, dest string) error {
dest, _ = filepath.Abs(dest)
_ = os.MkdirAll(dest, os.ModePerm)
// Closure to address file descriptors issue with all the deferred .Close() methods
extractAndWriteFile := func(f *zip.File) error {
rc, err := f.Open()
if err != nil {
return err
}
defer func() {
if err := rc.Close(); err != nil {
panic(err)
}
}()
path := filepath.Join(dest, f.Name)
// Check for ZipSlip (Directory traversal)
if !strings.HasPrefix(path, filepath.Clean(dest) + string(os.PathSeparator)) {
return fmt.Errorf("illegal file path: %s", path)
}
if f.FileInfo().IsDir() {
_ = os.MkdirAll(path, f.Mode())
} else {
_ = os.MkdirAll(filepath.Dir(path), f.Mode())
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
}
defer func() {
if err := f.Close(); err != nil {
panic(err)
}
}()
_, err = io.Copy(f, rc)
if err != nil {
return err
}
}
return nil
}
for _, f := range reader.File {
err := extractAndWriteFile(f)
if err != nil {
return err
}
}
return nil
}

View File

@ -8,7 +8,7 @@ var rootCmd = &cobra.Command{
Use: "mizu",
Short: "A web traffic viewer for kubernetes",
Long: `A web traffic viewer for kubernetes
Further info is available at https://github.com/up9inc/mizu`,
Further info is available at https://github.com/up9inc/mizu`,
}
// Execute adds all child commands to the root command and sets flags appropriately.

View File

@ -3,14 +3,23 @@ package cmd
import (
"errors"
"fmt"
"github.com/up9inc/mizu/cli/mizu"
"regexp"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/config"
"github.com/up9inc/mizu/cli/mizu"
)
type MizuTapOptions struct {
GuiPort uint16
Namespace string
KubeConfigPath string
MizuImage string
MizuPodPort uint16
}
var mizuTapOptions = &MizuTapOptions{}
var tapCmd = &cobra.Command{
Use: "tap [POD REGEX]",
Short: "Record ingoing traffic of a kubernetes pod",
@ -20,7 +29,7 @@ var tapCmd = &cobra.Command{
if len(args) == 0 {
return errors.New("POD REGEX argument is required")
} else if len(args) > 1 {
return errors.New("Unexpected number of arguments")
return errors.New("unexpected number of arguments")
}
regex, err := regexp.Compile(args[0])
@ -28,7 +37,7 @@ var tapCmd = &cobra.Command{
return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], err))
}
mizu.Run(regex)
RunMizuTap(regex, mizuTapOptions)
return nil
},
}
@ -36,9 +45,9 @@ var tapCmd = &cobra.Command{
func init() {
rootCmd.AddCommand(tapCmd)
tapCmd.Flags().Uint16VarP(&config.Configuration.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
tapCmd.Flags().StringVarP(&config.Configuration.Namespace, "namespace", "n", "", "Namespace selector")
tapCmd.Flags().StringVarP(&config.Configuration.KubeConfigPath, "kubeconfig", "k", "", "Path to kubeconfig file")
tapCmd.Flags().StringVarP(&config.Configuration.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector")
tapCmd.Flags().Uint16VarP(&config.Configuration.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod")
tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector")
tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector")
tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod")
}

View File

@ -1,11 +1,11 @@
package mizu
package cmd
import (
"context"
"fmt"
"github.com/up9inc/mizu/cli/config"
"github.com/up9inc/mizu/cli/kubernetes"
core "k8s.io/api/core/v1"
"github.com/up9inc/mizu/cli/mizu"
"os"
"os/signal"
"regexp"
@ -15,8 +15,9 @@ import (
var currentlyTappedPods []core.Pod
func Run(podRegexQuery *regexp.Regexp) {
kubernetesProvider := kubernetes.NewProvider(config.Configuration.KubeConfigPath, config.Configuration.Namespace)
func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath, tappingOptions.Namespace)
ctx, cancel := context.WithCancel(context.Background())
defer cancel() // cancel will be called when this function exits
@ -32,13 +33,14 @@ func Run(podRegexQuery *regexp.Regexp) {
cleanUpMizuResources(kubernetesProvider)
return
}
err = createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap)
err = createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions)
if err != nil {
cleanUpMizuResources(kubernetesProvider)
return
}
go portForwardApiPod(ctx, kubernetesProvider, cancel) //TODO convert this to job for built in pod ttl or have the running app handle this
go syncApiStatus(ctx, cancel, kubernetesProvider.Namespace)
go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions) //TODO convert this to job for built in pod ttl or have the running app handle this
go syncApiStatus(ctx, cancel, tappingOptions)
waitForFinish(ctx, cancel) //block until exit signal or error
// TODO handle incoming traffic from tapper using a channel
@ -48,19 +50,19 @@ func Run(podRegexQuery *regexp.Regexp) {
cleanUpMizuResources(kubernetesProvider)
}
func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {
func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error {
mizuServiceAccountExists := createRBACIfNecessary(ctx, kubernetesProvider)
_, err := kubernetesProvider.CreateMizuAggregatorPod(ctx, MizuResourcesNamespace, aggregatorPodName, config.Configuration.MizuImage, mizuServiceAccountExists)
_, err := kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists)
if err != nil {
fmt.Printf("Error creating mizu collector pod: %v\n", err)
return err
}
aggregatorService, err := kubernetesProvider.CreateService(ctx, MizuResourcesNamespace, aggregatorPodName, aggregatorPodName)
aggregatorService, err := kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, mizu.AggregatorPodName)
if err != nil {
fmt.Printf("Error creating mizu collector service: %v\n", err)
return err
}
err = kubernetesProvider.CreateMizuTapperDaemonSet(ctx, MizuResourcesNamespace, TapperDaemonSetName, config.Configuration.MizuImage, tapperPodName, fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace), nodeToTappedPodIPMap, mizuServiceAccountExists)
err = kubernetesProvider.CreateMizuTapperDaemonSet(ctx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName, tappingOptions.MizuImage, mizu.TapperPodName, fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace), nodeToTappedPodIPMap, mizuServiceAccountExists)
if err != nil {
fmt.Printf("Error creating mizu tapper daemonset: %v\n", err)
return err
@ -70,14 +72,14 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro
func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
removalCtx, _ := context.WithTimeout(context.Background(), 5 * time.Second)
if err := kubernetesProvider.RemovePod(removalCtx, MizuResourcesNamespace, aggregatorPodName); err != nil {
fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", aggregatorPodName, MizuResourcesNamespace, err, err, err)
if err := kubernetesProvider.RemovePod(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil {
fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err);
}
if err := kubernetesProvider.RemoveService(removalCtx, MizuResourcesNamespace, aggregatorPodName); err != nil {
fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", aggregatorPodName, MizuResourcesNamespace, err, err, err)
if err := kubernetesProvider.RemoveService(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil {
fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err);
}
if err := kubernetesProvider.RemoveDaemonSet(removalCtx, MizuResourcesNamespace, TapperDaemonSetName); err != nil {
fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", TapperDaemonSetName, MizuResourcesNamespace, err, err, err)
if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", mizu.TapperDaemonSetName, mizu.ResourcesNamespace, err, err, err);
}
}
@ -104,9 +106,9 @@ func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
// }
//}
func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", aggregatorPodName))
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, MizuResourcesNamespace), podExactRegex)
func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.AggregatorPodName))
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex)
isPodReady := false
var portForward *kubernetes.PortForward
for {
@ -114,15 +116,15 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
case <- added:
continue
case <- removed:
fmt.Printf("%s removed\n", aggregatorPodName)
fmt.Printf("%s removed\n", mizu.AggregatorPodName)
cancel()
return
case modifiedPod := <- modified:
if modifiedPod.Status.Phase == "Running" && !isPodReady {
isPodReady = true
var err error
portForward, err = kubernetes.NewPortForward(kubernetesProvider, MizuResourcesNamespace, aggregatorPodName, config.Configuration.GuiPort, config.Configuration.MizuPodPort, cancel)
fmt.Printf("Web interface is now available at http://localhost:%d\n", config.Configuration.GuiPort)
portForward, err = kubernetes.NewPortForward(kubernetesProvider, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.GuiPort, tappingOptions.MizuPodPort, cancel)
fmt.Printf("Web interface is now available at http://localhost:%d\n", tappingOptions.GuiPort)
if err != nil {
fmt.Printf("error forwarding port to pod %s\n", err)
cancel()
@ -131,7 +133,7 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
case <- time.After(25 * time.Second):
if !isPodReady {
fmt.Printf("error: %s pod was not ready in time", aggregatorPodName)
fmt.Printf("error: %s pod was not ready in time", mizu.AggregatorPodName)
cancel()
}
@ -148,17 +150,17 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
}
func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
mizuRBACExists, err := kubernetesProvider.DoesMizuRBACExist(ctx, MizuResourcesNamespace)
mizuRBACExists, err := kubernetesProvider.DoesMizuRBACExist(ctx, mizu.ResourcesNamespace)
if err != nil {
fmt.Printf("warning: could not ensure mizu rbac resources exist %v\n", err)
return false
}
if !mizuRBACExists {
var versionString = Version
if GitCommitHash != "" {
versionString += "-" + GitCommitHash
var versionString = mizu.Version
if mizu.GitCommitHash != "" {
versionString += "-" + mizu.GitCommitHash
}
err := kubernetesProvider.CreateMizuRBAC(ctx, MizuResourcesNamespace, versionString)
err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, versionString)
if err != nil {
fmt.Printf("warning: could not create mizu rbac resources %v\n", err)
return false
@ -193,8 +195,8 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
}
}
func syncApiStatus(ctx context.Context, cancel context.CancelFunc, namespace string) {
controlSocket, err := CreateControlSocket(fmt.Sprintf("ws://localhost:%d/ws", config.Configuration.GuiPort))
func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
controlSocket, err := mizu.CreateControlSocket(fmt.Sprintf("ws://localhost:%d/ws", tappingOptions.GuiPort))
if err != nil {
fmt.Printf("error establishing control socket connection %s\n", err)
cancel()

View File

@ -11,7 +11,7 @@ var versionCmd = &cobra.Command{
Use: "version",
Short: "Print version info",
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Printf("%s %s\n", mizu.Version, mizu.GitCommitHash)
fmt.Printf("%s (%s) %s\n", mizu.Version, mizu.Branch, mizu.GitCommitHash)
return nil
},
}

View File

@ -2,7 +2,6 @@ package cmd
import (
"fmt"
"github.com/spf13/cobra"
)

View File

@ -1,11 +0,0 @@
package config
type Options struct {
GuiPort uint16
Namespace string
KubeConfigPath string
MizuImage string
MizuPodPort uint16
}
var Configuration = &Options{}

View File

@ -7,8 +7,8 @@ var (
)
const (
MizuResourcesNamespace = "default"
ResourcesNamespace = "default"
TapperDaemonSetName = "mizu-tapper-daemon-set"
aggregatorPodName = "mizu-collector"
tapperPodName = "mizu-tapper"
AggregatorPodName = "mizu-collector"
TapperPodName = "mizu-tapper"
)