diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index e4f8888df..06f8665db 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" "net/http" "os/exec" "strings" @@ -12,7 +11,7 @@ import ( "time" ) -func TestTapAndFetch(t *testing.T) { +func TestTap(t *testing.T) { if testing.Short() { t.Skip("ignored acceptance test") } @@ -93,37 +92,6 @@ func TestTapAndFetch(t *testing.T) { t.Errorf("%v", err) return } - - fetchCmdArgs := getDefaultFetchCommandArgs() - fetchCmd := exec.Command(cliPath, fetchCmdArgs...) - t.Logf("running command: %v", fetchCmd.String()) - - if err := fetchCmd.Start(); err != nil { - t.Errorf("failed to start fetch command, err: %v", err) - return - } - - harCheckFunc := func() error { - harBytes, readFileErr := ioutil.ReadFile("./unknown_source.har") - if readFileErr != nil { - return fmt.Errorf("failed to read har file, err: %v", readFileErr) - } - - harEntries, err := getEntriesFromHarBytes(harBytes) - if err != nil { - return fmt.Errorf("failed to get entries from har, err: %v", err) - } - - if len(harEntries) == 0 { - return fmt.Errorf("unexpected har entries result - Expected more than 0 entries") - } - - return nil - } - if err := retriesExecute(shortRetriesCount, harCheckFunc); err != nil { - t.Errorf("%v", err) - return - } }) } } diff --git a/acceptanceTests/testsUtils.go b/acceptanceTests/testsUtils.go index 0d85a7353..d6a41db8d 100644 --- a/acceptanceTests/testsUtils.go +++ b/acceptanceTests/testsUtils.go @@ -76,13 +76,6 @@ func getDefaultTapNamespace() []string { return []string{"-n", "mizu-tests"} } -func getDefaultFetchCommandArgs() []string { - fetchCommand := "fetch" - defaultCmdArgs := getDefaultCommandArgs() - - return append([]string{fetchCommand}, defaultCmdArgs...) -} - func getDefaultConfigCommandArgs() []string { configCommand := "config" defaultCmdArgs := getDefaultCommandArgs() @@ -179,19 +172,6 @@ func cleanupCommand(cmd *exec.Cmd) error { return nil } -func getEntriesFromHarBytes(harBytes []byte) ([]interface{}, error) { - harInterface, convertErr := jsonBytesToInterface(harBytes) - if convertErr != nil { - return nil, convertErr - } - - har := harInterface.(map[string]interface{}) - harLog := har["log"].(map[string]interface{}) - harEntries := harLog["entries"].([]interface{}) - - return harEntries, nil -} - func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) { tapStatus := tapStatusInterface.(map[string]interface{}) podsInterface := tapStatus["pods"].([]interface{}) diff --git a/agent/README.md b/agent/README.md index f275a0988..87262d908 100644 --- a/agent/README.md +++ b/agent/README.md @@ -1,7 +1,6 @@ # mizu agent Agent for MIZU (API server and tapper) Basic APIs: -* /fetch - retrieve traffic data * /stats - retrieve statistics of collected data * /viewer - web ui diff --git a/agent/pkg/controllers/entries_controller.go b/agent/pkg/controllers/entries_controller.go index f4a225aa0..25416fff1 100644 --- a/agent/pkg/controllers/entries_controller.go +++ b/agent/pkg/controllers/entries_controller.go @@ -3,6 +3,7 @@ package controllers import ( "encoding/json" "fmt" + "github.com/google/martian/har" "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/providers" @@ -10,11 +11,9 @@ import ( "mizuserver/pkg/utils" "mizuserver/pkg/validation" "net/http" - "strings" "time" "github.com/gin-gonic/gin" - "github.com/google/martian/har" "github.com/romana/rlog" tapApi "github.com/up9inc/mizu/tap/api" @@ -64,93 +63,6 @@ func GetEntries(c *gin.Context) { c.JSON(http.StatusOK, baseEntries) } -func GetHARs(c *gin.Context) { - entriesFilter := &models.HarFetchRequestQuery{} - order := database.OrderDesc - if err := c.BindQuery(entriesFilter); err != nil { - c.JSON(http.StatusBadRequest, err) - } - err := validation.Validate(entriesFilter) - if err != nil { - c.JSON(http.StatusBadRequest, err) - } - - var timestampFrom, timestampTo int64 - - if entriesFilter.From < 0 { - timestampFrom = 0 - } else { - timestampFrom = entriesFilter.From - } - if entriesFilter.To <= 0 { - timestampTo = time.Now().UnixNano() / int64(time.Millisecond) - } else { - timestampTo = entriesFilter.To - } - - var entries []tapApi.MizuEntry - database.GetEntriesTable(). - Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)). - Order(fmt.Sprintf("timestamp %s", order)). - Find(&entries) - - if len(entries) > 0 { - // the entries always order from oldest to newest - we should reverse - utils.ReverseSlice(entries) - } - - harsObject := map[string]*models.ExtendedHAR{} - - for _, entryData := range entries { - var harEntry har.Entry - _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) - if entryData.ResolvedDestination != "" { - harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entryData.ResolvedDestination) - } - - var fileName string - sourceOfEntry := entryData.ResolvedSource - if sourceOfEntry != "" { - // naively assumes the proper service source is http - sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry) - //replace / from the file name because they end up creating a corrupted folder - fileName = fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_")) - } else { - fileName = "unknown_source.har" - } - if harOfSource, ok := harsObject[fileName]; ok { - harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry) - } else { - var entriesHar []*har.Entry - entriesHar = append(entriesHar, &harEntry) - harsObject[fileName] = &models.ExtendedHAR{ - Log: &models.ExtendedLog{ - Version: "1.2", - Creator: &models.ExtendedCreator{ - Creator: &har.Creator{ - Name: "mizu", - Version: "0.0.2", - }, - }, - Entries: entriesHar, - }, - } - // leave undefined when no source is present, otherwise modeler assumes source is empty string "" - if sourceOfEntry != "" { - harsObject[fileName].Log.Creator.Source = &sourceOfEntry - } - } - } - - retObj := map[string][]byte{} - for k, v := range harsObject { - bytesData, _ := json.Marshal(v) - retObj[k] = bytesData - } - buffer := utils.ZipData(retObj) - c.Data(http.StatusOK, "application/octet-stream", buffer.Bytes()) -} - func UploadEntries(c *gin.Context) { rlog.Infof("Upload entries - started\n") diff --git a/agent/pkg/routes/entries_routes.go b/agent/pkg/routes/entries_routes.go index af88b5bd4..12d901d0f 100644 --- a/agent/pkg/routes/entries_routes.go +++ b/agent/pkg/routes/entries_routes.go @@ -15,8 +15,6 @@ func EntriesRoutes(ginApp *gin.Engine) { routeGroup.GET("/uploadEntries", controllers.UploadEntries) routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation) - routeGroup.GET("/har", controllers.GetHARs) - routeGroup.GET("/resetDB", controllers.DeleteAllEntries) // get single (full) entry routeGroup.GET("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB diff --git a/cli/apiserver/provider.go b/cli/apiserver/provider.go index 5b8ca44cf..7555eb587 100644 --- a/cli/apiserver/provider.go +++ b/cli/apiserver/provider.go @@ -1,7 +1,6 @@ package apiserver import ( - "archive/zip" "bytes" "encoding/json" "fmt" @@ -131,28 +130,6 @@ func (provider *apiServerProvider) GetGeneralStats() (map[string]interface{}, er return generalStats, nil } -func (provider *apiServerProvider) GetHars(fromTimestamp int, toTimestamp int) (*zip.Reader, error) { - if !provider.isReady { - return nil, fmt.Errorf("trying to reach api server when not initialized yet") - } - resp, err := http.Get(fmt.Sprintf("%s/api/har?from=%v&to=%v", provider.url, fromTimestamp, toTimestamp)) - if err != nil { - return nil, fmt.Errorf("failed getting har from api server %w", err) - } - - defer func() { _ = resp.Body.Close() }() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed reading hars %w", err) - } - - zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) - if err != nil { - return nil, fmt.Errorf("failed craeting zip reader %w", err) - } - return zipReader, nil -} func (provider *apiServerProvider) GetVersion() (string, error) { if !provider.isReady { diff --git a/cli/cmd/fetch.go b/cli/cmd/fetch.go deleted file mode 100644 index 15d124726..000000000 --- a/cli/cmd/fetch.go +++ /dev/null @@ -1,46 +0,0 @@ -package cmd - -import ( - "github.com/creasty/defaults" - "github.com/spf13/cobra" - "github.com/up9inc/mizu/cli/apiserver" - "github.com/up9inc/mizu/cli/config" - "github.com/up9inc/mizu/cli/config/configStructs" - "github.com/up9inc/mizu/cli/logger" - "github.com/up9inc/mizu/cli/mizu/version" - "github.com/up9inc/mizu/cli/telemetry" - "github.com/up9inc/mizu/cli/uiUtils" -) - -var fetchCmd = &cobra.Command{ - Use: "fetch", - Short: "Download recorded traffic to files", - RunE: func(cmd *cobra.Command, args []string) error { - go telemetry.ReportRun("fetch", config.Config.Fetch) - - if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil { - logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, make sure one running") - return nil - } - - if isCompatible, err := version.CheckVersionCompatibility(); err != nil { - return err - } else if !isCompatible { - return nil - } - RunMizuFetch() - return nil - }, -} - -func init() { - rootCmd.AddCommand(fetchCmd) - - defaultFetchConfig := configStructs.FetchConfig{} - defaults.Set(&defaultFetchConfig) - - fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries") - fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries") - fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries") - fetchCmd.Flags().Uint16P(configStructs.GuiPortFetchName, "p", defaultFetchConfig.GuiPort, "Provide a custom port for the web interface webserver") -} diff --git a/cli/cmd/fetchRunner.go b/cli/cmd/fetchRunner.go deleted file mode 100644 index 0b539244a..000000000 --- a/cli/cmd/fetchRunner.go +++ /dev/null @@ -1,25 +0,0 @@ -package cmd - -import ( - "github.com/up9inc/mizu/cli/apiserver" - "github.com/up9inc/mizu/cli/config" - "github.com/up9inc/mizu/cli/logger" - "github.com/up9inc/mizu/cli/mizu/fsUtils" - "github.com/up9inc/mizu/cli/uiUtils" -) - -func RunMizuFetch() { - if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil { - logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs") - } - - zipReader, err := apiserver.Provider.GetHars(config.Config.Fetch.FromTimestamp, config.Config.Fetch.ToTimestamp) - if err != nil { - logger.Log.Errorf("Failed fetch data from API server %v", err) - return - } - - if err := fsUtils.Unzip(zipReader, config.Config.Fetch.Directory); err != nil { - logger.Log.Debugf("[ERROR] failed unzip %v", err) - } -} diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 00e59e90a..a02800853 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -64,7 +64,6 @@ func init() { tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies") tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values") tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size") - tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any") tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them") tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file with policy rules") } diff --git a/cli/config/configStruct.go b/cli/config/configStruct.go index bf3472adc..42987010b 100644 --- a/cli/config/configStruct.go +++ b/cli/config/configStruct.go @@ -18,7 +18,6 @@ const ( type ConfigStruct struct { Tap configStructs.TapConfig `yaml:"tap"` - Fetch configStructs.FetchConfig `yaml:"fetch"` Version configStructs.VersionConfig `yaml:"version"` View configStructs.ViewConfig `yaml:"view"` Logs configStructs.LogsConfig `yaml:"logs"` diff --git a/cli/config/configStructs/fetchConfig.go b/cli/config/configStructs/fetchConfig.go deleted file mode 100644 index e84fba8b6..000000000 --- a/cli/config/configStructs/fetchConfig.go +++ /dev/null @@ -1,15 +0,0 @@ -package configStructs - -const ( - DirectoryFetchName = "directory" - FromTimestampFetchName = "from" - ToTimestampFetchName = "to" - GuiPortFetchName = "gui-port" -) - -type FetchConfig struct { - Directory string `yaml:"directory" default:"."` - FromTimestamp int `yaml:"from" default:"0"` - ToTimestamp int `yaml:"to" default:"0"` - GuiPort uint16 `yaml:"gui-port" default:"8899"` -} diff --git a/cli/config/configStructs/tapConfig.go b/cli/config/configStructs/tapConfig.go index 8a999c37b..10ea29cc0 100644 --- a/cli/config/configStructs/tapConfig.go +++ b/cli/config/configStructs/tapConfig.go @@ -3,10 +3,8 @@ package configStructs import ( "errors" "fmt" - "regexp" - "strings" - "github.com/up9inc/mizu/shared/units" + "regexp" ) const ( @@ -17,7 +15,6 @@ const ( PlainTextFilterRegexesTapName = "regex-masking" DisableRedactionTapName = "no-redact" HumanMaxEntriesDBSizeTapName = "max-entries-db-size" - DirectionTapName = "direction" DryRunTapName = "dry-run" EnforcePolicyFile = "test-rules" ) @@ -34,7 +31,6 @@ type TapConfig struct { HealthChecksUserAgentHeaders []string `yaml:"ignored-user-agents"` DisableRedaction bool `yaml:"no-redact" default:"false"` HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"` - Direction string `yaml:"direction" default:"in"` DryRun bool `yaml:"dry-run" default:"false"` EnforcePolicyFile string `yaml:"test-rules"` ApiServerResources Resources `yaml:"api-server-resources"` @@ -69,10 +65,5 @@ func (config *TapConfig) Validate() error { return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize)) } - directionLowerCase := strings.ToLower(config.Direction) - if directionLowerCase != "any" && directionLowerCase != "in" { - return errors.New(fmt.Sprintf("%s is not a valid value for flag --%s. Acceptable values are in/any.", config.Direction, DirectionTapName)) - } - return nil }