From 74bd4b180f40d6f34011815462d05a8af5eb8acd Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Sun, 15 May 2022 15:23:03 +0300 Subject: [PATCH] Removing elastic dump experimental feature (#1086) * Removing elastic dump experimental feature --- agent/go.mod | 1 - agent/go.sum | 2 - agent/main.go | 2 - agent/pkg/api/main.go | 3 - agent/pkg/elastic/esClient.go | 116 ---------------------------------- cli/cmd/tapRunner.go | 1 - cli/config/configStruct.go | 1 - shared/models.go | 7 -- 8 files changed, 133 deletions(-) delete mode 100644 agent/pkg/elastic/esClient.go diff --git a/agent/go.mod b/agent/go.mod index 1f7dae7c8..cc0ae55f8 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -6,7 +6,6 @@ require ( github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b github.com/chanced/openapi v0.0.8 github.com/djherbis/atime v1.1.0 - github.com/elastic/go-elasticsearch/v7 v7.17.0 github.com/getkin/kin-openapi v0.89.0 github.com/gin-contrib/static v0.0.1 github.com/gin-gonic/gin v1.7.7 diff --git a/agent/go.sum b/agent/go.sum index 4b78e088d..2b3f360b3 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -166,8 +166,6 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/elastic/go-elasticsearch/v7 v7.17.0 h1:0fcSh4qeC/i1+7QU1KXpmq2iUAdMk4l0/vmbtW1+KJM= -github.com/elastic/go-elasticsearch/v7 v7.17.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= diff --git a/agent/main.go b/agent/main.go index 2dc82400c..3be77822c 100644 --- a/agent/main.go +++ b/agent/main.go @@ -17,7 +17,6 @@ import ( "github.com/gin-contrib/static" "github.com/gin-gonic/gin" "github.com/up9inc/mizu/agent/pkg/dependency" - "github.com/up9inc/mizu/agent/pkg/elastic" "github.com/up9inc/mizu/agent/pkg/entries" "github.com/up9inc/mizu/agent/pkg/middlewares" "github.com/up9inc/mizu/agent/pkg/models" @@ -205,7 +204,6 @@ func enableExpFeatureIfNeeded() { serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap) serviceMapGenerator.Enable() } - elastic.GetInstance().Configure(config.Config.Elastic) } func getSyncEntriesConfig() *shared.SyncEntriesConfig { diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index e8ff72861..afbaff051 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -14,7 +14,6 @@ import ( "github.com/up9inc/mizu/agent/pkg/models" "github.com/up9inc/mizu/agent/pkg/dependency" - "github.com/up9inc/mizu/agent/pkg/elastic" "github.com/up9inc/mizu/agent/pkg/har" "github.com/up9inc/mizu/agent/pkg/holder" "github.com/up9inc/mizu/agent/pkg/providers" @@ -153,8 +152,6 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink) serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol) - - elastic.GetInstance().PushEntry(mizuEntry) } } diff --git a/agent/pkg/elastic/esClient.go b/agent/pkg/elastic/esClient.go deleted file mode 100644 index 33a1775d6..000000000 --- a/agent/pkg/elastic/esClient.go +++ /dev/null @@ -1,116 +0,0 @@ -package elastic - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "net/http" - "sync" - "time" - - "github.com/elastic/go-elasticsearch/v7" - "github.com/up9inc/mizu/logger" - "github.com/up9inc/mizu/shared" - "github.com/up9inc/mizu/tap/api" -) - -type client struct { - es *elasticsearch.Client - index string - insertedCount int -} - -var instance *client -var once sync.Once - -func GetInstance() *client { - once.Do(func() { - instance = newClient() - }) - return instance -} - -func (client *client) Configure(config shared.ElasticConfig) { - if config.Url == "" || config.User == "" || config.Password == "" { - if client.es != nil { - client.es = nil - } - logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled") - return - } - transport := http.DefaultTransport - tlsClientConfig := &tls.Config{InsecureSkipVerify: true} - transport.(*http.Transport).TLSClientConfig = tlsClientConfig - cfg := elasticsearch.Config{ - Addresses: []string{config.Url}, - Username: config.User, - Password: config.Password, - Transport: transport, - } - - es, err := elasticsearch.NewClient(cfg) - if err != nil { - logger.Log.Errorf("Failed to initialize elastic client %v", err) - } - - // Have the client instance return a response - res, err := es.Info() - if err != nil { - logger.Log.Errorf("Elastic client.Info() ERROR: %v", err) - } else { - client.es = es - client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04") - client.insertedCount = 0 - logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res) - } - defer res.Body.Close() -} - -func newClient() *client { - return &client{ - es: nil, - index: "", - } -} - -type httpEntry struct { - Source *api.TCP `json:"src"` - Destination *api.TCP `json:"dst"` - Outgoing bool `json:"outgoing"` - CreatedAt time.Time `json:"createdAt"` - Request map[string]interface{} `json:"request"` - Response map[string]interface{} `json:"response"` - ElapsedTime int64 `json:"elapsedTime"` -} - -func (client *client) PushEntry(entry *api.Entry) { - if client.es == nil { - return - } - - if entry.Protocol.Name != "http" { - return - } - - entryToPush := httpEntry{ - Source: entry.Source, - Destination: entry.Destination, - Outgoing: entry.Outgoing, - CreatedAt: entry.StartTime, - Request: entry.Request, - Response: entry.Response, - ElapsedTime: entry.ElapsedTime, - } - - entryJson, err := json.Marshal(entryToPush) - if err != nil { - logger.Log.Errorf("json.Marshal ERROR: %v", err) - return - } - var buffer bytes.Buffer - buffer.WriteString(string(entryJson)) - res, _ := client.es.Index(client.index, &buffer) - if res.StatusCode == 201 { - client.insertedCount += 1 - } -} diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 5f8ecbaee..9bf833ca6 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -164,7 +164,6 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig { ServiceMap: config.Config.ServiceMap, OAS: config.Config.OAS, Telemetry: config.Config.Telemetry, - Elastic: config.Config.Elastic, } return &mizuAgentConfig diff --git a/cli/config/configStruct.go b/cli/config/configStruct.go index 99dbdb762..e809d3aec 100644 --- a/cli/config/configStruct.go +++ b/cli/config/configStruct.go @@ -41,7 +41,6 @@ type ConfigStruct struct { LogLevelStr string `yaml:"log-level,omitempty" default:"INFO" readonly:""` ServiceMap bool `yaml:"service-map" default:"true"` OAS bool `yaml:"oas" default:"true"` - Elastic shared.ElasticConfig `yaml:"elastic"` } func (config *ConfigStruct) validate() error { diff --git a/shared/models.go b/shared/models.go index 9fefe9453..3e7d47272 100644 --- a/shared/models.go +++ b/shared/models.go @@ -45,13 +45,6 @@ type MizuAgentConfig struct { ServiceMap bool `json:"serviceMap"` OAS bool `json:"oas"` Telemetry bool `json:"telemetry"` - Elastic ElasticConfig `json:"elastic"` -} - -type ElasticConfig struct { - User string `yaml:"user,omitempty" default:"" readonly:""` - Password string `yaml:"password,omitempty" default:"" readonly:""` - Url string `yaml:"url,omitempty" default:"" readonly:""` } type WebSocketMessageMetadata struct {