From fc5d6b2d0ab3f5241ef4c29faffda5bbc6ee3b17 Mon Sep 17 00:00:00 2001 From: lirazyehezkel <61656597+lirazyehezkel@users.noreply.github.com> Date: Thu, 27 May 2021 13:48:37 +0300 Subject: [PATCH 01/36] Show pod name and namespace (#61) --- ui/src/components/StatusBar.tsx | 25 ++++++++++++++++++---- ui/src/components/style/StatusBar.sass | 29 +++++++++++++++++++------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/ui/src/components/StatusBar.tsx b/ui/src/components/StatusBar.tsx index ee09369c5..0fb25b1e4 100644 --- a/ui/src/components/StatusBar.tsx +++ b/ui/src/components/StatusBar.tsx @@ -1,5 +1,5 @@ import './style/StatusBar.sass'; -import React from "react"; +import React, {useState} from "react"; export interface TappingStatusPod { name: string; @@ -15,14 +15,31 @@ export interface Props { } const pluralize = (noun: string, amount: number) => { - return `${noun}${amount != 1 ? 's' : ''}` + return `${noun}${amount !== 1 ? 's' : ''}` } export const StatusBar: React.FC = ({tappingStatus}) => { + + const [expandedBar, setExpandedBar] = useState(false); + const uniqueNamespaces = Array.from(new Set(tappingStatus.pods.map(pod => pod.namespace))); const amountOfPods = tappingStatus.pods.length; - return
- {`Tapping ${amountOfPods} ${pluralize('pod', amountOfPods)} in ${pluralize('namespace', uniqueNamespaces.length)} ${uniqueNamespaces.join(", ")}`} + return
setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)}> +
{`Tapping ${amountOfPods} ${pluralize('pod', amountOfPods)} in ${pluralize('namespace', uniqueNamespaces.length)} ${uniqueNamespaces.join(", ")}`}
+ {expandedBar &&
+ + + + + + + {tappingStatus.pods.map(pod => + + + )} + +
Pod nameNamespace
{pod.name}{pod.namespace}
+
}
; } diff --git a/ui/src/components/style/StatusBar.sass b/ui/src/components/style/StatusBar.sass index 470166b25..671975e8f 100644 --- a/ui/src/components/style/StatusBar.sass +++ b/ui/src/components/style/StatusBar.sass @@ -1,20 +1,35 @@ @import 'variables.module.scss' -.StatusBar +.statusBar position: absolute transform: translate(-50%, -3px) left: 50% z-index: 9999 min-width: 200px - height: 32px background: $blue-color - color: $light-blue-color + color: rgba(255,255,255,0.75) border-bottom-left-radius: 8px border-bottom-right-radius: 8px top: 0 - display: flex - align-items: center padding: 2px 10px - user-select: none font-size: 14px - opacity: 0.8 + transition: max-height 2s ease-out + width: auto + max-height: 32px + overflow: hidden + + .podsCount + display: flex + justify-content: center + padding: 8px + font-weight: 600 + + th + text-align: left + td + padding-right: 15px + padding-top: 5px + +.expandedStatusBar + max-height: 100vh + padding-bottom: 15px \ No newline at end of file From 07c19b5d6de1bdbd338f7b25094a5f4b4829350e Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Thu, 27 May 2021 18:17:26 +0300 Subject: [PATCH 02/36] WIP --- api/main.go | 12 +++++- api/pkg/utils/consts.go | 9 +++++ api/pkg/utils/utils.go | 87 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 api/pkg/utils/consts.go diff --git a/api/main.go b/api/main.go index 220b53a69..5ed4f395f 100644 --- a/api/main.go +++ b/api/main.go @@ -52,7 +52,9 @@ func main() { if err != nil { panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err)) } - go pipeChannelToSocket(socketConnection, harOutputChannel) + filteredHarChannel := make(chan *tap.OutputChannelItem) + go filterHarHeaders(harOutputChannel, filteredHarChannel) + go pipeChannelToSocket(socketConnection, filteredHarChannel) } else if *aggregator { socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) go api.StartReadingEntries(socketHarOutChannel, nil) @@ -98,6 +100,14 @@ func getTapTargets() []string { return tappedAddressesPerNodeDict[nodeName] } +func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem) { + for { + message := <- inChannel + utils.FilterSensitiveInfoFromHarRequest(message) + outChannel <- message + } +} + func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) { if connection == nil { panic("Websocket connection is nil") diff --git a/api/pkg/utils/consts.go b/api/pkg/utils/consts.go new file mode 100644 index 000000000..377920a51 --- /dev/null +++ b/api/pkg/utils/consts.go @@ -0,0 +1,9 @@ +package utils + +const maskedFieldPlaceholderValue = "[REDACTED]" +var personallyIdentifiableDataFields = []string {"token", "authorization", "authentication", "cookie", "userid", "password", + "username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt", + "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", + "zip", "zipcode", "address", "country", "city", "state", "residence", + "name", "firstname", "lastname", "suffix", "middlename", "fname", "lname", + "mname", "date", "birthday", "birthday", "bday", "sender", "receiver"} diff --git a/api/pkg/utils/utils.go b/api/pkg/utils/utils.go index 9ecb89522..8b2638fb1 100644 --- a/api/pkg/utils/utils.go +++ b/api/pkg/utils/utils.go @@ -4,12 +4,15 @@ import ( "encoding/json" "fmt" "github.com/gofiber/fiber/v2" + "github.com/google/martian/har" "log" "mizuserver/pkg/models" + "mizuserver/pkg/tap" "net/url" "os" "os/signal" "reflect" + "strings" "syscall" ) @@ -84,4 +87,86 @@ func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails { func GetBytesFromStruct(v interface{}) []byte{ a, _ := json.Marshal(v) return a -} \ No newline at end of file +} + +func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) { + filterHarHeaders(harOutputItem.HarEntry.Request.Headers) + filterHarHeaders(harOutputItem.HarEntry.Response.Headers) + + harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL) + + var requestJsonMap map[string] interface{} + err := json.Unmarshal([]byte(harOutputItem.HarEntry.Request.PostData.Text) ,&requestJsonMap) + if err == nil { + filterJsonMap(requestJsonMap) + } + // + //filterJsonMap(harOutputItem.HarEntry.Response.Content.Text) + + + // filter url query params + // filter bodies +} + +func filterHarHeaders(headers []har.Header) { + for _, header := range headers { + if isFieldNameSensitive(header.Name) { + header.Value = maskedFieldPlaceholderValue + } + } +} + +func isFieldNameSensitive(fieldName string) bool { + name := strings.ToLower(fieldName) + name = strings.ReplaceAll(name, "_", "") + name = strings.ReplaceAll(name, "-", "") + name = strings.ReplaceAll(name, " ", "") + + for _, sensitiveField := range personallyIdentifiableDataFields { + if strings.Contains(name, sensitiveField) { + return true + } + } + + return false +} + +func filterJsonMap(jsonMap map[string] interface{}) { + for key, value := range jsonMap { + if value == nil { + return + } + nestedMap, isNested := value.(map[string] interface{}) + if isNested { + filterJsonMap(nestedMap) + } else { + if isFieldNameSensitive(key) { + jsonMap[key] = maskedFieldPlaceholderValue + } + } + } +} + +func filterUrl(originalUrl string) string { + parsedUrl, err := url.Parse(originalUrl) + if err != nil { + return originalUrl + } else { + if len(parsedUrl.RawQuery) > 0 { + newQueryArgs := make([]string, 0) + for urlQueryParamName, urlQueryParamValues := range parsedUrl.Query() { + newValues := urlQueryParamValues + if isFieldNameSensitive(urlQueryParamName) { + newValues = []string {maskedFieldPlaceholderValue} + } + for value := range newValues { + newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, value)) + } + } + + parsedUrl.RawQuery = strings.Join(newQueryArgs, "&") + } + + return parsedUrl.String() + } +} From 19fba89ca5d8d830c3c3ab0bc31fb52818b96c02 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Sun, 30 May 2021 15:21:39 +0300 Subject: [PATCH 03/36] Update main.go, consts.go, and 2 more files... --- api/main.go | 4 +- api/pkg/utils/consts.go | 2 +- api/pkg/utils/messageSensitiveDataCleaner.go | 109 +++++++++++++++++++ api/pkg/utils/utils.go | 85 --------------- 4 files changed, 113 insertions(+), 87 deletions(-) create mode 100644 api/pkg/utils/messageSensitiveDataCleaner.go diff --git a/api/main.go b/api/main.go index 5ed4f395f..e43872afc 100644 --- a/api/main.go +++ b/api/main.go @@ -34,7 +34,9 @@ func main() { if *standalone { harOutputChannel := tap.StartPassiveTapper() - go api.StartReadingEntries(harOutputChannel, tap.HarOutputDir) + filteredHarChannel := make(chan *tap.OutputChannelItem) + go filterHarHeaders(harOutputChannel, filteredHarChannel) + go api.StartReadingEntries(filteredHarChannel, nil) hostApi(nil) } else if *shouldTap { if *aggregatorAddress == "" { diff --git a/api/pkg/utils/consts.go b/api/pkg/utils/consts.go index 377920a51..0b1a99f3d 100644 --- a/api/pkg/utils/consts.go +++ b/api/pkg/utils/consts.go @@ -6,4 +6,4 @@ var personallyIdentifiableDataFields = []string {"token", "authorization", "auth "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", "zip", "zipcode", "address", "country", "city", "state", "residence", "name", "firstname", "lastname", "suffix", "middlename", "fname", "lname", - "mname", "date", "birthday", "birthday", "bday", "sender", "receiver"} + "mname", "birthday", "birthday", "birthdate", "bday", "sender", "receiver"} diff --git a/api/pkg/utils/messageSensitiveDataCleaner.go b/api/pkg/utils/messageSensitiveDataCleaner.go new file mode 100644 index 000000000..595751ff7 --- /dev/null +++ b/api/pkg/utils/messageSensitiveDataCleaner.go @@ -0,0 +1,109 @@ +package utils + +import ( + "encoding/json" + "fmt" + "github.com/google/martian/har" + "mizuserver/pkg/tap" + "net/url" + "strings" + +) + +func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) { + filterHarHeaders(harOutputItem.HarEntry.Request.Headers) + filterHarHeaders(harOutputItem.HarEntry.Response.Headers) + + harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL) + for i, queryString := range harOutputItem.HarEntry.Request.QueryString { + if isFieldNameSensitive(queryString.Name) { + harOutputItem.HarEntry.Request.QueryString[i].Value = maskedFieldPlaceholderValue + } + } + + if harOutputItem.HarEntry.Request.PostData != nil { + filteredRequestBody, err := filterHttpBody([]byte(harOutputItem.HarEntry.Request.PostData.Text)) + if err == nil { + harOutputItem.HarEntry.Request.PostData.Text = string(filteredRequestBody) + } + } + if harOutputItem.HarEntry.Response.Content != nil { + filteredResponseBody, err := filterHttpBody(harOutputItem.HarEntry.Response.Content.Text) + if err == nil { + harOutputItem.HarEntry.Response.Content.Text = filteredResponseBody + } + } +} + +func filterHarHeaders(headers []har.Header) { + for i, header := range headers { + if isFieldNameSensitive(header.Name) { + headers[i].Value = maskedFieldPlaceholderValue + } + } +} + +func isFieldNameSensitive(fieldName string) bool { + name := strings.ToLower(fieldName) + name = strings.ReplaceAll(name, "_", "") + name = strings.ReplaceAll(name, "-", "") + name = strings.ReplaceAll(name, " ", "") + + for _, sensitiveField := range personallyIdentifiableDataFields { + if strings.Contains(name, sensitiveField) { + return true + } + } + + return false +} + +func filterHttpBody(bytes []byte) ([]byte, error){ + var bodyJsonMap map[string] interface{} + err := json.Unmarshal(bytes ,&bodyJsonMap) + if err != nil { + return nil, err + } + filterJsonMap(bodyJsonMap) + return json.Marshal(bodyJsonMap) +} + +func filterJsonMap(jsonMap map[string] interface{}) { + for key, value := range jsonMap { + if value == nil { + return + } + nestedMap, isNested := value.(map[string] interface{}) + if isNested { + filterJsonMap(nestedMap) + } else { + if isFieldNameSensitive(key) { + jsonMap[key] = maskedFieldPlaceholderValue + } + } + } +} + +func filterUrl(originalUrl string) string { + parsedUrl, err := url.Parse(originalUrl) + if err != nil { + return originalUrl + } else { + if len(parsedUrl.RawQuery) > 0 { + newQueryArgs := make([]string, 0) + for urlQueryParamName, urlQueryParamValues := range parsedUrl.Query() { + newValues := urlQueryParamValues + if isFieldNameSensitive(urlQueryParamName) { + newValues = []string {maskedFieldPlaceholderValue} + } + for _, paramValue := range newValues { + newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue)) + } + } + + parsedUrl.RawQuery = strings.Join(newQueryArgs, "&") + } + + return parsedUrl.String() + } +} diff --git a/api/pkg/utils/utils.go b/api/pkg/utils/utils.go index 8b2638fb1..ebf4ce9f1 100644 --- a/api/pkg/utils/utils.go +++ b/api/pkg/utils/utils.go @@ -4,15 +4,12 @@ import ( "encoding/json" "fmt" "github.com/gofiber/fiber/v2" - "github.com/google/martian/har" "log" "mizuserver/pkg/models" - "mizuserver/pkg/tap" "net/url" "os" "os/signal" "reflect" - "strings" "syscall" ) @@ -88,85 +85,3 @@ func GetBytesFromStruct(v interface{}) []byte{ a, _ := json.Marshal(v) return a } - -func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) { - filterHarHeaders(harOutputItem.HarEntry.Request.Headers) - filterHarHeaders(harOutputItem.HarEntry.Response.Headers) - - harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL) - - var requestJsonMap map[string] interface{} - err := json.Unmarshal([]byte(harOutputItem.HarEntry.Request.PostData.Text) ,&requestJsonMap) - if err == nil { - filterJsonMap(requestJsonMap) - } - // - //filterJsonMap(harOutputItem.HarEntry.Response.Content.Text) - - - // filter url query params - // filter bodies -} - -func filterHarHeaders(headers []har.Header) { - for _, header := range headers { - if isFieldNameSensitive(header.Name) { - header.Value = maskedFieldPlaceholderValue - } - } -} - -func isFieldNameSensitive(fieldName string) bool { - name := strings.ToLower(fieldName) - name = strings.ReplaceAll(name, "_", "") - name = strings.ReplaceAll(name, "-", "") - name = strings.ReplaceAll(name, " ", "") - - for _, sensitiveField := range personallyIdentifiableDataFields { - if strings.Contains(name, sensitiveField) { - return true - } - } - - return false -} - -func filterJsonMap(jsonMap map[string] interface{}) { - for key, value := range jsonMap { - if value == nil { - return - } - nestedMap, isNested := value.(map[string] interface{}) - if isNested { - filterJsonMap(nestedMap) - } else { - if isFieldNameSensitive(key) { - jsonMap[key] = maskedFieldPlaceholderValue - } - } - } -} - -func filterUrl(originalUrl string) string { - parsedUrl, err := url.Parse(originalUrl) - if err != nil { - return originalUrl - } else { - if len(parsedUrl.RawQuery) > 0 { - newQueryArgs := make([]string, 0) - for urlQueryParamName, urlQueryParamValues := range parsedUrl.Query() { - newValues := urlQueryParamValues - if isFieldNameSensitive(urlQueryParamName) { - newValues = []string {maskedFieldPlaceholderValue} - } - for value := range newValues { - newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, value)) - } - } - - parsedUrl.RawQuery = strings.Join(newQueryArgs, "&") - } - - return parsedUrl.String() - } -} From f958de6619b00783e05550601d1caf4dfa739d91 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Sun, 30 May 2021 15:49:02 +0300 Subject: [PATCH 04/36] Update messageSensitiveDataCleaner.go --- api/pkg/utils/messageSensitiveDataCleaner.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/pkg/utils/messageSensitiveDataCleaner.go b/api/pkg/utils/messageSensitiveDataCleaner.go index 595751ff7..d87e21e8d 100644 --- a/api/pkg/utils/messageSensitiveDataCleaner.go +++ b/api/pkg/utils/messageSensitiveDataCleaner.go @@ -14,6 +14,9 @@ func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) { filterHarHeaders(harOutputItem.HarEntry.Request.Headers) filterHarHeaders(harOutputItem.HarEntry.Response.Headers) + harOutputItem.HarEntry.Request.Cookies = nil + harOutputItem.HarEntry.Response.Cookies = nil + harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL) for i, queryString := range harOutputItem.HarEntry.Request.QueryString { if isFieldNameSensitive(queryString.Name) { From 3497dc057b43749ad1f21b96135192b2d2ad699c Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Sun, 30 May 2021 15:54:07 +0300 Subject: [PATCH 05/36] Update consts.go and messageSensitiveDataCleaner.go --- api/pkg/utils/consts.go | 5 ++--- api/pkg/utils/messageSensitiveDataCleaner.go | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/api/pkg/utils/consts.go b/api/pkg/utils/consts.go index 0b1a99f3d..fcaf5d5b5 100644 --- a/api/pkg/utils/consts.go +++ b/api/pkg/utils/consts.go @@ -4,6 +4,5 @@ const maskedFieldPlaceholderValue = "[REDACTED]" var personallyIdentifiableDataFields = []string {"token", "authorization", "authentication", "cookie", "userid", "password", "username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt", "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", - "zip", "zipcode", "address", "country", "city", "state", "residence", - "name", "firstname", "lastname", "suffix", "middlename", "fname", "lname", - "mname", "birthday", "birthday", "birthdate", "bday", "sender", "receiver"} + "zip", "zipcode", "address", "country", "firstname", "lastname", + "middlename", "fname", "lname", "birthdate"} diff --git a/api/pkg/utils/messageSensitiveDataCleaner.go b/api/pkg/utils/messageSensitiveDataCleaner.go index d87e21e8d..b1be71eb3 100644 --- a/api/pkg/utils/messageSensitiveDataCleaner.go +++ b/api/pkg/utils/messageSensitiveDataCleaner.go @@ -7,7 +7,6 @@ import ( "mizuserver/pkg/tap" "net/url" "strings" - ) func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) { @@ -87,6 +86,7 @@ func filterJsonMap(jsonMap map[string] interface{}) { } } +// replaces sensitive query params in url string (http://service/api?userId=bob&password=123&type=login -> http://service/api?userId=[REDACTED]&password=[REDACTED]&type=login) func filterUrl(originalUrl string) string { parsedUrl, err := url.Parse(originalUrl) if err != nil { From 5473f1121584a3c99f998a4a917dbdbd1ec7d62b Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Sun, 30 May 2021 15:54:45 +0300 Subject: [PATCH 06/36] Update messageSensitiveDataCleaner.go --- api/pkg/utils/messageSensitiveDataCleaner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/pkg/utils/messageSensitiveDataCleaner.go b/api/pkg/utils/messageSensitiveDataCleaner.go index b1be71eb3..2fc88ef00 100644 --- a/api/pkg/utils/messageSensitiveDataCleaner.go +++ b/api/pkg/utils/messageSensitiveDataCleaner.go @@ -86,7 +86,7 @@ func filterJsonMap(jsonMap map[string] interface{}) { } } -// replaces sensitive query params in url string (http://service/api?userId=bob&password=123&type=login -> http://service/api?userId=[REDACTED]&password=[REDACTED]&type=login) +// receives string representing url, returns string url without sensitive query param values (http://service/api?userId=bob&password=123&type=login -> http://service/api?userId=[REDACTED]&password=[REDACTED]&type=login) func filterUrl(originalUrl string) string { parsedUrl, err := url.Parse(originalUrl) if err != nil { From 27c7d66478a1bb3d1b58512380cb129b50222adc Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Mon, 31 May 2021 09:58:12 +0300 Subject: [PATCH 07/36] Update main.go, consts.go, and 3 more files... --- api/main.go | 6 +++--- api/pkg/sensitiveDataFiltering/consts.go | 10 ++++++++++ .../messageSensitiveDataCleaner.go | 4 ++-- api/pkg/utils/consts.go | 8 -------- 4 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 api/pkg/sensitiveDataFiltering/consts.go rename api/pkg/{utils => sensitiveDataFiltering}/messageSensitiveDataCleaner.go (97%) delete mode 100644 api/pkg/utils/consts.go diff --git a/api/main.go b/api/main.go index e43872afc..187af6d80 100644 --- a/api/main.go +++ b/api/main.go @@ -11,6 +11,7 @@ import ( "mizuserver/pkg/middleware" "mizuserver/pkg/models" "mizuserver/pkg/routes" + "mizuserver/pkg/sensitiveDataFiltering" "mizuserver/pkg/tap" "mizuserver/pkg/utils" "os" @@ -103,9 +104,8 @@ func getTapTargets() []string { } func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem) { - for { - message := <- inChannel - utils.FilterSensitiveInfoFromHarRequest(message) + for message := range inChannel { + sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message) outChannel <- message } } diff --git a/api/pkg/sensitiveDataFiltering/consts.go b/api/pkg/sensitiveDataFiltering/consts.go new file mode 100644 index 000000000..09ce6628c --- /dev/null +++ b/api/pkg/sensitiveDataFiltering/consts.go @@ -0,0 +1,10 @@ +package sensitiveDataFiltering + +const maskedFieldPlaceholderValue = "[REDACTED]" + +//these values MUST be all lower case +var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password", + "username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt", + "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", + "zip", "zipcode", "address", "country", "firstname", "lastname", + "middlename", "fname", "lname", "birthdate"} diff --git a/api/pkg/utils/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go similarity index 97% rename from api/pkg/utils/messageSensitiveDataCleaner.go rename to api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index 2fc88ef00..468f01d6a 100644 --- a/api/pkg/utils/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -1,4 +1,4 @@ -package utils +package sensitiveDataFiltering import ( "encoding/json" @@ -90,7 +90,7 @@ func filterJsonMap(jsonMap map[string] interface{}) { func filterUrl(originalUrl string) string { parsedUrl, err := url.Parse(originalUrl) if err != nil { - return originalUrl + return fmt.Sprintf("http://%s", maskedFieldPlaceholderValue) } else { if len(parsedUrl.RawQuery) > 0 { newQueryArgs := make([]string, 0) diff --git a/api/pkg/utils/consts.go b/api/pkg/utils/consts.go deleted file mode 100644 index fcaf5d5b5..000000000 --- a/api/pkg/utils/consts.go +++ /dev/null @@ -1,8 +0,0 @@ -package utils - -const maskedFieldPlaceholderValue = "[REDACTED]" -var personallyIdentifiableDataFields = []string {"token", "authorization", "authentication", "cookie", "userid", "password", - "username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt", - "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", - "zip", "zipcode", "address", "country", "firstname", "lastname", - "middlename", "fname", "lname", "birthdate"} From 47237f05a58236f220b26346f12243a803095bac Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Mon, 31 May 2021 17:53:21 +0300 Subject: [PATCH 08/36] WIP --- api/go.mod | 1 + api/go.sum | 2 + api/main.go | 28 ++++++- api/pkg/sensitiveDataFiltering/consts.go | 4 +- .../messageSensitiveDataCleaner.go | 84 +++++++++++++++++-- cli/cmd/tap.go | 12 +-- 6 files changed, 117 insertions(+), 14 deletions(-) diff --git a/api/go.mod b/api/go.mod index 4f4c055c0..dcb1edf86 100644 --- a/api/go.mod +++ b/api/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a + github.com/beevik/etree v1.1.0 github.com/djherbis/atime v1.0.0 github.com/fasthttp/websocket v1.4.3-beta.1 // indirect github.com/go-playground/locales v0.13.0 diff --git a/api/go.sum b/api/go.sum index bc655181d..efa2d4285 100644 --- a/api/go.sum +++ b/api/go.sum @@ -48,6 +48,8 @@ github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a h1:76llBl github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a/go.mod h1:QvDfsDQDmGxUsvEeWabVZ5pp2FMXpOkwQV0L6SE6cp0= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= +github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= diff --git a/api/main.go b/api/main.go index 187af6d80..7134733d6 100644 --- a/api/main.go +++ b/api/main.go @@ -16,6 +16,7 @@ import ( "mizuserver/pkg/utils" "os" "os/signal" + "regexp" ) var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API") @@ -25,6 +26,7 @@ var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu c const nodeNameEnvVar = "NODE_NAME" const tappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST" +const plainTextRegexesEnvVar = "PLAINTEXT_REGEXES" func main() { flag.Parse() @@ -98,14 +100,36 @@ func getTapTargets() []string { var tappedAddressesPerNodeDict map[string][]string err := json.Unmarshal([]byte(os.Getenv(tappedAddressesPerNodeDictEnvVar)), &tappedAddressesPerNodeDict) if err != nil { - panic(fmt.Sprintf("env var value of %s is invalid! must be map[string][]string %v", tappedAddressesPerNodeDict, err)) + panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", tappedAddressesPerNodeDictEnvVar, tappedAddressesPerNodeDict, err)) } return tappedAddressesPerNodeDict[nodeName] } +func getFilteringOptions() *sensitiveDataFiltering.FilteringOptions { + regexJsonArr := os.Getenv(plainTextRegexesEnvVar) + if regexJsonArr == "" { + return nil + } + var regexStrSlice []string + err := json.Unmarshal([]byte(regexJsonArr), ®exStrSlice) + if err != nil { + panic(fmt.Sprintf("env var %s's value of %s is invalid! must be []string %v", plainTextRegexesEnvVar, regexJsonArr, err)) + } + + parsedRegexSlice := make([]regexp.Regexp, 0) + for _, regexStr := range regexStrSlice { + regex, err := regexp.Compile(regexStr) + if err != nil { + panic(fmt.Sprintf("env var %s's value of %s is invalid! must be []string %v", plainTextRegexesEnvVar, regexJsonArr, err)) + } + parsedRegexSlice = append(parsedRegexSlice, *regex) + } + return &sensitiveDataFiltering.FilteringOptions{PlainTextFilterRegexes: parsedRegexSlice} +} + func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem) { for message := range inChannel { - sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message) + sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, nil) outChannel <- message } } diff --git a/api/pkg/sensitiveDataFiltering/consts.go b/api/pkg/sensitiveDataFiltering/consts.go index 09ce6628c..8cd0cacf0 100644 --- a/api/pkg/sensitiveDataFiltering/consts.go +++ b/api/pkg/sensitiveDataFiltering/consts.go @@ -2,9 +2,9 @@ package sensitiveDataFiltering const maskedFieldPlaceholderValue = "[REDACTED]" -//these values MUST be all lower case +//these values MUST be all lower case and contain no `-` or `_` characters var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password", "username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt", "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", "zip", "zipcode", "address", "country", "firstname", "lastname", - "middlename", "fname", "lname", "birthdate"} + "middlename", "fname", "lname", "birthdate", "title"} diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index 468f01d6a..fa247f196 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -3,13 +3,20 @@ package sensitiveDataFiltering import ( "encoding/json" "fmt" - "github.com/google/martian/har" "mizuserver/pkg/tap" "net/url" + "regexp" "strings" + + "github.com/beevik/etree" + "github.com/google/martian/har" ) -func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) { +type FilteringOptions struct { + PlainTextFilterRegexes []regexp.Regexp +} + +func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *FilteringOptions) { filterHarHeaders(harOutputItem.HarEntry.Request.Headers) filterHarHeaders(harOutputItem.HarEntry.Response.Headers) @@ -24,13 +31,15 @@ func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) { } if harOutputItem.HarEntry.Request.PostData != nil { - filteredRequestBody, err := filterHttpBody([]byte(harOutputItem.HarEntry.Request.PostData.Text)) + requestContentType := getContentTypeHeaderValue(harOutputItem.HarEntry.Request.Headers) + filteredRequestBody, err := filterHttpBody([]byte(harOutputItem.HarEntry.Request.PostData.Text), requestContentType, options) if err == nil { harOutputItem.HarEntry.Request.PostData.Text = string(filteredRequestBody) } } if harOutputItem.HarEntry.Response.Content != nil { - filteredResponseBody, err := filterHttpBody(harOutputItem.HarEntry.Response.Content.Text) + responseContentType := getContentTypeHeaderValue(harOutputItem.HarEntry.Response.Headers) + filteredResponseBody, err := filterHttpBody(harOutputItem.HarEntry.Response.Content.Text, responseContentType, options) if err == nil { harOutputItem.HarEntry.Response.Content.Text = filteredResponseBody } @@ -45,6 +54,15 @@ func filterHarHeaders(headers []har.Header) { } } +func getContentTypeHeaderValue(headers []har.Header) string { + for _, header := range headers { + if strings.ToLower(header.Name) == "content-type" { + return header.Value + } + } + return "" +} + func isFieldNameSensitive(fieldName string) bool { name := strings.ToLower(fieldName) name = strings.ReplaceAll(name, "_", "") @@ -60,7 +78,63 @@ func isFieldNameSensitive(fieldName string) bool { return false } -func filterHttpBody(bytes []byte) ([]byte, error){ +func filterHttpBody(bytes []byte, contentType string, options *FilteringOptions) ([]byte, error) { + mimeType := strings.Split(contentType, ";")[0] + switch strings.ToLower(mimeType) { + case "application/json": + return filterJsonBody(bytes) + case "text/html": + fallthrough + case "application/xhtml+xml": + fallthrough + case "text/xml": + fallthrough + case "application/xml": + return filterXmlEtree(bytes) + case "text/plain": + if options != nil && options.PlainTextFilterRegexes != nil { + return filterPlainText(bytes, options), nil + } + } + return bytes, nil +} + +func filterPlainText(bytes []byte, options *FilteringOptions) []byte { + for _, regex := range options.PlainTextFilterRegexes { + bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue)) + } + return bytes +} + +func filterXmlEtree(bytes []byte) ([]byte, error) { + xmlDoc := etree.NewDocument() + err := xmlDoc.ReadFromBytes(bytes) + if err != nil { + return nil, err + } else { + filterXmlElement(xmlDoc.Root()) + } + return xmlDoc.WriteToBytes() +} + +func filterXmlElement(element *etree.Element) { + for i, attribute := range element.Attr { + if isFieldNameSensitive(attribute.Key) { + element.Attr[i].Value = maskedFieldPlaceholderValue + } + } + if element.ChildElements() == nil || len(element.ChildElements()) == 0 { + if isFieldNameSensitive(element.Tag) { + element.SetText(maskedFieldPlaceholderValue) + } + } else { + for _, element := range element.ChildElements() { + filterXmlElement(element) + } + } +} + +func filterJsonBody(bytes []byte) ([]byte, error) { var bodyJsonMap map[string] interface{} err := json.Unmarshal(bytes ,&bodyJsonMap) if err != nil { diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index c392f99ba..ba6bcc0d0 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -10,11 +10,12 @@ import ( ) type MizuTapOptions struct { - GuiPort uint16 - Namespace string - KubeConfigPath string - MizuImage string - MizuPodPort uint16 + GuiPort uint16 + Namespace string + KubeConfigPath string + MizuImage string + MizuPodPort uint16 + PlainTextFilterRegexes []string } @@ -50,4 +51,5 @@ func init() { tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod") + tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "text-value-filter-regex", "", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies") } From 4bc16fa0b4ff860e8b7a7d8ab65e8632508acc0c Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 1 Jun 2021 14:25:52 +0300 Subject: [PATCH 09/36] Update main.go, messageSensitiveDataCleaner.go, and 6 more files... --- api/main.go | 46 +++++++------------ .../messageSensitiveDataCleaner.go | 12 ++--- api/pkg/tap/passive_tapper.go | 4 +- cli/cmd/tapRunner.go | 33 +++++++++++-- cli/kubernetes/provider.go | 26 +++++++---- shared/consts.go | 8 ++++ shared/models.go | 4 ++ shared/serializableRegexp.go | 34 ++++++++++++++ 8 files changed, 114 insertions(+), 53 deletions(-) create mode 100644 shared/consts.go create mode 100644 shared/serializableRegexp.go diff --git a/api/main.go b/api/main.go index 7134733d6..c257bb638 100644 --- a/api/main.go +++ b/api/main.go @@ -16,7 +16,6 @@ import ( "mizuserver/pkg/utils" "os" "os/signal" - "regexp" ) var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API") @@ -24,9 +23,6 @@ var aggregator = flag.Bool("aggregator", false, "Run in aggregator mode with API var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode") var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu collector for tapping") -const nodeNameEnvVar = "NODE_NAME" -const tappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST" -const plainTextRegexesEnvVar = "PLAINTEXT_REGEXES" func main() { flag.Parse() @@ -38,7 +34,7 @@ func main() { if *standalone { harOutputChannel := tap.StartPassiveTapper() filteredHarChannel := make(chan *tap.OutputChannelItem) - go filterHarHeaders(harOutputChannel, filteredHarChannel) + go filterHarHeaders(harOutputChannel, filteredHarChannel, getFilteringOptions()) go api.StartReadingEntries(filteredHarChannel, nil) hostApi(nil) } else if *shouldTap { @@ -57,12 +53,12 @@ func main() { if err != nil { panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err)) } - filteredHarChannel := make(chan *tap.OutputChannelItem) - go filterHarHeaders(harOutputChannel, filteredHarChannel) - go pipeChannelToSocket(socketConnection, filteredHarChannel) + go pipeChannelToSocket(socketConnection, harOutputChannel) } else if *aggregator { socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) - go api.StartReadingEntries(socketHarOutChannel, nil) + filteredHarChannel := make(chan *tap.OutputChannelItem) + go api.StartReadingEntries(filteredHarChannel, nil) + go filterHarHeaders(socketHarOutChannel, filteredHarChannel, getFilteringOptions()) hostApi(socketHarOutChannel) } @@ -96,40 +92,32 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) { func getTapTargets() []string { - nodeName := os.Getenv(nodeNameEnvVar) + nodeName := os.Getenv(shared.NodeNameEnvVar) var tappedAddressesPerNodeDict map[string][]string - err := json.Unmarshal([]byte(os.Getenv(tappedAddressesPerNodeDictEnvVar)), &tappedAddressesPerNodeDict) + err := json.Unmarshal([]byte(os.Getenv(shared.TappedAddressesPerNodeDictEnvVar)), &tappedAddressesPerNodeDict) if err != nil { - panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", tappedAddressesPerNodeDictEnvVar, tappedAddressesPerNodeDict, err)) + panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", shared.TappedAddressesPerNodeDictEnvVar, tappedAddressesPerNodeDict, err)) } return tappedAddressesPerNodeDict[nodeName] } -func getFilteringOptions() *sensitiveDataFiltering.FilteringOptions { - regexJsonArr := os.Getenv(plainTextRegexesEnvVar) - if regexJsonArr == "" { +func getFilteringOptions() *shared.FilteringOptions { + filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar) + if filteringOptionsJson == "" { return nil } - var regexStrSlice []string - err := json.Unmarshal([]byte(regexJsonArr), ®exStrSlice) + var filteringOptions shared.FilteringOptions + err := json.Unmarshal([]byte(filteringOptionsJson), &filteringOptions) if err != nil { - panic(fmt.Sprintf("env var %s's value of %s is invalid! must be []string %v", plainTextRegexesEnvVar, regexJsonArr, err)) + panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.FilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err)) } - parsedRegexSlice := make([]regexp.Regexp, 0) - for _, regexStr := range regexStrSlice { - regex, err := regexp.Compile(regexStr) - if err != nil { - panic(fmt.Sprintf("env var %s's value of %s is invalid! must be []string %v", plainTextRegexesEnvVar, regexJsonArr, err)) - } - parsedRegexSlice = append(parsedRegexSlice, *regex) - } - return &sensitiveDataFiltering.FilteringOptions{PlainTextFilterRegexes: parsedRegexSlice} + return &filteringOptions } -func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem) { +func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.FilteringOptions) { for message := range inChannel { - sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, nil) + sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions) outChannel <- message } } diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index fa247f196..0bd8b95bd 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -3,20 +3,16 @@ package sensitiveDataFiltering import ( "encoding/json" "fmt" + "github.com/up9inc/mizu/shared" "mizuserver/pkg/tap" "net/url" - "regexp" "strings" "github.com/beevik/etree" "github.com/google/martian/har" ) -type FilteringOptions struct { - PlainTextFilterRegexes []regexp.Regexp -} - -func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *FilteringOptions) { +func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.FilteringOptions) { filterHarHeaders(harOutputItem.HarEntry.Request.Headers) filterHarHeaders(harOutputItem.HarEntry.Response.Headers) @@ -78,7 +74,7 @@ func isFieldNameSensitive(fieldName string) bool { return false } -func filterHttpBody(bytes []byte, contentType string, options *FilteringOptions) ([]byte, error) { +func filterHttpBody(bytes []byte, contentType string, options *shared.FilteringOptions) ([]byte, error) { mimeType := strings.Split(contentType, ";")[0] switch strings.ToLower(mimeType) { case "application/json": @@ -99,7 +95,7 @@ func filterHttpBody(bytes []byte, contentType string, options *FilteringOptions) return bytes, nil } -func filterPlainText(bytes []byte, options *FilteringOptions) []byte { +func filterPlainText(bytes []byte, options *shared.FilteringOptions) []byte { for _, regex := range options.PlainTextFilterRegexes { bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue)) } diff --git a/api/pkg/tap/passive_tapper.go b/api/pkg/tap/passive_tapper.go index b5cacc9e8..72f5c1294 100644 --- a/api/pkg/tap/passive_tapper.go +++ b/api/pkg/tap/passive_tapper.go @@ -13,6 +13,7 @@ import ( "encoding/json" "flag" "fmt" + "github.com/up9inc/mizu/shared" "log" "os" "os/signal" @@ -34,7 +35,6 @@ import ( const AppPortsEnvVar = "APP_PORTS" const OutPortEnvVar = "WEB_SOCKET_PORT" const maxHTTP2DataLenEnvVar = "HTTP2_DATA_SIZE_LIMIT" -const hostModeEnvVar = "HOST_MODE" // default is 1MB, more than the max size accepted by collector and traffic-dumper const maxHTTP2DataLenDefault = 1 * 1024 * 1024 const cleanPeriod = time.Second * 10 @@ -258,7 +258,7 @@ func startPassiveTapper(harWriter *HarWriter) { maxHTTP2DataLen = convertedInt } } - hostMode = os.Getenv(hostModeEnvVar) == "1" + hostMode = os.Getenv(shared.HostModeEnvVar) == "1" fmt.Printf("App Ports: %v\n", appPorts) fmt.Printf("Tap output websocket port: %s\n", tapOutputPort) diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 040957ecb..44e38904d 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "github.com/up9inc/mizu/shared" "os" "os/signal" "regexp" @@ -26,6 +27,10 @@ const ( var currentlyTappedPods []core.Pod func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { + mizuApiFilteringOptions, err := getMizuApiFilteringOptions(tappingOptions) + if err != nil { + return + } kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath, tappingOptions.Namespace) defer cleanUpMizuResources(kubernetesProvider) @@ -43,7 +48,7 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { return } - if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil { + if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions, mizuApiFilteringOptions); err != nil { return } @@ -57,8 +62,8 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { // TODO handle incoming traffic from tapper using a channel } -func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error { - if err := createMizuAggregator(ctx, kubernetesProvider, tappingOptions); err != nil { +func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.FilteringOptions) error { + if err := createMizuAggregator(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil { return err } @@ -69,11 +74,11 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro return nil } -func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions) error { +func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.FilteringOptions) error { var err error mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider) - _, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists) + _, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists, mizuApiFilteringOptions) if err != nil { fmt.Printf("Error creating mizu collector pod: %v\n", err) return err @@ -88,6 +93,24 @@ func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Pr return nil } +func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.FilteringOptions, error) { + if tappingOptions.PlainTextFilterRegexes == nil || len(tappingOptions.PlainTextFilterRegexes) == 0 { + return nil, nil + } + + compiledRegexSlice := make([]*shared.SerializableRegexp, 0) + for _, regexStr := range tappingOptions.PlainTextFilterRegexes { + compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr) + if err != nil { + fmt.Printf("Regex %s is invalid: %v", regexStr, err) + return nil, err + } + compiledRegexSlice = append(compiledRegexSlice, compiledRegex) + } + + return &shared.FilteringOptions{PlainTextFilterRegexes: compiledRegexSlice}, nil +} + func createMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error { if err := kubernetesProvider.ApplyMizuTapperDaemonSet( ctx, diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 348068b4c..d6b598a27 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -6,19 +6,20 @@ import ( "encoding/json" "errors" "fmt" + "github.com/up9inc/mizu/shared" "path/filepath" "regexp" - applyconfapp "k8s.io/client-go/applyconfigurations/apps/v1" - applyconfmeta "k8s.io/client-go/applyconfigurations/meta/v1" - applyconfcore "k8s.io/client-go/applyconfigurations/core/v1" core "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/watch" + applyconfapp "k8s.io/client-go/applyconfigurations/apps/v1" + applyconfcore "k8s.io/client-go/applyconfigurations/core/v1" + applyconfmeta "k8s.io/client-go/applyconfigurations/meta/v1" "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth/azure" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -85,7 +86,11 @@ func (provider *Provider) GetPods(ctx context.Context, namespace string) { fmt.Printf("There are %d pods in Namespace %s\n", len(pods.Items), namespace) } -func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool) (*core.Pod, error) { +func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.FilteringOptions) (*core.Pod, error) { + marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions) + if err != nil { + return nil, err + } pod := &core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -101,9 +106,13 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace Command: []string {"./mizuagent", "--aggregator"}, Env: []core.EnvVar{ { - Name: "HOST_MODE", + Name: shared.HostModeEnvVar, Value: "1", }, + { + Name: shared.MizuFilteringOptionsEnvVar, + Value: string(marshaledFilteringOptions), + }, }, }, }, @@ -232,12 +241,11 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(privileged)) agentContainer.WithCommand("./mizuagent", "-i", "any", "--tap", "--hardump", "--aggregator-address", fmt.Sprintf("ws://%s/wsTapper", aggregatorPodIp)) agentContainer.WithEnv( - applyconfcore.EnvVar().WithName("HOST_MODE").WithValue("1"), - applyconfcore.EnvVar().WithName("AGGREGATOR_ADDRESS").WithValue(aggregatorPodIp), - applyconfcore.EnvVar().WithName("TAPPED_ADDRESSES_PER_HOST").WithValue(string(nodeToTappedPodIPMapJsonStr)), + applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"), + applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodIPMapJsonStr)), ) agentContainer.WithEnv( - applyconfcore.EnvVar().WithName("NODE_NAME").WithValueFrom( + applyconfcore.EnvVar().WithName(shared.NodeNameEnvVar).WithValueFrom( applyconfcore.EnvVarSource().WithFieldRef( applyconfcore.ObjectFieldSelector().WithAPIVersion("v1").WithFieldPath("spec.nodeName"), ), diff --git a/shared/consts.go b/shared/consts.go new file mode 100644 index 000000000..ccda67615 --- /dev/null +++ b/shared/consts.go @@ -0,0 +1,8 @@ +package shared + +const ( + MizuFilteringOptionsEnvVar = "SENSITIVE_DATA_FILTERING_OPTIONS" + HostModeEnvVar = "HOST_MODE" + NodeNameEnvVar = "NODE_NAME" + TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST" +) diff --git a/shared/models.go b/shared/models.go index dba9e1731..44e3ff9f5 100644 --- a/shared/models.go +++ b/shared/models.go @@ -33,3 +33,7 @@ func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessag TappingStatus: tappingStatus, } } + +type FilteringOptions struct { + PlainTextFilterRegexes []*SerializableRegexp +} diff --git a/shared/serializableRegexp.go b/shared/serializableRegexp.go new file mode 100644 index 000000000..a0a513ff6 --- /dev/null +++ b/shared/serializableRegexp.go @@ -0,0 +1,34 @@ +package shared + +import "regexp" + +type SerializableRegexp struct { + regexp.Regexp +} + +// CompileRegexToSerializableRegexp wraps the result of the standard library's +// regexp.Compile, for easy (un)marshaling. +func CompileRegexToSerializableRegexp(expr string) (*SerializableRegexp, error) { + re, err := regexp.Compile(expr) + if err != nil { + return nil, err + } + return &SerializableRegexp{*re}, nil +} + +// UnmarshalText satisfies the encoding.TextMarshaler interface, +// also used by json.Unmarshal. +func (r *SerializableRegexp) UnmarshalText(text []byte) error { + rr, err := CompileRegexToSerializableRegexp(string(text)) + if err != nil { + return err + } + *r = *rr + return nil +} + +// MarshalText satisfies the encoding.TextMarshaler interface, +// also used by json.Marshal. +func (r *SerializableRegexp) MarshalText() ([]byte, error) { + return []byte(r.String()), nil +} From 107c2d5b596376d870bcbf0db843623c443f9a10 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 1 Jun 2021 14:27:19 +0300 Subject: [PATCH 10/36] Update main.go, messageSensitiveDataCleaner.go, and 3 more files... --- api/main.go | 12 ++++++------ .../messageSensitiveDataCleaner.go | 10 +++++----- cli/cmd/tapRunner.go | 8 ++++---- cli/kubernetes/provider.go | 2 +- shared/models.go | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/api/main.go b/api/main.go index c257bb638..b1bd17a6c 100644 --- a/api/main.go +++ b/api/main.go @@ -34,7 +34,7 @@ func main() { if *standalone { harOutputChannel := tap.StartPassiveTapper() filteredHarChannel := make(chan *tap.OutputChannelItem) - go filterHarHeaders(harOutputChannel, filteredHarChannel, getFilteringOptions()) + go filterHarHeaders(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions()) go api.StartReadingEntries(filteredHarChannel, nil) hostApi(nil) } else if *shouldTap { @@ -58,7 +58,7 @@ func main() { socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) filteredHarChannel := make(chan *tap.OutputChannelItem) go api.StartReadingEntries(filteredHarChannel, nil) - go filterHarHeaders(socketHarOutChannel, filteredHarChannel, getFilteringOptions()) + go filterHarHeaders(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) hostApi(socketHarOutChannel) } @@ -101,21 +101,21 @@ func getTapTargets() []string { return tappedAddressesPerNodeDict[nodeName] } -func getFilteringOptions() *shared.FilteringOptions { +func getTrafficFilteringOptions() *shared.TrafficFilteringOptions { filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar) if filteringOptionsJson == "" { return nil } - var filteringOptions shared.FilteringOptions + var filteringOptions shared.TrafficFilteringOptions err := json.Unmarshal([]byte(filteringOptionsJson), &filteringOptions) if err != nil { - panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.FilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err)) + panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err)) } return &filteringOptions } -func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.FilteringOptions) { +func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { for message := range inChannel { sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions) outChannel <- message diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index 0bd8b95bd..2d881a5ad 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -12,7 +12,7 @@ import ( "github.com/google/martian/har" ) -func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.FilteringOptions) { +func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.TrafficFilteringOptions) { filterHarHeaders(harOutputItem.HarEntry.Request.Headers) filterHarHeaders(harOutputItem.HarEntry.Response.Headers) @@ -74,7 +74,7 @@ func isFieldNameSensitive(fieldName string) bool { return false } -func filterHttpBody(bytes []byte, contentType string, options *shared.FilteringOptions) ([]byte, error) { +func filterHttpBody(bytes []byte, contentType string, options *shared.TrafficFilteringOptions) ([]byte, error) { mimeType := strings.Split(contentType, ";")[0] switch strings.ToLower(mimeType) { case "application/json": @@ -88,15 +88,15 @@ func filterHttpBody(bytes []byte, contentType string, options *shared.FilteringO case "application/xml": return filterXmlEtree(bytes) case "text/plain": - if options != nil && options.PlainTextFilterRegexes != nil { + if options != nil && options.PlainTextMaskingRegexes != nil { return filterPlainText(bytes, options), nil } } return bytes, nil } -func filterPlainText(bytes []byte, options *shared.FilteringOptions) []byte { - for _, regex := range options.PlainTextFilterRegexes { +func filterPlainText(bytes []byte, options *shared.TrafficFilteringOptions) []byte { + for _, regex := range options.PlainTextMaskingRegexes { bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue)) } return bytes diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 44e38904d..1fb81edb4 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -62,7 +62,7 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { // TODO handle incoming traffic from tapper using a channel } -func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.FilteringOptions) error { +func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error { if err := createMizuAggregator(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil { return err } @@ -74,7 +74,7 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro return nil } -func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.FilteringOptions) error { +func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error { var err error mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider) @@ -93,7 +93,7 @@ func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Pr return nil } -func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.FilteringOptions, error) { +func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.TrafficFilteringOptions, error) { if tappingOptions.PlainTextFilterRegexes == nil || len(tappingOptions.PlainTextFilterRegexes) == 0 { return nil, nil } @@ -108,7 +108,7 @@ func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.Filteri compiledRegexSlice = append(compiledRegexSlice, compiledRegex) } - return &shared.FilteringOptions{PlainTextFilterRegexes: compiledRegexSlice}, nil + return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice}, nil } func createMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error { diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index d6b598a27..decee5e5f 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -86,7 +86,7 @@ func (provider *Provider) GetPods(ctx context.Context, namespace string) { fmt.Printf("There are %d pods in Namespace %s\n", len(pods.Items), namespace) } -func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.FilteringOptions) (*core.Pod, error) { +func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.TrafficFilteringOptions) (*core.Pod, error) { marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions) if err != nil { return nil, err diff --git a/shared/models.go b/shared/models.go index 44e3ff9f5..cbf73b594 100644 --- a/shared/models.go +++ b/shared/models.go @@ -34,6 +34,6 @@ func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessag } } -type FilteringOptions struct { - PlainTextFilterRegexes []*SerializableRegexp +type TrafficFilteringOptions struct { + PlainTextMaskingRegexes []*SerializableRegexp } From ff2131ea1ecddc59aebd081c0656570ce572866a Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 1 Jun 2021 14:39:22 +0300 Subject: [PATCH 11/36] Update consts.go, messageSensitiveDataCleaner.go, and tap.go --- api/pkg/sensitiveDataFiltering/consts.go | 2 +- api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go | 2 +- cli/cmd/tap.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pkg/sensitiveDataFiltering/consts.go b/api/pkg/sensitiveDataFiltering/consts.go index 8cd0cacf0..e5624de73 100644 --- a/api/pkg/sensitiveDataFiltering/consts.go +++ b/api/pkg/sensitiveDataFiltering/consts.go @@ -7,4 +7,4 @@ var personallyIdentifiableDataFields = []string{"token", "authorization", "authe "username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt", "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", "zip", "zipcode", "address", "country", "firstname", "lastname", - "middlename", "fname", "lname", "birthdate", "title"} + "middlename", "fname", "lname", "birthdate"} diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index 2d881a5ad..baa46e3e6 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -3,13 +3,13 @@ package sensitiveDataFiltering import ( "encoding/json" "fmt" - "github.com/up9inc/mizu/shared" "mizuserver/pkg/tap" "net/url" "strings" "github.com/beevik/etree" "github.com/google/martian/har" + "github.com/up9inc/mizu/shared" ) func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.TrafficFilteringOptions) { diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index ba6bcc0d0..b597d0ea9 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -51,5 +51,5 @@ func init() { tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod") - tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "text-value-filter-regex", "", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies") + tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "http-text-body-filter-regex", "", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies") } From 76bb3db55320bcb027f462aa97d5150a4b40d234 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 1 Jun 2021 14:40:34 +0300 Subject: [PATCH 12/36] Update provider.go --- cli/kubernetes/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index decee5e5f..20acd52f7 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -6,11 +6,11 @@ import ( "encoding/json" "errors" "fmt" - "github.com/up9inc/mizu/shared" "path/filepath" "regexp" + "github.com/up9inc/mizu/shared" core "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" From 1c4588a83cf8a124b0e3c792a8402817f638b5b4 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 1 Jun 2021 14:41:42 +0300 Subject: [PATCH 13/36] Update serializableRegexp.go --- shared/serializableRegexp.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/shared/serializableRegexp.go b/shared/serializableRegexp.go index a0a513ff6..e311fdeb5 100644 --- a/shared/serializableRegexp.go +++ b/shared/serializableRegexp.go @@ -6,8 +6,6 @@ type SerializableRegexp struct { regexp.Regexp } -// CompileRegexToSerializableRegexp wraps the result of the standard library's -// regexp.Compile, for easy (un)marshaling. func CompileRegexToSerializableRegexp(expr string) (*SerializableRegexp, error) { re, err := regexp.Compile(expr) if err != nil { @@ -16,8 +14,7 @@ func CompileRegexToSerializableRegexp(expr string) (*SerializableRegexp, error) return &SerializableRegexp{*re}, nil } -// UnmarshalText satisfies the encoding.TextMarshaler interface, -// also used by json.Unmarshal. +// UnmarshalText is by json.Unmarshal. func (r *SerializableRegexp) UnmarshalText(text []byte) error { rr, err := CompileRegexToSerializableRegexp(string(text)) if err != nil { @@ -27,8 +24,7 @@ func (r *SerializableRegexp) UnmarshalText(text []byte) error { return nil } -// MarshalText satisfies the encoding.TextMarshaler interface, -// also used by json.Marshal. +// MarshalText is used by json.Marshal. func (r *SerializableRegexp) MarshalText() ([]byte, error) { return []byte(r.String()), nil } From b3cfd20a78754d35f3accbfd34c8e178df4862c9 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 1 Jun 2021 17:18:57 +0300 Subject: [PATCH 14/36] Update tap.go --- cli/cmd/tap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index b597d0ea9..5c795d935 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -51,5 +51,5 @@ func init() { tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod") - tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "http-text-body-filter-regex", "", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies") + tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies") } From 2c8d1f854de2cf48a1edfeaa4b32b73ef7d6e5ac Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Wed, 2 Jun 2021 08:17:45 +0300 Subject: [PATCH 15/36] TRA-3234 fetch with _source + no hard limit (#64) * remove the HARD limit of 5000 --- api/pkg/controllers/entries_controller.go | 17 +++++++++------- api/pkg/models/models.go | 24 ++++++++++++++++++++++- api/pkg/routes/public_routes.go | 2 +- cli/Makefile | 2 +- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index e9702fe3b..70ec62714 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -64,7 +64,7 @@ func GetEntries(c *fiber.Ctx) error { return c.Status(fiber.StatusOK).JSON(baseEntries) } -func GetHAR(c *fiber.Ctx) error { +func GetHARs(c *fiber.Ctx) error { entriesFilter := &models.HarFetchRequestBody{} order := OrderDesc if err := c.QueryParser(entriesFilter); err != nil { @@ -87,7 +87,7 @@ func GetHAR(c *fiber.Ctx) error { utils.ReverseSlice(entries) } - harsObject := map[string]*har.HAR{} + harsObject := map[string]*models.ExtendedHAR{} for _, entryData := range entries { harEntryObject := []byte(entryData.Entry) @@ -101,12 +101,15 @@ func GetHAR(c *fiber.Ctx) error { } else { var entriesHar []*har.Entry entriesHar = append(entriesHar, &harEntry) - harsObject[sourceOfEntry] = &har.HAR{ - Log: &har.Log{ + harsObject[sourceOfEntry] = &models.ExtendedHAR{ + Log: &models.ExtendedLog{ Version: "1.2", - Creator: &har.Creator{ - Name: "mizu", - Version: "0.0.1", + Creator: &models.ExtendedCreator{ + Creator: &har.Creator{ + Name: "mizu", + Version: "0.0.2", + }, + Source: sourceOfEntry, }, Entries: entriesHar, }, diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index feb1a4b90..856d1aa4c 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -2,6 +2,7 @@ package models import ( "encoding/json" + "github.com/google/martian/har" "github.com/up9inc/mizu/shared" "mizuserver/pkg/tap" "time" @@ -47,7 +48,7 @@ type EntriesFilter struct { } type HarFetchRequestBody struct { - Limit int `query:"limit" validate:"max=5000"` + Limit int `query:"limit"` } type WebSocketEntryMessage struct { @@ -80,3 +81,24 @@ func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, err } return json.Marshal(message) } + + +// ExtendedHAR is the top level object of a HAR log. +type ExtendedHAR struct { + Log *ExtendedLog `json:"log"` +} + +// ExtendedLog is the HAR HTTP request and response log. +type ExtendedLog struct { + // Version number of the HAR format. + Version string `json:"version"` + // Creator holds information about the log creator application. + Creator *ExtendedCreator `json:"creator"` + // Entries is a list containing requests and responses. + Entries []*har.Entry `json:"entries"` +} + +type ExtendedCreator struct { + *har.Creator + Source string `json:"_source"` +} \ No newline at end of file diff --git a/api/pkg/routes/public_routes.go b/api/pkg/routes/public_routes.go index c15203189..042ea54c9 100644 --- a/api/pkg/routes/public_routes.go +++ b/api/pkg/routes/public_routes.go @@ -12,7 +12,7 @@ func EntriesRoutes(fiberApp *fiber.App) { routeGroup.Get("/entries", controllers.GetEntries) // get entries (base/thin entries) routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry - routeGroup.Get("/har", controllers.GetHAR) + routeGroup.Get("/har", controllers.GetHARs) routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB diff --git a/cli/Makefile b/cli/Makefile index a7af9999c..835ee9142 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -1,6 +1,6 @@ FOLDER=$(GOOS).$(GOARCH) COMMIT_HASH=$(shell git rev-parse HEAD) -GIT_BRANCH=$(shell git branch --show-current) +GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]') .PHONY: help .DEFAULT_GOAL := help From 1bf5bf0b31e1f7c6e733174a9f767c8fc6e2f9f0 Mon Sep 17 00:00:00 2001 From: nimrod-up9 <59927337+nimrod-up9@users.noreply.github.com> Date: Thu, 3 Jun 2021 19:48:12 +0300 Subject: [PATCH 16/36] TRA-3299 Reduce footprint and Add Tolerances(#65) * Use lib const for DNSClusterFirstWithHostNet. * Whitespace. * Break lines. * Added affinity to pod names. * Added tolerations to NoExecute and NoSchedule taints. --- cli/kubernetes/provider.go | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 20acd52f7..c8ddd8433 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -116,7 +116,7 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace }, }, }, - DNSPolicy: "ClusterFirstWithHostNet", + DNSPolicy: core.DNSClusterFirstWithHostNet, TerminationGracePeriodSeconds: new(int64), // Affinity: TODO: define node selector for all relevant nodes for this mizu instance }, @@ -252,12 +252,40 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac ), ) - podSpec := applyconfcore.PodSpec().WithHostNetwork(true).WithDNSPolicy("ClusterFirstWithHostNet").WithTerminationGracePeriodSeconds(0) + nodeNames := make([]string, 0, len(nodeToTappedPodIPMap)) + for nodeName := range nodeToTappedPodIPMap { + nodeNames = append(nodeNames, nodeName) + } + nodeSelectorRequirement := applyconfcore.NodeSelectorRequirement() + nodeSelectorRequirement.WithKey("kubernetes.io/hostname") + nodeSelectorRequirement.WithOperator(core.NodeSelectorOpIn) + nodeSelectorRequirement.WithValues(nodeNames...) + nodeSelectorTerm := applyconfcore.NodeSelectorTerm() + nodeSelectorTerm.WithMatchExpressions(nodeSelectorRequirement) + nodeSelector := applyconfcore.NodeSelector() + nodeSelector.WithNodeSelectorTerms(nodeSelectorTerm) + nodeAffinity := applyconfcore.NodeAffinity() + nodeAffinity.WithRequiredDuringSchedulingIgnoredDuringExecution(nodeSelector) + affinity := applyconfcore.Affinity() + affinity.WithNodeAffinity(nodeAffinity) + + noExecuteToleration := applyconfcore.Toleration() + noExecuteToleration.WithOperator(core.TolerationOpExists) + noExecuteToleration.WithEffect(core.TaintEffectNoExecute) + noScheduleToleration := applyconfcore.Toleration() + noScheduleToleration.WithOperator(core.TolerationOpExists) + noScheduleToleration.WithEffect(core.TaintEffectNoSchedule) + + podSpec := applyconfcore.PodSpec() + podSpec.WithHostNetwork(true) + podSpec.WithDNSPolicy(core.DNSClusterFirstWithHostNet) + podSpec.WithTerminationGracePeriodSeconds(0) if linkServiceAccount { podSpec.WithServiceAccountName(serviceAccountName) } podSpec.WithContainers(agentContainer) - + podSpec.WithAffinity(affinity) + podSpec.WithTolerations(noExecuteToleration, noScheduleToleration) podTemplate := applyconfcore.PodTemplateSpec() podTemplate.WithLabels(map[string]string{"app": tapperPodName}) From 88a5befd4bec1bb2a70ef512620673745ddebbfc Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Sun, 6 Jun 2021 17:00:23 +0300 Subject: [PATCH 17/36] Implementation of Mizu view command --- cli/cmd/view.go | 3 +- cli/cmd/viewRunner.go | 37 ++++++++ cli/go.sum | 172 +++++++++++++++++++++++++++++++++++++ cli/kubernetes/provider.go | 64 +++++++------- 4 files changed, 242 insertions(+), 34 deletions(-) create mode 100644 cli/cmd/viewRunner.go diff --git a/cli/cmd/view.go b/cli/cmd/view.go index aa4aa928d..529c756b1 100644 --- a/cli/cmd/view.go +++ b/cli/cmd/view.go @@ -1,7 +1,6 @@ package cmd import ( - "fmt" "github.com/spf13/cobra" ) @@ -9,7 +8,7 @@ var viewCmd = &cobra.Command{ Use: "view", Short: "Open GUI in browser", RunE: func(cmd *cobra.Command, args []string) error { - fmt.Println("Not implemented") + runMizuView() return nil }, } diff --git a/cli/cmd/viewRunner.go b/cli/cmd/viewRunner.go new file mode 100644 index 000000000..224457835 --- /dev/null +++ b/cli/cmd/viewRunner.go @@ -0,0 +1,37 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "github.com/up9inc/mizu/cli/kubernetes" + "github.com/up9inc/mizu/cli/mizu" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "net/http" +) + +func runMizuView() { + kubernetesProvider := kubernetes.NewProvider("", "") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, err := kubernetesProvider.ClientSet.CoreV1().Services(mizu.ResourcesNamespace).Get(ctx, mizu.AggregatorPodName, metav1.GetOptions{}) + var statusError *k8serrors.StatusError + if errors.As(err, &statusError) { + if statusError.ErrStatus.Reason == metav1.StatusReasonNotFound { + fmt.Printf("The %s service not found\n", mizu.AggregatorPodName) + return + } + panic(err) + } + + _, err = http.Get("http://localhost:8899/") + if err == nil { + fmt.Printf("Found a running service %s and open port 8899\n", mizu.AggregatorPodName) + return + } + fmt.Printf("Found service %s, creating port forwarding to 8899\n", mizu.AggregatorPodName) + portForwardApiPod(ctx, kubernetesProvider, cancel, &MizuTapOptions{GuiPort: 8899, MizuPodPort: 8899}) +} diff --git a/cli/go.sum b/cli/go.sum index 9610fbffe..654ce234e 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -12,16 +12,22 @@ cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0 h1:9x7Bx0A9R5/M9jibeJeZWqjeVEIxYW9fZYqB9a70/bY= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -37,82 +43,132 @@ github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1 h1:ocYkMQY5RrXTYgXl7ICpV0IXwlEQGwKIsery4gyXa1U= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -128,6 +184,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -139,86 +196,138 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3 h1:SRgJV+IoxM5MKyFdlSUeNy6/ycRUF2yBAKdAQswoHUk= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0 h1:lfGJxY7ToLJQjHHwi0EX6uYBdK78egf954SQl13PQJc= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= @@ -228,78 +337,119 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d h1:7PxY7LVfSZm7PEeBTyK1rj1gABdCO2mbri6GKO1cMDs= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af h1:gu+uRPtBe88sKxUCEXRoeCvVG90TJmwhiqRpvdhQFng= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -321,8 +471,10 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -333,14 +485,17 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -382,6 +537,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -465,6 +621,7 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -480,6 +637,7 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -505,6 +663,7 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -512,6 +671,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -523,18 +683,24 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -550,6 +716,7 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y= k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= @@ -557,15 +724,20 @@ k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag= k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac h1:sAvhNk5RRuc6FNYGqe7Ygz3PSo/2wGWbulskmzRX8Vs= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index c8ddd8433..e6d841d30 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -32,7 +32,7 @@ import ( ) type Provider struct { - clientSet *kubernetes.Clientset + ClientSet *kubernetes.Clientset kubernetesConfig clientcmd.ClientConfig clientConfig restclient.Config Namespace string @@ -63,7 +63,7 @@ func NewProvider(kubeConfigPath string, overrideNamespace string) *Provider { } return &Provider{ - clientSet: clientSet, + ClientSet: clientSet, kubernetesConfig: kubernetesConfig, clientConfig: *restClientConfig, Namespace: namespace, @@ -71,7 +71,7 @@ func NewProvider(kubeConfigPath string, overrideNamespace string) *Provider { } func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) watch.Interface { - watcher, err := provider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true}) + watcher, err := provider.ClientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true}) if err != nil { panic(err.Error()) } @@ -79,7 +79,7 @@ func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) w } func (provider *Provider) GetPods(ctx context.Context, namespace string) { - pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + pods, err := provider.ClientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) if err != nil { panic(err.Error()) } @@ -95,7 +95,7 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace ObjectMeta: metav1.ObjectMeta{ Name: podName, Namespace: namespace, - Labels: map[string]string{"app": podName}, + Labels: map[string]string{"app": podName}, }, Spec: core.PodSpec{ Containers: []core.Container{ @@ -103,20 +103,20 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace Name: podName, Image: podImage, ImagePullPolicy: core.PullAlways, - Command: []string {"./mizuagent", "--aggregator"}, + Command: []string{"./mizuagent", "--aggregator"}, Env: []core.EnvVar{ { - Name: shared.HostModeEnvVar, + Name: shared.HostModeEnvVar, Value: "1", }, { - Name: shared.MizuFilteringOptionsEnvVar, + Name: shared.MizuFilteringOptionsEnvVar, Value: string(marshaledFilteringOptions), }, }, }, }, - DNSPolicy: core.DNSClusterFirstWithHostNet, + DNSPolicy: core.DNSClusterFirstWithHostNet, TerminationGracePeriodSeconds: new(int64), // Affinity: TODO: define node selector for all relevant nodes for this mizu instance }, @@ -125,26 +125,26 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace if linkServiceAccount { pod.Spec.ServiceAccountName = serviceAccountName } - return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + return provider.ClientSet.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) } func (provider *Provider) CreateService(ctx context.Context, namespace string, serviceName string, appLabelValue string) (*core.Service, error) { service := core.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, + Name: serviceName, Namespace: namespace, }, Spec: core.ServiceSpec{ - Ports: []core.ServicePort {{TargetPort: intstr.FromInt(8899), Port: 80}}, - Type: core.ServiceTypeClusterIP, + Ports: []core.ServicePort{{TargetPort: intstr.FromInt(8899), Port: 80}}, + Type: core.ServiceTypeClusterIP, Selector: map[string]string{"app": appLabelValue}, }, } - return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{}) + return provider.ClientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{}) } -func (provider *Provider) DoesMizuRBACExist(ctx context.Context, namespace string) (bool, error){ - serviceAccount, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{}) +func (provider *Provider) DoesMizuRBACExist(ctx context.Context, namespace string) (bool, error) { + serviceAccount, err := provider.ClientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{}) var statusError *k8serrors.StatusError if errors.As(err, &statusError) { @@ -159,7 +159,7 @@ func (provider *Provider) DoesMizuRBACExist(ctx context.Context, namespace strin return serviceAccount != nil, nil } -func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string ,version string) error { +func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, version string) error { clusterRoleName := "mizu-cluster-role" serviceAccount := &core.ServiceAccount{ @@ -171,25 +171,25 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string , } clusterRole := &rbac.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: clusterRoleName, + Name: clusterRoleName, Labels: map[string]string{"mizu-cli-version": version}, }, Rules: []rbac.PolicyRule{ { - APIGroups: []string {"", "extensions", "apps"}, - Resources: []string {"pods", "services", "endpoints"}, - Verbs: []string {"list", "get", "watch"}, + APIGroups: []string{"", "extensions", "apps"}, + Resources: []string{"pods", "services", "endpoints"}, + Verbs: []string{"list", "get", "watch"}, }, }, } clusterRoleBinding := &rbac.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: "mizu-cluster-role-binding", + Name: "mizu-cluster-role-binding", Labels: map[string]string{"mizu-cli-version": version}, }, RoleRef: rbac.RoleRef{ - Name: clusterRoleName, - Kind: "ClusterRole", + Name: clusterRoleName, + Kind: "ClusterRole", APIGroup: "rbac.authorization.k8s.io", }, Subjects: []rbac.Subject{ @@ -200,15 +200,15 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string , }, }, } - _, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Create(ctx, serviceAccount, metav1.CreateOptions{}) + _, err := provider.ClientSet.CoreV1().ServiceAccounts(namespace).Create(ctx, serviceAccount, metav1.CreateOptions{}) if err != nil { return err } - _, err = provider.clientSet.RbacV1().ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{}) + _, err = provider.ClientSet.RbacV1().ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{}) if err != nil { return err } - _, err = provider.clientSet.RbacV1().ClusterRoleBindings().Create(ctx, clusterRoleBinding, metav1.CreateOptions{}) + _, err = provider.ClientSet.RbacV1().ClusterRoleBindings().Create(ctx, clusterRoleBinding, metav1.CreateOptions{}) if err != nil { return err } @@ -216,15 +216,15 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string , } func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error { - return provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) + return provider.ClientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) } func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error { - return provider.clientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{}) + return provider.ClientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{}) } func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error { - return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{}) + return provider.ClientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{}) } func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool) error { @@ -297,12 +297,12 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac daemonSet := applyconfapp.DaemonSet(daemonSetName, namespace) daemonSet.WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate)) - _, err = provider.clientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, metav1.ApplyOptions{FieldManager: fieldManagerName}) + _, err = provider.ClientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, metav1.ApplyOptions{FieldManager: fieldManagerName}) return err } func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp) ([]core.Pod, error) { - pods, err := provider.clientSet.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) + pods, err := provider.ClientSet.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } From f98185f0f03bfd1abff0bf76b8245b18413cd40a Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Sun, 6 Jun 2021 17:01:45 +0300 Subject: [PATCH 18/36] . --- cli/go.sum | 172 ----------------------------------------------------- 1 file changed, 172 deletions(-) diff --git a/cli/go.sum b/cli/go.sum index 654ce234e..9610fbffe 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -12,22 +12,16 @@ cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0 h1:9x7Bx0A9R5/M9jibeJeZWqjeVEIxYW9fZYqB9a70/bY= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -43,132 +37,82 @@ github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1 h1:ocYkMQY5RrXTYgXl7ICpV0IXwlEQGwKIsery4gyXa1U= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -184,7 +128,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -196,138 +139,86 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3 h1:SRgJV+IoxM5MKyFdlSUeNy6/ycRUF2yBAKdAQswoHUk= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0 h1:lfGJxY7ToLJQjHHwi0EX6uYBdK78egf954SQl13PQJc= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= @@ -337,119 +228,78 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d h1:7PxY7LVfSZm7PEeBTyK1rj1gABdCO2mbri6GKO1cMDs= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af h1:gu+uRPtBe88sKxUCEXRoeCvVG90TJmwhiqRpvdhQFng= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -471,10 +321,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -485,17 +333,14 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -537,7 +382,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -621,7 +465,6 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -637,7 +480,6 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -663,7 +505,6 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -671,7 +512,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -683,24 +523,18 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -716,7 +550,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y= k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= @@ -724,20 +557,15 @@ k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag= k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac h1:sAvhNk5RRuc6FNYGqe7Ygz3PSo/2wGWbulskmzRX8Vs= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= From 8316f8456f7f6e8287a73e254e866780ce490053 Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Sun, 6 Jun 2021 17:18:58 +0300 Subject: [PATCH 19/36] . --- cli/cmd/viewRunner.go | 16 +++++-------- cli/kubernetes/provider.go | 46 +++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/cli/cmd/viewRunner.go b/cli/cmd/viewRunner.go index 224457835..18e2bf9e0 100644 --- a/cli/cmd/viewRunner.go +++ b/cli/cmd/viewRunner.go @@ -2,12 +2,9 @@ package cmd import ( "context" - "errors" "fmt" "github.com/up9inc/mizu/cli/kubernetes" "github.com/up9inc/mizu/cli/mizu" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "net/http" ) @@ -17,15 +14,14 @@ func runMizuView() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, err := kubernetesProvider.ClientSet.CoreV1().Services(mizu.ResourcesNamespace).Get(ctx, mizu.AggregatorPodName, metav1.GetOptions{}) - var statusError *k8serrors.StatusError - if errors.As(err, &statusError) { - if statusError.ErrStatus.Reason == metav1.StatusReasonNotFound { - fmt.Printf("The %s service not found\n", mizu.AggregatorPodName) - return - } + exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName) + if err != nil { panic(err) } + if !exists { + fmt.Printf("The %s service not found\n", mizu.AggregatorPodName) + return + } _, err = http.Get("http://localhost:8899/") if err == nil { diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index e6d841d30..58d96b18c 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "path/filepath" "regexp" @@ -32,7 +31,7 @@ import ( ) type Provider struct { - ClientSet *kubernetes.Clientset + clientSet *kubernetes.Clientset kubernetesConfig clientcmd.ClientConfig clientConfig restclient.Config Namespace string @@ -63,7 +62,7 @@ func NewProvider(kubeConfigPath string, overrideNamespace string) *Provider { } return &Provider{ - ClientSet: clientSet, + clientSet: clientSet, kubernetesConfig: kubernetesConfig, clientConfig: *restClientConfig, Namespace: namespace, @@ -71,7 +70,7 @@ func NewProvider(kubeConfigPath string, overrideNamespace string) *Provider { } func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) watch.Interface { - watcher, err := provider.ClientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true}) + watcher, err := provider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true}) if err != nil { panic(err.Error()) } @@ -79,7 +78,7 @@ func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) w } func (provider *Provider) GetPods(ctx context.Context, namespace string) { - pods, err := provider.ClientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) if err != nil { panic(err.Error()) } @@ -125,7 +124,7 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace if linkServiceAccount { pod.Spec.ServiceAccountName = serviceAccountName } - return provider.ClientSet.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) } func (provider *Provider) CreateService(ctx context.Context, namespace string, serviceName string, appLabelValue string) (*core.Service, error) { @@ -140,11 +139,11 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s Selector: map[string]string{"app": appLabelValue}, }, } - return provider.ClientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{}) + return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{}) } func (provider *Provider) DoesMizuRBACExist(ctx context.Context, namespace string) (bool, error) { - serviceAccount, err := provider.ClientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{}) + serviceAccount, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{}) var statusError *k8serrors.StatusError if errors.As(err, &statusError) { @@ -159,6 +158,21 @@ func (provider *Provider) DoesMizuRBACExist(ctx context.Context, namespace strin return serviceAccount != nil, nil } +func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, serviceName string) (bool, error) { + service, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{}) + + var statusError *k8serrors.StatusError + if errors.As(err, &statusError) { + if statusError.ErrStatus.Reason == metav1.StatusReasonNotFound { + return false, nil + } + } + if err != nil { + return false, err + } + return service != nil, nil +} + func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, version string) error { clusterRoleName := "mizu-cluster-role" @@ -200,15 +214,15 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, }, }, } - _, err := provider.ClientSet.CoreV1().ServiceAccounts(namespace).Create(ctx, serviceAccount, metav1.CreateOptions{}) + _, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Create(ctx, serviceAccount, metav1.CreateOptions{}) if err != nil { return err } - _, err = provider.ClientSet.RbacV1().ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{}) + _, err = provider.clientSet.RbacV1().ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{}) if err != nil { return err } - _, err = provider.ClientSet.RbacV1().ClusterRoleBindings().Create(ctx, clusterRoleBinding, metav1.CreateOptions{}) + _, err = provider.clientSet.RbacV1().ClusterRoleBindings().Create(ctx, clusterRoleBinding, metav1.CreateOptions{}) if err != nil { return err } @@ -216,15 +230,15 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, } func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error { - return provider.ClientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) + return provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) } func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error { - return provider.ClientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{}) + return provider.clientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{}) } func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error { - return provider.ClientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{}) + return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{}) } func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool) error { @@ -297,12 +311,12 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac daemonSet := applyconfapp.DaemonSet(daemonSetName, namespace) daemonSet.WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate)) - _, err = provider.ClientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, metav1.ApplyOptions{FieldManager: fieldManagerName}) + _, err = provider.clientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, metav1.ApplyOptions{FieldManager: fieldManagerName}) return err } func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp) ([]core.Pod, error) { - pods, err := provider.ClientSet.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) + pods, err := provider.clientSet.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } From 931b6f4260743e6f191fda445a7962310eb767aa Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Mon, 7 Jun 2021 11:35:12 +0300 Subject: [PATCH 20/36] Update main.go and messageSensitiveDataCleaner.go --- api/main.go | 147 ------------------ .../messageSensitiveDataCleaner.go | 9 ++ 2 files changed, 9 insertions(+), 147 deletions(-) delete mode 100644 api/main.go diff --git a/api/main.go b/api/main.go deleted file mode 100644 index b1bd17a6c..000000000 --- a/api/main.go +++ /dev/null @@ -1,147 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "github.com/gofiber/fiber/v2" - "github.com/gorilla/websocket" - "github.com/up9inc/mizu/shared" - "mizuserver/pkg/api" - "mizuserver/pkg/middleware" - "mizuserver/pkg/models" - "mizuserver/pkg/routes" - "mizuserver/pkg/sensitiveDataFiltering" - "mizuserver/pkg/tap" - "mizuserver/pkg/utils" - "os" - "os/signal" -) - -var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API") -var aggregator = flag.Bool("aggregator", false, "Run in aggregator mode with API") -var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode") -var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu collector for tapping") - - -func main() { - flag.Parse() - - if !*shouldTap && !*aggregator && !*standalone{ - panic("One of the flags --tap, --api or --standalone must be provided") - } - - if *standalone { - harOutputChannel := tap.StartPassiveTapper() - filteredHarChannel := make(chan *tap.OutputChannelItem) - go filterHarHeaders(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions()) - go api.StartReadingEntries(filteredHarChannel, nil) - hostApi(nil) - } else if *shouldTap { - if *aggregatorAddress == "" { - panic("Aggregator address must be provided with --aggregator-address when using --tap") - } - - tapTargets := getTapTargets() - if tapTargets != nil { - tap.HostAppAddresses = tapTargets - fmt.Println("Filtering for the following addresses:", tap.HostAppAddresses) - } - - harOutputChannel := tap.StartPassiveTapper() - socketConnection, err := shared.ConnectToSocketServer(*aggregatorAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false) - if err != nil { - panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err)) - } - go pipeChannelToSocket(socketConnection, harOutputChannel) - } else if *aggregator { - socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) - filteredHarChannel := make(chan *tap.OutputChannelItem) - go api.StartReadingEntries(filteredHarChannel, nil) - go filterHarHeaders(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) - hostApi(socketHarOutChannel) - } - - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, os.Interrupt) - <-signalChan - - fmt.Println("Exiting") -} - -func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) { - app := fiber.New() - - - middleware.FiberMiddleware(app) // Register Fiber's middleware for app. - app.Static("/", "./site") - - //Simple route to know server is running - app.Get("/echo", func(c *fiber.Ctx) error { - return c.SendString("Hello, World 👋!") - }) - eventHandlers := api.RoutesEventHandlers{ - SocketHarOutChannel: socketHarOutputChannel, - } - routes.WebSocketRoutes(app, &eventHandlers) - routes.EntriesRoutes(app) - routes.NotFoundRoute(app) - - utils.StartServer(app) -} - - -func getTapTargets() []string { - nodeName := os.Getenv(shared.NodeNameEnvVar) - var tappedAddressesPerNodeDict map[string][]string - err := json.Unmarshal([]byte(os.Getenv(shared.TappedAddressesPerNodeDictEnvVar)), &tappedAddressesPerNodeDict) - if err != nil { - panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", shared.TappedAddressesPerNodeDictEnvVar, tappedAddressesPerNodeDict, err)) - } - return tappedAddressesPerNodeDict[nodeName] -} - -func getTrafficFilteringOptions() *shared.TrafficFilteringOptions { - filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar) - if filteringOptionsJson == "" { - return nil - } - var filteringOptions shared.TrafficFilteringOptions - err := json.Unmarshal([]byte(filteringOptionsJson), &filteringOptions) - if err != nil { - panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err)) - } - - return &filteringOptions -} - -func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { - for message := range inChannel { - sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions) - outChannel <- message - } -} - -func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) { - if connection == nil { - panic("Websocket connection is nil") - } - - if messageDataChannel == nil { - panic("Channel of captured messages is nil") - } - - for messageData := range messageDataChannel { - marshaledData, err := models.CreateWebsocketTappedEntryMessage(messageData) - if err != nil { - fmt.Printf("error converting message to json %s, (%v,%+v)\n", err, err, err) - continue - } - - err = connection.WriteMessage(websocket.TextMessage, marshaledData) - if err != nil { - fmt.Printf("error sending message through socket server %s, (%v,%+v)\n", err, err, err) - continue - } - } -} diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index baa46e3e6..7cf9660f9 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -2,6 +2,8 @@ package sensitiveDataFiltering import ( "encoding/json" + "encoding/xml" + "errors" "fmt" "mizuserver/pkg/tap" "net/url" @@ -103,6 +105,9 @@ func filterPlainText(bytes []byte, options *shared.TrafficFilteringOptions) []by } func filterXmlEtree(bytes []byte) ([]byte, error) { + if !IsValidXML(bytes) { + return nil, errors.New("Invalid XML") + } xmlDoc := etree.NewDocument() err := xmlDoc.ReadFromBytes(bytes) if err != nil { @@ -113,6 +118,10 @@ func filterXmlEtree(bytes []byte) ([]byte, error) { return xmlDoc.WriteToBytes() } +func IsValidXML(data []byte) bool { + return xml.Unmarshal(data, new(interface{})) == nil +} + func filterXmlElement(element *etree.Element) { for i, attribute := range element.Attr { if isFieldNameSensitive(attribute.Key) { From b7d3ff6eb8795ef430d09c4e46432745127cc980 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Mon, 7 Jun 2021 11:35:50 +0300 Subject: [PATCH 21/36] Update main.go --- api/main.go | 147 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 api/main.go diff --git a/api/main.go b/api/main.go new file mode 100644 index 000000000..b1bd17a6c --- /dev/null +++ b/api/main.go @@ -0,0 +1,147 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "github.com/gofiber/fiber/v2" + "github.com/gorilla/websocket" + "github.com/up9inc/mizu/shared" + "mizuserver/pkg/api" + "mizuserver/pkg/middleware" + "mizuserver/pkg/models" + "mizuserver/pkg/routes" + "mizuserver/pkg/sensitiveDataFiltering" + "mizuserver/pkg/tap" + "mizuserver/pkg/utils" + "os" + "os/signal" +) + +var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API") +var aggregator = flag.Bool("aggregator", false, "Run in aggregator mode with API") +var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode") +var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu collector for tapping") + + +func main() { + flag.Parse() + + if !*shouldTap && !*aggregator && !*standalone{ + panic("One of the flags --tap, --api or --standalone must be provided") + } + + if *standalone { + harOutputChannel := tap.StartPassiveTapper() + filteredHarChannel := make(chan *tap.OutputChannelItem) + go filterHarHeaders(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions()) + go api.StartReadingEntries(filteredHarChannel, nil) + hostApi(nil) + } else if *shouldTap { + if *aggregatorAddress == "" { + panic("Aggregator address must be provided with --aggregator-address when using --tap") + } + + tapTargets := getTapTargets() + if tapTargets != nil { + tap.HostAppAddresses = tapTargets + fmt.Println("Filtering for the following addresses:", tap.HostAppAddresses) + } + + harOutputChannel := tap.StartPassiveTapper() + socketConnection, err := shared.ConnectToSocketServer(*aggregatorAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false) + if err != nil { + panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err)) + } + go pipeChannelToSocket(socketConnection, harOutputChannel) + } else if *aggregator { + socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) + filteredHarChannel := make(chan *tap.OutputChannelItem) + go api.StartReadingEntries(filteredHarChannel, nil) + go filterHarHeaders(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) + hostApi(socketHarOutChannel) + } + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, os.Interrupt) + <-signalChan + + fmt.Println("Exiting") +} + +func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) { + app := fiber.New() + + + middleware.FiberMiddleware(app) // Register Fiber's middleware for app. + app.Static("/", "./site") + + //Simple route to know server is running + app.Get("/echo", func(c *fiber.Ctx) error { + return c.SendString("Hello, World 👋!") + }) + eventHandlers := api.RoutesEventHandlers{ + SocketHarOutChannel: socketHarOutputChannel, + } + routes.WebSocketRoutes(app, &eventHandlers) + routes.EntriesRoutes(app) + routes.NotFoundRoute(app) + + utils.StartServer(app) +} + + +func getTapTargets() []string { + nodeName := os.Getenv(shared.NodeNameEnvVar) + var tappedAddressesPerNodeDict map[string][]string + err := json.Unmarshal([]byte(os.Getenv(shared.TappedAddressesPerNodeDictEnvVar)), &tappedAddressesPerNodeDict) + if err != nil { + panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", shared.TappedAddressesPerNodeDictEnvVar, tappedAddressesPerNodeDict, err)) + } + return tappedAddressesPerNodeDict[nodeName] +} + +func getTrafficFilteringOptions() *shared.TrafficFilteringOptions { + filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar) + if filteringOptionsJson == "" { + return nil + } + var filteringOptions shared.TrafficFilteringOptions + err := json.Unmarshal([]byte(filteringOptionsJson), &filteringOptions) + if err != nil { + panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err)) + } + + return &filteringOptions +} + +func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { + for message := range inChannel { + sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions) + outChannel <- message + } +} + +func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) { + if connection == nil { + panic("Websocket connection is nil") + } + + if messageDataChannel == nil { + panic("Channel of captured messages is nil") + } + + for messageData := range messageDataChannel { + marshaledData, err := models.CreateWebsocketTappedEntryMessage(messageData) + if err != nil { + fmt.Printf("error converting message to json %s, (%v,%+v)\n", err, err, err) + continue + } + + err = connection.WriteMessage(websocket.TextMessage, marshaledData) + if err != nil { + fmt.Printf("error sending message through socket server %s, (%v,%+v)\n", err, err, err) + continue + } + } +} From fcf27e7c4d50ba661e6875540e1521db4d5d7041 Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Mon, 7 Jun 2021 15:19:12 +0300 Subject: [PATCH 22/36] String and not pointers (#68) --- api/pkg/api/main.go | 4 ++-- api/pkg/controllers/entries_controller.go | 10 ++++------ api/pkg/models/models.go | 6 +++--- api/pkg/resolver/resolver.go | 6 +++--- api/pkg/utils/utils.go | 6 +++--- 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/api/pkg/api/main.go b/api/pkg/api/main.go index 1aea81795..e5eb0b5c2 100644 --- a/api/pkg/api/main.go +++ b/api/pkg/api/main.go @@ -97,8 +97,8 @@ func saveHarToDb(entry *har.Entry, sender string) { serviceName, urlPath, serviceHostName := getServiceNameFromUrl(entry.Request.URL) entryId := primitive.NewObjectID().Hex() var ( - resolvedSource *string - resolvedDestination *string + resolvedSource string + resolvedDestination string ) if k8sResolver != nil { resolvedSource = k8sResolver.Resolve(sender) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index 70ec62714..293f02c29 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -90,12 +90,10 @@ func GetHARs(c *fiber.Ctx) error { harsObject := map[string]*models.ExtendedHAR{} for _, entryData := range entries { - harEntryObject := []byte(entryData.Entry) - var harEntry har.Entry - _ = json.Unmarshal(harEntryObject, &harEntry) + _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) - sourceOfEntry := *entryData.ResolvedSource + sourceOfEntry := entryData.ResolvedSource if harOfSource, ok := harsObject[sourceOfEntry]; ok { harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry) } else { @@ -137,8 +135,8 @@ func GetEntry(c *fiber.Ctx) error { unmarshallErr := json.Unmarshal([]byte(entryData.Entry), &fullEntry) utils.CheckErr(unmarshallErr) - if entryData.ResolvedDestination != nil { - fullEntry.Request.URL = utils.SetHostname(fullEntry.Request.URL, *entryData.ResolvedDestination) + if entryData.ResolvedDestination != "" { + fullEntry.Request.URL = utils.SetHostname(fullEntry.Request.URL, entryData.ResolvedDestination) } return c.Status(fiber.StatusOK).JSON(fullEntry) diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index 856d1aa4c..3ed03ebed 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -21,8 +21,8 @@ type MizuEntry struct { Service string `json:"service" gorm:"column:service"` Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` Path string `json:"path" gorm:"column:path"` - ResolvedSource *string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` - ResolvedDestination *string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` + ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` + ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` } type BaseEntryDetails struct { @@ -38,7 +38,7 @@ type BaseEntryDetails struct { type EntryData struct { Entry string `json:"entry,omitempty"` - ResolvedDestination *string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` + ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` } type EntriesFilter struct { diff --git a/api/pkg/resolver/resolver.go b/api/pkg/resolver/resolver.go index e45ce490a..14076a108 100644 --- a/api/pkg/resolver/resolver.go +++ b/api/pkg/resolver/resolver.go @@ -33,12 +33,12 @@ func (resolver *Resolver) Start(ctx context.Context) { } } -func (resolver *Resolver) Resolve(name string) *string { +func (resolver *Resolver) Resolve(name string) string { resolvedName, isFound := resolver.nameMap[name] if !isFound { - return nil + return "" } - return &resolvedName + return resolvedName } func (resolver *Resolver) watchPods(ctx context.Context) error { diff --git a/api/pkg/utils/utils.go b/api/pkg/utils/utils.go index ebf4ce9f1..9492c9b55 100644 --- a/api/pkg/utils/utils.go +++ b/api/pkg/utils/utils.go @@ -65,9 +65,9 @@ func SetHostname(address, newHostname string) string { func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails { entryUrl := entry.Url service := entry.Service - if entry.ResolvedDestination != nil { - entryUrl = SetHostname(entryUrl, *entry.ResolvedDestination) - service = SetHostname(service, *entry.ResolvedDestination) + if entry.ResolvedDestination != "" { + entryUrl = SetHostname(entryUrl, entry.ResolvedDestination) + service = SetHostname(service, entry.ResolvedDestination) } return models.BaseEntryDetails{ Id: entry.EntryId, From 31dcfc4b2e689dc6f532c8a0d719e7c955d7be5d Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Tue, 8 Jun 2021 11:17:02 +0300 Subject: [PATCH 23/36] TRA-3318 - Cookies not null and fix har file names (#69) * no message --- api/pkg/controllers/entries_controller.go | 5 +- .../messageSensitiveDataCleaner.go | 4 +- cli/cmd/tap.go | 2 + cli/cmd/tapRunner.go | 59 +++++++++++-------- cli/cmd/viewRunner.go | 2 +- cli/kubernetes/provider.go | 30 +++------- cli/mizu/consts.go | 5 +- 7 files changed, 54 insertions(+), 53 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index 293f02c29..06b96b5b8 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -94,12 +94,13 @@ func GetHARs(c *fiber.Ctx) error { _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) sourceOfEntry := entryData.ResolvedSource - if harOfSource, ok := harsObject[sourceOfEntry]; ok { + fileName := fmt.Sprintf("%s.har", sourceOfEntry) + if harOfSource, ok := harsObject[fileName]; ok { harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry) } else { var entriesHar []*har.Entry entriesHar = append(entriesHar, &harEntry) - harsObject[sourceOfEntry] = &models.ExtendedHAR{ + harsObject[fileName] = &models.ExtendedHAR{ Log: &models.ExtendedLog{ Version: "1.2", Creator: &models.ExtendedCreator{ diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index 7cf9660f9..f302660f2 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -18,8 +18,8 @@ func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, opt filterHarHeaders(harOutputItem.HarEntry.Request.Headers) filterHarHeaders(harOutputItem.HarEntry.Response.Headers) - harOutputItem.HarEntry.Request.Cookies = nil - harOutputItem.HarEntry.Response.Cookies = nil + harOutputItem.HarEntry.Request.Cookies = make([]har.Cookie, 0, 0) + harOutputItem.HarEntry.Response.Cookies = make([]har.Cookie, 0, 0) harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL) for i, queryString := range harOutputItem.HarEntry.Request.QueryString { diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 5c795d935..69c4b988a 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -12,6 +12,7 @@ import ( type MizuTapOptions struct { GuiPort uint16 Namespace string + AllNamespaces bool KubeConfigPath string MizuImage string MizuPodPort uint16 @@ -48,6 +49,7 @@ func init() { tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver") tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector") + tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces") tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod") diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 1fb81edb4..e01294e1d 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -31,7 +31,8 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { if err != nil { return } - kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath, tappingOptions.Namespace) + + kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath) defer cleanUpMizuResources(kubernetesProvider) ctx, cancel := context.WithCancel(context.Background()) @@ -43,7 +44,7 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { currentlyTappedPods = matchingPods } - nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(ctx, kubernetesProvider, currentlyTappedPods) + nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods) if err != nil { return } @@ -132,20 +133,20 @@ func createMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) { fmt.Printf("\nRemoving mizu resources\n") - removalCtx, _ := context.WithTimeout(context.Background(), 5 * time.Second) + removalCtx, _ := context.WithTimeout(context.Background(), 5*time.Second) if err := kubernetesProvider.RemovePod(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil { - fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err); + fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err) } if err := kubernetesProvider.RemoveService(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil { - fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err); + fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err) } if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil { - fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", mizu.TapperDaemonSetName, mizu.ResourcesNamespace, err, err, err); + fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", mizu.TapperDaemonSetName, mizu.ResourcesNamespace, err, err, err) } } func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions) { - added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, kubernetesProvider.Namespace), podRegex) + added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, getNamespace(tappingOptions, kubernetesProvider)), podRegex) restartTappers := func() { if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex); err != nil { @@ -155,7 +156,7 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro currentlyTappedPods = matchingPods } - nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(ctx, kubernetesProvider, currentlyTappedPods) + nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods) if err != nil { fmt.Printf("Error building node to ips map: %s (%v,%+v)\n", err, err, err) cancel() @@ -170,14 +171,14 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro for { select { - case newTarget := <- added: + case newTarget := <-added: fmt.Printf("+%s\n", newTarget.Name) - case removedTarget := <- removed: + case removedTarget := <-removed: fmt.Printf("-%s\n", removedTarget.Name) restartTappersDebouncer.SetOn() - case modifiedTarget := <- modified: + case modifiedTarget := <-modified: // Act only if the modified pod has already obtained an IP address. // After filtering for IPs, on a normal pod restart this includes the following events: // - Pod deletion @@ -188,11 +189,11 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro restartTappersDebouncer.SetOn() } - case <- errorChan: + case <-errorChan: // TODO: Does this also perform cleanup? cancel() - case <- ctx.Done(): + case <-ctx.Done(): return } } @@ -205,13 +206,13 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi var portForward *kubernetes.PortForward for { select { - case <- added: + case <-added: continue - case <- removed: + case <-removed: fmt.Printf("%s removed\n", mizu.AggregatorPodName) cancel() return - case modifiedPod := <- modified: + case modifiedPod := <-modified: if modifiedPod.Status.Phase == "Running" && !isPodReady { isPodReady = true var err error @@ -223,16 +224,16 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi } } - case <- time.After(25 * time.Second): + case <-time.After(25 * time.Second): if !isPodReady { fmt.Printf("error: %s pod was not ready in time", mizu.AggregatorPodName) cancel() } - case <- errorChan: + case <-errorChan: cancel() - case <- ctx.Done(): + case <-ctx.Done(): if portForward != nil { portForward.Stop() } @@ -261,12 +262,12 @@ func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.P return true } -func getNodeHostToTappedPodIpsMap(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappedPods []core.Pod) (map[string][]string, error) { +func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) (map[string][]string, error) { nodeToTappedPodIPMap := make(map[string][]string, 0) for _, pod := range tappedPods { existingList := nodeToTappedPodIPMap[pod.Spec.NodeName] if existingList == nil { - nodeToTappedPodIPMap[pod.Spec.NodeName] = []string {pod.Status.PodIP} + nodeToTappedPodIPMap[pod.Spec.NodeName] = []string{pod.Status.PodIP} } else { nodeToTappedPodIPMap[pod.Spec.NodeName] = append(nodeToTappedPodIPMap[pod.Spec.NodeName], pod.Status.PodIP) } @@ -280,9 +281,9 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) { // block until ctx cancel is called or termination signal is received select { - case <- ctx.Done(): + case <-ctx.Done(): break - case <- sigChan: + case <-sigChan: cancel() } } @@ -296,7 +297,7 @@ func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOption for { select { - case <- ctx.Done(): + case <-ctx.Done(): return default: err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods) @@ -308,3 +309,13 @@ func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOption } } + +func getNamespace(tappingOptions *MizuTapOptions, kubernetesProvider *kubernetes.Provider) string { + if tappingOptions.AllNamespaces { + return mizu.K8sAllNamespaces + } else if len(tappingOptions.Namespace) > 0 { + return tappingOptions.Namespace + } else { + return kubernetesProvider.CurrentNamespace() + } +} diff --git a/cli/cmd/viewRunner.go b/cli/cmd/viewRunner.go index 18e2bf9e0..059a9b2e2 100644 --- a/cli/cmd/viewRunner.go +++ b/cli/cmd/viewRunner.go @@ -9,7 +9,7 @@ import ( ) func runMizuView() { - kubernetesProvider := kubernetes.NewProvider("", "") + kubernetesProvider := kubernetes.NewProvider("") ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 58d96b18c..f7d7e68f7 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/up9inc/mizu/cli/mizu" "path/filepath" "regexp" @@ -42,7 +43,7 @@ const ( fieldManagerName = "mizu-manager" ) -func NewProvider(kubeConfigPath string, overrideNamespace string) *Provider { +func NewProvider(kubeConfigPath string) *Provider { kubernetesConfig := loadKubernetesConfiguration(kubeConfigPath) restClientConfig, err := kubernetesConfig.ClientConfig() if err != nil { @@ -50,25 +51,18 @@ func NewProvider(kubeConfigPath string, overrideNamespace string) *Provider { } clientSet := getClientSet(restClientConfig) - var namespace string - if len(overrideNamespace) > 0 { - namespace = overrideNamespace - } else { - configuredNamespace, _, err := kubernetesConfig.Namespace() - if err != nil { - panic(err) - } - namespace = configuredNamespace - } - return &Provider{ clientSet: clientSet, kubernetesConfig: kubernetesConfig, clientConfig: *restClientConfig, - Namespace: namespace, } } +func (provider *Provider) CurrentNamespace() string { + ns, _, _ := provider.kubernetesConfig.Namespace() + return ns +} + func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) watch.Interface { watcher, err := provider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true}) if err != nil { @@ -77,14 +71,6 @@ func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) w return watcher } -func (provider *Provider) GetPods(ctx context.Context, namespace string) { - pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - panic(err.Error()) - } - fmt.Printf("There are %d pods in Namespace %s\n", len(pods.Items), namespace) -} - func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.TrafficFilteringOptions) (*core.Pod, error) { marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions) if err != nil { @@ -316,7 +302,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac } func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp) ([]core.Pod, error) { - pods, err := provider.clientSet.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) + pods, err := provider.clientSet.CoreV1().Pods(mizu.K8sAllNamespaces).List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } diff --git a/cli/mizu/consts.go b/cli/mizu/consts.go index 22532d0e6..59d33f1b0 100644 --- a/cli/mizu/consts.go +++ b/cli/mizu/consts.go @@ -1,8 +1,8 @@ package mizu var ( - Version = "v0.0.1" - Branch = "develop" + Version = "v0.0.1" + Branch = "develop" GitCommitHash = "" // this var is overridden using ldflags in makefile when building ) @@ -11,4 +11,5 @@ const ( TapperDaemonSetName = "mizu-tapper-daemon-set" AggregatorPodName = "mizu-collector" TapperPodName = "mizu-tapper" + K8sAllNamespaces = "" ) From 135b1a5e1e89e83ab088ad7b6df82b4efeb581e8 Mon Sep 17 00:00:00 2001 From: nimrod-up9 <59927337+nimrod-up9@users.noreply.github.com> Date: Mon, 14 Jun 2021 13:22:44 +0300 Subject: [PATCH 24/36] TRA-3212 Passive-Tapper and Mizu share code (#70) * Use log in tap package instead of fmt. * Moved api/pkg/tap to root. * Added go.mod and go.sum for tap. * Added replace for shared. * api uses tap module instead of tap package. * Removed dependency of tap in shared by moving env var out of tap. * Fixed compilation bugs. * Fixed: Forgot to export struct field HostMode. * Removed unused flag. * Close har output channel when done. * Moved websocket out of mizu and into passive-tapper. * Send connection details over har output channel. * Fixed compilation errors. * Removed unused info from request response cache. * Renamed connection -> connectionID. * Fixed rename bug. * Export setters and getters for filter ips and ports. * Added tap dependency to Dockerfile. * Uncomment error messages. * Renamed `filterIpAddresses` -> `filterAuthorities`. * Renamed ConnectionID -> ConnectionInfo. * Fixed: Missed one replace. --- Dockerfile | 2 + api/go.mod | 2 + api/main.go | 20 +- api/pkg/api/main.go | 12 +- api/pkg/api/socket_server_handlers.go | 2 +- api/pkg/models/models.go | 4 +- .../messageSensitiveDataCleaner.go | 2 +- api/pkg/tap/http_matcher.go | 209 --------------- api/pkg/tap/tap_output.go | 239 ------------------ {api/pkg/tap => tap}/cleaner.go | 0 tap/go.mod | 12 + tap/go.sum | 31 +++ {api/pkg/tap => tap}/grpc_assembler.go | 10 +- {api/pkg/tap => tap}/har_writer.go | 45 ++-- tap/http_matcher.go | 138 ++++++++++ {api/pkg/tap => tap}/http_reader.go | 126 +++------ {api/pkg/tap => tap}/net_utils.go | 0 tap/outboundlinks.go | 29 +++ {api/pkg/tap => tap}/passive_tapper.go | 185 ++++++-------- tap/settings.go | 31 +++ {api/pkg/tap => tap}/stats_tracker.go | 0 {api/pkg/tap => tap}/tcp_stream.go | 24 +- {api/pkg/tap => tap}/tcp_stream_factory.go | 19 +- 23 files changed, 442 insertions(+), 700 deletions(-) delete mode 100644 api/pkg/tap/http_matcher.go delete mode 100644 api/pkg/tap/tap_output.go rename {api/pkg/tap => tap}/cleaner.go (100%) create mode 100644 tap/go.mod create mode 100644 tap/go.sum rename {api/pkg/tap => tap}/grpc_assembler.go (96%) rename {api/pkg/tap => tap}/har_writer.go (82%) create mode 100644 tap/http_matcher.go rename {api/pkg/tap => tap}/http_reader.go (60%) rename {api/pkg/tap => tap}/net_utils.go (100%) create mode 100644 tap/outboundlinks.go rename {api/pkg/tap => tap}/passive_tapper.go (70%) create mode 100644 tap/settings.go rename {api/pkg/tap => tap}/stats_tracker.go (100%) rename {api/pkg/tap => tap}/tcp_stream.go (83%) rename {api/pkg/tap => tap}/tcp_stream_factory.go (82%) diff --git a/Dockerfile b/Dockerfile index e85745689..157588ec9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,12 +18,14 @@ WORKDIR /app/api-build COPY api/go.mod api/go.sum ./ COPY shared/go.mod shared/go.mod ../shared/ +COPY tap/go.mod tap/go.mod ../tap/ RUN go mod download # cheap trick to make the build faster (As long as go.mod wasn't changes) RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get # Copy and build api code COPY shared ../shared +COPY tap ../tap COPY api . RUN go build -ldflags="-s -w" -o mizuagent . diff --git a/api/go.mod b/api/go.mod index dcb1edf86..f75ea5920 100644 --- a/api/go.mod +++ b/api/go.mod @@ -18,6 +18,7 @@ require ( github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/up9inc/mizu/shared v0.0.0 + github.com/up9inc/mizu/tap v0.0.0 go.mongodb.org/mongo-driver v1.5.1 golang.org/x/net v0.0.0-20210421230115-4e50805a0758 gorm.io/driver/sqlite v1.1.4 @@ -28,3 +29,4 @@ require ( ) replace github.com/up9inc/mizu/shared v0.0.0 => ../shared +replace github.com/up9inc/mizu/tap v0.0.0 => ../tap diff --git a/api/main.go b/api/main.go index b1bd17a6c..b7bbaa14d 100644 --- a/api/main.go +++ b/api/main.go @@ -7,12 +7,12 @@ import ( "github.com/gofiber/fiber/v2" "github.com/gorilla/websocket" "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/tap" "mizuserver/pkg/api" "mizuserver/pkg/middleware" "mizuserver/pkg/models" "mizuserver/pkg/routes" "mizuserver/pkg/sensitiveDataFiltering" - "mizuserver/pkg/tap" "mizuserver/pkg/utils" "os" "os/signal" @@ -26,16 +26,21 @@ var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu c func main() { flag.Parse() + hostMode := os.Getenv(shared.HostModeEnvVar) == "1" + tapOpts := &tap.TapOpts{HostMode: hostMode} if !*shouldTap && !*aggregator && !*standalone{ panic("One of the flags --tap, --api or --standalone must be provided") } if *standalone { - harOutputChannel := tap.StartPassiveTapper() + harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts) filteredHarChannel := make(chan *tap.OutputChannelItem) + go filterHarHeaders(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions()) go api.StartReadingEntries(filteredHarChannel, nil) + go api.StartReadingOutbound(outboundLinkOutputChannel) + hostApi(nil) } else if *shouldTap { if *aggregatorAddress == "" { @@ -44,21 +49,26 @@ func main() { tapTargets := getTapTargets() if tapTargets != nil { - tap.HostAppAddresses = tapTargets - fmt.Println("Filtering for the following addresses:", tap.HostAppAddresses) + tap.SetFilterAuthorities(tapTargets) + fmt.Println("Filtering for the following authorities:", tap.GetFilterIPs()) } - harOutputChannel := tap.StartPassiveTapper() + harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts) + socketConnection, err := shared.ConnectToSocketServer(*aggregatorAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false) if err != nil { panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err)) } + go pipeChannelToSocket(socketConnection, harOutputChannel) + go api.StartReadingOutbound(outboundLinkOutputChannel) } else if *aggregator { socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) filteredHarChannel := make(chan *tap.OutputChannelItem) + go api.StartReadingEntries(filteredHarChannel, nil) go filterHarHeaders(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) + hostApi(socketHarOutChannel) } diff --git a/api/pkg/api/main.go b/api/pkg/api/main.go index e5eb0b5c2..2a1e550d4 100644 --- a/api/pkg/api/main.go +++ b/api/pkg/api/main.go @@ -6,11 +6,11 @@ import ( "encoding/json" "fmt" "github.com/google/martian/har" + "github.com/up9inc/mizu/tap" "go.mongodb.org/mongo-driver/bson/primitive" "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/resolver" - "mizuserver/pkg/tap" "mizuserver/pkg/utils" "net/url" "os" @@ -88,10 +88,18 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) { } for item := range outputItems { - saveHarToDb(item.HarEntry, item.RequestSenderIp) + saveHarToDb(item.HarEntry, item.ConnectionInfo.ClientIP) } } +func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) { + // tcpStreamFactory will block on write to channel. Empty channel to unblock. + // TODO: Make write to channel optional. + for range outboundLinkChannel { + } +} + + func saveHarToDb(entry *har.Entry, sender string) { entryBytes, _ := json.Marshal(entry) serviceName, urlPath, serviceHostName := getServiceNameFromUrl(entry.Request.URL) diff --git a/api/pkg/api/socket_server_handlers.go b/api/pkg/api/socket_server_handlers.go index 12bf42adf..8164df004 100644 --- a/api/pkg/api/socket_server_handlers.go +++ b/api/pkg/api/socket_server_handlers.go @@ -5,10 +5,10 @@ import ( "fmt" "github.com/antoniodipinto/ikisocket" "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/tap" "mizuserver/pkg/controllers" "mizuserver/pkg/models" "mizuserver/pkg/routes" - "mizuserver/pkg/tap" ) var browserClientSocketUUIDs = make([]string, 0) diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index 3ed03ebed..d8dc90184 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -4,7 +4,7 @@ import ( "encoding/json" "github.com/google/martian/har" "github.com/up9inc/mizu/shared" - "mizuserver/pkg/tap" + "github.com/up9inc/mizu/tap" "time" ) @@ -101,4 +101,4 @@ type ExtendedLog struct { type ExtendedCreator struct { *har.Creator Source string `json:"_source"` -} \ No newline at end of file +} diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index f302660f2..15a8e5d8b 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -5,7 +5,7 @@ import ( "encoding/xml" "errors" "fmt" - "mizuserver/pkg/tap" + "github.com/up9inc/mizu/tap" "net/url" "strings" diff --git a/api/pkg/tap/http_matcher.go b/api/pkg/tap/http_matcher.go deleted file mode 100644 index c0a98fb1f..000000000 --- a/api/pkg/tap/http_matcher.go +++ /dev/null @@ -1,209 +0,0 @@ -package tap - -import ( - "fmt" - "net/http" - "strconv" - "strings" - "time" - - "github.com/orcaman/concurrent-map" -) - -type requestResponsePair struct { - Request httpMessage `json:"request"` - Response httpMessage `json:"response"` -} - -type envoyMessageWrapper struct { - HttpBufferedTrace requestResponsePair `json:"http_buffered_trace"` -} - -type headerKeyVal struct { - Key string `json:"key"` - Value string `json:"value"` -} - -type messageBody struct { - Truncated bool `json:"truncated"` - AsBytes string `json:"as_bytes"` -} - -type httpMessage struct { - IsRequest bool - Headers []headerKeyVal `json:"headers"` - HTTPVersion string `json:"httpVersion"` - Body messageBody `json:"body"` - captureTime time.Time - orig interface {} - requestSenderIp string -} - - -// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port} -type requestResponseMatcher struct { - openMessagesMap cmap.ConcurrentMap - -} - -func createResponseRequestMatcher() requestResponseMatcher { - newMatcher := &requestResponseMatcher{openMessagesMap: cmap.New()} - return *newMatcher -} - -func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time, body string, isHTTP2 bool) *envoyMessageWrapper { - split := splitIdent(ident) - key := genKey(split) - - messageExtraHeaders := []headerKeyVal{ - {Key: "x-up9-source", Value: split[0]}, - {Key: "x-up9-destination", Value: split[1] + ":" + split[3]}, - } - - requestHTTPMessage := requestToMessage(request, captureTime, body, &messageExtraHeaders, isHTTP2, split[0]) - - if response, found := matcher.openMessagesMap.Pop(key); found { - // Type assertion always succeeds because all of the map's values are of httpMessage type - responseHTTPMessage := response.(*httpMessage) - if responseHTTPMessage.IsRequest { - SilentError("Request-Duplicate", "Got duplicate request with same identifier\n") - return nil - } - Debug("Matched open Response for %s\n", key) - return matcher.preparePair(&requestHTTPMessage, responseHTTPMessage) - } - - matcher.openMessagesMap.Set(key, &requestHTTPMessage) - Debug("Registered open Request for %s\n", key) - return nil -} - -func (matcher *requestResponseMatcher) registerResponse(ident string, response *http.Response, captureTime time.Time, body string, isHTTP2 bool) *envoyMessageWrapper { - split := splitIdent(ident) - key := genKey(split) - - responseHTTPMessage := responseToMessage(response, captureTime, body, isHTTP2) - - if request, found := matcher.openMessagesMap.Pop(key); found { - // Type assertion always succeeds because all of the map's values are of httpMessage type - requestHTTPMessage := request.(*httpMessage) - if !requestHTTPMessage.IsRequest { - SilentError("Response-Duplicate", "Got duplicate response with same identifier\n") - return nil - } - Debug("Matched open Request for %s\n", key) - return matcher.preparePair(requestHTTPMessage, &responseHTTPMessage) - } - - matcher.openMessagesMap.Set(key, &responseHTTPMessage) - Debug("Registered open Response for %s\n", key) - return nil -} - -func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *httpMessage, responseHTTPMessage *httpMessage) *envoyMessageWrapper { - matcher.addDuration(requestHTTPMessage, responseHTTPMessage) - - return &envoyMessageWrapper{ - HttpBufferedTrace: requestResponsePair{ - Request: *requestHTTPMessage, - Response: *responseHTTPMessage, - }, - } -} - -func requestToMessage(request *http.Request, captureTime time.Time, body string, messageExtraHeaders *[]headerKeyVal, isHTTP2 bool, requestSenderIp string) httpMessage { - messageHeaders := make([]headerKeyVal, 0) - - for key, value := range request.Header { - messageHeaders = append(messageHeaders, headerKeyVal{Key: key, Value: value[0]}) - } - - if !isHTTP2 { - messageHeaders = append(messageHeaders, headerKeyVal{Key: ":method", Value: request.Method}) - messageHeaders = append(messageHeaders, headerKeyVal{Key: ":path", Value: request.RequestURI}) - messageHeaders = append(messageHeaders, headerKeyVal{Key: ":authority", Value: request.Host}) - messageHeaders = append(messageHeaders, headerKeyVal{Key: ":scheme", Value: "http"}) - } - - messageHeaders = append(messageHeaders, headerKeyVal{Key: "x-request-start", Value: fmt.Sprintf("%.3f", float64(captureTime.UnixNano()) / float64(1000000000))}) - - messageHeaders = append(messageHeaders, *messageExtraHeaders...) - - httpVersion := request.Proto - - requestBody := messageBody{Truncated: false, AsBytes: body} - - return httpMessage{ - IsRequest: true, - Headers: messageHeaders, - HTTPVersion: httpVersion, - Body: requestBody, - captureTime: captureTime, - orig: request, - requestSenderIp: requestSenderIp, - } -} - -func responseToMessage(response *http.Response, captureTime time.Time, body string, isHTTP2 bool) httpMessage { - messageHeaders := make([]headerKeyVal, 0) - - for key, value := range response.Header { - messageHeaders = append(messageHeaders, headerKeyVal{Key: key, Value: value[0]}) - } - - if !isHTTP2 { - messageHeaders = append(messageHeaders, headerKeyVal{Key: ":status", Value: strconv.Itoa(response.StatusCode)}) - } - - httpVersion := response.Proto - - requestBody := messageBody{Truncated: false, AsBytes: body} - - return httpMessage{ - IsRequest: false, - Headers: messageHeaders, - HTTPVersion: httpVersion, - Body: requestBody, - captureTime: captureTime, - orig: response, - } -} - -func (matcher *requestResponseMatcher) addDuration(requestHTTPMessage *httpMessage, responseHTTPMessage *httpMessage) { - durationMs := float64(responseHTTPMessage.captureTime.UnixNano() / 1000000) - float64(requestHTTPMessage.captureTime.UnixNano() / 1000000) - if durationMs < 1 { - durationMs = 1 - } - - responseHTTPMessage.Headers = append(responseHTTPMessage.Headers, headerKeyVal{Key: "x-up9-duration-ms", Value: fmt.Sprintf("%.0f", durationMs)}) -} - -func splitIdent(ident string) []string { - ident = strings.Replace(ident, "->", " ", -1) - return strings.Split(ident, " ") -} - -func genKey(split []string) string { - key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4]) - return key -} - -func (matcher *requestResponseMatcher) deleteOlderThan(t time.Time) int { - keysToPop := make([]string, 0) - for item := range matcher.openMessagesMap.IterBuffered() { - // Map only contains values of type httpMessage - message, _ := item.Val.(*httpMessage) - - if message.captureTime.Before(t) { - keysToPop = append(keysToPop, item.Key) - } - } - - numDeleted := len(keysToPop) - - for _, key := range keysToPop { - _, _ = matcher.openMessagesMap.Pop(key) - } - - return numDeleted -} diff --git a/api/pkg/tap/tap_output.go b/api/pkg/tap/tap_output.go deleted file mode 100644 index 561c445c0..000000000 --- a/api/pkg/tap/tap_output.go +++ /dev/null @@ -1,239 +0,0 @@ -package tap - -import ( - "bytes" - "encoding/json" - "fmt" - "log" - "net/http" - "time" - - "github.com/gorilla/websocket" - "github.com/patrickmn/go-cache" -) - - -const ( - // Time allowed to write a message to the peer. - writeWait = 10 * time.Second - - // Time allowed to read the next pong message from the peer. - pongWait = 60 * time.Second - - // Send pings to peer with this period. Must be less than pongWait. - pingPeriod = (pongWait * 9) / 10 - - // Maximum message size allowed from peer. - maxMessageSize = 512 -) - -var ( - newline = []byte{'\n'} - space = []byte{' '} - hub *Hub - outboundSocketNotifyExpiringCache = cache.New(outboundThrottleCacheExpiryPeriod, outboundThrottleCacheExpiryPeriod) -) - -var upgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, - CheckOrigin: func (_ *http.Request) bool { return true }, -} - -// Client is a middleman between the websocket connection and the hub. -type Client struct { - hub *Hub - - // The websocket connection. - conn *websocket.Conn - - // Buffered channel of outbound messages. - send chan []byte -} - -type OutBoundLinkMessage struct { - SourceIP string `json:"sourceIP"` - IP string `json:"ip"` - Port int `json:"port"` - Type string `json:"type"` -} - - -// readPump pumps messages from the websocket connection to the hub. -// -// The application runs readPump in a per-connection goroutine. The application -// ensures that there is at most one reader on a connection by executing all -// reads from this goroutine. -func (c *Client) readPump() { - defer func() { - c.hub.unregister <- c - c.conn.Close() - }() - c.conn.SetReadLimit(maxMessageSize) - c.conn.SetReadDeadline(time.Now().Add(pongWait)) - c.conn.SetPongHandler(func(string) error { c.conn.SetReadDeadline(time.Now().Add(pongWait)); return nil }) - for { - _, message, err := c.conn.ReadMessage() - if err != nil { - if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) { - log.Printf("error: %v", err) - } - break - } - message = bytes.TrimSpace(bytes.Replace(message, newline, space, -1)) - c.hub.onMessageCallback(message) - } -} - -// writePump pumps messages from the hub to the websocket connection. -// -// A goroutine running writePump is started for each connection. The -// application ensures that there is at most one writer to a connection by -// executing all writes from this goroutine. -func (c *Client) writePump() { - ticker := time.NewTicker(pingPeriod) - defer func() { - ticker.Stop() - c.conn.Close() - }() - for { - select { - case message, ok := <-c.send: - c.conn.SetWriteDeadline(time.Now().Add(writeWait)) - if !ok { - // The hub closed the channel. - c.conn.WriteMessage(websocket.CloseMessage, []byte{}) - return - } - - w, err := c.conn.NextWriter(websocket.TextMessage) - if err != nil { - return - } - w.Write(message) - - - if err := w.Close(); err != nil { - return - } - case <-ticker.C: - c.conn.SetWriteDeadline(time.Now().Add(writeWait)) - if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil { - return - } - } - } -} - -type Hub struct { - // Registered clients. - clients map[*Client]bool - - // Inbound messages from the clients. - broadcast chan []byte - - // Register requests from the clients. - register chan *Client - - // Unregister requests from clients. - unregister chan *Client - - // Handle messages from client - onMessageCallback func([]byte) - - -} - -func newHub(onMessageCallback func([]byte)) *Hub { - return &Hub{ - broadcast: make(chan []byte), - register: make(chan *Client), - unregister: make(chan *Client), - clients: make(map[*Client]bool), - onMessageCallback: onMessageCallback, - } -} - -func (h *Hub) run() { - for { - select { - case client := <-h.register: - h.clients[client] = true - case client := <-h.unregister: - if _, ok := h.clients[client]; ok { - delete(h.clients, client) - close(client.send) - } - case message := <-h.broadcast: - // matched messages counter is incremented in this thread instead of in multiple http reader - // threads in order to reduce contention. - statsTracker.incMatchedMessages() - - for client := range h.clients { - select { - case client.send <- message: - default: - close(client.send) - delete(h.clients, client) - } - } - } - } -} - - -// serveWs handles websocket requests from the peer. -func serveWs(hub *Hub, w http.ResponseWriter, r *http.Request) { - conn, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Println(err) - return - } - client := &Client{hub: hub, conn: conn, send: make(chan []byte, 256)} - client.hub.register <- client - - // Allow collection of memory referenced by the caller by doing all work in - // new goroutines. - go client.writePump() - go client.readPump() -} - -func startOutputServer(port string, messageCallback func([]byte)) { - hub = newHub(messageCallback) - go hub.run() - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - serveWs(hub, w, r) - }) - err := http.ListenAndServe("0.0.0.0:" + port, nil) - if err != nil { - log.Fatal("Output server error: ", err) - } -} - -func broadcastReqResPair(reqResJson []byte) { - hub.broadcast <- reqResJson -} - -func broadcastOutboundLink(srcIP string, dstIP string, dstPort int) { - cacheKey := fmt.Sprintf("%s -> %s:%d", srcIP, dstIP, dstPort) - _, isInCache := outboundSocketNotifyExpiringCache.Get(cacheKey) - if isInCache { - return - } else { - outboundSocketNotifyExpiringCache.SetDefault(cacheKey, true) - } - - socketMessage := OutBoundLinkMessage{ - SourceIP: srcIP, - IP: dstIP, - Port: dstPort, - Type: "outboundSocketDetected", - } - - jsonStr, err := json.Marshal(socketMessage) - if err != nil { - log.Printf("error marshalling outbound socket detection object: %v", err) - } else { - hub.broadcast <- jsonStr - } -} diff --git a/api/pkg/tap/cleaner.go b/tap/cleaner.go similarity index 100% rename from api/pkg/tap/cleaner.go rename to tap/cleaner.go diff --git a/tap/go.mod b/tap/go.mod new file mode 100644 index 000000000..d8c38a22c --- /dev/null +++ b/tap/go.mod @@ -0,0 +1,12 @@ +module github.com/up9inc/mizu/tap + +go 1.16 + +require ( + github.com/google/gopacket v1.1.19 + github.com/google/martian v2.1.0+incompatible + github.com/gorilla/websocket v1.4.2 + github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 + github.com/patrickmn/go-cache v2.1.0+incompatible + golang.org/x/net v0.0.0-20210421230115-4e50805a0758 +) diff --git a/tap/go.sum b/tap/go.sum new file mode 100644 index 000000000..a110e49b4 --- /dev/null +++ b/tap/go.sum @@ -0,0 +1,31 @@ +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc h1:Ak86L+yDSOzKFa7WM5bf5itSOo1e3Xh8bm5YCMUXIjQ= +github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/api/pkg/tap/grpc_assembler.go b/tap/grpc_assembler.go similarity index 96% rename from api/pkg/tap/grpc_assembler.go rename to tap/grpc_assembler.go index 076faf7cd..72b5665f1 100644 --- a/api/pkg/tap/grpc_assembler.go +++ b/tap/grpc_assembler.go @@ -84,14 +84,14 @@ type GrpcAssembler struct { framer *http2.Framer } -func (ga *GrpcAssembler) readMessage() (uint32, interface{}, string, error) { +func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) { // Exactly one Framer is used for each half connection. // (Instead of creating a new Framer for each ReadFrame operation) // This is needed in order to decompress the headers, // because the compression context is updated with each requests/response. frame, err := ga.framer.ReadFrame() if err != nil { - return 0, nil, "", err + return 0, nil, err } streamID := frame.Header().StreamID @@ -99,7 +99,7 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, string, error) { ga.fragmentsByStream.appendFrame(streamID, frame) if !(ga.isStreamEnd(frame)) { - return 0, nil, "", nil + return 0, nil, nil } headers, data := ga.fragmentsByStream.pop(streamID) @@ -137,10 +137,10 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, string, error) { ContentLength: int64(len(dataString)), } } else { - return 0, nil, "", errors.New("Failed to assemble stream: neither a request nor a message") + return 0, nil, errors.New("Failed to assemble stream: neither a request nor a message") } - return streamID, messageHTTP1, dataString, nil + return streamID, messageHTTP1, nil } func (ga *GrpcAssembler) isStreamEnd(frame http2.Frame) bool { diff --git a/api/pkg/tap/har_writer.go b/tap/har_writer.go similarity index 82% rename from api/pkg/tap/har_writer.go rename to tap/har_writer.go index 1b1eb1096..1bd2b5ee5 100644 --- a/api/pkg/tap/har_writer.go +++ b/tap/har_writer.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "log" "net/http" "os" "path/filepath" @@ -23,12 +24,13 @@ type PairChanItem struct { Response *http.Response ResponseTime time.Time RequestSenderIp string + ConnectionInfo *ConnectionInfo } func openNewHarFile(filename string) *HarFile { file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, readPermission) if err != nil { - panic(fmt.Sprintf("Failed to open output file: %s (%v,%+v)", err, err, err)) + log.Panicf("Failed to open output file: %s (%v,%+v)", err, err, err) } harFile := HarFile{file: file, entryCount: 0} @@ -45,13 +47,13 @@ type HarFile struct { func NewEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) (*har.Entry, error) { harRequest, err := har.NewRequest(request, true) if err != nil { - SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)\n", err, err, err) + SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err) return nil, errors.New("Failed converting request to HAR") } harResponse, err := har.NewResponse(response, true) if err != nil { - SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)\n", err, err, err) + SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err) return nil, errors.New("Failed converting response to HAR") } @@ -62,7 +64,7 @@ func NewEntry(request *http.Request, requestTime time.Time, response *http.Respo status, err := strconv.Atoi(response.Header.Get(":status")) if err != nil { - SilentError("convert-response-status-for-har", "Failed converting status to int %s (%v,%+v)\n", err, err, err) + SilentError("convert-response-status-for-har", "Failed converting status to int %s (%v,%+v)", err, err, err) return nil, errors.New("Failed converting response status to int for HAR") } harResponse.Status = status @@ -102,7 +104,7 @@ func NewEntry(request *http.Request, requestTime time.Time, response *http.Respo func (f *HarFile) WriteEntry(harEntry *har.Entry) { harEntryJson, err := json.Marshal(harEntry) if err != nil { - SilentError("har-entry-marshal", "Failed converting har entry object to JSON%s (%v,%+v)\n", err, err, err) + SilentError("har-entry-marshal", "Failed converting har entry object to JSON%s (%v,%+v)", err, err, err) return } @@ -116,7 +118,7 @@ func (f *HarFile) WriteEntry(harEntry *har.Entry) { harEntryString := append([]byte(separator), harEntryJson...) if _, err := f.file.Write(harEntryString); err != nil { - panic(fmt.Sprintf("Failed to write to output file: %s (%v,%+v)", err, err, err)) + log.Panicf("Failed to write to output file: %s (%v,%+v)", err, err, err) } f.entryCount++ @@ -131,21 +133,21 @@ func (f *HarFile) Close() { err := f.file.Close() if err != nil { - panic(fmt.Sprintf("Failed to close output file: %s (%v,%+v)", err, err, err)) + log.Panicf("Failed to close output file: %s (%v,%+v)", err, err, err) } } func (f*HarFile) writeHeader() { header := []byte(`{"log": {"version": "1.2", "creator": {"name": "Mizu", "version": "0.0.1"}, "entries": [`) if _, err := f.file.Write(header); err != nil { - panic(fmt.Sprintf("Failed to write header to output file: %s (%v,%+v)", err, err, err)) + log.Panicf("Failed to write header to output file: %s (%v,%+v)", err, err, err) } } func (f*HarFile) writeTrailer() { trailer := []byte("]}}") if _, err := f.file.Write(trailer); err != nil { - panic(fmt.Sprintf("Failed to write trailer to output file: %s (%v,%+v)", err, err, err)) + log.Panicf("Failed to write trailer to output file: %s (%v,%+v)", err, err, err) } } @@ -161,8 +163,8 @@ func NewHarWriter(outputDir string, maxEntries int) *HarWriter { } type OutputChannelItem struct { - HarEntry *har.Entry - RequestSenderIp string + HarEntry *har.Entry + ConnectionInfo *ConnectionInfo } type HarWriter struct { @@ -174,20 +176,20 @@ type HarWriter struct { done chan bool } -func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time, requestSenderIp string) { +func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time, connectionInfo *ConnectionInfo) { hw.PairChan <- &PairChanItem{ - Request: request, - RequestTime: requestTime, - Response: response, - ResponseTime: responseTime, - RequestSenderIp: requestSenderIp, + Request: request, + RequestTime: requestTime, + Response: response, + ResponseTime: responseTime, + ConnectionInfo: connectionInfo, } } func (hw *HarWriter) Start() { if hw.OutputDirPath != "" { if err := os.MkdirAll(hw.OutputDirPath, os.ModePerm); err != nil { - panic(fmt.Sprintf("Failed to create output directory: %s (%v,%+v)", err, err, err)) + log.Panicf("Failed to create output directory: %s (%v,%+v)", err, err, err) } } @@ -210,8 +212,8 @@ func (hw *HarWriter) Start() { } } else { hw.OutChan <- &OutputChannelItem{ - HarEntry: harEntry, - RequestSenderIp: pair.RequestSenderIp, + HarEntry: harEntry, + ConnectionInfo: pair.ConnectionInfo, } } } @@ -226,6 +228,7 @@ func (hw *HarWriter) Start() { func (hw *HarWriter) Stop() { close(hw.PairChan) <-hw.done + close(hw.OutChan) } func (hw *HarWriter) openNewFile() { @@ -241,7 +244,7 @@ func (hw *HarWriter) closeFile() { filename := buildFilename(hw.OutputDirPath, time.Now()) err := os.Rename(tmpFilename, filename) if err != nil { - SilentError("Rename-file", "cannot rename file: %s (%v,%+v)\n", err, err, err) + SilentError("Rename-file", "cannot rename file: %s (%v,%+v)", err, err, err) } } diff --git a/tap/http_matcher.go b/tap/http_matcher.go new file mode 100644 index 000000000..a0377cd46 --- /dev/null +++ b/tap/http_matcher.go @@ -0,0 +1,138 @@ +package tap + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/orcaman/concurrent-map" +) + +type requestResponsePair struct { + Request httpMessage `json:"request"` + Response httpMessage `json:"response"` +} + +type ConnectionInfo struct { + ClientIP string + ClientPort string + ServerIP string + ServerPort string +} + +type httpMessage struct { + isRequest bool + captureTime time.Time + orig interface{} + connectionInfo ConnectionInfo +} + + +// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port} +type requestResponseMatcher struct { + openMessagesMap cmap.ConcurrentMap + +} + +func createResponseRequestMatcher() requestResponseMatcher { + newMatcher := &requestResponseMatcher{openMessagesMap: cmap.New()} + return *newMatcher +} + +func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time) *requestResponsePair { + split := splitIdent(ident) + key := genKey(split) + + connectionInfo := &ConnectionInfo{ + ClientIP: split[0], + ClientPort: split[2], + ServerIP: split[1], + ServerPort: split[3], + } + + requestHTTPMessage := httpMessage{ + isRequest: true, + captureTime: captureTime, + orig: request, + connectionInfo: *connectionInfo, + } + + if response, found := matcher.openMessagesMap.Pop(key); found { + // Type assertion always succeeds because all of the map's values are of httpMessage type + responseHTTPMessage := response.(*httpMessage) + if responseHTTPMessage.isRequest { + SilentError("Request-Duplicate", "Got duplicate request with same identifier") + return nil + } + Debug("Matched open Response for %s", key) + return matcher.preparePair(&requestHTTPMessage, responseHTTPMessage) + } + + matcher.openMessagesMap.Set(key, &requestHTTPMessage) + Debug("Registered open Request for %s", key) + return nil +} + +func (matcher *requestResponseMatcher) registerResponse(ident string, response *http.Response, captureTime time.Time) *requestResponsePair { + split := splitIdent(ident) + key := genKey(split) + + responseHTTPMessage := httpMessage{ + isRequest: false, + captureTime: captureTime, + orig: response, + } + + if request, found := matcher.openMessagesMap.Pop(key); found { + // Type assertion always succeeds because all of the map's values are of httpMessage type + requestHTTPMessage := request.(*httpMessage) + if !requestHTTPMessage.isRequest { + SilentError("Response-Duplicate", "Got duplicate response with same identifier") + return nil + } + Debug("Matched open Request for %s", key) + return matcher.preparePair(requestHTTPMessage, &responseHTTPMessage) + } + + matcher.openMessagesMap.Set(key, &responseHTTPMessage) + Debug("Registered open Response for %s", key) + return nil +} + +func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *httpMessage, responseHTTPMessage *httpMessage) *requestResponsePair { + return &requestResponsePair{ + Request: *requestHTTPMessage, + Response: *responseHTTPMessage, + } +} + +func splitIdent(ident string) []string { + ident = strings.Replace(ident, "->", " ", -1) + return strings.Split(ident, " ") +} + +func genKey(split []string) string { + key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4]) + return key +} + +func (matcher *requestResponseMatcher) deleteOlderThan(t time.Time) int { + keysToPop := make([]string, 0) + for item := range matcher.openMessagesMap.IterBuffered() { + // Map only contains values of type httpMessage + message, _ := item.Val.(*httpMessage) + + if message.captureTime.Before(t) { + keysToPop = append(keysToPop, item.Key) + } + } + + numDeleted := len(keysToPop) + + for _, key := range keysToPop { + _, _ = matcher.openMessagesMap.Pop(key) + } + + return numDeleted +} diff --git a/api/pkg/tap/http_reader.go b/tap/http_reader.go similarity index 60% rename from api/pkg/tap/http_reader.go rename to tap/http_reader.go index 21ce9c788..00d20f425 100644 --- a/api/pkg/tap/http_reader.go +++ b/tap/http_reader.go @@ -3,10 +3,7 @@ package tap import ( "bufio" "bytes" - "compress/gzip" - b64 "encoding/base64" "encoding/hex" - "encoding/json" "fmt" "io" "io/ioutil" @@ -73,7 +70,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) { b := bufio.NewReader(h) if isHTTP2, err := checkIsHTTP2Connection(b, h.isClient); err != nil { - SilentError("HTTP/2-Prepare-Connection", "stream %s Failed to check if client is HTTP/2: %s (%v,%+v)\n", h.ident, err, err, err) + SilentError("HTTP/2-Prepare-Connection", "stream %s Failed to check if client is HTTP/2: %s (%v,%+v)", h.ident, err, err, err) // Do something? } else { h.isHTTP2 = isHTTP2 @@ -82,7 +79,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) { if h.isHTTP2 { err := prepareHTTP2Connection(b, h.isClient) if err != nil { - SilentError("HTTP/2-Prepare-Connection-After-Check", "stream %s error: %s (%v,%+v)\n", h.ident, err, err, err) + SilentError("HTTP/2-Prepare-Connection-After-Check", "stream %s error: %s (%v,%+v)", h.ident, err, err, err) } h.grpcAssembler = createGrpcAssembler(b) } @@ -93,7 +90,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) { if err == io.EOF || err == io.ErrUnexpectedEOF { break } else if err != nil { - SilentError("HTTP/2", "stream %s error: %s (%v,%+v)\n", h.ident, err, err, err) + SilentError("HTTP/2", "stream %s error: %s (%v,%+v)", h.ident, err, err, err) continue } } else if h.isClient { @@ -101,7 +98,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) { if err == io.EOF || err == io.ErrUnexpectedEOF { break } else if err != nil { - SilentError("HTTP-request", "stream %s Request error: %s (%v,%+v)\n", h.ident, err, err, err) + SilentError("HTTP-request", "stream %s Request error: %s (%v,%+v)", h.ident, err, err, err) continue } } else { @@ -109,7 +106,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) { if err == io.EOF || err == io.ErrUnexpectedEOF { break } else if err != nil { - SilentError("HTTP-response", "stream %s Response error: %s (%v,%+v)\n", h.ident, err, err, err) + SilentError("HTTP-response", "stream %s Response error: %s (%v,%+v)", h.ident, err, err, err) continue } } @@ -117,38 +114,34 @@ func (h *httpReader) run(wg *sync.WaitGroup) { } func (h *httpReader) handleHTTP2Stream() error { - streamID, messageHTTP1, body, err := h.grpcAssembler.readMessage() + streamID, messageHTTP1, err := h.grpcAssembler.readMessage() h.messageCount++ if err != nil { return err } - var reqResPair *envoyMessageWrapper + var reqResPair *requestResponsePair switch messageHTTP1 := messageHTTP1.(type) { case http.Request: ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.srcIP, h.tcpID.dstIP, h.tcpID.srcPort, h.tcpID.dstPort, streamID) - reqResPair = reqResMatcher.registerRequest(ident, &messageHTTP1, h.captureTime, body, true) + reqResPair = reqResMatcher.registerRequest(ident, &messageHTTP1, h.captureTime) case http.Response: ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.dstIP, h.tcpID.srcIP, h.tcpID.dstPort, h.tcpID.srcPort, streamID) - reqResPair = reqResMatcher.registerResponse(ident, &messageHTTP1, h.captureTime, body, true) + reqResPair = reqResMatcher.registerResponse(ident, &messageHTTP1, h.captureTime) } if reqResPair != nil { + statsTracker.incMatchedMessages() + if h.harWriter != nil { h.harWriter.WritePair( - reqResPair.HttpBufferedTrace.Request.orig.(*http.Request), - reqResPair.HttpBufferedTrace.Request.captureTime, - reqResPair.HttpBufferedTrace.Response.orig.(*http.Response), - reqResPair.HttpBufferedTrace.Response.captureTime, - reqResPair.HttpBufferedTrace.Request.requestSenderIp, + reqResPair.Request.orig.(*http.Request), + reqResPair.Request.captureTime, + reqResPair.Response.orig.(*http.Response), + reqResPair.Response.captureTime, + &reqResPair.Request.connectionInfo, ) - } else { - jsonStr, err := json.Marshal(reqResPair) - if err != nil { - return err - } - broadcastReqResPair(jsonStr) } } @@ -165,37 +158,29 @@ func (h *httpReader) handleHTTP1ClientStream(b *bufio.Reader) error { req.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind s := len(body) if err != nil { - SilentError("HTTP-request-body", "stream %s Got body err: %s\n", h.ident, err) + SilentError("HTTP-request-body", "stream %s Got body err: %s", h.ident, err) } else if h.hexdump { - Info("Body(%d/0x%x)\n%s\n", len(body), len(body), hex.Dump(body)) + Info("Body(%d/0x%x) - %s", len(body), len(body), hex.Dump(body)) } if err := req.Body.Close(); err != nil { - SilentError("HTTP-request-body-close", "stream %s Failed to close request body: %s\n", h.ident, err) + SilentError("HTTP-request-body-close", "stream %s Failed to close request body: %s", h.ident, err) } encoding := req.Header["Content-Encoding"] - bodyStr, err := readBody(body, encoding) - if err != nil { - SilentError("HTTP-request-body-decode", "stream %s Failed to decode body: %s\n", h.ident, err) - } - Info("HTTP/%s Request: %s %s (Body:%d)\n", h.ident, req.Method, req.URL, s) + Info("HTTP/1 Request: %s %s %s (Body:%d) -> %s", h.ident, req.Method, req.URL, s, encoding) ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.srcIP, h.tcpID.dstIP, h.tcpID.srcPort, h.tcpID.dstPort, h.messageCount) - reqResPair := reqResMatcher.registerRequest(ident, req, h.captureTime, bodyStr, false) + reqResPair := reqResMatcher.registerRequest(ident, req, h.captureTime) if reqResPair != nil { + statsTracker.incMatchedMessages() + if h.harWriter != nil { h.harWriter.WritePair( - reqResPair.HttpBufferedTrace.Request.orig.(*http.Request), - reqResPair.HttpBufferedTrace.Request.captureTime, - reqResPair.HttpBufferedTrace.Response.orig.(*http.Response), - reqResPair.HttpBufferedTrace.Response.captureTime, - reqResPair.HttpBufferedTrace.Request.requestSenderIp, + reqResPair.Request.orig.(*http.Request), + reqResPair.Request.captureTime, + reqResPair.Response.orig.(*http.Response), + reqResPair.Response.captureTime, + &reqResPair.Request.connectionInfo, ) - } else { - jsonStr, err := json.Marshal(reqResPair) - if err != nil { - SilentError("HTTP-marshal", "stream %s Error convert request response to json: %s\n", h.ident, err) - } - broadcastReqResPair(jsonStr) } } @@ -224,13 +209,13 @@ func (h *httpReader) handleHTTP1ServerStream(b *bufio.Reader) error { res.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind s := len(body) if err != nil { - SilentError("HTTP-response-body", "HTTP/%s: failed to get body(parsed len:%d): %s\n", h.ident, s, err) + SilentError("HTTP-response-body", "HTTP/%s: failed to get body(parsed len:%d): %s", h.ident, s, err) } if h.hexdump { - Info("Body(%d/0x%x)\n%s\n", len(body), len(body), hex.Dump(body)) + Info("Body(%d/0x%x) - %s", len(body), len(body), hex.Dump(body)) } if err := res.Body.Close(); err != nil { - SilentError("HTTP-response-body-close", "HTTP/%s: failed to close body(parsed len:%d): %s\n", h.ident, s, err) + SilentError("HTTP-response-body-close", "HTTP/%s: failed to close body(parsed len:%d): %s", h.ident, s, err) } sym := "," if res.ContentLength > 0 && res.ContentLength != int64(s) { @@ -241,54 +226,23 @@ func (h *httpReader) handleHTTP1ServerStream(b *bufio.Reader) error { contentType = []string{http.DetectContentType(body)} } encoding := res.Header["Content-Encoding"] - Info("HTTP/%s Response: %s URL:%s (%d%s%d%s) -> %s\n", h.ident, res.Status, req, res.ContentLength, sym, s, contentType, encoding) - bodyStr, err := readBody(body, encoding) - if err != nil { - SilentError("HTTP-response-body-decode", "stream %s Failed to decode body: %s\n", h.ident, err) - } + Info("HTTP/1 Response: %s %s URL:%s (%d%s%d%s) -> %s", h.ident, res.Status, req, res.ContentLength, sym, s, contentType, encoding) ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.dstIP, h.tcpID.srcIP, h.tcpID.dstPort, h.tcpID.srcPort, h.messageCount) - reqResPair := reqResMatcher.registerResponse(ident, res, h.captureTime, bodyStr, false) + reqResPair := reqResMatcher.registerResponse(ident, res, h.captureTime) if reqResPair != nil { + statsTracker.incMatchedMessages() + if h.harWriter != nil { h.harWriter.WritePair( - reqResPair.HttpBufferedTrace.Request.orig.(*http.Request), - reqResPair.HttpBufferedTrace.Request.captureTime, - reqResPair.HttpBufferedTrace.Response.orig.(*http.Response), - reqResPair.HttpBufferedTrace.Response.captureTime, - reqResPair.HttpBufferedTrace.Request.requestSenderIp, + reqResPair.Request.orig.(*http.Request), + reqResPair.Request.captureTime, + reqResPair.Response.orig.(*http.Response), + reqResPair.Response.captureTime, + &reqResPair.Request.connectionInfo, ) - } else { - jsonStr, err := json.Marshal(reqResPair) - if err != nil { - SilentError("HTTP-marshal", "stream %s Error convert request response to json: %s\n", h.ident, err) - } - broadcastReqResPair(jsonStr) } } return nil } - -func readBody(bodyBytes []byte, encoding []string) (string, error) { - var bodyBuffer io.Reader - bodyBuffer = bytes.NewBuffer(bodyBytes) - var err error - if len(encoding) > 0 && (encoding[0] == "gzip" || encoding[0] == "deflate") { - bodyBuffer, err = gzip.NewReader(bodyBuffer) - if err != nil { - SilentError("HTTP-gunzip", "Failed to gzip decode: %s\n", err) - return "", err - } - } - if _, ok := bodyBuffer.(*gzip.Reader); ok { - err = bodyBuffer.(*gzip.Reader).Close() - if err != nil { - return "", err - } - } - - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(bodyBuffer) - return b64.StdEncoding.EncodeToString(buf.Bytes()), err -} diff --git a/api/pkg/tap/net_utils.go b/tap/net_utils.go similarity index 100% rename from api/pkg/tap/net_utils.go rename to tap/net_utils.go diff --git a/tap/outboundlinks.go b/tap/outboundlinks.go new file mode 100644 index 000000000..0cb60bbd9 --- /dev/null +++ b/tap/outboundlinks.go @@ -0,0 +1,29 @@ +package tap + +type OutboundLink struct { + Src string + DstIP string + DstPort int +} + +func NewOutboundLinkWriter() *OutboundLinkWriter { + return &OutboundLinkWriter{ + OutChan: make(chan *OutboundLink), + } +} + +type OutboundLinkWriter struct { + OutChan chan *OutboundLink +} + +func (olw *OutboundLinkWriter) WriteOutboundLink(src string, DstIP string, DstPort int) { + olw.OutChan <- &OutboundLink{ + Src: src, + DstIP: DstIP, + DstPort: DstPort, + } +} + +func (olw *OutboundLinkWriter) Stop() { + close(olw.OutChan) +} diff --git a/api/pkg/tap/passive_tapper.go b/tap/passive_tapper.go similarity index 70% rename from api/pkg/tap/passive_tapper.go rename to tap/passive_tapper.go index 72f5c1294..3b53e524a 100644 --- a/api/pkg/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -10,10 +10,8 @@ package tap import ( "encoding/hex" - "encoding/json" "flag" "fmt" - "github.com/up9inc/mizu/shared" "log" "os" "os/signal" @@ -33,12 +31,10 @@ import ( ) const AppPortsEnvVar = "APP_PORTS" -const OutPortEnvVar = "WEB_SOCKET_PORT" const maxHTTP2DataLenEnvVar = "HTTP2_DATA_SIZE_LIMIT" // default is 1MB, more than the max size accepted by collector and traffic-dumper const maxHTTP2DataLenDefault = 1 * 1024 * 1024 const cleanPeriod = time.Second * 10 -const outboundThrottleCacheExpiryPeriod = time.Minute * 15 var remoteOnlyOutboundPorts = []int { 80, 443 } func parseAppPorts(appPortsList string) []int { @@ -46,7 +42,7 @@ func parseAppPorts(appPortsList string) []int { for _, portStr := range strings.Split(appPortsList, ",") { parsedInt, parseError := strconv.Atoi(portStr) if parseError != nil { - fmt.Println("Provided app port ", portStr, " is not a valid number!") + log.Printf("Provided app port %v is not a valid number!", portStr) } else { ports = append(ports, parsedInt) } @@ -54,13 +50,6 @@ func parseAppPorts(appPortsList string) []int { return ports } -func parseHostAppAddresses(hostAppAddressesString string) []string { - if len(hostAppAddressesString) == 0 { - return []string{} - } - return strings.Split(hostAppAddressesString, ",") -} - var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit") var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)") var statsevery = flag.Int("stats", 60, "Output statistics every N seconds") @@ -90,7 +79,6 @@ var tstype = flag.String("timestamp_type", "", "Type of timestamps to use") var promisc = flag.Bool("promisc", true, "Set promiscuous mode") var anydirection = flag.Bool("anydirection", false, "Capture http requests to other hosts") var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to keep connections which don't transmit data") -var hostAppAddressesString = flag.String("targets", "", "Comma separated list of ip:ports to tap") var memprofile = flag.String("memprofile", "", "Write memory profile") @@ -121,24 +109,20 @@ var stats struct { overlapPackets int } -type CollectorMessage struct { - MessageType string - Ports *[]int `json:"ports,omitempty"` - Addresses *[]string `json:"addresses,omitempty"` +type TapOpts struct { + HostMode bool } var outputLevel int var errorsMap map[string]uint var errorsMapMutex sync.Mutex var nErrors uint -var appPorts []int // global -var ownIps []string //global -var hostMode bool //global -var HostAppAddresses []string //global +var ownIps []string // global +var hostMode bool // global /* minOutputLevel: Error will be printed only if outputLevel is above this value * t: key for errorsMap (counting errors) - * s, a: arguments fmt.Printf + * s, a: arguments log.Printf * Note: Too bad for perf that a... is evaluated */ func logError(minOutputLevel int, t string, s string, a ...interface{}) { @@ -149,7 +133,7 @@ func logError(minOutputLevel int, t string, s string, a ...interface{}) { errorsMapMutex.Unlock() if outputLevel >= minOutputLevel { formatStr := fmt.Sprintf("%s: %s", t, s) - fmt.Printf(formatStr, a...) + log.Printf(formatStr, a...) } } func Error(t string, s string, a ...interface{}) { @@ -160,12 +144,12 @@ func SilentError(t string, s string, a ...interface{}) { } func Info(s string, a ...interface{}) { if outputLevel >= 1 { - fmt.Printf(s, a...) + log.Printf(s, a...) } } func Debug(s string, a ...interface{}) { if outputLevel >= 2 { - fmt.Printf(s, a...) + log.Printf(s, a...) } } @@ -187,9 +171,8 @@ func inArrayString(arr []string, valueToCheck string) bool { return false } -/* - * The assembler context - */ +// Context +// The assembler context type Context struct { CaptureInfo gopacket.CaptureInfo } @@ -198,22 +181,27 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo { return c.CaptureInfo } -func StartPassiveTapper() <-chan *OutputChannelItem { +func StartPassiveTapper(opts *TapOpts) (<-chan *OutputChannelItem, <-chan *OutboundLink) { + hostMode = opts.HostMode + var harWriter *HarWriter if *dumpToHar { harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile) } + outboundLinkWriter := NewOutboundLinkWriter() - go startPassiveTapper(harWriter) + go startPassiveTapper(harWriter, outboundLinkWriter) if harWriter != nil { - return harWriter.OutChan + return harWriter.OutChan, outboundLinkWriter.OutChan } - return nil + return nil, outboundLinkWriter.OutChan } -func startPassiveTapper(harWriter *HarWriter) { +func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWriter) { + log.SetFlags(log.LstdFlags | log.LUTC | log.Lshortfile) + defer util.Run()() if *debug { outputLevel = 2 @@ -226,68 +214,43 @@ func startPassiveTapper(harWriter *HarWriter) { if localhostIPs, err := getLocalhostIPs(); err != nil { // TODO: think this over - fmt.Println("Failed to get self IP addresses") - Error("Getting-Self-Address", "Error getting self ip address: %s (%v,%+v)\n", err, err, err) + log.Println("Failed to get self IP addresses") + Error("Getting-Self-Address", "Error getting self ip address: %s (%v,%+v)", err, err, err) ownIps = make([]string, 0) } else { ownIps = localhostIPs } appPortsStr := os.Getenv(AppPortsEnvVar) + var appPorts []int if appPortsStr == "" { - fmt.Println("Received empty/no APP_PORTS env var! only listening to http on port 80!") + log.Println("Received empty/no APP_PORTS env var! only listening to http on port 80!") appPorts = make([]int, 0) } else { appPorts = parseAppPorts(appPortsStr) } - tapOutputPort := os.Getenv(OutPortEnvVar) - if tapOutputPort == "" { - fmt.Println("Received empty/no WEB_SOCKET_PORT env var! falling back to port 8080") - tapOutputPort = "8080" - } + SetFilterPorts(appPorts) envVal := os.Getenv(maxHTTP2DataLenEnvVar) if envVal == "" { - fmt.Println("Received empty/no HTTP2_DATA_SIZE_LIMIT env var! falling back to", maxHTTP2DataLenDefault) + log.Println("Received empty/no HTTP2_DATA_SIZE_LIMIT env var! falling back to", maxHTTP2DataLenDefault) maxHTTP2DataLen = maxHTTP2DataLenDefault } else { if convertedInt, err := strconv.Atoi(envVal); err != nil { - fmt.Println("Received invalid HTTP2_DATA_SIZE_LIMIT env var! falling back to", maxHTTP2DataLenDefault) + log.Println("Received invalid HTTP2_DATA_SIZE_LIMIT env var! falling back to", maxHTTP2DataLenDefault) maxHTTP2DataLen = maxHTTP2DataLenDefault } else { - fmt.Println("Received HTTP2_DATA_SIZE_LIMIT env var:", maxHTTP2DataLenDefault) + log.Println("Received HTTP2_DATA_SIZE_LIMIT env var:", maxHTTP2DataLenDefault) maxHTTP2DataLen = convertedInt } } - hostMode = os.Getenv(shared.HostModeEnvVar) == "1" - fmt.Printf("App Ports: %v\n", appPorts) - fmt.Printf("Tap output websocket port: %s\n", tapOutputPort) - - var onCollectorMessage = func(message []byte) { - var parsedMessage CollectorMessage - err := json.Unmarshal(message, &parsedMessage) - if err == nil { - - if parsedMessage.MessageType == "setPorts" { - Debug("Got message from collector. Type: %s, Ports: %v\n", parsedMessage.MessageType, parsedMessage.Ports) - appPorts = *parsedMessage.Ports - } else if parsedMessage.MessageType == "setAddresses" { - Debug("Got message from collector. Type: %s, IPs: %v\n", parsedMessage.MessageType, parsedMessage.Addresses) - HostAppAddresses = *parsedMessage.Addresses - Info("Filtering for the following addresses: %s\n", HostAppAddresses) - } - } else { - Error("Collector-Message-Parsing", "Error parsing message from collector: %s (%v,%+v)\n", err, err, err) - } - } - - go startOutputServer(tapOutputPort, onCollectorMessage) + log.Printf("App Ports: %v", gSettings.filterPorts) var handle *pcap.Handle var err error if *fname != "" { if handle, err = pcap.OpenOffline(*fname); err != nil { - log.Fatal("PCAP OpenOffline error:", err) + log.Fatalf("PCAP OpenOffline error: %v", err) } } else { // This is a little complicated because we want to allow all possible options @@ -313,15 +276,15 @@ func startPassiveTapper(harWriter *HarWriter) { } } if handle, err = inactive.Activate(); err != nil { - log.Fatal("PCAP Activate error:", err) + log.Fatalf("PCAP Activate error: %v", err) } defer handle.Close() } if len(flag.Args()) > 0 { bpffilter := strings.Join(flag.Args(), " ") - Info("Using BPF filter %q\n", bpffilter) + Info("Using BPF filter %q", bpffilter) if err = handle.SetBPFFilter(bpffilter); err != nil { - log.Fatal("BPF filter error:", err) + log.Fatalf("BPF filter error: %v", err) } } @@ -329,6 +292,7 @@ func startPassiveTapper(harWriter *HarWriter) { harWriter.Start() defer harWriter.Stop() } + defer outboundLinkWriter.Stop() var dec gopacket.Decoder var ok bool @@ -342,13 +306,18 @@ func startPassiveTapper(harWriter *HarWriter) { source := gopacket.NewPacketSource(handle, dec) source.Lazy = *lazy source.NoCopy = true - Info("Starting to read packets\n") + Info("Starting to read packets") count := 0 bytes := int64(0) start := time.Now() defragger := ip4defrag.NewIPv4Defragmenter() - streamFactory := &tcpStreamFactory{doHTTP: !*nohttp, harWriter: harWriter} + streamFactory := &tcpStreamFactory{ + doHTTP: !*nohttp, + harWriter: harWriter, + outbountLinkWriter: outboundLinkWriter, + + } streamPool := reassembly.NewStreamPool(streamFactory) assembler := reassembly.NewAssembler(streamPool) var assemblerMutex sync.Mutex @@ -378,7 +347,7 @@ func startPassiveTapper(harWriter *HarWriter) { errorMapLen := len(errorsMap) errorsSummery := fmt.Sprintf("%v", errorsMap) errorsMapMutex.Unlock() - fmt.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)\nErrors Summary: %s\n", + log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v) - Errors Summary: %s", count, bytes, time.Since(start), @@ -390,8 +359,8 @@ func startPassiveTapper(harWriter *HarWriter) { // At this moment memStats := runtime.MemStats{} runtime.ReadMemStats(&memStats) - fmt.Printf( - "mem: %d, goroutines: %d, unmatched messages: %d\n", + log.Printf( + "mem: %d, goroutines: %d, unmatched messages: %d", memStats.HeapAlloc, runtime.NumGoroutine(), reqResMatcher.openMessagesMap.Count(), @@ -400,8 +369,8 @@ func startPassiveTapper(harWriter *HarWriter) { // Since the last print cleanStats := cleaner.dumpStats() appStats := statsTracker.dumpStats() - fmt.Printf( - "flushed connections %d, closed connections: %d, deleted messages: %d, matched messages: %d\n", + log.Printf( + "flushed connections %d, closed connections: %d, deleted messages: %d, matched messages: %d", cleanStats.flushed, cleanStats.closed, cleanStats.deleted, @@ -412,11 +381,11 @@ func startPassiveTapper(harWriter *HarWriter) { for packet := range source.Packets() { count++ - Debug("PACKET #%d\n", count) + Debug("PACKET #%d", count) data := packet.Data() bytes += int64(len(data)) if *hexdumppkt { - Debug("Packet content (%d/0x%x)\n%s\n", len(data), len(data), hex.Dump(data)) + Debug("Packet content (%d/0x%x) - %s", len(data), len(data), hex.Dump(data)) } // defrag the IPv4 packet if required @@ -431,18 +400,18 @@ func startPassiveTapper(harWriter *HarWriter) { if err != nil { log.Fatalln("Error while de-fragmenting", err) } else if newip4 == nil { - Debug("Fragment...\n") + Debug("Fragment...") continue // packet fragment, we don't have whole packet yet. } if newip4.Length != l { stats.ipdefrag++ - Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType()) + Debug("Decoding re-assembled packet: %s", newip4.NextLayerType()) pb, ok := packet.(gopacket.PacketBuilder) if !ok { - panic("Not a PacketBuilder") + log.Panic("Not a PacketBuilder") } nextDecoder := newip4.NextLayerType() - nextDecoder.Decode(newip4.Payload, pb) + _ = nextDecoder.Decode(newip4.Payload, pb) } } @@ -459,7 +428,7 @@ func startPassiveTapper(harWriter *HarWriter) { CaptureInfo: packet.Metadata().CaptureInfo, } stats.totalsz += len(tcp.Payload) - //fmt.Println(packet.NetworkLayer().NetworkFlow().Src(), ":", tcp.SrcPort, " -> ", packet.NetworkLayer().NetworkFlow().Dst(), ":", tcp.DstPort) + // log.Println(packet.NetworkLayer().NetworkFlow().Src(), ":", tcp.SrcPort, " -> ", packet.NetworkLayer().NetworkFlow().Dst(), ":", tcp.DstPort) assemblerMutex.Lock() assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c) assemblerMutex.Unlock() @@ -470,11 +439,11 @@ func startPassiveTapper(harWriter *HarWriter) { errorsMapMutex.Lock() errorMapLen := len(errorsMap) errorsMapMutex.Unlock() - fmt.Fprintf(os.Stderr, "Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)\n", count, bytes, time.Since(start), nErrors, errorMapLen) + log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)", count, bytes, time.Since(start), nErrors, errorMapLen) } select { case <-signalChan: - fmt.Fprintf(os.Stderr, "\nCaught SIGINT: aborting\n") + log.Printf("Caught SIGINT: aborting") done = true default: // NOP: continue @@ -497,34 +466,34 @@ func startPassiveTapper(harWriter *HarWriter) { if err != nil { log.Fatal(err) } - pprof.WriteHeapProfile(f) - f.Close() + _ = pprof.WriteHeapProfile(f) + _ = f.Close() } streamFactory.WaitGoRoutines() assemblerMutex.Lock() - Debug("%s\n", assembler.Dump()) + Debug("%s", assembler.Dump()) assemblerMutex.Unlock() if !*nodefrag { - fmt.Printf("IPdefrag:\t\t%d\n", stats.ipdefrag) + log.Printf("IPdefrag:\t\t%d", stats.ipdefrag) } - fmt.Printf("TCP stats:\n") - fmt.Printf(" missed bytes:\t\t%d\n", stats.missedBytes) - fmt.Printf(" total packets:\t\t%d\n", stats.pkt) - fmt.Printf(" rejected FSM:\t\t%d\n", stats.rejectFsm) - fmt.Printf(" rejected Options:\t%d\n", stats.rejectOpt) - fmt.Printf(" reassembled bytes:\t%d\n", stats.sz) - fmt.Printf(" total TCP bytes:\t%d\n", stats.totalsz) - fmt.Printf(" conn rejected FSM:\t%d\n", stats.rejectConnFsm) - fmt.Printf(" reassembled chunks:\t%d\n", stats.reassembled) - fmt.Printf(" out-of-order packets:\t%d\n", stats.outOfOrderPackets) - fmt.Printf(" out-of-order bytes:\t%d\n", stats.outOfOrderBytes) - fmt.Printf(" biggest-chunk packets:\t%d\n", stats.biggestChunkPackets) - fmt.Printf(" biggest-chunk bytes:\t%d\n", stats.biggestChunkBytes) - fmt.Printf(" overlap packets:\t%d\n", stats.overlapPackets) - fmt.Printf(" overlap bytes:\t\t%d\n", stats.overlapBytes) - fmt.Printf("Errors: %d\n", nErrors) + log.Printf("TCP stats:") + log.Printf(" missed bytes:\t\t%d", stats.missedBytes) + log.Printf(" total packets:\t\t%d", stats.pkt) + log.Printf(" rejected FSM:\t\t%d", stats.rejectFsm) + log.Printf(" rejected Options:\t%d", stats.rejectOpt) + log.Printf(" reassembled bytes:\t%d", stats.sz) + log.Printf(" total TCP bytes:\t%d", stats.totalsz) + log.Printf(" conn rejected FSM:\t%d", stats.rejectConnFsm) + log.Printf(" reassembled chunks:\t%d", stats.reassembled) + log.Printf(" out-of-order packets:\t%d", stats.outOfOrderPackets) + log.Printf(" out-of-order bytes:\t%d", stats.outOfOrderBytes) + log.Printf(" biggest-chunk packets:\t%d", stats.biggestChunkPackets) + log.Printf(" biggest-chunk bytes:\t%d", stats.biggestChunkBytes) + log.Printf(" overlap packets:\t%d", stats.overlapPackets) + log.Printf(" overlap bytes:\t\t%d", stats.overlapBytes) + log.Printf("Errors: %d", nErrors) for e := range errorsMap { - fmt.Printf(" %s:\t\t%d\n", e, errorsMap[e]) + log.Printf(" %s:\t\t%d", e, errorsMap[e]) } } diff --git a/tap/settings.go b/tap/settings.go new file mode 100644 index 000000000..cf89dd345 --- /dev/null +++ b/tap/settings.go @@ -0,0 +1,31 @@ +package tap + +type globalSettings struct { + filterPorts []int + filterAuthorities []string +} + +var gSettings = &globalSettings{ + filterPorts: []int{}, + filterAuthorities: []string{}, +} + +func SetFilterPorts(ports []int) { + gSettings.filterPorts = ports +} + +func GetFilterPorts() []int { + ports := make([]int, len(gSettings.filterPorts)) + copy(ports, gSettings.filterPorts) + return ports +} + +func SetFilterAuthorities(ipAddresses []string) { + gSettings.filterAuthorities = ipAddresses +} + +func GetFilterIPs() []string { + addresses := make([]string, len(gSettings.filterAuthorities)) + copy(addresses, gSettings.filterAuthorities) + return addresses +} diff --git a/api/pkg/tap/stats_tracker.go b/tap/stats_tracker.go similarity index 100% rename from api/pkg/tap/stats_tracker.go rename to tap/stats_tracker.go diff --git a/api/pkg/tap/tcp_stream.go b/tap/tcp_stream.go similarity index 83% rename from api/pkg/tap/tcp_stream.go rename to tap/tcp_stream.go index 7e96e9ed5..db5fb59ee 100644 --- a/api/pkg/tap/tcp_stream.go +++ b/tap/tcp_stream.go @@ -34,7 +34,7 @@ type tcpStream struct { func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool { // FSM if !t.tcpstate.CheckState(tcp, dir) { - //SilentError("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String()) + SilentError("FSM-rejection", "%s: Packet rejected by FSM (state:%s)", t.ident, t.tcpstate.String()) stats.rejectFsm++ if !t.fsmerr { t.fsmerr = true @@ -47,7 +47,7 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem // Options err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start) if err != nil { - //SilentError("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err) + SilentError("OptionChecker-rejection", "%s: Packet rejected by OptionChecker: %s", t.ident, err) stats.rejectOpt++ if !*nooptcheck { return false @@ -58,10 +58,10 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem if *checksum { c, err := tcp.ComputeChecksum() if err != nil { - SilentError("ChecksumCompute", "%s: Got error computing checksum: %s\n", t.ident, err) + SilentError("ChecksumCompute", "%s: Got error computing checksum: %s", t.ident, err) accept = false } else if c != 0x0 { - SilentError("Checksum", "%s: Invalid checksum: 0x%x\n", t.ident, c) + SilentError("Checksum", "%s: Invalid checksum: 0x%x", t.ident, c) accept = false } } @@ -95,7 +95,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass if sgStats.OverlapBytes != 0 && sgStats.OverlapPackets == 0 { // In the original example this was handled with panic(). // I don't know what this error means or how to handle it properly. - SilentError("Invalid-Overlap", "bytes:%d, pkts:%d\n", sgStats.OverlapBytes, sgStats.OverlapPackets) + SilentError("Invalid-Overlap", "bytes:%d, pkts:%d", sgStats.OverlapBytes, sgStats.OverlapPackets) } stats.overlapBytes += sgStats.OverlapBytes stats.overlapPackets += sgStats.OverlapPackets @@ -106,7 +106,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass } else { ident = fmt.Sprintf("%v %v(%s): ", t.net.Reverse(), t.transport.Reverse(), dir) } - Debug("%s: SG reassembled packet with %d bytes (start:%v,end:%v,skip:%d,saved:%d,nb:%d,%d,overlap:%d,%d)\n", ident, length, start, end, skip, saved, sgStats.Packets, sgStats.Chunks, sgStats.OverlapBytes, sgStats.OverlapPackets) + Debug("%s: SG reassembled packet with %d bytes (start:%v,end:%v,skip:%d,saved:%d,nb:%d,%d,overlap:%d,%d)", ident, length, start, end, skip, saved, sgStats.Packets, sgStats.Chunks, sgStats.OverlapBytes, sgStats.OverlapPackets) if skip == -1 && *allowmissinginit { // this is allowed } else if skip != 0 { @@ -125,18 +125,18 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass } dnsSize := binary.BigEndian.Uint16(data[:2]) missing := int(dnsSize) - len(data[2:]) - Debug("dnsSize: %d, missing: %d\n", dnsSize, missing) + Debug("dnsSize: %d, missing: %d", dnsSize, missing) if missing > 0 { - Info("Missing some bytes: %d\n", missing) + Info("Missing some bytes: %d", missing) sg.KeepFrom(0) return } p := gopacket.NewDecodingLayerParser(layers.LayerTypeDNS, dns) err := p.DecodeLayers(data[2:], &decoded) if err != nil { - SilentError("DNS-parser", "Failed to decode DNS: %v\n", err) + SilentError("DNS-parser", "Failed to decode DNS: %v", err) } else { - Debug("DNS: %s\n", gopacket.LayerDump(dns)) + Debug("DNS: %s", gopacket.LayerDump(dns)) } if len(data) > 2+int(dnsSize) { sg.KeepFrom(2 + int(dnsSize)) @@ -144,7 +144,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass } else if t.isHTTP { if length > 0 { if *hexdump { - Debug("Feeding http with:\n%s", hex.Dump(data)) + Debug("Feeding http with:%s", hex.Dump(data)) } // This is where we pass the reassembled information onwards // This channel is read by an httpReader object @@ -158,7 +158,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass } func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool { - Debug("%s: Connection closed\n", t.ident) + Debug("%s: Connection closed", t.ident) if t.isHTTP { close(t.client.msgQueue) close(t.server.msgQueue) diff --git a/api/pkg/tap/tcp_stream_factory.go b/tap/tcp_stream_factory.go similarity index 82% rename from api/pkg/tap/tcp_stream_factory.go rename to tap/tcp_stream_factory.go index 23bda51bd..6808cc13e 100644 --- a/api/pkg/tap/tcp_stream_factory.go +++ b/tap/tcp_stream_factory.go @@ -15,22 +15,23 @@ import ( * Generates a new tcp stream for each new tcp connection. Closes the stream when the connection closes. */ type tcpStreamFactory struct { - wg sync.WaitGroup - doHTTP bool - harWriter *HarWriter + wg sync.WaitGroup + doHTTP bool + harWriter *HarWriter + outbountLinkWriter *OutboundLinkWriter } func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream { - Debug("* NEW: %s %s\n", net, transport) + Debug("* NEW: %s %s", net, transport) fsmOptions := reassembly.TCPSimpleFSMOptions{ SupportMissingEstablishment: *allowmissinginit, } - Debug("Current App Ports: %v\n", appPorts) + Debug("Current App Ports: %v", gSettings.filterPorts) dstIp := net.Dst().String() dstPort := int(tcp.DstPort) if factory.shouldNotifyOnOutboundLink(dstIp, dstPort) { - broadcastOutboundLink(net.Src().String(), dstIp, dstPort) + factory.outbountLinkWriter.WriteOutboundLink(net.Src().String(), dstIp, dstPort) } isHTTP := factory.shouldTap(dstIp, dstPort) stream := &tcpStream{ @@ -85,14 +86,14 @@ func (factory *tcpStreamFactory) WaitGoRoutines() { func (factory *tcpStreamFactory) shouldTap(dstIP string, dstPort int) bool { if hostMode { - if inArrayString(HostAppAddresses, fmt.Sprintf("%s:%d", dstIP, dstPort)) == true { + if inArrayString(gSettings.filterAuthorities, fmt.Sprintf("%s:%d", dstIP, dstPort)) == true { return true - } else if inArrayString(HostAppAddresses, dstIP) == true { + } else if inArrayString(gSettings.filterAuthorities, dstIP) == true { return true } return false } else { - isTappedPort := dstPort == 80 || (appPorts != nil && (inArrayInt(appPorts, dstPort))) + isTappedPort := dstPort == 80 || (gSettings.filterPorts != nil && (inArrayInt(gSettings.filterPorts, dstPort))) if !isTappedPort { return false } From bc3efc6d4cac3a2fa5b38466364f6a977e2ef878 Mon Sep 17 00:00:00 2001 From: nimrod-up9 <59927337+nimrod-up9@users.noreply.github.com> Date: Tue, 15 Jun 2021 15:51:09 +0300 Subject: [PATCH 25/36] TRA-3342 Mizu/tap dump to har directory fails on Linux (#71) * Instead of saving incomplete temp har files in a temp dir, save them in the output dir with a *.har.tmp suffix. * API only loads har from *.har files (by extension). --- api/pkg/api/main.go | 21 +++++++++++++++------ tap/har_writer.go | 11 ++++++----- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/api/pkg/api/main.go b/api/pkg/api/main.go index 2a1e550d4..0907b0146 100644 --- a/api/pkg/api/main.go +++ b/api/pkg/api/main.go @@ -5,9 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/google/martian/har" - "github.com/up9inc/mizu/tap" - "go.mongodb.org/mongo-driver/bson/primitive" "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/resolver" @@ -16,7 +13,12 @@ import ( "os" "path" "sort" + "strings" "time" + + "github.com/google/martian/har" + "github.com/up9inc/mizu/tap" + "go.mongodb.org/mongo-driver/bson/primitive" ) var k8sResolver *resolver.Resolver @@ -57,14 +59,21 @@ func startReadingFiles(workingDir string) { for true { dir, _ := os.Open(workingDir) dirFiles, _ := dir.Readdir(-1) - sort.Sort(utils.ByModTime(dirFiles)) - if len(dirFiles) == 0 { + var harFiles []os.FileInfo + for _, fileInfo := range dirFiles { + if strings.HasSuffix(fileInfo.Name(), ".har") { + harFiles = append(harFiles, fileInfo) + } + } + sort.Sort(utils.ByModTime(harFiles)) + + if len(harFiles) == 0 { fmt.Printf("Waiting for new files\n") time.Sleep(3 * time.Second) continue } - fileInfo := dirFiles[0] + fileInfo := harFiles[0] inputFilePath := path.Join(workingDir, fileInfo.Name()) file, err := os.Open(inputFilePath) utils.CheckErr(err) diff --git a/tap/har_writer.go b/tap/har_writer.go index 1bd2b5ee5..3126ee4c0 100644 --- a/tap/har_writer.go +++ b/tap/har_writer.go @@ -16,7 +16,8 @@ import ( ) const readPermission = 0644 -const tempFilenamePrefix = "har_writer" +const harFilenameSuffix = ".har" +const tempFilenameSuffix = ".har.tmp" type PairChanItem struct { Request *http.Request @@ -232,7 +233,7 @@ func (hw *HarWriter) Stop() { } func (hw *HarWriter) openNewFile() { - filename := filepath.Join(os.TempDir(), fmt.Sprintf("%s_%d", tempFilenamePrefix, time.Now().UnixNano())) + filename := buildFilename(hw.OutputDirPath, time.Now(), tempFilenameSuffix) hw.currentFile = openNewHarFile(filename) } @@ -241,15 +242,15 @@ func (hw *HarWriter) closeFile() { tmpFilename := hw.currentFile.file.Name() hw.currentFile = nil - filename := buildFilename(hw.OutputDirPath, time.Now()) + filename := buildFilename(hw.OutputDirPath, time.Now(), harFilenameSuffix) err := os.Rename(tmpFilename, filename) if err != nil { SilentError("Rename-file", "cannot rename file: %s (%v,%+v)", err, err, err) } } -func buildFilename(dir string, t time.Time) string { +func buildFilename(dir string, t time.Time, suffix string) string { // (epoch time in nanoseconds)__(YYYY_Month_DD__hh-mm-ss).har - filename := fmt.Sprintf("%d__%s.har", t.UnixNano(), t.Format("2006_Jan_02__15-04-05")) + filename := fmt.Sprintf("%d__%s%s", t.UnixNano(), t.Format("2006_Jan_02__15-04-05"), suffix) return filepath.Join(dir, filename) } From 485bc7fd2b22fbe364ab6758d9e047cfa0ec2aac Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Thu, 17 Jun 2021 16:48:05 +0300 Subject: [PATCH 26/36] Add export entries endpoint for better up9 connect funcionality (#72) * no message * no message * no message --- api/pkg/controllers/entries_controller.go | 61 ++++++++++++++++++++++- api/pkg/models/models.go | 27 +++++----- api/pkg/routes/public_routes.go | 3 ++ cli/cmd/fetch.go | 10 ++-- cli/cmd/fetchRunner.go | 10 ++-- 5 files changed, 86 insertions(+), 25 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index 06b96b5b8..282271b65 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -9,6 +9,7 @@ import ( "mizuserver/pkg/models" "mizuserver/pkg/utils" "mizuserver/pkg/validation" + "time" ) const ( @@ -75,11 +76,23 @@ func GetHARs(c *fiber.Ctx) error { return c.Status(fiber.StatusBadRequest).JSON(err) } + var timestampFrom, timestampTo int64 + + if entriesFilter.From < 0 { + timestampFrom = 0 + } else { + timestampFrom = entriesFilter.From + } + if entriesFilter.To <= 0 { + timestampTo = time.Now().UnixNano() / int64(time.Millisecond) + } else { + timestampTo = entriesFilter.To + } + var entries []models.MizuEntry database.GetEntriesTable(). + Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)). Order(fmt.Sprintf("timestamp %s", order)). - // Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)). - Limit(1000). Find(&entries) if len(entries) > 0 { @@ -125,6 +138,50 @@ func GetHARs(c *fiber.Ctx) error { return c.Status(fiber.StatusOK).SendStream(buffer) } +func GetFullEntries(c *fiber.Ctx) error { + entriesFilter := &models.HarFetchRequestBody{} + order := OrderDesc + if err := c.QueryParser(entriesFilter); err != nil { + return c.Status(fiber.StatusBadRequest).JSON(err) + } + err := validation.Validate(entriesFilter) + if err != nil { + return c.Status(fiber.StatusBadRequest).JSON(err) + } + + var timestampFrom, timestampTo int64 + + if entriesFilter.From < 0 { + timestampFrom = 0 + } else { + timestampFrom = entriesFilter.From + } + if entriesFilter.To <= 0 { + timestampTo = time.Now().UnixNano() / int64(time.Millisecond) + } else { + timestampTo = entriesFilter.To + } + + var entries []models.MizuEntry + database.GetEntriesTable(). + Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)). + Order(fmt.Sprintf("timestamp %s", order)). + Find(&entries) + + if len(entries) > 0 { + // the entries always order from oldest to newest so we should revers + utils.ReverseSlice(entries) + } + + entriesArray := make([]har.Entry, 0) + for _, entryData := range entries { + var harEntry har.Entry + _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) + entriesArray = append(entriesArray, harEntry) + } + return c.Status(fiber.StatusOK).JSON(entriesArray) +} + func GetEntry(c *fiber.Ctx) error { var entryData models.EntryData database.GetEntriesTable(). diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index d8dc90184..5cde7622c 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -12,15 +12,15 @@ type MizuEntry struct { ID uint `gorm:"primarykey"` CreatedAt time.Time UpdatedAt time.Time - Entry string `json:"entry,omitempty" gorm:"column:entry"` - EntryId string `json:"entryId" gorm:"column:entryId"` - Url string `json:"url" gorm:"column:url"` - Method string `json:"method" gorm:"column:method"` - Status int `json:"status" gorm:"column:status"` - RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"` - Service string `json:"service" gorm:"column:service"` - Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` - Path string `json:"path" gorm:"column:path"` + Entry string `json:"entry,omitempty" gorm:"column:entry"` + EntryId string `json:"entryId" gorm:"column:entryId"` + Url string `json:"url" gorm:"column:url"` + Method string `json:"method" gorm:"column:method"` + Status int `json:"status" gorm:"column:status"` + RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"` + Service string `json:"service" gorm:"column:service"` + Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` + Path string `json:"path" gorm:"column:path"` ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` } @@ -37,7 +37,7 @@ type BaseEntryDetails struct { } type EntryData struct { - Entry string `json:"entry,omitempty"` + Entry string `json:"entry,omitempty"` ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` } @@ -48,7 +48,8 @@ type EntriesFilter struct { } type HarFetchRequestBody struct { - Limit int `query:"limit"` + From int64 `query:"from"` + To int64 `query:"to"` } type WebSocketEntryMessage struct { @@ -56,7 +57,6 @@ type WebSocketEntryMessage struct { Data *BaseEntryDetails `json:"data,omitempty"` } - type WebSocketTappedEntryMessage struct { *shared.WebSocketMessageMetadata Data *tap.OutputChannelItem @@ -82,7 +82,6 @@ func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, err return json.Marshal(message) } - // ExtendedHAR is the top level object of a HAR log. type ExtendedHAR struct { Log *ExtendedLog `json:"log"` @@ -100,5 +99,5 @@ type ExtendedLog struct { type ExtendedCreator struct { *har.Creator - Source string `json:"_source"` + Source string `json:"_source"` } diff --git a/api/pkg/routes/public_routes.go b/api/pkg/routes/public_routes.go index 042ea54c9..df589a62c 100644 --- a/api/pkg/routes/public_routes.go +++ b/api/pkg/routes/public_routes.go @@ -11,8 +11,11 @@ func EntriesRoutes(fiberApp *fiber.App) { routeGroup.Get("/entries", controllers.GetEntries) // get entries (base/thin entries) routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry + routeGroup.Get("/exportEntries", controllers.GetFullEntries) + routeGroup.Get("/har", controllers.GetHARs) + routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB diff --git a/cli/cmd/fetch.go b/cli/cmd/fetch.go index c4e8fb913..9e55f08fa 100644 --- a/cli/cmd/fetch.go +++ b/cli/cmd/fetch.go @@ -5,8 +5,10 @@ import ( ) type MizuFetchOptions struct { - Limit uint16 - Directory string + FromTimestamp int64 + ToTimestamp int64 + Directory string + MizuPort uint } var mizuFetchOptions = MizuFetchOptions{} @@ -23,6 +25,8 @@ var fetchCmd = &cobra.Command{ func init() { rootCmd.AddCommand(fetchCmd) - fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.Limit, "limit", "l", 1000, "Provide a custom limit for entries to fetch") fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries") + fetchCmd.Flags().Int64Var(&mizuFetchOptions.FromTimestamp, "from", 0, "Custom start timestamp for fetched entries") + fetchCmd.Flags().Int64Var(&mizuFetchOptions.ToTimestamp, "to", 0, "Custom end timestamp fetched entries") + fetchCmd.Flags().UintVarP(&mizuFetchOptions.MizuPort, "port", "p", 8899, "Custom port for mizu") } diff --git a/cli/cmd/fetchRunner.go b/cli/cmd/fetchRunner.go index 71db8364d..58db0352b 100644 --- a/cli/cmd/fetchRunner.go +++ b/cli/cmd/fetchRunner.go @@ -14,7 +14,7 @@ import ( ) func RunMizuFetch(fetch *MizuFetchOptions) { - resp, err := http.Get(fmt.Sprintf("http://localhost:8899/api/har?limit=%v", fetch.Limit)) + resp, err := http.Get(fmt.Sprintf("http://localhost:%v/api/har?from=%v&to=%v", fetch.MizuPort, fetch.FromTimestamp, fetch.ToTimestamp)) if err != nil { log.Fatal(err) } @@ -53,7 +53,7 @@ func Unzip(reader *zip.Reader, dest string) error { path := filepath.Join(dest, f.Name) // Check for ZipSlip (Directory traversal) - if !strings.HasPrefix(path, filepath.Clean(dest) + string(os.PathSeparator)) { + if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) { return fmt.Errorf("illegal file path: %s", path) } @@ -61,7 +61,7 @@ func Unzip(reader *zip.Reader, dest string) error { _ = os.MkdirAll(path, f.Mode()) } else { _ = os.MkdirAll(filepath.Dir(path), f.Mode()) - fmt.Print("writing HAR file [ ", path, " ] .. ") + fmt.Print("writing HAR file [ ", path, " ] .. ") f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { return err @@ -70,7 +70,7 @@ func Unzip(reader *zip.Reader, dest string) error { if err := f.Close(); err != nil { panic(err) } - fmt.Println(" done") + fmt.Println(" done") }() _, err = io.Copy(f, rc) @@ -90,5 +90,3 @@ func Unzip(reader *zip.Reader, dest string) error { return nil } - - From accad7c05829dd46d700bb3b9ea9a9d03108273d Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Sun, 20 Jun 2021 17:41:36 +0300 Subject: [PATCH 27/36] Filter 'cookie' header --- .../messageSensitiveDataCleaner.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index 15a8e5d8b..cc0e4d289 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -15,8 +15,8 @@ import ( ) func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.TrafficFilteringOptions) { - filterHarHeaders(harOutputItem.HarEntry.Request.Headers) - filterHarHeaders(harOutputItem.HarEntry.Response.Headers) + harOutputItem.HarEntry.Request.Headers = filterHarHeaders(harOutputItem.HarEntry.Request.Headers) + harOutputItem.HarEntry.Response.Headers = filterHarHeaders(harOutputItem.HarEntry.Response.Headers) harOutputItem.HarEntry.Request.Cookies = make([]har.Cookie, 0, 0) harOutputItem.HarEntry.Response.Cookies = make([]har.Cookie, 0, 0) @@ -44,12 +44,19 @@ func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, opt } } -func filterHarHeaders(headers []har.Header) { +func filterHarHeaders(headers []har.Header) []har.Header { + newHeaders := make([]har.Header, 0) for i, header := range headers { - if isFieldNameSensitive(header.Name) { + if strings.ToLower(header.Name) == "cookie" { + continue + } else if isFieldNameSensitive(header.Name) { + newHeaders = append(newHeaders, har.Header{Name: header.Name, Value: maskedFieldPlaceholderValue}) headers[i].Value = maskedFieldPlaceholderValue + } else { + newHeaders = append(newHeaders, header) } } + return newHeaders } func getContentTypeHeaderValue(headers []har.Header) string { From 9a1c17cc61c80701aa72d8c8e01a62ded67299f0 Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Mon, 21 Jun 2021 11:19:04 +0300 Subject: [PATCH 28/36] Release action (#73) * Create main.yml * Update main.yml * Update main.yml * Update main.yml * Update main.yml * trying new approach * no message * yaml error * no message * no message * no message * missing ) * no message * no message * remove main.yml and fix branches * Create tag-temp.yaml * Update tag-temp.yaml * Update tag-temp.yaml * no message * no message * no message * no message * no message * no message * no message * #minor * no message * no message * added checksum calc to CLI makefile * fixed build error - created bin directory upfront * using markdown for release text * use separate checksum files * fixed release readme * #minor * readme updated Co-authored-by: Alex Haiut --- .github/workflows/publish-cli.yml | 34 ++++++++++++++++++++++++--- README.md | 39 +++++++++++++++++++++++++++---- cli/Makefile | 12 ++++++++-- cli/cmd/tapRunner.go | 6 +---- cli/cmd/version.go | 21 ++++++++++++++++- cli/mizu/consts.go | 8 ++++--- 6 files changed, 101 insertions(+), 19 deletions(-) diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml index 3790568f2..d71b49a19 100644 --- a/.github/workflows/publish-cli.yml +++ b/.github/workflows/publish-cli.yml @@ -2,8 +2,9 @@ name: public-cli on: push: branches: - - 'develop' - - 'main' + - develop + - main + - my-temp-release-check jobs: docker: runs-on: ubuntu-latest @@ -15,5 +16,32 @@ jobs: with: service_account_key: ${{ secrets.GCR_JSON_KEY }} export_default_credentials: true + - uses: haya14busa/action-cond@v1 + id: condval + with: + cond: ${{ github.ref == 'refs/heads/main' }} + if_true: "minor" + if_false: "patch" + - name: Auto Increment Semver Action + uses: MCKanpolat/auto-semver-action@1.0.5 + id: versioning + with: + releaseType: ${{ steps.condval.outputs.value }} + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Get base image name + shell: bash + run: | + echo "##[set-output name=build_timestamp;]$(echo $(date +%s))" + echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: version_parameters - name: Build and Push CLI - run: make push-cli + run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}' + - name: publish + uses: ncipollo/release-action@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + artifacts: "cli/bin/*" + commit: ${{ steps.version_parameters.outputs.branch }} + tag: ${{ steps.versioning.outputs.version }} + prerelease: ${{ github.ref != 'refs/heads/main' }} + bodyFile: 'cli/bin/README.md' diff --git a/README.md b/README.md index efbcc082c..502fcde99 100644 --- a/README.md +++ b/README.md @@ -3,16 +3,45 @@ standalone web app traffic viewer for Kubernetes ## Download -Download `mizu` for your platform as +Download `mizu` for your platform and operating system -* for MacOS - `curl -o mizu https://static.up9.com/mizu/mizu-darwin-amd64 && chmod 755 mizu` -* for Linux - `curl -o mizu https://static.up9.com/mizu/mizu-linux-amd64 && chmod 755 mizu` +### Latest stable release -## Run +* for MacOS - Intel +`curl -Lo mizu https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_amd64 && chmod 755 mizu` + +* for MacOS - Apple Silicon + `curl -Lo mizu https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_arm64 && chmod 755 mizu` + +* for Linux - Intel 64bit + `curl -Lo mizu https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 && chmod 755 mizu` +SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page. + +### Development (unstable) build +Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. + +## How to run 1. Find pod you'd like to tap to in your Kubernetes cluster -2. Run `mizu --pod podname` +2. Run `mizu PODNAME` or `mizu REGEX` 3. Open browser on `http://localhost:8899` as instructed .. 4. Watch the WebAPI traffic flowing .. +5. Type ^C to stop + +## Examples + +To tap specific pod - +``` + $ kubectl get pods | grep front-end + NAME READY STATUS RESTARTS AGE + front-end-649fc5fd6-kqbtn 2/2 Running 0 7m + $ mizu tap front-end-649fc5fd6-kqbtn + +front-end-649fc5fd6-kqbtn + Web interface is now available at http://localhost:8899 + ^C +``` + + + diff --git a/cli/Makefile b/cli/Makefile index 835ee9142..3dd9f043a 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -1,6 +1,8 @@ -FOLDER=$(GOOS).$(GOARCH) +SUFFIX=$(GOOS)_$(GOARCH) COMMIT_HASH=$(shell git rev-parse HEAD) GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]') +GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]') +BUILD_TIMESTAMP=$(shell date +%s) .PHONY: help .DEFAULT_GOAL := help @@ -12,10 +14,16 @@ install: go install mizu.go build: ## build mizu CLI binary (select platform via GOOS / GOARCH env variables) - go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)'" -o bin/$(FOLDER)/mizu mizu.go + go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \ + -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \ + -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \ + -X 'github.com/up9inc/mizu/cli/mizu.SemVer=$(SEM_VER)'" \ + -o bin/mizu_$(SUFFIX) mizu.go + (cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256) build-all: ## build for all supported platforms @echo "Compiling for every OS and Platform" + @mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" > bin/README.md @$(MAKE) build GOOS=darwin GOARCH=amd64 @$(MAKE) build GOOS=linux GOARCH=amd64 @# $(MAKE) GOOS=windows GOARCH=amd64 diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index e01294e1d..679e46dbe 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -249,11 +249,7 @@ func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.P return false } if !mizuRBACExists { - var versionString = mizu.Version - if mizu.GitCommitHash != "" { - versionString += "-" + mizu.GitCommitHash - } - err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, versionString) + err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, mizu.RBACVersion) if err != nil { fmt.Printf("warning: could not create mizu rbac resources %v\n", err) return false diff --git a/cli/cmd/version.go b/cli/cmd/version.go index 11a348a27..366b05da2 100644 --- a/cli/cmd/version.go +++ b/cli/cmd/version.go @@ -3,19 +3,38 @@ package cmd import ( "fmt" "github.com/up9inc/mizu/cli/mizu" + "strconv" + "time" "github.com/spf13/cobra" ) +type MizuVersionOptions struct { + DebugInfo bool +} + + +var mizuVersionOptions = &MizuVersionOptions{} + var versionCmd = &cobra.Command{ Use: "version", Short: "Print version info", RunE: func(cmd *cobra.Command, args []string) error { - fmt.Printf("%s (%s) %s\n", mizu.Version, mizu.Branch, mizu.GitCommitHash) + if mizuVersionOptions.DebugInfo { + timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0) + fmt.Printf("Version: %s \nBranch: %s (%s) \n", mizu.SemVer, mizu.Branch, mizu.GitCommitHash) + fmt.Printf("Build Time: %s (%s)\n", mizu.BuildTimestamp, time.Unix(timeStampInt, 0)) + + } else { + fmt.Printf("Version: %s (%s)\n", mizu.SemVer, mizu.Branch) + } return nil }, } func init() { rootCmd.AddCommand(versionCmd) + + versionCmd.Flags().BoolVarP(&mizuVersionOptions.DebugInfo, "debug", "d", false, "Provide all information about version") + } diff --git a/cli/mizu/consts.go b/cli/mizu/consts.go index 59d33f1b0..9f72620ad 100644 --- a/cli/mizu/consts.go +++ b/cli/mizu/consts.go @@ -1,9 +1,11 @@ package mizu var ( - Version = "v0.0.1" - Branch = "develop" - GitCommitHash = "" // this var is overridden using ldflags in makefile when building + SemVer = "0.0.1" + Branch = "develop" + GitCommitHash = "" // this var is overridden using ldflags in makefile when building + BuildTimestamp = "" // this var is overridden using ldflags in makefile when building + RBACVersion = "v1" ) const ( From 756f5f57202b1cd48794345abb4884d292360c63 Mon Sep 17 00:00:00 2001 From: nimrod-up9 <59927337+nimrod-up9@users.noreply.github.com> Date: Mon, 21 Jun 2021 12:07:25 +0300 Subject: [PATCH 29/36] TRA-3360 Fix: Mizu ignores -n namespace flag and records traffic from all pods (#75) Do not tap pods in namespaces which were not requested. --- cli/cmd/tapRunner.go | 9 ++++++--- cli/kubernetes/provider.go | 5 ++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 679e46dbe..6c0cc7d9b 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -38,7 +38,8 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // cancel will be called when this function exits - if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery); err != nil { + targetNamespace := getNamespace(tappingOptions, kubernetesProvider) + if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery, targetNamespace); err != nil { return } else { currentlyTappedPods = matchingPods @@ -146,10 +147,12 @@ func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) { } func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions) { - added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, getNamespace(tappingOptions, kubernetesProvider)), podRegex) + targetNamespace := getNamespace(tappingOptions, kubernetesProvider) + + added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, targetNamespace), podRegex) restartTappers := func() { - if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex); err != nil { + if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex, targetNamespace); err != nil { fmt.Printf("Error getting pods by regex: %s (%v,%+v)\n", err, err, err) cancel() } else { diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index f7d7e68f7..09b000bcb 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/up9inc/mizu/cli/mizu" "path/filepath" "regexp" @@ -301,8 +300,8 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac return err } -func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp) ([]core.Pod, error) { - pods, err := provider.clientSet.CoreV1().Pods(mizu.K8sAllNamespaces).List(ctx, metav1.ListOptions{}) +func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespace string) ([]core.Pod, error) { + pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } From 5d205b50823792758d03fb55fe52b51f16cf73bd Mon Sep 17 00:00:00 2001 From: Alex Haiut Date: Mon, 21 Jun 2021 13:06:08 +0300 Subject: [PATCH 30/36] added apple/m1 binary, updated readme (#77) Co-authored-by: Alex Haiut --- .github/workflows/publish-cli.yml | 1 - README.md | 21 ++++++++++++++++++++- cli/Makefile | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml index d71b49a19..b9e814931 100644 --- a/.github/workflows/publish-cli.yml +++ b/.github/workflows/publish-cli.yml @@ -4,7 +4,6 @@ on: branches: - develop - main - - my-temp-release-check jobs: docker: runs-on: ubuntu-latest diff --git a/README.md b/README.md index 502fcde99..0c7e586c0 100644 --- a/README.md +++ b/README.md @@ -32,16 +32,35 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. ## Examples +Run `mizu help` for usage options + + To tap specific pod - ``` - $ kubectl get pods | grep front-end + $ kubectl get pods NAME READY STATUS RESTARTS AGE front-end-649fc5fd6-kqbtn 2/2 Running 0 7m + .. + $ mizu tap front-end-649fc5fd6-kqbtn +front-end-649fc5fd6-kqbtn Web interface is now available at http://localhost:8899 ^C ``` +To tap multiple pods using regex - +``` + $ kubectl get pods + NAME READY STATUS RESTARTS AGE + carts-66c77f5fbb-fq65r 2/2 Running 0 20m + catalogue-5f4cb7cf5-7zrmn 2/2 Running 0 20m + front-end-649fc5fd6-kqbtn 2/2 Running 0 20m + .. + $ mizu tap "^ca.*" + +carts-66c77f5fbb-fq65r + +catalogue-5f4cb7cf5-7zrmn + Web interface is now available at http://localhost:8899 + ^C +``` diff --git a/cli/Makefile b/cli/Makefile index 3dd9f043a..ef97249c7 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -29,7 +29,7 @@ build-all: ## build for all supported platforms @# $(MAKE) GOOS=windows GOARCH=amd64 @# $(MAKE) GOOS=linux GOARCH=386 @# $(MAKE) GOOS=windows GOARCH=386 - @# $(MAKE) GOOS=darwin GOARCH=arm64 + @$(MAKE) GOOS=darwin GOARCH=arm64 @# $(MAKE) GOOS=linux GOARCH=arm64 @# $(MAKE) GOOS=windows GOARCH=arm64 @echo "---------" From f604a3a35d4902ff86fb4a29d98345c0c1a3d375 Mon Sep 17 00:00:00 2001 From: Alex Haiut Date: Mon, 21 Jun 2021 13:53:08 +0300 Subject: [PATCH 31/36] Update README.md (#78) --- README.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0c7e586c0..0205f5726 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,25 @@ Download `mizu` for your platform and operating system ### Latest stable release * for MacOS - Intel -`curl -Lo mizu https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_amd64 && chmod 755 mizu` +``` +curl -Lo mizu \ +https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_amd64 \ +&& chmod 755 mizu +``` * for MacOS - Apple Silicon - `curl -Lo mizu https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_arm64 && chmod 755 mizu` +``` +curl -Lo mizu \ +https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_arm64 \ +&& chmod 755 mizu +``` * for Linux - Intel 64bit - `curl -Lo mizu https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 && chmod 755 mizu` +``` +curl -Lo mizu \ +https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \ +&& chmod 755 mizu +``` SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page. From 3e32c889d954c4d27217f3f37a8e3447be0f00c2 Mon Sep 17 00:00:00 2001 From: Alex Haiut Date: Mon, 21 Jun 2021 15:43:13 +0300 Subject: [PATCH 32/36] fixed Apple M1 - darwin.arm64 build (#80) Co-authored-by: Alex Haiut --- cli/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/Makefile b/cli/Makefile index ef97249c7..09684c5ab 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -26,10 +26,10 @@ build-all: ## build for all supported platforms @mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" > bin/README.md @$(MAKE) build GOOS=darwin GOARCH=amd64 @$(MAKE) build GOOS=linux GOARCH=amd64 + @$(MAKE) build GOOS=darwin GOARCH=arm64 @# $(MAKE) GOOS=windows GOARCH=amd64 @# $(MAKE) GOOS=linux GOARCH=386 @# $(MAKE) GOOS=windows GOARCH=386 - @$(MAKE) GOOS=darwin GOARCH=arm64 @# $(MAKE) GOOS=linux GOARCH=arm64 @# $(MAKE) GOOS=windows GOARCH=arm64 @echo "---------" From f18f3da99cf82328756aa7fd503733d2ee7ed0e9 Mon Sep 17 00:00:00 2001 From: Alex Haiut Date: Mon, 21 Jun 2021 16:23:14 +0300 Subject: [PATCH 33/36] Feature/fix darwin m1 build (#81) * fixed Apple M1 - darwin.arm64 build * removing Apple M1 build for now Co-authored-by: Alex Haiut --- README.md | 8 -------- cli/Makefile | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/README.md b/README.md index 0205f5726..1d0cd48ef 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,6 @@ curl -Lo mizu \ https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_amd64 \ && chmod 755 mizu ``` - -* for MacOS - Apple Silicon -``` -curl -Lo mizu \ -https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_arm64 \ -&& chmod 755 mizu -``` * for Linux - Intel 64bit ``` @@ -28,7 +21,6 @@ https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \ && chmod 755 mizu ``` - SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page. ### Development (unstable) build diff --git a/cli/Makefile b/cli/Makefile index 09684c5ab..b228e9fd1 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -26,7 +26,7 @@ build-all: ## build for all supported platforms @mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" > bin/README.md @$(MAKE) build GOOS=darwin GOARCH=amd64 @$(MAKE) build GOOS=linux GOARCH=amd64 - @$(MAKE) build GOOS=darwin GOARCH=arm64 + @# $(MAKE) build GOOS=darwin GOARCH=arm64 @# $(MAKE) GOOS=windows GOARCH=amd64 @# $(MAKE) GOOS=linux GOARCH=386 @# $(MAKE) GOOS=windows GOARCH=386 From 6f47ad862e79e51b507baf591e77f1508479c05a Mon Sep 17 00:00:00 2001 From: nimrod-up9 <59927337+nimrod-up9@users.noreply.github.com> Date: Thu, 24 Jun 2021 15:10:11 +0300 Subject: [PATCH 34/36] TRA-3317 Tap and show outbound traffic (#83) * Tap outgoing: If --anydirection flag is passed with HOST_MODE, tap by source IP. * Moved ConnectionInfo from http_matcher to http_reader. * Generalized shouldTap in stream factory to get more properties. * tap reports IsOutgoing property of tcp connection. * gofmt. * CLI instructs tapper to tap outgoing connections. * API saves IsOutgoing to DB and passes it to UI. * Add a visual marker in the HAR list for outgoing messages. * Fixed: Swapped src and dst. * Resolver keeps a list of all ClusterIP services. * Do not save HARs with destination ClusterIP services. * CLI accepts flag that controls traffic direction. * Indicate incoming/outgoing with icon instead of with border color. * Fixed: Didn't filter messages to services in aggregator. * Clearer syntax around the direction icon. Added title text. * Fixed width around direction icon. * Less repetition. * Removed TODO. * Renamed incoming -> ingoing. * More verbose title text to image. * Switched routine order for readability. --- api/main.go | 26 +++++++++--- api/pkg/api/main.go | 11 +++-- api/pkg/models/models.go | 2 + api/pkg/resolver/loader.go | 6 +-- api/pkg/resolver/resolver.go | 15 +++++++ api/pkg/utils/utils.go | 15 +++---- cli/cmd/tap.go | 6 +++ cli/cmd/tapRunner.go | 3 +- cli/kubernetes/provider.go | 15 ++++++- shared/models.go | 2 +- tap/http_matcher.go | 16 ------- tap/http_reader.go | 42 +++++++++++++++++-- tap/tcp_stream_factory.go | 35 ++++++++++------ ui/src/components/HarEntry.tsx | 16 ++++++- ui/src/components/assets/ingoing-traffic.svg | 1 + ui/src/components/assets/outgoing-traffic.svg | 1 + ui/src/components/style/HarEntry.module.sass | 15 ++++++- ui/src/components/style/variables.module.scss | 2 +- 18 files changed, 171 insertions(+), 58 deletions(-) create mode 100644 ui/src/components/assets/ingoing-traffic.svg create mode 100644 ui/src/components/assets/outgoing-traffic.svg diff --git a/api/main.go b/api/main.go index b7bbaa14d..2cf1da9e4 100644 --- a/api/main.go +++ b/api/main.go @@ -35,10 +35,12 @@ func main() { if *standalone { harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts) - filteredHarChannel := make(chan *tap.OutputChannelItem) + filteredHarChannel0 := make(chan *tap.OutputChannelItem) + filteredHarChannel1 := make(chan *tap.OutputChannelItem) - go filterHarHeaders(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions()) - go api.StartReadingEntries(filteredHarChannel, nil) + go filterServices(harOutputChannel, filteredHarChannel0, getTrafficFilteringOptions()) + go filterHarHeaders(filteredHarChannel0, filteredHarChannel1, getTrafficFilteringOptions()) + go api.StartReadingEntries(filteredHarChannel1, nil) go api.StartReadingOutbound(outboundLinkOutputChannel) hostApi(nil) @@ -64,10 +66,12 @@ func main() { go api.StartReadingOutbound(outboundLinkOutputChannel) } else if *aggregator { socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) - filteredHarChannel := make(chan *tap.OutputChannelItem) + filteredHarChannel0 := make(chan *tap.OutputChannelItem) + filteredHarChannel1 := make(chan *tap.OutputChannelItem) - go api.StartReadingEntries(filteredHarChannel, nil) - go filterHarHeaders(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) + go filterServices(socketHarOutChannel, filteredHarChannel0, getTrafficFilteringOptions()) + go filterHarHeaders(filteredHarChannel0, filteredHarChannel1, getTrafficFilteringOptions()) + go api.StartReadingEntries(filteredHarChannel1, nil) hostApi(socketHarOutChannel) } @@ -125,6 +129,16 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions { return &filteringOptions } +func filterServices(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { + for message := range inChannel { + if api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { + continue + } + + outChannel <- message + } +} + func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { for message := range inChannel { sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions) diff --git a/api/pkg/api/main.go b/api/pkg/api/main.go index 0907b0146..532344f3e 100644 --- a/api/pkg/api/main.go +++ b/api/pkg/api/main.go @@ -84,7 +84,7 @@ func startReadingFiles(workingDir string) { for _, entry := range inputHar.Log.Entries { time.Sleep(time.Millisecond * 250) - saveHarToDb(entry, fileInfo.Name()) + saveHarToDb(entry, fileInfo.Name(), false) } rmErr := os.Remove(inputFilePath) utils.CheckErr(rmErr) @@ -97,7 +97,7 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) { } for item := range outputItems { - saveHarToDb(item.HarEntry, item.ConnectionInfo.ClientIP) + saveHarToDb(item.HarEntry, item.ConnectionInfo.ClientIP, item.ConnectionInfo.IsOutgoing) } } @@ -109,7 +109,7 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) { } -func saveHarToDb(entry *har.Entry, sender string) { +func saveHarToDb(entry *har.Entry, sender string, isOutgoing bool) { entryBytes, _ := json.Marshal(entry) serviceName, urlPath, serviceHostName := getServiceNameFromUrl(entry.Request.URL) entryId := primitive.NewObjectID().Hex() @@ -133,6 +133,7 @@ func saveHarToDb(entry *har.Entry, sender string) { Timestamp: entry.StartedDateTime.UnixNano() / int64(time.Millisecond), ResolvedSource: resolvedSource, ResolvedDestination: resolvedDestination, + IsOutgoing: isOutgoing, } database.GetEntriesTable().Create(&mizuEntry) @@ -146,3 +147,7 @@ func getServiceNameFromUrl(inputUrl string) (string, string, string) { utils.CheckErr(err) return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path, parsed.Host } + +func CheckIsServiceIP(address string) bool { + return k8sResolver.CheckIsServiceIP(address) +} diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index 5cde7622c..108801e1e 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -23,6 +23,7 @@ type MizuEntry struct { Path string `json:"path" gorm:"column:path"` ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` + IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"` } type BaseEntryDetails struct { @@ -34,6 +35,7 @@ type BaseEntryDetails struct { StatusCode int `json:"statusCode,omitempty"` Method string `json:"method,omitempty"` Timestamp int64 `json:"timestamp,omitempty"` + IsOutgoing bool `json:"isOutgoing,omitempty"` } type EntryData struct { diff --git a/api/pkg/resolver/loader.go b/api/pkg/resolver/loader.go index c6836116c..f25fab3f0 100644 --- a/api/pkg/resolver/loader.go +++ b/api/pkg/resolver/loader.go @@ -21,7 +21,7 @@ func NewFromInCluster(errOut chan error) (*Resolver, error) { if err != nil { return nil, err } - return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), errOut: errOut}, nil + return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil } func NewFromOutOfCluster(kubeConfigPath string, errOut chan error) (*Resolver, error) { @@ -53,9 +53,9 @@ func NewFromOutOfCluster(kubeConfigPath string, errOut chan error) (*Resolver, e return nil, err } - return &Resolver{clientConfig: clientConfig, clientSet: clientset, nameMap: make(map[string]string), errOut: errOut}, nil + return &Resolver{clientConfig: clientConfig, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil } func NewFromExisting(clientConfig *restclient.Config, clientSet *kubernetes.Clientset, errOut chan error) *Resolver { - return &Resolver{clientConfig: clientConfig, clientSet: clientSet, nameMap: make(map[string]string), errOut: errOut} + return &Resolver{clientConfig: clientConfig, clientSet: clientSet, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut} } diff --git a/api/pkg/resolver/resolver.go b/api/pkg/resolver/resolver.go index 14076a108..88dbbc929 100644 --- a/api/pkg/resolver/resolver.go +++ b/api/pkg/resolver/resolver.go @@ -20,6 +20,7 @@ type Resolver struct { clientConfig *restclient.Config clientSet *kubernetes.Clientset nameMap map[string]string + serviceMap map[string]string isStarted bool errOut chan error } @@ -41,6 +42,11 @@ func (resolver *Resolver) Resolve(name string) string { return resolvedName } +func (resolver *Resolver) CheckIsServiceIP(address string) bool { + _, isFound := resolver.serviceMap[address] + return isFound +} + func (resolver *Resolver) watchPods(ctx context.Context) error { // empty namespace makes the client watch all namespaces watcher, err := resolver.clientSet.CoreV1().Pods("").Watch(ctx, metav1.ListOptions{Watch: true}) @@ -124,6 +130,7 @@ func (resolver *Resolver) watchServices(ctx context.Context) error { serviceHostname := fmt.Sprintf("%s.%s", service.Name, service.Namespace) if service.Spec.ClusterIP != "" && service.Spec.ClusterIP != kubClientNullString { resolver.saveResolvedName(service.Spec.ClusterIP, serviceHostname, event.Type) + resolver.saveServiceIP(service.Spec.ClusterIP, serviceHostname, event.Type) } if service.Status.LoadBalancer.Ingress != nil { for _, ingress := range service.Status.LoadBalancer.Ingress { @@ -147,6 +154,14 @@ func (resolver *Resolver) saveResolvedName(key string, resolved string, eventTyp } } +func (resolver *Resolver) saveServiceIP(key string, resolved string, eventType watch.EventType) { + if eventType == watch.Deleted { + delete(resolver.serviceMap, key) + } else { + resolver.serviceMap[key] = resolved + } +} + func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun func(ctx context.Context) error) { for { err := fun(ctx) diff --git a/api/pkg/utils/utils.go b/api/pkg/utils/utils.go index 9492c9b55..c50f65c9d 100644 --- a/api/pkg/utils/utils.go +++ b/api/pkg/utils/utils.go @@ -70,14 +70,15 @@ func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails { service = SetHostname(service, entry.ResolvedDestination) } return models.BaseEntryDetails{ - Id: entry.EntryId, - Url: entryUrl, - Service: service, - Path: entry.Path, - StatusCode: entry.Status, - Method: entry.Method, - Timestamp: entry.Timestamp, + Id: entry.EntryId, + Url: entryUrl, + Service: service, + Path: entry.Path, + StatusCode: entry.Status, + Method: entry.Method, + Timestamp: entry.Timestamp, RequestSenderIp: entry.RequestSenderIp, + IsOutgoing: entry.IsOutgoing, } } diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 69c4b988a..61e894e3a 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -17,6 +17,7 @@ type MizuTapOptions struct { MizuImage string MizuPodPort uint16 PlainTextFilterRegexes []string + Direction string } @@ -39,6 +40,10 @@ var tapCmd = &cobra.Command{ return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], err)) } + if mizuTapOptions.Direction != "in" && mizuTapOptions.Direction != "any" { + return errors.New(fmt.Sprintf("%s is not a valid value for flag --direction. Acceptable values are in/any.", mizuTapOptions.Direction)) + } + RunMizuTap(regex, mizuTapOptions) return nil }, @@ -54,4 +59,5 @@ func init() { tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod") tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies") + tapCmd.Flags().StringVarP(&mizuTapOptions.Direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any") } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 6c0cc7d9b..8fbfe72fa 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -60,8 +60,6 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { //block until exit signal or error waitForFinish(ctx, cancel) - - // TODO handle incoming traffic from tapper using a channel } func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error { @@ -123,6 +121,7 @@ func createMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace), nodeToTappedPodIPMap, mizuServiceAccountExists, + tappingOptions.Direction, ); err != nil { fmt.Printf("Error creating mizu tapper daemonset: %v\n", err) return err diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 09b000bcb..7e80288b6 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -226,19 +226,30 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{}) } -func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool) error { +func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool, direction string) error { nodeToTappedPodIPMapJsonStr, err := json.Marshal(nodeToTappedPodIPMap) if err != nil { return err } + mizuCmd := []string{ + "./mizuagent", + "-i", "any", + "--tap", + "--hardump", + "--aggregator-address", fmt.Sprintf("ws://%s/wsTapper", aggregatorPodIp), + } + if direction == "any" { + mizuCmd = append(mizuCmd, "--anydirection") + } + privileged := true agentContainer := applyconfcore.Container() agentContainer.WithName(tapperPodName) agentContainer.WithImage(podImage) agentContainer.WithImagePullPolicy(core.PullAlways) agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(privileged)) - agentContainer.WithCommand("./mizuagent", "-i", "any", "--tap", "--hardump", "--aggregator-address", fmt.Sprintf("ws://%s/wsTapper", aggregatorPodIp)) + agentContainer.WithCommand(mizuCmd...) agentContainer.WithEnv( applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"), applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodIPMapJsonStr)), diff --git a/shared/models.go b/shared/models.go index cbf73b594..854dbf1f6 100644 --- a/shared/models.go +++ b/shared/models.go @@ -3,7 +3,7 @@ package shared type WebSocketMessageType string const ( WebSocketMessageTypeEntry WebSocketMessageType = "entry" - WebSocketMessageTypeTappedEntry WebSocketMessageType = "tappedEntry" + WebSocketMessageTypeTappedEntry WebSocketMessageType = "tappedEntry" WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status" ) diff --git a/tap/http_matcher.go b/tap/http_matcher.go index a0377cd46..6e0393e5c 100644 --- a/tap/http_matcher.go +++ b/tap/http_matcher.go @@ -14,18 +14,10 @@ type requestResponsePair struct { Response httpMessage `json:"response"` } -type ConnectionInfo struct { - ClientIP string - ClientPort string - ServerIP string - ServerPort string -} - type httpMessage struct { isRequest bool captureTime time.Time orig interface{} - connectionInfo ConnectionInfo } @@ -44,18 +36,10 @@ func (matcher *requestResponseMatcher) registerRequest(ident string, request *ht split := splitIdent(ident) key := genKey(split) - connectionInfo := &ConnectionInfo{ - ClientIP: split[0], - ClientPort: split[2], - ServerIP: split[1], - ServerPort: split[3], - } - requestHTTPMessage := httpMessage{ isRequest: true, captureTime: captureTime, orig: request, - connectionInfo: *connectionInfo, } if response, found := matcher.openMessagesMap.Pop(key); found { diff --git a/tap/http_reader.go b/tap/http_reader.go index 00d20f425..774f4c5a2 100644 --- a/tap/http_reader.go +++ b/tap/http_reader.go @@ -24,6 +24,14 @@ type tcpID struct { dstPort string } +type ConnectionInfo struct { + ClientIP string + ClientPort string + ServerIP string + ServerPort string + IsOutgoing bool +} + func (tid *tcpID) String() string { return fmt.Sprintf("%s->%s %s->%s", tid.srcIP, tid.dstIP, tid.srcPort, tid.dstPort) } @@ -38,6 +46,7 @@ type httpReader struct { tcpID tcpID isClient bool isHTTP2 bool + isOutgoing bool msgQueue chan httpReaderDataMsg // Channel of captured reassembled tcp payload data []byte captureTime time.Time @@ -121,13 +130,28 @@ func (h *httpReader) handleHTTP2Stream() error { } var reqResPair *requestResponsePair + var connectionInfo *ConnectionInfo switch messageHTTP1 := messageHTTP1.(type) { case http.Request: ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.srcIP, h.tcpID.dstIP, h.tcpID.srcPort, h.tcpID.dstPort, streamID) + connectionInfo = &ConnectionInfo{ + ClientIP: h.tcpID.srcIP, + ClientPort: h.tcpID.srcPort, + ServerIP: h.tcpID.dstIP, + ServerPort: h.tcpID.dstPort, + IsOutgoing: h.isOutgoing, + } reqResPair = reqResMatcher.registerRequest(ident, &messageHTTP1, h.captureTime) case http.Response: ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.dstIP, h.tcpID.srcIP, h.tcpID.dstPort, h.tcpID.srcPort, streamID) + connectionInfo = &ConnectionInfo{ + ClientIP: h.tcpID.dstIP, + ClientPort: h.tcpID.dstPort, + ServerIP: h.tcpID.srcIP, + ServerPort: h.tcpID.srcPort, + IsOutgoing: h.isOutgoing, + } reqResPair = reqResMatcher.registerResponse(ident, &messageHTTP1, h.captureTime) } @@ -140,7 +164,7 @@ func (h *httpReader) handleHTTP2Stream() error { reqResPair.Request.captureTime, reqResPair.Response.orig.(*http.Response), reqResPair.Response.captureTime, - &reqResPair.Request.connectionInfo, + connectionInfo, ) } } @@ -179,7 +203,13 @@ func (h *httpReader) handleHTTP1ClientStream(b *bufio.Reader) error { reqResPair.Request.captureTime, reqResPair.Response.orig.(*http.Response), reqResPair.Response.captureTime, - &reqResPair.Request.connectionInfo, + &ConnectionInfo{ + ClientIP: h.tcpID.srcIP, + ClientPort: h.tcpID.srcPort, + ServerIP: h.tcpID.dstIP, + ServerPort: h.tcpID.dstPort, + IsOutgoing: h.isOutgoing, + }, ) } } @@ -239,7 +269,13 @@ func (h *httpReader) handleHTTP1ServerStream(b *bufio.Reader) error { reqResPair.Request.captureTime, reqResPair.Response.orig.(*http.Response), reqResPair.Response.captureTime, - &reqResPair.Request.connectionInfo, + &ConnectionInfo{ + ClientIP: h.tcpID.dstIP, + ClientPort: h.tcpID.dstPort, + ServerIP: h.tcpID.srcIP, + ServerPort: h.tcpID.srcPort, + IsOutgoing: h.isOutgoing, + }, ) } } diff --git a/tap/tcp_stream_factory.go b/tap/tcp_stream_factory.go index 6808cc13e..bb268a929 100644 --- a/tap/tcp_stream_factory.go +++ b/tap/tcp_stream_factory.go @@ -27,13 +27,15 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T SupportMissingEstablishment: *allowmissinginit, } Debug("Current App Ports: %v", gSettings.filterPorts) + srcIp := net.Src().String() dstIp := net.Dst().String() dstPort := int(tcp.DstPort) if factory.shouldNotifyOnOutboundLink(dstIp, dstPort) { factory.outbountLinkWriter.WriteOutboundLink(net.Src().String(), dstIp, dstPort) } - isHTTP := factory.shouldTap(dstIp, dstPort) + props := factory.getStreamProps(srcIp, dstIp, dstPort) + isHTTP := props.isTapTarget stream := &tcpStream{ net: net, transport: transport, @@ -57,6 +59,7 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T hexdump: *hexdump, parent: stream, isClient: true, + isOutgoing: props.isOutgoing, harWriter: factory.harWriter, } stream.server = httpReader{ @@ -70,6 +73,7 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T }, hexdump: *hexdump, parent: stream, + isOutgoing: props.isOutgoing, harWriter: factory.harWriter, } factory.wg.Add(2) @@ -84,28 +88,29 @@ func (factory *tcpStreamFactory) WaitGoRoutines() { factory.wg.Wait() } -func (factory *tcpStreamFactory) shouldTap(dstIP string, dstPort int) bool { +func (factory *tcpStreamFactory) getStreamProps(srcIP string, dstIP string, dstPort int) *streamProps { if hostMode { if inArrayString(gSettings.filterAuthorities, fmt.Sprintf("%s:%d", dstIP, dstPort)) == true { - return true + return &streamProps{isTapTarget: true, isOutgoing: false} } else if inArrayString(gSettings.filterAuthorities, dstIP) == true { - return true + return &streamProps{isTapTarget: true, isOutgoing: false} + } else if *anydirection && inArrayString(gSettings.filterAuthorities, srcIP) == true { + return &streamProps{isTapTarget: true, isOutgoing: true} } - return false + return &streamProps{isTapTarget: false} } else { isTappedPort := dstPort == 80 || (gSettings.filterPorts != nil && (inArrayInt(gSettings.filterPorts, dstPort))) if !isTappedPort { - return false + return &streamProps{isTapTarget: false, isOutgoing: false} } - if !*anydirection { - isDirectedHere := inArrayString(ownIps, dstIP) - if !isDirectedHere { - return false - } + isOutgoing := !inArrayString(ownIps, dstIP) + + if !*anydirection && isOutgoing { + return &streamProps{isTapTarget: false, isOutgoing: isOutgoing} } - return true + return &streamProps{isTapTarget: true} } } @@ -116,3 +121,9 @@ func (factory *tcpStreamFactory) shouldNotifyOnOutboundLink(dstIP string, dstPor } return true } + +type streamProps struct { + isTapTarget bool + isOutgoing bool +} + diff --git a/ui/src/components/HarEntry.tsx b/ui/src/components/HarEntry.tsx index 6ef8d76b9..95f3c6ffd 100644 --- a/ui/src/components/HarEntry.tsx +++ b/ui/src/components/HarEntry.tsx @@ -2,6 +2,8 @@ import React from "react"; import styles from './style/HarEntry.module.sass'; import StatusCode from "./StatusCode"; import {EndpointPath} from "./EndpointPath"; +import ingoingIcon from "./assets/ingoing-traffic.svg" +import outgoingIcon from "./assets/outgoing-traffic.svg" interface HAREntry { method?: string, @@ -12,6 +14,7 @@ interface HAREntry { url?: string; isCurrentRevision?: boolean; timestamp: Date; + isOutgoing?: boolean; } interface HAREntryProps { @@ -33,7 +36,18 @@ export const HarEntry: React.FC = ({entry, setFocusedEntryId, isS {entry.service}
+
+ {entry.isOutgoing ? +
+ outgoing traffic +
+ : +
+ ingoing traffic +
+ } +
{new Date(+entry.timestamp)?.toLocaleString()}
-}; \ No newline at end of file +}; diff --git a/ui/src/components/assets/ingoing-traffic.svg b/ui/src/components/assets/ingoing-traffic.svg new file mode 100644 index 000000000..7aaded9a8 --- /dev/null +++ b/ui/src/components/assets/ingoing-traffic.svg @@ -0,0 +1 @@ + diff --git a/ui/src/components/assets/outgoing-traffic.svg b/ui/src/components/assets/outgoing-traffic.svg new file mode 100644 index 000000000..1c8ec87d7 --- /dev/null +++ b/ui/src/components/assets/outgoing-traffic.svg @@ -0,0 +1 @@ + diff --git a/ui/src/components/style/HarEntry.module.sass b/ui/src/components/style/HarEntry.module.sass index 4f1a8185b..3e28bfee9 100644 --- a/ui/src/components/style/HarEntry.module.sass +++ b/ui/src/components/style/HarEntry.module.sass @@ -47,4 +47,17 @@ overflow: hidden padding-right: 10px padding-left: 10px - flex-grow: 1 \ No newline at end of file + flex-grow: 1 + +.directionContainer + display: flex + width: 28px + flex-direction: column + +.outgoingIcon + display: flex + align-self: flex-end + +.ingoingIcon + display: flex + align-self: flex-start diff --git a/ui/src/components/style/variables.module.scss b/ui/src/components/style/variables.module.scss index d1152084c..d2f3d89bd 100644 --- a/ui/src/components/style/variables.module.scss +++ b/ui/src/components/style/variables.module.scss @@ -19,4 +19,4 @@ $blue-gray: #494677; successColor: $success-color; failureColor: $failure-color; blueGray: $blue-gray; -} \ No newline at end of file +} From 6aaee4b519dd8b30de2bfc4131771f19ff8aec13 Mon Sep 17 00:00:00 2001 From: nimrod-up9 <59927337+nimrod-up9@users.noreply.github.com> Date: Sun, 27 Jun 2021 16:26:12 +0300 Subject: [PATCH 35/36] Refactoring (#84) * Only use one channel for filtering HARs. * Only check if dest is service ip if message is outgoing. * Parse direction flag on input. * Renamed filterHarHeaders -> filterHarItems. * Fixed compilation bugs. --- api/main.go | 27 +++++++++------------------ cli/cmd/tap.go | 18 +++++++++++++----- cli/cmd/tapRunner.go | 2 +- cli/kubernetes/provider.go | 4 ++-- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/api/main.go b/api/main.go index 2cf1da9e4..eebac00d1 100644 --- a/api/main.go +++ b/api/main.go @@ -35,12 +35,10 @@ func main() { if *standalone { harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts) - filteredHarChannel0 := make(chan *tap.OutputChannelItem) - filteredHarChannel1 := make(chan *tap.OutputChannelItem) + filteredHarChannel := make(chan *tap.OutputChannelItem) - go filterServices(harOutputChannel, filteredHarChannel0, getTrafficFilteringOptions()) - go filterHarHeaders(filteredHarChannel0, filteredHarChannel1, getTrafficFilteringOptions()) - go api.StartReadingEntries(filteredHarChannel1, nil) + go filterHarItems(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions()) + go api.StartReadingEntries(filteredHarChannel, nil) go api.StartReadingOutbound(outboundLinkOutputChannel) hostApi(nil) @@ -66,12 +64,10 @@ func main() { go api.StartReadingOutbound(outboundLinkOutputChannel) } else if *aggregator { socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) - filteredHarChannel0 := make(chan *tap.OutputChannelItem) - filteredHarChannel1 := make(chan *tap.OutputChannelItem) + filteredHarChannel := make(chan *tap.OutputChannelItem) - go filterServices(socketHarOutChannel, filteredHarChannel0, getTrafficFilteringOptions()) - go filterHarHeaders(filteredHarChannel0, filteredHarChannel1, getTrafficFilteringOptions()) - go api.StartReadingEntries(filteredHarChannel1, nil) + go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) + go api.StartReadingEntries(filteredHarChannel, nil) hostApi(socketHarOutChannel) } @@ -129,19 +125,14 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions { return &filteringOptions } -func filterServices(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { +func filterHarItems(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { for message := range inChannel { - if api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { + if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { continue } - outChannel <- message - } -} - -func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { - for message := range inChannel { sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions) + outChannel <- message } } diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 61e894e3a..c3e6fb747 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -3,8 +3,10 @@ package cmd import ( "errors" "fmt" - "github.com/up9inc/mizu/cli/mizu" "regexp" + "strings" + + "github.com/up9inc/mizu/cli/mizu" "github.com/spf13/cobra" ) @@ -17,11 +19,12 @@ type MizuTapOptions struct { MizuImage string MizuPodPort uint16 PlainTextFilterRegexes []string - Direction string + TapOutgoing bool } var mizuTapOptions = &MizuTapOptions{} +var direction string var tapCmd = &cobra.Command{ Use: "tap [POD REGEX]", @@ -40,8 +43,13 @@ var tapCmd = &cobra.Command{ return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], err)) } - if mizuTapOptions.Direction != "in" && mizuTapOptions.Direction != "any" { - return errors.New(fmt.Sprintf("%s is not a valid value for flag --direction. Acceptable values are in/any.", mizuTapOptions.Direction)) + directionLowerCase := strings.ToLower(direction) + if directionLowerCase == "any" { + mizuTapOptions.TapOutgoing = true + } else if directionLowerCase == "in" { + mizuTapOptions.TapOutgoing = false + } else { + return errors.New(fmt.Sprintf("%s is not a valid value for flag --direction. Acceptable values are in/any.", direction)) } RunMizuTap(regex, mizuTapOptions) @@ -59,5 +67,5 @@ func init() { tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod") tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies") - tapCmd.Flags().StringVarP(&mizuTapOptions.Direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any") + tapCmd.Flags().StringVarP(&direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any") } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 8fbfe72fa..2eddcbb51 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -121,7 +121,7 @@ func createMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace), nodeToTappedPodIPMap, mizuServiceAccountExists, - tappingOptions.Direction, + tappingOptions.TapOutgoing, ); err != nil { fmt.Printf("Error creating mizu tapper daemonset: %v\n", err) return err diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 7e80288b6..e416e940c 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -226,7 +226,7 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{}) } -func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool, direction string) error { +func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool, tapOutgoing bool) error { nodeToTappedPodIPMapJsonStr, err := json.Marshal(nodeToTappedPodIPMap) if err != nil { return err @@ -239,7 +239,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac "--hardump", "--aggregator-address", fmt.Sprintf("ws://%s/wsTapper", aggregatorPodIp), } - if direction == "any" { + if tapOutgoing { mizuCmd = append(mizuCmd, "--anydirection") } From c59aadb2214b957f29b9793cc47e7aca142f9d0e Mon Sep 17 00:00:00 2001 From: nimrod-up9 <59927337+nimrod-up9@users.noreply.github.com> Date: Sun, 27 Jun 2021 18:24:14 +0300 Subject: [PATCH 36/36] Fix crash when there are no pods matching the regex (#85) * Removed done todo. * Error when trying to apply tapper-daemonset with 0 pods in affinity. * Reorder imports. * Create/update mizu tappers if there are tapped pods. Delete if there are no tapped pods. * Skip deletion if tapper daemonset is not installed. * Renamed createMizuTappers -> updateMizuTappers. * Renamed IsDaemonSetApplied -> CheckDaemonSetExists. * Skip deletion if pod / service is not installed. * Fixed: Inverted logic. * Rename. * Fixed compilation bugs. * Warn if no pods are found. Suggest changing the namespace. * Use consts. * Removed empty line. --- cli/cmd/tapRunner.go | 58 +++++++++++++++++++--------- cli/kubernetes/provider.go | 77 +++++++++++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 18 deletions(-) diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 2eddcbb51..76011e1dc 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -3,13 +3,14 @@ package cmd import ( "context" "fmt" - "github.com/up9inc/mizu/shared" "os" "os/signal" "regexp" "syscall" "time" + "github.com/up9inc/mizu/shared" + core "k8s.io/api/core/v1" "github.com/up9inc/mizu/cli/debounce" @@ -45,6 +46,22 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { currentlyTappedPods = matchingPods } + var namespacesStr string + if targetNamespace != mizu.K8sAllNamespaces { + namespacesStr = fmt.Sprintf("namespace \"%s\"", targetNamespace) + } else { + namespacesStr = "all namespaces" + } + fmt.Printf("Tapping pods in %s\n", namespacesStr) + + if len(currentlyTappedPods) == 0 { + var suggestionStr string + if targetNamespace != mizu.K8sAllNamespaces { + suggestionStr = "\nSelect a different namespace with -n or tap all namespaces with -A" + } + fmt.Printf("Did not find any pods matching the regex argument%s\n", suggestionStr) + } + nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods) if err != nil { return @@ -67,7 +84,7 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro return err } - if err := createMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil { + if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil { return err } @@ -111,20 +128,27 @@ func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.Traffic return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice}, nil } -func createMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error { - if err := kubernetesProvider.ApplyMizuTapperDaemonSet( - ctx, - mizu.ResourcesNamespace, - mizu.TapperDaemonSetName, - tappingOptions.MizuImage, - mizu.TapperPodName, - fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace), - nodeToTappedPodIPMap, - mizuServiceAccountExists, - tappingOptions.TapOutgoing, - ); err != nil { - fmt.Printf("Error creating mizu tapper daemonset: %v\n", err) - return err +func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error { + if len(nodeToTappedPodIPMap) > 0 { + if err := kubernetesProvider.ApplyMizuTapperDaemonSet( + ctx, + mizu.ResourcesNamespace, + mizu.TapperDaemonSetName, + tappingOptions.MizuImage, + mizu.TapperPodName, + fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace), + nodeToTappedPodIPMap, + mizuServiceAccountExists, + tappingOptions.TapOutgoing, + ); err != nil { + fmt.Printf("Error creating mizu tapper daemonset: %v\n", err) + return err + } + } else { + if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil { + fmt.Printf("Error deleting mizu tapper daemonset: %v\n", err) + return err + } } return nil @@ -164,7 +188,7 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro cancel() } - if err := createMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil { + if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil { fmt.Printf("Error updating daemonset: %s (%v,%+v)\n", err, err, err) cancel() } diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index e416e940c..0bf40c1b8 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -102,7 +102,6 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace }, DNSPolicy: core.DNSClusterFirstWithHostNet, TerminationGracePeriodSeconds: new(int64), - // Affinity: TODO: define node selector for all relevant nodes for this mizu instance }, } //define the service account only when it exists to prevent pod crash @@ -215,18 +214,94 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, } func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error { + if isFound, err := provider.CheckPodExists(ctx, namespace, podName); + err != nil { + return err + } else if !isFound { + return nil + } + return provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) } func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error { + if isFound, err := provider.CheckServiceExists(ctx, namespace, serviceName); + err != nil { + return err + } else if !isFound { + return nil + } + return provider.clientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{}) } func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error { + if isFound, err := provider.CheckDaemonSetExists(ctx, namespace, daemonSetName); + err != nil { + return err + } else if !isFound { + return nil + } + return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{}) } +func (provider *Provider) CheckPodExists(ctx context.Context, namespace string, name string) (bool, error) { + listOptions := metav1.ListOptions{ + FieldSelector: fmt.Sprintf("metadata.name=%s", name), + Limit: 1, + } + resourceList, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, listOptions) + if err != nil { + return false, err + } + + if len(resourceList.Items) > 0 { + return true, nil + } + + return false, nil +} + +func (provider *Provider) CheckServiceExists(ctx context.Context, namespace string, name string) (bool, error) { + listOptions := metav1.ListOptions{ + FieldSelector: fmt.Sprintf("metadata.name=%s", name), + Limit: 1, + } + resourceList, err := provider.clientSet.CoreV1().Services(namespace).List(ctx, listOptions) + if err != nil { + return false, err + } + + if len(resourceList.Items) > 0 { + return true, nil + } + + return false, nil +} + +func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace string, name string) (bool, error) { + listOptions := metav1.ListOptions{ + FieldSelector: fmt.Sprintf("metadata.name=%s", name), + Limit: 1, + } + resourceList, err := provider.clientSet.AppsV1().DaemonSets(namespace).List(ctx, listOptions) + if err != nil { + return false, err + } + + if len(resourceList.Items) > 0 { + return true, nil + } + + return false, nil +} + func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool, tapOutgoing bool) error { + if len(nodeToTappedPodIPMap) == 0 { + return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName) + } + nodeToTappedPodIPMapJsonStr, err := json.Marshal(nodeToTappedPodIPMap) if err != nil { return err