github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-06-27 08:39:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Have own HAR library code (#695)

Browse Source
This commit is contained in:
Andrey Pokhilko 2022-01-26 11:57:18 +03:00 committed by GitHub
parent 843ac722c9
commit 0c56c0f541
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 526 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
agent/pkg/api/main.go
View File

@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"mizuserver/pkg/har"
"mizuserver/pkg/holder"
"mizuserver/pkg/providers"
"os"
@ -15,11 +16,9 @@ import (
"mizuserver/pkg/servicemap"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api"
"mizuserver/pkg/models"
"mizuserver/pkg/oas"
"mizuserver/pkg/resolver"
@ -132,7 +131,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
mizuEntry.ContractContent = contract.Content
}
harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
if err == nil {
rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
mizuEntry.Rules = rules

4
agent/pkg/controllers/entries_controller.go
View File

@ -2,8 +2,8 @@ package controllers
import (
"encoding/json"
"mizuserver/pkg/har"
"mizuserver/pkg/models"
"mizuserver/pkg/utils"
"mizuserver/pkg/validation"
"net/http"
"strconv"
@ -127,7 +127,7 @@ func GetEntry(c *gin.Context) {
var rules []map[string]interface{}
var isRulesEnabled bool
if entry.Protocol.Name == "http" {
harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
isRulesEnabled = _isRulesEnabled
inrec, _ := json.Marshal(rulesMatched)

375
agent/pkg/har/types.go Normal file
View File

@ -0,0 +1,375 @@
package har
import (
"encoding/base64"
"github.com/up9inc/mizu/shared/logger"
"time"
"unicode/utf8"
)
/*
HTTP Archive (HAR) format
https://w3c.github.io/web-performance/specs/HAR/Overview.html
*/
// HAR is a container type for deserialization
type HAR struct {
Log Log `json:"log"`
}
// Log represents the root of the exported data. This object MUST be present and its name MUST be "log".
type Log struct {
// The object contains the following name/value pairs:
// Required. Version number of the format.
Version string `json:"version"`
// Required. An object of type creator that contains the name and version
// information of the log creator application.
Creator Creator `json:"creator"`
// Optional. An object of type browser that contains the name and version
// information of the user agent.
Browser Browser `json:"browser"`
// Optional. An array of objects of type page, each representing one exported
// (tracked) page. Leave out this field if the application does not support
// grouping by pages.
Pages []Page `json:"pages,omitempty"`
// Required. An array of objects of type entry, each representing one
// exported (tracked) HTTP request.
Entries []Entry `json:"entries"`
// Optional. A comment provided by the user or the application. Sorting
// entries by startedDateTime (starting from the oldest) is preferred way how
// to export data since it can make importing faster. However the reader
// application should always make sure the array is sorted (if required for
// the import).
Comment string `json:"comment"`
}
// Creator contains information about the log creator application
type Creator struct {
// Required. The name of the application that created the log.
Name string `json:"name"`
// Required. The version number of the application that created the log.
Version string `json:"version"`
// Optional. A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Browser that created the log
type Browser struct {
// Required. The name of the browser that created the log.
Name string `json:"name"`
// Required. The version number of the browser that created the log.
Version string `json:"version"`
// Optional. A comment provided by the user or the browser.
Comment string `json:"comment"`
}
// Page object for every exported web page and one <entry> object for every HTTP request.
// In case when an HTTP trace tool isn't able to group requests by a page,
// the <pages> object is empty and individual requests doesn't have a parent page.
type Page struct {
/* There is one <page> object for every exported web page and one <entry>
object for every HTTP request. In case when an HTTP trace tool isn't able to
group requests by a page, the <pages> object is empty and individual
requests doesn't have a parent page.
*/
// Date and time stamp for the beginning of the page load
// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00).
StartedDateTime string `json:"startedDateTime"`
// Unique identifier of a page within the . Entries use it to refer the parent page.
ID string `json:"id"`
// Page title.
Title string `json:"title"`
// Detailed timing info about page load.
PageTiming PageTiming `json:"pageTiming"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// PageTiming describes timings for various events (states) fired during the page load.
// All times are specified in milliseconds. If a time info is not available appropriate field is set to -1.
type PageTiming struct {
// Content of the page loaded. Number of milliseconds since page load started
// (page.startedDateTime). Use -1 if the timing does not apply to the current
// request.
// Depeding on the browser, onContentLoad property represents DOMContentLoad
// event or document.readyState == interactive.
OnContentLoad int `json:"onContentLoad"`
// Page is loaded (onLoad event fired). Number of milliseconds since page
// load started (page.startedDateTime). Use -1 if the timing does not apply
// to the current request.
OnLoad int `json:"onLoad"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment"`
}
// Entry is a unique, optional Reference to the parent page.
// Leave out this field if the application does not support grouping by pages.
type Entry struct {
Pageref string `json:"pageref,omitempty"`
// Date and time stamp of the request start
// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD).
StartedDateTime string `json:"startedDateTime"`
// Total elapsed time of the request in milliseconds. This is the sum of all
// timings available in the timings object (i.e. not including -1 values) .
Time int `json:"time"`
// Detailed info about the request.
Request Request `json:"request"`
// Detailed info about the response.
Response Response `json:"response"`
// Info about cache usage.
Cache Cache `json:"cache"`
// Detailed timing info about request/response round trip.
PageTimings PageTimings `json:"pageTimings"`
// optional (new in 1.2) IP address of the server that was connected
// (result of DNS resolution).
ServerIPAddress string `json:"serverIPAddress,omitempty"`
// optional (new in 1.2) Unique ID of the parent TCP/IP connection, can be
// the client port number. Note that a port number doesn't have to be unique
// identifier in cases where the port is shared for more connections. If the
// port isn't available for the application, any other unique connection ID
// can be used instead (e.g. connection index). Leave out this field if the
// application doesn't support this info.
Connection string `json:"connection,omitempty"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Request contains detailed info about performed request.
type Request struct {
// Request method (GET, POST, ...).
Method string `json:"method"`
// Absolute URL of the request (fragments are not included).
URL string `json:"url"`
// Request HTTP Version.
HTTPVersion string `json:"httpVersion"`
// List of cookie objects.
Cookies []Cookie `json:"cookies"`
// List of header objects.
Headers []NVP `json:"headers"`
// List of query parameter objects.
QueryString []NVP `json:"queryString"`
// Posted data.
PostData PostData `json:"postData"`
// Total number of bytes from the start of the HTTP request message until
// (and including) the double CRLF before the body. Set to -1 if the info
// is not available.
HeaderSize int `json:"headerSize"`
// Size of the request body (POST data payload) in bytes. Set to -1 if the
// info is not available.
BodySize int `json:"bodySize"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment"`
}
// Response contains detailed info about the response.
type Response struct {
// Response status.
Status int `json:"status"`
// Response status description.
StatusText string `json:"statusText"`
// Response HTTP Version.
HTTPVersion string `json:"httpVersion"`
// List of cookie objects.
Cookies []Cookie `json:"cookies"`
// List of header objects.
Headers []NVP `json:"headers"`
// Details about the response body.
Content Content `json:"content"`
// Redirection target URL from the Location response header.
RedirectURL string `json:"redirectURL"`
// Total number of bytes from the start of the HTTP response message until
// (and including) the double CRLF before the body. Set to -1 if the info is
// not available.
// The size of received response-headers is computed only from headers that
// are really received from the server. Additional headers appended by the
// browser are not included in this number, but they appear in the list of
// header objects.
HeadersSize int `json:"headersSize"`
// Size of the received response body in bytes. Set to zero in case of
// responses coming from the cache (304). Set to -1 if the info is not
// available.
BodySize int `json:"bodySize"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Cookie contains list of all cookies (used in <request> and <response> objects).
type Cookie struct {
// The name of the cookie.
Name string `json:"name"`
// The cookie value.
Value string `json:"value"`
// optional The path pertaining to the cookie.
Path string `json:"path,omitempty"`
// optional The host of the cookie.
Domain string `json:"domain,omitempty"`
// optional Cookie expiration time.
// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00).
Expires string `json:"expires,omitempty"`
// optional Set to true if the cookie is HTTP only, false otherwise.
HTTPOnly bool `json:"httpOnly,omitempty"`
// optional (new in 1.2) True if the cookie was transmitted over ssl, false
// otherwise.
Secure bool `json:"secure,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// NVP is simply a name/value pair with a comment
type NVP struct {
Name string `json:"name"`
Value string `json:"value"`
Comment string `json:"comment,omitempty"`
}
// PostData describes posted data, if any (embedded in <request> object).
type PostData struct {
// Mime type of posted data.
MimeType string `json:"mimeType"`
// List of posted parameters (in case of URL encoded parameters).
Params []PostParam `json:"params"`
// Plain text posted data
Text string `json:"text"`
// optional (new in 1.2) A comment provided by the user or the
// application.
Comment string `json:"comment,omitempty"`
}
func (d PostData) B64Decoded() (bool, []byte, string) {
// there is a weird gap in HAR spec 1.2, that does not define encoding for binary POST bodies
// we have own convention of putting `base64` into comment field to handle it similar to response `Content`
return b64Decoded(d.Comment, d.Text)
}
// PostParam is a list of posted parameters, if any (embedded in <postData> object).
type PostParam struct {
// name of a posted parameter.
Name string `json:"name"`
// optional value of a posted parameter or content of a posted file.
Value string `json:"value,omitempty"`
// optional name of a posted file.
FileName string `json:"fileName,omitempty"`
// optional content type of a posted file.
ContentType string `json:"contentType,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Content describes details about response content (embedded in <response> object).
type Content struct {
// Length of the returned content in bytes. Should be equal to
// response.bodySize if there is no compression and bigger when the content
// has been compressed.
Size int `json:"size"`
// optional Number of bytes saved. Leave out this field if the information
// is not available.
Compression int `json:"compression,omitempty"`
// MIME type of the response text (value of the Content-Type response
// header). The charset attribute of the MIME type is included (if
// available).
MimeType string `json:"mimeType"`
// optional Response body sent from the server or loaded from the browser
// cache. This field is populated with textual content only. The text field
// is either HTTP decoded text or a encoded (e.g. "base64") representation of
// the response body. Leave out this field if the information is not
// available.
Text string `json:"text,omitempty"`
// optional (new in 1.2) Encoding used for response text field e.g
// "base64". Leave out this field if the text field is HTTP decoded
// (decompressed & unchunked), than trans-coded from its original character
// set into UTF-8.
Encoding string `json:"encoding,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
func (c Content) B64Decoded() (bool, []byte, string) {
return b64Decoded(c.Encoding, c.Text)
}
func b64Decoded(enc string, text string) (isBinary bool, asBytes []byte, asString string) {
if enc == "base64" {
decoded, err := base64.StdEncoding.DecodeString(text)
if err != nil {
logger.Log.Warningf("Failed to decode content as base64: %s", text)
return false, []byte(text), text
}
valid := utf8.Valid(decoded)
return !valid, decoded, string(decoded)
} else {
return false, nil, text
}
}
// Cache contains info about a request coming from browser cache.
type Cache struct {
// optional State of a cache entry before the request. Leave out this field
// if the information is not available.
BeforeRequest CacheObject `json:"beforeRequest,omitempty"`
// optional State of a cache entry after the request. Leave out this field if
// the information is not available.
AfterRequest CacheObject `json:"afterRequest,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// CacheObject is used by both beforeRequest and afterRequest
type CacheObject struct {
// optional - Expiration time of the cache entry.
Expires string `json:"expires,omitempty"`
// The last time the cache entry was opened.
LastAccess string `json:"lastAccess"`
// Etag
ETag string `json:"eTag"`
// The number of times the cache entry has been opened.
HitCount int `json:"hitCount"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// PageTimings describes various phases within request-response round trip.
// All times are specified in milliseconds.
type PageTimings struct {
Blocked int `json:"blocked,omitempty"`
// optional - Time spent in a queue waiting for a network connection. Use -1
// if the timing does not apply to the current request.
DNS int `json:"dns,omitempty"`
// optional - DNS resolution time. The time required to resolve a host name.
// Use -1 if the timing does not apply to the current request.
Connect int `json:"connect,omitempty"`
// optional - Time required to create TCP connection. Use -1 if the timing
// does not apply to the current request.
Send int `json:"send"`
// Time required to send HTTP request to the server.
Wait int `json:"wait"`
// Waiting for a response from the server.
Receive int `json:"receive"`
// Time required to read entire response from the server (or cache).
Ssl int `json:"ssl,omitempty"`
// optional (new in 1.2) - Time required for SSL/TLS negotiation. If this
// field is defined then the time is also included in the connect field (to
// ensure backward compatibility with HAR 1.1). Use -1 if the timing does not
// apply to the current request.
Comment string `json:"comment,omitempty"`
// optional (new in 1.2) - A comment provided by the user or the application.
}
// TestResult contains results for an individual HTTP request
type TestResult struct {
URL string `json:"url"`
Status int `json:"status"` // 200, 500, etc.
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime"`
Latency int `json:"latency"` // milliseconds
Method string `json:"method"`
HarFile string `json:"harfile"`
}
// aliases for martian lib compatibility
type Header = NVP
type QueryString = NVP
type Param = PostParam
type Timings = PageTimings

38
agent/pkg/har/types_test.go Normal file
View File

@ -0,0 +1,38 @@
package har
import "testing"
func TestContentEncoded(t *testing.T) {
testCases := []struct {
text string
isBinary bool
expectedStr string
binaryLen int
}{
{"not-base64", false, "not-base64", 10},
{"dGVzdA==", false, "test", 4},
{"test", true, "\f@A", 3}, // valid UTF-8 with some non-printable chars
{"IsDggPCAgPiAgID8gICAgN/vv/e/v/u/v7/9v7+/vyIKIu+3kO+3ke+3ku+3k++3lO+3le+3lu+3l++3mO+3me+3mu+3m++3nO+3ne+3nu+3n++3oO+3oe+3ou+3o++3pO+3pe+3pu+3p++3qO+3qe+3qu+3q++3rO+3re+3ru+3ryIK", true, "test", 132}, // invalid UTF-8 (thus binary), taken from https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
}
for _, tc := range testCases {
c := Content{
Encoding: "base64",
Text: tc.text,
}
isBinary, asBytes, asString := c.B64Decoded()
_ = asBytes
if tc.isBinary != isBinary {
t.Errorf("Binary flag mismatch: %t != %t", tc.isBinary, isBinary)
}
if !isBinary && tc.expectedStr != asString {
t.Errorf("Decode value mismatch: %s != %s", tc.expectedStr, asString)
}
if tc.binaryLen != len(asBytes) {
t.Errorf("Binary len mismatch: %d != %d", tc.binaryLen, len(asBytes))
}
}
}

69
agent/pkg/utils/har.go → agent/pkg/har/utils.go
View File

@ -1,4 +1,4 @@
package utils
package har
import (
"bytes"
@ -8,7 +8,6 @@ import (
"strings"
"time"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared/logger"
)
@ -55,14 +54,14 @@ import (
// return cookies
//}
func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, string, string, string) {
func BuildHeaders(rawHeaders []interface{}) ([]Header, string, string, string, string, string) {
var host, scheme, authority, path, status string
headers := make([]har.Header, 0, len(rawHeaders))
headers := make([]Header, 0, len(rawHeaders))
for _, header := range rawHeaders {
h := header.(map[string]interface{})
headers = append(headers, har.Header{
headers = append(headers, Header{
Name: h["name"].(string),
Value: h["value"].(string),
})
@ -87,8 +86,8 @@ func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, strin
return headers, host, scheme, authority, path, status
}
func BuildPostParams(rawParams []interface{}) []har.Param {
params := make([]har.Param, 0, len(rawParams))
func BuildPostParams(rawParams []interface{}) []Param {
params := make([]Param, 0, len(rawParams))
for _, param := range rawParams {
p := param.(map[string]interface{})
name := ""
@ -108,10 +107,10 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
contentType = p["contentType"].(string)
}
params = append(params, har.Param{
params = append(params, Param{
Name: name,
Value: value,
Filename: fileName,
FileName: fileName,
ContentType: contentType,
})
}
@ -119,9 +118,9 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
return params
}
func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) {
func NewRequest(request map[string]interface{}) (harRequest *Request, err error) {
headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
postData, _ := request["postData"].(map[string]interface{})
mimeType, _ := postData["mimeType"]
@ -134,10 +133,10 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
postDataText = text.(string)
}
queryString := make([]har.QueryString, 0)
queryString := make([]QueryString, 0)
for _, _qs := range request["_queryString"].([]interface{}) {
qs := _qs.(map[string]interface{})
queryString = append(queryString, har.QueryString{
queryString = append(queryString, QueryString{
Name: qs["name"].(string),
Value: qs["value"].(string),
})
@ -148,21 +147,21 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
}
harParams := make([]har.Param, 0)
harParams := make([]Param, 0)
if postData["params"] != nil {
harParams = BuildPostParams(postData["params"].([]interface{}))
}
harRequest = &har.Request{
harRequest = &Request{
Method: request["method"].(string),
URL: url,
HTTPVersion: request["httpVersion"].(string),
HeadersSize: -1,
BodySize: int64(bytes.NewBufferString(postDataText).Len()),
HeaderSize: -1,
BodySize: bytes.NewBufferString(postDataText).Len(),
QueryString: queryString,
Headers: headers,
Cookies: cookies,
PostData: &har.PostData{
PostData: PostData{
MimeType: mimeType.(string),
Params: harParams,
Text: postDataText,
@ -172,9 +171,9 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
return
}
func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) {
func NewResponse(response map[string]interface{}) (harResponse *Response, err error) {
headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
content, _ := response["content"].(map[string]interface{})
mimeType, _ := content["mimeType"]
@ -188,11 +187,11 @@ func NewResponse(response map[string]interface{}) (harResponse *har.Response, er
bodyText = text.(string)
}
harContent := &har.Content{
harContent := &Content{
Encoding: encoding.(string),
MimeType: mimeType.(string),
Text: []byte(bodyText),
Size: int64(len(bodyText)),
Text: bodyText,
Size: len(bodyText),
}
status := int(response["status"].(float64))
@ -206,20 +205,20 @@ func NewResponse(response map[string]interface{}) (harResponse *har.Response, er
}
}
harResponse = &har.Response{
harResponse = &Response{
HTTPVersion: response["httpVersion"].(string),
Status: status,
StatusText: response["statusText"].(string),
HeadersSize: -1,
BodySize: int64(bytes.NewBufferString(bodyText).Len()),
BodySize: bytes.NewBufferString(bodyText).Len(),
Headers: headers,
Cookies: cookies,
Content: harContent,
Content: *harContent,
}
return
}
func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) {
func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*Entry, error) {
harRequest, err := NewRequest(request)
if err != nil {
logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
@ -236,16 +235,16 @@ func NewEntry(request map[string]interface{}, response map[string]interface{}, s
elapsedTime = 1
}
harEntry := har.Entry{
StartedDateTime: startTime,
Time: elapsedTime,
Request: harRequest,
Response: harResponse,
Cache: &har.Cache{},
Timings: &har.Timings{
harEntry := Entry{
StartedDateTime: startTime.Format(time.RFC3339),
Time: int(elapsedTime),
Request: *harRequest,
Response: *harResponse,
Cache: Cache{},
PageTimings: PageTimings{
Send: -1,
Wait: -1,
Receive: elapsedTime,
Receive: int(elapsedTime),
},
}

5
agent/pkg/models/models.go
View File

@ -2,11 +2,10 @@ package models
import (
"encoding/json"
tapApi "github.com/up9inc/mizu/tap/api"
"mizuserver/pkg/har"
"mizuserver/pkg/rules"
tapApi "github.com/up9inc/mizu/tap/api"
"github.com/google/martian/har"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"

4
agent/pkg/oas/feeder_test.go
View File

@ -4,10 +4,10 @@ import (
"bufio"
"encoding/json"
"errors"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared/logger"
"io"
"io/ioutil"
"mizuserver/pkg/har"
"os"
"path/filepath"
"sort"
@ -96,7 +96,7 @@ func feedFromHAR(file string) error {
}
for _, entry := range harDoc.Log.Entries {
GetOasGeneratorInstance().PushEntry(entry)
GetOasGeneratorInstance().PushEntry(&entry)
}
return nil

4
agent/pkg/oas/oas_generator.go
View File

@ -3,8 +3,8 @@ package oas
import (
"context"
"encoding/json"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared/logger"
"mizuserver/pkg/har"
"net/url"
"sync"
)
@ -17,7 +17,7 @@ var (
func GetOasGeneratorInstance() *oasGenerator {
syncOnce.Do(func() {
instance = newOasGenerator()
logger.Log.Debug("Oas Generator Initialized")
logger.Log.Debug("OAS Generator Initialized")
})
return instance
}

62
agent/pkg/oas/specgen.go
View File

@ -1,14 +1,13 @@
package oas
import (
"encoding/base64"
"encoding/json"
"errors"
"github.com/chanced/openapi"
"github.com/google/martian/har"
"github.com/google/uuid"
"github.com/up9inc/mizu/shared/logger"
"mime"
"mizuserver/pkg/har"
"net/url"
"strconv"
"strings"
@ -178,7 +177,7 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
}
ctype := getRespCtype(entry.Response)
ctype := getRespCtype(&entry.Response)
if isCtypeIgnored(ctype) {
logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
}
@ -216,12 +215,12 @@ func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation
return nil, nil
}
err = handleRequest(entry.Request, opObj, isSuccess)
err = handleRequest(&entry.Request, opObj, isSuccess)
if err != nil {
return nil, err
}
err = handleResponse(entry.Response, opObj, isSuccess)
err = handleResponse(&entry.Response, opObj, isSuccess)
if err != nil {
return nil, err
}
@ -252,7 +251,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
}
handleNameVals(hdrGW, &opObj.Parameters)
if req.PostData != nil && req.PostData.Text != "" && isSuccess {
if req.PostData.Text != "" && isSuccess {
reqBody, err := getRequestBody(req, opObj, isSuccess)
if err != nil {
return err
@ -342,43 +341,34 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err
}
var text string
var isBinary bool
if reqResp.Req != nil {
text = reqResp.Req.PostData.Text
isBinary, _, text = reqResp.Req.PostData.B64Decoded()
} else {
text = decRespText(reqResp.Resp.Content)
isBinary, _, text = reqResp.Resp.Content.B64Decoded()
}
var exampleMsg []byte
// try treating it as json
any, isJSON := anyJSON(text)
if isJSON {
// re-marshal with forced indent
exampleMsg, err = json.MarshalIndent(any, "", "\t")
if err != nil {
panic("Failed to re-marshal value, super-strange")
}
} else {
exampleMsg, err = json.Marshal(text)
if err != nil {
return nil, err
}
}
content.Example = exampleMsg
return respContent[ctype], nil
}
func decRespText(content *har.Content) (res string) {
res = string(content.Text)
if content.Encoding == "base64" {
data, err := base64.StdEncoding.DecodeString(res)
if err != nil {
logger.Log.Warningf("error decoding response text as base64: %s", err)
if !isBinary {
var exampleMsg []byte
// try treating it as json
any, isJSON := anyJSON(text)
if isJSON {
// re-marshal with forced indent
exampleMsg, err = json.MarshalIndent(any, "", "\t")
if err != nil {
panic("Failed to re-marshal value, super-strange")
}
} else {
res = string(data)
exampleMsg, err = json.Marshal(text)
if err != nil {
return nil, err
}
}
content.Example = exampleMsg
}
return
return respContent[ctype], nil
}
func getRespCtype(resp *har.Response) string {

2
agent/pkg/oas/specgen_test.go
View File

@ -3,9 +3,9 @@ package oas
import (
"encoding/json"
"github.com/chanced/openapi"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared/logger"
"io/ioutil"
"mizuserver/pkg/har"
"os"
"strings"
"testing"

18
agent/pkg/oas/tree.go
View File

@ -27,7 +27,7 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod
chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
var paramObj *openapi.ParameterObj
if chunkIsParam && pathObjToSet != nil {
if chunkIsParam && pathObjToSet != nil && pathObjToSet.Parameters != nil {
paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
}
@ -82,13 +82,15 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod
func (n *Node) createParam() *openapi.ParameterObj {
name := "param"
// REST assumption, not always correct
if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
name = *n.constant
name = name[:len(name)-2] + "Id"
} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
name = *n.constant
name = name[:len(name)-1] + "Id"
if n.constant != nil { // the node is already a param
// REST assumption, not always correct
if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
name = *n.constant
name = name[:len(name)-2] + "Id"
} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
name = *n.constant
name = name[:len(name)-1] + "Id"
}
}
newParam := createSimpleParam(name, "path", "string")

26
agent/pkg/oas/tree_test.go Normal file
View File

@ -0,0 +1,26 @@
package oas
import (
"github.com/chanced/openapi"
"strings"
"testing"
)
func TestTree(t *testing.T) {
testCases := []struct {
inp string
}{
{"/"},
{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation"},
}
tree := new(Node)
for _, tc := range testCases {
split := strings.Split(tc.inp, "/")
node := tree.getOrSet(split, new(openapi.PathObj))
if node.constant == nil {
t.Errorf("nil constant: %s", tc.inp)
}
}
}

2
agent/pkg/oas/utils.go
View File

@ -4,8 +4,8 @@ import (
"encoding/json"
"errors"
"github.com/chanced/openapi"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared/logger"
"mizuserver/pkg/har"
"strconv"
"strings"
)

4
agent/pkg/rules/rulesHTTP.go
View File

@ -4,15 +4,15 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"mizuserver/pkg/har"
"reflect"
"regexp"
"strings"
"github.com/up9inc/mizu/shared/logger"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
jsonpath "github.com/yalp/jsonpath"
"github.com/yalp/jsonpath"
)
type RulesMatched struct {

10
agent/pkg/up9/main.go
View File

@ -3,10 +3,10 @@ package up9
import (
"bytes"
"compress/zlib"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"mizuserver/pkg/har"
"mizuserver/pkg/utils"
"net/http"
"net/url"
@ -15,7 +15,6 @@ import (
"sync"
"time"
"github.com/google/martian/har"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
@ -247,7 +246,7 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
continue
}
harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
harEntry, err := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
if err != nil {
continue
}
@ -259,11 +258,6 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.Destination.Name)
}
// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
continue
}
batch = append(batch, *harEntry)
now := time.Now()

3
tap/api/api.go
View File

@ -6,12 +6,11 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/google/martian/har"
"io/ioutil"
"net/http"
"sync"
"time"
"github.com/google/martian/har"
)
type Protocol struct {