From abee5d4eeec5a7a814dde11e7e91a06e459c1727 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Wed, 28 Apr 2021 13:47:56 +0300 Subject: [PATCH 1/4] no message --- api/Dockerfile | 17 +++++++----- api/go.mod | 2 ++ api/go.sum | 33 +++++++++++++++++++++- api/main.go | 60 ++++++++++++++++++++++++++++++++++------ api/pkg/inserter/main.go | 20 ++++++++++++-- 5 files changed, 112 insertions(+), 20 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 443e010b6..5fac361f5 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,6 +1,6 @@ -FROM golang:1.16-alpine AS builder +FROM golang:1.16 AS builder -# Move to working directory (/build). +# Move to working directory (/site). WORKDIR /build # Copy and download dependency using go mod. @@ -10,17 +10,20 @@ RUN go mod download # Copy the code into the container. COPY . . -# Set necessary environmet variables needed for our image and build the API server. -ENV CGO_ENABLED=0 GOOS=linux GOARCH=amd64 +# Set necessary environmet variables needed for our image and site the API server. +ENV GOOS=linux GOARCH=amd64 RUN go build -ldflags="-s -w" -o apiserver . -FROM scratch +FROM golang:1.16 -# Copy binary and config files from /build to root folder of scratch container. +# Copy binary and config files from /site to root folder of scratch container. COPY --from=builder ["/build/apiserver", "/"] # Export necessary port. -EXPOSE 5000 +EXPOSE 8899 + +COPY site /go/site +COPY entries.db /go/entries.db # Command to run when starting the container. ENTRYPOINT ["/apiserver"] diff --git a/api/go.mod b/api/go.mod index 6ceb5579f..7b0075d02 100644 --- a/api/go.mod +++ b/api/go.mod @@ -3,7 +3,9 @@ module mizuserver go 1.16 require ( + github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a github.com/djherbis/atime v1.0.0 + github.com/fasthttp/websocket v1.4.3-beta.1 // indirect github.com/gofiber/fiber/v2 v2.8.0 github.com/google/martian v2.1.0+incompatible go.mongodb.org/mongo-driver v1.5.1 diff --git a/api/go.sum b/api/go.sum index b4a4894b1..070e59a2b 100644 --- a/api/go.sum +++ b/api/go.sum @@ -1,12 +1,18 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a h1:76llBleIE3fkdqaJFDzdirtiYhQPdIQem8H8r2iwA1Q= +github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a/go.mod h1:QvDfsDQDmGxUsvEeWabVZ5pp2FMXpOkwQV0L6SE6cp0= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg= github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= +github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y= +github.com/fasthttp/websocket v1.4.3-beta.1 h1:stc4P2aoxYKsdmbe1AJ5mAm73Fxc1NOgrZpPftvZIXQ= +github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2tkCJZQh7yJ5/XCXO2g= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -33,8 +39,12 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gofiber/fiber/v2 v2.1.3/go.mod h1:MMiSv1HrDkN8Pv7NeVDYK+T/lwXOEKAvPBbLvJPCEfA= +github.com/gofiber/fiber/v2 v2.7.1/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0= github.com/gofiber/fiber/v2 v2.8.0 h1:BdWvZmg/WY/Vjtjm38aXOp1Lks1BhuyS2b7lSWSPAzk= github.com/gofiber/fiber/v2 v2.8.0/go.mod h1:Ah3IJikrKNRepl/HuVawppS25X7FWohwfCSRn7kJG28= +github.com/gofiber/websocket/v2 v2.0.3 h1:nqPGHB4LQhxKX5KJUjayOd2xiiENieS/dn6TPfCL8uk= +github.com/gofiber/websocket/v2 v2.0.3/go.mod h1:/OTEImCxORKE5unw0dWqJYovid6vZF+wB1W0aaMKs2M= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -51,10 +61,13 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -74,20 +87,29 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= +github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c h1:KKqhycXW1WVNkX7r4ekTV2gFkbhdyihlWD8c0/FiWmk= +github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasthttp v1.15.1/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= +github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= +github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs= github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= @@ -102,10 +124,14 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -119,7 +145,12 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/api/main.go b/api/main.go index bef9a79e4..47fa001dc 100644 --- a/api/main.go +++ b/api/main.go @@ -1,33 +1,75 @@ package main import ( + "fmt" + "github.com/antoniodipinto/ikisocket" "github.com/gofiber/fiber/v2" + "mizuserver/pkg/inserter" "mizuserver/pkg/middleware" "mizuserver/pkg/routes" "mizuserver/pkg/utils" ) func main() { - // TODO: to generate data - //path := "/Users/roeegadot/Downloads/output2" - //api.TestHarSavingFromFolder(path) - // TODO: disabling this line for now (this should be as part of the MAIN - // go inserter.StartReadingFiles("/var/up9hars") app := fiber.New() - middleware.FiberMiddleware(app) // Register Fiber's middleware for app. + // TODO: disabling this line for now (this should be as part of the MAIN + go inserter.StartReadingFiles("/var/up9hars") + // TODO: to generate data + // path := "/Users/roeegadot/Downloads/output2" + // api.TestHarSavingFromFolder(path) + + middleware.FiberMiddleware(app) // Register Fiber's middleware for app. // TODO: Check if working app.Static("/", "./site") - - // Simple route to know server is running + //Simple route to know server is running app.Get("/echo", func(c *fiber.Ctx) error { return c.SendString("Hello, World 👋!") }) + // Multiple event handling supported + ikisocket.On(ikisocket.EventConnect, func(ep *ikisocket.EventPayload) { + fmt.Println(fmt.Sprintf("Connection event 1 - User: %s", ep.Kws.GetStringAttribute("user_id"))) + }) + + ikisocket.On(ikisocket.EventConnect, func(ep *ikisocket.EventPayload) { + fmt.Println(fmt.Sprintf("Connection event 2 - User: %s", ep.Kws.GetStringAttribute("user_id"))) + }) + + // On message event + ikisocket.On(ikisocket.EventMessage, func(ep *ikisocket.EventPayload) { + + ikisocket.Broadcast([]byte("bla")) + fmt.Println(fmt.Sprintf("Message event - User: %s - Message: %s", ep.Kws.GetStringAttribute("user_id"), string(ep.Data))) + }) + + // On disconnect event + ikisocket.On(ikisocket.EventDisconnect, func(ep *ikisocket.EventPayload) { + // Remove the user from the local clients + fmt.Println(fmt.Sprintf("Disconnection event - User: %s", ep.Kws.GetStringAttribute("user_id"))) + }) + + // On close event + // This event is called when the server disconnects the user actively with .Close() method + ikisocket.On(ikisocket.EventClose, func(ep *ikisocket.EventPayload) { + // Remove the user from the local clients + fmt.Println(fmt.Sprintf("Close event - User: %s", ep.Kws.GetStringAttribute("user_id"))) + }) + + // On error event + ikisocket.On(ikisocket.EventError, func(ep *ikisocket.EventPayload) { + fmt.Println(fmt.Sprintf("Error event - User: %s", ep.Kws.GetStringAttribute("user_id"))) + }) + + app.Get("/ws", ikisocket.New(func(kws *ikisocket.Websocket) { + // kws.Broadcast([]byte(fmt.Sprintf("New user connected: %s and UUID: %s", userId, kws.UUID)), true) + // kws.Emit([]byte(fmt.Sprintf("Hello user with UUID: %s", kws.UUID))) + fmt.Println("User connected") + })) + routes.EntriesRoutes(app) routes.NotFoundRoute(app) utils.StartServer(app) } - diff --git a/api/pkg/inserter/main.go b/api/pkg/inserter/main.go index d236ce550..ee789cef5 100644 --- a/api/pkg/inserter/main.go +++ b/api/pkg/inserter/main.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/json" "fmt" + "github.com/antoniodipinto/ikisocket" "github.com/google/martian/har" "go.mongodb.org/mongo-driver/bson/primitive" "io" @@ -18,7 +19,6 @@ import ( "time" ) - func IsEmpty(name string) bool { f, err := os.Open(name) if err != nil { @@ -57,7 +57,7 @@ func StartReadingFiles(workingDir string) { decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar) utils.CheckErr(decErr) - for _, entry := range inputHar.Log.Entries { + for _, entry := range inputHar.Log.Entries { SaveHarToDb(*entry, "") } rmErr := os.Remove(inputFilePath) @@ -69,8 +69,9 @@ func StartReadingFiles(workingDir string) { func SaveHarToDb(entry har.Entry, source string) { entryBytes, _ := json.Marshal(entry) serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL) + entryId := primitive.NewObjectID().Hex() mizuEntry := models.MizuEntry{ - EntryId: primitive.NewObjectID().Hex(), + EntryId: entryId, Entry: string(entryBytes), // simple way to store it and not convert to bytes Service: serviceName, Url: entry.Request.URL, @@ -81,6 +82,19 @@ func SaveHarToDb(entry har.Entry, source string) { Timestamp: entry.StartedDateTime.Unix(), } database.GetEntriesTable().Create(&mizuEntry) + + baseEntry := &models.BaseEntryDetails{ + Id: entryId, + Url: entry.Request.URL, + Service: serviceName, + Path: urlPath, + StatusCode: entry.Response.Status, + Method: entry.Request.Method, + Timestamp: entry.StartedDateTime.Unix(), + } + baseEntryBytes, _ := json.Marshal(&baseEntry) + ikisocket.Broadcast(baseEntryBytes) + } func getServiceNameFromUrl(inputUrl string) (string, string) { From 5f71fcfb9fd832b4755c9d81c9c85b882033f0d8 Mon Sep 17 00:00:00 2001 From: Liraz Yehezkel Date: Wed, 28 Apr 2021 13:49:07 +0300 Subject: [PATCH 2/4] stating with web socket --- ui/package-lock.json | 5 +++++ ui/package.json | 1 + ui/src/components/HarPage.tsx | 17 +++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/ui/package-lock.json b/ui/package-lock.json index e9be368b3..50ae533bc 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -13443,6 +13443,11 @@ "prop-types": "^15.6.2" } }, + "react-use-websocket": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/react-use-websocket/-/react-use-websocket-2.6.1.tgz", + "integrity": "sha512-Nx1jUab+7eHpVftBpscgVG26UMVjy4P8ss+I3sE6LijHXC0chFej7unzH9YXElN9stEm1of+qZA+YX/ZlPIQBQ==" + }, "read-pkg": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", diff --git a/ui/package.json b/ui/package.json index 2fa70ffbf..b097d5a38 100644 --- a/ui/package.json +++ b/ui/package.json @@ -18,6 +18,7 @@ "react-dom": "^17.0.2", "react-scripts": "4.0.3", "react-syntax-highlighter": "^15.4.3", + "react-use-websocket": "^2.6.1", "typescript": "^4.2.4", "web-vitals": "^1.1.1" }, diff --git a/ui/src/components/HarPage.tsx b/ui/src/components/HarPage.tsx index 276a9219f..3e75182f7 100644 --- a/ui/src/components/HarPage.tsx +++ b/ui/src/components/HarPage.tsx @@ -6,6 +6,7 @@ import "./style/HarPage.sass"; import styles from './style/HarEntriesList.module.sass'; import {HAREntryDetailed} from "./HarEntryDetailed"; // import {HarPaging} from "./HarPaging"; +import useWebSocket from 'react-use-websocket'; const useLayoutStyles = makeStyles(() => ({ details: { @@ -30,6 +31,22 @@ export const HarPage: React.FC = () => { const [focusedEntryId, setFocusedEntryId] = useState(null); const [selectedHarEntry, setSelectedHarEntry] = useState(null); + const socketUrl = 'ws://localhost:8899/ws/1'; + const { + sendMessage, + sendJsonMessage, + lastMessage, + lastJsonMessage, + readyState, + getWebSocket + } = useWebSocket(socketUrl, { + onOpen: () => console.log('opened'), + //Will attempt to reconnect on all close events, such as server shutting down + shouldReconnect: (closeEvent) => true, + }); + + console.log(lastMessage?.data); + useEffect(() => { fetch("http://localhost:8899/api/entries") .then(response => response.json()) From 37a169b371252f8632dd642294e3af7611c2e8c1 Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Wed, 28 Apr 2021 14:47:28 +0300 Subject: [PATCH 3/4] Improvements (#12) * no message * no message --- api/Dockerfile | 17 ++++++----- api/go.mod | 2 ++ api/go.sum | 33 +++++++++++++++++++++- api/main.go | 16 +++++------ api/pkg/inserter/main.go | 50 +++++++++++++++------------------ api/pkg/routes/socket_routes.go | 43 ++++++++++++++++++++++++++++ 6 files changed, 118 insertions(+), 43 deletions(-) create mode 100644 api/pkg/routes/socket_routes.go diff --git a/api/Dockerfile b/api/Dockerfile index 443e010b6..5fac361f5 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,6 +1,6 @@ -FROM golang:1.16-alpine AS builder +FROM golang:1.16 AS builder -# Move to working directory (/build). +# Move to working directory (/site). WORKDIR /build # Copy and download dependency using go mod. @@ -10,17 +10,20 @@ RUN go mod download # Copy the code into the container. COPY . . -# Set necessary environmet variables needed for our image and build the API server. -ENV CGO_ENABLED=0 GOOS=linux GOARCH=amd64 +# Set necessary environmet variables needed for our image and site the API server. +ENV GOOS=linux GOARCH=amd64 RUN go build -ldflags="-s -w" -o apiserver . -FROM scratch +FROM golang:1.16 -# Copy binary and config files from /build to root folder of scratch container. +# Copy binary and config files from /site to root folder of scratch container. COPY --from=builder ["/build/apiserver", "/"] # Export necessary port. -EXPOSE 5000 +EXPOSE 8899 + +COPY site /go/site +COPY entries.db /go/entries.db # Command to run when starting the container. ENTRYPOINT ["/apiserver"] diff --git a/api/go.mod b/api/go.mod index 6ceb5579f..7b0075d02 100644 --- a/api/go.mod +++ b/api/go.mod @@ -3,7 +3,9 @@ module mizuserver go 1.16 require ( + github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a github.com/djherbis/atime v1.0.0 + github.com/fasthttp/websocket v1.4.3-beta.1 // indirect github.com/gofiber/fiber/v2 v2.8.0 github.com/google/martian v2.1.0+incompatible go.mongodb.org/mongo-driver v1.5.1 diff --git a/api/go.sum b/api/go.sum index b4a4894b1..070e59a2b 100644 --- a/api/go.sum +++ b/api/go.sum @@ -1,12 +1,18 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a h1:76llBleIE3fkdqaJFDzdirtiYhQPdIQem8H8r2iwA1Q= +github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a/go.mod h1:QvDfsDQDmGxUsvEeWabVZ5pp2FMXpOkwQV0L6SE6cp0= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg= github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= +github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y= +github.com/fasthttp/websocket v1.4.3-beta.1 h1:stc4P2aoxYKsdmbe1AJ5mAm73Fxc1NOgrZpPftvZIXQ= +github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2tkCJZQh7yJ5/XCXO2g= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -33,8 +39,12 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gofiber/fiber/v2 v2.1.3/go.mod h1:MMiSv1HrDkN8Pv7NeVDYK+T/lwXOEKAvPBbLvJPCEfA= +github.com/gofiber/fiber/v2 v2.7.1/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0= github.com/gofiber/fiber/v2 v2.8.0 h1:BdWvZmg/WY/Vjtjm38aXOp1Lks1BhuyS2b7lSWSPAzk= github.com/gofiber/fiber/v2 v2.8.0/go.mod h1:Ah3IJikrKNRepl/HuVawppS25X7FWohwfCSRn7kJG28= +github.com/gofiber/websocket/v2 v2.0.3 h1:nqPGHB4LQhxKX5KJUjayOd2xiiENieS/dn6TPfCL8uk= +github.com/gofiber/websocket/v2 v2.0.3/go.mod h1:/OTEImCxORKE5unw0dWqJYovid6vZF+wB1W0aaMKs2M= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -51,10 +61,13 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -74,20 +87,29 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= +github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c h1:KKqhycXW1WVNkX7r4ekTV2gFkbhdyihlWD8c0/FiWmk= +github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasthttp v1.15.1/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= +github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= +github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs= github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= @@ -102,10 +124,14 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -119,7 +145,12 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/api/main.go b/api/main.go index bef9a79e4..e1890d668 100644 --- a/api/main.go +++ b/api/main.go @@ -2,32 +2,32 @@ package main import ( "github.com/gofiber/fiber/v2" + "mizuserver/pkg/inserter" "mizuserver/pkg/middleware" "mizuserver/pkg/routes" "mizuserver/pkg/utils" ) -func main() { - // TODO: to generate data - //path := "/Users/roeegadot/Downloads/output2" - //api.TestHarSavingFromFolder(path) - // TODO: disabling this line for now (this should be as part of the MAIN - // go inserter.StartReadingFiles("/var/up9hars") + +func main() { + app := fiber.New() + go inserter.StartReadingFiles("/tmp/up9hars") // process to read files and insert to DB + middleware.FiberMiddleware(app) // Register Fiber's middleware for app. app.Static("/", "./site") - // Simple route to know server is running + //Simple route to know server is running app.Get("/echo", func(c *fiber.Ctx) error { return c.SendString("Hello, World 👋!") }) + routes.WebSocketRoutes(app) routes.EntriesRoutes(app) routes.NotFoundRoute(app) utils.StartServer(app) } - diff --git a/api/pkg/inserter/main.go b/api/pkg/inserter/main.go index d236ce550..16969afbc 100644 --- a/api/pkg/inserter/main.go +++ b/api/pkg/inserter/main.go @@ -4,10 +4,9 @@ import ( "bufio" "encoding/json" "fmt" + "github.com/antoniodipinto/ikisocket" "github.com/google/martian/har" "go.mongodb.org/mongo-driver/bson/primitive" - "io" - "io/fs" "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/utils" @@ -18,36 +17,20 @@ import ( "time" ) - -func IsEmpty(name string) bool { - f, err := os.Open(name) - if err != nil { - return false - } - defer f.Close() - - _, err = f.Readdirnames(1) // Or f.Readdir(1) - if err == io.EOF { - return true - } - return false // Either not empty or error, suits both cases -} - func StartReadingFiles(workingDir string) { - err := os.MkdirAll(workingDir, fs.ModeDir) + err := os.MkdirAll(workingDir, 777) utils.CheckErr(err) for true { - if IsEmpty(workingDir) { - fmt.Printf("Waiting for new files\n") - time.Sleep(5 * time.Second) - continue - } - dir, _ := os.Open(workingDir) dirFiles, _ := dir.Readdir(-1) sort.Sort(utils.ByModTime(dirFiles)) + if len(dirFiles) == 0{ + fmt.Printf("Waiting for new files\n") + time.Sleep(3 * time.Second) + continue + } fileInfo := dirFiles[0] inputFilePath := path.Join(workingDir, fileInfo.Name()) file, err := os.Open(inputFilePath) @@ -57,20 +40,20 @@ func StartReadingFiles(workingDir string) { decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar) utils.CheckErr(decErr) - for _, entry := range inputHar.Log.Entries { + for _, entry := range inputHar.Log.Entries { SaveHarToDb(*entry, "") } rmErr := os.Remove(inputFilePath) utils.CheckErr(rmErr) } - } func SaveHarToDb(entry har.Entry, source string) { entryBytes, _ := json.Marshal(entry) serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL) + entryId := primitive.NewObjectID().Hex() mizuEntry := models.MizuEntry{ - EntryId: primitive.NewObjectID().Hex(), + EntryId: entryId, Entry: string(entryBytes), // simple way to store it and not convert to bytes Service: serviceName, Url: entry.Request.URL, @@ -81,6 +64,19 @@ func SaveHarToDb(entry har.Entry, source string) { Timestamp: entry.StartedDateTime.Unix(), } database.GetEntriesTable().Create(&mizuEntry) + + baseEntry := &models.BaseEntryDetails{ + Id: entryId, + Url: entry.Request.URL, + Service: serviceName, + Path: urlPath, + StatusCode: entry.Response.Status, + Method: entry.Request.Method, + Timestamp: entry.StartedDateTime.Unix(), + } + baseEntryBytes, _ := json.Marshal(&baseEntry) + ikisocket.Broadcast(baseEntryBytes) + } func getServiceNameFromUrl(inputUrl string) (string, string) { diff --git a/api/pkg/routes/socket_routes.go b/api/pkg/routes/socket_routes.go new file mode 100644 index 000000000..c912253ed --- /dev/null +++ b/api/pkg/routes/socket_routes.go @@ -0,0 +1,43 @@ +package routes + +import ( + "fmt" + "github.com/antoniodipinto/ikisocket" + "github.com/gofiber/fiber/v2" +) + +func webSocketConnect(ep *ikisocket.EventPayload) { + fmt.Println(fmt.Sprintf("Connection event 1 - User: %s", ep.Kws.GetStringAttribute("user_id"))) +} + +func webSocketDisconnect(ep *ikisocket.EventPayload) { + fmt.Println(fmt.Sprintf("Disconnection event - User: %s", ep.Kws.GetStringAttribute("user_id"))) +} + +func webSocketClose(ep *ikisocket.EventPayload) { + fmt.Println(fmt.Sprintf("Close event - User: %s", ep.Kws.GetStringAttribute("user_id"))) +} + +func webSocketError(ep *ikisocket.EventPayload) { + fmt.Println(fmt.Sprintf("Error event - User: %s", ep.Kws.GetStringAttribute("user_id"))) +} + +func webSocketMessage(ep *ikisocket.EventPayload) { + fmt.Println("Web socket message") + // fmt.Println(fmt.Sprintf("Message event - User: %s - Message: %s", ep.Kws.GetStringAttribute("user_id"), string(ep.Data))) +} + +func WebSocketRoutes(app *fiber.App) { + + app.Get("/ws", ikisocket.New(func(kws *ikisocket.Websocket) { + // kws.Broadcast([]byte(fmt.Sprintf("New user connected: %s and UUID: %s", userId, kws.UUID)), true) + // kws.Emit([]byte(fmt.Sprintf("Hello user with UUID: %s", kws.UUID))) + kws.SetAttribute("user_id", kws.UUID) + })) + + ikisocket.On(ikisocket.EventMessage, webSocketMessage) + ikisocket.On(ikisocket.EventConnect, webSocketConnect) + ikisocket.On(ikisocket.EventDisconnect, webSocketDisconnect) + ikisocket.On(ikisocket.EventClose, webSocketClose) // This event is called when the server disconnects the user actively with .Close() method + ikisocket.On(ikisocket.EventError, webSocketError) // On error event +} From 96cb0ec9c193c89a68a197cffaf83a73f637c037 Mon Sep 17 00:00:00 2001 From: Alex Haiut Date: Wed, 28 Apr 2021 15:55:57 +0300 Subject: [PATCH 4/4] Feature/makefile (#11) * minor fixes * makefile fixes - docker build * minor fix in Makefile Co-authored-by: Alex Haiut --- .gitignore | 1 + Makefile | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 179f9bdd9..5c657649f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ # Dependency directories (remove the comment below to include it) # vendor/ .idea/ +build *.db # Build directories diff --git a/Makefile b/Makefile index bd71b972c..171308c5b 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,8 @@ help: ## This help. # Variables and lists TS_SUFFIX="$(shell date '+%s')" - +DOCKER_IMG="up9inc/mizu" +DOCKER_TAG="latest" ui: ## build UI @(cd ui; npm i ; npm run build; ) @@ -27,6 +28,7 @@ cli: # build CLI @(cd cli; echo "building cli" ) api: ## build API server + @(echo "building API server .." ) @(cd api; go build -o build/apiserver main.go) @ls -l api/build @@ -36,19 +38,21 @@ tap: ## build tap binary docker: ## build Docker image @(echo "building docker image" ) + docker build -t ${DOCKER_IMG}:${DOCKER_TAG} api + docker images ${DOCKER_IMG} publish: ## build and publish Mizu docker image & CLI @echo "publishing Docker image .. " @echo "publishing CLI .. " -clean: clean-api clean-cli clean-ui clean-docker ## Clean all build artifacts +clean: clean-ui clean-api clean-cli clean-docker ## Clean all build artifacts clean-ui: - @(cd ui; rm -rf build ; echo "ui cleanup done" ) + @(rm -rf ui/build ; echo "UI cleanup done" ) clean-api: - @(cd api; rm -rf build ; echo "api cleanup done" ) + @(rm -rf api/build ; echo "api cleanup done" ) clean-cli: @(echo "CLI cleanup - NOT IMPLEMENTED YET " )