diff --git a/.gitignore b/.gitignore index ef835de11..cc606960f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,5 @@ build *.db -# Build directories -build - # Mac OS .DS_Store diff --git a/Dockerfile b/Dockerfile index 4e9bcc8d1..f2223fcaf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,4 +38,4 @@ COPY --from=site-build ["/ui-build/build", "site"] COPY api/start.sh . # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process -CMD "./start.sh" +CMD "./start.sh" \ No newline at end of file diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index d972ac98e..caa055e24 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -2,26 +2,59 @@ package controllers import ( "encoding/json" + "fmt" "github.com/gofiber/fiber/v2" "github.com/google/martian/har" "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/utils" "strconv" + "strings" +) + +const ( + HardLimit = 200 ) func GetEntries(c *fiber.Ctx) error { - limit, e := strconv.Atoi(c.Query("limit", "100")) + limit, e := strconv.Atoi(c.Query("limit", "200")) + utils.CheckErr(e) + if limit > HardLimit { + limit = HardLimit + } + + sortOption := c.Query("operator", "lt") + var sortingOperator string + var ordering string + if strings.ToLower(sortOption) == "gt" { + sortingOperator = ">" + ordering = "asc" + } else if strings.ToLower(sortOption) == "lt" { + sortingOperator = "<" + ordering = "desc" + } else { + fmt.Println("Unsupported") + return nil + } + + timestamp, e := strconv.Atoi(c.Query("timestamp", "-1")) utils.CheckErr(e) var entries []models.MizuEntry + database.GetEntriesTable(). + Order(fmt.Sprintf("timestamp %s", ordering)). + Where(fmt.Sprintf("timestamp %s %v",sortingOperator, timestamp)). Omit("entry"). // remove the "big" entry field Limit(limit). Find(&entries) + if len(entries) > 0 && ordering == "desc"{ + utils.ReverseSlice(entries) + } + // Convert to base entries - baseEntries := make([]models.BaseEntryDetails, 0) + baseEntries := make([]models.BaseEntryDetails, 0, limit) for _, entry := range entries { baseEntries = append(baseEntries, models.BaseEntryDetails{ Id: entry.EntryId, @@ -51,7 +84,6 @@ func GetEntry(c *fiber.Ctx) error { return c.Status(fiber.StatusOK).JSON(fullEntry) } - func DeleteAllEntries(c *fiber.Ctx) error { database.GetEntriesTable(). Where("1 = 1"). diff --git a/api/pkg/inserter/main.go b/api/pkg/inserter/main.go index 70c6e6652..bf6ae6ae1 100644 --- a/api/pkg/inserter/main.go +++ b/api/pkg/inserter/main.go @@ -76,7 +76,7 @@ func SaveHarToDb(entry har.Entry, source string) { Method: entry.Request.Method, Status: entry.Response.Status, Source: source, - Timestamp: entry.StartedDateTime.Unix(), + Timestamp: entry.StartedDateTime.UnixNano() / int64(time.Millisecond), } database.GetEntriesTable().Create(&mizuEntry) @@ -87,7 +87,7 @@ func SaveHarToDb(entry har.Entry, source string) { Path: urlPath, StatusCode: entry.Response.Status, Method: entry.Request.Method, - Timestamp: entry.StartedDateTime.Unix(), + Timestamp: entry.StartedDateTime.UnixNano() / int64(time.Millisecond), } baseEntryBytes, _ := json.Marshal(&baseEntry) ikisocket.Broadcast(baseEntryBytes) diff --git a/api/pkg/routes/public_routes.go b/api/pkg/routes/public_routes.go index 15298b0d1..1f465baf2 100644 --- a/api/pkg/routes/public_routes.go +++ b/api/pkg/routes/public_routes.go @@ -13,5 +13,4 @@ func EntriesRoutes(fiberApp *fiber.App) { routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry - } diff --git a/api/pkg/utils/utils.go b/api/pkg/utils/utils.go index 789756bcb..c5de0fc9f 100644 --- a/api/pkg/utils/utils.go +++ b/api/pkg/utils/utils.go @@ -6,6 +6,7 @@ import ( "log" "os" "os/signal" + "reflect" "syscall" ) @@ -29,6 +30,17 @@ func StartServer(app *fiber.App) { } } +func ReverseSlice(data interface{}) { + value := reflect.ValueOf(data) + valueLen := value.Len() + for i := 0; i <= int((valueLen-1)/2); i++ { + reverseIndex := valueLen - 1 - i + tmp := value.Index(reverseIndex).Interface() + value.Index(reverseIndex).Set(value.Index(i)) + value.Index(i).Set(reflect.ValueOf(tmp)) + } +} + func CheckErr(e error) { if e != nil { diff --git a/ui/package-lock.json b/ui/package-lock.json index af88e46fc..4d00ab30a 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -13448,11 +13448,6 @@ "prop-types": "^15.6.2" } }, - "react-use-websocket": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/react-use-websocket/-/react-use-websocket-2.6.1.tgz", - "integrity": "sha512-Nx1jUab+7eHpVftBpscgVG26UMVjy4P8ss+I3sE6LijHXC0chFej7unzH9YXElN9stEm1of+qZA+YX/ZlPIQBQ==" - }, "read-pkg": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", diff --git a/ui/package.json b/ui/package.json index 73debb27f..134655049 100644 --- a/ui/package.json +++ b/ui/package.json @@ -19,7 +19,6 @@ "react-scripts": "4.0.3", "react-scrollable-feed": "^1.3.0", "react-syntax-highlighter": "^15.4.3", - "react-use-websocket": "^2.6.1", "typescript": "^4.2.4", "web-vitals": "^1.1.1" }, diff --git a/ui/src/components/HarEntriesList.tsx b/ui/src/components/HarEntriesList.tsx index 3dd54aaf4..efc6ba52a 100644 --- a/ui/src/components/HarEntriesList.tsx +++ b/ui/src/components/HarEntriesList.tsx @@ -1,28 +1,112 @@ import {HarEntry} from "./HarEntry"; -import React, {useEffect, useRef} from "react"; +import React, {useEffect, useState} from "react"; import styles from './style/HarEntriesList.module.sass'; -import ScrollableFeed from 'react-scrollable-feed' +import spinner from './assets/spinner.svg'; +import ScrollableFeed from "react-scrollable-feed"; interface HarEntriesListProps { entries: any[]; + setEntries: (entries: any[]) => void; focusedEntryId: string; - setFocusedEntryId: (id: string) => void + setFocusedEntryId: (id: string) => void; + connectionOpen: boolean; + noMoreDataTop: boolean; + setNoMoreDataTop: (flag: boolean) => void; + noMoreDataBottom: boolean; + setNoMoreDataBottom: (flag: boolean) => void; } -export const HarEntriesList: React.FC = ({entries, focusedEntryId, setFocusedEntryId}) => { +enum FetchOperator { + LT = "lt", + GT = "gt" +} + +export const HarEntriesList: React.FC = ({entries, setEntries, focusedEntryId, setFocusedEntryId, connectionOpen, noMoreDataTop, setNoMoreDataTop, noMoreDataBottom, setNoMoreDataBottom}) => { + + const [loadMoreTop, setLoadMoreTop] = useState(false); + const [isLoadingTop, setIsLoadingTop] = useState(false); + + useEffect(() => { + if(loadMoreTop && !connectionOpen && !noMoreDataTop) + fetchData(FetchOperator.LT); + }, [loadMoreTop, connectionOpen, noMoreDataTop]); + + useEffect(() => { + const list = document.getElementById('list').firstElementChild; + list.addEventListener('scroll', (e) => { + const el: any = e.target; + if(el.scrollTop === 0) { + setLoadMoreTop(true); + } + }); + }, []); + + const fetchData = async (operator) => { + + const timestamp = operator === FetchOperator.LT ? entries[0].timestamp : entries[entries.length - 1].timestamp; + if(operator === FetchOperator.LT) + setIsLoadingTop(true); + + fetch(`http://localhost:8899/api/entries?limit=50&operator=${operator}×tamp=${timestamp}`) + .then(response => response.json()) + .then((data: any[]) => { + let scrollTo; + if(operator === FetchOperator.LT) { + if(data.length === 0) { + setNoMoreDataTop(true); + scrollTo = document.getElementById("noMoreDataTop"); + } else { + scrollTo = document.getElementById(entries[0].id); + } + const newEntries = [...data, ...entries]; + if(newEntries.length >= 1000) { + newEntries.splice(1000); + } + setEntries(newEntries); + setLoadMoreTop(false); + setIsLoadingTop(false) + if(scrollTo) { + scrollTo.scrollIntoView(); + } + } + + if(operator === FetchOperator.GT) { + if(data.length === 0) { + setNoMoreDataBottom(true); + } + scrollTo = document.getElementById(entries[entries.length -1].id); + let newEntries = [...entries, ...data]; + if(newEntries.length >= 1000) { + setNoMoreDataTop(false); + newEntries = newEntries.slice(-1000); + } + setEntries(newEntries); + if(scrollTo) { + scrollTo.scrollIntoView({behavior: "smooth"}); + } + } + }); + }; return <>
- - {entries?.map(entry => + {isLoadingTop &&
spinner
} + + {noMoreDataTop && !connectionOpen &&
No more data available
} + {entries?.map(entry => )} - + isSelected={focusedEntryId === entry.id}/>)} + {!connectionOpen && !noMoreDataBottom &&
+
fetchData(FetchOperator.GT)}>Fetch more entries
+
} + +
+ {entries?.length > 0 &&
{entries?.length} requests
-
Started listening at {new Date(+entries[0].timestamp*1000)?.toLocaleString()}
+
Started listening at {new Date(+entries[0].timestamp)?.toLocaleString()}
} ; diff --git a/ui/src/components/HarEntry.tsx b/ui/src/components/HarEntry.tsx index c9dcda0c4..6ef8d76b9 100644 --- a/ui/src/components/HarEntry.tsx +++ b/ui/src/components/HarEntry.tsx @@ -23,7 +23,7 @@ interface HAREntryProps { export const HarEntry: React.FC = ({entry, setFocusedEntryId, isSelected}) => { return <> -
setFocusedEntryId(entry.id)}> +
setFocusedEntryId(entry.id)}> {entry.statusCode &&
} @@ -33,7 +33,7 @@ export const HarEntry: React.FC = ({entry, setFocusedEntryId, isS {entry.service}
-
{new Date(+entry.timestamp*1000)?.toLocaleString()}
+
{new Date(+entry.timestamp)?.toLocaleString()}
}; \ No newline at end of file diff --git a/ui/src/components/HarPage.tsx b/ui/src/components/HarPage.tsx index 22355c48e..3e0d80ff9 100644 --- a/ui/src/components/HarPage.tsx +++ b/ui/src/components/HarPage.tsx @@ -1,11 +1,12 @@ -import React, {useEffect, useState} from "react"; +import React, {useEffect, useRef, useState} from "react"; // import {HarFilters} from "./HarFilters"; import {HarEntriesList} from "./HarEntriesList"; import {makeStyles} from "@material-ui/core"; import "./style/HarPage.sass"; import styles from './style/HarEntriesList.module.sass'; import {HAREntryDetailed} from "./HarEntryDetailed"; -import useWebSocket from 'react-use-websocket'; +import playIcon from './assets/play.svg'; +import pauseIcon from './assets/pause.svg'; const useLayoutStyles = makeStyles(() => ({ details: { @@ -25,6 +26,12 @@ const useLayoutStyles = makeStyles(() => ({ } })); +enum ConnectionStatus { + Closed, + Connected, + Paused +} + export const HarPage: React.FC = () => { const classes = useLayoutStyles(); @@ -32,24 +39,41 @@ export const HarPage: React.FC = () => { const [entries, setEntries] = useState([] as any); const [focusedEntryId, setFocusedEntryId] = useState(null); const [selectedHarEntry, setSelectedHarEntry] = useState(null); - const [connectionOpen, setConnectionOpen] = useState(false); + const [connection, setConnection] = useState(ConnectionStatus.Closed); + const [noMoreDataTop, setNoMoreDataTop] = useState(false); + const [noMoreDataBottom, setNoMoreDataBottom] = useState(false); - const socketUrl = 'ws://localhost:8899/ws'; - const {lastMessage} = useWebSocket(socketUrl, { - onOpen: () => setConnectionOpen(true), - onClose: () => setConnectionOpen(false), - shouldReconnect: (closeEvent) => true}); + const ws = useRef(null); + + const openWebSocket = () => { + ws.current = new WebSocket("ws://localhost:8899/ws"); + ws.current.onopen = () => setConnection(ConnectionStatus.Connected); + ws.current.onclose = () => setConnection(ConnectionStatus.Closed); + } + + if(ws.current) { + ws.current.onmessage = e => { + console.log(connection); + if(!e?.data) return; + const entry = JSON.parse(e.data); + if(connection === ConnectionStatus.Paused) { + setNoMoreDataBottom(false) + return; + } + if(!focusedEntryId) setFocusedEntryId(entry.id) + let newEntries = [...entries]; + if(entries.length === 1000) { + newEntries = newEntries.splice(1); + setNoMoreDataTop(false); + } + setEntries([...newEntries, entry]) + } + } useEffect(() => { - if(!lastMessage?.data) return; - const entry = JSON.parse(lastMessage.data); - if(!focusedEntryId) setFocusedEntryId(entry.id) - let newEntries = [...entries]; - if(entries.length === 1000) { - newEntries = newEntries.splice(1) - } - setEntries([...newEntries, entry]) - },[lastMessage?.data]) + openWebSocket(); + }, []); + useEffect(() => { if(!focusedEntryId) return; @@ -59,19 +83,43 @@ export const HarPage: React.FC = () => { .then(data => setSelectedHarEntry(data)); },[focusedEntryId]) + const toggleConnection = () => { + setConnection(connection === ConnectionStatus.Connected ? ConnectionStatus.Paused : ConnectionStatus.Connected ); + } + + const getConnectionStatusClass = () => { + switch (connection) { + case ConnectionStatus.Paused: + return "orangeIndicator"; + case ConnectionStatus.Connected: + return "greenIndicator" + default: + return "redIndicator"; + } + } + return (
-
+
+ pause
- {connectionOpen ? "connected, waiting for traffic" : "not connected"} -
+ {connection === ConnectionStatus.Connected ? "connected, waiting for traffic" : "not connected"} +
{entries.length > 0 &&
{/**/}
- +
diff --git a/ui/src/components/assets/pause.svg b/ui/src/components/assets/pause.svg new file mode 100644 index 000000000..bb8b5599d --- /dev/null +++ b/ui/src/components/assets/pause.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/ui/src/components/assets/play.svg b/ui/src/components/assets/play.svg new file mode 100644 index 000000000..66b964f6e --- /dev/null +++ b/ui/src/components/assets/play.svg @@ -0,0 +1,9 @@ + + + + + + + diff --git a/ui/src/components/assets/spinner.svg b/ui/src/components/assets/spinner.svg new file mode 100644 index 000000000..16ac582fa --- /dev/null +++ b/ui/src/components/assets/spinner.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/ui/src/components/style/HarEntriesList.module.sass b/ui/src/components/style/HarEntriesList.module.sass index 2c5eb782d..5b0d9830c 100644 --- a/ui/src/components/style/HarEntriesList.module.sass +++ b/ui/src/components/style/HarEntriesList.module.sass @@ -20,5 +20,19 @@ border-top: 1px solid rgba(255,255,255,0.5) align-items: center padding-top: 10px - margin-top: 10px - margin-right: 15px \ No newline at end of file + margin-right: 15px + +.styledButton + cursor: pointer + line-height: 1 + border-radius: 20px + letter-spacing: .02857em + color: #627ef7 + border: 1px solid rgba(98, 126, 247, 0.5) + padding: 5px 18px + transition: background-color 250ms cubic-bezier(0.4, 0, 0.2, 1) 0ms,box-shadow 250ms cubic-bezier(0.4, 0, 0.2, 1) 0ms,border 250ms cubic-bezier(0.4, 0, 0.2, 1) 0ms + font-weight: 600 + +.styledButton:hover + border: 1px solid #627ef7 + background-color: rgba(255, 255, 255, 0.06) \ No newline at end of file diff --git a/ui/src/components/style/HarPage.sass b/ui/src/components/style/HarPage.sass index 6c9e4ca68..31b6978ae 100644 --- a/ui/src/components/style/HarPage.sass +++ b/ui/src/components/style/HarPage.sass @@ -87,6 +87,15 @@ box-shadow: 0 0 3px 3px #ff3a3073 margin-left: 10px +.orangeIndicator + background-color: #ffb530 + height: 10px + width: 10px + border-radius: 50% + box-shadow: 0 0 3px 3px #ffb33073 + margin-left: 10px + + .connectionText display: flex align-items: center