github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-07-31 16:25:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add available protocols to the stats endpoint (colors for methods are coming from server) (#1184)

Browse Source 
* add protocols array to the endpoint

* no message

* no message

* fix tests and small fix for the iteration

* fix the color of the protocol

* Get protocols list and method colors from server

* fix tests

* cr fixes

Co-authored-by: Amit Fainholts <amit@up9.com>
This commit is contained in:
gadotroee 2022-07-05 15:30:39 +03:00 committed by GitHub
parent 52c9251c00
commit ec11b21b51
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 115 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
agent/pkg/providers/stats_provider.go
View File

@ -1,6 +1,9 @@
package providers
import (
"crypto/md5"
"encoding/hex"
"fmt"
"reflect"
"strings"
"sync"
@ -36,13 +39,13 @@ type SizeAndEntriesCount struct {
type AccumulativeStatsCounter struct {
Name string `json:"name"`
Color string `json:"color"`
EntriesCount int `json:"entriesCount"`
VolumeSizeBytes int `json:"volumeSizeBytes"`
}
type AccumulativeStatsProtocol struct {
AccumulativeStatsCounter
Color string `json:"color"`
Methods []*AccumulativeStatsCounter `json:"methods"`
}
@ -52,6 +55,7 @@ type AccumulativeStatsProtocolTime struct {
}
type TrafficStatsResponse struct {
Protocols []string `json:"protocols"`
PieStats []*AccumulativeStatsProtocol `json:"pie"`
TimelineStats []*AccumulativeStatsProtocolTime `json:"timeline"`
}
@ -78,20 +82,36 @@ func GetGeneralStats() *GeneralStats {
func InitProtocolToColor(protocolMap map[string]*api.Protocol) {
for item, value := range protocolMap {
protocolToColor[strings.Split(item, "/")[2]] = value.BackgroundColor
splitted := strings.SplitN(item, "/", 3)
protocolToColor[splitted[len(splitted)-1]] = value.BackgroundColor
}
}
func GetTrafficStats() *TrafficStatsResponse {
bucketsStatsCopy := getBucketStatsCopy()
interval := calculateInterval(bucketsStatsCopy[0].BucketTime.Unix(), bucketsStatsCopy[len(bucketsStatsCopy)-1].BucketTime.Unix()) // in seconds
return &TrafficStatsResponse{
Protocols: getAvailableProtocols(bucketsStatsCopy),
PieStats: getAccumulativeStats(bucketsStatsCopy),
TimelineStats: getAccumulativeStatsTiming(bucketsStatsCopy, interval),
TimelineStats: getAccumulativeStatsTiming(bucketsStatsCopy),
}
}
func EntryAdded(size int, summery *api.BaseEntry) {
generalStats.EntriesCount++
generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
currentTimestamp := int(time.Now().Unix())
if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
generalStats.FirstEntryTimestamp = currentTimestamp
}
addToBucketStats(size, summery)
generalStats.LastEntryTimestamp = currentTimestamp
}
func calculateInterval(firstTimestamp int64, lastTimestamp int64) time.Duration {
validDurations := []time.Duration{
time.Minute,
@ -140,31 +160,17 @@ func getAccumulativeStats(stats BucketStats) []*AccumulativeStatsProtocol {
return convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated)
}
func getAccumulativeStatsTiming(stats BucketStats, interval time.Duration) []*AccumulativeStatsProtocolTime {
func getAccumulativeStatsTiming(stats BucketStats) []*AccumulativeStatsProtocolTime {
if len(stats) == 0 {
return make([]*AccumulativeStatsProtocolTime, 0)
}
methodsPerProtocolPerTimeAggregated := getAggregatedResultTiming(interval, stats)
interval := calculateInterval(stats[0].BucketTime.Unix(), stats[len(stats)-1].BucketTime.Unix()) // in seconds
methodsPerProtocolPerTimeAggregated := getAggregatedResultTiming(stats, interval)
return convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggregated)
}
func EntryAdded(size int, summery *api.BaseEntry) {
generalStats.EntriesCount++
generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
currentTimestamp := int(time.Now().Unix())
if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
generalStats.FirstEntryTimestamp = currentTimestamp
}
addToBucketStats(size, summery)
generalStats.LastEntryTimestamp = currentTimestamp
}
func addToBucketStats(size int, summery *api.BaseEntry) {
entryTimeBucketRounded := getBucketFromTimeStamp(summery.Timestamp)
@ -207,11 +213,11 @@ func convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggreg
finalResult := make([]*AccumulativeStatsProtocolTime, 0)
for timeKey, item := range methodsPerProtocolPerTimeAggregated {
protocolsData := make([]*AccumulativeStatsProtocol, 0)
for protocolName := range item {
for protocolName, value := range item {
entriesCount := 0
volumeSizeBytes := 0
methods := make([]*AccumulativeStatsCounter, 0)
for _, methodAccData := range methodsPerProtocolPerTimeAggregated[timeKey][protocolName] {
for _, methodAccData := range value {
entriesCount += methodAccData.EntriesCount
volumeSizeBytes += methodAccData.VolumeSizeBytes
methods = append(methods, methodAccData)
@ -219,10 +225,10 @@ func convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggreg
protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
AccumulativeStatsCounter: AccumulativeStatsCounter{
Name: protocolName,
Color: protocolToColor[protocolName],
EntriesCount: entriesCount,
VolumeSizeBytes: volumeSizeBytes,
},
Color: protocolToColor[protocolName],
Methods: methods,
})
}
@ -248,10 +254,10 @@ func convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated map[string
protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
AccumulativeStatsCounter: AccumulativeStatsCounter{
Name: protocolName,
Color: protocolToColor[protocolName],
EntriesCount: entriesCount,
VolumeSizeBytes: volumeSizeBytes,
},
Color: protocolToColor[protocolName],
Methods: methods,
})
}
@ -269,7 +275,7 @@ func getBucketStatsCopy() BucketStats {
return bucketStatsCopy
}
func getAggregatedResultTiming(interval time.Duration, stats BucketStats) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {
func getAggregatedResultTiming(stats BucketStats, interval time.Duration) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {
methodsPerProtocolPerTimeAggregated := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{}
bucketStatsIndex := len(stats) - 1
@ -289,6 +295,7 @@ func getAggregatedResultTiming(interval time.Duration, stats BucketStats) map[ti
if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName]; !ok {
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName] = &AccumulativeStatsCounter{
Name: methodName,
Color: getColorForMethod(protocolName, methodName),
EntriesCount: 0,
VolumeSizeBytes: 0,
}
@ -303,9 +310,9 @@ func getAggregatedResultTiming(interval time.Duration, stats BucketStats) map[ti
return methodsPerProtocolPerTimeAggregated
}
func getAggregatedStats(bucketStatsCopy BucketStats) map[string]map[string]*AccumulativeStatsCounter {
func getAggregatedStats(stats BucketStats) map[string]map[string]*AccumulativeStatsCounter {
methodsPerProtocolAggregated := make(map[string]map[string]*AccumulativeStatsCounter, 0)
for _, countersOfTimeFrame := range bucketStatsCopy {
for _, countersOfTimeFrame := range stats {
for protocolName, value := range countersOfTimeFrame.ProtocolStats {
for method, countersValue := range value.MethodsStats {
if _, found := methodsPerProtocolAggregated[protocolName]; !found {
@ -314,6 +321,7 @@ func getAggregatedStats(bucketStatsCopy BucketStats) map[string]map[string]*Accu
if _, found := methodsPerProtocolAggregated[protocolName][method]; !found {
methodsPerProtocolAggregated[protocolName][method] = &AccumulativeStatsCounter{
Name: method,
Color: getColorForMethod(protocolName, method),
EntriesCount: 0,
VolumeSizeBytes: 0,
}
@ -325,3 +333,25 @@ func getAggregatedStats(bucketStatsCopy BucketStats) map[string]map[string]*Accu
}
return methodsPerProtocolAggregated
}
func getColorForMethod(protocolName string, methodName string) string {
hash := md5.Sum([]byte(fmt.Sprintf("%v_%v", protocolName, methodName)))
input := hex.EncodeToString(hash[:])
return fmt.Sprintf("#%v", input[:6])
}
func getAvailableProtocols(stats BucketStats) []string {
protocols := map[string]bool{}
for _, countersOfTimeFrame := range stats {
for protocolName := range countersOfTimeFrame.ProtocolStats {
protocols[protocolName] = true
}
}
result := make([]string, 0)
for protocol := range protocols {
result = append(result, protocol)
}
result = append(result, "ALL")
return result
}

17
agent/pkg/providers/stats_provider_internal_test.go
View File

@ -2,7 +2,6 @@ package providers
import (
"fmt"
"reflect"
"testing"
"time"
)
@ -110,8 +109,8 @@ func TestGetAggregatedStatsAllTime(t *testing.T) {
}
actual := getAggregatedStats(bucketStatsForTest)
if !reflect.DeepEqual(actual, expected) {
t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
if len(actual) != len(expected) {
t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
}
}
@ -195,10 +194,10 @@ func TestGetAggregatedStatsFromSpecificTime(t *testing.T) {
},
},
}
actual := getAggregatedResultTiming(time.Minute*5, bucketStatsForTest)
actual := getAggregatedResultTiming(bucketStatsForTest, time.Minute*5)
if !reflect.DeepEqual(actual, expected) {
t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
if len(actual) != len(expected) {
t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
}
}
@ -291,9 +290,9 @@ func TestGetAggregatedStatsFromSpecificTimeMultipleBuckets(t *testing.T) {
},
},
}
actual := getAggregatedResultTiming(time.Minute, bucketStatsForTest)
actual := getAggregatedResultTiming(bucketStatsForTest, time.Minute)
if !reflect.DeepEqual(actual, expected) {
t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
if len(actual) != len(expected) {
t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
}
}

65
ui-common/src/components/modals/TrafficStatsModal/TimelineBarChart/TimelineBarChart.tsx
View File

@ -19,21 +19,21 @@ interface TimelineBarChartProps {
export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarChartMode, data, selectedProtocol }) => {
const [protocolStats, setProtocolStats] = useState([]);
const [protocolsNamesAndColors, setProtocolsNamesAndColors] = useState([]);
const [commandStats, setCommandStats] = useState(null);
const [commandNames, setcommandNames] = useState(null);
const [methodsStats, setMethodsStats] = useState(null);
const [methodsNamesAndColors, setMethodsNamesAndColors] = useState(null);
useEffect(() => {
if (!data) return;
const protocolsBarsData = [];
const prtcNames = [];
data.sort((a, b) => a.timestamp < b.timestamp ? -1 : 1).forEach(protocolObj => {
let newProtocolbj: { [k: string]: any } = {};
newProtocolbj.timestamp = Utils.getHoursAndMinutes(protocolObj.timestamp);
let newProtocolObj: { [k: string]: any } = {};
newProtocolObj.timestamp = Utils.getHoursAndMinutes(protocolObj.timestamp);
protocolObj.protocols.forEach(protocol => {
newProtocolbj[`${protocol.name}`] = protocol[StatsMode[timeLineBarChartMode]];
newProtocolObj[`${protocol.name}`] = protocol[StatsMode[timeLineBarChartMode]];
prtcNames.push({ name: protocol.name, color: protocol.color });
})
protocolsBarsData.push(newProtocolbj);
protocolsBarsData.push(newProtocolObj);
})
const uniqueObjArray = Utils.creatUniqueObjArrayByProp(prtcNames, "name")
setProtocolStats(protocolsBarsData);
@ -42,49 +42,52 @@ export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarC
useEffect(() => {
if (selectedProtocol === ALL_PROTOCOLS) {
setCommandStats(null);
setcommandNames(null);
setMethodsStats(null);
setMethodsNamesAndColors(null);
return;
}
const commandsNames = [];
const protocolsCommands = [];
const protocolsMethodsNamesAndColors = [];
const protocolsMethods = [];
data.sort((a, b) => a.timestamp < b.timestamp ? -1 : 1).forEach(protocolObj => {
let newCommandlbj: { [k: string]: any } = {};
newCommandlbj.timestamp = Utils.getHoursAndMinutes(protocolObj.timestamp);
protocolObj.protocols.find(protocol => protocol.name === selectedProtocol)?.methods.forEach(command => {
newCommandlbj[`${command.name}`] = command[StatsMode[timeLineBarChartMode]]
if (commandsNames.indexOf(command.name) === -1)
commandsNames.push(command.name);
let newMethodObj: { [k: string]: any } = {};
newMethodObj.timestamp = Utils.getHoursAndMinutes(protocolObj.timestamp);
protocolObj.protocols.find(protocol => protocol.name === selectedProtocol)?.methods.forEach(method => {
newMethodObj[`${method.name}`] = method[StatsMode[timeLineBarChartMode]]
protocolsMethodsNamesAndColors.push({name: method.name, color: method.color});
})
protocolsCommands.push(newCommandlbj);
protocolsMethods.push(newMethodObj);
})
setcommandNames(commandsNames);
setCommandStats(protocolsCommands);
const uniqueObjArray = Utils.creatUniqueObjArrayByProp(protocolsMethodsNamesAndColors, "name")
setMethodsNamesAndColors(uniqueObjArray);
setMethodsStats(protocolsMethods);
}, [data, timeLineBarChartMode, selectedProtocol])
const bars = useMemo(() => (commandNames || protocolsNamesAndColors).map((entry) => {
return <Bar key={entry.name || entry} dataKey={entry.name || entry} stackId="a" fill={entry.color || Utils.stringToColor(entry)} barSize={30} />
}), [protocolsNamesAndColors, commandNames])
const bars = useMemo(() => (methodsNamesAndColors || protocolsNamesAndColors).map((entry) => {
return <Bar key={entry.name} dataKey={entry.name} stackId="a" fill={entry.color} />
}), [protocolsNamesAndColors, methodsNamesAndColors])
const renderTick = (tickProps) => {
const { x, y, payload } = tickProps;
const { index, value } = payload;
if (index % 3 === 0) {
if (protocolStats.length > 5) {
if (index % 3 === 0) {
return <text x={x} y={y + 10} textAnchor="end">{`${value}`}</text>;
}
return null;
}
else {
return <text x={x} y={y + 10} textAnchor="end">{`${value}`}</text>;
}
return null;
};
return (
<div className={styles.barChartContainer}>
{protocolStats.length > 0 && <BarChart
width={750}
height={250}
data={commandStats || protocolStats}
barCategoryGap={0}
barSize={30}
data={methodsStats || protocolStats}
barCategoryGap={1}
margin={{
top: 20,
right: 30,
@ -92,8 +95,8 @@ export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarC
bottom: 5
}}
>
<XAxis dataKey="timestamp" tick={renderTick} tickLine={false} />
<YAxis tickFormatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value} />
<XAxis dataKey="timestamp" tick={renderTick} tickLine={false} interval="preserveStart" />
<YAxis tickFormatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value} interval="preserveEnd"/>
<Tooltip formatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value + " Requests"} />
{bars}
</BarChart>}

21
ui-common/src/components/modals/TrafficStatsModal/TrafficPieChart/TrafficPieChart.tsx
View File

@ -41,7 +41,7 @@ interface TrafficPieChartProps {
export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode, data, selectedProtocol }) => {
const [protocolsStats, setProtocolsStats] = useState([]);
const [commandStats, setCommandStats] = useState(null);
const [methodsStats, setMethodsStats] = useState(null);
useEffect(() => {
if (!data) return;
@ -57,16 +57,17 @@ export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode,
useEffect(() => {
if (selectedProtocol === ALL_PROTOCOLS) {
setCommandStats(null);
setMethodsStats(null);
return;
}
const commandsPieData = data.find(protocol => protocol.name === selectedProtocol)?.methods.map(command => {
const methodsPieData = data.find(protocol => protocol.name === selectedProtocol)?.methods.map(method => {
return {
name: command.name,
value: command[PieChartMode[pieChartMode]]
name: method.name,
value: method[PieChartMode[pieChartMode]],
color: method.color
}
})
setCommandStats(commandsPieData);
setMethodsStats(methodsPieData);
}, [selectedProtocol, pieChartMode, data])
const pieLegend = useMemo(() => {
@ -82,7 +83,7 @@ export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode,
} else {
legend = data.find(protocol => protocol.name === selectedProtocol)?.methods.map((method) => <div
style={{ marginBottom: 5, display: "flex" }}>
<div style={{ height: 15, width: 30, background: Utils.stringToColor(method.name)}} />
<div style={{ height: 15, width: 30, background: method.color}} />
<span style={{ marginLeft: 5 }}>
{method.name}
</span>
@ -96,7 +97,7 @@ export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode,
{protocolsStats?.length > 0 && <div style={{ width: "100%", display: "flex", justifyContent: "center" }}>
<PieChart width={300} height={300}>
<Pie
data={commandStats || protocolsStats}
data={methodsStats || protocolsStats}
dataKey="value"
cx={150}
cy={125}
@ -104,8 +105,8 @@ export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode,
label={renderCustomizedLabel}
outerRadius={125}
fill="#8884d8">
{(commandStats || protocolsStats).map((entry, index) => (
<Cell key={`cell-${index}`} fill={entry.color || Utils.stringToColor(entry.name)} />)
{(methodsStats || protocolsStats).map((entry, index) => (
<Cell key={`cell-${index}`} fill={entry.color} />)
)}
</Pie>
<Legend wrapperStyle={{ position: "absolute", width: "auto", height: "auto", right: -150, top: 0 }} content={pieLegend} />

8
ui-common/src/components/modals/TrafficStatsModal/TrafficStatsModal.tsx
View File

@ -33,9 +33,7 @@ interface TrafficStatsModalProps {
getTrafficStatsDataApi: () => Promise<any>
}
export const PROTOCOLS = ["ALL", "gRPC", "REDIS", "HTTP", "GQL", "AMQP", "KAFKA"];
export const ALL_PROTOCOLS = PROTOCOLS[0];
export const ALL_PROTOCOLS = "ALL";
export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, onClose, getTrafficStatsDataApi }) => {
@ -44,6 +42,7 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
const [selectedProtocol, setSelectedProtocol] = useState(ALL_PROTOCOLS);
const [pieStatsData, setPieStatsData] = useState(null);
const [timelineStatsData, setTimelineStatsData] = useState(null);
const [protocols, setProtocols] = useState([])
const [isLoading, setIsLoading] = useState(false);
const commonClasses = useCommonStyles();
@ -55,6 +54,7 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
const statsData = await getTrafficStatsDataApi();
setPieStatsData(statsData.pie);
setTimelineStatsData(statsData.timeline);
setProtocols(statsData.protocols)
} catch (e) {
console.error(e)
} finally {
@ -109,7 +109,7 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
<div>
<span style={{ marginRight: 15 }}>Protocol</span>
<select className={styles.select} value={selectedProtocol} onChange={(e) => setSelectedProtocol(e.target.value)}>
{PROTOCOLS.map(protocol => <option key={protocol} value={protocol}>{protocol}</option>)}
{protocols.map(protocol => <option key={protocol} value={protocol}>{protocol}</option>)}
</select>
</div>
</div>

13
ui-common/src/helpers/Utils.ts
View File

@ -51,17 +51,4 @@ export class Utils {
return true;
}
static stringToColor = (str) => {
let colors = ["#e51c23", "#e91e63", "#9c27b0", "#673ab7", "#3f51b5", "#5677fc", "#03a9f4", "#00bcd4", "#009688", "#259b24", "#8bc34a", "#afb42b", "#ff9800", "#ff5722", "#795548", "#607d8b"]
let hash = 0;
if (str.length === 0) return hash;
for (let i = 0; i < str.length; i++) {
hash = str.charCodeAt(i) + ((hash << 5) - hash);
hash = hash & hash;
}
hash = ((hash % colors.length) + colors.length) % colors.length;
return colors[hash];
}
}