Add available protocols to the stats endpoint (colors for methods are coming from server) (#1184)

* add protocols array to the endpoint

* no message

* no message

* fix tests and small fix for the iteration

* fix the color of the protocol

* Get protocols list and method colors from server

* fix tests

* cr fixes

Co-authored-by: Amit Fainholts <amit@up9.com>

agent/pkg/providers/stats_provider.go

@@ -1,6 +1,9 @@
 package providers
 
 import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
 	"reflect"
 	"strings"
 	"sync"
@@ -36,13 +39,13 @@ type SizeAndEntriesCount struct {
 
 type AccumulativeStatsCounter struct {
 	Name            string `json:"name"`
+	Color           string `json:"color"`
 	EntriesCount    int    `json:"entriesCount"`
 	VolumeSizeBytes int    `json:"volumeSizeBytes"`
 }
 
 type AccumulativeStatsProtocol struct {
 	AccumulativeStatsCounter
-	Color   string                      `json:"color"`
 	Methods []*AccumulativeStatsCounter `json:"methods"`
 }
 
@@ -52,6 +55,7 @@ type AccumulativeStatsProtocolTime struct {
 }
 
 type TrafficStatsResponse struct {
+	Protocols     []string                         `json:"protocols"`
 	PieStats      []*AccumulativeStatsProtocol     `json:"pie"`
 	TimelineStats []*AccumulativeStatsProtocolTime `json:"timeline"`
 }
@@ -78,20 +82,36 @@ func GetGeneralStats() *GeneralStats {
 
 func InitProtocolToColor(protocolMap map[string]*api.Protocol) {
 	for item, value := range protocolMap {
-		protocolToColor[strings.Split(item, "/")[2]] = value.BackgroundColor
+		splitted := strings.SplitN(item, "/", 3)
+		protocolToColor[splitted[len(splitted)-1]] = value.BackgroundColor
 	}
 }
 
 func GetTrafficStats() *TrafficStatsResponse {
 	bucketsStatsCopy := getBucketStatsCopy()
-	interval := calculateInterval(bucketsStatsCopy[0].BucketTime.Unix(), bucketsStatsCopy[len(bucketsStatsCopy)-1].BucketTime.Unix()) // in seconds
 
 	return &TrafficStatsResponse{
+		Protocols:     getAvailableProtocols(bucketsStatsCopy),
 		PieStats:      getAccumulativeStats(bucketsStatsCopy),
-		TimelineStats: getAccumulativeStatsTiming(bucketsStatsCopy, interval),
+		TimelineStats: getAccumulativeStatsTiming(bucketsStatsCopy),
 	}
 }
 
+func EntryAdded(size int, summery *api.BaseEntry) {
+	generalStats.EntriesCount++
+	generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
+
+	currentTimestamp := int(time.Now().Unix())
+
+	if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
+		generalStats.FirstEntryTimestamp = currentTimestamp
+	}
+
+	addToBucketStats(size, summery)
+
+	generalStats.LastEntryTimestamp = currentTimestamp
+}
+
 func calculateInterval(firstTimestamp int64, lastTimestamp int64) time.Duration {
 	validDurations := []time.Duration{
 		time.Minute,
@@ -140,31 +160,17 @@ func getAccumulativeStats(stats BucketStats) []*AccumulativeStatsProtocol {
 	return convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated)
 }
 
-func getAccumulativeStatsTiming(stats BucketStats, interval time.Duration) []*AccumulativeStatsProtocolTime {
+func getAccumulativeStatsTiming(stats BucketStats) []*AccumulativeStatsProtocolTime {
 	if len(stats) == 0 {
 		return make([]*AccumulativeStatsProtocolTime, 0)
 	}
 
-	methodsPerProtocolPerTimeAggregated := getAggregatedResultTiming(interval, stats)
+	interval := calculateInterval(stats[0].BucketTime.Unix(), stats[len(stats)-1].BucketTime.Unix()) // in seconds
+	methodsPerProtocolPerTimeAggregated := getAggregatedResultTiming(stats, interval)
 
 	return convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggregated)
 }
 
-func EntryAdded(size int, summery *api.BaseEntry) {
-	generalStats.EntriesCount++
-	generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
-
-	currentTimestamp := int(time.Now().Unix())
-
-	if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
-		generalStats.FirstEntryTimestamp = currentTimestamp
-	}
-
-	addToBucketStats(size, summery)
-
-	generalStats.LastEntryTimestamp = currentTimestamp
-}
-
 func addToBucketStats(size int, summery *api.BaseEntry) {
 	entryTimeBucketRounded := getBucketFromTimeStamp(summery.Timestamp)
 
@@ -207,11 +213,11 @@ func convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggreg
 	finalResult := make([]*AccumulativeStatsProtocolTime, 0)
 	for timeKey, item := range methodsPerProtocolPerTimeAggregated {
 		protocolsData := make([]*AccumulativeStatsProtocol, 0)
-		for protocolName := range item {
+		for protocolName, value := range item {
 			entriesCount := 0
 			volumeSizeBytes := 0
 			methods := make([]*AccumulativeStatsCounter, 0)
-			for _, methodAccData := range methodsPerProtocolPerTimeAggregated[timeKey][protocolName] {
+			for _, methodAccData := range value {
 				entriesCount += methodAccData.EntriesCount
 				volumeSizeBytes += methodAccData.VolumeSizeBytes
 				methods = append(methods, methodAccData)
@@ -219,10 +225,10 @@ func convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggreg
 			protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
 				AccumulativeStatsCounter: AccumulativeStatsCounter{
 					Name:            protocolName,
+					Color:           protocolToColor[protocolName],
 					EntriesCount:    entriesCount,
 					VolumeSizeBytes: volumeSizeBytes,
 				},
-				Color:   protocolToColor[protocolName],
 				Methods: methods,
 			})
 		}
@@ -248,10 +254,10 @@ func convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated map[string
 		protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
 			AccumulativeStatsCounter: AccumulativeStatsCounter{
 				Name:            protocolName,
+				Color:           protocolToColor[protocolName],
 				EntriesCount:    entriesCount,
 				VolumeSizeBytes: volumeSizeBytes,
 			},
-			Color:   protocolToColor[protocolName],
 			Methods: methods,
 		})
 	}
@@ -269,7 +275,7 @@ func getBucketStatsCopy() BucketStats {
 	return bucketStatsCopy
 }
 
-func getAggregatedResultTiming(interval time.Duration, stats BucketStats) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {
+func getAggregatedResultTiming(stats BucketStats, interval time.Duration) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {
 	methodsPerProtocolPerTimeAggregated := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{}
 
 	bucketStatsIndex := len(stats) - 1
@@ -289,6 +295,7 @@ func getAggregatedResultTiming(interval time.Duration, stats BucketStats) map[ti
 				if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName]; !ok {
 					methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName] = &AccumulativeStatsCounter{
 						Name:            methodName,
+						Color:           getColorForMethod(protocolName, methodName),
 						EntriesCount:    0,
 						VolumeSizeBytes: 0,
 					}
@@ -303,9 +310,9 @@ func getAggregatedResultTiming(interval time.Duration, stats BucketStats) map[ti
 	return methodsPerProtocolPerTimeAggregated
 }
 
-func getAggregatedStats(bucketStatsCopy BucketStats) map[string]map[string]*AccumulativeStatsCounter {
+func getAggregatedStats(stats BucketStats) map[string]map[string]*AccumulativeStatsCounter {
 	methodsPerProtocolAggregated := make(map[string]map[string]*AccumulativeStatsCounter, 0)
-	for _, countersOfTimeFrame := range bucketStatsCopy {
+	for _, countersOfTimeFrame := range stats {
 		for protocolName, value := range countersOfTimeFrame.ProtocolStats {
 			for method, countersValue := range value.MethodsStats {
 				if _, found := methodsPerProtocolAggregated[protocolName]; !found {
@@ -314,6 +321,7 @@ func getAggregatedStats(bucketStatsCopy BucketStats) map[string]map[string]*Accu
 				if _, found := methodsPerProtocolAggregated[protocolName][method]; !found {
 					methodsPerProtocolAggregated[protocolName][method] = &AccumulativeStatsCounter{
 						Name:            method,
+						Color:           getColorForMethod(protocolName, method),
 						EntriesCount:    0,
 						VolumeSizeBytes: 0,
 					}
@@ -325,3 +333,25 @@ func getAggregatedStats(bucketStatsCopy BucketStats) map[string]map[string]*Accu
 	}
 	return methodsPerProtocolAggregated
 }
+
+func getColorForMethod(protocolName string, methodName string) string {
+	hash := md5.Sum([]byte(fmt.Sprintf("%v_%v", protocolName, methodName)))
+	input := hex.EncodeToString(hash[:])
+	return fmt.Sprintf("#%v", input[:6])
+}
+
+func getAvailableProtocols(stats BucketStats) []string {
+	protocols := map[string]bool{}
+	for _, countersOfTimeFrame := range stats {
+		for protocolName := range countersOfTimeFrame.ProtocolStats {
+			protocols[protocolName] = true
+		}
+	}
+
+	result := make([]string, 0)
+	for protocol := range protocols {
+		result = append(result, protocol)
+	}
+	result = append(result, "ALL")
+	return result
+}

agent/pkg/providers/stats_provider_internal_test.go

@@ -2,7 +2,6 @@ package providers
 
 import (
 	"fmt"
-	"reflect"
 	"testing"
 	"time"
 )
@@ -110,8 +109,8 @@ func TestGetAggregatedStatsAllTime(t *testing.T) {
 	}
 	actual := getAggregatedStats(bucketStatsForTest)
 
-	if !reflect.DeepEqual(actual, expected) {
+	if len(actual) != len(expected) {
-		t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
+		t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
 	}
 }
 
@@ -195,10 +194,10 @@ func TestGetAggregatedStatsFromSpecificTime(t *testing.T) {
 			},
 		},
 	}
-	actual := getAggregatedResultTiming(time.Minute*5, bucketStatsForTest)
+	actual := getAggregatedResultTiming(bucketStatsForTest, time.Minute*5)
 
-	if !reflect.DeepEqual(actual, expected) {
+	if len(actual) != len(expected) {
-		t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
+		t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
 	}
 }
 
@@ -291,9 +290,9 @@ func TestGetAggregatedStatsFromSpecificTimeMultipleBuckets(t *testing.T) {
 			},
 		},
 	}
-	actual := getAggregatedResultTiming(time.Minute, bucketStatsForTest)
+	actual := getAggregatedResultTiming(bucketStatsForTest, time.Minute)
 
-	if !reflect.DeepEqual(actual, expected) {
+	if len(actual) != len(expected) {
-		t.Errorf("unexpected result - expected: %v, actual: %v", 3, len(actual))
+		t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
 	}
 }

ui-common/src/components/modals/TrafficStatsModal/TimelineBarChart/TimelineBarChart.tsx

@@ -19,21 +19,21 @@ interface TimelineBarChartProps {
 export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarChartMode, data, selectedProtocol }) => {
     const [protocolStats, setProtocolStats] = useState([]);
     const [protocolsNamesAndColors, setProtocolsNamesAndColors] = useState([]);
-    const [commandStats, setCommandStats] = useState(null);
+    const [methodsStats, setMethodsStats] = useState(null);
-    const [commandNames, setcommandNames] = useState(null);
+    const [methodsNamesAndColors, setMethodsNamesAndColors] = useState(null);
     
     useEffect(() => {
         if (!data) return;
         const protocolsBarsData = [];
         const prtcNames = [];
         data.sort((a, b) => a.timestamp < b.timestamp ? -1 : 1).forEach(protocolObj => {
-            let newProtocolbj: { [k: string]: any } = {};
+            let newProtocolObj: { [k: string]: any } = {};
-            newProtocolbj.timestamp = Utils.getHoursAndMinutes(protocolObj.timestamp);
+            newProtocolObj.timestamp = Utils.getHoursAndMinutes(protocolObj.timestamp);
             protocolObj.protocols.forEach(protocol => {
-                newProtocolbj[`${protocol.name}`] = protocol[StatsMode[timeLineBarChartMode]];
+                newProtocolObj[`${protocol.name}`] = protocol[StatsMode[timeLineBarChartMode]];
                 prtcNames.push({ name: protocol.name, color: protocol.color });
             })
-            protocolsBarsData.push(newProtocolbj);
+            protocolsBarsData.push(newProtocolObj);
         })
         const uniqueObjArray = Utils.creatUniqueObjArrayByProp(prtcNames, "name")
         setProtocolStats(protocolsBarsData);
@@ -42,49 +42,52 @@ export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarC
 
     useEffect(() => {
         if (selectedProtocol === ALL_PROTOCOLS) {
-            setCommandStats(null);
+            setMethodsStats(null);
-            setcommandNames(null);
+            setMethodsNamesAndColors(null);
             return;
         }
-        const commandsNames = [];
+        const protocolsMethodsNamesAndColors = [];
-        const protocolsCommands = [];
+        const protocolsMethods = [];
         data.sort((a, b) => a.timestamp < b.timestamp ? -1 : 1).forEach(protocolObj => {
-            let newCommandlbj: { [k: string]: any } = {};
+            let newMethodObj: { [k: string]: any } = {};
-            newCommandlbj.timestamp = Utils.getHoursAndMinutes(protocolObj.timestamp);
+            newMethodObj.timestamp = Utils.getHoursAndMinutes(protocolObj.timestamp);
-            protocolObj.protocols.find(protocol => protocol.name === selectedProtocol)?.methods.forEach(command => {
+            protocolObj.protocols.find(protocol => protocol.name === selectedProtocol)?.methods.forEach(method => {
-                newCommandlbj[`${command.name}`] = command[StatsMode[timeLineBarChartMode]]
+                newMethodObj[`${method.name}`] = method[StatsMode[timeLineBarChartMode]]
-                if (commandsNames.indexOf(command.name) === -1)
+                protocolsMethodsNamesAndColors.push({name: method.name, color: method.color});
-                    commandsNames.push(command.name);
             })
-            protocolsCommands.push(newCommandlbj);
+            protocolsMethods.push(newMethodObj);
         })
-        setcommandNames(commandsNames);
+        const uniqueObjArray = Utils.creatUniqueObjArrayByProp(protocolsMethodsNamesAndColors, "name")
-        setCommandStats(protocolsCommands);
+        setMethodsNamesAndColors(uniqueObjArray);
+        setMethodsStats(protocolsMethods);
     }, [data, timeLineBarChartMode, selectedProtocol])
 
-    const bars = useMemo(() => (commandNames || protocolsNamesAndColors).map((entry) => {
+    const bars = useMemo(() => (methodsNamesAndColors || protocolsNamesAndColors).map((entry) => {
-        return <Bar key={entry.name || entry} dataKey={entry.name || entry} stackId="a" fill={entry.color || Utils.stringToColor(entry)} barSize={30} />
+        return <Bar key={entry.name} dataKey={entry.name} stackId="a" fill={entry.color} />
-    }), [protocolsNamesAndColors, commandNames])
+    }), [protocolsNamesAndColors, methodsNamesAndColors])
 
     const renderTick = (tickProps) => {
         const { x, y, payload } = tickProps;
         const { index, value } = payload;
 
-        if (index % 3 === 0) {
+        if (protocolStats.length > 5) {
+            if (index % 3 === 0) {
+                return <text x={x} y={y + 10} textAnchor="end">{`${value}`}</text>;
+            }
+            return null;
+        }
+        else {
             return <text x={x} y={y + 10} textAnchor="end">{`${value}`}</text>;
         }
-        return null;
     };
 
-
     return (
         <div className={styles.barChartContainer}>
             {protocolStats.length > 0 && <BarChart
                 width={750}
                 height={250}
-                data={commandStats || protocolStats}
+                data={methodsStats || protocolStats}
-                barCategoryGap={0}
+                barCategoryGap={1}
-                barSize={30}
                 margin={{
                     top: 20,
                     right: 30,
@@ -92,8 +95,8 @@ export const TimelineBarChart: React.FC<TimelineBarChartProps> = ({ timeLineBarC
                     bottom: 5
                 }}
             >
-                <XAxis dataKey="timestamp" tick={renderTick} tickLine={false} />
+                <XAxis dataKey="timestamp" tick={renderTick} tickLine={false} interval="preserveStart" />
-                <YAxis tickFormatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value} />
+                <YAxis tickFormatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value} interval="preserveEnd"/>
                 <Tooltip formatter={(value) => timeLineBarChartMode === "VOLUME" ? Utils.humanFileSize(value) : value + " Requests"} />
                 {bars}
             </BarChart>}

ui-common/src/components/modals/TrafficStatsModal/TrafficPieChart/TrafficPieChart.tsx

@@ -41,7 +41,7 @@ interface TrafficPieChartProps {
 export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode, data, selectedProtocol }) => {
 
   const [protocolsStats, setProtocolsStats] = useState([]);
-  const [commandStats, setCommandStats] = useState(null);
+  const [methodsStats, setMethodsStats] = useState(null);
 
   useEffect(() => {
     if (!data) return;
@@ -57,16 +57,17 @@ export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode,
 
   useEffect(() => {
     if (selectedProtocol === ALL_PROTOCOLS) {
-      setCommandStats(null);
+      setMethodsStats(null);
       return;
     }
-    const commandsPieData = data.find(protocol => protocol.name === selectedProtocol)?.methods.map(command => {
+    const methodsPieData = data.find(protocol => protocol.name === selectedProtocol)?.methods.map(method => {
       return {
-        name: command.name,
+        name: method.name,
-        value: command[PieChartMode[pieChartMode]]
+        value: method[PieChartMode[pieChartMode]],
+        color: method.color
       }
     })
-    setCommandStats(commandsPieData);
+    setMethodsStats(methodsPieData);
   }, [selectedProtocol, pieChartMode, data])
 
   const pieLegend = useMemo(() => {
@@ -82,7 +83,7 @@ export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode,
     } else {
       legend = data.find(protocol => protocol.name === selectedProtocol)?.methods.map((method) => <div
         style={{ marginBottom: 5, display: "flex" }}>
-        <div style={{ height: 15, width: 30, background: Utils.stringToColor(method.name)}} />
+        <div style={{ height: 15, width: 30, background: method.color}} />
         <span style={{ marginLeft: 5 }}>
           {method.name}
         </span>
@@ -96,7 +97,7 @@ export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode,
       {protocolsStats?.length > 0 && <div style={{ width: "100%", display: "flex", justifyContent: "center" }}>
         <PieChart width={300} height={300}>
           <Pie
-            data={commandStats || protocolsStats}
+            data={methodsStats || protocolsStats}
             dataKey="value"
             cx={150}
             cy={125}
@@ -104,8 +105,8 @@ export const TrafficPieChart: React.FC<TrafficPieChartProps> = ({ pieChartMode,
             label={renderCustomizedLabel}
             outerRadius={125}
             fill="#8884d8">
-            {(commandStats || protocolsStats).map((entry, index) => (
+            {(methodsStats || protocolsStats).map((entry, index) => (
-              <Cell key={`cell-${index}`} fill={entry.color || Utils.stringToColor(entry.name)} />)
+              <Cell key={`cell-${index}`} fill={entry.color} />)
             )}
           </Pie>
           <Legend wrapperStyle={{ position: "absolute", width: "auto", height: "auto", right: -150, top: 0 }} content={pieLegend} />

ui-common/src/components/modals/TrafficStatsModal/TrafficStatsModal.tsx

@@ -33,9 +33,7 @@ interface TrafficStatsModalProps {
   getTrafficStatsDataApi: () => Promise<any>
 }
 
-
+export const ALL_PROTOCOLS = "ALL";
-export const PROTOCOLS = ["ALL", "gRPC", "REDIS", "HTTP", "GQL", "AMQP", "KAFKA"];
-export const ALL_PROTOCOLS = PROTOCOLS[0];
 
 export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, onClose, getTrafficStatsDataApi }) => {
 
@@ -44,6 +42,7 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
   const [selectedProtocol, setSelectedProtocol] = useState(ALL_PROTOCOLS);
   const [pieStatsData, setPieStatsData] = useState(null);
   const [timelineStatsData, setTimelineStatsData] = useState(null);
+  const [protocols, setProtocols] = useState([])
   const [isLoading, setIsLoading] = useState(false);
   const commonClasses = useCommonStyles();
 
@@ -55,6 +54,7 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
           const statsData = await getTrafficStatsDataApi();
           setPieStatsData(statsData.pie);
           setTimelineStatsData(statsData.timeline);
+          setProtocols(statsData.protocols)
         } catch (e) {
           console.error(e)
         } finally {
@@ -109,7 +109,7 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
               <div>
                 <span style={{ marginRight: 15 }}>Protocol</span>
                 <select className={styles.select} value={selectedProtocol} onChange={(e) => setSelectedProtocol(e.target.value)}>
-                  {PROTOCOLS.map(protocol => <option key={protocol} value={protocol}>{protocol}</option>)}
+                  {protocols.map(protocol => <option key={protocol} value={protocol}>{protocol}</option>)}
                 </select>
               </div>
             </div>

ui-common/src/helpers/Utils.ts

@@ -51,17 +51,4 @@ export class Utils {
     return true;
   }
 
-  static stringToColor = (str) => {
-    let colors = ["#e51c23", "#e91e63", "#9c27b0", "#673ab7", "#3f51b5", "#5677fc", "#03a9f4", "#00bcd4", "#009688", "#259b24", "#8bc34a", "#afb42b", "#ff9800", "#ff5722", "#795548", "#607d8b"]
-
-    let hash = 0;
-    if (str.length === 0) return hash;
-    for (let i = 0; i < str.length; i++) {
-      hash = str.charCodeAt(i) + ((hash << 5) - hash);
-      hash = hash & hash;
-    }
-    hash = ((hash % colors.length) + colors.length) % colors.length;
-    return colors[hash];
-  }
-
 }