From b29c6e45825807d07dd6fdb954457772f40b1b0e Mon Sep 17 00:00:00 2001 From: Thomas Schuetz Date: Sat, 25 Mar 2023 14:25:22 +0100 Subject: [PATCH 1/3] feat: find parent objects Signed-off-by: Thomas Schuetz --- cmd/find/problems.go | 3 +- pkg/analyzer/analysis.go | 22 ++++++--- pkg/analyzer/podAnalyzer.go | 89 ++++++++++++++++++++++++++++++------- pkg/analyzer/rsAnalyzer.go | 20 ++++++--- 4 files changed, 105 insertions(+), 29 deletions(-) diff --git a/cmd/find/problems.go b/cmd/find/problems.go index 7a6c947..8f40118 100644 --- a/cmd/find/problems.go +++ b/cmd/find/problems.go @@ -79,8 +79,7 @@ var problemsCmd = &cobra.Command{ } fmt.Println(string(j)) default: - fmt.Printf("%s %s: %s \n%s\n", color.CyanString("%d", n), color.YellowString(analysis.Name), color.RedString(analysis.Error), color.GreenString(analysis.Details)) - + fmt.Printf("%s %s(%s): %s \n%s\n", color.CyanString("%d", n), color.YellowString(analysis.Name), color.CyanString(analysis.ParentObject), color.RedString(analysis.Error), color.GreenString(analysis.Details)) } } diff --git a/pkg/analyzer/analysis.go b/pkg/analyzer/analysis.go index b627925..7a35075 100644 --- a/pkg/analyzer/analysis.go +++ b/pkg/analyzer/analysis.go @@ -1,8 +1,20 @@ package analyzer -type Analysis struct { - Kind string - Name string - Error string - Details string +import ( + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" +) + +type PreAnalysis struct { + Pod v1.Pod + FailureDetails []string + ReplicaSet appsv1.ReplicaSet +} + +type Analysis struct { + Kind string `json:"kind"` + Name string `json:"name"` + Error string `json:"error"` + Details string `json:"details"` + ParentObject string `json:"parentObject"` } diff --git a/pkg/analyzer/podAnalyzer.go b/pkg/analyzer/podAnalyzer.go index ec01d08..64e17d6 100644 --- a/pkg/analyzer/podAnalyzer.go +++ b/pkg/analyzer/podAnalyzer.go @@ -22,30 +22,30 @@ func AnalyzePod(ctx context.Context, client *kubernetes.Client, aiClient ai.IAI, if err != nil { return err } - - var brokenPods = map[string][]string{} + var preAnalysis = map[string]PreAnalysis{} for _, pod := range list.Items { - + var failures []string // Check for pending pods if pod.Status.Phase == "Pending" { // Check through container status to check for crashes for _, containerStatus := range pod.Status.Conditions { if containerStatus.Type == "PodScheduled" && containerStatus.Reason == "Unschedulable" { - brokenPods[fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)] = []string{containerStatus.Message} + if containerStatus.Message != "" { + failures = []string{containerStatus.Message} + } } } } // Check through container status to check for crashes - var failureDetails = []string{} for _, containerStatus := range pod.Status.ContainerStatuses { if containerStatus.State.Waiting != nil { if containerStatus.State.Waiting.Reason == "CrashLoopBackOff" || containerStatus.State.Waiting.Reason == "ImagePullBackOff" { - - failureDetails = append(failureDetails, containerStatus.State.Waiting.Message) - brokenPods[fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)] = failureDetails + if containerStatus.State.Waiting.Message != "" { + failures = append(failures, containerStatus.State.Waiting.Message) + } } // This represents a container that is still being created or blocked due to conditions such as OOMKilled if containerStatus.State.Waiting.Reason == "ContainerCreating" && pod.Status.Phase == "Pending" { @@ -55,24 +55,30 @@ func AnalyzePod(ctx context.Context, client *kubernetes.Client, aiClient ai.IAI, if err != nil { continue } - if evt.Reason == "FailedCreatePodSandBox" { - failureDetails = append(failureDetails, evt.Message) - brokenPods[fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)] = failureDetails + if evt.Reason == "FailedCreatePodSandBox" && evt.Message != "" { + failures = append(failures, evt.Message) } } - } } - + if len(failures) > 0 { + preAnalysis[fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)] = PreAnalysis{ + Pod: pod, + FailureDetails: failures, + } + } } - for key, value := range brokenPods { - inputValue := strings.Join(value, " ") + for key, value := range preAnalysis { + inputValue := strings.Join(value.FailureDetails, " ") var currentAnalysis = Analysis{ Kind: "Pod", Name: key, - Error: value[0], + Error: value.FailureDetails[0], } + + parent, _ := getParent(client, value.Pod.ObjectMeta) + if explain { s := spinner.New(spinner.CharSets[35], 100*time.Millisecond) // Build our new spinner s.Start() @@ -112,10 +118,59 @@ func AnalyzePod(ctx context.Context, client *kubernetes.Client, aiClient ai.IAI, } } currentAnalysis.Details = response - } + currentAnalysis.ParentObject = parent *analysisResults = append(*analysisResults, currentAnalysis) } return nil } + +func getParent(client *kubernetes.Client, meta metav1.ObjectMeta) (string, bool) { + if meta.OwnerReferences != nil { + for _, owner := range meta.OwnerReferences { + switch owner.Kind { + case "ReplicaSet": + rs, err := client.GetClient().AppsV1().ReplicaSets(meta.Namespace).Get(context.Background(), owner.Name, metav1.GetOptions{}) + if err != nil { + return "", false + } + if rs.OwnerReferences != nil { + return getParent(client, rs.ObjectMeta) + } + return "ReplicaSet/" + rs.Name, false + + case "Deployment": + dep, err := client.GetClient().AppsV1().Deployments(meta.Namespace).Get(context.Background(), owner.Name, metav1.GetOptions{}) + if err != nil { + return "", false + } + if dep.OwnerReferences != nil { + return getParent(client, dep.ObjectMeta) + } + return "Deployment/" + dep.Name, false + + case "StatefulSet": + sts, err := client.GetClient().AppsV1().StatefulSets(meta.Namespace).Get(context.Background(), owner.Name, metav1.GetOptions{}) + if err != nil { + return "", false + } + if sts.OwnerReferences != nil { + return getParent(client, sts.ObjectMeta) + } + return "StatefulSet/" + sts.Name, false + + case "DaemonSet": + ds, err := client.GetClient().AppsV1().DaemonSets(meta.Namespace).Get(context.Background(), owner.Name, metav1.GetOptions{}) + if err != nil { + return "", false + } + if ds.OwnerReferences != nil { + return getParent(client, ds.ObjectMeta) + } + return "DaemonSet/" + ds.Name, false + } + } + } + return meta.Name, false +} diff --git a/pkg/analyzer/rsAnalyzer.go b/pkg/analyzer/rsAnalyzer.go index bebde09..12c3094 100644 --- a/pkg/analyzer/rsAnalyzer.go +++ b/pkg/analyzer/rsAnalyzer.go @@ -23,9 +23,10 @@ func AnalyzeReplicaSet(ctx context.Context, client *kubernetes.Client, aiClient return err } - var brokenRS = map[string][]string{} + var preAnalysis = map[string]PreAnalysis{} for _, rs := range list.Items { + var failures []string // Check for empty rs if rs.Status.Replicas == 0 { @@ -33,24 +34,32 @@ func AnalyzeReplicaSet(ctx context.Context, client *kubernetes.Client, aiClient // Check through container status to check for crashes for _, rsStatus := range rs.Status.Conditions { if rsStatus.Type == "ReplicaFailure" && rsStatus.Reason == "FailedCreate" { - brokenRS[fmt.Sprintf("%s/%s", rs.Namespace, rs.Name)] = []string{rsStatus.Message} + failures = []string{rsStatus.Message} } } } + if len(failures) > 0 { + preAnalysis[fmt.Sprintf("%s/%s", rs.Namespace, rs.Name)] = PreAnalysis{ + ReplicaSet: rs, + FailureDetails: failures, + } + } } - for key, value := range brokenRS { + for key, value := range preAnalysis { var currentAnalysis = Analysis{ Kind: "ReplicaSet", Name: key, - Error: value[0], + Error: value.FailureDetails[0], } + parent, _ := getParent(client, value.ReplicaSet.ObjectMeta) + if explain { s := spinner.New(spinner.CharSets[35], 100*time.Millisecond) // Build our new spinner s.Start() - inputValue := strings.Join(value, " ") + inputValue := strings.Join(value.FailureDetails, " ") // Check for cached data sEnc := base64.StdEncoding.EncodeToString([]byte(inputValue)) @@ -88,6 +97,7 @@ func AnalyzeReplicaSet(ctx context.Context, client *kubernetes.Client, aiClient } currentAnalysis.Details = response } + currentAnalysis.ParentObject = parent *analysisResults = append(*analysisResults, currentAnalysis) } From 9c7d55955b777ad201307cb946e0fc81cf9c4b99 Mon Sep 17 00:00:00 2001 From: Thomas Schuetz Date: Sat, 25 Mar 2023 15:57:48 +0100 Subject: [PATCH 2/3] fix: missing parent when explain is used Signed-off-by: Thomas Schuetz --- pkg/analyzer/podAnalyzer.go | 1 + pkg/analyzer/rsAnalyzer.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/analyzer/podAnalyzer.go b/pkg/analyzer/podAnalyzer.go index 64e17d6..fabb1e8 100644 --- a/pkg/analyzer/podAnalyzer.go +++ b/pkg/analyzer/podAnalyzer.go @@ -100,6 +100,7 @@ func AnalyzePod(ctx context.Context, client *kubernetes.Client, aiClient ai.IAI, continue } currentAnalysis.Details = string(output) + currentAnalysis.ParentObject = parent *analysisResults = append(*analysisResults, currentAnalysis) continue } diff --git a/pkg/analyzer/rsAnalyzer.go b/pkg/analyzer/rsAnalyzer.go index 12c3094..94e5bce 100644 --- a/pkg/analyzer/rsAnalyzer.go +++ b/pkg/analyzer/rsAnalyzer.go @@ -78,6 +78,7 @@ func AnalyzeReplicaSet(ctx context.Context, client *kubernetes.Client, aiClient continue } currentAnalysis.Details = string(output) + currentAnalysis.ParentObject = parent *analysisResults = append(*analysisResults, currentAnalysis) continue } From 961fb6c555f59f1276531f462739b76b1508830e Mon Sep 17 00:00:00 2001 From: Thomas Schuetz Date: Sat, 25 Mar 2023 22:53:14 +0100 Subject: [PATCH 3/3] feat: add service analysis Signed-off-by: Thomas Schuetz --- pkg/analyzer/analysis.go | 1 + pkg/analyzer/analyzer.go | 5 ++ pkg/analyzer/serviceAnalyzer.go | 121 ++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 pkg/analyzer/serviceAnalyzer.go diff --git a/pkg/analyzer/analysis.go b/pkg/analyzer/analysis.go index 7a35075..11ddc72 100644 --- a/pkg/analyzer/analysis.go +++ b/pkg/analyzer/analysis.go @@ -9,6 +9,7 @@ type PreAnalysis struct { Pod v1.Pod FailureDetails []string ReplicaSet appsv1.ReplicaSet + Endpoint v1.Endpoints } type Analysis struct { diff --git a/pkg/analyzer/analyzer.go b/pkg/analyzer/analyzer.go index 595a27f..db4df4b 100644 --- a/pkg/analyzer/analyzer.go +++ b/pkg/analyzer/analyzer.go @@ -18,5 +18,10 @@ func RunAnalysis(ctx context.Context, client *kubernetes.Client, aiClient ai.IAI if err != nil { return err } + + err = AnalyzeEndpoints(ctx, client, aiClient, explain, analysisResults) + if err != nil { + return err + } return nil } diff --git a/pkg/analyzer/serviceAnalyzer.go b/pkg/analyzer/serviceAnalyzer.go new file mode 100644 index 0000000..085bb8f --- /dev/null +++ b/pkg/analyzer/serviceAnalyzer.go @@ -0,0 +1,121 @@ +package analyzer + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + "time" + + "github.com/briandowns/spinner" + "github.com/fatih/color" + "github.com/k8sgpt-ai/k8sgpt/pkg/ai" + "github.com/k8sgpt-ai/k8sgpt/pkg/kubernetes" + "github.com/spf13/viper" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func AnalyzeEndpoints(ctx context.Context, client *kubernetes.Client, aiClient ai.IAI, explain bool, analysisResults *[]Analysis) error { + + // search all namespaces for pods that are not running + list, err := client.GetClient().CoreV1().Endpoints("").List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + + var preAnalysis = map[string]PreAnalysis{} + + for _, ep := range list.Items { + var failures []string + + // Check for empty service + if len(ep.Subsets) == 0 { + svc, err := client.GetClient().CoreV1().Services(ep.Namespace).Get(ctx, ep.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + for k, v := range svc.Spec.Selector { + failures = append(failures, fmt.Sprintf("Service has no endpoints, expected label %s=%s", k, v)) + } + } else { + count := 0 + pods := []string{} + + // Check through container status to check for crashes + for _, epSubset := range ep.Subsets { + if len(epSubset.NotReadyAddresses) > 0 { + for _, addresses := range epSubset.NotReadyAddresses { + count++ + pods = append(pods, addresses.TargetRef.Kind+"/"+addresses.TargetRef.Name) + } + failures = append(failures, fmt.Sprintf("Service has not ready endpoints, pods: %s, expected %d", pods, count)) + } + } + } + + if len(failures) > 0 { + preAnalysis[fmt.Sprintf("%s/%s", ep.Namespace, ep.Name)] = PreAnalysis{ + Endpoint: ep, + FailureDetails: failures, + } + } + } + + for key, value := range preAnalysis { + var currentAnalysis = Analysis{ + Kind: "Service", + Name: key, + Error: value.FailureDetails[0], + } + + parent, _ := getParent(client, value.Endpoint.ObjectMeta) + + if explain { + s := spinner.New(spinner.CharSets[35], 100*time.Millisecond) // Build our new spinner + s.Start() + + inputValue := strings.Join(value.FailureDetails, " ") + + // Check for cached data + sEnc := base64.StdEncoding.EncodeToString([]byte(inputValue)) + // find in viper cache + if viper.IsSet(sEnc) { + s.Stop() + // retrieve data from cache + response := viper.GetString(sEnc) + if response == "" { + color.Red("error retrieving cached data") + continue + } + output, err := base64.StdEncoding.DecodeString(response) + if err != nil { + color.Red("error decoding cached data: %v", err) + continue + } + currentAnalysis.Details = string(output) + currentAnalysis.ParentObject = parent + *analysisResults = append(*analysisResults, currentAnalysis) + continue + } + + response, err := aiClient.GetCompletion(ctx, inputValue) + s.Stop() + if err != nil { + color.Red("error getting completion: %v", err) + continue + } + + if !viper.IsSet(sEnc) { + viper.Set(sEnc, base64.StdEncoding.EncodeToString([]byte(response))) + if err := viper.WriteConfig(); err != nil { + return err + } + } + currentAnalysis.Details = response + } + currentAnalysis.ParentObject = parent + *analysisResults = append(*analysisResults, currentAnalysis) + } + return nil +}