feat: implement Top-K sampling for improved user control (#1110)

This commit adds Top-K sampling, a feature that allows users to control
the randomness of the generated text by specifying the number of most
probable next words considered by the model. This enhances user control
and potentially improves the quality of the generated outputs.

Fixes: https://github.com/k8sgpt-ai/k8sgpt/issues/1105

Signed-off-by: VaibhavMalik4187 <vaibhavmalik2018@gmail.com>
Co-authored-by: Alex Jones <alexsimonjones@gmail.com>

cmd/serve/serve.go

@@ -28,6 +28,7 @@ import (
 const (
 	defaultTemperature float32 = 0.7
 	defaultTopP        float32 = 1.0
+	defaultTopK        int32   = 50
 )
 
 var (
@@ -84,6 +85,22 @@ var ServeCmd = &cobra.Command{
 				}
 				return float32(topP)
 			}
+			topK := func() int32 {
+				env := os.Getenv("K8SGPT_TOP_K")
+				if env == "" {
+					return defaultTopK
+				}
+				topK, err := strconv.ParseFloat(env, 32)
+				if err != nil {
+					color.Red("Unable to convert topK value: %v", err)
+					os.Exit(1)
+				}
+				if topK < 10 || topK > 100 {
+					color.Red("Error: topK ranges from 1 to 100.")
+					os.Exit(1)
+				}
+				return int32(topK)
+			}
 			// Check for env injection
 			backend = os.Getenv("K8SGPT_BACKEND")
 			password := os.Getenv("K8SGPT_PASSWORD")
@@ -104,6 +121,7 @@ var ServeCmd = &cobra.Command{
 					ProxyEndpoint: proxyEndpoint,
 					Temperature:   temperature(),
 					TopP:          topP(),
+					TopK:          topK(),
 				}
 
 				configAI.Providers = append(configAI.Providers, *aiProvider)