[Online Server] Chat Api for streaming and not streaming response (#5470)

* fix bugs

* fix bugs

* fix api server

* fix api server

* add chat api and test

* del request.n

examples/inference/client/locustfile.py

@@ -14,9 +14,37 @@ class QuickstartUser(HttpUser):
     def completion_streaming(self):
         self.client.post("/v1/completion", json={"prompt": "hello, who are you? ", "stream": "True"})
 
+    @tag("online-chat")
+    @task(5)
+    def chat(self):
+        self.client.post(
+            "v1/chat",
+            json={
+                "converation": [
+                    {"role": "system", "content": "you are a helpful assistant"},
+                    {"role": "user", "content": "what is 1+1?"},
+                ],
+                "stream": "False",
+            },
+        )
+
+    @tag("online-chat")
+    @task(5)
+    def chat_streaming(self):
+        self.client.post(
+            "v1/chat",
+            json={
+                "converation": [
+                    {"role": "system", "content": "you are a helpful assistant"},
+                    {"role": "user", "content": "what is 1+1?"},
+                ],
+                "stream": "True",
+            },
+        )
+
     @tag("offline-generation")
     @task(5)
-    def generate_stream(self):
+    def generate_streaming(self):
         self.client.post("/generate", json={"prompt": "Can you help me? ", "stream": "True"})
 
     @tag("offline-generation")