feat(model): Support Llama-3 (#1436)

README.md

@@ -158,6 +158,8 @@ At present, we have introduced several key features to showcase our current capa
   We offer extensive model support, including dozens of large language models (LLMs) from both open-source and API agents, such as LLaMA/LLaMA2, Baichuan, ChatGLM, Wenxin, Tongyi, Zhipu, and many more. 
 
   - News
+    - 🔥🔥🔥  [Meta-Llama-3-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct)
+    - 🔥🔥🔥  [Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct)
     - 🔥🔥🔥  [CodeQwen1.5-7B-Chat](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat)
     - 🔥🔥🔥  [Qwen1.5-32B-Chat](https://huggingface.co/Qwen/Qwen1.5-32B-Chat)
     - 🔥🔥🔥  [Starling-LM-7B-beta](https://huggingface.co/Nexusflow/Starling-LM-7B-beta)

README.zh.md

@@ -152,6 +152,8 @@
   海量模型支持，包括开源、API代理等几十种大语言模型。如LLaMA/LLaMA2、Baichuan、ChatGLM、文心、通义、智谱等。当前已支持如下模型: 
 
   - 新增支持模型
+    - 🔥🔥🔥  [Meta-Llama-3-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct)
+    - 🔥🔥🔥  [Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct)
     - 🔥🔥🔥  [CodeQwen1.5-7B-Chat](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat)
     - 🔥🔥🔥  [Qwen1.5-32B-Chat](https://huggingface.co/Qwen/Qwen1.5-32B-Chat)
     - 🔥🔥🔥  [Starling-LM-7B-beta](https://huggingface.co/Nexusflow/Starling-LM-7B-beta)
@@ -164,6 +166,7 @@
   - [更多开源模型](https://www.yuque.com/eosphoros/dbgpt-docs/iqaaqwriwhp6zslc#qQktR)
 
   - 支持在线代理模型 
+    - [x] [月之暗面.Moonshot](https://platform.moonshot.cn/docs/)
     - [x] [零一万物.Yi](https://platform.lingyiwanwu.com/docs)
     - [x] [OpenAI·ChatGPT](https://api.openai.com/)
     - [x] [百川·Baichuan](https://platform.baichuan-ai.com/)

dbgpt/configs/model_config.py

@@ -72,6 +72,10 @@ LLM_MODEL_CONFIG = {
     "llama-2-7b": os.path.join(MODEL_PATH, "Llama-2-7b-chat-hf"),
     "llama-2-13b": os.path.join(MODEL_PATH, "Llama-2-13b-chat-hf"),
     "llama-2-70b": os.path.join(MODEL_PATH, "Llama-2-70b-chat-hf"),
+    # https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct
+    "meta-llama-3-8b-instruct": os.path.join(MODEL_PATH, "Meta-Llama-3-8B-Instruct"),
+    # https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct
+    "meta-llama-3-70b-instruct": os.path.join(MODEL_PATH, "Meta-Llama-3-70B-Instruct"),
     "baichuan-13b": os.path.join(MODEL_PATH, "Baichuan-13B-Chat"),
     # please rename "fireballoon/baichuan-vicuna-chinese-7b" to "baichuan-7b"
     "baichuan-7b": os.path.join(MODEL_PATH, "baichuan-7b"),

dbgpt/datasource/rdbms/conn_clickhouse.py

@@ -153,7 +153,7 @@ class ClickhouseConnector(RDBMSConnector):
     @property
     def dialect(self) -> str:
         """Return string representation of dialect to use."""
-        pass
+        return ""
 
     def get_fields(self, table_name) -> List[Tuple]:
         """Get column fields about specified table."""

dbgpt/model/adapter/hf_adapter.py

@@ -270,9 +270,48 @@ class QwenAdapter(NewHFChatModelAdapter):
         )
 
 
+class Llama3Adapter(NewHFChatModelAdapter):
+    """
+    https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct
+    https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct
+    """
+
+    support_4bit: bool = True
+    support_8bit: bool = True
+
+    def do_match(self, lower_model_name_or_path: Optional[str] = None):
+        return lower_model_name_or_path and "llama-3" in lower_model_name_or_path
+
+    def get_str_prompt(
+        self,
+        params: Dict,
+        messages: List[ModelMessage],
+        tokenizer: Any,
+        prompt_template: str = None,
+        convert_to_compatible_format: bool = False,
+    ) -> Optional[str]:
+        str_prompt = super().get_str_prompt(
+            params,
+            messages,
+            tokenizer,
+            prompt_template,
+            convert_to_compatible_format,
+        )
+        terminators = [
+            tokenizer.eos_token_id,
+            tokenizer.convert_tokens_to_ids("<|eot_id|>"),
+        ]
+        exist_token_ids = params.get("stop_token_ids", [])
+        terminators.extend(exist_token_ids)
+        # TODO(fangyinc): We should modify the params in the future
+        params["stop_token_ids"] = terminators
+        return str_prompt
+
+
 register_model_adapter(YiAdapter)
 register_model_adapter(Mixtral8x7BAdapter)
 register_model_adapter(SOLARAdapter)
 register_model_adapter(GemmaAdapter)
 register_model_adapter(StarlingLMAdapter)
 register_model_adapter(QwenAdapter)
+register_model_adapter(Llama3Adapter)

dbgpt/model/llm_out/hf_chat_llm.py

@@ -20,6 +20,8 @@ def huggingface_chat_generate_stream(
     top_p = float(params.get("top_p", 1.0))
     echo = params.get("echo", False)
     max_new_tokens = int(params.get("max_new_tokens", 2048))
+    stop_token_ids = params.get("stop_token_ids", [])
+    do_sample = params.get("do_sample", None)
 
     input_ids = tokenizer(prompt).input_ids
     # input_ids = input_ids.to(device)
@@ -39,13 +41,22 @@ def huggingface_chat_generate_stream(
     streamer = TextIteratorStreamer(
         tokenizer, skip_prompt=not echo, skip_special_tokens=True
     )
-    generate_kwargs = {
+
-        "input_ids": input_ids,
+    base_kwargs = {
         "max_length": context_len,
         "temperature": temperature,
         "streamer": streamer,
+        "top_p": top_p,
     }
 
+    if stop_token_ids:
+        base_kwargs["eos_token_id"] = stop_token_ids
+    if do_sample is not None:
+        base_kwargs["do_sample"] = do_sample
+
+    logger.info(f"Predict with parameters: {base_kwargs}")
+
+    generate_kwargs = {"input_ids": input_ids, **base_kwargs}
     thread = Thread(target=model.generate, kwargs=generate_kwargs)
     thread.start()
     out = ""

setup.py

@@ -30,6 +30,7 @@ BUILD_FROM_SOURCE_URL_FAST_CHAT = os.getenv(
     "BUILD_FROM_SOURCE_URL_FAST_CHAT", "git+https://github.com/lm-sys/FastChat.git"
 )
 BUILD_VERSION_OPENAI = os.getenv("BUILD_VERSION_OPENAI")
+INCLUDE_QUANTIZATION = os.getenv("INCLUDE_QUANTIZATION", "true").lower() == "true"
 
 
 def parse_requirements(file_name: str) -> List[str]:
@@ -552,7 +553,9 @@ def quantization_requires():
             # TODO(yyhhyy): Add autoawq install method for CUDA version 11.8
             quantization_pkgs.extend(["autoawq", _build_autoawq_requires(), "optimum"])
 
-    setup_spec.extras["quantization"] = ["cpm_kernels"] + quantization_pkgs
+    setup_spec.extras["quantization"] = (
+        ["cpm_kernels"] + quantization_pkgs + setup_spec.extras["bitsandbytes"]
+    )
 
 
 def all_vector_store_requires():
@@ -659,7 +662,9 @@ def default_requires():
     setup_spec.extras["default"] += setup_spec.extras["rag"]
     setup_spec.extras["default"] += setup_spec.extras["datasource"]
     setup_spec.extras["default"] += setup_spec.extras["torch"]
-    setup_spec.extras["default"] += setup_spec.extras["quantization"]
+    if INCLUDE_QUANTIZATION:
+        # Add quantization extra to default, default is True
+        setup_spec.extras["default"] += setup_spec.extras["quantization"]
     setup_spec.extras["default"] += setup_spec.extras["cache"]