mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-23 11:30:37 +00:00
community[minor]: migrate bigdl-llm to ipex-llm (#19518)
- **Description**: `bigdl-llm` library has been renamed to [`ipex-llm`](https://github.com/intel-analytics/ipex-llm). This PR migrates the `bigdl-llm` integration to `ipex-llm` . - **Issue**: N/A. The original PR of `bigdl-llm` is https://github.com/langchain-ai/langchain/pull/17953 - **Dependencies**: `ipex-llm` library - **Contribution maintainer**: @shane-huang Updated doc: docs/docs/integrations/llms/ipex_llm.ipynb Updated test: libs/community/tests/integration_tests/llms/test_ipex_llm.py
This commit is contained in:
Shengsheng Huang
GitHub
223
libs/community/langchain_community/llms/ipex_llm.py
Normal file
223
libs/community/langchain_community/llms/ipex_llm.py
Normal file
@@ -0,0 +1,223 @@
|
||||
import logging
|
||||
from typing import Any, List, Mapping, Optional
|
||||
|
||||
from langchain_core.callbacks import CallbackManagerForLLMRun
|
||||
from langchain_core.language_models.llms import LLM
|
||||
from langchain_core.pydantic_v1 import Extra
|
||||
|
||||
DEFAULT_MODEL_ID = "gpt2"
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IpexLLM(LLM):
|
||||
"""Wrapper around the IpexLLM model
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
|
||||
from langchain_community.llms import IpexLLM
|
||||
llm = IpexLLM.from_model_id(model_id="THUDM/chatglm-6b")
|
||||
"""
|
||||
|
||||
model_id: str = DEFAULT_MODEL_ID
|
||||
"""Model name or model path to use."""
|
||||
model_kwargs: Optional[dict] = None
|
||||
"""Keyword arguments passed to the model."""
|
||||
model: Any #: :meta private:
|
||||
"""IpexLLM model."""
|
||||
tokenizer: Any #: :meta private:
|
||||
"""Huggingface tokenizer model."""
|
||||
streaming: bool = True
|
||||
"""Whether to stream the results, token by token."""
|
||||
|
||||
class Config:
|
||||
"""Configuration for this pydantic object."""
|
||||
|
||||
extra = Extra.forbid
|
||||
|
||||
@classmethod
|
||||
def from_model_id(
|
||||
cls,
|
||||
model_id: str,
|
||||
model_kwargs: Optional[dict] = None,
|
||||
**kwargs: Any,
|
||||
) -> LLM:
|
||||
"""
|
||||
Construct object from model_id
|
||||
|
||||
Args:
|
||||
model_id: Path for the huggingface repo id to be downloaded or
|
||||
the huggingface checkpoint folder.
|
||||
model_kwargs: Keyword arguments to pass to the model and tokenizer.
|
||||
kwargs: Extra arguments to pass to the model and tokenizer.
|
||||
|
||||
Returns:
|
||||
An object of IpexLLM.
|
||||
"""
|
||||
try:
|
||||
from ipex_llm.transformers import (
|
||||
AutoModel,
|
||||
AutoModelForCausalLM,
|
||||
)
|
||||
from transformers import AutoTokenizer, LlamaTokenizer
|
||||
|
||||
except ImportError:
|
||||
raise ValueError(
|
||||
"Could not import ipex-llm or transformers. "
|
||||
"Please install it with `pip install --pre --upgrade ipex-llm[all]`."
|
||||
)
|
||||
|
||||
_model_kwargs = model_kwargs or {}
|
||||
|
||||
try:
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs)
|
||||
except Exception:
|
||||
tokenizer = LlamaTokenizer.from_pretrained(model_id, **_model_kwargs)
|
||||
|
||||
try:
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_id, load_in_4bit=True, **_model_kwargs
|
||||
)
|
||||
except Exception:
|
||||
model = AutoModel.from_pretrained(
|
||||
model_id, load_in_4bit=True, **_model_kwargs
|
||||
)
|
||||
|
||||
if "trust_remote_code" in _model_kwargs:
|
||||
_model_kwargs = {
|
||||
k: v for k, v in _model_kwargs.items() if k != "trust_remote_code"
|
||||
}
|
||||
|
||||
return cls(
|
||||
model_id=model_id,
|
||||
model=model,
|
||||
tokenizer=tokenizer,
|
||||
model_kwargs=_model_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_model_id_low_bit(
|
||||
cls,
|
||||
model_id: str,
|
||||
model_kwargs: Optional[dict] = None,
|
||||
**kwargs: Any,
|
||||
) -> LLM:
|
||||
"""
|
||||
Construct low_bit object from model_id
|
||||
|
||||
Args:
|
||||
|
||||
model_id: Path for the ipex-llm transformers low-bit model folder.
|
||||
model_kwargs: Keyword arguments to pass to the model and tokenizer.
|
||||
kwargs: Extra arguments to pass to the model and tokenizer.
|
||||
|
||||
Returns:
|
||||
An object of IpexLLM.
|
||||
"""
|
||||
try:
|
||||
from ipex_llm.transformers import (
|
||||
AutoModel,
|
||||
AutoModelForCausalLM,
|
||||
)
|
||||
from transformers import AutoTokenizer, LlamaTokenizer
|
||||
|
||||
except ImportError:
|
||||
raise ValueError(
|
||||
"Could not import ipex-llm or transformers. "
|
||||
"Please install it with `pip install --pre --upgrade ipex-llm[all]`."
|
||||
)
|
||||
|
||||
_model_kwargs = model_kwargs or {}
|
||||
try:
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs)
|
||||
except Exception:
|
||||
tokenizer = LlamaTokenizer.from_pretrained(model_id, **_model_kwargs)
|
||||
|
||||
try:
|
||||
model = AutoModelForCausalLM.load_low_bit(model_id, **_model_kwargs)
|
||||
except Exception:
|
||||
model = AutoModel.load_low_bit(model_id, **_model_kwargs)
|
||||
|
||||
if "trust_remote_code" in _model_kwargs:
|
||||
_model_kwargs = {
|
||||
k: v for k, v in _model_kwargs.items() if k != "trust_remote_code"
|
||||
}
|
||||
|
||||
return cls(
|
||||
model_id=model_id,
|
||||
model=model,
|
||||
tokenizer=tokenizer,
|
||||
model_kwargs=_model_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def _identifying_params(self) -> Mapping[str, Any]:
|
||||
"""Get the identifying parameters."""
|
||||
return {
|
||||
"model_id": self.model_id,
|
||||
"model_kwargs": self.model_kwargs,
|
||||
}
|
||||
|
||||
@property
|
||||
def _llm_type(self) -> str:
|
||||
return "ipex-llm"
|
||||
|
||||
def _call(
|
||||
self,
|
||||
prompt: str,
|
||||
stop: Optional[List[str]] = None,
|
||||
run_manager: Optional[CallbackManagerForLLMRun] = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
if self.streaming:
|
||||
from transformers import TextStreamer
|
||||
|
||||
input_ids = self.tokenizer.encode(prompt, return_tensors="pt")
|
||||
streamer = TextStreamer(
|
||||
self.tokenizer, skip_prompt=True, skip_special_tokens=True
|
||||
)
|
||||
if stop is not None:
|
||||
from transformers.generation.stopping_criteria import (
|
||||
StoppingCriteriaList,
|
||||
)
|
||||
from transformers.tools.agents import StopSequenceCriteria
|
||||
|
||||
# stop generation when stop words are encountered
|
||||
# TODO: stop generation when the following one is stop word
|
||||
stopping_criteria = StoppingCriteriaList(
|
||||
[StopSequenceCriteria(stop, self.tokenizer)]
|
||||
)
|
||||
else:
|
||||
stopping_criteria = None
|
||||
output = self.model.generate(
|
||||
input_ids,
|
||||
streamer=streamer,
|
||||
stopping_criteria=stopping_criteria,
|
||||
**kwargs,
|
||||
)
|
||||
text = self.tokenizer.decode(output[0], skip_special_tokens=True)
|
||||
return text
|
||||
else:
|
||||
input_ids = self.tokenizer.encode(prompt, return_tensors="pt")
|
||||
if stop is not None:
|
||||
from transformers.generation.stopping_criteria import (
|
||||
StoppingCriteriaList,
|
||||
)
|
||||
from transformers.tools.agents import StopSequenceCriteria
|
||||
|
||||
stopping_criteria = StoppingCriteriaList(
|
||||
[StopSequenceCriteria(stop, self.tokenizer)]
|
||||
)
|
||||
else:
|
||||
stopping_criteria = None
|
||||
output = self.model.generate(
|
||||
input_ids, stopping_criteria=stopping_criteria, **kwargs
|
||||
)
|
||||
text = self.tokenizer.decode(output[0], skip_special_tokens=True)[
|
||||
len(prompt) :
|
||||
]
|
||||
return text
|
||||
Reference in New Issue
Block a user