mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-17 07:26:16 +00:00
Harrison/official pre release (#8106)
This commit is contained in:
Harrison Chase
GitHub
@@ -0,0 +1,10 @@
|
||||
"""Implements Program-Aided Language Models.
|
||||
|
||||
As in https://arxiv.org/pdf/2211.10435.pdf.
|
||||
|
||||
This is vulnerable to arbitrary code execution:
|
||||
https://github.com/hwchase17/langchain/issues/5872
|
||||
"""
|
||||
from langchain_experimental.pal_chain.base import PALChain
|
||||
|
||||
__all__ = ["PALChain"]
|
||||
314
libs/experimental/langchain_experimental/pal_chain/base.py
Normal file
314
libs/experimental/langchain_experimental/pal_chain/base.py
Normal file
@@ -0,0 +1,314 @@
|
||||
"""Implements Program-Aided Language Models.
|
||||
|
||||
This module implements the Program-Aided Language Models (PAL) for generating code
|
||||
solutions. PAL is a technique described in the paper "Program-Aided Language Models"
|
||||
(https://arxiv.org/pdf/2211.10435.pdf).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import warnings
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from langchain.callbacks.manager import CallbackManagerForChainRun
|
||||
from langchain.chains.base import Chain
|
||||
from langchain.chains.llm import LLMChain
|
||||
from langchain.schema import BasePromptTemplate
|
||||
from langchain.schema.language_model import BaseLanguageModel
|
||||
from langchain.utilities import PythonREPL
|
||||
from pydantic import Extra, Field, root_validator
|
||||
|
||||
from langchain_experimental.pal_chain.colored_object_prompt import COLORED_OBJECT_PROMPT
|
||||
from langchain_experimental.pal_chain.math_prompt import MATH_PROMPT
|
||||
|
||||
COMMAND_EXECUTION_FUNCTIONS = ["system", "exec", "execfile", "eval"]
|
||||
|
||||
|
||||
class PALValidation:
|
||||
SOLUTION_EXPRESSION_TYPE_FUNCTION = ast.FunctionDef
|
||||
SOLUTION_EXPRESSION_TYPE_VARIABLE = ast.Name
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
solution_expression_name: Optional[str] = None,
|
||||
solution_expression_type: Optional[type] = None,
|
||||
allow_imports: bool = False,
|
||||
allow_command_exec: bool = False,
|
||||
):
|
||||
"""Initialize a PALValidation instance.
|
||||
|
||||
Args:
|
||||
solution_expression_name (str): Name of the expected solution expression.
|
||||
If passed, solution_expression_type must be passed as well.
|
||||
solution_expression_type (str): AST type of the expected solution
|
||||
expression. If passed, solution_expression_name must be passed as well.
|
||||
Must be one of PALValidation.SOLUTION_EXPRESSION_TYPE_FUNCTION,
|
||||
PALValidation.SOLUTION_EXPRESSION_TYPE_VARIABLE.
|
||||
allow_imports (bool): Allow import statements.
|
||||
allow_command_exec (bool): Allow using known command execution functions.
|
||||
"""
|
||||
self.solution_expression_name = solution_expression_name
|
||||
self.solution_expression_type = solution_expression_type
|
||||
|
||||
if solution_expression_name is not None:
|
||||
if not isinstance(self.solution_expression_name, str):
|
||||
raise ValueError(
|
||||
f"Expected solution_expression_name to be str, "
|
||||
f"instead found {type(self.solution_expression_name)}"
|
||||
)
|
||||
if solution_expression_type is not None:
|
||||
if (
|
||||
self.solution_expression_type
|
||||
is not self.SOLUTION_EXPRESSION_TYPE_FUNCTION
|
||||
and self.solution_expression_type
|
||||
is not self.SOLUTION_EXPRESSION_TYPE_VARIABLE
|
||||
):
|
||||
raise ValueError(
|
||||
f"Expected solution_expression_type to be one of "
|
||||
f"({self.SOLUTION_EXPRESSION_TYPE_FUNCTION},"
|
||||
f"{self.SOLUTION_EXPRESSION_TYPE_VARIABLE}),"
|
||||
f"instead found {self.solution_expression_type}"
|
||||
)
|
||||
|
||||
if solution_expression_name is not None and solution_expression_type is None:
|
||||
raise TypeError(
|
||||
"solution_expression_name "
|
||||
"requires solution_expression_type to be passed as well"
|
||||
)
|
||||
if solution_expression_name is None and solution_expression_type is not None:
|
||||
raise TypeError(
|
||||
"solution_expression_type "
|
||||
"requires solution_expression_name to be passed as well"
|
||||
)
|
||||
|
||||
self.allow_imports = allow_imports
|
||||
self.allow_command_exec = allow_command_exec
|
||||
|
||||
|
||||
class PALChain(Chain):
|
||||
"""Implements Program-Aided Language Models (PAL).
|
||||
|
||||
This class implements the Program-Aided Language Models (PAL) for generating code
|
||||
solutions. PAL is a technique described in the paper "Program-Aided Language Models"
|
||||
(https://arxiv.org/pdf/2211.10435.pdf).
|
||||
"""
|
||||
|
||||
llm_chain: LLMChain
|
||||
llm: Optional[BaseLanguageModel] = None
|
||||
"""[Deprecated]"""
|
||||
prompt: BasePromptTemplate = MATH_PROMPT
|
||||
"""[Deprecated]"""
|
||||
stop: str = "\n\n"
|
||||
"""Stop token to use when generating code."""
|
||||
get_answer_expr: str = "print(solution())"
|
||||
"""Expression to use to get the answer from the generated code."""
|
||||
python_globals: Optional[Dict[str, Any]] = None
|
||||
"""Python globals and locals to use when executing the generated code."""
|
||||
python_locals: Optional[Dict[str, Any]] = None
|
||||
"""Python globals and locals to use when executing the generated code."""
|
||||
output_key: str = "result" #: :meta private:
|
||||
return_intermediate_steps: bool = False
|
||||
"""Whether to return intermediate steps in the generated code."""
|
||||
code_validations: PALValidation = Field(default_factory=PALValidation)
|
||||
"""Validations to perform on the generated code."""
|
||||
timeout: Optional[int] = 10
|
||||
"""Timeout in seconds for the generated code to execute."""
|
||||
|
||||
class Config:
|
||||
"""Configuration for this pydantic object."""
|
||||
|
||||
extra = Extra.forbid
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
@root_validator(pre=True)
|
||||
def raise_deprecation(cls, values: Dict) -> Dict:
|
||||
if "llm" in values:
|
||||
warnings.warn(
|
||||
"Directly instantiating a PALChain with an llm is deprecated. "
|
||||
"Please instantiate with llm_chain argument or using one of "
|
||||
"the class method constructors from_math_prompt, "
|
||||
"from_colored_object_prompt."
|
||||
)
|
||||
if "llm_chain" not in values and values["llm"] is not None:
|
||||
values["llm_chain"] = LLMChain(llm=values["llm"], prompt=MATH_PROMPT)
|
||||
return values
|
||||
|
||||
@property
|
||||
def input_keys(self) -> List[str]:
|
||||
"""Return the singular input key.
|
||||
|
||||
:meta private:
|
||||
"""
|
||||
return self.prompt.input_variables
|
||||
|
||||
@property
|
||||
def output_keys(self) -> List[str]:
|
||||
"""Return the singular output key.
|
||||
|
||||
:meta private:
|
||||
"""
|
||||
if not self.return_intermediate_steps:
|
||||
return [self.output_key]
|
||||
else:
|
||||
return [self.output_key, "intermediate_steps"]
|
||||
|
||||
def _call(
|
||||
self,
|
||||
inputs: Dict[str, Any],
|
||||
run_manager: Optional[CallbackManagerForChainRun] = None,
|
||||
) -> Dict[str, str]:
|
||||
_run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
|
||||
code = self.llm_chain.predict(
|
||||
stop=[self.stop], callbacks=_run_manager.get_child(), **inputs
|
||||
)
|
||||
_run_manager.on_text(code, color="green", end="\n", verbose=self.verbose)
|
||||
PALChain.validate_code(code, self.code_validations)
|
||||
repl = PythonREPL(_globals=self.python_globals, _locals=self.python_locals)
|
||||
res = repl.run(code + f"\n{self.get_answer_expr}", timeout=self.timeout)
|
||||
output = {self.output_key: res.strip()}
|
||||
if self.return_intermediate_steps:
|
||||
output["intermediate_steps"] = code
|
||||
return output
|
||||
|
||||
@classmethod
|
||||
def validate_code(cls, code: str, code_validations: PALValidation) -> None:
|
||||
try:
|
||||
code_tree = ast.parse(code)
|
||||
except (SyntaxError, UnicodeDecodeError):
|
||||
raise ValueError(f"Generated code is not valid python code: {code}")
|
||||
except TypeError:
|
||||
raise ValueError(
|
||||
f"Generated code is expected to be a string, "
|
||||
f"instead found {type(code)}"
|
||||
)
|
||||
except OverflowError:
|
||||
raise ValueError(
|
||||
f"Generated code too long / complex to be parsed by ast: {code}"
|
||||
)
|
||||
|
||||
found_solution_expr = False
|
||||
if code_validations.solution_expression_name is None:
|
||||
# Skip validation if no solution_expression_name was given
|
||||
found_solution_expr = True
|
||||
|
||||
has_imports = False
|
||||
top_level_nodes = list(ast.iter_child_nodes(code_tree))
|
||||
for node in top_level_nodes:
|
||||
if (
|
||||
code_validations.solution_expression_name is not None
|
||||
and code_validations.solution_expression_type is not None
|
||||
):
|
||||
# Check root nodes (like func def)
|
||||
if (
|
||||
isinstance(node, code_validations.solution_expression_type)
|
||||
and hasattr(node, "name")
|
||||
and node.name == code_validations.solution_expression_name
|
||||
):
|
||||
found_solution_expr = True
|
||||
# Check assigned nodes (like answer variable)
|
||||
if isinstance(node, ast.Assign):
|
||||
for target_node in node.targets:
|
||||
if (
|
||||
isinstance(
|
||||
target_node, code_validations.solution_expression_type
|
||||
)
|
||||
and hasattr(target_node, "id")
|
||||
and target_node.id
|
||||
== code_validations.solution_expression_name
|
||||
):
|
||||
found_solution_expr = True
|
||||
if isinstance(node, ast.Import) or isinstance(node, ast.ImportFrom):
|
||||
has_imports = True
|
||||
|
||||
if not found_solution_expr:
|
||||
raise ValueError(
|
||||
f"Generated code is missing the solution expression: "
|
||||
f"{code_validations.solution_expression_name} of type: "
|
||||
f"{code_validations.solution_expression_type}"
|
||||
)
|
||||
|
||||
if not code_validations.allow_imports and has_imports:
|
||||
raise ValueError(f"Generated code has disallowed imports: {code}")
|
||||
|
||||
if (
|
||||
not code_validations.allow_command_exec
|
||||
or not code_validations.allow_imports
|
||||
):
|
||||
for node in ast.walk(code_tree):
|
||||
if (
|
||||
(not code_validations.allow_command_exec)
|
||||
and isinstance(node, ast.Call)
|
||||
and (
|
||||
(
|
||||
hasattr(node.func, "id")
|
||||
and node.func.id in COMMAND_EXECUTION_FUNCTIONS
|
||||
)
|
||||
or (
|
||||
isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr in COMMAND_EXECUTION_FUNCTIONS
|
||||
)
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
f"Found illegal command execution function "
|
||||
f"{node.func.id} in code {code}"
|
||||
)
|
||||
|
||||
if (not code_validations.allow_imports) and (
|
||||
isinstance(node, ast.Import) or isinstance(node, ast.ImportFrom)
|
||||
):
|
||||
raise ValueError(f"Generated code has disallowed imports: {code}")
|
||||
|
||||
@classmethod
|
||||
def from_math_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PALChain:
|
||||
"""Load PAL from math prompt.
|
||||
|
||||
Args:
|
||||
llm (BaseLanguageModel): The language model to use for generating code.
|
||||
|
||||
Returns:
|
||||
PALChain: An instance of PALChain.
|
||||
"""
|
||||
llm_chain = LLMChain(llm=llm, prompt=MATH_PROMPT)
|
||||
code_validations = PALValidation(
|
||||
solution_expression_name="solution",
|
||||
solution_expression_type=PALValidation.SOLUTION_EXPRESSION_TYPE_FUNCTION,
|
||||
)
|
||||
|
||||
return cls(
|
||||
llm_chain=llm_chain,
|
||||
stop="\n\n",
|
||||
get_answer_expr="print(solution())",
|
||||
code_validations=code_validations,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_colored_object_prompt(
|
||||
cls, llm: BaseLanguageModel, **kwargs: Any
|
||||
) -> PALChain:
|
||||
"""Load PAL from colored object prompt.
|
||||
|
||||
Args:
|
||||
llm (BaseLanguageModel): The language model to use for generating code.
|
||||
|
||||
Returns:
|
||||
PALChain: An instance of PALChain.
|
||||
"""
|
||||
llm_chain = LLMChain(llm=llm, prompt=COLORED_OBJECT_PROMPT)
|
||||
code_validations = PALValidation(
|
||||
solution_expression_name="answer",
|
||||
solution_expression_type=PALValidation.SOLUTION_EXPRESSION_TYPE_VARIABLE,
|
||||
)
|
||||
return cls(
|
||||
llm_chain=llm_chain,
|
||||
stop="\n\n\n",
|
||||
get_answer_expr="print(answer)",
|
||||
code_validations=code_validations,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def _chain_type(self) -> str:
|
||||
return "pal_chain"
|
||||
@@ -0,0 +1,77 @@
|
||||
# flake8: noqa
|
||||
from langchain.prompts.prompt import PromptTemplate
|
||||
|
||||
template = (
|
||||
"""
|
||||
# Generate Python3 Code to solve problems
|
||||
# Q: On the nightstand, there is a red pencil, a purple mug, a burgundy keychain, a fuchsia teddy bear, a black plate, and a blue stress ball. What color is the stress ball?
|
||||
# Put objects into a dictionary for quick look up
|
||||
objects = dict()
|
||||
objects['pencil'] = 'red'
|
||||
objects['mug'] = 'purple'
|
||||
objects['keychain'] = 'burgundy'
|
||||
objects['teddy bear'] = 'fuchsia'
|
||||
objects['plate'] = 'black'
|
||||
objects['stress ball'] = 'blue'
|
||||
|
||||
# Look up the color of stress ball
|
||||
stress_ball_color = objects['stress ball']
|
||||
answer = stress_ball_color
|
||||
|
||||
|
||||
# Q: On the table, you see a bunch of objects arranged in a row: a purple paperclip, a pink stress ball, a brown keychain, a green scrunchiephone charger, a mauve fidget spinner, and a burgundy pen. What is the color of the object directly to the right of the stress ball?
|
||||
# Put objects into a list to record ordering
|
||||
objects = []
|
||||
objects += [('paperclip', 'purple')] * 1
|
||||
objects += [('stress ball', 'pink')] * 1
|
||||
objects += [('keychain', 'brown')] * 1
|
||||
objects += [('scrunchiephone charger', 'green')] * 1
|
||||
objects += [('fidget spinner', 'mauve')] * 1
|
||||
objects += [('pen', 'burgundy')] * 1
|
||||
|
||||
# Find the index of the stress ball
|
||||
stress_ball_idx = None
|
||||
for i, object in enumerate(objects):
|
||||
if object[0] == 'stress ball':
|
||||
stress_ball_idx = i
|
||||
break
|
||||
|
||||
# Find the directly right object
|
||||
direct_right = objects[i+1]
|
||||
|
||||
# Check the directly right object's color
|
||||
direct_right_color = direct_right[1]
|
||||
answer = direct_right_color
|
||||
|
||||
|
||||
# Q: On the nightstand, you see the following items arranged in a row: a teal plate, a burgundy keychain, a yellow scrunchiephone charger, an orange mug, a pink notebook, and a grey cup. How many non-orange items do you see to the left of the teal item?
|
||||
# Put objects into a list to record ordering
|
||||
objects = []
|
||||
objects += [('plate', 'teal')] * 1
|
||||
objects += [('keychain', 'burgundy')] * 1
|
||||
objects += [('scrunchiephone charger', 'yellow')] * 1
|
||||
objects += [('mug', 'orange')] * 1
|
||||
objects += [('notebook', 'pink')] * 1
|
||||
objects += [('cup', 'grey')] * 1
|
||||
|
||||
# Find the index of the teal item
|
||||
teal_idx = None
|
||||
for i, object in enumerate(objects):
|
||||
if object[1] == 'teal':
|
||||
teal_idx = i
|
||||
break
|
||||
|
||||
# Find non-orange items to the left of the teal item
|
||||
non_orange = [object for object in objects[:i] if object[1] != 'orange']
|
||||
|
||||
# Count number of non-orange objects
|
||||
num_non_orange = len(non_orange)
|
||||
answer = num_non_orange
|
||||
|
||||
|
||||
# Q: {question}
|
||||
""".strip()
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
COLORED_OBJECT_PROMPT = PromptTemplate(input_variables=["question"], template=template)
|
||||
@@ -0,0 +1,157 @@
|
||||
# flake8: noqa
|
||||
from langchain.prompts.prompt import PromptTemplate
|
||||
|
||||
template = (
|
||||
'''
|
||||
Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?
|
||||
|
||||
# solution in Python:
|
||||
|
||||
|
||||
def solution():
|
||||
"""Olivia has $23. She bought five bagels for $3 each. How much money does she have left?"""
|
||||
money_initial = 23
|
||||
bagels = 5
|
||||
bagel_cost = 3
|
||||
money_spent = bagels * bagel_cost
|
||||
money_left = money_initial - money_spent
|
||||
result = money_left
|
||||
return result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?
|
||||
|
||||
# solution in Python:
|
||||
|
||||
|
||||
def solution():
|
||||
"""Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?"""
|
||||
golf_balls_initial = 58
|
||||
golf_balls_lost_tuesday = 23
|
||||
golf_balls_lost_wednesday = 2
|
||||
golf_balls_left = golf_balls_initial - golf_balls_lost_tuesday - golf_balls_lost_wednesday
|
||||
result = golf_balls_left
|
||||
return result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Q: There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?
|
||||
|
||||
# solution in Python:
|
||||
|
||||
|
||||
def solution():
|
||||
"""There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?"""
|
||||
computers_initial = 9
|
||||
computers_per_day = 5
|
||||
num_days = 4 # 4 days between monday and thursday
|
||||
computers_added = computers_per_day * num_days
|
||||
computers_total = computers_initial + computers_added
|
||||
result = computers_total
|
||||
return result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?
|
||||
|
||||
# solution in Python:
|
||||
|
||||
|
||||
def solution():
|
||||
"""Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?"""
|
||||
toys_initial = 5
|
||||
mom_toys = 2
|
||||
dad_toys = 2
|
||||
total_received = mom_toys + dad_toys
|
||||
total_toys = toys_initial + total_received
|
||||
result = total_toys
|
||||
return result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?
|
||||
|
||||
# solution in Python:
|
||||
|
||||
|
||||
def solution():
|
||||
"""Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?"""
|
||||
jason_lollipops_initial = 20
|
||||
jason_lollipops_after = 12
|
||||
denny_lollipops = jason_lollipops_initial - jason_lollipops_after
|
||||
result = denny_lollipops
|
||||
return result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?
|
||||
|
||||
# solution in Python:
|
||||
|
||||
|
||||
def solution():
|
||||
"""Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?"""
|
||||
leah_chocolates = 32
|
||||
sister_chocolates = 42
|
||||
total_chocolates = leah_chocolates + sister_chocolates
|
||||
chocolates_eaten = 35
|
||||
chocolates_left = total_chocolates - chocolates_eaten
|
||||
result = chocolates_left
|
||||
return result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?
|
||||
|
||||
# solution in Python:
|
||||
|
||||
|
||||
def solution():
|
||||
"""If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?"""
|
||||
cars_initial = 3
|
||||
cars_arrived = 2
|
||||
total_cars = cars_initial + cars_arrived
|
||||
result = total_cars
|
||||
return result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?
|
||||
|
||||
# solution in Python:
|
||||
|
||||
|
||||
def solution():
|
||||
"""There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?"""
|
||||
trees_initial = 15
|
||||
trees_after = 21
|
||||
trees_added = trees_after - trees_initial
|
||||
result = trees_added
|
||||
return result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Q: {question}
|
||||
|
||||
# solution in Python:
|
||||
'''.strip()
|
||||
+ "\n\n\n"
|
||||
)
|
||||
MATH_PROMPT = PromptTemplate(input_variables=["question"], template=template)
|
||||
Reference in New Issue
Block a user