add LLMBashChain to experimental (#11305)

Add LLMBashChain to experimental

libs/experimental/tests/unit_tests/test_bash.py

@@ -0,0 +1,102 @@
+"""Test the bash utility."""
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from langchain_experimental.llm_bash.bash import BashProcess
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="Test not supported on Windows"
+)
+def test_pwd_command() -> None:
+    """Test correct functionality."""
+    session = BashProcess()
+    commands = ["pwd"]
+    output = session.run(commands)
+
+    assert output == subprocess.check_output("pwd", shell=True).decode()
+
+
+@pytest.mark.skip(reason="flaky on GHA, TODO to fix")
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="Test not supported on Windows"
+)
+def test_pwd_command_persistent() -> None:
+    """Test correct functionality when the bash process is persistent."""
+    session = BashProcess(persistent=True, strip_newlines=True)
+    commands = ["pwd"]
+    output = session.run(commands)
+
+    assert subprocess.check_output("pwd", shell=True).decode().strip() in output
+
+    session.run(["cd .."])
+    new_output = session.run(["pwd"])
+    # Assert that the new_output is a parent of the old output
+    assert Path(output).parent == Path(new_output)
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="Test not supported on Windows"
+)
+def test_incorrect_command() -> None:
+    """Test handling of incorrect command."""
+    session = BashProcess()
+    output = session.run(["invalid_command"])
+    assert output == "Command 'invalid_command' returned non-zero exit status 127."
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="Test not supported on Windows"
+)
+def test_incorrect_command_return_err_output() -> None:
+    """Test optional returning of shell output on incorrect command."""
+    session = BashProcess(return_err_output=True)
+    output = session.run(["invalid_command"])
+    assert re.match(
+        r"^/bin/sh:.*invalid_command.*(?:not found|Permission denied).*$", output
+    )
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="Test not supported on Windows"
+)
+def test_create_directory_and_files(tmp_path: Path) -> None:
+    """Test creation of a directory and files in a temporary directory."""
+    session = BashProcess(strip_newlines=True)
+
+    # create a subdirectory in the temporary directory
+    temp_dir = tmp_path / "test_dir"
+    temp_dir.mkdir()
+
+    # run the commands in the temporary directory
+    commands = [
+        f"touch {temp_dir}/file1.txt",
+        f"touch {temp_dir}/file2.txt",
+        f"echo 'hello world' > {temp_dir}/file2.txt",
+        f"cat {temp_dir}/file2.txt",
+    ]
+
+    output = session.run(commands)
+    assert output == "hello world"
+
+    # check that the files were created in the temporary directory
+    output = session.run([f"ls {temp_dir}"])
+    assert output == "file1.txt\nfile2.txt"
+
+
+@pytest.mark.skip(reason="flaky on GHA, TODO to fix")
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="Test not supported on Windows"
+)
+def test_create_bash_persistent() -> None:
+    """Test the pexpect persistent bash terminal"""
+    session = BashProcess(persistent=True)
+    response = session.run("echo hello")
+    response += session.run("echo world")
+
+    assert "hello" in response
+    assert "world" in response

libs/experimental/tests/unit_tests/test_llm_bash.py

@@ -0,0 +1,109 @@
+"""Test LLM Bash functionality."""
+import sys
+
+import pytest
+from langchain.schema import OutputParserException
+
+from langchain_experimental.llm_bash.base import LLMBashChain
+from langchain_experimental.llm_bash.prompt import _PROMPT_TEMPLATE, BashOutputParser
+from tests.unit_tests.fake_llm import FakeLLM
+
+_SAMPLE_CODE = """
+Unrelated text
+```bash
+echo hello
+```
+Unrelated text
+"""
+
+
+_SAMPLE_CODE_2_LINES = """
+Unrelated text
+```bash
+echo hello
+
+echo world
+```
+Unrelated text
+"""
+
+
+@pytest.fixture
+def output_parser() -> BashOutputParser:
+    """Output parser for testing."""
+    return BashOutputParser()
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="Test not supported on Windows"
+)
+def test_simple_question() -> None:
+    """Test simple question that should not need python."""
+    question = "Please write a bash script that prints 'Hello World' to the console."
+    prompt = _PROMPT_TEMPLATE.format(question=question)
+    queries = {prompt: "```bash\nexpr 1 + 1\n```"}
+    fake_llm = FakeLLM(queries=queries)
+    fake_llm_bash_chain = LLMBashChain.from_llm(fake_llm, input_key="q", output_key="a")
+    output = fake_llm_bash_chain.run(question)
+    assert output == "2\n"
+
+
+def test_get_code(output_parser: BashOutputParser) -> None:
+    """Test the parser."""
+    code_lines = output_parser.parse(_SAMPLE_CODE)
+    code = [c for c in code_lines if c.strip()]
+    assert code == code_lines
+    assert code == ["echo hello"]
+
+    code_lines = output_parser.parse(_SAMPLE_CODE + _SAMPLE_CODE_2_LINES)
+    assert code_lines == ["echo hello", "echo hello", "echo world"]
+
+
+def test_parsing_error() -> None:
+    """Test that LLM Output without a bash block raises an exce"""
+    question = "Please echo 'hello world' to the terminal."
+    prompt = _PROMPT_TEMPLATE.format(question=question)
+    queries = {
+        prompt: """
+```text
+echo 'hello world'
+```
+"""
+    }
+    fake_llm = FakeLLM(queries=queries)
+    fake_llm_bash_chain = LLMBashChain.from_llm(fake_llm, input_key="q", output_key="a")
+    with pytest.raises(OutputParserException):
+        fake_llm_bash_chain.run(question)
+
+
+def test_get_code_lines_mixed_blocks(output_parser: BashOutputParser) -> None:
+    text = """
+Unrelated text
+```bash
+echo hello
+ls && pwd && ls
+```
+
+```python
+print("hello")
+```
+
+```bash
+echo goodbye
+```
+"""
+    code_lines = output_parser.parse(text)
+    assert code_lines == ["echo hello", "ls && pwd && ls", "echo goodbye"]
+
+
+def test_get_code_lines_simple_nested_ticks(output_parser: BashOutputParser) -> None:
+    """Test that backticks w/o a newline are ignored."""
+    text = """
+Unrelated text
+```bash
+echo hello
+echo "```bash is in this string```"
+```
+"""
+    code_lines = output_parser.parse(text)
+    assert code_lines == ["echo hello", 'echo "```bash is in this string```"']