Compare commits

...

6 Commits

Author SHA1 Message Date
William Fu-Hinthorn
7c61793374 merge 2023-07-25 14:57:15 -07:00
William Fu-Hinthorn
db31cbad4a tmp 2023-07-19 11:39:33 -07:00
William Fu-Hinthorn
4dd86bfd43 Merge branch 'master' into wfh/docslinks 2023-07-19 07:34:21 -07:00
William Fu-Hinthorn
bc0d18fa68 update 2023-07-17 10:37:52 -07:00
William Fu-Hinthorn
a1013d1c15 update 2023-07-17 09:19:16 -07:00
William Fu-Hinthorn
c10d9fcbef Add link generator 2023-07-17 08:45:11 -07:00
20 changed files with 600 additions and 61 deletions

View File

@@ -10,4 +10,6 @@ sphinx-panels
toml
myst_nb
sphinx_copybutton
pydata-sphinx-theme==0.13.1
pydata-sphinx-theme==0.13.1
nbdoc
urllib3<2

View File

@@ -0,0 +1,44 @@
import os
import re
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# Directory where generated markdown files are stored
docs_dir = "docs_skeleton/docs"
# Regular expression to match Python import lines
link_re = re.compile(r"\[(from langchain.+?)\]\(.+?\)")
def find_files(path):
"""Find all MDX files in the given path"""
for root, _, files in os.walk(path):
for file in files:
if file.endswith(".mdx") or file.endswith(".md"):
yield os.path.join(root, file)
def unreplace_imports(file):
"""Replace import links in a markdown file with just the import text"""
with open(file, "r") as f:
data = f.read()
for match in link_re.finditer(data):
# Replace the markdown link with just the link text
data = data.replace(match.group(0), match.group(1))
with open(file, "w") as f:
f.write(data)
def main():
"""Main function"""
for file in find_files(docs_dir):
print(f"Removing links from imports in {file}")
unreplace_imports(file)
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,75 @@
import importlib
import inspect
import logging
import os
import re
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# Base URL for all class documentation
base_url = "https://api.python.langchain.com/en/latest/"
# Directory where generated markdown files are stored
docs_dir = "docs_skeleton/docs"
# Regular expression to match Python import lines
import_re = re.compile(r"(from\s+(langchain\.\w+(\.\w+)*?)\s+import\s+)(\w+)")
def find_files(path):
"""Find all MDX files in the given path"""
for root, _, files in os.walk(path):
for file in files:
if file.endswith(".mdx") or file.endswith(".md"):
yield os.path.join(root, file)
def get_full_module_name(module_path, class_name):
"""Get full module name using inspect"""
module = importlib.import_module(module_path)
class_ = getattr(module, class_name)
return inspect.getmodule(class_).__name__
def replace_imports(file):
"""Replace imports in a markdown file with links to their documentation"""
with open(file, "r") as f:
data = f.read()
for match in import_re.finditer(data):
# Create the URL for this class's documentation
class_name = match.group(4)
try:
module_path = get_full_module_name(match.group(2), class_name)
except AttributeError as e:
logger.error(f"Could not find module for {class_name}", e)
continue
module_path_parts = module_path.replace(".", "/")
url = (
base_url
+ module_path_parts
+ "/"
+ module_path_parts
+ "."
+ class_name
+ ".html"
)
# Replace the class name with a markdown link
link = f"[{match.group(1)}{class_name}]({url})"
data = data.replace(match.group(0), f"{match.group(0)}\n{link}")
with open(file, "w") as f:
f.write(data)
def main():
"""Main function"""
for file in find_files(docs_dir):
print(f"Adding links for imports in {file}")
replace_imports(file)
if __name__ == "__main__":
main()

View File

@@ -5,7 +5,7 @@ tst_flags = notest
user = hwchase17
doc_host = https://python.langchain.com
doc_baseurl = /docs
module_baseurls = metaflow=https://github.com/Netflix/metaflow/tree/master/
module_baseurls = langchain=https://github.com/hwchase17/langchain/tree/master
fastcore=https://github.com/fastcore/tree/master
host = github

View File

@@ -0,0 +1,21 @@
import React from 'react';
import clsx from 'clsx';
import {ThemeClassNames, usePrismTheme} from '@docusaurus/theme-common';
import {getPrismCssVariables} from '@docusaurus/theme-common/internal';
import styles from './styles.module.css';
export default function CodeBlockContainer({as: As, ...props}) {
const prismTheme = usePrismTheme();
const prismCssVariables = getPrismCssVariables(prismTheme);
return (
<As
// Polymorphic components are hard to type, without `oneOf` generics
{...props}
style={prismCssVariables}
className={clsx(
props.className,
styles.codeBlockContainer,
ThemeClassNames.common.codeBlock,
)}
/>
);
}

View File

@@ -0,0 +1,7 @@
.codeBlockContainer {
background: var(--prism-background-color);
color: var(--prism-color);
margin-bottom: var(--ifm-leading);
box-shadow: var(--ifm-global-shadow-lw);
border-radius: var(--ifm-code-border-radius);
}

View File

@@ -0,0 +1,17 @@
import React from 'react';
import clsx from 'clsx';
import Container from '@theme/CodeBlock/Container';
import styles from './styles.module.css';
// <pre> tags in markdown map to CodeBlocks. They may contain JSX children. When
// the children is not a simple string, we just return a styled block without
// actually highlighting.
export default function CodeBlockJSX({children, className}) {
return (
<Container
as="pre"
tabIndex={0}
className={clsx(styles.codeBlockStandalone, 'thin-scrollbar', className)}>
<code className={styles.codeBlockLines}>{children}</code>
</Container>
);
}

View File

@@ -0,0 +1,97 @@
import React from 'react';
import clsx from 'clsx';
import {useThemeConfig, usePrismTheme} from '@docusaurus/theme-common';
import {
parseCodeBlockTitle,
parseLanguage,
parseLines,
containsLineNumbers,
useCodeWordWrap,
} from '@docusaurus/theme-common/internal';
import Highlight, {defaultProps} from 'prism-react-renderer';
import Line from '@theme/CodeBlock/Line';
import CopyButton from '@theme/CodeBlock/CopyButton';
import WordWrapButton from '@theme/CodeBlock/WordWrapButton';
import Container from '@theme/CodeBlock/Container';
import styles from './styles.module.css';
export default function CodeBlockString({
children,
className: blockClassName = '',
metastring,
title: titleProp,
showLineNumbers: showLineNumbersProp,
language: languageProp,
}) {
const {
prism: {defaultLanguage, magicComments},
} = useThemeConfig();
const language =
languageProp ?? parseLanguage(blockClassName) ?? defaultLanguage;
const prismTheme = usePrismTheme();
const wordWrap = useCodeWordWrap();
// We still parse the metastring in case we want to support more syntax in the
// future. Note that MDX doesn't strip quotes when parsing metastring:
// "title=\"xyz\"" => title: "\"xyz\""
const title = parseCodeBlockTitle(metastring) || titleProp;
const {lineClassNames, code} = parseLines(children, {
metastring,
language,
magicComments,
});
const showLineNumbers =
showLineNumbersProp ?? containsLineNumbers(metastring);
return (
<Container
as="div"
className={clsx(
blockClassName,
language &&
!blockClassName.includes(`language-${language}`) &&
`language-${language}`,
)}>
{title && <div className={styles.codeBlockTitle}>{title}</div>}
<div className={styles.codeBlockContent}>
<Highlight
{...defaultProps}
theme={prismTheme}
code={code}
language={language ?? 'text'}>
{({className, tokens, getLineProps, getTokenProps}) => (
<pre
/* eslint-disable-next-line jsx-a11y/no-noninteractive-tabindex */
tabIndex={0}
ref={wordWrap.codeBlockRef}
className={clsx(className, styles.codeBlock, 'thin-scrollbar')}>
<code
className={clsx(
styles.codeBlockLines,
showLineNumbers && styles.codeBlockLinesWithNumbering,
)}>
{tokens.map((line, i) => (
<Line
key={i}
line={line}
getLineProps={getLineProps}
getTokenProps={getTokenProps}
classNames={lineClassNames[i]}
showLineNumbers={showLineNumbers}
/>
))}
</code>
</pre>
)}
</Highlight>
<div className={styles.buttonGroup}>
{(wordWrap.isEnabled || wordWrap.isCodeScrollable) && (
<WordWrapButton
className={styles.codeButton}
onClick={() => wordWrap.toggle()}
isEnabled={wordWrap.isEnabled}
/>
)}
<CopyButton className={styles.codeButton} code={code} />
</div>
</div>
</Container>
);
}

View File

@@ -0,0 +1,80 @@
.codeBlockContent {
position: relative;
/* rtl:ignore */
direction: ltr;
border-radius: inherit;
}
.codeBlockTitle {
border-bottom: 1px solid var(--ifm-color-emphasis-300);
font-size: var(--ifm-code-font-size);
font-weight: 500;
padding: 0.75rem var(--ifm-pre-padding);
border-top-left-radius: inherit;
border-top-right-radius: inherit;
}
.codeBlock {
--ifm-pre-background: var(--prism-background-color);
margin: 0;
padding: 0;
}
.codeBlockTitle + .codeBlockContent .codeBlock {
border-top-left-radius: 0;
border-top-right-radius: 0;
}
.codeBlockStandalone {
padding: 0;
}
.codeBlockLines {
font: inherit;
/* rtl:ignore */
float: left;
min-width: 100%;
padding: var(--ifm-pre-padding);
}
.codeBlockLinesWithNumbering {
display: table;
padding: var(--ifm-pre-padding) 0;
}
@media print {
.codeBlockLines {
white-space: pre-wrap;
}
}
.buttonGroup {
display: flex;
column-gap: 0.2rem;
position: absolute;
/* rtl:ignore */
right: calc(var(--ifm-pre-padding) / 2);
top: calc(var(--ifm-pre-padding) / 2);
}
.buttonGroup button {
display: flex;
align-items: center;
background: var(--prism-background-color);
color: var(--prism-color);
border: 1px solid var(--ifm-color-emphasis-300);
border-radius: var(--ifm-global-radius);
padding: 0.4rem;
line-height: 0;
transition: opacity var(--ifm-transition-fast) ease-in-out;
opacity: 0;
}
.buttonGroup button:focus-visible,
.buttonGroup button:hover {
opacity: 1 !important;
}
:global(.theme-code-block:hover) .buttonGroup button {
opacity: 0.4;
}

View File

@@ -0,0 +1,56 @@
import React, {useCallback, useState, useRef, useEffect} from 'react';
import clsx from 'clsx';
// @ts-expect-error: TODO, we need to make theme-classic have type: module
import copy from 'copy-text-to-clipboard';
import {translate} from '@docusaurus/Translate';
import styles from './styles.module.css';
export default function CopyButton({code, className}) {
const [isCopied, setIsCopied] = useState(false);
const copyTimeout = useRef(undefined);
const handleCopyCode = useCallback(() => {
copy(code);
setIsCopied(true);
copyTimeout.current = window.setTimeout(() => {
setIsCopied(false);
}, 1000);
}, [code]);
useEffect(() => () => window.clearTimeout(copyTimeout.current), []);
return (
<button
type="button"
aria-label={
isCopied
? translate({
id: 'theme.CodeBlock.copied',
message: 'Copied',
description: 'The copied button label on code blocks',
})
: translate({
id: 'theme.CodeBlock.copyButtonAriaLabel',
message: 'Copy code to clipboard',
description: 'The ARIA label for copy code blocks button',
})
}
title={translate({
id: 'theme.CodeBlock.copy',
message: 'Copy',
description: 'The copy button label on code blocks',
})}
className={clsx(
'clean-btn',
className,
styles.copyButton,
isCopied && styles.copyButtonCopied,
)}
onClick={handleCopyCode}>
<span className={styles.copyButtonIcons} aria-hidden="true">
<svg className={styles.copyButtonIcon} viewBox="0 0 24 24">
<path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z" />
</svg>
<svg className={styles.copyButtonSuccessIcon} viewBox="0 0 24 24">
<path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z" />
</svg>
</span>
</button>
);
}

View File

@@ -0,0 +1,40 @@
:global(.theme-code-block:hover) .copyButtonCopied {
opacity: 1 !important;
}
.copyButtonIcons {
position: relative;
width: 1.125rem;
height: 1.125rem;
}
.copyButtonIcon,
.copyButtonSuccessIcon {
position: absolute;
top: 0;
left: 0;
fill: currentColor;
opacity: inherit;
width: inherit;
height: inherit;
transition: all var(--ifm-transition-fast) ease;
}
.copyButtonSuccessIcon {
top: 50%;
left: 50%;
transform: translate(-50%, -50%) scale(0.33);
opacity: 0;
color: #00d600;
}
.copyButtonCopied .copyButtonIcon {
transform: scale(0.33);
opacity: 0;
}
.copyButtonCopied .copyButtonSuccessIcon {
transform: translate(-50%, -50%) scale(1);
opacity: 1;
transition-delay: 0.075s;
}

View File

@@ -0,0 +1,34 @@
import React from 'react';
import clsx from 'clsx';
import styles from './styles.module.css';
export default function CodeBlockLine({
line,
classNames,
showLineNumbers,
getLineProps,
getTokenProps,
}) {
if (line.length === 1 && line[0].content === '\n') {
line[0].content = '';
}
const lineProps = getLineProps({
line,
className: clsx(classNames, showLineNumbers && styles.codeLine),
});
const lineTokens = line.map((token, key) => (
<span key={key} {...getTokenProps({token, key})} />
));
return (
<span {...lineProps}>
{showLineNumbers ? (
<>
<span className={styles.codeLineNumber} />
<span className={styles.codeLineContent}>{lineTokens}</span>
</>
) : (
lineTokens
)}
<br />
</span>
);
}

View File

@@ -0,0 +1,45 @@
/* Intentionally has zero specificity, so that to be able to override
the background in custom CSS file due bug https://github.com/facebook/docusaurus/issues/3678 */
:where(:root) {
--docusaurus-highlighted-code-line-bg: rgb(72 77 91);
}
:where([data-theme='dark']) {
--docusaurus-highlighted-code-line-bg: rgb(100 100 100);
}
:global(.theme-code-block-highlighted-line) {
background-color: var(--docusaurus-highlighted-code-line-bg);
display: block;
margin: 0 calc(-1 * var(--ifm-pre-padding));
padding: 0 var(--ifm-pre-padding);
}
.codeLine {
display: table-row;
counter-increment: line-count;
}
.codeLineNumber {
display: table-cell;
text-align: right;
width: 1%;
position: sticky;
left: 0;
padding: 0 var(--ifm-pre-padding);
background: var(--ifm-pre-background);
overflow-wrap: normal;
}
.codeLineNumber::before {
content: counter(line-count);
opacity: 0.4;
}
:global(.theme-code-block-highlighted-line) .codeLineNumber::before {
opacity: 0.8;
}
.codeLineContent {
padding-right: var(--ifm-pre-padding);
}

View File

@@ -0,0 +1,34 @@
import React from 'react';
import clsx from 'clsx';
import {translate} from '@docusaurus/Translate';
import styles from './styles.module.css';
export default function WordWrapButton({className, onClick, isEnabled}) {
const title = translate({
id: 'theme.CodeBlock.wordWrapToggle',
message: 'Toggle word wrap',
description:
'The title attribute for toggle word wrapping button of code block lines',
});
return (
<button
type="button"
onClick={onClick}
className={clsx(
'clean-btn',
className,
isEnabled && styles.wordWrapButtonEnabled,
)}
aria-label={title}
title={title}>
<svg
className={styles.wordWrapButtonIcon}
viewBox="0 0 24 24"
aria-hidden="true">
<path
fill="currentColor"
d="M4 19h6v-2H4v2zM20 5H4v2h16V5zm-3 6H4v2h13.25c1.1 0 2 .9 2 2s-.9 2-2 2H15v-2l-3 3l3 3v-2h2c2.21 0 4-1.79 4-4s-1.79-4-4-4z"
/>
</svg>
</button>
);
}

View File

@@ -0,0 +1,8 @@
.wordWrapButtonIcon {
width: 1.2rem;
height: 1.2rem;
}
.wordWrapButtonEnabled .wordWrapButtonIcon {
color: var(--ifm-color-primary);
}

View File

@@ -1,47 +1,32 @@
/* eslint-disable react/jsx-props-no-spreading */
import React from "react";
import CodeBlock from "@theme-original/CodeBlock";
function Imports({ imports }) {
return (
<div
style={{
paddingTop: "1.3rem",
background: "var(--prism-background-color)",
color: "var(--prism-color)",
marginTop: "calc(-1 * var(--ifm-leading) - 5px)",
marginBottom: "var(--ifm-leading)",
boxShadow: "var(--ifm-global-shadow-lw)",
borderBottomLeftRadius: "var(--ifm-code-border-radius)",
borderBottomRightRadius: "var(--ifm-code-border-radius)",
}}
>
<h4 style={{ paddingLeft: "0.65rem", marginBottom: "0.45rem" }}>
API Reference:
</h4>
<ul style={{ paddingBottom: "1rem" }}>
{imports.map(({ imported, source, docs }) => (
<li>
<a href={docs}>
<span>{imported}</span>
</a>{" "}
from <code>{source}</code>
</li>
))}
</ul>
</div>
);
}
export default function CodeBlockWrapper({ children, ...props }) {
if (typeof children === "string") {
return <CodeBlock {...props}>{children}</CodeBlock>;
import React, {isValidElement} from 'react';
import useIsBrowser from '@docusaurus/useIsBrowser';
import ElementContent from '@theme/CodeBlock/Content/Element';
import StringContent from '@theme/CodeBlock/Content/String';
/**
* Best attempt to make the children a plain string so it is copyable. If there
* are react elements, we will not be able to copy the content, and it will
* return `children` as-is; otherwise, it concatenates the string children
* together.
*/
function maybeStringifyChildren(children) {
if (React.Children.toArray(children).some((el) => isValidElement(el))) {
return children;
}
// The children is now guaranteed to be one/more plain strings
return Array.isArray(children) ? children.join('') : children;
}
export default function CodeBlock({children: rawChildren, ...props}) {
// The Prism theme on SSR is always the default theme but the site theme can
// be in a different mode. React hydration doesn't update DOM styles that come
// from SSR. Hence force a re-render after mounting to apply the current
// relevant styles.
const isBrowser = useIsBrowser();
const children = maybeStringifyChildren(rawChildren);
const CodeBlockComp =
typeof children === 'string' ? StringContent : ElementContent;
return (
<>
<CodeBlock {...props}>{children.content}</CodeBlock>
<Imports imports={children.imports} />
</>
<CodeBlockComp key={String(isBrowser)} {...props}>
{children}
</CodeBlockComp>
);
}

View File

@@ -6,5 +6,6 @@ python3 -m venv .venv
source .venv/bin/activate
python3 -m pip install -r vercel_requirements.txt
cp -r extras/* docs_skeleton/docs
python3 docs_skeleton/link_generator.py
cd docs_skeleton
nbdoc_build

View File

@@ -15,7 +15,7 @@ pip install rockset
See a [usage example](/docs/integrations/vectorstores/rockset).
```python
from langchain.vectorstores import RocksetDB
from langchain.vectorstores import Rockset
```
## Document Loader

View File

@@ -124,7 +124,7 @@
"from langchain.embeddings.openai import OpenAIEmbeddings\n",
"from langchain.text_splitter import CharacterTextSplitter\n",
"from langchain.document_loaders import TextLoader\n",
"from langchain.vectorstores.rocksetdb import RocksetDB\n",
"from langchain.vectorstores.rocksetdb import Rockset\n",
"\n",
"loader = TextLoader(\"../../../state_of_the_union.txt\")\n",
"documents = loader.load()\n",
@@ -150,7 +150,7 @@
"# Make sure the environment variable OPENAI_API_KEY is set up\n",
"embeddings = OpenAIEmbeddings()\n",
"\n",
"docsearch = RocksetDB(\n",
"docsearch = Rockset(\n",
" client=rockset_client,\n",
" embeddings=embeddings,\n",
" collection_name=COLLECTION_NAME,\n",
@@ -185,7 +185,7 @@
"source": [
"query = \"What did the president say about Ketanji Brown Jackson\"\n",
"output = docsearch.similarity_search_with_relevance_scores(\n",
" query, 4, RocksetDB.DistanceFunction.COSINE_SIM\n",
" query, 4, Rockset.DistanceFunction.COSINE_SIM\n",
")\n",
"print(\"output length:\", len(output))\n",
"for d, dist in output:\n",
@@ -221,7 +221,7 @@
"output = docsearch.similarity_search_with_relevance_scores(\n",
" query,\n",
" 4,\n",
" RocksetDB.DistanceFunction.COSINE_SIM,\n",
" Rockset.DistanceFunction.COSINE_SIM,\n",
" where_str=\"{} NOT LIKE '%citizens%'\".format(TEXT_KEY),\n",
")\n",
"print(\"output length:\", len(output))\n",
@@ -243,9 +243,9 @@
"source": [
"### 3. [Optional] Drop all inserted documents\n",
"\n",
"In order to delete texts from the Rockset collection, you need to know the unique ID associated with each document inside Rockset. These ids can either be supplied directly by the user while inserting the texts (in the `RocksetDB.add_texts()` function), else Rockset will generate a unique ID or each document. Either way, `Rockset.add_texts()` returns the ids for the inserted documents.\n",
"In order to delete texts from the Rockset collection, you need to know the unique ID associated with each document inside Rockset. These ids can either be supplied directly by the user while inserting the texts (in the `Rockset.add_texts()` function), else Rockset will generate a unique ID or each document. Either way, `Rockset.add_texts()` returns the ids for the inserted documents.\n",
"\n",
"To delete these docs, simply use the `RocksetDB.delete_texts()` function."
"To delete these docs, simply use the `Rockset.delete_texts()` function."
]
},
{

View File

@@ -15,10 +15,7 @@
"execution_count": 11,
"id": "c19c736e-ca74-4726-bb77-0a849bcc2960",
"metadata": {
"tags": [],
"vscode": {
"languageId": "python"
}
"tags": []
},
"outputs": [],
"source": [
@@ -130,11 +127,7 @@
"cell_type": "code",
"execution_count": 12,
"id": "18361f89",
"metadata": {
"vscode": {
"languageId": "python"
}
},
"metadata": {},
"outputs": [
{
"name": "stdout",