infra: cleanup docs build (#21134)

Refactors the docs build in order to:
- run the same `make build` command in both vercel and local build
- incrementally build artifacts in 2 distinct steps, instead of building
all docs in-place (in vercel) or in a _dist dir (locally)

Highlights:
- introduces `make build` in order to build the docs
- collects and generates all files for the build in
`docs/build/intermediate`
- renders those jupyter notebook + markdown files into
`docs/build/outputs`

And now the outputs to host are in `docs/build/outputs`, which will need
a vercel settings change.

Todo:
- [ ] figure out how to point the right directory (right now deleting
and moving docs dir in vercel_build.sh isn't great)

docs/scripts/copy_templates.py

@@ -2,35 +2,44 @@ import glob
 import os
 import re
 import shutil
+import sys
 from pathlib import Path
 
-TEMPLATES_DIR = Path(os.path.abspath(__file__)).parents[2] / "templates"
-DOCS_TEMPLATES_DIR = Path(os.path.abspath(__file__)).parents[1] / "docs" / "templates"
+if __name__ == "__main__":
+    intermediate_dir = Path(sys.argv[1])
 
+    templates_source_dir = Path(os.path.abspath(__file__)).parents[2] / "templates"
+    templates_intermediate_dir = intermediate_dir / "templates"
 
-readmes = list(glob.glob(str(TEMPLATES_DIR) + "/*/README.md"))
-destinations = [readme[len(str(TEMPLATES_DIR)) + 1 : -10] + ".md" for readme in readmes]
-for source, destination in zip(readmes, destinations):
-    full_destination = DOCS_TEMPLATES_DIR / destination
-    shutil.copyfile(source, full_destination)
-    with open(full_destination, "r") as f:
-        content = f.read()
-    # remove images
-    content = re.sub("\!\[.*?\]\((.*?)\)", "", content)
-    with open(full_destination, "w") as f:
-        f.write(content)
+    readmes = list(glob.glob(str(templates_source_dir) + "/*/README.md"))
+    destinations = [
+        readme[len(str(templates_source_dir)) + 1 : -10] + ".md" for readme in readmes
+    ]
+    for source, destination in zip(readmes, destinations):
+        full_destination = templates_intermediate_dir / destination
+        shutil.copyfile(source, full_destination)
+        with open(full_destination, "r") as f:
+            content = f.read()
+        # remove images
+        content = re.sub("\!\[.*?\]\((.*?)\)", "", content)
+        with open(full_destination, "w") as f:
+            f.write(content)
 
-sidebar_hidden = """---
+    sidebar_hidden = """---
 sidebar_class_name: hidden
 ---
 
 """
-TEMPLATES_INDEX_DESTINATION = DOCS_TEMPLATES_DIR / "index.md"
-with open(TEMPLATES_INDEX_DESTINATION, "r") as f:
-    content = f.read()
 
-# replace relative links
-content = re.sub("\]\(\.\.\/", "](/docs/templates/", content)
+    # handle index file
+    templates_index_source = templates_source_dir / "docs" / "INDEX.md"
+    templates_index_intermediate = templates_intermediate_dir / "index.md"
 
-with open(TEMPLATES_INDEX_DESTINATION, "w") as f:
-    f.write(sidebar_hidden + content)
+    with open(templates_index_source, "r") as f:
+        content = f.read()
+
+    # replace relative links
+    content = re.sub("\]\(\.\.\/", "](/docs/templates/", content)
+
+    with open(templates_index_intermediate, "w") as f:
+        f.write(sidebar_hidden + content)