infra: cleanup docs build (#21134)

Refactors the docs build in order to:
- run the same `make build` command in both vercel and local build
- incrementally build artifacts in 2 distinct steps, instead of building
all docs in-place (in vercel) or in a _dist dir (locally)

Highlights:
- introduces `make build` in order to build the docs
- collects and generates all files for the build in
`docs/build/intermediate`
- renders those jupyter notebook + markdown files into
`docs/build/outputs`

And now the outputs to host are in `docs/build/outputs`, which will need
a vercel settings change.

Todo:
- [ ] figure out how to point the right directory (right now deleting
and moving docs dir in vercel_build.sh isn't great)

docs/scripts/generate_api_reference_links.py

@@ -25,7 +25,6 @@ _IMPORT_RE = re.compile(
 _CURRENT_PATH = Path(__file__).parent.absolute()
 # Directory where generated markdown files are stored
 _DOCS_DIR = _CURRENT_PATH / "docs"
-_JSON_PATH = _CURRENT_PATH / "api_reference" / "guide_imports.json"
 
 
 def find_files(path):
@@ -55,6 +54,12 @@ def get_args():
         default=_DOCS_DIR,
         help="Directory where generated markdown files are stored",
     )
+    parser.add_argument(
+        "--json_path",
+        type=str,
+        default=None,
+        help="Path to store the generated JSON file",
+    )
     return parser.parse_args()
 
 
@@ -83,9 +88,11 @@ def main():
                 global_imports[class_name][doc_title] = doc_url
 
     # Write the global imports information to a JSON file
-    _JSON_PATH.parent.mkdir(parents=True, exist_ok=True)
-    with _JSON_PATH.open("w") as f:
-        json.dump(global_imports, f)
+    if args.json_path:
+        json_path = Path(args.json_path)
+        json_path.parent.mkdir(parents=True, exist_ok=True)
+        with json_path.open("w") as f:
+            json.dump(global_imports, f)
 
 
 def _get_doc_title(data: str, file_name: str) -> str: