From 42ff45b865b1744e3f37a821bbcbc3d2ece1513c Mon Sep 17 00:00:00 2001 From: Scott Linder Date: Wed, 24 Jun 2026 15:12:48 -0400 Subject: [PATCH] [docs] Create utils/docs (#203962) llvm-project is home to many sphinx documentation sites, each with configuration quirks and bespoke extensions. The sphinx config model makes sharing code somewhat difficult. There are options like sphinx-multiproject, but some of our docs builds are out of the source tree while some are done out of the binary tree, so the multiproject configuration itself would need to be generated. It also would impose more uniformity around extensions than required. This change instead creates a python package at utils/docs/llvm_sphinx and makes it available to all sphinx-build processes via PYTHONPATH. Each conf.py does not modify its own sys.path because not all builds are out of the source tree, so there isn't a stable relative path to use to refer to the utils/docs/ directory. Type checking via pyright in new package is pinned to being python 3.8 compatible. The myst slug script from llvm/docs is promoted to be the default, with a slight edit to targets needed in FlangDriver.md to adapt to the change. The ghlinks extension from clang gets lifted to be the first generic extension, although it is still only used in clang currently. I used it as an example of a general extension that might be useful for other projects, but I also intend to add a new generic extension in a future patch. As part of the move ghlinks is refactored a bit: * Rather than have a main function directly to run its tests, those tests are now run via the command-line entrypoint to the new package, e.g. `python utils/docs --test` will now replace `python clang/docs/ghlinks.py --test`. This makes the command-line more pithy (`python utils/docs/llvm_sphinx/ext/ghlinks --test` is a bit long), and makes the default behavior when running be to include utils/docs in the sys.path, so imports can be absolute and the behavior is as close to the CMake targets as possible. * The static files that were previously written to a tempdir during testing are just committed directly in git. The assumption here is that the original version was taking pains to avoid polluting the clang/docs directory with test files that would confuse sphinx-build, and without that limitation the workaround can be removed. * Some typing-related changes to make it strictly python 3.8 compatible. Change-Id: Ie16de273f74973f163dbf7fe9d22d80a466d051f --- bolt/docs/conf.py | 24 +--- clang-tools-extra/docs/conf.py | 35 +---- clang/docs/conf.py | 39 ++--- flang/docs/FlangDriver.md | 4 +- flang/docs/conf.py | 37 +---- libc/docs/conf.py | 25 +--- libcxx/docs/conf.py | 24 +--- libunwind/docs/conf.py | 24 +--- lld/docs/conf.py | 24 +--- lldb/docs/_ext/lldb_setting.py | 4 +- lldb/docs/conf.py | 45 +----- llvm/cmake/modules/AddSphinxTarget.cmake | 1 + llvm/docs/conf.py | 56 ++------ llvm/docs/llvm_slug.py | 16 --- openmp/docs/conf.py | 35 +---- orc-rt/docs/conf.py | 31 +--- polly/docs/conf.py | 24 +--- pyproject.toml | 3 +- utils/docs/README.md | 20 +++ utils/docs/__main__.py | 31 ++++ utils/docs/llvm_sphinx/__init__.py | 71 +++++++++ utils/docs/llvm_sphinx/ext/__init__.py | 0 .../docs/llvm_sphinx/ext/ghlinks/__init__.py | 136 +----------------- .../docs/llvm_sphinx/ext/ghlinks/test/conf.py | 7 + .../llvm_sphinx/ext/ghlinks/test/index.rst | 8 ++ .../llvm_sphinx/ext/ghlinks/test/markdown.md | 14 ++ .../llvm_sphinx/ext/ghlinks/test/rest.rst | 19 +++ utils/docs/llvm_sphinx/help.py | 27 ++++ 28 files changed, 279 insertions(+), 505 deletions(-) delete mode 100644 llvm/docs/llvm_slug.py create mode 100644 utils/docs/README.md create mode 100644 utils/docs/__main__.py create mode 100644 utils/docs/llvm_sphinx/__init__.py create mode 100644 utils/docs/llvm_sphinx/ext/__init__.py rename clang/docs/ghlinks.py => utils/docs/llvm_sphinx/ext/ghlinks/__init__.py (60%) create mode 100644 utils/docs/llvm_sphinx/ext/ghlinks/test/conf.py create mode 100644 utils/docs/llvm_sphinx/ext/ghlinks/test/index.rst create mode 100644 utils/docs/llvm_sphinx/ext/ghlinks/test/markdown.md create mode 100644 utils/docs/llvm_sphinx/ext/ghlinks/test/rest.rst create mode 100644 utils/docs/llvm_sphinx/help.py diff --git a/bolt/docs/conf.py b/bolt/docs/conf.py index ededa1ccbb0b..14b6c1c8eff3 100644 --- a/bolt/docs/conf.py +++ b/bolt/docs/conf.py @@ -13,31 +13,15 @@ import sys, os from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.NEVER)) # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.intersphinx", "sphinx.ext.todo"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" +extensions += ["sphinx.ext.intersphinx", "sphinx.ext.todo"] # General information about the project. project = "BOLT" diff --git a/clang-tools-extra/docs/conf.py b/clang-tools-extra/docs/conf.py index 8247cd173fcf..a80bbcf8638b 100644 --- a/clang-tools-extra/docs/conf.py +++ b/clang-tools-extra/docs/conf.py @@ -14,42 +14,15 @@ import sys, os from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.EXCEPT_MAN)) # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.todo", "sphinx.ext.mathjax"] - -# When building man pages, we do not use the markdown pages, -# So, we can continue without the myst_parser dependencies. -# Doing so reduces dependencies of some packaged llvm distributions. -try: - import myst_parser - - extensions.append("myst_parser") -except ImportError: - if not tags.has("builder-man"): - raise - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = [".rst", ".md"] - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" +extensions += ["sphinx.ext.todo", "sphinx.ext.mathjax"] # General information about the project. project = "Extra Clang Tools" diff --git a/clang/docs/conf.py b/clang/docs/conf.py index 23e059baeb86..cac0686d7f42 100644 --- a/clang/docs/conf.py +++ b/clang/docs/conf.py @@ -15,44 +15,23 @@ from __future__ import absolute_import, division, print_function import sys, os from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath(".")) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.EXCEPT_MAN)) # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.todo", "sphinx.ext.mathjax", "sphinx.ext.graphviz", "ghlinks"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - +extensions += [ + "sphinx.ext.todo", + "sphinx.ext.mathjax", + "sphinx.ext.graphviz", + "llvm_sphinx.ext.ghlinks", +] import sphinx -# When building man pages, we do not use the markdown pages, -# So, we can continue without the myst_parser dependencies. -# Doing so reduces dependencies of some packaged llvm distributions. -try: - import myst_parser - - extensions.append("myst_parser") -except ImportError: - if not tags.has("builder-man"): - raise - - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - # General information about the project. project = "Clang" copyright = "2007-%d, The Clang Team" % date.today().year diff --git a/flang/docs/FlangDriver.md b/flang/docs/FlangDriver.md index 2d1140510afc..4edc99944ad4 100644 --- a/flang/docs/FlangDriver.md +++ b/flang/docs/FlangDriver.md @@ -463,9 +463,9 @@ static FrontendPluginRegistry::Add X( ### Loading and Running a Plugin In order to use plugins, there are 2 command line options made available to the frontend driver, `flang -fc1`: -* [`-load `](#the--load-dsopath-option) for loading the dynamic shared +* [`-load `](#the-load-dsopath-option) for loading the dynamic shared object of the plugin -* [`-plugin `](#the--plugin-name-option) for calling the registered plugin +* [`-plugin `](#the-plugin-name-option) for calling the registered plugin Invocation of the example plugin is done through: ```bash diff --git a/flang/docs/conf.py b/flang/docs/conf.py index b8a30aff6bf9..1ad0c19ecb3f 100644 --- a/flang/docs/conf.py +++ b/flang/docs/conf.py @@ -11,10 +11,9 @@ from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags)) # -- General configuration ----------------------------------------------------- @@ -22,47 +21,19 @@ from datetime import date # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ +extensions += [ "sphinx.ext.todo", "sphinx.ext.mathjax", "sphinx.ext.intersphinx", "sphinx.ext.autodoc", ] -try: - import myst_parser - - extensions.append("myst_parser") -except ImportError: - raise ImportError( - "myst_parser is required to build documentation, including man pages." - ) - - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] -source_suffix = { - ".rst": "restructuredtext", - ".md": "markdown", -} -myst_heading_anchors = 6 - -# Enable myst's substitution extension since markdown files cannot use the -# |version| and |release| substitutions available to .rst files. -myst_enable_extensions = ["substitution"] - # The substitutions to use in markdown files. This contains unconditional # substitutions, but more may be added once the configuration is obtained. myst_substitutions = {"in_progress": "(In-Progress) " if tags.has("PreRelease") else ""} import sphinx -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - # General information about the project. project = "Flang" copyright = "2017-%d, The Flang Team" % date.today().year diff --git a/libc/docs/conf.py b/libc/docs/conf.py index 4608a0282c24..1490e7bde741 100644 --- a/libc/docs/conf.py +++ b/libc/docs/conf.py @@ -13,37 +13,20 @@ import sys, os from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.EXCEPT_MAN)) # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ +extensions += [ "sphinx.ext.intersphinx", "sphinx.ext.todo", "sphinx_reredirects", - "myst_parser", ] -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = [".rst", ".md"] - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - # General information about the project. project = "libc" copyright = "2011-%d, LLVM Project" % date.today().year diff --git a/libcxx/docs/conf.py b/libcxx/docs/conf.py index d635bac57e27..9051bce7cf35 100644 --- a/libcxx/docs/conf.py +++ b/libcxx/docs/conf.py @@ -13,31 +13,15 @@ import sys, os from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.NEVER)) # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.intersphinx", "sphinx.ext.todo"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" +extensions += ["sphinx.ext.intersphinx", "sphinx.ext.todo"] # General information about the project. project = "libc++" diff --git a/libunwind/docs/conf.py b/libunwind/docs/conf.py index 29f9c24a7ee2..2d91df6753d6 100644 --- a/libunwind/docs/conf.py +++ b/libunwind/docs/conf.py @@ -13,31 +13,15 @@ import sys, os from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.NEVER)) # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.intersphinx", "sphinx.ext.todo"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" +extensions += ["sphinx.ext.intersphinx", "sphinx.ext.todo"] # General information about the project. project = "libunwind" diff --git a/lld/docs/conf.py b/lld/docs/conf.py index 6f411ed3863d..962424a58954 100644 --- a/lld/docs/conf.py +++ b/lld/docs/conf.py @@ -13,31 +13,15 @@ import sys, os from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.EXCEPT_MAN)) # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.intersphinx", "sphinx.ext.todo"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" +extensions += ["sphinx.ext.intersphinx", "sphinx.ext.todo"] # General information about the project. project = "lld" diff --git a/lldb/docs/_ext/lldb_setting.py b/lldb/docs/_ext/lldb_setting.py index 1d6171ab482e..82b04505eb1f 100644 --- a/lldb/docs/_ext/lldb_setting.py +++ b/lldb/docs/_ext/lldb_setting.py @@ -5,7 +5,7 @@ from sphinx import addnodes from sphinx.application import Sphinx from sphinx.directives import ObjectDescription from sphinx.util.docfields import Field, GroupedField -import llvm_slug +from llvm_sphinx import make_slug # Example: @@ -37,7 +37,7 @@ class LLDBSetting(ObjectDescription): typ, classes=[ "lldb-setting-type", - f"lldb-setting-type-{llvm_slug.make_slug(typ)}", + f"lldb-setting-type-{make_slug(typ)}", ], ) signode["ids"].append(sig) diff --git a/lldb/docs/conf.py b/lldb/docs/conf.py index a0ce2807d231..e8d71ce4c4e6 100644 --- a/lldb/docs/conf.py +++ b/lldb/docs/conf.py @@ -14,8 +14,9 @@ import sys, os, re, shutil from datetime import date from pathlib import Path -# Add path for llvm_slug module. -sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "llvm", "docs"))) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.EXCEPT_MAN)) building_man_page = tags.has("builder-man") @@ -39,14 +40,12 @@ automodapi_toctreedirnm = "python_api" # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - +# lldb specific sphinx extensions sys.path.append(str(Path("_ext").resolve())) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ +extensions += [ "sphinx.ext.todo", # Support for todo items. "sphinx.ext.mathjax", # Render math via JavaScript. "sphinx.ext.intersphinx", # Link to other projects’ documentation. @@ -57,29 +56,10 @@ if "LLDB_BUILD_DIR" in os.environ: # Include a file from $LLDB_BUILD_DIR (see _ext/build_include.py) extensions.append("build_include") -# When building man pages, we do not use the markdown pages, -# So, we can continue without the myst_parser dependencies. -# Doing so reduces dependencies of some packaged llvm distributions. -try: - import myst_parser - - extensions.append("myst_parser") -except ImportError: - if not tags.has("builder-man"): - raise - -# Automatic anchors for markdown titles -myst_heading_anchors = 6 -myst_heading_slug_func = "llvm_slug.make_slug" -myst_enable_extensions = ["fieldlist", "colon_fence", "deflist"] +myst_enable_extensions += ["fieldlist", "colon_fence", "deflist"] autodoc_default_options = {"special-members": True} -# The suffix of source filenames. -source_suffix = { - ".rst": "restructuredtext", -} - # Unless we only generate the basic manpage we need the plugin for generating # the Python API documentation. if not building_man_page: @@ -99,19 +79,6 @@ if not building_man_page: # a list of builtin themes. html_theme = "furo" - # Since man pages do not use markdown, we do not need to register a markdown - # parser. - source_suffix[".md"] = "markdown" - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - # General information about the project. project = "LLDB" copyright = "2007-%d, The LLDB Team" % date.today().year diff --git a/llvm/cmake/modules/AddSphinxTarget.cmake b/llvm/cmake/modules/AddSphinxTarget.cmake index 9de169d7297c..379e36db48a2 100644 --- a/llvm/cmake/modules/AddSphinxTarget.cmake +++ b/llvm/cmake/modules/AddSphinxTarget.cmake @@ -46,6 +46,7 @@ function (add_sphinx_target builder project) add_custom_target(${SPHINX_TARGET_NAME} COMMAND ${CMAKE_COMMAND} -E env ${ARG_ENV_VARS} + --modify "PYTHONPATH=path_list_append:${LLVM_MAIN_SRC_DIR}/../utils/docs" ${SPHINX_EXECUTABLE} -b ${builder} -d "${SPHINX_DOC_TREE_DIR}" diff --git a/llvm/docs/conf.py b/llvm/docs/conf.py index af9781b66c69..2727f2248e80 100644 --- a/llvm/docs/conf.py +++ b/llvm/docs/conf.py @@ -14,59 +14,27 @@ from __future__ import print_function import sys, os, re from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath(".")) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.EXCEPT_MAN)) # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.intersphinx", "sphinx.ext.todo"] +extensions += ["sphinx.ext.intersphinx", "sphinx.ext.todo"] -# When building man pages, we do not use the markdown pages, -# So, we can continue without the myst_parser dependencies. -# Doing so reduces dependencies of some packaged llvm distributions. -try: - import myst_parser - - extensions.append("myst_parser") -except ImportError: - if not tags.has("builder-man"): - raise -else: - myst_enable_extensions = ["deflist", "substitution"] - myst_url_schemes = { - "http": None, - "https": None, - "mailto": None, - "ftp": None, - "doxygen": {"url": "/doxygen/{{path}}"}, - } - -# Automatic anchors for markdown titles -myst_heading_anchors = 6 -myst_heading_slug_func = "llvm_slug.make_slug" - - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -source_suffix = [".rst", ".md"] +myst_enable_extensions += ["deflist"] +myst_url_schemes = { + "http": None, + "https": None, + "mailto": None, + "ftp": None, + "doxygen": {"url": "/doxygen/{{path}}"}, +} import sphinx -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. project = "LLVM" copyright = "2003-%d, LLVM Project" % date.today().year diff --git a/llvm/docs/llvm_slug.py b/llvm/docs/llvm_slug.py deleted file mode 100644 index 9dd8b1cd3f1a..000000000000 --- a/llvm/docs/llvm_slug.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# -# LLVM documentation anchor slug formatting - -# Some of our markdown documentation numbers section titles -# This helpers is used by myst to remove that numbering from the anchor links. - -from docutils.nodes import make_id - - -def make_slug(str): - import re - - str = re.sub(r"^\s*(\w\.)+\w\s", "", str) - str = re.sub(r"^\s*\w\.\s", "", str) - return make_id(str) diff --git a/openmp/docs/conf.py b/openmp/docs/conf.py index d7002ee03314..ec5cbc3e4f2b 100644 --- a/openmp/docs/conf.py +++ b/openmp/docs/conf.py @@ -14,46 +14,21 @@ import sys, os from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags)) # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ +extensions += [ "sphinx.ext.todo", "sphinx.ext.mathjax", "sphinx.ext.intersphinx", - "myst_parser", ] -# Add path for llvm_slug module. -sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "llvm", "docs"))) - - -myst_enable_extensions = ["substitution", "colon_fence", "deflist"] - -# Automatic anchors for markdown titles -myst_heading_anchors = 6 -myst_heading_slug_func = "llvm_slug.make_slug" - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = [".rst", ".md"] - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" +myst_enable_extensions += ["colon_fence", "deflist"] # General information about the project. project = "LLVM/OpenMP" diff --git a/orc-rt/docs/conf.py b/orc-rt/docs/conf.py index b9f197d85b84..8d307024133a 100644 --- a/orc-rt/docs/conf.py +++ b/orc-rt/docs/conf.py @@ -11,10 +11,9 @@ from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.EXCEPT_MAN)) # -- General configuration ----------------------------------------------------- @@ -22,37 +21,15 @@ from datetime import date # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ +extensions += [ "sphinx.ext.todo", "sphinx.ext.mathjax", "sphinx.ext.intersphinx", "sphinx.ext.autodoc", ] -# When building man pages, we do not use the markdown pages, -# So, we can continue without the myst_parser dependencies. -# Doing so reduces dependencies of some packaged llvm distributions. -try: - import myst_parser - - extensions.append("myst_parser") -except ImportError: - if not tags.has("builder-man"): - raise - - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] -myst_heading_anchors = 6 - import sphinx -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - # General information about the project. project = "ORC-RT" copyright = "2025-%d, The ORC-RT Team" % date.today().year diff --git a/polly/docs/conf.py b/polly/docs/conf.py index 908f4d6fa6e1..7d3ece63534a 100644 --- a/polly/docs/conf.py +++ b/polly/docs/conf.py @@ -14,31 +14,15 @@ import sys, os from datetime import date -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +from llvm_sphinx import * # see llvm-project/utils/docs/README.md + +globals().update(common_conf(tags, markdown=Markdown.NEVER)) # -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.todo", "sphinx.ext.mathjax"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" +extensions += ["sphinx.ext.todo", "sphinx.ext.mathjax"] # General information about the project. project = "Polly" diff --git a/pyproject.toml b/pyproject.toml index c84c7ffe2dc1..2992b81d026e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,8 @@ extend-exclude = ''' [tool.pyright] executionEnvironments = [ - { root = "lldb/packages/Python", pythonVersion = "3.8" } + { root = "lldb/packages/Python", pythonVersion = "3.8" }, + { root = "utils/docs", pythonVersion = "3.8" }, ] [dependency-groups] diff --git a/utils/docs/README.md b/utils/docs/README.md new file mode 100644 index 000000000000..e14b2d89d8a9 --- /dev/null +++ b/utils/docs/README.md @@ -0,0 +1,20 @@ +# utils/docs + +Common configuration and extensions for all of the sphinx projects in the +llvm-project monorepo (e.g. `llvm/docs/`, `clang/docs/`, `bolt/docs`, ...). + +This directory is injected into the `PYTHONPATH` of `sphinx-build` processes as +part of the llvm-project CMake build. + +**Note:** On the command-line, you must manually ensure this directory is part +of your `PYTHONPATH`. For example, to use `sphinx-autobuild` for projects which +support it, you might use: + +``` +$ PYTHONPATH=$PWD/utils/docs sphinx-autobuild llvm/docs/ /tmp/sphinx-build +``` + +## Testing your Environment + +To smoke-test your sphinx environment outside of a CMake build, you can use the +`utils/docs/__main__.py` entrypoint (e.g. `python3 utils/docs --test`). diff --git a/utils/docs/__main__.py b/utils/docs/__main__.py new file mode 100644 index 000000000000..9fdcbcda35bb --- /dev/null +++ b/utils/docs/__main__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +"""Command-line entrypoint to utils/docs + +Use this as e.g. `python utils/docs --test` to run docs smoke tests. +""" + +import sys +import argparse +from llvm_sphinx.ext import ghlinks +from typing import List + + +def main(argv: List[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--test", action="store_true", help="run sphinx self-tests") + args = parser.parse_args(argv) + + if args.test: + ghlinks.run_tests() + print( + "ghlinks.py: tests passed; next, rebuild docs-clang-html and spot check the release notes" + ) + return 0 + + parser.print_help(sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/utils/docs/llvm_sphinx/__init__.py b/utils/docs/llvm_sphinx/__init__.py new file mode 100644 index 000000000000..3b63964e5d3c --- /dev/null +++ b/utils/docs/llvm_sphinx/__init__.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +"""Shared configuration and extensions for llvm-project sphinx documentation + +Every sphinx `conf.py` in llvm-project is intended to use `common_conf` as a +baseline configuration. The sphinx module-as-conf model means the easiest +way to achieve this is by updating `globals()` directly, as in: + + from llvm_sphinx import * # see llvm-project/utils/docs/README.md + + globals().update(common_conf(tags)) + +Note: common settings like `extensions` should not be reassigned after the +call to `common_conf`, they should be modified/appended to, as in: + + extensions += ["foo"] + +""" + +import sys +from typing import Any, Dict, TYPE_CHECKING +from enum import Enum, auto +from sphinx.util.tags import Tags +from llvm_sphinx.help import venv_help + + +class Markdown(Enum): + ALWAYS = auto() + EXCEPT_MAN = auto() + NEVER = auto() + + +def common_conf(tags: Tags, markdown=Markdown.ALWAYS) -> Dict[str, Any]: + # If your documentation needs a minimal Sphinx version, state it here. + # needs_sphinx = '1.0' + # The encoding of source files. + # source_encoding = 'utf-8-sig' + extensions = [] + source_suffix = {".rst": "restructuredtext"} + if markdown != Markdown.NEVER: + # When building man pages, we do not use the markdown pages, + # So, we can continue without the myst_parser dependencies. + # Doing so reduces dependencies of some packaged llvm distributions. + try: + import myst_parser + + extensions.append("myst_parser") + except ImportError as err: + if markdown == Markdown.ALWAYS or not tags.has("builder-man"): + print(venv_help(err), file=sys.stderr) + raise + else: + source_suffix[".md"] = "markdown" + myst_enable_extensions = ["substitution"] + myst_heading_anchors = 6 + myst_heading_slug_func = "llvm_sphinx.make_slug" + templates_path = ["_templates"] + master_doc = "index" + + return locals() + + +# Some of our markdown documentation numbers section titles +# This helpers is used by myst to remove that numbering from the anchor links. +def make_slug(s: str) -> str: + from docutils.nodes import make_id + from re import sub + + s = sub(r"^\s*(\w\.)+\w\s", "", s) + s = sub(r"^\s*\w\.\s", "", s) + return make_id(s) diff --git a/utils/docs/llvm_sphinx/ext/__init__.py b/utils/docs/llvm_sphinx/ext/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/clang/docs/ghlinks.py b/utils/docs/llvm_sphinx/ext/ghlinks/__init__.py similarity index 60% rename from clang/docs/ghlinks.py rename to utils/docs/llvm_sphinx/ext/ghlinks/__init__.py index 5ba4c9b6d323..752ad8291ac6 100644 --- a/clang/docs/ghlinks.py +++ b/utils/docs/llvm_sphinx/ext/ghlinks/__init__.py @@ -14,52 +14,22 @@ import re import sys import tempfile import textwrap +from typing import Dict, List, Type, Tuple from pathlib import Path - - -def _find_docs_requirements() -> str: - for parent in Path(__file__).resolve().parents: - docs_requirements = parent / "llvm/docs/requirements.txt" - if docs_requirements.exists(): - return str(docs_requirements) - return "llvm/docs/requirements.txt" - +from llvm_sphinx.help import venv_help try: from docutils import nodes from sphinx.application import Sphinx except ImportError as err: - print( - textwrap.dedent( - f""" - ghlinks.py requires the LLVM documentation build dependencies. - - Import failed with: - {err} - - The standard requirements file is: - {_find_docs_requirements()} - - From an llvm-project checkout, a typical pip setup is: - python3 -m venv .venv - . .venv/bin/activate - python3 -m pip install -r llvm/docs/requirements.txt - python3 clang/docs/ghlinks.py --test - - With uv, a typical one-shot command is: - uv run --with-requirements llvm/docs/requirements.txt \\ - python clang/docs/ghlinks.py --test - """ - ).strip(), - file=sys.stderr, - ) + print(venv_help(err), file=sys.stderr) raise __version__ = "1.0" GH_LINK_RE = re.compile("#GH([1-9][0-9]+)") GH_LINK_TMPL = "https://github.com/llvm/llvm-project/issues/{}" -SKIP_NODES: tuple[type[nodes.Node], ...] = ( +SKIP_NODES: Tuple[Type[nodes.Node], ...] = ( nodes.FixedTextElement, nodes.literal, nodes.raw, @@ -123,109 +93,21 @@ def subst_gh_links(_app: Sphinx, doctree: nodes.document) -> None: replace_gh_links_in_subtree(doctree) -def setup(app: Sphinx) -> dict[str, object]: +def setup(app: Sphinx) -> Dict[str, object]: app.connect("doctree-read", subst_gh_links) return dict(version=__version__, parallel_read_safe=True, parallel_write_safe=True) -def main(argv: list[str]) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--test", action="store_true", help="run ghlinks self-tests") - args = parser.parse_args(argv) - - if args.test: - run_tests() - print( - "ghlinks.py: tests passed; next, rebuild docs-clang-html and spot check the release notes" - ) - return 0 - - parser.print_help(sys.stderr) - return 0 - - # ----------------------------------------------------------------------------- # Test code only below: # ----------------------------------------------------------------------------- - -REST_TEST_DOC = r""" -GHLink reST Test -================ - -A paragraph links #GH123 and #GH456. - -No leading zero link #GH0123. - -Inline literal ``#GH333`` stays text. - -Existing link `#GH666 `_ stays existing. - -Code block:: - - #GH777 - -.. raw:: html - - #GH888 -""" - - -MARKDOWN_TEST_DOC = r""" -# GHLink Markdown Test - -A paragraph links #GH234 and #GH567. - -No leading zero link #GH0234. - -Inline code `#GH444` stays text. - -Existing link [#GH999](https://example.com/md) stays existing. - -```c -#GH778 -``` -""" - - -def _build_test_docs() -> tuple[str, str]: +def _build_test_docs() -> Tuple[str, str]: """Build in-file reST and Markdown test strings and return their HTML.""" - conf = f""" -import sys -sys.path.insert(0, {str(Path(__file__).parent)!r}) - -extensions = ["ghlinks", "myst_parser"] -master_doc = "index" -project = "ghlinks test" -source_suffix = {{ - ".rst": "restructuredtext", - ".md": "markdown", -}} -""" - index = """ -GHLink Tests -============ - -.. toctree:: - - rest - markdown -""" - + srcdir = Path(__file__).resolve().parent / "test" with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) - srcdir = tmp_path / "src" outdir = tmp_path / "out" doctreedir = tmp_path / "doctrees" - srcdir.mkdir() - (srcdir / "conf.py").write_text(textwrap.dedent(conf), encoding="utf-8") - (srcdir / "index.rst").write_text(textwrap.dedent(index), encoding="utf-8") - (srcdir / "rest.rst").write_text( - textwrap.dedent(REST_TEST_DOC), encoding="utf-8" - ) - (srcdir / "markdown.md").write_text( - textwrap.dedent(MARKDOWN_TEST_DOC), encoding="utf-8" - ) - app = Sphinx( srcdir=srcdir, confdir=srcdir, @@ -267,7 +149,3 @@ def run_tests() -> None: for issue in ("0234", "444", "999", "778"): _check_contains(markdown_html, f"#GH{issue}") _check_contains(markdown_html, 'href="https://example.com/md"') - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/utils/docs/llvm_sphinx/ext/ghlinks/test/conf.py b/utils/docs/llvm_sphinx/ext/ghlinks/test/conf.py new file mode 100644 index 000000000000..01d309121161 --- /dev/null +++ b/utils/docs/llvm_sphinx/ext/ghlinks/test/conf.py @@ -0,0 +1,7 @@ +extensions = ["llvm_sphinx.ext.ghlinks", "myst_parser"] +master_doc = "index" +project = "ghlinks test" +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} diff --git a/utils/docs/llvm_sphinx/ext/ghlinks/test/index.rst b/utils/docs/llvm_sphinx/ext/ghlinks/test/index.rst new file mode 100644 index 000000000000..45b6ef8f145f --- /dev/null +++ b/utils/docs/llvm_sphinx/ext/ghlinks/test/index.rst @@ -0,0 +1,8 @@ + +GHLink Tests +============ + +.. toctree:: + + rest + markdown diff --git a/utils/docs/llvm_sphinx/ext/ghlinks/test/markdown.md b/utils/docs/llvm_sphinx/ext/ghlinks/test/markdown.md new file mode 100644 index 000000000000..8b331071c128 --- /dev/null +++ b/utils/docs/llvm_sphinx/ext/ghlinks/test/markdown.md @@ -0,0 +1,14 @@ + +# GHLink Markdown Test + +A paragraph links #GH234 and #GH567. + +No leading zero link #GH0234. + +Inline code `#GH444` stays text. + +Existing link [#GH999](https://example.com/md) stays existing. + +```c +#GH778 +``` diff --git a/utils/docs/llvm_sphinx/ext/ghlinks/test/rest.rst b/utils/docs/llvm_sphinx/ext/ghlinks/test/rest.rst new file mode 100644 index 000000000000..465c61b35eee --- /dev/null +++ b/utils/docs/llvm_sphinx/ext/ghlinks/test/rest.rst @@ -0,0 +1,19 @@ + +GHLink reST Test +================ + +A paragraph links #GH123 and #GH456. + +No leading zero link #GH0123. + +Inline literal ``#GH333`` stays text. + +Existing link `#GH666 `_ stays existing. + +Code block:: + + #GH777 + +.. raw:: html + + #GH888 diff --git a/utils/docs/llvm_sphinx/help.py b/utils/docs/llvm_sphinx/help.py new file mode 100644 index 000000000000..4648482f9045 --- /dev/null +++ b/utils/docs/llvm_sphinx/help.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +import textwrap + + +def venv_help(err) -> str: + return textwrap.dedent( + f""" + Missing LLVM documentation build dependencies. + + Import failed with: + {err} + + The standard requirements file is: + llvm-project/llvm/docs/requirements.txt + + From an llvm-project checkout, a typical pip setup is: + python3 -m venv .venv + . .venv/bin/activate + python3 -m pip install -r llvm/docs/requirements.txt + python3 utils/docs --test + + With uv, a typical one-shot command is: + uv run --with-requirements llvm/docs/requirements.txt \\ + python utils/docs --test + """ + ).strip()