Split sphinx call logic from docs Makefile

As we're reorganizing the place where doc scripts are located, move this one to tools/docs.

No functional changes.

Signed-off-by Mauro Carvalho Chehab <[email protected]>

---
 Documentation/Makefile                             | 14 +++++++-------
 Documentation/doc-guide/sphinx.rst                 |  4 ++--
 Documentation/sphinx/kerneldoc-preamble.sty        |  2 +-
 .../translations/it_IT/doc-guide/sphinx.rst        |  4 ++--
 .../translations/zh_CN/doc-guide/sphinx.rst        |  4 ++--
 Documentation/translations/zh_CN/how-to.rst        |  2 +-

Quoted 5 lines · Mauro Carvalho Chehab

As we're reorganizing the place where doc scripts are located, move this one to tools/docs.

No functional changes.

Quoted 8 lines

Signed-off-by Mauro Carvalho Chehab <[email protected]>

---
 Documentation/Makefile                             | 14 +++++++-------
 Documentation/doc-guide/sphinx.rst                 |  4 ++--
 Documentation/sphinx/kerneldoc-preamble.sty        |  2 +-
 .../translations/it_IT/doc-guide/sphinx.rst        |  4 ++--
 .../translations/zh_CN/doc-guide/sphinx.rst        |  4 ++--
 Documentation/translations/zh_CN/how-to.rst        |  2 +-  

This one fails to apply to the translations - it looks like some sort of encoding problem...?

Quoted 1 line

Quoted 1 line

with open("Documentation/translations/zh_CN/how-to.rst", "r", encoding="utf-8") as fp: c=fp.read()

Quoted 1 line

Quoted 1 line

Quoted 5 lines · Mauro Carvalho Chehab

As we're reorganizing the place where doc scripts are located, move this one to tools/docs.

No functional changes.

Quoted 8 lines

Signed-off-by Mauro Carvalho Chehab <[email protected]>

---
 Documentation/Makefile                             | 14 +++++++-------
 Documentation/doc-guide/sphinx.rst                 |  4 ++--
 Documentation/sphinx/kerneldoc-preamble.sty        |  2 +-
 .../translations/it_IT/doc-guide/sphinx.rst        |  4 ++--
 .../translations/zh_CN/doc-guide/sphinx.rst        |  4 ++--
 Documentation/translations/zh_CN/how-to.rst        |  2 +-  

This one fails to apply to the translations - it looks like some sort of encoding problem...?

The sphinx-pre-install code has some logic to deal with Python version, which ensures that a minimal version will be enforced for documentation build logic.

Move it to a separate library to allow re-using its code.

No functional changes.

Signed-off-by Mauro Carvalho Chehab <[email protected]>

---
 tools/docs/lib/python_version.py | 133 +++++++++++++++++++++++++++++++
 tools/docs/sphinx-pre-install    | 120 +++-------------------------
 2 files changed, 146 insertions(+), 107 deletions(-)
 create mode 100644 tools/docs/lib/python_version.py

tools/docs/lib/python_version.py new file +30 -0

new file mode 100644
index 000000000000..0519d524e547
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (c) 2017-2025 Mauro Carvalho Chehab <[email protected]>
+
+"""
+Handle Python version check logic.
+
+Not all Python versions are supported by scripts. Yet, on some cases,
+like during documentation build, a newer version of python could be
+available.
+
+This class allows checking if the minimal requirements are followed.
+
+Better than that, PythonVersion.check_python() not only checks the minimal
+requirements, but it automatically switches to a the newest available
+Python version if present.
+
+"""
+
+import os
+import re
+import subprocess
+import sys
+
+from glob import glob
+
+class PythonVersion:
+    """
+    Ancillary methods that checks for missing dependencies for different
+    types of types, like binaries, python modules, rpm deps, etc.

tools/docs/lib/python_version.py new file +9 -0

+
+    @staticmethod
+    def ver_str(version):
+        """Returns a version tuple as major.minor.patch"""
+        return ".".join([str(x) for x in version])
+
+    def __str__(self):
+        """Returns a version tuple as major.minor.patch from self.version"""
+        return self.ver_str(self.version)

tools/docs/lib/python_version.py new file +55 -0

+            kwargs['universal_newlines'] = True
+        else:
+            kwargs['text'] = True
+
+        result = subprocess.run([cmd, "--version"],
+                                stdout = subprocess.PIPE,
+                                stderr = subprocess.PIPE,
+                                **kwargs, check=False)
+
+        version = result.stdout.strip()
+
+        match = re.search(r"(\d+\.\d+\.\d+)", version)
+        if match:
+            return PythonVersion.parse_version(match.group(1))
+
+        print(f"Can't parse version {version}")
+        return (0, 0, 0)
+
+    @staticmethod
+    def find_python(min_version):
+        """
+        Detect if are out there any python 3.xy version newer than the
+        current one.
+
+        Note: this routine is limited to up to 2 digits for python3. We
+        may need to update it one day, hopefully on a distant future.
+        """
+        patterns = [
+            "python3.[0-9]",
+            "python3.[0-9][0-9]",
+        ]
+
+        # Seek for a python binary newer than min_version
+        for path in os.getenv("PATH", "").split(":"):
+            for pattern in patterns:
+                for cmd in glob(os.path.join(path, pattern)):
+                    if os.path.isfile(cmd) and os.access(cmd, os.X_OK):
+                        version = PythonVersion.get_python_version(cmd)
+                        if version >= min_version:
+                            return cmd
+
+        return None
+
+    @staticmethod
+    def check_python(min_version):
+        """
+        Check if the current python binary satisfies our minimal requirement
+        for Sphinx build. If not, re-run with a newer version if found.
+        """
+        cur_ver = sys.version_info[:3]
+        if cur_ver >= min_version:
+            ver = PythonVersion.ver_str(cur_ver)
+            print(f"Python version: {ver}")
+
+            return

tools/docs/lib/python_version.py new file +9 -0

+        script_path = os.path.abspath(sys.argv[0])
+        args = [new_python_cmd, script_path] + sys.argv[1:]
+
+        print(f"Python {python_ver} not supported. Changing to {new_python_cmd}")
+
+        try:
+            os.execv(new_python_cmd, args)
+        except OSError as e:
+            sys.exit(f"Failed to restart with {new_python_cmd}: {e}")

tools/docs/sphinx-pre-install +13 -107

index 954ed3dc0645..d6d673b7945c 100755
@@ -32,20 +32,10 @@ import subprocess
 import sys
 from glob import glob
 
+from lib.python_version import PythonVersion
 
-def parse_version(version):
-    """Convert a major.minor.patch version into a tuple"""
-    return tuple(int(x) for x in version.split("."))
-
-
-def ver_str(version):
-    """Returns a version tuple as major.minor.patch"""
-
-    return ".".join([str(x) for x in version])
-
-
-RECOMMENDED_VERSION = parse_version("3.4.3")
-MIN_PYTHON_VERSION = parse_version("3.7")
+RECOMMENDED_VERSION = PythonVersion("3.4.3").version
+MIN_PYTHON_VERSION = PythonVersion("3.7").version
 
 
 class DepManager:

@@ -235,95 +225,11 @@ class AncillaryMethods:
 
         return None
 
-    @staticmethod
-    def get_python_version(cmd):
-        """
-        Get python version from a Python binary. As we need to detect if
-        are out there newer python binaries, we can't rely on sys.release here.
-        """
-
-        result = SphinxDependencyChecker.run([cmd, "--version"],
-                                            capture_output=True, text=True)
-        version = result.stdout.strip()
-
-        match = re.search(r"(\d+\.\d+\.\d+)", version)
-        if match:
-            return parse_version(match.group(1))
-
-        print(f"Can't parse version {version}")
-        return (0, 0, 0)
-
-    @staticmethod
-    def find_python():
-        """
-        Detect if are out there any python 3.xy version newer than the
-        current one.
-
-        Note: this routine is limited to up to 2 digits for python3. We
-        may need to update it one day, hopefully on a distant future.
-        """
-        patterns = [
-            "python3.[0-9]",
-            "python3.[0-9][0-9]",
-        ]
-
-        # Seek for a python binary newer than MIN_PYTHON_VERSION
-        for path in os.getenv("PATH", "").split(":"):
-            for pattern in patterns:
-                for cmd in glob(os.path.join(path, pattern)):
-                    if os.path.isfile(cmd) and os.access(cmd, os.X_OK):
-                        version = SphinxDependencyChecker.get_python_version(cmd)
-                        if version >= MIN_PYTHON_VERSION:
-                            return cmd
-
-    @staticmethod
-    def check_python():
-        """
-        Check if the current python binary satisfies our minimal requirement
-        for Sphinx build. If not, re-run with a newer version if found.
-        """
-        cur_ver = sys.version_info[:3]
-        if cur_ver >= MIN_PYTHON_VERSION:
-            ver = ver_str(cur_ver)
-            print(f"Python version: {ver}")
-
-            # This could be useful for debugging purposes
-            if SphinxDependencyChecker.which("docutils"):
-                result = SphinxDependencyChecker.run(["docutils", "--version"],
-                                                    capture_output=True, text=True)
-                ver = result.stdout.strip()
-                match = re.search(r"(\d+\.\d+\.\d+)", ver)
-                if match:
-                    ver = match.group(1)
-
-                print(f"Docutils version: {ver}")
-
-            return
-
-        python_ver = ver_str(cur_ver)
-
-        new_python_cmd = SphinxDependencyChecker.find_python()
-        if not new_python_cmd:
-            print(f"ERROR: Python version {python_ver} is not spported anymore\n")
-            print("       Can't find a new version. This script may fail")
-            return
-
-        # Restart script using the newer version
-        script_path = os.path.abspath(sys.argv[0])
-        args = [new_python_cmd, script_path] + sys.argv[1:]
-
-        print(f"Python {python_ver} not supported. Changing to {new_python_cmd}")
-
-        try:
-            os.execv(new_python_cmd, args)
-        except OSError as e:
-            sys.exit(f"Failed to restart with {new_python_cmd}: {e}")
-
     @staticmethod
     def run(*args, **kwargs):
         """
         Excecute a command, hiding its output by default.
-        Preserve comatibility with older Python versions.
+        Preserve compatibility with older Python versions.
         """
 
         capture_output = kwargs.pop('capture_output', False)

@@ -527,11 +433,11 @@ class MissingCheckers(AncillaryMethods):
         for line in result.stdout.split("\n"):
             match = re.match(r"^sphinx-build\s+([\d\.]+)(?:\+(?:/[\da-f]+)|b\d+)?\s*$", line)
             if match:
-                return parse_version(match.group(1))
+                return PythonVersion.parse_version(match.group(1))
 
             match = re.match(r"^Sphinx.*\s+([\d\.]+)\s*$", line)
             if match:
-                return parse_version(match.group(1))
+                return PythonVersion.parse_version(match.group(1))
 
     def check_sphinx(self, conf):
         """

@@ -542,7 +448,7 @@ class MissingCheckers(AncillaryMethods):
                 for line in f:
                     match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line)
                     if match:
-                        self.min_version = parse_version(match.group(1))
+                        self.min_version = PythonVersion.parse_version(match.group(1))
                         break
         except IOError:
             sys.exit(f"Can't open {conf}")

@@ -562,8 +468,8 @@ class MissingCheckers(AncillaryMethods):
             sys.exit(f"{sphinx} didn't return its version")
 
         if self.cur_version < self.min_version:
-            curver = ver_str(self.cur_version)
-            minver = ver_str(self.min_version)
+            curver = PythonVersion.ver_str(self.cur_version)
+            minver = PythonVersion.ver_str(self.min_version)
 
             print(f"ERROR: Sphinx version is {curver}. It should be >= {minver}")
             self.need_sphinx = 1

@@ -1304,7 +1210,7 @@ class SphinxDependencyChecker(MissingCheckers):
             else:
                 if self.need_sphinx and ver >= self.min_version:
                     return (f, ver)
-                elif parse_version(ver) > self.cur_version:
+                elif PythonVersion.parse_version(ver) > self.cur_version:
                     return (f, ver)
 
         return ("", ver)

@@ -1411,7 +1317,7 @@ class SphinxDependencyChecker(MissingCheckers):
             return
 
         if self.latest_avail_ver:
-            latest_avail_ver = ver_str(self.latest_avail_ver)
+            latest_avail_ver = PythonVersion.ver_str(self.latest_avail_ver)
 
         if not self.need_sphinx:
             # sphinx-build is present and its version is >= $min_version

@@ -1507,7 +1413,7 @@ class SphinxDependencyChecker(MissingCheckers):
         else:
             print("Unknown OS")
         if self.cur_version != (0, 0, 0):
-            ver = ver_str(self.cur_version)
+            ver = PythonVersion.ver_str(self.cur_version)
             print(f"Sphinx version: {ver}\n")
 
         # Check the type of virtual env, depending on Python version

@@ -1613,7 +1519,7 @@ def main():
 
     checker = SphinxDependencyChecker(args)
 
-    checker.check_python()
+    PythonVersion.check_python(MIN_PYTHON_VERSION)
     checker.check_needs()
 
 # Call main if not used as module

Quoted 51 lines

The sphinx-pre-install code has some logic to deal with Python version, which ensures that a minimal version will be enforced for documentation build logic.

Move it to a separate library to allow re-using its code.

No functional changes.

Signed-off-by Mauro Carvalho Chehab <[email protected]>

---
 tools/docs/lib/python_version.py | 133 +++++++++++++++++++++++++++++++
 tools/docs/sphinx-pre-install    | 120 +++-------------------------
 2 files changed, 146 insertions(+), 107 deletions(-)
 create mode 100644 tools/docs/lib/python_version.py

tools/docs/lib/python_version.py new file +30 -0

new file mode 100644
index 000000000000..0519d524e547
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (c) 2017-2025 Mauro Carvalho Chehab <[email protected]>
+
+"""
+Handle Python version check logic.
+
+Not all Python versions are supported by scripts. Yet, on some cases,
+like during documentation build, a newer version of python could be
+available.
+
+This class allows checking if the minimal requirements are followed.
+
+Better than that, PythonVersion.check_python() not only checks the minimal
+requirements, but it automatically switches to a the newest available
+Python version if present.
+
+"""
+
+import os
+import re
+import subprocess
+import sys
+
+from glob import glob
+
+class PythonVersion:
+    """
+    Ancillary methods that checks for missing dependencies for different
+    types of types, like binaries, python modules, rpm deps, etc.

tools/docs/lib/python_version.py line 40

Quoted 10 lines

tools/docs/lib/python_version.py new file +10 -0

+    """
+
+    def __init__(self, version):
+        """��nitialize self.version tuple from a version string"""
+        self.version = self.parse_version(version)
+
+    @staticmethod
+    def parse_version(version):
+        """Convert a major.minor.patch version into a tuple"""
+        return tuple(int(x) for x in version.split("."))  

I've written a few quick and dirty semver parsers myself, and it saddens me to think of adding a simplistic one in the kernel.

I'm just wondering, are we doomed to reinventing the wheels in our reluctance to depend on anything else?

Quoted 10 lines

tools/docs/lib/python_version.py new file +9 -0

+
+    @staticmethod
+    def ver_str(version):
+        """Returns a version tuple as major.minor.patch"""
+        return ".".join([str(x) for x in version])
+
+    def __str__(self):
+        """Returns a version tuple as major.minor.patch from self.version"""
+        return self.ver_str(self.version)

tools/docs/lib/python_version.py line 59

Quoted 10 lines

tools/docs/lib/python_version.py new file +10 -0

+
+    @staticmethod
+    def get_python_version(cmd):
+        """
+        Get python version from a Python binary. As we need to detect if
+        are out there newer python binaries, we can't rely on sys.release here.
+        """
+
+        kwargs = {}
+        if sys.version_info < (3, 7):  

Checking for things that EOL'd four years ago. Why are we doing this to ourselves? Why should we take on maintenance of code that jumps through hoops for things that nobody supports anymore?

In Documentation/process/changes.rst we've declared Python 3.9 as minimum, which is also the oldest version supported by upstream (until next month). Even Debian oldoldstable (that's two olds) has 3.9.

We're talking about the documentation build. I can undertand being more conservative about the kernel build in general, but IMHO this is just extra work for absolutely no users out there. And I'm not advocating for bleeding edge here.

We could just throw out a lot of crap by setting newer but still moderately concervative required Python (and Sphinx) versions, and bail out on older version. Let the user figure out how to get them.

We don't do this for any other tools either.

I'm saying that instead of refactoring this overgrown logic to a separate file and class, it should be nuked out of the kernel completely.

Quoted 60 lines

tools/docs/lib/python_version.py new file +60 -0

+            kwargs['universal_newlines'] = True
+        else:
+            kwargs['text'] = True
+
+        result = subprocess.run([cmd, "--version"],
+                                stdout = subprocess.PIPE,
+                                stderr = subprocess.PIPE,
+                                **kwargs, check=False)
+
+        version = result.stdout.strip()
+
+        match = re.search(r"(\d+\.\d+\.\d+)", version)
+        if match:
+            return PythonVersion.parse_version(match.group(1))
+
+        print(f"Can't parse version {version}")
+        return (0, 0, 0)
+
+    @staticmethod
+    def find_python(min_version):
+        """
+        Detect if are out there any python 3.xy version newer than the
+        current one.
+
+        Note: this routine is limited to up to 2 digits for python3. We
+        may need to update it one day, hopefully on a distant future.
+        """
+        patterns = [
+            "python3.[0-9]",
+            "python3.[0-9][0-9]",
+        ]
+
+        # Seek for a python binary newer than min_version
+        for path in os.getenv("PATH", "").split(":"):
+            for pattern in patterns:
+                for cmd in glob(os.path.join(path, pattern)):
+                    if os.path.isfile(cmd) and os.access(cmd, os.X_OK):
+                        version = PythonVersion.get_python_version(cmd)
+                        if version >= min_version:
+                            return cmd
+
+        return None
+
+    @staticmethod
+    def check_python(min_version):
+        """
+        Check if the current python binary satisfies our minimal requirement
+        for Sphinx build. If not, re-run with a newer version if found.
+        """
+        cur_ver = sys.version_info[:3]
+        if cur_ver >= min_version:
+            ver = PythonVersion.ver_str(cur_ver)
+            print(f"Python version: {ver}")
+
+            return
+
+        python_ver = PythonVersion.ver_str(cur_ver)
+
+        new_python_cmd = PythonVersion.find_python(min_version)
+        if not new_python_cmd:

tools/docs/lib/python_version.py line 124

Quoted 5 lines

tools/docs/lib/python_version.py new file +5 -0

+            print(f"ERROR: Python version {python_ver} is not spported anymore\n")
+            print("       Can't find a new version. This script may fail")
+            return
+
+        # Restart script using the newer version  

I thought the whole idea of restarting was completely rejected by approximately everyone?!

BR,
Jani.

Quoted 197 lines

tools/docs/lib/python_version.py new file +9 -0

+        script_path = os.path.abspath(sys.argv[0])
+        args = [new_python_cmd, script_path] + sys.argv[1:]
+
+        print(f"Python {python_ver} not supported. Changing to {new_python_cmd}")
+
+        try:
+            os.execv(new_python_cmd, args)
+        except OSError as e:
+            sys.exit(f"Failed to restart with {new_python_cmd}: {e}")

tools/docs/sphinx-pre-install +12 -106

index 954ed3dc0645..d6d673b7945c 100755
@@ -32,20 +32,10 @@ import subprocess
 import sys
 from glob import glob
 
+from lib.python_version import PythonVersion
 
-def parse_version(version):
-    """Convert a major.minor.patch version into a tuple"""
-    return tuple(int(x) for x in version.split("."))
-
-
-def ver_str(version):
-    """Returns a version tuple as major.minor.patch"""
-
-    return ".".join([str(x) for x in version])
-
-
-RECOMMENDED_VERSION = parse_version("3.4.3")
-MIN_PYTHON_VERSION = parse_version("3.7")
+RECOMMENDED_VERSION = PythonVersion("3.4.3").version
+MIN_PYTHON_VERSION = PythonVersion("3.7").version
 
 
 class DepManager:

@@ -235,95 +225,11 @@ class AncillaryMethods:
 
         return None
 
-    @staticmethod
-    def get_python_version(cmd):
-        """
-        Get python version from a Python binary. As we need to detect if
-        are out there newer python binaries, we can't rely on sys.release here.
-        """
-
-        result = SphinxDependencyChecker.run([cmd, "--version"],
-                                            capture_output=True, text=True)
-        version = result.stdout.strip()
-
-        match = re.search(r"(\d+\.\d+\.\d+)", version)
-        if match:
-            return parse_version(match.group(1))
-
-        print(f"Can't parse version {version}")
-        return (0, 0, 0)
-
-    @staticmethod
-    def find_python():
-        """
-        Detect if are out there any python 3.xy version newer than the
-        current one.
-
-        Note: this routine is limited to up to 2 digits for python3. We
-        may need to update it one day, hopefully on a distant future.
-        """
-        patterns = [
-            "python3.[0-9]",
-            "python3.[0-9][0-9]",
-        ]
-
-        # Seek for a python binary newer than MIN_PYTHON_VERSION
-        for path in os.getenv("PATH", "").split(":"):
-            for pattern in patterns:
-                for cmd in glob(os.path.join(path, pattern)):
-                    if os.path.isfile(cmd) and os.access(cmd, os.X_OK):
-                        version = SphinxDependencyChecker.get_python_version(cmd)
-                        if version >= MIN_PYTHON_VERSION:
-                            return cmd
-
-    @staticmethod
-    def check_python():
-        """
-        Check if the current python binary satisfies our minimal requirement
-        for Sphinx build. If not, re-run with a newer version if found.
-        """
-        cur_ver = sys.version_info[:3]
-        if cur_ver >= MIN_PYTHON_VERSION:
-            ver = ver_str(cur_ver)
-            print(f"Python version: {ver}")
-
-            # This could be useful for debugging purposes
-            if SphinxDependencyChecker.which("docutils"):
-                result = SphinxDependencyChecker.run(["docutils", "--version"],
-                                                    capture_output=True, text=True)
-                ver = result.stdout.strip()
-                match = re.search(r"(\d+\.\d+\.\d+)", ver)
-                if match:
-                    ver = match.group(1)
-
-                print(f"Docutils version: {ver}")
-
-            return
-
-        python_ver = ver_str(cur_ver)
-
-        new_python_cmd = SphinxDependencyChecker.find_python()
-        if not new_python_cmd:
-            print(f"ERROR: Python version {python_ver} is not spported anymore\n")
-            print("       Can't find a new version. This script may fail")
-            return
-
-        # Restart script using the newer version
-        script_path = os.path.abspath(sys.argv[0])
-        args = [new_python_cmd, script_path] + sys.argv[1:]
-
-        print(f"Python {python_ver} not supported. Changing to {new_python_cmd}")
-
-        try:
-            os.execv(new_python_cmd, args)
-        except OSError as e:
-            sys.exit(f"Failed to restart with {new_python_cmd}: {e}")
-
     @staticmethod
     def run(*args, **kwargs):
         """
         Excecute a command, hiding its output by default.
-        Preserve comatibility with older Python versions.
+        Preserve compatibility with older Python versions.
         """
 
         capture_output = kwargs.pop('capture_output', False)

@@ -527,11 +433,11 @@ class MissingCheckers(AncillaryMethods):
         for line in result.stdout.split("\n"):
             match = re.match(r"^sphinx-build\s+([\d\.]+)(?:\+(?:/[\da-f]+)|b\d+)?\s*$", line)
             if match:
-                return parse_version(match.group(1))
+                return PythonVersion.parse_version(match.group(1))
 
             match = re.match(r"^Sphinx.*\s+([\d\.]+)\s*$", line)
             if match:
-                return parse_version(match.group(1))
+                return PythonVersion.parse_version(match.group(1))
 
     def check_sphinx(self, conf):
         """

@@ -542,7 +448,7 @@ class MissingCheckers(AncillaryMethods):
                 for line in f:
                     match = re.match(r"^\s*needs_sphinx\s*=\s*[\'\"]([\d\.]+)[\'\"]", line)
                     if match:
-                        self.min_version = parse_version(match.group(1))
+                        self.min_version = PythonVersion.parse_version(match.group(1))
                         break
         except IOError:
             sys.exit(f"Can't open {conf}")

@@ -562,8 +468,8 @@ class MissingCheckers(AncillaryMethods):
             sys.exit(f"{sphinx} didn't return its version")
 
         if self.cur_version < self.min_version:
-            curver = ver_str(self.cur_version)
-            minver = ver_str(self.min_version)
+            curver = PythonVersion.ver_str(self.cur_version)
+            minver = PythonVersion.ver_str(self.min_version)
 
             print(f"ERROR: Sphinx version is {curver}. It should be >= {minver}")
             self.need_sphinx = 1

@@ -1304,7 +1210,7 @@ class SphinxDependencyChecker(MissingCheckers):
             else:
                 if self.need_sphinx and ver >= self.min_version:
                     return (f, ver)
-                elif parse_version(ver) > self.cur_version:
+                elif PythonVersion.parse_version(ver) > self.cur_version:
                     return (f, ver)
 
         return ("", ver)

@@ -1411,7 +1317,7 @@ class SphinxDependencyChecker(MissingCheckers):
             return
 
         if self.latest_avail_ver:
-            latest_avail_ver = ver_str(self.latest_avail_ver)
+            latest_avail_ver = PythonVersion.ver_str(self.latest_avail_ver)
 
         if not self.need_sphinx:
             # sphinx-build is present and its version is >= $min_version

@@ -1507,7 +1413,7 @@ class SphinxDependencyChecker(MissingCheckers):
         else:
             print("Unknown OS")
         if self.cur_version != (0, 0, 0):
-            ver = ver_str(self.cur_version)
+            ver = PythonVersion.ver_str(self.cur_version)
             print(f"Sphinx version: {ver}\n")
 
         # Check the type of virtual env, depending on Python version

@@ -1613,7 +1519,7 @@ def main():
 

tools/docs/sphinx-pre-install def main(): · line 1525

Quoted 7 lines

tools/docs/sphinx-pre-install +1 -1

     checker = SphinxDependencyChecker(args)
 
-    checker.check_python()
+    PythonVersion.check_python(MIN_PYTHON_VERSION)
     checker.check_needs()
 
 # Call main if not used as module  

Em Wed, 10 Sep 2025 13:14:33 +0300
Jani Nikula <[email protected]> escreveu:

Quoted 5 lines · Mauro Carvalho Chehab

Quoted 1 line

+ # Restart script using the newer version

I thought the whole idea of restarting was completely rejected by approximately everyone?!

This patch is just moving the code. There is a patch after this one changing the behavior.

Quoted 6 lines · Mauro Carvalho Chehab

Em Wed, 10 Sep 2025 13:14:33 +0300
Jani Nikula <[email protected]> escreveu:

Quoted 3 lines · Mauro Carvalho Chehab

Quoted 1 line

+ # Restart script using the newer version

Quoted 5 lines

Quoted 2 lines

I thought the whole idea of restarting was completely rejected by approximately everyone?!

This patch is just moving the code. There is a patch after this one changing the behavior.

Shouldn't we fix that kind of problems first, instead of refactoring?

#
# As we handle number of jobs and quiet in separate, we need to pick # both the same way as sphinx-build would pick, optionally accepts # whitespaces or not. So let's use argparse to handle argument expansion #
parser = argparse.ArgumentParser() parser.add_argument('-j', '--jobs', type=int) parser.add_argument('-q', '--quiet', type=int)

#
# Other sphinx-build arguments go as-is, so place them # at self.sphinxopts, using shell parser
#
sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "))

#
# Build a list of sphinx args
#
sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) if sphinx_args.quiet is True:
self.verbose = False

if sphinx_args.jobs:
self.n_jobs = sphinx_args.jobs

#
# If the command line argument "-j" is used override SPHINXOPTS #

self.n_jobs = n_jobs

Finally beginning to look at this. I'm working from the pulled version, rather than the commentless patch (please don't do that again :).

A nit
from SphinxBuilder::__init__():

Quoted 25 lines

#
# As we handle number of jobs and quiet in separate, we need to pick # both the same way as sphinx-build would pick, optionally accepts # whitespaces or not. So let's use argparse to handle argument expansion #
parser = argparse.ArgumentParser() parser.add_argument('-j', '--jobs', type=int) parser.add_argument('-q', '--quiet', type=int)

#
# Other sphinx-build arguments go as-is, so place them # at self.sphinxopts, using shell parser
#
sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "))

#
# Build a list of sphinx args
#
sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) if sphinx_args.quiet is True:
self.verbose = False

if sphinx_args.jobs:
self.n_jobs = sphinx_args.jobs

Quoted 5 lines

#
# If the command line argument "-j" is used override SPHINXOPTS #

self.n_jobs = n_jobs

First of all, I do wish you would isolate this sort of concern into its own function.

But, beyond that, you go to all that effort to parse the
--jobs flag, but that last line just throws it all away. What was the real purpose here?

Basically, what happens is that the number of jobs can be on different places:

Quoted 2 lines · Mauro Carvalho Chehab

Basically, what happens is that the number of jobs can be on different places:

There is a lot of complexity there, and spread out between __init__(), run_sphinx(), and handle_pdf(). Is there any way to create a single figure_out_how_many_damn_jobs() and coalesce that logic there? That would help make that part of the system a bit more comprehensible.

That said, I've been unable to make this change break in my testing. I guess I'm not seeing a lot of impediments to applying the next version at this point.

        # sphinxdirs can be a list or a whitespace-separated string
        #
        sphinxdirs_list = []
        for sphinxdir in sphinxdirs:
            if isinstance(sphinxdir, list):
                sphinxdirs_list += sphinxdir
            else:
                for name in sphinxdir.split(" "):
                    sphinxdirs_list.append(name)

Another nit:

Quoted 9 lines

        # sphinxdirs can be a list or a whitespace-separated string
        #
        sphinxdirs_list = []
        for sphinxdir in sphinxdirs:
            if isinstance(sphinxdir, list):
                sphinxdirs_list += sphinxdir
            else:
                for name in sphinxdir.split(" "):
                    sphinxdirs_list.append(name)

That inner loop just seems like a complicated way of saying:

	sphinxdirs_list += sphinxdir.split()

There are too much magic inside docs Makefile to properly run sphinx-build. Create an ancillary script that contains all kernel-related sphinx-build call logic currently at Makefile.

Such script is designed to work both as an standalone command and as part of a Makefile. As such, it properly handles POSIX jobserver used by GNU make.

On a side note, there was a line number increase due to the conversion:

 Documentation/Makefile          |  131 +++----------
 tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
 2 files changed, 323 insertions(+), 101 deletions(-)

This is because some things are more verbosed on Python and because it requires reading env vars from Makefile. Besides it, this script has some extra features that don't exist at the Makefile:

- It can be called directly from command line;
- It properly return PDF build errors.

When running the script alone, it will only take handle sphinx-build targets. On other words, it won't runn make rustdoc after building htmlfiles, nor it will run the extra check scripts.

Signed-off-by Mauro Carvalho Chehab <[email protected]>

---
 Documentation/Makefile          | 131 ++++----------
 tools/docs/sphinx-build-wrapper | 293 ++++++++++++++++++++++++++++++++
 2 files changed, 323 insertions(+), 101 deletions(-)
 create mode 100755 tools/docs/sphinx-build-wrapper

Documentation/Makefile +30 -101

index deb2029228ed..4736f02b6c9e 100644
@@ -23,21 +23,22 @@ SPHINXOPTS    =
 SPHINXDIRS    = .
 DOCS_THEME    =
 DOCS_CSS      =
-_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
 SPHINX_CONF   = conf.py
 PAPER         =
 BUILDDIR      = $(obj)/output
 PDFLATEX      = xelatex
 LATEXOPTS     = -interaction=batchmode -no-shell-escape
 
+PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
+
+# Wrapper for sphinx-build
+
+BUILD_WRAPPER = $(srctree)/tools/docs/sphinx-build-wrapper
+
 # For denylisting "variable font" files
 # Can be overridden by setting as an env variable
 FONTS_CONF_DENY_VF ?= $(HOME)/deny-vf
 
-ifeq ($(findstring 1, $(KBUILD_VERBOSE)),)
-SPHINXOPTS    += "-q"
-endif
-
 # User-friendly check for sphinx-build
 HAVE_SPHINX := $(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi)
 

@@ -51,63 +52,31 @@ ifeq ($(HAVE_SPHINX),0)
 
 else # HAVE_SPHINX
 
-# User-friendly check for pdflatex and latexmk
-HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi)
-HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi)
+# Common documentation targets
+infodocs texinfodocs latexdocs epubdocs xmldocs pdfdocs linkcheckdocs:
+	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
+	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
+		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
+		--builddir="$(BUILDDIR)" \
+		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
 
-ifeq ($(HAVE_LATEXMK),1)
-	PDFLATEX := latexmk -$(PDFLATEX)
-endif #HAVE_LATEXMK
-
-# Internal variables.
-PAPEROPT_a4     = -D latex_elements.papersize=a4paper
-PAPEROPT_letter = -D latex_elements.papersize=letterpaper
-ALLSPHINXOPTS   = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC)
-ALLSPHINXOPTS   += $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)
-ifneq ($(wildcard $(srctree)/.config),)
-ifeq ($(CONFIG_RUST),y)
-	# Let Sphinx know we will include rustdoc
-	ALLSPHINXOPTS   +=  -t rustdoc
-endif
+# Special handling for pdfdocs
+ifeq ($(shell which $(PDFLATEX) >/dev/null 2>&1; echo $$?),0)
+pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
+else
+pdfdocs:
+	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
+	@echo "  SKIP    Sphinx $@ target."
 endif
-# the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-# commands; the 'cmd' from scripts/Kbuild.include is not *loopable*
-loop_cmd = $(echo-cmd) $(cmd_$(1)) || exit;
-
-# $2 sphinx builder e.g. "html"
-# $3 name of the build subfolder / e.g. "userspace-api/media", used as:
-#    * dest folder relative to $(BUILDDIR) and
-#    * cache folder relative to $(BUILDDIR)/.doctrees
-# $4 dest subfolder e.g. "man" for man pages at userspace-api/media/man
-# $5 reST source folder relative to $(src),
-#    e.g. "userspace-api/media" for the linux-tv book-set at ./Documentation/userspace-api/media
-
-PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
-
-quiet_cmd_sphinx = SPHINX  $@ --> file://$(abspath $(BUILDDIR)/$3/$4)
-      cmd_sphinx = \
-	PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \
-	BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(src)/$5/$(SPHINX_CONF)) \
-	$(PYTHON3) $(srctree)/scripts/jobserver-exec \
-	$(CONFIG_SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \
-	$(SPHINXBUILD) \
-	-b $2 \
-	-c $(abspath $(src)) \
-	-d $(abspath $(BUILDDIR)/.doctrees/$3) \
-	-D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) \
-	$(ALLSPHINXOPTS) \
-	$(abspath $(src)/$5) \
-	$(abspath $(BUILDDIR)/$3/$4) && \
-	if [ "x$(DOCS_CSS)" != "x" ]; then \
-		cp $(if $(patsubst /%,,$(DOCS_CSS)),$(abspath $(srctree)/$(DOCS_CSS)),$(DOCS_CSS)) $(BUILDDIR)/$3/_static/; \
-	fi
 
+# HTML main logic is identical to other targets. However, if rust is enabled,
+# an extra step at the end is required to generate rustdoc.
 htmldocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,html,$(var),,$(var)))
-
+	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
+	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
+		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
+		--builddir="$(BUILDDIR)" \
+		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
 # If Rust support is available and .config exists, add rustdoc generated contents.
 # If there are any, the errors from this make rustdoc will be displayed but
 # won't stop the execution of htmldocs

@@ -118,49 +87,6 @@ ifeq ($(CONFIG_RUST),y)
 endif
 endif
 
-texinfodocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,texinfo,$(var),texinfo,$(var)))
-
-# Note: the 'info' Make target is generated by sphinx itself when
-# running the texinfodocs target define above.
-infodocs: texinfodocs
-	$(MAKE) -C $(BUILDDIR)/texinfo info
-
-linkcheckdocs:
-	@$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,linkcheck,$(var),,$(var)))
-
-latexdocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,latex,$(var),latex,$(var)))
-
-ifeq ($(HAVE_PDFLATEX),0)
-
-pdfdocs:
-	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
-	@echo "  SKIP    Sphinx $@ target."
-
-else # HAVE_PDFLATEX
-
-pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
-pdfdocs: latexdocs
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	$(foreach var,$(SPHINXDIRS), \
-	   $(MAKE) PDFLATEX="$(PDFLATEX)" LATEXOPTS="$(LATEXOPTS)" $(DENY_VF) -C $(BUILDDIR)/$(var)/latex || sh $(srctree)/scripts/check-variable-fonts.sh || exit; \
-	   mkdir -p $(BUILDDIR)/$(var)/pdf; \
-	   mv $(subst .tex,.pdf,$(wildcard $(BUILDDIR)/$(var)/latex/*.tex)) $(BUILDDIR)/$(var)/pdf/; \
-	)
-
-endif # HAVE_PDFLATEX
-
-epubdocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,epub,$(var),epub,$(var)))
-
-xmldocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,xml,$(var),xml,$(var)))
-
 endif # HAVE_SPHINX
 
 # The following targets are independent of HAVE_SPHINX, and the rules should

@@ -172,6 +98,9 @@ refcheckdocs:
 cleandocs:
 	$(Q)rm -rf $(BUILDDIR)
 
+# Used only on help
+_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
+
 dochelp:
 	@echo  ' Linux kernel internal documentation in different formats from ReST:'
 	@echo  '  htmldocs        - HTML'

tools/docs/sphinx-build-wrapper new file +293 -0

new file mode 100755
index 000000000000..3256418d8dc5
@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+import argparse
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+from lib.python_version import PythonVersion
+
+LIB_DIR = "../../scripts/lib"
+SRC_DIR = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
+
+from jobserver import JobserverExec
+
+MIN_PYTHON_VERSION = PythonVersion("3.7").version
+PAPER = ["", "a4", "letter"]
+TARGETS = {
+    "cleandocs":     { "builder": "clean" },
+    "linkcheckdocs": { "builder": "linkcheck" },
+    "htmldocs":      { "builder": "html" },
+    "epubdocs":      { "builder": "epub",    "out_dir": "epub" },
+    "texinfodocs":   { "builder": "texinfo", "out_dir": "texinfo" },
+    "infodocs":      { "builder": "texinfo", "out_dir": "texinfo" },
+    "latexdocs":     { "builder": "latex",   "out_dir": "latex" },
+    "pdfdocs":       { "builder": "latex",   "out_dir": "latex" },
+    "xmldocs":       { "builder": "xml",     "out_dir": "xml" },
+}
+
+class SphinxBuilder:
+    def is_rust_enabled(self):
+        config_path = os.path.join(self.srctree, ".config")
+        if os.path.isfile(config_path):
+            with open(config_path, "r", encoding="utf-8") as f:
+                return "CONFIG_RUST=y" in f.read()
+        return False
+
+    def get_path(self, path, use_cwd=False, abs_path=False):
+        path = os.path.expanduser(path)
+        if not path.startswith("/"):
+            if use_cwd:
+                base = os.getcwd()
+            else:
+                base = self.srctree
+            path = os.path.join(base, path)
+        if abs_path:
+            return os.path.abspath(path)
+        return path
+
+    def __init__(self, builddir, verbose=False, n_jobs=None):
+        self.verbose = None
+        self.kernelversion = os.environ.get("KERNELVERSION", "unknown")
+        self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown")
+        self.pdflatex = os.environ.get("PDFLATEX", "xelatex")
+        self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape")
+        if not verbose:
+            verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "")
+        if verbose is not None:
+            self.verbose = verbose
+        parser = argparse.ArgumentParser()
+        parser.add_argument('-j', '--jobs', type=int)
+        parser.add_argument('-q', '--quiet', type=int)
+        sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", ""))
+        sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)
+        if sphinx_args.quiet is True:
+            self.verbose = False
+        if sphinx_args.jobs:
+            self.n_jobs = sphinx_args.jobs
+        self.n_jobs = n_jobs
+        self.srctree = os.environ.get("srctree")
+        if not self.srctree:
+            self.srctree = "."
+            os.environ["srctree"] = self.srctree
+        self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build")
+        self.kerneldoc = self.get_path(os.environ.get("KERNELDOC",
+                                                      "scripts/kernel-doc.py"))
+        self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True)
+
+        self.config_rust = self.is_rust_enabled()
+
+        self.pdflatex_cmd = shutil.which(self.pdflatex)
+        self.latexmk_cmd = shutil.which("latexmk")
+
+        self.env = os.environ.copy()
+
+    def run_sphinx(self, sphinx_build, build_args, *args, **pwargs):
+        with JobserverExec() as jobserver:
+            if jobserver.claim:
+                n_jobs = str(jobserver.claim)
+            else:
+                n_jobs = "auto" # Supported since Sphinx 1.7
+            cmd = []
+            cmd.append(sys.executable)
+            cmd.append(sphinx_build)
+            if self.n_jobs:
+                n_jobs = str(self.n_jobs)
+
+            if n_jobs:
+                cmd += [f"-j{n_jobs}"]
+
+            if not self.verbose:
+                cmd.append("-q")
+            cmd += self.sphinxopts
+            cmd += build_args
+            if self.verbose:
+                print(" ".join(cmd))
+            return subprocess.call(cmd, *args, **pwargs)
+
+    def handle_html(self, css, output_dir):
+        if not css:
+            return
+        css = os.path.expanduser(css)
+        if not css.startswith("/"):
+            css = os.path.join(self.srctree, css)
+        static_dir = os.path.join(output_dir, "_static")
+        os.makedirs(static_dir, exist_ok=True)
+        try:
+            shutil.copy2(css, static_dir)
+        except (OSError, IOError) as e:
+            print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr)
+
+    def handle_pdf(self, output_dirs):
+        builds = {}
+        max_len = 0
+        for from_dir in output_dirs:
+            pdf_dir = os.path.join(from_dir, "../pdf")
+            os.makedirs(pdf_dir, exist_ok=True)
+            if self.latexmk_cmd:
+                latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"]
+            else:
+                latex_cmd = [self.pdflatex]
+            latex_cmd.extend(shlex.split(self.latexopts))
+            tex_suffix = ".tex"
+            has_tex = False
+            build_failed = False
+            with os.scandir(from_dir) as it:
+                for entry in it:
+                    if not entry.name.endswith(tex_suffix):
+                        continue
+                    name = entry.name[:-len(tex_suffix)]
+                    has_tex = True
+                    try:
+                        subprocess.run(latex_cmd + [entry.path],
+                                       cwd=from_dir, check=True)
+                    except subprocess.CalledProcessError:
+                        pass
+                    pdf_name = name + ".pdf"
+                    pdf_from = os.path.join(from_dir, pdf_name)
+                    pdf_to = os.path.join(pdf_dir, pdf_name)
+                    if os.path.exists(pdf_from):
+                        os.rename(pdf_from, pdf_to)
+                        builds[name] = os.path.relpath(pdf_to, self.builddir)
+                    else:
+                        builds[name] = "FAILED"
+                        build_failed = True
+                    name = entry.name.removesuffix(".tex")
+                    max_len = max(max_len, len(name))
+
+            if not has_tex:
+                name = os.path.basename(from_dir)
+                max_len = max(max_len, len(name))
+                builds[name] = "FAILED (no .tex)"
+                build_failed = True
+        msg = "Summary"
+        msg += "\n" + "=" * len(msg)
+        print()
+        print(msg)
+        for pdf_name, pdf_file in builds.items():
+            print(f"{pdf_name:<{max_len}}: {pdf_file}")
+        print()
+        if build_failed:
+            sys.exit("PDF build failed: not all PDF files were created.")
+        else:
+            print("All PDF files were built.")
+
+    def handle_info(self, output_dirs):
+        for output_dir in output_dirs:
+            try:
+                subprocess.run(["make", "info"], cwd=output_dir, check=True)
+            except subprocess.CalledProcessError as e:
+                sys.exit(f"Error generating info docs: {e}")
+
+    def cleandocs(self, builder):
+        shutil.rmtree(self.builddir, ignore_errors=True)
+
+    def build(self, target, sphinxdirs=None, conf="conf.py",
+              theme=None, css=None, paper=None):
+        builder = TARGETS[target]["builder"]
+        out_dir = TARGETS[target].get("out_dir", "")
+        if target == "cleandocs":
+            self.cleandocs(builder)
+            return
+        if theme:
+                os.environ["DOCS_THEME"] = theme
+        sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"])
+        if not sphinxbuild:
+            sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n")
+        if builder == "latex":
+            if not self.pdflatex_cmd and not self.latexmk_cmd:
+                sys.exit("Error: pdflatex or latexmk required for PDF generation")
+        docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation"))
+        kerneldoc = self.kerneldoc
+        if kerneldoc.startswith(self.srctree):
+            kerneldoc = os.path.relpath(kerneldoc, self.srctree)
+        args = [ "-b", builder, "-c", docs_dir ]
+        if builder == "latex":
+            if not paper:
+                paper = PAPER[1]
+            args.extend(["-D", f"latex_elements.papersize={paper}paper"])
+        if self.config_rust:
+            args.extend(["-t", "rustdoc"])
+        if conf:
+            self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True)
+        if not sphinxdirs:
+            sphinxdirs = os.environ.get("SPHINXDIRS", ".")
+        sphinxdirs_list = []
+        for sphinxdir in sphinxdirs:
+            if isinstance(sphinxdir, list):
+                sphinxdirs_list += sphinxdir
+            else:
+                for name in sphinxdir.split(" "):
+                    sphinxdirs_list.append(name)
+        output_dirs = []
+        for sphinxdir in sphinxdirs_list:
+            src_dir = os.path.join(docs_dir, sphinxdir)
+            doctree_dir = os.path.join(self.builddir, ".doctrees")
+            output_dir = os.path.join(self.builddir, sphinxdir, out_dir)
+            src_dir = os.path.normpath(src_dir)
+            doctree_dir = os.path.normpath(doctree_dir)
+            output_dir = os.path.normpath(output_dir)
+            os.makedirs(doctree_dir, exist_ok=True)
+            os.makedirs(output_dir, exist_ok=True)
+            output_dirs.append(output_dir)
+            build_args = args + [
+                "-d", doctree_dir,
+                "-D", f"kerneldoc_bin={kerneldoc}",
+                "-D", f"version={self.kernelversion}",
+                "-D", f"release={self.kernelrelease}",
+                "-D", f"kerneldoc_srctree={self.srctree}",
+                src_dir,
+                output_dir,
+            ]
+            try:
+                self.run_sphinx(sphinxbuild, build_args, env=self.env)
+            except (OSError, ValueError, subprocess.SubprocessError) as e:
+                sys.exit(f"Build failed: {repr(e)}")
+            if target in ["htmldocs", "epubdocs"]:
+                self.handle_html(css, output_dir)
+        if target == "pdfdocs":
+            self.handle_pdf(output_dirs)
+        elif target == "infodocs":
+            self.handle_info(output_dirs)
+
+def jobs_type(value):
+    if value is None:
+        return None
+    if value.lower() == 'auto':
+        return value.lower()
+    try:
+        if int(value) >= 1:
+            return value
+        raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}")
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Kernel documentation builder")
+    parser.add_argument("target", choices=list(TARGETS.keys()),
+                        help="Documentation target to build")
+    parser.add_argument("--sphinxdirs", nargs="+",
+                        help="Specific directories to build")
+    parser.add_argument("--conf", default="conf.py",
+                        help="Sphinx configuration file")
+    parser.add_argument("--builddir", default="output",
+                        help="Sphinx configuration file")
+    parser.add_argument("--theme", help="Sphinx theme to use")
+    parser.add_argument("--css", help="Custom CSS file for HTML/EPUB")
+    parser.add_argument("--paper", choices=PAPER, default=PAPER[0],
+                        help="Paper size for LaTeX/PDF output")
+    parser.add_argument("-v", "--verbose", action='store_true',
+                        help="place build in verbose mode")
+    parser.add_argument('-j', '--jobs', type=jobs_type,
+                        help="Sets number of jobs to use with sphinx-build")
+    args = parser.parse_args()
+    PythonVersion.check_python(MIN_PYTHON_VERSION)
+    builder = SphinxBuilder(builddir=args.builddir,
+                            verbose=args.verbose, n_jobs=args.jobs)
+    builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf,
+                  theme=args.theme, css=args.css, paper=args.paper)
+
+if __name__ == "__main__":
+    main()

Quoted 19 lines

There are too much magic inside docs Makefile to properly run sphinx-build. Create an ancillary script that contains all kernel-related sphinx-build call logic currently at Makefile.

Such script is designed to work both as an standalone command and as part of a Makefile. As such, it properly handles POSIX jobserver used by GNU make.

On a side note, there was a line number increase due to the conversion:

 Documentation/Makefile          |  131 +++----------
 tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
 2 files changed, 323 insertions(+), 101 deletions(-)

This is because some things are more verbosed on Python and because it requires reading env vars from Makefile. Besides it, this script has some extra features that don't exist at the Makefile:

Quoted 6 lines

- It can be called directly from command line;
- It properly return PDF build errors.

When running the script alone, it will only take handle sphinx-build targets. On other words, it won't runn make rustdoc after building htmlfiles, nor it will run the extra check scripts.

I've always strongly believed we should aim to make it possible to build the documentation by running sphinx-build directly on the command-line. Not that it would be the common way to run it, but to not accumulate things in the Makefile that need to happen before or after. To promote handling the documentation build in Sphinx. To be able to debug issues and try new Sphinx versions without all the hacks.

This patch moves a bunch of that logic into a Python wrapper, and I feel like it complicates matters. You can no longer rely on 'make V=1' to get the build commands, for instance.

Newer Sphinx versions have the -M option for "make mode". The Makefiles produced by sphinx-quickstart only have one build target:

# Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).

%: Makefile

        @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

That's all.

The proposed wrapper duplicates loads of code that's supposed to be handled by sphinx-build directly.

Including the target/builder names.

Seems to me the goal should be to figure out *generic* wrappers for handling parallelism, not Sphinx aware/specific.

BR,
Jani.

Quoted 476 lines

Signed-off-by Mauro Carvalho Chehab <[email protected]>

---
 Documentation/Makefile          | 131 ++++----------
 tools/docs/sphinx-build-wrapper | 293 ++++++++++++++++++++++++++++++++
 2 files changed, 323 insertions(+), 101 deletions(-)
 create mode 100755 tools/docs/sphinx-build-wrapper

Documentation/Makefile +30 -101

index deb2029228ed..4736f02b6c9e 100644
@@ -23,21 +23,22 @@ SPHINXOPTS    =
 SPHINXDIRS    = .
 DOCS_THEME    =
 DOCS_CSS      =
-_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
 SPHINX_CONF   = conf.py
 PAPER         =
 BUILDDIR      = $(obj)/output
 PDFLATEX      = xelatex
 LATEXOPTS     = -interaction=batchmode -no-shell-escape
 
+PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
+
+# Wrapper for sphinx-build
+
+BUILD_WRAPPER = $(srctree)/tools/docs/sphinx-build-wrapper
+
 # For denylisting "variable font" files
 # Can be overridden by setting as an env variable
 FONTS_CONF_DENY_VF ?= $(HOME)/deny-vf
 
-ifeq ($(findstring 1, $(KBUILD_VERBOSE)),)
-SPHINXOPTS    += "-q"
-endif
-
 # User-friendly check for sphinx-build
 HAVE_SPHINX := $(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi)
 

@@ -51,63 +52,31 @@ ifeq ($(HAVE_SPHINX),0)
 
 else # HAVE_SPHINX
 
-# User-friendly check for pdflatex and latexmk
-HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi)
-HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi)
+# Common documentation targets
+infodocs texinfodocs latexdocs epubdocs xmldocs pdfdocs linkcheckdocs:
+	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
+	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
+		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
+		--builddir="$(BUILDDIR)" \
+		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
 
-ifeq ($(HAVE_LATEXMK),1)
-	PDFLATEX := latexmk -$(PDFLATEX)
-endif #HAVE_LATEXMK
-
-# Internal variables.
-PAPEROPT_a4     = -D latex_elements.papersize=a4paper
-PAPEROPT_letter = -D latex_elements.papersize=letterpaper
-ALLSPHINXOPTS   = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC)
-ALLSPHINXOPTS   += $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)
-ifneq ($(wildcard $(srctree)/.config),)
-ifeq ($(CONFIG_RUST),y)
-	# Let Sphinx know we will include rustdoc
-	ALLSPHINXOPTS   +=  -t rustdoc
-endif
+# Special handling for pdfdocs
+ifeq ($(shell which $(PDFLATEX) >/dev/null 2>&1; echo $$?),0)
+pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
+else
+pdfdocs:
+	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
+	@echo "  SKIP    Sphinx $@ target."
 endif
-# the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-# commands; the 'cmd' from scripts/Kbuild.include is not *loopable*
-loop_cmd = $(echo-cmd) $(cmd_$(1)) || exit;
-
-# $2 sphinx builder e.g. "html"
-# $3 name of the build subfolder / e.g. "userspace-api/media", used as:
-#    * dest folder relative to $(BUILDDIR) and
-#    * cache folder relative to $(BUILDDIR)/.doctrees
-# $4 dest subfolder e.g. "man" for man pages at userspace-api/media/man
-# $5 reST source folder relative to $(src),
-#    e.g. "userspace-api/media" for the linux-tv book-set at ./Documentation/userspace-api/media
-
-PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
-
-quiet_cmd_sphinx = SPHINX  $@ --> file://$(abspath $(BUILDDIR)/$3/$4)
-      cmd_sphinx = \
-	PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \
-	BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(src)/$5/$(SPHINX_CONF)) \
-	$(PYTHON3) $(srctree)/scripts/jobserver-exec \
-	$(CONFIG_SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \
-	$(SPHINXBUILD) \
-	-b $2 \
-	-c $(abspath $(src)) \
-	-d $(abspath $(BUILDDIR)/.doctrees/$3) \
-	-D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) \
-	$(ALLSPHINXOPTS) \
-	$(abspath $(src)/$5) \
-	$(abspath $(BUILDDIR)/$3/$4) && \
-	if [ "x$(DOCS_CSS)" != "x" ]; then \
-		cp $(if $(patsubst /%,,$(DOCS_CSS)),$(abspath $(srctree)/$(DOCS_CSS)),$(DOCS_CSS)) $(BUILDDIR)/$3/_static/; \
-	fi
 
+# HTML main logic is identical to other targets. However, if rust is enabled,
+# an extra step at the end is required to generate rustdoc.
 htmldocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,html,$(var),,$(var)))
-
+	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
+	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
+		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
+		--builddir="$(BUILDDIR)" \
+		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
 # If Rust support is available and .config exists, add rustdoc generated contents.
 # If there are any, the errors from this make rustdoc will be displayed but
 # won't stop the execution of htmldocs

@@ -118,49 +87,6 @@ ifeq ($(CONFIG_RUST),y)
 endif
 endif
 
-texinfodocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,texinfo,$(var),texinfo,$(var)))
-
-# Note: the 'info' Make target is generated by sphinx itself when
-# running the texinfodocs target define above.
-infodocs: texinfodocs
-	$(MAKE) -C $(BUILDDIR)/texinfo info
-
-linkcheckdocs:
-	@$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,linkcheck,$(var),,$(var)))
-
-latexdocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,latex,$(var),latex,$(var)))
-
-ifeq ($(HAVE_PDFLATEX),0)
-
-pdfdocs:
-	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
-	@echo "  SKIP    Sphinx $@ target."
-
-else # HAVE_PDFLATEX
-
-pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
-pdfdocs: latexdocs
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	$(foreach var,$(SPHINXDIRS), \
-	   $(MAKE) PDFLATEX="$(PDFLATEX)" LATEXOPTS="$(LATEXOPTS)" $(DENY_VF) -C $(BUILDDIR)/$(var)/latex || sh $(srctree)/scripts/check-variable-fonts.sh || exit; \
-	   mkdir -p $(BUILDDIR)/$(var)/pdf; \
-	   mv $(subst .tex,.pdf,$(wildcard $(BUILDDIR)/$(var)/latex/*.tex)) $(BUILDDIR)/$(var)/pdf/; \
-	)
-
-endif # HAVE_PDFLATEX
-
-epubdocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,epub,$(var),epub,$(var)))
-
-xmldocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,xml,$(var),xml,$(var)))
-
 endif # HAVE_SPHINX
 
 # The following targets are independent of HAVE_SPHINX, and the rules should

@@ -172,6 +98,9 @@ refcheckdocs:
 cleandocs:
 	$(Q)rm -rf $(BUILDDIR)
 
+# Used only on help
+_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
+
 dochelp:
 	@echo  ' Linux kernel internal documentation in different formats from ReST:'
 	@echo  '  htmldocs        - HTML'

tools/docs/sphinx-build-wrapper new file +285 -0

new file mode 100755
index 000000000000..3256418d8dc5
@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+import argparse
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+from lib.python_version import PythonVersion
+
+LIB_DIR = "../../scripts/lib"
+SRC_DIR = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
+
+from jobserver import JobserverExec
+
+MIN_PYTHON_VERSION = PythonVersion("3.7").version
+PAPER = ["", "a4", "letter"]
+TARGETS = {
+    "cleandocs":     { "builder": "clean" },
+    "linkcheckdocs": { "builder": "linkcheck" },
+    "htmldocs":      { "builder": "html" },
+    "epubdocs":      { "builder": "epub",    "out_dir": "epub" },
+    "texinfodocs":   { "builder": "texinfo", "out_dir": "texinfo" },
+    "infodocs":      { "builder": "texinfo", "out_dir": "texinfo" },
+    "latexdocs":     { "builder": "latex",   "out_dir": "latex" },
+    "pdfdocs":       { "builder": "latex",   "out_dir": "latex" },
+    "xmldocs":       { "builder": "xml",     "out_dir": "xml" },
+}
+
+class SphinxBuilder:
+    def is_rust_enabled(self):
+        config_path = os.path.join(self.srctree, ".config")
+        if os.path.isfile(config_path):
+            with open(config_path, "r", encoding="utf-8") as f:
+                return "CONFIG_RUST=y" in f.read()
+        return False
+
+    def get_path(self, path, use_cwd=False, abs_path=False):
+        path = os.path.expanduser(path)
+        if not path.startswith("/"):
+            if use_cwd:
+                base = os.getcwd()
+            else:
+                base = self.srctree
+            path = os.path.join(base, path)
+        if abs_path:
+            return os.path.abspath(path)
+        return path
+
+    def __init__(self, builddir, verbose=False, n_jobs=None):
+        self.verbose = None
+        self.kernelversion = os.environ.get("KERNELVERSION", "unknown")
+        self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown")
+        self.pdflatex = os.environ.get("PDFLATEX", "xelatex")
+        self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape")
+        if not verbose:
+            verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "")
+        if verbose is not None:
+            self.verbose = verbose
+        parser = argparse.ArgumentParser()
+        parser.add_argument('-j', '--jobs', type=int)
+        parser.add_argument('-q', '--quiet', type=int)
+        sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", ""))
+        sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)
+        if sphinx_args.quiet is True:
+            self.verbose = False
+        if sphinx_args.jobs:
+            self.n_jobs = sphinx_args.jobs
+        self.n_jobs = n_jobs
+        self.srctree = os.environ.get("srctree")
+        if not self.srctree:
+            self.srctree = "."
+            os.environ["srctree"] = self.srctree
+        self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build")
+        self.kerneldoc = self.get_path(os.environ.get("KERNELDOC",
+                                                      "scripts/kernel-doc.py"))
+        self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True)
+
+        self.config_rust = self.is_rust_enabled()
+
+        self.pdflatex_cmd = shutil.which(self.pdflatex)
+        self.latexmk_cmd = shutil.which("latexmk")
+
+        self.env = os.environ.copy()
+
+    def run_sphinx(self, sphinx_build, build_args, *args, **pwargs):
+        with JobserverExec() as jobserver:
+            if jobserver.claim:
+                n_jobs = str(jobserver.claim)
+            else:
+                n_jobs = "auto" # Supported since Sphinx 1.7
+            cmd = []
+            cmd.append(sys.executable)
+            cmd.append(sphinx_build)
+            if self.n_jobs:
+                n_jobs = str(self.n_jobs)
+
+            if n_jobs:
+                cmd += [f"-j{n_jobs}"]
+
+            if not self.verbose:
+                cmd.append("-q")
+            cmd += self.sphinxopts
+            cmd += build_args
+            if self.verbose:
+                print(" ".join(cmd))
+            return subprocess.call(cmd, *args, **pwargs)
+
+    def handle_html(self, css, output_dir):
+        if not css:
+            return
+        css = os.path.expanduser(css)
+        if not css.startswith("/"):
+            css = os.path.join(self.srctree, css)
+        static_dir = os.path.join(output_dir, "_static")
+        os.makedirs(static_dir, exist_ok=True)
+        try:
+            shutil.copy2(css, static_dir)
+        except (OSError, IOError) as e:
+            print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr)
+
+    def handle_pdf(self, output_dirs):
+        builds = {}
+        max_len = 0
+        for from_dir in output_dirs:
+            pdf_dir = os.path.join(from_dir, "../pdf")
+            os.makedirs(pdf_dir, exist_ok=True)
+            if self.latexmk_cmd:
+                latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"]
+            else:
+                latex_cmd = [self.pdflatex]
+            latex_cmd.extend(shlex.split(self.latexopts))
+            tex_suffix = ".tex"
+            has_tex = False
+            build_failed = False
+            with os.scandir(from_dir) as it:
+                for entry in it:
+                    if not entry.name.endswith(tex_suffix):
+                        continue
+                    name = entry.name[:-len(tex_suffix)]
+                    has_tex = True
+                    try:
+                        subprocess.run(latex_cmd + [entry.path],
+                                       cwd=from_dir, check=True)
+                    except subprocess.CalledProcessError:
+                        pass
+                    pdf_name = name + ".pdf"
+                    pdf_from = os.path.join(from_dir, pdf_name)
+                    pdf_to = os.path.join(pdf_dir, pdf_name)
+                    if os.path.exists(pdf_from):
+                        os.rename(pdf_from, pdf_to)
+                        builds[name] = os.path.relpath(pdf_to, self.builddir)
+                    else:
+                        builds[name] = "FAILED"
+                        build_failed = True
+                    name = entry.name.removesuffix(".tex")
+                    max_len = max(max_len, len(name))
+
+            if not has_tex:
+                name = os.path.basename(from_dir)
+                max_len = max(max_len, len(name))
+                builds[name] = "FAILED (no .tex)"
+                build_failed = True
+        msg = "Summary"
+        msg += "\n" + "=" * len(msg)
+        print()
+        print(msg)
+        for pdf_name, pdf_file in builds.items():
+            print(f"{pdf_name:<{max_len}}: {pdf_file}")
+        print()
+        if build_failed:
+            sys.exit("PDF build failed: not all PDF files were created.")
+        else:
+            print("All PDF files were built.")
+
+    def handle_info(self, output_dirs):
+        for output_dir in output_dirs:
+            try:
+                subprocess.run(["make", "info"], cwd=output_dir, check=True)
+            except subprocess.CalledProcessError as e:
+                sys.exit(f"Error generating info docs: {e}")
+
+    def cleandocs(self, builder):
+        shutil.rmtree(self.builddir, ignore_errors=True)
+
+    def build(self, target, sphinxdirs=None, conf="conf.py",
+              theme=None, css=None, paper=None):
+        builder = TARGETS[target]["builder"]
+        out_dir = TARGETS[target].get("out_dir", "")
+        if target == "cleandocs":
+            self.cleandocs(builder)
+            return
+        if theme:
+                os.environ["DOCS_THEME"] = theme
+        sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"])
+        if not sphinxbuild:
+            sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n")
+        if builder == "latex":
+            if not self.pdflatex_cmd and not self.latexmk_cmd:
+                sys.exit("Error: pdflatex or latexmk required for PDF generation")
+        docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation"))
+        kerneldoc = self.kerneldoc
+        if kerneldoc.startswith(self.srctree):
+            kerneldoc = os.path.relpath(kerneldoc, self.srctree)
+        args = [ "-b", builder, "-c", docs_dir ]
+        if builder == "latex":
+            if not paper:
+                paper = PAPER[1]
+            args.extend(["-D", f"latex_elements.papersize={paper}paper"])
+        if self.config_rust:
+            args.extend(["-t", "rustdoc"])
+        if conf:
+            self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True)
+        if not sphinxdirs:
+            sphinxdirs = os.environ.get("SPHINXDIRS", ".")
+        sphinxdirs_list = []
+        for sphinxdir in sphinxdirs:
+            if isinstance(sphinxdir, list):
+                sphinxdirs_list += sphinxdir
+            else:
+                for name in sphinxdir.split(" "):
+                    sphinxdirs_list.append(name)
+        output_dirs = []
+        for sphinxdir in sphinxdirs_list:
+            src_dir = os.path.join(docs_dir, sphinxdir)
+            doctree_dir = os.path.join(self.builddir, ".doctrees")
+            output_dir = os.path.join(self.builddir, sphinxdir, out_dir)
+            src_dir = os.path.normpath(src_dir)
+            doctree_dir = os.path.normpath(doctree_dir)
+            output_dir = os.path.normpath(output_dir)
+            os.makedirs(doctree_dir, exist_ok=True)
+            os.makedirs(output_dir, exist_ok=True)
+            output_dirs.append(output_dir)
+            build_args = args + [
+                "-d", doctree_dir,
+                "-D", f"kerneldoc_bin={kerneldoc}",
+                "-D", f"version={self.kernelversion}",
+                "-D", f"release={self.kernelrelease}",
+                "-D", f"kerneldoc_srctree={self.srctree}",
+                src_dir,
+                output_dir,
+            ]
+            try:
+                self.run_sphinx(sphinxbuild, build_args, env=self.env)
+            except (OSError, ValueError, subprocess.SubprocessError) as e:
+                sys.exit(f"Build failed: {repr(e)}")
+            if target in ["htmldocs", "epubdocs"]:
+                self.handle_html(css, output_dir)
+        if target == "pdfdocs":
+            self.handle_pdf(output_dirs)
+        elif target == "infodocs":
+            self.handle_info(output_dirs)
+
+def jobs_type(value):
+    if value is None:
+        return None
+    if value.lower() == 'auto':
+        return value.lower()
+    try:
+        if int(value) >= 1:
+            return value
+        raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}")
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Kernel documentation builder")
+    parser.add_argument("target", choices=list(TARGETS.keys()),
+                        help="Documentation target to build")
+    parser.add_argument("--sphinxdirs", nargs="+",
+                        help="Specific directories to build")
+    parser.add_argument("--conf", default="conf.py",
+                        help="Sphinx configuration file")
+    parser.add_argument("--builddir", default="output",
+                        help="Sphinx configuration file")
+    parser.add_argument("--theme", help="Sphinx theme to use")
+    parser.add_argument("--css", help="Custom CSS file for HTML/EPUB")
+    parser.add_argument("--paper", choices=PAPER, default=PAPER[0],
+                        help="Paper size for LaTeX/PDF output")
+    parser.add_argument("-v", "--verbose", action='store_true',
+                        help="place build in verbose mode")
+    parser.add_argument('-j', '--jobs', type=jobs_type,
+                        help="Sets number of jobs to use with sphinx-build")
+    args = parser.parse_args()

tools/docs/sphinx-build-wrapper line 293

Quoted 8 lines

tools/docs/sphinx-build-wrapper new file +8 -0

+    PythonVersion.check_python(MIN_PYTHON_VERSION)
+    builder = SphinxBuilder(builddir=args.builddir,
+                            verbose=args.verbose, n_jobs=args.jobs)
+    builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf,
+                  theme=args.theme, css=args.css, paper=args.paper)
+
+if __name__ == "__main__":
+    main()  

Em Wed, 10 Sep 2025 13:46:17 +0300
Jani Nikula <[email protected]> escreveu:

Quoted 27 lines · Mauro Carvalho Chehab

Quoted 19 lines

There are too much magic inside docs Makefile to properly run sphinx-build. Create an ancillary script that contains all kernel-related sphinx-build call logic currently at Makefile.

Such script is designed to work both as an standalone command and as part of a Makefile. As such, it properly handles POSIX jobserver used by GNU make.

On a side note, there was a line number increase due to the conversion:

 Documentation/Makefile          |  131 +++----------
 tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
 2 files changed, 323 insertions(+), 101 deletions(-)

This is because some things are more verbosed on Python and because it requires reading env vars from Makefile. Besides it, this script has some extra features that don't exist at the Makefile:

Quoted 6 lines

- It can be called directly from command line;
- It properly return PDF build errors.

When running the script alone, it will only take handle sphinx-build targets. On other words, it won't runn make rustdoc after building htmlfiles, nor it will run the extra check scripts.

Quoted 6 lines

I've always strongly believed we should aim to make it possible to build the documentation by running sphinx-build directly on the command-line. Not that it would be the common way to run it, but to not accumulate things in the Makefile that need to happen before or after. To promote handling the documentation build in Sphinx. To be able to debug issues and try new Sphinx versions without all the hacks.

That would be the better, but, unfortunately, this is not possible, for several reasons:

1. SPHINXDIRS. It needs a lot of magic to work, both before running sphinx-build and after (inside conf.py);
2. Several extensions require kernel-specific environment variables to work. Calling sphinx-build directly breaks them;
3. Sphinx itself doesn't build several targets alone. Instead, they create a Makefile, and an extra step is needed to finish the build. That's the case for pdf and texinfo, for instance;
4. Man pages generation. Sphinx support to generate it is very poor;
5. Rust integration adds more complexity to the table;

I'm not seeing sphinx-build supporting the above needs anytime soon, and, even if we push our needs to Sphinx and it gets accepted there, we'll still need to wait for quite a while until LTS distros merge them.

Quoted 3 lines

This patch moves a bunch of that logic into a Python wrapper, and I feel like it complicates matters. You can no longer rely on 'make V=1' to get the build commands, for instance.

Quite the opposite. if you try using "make V=1", it won't show the command line used to call sphinx-build anymore.

This series restore it.

See, if you build with this series with V=1, you will see exactly what commands are used on the build:

	$ make V=1 htmldocs
	...
	python3 ./tools/docs/sphinx-build-wrapper htmldocs \
	        --sphinxdirs="." --conf="conf.py" \
        	--builddir="Documentation/output" \
	        --theme= --css= --paper=
	python3 /new_devel/docs/sphinx_latest/bin/sphinx-build -j25 -b html -c /new_devel/docs/Documentation -d /new_devel/docs/Documentation/output/.doctrees -D kerneldoc_bin=scripts/kernel-doc.py -D version=6.17.0-rc1 -D release=6.17.0-rc1+ -D kerneldoc_srctree=. /new_devel/docs/Documentation /new_devel/docs/Documentation/output
	...

Quoted 5 lines

Newer Sphinx versions have the -M option for "make mode". The Makefiles produced by sphinx-quickstart only have one build target:

# Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).

I didn't know about this, but from [1] it sounds it covers just two
targets: "latexpdf" and "info".

The most complex scenario is still not covered: SPHINXDIRS.

[1] https://www.sphinx-doc.org/en/master/man/sphinx-build.html

Quoted 4 lines

%: Makefile

        @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

That's all.

Try doing such change on your makefile. it will break:

- SPHINXDIRS;
- V=1;
- rustdoc

and will still be dependent on variables that are passed via env from Kernel makefile. So, stil you can't run from command line. Also, if you call sphinx-build from command line:

	$ sphinx-build -j25 -b html Documentation Documentation/output
	...
	      File "<frozen os>", line 717, in __getitem__
	    KeyError: 'srctree'

It won't work, as several parameters that are required by conf.py and by Sphinx extensions would be missing (the most important one is srctree, but there are others in the line too).

Quoted 2 lines

The proposed wrapper duplicates loads of code that's supposed to be handled by sphinx-build directly.

Once we get the wrapper, we can work to simplify it, but still I can't see how to get rid of it.

Quoted 1 line

Including the target/builder names.

True, but this was a design decision taken lots of years ago: instead of:
make html

we're using:

make htmldocs

This series doesn't change that: either makefile or the script need to tho the namespace conversion.

Quoted 483 lines

Seems to me the goal should be to figure out *generic* wrappers for handling parallelism, not Sphinx aware/specific.

BR,
Jani.

Quoted 476 lines

Signed-off-by Mauro Carvalho Chehab <[email protected]>

---
 Documentation/Makefile          | 131 ++++----------
 tools/docs/sphinx-build-wrapper | 293 ++++++++++++++++++++++++++++++++
 2 files changed, 323 insertions(+), 101 deletions(-)
 create mode 100755 tools/docs/sphinx-build-wrapper

Documentation/Makefile +30 -101

index deb2029228ed..4736f02b6c9e 100644
@@ -23,21 +23,22 @@ SPHINXOPTS    =
 SPHINXDIRS    = .
 DOCS_THEME    =
 DOCS_CSS      =
-_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
 SPHINX_CONF   = conf.py
 PAPER         =
 BUILDDIR      = $(obj)/output
 PDFLATEX      = xelatex
 LATEXOPTS     = -interaction=batchmode -no-shell-escape
 
+PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
+
+# Wrapper for sphinx-build
+
+BUILD_WRAPPER = $(srctree)/tools/docs/sphinx-build-wrapper
+
 # For denylisting "variable font" files
 # Can be overridden by setting as an env variable
 FONTS_CONF_DENY_VF ?= $(HOME)/deny-vf
 
-ifeq ($(findstring 1, $(KBUILD_VERBOSE)),)
-SPHINXOPTS    += "-q"
-endif
-
 # User-friendly check for sphinx-build
 HAVE_SPHINX := $(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi)
 

@@ -51,63 +52,31 @@ ifeq ($(HAVE_SPHINX),0)
 
 else # HAVE_SPHINX
 
-# User-friendly check for pdflatex and latexmk
-HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi)
-HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi)
+# Common documentation targets
+infodocs texinfodocs latexdocs epubdocs xmldocs pdfdocs linkcheckdocs:
+	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
+	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
+		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
+		--builddir="$(BUILDDIR)" \
+		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
 
-ifeq ($(HAVE_LATEXMK),1)
-	PDFLATEX := latexmk -$(PDFLATEX)
-endif #HAVE_LATEXMK
-
-# Internal variables.
-PAPEROPT_a4     = -D latex_elements.papersize=a4paper
-PAPEROPT_letter = -D latex_elements.papersize=letterpaper
-ALLSPHINXOPTS   = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC)
-ALLSPHINXOPTS   += $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)
-ifneq ($(wildcard $(srctree)/.config),)
-ifeq ($(CONFIG_RUST),y)
-	# Let Sphinx know we will include rustdoc
-	ALLSPHINXOPTS   +=  -t rustdoc
-endif
+# Special handling for pdfdocs
+ifeq ($(shell which $(PDFLATEX) >/dev/null 2>&1; echo $$?),0)
+pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
+else
+pdfdocs:
+	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
+	@echo "  SKIP    Sphinx $@ target."
 endif
-# the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-# commands; the 'cmd' from scripts/Kbuild.include is not *loopable*
-loop_cmd = $(echo-cmd) $(cmd_$(1)) || exit;
-
-# $2 sphinx builder e.g. "html"
-# $3 name of the build subfolder / e.g. "userspace-api/media", used as:
-#    * dest folder relative to $(BUILDDIR) and
-#    * cache folder relative to $(BUILDDIR)/.doctrees
-# $4 dest subfolder e.g. "man" for man pages at userspace-api/media/man
-# $5 reST source folder relative to $(src),
-#    e.g. "userspace-api/media" for the linux-tv book-set at ./Documentation/userspace-api/media
-
-PYTHONPYCACHEPREFIX ?= $(abspath $(BUILDDIR)/__pycache__)
-
-quiet_cmd_sphinx = SPHINX  $@ --> file://$(abspath $(BUILDDIR)/$3/$4)
-      cmd_sphinx = \
-	PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \
-	BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(src)/$5/$(SPHINX_CONF)) \
-	$(PYTHON3) $(srctree)/scripts/jobserver-exec \
-	$(CONFIG_SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \
-	$(SPHINXBUILD) \
-	-b $2 \
-	-c $(abspath $(src)) \
-	-d $(abspath $(BUILDDIR)/.doctrees/$3) \
-	-D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) \
-	$(ALLSPHINXOPTS) \
-	$(abspath $(src)/$5) \
-	$(abspath $(BUILDDIR)/$3/$4) && \
-	if [ "x$(DOCS_CSS)" != "x" ]; then \
-		cp $(if $(patsubst /%,,$(DOCS_CSS)),$(abspath $(srctree)/$(DOCS_CSS)),$(DOCS_CSS)) $(BUILDDIR)/$3/_static/; \
-	fi
 
+# HTML main logic is identical to other targets. However, if rust is enabled,
+# an extra step at the end is required to generate rustdoc.
 htmldocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,html,$(var),,$(var)))
-
+	$(Q)@$(srctree)/tools/docs/sphinx-pre-install --version-check
+	+$(Q)$(PYTHON3) $(BUILD_WRAPPER) $@ \
+		--sphinxdirs="$(SPHINXDIRS)" --conf="$(SPHINX_CONF)" \
+		--builddir="$(BUILDDIR)" \
+		--theme=$(DOCS_THEME) --css=$(DOCS_CSS) --paper=$(PAPER)
 # If Rust support is available and .config exists, add rustdoc generated contents.
 # If there are any, the errors from this make rustdoc will be displayed but
 # won't stop the execution of htmldocs

@@ -118,49 +87,6 @@ ifeq ($(CONFIG_RUST),y)
 endif
 endif
 
-texinfodocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,texinfo,$(var),texinfo,$(var)))
-
-# Note: the 'info' Make target is generated by sphinx itself when
-# running the texinfodocs target define above.
-infodocs: texinfodocs
-	$(MAKE) -C $(BUILDDIR)/texinfo info
-
-linkcheckdocs:
-	@$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,linkcheck,$(var),,$(var)))
-
-latexdocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,latex,$(var),latex,$(var)))
-
-ifeq ($(HAVE_PDFLATEX),0)
-
-pdfdocs:
-	$(warning The '$(PDFLATEX)' command was not found. Make sure you have it installed and in PATH to produce PDF output.)
-	@echo "  SKIP    Sphinx $@ target."
-
-else # HAVE_PDFLATEX
-
-pdfdocs: DENY_VF = XDG_CONFIG_HOME=$(FONTS_CONF_DENY_VF)
-pdfdocs: latexdocs
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	$(foreach var,$(SPHINXDIRS), \
-	   $(MAKE) PDFLATEX="$(PDFLATEX)" LATEXOPTS="$(LATEXOPTS)" $(DENY_VF) -C $(BUILDDIR)/$(var)/latex || sh $(srctree)/scripts/check-variable-fonts.sh || exit; \
-	   mkdir -p $(BUILDDIR)/$(var)/pdf; \
-	   mv $(subst .tex,.pdf,$(wildcard $(BUILDDIR)/$(var)/latex/*.tex)) $(BUILDDIR)/$(var)/pdf/; \
-	)
-
-endif # HAVE_PDFLATEX
-
-epubdocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,epub,$(var),epub,$(var)))
-
-xmldocs:
-	@$(srctree)/tools/docs/sphinx-pre-install --version-check
-	@+$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,xml,$(var),xml,$(var)))
-
 endif # HAVE_SPHINX
 
 # The following targets are independent of HAVE_SPHINX, and the rules should

@@ -172,6 +98,9 @@ refcheckdocs:
 cleandocs:
 	$(Q)rm -rf $(BUILDDIR)
 
+# Used only on help
+_SPHINXDIRS   = $(sort $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)))
+
 dochelp:
 	@echo  ' Linux kernel internal documentation in different formats from ReST:'
 	@echo  '  htmldocs        - HTML'

tools/docs/sphinx-build-wrapper new file +285 -0

new file mode 100755
index 000000000000..3256418d8dc5
@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+import argparse
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+from lib.python_version import PythonVersion
+
+LIB_DIR = "../../scripts/lib"
+SRC_DIR = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
+
+from jobserver import JobserverExec
+
+MIN_PYTHON_VERSION = PythonVersion("3.7").version
+PAPER = ["", "a4", "letter"]
+TARGETS = {
+    "cleandocs":     { "builder": "clean" },
+    "linkcheckdocs": { "builder": "linkcheck" },
+    "htmldocs":      { "builder": "html" },
+    "epubdocs":      { "builder": "epub",    "out_dir": "epub" },
+    "texinfodocs":   { "builder": "texinfo", "out_dir": "texinfo" },
+    "infodocs":      { "builder": "texinfo", "out_dir": "texinfo" },
+    "latexdocs":     { "builder": "latex",   "out_dir": "latex" },
+    "pdfdocs":       { "builder": "latex",   "out_dir": "latex" },
+    "xmldocs":       { "builder": "xml",     "out_dir": "xml" },
+}
+
+class SphinxBuilder:
+    def is_rust_enabled(self):
+        config_path = os.path.join(self.srctree, ".config")
+        if os.path.isfile(config_path):
+            with open(config_path, "r", encoding="utf-8") as f:
+                return "CONFIG_RUST=y" in f.read()
+        return False
+
+    def get_path(self, path, use_cwd=False, abs_path=False):
+        path = os.path.expanduser(path)
+        if not path.startswith("/"):
+            if use_cwd:
+                base = os.getcwd()
+            else:
+                base = self.srctree
+            path = os.path.join(base, path)
+        if abs_path:
+            return os.path.abspath(path)
+        return path
+
+    def __init__(self, builddir, verbose=False, n_jobs=None):
+        self.verbose = None
+        self.kernelversion = os.environ.get("KERNELVERSION", "unknown")
+        self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown")
+        self.pdflatex = os.environ.get("PDFLATEX", "xelatex")
+        self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape")
+        if not verbose:
+            verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "")
+        if verbose is not None:
+            self.verbose = verbose
+        parser = argparse.ArgumentParser()
+        parser.add_argument('-j', '--jobs', type=int)
+        parser.add_argument('-q', '--quiet', type=int)
+        sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", ""))
+        sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts)
+        if sphinx_args.quiet is True:
+            self.verbose = False
+        if sphinx_args.jobs:
+            self.n_jobs = sphinx_args.jobs
+        self.n_jobs = n_jobs
+        self.srctree = os.environ.get("srctree")
+        if not self.srctree:
+            self.srctree = "."
+            os.environ["srctree"] = self.srctree
+        self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build")
+        self.kerneldoc = self.get_path(os.environ.get("KERNELDOC",
+                                                      "scripts/kernel-doc.py"))
+        self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True)
+
+        self.config_rust = self.is_rust_enabled()
+
+        self.pdflatex_cmd = shutil.which(self.pdflatex)
+        self.latexmk_cmd = shutil.which("latexmk")
+
+        self.env = os.environ.copy()
+
+    def run_sphinx(self, sphinx_build, build_args, *args, **pwargs):
+        with JobserverExec() as jobserver:
+            if jobserver.claim:
+                n_jobs = str(jobserver.claim)
+            else:
+                n_jobs = "auto" # Supported since Sphinx 1.7
+            cmd = []
+            cmd.append(sys.executable)
+            cmd.append(sphinx_build)
+            if self.n_jobs:
+                n_jobs = str(self.n_jobs)
+
+            if n_jobs:
+                cmd += [f"-j{n_jobs}"]
+
+            if not self.verbose:
+                cmd.append("-q")
+            cmd += self.sphinxopts
+            cmd += build_args
+            if self.verbose:
+                print(" ".join(cmd))
+            return subprocess.call(cmd, *args, **pwargs)
+
+    def handle_html(self, css, output_dir):
+        if not css:
+            return
+        css = os.path.expanduser(css)
+        if not css.startswith("/"):
+            css = os.path.join(self.srctree, css)
+        static_dir = os.path.join(output_dir, "_static")
+        os.makedirs(static_dir, exist_ok=True)
+        try:
+            shutil.copy2(css, static_dir)
+        except (OSError, IOError) as e:
+            print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr)
+
+    def handle_pdf(self, output_dirs):
+        builds = {}
+        max_len = 0
+        for from_dir in output_dirs:
+            pdf_dir = os.path.join(from_dir, "../pdf")
+            os.makedirs(pdf_dir, exist_ok=True)
+            if self.latexmk_cmd:
+                latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"]
+            else:
+                latex_cmd = [self.pdflatex]
+            latex_cmd.extend(shlex.split(self.latexopts))
+            tex_suffix = ".tex"
+            has_tex = False
+            build_failed = False
+            with os.scandir(from_dir) as it:
+                for entry in it:
+                    if not entry.name.endswith(tex_suffix):
+                        continue
+                    name = entry.name[:-len(tex_suffix)]
+                    has_tex = True
+                    try:
+                        subprocess.run(latex_cmd + [entry.path],
+                                       cwd=from_dir, check=True)
+                    except subprocess.CalledProcessError:
+                        pass
+                    pdf_name = name + ".pdf"
+                    pdf_from = os.path.join(from_dir, pdf_name)
+                    pdf_to = os.path.join(pdf_dir, pdf_name)
+                    if os.path.exists(pdf_from):
+                        os.rename(pdf_from, pdf_to)
+                        builds[name] = os.path.relpath(pdf_to, self.builddir)
+                    else:
+                        builds[name] = "FAILED"
+                        build_failed = True
+                    name = entry.name.removesuffix(".tex")
+                    max_len = max(max_len, len(name))
+
+            if not has_tex:
+                name = os.path.basename(from_dir)
+                max_len = max(max_len, len(name))
+                builds[name] = "FAILED (no .tex)"
+                build_failed = True
+        msg = "Summary"
+        msg += "\n" + "=" * len(msg)
+        print()
+        print(msg)
+        for pdf_name, pdf_file in builds.items():
+            print(f"{pdf_name:<{max_len}}: {pdf_file}")
+        print()
+        if build_failed:
+            sys.exit("PDF build failed: not all PDF files were created.")
+        else:
+            print("All PDF files were built.")
+
+    def handle_info(self, output_dirs):
+        for output_dir in output_dirs:
+            try:
+                subprocess.run(["make", "info"], cwd=output_dir, check=True)
+            except subprocess.CalledProcessError as e:
+                sys.exit(f"Error generating info docs: {e}")
+
+    def cleandocs(self, builder):
+        shutil.rmtree(self.builddir, ignore_errors=True)
+
+    def build(self, target, sphinxdirs=None, conf="conf.py",
+              theme=None, css=None, paper=None):
+        builder = TARGETS[target]["builder"]
+        out_dir = TARGETS[target].get("out_dir", "")
+        if target == "cleandocs":
+            self.cleandocs(builder)
+            return
+        if theme:
+                os.environ["DOCS_THEME"] = theme
+        sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"])
+        if not sphinxbuild:
+            sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n")
+        if builder == "latex":
+            if not self.pdflatex_cmd and not self.latexmk_cmd:
+                sys.exit("Error: pdflatex or latexmk required for PDF generation")
+        docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation"))
+        kerneldoc = self.kerneldoc
+        if kerneldoc.startswith(self.srctree):
+            kerneldoc = os.path.relpath(kerneldoc, self.srctree)
+        args = [ "-b", builder, "-c", docs_dir ]
+        if builder == "latex":
+            if not paper:
+                paper = PAPER[1]
+            args.extend(["-D", f"latex_elements.papersize={paper}paper"])
+        if self.config_rust:
+            args.extend(["-t", "rustdoc"])
+        if conf:
+            self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True)
+        if not sphinxdirs:
+            sphinxdirs = os.environ.get("SPHINXDIRS", ".")
+        sphinxdirs_list = []
+        for sphinxdir in sphinxdirs:
+            if isinstance(sphinxdir, list):
+                sphinxdirs_list += sphinxdir
+            else:
+                for name in sphinxdir.split(" "):
+                    sphinxdirs_list.append(name)
+        output_dirs = []
+        for sphinxdir in sphinxdirs_list:
+            src_dir = os.path.join(docs_dir, sphinxdir)
+            doctree_dir = os.path.join(self.builddir, ".doctrees")
+            output_dir = os.path.join(self.builddir, sphinxdir, out_dir)
+            src_dir = os.path.normpath(src_dir)
+            doctree_dir = os.path.normpath(doctree_dir)
+            output_dir = os.path.normpath(output_dir)
+            os.makedirs(doctree_dir, exist_ok=True)
+            os.makedirs(output_dir, exist_ok=True)
+            output_dirs.append(output_dir)
+            build_args = args + [
+                "-d", doctree_dir,
+                "-D", f"kerneldoc_bin={kerneldoc}",
+                "-D", f"version={self.kernelversion}",
+                "-D", f"release={self.kernelrelease}",
+                "-D", f"kerneldoc_srctree={self.srctree}",
+                src_dir,
+                output_dir,
+            ]
+            try:
+                self.run_sphinx(sphinxbuild, build_args, env=self.env)
+            except (OSError, ValueError, subprocess.SubprocessError) as e:
+                sys.exit(f"Build failed: {repr(e)}")
+            if target in ["htmldocs", "epubdocs"]:
+                self.handle_html(css, output_dir)
+        if target == "pdfdocs":
+            self.handle_pdf(output_dirs)
+        elif target == "infodocs":
+            self.handle_info(output_dirs)
+
+def jobs_type(value):
+    if value is None:
+        return None
+    if value.lower() == 'auto':
+        return value.lower()
+    try:
+        if int(value) >= 1:
+            return value
+        raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}")
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Kernel documentation builder")
+    parser.add_argument("target", choices=list(TARGETS.keys()),
+                        help="Documentation target to build")
+    parser.add_argument("--sphinxdirs", nargs="+",
+                        help="Specific directories to build")
+    parser.add_argument("--conf", default="conf.py",
+                        help="Sphinx configuration file")
+    parser.add_argument("--builddir", default="output",
+                        help="Sphinx configuration file")
+    parser.add_argument("--theme", help="Sphinx theme to use")
+    parser.add_argument("--css", help="Custom CSS file for HTML/EPUB")
+    parser.add_argument("--paper", choices=PAPER, default=PAPER[0],
+                        help="Paper size for LaTeX/PDF output")
+    parser.add_argument("-v", "--verbose", action='store_true',
+                        help="place build in verbose mode")
+    parser.add_argument('-j', '--jobs', type=jobs_type,
+                        help="Sets number of jobs to use with sphinx-build")
+    args = parser.parse_args()

Quoted 9 lines

tools/docs/sphinx-build-wrapper line 293

Quoted 8 lines

tools/docs/sphinx-build-wrapper new file +8 -0

+    PythonVersion.check_python(MIN_PYTHON_VERSION)
+    builder = SphinxBuilder(builddir=args.builddir,
+                            verbose=args.verbose, n_jobs=args.jobs)
+    builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf,
+                  theme=args.theme, css=args.css, paper=args.paper)
+
+if __name__ == "__main__":
+    main()    

Thanks,
Mauro

The current approach of using LaTeX for PDF is dirty:

- Sphinx can't produce a LaTeX file from the Kernel trees without hundreds of warnings;
- latexmk hides some of them, but even it just one warning is reported, the return status is not zero.

[-CC: rust people and list]

OK, Looks like I have to bite.

On Wed, 10 Sep 2025 15:33:34 +0200, Mauro Carvalho Chehab wrote:
[...]

Quoted 2 lines

The current approach of using LaTeX for PDF is dirty:

Quoted 4 lines

- Sphinx can't produce a LaTeX file from the Kernel trees without hundreds of warnings;
- latexmk hides some of them, but even it just one warning is reported, the return status is not zero.

This sounds interesting to me.
As far I remember, I have never seen such strange results of latexmk under build envs where all the necessary packages are properly installed.

I think what you are trying here is to paper over whatever bug in latexmk/ xelatex by always ignoring their exit status. Am I guessing right?

If that is the case, I'd rather report such an issue at upstream lagtexmk/xelatex.

Em Fri, 12 Sep 2025 19:15:44 +0900
Akira Yokosawa <[email protected]> escreveu:

Quoted 14 lines

[-CC: rust people and list]

OK, Looks like I have to bite.

On Wed, 10 Sep 2025 15:33:34 +0200, Mauro Carvalho Chehab wrote:
[...]

Quoted 6 lines

The current approach of using LaTeX for PDF is dirty:

- Sphinx can't produce a LaTeX file from the Kernel trees without hundreds of warnings;
- latexmk hides some of them, but even it just one warning is reported, the return status is not zero.

Quoted 3 lines

This sounds interesting to me.
As far I remember, I have never seen such strange results of latexmk under build envs where all the necessary packages are properly installed.

I saw it here on multiple distros including Fedora (which is the one I use on my desktop). Media jenkins CI running on Debian always suffered from such issues, up to the point I started ignoring pdf build results.

Quoted 21 lines · Mauro Carvalho Chehab

Em Fri, 12 Sep 2025 19:15:44 +0900
Akira Yokosawa <[email protected]> escreveu:

Quoted 17 lines

[-CC: rust people and list]

OK, Looks like I have to bite.

On Wed, 10 Sep 2025 15:33:34 +0200, Mauro Carvalho Chehab wrote:
[...]

Quoted 6 lines

The current approach of using LaTeX for PDF is dirty:

- Sphinx can't produce a LaTeX file from the Kernel trees without hundreds of warnings;
- latexmk hides some of them, but even it just one warning is reported, the return status is not zero.

This sounds interesting to me.
As far I remember, I have never seen such strange results of latexmk under build envs where all the necessary packages are properly installed.

Quoted 4 lines

I saw it here on multiple distros including Fedora (which is the one I use on my desktop). Media jenkins CI running on Debian always suffered from such issues, up to the point I started ignoring pdf build results.

So please provide exact steps for me to see such errors.

I don't have any issues after strictly following the suggestions from sphinx-pre-install under Fedora.

I even invoked [...]/output/latex/Makefile manually after running "make latexdocs" by:

- cd [...]/output/latex/
- make PDFLATEX="latexmk -xelatex" LATEXOPTS="-interaction=batchmode -no-shell-escape" -j6 -O all

, and all the PDFs were built without any issues.

Quite puzzling ...

Or does your Fedora have some Noto CJK variable fonts installed?

Quoted 1 line · Akira Yokosawa

On Fri, 12 Sep 2025 13:04:20 +0200, Mauro Carvalho Chehab wrote:

Quoted 6 lines

Quoted 4 lines

I saw it here on multiple distros including Fedora (which is the one I use on my desktop). Media jenkins CI running on Debian always suffered from such issues, up to the point I started ignoring pdf build results.

So please provide exact steps for me to see such errors.

Sorry, but I don't have enough time to try reproducing it again (plus, I'm ran out of disk space on my /var partition forcing me to reclaim the space used by my test containers).

Quoted 13 lines

I don't have any issues after strictly following the suggestions from sphinx-pre-install under Fedora.

I even invoked [...]/output/latex/Makefile manually after running "make latexdocs" by:

- cd [...]/output/latex/
- make PDFLATEX="latexmk -xelatex" LATEXOPTS="-interaction=batchmode -no-shell-escape" -j6 -O all

, and all the PDFs were built without any issues.

Quoted 3 lines

Quite puzzling ...

Or does your Fedora have some Noto CJK variable fonts installed?

On my main desktop, yes, that's the case: it currently has some Noto CJK fonts installed. I wasn't aware about such issues with Fedora until today, when I noticed your check script.

In any case, this changeset fix it on several ways:

- A failure while building one pdf doesn't prevent building other files. With make, it may stop before building them all (if we drop the "|| exit");

- It prints a summary reporting what PDF files were actually built, so it is easy for the developer to know what broke (and it is also easily parsed by a CI);
- The return code is zero only if all docs were built.

Quoted 2 lines · Akira Yokosawa

Quoted 1 line · Mauro Carvalho Chehab

[...]

Quoted 10 lines

Quoted 6 lines

Quoted 4 lines

I saw it here on multiple distros including Fedora (which is the one I use on my desktop). Media jenkins CI running on Debian always suffered from such issues, up to the point I started ignoring pdf build results.

So please provide exact steps for me to see such errors.

Sorry, but I don't have enough time to try reproducing it again (plus, I'm ran out of disk space on my /var partition forcing me to reclaim the space used by my test containers).

There is no urgency on my side. Please take your time.

Quoted 13 lines

Quoted 13 lines

I don't have any issues after strictly following the suggestions from sphinx-pre-install under Fedora.

I even invoked [...]/output/latex/Makefile manually after running "make latexdocs" by:

- cd [...]/output/latex/
- make PDFLATEX="latexmk -xelatex" LATEXOPTS="-interaction=batchmode -no-shell-escape" -j6 -O all

, and all the PDFs were built without any issues.

Quoted 8 lines

Quoted 3 lines

Quite puzzling ...

Or does your Fedora have some Noto CJK variable fonts installed?

On my main desktop, yes, that's the case: it currently has some Noto CJK fonts installed. I wasn't aware about such issues with Fedora until today, when I noticed your check script.

Good. That should be a step forward, I guess.

I know you have repeatedly said it is not the purpose of this series to fix those issues in your images, but I have an impression that it is closely related to testing sphinx-pre-install, and the objective of this series is to make the testing/debugging of sphinx-pre-install easier for you.

But, at least for me, the behavior you want for "pdfdocs" is not ideal for regular testing of .rst and kernel-doc changes in kernel source side. I want "make pdfdocs" to give up earlier rather than later. It should leave relevant info near the bottom of terminal log.

Now, here are my responses to your arguments:

Quoted 5 lines

In any case, this changeset fix it on several ways:

- A failure while building one pdf doesn't prevent building other files. With make, it may stop before building them all (if we drop the "|| exit");

Didn't you mean "(if we keep the "|| exit"); ???

If you drop the "|| exit", which will cause false-negatives. And you are going to test every PDFs at the final stage of pdfdocs to catch such false-positives.

Sounds like a plan.

Quoted 4 lines

- It prints a summary reporting what PDF files were actually built, so it is easy for the developer to know what broke (and it is also easily parsed by a CI);
- The return code is zero only if all docs were built.

I agree this is an improvement, but if we get rid of the loop construct in the Makefile, we can forget about said false-negatives, can't we?

How about something like the following approach?

Let's think of SPHINXDIRS="admin-guide core-api driver-api userspace-api"

In this case "make latexdocs" will generate

output/admin-guide/latex/
output/core-api/latex/
output/driver-api/latex/
output/userspace-api/latex/

They will have identical boiler-plate files latex builder would emit, and subdir-specific files such as:

    output/admin-guide/latex/admin-guide.tex
                             c3-isp.dot
                             c3-isp.pdf
                             c3-isp.svg
                             conn-states-8.dot
                             conn-states-8.pdf
                             conn-states-8.svg
                             disk-states-8.dot
                             disk-states-8.pdf
                             disk-states-8.svg
                             ...
    output/core-api/latex/core-api.tex
    output/driver-api/latex/driver-api.tex
                            DOT-1e98886fceca6e25a115532f5efebb44c09dc98b.dot
                            DOT-1e98886fceca6e25a115532f5efebb44c09dc98b.pdf
                            DOT-1e98886fceca6e25a115532f5efebb44c09dc98b.svg
                            DOT-289c17ebc0291f90ccaf431961707504464a78d4.dot
                            DOT-289c17ebc0291f90ccaf431961707504464a78d4.pdf
                            DOT-289c17ebc0291f90ccaf431961707504464a78d4.svg
                            ...
    output/userspace-api/latex/userspace-api.tex
                               DOT-1e98886fceca6e25a115532f5efebb44c09dc98b.dot
                               DOT-1e98886fceca6e25a115532f5efebb44c09dc98b.pdf
                               DOT-1e98886fceca6e25a115532f5efebb44c09dc98b.svg
                               DOT-289c17ebc0291f90ccaf431961707504464a78d4.dot
                               DOT-289c17ebc0291f90ccaf431961707504464a78d4.pdf
                               DOT-289c17ebc0291f90ccaf431961707504464a78d4.svg
                               ...

At a pre-processing stage of pdfdocs, you create output/latex/ and symlink subdir-specific files needed for latexmk/xelatex into there. (Copying them should work too.)

    output/latex/admin-guide.tex --> ../admin-guide/latex/
                 c3-isp.pdf      --> ../admin-guide/latex/
                 conn-states-8.pdf --> ../admin-guide/latex/
                 disk-states-8.pdf --> ../admin-guide/latex/
                 ...
                 core-api.tex --> ../core-api/latex/
                 driver-api.tex --> ../driver-api/latex/
                 DOT-1e98886fceca6e25a115532f5efebb44c09dc98b.pdf --> ../driver-api/latex/
                 DOT-289c17ebc0291f90ccaf431961707504464a78d4.pdf --> ../driver-api/latex/
                 ...
                 userspace-api.tex --> ../userspace-api/
                 DOT-1e98886fceca6e25a115532f5efebb44c09dc98b.pdf --> ../userspace-api/latex/
                 DOT-289c17ebc0291f90ccaf431961707504464a78d4.pdf --> ../userspace-api/latex/
                 ...

The latexmk stage would now be identical to the SPHINXDIRS="." case, meaning it won't need the loop in the recipe. Furthermore, post-processing would be almost the same as the default case.

As a bonus, "make -j -O" will work as the same as full pdfdocs build.

If you are interested, I think I can prepare a PoC patch, probably after v6.18-rc1.

If your sphinx-build-wrapper's latexdocs stage can be adjusted so that said pre-processing of pdfdocs can be made unnecessary, that would be even better.

IMHO, long term solution is to change SPHINXDIRS into something like:

make O=doc_build SPHINXTITLE="Media docs" SPHINXDIRS="admin-guide/media userspace-api/media driver-api/media/"

would create something similar to this(*):

	doc_build/sphindirs/
		|
		+--> index.rst
		+--> admin-guide -> {srcdir}/Documentation/admin-guide/media/
		+--> usespace-api -> {srcdir}/Documentation/admin-guide/media/
		\--> driver-api -> {srcdir}/Documentation/admin-guide/media/

Quoted 5 lines · Mauro Carvalho Chehab

IMHO, long term solution is to change SPHINXDIRS into something like:

make O=doc_build SPHINXTITLE="Media docs" SPHINXDIRS="admin-guide/media userspace-api/media driver-api/media/"

Quoted 8 lines

would create something similar to this(*):

	doc_build/sphindirs/
		|
		+--> index.rst
		+--> admin-guide -> {srcdir}/Documentation/admin-guide/media/
		+--> usespace-api -> {srcdir}/Documentation/admin-guide/media/
		\--> driver-api -> {srcdir}/Documentation/admin-guide/media/

So you're basically suggesting the documentation build should support cherry-picking parts of the documentation with categories different from what the upstream documentation has?

I.e. even if we figured out how to
do intersphinx books, you'd want to grab parts from them and turn them into something else?

Ugh.

BR,
Jani.

Signature · 1 line

Jani Nikula, Intel

Quoted 15 lines · Jani Nikula

Quoted 5 lines · Mauro Carvalho Chehab

IMHO, long term solution is to change SPHINXDIRS into something like:

make O=doc_build SPHINXTITLE="Media docs" SPHINXDIRS="admin-guide/media userspace-api/media driver-api/media/"

Quoted 8 lines

would create something similar to this(*):

	doc_build/sphindirs/
		|
		+--> index.rst
		+--> admin-guide -> {srcdir}/Documentation/admin-guide/media/
		+--> usespace-api -> {srcdir}/Documentation/admin-guide/media/
		\--> driver-api -> {srcdir}/Documentation/admin-guide/media/

Quoted 3 lines

So you're basically suggesting the documentation build should support cherry-picking parts of the documentation with categories different from what the upstream documentation has?

No. I'm saying that, if we want to have a single build process for multiple sphinxdirs, that sounds to be the better way to do it to override sphinx-build limitation of having single source directory.

The advantages is that:
- brings more performance, as a single build would be enough;
- cross-references between them will be properly solved.

The disadvantages are:
- it would very likely need to create copies (or hard symlinks) at the build dir, which may reduce performance;
- yet-another-hack;
- increased build complexity.

I'm not convinced myself about doing it or not. I didn't like when I had to do that after the media book was split on 3 books. If one thinks that having for loops to build targets is a problem, we need a separate discussion about how to avoid it. Also, this is outside of the scope of this series.

-

Another alternative to achieve such goal of not needing a loop at Sphinx to handle multiple books in parallel would be to submit a patch for Sphinx to get rid of the current limitation of having a single book with everything on a single directory. Sphinx has already hacks for it with "latex_documents", "man_pages", "texinfo_documents" conf.py variables that are specific for non-html builders.

Still, when such variables are used, a post-sphinx-build logic with a per-output-file loop is needed.

Quoted 3 lines

I.e. even if we figured out how to
do intersphinx books, you'd want to grab parts from them and turn them into something else?

Either doing it or not, intersphinx is intestesting.

Quoted 9 lines

Ugh.

BR,
Jani.

Signature · 1 line

Jani Nikula, Intel

Quoted 34 lines · Mauro Carvalho Chehab

Quoted 15 lines · Jani Nikula

Quoted 5 lines · Mauro Carvalho Chehab

IMHO, long term solution is to change SPHINXDIRS into something like:

make O=doc_build SPHINXTITLE="Media docs" SPHINXDIRS="admin-guide/media userspace-api/media driver-api/media/"

Quoted 8 lines

would create something similar to this(*):

	doc_build/sphindirs/
		|
		+--> index.rst
		+--> admin-guide -> {srcdir}/Documentation/admin-guide/media/
		+--> usespace-api -> {srcdir}/Documentation/admin-guide/media/
		\--> driver-api -> {srcdir}/Documentation/admin-guide/media/

Quoted 3 lines

So you're basically suggesting the documentation build should support cherry-picking parts of the documentation with categories different from what the upstream documentation has?

No. I'm saying that, if we want to have a single build process for multiple sphinxdirs, that sounds to be the better way to do it to override sphinx-build limitation of having single source directory.

The advantages is that:
- brings more performance, as a single build would be enough;
- cross-references between them will be properly solved.

The disadvantages are:
- it would very likely need to create copies (or hard symlinks) at the build dir, which may reduce performance;
- yet-another-hack;
- increased build complexity.

Quoted 5 lines

I'm not convinced myself about doing it or not. I didn't like when I had to do that after the media book was split on 3 books. If one thinks that having for loops to build targets is a problem, we need a separate discussion about how to avoid it. Also, this is outside of the scope of this series.

I honestly don't even understand what you're saying above

and how it
contradicts with what I said about cherry-picking the documentation to build.

Em Wed, 10 Sep 2025 13:46:17 +0300
Jani Nikula <[email protected]> escreveu:

Quoted 27 lines · Mauro Carvalho Chehab

Quoted 19 lines

There are too much magic inside docs Makefile to properly run sphinx-build. Create an ancillary script that contains all kernel-related sphinx-build call logic currently at Makefile.

Such script is designed to work both as an standalone command and as part of a Makefile. As such, it properly handles POSIX jobserver used by GNU make.

On a side note, there was a line number increase due to the conversion:

 Documentation/Makefile          |  131 +++----------
 tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
 2 files changed, 323 insertions(+), 101 deletions(-)

This is because some things are more verbosed on Python and because it requires reading env vars from Makefile. Besides it, this script has some extra features that don't exist at the Makefile:

Quoted 6 lines

- It can be called directly from command line;
- It properly return PDF build errors.

When running the script alone, it will only take handle sphinx-build targets. On other words, it won't runn make rustdoc after building htmlfiles, nor it will run the extra check scripts.

Quoted 6 lines

I've always strongly believed we should aim to make it possible to build the documentation by running sphinx-build directly on the command-line. Not that it would be the common way to run it, but to not accumulate things in the Makefile that need to happen before or after. To promote handling the documentation build in Sphinx. To be able to debug issues and try new Sphinx versions without all the hacks.

That would be the better, but, unfortunately, this is not possible, for several reasons:

1. SPHINXDIRS. It needs a lot of magic to work, both before running sphinx-build and after (inside conf.py);

2. Several extensions require kernel-specific environment variables to work. Calling sphinx-build directly breaks them;

3. Sphinx itself doesn't build several targets alone. Instead, they create a Makefile, and an extra step is needed to finish the build. That's the case for pdf and texinfo, for instance;

4. Man pages generation. Sphinx support to generate it is very poor;

5. Rust integration adds more complexity to the table;

I'm not seeing sphinx-build supporting the above needs anytime soon, and, even if we push our needs to Sphinx and it gets accepted there, we'll still need to wait for quite a while until LTS distros merge them.

Quoted 3 lines

This patch moves a bunch of that logic into a Python wrapper, and I feel like it complicates matters. You can no longer rely on 'make V=1' to get the build commands, for instance.

Quite the opposite. if you try using "make V=1", it won't show the command line used to call sphinx-build anymore.

This series restore it.

See, if you build with this series with V=1, you will see exactly what commands are used on the build:

	$ make V=1 htmldocs
	...
	python3 ./tools/docs/sphinx-build-wrapper htmldocs \
	        --sphinxdirs="." --conf="conf.py" \
        	--builddir="Documentation/output" \
	        --theme= --css= --paper=
	python3 /new_devel/docs/sphinx_latest/bin/sphinx-build -j25 -b html -c /new_devel/docs/Documentation -d /new_devel/docs/Documentation/output/.doctrees -D kerneldoc_bin=scripts/kernel-doc.py -D version=6.17.0-rc1 -D release=6.17.0-rc1+ -D kerneldoc_srctree=. /new_devel/docs/Documentation /new_devel/docs/Documentation/output
	...

Quoted 5 lines

Newer Sphinx versions have the -M option for "make mode". The Makefiles produced by sphinx-quickstart only have one build target:

# Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).

I didn't know about this, but from [1] it sounds it covers just two
targets: "latexpdf" and "info".

The most complex scenario is still not covered: SPHINXDIRS.

[1] https://www.sphinx-doc.org/en/master/man/sphinx-build.html

Quoted 3 lines

%: Makefile

        @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

Quoted 1 line

That's all.

Try doing such change on your makefile. it will break:

- SPHINXDIRS;
- V=1;
- rustdoc

and will still be dependent on variables that are passed via env from Kernel makefile. So, stil you can't run from command line. Also, if you call sphinx-build from command line:

	$ sphinx-build -j25 -b html Documentation Documentation/output
	...
	      File "<frozen os>", line 717, in __getitem__
	    KeyError: 'srctree'

It won't work, as several parameters that are required by conf.py and by Sphinx extensions would be missing (the most important one is srctree, but there are others in the line too).

Quoted 2 lines

The proposed wrapper duplicates loads of code that's supposed to be handled by sphinx-build directly.

Once we get the wrapper, we can work to simplify it, but still I can't see how to get rid of it.

Quoted 1 line

Including the target/builder names.

True, but this was a design decision taken lots of years ago: instead of:
make html

we're using:

make htmldocs

This series doesn't change that: either makefile or the script need to tho the namespace conversion.

Quoted 6 lines

Seems to me the goal should be to figure out *generic* wrappers for handling parallelism, not Sphinx aware/specific.

BR,
Jani.

Quoted 37 lines · Mauro Carvalho Chehab

Em Wed, 10 Sep 2025 13:46:17 +0300
Jani Nikula <[email protected]> escreveu:

Quoted 33 lines · Mauro Carvalho Chehab

Quoted 19 lines

There are too much magic inside docs Makefile to properly run sphinx-build. Create an ancillary script that contains all kernel-related sphinx-build call logic currently at Makefile.

Such script is designed to work both as an standalone command and as part of a Makefile. As such, it properly handles POSIX jobserver used by GNU make.

On a side note, there was a line number increase due to the conversion:

 Documentation/Makefile          |  131 +++----------
 tools/docs/sphinx-build-wrapper |  293 +++++++++++++++++++++++++++++++
 2 files changed, 323 insertions(+), 101 deletions(-)

This is because some things are more verbosed on Python and because it requires reading env vars from Makefile. Besides it, this script has some extra features that don't exist at the Makefile:

Quoted 6 lines

- It can be called directly from command line;
- It properly return PDF build errors.

When running the script alone, it will only take handle sphinx-build targets. On other words, it won't runn make rustdoc after building htmlfiles, nor it will run the extra check scripts.

I've always strongly believed we should aim to make it possible to build the documentation by running sphinx-build directly on the command-line. Not that it would be the common way to run it, but to not accumulate things in the Makefile that need to happen before or after. To promote handling the documentation build in Sphinx. To be able to debug issues and try new Sphinx versions without all the hacks.

Quoted 5 lines

That would be the better, but, unfortunately, this is not possible, for several reasons:

1. SPHINXDIRS. It needs a lot of magic to work, both before running sphinx-build and after (inside conf.py);

Makes you wonder if that's the right solution to the original problem. It was added as a kind of hack, and it stuck.

Quoted 3 lines

2. Several extensions require kernel-specific environment variables to work. Calling sphinx-build directly breaks them;

The extensions shouldn't be using environment variables for configuration anyway. Add config options and set them in conf.py like everything else?

Quoted 3 lines

3. Sphinx itself doesn't build several targets alone. Instead, they create a Makefile, and an extra step is needed to finish the build. That's the case for pdf and texinfo, for instance;

That's not true for the Makefile currently generated by sphinx-quickstart. Granted, I haven't used Sphinx much for pdf output.

Quoted 2 lines

4. Man pages generation. Sphinx support to generate it is very poor;

In what way?

Quoted 6 lines

5. Rust integration adds more complexity to the table;

I'm not seeing sphinx-build supporting the above needs anytime soon, and, even if we push our needs to Sphinx and it gets accepted there, we'll still need to wait for quite a while until LTS distros merge them.

I'm not suggesting to add anything to Sphinx upstream.

Quoted 26 lines

Quoted 3 lines

This patch moves a bunch of that logic into a Python wrapper, and I feel like it complicates matters. You can no longer rely on 'make V=1' to get the build commands, for instance.

Quite the opposite. if you try using "make V=1", it won't show the command line used to call sphinx-build anymore.

This series restore it.

See, if you build with this series with V=1, you will see exactly what commands are used on the build:

	$ make V=1 htmldocs
	...
	python3 ./tools/docs/sphinx-build-wrapper htmldocs \
	        --sphinxdirs="." --conf="conf.py" \
        	--builddir="Documentation/output" \
	        --theme= --css= --paper=
	python3 /new_devel/docs/sphinx_latest/bin/sphinx-build -j25 -b html -c /new_devel/docs/Documentation -d /new_devel/docs/Documentation/output/.doctrees -D kerneldoc_bin=scripts/kernel-doc.py -D version=6.17.0-rc1 -D release=6.17.0-rc1+ -D kerneldoc_srctree=. /new_devel/docs/Documentation /new_devel/docs/Documentation/output
	...

Quoted 3 lines

Newer Sphinx versions have the -M option for "make mode". The Makefiles produced by sphinx-quickstart only have one build target:

Quoted 5 lines

Quoted 2 lines

# Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).

I didn't know about this, but from [1] it sounds it covers just two
targets: "latexpdf" and "info".

sphinx-build -M help gives a list of 24 targets.

Quoted 9 lines

The most complex scenario is still not covered: SPHINXDIRS.

[1] https://www.sphinx-doc.org/en/master/man/sphinx-build.html

Quoted 4 lines

%: Makefile

        @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

That's all.

Quoted 5 lines

Try doing such change on your makefile. it will break:

- SPHINXDIRS;
- V=1;
- rustdoc

I know it does. That's the problem.

Quoted 14 lines

and will still be dependent on variables that are passed via env from Kernel makefile. So, stil you can't run from command line. Also, if you call sphinx-build from command line:

	$ sphinx-build -j25 -b html Documentation Documentation/output
	...
	      File "<frozen os>", line 717, in __getitem__
	    KeyError: 'srctree'

It won't work, as several parameters that are required by conf.py and by Sphinx extensions would be missing (the most important one is srctree, but there are others in the line too).

Quoted 5 lines

Quoted 2 lines

The proposed wrapper duplicates loads of code that's supposed to be handled by sphinx-build directly.

Once we get the wrapper, we can work to simplify it, but still I can't see how to get rid of it.

I just don't understand the mentality of first adding something complex, and then working to simplify it.

Don't make it a Rube Goldberg machine in the first place.

Quoted 6 lines

Quoted 1 line

Including the target/builder names.

True, but this was a design decision taken lots of years ago: instead of:
make html

Quoted 6 lines

we're using:

make htmldocs

This series doesn't change that: either makefile or the script need to tho the namespace conversion.

In the above Makefile snippet that conversion would be $(@:docs=)

The clean Makefile way of checking for having Sphinx and the required versions of Python and dependencies etc. would be a .PHONY target that just checks, and doesn't do *anything* else. It shouldn't be part of the sphinx-build rules.

PHONY += check-versions
check-versions:

	sphinx-pre-install --version-check

htmldocs: check-versions

	...

Or something like that.

Quoted 5 lines · Jani Nikula

Quoted 2 lines

1. SPHINXDIRS. It needs a lot of magic to work, both before running sphinx-build and after (inside conf.py);

Makes you wonder if that's the right solution to the original problem. It was added as a kind of hack, and it stuck.

The problem is, IMHO, due to the lack of flexibility of sphinx-build:
It should have a way on it to do partial documentation builds.

Quoted 7 lines · Mauro Carvalho Chehab

Quoted 5 lines · Jani Nikula

Quoted 2 lines

1. SPHINXDIRS. It needs a lot of magic to work, both before running sphinx-build and after (inside conf.py);

Makes you wonder if that's the right solution to the original problem. It was added as a kind of hack, and it stuck.

Quoted 2 lines

The problem is, IMHO, due to the lack of flexibility of sphinx-build:
It should have a way on it to do partial documentation builds.

A couple of times I have looked into using intersphinx, making each book into an actually separate book. The thing I always run into is that doing a complete docs build, with working references, would require building everything twice. This is probably worth another attempt one of these years...

Quoted 5 lines · Jonathan Corbet

A couple of times I have looked into using intersphinx, making each book into an actually separate book. The thing I always run into is that doing a complete docs build, with working references, would require building everything twice. This is probably worth another attempt one of these years...

I think the main factor in that should be whether it makes sense from overall documentation standpoint, not the technical details.

Having several books might make sense. It might even be helpful in organizing the documentation by audiences. But having the granularity of SPHINXDIRS with that would be overkill. And there needs to be a book to bring them together, and link to the other books, acting as the landing page.

I believe it should be possible to generate the intersphinx inventory without generating the full html or pdf documentation. So I don't think it's actually two complete docs builds. It might speed things up to have a number of independent documentation builds.

As to the working references, IIUC partial builds with SPHINXDIRS doesn't get that part right if there are references outside of the designated dirs, leading to warnings.

Quoted 6 lines · Jani Nikula

Quoted 5 lines · Jonathan Corbet

A couple of times I have looked into using intersphinx, making each book into an actually separate book. The thing I always run into is that doing a complete docs build, with working references, would require building everything twice. This is probably worth another attempt one of these years...

Quoted 2 lines

I think the main factor in that should be whether it makes sense from overall documentation standpoint, not the technical details.

Quoted 3 lines

Having several books might make sense. It might even be helpful in organizing the documentation by audiences. But having the granularity of SPHINXDIRS with that would be overkill.

Quoted 3 lines

And there needs to be a book to
bring them together, and link to the other books, acting as the landing page.

Well, I think that the number of existing directories needs to be reduced rather further. I made progress in that direction by coalescing all the arch docs under Documentation/arch/. I would like to do something similar with all the device-specific docs, creating Documentation/devices/. Then we start to get to a reasonable number of books.

Quoted 4 lines

I believe it should be possible to generate the intersphinx inventory without generating the full html or pdf documentation. So I don't think it's actually two complete docs builds. It might speed things up to have a number of independent documentation builds.

That's a good point, I hadn't looked into that part. The builder phase takes a lot of the time, if that could be cut out things would go faster.

Quoted 3 lines

As to the working references, IIUC partial builds with SPHINXDIRS doesn't get that part right if there are references outside of the designated dirs, leading to warnings.

That is true. My point though is that, to get the references right with a *full* build, a two-pass approach is needed though, as you suggest, perhaps the first pass could be faster.

Em Thu, 11 Sep 2025 13:47:54 -0600
Jonathan Corbet <[email protected]> escreveu:

Quoted 8 lines

Quoted 6 lines · Jani Nikula

Quoted 5 lines · Jonathan Corbet

A couple of times I have looked into using intersphinx, making each book into an actually separate book. The thing I always run into is that doing a complete docs build, with working references, would require building everything twice. This is probably worth another attempt one of these years...

There are a couple of different usecase scenarios for building docs.

1) The first and most important one is to produce book(s) for people to use. This is usually done by some automation, and the result is placed on places like:
- https://docs.kernel.org/

and on subsystem-specific places like:
- https://linuxtv.org/downloads/v4l-dvb-apis-new/

for scenario (1), taking twice the time to build is not an issue, as nobody will be sitting on a chair waiting for the build to finish.

On such scenario, SPHINXDIRS is important on subsystem-specific docs. For instance, on media, we use SPHINXDIRS to pick parts of 3 different books:

- Documentation/admin-guide/media/
- Documentation/driver-api/media/
- Documentation/userspace-api/media/

What media automation does, once per day, is:

	# Non-essencial parts of index.rst dropped
	cat <<END >Documentation/media/index.rst
	================================
	Linux Kernel Media Documentation
	================================

	.. toctree::

	        admin-guide/index
        	driver-api/index
	        userspace-api/index
	END

	rsync -uAXEHlaSx -W --inplace --delete Documentation/admin-guide/media/ Documentation/media/admin-guide
	rsync -uAXEHlaSx -W --inplace --delete Documentation/driver-api/media/ Documentation/media/driver-api
	rsync -uAXEHlaSx -W --inplace --delete Documentation/userspace-api/media/ Documentation/media/userspace-api

	make SPHINXDIRS='media' CSS='$CSS' DOCS_THEME='$DOCS_THEME' htmldocs
	make SPHINXDIRS='media' pdfdocs
	make SPHINXDIRS='media' epubdocs

2) CI tests. Here, taking more time usually is not a problem, except when CI is used before pushing stuff, and the developer has to wait it to finish before pushing.

For scenario (2), a build time increase is problematic, as, if it now takes twice the time, a change like that will require twice the resources for the build with may increase costs.

3) developers who touched docs. They want a way to quickly build and verify the output for their changes.

Here, any time increase is problematic, and SPHINXDIRS play an important hole by allowing them to build only the touched documents.

For instance, when I was developing Netlink yaml plugin, I had to use dozens of times:

make SPINXDRS=Documentation/netlink/specs/ htmldocs

If I had to build the entire documentation every time, the development time would increase from days to weeks.

Looking on those three scenarios, the only one where intersphinx is useful is (1).

From my PoV, we should support intersphinx, but this should be optional.

Also, one has to point from where intersphinx will point unsolved symbols. So, we would need something like:

make SPHINXREFMAP=intersphinx_mapping.py htmldocs

where intersphinx_mapping.py would be a file containing intersphinx configuration. We would add a default map at Documentation/, while letting it to be overridden if some subsystem has different requirements or is using a different CSS tamplate or not using alabaster.

Quoted 2 lines

Quoted 2 lines

I think the main factor in that should be whether it makes sense from overall documentation standpoint, not the technical details.

Agreed.

Quoted 3 lines

Quoted 3 lines

Having several books might make sense. It might even be helpful in organizing the documentation by audiences. But having the granularity of SPHINXDIRS with that would be overkill.

On the contrary. SPHINXDIRS granuarity is very important for scenario (3).

Quoted 10 lines

Quoted 3 lines

And there needs to be a book to
bring them together, and link to the other books, acting as the landing page.

Well, I think that the number of existing directories needs to be reduced rather further. I made progress in that direction by coalescing all the arch docs under Documentation/arch/. I would like to do something similar with all the device-specific docs, creating Documentation/devices/. Then we start to get to a reasonable number of books.

I don't think reducing the number of books should be the goal, but, instead, to have them with a clear and coherent organization with focus on the audience that will be actually using them.

After reorg, we may have less books. That's fine. But it is also fine if we end with more books.

I lost the battle years ago, but I still believe that, at least for some subsystems like media, i2c, DRM, security and others, a subsystem-specific book could be better. After all, the audience for such subsystems is very specialized.

Quoted 8 lines

Quoted 4 lines

I believe it should be possible to generate the intersphinx inventory without generating the full html or pdf documentation. So I don't think it's actually two complete docs builds. It might speed things up to have a number of independent documentation builds.

That's a good point, I hadn't looked into that part. The builder phase takes a lot of the time, if that could be cut out things would go faster.

Indeed, but we need to double check if .doctree cache expiration will happen the right way for all books affected by a partial build.

During this merge window, I sent a RFC patch in the middle of a comment with a conf.py logic to detect Sphinx cache expiration. I remember I added a comment asking if we should upstream it or not, but, as nobody answered, I ended forgetting about it.

If we're willing to experiment with that, I recommend looking on such patch and add a variant of it, enabled via V=1 or via some debug parameter.

The goal would be to check if a change on a file will ensure that all books using it will have cache expiration and be rebuilt.

Quoted 7 lines

Quoted 3 lines

As to the working references, IIUC partial builds with SPHINXDIRS doesn't get that part right if there are references outside of the designated dirs, leading to warnings.

That is true. My point though is that, to get the references right with a *full* build, a two-pass approach is needed though, as you suggest, perhaps the first pass could be faster.

How fast? during development time, SPHINXDIRS means a couple of seconds:

	$ make clean; time make SPHINXDIRS="peci" htmldocs
	...
	real    0m1,373s
	user    0m1,348s

Even more complex builds, even when picking more than one book, like this:

	$ make clean; time make SPHINXDIRS="driver-api/media/ userspace-api/media/" htmldocs
	...
	real    0m11,801s
	user    0m31,381s
	sys     0m6,880s

it still fits at the seconds range. Can interphinx first pass have a similar build time?

Quoted 2 lines

Here, any time increase is problematic, and SPHINXDIRS play an important hole by allowing them to build only the touched documents.

This is actually problematic, because the SPHINXDIRS partial builds will give you warnings for unresolved references that are just fine if the entire documentation gets built.

Quoted 5 lines

Quoted 2 lines

Here, any time increase is problematic, and SPHINXDIRS play an important hole by allowing them to build only the touched documents.

This is actually problematic, because the SPHINXDIRS partial builds will give you warnings for unresolved references that are just fine if the entire documentation gets built.

I admit I don't have a full overview of all the problems that are being solved here (in existing and proposed code), but how hard would it be to convert the whole SPHINXDIRS thing into a Sphinx plugin that runs early and discards documents outside of what the user wants to build? By "discards" I mean in some useful way that reduces runtime compared to a full build while retaining some benefits of a full build (reference checking)?

Quoted 52 lines

Em Thu, 11 Sep 2025 13:47:54 -0600
Jonathan Corbet <[email protected]> escreveu:

Quoted 8 lines

Quoted 6 lines · Jani Nikula

Quoted 5 lines · Jonathan Corbet

A couple of times I have looked into using intersphinx, making each book into an actually separate book. The thing I always run into is that doing a complete docs build, with working references, would require building everything twice. This is probably worth another attempt one of these years...

There are a couple of different usecase scenarios for building docs.

1) The first and most important one is to produce book(s) for people to use. This is usually done by some automation, and the result is placed on places like:
- https://docs.kernel.org/

and on subsystem-specific places like:
- https://linuxtv.org/downloads/v4l-dvb-apis-new/

for scenario (1), taking twice the time to build is not an issue, as nobody will be sitting on a chair waiting for the build to finish.

On such scenario, SPHINXDIRS is important on subsystem-specific docs. For instance, on media, we use SPHINXDIRS to pick parts of 3 different books:

- Documentation/admin-guide/media/
- Documentation/driver-api/media/
- Documentation/userspace-api/media/

What media automation does, once per day, is:

	# Non-essencial parts of index.rst dropped
	cat <<END >Documentation/media/index.rst
	================================
	Linux Kernel Media Documentation
	================================

	.. toctree::

	        admin-guide/index
        	driver-api/index
	        userspace-api/index
	END

	rsync -uAXEHlaSx -W --inplace --delete Documentation/admin-guide/media/ Documentation/media/admin-guide
	rsync -uAXEHlaSx -W --inplace --delete Documentation/driver-api/media/ Documentation/media/driver-api
	rsync -uAXEHlaSx -W --inplace --delete Documentation/userspace-api/media/ Documentation/media/userspace-api

Quoted 3 lines

make SPHINXDIRS='media' CSS='$CSS' DOCS_THEME='$DOCS_THEME' htmldocs make SPHINXDIRS='media' pdfdocs
make SPHINXDIRS='media' epubdocs

I was actually wondering how [1] was built. So it's not a complete build of anything upstream, but rather something cobbled together downstream.

So your scenario (1) above is actually *two* wildly different scenarios.

And if upstream needs to cater for pretty much random subsets of documentation being built, cherry-picking documentation from here and there, I don't know what hope there is in radically refactoring how documentation gets built upstream.

I presume you have one or more of a) get bunch of broken link warnings at build, b) get broken links in the output, c) avoid links outside of your subset altogether.

[1] https://linuxtv.org/downloads/v4l-dvb-apis-new/

Quoted 11 lines

2) CI tests. Here, taking more time usually is not a problem, except when CI is used before pushing stuff, and the developer has to wait it to finish before pushing.

For scenario (2), a build time increase is problematic, as, if it now takes twice the time, a change like that will require twice the resources for the build with may increase costs.

3) developers who touched docs. They want a way to quickly build and verify the output for their changes.

Quoted 2 lines

Here, any time increase is problematic, and SPHINXDIRS play an important hole by allowing them to build only the touched documents.

This is actually problematic, because the SPHINXDIRS partial builds will give you warnings for unresolved references that are just fine if the entire documentation gets built.

Quoted 5 lines

For instance, when I was developing Netlink yaml plugin, I had to use dozens of times:

make SPINXDRS=Documentation/netlink/specs/ htmldocs

Quoted 5 lines

If I had to build the entire documentation every time, the development time would increase from days to weeks.

Looking on those three scenarios, the only one where intersphinx is useful is (1).

It's also helpful for 3, and it could be helpful for 2 if CI only checks some parts of the documentation.

Quoted 1 line

From my PoV, we should support intersphinx, but this should be optional.

Per my understanding making this somehow optional is not easily achieved. And you end up with a bunch of extra complexity.

Quoted 19 lines

Also, one has to point from where intersphinx will point unsolved symbols. So, we would need something like:

make SPHINXREFMAP=intersphinx_mapping.py htmldocs

where intersphinx_mapping.py would be a file containing intersphinx configuration. We would add a default map at Documentation/, while letting it to be overridden if some subsystem has different requirements or is using a different CSS tamplate or not using alabaster.

Quoted 2 lines

Quoted 2 lines

I think the main factor in that should be whether it makes sense from overall documentation standpoint, not the technical details.

Agreed.

Quoted 3 lines

Quoted 3 lines

Having several books might make sense. It might even be helpful in organizing the documentation by audiences. But having the granularity of SPHINXDIRS with that would be overkill.

Quoted 1 line

On the contrary. SPHINXDIRS granuarity is very important for scenario (3).

Sphinx does support incremental builds, and it's only the very first build that's slow. IMO a handful of books that you can actually build without warnings (unlike SPHINXDIRS) with incremental builds is a good compromise.

Quoted 70 lines

Quoted 10 lines

Quoted 3 lines

And there needs to be a book to
bring them together, and link to the other books, acting as the landing page.

Well, I think that the number of existing directories needs to be reduced rather further. I made progress in that direction by coalescing all the arch docs under Documentation/arch/. I would like to do something similar with all the device-specific docs, creating Documentation/devices/. Then we start to get to a reasonable number of books.

I don't think reducing the number of books should be the goal, but, instead, to have them with a clear and coherent organization with focus on the audience that will be actually using them.

After reorg, we may have less books. That's fine. But it is also fine if we end with more books.

I lost the battle years ago, but I still believe that, at least for some subsystems like media, i2c, DRM, security and others, a subsystem-specific book could be better. After all, the audience for such subsystems is very specialized.

Quoted 8 lines

Quoted 4 lines

I believe it should be possible to generate the intersphinx inventory without generating the full html or pdf documentation. So I don't think it's actually two complete docs builds. It might speed things up to have a number of independent documentation builds.

That's a good point, I hadn't looked into that part. The builder phase takes a lot of the time, if that could be cut out things would go faster.

Indeed, but we need to double check if .doctree cache expiration will happen the right way for all books affected by a partial build.

During this merge window, I sent a RFC patch in the middle of a comment with a conf.py logic to detect Sphinx cache expiration. I remember I added a comment asking if we should upstream it or not, but, as nobody answered, I ended forgetting about it.

If we're willing to experiment with that, I recommend looking on such patch and add a variant of it, enabled via V=1 or via some debug parameter.

The goal would be to check if a change on a file will ensure that all books using it will have cache expiration and be rebuilt.

Quoted 7 lines

Quoted 3 lines

As to the working references, IIUC partial builds with SPHINXDIRS doesn't get that part right if there are references outside of the designated dirs, leading to warnings.

That is true. My point though is that, to get the references right with a *full* build, a two-pass approach is needed though, as you suggest, perhaps the first pass could be faster.

How fast? during development time, SPHINXDIRS means a couple of seconds:

	$ make clean; time make SPHINXDIRS="peci" htmldocs
	...
	real    0m1,373s
	user    0m1,348s

Even more complex builds, even when picking more than one book, like this:

	$ make clean; time make SPHINXDIRS="driver-api/media/ userspace-api/media/" htmldocs
	...
	real    0m11,801s
	user    0m31,381s
	sys     0m6,880s

Quoted 2 lines

it still fits at the seconds range. Can interphinx first pass have a similar build time?

Probably not. Can you add links from media to non-media documentation without warnings? Probably not also.

Quoted 7 lines · Mauro Carvalho Chehab

Quoted 5 lines · Jani Nikula

Quoted 2 lines

1. SPHINXDIRS. It needs a lot of magic to work, both before running sphinx-build and after (inside conf.py);

Makes you wonder if that's the right solution to the original problem. It was added as a kind of hack, and it stuck.

Quoted 2 lines

The problem is, IMHO, due to the lack of flexibility of sphinx-build:
It should have a way on it to do partial documentation builds.

A couple of times I have looked into using intersphinx, making each book into an actually separate book. The thing I always run into is that doing a complete docs build, with working references, would require building everything twice. This is probably worth another attempt one of these years...

Sometimes, it is desired to run Sphinx from a virtual environment. Add a command line parameter to automatically build Sphinx from such environment.

Signed-off-by Mauro Carvalho Chehab <[email protected]>

---
 tools/docs/sphinx-build-wrapper | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

tools/docs/sphinx-build-wrapper +27 -3

index ea9f8e17b0bc..cf7b30bc40ff 100755
@@ -63,6 +63,7 @@ from jobserver import JobserverExec         # pylint: disable=C0413,C0411,E0401
 #
 #  Some constants
 #
+VENV_DEFAULT = "sphinx_latest"
 MIN_PYTHON_VERSION = PythonVersion("3.7").version
 PAPER = ["", "a4", "letter"]
 

@@ -119,8 +120,9 @@ class SphinxBuilder:
 
         return path
 
-    def __init__(self, builddir, verbose=False, n_jobs=None):
+    def __init__(self, builddir, venv=None, verbose=False, n_jobs=None):
         """Initialize internal variables"""
+        self.venv = venv
         self.verbose = None
 
         #

@@ -195,6 +197,21 @@ class SphinxBuilder:
 
         self.env = os.environ.copy()
 
+        #
+        # If venv command line argument is specified, run Sphinx from venv
+        #
+        if venv:
+            bin_dir = os.path.join(venv, "bin")
+            if not os.path.isfile(os.path.join(bin_dir, "activate")):
+                sys.exit(f"Venv {venv} not found.")
+
+            # "activate" virtual env
+            self.env["PATH"] = bin_dir + ":" + self.env["PATH"]
+            self.env["VIRTUAL_ENV"] = venv
+            if "PYTHONHOME" in self.env:
+                del self.env["PYTHONHOME"]
+            print(f"Setting venv to {venv}")
+
     def run_sphinx(self, sphinx_build, build_args, *args, **pwargs):
         """
         Executes sphinx-build using current python3 command and setting

@@ -209,7 +226,10 @@ class SphinxBuilder:
 
             cmd = []
 
-            cmd.append(sys.executable)
+            if self.venv:
+                cmd.append("python")
+            else:
+                cmd.append(sys.executable)
 
             cmd.append(sphinx_build)
 

@@ -533,11 +553,15 @@ def main():
     parser.add_argument('-j', '--jobs', type=jobs_type,
                         help="Sets number of jobs to use with sphinx-build")
 
+    parser.add_argument("-V", "--venv", nargs='?', const=f'{VENV_DEFAULT}',
+                        default=None,
+                        help=f'If used, run Sphinx from a venv dir (default dir: {VENV_DEFAULT})')
+
     args = parser.parse_args()
 
     PythonVersion.check_python(MIN_PYTHON_VERSION)
 
-    builder = SphinxBuilder(builddir=args.builddir,
+    builder = SphinxBuilder(builddir=args.builddir, venv=args.venv,
                             verbose=args.verbose, n_jobs=args.jobs)
 
     builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf,