Skip to contentSkip to conversation

rust: add missing macros deps as host crates

PATCH Superseded Tamir Duberstein 1 participant 4 messages Feb 9, 2025

Subsystems (2)

v1 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

cover rust: add missing macros deps as host crates

This is an alternative of (or follow-up to) a similar patch based on rust-fixes:

https://lore.kernel.org/all/20250209-rust-analyzer-macros-core-dep-v2-1-897338344d16@gmail.com/

This one contains a larger refactor that allows host crates to be declared separately from target crates as discussed in v1 of the above:

https://lore.kernel.org/all/20250209-rust-analyzer-macros-core-dep-v1-1-5ebeb3eb60a9@gmail.com/

In my testing rust-analyzer seems fine with multiple crates with the same display_name. Crates are identified by their index.

---
Tamir Duberstein (3):
scripts: generate_rust_analyzer.py: add type hints
scripts: generate_rust_analyzer.py: identify crates explicitly
scripts: generate_rust_analyzer.py: add missing macros deps

 scripts/generate_rust_analyzer.py | 143 ++++++++++++++++++++++++++------------
 1 file changed, 97 insertions(+), 46 deletions(-)

---
base-commit: beeb78d46249cab8b2b8359a2ce8fa5376b5ad2d
change-id: 20250209-rust-analyzer-host-43b108655578

Best regards,

Signature · 1 line
Tamir Duberstein <[email protected]>
#1 lore raw

PATCH 1/3 scripts: generate_rust_analyzer.py: add type hints

Python type hints allow static analysis tools like mypy to detect type errors during development, improving the developer experience.

Python type hints have been present in the kernel since 2019 at the latest; see commit 6ebf5866f2e8 ("kunit: tool: add Python wrappers for running KUnit tests").

Run uv tool run mypy --strict scripts/generate_rust_analyzer.py to verify.

---
 scripts/generate_rust_analyzer.py | 83 ++++++++++++++++++++++++++++-----------
 1 file changed, 61 insertions(+), 22 deletions(-)
scripts/generate_rust_analyzer.py +61 -22
index aa8ea1a4dbe5..28ab05de52b8 100755
@@ -10,8 +10,9 @@ import os
 import pathlib
 import subprocess
 import sys
+import typing as T
 
-def args_crates_cfgs(cfgs):
+def args_crates_cfgs(cfgs: T.Iterable[str]) -> dict[str, list[str]]:
     crates_cfgs = {}
     for cfg in cfgs:
         crate, vals = cfg.split("=", 1)
@@ -19,7 +20,34 @@ def args_crates_cfgs(cfgs):
 
     return crates_cfgs
 
-def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs):
+class Dependency(T.TypedDict):
+    crate: int
+    name: str
+
+class Source(T.TypedDict):
+    include_dirs: list[str]
+    exclude_dirs: list[str]
+
+class Crate(T.TypedDict):
+    display_name: str
+    root_module: str
+    is_workspace_member: bool
+    is_proc_macro: bool
+    deps: list[Dependency]
+    cfg: list[str]
+    edition: T.Literal["2021"]
+    env: dict[str, str]
+    # `NotRequired` would be better but was added in 3.11.
+    proc_macro_dylib_path: T.Optional[str]
+    source: T.Optional[Source]
+
+def generate_crates(
+    srctree: pathlib.Path,
+    objtree: pathlib.Path,
+    sysroot_src: pathlib.Path,
+    external_src: pathlib.Path,
+    cfgs: list[str],
+) -> list[Crate]:
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
@@ -31,31 +59,40 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs):
     # Now fill the crates list -- dependencies need to come first.
     #
     # Avoid O(n^2) iterations by keeping a map of indexes.
-    crates = []
-    crates_indexes = {}
+    crates: list[Crate] = []
+    crates_indexes: dict[str, int] = {}
     crates_cfgs = args_crates_cfgs(cfgs)
 
-    def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=True, is_proc_macro=False):
-        crate = {
-            "display_name": display_name,
-            "root_module": str(root_module),
-            "is_workspace_member": is_workspace_member,
-            "is_proc_macro": is_proc_macro,
-            "deps": [{"crate": crates_indexes[dep], "name": dep} for dep in deps],
-            "cfg": cfg,
-            "edition": "2021",
-            "env": {
-                "RUST_MODFILE": "This is only for rust-analyzer"
-            }
-        }
+    def append_crate(
+        display_name: str,
+        root_module: pathlib.Path,
+        deps: list[str],
+        cfg: list[str] = [],
+        is_workspace_member: bool = True,
+        is_proc_macro: bool = False,
+    ) -> None:
+        proc_macro_dylib_path = None
         if is_proc_macro:
             proc_macro_dylib_name = subprocess.check_output(
                 [os.environ["RUSTC"], "--print", "file-names", "--crate-name", display_name, "--crate-type", "proc-macro", "-"],
                 stdin=subprocess.DEVNULL,
             ).decode('utf-8').strip()
-            crate["proc_macro_dylib_path"] = f"{objtree}/rust/{proc_macro_dylib_name}"
+            proc_macro_dylib_path = f"{objtree}/rust/{proc_macro_dylib_name}"
         crates_indexes[display_name] = len(crates)
-        crates.append(crate)
+        crates.append(
+            {
+                "display_name": display_name,
+                "root_module": str(root_module),
+                "is_workspace_member": is_workspace_member,
+                "is_proc_macro": is_proc_macro,
+                "deps": [{"crate": crates_indexes[dep], "name": dep} for dep in deps],
+                "cfg": cfg,
+                "edition": "2021",
+                "env": {"RUST_MODFILE": "This is only for rust-analyzer"},
+                "proc_macro_dylib_path": proc_macro_dylib_path,
+                "source": None,
+            }
+        )
 
     # First, the ones in `rust/` since they are a bit special.
     append_crate(
@@ -107,7 +144,7 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs):
         "exclude_dirs": [],
     }
 
-    def is_root_crate(build_file, target):
+    def is_root_crate(build_file: pathlib.Path, target: str) -> bool:
         try:
             return f"{target}.o" in open(build_file).read()
         except FileNotFoundError:
@@ -116,7 +153,9 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs):
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    extra_dirs = map(lambda dir: srctree / dir, ("samples", "drivers"))
+    extra_dirs: T.Iterable[pathlib.Path] = map(
+        lambda dir: srctree / dir, ("samples", "drivers")
+    )
     if external_src is not None:
         extra_dirs = [external_src]
     for folder in extra_dirs:
@@ -139,7 +178,7 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs):
 
     return crates
 
-def main():
+def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument('--verbose', '-v', action='store_true')
     parser.add_argument('--cfgs', action='append', default=[])
Signature · 1 line
2.48.1
#2 lore raw

PATCH 2/3 scripts: generate_rust_analyzer.py: identify crates explicitly

In reply to Tamir Duberstein

Use the return of append_crate to declare dependency on that crate. This allows multiple crates with the same display_name be defined, which we'll use to define host crates separately from target crates.

---
 scripts/generate_rust_analyzer.py | 34 ++++++++++++++++------------------
 1 file changed, 16 insertions(+), 18 deletions(-)
scripts/generate_rust_analyzer.py +16 -18
index 28ab05de52b8..cb49f0b07c97 100755
@@ -56,21 +56,18 @@ def generate_crates(
             line = line.replace("\n", "")
             cfg.append(line)
 
-    # Now fill the crates list -- dependencies need to come first.
-    #
-    # Avoid O(n^2) iterations by keeping a map of indexes.
+    # Now fill the crates list.
     crates: list[Crate] = []
-    crates_indexes: dict[str, int] = {}
     crates_cfgs = args_crates_cfgs(cfgs)
 
     def append_crate(
         display_name: str,
         root_module: pathlib.Path,
-        deps: list[str],
+        deps: list[Dependency],
         cfg: list[str] = [],
         is_workspace_member: bool = True,
         is_proc_macro: bool = False,
-    ) -> None:
+    ) -> Dependency:
         proc_macro_dylib_path = None
         if is_proc_macro:
             proc_macro_dylib_name = subprocess.check_output(
@@ -78,14 +75,14 @@ def generate_crates(
                 stdin=subprocess.DEVNULL,
             ).decode('utf-8').strip()
             proc_macro_dylib_path = f"{objtree}/rust/{proc_macro_dylib_name}"
-        crates_indexes[display_name] = len(crates)
+        index = len(crates)
         crates.append(
             {
                 "display_name": display_name,
                 "root_module": str(root_module),
                 "is_workspace_member": is_workspace_member,
                 "is_proc_macro": is_proc_macro,
-                "deps": [{"crate": crates_indexes[dep], "name": dep} for dep in deps],
+                "deps": deps,
                 "cfg": cfg,
                 "edition": "2021",
                 "env": {"RUST_MODFILE": "This is only for rust-analyzer"},
@@ -93,9 +90,10 @@ def generate_crates(
                 "source": None,
             }
         )
+        return {"crate": index, "name": display_name}
 
     # First, the ones in `rust/` since they are a bit special.
-    append_crate(
+    core = append_crate(
         "core",
         sysroot_src / "core" / "src" / "lib.rs",
         [],
@@ -103,37 +101,37 @@ def generate_crates(
         is_workspace_member=False,
     )
 
-    append_crate(
+    compiler_builtins = append_crate(
         "compiler_builtins",
         srctree / "rust" / "compiler_builtins.rs",
         [],
     )
 
-    append_crate(
+    macros = append_crate(
         "macros",
         srctree / "rust" / "macros" / "lib.rs",
         [],
         is_proc_macro=True,
     )
 
-    append_crate(
+    build_error = append_crate(
         "build_error",
         srctree / "rust" / "build_error.rs",
-        ["core", "compiler_builtins"],
+        [core, compiler_builtins],
     )
 
-    append_crate(
+    bindings = append_crate(
         "bindings",
         srctree / "rust"/ "bindings" / "lib.rs",
-        ["core"],
+        [core],
         cfg=cfg,
     )
     crates[-1]["env"]["OBJTREE"] = str(objtree.resolve(True))
 
-    append_crate(
+    kernel = append_crate(
         "kernel",
         srctree / "rust" / "kernel" / "lib.rs",
-        ["core", "macros", "build_error", "bindings"],
+        [core, macros, build_error, bindings],
         cfg=cfg,
     )
     crates[-1]["source"] = {
@@ -172,7 +170,7 @@ def generate_crates(
             append_crate(
                 name,
                 path,
-                ["core", "kernel"],
+                [core, kernel],
                 cfg=cfg,
             )
 
Signature · 1 line
2.48.1
#3 lore raw

PATCH 3/3 scripts: generate_rust_analyzer.py: add missing macros deps

In reply to Tamir Duberstein

The macros crate has depended on std and proc_macro since its introduction in commit 1fbde52bde73 ("rust: add macros crate"). These dependencies were omitted from commit 8c4555ccc55c ("scripts: add generate_rust_analyzer.py") resulting in missing go-to-definition and autocomplete, and false-positive warnings emitted from rust-analyzer such as:

  [{
  	"resource": "/Users/tamird/src/linux/rust/macros/module.rs",
  	"owner": "_generated_diagnostic_collection_name_#1",
  	"code": {
  		"value": "non_snake_case",
  		"target": {
  			"$mid": 1,
  			"path": "/rustc/",
  			"scheme": "https",
  			"authority": "doc.rust-lang.org",
  			"query": "search=non_snake_case"
  		}
  	},
  	"severity": 4,
  	"message": "Variable `None` should have snake_case name, e.g. `none`",
  	"source": "rust-analyzer",
  	"startLineNumber": 123,
  	"startColumn": 17,
  	"endLineNumber": 123,
  	"endColumn": 21
  }]

Define and add the missing *host* dependencies to improve the developer experience. Define the *target* core crate using the new append_sysroot_crate helper.

Fixes 8c4555ccc55c ("scripts: add `generate_rust_analyzer.py`")
---
 scripts/generate_rust_analyzer.py | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)
scripts/generate_rust_analyzer.py +23 -9
index cb49f0b07c97..124f6b0334db 100755
@@ -92,14 +92,28 @@ def generate_crates(
         )
         return {"crate": index, "name": display_name}
 
-    # First, the ones in `rust/` since they are a bit special.
-    core = append_crate(
-        "core",
-        sysroot_src / "core" / "src" / "lib.rs",
-        [],
-        cfg=crates_cfgs.get("core", []),
-        is_workspace_member=False,
-    )
+    def append_sysroot_crate(
+        display_name: str,
+        deps: list[Dependency],
+        cfg: list[str] = [],
+    ) -> Dependency:
+        return append_crate(
+            display_name,
+            sysroot_src / display_name / "src" / "lib.rs",
+            deps,
+            cfg,
+            is_workspace_member=False,
+        )
+
+    # NB: sysroot crates reexport items from one another so setting up our transitive dependencies
+    # here is important for ensuring that rust-analyzer can resolve symbols. The sources of truth
+    # for this dependency graph are `(sysroot_src / crate / "Cargo.toml" for crate in crates)`.
+    host_core = append_sysroot_crate("core", [])
+    host_alloc = append_sysroot_crate("alloc", [host_core])
+    host_std = append_sysroot_crate("std", [host_alloc, host_core])
+    host_proc_macro = append_sysroot_crate("proc_macro", [host_core, host_std])
+
+    core = append_sysroot_crate("core", [], cfg=crates_cfgs.get("core", []))
 
     compiler_builtins = append_crate(
         "compiler_builtins",
@@ -110,7 +124,7 @@ def generate_crates(
     macros = append_crate(
         "macros",
         srctree / "rust" / "macros" / "lib.rs",
-        [],
+        [host_std, host_proc_macro],
         is_proc_macro=True,
     )
 
Signature · 1 line
2.48.1