scripts: `make rust-analyzer` for out-of-tree modules

PATCH Superseded Vinay Varma 5 participants 16 messages Jan 18, 2023 – Aug 2, 2023

Subsystems (3)

Kbuild Rust rust-analyzer

lore

Share & tools

lore

Vinay Varma

author

2023-01-18 16:06 UTC #0 lore raw

PATCH 1/1 scripts: `make rust-analyzer` for out-of-tree modules

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.

The change involves adding an optional parameter external_src to the generate_rust_analyzer.py which expects the path to the out-of-tree module's source directory. When this parameter is passed, I have chosen not to add the non-core modules (samples and drivers) into the result since these are not expected to be used in third party modules. Related changes are also made to the Makefile and rust/Makefile allowing the rust-analyzer target to be used for out-of-tree modules as well.

Signed-off-by Vinay Varma <[email protected]>

---
 Makefile                          | 12 +++++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 14 +++++++++-----
 3 files changed, 20 insertions(+), 12 deletions(-)

Makefile +7 -5

index f41ec8c8426b..a055a316d2a4 100644
@@ -1831,11 +1831,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-	$(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------

@@ -1888,6 +1883,7 @@ help:
 	@echo  '  modules         - default target, build the module(s)'
 	@echo  '  modules_install - install the module'
 	@echo  '  clean           - remove generated files in module directory only'
+	@echo  '  rust-analyzer	  - generate rust-project.json rust-analyzer support file'
 	@echo  ''
 
 endif # KBUILD_EXTMOD

@@ -2022,6 +2018,12 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
 	$(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
+
 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------

rust/Makefile +4 -2

index 8f598a904f38..41c1435cd8d4 100644
@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+		$(abs_srctree) $(abs_objtree) \
+		$(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+		$(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
 
 $(obj)/core.o: private skip_clippy = 1
 $(obj)/core.o: private skip_flags = -Dunreachable_pub

scripts/generate_rust_analyzer.py +9 -5

index ecc7ea9a4dcf..1546b80db554 100755
@@ -8,8 +8,9 @@ import json
 import logging
 import pathlib
 import sys
+import os
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:

@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",

@@ -98,13 +99,15 @@ def generate_crates(srctree, objtree, sysroot_src):
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    for folder in ("samples", "drivers"):
+    extra_src_dirs = ["samples", "drivers"] if external_src is None else [external_src]
+
+    for folder in extra_src_dirs:
         for path in (srctree / folder).rglob("*.rs"):
             logging.info("Checking %s", path)
             name = path.name.replace(".rs", "")
 
             # Skip those that are not crate roots.
-            if f"{name}.o" not in open(path.parent / "Makefile").read():
+            if os.path.exists(path.parent / "Makefile") and f"{name}.o" not in open(path.parent / "Makefile").read():
                 continue
 
             logging.info("Adding %s", name)

@@ -123,6 +126,7 @@ def main():
     parser.add_argument("srctree", type=pathlib.Path)
     parser.add_argument("objtree", type=pathlib.Path)
     parser.add_argument("sysroot_src", type=pathlib.Path)
+    parser.add_argument("exttree", type=pathlib.Path, nargs='?')
     args = parser.parse_args()
 
     logging.basicConfig(

@@ -131,7 +135,7 @@ def main():
     )
 
     rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
         "sysroot_src": str(args.sysroot_src),
     }

Signature · 1 line

2.39.0

Miguel Ojeda

2023-01-19 19:15 UTC #1 lore raw

Quoted 3 lines · Vinay Varma

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.

Thanks for sending this! A few nits below...

Quoted 3 lines

When this parameter is passed, I have chosen
not to add the non-core modules (samples and drivers) into the result since these are not expected to be used in third party modules.

Sounds good to me.

Quoted 4 lines

changes are also made to the Makefile and rust/Makefile allowing the rust-analyzer target to be used for out-of-tree modules as well.
Signed-off-by Vinay Varma <[email protected]>

A Link: to the GitHub PR would be nice here:
https://github.com/Rust-for-Linux/linux/pull/914.

Possibly another one to the out-of-tree approach at
https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2.

Quoted 4 lines

import logging
import pathlib
import sys
+import os

Please keep the imports sorted.

scripts/generate_rust_analyzer.py def generate_crates(srctree, objtree, sysroot_src): · line 102

Quoted 2 lines

scripts/generate_rust_analyzer.py +1 -1

-    for folder in ("samples", "drivers"):
+    extra_src_dirs = ["samples", "drivers"] if external_src is None else [external_src]

If you send a v2 for the above, this could be a tuple like in the original line, to minimize the diff.

Quoted 1 line

+ if os.path.exists(path.parent / "Makefile") and f"{name}.o" not in open(path.parent / "Makefile").read():

In Python one would typically go with the EAFP style instead
(https://docs.python.org/3/glossary.html#term-EAFP), which would also reduce the duplication of the path computation. But it would take more lines... Not a big deal in any case.

Thanks!

Cheers,
Miguel

Björn Roy Baron

reviewer

2023-01-20 13:19 UTC #2 lore raw

In reply to Vinay Varma

------- Original Message -------

Quoted 11 lines · Vinay Varma

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.
The change involves adding an optional parameter external_src to the generate_rust_analyzer.py which expects the path to the out-of-tree module's source directory. When this parameter is passed, I have chosen not to add the non-core modules (samples and drivers) into the result since these are not expected to be used in third party modules. Related changes are also made to the Makefile and rust/Makefile allowing the rust-analyzer target to be used for out-of-tree modules as well.

Quoted 3 lines

Signed-off-by Vinay Varma [email protected]
---

I tested this with https://github.com/Rust-for-Linux/rust-out-of-tree-module using "make -C ../rust-for-linux M=$(pwd) rust-analyzer". It produces a rust-project.json file, however it misses the actual rust_out_of_tree crate. This is due to the fact that generate_rust_analyzer.py only checks Makefile to find the crate roots, but rust-out-of-tree-module defines it in Kbuild instead.

Apart from this issue, all paths I checked in the generated rust-project.json are correct.

Cheers,
Bjorn

Miguel Ojeda

2023-01-20 13:35 UTC #3 lore raw

Quoted 2 lines · Björn Roy Baron

I tested this with https://github.com/Rust-for-Linux/rust-out-of-tree-module using "make -C ../rust-for-linux M=$(pwd) rust-analyzer". It produces a rust-project.json file, however it misses the actual rust_out_of_tree crate. This is due to the fact that generate_rust_analyzer.py only checks Makefile to find the crate roots, but rust-out-of-tree-module defines it in Kbuild instead.

Thanks for testing it! Indeed, we should support out-of-tree modules using the Kbuild + Makefile approach.

Having said that, I wonder if we should even attempt to perform the search for out-of-tree modules. The search loop is a simple hack which works fine for in-tree code because we know the structure of the code around us, but we cannot for out-of-tree modules. For instance, they may be using a different build system, or not mentioning the object file, etc.

Perhaps we should simply ask them to give us the roots directly as a list of arguments to the script or similar.

Cheers,
Miguel

Vinay Varma

author

2023-01-21 05:25 UTC #4 lore raw

PATCH v2 1/1 scripts: `make rust-analyzer` for out-of-tree modules

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.

Link https://github.com/Rust-for-Linux/linux/pull/914

Link https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2

Signed-off-by Vinay Varma <[email protected]>

---
 Makefile                          | 12 +++++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
 3 files changed, 29 insertions(+), 20 deletions(-)

Makefile +7 -5

index f41ec8c8426b..a055a316d2a4 100644
@@ -1831,11 +1831,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-	$(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------

@@ -1888,6 +1883,7 @@ help:
 	@echo  '  modules         - default target, build the module(s)'
 	@echo  '  modules_install - install the module'
 	@echo  '  clean           - remove generated files in module directory only'
+	@echo  '  rust-analyzer	  - generate rust-project.json rust-analyzer support file'
 	@echo  ''
 
 endif # KBUILD_EXTMOD

@@ -2022,6 +2018,12 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
 	$(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
+
 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------

rust/Makefile +4 -2

index 8f598a904f38..41c1435cd8d4 100644
@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+		$(abs_srctree) $(abs_objtree) \
+		$(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+		$(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
 
 $(obj)/core.o: private skip_clippy = 1
 $(obj)/core.o: private skip_flags = -Dunreachable_pub

scripts/generate_rust_analyzer.py +18 ~~-13~~

index ecc7ea9a4dcf..1792f379ee4e 100755
@@ -6,10 +6,11 @@
 import argparse
 import json
 import logging
+import os
 import pathlib
 import sys
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:

@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",

@@ -95,25 +96,28 @@ def generate_crates(srctree, objtree, sysroot_src):
         "exclude_dirs": [],
     }
 
+    def is_root_crate(build_file, target):
+        return os.path.exists(build_file) and target in open(build_file).read()
+
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    for folder in ("samples", "drivers"):
+    for folder in ("samples", "drivers") if external_src is None else [external_src]:
         for path in (srctree / folder).rglob("*.rs"):
             logging.info("Checking %s", path)
             name = path.name.replace(".rs", "")
 
             # Skip those that are not crate roots.
-            if f"{name}.o" not in open(path.parent / "Makefile").read():
-                continue
+            if is_root_crate(path.parent / "Makefile", f"{name}.o") or \
+               is_root_crate(path.parent / "Kbuild", f"{name}.o"):
 
-            logging.info("Adding %s", name)
-            append_crate(
-                name,
-                path,
-                ["core", "alloc", "kernel"],
-                cfg=cfg,
-            )
+                logging.info("Adding %s", name)
+                append_crate(
+                    name,
+                    path,
+                    ["core", "alloc", "kernel"],
+                    cfg=cfg,
+                )
 
     return crates

@@ -123,6 +127,7 @@ def main():
     parser.add_argument("srctree", type=pathlib.Path)
     parser.add_argument("objtree", type=pathlib.Path)
     parser.add_argument("sysroot_src", type=pathlib.Path)
+    parser.add_argument("exttree", type=pathlib.Path, nargs='?')
     args = parser.parse_args()
 
     logging.basicConfig(

@@ -131,7 +136,7 @@ def main():
     )
 
     rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
         "sysroot_src": str(args.sysroot_src),
     }

Signature · 1 line

2.39.0

Masahiro Yamada

2023-01-21 09:18 UTC #5 lore raw

Quoted 47 lines · Vinay Varma

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.
The change involves adding an optional parameter external_src to the generate_rust_analyzer.py which expects the path to the out-of-tree module's source directory. When this parameter is passed, I have chosen not to add the non-core modules (samples and drivers) into the result since these are not expected to be used in third party modules. Related changes are also made to the Makefile and rust/Makefile allowing the rust-analyzer target to be used for out-of-tree modules as well.
Link https://github.com/Rust-for-Linux/linux/pull/914
Link https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2
Signed-off-by Vinay Varma <[email protected]>
---
 Makefile                          | 12 +++++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
 3 files changed, 29 insertions(+), 20 deletions(-)
Makefile +1 -5
index f41ec8c8426b..a055a316d2a4 100644
@@ -1831,11 +1831,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-       $(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------
 
@@ -1888,6 +1883,7 @@ help:
        @echo  '  modules         - default target, build the module(s)'
        @echo  '  modules_install - install the module'
        @echo  '  clean           - remove generated files in module directory only'
+       @echo  '  rust-analyzer   - generate rust-project.json rust-analyzer support file'
        @echo  ''
 
 endif # KBUILD_EXTMOD

Makefile quiet_cmd_tags = GEN $@ · line 2026

Quoted 9 lines

Makefile +6 -0

@@ -2022,6 +2018,12 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
        $(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+       $(Q)$(MAKE) $(build)=rust $@
+
+

Extra empty line.

Quoted 8 lines

Makefile +0 -0

 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------

rust/Makefile +0 -0

index 8f598a904f38..41c1435cd8d4 100644

rust/Makefile quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L · line 395

Quoted 9 lines

rust/Makefile +4 -2

@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
        $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-       $(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-               $(RUST_LIB_SRC) > $(objtree)/rust-project.json
+       $(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+               $(abs_srctree) $(abs_objtree) \
+               $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+               $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json

This is equivalent to:

rust/Makefile quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L · line 396

Quoted 1 line

rust/Makefile +0 -0

 $(extmod_prefix)/rust-project.json

See the rule of 'compile_commands.json'.

Quoted 29 lines

rust/Makefile +0 -0

 $(obj)/core.o: private skip_clippy = 1
 $(obj)/core.o: private skip_flags = -Dunreachable_pub

scripts/generate_rust_analyzer.py +3 -2

index ecc7ea9a4dcf..1792f379ee4e 100755
@@ -6,10 +6,11 @@
 import argparse
 import json
 import logging
+import os
 import pathlib
 import sys
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:

@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",

scripts/generate_rust_analyzer.py def generate_crates(srctree, objtree, sysroot_src): · line 104

Quoted 9 lines

scripts/generate_rust_analyzer.py +3 -0

@@ -95,25 +96,28 @@ def generate_crates(srctree, objtree, sysroot_src):
         "exclude_dirs": [],
     }
 
+    def is_root_crate(build_file, target):
+        return os.path.exists(build_file) and target in open(build_file).read()
+
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.

Huh, not maintainable, unfortunately.

scripts/generate_rust_analyzer.py def generate_crates(srctree, objtree, sysroot_src): · line 106

Quoted 3 lines

scripts/generate_rust_analyzer.py +1 -1

-    for folder in ("samples", "drivers"):
+    for folder in ("samples", "drivers") if external_src is None else [external_src]:
         for path in (srctree / folder).rglob("*.rs"):

It is odd to add 'srctree' prefix to external module sources.

I think rust-project.json is a similar concept to
compile_commands.json, but it was implemented
in a different way.

Signature · 2 lines

Best Regards
Masahiro Yamada

Vinay Varma

author

2023-03-07 15:28 UTC #6 lore raw

Sorry, got caught up with another project and lost track of this thread. I have updated the patch and replied to some of the threads inline.

Quoted 142 lines · Masahiro Yamada

Quoted 47 lines · Vinay Varma

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.
The change involves adding an optional parameter external_src to the generate_rust_analyzer.py which expects the path to the out-of-tree module's source directory. When this parameter is passed, I have chosen not to add the non-core modules (samples and drivers) into the result since these are not expected to be used in third party modules. Related changes are also made to the Makefile and rust/Makefile allowing the rust-analyzer target to be used for out-of-tree modules as well.
Link https://github.com/Rust-for-Linux/linux/pull/914
Link https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2
Signed-off-by Vinay Varma <[email protected]>
---
 Makefile                          | 12 +++++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
 3 files changed, 29 insertions(+), 20 deletions(-)
Makefile +1 -5
index f41ec8c8426b..a055a316d2a4 100644
@@ -1831,11 +1831,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-       $(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------
 
@@ -1888,6 +1883,7 @@ help:
        @echo  '  modules         - default target, build the module(s)'
        @echo  '  modules_install - install the module'
        @echo  '  clean           - remove generated files in module directory only'
+       @echo  '  rust-analyzer   - generate rust-project.json rust-analyzer support file'
        @echo  ''
 
 endif # KBUILD_EXTMOD

Makefile quiet_cmd_tags = GEN $@ · line 2026

Quoted 9 lines

Makefile +6 -0

@@ -2022,6 +2018,12 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
        $(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+       $(Q)$(MAKE) $(build)=rust $@
+
+

Extra empty line.

Quoted 8 lines

Makefile +0 -0

 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------

rust/Makefile +0 -0

index 8f598a904f38..41c1435cd8d4 100644

rust/Makefile quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L · line 395

Quoted 9 lines

rust/Makefile +4 -2

@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
        $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-       $(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-               $(RUST_LIB_SRC) > $(objtree)/rust-project.json
+       $(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+               $(abs_srctree) $(abs_objtree) \
+               $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+               $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json

This is equivalent to:

rust/Makefile quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L · line 396

Quoted 1 line

rust/Makefile +0 -0

 $(extmod_prefix)/rust-project.json

See the rule of 'compile_commands.json'.

Quoted 29 lines

rust/Makefile +0 -0

 $(obj)/core.o: private skip_clippy = 1
 $(obj)/core.o: private skip_flags = -Dunreachable_pub

scripts/generate_rust_analyzer.py +3 -2

index ecc7ea9a4dcf..1792f379ee4e 100755
@@ -6,10 +6,11 @@
 import argparse
 import json
 import logging
+import os
 import pathlib
 import sys
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:

@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",

scripts/generate_rust_analyzer.py def generate_crates(srctree, objtree, sysroot_src): · line 104

Quoted 9 lines

scripts/generate_rust_analyzer.py +3 -0

@@ -95,25 +96,28 @@ def generate_crates(srctree, objtree, sysroot_src):
         "exclude_dirs": [],
     }
 
+    def is_root_crate(build_file, target):
+        return os.path.exists(build_file) and target in open(build_file).read()
+
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.

Huh, not maintainable, unfortunately.

Quoted 7 lines

scripts/generate_rust_analyzer.py def generate_crates(srctree, objtree, sysroot_src): · line 106
Quoted 3 lines
scripts/generate_rust_analyzer.py +1 -1
-    for folder in ("samples", "drivers"):
+    for folder in ("samples", "drivers") if external_src is None else [external_src]:
         for path in (srctree / folder).rglob("*.rs"):
It is odd to add 'srctree' prefix to external module sources.

For external module sources, external_src is an absolute path and hence srctree is ignored in this call.

Quoted 7 lines

I think rust-project.json is a similar concept to
compile_commands.json, but it was implemented
in a different way.

I have not included the changes mentioned to refactor the rust-analyzer target along the lines of how compile_commands.json has been solved since it was beyond the scope of this changeset. However, I can take this up as a follow up changeset.

Quoted 9 lines

Signature · 3 lines
-- 
Best Regards
Masahiro Yamada

Miguel Ojeda

2023-03-07 19:59 UTC #7 lore raw

Quoted 3 lines · Vinay Varma

Sorry, got caught up with another project and lost track of this thread. I have updated the patch and replied to some of the threads inline.

No problem -- thanks!

Note that we may need a rebase after
https://lore.kernel.org/rust-for-linux/[email protected]/
(or equivalent) lands in rust-fixes. So perhaps wait for that, then rebase (especially if we go with the EAFP style), and then others may want to give it a spin for a re-test.

When you rebase, please consider putting f"{name}.o" inside the function to avoid repetition, and perhaps consider inverting the condition to avoid indenting the file further (and reducing the diff). It makes the "Skip ..." comment more understandable (otherwise, you may want to change the comment to "Only process ..." or similar instead of using "Skip ...").

Cheers,
Miguel

Miguel Ojeda

2023-04-08 00:00 UTC #8 lore raw

Quoted 9 lines · Miguel Ojeda

No problem -- thanks!
Note that we may need a rebase after
https://lore.kernel.org/rust-for-linux/[email protected]/
(or equivalent) lands in rust-fixes. So perhaps wait for that, then rebase (especially if we go with the EAFP style), and then others may want to give it a spin for a re-test.

Quoted 6 lines

When you rebase, please consider putting f"{name}.o" inside the function to avoid repetition, and perhaps consider inverting the condition to avoid indenting the file further (and reducing the diff). It makes the "Skip ..." comment more understandable (otherwise, you may want to change the comment to "Only process ..." or similar instead of using "Skip ...").

The EAFP style change landed in rust-fixes at
https://github.com/Rust-for-Linux/linux/commit/5c7548d5a25306dcdb97689479be81cacc8ce596.

Cheers,
Miguel

Vinay Varma

author

2023-04-11 09:19 UTC #9 lore raw

PATCH v5 1/1 scripts: `make rust-analyzer` for out-of-tree modules

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.

Link https://github.com/Rust-for-Linux/linux/pull/914

Link https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2

Signed-off-by Vinay Varma <[email protected]>

---
 Makefile                          | 11 ++++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 27 ++++++++++++++++++---------
 3 files changed, 28 insertions(+), 16 deletions(-)

Makefile +6 -5

index ef4e96b9cd5b..1c57360e14f2 100644
@@ -1856,11 +1856,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-	$(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------

@@ -1921,6 +1916,7 @@ help:
 	@echo  '  modules         - default target, build the module(s)'
 	@echo  '  modules_install - install the module'
 	@echo  '  clean           - remove generated files in module directory only'
+	@echo  '  rust-analyzer	  - generate rust-project.json rust-analyzer support file'
 	@echo  ''
 
 endif # KBUILD_EXTMOD

@@ -2058,6 +2054,11 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
 	$(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------

rust/Makefile +4 -2

index aef85e9e8eeb..45c92acd13bc 100644
@@ -360,8 +360,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+		$(abs_srctree) $(abs_objtree) \
+		$(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+		$(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
 
 redirect-intrinsics = \
 	__eqsf2 __gesf2 __lesf2 __nesf2 __unordsf2 \

scripts/generate_rust_analyzer.py +18 -9

index 946e250c1b2a..848fa1ad92ba 100755
@@ -6,10 +6,11 @@
 import argparse
 import json
 import logging
+import os
 import pathlib
 import sys
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:

@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",

@@ -95,19 +96,26 @@ def generate_crates(srctree, objtree, sysroot_src):
         "exclude_dirs": [],
     }
 
+    def is_root_crate(build_file, target):
+        try:
+            return f"{target}.o" in open(build_file).read()
+        except FileNotFoundError:
+            return False
+
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    for folder in ("samples", "drivers"):
-        for path in (srctree / folder).rglob("*.rs"):
+    extra_dirs = map(lambda dir: srctree / dir, ("samples", "drivers"))
+    if external_src is not None:
+        extra_dirs = [external_src]
+    for folder in extra_dirs:
+        for path in folder.rglob("*.rs"):
             logging.info("Checking %s", path)
             name = path.name.replace(".rs", "")
 
             # Skip those that are not crate roots.
-            try:
-                if f"{name}.o" not in open(path.parent / "Makefile").read():
-                    continue
-            except FileNotFoundError:
+            if not is_root_crate(path.parent / "Makefile", name) and \
+               not is_root_crate(path.parent / "Kbuild", name):
                 continue
 
             logging.info("Adding %s", name)

@@ -126,6 +134,7 @@ def main():
     parser.add_argument("srctree", type=pathlib.Path)
     parser.add_argument("objtree", type=pathlib.Path)
     parser.add_argument("sysroot_src", type=pathlib.Path)
+    parser.add_argument("exttree", type=pathlib.Path, nargs="?")
     args = parser.parse_args()
 
     logging.basicConfig(

@@ -134,7 +143,7 @@ def main():
     )
 
     rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
         "sysroot_src": str(args.sysroot_src),
     }

Signature · 1 line

2.40.0

Miguel Ojeda

2023-08-02 17:37 UTC #10 lore raw

Quoted 12 lines · Vinay Varma

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.
The change involves adding an optional parameter external_src to the generate_rust_analyzer.py which expects the path to the out-of-tree module's source directory. When this parameter is passed, I have chosen not to add the non-core modules (samples and drivers) into the result since these are not expected to be used in third party modules. Related changes are also made to the Makefile and rust/Makefile allowing the rust-analyzer target to be used for out-of-tree modules as well.

Quoted 4 lines

Link https://github.com/Rust-for-Linux/linux/pull/914
Link https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2
Signed-off-by Vinay Varma <[email protected]>

Applied to rust-next, thanks! I tested it and the generation of the file seems to work on the sample out-of-tree module. We can do the other improvements later on. I removed the newline above your SoB, by the way.

Cheers,
Miguel

Björn Roy Baron

reviewer Tested-by on v2 1/1

2023-01-21 16:40 UTC #11 lore raw

In reply to Vinay Varma

------- Original Message -------

Quoted 11 lines · Vinay Varma

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.
The change involves adding an optional parameter external_src to the generate_rust_analyzer.py which expects the path to the out-of-tree module's source directory. When this parameter is passed, I have chosen not to add the non-core modules (samples and drivers) into the result since these are not expected to be used in third party modules. Related changes are also made to the Makefile and rust/Makefile allowing the rust-analyzer target to be used for out-of-tree modules as well.

Quoted 4 lines

Link https://github.com/Rust-for-Linux/linux/pull/914
Link https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2
Signed-off-by Vinay Varma [email protected]

"make -C ../rust-for-linux M=$(pwd) rust-analyzer" produces a rust-project.json file in the source dir of the out of tree kernel module with a crate definition for the kernel module. In addition rust-analyzer is able to resolve definitions from the kernel crate as expected.

Tested-by Björn Roy Baron <[email protected]>

Quoted 146 lines

---
Makefile | 12 +++++++-----
rust/Makefile | 6 ++++--
scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
3 files changed, 29 insertions(+), 20 deletions(-)
Makefile +0 -0
index f41ec8c8426b..a055a316d2a4 100644
@@ -1831,11 +1831,6 @@ rustfmt:
rustfmtcheck: rustfmt_flags = --check
rustfmtcheck: rustfmt
Makefile +0 -5
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
- $(Q)$(MAKE) $(build)=rust $@
-
# Misc
# ---------------------------------------------------------------------------
Makefile +0 -0
@@ -1888,6 +1883,7 @@ help:
@echo ' modules - default target, build the module(s)'
@echo ' modules_install - install the module'
@echo ' clean - remove generated files in module directory only'
+ @echo ' rust-analyzer - generate rust-project.json rust-analyzer support file' @echo ''
endif # KBUILD_EXTMOD
Makefile +0 -0
@@ -2022,6 +2018,12 @@ quiet_cmd_tags = GEN $@
tags TAGS cscope gtags: FORCE
$(call cmd,tags)
Makefile +6 -0
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+ $(Q)$(MAKE) $(build)=rust $@
+
+
# Script to generate missing namespace dependencies
# ---------------------------------------------------------------------------
rust/Makefile +0 -0
index 8f598a904f38..41c1435cd8d4 100644
@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
rust-analyzer:
- $(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
- $(RUST_LIB_SRC) > $(objtree)/rust-project.json
+ $(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+ $(abs_srctree) $(abs_objtree) \
+ $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+ $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
$(obj)/core.o: private skip_clippy = 1 $(obj)/core.o: private skip_flags = -Dunreachable_pub
scripts/generate_rust_analyzer.py +0 -0
index ecc7ea9a4dcf..1792f379ee4e 100755
@@ -6,10 +6,11 @@
import argparse
import json
import logging
+import os
import pathlib
import sys
scripts/generate_rust_analyzer.py +1 -1
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
# Generate the configuration list.
cfg = []
with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
scripts/generate_rust_analyzer.py +1 -1
@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
 [],
 is_proc_macro=True,
 )
- crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+ crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
append_crate(
"build_error",
scripts/generate_rust_analyzer.py +4 -1
@@ -95,25 +96,28 @@ def generate_crates(srctree, objtree, sysroot_src):
 "exclude_dirs": [],
 }
 
+ def is_root_crate(build_file, target):
+ return os.path.exists(build_file) and target in open(build_file).read()
+
 # Then, the rest outside of `rust/`.
 #
 # We explicitly mention the top-level folders we want to cover.
- for folder in ("samples", "drivers"):
+ for folder in ("samples", "drivers") if external_src is None else [external_src]:
for path in (srctree / folder).rglob("*.rs"):
logging.info("Checking %s", path)
name = path.name.replace(".rs", "")
# Skip those that are not crate roots.
- if f"{name}.o" not in open(path.parent / "Makefile").read():
- continue
+ if is_root_crate(path.parent / "Makefile", f"{name}.o") or \
+ is_root_crate(path.parent / "Kbuild", f"{name}.o"):
- logging.info("Adding %s", name)
- append_crate(
- name,
- path,
- ["core", "alloc", "kernel"],
- cfg=cfg,
- )
+ logging.info("Adding %s", name)
+ append_crate(
+ name,
+ path,
+ ["core", "alloc", "kernel"],
+ cfg=cfg,
+ )
return crates
scripts/generate_rust_analyzer.py +1 -0
@@ -123,6 +127,7 @@ def main():
 parser.add_argument("srctree", type=pathlib.Path)
 parser.add_argument("objtree", type=pathlib.Path)
 parser.add_argument("sysroot_src", type=pathlib.Path)
+ parser.add_argument("exttree", type=pathlib.Path, nargs='?')
 args = parser.parse_args()
 
logging.basicConfig(
scripts/generate_rust_analyzer.py +1 -2
@@ -131,7 +136,7 @@ def main():
 )
 
 rust_project = {
- "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+ "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
 "sysroot_src": str(args.sysroot_src),
 }
 
--

Quoted 1 line

2.39.0

Cheers,
Björn

(resend as I forgot to do reply to all)

Miguel Ojeda

2023-03-07 12:25 UTC #12 lore raw

In reply to Vinay Varma

Hi Vinay,

Quoted 3 lines · Vinay Varma

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.

Could you please address (some of) Masahiro's and Alice's feedback in a new version?

Thanks!

Cheers,
Miguel

Vinay Varma

author

2023-03-07 15:00 UTC #13 lore raw

PATCH v3 1/1 scripts: `make rust-analyzer` for out-of-tree modules

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.

Link https://github.com/Rust-for-Linux/linux/pull/914

Link https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2

Signed-off-by Vinay Varma <[email protected]>

---
 Makefile                          | 11 ++++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 31 ++++++++++++++++++-------------
 3 files changed, 28 insertions(+), 20 deletions(-)

Makefile +6 -5

index 3f6628780eb2..ee7b61dc6075 100644
@@ -1860,11 +1860,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-	$(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------

@@ -1917,6 +1912,7 @@ help:
 	@echo  '  modules         - default target, build the module(s)'
 	@echo  '  modules_install - install the module'
 	@echo  '  clean           - remove generated files in module directory only'
+	@echo  '  rust-analyzer	  - generate rust-project.json rust-analyzer support file'
 	@echo  ''
 
 endif # KBUILD_EXTMOD

@@ -2053,6 +2049,11 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
 	$(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------

rust/Makefile +4 -2

index 8f598a904f38..41c1435cd8d4 100644
@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+		$(abs_srctree) $(abs_objtree) \
+		$(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+		$(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
 
 $(obj)/core.o: private skip_clippy = 1
 $(obj)/core.o: private skip_flags = -Dunreachable_pub

scripts/generate_rust_analyzer.py +18 ~~-13~~

index ecc7ea9a4dcf..741ccd6efe2b 100755
@@ -6,10 +6,11 @@
 import argparse
 import json
 import logging
+import os
 import pathlib
 import sys
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:

@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",

@@ -95,25 +96,28 @@ def generate_crates(srctree, objtree, sysroot_src):
         "exclude_dirs": [],
     }
 
+    def is_root_crate(build_file, target):
+        return os.path.exists(build_file) and target in open(build_file).read()
+
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    for folder in ("samples", "drivers"):
+    for folder in ("samples", "drivers") if external_src is None else [external_src]:
         for path in (srctree / folder).rglob("*.rs"):
             logging.info("Checking %s", path)
             name = path.name.replace(".rs", "")
 
             # Skip those that are not crate roots.
-            if f"{name}.o" not in open(path.parent / "Makefile").read():
-                continue
+            if is_root_crate(path.parent / "Makefile", f"{name}.o") or \
+               is_root_crate(path.parent / "Kbuild", f"{name}.o"):
 
-            logging.info("Adding %s", name)
-            append_crate(
-                name,
-                path,
-                ["core", "alloc", "kernel"],
-                cfg=cfg,
-            )
+                logging.info("Adding %s", name)
+                append_crate(
+                    name,
+                    path,
+                    ["core", "alloc", "kernel"],
+                    cfg=cfg,
+                )
 
     return crates

@@ -123,6 +127,7 @@ def main():
     parser.add_argument("srctree", type=pathlib.Path)
     parser.add_argument("objtree", type=pathlib.Path)
     parser.add_argument("sysroot_src", type=pathlib.Path)
+    parser.add_argument("exttree", type=pathlib.Path, nargs="?")
     args = parser.parse_args()
 
     logging.basicConfig(

@@ -131,7 +136,7 @@ def main():
     )
 
     rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
         "sysroot_src": str(args.sysroot_src),
     }

Signature · 1 line

2.39.2

Vinay Varma

author

2023-03-07 15:59 UTC #14 lore raw

PATCH v4 1/1 scripts: `make rust-analyzer` for out-of-tree modules

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.

Link https://github.com/Rust-for-Linux/linux/pull/914

Link https://github.com/Rust-for-Linux/rust-out-of-tree-module/pull/2

Signed-off-by Vinay Varma <[email protected]>

---
 Makefile                          | 11 +++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 36 +++++++++++++++++++------------
 3 files changed, 32 insertions(+), 21 deletions(-)

Makefile +6 -5

index 3f6628780eb2..ee7b61dc6075 100644
@@ -1860,11 +1860,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-	$(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------

@@ -1917,6 +1912,7 @@ help:
 	@echo  '  modules         - default target, build the module(s)'
 	@echo  '  modules_install - install the module'
 	@echo  '  clean           - remove generated files in module directory only'
+	@echo  '  rust-analyzer	  - generate rust-project.json rust-analyzer support file'
 	@echo  ''
 
 endif # KBUILD_EXTMOD

@@ -2053,6 +2049,11 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
 	$(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------

rust/Makefile +4 -2

index 8f598a904f38..41c1435cd8d4 100644
@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+		$(abs_srctree) $(abs_objtree) \
+		$(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+		$(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
 
 $(obj)/core.o: private skip_clippy = 1
 $(obj)/core.o: private skip_flags = -Dunreachable_pub

scripts/generate_rust_analyzer.py +22 ~~-14~~

index ecc7ea9a4dcf..7a1a958b1f8a 100755
@@ -6,10 +6,11 @@
 import argparse
 import json
 import logging
+import os
 import pathlib
 import sys
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:

@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",

@@ -95,25 +96,31 @@ def generate_crates(srctree, objtree, sysroot_src):
         "exclude_dirs": [],
     }
 
+    def is_root_crate(build_file, target):
+        return os.path.exists(build_file) and target in open(build_file).read()
+
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    for folder in ("samples", "drivers"):
-        for path in (srctree / folder).rglob("*.rs"):
+    extra_dirs = map(lambda dir: srctree / dir, ("samples", "drivers"))
+    if external_src is not None:
+        extra_dirs = [external_src]
+    for folder in extra_dirs:
+        for path in folder.rglob("*.rs"):
             logging.info("Checking %s", path)
             name = path.name.replace(".rs", "")
 
             # Skip those that are not crate roots.
-            if f"{name}.o" not in open(path.parent / "Makefile").read():
-                continue
+            if is_root_crate(path.parent / "Makefile", f"{name}.o") or \
+               is_root_crate(path.parent / "Kbuild", f"{name}.o"):
 
-            logging.info("Adding %s", name)
-            append_crate(
-                name,
-                path,
-                ["core", "alloc", "kernel"],
-                cfg=cfg,
-            )
+                logging.info("Adding %s", name)
+                append_crate(
+                    name,
+                    path,
+                    ["core", "alloc", "kernel"],
+                    cfg=cfg,
+                )
 
     return crates

@@ -123,6 +130,7 @@ def main():
     parser.add_argument("srctree", type=pathlib.Path)
     parser.add_argument("objtree", type=pathlib.Path)
     parser.add_argument("sysroot_src", type=pathlib.Path)
+    parser.add_argument("exttree", type=pathlib.Path, nargs="?")
     args = parser.parse_args()
 
     logging.basicConfig(

@@ -131,7 +139,7 @@ def main():
     )
 
     rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
         "sysroot_src": str(args.sysroot_src),
     }

Signature · 1 line

2.39.2

Alice Ferrazzi

Reviewed-by Tested-by on v1 1/1

2023-01-20 15:32 UTC #15 lore raw

In reply to Vinay Varma

Quoted 115 lines · Vinay Varma

Adds support for out-of-tree rust modules to use the rust-analyzer make target to generate the rust-project.json file.

Signed-off-by Vinay Varma <[email protected]>

---
 Makefile                          | 12 +++++++-----
 rust/Makefile                     |  6 ++++--
 scripts/generate_rust_analyzer.py | 14 +++++++++-----
 3 files changed, 20 insertions(+), 12 deletions(-)

Makefile +7 -5

index f41ec8c8426b..a055a316d2a4 100644
@@ -1831,11 +1831,6 @@ rustfmt:
 rustfmtcheck: rustfmt_flags = --check
 rustfmtcheck: rustfmt
 
-# IDE support targets
-PHONY += rust-analyzer
-rust-analyzer:
-	$(Q)$(MAKE) $(build)=rust $@
-
 # Misc
 # ---------------------------------------------------------------------------

@@ -1888,6 +1883,7 @@ help:
 	@echo  '  modules         - default target, build the module(s)'
 	@echo  '  modules_install - install the module'
 	@echo  '  clean           - remove generated files in module directory only'
+	@echo  '  rust-analyzer	  - generate rust-project.json rust-analyzer support file'
 	@echo  ''
 
 endif # KBUILD_EXTMOD

@@ -2022,6 +2018,12 @@ quiet_cmd_tags = GEN     $@
 tags TAGS cscope gtags: FORCE
 	$(call cmd,tags)
 
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
+
 # Script to generate missing namespace dependencies
 # ---------------------------------------------------------------------------

rust/Makefile +4 -2

index 8f598a904f38..41c1435cd8d4 100644
@@ -389,8 +389,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
 
 rust-analyzer:
-	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
-		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py \
+		$(abs_srctree) $(abs_objtree) \
+		$(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
+		$(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
 
 $(obj)/core.o: private skip_clippy = 1
 $(obj)/core.o: private skip_flags = -Dunreachable_pub

scripts/generate_rust_analyzer.py +7 -4

index ecc7ea9a4dcf..1546b80db554 100755
@@ -8,8 +8,9 @@ import json
 import logging
 import pathlib
 import sys
+import os
 
-def generate_crates(srctree, objtree, sysroot_src):
+def generate_crates(srctree, objtree, sysroot_src, external_src):
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:

@@ -65,7 +66,7 @@ def generate_crates(srctree, objtree, sysroot_src):
         [],
         is_proc_macro=True,
     )
-    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+    crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so"
 
     append_crate(
         "build_error",

@@ -98,13 +99,15 @@ def generate_crates(srctree, objtree, sysroot_src):
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    for folder in ("samples", "drivers"):
+    extra_src_dirs = ["samples", "drivers"] if external_src is None else [external_src]
+
+    for folder in extra_src_dirs:
         for path in (srctree / folder).rglob("*.rs"):
             logging.info("Checking %s", path)
             name = path.name.replace(".rs", "")
 
             # Skip those that are not crate roots.
-            if f"{name}.o" not in open(path.parent / "Makefile").read():
+            if os.path.exists(path.parent / "Makefile") and f"{name}.o" not in open(path.parent / "Makefile").read():
                 continue
 
             logging.info("Adding %s", name)

scripts/generate_rust_analyzer.py def main(): · line 129

Quoted 4 lines

scripts/generate_rust_analyzer.py +1 -0

@@ -123,6 +126,7 @@ def main():
     parser.add_argument("srctree", type=pathlib.Path)
     parser.add_argument("objtree", type=pathlib.Path)
     parser.add_argument("sysroot_src", type=pathlib.Path)
+    parser.add_argument("exttree", type=pathlib.Path, nargs='?')

I would prefer to don't mix '' and "" unless necessary

Quoted 14 lines

scripts/generate_rust_analyzer.py +1 -1

     args = parser.parse_args()
 
     logging.basicConfig(

@@ -131,7 +135,7 @@ def main():
     )
 
     rust_project = {
-        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree),
         "sysroot_src": str(args.sysroot_src),
     }

Signature · 1 line

2.39.0

I could test it on my enviroment with rust-out-of-tree-module and rustc 1.66
it correctly generated a json file rust-project.json with correct path

As following from github conversation:

Reviewed-by Alice Ferrazzi <[email protected]>

Tested-by Alice Ferrazzi <[email protected]>

Thanks,
Alicef