package/pkg-python: add pyo3(setuptools-rust/maturin) infrastructure
PATCH v4 1/5 package/pkg-python: add pyo3(setuptools-rust/maturin) infrastructure
The pyo3 package infrastructure provides two build backends, one of which is based on setuptools while the other is a pep517 backend.
The setuptools-rust build backend is a setuptools extension that is capable of building python rust extensions.
The maturin build backend is a pep517 build extension that is itself written in rust, it is itself bootstrapped using setuptools-rust but is not itself a setuptools extension and uses pep517 infrastructure.
As both backends use pyo3 we need to set up the pyo3 environment in addition to cargo vendoring for each.
---
Changes v3 -> v4:
- generalize infrastructure to support both pyo3 build backends
--- package/pkg-python.mk | 92 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 5 deletions(-)
package/pkg-python.mk +87 -5
@@ -113,6 +113,24 @@ HOST_PKG_PYTHON_SETUPTOOLS_INSTALL_OPTS = \ --root=/ \ --single-version-externally-managed +PKG_PYTHON_PYO3_ENV = \ + $(PKG_CARGO_ENV) \ + PYO3_CROSS_LIB_DIR="$(STAGING_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)" + +HOST_PKG_PYTHON_PYO3_ENV = \ + $(HOST_PKG_CARGO_ENV) \ + PYO3_CROSS_LIB_DIR="$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)" + +# Target setuptools-rust-based packages +PKG_PYTHON_SETUPTOOLS_RUST_ENV = \ + $(PKG_PYTHON_SETUPTOOLS_ENV) \ + $(PKG_PYTHON_PYO3_ENV) + +# Host setuptools-rust-based packages +HOST_PKG_PYTHON_SETUPTOOLS_RUST_ENV = \ + $(HOST_PKG_PYTHON_SETUPTOOLS_ENV) \ + $(HOST_PKG_PYTHON_PYO3_ENV) + # Target pep517-based packages PKG_PYTHON_PEP517_ENV = \ $(PKG_PYTHON_ENV)
@@ -142,6 +160,16 @@ HOST_PKG_PYTHON_PEP517_ENV = \ HOST_PKG_PYTHON_PEP517_BOOTSTRAP_INSTALL_OPTS = \ --installdir=$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages +# Target maturin packages +PKG_PYTHON_MATURIN_ENV = \ + $(PKG_PYTHON_PEP517_ENV) \ + $(PKG_PYTHON_PYO3_ENV) + +# Host maturin packages +HOST_PKG_PYTHON_MATURIN_ENV = \ + $(HOST_PKG_PYTHON_PEP517_ENV) \ + $(HOST_PKG_PYTHON_PYO3_ENV) + ################################################################################ # inner-python-package -- defines how the configuration, compilation # and installation of a Python package should be done, implements a
@@ -180,25 +208,41 @@ $(2)_BASE_BUILD_CMD = setup.py build $(2)_BASE_INSTALL_CMD = setup.py install $$(HOST_PKG_PYTHON_DISTUTILS_INSTALL_OPTS) endif # Setuptools -else ifeq ($$($(2)_SETUP_TYPE),setuptools) +else ifneq ($$(filter setuptools setuptools-rust,$$($(2)_SETUP_TYPE)),) ifeq ($(4),target) +ifeq ($$($(2)_SETUP_TYPE),setuptools-rust) +$(2)_BASE_ENV = $$(PKG_PYTHON_SETUPTOOLS_RUST_ENV) +else $(2)_BASE_ENV = $$(PKG_PYTHON_SETUPTOOLS_ENV) +endif $(2)_BASE_BUILD_CMD = $$(PKG_PYTHON_SETUPTOOLS_CMD) build $(2)_BASE_INSTALL_TARGET_CMD = $$(PKG_PYTHON_SETUPTOOLS_CMD) install --no-compile $$(PKG_PYTHON_SETUPTOOLS_INSTALL_TARGET_OPTS) $(2)_BASE_INSTALL_STAGING_CMD = $$(PKG_PYTHON_SETUPTOOLS_CMD) install $$(PKG_PYTHON_SETUPTOOLS_INSTALL_STAGING_OPTS) else +ifeq ($$($(2)_SETUP_TYPE),setuptools-rust) +$(2)_BASE_ENV = $$(HOST_PKG_PYTHON_SETUPTOOLS_RUST_ENV) +else $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_SETUPTOOLS_ENV) +endif $(2)_BASE_BUILD_CMD = $$(PKG_PYTHON_SETUPTOOLS_CMD) build $(2)_BASE_INSTALL_CMD = $$(PKG_PYTHON_SETUPTOOLS_CMD) install $$(HOST_PKG_PYTHON_SETUPTOOLS_INSTALL_OPTS) endif -else ifneq ($$(filter flit pep517,$$($(2)_SETUP_TYPE)),) +else ifneq ($$(filter flit maturin pep517,$$($(2)_SETUP_TYPE)),) ifeq ($(4),target) +ifeq ($$($(2)_SETUP_TYPE),maturin) +$(2)_BASE_ENV = $$(PKG_PYTHON_MATURIN_ENV) +else $(2)_BASE_ENV = $$(PKG_PYTHON_PEP517_ENV) +endif $(2)_BASE_BUILD_CMD = -m build -n -w $(2)_BASE_INSTALL_TARGET_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(PKG_PYTHON_PEP517_INSTALL_TARGET_OPTS) $(2)_BASE_INSTALL_STAGING_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(PKG_PYTHON_PEP517_INSTALL_STAGING_OPTS) else +ifeq ($$($(2)_SETUP_TYPE),maturin) +$(2)_BASE_ENV = $$(HOST_PKG_PYTHON_MATURIN_ENV) +else $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV) +endif $(2)_BASE_BUILD_CMD = -m build -n -w $(2)_BASE_INSTALL_CMD = -m installer dist/* endif
@@ -211,7 +255,29 @@ $(2)_BASE_BUILD_CMD = -m flit_core.wheel $(2)_BASE_INSTALL_CMD ?= -m installer dist/* endif else -$$(error "Invalid $(2)_SETUP_TYPE. Valid options are 'distutils', 'setuptools', 'pep517' or 'flit'.") +$$(error "Invalid $(2)_SETUP_TYPE. Valid options are 'distutils', 'maturin', 'setuptools', 'setuptools-rust', 'pep517' or 'flit'.") +endif + +# We need to vendor the Cargo crates at download time for pyo3 based +# packages. +# +ifneq ($$(filter maturin setuptools-rust,$$($(2)_SETUP_TYPE)),) +ifeq ($(4),target) +$(2)_DL_ENV = $$(PKG_CARGO_ENV) +else +$(2)_DL_ENV = $$(HOST_PKG_CARGO_ENV) +endif +ifndef $(2)_CARGO_MANIFEST_PATH +ifdef $(3)_CARGO_MANIFEST_PATH +$(2)_DL_ENV += BR_CARGO_MANIFEST_PATH=$$($(3)_CARGO_MANIFEST_PATH) +else +ifneq ($$($(2)_SUBDIR),) +$(2)_DL_ENV += BR_CARGO_MANIFEST_PATH=$$($(2)_SUBDIR)/Cargo.toml +endif +endif +else +$(2)_DL_ENV += BR_CARGO_MANIFEST_PATH=$$($(2)_CARGO_MANIFEST_PATH) +endif endif # Target packages need both the python interpreter on the target (for
@@ -228,19 +294,35 @@ endif # ($(4),target) # Setuptools based packages will need setuptools for the host Python # interpreter (both host and target). # -ifeq ($$($(2)_SETUP_TYPE),setuptools) +ifneq ($$(filter setuptools setuptools-rust,$$($(2)_SETUP_TYPE)),) $(2)_DEPENDENCIES += $$(if $$(filter host-python-setuptools,$(1)),,host-python-setuptools) -else ifneq ($$(filter flit pep517,$$($(2)_SETUP_TYPE)),) +ifeq ($$($(2)_SETUP_TYPE),setuptools-rust) +$(2)_DEPENDENCIES += host-python-setuptools-rust +endif +else ifneq ($$(filter flit maturin pep517,$$($(2)_SETUP_TYPE)),) $(2)_DEPENDENCIES += host-python-pypa-build host-python-installer ifeq ($$($(2)_SETUP_TYPE),flit) $(2)_DEPENDENCIES += host-python-flit-core endif +ifeq ($$($(2)_SETUP_TYPE),maturin) +$(2)_DEPENDENCIES += host-python-maturin +endif else ifeq ($$($(2)_SETUP_TYPE),flit-bootstrap) # Don't add dependency on host-python-installer for # host-python-installer itself, and its dependencies. ifeq ($$(filter host-python-flit-core host-python-installer,$(1)),) $(2)_DEPENDENCIES += host-python-installer endif +endif + +# Pyo3 based packages(setuptools-rust and maturin) will need rust +# toolchain dependencies for the host Python interpreter (both host +# and target). +# +ifneq ($$(filter maturin setuptools-rust,$$($(2)_SETUP_TYPE)),) +$(2)_DEPENDENCIES += host-rustc +$(2)_DOWNLOAD_POST_PROCESS = cargo +$(2)_DOWNLOAD_DEPENDENCIES = host-rustc endif # SETUP_TYPE # Python interpreter to use for building the package.
Signature · 6 lines
2.34.1 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
PATCH v4 2/5 package/python-maturin: migrate to setuptools-rust infrastructure
The python-maturin build backend while itself is a pep517 backend uses setuptools-rust for bootstrapping purposes.
--- package/python-maturin/python-maturin.mk | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-)
package/python-maturin/python-maturin.mk +2 -12
@@ -7,19 +7,9 @@ PYTHON_MATURIN_VERSION = 0.12.20 PYTHON_MATURIN_SOURCE = maturin-$(PYTHON_MATURIN_VERSION).tar.gz PYTHON_MATURIN_SITE = https://files.pythonhosted.org/packages/39/ba/fec598956393230341def42367c6ad0efbee6ddc899f8858e5b25a7f37a5 -PYTHON_MATURIN_SETUP_TYPE = setuptools +PYTHON_MATURIN_SETUP_TYPE = setuptools-rust PYTHON_MATURIN_LICENSE = Apache-2.0 or MIT PYTHON_MATURIN_LICENSE_FILES = license-apache license-mit -HOST_PYTHON_MATURIN_DEPENDENCIES = \ - host-python-setuptools-rust \ - host-python-tomli \ - host-rustc -HOST_PYTHON_MATURIN_ENV = \ - $(HOST_PKG_CARGO_ENV) \ - PYO3_CROSS_LIB_DIR="$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)" -# We need to vendor the Cargo crates at download time -PYTHON_MATURIN_DOWNLOAD_POST_PROCESS = cargo -PYTHON_MATURIN_DOWNLOAD_DEPENDENCIES = host-rustc -HOST_PYTHON_MATURIN_DL_ENV = $(HOST_PKG_CARGO_ENV) +HOST_PYTHON_MATURIN_DEPENDENCIES = host-python-tomli $(eval $(host-python-package))
Signature · 6 lines
2.34.1 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
PATCH v4 3/5 package/python-cryptography: migrate to setuptools-rust infrastructure
In reply to James HilliardWe can now significantly simplify the python-cryptography build using the new setuptools-rust setup type introduced in the python package infrastructure.
---
Changes v2 -> v3:
- rebase
--- .../python-cryptography.mk | 30 +++---------------- 1 file changed, 4 insertions(+), 26 deletions(-)
package/python-cryptography/python-cryptography.mk +4 -26
@@ -7,36 +7,14 @@ PYTHON_CRYPTOGRAPHY_VERSION = 37.0.4 PYTHON_CRYPTOGRAPHY_SOURCE = cryptography-$(PYTHON_CRYPTOGRAPHY_VERSION).tar.gz PYTHON_CRYPTOGRAPHY_SITE = https://files.pythonhosted.org/packages/89/d9/5fcd312d5cce0b4d7ee8b551a0ea99e4ea9db0fdbf6dd455a19042e3370b -PYTHON_CRYPTOGRAPHY_SETUP_TYPE = setuptools +PYTHON_CRYPTOGRAPHY_SETUP_TYPE = setuptools-rust PYTHON_CRYPTOGRAPHY_LICENSE = Apache-2.0 or BSD-3-Clause PYTHON_CRYPTOGRAPHY_LICENSE_FILES = LICENSE LICENSE.APACHE LICENSE.BSD PYTHON_CRYPTOGRAPHY_CPE_ID_VENDOR = cryptography_project PYTHON_CRYPTOGRAPHY_CPE_ID_PRODUCT = cryptography -PYTHON_CRYPTOGRAPHY_DEPENDENCIES = \ - host-python-setuptools-rust \ - host-python-cffi \ - host-rustc \ - openssl -HOST_PYTHON_CRYPTOGRAPHY_DEPENDENCIES = \ - host-python-setuptools-rust \ - host-python-cffi \ - host-rustc \ - host-openssl -PYTHON_CRYPTOGRAPHY_ENV = \ - $(PKG_CARGO_ENV) \ - PYO3_CROSS_LIB_DIR="$(STAGING_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)" -HOST_PYTHON_CRYPTOGRAPHY_ENV = \ - $(HOST_PKG_CARGO_ENV) \ - PYO3_CROSS_LIB_DIR="$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)" -# We need to vendor the Cargo crates at download time -PYTHON_CRYPTOGRAPHY_DOWNLOAD_POST_PROCESS = cargo -PYTHON_CRYPTOGRAPHY_DOWNLOAD_DEPENDENCIES = host-rustc -PYTHON_CRYPTOGRAPHY_DL_ENV = \ - $(PKG_CARGO_ENV) \ - BR_CARGO_MANIFEST_PATH=src/rust/Cargo.toml -HOST_PYTHON_CRYPTOGRAPHY_DL_ENV = \ - $(HOST_PKG_CARGO_ENV) \ - BR_CARGO_MANIFEST_PATH=src/rust/Cargo.toml +PYTHON_CRYPTOGRAPHY_CARGO_MANIFEST_PATH = src/rust/Cargo.toml +PYTHON_CRYPTOGRAPHY_DEPENDENCIES = host-python-cffi openssl +HOST_PYTHON_CRYPTOGRAPHY_DEPENDENCIES = host-python-cffi host-openssl $(eval $(python-package)) $(eval $(host-python-package))
Signature · 6 lines
2.34.1 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
PATCH v4 4/5 package/python-orjson: migrate to maturin infrastructure
In reply to James HilliardWe can now significantly simplify the python-orjson build using the new maturin setup type introduced in the python package infrastructure.
--- package/python-orjson/python-orjson.mk | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)
package/python-orjson/python-orjson.mk +2 -9
@@ -7,16 +7,9 @@ PYTHON_ORJSON_VERSION = 3.7.7 PYTHON_ORJSON_SOURCE = orjson-$(PYTHON_ORJSON_VERSION).tar.gz PYTHON_ORJSON_SITE = https://files.pythonhosted.org/packages/8a/68/47bc5e624431d77445f06a2d3ba32777fda650f05a2f803ea1dc2b07b65a -PYTHON_ORJSON_SETUP_TYPE = pep517 +PYTHON_ORJSON_SETUP_TYPE = maturin PYTHON_ORJSON_LICENSE = Apache-2.0 or MIT PYTHON_ORJSON_LICENSE_FILES = LICENSE-APACHE LICENSE-MIT -PYTHON_ORJSON_DEPENDENCIES = host-python-cffi host-python-maturin -PYTHON_ORJSON_ENV = \ - $(PKG_CARGO_ENV) \ - PYO3_CROSS_LIB_DIR="$(STAGING_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)" -# We need to vendor the Cargo crates at download time -PYTHON_ORJSON_DOWNLOAD_POST_PROCESS = cargo -PYTHON_ORJSON_DOWNLOAD_DEPENDENCIES = host-rustc -PYTHON_ORJSON_DL_ENV = $(PKG_CARGO_ENV) +PYTHON_ORJSON_DEPENDENCIES = host-python-cffi $(eval $(python-package))
Signature · 6 lines
2.34.1 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
PATCH v4 5/5 package/python-rtoml: migrate to setuptools-rust infrastructure
In reply to James HilliardWe can now significantly simplify the python-rtoml build using the new setuptools-rust setup type introduced in the python package infrastructure.
--- package/python-rtoml/python-rtoml.mk | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-)
package/python-rtoml/python-rtoml.mk +1 -9
@@ -7,16 +7,8 @@ PYTHON_RTOML_VERSION = 0.8.0 PYTHON_RTOML_SOURCE = rtoml-$(PYTHON_RTOML_VERSION).tar.gz PYTHON_RTOML_SITE = https://files.pythonhosted.org/packages/33/a6/b42d8e0e28bec9fd7fdbafb2d76db3f8578f151a669eba564d422756d909 -PYTHON_RTOML_SETUP_TYPE = setuptools +PYTHON_RTOML_SETUP_TYPE = setuptools-rust PYTHON_RTOML_LICENSE = MIT PYTHON_RTOML_LICENSE_FILES = LICENSE -PYTHON_RTOML_DEPENDENCIES = host-python-setuptools-rust host-rustc -PYTHON_RTOML_ENV = \ - $(PKG_CARGO_ENV) \ - PYO3_CROSS_LIB_DIR="$(STAGING_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)" -# We need to vendor the Cargo crates at download time -PYTHON_RTOML_DOWNLOAD_POST_PROCESS = cargo -PYTHON_RTOML_DOWNLOAD_DEPENDENCIES = host-rustc -PYTHON_RTOML_DL_ENV = $(PKG_CARGO_ENV) $(eval $(python-package))
Signature · 6 lines
2.34.1 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 6 lines · James Hilliard
The pyo3 package infrastructure provides two build backends, one of which is based on setuptools while the other is a pep517 backend.
The setuptools-rust build backend is a setuptools extension that is capable of building python rust extensions.
Quoted 8 lines
The maturin build backend is a pep517 build extension that is itself written in rust, it is itself bootstrapped using setuptools-rust but is not itself a setuptools extension and uses pep517 infrastructure.
As both backends use pyo3 we need to set up the pyo3 environment in addition to cargo vendoring for each.
Signed-off-by James Hilliard <[email protected]>
Series _finally_ applied to master, thanks!
Quoted 12 lines
---
Changes v3 -> v4:
- generalize infrastructure to support both pyo3 build backends--- package/pkg-python.mk | 92 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 5 deletions(-)package/pkg-python.mk +0
-0
package/pkg-python.mk HOST_PKG_PYTHON_SETUPTOOLS_INSTALL_OPTS = \ · line 118
Quoted 6 lines
package/pkg-python.mk +3
-0@@ -113,6 +113,24 @@ HOST_PKG_PYTHON_SETUPTOOLS_INSTALL_OPTS = \ --root=/ \ --single-version-externally-managed +PKG_PYTHON_PYO3_ENV = \ + $(PKG_CARGO_ENV) \ + PYO3_CROSS_LIB_DIR="$(STAGING_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)"
This _PYO3_ENV variables aren't contributing much and make the code even more
difficult to read, so I removed them.
Quoted 36 lines
package/pkg-python.mk +25
-0+ +HOST_PKG_PYTHON_PYO3_ENV = \ + $(HOST_PKG_CARGO_ENV) \ + PYO3_CROSS_LIB_DIR="$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)" + +# Target setuptools-rust-based packages +PKG_PYTHON_SETUPTOOLS_RUST_ENV = \ + $(PKG_PYTHON_SETUPTOOLS_ENV) \ + $(PKG_PYTHON_PYO3_ENV) + +# Host setuptools-rust-based packages +HOST_PKG_PYTHON_SETUPTOOLS_RUST_ENV = \ + $(HOST_PKG_PYTHON_SETUPTOOLS_ENV) \ + $(HOST_PKG_PYTHON_PYO3_ENV) + # Target pep517-based packages PKG_PYTHON_PEP517_ENV = \ $(PKG_PYTHON_ENV)@@ -142,6 +160,16 @@ HOST_PKG_PYTHON_PEP517_ENV = \ HOST_PKG_PYTHON_PEP517_BOOTSTRAP_INSTALL_OPTS = \ --installdir=$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages +# Target maturin packages +PKG_PYTHON_MATURIN_ENV = \ + $(PKG_PYTHON_PEP517_ENV) \ + $(PKG_PYTHON_PYO3_ENV) + +# Host maturin packages +HOST_PKG_PYTHON_MATURIN_ENV = \ + $(HOST_PKG_PYTHON_PEP517_ENV) \ + $(HOST_PKG_PYTHON_PYO3_ENV) + ################################################################################ # inner-python-package -- defines how the configuration, compilation # and installation of a Python package should be done, implements a
package/pkg-python.mk $(2)_BASE_BUILD_CMD = setup.py build · line 211
Quoted 5 lines
package/pkg-python.mk +1
-1@@ -180,25 +208,41 @@ $(2)_BASE_BUILD_CMD = setup.py build $(2)_BASE_INSTALL_CMD = setup.py install $$(HOST_PKG_PYTHON_DISTUTILS_INSTALL_OPTS) endif # Setuptools -else ifeq ($$($(2)_SETUP_TYPE),setuptools) +else ifneq ($$(filter setuptools setuptools-rust,$$($(2)_SETUP_TYPE)),)
This stuff is becoming increasingly difficult to read with all the different
setup types. I'm going to try to refactor it a little.
Regards,
Arnout
[snip]
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot