Skip to contentSkip to conversation

gentargets: add scp and Mercurial download helpers

PATCH Superseded Thomas De Schampheleire 2 participants 7 messages Sep 21, 2011 – Sep 22, 2011

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Thomas De Schampheleire

author
#0 lore raw

PATCH 1/1 gentargets: add scp and Mercurial download helpers

This patch series adds download methods for scp and Mercurial, and updates the documentation (including some typo fixes)

Signed-off-by Thomas De Schampheleire <[email protected]>
---
 Config.in                   |  12 ++++++++++++
 docs/buildroot.html         |  29 +++++++++++++++++------------
 package/Makefile.package.in |  62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 3 files changed, 86 insertions(+), 17 deletions(-)
author
#1 lore raw

PATCH 1/1 GENTARGETS: add support for scp://

This patch adds support for scp:// both for use in the package Makefiles, as for the BR2_PRIMARY_SITE variable.

This patch was based on the work of Richard Guy Briggs (see https://bugs.busybox.net/show_bug.cgi?id=3343).

Signed-off-by Thomas De Schampheleire <[email protected]>
---
 Config.in                   |   8 ++++++++
 package/Makefile.package.in |  34 +++++++++++++++++++++++++++++-----
 2 files changed, 37 insertions(+), 5 deletions(-)
Config.in +8 -0
@@ -32,6 +32,14 @@
 	string "Git command"
 	default "git"
 
+config BR2_SCP
+	string "Secure copy (scp) command"
+	default "scp"
+
+config BR2_SSH
+	string "Secure shell (ssh) command"
+	default "ssh"
+
 config BR2_ZCAT
 	string "zcat command"
 	default "gzip -d -c"
package/Makefile.package.in +31 -4
@@ -74,6 +74,8 @@
 SVN:=$(call qstrip,$(BR2_SVN)) $(QUIET)
 BZR:=$(call qstrip,$(BR2_BZR)) $(QUIET)
 GIT:=$(call qstrip,$(BR2_GIT)) $(QUIET)
+SCP:=$(call qstrip,$(BR2_SCP)) $(QUIET)
+SSH:=$(call qstrip,$(BR2_SSH)) $(QUIET)
 
 # Default spider mode is 'DOWNLOAD'. Other possible values are 'SOURCE_CHECK'
 # used by the _source-check target and 'SHOW_EXTERNAL_DEPS', used by the

@@ -85,19 +87,24 @@
 DL_DIR:=$(TOPDIR)/dl
 endif
 
+# URI scheme helper functions
+geturischeme=$(firstword $(subst ://, ,$(call qstrip,$(1))))
+stripurischeme=$(lastword $(subst ://, ,$(call qstrip,$(1))))
+
 ################################################################################
 # The DOWNLOAD_{GIT,SVN,BZR} helpers are in charge of getting a
 # working copy of the source repository for their corresponding SCM,
 # checking out the requested version / commit / tag, and create an
-# archive out of it. DOWNLOAD_WGET is the normal wget-based download
+# archive out of it. DOWNLOAD_SCP uses scp to obtain a remote file with
+# ssh authentication. DOWNLOAD_WGET is the normal wget-based download
 # mechanism.
 #
-# The SOURCE_CHECK_{GIT,SVN,BZR,WGET} helpers are in charge of simply
+# The SOURCE_CHECK_{GIT,SVN,BZR,SCP,WGET} helpers are in charge of simply
 # checking that the source is available for download. This can be used
 # to make sure one will be able to get all the sources needed for
 # one's build configuration.
 #
-# The SHOW_EXTERNAL_DEPS_{GIT,SVN,BZR,WGET} helpers simply output to
+# The SHOW_EXTERNAL_DEPS_{GIT,SVN,BZR,SCP,WGET} helpers simply output to
 # the console the names of the files that will be downloaded, or path
 # and revision of the source repositories, producing a list of all the
 # "external dependencies" of a given build configuration.

@@ -158,6 +165,20 @@
 endef
 
 
+define DOWNLOAD_SCP
+	test -e $(DL_DIR)/$(2) || \
+	$(SCP) $(call stripurischeme,$(call qstrip,$(1)))/$(2) $(DL_DIR)
+endef
+
+define SOURCE_CHECK_SCP
+	$(SSH) $(call stripurischeme,`echo "$(call qstrip,$(1))" | cut -d/ -f1`) ls /`echo "$(call qstrip,$(1))/$(2)" | cut -d/ -f2-` > /dev/null
+endef
+
+define SHOW_EXTERNAL_DEPS_SCP
+	echo "$($(PKG)_SITE) [scp: $($(PKG)_DL_VERSION)]"
+endef
+
+
 define DOWNLOAD_WGET
 	test -e $(DL_DIR)/$(2) || \
 	$(WGET) -P $(DL_DIR) $(call qstrip,$(1))/$(2)

@@ -186,13 +207,17 @@
 
 define DOWNLOAD
 	$(Q)if test -n "$(call qstrip,$(BR2_PRIMARY_SITE))" ; then \
-		$(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ; \
+		case "$(call geturischeme,$(BR2_PRIMARY_SITE))" in \
+			scp) $(call $(DL_MODE)_SCP,$(BR2_PRIMARY_SITE),$(2)) && exit ;; \
+			*) $(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ;; \
+		esac ; \
 	fi ; \
 	if test -n "$(1)" ; then \
 		case "$($(PKG)_SITE_METHOD)" in \
 			git) $($(DL_MODE)_GIT) && exit ;; \
 			svn) $($(DL_MODE)_SVN) && exit ;; \
 			bzr) $($(DL_MODE)_BZR) && exit ;; \
+			scp) $($(DL_MODE)_SCP) && exit ;; \
 			*) $(call $(DL_MODE)_WGET,$(1),$(2)) && exit ;; \
 		esac ; \
 	fi ; \

@@ -548,6 +573,8 @@
 DL_TOOLS_DEPENDENCIES += git
 else ifeq ($$($(2)_SITE_METHOD),bzr)
 DL_TOOLS_DEPENDENCIES += bzr
+else ifeq ($$($(2)_SITE_METHOD),scp)
+DL_TOOLS_DEPENDENCIES += scp ssh
 endif # SITE_METHOD
 
 endif # $(2)_KCONFIG_VAR
#2 lore raw

PATCH 1/1 GENTARGETS: add support for scp://

Le Wed, 21 Sep 2011 08:42:12 +0200,
Thomas De Schampheleire <[email protected]> a ?crit :

Quoted 2 lines

This patch adds support for scp:// both for use in the package Makefiles, as for the BR2_PRIMARY_SITE variable.

I'm ok with the principle. Some implementation comments below.

package/Makefile.package.in line 175

Quoted 8 lines
package/Makefile.package.in +8 -0
+define DOWNLOAD_SCP
+	test -e $(DL_DIR)/$(2) || \
+	$(SCP) $(call stripurischeme,$(call qstrip,$(1)))/$(2) $(DL_DIR)
+endef
+
+define SOURCE_CHECK_SCP
+	$(SSH) $(call stripurischeme,`echo "$(call qstrip,$(1))" | cut -d/ -f1`) ls /`echo "$(call qstrip,$(1))/$(2)" | cut -d/ -f2-` > /dev/null
+endef

Ouch, this looks ugly. I guess you can use "test -f" instead of using ls. Maybe something like:

scphost=$(firstword $(subst /, ,$(call stripurischeme,$(1)))) scpfile=$(patsubst $(call scphost,$(1))/%,/%/$(2),$(call stripurischeme,$(1)))

define SOURCE_CHECK_SCP

	$(SSH) $(call scphost,$(1),$(2)) test -f $(call scpfile,$(1),$(2))

endef

Maybe it's possible to do better and/or to make those scphost/scpfile macros a bit more generic. But the general comment is :

* Try to use make functions instead of shell functions when possible. Every shell function requires a fork+exec and is very costly compared to make functions. In the past, we had a single shell function call for every package, and this was causing a 5-10 seconds startup delay for the build process.

* Try to split in several small macros, whose name allows to understand what it is doing.

package/Makefile.package.in line 179

Quoted 3 lines
package/Makefile.package.in +3 -0
+define SHOW_EXTERNAL_DEPS_SCP
+	echo "$($(PKG)_SITE) [scp: $($(PKG)_DL_VERSION)]"
+endef

As per a recent discussion with Peter, external-deps shouldn't show the URL of the tarball of the package, but rather simply the tarball name, as found in the $(DL_DIR). I know this is not consistent with the current GIT/SVN/BZR helpers, but I have sent patches to fix this.

package/Makefile.package.in line 213

Quoted 12 lines

define DOWNLOAD_WGET

package/Makefile.package.in +4 -1
 	test -e $(DL_DIR)/$(2) || \
 	$(WGET) -P $(DL_DIR) $(call qstrip,$(1))/$(2)

@@ -186,13 +207,17 @@
 
 define DOWNLOAD
 	$(Q)if test -n "$(call qstrip,$(BR2_PRIMARY_SITE))" ; then \
-		$(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ; \
+		case "$(call geturischeme,$(BR2_PRIMARY_SITE))" in \
+			scp) $(call $(DL_MODE)_SCP,$(BR2_PRIMARY_SITE),$(2)) && exit ;; \
+			*) $(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ;; \
+		esac ; \

It would be good to update the BR2_PRIMARY_SITE help text to mention which URL types are possible.

Regards,

Thomas

Signature · 4 lines
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com
author
#3 lore raw

PATCH 1/1 Add support for packages stored in Mercurial (hg) repositories

In reply to Thomas De Schampheleire

Add support for packages stored in Mercurial (hg) repositories.

Signed-off-by Thomas De Schampheleire <[email protected]>
---
 Config.in                   |   4 ++++
 package/Makefile.package.in |  31 ++++++++++++++++++++++++++++---
 2 files changed, 32 insertions(+), 3 deletions(-)
Config.in +4 -0
@@ -40,6 +40,10 @@
 	string "Secure shell (ssh) command"
 	default "ssh"
 
+config BR2_HG
+	string "Mercurial (hg) command"
+	default "hg"
+
 config BR2_ZCAT
 	string "zcat command"
 	default "gzip -d -c"
package/Makefile.package.in +28 -3
@@ -76,6 +76,7 @@
 GIT:=$(call qstrip,$(BR2_GIT)) $(QUIET)
 SCP:=$(call qstrip,$(BR2_SCP)) $(QUIET)
 SSH:=$(call qstrip,$(BR2_SSH)) $(QUIET)
+HG:=$(call qstrip,$(BR2_HG)) $(QUIET)
 
 # Default spider mode is 'DOWNLOAD'. Other possible values are 'SOURCE_CHECK'
 # used by the _source-check target and 'SHOW_EXTERNAL_DEPS', used by the

@@ -92,19 +93,19 @@
 stripurischeme=$(lastword $(subst ://, ,$(call qstrip,$(1))))
 
 ################################################################################
-# The DOWNLOAD_{GIT,SVN,BZR} helpers are in charge of getting a
+# The DOWNLOAD_{GIT,SVN,BZR,HG} helpers are in charge of getting a
 # working copy of the source repository for their corresponding SCM,
 # checking out the requested version / commit / tag, and create an
 # archive out of it. DOWNLOAD_SCP uses scp to obtain a remote file with
 # ssh authentication. DOWNLOAD_WGET is the normal wget-based download
 # mechanism.
 #
-# The SOURCE_CHECK_{GIT,SVN,BZR,SCP,WGET} helpers are in charge of simply
+# The SOURCE_CHECK_{GIT,SVN,BZR,HG,SCP,WGET} helpers are in charge of simply
 # checking that the source is available for download. This can be used
 # to make sure one will be able to get all the sources needed for
 # one's build configuration.
 #
-# The SHOW_EXTERNAL_DEPS_{GIT,SVN,BZR,SCP,WGET} helpers simply output to
+# The SHOW_EXTERNAL_DEPS_{GIT,SVN,BZR,HG,SCP,WGET} helpers simply output to
 # the console the names of the files that will be downloaded, or path
 # and revision of the source repositories, producing a list of all the
 # "external dependencies" of a given build configuration.

@@ -179,6 +180,27 @@
 endef
 
 
+define DOWNLOAD_HG
+	test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
+	(pushd $(DL_DIR) > /dev/null && \
+	$(HG) clone --noupdate --rev $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_BASE_NAME) && \
+	$(HG) archive --repository $($(PKG)_BASE_NAME) --type tgz --prefix $($(PKG)_BASE_NAME)/ \
+	              --rev $($(PKG)_DL_VERSION) $(DL_DIR)/$($(PKG)_SOURCE) && \
+	rm -rf $($(PKG)_DL_DIR) && \
+	popd > /dev/null)
+endef
+
+# TODO: improve to check that the given PKG_DL_VERSION exists on the remote
+# repository
+define SOURCE_CHECK_HG
+  $(HG) incoming --force -l1 $($(PKG)_SITE) > /dev/null
+endef
+
+define SHOW_EXTERNAL_DEPS_HG
+  echo "$($(PKG)_SITE) [hg: $($(PKG)_DL_VERSION)]"
+endef
+
+
 define DOWNLOAD_WGET
 	test -e $(DL_DIR)/$(2) || \
 	$(WGET) -P $(DL_DIR) $(call qstrip,$(1))/$(2)

@@ -218,6 +240,7 @@
 			svn) $($(DL_MODE)_SVN) && exit ;; \
 			bzr) $($(DL_MODE)_BZR) && exit ;; \
 			scp) $($(DL_MODE)_SCP) && exit ;; \
+			hg) $($(DL_MODE)_HG) && exit ;; \
 			*) $(call $(DL_MODE)_WGET,$(1),$(2)) && exit ;; \
 		esac ; \
 	fi ; \

@@ -575,6 +598,8 @@
 DL_TOOLS_DEPENDENCIES += bzr
 else ifeq ($$($(2)_SITE_METHOD),scp)
 DL_TOOLS_DEPENDENCIES += scp ssh
+else ifeq ($$($(2)_SITE_METHOD),hg)
+DL_TOOLS_DEPENDENCIES += hg
 endif # SITE_METHOD
 
 endif # $(2)_KCONFIG_VAR
author
#4 lore raw

PATCH 1/1 Update documentation for scp and Mercurial support

In reply to Thomas De Schampheleire

Update documentation for scp and Mercurial support.
Along with the documentation update, a few typos were fixed as well.

Signed-off-by Thomas De Schampheleire <[email protected]>
---
 docs/buildroot.html |  29 +++++++++++++++++------------
 1 files changed, 17 insertions(+), 12 deletions(-)
docs/buildroot.html +17 -12
@@ -1025,8 +1025,8 @@
       version of the package. Note that
       if <code>HOST_LIBFOO_VERSION</code> doesn't exist, it is assumed
       to be the same as <code>LIBFOO_VERSION</code>. It can also be a
-      Subversion or Git branch or tag, for packages that are fetched
-      directly from their revision control system.<br/>
+      Subversion, Git, Bazaar or Mercurial branch or tag, for packages
+      that are fetched directly from their revision control system.<br/>
       Example: <code>LIBFOO_VERSION = 0.1.2</code></li>
 
       <li><code>LIBFOO_SOURCE</code> may contain the name of the tarball of

@@ -1047,7 +1047,8 @@
 
       <li><code>LIBFOO_SITE</code> may contain the Internet location
       of the package. It can either be the HTTP or FTP location of a
-      tarball, or the URL of a Git or Subversion repository
+      tarball, an SCP URI of a tarball accessible with Secure Copy, or
+      the URL of a Git, Subversion, Bazaar or Mercurial repository.
       (see <code>LIBFOO_SITE_METHOD</code>
       below). If <code>HOST_LIBFOO_SITE</code> is not specified, it
       defaults to <code>LIBFOO_SITE</code>. If none are specified,

@@ -1055,22 +1056,26 @@
       <code>http://$$(BR2_SOURCEFORGE_MIRROR).dl.sourceforge.net/sourceforge/packagename</code>.
       <br/>Examples:<br/>
       <code>LIBFOO_SITE=http://www.libfoosoftware.org/libfoo</code><br/>
-      <code>LIBFOO_SITE=http://svn.xiph.org/trunk/Tremor/</code></li>
+      <code>LIBFOO_SITE=http://svn.xiph.org/trunk/Tremor/</code><br/>
+      <code>LIBFOO_SITE=scp://machinename:/path/to/tarball</code><br/>
+      <code>LIBFOO_SITE=git://github.com/user/test.git</code></li>
 
       <li><code>LIBFOO_SITE_METHOD</code> may contain the method to
       fetch the package source code. It can either
       be <code>wget</code> (for normal FTP/HTTP downloads of
-      tarballs), <code>svn</code>, <code>git</code> or <code>bzr</code>.
+      tarballs), <code>scp</code>, <code>svn</code>, <code>git</code>,
+      <code>bzr</code> or <code>hg</code>.
       When not specified, it is guessed from the URL given
       in <code>LIBFOO_SITE</code>: <code>svn://</code>, <code>git://</code>
-      and <code>bzr://</code> URLs will use the <code>svn</code>,
-      <code>git</code> and <code>bzr</code> methods respectively. All other
+      <code>bzr://</code> and <code>scp://</code> URLs will use the
+      <code>svn</code>, <code>git</code>, <code>bzr</code> and
+      <code>scp</code>methods respectively. All other
       URL-types will use the <code>wget</code> method. So for example, in the
       case of a package whose source code is available through
       Subversion repository on HTTP, one <i>must</i>
-      specifiy <code>LIBFOO_SITE_METHOD=svn</code>. For <code>svn</code>
-      and <code>git</code> methods, what Buildroot does is a
-      checkout/clone of the repository which is then tarballed and
+      specify <code>LIBFOO_SITE_METHOD=svn</code>. For <code>svn</code>,
+      <code>git</code>, <code>bzr</code> and <code>hg</code> methods, what
+      Buildroot does is a checkout/clone of the repository which is then tarballed and
       stored into the download cache. Next builds will not
       checkout/clone again, but will use the tarball
       directly. When <code>HOST_LIBFOO_SITE_METHOD</code> is not

@@ -1184,7 +1189,7 @@
     However, since they are provided by the generic infrastructure, they are
     documented here. The exception is <code>LIBFOO_POST_PATCH_HOOKS</code>.
     Patching the package is not user definable, so
-    <code>LIBFOO_POST_PATCH_HOOKS</code> will be userful for generic packages.
+    <code>LIBFOO_POST_PATCH_HOOKS</code> will be useful for generic packages.
     </p>
 
     <p>The following hook points are available:</p>

@@ -1776,7 +1781,7 @@
     network...</code></h3>
 
     <p>If the boot process seems to hang after the following messages
-    (messages not necessarly exactly similar, depending on the list of
+    (messages not necessarily exactly similar, depending on the list of
     packages selected):</p>
 
     <pre>Freeing init memory: 3972K
#5 lore raw

PATCH 1/1 Update documentation for scp and Mercurial support

Hello Thomas,

Le Wed, 21 Sep 2011 08:42:14 +0200,
Thomas De Schampheleire <[email protected]> a ?crit :

Quoted 2 lines

Update documentation for scp and Mercurial support.
Along with the documentation update, a few typos were fixed as well.

Please don't make any changes to the HTML documentation, as we are moving to the AsciiDoc version. I'd prefer to have the AsciiDoc version merge, and from there update the documentation with all the latest developments of the package infrastructure.

Regards,

Thomas

Signature · 4 lines
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com
author
#6 lore raw

PATCH 1/1 Update documentation for scp and Mercurial support

Hi Thomas,

Quoted 5 lines · Thomas Petazzoni

Hello Thomas,

Le Wed, 21 Sep 2011 08:42:14 +0200,
Thomas De Schampheleire <[email protected]> a ?crit :

Quoted 7 lines
Quoted 2 lines

Update documentation for scp and Mercurial support.
Along with the documentation update, a few typos were fixed as well.

Please don't make any changes to the HTML documentation, as we are moving to the AsciiDoc version. I'd prefer to have the AsciiDoc version merge, and from there update the documentation with all the latest developments of the package infrastructure.

I am aware of your AsciiDoc patches, but given that they are not yet merged, I though I'd still update the HTML docs.

What is blocking the merge of the AsciiDoc patches?

Thanks,
Thomas