Skip to contentSkip to conversation

perforce: add basic progress handler for perforce

PATCH New Alexandru N. Onea 1 participant 3 messages Jun 23, 2020

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/3 perforce: add basic progress handler for perforce

This patch adds a basic implementation of a progress handler for the perforce fetcher, based on the number of files to be downloaded and the output behavior of the p4 print command used in the fetcher implementation.

---
 lib/bb/fetch2/perforce.py | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)
lib/bb/fetch2/perforce.py +33 -1
index f57c2a4f..83128cde 100644
@@ -17,6 +17,36 @@ from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
 
+class PerforceProgressHandler (bb.progress.BasicProgressHandler):
+    """
+    Implements basic progress information for perforce, based on the number of
+    files to be downloaded.
+
+    The p4 print command will print one line per file, therefore it can be used
+    to "count" the number of files already completed and give an indication of
+    the progress.
+    """
+    def __init__(self, d, num_files):
+        self._num_files = num_files
+        self._count = 0
+        super(PerforceProgressHandler, self).__init__(d)
+
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(-1)
+
+    def write(self, string):
+        self._count = self._count + 1
+
+        percent = int(100.0 * float(self._count) / float(self._num_files))
+
+        # In case something goes wrong, we try to preserve our sanity
+        if percent > 100:
+            percent = 100
+
+        self.update(percent)
+
+        super(PerforceProgressHandler, self).write(string)
+
 class Perforce(FetchMethod):
     """ Class to fetch from perforce repositories """
     def supports(self, ud, d):
@@ -150,10 +180,12 @@ class Perforce(FetchMethod):
         bb.utils.remove(ud.pkgdir, True)
         bb.utils.mkdirhier(ud.pkgdir)
 
+        progresshandler = PerforceProgressHandler(d, len(filelist))
+
         for afile in filelist:
             p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
             bb.fetch2.check_network_access(d, p4fetchcmd, ud.url)
-            runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir)
+            runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler)
 
         runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
 
Signature · 1 line
2.25.1
#1 lore raw

PATCH v2 2/3 perforce: add local path handling SRC_URI options

This patch implements three new SRC_URI options for the perforce fetcher, namely:

* module
* remotepath

The options are intended to provide the user more control over the downloaded file paths by allowing the user to specify how much of the remote path should be preserved locally.

The changes in this patch are backwards compatible, i.e. if none of the introduced options is specified, the default (old) behavior is enforced.

---
 lib/bb/fetch2/perforce.py | 47 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)
lib/bb/fetch2/perforce.py +45 -2
index 83128cde..6f3c95b6 100644
@@ -1,6 +1,20 @@
 """
 BitBake 'Fetch' implementation for perforce
 
+Supported SRC_URI options are:
+
+- module
+   The top-level location to fetch while preserving the remote paths
+
+   The value of module can point to either a directory or a file. The result,
+   in both cases, is that the fetcher will preserve all file paths starting
+   from the module path. That is, the top-level directory in the module value
+   will also be the top-level directory in P4DIR.
+
+- remotepath
+   If the value "keep" is given, the full depot location of each file is
+   preserved in P4DIR. This option overrides the effect of the module option.
+
 """
 
 # Copyright (C) 2003, 2004  Chris Larson
@@ -88,14 +102,33 @@ class Perforce(FetchMethod):
             logger.debug(1, 'Determined P4PORT to be: %s' % ud.host)
             if not ud.host:
                 raise FetchError('Could not determine P4PORT from P4CONFIG')
-	
+
+        # Fetcher options
+        ud.module = ud.parm.get('module')
+        ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep')
+
         if ud.path.find('/...') >= 0:
             ud.pathisdir = True
         else:
             ud.pathisdir = False
 
+        # Avoid using the "/..." syntax in SRC_URI when a module value is given
+        if ud.pathisdir and ud.module:
+            raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given')
+
         cleanedpath = ud.path.replace('/...', '').replace('/', '.')
         cleanedhost = ud.host.replace(':', '.')
+
+        # Merge the path and module into the final depot location
+        if ud.module:
+            if ud.module.find('/') == 0:
+                raise FetchError('module cannot begin with /')
+            ud.path = os.path.join(ud.path, ud.module)
+
+            # Append the module path to the local pkg name
+            cleanedmodule = ud.module.replace('/...', '').replace('/', '.')
+            cleanedpath += '--%s' % cleanedmodule
+
         ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath)
 
         ud.setup_revisions(d)
@@ -125,10 +158,20 @@ class Perforce(FetchMethod):
             pathnrev = '%s' % (ud.path)
 
         if depot_filename:
-            if ud.pathisdir: # Remove leading path to obtain filename
+            if ud.keepremotepath:
+                # preserve everything, remove the leading //
+                filename = depot_filename.lstrip('/')
+            elif ud.module:
+                # remove everything up to the module path
+                modulepath = ud.module.rstrip('/...')
+                filename = depot_filename[depot_filename.rfind(modulepath):]
+            elif ud.pathisdir:
+                # Remove leading (visible) path to obtain the filepath
                 filename = depot_filename[len(ud.path)-1:]
             else:
+                # Remove everything, except the filename
                 filename = depot_filename[depot_filename.rfind('/'):]
+
             filename = filename[:filename.find('#')] # Remove trailing '#rev'
 
         if command == 'changes':
Signature · 1 line
2.25.1
#2 lore raw

PATCH v2 3/3 bitbake-user-manual: update perforce fetcher docs

In reply to Alexandru N. Onea

This change updates the perforce documentation by describing two new
parameters: module and remotepath.

Additionally, a general statement regarding the fetcher implementation has been added, to make it clear that the fetcher does not use a perforce client for the job.

---
 .../bitbake-user-manual-fetching.xml          | 60 +++++++++++++++++++
 1 file changed, 60 insertions(+)
doc/bitbake-user-manual/bitbake-user-manual-fetching.xml +60 -0
index d1bfc233..fe4372ad 100644
@@ -736,6 +736,10 @@
                 The fetcher's temporary working directory is set by
                 <link linkend='var-bb-P4DIR'><filename>P4DIR</filename></link>,
                 which defaults to "DL_DIR/p4".
+                The fetcher does not make use of a perforce client, instead it
+                relies on <filename>p4 files</filename> to retrieve a list of
+                files and <filename>p4 print</filename> to transfer the content
+                of those files locally.
             </para>
 
             <para>
@@ -783,6 +787,62 @@
                     to <filename>"${WORKDIR}/p4"</filename> in your recipe.
                 </note>
             </para>
+
+            <para>
+                By default, the fetcher strips the depot location from the
+                local file paths. In the above example, the content of
+                <filename>example-depot/main/source/</filename>
+                will be placed in <filename>${WORKDIR}/p4</filename>.
+                For situations where preserving parts of the remote depot paths
+                locally is desirable, the fetcher supports two parameters:
+
+                <itemizedlist>
+                    <listitem><para>
+                        <emphasis>"module":</emphasis>
+                        The top-level depot location or directory to fetch. The
+                        value of this parameter can also point to a single file
+                        within the depot, in which case the local file path will
+                        include the module path.
+                    </para></listitem>
+                    <listitem><para>
+                        <emphasis>"remotepath":</emphasis>
+                        When used with the value "<filename>keep</filename>",
+                        the fetcher will mirror the full depot paths locally
+                        for the specified location, even in combination with
+                        the <filename>module</filename> parameter.
+                    </para></listitem>
+                </itemizedlist>
+            </para>
+
+            <para>
+                Here is an example use of the the <filename>module</filename>
+                parameter:
+
+                <literallayout class='monospaced'>
+    SRC_URI = "p4://user:passwd@example-depot/main;module=source/..."
+                </literallayout>
+
+                In this case, the content of the top-level directory
+                <filename>source/</filename>  will be fetched to
+                <filename>${P4DIR}</filename>, including the directory itself.
+                The top-level directory will be accesible at
+                <filename>${P4DIR}/source/</filename>.
+            </para>
+
+            <para>
+                Here is an example use of the the <filename>remotepath</filename>
+                parameter:
+
+                <literallayout class='monospaced'>
+    SRC_URI = "p4://user:passwd@example-depot/main;module=source/...;remotepath=keep"
+                </literallayout>
+
+                In this case, the content of the top-level directory
+                <filename>source/</filename>  will be fetched to
+                <filename>${P4DIR}</filename>, but the complete depot paths will
+                be mirrored locally. The top-level directory will be accessible
+                at <filename>${P4DIR}/example-depot/main/source/</filename>.
+            </para>
         </section>
 
         <section id='repo-fetcher'>
Signature · 1 line
2.25.1