Skip to contentSkip to conversation

perforce: add basic progress handler for perforce

PATCH Superseded Alexandru N. Onea 1 participant 2 messages Jun 21, 2020

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 perforce: add basic progress handler for perforce

This patch adds a basic implementation of a progress handler for the perforce fetcher, based on the number of files to be downloaded and the output behavior of the p4 print command used in the fetcher implementation.

---
 lib/bb/fetch2/perforce.py | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)
lib/bb/fetch2/perforce.py +33 -1
index f57c2a4f..83128cde 100644
@@ -17,6 +17,36 @@ from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
 
+class PerforceProgressHandler (bb.progress.BasicProgressHandler):
+    """
+    Implements basic progress information for perforce, based on the number of
+    files to be downloaded.
+
+    The p4 print command will print one line per file, therefore it can be used
+    to "count" the number of files already completed and give an indication of
+    the progress.
+    """
+    def __init__(self, d, num_files):
+        self._num_files = num_files
+        self._count = 0
+        super(PerforceProgressHandler, self).__init__(d)
+
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(-1)
+
+    def write(self, string):
+        self._count = self._count + 1
+
+        percent = int(100.0 * float(self._count) / float(self._num_files))
+
+        # In case something goes wrong, we try to preserve our sanity
+        if percent > 100:
+            percent = 100
+
+        self.update(percent)
+
+        super(PerforceProgressHandler, self).write(string)
+
 class Perforce(FetchMethod):
     """ Class to fetch from perforce repositories """
     def supports(self, ud, d):
@@ -150,10 +180,12 @@ class Perforce(FetchMethod):
         bb.utils.remove(ud.pkgdir, True)
         bb.utils.mkdirhier(ud.pkgdir)
 
+        progresshandler = PerforceProgressHandler(d, len(filelist))
+
         for afile in filelist:
             p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
             bb.fetch2.check_network_access(d, p4fetchcmd, ud.url)
-            runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir)
+            runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler)
 
         runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
 
Signature · 1 line
2.25.1
#1 lore raw

PATCH 2/2 perforce: add local path handling SRC_URI options

This patch implements three new SRC_URI options for the perforce fetcher, namely:

* module
* destdir
* remotepath

The options are intended to provide the user more control over the downloaded file paths. The module and remotepath options allow the user to specify how much of the remote path should be preserved locally. The destdir option provides a mechanism for grouping and separating downloads in case of multiple perforce entries in SRC_URI.

The changes in this patch are backwards compatible, i.e. if none of the introduced options is specified, the default (old) behavior is enforced.

---
 lib/bb/fetch2/perforce.py | 55 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)
lib/bb/fetch2/perforce.py +53 -2
index 83128cde..3237b038 100644
@@ -1,6 +1,25 @@
 """
 BitBake 'Fetch' implementation for perforce
 
+Supported SRC_URI options are:
+
+- module
+   The top-level location to fetch while preserving the remote paths
+
+   The value of module can point to either a directory or a file. The result,
+   in both cases, is that the fetcher will preserve all file paths starting
+   from the module path. That is, the top-level directory in the module value
+   will also be the top-level directory in P4DIR.
+
+- destdir
+   An additional path element to be prefixed to the final file paths, for each
+   file in the download, helping with grouping files in case of multiple
+   SRC_URI perforce entries.
+
+- remotepath
+   If the value "keep" is given, the full depot location of each file is
+   preserved in P4DIR. This option overrides the effect of the module option.
+
 """
 
 # Copyright (C) 2003, 2004  Chris Larson
@@ -88,12 +107,31 @@ class Perforce(FetchMethod):
             logger.debug(1, 'Determined P4PORT to be: %s' % ud.host)
             if not ud.host:
                 raise FetchError('Could not determine P4PORT from P4CONFIG')
-	
+
+        # Fetcher options
+        ud.module = ud.parm.get('module')
+        ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep')
+        ud.destdir = ud.parm.get('destdir', '')
+
         if ud.path.find('/...') >= 0:
             ud.pathisdir = True
         else:
             ud.pathisdir = False
 
+        # Avoid using the "/..." syntax in SRC_URI when a module value is given
+        if ud.pathisdir and ud.module:
+            raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given')
+
+        # Avoid destdir values which don't reference a single directory
+        if ud.destdir.find('/') >= 0:
+            raise FetchError('destdir cannot contain /')
+
+        # Merge the path and module into the final depot location
+        if ud.module:
+            if ud.module.find('/') == 0:
+                raise FetchError('module cannot begin with /')
+            ud.path = os.path.join(ud.path, ud.module)
+
         cleanedpath = ud.path.replace('/...', '').replace('/', '.')
         cleanedhost = ud.host.replace(':', '.')
         ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath)
@@ -125,10 +163,23 @@ class Perforce(FetchMethod):
             pathnrev = '%s' % (ud.path)
 
         if depot_filename:
-            if ud.pathisdir: # Remove leading path to obtain filename
+            if ud.keepremotepath:
+                # preserve everything (the leading // is removed below)
+                filename = depot_filename
+            elif ud.module:
+                # remove everything up to the module path
+                modulepath = ud.module.rstrip('/...')
+                filename = depot_filename[depot_filename.rfind(modulepath):]
+            elif ud.pathisdir:
+                # Remove leading (visible) path to obtain the filepath
                 filename = depot_filename[len(ud.path)-1:]
             else:
+                # Remove everything, except the filename
                 filename = depot_filename[depot_filename.rfind('/'):]
+
+            # Prepend destdir to the final file path, also remove leading //
+            filename = os.path.join(ud.destdir, filename.lstrip('/'))
+
             filename = filename[:filename.find('#')] # Remove trailing '#rev'
 
         if command == 'changes':
Signature · 1 line
2.25.1