perforce: add basic progress handler for perforce
PATCH v2 1/3 perforce: add basic progress handler for perforce
This patch adds a basic implementation of a progress handler for the perforce fetcher, based on the number of files to be downloaded and the output behavior of the p4 print command used in the fetcher implementation.
--- lib/bb/fetch2/perforce.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-)
lib/bb/fetch2/perforce.py +33 -1
@@ -17,6 +17,36 @@ from bb.fetch2 import FetchError from bb.fetch2 import logger from bb.fetch2 import runfetchcmd +class PerforceProgressHandler (bb.progress.BasicProgressHandler): + """ + Implements basic progress information for perforce, based on the number of + files to be downloaded. + + The p4 print command will print one line per file, therefore it can be used + to "count" the number of files already completed and give an indication of + the progress. + """ + def __init__(self, d, num_files): + self._num_files = num_files + self._count = 0 + super(PerforceProgressHandler, self).__init__(d) + + # Send an initial progress event so the bar gets shown + self._fire_progress(-1) + + def write(self, string): + self._count = self._count + 1 + + percent = int(100.0 * float(self._count) / float(self._num_files)) + + # In case something goes wrong, we try to preserve our sanity + if percent > 100: + percent = 100 + + self.update(percent) + + super(PerforceProgressHandler, self).write(string) + class Perforce(FetchMethod): """ Class to fetch from perforce repositories """ def supports(self, ud, d):
@@ -150,10 +180,12 @@ class Perforce(FetchMethod): bb.utils.remove(ud.pkgdir, True) bb.utils.mkdirhier(ud.pkgdir) + progresshandler = PerforceProgressHandler(d, len(filelist)) + for afile in filelist: p4fetchcmd = self._buildp4command(ud, d, 'print', afile) bb.fetch2.check_network_access(d, p4fetchcmd, ud.url) - runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir) + runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler) runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
Signature · 1 line
2.25.1
PATCH v2 2/3 perforce: add local path handling SRC_URI options
This patch implements three new SRC_URI options for the perforce fetcher, namely:
* module
* remotepath
The options are intended to provide the user more control over the downloaded file paths by allowing the user to specify how much of the remote path should be preserved locally.
The changes in this patch are backwards compatible, i.e. if none of the introduced options is specified, the default (old) behavior is enforced.
--- lib/bb/fetch2/perforce.py | 47 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-)
lib/bb/fetch2/perforce.py +45 -2
@@ -1,6 +1,20 @@ """ BitBake 'Fetch' implementation for perforce +Supported SRC_URI options are: + +- module + The top-level location to fetch while preserving the remote paths + + The value of module can point to either a directory or a file. The result, + in both cases, is that the fetcher will preserve all file paths starting + from the module path. That is, the top-level directory in the module value + will also be the top-level directory in P4DIR. + +- remotepath + If the value "keep" is given, the full depot location of each file is + preserved in P4DIR. This option overrides the effect of the module option. + """ # Copyright (C) 2003, 2004 Chris Larson
@@ -88,14 +102,33 @@ class Perforce(FetchMethod): logger.debug(1, 'Determined P4PORT to be: %s' % ud.host) if not ud.host: raise FetchError('Could not determine P4PORT from P4CONFIG') - + + # Fetcher options + ud.module = ud.parm.get('module') + ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep') + if ud.path.find('/...') >= 0: ud.pathisdir = True else: ud.pathisdir = False + # Avoid using the "/..." syntax in SRC_URI when a module value is given + if ud.pathisdir and ud.module: + raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given') + cleanedpath = ud.path.replace('/...', '').replace('/', '.') cleanedhost = ud.host.replace(':', '.') + + # Merge the path and module into the final depot location + if ud.module: + if ud.module.find('/') == 0: + raise FetchError('module cannot begin with /') + ud.path = os.path.join(ud.path, ud.module) + + # Append the module path to the local pkg name + cleanedmodule = ud.module.replace('/...', '').replace('/', '.') + cleanedpath += '--%s' % cleanedmodule + ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath) ud.setup_revisions(d)
@@ -125,10 +158,20 @@ class Perforce(FetchMethod): pathnrev = '%s' % (ud.path) if depot_filename: - if ud.pathisdir: # Remove leading path to obtain filename + if ud.keepremotepath: + # preserve everything, remove the leading // + filename = depot_filename.lstrip('/') + elif ud.module: + # remove everything up to the module path + modulepath = ud.module.rstrip('/...') + filename = depot_filename[depot_filename.rfind(modulepath):] + elif ud.pathisdir: + # Remove leading (visible) path to obtain the filepath filename = depot_filename[len(ud.path)-1:] else: + # Remove everything, except the filename filename = depot_filename[depot_filename.rfind('/'):] + filename = filename[:filename.find('#')] # Remove trailing '#rev' if command == 'changes':
Signature · 1 line
2.25.1
PATCH v2 3/3 bitbake-user-manual: update perforce fetcher docs
In reply to Alexandru N. OneaThis change updates the perforce documentation by describing two new
parameters: module and remotepath.
Additionally, a general statement regarding the fetcher implementation has been added, to make it clear that the fetcher does not use a perforce client for the job.
--- .../bitbake-user-manual-fetching.xml | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+)
doc/bitbake-user-manual/bitbake-user-manual-fetching.xml +60 -0
@@ -736,6 +736,10 @@ The fetcher's temporary working directory is set by <link linkend='var-bb-P4DIR'><filename>P4DIR</filename></link>, which defaults to "DL_DIR/p4". + The fetcher does not make use of a perforce client, instead it + relies on <filename>p4 files</filename> to retrieve a list of + files and <filename>p4 print</filename> to transfer the content + of those files locally. </para> <para>
@@ -783,6 +787,62 @@ to <filename>"${WORKDIR}/p4"</filename> in your recipe. </note> </para> + + <para> + By default, the fetcher strips the depot location from the + local file paths. In the above example, the content of + <filename>example-depot/main/source/</filename> + will be placed in <filename>${WORKDIR}/p4</filename>. + For situations where preserving parts of the remote depot paths + locally is desirable, the fetcher supports two parameters: + + <itemizedlist> + <listitem><para> + <emphasis>"module":</emphasis> + The top-level depot location or directory to fetch. The + value of this parameter can also point to a single file + within the depot, in which case the local file path will + include the module path. + </para></listitem> + <listitem><para> + <emphasis>"remotepath":</emphasis> + When used with the value "<filename>keep</filename>", + the fetcher will mirror the full depot paths locally + for the specified location, even in combination with + the <filename>module</filename> parameter. + </para></listitem> + </itemizedlist> + </para> + + <para> + Here is an example use of the the <filename>module</filename> + parameter: + + <literallayout class='monospaced'> + SRC_URI = "p4://user:passwd@example-depot/main;module=source/..." + </literallayout> + + In this case, the content of the top-level directory + <filename>source/</filename> will be fetched to + <filename>${P4DIR}</filename>, including the directory itself. + The top-level directory will be accesible at + <filename>${P4DIR}/source/</filename>. + </para> + + <para> + Here is an example use of the the <filename>remotepath</filename> + parameter: + + <literallayout class='monospaced'> + SRC_URI = "p4://user:passwd@example-depot/main;module=source/...;remotepath=keep" + </literallayout> + + In this case, the content of the top-level directory + <filename>source/</filename> will be fetched to + <filename>${P4DIR}</filename>, but the complete depot paths will + be mirrored locally. The top-level directory will be accessible + at <filename>${P4DIR}/example-depot/main/source/</filename>. + </para> </section> <section id='repo-fetcher'>
Signature · 1 line
2.25.1