Minor npm fetcher optimisation
cover Minor npm fetcher optimisation
The following changes since commit 57912de63fa83550c0ae658eb99b76e9cc91a8d1:
cooker: record events on cooker exit (2016-09-06 10:36:34 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib paule/bb-npm-download-fix
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/bb-npm-download-fix
Paul Eggleton (1):
fetch2/npm: don't download same URL multiple times
lib/bb/fetch2/npm.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-)
Signature · 1 line
2.5.5
PATCH 1/1 fetch2/npm: don't download same URL multiple times
If we've already fetched a particular URL then we do not need to do so again within in the same operation. Maintain an internal list of fetched URLs to avoid doing that.
Signed-off-by Paul Eggleton <[email protected]>
--- lib/bb/fetch2/npm.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-)
lib/bb/fetch2/npm.py +8 -4
@@ -165,7 +165,9 @@ class Npm(FetchMethod): pdata = json.loads('\n'.join(datalines)) return pdata - def _getdependencies(self, pkg, data, version, d, ud, optional=False): + def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None): + if fetchedlist is None: + fetchedlist = [] pkgfullname = pkg if version != '*' and not '/' in version: pkgfullname += "@'%s'" % version
@@ -187,7 +189,9 @@ class Npm(FetchMethod): outputurl = pdata['dist']['tarball'] data[pkg] = {} data[pkg]['tgz'] = os.path.basename(outputurl) - self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) + if not outputurl in fetchedlist: + self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) + fetchedlist.append(outputurl) dependencies = pdata.get('dependencies', {}) optionalDependencies = pdata.get('optionalDependencies', {})
@@ -200,9 +204,9 @@ class Npm(FetchMethod): else: depsfound[dep] = dependencies[dep] for dep, version in optdepsfound.items(): - self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True) + self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist) for dep, version in depsfound.items(): - self._getdependencies(dep, data[pkg]['deps'], version, d, ud) + self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist) def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True): logger.debug(2, "NPM shrinkwrap file is %s" % data)
Signature · 1 line
2.5.5