bb/fetch2: fix the "filename too long" error
cover bb/fetch2: fix the "filename too long" error
The following changes since commit 78b91ab23d9856525fc7ac1bb8da2975382813bf:
pigz: bump to 2.3.1 (2013-10-26 16:08:41 +0100)
are available in the git repository at:
git://git.pokylinux.org/poky-contrib robert/name_max
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=robert/name_max
Robert Yang (1):
bb/fetch2: fix the "filename too long" error
bitbake/lib/bb/fetch2/__init__.py | 18 ++++++++++++++++++ bitbake/lib/bb/fetch2/bzr.py | 2 ++ bitbake/lib/bb/fetch2/git.py | 6 ++++++ bitbake/lib/bb/fetch2/hg.py | 3 +++ bitbake/lib/bb/fetch2/osc.py | 2 ++ bitbake/lib/bb/fetch2/repo.py | 4 ++++ bitbake/lib/bb/fetch2/svk.py | 2 ++ bitbake/lib/bb/fetch2/svn.py | 3 +++ 8 files changed, 40 insertions(+)
Signature · 1 line
1.8.3.1
PATCH 1/1 bb/fetch2: fix the "filename too long" error
When the SRC_URI is very long or the PREMIRROR/MIRROR is in a very deep directory, the git, svn and other fetchers will covert the path into the filename, so if:
len(path) > NAME_MAX,
then we will get the "filename too long" error, the oe-core supports to build when len(TMPDIR) <= 410, but the NAME_MAX is usually 256, so we can easily get this error when we put the PREMIRROR/MIRROR in a deep dir.
Truncate the filename to NAME_MAX - 10 will fix the problem, the "-10" is used for the ".done" and ".lock", in fact, -5 would be OK, but use "-10" in case we will have other longer suffix in the future.
[YOCTO #5389]
--- bitbake/lib/bb/fetch2/__init__.py | 18 ++++++++++++++++++ bitbake/lib/bb/fetch2/bzr.py | 2 ++ bitbake/lib/bb/fetch2/git.py | 6 ++++++ bitbake/lib/bb/fetch2/hg.py | 3 +++ bitbake/lib/bb/fetch2/osc.py | 2 ++ bitbake/lib/bb/fetch2/repo.py | 4 ++++ bitbake/lib/bb/fetch2/svk.py | 2 ++ bitbake/lib/bb/fetch2/svn.py | 3 +++ 8 files changed, 40 insertions(+)
bitbake/lib/bb/fetch2/__init__.py +18 -0
@@ -959,6 +959,24 @@ def get_file_checksums(filelist, pn): checksums.sort(key=operator.itemgetter(1)) return checksums +def truncate_to_name_max(dirpath, filename): + """ + Truncate the filename to meet len(filename) <= NAME_MAX -10, return + the last NAME_MAX - 10 characters if it doesn't, the -10 is for the + '.done', '.lock' and perhaps others in the future. + """ + if not os.path.exists(dirpath): + bb.utils.mkdirhier(dirpath) + + name_max = os.pathconf(dirpath, 'PC_NAME_MAX') + new_max = name_max - 10 + if len(filename) > new_max: + new_name = filename[-new_max:] + logger.debug(1, "Truncate %s to %s to meet %s (NAME_MAX - 10) characters" % \ + (filename, new_name, new_max)) + filename = new_name + + return filename class FetchData(object): """
bitbake/lib/bb/fetch2/bzr.py +2 -0
@@ -51,6 +51,8 @@ class Bzr(FetchMethod): ud.revision = self.latest_revision(ud.url, ud, d) ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) def _buildbzrcommand(self, ud, d, command): """
bitbake/lib/bb/fetch2/git.py +6 -0
@@ -135,9 +135,15 @@ class Git(FetchMethod): if ud.rebaseable: for name in ud.names: gitsrcname = gitsrcname + '_' + ud.revisions[name] + ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname) + # The length of the filename should be less than NAME_MAX - 10 + ud.mirrortarball = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.mirrortarball) ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) + gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/") + # The length of the filename should be less than NAME_MAX - 10 + gitsrcname = bb.fetch2.truncate_to_name_max(gitdir, gitsrcname) ud.clonedir = os.path.join(gitdir, gitsrcname) ud.localfile = ud.clonedir
bitbake/lib/bb/fetch2/hg.py +3 -0
@@ -66,6 +66,9 @@ class Hg(FetchMethod): ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) + def need_update(self, url, ud, d): revTag = ud.parm.get('rev', 'tip') if revTag == "tip":
bitbake/lib/bb/fetch2/osc.py +2 -0
@@ -48,6 +48,8 @@ class Osc(FetchMethod): ud.revision = "" ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) def _buildosccommand(self, ud, d, command): """
bitbake/lib/bb/fetch2/repo.py +4 -0
@@ -52,6 +52,8 @@ class Repo(FetchMethod): ud.manifest += '.xml' ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile) def download(self, loc, ud, d): """Fetch url"""
@@ -62,6 +64,8 @@ class Repo(FetchMethod): gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo") + # The length of the filename should be less than NAME_MAX - 10 + gitsrcname = bb.fetch2.truncate_to_name_max(repodir, gitsrcname) codir = os.path.join(repodir, gitsrcname, ud.manifest) if ud.user:
bitbake/lib/bb/fetch2/svk.py +2 -0
@@ -53,6 +53,8 @@ class Svk(FetchMethod): ud.revision = ud.parm.get('rev', "") ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile) def need_update(self, url, ud, d): if ud.date == "now":
bitbake/lib/bb/fetch2/svn.py +3 -0
@@ -65,6 +65,9 @@ class Svn(FetchMethod): ud.localfile = data.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) + def _buildsvncommand(self, ud, d, command): """ Build up an svn commandline based on ud
Signature · 1 line
1.8.3.1
Quoted 11 lines · Robert Yang
When the
SRC_URIis very long or the PREMIRROR/MIRROR is in a very deep directory, the git, svn and other fetchers will covert the path into the filename, so if:len(path) >
NAME_MAX,then we will get the "filename too long" error, the oe-core supports to build when len(TMPDIR) <= 410, but the
NAME_MAXis usually 256, so we can easily get this error when we put the PREMIRROR/MIRROR in a deep dir.
Quoted 5 lines
Truncate the filename to
NAME_MAX- 10 will fix the problem, the "-10" is used for the ".done" and ".lock", in fact, -5 would be OK, but use "-10" in case we will have other longer suffix in the future.[YOCTO #5389]
Aren't the "long SRC_URI" and deep PREMIRROR/MIRROR problems different issues?
As far as I know, we don't have any long SRC_URIs in use anywhere within the project. We could just error if we found one with a path we couldn't cope with.
The deep PREMIRROR/MIRROR issue on the other hand sounds like something we should adjust the code to handle properly?
Randomly truncating path names like this looks like a rather hacky solution and it loses key information, there is a reason these long names are being used as keys.
Put another way, the mirror tarball name should not change depending on how deep the DL_DIR or other directories are.
So no, there is no way this patch is correct or should be merged, sorry.
Cheers,
Richard
Quoted 150 lines
Signed-off-by Robert Yang <[email protected]>--- bitbake/lib/bb/fetch2/__init__.py | 18 ++++++++++++++++++ bitbake/lib/bb/fetch2/bzr.py | 2 ++ bitbake/lib/bb/fetch2/git.py | 6 ++++++ bitbake/lib/bb/fetch2/hg.py | 3 +++ bitbake/lib/bb/fetch2/osc.py | 2 ++ bitbake/lib/bb/fetch2/repo.py | 4 ++++ bitbake/lib/bb/fetch2/svk.py | 2 ++ bitbake/lib/bb/fetch2/svn.py | 3 +++ 8 files changed, 40 insertions(+)bitbake/lib/bb/fetch2/__init__.py +18
-0@@ -959,6 +959,24 @@ def get_file_checksums(filelist, pn): checksums.sort(key=operator.itemgetter(1)) return checksums +def truncate_to_name_max(dirpath, filename): + """ + Truncate the filename to meet len(filename) <= NAME_MAX -10, return + the last NAME_MAX - 10 characters if it doesn't, the -10 is for the + '.done', '.lock' and perhaps others in the future. + """ + if not os.path.exists(dirpath): + bb.utils.mkdirhier(dirpath) + + name_max = os.pathconf(dirpath, 'PC_NAME_MAX') + new_max = name_max - 10 + if len(filename) > new_max: + new_name = filename[-new_max:] + logger.debug(1, "Truncate %s to %s to meet %s (NAME_MAX - 10) characters" % \ + (filename, new_name, new_max)) + filename = new_name + + return filename class FetchData(object): """bitbake/lib/bb/fetch2/bzr.py +2
-0@@ -51,6 +51,8 @@ class Bzr(FetchMethod): ud.revision = self.latest_revision(ud.url, ud, d) ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) def _buildbzrcommand(self, ud, d, command): """bitbake/lib/bb/fetch2/git.py +6
-0@@ -135,9 +135,15 @@ class Git(FetchMethod): if ud.rebaseable: for name in ud.names: gitsrcname = gitsrcname + '_' + ud.revisions[name] + ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname) + # The length of the filename should be less than NAME_MAX - 10 + ud.mirrortarball = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.mirrortarball) ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) + gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/") + # The length of the filename should be less than NAME_MAX - 10 + gitsrcname = bb.fetch2.truncate_to_name_max(gitdir, gitsrcname) ud.clonedir = os.path.join(gitdir, gitsrcname) ud.localfile = ud.clonedirbitbake/lib/bb/fetch2/hg.py +3
-0@@ -66,6 +66,9 @@ class Hg(FetchMethod): ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) + def need_update(self, url, ud, d): revTag = ud.parm.get('rev', 'tip') if revTag == "tip":bitbake/lib/bb/fetch2/osc.py +2
-0@@ -48,6 +48,8 @@ class Osc(FetchMethod): ud.revision = "" ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) def _buildosccommand(self, ud, d, command): """bitbake/lib/bb/fetch2/repo.py +4
-0@@ -52,6 +52,8 @@ class Repo(FetchMethod): ud.manifest += '.xml' ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile) def download(self, loc, ud, d): """Fetch url"""@@ -62,6 +64,8 @@ class Repo(FetchMethod): gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo") + # The length of the filename should be less than NAME_MAX - 10 + gitsrcname = bb.fetch2.truncate_to_name_max(repodir, gitsrcname) codir = os.path.join(repodir, gitsrcname, ud.manifest) if ud.user:bitbake/lib/bb/fetch2/svk.py +2
-0@@ -53,6 +53,8 @@ class Svk(FetchMethod): ud.revision = ud.parm.get('rev', "") ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile) def need_update(self, url, ud, d): if ud.date == "now":bitbake/lib/bb/fetch2/svn.py +3
-0@@ -65,6 +65,9 @@ class Svn(FetchMethod): ud.localfile = data.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) + def _buildsvncommand(self, ud, d, command): """ Build up an svn commandline based on ud
Quoted 12 lines · Richard Purdie
Quoted 11 lines · Robert Yang
When the
SRC_URIis very long or the PREMIRROR/MIRROR is in a very deep directory, the git, svn and other fetchers will covert the path into the filename, so if:len(path) >
NAME_MAX,then we will get the "filename too long" error, the oe-core supports to build when len(TMPDIR) <= 410, but the
NAME_MAXis usually 256, so we can easily get this error when we put the PREMIRROR/MIRROR in a deep dir.
Quoted 9 lines
Quoted 5 lines
Truncate the filename to
NAME_MAX- 10 will fix the problem, the "-10" is used for the ".done" and ".lock", in fact, -5 would be OK, but use "-10" in case we will have other longer suffix in the future.[YOCTO #5389]
Aren't the "long
SRC_URI" and deep PREMIRROR/MIRROR problems different issues?
They are slightly different on the stamps (.lock and .done):
We use the SRC_URI.done as the stamp (with the "/" converted to ".", and the protocol e.g., "git://" removed), when len(SRC_URI) >= 256, we can only use 256 - 5 which is 251 characters since the stamp is SRC_URI.done, and len(".done") = 5, for example, the gtk-doc-stub_git.bb:
SRC_URI = "git://git.gnome.org/${BPN}"
The stamp is: git.gnome.org.gtk-doc-stub.done
When it uses the src from the PREMIRRORS, the path of the PREMIRRORS is not used by the stamp filename.
Quoted 3 lines
As far as I know, we don't have any long
SRC_URIsin use anywhere within the project. We could just error if we found one with a path we couldn't cope with.
Sounds great.
Quoted 11 lines
The deep PREMIRROR/MIRROR issue on the other hand sounds like something we should adjust the code to handle properly?
Randomly truncating path names like this looks like a rather hacky solution and it loses key information, there is a reason these long names are being used as keys.
Put another way, the mirror tarball name should not change depending on how deep the
DL_DIRor other directories are.
The tarball doesn't have this issue since it uses the tarball name as the symlink, only the repo fetchers (for example, the git) have this problem.
The current process for the PREMIRRORS are:
1) git clone from the PREMIRRORS into DL_DIR/git2/localname, while:
localname = echo PREMIRRORS/reponame | sed '#/#.#"
So if PREMIRRORS is in a deep directory, the localname will be too long.
2) ln -s localname DL_DIR/git2/reponame
Maybe we can:
1) Remove step 1, Don't clone, but directly link to the srcrepo when the protocol is file://.
Or
2) Remove the step 2, and directly use the reponame as the localname.
Or
3) Use the md5sum(PREMIRRORS) as a prefix or suffix for the filename
// Robert
Quoted 162 lines
So no, there is no way this patch is correct or should be merged, sorry.
Cheers,
Richard
Quoted 150 lines
Signed-off-by Robert Yang <[email protected]>--- bitbake/lib/bb/fetch2/__init__.py | 18 ++++++++++++++++++ bitbake/lib/bb/fetch2/bzr.py | 2 ++ bitbake/lib/bb/fetch2/git.py | 6 ++++++ bitbake/lib/bb/fetch2/hg.py | 3 +++ bitbake/lib/bb/fetch2/osc.py | 2 ++ bitbake/lib/bb/fetch2/repo.py | 4 ++++ bitbake/lib/bb/fetch2/svk.py | 2 ++ bitbake/lib/bb/fetch2/svn.py | 3 +++ 8 files changed, 40 insertions(+)bitbake/lib/bb/fetch2/__init__.py +18
-0@@ -959,6 +959,24 @@ def get_file_checksums(filelist, pn): checksums.sort(key=operator.itemgetter(1)) return checksums +def truncate_to_name_max(dirpath, filename): + """ + Truncate the filename to meet len(filename) <= NAME_MAX -10, return + the last NAME_MAX - 10 characters if it doesn't, the -10 is for the + '.done', '.lock' and perhaps others in the future. + """ + if not os.path.exists(dirpath): + bb.utils.mkdirhier(dirpath) + + name_max = os.pathconf(dirpath, 'PC_NAME_MAX') + new_max = name_max - 10 + if len(filename) > new_max: + new_name = filename[-new_max:] + logger.debug(1, "Truncate %s to %s to meet %s (NAME_MAX - 10) characters" % \ + (filename, new_name, new_max)) + filename = new_name + + return filename class FetchData(object): """bitbake/lib/bb/fetch2/bzr.py +2
-0@@ -51,6 +51,8 @@ class Bzr(FetchMethod): ud.revision = self.latest_revision(ud.url, ud, d) ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) def _buildbzrcommand(self, ud, d, command): """bitbake/lib/bb/fetch2/git.py +6
-0@@ -135,9 +135,15 @@ class Git(FetchMethod): if ud.rebaseable: for name in ud.names: gitsrcname = gitsrcname + '_' + ud.revisions[name] + ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname) + # The length of the filename should be less than NAME_MAX - 10 + ud.mirrortarball = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.mirrortarball) ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) + gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/") + # The length of the filename should be less than NAME_MAX - 10 + gitsrcname = bb.fetch2.truncate_to_name_max(gitdir, gitsrcname) ud.clonedir = os.path.join(gitdir, gitsrcname) ud.localfile = ud.clonedirbitbake/lib/bb/fetch2/hg.py +3
-0@@ -66,6 +66,9 @@ class Hg(FetchMethod): ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) + def need_update(self, url, ud, d): revTag = ud.parm.get('rev', 'tip') if revTag == "tip":bitbake/lib/bb/fetch2/osc.py +2
-0@@ -48,6 +48,8 @@ class Osc(FetchMethod): ud.revision = "" ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) def _buildosccommand(self, ud, d, command): """bitbake/lib/bb/fetch2/repo.py +4
-0@@ -52,6 +52,8 @@ class Repo(FetchMethod): ud.manifest += '.xml' ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile) def download(self, loc, ud, d): """Fetch url"""@@ -62,6 +64,8 @@ class Repo(FetchMethod): gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo") + # The length of the filename should be less than NAME_MAX - 10 + gitsrcname = bb.fetch2.truncate_to_name_max(repodir, gitsrcname) codir = os.path.join(repodir, gitsrcname, ud.manifest) if ud.user:bitbake/lib/bb/fetch2/svk.py +2
-0@@ -53,6 +53,8 @@ class Svk(FetchMethod): ud.revision = ud.parm.get('rev', "") ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile) def need_update(self, url, ud, d): if ud.date == "now":bitbake/lib/bb/fetch2/svn.py +3
-0@@ -65,6 +65,9 @@ class Svn(FetchMethod): ud.localfile = data.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) + def _buildsvncommand(self, ud, d, command): """ Build up an svn commandline based on ud