fetcher: Add support for search latest version of

index 799fb6c..ce647c2 100644
@@ -67,6 +67,7 @@ Supported SRC_URI options are:
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
+import re
 import bb
 from   bb    import data
 from   bb.fetch2 import FetchMethod

@@ -343,6 +344,43 @@ class Git(FetchMethod):
         output = self._lsremote(ud, d, search)
         return output.split()[0]
 
+    def latest_versionstring(self, ud, d):
+        """
+        Compute the latest release name like "x.y.x" in "x.y.x+gitHASH"
+        by searching through the tags output of ls-remote, comparing
+        versions and returning the highest match.
+        """
+        verstring = ""
+        tagregex = re.compile(d.getVar('GITTAGREGEX', True) or "(?P<pver>([0-9][\.|_]?)+)")
+        try:
+            output = self._lsremote(ud, d, "refs/tags/*^{}")
+        except bb.fetch2.FetchError or bb.fetch2.NetworkAccess:
+            return ""
+
+        for line in output.split("\n"):
+            if not line:
+                break
+
+            line = line.split("/")[-1]
+            # Ignore non-released branches
+            m = re.search("(alpha|beta|rc|final)+", line)
+            if m:
+                continue
+
+            # search for version in the line
+            tag = tagregex.search(line)
+            if tag == None:
+                continue
+
+            tag = tag.group('pver')
+            tag = tag.replace("_", ".")
+
+            if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
+                continue
+            verstring = tag
+
+        return verstring
+
     def _build_revision(self, ud, d, name):
         return ud.revisions[name]
 

54 lines not on this page · raw message

index 0456490..2f88332 100644
@@ -25,6 +25,9 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
+import re
+import tempfile
+import subprocess
 import os
 import logging
 import bb

@@ -34,6 +37,7 @@ from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
+from   bs4 import BeautifulSoup
 
 class Wget(FetchMethod):
     """Class to fetch urls via 'wget'"""

@@ -104,3 +108,238 @@ class Wget(FetchMethod):
         self._runwget(ud, d, fetchcmd, True)
 
         return True
+
+
+    def _parse_path(self, regex, s):
+        """
+        Find and group name, version and archive type in the given string s
+        """
+        bb.debug("parse_path(%s, %s)" % (regex.pattern, s))
+        m = regex.search(s)
+        if m:
+            bb.debug("%s, %s, %s" % (m.group('name'), m.group('ver'), m.group('type')))
+            return (m.group('name'), m.group('ver'), m.group('type'))
+        return None
+
+    def _modelate_version(self, version):
+        if version[0] in ['.', '-']:
+            if version[1].isdigit():
+                version = version[1] + version[0] + version[2:len(version)]
+            else:
+                version = version[1:len(version)]
+
+        version = re.sub('\-', '.', version)
+        version = re.sub('_', '.', version)
+        version = re.sub('(rc)+', '.-1.', version)
+        version = re.sub('(alpha)+', '.-3.', version)
+        version = re.sub('(beta)+', '.-2.', version)
+        if version[0] == 'v':
+            version = version[1:len(version)]
+        return version
+
+    def _vercmp(self, old, new):
+        """
+        Check whether 'new' is newer than 'old' version. We use existing vercmp() for the
+        purpose. PE is cleared in comparison as it's not for build, and PR is cleared too
+        for simplicity as it's somehow difficult to get from various upstream format
+        """
+
+        (oldpn, oldpv, oldsuffix) = old
+        (newpn, newpv, newsuffix) = new
+
+        """
+        Check for a new suffix type that we have never heard of before
+        """
+        if (newsuffix):
+            m = self.suffixregex.search(newsuffix)
+            if not m:
+                bb.warn("%s has a possible unknown suffix: %s" % (newpn, newsuffix))
+                return False
+
+        """
+        Not our package so ignore it
+        """
+        if oldpn != newpn:
+            return False
+
+        oldpv = self._modelate_version(oldpv)
+        newpv = self._modelate_version(newpv)
+
+        if bb.utils.vercmp(("0", oldpv, ""), ("0", newpv, "")) < 0:
+            return True
+        else:
+            return False
+
+    def _fetch_index(self, uri, ud, d):
+        """
+        Run fetch checkstatus to get directory information
+        """
+        f = tempfile.NamedTemporaryFile(dir="/tmp/s/", delete=False)
+
+        agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
+        fetchcmd = self.basecmd
+        fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
+        try:
+            self._runwget(ud, d, fetchcmd, True)
+            fetchresult = f.read()
+        except bb.fetch2.BBFetchException:
+            fetchresult = ""
+
+        f.close()
+        # os.unlink(f.name)
+        return fetchresult
+
+    def _check_latest_dir(self, url, versionstring, ud, d):
+        """
+        Return the name of the directory with the greatest package version
+        If error or no version, return ""
+        """
+        bb.debug("DirURL: %s, %s" % (url, versionstring))
+        soup = BeautifulSoup(self._fetch_index(url, ud, d))
+        if not soup:
+            return ""
+
+        valid = 0
+        prefix = ''
+        regex = re.compile("(\D*)((\d+[\.\-_])+(\d+))")
+        m = regex.search(versionstring)
+        if m:
+            version = ('', m.group(2), '')
+            prefix = m.group(1)
+            bb.debug("version: %s, prefix: %s" % (version, prefix))
+        else:
+            version = ('', versionstring, '')
+
+        for href in soup.find_all('a', href=True):
+            bb.debug("href: %s" % (href['href']))
+            if href['href'].find(versionstring) >= 0:
+                valid = 1
+            m = regex.search(href['href'].strip("/"))
+            if m:
+                thisversion = ('', m.group(2), '')
+                if thisversion and self._vercmp(version, thisversion) == True:
+                    version = thisversion
+
+        if valid:
+            bb.debug("Would return %s" % (prefix+version[1]))
+            return prefix+version[1]
+        else:
+            bb.debug("Not Valid")
+            return ""
+
+    def _check_latest_version(self, url, packagename, ud, d):
+        """
+        Return the latest version of a package inside a given directory path
+        If error or no version, return ""
+        """
+        valid = 0
+        version = self._parse_path(self.name_version_type_regex, packagename)
+
+        bb.debug("VersionURL: %s" % (url))
+        soup = BeautifulSoup(self._fetch_index(url, ud, d))
+        if not soup:
+            bb.debug("*** %s NO SOUP" % (packagename))
+            return ""
+
+        pn_regex = d.getVar('REGEX', True)
+        if pn_regex:
+            testversion = version
+            pn_regex = re.compile(pn_regex)
+            bb.debug("pn_regex = '%s'" % (pn_regex.pattern))
+            
+        for line in soup.find_all('a', href=True):
+            newver = ('', '', '')
+            bb.debug("line = '%s'" % (line['href']))
+            if pn_regex:
+                m = pn_regex.search(line['href'])
+                if m:
+                    bb.debug("Name = '%s', Pver = '%s'" % (m.group('name'), m.group('pver')))
+                    newver = (m.group('name'), m.group('pver'), '')
+                else:
+                    continue
+            else:
+                newver = self._parse_path(self.name_version_type_regex, line['href'])
+            valid = 1
+            if newver and self._vercmp(version, newver) == True:
+                version = newver
+                
+        # check whether a valid package and version were found
+
+        if not valid:
+            version = ('', '', '')
+        if not pn_regex:
+            testversion = ('', '', '')
+        bb.debug("*** %s -> %s (TestVersion = %s)" % (packagename, version[1], testversion[1]))
+        if valid and version:
+            return re.sub('_', '.', version[1])
+
+    def latest_versionstring(self, ud, d):
+        """
+        Manipulate the URL and try to obtain the latest package version
+
+        sanity check to ensure same name and type. Match as many patterns as possible
+        such as:
+                gnome-common-2.20.0.tar.gz (most common format)
+                gtk+-2.90.1.tar.gz
+                xf86-input-synaptics-12.6.9.tar.gz
+                dri2proto-2.3.tar.gz
+                blktool_4.orig.tar.gz
+                libid3tag-0.15.1b.tar.gz
+                unzip552.tar.gz
+                icu4c-3_6-src.tgz
+                genext2fs_1.3.orig.tar.gz
+                gst-fluendo-mp3
+        """
+        # match most patterns which uses "-" as separator to version digits
+        pn_prefix1 = "[a-zA-Z][a-zA-Z0-9]*([\-_][a-zA-Z]\w+)*\+?[\-_]"
+        # a loose pattern such as for unzip552.tar.gz
+        pn_prefix2 = "[a-zA-Z]+"
+        # a loose pattern such as for 80325-quicky-0.4.tar.gz
+        pn_prefix3 = "[0-9]+[\-]?[a-zA-Z]+"
+        # Save the Package Name (pn) Regex for use later
+        self.pn_regex = "(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3)
+
+        # match version
+        version_regex = "(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)"
+
+        # src.rpm extension was added only for rpm package. Can be removed if the rpm
+        # packaged will always be considered as having to be manually upgraded
+        suffixlist = "(tar\.gz|tgz|tar\.bz2|zip|xz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
+        self.suffixregex = re.compile(suffixlist)
+
+        # match name, version and archive type of a package
+        self.name_version_type_regex = re.compile("(?P<name>%s?)\.?v?(?P<ver>%s)(\-source)?[\.\-](?P<type>%s$)" % (self.pn_regex, version_regex, suffixlist))
+
+        regex_uri = d.getVar("REGEX_URI", True)
+        newpath = ud.path
+        pupver = ""
+
+        # search for version matches on folders inside the path, like:
+        # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
+        m = re.search("(?P<dirver>[^/]*(\d+\.)*\d+([\-_]r\d+)*)/", ud.path)
+        bb.debug("path = %s" % (ud.path))
+        bb.debug("Regex: %s" % (self.name_version_type_regex.pattern))
+        if m and not regex_uri:
+            dirver = m.group('dirver')
+            # generate the new uri after removing version directory name
+            newuri = bb.fetch.encodeurl([ud.type, ud.host, ud.path.split(dirver)[0], ud.user, ud.pswd, {}])
+            newversion = self._check_latest_dir(newuri, dirver, ud, d)
+            if newversion and dirver != newversion:
+                newpath = ud.path.replace(dirver, newversion, True)
+
+        # try to acquire all remote files in current directory
+        packagename = newpath.split("/")[-1]            # current package name
+        newpath = newpath.split(packagename)[0] or "/"  # path to directory
+
+        # generate the new uri with the appropriate latest directory
+        newuri = regex_uri or bb.fetch.encodeurl([ud.type, ud.host, newpath, ud.user, ud.pswd, {}])
+        newversion = self._check_latest_version(newuri, packagename, ud, d)
+        while not newversion:
+            # maybe it's hiding in a download directory so try there
+            newuri = "/".join(newuri.split("/")[0:-2]) + "/download"
+            if newuri == "/download" or newuri == "http://download":
+                break
+            newversion = self._check_latest_version(newuri, packagename, ud, d)
+
+        return newversion
+

index e0eefd3..f059d87 100644
@@ -593,6 +593,31 @@ class FetchMethodTest(FetcherTest):
             : "3.82+dbg0.9",
     }
 
+    test_wget_uris = {
+        # packages with versions inside directory name
+        ("util-linux", "http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "")
+            : "2.24.2",
+        ("enchant", "http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "")
+            : "1.6.0",
+        ("cmake", "http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
+            : "2.8.12.1",
+        # packages with versions only in current directory
+        ("eglic", "http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
+            : "2.19",
+        ("gnu-config", "http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
+            : "20120814",
+        # packages with "99" in the name of possible version
+        ("pulseaudio", "http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "")
+            : "5.0",
+        ("xserver-xorg", "http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "")
+            : "1.15.1",
+        # packages with valid REGEX_URI and REGEX
+        ("cups", "http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2", "http://www.cups.org/software.php", "/software\.php\?VERSION=2\.0\.0&FILE=2\.0\.0/(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
+            : "2.0.0",
+        ("db", "http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz", "http://www.oracle.com/technetwork/products/berkeleydb/downloads/index-082944.html", "http://download.oracle.com/otn/berkeley-db/(?P<name>db-)(?P<pver>((\d+[\.\-_]*)+))\.tar\.gz")
+            : "6.1.19",
+    }
+
     def test_git_latest_versionstring(self):
         for k, v in self.test_git_uris.items():
             self.d.setVar("SRCREV", k[2])

@@ -602,3 +627,13 @@ class FetchMethodTest(FetcherTest):
             print("Package %s, version: %s <= %s" % (k[0], v, verstring))
             r = bb.utils.vercmp_string(v, verstring)
             self.assertTrue(r == -1 or r == 0)
+
+    def test_wget_latest_versionstring(self):
+        for k, v in self.test_wget_uris.items():
+            self.d.setVar("REGEX_URI", k[2])
+            self.d.setVar("REGEX", k[3])
+            ud = bb.fetch2.FetchData(k[1], self.d)
+            verstring = ud.method.latest_versionstring(ud, self.d)
+            print("Package %s, version: %s <= %s" % (k[0], v, verstring))
+            r = bb.utils.vercmp_string(v, verstring)
+            self.assertTrue(r == -1 or r == 0)