Skip to contentSkip to conversation

Please help to review the Yocto 1656: Recipe creation/import script

DISCUSSION Kang Kai 4 participants 5 messages Jan 19, 2012 – Jan 30, 2012 1 message not in the archive

lore
Share & tools
lore

Kang Kai

author
#1 lore raw
In reply to a message not in the archive

Hi Josh,

The attachment is the the design document V2 and implement.

The V2 document changes are:
1 remove patch_set from argument list
2 deb is not support right now

Any comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.

Thanks& Regards,
Kai

Design document for Recipe creation/import script

* Purpose

The feature is from Yocto 1.2 Bug 1656. A script or similar system that would allow you to give is an upstream URL, tarball, patch set, package (SRPM, or debian style) and generate a recipe based on those instructions.

Name it with bitbake-createbb, and integrate into the build system to use the build system infrastructure that similiar with bitbake-runtask.

* Usage:
bitbake-createbb <SRC_URI>

* Design Flow
1 download the source package from the SRC_URI
2 get package name and version from package or tarball name.
3 unpack to /tmp/bitbake-createbb-pid
4 if tarball, get summary and description from freshmeat.org, README files and pkgconfig files;
otherwise get them from SRPM spec file or debian control file
5 get license, license file and checksum from LICENSE/COPYING files or freshmeat.org. 6 get dependent packages
|-- get them by parse configure files
|-- just parse spec file
8 remove downloaded files

#!/usr/bin/env python

import os
import sys
import shutil
import re
import urllib
import sha, md5, hashlib

tmpdir = "/tmp/bitbake_createbb.%s" % os.getpid()
bin_file_name = ''
pkgname = ''
pkgversion = ''
summary = ''
description = ''
homepage = ''
license = []
license_files = {}
licenses = {}
depends = []
src_uri = ''
src_uri_md5sum = ''
src_uri_sha256sum = ''
inherits = []

# support 2 package types: tar and rpm
pkgtype = ""

def usage():
print "This is the bitbake import bb file script. Usage:\n" print "\tbitbake-createbb <SRC_URI> \n"

def download(pkg_uri):
try:
shutil.rmtree(tmpdir)
except:
pass

cwd = os.getcwd()
os.mkdir(tmpdir)
os.chdir(tmpdir)
print 'Downloading package: ', pkg_uri, ' to ', tmpdir ret = os.system("wget --quiet " + pkg_uri) os.chdir(cwd)

return ret

def guess_name_by_uri(uri):
global bin_file_name, pkgname, pkgversion, pkgtype

elems = uri.rsplit('/', 1)
pkg = bin_file_name = elems[1]

if pkg.endswith("tgz"):
pkg = re.sub("tgz$", "tar.gz", pkg)

match = re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg) if match:
pkgtype = 'tar'
pkgname = match.group(1)
pkgversion = match.group(2)

match = re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg) if match:
pkgtype = 'rpm'
pkgname = match.group(1)
pkgversion = match.group(2)

def get_md5_sha256_sum():

    global src_uri_md5sum, src_uri_sha256sum

    curdir = os.getcwd()
    os.chdir(tmpdir)

    f = file(bin_file_name, 'r')
    content = f.read()
    f.close()

    md_five = md5.new()
    md_five.update(content)
    src_uri_md5sum = md_five.hexdigest()

    sha_256 = hashlib.sha256()
    sha_256.update(content)
    src_uri_sha256sum = sha_256.hexdigest()

    os.chdir(curdir)

def unpack_package():
global bin_file_name

cwd = os.getcwd()
os.chdir(tmpdir)
if pkgtype == 'rpm':
cmdline = "rpm2cpio " + bin_file_name + " | cpio -id" ret = os.system(cmdline)
if ret != 0:
return ret
for item in os.listdir('.'):
if re.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
bin_file_name = item
break

cmdline = "tar axf " + bin_file_name
ret = os.system(cmdline)

os.chdir(cwd)
return ret

# get the fall-back description when other way fail
# check homepage at same time
def guess_description_from_readme(readme):
    global description, homepage
    f = file(readme)
    state = 0
    desc = ''

for line in f:
if state == 1 and re.match('^\n', line) and len(desc) > 80:
state = 2
if state == 0 and len(line) > 1:
state = 1
if state == 1:
desc += line

match = re.search('(http\:\/\/.*$name.*\.org)', line) if match:
url = match.group(1)
if re.search('bug') or len(homepage) > 1:
pass
else:
homepage = url

f.close()
if (len(desc) > 4 and len(description) < 3):
description = desc

def guess_description_from_freecode(pkgname):

    global description
    desc = ''
    state = 0

    html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
    for line in html:
        if state == 1:
            desc += line
        if state == 0 and re.search('\<div class\=\"project-detail\"\>', line):
            state = 1
        if state == 1 and re.search('\<\/p\>', line):
            state = 2

    # deal the description
    desc = re.sub('\<p\>', '', desc)
    desc = re.sub('\<\/p\>', '', desc)
    desc = re.sub('\r', '', desc)
    desc = desc.strip()
    if len(desc) > 10:
       description = desc
# get Summary from pkgconfig file
def guess_summary_from_pc(pc):
    global summary

fn = file(pc)
for line in fn:
match = re.match('Description:\s*(.*)', line) if match and len(summary) < 2:
summary = match.group(1)
break

summary = summary.strip()
fn.close()

def guess_description(pkgdir):
global pkgname, description, summary

readmes = []
pcs = []

for subdir in os.listdir(pkgdir):
if re.match('^README$', subdir):
readmes.insert(0, os.path.join(pkgdir, subdir))
elif re.match('README.*', subdir):
readmes.append(os.path.join(pkgdir, subdir))
elif re.match('.*\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elif re.match(pkgname + '\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elif re.match('.*\.pc', subdir):
pcs.append(os.path.join(pkgdir, subdir))

for readme in readmes:
guess_description_from_readme(os.path.join(pkgdir, readme))

if (len(pkgname) > 2):
guess_description_from_freecode(pkgname)

for pc in pcs:
guess_summary_from_pc(pc)

# if didn't get summary, use first line of description if len(summary) < 2:
summary = description
summary = re.sub("\n", " ", summary) summary = re.sub("\s+", " ", summary) match = re.match("(.*?)\.", summary) summary = match.group(1)

# the sha1sum values are from autospectacle
def setup_licenses():
    licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] =  "GPLv2"
    licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2"
    licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2"
    licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2"
    licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1"
    licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2"
    licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2"
    licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2"
    licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2"
    licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1"
    licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2"
    licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1"
    licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3"
    licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT"
    licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD"
    licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1"
    licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2"
    licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1"
    licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2"
    licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1"
    licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"

def guess_licenses_from_file(copying, relname):
global licenses, license

sha1 = sha.new()

fn = open(copying)
content = fn.read()
fn.close()

sha1.update(content)
digest = sha1.hexdigest()

if digest in licenses:
license.append(licenses[digest]) md_five = md5.new() md_five.update(content) license_files[relname] = md_five.hexdigest()

def guess_licenses_from_freecode():
global license

lic = ''
state = 0

html = urllib.urlopen("http://freecode.com/projects/" + pkgname) for line in html:
if state == 1:
lic += line
if state == 0 and re.search('<th><span>Licenses</span></th>', line):
state = 1
if state == 1 and re.search('</a>', line):
state = 2

# deal the license
lic = lic.strip()
match = re.search('<a.*?>(.*?)</a>', lic)
if match:
license.append(match.group(1))

def guess_license(pkgdir):

    for item in  os.listdir(pkgdir):
        realpath = os.path.join(pkgdir, item)
        if os.path.isfile(realpath) and (re.match('COPY.*', item) \
                or re.match('LICENSE.*', item) or re.match('GPL.*', item)):
            guess_licenses_from_file(realpath, item)

    if len(license) == 0:
        guess_licenses_from_freecode()

def process_configure(pkgdir):
if os.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
os.system("cd " + pkgdir + " ; ./autogen.sh &> /dev/null");

configure = os.path.join(pkgdir, 'configure')
if os.path.isfile(configure):
if 'autotools' not in inherits:
inherits.append('autotools')

fn = open(configure)
for line in fn:
match = re.match('PACKAGE_NAME=\'(.*?)\'', line) if match and len(pkgname) == 0:
pkgname = match.group(1)

match = re.match('PACKAGE_TARNAME=\'(.*?)\'', line) if match:
pkgname = match.group(1)

match = re.match('PACKAGE_VERSION=\'(.*?)\'', line) if match:
pkgversion = match.group(1)

match = re.match('PACKAGE_URL=\'(.*?)\'', line) if match:
homepage = match.group(1)

fn.close()

def push_buildreq(dep):

    global depends

    # remove collateral ] ) etc damage in the string
    dep = re.sub("\"", "", dep)
    dep = re.sub("\)", "", dep)
    dep = re.sub("\]", "", dep)
    dep = re.sub("\[", "", dep)

    # first, undo the space packing
    dep = re.sub(">=", " >= ", dep)
    dep = re.sub("<=", " <= ", dep)

    items = dep.split(' ')
    dep = items[0]

    # don't show configure variables, we can't deal with them
    if re.search("\$", dep):
        return
    if re.search("AC_SUBST", dep):
        return

    if dep not in depends:
        depends.append(dep)

def parse_configure_ac(ac_file):

    depth = 0
    clause = ""

    fac = file(ac_file)

    for line in fac:
        line = line.strip()
        i = 0
        while i < len(line):
            if line[i] == '(':
                depth += 1
            if line[i] == ')' and depth > 0:
                depth -= 1
            clause += line[i]
            i += 1
        if depth > 0:
            continue

        # remove '\n'
        clause = re.sub('\n', '', clause)
        clause = clause.strip()

        match = re.match('PKG_CHECK_MODULES\((.*)\)', clause)
        if match:
            modules = match.group(1)
            pkg = modules.split(',', 1)[1].strip()
            match2 = re.match('\[(.*)\]', pkg)
            if match2:
                pkg = match2.group(1)

            # split the build dependencies
            pkg = pkg.replace('\s+', ' ')
            pkg = re.sub('\s>\s', '>', pkg)
            pkg = re.sub('\s>=\s', '>=', pkg)
            pkg = re.sub('\s=\s', '=', pkg)
            pkg = re.sub('\s<=\s', '<=', pkg)
            pkg = re.sub('\s<\s', '<', pkg)
            pkglist = pkg.split(' ')
            for dep in pkglist:
                push_buildreq(dep)

        match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause)
        if match:
            exists = match.group(1)
            pkg = exists.split(',', 1)[0].strip()
            match2 = re.match('\[(.*)\]', pkg)
            if match2:
                pkg = match2.group(1)
                pkg = pkg.strip()

            pkg = re.sub('\s+', ' ', pkg)
            pkg = re.sub('\s>\s', '>', pkg)
            pkg = re.sub('\s>=\s', '>=', pkg)
            pkg = re.sub('\s=\s', '=', pkg)
            pkg = re.sub('\s<=\s', '<=', pkg)
            pkg = re.sub('\s<\s', '<', pkg)
            pkglist = pkg.split(' ')
            for dep in pkglist:
                push_buildreq(dep)

        # these items are from autospectacle.pl
        if re.search('_PROG_INTLTOOL', clause):
            push_buildreq("intltool")
        if re.search('GETTEXT_PACKAGE', clause):
            push_buildreq('gettext')
        if re.search('GTK_DOC_CHECK', clause):
            push_buildreq('gtk-doc')
        if re.search('GNOME_DOC_INIT', clause):
            push_buildreq('gnome-doc-utils')
        if re.search('AM_GLIB_GNU_GETTEXT', clause):
            push_buildreq('gettext')

        match = re.search('AC_INIT\((.*)\)', clause)
        if match:
            ac_init = match.group(1)
            ac_init = ac_init.strip()
            ac_init = re.sub('\s+', ' ', ac_init)
            version = ac_init.split(',')[1].strip()
            if re.match('\d+(\.\d+)*', version):
                pkgversion = version

        match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause)
        if match:
            am_init = match.group(1)
            am_init = re.sub('\s+', ' ', am_init)
            items = am_init.split(',')
            if len(items) >= 2:
                ver = items[1]
                ver = re.sub('\[', '', ver)
                ver = re.sub('\]', '', ver)
                if re.match('\d(\.\d+)*', ver):
                    pkgversion = ver
        clause = ''
    fac.close

def process_configure_ac(pkgdir):
configure_acs = []
for root, dirnames, filenames in os.walk(pkgdir):
for f in filenames:
if re.match('.*\.ac', f):
configure_acs.append(os.path.join(root, f)) if 'autotools' not in inherits:
inherits.append('autotools')

for ac in configure_acs:
parse_configure_ac(ac)

# check the *.pro files and get build requires from 'PKGCONFIG'
def process_qmake_pro(pkgdir):
    global pkgname
    pro_files = []
    depth = 0
    clause = ''
    pre_char = ''

    if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and 'qmake2' not in inherits:
        inherits.append('qmake2')
    for root, dirnames, filenames in os.walk(pkgdir):
        for f in filenames:
            if re.match('.*\.pro', f):
                pro_files.append(os.path.join(pkgdir, f))

    for pro in pro_files:
        need_next_line = False
        f = file(pro)
        for line in f:
            if line.startswith('#'):
                continue
            line = re.sub('\n', '', line)
            clause += line
            if line.endswith('\\'):
                continue

            clause = clause.strip()
            match =  re.match('PKGCONFIG\s*=(.*)', clause)
            if match:
                dep = match.group(1)
                push_buildreq(dep)
            match = re.match('PKGCONFIG\s*\+=(.*)', clause)
            if match:
                dep = match.group(1)
                dep = dep.strip()
                dep = re.sub('\\\\', '', dep)
                dep = re.sub('\s+', ' ', dep)
                items = dep.split(' ')
                for item in items:
                    push_buildreq(item)

            clause = ''

        f.close()

def parse_spec_file(tmpdir):

    # find the spec file first
    spec_file = ''
    for item in os.listdir(tmpdir):
        if re.match('.*\.spec', item):
            spec_file = os.path.join(tmpdir, item)
            break
    if len(spec_file) == 0:
        return 1

    global pkgname, pkgversion, summary, description, depends
    global src_uri, homepage

    f = file(spec_file)
    for line in f:
        match = re.match('Summary\s*:(.*)', line)
        if match:
            if len(summary) == 0:
                summary = match.group(1).strip()
            continue
        match = re.match('Name\s*:(.*)', line)
        if match:
            pkgname = match.group(1).strip()
            continue
        match = re.match('Version\s*:(.*)', line)
        if match:
            version = match.group(1).strip()
            if re.match('\d+(\.\d+)?', version):
                pkgversion = version
            continue
        match = re.match('License\s*:(.*)', line)
        if match:
            license.append(match.group(1).strip())
            continue
        match = re.match('URL(?i)\s*:(.*)', line)
        if match:
            homepage = match.group(1).strip()
            continue
        match = re.match('Source0*\s*:(.*)', line)
        if match:
            src_uri = match.group(1).strip()
            src_uri = re.sub('\%\{name\}', pkgname, src_uri)
            src_uri = re.sub('\%\{version\}', pkgversion, src_uri)
            continue
        match = re.match('BuildRequires\s*:(.*)', line)
        if match:
            deps = match.group(1).strip()
            dpes = re.sub(',', ' ', deps)
            deps = re.sub('\s+', ' ', deps)
            for item in deps.split(' '):
                push_buildreq(item)

        match = re.match('%description\s*$', line)
        if match:
            for l in f:
                if l.startswith('%'):
                    break
                description += l
            description = re.sub('\n', '', description)

    f.close()

def write_bbfile():

    if len(pkgname) == 0 or len(pkgversion) == 0:
        fail_quit("Can't get package name or version.")

    content = ''
    content += 'DESCRIPTION = "' + description + '"\n'
    content += 'SUMMARY = "' + summary + '"\n'
    if len(homepage) > 0:
        content += 'HOMEPAGE = "' + homepage + '"\n'

    content += 'LICENSE = "'
    for lic in license:
        content += lic + ' '
    content += '"\n'
    content += 'LIC_FILES_CHKSUM = "'
    for key in license_files:
        content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n';
    content += '\t\t"\n\n';

    if len(depends) > 0:
        content += 'DEPENDS = "'
        for dep in depends:
            content += dep + ' '
        content += ' "\n'

    content += 'PR = "r0"\n\n'

    content += 'SRC_URI = "' + src_uri + '"\n'
    content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n'
    content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n'

    if len(inherits) > 0:
        content += '\ninherit '
        for inherit in inherits:
            content += inherit + ' '
        content += '\n'

    bb_filename = pkgname + '_' + pkgversion + '.bb'
    bb_file = open(bb_filename, 'w')
    bb_file.write(content)
    bb_file.close()
    print 'Create bb file : ', bb_filename

def fail_quit(msg):

    print msg
    clean_up()
    exit(1)

def clean_up():
shutil.rmtree(tmpdir)
pass

##########################################################

if __name__ == '__main__':

    if (len(sys.argv) < 2):
        usage()
        exit(1)

    src_uri = sys.argv[1]

    guess_name_by_uri(src_uri)
    if pkgname is None:
        print "Can't get the package name."
        exit(1)

    ret = download(src_uri)
    if ret != 0:
        fail_quit('Download package failed. Make sure the SRC_URI is valid.')

    if unpack_package() != 0:
        fail_quit('Unpack package failed.')

    if pkgtype == 'rpm':
        parse_spec_file(tmpdir)

    get_md5_sha256_sum()

    pkgdir = ''
    for subdir in os.listdir(tmpdir):
        subdir = os.path.join(tmpdir, subdir)
        if os.path.isdir(subdir):
            pkgdir = subdir
            break

    if pkgtype == 'tar':
        parse_spec_file(pkgdir)

    guess_description(pkgdir)
    process_configure(pkgdir)

    setup_licenses()
    guess_license(pkgdir)

    process_configure_ac(pkgdir)
    process_qmake_pro(pkgdir)

    write_bbfile()
    clean_up()
#2 lore raw

Hi Kai,

Quoted 8 lines · On Thursday 19 January 2012 17:33:47 Kang Kai wrote:

The attachment is the the design document V2 and implement.

The V2 document changes are:
1 remove patch_set from argument list
2 deb is not support right now

Any comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.

Great work! I can see this being really useful to help people get started with new recipes. A few comments:

1) I tried it on hdparm and it worked very well. However, obexftp caused an error:

----------------
Downloading package:
http://downloads.sourceforge.net/openobex/obexftp-0.23.tar.bz2 to /tmp/bitbake_createbb.5729
Traceback (most recent call last):
File "./bitbake-createbb", line 643, in <module>
process_configure(pkgdir)
File "./bitbake-createbb", line 295, in process_configure

    if match and len(pkgname) == 0:

UnboundLocalError: local variable 'pkgname' referenced before assignment
----------------

2) If it is able to find license file(s) but unable to find any appropriate for LIC_FILES_CHKSUM, it would be nice if it would print a warning just so the user knows they will need to set this themselves (bonus points if you still list LIC_FILES_CHKSUM entries for license files that did not match the known checksums either in a comment in the output or in the warning message, to save the user work if they know the files are appropriate). If you prefer I can send a patch afterwards for this.

3) If you have an RPM spec file it is possible to extract PACKAGES and FILES and possibly even EXTRA_OECONF. Somewhere around here I have a very basic script to do this I think, I could send it to you if you like (or we can leave this for later).

Cheers,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre
#3 lore raw
In reply to Kang Kai
Quoted 24 lines · Kang Kai

Hi Josh,

The attachment is the the design document V2 and implement.

The V2 document changes are:
1 remove patch_set from argument list
2 deb is not support right now

Any comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.

Thanks& Regards,
Kai

recipe_creation_bb.txt

Design document for Recipe creation/import script

* Purpose

Quoted 7 lines

The feature is from Yocto 1.2 Bug 1656. A script or similar system that would allow you to give is an upstream URL, tarball, patch set, package (SRPM, or debian style) and generate a recipe based on those instructions.

Name it with bitbake-createbb, and integrate into the build system to use the build system infrastructure that similiar with bitbake-runtask.

I'd suggest changing the name to recipe-createbb or something. Since this isn't a "bitbake" program (or is it?)

Quoted 10 lines

* Usage:
bitbake-createbb<SRC_URI>

* Design Flow
1 download the source package from the SRC_URI
2 get package name and version from package or tarball name.
3 unpack to /tmp/bitbake-createbb-pid
4 if tarball, get summary and description from freshmeat.org, README files and pkgconfig files;
otherwise get them from SRPM spec file or debian control file
5 get license, license file and checksum from LICENSE/COPYING files or freshmeat.org.

For license files, I would suggest we scan for common filenames:

LICENSE, COPYING, COPYING.LIB, and COPYRIGHT

If it doesn't match the known checksums, we tell the user and let them manually figure out the license type and verify things are consisten, but we should be able to fill out the license filename for them.

Otherwise, it all looks fine to me.

--Mark

Quoted 672 lines

6 get dependent packages
|-- get them by parse configure files
|-- just parse spec file
8 remove downloaded files

bitbake-createbb

#!/usr/bin/env python

import os
import sys
import shutil
import re
import urllib
import sha, md5, hashlib

tmpdir = "/tmp/bitbake_createbb.%s" % os.getpid()
bin_file_name = ''
pkgname = ''
pkgversion = ''
summary = ''
description = ''
homepage = ''
license = []
license_files = {}
licenses = {}
depends = []
src_uri = ''
src_uri_md5sum = ''
src_uri_sha256sum = ''
inherits = []

# support 2 package types: tar and rpm
pkgtype = ""

def usage():
print "This is the bitbake import bb file script. Usage:\n" print "\tbitbake-createbb<SRC_URI> \n"

def download(pkg_uri):
try:
shutil.rmtree(tmpdir)
except:
pass

cwd = os.getcwd()
os.mkdir(tmpdir)
os.chdir(tmpdir)
print 'Downloading package: ', pkg_uri, ' to ', tmpdir ret = os.system("wget --quiet " + pkg_uri) os.chdir(cwd)

return ret

def guess_name_by_uri(uri):
global bin_file_name, pkgname, pkgversion, pkgtype

elems = uri.rsplit('/', 1)
pkg = bin_file_name = elems[1]

if pkg.endswith("tgz"):
pkg = re.sub("tgz$", "tar.gz", pkg)

match = re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg) if match:
pkgtype = 'tar'
pkgname = match.group(1)
pkgversion = match.group(2)

match = re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg) if match:
pkgtype = 'rpm'
pkgname = match.group(1)
pkgversion = match.group(2)

def get_md5_sha256_sum():

     global src_uri_md5sum, src_uri_sha256sum

     curdir = os.getcwd()
     os.chdir(tmpdir)

     f = file(bin_file_name, 'r')
     content = f.read()
     f.close()

     md_five = md5.new()
     md_five.update(content)
     src_uri_md5sum = md_five.hexdigest()

     sha_256 = hashlib.sha256()
     sha_256.update(content)
     src_uri_sha256sum = sha_256.hexdigest()

     os.chdir(curdir)

def unpack_package():
global bin_file_name

cwd = os.getcwd()
os.chdir(tmpdir)
if pkgtype == 'rpm':
cmdline = "rpm2cpio " + bin_file_name + " | cpio -id" ret = os.system(cmdline)
if ret != 0:
return ret
for item in os.listdir('.'):
if re.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
bin_file_name = item
break

cmdline = "tar axf " + bin_file_name
ret = os.system(cmdline)

os.chdir(cwd)
return ret

# get the fall-back description when other way fail
# check homepage at same time
def guess_description_from_readme(readme):
     global description, homepage
     f = file(readme)
     state = 0
     desc = ''

for line in f:
if state == 1 and re.match('^\n', line) and len(desc)> 80:
state = 2
if state == 0 and len(line)> 1:
state = 1
if state == 1:
desc += line

match = re.search('(http\:\/\/.*$name.*\.org)', line) if match:
url = match.group(1)
if re.search('bug') or len(homepage)> 1:
pass
else:
homepage = url

f.close()
if (len(desc)> 4 and len(description)< 3):
description = desc

def guess_description_from_freecode(pkgname):

     global description
     desc = ''
     state = 0

     html = urllib.urlopen("http://freecode.com/projects/"  + pkgname)
     for line in html:
         if state == 1:
             desc += line
         if state == 0 and re.search('\<div class\=\"project-detail\"\>', line):
             state = 1
         if state == 1 and re.search('\<\/p\>', line):
             state = 2

     # deal the description
     desc = re.sub('\<p\>', '', desc)
     desc = re.sub('\<\/p\>', '', desc)
     desc = re.sub('\r', '', desc)
     desc = desc.strip()
     if len(desc)>  10:
        description = desc
# get Summary from pkgconfig file
def guess_summary_from_pc(pc):
     global summary

fn = file(pc)
for line in fn:
match = re.match('Description:\s*(.*)', line) if match and len(summary)< 2:
summary = match.group(1)
break

summary = summary.strip()
fn.close()

def guess_description(pkgdir):
global pkgname, description, summary

readmes = []
pcs = []

for subdir in os.listdir(pkgdir):
if re.match('^README$', subdir):
readmes.insert(0, os.path.join(pkgdir, subdir))
elif re.match('README.*', subdir):
readmes.append(os.path.join(pkgdir, subdir))
elif re.match('.*\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elif re.match(pkgname + '\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elif re.match('.*\.pc', subdir):
pcs.append(os.path.join(pkgdir, subdir))

for readme in readmes:
guess_description_from_readme(os.path.join(pkgdir, readme))

if (len(pkgname)> 2):
guess_description_from_freecode(pkgname)

for pc in pcs:
guess_summary_from_pc(pc)

# if didn't get summary, use first line of description if len(summary)< 2:
summary = description
summary = re.sub("\n", " ", summary) summary = re.sub("\s+", " ", summary) match = re.match("(.*?)\.", summary) summary = match.group(1)

# the sha1sum values are from autospectacle
def setup_licenses():
     licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] =  "GPLv2"
     licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2"
     licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2"
     licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2"
     licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1"
     licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2"
     licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2"
     licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2"
     licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2"
     licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1"
     licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2"
     licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1"
     licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3"
     licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT"
     licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD"
     licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1"
     licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2"
     licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1"
     licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2"
     licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1"
     licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"

def guess_licenses_from_file(copying, relname):
global licenses, license

sha1 = sha.new()

fn = open(copying)
content = fn.read()
fn.close()

sha1.update(content)
digest = sha1.hexdigest()

if digest in licenses:
license.append(licenses[digest]) md_five = md5.new() md_five.update(content) license_files[relname] = md_five.hexdigest()

def guess_licenses_from_freecode():
global license

lic = ''
state = 0

html = urllib.urlopen("http://freecode.com/projects/" + pkgname) for line in html:
if state == 1:
lic += line
if state == 0 and re.search('<th><span>Licenses</span></th>', line):
state = 1
if state == 1 and re.search('</a>', line):
state = 2

# deal the license
lic = lic.strip()
match = re.search('<a.*?>(.*?)</a>', lic)
if match:
license.append(match.group(1))

def guess_license(pkgdir):

     for item in  os.listdir(pkgdir):
         realpath = os.path.join(pkgdir, item)
         if os.path.isfile(realpath) and (re.match('COPY.*', item) \
                 or re.match('LICENSE.*', item) or re.match('GPL.*', item)):
             guess_licenses_from_file(realpath, item)

     if len(license) == 0:
         guess_licenses_from_freecode()

def process_configure(pkgdir):
if os.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
os.system("cd " + pkgdir + " ; ./autogen.sh&> /dev/null");

configure = os.path.join(pkgdir, 'configure') if os.path.isfile(configure):
if 'autotools' not in inherits:
inherits.append('autotools')

fn = open(configure)
for line in fn:
match = re.match('PACKAGE_NAME=\'(.*?)\'', line) if match and len(pkgname) == 0:
pkgname = match.group(1)

match = re.match('PACKAGE_TARNAME=\'(.*?)\'', line) if match:
pkgname = match.group(1)

match = re.match('PACKAGE_VERSION=\'(.*?)\'', line) if match:
pkgversion = match.group(1)

match = re.match('PACKAGE_URL=\'(.*?)\'', line) if match:
homepage = match.group(1)

fn.close()

def push_buildreq(dep):

     global depends

     # remove collateral ] ) etc damage in the string
     dep = re.sub("\"", "", dep)
     dep = re.sub("\)", "", dep)
     dep = re.sub("\]", "", dep)
     dep = re.sub("\[", "", dep)

     # first, undo the space packing
     dep = re.sub(">=", ">= ", dep)
     dep = re.sub("<=", "<= ", dep)

     items = dep.split(' ')
     dep = items[0]

     # don't show configure variables, we can't deal with them
     if re.search("\$", dep):
         return
     if re.search("AC_SUBST", dep):
         return

     if dep not in depends:
         depends.append(dep)

def parse_configure_ac(ac_file):

     depth = 0
     clause = ""

     fac = file(ac_file)

     for line in fac:
         line = line.strip()
         i = 0
         while i<  len(line):
             if line[i] == '(':
                 depth += 1
             if line[i] == ')' and depth>  0:
                 depth -= 1
             clause += line[i]
             i += 1
         if depth>  0:
             continue

         # remove '\n'
         clause = re.sub('\n', '', clause)
         clause = clause.strip()

         match = re.match('PKG_CHECK_MODULES\((.*)\)', clause)
         if match:
             modules = match.group(1)
             pkg = modules.split(',', 1)[1].strip()
             match2 = re.match('\[(.*)\]', pkg)
             if match2:
                 pkg = match2.group(1)

             # split the build dependencies
             pkg = pkg.replace('\s+', ' ')
             pkg = re.sub('\s>\s', '>', pkg)
             pkg = re.sub('\s>=\s', '>=', pkg)
             pkg = re.sub('\s=\s', '=', pkg)
             pkg = re.sub('\s<=\s', '<=', pkg)
             pkg = re.sub('\s<\s', '<', pkg)
             pkglist = pkg.split(' ')
             for dep in pkglist:
                 push_buildreq(dep)

         match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause)
         if match:
             exists = match.group(1)
             pkg = exists.split(',', 1)[0].strip()
             match2 = re.match('\[(.*)\]', pkg)
             if match2:
                 pkg = match2.group(1)
                 pkg = pkg.strip()

             pkg = re.sub('\s+', ' ', pkg)
             pkg = re.sub('\s>\s', '>', pkg)
             pkg = re.sub('\s>=\s', '>=', pkg)
             pkg = re.sub('\s=\s', '=', pkg)
             pkg = re.sub('\s<=\s', '<=', pkg)
             pkg = re.sub('\s<\s', '<', pkg)
             pkglist = pkg.split(' ')
             for dep in pkglist:
                 push_buildreq(dep)

         # these items are from autospectacle.pl
         if re.search('_PROG_INTLTOOL', clause):
             push_buildreq("intltool")
         if re.search('GETTEXT_PACKAGE', clause):
             push_buildreq('gettext')
         if re.search('GTK_DOC_CHECK', clause):
             push_buildreq('gtk-doc')
         if re.search('GNOME_DOC_INIT', clause):
             push_buildreq('gnome-doc-utils')
         if re.search('AM_GLIB_GNU_GETTEXT', clause):
             push_buildreq('gettext')

         match = re.search('AC_INIT\((.*)\)', clause)
         if match:
             ac_init = match.group(1)
             ac_init = ac_init.strip()
             ac_init = re.sub('\s+', ' ', ac_init)
             version = ac_init.split(',')[1].strip()
             if re.match('\d+(\.\d+)*', version):
                 pkgversion = version

         match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause)
         if match:
             am_init = match.group(1)
             am_init = re.sub('\s+', ' ', am_init)
             items = am_init.split(',')
             if len(items)>= 2:
                 ver = items[1]
                 ver = re.sub('\[', '', ver)
                 ver = re.sub('\]', '', ver)
                 if re.match('\d(\.\d+)*', ver):
                     pkgversion = ver
         clause = ''
     fac.close

def process_configure_ac(pkgdir):
configure_acs = []
for root, dirnames, filenames in os.walk(pkgdir):
for f in filenames:
if re.match('.*\.ac', f):
configure_acs.append(os.path.join(root, f)) if 'autotools' not in inherits:
inherits.append('autotools')

for ac in configure_acs:
parse_configure_ac(ac)

# check the *.pro files and get build requires from 'PKGCONFIG'
def process_qmake_pro(pkgdir):
     global pkgname
     pro_files = []
     depth = 0
     clause = ''
     pre_char = ''

     if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and 'qmake2' not in inherits:
         inherits.append('qmake2')
     for root, dirnames, filenames in os.walk(pkgdir):
         for f in filenames:
             if re.match('.*\.pro', f):
                 pro_files.append(os.path.join(pkgdir, f))

     for pro in pro_files:
         need_next_line = False
         f = file(pro)
         for line in f:
             if line.startswith('#'):
                 continue
             line = re.sub('\n', '', line)
             clause += line
             if line.endswith('\\'):
                 continue

             clause = clause.strip()
             match =  re.match('PKGCONFIG\s*=(.*)', clause)
             if match:
                 dep = match.group(1)
                 push_buildreq(dep)
             match = re.match('PKGCONFIG\s*\+=(.*)', clause)
             if match:
                 dep = match.group(1)
                 dep = dep.strip()
                 dep = re.sub('\\\\', '', dep)
                 dep = re.sub('\s+', ' ', dep)
                 items = dep.split(' ')
                 for item in items:
                     push_buildreq(item)

             clause = ''

         f.close()

def parse_spec_file(tmpdir):

     # find the spec file first
     spec_file = ''
     for item in os.listdir(tmpdir):
         if re.match('.*\.spec', item):
             spec_file = os.path.join(tmpdir, item)
             break
     if len(spec_file) == 0:
         return 1

     global pkgname, pkgversion, summary, description, depends
     global src_uri, homepage

     f = file(spec_file)
     for line in f:
         match = re.match('Summary\s*:(.*)', line)
         if match:
             if len(summary) == 0:
                 summary = match.group(1).strip()
             continue
         match = re.match('Name\s*:(.*)', line)
         if match:
             pkgname = match.group(1).strip()
             continue
         match = re.match('Version\s*:(.*)', line)
         if match:
             version = match.group(1).strip()
             if re.match('\d+(\.\d+)?', version):
                 pkgversion = version
             continue
         match = re.match('License\s*:(.*)', line)
         if match:
             license.append(match.group(1).strip())
             continue
         match = re.match('URL(?i)\s*:(.*)', line)
         if match:
             homepage = match.group(1).strip()
             continue
         match = re.match('Source0*\s*:(.*)', line)
         if match:
             src_uri = match.group(1).strip()
             src_uri = re.sub('\%\{name\}', pkgname, src_uri)
             src_uri = re.sub('\%\{version\}', pkgversion, src_uri)
             continue
         match = re.match('BuildRequires\s*:(.*)', line)
         if match:
             deps = match.group(1).strip()
             dpes = re.sub(',', ' ', deps)
             deps = re.sub('\s+', ' ', deps)
             for item in deps.split(' '):
                 push_buildreq(item)

         match = re.match('%description\s*$', line)
         if match:
             for l in f:
                 if l.startswith('%'):
                     break
                 description += l
             description = re.sub('\n', '', description)

     f.close()

def write_bbfile():

     if len(pkgname) == 0 or len(pkgversion) == 0:
         fail_quit("Can't get package name or version.")

     content = ''
     content += 'DESCRIPTION = "' + description + '"\n'
     content += 'SUMMARY = "' + summary + '"\n'
     if len(homepage)>  0:
         content += 'HOMEPAGE = "' + homepage + '"\n'

     content += 'LICENSE = "'
     for lic in license:
         content += lic + ' '
     content += '"\n'
     content += 'LIC_FILES_CHKSUM = "'
     for key in license_files:
         content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n';
     content += '\t\t"\n\n';

     if len(depends)>  0:
         content += 'DEPENDS = "'
         for dep in depends:
             content += dep + ' '
         content += ' "\n'

     content += 'PR = "r0"\n\n'

     content += 'SRC_URI = "' + src_uri + '"\n'
     content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n'
     content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n'

     if len(inherits)>  0:
         content += '\ninherit '
         for inherit in inherits:
             content += inherit + ' '
         content += '\n'

     bb_filename = pkgname + '_' + pkgversion + '.bb'
     bb_file = open(bb_filename, 'w')
     bb_file.write(content)
     bb_file.close()
     print 'Create bb file : ', bb_filename

def fail_quit(msg):

     print msg
     clean_up()
     exit(1)

def clean_up():
shutil.rmtree(tmpdir)
pass

##########################################################

if __name__ == '__main__':

     if (len(sys.argv)<  2):
         usage()
         exit(1)

     src_uri = sys.argv[1]

     guess_name_by_uri(src_uri)
     if pkgname is None:
         print "Can't get the package name."
         exit(1)

     ret = download(src_uri)
     if ret != 0:
         fail_quit('Download package failed. Make sure the SRC_URI is valid.')

     if unpack_package() != 0:
         fail_quit('Unpack package failed.')

     if pkgtype == 'rpm':
         parse_spec_file(tmpdir)

     get_md5_sha256_sum()

     pkgdir = ''
     for subdir in os.listdir(tmpdir):
         subdir = os.path.join(tmpdir, subdir)
         if os.path.isdir(subdir):
             pkgdir = subdir
             break

     if pkgtype == 'tar':
         parse_spec_file(pkgdir)

     guess_description(pkgdir)
     process_configure(pkgdir)

     setup_licenses()
     guess_license(pkgdir)

     process_configure_ac(pkgdir)
     process_qmake_pro(pkgdir)

     write_bbfile()
     clean_up()

_______________________________________________
poky mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/poky

#5 lore raw
In reply to Kang Kai
Quoted 10 lines · Kang Kai

Hi Josh,

The attachment is the the design document V2 and implement.

The V2 document changes are:
1 remove patch_set from argument list
2 deb is not support right now

Any comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.

This looks like a good start, Paul has offered some great feedback. In addition I'd add that we need a license header on the script, we should probably attribute autospectacle in the header too (Inspired by? Based on?).

I haven't had chance to do a thorough code review but will try and do so whilst you're on your break.

Cheers,
Joshua

Signature · 3 lines
Joshua Lock
        Yocto Project "Johannes factotum"
        Intel Open Source Technology Centre
author
#4 lore raw
In reply to Mark Hatle
Quoted 25 lines · Mark Hatle
Quoted 24 lines · Kang Kai

Hi Josh,

The attachment is the the design document V2 and implement.

The V2 document changes are:
1 remove patch_set from argument list
2 deb is not support right now

Any comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.

Thanks& Regards,
Kai

recipe_creation_bb.txt

Design document for Recipe creation/import script

* Purpose

Quoted 8 lines
Quoted 7 lines

The feature is from Yocto 1.2 Bug 1656. A script or similar system that would allow you to give is an upstream URL, tarball, patch set, package (SRPM, or debian style) and generate a recipe based on those instructions.

Name it with bitbake-createbb, and integrate into the build system to use the build system infrastructure that similiar with bitbake-runtask.

Hi Mark,

Thanks for your comments.

Quoted 1 line

I'd suggest changing the name to recipe-createbb or something. Since this isn't a "bitbake" program (or is it?)

How about name it with "create-recipe"?

Quoted 12 lines
Quoted 11 lines

* Usage:
bitbake-createbb<SRC_URI>

* Design Flow
1 download the source package from the SRC_URI
2 get package name and version from package or tarball name.
3 unpack to /tmp/bitbake-createbb-pid
4 if tarball, get summary and description from freshmeat.org, README files and pkgconfig files; otherwise get them from SRPM spec file or debian control file
5 get license, license file and checksum from LICENSE/COPYING files or freshmeat.org.

Quoted 5 lines

For license files, I would suggest we scan for common filenames:

LICENSE, COPYING, COPYING.LIB, and COPYRIGHT

If it doesn't match the known checksums, we tell the user and let them manually figure out the license type and verify things are consisten, but we should be able to fill out the license filename for them.

Ok, if can't get license from known checksums I'll left messages for user to check manually.

Regards,
Kai

Quoted 682 lines

Otherwise, it all looks fine to me.

--Mark

Quoted 667 lines

6 get dependent packages
|-- get them by parse configure files
|-- just parse spec file
8 remove downloaded files

bitbake-createbb

#!/usr/bin/env python

import os
import sys
import shutil
import re
import urllib
import sha, md5, hashlib

tmpdir = "/tmp/bitbake_createbb.%s" % os.getpid()
bin_file_name = ''
pkgname = ''
pkgversion = ''
summary = ''
description = ''
homepage = ''
license = []
license_files = {}
licenses = {}
depends = []
src_uri = ''
src_uri_md5sum = ''
src_uri_sha256sum = ''
inherits = []

# support 2 package types: tar and rpm
pkgtype = ""

def usage():
print "This is the bitbake import bb file script. Usage:\n" print "\tbitbake-createbb<SRC_URI> \n"

def download(pkg_uri):
try:
shutil.rmtree(tmpdir)
except:
pass

cwd = os.getcwd()
os.mkdir(tmpdir)
os.chdir(tmpdir)
print 'Downloading package: ', pkg_uri, ' to ', tmpdir ret = os.system("wget --quiet " + pkg_uri) os.chdir(cwd)

return ret

def guess_name_by_uri(uri):
global bin_file_name, pkgname, pkgversion, pkgtype

elems = uri.rsplit('/', 1)
pkg = bin_file_name = elems[1]

if pkg.endswith("tgz"):
pkg = re.sub("tgz$", "tar.gz", pkg)

match = re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg) if match:
pkgtype = 'tar'
pkgname = match.group(1)
pkgversion = match.group(2)

match = re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg) if match:
pkgtype = 'rpm'
pkgname = match.group(1)
pkgversion = match.group(2)

def get_md5_sha256_sum():
global src_uri_md5sum, src_uri_sha256sum

curdir = os.getcwd()
os.chdir(tmpdir)

f = file(bin_file_name, 'r')
content = f.read()
f.close()

md_five = md5.new()
md_five.update(content)
src_uri_md5sum = md_five.hexdigest()

sha_256 = hashlib.sha256()
sha_256.update(content)
src_uri_sha256sum = sha_256.hexdigest()

os.chdir(curdir)

def unpack_package():
global bin_file_name

cwd = os.getcwd()
os.chdir(tmpdir)
if pkgtype == 'rpm':
cmdline = "rpm2cpio " + bin_file_name + " | cpio -id" ret = os.system(cmdline)
if ret != 0:
return ret
for item in os.listdir('.'):
if re.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
bin_file_name = item
break

cmdline = "tar axf " + bin_file_name
ret = os.system(cmdline)

os.chdir(cwd)
return ret

# get the fall-back description when other way fail
# check homepage at same time
def guess_description_from_readme(readme):
global description, homepage
f = file(readme)
state = 0
desc = ''

for line in f:
if state == 1 and re.match('^\n', line) and len(desc)> 80:
state = 2
if state == 0 and len(line)> 1:
state = 1
if state == 1:
desc += line

match = re.search('(http\:\/\/.*$name.*\.org)', line) if match:
url = match.group(1)
if re.search('bug') or len(homepage)> 1:
pass
else:
homepage = url

f.close()
if (len(desc)> 4 and len(description)< 3):
description = desc

def guess_description_from_freecode(pkgname):
global description
desc = ''
state = 0

html = urllib.urlopen("http://freecode.com/projects/" + pkgname) for line in html:
if state == 1:
desc += line
if state == 0 and re.search('\<div class\=\"project-detail\"\>', line):
state = 1
if state == 1 and re.search('\<\/p\>', line):
state = 2

# deal the description
desc = re.sub('\<p\>', '', desc)
desc = re.sub('\<\/p\>', '', desc)
desc = re.sub('\r', '', desc)
desc = desc.strip()
if len(desc)> 10:
description = desc
# get Summary from pkgconfig file
def guess_summary_from_pc(pc):
global summary

fn = file(pc)
for line in fn:
match = re.match('Description:\s*(.*)', line)
if match and len(summary)< 2:
summary = match.group(1)
break

summary = summary.strip()
fn.close()

def guess_description(pkgdir):
global pkgname, description, summary

readmes = []
pcs = []

for subdir in os.listdir(pkgdir):
if re.match('^README$', subdir):
readmes.insert(0, os.path.join(pkgdir, subdir))
elif re.match('README.*', subdir):
readmes.append(os.path.join(pkgdir, subdir))
elif re.match('.*\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elif re.match(pkgname + '\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elif re.match('.*\.pc', subdir):
pcs.append(os.path.join(pkgdir, subdir))

for readme in readmes:
guess_description_from_readme(os.path.join(pkgdir, readme))

if (len(pkgname)> 2):
guess_description_from_freecode(pkgname)

for pc in pcs:
guess_summary_from_pc(pc)

# if didn't get summary, use first line of description
if len(summary)< 2:
summary = description
summary = re.sub("\n", " ", summary)
summary = re.sub("\s+", " ", summary)
match = re.match("(.*?)\.", summary)
summary = match.group(1)
# the sha1sum values are from autospectacle
def setup_licenses():
licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] = "GPLv2"
licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2"
licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2"
licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2"
licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1"
licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2"
licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2"
licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2"
licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2"
licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1"
licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2"
licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1"
licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3"
licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT"
licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD"
licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1"
licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2"
licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1"
licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2"
licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1"
licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"

def guess_licenses_from_file(copying, relname):
global licenses, license

sha1 = sha.new()

fn = open(copying)
content = fn.read()
fn.close()

sha1.update(content)
digest = sha1.hexdigest()

if digest in licenses:
license.append(licenses[digest])
md_five = md5.new()
md_five.update(content)
license_files[relname] = md_five.hexdigest()

def guess_licenses_from_freecode():
global license

lic = ''
state = 0

html = urllib.urlopen("http://freecode.com/projects/" + pkgname) for line in html:
if state == 1:
lic += line
if state == 0 and re.search('<th><span>Licenses</span></th>', line):
state = 1
if state == 1 and re.search('</a>', line):
state = 2

# deal the license
lic = lic.strip()
match = re.search('<a.*?>(.*?)</a>', lic)
if match:
license.append(match.group(1))

def guess_license(pkgdir):
for item in os.listdir(pkgdir):
realpath = os.path.join(pkgdir, item)
if os.path.isfile(realpath) and (re.match('COPY.*', item) \
or re.match('LICENSE.*', item) or re.match('GPL.*', item)):
guess_licenses_from_file(realpath, item)

if len(license) == 0:
guess_licenses_from_freecode()

def process_configure(pkgdir):
if os.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
os.system("cd " + pkgdir + " ; ./autogen.sh&> /dev/null");

configure = os.path.join(pkgdir, 'configure')
if os.path.isfile(configure):
if 'autotools' not in inherits:
inherits.append('autotools')

fn = open(configure)
for line in fn:
match = re.match('PACKAGE_NAME=\'(.*?)\'', line)
if match and len(pkgname) == 0:
pkgname = match.group(1)

match = re.match('PACKAGE_TARNAME=\'(.*?)\'', line) if match:
pkgname = match.group(1)

match = re.match('PACKAGE_VERSION=\'(.*?)\'', line) if match:
pkgversion = match.group(1)

match = re.match('PACKAGE_URL=\'(.*?)\'', line)
if match:
homepage = match.group(1)

fn.close()

def push_buildreq(dep):
global depends

# remove collateral ] ) etc damage in the string
dep = re.sub("\"", "", dep)
dep = re.sub("\)", "", dep)
dep = re.sub("\]", "", dep)
dep = re.sub("\[", "", dep)

# first, undo the space packing
dep = re.sub(">=", ">= ", dep)
dep = re.sub("<=", "<= ", dep)

items = dep.split(' ')
dep = items[0]

# don't show configure variables, we can't deal with them if re.search("\$", dep):
return
if re.search("AC_SUBST", dep):
return

if dep not in depends:
depends.append(dep)

def parse_configure_ac(ac_file):
depth = 0
clause = ""

fac = file(ac_file)

for line in fac:
line = line.strip()
i = 0
while i< len(line):
if line[i] == '(':
depth += 1
if line[i] == ')' and depth> 0:
depth -= 1
clause += line[i]
i += 1
if depth> 0:
continue

# remove '\n'
clause = re.sub('\n', '', clause)
clause = clause.strip()

match = re.match('PKG_CHECK_MODULES\((.*)\)', clause) if match:
modules = match.group(1)
pkg = modules.split(',', 1)[1].strip()
match2 = re.match('\[(.*)\]', pkg)
if match2:
pkg = match2.group(1)

# split the build dependencies
pkg = pkg.replace('\s+', ' ')
pkg = re.sub('\s>\s', '>', pkg)
pkg = re.sub('\s>=\s', '>=', pkg)
pkg = re.sub('\s=\s', '=', pkg)
pkg = re.sub('\s<=\s', '<=', pkg)
pkg = re.sub('\s<\s', '<', pkg)
pkglist = pkg.split(' ')
for dep in pkglist:
push_buildreq(dep)

match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause) if match:
exists = match.group(1)
pkg = exists.split(',', 1)[0].strip()
match2 = re.match('\[(.*)\]', pkg)
if match2:
pkg = match2.group(1)
pkg = pkg.strip()

pkg = re.sub('\s+', ' ', pkg)
pkg = re.sub('\s>\s', '>', pkg)
pkg = re.sub('\s>=\s', '>=', pkg)
pkg = re.sub('\s=\s', '=', pkg)
pkg = re.sub('\s<=\s', '<=', pkg)
pkg = re.sub('\s<\s', '<', pkg)
pkglist = pkg.split(' ')
for dep in pkglist:
push_buildreq(dep)

# these items are from autospectacle.pl
if re.search('_PROG_INTLTOOL', clause):
push_buildreq("intltool")
if re.search('GETTEXT_PACKAGE', clause):
push_buildreq('gettext')
if re.search('GTK_DOC_CHECK', clause):
push_buildreq('gtk-doc')
if re.search('GNOME_DOC_INIT', clause):
push_buildreq('gnome-doc-utils')
if re.search('AM_GLIB_GNU_GETTEXT', clause):
push_buildreq('gettext')

match = re.search('AC_INIT\((.*)\)', clause)
if match:
ac_init = match.group(1)
ac_init = ac_init.strip()
ac_init = re.sub('\s+', ' ', ac_init)
version = ac_init.split(',')[1].strip()
if re.match('\d+(\.\d+)*', version):
pkgversion = version

match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause) if match:
am_init = match.group(1)
am_init = re.sub('\s+', ' ', am_init)
items = am_init.split(',')
if len(items)>= 2:
ver = items[1]
ver = re.sub('\[', '', ver)
ver = re.sub('\]', '', ver)
if re.match('\d(\.\d+)*', ver):
pkgversion = ver
clause = ''
fac.close

def process_configure_ac(pkgdir):
configure_acs = []
for root, dirnames, filenames in os.walk(pkgdir):
for f in filenames:
if re.match('.*\.ac', f):
configure_acs.append(os.path.join(root, f))
if 'autotools' not in inherits:
inherits.append('autotools')

for ac in configure_acs:
parse_configure_ac(ac)

# check the *.pro files and get build requires from 'PKGCONFIG'
def process_qmake_pro(pkgdir):
global pkgname

pro_files = []
depth = 0
clause = ''
pre_char = ''

if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and 'qmake2' not in inherits:
inherits.append('qmake2')
for root, dirnames, filenames in os.walk(pkgdir):
for f in filenames:
if re.match('.*\.pro', f):
pro_files.append(os.path.join(pkgdir, f))

for pro in pro_files:
need_next_line = False
f = file(pro)
for line in f:
if line.startswith('#'):
continue
line = re.sub('\n', '', line)
clause += line
if line.endswith('\\'):
continue

clause = clause.strip()
match = re.match('PKGCONFIG\s*=(.*)', clause)
if match:
dep = match.group(1)
push_buildreq(dep)
match = re.match('PKGCONFIG\s*\+=(.*)', clause)
if match:
dep = match.group(1)
dep = dep.strip()
dep = re.sub('\\\\', '', dep)
dep = re.sub('\s+', ' ', dep)
items = dep.split(' ')
for item in items:
push_buildreq(item)

clause = ''

f.close()

def parse_spec_file(tmpdir):

# find the spec file first
spec_file = ''
for item in os.listdir(tmpdir):
if re.match('.*\.spec', item):
spec_file = os.path.join(tmpdir, item)
break

if len(spec_file) == 0:
return 1

global pkgname, pkgversion, summary, description, depends global src_uri, homepage

f = file(spec_file)
for line in f:
match = re.match('Summary\s*:(.*)', line)
if match:
if len(summary) == 0:
summary = match.group(1).strip()
continue
match = re.match('Name\s*:(.*)', line)
if match:
pkgname = match.group(1).strip()
continue
match = re.match('Version\s*:(.*)', line)
if match:
version = match.group(1).strip()
if re.match('\d+(\.\d+)?', version):
pkgversion = version
continue
match = re.match('License\s*:(.*)', line)
if match:
license.append(match.group(1).strip()) continue
match = re.match('URL(?i)\s*:(.*)', line)
if match:
homepage = match.group(1).strip()
continue
match = re.match('Source0*\s*:(.*)', line)
if match:
src_uri = match.group(1).strip()
src_uri = re.sub('\%\{name\}', pkgname, src_uri) src_uri = re.sub('\%\{version\}', pkgversion, src_uri) continue
match = re.match('BuildRequires\s*:(.*)', line) if match:
deps = match.group(1).strip()
dpes = re.sub(',', ' ', deps)
deps = re.sub('\s+', ' ', deps)
for item in deps.split(' '):
push_buildreq(item)

match = re.match('%description\s*$', line)
if match:
for l in f:
if l.startswith('%'):
break
description += l
description = re.sub('\n', '', description)

f.close()

def write_bbfile():
if len(pkgname) == 0 or len(pkgversion) == 0:
fail_quit("Can't get package name or version.")

content = ''
content += 'DESCRIPTION = "' + description + '"\n' content += 'SUMMARY = "' + summary + '"\n' if len(homepage)> 0:
content += 'HOMEPAGE = "' + homepage + '"\n'

content += 'LICENSE = "'
for lic in license:
content += lic + ' '
content += '"\n'
content += 'LIC_FILES_CHKSUM = "'
for key in license_files:
content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n'; content += '\t\t"\n\n';

if len(depends)> 0:
content += 'DEPENDS = "'
for dep in depends:
content += dep + ' '
content += ' "\n'

content += 'PR = "r0"\n\n'

content += 'SRC_URI = "' + src_uri + '"\n'
content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n' content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n'

if len(inherits)> 0:
content += '\ninherit '
for inherit in inherits:
content += inherit + ' '
content += '\n'

bb_filename = pkgname + '_' + pkgversion + '.bb'
bb_file = open(bb_filename, 'w')
bb_file.write(content)
bb_file.close()
print 'Create bb file : ', bb_filename

def fail_quit(msg):
print msg
clean_up()
exit(1)

def clean_up():
shutil.rmtree(tmpdir)
pass

##########################################################

if __name__ == '__main__':
if (len(sys.argv)< 2):
usage()
exit(1)

src_uri = sys.argv[1]

guess_name_by_uri(src_uri)
if pkgname is None:
print "Can't get the package name."
exit(1)

ret = download(src_uri)
if ret != 0:
fail_quit('Download package failed. Make sure the SRC_URI is valid.')

if unpack_package() != 0:
fail_quit('Unpack package failed.')

if pkgtype == 'rpm':
parse_spec_file(tmpdir)

get_md5_sha256_sum()

pkgdir = ''
for subdir in os.listdir(tmpdir):
subdir = os.path.join(tmpdir, subdir)
if os.path.isdir(subdir):
pkgdir = subdir
break

if pkgtype == 'tar':
parse_spec_file(pkgdir)

guess_description(pkgdir)
process_configure(pkgdir)

setup_licenses()
guess_license(pkgdir)

process_configure_ac(pkgdir)
process_qmake_pro(pkgdir)

write_bbfile()
clean_up()

Quoted 5 lines

_______________________________________________
poky mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/poky

_______________________________________________
poky mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/poky