Please help to review the Yocto 1656: Recipe creation/import script
Hi Josh,
The attachment is the the design document V2 and implement.
The V2 document changes are:
1 remove patch_set from argument list
2 deb is not support right now
Any comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.
Thanks& Regards,
Kai
Design document for Recipe creation/import script
* Purpose
The feature is from Yocto 1.2 Bug 1656. A script or similar system that would allow you to give is an upstream URL, tarball, patch set, package (SRPM, or debian style) and generate a recipe based on those instructions.
Name it with bitbake-createbb, and integrate into the build system to use the build system infrastructure that similiar with bitbake-runtask.
* Usage:
bitbake-createbb <SRC_URI>
* Design Flow
1 download the source package from the SRC_URI
2 get package name and version from package or tarball name.
3 unpack to /tmp/bitbake-createbb-pid
4 if tarball, get summary and description from freshmeat.org, README files and pkgconfig files;
otherwise get them from SRPM spec file or debian control file
5 get license, license file and checksum from LICENSE/COPYING files or freshmeat.org. 6 get dependent packages
|-- get them by parse configure files
|-- just parse spec file
8 remove downloaded files
#!/usr/bin/env python
import os
import sys
import shutil
import re
import urllib
import sha, md5, hashlib
tmpdir = "/tmp/bitbake_createbb.%s" % os.getpid()bin_file_name = ''
pkgname = ''
pkgversion = ''
summary = ''
description = ''
homepage = ''
license = []license_files = {}
licenses = {}
depends = []src_uri = ''src_uri_md5sum = ''src_uri_sha256sum = ''
inherits = []
# support 2 package types: tar and rpm pkgtype = ""
def usage():
print "This is the bitbake import bb file script. Usage:\n" print "\tbitbake-createbb <SRC_URI> \n"
def download(pkg_uri):
try:
shutil.rmtree(tmpdir)
except:
pass
cwd = os.getcwd()
os.mkdir(tmpdir)
os.chdir(tmpdir)
print 'Downloading package: ', pkg_uri, ' to ', tmpdir ret = os.system("wget --quiet " + pkg_uri) os.chdir(cwd)
return ret
def guess_name_by_uri(uri):
global bin_file_name, pkgname, pkgversion, pkgtype
elems = uri.rsplit('/', 1)
pkg = bin_file_name = elems[1]
if pkg.endswith("tgz"):
pkg = re.sub("tgz$", "tar.gz", pkg)
match = re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg) if match:
pkgtype = 'tar'
pkgname = match.group(1)
pkgversion = match.group(2)
match = re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg) if match:
pkgtype = 'rpm'
pkgname = match.group(1)
pkgversion = match.group(2)
def get_md5_sha256_sum():
global src_uri_md5sum, src_uri_sha256sum
curdir = os.getcwd()
os.chdir(tmpdir)
f = file(bin_file_name, 'r')
content = f.read()
f.close()
md_five = md5.new()
md_five.update(content)
src_uri_md5sum = md_five.hexdigest()
sha_256 = hashlib.sha256()
sha_256.update(content)
src_uri_sha256sum = sha_256.hexdigest()
os.chdir(curdir)def unpack_package():
global bin_file_name
cwd = os.getcwd()
os.chdir(tmpdir)
if pkgtype == 'rpm':
cmdline = "rpm2cpio " + bin_file_name + " | cpio -id" ret = os.system(cmdline)
if ret != 0:
return ret
for item in os.listdir('.'):
if re.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
bin_file_name = item
break
cmdline = "tar axf " + bin_file_name
ret = os.system(cmdline)
os.chdir(cwd)
return ret
# get the fall-back description when other way fail
# check homepage at same time
def guess_description_from_readme(readme):
global description, homepage
f = file(readme)
state = 0
desc = '' for line in f:
if state == 1 and re.match('^\n', line) and len(desc) > 80:
state = 2
if state == 0 and len(line) > 1:
state = 1
if state == 1:
desc += line
match = re.search('(http\:\/\/.*$name.*\.org)', line) if match:
url = match.group(1)
if re.search('bug') or len(homepage) > 1:
pass
else:
homepage = url
f.close()
if (len(desc) > 4 and len(description) < 3):
description = desc
def guess_description_from_freecode(pkgname):
global description
desc = ''
state = 0
html = urllib.urlopen("http://freecode.com/projects/" + pkgname)
for line in html:
if state == 1:
desc += line
if state == 0 and re.search('\<div class\=\"project-detail\"\>', line):
state = 1
if state == 1 and re.search('\<\/p\>', line):
state = 2
# deal the description
desc = re.sub('\<p\>', '', desc)
desc = re.sub('\<\/p\>', '', desc)
desc = re.sub('\r', '', desc)
desc = desc.strip()
if len(desc) > 10:
description = desc# get Summary from pkgconfig file
def guess_summary_from_pc(pc):
global summary fn = file(pc)
for line in fn:
match = re.match('Description:\s*(.*)', line) if match and len(summary) < 2:
summary = match.group(1)
break
summary = summary.strip()
fn.close()
def guess_description(pkgdir):
global pkgname, description, summary
readmes = []
pcs = []
for subdir in os.listdir(pkgdir):
if re.match('^README$', subdir):
readmes.insert(0, os.path.join(pkgdir, subdir))
elif re.match('README.*', subdir):
readmes.append(os.path.join(pkgdir, subdir))
elif re.match('.*\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elif re.match(pkgname + '\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elif re.match('.*\.pc', subdir):
pcs.append(os.path.join(pkgdir, subdir))
for readme in readmes:
guess_description_from_readme(os.path.join(pkgdir, readme))
if (len(pkgname) > 2):
guess_description_from_freecode(pkgname)
for pc in pcs:
guess_summary_from_pc(pc)
# if didn't get summary, use first line of description if len(summary) < 2:
summary = description
summary = re.sub("\n", " ", summary) summary = re.sub("\s+", " ", summary) match = re.match("(.*?)\.", summary) summary = match.group(1)
# the sha1sum values are from autospectacle
def setup_licenses():
licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] = "GPLv2"
licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2"
licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2"
licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2"
licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1"
licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2"
licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2"
licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2"
licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2"
licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1"
licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2"
licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1"
licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3"
licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT"
licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD"
licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1"
licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2"
licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1"
licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2"
licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1"
licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"def guess_licenses_from_file(copying, relname):
global licenses, license
sha1 = sha.new()
fn = open(copying)
content = fn.read()
fn.close()
sha1.update(content)
digest = sha1.hexdigest()
if digest in licenses:
license.append(licenses[digest]) md_five = md5.new() md_five.update(content) license_files[relname] = md_five.hexdigest()
def guess_licenses_from_freecode():
global license
lic = ''
state = 0
html = urllib.urlopen("http://freecode.com/projects/" + pkgname) for line in html:
if state == 1:
lic += line
if state == 0 and re.search('<th><span>Licenses</span></th>', line):
state = 1
if state == 1 and re.search('</a>', line):
state = 2
# deal the license
lic = lic.strip()
match = re.search('<a.*?>(.*?)</a>', lic)
if match:
license.append(match.group(1))
def guess_license(pkgdir):
for item in os.listdir(pkgdir):
realpath = os.path.join(pkgdir, item)
if os.path.isfile(realpath) and (re.match('COPY.*', item) \
or re.match('LICENSE.*', item) or re.match('GPL.*', item)):
guess_licenses_from_file(realpath, item)
if len(license) == 0:
guess_licenses_from_freecode()def process_configure(pkgdir):
if os.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
os.system("cd " + pkgdir + " ; ./autogen.sh &> /dev/null");
configure = os.path.join(pkgdir, 'configure')
if os.path.isfile(configure):
if 'autotools' not in inherits:
inherits.append('autotools')
fn = open(configure)
for line in fn:
match = re.match('PACKAGE_NAME=\'(.*?)\'', line) if match and len(pkgname) == 0:
pkgname = match.group(1)
match = re.match('PACKAGE_TARNAME=\'(.*?)\'', line) if match:
pkgname = match.group(1)
match = re.match('PACKAGE_VERSION=\'(.*?)\'', line) if match:
pkgversion = match.group(1)
match = re.match('PACKAGE_URL=\'(.*?)\'', line) if match:
homepage = match.group(1)
fn.close()
def push_buildreq(dep):
global depends
# remove collateral ] ) etc damage in the string
dep = re.sub("\"", "", dep)
dep = re.sub("\)", "", dep)
dep = re.sub("\]", "", dep)
dep = re.sub("\[", "", dep)
# first, undo the space packing
dep = re.sub(">=", " >= ", dep)
dep = re.sub("<=", " <= ", dep)
items = dep.split(' ')
dep = items[0]
# don't show configure variables, we can't deal with them
if re.search("\$", dep):
return
if re.search("AC_SUBST", dep):
return
if dep not in depends:
depends.append(dep)def parse_configure_ac(ac_file):
depth = 0
clause = ""
fac = file(ac_file)
for line in fac:
line = line.strip()
i = 0
while i < len(line):
if line[i] == '(':
depth += 1
if line[i] == ')' and depth > 0:
depth -= 1
clause += line[i]
i += 1
if depth > 0:
continue
# remove '\n'
clause = re.sub('\n', '', clause)
clause = clause.strip()
match = re.match('PKG_CHECK_MODULES\((.*)\)', clause)
if match:
modules = match.group(1)
pkg = modules.split(',', 1)[1].strip()
match2 = re.match('\[(.*)\]', pkg)
if match2:
pkg = match2.group(1)
# split the build dependencies
pkg = pkg.replace('\s+', ' ')
pkg = re.sub('\s>\s', '>', pkg)
pkg = re.sub('\s>=\s', '>=', pkg)
pkg = re.sub('\s=\s', '=', pkg)
pkg = re.sub('\s<=\s', '<=', pkg)
pkg = re.sub('\s<\s', '<', pkg)
pkglist = pkg.split(' ')
for dep in pkglist:
push_buildreq(dep)
match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause)
if match:
exists = match.group(1)
pkg = exists.split(',', 1)[0].strip()
match2 = re.match('\[(.*)\]', pkg)
if match2:
pkg = match2.group(1)
pkg = pkg.strip()
pkg = re.sub('\s+', ' ', pkg)
pkg = re.sub('\s>\s', '>', pkg)
pkg = re.sub('\s>=\s', '>=', pkg)
pkg = re.sub('\s=\s', '=', pkg)
pkg = re.sub('\s<=\s', '<=', pkg)
pkg = re.sub('\s<\s', '<', pkg)
pkglist = pkg.split(' ')
for dep in pkglist:
push_buildreq(dep)
# these items are from autospectacle.pl
if re.search('_PROG_INTLTOOL', clause):
push_buildreq("intltool")
if re.search('GETTEXT_PACKAGE', clause):
push_buildreq('gettext')
if re.search('GTK_DOC_CHECK', clause):
push_buildreq('gtk-doc')
if re.search('GNOME_DOC_INIT', clause):
push_buildreq('gnome-doc-utils')
if re.search('AM_GLIB_GNU_GETTEXT', clause):
push_buildreq('gettext')
match = re.search('AC_INIT\((.*)\)', clause)
if match:
ac_init = match.group(1)
ac_init = ac_init.strip()
ac_init = re.sub('\s+', ' ', ac_init)
version = ac_init.split(',')[1].strip()
if re.match('\d+(\.\d+)*', version):
pkgversion = version
match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause)
if match:
am_init = match.group(1)
am_init = re.sub('\s+', ' ', am_init)
items = am_init.split(',')
if len(items) >= 2:
ver = items[1]
ver = re.sub('\[', '', ver)
ver = re.sub('\]', '', ver)
if re.match('\d(\.\d+)*', ver):
pkgversion = ver
clause = ''
fac.closedef process_configure_ac(pkgdir):
configure_acs = []
for root, dirnames, filenames in os.walk(pkgdir):
for f in filenames:
if re.match('.*\.ac', f):
configure_acs.append(os.path.join(root, f)) if 'autotools' not in inherits:
inherits.append('autotools')
for ac in configure_acs:
parse_configure_ac(ac)
# check the *.pro files and get build requires from 'PKGCONFIG'
def process_qmake_pro(pkgdir):
global pkgname pro_files = []
depth = 0
clause = ''
pre_char = ''
if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and 'qmake2' not in inherits:
inherits.append('qmake2')
for root, dirnames, filenames in os.walk(pkgdir):
for f in filenames:
if re.match('.*\.pro', f):
pro_files.append(os.path.join(pkgdir, f))
for pro in pro_files:
need_next_line = False
f = file(pro)
for line in f:
if line.startswith('#'):
continue
line = re.sub('\n', '', line)
clause += line
if line.endswith('\\'):
continue
clause = clause.strip()
match = re.match('PKGCONFIG\s*=(.*)', clause)
if match:
dep = match.group(1)
push_buildreq(dep)
match = re.match('PKGCONFIG\s*\+=(.*)', clause)
if match:
dep = match.group(1)
dep = dep.strip()
dep = re.sub('\\\\', '', dep)
dep = re.sub('\s+', ' ', dep)
items = dep.split(' ')
for item in items:
push_buildreq(item)
clause = ''
f.close()def parse_spec_file(tmpdir):
# find the spec file first
spec_file = ''
for item in os.listdir(tmpdir):
if re.match('.*\.spec', item):
spec_file = os.path.join(tmpdir, item)
break if len(spec_file) == 0:
return 1
global pkgname, pkgversion, summary, description, depends
global src_uri, homepage
f = file(spec_file)
for line in f:
match = re.match('Summary\s*:(.*)', line)
if match:
if len(summary) == 0:
summary = match.group(1).strip()
continue
match = re.match('Name\s*:(.*)', line)
if match:
pkgname = match.group(1).strip()
continue
match = re.match('Version\s*:(.*)', line)
if match:
version = match.group(1).strip()
if re.match('\d+(\.\d+)?', version):
pkgversion = version
continue
match = re.match('License\s*:(.*)', line)
if match:
license.append(match.group(1).strip())
continue
match = re.match('URL(?i)\s*:(.*)', line)
if match:
homepage = match.group(1).strip()
continue
match = re.match('Source0*\s*:(.*)', line)
if match:
src_uri = match.group(1).strip()
src_uri = re.sub('\%\{name\}', pkgname, src_uri)
src_uri = re.sub('\%\{version\}', pkgversion, src_uri)
continue
match = re.match('BuildRequires\s*:(.*)', line)
if match:
deps = match.group(1).strip()
dpes = re.sub(',', ' ', deps)
deps = re.sub('\s+', ' ', deps)
for item in deps.split(' '):
push_buildreq(item)
match = re.match('%description\s*$', line)
if match:
for l in f:
if l.startswith('%'):
break
description += l
description = re.sub('\n', '', description)
f.close()def write_bbfile():
if len(pkgname) == 0 or len(pkgversion) == 0:
fail_quit("Can't get package name or version.")
content = ''
content += 'DESCRIPTION = "' + description + '"\n'
content += 'SUMMARY = "' + summary + '"\n'
if len(homepage) > 0:
content += 'HOMEPAGE = "' + homepage + '"\n'
content += 'LICENSE = "'
for lic in license:
content += lic + ' '
content += '"\n'
content += 'LIC_FILES_CHKSUM = "'
for key in license_files:
content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n';
content += '\t\t"\n\n';
if len(depends) > 0:
content += 'DEPENDS = "'
for dep in depends:
content += dep + ' '
content += ' "\n'
content += 'PR = "r0"\n\n'
content += 'SRC_URI = "' + src_uri + '"\n'
content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n'
content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n'
if len(inherits) > 0:
content += '\ninherit '
for inherit in inherits:
content += inherit + ' '
content += '\n'
bb_filename = pkgname + '_' + pkgversion + '.bb'
bb_file = open(bb_filename, 'w')
bb_file.write(content)
bb_file.close()
print 'Create bb file : ', bb_filenamedef fail_quit(msg):
print msg
clean_up()
exit(1)def clean_up():
shutil.rmtree(tmpdir)
pass
##########################################################
if __name__ == '__main__':
if (len(sys.argv) < 2):
usage()
exit(1)
src_uri = sys.argv[1]
guess_name_by_uri(src_uri)
if pkgname is None:
print "Can't get the package name."
exit(1)
ret = download(src_uri)
if ret != 0:
fail_quit('Download package failed. Make sure the SRC_URI is valid.')
if unpack_package() != 0:
fail_quit('Unpack package failed.')
if pkgtype == 'rpm':
parse_spec_file(tmpdir)
get_md5_sha256_sum()
pkgdir = ''
for subdir in os.listdir(tmpdir):
subdir = os.path.join(tmpdir, subdir)
if os.path.isdir(subdir):
pkgdir = subdir
break
if pkgtype == 'tar':
parse_spec_file(pkgdir)
guess_description(pkgdir)
process_configure(pkgdir)
setup_licenses()
guess_license(pkgdir)
process_configure_ac(pkgdir)
process_qmake_pro(pkgdir)
write_bbfile()
clean_up()Hi Kai,
Quoted 8 lines · On Thursday 19 January 2012 17:33:47 Kang Kai wrote:
The attachment is the the design document V2 and implement.
The V2 document changes are:
1 removepatch_setfrom argument list
2 deb is not support right nowAny comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.
Great work! I can see this being really useful to help people get started with new recipes. A few comments:
1) I tried it on hdparm and it worked very well. However, obexftp caused an error:
----------------
Downloading package:
http://downloads.sourceforge.net/openobex/obexftp-0.23.tar.bz2 to /tmp/bitbake_createbb.5729
Traceback (most recent call last):
File "./bitbake-createbb", line 643, in <module>
process_configure(pkgdir)
File "./bitbake-createbb", line 295, in process_configure
if match and len(pkgname) == 0:
UnboundLocalError: local variable 'pkgname' referenced before assignment
----------------
2) If it is able to find license file(s) but unable to find any appropriate for LIC_FILES_CHKSUM, it would be nice if it would print a warning just so the user knows they will need to set this themselves (bonus points if you still list LIC_FILES_CHKSUM entries for license files that did not match the known checksums either in a comment in the output or in the warning message, to save the user work if they know the files are appropriate). If you prefer I can send a patch afterwards for this.
3) If you have an RPM spec file it is possible to extract PACKAGES and FILES and possibly even EXTRA_OECONF. Somewhere around here I have a very basic script to do this I think, I could send it to you if you like (or we can leave this for later).
Cheers,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre
Quoted 24 lines · Kang Kai
Hi Josh,
The attachment is the the design document V2 and implement.
The V2 document changes are:
1 removepatch_setfrom argument list
2 deb is not support right nowAny comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.Thanks& Regards,
Kairecipe_creation_bb.txt
Design document for Recipe creation/import script
* Purpose
Quoted 7 lines
The feature is from Yocto 1.2 Bug 1656. A script or similar system that would allow you to give is an upstream URL, tarball, patch set, package (SRPM, or debian style) and generate a recipe based on those instructions.
Name it with bitbake-createbb, and integrate into the build system to use the build system infrastructure that similiar with bitbake-runtask.
I'd suggest changing the name to recipe-createbb or something. Since this isn't a "bitbake" program (or is it?)
Quoted 10 lines
* Usage:
bitbake-createbb<SRC_URI>* Design Flow
1 download the source package from theSRC_URI
2 get package name and version from package or tarball name.
3 unpack to/tmp/bitbake-createbb-pid
4 if tarball, get summary and description from freshmeat.org, README files and pkgconfig files;
otherwise get them from SRPM spec file or debian control file
5 get license, license file and checksum from LICENSE/COPYING files or freshmeat.org.
For license files, I would suggest we scan for common filenames:
LICENSE, COPYING, COPYING.LIB, and COPYRIGHT
If it doesn't match the known checksums, we tell the user and let them manually figure out the license type and verify things are consisten, but we should be able to fill out the license filename for them.
Otherwise, it all looks fine to me.
--Mark
Quoted 672 lines
6 get dependent packages
|-- get them by parse configure files
|-- just parse spec file
8 remove downloaded filesbitbake-createbb
#!/usr/bin/env python
import os
import sys
import shutil
import re
import urllib
import sha, md5, hashlibtmpdir = "/tmp/bitbake_createbb.%s" %
os.getpid()bin_file_name= ''
pkgname = ''
pkgversion = ''
summary = ''
description = ''
homepage = ''
license = []license_files= {}
licenses = {}
depends = []src_uri= ''src_uri_md5sum= ''src_uri_sha256sum= ''
inherits = []# support 2 package types: tar and rpm pkgtype = ""def
usage():
print "This is the bitbake import bb file script. Usage:\n" print "\tbitbake-createbb<SRC_URI> \n"def
download(pkg_uri):
try:
shutil.rmtree(tmpdir)
except:
passcwd =
os.getcwd()
os.mkdir(tmpdir)
os.chdir(tmpdir)
print 'Downloading package: ',pkg_uri, ' to ', tmpdir ret =os.system("wget --quiet " + pkg_uri)os.chdir(cwd)return ret
def
guess_name_by_uri(uri):
globalbin_file_name, pkgname, pkgversion, pkgtypeelems =
uri.rsplit('/', 1)
pkg =bin_file_name= elems[1]if
pkg.endswith("tgz"):
pkg =re.sub("tgz$", "tar.gz", pkg)match =
re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg)if match:
pkgtype = 'tar'
pkgname =match.group(1)
pkgversion =match.group(2)match =
re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg)if match:
pkgtype = 'rpm'
pkgname =match.group(1)
pkgversion =match.group(2)def
get_md5_sha256_sum():global src_uri_md5sum, src_uri_sha256sum curdir = os.getcwd() os.chdir(tmpdir) f = file(bin_file_name, 'r') content = f.read() f.close() md_five = md5.new() md_five.update(content) src_uri_md5sum = md_five.hexdigest() sha_256 = hashlib.sha256() sha_256.update(content) src_uri_sha256sum = sha_256.hexdigest() os.chdir(curdir)def
unpack_package():
globalbin_file_namecwd =
os.getcwd()
os.chdir(tmpdir)
if pkgtype == 'rpm':
cmdline = "rpm2cpio " +bin_file_name+ " | cpio -id" ret =os.system(cmdline)
if ret != 0:
return ret
for item inos.listdir('.'):
ifre.match(pkgname + '([0-9\.\-\~]*)\.tar', item):
bin_file_name= item
breakcmdline = "tar axf " +
bin_file_name
ret =os.system(cmdline)
os.chdir(cwd)
return ret# get the fall-back description when other way fail # check homepage at same time def guess_description_from_readme(readme): global description, homepage f = file(readme) state = 0 desc = ''for line in f:
if state == 1 andre.match('^\n', line)and len(desc)> 80:
state = 2
if state == 0 and len(line)> 1:
state = 1
if state == 1:
desc += linematch =
re.search('(http\:\/\/.*$name.*\.org)', line)if match:
url =match.group(1)
ifre.search('bug')or len(homepage)> 1:
pass
else:
homepage = url
f.close()
if (len(desc)> 4 and len(description)< 3):
description = descdef
guess_description_from_freecode(pkgname):global description desc = '' state = 0 html = urllib.urlopen("http://freecode.com/projects/" + pkgname) for line in html: if state == 1: desc += line if state == 0 and re.search('\<div class\=\"project-detail\"\>', line): state = 1 if state == 1 and re.search('\<\/p\>', line): state = 2 # deal the description desc = re.sub('\<p\>', '', desc) desc = re.sub('\<\/p\>', '', desc) desc = re.sub('\r', '', desc) desc = desc.strip() if len(desc)> 10: description = desc# get Summary from pkgconfig file def guess_summary_from_pc(pc): global summaryfn = file(pc)
for line in fn:
match =re.match('Description:\s*(.*)', line)if match and len(summary)< 2:
summary =match.group(1)
breaksummary =
summary.strip()
fn.close()def
guess_description(pkgdir):
global pkgname, description, summaryreadmes = []
pcs = []for subdir in
os.listdir(pkgdir):
ifre.match('^README$', subdir):
readmes.insert(0, os.path.join(pkgdir, subdir))
elifre.match('README.*', subdir):
readmes.append(os.path.join(pkgdir, subdir))
elifre.match('.*\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elifre.match(pkgname + '\.pc.*', subdir):
pcs.insert(0, os.path.join(pkgdir, subdir))
elifre.match('.*\.pc', subdir):
pcs.append(os.path.join(pkgdir, subdir))for readme in readmes:
guess_description_from_readme(os.path.join(pkgdir, readme))if (len(pkgname)> 2):
guess_description_from_freecode(pkgname)for pc in pcs:
guess_summary_from_pc(pc)# if didn't get summary, use first line of description if len(summary)< 2:
summary = description
summary =re.sub("\n", " ", summary)summary =re.sub("\s+", " ", summary)match =re.match("(.*?)\.", summary)summary =match.group(1)# the sha1sum values are from autospectacle def setup_licenses(): licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] = "GPLv2" licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2" licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2" licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2" licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1" licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2" licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2" licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2" licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2" licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1" licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2" licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1" licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3" licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT" licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD" licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1" licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2" licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1" licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2" licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1" licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"def
guess_licenses_from_file(copying, relname):
global licenses, licensesha1 =
sha.new()fn = open(copying)
content =fn.read()
fn.close()
sha1.update(content)
digest =sha1.hexdigest()if digest in licenses:
license.append(licenses[digest])md_five=md5.new()md_five.update(content)license_files[relname] =md_five.hexdigest()def
guess_licenses_from_freecode():
global licenselic = ''
state = 0html = urllib.urlopen("http://freecode.com/projects/" + pkgname) for line in html:
if state == 1:
lic += line
if state == 0 andre.search('<th><span>Licenses</span></th>', line):
state = 1
if state == 1 andre.search('</a>', line):
state = 2# deal the license
lic =lic.strip()
match =re.search('<a.*?>(.*?)</a>', lic)
if match:
license.append(match.group(1))def
guess_license(pkgdir):for item in os.listdir(pkgdir): realpath = os.path.join(pkgdir, item) if os.path.isfile(realpath) and (re.match('COPY.*', item) \ or re.match('LICENSE.*', item) or re.match('GPL.*', item)): guess_licenses_from_file(realpath, item) if len(license) == 0: guess_licenses_from_freecode()def
process_configure(pkgdir):
ifos.path.isfile(os.path.join(pkgdir, 'autogen.sh')):
os.system("cd " + pkgdir + " ; ./autogen.sh&> /dev/null");configure =
os.path.join(pkgdir, 'configure')ifos.path.isfile(configure):
if 'autotools' not in inherits:
inherits.append('autotools')fn = open(configure)
for line in fn:
match =re.match('PACKAGE_NAME=\'(.*?)\'', line)if match and len(pkgname) == 0:
pkgname =match.group(1)match =
re.match('PACKAGE_TARNAME=\'(.*?)\'', line)if match:
pkgname =match.group(1)match =
re.match('PACKAGE_VERSION=\'(.*?)\'', line)if match:
pkgversion =match.group(1)match =
re.match('PACKAGE_URL=\'(.*?)\'', line)if match:
homepage =match.group(1)
fn.close()def
push_buildreq(dep):global depends # remove collateral ] ) etc damage in the string dep = re.sub("\"", "", dep) dep = re.sub("\)", "", dep) dep = re.sub("\]", "", dep) dep = re.sub("\[", "", dep) # first, undo the space packing dep = re.sub(">=", ">= ", dep) dep = re.sub("<=", "<= ", dep) items = dep.split(' ') dep = items[0] # don't show configure variables, we can't deal with them if re.search("\$", dep): return if re.search("AC_SUBST", dep): return if dep not in depends: depends.append(dep)def
parse_configure_ac(ac_file):depth = 0 clause = "" fac = file(ac_file) for line in fac: line = line.strip() i = 0 while i< len(line): if line[i] == '(': depth += 1 if line[i] == ')' and depth> 0: depth -= 1 clause += line[i] i += 1 if depth> 0: continue # remove '\n' clause = re.sub('\n', '', clause) clause = clause.strip() match = re.match('PKG_CHECK_MODULES\((.*)\)', clause) if match: modules = match.group(1) pkg = modules.split(',', 1)[1].strip() match2 = re.match('\[(.*)\]', pkg) if match2: pkg = match2.group(1) # split the build dependencies pkg = pkg.replace('\s+', ' ') pkg = re.sub('\s>\s', '>', pkg) pkg = re.sub('\s>=\s', '>=', pkg) pkg = re.sub('\s=\s', '=', pkg) pkg = re.sub('\s<=\s', '<=', pkg) pkg = re.sub('\s<\s', '<', pkg) pkglist = pkg.split(' ') for dep in pkglist: push_buildreq(dep) match = re.match('PKG_CHECK_EXISTS\((.*)\)', clause) if match: exists = match.group(1) pkg = exists.split(',', 1)[0].strip() match2 = re.match('\[(.*)\]', pkg) if match2: pkg = match2.group(1) pkg = pkg.strip() pkg = re.sub('\s+', ' ', pkg) pkg = re.sub('\s>\s', '>', pkg) pkg = re.sub('\s>=\s', '>=', pkg) pkg = re.sub('\s=\s', '=', pkg) pkg = re.sub('\s<=\s', '<=', pkg) pkg = re.sub('\s<\s', '<', pkg) pkglist = pkg.split(' ') for dep in pkglist: push_buildreq(dep) # these items are from autospectacle.pl if re.search('_PROG_INTLTOOL', clause): push_buildreq("intltool") if re.search('GETTEXT_PACKAGE', clause): push_buildreq('gettext') if re.search('GTK_DOC_CHECK', clause): push_buildreq('gtk-doc') if re.search('GNOME_DOC_INIT', clause): push_buildreq('gnome-doc-utils') if re.search('AM_GLIB_GNU_GETTEXT', clause): push_buildreq('gettext') match = re.search('AC_INIT\((.*)\)', clause) if match: ac_init = match.group(1) ac_init = ac_init.strip() ac_init = re.sub('\s+', ' ', ac_init) version = ac_init.split(',')[1].strip() if re.match('\d+(\.\d+)*', version): pkgversion = version match = re.search('AM_INIT_AUTOMAKE\((.*)\)', clause) if match: am_init = match.group(1) am_init = re.sub('\s+', ' ', am_init) items = am_init.split(',') if len(items)>= 2: ver = items[1] ver = re.sub('\[', '', ver) ver = re.sub('\]', '', ver) if re.match('\d(\.\d+)*', ver): pkgversion = ver clause = '' fac.closedef
process_configure_ac(pkgdir):
configure_acs= []
for root, dirnames, filenames inos.walk(pkgdir):
for f in filenames:
ifre.match('.*\.ac', f):
configure_acs.append(os.path.join(root, f))if 'autotools' not in inherits:
inherits.append('autotools')for ac in
configure_acs:
parse_configure_ac(ac)# check the *.pro files and get build requires from 'PKGCONFIG' def process_qmake_pro(pkgdir): global pkgnamepro_files = [] depth = 0 clause = '' pre_char = '' if os.path.isfile(os.path.join(pkgdir, pkgname + '.pro')) and 'qmake2' not in inherits: inherits.append('qmake2') for root, dirnames, filenames in os.walk(pkgdir): for f in filenames: if re.match('.*\.pro', f): pro_files.append(os.path.join(pkgdir, f)) for pro in pro_files: need_next_line = False f = file(pro) for line in f: if line.startswith('#'): continue line = re.sub('\n', '', line) clause += line if line.endswith('\\'): continue clause = clause.strip() match = re.match('PKGCONFIG\s*=(.*)', clause) if match: dep = match.group(1) push_buildreq(dep) match = re.match('PKGCONFIG\s*\+=(.*)', clause) if match: dep = match.group(1) dep = dep.strip() dep = re.sub('\\\\', '', dep) dep = re.sub('\s+', ' ', dep) items = dep.split(' ') for item in items: push_buildreq(item) clause = '' f.close()def
parse_spec_file(tmpdir):# find the spec file first spec_file = '' for item in os.listdir(tmpdir): if re.match('.*\.spec', item): spec_file = os.path.join(tmpdir, item) breakif len(spec_file) == 0: return 1 global pkgname, pkgversion, summary, description, depends global src_uri, homepage f = file(spec_file) for line in f: match = re.match('Summary\s*:(.*)', line) if match: if len(summary) == 0: summary = match.group(1).strip() continue match = re.match('Name\s*:(.*)', line) if match: pkgname = match.group(1).strip() continue match = re.match('Version\s*:(.*)', line) if match: version = match.group(1).strip() if re.match('\d+(\.\d+)?', version): pkgversion = version continue match = re.match('License\s*:(.*)', line) if match: license.append(match.group(1).strip()) continue match = re.match('URL(?i)\s*:(.*)', line) if match: homepage = match.group(1).strip() continue match = re.match('Source0*\s*:(.*)', line) if match: src_uri = match.group(1).strip() src_uri = re.sub('\%\{name\}', pkgname, src_uri) src_uri = re.sub('\%\{version\}', pkgversion, src_uri) continue match = re.match('BuildRequires\s*:(.*)', line) if match: deps = match.group(1).strip() dpes = re.sub(',', ' ', deps) deps = re.sub('\s+', ' ', deps) for item in deps.split(' '): push_buildreq(item) match = re.match('%description\s*$', line) if match: for l in f: if l.startswith('%'): break description += l description = re.sub('\n', '', description) f.close()def
write_bbfile():if len(pkgname) == 0 or len(pkgversion) == 0: fail_quit("Can't get package name or version.") content = '' content += 'DESCRIPTION = "' + description + '"\n' content += 'SUMMARY = "' + summary + '"\n' if len(homepage)> 0: content += 'HOMEPAGE = "' + homepage + '"\n' content += 'LICENSE = "' for lic in license: content += lic + ' ' content += '"\n' content += 'LIC_FILES_CHKSUM = "' for key in license_files: content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n'; content += '\t\t"\n\n'; if len(depends)> 0: content += 'DEPENDS = "' for dep in depends: content += dep + ' ' content += ' "\n' content += 'PR = "r0"\n\n' content += 'SRC_URI = "' + src_uri + '"\n' content += 'SRC_URI[md5sum] = "' + src_uri_md5sum + '"\n' content += 'SRC_URI[sha256sum] = "' + src_uri_sha256sum + '"\n' if len(inherits)> 0: content += '\ninherit ' for inherit in inherits: content += inherit + ' ' content += '\n' bb_filename = pkgname + '_' + pkgversion + '.bb' bb_file = open(bb_filename, 'w') bb_file.write(content) bb_file.close() print 'Create bb file : ', bb_filenamedef
fail_quit(msg):print msg clean_up() exit(1)def
clean_up():
shutil.rmtree(tmpdir)
pass##########################################################
if
__name__== '__main__':if (len(sys.argv)< 2): usage() exit(1) src_uri = sys.argv[1] guess_name_by_uri(src_uri) if pkgname is None: print "Can't get the package name." exit(1) ret = download(src_uri) if ret != 0: fail_quit('Download package failed. Make sure the SRC_URI is valid.') if unpack_package() != 0: fail_quit('Unpack package failed.') if pkgtype == 'rpm': parse_spec_file(tmpdir) get_md5_sha256_sum() pkgdir = '' for subdir in os.listdir(tmpdir): subdir = os.path.join(tmpdir, subdir) if os.path.isdir(subdir): pkgdir = subdir break if pkgtype == 'tar': parse_spec_file(pkgdir) guess_description(pkgdir) process_configure(pkgdir) setup_licenses() guess_license(pkgdir) process_configure_ac(pkgdir) process_qmake_pro(pkgdir) write_bbfile() clean_up()_______________________________________________
poky mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/poky
Quoted 10 lines · Kang Kai
Hi Josh,
The attachment is the the design document V2 and implement.
The V2 document changes are:
1 removepatch_setfrom argument list
2 deb is not support right nowAny comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.
This looks like a good start, Paul has offered some great feedback. In addition I'd add that we need a license header on the script, we should probably attribute autospectacle in the header too (Inspired by? Based on?).
I haven't had chance to do a thorough code review but will try and do so whilst you're on your break.
Cheers,
Joshua
Signature · 3 lines
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology CentreQuoted 25 lines · Mark Hatle
Quoted 24 lines · Kang Kai
Hi Josh,
The attachment is the the design document V2 and implement.
The V2 document changes are:
1 removepatch_setfrom argument list
2 deb is not support right nowAny comment is welcome.
And I will on vacation until Jan 30, so my reply may not in time.Thanks& Regards,
Kairecipe_creation_bb.txt
Design document for Recipe creation/import script
* Purpose
Quoted 8 lines
Quoted 7 lines
The feature is from Yocto 1.2 Bug 1656. A script or similar system that would allow you to give is an upstream URL, tarball, patch set, package (SRPM, or debian style) and generate a recipe based on those instructions.
Name it with bitbake-createbb, and integrate into the build system to use the build system infrastructure that similiar with bitbake-runtask.
Hi Mark,
Thanks for your comments.
Quoted 1 line
I'd suggest changing the name to recipe-createbb or something. Since this isn't a "bitbake" program (or is it?)
How about name it with "create-recipe"?
Quoted 12 lines
Quoted 11 lines
* Usage:
bitbake-createbb<SRC_URI>* Design Flow
1 download the source package from theSRC_URI
2 get package name and version from package or tarball name.
3 unpack to/tmp/bitbake-createbb-pid
4 if tarball, get summary and description from freshmeat.org, README files and pkgconfig files; otherwise get them from SRPM spec file or debian control file
5 get license, license file and checksum from LICENSE/COPYING files or freshmeat.org.
Quoted 5 lines
For license files, I would suggest we scan for common filenames:
LICENSE, COPYING, COPYING.LIB, and COPYRIGHT
If it doesn't match the known checksums, we tell the user and let them manually figure out the license type and verify things are consisten, but we should be able to fill out the license filename for them.
Ok, if can't get license from known checksums I'll left messages for user to check manually.
Regards,
Kai
Quoted 682 lines
Otherwise, it all looks fine to me.
--Mark
Quoted 667 lines
6 get dependent packages
|-- get them by parse configure files
|-- just parse spec file
8 remove downloaded filesbitbake-createbb
#!/usr/bin/env python
import os
import sys
import shutil
import re
import urllib
import sha, md5, hashlibtmpdir = "/tmp/bitbake_createbb.%s" %
os.getpid()bin_file_name= ''
pkgname = ''
pkgversion = ''
summary = ''
description = ''
homepage = ''
license = []license_files= {}
licenses = {}
depends = []src_uri= ''src_uri_md5sum= ''src_uri_sha256sum= ''
inherits = []# support 2 package types: tar and rpm pkgtype = ""def
usage():
print "This is the bitbake import bb file script. Usage:\n" print "\tbitbake-createbb<SRC_URI> \n"def
download(pkg_uri):
try:shutil.rmtree(tmpdir)
except:
passcwd =
os.getcwd()os.mkdir(tmpdir)os.chdir(tmpdir)
print 'Downloading package: ',pkg_uri, ' to ', tmpdir ret =os.system("wget --quiet " + pkg_uri)os.chdir(cwd)return ret
def
guess_name_by_uri(uri):
globalbin_file_name, pkgname, pkgversion, pkgtypeelems =
uri.rsplit('/', 1)
pkg =bin_file_name= elems[1]if
pkg.endswith("tgz"):
pkg =re.sub("tgz$", "tar.gz", pkg)match =
re.match('(.*?)\-([0-9\.\-\~]+)\.tar', pkg)if match:
pkgtype = 'tar'
pkgname =match.group(1)
pkgversion =match.group(2)match =
re.match('(.*?)\-([0-9\.\~]+).*?\.src.rpm', pkg)if match:
pkgtype = 'rpm'
pkgname =match.group(1)
pkgversion =match.group(2)def
get_md5_sha256_sum():
globalsrc_uri_md5sum,src_uri_sha256sumcurdir =
os.getcwd()os.chdir(tmpdir)f =
file(bin_file_name, 'r')
content =f.read()f.close()
md_five=md5.new()md_five.update(content)src_uri_md5sum=md_five.hexdigest()
sha_256=hashlib.sha256()sha_256.update(content)src_uri_sha256sum=sha_256.hexdigest()
os.chdir(curdir)def
unpack_package():
globalbin_file_namecwd =
os.getcwd()os.chdir(tmpdir)
if pkgtype == 'rpm':
cmdline = "rpm2cpio " +bin_file_name+ " | cpio -id" ret =os.system(cmdline)
if ret != 0:
return ret
for item inos.listdir('.'):
ifre.match(pkgname + '([0-9\.\-\~]*)\.tar', item):bin_file_name= item
breakcmdline = "tar axf " +
bin_file_name
ret =os.system(cmdline)
os.chdir(cwd)
return ret# get the fall-back description when other way fail # check homepage at same time def guess_description_from_readme(readme): global description, homepage f = file(readme) state = 0 desc = ''for line in f:
if state == 1 andre.match('^\n', line)and len(desc)> 80:
state = 2
if state == 0 and len(line)> 1:
state = 1
if state == 1:
desc += linematch =
re.search('(http\:\/\/.*$name.*\.org)', line)if match:
url =match.group(1)
ifre.search('bug')or len(homepage)> 1:
pass
else:
homepage = url
f.close()
if (len(desc)> 4 and len(description)< 3):
description = descdef
guess_description_from_freecode(pkgname):
global description
desc = ''
state = 0html = urllib.urlopen("http://freecode.com/projects/" + pkgname) for line in html:
if state == 1:
desc += line
if state == 0 andre.search('\<div class\=\"project-detail\"\>', line):
state = 1
if state == 1 andre.search('\<\/p\>', line):
state = 2# deal the description desc = re.sub('\<p\>', '', desc) desc = re.sub('\<\/p\>', '', desc) desc = re.sub('\r', '', desc) desc = desc.strip() if len(desc)> 10: description = desc# get Summary from pkgconfig file def guess_summary_from_pc(pc): global summaryfn = file(pc)
for line in fn:
match =re.match('Description:\s*(.*)', line)
if match and len(summary)< 2:
summary =match.group(1)
breaksummary =
summary.strip()fn.close()def
guess_description(pkgdir):
global pkgname, description, summaryreadmes = []
pcs = []for subdir in
os.listdir(pkgdir):
ifre.match('^README$', subdir):readmes.insert(0, os.path.join(pkgdir, subdir))
elifre.match('README.*', subdir):readmes.append(os.path.join(pkgdir, subdir))
elifre.match('.*\.pc.*', subdir):pcs.insert(0, os.path.join(pkgdir, subdir))
elifre.match(pkgname + '\.pc.*', subdir):pcs.insert(0, os.path.join(pkgdir, subdir))
elifre.match('.*\.pc', subdir):pcs.append(os.path.join(pkgdir, subdir))for readme in readmes:
guess_description_from_readme(os.path.join(pkgdir, readme))if (len(pkgname)> 2):
guess_description_from_freecode(pkgname)for pc in pcs:
guess_summary_from_pc(pc)# if didn't get summary, use first line of description if len(summary)< 2: summary = description summary = re.sub("\n", " ", summary) summary = re.sub("\s+", " ", summary) match = re.match("(.*?)\.", summary) summary = match.group(1)# the sha1sum values are from autospectacle def setup_licenses(): licenses['06877624ea5c77efe3b7e39b0f909eda6e25a4ec'] = "GPLv2" licenses["075d599585584bb0e4b526f5c40cb6b17e0da35a"] = "GPLv2" licenses["10782dd732f42f49918c839e8a5e2894c508b079"] = "GPLv2" licenses["2d29c273fda30310211bbf6a24127d589be09b6c"] = "GPLv2" licenses["4df5d4b947cf4e63e675729dd3f168ba844483c7"] = "LGPLv2.1" licenses["503df7650052cf38efde55e85f0fe363e59b9739"] = "GPLv2" licenses["5405311284eab5ab51113f87c9bfac435c695bb9"] = "GPLv2" licenses["5fb362ef1680e635fe5fb212b55eef4db9ead48f"] = "LGPLv2" licenses["68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1"] = "GPLv2" licenses["66c77efd1cf9c70d4f982ea59487b2eeb6338e26"] = "LGPLv2.1" licenses["74a8a6531a42e124df07ab5599aad63870fa0bd4"] = "GPLv2" licenses["8088b44375ef05202c0fca4e9e82d47591563609"] = "LGPLv2.1" licenses["8624bcdae55baeef00cd11d5dfcfa60f68710a02"] = "GPLv3" licenses["8e57ffebd0ed4417edc22e3f404ea3664d7fed27"] = "MIT" licenses["99b5245b4714b9b89e7584bfc88da64e2d315b81"] = "BSD" licenses["aba8d76d0af67d57da3c3c321caa59f3d242386b"] = "MPLv1.1" licenses["bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f"] = "LGPLv2" licenses["caeb68c46fa36651acf592771d09de7937926bb3"] = "LGPLv2.1" licenses["dfac199a7539a404407098a2541b9482279f690d"] = "GPLv2" licenses["e60c2e780886f95df9c9ee36992b8edabec00bcc"] = "LGPLv2.1" licenses["c931aad3017d975b7f20666cde0953234a9efde3"] = "GPLv2"def
guess_licenses_from_file(copying, relname):
global licenses, licensesha1 =
sha.new()fn = open(copying)
content =fn.read()fn.close()
sha1.update(content)
digest =sha1.hexdigest()if digest in licenses:
license.append(licenses[digest])md_five=md5.new()md_five.update(content)
license_files[relname] =md_five.hexdigest()def
guess_licenses_from_freecode():
global licenselic = ''
state = 0html = urllib.urlopen("http://freecode.com/projects/" + pkgname) for line in html:
if state == 1:
lic += line
if state == 0 andre.search('<th><span>Licenses</span></th>', line):
state = 1
if state == 1 andre.search('</a>', line):
state = 2# deal the license lic = lic.strip() match = re.search('<a.*?>(.*?)</a>', lic) if match: license.append(match.group(1))def
guess_license(pkgdir):
for item inos.listdir(pkgdir):
realpath =os.path.join(pkgdir, item)
ifos.path.isfile(realpath)and (re.match('COPY.*', item)\
orre.match('LICENSE.*', item)orre.match('GPL.*', item)):guess_licenses_from_file(realpath, item)if len(license) == 0:
guess_licenses_from_freecode()def
process_configure(pkgdir):
ifos.path.isfile(os.path.join(pkgdir, 'autogen.sh')):os.system("cd " + pkgdir + " ; ./autogen.sh&> /dev/null");configure =
os.path.join(pkgdir, 'configure')
ifos.path.isfile(configure):
if 'autotools' not in inherits:inherits.append('autotools')fn = open(configure)
for line in fn:
match =re.match('PACKAGE_NAME=\'(.*?)\'', line)
if match and len(pkgname) == 0:
pkgname =match.group(1)match =
re.match('PACKAGE_TARNAME=\'(.*?)\'', line)if match:
pkgname =match.group(1)match =
re.match('PACKAGE_VERSION=\'(.*?)\'', line)if match:
pkgversion =match.group(1)match =
re.match('PACKAGE_URL=\'(.*?)\'', line)
if match:
homepage =match.group(1)
fn.close()def
push_buildreq(dep):
global depends# remove collateral ] ) etc damage in the string dep = re.sub("\"", "", dep) dep = re.sub("\)", "", dep) dep = re.sub("\]", "", dep) dep = re.sub("\[", "", dep)# first, undo the space packing
dep =re.sub(">=", ">= ", dep)
dep =re.sub("<=", "<= ", dep)items =
dep.split(' ')
dep = items[0]# don't show configure variables, we can't deal with them if
re.search("\$", dep):
return
ifre.search("AC_SUBST", dep):
returnif dep not in depends:
depends.append(dep)def
parse_configure_ac(ac_file):
depth = 0
clause = ""fac =
file(ac_file)for line in fac:
line =line.strip()
i = 0
while i< len(line):
if line[i] == '(':
depth += 1
if line[i] == ')' and depth> 0:
depth -= 1
clause += line[i]
i += 1
if depth> 0:
continue# remove '\n' clause = re.sub('\n', '', clause) clause = clause.strip()match =
re.match('PKG_CHECK_MODULES\((.*)\)', clause)if match:
modules =match.group(1)
pkg =modules.split(',', 1)[1].strip()
match2 =re.match('\[(.*)\]', pkg)
if match2:
pkg =match2.group(1)# split the build dependencies pkg = pkg.replace('\s+', ' ') pkg = re.sub('\s>\s', '>', pkg) pkg = re.sub('\s>=\s', '>=', pkg) pkg = re.sub('\s=\s', '=', pkg) pkg = re.sub('\s<=\s', '<=', pkg) pkg = re.sub('\s<\s', '<', pkg) pkglist = pkg.split(' ') for dep in pkglist: push_buildreq(dep)match =
re.match('PKG_CHECK_EXISTS\((.*)\)', clause)if match:
exists =match.group(1)
pkg =exists.split(',', 1)[0].strip()
match2 =re.match('\[(.*)\]', pkg)
if match2:
pkg =match2.group(1)
pkg =pkg.strip()pkg =
re.sub('\s+', ' ', pkg)
pkg =re.sub('\s>\s', '>', pkg)
pkg =re.sub('\s>=\s', '>=', pkg)
pkg =re.sub('\s=\s', '=', pkg)
pkg =re.sub('\s<=\s', '<=', pkg)
pkg =re.sub('\s<\s', '<', pkg)
pkglist =pkg.split(' ')
for dep in pkglist:push_buildreq(dep)# these items are from autospectacle.pl if re.search('_PROG_INTLTOOL', clause): push_buildreq("intltool") if re.search('GETTEXT_PACKAGE', clause): push_buildreq('gettext') if re.search('GTK_DOC_CHECK', clause): push_buildreq('gtk-doc') if re.search('GNOME_DOC_INIT', clause): push_buildreq('gnome-doc-utils') if re.search('AM_GLIB_GNU_GETTEXT', clause): push_buildreq('gettext')match =
re.search('AC_INIT\((.*)\)', clause)
if match:ac_init=match.group(1)ac_init=ac_init.strip()ac_init=re.sub('\s+', ' ', ac_init)
version = ac_init.split(',')[1].strip()
ifre.match('\d+(\.\d+)*', version):
pkgversion = versionmatch =
re.search('AM_INIT_AUTOMAKE\((.*)\)', clause)if match:am_init=match.group(1)am_init=re.sub('\s+', ' ', am_init)
items =am_init.split(',')
if len(items)>= 2:
ver = items[1]
ver =re.sub('\[', '', ver)
ver =re.sub('\]', '', ver)
ifre.match('\d(\.\d+)*', ver):
pkgversion = ver
clause = ''
fac.closedef
process_configure_ac(pkgdir):configure_acs= []
for root, dirnames, filenames inos.walk(pkgdir):
for f in filenames:
ifre.match('.*\.ac', f):configure_acs.append(os.path.join(root, f))
if 'autotools' not in inherits:inherits.append('autotools')for ac in
configure_acs:parse_configure_ac(ac)# check the *.pro files and get build requires from 'PKGCONFIG' def process_qmake_pro(pkgdir): global pkgname
pro_files= []
depth = 0
clause = ''pre_char= ''if
os.path.isfile(os.path.join(pkgdir, pkgname + '.pro'))and 'qmake2' not in inherits:inherits.append('qmake2')
for root, dirnames, filenames inos.walk(pkgdir):
for f in filenames:
ifre.match('.*\.pro', f):pro_files.append(os.path.join(pkgdir, f))for pro in
pro_files:need_next_line= False
f = file(pro)
for line in f:
ifline.startswith('#'):
continue
line =re.sub('\n', '', line)
clause += line
ifline.endswith('\\'):
continueclause =
clause.strip()
match =re.match('PKGCONFIG\s*=(.*)', clause)
if match:
dep =match.group(1)push_buildreq(dep)
match =re.match('PKGCONFIG\s*\+=(.*)', clause)
if match:
dep =match.group(1)
dep =dep.strip()
dep =re.sub('\\\\', '', dep)
dep =re.sub('\s+', ' ', dep)
items =dep.split(' ')
for item in items:push_buildreq(item)clause = ''
f.close()def
parse_spec_file(tmpdir):# find the spec file first spec_file = '' for item in os.listdir(tmpdir): if re.match('.*\.spec', item): spec_file = os.path.join(tmpdir, item) breakif
len(spec_file)== 0:
return 1global pkgname, pkgversion, summary, description, depends global
src_uri, homepagef =
file(spec_file)
for line in f:
match =re.match('Summary\s*:(.*)', line)
if match:
if len(summary) == 0:
summary = match.group(1).strip()
continue
match =re.match('Name\s*:(.*)', line)
if match:
pkgname = match.group(1).strip()
continue
match =re.match('Version\s*:(.*)', line)
if match:
version = match.group(1).strip()
ifre.match('\d+(\.\d+)?', version):
pkgversion = version
continue
match =re.match('License\s*:(.*)', line)
if match:
license.append(match.group(1).strip()) continue
match =re.match('URL(?i)\s*:(.*)', line)
if match:
homepage = match.group(1).strip()
continue
match =re.match('Source0*\s*:(.*)', line)
if match:src_uri= match.group(1).strip()src_uri=re.sub('\%\{name\}', pkgname, src_uri)src_uri=re.sub('\%\{version\}', pkgversion, src_uri)continue
match =re.match('BuildRequires\s*:(.*)', line)if match:
deps = match.group(1).strip()
dpes =re.sub(',', ' ', deps)
deps =re.sub('\s+', ' ', deps)
for item indeps.split(' '):push_buildreq(item)match =
re.match('%description\s*$', line)
if match:
for l in f:
ifl.startswith('%'):
break
description += l
description =re.sub('\n', '', description)
f.close()def
write_bbfile():
if len(pkgname) == 0 or len(pkgversion) == 0:fail_quit("Can't get package name or version.")content = ''
content += 'DESCRIPTION = "' + description + '"\n' content += 'SUMMARY = "' + summary + '"\n' if len(homepage)> 0:
content += 'HOMEPAGE = "' + homepage + '"\n'content += 'LICENSE = "'
for lic in license:
content += lic + ' '
content += '"\n'
content += 'LIC_FILES_CHKSUM= "'
for key inlicense_files:
content += 'file://' + key + ';md5=' + license_files[key] + ' \\\n'; content += '\t\t"\n\n';if len(depends)> 0:
content += 'DEPENDS = "'
for dep in depends:
content += dep + ' '
content += ' "\n'content += 'PR = "r0"\n\n'
content += '
SRC_URI= "' +src_uri+ '"\n'
content += 'SRC_URI[md5sum] = "' +src_uri_md5sum+ '"\n' content += 'SRC_URI[sha256sum] = "' +src_uri_sha256sum+ '"\n'if len(inherits)> 0:
content += '\ninherit '
for inherit in inherits:
content += inherit + ' '
content += '\n'
bb_filename= pkgname + '_' + pkgversion + '.bb'bb_file=open(bb_filename, 'w')bb_file.write(content)bb_file.close()
print 'Create bb file : ',bb_filenamedef
fail_quit(msg):
print msgclean_up()
exit(1)def
clean_up():shutil.rmtree(tmpdir)
pass##########################################################
if
__name__== '__main__':
if (len(sys.argv)< 2):usage()
exit(1)
src_uri= sys.argv[1]
guess_name_by_uri(src_uri)
if pkgname is None:
print "Can't get the package name."
exit(1)ret =
download(src_uri)
if ret != 0:
fail_quit('Download package failed. Make sure theSRC_URIis valid.')if
unpack_package()!= 0:fail_quit('Unpack package failed.')if pkgtype == 'rpm':
parse_spec_file(tmpdir)
get_md5_sha256_sum()pkgdir = ''
for subdir inos.listdir(tmpdir):
subdir =os.path.join(tmpdir, subdir)
ifos.path.isdir(subdir):
pkgdir = subdir
breakif pkgtype == 'tar':
parse_spec_file(pkgdir)
guess_description(pkgdir)process_configure(pkgdir)
setup_licenses()guess_license(pkgdir)
process_configure_ac(pkgdir)process_qmake_pro(pkgdir)
write_bbfile()clean_up()Quoted 5 lines
_______________________________________________
poky mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/poky_______________________________________________
poky mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/poky