Add checklicense task to get missing license text
cover Add checklicense task to get missing license text
From: Mei Lei <[email protected]>
Hi Saul,
For adding the information of missing license text into pkg-report system, I add a new task called checklicense in distrodata.bbclass. This check logic is writen by Beth in license.bbclass, avoid affecting the build time, I rewrite this task in distrodata.bbclass. And there are a lot of repeat code in distrodata.bbclass, create a new function to reduce the repeat code.
Thanks
Lei
Pull URL: git://git.pokylinux.org/poky-contrib.git
Branch: lmei3/licensecheck
Browse: http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=lmei3/licensecheck
Thanks,
Mei Lei <[email protected]>
---
Mei Lei (2):
distro_check.py: Add a new function to reduce the repeat code distrodata.bbclass: Add checklicense task
meta/classes/distrodata.bbclass | 88 ++++++++++++++++++++++++--------------- meta/lib/oe/distro_check.py | 15 +++++++ 2 files changed, 69 insertions(+), 34 deletions(-)
PATCH 1/2 distro_check.py: Add a new function to reduce the repeat code
From: Mei Lei <[email protected]>
A lot of repeat code in distrodata.bbclass when creating log file, we can define a function called create_log_file to do it.
--- meta/lib/oe/distro_check.py | 15 +++++++++++++++ 1 files changed, 15 insertions(+), 0 deletions(-)
meta/lib/oe/distro_check.py +15 -0
@@ -341,6 +341,21 @@ def compare_in_distro_packages_list(distro_check_dir, d): bb.note("Matching: %s" % matching_distros) return matching_distros +def create_log_file(d, logname): + #logpath = bb.data.getVar('LOG_DIR', e.data, 1) + logpath = bb.data.getVar('LOG_DIR', d, 1) + bb.utils.mkdirhier(logpath) + logfile = logpath +"/" + logname.split(".")[0] + ".%s.csv" % bb.data.getVar('DATETIME', d, 1) + if not os.path.exists(logfile): + slogfile = os.path.join(logpath, logname) + if os.path.exists(slogfile): + os.remove(slogfile) + os.system("touch %s" % logfile) + os.symlink(logfile, slogfile) + bb.data.setVar('LOG_FILE', logfile, d) + return logfile + + def save_distro_check_result(result, datetime, d): pn = bb.data.getVar('PN', d, True) logdir = bb.data.getVar('LOG_DIR', d, True)
Signature · 1 line
1.6.3.3
Re: [PATCH 1/2] distro_check.py: Add a new function to reduce the repeat code
Quoted 4 lines · Mei Lei
From: Mei Lei <[email protected]>
A lot of repeat code in distrodata.bbclass when creating log file, we can define a function called
create_log_fileto do it.
I like the intent of the patch, some comments on coding style below.
Quoted 11 lines
Signed-off-by Mei Lei <[email protected]>--- meta/lib/oe/distro_check.py | 15 +++++++++++++++ 1 files changed, 15 insertions(+), 0 deletions(-)meta/lib/oe/distro_check.py +0
-0
meta/lib/oe/distro_check.py def compare_in_distro_packages_list(distro_check_dir, d): · line 345
Quoted 5 lines
meta/lib/oe/distro_check.py +2
-0@@ -341,6 +341,21 @@ def compare_in_distro_packages_list(distro_check_dir, d): bb.note("Matching: %s" % matching_distros) return matching_distros +def create_log_file(d, logname): + #logpath = bb.data.getVar('LOG_DIR', e.data, 1)
No need for this commented line.
meta/lib/oe/distro_check.py def compare_in_distro_packages_list(distro_check_dir, d): · line 346
Quoted 1 line
meta/lib/oe/distro_check.py +1
-0+ logpath = bb.data.getVar('LOG_DIR', d, 1)
We are trying to move towards using Boolean values where appropriate to aid readability, this line should be:
logpath = bb.data.getVar('LOG_DIR', d, True)
meta/lib/oe/distro_check.py def compare_in_distro_packages_list(distro_check_dir, d): · line 348
Quoted 2 lines
meta/lib/oe/distro_check.py +2
-0+ bb.utils.mkdirhier(logpath) + logfile = logpath +"/" + logname.split(".")[0] + ".%s.csv" % bb.data.getVar('DATETIME', d, 1)
Can you use the Python standard library functions for handling filenames here? And a boolean True in the getVar() call.
Something like the following feels more Pythonic:
logfn, logext = os.path.splitext(logname) logfile = os.path.join(logpath, "%s.%s.csv" % (logfn, bb.data.getVar('DATETIME', d, True))
Quoted 13 lines
meta/lib/oe/distro_check.py +10
-0+ if not os.path.exists(logfile): + slogfile = os.path.join(logpath, logname) + if os.path.exists(slogfile): + os.remove(slogfile) + os.system("touch %s" % logfile) + os.symlink(logfile, slogfile) + bb.data.setVar('LOG_FILE', logfile, d) + return logfile + + def save_distro_check_result(result, datetime, d): pn = bb.data.getVar('PN', d, True) logdir = bb.data.getVar('LOG_DIR', d, True)
Signature · 3 lines
Joshua Lock
Yocto Build System Monkey
Intel Open Source Technology CentrePATCH 2/2 distrodata.bbclass: Add checklicense task
In reply to Mei LeiFrom: Mei Lei <[email protected]>
This task will check which license text is missing
--- meta/classes/distrodata.bbclass | 88 ++++++++++++++++++++++++--------------- 1 files changed, 54 insertions(+), 34 deletions(-)
meta/classes/distrodata.bbclass +54 -34
@@ -4,18 +4,8 @@ addhandler distro_eventhandler python distro_eventhandler() { if bb.event.getName(e) == "BuildStarted": - """initialize log files.""" - logpath = bb.data.getVar('LOG_DIR', e.data, 1) - bb.utils.mkdirhier(logpath) - logfile = os.path.join(logpath, "distrodata.%s.csv" % bb.data.getVar('DATETIME', e.data, 1)) - if not os.path.exists(logfile): - slogfile = os.path.join(logpath, "distrodata.csv") - if os.path.exists(slogfile): - os.remove(slogfile) - os.system("touch %s" % logfile) - os.symlink(logfile, slogfile) - bb.data.setVar('LOG_FILE', logfile, e.data) - + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"distrodata.csv") lf = bb.utils.lockfile(logfile + ".lock") f = open(logfile, "a") f.write("Package,Description,Owner,License,ChkSum,Status,VerMatch,Version,Upsteam,Non-Update,Reason,Recipe Status\n")
@@ -211,17 +201,8 @@ do_distrodataall() { addhandler checkpkg_eventhandler python checkpkg_eventhandler() { if bb.event.getName(e) == "BuildStarted": - """initialize log files.""" - logpath = bb.data.getVar('LOG_DIR', e.data, 1) - bb.utils.mkdirhier(logpath) - logfile = os.path.join(logpath, "checkpkg.%s.csv" % bb.data.getVar('DATETIME', e.data, 1)) - if not os.path.exists(logfile): - slogfile = os.path.join(logpath, "checkpkg.csv") - if os.path.exists(slogfile): - os.remove(slogfile) - os.system("touch %s" % logfile) - os.symlink(logfile, slogfile) - bb.data.setVar('LOG_FILE', logfile, e.data) + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"checkpkg.csv") lf = bb.utils.lockfile(logfile + ".lock") f = open(logfile, "a")
@@ -662,17 +643,8 @@ python check_eventhandler() { distro_check_dir = os.path.join(tmpdir, "distro_check") datetime = bb.data.getVar('DATETIME', e.data, 1) """initialize log files.""" - logpath = bb.data.getVar('LOG_DIR', e.data, 1) - bb.utils.mkdirhier(logpath) - logfile = os.path.join(logpath, "distrocheck.%s.csv" % bb.data.getVar('DATETIME', e.data, 1)) - if not os.path.exists(logfile): - slogfile = os.path.join(logpath, "distrocheck.csv") - if os.path.exists(slogfile): - os.remove(slogfile) - os.system("touch %s" % logfile) - os.symlink(logfile, slogfile) - bb.data.setVar('LOG_FILE', logfile, e.data) - + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"distrocheck.csv") return }
@@ -701,3 +673,51 @@ do_distro_checkall[nostamp] = "1" do_distro_checkall() { : } +##Check Missing License Text +addhandler checklicense_eventhandler +python checklicense_eventhandler() { + if bb.event.getName(e) == "BuildStarted": + """initialize log files.""" + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"missinglicense.csv") + lf = bb.utils.lockfile(logfile + ".lock") + f = open(logfile, "a") + f.write("Package\tLicense\tMissingLicense\n") + f.close() + bb.utils.unlockfile(lf) + return +} + +addtask checklicense +do_checklicense[nostamp] = "1" +python do_checklicense() { + import os + import bb + import shutil + logpath = bb.data.getVar('LOG_DIR', d, 1) + bb.utils.mkdirhier(logpath) + pn = bb.data.getVar('PN',d,1) + logfile = os.path.join(logpath, "missinglicense.csv") + generic_directory = bb.data.getVar('COMMON_LICENSE_DIR', d, True) + license_types = bb.data.getVar('LICENSE', d, True) + for license_type in ((license_types.replace('+', '').replace('|', '&') + .replace('(', '').replace(')', '').replace(';', '') + .replace(',', '').replace(" ", "").split("&"))): + if not os.path.isfile(os.path.join(generic_directory, license_type)): + lf = bb.utils.lockfile(logfile + ".lock") + f = open(logfile,"a") + f.write("%s\t%s\t%s\n" % \ + (pn,license_types,license_type)) + f.close() + bb.utils.unlockfile(lf) + return +} + +addtask checklicenseall after do_checklicense +do_checklicenseall[recrdeptask] = "do_checklicense" +do_checklicenseall[nostamp] = "1" +do_checklicenseall() { + : +} + +
Signature · 1 line
1.6.3.3
Re: [PATCH 2/2] distrodata.bbclass: Add checklicense task
Quoted 3 lines · Mei Lei
From: Mei Lei <[email protected]>
This task will check which license text is missing
Can you flesh out this commit log some please? Why is this task useful? When should it be used? Etc.
Code style comments in line below, thanks!
Quoted 72 lines
Signed-off-by Mei Lei <[email protected]>--- meta/classes/distrodata.bbclass | 88 ++++++++++++++++++++++++--------------- 1 files changed, 54 insertions(+), 34 deletions(-)meta/classes/distrodata.bbclass +6
-34@@ -4,18 +4,8 @@ addhandler distro_eventhandler python distro_eventhandler() { if bb.event.getName(e) == "BuildStarted": - """initialize log files.""" - logpath = bb.data.getVar('LOG_DIR', e.data, 1) - bb.utils.mkdirhier(logpath) - logfile = os.path.join(logpath, "distrodata.%s.csv" % bb.data.getVar('DATETIME', e.data, 1)) - if not os.path.exists(logfile): - slogfile = os.path.join(logpath, "distrodata.csv") - if os.path.exists(slogfile): - os.remove(slogfile) - os.system("touch %s" % logfile) - os.symlink(logfile, slogfile) - bb.data.setVar('LOG_FILE', logfile, e.data) - + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"distrodata.csv") lf = bb.utils.lockfile(logfile + ".lock") f = open(logfile, "a") f.write("Package,Description,Owner,License,ChkSum,Status,VerMatch,Version,Upsteam,Non-Update,Reason,Recipe Status\n")@@ -211,17 +201,8 @@ do_distrodataall() { addhandler checkpkg_eventhandler python checkpkg_eventhandler() { if bb.event.getName(e) == "BuildStarted": - """initialize log files.""" - logpath = bb.data.getVar('LOG_DIR', e.data, 1) - bb.utils.mkdirhier(logpath) - logfile = os.path.join(logpath, "checkpkg.%s.csv" % bb.data.getVar('DATETIME', e.data, 1)) - if not os.path.exists(logfile): - slogfile = os.path.join(logpath, "checkpkg.csv") - if os.path.exists(slogfile): - os.remove(slogfile) - os.system("touch %s" % logfile) - os.symlink(logfile, slogfile) - bb.data.setVar('LOG_FILE', logfile, e.data) + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"checkpkg.csv") lf = bb.utils.lockfile(logfile + ".lock") f = open(logfile, "a")@@ -662,17 +643,8 @@ python check_eventhandler() { distro_check_dir = os.path.join(tmpdir, "distro_check") datetime = bb.data.getVar('DATETIME', e.data, 1) """initialize log files.""" - logpath = bb.data.getVar('LOG_DIR', e.data, 1) - bb.utils.mkdirhier(logpath) - logfile = os.path.join(logpath, "distrocheck.%s.csv" % bb.data.getVar('DATETIME', e.data, 1)) - if not os.path.exists(logfile): - slogfile = os.path.join(logpath, "distrocheck.csv") - if os.path.exists(slogfile): - os.remove(slogfile) - os.system("touch %s" % logfile) - os.symlink(logfile, slogfile) - bb.data.setVar('LOG_FILE', logfile, e.data) - + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"distrocheck.csv") return }
meta/classes/distrodata.bbclass do_distro_checkall[nostamp] = "1" · line 682
Quoted 10 lines
meta/classes/distrodata.bbclass +7
-0@@ -701,3 +673,51 @@ do_distro_checkall[nostamp] = "1" do_distro_checkall() { : } +##Check Missing License Text +addhandler checklicense_eventhandler +python checklicense_eventhandler() { + if bb.event.getName(e) == "BuildStarted": + """initialize log files.""" + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"missinglicense.csv")
Whitespace: logfile = dc.create_log_file(e.data, "missinglicense.csv")
meta/classes/distrodata.bbclass do_distro_checkall[nostamp] = "1" · line 683
Quoted 1 line
meta/classes/distrodata.bbclass +1
-0+ lf = bb.utils.lockfile(logfile + ".lock")
Personally I prefer to use string formatting rather for this sort of thing:
+ lf = bb.utils.lockfile("%s.lock" % logfile)
Quoted 6 lines
meta/classes/distrodata.bbclass +6
-0+ f = open(logfile, "a") + f.write("Package\tLicense\tMissingLicense\n") + f.close() + bb.utils.unlockfile(lf) + return +}
meta/classes/distrodata.bbclass do_distro_checkall[nostamp] = "1" · line 699
Quoted 10 lines
meta/classes/distrodata.bbclass +10
-0+ +addtask checklicense +do_checklicense[nostamp] = "1" +python do_checklicense() { + import os + import bb + import shutil + logpath = bb.data.getVar('LOG_DIR', d, 1) + bb.utils.mkdirhier(logpath) + pn = bb.data.getVar('PN',d,1)
pn = bb.data.getVar('PN', d, True)
Please try and maintain whitespace style and use boolean values where appropriate.
meta/classes/distrodata.bbclass do_distro_checkall[nostamp] = "1" · line 707
Quoted 8 lines
meta/classes/distrodata.bbclass +8
-0+ logfile = os.path.join(logpath, "missinglicense.csv") + generic_directory = bb.data.getVar('COMMON_LICENSE_DIR', d, True) + license_types = bb.data.getVar('LICENSE', d, True) + for license_type in ((license_types.replace('+', '').replace('|', '&') + .replace('(', '').replace(')', '').replace(';', '') + .replace(',', '').replace(" ", "").split("&"))): + if not os.path.isfile(os.path.join(generic_directory, license_type)): + lf = bb.utils.lockfile(logfile + ".lock")
Maybe use string formatters here too:
+ lf = bb.utils.lockfile("%s.lock" % logfile)
Quoted 16 lines
meta/classes/distrodata.bbclass +16
-0+ f = open(logfile,"a") + f.write("%s\t%s\t%s\n" % \ + (pn,license_types,license_type)) + f.close() + bb.utils.unlockfile(lf) + return +} + +addtask checklicenseall after do_checklicense +do_checklicenseall[recrdeptask] = "do_checklicense" +do_checklicenseall[nostamp] = "1" +do_checklicenseall() { + : +} + +
Signature · 3 lines
Joshua Lock
Yocto Build System Monkey
Intel Open Source Technology CentreRe: [PATCH 2/2] distrodata.bbclass: Add checklicense task
Hi Josh,
Thanks for your comments, I am very appreciate of that.
I have revised these patches and sent a new pull-request to mail list, please review it.
Thanks
Lei
Quoted 168 lines
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Joshua Lock
Sent: Saturday, May 14, 2011 1:14 AM
To: [email protected]
Subject: Re: [poky] [PATCH 2/2] distrodata.bbclass: Add checklicense taskQuoted 3 lines · Mei Lei
From: Mei Lei <[email protected]>
This task will check which license text is missing
Can you flesh out this commit log some please? Why is this task useful? When should it be used? Etc.
Code style comments in line below, thanks!
Quoted 4 lines
Signed-off-by Mei Lei <[email protected]>--- meta/classes/distrodata.bbclass | 88++++++++++++++++++++++++---------------
Quoted 7 lines
1 files changed, 54 insertions(+), 34 deletions(-)meta/classes/distrodata.bbclass +0
-0meta/classes/distrodata.bbclass addhandler distro_eventhandler · line 10
Quoted 7 lines
meta/classes/distrodata.bbclass +0
-4@@ -4,18 +4,8 @@ addhandler distro_eventhandler python distro_eventhandler() { if bb.event.getName(e) == "BuildStarted": - """initialize log files.""" - logpath = bb.data.getVar('LOG_DIR', e.data, 1) - bb.utils.mkdirhier(logpath) - logfile = os.path.join(logpath, "distrodata.%s.csv" %
bb.data.getVar('DATETIME', e.data, 1))meta/classes/distrodata.bbclass addhandler distro_eventhandler
Quoted 13 lines
meta/classes/distrodata.bbclass +2
-8- if not os.path.exists(logfile): - slogfile = os.path.join(logpath, "distrodata.csv") - if os.path.exists(slogfile): - os.remove(slogfile) - os.system("touch %s" % logfile) - os.symlink(logfile, slogfile) - bb.data.setVar('LOG_FILE', logfile, e.data) - + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"distrodata.csv") lf = bb.utils.lockfile(logfile + ".lock") f = open(logfile, "a")f.write("Package,Description,Owner,License,ChkSum,Status,VerMatch,Versio
n,Upsteam,Non-Update,Reason,Recipe Status\n")meta/classes/distrodata.bbclass do_distrodataall() { · line 217
Quoted 8 lines
meta/classes/distrodata.bbclass +0
-4@@ -211,17 +201,8 @@ do_distrodataall() { addhandler checkpkg_eventhandler python checkpkg_eventhandler() { if bb.event.getName(e) == "BuildStarted": - """initialize log files.""" - logpath = bb.data.getVar('LOG_DIR', e.data, 1) - bb.utils.mkdirhier(logpath) - logfile = os.path.join(logpath, "checkpkg.%s.csv" %
bb.data.getVar('DATETIME', e.data, 1))Quoted 13 lines
meta/classes/distrodata.bbclass +2
-7- if not os.path.exists(logfile): - slogfile = os.path.join(logpath, "checkpkg.csv") - if os.path.exists(slogfile): - os.remove(slogfile) - os.system("touch %s" % logfile) - os.symlink(logfile, slogfile) - bb.data.setVar('LOG_FILE', logfile, e.data) + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"checkpkg.csv") lf = bb.utils.lockfile(logfile + ".lock") f = open(logfile, "a")meta/classes/distrodata.bbclass python check_eventhandler() { · line 667
Quoted 6 lines
meta/classes/distrodata.bbclass +0
-3@@ -662,17 +643,8 @@ python check_eventhandler() { distro_check_dir = os.path.join(tmpdir, "distro_check") datetime = bb.data.getVar('DATETIME', e.data, 1) """initialize log files.""" - logpath = bb.data.getVar('LOG_DIR', e.data, 1) - bb.utils.mkdirhier(logpath) - logfile = os.path.join(logpath, "distrocheck.%s.csv" %
bb.data.getVar('DATETIME', e.data, 1))Quoted 14 lines
meta/classes/distrodata.bbclass +2
-8- if not os.path.exists(logfile): - slogfile = os.path.join(logpath, "distrocheck.csv") - if os.path.exists(slogfile): - os.remove(slogfile) - os.system("touch %s" % logfile) - os.symlink(logfile, slogfile) - bb.data.setVar('LOG_FILE', logfile, e.data) - + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"distrocheck.csv") return }meta/classes/distrodata.bbclass do_distro_checkall[nostamp] = "1" · line 682
Quoted 10 lines
meta/classes/distrodata.bbclass +7
-0@@ -701,3 +673,51 @@ do_distro_checkall[nostamp] = "1" do_distro_checkall() { : } +##Check Missing License Text +addhandler checklicense_eventhandler +python checklicense_eventhandler() { + if bb.event.getName(e) == "BuildStarted": + """initialize log files.""" + import oe.distro_check as dc + logfile = dc.create_log_file(e.data,"missinglicense.csv")Whitespace: logfile =
dc.create_log_file(e.data, "missinglicense.csv")meta/classes/distrodata.bbclass do_distro_checkall[nostamp] = "1" · line 683
Quoted 1 line
meta/classes/distrodata.bbclass +1
-0+ lf = bb.utils.lockfile(logfile + ".lock")Personally I prefer to use string formatting rather for this sort of thing:
+ lf =bb.utils.lockfile("%s.lock" % logfile)Quoted 6 lines
meta/classes/distrodata.bbclass +6
-0+ f = open(logfile, "a") + f.write("Package\tLicense\tMissingLicense\n") + f.close() + bb.utils.unlockfile(lf) + return +}meta/classes/distrodata.bbclass do_distro_checkall[nostamp] = "1" · line 699
Quoted 10 lines
meta/classes/distrodata.bbclass +10
-0+ +addtask checklicense +do_checklicense[nostamp] = "1" +python do_checklicense() { + import os + import bb + import shutil + logpath = bb.data.getVar('LOG_DIR', d, 1) + bb.utils.mkdirhier(logpath) + pn = bb.data.getVar('PN',d,1)pn =
bb.data.getVar('PN', d, True)Please try and maintain whitespace style and use boolean values where appropriate.
meta/classes/distrodata.bbclass do_distro_checkall[nostamp] = "1" · line 707
Quoted 8 lines
meta/classes/distrodata.bbclass +8
-0+ logfile = os.path.join(logpath, "missinglicense.csv") + generic_directory = bb.data.getVar('COMMON_LICENSE_DIR', d, True) + license_types = bb.data.getVar('LICENSE', d, True) + for license_type in ((license_types.replace('+', '').replace('|', '&') + .replace('(', '').replace(')', '').replace(';', '') + .replace(',', '').replace(" ", "").split("&"))): + if not os.path.isfile(os.path.join(generic_directory, license_type)): + lf = bb.utils.lockfile(logfile + ".lock")Maybe use string formatters here too:
+ lf =bb.utils.lockfile("%s.lock" % logfile)Quoted 6 lines
meta/classes/distrodata.bbclass +6
-0+ f = open(logfile,"a") + f.write("%s\t%s\t%s\n" % \ + (pn,license_types,license_type)) + f.close() + bb.utils.unlockfile(lf) + returnmeta/classes/distrodata.bbclass do_distro_checkall[nostamp] = "1" · line 723
Quoted 10 lines
meta/classes/distrodata.bbclass +10
-0+} + +addtask checklicenseall after do_checklicense +do_checklicenseall[recrdeptask] = "do_checklicense" +do_checklicenseall[nostamp] = "1" +do_checklicenseall() { + : +} + +--
Joshua Lock
Yocto Build System Monkey
Intel Open Source Technology Centre_______________________________________________
poky mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/poky