Realize archiving functions

From: Xiaofeng Yan <[email protected]>

1 Archive sources in ${S} in the different stage (do_unpack,do_patch,do_configure).
2 Archive patches including series
3 Archive logs including scripts (.bb and .inc files)
4 dump environment resources which show all variable and functions used to xxx.showdata.dump when running a task
5 dump all content in 's' including patches to file xxx.diff.gz

All archiving packages  will be deployed to ${DEPLOY_DIR}/sources/

[YOCTO #1977]

Signed-off-by Xiaofeng Yan <[email protected]>

---
 meta/classes/archiver.bbclass |  460 +++++++++++++++++++++++++++++++++++++++++

 1 files changed, 460 insertions(+), 0 deletions(-)

 create mode 100644 meta/classes/archiver.bbclass

meta/classes/archiver.bbclass new file +0 -0

new file mode 100644
index 0000000..471430e

meta/classes/archiver.bbclass new file +6 -0

+def parse_var(d,var):
+       ''' parse variable like ${PV}  in "require xxx_${PV}.inc"  to a real value. for example, change "require xxx_${PV}.inc" to "require xxx_1.2.inc" '''
+       import re
+       pat = re.compile('.*\$({(.*)}).*')
+       if '$' not in var and '/' not in var:
+               return var

meta/classes/archiver.bbclass new file +19 -0

+       bb.mkdirhier("script-logs")
+       os.chdir(bbdir)
+       bbfile = os.path.basename(file_dir)
+       bbinc.append(bbfile)
+
+       def get_inc (file):
+               f = open(file,'r')
+               for line in f.readlines():
+                       if 'require' not  in line:
+                               bbinc.append(file)
+                       else:
+                               try:
+                                       incfile = pat.match(line).group(1)
+                                       incfile = parse_var(d,incfile)
+                                       bbinc.append(incfile)
+                                       get_inc(incfile)
+                               except (IOError,AttributeError):
+                                       pass
+       get_inc(bbfile)

meta/classes/archiver.bbclass new file +50 -0

+       import shutil
+
+       src_patches=[]
+       pf = d.getVar('PF', True)
+       work_dir = d.getVar('WORKDIR', True)
+       s = d.getVar('S',True)
+       dest = os.path.join(work_dir, pf + '-patches')
+       shutil.rmtree(dest, ignore_errors=True)
+       bb.mkdirhier(dest)
+
+       src_uri = d.getVar('SRC_URI', 1).split()
+       fetch = bb.fetch2.Fetch(src_uri, d)
+       locals = (fetch.localpath(url) for url in fetch.urls)
+       for local in locals:
+               src_patches.append(local)
+       if not cmp(work_dir,s):
+               tmp_list = src_patches
+       else:
+               tmp_list = src_patches[1:]
+
+       for patch in tmp_list:
+               try:
+                       shutil.copy(patch,dest)
+               except IOError:
+                       if os.path.isdir(patch):
+                               oe.path.copytree(patch,dest)
+       return dest
+
+def get_applying_patches(d):
+       """only copy applying patches to a pointed directory which will be archived to tarball"""
+       import os
+       import shutil
+
+
+       pf = d.getVar('PF', True)
+       work_dir = d.getVar('WORKDIR', True)
+       dest = os.path.join(work_dir, pf + '-patches')
+       shutil.rmtree(dest, ignore_errors=True)
+       bb.mkdirhier(dest)
+
+
+       patches = src_patches(d)
+       for patch in patches:
+               _, _, local, _, _, parm = bb.decodeurl(patch)
+               if local:
+                        shutil.copy(local,dest)
+       return dest
+
+def not_tarball(d):
+       '''packages including key words 'work-shared','native', 'task-' will be passed'''

meta/classes/archiver.bbclass new file +28 -0

+               if os.path.exists(source) and not os.path.isdir(source):
+                       return source
+       except (IndexError, OSError):
+               pass
+
+def archive_sources_from_directory(d,middle_name):
+       '''archive sources codes tree to tarball when tarball of $P doesn't exist in $DL_DIR'''
+       import tarfile
+       import shutil
+
+       s = d.getVar('S',True)
+       workdir=d.getVar('WORKDIR', True)
+       PF = d.getVar('PF',True)
+       tarname = PF + '-' + middle_name + ".tar.gz"
+
+       if os.path.exists(s) and s is not workdir:
+               sourcedir = os.path.basename(s)
+               tarbase = os.path.dirname(s)
+               if not sourcedir or os.path.dirname(tarbase) == workdir:
+                       sourcedir = os.path.basename(os.path.dirname(s))
+                       tarbase = os.path.dirname(os.path.dirname(s))
+               os.chdir(tarbase)
+       else:
+               sourcedir = os.path.basename(s)
+               if not os.path.exists(sourcedir):
+                       os.mkdir(sourcedir)
+               try:
+                       for file in os.listdir(s):

meta/classes/archiver.bbclass new file +65 -0

+       else:
+               return
+       return tarname
+
+def archive_sources(d,middle_name):
+       '''copy tarball from $DL_DIR to $WORKDIR if have tarball, archive source codes tree in $WORKDIR if $P is directory instead of tarball'''
+       import shutil
+       work_dir = d.getVar('WORKDIR',True)
+       file = get_source_from_downloads(d,middle_name)
+       if file:
+               shutil.copy(file,work_dir)
+       else:
+               file = archive_sources_from_directory(d,middle_name)
+       return file
+
+
+def archive_patches(d,patchdir,series):
+       '''archive patches to tarball and also include series files if 'series' is True'''
+       import tarfile
+       import shutil
+
+       s = d.getVar('S',True)
+       work_dir = d.getVar('WORKDIR', True)
+       os.chdir(work_dir)
+       patch_dir = os.path.basename(patchdir)
+       tarname = patch_dir + ".tar.gz"
+       if series  == 'all' and os.path.exists(s + '/patches/series'):
+                       shutil.copy(s + '/patches/series',patch_dir)
+       tar = tarfile.open(tarname, "w:gz")
+       tar.add(patch_dir)
+       tar.close()
+       shutil.rmtree(patch_dir, ignore_errors=True)
+       return tarname
+
+def select_archive_patches(d,option):
+       '''select to archive all patches including non-applying and series or applying patches '''
+       if option == "all":
+               patchdir = get_all_patches(d)
+       elif option == "applying":
+               patchdir = get_applying_patches(d)
+       try:
+               os.rmdir(patchdir)
+       except OSError:
+                       tarpatch = archive_patches(d,patchdir,option)
+                       return tarpatch
+       return
+
+def archive_logs(d,logdir,bbinc=False):
+       '''archive logs in temp to tarball and .bb and .inc files if bbinc is True '''
+       import tarfile
+       import shutil
+
+       log_dir = os.path.basename(logdir)
+       pf = d.getVar('PF',True)
+       tarname = pf + '-' + log_dir + ".tar.gz"
+       tar = tarfile.open(tarname, "w:gz")
+       tar.add(log_dir)
+       tar.close()
+       if bbinc:
+               shutil.rmtree(log_dir, ignore_errors=True)
+       return tarname
+
+def get_licenses(d):
+       '''get licenses for running .bb file'''
+       licenses = d.getVar('LICENSE', 1).replace('&', '|')

meta/classes/archiver.bbclass new file +7 -0

+       import shutil
+
+       if tarball_list is []:
+               return
+       target_sys = d.getVar('TARGET_SYS', True)
+       pf = d.getVar('PF', True)
+       licenses = get_licenses(d)

meta/classes/archiver.bbclass new file +160 -0

+               f = open('tar-package','r')
+               line = list(set(f.readline().replace('\n','').split()))
+               f.close()
+               return line
+       except IOError:
+               pass
+
+
+def archive_sources_patches(d,middle_name):
+       '''archive sources and patches to tarball. middle_name will append strings ${middle_name} to ${PR} as middle name. for example, zlib-1.4.6-prepatch(middle_name).tar.gz '''
+       import shutil
+       verify_var(d)
+       if not_tarball(d):
+               return
+
+       source_tar_name = archive_sources(d,middle_name)
+       if middle_name == "prepatch":
+               if d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'TRUE':
+                       patch_tar_name = select_archive_patches(d,"all")
+               elif d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'FALSE':
+                       patch_tar_name = select_archive_patches(d,"applying")
+               else:
+                       bb.fatal("Please define 'PATCHES_ARCHIVE_WITH_SERIES' is strings 'True' or 'False' ")
+       else:
+               patch_tar_name = ''
+
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+               move_tarball_deploy(d,[source_tar_name,patch_tar_name])
+       else:
+               tarpackage = d.getVar('WORKDIR', True) + '/tar-package'
+               if os.path.exists(tarpackage):
+                       os.remove(tarpackage)
+               for package in source_tar_name, patch_tar_name:
+                       if package:
+                               store_package(d,str(package) + ' ')
+
+def archive_scripts_logs(d):
+       '''archive scripts and logs. scripts include .bb and .inc files and logs include stuff in "temp".'''
+
+       work_dir = d.getVar('WORKDIR', True)
+       os.chdir(work_dir)
+       source_archive_log_with_scripts = d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True)
+       if source_archive_log_with_scripts == 'logs_with_scripts':
+               logdir = get_bb_inc(d)
+               tarlog = archive_logs(d,logdir,True)
+       elif source_archive_log_with_scripts == 'logs':
+               if os.path.exists('temp'):
+                       tarlog = archive_logs(d,'temp',False)
+       else:
+               return
+
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+               move_tarball_deploy(d,[tarlog])
+
+       else:
+               store_package(d,tarlog)
+
+def dumpdata(d):
+       '''dump environment to "${P}-${PR}.showdata.dump" including all kinds of variables and functions when running a task'''
+       workdir = bb.data.getVar('WORKDIR', d, 1)
+       distro = bb.data.getVar('DISTRO', d, 1)
+       s = d.getVar('S', True)
+       pf = d.getVar('PF', True)
+       target_sys = d.getVar('TARGET_SYS', True)
+       licenses = get_licenses(d)
+       dumpdir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+       if not os.path.exists(dumpdir):
+               bb.mkdirhier(dumpdir)
+
+       dumpfile = os.path.join(dumpdir, bb.data.expand("${P}-${PR}.showdata.dump",d))
+
+       bb.note("Dumping metadata into '%s'" % dumpfile)
+       f = open(dumpfile, "w")
+       # emit variables and shell functions
+        bb.data.emit_env(f, d, True)
+       # emit the metadata which isnt valid shell
+       for e in d.keys():
+               if bb.data.getVarFlag(e, 'python', d):
+                       f.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, 1)))
+       f.close()
+
+def create_diff_gz(d):
+       '''creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.g gz for mapping all content in 's' including patches to  xxx.diff.gz'''
+       import shutil
+
+       work_dir = d.getVar('WORKDIR', True)
+       exclude_from = d.getVar('EXCLUDE_FROM', True).split()
+       pf = d.getVar('PF', True)
+       licenses = get_licenses(d)
+       target_sys = d.getVar('TARGET_SYS', True)
+       diff_dir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+       diff_file = os.path.join(diff_dir, bb.data.expand("${P}-${PR}.diff.gz",d))
+       os.chdir(work_dir)
+       f = open('temp/exclude-from-file', 'a')
+       for i in exclude_from:
+               f.write(i)
+               f.write("\n")
+       f.close()
+
+       s=d.getVar('S', True)
+       distro = d.getVar('DISTRO',True)
+       dest = s + '/' + distro + '/files'
+       if not os.path.exists(dest):
+               bb.mkdirhier(dest)
+       for i in os.listdir(os.getcwd()):
+               if os.path.isfile(i):
+                       shutil.copy(i, dest)
+
+       bb.note("Creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz")
+       cmd = "LC_ALL=C TZ=UTC0 diff --exclude-from=" + work_dir + "/temp/exclude-from-file -Naur " + s + '.org' + ' ' +  s + " | gzip -c > " + diff_file
+       d.setVar('DIFF', cmd + "\n")
+       d.setVarFlag('DIFF', 'func', '1')
+       bb.build.exec_func('DIFF', d)
+       shutil.rmtree(s + '.org', ignore_errors=True)
+
+# This function will run when user want to get tarball for sources and patches after do_unpack
+python do_archive_original_sources_patches(){
+       archive_sources_patches(d,'prepatch')
+}
+
+# This function will run when user want to get tarball for patched sources after do_patch
+python do_archive_patched_sources(){
+       archive_sources_patches(d,'patched')
+}
+
+# This function will run when user want to get tarball for configured sources after do_configure
+python do_archive_configured_sources(){
+       archive_sources_patches(d,'configured')
+}
+
+# This function will run when user want to get tarball for logs or both logs and scripts(.bb and .inc files)
+python do_archive_scripts_logs(){
+       archive_scripts_logs(d)
+}
+
+# This function will run when user want to know what variable and functions in a running task are and also can get a diff file including
+# all content a package should include.
+python do_dumpdata_create_diff_gz(){
+       dumpdata(d)
+       create_diff_gz(d)
+}
+
+# This functions prepare for archiving "linux-yocto" because this package create directory 's' before do_patch instead of after do_unpack.
+# This is special control for archiving linux-yocto only.
+python do_archive_linux_yocto(){
+       s = d.getVar('S', True)
+       if 'linux-yocto' in s:
+               source_tar_name = archive_sources(d,'')
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+               move_tarball_deploy(d,[source_tar_name,''])
+}
+do_kernel_checkout[postfuncs] += "do_archive_linux_yocto "
+
+# remove tarball for sources, patches and logs after creating srpm.
+python do_remove_tarball(){
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+               work_dir = d.getVar('WORKDIR', True)
+               os.chdir(work_dir)
+               try:
+                       for file in os.listdir(os.getcwd()):

Quoted 26 lines · Xiaofeng Yan

From: Xiaofeng Yan<[email protected]>

1 Archive sources in ${S} in the different stage
(do_unpack,do_patch,do_configure).
2 Archive patches including series
3 Archive logs including scripts (.bb and .inc files)
4 dump environment resources which show all variable and functions
used to xxx.showdata.dump when running a task
5 dump all content in 's' including patches to file xxx.diff.gz

All archiving packages will be deployed to ${DEPLOY_DIR}/sources/

[YOCTO #1977]

Signed-off-by Xiaofeng Yan<[email protected]>

---
  meta/classes/archiver.bbclass |  460 +++++++++++++++++++++++++++++++++++++++++
  1 files changed, 460 insertions(+), 0 deletions(-)

create mode 100644 meta/classes/archiver.bbclass

meta/classes/archiver.bbclass new file +0 -0

new file mode 100644
index 0000000..471430e

meta/classes/archiver.bbclass line 8

Quoted 8 lines

meta/classes/archiver.bbclass new file +8 -0

@@ -0,0 +1,460 @@
+# This file is used for archiving sources ,patches,and logs to tarball.
+# It also output building environment to xxx.dump.data and create xxx.diff.gz to record
+# all content in ${S} to a diff file.
+
+EXCLUDE_FROM ?= ".pc autom4te.cache"
+ARCHIVE_TYPE ?= "TAR SRPM"
+DISTRO ?= "poky"
+PATCHES_ARCHIVE_WITH_SERIES = 'TRUE'

I'm a bit concerned about the variable names here. Perhaps we should prefix them with ARCHIVER_? "EXCLUDE_FROM" is a bit generic.

Quoted 6 lines

meta/classes/archiver.bbclass new file +6 -0

+def parse_var(d,var):
+       ''' parse variable like ${PV}  in "require xxx_${PV}.inc"  to a real value. for example, change "require xxx_${PV}.inc" to "require xxx_1.2.inc" '''
+       import re
+       pat = re.compile('.*\$({(.*)}).*')
+       if '$' not in var and '/' not in var:
+               return var

meta/classes/archiver.bbclass line 25

Quoted 10 lines

meta/classes/archiver.bbclass new file +10 -0

+       else:
+               if '/' in var:
+                       return [i for i in var.split('/') if i.endswith('.inc')][0]
+               elif '$' in  var:
+                       m = pat.match(var)
+                       patstr = '\$' + m.group(1)
+                       var_str = m.group(2)
+                       return re.sub(patstr,d.getVar(var_str,True),var)
+               else:
+                       return var

Why not using bb.data.expand(), which is designed for this? I see one usage of parse_var here which has access to 'd', so I see no reason not to use the standard mechanisms.

meta/classes/archiver.bbclass line 38

Quoted 12 lines

meta/classes/archiver.bbclass new file +12 -0

+def get_bb_inc(d):
+       '''create a directory "script-logs" including .bb and .inc file in ${WORKDIR}'''
+       import re
+       import os
+       import shutil
+
+       bbinc = []
+       pat=re.compile('require\s*([^\s]*\.*)(.*)')
+       file_dir = d.getVar('FILE', True)
+       bbdir = os.path.dirname(file_dir)
+       work_dir = d.getVar('WORKDIR', True)
+       os.chdir(work_dir)

I'd recommend against using chdir() unless there's no way around it. I'd suggest working with the absolute paths instead.

Quoted 19 lines

meta/classes/archiver.bbclass new file +19 -0

+       bb.mkdirhier("script-logs")
+       os.chdir(bbdir)
+       bbfile = os.path.basename(file_dir)
+       bbinc.append(bbfile)
+
+       def get_inc (file):
+               f = open(file,'r')
+               for line in f.readlines():
+                       if 'require' not  in line:
+                               bbinc.append(file)
+                       else:
+                               try:
+                                       incfile = pat.match(line).group(1)
+                                       incfile = parse_var(d,incfile)
+                                       bbinc.append(incfile)
+                                       get_inc(incfile)
+                               except (IOError,AttributeError):
+                                       pass
+       get_inc(bbfile)

meta/classes/archiver.bbclass line 67

Quoted 10 lines

meta/classes/archiver.bbclass new file +10 -0

+       os.chdir(work_dir)
+       for root, dirs, files in os.walk(bbdir):
+               for file in bbinc:
+                       if file in files:
+                               shutil.copy(root + '/' + file,'script-logs')
+       oe.path.copytree('temp', 'script-logs')
+       return work_dir + '/script-logs'
+
+def get_all_patches(d):
+       '''copy patches and series file to a pointed directory which will be archived to tarball in ${WORKDIR}'''

This claims to be working with a series file, but I don't see anything here which does anything of the sort. Further, as you aren't determining what is and is not a patch, this will capture more than just patches, also config files, etc, which makes either the function name or its implementation wrong, depending on which is correct.

Quoted 50 lines

meta/classes/archiver.bbclass new file +50 -0

+       import shutil
+
+       src_patches=[]
+       pf = d.getVar('PF', True)
+       work_dir = d.getVar('WORKDIR', True)
+       s = d.getVar('S',True)
+       dest = os.path.join(work_dir, pf + '-patches')
+       shutil.rmtree(dest, ignore_errors=True)
+       bb.mkdirhier(dest)
+
+       src_uri = d.getVar('SRC_URI', 1).split()
+       fetch = bb.fetch2.Fetch(src_uri, d)
+       locals = (fetch.localpath(url) for url in fetch.urls)
+       for local in locals:
+               src_patches.append(local)
+       if not cmp(work_dir,s):
+               tmp_list = src_patches
+       else:
+               tmp_list = src_patches[1:]
+
+       for patch in tmp_list:
+               try:
+                       shutil.copy(patch,dest)
+               except IOError:
+                       if os.path.isdir(patch):
+                               oe.path.copytree(patch,dest)
+       return dest
+
+def get_applying_patches(d):
+       """only copy applying patches to a pointed directory which will be archived to tarball"""
+       import os
+       import shutil
+
+
+       pf = d.getVar('PF', True)
+       work_dir = d.getVar('WORKDIR', True)
+       dest = os.path.join(work_dir, pf + '-patches')
+       shutil.rmtree(dest, ignore_errors=True)
+       bb.mkdirhier(dest)
+
+
+       patches = src_patches(d)
+       for patch in patches:
+               _, _, local, _, _, parm = bb.decodeurl(patch)
+               if local:
+                        shutil.copy(local,dest)
+       return dest
+
+def not_tarball(d):
+       '''packages including key words 'work-shared','native', 'task-' will be passed'''

meta/classes/archiver.bbclass line 127

Quoted 10 lines

meta/classes/archiver.bbclass new file +10 -0

+       import os
+
+       workdir = d.getVar('WORKDIR',True)
+       s = d.getVar('S',True)
+       if 'work-shared' in s or 'task-' in workdir or 'native' in workdir:
+               return True
+       else:
+               return False
+
+def get_source_from_downloads(d,middle_name):

I'd encourage you to find a better variable name than "middle_name". It's vague at best, confusing at worst.

meta/classes/archiver.bbclass line 134

Quoted 7 lines

meta/classes/archiver.bbclass new file +7 -0

+       '''copy tarball of $P to $WORKDIR when this tarball exists in $DL_DIR'''
+       if middle_name in 'patched' 'configured':
+               return
+       pf = d.getVar('PF', True)
+       dl_dir = d.getVar('DL_DIR',True)
+       try:
+               source = os.path.basename(d.getVar('SRC_URI', 1).split()[0])

Incorrect getVar call, use True.

meta/classes/archiver.bbclass line 135

Quoted 1 line

meta/classes/archiver.bbclass new file +1 -0

+               os.chdir(dl_dir)

You do a chdir here, but don't return it to the previous path afterwards.

Quoted 28 lines

meta/classes/archiver.bbclass new file +28 -0

+               if os.path.exists(source) and not os.path.isdir(source):
+                       return source
+       except (IndexError, OSError):
+               pass
+
+def archive_sources_from_directory(d,middle_name):
+       '''archive sources codes tree to tarball when tarball of $P doesn't exist in $DL_DIR'''
+       import tarfile
+       import shutil
+
+       s = d.getVar('S',True)
+       workdir=d.getVar('WORKDIR', True)
+       PF = d.getVar('PF',True)
+       tarname = PF + '-' + middle_name + ".tar.gz"
+
+       if os.path.exists(s) and s is not workdir:
+               sourcedir = os.path.basename(s)
+               tarbase = os.path.dirname(s)
+               if not sourcedir or os.path.dirname(tarbase) == workdir:
+                       sourcedir = os.path.basename(os.path.dirname(s))
+                       tarbase = os.path.dirname(os.path.dirname(s))
+               os.chdir(tarbase)
+       else:
+               sourcedir = os.path.basename(s)
+               if not os.path.exists(sourcedir):
+                       os.mkdir(sourcedir)
+               try:
+                       for file in os.listdir(s):

meta/classes/archiver.bbclass line 173

Quoted 10 lines

meta/classes/archiver.bbclass new file +10 -0

+                               if file is not 'temp' and file is not sourcedir:
+                                       shutil.copy(file,sourcedir)
+               except (IOError,OSError):
+                       pass
+
+       if (len(os.listdir(sourcedir))) != 0:
+               tar = tarfile.open( tarname, "w:gz")
+               tar.add(sourcedir)
+               tar.close()
+               if cmp(workdir,os.path.dirname(s)) and not os.path.exists(workdir + '/' + tarname):

Use os.path.join, not + '/'.

meta/classes/archiver.bbclass line 174

Quoted 1 line

meta/classes/archiver.bbclass new file +1 -0

+                       shutil.move(os.path.dirname(s) + '/' + tarname,workdir)

Same as above.

Quoted 65 lines

meta/classes/archiver.bbclass new file +65 -0

+       else:
+               return
+       return tarname
+
+def archive_sources(d,middle_name):
+       '''copy tarball from $DL_DIR to $WORKDIR if have tarball, archive source codes tree in $WORKDIR if $P is directory instead of tarball'''
+       import shutil
+       work_dir = d.getVar('WORKDIR',True)
+       file = get_source_from_downloads(d,middle_name)
+       if file:
+               shutil.copy(file,work_dir)
+       else:
+               file = archive_sources_from_directory(d,middle_name)
+       return file
+
+
+def archive_patches(d,patchdir,series):
+       '''archive patches to tarball and also include series files if 'series' is True'''
+       import tarfile
+       import shutil
+
+       s = d.getVar('S',True)
+       work_dir = d.getVar('WORKDIR', True)
+       os.chdir(work_dir)
+       patch_dir = os.path.basename(patchdir)
+       tarname = patch_dir + ".tar.gz"
+       if series  == 'all' and os.path.exists(s + '/patches/series'):
+                       shutil.copy(s + '/patches/series',patch_dir)
+       tar = tarfile.open(tarname, "w:gz")
+       tar.add(patch_dir)
+       tar.close()
+       shutil.rmtree(patch_dir, ignore_errors=True)
+       return tarname
+
+def select_archive_patches(d,option):
+       '''select to archive all patches including non-applying and series or applying patches '''
+       if option == "all":
+               patchdir = get_all_patches(d)
+       elif option == "applying":
+               patchdir = get_applying_patches(d)
+       try:
+               os.rmdir(patchdir)
+       except OSError:
+                       tarpatch = archive_patches(d,patchdir,option)
+                       return tarpatch
+       return
+
+def archive_logs(d,logdir,bbinc=False):
+       '''archive logs in temp to tarball and .bb and .inc files if bbinc is True '''
+       import tarfile
+       import shutil
+
+       log_dir = os.path.basename(logdir)
+       pf = d.getVar('PF',True)
+       tarname = pf + '-' + log_dir + ".tar.gz"
+       tar = tarfile.open(tarname, "w:gz")
+       tar.add(log_dir)
+       tar.close()
+       if bbinc:
+               shutil.rmtree(log_dir, ignore_errors=True)
+       return tarname
+
+def get_licenses(d):
+       '''get licenses for running .bb file'''
+       licenses = d.getVar('LICENSE', 1).replace('&', '|')

meta/classes/archiver.bbclass line 249

Quoted 10 lines

meta/classes/archiver.bbclass new file +10 -0

+       licenses = licenses.replace('(', '').replace(')', '')
+       clean_licenses = ""
+       for x in licenses.split():
+               if x.strip() == '' or x == 'CLOSED':
+                       continue
+               if x != "|":
+                       clean_licenses += x
+       if '|' in clean_licenses:
+               clean_licenses = clean_licenses.replace('|','')
+       return clean_licenses

We have a license parser for a reason, is there a reason behind not using it here? If it's potential performance benefit, I'd encourage you to do the right thing, profile, *then* optimize.

meta/classes/archiver.bbclass line 253

Quoted 2 lines

meta/classes/archiver.bbclass new file +2 -0

+def move_tarball_deploy(d,tarball_list):
+       '''move tarball in location to ${DEPLOY_DIR}/sources'''

"in location" is extremely vague here. Please improve the function documentation.

Quoted 7 lines

meta/classes/archiver.bbclass new file +7 -0

+       import shutil
+
+       if tarball_list is []:
+               return
+       target_sys = d.getVar('TARGET_SYS', True)
+       pf = d.getVar('PF', True)
+       licenses = get_licenses(d)

meta/classes/archiver.bbclass line 270

Quoted 10 lines

meta/classes/archiver.bbclass new file +10 -0

+       tar_sources = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+       if not os.path.exists(tar_sources):
+               bb.mkdirhier(tar_sources)
+       for source in tarball_list:
+               if source:
+                       if os.path.exists(tar_sources + '/' + source):
+                               os.remove(tar_sources + '/' + source)
+                       shutil.move(source,tar_sources)
+
+def verify_var(d):

This function name doesn't correspond to what it does. Please make it more clear.

meta/classes/archiver.bbclass line 279

Quoted 9 lines

meta/classes/archiver.bbclass new file +9 -0

+       '''check the type for archiving package('tar' or 'srpm')'''
+       try:
+               if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in d.getVar('ARCHIVE_TYPE', True).split():
+                       raise AttributeError
+       except AttributeError:
+                       bb.fatal("\"SOURCE_ARCHIVE_PACKAGE_TYPE\" is \'tar\' or \'srpm\', no other types")
+
+def store_package(d,package_name):
+       '''store tarbablls name to file "tar-package"'''

Typo in docstring.

meta/classes/archiver.bbclass line 277

Quoted 2 lines

meta/classes/archiver.bbclass new file +2 -0

+       try:
+               f = open(d.getVar('WORKDIR',True )+ '/tar-package','a')

More use of string concatenation rather than os.path.join.

meta/classes/archiver.bbclass line 290

Quoted 9 lines

meta/classes/archiver.bbclass new file +9 -0

+               f.write(package_name + ' ')
+               f.close()
+       except IOError:
+               pass
+
+def get_package(d):
+       '''get tarballs name from "tar-package"'''
+       try:
+               os.chdir(d.getVar('WORKDIR', True))

Another chdir without returning to the previous path. Functions shouldn't have process-wide side effects like this, in my opinion. Either work with the absolute paths (preferred) or return to the old path.

Quoted 160 lines

meta/classes/archiver.bbclass new file +160 -0

+               f = open('tar-package','r')
+               line = list(set(f.readline().replace('\n','').split()))
+               f.close()
+               return line
+       except IOError:
+               pass
+
+
+def archive_sources_patches(d,middle_name):
+       '''archive sources and patches to tarball. middle_name will append strings ${middle_name} to ${PR} as middle name. for example, zlib-1.4.6-prepatch(middle_name).tar.gz '''
+       import shutil
+       verify_var(d)
+       if not_tarball(d):
+               return
+
+       source_tar_name = archive_sources(d,middle_name)
+       if middle_name == "prepatch":
+               if d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'TRUE':
+                       patch_tar_name = select_archive_patches(d,"all")
+               elif d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'FALSE':
+                       patch_tar_name = select_archive_patches(d,"applying")
+               else:
+                       bb.fatal("Please define 'PATCHES_ARCHIVE_WITH_SERIES' is strings 'True' or 'False' ")
+       else:
+               patch_tar_name = ''
+
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+               move_tarball_deploy(d,[source_tar_name,patch_tar_name])
+       else:
+               tarpackage = d.getVar('WORKDIR', True) + '/tar-package'
+               if os.path.exists(tarpackage):
+                       os.remove(tarpackage)
+               for package in source_tar_name, patch_tar_name:
+                       if package:
+                               store_package(d,str(package) + ' ')
+
+def archive_scripts_logs(d):
+       '''archive scripts and logs. scripts include .bb and .inc files and logs include stuff in "temp".'''
+
+       work_dir = d.getVar('WORKDIR', True)
+       os.chdir(work_dir)
+       source_archive_log_with_scripts = d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True)
+       if source_archive_log_with_scripts == 'logs_with_scripts':
+               logdir = get_bb_inc(d)
+               tarlog = archive_logs(d,logdir,True)
+       elif source_archive_log_with_scripts == 'logs':
+               if os.path.exists('temp'):
+                       tarlog = archive_logs(d,'temp',False)
+       else:
+               return
+
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+               move_tarball_deploy(d,[tarlog])
+
+       else:
+               store_package(d,tarlog)
+
+def dumpdata(d):
+       '''dump environment to "${P}-${PR}.showdata.dump" including all kinds of variables and functions when running a task'''
+       workdir = bb.data.getVar('WORKDIR', d, 1)
+       distro = bb.data.getVar('DISTRO', d, 1)
+       s = d.getVar('S', True)
+       pf = d.getVar('PF', True)
+       target_sys = d.getVar('TARGET_SYS', True)
+       licenses = get_licenses(d)
+       dumpdir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+       if not os.path.exists(dumpdir):
+               bb.mkdirhier(dumpdir)
+
+       dumpfile = os.path.join(dumpdir, bb.data.expand("${P}-${PR}.showdata.dump",d))
+
+       bb.note("Dumping metadata into '%s'" % dumpfile)
+       f = open(dumpfile, "w")
+       # emit variables and shell functions
+        bb.data.emit_env(f, d, True)
+       # emit the metadata which isnt valid shell
+       for e in d.keys():
+               if bb.data.getVarFlag(e, 'python', d):
+                       f.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, 1)))
+       f.close()
+
+def create_diff_gz(d):
+       '''creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.g gz for mapping all content in 's' including patches to  xxx.diff.gz'''
+       import shutil
+
+       work_dir = d.getVar('WORKDIR', True)
+       exclude_from = d.getVar('EXCLUDE_FROM', True).split()
+       pf = d.getVar('PF', True)
+       licenses = get_licenses(d)
+       target_sys = d.getVar('TARGET_SYS', True)
+       diff_dir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+       diff_file = os.path.join(diff_dir, bb.data.expand("${P}-${PR}.diff.gz",d))
+       os.chdir(work_dir)
+       f = open('temp/exclude-from-file', 'a')
+       for i in exclude_from:
+               f.write(i)
+               f.write("\n")
+       f.close()
+
+       s=d.getVar('S', True)
+       distro = d.getVar('DISTRO',True)
+       dest = s + '/' + distro + '/files'
+       if not os.path.exists(dest):
+               bb.mkdirhier(dest)
+       for i in os.listdir(os.getcwd()):
+               if os.path.isfile(i):
+                       shutil.copy(i, dest)
+
+       bb.note("Creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz")
+       cmd = "LC_ALL=C TZ=UTC0 diff --exclude-from=" + work_dir + "/temp/exclude-from-file -Naur " + s + '.org' + ' ' +  s + " | gzip -c>  " + diff_file
+       d.setVar('DIFF', cmd + "\n")
+       d.setVarFlag('DIFF', 'func', '1')
+       bb.build.exec_func('DIFF', d)
+       shutil.rmtree(s + '.org', ignore_errors=True)
+
+# This function will run when user want to get tarball for sources and patches after do_unpack
+python do_archive_original_sources_patches(){
+       archive_sources_patches(d,'prepatch')
+}
+
+# This function will run when user want to get tarball for patched sources after do_patch
+python do_archive_patched_sources(){
+       archive_sources_patches(d,'patched')
+}
+
+# This function will run when user want to get tarball for configured sources after do_configure
+python do_archive_configured_sources(){
+       archive_sources_patches(d,'configured')
+}
+
+# This function will run when user want to get tarball for logs or both logs and scripts(.bb and .inc files)
+python do_archive_scripts_logs(){
+       archive_scripts_logs(d)
+}
+
+# This function will run when user want to know what variable and functions in a running task are and also can get a diff file including
+# all content a package should include.
+python do_dumpdata_create_diff_gz(){
+       dumpdata(d)
+       create_diff_gz(d)
+}
+
+# This functions prepare for archiving "linux-yocto" because this package create directory 's' before do_patch instead of after do_unpack.
+# This is special control for archiving linux-yocto only.
+python do_archive_linux_yocto(){
+       s = d.getVar('S', True)
+       if 'linux-yocto' in s:
+               source_tar_name = archive_sources(d,'')
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+               move_tarball_deploy(d,[source_tar_name,''])
+}
+do_kernel_checkout[postfuncs] += "do_archive_linux_yocto "
+
+# remove tarball for sources, patches and logs after creating srpm.
+python do_remove_tarball(){
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+               work_dir = d.getVar('WORKDIR', True)
+               os.chdir(work_dir)
+               try:
+                       for file in os.listdir(os.getcwd()):

meta/classes/archiver.bbclass line 460

Quoted 10 lines

meta/classes/archiver.bbclass new file +10 -0

+                               if file in get_package(d):
+                                       os.remove(file)
+                       os.remove('tar-package')
+               except (TypeError,OSError):
+                               pass
+}
+do_remove_taball[deptask] = "do_archive_scripts_logs"
+do_package_write_rpm[postfuncs] += "do_remove_tarball "
+export get_licenses
+export get_package

export? This isn't making any sense to me. Why are you exporting these functions to shell tasks?

From: Xiaofeng Yan <[email protected]>

1 Archive sources in ${S} in the different stage
(do_unpack,do_patch,do_configure).
2 Archive patches including series
3 Archive logs including scripts (.bb and .inc files)
4 dump environment resources which show all variable and functions
used to xxx.showdata.dump when running a task
5 dump all content in 's' including patches to file xxx.diff.gz

All archiving packages will be deployed to ${DEPLOY_DIR}/sources/

[YOCTO #1977]

Signed-off-by Xiaofeng Yan <[email protected]>

---
 meta/classes/archiver.bbclass |  460
+++++++++++++++++++++++++++++++++++++++++
 1 files changed, 460 insertions(+), 0 deletions(-)
 create mode 100644 meta/classes/archiver.bbclass

meta/classes/archiver.bbclass new file +2 -0

new file mode 100644
index 0000000..471430e
@@ -0,0 +1,460 @@
+# This file is used for archiving sources ,patches,and logs to tarball.
+# It also output building environment to xxx.dump.data and create

xxx.diff.gz to record

meta/classes/archiver.bbclass new file +9 -0

+# all content in ${S} to a diff file.
+
+EXCLUDE_FROM ?= ".pc autom4te.cache"
+ARCHIVE_TYPE ?= "TAR SRPM"
+DISTRO ?= "poky"
+PATCHES_ARCHIVE_WITH_SERIES = 'TRUE'
+
+def parse_var(d,var):
+       ''' parse variable like ${PV}  in "require xxx_${PV}.inc"  to a

real value. for example, change "require xxx_${PV}.inc" to "require xxx_1.2.inc" '''

meta/classes/archiver.bbclass new file +7 -0

+       import re
+       pat = re.compile('.*\$({(.*)}).*')
+       if '$' not in var and '/' not in var:
+               return var
+       else:
+               if '/' in var:
+                       return [i for i in var.split('/') if

i.endswith('.inc')][0]

meta/classes/archiver.bbclass new file +10 -0

+               elif '$' in  var:
+                       m = pat.match(var)
+                       patstr = '\$' + m.group(1)
+                       var_str = m.group(2)
+                       return re.sub(patstr,d.getVar(var_str,True),var)
+               else:
+                       return var
+
+def get_bb_inc(d):
+       '''create a directory "script-logs" including .bb and .inc file in

${WORKDIR}'''

meta/classes/archiver.bbclass new file +34 -0

+       import re
+       import os
+       import shutil
+
+       bbinc = []
+       pat=re.compile('require\s*([^\s]*\.*)(.*)')
+       file_dir = d.getVar('FILE', True)
+       bbdir = os.path.dirname(file_dir)
+       work_dir = d.getVar('WORKDIR', True)
+       os.chdir(work_dir)
+       bb.mkdirhier("script-logs")
+       os.chdir(bbdir)
+       bbfile = os.path.basename(file_dir)
+       bbinc.append(bbfile)
+
+       def get_inc (file):
+               f = open(file,'r')
+               for line in f.readlines():
+                       if 'require' not  in line:
+                               bbinc.append(file)
+                       else:
+                               try:
+                                       incfile = pat.match(line).group(1)
+                                       incfile = parse_var(d,incfile)
+                                       bbinc.append(incfile)
+                                       get_inc(incfile)
+                               except (IOError,AttributeError):
+                                       pass
+       get_inc(bbfile)
+       os.chdir(work_dir)
+       for root, dirs, files in os.walk(bbdir):
+               for file in bbinc:
+                       if file in files:
+                               shutil.copy(root + '/' +

file,'script-logs')

meta/classes/archiver.bbclass new file +5 -0

+       oe.path.copytree('temp', 'script-logs')
+       return work_dir + '/script-logs'
+
+def get_all_patches(d):
+       '''copy patches and series file to a pointed directory which will

be archived to tarball in ${WORKDIR}'''

meta/classes/archiver.bbclass new file +30 -0

+       import shutil
+
+       src_patches=[]
+       pf = d.getVar('PF', True)
+       work_dir = d.getVar('WORKDIR', True)
+       s = d.getVar('S',True)
+       dest = os.path.join(work_dir, pf + '-patches')
+       shutil.rmtree(dest, ignore_errors=True)
+       bb.mkdirhier(dest)
+
+       src_uri = d.getVar('SRC_URI', 1).split()
+       fetch = bb.fetch2.Fetch(src_uri, d)
+       locals = (fetch.localpath(url) for url in fetch.urls)
+       for local in locals:
+               src_patches.append(local)
+       if not cmp(work_dir,s):
+               tmp_list = src_patches
+       else:
+               tmp_list = src_patches[1:]
+
+       for patch in tmp_list:
+               try:
+                       shutil.copy(patch,dest)
+               except IOError:
+                       if os.path.isdir(patch):
+                               oe.path.copytree(patch,dest)
+       return dest
+
+def get_applying_patches(d):
+       """only copy applying patches to a pointed directory which will be

archived to tarball"""

meta/classes/archiver.bbclass new file +20 -0

+       import os
+       import shutil
+
+
+       pf = d.getVar('PF', True)
+       work_dir = d.getVar('WORKDIR', True)
+       dest = os.path.join(work_dir, pf + '-patches')
+       shutil.rmtree(dest, ignore_errors=True)
+       bb.mkdirhier(dest)
+
+
+       patches = src_patches(d)
+       for patch in patches:
+               _, _, local, _, _, parm = bb.decodeurl(patch)
+               if local:
+                        shutil.copy(local,dest)
+       return dest
+
+def not_tarball(d):
+       '''packages including key words 'work-shared','native', 'task-'

will be passed'''

meta/classes/archiver.bbclass new file +11 -0

+       import os
+
+       workdir = d.getVar('WORKDIR',True)
+       s = d.getVar('S',True)
+       if 'work-shared' in s or 'task-' in workdir or 'native' in workdir:
+               return True
+       else:
+               return False
+
+def get_source_from_downloads(d,middle_name):
+       '''copy tarball of $P to $WORKDIR when this tarball exists in

$DL_DIR'''

meta/classes/archiver.bbclass new file +6 -0

+       if middle_name in 'patched' 'configured':
+               return
+       pf = d.getVar('PF', True)
+       dl_dir = d.getVar('DL_DIR',True)
+       try:
+               source = os.path.basename(d.getVar('SRC_URI',

1).split()[0])

meta/classes/archiver.bbclass new file +8 -0

+               os.chdir(dl_dir)
+               if os.path.exists(source) and not os.path.isdir(source):
+                       return source
+       except (IndexError, OSError):
+               pass
+
+def archive_sources_from_directory(d,middle_name):
+       '''archive sources codes tree to tarball when tarball of $P

doesn't exist in $DL_DIR'''

meta/classes/archiver.bbclass new file +22 -0

+       import tarfile
+       import shutil
+
+       s = d.getVar('S',True)
+       workdir=d.getVar('WORKDIR', True)
+       PF = d.getVar('PF',True)
+       tarname = PF + '-' + middle_name + ".tar.gz"
+
+       if os.path.exists(s) and s is not workdir:
+               sourcedir = os.path.basename(s)
+               tarbase = os.path.dirname(s)
+               if not sourcedir or os.path.dirname(tarbase) == workdir:
+                       sourcedir = os.path.basename(os.path.dirname(s))
+                       tarbase = os.path.dirname(os.path.dirname(s))
+               os.chdir(tarbase)
+       else:
+               sourcedir = os.path.basename(s)
+               if not os.path.exists(sourcedir):
+                       os.mkdir(sourcedir)
+               try:
+                       for file in os.listdir(s):
+                               if file is not 'temp' and file is not

sourcedir:

meta/classes/archiver.bbclass new file +9 -0

+                                       shutil.copy(file,sourcedir)
+               except (IOError,OSError):
+                       pass
+
+       if (len(os.listdir(sourcedir))) != 0:
+               tar = tarfile.open( tarname, "w:gz")
+               tar.add(sourcedir)
+               tar.close()
+               if cmp(workdir,os.path.dirname(s)) and not

os.path.exists(workdir + '/' + tarname):
+ shutil.move(os.path.dirname(s) + '/' +
tarname,workdir)

meta/classes/archiver.bbclass new file +6 -0

+       else:
+               return
+       return tarname
+
+def archive_sources(d,middle_name):
+       '''copy tarball from $DL_DIR to $WORKDIR if have tarball, archive

source codes tree in $WORKDIR if $P is directory instead of tarball'''

meta/classes/archiver.bbclass new file +12 -0

+       import shutil
+       work_dir = d.getVar('WORKDIR',True)
+       file = get_source_from_downloads(d,middle_name)
+       if file:
+               shutil.copy(file,work_dir)
+       else:
+               file = archive_sources_from_directory(d,middle_name)
+       return file
+
+
+def archive_patches(d,patchdir,series):
+       '''archive patches to tarball and also include series files if

'series' is True'''

meta/classes/archiver.bbclass new file +18 -0

+       import tarfile
+       import shutil
+
+       s = d.getVar('S',True)
+       work_dir = d.getVar('WORKDIR', True)
+       os.chdir(work_dir)
+       patch_dir = os.path.basename(patchdir)
+       tarname = patch_dir + ".tar.gz"
+       if series  == 'all' and os.path.exists(s + '/patches/series'):
+                       shutil.copy(s + '/patches/series',patch_dir)
+       tar = tarfile.open(tarname, "w:gz")
+       tar.add(patch_dir)
+       tar.close()
+       shutil.rmtree(patch_dir, ignore_errors=True)
+       return tarname
+
+def select_archive_patches(d,option):
+       '''select to archive all patches including non-applying and series

or applying patches '''

meta/classes/archiver.bbclass new file +13 -0

+       if option == "all":
+               patchdir = get_all_patches(d)
+       elif option == "applying":
+               patchdir = get_applying_patches(d)
+       try:
+               os.rmdir(patchdir)
+       except OSError:
+                       tarpatch = archive_patches(d,patchdir,option)
+                       return tarpatch
+       return
+
+def archive_logs(d,logdir,bbinc=False):
+       '''archive logs in temp to tarball and .bb and .inc files if bbinc

is True '''

meta/classes/archiver.bbclass new file +13 -0

+       import tarfile
+       import shutil
+
+       log_dir = os.path.basename(logdir)
+       pf = d.getVar('PF',True)
+       tarname = pf + '-' + log_dir + ".tar.gz"
+       tar = tarfile.open(tarname, "w:gz")
+       tar.add(log_dir)
+       tar.close()
+       if bbinc:
+               shutil.rmtree(log_dir, ignore_errors=True)
+       return tarname
+

meta/classes/archiver.bbclass new file +14 -0

+               clean_licenses = clean_licenses.replace('|','')
+       return clean_licenses
+
+
+def move_tarball_deploy(d,tarball_list):
+       '''move tarball in location to ${DEPLOY_DIR}/sources'''
+       import shutil
+
+       if tarball_list is []:
+               return
+       target_sys = d.getVar('TARGET_SYS', True)
+       pf = d.getVar('PF', True)
+       licenses = get_licenses(d)
+       tar_sources = d.getVar('DEPLOY_DIR', True) + '/sources/' +

target_sys + '/' + licenses + '/' + pf

meta/classes/archiver.bbclass new file +11 -0

+               bb.mkdirhier(tar_sources)
+       for source in tarball_list:
+               if source:
+                       if os.path.exists(tar_sources + '/' + source):
+                               os.remove(tar_sources + '/' + source)
+                       shutil.move(source,tar_sources)
+
+def verify_var(d):
+       '''check the type for archiving package('tar' or 'srpm')'''
+       try:
+               if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper()

not in d.getVar('ARCHIVE_TYPE', True).split():

meta/classes/archiver.bbclass new file +3 -0

+                       raise AttributeError
+       except AttributeError:
+                       bb.fatal("\"SOURCE_ARCHIVE_PACKAGE_TYPE\" is

\'tar\' or \'srpm\', no other types")

meta/classes/archiver.bbclass new file +24 -0

+
+def store_package(d,package_name):
+       '''store tarbablls name to file "tar-package"'''
+       try:
+               f = open(d.getVar('WORKDIR',True )+ '/tar-package','a')
+               f.write(package_name + ' ')
+               f.close()
+       except IOError:
+               pass
+
+def get_package(d):
+       '''get tarballs name from "tar-package"'''
+       try:
+               os.chdir(d.getVar('WORKDIR', True))
+               f = open('tar-package','r')
+               line = list(set(f.readline().replace('\n','').split()))
+               f.close()
+               return line
+       except IOError:
+               pass
+
+
+def archive_sources_patches(d,middle_name):
+       '''archive sources and patches to tarball. middle_name will append

strings ${middle_name} to ${PR} as middle name. for example, zlib-1.4.6-prepatch(middle_name).tar.gz '''

meta/classes/archiver.bbclass new file +8 -0

+       import shutil
+       verify_var(d)
+       if not_tarball(d):
+               return
+
+       source_tar_name = archive_sources(d,middle_name)
+       if middle_name == "prepatch":
+               if d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() ==

'TRUE':

meta/classes/archiver.bbclass new file +2 -0

+                       patch_tar_name = select_archive_patches(d,"all")
+               elif d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper()

== 'FALSE':
+ patch_tar_name =
select_archive_patches(d,"applying")

meta/classes/archiver.bbclass new file +2 -0

+               else:
+                       bb.fatal("Please define

'PATCHES_ARCHIVE_WITH_SERIES' is strings 'True' or 'False' ")

meta/classes/archiver.bbclass new file +4 -0

+       else:
+               patch_tar_name = ''
+
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in

'SRPM':

meta/classes/archiver.bbclass new file +11 -0

+               move_tarball_deploy(d,[source_tar_name,patch_tar_name])
+       else:
+               tarpackage = d.getVar('WORKDIR', True) + '/tar-package'
+               if os.path.exists(tarpackage):
+                       os.remove(tarpackage)
+               for package in source_tar_name, patch_tar_name:
+                       if package:
+                               store_package(d,str(package) + ' ')
+
+def archive_scripts_logs(d):
+       '''archive scripts and logs. scripts include .bb and .inc files

and logs include stuff in "temp".'''

meta/classes/archiver.bbclass new file +4 -0

+
+       work_dir = d.getVar('WORKDIR', True)
+       os.chdir(work_dir)
+       source_archive_log_with_scripts =

d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True)

meta/classes/archiver.bbclass new file +10 -0

+       if source_archive_log_with_scripts == 'logs_with_scripts':
+               logdir = get_bb_inc(d)
+               tarlog = archive_logs(d,logdir,True)
+       elif source_archive_log_with_scripts == 'logs':
+               if os.path.exists('temp'):
+                       tarlog = archive_logs(d,'temp',False)
+       else:
+               return
+
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in

'SRPM':

meta/classes/archiver.bbclass new file +7 -0

+               move_tarball_deploy(d,[tarlog])
+
+       else:
+               store_package(d,tarlog)
+
+def dumpdata(d):
+       '''dump environment to "${P}-${PR}.showdata.dump" including all

kinds of variables and functions when running a task'''

meta/classes/archiver.bbclass new file +12 -0

+       workdir = bb.data.getVar('WORKDIR', d, 1)
+       distro = bb.data.getVar('DISTRO', d, 1)
+       s = d.getVar('S', True)
+       pf = d.getVar('PF', True)
+       target_sys = d.getVar('TARGET_SYS', True)
+       licenses = get_licenses(d)
+       dumpdir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys
+ '/' + licenses + '/' + pf
+       if not os.path.exists(dumpdir):
+               bb.mkdirhier(dumpdir)
+
+       dumpfile = os.path.join(dumpdir,

bb.data.expand("${P}-${PR}.showdata.dump",d))

meta/classes/archiver.bbclass new file +9 -0

+
+       bb.note("Dumping metadata into '%s'" % dumpfile)
+       f = open(dumpfile, "w")
+       # emit variables and shell functions
+        bb.data.emit_env(f, d, True)
+       # emit the metadata which isnt valid shell
+       for e in d.keys():
+               if bb.data.getVarFlag(e, 'python', d):
+                       f.write("\npython %s () {\n%s}\n" % (e,

bb.data.getVar(e, d, 1)))

meta/classes/archiver.bbclass new file +4 -0

+       f.close()
+
+def create_diff_gz(d):
+       '''creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.g gz for

mapping all content in 's' including patches to xxx.diff.gz'''

meta/classes/archiver.bbclass new file +10 -0

+       import shutil
+
+       work_dir = d.getVar('WORKDIR', True)
+       exclude_from = d.getVar('EXCLUDE_FROM', True).split()
+       pf = d.getVar('PF', True)
+       licenses = get_licenses(d)
+       target_sys = d.getVar('TARGET_SYS', True)
+       diff_dir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys
+ '/' + licenses + '/' + pf
+       diff_file = os.path.join(diff_dir,

bb.data.expand("${P}-${PR}.diff.gz",d))

meta/classes/archiver.bbclass new file +17 -0

+       os.chdir(work_dir)
+       f = open('temp/exclude-from-file', 'a')
+       for i in exclude_from:
+               f.write(i)
+               f.write("\n")
+       f.close()
+
+       s=d.getVar('S', True)
+       distro = d.getVar('DISTRO',True)
+       dest = s + '/' + distro + '/files'
+       if not os.path.exists(dest):
+               bb.mkdirhier(dest)
+       for i in os.listdir(os.getcwd()):
+               if os.path.isfile(i):
+                       shutil.copy(i, dest)
+
+       bb.note("Creating .diff.gz in

${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz")
+ cmd = "LC_ALL=C TZ=UTC0 diff --exclude-from=" + work_dir +
"/temp/exclude-from-file -Naur " + s + '.org' + ' ' + s + " | gzip -c > "
+ diff_file

meta/classes/archiver.bbclass new file +6 -0

+       d.setVar('DIFF', cmd + "\n")
+       d.setVarFlag('DIFF', 'func', '1')
+       bb.build.exec_func('DIFF', d)
+       shutil.rmtree(s + '.org', ignore_errors=True)
+
+# This function will run when user want to get tarball for sources and

patches after do_unpack

meta/classes/archiver.bbclass new file +5 -0

+python do_archive_original_sources_patches(){
+       archive_sources_patches(d,'prepatch')
+}
+
+# This function will run when user want to get tarball for patched

sources after do_patch

meta/classes/archiver.bbclass new file +5 -0

+python do_archive_patched_sources(){
+       archive_sources_patches(d,'patched')
+}
+
+# This function will run when user want to get tarball for configured

sources after do_configure

meta/classes/archiver.bbclass new file +5 -0

+python do_archive_configured_sources(){
+       archive_sources_patches(d,'configured')
+}
+
+# This function will run when user want to get tarball for logs or both

logs and scripts(.bb and .inc files)

meta/classes/archiver.bbclass new file +5 -0

+python do_archive_scripts_logs(){
+       archive_scripts_logs(d)
+}
+
+# This function will run when user want to know what variable and

functions in a running task are and also can get a diff file including

meta/classes/archiver.bbclass new file +7 -0

+# all content a package should include.
+python do_dumpdata_create_diff_gz(){
+       dumpdata(d)
+       create_diff_gz(d)
+}
+
+# This functions prepare for archiving "linux-yocto" because this package

create directory 's' before do_patch instead of after do_unpack.

meta/classes/archiver.bbclass new file +6 -0

+# This is special control for archiving linux-yocto only.
+python do_archive_linux_yocto(){
+       s = d.getVar('S', True)
+       if 'linux-yocto' in s:
+               source_tar_name = archive_sources(d,'')
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in

'SRPM':

meta/classes/archiver.bbclass new file +21 -1

+               move_tarball_deploy(d,[source_tar_name,''])
+}
+do_kernel_checkout[postfuncs] += "do_archive_linux_yocto "
+
+# remove tarball for sources, patches and logs after creating srpm.
+python do_remove_tarball(){
+       if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+               work_dir = d.getVar('WORKDIR', True)
+               os.chdir(work_dir)
+               try:
+                       for file in os.listdir(os.getcwd()):
+                               if file in get_package(d):
+                                       os.remove(file)
+                       os.remove('tar-package')
+               except (TypeError,OSError):
+                               pass
+}
+do_remove_taball[deptask] = "do_archive_scripts_logs"
+do_package_write_rpm[postfuncs] += "do_remove_tarball "
+export get_licenses
+export get_package
--

1.7.0.4

_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core

From: Xiaofeng Yan <[email protected]>

User can use these variables to get atchiving packages they want.

Signed-off-by Xiaofeng Yan <[email protected]>

---
 meta-yocto/conf/local.conf.sample.extended |   32 ++++++++++++++++++++++++++++

 1 files changed, 32 insertions(+), 0 deletions(-)

meta-yocto/conf/local.conf.sample.extended +32 -1

index 9c5b974..731b7d6 100644
@@ -182,3 +182,35 @@
  # when the disk space reduces 50M (or the amount of inode reduces 5k).
  #BB_DISKMON_WARNINTERVAL = "50M,5K"
 
+# Archiving packages configuration
+#
+# The following variables lists which files to archive and the package type to archive.
+# One bbclass of them (archive-original-source.bbclass,archive-patched-source.bbclass
+# and archive-configured-source.bbclass) should be inherited in a global bbclass, for example,
+# intherit archive-original-source in packages_rpm.bbclass
+#
+# SOURCE_ARCHIVE_PACKAGE_TYPE = {'tar','srpm'}
+#SOURCE_ARCHIVE_PACKAGE_TYPE ?= 'tar'
+#
+# SOURCE_ARCHIVE_LOG_WITH_SCRIPTS = {'logs_with_scripts', 'logs'}
+# String 'logs_with_scripts' include temp directory and .bb and .inc files
+# String 'logs' only include temp
+#SOURCE_ARCHIVE_LOG_WITH_SCRIPTS ?= 'logs_with_scripts'
+
+#
+# Archiving packages configuration
+#
+# The following variables lists which files to archive and the package type to archive.
+# One bbclass of them (archive-original-source.bbclass,archive-patched-source.bbclass
+# and archive-configured-source.bbclass) should be inherited in a globale bbclass, for example,
+# intherit archive-original-source in packages_rpm.bbclass
+#
+# SOURCE_ARCHIVE_PACKAGE_TYPE = {'tar','srpm'}
+#SOURCE_ARCHIVE_PACKAGE_TYPE ?= 'tar'
+#
+# SOURCE_ARCHIVE_LOG_WITH_SCRIPTS = {'logs_with_scripts', 'logs'}
+# String 'logs_with_scripts' include temp directory and .bb and .inc files
+# String 'logs' only include temp
+#SOURCE_ARCHIVE_LOG_WITH_SCRIPTS ?= 'logs_with_scripts'
+#
+
--

1.7.0.4

From: Xiaofeng Yan <[email protected]>

User can use these variables to get atchiving packages they want.

Signed-off-by Xiaofeng Yan <[email protected]>

---
 meta-yocto/conf/local.conf.sample.extended |   32 ++++++++++++++++++++++++++++
 1 files changed, 32 insertions(+), 0 deletions(-)