Realize archiving functions

From: Xiaofeng Yan<[email protected]>

1 Archive sources in ${S} in the different stage (do_unpack,do_patch,do_configure). 2 Archive patches including series
3 Archive logs including scripts (.bb and .inc files)
4 dump environment resources which show all variable and
functions used to xxx.showdata.dump when running a task
5 dump all content in 's' including patches to file xxx.diff.gz

All archiving packages will be deployed to ${DEPLOY_DIR}/sources/

[#YOCTO 1977]

Signed-off-by Xiaofeng Yan<[email protected]>

---
  meta/classes/archiver.bbclass |  419 +++++++++++++++++++++++++++++++++++++++++
  1 files changed, 419 insertions(+), 0 deletions(-)

create mode 100644 meta/classes/archiver.bbclass

meta/classes/archiver.bbclass new file +356 -0

new file mode 100644
index 0000000..de90c75
@@ -0,0 +1,419 @@
+# This file is used for archiving sources ,patches,and logs to tarball.
+# It also output building environment to xxx.dump.data and create xxx.diff.gz to record
+# all content in ${S} to a diff file.
+
+EXCLUDE_FROM ?= ".pc autom4te.cache"
+ARCHIVE_TYPE ?= "TAR SRPM"
+DISTRO ?= "poky"
+PATCHES_ARCHIVE_WITH_SERIES = 'TRUE'
+
+def parse_var(d,var):
+	''' parse variable like ${PV}  in "require xxx_${PV}.inc"  to a real value. for example, change "require xxx_${PV}.inc" to "require xxx_1.2.inc" '''
+	import re
+	pat = re.compile('.*\$({(.*)}).*')
+	if '$' not in var and '/' not in var:
+		return var
+	else:
+		if '/' in var:
+			return [i for i in var.split('/') if i.endswith('.inc')][0]
+		elif '$' in  var:
+			m = pat.match(var)
+			patstr = '\$' + m.group(1)
+			var_str = m.group(2)
+			return re.sub(patstr,d.getVar(var_str,True),var)
+		else:
+			return var
+		
+def get_bb_inc(d):
+	'''create a directory "script-logs" including .bb and .inc file in ${WORKDIR}'''
+	import re
+	import os
+	import shutil
+	
+	bbinc = []
+	pat=re.compile('require\s*([^\s]*\.*)(.*)')
+	file_dir = d.getVar('FILE', True)
+	bbdir = os.path.dirname(file_dir)
+	work_dir = d.getVar('WORKDIR', True)
+	os.chdir(work_dir)
+	bb.mkdirhier("script-logs")
+	os.chdir(bbdir)
+	bbfile = os.path.basename(file_dir)
+	bbinc.append(bbfile)
+	
+	def get_inc (file):
+		f = open(file,'r')
+		for line in f.readlines():
+			if 'require' not  in line:
+				bbinc.append(file)
+			else:
+				try:
+					incfile = pat.match(line).group(1)
+					incfile = parse_var(d,incfile)
+					bbinc.append(incfile)
+					get_inc(incfile)
+				except (IOError,AttributeError):
+					pass
+	get_inc(bbfile)
+	os.chdir(work_dir)
+	for root, dirs, files in os.walk(bbdir):
+		for file in bbinc:
+			if file in files:
+				shutil.copy(root + '/' + file,'script-logs')
+	oe.path.copytree('temp', 'script-logs')
+	return work_dir + '/script-logs'
+
+def get_all_patches(d):
+	'''copy patches and series file to a pointed directory which will be archived to tarball in ${WORKDIR}'''
+	import shutil
+	
+	src_patches=[]
+	pf = d.getVar('PF', True)
+	work_dir = d.getVar('WORKDIR', True)
+	dest = os.path.join(work_dir, pf + '-patches')
+	shutil.rmtree(dest, ignore_errors=True)
+	bb.mkdirhier(dest)
+	
+	src_uri = d.getVar('SRC_URI', 1).split()
+	fetch = bb.fetch2.Fetch(src_uri, d)
+	locals = (fetch.localpath(url) for url in fetch.urls)
+	for local in locals:
+		src_patches.append(local)
+	for patch in src_patches[1:]:
+		 shutil.copy(patch,dest)
+	return dest
+
+def get_applying_patches(d):
+	"""only copy applying patches to a pointed directory which will be archived to tarball"""
+	import os
+	import shutil
+
+
+	pf = d.getVar('PF', True)
+	work_dir = d.getVar('WORKDIR', True)
+	dest = os.path.join(work_dir, pf + '-patches')
+	shutil.rmtree(dest, ignore_errors=True)
+	bb.mkdirhier(dest)
+
+
+	patches = src_patches(d)
+	for patch in patches:
+		_, _, local, _, _, parm = bb.decodeurl(patch)
+		if local:
+			 shutil.copy(local,dest)
+	return dest
+
+def not_tarball(d):
+	'''packages including key words 'work-shared','native', 'task-' will be passed'''
+	import os
+
+	workdir = d.getVar('WORKDIR',True)
+	s = d.getVar('S',True)
+	if 'work-shared' in s or 'task-' in workdir or 'native' in workdir:
+		return True
+	else:
+		return False
+
+def archive_sources(d,middle_name):
+	'''archive sources codes tree to tarball'''
+	import tarfile
+	import shutil
+	
+	s = d.getVar('S',True)
+	workdir=d.getVar('WORKDIR', True)
+	PF = d.getVar('PF',True)
+	tarname = PF + '-' + middle_name + ".tar.gz"
+
+	if os.path.exists(s) and s is not workdir:
+		sourcedir = os.path.basename(s)
+		tarbase = os.path.dirname(s)
+		if not sourcedir or os.path.dirname(tarbase) == workdir:
+			sourcedir = os.path.basename(os.path.dirname(s))
+			tarbase = os.path.dirname(os.path.dirname(s))
+		os.chdir(tarbase)
+	else:
+		sourcedir = os.path.basename(s)
+		if not os.path.exists(sourcedir):
+			os.mkdir(sourcedir)
+		try:
+			for file in os.listdir(s):
+				if file is not 'temp' and file is not sourcedir:
+					shutil.copy(file,sourcedir)
+		except (IOError,OSError):
+			pass
+
+	if (len(os.listdir(sourcedir))) != 0:
+		tar = tarfile.open( tarname, "w:gz")
+		tar.add(sourcedir)
+		tar.close()
+		if cmp(workdir,os.path.dirname(s)) and not os.path.exists(workdir + '/' + tarname):
+			shutil.move(os.path.dirname(s) + '/' + tarname,workdir)
+	else:
+		return
+	return tarname
+
+def archive_patches(d,patchdir,series):
+	'''archive patches to tarball and also include series files if 'series' is True'''
+	import tarfile
+	import shutil
+
+	s = d.getVar('S',True)
+	work_dir = d.getVar('WORKDIR', True)
+	os.chdir(work_dir)
+	patch_dir = os.path.basename(patchdir)
+	tarname = patch_dir + ".tar.gz"
+	if series  == 'all' and os.path.exists(s + '/patches/series'):
+			shutil.copy(s + '/patches/series',patch_dir)
+	tar = tarfile.open(tarname, "w:gz")
+	tar.add(patch_dir)
+	tar.close()
+	shutil.rmtree(patch_dir, ignore_errors=True)
+	return tarname
+
+def select_archive_patches(d,option):
+	'''select to archive all patches including non-applying and series or applying patches '''
+	if option == "all":
+		patchdir = get_all_patches(d)
+	elif option == "applying":
+		patchdir = get_applying_patches(d)
+	try:
+		os.rmdir(patchdir)
+	except OSError:
+			tarpatch = archive_patches(d,patchdir,option)
+			return tarpatch
+	return
+
+def archive_logs(d,logdir,bbinc=False):
+	'''archive logs in temp to tarball and .bb and .inc files if bbinc is True '''
+	import tarfile
+	import shutil
+
+	log_dir = os.path.basename(logdir)
+	pf = d.getVar('PF',True)
+	tarname = pf + '-' + log_dir + ".tar.gz"
+	tar = tarfile.open(tarname, "w:gz")
+	tar.add(log_dir)
+	tar.close()
+	if bbinc:
+		shutil.rmtree(log_dir, ignore_errors=True)
+	return tarname
+
+def get_licenses(d):
+	'''get licenses for running .bb file'''
+	licenses = d.getVar('LICENSE', 1).replace('&', '|')
+	licenses = licenses.replace('(', '').replace(')', '')
+	clean_licenses = ""
+	for x in licenses.split():
+		if x.strip() == '' or x == 'CLOSED':
+			continue
+		if x != "|":
+			clean_licenses += x
+	if '|' in clean_licenses:
+		clean_licenses = clean_licenses.replace('|','')
+	return clean_licenses
+	
+
+def move_tarball_deploy(d,tarball_list):
+	'''move tarball in location to ${DEPLOY_DIR}/sources'''
+	import shutil
+	
+	if tarball_list is []:
+		return
+	target_sys = d.getVar('TARGET_SYS', True)
+	pf = d.getVar('PF', True)
+	licenses = get_licenses(d)
+	tar_sources = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+	if not os.path.exists(tar_sources):
+		bb.mkdirhier(tar_sources)
+	for source in tarball_list:
+		if source:
+			if os.path.exists(tar_sources + '/' + source):
+				os.remove(tar_sources + '/' + source)
+			shutil.move(source,tar_sources)
+
+def verify_var(d):
+	'''check the type for archiving package('tar' or 'srpm')'''
+	try:
+		if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in d.getVar('ARCHIVE_TYPE', True).split():
+			raise AttributeError	
+	except AttributeError:
+			bb.fatal("\"SOURCE_ARCHIVE_PACKAGE_TYPE\" is \'tar\' or \'srpm\', no other types")
+
+def store_package(d,package_name):
+	'''store tarbablls name to file "tar-package"'''
+	f = open(d.getVar('WORKDIR',True )+ '/tar-package','a')
+	f.write(package_name + ' ')
+	f.close()
+
+def get_package(d):
+	'''get tarballs name from "tar-package"'''
+	os.chdir(d.getVar('WORKDIR', True))
+	f = open('tar-package','r')
+	line = list(set(f.readline().replace('\n','').split()))
+	f.close()
+	return line
+
+
+def archive_sources_patches(d,middle_name):
+	'''archive sources and patches to tarball. middle_name will append strings ${middle_name} to ${PR} as middle name. for example, zlib-1.4.6-prepatch(middle_name).tar.gz '''
+	import shutil
+	verify_var(d)	
+	if not_tarball(d):
+		return
+	
+	source_tar_name = archive_sources(d,middle_name)
+	if middle_name == "prepatch":
+		if d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'TRUE':
+			patch_tar_name = select_archive_patches(d,"all")
+		elif d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'FALSE':
+			patch_tar_name = select_archive_patches(d,"applying")
+		else:
+			bb.fatal("Please define 'PATCHES_ARCHIVE_WITH_SERIES' is strings 'True' or 'False' ")
+	else:
+		patch_tar_name = ''
+
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+		move_tarball_deploy(d,[source_tar_name,patch_tar_name])
+	else:
+		tarpackage = d.getVar('WORKDIR', True) + '/tar-package'
+		if os.path.exists(tarpackage):
+			os.remove(tarpackage)
+		for package in source_tar_name, patch_tar_name:
+			if package:
+				store_package(d,str(package) + ' ')
+
+def archive_scripts_logs(d):
+	'''archive scripts and logs. scripts include .bb and .inc files and logs include stuff in "temp".'''
+
+	work_dir = d.getVar('WORKDIR', True)
+	os.chdir(work_dir)
+	source_archive_log_with_scripts = d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True)
+	if source_archive_log_with_scripts == 'logs_with_scripts':
+		logdir = get_bb_inc(d)
+		tarlog = archive_logs(d,logdir,True)
+	elif source_archive_log_with_scripts == 'logs':
+		if os.path.exists('temp'):
+			tarlog = archive_logs(d,'temp',False)
+	else:
+		return
+		
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+		move_tarball_deploy(d,[tarlog])
+
+	else:
+		store_package(d,tarlog)
+
+def dumpdata(d):
+	'''dump environment to "${P}-${PR}.showdata.dump" including all kinds of variables and functions when running a task'''
+	workdir = bb.data.getVar('WORKDIR', d, 1)
+	distro = bb.data.getVar('DISTRO', d, 1)
+	s = d.getVar('S', True)
+	pf = d.getVar('PF', True)
+	target_sys = d.getVar('TARGET_SYS', True)
+	licenses = get_licenses(d)
+	dumpdir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+	if not os.path.exists(dumpdir):
+		bb.mkdirhier(dumpdir)
+
+	dumpfile = os.path.join(dumpdir, bb.data.expand("${P}-${PR}.showdata.dump",d))
+
+	bb.note("Dumping metadata into '%s'" % dumpfile)
+	f = open(dumpfile, "w")
+	# emit variables and shell functions
+        bb.data.emit_env(f, d, True)
+	# emit the metadata which isnt valid shell
+	for e in d.keys():
+		if bb.data.getVarFlag(e, 'python', d):
+			f.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, 1)))
+	f.close()
+
+def create_diff_gz(d):
+	'''creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.g gz for mapping all content in 's' including patches to  xxx.diff.gz'''
+	import shutil
+
+	work_dir = d.getVar('WORKDIR', True)
+	exclude_from = d.getVar('EXCLUDE_FROM', True).split()
+	pf = d.getVar('PF', True)
+	licenses = get_licenses(d)
+	target_sys = d.getVar('TARGET_SYS', True)
+	diff_dir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+	diff_file = os.path.join(diff_dir, bb.data.expand("${P}-${PR}.diff.gz",d))
+	os.chdir(work_dir)
+	f = open('temp/exclude-from-file', 'a')
+	for i in exclude_from:
+		f.write(i)
+		f.write("\n")
+	f.close()
+
+	s=d.getVar('S', True)
+	distro = d.getVar('DISTRO',True)
+	dest = s + '/' + distro + '/files'
+	if not os.path.exists(dest):
+		bb.mkdirhier(dest)
+	for i in os.listdir(os.getcwd()):
+		if os.path.isfile(i):
+			shutil.copy(i, dest)
+	

meta/classes/archiver.bbclass new file +53 -0

+}
+
+# This function will run when user want to get tarball for patched sources after do_patch
+python do_archive_patched_sources(){
+	archive_sources_patches(d,'patched')
+}
+
+# This function will run when user want to get tarball for configured sources after do_configure
+python do_archive_configured_sources(){
+	archive_sources_patches(d,'configured')
+}
+
+# This function will run when user want to get tarball for logs or both logs and scripts(.bb and .inc files)
+python do_archive_scripts_logs(){
+	archive_scripts_logs(d)
+}
+
+# This function will run when user want to know what variable and functions in a running task are and also can get a diff file including
+# all content a package should include.
+python do_dumpdata_create_diff_gz(){
+	dumpdata(d)
+	create_diff_gz(d)
+}
+
+# This functions prepare for archiving "linux-yocto" because this package create directory 's' before do_patch instead of after do_unpack.
+# This is special control for archiving linux-yocto only.
+python do_archive_linux_yocto(){
+	s = d.getVar('S', True)
+	if 'linux-yocto' in s:
+		source_tar_name = archive_sources(d,'')
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+		move_tarball_deploy(d,[source_tar_name,''])
+}
+do_kernel_checkout[postfuncs] += "do_archive_linux_yocto "
+
+# remove tarball for sources, patches and logs after creating srpm.
+python do_remove_tarball(){
+	if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+		work_dir = d.getVar('WORKDIR', True)
+		os.chdir(work_dir)
+		for file in os.listdir(os.getcwd()):
+			if file in get_package(d):
+				try:
+					os.remove(file)
+				except OSError:
+					pass
+		if os.path.exists('tar-package'):
+			os.remove('tar-package')
+}
+do_remove_taball[deptask] = "do_archive_scripts_logs"
+do_package_write_rpm[postfuncs] += "do_remove_tarball "
+export get_licenses
+export get_package

Quoted 413 lines · Xiaofeng Yan

From: Xiaofeng Yan<[email protected]>

1 Archive sources in ${S} in the different stage (do_unpack,do_patch,do_configure). 2 Archive patches including series
3 Archive logs including scripts (.bb and .inc files)
4 dump environment resources which show all variable and
functions used to xxx.showdata.dump when running a task
5 dump all content in 's' including patches to file xxx.diff.gz

All archiving packages will be deployed to ${DEPLOY_DIR}/sources/

[#YOCTO 1977]

Signed-off-by Xiaofeng Yan<[email protected]>

---
meta/classes/archiver.bbclass | 419
+++++++++++++++++++++++++++++++++++++++++

1 files changed, 419 insertions(+), 0 deletions(-)

create mode 100644 meta/classes/archiver.bbclass

meta/classes/archiver.bbclass new file +2 -0

new file mode 100644
index 0000000..de90c75
@@ -0,0 +1,419 @@
+# This file is used for archiving sources ,patches,and logs to tarball.
+# It also output building environment to xxx.dump.data and create 

xxx.diff.gz to record

meta/classes/archiver.bbclass new file +9 -0

+# all content in ${S} to a diff file.
+
+EXCLUDE_FROM ?= ".pc autom4te.cache"
+ARCHIVE_TYPE ?= "TAR SRPM"
+DISTRO ?= "poky"
+PATCHES_ARCHIVE_WITH_SERIES = 'TRUE'
+
+def parse_var(d,var):
+ ''' parse variable like ${PV} in "require xxx_${PV}.inc" to a real 

value. for example, change "require xxx_${PV}.inc" to "require xxx_1.2.inc" '''
+ import re
+ pat = re.compile('.*\$({(.*)}).*')
+ if '$' not in var and '/' not in var:
+ return var
+ else:
+ if '/' in var:
+ return [i for i in var.split('/') if i.endswith('.inc')][0]
+ elif '$' in var:
+ m = pat.match(var)
+ patstr = '\$' + m.group(1)
+ var_str = m.group(2)
+ return re.sub(patstr,d.getVar(var_str,True),var)
+ else:
+ return var

meta/classes/archiver.bbclass new file +3 -0

+
+def get_bb_inc(d):
+ '''create a directory "script-logs" including .bb and .inc file in 

${WORKDIR}'''
+ import re
+ import os
+ import shutil

meta/classes/archiver.bbclass new file +36 -0

+
+ bbinc = []
+ pat=re.compile('require\s*([^\s]*\.*)(.*)')
+ file_dir = d.getVar('FILE', True)
+ bbdir = os.path.dirname(file_dir)
+ work_dir = d.getVar('WORKDIR', True)
+ os.chdir(work_dir)
+ bb.mkdirhier("script-logs")
+ os.chdir(bbdir)
+ bbfile = os.path.basename(file_dir)
+ bbinc.append(bbfile)
+
+ def get_inc (file):
+ f = open(file,'r')
+ for line in f.readlines():
+ if 'require' not in line:
+ bbinc.append(file)
+ else:
+ try:
+ incfile = pat.match(line).group(1)
+ incfile = parse_var(d,incfile)
+ bbinc.append(incfile)
+ get_inc(incfile)
+ except (IOError,AttributeError):
+ pass
+ get_inc(bbfile)
+ os.chdir(work_dir)
+ for root, dirs, files in os.walk(bbdir):
+ for file in bbinc:
+ if file in files:
+ shutil.copy(root + '/' + file,'script-logs')
+ oe.path.copytree('temp', 'script-logs')
+ return work_dir + '/script-logs'
+
+def get_all_patches(d):
+ '''copy patches and series file to a pointed directory which will 

be archived to tarball in ${WORKDIR}'''
+ import shutil

meta/classes/archiver.bbclass new file +19 -0

+
+ src_patches=[]
+ pf = d.getVar('PF', True)
+ work_dir = d.getVar('WORKDIR', True)
+ dest = os.path.join(work_dir, pf + '-patches')
+ shutil.rmtree(dest, ignore_errors=True)
+ bb.mkdirhier(dest)
+
+ src_uri = d.getVar('SRC_URI', 1).split()
+ fetch = bb.fetch2.Fetch(src_uri, d)
+ locals = (fetch.localpath(url) for url in fetch.urls)
+ for local in locals:
+ src_patches.append(local)
+ for patch in src_patches[1:]:
+ shutil.copy(patch,dest)
+ return dest
+
+def get_applying_patches(d):
+ """only copy applying patches to a pointed directory which will be 

archived to tarball"""
+ import os
+ import shutil

meta/classes/archiver.bbclass new file +18 -0

+
+
+ pf = d.getVar('PF', True)
+ work_dir = d.getVar('WORKDIR', True)
+ dest = os.path.join(work_dir, pf + '-patches')
+ shutil.rmtree(dest, ignore_errors=True)
+ bb.mkdirhier(dest)
+
+
+ patches = src_patches(d)
+ for patch in patches:
+ _, _, local, _, _, parm = bb.decodeurl(patch)
+ if local:
+ shutil.copy(local,dest)
+ return dest
+
+def not_tarball(d):
+ '''packages including key words 'work-shared','native', 'task-' 

will be passed'''
+ import os

meta/classes/archiver.bbclass new file +41 -0

+
+ workdir = d.getVar('WORKDIR',True)
+ s = d.getVar('S',True)
+ if 'work-shared' in s or 'task-' in workdir or 'native' in workdir:
+ return True
+ else:
+ return False
+
+def archive_sources(d,middle_name):
+ '''archive sources codes tree to tarball'''
+ import tarfile
+ import shutil
+
+ s = d.getVar('S',True)
+ workdir=d.getVar('WORKDIR', True)
+ PF = d.getVar('PF',True)
+ tarname = PF + '-' + middle_name + ".tar.gz"
+
+ if os.path.exists(s) and s is not workdir:
+ sourcedir = os.path.basename(s)
+ tarbase = os.path.dirname(s)
+ if not sourcedir or os.path.dirname(tarbase) == workdir:
+ sourcedir = os.path.basename(os.path.dirname(s))
+ tarbase = os.path.dirname(os.path.dirname(s))
+ os.chdir(tarbase)
+ else:
+ sourcedir = os.path.basename(s)
+ if not os.path.exists(sourcedir):
+ os.mkdir(sourcedir)
+ try:
+ for file in os.listdir(s):
+ if file is not 'temp' and file is not sourcedir:
+ shutil.copy(file,sourcedir)
+ except (IOError,OSError):
+ pass
+
+ if (len(os.listdir(sourcedir))) != 0:
+ tar = tarfile.open( tarname, "w:gz")
+ tar.add(sourcedir)
+ tar.close()
+ if cmp(workdir,os.path.dirname(s)) and not os.path.exists(workdir + 

'/' + tarname):
+ shutil.move(os.path.dirname(s) + '/' + tarname,workdir)
+ else:
+ return
+ return tarname

meta/classes/archiver.bbclass new file +3 -0

+
+def archive_patches(d,patchdir,series):
+ '''archive patches to tarball and also include series files if 

'series' is True'''
+ import tarfile
+ import shutil

meta/classes/archiver.bbclass new file +16 -0

+
+ s = d.getVar('S',True)
+ work_dir = d.getVar('WORKDIR', True)
+ os.chdir(work_dir)
+ patch_dir = os.path.basename(patchdir)
+ tarname = patch_dir + ".tar.gz"
+ if series == 'all' and os.path.exists(s + '/patches/series'):
+ shutil.copy(s + '/patches/series',patch_dir)
+ tar = tarfile.open(tarname, "w:gz")
+ tar.add(patch_dir)
+ tar.close()
+ shutil.rmtree(patch_dir, ignore_errors=True)
+ return tarname
+
+def select_archive_patches(d,option):
+ '''select to archive all patches including non-applying and series 

or applying patches '''
+ if option == "all":
+ patchdir = get_all_patches(d)
+ elif option == "applying":
+ patchdir = get_applying_patches(d)
+ try:
+ os.rmdir(patchdir)
+ except OSError:
+ tarpatch = archive_patches(d,patchdir,option)
+ return tarpatch
+ return

meta/classes/archiver.bbclass new file +3 -0

+
+def archive_logs(d,logdir,bbinc=False):
+ '''archive logs in temp to tarball and .bb and .inc files if bbinc 

is True '''
+ import tarfile
+ import shutil

meta/classes/archiver.bbclass new file +36 -0

+
+ log_dir = os.path.basename(logdir)
+ pf = d.getVar('PF',True)
+ tarname = pf + '-' + log_dir + ".tar.gz"
+ tar = tarfile.open(tarname, "w:gz")
+ tar.add(log_dir)
+ tar.close()
+ if bbinc:
+ shutil.rmtree(log_dir, ignore_errors=True)
+ return tarname
+
+def get_licenses(d):
+ '''get licenses for running .bb file'''
+ licenses = d.getVar('LICENSE', 1).replace('&', '|')
+ licenses = licenses.replace('(', '').replace(')', '')
+ clean_licenses = ""
+ for x in licenses.split():
+ if x.strip() == '' or x == 'CLOSED':
+ continue
+ if x != "|":
+ clean_licenses += x
+ if '|' in clean_licenses:
+ clean_licenses = clean_licenses.replace('|','')
+ return clean_licenses
+
+
+def move_tarball_deploy(d,tarball_list):
+ '''move tarball in location to ${DEPLOY_DIR}/sources'''
+ import shutil
+
+ if tarball_list is []:
+ return
+ target_sys = d.getVar('TARGET_SYS', True)
+ pf = d.getVar('PF', True)
+ licenses = get_licenses(d)
+ tar_sources = d.getVar('DEPLOY_DIR', True) + '/sources/' + 

target_sys + '/' + licenses + '/' + pf
+ if not os.path.exists(tar_sources):
+ bb.mkdirhier(tar_sources)
+ for source in tarball_list:
+ if source:
+ if os.path.exists(tar_sources + '/' + source):
+ os.remove(tar_sources + '/' + source)
+ shutil.move(source,tar_sources)

meta/classes/archiver.bbclass new file +5 -0

+
+def verify_var(d):
+ '''check the type for archiving package('tar' or 'srpm')'''
+ try:
+ if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 

d.getVar('ARCHIVE_TYPE', True).split():
+ raise AttributeError
+ except AttributeError:
+ bb.fatal("\"SOURCE_ARCHIVE_PACKAGE_TYPE\" is \'tar\' or \'srpm\', no other types")

meta/classes/archiver.bbclass new file +18 -0

+
+def store_package(d,package_name):
+ '''store tarbablls name to file "tar-package"'''
+ f = open(d.getVar('WORKDIR',True )+ '/tar-package','a')
+ f.write(package_name + ' ')
+ f.close()
+
+def get_package(d):
+ '''get tarballs name from "tar-package"'''
+ os.chdir(d.getVar('WORKDIR', True))
+ f = open('tar-package','r')
+ line = list(set(f.readline().replace('\n','').split()))
+ f.close()
+ return line
+
+
+def archive_sources_patches(d,middle_name):
+ '''archive sources and patches to tarball. middle_name will append 

strings ${middle_name} to ${PR} as middle name. for example, zlib-1.4.6-prepatch(middle_name).tar.gz '''
+ import shutil
+ verify_var(d)
+ if not_tarball(d):
+ return
+
+ source_tar_name = archive_sources(d,middle_name)
+ if middle_name == "prepatch":
+ if d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'TRUE':
+ patch_tar_name = select_archive_patches(d,"all")
+ elif d.getVar('PATCHES_ARCHIVE_WITH_SERIES',True).upper() == 'FALSE':
+ patch_tar_name = select_archive_patches(d,"applying")
+ else:
+ bb.fatal("Please define 'PATCHES_ARCHIVE_WITH_SERIES' is strings
'True' or 'False' ")
+ else:
+ patch_tar_name = ''
+
+ if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in
'SRPM':
+ move_tarball_deploy(d,[source_tar_name,patch_tar_name])
+ else:
+ tarpackage = d.getVar('WORKDIR', True) + '/tar-package'
+ if os.path.exists(tarpackage):
+ os.remove(tarpackage)
+ for package in source_tar_name, patch_tar_name:
+ if package:
+ store_package(d,str(package) + ' ')

meta/classes/archiver.bbclass new file +3 -0

+
+def archive_scripts_logs(d):
+ '''archive scripts and logs. scripts include .bb and .inc files and 

logs include stuff in "temp".'''
+
+ work_dir = d.getVar('WORKDIR', True)
+ os.chdir(work_dir)
+ source_archive_log_with_scripts = d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True)
+ if source_archive_log_with_scripts == 'logs_with_scripts':
+ logdir = get_bb_inc(d)
+ tarlog = archive_logs(d,logdir,True)
+ elif source_archive_log_with_scripts == 'logs':
+ if os.path.exists('temp'):
+ tarlog = archive_logs(d,'temp',False)
+ else:
+ return
+
+ if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 'SRPM':
+ move_tarball_deploy(d,[tarlog])

meta/classes/archiver.bbclass new file +6 -0

+
+ else:
+ store_package(d,tarlog)
+
+def dumpdata(d):
+ '''dump environment to "${P}-${PR}.showdata.dump" including all 

kinds of variables and functions when running a task'''
+ workdir = bb.data.getVar('WORKDIR', d, 1)
+ distro = bb.data.getVar('DISTRO', d, 1)
+ s = d.getVar('S', True)
+ pf = d.getVar('PF', True)
+ target_sys = d.getVar('TARGET_SYS', True)
+ licenses = get_licenses(d)
+ dumpdir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+ if not os.path.exists(dumpdir):
+ bb.mkdirhier(dumpdir)
+
+ dumpfile = os.path.join(dumpdir, bb.data.expand("${P}-${PR}.showdata.dump",d))

meta/classes/archiver.bbclass new file +13 -0

+
+ bb.note("Dumping metadata into '%s'" % dumpfile)
+ f = open(dumpfile, "w")
+ # emit variables and shell functions
+ bb.data.emit_env(f, d, True)
+ # emit the metadata which isnt valid shell
+ for e in d.keys():
+ if bb.data.getVarFlag(e, 'python', d):
+ f.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, 1)))
+ f.close()
+
+def create_diff_gz(d):
+ '''creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.g gz for 

mapping all content in 's' including patches to xxx.diff.gz'''
+ import shutil
+
+ work_dir = d.getVar('WORKDIR', True)
+ exclude_from = d.getVar('EXCLUDE_FROM', True).split()
+ pf = d.getVar('PF', True)
+ licenses = get_licenses(d)
+ target_sys = d.getVar('TARGET_SYS', True)
+ diff_dir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys
+ '/' + licenses + '/' + pf
+ diff_file = os.path.join(diff_dir, bb.data.expand("${P}-${PR}.diff.gz",d))
+ os.chdir(work_dir)
+ f = open('temp/exclude-from-file', 'a')
+ for i in exclude_from:
+ f.write(i)
+ f.write("\n")
+ f.close()

meta/classes/archiver.bbclass new file +12 -0

+
+ s=d.getVar('S', True)
+ distro = d.getVar('DISTRO',True)
+ dest = s + '/' + distro + '/files'
+ if not os.path.exists(dest):
+ bb.mkdirhier(dest)
+ for i in os.listdir(os.getcwd()):
+ if os.path.isfile(i):
+ shutil.copy(i, dest)
+
+ bb.note("Creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz")
+ cmd = "LC_ALL=C TZ=UTC0 diff --exclude-from=" + work_dir + 

"/temp/exclude-from-file -Naur " + s + '.org' + ' ' + s + " | gzip -c> " + diff_file
+ d.setVar('DIFF', cmd + "\n")
+ d.setVarFlag('DIFF', 'func', '1')

Quoted 7 lines

+ bb.build.exec_func('DIFF', d)
+ shutil.rmtree(s + '.org', ignore_errors=True)

meta/classes/archiver.bbclass new file +2 -0

+
+# This function will run when user want to get tarball for sources 

and patches after do_unpack
+python do_archive_original_sources_patches(){
+ archive_sources_patches(d,'prepatch')

Why do you rebuild the tarball here are rename it instead of just copying the original tarball?

I could understand this for SCM based packages, and even then we have tarballs created in the DL_DIR.

I think I understand why you name it _prepatch, since it's not the exact original tarball.

Sau!

Quoted 59 lines

meta/classes/archiver.bbclass new file +3 -0

+}
+
+# This function will run when user want to get tarball for patched 

sources after do_patch

meta/classes/archiver.bbclass new file +5 -0

+python do_archive_patched_sources(){
+ archive_sources_patches(d,'patched')
+}
+
+# This function will run when user want to get tarball for 

configured sources after do_configure

meta/classes/archiver.bbclass new file +5 -0

+python do_archive_configured_sources(){
+ archive_sources_patches(d,'configured')
+}
+
+# This function will run when user want to get tarball for logs or 

both logs and scripts(.bb and .inc files)

meta/classes/archiver.bbclass new file +5 -0

+python do_archive_scripts_logs(){
+ archive_scripts_logs(d)
+}
+
+# This function will run when user want to know what variable and 

functions in a running task are and also can get a diff file including

meta/classes/archiver.bbclass new file +7 -0

+# all content a package should include.
+python do_dumpdata_create_diff_gz(){
+ dumpdata(d)
+ create_diff_gz(d)
+}
+
+# This functions prepare for archiving "linux-yocto" because this 

package create directory 's' before do_patch instead of after do_unpack.

meta/classes/archiver.bbclass new file +6 -0

+# This is special control for archiving linux-yocto only.
+python do_archive_linux_yocto(){
+ s = d.getVar('S', True)
+ if 'linux-yocto' in s:
+ source_tar_name = archive_sources(d,'')
+ if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() not in 

'SRPM':
+ move_tarball_deploy(d,[source_tar_name,''])

meta/classes/archiver.bbclass new file +21 -0

+}
+do_kernel_checkout[postfuncs] += "do_archive_linux_yocto "
+
+# remove tarball for sources, patches and logs after creating srpm.
+python do_remove_tarball(){
+ if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True).upper() == 'SRPM':
+ work_dir = d.getVar('WORKDIR', True)
+ os.chdir(work_dir)
+ for file in os.listdir(os.getcwd()):
+ if file in get_package(d):
+ try:
+ os.remove(file)
+ except OSError:
+ pass
+ if os.path.exists('tar-package'):
+ os.remove('tar-package')
+}
+do_remove_taball[deptask] = "do_archive_scripts_logs"
+do_package_write_rpm[postfuncs] += "do_remove_tarball "
+export get_licenses
+export get_package

Quoted 5 lines · Xiaofeng Yan

Quoted 2 lines

From: Xiaofeng Yan<[email protected]>

Why do you rebuild the tarball here are rename it instead of just copying the original tarball?

because many packages come from non-tarball and they could be git sources , So I archive these sources after do_unpack.

Op 20 mrt. 2012, om 06:50 heeft Xiaofeng Yan het volgende geschreven:

Quoted 7 lines · Saul Wold

Quoted 5 lines · Xiaofeng Yan

Quoted 2 lines

From: Xiaofeng Yan<[email protected]>

Why do you rebuild the tarball here are rename it instead of just copying the original tarball?

because many packages come from non-tarball and they could be git sources , So I archive these sources after do_unpack.

But if you have the original tarball and reuse it, you'll get the warm fuzzy feeling of matching checksums.

regards,

Koen
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core

Quoted 3 lines · Koen Kooi

Op 20 mrt. 2012, om 06:50 heeft Xiaofeng Yan het volgende geschreven:

Quoted 1 line · Saul Wold

Quoted 7 lines

Quoted 6 lines

Quoted 5 lines · Xiaofeng Yan

Quoted 2 lines

From: Xiaofeng Yan<[email protected]>

Why do you rebuild the tarball here are rename it instead of just copying the original tarball?

because many packages come from non-tarball and they could be git sources , So I archive these sources after do_unpack.

But if you have the original tarball and reuse it, you'll get the warm fuzzy feeling of matching checksums.

So How about renaming archive-original-source to archive-prepatched-source?

Quoted 7 lines

regards,

Koen
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core

_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core

Quoted 20 lines · Xiaofeng Yan

Quoted 17 lines · Koen Kooi

Op 20 mrt. 2012, om 06:50 heeft Xiaofeng Yan het volgende geschreven:

Quoted 11 lines · Saul Wold

Quoted 8 lines · Xiaofeng Yan

Quoted 3 lines

From: Xiaofeng Yan<[email protected]>

Why do you rebuild the tarball here are rename it instead of just copying the original tarball?

because many packages come from non-tarball and they could be git sources , So I archive these sources after do_unpack.

But if you have the original tarball and reuse it, you'll get the warm fuzzy feeling of matching checksums.

Quoted 2 lines

So How about  renaming archive-original-source to
archive-prepatched-source?

Hi Saul,

Or I can add a new functions to get tarball from downloads if have tarball and archive source codes directory if have directory without tarball. Do you think it is necessary for us to add this function?

Quoted 21 lines · Xiaofeng Yan

Quoted 20 lines · Xiaofeng Yan

Quoted 17 lines · Koen Kooi

Op 20 mrt. 2012, om 06:50 heeft Xiaofeng Yan het volgende geschreven:

Quoted 11 lines · Saul Wold

Quoted 8 lines · Xiaofeng Yan

Quoted 3 lines

From: Xiaofeng Yan<[email protected]>

Why do you rebuild the tarball here are rename it instead of just copying the original tarball?

because many packages come from non-tarball and they could be git sources , So I archive these sources after do_unpack.

But if you have the original tarball and reuse it, you'll get the warm fuzzy feeling of matching checksums.

Quoted 8 lines

Quoted 2 lines

So How about renaming archive-original-source to
archive-prepatched-source?

Hi Saul,

Or I can add a new functions to get tarball from downloads if have tarball and archive source codes directory if have directory without tarball. Do you think it is necessary for us to add this function?

I'd say that this is a change that should be made. If nothing else, just to get the assurance that the archived tarball's are the same as the ones that was used to build the system. (This is beneficial both when it comes to using the archived tarballs for rebuilding the system, as well as for ensuring licence compliance).

/Anders

_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core

Quoted 31 lines · Anders Darander

Quoted 21 lines · Xiaofeng Yan

Quoted 20 lines · Xiaofeng Yan

Quoted 17 lines · Koen Kooi

Op 20 mrt. 2012, om 06:50 heeft Xiaofeng Yan het volgende geschreven:

Quoted 11 lines · Saul Wold

Quoted 8 lines · Xiaofeng Yan

Quoted 3 lines

From: Xiaofeng Yan<[email protected]>

Why do you rebuild the tarball here are rename it instead of just copying the original tarball?

because many packages come from non-tarball and they could be git sources , So I archive these sources after do_unpack.

But if you have the original tarball and reuse it, you'll get the warm fuzzy feeling of matching checksums.

Quoted 8 lines

Quoted 2 lines

So How about renaming archive-original-source to
archive-prepatched-source?

Hi Saul,

Or I can add a new functions to get tarball from downloads if have tarball and archive source codes directory if have directory without tarball. Do you think it is necessary for us to add this function?

Quoted 6 lines

I'd say that this is a change that should be made. If nothing else, just to get the assurance that the archived tarball's are the same as the ones that was used to build the system. (This is beneficial both when it comes to using the archived tarballs for rebuilding the system, as well as for ensuring licence compliance).

I agree here, as Koen also pointed out, it will have matching checksums.

As to the name of the tarball, again best to keep it named the same as the original tarball.

Sau!

Quoted 7 lines

/Anders

_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core

_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core