Fix writing events to file
cover Fix writing events to file
Hi,
This patchset fixes "object is not JSON serializable" error caused by attempt to convert binary pickled event object to JSON when bitbake is called with -w command line option.
It also fixes error caused writing the same event to the file multiple times and refactoring of event writing functionality.
The following changes since commit 09d08340fe0fca6f551c33d2fdcafb72f52812d2:
Revert "cmake.bbclass: set the modules directory correctly" (2016-06-20 08:40:21 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib ed/bitbake/cooker-event-pickling-9803
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=ed/bitbake/cooker-event-pickling-9803
Ed Bartosh (5):
cooker: move EventLogWriteHandler to the top module level
cooker: encode event objects to base64
cooker: don't remove event file
cooker: replace EventLogWriteHandler with namedtuple
cooker: clean up EvertWriter
bitbake/lib/bb/cooker.py | 114 +++++++++++++++++++++-------------------------- 1 file changed, 51 insertions(+), 63 deletions(-)
--
Regards,
Ed
PATCH 1/5 cooker: move EventLogWriteHandler to the top module level
EventLogWriteHandler object was created and used in BBCooker.initConfigurationData.
This causes creation of multiple EventLogWriteHandler objects and results in duplicated entries in the output event file as BBCooker.initConfigurationData is called multiple times.
Added eventlogfile parameter to EventLogWriteHandler to avoid using global variable DEFAULT_EVENTFILE.
Moved EventLogWriteHandler to the module level.
Created EventLogWriteHandler object in BBCooker.__init__ to ensure that only one handler object is created.
Signed-off-by Ed Bartosh <[email protected]>
--- bitbake/lib/bb/cooker.py | 126 ++++++++++++++++++++++++----------------------- 1 file changed, 64 insertions(+), 62 deletions(-)
bitbake/lib/bb/cooker.py +64 -62
@@ -42,6 +42,8 @@ import subprocess import errno import prserv.serv import pyinotify +import json +import pickle logger = logging.getLogger("BitBake") collectlog = logging.getLogger("BitBake.Collection")
@@ -114,6 +116,63 @@ class CookerFeatures(object): return next(self._features) +class EventLogWriteHandler: + + def __init__(self, cooker, eventfile): + # set our handler's event processor + self.event = self.EventWriter(cooker, eventfile) + + class EventWriter: + def __init__(self, cooker, eventfile): + self.file_inited = None + self.cooker = cooker + self.eventfile = eventfile + self.event_queue = [] + + def init_file(self): + try: + # delete the old log + os.remove(self.eventfile) + except: + pass + + # write current configuration data + with open(eventfile, "w") as f: + f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) + + def write_event(self, event): + with open(self.eventfile, "a") as f: + try: + f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) })) + except Exception as e: + import traceback + print(e, traceback.format_exc()) + + + def send(self, event): + event_class = event.__module__ + "." + event.__class__.__name__ + + # init on bb.event.BuildStarted + if self.file_inited is None: + if event_class == "bb.event.BuildStarted": + self.init_file() + self.file_inited = True + + # write pending events + for e in self.event_queue: + self.write_event(e) + + # also write the current event + self.write_event(event) + + else: + # queue all events until the file is inited + self.event_queue.append(event) + + else: + # we have the file, just write the event + self.write_event(event) + #============================================================================# # BBCooker #============================================================================#
@@ -151,6 +210,11 @@ class BBCooker: self.initConfigurationData() + # we log all events to a file if so directed + if self.configuration.writeeventlog: + # register the log file writer as UI Handler + bb.event.register_UIHhandler(EventLogWriteHandler(self, self.configuration.writeeventlog)) + self.inotify_modified_files = [] def _process_inotify_updates(server, notifier_list, abort):
@@ -301,68 +365,6 @@ class BBCooker: if consolelog: self.data.setVar("BB_CONSOLELOG", consolelog) - # we log all events to a file if so directed - if self.configuration.writeeventlog: - import json, pickle - DEFAULT_EVENTFILE = self.configuration.writeeventlog - class EventLogWriteHandler(): - - class EventWriter(): - def __init__(self, cooker): - self.file_inited = None - self.cooker = cooker - self.event_queue = [] - - def init_file(self): - try: - # delete the old log - os.remove(DEFAULT_EVENTFILE) - except: - pass - - # write current configuration data - with open(DEFAULT_EVENTFILE, "w") as f: - f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) - - def write_event(self, event): - with open(DEFAULT_EVENTFILE, "a") as f: - try: - f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) })) - except Exception as e: - import traceback - print(e, traceback.format_exc()) - - - def send(self, event): - event_class = event.__module__ + "." + event.__class__.__name__ - - # init on bb.event.BuildStarted - if self.file_inited is None: - if event_class == "bb.event.BuildStarted": - self.init_file() - self.file_inited = True - - # write pending events - for e in self.event_queue: - self.write_event(e) - - # also write the current event - self.write_event(event) - - else: - # queue all events until the file is inited - self.event_queue.append(event) - - else: - # we have the file, just write the event - self.write_event(event) - - # set our handler's event processor - event = EventWriter(self) # self is the cooker here - - # register the log file writer as UI Handler - bb.event.register_UIHhandler(EventLogWriteHandler()) - # # Copy of the data store which has been expanded. # Used for firing events and accessing variables where expansion needs to be accounted for
Signature · 1 line
2.1.4
PATCH 2/5 cooker: encode event objects to base64
In reply to Ed Bartoshpickle converts python objects into the binary form that can't be decoded to text and therefore can't be converted to JSON format.
Attempt to convert event objects raises this error:
TypeError:
b'\x80\x03cbb.runqueue\nrunQueueTaskSkipped\nq\x00)... is not JSON serializable
Encoded pickled event objects to base64 to be able to convert data structure to JSON.
[YOCTO #9803]
Signed-off-by Ed Bartosh <[email protected]>
--- bitbake/lib/bb/cooker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
bitbake/lib/bb/cooker.py +4 -1
@@ -44,6 +44,7 @@ import prserv.serv import pyinotify import json import pickle +import codecs logger = logging.getLogger("BitBake") collectlog = logging.getLogger("BitBake.Collection")
@@ -143,7 +144,9 @@ class EventLogWriteHandler: def write_event(self, event): with open(self.eventfile, "a") as f: try: - f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) })) + str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8') + f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__, + "vars": str_event})) except Exception as e: import traceback print(e, traceback.format_exc())
Signature · 1 line
2.1.4
PATCH 3/5 cooker: don't remove event file
In reply to Ed BartoshThere is no need to remove output file as it gets rewritten by open(self.eventfile, 'w') anyway.
Signed-off-by Ed Bartosh <[email protected]>
--- bitbake/lib/bb/cooker.py | 6 ------ 1 file changed, 6 deletions(-)
bitbake/lib/bb/cooker.py +0 -6
@@ -131,12 +131,6 @@ class EventLogWriteHandler: self.event_queue = [] def init_file(self): - try: - # delete the old log - os.remove(self.eventfile) - except: - pass - # write current configuration data with open(eventfile, "w") as f: f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
Signature · 1 line
2.1.4
PATCH 4/5 cooker: replace EventLogWriteHandler with namedtuple
In reply to Ed Bartoshclass EventLogWriteHandler is a simple wrapper class with only one class member. Replacing it with namedtuple makes code less nested and more readable.
Signed-off-by Ed Bartosh <[email protected]>
--- bitbake/lib/bb/cooker.py | 86 +++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 45 deletions(-)
bitbake/lib/bb/cooker.py +41 -45
@@ -33,7 +33,7 @@ import threading from io import StringIO from contextlib import closing from functools import wraps -from collections import defaultdict +from collections import defaultdict, namedtuple import bb, bb.exceptions, bb.command from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build import queue
@@ -117,58 +117,52 @@ class CookerFeatures(object): return next(self._features) -class EventLogWriteHandler: - +class EventWriter: def __init__(self, cooker, eventfile): - # set our handler's event processor - self.event = self.EventWriter(cooker, eventfile) - - class EventWriter: - def __init__(self, cooker, eventfile): - self.file_inited = None - self.cooker = cooker - self.eventfile = eventfile - self.event_queue = [] - - def init_file(self): - # write current configuration data - with open(eventfile, "w") as f: - f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) - - def write_event(self, event): - with open(self.eventfile, "a") as f: - try: - str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8') - f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__, - "vars": str_event})) - except Exception as e: - import traceback - print(e, traceback.format_exc()) + self.file_inited = None + self.cooker = cooker + self.eventfile = eventfile + self.event_queue = [] + def init_file(self): + # write current configuration data + with open(eventfile, "w") as f: + f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) + + def write_event(self, event): + with open(self.eventfile, "a") as f: + try: + str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8') + f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__, + "vars": str_event})) + except Exception as e: + import traceback + print(e, traceback.format_exc()) - def send(self, event): - event_class = event.__module__ + "." + event.__class__.__name__ - # init on bb.event.BuildStarted - if self.file_inited is None: - if event_class == "bb.event.BuildStarted": - self.init_file() - self.file_inited = True + def send(self, event): + event_class = event.__module__ + "." + event.__class__.__name__ - # write pending events - for e in self.event_queue: - self.write_event(e) + # init on bb.event.BuildStarted + if self.file_inited is None: + if event_class == "bb.event.BuildStarted": + self.init_file() + self.file_inited = True - # also write the current event - self.write_event(event) + # write pending events + for e in self.event_queue: + self.write_event(e) - else: - # queue all events until the file is inited - self.event_queue.append(event) + # also write the current event + self.write_event(event) else: - # we have the file, just write the event - self.write_event(event) + # queue all events until the file is inited + self.event_queue.append(event) + + else: + # we have the file, just write the event + self.write_event(event) #============================================================================# # BBCooker
@@ -210,7 +204,9 @@ class BBCooker: # we log all events to a file if so directed if self.configuration.writeeventlog: # register the log file writer as UI Handler - bb.event.register_UIHhandler(EventLogWriteHandler(self, self.configuration.writeeventlog)) + writer = EventWriter(self, self.configuration.writeeventlog) + EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event']) + bb.event.register_UIHhandler(EventLogWriteHandler(writer)) self.inotify_modified_files = []
Signature · 1 line
2.1.4
PATCH 5/5 cooker: clean up EvertWriter
In reply to Ed BartoshRestructured EventWriter code to make it more readable:
- got rid of init_file method as it's called only once
- renamed exception variable e -> err
- renamed event variable e -> evt
- simplified main 'if' structure of send method
Signed-off-by Ed Bartosh <[email protected]>
--- bitbake/lib/bb/cooker.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-)
bitbake/lib/bb/cooker.py +13 -20
@@ -124,46 +124,39 @@ class EventWriter: self.eventfile = eventfile self.event_queue = [] - def init_file(self): - # write current configuration data - with open(eventfile, "w") as f: - f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) - def write_event(self, event): with open(self.eventfile, "a") as f: try: str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8') f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__, "vars": str_event})) - except Exception as e: + except Exception as err: import traceback - print(e, traceback.format_exc()) - + print(err, traceback.format_exc()) def send(self, event): - event_class = event.__module__ + "." + event.__class__.__name__ + if self.file_inited: + # we have the file, just write the event + self.write_event(event) + else: + # init on bb.event.BuildStarted + name = "%s.%s" % (event.__module__, event.__class__.__name__) + if name == "bb.event.BuildStarted": + with open(self.eventfile, "w") as f: + f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) - # init on bb.event.BuildStarted - if self.file_inited is None: - if event_class == "bb.event.BuildStarted": - self.init_file() self.file_inited = True # write pending events - for e in self.event_queue: - self.write_event(e) + for evt in self.event_queue: + self.write_event(evt) # also write the current event self.write_event(event) - else: # queue all events until the file is inited self.event_queue.append(event) - else: - # we have the file, just write the event - self.write_event(event) - #============================================================================# # BBCooker #============================================================================#
Signature · 1 line
2.1.4