Read correct cooker log
cover Read correct cooker log
this patch selects the correct cooker log in both analysis and managed mode.
The following changes since commit d4585fa23d56a58c583ee33db067a23177a936b1:
bsps: update 3.14 SRCREVs (2015-09-09 14:27:57 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib bavery/submit/elliot/20150911_read-correct-cooker-log
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=bavery/submit/elliot/20150911_read-correct-cooker-log
Elliot Smith (3):
toaster: Read correct cooker log path from toasterui
toaster: Avoid unnecessary local to local copy of cooker log
toaster: Don't def a function for each call to build_artifact()
lib/bb/ui/buildinfohelper.py | 8 ++-- lib/bb/ui/toasterui.py | 2 +- .../management/commands/checksettings.py | 22 --------- .../bldcontrol/management/commands/runbuilds.py | 24 +--------- lib/toaster/toastergui/views.py | 53 ++++++---------------- toaster-requirements.txt | 1 + 6 files changed, 23 insertions(+), 87 deletions(-)
Signature · 1 line
1.9.1
PATCH 1/3 toaster: Read correct cooker log path from toasterui
From: Elliot Smith <[email protected]>
The BB_CONSOLELOG variable changes by the time we read it in BuildInfoHelper. This means that the log file location we are using is incorrect, so the links to the cooker logs don't work.
Instead, read it at the point when the BuildStarted event occurs in toasterui. The BB_CONSOLELOG variable has the correct value here, so pass that to BuildInfoHelper.
[YOCTO #8209]
--- lib/bb/ui/buildinfohelper.py | 8 ++++---- lib/bb/ui/toasterui.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-)
lib/bb/ui/buildinfohelper.py +4 -4
@@ -704,7 +704,7 @@ class BuildInfoHelper(object): ## methods to convert event/external info into objects that the ORM layer uses - def _get_build_information(self): + def _get_build_information(self, consolelogfile): build_info = {} # Generate an identifier for each new build
@@ -713,7 +713,7 @@ class BuildInfoHelper(object): build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] build_info['started_on'] = timezone.now() build_info['completed_on'] = timezone.now() - build_info['cooker_log_path'] = self.server.runCommand(["getVariable", "BB_CONSOLELOG"])[0] + build_info['cooker_log_path'] = consolelogfile build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
@@ -847,9 +847,9 @@ class BuildInfoHelper(object): logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee) - def store_started_build(self, event): + def store_started_build(self, event, consolelogfile): assert '_pkgs' in vars(event) - build_information = self._get_build_information() + build_information = self._get_build_information(consolelogfile) build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
lib/bb/ui/toasterui.py +1 -1
@@ -126,7 +126,7 @@ def main(server, eventHandler, params ): # the code will look into the protected variables of the event; no easy way around this if isinstance(event, bb.event.BuildStarted): - buildinfohelper.store_started_build(event) + buildinfohelper.store_started_build(event, consolelogfile) if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): buildinfohelper.update_and_store_task(event)
Signature · 1 line
1.9.1
PATCH 2/3 toaster: Avoid unnecessary local to local copy of cooker log
From: Elliot Smith <[email protected]>
The cooker log is copied from its original (bitbake) location to an artifact directory chosen by the user (chosen when checksettings.py runs as part of the managed mode; it's one of the annoying questions you're asked at startup). The copy happens as part of the runbuilds script run, which is started in a loop from the toaster startup script in managed mode.
When a user requests the log for a build via toaster, they are getting the log which has been copied to the artifact directory, not the original bitbake log.
This works for the managed case, where the runbuilds command is running in a loop and copying log files for completed builds to the artifact directory. However, in analysis mode, there are two problems:
1. checksettings isn't run, so the artifacts directory isn't set. toaster is then unable to figure out where the log files should have been copied to.
2. The log files aren't copied to the artifacts directory anyway, as runbuilds isn't running in analysis mode.
To fix this, just point the user to the local bitbake log file in its original location. This avoids the copy step, and means we can remove a whole question from the toaster startup sequence, as we no longer need an artifact directory.
The only downside to this is that it's not appropriate for remote bitbake servers. We will need to revisit this and possibly reinstate the copy step once we have to reconcile local and remote builds and make their logs available in the same way.
[YOCTO #8209]
--- .../management/commands/checksettings.py | 22 -------------------- .../bldcontrol/management/commands/runbuilds.py | 24 ++-------------------- lib/toaster/toastergui/views.py | 13 ++++++------ 3 files changed, 8 insertions(+), 51 deletions(-)
lib/toaster/bldcontrol/management/commands/checksettings.py +0 -22
@@ -64,27 +64,6 @@ class Command(NoArgsCommand): return "" return DN(self._find_first_path_for_file(DN(self.guesspath), "bblayers.conf", 4)) - - def _verify_artifact_storage_dir(self): - # verify that we have a settings for downloading artifacts - while ToasterSetting.objects.filter(name="ARTIFACTS_STORAGE_DIR").count() == 0: - guessedpath = os.getcwd() + "/toaster_build_artifacts/" - print("\nToaster needs to know in which directory it can download build log files and other artifacts.\nToaster suggests \"%s\"." % guessedpath) - artifacts_storage_dir = raw_input("Press Enter to select \"%s\" or type the full path to a different directory: " % guessedpath) - if len(artifacts_storage_dir) == 0: - artifacts_storage_dir = guessedpath - if len(artifacts_storage_dir) > 0 and artifacts_storage_dir.startswith("/"): - try: - os.makedirs(artifacts_storage_dir) - except OSError as ose: - if "File exists" in str(ose): - pass - else: - raise ose - ToasterSetting.objects.create(name="ARTIFACTS_STORAGE_DIR", value=artifacts_storage_dir) - return 0 - - def _verify_build_environment(self): # refuse to start if we have no build environments while BuildEnvironment.objects.count() == 0:
@@ -239,7 +218,6 @@ class Command(NoArgsCommand): def handle_noargs(self, **options): retval = 0 - retval += self._verify_artifact_storage_dir() retval += self._verify_build_environment() retval += self._verify_default_settings() retval += self._verify_builds_in_progress()
lib/toaster/bldcontrol/management/commands/runbuilds.py +2 -22
@@ -77,31 +77,11 @@ class Command(NoArgsCommand): bec.be.save() def archive(self): - ''' archives data from the builds ''' - artifact_storage_dir = ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): - # save cooker log if br.build == None: br.state = BuildRequest.REQ_FAILED - br.save() - continue - build_artifact_storage_dir = os.path.join(artifact_storage_dir, "%d" % br.build.pk) - try: - os.makedirs(build_artifact_storage_dir) - except OSError as ose: - if "File exists" in str(ose): - pass - else: - raise ose - - file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") - try: - with open(file_name, "w") as f: - f.write(br.environment.get_artifact(br.build.cooker_log_path).read()) - except IOError: - os.unlink(file_name) - - br.state = BuildRequest.REQ_COMPLETED + else: + br.state = BuildRequest.REQ_COMPLETED br.save() def cleanup(self):
lib/toaster/toastergui/views.py +6 -7
@@ -2710,7 +2710,6 @@ if True: def build_artifact(request, build_id, artifact_type, artifact_id): if artifact_type in ["cookerlog"]: - # these artifacts are saved after building, so they are on the server itself def _mimetype_for_artifact(path): try: import magic
@@ -2741,16 +2740,16 @@ if True: except ImportError: return "binary/octet-stream" try: - # match code with runbuilds.Command.archive() - build_artifact_storage_dir = os.path.join(ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value, "%d" % int(build_id)) - file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") - + build = Build.objects.get(pk = build_id) + file_name = build.cooker_log_path fsock = open(file_name, "r") - content_type=_mimetype_for_artifact(file_name) + content_type = _mimetype_for_artifact(file_name) response = HttpResponse(fsock, content_type = content_type) - response['Content-Disposition'] = 'attachment; filename=' + os.path.basename(file_name) + disposition = 'attachment; filename=cooker.log' + response['Content-Disposition'] = disposition + return response except IOError: context = {
Signature · 1 line
1.9.1
PATCH 3/3 toaster: Don't def a function for each call to build_artifact()
In reply to brian averyFrom: Elliot Smith <[email protected]>
Cache the mimetype object and only define the function for getting a mimetype once.
Also ensure that filemagic is listed as a requirement of toaster. Doing this also means we can remove the code which tries multiple different "magic" libraries, as we know we have the right version available.
--- lib/toaster/toastergui/views.py | 42 ++++++++++------------------------------- toaster-requirements.txt | 1 + 2 files changed, 11 insertions(+), 32 deletions(-)
lib/toaster/toastergui/views.py +10 -32
@@ -47,10 +47,19 @@ import json from os.path import dirname import itertools +import magic import logging logger = logging.getLogger("toaster") +class MimeTypeFinder(object): + _magic = magic.Magic(flags = magic.MAGIC_MIME_TYPE) + + # returns the mimetype for a file path + @classmethod + def get_mimetype(self, path): + return self._magic.id_filename(path) + # all new sessions should come through the landing page; # determine in which mode we are running in, and redirect appropriately def landing(request):
@@ -68,8 +77,6 @@ def landing(request): return render(request, 'landing.html', context) - - # returns a list for most recent builds; def _get_latest_builds(prj=None): queryset = Build.objects.all()
@@ -2710,40 +2717,11 @@ if True: def build_artifact(request, build_id, artifact_type, artifact_id): if artifact_type in ["cookerlog"]: - def _mimetype_for_artifact(path): - try: - import magic - - # fair warning: this is a mess; there are multiple competing and incompatible - # magic modules floating around, so we try some of the most common combinations - - try: # we try ubuntu's python-magic 5.4 - m = magic.open(magic.MAGIC_MIME_TYPE) - m.load() - return m.file(path) - except AttributeError: - pass - - try: # we try python-magic 0.4.6 - m = magic.Magic(magic.MAGIC_MIME) - return m.from_file(path) - except AttributeError: - pass - - try: # we try pip filemagic 1.6 - m = magic.Magic(flags=magic.MAGIC_MIME_TYPE) - return m.id_filename(path) - except AttributeError: - pass - - return "binary/octet-stream" - except ImportError: - return "binary/octet-stream" try: build = Build.objects.get(pk = build_id) file_name = build.cooker_log_path fsock = open(file_name, "r") - content_type = _mimetype_for_artifact(file_name) + content_type = MimeTypeFinder.get_mimetype(file_name) response = HttpResponse(fsock, content_type = content_type)
toaster-requirements.txt +1 -0
@@ -2,3 +2,4 @@ Django==1.6 South==0.8.4 argparse==1.2.1 wsgiref==0.1.2 +filemagic==1.6
Signature · 1 line
1.9.1