admin: add an action to duplicate a branch
PATCH 1/4 admin: add an action to duplicate a branch
Add an action to duplicate a Branch object, along with all of the LayerBranches (and LayerMaintainers and LayerDependencies) underneath it.
--- layerindex/admin.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+)
layerindex/admin.py +26 -0
@@ -17,6 +17,32 @@ class LayerDependencyInline(admin.StackedInline): class BranchAdmin(CompareVersionAdmin): model = Branch + actions = ['duplicate'] + + def duplicate(self, request, queryset): + for branch in queryset: + layerbranches = branch.layerbranch_set.all() + branch.pk = None + branch.name += '-copy' + branch.save() + for layerbranch in layerbranches: + layerbranch_maintainers = layerbranch.layermaintainer_set.all() + layerbranch_dependencies = layerbranch.dependencies_set.all() + layerbranch.pk = None + layerbranch.branch = branch + layerbranch.vcs_last_fetch = None + layerbranch.vcs_last_rev = '' + layerbranch.vcs_last_commit = None + layerbranch.save() + for layermaintainer in layerbranch_maintainers: + layermaintainer.pk = None + layermaintainer.layerbranch = layerbranch + layermaintainer.save() + for layerdependency in layerbranch_dependencies: + layerdependency.pk = None + layerdependency.layerbranch = layerbranch + layerdependency.save() + duplicate.short_description = "Duplicate selected Branches" class LayerItemAdmin(CompareVersionAdmin): list_filter = ['status', 'layer_type']
Signature · 1 line
2.5.5
PATCH 2/4 utils: fix error in runcmd() if printerr=False
If you specified printerr=False we were referring to the output variable that hadn't been set. Looks like I broke this back in 2013 in 93ce26f21cdbbd8a645792359cde87acf05144d7.
--- layerindex/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
layerindex/utils.py +2 -2
@@ -190,9 +190,9 @@ def runcmd(cmd, destdir=None, printerr=True, logger=None): subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True) except subprocess.CalledProcessError as e: out.seek(0) + output = out.read() + output = output.decode('ascii').strip() if printerr: - output = out.read() - output = output.decode('ascii').strip() if logger: logger.error("%s" % output) else:
Signature · 1 line
2.5.5
PATCH 3/4 Record and display update logs
In reply to Paul EggletonAt the moment it's a bit difficult to get update logs out of the environment in which the update script is being run. In order to make the logs more accessible, create a LayerUpdate model to record the output of update_layer.py separately for each layerbranch and tie the created LayerUpdates together with a single Update model per session.
We provide two ways to look at this - a Tools->Updates page for logged-in users, and there's also an "Updates" tab on each layer that is accessible to anyone; which one is useful depends on whether you are looking at the index as a whole or an individual layer.
Update records older than 30 days are deleted automatically by default.
--- TODO | 2 - layerindex/admin.py | 8 + layerindex/migrations/0005_layerupdate.py | 46 +++++ layerindex/models.py | 37 +++- layerindex/update.py | 286 ++++++++++++++++++------------ layerindex/urls.py | 14 +- layerindex/urls_branch.py | 4 +- layerindex/utils.py | 13 ++ layerindex/views.py | 30 +++- settings.py | 3 + templates/base.html | 1 + templates/layerindex/detail.html | 37 ++++ templates/layerindex/layerupdate.html | 30 ++++ templates/layerindex/updatedetail.html | 47 +++++ templates/layerindex/updatelist.html | 57 ++++++ 15 files changed, 491 insertions(+), 124 deletions(-) create mode 100644 layerindex/migrations/0005_layerupdate.py create mode 100644 templates/layerindex/layerupdate.html create mode 100644 templates/layerindex/updatedetail.html create mode 100644 templates/layerindex/updatelist.html
TODO +0 -2
@@ -35,10 +35,8 @@ Other * Create simple script to check for unlisted layer subdirectories in all repos * Auto-detect more values from github pages? * Ability for submitters to get email notification about publication? -* Update script still seems not to be always printing layer name on parsing warnings/errors * Update script could send warnings when parsing layers to maintainers? (optional) * Click on OE-Classic graph element to go to query? * Use bar instead of pie graphs for OE-Classic statistics * Ensure OE-Core appears before meta-oe in layer list * Ability for reviewers to comment before publishing a layer? -* Record update & parse errors against recipe/layer
layerindex/admin.py +8 -0
@@ -74,6 +74,12 @@ class LayerDependencyAdmin(CompareVersionAdmin): class LayerNoteAdmin(CompareVersionAdmin): list_filter = ['layer__name'] +class UpdateAdmin(admin.ModelAdmin): + pass + +class LayerUpdateAdmin(admin.ModelAdmin): + list_filter = ['update__started', 'layerbranch__layer__name', 'layerbranch__branch__name'] + class RecipeAdmin(admin.ModelAdmin): search_fields = ['filename', 'pn'] list_filter = ['layerbranch__layer__name', 'layerbranch__branch__name']
@@ -144,6 +150,8 @@ admin.site.register(LayerBranch, LayerBranchAdmin) admin.site.register(LayerMaintainer, LayerMaintainerAdmin) admin.site.register(LayerDependency, LayerDependencyAdmin) admin.site.register(LayerNote, LayerNoteAdmin) +admin.site.register(Update, UpdateAdmin) +admin.site.register(LayerUpdate, LayerUpdateAdmin) admin.site.register(Recipe, RecipeAdmin) admin.site.register(RecipeFileDependency) admin.site.register(Machine, MachineAdmin)
layerindex/migrations/0005_layerupdate.py new file +46 -0
46 lines not on this page · raw message
layerindex/models.py +36 -1
@@ -31,7 +31,7 @@ class PythonEnvironment(models.Model): class Branch(models.Model): - name = models.CharField(max_length=50) + name = models.CharField('Branch name', max_length=50) bitbake_branch = models.CharField(max_length=50) short_description = models.CharField(max_length=50, blank=True) sort_priority = models.IntegerField(blank=True, null=True)
@@ -47,6 +47,16 @@ class Branch(models.Model): return self.name +class Update(models.Model): + started = models.DateTimeField() + finished = models.DateTimeField(blank=True, null=True) + log = models.TextField(blank=True) + reload = models.BooleanField('Reloaded', default=False, help_text='Was this update a reload?') + + def __str__(self): + return '%s' % self.started + + class LayerItem(models.Model): LAYER_STATUS_CHOICES = ( ('N', 'New'),
@@ -255,6 +265,31 @@ class LayerNote(models.Model): return "%s: %s" % (self.layer.name, self.text) +class LayerUpdate(models.Model): + layerbranch = models.ForeignKey(LayerBranch) + update = models.ForeignKey(Update) + started = models.DateTimeField() + finished = models.DateTimeField() + errors = models.IntegerField(default=0) + warnings = models.IntegerField(default=0) + log = models.TextField(blank=True) + + def save(self): + warnings = 0 + errors = 0 + for line in self.log.splitlines(): + if line.startswith('WARNING:'): + warnings += 1 + elif line.startswith('ERROR:'): + errors += 1 + self.warnings = warnings + self.errors = errors + super(LayerUpdate, self).save() + + def __str__(self): + return "%s: %s: %s" % (self.layerbranch.layer.name, self.layerbranch.branch.name, self.started) + + class Recipe(models.Model): layerbranch = models.ForeignKey(LayerBranch) filename = models.CharField(max_length=255)
layerindex/update.py +171 -115
324 lines not on this page · raw message
layerindex/urls.py +13 -1
@@ -8,7 +8,7 @@ from django.conf.urls import * from django.views.generic import TemplateView, DetailView, ListView, RedirectView from django.views.defaults import page_not_found from django.core.urlresolvers import reverse_lazy -from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView +from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, UpdateDetailView from layerindex.models import LayerItem, Recipe, RecipeChangeset from rest_framework import routers from . import restviews
@@ -67,6 +67,10 @@ urlpatterns = patterns('', template_name='layerindex/recipedetail.html'), name='recipe'), url(r'^layer/(?P<name>[-\w]+)/publish/$', 'layerindex.views.publish', name="publish"), + url(r'^layerupdate/(?P<pk>[-\w]+)/$', + LayerUpdateDetailView.as_view( + template_name='layerindex/layerupdate.html'), + name='layerupdate'), url(r'^bulkchange/$', BulkChangeView.as_view( template_name='layerindex/bulkchange.html'),
@@ -97,6 +101,14 @@ urlpatterns = patterns('', # context_object_name='recipe_list', # template_name='layerindex/rawrecipes.txt'), # name='recipe_list_raw'), + url(r'^updates/$', + UpdateListView.as_view( + template_name='layerindex/updatelist.html'), + name='update_list'), + url(r'^updates/(?P<pk>[-\w]+)/$', + UpdateDetailView.as_view( + template_name='layerindex/updatedetail.html'), + name='update'), url(r'^history/$', HistoryListView.as_view( template_name='layerindex/history.html'),
layerindex/urls_branch.py +2 -2
@@ -1,13 +1,13 @@ # layerindex-web - Branch-based URL definitions # -# Copyright (C) 2013 Intel Corporation +# Copyright (C) 2013-2016 Intel Corporation # # Licensed under the MIT license, see COPYING.MIT for details from django.conf.urls import * from django.views.defaults import page_not_found from django.core.urlresolvers import reverse_lazy -from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView +from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView urlpatterns = patterns('', url(r'^$',
layerindex/utils.py +13 -0
@@ -223,6 +223,19 @@ def logger_create(name): logger.setLevel(logging.INFO) return logger +class ListHandler(logging.Handler): + """Logging handler which accumulates formatted log records in a list, returning the list on demand""" + def __init__(self): + self.log = [] + logging.Handler.__init__(self, logging.WARNING) + def emit(self, record): + self.log.append('%s\n' % self.format(record)) + def read(self): + log = self.log + self.log = [] + return log + + def lock_file(fn): starttime = time.time() while True:
layerindex/views.py +27 -3
@@ -1,6 +1,6 @@ # layerindex-web - view definitions # -# Copyright (C) 2013-2014 Intel Corporation +# Copyright (C) 2013-2016 Intel Corporation # # Licensed under the MIT license, see COPYING.MIT for details
@@ -10,7 +10,7 @@ from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidde from django.core.urlresolvers import reverse, reverse_lazy, resolve from django.core.exceptions import PermissionDenied from django.template import RequestContext -from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Recipe, Machine, Distro, BBClass, BBAppend, RecipeChange, RecipeChangeset, ClassicRecipe +from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Update, LayerUpdate, Recipe, Machine, Distro, BBClass, BBAppend, RecipeChange, RecipeChangeset, ClassicRecipe from datetime import datetime from django.views.generic import TemplateView, DetailView, ListView from django.views.generic.edit import CreateView, DeleteView, UpdateView
@@ -18,7 +18,7 @@ from django.views.generic.base import RedirectView from layerindex.forms import EditLayerForm, LayerMaintainerFormSet, EditNoteForm, EditProfileForm, RecipeChangesetForm, AdvancedRecipeSearchForm, BulkChangeEditFormSet, ClassicRecipeForm, ClassicRecipeSearchForm from django.db import transaction from django.contrib.auth.models import User, Permission -from django.db.models import Q, Count +from django.db.models import Q, Count, Sum from django.core.mail import EmailMessage from django.template.loader import get_template from django.template import Context
@@ -328,6 +328,7 @@ class LayerDetailView(DetailView): context['distros'] = layerbranch.distro_set.order_by('name') context['appends'] = layerbranch.bbappend_set.order_by('filename') context['classes'] = layerbranch.bbclass_set.order_by('name') + context['updates'] = layerbranch.layerupdate_set.order_by('-started') context['url_branch'] = self.kwargs['branch'] context['this_url_name'] = resolve(self.request.path_info).url_name return context
@@ -599,6 +600,29 @@ class MachineSearchView(ListView): return context +class UpdateListView(ListView): + context_object_name = "updates" + paginate_by = 50 + + def get_queryset(self): + return Update.objects.all().order_by('-started').annotate(errors=Sum('layerupdate__errors'), warnings=Sum('layerupdate__warnings')) + + +class UpdateDetailView(DetailView): + model = Update + + def get_context_data(self, **kwargs): + context = super(UpdateDetailView, self).get_context_data(**kwargs) + update = self.get_object() + if update: + context['layerupdates'] = update.layerupdate_set.exclude(log__isnull=True).exclude(log__exact='') + return context + + +class LayerUpdateDetailView(DetailView): + model = LayerUpdate + + class DistroSearchView(ListView): context_object_name = 'distro_list' paginate_by = 50
settings.py +3 -0
@@ -211,6 +211,9 @@ BITBAKE_REPO_URL = "git://git.openembedded.org/bitbake" # Core layer to be used by the update script for basic BitBake configuration CORE_LAYER_NAME = "openembedded-core" +# Update records older than this number of days will be deleted every update +UPDATE_PURGE_DAYS = 30 + # Settings for layer submission feature SUBMIT_EMAIL_FROM = '[email protected]' SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission'
templates/base.html +1 -0
@@ -72,6 +72,7 @@ <ul class="dropdown-menu"> <li><a href="{% url 'bulk_change' %}">Bulk Change</a></li> <li><a href="{% url 'duplicates' 'master' %}">Duplicates</a></li> + <li><a href="{% url 'update_list' %}">Updates</a></li> </ul> </li> {% endif %}
templates/layerindex/detail.html +37 -0
@@ -181,6 +181,9 @@ {% if distros.count > 0 %} <li><a href="#distros" data-toggle="tab">Distros</a></li> {% endif %} + {% if updates.count > 0 %} + <li><a href="#updates" data-toggle="tab">Updates</a></li> + {% endif %} </ul> <div class="tab-content">
@@ -298,6 +301,40 @@ </table> </div> {% endif %} + {% if updates.count > 0 %} + <div class="tab-pane" id="updates"> + <div class="navbar"> + <div class="navbar-inner"> + <a class="brand pull-left">{{ layeritem.name }} updates</a> + </div> + </div> + + <table class="table table-bordered"> + <thead> + <tr> + <th>Date/time</th> + <th>Errors</th> + <th>Warnings</th> + </tr> + </thead> + <tbody> + {% for update in updates %} + <tr> + <td> + {% if update.log %} + <a href="{% url 'layerupdate' update.id %}">{{ update.started }}{% if update.update.reload %} (reload){% endif%}</a> + {% else %} + <span class="muted">{{ update.started }}{% if update.update.reload %} (reload){% endif%}</span> + {% endif %} + </td> + <td>{% if update.errors %}<span class="badge badge-important">{{ update.errors }}</span>{% endif %}</td> + <td>{% if update.warnings %}<span class="badge badge-warning">{{ update.warnings }}</span>{% endif %}</td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + {% endif %} </div>
templates/layerindex/layerupdate.html new file +30 -0
@@ -0,0 +1,30 @@ +{% extends "base.html" %} +{% load i18n %} + +{% comment %} + + layerindex-web - layer update page + + Copyright (C) 2016 Intel Corporation + Licensed under the MIT license, see COPYING.MIT for details + +{% endcomment %} + +<!-- +{% block title_append %} - {{ layerupdate.layerbranch.layer.name }} {{ layerupdate.layerbranch.branch.name }} - {{ layerupdate.started }} {% endblock %} +--> + +{% block content %} +{% autoescape on %} + +<h2>{{ layerupdate.layerbranch.layer.name }} {{ layerupdate.layerbranch.branch.name }} - {{ layerupdate.started }}</h2> + +<pre>{{ layerupdate.log }}</pre> + +{% endautoescape %} + +{% endblock %} + + +{% block scripts %} +{% endblock %}
templates/layerindex/updatedetail.html new file +47 -0
47 lines not on this page · raw message
templates/layerindex/updatelist.html new file +57 -0
57 lines not on this page · raw message
Signature · 1 line
2.5.5
PATCH 4/4 views: support querying class inheritance
In reply to Paul EggletonIt's a little crude and certainly not optimal performance-wise, but we can support querying for recipes that inherit a particular class without too much trouble. This allows you to add "inherits:cmake" to the query and have it return only recipes that inherit the cmake class. You can use more than one inherits: item to filter down to recipes that inherit all of the specified classes.
Note: this does not otherwise change the behaviour of specifying multiple words - all of the words other than those that start with "inherits:" are treated as part of a single phrase that will be searched for - not separate keywords.
Fixes [YOCTO #9879].
--- layerindex/views.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
layerindex/views.py +16 -0
@@ -376,6 +376,22 @@ class RecipeSearchView(ListView): _check_url_branch(self.kwargs) query_string = self.request.GET.get('q', '') init_qs = Recipe.objects.filter(layerbranch__branch__name=self.kwargs['branch']) + + # Support slightly crude search on inherits field + query_items = query_string.split() + inherits = [] + query_terms = [] + for item in query_items: + if item.startswith('inherits:'): + inherits.append(item.split(':')[1]) + else: + query_terms.append(item) + if inherits: + # FIXME This is a bit ugly, perhaps we should consider having this as a one-many relationship instead + for inherit in inherits: + init_qs = init_qs.filter(Q(inherits=inherit) | Q(inherits__startswith=inherit + ' ') | Q(inherits__endswith=' ' + inherit) | Q(inherits__contains=' %s ' % inherit)) + query_string = ' '.join(query_terms) + if query_string.strip(): order_by = ('pn', 'layerbranch__layer')
Signature · 1 line
2.5.5