Implement CSV export for layer recipes
cover Implement CSV export for layer recipes
Ross asked for the ability to export a layer's recipes in CSV format, and here it is - as well as cleanup of some old cruft that was supposed to do a similar thing but wasn't finished.
The following changes since commit 837359e6f18aac985c2967344d035d70c05e427a:
detail: Fix layer description not wrapping properly (2018-05-04 23:52:51 +1200)
are available in the Git repository at:
git://git.yoctoproject.org/layerindex-web paule/recipe-csv-export
http://git.yoctoproject.org/cgit.cgi//log/?h=paule/recipe-csv-export
Paul Eggleton (2):
Drop old raw recipe export code
Add CSV export for layer recipes
layerindex/urls.py | 8 +------- layerindex/urls_branch.py | 5 ++++- layerindex/views.py | 29 ++++++++++++++++++++++------- templates/layerindex/detail.html | 14 +++++++------- templates/layerindex/rawrecipes.txt | 2 -- 5 files changed, 34 insertions(+), 24 deletions(-) delete mode 100644 templates/layerindex/rawrecipes.txt
Signature · 1 line
2.14.3
PATCH 1/2 Drop old raw recipe export code
We're about to replace this with a proper CSV export function, so we don't need this dead code hanging around anymore.
Signed-off-by Paul Eggleton <[email protected]>
--- layerindex/urls.py | 8 +------- layerindex/urls_branch.py | 2 +- layerindex/views.py | 7 ------- templates/layerindex/rawrecipes.txt | 2 -- 4 files changed, 2 insertions(+), 17 deletions(-) delete mode 100644 templates/layerindex/rawrecipes.txt
layerindex/urls.py +1 -7
@@ -8,7 +8,7 @@ from django.conf.urls import * from django.views.generic import TemplateView, DetailView, ListView, RedirectView from django.views.defaults import page_not_found from django.core.urlresolvers import reverse_lazy -from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, UpdateDetailView, StatsView, publish_view +from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, UpdateDetailView, StatsView, publish_view from layerindex.models import LayerItem, Recipe, RecipeChangeset from rest_framework import routers from . import restviews
@@ -101,12 +101,6 @@ urlpatterns = [ name="bulk_change_delete"), url(r'^branch/(?P<branch>[-\w]+)/', include('layerindex.urls_branch')), - #url(r'^raw/recipes.txt$', - # PlainTextListView.as_view( - # queryset=Recipe.objects.order_by('pn', 'layerbranch__layer'), - # context_object_name='recipe_list', - # template_name='layerindex/rawrecipes.txt'), - # name='recipe_list_raw'), url(r'^updates/$', UpdateListView.as_view( template_name='layerindex/updatelist.html'),
layerindex/urls_branch.py +1 -1
@@ -7,7 +7,7 @@ from django.conf.urls import * from django.views.defaults import page_not_found from django.core.urlresolvers import reverse_lazy -from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView +from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView urlpatterns = [ url(r'^$',
layerindex/views.py +0 -7
@@ -771,13 +771,6 @@ class ClassSearchView(ListView): context['this_url_name'] = resolve(self.request.path_info).url_name return context -class PlainTextListView(ListView): - def render_to_response(self, context): - "Returns a plain text response rendering of the template" - template = get_template(self.template_name) - return HttpResponse(template.render(context), - content_type='text/plain') - class HistoryListView(ListView): context_object_name = "revisions" paginate_by = 50
templates/layerindex/rawrecipes.txt deleted +0 -2
@@ -1,2 +0,0 @@ -{% for recipe in recipe_list %}{{ recipe.layerbranch.layer.name }} {{ recipe.pn }} {{ recipe.pv }} {{ recipe.full_path }} -{% endfor %}
Signature · 1 line
2.14.3
PATCH 2/2 Add CSV export for layer recipes
In reply to Paul EggletonAdd the ability to export the recipe listing for a layer to a CSV file for importing into external tools. At the moment we include name, version and license, but there is a parameter that lets you specify the fields to include in the URL if desired.
Implements [YOCTO #12722].
Signed-off-by Paul Eggleton <[email protected]>
--- layerindex/urls_branch.py | 5 ++++- layerindex/views.py | 22 ++++++++++++++++++++++ templates/layerindex/detail.html | 14 +++++++------- 3 files changed, 33 insertions(+), 8 deletions(-)
layerindex/urls_branch.py +4 -1
@@ -7,7 +7,7 @@ from django.conf.urls import * from django.views.defaults import page_not_found from django.core.urlresolvers import reverse_lazy -from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView +from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView, layer_export_recipes_csv_view urlpatterns = [ url(r'^$',
@@ -20,6 +20,9 @@ urlpatterns = [ LayerDetailView.as_view( template_name='layerindex/detail.html'), name='layer_item'), + url(r'^layer/(?P<slug>[-\w]+)/recipes/csv/$', + layer_export_recipes_csv_view, + name='layer_export_recipes_csv'), url(r'^recipes/$', RecipeSearchView.as_view( template_name='layerindex/recipes.html'),
layerindex/views.py +22 -0
@@ -1022,3 +1022,25 @@ class StatsView(TemplateView): machine_count=Count('layerbranch__machine', distinct=True), distro_count=Count('layerbranch__distro', distinct=True)) return context + + +def layer_export_recipes_csv_view(request, branch, slug): + import csv + layer = get_object_or_404(LayerItem, name=slug) + layerbranch = layer.get_layerbranch(branch) + + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename="recipes_%s_%s.csv"' % (layer.name, layerbranch.branch.name) + + fieldlist = request.GET.get('fields', 'pn,pv,license').split(',') + recipe_fields = [f.name for f in Recipe._meta.get_fields() if not (f.auto_created and f.is_relation)] + for field in fieldlist: + if field not in recipe_fields: + return HttpResponse('Field %s is invalid' % field) + + writer = csv.writer(response) + for recipe in layerbranch.sorted_recipes(): + values = [getattr(recipe, field) for field in fieldlist] + writer.writerow(values) + + return response
templates/layerindex/detail.html +7 -7
@@ -199,13 +199,13 @@ <div class="navbar-inner"> <a class="brand pull-left">{{ layeritem.name }} recipes <span class="muted">({{ layerbranch.sorted_recipes.count }})</span></a> - <ul class="nav pull-right"> - <li> - <form action="" class="navbar-search pull-right" id="filter-form"> - <input type="text" placeholder="Search recipes" class="search-query" id="filter"> - </form> - </li> - </ul> + <div class="pull-right"> + <a href="{% url 'layer_export_recipes_csv' layerbranch.branch.name layerbranch.layer.name %}" class="btn"><i class="icon-file"></i> Export CSV</a> + + <form action="" class="navbar-search nav-spacer pull-right" id="filter-form"> + <input type="text" placeholder="Search recipes" class="search-query" id="filter"> + </form> + </div> </div> </div>
Signature · 1 line
2.14.3
Re: [layerindex-web][PATCH 2/2] Add CSV export for layer recipes
Quoted 6 lines · Paul Eggleton
Add the ability to export the recipe listing for a layer to a CSV file for importing into external tools. At the moment we include name, version and license, but there is a parameter that lets you specify the fields to include in the URL if desired.
Implements [YOCTO #12722].
With the bitbake layer library I'm working on, it should be possible to do this directly out of bitbake.
This wouldn't limit it to just recipes either, but any data. Loading (querying) the json data from the layer index is easy, and the library would know how to filter to the right branch/layer/etc.
Something to consider as we go forward with this and other work.
--Mark
Quoted 86 lines
Signed-off-by Paul Eggleton <[email protected]>--- layerindex/urls_branch.py | 5 ++++- layerindex/views.py | 22 ++++++++++++++++++++++ templates/layerindex/detail.html | 14 +++++++------- 3 files changed, 33 insertions(+), 8 deletions(-)layerindex/urls_branch.py +4
-1@@ -7,7 +7,7 @@ from django.conf.urls import * from django.views.defaults import page_not_found from django.core.urlresolvers import reverse_lazy -from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView +from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView, layer_export_recipes_csv_view urlpatterns = [ url(r'^$',@@ -20,6 +20,9 @@ urlpatterns = [ LayerDetailView.as_view( template_name='layerindex/detail.html'), name='layer_item'), + url(r'^layer/(?P<slug>[-\w]+)/recipes/csv/$', + layer_export_recipes_csv_view, + name='layer_export_recipes_csv'), url(r'^recipes/$', RecipeSearchView.as_view( template_name='layerindex/recipes.html'),layerindex/views.py +22
-0@@ -1022,3 +1022,25 @@ class StatsView(TemplateView): machine_count=Count('layerbranch__machine', distinct=True), distro_count=Count('layerbranch__distro', distinct=True)) return context + + +def layer_export_recipes_csv_view(request, branch, slug): + import csv + layer = get_object_or_404(LayerItem, name=slug) + layerbranch = layer.get_layerbranch(branch) + + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename="recipes_%s_%s.csv"' % (layer.name, layerbranch.branch.name) + + fieldlist = request.GET.get('fields', 'pn,pv,license').split(',') + recipe_fields = [f.name for f in Recipe._meta.get_fields() if not (f.auto_created and f.is_relation)] + for field in fieldlist: + if field not in recipe_fields: + return HttpResponse('Field %s is invalid' % field) + + writer = csv.writer(response) + for recipe in layerbranch.sorted_recipes(): + values = [getattr(recipe, field) for field in fieldlist] + writer.writerow(values) + + return responsetemplates/layerindex/detail.html +7
-7@@ -199,13 +199,13 @@ <div class="navbar-inner"> <a class="brand pull-left">{{ layeritem.name }} recipes <span class="muted">({{ layerbranch.sorted_recipes.count }})</span></a> - <ul class="nav pull-right"> - <li> - <form action="" class="navbar-search pull-right" id="filter-form"> - <input type="text" placeholder="Search recipes" class="search-query" id="filter"> - </form> - </li> - </ul> + <div class="pull-right"> + <a href="{% url 'layer_export_recipes_csv' layerbranch.branch.name layerbranch.layer.name %}" class="btn"><i class="icon-file"></i> Export CSV</a> + + <form action="" class="navbar-search nav-spacer pull-right" id="filter-form"> + <input type="text" placeholder="Search recipes" class="search-query" id="filter"> + </form> + </div> </div> </div>