REST API fixes
cover REST API fixes
Some fixes relating to the REST API performance issues experienced with the recent upgrade on layers.openembedded.org.
The following changes since commit 73198bd5483c16b9852121c6855dc6dac96a98a0:
README: add reference to using import_layers to update (2019-08-06 12:04:17 +1200)
are available in the Git repository at:
git://git.yoctoproject.org/layerindex-web paule/restapi-fixes
http://git.yoctoproject.org/cgit.cgi/layerindex-web/log/?h=paule/restapi-fixes
Paul Eggleton (3):
API: fix recipes API performance regression
import_layers: use recipesExtended viewset
import_layers: fix output when importing layers from scratch
layerindex/restviews.py | 25 +++++++- layerindex/tools/import_layers.py | 94 ++++++++++++++++++------------- layerindex/urls.py | 1 + 3 files changed, 79 insertions(+), 41 deletions(-)
Signature · 1 line
2.20.1
PATCH 1/3 API: fix recipes API performance regression
Adding these extra child items to the standard "recipes" viewset (which we did recently in 684a06a383fd2a8da80490dff5f0dbc47750934e) means that some current usage is impractical due to the size of the returned list of items. Instead, create a recipesExtended viewset, move the new child items to that and add pagination to avoid result size issues.
--- layerindex/restviews.py | 25 ++++++++++++++++++++++--- layerindex/urls.py | 1 + 2 files changed, 23 insertions(+), 3 deletions(-)
layerindex/restviews.py +22 -3
@@ -1,7 +1,16 @@ +# OpenEmbedded Layer Index REST API implementation +# +# Copyright (C) 2014, 2016-2019 Intel Corporation +# +# Licensed under the MIT license, see COPYING.MIT for details + from layerindex.models import Branch, LayerItem, LayerMaintainer, YPCompatibleVersion, LayerNote, LayerBranch, LayerDependency, Recipe, Machine, Distro, BBClass, Source, Patch, PackageConfig, StaticBuildDep, DynamicBuildDep, RecipeFileDependency, BBAppend, IncFile -from rest_framework import viewsets, serializers +from rest_framework import viewsets, serializers, pagination from layerindex.querysethelper import params_to_queryset, get_search_tuple +class LayerIndexPagination(pagination.PageNumberPagination): + page_size = 200 + class DynamicFieldsModelSerializer(serializers.ModelSerializer): """ A ModelSerializer that takes an additional "fields" argument that
@@ -114,6 +123,15 @@ class RecipeSerializer(serializers.ModelSerializer): model = Recipe fields = '__all__' +class RecipeViewSet(ParametricSearchableModelViewSet): + queryset = Recipe.objects.all() + serializer_class = RecipeSerializer + +class RecipeExtendedSerializer(serializers.ModelSerializer): + class Meta: + model = Recipe + fields = '__all__' + sources = serializers.SerializerMethodField() patches = serializers.SerializerMethodField() package_configs = serializers.SerializerMethodField()
@@ -143,9 +161,10 @@ class RecipeSerializer(serializers.ModelSerializer): serializer = RecipeFileDependencySerializer(instance=qs, many=True, read_only=True, fields=('layerbranch', 'path')) return serializer.data -class RecipeViewSet(ParametricSearchableModelViewSet): +class RecipeExtendedViewSet(ParametricSearchableModelViewSet): queryset = Recipe.objects.all() - serializer_class = RecipeSerializer + serializer_class = RecipeExtendedSerializer + pagination_class = LayerIndexPagination class MachineSerializer(serializers.ModelSerializer): class Meta:
layerindex/urls.py +1 -0
@@ -28,6 +28,7 @@ router.register(r'layerDependencies', restviews.LayerDependencyViewSet) router.register(r'layerMaintainers', restviews.LayerMaintainerViewSet) router.register(r'layerNotes', restviews.LayerNoteViewSet) router.register(r'recipes', restviews.RecipeViewSet) +router.register(r'recipesExtended', restviews.RecipeExtendedViewSet) router.register(r'machines', restviews.MachineViewSet) router.register(r'distros', restviews.DistroViewSet) router.register(r'classes', restviews.ClassViewSet)
Signature · 1 line
2.20.1
PATCH 2/3 import_layers: use recipesExtended viewset
In reply to Paul EggletonThe new viewset has a different URL and uses pagination, so we need to take that into account.
--- layerindex/tools/import_layers.py | 92 ++++++++++++++++++------------- 1 file changed, 55 insertions(+), 37 deletions(-)
layerindex/tools/import_layers.py +55 -37
@@ -89,7 +89,7 @@ def main(): layerbranches_url = jsdata['layerBranches'] layermaintainers_url = jsdata.get('layerMaintainers', None) layernotes_url = jsdata.get('layerNotes', None) - recipes_url = jsdata.get('recipes', None) + recipes_url = jsdata.get('recipesExtended', None) machines_url = jsdata.get('machines', None) distros_url = jsdata.get('distros', None) classes_url = jsdata.get('classes', None)
@@ -186,55 +186,73 @@ def main(): # The parent field always needs to be part of the keys keys = key_fields + [parentfield] + def fetch_api_url(api_url): + rq = urllib.request.Request(api_url) + data = urllib.request.urlopen(rq).read() + return json.loads(data.decode('utf-8')) + if url: if parent_orig_id is None: raise Exception('import_child_items: if url is specified then parent_orig_id must also be specified') - rq = urllib.request.Request(url + '?filter=%s:%s' % (parentfield, parent_orig_id)) - data = urllib.request.urlopen(rq).read() - childjslist = json.loads(data.decode('utf-8')) + childjsdata = fetch_api_url(url + '?filter=%s:%s' % (parentfield, parent_orig_id)) elif childlist is not None: - childjslist = childlist + childjsdata = childlist else: raise Exception('import_child_items: either url or childlist must be specified') manager = getattr(parentobj, objclass.__name__.lower() + '_set') existing_ids = list(manager.values_list('id', flat=True)) updated_ids = [] - for childjs in childjslist: - vals = {} - for key, value in childjs.items(): - if key in exclude: - continue - vals[key] = value - vals[parentfield] = parentobj - if keys: - keyvals = {k: vals[k] for k in keys} - else: - keyvals = vals - - # In the case of multiple records with the same keys (e.g. multiple recipes with same pn), - # we need to skip ones we've already touched - obj = None - created = False - for entry in manager.filter(**keyvals): - if entry.id not in updated_ids: - obj = entry + def import_list(childjslist): + for childjs in childjslist: + vals = {} + for key, value in childjs.items(): + if key in exclude: + continue + vals[key] = value + vals[parentfield] = parentobj + + if keys: + keyvals = {k: vals[k] for k in keys} + else: + keyvals = vals + + # In the case of multiple records with the same keys (e.g. multiple recipes with same pn), + # we need to skip ones we've already touched + obj = None + created = False + for entry in manager.filter(**keyvals): + if entry.id not in updated_ids: + obj = entry + break + else: + created = True + obj = objclass(**keyvals) + + for key, value in vals.items(): + setattr(obj, key, value) + # Need to have saved before calling custom_field_cb since the function might be adding child objects + obj.save() + updated_ids.append(obj.id) + if custom_field_cb is not None: + custom_field_cb(obj, childjs) + if not created: + if obj.id in existing_ids: + existing_ids.remove(obj.id) + + if 'results' in childjsdata: + while True: + import_list(childjsdata['results']) + if childjsdata.get('next', None): + childjsdata = fetch_api_url(childjsdata['next']) + if not 'results' in childjsdata: + break + else: break - else: - created = True - obj = objclass(**keyvals) + else: + import_list(childjsdata) - for key, value in vals.items(): - setattr(obj, key, value) - # Need to have saved before calling custom_field_cb since the function might be adding child objects - obj.save() - updated_ids.append(obj.id) - if custom_field_cb is not None: - custom_field_cb(obj, childjs) - if not created: - if obj.id in existing_ids: - existing_ids.remove(obj.id) for idv in existing_ids: objclass.objects.filter(id=idv).delete()
Signature · 1 line
2.20.1
PATCH 3/3 import_layers: fix output when importing layers from scratch
In reply to Paul EggletonFixes "Importing None" message since we had not yet initialised the layerbranch variable.
--- layerindex/tools/import_layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
layerindex/tools/import_layers.py +1 -1
@@ -329,10 +329,10 @@ def main(): continue logger.info('Updating %s (%d/%d)' % (layerbranch, i+1, layercount)) else: - logger.info('Importing %s (%d/%d)' % (layerbranch, i+1, layercount)) layerbranch = LayerBranch() layerbranch.branch = branch layerbranch.layer = layer + logger.info('Importing %s (%d/%d)' % (layerbranch, i+1, layercount)) for key, value in layerbranchjs.items():
Signature · 1 line
2.20.1