purge.py: Create script that will purge database
cover purge.py: Create script that will purge database
The following changes since commit 9c9e34d0fdcc543945951e374e75b8678e71c35d:
latest-errors: Use the key value instead of display value for filter (2016-06-23 12:24:17 +0100)
are available in the git repository at:
git://push.yoctoproject.org/error-report-web abrindle/purge_script
Amanda Brindle (1):
purge.py: Create script that will purge database
Post/migrations/0006_buildfailure_referer.py | 19 ++++++++++++++++++ Post/models.py | 10 ++++++++++ Post/purge.py | 29 ++++++++++++++++++++++++++++ Post/views.py | 16 +++++++++++++-- 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 Post/migrations/0006_buildfailure_referer.py create mode 100644 Post/purge.py
Signature · 1 line
2.7.4
PATCH 1/1 purge.py: Create script that will purge database
The script will be run on a regular basis to get rid of old reports that we don't need. This will improve the performance of the application since the database has grown to a huge size.
The script will remove reports older than thirty days if they have not been referred to by a host other than the Error Reporting Tool.
The function details() in views.py will keep trick of the referer when a build failure report is accessed. If there is no referer, we will note that in the database in order to determine how often there is no referer. It's possible that crawlers will access reports with no referer; if that's the case, a lack of a referer should not determine whether we save an older report or not.
Fixes [YOCTO #12332]
--- Post/migrations/0006_buildfailure_referer.py | 19 ++++++++++++++++++ Post/models.py | 10 ++++++++++ Post/purge.py | 29 ++++++++++++++++++++++++++++ Post/views.py | 16 +++++++++++++-- 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 Post/migrations/0006_buildfailure_referer.py create mode 100644 Post/purge.py
Post/migrations/0006_buildfailure_referer.py new file +19 -0
@@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('Post', '0005_build_error_type'), + ] + + operations = [ + migrations.AddField( + model_name='buildfailure', + name='REFERER', + field=models.CharField(default=b'NOT_VISITED', max_length=14, choices=[(b'NO_REFERER', b'no_referer'), (b'OTHER', b'other'), (b'NOT_VISITED', b'not_visited')]), + ), + ]
Post/models.py +10 -0
@@ -59,6 +59,16 @@ class BuildFailure(models.Model): ERROR_DETAILS = models.TextField(max_length=int(settings.MAX_UPLOAD_SIZE)) BUILD = models.ForeignKey(Build) LEV_DISTANCE = models.IntegerField(blank=True, null=True) + REFERER_CHOICES = ( + ('NO_REFERER', 'no_referer'), + ('OTHER', 'other'), + ('NOT_VISITED', 'not_visited') + ) + REFERER = models.CharField( + max_length = 14, + choices = REFERER_CHOICES, + default = 'NOT_VISITED' + ) def get_similar_fails(self): if self.LEV_DISTANCE is None:
Post/purge.py new file +29 -0
@@ -0,0 +1,29 @@ +from datetime import datetime +from django.utils import timezone +import os +import sys + +def setup_django(): + import django + # Get access to our Django model + newpath = os.path.abspath(os.path.dirname(__file__)) + '/..' + sys.path.append(newpath) + os.environ['DJANGO_SETTINGS_MODULE'] = 'project.settings' + django.setup() + +def main(): + setup_django() + from Post.models import BuildFailure + items = BuildFailure.objects.all() + now = timezone.now() + for item in items: + if item.REFERER == 'OTHER' or item.REFERER == 'NO_REFERER': + continue + difference = now - item.BUILD.DATE + if difference.days > 30: + item.delete() + +if __name__ == "__main__": + main() + +
Post/views.py +14 -2
@@ -22,6 +22,7 @@ from django.http import JsonResponse from django.db.models import Q import json import urllib +from urlparse import urlparse class results_mode(object): LATEST = 0
@@ -256,9 +257,20 @@ def search(request, mode=results_mode.LATEST, **kwargs): def details(request, fail_id): try: - build_failure = BuildFailure.objects.get(id=fail_id) + build_failure = BuildFailure.objects.get(id=fail_id) except ObjectDoesNotExist: - build_failure = None + build_failure = None + try: + referer = urlparse(request.META['HTTP_REFERER']) + referer_hostname = referer.hostname + if referer.port: + referer_hostname += ":" + str(referer.port) + if referer_hostname != request.get_host(): + build_failure.REFERER = 'OTHER' + except KeyError: + # There is no referer + build_failure.REFERER = 'NO_REFERER' + build_failure.save() context = {'detail' : build_failure, 'error_types' : ErrorType }
Signature · 1 line
2.7.4