Bugfix, tests, doc update

new file mode 100644
index 00000000..5abe2c2d
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+# Fix recipes that were moved out
+#
+# Copyright (C) 2017 Intel Corporation
+# Author: Paul Eggleton <[email protected]>
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+
+
+import sys
+import os
+
+sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
+
+import optparse
+import utils
+import logging
+
+class DryRunRollbackException(Exception):
+    pass
+
+logger = utils.logger_create('LayerIndexFixup')
+
+
+
+def main():
+    parser = optparse.OptionParser(
+        usage = """
+    %prog [options""")
+
+    parser.add_option("-n", "--dry-run",
+            help = "Don't write any data back to the database",
+            action="store_true", dest="dryrun")
+    parser.add_option("-d", "--debug",
+            help = "Enable debug output",
+            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
+    parser.add_option("-q", "--quiet",
+            help = "Hide all output except error messages",
+            action="store_const", const=logging.ERROR, dest="loglevel")
+
+    options, args = parser.parse_args(sys.argv)
+
+    utils.setup_django()
+    import settings
+    from layerindex.models import Recipe
+    from django.db import transaction
+
+    logger.setLevel(options.loglevel)
+
+    try:
+        with transaction.atomic():
+            #LayerBranch.objects.filter(layermaintainer__isnull=True).delete()
+            #LayerItem.objects.filter(layerbranch__isnull=True).filter(classic=False).delete()
+            #LayerItem.objects.filter(layerbranch__isnull=True).filter(classic=False).delete()
+            for recipe in Recipe.objects.filter(filepath__startswith='../'):
+                print('Deleting erroneous recipe %s %s' % (recipe.layerbranch, recipe))
+                recipe.delete()
+
+            if options.dryrun:
+                raise DryRunRollbackException()
+    except DryRunRollbackException:
+        pass
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

index d34d8a59..2b39e8ef 100644
@@ -502,6 +502,10 @@ def main():
                         if skip:
                             continue
                         if oldpath.startswith(subdir_start):
+                            if not newpath.startswith(subdir_start):
+                                logger.debug("Treating rename of %s to %s as a delete since new path is outside layer" % (oldpath, newpath))
+                                other_deletes.append(diffitem)
+                                continue
                             (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
                             (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
                             if oldtypename != newtypename:

@@ -684,16 +688,21 @@ def main():
                         # First, check which recipes still exist
                         layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
                         for v in layerrecipe_values:
-                            root = os.path.join(layerdir, v['filepath'])
-                            fullpath = os.path.join(root, v['filename'])
-                            preserve = True
-                            if os.path.exists(fullpath):
-                                for removedir in removedirs:
-                                    if fullpath.startswith(removedir):
-                                        preserve = False
-                                        break
-                            else:
+                            if v['filepath'].startswith('../'):
+                                # FIXME: These recipes were present due to a bug (not handling renames
+                                # to paths outside the layer) - this can be removed at some point in the future
                                 preserve = False
+                            else:
+                                root = os.path.join(layerdir, v['filepath'])
+                                fullpath = os.path.join(root, v['filename'])
+                                if os.path.exists(fullpath):
+                                    preserve = True
+                                    for removedir in removedirs:
+                                        if fullpath.startswith(removedir):
+                                            preserve = False
+                                            break
+                                else:
+                                    preserve = False
 
                             if preserve:
                                 # Recipe still exists, update it

index 0010cdb6..edfc2ac1 100644
@@ -1,4 +1,5 @@
 *.pyc
 *.db3
 *.swp
+.pytest_cache
 venv/

new file mode 100644
index 00000000..c067b059
@@ -0,0 +1,4 @@
+[pytest]
+DJANGO_SETTINGS_MODULE = settings
+norecursedirs = .git layers
+

195 lines not on this page · raw message

index 72dce71d..d1b1f9bd 100644
@@ -29,8 +29,41 @@ In order to make use of this application you will need:
   in settings.py and have access to the database used by the web
   application):
   * Python 2.7.6+ / Python 3.4+ to match with the version of BitBake
-    for the OpenEmbedded branch being parsed
-  * GitPython (python-git) version 2.0 or later
+    for the OpenEmbedded branch being parsed (for modern versions it's
+    Python 3.)
+  * Python dependencies as per requirements.txt (we still need Django
+    etc. here since we interact with the database through Django's ORM.)
+
+Example virtualenv-based setup for the above:
+
+Python's virtualenv provides an easy way to isolate the python dependencies
+of applications such as the layer index. Here's an example of setting up a
+virtualenv for the layer index that's particularly useful for development.
+(This assumes a Debian-based distribution, adjust accordingly for other
+distros).
+
+1. Install required host distro packages (some of these are required by
+   pip to build the dependencies; it's also assumed you want MariaDB as
+   the database backend):
+
+   sudo apt-get install virtualenv libmariadb-client-lgpl-dev build-essential python3-dev libjpeg-dev libz-dev libfreetype6-dev mariadb-server rabbitmq-server
+
+2. Work around path issues (you may not need this):
+
+   sudo ln -s /usr/bin/mariadb_config /usr/bin/mysql_config
+
+3. Create a Python 3 virtualenv (path can be anywhere you like):
+
+   virtualenv -p python3 /path/to/desired/venv
+
+4. Activate the virtualenv:
+
+   . /path/to/desired/venv/bin/activate
+
+5. Install requirements:
+
+   pip install -r requirements.txt
+
 
 Setup instructions: