Merge branch 'feature-django18' into 'master'

Django18 ✅

See merge request !331

circle/acl/models.py

@@ -43,6 +43,7 @@ class Level(Model):
         return "<%s/%s>" % (unicode(self.content_type), self.name)
 
     class Meta:
+        app_label = 'acl'
         unique_together = (('content_type', 'codename'),
                            # ('content_type', 'weight'),
                            # TODO find a way of temp. disabling this constr.
@@ -63,6 +64,7 @@ class ObjectLevel(Model):
         return "<%s: %s>" % (unicode(self.content_object), unicode(self.level))
 
     class Meta:
+        app_label = 'acl'
         unique_together = (('content_type', 'object_id', 'level'),)
 
 
@@ -182,7 +184,7 @@ class AclBase(Model):
     def get_users_with_level(self, **kwargs):
         logger.debug('%s.get_users_with_level() called', unicode(self))
         object_levels = (self.object_level_set.filter(**kwargs).select_related(
-            'users', 'level').all())
+            'level').prefetch_related('users').all())
         users = []
         for object_level in object_levels:
             name = object_level.level.codename
@@ -194,7 +196,7 @@ class AclBase(Model):
     def get_groups_with_level(self):
         logger.debug('%s.get_groups_with_level() called', unicode(self))
         object_levels = (self.object_level_set.select_related(
-            'groups', 'level').all())
+            'level').prefetch_related('groups').all())
         groups = []
         for object_level in object_levels:
             name = object_level.level.codename

circle/circle/settings/base.py

@@ -337,7 +337,6 @@ DJANGO_APPS = (
 THIRD_PARTY_APPS = (
     'django_tables2',
     'crispy_forms',
-    'djcelery',
     'sizefield',
     'taggit',
     'statici18n',
@@ -564,3 +563,5 @@ ADMIN_ENABLED = False
 BLACKLIST_PASSWORD = get_env_variable("BLACKLIST_PASSWORD", "")
 BLACKLIST_HOOK_URL = get_env_variable("BLACKLIST_HOOK_URL", "")
 REQUEST_HOOK_URL = get_env_variable("REQUEST_HOOK_URL", "")
+
+SSHKEY_EMAIL_ADD_KEY = False

circle/circle/settings/selenium_test.py

@@ -14,9 +14,31 @@
 #
 # You should have received a copy of the GNU General Public License along
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
-import os# noqa
+import os
+
 from .base import *  # noqa
-# flake8: noqa
+
+
+# fix https://github.com/django-nose/django-nose/issues/197
+# AttributeError: 'module' object has no attribute 'commit_unless_managed'
+# TypeError: _skip_create_test_db() got an unexpected keyword argument 'keepdb'
+
+from django.db import transaction
+from django_nose import runner
+
+
+def _skip_create_test_db(self, verbosity=1, autoclobber=False, serialize=True,
+                         keepdb=True):
+    return old_skip_create_test_db(
+        self, verbosity=verbosity, autoclobber=autoclobber,
+        serialize=serialize)
+
+
+setattr(transaction, "commit_unless_managed", lambda using: using)
+old_skip_create_test_db = runner._skip_create_test_db
+setattr(runner, "_skip_create_test_db", _skip_create_test_db)
+
+
 os.environ['REUSE_DB'] = "1"
 os.environ['DJANGO_TEST_DB_NAME'] = "circle"
 DATABASES = {
@@ -34,22 +56,23 @@ DATABASES = {
 SOUTH_TESTS_MIGRATE = False
 
 INSTALLED_APPS += (
-        'acl.tests',
-        'django_nose',
-        'django_jenkins',
+    'acl.tests',
+    'django_nose',
+    'django_jenkins',
 )
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
 
 path_to_selenium_test = os.path.join(SITE_ROOT, "dashboard/tests/selenium")
-NOSE_ARGS = ['--stop', '--with-doctest', '--with-selenium-driver', '--selenium-driver=firefox', '-w%s' % path_to_selenium_test]
+NOSE_ARGS = ['--stop', '--with-doctest', '--with-selenium-driver',
+             '--selenium-driver=firefox', '-w%s' % path_to_selenium_test]
 
 PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher']
 
 CACHES = {
-        'default': {
-            'BACKEND': 'django.core.cache.backends.dummy.DummyCache'
-        }
+    'default': {
+        'BACKEND': 'django.core.cache.backends.dummy.DummyCache'
+    }
 }
 
 LOGGING['loggers']['djangosaml2'] = {'handlers': ['console'],

circle/common/models.py

@@ -356,8 +356,6 @@ class HumanSortField(CharField):
     Code is based on carljm's django-model-utils.
     """
     def __init__(self, *args, **kwargs):
-        logger.debug('Initing HumanSortField(%s %s)',
-                     unicode(args), unicode(kwargs))
         kwargs.setdefault('default', "")
         self.maximum_number_length = kwargs.pop('maximum_number_length', 4)
         monitor = kwargs.pop('monitor', None)

circle/dashboard/forms.py

@@ -1326,16 +1326,16 @@ class UserEditForm(forms.ModelForm):
 class AclUserOrGroupAddForm(forms.Form):
     name = forms.CharField(widget=autocomplete_light.TextWidget(
         'AclUserGroupAutocomplete',
-        autocomplete_js_attributes={'placeholder': _("Name of group or user")},
-        attrs={'class': 'form-control'}))
+        attrs={'class': 'form-control',
+               'placeholder': _("Name of group or user")}))
 
 
 class TransferOwnershipForm(forms.Form):
     name = forms.CharField(
         widget=autocomplete_light.TextWidget(
             'AclUserAutocomplete',
-            autocomplete_js_attributes={"placeholder": _("Name of user")},
-            attrs={'class': 'form-control'}),
+            attrs={'class': 'form-control',
+                   'placeholder': _("Name of user")}),
         label=_("E-mail address or identifier of user"))
 
 
@@ -1343,8 +1343,8 @@ class AddGroupMemberForm(forms.Form):
     new_member = forms.CharField(
         widget=autocomplete_light.TextWidget(
             'AclUserAutocomplete',
-            autocomplete_js_attributes={"placeholder": _("Name of user")},
-            attrs={'class': 'form-control'}),
+            attrs={'class': 'form-control',
+                   'placeholder': _("Name of user")}),
         label=_("E-mail address or identifier of user"))
 
 

circle/dashboard/tests/test_templates.py

@@ -19,9 +19,8 @@ from os import listdir
 from os.path import isfile, isdir, join
 import unittest
 
-from django.conf import settings
 from django.template import Template, Context, VariableDoesNotExist
-from django.template.loader import find_template_loader
+from django.template.engine import Engine
 from django.core.urlresolvers import NoReverseMatch
 
 
@@ -29,9 +28,8 @@ class TemplateSyntaxTestCase(unittest.TestCase):
 
     def test_templates(self):
         """Test all templates for syntax errors."""
-        for loader_name in settings.TEMPLATE_LOADERS:
-            print loader_name
-            loader = find_template_loader(loader_name)
+        for loader in Engine.get_default().template_loaders:
+            print loader
             self._test_dir(loader.get_template_sources(''))
 
     def _test_dir(self, dir, path="/"):

circle/dashboard/tests/test_views.py

@@ -1706,7 +1706,11 @@ class SshKeyTest(LoginMixin, TestCase):
         self.u2.set_password('password')
         self.u2.profile = Profile()
         self.u2.save()
-        self.k1 = UserKey(key='ssh-rsa AAAAB3NzaC1yc2EC asd', user=self.u1)
+        self.valid_key = (
+            "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAYQDGqQy86fpVL3fAPE9ExTSvg4"
+            "Me7bpzH4azerTwWl8u9KKbhYe8XnC+cpvzbRxinFE9SqgQtKJzuxE0f/hHsNCQ"
+            "t3zDLHsqfFUdFQzkImXJ+duUKGyHKIsx6Os0j6nl+3c= asd")
+        self.k1 = UserKey(key=self.valid_key, user=self.u1)
         self.k1.save()
 
     def tearDown(self):
@@ -1719,7 +1723,7 @@ class SshKeyTest(LoginMixin, TestCase):
         self.login(c, self.u1)
 
         resp = c.post("/dashboard/sshkey/1/",
-                      {'key': 'ssh-rsa AAAAB3NzaC1yc2EC'})
+                      {'key': self.valid_key})
         self.assertEqual(UserKey.objects.get(id=1).user, self.u1)
         self.assertEqual(200, resp.status_code)
 
@@ -1728,7 +1732,7 @@ class SshKeyTest(LoginMixin, TestCase):
         self.login(c, self.u2)
 
         resp = c.post("/dashboard/sshkey/1/",
-                      {'key': 'ssh-rsa AAAAB3NzaC1yc2EC'})
+                      {'key': self.valid_key})
         self.assertEqual(UserKey.objects.get(id=1).user, self.u1)
         self.assertEqual(403, resp.status_code)
 
@@ -1737,7 +1741,7 @@ class SshKeyTest(LoginMixin, TestCase):
         self.login(c, self.u1)
 
         resp = c.post("/dashboard/sshkey/create/",
-                      {'name': 'asd', 'key': 'ssh-rsa AAAAB3NzaC1yc2EC'})
+                      {'name': 'asd', 'key': self.valid_key})
         self.assertEqual(UserKey.objects.get(id=2).user, self.u1)
         self.assertEqual(302, resp.status_code)
 

circle/dashboard/views/vm.py

@@ -1109,8 +1109,10 @@ class VmCreate(LoginRequiredMixin, TemplateView):
         return self.__deploy(request, instances)
 
     def __deploy(self, request, instances, *args, **kwargs):
+        # workaround EncodeError: dictionary changed size during iteration
+        user = User.objects.get(pk=request.user.pk)
         for i in instances:
-            i.deploy.async(user=request.user)
+            i.deploy.async(user=user)
 
         if len(instances) > 1:
             messages.success(request, ungettext_lazy(

circle/fabfile.py

@@ -75,7 +75,7 @@ def flake8():
 def migrate():
     "Run db migrations"
     with _workon("circle"), cd("~/circle/circle"):
-        run("./manage.py migrate")
+        run("./manage.py migrate --fake-initial")
 
 
 @roles('portal')
@@ -166,6 +166,7 @@ def update_portal(test=False, git=True):
             pull()
         cleanup()
         pip("circle", "~/circle/requirements.txt")
+        sudo("cp ~/circle/miscellaneous/*celery.conf /etc/init/")
         bower()
         migrate()
         compile_things()

circle/firewall/migrations/0005_auto_20150520_2250.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('firewall', '0004_auto_20150318_1317'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='host',
+            name='groups',
+            field=models.ManyToManyField(help_text='Host groups the machine is part of.', to='firewall.Group', verbose_name='groups', blank=True),
+        ),
+        migrations.AlterField(
+            model_name='vlan',
+            name='snat_to',
+            field=models.ManyToManyField(help_text='Connections to these networks should be network address translated, i.e. their source address is rewritten to the value of NAT IP address.', to='firewall.Vlan', verbose_name='NAT to', blank=True),
+        ),
+        migrations.AlterField(
+            model_name='vlangroup',
+            name='vlans',
+            field=models.ManyToManyField(help_text='The vlans which are members of the group.', to='firewall.Vlan', verbose_name='vlans', blank=True),
+        ),
+    ]

circle/firewall/models.py

@@ -255,6 +255,7 @@ class Rule(models.Model):
         return qs
 
     class Meta:
+        app_label = 'firewall'
         verbose_name = _("rule")
         verbose_name_plural = _("rules")
         ordering = (
@@ -330,7 +331,7 @@ class Vlan(AclBase, models.Model):
                                                'selected below '
                                                '(typically to the internet).'))
     snat_to = models.ManyToManyField('self', symmetrical=False, blank=True,
-                                     null=True, verbose_name=_('NAT to'),
+                                     verbose_name=_('NAT to'),
                                      help_text=_(
                                          'Connections to these networks '
                                          'should be network address '
@@ -392,6 +393,7 @@ class Vlan(AclBase, models.Model):
                                        verbose_name=_('modified at'))
 
     class Meta:
+        app_label = 'firewall'
         verbose_name = _("vlan")
         verbose_name_plural = _("vlans")
         ordering = ('vid', )
@@ -533,7 +535,7 @@ class VlanGroup(models.Model):
     name = models.CharField(max_length=20, unique=True, verbose_name=_('name'),
                             help_text=_('The name of the group.'))
     vlans = models.ManyToManyField('Vlan', symmetrical=False, blank=True,
-                                   null=True, verbose_name=_('vlans'),
+                                   verbose_name=_('vlans'),
                                    help_text=_('The vlans which are members '
                                                'of the group.'))
     description = models.TextField(blank=True, verbose_name=_('description'),
@@ -546,6 +548,7 @@ class VlanGroup(models.Model):
                                        verbose_name=_('modified at'))
 
     class Meta:
+        app_label = 'firewall'
         verbose_name = _("vlan group")
         verbose_name_plural = _("vlan groups")
         ordering = ('id', )
@@ -573,6 +576,7 @@ class Group(models.Model):
                                        verbose_name=_('modified at'))
 
     class Meta:
+        app_label = 'firewall'
         verbose_name = _("host group")
         verbose_name_plural = _("host groups")
         ordering = ('id', )
@@ -639,7 +643,7 @@ class Host(models.Model):
                               help_text=_(
                                   'The person responsible for this host.'))
     groups = models.ManyToManyField('Group', symmetrical=False, blank=True,
-                                    null=True, verbose_name=_('groups'),
+                                    verbose_name=_('groups'),
                                     help_text=_(
                                         'Host groups the machine is part of.'))
     created_at = models.DateTimeField(auto_now_add=True,
@@ -648,6 +652,7 @@ class Host(models.Model):
                                        verbose_name=_('modified at'))
 
     class Meta(object):
+        app_label = 'firewall'
         unique_together = ('hostname', 'vlan')
         ordering = ('normalized_hostname', 'vlan')
 
@@ -951,6 +956,7 @@ class Firewall(models.Model):
                             verbose_name=_('name'))
 
     class Meta:
+        app_label = 'firewall'
         verbose_name = _("firewall")
         verbose_name_plural = _("firewalls")
         ordering = ('id', )
@@ -1002,6 +1008,7 @@ class Domain(models.Model):
     description = models.TextField(blank=True, verbose_name=_('description'))
 
     class Meta:
+        app_label = 'firewall'
         verbose_name = _("domain")
         verbose_name_plural = _("domains")
         ordering = ('id', )
@@ -1083,6 +1090,7 @@ class Record(models.Model):
         return reverse('network.record', kwargs={'pk': self.pk})
 
     class Meta:
+        app_label = 'firewall'
         verbose_name = _("record")
         verbose_name_plural = _("records")
         ordering = (
@@ -1105,6 +1113,7 @@ class SwitchPort(models.Model):
                                        verbose_name=_('modified_at'))
 
     class Meta:
+        app_label = 'firewall'
         verbose_name = _("switch port")
         verbose_name_plural = _("switch ports")
         ordering = ('id', )
@@ -1137,6 +1146,7 @@ class EthernetDevice(models.Model):
                                        verbose_name=_('modified_at'))
 
     class Meta:
+        app_label = 'firewall'
         verbose_name = _("ethernet device")
         verbose_name_plural = _("ethernet devices")
         ordering = ('id', )
@@ -1172,6 +1182,7 @@ class BlacklistItem(models.Model):
         return self.ipv4
 
     class Meta(object):
+        app_label = 'firewall'
         verbose_name = _('blacklist item')
         verbose_name_plural = _('blacklist items')
         ordering = ('id', )

miscellaneous/managercelery@.service

@@ -11,5 +11,5 @@ TimeoutStopSec=600
 Restart=always
 
 WorkingDirectory=/home/cloud/circle/circle
-ExecStartPre=/bin/bash -c "source /etc/profile; workon circle; ./manage.py celery -f --app=manager.%I purge"
-ExecStart=/bin/bash -c "source /etc/profile; workon circle; exec ./manage.py celery --app=manager.%I worker --autoreload --loglevel=info --hostname=%I -B -c 3"
+ExecStartPre=/bin/bash -c "source /etc/profile; workon circle; celery -f --app=manager.%I purge"
+ExecStart=/bin/bash -c "source /etc/profile; workon circle; exec celery --app=manager.%I worker --loglevel=info --hostname=%I -B -c 3"

miscellaneous/mancelery.conf

@@ -9,11 +9,12 @@ setuid cloud
 kill timeout 360
 kill signal SIGTERM
 
+
 script
     cd /home/cloud/circle/circle
     . /home/cloud/.virtualenvs/circle/bin/activate
     . /home/cloud/.virtualenvs/circle/bin/postactivate
-    ./manage.py celery -f --app=manager.mancelery purge
-    exec ./manage.py celery --app=manager.mancelery worker --autoreload --loglevel=info --hostname=mancelery -B -c 3
+    celery -f --app=manager.mancelery purge
+    exec celery --app=manager.mancelery worker --autoreload --loglevel=info --hostname=mancelery -B -c 3
 end script
 

miscellaneous/moncelery.conf

@@ -3,15 +3,18 @@ description     "CIRCLE moncelery for monitoring jobs"
 respawn
 respawn limit 30 30
 
-
 setgid cloud
 setuid cloud
 
+kill timeout 360
+kill signal SIGTERM
+
+
 script
     cd /home/cloud/circle/circle
     . /home/cloud/.virtualenvs/circle/bin/activate
     . /home/cloud/.virtualenvs/circle/bin/postactivate
-    ./manage.py celery -f --app=manager.moncelery purge
-    exec ./manage.py celery --app=manager.moncelery worker --autoreload --loglevel=info --hostname=moncelery -B -c 2
+    celery -f --app=manager.moncelery purge
+    exec celery --app=manager.moncelery worker --autoreload --loglevel=info --hostname=moncelery -B -c 2
 end script
 

miscellaneous/slowcelery.conf

@@ -14,7 +14,7 @@ script
     cd /home/cloud/circle/circle
     . /home/cloud/.virtualenvs/circle/bin/activate
     . /home/cloud/.virtualenvs/circle/bin/postactivate
-    ./manage.py celery -f --app=manager.slowcelery purge
-    exec ./manage.py celery --app=manager.slowcelery worker --autoreload --loglevel=info --hostname=slowcelery -B -c 1
+    celery -f --app=manager.slowcelery purge
+    exec celery --app=manager.slowcelery worker --autoreload --loglevel=info --hostname=slowcelery -B -c 1
 end script
 

requirements/base.txt

 amqp==1.4.6
 anyjson==0.3.3
 arrow==0.5.4
-billiard==3.3.0.18
-bpython==0.13.1
-celery==3.1.17
-Django==1.7.5
-django-appconf==0.6
-django-autocomplete-light==1.4.14
-django-braces==1.4.0
-django-celery==3.1.16
+billiard==3.3.0.20
+bpython==0.14.1
+celery==3.1.18
+Django==1.8.2
+django-appconf==1.0.1
+django-autocomplete-light==2.1.1
+django-braces==1.8.0
 django-crispy-forms==1.4.0
 django-model-utils==2.2
 djangosaml2==0.13.0
-django-sizefield==0.6
-django-sshkey==2.2.0
-django-statici18n==1.1
-django-tables2==0.15.0
-django-taggit==0.13.0
+django-sizefield==0.7
+git+https://git.ik.bme.hu/circle/django-sshkey.git
+django-statici18n==1.1.3
+django-tables2==0.16.0
+django-taggit==0.14.0
 docutils==0.12
 Jinja2==2.7.3
-jsonfield==1.0.0
-kombu==3.0.23
+jsonfield==1.0.3
+kombu==3.0.26
 logutils==0.3.3
 MarkupSafe==0.23
-netaddr==0.7.12
-pip-tools==0.3.5
-psycopg2==2.5.4
-Pygments==1.6
-pylibmc==1.3.0
-python-dateutil==2.2
-pyinotify==0.9.4
-pytz==2014.7
-requests==2.5.3
-salt==2014.1.0
-shutilwhich==1.0.1
-simplejson==3.6.5
-six==1.8.0
+netaddr==0.7.14
+pip-tools==0.3.6
+psycopg2==2.6
+Pygments==2.0.2
+pylibmc==1.4.3
+python-dateutil==2.4.2
+pyinotify==0.9.5
+pytz==2015.4
+requests==2.7.0
+salt==2015.5.1
+shutilwhich==1.1.0
+simplejson==3.7.2
+six==1.9.0
 slimit==0.8.1
-South==0.8.4
-sqlparse==0.1.13
+sqlparse==0.1.15
 pika==0.9.14
-Fabric==1.10.0
-lxml==3.4.0
 django-pipeline==1.4.7

requirements/local.txt

 # Local development dependencies go here
 -r base.txt
 coverage==3.7.1
-django-debug-toolbar==1.2.1
-django-rosetta==0.7.4
-Sphinx==1.2.2
+django-debug-toolbar==1.3.0
+django-rosetta==0.7.6
+Sphinx==1.3.1

requirements/production.txt

 # Pro-tip: Try not to put anything here. There should be no dependency in
 # production that isn't in development.
 -r base.txt
-uWSGI==2.0.8
+uWSGI==2.0.10

requirements/test.txt

@@ -3,8 +3,8 @@
 coverage==3.7.1
 factory-boy==2.4.1
 mock==1.0.1
-django-nose==1.3
-nose==1.3.4
+django-nose==1.4
+nose==1.3.6
 nose-exclude==0.2.0
 selenium==2.45.0
 selenose==1.3