Merge branch 'feature-node-ordering' into 'master'

Natural ordering of hostnames

ready to merge

circle/common/models.py

+from collections import deque
 from hashlib import sha224
 from hashlib import sha224
 from logging import getLogger
 from logging import getLogger
 from time import time
 from time import time
@@ -139,3 +140,91 @@ def method_cache(memcached_seconds=60, instance_seconds=5):  # noqa
         return x
         return x
 
 
     return inner_cache
     return inner_cache
+
+
+class HumanSortField(CharField):
+    """
+    A CharField that monitors another field on the same model and sets itself
+    to a normalized value, which can be used for sensible lexicographycal
+    sorting for fields containing numerals. (Avoiding technically correct
+    orderings like [a1, a10, a2], which can be annoying for file or host
+    names.)
+
+    Apart from CharField's default arguments, an argument is requered:
+        - monitor   sets the base field, whose value is normalized.
+        - maximum_number_length    can also be provided, and defaults to 4. If
+        you have to sort values containing numbers greater than 9999, you
+        should increase it.
+
+    Code is based on carljm's django-model-utils.
+    """
+    def __init__(self, *args, **kwargs):
+        logger.debug('Initing HumanSortField(%s %s)',
+                     unicode(args), unicode(kwargs))
+        kwargs.setdefault('default', "")
+        self.maximum_number_length = kwargs.pop('maximum_number_length', 4)
+        monitor = kwargs.pop('monitor', None)
+        if not monitor:
+            raise TypeError(
+                '%s requires a "monitor" argument' % self.__class__.__name__)
+        self.monitor = monitor
+        kwargs['blank'] = True
+        super(HumanSortField, self).__init__(*args, **kwargs)
+
+    def get_monitored_value(self, instance):
+        return getattr(instance, self.monitor)
+
+    @staticmethod
+    def _partition(s, pred):
+        """Partition a deque of chars to a tuple of a
+           - string of the longest prefix matching pred,
+           - string of the longest prefix after the former one not matching,
+           - deque of the remaining characters.
+
+           >>> HumanSortField._partition(deque("1234abc567"),
+           ...                           lambda s: s.isdigit())
+           ('1234', 'abc', deque(['5', '6', '7']))
+           >>> HumanSortField._partition(deque("12ab"), lambda s: s.isalpha())
+           ('', '12', deque(['a', 'b']))
+        """
+        match, notmatch = deque(), deque()
+        while s and pred(s[0]):
+            match.append(s.popleft())
+        while s and not pred(s[0]):
+            notmatch.append(s.popleft())
+        return (''.join(match), ''.join(notmatch), s)
+
+    def get_normalized_value(self, val):
+        logger.debug('Normalizing value: %s', val)
+        norm = ""
+        val = deque(val)
+        while val:
+            numbers, letters, val = self._partition(val,
+                                                    lambda s: s[0].isdigit())
+            if numbers:
+                norm += numbers.rjust(self.maximum_number_length, '0')
+            norm += letters
+        logger.debug('Normalized value: %s', norm)
+        return norm
+
+    def pre_save(self, model_instance, add):
+        logger.debug('Pre-saving %s.%s. %s',
+                     model_instance, self.attname, add)
+        value = self.get_normalized_value(
+            self.get_monitored_value(model_instance))
+        setattr(model_instance, self.attname, value[:self.max_length])
+        return super(HumanSortField, self).pre_save(model_instance, add)
+
+# allow South to handle these fields smoothly
+try:
+    from south.modelsinspector import add_introspection_rules
+    add_introspection_rules(rules=[
+        (
+            (HumanSortField,),
+            [],
+            {'monitor': ('monitor', {}),
+             'maximum_number_length': ('maximum_number_length', {}), }
+        ),
+    ], patterns=['common\.models\.'])
+except ImportError:
+    pass

circle/common/tests/test_models.py

+from collections import deque
+
 from django.test import TestCase
 from django.test import TestCase
+from mock import MagicMock
+
 from .models import TestClass
 from .models import TestClass
+from ..models import HumanSortField
 
 
 
 
 class MethodCacheTestCase(TestCase):
 class MethodCacheTestCase(TestCase):
@@ -28,3 +33,33 @@ class MethodCacheTestCase(TestCase):
         t1.method('a')
         t1.method('a')
         self.assertEqual(val1a, val1b)
         self.assertEqual(val1a, val1b)
         self.assertEqual(t1.called, 2)
         self.assertEqual(t1.called, 2)
+
+
+class TestHumanSortField(TestCase):
+
+    def test_partition(self):
+        values = {(lambda s: s.isdigit(), "1234abc56"): ("1234", "abc", "56"),
+                  (lambda s: s.isalpha(), "abc567"): ("abc", "567", ""),
+                  (lambda s: s == "a", "aaababaa"): ("aaa", "b", "abaa"),
+                  (lambda s: s == "a", u"aaababaa"): ("aaa", "b", "abaa"),
+                  }
+        for (pred, val), result in values.iteritems():
+            a, b, c = HumanSortField._partition(deque(val), pred)
+            assert isinstance(c, deque)
+            c = ''.join(c)
+            # print "%s, %s => %s" % (val, str(pred), str((a, b, c)))
+            self.assertEquals((a, b, c), result)
+
+    def test_get_normalized(self):
+        values = {("1234abc56", 4): "1234abc0056",
+                  ("abc567", 2): "abc567",
+                  ("aaababaa", 8): "aaababaa",
+                  ("aa4ababaa", 2): "aa04ababaa",
+                  ("aa4aba24baa4", 4): "aa0004aba0024baa0004",
+                  }
+        for (val, length), result in values.iteritems():
+            obj = MagicMock(spec=HumanSortField, maximum_number_length=length,
+                            _partition=HumanSortField._partition)
+
+            test_result = HumanSortField.get_normalized_value(obj, val)
+            self.assertEquals(test_result, result)

circle/dashboard/tables.py

@@ -66,7 +66,8 @@ class NodeListTable(Table):
     )
     )
 
 
     name = TemplateColumn(
     name = TemplateColumn(
-        template_name="dashboard/node-list/column-name.html"
+        template_name="dashboard/node-list/column-name.html",
+        order_by="normalized_name"
     )
     )
 
 
     priority = Column(
     priority = Column(

circle/firewall/models.py

@@ -17,6 +17,7 @@ import django.conf
 from django.db.models.signals import post_save, post_delete
 from django.db.models.signals import post_save, post_delete
 import random
 import random
 
 
+from common.models import HumanSortField
 from firewall.tasks.local_tasks import reloadtask
 from firewall.tasks.local_tasks import reloadtask
 from acl.models import AclBase
 from acl.models import AclBase
 logger = logging.getLogger(__name__)
 logger = logging.getLogger(__name__)
@@ -387,6 +388,7 @@ class Host(models.Model):
                                             'the host, the first part of '
                                             'the host, the first part of '
                                             'the FQDN.'),
                                             'the FQDN.'),
                                 validators=[val_alfanum])
                                 validators=[val_alfanum])
+    normalized_hostname = HumanSortField(monitor='hostname', max_length=80)
     reverse = models.CharField(max_length=40, validators=[val_domain],
     reverse = models.CharField(max_length=40, validators=[val_domain],
                                verbose_name=_('reverse'),
                                verbose_name=_('reverse'),
                                help_text=_('The fully qualified reverse '
                                help_text=_('The fully qualified reverse '
@@ -440,6 +442,7 @@ class Host(models.Model):
 
 
     class Meta(object):
     class Meta(object):
         unique_together = ('hostname', 'vlan')
         unique_together = ('hostname', 'vlan')
+        ordering = ('normalized_hostname', 'vlan')
 
 
     def __unicode__(self):
     def __unicode__(self):
         return self.hostname
         return self.hostname

circle/vm/models/node.py

@@ -11,7 +11,7 @@ from celery.exceptions import TimeoutError
 from model_utils.models import TimeStampedModel
 from model_utils.models import TimeStampedModel
 from taggit.managers import TaggableManager
 from taggit.managers import TaggableManager
 
 
-from common.models import method_cache, WorkerNotFound
+from common.models import method_cache, WorkerNotFound, HumanSortField
 from firewall.models import Host
 from firewall.models import Host
 from ..tasks import vm_tasks, local_tasks
 from ..tasks import vm_tasks, local_tasks
 from .common import Trait
 from .common import Trait
@@ -43,6 +43,7 @@ class Node(TimeStampedModel):
     name = CharField(max_length=50, unique=True,
     name = CharField(max_length=50, unique=True,
                      verbose_name=_('name'),
                      verbose_name=_('name'),
                      help_text=_('Human readable name of node.'))
                      help_text=_('Human readable name of node.'))
+    normalized_name = HumanSortField(monitor='name', max_length=100)
     priority = IntegerField(verbose_name=_('priority'),
     priority = IntegerField(verbose_name=_('priority'),
                             help_text=_('Node usage priority.'))
                             help_text=_('Node usage priority.'))
     host = ForeignKey(Host, verbose_name=_('host'),
     host = ForeignKey(Host, verbose_name=_('host'),
@@ -62,6 +63,7 @@ class Node(TimeStampedModel):
         app_label = 'vm'
         app_label = 'vm'
         db_table = 'vm_node'
         db_table = 'vm_node'
         permissions = ()
         permissions = ()
+        ordering = ('-enabled', 'normalized_name')
 
 
     def __unicode__(self):
     def __unicode__(self):
         return self.name
         return self.name