vm: add refresh credential node operation, extend migrate operation

storage, dashboard: automatic credential refresh if ceph specific attributes changes,
minimal refactor on DataStore

circle/dashboard/fixtures/test-vm-fixture.json

@@ -23,7 +23,7 @@
     "endpoints": [],
     "type": "file",
     "ceph_user": null,
-    "secret_uuid": null,
+    "secret": null,
     "path": "/disks", 
     "hostname": "wut", 
     "name": "diszkek"

circle/dashboard/forms.py

@@ -1661,7 +1661,7 @@ class CephDataStoreForm(DataStoreForm):
             Fieldset(
                 '',
                 'ceph_user',
-                'secret_uuid',
+                'secret',
             )
         )
         return helper
@@ -1669,7 +1669,7 @@ class CephDataStoreForm(DataStoreForm):
     class Meta:
         model = DataStore
         fields = ("type", "name", "path", "hostname",
-                  "ceph_user", "secret_uuid", "endpoints")
+                  "ceph_user", "secret", "endpoints")
         widgets = {"endpoints": FilteredSelectMultiple(_("Endpoints"),
                                                        is_stacked=True)}
 

circle/dashboard/templates/dashboard/storage/form_chunk.html

@@ -17,7 +17,7 @@
 <fieldset>
   <legend>{% trans "Ceph block storage authentication settings" %}</legend>
     {{ form.ceph_user|as_crispy_field }}
-    {{ form.secret_uuid|as_crispy_field }}
+    {{ form.secret|as_crispy_field }}
 </fieldset>
 <fieldset>
   <legend>{% trans "Select or add new Ceph monitor endpoints(s)" %}</legend>

circle/dashboard/views/storage.py

@@ -47,6 +47,8 @@ from .util import FilterMixin
 import json
 from celery.exceptions import TimeoutError
 
+from vm.models import Node
+
 logger = logging.getLogger(__name__)
 
 
@@ -282,6 +284,23 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
         ds = self.get_object()
         return reverse("dashboard.views.storage-detail", kwargs={"pk": ds.id})
 
+    def form_valid(self, form):
+        # automatic credential refresh
+        changed = False
+        if self.object.type == "ceph_block":
+            changed = (self.object.tracker.has_changed("secret")
+                       or self.object.tracker.has_changed("ceph_user"))
+        response = super(StorageDetail, self).form_valid(form)
+        if changed:
+            nodes = Node.objects.all()
+            for node in nodes:
+                if node.get_online():
+                    node.refresh_credential(
+                        user=self.request.user,
+                        username=self.object.ceph_user,
+                        secret=self.object.secret)
+        return response
+
 
 class StorageDelete(SuperuserRequiredMixin, DeleteView):
     model = DataStore

circle/storage/migrations/0008_auto_20160609_2338.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('storage', '0007_datastore_destroyed'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='datastore',
+            name='secret_uuid',
+        ),
+        migrations.AddField(
+            model_name='datastore',
+            name='secret',
+            field=models.CharField(max_length=255, null=True, verbose_name='secret key', blank=True),
+        ),
+    ]

circle/storage/models.py

@@ -33,6 +33,7 @@ from django.core.urlresolvers import reverse
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _, ugettext_noop
 from model_utils.models import TimeStampedModel
+from model_utils import FieldTracker
 from sizefield.models import FileSizeField
 
 from .tasks import local_tasks, storage_tasks
@@ -88,10 +89,12 @@ class DataStore(Model):
                                 verbose_name=_('endpoints'))
     ceph_user = CharField(max_length=255, null=True, blank=True,
                           verbose_name=_('Ceph username'))
-    secret_uuid = CharField(max_length=255, null=True, blank=True,
-                            verbose_name=_('uuid of secret key'))
+    secret = CharField(max_length=255, null=True, blank=True,
+                       verbose_name=_('secret key'))
     destroyed = DateTimeField(blank=True, default=None, null=True)
 
+    tracker = FieldTracker(fields=["ceph_user", "secret"])
+
     class Meta:
         ordering = ['name']
         verbose_name = _('datastore')
@@ -480,7 +483,7 @@ class Disk(TimeStampedModel):
         desc = self.get_vmdisk_desc_for_filesystem()
         desc["endpoints"] = self.datastore.get_endpoints()
         desc["ceph_user"] = self.datastore.ceph_user
-        desc["secret_uuid"] = self.datastore.secret_uuid
+        desc["secret"] = self.datastore.secret
 
         return desc
 

circle/vm/models/node.py

@@ -333,7 +333,6 @@ class Node(OperatedMixin, TimeStampedModel):
         try:
             logger.info('%s %s', settings.GRAPHITE_URL, params)
             response = requests.get(settings.GRAPHITE_URL, params=params)
-
             retval = {}
             for target in response.json():
                 # Example:
@@ -364,12 +363,12 @@ class Node(OperatedMixin, TimeStampedModel):
     @property
     @node_available
     def cpu_usage(self):
-        return self.monitor_info.get('cpu.percent') / 100
+        return self.monitor_info.get('cpu.percent', 0) / 100
 
     @property
     @node_available
     def ram_usage(self):
-        return self.monitor_info.get('memory.usage') / 100
+        return self.monitor_info.get('memory.usage', 0) / 100
 
     @property
     @node_available

circle/vm/operations.py

@@ -562,7 +562,7 @@ class MigrateOperation(RemoteInstanceOperation):
                 "redeploy network (rollback)")):
             self.instance.deploy_net()
 
-    def _operation(self, activity, to_node=None, live_migration=True):
+    def _operation(self, activity, user, to_node=None, live_migration=True):
         if not to_node:
             with activity.sub_activity('scheduling',
                                        readable_name=ugettext_noop(
@@ -572,6 +572,17 @@ class MigrateOperation(RemoteInstanceOperation):
 
         try:
             with activity.sub_activity(
+                'refresh_credential', readable_name=create_readable(
+                    ugettext_noop("refresh credential on %(node)s"),
+                    node=to_node)):
+                ceph_blocks = self.instance.disks.filter(
+                    datastore__type="ceph_block")
+                if ceph_blocks.exists():
+                    ds = ceph_blocks[0].datastore
+                    to_node.refresh_credential(user=user,
+                                               username=ds.ceph_user,
+                                               secret=ds.secret)
+            with activity.sub_activity(
                 'migrate_vm', readable_name=create_readable(
                     ugettext_noop("migrate to %(node)s"), node=to_node)):
                 super(MigrateOperation, self)._operation(
@@ -1143,6 +1154,14 @@ class NodeOperation(Operation):
                 user=user, readable_name=name)
 
 
+class RemoteNodeOperation(RemoteOperationMixin, NodeOperation):
+
+    remote_queue = ('vm', 'fast')
+
+    def _get_remote_queue(self):
+        return self.node.get_remote_queue_name(*self.remote_queue)
+
+
 @register_operation
 class ResetNodeOperation(NodeOperation):
     id = 'reset'
@@ -1333,6 +1352,22 @@ class UpdateNodeOperation(NodeOperation):
 
 
 @register_operation
+class RefreshCredentialOperation(RemoteNodeOperation):
+    id = 'refresh_credential'
+    name = _("refresh credential")
+    description = _("Refresh credential.")
+    required_perms = ()
+    task = vm_tasks.refresh_credential
+
+    def _get_remote_args(self, **kwargs):
+        return [kwargs["username"], kwargs["secret"]]
+
+    def _operation(self, activity, username, secret):
+        super(RefreshCredentialOperation, self)._operation(
+            username=username, secret=secret)
+
+
+@register_operation
 class ScreenshotOperation(RemoteInstanceOperation):
     id = 'screenshot'
     name = _("screenshot")

circle/vm/tasks/vm_tasks.py

@@ -185,3 +185,8 @@ def get_node_metrics(params):
 @celery.task(name='vmdriver.screenshot')
 def screenshot(params):
     pass
+
+
+@celery.task(name='vmdriver.refresh_secret')
+def refresh_credential(user, secret):
+    pass

circle/vm/tests/test_models.py

@@ -36,6 +36,15 @@ from ..operations import (
 )
 
 
+class DiskQuerySet(list):
+
+    def filter(self, *args, **kwargs):
+        return DiskQuerySet()
+
+    def exists(self):
+        return False
+
+
 class PortFinderTestCase(TestCase):
 
     def test_find_unused_port_without_used_ports(self):
@@ -106,6 +115,7 @@ class InstanceTestCase(TestCase):
         inst = Mock(destroyed_at=None, spec=Instance)
         inst.interface_set.all.return_value = []
         inst.node = MagicMock(spec=Node)
+        inst.disks = DiskQuerySet()
         inst.status = 'RUNNING'
         migrate_op = MigrateOperation(inst)
         with patch('vm.operations.vm_tasks.migrate') as migr, \
@@ -124,6 +134,7 @@ class InstanceTestCase(TestCase):
         inst.interface_set.all.return_value = []
         inst.node = MagicMock(spec=Node)
         inst.status = 'RUNNING'
+        inst.disks = DiskQuerySet()
         migrate_op = MigrateOperation(inst)
         with patch('vm.operations.vm_tasks.migrate') as migr, \
                 patch.object(RemoteOperationMixin, "_operation"):

circle/vm/tests/test_operations.py

@@ -25,6 +25,7 @@ from vm.operations import (
     RebootOperation, ResetOperation, SaveAsTemplateOperation,
     ShutdownOperation, ShutOffOperation, SleepOperation, WakeUpOperation,
 )
+from test_models import DiskQuerySet
 
 
 class DeployOperationTestCase(TestCase):
@@ -55,9 +56,10 @@ class MigrateOperationTestCase(TestCase):
         op = MigrateOperation(inst)
         op._get_remote_args = MagicMock(side_effect=MigrateException())
         inst.select_node = MagicMock(return_value='test')
+        inst.disks = DiskQuerySet()
         self.assertRaises(
             MigrateException, op._operation,
-            act, to_node=None)
+            act, user=None, to_node=None)
         assert inst.select_node.called
         op._get_remote_args.assert_called_once_with(
             to_node='test', live_migration=True)