Merge branch 'feature-node-activity-2' into 'master'

Feature Node Activity

Fixing activity list in node details activity.

circle/dashboard/templates/dashboard/node-detail.html

@@ -9,10 +9,10 @@
   <div class="row">
   <div class="row">
     <div class="col-md-4" id="node-info-pane">
     <div class="col-md-4" id="node-info-pane">
       <div id="node-info-data" class="big">
       <div id="node-info-data" class="big">
-	      <span id="node-status-label" class="label {% if node.state == 'Online' %}label-success
-			{% elif node.state == 'Missing' %}label-danger
-			{% elif node.state == 'Disabled' %}label-warning
-			{% elif node.state == 'Offline' %}label-warning
+      <span id="node-details-state" class="label {% if node.state == 'ONLINE' %}label-success
+			{% elif node.state == 'MISSING' %}label-danger
+			{% elif node.state == 'DISABLED' %}label-warning
+			{% elif node.state == 'OFFLINE' %}label-warning
 			{% endif %}">{{ node.state }}</span>
 			{% endif %}">{{ node.state }}</span>
 		<div class="btn-group">
 		<div class="btn-group">
 			{% with node as record %}
 			{% with node as record %}
@@ -43,10 +43,10 @@
 	</ul>
 	</ul>
 
 
         <div id="panel-body"  class="tab-content panel-body">
         <div id="panel-body"  class="tab-content panel-body">
-          <div class="tab-pane active" id="home">{% include "dashboard/node-detail-home.html" %}</div>
-	  <div class="tab-pane" id="resources">{% include "dashboard/node-detail-resources.html" %}</div>
-          <div class="tab-pane" id="activity">{% include "dashboard/node-detail-activity.html" %}</div>
-          <div class="tab-pane" id="virtualmachines">{% include "dashboard/node-detail-vm.html" %}</div>
+          <div class="tab-pane active" id="home">{% include "dashboard/node-detail/home.html" %}</div>
+	  <div class="tab-pane" id="resources">{% include "dashboard/node-detail/resources.html" %}</div>
+          <div class="tab-pane" id="activity">{% include "dashboard/node-detail/activity.html" %}</div>
+          <div class="tab-pane" id="virtualmachines">{% include "dashboard/node-detail/vm.html" %}</div>
   	</div>
   	</div>
       </div>
       </div>
     </div>
     </div>

circle/dashboard/templates/dashboard/node-detail-activity.html → circle/dashboard/templates/dashboard/node-detail/_activity-timeline.html

 {% load i18n %}
 {% load i18n %}
-<h3>{% trans "Activity" %}</h3>
-<style>
-  .sub-timeline {
-    border-left: 3px solid green;
-    margin-left: 30px;
-    padding-left: 10px;
-  }
-</style>
-
-<div class="timeline">
-  {% for a in activity %}
-  <div class="activity" data-activity-id="{{ a.pk }}">
-    <span class="timeline-icon">
-      <i class="{% if not a.finished %} icon-refresh icon-spin {% else %}icon-plus{% endif %}"></i>
-    </span> 
-    <strong>{{ a.get_readable_name }}</strong>
-    {{ a.started|date:"Y-m-d. H:i" }}, {{ a.user }}
-      {% if a.children.count > 0 %}
-        <div class="sub-timeline">
-          {% for s in a.children.all %}
-            <div data-activity-id="{{ s.pk }}" class="sub-activity">
-            {{ s.get_readable_name }} - 
-            {% if s.finished %}
-              {{ s.finished|time:"H:i:s" }}
-            {% else %}
-              <i class="icon-refresh icon-spin" class="sub-activity-loading-icon"></i>
-            {% endif %}
-            </div>
-          {% endfor %}
+{% for a in activities %}
+<div class="activity" data-activity-id="{{ a.pk }}">
+  <span class="timeline-icon{% if a.has_failed %} timeline-icon-failed{% endif %}">
+  <i class="{% if not a.finished %} icon-refresh icon-spin {% else %}icon-plus{% endif %}"></i>
+</span>
+<strong>{{ a.get_readable_name }}</strong>
+{{ a.started|date:"Y-m-d H:i" }}, {{ a.user }}
+  {% if a.children.count > 0 %}
+    <div class="sub-timeline">
+      {% for s in a.children.all %}
+      <div data-activity-id="{{ s.pk }}" class="sub-activity{% if s.has_failed %} sub-activity-failed{% endif %}">
+        {{ s.get_readable_name }} - 
+        {% if s.finished %}
+          {{ s.finished|time:"H:i:s" }}
+        {% else %}
+          <i class="icon-refresh icon-spin" class="sub-activity-loading-icon"></i>
+        {% endif %}
+        {% if s.has_failed %}
+          <div class="label label-danger">{% trans "failed" %}</div>
+        {% endif %}
         </div>
         </div>
-      {% endif %}
-  </div>
-  {% endfor %}
-  <div><span class="timeline-icon timeline-warning"><i class="icon-remove"></i></span> <strong>Removing</strong> 2013-11-21 15:32</div>
-  <div><span class="timeline-icon timeline-warning"><i class="icon-pause"></i></span> <strong>Suspending</strong> 2013-09-21 15:32</div>
-  <div><span class="timeline-icon"><i class="icon-ellipsis-vertical" ></i></span> <strong>(now)</strong></div>
-  <div><span class="timeline-icon"><i class="icon-truck"></i></span> <strong>Migrated to mega5</strong> 2013-04-21 15:32, ABC123</div>
-  <div><span class="timeline-icon"><i class="icon-refresh"></i></span> <strong>Forced reboot</strong> 2013-04-21 15:32, ABC123</div>
-  <div><span class="timeline-icon"><i class="icon-plus"></i></span> <strong>Created</strong> 2013-04-21 15:32, ABC123</div>
+      {% endfor %}
+    </div>
+  {% endif %}
 </div>
 </div>
-
-{% block extra_js %}
-<script src="{{ STATIC_URL }}dashboard/vm-details.js"></script>
-{% endblock %}
+{% endfor %}

circle/dashboard/templates/dashboard/node-detail/activity.html

+{% load i18n %}
+
+<h3>{% trans "Activity" %}</h3>
+
+<div id="activity-timeline" class="timeline">
+  {% include "dashboard/node-detail/_activity-timeline.html" %}
+</div>

circle/dashboard/views.py

@@ -35,7 +35,7 @@ from .tables import (VmListTable, NodeListTable, NodeVmListTable,
                      TemplateListTable, LeaseListTable, GroupListTable,)
                      TemplateListTable, LeaseListTable, GroupListTable,)
 from vm.models import (Instance, InstanceTemplate, InterfaceTemplate,
 from vm.models import (Instance, InstanceTemplate, InterfaceTemplate,
                        InstanceActivity, Node, instance_activity, Lease,
                        InstanceActivity, Node, instance_activity, Lease,
-                       Interface)
+                       Interface, NodeActivity)
 from firewall.models import Vlan, Host, Rule
 from firewall.models import Vlan, Host, Rule
 from dashboard.models import Favourite
 from dashboard.models import Favourite
 
 
@@ -444,6 +444,10 @@ class NodeDetailView(LoginRequiredMixin, SuperuserRequiredMixin, DetailView):
         context = super(NodeDetailView, self).get_context_data(**kwargs)
         context = super(NodeDetailView, self).get_context_data(**kwargs)
         instances = Instance.active.filter(node=self.object)
         instances = Instance.active.filter(node=self.object)
         context['table'] = NodeVmListTable(instances)
         context['table'] = NodeVmListTable(instances)
+        na = NodeActivity.objects.filter(
+            node=self.object, parent=None
+        ).order_by('-started').select_related()
+        context['activities'] = na
         return context
         return context
 
 
     def post(self, request, *args, **kwargs):
     def post(self, request, *args, **kwargs):

circle/vm/models/activity.py

@@ -98,6 +98,18 @@ class NodeActivity(ActivityModel):
         app_label = 'vm'
         app_label = 'vm'
         db_table = 'vm_nodeactivity'
         db_table = 'vm_nodeactivity'
 
 
+    def __unicode__(self):
+        if self.parent:
+            return '{}({})->{}'.format(self.parent.activity_code,
+                                       self.node,
+                                       self.activity_code)
+        else:
+            return '{}({})'.format(self.activity_code,
+                                   self.node)
+
+    def get_readable_name(self):
+        return self.activity_code.split('.')[-1].replace('_', ' ').capitalize()
+
     @classmethod
     @classmethod
     def create(cls, code_suffix, node, task_uuid=None, user=None):
     def create(cls, code_suffix, node, task_uuid=None, user=None):
         act = cls(activity_code='vm.Node.' + code_suffix,
         act = cls(activity_code='vm.Node.' + code_suffix,

circle/vm/models/node.py

@@ -16,10 +16,11 @@ from firewall.models import Host
 from ..tasks import vm_tasks
 from ..tasks import vm_tasks
 from .common import Trait
 from .common import Trait
 
 
-from .activity import node_activity
+from .activity import node_activity, NodeActivity
 
 
 from monitor.calvin.calvin import Query
 from monitor.calvin.calvin import Query
 from monitor.calvin.calvin import GraphiteHandler
 from monitor.calvin.calvin import GraphiteHandler
+from django.utils import timezone
 
 
 logger = getLogger(__name__)
 logger = getLogger(__name__)
 
 
@@ -56,8 +57,14 @@ class Node(TimeStampedModel):
 
 
     @method_cache(10, 5)
     @method_cache(10, 5)
     def get_online(self):
     def get_online(self):
+        """Check if the node is online.
 
 
-        return self.remote_query(vm_tasks.ping, timeout=1, default=False)
+        Runs a remote ping task if the worker is running.
+        """
+        try:
+            return self.remote_query(vm_tasks.ping, timeout=1, default=False)
+        except WorkerNotFound:
+            return False
 
 
     online = property(get_online)
     online = property(get_online)
 
 
@@ -66,42 +73,44 @@ class Node(TimeStampedModel):
         """Number of CPU threads available to the virtual machines.
         """Number of CPU threads available to the virtual machines.
         """
         """
 
 
-        return self.remote_query(vm_tasks.get_core_num)
+        return self.remote_query(vm_tasks.get_core_num, default=0)
 
 
     num_cores = property(get_num_cores)
     num_cores = property(get_num_cores)
 
 
     @property
     @property
     def state(self):
     def state(self):
-        """Node state.
+        """The state combined of online and enabled attributes.
         """
         """
-
         if self.enabled and self.online:
         if self.enabled and self.online:
-            return 'Online'
+            return 'ONLINE'
         elif self.enabled and not self.online:
         elif self.enabled and not self.online:
-            return 'Missing'
+            return 'MISSING'
         elif not self.enabled and self.online:
         elif not self.enabled and self.online:
-            return 'Disabled'
+            return 'DISABLED'
         else:
         else:
-            return 'Offline'
+            return 'OFFLINE'
 
 
     def disable(self, user=None):
     def disable(self, user=None):
         ''' Disable the node.'''
         ''' Disable the node.'''
-        with node_activity(code_suffix='disable', node=self, user=user):
-            self.enabled = False
-            self.save()
+        if self.enabled is True:
+            with node_activity(code_suffix='disable', node=self, user=user):
+                self.enabled = False
+                self.save()
 
 
     def enable(self, user=None):
     def enable(self, user=None):
         ''' Enable the node. '''
         ''' Enable the node. '''
-        with node_activity(code_suffix='enable', node=self, user=user):
-            self.enabled = True
-            self.save()
+        if self.enabled is not True:
+            with node_activity(code_suffix='enable', node=self, user=user):
+                self.enabled = True
+                self.save()
+            self.get_num_cores(invalidate_cache=True)
+            self.get_ram_size(invalidate_cache=True)
 
 
     @method_cache(300)
     @method_cache(300)
     def get_ram_size(self):
     def get_ram_size(self):
         """Bytes of total memory in the node.
         """Bytes of total memory in the node.
         """
         """
-
-        return self.remote_query(vm_tasks.get_ram_size)
+        return self.remote_query(vm_tasks.get_ram_size, default=0)
 
 
     ram_size = property(get_ram_size)
     ram_size = property(get_ram_size)
 
 
@@ -113,25 +122,77 @@ class Node(TimeStampedModel):
 
 
     @method_cache(30)
     @method_cache(30)
     def get_remote_queue_name(self, queue_id):
     def get_remote_queue_name(self, queue_id):
-        """ Return the remote queue name
+        """Return the name of the remote celery queue for this node.
+
         throws Exception if there is no worker on the queue.
         throws Exception if there is no worker on the queue.
-        Until the cache provide reult there can be dead quques.
+        Until the cache provide reult there can be dead queues.
         """
         """
+
         if vm_tasks.check_queue(self.host.hostname, queue_id):
         if vm_tasks.check_queue(self.host.hostname, queue_id):
+            self.node_online()
             return self.host.hostname + "." + queue_id
             return self.host.hostname + "." + queue_id
         else:
         else:
+            if self.enabled is True:
+                self.node_offline()
             raise WorkerNotFound()
             raise WorkerNotFound()
 
 
+    def node_online(self):
+        """Create activity and log entry when node reappears.
+        """
+
+        try:
+            act = self.activity_log.order_by('-pk')[0]
+        except IndexError:
+            pass  # no monitoring activity at all
+        else:
+            logger.debug("The last activity was %s" % act)
+            if act.activity_code.endswith("offline"):
+                act = NodeActivity.create(code_suffix='monitor_succes_online',
+                                          node=self, user=None)
+                act.started = timezone.now()
+                act.finished = timezone.now()
+                act.succeeded = True
+                act.save()
+                logger.info("Node %s is ONLINE." % self.name)
+                self.get_num_cores(invalidate_cache=True)
+                self.get_ram_size(invalidate_cache=True)
+
+    def node_offline(self):
+        """Called when a node disappears.
+
+        If the node is not already offline, record an activity and a log entry.
+        """
+
+        try:
+            act = self.activity_log.order_by('-pk')[0]
+        except IndexError:
+            pass  # no activity at all
+        else:
+            logger.debug("The last activity was %s" % act)
+            if act.activity_code.endswith("offline"):
+                return
+        act = NodeActivity.create(code_suffix='monitor_failed_offline',
+                                  node=self, user=None)
+        act.started = timezone.now()
+        act.finished = timezone.now()
+        act.succeeded = False
+        act.save()
+        logger.critical("Node %s is OFFLINE%s.", self.name,
+                        ", but enabled" if self.enabled else "")
+        # TODO: check if we should reschedule any VMs?
+
     def remote_query(self, task, timeout=30, raise_=False, default=None):
     def remote_query(self, task, timeout=30, raise_=False, default=None):
         """Query the given task, and get the result.
         """Query the given task, and get the result.
 
 
-        If the result is not ready in timeout secs, return default value or
-        raise a TimeoutError."""
-        r = task.apply_async(
-            queue=self.get_remote_queue_name('vm'), expires=timeout + 60)
+        If the result is not ready or worker not reachable
+        in timeout secs, return default value or raise a
+        TimeoutError or WorkerNotFound exception.
+        """
         try:
         try:
+            r = task.apply_async(
+                queue=self.get_remote_queue_name('vm'), expires=timeout + 60)
             return r.get(timeout=timeout)
             return r.get(timeout=timeout)
-        except TimeoutError:
+        except (TimeoutError, WorkerNotFound):
             if raise_:
             if raise_:
                 raise
                 raise
             else:
             else:
@@ -175,6 +236,11 @@ class Node(TimeStampedModel):
         return float(self.get_monitor_info()["memory.usage"]) / 100
         return float(self.get_monitor_info()["memory.usage"]) / 100
 
 
     def update_vm_states(self):
     def update_vm_states(self):
+        """Update state of Instances running on this Node.
+
+        Query state of all libvirt domains, and notify Instances by their
+        vm_state_changed hook.
+        """
         domains = {}
         domains = {}
         domain_list = self.remote_query(vm_tasks.list_domains_info, timeout=5)
         domain_list = self.remote_query(vm_tasks.list_domains_info, timeout=5)
         if domain_list is None:
         if domain_list is None: