Merge branch 'feature-more-stats' into 'master'

Disk and template stats

related: circle/storagedriver!14

See merge request !391

circle/bower.json

@@ -20,6 +20,7 @@
     "bootbox": "~4.3.0",
     "intro.js": "0.9.0",
     "favico.js": "~0.3.5",
-    "datatables": "~1.10.4"
+    "datatables": "~1.10.4",
+    "chart.js": "2.3.0"
   }
 }

circle/circle/settings/base.py

@@ -245,6 +245,12 @@ PIPELINE_JS = {
     ),
         "output_filename": "vm-detail.js",
     },
+    "datastore": {"source_filenames": (
+        "chart.js/dist/Chart.min.js",
+        "dashboard/datastore-details.js"
+    ),
+        "output_filename": "datastore.js",
+    },
 }
 
 

circle/dashboard/static/dashboard/dashboard.less

@@ -1563,3 +1563,28 @@ textarea[name="new_members"] {
     margin: 15px 0 2px 0;
   }
 }
+
+
+#datastore-chart-legend {
+  width: 350px;
+  margin-top: 100px;
+  margin-left: -120px;
+
+  /* Landscape phones and down */
+  @media (max-width: 992px) {
+    margin-left: -25px;
+  }
+  ul {
+    list-style: none;
+  }
+  li {
+    font-size: 18px;
+      margin-bottom: 2px;
+    span {
+      display: inline-block;
+      width: 30px;
+      height: 18px;
+      margin-right: 8px;
+    }
+  }
+}

circle/dashboard/static/dashboard/datastore-details.js

+$(function() {
+  var data = JSON.parse($("#chart-data").data("data"));
+  var labels = [];
+  for(var i=0; i<data.labels.length; i++) {
+    labels.push(data.labels[i] + " (" + data.readable_data[i] + ")");
+  }
+  var pieChart = new Chart(document.getElementById("datastore-chart"), {
+      type: 'pie',
+      data: {
+        labels: labels,
+        datasets: [{
+          data: data.data,
+          backgroundColor: [
+              "#57b257",
+              "#538ccc",
+              "#f0df24",
+              "#ff9a38",
+              "#7f7f7f",
+          ]
+        }]
+      },
+      options: {
+        legend: {
+          display: false,
+        },
+        tooltips: {
+          callbacks: {
+            label: function(item, chartData) {
+              return data.labels[item.index] + ": " + data.readable_data[item.index];
+            }
+          }
+        },
+      }
+  });
+  $("#datastore-chart-legend").html(pieChart.generateLegend());
+});

circle/dashboard/templates/dashboard/_list-templates.html

+{% for t in templates %}
+  <a href="{{ t.get_absolute_url }}">
+    {{ t.name }}</a>{% if not forloop.last %},{% endif %}
+{% empty %}
+  -
+{% endfor %}

circle/dashboard/templates/dashboard/storage/detail.html

 {% extends "dashboard/base.html" %}
-{% load staticfiles %}
+{% load pipeline %}
+{% load sizefieldtags %}
 {% load i18n %}
 {% load render_table from django_tables2 %}
 {% load crispy_forms_tags %}
@@ -105,4 +106,58 @@
   </div>
 </div>
 
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h3 class="no-margin">
+          <i class="fa fa-pie-chart"></i>
+          {% trans "Disk usage breakdown" %}
+        </h3>
+      </div>
+      <div class="panel-body">
+        <div class="row">
+          <div class="col-md-9">
+            <canvas id="datastore-chart"></canvas>
+          </div>
+          <div class="col-md-3">
+            <div id="datastore-chart-legend"></div>
+          </div>
+        </div>
+        <div id="chart-data" data-data='{
+          "data": [{{stats.template_actual_size}},
+                   {{stats.vm_actual_size}},
+                   {{stats.dumps}},
+                   {{stats.iso_raw}},
+                   {{stats.trash}}],
+          "readable_data": ["{{stats.template_actual_size|filesize}}",
+                            "{{stats.vm_actual_size|filesize}}",
+                            "{{stats.dumps|filesize}}",
+                            "{{stats.dumps|filesize}}",
+                            "{{stats.iso_raw|filesize}}",
+                            "{{stats.trash|filesize}}"],
+          "labels": ["{% trans "Templates" %}",
+                     "{% trans "Virtual machines" %}",
+                     "{% trans "Memory dumps" %}",
+                     "{% trans "ISO + Raw images" %}",
+                     "{% trans "Trash" %}"]
+          }
+          '>
+        </div>
+        <div>
+          {% trans "Total disk usage of virtual machines" %}:
+          <strong>{{ stats.vm_actual_size|filesize }}</strong>
+          <br />
+          {% trans "Total virtual disk usage of virtual machines" %}:
+          <strong>{{ stats.vm_size|filesize}}</strong>
+        </div>
+      </div><!-- .panel-body -->
+    </div>
+  </div>
+</div>
+
+{% endblock %}
+
+{% block extra_js %}
+  {% javascript "datastore" %}
 {% endblock %}

circle/dashboard/templates/dashboard/template-list.html

@@ -63,18 +63,36 @@
     </div>
   </div>
 
-  {% comment %}
   <div class="col-md-6">
     <div class="panel panel-default">
       <div class="panel-heading">
-        <h3 class="no-margin"><i class="fa fa-desktop"></i> Placeholder</h3>
+        <h3 class="no-margin">
+          <i class="fa fa-puzzle-piece"></i>
+          {% trans "Rarely used templates" %}
+        </h3>
       </div>
       <div class="panel-body">
-        ???
+        <dl>
+          <dt>{% trans "Never instantiated" %}</dd>
+          <dd>
+            {% include "dashboard/_list-templates.html" with templates=unused_templates.never_instantiated %}
+          </dd>
+          <dt>{% trans "Templates without running instances" %}</dd>
+          <dd>
+            {% include "dashboard/_list-templates.html" with templates=unused_templates.templates_wo_instances %}
+          </dd>
+          <dt>{% trans "Templates without instances, last instance created more than 90 days ago" %}</dd>
+          <dd>
+            {% include "dashboard/_list-templates.html" with templates=unused_templates.templates_wo_instances_90 %}
+          </dd>
+          <dt>{% trans "Templates without instances, last instance created more than 180 days ago" %}</dd>
+          <dd>
+            {% include "dashboard/_list-templates.html" with templates=unused_templates.templates_wo_instances_180 %}
+          </dd>
+        </dl>
       </div>
     </div>
   </div>
-  {% endcomment %}
 </div>
 {% endif %}
 {% endblock %}

circle/dashboard/views/storage.py

@@ -91,6 +91,7 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
         return qs
 
     def _get_stats(self):
+        # datastore stats
         stats = self.object.get_statistics()
         free_space = int(stats['free_space'])
         free_percent = float(stats['free_percent'])
@@ -98,11 +99,32 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
         total_space = free_space / (free_percent/100.0)
         used_space = total_space - free_space
 
+        # file stats
+        data = self.get_object().get_file_statistics()
+        dumps_size = sum(d['size'] for d in data['dumps'])
+        trash = sum(d['size'] for d in data['trash'])
+        iso_raw = sum(d['size'] for d in data['disks']
+                      if d['format'] in ("iso", "raw"))
+
+        vm_size = vm_actual_size = template_actual_size = 0
+        for d in data['disks']:
+            if d['format'] == "qcow2" and d['type'] == "normal":
+                template_actual_size += d['actual_size']
+            else:
+                vm_size += d['size']
+                vm_actual_size += d['actual_size']
+
         return {
             'used_percent': int(100 - free_percent),
             'free_space': filesizeformat(free_space),
             'used_space': filesizeformat(used_space),
             'total_space': filesizeformat(total_space),
+            'dumps': dumps_size,
+            'trash': trash,
+            'iso_raw': iso_raw,
+            'vm_size': vm_size,
+            'vm_actual_size': vm_actual_size,
+            'template_actual_size': template_actual_size,
         }
 
     def get_success_url(self):

circle/dashboard/views/template.py

@@ -16,6 +16,7 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import unicode_literals, absolute_import
 
+from datetime import timedelta
 import json
 import logging
 
@@ -24,8 +25,10 @@ from django.contrib.auth.models import User
 from django.contrib.messages.views import SuccessMessageMixin
 from django.core.urlresolvers import reverse, reverse_lazy
 from django.core.exceptions import PermissionDenied, SuspiciousOperation
+from django.db.models import Count
 from django.http import HttpResponse, HttpResponseRedirect
 from django.shortcuts import redirect, get_object_or_404
+from django.utils import timezone
 from django.utils.translation import ugettext as _, ugettext_noop
 from django.views.generic import (
     TemplateView, CreateView, UpdateView,
@@ -36,7 +39,9 @@ from braces.views import (
 )
 from django_tables2 import SingleTableView
 
-from vm.models import InstanceTemplate, InterfaceTemplate, Instance, Lease
+from vm.models import (
+    InstanceTemplate, InterfaceTemplate, Instance, Lease, InstanceActivity
+)
 from storage.models import Disk
 
 from ..forms import (
@@ -203,6 +208,41 @@ class TemplateList(LoginRequiredMixin, FilterMixin, SingleTableView):
 
         context['search_form'] = self.search_form
 
+        # templates without any instances
+        # [t for t in InstanceTemplate.objects.all()
+        #  if t.instance_set.count() < 1]
+        never_instantiated = context['object_list'].annotate(
+            instance_count=Count("instance_set")).filter(instance_count__lt=1)
+
+        # templates without active virtual machines
+        active_statuses = Instance.STATUS._db_values - set(["DESTROYED"])
+        templates_wo_instances = context['object_list'].exclude(
+            pk__in=InstanceTemplate.objects.filter(
+                instance_set__status__in=active_statuses)
+        ).exclude(pk__in=never_instantiated)
+
+        def get_create_acts_younger_than(days):
+            return InstanceActivity.objects.filter(
+                activity_code="vm.Instance.create",
+                finished__gt=timezone.now() - timedelta(days=days))
+
+        # templates without active virtual machines
+        # last machine started later than 90 days
+        templates_wo_i_90 = templates_wo_instances.exclude(
+            instance_set__activity_log__in=get_create_acts_younger_than(90))
+
+        # templates without active virtual machines
+        # last machine started later than 180 days
+        templates_wo_i_180 = templates_wo_instances.exclude(
+            instance_set__activity_log__in=get_create_acts_younger_than(180))
+
+        context['unused_templates'] = {
+            'never_instantiated': never_instantiated,
+            'templates_wo_instances': templates_wo_instances,
+            'templates_wo_instances_90': templates_wo_i_90,
+            'templates_wo_instances_180': templates_wo_i_180,
+        }
+
         return context
 
     def get(self, *args, **kwargs):

circle/storage/models.py

@@ -110,6 +110,13 @@ class DataStore(Model):
         disks = Disk.objects.filter(destroyed__isnull=True, is_ready=True)
         return disks.exclude(filename__in=files)
 
+    @method_cache(30)
+    def get_file_statistics(self, timeout=15):
+        queue_name = self.get_remote_queue_name('storage', "slow")
+        data = storage_tasks.get_file_statistics.apply_async(
+            args=[self.path], queue=queue_name).get(timeout=timeout)
+        return data
+
 
 class Disk(TimeStampedModel):
 

circle/storage/tasks/storage_tasks.py

@@ -81,3 +81,8 @@ def recover_from_trash(datastore, disk_path):
 @celery.task(name='storagedriver.get_storage_stat')
 def get_storage_stat(path):
     pass
+
+
+@celery.task(name='storagedriver.get_file_statistics')
+def get_file_statistics(datastore):
+    pass