Merge branch 'feature-store' into 'master'

Feature Store

✅  Add SMB and Quota parameter
✅   Handle private_keys for sshfs and pair it with django_sshkey (Is the hook ready?)
✅ Web interface
🚧 Show user scp/sftp info
✅ Fix tests and it's complete.

circle/circle/settings/base.py

@@ -447,5 +447,14 @@ if graphite_host and graphite_port:
 else:
     GRAPHITE_URL = None
 
+STORE_BASIC_AUTH = get_env_variable("STORE_BASIC_AUTH", "") == "True"
+STORE_VERIFY_SSL = get_env_variable("STORE_VERIFY_SSL", "") == "True"
+STORE_SSL_AUTH = get_env_variable("STORE_SSL_AUTH", "") == "True"
+STORE_CLIENT_USER = get_env_variable("STORE_CLIENT_USER", "")
+STORE_CLIENT_PASSWORD = get_env_variable("STORE_CLIENT_PASSWORD", "")
+STORE_CLIENT_KEY = get_env_variable("STORE_CLIENT_KEY", "")
+STORE_CLIENT_CERT = get_env_variable("STORE_CLIENT_CERT", "")
+STORE_URL = get_env_variable("STORE_URL", "")
+
 SESSION_COOKIE_NAME = "csessid%x" % (((getnode() // 139) ^
                                       (getnode() % 983)) & 0xffff)

circle/circle/settings/test.py

@@ -56,3 +56,5 @@ LOGGING['handlers']['console'] = {'level': level,
                                   'formatter': 'simple'}
 for i in LOCAL_APPS:
     LOGGING['loggers'][i] = {'handlers': ['console'], 'level': level}
+# Forbid store usage
+STORE_URL = ""

circle/dashboard/migrations/0010_auto__add_futuremember__add_unique_futuremember_org_id_group.py

@@ -97,11 +97,13 @@ class Migration(SchemaMigration):
         },
         u'dashboard.profile': {
             'Meta': {'object_name': 'Profile'},
+            'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
             'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
             'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
             'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
+            'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
             'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
         },

circle/dashboard/migrations/0011_auto__add_field_notification_subject_data__add_field_notification_mess.py

@@ -98,11 +98,13 @@ class Migration(SchemaMigration):
         },
         u'dashboard.profile': {
             'Meta': {'object_name': 'Profile'},
+            'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
             'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
             'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
             'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
+            'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
             'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
         },

circle/dashboard/migrations/0012_migrate_messages.py

@@ -93,11 +93,13 @@ class Migration(DataMigration):
         },
         u'dashboard.profile': {
             'Meta': {'object_name': 'Profile'},
+            'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
             'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
             'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
             'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
+            'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
             'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
         },

circle/dashboard/migrations/0013_auto__del_field_notification_message__del_field_notification_subject.py

@@ -96,11 +96,13 @@ class Migration(SchemaMigration):
         },
         u'dashboard.profile': {
             'Meta': {'object_name': 'Profile'},
+            'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
             'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
             'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
             'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
+            'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
             'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
         },

circle/dashboard/models.py

@@ -29,10 +29,13 @@ from django.db.models import (
     Model, ForeignKey, OneToOneField, CharField, IntegerField, TextField,
     DateTimeField, permalink, BooleanField
 )
-from django.db.models.signals import post_save, pre_delete
+from django.db.models.signals import post_save, pre_delete, post_delete
 from django.templatetags.static import static
 from django.utils.translation import ugettext_lazy as _
 from django_sshkey.models import UserKey
+from django.core.exceptions import ObjectDoesNotExist
+
+from sizefield.models import FileSizeField
 
 from jsonfield import JSONField
 from model_utils.models import TimeStampedModel
@@ -44,8 +47,12 @@ from common.models import HumanReadableObject, create_readable, Encoder
 
 from vm.tasks.agent_tasks import add_keys, del_keys
 
+from .store_api import Store, NoStoreException, NotOkException
+
 logger = getLogger(__name__)
 
+pwgen = User.objects.make_random_password
+
 
 class Favourite(Model):
     instance = ForeignKey("vm.Instance")
@@ -109,6 +116,18 @@ class Profile(Model):
     email_notifications = BooleanField(
         verbose_name=_("Email notifications"), default=True,
         help_text=_('Whether user wants to get digested email notifications.'))
+    smb_password = CharField(
+        max_length=20,
+        verbose_name=_('Samba password'),
+        help_text=_(
+            'Generated password for accessing store from '
+            'virtual machines.'),
+        default=pwgen,
+    )
+    disk_quota = FileSizeField(
+        verbose_name=_('disk quota'),
+        default=2048 * 1024 * 1024,
+        help_text=_('Disk quota in mebibytes.'))
 
     def notify(self, subject, template, context=None, valid_until=None,
                **kwargs):
@@ -201,6 +220,11 @@ def create_profile(sender, user, request, **kwargs):
     if not user.pk:
         return False
     profile, created = Profile.objects.get_or_create(user=user)
+
+    try:
+        Store(user).create_user(profile.smb_password, None, profile.disk_quota)
+    except:
+        logger.exception("Can't create user %s", unicode(user))
     return created
 
 user_logged_in.connect(create_profile)
@@ -268,6 +292,44 @@ else:
     logger.debug("Do not register save_org_id to djangosaml2 pre_user_save")
 
 
+def update_store_profile(sender, **kwargs):
+    profile = kwargs.get('instance')
+    keys = [i.key for i in profile.user.userkey_set.all()]
+    try:
+        s = Store(profile.user)
+        s.create_user(profile.smb_password, keys,
+                      profile.disk_quota)
+    except NoStoreException:
+        logger.debug("Store is not available.")
+    except NotOkException:
+        logger.critical("Store is not accepting connections.")
+
+
+post_save.connect(update_store_profile, sender=Profile)
+
+
+def update_store_keys(sender, **kwargs):
+    userkey = kwargs.get('instance')
+    try:
+        profile = userkey.user.profile
+    except ObjectDoesNotExist:
+        pass  # If there is no profile the user is deleted
+    else:
+        keys = [i.key for i in profile.user.userkey_set.all()]
+        try:
+            s = Store(userkey.user)
+            s.create_user(profile.smb_password, keys,
+                          profile.disk_quota)
+        except NoStoreException:
+            logger.debug("Store is not available.")
+        except NotOkException:
+            logger.critical("Store is not accepting connections.")
+
+
+post_save.connect(update_store_keys, sender=UserKey)
+post_delete.connect(update_store_keys, sender=UserKey)
+
+
 def add_ssh_keys(sender, **kwargs):
     from vm.models import Instance
 

circle/dashboard/static/dashboard/dashboard.css

@@ -723,6 +723,82 @@ textarea[name="list-new-namelist"] {
 
 }
 
+#store-list-list {
+  list-style: none;
+}
+
+.store-list-item {
+  cursor: pointer;
+}
+
+.store-list-item:hover {
+  background: rgba(0, 0, 0, 0.6);
+}
+
+.store-list-item-icon {
+  width: 20px;
+  text-align: center;
+  display: inline-block;
+  margin-right: 15px;
+  float: left;
+}
+
+.store-list-item-size {
+  width: 70px;
+  text-align: right;
+  float: right;
+}
+
+.store-list-file-infos {
+  padding: 15px;
+  display: none;
+  border-left: 1px solid #ddd;
+  border-right: 1px solid #ddd;
+  position: relative;
+}
+
+.store-list-item-new {
+  display: inline-block;
+}
+
+.store-list-item-new .badge {
+  margin-left: 5px;
+  background: #5bc0dc;
+}
+
+.store-list-item-icon-directory {
+  color: #ff8c00;
+}
+
+.store-remove-button {
+  margin-top: 8px;
+}
+
+#dashboard-files-toplist div.list-group-item {
+  color: #555;
+}
+
+#dashboard-files-toplist div.list-group-item:hover {
+  background: #eee;
+}
+
+.store-list-item-name {
+  max-width: 70%;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+  overflow: hidden;
+  float: left;
+}
+
+.dashboard-toplist-icon {
+  float: left;
+  padding: 2px 5px 0 0;
+}
+
+.no-hover:hover {
+  background: none !important;
+}
+
 #group-detail-permissions .filtered {
   margin: 2px 0;
   padding: 2px 3px;
@@ -752,6 +828,23 @@ textarea[name="list-new-namelist"] {
   margin-top: -6px;
 }
 
+.store-action-button {
+  margin-left: 5px;
+}
+
+#progress-marker-hard {
+  border-top-right-radius: 4px;
+  border-bottom-right-radius: 4px;
+  right: 0;
+  background: red;
+}
+
+.progress-marker {
+  width: 6px;
+  height: 20px;
+  position: absolute;
+}
+
 #show-all-activities-container {
   margin: 20px 0 0 10px;
 }

circle/dashboard/static/dashboard/dashboard.js

@@ -112,6 +112,7 @@ $(function () {
 
 
   /* no js compatibility */
+  noJS();
   $('.no-js-hidden').show();
   $('.js-hidden').hide();
 
@@ -561,3 +562,10 @@ function getCookie(name) {
   }                                                                         
   return cookieValue;                                                       
 }
+
+
+/* no js compatibility */
+function noJS() {
+  $('.no-js-hidden').show();
+  $('.js-hidden').hide();
+}

circle/dashboard/static/dashboard/store.js

+$(function() {
+  $("#store-list-container").on("click", ".store-list-item", function() {
+    if($(this).data("item-type") == "D") {
+      $("#store-list-up-icon").removeClass("fa-reply").addClass("fa-refresh fa-spin");
+      var url = $(this).prop("href");
+      $.get(url, function(result) {
+        $("#store-list-container").html(result);
+        noJS();
+        $("[title]").tooltip();
+        history.pushState({}, "", url);
+      });
+    } else {
+      $(this).next(".store-list-file-infos").stop().slideToggle();
+    }
+    return false;
+  });
+
+  /* how upload works
+   * - user clicks on a "fake" browse button, this triggers a click event on the file upload
+   * - if the file input changes it adds the name of the file to form (or number of files if multiple is enabled)
+   * - and finally when we click on the upload button (this event handler) it firsts ask the store api where to upload
+   *   then changes the form's action attr before sending the form itself
+   */
+  $("#store-list-container").on("click", '#store-upload-form button[type="submit"]', function() {
+    var current_dir = $("#store-upload-form").find('[name="current_dir"]').val();
+    $.get($("#store-upload-form").data("action") + "?current_dir=" + current_dir, function(result) {
+      $('#store-upload-form button[type="submit"] i').addClass("fa-spinner fa-spin");
+      $("#store-upload-form").get(0).setAttribute("action", result['url']);
+      $("#store-upload-form").submit();
+    });
+
+    return false;
+  });
+  
+  /* "fake" browse button */
+  $("#store-list-container").on("click", "#store-upload-browse", function() {
+    $('#store-upload-form input[type="file"]').click();
+  });
+  
+  $("#store-list-container").on("change", "#store-upload-file", function() {
+    var input = $(this);
+    var numFiles = input.get(0).files ? input.get(0).files.length : 1;
+    var label = input.val().replace(/\\/g, '/').replace(/.*\//, '');
+    input.trigger('fileselect', [numFiles, label]);
+  });
+
+  $("#store-list-container").on("fileselect", "#store-upload-file", function(event, numFiles, label) {
+    var input = $("#store-upload-filename");
+    var log = numFiles > 1 ? numFiles + ' files selected' : label;
+    if(input.length) {
+      input.val(log);
+    }
+    if(log) {
+      $('#store-upload-form button[type="submit"]').prop("disabled", false);
+    } else {
+      $('#store-upload-form button[type="submit"]').prop("disabled", true);
+    }
+
+  });
+});

circle/dashboard/store_api.py

+from os.path import splitext
+import json
+import logging
+from urlparse import urljoin
+from datetime import datetime
+
+from django.http import Http404
+from django.conf import settings
+from requests import get, post, codes
+from sizefield.utils import filesizeformat
+
+logger = logging.getLogger(__name__)
+
+
+class StoreApiException(Exception):
+    pass
+
+
+class NotOkException(StoreApiException):
+    def __init__(self, status, *args, **kwargs):
+        self.status = status
+        super(NotOkException, self).__init__(*args, **kwargs)
+
+
+class NoStoreException(StoreApiException):
+    pass
+
+
+class Store(object):
+
+    def __init__(self, user, default_timeout=0.5):
+        self.request_args = {'verify': settings.STORE_VERIFY_SSL}
+        if settings.STORE_SSL_AUTH:
+            self.request_args['cert'] = (settings.STORE_CLIENT_CERT,
+                                         settings.STORE_CLIENT_KEY)
+        if settings.STORE_BASIC_AUTH:
+            self.request_args['auth'] = (settings.STORE_CLIENT_USER,
+                                         settings.STORE_CLIENT_PASSWORD)
+        self.username = "u-%d" % user.pk
+        self.default_timeout = default_timeout
+        self.store_url = settings.STORE_URL
+        if not self.store_url:
+            raise NoStoreException
+
+    def _request(self, url, method=get, timeout=None,
+                 raise_status_code=True, **kwargs):
+        url = urljoin(self.store_url, url)
+        if timeout is None:
+            timeout = self.default_timeout
+        payload = json.dumps(kwargs) if kwargs else None
+        try:
+            headers = {'content-type': 'application/json'}
+            response = method(url, data=payload, headers=headers,
+                              timeout=timeout, **self.request_args)
+        except Exception:
+            logger.exception("Error in store %s loading %s",
+                             unicode(method), url)
+            raise
+        else:
+            if raise_status_code and response.status_code != codes.ok:
+                if response.status_code == 404:
+                    raise Http404()
+                else:
+                    raise NotOkException(response.status_code)
+            return response
+
+    def _request_cmd(self, cmd, **kwargs):
+        return self._request(self.username, post, CMD=cmd, **kwargs)
+
+    def list(self, path, process=True):
+        r = self._request_cmd("LIST", PATH=path)
+        result = r.json()
+        if process:
+            return self._process_list(result)
+        else:
+            return result
+
+    def toplist(self, process=True):
+        r = self._request_cmd("TOPLIST")
+        result = r.json()
+        if process:
+            return self._process_list(result)
+        else:
+            return result
+
+    def request_download(self, path):
+        r = self._request_cmd("DOWNLOAD", PATH=path, timeout=10)
+        return r.json()['LINK']
+
+    def request_upload(self, path):
+            r = self._request_cmd("UPLOAD", PATH=path)
+            return r.json()['LINK']
+
+    def remove(self, path):
+        self._request_cmd("REMOVE", PATH=path)
+
+    def new_folder(self, path):
+        self._request_cmd("NEW_FOLDER", PATH=path)
+
+    def rename(self, old_path, new_name):
+        self._request_cmd("RENAME", PATH=old_path, NEW_NAME=new_name)
+
+    def get_quota(self):  # no CMD? :o
+        r = self._request(self.username)
+        quota = r.json()
+        quota.update({
+            'readable_used': filesizeformat(float(quota['used'])),
+            'readable_soft': filesizeformat(float(quota['soft'])),
+            'readable_hard': filesizeformat(float(quota['hard'])),
+        })
+        return quota
+
+    def set_quota(self, quota):
+        self._request("/quota/" + self.username, post, QUOTA=quota)
+
+    def user_exist(self):
+        try:
+            self._request(self.username)
+            return True
+        except NotOkException:
+            return False
+
+    def create_user(self, password, keys, quota):
+        self._request("/new/" + self.username, method=post,
+                      SMBPASSWD=password, KEYS=keys, QUOTA=quota)
+
+    @staticmethod
+    def _process_list(content):
+        for d in content:
+            d['human_readable_date'] = datetime.utcfromtimestamp(float(
+                d['MTIME']))
+            delta = (datetime.utcnow() -
+                     d['human_readable_date']).total_seconds()
+            d['is_new'] = 0 < delta < 5
+            d['human_readable_size'] = (
+                "directory" if d['TYPE'] == "D" else
+                filesizeformat(float(d['SIZE'])))
+
+            if d['DIR'] == ".":
+                d['directory'] = "/"
+            else:
+                d['directory'] = "/" + d['DIR'] + "/"
+
+            d['path'] = d['directory']
+            d['path'] += d['NAME']
+            if d['TYPE'] == "D":
+                d['path'] += "/"
+
+            d['ext'] = splitext(d['path'])[1]
+            d['icon'] = ("folder-open" if not d['TYPE'] == "F"
+                         else file_icons.get(d['ext'], "file-o"))
+
+        return sorted(content, key=lambda k: k['TYPE'])
+
+
+file_icons = {
+    '.txt': "file-text-o",
+    '.pdf': "file-pdf-o",
+
+    '.jpg': "file-image-o",
+    '.jpeg': "file-image-o",
+    '.png': "file-image-o",
+    '.gif': "file-image-o",
+
+    '.avi': "file-video-o",
+    '.mkv': "file-video-o",
+    '.mp4': "file-video-o",
+    '.mov': "file-video-o",
+
+    '.mp3': "file-sound-o",
+    '.flac': "file-sound-o",
+    '.wma': "file-sound-o",
+
+    '.pptx': "file-powerpoint-o",
+    '.ppt': "file-powerpoint-o",
+    '.doc': "file-word-o",
+    '.docx': "file-word-o",
+    '.xlsx': "file-excel-o",
+    '.xls': "file-excel-o",
+
+    '.rar': "file-archive-o",
+    '.zip': "file-archive-o",
+    '.7z': "file-archive-o",
+    '.tar': "file-archive-o",
+    '.gz': "file-archive-o",
+
+    '.py': "file-code-o",
+    '.html': "file-code-o",
+    '.js': "file-code-o",
+    '.css': "file-code-o",
+    '.c': "file-code-o",
+    '.cpp': "file-code-o",
+    '.h': "file-code-o",
+    '.sh': "file-code-o",
+}

circle/dashboard/templates/dashboard/index.html

@@ -23,11 +23,11 @@
     </div>
     {% endif %}
 
-    {% comment %}
+    {% if not no_store %}
     <div class="col-lg-4 col-sm-6">
-      {% include "dashboard/index-files.html" %}
+      {% include "dashboard/store/index-files.html" %}
     </div>
-    {% endcomment %}
+    {% endif %}
 
     {% if perms.vm.create_template %}
     <div class="col-lg-4 col-sm-6">

circle/dashboard/templates/dashboard/store/_list-box.html

+{% load i18n %}
+
+<div class="list-group">
+  <div class="list-group-item">
+    <div class="row">
+      <div class="col-sm-6">
+        <a href="{% url "dashboard.views.store-upload"%}?directory={{ current }}"
+          class="btn btn-info btn-xs js-hidden">
+          {% trans "Upload" %}
+        </a>
+        <form action="" data-action="{% url "dashboard.views.store-upload-url" %}" 
+          method="POST" enctype="multipart/form-data" class="no-js-hidden"
+          id="store-upload-form">
+          {% csrf_token %}
+          <input type="hidden" name="current_dir" value="{{ current }}"/>
+          <input type="hidden" name="next" value="{{ next_url }}"/>
+          <div class="input-group" style="max-width: 350px;">
+            <span class="input-group-btn" id="store-upload-browse">
+              <span class="btn btn-primary btn-xs">
+                {% trans "Browse..." %}
+              </span>     
+            </span>
+            <input type="text" class="form-control input-tags" 
+             id="store-upload-filename"/>
+            <span class="input-group-btn">
+              <button type="submit" class="btn btn-primary btn-xs" disabled>
+                <i class="fa fa-cloud-upload"></i> {% trans "Upload" %}
+              </button>
+            </span>
+          </div>
+          <input id="store-upload-file" name="data" type="file" style="display:none">
+        </form>
+      </div><!-- .col-sm-6 upload -->
+
+      <div class="col-sm-6">
+        <a href="{% url "dashboard.views.store-remove" %}?path={{ current }}" 
+          class="btn btn-danger btn-xs pull-right store-action-button" 
+          title="{% trans "Remove directory" %}">
+          <i class="fa fa-times"></i>
+        </a>
+        <a href="{% url "dashboard.views.store-download" %}?path={{ current }}" 
+          class="btn btn-primary btn-xs pull-right store-action-button" 
+          title="{% trans "Download directory" %}">
+          <i class="fa fa-cloud-download"></i>
+        </a>
+
+        <form method="POST" action="{% url "dashboard.views.store-new-directory" %}">
+          {% csrf_token %}
+          <input type="hidden" name="path" value="{{ current }}"/>
+          <div class="input-group" style="max-width: 300px;">
+            <span class="input-group-addon input-tags" title="{% trans "New directory" %}">
+              <i class="fa fa-folder-open"></i>
+            </span>
+            <input type="text" class="form-control input-tags" name="name" 
+             placeholder="{% trans "Name "%}" required/>
+            <span class="input-group-btn">
+              <input type="submit" class="btn btn-success btn-xs" value="{% trans "Create" %}"/>
+            </span>
+          </div>
+        </form>
+      </div><!-- .col-sm-6 -->
+    </div><!-- .row -->
+  </div><!-- .list-group-item -->
+</div><!-- .list-group -->
+
+<div class="list-group" id="store-list-list">
+  <a href="{% url "dashboard.views.store-list" %}?directory={{ up_url }}" 
+    class="list-group-item store-list-item" data-item-type="D">
+    {% if current == "/" %}
+      <div class="store-list-item-icon">
+        <i class="fa fa-refresh" id="store-list-up-icon"></i>
+      </div>
+      {% trans "Refresh" %}
+    {% else %}
+      <div class="store-list-item-icon">
+        <i class="fa fa-reply" id="store-list-up-icon"></i>
+      </div>
+      ..
+    {% endif %}
+
+    <div class="pull-right">
+      {{ current }}
+    </div>
+  </a>
+
+  {% for f in root %}
+  <a class="list-group-item store-list-item" data-item-type="{{ f.TYPE }}"
+    href="{% if f.TYPE == "D" %}{% url "dashboard.views.store-list" %}?directory={{ f.path }}{% else %}
+    {% url "dashboard.views.store-download" %}?path={{ f.path }}{% endif %}"
+    >
+    <div class="store-list-item-icon">
+      <i class="
+        fa fa-{{ f.icon }}{% if f.TYPE == "D" %} store-list-item-icon-directory{% endif %}"
+        ></i>
+    </div>
+    <div class="store-list-item-name">
+      {{ f.NAME }}
+    </div>
+    <div class="store-list-item-new">
+      {% if f.is_new and f.TYPE == "F" %}
+        <span class="badge badge-pulse">{% trans "new" %}</span>
+      {% endif %}
+    </div>
+    
+    <div class="store-list-item-size">
+      {{ f.human_readable_size }}
+    </div>
+    <div class="clearfix"></div>
+  </a>
+  <div class="store-list-file-infos">
+    <div class="row">
+      <div class="col-sm-10">
+        <dl class="dl-horizontal" style="margin: 0; padding: 0;">
+          <dt>{% trans "Filename" %}</dt>
+          <dd>{{ f.NAME }}</dd>
+
+          <dt>{% trans "Size" %}</dt>
+          <dd>{{ f.human_readable_size }}</dd>
+
+          <dt>{% trans "Latest modification" %}</dt>
+          <dd>{{ f.human_readable_date }}</dd>
+        </dl>
+      </div>
+      <div class="col-sm-2" style="text-align: right;">
+        <a href="{% url "dashboard.views.store-download" %}?path={{ f.path }}" 
+          class="btn btn-primary btn-sm store-download-button">
+          <i class="fa fa-download"></i>
+          {% trans "Download" %}
+        </a>
+        <a href="{% url "dashboard.views.store-remove" %}?path={{ f.path }}" 
+          class="btn btn-danger btn-xs store-remove-button">
+          <i class="fa fa-times"></i>
+          {% trans "Remove" %}
+        </a>
+      </div>
+    </div><!-- .row -->
+  </div>
+  {% empty %}
+  <a class="list-group-item">
+    {% trans "This folder is empty." %}
+  </a>
+  {% endfor %}
+</div><!-- closing list-group -->

circle/dashboard/templates/dashboard/store/index-files.html

+{% load i18n %}
+
+<div class="panel panel-default">
+  <div class="panel-heading">
+    <span class="btn btn-default btn-xs infobtn pull-right store-action-button" 
+      title="{% trans "A list of your most recent files." %}">
+      <i class="fa fa-info-circle"></i>
+    </span>
+    <span class="btn btn-default btn-xs infobtn pull-right" 
+      title="
+      {% blocktrans with used=files.quota.readable_used soft=files.quota.readable_soft hard=files.quota.readable_hard %}
+      You are currently using {{ used }}, your soft limit is {{ soft }}, your hard limit is {{ hard }}.
+      {% endblocktrans %}">
+      <i class="fa fa-adjust"></i>
+    </span>
+    <h3 class="no-margin"><i class="fa fa-briefcase"></i> {% trans "Files" %}
+    </h3>
+  </div>
+  <div class="list-group" id="dashboard-files-toplist">
+    {% for t in files.toplist %}
+    {% if t.TYPE == "F" %}
+      <div class="list-group-item">
+        <i class="fa fa-{{ t.icon }} dashboard-toplist-icon"></i> 
+        <div class="store-list-item-name">
+          {{ t.NAME }}
+        </div>
+        <a href="{% url "dashboard.views.store-download" %}?path={{ t.path }}" 
+          class="pull-right btn btn-xs" style="color: black;">
+          <i class="fa fa-cloud-download" title="{% trans "Download" %}"></i>
+        </a>
+        <a href="{% url "dashboard.views.store-list" %}?directory={{ t.directory }}" 
+          class="pull-right btn btn-xs" style="color: black;">
+          <i class="fa fa-folder-open" title="{% trans "Show in directory" %}"></i>
+        </a>
+        <div style="clear: both;"></div> 
+      </div>
+    {% else %}
+    <a href="{% url "dashboard.views.store-list" %}?directory={{ t.path }}" 
+      class="list-group-item">
+      <i class="fa fa-{{ t.icon }} dashboard-toplist-icon"></i>
+        <div class="store-list-item-name">
+        {{ t.NAME }}
+      </div>
+      <div style="clear: both;"></div>
+    </a>
+    {% endif %}
+    {% endfor %}
+    <div class="list-group-item text-right no-hover">
+      <form class="pull-left" method="POST" action="{% url "dashboard.views.store-refresh-toplist" %}">
+        {% csrf_token %}
+        <button class="btn btn-success btn-xs" type="submit" title="{% trans "Refresh" %}"/>
+          <i class="fa fa-refresh"></i>
+        </button>
+      </form>
+      <a href="{% url "dashboard.views.store-list" %}" class="btn btn-primary btn-xs">
+        <i class="fa fa-chevron-circle-right"></i> {% trans "show my files" %}
+      </a>
+      <a href="{% url "dashboard.views.store-upload" %}" class="btn btn-success btn-xs">
+        <i class="fa fa-cloud-upload"></i> {% trans "upload" %}
+      </a>
+    </div>
+  </div>
+</div>

circle/dashboard/templates/dashboard/store/list.html

+{% extends "dashboard/base.html" %}
+{% load i18n %}
+
+{% block title-page %}{% trans "List" %} | {% trans "Store" %}{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col-md-12">
+    <div id="store-list-container">
+      {% include "dashboard/store/_list-box.html" %}
+    </div>
+  </div>
+</div>
+
+<div style="position: relative;">
+  <div class="progress" style="width: 100%">
+    <div class="progress-bar" role="progressbar" 
+      aria-valuenow="{{ quota.used }}" aria-valuemin="0" aria-valuemax="{{ quota.hard }}" 
+      style="width: {% widthratio quota.used quota.hard 100 %}%; min-width: 150px;">
+      <div style="padding-top: 2px;">
+        {% blocktrans with used=quota.readable_used %}
+          {{ used }} used
+        {% endblocktrans %}
+      </div>
+    </div>
+
+    <div class="progress-marker" id="progress-marker-hard" data-placement="left"
+      title="{% trans "Hard limit" %}: {{ quota.readable_hard }}">
+    </div>
+    <div class="progress-marker" id="progress-marker-soft" style="background: orange; 
+      left: {% widthratio quota.soft quota.hard 100 %}%"
+      title="{% trans "Soft limit" %}: {{ quota.readable_soft }}"
+      data-placement="top">
+    </div>
+  </div>
+</div>
+
+{% endblock %}
+
+{% block extra_js %}
+  <script src="{{ STATIC_URL}}dashboard/store.js"></script>
+{% endblock %}

circle/dashboard/templates/dashboard/store/remove.html

+{% extends "dashboard/base.html" %}
+{% load i18n %}
+
+{% block content %}
+  <div class="body-content">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h3 class="no-margin">
+          <i class="fa fa-times"></i>
+          {% if is_dir %}
+            {% trans "Directory removal confirmation" %}
+          {% else %}
+            {% trans "File removal confirmation" %}
+          {% endif %}
+        </h3>
+      </div>
+      <div class="panel-body">
+        {% if not is_dir %}
+          <h4>{% trans "File directory" %}: {{ directory }}</h4>
+          <h4>{% trans "File name" %}: {{ name }}</h4>
+
+          {% blocktrans with path=path %}
+          Are you sure you want to remove the file at <strong>{{ path }}</strong>?
+          {% endblocktrans %}
+        {% else %}
+          {% blocktrans with directory=directory %}
+          Are you sure you want to remove the directory <strong>{{ directory }}</strong>?
+          {% endblocktrans %}
+        {% endif %}
+        <div class="pull-right">
+          <form action="" method="POST">
+            {% csrf_token %}
+            <a href="{% url "dashboard.views.store-list" %}?directory={{ directory }}"
+              class="btn btn-default">{% trans "Cancel" %}</a>
+            <input type="hidden" name="path" value="{{ path }}"/>
+            <button class="btn btn-danger">{% trans "Remove" %}</button>
+          </form>
+        </div>
+      </div>
+    </div>
+  </div>
+{% endblock %}

circle/dashboard/templates/dashboard/store/upload.html

+{% extends "dashboard/base.html" %}
+{% load i18n %}
+
+{% block title-page %}{% trans "Upload" %} | {% trans "Store" %}{% endblock %}
+
+{% block content %}
+
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <a class="btn btn-default pull-right btn-xs" 
+          href="{% url "dashboard.views.store-list" %}">Back</a>
+        <h3 class="no-margin">
+          <i class="fa fa-cloud-upload"></i>
+          {% trans "File upload" %}
+        </h3>
+      </div>
+      <div class="panel-body">
+        <div style="text-align: center; margin: 0 0 20px 0;">
+          <div class="label label-info" style="padding: 5px;">
+            {% trans "Curently uploading to" %}: {{ directory }}
+          </div>
+        </div>
+        <form method="POST" action="{{ action }}" enctype="multipart/form-data">
+          <input type="hidden" name="next" value="{{ next_url }}"/>
+          <div>
+            <input class="btn btn-default btn-sm pull-right" 
+            type="submit" value="{% trans "Upload" %}"/>
+            <input type="file" name="data" style="padding-top: 5px;" required/>
+          </div>
+        </form>
+      </div>
+    </div>
+  </div>
+</div>
+
+{% endblock %}

circle/dashboard/tests/test_views.py

@@ -1776,9 +1776,11 @@ class SshKeyTest(LoginMixin, TestCase):
     def setUp(self):
         self.u1 = User.objects.create(username='user1')
         self.u1.set_password('password')
+        self.u1.profile = Profile()
         self.u1.save()
         self.u2 = User.objects.create(username='user2')
         self.u2.set_password('password')
+        self.u2.profile = Profile()
         self.u2.save()
         self.k1 = UserKey(key='ssh-rsa AAAAB3NzaC1yc2EC asd', user=self.u1)
         self.k1.save()

circle/dashboard/urls.py

@@ -39,6 +39,8 @@ from .views import (
     get_vm_screenshot,
     ProfileView, toggle_use_gravatar, UnsubscribeFormView,
     UserKeyDelete, UserKeyDetail, UserKeyCreate,
+    StoreList, store_download, store_upload, store_get_upload_url, StoreRemove,
+    store_new_directory, store_refresh_toplist,
     VmTraitsUpdate, VmRawDataUpdate,
     GroupPermissionsView,
     LeaseAclUpdateView,
@@ -177,5 +179,21 @@ urlpatterns = patterns(
     url(r'^sshkey/create/$',
         UserKeyCreate.as_view(),
         name="dashboard.views.userkey-create"),
+
     url(r'^autocomplete/', include('autocomplete_light.urls')),
+
+    url(r"^store/list/$", StoreList.as_view(),
+        name="dashboard.views.store-list"),
+    url(r"^store/download/$", store_download,
+        name="dashboard.views.store-download"),
+    url(r"^store/upload/url$", store_get_upload_url,
+        name="dashboard.views.store-upload-url"),
+    url(r"^store/upload/$", store_upload,
+        name="dashboard.views.store-upload"),
+    url(r"^store/remove/$", StoreRemove.as_view(),
+        name="dashboard.views.store-remove"),
+    url(r"^store/new_directory/$", store_new_directory,
+        name="dashboard.views.store-new-directory"),
+    url(r"^store/refresh_toplist$", store_refresh_toplist,
+        name="dashboard.views.store-refresh-toplist"),
 )

circle/dashboard/views.py

 # Copyright 2014 Budapest University of Technology and Economics (BME IK)
+
 #
 # This file is part of CIRCLE Cloud.
 #
@@ -20,6 +21,7 @@ from __future__ import unicode_literals, absolute_import
 from collections import OrderedDict
 from itertools import chain
 from os import getenv
+from os.path import join, normpath, dirname, basename
 from urlparse import urljoin
 import json
 import logging
@@ -29,15 +31,19 @@ import requests
 from django.conf import settings
 from django.contrib.auth.models import User, Group
 from django.contrib.auth.views import login, redirect_to_login
+from django.contrib.auth.decorators import login_required
 from django.contrib.messages.views import SuccessMessageMixin
 from django.core.exceptions import (
     PermissionDenied, SuspiciousOperation,
 )
+from django.core.cache import get_cache
 from django.core import signing
 from django.core.urlresolvers import reverse, reverse_lazy
 from django.db.models import Count, Q
 from django.http import HttpResponse, HttpResponseRedirect, Http404
-from django.shortcuts import redirect, render, get_object_or_404
+from django.shortcuts import (
+    redirect, render, get_object_or_404, render_to_response,
+)
 from django.views.decorators.http import require_GET, require_POST
 from django.views.generic.detail import SingleObjectMixin
 from django.views.generic import (TemplateView, DetailView, View, DeleteView,
@@ -80,6 +86,8 @@ from storage.models import Disk
 from firewall.models import Vlan, Host, Rule
 from .models import Favourite, Profile, GroupProfile, FutureMember
 
+from .store_api import Store, NoStoreException
+
 logger = logging.getLogger(__name__)
 saml_available = hasattr(settings, "SAML_CONFIG")
 
@@ -219,6 +227,27 @@ class IndexView(LoginRequiredMixin, TemplateView):
             context['templates'] = InstanceTemplate.get_objects_with_level(
                 'operator', user).all()[:5]
 
+        # toplist
+        if settings.STORE_URL:
+            cache_key = "files-%d" % self.request.user.pk
+            cache = get_cache("default")
+            files = cache.get(cache_key)
+            if not files:
+                try:
+                    store = Store(self.request.user)
+                    toplist = store.toplist()
+                    quota = store.get_quota()
+                    files = {'toplist': toplist, 'quota': quota}
+                except Exception:
+                    logger.exception("Unable to get tolist for %s",
+                                     unicode(self.request.user))
+                    files = {'toplist': []}
+                cache.set(cache_key, files, 300)
+
+            context['files'] = files
+        else:
+            context['no_store'] = True
+
         return context
 
 
@@ -3092,5 +3121,165 @@ class UserKeyCreate(LoginRequiredMixin, SuccessMessageMixin, CreateView):
         return kwargs
 
 
+class StoreList(LoginRequiredMixin, TemplateView):
+    template_name = "dashboard/store/list.html"
+
+    def get_context_data(self, **kwargs):
+        context = super(StoreList, self).get_context_data(**kwargs)
+        directory = self.request.GET.get("directory", "/")
+        directory = "/" if not len(directory) else directory
+
+        store = Store(self.request.user)
+        context['root'] = store.list(directory)
+        context['quota'] = store.get_quota()
+        context['up_url'] = self.create_up_directory(directory)
+        context['current'] = directory
+        context['next_url'] = "%s%s?directory=%s" % (
+            settings.DJANGO_URL.rstrip("/"),
+            reverse("dashboard.views.store-list"), directory)
+        return context
+
+    def get(self, *args, **kwargs):
+        try:
+            if self.request.is_ajax():
+                context = self.get_context_data(**kwargs)
+                return render_to_response(
+                    "dashboard/store/_list-box.html",
+                    RequestContext(self.request, context),
+                )
+            else:
+                return super(StoreList, self).get(*args, **kwargs)
+        except NoStoreException:
+            messages.warning(self.request, _("No store."))
+            return redirect("/")
+
+    def create_up_directory(self, directory):
+        path = normpath(join('/', directory, '..'))
+        if not path.endswith("/"):
+            path += "/"
+        return path
+
+
+@require_GET
+@login_required
+def store_download(request):
+    path = request.GET.get("path")
+    try:
+        url = Store(request.user).request_download(path)
+    except Exception:
+        messages.error(request, _("Something went wrong during download."))
+        logger.exception("Unable to download, "
+                         "maybe it is already deleted")
+        return redirect(reverse("dashboard.views.store-list"))
+    return redirect(url)
+
+
+@require_GET
+@login_required
+def store_upload(request):
+    directory = request.GET.get("directory", "/")
+    try:
+        action = Store(request.user).request_upload(directory)
+    except Exception:
+        logger.exception("Unable to upload")
+        messages.error(request, _("Unable to upload file."))
+        return redirect("/")
+
+    next_url = "%s%s?directory=%s" % (
+        settings.DJANGO_URL.rstrip("/"),
+        reverse("dashboard.views.store-list"), directory)
+
+    return render(request, "dashboard/store/upload.html",
+                  {'directory': directory, 'action': action,
+                   'next_url': next_url})
+
+
+@require_GET
+@login_required
+def store_get_upload_url(request):
+    current_dir = request.GET.get("current_dir")
+    try:
+        url = Store(request.user).request_upload(current_dir)
+    except Exception:
+        logger.exception("Unable to upload")
+        messages.error(request, _("Unable to upload file."))
+        return redirect("/")
+    return HttpResponse(
+        json.dumps({'url': url}), content_type="application/json")
+
+
+class StoreRemove(LoginRequiredMixin, TemplateView):
+    template_name = "dashboard/store/remove.html"
+
+    def get_context_data(self, *args, **kwargs):
+        context = super(StoreRemove, self).get_context_data(*args, **kwargs)
+        path = self.request.GET.get("path", "/")
+        if path == "/":
+            SuspiciousOperation()
+
+        context['path'] = path
+        context['is_dir'] = path.endswith("/")
+        if context['is_dir']:
+            context['directory'] = path
+        else:
+            context['directory'] = dirname(path)
+            context['name'] = basename(path)
+
+        return context
+
+    def get(self, *args, **kwargs):
+        try:
+            return super(StoreRemove, self).get(*args, **kwargs)
+        except NoStoreException:
+            return redirect("/")
+
+    def post(self, *args, **kwargs):
+        path = self.request.POST.get("path")
+        try:
+            Store(self.request.user).remove(path)
+        except Exception:
+            logger.exception("Unable to remove %s", path)
+            messages.error(self.request, _("Unable to remove %s.") % path)
+
+        return redirect("%s?directory=%s" % (
+            reverse("dashboard.views.store-list"),
+            dirname(dirname(path)),
+        ))
+
+
+@require_POST
+@login_required
+def store_new_directory(request):
+    path = request.POST.get("path")
+    name = request.POST.get("name")
+
+    try:
+        Store(request.user).new_folder(join(path, name))
+    except Exception:
+        logger.exception("Unable to create folder %s in %s for %s",
+                         name, path, unicode(request.user))
+        messages.error(request, _("Unable to create folder."))
+    return redirect("%s?directory=%s" % (
+        reverse("dashboard.views.store-list"), path))
+
+
+@require_POST
+@login_required
+def store_refresh_toplist(request):
+    cache_key = "files-%d" % request.user.pk
+    cache = get_cache("default")
+    try:
+        store = Store(request.user)
+        toplist = store.toplist()
+        quota = store.get_quota()
+        files = {'toplist': toplist, 'quota': quota}
+    except Exception:
+        logger.exception("Can't get toplist of %s", unicode(request.user))
+        files = {'toplist': []}
+    cache.set(cache_key, files, 300)
+
+    return redirect(reverse("dashboard.index"))
+
+
 def absolute_url(url):
     return urljoin(settings.DJANGO_URL, url)