Commit c9d8c0ae by Bach Dániel

Merge branch 'feature-store' into 'master'

Feature Store

  Add SMB and Quota parameter
   Handle private_keys for sshfs and pair it with django_sshkey (Is the hook ready?)
 Web interface
🚧 Show user scp/sftp info
 Fix tests and it's complete.
parents 89d48699 090a6a97
......@@ -447,5 +447,14 @@ if graphite_host and graphite_port:
else:
GRAPHITE_URL = None
STORE_BASIC_AUTH = get_env_variable("STORE_BASIC_AUTH", "") == "True"
STORE_VERIFY_SSL = get_env_variable("STORE_VERIFY_SSL", "") == "True"
STORE_SSL_AUTH = get_env_variable("STORE_SSL_AUTH", "") == "True"
STORE_CLIENT_USER = get_env_variable("STORE_CLIENT_USER", "")
STORE_CLIENT_PASSWORD = get_env_variable("STORE_CLIENT_PASSWORD", "")
STORE_CLIENT_KEY = get_env_variable("STORE_CLIENT_KEY", "")
STORE_CLIENT_CERT = get_env_variable("STORE_CLIENT_CERT", "")
STORE_URL = get_env_variable("STORE_URL", "")
SESSION_COOKIE_NAME = "csessid%x" % (((getnode() // 139) ^
(getnode() % 983)) & 0xffff)
......@@ -56,3 +56,5 @@ LOGGING['handlers']['console'] = {'level': level,
'formatter': 'simple'}
for i in LOCAL_APPS:
LOGGING['loggers'][i] = {'handlers': ['console'], 'level': level}
# Forbid store usage
STORE_URL = ""
......@@ -97,11 +97,13 @@ class Migration(SchemaMigration):
},
u'dashboard.profile': {
'Meta': {'object_name': 'Profile'},
'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
},
......@@ -269,4 +271,4 @@ class Migration(SchemaMigration):
}
}
complete_apps = ['dashboard']
\ No newline at end of file
complete_apps = ['dashboard']
......@@ -98,11 +98,13 @@ class Migration(SchemaMigration):
},
u'dashboard.profile': {
'Meta': {'object_name': 'Profile'},
'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
},
......@@ -270,4 +272,4 @@ class Migration(SchemaMigration):
}
}
complete_apps = ['dashboard']
\ No newline at end of file
complete_apps = ['dashboard']
......@@ -93,11 +93,13 @@ class Migration(DataMigration):
},
u'dashboard.profile': {
'Meta': {'object_name': 'Profile'},
'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
},
......
......@@ -96,11 +96,13 @@ class Migration(SchemaMigration):
},
u'dashboard.profile': {
'Meta': {'object_name': 'Profile'},
'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
},
......@@ -268,4 +270,4 @@ class Migration(SchemaMigration):
}
}
complete_apps = ['dashboard']
\ No newline at end of file
complete_apps = ['dashboard']
......@@ -29,10 +29,13 @@ from django.db.models import (
Model, ForeignKey, OneToOneField, CharField, IntegerField, TextField,
DateTimeField, permalink, BooleanField
)
from django.db.models.signals import post_save, pre_delete
from django.db.models.signals import post_save, pre_delete, post_delete
from django.templatetags.static import static
from django.utils.translation import ugettext_lazy as _
from django_sshkey.models import UserKey
from django.core.exceptions import ObjectDoesNotExist
from sizefield.models import FileSizeField
from jsonfield import JSONField
from model_utils.models import TimeStampedModel
......@@ -44,8 +47,12 @@ from common.models import HumanReadableObject, create_readable, Encoder
from vm.tasks.agent_tasks import add_keys, del_keys
from .store_api import Store, NoStoreException, NotOkException
logger = getLogger(__name__)
pwgen = User.objects.make_random_password
class Favourite(Model):
instance = ForeignKey("vm.Instance")
......@@ -109,6 +116,18 @@ class Profile(Model):
email_notifications = BooleanField(
verbose_name=_("Email notifications"), default=True,
help_text=_('Whether user wants to get digested email notifications.'))
smb_password = CharField(
max_length=20,
verbose_name=_('Samba password'),
help_text=_(
'Generated password for accessing store from '
'virtual machines.'),
default=pwgen,
)
disk_quota = FileSizeField(
verbose_name=_('disk quota'),
default=2048 * 1024 * 1024,
help_text=_('Disk quota in mebibytes.'))
def notify(self, subject, template, context=None, valid_until=None,
**kwargs):
......@@ -201,6 +220,11 @@ def create_profile(sender, user, request, **kwargs):
if not user.pk:
return False
profile, created = Profile.objects.get_or_create(user=user)
try:
Store(user).create_user(profile.smb_password, None, profile.disk_quota)
except:
logger.exception("Can't create user %s", unicode(user))
return created
user_logged_in.connect(create_profile)
......@@ -268,6 +292,44 @@ else:
logger.debug("Do not register save_org_id to djangosaml2 pre_user_save")
def update_store_profile(sender, **kwargs):
profile = kwargs.get('instance')
keys = [i.key for i in profile.user.userkey_set.all()]
try:
s = Store(profile.user)
s.create_user(profile.smb_password, keys,
profile.disk_quota)
except NoStoreException:
logger.debug("Store is not available.")
except NotOkException:
logger.critical("Store is not accepting connections.")
post_save.connect(update_store_profile, sender=Profile)
def update_store_keys(sender, **kwargs):
userkey = kwargs.get('instance')
try:
profile = userkey.user.profile
except ObjectDoesNotExist:
pass # If there is no profile the user is deleted
else:
keys = [i.key for i in profile.user.userkey_set.all()]
try:
s = Store(userkey.user)
s.create_user(profile.smb_password, keys,
profile.disk_quota)
except NoStoreException:
logger.debug("Store is not available.")
except NotOkException:
logger.critical("Store is not accepting connections.")
post_save.connect(update_store_keys, sender=UserKey)
post_delete.connect(update_store_keys, sender=UserKey)
def add_ssh_keys(sender, **kwargs):
from vm.models import Instance
......
......@@ -723,6 +723,82 @@ textarea[name="list-new-namelist"] {
}
#store-list-list {
list-style: none;
}
.store-list-item {
cursor: pointer;
}
.store-list-item:hover {
background: rgba(0, 0, 0, 0.6);
}
.store-list-item-icon {
width: 20px;
text-align: center;
display: inline-block;
margin-right: 15px;
float: left;
}
.store-list-item-size {
width: 70px;
text-align: right;
float: right;
}
.store-list-file-infos {
padding: 15px;
display: none;
border-left: 1px solid #ddd;
border-right: 1px solid #ddd;
position: relative;
}
.store-list-item-new {
display: inline-block;
}
.store-list-item-new .badge {
margin-left: 5px;
background: #5bc0dc;
}
.store-list-item-icon-directory {
color: #ff8c00;
}
.store-remove-button {
margin-top: 8px;
}
#dashboard-files-toplist div.list-group-item {
color: #555;
}
#dashboard-files-toplist div.list-group-item:hover {
background: #eee;
}
.store-list-item-name {
max-width: 70%;
text-overflow: ellipsis;
white-space: nowrap;
overflow: hidden;
float: left;
}
.dashboard-toplist-icon {
float: left;
padding: 2px 5px 0 0;
}
.no-hover:hover {
background: none !important;
}
#group-detail-permissions .filtered {
margin: 2px 0;
padding: 2px 3px;
......@@ -752,6 +828,23 @@ textarea[name="list-new-namelist"] {
margin-top: -6px;
}
.store-action-button {
margin-left: 5px;
}
#progress-marker-hard {
border-top-right-radius: 4px;
border-bottom-right-radius: 4px;
right: 0;
background: red;
}
.progress-marker {
width: 6px;
height: 20px;
position: absolute;
}
#show-all-activities-container {
margin: 20px 0 0 10px;
}
......@@ -112,6 +112,7 @@ $(function () {
/* no js compatibility */
noJS();
$('.no-js-hidden').show();
$('.js-hidden').hide();
......@@ -561,3 +562,10 @@ function getCookie(name) {
}
return cookieValue;
}
/* no js compatibility */
function noJS() {
$('.no-js-hidden').show();
$('.js-hidden').hide();
}
$(function() {
$("#store-list-container").on("click", ".store-list-item", function() {
if($(this).data("item-type") == "D") {
$("#store-list-up-icon").removeClass("fa-reply").addClass("fa-refresh fa-spin");
var url = $(this).prop("href");
$.get(url, function(result) {
$("#store-list-container").html(result);
noJS();
$("[title]").tooltip();
history.pushState({}, "", url);
});
} else {
$(this).next(".store-list-file-infos").stop().slideToggle();
}
return false;
});
/* how upload works
* - user clicks on a "fake" browse button, this triggers a click event on the file upload
* - if the file input changes it adds the name of the file to form (or number of files if multiple is enabled)
* - and finally when we click on the upload button (this event handler) it firsts ask the store api where to upload
* then changes the form's action attr before sending the form itself
*/
$("#store-list-container").on("click", '#store-upload-form button[type="submit"]', function() {
var current_dir = $("#store-upload-form").find('[name="current_dir"]').val();
$.get($("#store-upload-form").data("action") + "?current_dir=" + current_dir, function(result) {
$('#store-upload-form button[type="submit"] i').addClass("fa-spinner fa-spin");
$("#store-upload-form").get(0).setAttribute("action", result['url']);
$("#store-upload-form").submit();
});
return false;
});
/* "fake" browse button */
$("#store-list-container").on("click", "#store-upload-browse", function() {
$('#store-upload-form input[type="file"]').click();
});
$("#store-list-container").on("change", "#store-upload-file", function() {
var input = $(this);
var numFiles = input.get(0).files ? input.get(0).files.length : 1;
var label = input.val().replace(/\\/g, '/').replace(/.*\//, '');
input.trigger('fileselect', [numFiles, label]);
});
$("#store-list-container").on("fileselect", "#store-upload-file", function(event, numFiles, label) {
var input = $("#store-upload-filename");
var log = numFiles > 1 ? numFiles + ' files selected' : label;
if(input.length) {
input.val(log);
}
if(log) {
$('#store-upload-form button[type="submit"]').prop("disabled", false);
} else {
$('#store-upload-form button[type="submit"]').prop("disabled", true);
}
});
});
from os.path import splitext
import json
import logging
from urlparse import urljoin
from datetime import datetime
from django.http import Http404
from django.conf import settings
from requests import get, post, codes
from sizefield.utils import filesizeformat
logger = logging.getLogger(__name__)
class StoreApiException(Exception):
pass
class NotOkException(StoreApiException):
def __init__(self, status, *args, **kwargs):
self.status = status
super(NotOkException, self).__init__(*args, **kwargs)
class NoStoreException(StoreApiException):
pass
class Store(object):
def __init__(self, user, default_timeout=0.5):
self.request_args = {'verify': settings.STORE_VERIFY_SSL}
if settings.STORE_SSL_AUTH:
self.request_args['cert'] = (settings.STORE_CLIENT_CERT,
settings.STORE_CLIENT_KEY)
if settings.STORE_BASIC_AUTH:
self.request_args['auth'] = (settings.STORE_CLIENT_USER,
settings.STORE_CLIENT_PASSWORD)
self.username = "u-%d" % user.pk
self.default_timeout = default_timeout
self.store_url = settings.STORE_URL
if not self.store_url:
raise NoStoreException
def _request(self, url, method=get, timeout=None,
raise_status_code=True, **kwargs):
url = urljoin(self.store_url, url)
if timeout is None:
timeout = self.default_timeout
payload = json.dumps(kwargs) if kwargs else None
try:
headers = {'content-type': 'application/json'}
response = method(url, data=payload, headers=headers,
timeout=timeout, **self.request_args)
except Exception:
logger.exception("Error in store %s loading %s",
unicode(method), url)
raise
else:
if raise_status_code and response.status_code != codes.ok:
if response.status_code == 404:
raise Http404()
else:
raise NotOkException(response.status_code)
return response
def _request_cmd(self, cmd, **kwargs):
return self._request(self.username, post, CMD=cmd, **kwargs)
def list(self, path, process=True):
r = self._request_cmd("LIST", PATH=path)
result = r.json()
if process:
return self._process_list(result)
else:
return result
def toplist(self, process=True):
r = self._request_cmd("TOPLIST")
result = r.json()
if process:
return self._process_list(result)
else:
return result
def request_download(self, path):
r = self._request_cmd("DOWNLOAD", PATH=path, timeout=10)
return r.json()['LINK']
def request_upload(self, path):
r = self._request_cmd("UPLOAD", PATH=path)
return r.json()['LINK']
def remove(self, path):
self._request_cmd("REMOVE", PATH=path)
def new_folder(self, path):
self._request_cmd("NEW_FOLDER", PATH=path)
def rename(self, old_path, new_name):
self._request_cmd("RENAME", PATH=old_path, NEW_NAME=new_name)
def get_quota(self): # no CMD? :o
r = self._request(self.username)
quota = r.json()
quota.update({
'readable_used': filesizeformat(float(quota['used'])),
'readable_soft': filesizeformat(float(quota['soft'])),
'readable_hard': filesizeformat(float(quota['hard'])),
})
return quota
def set_quota(self, quota):
self._request("/quota/" + self.username, post, QUOTA=quota)
def user_exist(self):
try:
self._request(self.username)
return True
except NotOkException:
return False
def create_user(self, password, keys, quota):
self._request("/new/" + self.username, method=post,
SMBPASSWD=password, KEYS=keys, QUOTA=quota)
@staticmethod
def _process_list(content):
for d in content:
d['human_readable_date'] = datetime.utcfromtimestamp(float(
d['MTIME']))
delta = (datetime.utcnow() -
d['human_readable_date']).total_seconds()
d['is_new'] = 0 < delta < 5
d['human_readable_size'] = (
"directory" if d['TYPE'] == "D" else
filesizeformat(float(d['SIZE'])))
if d['DIR'] == ".":
d['directory'] = "/"
else:
d['directory'] = "/" + d['DIR'] + "/"
d['path'] = d['directory']
d['path'] += d['NAME']
if d['TYPE'] == "D":
d['path'] += "/"
d['ext'] = splitext(d['path'])[1]
d['icon'] = ("folder-open" if not d['TYPE'] == "F"
else file_icons.get(d['ext'], "file-o"))
return sorted(content, key=lambda k: k['TYPE'])
file_icons = {
'.txt': "file-text-o",
'.pdf': "file-pdf-o",
'.jpg': "file-image-o",
'.jpeg': "file-image-o",
'.png': "file-image-o",
'.gif': "file-image-o",
'.avi': "file-video-o",
'.mkv': "file-video-o",
'.mp4': "file-video-o",
'.mov': "file-video-o",
'.mp3': "file-sound-o",
'.flac': "file-sound-o",
'.wma': "file-sound-o",
'.pptx': "file-powerpoint-o",
'.ppt': "file-powerpoint-o",
'.doc': "file-word-o",
'.docx': "file-word-o",
'.xlsx': "file-excel-o",
'.xls': "file-excel-o",
'.rar': "file-archive-o",
'.zip': "file-archive-o",
'.7z': "file-archive-o",
'.tar': "file-archive-o",
'.gz': "file-archive-o",
'.py': "file-code-o",
'.html': "file-code-o",
'.js': "file-code-o",
'.css': "file-code-o",
'.c': "file-code-o",
'.cpp': "file-code-o",
'.h': "file-code-o",
'.sh': "file-code-o",
}
......@@ -23,11 +23,11 @@
</div>
{% endif %}
{% comment %}
{% if not no_store %}
<div class="col-lg-4 col-sm-6">
{% include "dashboard/index-files.html" %}
{% include "dashboard/store/index-files.html" %}
</div>
{% endcomment %}
{% endif %}
{% if perms.vm.create_template %}
<div class="col-lg-4 col-sm-6">
......
{% load i18n %}
<div class="list-group">
<div class="list-group-item">
<div class="row">
<div class="col-sm-6">
<a href="{% url "dashboard.views.store-upload"%}?directory={{ current }}"
class="btn btn-info btn-xs js-hidden">
{% trans "Upload" %}
</a>
<form action="" data-action="{% url "dashboard.views.store-upload-url" %}"
method="POST" enctype="multipart/form-data" class="no-js-hidden"
id="store-upload-form">
{% csrf_token %}
<input type="hidden" name="current_dir" value="{{ current }}"/>
<input type="hidden" name="next" value="{{ next_url }}"/>
<div class="input-group" style="max-width: 350px;">
<span class="input-group-btn" id="store-upload-browse">
<span class="btn btn-primary btn-xs">
{% trans "Browse..." %}
</span>
</span>
<input type="text" class="form-control input-tags"
id="store-upload-filename"/>
<span class="input-group-btn">
<button type="submit" class="btn btn-primary btn-xs" disabled>
<i class="fa fa-cloud-upload"></i> {% trans "Upload" %}
</button>
</span>
</div>
<input id="store-upload-file" name="data" type="file" style="display:none">
</form>
</div><!-- .col-sm-6 upload -->
<div class="col-sm-6">
<a href="{% url "dashboard.views.store-remove" %}?path={{ current }}"
class="btn btn-danger btn-xs pull-right store-action-button"
title="{% trans "Remove directory" %}">
<i class="fa fa-times"></i>
</a>
<a href="{% url "dashboard.views.store-download" %}?path={{ current }}"
class="btn btn-primary btn-xs pull-right store-action-button"
title="{% trans "Download directory" %}">
<i class="fa fa-cloud-download"></i>
</a>
<form method="POST" action="{% url "dashboard.views.store-new-directory" %}">
{% csrf_token %}
<input type="hidden" name="path" value="{{ current }}"/>
<div class="input-group" style="max-width: 300px;">
<span class="input-group-addon input-tags" title="{% trans "New directory" %}">
<i class="fa fa-folder-open"></i>
</span>
<input type="text" class="form-control input-tags" name="name"
placeholder="{% trans "Name "%}" required/>
<span class="input-group-btn">
<input type="submit" class="btn btn-success btn-xs" value="{% trans "Create" %}"/>
</span>
</div>
</form>
</div><!-- .col-sm-6 -->
</div><!-- .row -->
</div><!-- .list-group-item -->
</div><!-- .list-group -->
<div class="list-group" id="store-list-list">
<a href="{% url "dashboard.views.store-list" %}?directory={{ up_url }}"
class="list-group-item store-list-item" data-item-type="D">
{% if current == "/" %}
<div class="store-list-item-icon">
<i class="fa fa-refresh" id="store-list-up-icon"></i>
</div>
{% trans "Refresh" %}
{% else %}
<div class="store-list-item-icon">
<i class="fa fa-reply" id="store-list-up-icon"></i>
</div>
..
{% endif %}
<div class="pull-right">
{{ current }}
</div>
</a>
{% for f in root %}
<a class="list-group-item store-list-item" data-item-type="{{ f.TYPE }}"
href="{% if f.TYPE == "D" %}{% url "dashboard.views.store-list" %}?directory={{ f.path }}{% else %}
{% url "dashboard.views.store-download" %}?path={{ f.path }}{% endif %}"
>
<div class="store-list-item-icon">
<i class="
fa fa-{{ f.icon }}{% if f.TYPE == "D" %} store-list-item-icon-directory{% endif %}"
></i>
</div>
<div class="store-list-item-name">
{{ f.NAME }}
</div>
<div class="store-list-item-new">
{% if f.is_new and f.TYPE == "F" %}
<span class="badge badge-pulse">{% trans "new" %}</span>
{% endif %}
</div>
<div class="store-list-item-size">
{{ f.human_readable_size }}
</div>
<div class="clearfix"></div>
</a>
<div class="store-list-file-infos">
<div class="row">
<div class="col-sm-10">
<dl class="dl-horizontal" style="margin: 0; padding: 0;">
<dt>{% trans "Filename" %}</dt>
<dd>{{ f.NAME }}</dd>
<dt>{% trans "Size" %}</dt>
<dd>{{ f.human_readable_size }}</dd>
<dt>{