Merge branch 'master' into featura-mount-store-op

b226eeef · Kálmán Viktor · 2c38e827 · fec75c1e · b226eeef · b226eeef
Commit b226eeef authored Aug 01, 2014 by Kálmán Viktor
72 changed files
--- a/circle/circle/settings/base.py
+++ b/circle/circle/settings/base.py
@@ -447,5 +447,16 @@ if graphite_host and graphite_port:
 else:
    GRAPHITE_URL = None
+STORE_BASIC_AUTH = get_env_variable("STORE_BASIC_AUTH", "") == "True"
+STORE_VERIFY_SSL = get_env_variable("STORE_VERIFY_SSL", "") == "True"
+STORE_SSL_AUTH = get_env_variable("STORE_SSL_AUTH", "") == "True"
+STORE_CLIENT_USER = get_env_variable("STORE_CLIENT_USER", "")
+STORE_CLIENT_PASSWORD = get_env_variable("STORE_CLIENT_PASSWORD", "")
+STORE_CLIENT_KEY = get_env_variable("STORE_CLIENT_KEY", "")
+STORE_CLIENT_CERT = get_env_variable("STORE_CLIENT_CERT", "")
+STORE_URL = get_env_variable("STORE_URL", "")
 SESSION_COOKIE_NAME = "csessid%x" % (((getnode() // 139) ^
                                      (getnode() % 983)) & 0xffff)
+MAX_NODE_RAM = get_env_variable("MAX_NODE_RAM", 1024)
--- a/circle/circle/settings/production.py
+++ b/circle/circle/settings/production.py
@@ -70,20 +70,14 @@ SERVER_EMAIL = EMAIL_HOST_USER
 ########## CACHE CONFIGURATION
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#caches
-try:
+from urlparse import urlsplit
-    CACHES = {
-        'default': {
+CACHES = {
-            'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+    'default': {
-            'LOCATION': get_env_variable('DJANGO_MEMCACHED'),
+        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
-        }
+        'LOCATION': urlsplit(get_env_variable('CACHE_URI')).netloc,
-    }
-except ImproperlyConfigured:
-    CACHES = {
-        'default': {
-            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-            'LOCATION': SITE_NAME,
-        }
    }
+}
 ########## END CACHE CONFIGURATION

--- a/circle/circle/settings/test.py
+++ b/circle/circle/settings/test.py
@@ -35,7 +35,11 @@ SOUTH_TESTS_MIGRATE = False
 INSTALLED_APPS += (
    'acl.tests',
+    'django_nose',
 )
+TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
+NOSE_ARGS = ['--with-doctest']
+PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher']
 CACHES = {
    'default': {
@@ -52,3 +56,5 @@ LOGGING['handlers']['console'] = {'level': level,
                                  'formatter': 'simple'}
 for i in LOCAL_APPS:
    LOGGING['loggers'][i] = {'handlers': ['console'], 'level': level}
+# Forbid store usage
+STORE_URL = ""
--- a/circle/common/models.py
+++ b/circle/common/models.py
@@ -23,6 +23,7 @@ from logging import getLogger
 from time import time
 from warnings import warn
+from django.contrib import messages
 from django.contrib.auth.models import User
 from django.core.cache import cache
 from django.core.serializers.json import DjangoJSONEncoder
@@ -46,17 +47,24 @@ class WorkerNotFound(Exception):
 def activitycontextimpl(act, on_abort=None, on_commit=None):
    try:
-        yield act
+        try:
-    except BaseException as e:
+            yield act
-        # BaseException is the common parent of Exception and
+        except HumanReadableException as e:
-        # system-exiting exceptions, e.g. KeyboardInterrupt
+            result = e
+            raise
+        except BaseException as e:
+            # BaseException is the common parent of Exception and
+            # system-exiting exceptions, e.g. KeyboardInterrupt
+            result = create_readable(
+                ugettext_noop("Failure."),
+                ugettext_noop("Unhandled exception: %(error)s"),
+                error=unicode(e))
+            raise
+    except:
+        logger.exception("Failed activity %s" % unicode(act))
        handler = None if on_abort is None else lambda a: on_abort(a, e)
-        result = create_readable(ugettext_noop("Failure."),
-                                 ugettext_noop("Unhandled exception: "
-                                               "%(error)s"),
-                                 error=unicode(e))
        act.finish(succeeded=False, result=result, event_handler=handler)
-        raise e
+        raise
    else:
        act.finish(succeeded=True, event_handler=on_commit)
@@ -70,11 +78,11 @@ activity_code_separator = '.'
 def has_prefix(activity_code, *prefixes):
    """Determine whether the activity code has the specified prefix.
-    E.g.: has_prefix('foo.bar.buz', 'foo.bar') == True
+    >>> assert has_prefix('foo.bar.buz', 'foo.bar')
-          has_prefix('foo.bar.buz', 'foo', 'bar') == True
+    >>> assert has_prefix('foo.bar.buz', 'foo', 'bar')
-          has_prefix('foo.bar.buz', 'foo.bar', 'buz') == True
+    >>> assert has_prefix('foo.bar.buz', 'foo.bar', 'buz')
-          has_prefix('foo.bar.buz', 'foo', 'bar', 'buz') == True
+    >>> assert has_prefix('foo.bar.buz', 'foo', 'bar', 'buz')
-          has_prefix('foo.bar.buz', 'foo', 'buz') == False
+    >>> assert not has_prefix('foo.bar.buz', 'foo', 'buz')
    """
    equal = lambda a, b: a == b
    act_code_parts = split_activity_code(activity_code)
@@ -85,11 +93,11 @@ def has_prefix(activity_code, *prefixes):
 def has_suffix(activity_code, *suffixes):
    """Determine whether the activity code has the specified suffix.
-    E.g.: has_suffix('foo.bar.buz', 'bar.buz') == True
+    >>> assert has_suffix('foo.bar.buz', 'bar.buz')
-          has_suffix('foo.bar.buz', 'bar', 'buz') == True
+    >>> assert has_suffix('foo.bar.buz', 'bar', 'buz')
-          has_suffix('foo.bar.buz', 'foo.bar', 'buz') == True
+    >>> assert has_suffix('foo.bar.buz', 'foo.bar', 'buz')
-          has_suffix('foo.bar.buz', 'foo', 'bar', 'buz') == True
+    >>> assert has_suffix('foo.bar.buz', 'foo', 'bar', 'buz')
-          has_suffix('foo.bar.buz', 'foo', 'buz') == False
+    >>> assert not has_suffix('foo.bar.buz', 'foo', 'buz')
    """
    equal = lambda a, b: a == b
    act_code_parts = split_activity_code(activity_code)
@@ -196,6 +204,10 @@ class ActivityModel(TimeStampedModel):
                 DeprecationWarning, stacklevel=2)
            value = create_readable(user_text_template="",
                                    admin_text_template=value)
+        elif not hasattr(value, "to_dict"):
+            warn("Use HumanReadableObject.", DeprecationWarning, stacklevel=2)
+            value = create_readable(user_text_template="",
+                                    admin_text_template=unicode(value))
        self.result_data = None if value is None else value.to_dict()
@@ -361,8 +373,9 @@ class HumanReadableObject(object):
    @classmethod
    def create(cls, user_text_template, admin_text_template=None, **params):
-        return cls(user_text_template,
+        return cls(user_text_template=user_text_template,
-                   admin_text_template or user_text_template, params)
+                   admin_text_template=(admin_text_template
+                                        or user_text_template), params=params)
    def set(self, user_text_template, admin_text_template=None, **params):
        self._set_values(user_text_template,
@@ -407,10 +420,29 @@ create_readable = HumanReadableObject.create
 class HumanReadableException(HumanReadableObject, Exception):
    """HumanReadableObject that is an Exception so can used in except clause.
    """
-    pass
+    def __init__(self, level=None, *args, **kwargs):
+        super(HumanReadableException, self).__init__(*args, **kwargs)
+        if level is not None:
+            if hasattr(messages, level):
+                self.level = level
+            else:
+                raise ValueError(
+                    "Level should be the name of an attribute of django."
+                    "contrib.messages (and it should be callable with "
+                    "(request, message)). Like 'error', 'warning'.")
+        else:
+            self.level = "error"
+    def send_message(self, request, level=None):
+        if request.user and request.user.is_superuser:
+            msg = self.get_admin_text()
+        else:
+            msg = self.get_user_text()
+        getattr(messages, level or self.level)(request, msg)
-def humanize_exception(message, exception=None, **params):
+def humanize_exception(message, exception=None, level=None, **params):
    """Return new dynamic-class exception which is based on
    HumanReadableException and the original class with the dict of exception.
@@ -423,4 +455,7 @@ def humanize_exception(message, exception=None, **params):
    Ex = type("HumanReadable" + type(exception).__name__,
              (HumanReadableException, type(exception)),
              exception.__dict__)
-    return Ex.create(message, **params)
+    ex = Ex.create(message, **params)
+    if level:
+        ex.level = level
+    return ex
--- a/circle/dashboard/fixtures/test-vm-fixture.json
+++ b/circle/dashboard/fixtures/test-vm-fixture.json
@@ -1322,7 +1322,7 @@
    "user_permissions": [
      115
    ], 
-    "password": "pbkdf2_sha256$10000$KIoeMs78MiOj$PnVXn3YJMehbOciBO32CMzqL0ZnQrzrdb7+b5dE13os=", 
+    "password": "md5$qLN4mQMOrsUJ$f07129fd1a289a0afb4e09f7a6816a4f",
    "email": "test@example.org", 
    "date_joined": "2013-09-04T15:29:49.914Z"
  }
@@ -1382,7 +1382,7 @@
    "pw": "ads", 
    "time_of_suspend": null, 
    "ram_size": 200, 
-    "priority": 4, 
+    "priority": 10, 
    "active_since": null, 
    "template": null, 
    "access_method": "nx", 
@@ -1412,7 +1412,7 @@
    "pw": "ads", 
    "time_of_suspend": null, 
    "ram_size": 200, 
-    "priority": 4, 
+    "priority": 10, 
    "active_since": null, 
    "template": null, 
    "access_method": "nx", 
@@ -1518,7 +1518,7 @@
    "ram_size": 1024, 
    "modified": "2014-01-24T00:58:19.654Z", 
    "system": "bubuntu", 
-    "priority": 20, 
+    "priority": 10, 
    "access_method": "ssh", 
    "raw_data": "", 
    "arch": "x86_64", 

--- a/circle/dashboard/forms.py
+++ b/circle/dashboard/forms.py
--- a/circle/dashboard/migrations/0010_auto__add_field_profile_smb_password__add_field_profile_disk_quota.py
+++ b/circle/dashboard/migrations/0010_auto__add_field_profile_smb_password__add_field_profile_disk_quota.py
--- a/circle/dashboard/migrations/0010_auto__add_futuremember__add_unique_futuremember_org_id_group.py
+++ b/circle/dashboard/migrations/0010_auto__add_futuremember__add_unique_futuremember_org_id_group.py
@@ -97,11 +97,13 @@ class Migration(SchemaMigration):
        },
        u'dashboard.profile': {
            'Meta': {'object_name': 'Profile'},
+            'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
            'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
            'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
            'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
+            'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
            'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
        },
@@ -269,4 +271,4 @@ class Migration(SchemaMigration):
        }
    }
    complete_apps = ['dashboard']
\ No newline at end of file
--- a/circle/dashboard/migrations/0011_auto__add_field_notification_subject_data__add_field_notification_mess.py
+++ b/circle/dashboard/migrations/0011_auto__add_field_notification_subject_data__add_field_notification_mess.py
@@ -98,11 +98,13 @@ class Migration(SchemaMigration):
        },
        u'dashboard.profile': {
            'Meta': {'object_name': 'Profile'},
+            'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
            'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
            'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
            'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
+            'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
            'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
        },
@@ -270,4 +272,4 @@ class Migration(SchemaMigration):
        }
    }
    complete_apps = ['dashboard']
\ No newline at end of file
--- a/circle/dashboard/migrations/0012_migrate_messages.py
+++ b/circle/dashboard/migrations/0012_migrate_messages.py
@@ -93,11 +93,13 @@ class Migration(DataMigration):
        },
        u'dashboard.profile': {
            'Meta': {'object_name': 'Profile'},
+            'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
            'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
            'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
            'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
+            'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
            'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
        },

--- a/circle/dashboard/migrations/0013_auto__del_field_notification_message__del_field_notification_subject.py
+++ b/circle/dashboard/migrations/0013_auto__del_field_notification_message__del_field_notification_subject.py
@@ -96,11 +96,13 @@ class Migration(SchemaMigration):
        },
        u'dashboard.profile': {
            'Meta': {'object_name': 'Profile'},
+            'disk_quota': ('django.db.models.fields.IntegerField', [], {'default': '2048'}),
            'email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance_limit': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
            'org_id': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
            'preferred_language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '32'}),
+            'smb_password': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
            'use_gravatar': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'user': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['auth.User']", 'unique': 'True'})
        },
@@ -268,4 +270,4 @@ class Migration(SchemaMigration):
        }
    }
    complete_apps = ['dashboard']
\ No newline at end of file
--- a/circle/dashboard/migrations/0014_auto__chg_field_profile_disk_quota.py
+++ b/circle/dashboard/migrations/0014_auto__chg_field_profile_disk_quota.py
--- a/circle/dashboard/models.py
+++ b/circle/dashboard/models.py
@@ -29,10 +29,13 @@ from django.db.models import (
    Model, ForeignKey, OneToOneField, CharField, IntegerField, TextField,
    DateTimeField, permalink, BooleanField
 )
-from django.db.models.signals import post_save, pre_delete
+from django.db.models.signals import post_save, pre_delete, post_delete
 from django.templatetags.static import static
 from django.utils.translation import ugettext_lazy as _
 from django_sshkey.models import UserKey
+from django.core.exceptions import ObjectDoesNotExist
+from sizefield.models import FileSizeField
 from jsonfield import JSONField
 from model_utils.models import TimeStampedModel
@@ -44,8 +47,12 @@ from common.models import HumanReadableObject, create_readable, Encoder
 from vm.tasks.agent_tasks import add_keys, del_keys
+from .store_api import Store, NoStoreException, NotOkException
 logger = getLogger(__name__)
+pwgen = User.objects.make_random_password
 class Favourite(Model):
    instance = ForeignKey("vm.Instance")
@@ -109,6 +116,18 @@ class Profile(Model):
    email_notifications = BooleanField(
        verbose_name=_("Email notifications"), default=True,
        help_text=_('Whether user wants to get digested email notifications.'))
+    smb_password = CharField(
+        max_length=20,
+        verbose_name=_('Samba password'),
+        help_text=_(
+            'Generated password for accessing store from '
+            'virtual machines.'),
+        default=pwgen,
+    )
+    disk_quota = FileSizeField(
+        verbose_name=_('disk quota'),
+        default=2048 * 1024 * 1024,
+        help_text=_('Disk quota in mebibytes.'))
    def notify(self, subject, template, context=None, valid_until=None,
               **kwargs):
@@ -201,6 +220,11 @@ def create_profile(sender, user, request, **kwargs):
    if not user.pk:
        return False
    profile, created = Profile.objects.get_or_create(user=user)
+    try:
+        Store(user).create_user(profile.smb_password, None, profile.disk_quota)
+    except:
+        logger.exception("Can't create user %s", unicode(user))
    return created
 user_logged_in.connect(create_profile)
@@ -268,6 +292,44 @@ else:
    logger.debug("Do not register save_org_id to djangosaml2 pre_user_save")
+def update_store_profile(sender, **kwargs):
+    profile = kwargs.get('instance')
+    keys = [i.key for i in profile.user.userkey_set.all()]
+    try:
+        s = Store(profile.user)
+        s.create_user(profile.smb_password, keys,
+                      profile.disk_quota)
+    except NoStoreException:
+        logger.debug("Store is not available.")
+    except NotOkException:
+        logger.critical("Store is not accepting connections.")
+post_save.connect(update_store_profile, sender=Profile)
+def update_store_keys(sender, **kwargs):
+    userkey = kwargs.get('instance')
+    try:
+        profile = userkey.user.profile
+    except ObjectDoesNotExist:
+        pass  # If there is no profile the user is deleted
+    else:
+        keys = [i.key for i in profile.user.userkey_set.all()]
+        try:
+            s = Store(userkey.user)
+            s.create_user(profile.smb_password, keys,
+                          profile.disk_quota)
+        except NoStoreException:
+            logger.debug("Store is not available.")
+        except NotOkException:
+            logger.critical("Store is not accepting connections.")
+post_save.connect(update_store_keys, sender=UserKey)
+post_delete.connect(update_store_keys, sender=UserKey)
 def add_ssh_keys(sender, **kwargs):
    from vm.models import Instance

--- a/circle/dashboard/static/dashboard/dashboard.css
+++ b/circle/dashboard/static/dashboard/dashboard.css
@@ -186,42 +186,6 @@ html {
  text-decoration: none !important;
 }
-.slider {
-    display: inline-block;
-}
-.slider .track {
-    height: 20px;
-    top: 50%;
-}
-.slider > .dragger, .slider > .dragger:hover {
-    border-radius: 0px;
-    -moz-border-radius: 0px;
-    -webkit-border-radius: 0px;
-    width: 8px;
-    height: 24px;
-    margin-top: -12px!important;
-    text-shadow: 0 1px 0 #fff;
-    background-image: -webkit-gradient(linear, left 0%, left 100%, from(#428bca), to(#3071a9));
-    background-image: -webkit-linear-gradient(top, #428bca, 0%, #3071a9, 100%);
-    background-image: -moz-linear-gradient(top, #428bca 0%, #3071a9 100%);
-    background-image: linear-gradient(to bottom, #428bca 0%, #3071a9 100%);
-    background-repeat: repeat-x;
-    border-color: #2d6ca2;
-    filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff428bca', endColorstr='#ff3071a9', GradientType=0);
-}
-.slider > .dragger:hover {
-    background-color: #3071a9;
-    background-image: none;
-    border-color: #2d6ca2;
-}
-.slider > .highlight-track {
-    height: 20px;
-    top: 50%;
-}
-.slider + .output {
-}
 .rule-table tr >:nth-child(1) {
    text-align: right;
 }
@@ -723,6 +687,82 @@ textarea[name="list-new-namelist"] {
 }
+#store-list-list {
+  list-style: none;
+}
+.store-list-item {
+  cursor: pointer;
+}
+.store-list-item:hover {
+  background: rgba(0, 0, 0, 0.6);
+}
+.store-list-item-icon {
+  width: 20px;
+  text-align: center;
+  display: inline-block;
+  margin-right: 15px;
+  float: left;
+}
+.store-list-item-size {
+  width: 70px;
+  text-align: right;
+  float: right;
+}
+.store-list-file-infos {
+  padding: 15px;
+  display: none;
+  border-left: 1px solid #ddd;
+  border-right: 1px solid #ddd;
+  position: relative;
+}
+.store-list-item-new {
+  display: inline-block;
+}
+.store-list-item-new .badge {
+  margin-left: 5px;
+  background: #5bc0dc;
+}
+.store-list-item-icon-directory {
+  color: #ff8c00;
+}
+.store-remove-button {
+  margin-top: 8px;
+}
+#dashboard-files-toplist div.list-group-item {
+  color: #555;
+}
+#dashboard-files-toplist div.list-group-item:hover {
+  background: #eee;
+}
+.store-list-item-name {
+  max-width: 70%;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+  overflow: hidden;
+  float: left;
+}
+.dashboard-toplist-icon {
+  float: left;
+  padding: 2px 5px 0 0;
+}
+.no-hover:hover {
+  background: none !important;
+}
 #group-detail-permissions .filtered {
  margin: 2px 0;
  padding: 2px 3px;
@@ -752,6 +792,74 @@ textarea[name="list-new-namelist"] {
  margin-top: -6px;
 }
+.store-action-button {
+  margin-left: 5px;
+}
+#progress-marker-hard {
+  border-top-right-radius: 4px;
+  border-bottom-right-radius: 4px;
+  right: 0;
+  background: red;
+}
+.progress-marker {
+  width: 6px;
+  height: 20px;
+  position: absolute;
+}
 #show-all-activities-container {
  margin: 20px 0 0 10px;
 }