Merge branch 'master' into export_import_disk

5179872f · Bálint Máhonfai · 7a8c2ac5 · 7a49024a · 5179872f · 5179872f
Commit 5179872f authored Nov 12, 2019 by Bálint Máhonfai
9 changed files
--- a/circle/dashboard/models.py
+++ b/circle/dashboard/models.py
@@ -162,7 +162,7 @@ class ConnectCommand(Model):
                         validators=[connect_command_template_validator])
    class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
    def __unicode__(self):
        return self.template
@@ -218,7 +218,7 @@ class Profile(Model):
                    'id': command.id,
                    'cmd': command.template % {
                        'port': instance.get_connect_port(use_ipv6=use_ipv6),
-                        'host':  instance.get_connect_host(use_ipv6=use_ipv6),
+                        'host': instance.get_connect_host(use_ipv6=use_ipv6),
                        'password': instance.pw,
                        'username': 'cloud',
                    }} for command in commands]
@@ -263,7 +263,7 @@ class Profile(Model):
        super(Profile, self).save(*args, **kwargs)
    class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
        permissions = (
            ('use_autocomplete', _('Can use autocomplete.')),
        )
@@ -275,7 +275,7 @@ class FutureMember(Model):
    group = ForeignKey(Group)
    class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
        unique_together = ('org_id', 'group')
    def __unicode__(self):
@@ -295,7 +295,7 @@ class GroupProfile(AclBase):
    description = TextField(blank=True)
    class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
    def __unicode__(self):
        return self.group.name
@@ -331,7 +331,11 @@ def create_profile(user):
    profile, created = Profile.objects.get_or_create(user=user)
    try:
-        Store(user).create_user(profile.smb_password, None, profile.disk_quota)
+        store = Store(user)
+        if store.user_exist():
+            profile.disk_quota = store.get_quota()['soft']
+            profile.save()
+        store.create_user(profile.smb_password, None, profile.disk_quota)
    except:
        logger.exception("Can't create user %s", unicode(user))
    return created

--- a/circle/dashboard/store_api.py
+++ b/circle/dashboard/store_api.py
@@ -44,9 +44,19 @@ class NoStoreException(StoreApiException):
    pass
+class NoOrgIdException(StoreApiException):
+    pass
 class Store(object):
    def __init__(self, user, default_timeout=0.5):
+        self.store_url = settings.STORE_URL
+        if not self.store_url:
+            raise NoStoreException
+        if not user.profile.org_id:
+            raise NoOrgIdException
+        self.username = 'u-%s' % user.profile.org_id
        self.request_args = {'verify': settings.STORE_VERIFY_SSL}
        if settings.STORE_SSL_AUTH:
            self.request_args['cert'] = (settings.STORE_CLIENT_CERT,
@@ -54,18 +64,15 @@ class Store(object):
        if settings.STORE_BASIC_AUTH:
            self.request_args['auth'] = (settings.STORE_CLIENT_USER,
                                         settings.STORE_CLIENT_PASSWORD)
-        self.username = "u-%d" % user.pk
        self.default_timeout = default_timeout
-        self.store_url = settings.STORE_URL
-        if not self.store_url:
-            raise NoStoreException
    def _request(self, url, method=get, timeout=None,
                 raise_status_code=True, **kwargs):
        url = urljoin(self.store_url, url)
        if timeout is None:
            timeout = self.default_timeout
-        payload = json.dumps(kwargs) if kwargs else None
+        kwargs['USER'] = self.username
+        payload = json.dumps(kwargs)
        try:
            headers = {'content-type': 'application/json'}
            response = method(url, data=payload, headers=headers,
@@ -83,7 +90,7 @@ class Store(object):
            return response
    def _request_cmd(self, cmd, **kwargs):
-        return self._request(self.username, post, CMD=cmd, **kwargs)
+        return self._request("/user/", post, CMD=cmd, **kwargs)
    def list(self, path, process=True):
        r = self._request_cmd("LIST", PATH=path)
@@ -119,7 +126,7 @@ class Store(object):
        self._request_cmd("RENAME", PATH=old_path, NEW_NAME=new_name)
    def get_quota(self):  # no CMD? :o
-        r = self._request(self.username)
+        r = self._request("/user/")
        quota = r.json()
        quota.update({
            'readable_used': filesizeformat(float(quota['used'])),
@@ -129,17 +136,17 @@ class Store(object):
        return quota
    def set_quota(self, quota):
-        self._request("/quota/" + self.username, post, QUOTA=quota)
+        self._request("/quota/", post, QUOTA=quota)
    def user_exist(self):
        try:
-            self._request(self.username)
+            self._request("/user/")
            return True
        except NotOkException:
            return False
    def create_user(self, password, keys, quota):
-        self._request("/new/" + self.username, method=post,
+        self._request("/new/", method=post,
                      SMBPASSWD=password, KEYS=keys, QUOTA=quota)
    @staticmethod

--- a/circle/dashboard/views/store.py
+++ b/circle/dashboard/views/store.py
@@ -35,7 +35,8 @@ from django.views.generic import TemplateView
 from braces.views import LoginRequiredMixin
-from ..store_api import Store, NoStoreException, NotOkException
+from ..store_api import (Store, NoStoreException,
+                         NotOkException, NoOrgIdException)
 logger = logging.getLogger(__name__)
@@ -70,6 +71,11 @@ class StoreList(LoginRequiredMixin, TemplateView):
                return super(StoreList, self).get(*args, **kwargs)
        except NoStoreException:
            messages.warning(self.request, _("No store."))
+        except NoOrgIdException:
+            messages.warning(self.request,
+                             _("Your organization ID is not set."
+                               " To use the store, you need a"
+                               " unique organization ID."))
        except NotOkException:
            messages.warning(self.request, _("Store has some problems now."
                                             " Try again later."))

--- a/circle/firewall/tasks/local_tasks.py
+++ b/circle/firewall/tasks/local_tasks.py
@@ -39,7 +39,7 @@ def _apply_once(name, tasks, queues, task, data):
    data = data()
    for queue in queues:
        try:
-            task.apply_async(args=data, queue=queue, expires=60).get(timeout=2)
+            task.apply_async(args=data, queue=queue, expires=60).get(timeout=5)
            logger.info("%s configuration is reloaded. (queue: %s)",
                        name, queue)
        except TimeoutError as e:
@@ -76,8 +76,14 @@ def reloadtask_worker():
    logger.info("reloadtask_worker: Reload %s", ", ".join(tasks))
    firewall_queues = get_firewall_queues()
-    dns_queues = [("%s.dns" % i) for i in
-                  settings.get('dns_queues', [gethostname()])]
+    dns_queues = settings.get('dns_queues', [gethostname()])
+    if isinstance(dns_queues, (str, unicode)):
+        dns_queues = [dns_queues]
+    dns_queues = [("%s.dns" % i) for i in dns_queues]
+#    dns_queues = [("%s.dns" % i) for i in
+#                  settings.get('dns_queues', [gethostname()])]
    _apply_once('dns', tasks, dns_queues, reload_dns,
                lambda: (dns(), ))

--- a/circle/manager/mancelery.py
+++ b/circle/manager/mancelery.py
@@ -18,6 +18,7 @@
 from celery import Celery
 from celery.signals import worker_ready
 from datetime import timedelta
+from celery.schedules import crontab
 from kombu import Queue, Exchange
 from os import getenv
@@ -52,7 +53,7 @@ celery.conf.update(
        'dashboard.send_email_notifications': {
            'task': 'dashboard.tasks.local_periodic_tasks.'
            'send_email_notifications',
-            'schedule': timedelta(hours=24),
+            'schedule': crontab(minute=10, hour=1),
            'options': {'queue': 'localhost.man'}
        },
    }

--- a/circle/storage/models.py
+++ b/circle/storage/models.py
@@ -86,7 +86,7 @@ class DataStore(Model):
            args=[self.path], queue=q).get(timeout=timeout)
    @method_cache(30)
-    def get_orphan_disks(self, timeout=15):
+    def get_orphan_disks(self, timeout=25):
        """Disk image files without Disk object in the database.
        """
        queue_name = self.get_remote_queue_name('storage', "slow")
@@ -101,7 +101,7 @@ class DataStore(Model):
        return orphans
    @method_cache(30)
-    def get_missing_disks(self, timeout=15):
+    def get_missing_disks(self, timeout=25):
        """Disk objects without disk image files.
        """
        queue_name = self.get_remote_queue_name('storage', "slow")
@@ -111,7 +111,7 @@ class DataStore(Model):
        return disks.exclude(filename__in=files)
    @method_cache(120)
-    def get_file_statistics(self, timeout=30):
+    def get_file_statistics(self, timeout=90):
        queue_name = self.get_remote_queue_name('storage', "slow")
        data = storage_tasks.get_file_statistics.apply_async(
            args=[self.path], queue=queue_name).get(timeout=timeout)

--- a/circle/vm/models/node.py
+++ b/circle/vm/models/node.py
@@ -349,7 +349,7 @@ class Node(OperatedMixin, TimeStampedModel):
                        continue
                    value = target['datapoints'][-2][0]
                    retval[metric] = float(value)
-                except (KeyError, IndexError, ValueError):
+                except (KeyError, IndexError, ValueError, TypeError):
                    continue
            return retval

--- a/circle/vm/operations.py
+++ b/circle/vm/operations.py
@@ -460,7 +460,7 @@ class DeployOperation(InstanceOperation):
        description = _("Deploy virtual machine.")
        remote_queue = ("vm", "slow")
        task = vm_tasks.deploy
+	remote_timeout = 120
        def _get_remote_args(self, **kwargs):
            return [self.instance.get_vm_desc()]
            # intentionally not calling super

--- a/circle/vm/tasks/local_periodic_tasks.py
+++ b/circle/vm/tasks/local_periodic_tasks.py
@@ -58,9 +58,9 @@ def garbage_collector(timeout=15):
                             i.pk, unicode(e))
        elif (i.time_of_suspend and now > i.time_of_suspend and
              i.state == 'RUNNING'):
-            i.sleep.async(system=True)
            logger.info("Expired instance %d suspended." % i.pk)
            try:
+            	i.sleep.async(system=True)
                i.owner.profile.notify(
                    ugettext_noop('%(instance)s suspended'),
                    ugettext_noop(
@@ -68,8 +68,10 @@ def garbage_collector(timeout=15):
                        'has been suspended due to expiration. '
                        'You can resume or destroy it.'),
                    instance=i.name, url=i.get_absolute_url())
+ 	    except ActivityInProgressError:
+            	logger.error("Expired instance %d can't be destroyed due the AtctivityInPorgressError.", i.pk)
            except Exception as e:
-                logger.debug('Could not notify owner of instance %d .%s',
+                logger.info('Could not notify owner of instance %d .%s',
                             i.pk, unicode(e))
        elif i.is_expiring():
            logger.debug("Instance %d expires soon." % i.pk)