portal: add timeout handling for StorageDetail view

storage: synchronize tasks with storagedriver, rework garbage collector logic

circle/dashboard/views/storage.py

@@ -41,7 +41,7 @@ from ..forms import (
 )
 from .util import FilterMixin
 import json
-
+from celery.exceptions import TimeoutError
 
 logger = logging.getLogger(__name__)
 
@@ -199,6 +199,13 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
             context['orphan_disks'] = ds.get_orphan_disks()
         except WorkerNotFound:
             messages.error(self.request, _("The DataStore is offline."))
+        except TimeoutError:
+            messages.error(self.request, _("Operation timed out, "
+                                           "some data may insufficient."))
+        except Exception as e:
+             messages.error(self.request, _("Error occured: %s, "
+                                           "some data may insufficient."
+                                           % unicode(e)))
 
         context['disk_table'] = DiskListTable(
             self.get_table_data(), request=self.request,

circle/storage/models.py

@@ -106,9 +106,11 @@ class DataStore(Model):
             raise WorkerNotFound()
 
     def get_deletable_disks(self):
-        return [disk.filename for disk in
-                self.disk_set.filter(
+        deletables = [disk for disk in self.disk_set.filter(
                       destroyed__isnull=False) if disk.is_deletable]
+        deletables = sorted(deletables, key=lambda disk: disk.destroyed)
+
+        return [disk.filename for disk in deletables]
 
     def get_hosts(self):
 
@@ -127,7 +129,7 @@ class DataStore(Model):
         try:
             return storage_tasks.get_storage_stat.apply_async(
                 args=[self.type, self.path], queue=q).get(timeout=timeout)
-        except TimeoutError:
+        except Exception:
             return {'free_space': -1,
                     'free_percent': -1}
 
@@ -593,11 +595,17 @@ class Disk(TimeStampedModel):
         """
         queue_name = self.datastore.get_remote_queue_name(
             'storage', priority='slow')
-        logger.info("Image: %s at Datastore: %s recovered from trash." %
-                    (self.filename, self.datastore.path))
-        storage_tasks.recover_from_trash.apply_async(
-            args=[self.datastore.path, self.filename],
+        res = storage_tasks.is_exists.apply_async(
+            args=[self.datastore.type,
+                  self.datastore.path,
+                  self.filename],
             queue=queue_name).get(timeout=timeout)
+        if res:
+            logger.info("Image: %s at Datastore: %s recovered." %
+                        (self.filename, self.datastore.path))
+        else:
+            logger.info("Image: %s at Datastore: %s not recovered." %
+                        (self.filename, self.datastore.path))
 
     def save_as(self, task=None, user=None, task_uuid=None, timeout=300):
         """Save VM as template.

circle/storage/tasks/periodic_tasks.py

@@ -36,17 +36,18 @@ def garbage_collector(timeout=15):
     for ds in DataStore.objects.all():
         queue_name = ds.get_remote_queue_name('storage', priority='fast')
         files = set(storage_tasks.list_files.apply_async(
-            args=[ds.path], queue=queue_name).get(timeout=timeout))
-        disks = set(ds.get_deletable_disks())
+            args=[ds.type, ds.path], queue=queue_name).get(timeout=timeout))
+        disks = ds.get_deletable_disks()
         queue_name = ds.get_remote_queue_name('storage', priority='slow')
-        for i in disks & files:
-            logger.info("Image: %s at Datastore: %s moved to trash folder." %
+
+        deletable_disks = [disk for disk in disks if disk in files]
+        for i in deletable_disks:
+            logger.info("Image: %s at Datastore: %s fetch for destroy." %
                         (i, ds.path))
-            storage_tasks.move_to_trash.apply_async(
-                args=[ds.path, i], queue=queue_name).get(timeout=timeout)
         try:
             storage_tasks.make_free_space.apply_async(
-                args=[ds.path], queue=queue_name).get(timeout=timeout)
+                args=[ds.type, ds.path, deletable_disks],
+                queue=queue_name).get(timeout=timeout)
         except Exception as e:
             logger.warning(str(e))
 

circle/storage/tasks/storage_tasks.py

@@ -39,12 +39,12 @@ def download(disk_desc, url):
 
 
 @celery.task(name='storagedriver.delete')
-def delete(path):
+def delete(disk_desc):
     pass
 
 
 @celery.task(name='storagedriver.delete_dump')
-def delete_dump(path):
+def delete_dump(data_store_type, path):
     pass
 
 
@@ -54,7 +54,7 @@ def snapshot(disk_desc):
 
 
 @celery.task(name='storagedriver.get')
-def get(json_data):
+def get(disk_desc):
     pass
 
 
@@ -63,18 +63,13 @@ def merge(src_disk_desc, dst_disk_desc):
     pass
 
 
-@celery.task(name='storagedriver.make_free_space')
-def make_free_space(datastore, percent):
+@celery.task(name='storagedriver.is_exists')
+def is_exists(data_store_type, path, disk_name):
     pass
 
 
-@celery.task(name='storagedriver.move_to_trash')
-def move_to_trash(datastore, disk_path):
-    pass
-
-
-@celery.task(name='storagedriver.recover_from_trash')
-def recover_from_trash(datastore, disk_path):
+@celery.task(name='storagedriver.make_free_space')
+def make_free_space(datastore, path, deletable_disks, percent):
     pass