storage: add activity cleanup code

circle/storage/models.py

@@ -24,6 +24,7 @@ import logging
 from os.path import join
 import uuid
 
+from celery.signals import worker_ready
 from django.db.models import (Model, CharField, DateTimeField,
                               ForeignKey)
 from django.utils import timezone
@@ -631,3 +632,12 @@ def disk_activity(code_suffix, disk, task_uuid=None, user=None,
                   on_abort=None, on_commit=None):
     act = DiskActivity.create(code_suffix, disk, task_uuid, user)
     return activitycontextimpl(act, on_abort=on_abort, on_commit=on_commit)
+
+
+@worker_ready.connect()
+def cleanup(conf=None, **kwargs):
+    # TODO check if other manager workers are running
+    for i in DiskActivity.objects.filter(finished__isnull=True):
+        i.finish(False, "Manager is restarted, activity is cleaned up. "
+                 "You can try again now.")
+        logger.error('Forced finishing stale activity %s', i)