vm: clean up activities in progress on manager init

circle/vm/models/activity.py

@@ -19,6 +19,7 @@ from __future__ import absolute_import, unicode_literals
 from contextlib import contextmanager
 from logging import getLogger
 
+from celery.signals import worker_ready
 from django.core.urlresolvers import reverse
 from django.db.models import CharField, ForeignKey
 from django.utils import timezone
@@ -179,3 +180,14 @@ class NodeActivity(ActivityModel):
 def node_activity(code_suffix, node, task_uuid=None, user=None):
     act = NodeActivity.create(code_suffix, node, task_uuid, user)
     return activitycontextimpl(act)
+
+
+@worker_ready.connect()
+def cleanup(conf=None, **kwargs):
+    # TODO check if other manager workers are running
+    for i in InstanceActivity.objects.filter(finished__isnull=True):
+        i.finish(False, "Manager is restarted, activity is cleaned up. "
+                 "You can try again now.")
+    for i in NodeActivity.objects.filter(finished__isnull=True):
+        i.finish(False, "Manager is restarted, activity is cleaned up. "
+                 "You can try again now.")