add persistent disks

circle/common/models.py

@@ -250,6 +250,7 @@ class ActivityModel(TimeStampedModel):
             return 'failed'
 
 
+
 @celery.task()
 def compute_cached(method, instance, memcached_seconds,
                    key, start, *args, **kwargs):

circle/dashboard/serializers.py

@@ -4,7 +4,7 @@ from django.contrib.auth.models import Group, User
 
 from vm.models import Instance, InstanceTemplate, Lease, Interface, Node, InstanceActivity
 from firewall.models import Vlan
-from storage.models import Disk
+from storage.models import Disk, StorageActivity
 
 class InstanceActivitySerializer(serializers.ModelSerializer):
     get_percentage = serializers.IntegerField()
@@ -15,6 +15,12 @@ class InstanceActivitySerializer(serializers.ModelSerializer):
         fields = ('id', 'instance', 'resultant_state', 'interruptible', 'activity_code', 'parent', 
                     'task_uuid', 'user', 'started', 'finished', 'succeeded', 'result_data', 'created', 'modified', 'get_percentage')
 
+class StorageActivitySerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = StorageActivity
+        fields = ('id', 'parent', 'task_uuid', 'user', 'started', 'finished', 'succeeded', 'result_data', 'created', 'modified', 'disk')
+
     
 
 class GroupSerializer(serializers.ModelSerializer): 

circle/dashboard/urls.py

@@ -61,6 +61,8 @@ from .views import (
     VlanREST, ResizeDiskREST, GetVlanREST, DestroyDiskREST,
     StorageDetail, DiskDetail, UserREST, GroupREST,
     InstanceActivityREST, GetInstanceActivityREST,
+    SleepInstanceREST, WakeUpInstanceREST, DownloadPersistentDiskREST,
+    CreatePersistentDiskREST, GetStorageActivityREST,
     MessageList, MessageDetail, MessageCreate, MessageDelete,
     EnableTwoFactorView, DisableTwoFactorView,
     AclUserGroupAutocomplete, AclUserAutocomplete,
@@ -70,6 +72,9 @@ from .views.node import node_ops, NodeREST, GetNodeREST
 from .views.vm import vm_ops, vm_mass_ops
 
 urlpatterns = [
+    path('acpi/stact/<int:pk>/', GetStorageActivityREST.as_view()),
+    path('acpi/pddisk/', DownloadPersistentDiskREST.as_view()),
+    path('acpi/pcdisk/', CreatePersistentDiskREST.as_view()),
     path('acpi/vmact/', InstanceActivityREST.as_view()),
     path('acpi/vmact/<int:pk>/', GetInstanceActivityREST.as_view()),
     path('acpi/user/', UserREST.as_view()),
@@ -93,6 +98,8 @@ urlpatterns = [
     path('acpi/vm/<int:pk>/createdisk/', CreateDiskREST.as_view()),
     path('acpi/vm/<int:pk>/deploy/', DeployInstanceREST.as_view()),
     path('acpi/vm/<int:pk>/shutdown/', ShutdownInstanceREST.as_view()),
+    path('acpi/vm/<int:pk>/sleep/', SleepInstanceREST.as_view()),
+    path('acpi/vm/<int:pk>/wakeup/', WakeUpInstanceREST.as_view()),
     path('acpi/vm/<int:pk>/resizedisk/', ResizeDiskREST.as_view()),
     path('acpi/vm/<int:pk>/destroydisk/', DestroyDiskREST.as_view()),
     url(r'^$', IndexView.as_view(), name="dashboard.index"),

circle/dashboard/views/util.py

@@ -56,8 +56,9 @@ from rest_framework.views import APIView
 from rest_framework.parsers import JSONParser
 from rest_framework.authentication import TokenAuthentication, BasicAuthentication
 from rest_framework.permissions import IsAdminUser
+from storage.models import StorageActivity
 
-from dashboard.serializers import InstanceActivitySerializer
+from dashboard.serializers import InstanceActivitySerializer, StorageActivitySerializer
 
 from common.models import HumanReadableException, HumanReadableObject
 from ..models import GroupProfile, Profile
@@ -96,6 +97,16 @@ class GetInstanceActivityREST(APIView):
         return JsonResponse(serializer.data, safe=False)
 
 
+class GetStorageActivityREST(APIView):
+    authentication_classes = [TokenAuthentication,BasicAuthentication]
+    permission_classes = [IsAdminUser]
+
+    def get(self, request, pk, format=None):
+        act = StorageActivity.objects.get(pk=pk)
+        serializer = StorageActivitySerializer(act, many=False)
+        return JsonResponse(serializer.data, safe=False)
+
+
 class RedirectToLoginMixin(AccessMixin):
 
     redirect_exception_classes = (PermissionDenied, )

circle/dashboard/views/vm.py

@@ -17,6 +17,8 @@
 
 
 import json
+import queue
+import string
 import logging
 from collections import OrderedDict
 from os import getenv
@@ -44,8 +46,10 @@ from django.views.generic import (
 )
 
 from braces.views import SuperuserRequiredMixin, LoginRequiredMixin
+from storage.tasks import storage_tasks
+from vm.tasks.local_tasks import abortable_async_downloaddisk_operation
 from vm.operations import (DeployOperation, DestroyOperation, DownloadDiskOperation, ShutdownOperation, RenewOperation, 
-    ResizeDiskOperation, RemoveDiskOperation
+    ResizeDiskOperation, RemoveDiskOperation, SleepOperation, WakeUpOperation
 )
 
 from common.models import (
@@ -54,7 +58,7 @@ from common.models import (
 )
 from firewall.models import Vlan, Host, Rule
 from manager.scheduler import SchedulerError
-from storage.models import Disk
+from storage.models import Disk, StorageActivity
 from vm.models import (
     Instance, InstanceActivity, Node, Lease,
     InstanceTemplate, InterfaceTemplate, Interface,
@@ -100,9 +104,59 @@ from rest_framework.permissions import IsAdminUser
 
 from dashboard.serializers import (
     DiskSerializer, InstanceSerializer, InterfaceSerializer, CreateDiskSerializer, DownloadDiskSerializer, 
-    VMDeploySerializer, VlanSerializer, ResizeDiskSerializer, InstanceActivitySerializer, DestroyDiskSerializer,
+    VMDeploySerializer, VlanSerializer, ResizeDiskSerializer, InstanceActivitySerializer, DestroyDiskSerializer, StorageActivitySerializer,
 )
 
+def size_util(size: str):
+    size_dict = {
+        "GB": 1000000000,
+        "Gi": 1073741824,
+        "MB": 1000000,
+        "Mi": 1048576,
+        "KB": 1000,
+        "Ki": 1024,
+    }
+    res = re.search(r"(\d*)\s*(\w*)", size)
+    if res and res.group(1) and res.group(2):
+        return int(res.group(1)) * size_dict[str(res.group(2))]
+    raise ValidationErr()
+
+
+class CreatePersistentDiskREST(APIView):
+    authentication_classes = [TokenAuthentication,BasicAuthentication]
+    permission_classes = [IsAdminUser]
+
+    def post(self, request, format=None):
+        data = JSONParser().parse(request)
+        serializer = CreateDiskSerializer(data=data)
+        if serializer.is_valid():
+            disk_size = str(size_util(str(data['size'])))
+            disk_name = str(data['name'])
+            disk = Disk.create(size=disk_size, name=disk_name, type="qcow2-norm")
+            disk.full_clean()
+            disk.dev_num = 'f'
+            disk.save()
+            ret = DiskSerializer(disk, many=False)
+            return JsonResponse(ret.data, status=201)
+        return JsonResponse(serializer.errors, status=400)
+
+
+class DownloadPersistentDiskREST(APIView):
+    authentication_classes = [TokenAuthentication,BasicAuthentication]
+    permission_classes = [IsAdminUser]
+
+    def post(self, request, format=None):
+        data = JSONParser().parse(request)
+        serializer = DownloadDiskSerializer(data=data)
+        if serializer.is_valid():
+            disk_url = str(data['url'])
+            disk_name = str(data['name'])
+            store_act = StorageActivity.create(code_suffix="download_disk", user=request.user)
+            abortable_async_downloaddisk_operation.apply_async(args=(store_act.id, disk_url, disk_name), queue='localhost.man.slow')
+            serializer = StorageActivitySerializer(store_act, many=False)
+            return JsonResponse(serializer.data, status=201)
+        return JsonResponse(serializer.errors, status=400)
+
 class VlanREST(APIView):
     authentication_classes = [TokenAuthentication,BasicAuthentication]
     permission_classes = [IsAdminUser]
@@ -244,7 +298,29 @@ class DeployInstanceREST(APIView):
                 DeployOperation(instance).call(node=None, user=instance.owner)
             serializer = InstanceSerializer(instance, many=False)
             return JsonResponse(serializer.data, safe=False)
-        return JsonResponse(deploy.errors, status=400)
+        return JsonResponse(deploy.errors, status=201)
+
+
+class SleepInstanceREST(APIView):
+    authentication_classes = [TokenAuthentication,BasicAuthentication]
+    permission_classes = [IsAdminUser]
+
+    def post(self, request, pk, format=None):
+        instance = Instance.objects.get(pk=pk)
+        SleepOperation(instance).call(user=instance.owner)
+        serializer = InstanceSerializer(instance, many=False)
+        return JsonResponse(serializer.data, safe=False, status=201)
+
+
+class WakeUpInstanceREST(APIView):
+    authentication_classes = [TokenAuthentication,BasicAuthentication]
+    permission_classes = [IsAdminUser]
+
+    def post(self, request, pk, format=None):
+        instance = Instance.objects.get(pk=pk)
+        WakeUpOperation(instance).call(user=instance.owner)
+        serializer = InstanceSerializer(instance, many=False)
+        return JsonResponse(serializer.data, safe=False, status=201)
 
 
 class ShutdownInstanceREST(APIView):
@@ -255,7 +331,7 @@ class ShutdownInstanceREST(APIView):
         instance = Instance.objects.get(pk=pk)
         ShutdownOperation(instance).call(user=instance.owner)
         serializer = InstanceSerializer(instance, many=False)
-        return JsonResponse(serializer.data, status=400)
+        return JsonResponse(serializer.data, status=201)
 
 
 class DownloadDiskREST(APIView):
@@ -276,20 +352,6 @@ class DownloadDiskREST(APIView):
             return JsonResponse(serializer.data, status=201)
         return JsonResponse(serializer.errors, status=400)
 
-def size_util(size: str):
-    size_dict = {
-        "GB": 1000000000,
-        "Gi": 1073741824,
-        "MB": 1000000,
-        "Mi": 1048576,
-        "KB": 1000,
-        "Ki": 1024,
-    }
-    res = re.search(r"(\d*)\s*(\w*)", size)
-    if res and res.group(1) and res.group(2):
-        return int(res.group(1)) * size_dict[str(res.group(2))]
-    raise ValidationErr()
-
 class CreateDiskREST(APIView):
     authentication_classes = [TokenAuthentication,BasicAuthentication]
     permission_classes = [IsAdminUser]

circle/storage/admin.py

@@ -18,7 +18,7 @@
 from django import contrib
 # from django.utils.translation import ugettext_lazy as _
 
-from .models import Disk, DataStore
+from .models import Disk, DataStore, StorageActivity
 
 
 class DiskAdmin(contrib.admin.ModelAdmin):
@@ -32,3 +32,4 @@ class DataStoreAdmin(contrib.admin.ModelAdmin):
 
 contrib.admin.site.register(Disk, DiskAdmin)
 contrib.admin.site.register(DataStore, DataStoreAdmin)
+contrib.admin.site.register(StorageActivity)
\ No newline at end of file

circle/storage/migrations/0005_storageactivity.py

+# Generated by Django 3.2.3 on 2022-09-14 15:32
+
+import common.models
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import jsonfield.fields
+import model_utils.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('storage', '0004_disk_ci_disk'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='StorageActivity',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+                ('activity_code', models.CharField(max_length=100, verbose_name='activity code')),
+                ('readable_name_data', jsonfield.fields.JSONField(blank=True, dump_kwargs={'cls': common.models.Encoder}, help_text='Human readable name of activity.', null=True, verbose_name='human readable name')),
+                ('task_uuid', models.CharField(blank=True, help_text='Celery task unique identifier.', max_length=50, null=True, unique=True, verbose_name='task_uuid')),
+                ('started', models.DateTimeField(blank=True, help_text='Time of activity initiation.', null=True, verbose_name='started at')),
+                ('finished', models.DateTimeField(blank=True, help_text='Time of activity finalization.', null=True, verbose_name='finished at')),
+                ('succeeded', models.BooleanField(blank=True, help_text='True, if the activity has finished successfully.', null=True)),
+                ('result_data', jsonfield.fields.JSONField(blank=True, dump_kwargs={'cls': common.models.Encoder}, help_text='Human readable result of activity.', null=True, verbose_name='result')),
+                ('disk', models.ForeignKey(blank=True, help_text='Disks which are to be mounted.', null=True, on_delete=django.db.models.deletion.CASCADE, to='storage.disk', verbose_name='disk')),
+                ('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='storage.storageactivity')),
+                ('user', models.ForeignKey(blank=True, help_text='The person who started this activity.', null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='user')),
+            ],
+            options={
+                'db_table': 'st_act',
+                'ordering': ['-finished', '-started', '-id'],
+            },
+        ),
+    ]

circle/storage/models.py

@@ -39,7 +39,7 @@ from os.path import join
 from sizefield.models import FileSizeField
 
 from common.models import (
-    WorkerNotFound, HumanReadableException, humanize_exception, method_cache
+    WorkerNotFound, HumanReadableException, humanize_exception, join_activity_code, method_cache
 )
 from .tasks import local_tasks, storage_tasks
 
@@ -622,3 +622,23 @@ class Disk(TimeStampedModel):
     @property
     def is_exportable(self):
         return self.type in ('qcow2-norm', 'qcow2-snap', 'raw-rw', 'raw-ro')
+
+from common.models import ActivityModel
+
+class StorageActivity(ActivityModel):
+    disk = ForeignKey('storage.Disk', blank=True, null=True, verbose_name=_('disk'), on_delete=models.CASCADE,
+                            help_text=_('Disks which are to be mounted.'))
+    ACTIVITY_CODE_BASE = join_activity_code('st', 'Storage')
+
+    class Meta:
+        db_table = 'st_act'
+        ordering = ['-finished', '-started', '-id']
+
+    @classmethod
+    def create(cls, code_suffix, task_uuid=None, user=None):
+
+        activity_code = cls.construct_activity_code(code_suffix)
+        act = cls(activity_code=activity_code, parent=None,
+                    started=timezone.now(), task_uuid=task_uuid, user=user)
+        act.save()
+        return act
\ No newline at end of file

circle/vm/migrations/0010_auto_20220914_1532.py

+# Generated by Django 3.2.3 on 2022-09-14 15:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vm', '0009_auto_20220721_1118'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instance',
+            name='ci_meta_data',
+            field=models.TextField(blank=True, default='instance-id: {{ hostname }} \nlocal-hostname: {{ hostname }} \ncloud-name: circle3\nplatform: circle3', help_text='When cloud-init is active, set meta-data (YAML format)', verbose_name='CI Meta Data'),
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='ci_user_data',
+            field=models.TextField(blank=True, default='#cloud-config\n\nusers:\n  - name: {{ sysuser }} \n    sudo: [\'ALL=(ALL) NOPASSWD:ALL\']\n    groups: sudo\n    shell: /bin/bash\n    ssh_pwauth: True\n    chpasswd: { expire: False }\n    lock-passwd: false\n    passwd: "{{ password | hash }}"', help_text='When cloud-init is active, set user-data (YAML format)', verbose_name='CI User Data'),
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='has_agent',
+            field=models.BooleanField(default=False, help_text='If the machine has agent installed, and the manager should wait for its start.', verbose_name='has agent'),
+        ),
+        migrations.AlterField(
+            model_name='instancetemplate',
+            name='ci_meta_data',
+            field=models.TextField(blank=True, default='instance-id: {{ hostname }} \nlocal-hostname: {{ hostname }} \ncloud-name: circle3\nplatform: circle3', help_text='When cloud-init is active, set meta-data (YAML format)', verbose_name='CI Meta Data'),
+        ),
+        migrations.AlterField(
+            model_name='instancetemplate',
+            name='ci_user_data',
+            field=models.TextField(blank=True, default='#cloud-config\n\nusers:\n  - name: {{ sysuser }} \n    sudo: [\'ALL=(ALL) NOPASSWD:ALL\']\n    groups: sudo\n    shell: /bin/bash\n    ssh_pwauth: True\n    chpasswd: { expire: False }\n    lock-passwd: false\n    passwd: "{{ password | hash }}"', help_text='When cloud-init is active, set user-data (YAML format)', verbose_name='CI User Data'),
+        ),
+        migrations.AlterField(
+            model_name='instancetemplate',
+            name='has_agent',
+            field=models.BooleanField(default=False, help_text='If the machine has agent installed, and the manager should wait for its start.', verbose_name='has agent'),
+        ),
+    ]

circle/vm/operations.py

@@ -40,7 +40,7 @@ from re import search
 from sizefield.utils import filesizeformat
 
 from common.models import (
-    create_readable, humanize_exception, HumanReadableException
+    ActivityModel, create_readable, humanize_exception, HumanReadableException
 )
 from common.operations import Operation, register_operation, SubOperationMixin
 from dashboard.store_api import Store, NoStoreException
@@ -172,6 +172,38 @@ class InstanceOperation(Operation):
         return False
 
 
+class StorageOperation(Operation):
+    acl_level = 'owner'
+    async_operation = abortable_async_instance_operation
+    host_cls = Disk
+    concurrency_check = False
+    accept_states = None
+    deny_states = None
+    resultant_state = None
+
+    def __init__(self):
+        super(InstanceOperation, self).__init__(subject=None)
+
+    def check_precond(self):
+        pass
+
+    def check_auth(self, user):
+        if not user.is_superuser:
+            raise Exception()
+
+    def create_activity(self, parent, user, kwargs):
+        name = self.get_activity_name(kwargs)
+        return ActivityModel.create(
+            readable_name=name, user=user,
+            concurrency_check=self.concurrency_check,
+            resultant_state=self.resultant_state)
+
+    def is_preferred(self):
+        """If this is the recommended op in the current state of the instance.
+        """
+        return False
+
+
 class RemoteInstanceOperation(RemoteOperationMixin, InstanceOperation):
     remote_queue = ('vm', 'fast')
 
@@ -258,6 +290,7 @@ class CreateDiskOperation(InstanceOperation):
     description = _("Create and attach empty disk to the virtual machine.")
     required_perms = ('storage.create_empty_disk',)
     accept_states = ('STOPPED', 'PENDING', 'RUNNING')
+    concurrency_check = False
 
     def _operation(self, user, size, activity, name=None):
         from storage.models import Disk
@@ -330,6 +363,7 @@ class DownloadDiskOperation(InstanceOperation):
     required_perms = ('storage.download_disk',)
     accept_states = ('STOPPED', 'PENDING', 'RUNNING')
     async_queue = "localhost.man.slow"
+    concurrency_check = False # warning!!!
 
     def _operation(self, user, url, task, activity, name=None):
         disk = Disk.download(url=url, name=name, task=task)
@@ -354,6 +388,8 @@ class DownloadDiskOperation(InstanceOperation):
         
 
 
+
+
 @register_operation
 class ImportDiskOperation(InstanceOperation):
     id = 'import_disk'
@@ -915,7 +951,7 @@ class ShutdownOperation(AbortableRemoteOperationMixin,
     remote_queue = ("vm", "slow")
     remote_timeout = 180
 
-    def _operation(self, task):
+    def _operation(self, task=vm_tasks.shutdown):
         super(ShutdownOperation, self)._operation(task=task)
         self.instance.yield_node()
 

circle/vm/tasks/local_tasks.py

@@ -15,7 +15,10 @@
 # You should have received a copy of the GNU General Public License along
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
+from datetime import timezone
 from celery.contrib.abortable import AbortableTask
+from common.models import ActivityModel
+from storage.models import Disk, StorageActivity
 from manager.mancelery import celery
 
 
@@ -53,3 +56,21 @@ def abortable_async_node_operation(task, operation_id, node_pk, activity_pk,
     allargs['task'] = task
 
     return operation._exec_op(allargs, auxargs)
+
+
+@celery.task(base=AbortableTask, bind=True)
+def abortable_async_downloaddisk_operation(task, activity_pk, url, name):
+    activity = StorageActivity.objects.get(pk=activity_pk)
+    activity.task_uuid = task.request.id
+    activity.save()
+
+    disk = Disk.download(url=url, name=name, task=task)
+    disk.dev_num = 'g'
+    disk.full_clean()
+    disk.save()
+
+    activity.disk = disk
+    activity.succeeded = True
+    activity.finished =  timezone.now()
+    activity.save()
+    return
\ No newline at end of file