Merge branch 'balancervm' into 'master'

Balancervm

See merge request !25

circle/dashboard/forms.py

@@ -92,12 +92,15 @@ class VmSaveForm(OperationForm):
                            help_text=_('Human readable name of template.'))
     datastore = forms.ModelChoiceField(queryset=None, initial=0, empty_label=None, 
                     help_text=_('Backing file location'))
+    overlay_path = forms.ModelChoiceField(queryset=None, initial=0, empty_label=None, 
+                    help_text=_('Overlay image dir'))
 
     def __init__(self, *args, **kwargs):
         default = kwargs.pop('default', None)
         clone = kwargs.pop('clone', False)
         super(VmSaveForm, self).__init__(*args, **kwargs)
         self.fields['datastore'].queryset = DataStore.objects.all()
+        self.fields['overlay_path'].queryset = DataStore.objects.all()
         if default:
             self.fields['name'].initial = default
         if clone:
@@ -856,6 +859,14 @@ class VmCreateDiskForm(OperationForm):
         help_text=_('Size of disk to create in bytes or with units '
                     'like MB or GB.'))
     datastore = forms.ModelChoiceField(queryset=None, initial=0, empty_label=None)
+    cache_size = forms.CharField(
+        widget=FileSizeWidget, initial=(1 << 20), label=_('Metadata cache size'),
+        help_text=_('Metadata cache size, '
+                    'like KB, MB'))
+    cluster_size = forms.CharField(
+        widget=FileSizeWidget, initial=(1 << 16), label=_('Cluster size'),
+        help_text=_('Disk cluster (block) size, '
+                    'like KB'))
 
     def __init__(self, *args, **kwargs):
         default = kwargs.pop('default', None)
@@ -871,6 +882,20 @@ class VmCreateDiskForm(OperationForm):
                                           " GB or MB!"))
         return size_in_bytes
 
+    def clean_cache_size(self):
+        size_in_kbytes = self.cleaned_data.get("cache_size")
+        if not size_in_kbytes.isdigit() and len(size_in_kbytes) > 0:
+            raise forms.ValidationError(_("Invalid format, you can use "
+                                          " KB or MB!"))
+        return int(size_in_kbytes) / 1024
+    
+    def clean_cluster_size(self):
+        size_in_kbytes = self.cleaned_data.get("cluster_size")
+        if not size_in_kbytes.isdigit() and len(size_in_kbytes) > 0:
+            raise forms.ValidationError(_("Invalid format, you can use "
+                                          " KB or MB!"))
+        return int(size_in_kbytes) / 1024
+
 
 class VmDiskExportForm(OperationForm):
     exported_name = forms.CharField(max_length=100, label=_('Filename'))

circle/dashboard/templates/dashboard/template-edit.html

@@ -69,6 +69,7 @@
         {{ form.lease|as_crispy_field }}
 
         {{ form.tags|as_crispy_field }}
+        {{ form.overlay_path|as_crispy_field }}
         </fieldset>
 
         <input type="submit" value="{% trans "Save changes" %}" class="btn btn-primary">

circle/dashboard/urls.py

@@ -64,7 +64,7 @@ from .views import (
     SleepInstanceREST, WakeUpInstanceREST, DownloadPersistentDiskREST,
     CreatePersistentDiskREST, GetStorageActivityREST, GetTemplateREST,
     MessageList, MessageDetail, MessageCreate, MessageDelete,
-    SetupPortREST, RulesREST,
+    SetupPortREST, RulesREST, InstanceBalancerTemplateREST,
     EnableTwoFactorView, DisableTwoFactorView,
     AclUserGroupAutocomplete, AclUserAutocomplete,
     RescheduleView, GroupImportView, GroupExportView,
@@ -91,6 +91,7 @@ urlpatterns = [
     path('acpi/vm/<int:pk>/', GetInstanceREST.as_view()),
     path('acpi/template/<int:pk>/', GetTemplateREST.as_view()),
     path('acpi/template/', TemplateREST.as_view()),
+    path('acpi/bvm/', InstanceBalancerTemplateREST.as_view()),
     path('acpi/ft/', InstanceFromTemplateREST.as_view()),
     path('acpi/lease/', LeaseREST.as_view()),
     path('acpi/lease/<int:pk>/', GetLeaseREST.as_view()),

circle/dashboard/views/template.py

@@ -230,6 +230,41 @@ class GetTemplateREST(APIView):
     def delete(self, request, pk, format=None):
         return JsonResponse(status=400)
 
+class InstanceBalancerTemplateREST(APIView):
+    authentication_classes = [TokenAuthentication,BasicAuthentication]
+    permission_classes = [IsAdminUser]
+
+    def post(self, request, format=None):
+        data = JSONParser().parse(request)
+        user = User.objects.filter(username=data['username']).get()
+        template = InstanceTemplate.objects.filter(name=data['template_name']).get()
+        ikwargs = {
+            'name': data['name'],
+            'template': template,
+            'owner': user,
+        }
+        amount = data.get("amount", 1)
+        if 'num_cores' in data:
+            ikwargs.update({'num_cores':data['num_cores']})
+        
+        if 'ram_size' in data:
+            ikwargs.update({'ram_size':data['ram_size']})
+
+        if 'priority' in data:
+            ikwargs.update({'priority':data['priority']})
+
+        if 'max_ram_size' in data:
+            ikwargs.update({'max_ram_size':data['ram_size']})
+
+        instances = Instance.mass_create_from_template(amount=amount,
+                                                       **ikwargs)
+        for i in instances:
+            i.deploy._async(user=user)
+        if amount == 1:
+            serializer = InstanceSerializer(instances, many=True)
+            return JsonResponse(serializer.data, status=201, safe=False)
+        serializer = InstanceSerializer(instances, many=True)
+        return JsonResponse(serializer.data, status=201, safe=False)
 
 class InstanceFromTemplateREST(APIView):
     authentication_classes = [TokenAuthentication,BasicAuthentication]

circle/storage/migrations/0008_disk_cluster_size.py

+# Generated by Django 3.2.3 on 2023-09-18 14:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('storage', '0007_disk_cache_size'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='disk',
+            name='cluster_size',
+            field=models.IntegerField(default=64, help_text='Disk cluster (block) size (Kbyte)', verbose_name='cluster size'),
+        ),
+    ]

circle/storage/models.py

@@ -150,6 +150,9 @@ class Disk(TimeStampedModel):
     cache_size = IntegerField(default=1024,
                             help_text=_("Disk metadata cache max size (Kbyte)"),
                             verbose_name=_('cache size'))
+    cluster_size = IntegerField(default=64,
+                            help_text=_("Disk cluster (block) size (Kbyte)"),
+                            verbose_name=_('cluster size'))
 
     class Meta:
         ordering = ['name']
@@ -319,7 +322,7 @@ class Disk(TimeStampedModel):
         except ObjectDoesNotExist:
             return None
 
-    def get_exclusive(self):
+    def get_exclusive(self, datastore=None):
         """Get an instance of the disk for exclusive usage.
 
         This method manipulates the database only.
@@ -335,7 +338,7 @@ class Disk(TimeStampedModel):
 
         new_type = type_mapping[self.type]
 
-        return Disk.create(base=self, datastore=self.datastore,
+        return Disk.create(base=self, datastore=self.datastore if datastore is None else datastore,
                            name=self.name, size=self.size,
                            type=new_type, dev_num=self.dev_num)
 
@@ -350,7 +353,7 @@ class Disk(TimeStampedModel):
             'target_device': self.device_type + self.dev_num,
             'target_bus': self.device_bus,
             'disk_device': 'cdrom' if self.type == 'iso' else 'disk',
-            'cache_size': self.cache_size
+            'cache_size': self.cache_size,
         }
 
     def get_disk_desc(self):
@@ -363,7 +366,8 @@ class Disk(TimeStampedModel):
             'size': self.size,
             'base_name': self.base.filename if self.base else None,
             'type': 'snapshot' if self.base else 'normal',
-            'cache_size': self.cache_size
+            'base_dir':  self.base.datastore.path if self.base else None,
+            'cluster_size': self.cluster_size
         }
 
     def get_remote_queue_name(self, queue_id='storage', priority=None,

circle/vm/migrations/0018_instancetemplate_overlay_path.py

+# Generated by Django 3.2.3 on 2023-09-14 15:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vm', '0017_auto_20221231_1011'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instancetemplate',
+            name='overlay_path',
+            field=models.CharField(default='', help_text='Overlay images destination path. If empty, the location of the base image.', max_length=500, verbose_name='overlay_path'),
+        ),
+    ]

circle/vm/migrations/0019_alter_instancetemplate_overlay_path.py

+# Generated by Django 3.2.3 on 2023-09-18 08:53
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('storage', '0007_disk_cache_size'),
+        ('vm', '0018_instancetemplate_overlay_path'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instancetemplate',
+            name='overlay_path',
+            field=models.ForeignKey(blank=True, help_text='Overlay images destination path. If empty, the location of the base image.', null=True, on_delete=django.db.models.deletion.CASCADE, to='storage.datastore', verbose_name='overlay destination'),
+        ),
+    ]

circle/vm/models/instance.py

@@ -209,6 +209,8 @@ class InstanceTemplate(AclBase, VirtualMachineDescModel, TimeStampedModel):
                             related_name='template_set',
                             help_text=_('Disks which are to be mounted.'))
     owner = ForeignKey(User, on_delete=models.CASCADE)
+    overlay_path = ForeignKey('storage.DataStore', verbose_name=_("overlay destination"),  null=True, blank=True,
+                           help_text=_("Overlay images destination path. If empty, the location of the base image."), on_delete=models.CASCADE)
 
     class Meta:
         app_label = 'vm'
@@ -547,6 +549,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
                 raise PermissionDenied()
             
         # create instance and do additional setup
+        datastore = params.pop("overlay_path", None)
         inst = cls(**params)
         #if not params["num_cores_max"]:
         #inst.num_cores_max = inst.num_cores
@@ -562,7 +565,8 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
                            readable_name=ugettext_noop("create instance"),
                            on_commit=__on_commit, user=inst.owner) as act:
             # create related entities
-            inst.disks.add(*[disk.get_exclusive() for disk in disks])
+            inst.disks.add(*[disk.get_exclusive(datastore=None if datastore == "" 
+                                    else datastore) for disk in disks])
 
             for net in networks:
                 Interface.create(instance=inst, vlan=net.vlan,
@@ -610,7 +614,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
         common_fields = ['name', 'description', 'num_cores', 'num_cores_max','ram_size',
                          'max_ram_size', 'arch', 'priority', 'boot_menu',
                          'raw_data', 'lease', 'access_method', 'system',
-                         'cloud_init', 'ci_meta_data', 'ci_user_data', 'ci_network_config', 'has_agent']
+                         'cloud_init', 'ci_meta_data', 'ci_user_data', 'ci_network_config', 'has_agent', 'overlay_path']
         params = dict(template=template, owner=owner, pw=pwgen())
         params.update([(f, getattr(template, f)) for f in common_fields])
         params.update(kwargs)  # override defaults w/ user supplied values

circle/vm/operations.py

@@ -292,12 +292,12 @@ class CreateDiskOperation(InstanceOperation):
     accept_states = ('STOPPED', 'PENDING', 'RUNNING')
     concurrency_check = False
 
-    def _operation(self, user, size, activity, datastore, name=None):
+    def _operation(self, user, size, activity, datastore, name=None, cache_size=1024, cluster_size=64):
         from storage.models import Disk
 
         if not name:
             name = "new disk"
-        disk = Disk.create(size=size, name=name, datastore=datastore.name, type="qcow2-norm")
+        disk = Disk.create(size=size, name=name, datastore=datastore.name, type="qcow2-norm", cache_size=cache_size, cluster_size=cluster_size)
         disk.full_clean()
         devnums = list(ascii_lowercase)
         for d in self.instance.disks.all():