Add azure built in template models. Refresh azure built-ins every

24 hours.

circle/manager/mancelery.py

@@ -55,6 +55,12 @@ celery.conf.update(
             'schedule': timedelta(hours=24),
             'options': {'queue': 'localhost.man'}
         },
+        'vm.update_azure_builtin_templates': {
+            'task': 'vm.tasks.local_periodic_tasks.'
+            'update_azure_builtin_templates',
+            'schedule': timedelta(hours=24),
+            'options': {'queue': 'localhost.man'}
+        },
     }
 
 )

circle/vm/migrations/0003_auto_20161006_2159.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vm', '0002_interface_model'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AzureBuiltinOffer',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('azure_id', models.TextField(verbose_name='azure_id')),
+                ('name', models.TextField(verbose_name='name')),
+            ],
+            options={
+                'db_table': 'vm_azure_offer',
+            },
+        ),
+        migrations.CreateModel(
+            name='AzureBuiltinPublisher',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('azure_id', models.TextField(verbose_name='azure_id')),
+                ('name', models.TextField(verbose_name='name')),
+            ],
+            options={
+                'db_table': 'vm_azure_publisher',
+            },
+        ),
+        migrations.CreateModel(
+            name='AzureBuiltinSKU',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('azure_id', models.TextField(verbose_name='azure_id')),
+                ('name', models.TextField(verbose_name='name')),
+                ('offer', models.ForeignKey(related_name='azure_sku', verbose_name='azure_offer', to='vm.AzureBuiltinOffer')),
+            ],
+            options={
+                'db_table': 'vm_azure_sku',
+            },
+        ),
+        migrations.CreateModel(
+            name='AzureBuiltinVersion',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('azure_id', models.TextField(verbose_name='azure_id')),
+                ('name', models.TextField(verbose_name='name')),
+                ('offer', models.ForeignKey(related_name='azure_version', verbose_name='azure_sku', to='vm.AzureBuiltinSKU')),
+            ],
+            options={
+                'db_table': 'vm_azure_version',
+            },
+        ),
+        migrations.AddField(
+            model_name='azurebuiltinoffer',
+            name='publisher',
+            field=models.ForeignKey(related_name='azure_offer', verbose_name='azure_publisher', to='vm.AzureBuiltinPublisher'),
+        ),
+    ]

circle/vm/migrations/0004_auto_20161006_2201.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vm', '0003_auto_20161006_2159'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='azurebuiltinversion',
+            old_name='offer',
+            new_name='sku',
+        ),
+    ]

circle/vm/models/__init__.py

@@ -15,11 +15,16 @@ from .instance import pwgen
 from .network import InterfaceTemplate
 from .network import Interface
 from .node import Node
+from .azure_builtin_template import AzureBuiltinPublisher
+from .azure_builtin_template import AzureBuiltinOffer
+from .azure_builtin_template import AzureBuiltinSKU
+from .azure_builtin_template import AzureBuiltinVersion
 
 __all__ = [
     'InstanceActivity', 'BaseResourceConfigModel',
     'NamedBaseResourceConfig', 'VirtualMachineDescModel', 'InstanceTemplate',
     'Instance', 'post_state_changed', 'pre_state_changed', 'InterfaceTemplate',
     'Interface', 'Trait', 'Node', 'NodeActivity', 'Lease', 'node_activity',
-    'pwgen'
+    'pwgen', 'AzureBuiltinPublisher', 'AzureBuiltinOffer', 'AzureBuiltinSKU',
+    'AzureBuiltinVersion',
 ]

circle/vm/models/azure_builtin_template.py

+# Copyright 2014 Budapest University of Technology and Economics (BME IK)
+#
+# This file is part of CIRCLE Cloud.
+#
+# CIRCLE is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along
+# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import absolute_import, unicode_literals
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+class AzureBuiltinPublisher(models.Model):
+    """
+    Azure Built-in publishers (for templates)
+    """
+    azure_id = models.TextField(verbose_name=_('azure_id'))
+    name = models.TextField(verbose_name=_('name'))
+
+    class Meta:
+        app_label = 'vm'
+        db_table = 'vm_azure_publisher'
+
+    def __unicode__(self):
+        return 'azure_publisher-' + str(self.name)
+
+class AzureBuiltinOffer(models.Model):
+    """
+    Azure Built-in offers (for templates)
+    """
+    azure_id = models.TextField(verbose_name=_('azure_id'))
+    name = models.TextField(verbose_name=_('name'))
+    publisher = models.ForeignKey(
+        AzureBuiltinPublisher,
+        verbose_name=_('azure_publisher'),
+        related_name="azure_offer",
+        on_delete=models.CASCADE,
+    )
+
+    class Meta:
+        app_label = 'vm'
+        db_table = 'vm_azure_offer'
+
+    def __unicode__(self):
+        return 'azure_offer-' + str(self.name)
+
+class AzureBuiltinSKU(models.Model):
+    """
+    Azure Built-in SKUs (for templates)
+    """
+    azure_id = models.TextField(verbose_name=_('azure_id'))
+    name = models.TextField(verbose_name=_('name'))
+    offer = models.ForeignKey(
+        AzureBuiltinOffer,
+        verbose_name=_('azure_offer'),
+        related_name="azure_sku",
+        on_delete=models.CASCADE,
+    )
+
+    class Meta:
+        app_label = 'vm'
+        db_table = 'vm_azure_sku'
+
+    def __unicode__(self):
+        return 'azure_sku-' + str(self.name)
+
+class AzureBuiltinVersion(models.Model):
+    """
+    Azure Built-in versions (for templates)
+    """
+    azure_id = models.TextField(verbose_name=_('azure_id'))
+    name = models.TextField(verbose_name=_('name'))
+    sku = models.ForeignKey(
+        AzureBuiltinSKU,
+        verbose_name=_('azure_sku'),
+        related_name="azure_version",
+        on_delete=models.CASCADE,
+    )
+
+    class Meta:
+        app_label = 'vm'
+        db_table = 'vm_azure_version'
+
+    def __unicode__(self):
+        return 'azure_version-' + str(self.name)
\ No newline at end of file

circle/vm/tasks/local_periodic_tasks.py

@@ -16,14 +16,38 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
+from os import getenv
+
 from django.utils import timezone
 from django.utils.translation import ugettext_noop
 
 from manager.mancelery import celery
-from vm.models import Node, Instance
+from vm.models import (
+    Node, Instance, AzureBuiltinPublisher,
+    AzureBuiltinOffer, AzureBuiltinSKU, AzureBuiltinVersion,
+)
+from azure.common.credentials import ServicePrincipalCredentials
+from azure.mgmt.compute import ComputeManagementClient
 
 logger = logging.getLogger(__name__)
 
+SUBSCRIPTION_ID = getenv('SUBSCRIPTION_ID')
+CLIENT_ID = getenv('CLIENT_ID')
+SECRET = getenv('SECRET')
+TENANT = getenv('TENANT')
+GROUP_NAME = getenv('GROUP_NAME')
+REGION = getenv('REGION')
+
+credentials = ServicePrincipalCredentials(
+    client_id = CLIENT_ID,
+    secret = SECRET,
+    tenant = TENANT,
+)
+
+compute_client = ComputeManagementClient(
+    credentials,
+    SUBSCRIPTION_ID
+)
 
 @celery.task(ignore_result=True)
 def update_domain_states():
@@ -76,3 +100,71 @@ def garbage_collector(timeout=15):
             i.notify_owners_about_expiration()
         else:
             logger.debug("Instance %d didn't expire." % i.pk)
+
+@celery.task(ignore_result=True)
+def update_azure_builtin_templates():
+    logger.info("beginning azure built-in template imports")
+
+    current_publisher_ids = []
+    current_offer_ids = []
+    current_sku_ids = []
+    current_version_ids = []
+
+    # get currently available publishers, offers, skus, and versions
+    result_list_pub = compute_client.virtual_machine_images.list_publishers(
+        REGION,
+    )
+    for publisher in result_list_pub:
+        current_publisher = AzureBuiltinPublisher(
+            azure_id=publisher.id, name=publisher.name,
+        )
+        current_publisher.save()
+        current_publisher_ids.append(current_publisher.azure_id)
+
+        result_list_offers = compute_client.virtual_machine_images.list_offers(
+            REGION,
+            publisher.name,
+        )
+        for offer in result_list_offers:
+            current_offer = AzureBuiltinOffer(
+                azure_id=offer.id, name=offer.name, publisher=current_publisher
+            )
+            current_offer.save()
+            current_offer_ids.append(current_offer.azure_id)
+        
+            result_list_skus = compute_client.virtual_machine_images.list_skus(
+                REGION,
+                publisher.name,
+                offer.name,
+            )
+            for sku in result_list_skus:
+                current_sku = AzureBuiltinSKU(
+                    azure_id=sku.id, name=sku.name, offer=current_offer
+                )
+                current_sku.save()
+                current_sku_ids.append(current_sku.azure_id)
+        
+                result_list = compute_client.virtual_machine_images.list(
+                    REGION,
+                    publisher.name,
+                    offer.name,
+                    sku.name,
+                )
+                for version in result_list:
+                    current_version = AzureBuiltinVersion(
+                        azure_id=version.id, name=version.name, sku=current_sku
+                    )
+                    current_version.save()
+                    current_version_ids.append(current_version.azure_id)
+    
+    # delete old publishers, offers, skus, and versions
+    AzureBuiltinPublisher.objects.filter(
+        azure_id__in=current_publisher_ids).delete()
+    AzureBuiltinOffer.objects.filter(
+        azure_id__in=current_offer_ids).delete()
+    AzureBuiltinSKU.objects.filter(
+        azure_id__in=current_sku_ids).delete()
+    AzureBuiltinVersion.objects.filter(
+        azure_id__in=current_version_ids).delete()
+    
+    logger.info("refreshed azure built-in template imports")
\ No newline at end of file