Implement diagnostics collection from Azure VMs

.gitignore

 *.pyc
 *.swp
+.ropeproject/*

src/__init__.py

-__all__ = ["client", "collectables", "metrics", "cnfparse"]
+__all__ = ["azureperformancecounters","client", "collectables", "metrics", "cnfparse"]

src/azureperformancecounters.py

+import logging
+import requests
+import azure.mgmt.storage
+import re
+from azure.storage.table import TableService, Entity
+
+logger = logging.getLogger(__name__)
+
+
+class AzureVmPerformanceCounters(object):
+
+    """
+    Get performance counters for Azure VMs
+    For Linux:
+        - Processor usage
+        - Memory usage
+        - Bytes recieved, sent
+        - Packets recieved, sent
+    For Windows:
+        - Processor usage
+        - Memory usage
+    """
+
+    # performance counters for linux
+    __linux_performance_counters = {
+        "\\Processor\\PercentProcessorTime": "processor_usage",
+        "\\Memory\\PercentUsedMemory": "memory_usage",
+        "\\NetworkInterface\\BytesReceived": "bytes_recv",
+        "\\NetworkInterface\\BytesTransmitted": "bytes_sent",
+        "\\NetworkInterface\\PacketsReceived": "packets_recv",
+        "\\NetworkInterface\\PacketsTransmitted": "packets_sent",
+    }
+
+    # performance counters for windows
+    __windows_performance_counters = {
+        "\\Processor(_Total)\\% Processor Time": "processor_usage",
+        "\\Memory\\% Committed Bytes In Use": "memory_usage",
+    }
+
+    def __get_storage_table_infos(self):
+        """
+        Get information about the storage table of the VM through
+                Insights API
+                Relevant information is: storage account name, storage table names
+        """
+        # filter string for performance counters
+        filter_string = None
+        for pc in self.__performance_counters.keys():
+            if filter_string is None:
+                filter_string = "&$filter=name.value eq '%s'" % (pc)
+            else:
+                filter_string += " or name.value eq '%s'" % (pc)
+
+        # rest api url to query
+        list_metrics_url = "https://management.azure.com/subscriptions/" \
+            "%s/resourceGroups/%s/providers/" \
+            "Microsoft.Compute/virtualMachines/%s/metricDefinitions" \
+            "?api-version=2014-04-01%s" % (
+                self.subscription_id, self.resource_group,
+                self.vm.name, filter_string,
+            )
+        headers = {"Authorization": 'Bearer ' + self.accessToken}
+        json_output = requests.get(list_metrics_url, headers=headers).json()
+
+        if json_output.has_key("value"):
+            return json_output["value"]
+        else:
+            return None
+
+    def __get_access_key_for_storage_account(self):
+        """
+        Get acccess key for the storage account related to our VM
+        """
+        storage_client = azure.mgmt.storage.StorageManagementClient(
+            self.credentials,
+            self.subscription_id,
+        )
+
+        keys = storage_client.storage_accounts.list_keys(
+            self.resource_group, self.account_name
+        )
+
+        return keys.keys[0].value
+
+    def __get_performance_counter_values(self):
+        """
+        Get all the performance counters we are interested in,
+                in one OData request
+        """
+        table_service = TableService(
+            account_name=self.account_name,
+            account_key=self.account_key
+        )
+
+        filter_string = "PartitionKey eq '%s' and ( " % (
+            self.partition_key
+        )
+        first_pc_id = True
+        for pc_id in self.counter_ids:
+            if first_pc_id:
+                first_pc_id = False
+            else:
+                filter_string += " or "
+            filter_string += "CounterName eq '%s'" % (pc_id)
+        filter_string += " )"
+
+        pcs = table_service.query_entities(
+            self.table_name,
+            filter=filter_string,
+        )
+
+        most_recent_pcs = []
+        most_recent_pcs_names = []
+        for pc in pcs:
+            if pc["CounterName"] in most_recent_pcs_names:
+                break
+
+            most_recent_pcs_names.append(pc["CounterName"])
+            most_recent_pcs.append(pc)
+
+        return most_recent_pcs
+
+    def __init__(self, credentials, accessToken, subscription_id,
+                 resource_group, vm):
+        """
+        Get relevant performance for Azure Virtual Machine
+        Relevant performance counters are:
+                - Processor usage,
+                - Memory usage,
+                - Bytes, and packets recieved and transmitted over the network
+                  interfaces
+        Parameters:
+                :credentials: <azure.common.credentials> object for SDK calls
+                :accessToken: will be passed to 'Authorization'
+                  in REST API calls
+                :subscription_id: subscription id
+                :resource_group: vm's resource group's name
+                :vm: <azure.mgmt.compute.models.VirtualMachine> object
+        """
+        self.credentials = credentials
+        self.accessToken = accessToken
+        self.subscription_id = subscription_id
+        self.resource_group = resource_group
+        self.vm = vm
+        self.account_key = None
+        self.counter_ids = []
+
+        self.performance_counters = {}
+
+        # set performance counter dictionary based on OS
+        if vm.os_profile.linux_configuration is not None:
+            self.__performance_counters = \
+                AzureVmPerformanceCounters.__linux_performance_counters
+            self.__aggregation_type = "Last"
+        else:
+            self.__performance_counters = \
+                AzureVmPerformanceCounters.__windows_performance_counters
+            self.__aggregation_type = "Average"
+
+        # get storage table infos
+        logger.info("get storage table infos vm '%s'" % (vm.name))
+        storage_table_infos = self.__get_storage_table_infos()
+
+        # summarized performance counters by account_name
+        if storage_table_infos is not None:
+            for table_info in storage_table_infos:
+                table_minute_grain = (
+                    t_grained for t_grained
+                    in table_info["metricAvailabilities"]
+                    if t_grained["timeGrain"] == "PT1M"
+                ).next()
+
+                table_endpoint = \
+                    table_minute_grain["location"]["tableEndpoint"]
+
+                # get account name for table endpoint string
+                self.account_name = re.match(
+                    r'https?://(.*).table.core.windows.net/',
+                    table_endpoint,
+                ).groups(0)[0]
+
+                # get account key if we hadn't got it before
+                if self.account_key is None:
+                    logger.info("get access key for account '%s'" % (
+                        self.account_name
+                    ))
+                    self.account_key = \
+                        self.__get_access_key_for_storage_account()
+
+                # get most recent table
+                latest_table = None
+                for t in table_minute_grain["location"]["tableInfo"]:
+                    if latest_table == None \
+                            or latest_table["startTime"] < t["startTime"]:
+                        latest_table = t
+
+                        # get table name
+                self.table_name = latest_table["tableName"]
+
+                # get partition key
+                self.partition_key = \
+                    table_minute_grain["location"]["partitionKey"]
+
+                # get counter id
+                counter_id = \
+                    table_info["name"]["value"].replace("\\\\", "\\")
+                self.counter_ids.append(counter_id)
+
+            logger.info("get value of performance counters")
+            most_recent_pcs = self.__get_performance_counter_values()
+
+            # store counters
+            for pc in most_recent_pcs:
+                counter_name = \
+                    self.__performance_counters[pc["CounterName"]]
+                self.performance_counters[counter_name] = {
+                    "value": pc[self.__aggregation_type],
+                    "timestamp": pc["Timestamp"],
+                }