add upstart script, reload firewall on start-up, random fixes

.gitignore

@@ -17,3 +17,6 @@ _build
 
 # Logs:
 *.log
+
+# config
+*.conf

celeryconfig.py

-from os import getenv
-
-CELERY_TASK_RESULT_EXPIRES = 3600
-BROKER_URL = getenv("AMQP_URI")

fw.py

 from celery import Celery, task
+from os import getenv
 import subprocess
 import re
+import json
 import socket
 from ovs import Switch
 
-IRC_CHANNEL = '/home/cloud/irc/irc.atw.hu/#ik/in'
-DHCP_LOGFILE = '/home/cloud/dhcp.log'
+IRC_CHANNEL = getenv('IRC_CHANNEL', '/home/cloud/irc/irc.atw.hu/#ik/in')
+DHCP_LOGFILE = getenv('DHCP_LOGFILE', '/home/cloud/dhcp.log')
+VLAN_CONF = getenv('VLAN_CONF', 'vlan.conf')
+FIREWALL_CONF = getenv('FIREWALL_CONF', 'firewall.conf')
 
-CELERY_CREATE_MISSING_QUEUES = True
-celery = Celery('tasks', backend='amqp')
-celery.config_from_object('celeryconfig')
+
+celery = Celery('tasks', backend='amqp', )
+celery.conf.update(CELERY_TASK_RESULT_EXPIRES=3600,
+                   BROKER_URL=getenv("AMQP_URI"),
+                   CELERY_CREATE_MISSING_QUEUES=True)
 
 
 @task(name="firewall.reload_firewall")
-def reload_firewall(data4, data6):
+def reload_firewall(data4, data6, onstart=False):
     print "fw"
-
     process = subprocess.Popen(['/usr/bin/sudo',
                                 '/sbin/ip6tables-restore', '-c'],
                                shell=False, stdin=subprocess.PIPE)
@@ -26,21 +31,26 @@ def reload_firewall(data4, data6):
                                shell=False, stdin=subprocess.PIPE)
     process.communicate("\n".join(data4['filter'])
                         + "\n" + "\n".join(data4['nat']) + "\n")
+    if onstart is False:
+        with open(FIREWALL_CONF, 'w') as f:
+            json.dump([data4, data6], f)
 
 
 @task(name="firewall.reload_firewall_vlan")
-def reload_firewall_vlan(data):
+def reload_firewall_vlan(data, onstart=False):
     print "fw vlan"
-    print data
-    br = Switch('cloud')
+#    print data
+    br = Switch('firewall')
     br.migrate(data)
-    print br.list_ports()
+#    print br.list_ports()
+    if onstart is False:
+        with open(VLAN_CONF, 'w') as f:
+            json.dump(data, f)
 
 
 @task(name="firewall.reload_dhcp")
 def reload_dhcp(data):
     print "dhcp"
-
     with open('/tools/dhcp3/dhcpd.conf.generated', 'w') as f:
         f.write("\n".join(data) + "\n")
     subprocess.call(['sudo', '/etc/init.d/isc-dhcp-server',
@@ -145,3 +155,31 @@ def get_dhcp_clients():
             clients[mac] = (ip, hostname, interface)
 
     return clients
+
+
+def start_firewall():
+    try:
+        subprocess.call('sudo ipset create blacklist hash:ip family '
+                        'inet hashsize 4096 maxelem 65536 2>/dev/null',
+                        shell=True)
+        with open(FIREWALL_CONF, 'r') as f:
+            data4, data6 = json.load(f)
+            reload_firewall(data4, data6, True)
+    except:
+        print 'nemsikerult:('
+
+
+def start_networking():
+    try:
+        with open(VLAN_CONF, 'r') as f:
+            data = json.load(f)
+            reload_firewall_vlan(data, True)
+    except:
+        print 'nemsikerult:('
+
+
+def main():
+    start_networking()
+    start_firewall()
+
+main()

miscellaneous/firewall.conf

+description     "IK Cloud Django Development Server"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+respawn limit 30 30
+setuid cloud
+
+chdir /home/cloud/fwdriver
+
+script
+    . /home/cloud/.virtualenvs/fwdriver/local/bin/postactivate
+    exec /home/cloud/.virtualenvs/fwdriver/bin/celeryd -A fw -Q firewall --loglevel=info --logfile=/tmp/fwcelery.log
+end script
+

ovs.py

 import subprocess
 from netaddr import IPNetwork
-
-# data = subprocess.check_output('sudo ovs-vsctl --format=json --data=json '
-#                                '--no-headings find Interface', shell=True)
-
-# obj = json.loads(data)
-
-# print json.dumps(obj['data'][0], indent=4)
+import logging
 
 
 class IPDevice:
@@ -15,7 +9,7 @@ class IPDevice:
 
     def _run(self, *args):
         args = ('sudo', 'ip', 'addr', ) + args
-#        print args
+        logging.debug('subprocess_check_output: {}'.format(args))
         return subprocess.check_output(args)
 
     def show(self):
@@ -25,6 +19,7 @@ class IPDevice:
             t = line.split()
             if len(t) > 0 and t[0] in ('inet', 'inet6'):
                 retval.append(IPNetwork(t[1]))
+        logging.debug('[ip-%s] show: %s' % (self.devname, str(retval)))
         return retval
 
     def delete(self, address):
@@ -39,7 +34,8 @@ class IPDevice:
         delete = list(set(old_addresses) - set(new_addresses))
         add = list(set(new_addresses) - set(old_addresses))
 
-        print delete, add
+        logging.debug('[ip-%s] delete: %s' % (self.devname, str(delete)))
+        logging.debug('[ip-%s] add: %s' % (self.devname, str(add)))
 
         for i in delete:
             self.delete(i)
@@ -51,6 +47,10 @@ class IPDevice:
 class Switch:
     def __init__(self, brname):
         self.brname = brname
+        try:
+            self._run('add-br', brname)
+        except:
+            pass
 
     def _run(self, *args):
         args = ('sudo', 'ovs-vsctl', ) + args
@@ -58,26 +58,52 @@ class Switch:
 
     def list_ports(self):
         retval = {}
-        c_bridge = None
-        c_port = None
+        bridge = None
+        port = None
         for line in self._run('show').splitlines():
             t = line.split()
             if t[0] == 'Bridge':
-                c_bridge = t[1]
-                retval[c_bridge] = {}
+                bridge = t[1]
+                retval[bridge] = {}
             elif t[0] == 'Port':
-                c_port = t[1]
-                retval[c_bridge][c_port] = {}
+                port = t[1].replace('"', '')  # valahol idezojel van
+                retval[bridge][port] = {}
+                retval[bridge][port]['interfaces'] = []
+            elif t[0] == 'Interface':
+                interface = t[1].replace('"', '')  # valahol idezojel van
+                retval[bridge][port]['interfaces'].append(interface)
             elif t[0] == 'tag:':
-                retval[c_bridge][c_port]['tag'] = int(t[1])
+                tag = int(t[1])
+                retval[bridge][port]['tag'] = tag
             elif t[0] == 'type:':
-                retval[c_bridge][c_port]['type'] = t[1]
+                retval[bridge][port]['type'] = t[1]
+            elif t[0] == 'trunks:':
+                trunks = [int(p.strip('[,]')) for p in t[1:]]
+                retval[bridge][port]['trunks'] = trunks
         return retval.get(self.brname, {})
 
-    def add_port(self, name, tag):
-        self._run('add-port', self.brname, name, 'tag=%d' % int(tag), '--',
-                  'set', 'Interface', name, 'type=internal')
-        subprocess.check_output(['sudo', 'ip', 'link', 'set', 'up', name])
+    def add_port(self, name, interfaces, tag, trunks, internal=True):
+        if len(interfaces) > 1:
+            # bond
+            params = ['add-bond', self.brname,
+                      name] + interfaces + ['tag=%d' % int(tag)]
+        else:
+            params = ['add-port', self.brname, name, 'tag=%d' % int(tag)]
+        if internal:
+            params = params + ['--',  'set', 'Interface', interfaces[0],
+                               'type=internal']
+        if trunks is not None and len(trunks) > 0:
+            params.append('trunks=%s' % trunks)
+        self._run(*params)
+        self.ip_link_up(interfaces)
+
+    def ip_link_up(self, interfaces):
+        for interface in interfaces:
+            try:
+                subprocess.check_output(['sudo', 'ip', 'link',
+                                         'set', 'up', interface])
+            except:
+                pass
 
     def delete_port(self, name):
         self._run('del-port', self.brname, name)
@@ -89,9 +115,15 @@ class Switch:
 
         for port, data in new_ports.items():
             if port not in old_ports:
+                # new port
                 add.append(port)
             elif (old_ports[port].get('tag', None) !=
-                    new_ports[port].get('tag', None)):
+                    new_ports[port].get('tag', None) or
+                    old_ports[port].get('trunks', None) !=
+                    new_ports[port].get('trunks', None) or
+                    old_ports[port].get('interfaces', None) !=
+                    new_ports[port].get('interfaces', None)):
+                # modified port
                 delete.append(port)
                 add.append(port)
 
@@ -99,15 +131,23 @@ class Switch:
                                set(new_ports.keys()))
         delete.remove(self.brname)
 
-        print delete, add
+        logging.debug('[ovs delete: %s' % (delete, ))
+        logging.debug('[ovs] add: %s' % (add, ))
 
         for i in delete:
             self.delete_port(i)
         for i in add:
-            self.add_port(i, new_ports[i]['tag'])
+            internal = new_ports[i].get('type', '') == 'internal'
+            tag = new_ports[i]['tag']
+            trunks = new_ports[i].get('trunks', [])
+            interfaces = new_ports[i]['interfaces']
+            self.add_port(i, interfaces, tag, trunks, internal)
 
         for port, data in new_ports.items():
             interface = IPDevice(devname=port)
-            interface.migrate([IPNetwork(x)
-                               for x in data['addresses']
-                               if x != 'None'])
+            try:
+                interface.migrate([IPNetwork(x)
+                                   for x in data.get('addresses', [])
+                                  if x != 'None'])
+            except:
+                pass