add upstart script, reload firewall on start-up, random fixes

07624251 · Bach Dániel · 6e4c550d · 07624251 · 6e4c550d · 07624251
Commit 07624251 authored Oct 17, 2013 by Bach Dániel
Hide whitespace changes
Inline Side-by-side

Showing with 136 additions and 43 deletions

.gitignore
+3 -0

celeryconfig.py
+0 -4

fw.py
+50 -12

miscellaneous/firewall.conf
+16 -0

ovs.py
+67 -27

No files found.
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,6 @@ _build
 # Logs:
 *.log
+# config
+*.conf
--- a/celeryconfig.py
+++ b/celeryconfig.py
-from os import getenv
-CELERY_TASK_RESULT_EXPIRES = 3600
-BROKER_URL = getenv("AMQP_URI")
--- a/fw.py
+++ b/fw.py
 from celery import Celery, task
+from os import getenv
 import subprocess
 import re
+import json
 import socket
 from ovs import Switch
-IRC_CHANNEL = '/home/cloud/irc/irc.atw.hu/#ik/in'
+IRC_CHANNEL = getenv('IRC_CHANNEL', '/home/cloud/irc/irc.atw.hu/#ik/in')
-DHCP_LOGFILE = '/home/cloud/dhcp.log'
+DHCP_LOGFILE = getenv('DHCP_LOGFILE', '/home/cloud/dhcp.log')
+VLAN_CONF = getenv('VLAN_CONF', 'vlan.conf')
+FIREWALL_CONF = getenv('FIREWALL_CONF', 'firewall.conf')
-CELERY_CREATE_MISSING_QUEUES = True
-celery = Celery('tasks', backend='amqp')
+celery = Celery('tasks', backend='amqp', )
-celery.config_from_object('celeryconfig')
+celery.conf.update(CELERY_TASK_RESULT_EXPIRES=3600,
+                   BROKER_URL=getenv("AMQP_URI"),
+                   CELERY_CREATE_MISSING_QUEUES=True)
 @task(name="firewall.reload_firewall")
-def reload_firewall(data4, data6):
+def reload_firewall(data4, data6, onstart=False):
    print "fw"
    process = subprocess.Popen(['/usr/bin/sudo',
                                '/sbin/ip6tables-restore', '-c'],
                               shell=False, stdin=subprocess.PIPE)
@@ -26,21 +31,26 @@ def reload_firewall(data4, data6):
                               shell=False, stdin=subprocess.PIPE)
    process.communicate("\n".join(data4['filter'])
                        + "\n" + "\n".join(data4['nat']) + "\n")
+    if onstart is False:
+        with open(FIREWALL_CONF, 'w') as f:
+            json.dump([data4, data6], f)
 @task(name="firewall.reload_firewall_vlan")
-def reload_firewall_vlan(data):
+def reload_firewall_vlan(data, onstart=False):
    print "fw vlan"
-    print data
+#    print data
-    br = Switch('cloud')
+    br = Switch('firewall')
    br.migrate(data)
-    print br.list_ports()
+#    print br.list_ports()
+    if onstart is False:
+        with open(VLAN_CONF, 'w') as f:
+            json.dump(data, f)
 @task(name="firewall.reload_dhcp")
 def reload_dhcp(data):
    print "dhcp"
    with open('/tools/dhcp3/dhcpd.conf.generated', 'w') as f:
        f.write("\n".join(data) + "\n")
    subprocess.call(['sudo', '/etc/init.d/isc-dhcp-server',
@@ -145,3 +155,31 @@ def get_dhcp_clients():
            clients[mac] = (ip, hostname, interface)
    return clients
+def start_firewall():
+    try:
+        subprocess.call('sudo ipset create blacklist hash:ip family '
+                        'inet hashsize 4096 maxelem 65536 2>/dev/null',
+                        shell=True)
+        with open(FIREWALL_CONF, 'r') as f:
+            data4, data6 = json.load(f)
+            reload_firewall(data4, data6, True)
+    except:
+        print 'nemsikerult:('
+def start_networking():
+    try:
+        with open(VLAN_CONF, 'r') as f:
+            data = json.load(f)
+            reload_firewall_vlan(data, True)
+    except:
+        print 'nemsikerult:('
+def main():
+    start_networking()
+    start_firewall()
+main()
--- a/miscellaneous/firewall.conf
+++ b/miscellaneous/firewall.conf
+description     "IK Cloud Django Development Server"
+start on runlevel [2345]
+stop on runlevel [!2345]
+respawn
+respawn limit 30 30
+setuid cloud
+chdir /home/cloud/fwdriver
+script
+    . /home/cloud/.virtualenvs/fwdriver/local/bin/postactivate
+    exec /home/cloud/.virtualenvs/fwdriver/bin/celeryd -A fw -Q firewall --loglevel=info --logfile=/tmp/fwcelery.log
+end script
--- a/ovs.py
+++ b/ovs.py
 import subprocess
 from netaddr import IPNetwork
+import logging
-# data = subprocess.check_output('sudo ovs-vsctl --format=json --data=json '
-#                                '--no-headings find Interface', shell=True)
-# obj = json.loads(data)
-# print json.dumps(obj['data'][0], indent=4)
 class IPDevice:
@@ -15,7 +9,7 @@ class IPDevice:
    def _run(self, *args):
        args = ('sudo', 'ip', 'addr', ) + args
-#        print args
+        logging.debug('subprocess_check_output: {}'.format(args))
        return subprocess.check_output(args)
    def show(self):
@@ -25,6 +19,7 @@ class IPDevice:
            t = line.split()
            if len(t) > 0 and t[0] in ('inet', 'inet6'):
                retval.append(IPNetwork(t[1]))
+        logging.debug('[ip-%s] show: %s' % (self.devname, str(retval)))
        return retval
    def delete(self, address):
@@ -39,7 +34,8 @@ class IPDevice:
        delete = list(set(old_addresses) - set(new_addresses))
        add = list(set(new_addresses) - set(old_addresses))
-        print delete, add
+        logging.debug('[ip-%s] delete: %s' % (self.devname, str(delete)))
+        logging.debug('[ip-%s] add: %s' % (self.devname, str(add)))
        for i in delete:
            self.delete(i)
@@ -51,6 +47,10 @@ class IPDevice:
 class Switch:
    def __init__(self, brname):
        self.brname = brname
+        try:
+            self._run('add-br', brname)
+        except:
+            pass
    def _run(self, *args):
        args = ('sudo', 'ovs-vsctl', ) + args
@@ -58,26 +58,52 @@ class Switch:
    def list_ports(self):
        retval = {}
-        c_bridge = None
+        bridge = None
-        c_port = None
+        port = None
        for line in self._run('show').splitlines():
            t = line.split()
            if t[0] == 'Bridge':
-                c_bridge = t[1]
+                bridge = t[1]
-                retval[c_bridge] = {}
+                retval[bridge] = {}
            elif t[0] == 'Port':
-                c_port = t[1]
+                port = t[1].replace('"', '')  # valahol idezojel van
-                retval[c_bridge][c_port] = {}
+                retval[bridge][port] = {}
+                retval[bridge][port]['interfaces'] = []
+            elif t[0] == 'Interface':
+                interface = t[1].replace('"', '')  # valahol idezojel van
+                retval[bridge][port]['interfaces'].append(interface)
            elif t[0] == 'tag:':
-                retval[c_bridge][c_port]['tag'] = int(t[1])
+                tag = int(t[1])
+                retval[bridge][port]['tag'] = tag
            elif t[0] == 'type:':
-                retval[c_bridge][c_port]['type'] = t[1]
+                retval[bridge][port]['type'] = t[1]
+            elif t[0] == 'trunks:':
+                trunks = [int(p.strip('[,]')) for p in t[1:]]
+                retval[bridge][port]['trunks'] = trunks
        return retval.get(self.brname, {})
-    def add_port(self, name, tag):
+    def add_port(self, name, interfaces, tag, trunks, internal=True):
-        self._run('add-port', self.brname, name, 'tag=%d' % int(tag), '--',
+        if len(interfaces) > 1:
-                  'set', 'Interface', name, 'type=internal')
+            # bond
-        subprocess.check_output(['sudo', 'ip', 'link', 'set', 'up', name])
+            params = ['add-bond', self.brname,
+                      name] + interfaces + ['tag=%d' % int(tag)]
+        else:
+            params = ['add-port', self.brname, name, 'tag=%d' % int(tag)]
+        if internal:
+            params = params + ['--',  'set', 'Interface', interfaces[0],
+                               'type=internal']
+        if trunks is not None and len(trunks) > 0:
+            params.append('trunks=%s' % trunks)
+        self._run(*params)
+        self.ip_link_up(interfaces)
+    def ip_link_up(self, interfaces):
+        for interface in interfaces:
+            try:
+                subprocess.check_output(['sudo', 'ip', 'link',
+                                         'set', 'up', interface])
+            except:
+                pass
    def delete_port(self, name):
        self._run('del-port', self.brname, name)
@@ -89,9 +115,15 @@ class Switch:
        for port, data in new_ports.items():
            if port not in old_ports:
+                # new port
                add.append(port)
            elif (old_ports[port].get('tag', None) !=
-                    new_ports[port].get('tag', None)):
+                    new_ports[port].get('tag', None) or
+                    old_ports[port].get('trunks', None) !=
+                    new_ports[port].get('trunks', None) or
+                    old_ports[port].get('interfaces', None) !=
+                    new_ports[port].get('interfaces', None)):
+                # modified port
                delete.append(port)
                add.append(port)
@@ -99,15 +131,23 @@ class Switch:
                               set(new_ports.keys()))
        delete.remove(self.brname)
-        print delete, add
+        logging.debug('[ovs delete: %s' % (delete, ))
+        logging.debug('[ovs] add: %s' % (add, ))
        for i in delete:
            self.delete_port(i)
        for i in add:
-            self.add_port(i, new_ports[i]['tag'])
+            internal = new_ports[i].get('type', '') == 'internal'
+            tag = new_ports[i]['tag']
+            trunks = new_ports[i].get('trunks', [])
+            interfaces = new_ports[i]['interfaces']
+            self.add_port(i, interfaces, tag, trunks, internal)
        for port, data in new_ports.items():
            interface = IPDevice(devname=port)
-            interface.migrate([IPNetwork(x)
+            try:
-                               for x in data['addresses']
+                interface.migrate([IPNetwork(x)
-                               if x != 'None'])
+                                   for x in data.get('addresses', [])
+                                  if x != 'None'])
+            except:
+                pass