Merge branch 'feature-improve-firewall' into 'master'

Feature: Improve Firewall

✅ tests

circle/dashboard/fixtures/node.json

@@ -33,7 +33,6 @@
     "hostname": "devenv", 
     "modified_at": "2014-02-24T15:55:01.412Z", 
     "location": "", 
-    "pub_ipv4": null, 
     "mac": "11:22:33:44:55:66", 
     "shared_ip": false, 
     "ipv4": "10.7.0.96", 

circle/firewall/admin.py

@@ -16,7 +16,7 @@ class RecordInline(contrib.admin.TabularInline):
 
 
 class HostAdmin(admin.ModelAdmin):
-    list_display = ('hostname', 'vlan', 'ipv4', 'ipv6', 'pub_ipv4', 'mac',
+    list_display = ('hostname', 'vlan', 'ipv4', 'ipv6', 'external_ipv4', 'mac',
                     'shared_ip', 'owner', 'description', 'reverse',
                     'list_groups')
     ordering = ('hostname', )
@@ -25,6 +25,10 @@ class HostAdmin(admin.ModelAdmin):
     filter_horizontal = ('groups', )
     inlines = (RuleInline, RecordInline)
 
+    def queryset(self, request):
+        qs = super(HostAdmin, self).queryset(request)
+        return qs.prefetch_related('groups')
+
     @staticmethod
     def list_groups(instance):
         """Returns instance's groups' names as a comma-separated list."""
@@ -48,9 +52,9 @@ class VlanAdmin(admin.ModelAdmin):
 
 class RuleAdmin(admin.ModelAdmin):
     list_display = ('r_type', 'color_desc', 'owner', 'extra', 'direction',
-                    'accept', 'proto', 'sport', 'dport', 'nat',
-                    'nat_dport', 'used_in')
-    list_filter = ('vlan', 'owner', 'direction', 'accept',
+                    'action', 'proto', 'sport', 'dport', 'nat',
+                    'nat_external_port', 'used_in')
+    list_filter = ('vlan', 'owner', 'direction', 'action',
                    'proto', 'nat')
 
     def color_desc(self, instance):

circle/firewall/iptables.py

+import logging
+import re
+from collections import OrderedDict
+
+logger = logging.getLogger()
+
+ipv4_re = re.compile(
+    r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}')
+
+
+class InvalidRuleExcepion(Exception):
+    pass
+
+
+class IptRule(object):
+
+    def __init__(self, priority=1000, action=None, src=None, dst=None,
+                 proto=None, sport=None, dport=None, extra=None,
+                 ipv4_only=False, comment=None):
+        if proto not in ['tcp', 'udp', 'icmp', None]:
+            raise InvalidRuleExcepion()
+        if proto not in ['tcp', 'udp'] and (sport is not None or
+                                            dport is not None):
+            raise InvalidRuleExcepion()
+
+        self.priority = int(priority)
+        self.action = action
+
+        (self.src4, self.src6) = (None, None)
+        if isinstance(src, tuple):
+            (self.src4, self.src6) = src
+            if not self.src6:
+                ipv4_only = True
+        (self.dst4, self.dst6) = (None, None)
+        if isinstance(dst, tuple):
+            (self.dst4, self.dst6) = dst
+            if not self.dst6:
+                ipv4_only = True
+
+        self.proto = proto
+        self.sport = sport
+        self.dport = dport
+
+        self.extra = extra
+        self.ipv4_only = (ipv4_only or
+                          extra is not None and bool(ipv4_re.search(extra)))
+        self.comment = comment
+
+    def __hash__(self):
+        return hash(frozenset(self.__dict__.items()))
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__
+
+    def __lt__(self, other):
+        return self.priority < other.priority
+
+    def __repr__(self):
+        return '<IptRule: @%d %s >' % (self.priority, self.compile())
+
+    def __unicode__(self):
+        return self.__repr__()
+
+    def compile(self, proto='ipv4'):
+        opts = OrderedDict([('src4' if proto == 'ipv4' else 'src6', '-s %s'),
+                            ('dst4' if proto == 'ipv4' else 'dst6', '-d %s'),
+                            ('proto', '-p %s'),
+                            ('sport', '--sport %s'),
+                            ('dport', '--dport %s'),
+                            ('extra', '%s'),
+                            ('comment', '-m comment --comment "%s"'),
+                            ('action', '-g %s')])
+        params = [opts[param] % getattr(self, param)
+                  for param in opts
+                  if getattr(self, param) is not None]
+        return ' '.join(params)
+
+
+class IptChain(object):
+    nat_chains = ('PREROUTING', 'POSTROUTING')
+    builtin_chains = ('FORWARD', 'INPUT', 'OUTPUT') + nat_chains
+
+    def __init__(self, name):
+        self.rules = set()
+        self.name = name
+
+    def add(self, *args, **kwargs):
+        for rule in args:
+            self.rules.add(rule)
+
+    def sort(self):
+        return sorted(list(self.rules))
+
+    def __len__(self):
+        return len(self.rules)
+
+    def __repr__(self):
+        return '<IptChain: %s %s>' % (self.name, self.rules)
+
+    def __unicode__(self):
+        return self.__repr__()
+
+    def compile(self, proto='ipv4'):
+        assert proto in ('ipv4', 'ipv6')
+        prefix = '-A %s ' % self.name
+        return '\n'.join([prefix + rule.compile(proto)
+                          for rule in self.sort()
+                          if not (proto == 'ipv6' and rule.ipv4_only)])
+
+    def compile_v6(self):
+        return self.compile('ipv6')

circle/firewall/tasks/local_tasks.py

@@ -26,7 +26,7 @@ def _apply_once(name, queues, task, data):
 
 @celery.task(ignore_result=True)
 def periodic_task():
-    from firewall.fw import Firewall, dhcp, dns, ipset, vlan
+    from firewall.fw import BuildFirewall, dhcp, dns, ipset, vlan
     from remote_tasks import (reload_dns, reload_dhcp, reload_firewall,
                               reload_firewall_vlan, reload_blacklist)
 
@@ -40,7 +40,7 @@ def periodic_task():
     _apply_once('dhcp', firewall_queues, reload_dhcp,
                 lambda: (dhcp(), ))
     _apply_once('firewall', firewall_queues, reload_firewall,
-                lambda: (Firewall(proto=4).get(), Firewall(proto=6).get()))
+                lambda: (BuildFirewall().build_ipt()))
     _apply_once('firewall_vlan', firewall_queues, reload_firewall_vlan,
                 lambda: (vlan(), ))
     _apply_once('blacklist', firewall_queues, reload_blacklist,
@@ -48,7 +48,7 @@ def periodic_task():
 
 
 @celery.task
-def reloadtask(type='Host'):
+def reloadtask(type='Host', timeout=15):
     reload = {
         'Host': ['dns', 'dhcp', 'firewall'],
         'Record': ['dns'],

circle/firewall/templates/firewall/iptables.conf

+{% if proto == "ipv4" %}
+*nat
+:PREROUTING ACCEPT [0:0]
+:INPUT ACCEPT [0:0]
+:OUTPUT ACCEPT [0:0]
+:POSTROUTING ACCEPT [0:0]
+{% for chain in nat %}
+{{ chain.compile|safe }}
+{% endfor %}
+COMMIT
+{% endif %}
+
+*filter
+:INPUT DROP [0:0]
+:FORWARD DROP [0:0]
+:OUTPUT DROP [0:0]
+
+# initialize logging
+-N LOG_DROP
+# windows port scan are silently dropped
+-A LOG_DROP -p tcp --dport 445 -j DROP
+-A LOG_DROP -p udp --dport 137 -j DROP
+-A LOG_DROP -j LOG --log-level 7 --log-prefix "[ipt][drop]"
+-A LOG_DROP -j DROP
+-N LOG_ACC
+-A LOG_ACC -j LOG --log-level 7 --log-prefix "[ipt][isok]"
+-A LOG_ACC -j ACCEPT
+
+# initialize FORWARD chain
+-A FORWARD -m set --match-set blacklist src,dst -j DROP
+-A FORWARD -m state --state INVALID -g LOG_DROP
+-A FORWARD -m state --state ESTABLISHED,RELATED -j ACCEPT
+-A FORWARD -p icmp --icmp-type echo-request -g LOG_ACC
+
+# initialize INPUT chain
+-A INPUT -m set --match-set blacklist src -j DROP
+-A INPUT -m state --state INVALID -g LOG_DROP
+-A INPUT -i lo -j ACCEPT
+-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
+
+# initialize OUTPUT chain
+-A OUTPUT -m state --state INVALID -g LOG_DROP
+-A OUTPUT -o lo -j ACCEPT
+-A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
+
+{% for chain in filter %}
+{% if chain.name not in chain.builtin_chains %}-N {{ chain.name }}{% endif %}
+{% if proto == "ipv4" %}
+{{ chain.compile|safe }}
+{% else %}
+{{ chain.compile_v6|safe }}
+{% endif %}
+{% endfor %}
+
+# close all chains
+-A FORWARD -g LOG_DROP
+-A INPUT -g LOG_DROP
+-A OUTPUT -g LOG_DROP
+COMMIT

circle/firewall/tests/test_firewall.py

-from netaddr import IPSet
+from netaddr import IPSet, AddrFormatError
 
 from django.test import TestCase
 from django.contrib.auth.models import User
 from ..admin import HostAdmin
-from firewall.models import Vlan, Domain, Record, Host
+from firewall.models import (Vlan, Domain, Record, Host, VlanGroup, Group,
+                             Rule, Firewall)
+from firewall.fw import dns, ipv6_to_octal
+from firewall.tasks.local_tasks import periodic_task, reloadtask
 from django.forms import ValidationError
+from ..iptables import IptRule, IptChain, InvalidRuleExcepion
+from mock import patch
+
+import django.conf
+settings = django.conf.settings.FIREWALL_SETTINGS
 
 
 class MockInstance:
@@ -96,12 +104,12 @@ class HostGetHostnameTestCase(TestCase):
         self.vlan.save()
         self.h = Host(hostname='h', mac='01:02:03:04:05:00', ipv4='10.0.0.1',
                       vlan=self.vlan, owner=self.u1, shared_ip=True,
-                      pub_ipv4=self.vlan.snat_ip)
+                      external_ipv4=self.vlan.snat_ip)
         self.h.save()
 
     def test_issue_93_wo_record(self):
         self.assertEqual(self.h.get_hostname(proto='ipv4', public=True),
-                         unicode(self.h.pub_ipv4))
+                         unicode(self.h.external_ipv4))
 
     def test_issue_93_w_record(self):
         self.r = Record(name='vm', type='A', domain=self.d, owner=self.u1,
@@ -109,3 +117,192 @@ class HostGetHostnameTestCase(TestCase):
         self.r.save()
         self.assertEqual(self.h.get_hostname(proto='ipv4', public=True),
                          self.r.fqdn)
+
+
+class IptablesTestCase(TestCase):
+    def setUp(self):
+        self.r = [IptRule(priority=4, action='ACCEPT',
+                          src=('127.0.0.4', None)),
+                  IptRule(priority=4, action='ACCEPT',
+                          src=('127.0.0.4', None)),
+                  IptRule(priority=2, action='ACCEPT',
+                          dst=('127.0.0.2', None),
+                          extra='-p icmp'),
+                  IptRule(priority=6, action='ACCEPT',
+                          dst=('127.0.0.6', None),
+                          proto='tcp', dport=80),
+                  IptRule(priority=1, action='ACCEPT',
+                          dst=('127.0.0.1', None),
+                          proto='udp', dport=53),
+                  IptRule(priority=5, action='ACCEPT',
+                          dst=('127.0.0.5', None),
+                          proto='tcp', dport=443),
+                  IptRule(priority=2, action='ACCEPT',
+                          dst=('127.0.0.2', None),
+                          proto='icmp'),
+                  IptRule(priority=6, action='ACCEPT',
+                          dst=('127.0.0.6', None),
+                          proto='tcp', dport='1337')]
+
+    def test_chain_add(self):
+        ch = IptChain(name='test')
+        ch.add(*self.r)
+        self.assertEqual(len(ch), len(self.r) - 1)
+
+    def test_rule_compile_ok(self):
+        assert unicode(self.r[5])
+        self.assertEqual(self.r[5].compile(),
+                         '-d 127.0.0.5 -p tcp --dport 443 -g ACCEPT')
+
+    def test_rule_compile_fail(self):
+        self.assertRaises(InvalidRuleExcepion,
+                          IptRule, **{'proto': 'test'})
+        self.assertRaises(InvalidRuleExcepion,
+                          IptRule, **{'priority': 5, 'action': 'ACCEPT',
+                                      'dst': '127.0.0.5',
+                                      'proto': 'icmp', 'dport': 443})
+
+    def test_chain_compile(self):
+        ch = IptChain(name='test')
+        ch.add(*self.r)
+        compiled = ch.compile()
+        compiled_v6 = ch.compile_v6()
+        assert unicode(ch)
+        self.assertEqual(len(compiled.splitlines()), len(ch))
+        self.assertEqual(len(compiled_v6.splitlines()), 0)
+
+
+class ReloadTestCase(TestCase):
+    def setUp(self):
+        self.u1 = User.objects.create(username='user1')
+        self.u1.save()
+        d = Domain.objects.create(name='example.org', owner=self.u1)
+        self.vlan = Vlan(vid=1, name='test', network4='10.0.0.0/29',
+                         snat_ip='152.66.243.99',
+                         network6='2001:738:2001:4031::/80', domain=d,
+                         owner=self.u1, network_type='portforward',
+                         dhcp_pool='manual')
+        self.vlan.save()
+        self.vlan2 = Vlan(vid=2, name='pub', network4='10.1.0.0/29',
+                          network6='2001:738:2001:4032::/80', domain=d,
+                          owner=self.u1, network_type='public')
+        self.vlan2.save()
+        self.vlan.snat_to.add(self.vlan2)
+
+        settings["default_vlangroup"] = 'public'
+        settings["default_host_groups"] = ['netezhet']
+        vlg = VlanGroup.objects.create(name='public')
+        vlg.vlans.add(self.vlan, self.vlan2)
+        self.hg = Group.objects.create(name='netezhet')
+        Rule.objects.create(action='accept', hostgroup=self.hg,
+                            foreign_network=vlg)
+
+        firewall = Firewall.objects.create(name='fw')
+        Rule.objects.create(action='accept', firewall=firewall,
+                            foreign_network=vlg)
+
+        for i in range(1, 6):
+            h = Host.objects.create(hostname='h-%d' % i, vlan=self.vlan,
+                                    mac='01:02:03:04:05:%02d' % i,
+                                    ipv4='10.0.0.%d' % i, owner=self.u1)
+            h.enable_net()
+            h.groups.add(self.hg)
+            if i == 5:
+                h.vlan = self.vlan2
+                h.save()
+                self.h5 = h
+            if i == 1:
+                self.h1 = h
+
+        self.r1 = Record(name='tst', type='A', address='127.0.0.1',
+                         domain=d, owner=self.u1)
+        self.rb = Record(name='tst', type='AAAA', address='1.0.0.1',
+                         domain=d, owner=self.u1)
+        self.r2 = Record(name='ts', type='AAAA', address='2001:123:45::6',
+                         domain=d, owner=self.u1)
+        self.rm = Record(name='asd', type='MX', address='10:teszthu',
+                         domain=d, owner=self.u1)
+        self.rt = Record(name='asd', type='TXT', address='ASD',
+                         domain=d, owner=self.u1)
+        self.r1.save()
+        self.r2.save()
+        with patch('firewall.models.Record.clean'):
+            self.rb.save()
+        self.rm.save()
+        self.rt.save()
+
+    def test_bad_aaaa_record(self):
+        self.assertRaises(AddrFormatError, ipv6_to_octal, self.rb.address)
+
+    def test_good_aaaa_record(self):
+        ipv6_to_octal(self.r2.address)
+
+    def test_dns_func(self):
+        records = dns()
+        self.assertEqual(Host.objects.count() * 2 +         # soa
+                         len((self.r1, self.r2, self.rm, self.rt)) + 1,
+                         len(records))
+
+    def test_host_add_port(self):
+        h = self.h1
+        h.ipv6 = '2001:2:3:4::0'
+        assert h.behind_nat
+        h.save()
+        old_rules = h.rules.count()
+        h.add_port('tcp', private=22)
+        new_rules = h.rules.count()
+        self.assertEqual(new_rules, old_rules + 1)
+        self.assertEqual(len(h.list_ports()), old_rules + 1)
+        endp = h.get_public_endpoints(22)
+        self.assertEqual(endp['ipv4'][0], h.ipv4)
+        assert int(endp['ipv4'][1])
+        self.assertEqual(endp['ipv6'][0], h.ipv6)
+        assert int(endp['ipv6'][1])
+
+    def test_host_add_port2(self):
+        h = self.h5
+        h.ipv6 = '2001:2:3:4::1'
+        h.save()
+        assert not h.behind_nat
+        old_rules = h.rules.count()
+        h.add_port('tcp', private=22)
+        new_rules = h.rules.count()
+        self.assertEqual(new_rules, old_rules + 1)
+        self.assertEqual(len(h.list_ports()), old_rules + 1)
+        endp = h.get_public_endpoints(22)
+        self.assertEqual(endp['ipv4'][0], h.ipv4)
+        assert int(endp['ipv4'][1])
+        self.assertEqual(endp['ipv6'][0], h.ipv6)
+        assert int(endp['ipv6'][1])
+
+    def test_host_del_port(self):
+        h = self.h1
+        h.ipv6 = '2001:2:3:4::0'
+        h.save()
+        h.add_port('tcp', private=22)
+        old_rules = h.rules.count()
+        h.del_port('tcp', private=22)
+        new_rules = h.rules.count()
+        self.assertEqual(new_rules, old_rules - 1)
+
+    def test_host_add_port_wo_vlangroup(self):
+        VlanGroup.objects.filter(name='public').delete()
+        h = self.h1
+        old_rules = h.rules.count()
+        h.add_port('tcp', private=22)
+        new_rules = h.rules.count()
+        self.assertEqual(new_rules, old_rules)
+
+    def test_host_add_port_w_validationerror(self):
+        h = self.h1
+        self.assertRaises(ValidationError, h.add_port,
+                          'tcp', public=1000, private=22)
+
+    def test_periodic_task(self):
+        #TODO
+        with patch('firewall.tasks.local_tasks.cache') as cache:
+            self.test_host_add_port()
+            self.test_host_add_port2()
+            periodic_task()
+            reloadtask()
+            assert cache.delete.called

circle/network/forms.py

@@ -106,7 +106,7 @@ class HostForm(ModelForm):
                 'ipv4',
                 'ipv6',
                 'shared_ip',
-                'pub_ipv4',
+                'external_ipv4',
             ),
             Fieldset(
                 'Information',
@@ -162,12 +162,14 @@ class RuleForm(ModelForm):
                 'foreign_network',
                 'dport',
                 'sport',
+                'weight',
                 'proto',
                 'extra',
-                'accept',
+                'action',
                 'owner',
                 'nat',
-                'nat_dport',
+                'nat_external_port',
+                'nat_external_ipv4',
             ),
             Fieldset(
                 'External',
@@ -232,6 +234,7 @@ class VlanForm(ModelForm):
                 'IPv6',
                 'network6',
                 'ipv6_template',
+                'host_ipv6_prefixlen',
             ),
             Fieldset(
                 'Domain name service',

circle/network/tables.py

@@ -44,7 +44,7 @@ class HostTable(Table):
         model = Host
         attrs = {'class': 'table table-striped table-condensed'}
         fields = ('hostname', 'vlan', 'mac', 'ipv4', 'ipv6',
-                  'pub_ipv4', 'created_at', 'owner', )
+                  'external_ipv4', 'created_at', 'owner', )
         order_by = ('vlan', 'hostname', )
 
 
@@ -128,7 +128,8 @@ class RuleTable(Table):
         model = Rule
         attrs = {'class': 'table table-striped table-hover table-condensed'}
         fields = ('r_type', 'color_desc', 'owner', 'extra', 'direction',
-                  'accept', 'proto', 'sport', 'dport', 'nat', 'nat_dport', )
+                  'action', 'proto', 'sport', 'dport', 'nat',
+                  'nat_external_port', )
         order_by = 'direction'
 
 

circle/network/templates/network/columns/host-rule.html

@@ -33,5 +33,6 @@
 
 {% if record.nat %}
     <span class="label label-success">NAT
-        [ {{ record.dport }} <i class="icon-arrow-right"></i> {{record.nat_dport}} ]</span>
+        [ {{ record.dport }} <i class="icon-arrow-right"></i>
+        {{record.nat_external_port}} ]</span>
 {% endif %}

circle/network/views.py

@@ -503,6 +503,11 @@ class RuleList(LoginRequiredMixin, SuperuserRequiredMixin, SingleTableView):
     template_name = "network/rule-list.html"
     table_pagination = False
 
+    def get_table_data(self):
+        return Rule.objects.select_related('host', 'hostgroup', 'vlan',
+                                           'vlangroup', 'firewall',
+                                           'foreign_network', 'owner')
+
 
 class RuleDetail(LoginRequiredMixin, SuperuserRequiredMixin,
                  SuccessMessageMixin, UpdateView):