Merge branch 'debian' into multinode

Conflicts:
	README.md
	pillar/firewall.sls

README.md

@@ -7,6 +7,7 @@
     * Cent OS 7+
     * Scientific Linux 7+
 * Debian Linux family:
+    * Debian linux 8+
     * Ubuntu linux 14.04 LTS
 
 ## Prerequisites
@@ -28,7 +29,7 @@ sudo yum install python-pip gcc vim git
 Install some important packages:
 ```bash
 sudo apt-get update
-sudo apt-get install python-pip git
+sudo apt-get install python-pip vim git
 ```
 
 ## Install Salt
@@ -61,7 +62,7 @@ Most used variables
 * **database**:
     * **password**: database user’s password
 * **amqp**:
-    * **password**: ampq user’s password
+    * **password**: amqp user’s password
     * **host**: amqp server IP - usually runs at localhost
 * **graphite**:
     * **password**: graphite user’s password
@@ -103,6 +104,7 @@ Other variables
     * vm_if: vm interface
     * vm_et: vm network
     * management_if: management interface
+    * reload_firewall_timeout: timeout for synchronous firewall reload
 * graphite:
     * user: graphite user
     * port: graphite server’s port
@@ -192,10 +194,11 @@ Finally, we can run the machine. Click on 'deploy' and start it. You can choose,
 
 ## Troubleshooting ##
 
-### Portal won't loaded
-Maybe port 443 closed. Check and open it.
+### Portal won't load
+Maybe port 443 is closed. Check and open it.
 
-### Portal wont loaded on Ubuntu 14.04
+
+### Portal won't load on Ubuntu 14.04
 ```bash
 sudo service nginx restart
 ```

pillar/firewall.sls

@@ -18,3 +18,5 @@ fwdriver:
   external_net: 10.0.0.97/16
   gateway: 10.0.255.254
   open_ports: False
+
+  reload_firewall_timeout: 120

salt/agentdriver/configuration.sls

@@ -13,7 +13,7 @@
     - user: root
     - group: root
 
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/agentdriver.service:
   file.managed:
     - user: root
@@ -37,7 +37,7 @@ incrond:
 incron:
 {% endif %}
   service:
-    - reload: true
+    - full_restart: true
     - enable: true
     - running
     - watch:

salt/allinone.sls

 include:
+  - profile
   - agentdriver
   - manager
   - graphite

salt/fwdriver/configuration.sls

@@ -9,7 +9,7 @@ include:
     - group: {{ pillar['fwdriver']['user'] }}
     - mode: 700
 
-{% if grains['os_family'] == 'RedHat' %} 
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %} 
 /etc/systemd/system/firewall.service:
   file.managed:
     - user: root
@@ -22,7 +22,7 @@ include:
     - user: root
     - group: root
     - template: jinja
-    - source: file:///home/{{ pillar['fwdriver']['user'] }}/fwdriver/miscellaneous/firewall-init.service
+    - source: salt://fwdriver/files/firewall-init.service
 {% else %}
 /etc/init/firewall.conf:
   file.managed:
@@ -55,7 +55,7 @@ include:
     - user: {{ pillar['fwdriver']['user'] }}
     - group: {{ pillar['fwdriver']['user'] }}
 
-{% if grains['os_family'] != 'RedHat' %}
+{% if grains['os_family'] != 'RedHat' and grains['os'] != 'Debian' %}
 /etc/init.d/isc-dhcp-server:
   file.symlink:
     - target: /lib/init/upstart-job
@@ -77,12 +77,18 @@ include:
     - source: salt://fwdriver/files/sudoers
 
 
-{% if grains['os_family'] == 'RedHat' %}
-
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 systemd-sysctl:
+  cmd.run:
+    - name: /bin/systemctl restart systemd-sysctl
   service.running:
-    - reload: True
+    - watch:
+      - file: /etc/sysctl.d/60-circle-firewall.conf
+    - require:
+      - cmd: systemd-sysctl
+{% endif %}
 
+{% if grains['os_family'] == 'RedHat' %}
 /root/firewall-init.te:
   file.managed:
     - source: salt://fwdriver/files/firewall-init.te

salt/fwdriver/files/firewall-init.service

+[Unit]
+Description=CIRCLE firewall init
+After=network.target
+#Before=firewall.service
+
+[Service]
+User=root
+Group=root
+
+Type=oneshot
+ExecStart=/bin/bash -c "/bin/systemctl restart openvswitch{%if grains['os']=='Debian'%}-switch{%endif%} ; /sbin/ip netns add fw || true; ovs-vsctl del-br firewall || true; /sbin/ip netns exec fw sysctl -f /etc/sysctl.d/60-circle-firewall.conf ; /sbin/ip netns exec fw ip link set lo up"
+
+[Install]
+WantedBy=multi-user.target
+

salt/fwdriver/init.sls

@@ -28,7 +28,10 @@ firewall:
       - python-dev
       - libmemcached-dev
       - openvswitch-switch
+      {% if grains['os'] != 'Debian' %}
+      {# No such package in Debian Jessie! #}
       - openvswitch-controller
+      {% endif %}
       - isc-dhcp-server
       {% endif %}
       - git

salt/graphite/configuration.sls

@@ -17,7 +17,7 @@ requirements:
       - require:
         - user: {{ pillar['graphite']['user'] }}
   
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 
 /etc/systemd/system/graphite.service:
   file.managed:

salt/manager/configuration.sls

@@ -8,7 +8,7 @@ manager_postactivate:
 
 portal.conf:
   file.managed:
-    {% if grains['os_family'] == 'RedHat' %}
+    {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
     - name: /etc/systemd/system/portal.service
     {% else %}
     - name: /etc/init/portal.conf
@@ -16,7 +16,7 @@ portal.conf:
     - user: root
     - group: root
     - template: jinja
-    {% if grains['os_family'] == 'RedHat' %}
+    {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 
     {% if pillar['deployment_type'] == 'production' %}
     - source: file:///home/{{ pillar['user'] }}/circle/miscellaneous/portal-uwsgi.service
@@ -34,7 +34,7 @@ portal.conf:
 
     {% endif %}
 
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/manager.service:
   file.managed:
     - user: root

salt/manager/files/nginx-default-site.conf

@@ -14,9 +14,6 @@ server {
         alias /home/{{ pillar['user'] }}/circle/circle/static_collected; # your Django project's static files
     }
 {% endif %}
-    location /doc {
-        alias /home/cloud/circle-website/_build/html;
-    }
 
     location / {
 {% if pillar['deployment_type'] == "production" %}

salt/manager/files/nginx.te

+
+module nginx 1.0;
+
+require {
+	type initrc_tmp_t;
+	type httpd_t;
+	type initrc_t;
+	class sock_file write;
+	class unix_stream_socket connectto;
+}
+
+#============= httpd_t ==============
+allow httpd_t initrc_t:unix_stream_socket connectto;
+
+#!!!! This avc is allowed in the current policy
+allow httpd_t initrc_tmp_t:sock_file write;

salt/manager/init.sls

@@ -61,7 +61,7 @@ manager:
     - enable: True
     - watch:
       - file: manager_postactivate
-      {% if grains['os_family'] == 'RedHat' %}
+      {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
       - file: /etc/systemd/system/manager.service
       - file: /etc/systemd/system/managercelery@.service
       {% else %}

salt/manager/nginx.sls

 nginx:
   service.running:
     - enable: True
-    - require:
+    - watch:
        - pkg: nginx
        - cmd: circlecert
        - file: nginxdefault
+       - file: nginx_home_permission
        {% if grains['os_family'] == 'RedHat' %}
        - file: nginxconf
+       - cmd: nginx_no_private_temp
        {% endif %}
   pkg:
    - installed
 
+nginx_home_permission:
+  file.directory:
+    - name: /home/{{ pillar['user'] }}
+    - user: {{ pillar['user'] }}
+    - dir_mode: 711
+
 circlecert:
   cmd.run:
     {% if grains['os_family'] == 'RedHat' %}
@@ -22,17 +30,50 @@ circlecert:
     - creates: /etc/ssl/certs/circle.pem
 
 {% if grains['os_family'] == 'RedHat' %}
-nginx_selinux:
+nginx_selinux_pkgs:
   pkg.installed:
     - pkgs:
       - policycoreutils
       - policycoreutils-python
+
+nginx_httpd_can_network_connect:
   selinux.boolean:
     - name: httpd_can_network_connect
     - value: True
     - persist: True
     - require:
-      - pkg: nginx_selinux
+      - pkg: nginx_selinux_pkgs
+
+nginx_httpd_read_user_content:
+  selinux.boolean:
+    - name: httpd_read_user_content
+    - value: True
+    - persist: True
+    - require:
+      - pkg: nginx_selinux_pkgs
+
+/root/nginx.te:
+  file.managed:
+    - source: salt://manager/files/nginx.te
+    - template: jinja
+    - mode: 644
+
+nginx_semodule:
+  cmd.run:
+    - cwd: /root
+    - user: root
+    - name: checkmodule -M -m -o nginx.mod nginx.te; semodule_package -o nginx.pp -m nginx.mod; semodule -i nginx.pp
+    - unless: semodule -l |grep -qs ^nginx
+    - require:
+      - file: /root/nginx.te
+      - pkg: nginx_selinux_pkgs
+
+nginx_no_private_temp:
+  cmd.run:
+    - user: root
+    - name: sed -i "/PrivateTmp/d" /usr/lib/systemd/system/nginx.service
+    - require:
+      - pkg: nginx
 {% endif %}
 
 nginxdefault:

salt/manager/pipeline.sls

-{% if grains['os'] == 'Ubuntu' %}
+{% if grains['os'] == 'Ubuntu' or grains['os'] == 'Debian' %}
 nodejs-legacy:
   pkg.installed
 {% endif %}
 
 npm:
-  {% if grains['os'] == 'Ubuntu' %}
+  {% if grains['os'] == 'Ubuntu' or grains['os'] == 'Debian' %}
   pkg.installed:
     - require:
       - pkg: nodejs-legacy

salt/monitor-client/configuration.sls

@@ -6,7 +6,7 @@
     - group: {{ pillar['user'] }}
     - mode: 700
 
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/monitor-client.service:
   file.managed:
     - user: root

salt/network/files/fix_dhcp.sh

 #!/bin/bash
 sed -i '/HWADDR=.*/d' /etc/sysconfig/network-scripts/ifcfg-vm
 sed -i -e \$aNM_CONTROLLED=\"no\" /etc/sysconfig/network-scripts/ifcfg-vm
+/bin/systemctl daemon-reload
 ifup vm
 systemctl restart firewall
 systemctl restart dhcpd
-
 exit 0
 

salt/network/files/fix_dhcp_Debian.conf

+# systemd service file extras added by CIRCLE Salt installer:
+# openvswitch and virtual network interface must be up before
+# dhcpd is started
+
+[Unit]
+After=openvswitch-switch.service
+[Service]
+ExecStartPre=-/sbin/ifup vm
+{# TODO: change 'vm' to pillar['fwdriver']['vm_if'] ? #}
+{# TODO: similar patch for firewall.service ? #}

salt/network/files/network

+NETWORKING_IPV6=yes
+IPV6FORWARDING=yes

salt/network/files/reload_firewall.sh

@@ -2,4 +2,4 @@
 
 source /home/{{ pillar['user'] }}/.virtualenvs/circle/bin/activate
 source /home/{{ pillar['user'] }}/.virtualenvs/circle/bin/postactivate
-python /home/{{ pillar['user'] }}/circle/circle/manage.py reload_firewall --sync
+python /home/{{ pillar['user'] }}/circle/circle/manage.py reload_firewall --sync --timeout={{ pillar['fwdriver']['reload_firewall_timeout'] }}

salt/network/init.sls

@@ -13,12 +13,26 @@ vm:
     - pre_up_cmds:
       {% if grains['os_family'] == 'RedHat' %}
       - /bin/systemctl restart openvswitch
+      {% elif grains['os'] == 'Debian' %}
+      - /bin/systemctl restart openvswitch-switch
       {% else %}  
       - /etc/init.d/openvswitch-switch restart
       {% endif %} 
     - require:
       - cmd: ovs-if
 
+{% if grains['os'] == 'Debian' %}
+symlink_dhcpd:
+  file.symlink:
+    - name: /etc/init.d/dhcpd
+    - target: /etc/init.d/isc-dhcp-server
+    - force: True
+  cmd.run:
+    - name: /bin/systemctl daemon-reload
+    - require:
+      - file: symlink_dhcpd
+{% endif %}
+
 firewall2:
   service:
     - name: firewall
@@ -26,24 +40,75 @@ firewall2:
     - require:
       - network: vm
 
-salt://network/files/reload_firewall.sh:
+reload_firewall:
   cmd.script:
+    - name: salt://network/files/reload_firewall.sh
     - template: jinja
     - user: {{ pillar['user'] }}
     - require:
       - service: firewall2
+      {% if grains['os'] == 'Debian' %}
+      - cmd: symlink_dhcpd
+      {% endif %}
 
 {% if grains['os_family'] == 'RedHat' %}
-salt://network/files/fix_dhcp.sh:
-  cmd.script
+net_config:
+  file.managed:
+    - name: /etc/sysconfig/network
+    - source: salt://network/files/network
+    - user: root
+    - group: root
+    - mode: 644
+
+fix_dhcp:
+  cmd.script:
+    - name: salt://network/files/fix_dhcp.sh
+    - require:
+      - cmd: reload_firewall
+      - file: net_config
 {% endif %}
 
 isc-dhcp-server:
-  service:
-    - running
+  {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
+  cmd.run:
+    - name: /bin/systemctl restart dhcpd
     {% if grains['os_family'] == 'RedHat' %}
-    - name: dhcpd
+    - watch:
+      - cmd: fix_dhcp
+    {% elif grains['os'] == 'Debian' %}
+    - watch:
+      - cmd: fix_dhcp_daemon_reload
     {% endif %}
+  {% endif %}
+  service.running:
     - enable: True
-    - reload: True
+    {% if grains['os_family'] == 'RedHat' %}
+    - watch:
+      - cmd: fix_dhcp
+    {% elif grains['os'] == 'Debian' %}
+    - watch:
+      - cmd: fix_dhcp_daemon_reload
+    {% endif %}
+    {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
+    - name: dhcpd
+    - require:
+      - cmd: isc-dhcp-server
+    {% endif %}
 
+{% if grains['os'] == 'Debian' %}
+{# For next reboot #}
+after_openvswitch_conf:
+  file.managed:
+    - name: /etc/systemd/system/isc-dhcp-server.service.d/after_openvswitch.conf
+    - source: salt://network/files/fix_dhcp_Debian.conf
+    - user: root
+    - group: root
+    - template: jinja
+    - makedirs: True
+
+fix_dhcp_daemon_reload:
+  cmd.run:
+    - name: /bin/systemctl daemon-reload
+    - require:
+      - file: after_openvswitch_conf
+{% endif %}

salt/node.sls

 include:
+    - profile
     - agentdriver
     - monitor-client
     - vmdriver

salt/profile.sls

+{% if grains['os']=='Debian' %}
+{# For non-interactive shells, virtualenvwrapper commands 
+   ('workon' etc.) are not sourced automatically #}
+/etc/profile:
+  file.append:
+    - text:
+      - "#Line below added for Debian by CIRCLE Salt installer"
+      - . /etc/bash_completion
+{% endif %}
+

salt/storagedriver/configuration.sls

@@ -6,7 +6,7 @@
     - group: {{ pillar['user'] }}
     - mode: 700
 
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/storagecelery@.service:
   file.managed:
     - user: root

salt/vmdriver/configuration.sls

@@ -9,9 +9,10 @@ include:
     - group: {{ pillar['user'] }}
     - mode: 700
 
-{% set service_dir = "/etc/systemd/system/" if grains['os_family'] == 'RedHat' else "/etc/init/" %}
+{% set service_dir = "/etc/systemd/system/" if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' else "/etc/init/" %}
 {% set service_files = (("vmcelery@.service", "netcelery@.service", "node.service")
-                        if grains['os_family'] == 'RedHat' else
+                        if grains['os_family'] == 'RedHat' 
+                        or grains['os'] == 'Debian' else
                         ("vmcelery.conf", "netcelery.conf", "node.conf")) %}
 
 {% for file in service_files %}

salt/vmdriver/files/org.libvirt.unix.manage.pkla

+[Allow cloud libvirt management permissions]
+Identity=unix-user:cloud
+Action=org.libvirt.unix.manage;org.libvirt.unix.monitor
+ResultAny=yes
+ResultInactive=yes
+ResultActive=yes
+{# TODO: change 'cloud' to ? #}

salt/vmdriver/files/sudoers

 {{ pillar['user'] }}	ALL = (ALL) NOPASSWD: /usr/bin/ovs-ofctl, /usr/bin/ovs-vsctl, /sbin/ip link set *
-Defaults: cloud !requiretty
+Defaults: {{ pillar['user'] }} !requiretty
 

salt/vmdriver/init.sls

@@ -32,7 +32,10 @@ vmdriver:
       - libxslt1-dev
       - openvswitch-common
       - openvswitch-switch
+      {% if grains['os'] != 'Debian' %}
+      {# No such package in Debian Jessie! #}
       - openvswitch-controller
+      {% endif %}
       - python-dev
       - python-libvirt
       - virtualenvwrapper
@@ -41,7 +44,7 @@ vmdriver:
       {% endif %}
     - require_in:
       - file: /etc/default/libvirt-bin
-      {% if grains['os_family'] == 'RedHat' %}
+      {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
       - service: libvirtd
       {% else %}
       - file: /etc/apparmor.d/libvirt/TEMPLATE

salt/vmdriver/libvirt.sls

@@ -15,7 +15,7 @@ libvirtconf:
   file.append:
     - text: libvirtd_opts="-d -l"
 
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 libvirtd:
 {% else %}
 libvirt-bin:
@@ -68,6 +68,32 @@ vmdriver_semodule:
       - file: /root/vmdriver.te
       - pkg: selinux_pkgs
 
+{% elif grains['os'] == 'Debian' %}
+
+/usr/bin/kvm:
+  file.replace:
+    - pattern: -enable-kvm
+    - repl: ""
+    - watch:
+      - pkg: vmdriver
+
+policycoreutils:
+  pkg.installed
+
+{# Note: Debian Jessie has polkit 0.105, which uses pkla format instead of js #}
+/etc/polkit-1/localauthority/50-local.d/org.libvirt.unix.manage.pkla:
+  file.managed:
+    - source: salt://vmdriver/files/org.libvirt.unix.manage.pkla
+    - user: root
+    - group: root
+    - template: jinja
+
+polkitd:
+  service:
+    - running
+    - watch:
+      - file: /etc/polkit-1/localauthority/50-local.d/org.libvirt.unix.manage.pkla
+
 {% else %}
 
 /etc/apparmor.d/libvirt/TEMPLATE:

salt/vmdriver/virtualenv.sls

@@ -7,10 +7,14 @@ virtualenv_vmdriver:
 
 {% set libvirt_dir = "/usr/lib64/python2.7/site-packages/" if grains['os_family'] == 'RedHat' else "/usr/lib/python2.7/dist-packages/" %}
 
+{% set targets = { 'libvirtmod_qemu.so': 'libvirtmod_qemu.x86_64-linux-gnu.so',
+                   'libvirtmod.so': 'libvirtmod.x86_64-linux-gnu.so'
+                 } if grains['os'] == 'Debian' else {} %}
+
 {% for file in ("libvirtmod_qemu.so", "libvirtmod.so", "libvirt_qemu.py", "libvirt.py", "libvirt_qemu.pyc", "libvirt.pyc") %}
 /home/{{ pillar['user'] }}/.virtualenvs/vmdriver/lib/python2.7/site-packages/{{ file }}:
   file.symlink:
-    - target: {{ libvirt_dir + file }}
+    - target: {{ libvirt_dir + targets[file]|default(file) }}
     - require:
       - virtualenv: virtualenv_vmdriver
 {% endfor %}

salt/vncproxy/configuration.sls

@@ -6,7 +6,7 @@
     - group: {{ pillar['user'] }}
     - mode: 700
 
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/vncproxy.service:
   file.managed:
     - user: root