Merge branch 'debian' into ceph

Conflicts: README.md

Merge branch 'debian' into ceph
Conflicts: README.md
15d544d2 · Czémán Arnold · b01ae5bd · 281aa630 · 15d544d2 · 15d544d2
Commit 15d544d2 authored Jun 13, 2016 by Czémán Arnold
31 changed files
--- a/README.md
+++ b/README.md
@@ -7,6 +7,7 @@
    * Cent OS 7+
    * Scientific Linux 7+
 * Debian Linux family:
+    * Debian linux 8+
    * Ubuntu linux 14.04 LTS
 ## Prerequisites
@@ -60,7 +61,7 @@ Most used variables
 * **database**:
    * **password**: database user’s password
 * **amqp**:
-    * **password**: ampq user’s password
+    * **password**: amqp user’s password
    * **host**: amqp server IP - usually runs at localhost
 * **graphite**:
    * **password**: graphite user’s password
@@ -101,6 +102,7 @@ Other variables
    * vm_if: vm interface
    * vm_et: vm network
    * management_if: management interface
+    * reload_firewall_timeout: timeout for synchronous firewall reload
 * graphite:
    * user: graphite user
    * port: graphite server’s port
@@ -129,7 +131,7 @@ Other variables
 Run the following installation command:
 ```bash
-salt/install.sh allinone
+sudo salt-call state.sls allinone --local --file-root=/home/$USER/salt/salt --pillar-root=/home/$USER/salt/pillar
 ```
 After this finished, you have to get "Failed: 0" message.
 If installer fails, please visit the [Troubleshooting](#troubleshooting) paragraph.
@@ -191,10 +193,11 @@ Finally, we can run the machine. Click on 'deploy' and start it. You can choose,
 ## Troubleshooting ##
-### Portal won't loaded
+### Portal won't load
-Maybe port 443 closed. Check and open it.
+Maybe port 443 is closed. Check and open it.
-### Portal wont loaded on Ubuntu 14.04
+### Portal won't load on Ubuntu 14.04
 ```bash
 sudo service nginx restart
 ```

--- a/pillar/firewall.sls
+++ b/pillar/firewall.sls
@@ -17,3 +17,5 @@ fwdriver:
  external_if: eth0
  external_net: 10.0.0.97/16
  gateway: 10.0.255.254
+  reload_firewall_timeout: 120
--- a/pillar/installer.sls
+++ b/pillar/installer.sls
--- a/salt/agentdriver/configuration.sls
+++ b/salt/agentdriver/configuration.sls
@@ -13,7 +13,7 @@
    - user: root
    - group: root
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/agentdriver.service:
  file.managed:
    - user: root
@@ -37,7 +37,7 @@ incrond:
 incron:
 {% endif %}
  service:
-    - reload: true
+    - full_restart: true
    - enable: true
    - running
    - watch:

--- a/salt/allinone.sls
+++ b/salt/allinone.sls
 include:
+  - profile
  - agentdriver
  - manager
  - graphite

--- a/salt/fwdriver/configuration.sls
+++ b/salt/fwdriver/configuration.sls
@@ -9,7 +9,7 @@ include:
    - group: {{ pillar['fwdriver']['user'] }}
    - mode: 700
-{% if grains['os_family'] == 'RedHat' %} 
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %} 
 /etc/systemd/system/firewall.service:
  file.managed:
    - user: root
@@ -22,7 +22,7 @@ include:
    - user: root
    - group: root
    - template: jinja
-    - source: file:///home/{{ pillar['fwdriver']['user'] }}/fwdriver/miscellaneous/firewall-init.service
+    - source: salt://fwdriver/files/firewall-init.service
 {% else %}
 /etc/init/firewall.conf:
  file.managed:
@@ -55,7 +55,7 @@ include:
    - user: {{ pillar['fwdriver']['user'] }}
    - group: {{ pillar['fwdriver']['user'] }}
-{% if grains['os_family'] != 'RedHat' %}
+{% if grains['os_family'] != 'RedHat' and grains['os'] != 'Debian' %}
 /etc/init.d/isc-dhcp-server:
  file.symlink:
    - target: /lib/init/upstart-job
@@ -77,12 +77,18 @@ include:
    - source: salt://fwdriver/files/sudoers
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 systemd-sysctl:
+  cmd.run:
+    - name: /bin/systemctl restart systemd-sysctl
  service.running:
-    - reload: True
+    - watch:
+      - file: /etc/sysctl.d/60-circle-firewall.conf
+    - require:
+      - cmd: systemd-sysctl
+{% endif %}
+{% if grains['os_family'] == 'RedHat' %}
 /root/firewall-init.te:
  file.managed:
    - source: salt://fwdriver/files/firewall-init.te

--- a/salt/fwdriver/files/firewall-init.service
+++ b/salt/fwdriver/files/firewall-init.service
+[Unit]
+Description=CIRCLE firewall init
+After=network.target
+#Before=firewall.service
+[Service]
+User=root
+Group=root
+Type=oneshot
+ExecStart=/bin/bash -c "/bin/systemctl restart openvswitch{%if grains['os']=='Debian'%}-switch{%endif%} ; /sbin/ip netns add fw || true; ovs-vsctl del-br firewall || true; /sbin/ip netns exec fw sysctl -f /etc/sysctl.d/60-circle-firewall.conf ; /sbin/ip netns exec fw ip link set lo up"
+[Install]
+WantedBy=multi-user.target
--- a/salt/fwdriver/init.sls
+++ b/salt/fwdriver/init.sls
@@ -18,7 +18,10 @@ firewall:
      - python-dev
      - libmemcached-dev
      - openvswitch-switch
+      {% if grains['os'] != 'Debian' %}
+      {# No such package in Debian Jessie! #}
      - openvswitch-controller
+      {% endif %}
      - isc-dhcp-server
      {% endif %}
      - git

--- a/salt/graphite/configuration.sls
+++ b/salt/graphite/configuration.sls
@@ -17,7 +17,7 @@ requirements:
      - require:
        - user: {{ pillar['graphite']['user'] }}
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/graphite.service:
  file.managed:

--- a/salt/manager/configuration.sls
+++ b/salt/manager/configuration.sls
@@ -8,7 +8,7 @@ manager_postactivate:
 portal.conf:
  file.managed:
-    {% if grains['os_family'] == 'RedHat' %}
+    {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
    - name: /etc/systemd/system/portal.service
    {% else %}
    - name: /etc/init/portal.conf
@@ -16,7 +16,7 @@ portal.conf:
    - user: root
    - group: root
    - template: jinja
-    {% if grains['os_family'] == 'RedHat' %}
+    {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
    {% if pillar['deployment_type'] == 'production' %}
    - source: file:///home/{{ pillar['user'] }}/circle/miscellaneous/portal-uwsgi.service
@@ -34,7 +34,7 @@ portal.conf:
    {% endif %}
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/manager.service:
  file.managed:
    - user: root

--- a/salt/manager/files/nginx-default-site.conf
+++ b/salt/manager/files/nginx-default-site.conf
@@ -14,9 +14,6 @@ server {
        alias /home/{{ pillar['user'] }}/circle/circle/static_collected; # your Django project's static files
    }
 {% endif %}
-    location /doc {
-        alias /home/cloud/circle-website/_build/html;
-    }
    location / {
 {% if pillar['deployment_type'] == "production" %}

--- a/salt/manager/files/nginx.te
+++ b/salt/manager/files/nginx.te
+module nginx 1.0;
+require {
+	type initrc_tmp_t;
+	type httpd_t;
+	type initrc_t;
+	class sock_file write;
+	class unix_stream_socket connectto;
+}
+#============= httpd_t ==============
+allow httpd_t initrc_t:unix_stream_socket connectto;
+#!!!! This avc is allowed in the current policy
+allow httpd_t initrc_tmp_t:sock_file write;
--- a/salt/manager/init.sls
+++ b/salt/manager/init.sls
@@ -61,7 +61,7 @@ manager:
    - enable: True
    - watch:
      - file: manager_postactivate
-      {% if grains['os_family'] == 'RedHat' %}
+      {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
      - file: /etc/systemd/system/manager.service
      - file: /etc/systemd/system/managercelery@.service
      {% else %}

--- a/salt/manager/nginx.sls
+++ b/salt/manager/nginx.sls
 nginx:
  service.running:
    - enable: True
-    - require:
+    - watch:
       - pkg: nginx
       - cmd: circlecert
       - file: nginxdefault
+       - file: nginx_home_permission
       {% if grains['os_family'] == 'RedHat' %}
       - file: nginxconf
+       - cmd: nginx_no_private_temp
       {% endif %}
  pkg:
   - installed
+nginx_home_permission:
+  file.directory:
+    - name: /home/{{ pillar['user'] }}
+    - user: {{ pillar['user'] }}
+    - dir_mode: 711
 circlecert:
  cmd.run:
    {% if grains['os_family'] == 'RedHat' %}
@@ -22,17 +30,50 @@ circlecert:
    - creates: /etc/ssl/certs/circle.pem
 {% if grains['os_family'] == 'RedHat' %}
-nginx_selinux:
+nginx_selinux_pkgs:
  pkg.installed:
    - pkgs:
      - policycoreutils
      - policycoreutils-python
+nginx_httpd_can_network_connect:
  selinux.boolean:
    - name: httpd_can_network_connect
    - value: True
    - persist: True
    - require:
-      - pkg: nginx_selinux
+      - pkg: nginx_selinux_pkgs
+nginx_httpd_read_user_content:
+  selinux.boolean:
+    - name: httpd_read_user_content
+    - value: True
+    - persist: True
+    - require:
+      - pkg: nginx_selinux_pkgs
+/root/nginx.te:
+  file.managed:
+    - source: salt://manager/files/nginx.te
+    - template: jinja
+    - mode: 644
+nginx_semodule:
+  cmd.run:
+    - cwd: /root
+    - user: root
+    - name: checkmodule -M -m -o nginx.mod nginx.te; semodule_package -o nginx.pp -m nginx.mod; semodule -i nginx.pp
+    - unless: semodule -l |grep -qs ^nginx
+    - require:
+      - file: /root/nginx.te
+      - pkg: nginx_selinux_pkgs
+nginx_no_private_temp:
+  cmd.run:
+    - user: root
+    - name: sed -i "/PrivateTmp/d" /usr/lib/systemd/system/nginx.service
+    - require:
+      - pkg: nginx
 {% endif %}
 nginxdefault:

--- a/salt/manager/pipeline.sls
+++ b/salt/manager/pipeline.sls
-{% if grains['os'] == 'Ubuntu' %}
+{% if grains['os'] == 'Ubuntu' or grains['os'] == 'Debian' %}
 nodejs-legacy:
  pkg.installed
 {% endif %}
 npm:
-  {% if grains['os'] == 'Ubuntu' %}
+  {% if grains['os'] == 'Ubuntu' or grains['os'] == 'Debian' %}
  pkg.installed:
    - require:
      - pkg: nodejs-legacy

--- a/salt/monitor-client/configuration.sls
+++ b/salt/monitor-client/configuration.sls
@@ -6,7 +6,7 @@
    - group: {{ pillar['user'] }}
    - mode: 700
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/monitor-client.service:
  file.managed:
    - user: root

--- a/salt/network/files/fix_dhcp.sh
+++ b/salt/network/files/fix_dhcp.sh
 #!/bin/bash
 sed -i '/HWADDR=.*/d' /etc/sysconfig/network-scripts/ifcfg-vm
 sed -i -e \$aNM_CONTROLLED=\"no\" /etc/sysconfig/network-scripts/ifcfg-vm
+/bin/systemctl daemon-reload
 ifup vm
 systemctl restart firewall
 systemctl restart dhcpd
 exit 0
--- a/salt/network/files/fix_dhcp_Debian.conf
+++ b/salt/network/files/fix_dhcp_Debian.conf
+# systemd service file extras added by CIRCLE Salt installer:
+# openvswitch and virtual network interface must be up before
+# dhcpd is started
+[Unit]
+After=openvswitch-switch.service
+[Service]
+ExecStartPre=-/sbin/ifup vm
+{# TODO: change 'vm' to pillar['fwdriver']['vm_if'] ? #}
+{# TODO: similar patch for firewall.service ? #}
--- a/salt/network/files/network
+++ b/salt/network/files/network
+NETWORKING_IPV6=yes
+IPV6FORWARDING=yes
--- a/salt/network/files/reload_firewall.sh
+++ b/salt/network/files/reload_firewall.sh
@@ -2,4 +2,4 @@
 source /home/{{ pillar['user'] }}/.virtualenvs/circle/bin/activate
 source /home/{{ pillar['user'] }}/.virtualenvs/circle/bin/postactivate
-python /home/{{ pillar['user'] }}/circle/circle/manage.py reload_firewall --sync
+python /home/{{ pillar['user'] }}/circle/circle/manage.py reload_firewall --sync --timeout={{ pillar['fwdriver']['reload_firewall_timeout'] }}
--- a/salt/network/init.sls
+++ b/salt/network/init.sls
@@ -13,12 +13,26 @@ vm:
    - pre_up_cmds:
      {% if grains['os_family'] == 'RedHat' %}
      - /bin/systemctl restart openvswitch
+      {% elif grains['os'] == 'Debian' %}
+      - /bin/systemctl restart openvswitch-switch
      {% else %}  
      - /etc/init.d/openvswitch-switch restart
      {% endif %} 
    - require:
      - cmd: ovs-if
+{% if grains['os'] == 'Debian' %}
+symlink_dhcpd:
+  file.symlink:
+    - name: /etc/init.d/dhcpd
+    - target: /etc/init.d/isc-dhcp-server
+    - force: True
+  cmd.run:
+    - name: /bin/systemctl daemon-reload
+    - require:
+      - file: symlink_dhcpd
+{% endif %}
 firewall2:
  service:
    - name: firewall
@@ -26,24 +40,75 @@ firewall2:
    - require:
      - network: vm
-salt://network/files/reload_firewall.sh:
+reload_firewall:
  cmd.script:
+    - name: salt://network/files/reload_firewall.sh
    - template: jinja
    - user: {{ pillar['user'] }}
    - require:
      - service: firewall2
+      {% if grains['os'] == 'Debian' %}
+      - cmd: symlink_dhcpd
+      {% endif %}
 {% if grains['os_family'] == 'RedHat' %}
-salt://network/files/fix_dhcp.sh:
+net_config:
-  cmd.script
+  file.managed:
+    - name: /etc/sysconfig/network
+    - source: salt://network/files/network
+    - user: root
+    - group: root
+    - mode: 644
+fix_dhcp:
+  cmd.script:
+    - name: salt://network/files/fix_dhcp.sh
+    - require:
+      - cmd: reload_firewall
+      - file: net_config
 {% endif %}
 isc-dhcp-server:
-  service:
+  {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
-    - running
+  cmd.run:
+    - name: /bin/systemctl restart dhcpd
    {% if grains['os_family'] == 'RedHat' %}
-    - name: dhcpd
+    - watch:
+      - cmd: fix_dhcp
+    {% elif grains['os'] == 'Debian' %}
+    - watch:
+      - cmd: fix_dhcp_daemon_reload
    {% endif %}
+  {% endif %}
+  service.running:
    - enable: True
-    - reload: True
+    {% if grains['os_family'] == 'RedHat' %}
+    - watch:
+      - cmd: fix_dhcp
+    {% elif grains['os'] == 'Debian' %}
+    - watch:
+      - cmd: fix_dhcp_daemon_reload
+    {% endif %}
+    {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
+    - name: dhcpd
+    - require:
+      - cmd: isc-dhcp-server
+    {% endif %}
+{% if grains['os'] == 'Debian' %}
+{# For next reboot #}
+after_openvswitch_conf:
+  file.managed:
+    - name: /etc/systemd/system/isc-dhcp-server.service.d/after_openvswitch.conf
+    - source: salt://network/files/fix_dhcp_Debian.conf
+    - user: root
+    - group: root
+    - template: jinja
+    - makedirs: True
+fix_dhcp_daemon_reload:
+  cmd.run:
+    - name: /bin/systemctl daemon-reload
+    - require:
+      - file: after_openvswitch_conf
+{% endif %}
--- a/salt/node.sls
+++ b/salt/node.sls
 include:
+    - profile
    - agentdriver
    - monitor-client
    - vmdriver

--- a/salt/profile.sls
+++ b/salt/profile.sls
+{% if grains['os']=='Debian' %}
+{# For non-interactive shells, virtualenvwrapper commands 
+   ('workon' etc.) are not sourced automatically #}
+/etc/profile:
+  file.append:
+    - text:
+      - "#Line below added for Debian by CIRCLE Salt installer"
+      - . /etc/bash_completion
+{% endif %}
--- a/salt/storagedriver/configuration.sls
+++ b/salt/storagedriver/configuration.sls
@@ -6,7 +6,7 @@
    - group: {{ pillar['user'] }}
    - mode: 700
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/storagecelery@.service:
  file.managed:
    - user: root

--- a/salt/vmdriver/configuration.sls
+++ b/salt/vmdriver/configuration.sls
@@ -9,9 +9,10 @@ include:
    - group: {{ pillar['user'] }}
    - mode: 700
-{% set service_dir = "/etc/systemd/system/" if grains['os_family'] == 'RedHat' else "/etc/init/" %}
+{% set service_dir = "/etc/systemd/system/" if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' else "/etc/init/" %}
 {% set service_files = (("vmcelery@.service", "netcelery@.service", "node.service")
-                        if grains['os_family'] == 'RedHat' else
+                        if grains['os_family'] == 'RedHat' 
+                        or grains['os'] == 'Debian' else
                        ("vmcelery.conf", "netcelery.conf", "node.conf")) %}
 {% for file in service_files %}

--- a/salt/vmdriver/files/org.libvirt.unix.manage.pkla
+++ b/salt/vmdriver/files/org.libvirt.unix.manage.pkla
+[Allow cloud libvirt management permissions]
+Identity=unix-user:cloud
+Action=org.libvirt.unix.manage;org.libvirt.unix.monitor
+ResultAny=yes
+ResultInactive=yes
+ResultActive=yes
+{# TODO: change 'cloud' to ? #}
--- a/salt/vmdriver/files/sudoers
+++ b/salt/vmdriver/files/sudoers
 {{ pillar['user'] }}	ALL = (ALL) NOPASSWD: /usr/bin/ovs-ofctl, /usr/bin/ovs-vsctl, /sbin/ip link set *
-Defaults: cloud !requiretty
+Defaults: {{ pillar['user'] }} !requiretty
--- a/salt/vmdriver/init.sls
+++ b/salt/vmdriver/init.sls
@@ -32,7 +32,10 @@ vmdriver:
      - libxslt1-dev
      - openvswitch-common
      - openvswitch-switch
+      {% if grains['os'] != 'Debian' %}
+      {# No such package in Debian Jessie! #}
      - openvswitch-controller
+      {% endif %}
      - python-dev
      - python-libvirt
      - virtualenvwrapper
@@ -41,7 +44,7 @@ vmdriver:
      {% endif %}
    - require_in:
      - file: /etc/default/libvirt-bin
-      {% if grains['os_family'] == 'RedHat' %}
+      {% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
      - service: libvirtd
      {% else %}
      - file: /etc/apparmor.d/libvirt/TEMPLATE

--- a/salt/vmdriver/libvirt.sls
+++ b/salt/vmdriver/libvirt.sls
@@ -15,7 +15,7 @@ libvirtconf:
  file.append:
    - text: libvirtd_opts="-d -l"
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 libvirtd:
 {% else %}
 libvirt-bin:
@@ -65,6 +65,32 @@ vmdriver_semodule:
      - file: /root/vmdriver.te
      - pkg: selinux_pkgs
+{% elif grains['os'] == 'Debian' %}
+/usr/bin/kvm:
+  file.replace:
+    - pattern: -enable-kvm
+    - repl: ""
+    - watch:
+      - pkg: vmdriver
+policycoreutils:
+  pkg.installed
+{# Note: Debian Jessie has polkit 0.105, which uses pkla format instead of js #}
+/etc/polkit-1/localauthority/50-local.d/org.libvirt.unix.manage.pkla:
+  file.managed:
+    - source: salt://vmdriver/files/org.libvirt.unix.manage.pkla
+    - user: root
+    - group: root
+    - template: jinja
+polkitd:
+  service:
+    - running
+    - watch:
+      - file: /etc/polkit-1/localauthority/50-local.d/org.libvirt.unix.manage.pkla
 {% else %}
 /etc/apparmor.d/libvirt/TEMPLATE:

--- a/salt/vmdriver/virtualenv.sls
+++ b/salt/vmdriver/virtualenv.sls
@@ -7,10 +7,14 @@ virtualenv_vmdriver:
 {% set libvirt_dir = "/usr/lib64/python2.7/site-packages/" if grains['os_family'] == 'RedHat' else "/usr/lib/python2.7/dist-packages/" %}
+{% set targets = { 'libvirtmod_qemu.so': 'libvirtmod_qemu.x86_64-linux-gnu.so',
+                   'libvirtmod.so': 'libvirtmod.x86_64-linux-gnu.so'
+                 } if grains['os'] == 'Debian' else {} %}
 {% for file in ("libvirtmod_qemu.so", "libvirtmod.so", "libvirt_qemu.py", "libvirt.py", "libvirt_qemu.pyc", "libvirt.pyc") %}
 /home/{{ pillar['user'] }}/.virtualenvs/vmdriver/lib/python2.7/site-packages/{{ file }}:
  file.symlink:
-    - target: {{ libvirt_dir + file }}
+    - target: {{ libvirt_dir + targets[file]|default(file) }}
    - require:
      - virtualenv: virtualenv_vmdriver
 {% endfor %}
--- a/salt/vncproxy/configuration.sls
+++ b/salt/vncproxy/configuration.sls
@@ -6,7 +6,7 @@
    - group: {{ pillar['user'] }}
    - mode: 700
-{% if grains['os_family'] == 'RedHat' %}
+{% if grains['os_family'] == 'RedHat' or grains['os'] == 'Debian' %}
 /etc/systemd/system/vncproxy.service:
  file.managed:
    - user: root