openbsd: add semi-finished openbsd support

agent.py

@@ -9,7 +9,7 @@ import sys
 
 system = platform.system()  # noqa
 
-if system == "Linux" or system == "FreeBSD":  # noqa
+if system in ["Linux", "FreeBSD", "OpenBSD"]:  # noqa
     try:  # noqa
         chdir(sys.path[0])  # noqa
         subprocess.call(('pip', 'install', '-r', 'requirements.txt'))  # noqa
@@ -85,7 +85,7 @@ class SerialLineReceiver(SerialLineReceiverBase):
                 "swap": dict(psutil.swap_memory()._asdict()),
                 "uptime": {"seconds": uptime.uptime()},
                 "disk": disk_usage,
-                "user": {"count": len(psutil.get_users())}}
+                "user": {"count": len(psutil.users())}}
         self.send_response(response='status', args=args)
         logger.debug("send_status finished")
 

bootstrap-openbsd.sh

+#!/bin/sh
+
+#export LOGLEVEL=DEBUG
+
+pkg_add -z `cat bootstrap/openbsd/pkg-requirements.txt`
+
+# Necessary symlinks
+ln -sf /usr/local/bin/pip2.7 /usr/local/bin/pip
+#ln -sf /usr/local/bin/python2.7 /usr/local/bin/python
+#ln -sf /usr/local/bin/python2.7-2to3 /usr/local/bin/2to3
+#ln -sf /usr/local/bin/python2.7-config /usr/local/bin/python-config
+#ln -sf /usr/local/bin/pydoc2.7  /usr/local/bin/pydoc
+
+
+grep "^cloud:" /etc/passwd > /dev/null
+ret=$?
+if [ $ret -ne 0 ]
+then
+	# create the required backdoor user
+	groupadd cloud
+	useradd -g cloud -G wheel -m cloud
+fi
+
+#sed -i '.orig' -e 's/# \(%wheel ALL=(ALL) ALL\)/\1/g' /usr/local/etc/sudoers
+
+mkdir -p /store
+mkdir -p /etc/circle
+
+if [ ! -d /root/agent ]
+then
+	cd /root
+	git clone https://git.ik.bme.hu/circle/agent.git agent
+fi
+
+cp bootstrap/openbsd/usr/sbin/mount_circle-userstore /usr/sbin/mount_circle-userstore
+cp bootstrap/openbsd/rc.d/agent /etc/rc.d/agent
+
+rcctl enable agent
+rcctl start agent

bootstrap/openbsd/pkg-requirements.txt

+git
+python-2.7
+py-pip

bootstrap/openbsd/rc.d/agent

+#!/bin/ksh
+#
+# $OpenBSD: agent $
+
+daemon="/usr/local/bin/python2.7 /root/agent/agent.py"
+
+. /etc/rc.d/rc.subr
+
+rc_cmd $1

bootstrap/openbsd/usr/sbin/mount_circle-userstore

+#!/bin/sh
+
+if [ -eq x"$@" x"-o rw none /store" ]; then
+	/usr/local/bin/usmb -c /etc/circle/usmb.conf userstore
+else:
+	echo "It only works with following fstab line:"
+	echo "none /store circle-userstore rw 0 0"
+fi

context.py

@@ -29,6 +29,8 @@ def get_context():
         from linux._linuxcontext import Context
     elif system == "FreeBSD":
         from freebsd._freebsdcontext import Context
+    elif system == "OpenBSD":
+        from openbsd._openbsdcontext import Context
     else:
         raise NotImplementedError("Platform %s is not supported.", system)
     return Context
@@ -64,6 +66,15 @@ def get_serial():
             port = '/dev/ttyu0'
         else:
             from freebsd.posixvirtio import SerialPort
+    elif system == "OpenBSD":
+        port = "/dev/ttyVI00"
+        try:
+            open(port, 'rw').close()
+        except (OSError, IOError):
+            from twisted.internet.serialport import SerialPort
+            port = '/dev/tty00'
+        else:
+            from openbsd.posixvirtio import SerialPort
     else:
         raise NotImplementedError("Platform %s is not supported.", system)
     return (SerialPort, port)

openbsd/FEATURES

+serial:		ok
+ipv4:		ok
+ipv6:		ok
+hostname:	?
+time:		ok
+smb:		ok
+passwd update:	?
+get-ssh-keys:	ok

openbsd/_openbsdcontext.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from os import mkdir, environ, chmod
+from shutil import rmtree, move
+import subprocess
+import sys
+import pwd
+import logging
+import fileinput
+import tarfile
+from os.path import expanduser, join, exists
+from glob import glob
+from StringIO import StringIO
+from base64 import decodestring
+from hashlib import md5
+
+
+from ssh import PubKey
+from .network import change_ip_openbsd
+from context import BaseContext
+
+from twisted.internet import reactor
+import jinja2
+
+
+logging.basicConfig()
+logger = logging.getLogger()
+level = environ.get('LOGLEVEL', 'INFO')
+logger.setLevel(level)
+
+SSH_DIR = expanduser('~cloud/.ssh')
+AUTHORIZED_KEYS = join(SSH_DIR, 'authorized_keys')
+
+STORE_DIR = '/store'
+
+USMB_CONF = '/etc/circle/usmb.conf'
+
+MOUNT_TEMPLATE_OPENBSD = "none /store circle-userstore rw 0 0\n"
+
+
+def render_template_to_file(template_name, filename, context):
+    with open(filename, 'w') as f:
+        with open(template_name) as tf:
+            template = jinja2.Template(''.join(tf.readlines()))
+            template.stream(context).dump(f)
+
+
+class Context(BaseContext):
+
+    # http://stackoverflow.com/questions/12081310/
+    # python-module-to-change-system-date-and-time
+    @staticmethod
+    def _openbsd_set_time(time):
+        import ctypes
+        import ctypes.util
+
+        CLOCK_REALTIME = 0
+
+        class timespec(ctypes.Structure):
+            _fields_ = [('tv_sec', ctypes.c_long),
+                        ('tv_nsec', ctypes.c_long)]
+
+        libc = ctypes.CDLL(ctypes.util.find_library('libc'))
+
+        ts = timespec()
+        ts.tv_sec = int(time)
+        ts.tv_nsec = 0
+
+        libc.clock_settime(CLOCK_REALTIME, ctypes.byref(ts))
+
+    @staticmethod
+    def change_password(password):
+        # TODO: root password
+        enc_proc = subprocess.Popen(
+            ['/usr/bin/encrypt'],
+            stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        enc_password, _ = enc_proc.communicate('%s\n' % password)
+        subprocess.Popen(
+            ['/usr/sbin/usermod', '-p', enc_password.strip(), 'cloud'])
+
+    @staticmethod
+    def restart_networking():
+        logger.debug("restart_networking")
+
+    @staticmethod
+    def change_ip(interfaces, dns):
+        nameservers = dns.replace(' ', '').split(',')
+        change_ip_openbsd(interfaces, nameservers)
+
+    @staticmethod
+    def set_time(time):
+        Context._openbsd_set_time(float(time))
+        try:
+            subprocess.call(['rcctl' 'restart', 'ntpd'])
+        except Exception:
+            pass
+
+    @staticmethod
+    def set_hostname(hostname):
+        with open('/etc/myname', 'w') as f:
+            f.write(hostname)
+
+        with open('/etc/hosts', 'w') as f:
+            f.write("127.0.0.1 localhost\n"
+                    "127.0.1.1 %s\n" % hostname)
+
+        subprocess.call(['/bin/hostname', hostname])
+
+    @staticmethod
+    def mount_store(host, username, password):
+        uid = pwd.getpwnam('cloud').pw_uid
+        data = {'host': host, 'username': username,
+                'password': password, 'uid': uid}
+        data['dir'] = STORE_DIR
+        if not exists(STORE_DIR):
+            mkdir(STORE_DIR)
+
+        render_template_to_file('openbsd/usmb.conf', USMB_CONF, data)
+        chmod(USMB_CONF, 0600)
+
+        for line in fileinput.input('/etc/fstab', inplace=True):
+            if not (line.startswith('none') and ' circle-userstore ' in line):
+                print line.rstrip()
+
+        with open('/etc/fstab', 'a') as f:
+            f.write(MOUNT_TEMPLATE_OPENBSD % data)
+
+        subprocess.call('/sbin/mount /store', shell=True)
+
+    @staticmethod
+    def get_keys():
+        retval = []
+        try:
+            with open(AUTHORIZED_KEYS, 'r') as f:
+                for line in f.readlines():
+                    try:
+                        retval.append(PubKey.from_str(line))
+                    except Exception:
+                        logger.exception(u'Invalid ssh key: ')
+        except IOError:
+            pass
+        return retval
+
+    @staticmethod
+    def _save_keys(keys):
+        print keys
+        try:
+            mkdir(SSH_DIR)
+        except OSError:
+            pass
+        with open(AUTHORIZED_KEYS, 'w') as f:
+            for key in keys:
+                f.write(unicode(key) + '\n')
+
+    @staticmethod
+    def add_keys(keys):
+        new_keys = Context.get_keys()
+        for key in keys:
+            try:
+                p = PubKey.from_str(key)
+                if p not in new_keys:
+                    new_keys.append(p)
+            except Exception:
+                logger.exception(u'Invalid ssh key: ')
+        Context._save_keys(new_keys)
+
+    @staticmethod
+    def del_keys(keys):
+        new_keys = Context.get_keys()
+        for key in keys:
+            try:
+                p = PubKey.from_str(key)
+                try:
+                    new_keys.remove(p)
+                except ValueError:
+                    pass
+            except Exception:
+                logger.exception(u'Invalid ssh key: ')
+        Context._save_keys(new_keys)
+
+    @staticmethod
+    def cleanup():
+        filelist = ([
+            '/root/.bash_history',
+            '/home/cloud/.bash_history',
+            '/root/.ssh',
+            '/home/cloud/.ssh',
+            '/root/.lesshst',
+            '/home/cloud/.lesshst',
+            '/root/.history',
+            '/home/cloud/.history',
+            '/root/.viminfo',
+            '/home/cloud/.viminfo',
+            USMB_CONF,
+            ]
+            + glob('/etc/ssh/ssh_host_*'))
+        for f in filelist:
+            rmtree(f, ignore_errors=True)
+
+        subprocess.call(('/usr/bin/ssh-keygen', '-A'))
+
+    @staticmethod
+    def start_access_server():
+        subprocess.call(('/usr/sbin/rcctl', 'start', 'sshd'))
+
+    @staticmethod
+    def append(data, filename, chunk_number, uuid):
+        if chunk_number == 0:
+            flag = "w"
+        else:
+            flag = "a"
+        with open(filename, flag) as myfile:
+            myfile.write(data)
+
+    @staticmethod
+    def update(filename, executable, checksum, uuid):
+        working_directory = sys.path[0]
+        new_dir = working_directory + '.new'
+        old_dir = working_directory + '.old.%s' % uuid
+        with open(filename, "r") as f:
+            data = f.read()
+            local_checksum = md5(data).hexdigest()
+            if local_checksum != checksum:
+                raise Exception("Checksum missmatch the file is damaged.")
+            decoded = StringIO(decodestring(data))
+        try:
+            tar = tarfile.TarFile.open("dummy", fileobj=decoded, mode='r|gz')
+            tar.extractall(new_dir)
+        except tarfile.ReadError as e:
+            logger.error(e)
+        move(working_directory, old_dir)
+        move(new_dir, working_directory)
+        logger.info("Transfer completed!")
+        reactor.stop()
+
+    @staticmethod
+    def ipaddresses():
+        import netifaces
+        args = {}
+        interfaces = netifaces.interfaces()
+        for i in interfaces:
+            if i == 'lo0':
+                continue
+            args[i] = []
+            addresses = netifaces.ifaddresses(i)
+            args[i] = ([x['addr']
+                        for x in addresses.get(netifaces.AF_INET, [])] +
+                       [x['addr']
+                        for x in addresses.get(netifaces.AF_INET6, [])
+                        if '%' not in x['addr']])
+        return args
+
+    @staticmethod
+    def get_agent_version():
+        try:
+            with open('version.txt') as f:
+                return f.readline()
+        except IOError:
+            return None
+
+    @staticmethod
+    def send_expiration(url):
+        import notify
+        notify.notify(url)

openbsd/network.py

+import netifaces
+from netaddr import IPNetwork
+import logging
+import subprocess
+import os
+import os.path
+
+
+logger = logging.getLogger()
+
+IF_CONF_PREFIX = '/etc/hostname.'
+
+
+def get_interfaces_openbsd(interfaces):
+    for ifname in netifaces.interfaces():
+        if ifname == 'lo0':
+            continue
+        logger.debug("get_interfaces: " + ifname)
+        netif = netifaces.ifaddresses(ifname)
+        conf = None
+        if netif:
+            mac_list = netif.get(netifaces.AF_LINK)
+            if mac_list:
+                mac = mac_list[0]['addr']
+                logger.debug("get_interfaces: " + mac)
+                conf = interfaces.get(mac.upper())
+        if conf:
+            yield ifname, conf
+
+
+def remove_interfaces_openbsd(devices):
+    for device in devices:
+        if_file = IF_CONF_PREFIX + device
+        subprocess.call(('/sbin/ifconfig', device, '-inet', '-inet6'))
+        subprocess.call(('/sbin/ifconfig', device, 'destroy'))
+        if os.path.isfile(if_file):
+            logger.debug("remove interface configuration: " + if_file)
+            os.unlink(if_file)
+        else:
+            logger.debug(
+                "unable to remove interface configuration: " + if_file)
+
+
+def change_ip_openbsd(interfaces, nameservers):
+    data = list(get_interfaces_openbsd(interfaces))
+    remove_interfaces_openbsd(dict(data).keys())
+
+    # Set IPs
+    for device, conf in data:
+        if_file = IF_CONF_PREFIX + device
+        with open(if_file, 'w') as f:
+            ipv4_alias_counter = ipv6_alias_counter = 0
+            for i in conf['addresses']:
+                ip = IPNetwork(i)
+                alias = ''
+                typ = 'inet' if ip.version == 4 else 'inet6'
+                if ip.version == 6 and ipv6_alias_counter > 0:
+                    alias = 'alias'
+                    ipv6_alias_counter += 1
+                if ip.version == 4 and ipv4_alias_counter > 0:
+                    alias = 'alias'
+                    ipv4_alias_counter += 1
+                f.write('%(type)s %(ip)s %(alias)s\n'
+                        % {
+                            'type': typ,
+                            'ip': ip,
+                            'alias': alias
+                        })
+        os.chmod(if_file, 0540)
+
+    # Set default gateway from last config
+    if len(data) > 0:
+        device, conf = data[len(data) - 1]
+        with open('/etc/mygate', 'w') as f:
+            f.write(conf['gw4'] + '\n')
+            f.write(conf['gw6'] + '\n')
+
+    # Set nameservers
+    with open('/etc/resolv.conf', 'w') as f:
+        for ns in nameservers:
+            f.write('nameserver %s\n' % ns)
+
+    # Restart network
+    subprocess.call(('/bin/sh', '/etc/netstart'))
+
+
+# example:
+# change_ip_ubuntu({
+#    u'02:00:00:02:A3:E8': {
+#        u'gw4': u'10.1.0.254', 'gw6': '2001::ffff',
+#        u'addresses': [u'10.1.0.84/24', '10.1.0.1/24', '2001::1/48']},
+#    u'02:00:00:02:A3:E9': {
+#        u'gw4': u'10.255.255.1', u'addresses': [u'10.255.255.9']}},
+#    '8.8.8.8')

openbsd/posixvirtio.py

+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+
+"""
+Virtio-Serial Port Protocol
+"""
+
+# system imports
+import os
+
+# dependent on pyserial ( http://pyserial.sf.net/ )
+# only tested w/ 1.18 (5 Dec 2002)
+
+# twisted imports
+from twisted.internet import abstract, fdesc
+
+
+class SerialPort(abstract.FileDescriptor):
+    """
+    A select()able serial device, acting as a transport.
+    """
+
+    connected = 1
+
+    def __init__(self, protocol, deviceNameOrPortNumber, reactor):
+        abstract.FileDescriptor.__init__(self, reactor)
+        self.port = deviceNameOrPortNumber
+        self._serial = os.open(
+            self.port, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
+        self.reactor = reactor
+        self.protocol = protocol
+        self.protocol.makeConnection(self)
+        self.startReading()
+
+    def fileno(self):
+        return self._serial
+
+    def writeSomeData(self, data):
+        """
+        Write some data to the serial device.
+        """
+        return fdesc.writeToFD(self.fileno(), data)
+
+    def doRead(self):
+        """
+        Some data's readable from serial device.
+        """
+        return fdesc.readFromFD(self.fileno(), self.protocol.dataReceived)
+
+    def connectionLost(self, reason):
+        """
+        Called when the serial port disconnects.
+
+        Will call C{connectionLost} on the protocol that is handling the
+        serial data.
+        """
+        abstract.FileDescriptor.connectionLost(self, reason)
+        os.close(self._serial)
+        self.protocol.connectionLost(reason)

openbsd/ssh.py

+from base64 import decodestring
+from struct import unpack
+import binascii
+import unittest
+
+
+class InvalidKeyType(Exception):
+    pass
+
+
+class InvalidKey(Exception):
+    pass
+
+
+class PubKey(object):
+
+    key_types = ('ssh-rsa', 'ssh-dsa', 'ssh-ecdsa')
+
+    # http://stackoverflow.com/questions/2494450/ssh-rsa-public-key-
+    # validation-using-a-regular-expression
+    @classmethod
+    def validate_key(cls, key_type, key):
+        try:
+            data = decodestring(key)
+        except binascii.Error:
+            raise InvalidKey()
+        int_len = 4
+        str_len = unpack('>I', data[:int_len])[0]
+        if data[int_len:int_len + str_len] != key_type:
+            raise InvalidKey()
+
+    def __init__(self, key_type, key, comment):
+        if key_type not in self.key_types:
+            raise InvalidKeyType()
+        self.key_type = key_type
+
+        PubKey.validate_key(key_type, key)
+        self.key = key
+
+        self.comment = unicode(comment)
+
+    def __hash__(self):
+        return hash(frozenset(self.__dict__.items()))
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__
+
+    @classmethod
+    def from_str(cls, line):
+        key_type, key, comment = line.split()
+        return PubKey(key_type, key, comment)
+
+    def __unicode__(self):
+        return u' '.join((self.key_type, self.key, self.comment))
+
+    def __repr__(self):
+        return u'<PubKey: %s>' % unicode(self)
+
+
+# Unit tests
+
+class SshTestCase(unittest.TestCase):
+    def setUp(self):
+        self.p1 = PubKey.from_str('ssh-rsa AAAAB3NzaC1yc2EA comment')
+        self.p2 = PubKey.from_str('ssh-rsa AAAAB3NzaC1yc2EA comment')
+        self.p3 = PubKey.from_str('ssh-rsa AAAAB3NzaC1yc2EC comment')
+
+    def test_invalid_key_type(self):
+        self.assertRaises(InvalidKeyType, PubKey, 'ssh-inv', 'x', 'comment')
+
+    def test_valid_key(self):
+        PubKey('ssh-rsa', 'AAAAB3NzaC1yc2EA', 'comment')
+
+    def test_invalid_key(self):
+        self.assertRaises(InvalidKey, PubKey, 'ssh-rsa', 'x', 'comment')
+
+    def test_invalid_key2(self):
+        self.assertRaises(InvalidKey, PubKey, 'ssh-rsa',
+                          'AAAAB3MzaC1yc2EA', 'comment')
+
+    def test_repr(self):
+        p = PubKey('ssh-rsa', 'AAAAB3NzaC1yc2EA', 'comment')
+        self.assertEqual(
+            repr(p), '<PubKey: ssh-rsa AAAAB3NzaC1yc2EA comment>')
+
+    def test_unicode(self):
+        p = PubKey('ssh-rsa', 'AAAAB3NzaC1yc2EA', 'comment')
+        self.assertEqual(unicode(p), 'ssh-rsa AAAAB3NzaC1yc2EA comment')
+
+    def test_from_str(self):
+        p = PubKey.from_str('ssh-rsa AAAAB3NzaC1yc2EA comment')
+        self.assertEqual(unicode(p), 'ssh-rsa AAAAB3NzaC1yc2EA comment')
+
+    def test_eq(self):
+        self.assertEqual(self.p1, self.p2)
+        self.assertNotEqual(self.p1, self.p3)
+
+    def test_hash(self):
+        s = set()
+        s.add(self.p1)
+        s.add(self.p2)
+        s.add(self.p3)
+        self.assertEqual(len(s), 2)
+
+
+if __name__ == '__main__':
+    unittest.main()

openbsd/usmb.conf

+<?xml version="1.0" encoding="UTF-8"?>
+<usmbconfig>
+
+  <credentials id="cloud">
+    <!--<domain>CLOUD</domain>-->
+    <username>{{ username }}</username>
+    <password>{{ password }}</password>
+  </credentials>
+
+  <mount id="userstore" credentials="cloud">
+    <server>{{ host }}</server>
+    <share>{{ username }}</share>
+    <mountpoint>{{ dir }}</mountpoint>
+    <options>uid={{ uid }},nonempty</options>
+  </mount>
+
+</usmbconfig>

requirements.txt

 Twisted==13.2.0
 pyserial==2.7
-psutil==1.2.1
+psutil==5.4.8
 uptime==3.0.1
 netifaces==0.10.4
 netaddr==0.7.12
+Jinja2==2.10
 infi.devicemanager
 tzlocal
 pytz