move to python package with debian build

.gitignore

+.idea
 *.env
 __pycache__
 containers.json
+.mypy_cache

.gitlab-ci.yml

+default:
+  image: debian-python-build:v2
+
+# Is performed before the scripts in the stages step
+before_script:
+  - source /etc/profile
+  # Load the ssh private key from the gitlab build variables to enable dupload
+  # to connect to the repo via scp
+  - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+  - eval $(ssh-agent -s)
+  - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
+  - mkdir -p ~/.ssh
+  - chmod 700 ~/.ssh
+  - ssh-keyscan repo.stusta.mhn.de >> ~/.ssh/known_hosts
+  - chmod 644 ~/.ssh/known_hosts
+
+# Defines stages which are to be executed
+stages:
+  - build_buster
+  - upload_to_repo
+
+# Stage "build_buster"
+build_buster:
+  stage: build_buster
+  script:
+    - apt install python3-stdeb
+    - ./build.sh
+    - mkdir -p build/
+    - mv ../python3-lustmolch*.deb build/
+    - mv ../python3-lustmolch*.changes build/
+    - mv ../python3-lustmolch*.tar.gz build/
+    - mv ../python3-lustmolch*.dsc build/
+    - mv ../python3-lustmolch*.buildinfo build/
+
+  # The files which are to be made available in GitLab
+  artifacts:
+    paths:
+      - build/*
+
+upload_to_repo:
+  stage: upload_to_repo
+  script:
+    - echo "Uploading debian package to ssn repository"
+    - ssh repo@repo.stusta.mhn.de "echo SSH-Connection working"
+    - dupload -f -t ssn_repo build/python3-lustmolch*.changes
+  when: manual
+  only:
+    - master

MANIFEST.in

+include README.md
+include templates
\ No newline at end of file

build.sh

+#!/usr/bin/env sh
+
+python3 setup.py --command-packages=stdeb.command bdist_deb
\ No newline at end of file

deploy.sh

-#!/bin/bash
-
-mkdir -p /var/www/lustmolch
-cp -rf www/* /var/www/lustmolch/ 

lustmolch/__main__.py

+from . import cli
+
+if __name__ == '__main__':
+    cli.cli()

lustmolch/cli.py

+import click
+
+from lustmolch.config import DEFAULT_CONF_FILE, init_config
+from lustmolch.lustmolch import (
+    list_containers,
+    add_user,
+    create_container,
+    remove_container,
+    remove_user,
+    update_containers
+)
+
+
+@click.group()
+@click.option(
+    '--config-file',
+    default=DEFAULT_CONF_FILE,
+    help='Container configuration file')
+def cli(config_file: str):
+    init_config(config_file)
+
+
+@cli.command()
+def list_containers():
+    list_containers()
+
+
+@cli.command()
+@click.option('--dry-run', is_flag=True, default=False)
+@click.argument('name')
+def create_container(dry_run: bool, name: str):
+    create_container(dry_run, name)
+
+
+@cli.command()
+@click.option('--key-string', is_flag=True, default=False)
+@click.argument('name')
+@click.argument('key')
+def add_user(key_string: bool, name: str, key: str) -> None:
+    add_user(key_string, name, key)
+
+
+@cli.command()
+@click.argument('name')
+def remove_user(name: str) -> None:
+    remove_user(name)
+
+
+@cli.command()
+@click.option('--dry-run', is_flag=True, default=False)
+def update_containers(dry_run: bool) -> None:
+    update_containers(dry_run)
+
+
+@cli.command()
+@click.option('--dry-run', is_flag=True, default=False)
+@click.argument('name')
+def remove_container(dry_run: bool, name: str) -> None:
+    remove_container(dry_run, name)

lustmolch/config.py

+import json
+import logging
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+DEFAULT_TEMPLATE_DIR = '/srv/lustmolch-tools/templates'
+DEFAULT_CONF_FILE = '/etc/ssn/lustmolch-containers.json'
+
+DEFAULTS = {
+    'debian_flavour': 'buster',
+    'debian_mirror': 'http://mirror.stusta.de/debian',
+    'ssn_ip_ranges': ['10.150.0.0/17', '141.84.69.0/24'],
+    'www_root': '/var/www',
+    'ssh_start_port': 10022,
+    'ssh_port_increment': 1000,
+    'host_ip': '141.84.69.235',
+    'ip_start_host': (192, 168, 0, 1),
+    'ip_subnet_length': 30,
+    'log_level': logging.INFO
+}
+
+
+class Config:
+    def __init__(self, file_path: Path, cfg: Dict[str, Any]):
+        self.config = cfg
+        self.file_path = file_path
+
+    def get(self, key: str, default: Optional[Any] = None) -> Any:
+        if key not in self.config:
+            return DEFAULTS[key]
+
+        return self.config.get(key, default)
+
+    def set(self, key: str, value: Any):
+        self.config[key] = value
+
+    def __getitem__(self, key: str) -> Any:
+        return self.config[key]
+
+    def __setitem__(self, key: str, value: Any) -> Any:
+        self.config[key] = value
+
+    def __delitem__(self, key):
+        del self.config[key]
+
+    def save(self):
+        with self.file_path.open('w+') as f:
+            json.dump(self.config, f, indent=2)
+
+    @classmethod
+    def from_defaults(cls, file_name: str) -> 'Config':
+        cfg: Dict[str, Any] = {
+            'users': {},
+            'containers': {}
+        }
+
+        return cls(Path(file_name), cfg)
+
+    @classmethod
+    def from_file(cls, file_name: str) -> 'Config':
+        path = Path(file_name)
+        if not path.exists():
+            logging.info('Config file does not exist, generating defaults')
+            return cls.from_defaults(file_name)
+
+        with path.open('r') as f:
+            cfg = json.load(f)
+
+        return cls(path, cfg)
+
+
+config = Config.from_file(DEFAULT_CONF_FILE)
+
+
+def init_config(config_file: str) -> None:
+    global config
+    config = Config.from_file(config_file)

lustmolch.py → lustmolch/lustmolch.py

-#!/usr/bin/env python3
 import json
+import logging
 import shutil
 from collections import namedtuple
-from subprocess import run
 from pathlib import Path
+from subprocess import run
+from typing import Tuple
+
 from jinja2 import Environment, PackageLoader
-import click
 
+from .config import config, DEFAULT_TEMPLATE_DIR
+
+logging.basicConfig(level=config['log_level'])
 
-env = Environment(loader=PackageLoader('lustmolch', 'templates'))
+env = Environment(loader=PackageLoader('lustmolch', str(Path(__file__).parent.parent / 'templates')))
 cfg_template = namedtuple('cfg_template', ['source', 'path', 'filename'])
 
 template_files_host = [
@@ -24,94 +28,58 @@ template_files_container = [
 nspawn_config = cfg_template('nspawn', Path(
     '/etc/systemd/nspawn'), '{name}.nspawn')
 
-DEFAULT_TEMPLATE_DIR = '/srv/lustmolch-tools/templates'
-DEFAULT_CONF_FILE = '/etc/ssn/lustmolch-containers.json'
-
-FLAVOUR = 'buster'
-DEBIAN_MIRROR = 'http://mirror.stusta.de/debian'
-
-SSN_IP_RANGES = ['10.150.0.0/17', '141.84.69.0/24']
-www_root = Path('/var/www')
-
-SSH_START_PORT = 10022
-SSH_PORT_INCREMENT = 1000
-
-IP_LUSTMOLCH = '141.84.69.235'  # TODO: find out dynamically
-
-IP_START_HOST = (192, 168, 0, 1)
-IP_SUBNET_LENGTH = 30
-
-
-def gen_default_conf():
-    return {
-        'users': {},
-        'containers': {}
-    }
-
-
-def get_config(file_path):
-    if not Path(file_path).exists():
-        cfg = gen_default_conf()
-    else:
-        with open(file_path, 'r') as f:
-            cfg = json.load(f)
-
-    return cfg
 
-
-def next_ssh_port(cfg, name):
+def next_ssh_port(name: str) -> int:
     """
     Return the next available port for the containers ssh server to run on.
     If the container is already present in the list of installed containers
     returns the configured port.
     Args:
-        config_file: Path to container configuration file (containers.json)
         name: Container name
 
     Returns: SSH port
     """
 
-    if name in cfg['containers']:
-        return cfg['containers'][name].get('ssh_port')
+    if name in config['containers']:
+        return config['containers'][name].get('ssh_port')
 
-    port = SSH_START_PORT
-    for container in cfg['containers'].values():
+    port = config['ssh_start_port']
+    for container in config['containers'].values():
         if container['ssh_port'] >= port:
-            port = container['ssh_port'] + SSH_PORT_INCREMENT
+            port = container['ssh_port'] + config['ssh_port_increment']
 
     return port
 
 
-def next_ip_address(cfg, name):
+def next_ip_address(name: str) -> Tuple[str, str]:
     """
     Return the next available (local) IP address to be assigned to the
     container and the host interfaces.
     Args:
-        config_file: Path to container configuration file (containers.json)
         name: Container name
 
     Returns (tuple): host_ip, container_ip
     """
 
-    if name in cfg['containers']:
-        c = cfg['containers'][name]
+    if name in config['containers']:
+        c = config['containers'][name]
         return (c.get('ip_address_host').split('/')[0],
                 c.get('ip_address_container').split('/')[0])
 
-    ip_host = list(IP_START_HOST)
+    ip_host = list(config['ip_start_host'])
 
-    container_ips = [container['ip_address_host'].split('/')[0].split('.') 
-            for container in cfg['containers'].values()]
+    container_ips = [container['ip_address_host'].split('/')[0].split('.')
+                     for container in config['containers'].values()]
 
     ip_host[2] = max([int(ip[2]) for ip in container_ips])
     ip_host[3] = max([int(ip[3]) for ip in container_ips])
     ip_host[3] += 4
 
     if ip_host[3] >= 254:
-        ip_host[3] == 1
+        ip_host[3] = 1
         ip_host[2] += 1
     if ip_host[2] == 254:
-        click.echo('Error no available IP addresses found')
+        logging.error('Error no available IP addresses found')
         raise Exception()
 
     ip_container = list(ip_host)
@@ -122,89 +90,56 @@ def next_ip_address(cfg, name):
             str(x) for x in ip_container))
 
 
-def update_config(config_file, container):
-    if not Path(config_file).exists():
-        with open(config_file, 'w+') as f:
-            cfg = gen_default_conf()
-            cfg['containers'][container['name']] = container
-            json.dump(cfg, f, indent=4)
-    else:
-        with open(config_file, 'r') as f:
-            cfg = json.load(f)
-            cfg['containers'][container['name']] = container
-
-        with open(config_file, 'w') as f:
-            json.dump(cfg, f, indent=4)
-
-
-@click.group()
-def cli():
-    pass
-
-
-@cli.command()
-@click.option(
-    '--config-file',
-    default=DEFAULT_CONF_FILE,
-    help='Container configuration file')
-def list_containers(config_file):
+def list_containers():
     """output lustmolch configuration file"""
-    cfg = get_config(config_file)
+    # TODO: make nice
 
-    click.echo('Currently registered containers:\n')
-    click.echo(json.dumps(cfg, indent=4))
+    print('Currently registered containers:\n')
+    print(json.dumps(config.config, indent=2))
 
 
-@cli.command()
-@click.option('--dry-run', is_flag=True, default=False)
-@click.option(
-    '--config-file',
-    default=DEFAULT_CONF_FILE,
-    help='Container configuration file')
-@click.argument('name')
-def create_container(dry_run, config_file, name):
+def create_container(dry_run, name):
     """Creates a systemd-nspawn container."""
     if dry_run:
-        click.echo(f'Doing a dry run')
+        logging.info(f'Doing a dry run')
 
     # create shared folder for html static files
-    www_dir = www_root / name
-    click.echo(f'Creating shared www directory "{www_dir}"')
+    www_dir = Path(config['www_root']) / name
+    logging.info(f'Creating shared www directory "{www_dir}"')
     if not dry_run:
         www_dir.mkdir(parents=True, exist_ok=True)
 
     # place configuration files
-    cfg = get_config(config_file)
-    ip_address_host, ip_address_container = next_ip_address(cfg, name)
-    ssh_port = next_ssh_port(cfg, name)
+    ip_address_host, ip_address_container = next_ip_address(name)
+    ssh_port = next_ssh_port(name)
     context = {
         'name': name,
         'ssh_port': ssh_port,
         'ip_address_host': ip_address_host,
         'ip_address_container': ip_address_container,
-        'ip_subnet_length': IP_SUBNET_LENGTH,
+        'ip_subnet_length': config['ip_subnet_lenght'],
         'url': f'{name}.stusta.de',
         'users': []
     }
 
-    click.echo(f'Generated context values for container: {repr(context)}')
+    logging.info(f'Generated context values for container: {repr(context)}')
 
     for cfg in template_files_host:
         template = env.get_template('host/' + cfg.source)
         file_name = cfg.path / (cfg.filename.format(**context))
-        click.echo(f'Placing config file {file_name}')
+        logging.info(f'Placing config file {file_name}')
         if not dry_run:
             with open(file_name, 'w+') as cfg_file:
                 cfg_file.write(template.render(context))
 
     # create machine
     machine_path = Path('/var/lib/machines', name)
-    click.echo('Running debootstrap')
+    logging.info('Running debootstrap')
     if not dry_run:
-        run(['debootstrap', FLAVOUR, machine_path, DEBIAN_MIRROR],
+        run(['debootstrap', config['debian_flavour'], machine_path, config['debian_mirror']],
             capture_output=True, check=True)
 
-    click.echo('Bootstrapping container')
+    logging.info('Bootstrapping container')
     if not dry_run:
         # copy and run bootstrap shell script
         script_location = '/opt/bootstrap.sh'
@@ -217,200 +152,127 @@ def create_container(dry_run, config_file, name):
         run(['systemd-nspawn', '-D', str(machine_path), script_location],
             check=True)
 
-    click.echo(f'Installing systemd-nspawn config for container {name}')
+    logging.info(f'Installing systemd-nspawn config for container {name}')
     if not dry_run:
         template = env.get_template('host/' + nspawn_config.source)
-        file_name = nspawn_config.path / \
-            (nspawn_config.filename.format(**context))
+        file_name = nspawn_config.path / (nspawn_config.filename.format(**context))
         with open(file_name, 'w+') as cfg_file:
             cfg_file.write(template.render(context))
 
-    click.echo('Copying config files into container')
+    logging.info('Copying config files into container')
     if not dry_run:
         for cfg in template_files_container:
             template = env.get_template('container/' + cfg.source)
             file_name = cfg.filename.format(**context)
-            click.echo(f'Placing config file {file_name}')
+            logging.info(f'Placing config file {file_name}')
             if not dry_run:
                 with open(Path(f'{machine_path}{cfg.path}/{file_name}'), 'w+') as f:
                     f.write(template.render(context))
 
-    click.echo(f'Updating Iptable rules (filter, nat)')
+    logging.info(f'Updating Iptable rules (filter, nat)')
     if not dry_run:
-        for ip_range in SSN_IP_RANGES:
+        for ip_range in config['ssn_ip_ranges']:
             run(['iptables', '-A', 'INPUT', '-p', 'tcp', '-m', 'tcp',
                  '--dport', str(ssh_port), '-s', ip_range, '-j', 'ACCEPT'])
             run(['iptables', '-t', 'nat', '-A', 'PREROUTING', '-p', 'tcp',
                  '-m', 'tcp', '--dport', str(
-                     ssh_port), '-s', ip_range, '-j', 'DNAT',
+                    ssh_port), '-s', ip_range, '-j', 'DNAT',
                  '--to-destination', f'{ip_address_container}:22'])
         run(['iptables', '-t', 'nat', '-A', 'POSTROUTING', '-o', f've-{name}',
-             '-j', 'SNAT', '--to-source', IP_LUSTMOLCH])
+             '-j', 'SNAT', '--to-source', config['host_ip']])
 
-    click.echo('Starting container')
+    logging.info('Starting container')
     if not dry_run:
         run(['machinectl', 'start', name], capture_output=True, check=True)
 
-    click.echo('Updating container configuration file')
+    logging.info('Updating container configuration file')
     if not dry_run:
-        update_config(config_file, container=context)
+        config.save()
 
-    click.echo(f'All done, ssh server running on port {ssh_port}\n'
-               'To finish please run "iptables-save".')
+    logging.info(f'All done, ssh server running on port {ssh_port}\n'
+                 'To finish please run "iptables-save".')
 
 
-@cli.command()
-@click.option(
-    '--config-file',
-    default=DEFAULT_CONF_FILE,
-    help='Container configuration file')
-@click.option('--key-string', is_flag=True, default=False)
-@click.argument('name')
-@click.argument('key')
-def add_user(config_file, key_string, name, key):
+def add_user(key_string: bool, name: str, key: str) -> None:
     """add user to lustmolch management"""
-    if key_string:
-        key_string = key
-    else:
+    if not key_string:
         with open(key, 'r') as f:
-            key_string = f.read()
-    cfg = get_config(config_file)
+            key = f.read()
 
-    cfg['users'][name] = {
+    config['users'][name] = {
         'name': name,
         'key': key
     }
 
-    with open(config_file, 'w') as f:
-        json.dump(cfg, f, indent=4)
+    config.save()
 
 
-@cli.command()
-@click.option(
-    '--config-file',
-    default=DEFAULT_CONF_FILE,
-    help='Container configuration file')
-@click.argument('name')
-def remove_user(config_file, name):
+def remove_user(name: str) -> None:
     """remove a user, doesn't remove the user from all containers"""
-    cfg = get_config(config_file)
-    if name in cfg['users']:
-        del cfg['users'][name]
-    
-    with open(config_file, 'w') as f:
-        json.dump(cfg, f, indent=4)
-
-
-@cli.command()
-@click.option('--dry-run', is_flag=True, default=False)
-@click.option(
-    '--config-file',
-    default=DEFAULT_CONF_FILE,
-    help='Container configuration file')
-def update_containers(dry_run, config_file):
+    if name in config['users']:
+        del config['users'][name]
+
+    config.save()
+
+
+def update_containers(dry_run: bool) -> None:
     """update users on all containers"""
-    cfg = get_config(config_file)
 
-    for container in cfg['containers'].values():
+    for container in config['containers'].values():
         ssh_dir = Path('/var/lib/machines', container['name'], 'root/.ssh')
         authorized_keys = ssh_dir / 'authorized_keys'
-       
-        keys = [user['key'] for user in cfg['users'].values() if user['name'] in container['users']]
+
+        keys = [user['key'] for user in config['users'].values() if user['name'] in container['users']]
         keys = '\n'.join(keys)
 
-        click.echo(f'Writing\n{keys}\n to authorized key file {authorized_keys}')
+        logging.info(f'Writing\n{keys}\n to authorized key file {authorized_keys}')
         if not dry_run:
             ssh_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
             authorized_keys.touch(mode=0o600, exist_ok=True)
             authorized_keys.write_text(keys)
 
 
-@cli.command()
-@click.option(
-    '--config-file',
-    default=DEFAULT_CONF_FILE,
-    help='Container configuration file')
-@click.option('--key-string', is_flag=True, default=False)
-@click.argument('name')
-@click.argument('key')
-def install_ssh_key(config_file, key_string, name, key):
-    """copy a ssh key into a containers authorized_keys"""
-    click.echo('DEPRECATED: Use add-user, update-contaienrs instead. Will be removed soon!')
-    ssh_dir = Path('/var/lib/machines', name, 'root/.ssh')
-    authorized_keys = ssh_dir / 'authorized_keys'
-    if key_string: