Commit 070de26b authored by Michael Loipführer's avatar Michael Loipführer
Browse files

move to python package with debian build

parent 66ec4648
Pipeline #331 failed with stages
in 20 seconds
.idea
*.env
__pycache__
containers.json
.mypy_cache
default:
image: debian-python-build:v2
# Is performed before the scripts in the stages step
before_script:
- source /etc/profile
# Load the ssh private key from the gitlab build variables to enable dupload
# to connect to the repo via scp
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- ssh-keyscan repo.stusta.mhn.de >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
# Defines stages which are to be executed
stages:
- build_buster
- upload_to_repo
# Stage "build_buster"
build_buster:
stage: build_buster
script:
- apt install python3-stdeb
- ./build.sh
- mkdir -p build/
- mv ../python3-lustmolch*.deb build/
- mv ../python3-lustmolch*.changes build/
- mv ../python3-lustmolch*.tar.gz build/
- mv ../python3-lustmolch*.dsc build/
- mv ../python3-lustmolch*.buildinfo build/
# The files which are to be made available in GitLab
artifacts:
paths:
- build/*
upload_to_repo:
stage: upload_to_repo
script:
- echo "Uploading debian package to ssn repository"
- ssh repo@repo.stusta.mhn.de "echo SSH-Connection working"
- dupload -f -t ssn_repo build/python3-lustmolch*.changes
when: manual
only:
- master
include README.md
include templates
\ No newline at end of file
#!/usr/bin/env sh
python3 setup.py --command-packages=stdeb.command bdist_deb
\ No newline at end of file
#!/bin/bash
mkdir -p /var/www/lustmolch
cp -rf www/* /var/www/lustmolch/
from . import cli
if __name__ == '__main__':
cli.cli()
import click
from lustmolch.config import DEFAULT_CONF_FILE, init_config
from lustmolch.lustmolch import (
list_containers,
add_user,
create_container,
remove_container,
remove_user,
update_containers
)
@click.group()
@click.option(
'--config-file',
default=DEFAULT_CONF_FILE,
help='Container configuration file')
def cli(config_file: str):
init_config(config_file)
@cli.command()
def list_containers():
list_containers()
@cli.command()
@click.option('--dry-run', is_flag=True, default=False)
@click.argument('name')
def create_container(dry_run: bool, name: str):
create_container(dry_run, name)
@cli.command()
@click.option('--key-string', is_flag=True, default=False)
@click.argument('name')
@click.argument('key')
def add_user(key_string: bool, name: str, key: str) -> None:
add_user(key_string, name, key)
@cli.command()
@click.argument('name')
def remove_user(name: str) -> None:
remove_user(name)
@cli.command()
@click.option('--dry-run', is_flag=True, default=False)
def update_containers(dry_run: bool) -> None:
update_containers(dry_run)
@cli.command()
@click.option('--dry-run', is_flag=True, default=False)
@click.argument('name')
def remove_container(dry_run: bool, name: str) -> None:
remove_container(dry_run, name)
import json
import logging
from pathlib import Path
from typing import Dict, Any, Optional
DEFAULT_TEMPLATE_DIR = '/srv/lustmolch-tools/templates'
DEFAULT_CONF_FILE = '/etc/ssn/lustmolch-containers.json'
DEFAULTS = {
'debian_flavour': 'buster',
'debian_mirror': 'http://mirror.stusta.de/debian',
'ssn_ip_ranges': ['10.150.0.0/17', '141.84.69.0/24'],
'www_root': '/var/www',
'ssh_start_port': 10022,
'ssh_port_increment': 1000,
'host_ip': '141.84.69.235',
'ip_start_host': (192, 168, 0, 1),
'ip_subnet_length': 30,
'log_level': logging.INFO
}
class Config:
def __init__(self, file_path: Path, cfg: Dict[str, Any]):
self.config = cfg
self.file_path = file_path
def get(self, key: str, default: Optional[Any] = None) -> Any:
if key not in self.config:
return DEFAULTS[key]
return self.config.get(key, default)
def set(self, key: str, value: Any):
self.config[key] = value
def __getitem__(self, key: str) -> Any:
return self.config[key]
def __setitem__(self, key: str, value: Any) -> Any:
self.config[key] = value
def __delitem__(self, key):
del self.config[key]
def save(self):
with self.file_path.open('w+') as f:
json.dump(self.config, f, indent=2)
@classmethod
def from_defaults(cls, file_name: str) -> 'Config':
cfg: Dict[str, Any] = {
'users': {},
'containers': {}
}
return cls(Path(file_name), cfg)
@classmethod
def from_file(cls, file_name: str) -> 'Config':
path = Path(file_name)
if not path.exists():
logging.info('Config file does not exist, generating defaults')
return cls.from_defaults(file_name)
with path.open('r') as f:
cfg = json.load(f)
return cls(path, cfg)
config = Config.from_file(DEFAULT_CONF_FILE)
def init_config(config_file: str) -> None:
global config
config = Config.from_file(config_file)
#!/usr/bin/env python3
import json
import logging
import shutil
from collections import namedtuple
from subprocess import run
from pathlib import Path
from subprocess import run
from typing import Tuple
from jinja2 import Environment, PackageLoader
import click
from .config import config, DEFAULT_TEMPLATE_DIR
logging.basicConfig(level=config['log_level'])
env = Environment(loader=PackageLoader('lustmolch', 'templates'))
env = Environment(loader=PackageLoader('lustmolch', str(Path(__file__).parent.parent / 'templates')))
cfg_template = namedtuple('cfg_template', ['source', 'path', 'filename'])
template_files_host = [
......@@ -24,94 +28,58 @@ template_files_container = [
nspawn_config = cfg_template('nspawn', Path(
'/etc/systemd/nspawn'), '{name}.nspawn')
DEFAULT_TEMPLATE_DIR = '/srv/lustmolch-tools/templates'
DEFAULT_CONF_FILE = '/etc/ssn/lustmolch-containers.json'
FLAVOUR = 'buster'
DEBIAN_MIRROR = 'http://mirror.stusta.de/debian'
SSN_IP_RANGES = ['10.150.0.0/17', '141.84.69.0/24']
www_root = Path('/var/www')
SSH_START_PORT = 10022
SSH_PORT_INCREMENT = 1000
IP_LUSTMOLCH = '141.84.69.235' # TODO: find out dynamically
IP_START_HOST = (192, 168, 0, 1)
IP_SUBNET_LENGTH = 30
def gen_default_conf():
return {
'users': {},
'containers': {}
}
def get_config(file_path):
if not Path(file_path).exists():
cfg = gen_default_conf()
else:
with open(file_path, 'r') as f:
cfg = json.load(f)
return cfg
def next_ssh_port(cfg, name):
def next_ssh_port(name: str) -> int:
"""
Return the next available port for the containers ssh server to run on.
If the container is already present in the list of installed containers
returns the configured port.
Args:
config_file: Path to container configuration file (containers.json)
name: Container name
Returns: SSH port
"""
if name in cfg['containers']:
return cfg['containers'][name].get('ssh_port')
if name in config['containers']:
return config['containers'][name].get('ssh_port')
port = SSH_START_PORT
for container in cfg['containers'].values():
port = config['ssh_start_port']
for container in config['containers'].values():
if container['ssh_port'] >= port:
port = container['ssh_port'] + SSH_PORT_INCREMENT
port = container['ssh_port'] + config['ssh_port_increment']
return port
def next_ip_address(cfg, name):
def next_ip_address(name: str) -> Tuple[str, str]:
"""
Return the next available (local) IP address to be assigned to the
container and the host interfaces.
Args:
config_file: Path to container configuration file (containers.json)
name: Container name
Returns (tuple): host_ip, container_ip
"""
if name in cfg['containers']:
c = cfg['containers'][name]
if name in config['containers']:
c = config['containers'][name]
return (c.get('ip_address_host').split('/')[0],
c.get('ip_address_container').split('/')[0])
ip_host = list(IP_START_HOST)
ip_host = list(config['ip_start_host'])
container_ips = [container['ip_address_host'].split('/')[0].split('.')
for container in cfg['containers'].values()]
container_ips = [container['ip_address_host'].split('/')[0].split('.')
for container in config['containers'].values()]
ip_host[2] = max([int(ip[2]) for ip in container_ips])
ip_host[3] = max([int(ip[3]) for ip in container_ips])
ip_host[3] += 4
if ip_host[3] >= 254:
ip_host[3] == 1
ip_host[3] = 1
ip_host[2] += 1
if ip_host[2] == 254:
click.echo('Error no available IP addresses found')
logging.error('Error no available IP addresses found')
raise Exception()
ip_container = list(ip_host)
......@@ -122,89 +90,56 @@ def next_ip_address(cfg, name):
str(x) for x in ip_container))
def update_config(config_file, container):
if not Path(config_file).exists():
with open(config_file, 'w+') as f:
cfg = gen_default_conf()
cfg['containers'][container['name']] = container
json.dump(cfg, f, indent=4)
else:
with open(config_file, 'r') as f:
cfg = json.load(f)
cfg['containers'][container['name']] = container
with open(config_file, 'w') as f:
json.dump(cfg, f, indent=4)
@click.group()
def cli():
pass
@cli.command()
@click.option(
'--config-file',
default=DEFAULT_CONF_FILE,
help='Container configuration file')
def list_containers(config_file):
def list_containers():
"""output lustmolch configuration file"""
cfg = get_config(config_file)
# TODO: make nice
click.echo('Currently registered containers:\n')
click.echo(json.dumps(cfg, indent=4))
print('Currently registered containers:\n')
print(json.dumps(config.config, indent=2))
@cli.command()
@click.option('--dry-run', is_flag=True, default=False)
@click.option(
'--config-file',
default=DEFAULT_CONF_FILE,
help='Container configuration file')
@click.argument('name')
def create_container(dry_run, config_file, name):
def create_container(dry_run, name):
"""Creates a systemd-nspawn container."""
if dry_run:
click.echo(f'Doing a dry run')
logging.info(f'Doing a dry run')
# create shared folder for html static files
www_dir = www_root / name
click.echo(f'Creating shared www directory "{www_dir}"')
www_dir = Path(config['www_root']) / name
logging.info(f'Creating shared www directory "{www_dir}"')
if not dry_run:
www_dir.mkdir(parents=True, exist_ok=True)
# place configuration files
cfg = get_config(config_file)
ip_address_host, ip_address_container = next_ip_address(cfg, name)
ssh_port = next_ssh_port(cfg, name)
ip_address_host, ip_address_container = next_ip_address(name)
ssh_port = next_ssh_port(name)
context = {
'name': name,
'ssh_port': ssh_port,
'ip_address_host': ip_address_host,
'ip_address_container': ip_address_container,
'ip_subnet_length': IP_SUBNET_LENGTH,
'ip_subnet_length': config['ip_subnet_lenght'],
'url': f'{name}.stusta.de',
'users': []
}
click.echo(f'Generated context values for container: {repr(context)}')
logging.info(f'Generated context values for container: {repr(context)}')
for cfg in template_files_host:
template = env.get_template('host/' + cfg.source)
file_name = cfg.path / (cfg.filename.format(**context))
click.echo(f'Placing config file {file_name}')
logging.info(f'Placing config file {file_name}')
if not dry_run:
with open(file_name, 'w+') as cfg_file:
cfg_file.write(template.render(context))
# create machine
machine_path = Path('/var/lib/machines', name)
click.echo('Running debootstrap')
logging.info('Running debootstrap')
if not dry_run:
run(['debootstrap', FLAVOUR, machine_path, DEBIAN_MIRROR],
run(['debootstrap', config['debian_flavour'], machine_path, config['debian_mirror']],
capture_output=True, check=True)
click.echo('Bootstrapping container')
logging.info('Bootstrapping container')
if not dry_run:
# copy and run bootstrap shell script
script_location = '/opt/bootstrap.sh'
......@@ -217,200 +152,127 @@ def create_container(dry_run, config_file, name):
run(['systemd-nspawn', '-D', str(machine_path), script_location],
check=True)
click.echo(f'Installing systemd-nspawn config for container {name}')
logging.info(f'Installing systemd-nspawn config for container {name}')
if not dry_run:
template = env.get_template('host/' + nspawn_config.source)
file_name = nspawn_config.path / \
(nspawn_config.filename.format(**context))
file_name = nspawn_config.path / (nspawn_config.filename.format(**context))
with open(file_name, 'w+') as cfg_file:
cfg_file.write(template.render(context))
click.echo('Copying config files into container')
logging.info('Copying config files into container')
if not dry_run:
for cfg in template_files_container:
template = env.get_template('container/' + cfg.source)
file_name = cfg.filename.format(**context)
click.echo(f'Placing config file {file_name}')
logging.info(f'Placing config file {file_name}')
if not dry_run:
with open(Path(f'{machine_path}{cfg.path}/{file_name}'), 'w+') as f:
f.write(template.render(context))
click.echo(f'Updating Iptable rules (filter, nat)')
logging.info(f'Updating Iptable rules (filter, nat)')
if not dry_run:
for ip_range in SSN_IP_RANGES:
for ip_range in config['ssn_ip_ranges']:
run(['iptables', '-A', 'INPUT', '-p', 'tcp', '-m', 'tcp',
'--dport', str(ssh_port), '-s', ip_range, '-j', 'ACCEPT'])
run(['iptables', '-t', 'nat', '-A', 'PREROUTING', '-p', 'tcp',
'-m', 'tcp', '--dport', str(
ssh_port), '-s', ip_range, '-j', 'DNAT',
ssh_port), '-s', ip_range, '-j', 'DNAT',
'--to-destination', f'{ip_address_container}:22'])
run(['iptables', '-t', 'nat', '-A', 'POSTROUTING', '-o', f've-{name}',
'-j', 'SNAT', '--to-source', IP_LUSTMOLCH])
'-j', 'SNAT', '--to-source', config['host_ip']])
click.echo('Starting container')
logging.info('Starting container')
if not dry_run:
run(['machinectl', 'start', name], capture_output=True, check=True)
click.echo('Updating container configuration file')
logging.info('Updating container configuration file')
if not dry_run:
update_config(config_file, container=context)
config.save()
click.echo(f'All done, ssh server running on port {ssh_port}\n'
'To finish please run "iptables-save".')
logging.info(f'All done, ssh server running on port {ssh_port}\n'
'To finish please run "iptables-save".')
@cli.command()
@click.option(
'--config-file',
default=DEFAULT_CONF_FILE,
help='Container configuration file')
@click.option('--key-string', is_flag=True, default=False)
@click.argument('name')
@click.argument('key')
def add_user(config_file, key_string, name, key):
def add_user(key_string: bool, name: str, key: str) -> None:
"""add user to lustmolch management"""
if key_string:
key_string = key
else:
if not key_string:
with open(key, 'r') as f:
key_string = f.read()
cfg = get_config(config_file)
key = f.read()
cfg['users'][name] = {
config['users'][name] = {
'name': name,
'key': key
}
with open(config_file, 'w') as f:
json.dump(cfg, f, indent=4)
config.save()
@cli.command()
@click.option(
'--config-file',
default=DEFAULT_CONF_FILE,
help='Container configuration file')
@click.argument('name')
def remove_user(config_file, name):
def remove_user(name: str) -> None:
"""remove a user, doesn't remove the user from all containers"""
cfg = get_config(config_file)
if name in cfg['users']:
del cfg['users'][name]
with open(config_file, 'w') as f:
json.dump(cfg, f, indent=4)
@cli.command()
@click.option('--dry-run', is_flag=True, default=False)
@click.option(
'--config-file',
default=DEFAULT_CONF_FILE,
help='Container configuration file')
def update_containers(dry_run, config_file):
if name in config['users']:
del config['users'][name]
config.save()
def update_containers(dry_run: bool) -> None:
"""update users on all containers"""
cfg = get_config(config_file)
for container in cfg['containers'].values():
for container in config['containers'].values():
ssh_dir = Path('/var/lib/machines', container['name'], 'root/.ssh')
authorized_keys = ssh_dir / 'authorized_keys'
keys = [user['key'] for user in cfg['users'].values() if user['name'] in container['users']]
keys = [user['key'] for user in config['users'].values() if user['name'] in container['users']]
keys = '\n'.join(keys)
click.echo(f'Writing\n{keys}\n to authorized key file {authorized_keys}')
logging.info(f'Writing\n{keys}\n to authorized key file {authorized_keys}')
if not dry_run:
ssh_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
authorized_keys.touch(mode=0o600, exist_ok=True)
authorized_keys.write_text(keys)
@cli.command()
@click.option(
'--config-file',
default=DEFAULT_CONF_FILE,
help='Container configuration file')
@click.option('--key-string', is_flag=True, default=False)
@click.argument('name')
@click.argument('key')
def install_ssh_key(config_file, key_string, name, key):
"""copy a ssh key into a containers authorized_keys"""
click.echo('DEPRECATED: Use add-user, update-contaienrs instead. Will be removed soon!')
ssh_dir = Path('/var/lib/machines', name, 'root/.ssh')
authorized_keys = ssh_dir / 'authorized_keys'
if key_string: