tester

#!/usr/bin/env python3
# vim: expandtab shiftwidth=4 softtabstop=4

### CTF-specific parameters ###

FLAG_RE = r"OOO{[^}]*}\Z"                   # "normal" flag format
SERVICE_NAME_RE = r"[a-z][a-z0-9-]+\Z"      # so it's the same in Kubernetes, DNS, git, ...
PUBLIC_FILENAME_RE = r"[a-zA-Z0-9_.@-]+\Z"  # mostly for sanity in setting env vars & co.
NAME_PREFIX = "dc2021q"                     # for github, image names, etc.
INTERNAL_SUBDOMAIN = ".proxy.31337.ooo"        # for test_deployed (internal by default)
PUBLIC_SUBDOMAIN = ".challenges.ooo"        # for test_deployed public
GITHUB_ORG = "o-o-overflow"                 # Used to enforce the repo name -- we found it useful to have the same name everywhere
K8S_MASTER = 'master.31337.ooo'    # Used to delete pods on './tester deploy'
POW_DETECTION_STRING = b"(Pssst, http://oooverflow.io/pow.py)"   # If detected, will test on port+1 (which is expected to have no POW req.)
REGISTRY = "registry.31337.ooo:5000"

# A JSON updated via github webhooks -- this allows the tester to live in copy in each repo, and yet be kept up to date with the template
SELFUPDATE_INFO_URL = "https://jacopo.cc/ooo_latest_tester.json"

SHORTREAD_ALLOWED_DIFF = 2    # Accept this number of remaining processes after the short-read test (override: shortread_allowed_diff: -1)



import concurrent.futures
import urllib.request
import subprocess
import argparse
import hashlib
import logging
import tarfile
import socket
import shlex
import json
import yaml
import time
import sys
import re
import os
import traceback


def in_github_ci():
    return os.getenv('GITHUB_ACTIONS') == 'true' and os.getenv('GITHUB_WORKFLOW')

if in_github_ci():
    class GithubLogFormatter(logging.Formatter):
        def format(self, r):
            lvl = "error" if r.levelno>=logging.ERROR else ("warning" if r.levelno>=logging.WARNING else None)
            if lvl and not getattr(r, "no_github_ci", False):
                return "::{} file={},line={}::{}".format(lvl, r.pathname, r.lineno, r.getMessage())
            return super().format(r)  #.getMessage()
    _LOG = logging.getLogger("OOO")
    _LOG.setLevel("DEBUG")
    handler = logging.StreamHandler()
    handler.setLevel("DEBUG")
    handler.setFormatter(GithubLogFormatter())
    _LOG.addHandler(handler)
else:
    logging.basicConfig()
    _LOG = logging.getLogger("OOO")
    _LOG.setLevel("DEBUG")
    try:
        import coloredlogs
        coloredlogs.install(logger=_LOG, level=_LOG.level)
    except ImportError:
        pass



service_dir = os.path.dirname(__file__)
git_desc = "didnt_ask_git"
git_clean = False


dsystem = os.system  # But see cmdline options
def system_without_stdout(cmd):
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    out, _ = p.communicate()
    if p.returncode != 0:
        _LOG.warning("Command %s failed (%d). Stdout was: %s", cmd, p.returncode, out)
    return p.returncode


def file_sha256(filename):
    with open(filename, "rb") as mf:
        return hashlib.sha256(mf.read()).hexdigest().lower()


def grep_for_exposed_port():
    # NOTE: should match the logic in chalmanager
    service_Dockerfile = os.path.join(service_dir, 'service', 'Dockerfile')
    expose_grep = subprocess.check_output(['egrep','-i','^[[:space:]]*expose',service_Dockerfile]).strip()
    assert len(expose_grep.splitlines()) == 1, "More than one EXPOSE line in the service Dockerfile? Found: {}".format(expose_grep)
    m = re.match(br'\s*EXPOSE\s*([0-9]+)(/(tcp|udp))?', expose_grep, re.I)
    assert m, "I didn't understand the expose statement in the service Dockerfile ('{}')".format(expose_grep)
    ret = int(m.group(1))
    assert ret != 22
    return ret


def update_myself():
    if not SELFUPDATE_INFO_URL:
        return
    _LOG.info("Checking for tester updates...")
    my_sha256 = file_sha256(__file__)
    try:
        with urllib.request.urlopen(SELFUPDATE_INFO_URL, timeout=5) as u:
            newinfo = json.loads(u.read().decode("utf-8"))
        if newinfo['tester_sha256'].lower() == my_sha256 or newinfo.get("ignore", False):
            return
        _LOG.critical("THERE IS A NEW tester SCRIPT (or yours is different) -- please ./apply_template_commits.sh")
        if 'commit_id' in newinfo:
            _LOG.info("The last upstream tester commit was %s", newinfo['commit_id'])
        if 'commit_unixdate' in newinfo:
            _LOG.info("The last upstream tester commit happened at " +
                    time.strftime('%a, %d %b %Y %H:%M:%S %Z', time.localtime(newinfo['commit_unixdate'])))
        if in_github_ci() or ('CI' in os.environ) or not (sys.stdin.isatty() and sys.stdout.isatty() and sys.stderr.isatty()):
            _LOG.warning("Not interactive, I won't ask to upgrade")
            return
        if input("Continue without updating? ") in ['y','Y','yes','YES','Yes']:
            _LOG.info("OK. BTW, you can run me with --no-self-update if you don't want this check to happen.")
            return
        print("Good choice :) Plase run ./apply_template_commits.sh")
        print("In a pinch, run with --no-self-update")
        sys.exit(11)
    except Exception as e:
        _LOG.warning("Skipping self-update, got an exception: %s %s", type(e), str(e))


def validate_git():  # Called as part of validate_yaml
    global git_desc, git_clean
    try:
        git_desc = subprocess.check_output(['git','-P','-C',service_dir,
            'describe','--long','--always','--dirty'], universal_newlines=True).strip()
        _LOG.info("git description: %s", git_desc)
        if git_desc.endswith('dirty'):
            _LOG.info("The git tree is dirty -- that's fine locally, shouldn't happen in the infra")
            assert not in_github_ci()
        else:
            git_clean = True
    except subprocess.CalledProcessError as e:
        _LOG.warning('git-describe failed! Are you in git? exit code=%d stdout=%s',
                e.returncode, e.output)
        git_desc = "git_describe_errored"
    if not GITHUB_ORG:
        _LOG.debug("No GITHUB_ORG specified, skipping the repo name check")
        return
    if not os.path.isdir(os.path.join(service_dir,'.github','workflows')):
        _LOG.critical("No github test? Just copy the .github dir from the template")
    if not os.path.exists(os.path.join(service_dir, ".git")):
        _LOG.warning("I don't think I am in git -- skipping the github name check")
        return
    remotes_p = subprocess.run("git -P -C {} remote -v show -n|grep push|grep -oE '\S+github.com\S+'|grep '{}'".format(
            shlex.quote(service_dir), GITHUB_ORG),
            shell=True, stdout=subprocess.PIPE, cwd=service_dir, universal_newlines=True)
    if remotes_p.returncode != 0 or not remotes_p.stdout:
        _LOG.warning("I don't see github among the remotes -- skipping the github name check")
        return
    def remote_ok(remote):
        if ('2020' in remote) or ('2019' in remote) or ('2018' in remote):
            _LOG.debug("Probably an old repo, skipping github repo name check")
            return True
        if remote.endswith('.git'):
            remote = remote[:-4]
        return remote.endswith(GITHUB_ORG + '/' + NAME_PREFIX + "-" + service_conf['service_name'])
    if not any(remote_ok(r) for r in remotes_p.stdout.splitlines()):
        _LOG.critical("The github repo name does not conform to the format! I'm expecting %s-(service_name as in yaml) Either change the name or ask around. Remotes found: %s", NAME_PREFIX, remotes_p.stdout)
        if not service_conf.get('violates_github_format'):
            assert False, "you violated the github name format!"


def get_healthcheck_info():
    hc = [ k for k in service_conf.keys() if k.startswith('healthcheck_') ]
    tcp_send = None
    if 'healthcheck_tcp_send' in hc:
        tcp_send = service_conf['healthcheck_tcp_send'].encode('ascii','strict')
        hc.remove('healthcheck_tcp_send')
    if not hc: return None
    assert len(hc) == 1, "More than one healthcheck_xxx line?!?"
    protocol = hc[0][len('healthcheck_'):]
    if protocol not in ('tcp','http'):
        _LOG.critical("Are you sure protocol %s is supported for healthchecks?", protocol)
    rgx = ""
    if service_conf[hc[0]]:
        rgx = service_conf[hc[0]].encode('ascii','strict')  # TODO: matches reality?
    return protocol, re.compile(rgx), tcp_send

def simulate_healthcheck(protocol, regex, tcp_send, host, port):
    # Ideally match https://github.com/prometheus/blackbox_exporter
    # + infa's roles/blackbox_exporter/templates/config.yml.j2
    _LOG.info("Simulating a %s healthcheck %s:%d -> regex %s", protocol, host, port, repr(regex))
    if protocol not in ("tcp","http"):
        _LOG.warning("TODO: missing %s healthcheck simulation", protocol)
        return None
    try:
        if protocol == 'http':
            assert tcp_send is None
            with urllib.request.urlopen('http://{}:{}/'.format(host,port), timeout=5) as u:
                if u.getcode() != 200:
                    _LOG.critical('Got %d %s [!= 200] for %s (info: %s)',
                            u.getcode(), u.reason, u.geturl(), u.info())
                else:
                    _LOG.debug('Got %d %s for %s',
                            u.getcode(), u.reason, u.geturl())
                rdata = u.read()
        else:
            with socket.create_connection((host,port), timeout=5) as c:
                c.settimeout(5)
                if tcp_send is not None:
                    _LOG.debug("Sending %s ...", tcp_send.decode('ascii','backslashreplace'))
                    c.sendall(tcp_send)
                if regex.pattern:  # Empty healthcheck_tcp => just try connecting
                    rdata = c.recv(1024)  # TODO: loop over received lines instead
                    _LOG.debug("TCP healthcheck received: %s", rdata.decode('ascii','backslashreplace'))
        if regex.pattern:  # Empty healthcheck_tcp => just try connecting
            rdata_msgstr = rdata.decode('ascii','backslashreplace')
            m = regex.search(rdata)
            if m:
                _LOG.debug("Matched: %s", str(m))
            else:
                _LOG.error("Simulated healthcheck failed -- received %s (didn't match %s)", rdata_msgstr, repr(regex))
                return False
        _LOG.info("Simulated healthcheck passed, good!")
        return True
    except Exception as e:
        _LOG.critical("Got an exception while simulating a healthcheck on (%s:%d) -> %s %s", host, port, type(e), str(e))



def validate_game_network_info():  # Called as part of validate_yaml
    if 'game_network_info' not in service_conf:
        _LOG.warning("game_network_info not specified: THIS SHOULD ONLY HAPPEN FOR OFFLINE SERVICES")
        _LOG.debug("^^^ If that's wrong just copy the defaults from the template")
        assert get_healthcheck_info() is None, "Can't have healthchecks if offline!"
        return None, None
    assert "host" in service_conf["game_network_info"], "Missing game_network_info.hostname -- should normally be 'default'"
    host = service_conf["game_network_info"]["host"]
    assert "port" in service_conf["game_network_info"]
    port = service_conf["game_network_info"]["port"]
    if "hidden" in service_conf["game_network_info"]:
        assert service_conf["game_network_info"]["hidden"]
        _LOG.debug("The public description will NOT include the hostname and port")
    else:
        _LOG.debug("The public description will automatically include the hostname and port")
    if port == "guess":
        port = grep_for_exposed_port()
        _LOG.info("Guessed port for your service: %d", grep_for_exposed_port())
    else:
        port = int(port) # Put 'guess' if you want us to grep for EXPOSE
        assert port != 22
    if host == "default":
        _LOG.debug("You'll be using the default deployment -- good. Remember to ./tester test_deployed")
        hc = get_healthcheck_info()
        if not hc:
            _LOG.warning("Your service has no healthcheck -- this should happen only if offline or custom-deployed")
        else:
            _LOG.debug("You have suggested as healthcheck: %s", str(get_healthcheck_info()))
        return host,port
    assert get_healthcheck_info() is None, "Can't have healthchecks if custom-deployed! If using our infrastructure, put host: default"
    try:
        ip = socket.gethostbyname(host)
        _LOG.debug("Your custom host %s resolves to %s", host, ip)
        c = socket.create_connection((host,port), timeout=5)
        _LOG.info("Good, I TCP-connected to your custom %s:%d", host, port)
        c.close()
    except Exception as e:
        _LOG.critical("Got an exception while trying to TCP-connect to your custom game_network_info (%s:%d) -> %s %s", host, port, type(e), str(e))
    return host,port


def validate_yaml():
    _LOG.info("Validating yaml...")
    assert 'service_name' in service_conf, "no service name specified"
    if 'template' in service_conf['service_name'] and not os.path.abspath(service_dir).endswith('template'):
        _LOG.critical("Looks like you didn't change the service_name from the template (it's: %s)", service_conf['service_name'])
        assert False, "you must change the service_name"
    if not re.match(SERVICE_NAME_RE, service_conf['service_name']):
        _LOG.critical("Service name %s is unusual, will create issues with docker & co. -- can you change it? Regex: %s", service_conf['service_name'], SERVICE_NAME_RE)
        if not service_conf['violates_name_format']:
            assert False, "you violated the name format! Either change the name or ask around"
    validate_git()
    assert 'flag' in service_conf, "no service flag specified"
    if 'test flag' in service_conf['flag']:
        _LOG.critical("REMEMBER TO CHANGE THE FLAG: %s looks like the test flag", service_conf['flag'])
    if not re.match(FLAG_RE, service_conf['flag']):
        _LOG.critical("FLAG %s DOES NOT CONFORM TO THE FLAG FORMAT", service_conf['flag'])
        if not service_conf['violates_flag_format']:
            assert False, "you violated the flag format!"
    validate_game_network_info()

    if 'request_memory' in service_conf:
        assert re.match(r"[0-9]+[bkmg]\Z", service_conf['request_memory']), "request_memory example: 512m"
    else:
        _LOG.error("You did not specify request_memory -- defaulting to 512 MB during tests")
    if 'limit_memory' in service_conf:
        assert re.match(r"[0-9]+[bkmg]\Z", service_conf['limit_memory']), "limit_memory example: 512m"
    else:
        _LOG.error("You did not specify limit_memory -- defaulting to 512 MB during tests")

    assert service_conf['type'] in ('normal','jeopardy') # 'king_of_the_hill'


    for k in service_conf.keys():
        kregex = "^" + re.escape(k) + ":"
        ymlpath = os.path.join(service_dir, "info.yml")
        kgrepc = subprocess.check_output(['egrep','-c',kregex, ymlpath], universal_newlines=True)
        if int(kgrepc) != 1:
            _LOG.critical("Looks like '%s' is present more than once in the info.yml!", k)
            subprocess.run(['egrep',kregex, ymlpath])
            sys.exit(4)


def docker_label_arg():
    # TODO: hash of the public files?
    ret = ""
    ret += " --label built-local="+shlex.quote(time.strftime('%a, %d %b %Y %H:%M:%S %Z'))
    ret += " --label built-utc="+shlex.quote(time.strftime('%Y-%m-%d %H:%M:%S %Z',time.gmtime()))
    ret += " --label built-unixtime="+shlex.quote(str(time.time()))
    ret += " --label built-on-hostname="+shlex.quote(socket.gethostname())
    ret += " --label git-desc="+shlex.quote(git_desc)
    return ret


def get_docker_img_git_desc(tag):
    p = subprocess.run(['docker','inspect','--format','{{ index .Config.Labels "git-desc"}}',tag],
            universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
    if p.returncode != 0 or not p.stdout:
        return None  # Missing image or label
    return p.stdout.strip()

def image_matches_git(tag, unclean_ok=False):
    assert git_desc != 'didnt_ask_git'
    img_git_desc = get_docker_img_git_desc(tag)
    if img_git_desc is None: return None  # Never built or missing label
    return (git_desc == img_git_desc) and (git_clean or unclean_ok)

def warn_on_old_image(tag):
    ok = image_matches_git(tag)
    if ok:
        _LOG.debug("Local docker image %s is up-to-date with the current git commit (%s), good.", tag, git_desc)
        return
    if ok is None:
        _LOG.debug("I'm not sure if docker image %s is up-to-date or not, never built?", tag)
        return
    loc = git_desc
    im = get_docker_img_git_desc(tag)
    if loc and loc.endswith('dirty'): loc = loc[:-5]
    if im and im.endswith('dirty'): im = im[:-5]
    if git_clean or (loc != im):  # Certain
        _LOG.critical("The %s image you have is outdated! (local git %s != img %s)",
                tag, git_desc, get_docker_img_git_desc(tag))
        return
    _LOG.warning("Careful, the %s image you have may be outdated", tag)  # Unsure (dirty git)

def warn_on_old_images():
    warn_on_old_image(image_tag)
    warn_on_old_image(interaction_image_tag)


def check_container_own_log(ignore_missing=False):
    if ignore_missing and subprocess.call(['docker','inspect',container_tag], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) != 0:
        return
    o = subprocess.check_output(["docker","logs",container_tag]).decode('utf-8','replace')
    for l in o.splitlines():
        ll = l.lower()
        if 'error' in ll or 'crit' in ll or 'alert' in ll or 'emerg' in ll:
            # XXX: exit with failure?
            _LOG.error("From the container: %s", l)
        elif 'warn' in ll:
            _LOG.warning("From the container: %s", l)
    if os.getenv('SHOW_DOCKER_CONTAINER_LOGS') == "1":
        _LOG.debug("container's own log: %s", o)


def build_service():
    if os.path.exists(os.path.join(service_dir, "service", "Dockerfile")):
        if in_github_ci():
            print('::group::Docker-build the service container', flush=True)
        _LOG.info("Building service image...")
        build_args = ""
        if service_conf.get('copy_flag_using_build_arg'):
            build_args = "--build-arg THE_FLAG='%s'" % service_conf["flag"]
        assert dsystem("docker build %s %s -t %s %s/service" % (build_args, docker_label_arg(), image_tag, service_dir)) == 0, "service docker image build failed"
        if in_github_ci():
            print('::endgroup::', flush=True)
        assert image_matches_git(image_tag, unclean_ok=True)
    else:
        _LOG.warning("no dockerfile found for service...")

def build_interactions():
    if os.path.exists(os.path.join(service_dir, "interaction", "Dockerfile")):
        if in_github_ci():
            print('::group::Docker-build the interaction container', flush=True)
        _LOG.info("Building interaction image...")
        assert dsystem("docker build %s -t %s %s/interaction" % (docker_label_arg(), interaction_image_tag, service_dir)) == 0, "interaction docker image build failed"
        if in_github_ci():
            print('::endgroup::', flush=True)
        assert image_matches_git(interaction_image_tag, unclean_ok=True)

def _start_container():
    _stop_container()
    limits = " -m " + str(service_conf.get('limit_memory', '512m'))
    limits += " --memory-reservation " + str(service_conf.get('request_memory', '512m'))
    # -t (pseudo-tty) so xinetd stderr log works (uses the real terminal otherwise!)
    assert dsystem("docker run %s --name %s --rm -t -d --oom-score-adj=500 --log-driver local --log-opt mode=non-blocking %s" % (limits, container_tag, image_tag)) == 0, "service container failed to start"
def _stop_container():
    check_container_own_log(ignore_missing=True)
    dsystem("docker kill %s 2>/dev/null >/dev/null | true" % container_tag)
    dsystem("docker rm %s 2>/dev/null >/dev/null | true" % container_tag)

def launch_service():
    _LOG.debug("starting container")
    _start_container()
    time.sleep(1)
    container_config = json.loads(subprocess.check_output("docker inspect %s" % container_tag, shell=True).decode('utf-8'))
    ip_address = container_config[0]['NetworkSettings']['Networks']['bridge']['IPAddress']
    port = list(container_config[0]['Config']['ExposedPorts'].keys())[0].split("/")[0]
    _LOG.debug("network endpoint: %s:%s", ip_address, port)
    check_container_own_log()
    return ip_address, port

def test_service(ip_address, port, exploit=True, host_net=False):
    _LOG.info("Testing container...")

    _LOG.info("launching interaction container")
    dsystem("docker kill %s 2>/dev/null >/dev/null | true" % interaction_container_tag)
    dsystem("docker rm %s 2>/dev/null >/dev/null | true" % interaction_container_tag)
    interaction_container = subprocess.check_output("docker run %s --name %s --rm --oom-score-adj=600 -i -d %s" %
            ("--net=host" if host_net else "", interaction_container_tag, interaction_image_tag), shell=True).strip().decode('utf-8')

    def _run_interaction(script):
        if os.path.basename(script).startswith("exploit") and not exploit:
            _LOG.info("skipping exploit script %s", script)
            return
        _LOG.info("launching %s", script)

        try:
            output = subprocess.check_output("docker exec %s %s %s %s" % (interaction_container, script, ip_address, port), shell=True,
                    stderr=None if _LOG.getEffectiveLevel() < logging.WARNING else subprocess.STDOUT)
            _LOG.debug("%s -out-> %s", script, output)
        except subprocess.CalledProcessError as e:
            _LOG.critical("ERROR: %s returned %d -- full cmd: %s output: %s", script, e.returncode, e.cmd, e.output)
            if os.path.basename(script).startswith("exploit"):
                if service_conf['flag'].encode('utf-8') in e.output:
                    _LOG.critical("Despite the error, the exploit did actually print the flag, so... kinda worked?")
            sys.exit(e.returncode)

        if os.path.basename(script).startswith("exploit"):
            _LOG.info("... checking for the flag")
            assert service_conf['flag'].encode('utf-8') in output, "exploit %s did not print the flag"%script #pylint:disable=unsupported-membership-test

    _LOG.info("launching interaction scripts")
    interaction_files = service_conf['interactions']
    for f in interaction_files:
        _run_interaction(f)

    _LOG.info("STRESS TEST TIME")
    n = 2
    old_level = _LOG.level
    while n <= service_conf['concurrent_connections']:
        _LOG.info("stress testing with %d concurrent connections!", n)
        _LOG.setLevel(max(logging.WARNING, old_level))
        with concurrent.futures.ThreadPoolExecutor(max_workers=n) as pool:
            results = pool.map(_run_interaction, (interaction_files*n)[:n])
        try:
            for result in results:
                pass
        except Exception as e:
            _LOG.error('One iteration returns an exception: %s' % str(e))
            _LOG.error(traceback.format_exc())
            sys.exit(1)

        _LOG.setLevel(old_level)

        n *= 2

    _LOG.info("SHORT-READ SANITY CHECK")
    allowed = service_conf.get('shortread_allowed_diff', SHORTREAD_ALLOWED_DIFF)
    if SHORTREAD_ALLOWED_DIFF >= 0 and allowed >= 0:
        start_num_procs = len(subprocess.check_output("docker exec %s ps aux" % container_tag, shell=True).splitlines())
        assert os.system('docker run --rm ubuntu bash -ec "for i in {1..128}; do echo > /dev/tcp/%s/%s; done"' % (ip_address, port)) == 0
        _LOG.info("waiting for service to clean up after short reads")
        time.sleep(15)
        final_num_procs = len(subprocess.check_output("docker exec %s ps aux" % container_tag, shell=True).splitlines())
        assert final_num_procs < (start_num_procs + allowed), "your service did not clean up after short reads -- starting procs = {sp} final={fp}".format(sp=start_num_procs, fp=final_num_procs)
    else:
        _LOG.info("The short-read test is disabled")

    _LOG.info("stopping interaction container")
    dsystem("docker kill %s" % interaction_container)

    hck = get_healthcheck_info()
    if hck is not None:
        protocol, regex, tcp_send = hck
        simulate_healthcheck(protocol, regex, tcp_send, ip_address, int(port))



def build_bundle():
    # Do we ever actually use this tgz?
    #
    # Tries to make an anonymous and reproducible tar file, also for
    # repositories that include public_bundle.tar.gz in git
    # If we don't do this, running the tester will always make git dirty, with
    # annoying consequences on the docker image labels and other scripts.
    #
    public_bundle_path = os.path.join(service_dir, "public_bundle.tar")
    try:
        os.remove(public_bundle_path)
    except FileNotFoundError:
        pass
    with tarfile.open(public_bundle_path, "w", format=tarfile.PAX_FORMAT) as tar:
        list_public_files(tar=tar)
    subprocess.check_output(["tar", "tvf", public_bundle_path])
    subprocess.check_call(['gzip','-f','-n',public_bundle_path])
    public_bundle_path += ".gz"
    subprocess.check_output(["tar", "tvzf", public_bundle_path])
    _LOG.info("Created public_bundle.tar.gz -- but remember that the scoreboard is updated differently!")


def list_public_files(tar=None):
    if not ('public_files' in service_conf and service_conf['public_files']):
        print("")
        print("")
        print("^^^ \033[36m No Public Files Found \033[0m")
        print("")
        print("")
        return ""

    _LOG.info("Looking at public files...")

    ret = {}  #  basename -> sha256
    for f in service_conf['public_files']:
        bname = os.path.basename(f)  # chalmanager will only use the basename
        _LOG.warning("Public file: %s <-- %s", bname, f, extra={"no_github_ci":True})
        assert os.path.exists(f), "Public file not found: {} -- remember that all public files must be pre-built and checked into git".format(f)
        assert os.path.isfile(f), "Only regular files for the public: {}".format(f)
        assert not os.path.islink(f), "No symlinks for the public: {}".format(f)
        assert bname not in ret, "There was already a public file named {} (public files go by basename only)".format(f)
        assert re.match(PUBLIC_FILENAME_RE, bname), "Weird name for a public file: {} -- can it match '{}' instead?".format(bname, PUBLIC_FILENAME_RE)
        ret[bname] = file_sha256(f)
        if tar:
            def anonymize(t):
                t.mtime = t.uid = t.gid = 0; t.uname = t.gname = ""; t.pax_headers.clear()
                return t
            tar.add(f, arcname=bname, filter=anonymize)

    _LOG.warning("^^^^ PLEASE VERIFY THAT THE PUBLIC FILES ARE CORRECT ^^^^", extra={"no_github_ci":True})
    return ret



def test_deployed(host, do_exploits=None, port=None, host_net=False):
    test_exploits = False
    if do_exploits is None:
        if not (sys.stdin.isatty() and sys.stdout.isatty() and sys.stderr.isatty()):
            _LOG.warning("Not interactive, I won't ask to exploit")
        elif input("Also run exploit scripts? ") in ['y','Y','yes','YES','Yes']:
            _LOG.info("OK. will do.")
            test_exploits = True
    else:
        test_exploits = do_exploits

    if not host or not port:
        yaml_host, yaml_port = validate_game_network_info()

    if port is None:
        port = yaml_port
    assert port, "No game_network_info?!? Then it's an offline service! See the template."

    if host is None:
        host = yaml_host
        assert host not in ('lb','public')  # Only "default" is valid in info.yaml
    if host in ("lb","default"):  # Internal load-balancer by default
        assert INTERNAL_SUBDOMAIN.startswith('.')
        host = service_name + INTERNAL_SUBDOMAIN
    elif host == "public":
        assert PUBLIC_SUBDOMAIN.startswith('.')
        host = service_name + PUBLIC_SUBDOMAIN

    try:
        fyi_ip = socket.gethostbyname(host)  # Just a courtesy check, scripts get it as-is
        _LOG.debug("FYI: %s -> %s", host, fyi_ip)
    except Exception as e:
        _LOG.critical("I couldn't gethostbyname(%s) -> %s %s", host, type(e), str(e))
        _LOG.debug("I'll continue but... most likely things will fail")

    try:
        with socket.create_connection((host,port), timeout=5) as c:
            c.settimeout(3)
            rdata = c.recv(100)
        if POW_DETECTION_STRING in rdata:
            _LOG.info("POW detected, I will test on port + 1 = %d + 1 = %d", port, port+1)
            port += 1
            assert PUBLIC_SUBDOMAIN not in host, "TODO: solve pow/backdoor to test public endpoints"
    except Exception as e:
        _LOG.info("POW auto-detection timed-out, assuming no POW (%s %s)", type(e), e)

    global SHORTREAD_ALLOWED_DIFF  # TODO: ps aux using kubectl
    SHORTREAD_ALLOWED_DIFF = -1  # TODO: ps aux using kubectl
    _LOG.info("Testing deployed version on %s:%d (%s exploits)",
            host, port, 'WITH' if test_exploits else 'without')
    test_service(host, port, exploit=test_exploits, host_net=host_net)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--no-self-update", action="store_true", help="No self-update remote check")
    parser.add_argument("--log-level", metavar='LVL', help="WARNING will also sink docker output. Default: DEBUG")
    parser.add_argument("--use-cwd", action="store_true", help="Use CWD instead of script location for service directory")
    parser.add_argument("--force-color", action="store_true", help="Force color even if not on a TTY")
    parser.add_argument("cmds", metavar='CMD...', nargs=argparse.REMAINDER, help="Usual tester stuff: nothing / bundle, build, test, launch, test_deployed, deploy, ...")

    args = parser.parse_args()
    if args.force_color:
        if globals().get("coloredlogs"):
            coloredlogs.install(logger=_LOG, level=_LOG.level, isatty=True)
        else:
            _LOG.warning("Specified --force-color, but coloredlogs is not installed")
    if args.log_level:
        _LOG.setLevel(args.log_level)
    if _LOG.getEffectiveLevel() >= logging.WARNING:
        dsystem = system_without_stdout

    if not args.no_self_update:
        update_myself()

    if args.use_cwd:
        service_dir = os.getcwd()

    _LOG.info("USING YAML: %s/info.yml", service_dir)
    with open(os.path.join(service_dir, "info.yml")) as yf:
        service_conf = yaml.safe_load(yf)
    service_name = service_conf['service_name']
    _LOG.info("SERVICE ID: %s", service_name)

    image_tag = NAME_PREFIX + ":" + service_name     # also see deploy -- registry names are different
    interaction_image_tag = image_tag + '-interaction'
    container_tag = "running-%s" % service_name
    interaction_container_tag = "running-interaction-%s" % service_name

    validate_yaml()

    assert not any(('--' in c) for c in args.cmds)   # XXX: import the subparser stuff from the finals' tester
    sys.argv = [sys.argv[0]] + args.cmds
    arg = sys.argv[1] if len(sys.argv) >= 2 else ""
    if arg == 'bundle':
        warn_on_old_images()
        build_bundle()
    elif arg == 'list_public_files':
        warn_on_old_images()
        list_public_files()
    elif arg == 'build':
        build_service()
        build_interactions()
        list_public_files()
    elif arg == 'test':
        # ./tester test           [noexploit] [[no-]rebuild]
        # ./tester test ip [port] [noexploit] [[no-]rebuild]
        warn_on_old_images()
        rebuild_first = None
        if image_matches_git(image_tag) and image_matches_git(interaction_image_tag):
            rebuild_first = False
        elif in_github_ci() or ('CI' in os.environ) or not (sys.stdin.isatty() and sys.stdout.isatty() and sys.stderr.isatty()):
            _LOG.info("Not interactive, I won't ask to rebuild first (you can also pass no-rebuild)")
            rebuild_first = False
        if len(sys.argv) >= 2 and sys.argv[-1] == "rebuild":
            del sys.argv[-1]
            rebuild_first = True
        elif len(sys.argv) >= 2 and sys.argv[-1] == "no-rebuild":
            del sys.argv[-1]
            rebuild_first = False
        if rebuild_first is None and input("Do you want to rebuild first (cmdline argument: [no-]rebuild)? ") in ['y','Y','yes','YES','Yes']:
            rebuild_first = True
        if rebuild_first:
            build_service()
            build_interactions()

        test_exploits = True
        if len(sys.argv) >= 2 and sys.argv[-1] == "noexploit":
            del sys.argv[-1]
            test_exploits = False

        if len(sys.argv) == 2:
            _LOG.debug("I'm going to test against a local container (%s exploits)", "with" if test_exploits else "no")
            _ip_address, _port = launch_service()
            test_service(_ip_address, _port, exploit=test_exploits)
        else:
            port = sys.argv[3] if len(sys.argv)>=4 else grep_for_exposed_port()
            _LOG.debug("I'm going to test a remote instance at %s %s (%s exploits)", sys.argv[2], port, "with" if test_exploits else "no")
            test_service(sys.argv[2], int(port), exploit=test_exploits)
    elif arg == 'test_deployed':
        # ./tester test_deployed [[no]exploit [host [port [host_net]]]]
        #                                      ^ special host names: lb, public
        #                                            ^ special port: pow (guessed_port+1)
        if not image_matches_git(interaction_image_tag):
            build_interactions()
        test_exploits = None
        if len(sys.argv) >= 3:
            assert sys.argv[2] in ('exploit','noexploit')
            test_exploits = (sys.argv[2] == 'exploit')
        host = sys.argv[3] if len(sys.argv) >= 4 else None
        force_port = None
        if len(sys.argv) >= 5:
            if sys.argv[4] == 'pow':
                force_port = grep_for_exposed_port()+1
                _LOG.debug("POW-less port = exposed_port+1 = %d+1 = %d", force_port-1, force_port)
            else:
                force_port = int(sys.argv[4])
        host_net = False
        if len(sys.argv) >= 6:
            assert sys.argv[5] == 'host_net'
            host_net = True
        test_deployed(host, test_exploits, port=force_port, host_net=host_net)
        if len(sys.argv) < 4:
            _LOG.debug("If applicable, I suggest to also run:")
            _LOG.debug("  ./tester test_deployed noexploit public")
            _LOG.debug("  ./tester test_deployed noexploit INDIVIDUAL_POD_IPs FORWARDED_PORT host_net")
    elif arg == 'launch':
        build_service()
        try:
            _ip_address, _port = launch_service()
            print("")
            print("SERVICE RUNNING AT: %s %s" % (_ip_address, _port))
            print("nc %s %s" % (_ip_address, _port))
            print("./tester test %s %s" % (_ip_address, _port))
            print("%s:%s" % (_ip_address, _port))
            input()
        finally:
            _stop_container()
    elif arg == 'deploy':
        warn_on_old_image(image_tag)
        rebuild_first = None
        if in_github_ci() or ('CI' in os.environ) or not (sys.stdin.isatty() and sys.stdout.isatty() and sys.stderr.isatty()):
            _LOG.info("Not interactive, I won't ask to build first")
            rebuild_first = False
        if len(sys.argv) >= 3 and sys.argv[2] == "rebuild":
            del sys.argv[2]
            rebuild_first = True
        elif len(sys.argv) >= 3 and sys.argv[2] == "no-rebuild":
            del sys.argv[2]
            rebuild_first = False
        elif image_matches_git(image_tag):
            rebuild_first = False
        if rebuild_first is None and input("Do you want to rebuild first? ") in ['y','Y','yes','YES','Yes']:
            rebuild_first = True

        if rebuild_first:
            build_service()

        if not git_clean:
            _LOG.critical("You're deploying from a dirty git tree, it will be harder to check what images are deployed (label=%s)", git_desc)


        assert REGISTRY, "Missing REGISTRY setting"
        tagcmd = "docker tag %s:%s %s/%s-%s:latest" % (NAME_PREFIX, service_name.lower(), REGISTRY, NAME_PREFIX, service_name.lower())
        print(tagcmd)
        assert dsystem(tagcmd) == 0
        pushcmd = "docker push %s/%s-%s:latest" % (REGISTRY, NAME_PREFIX, service_name.lower())
        print(pushcmd)
        if dsystem(pushcmd) != 0:
            exst = os.path.exists("/etc/docker/daemon.json")
            insec = False
            if exst:
                with open("/etc/docker/daemon.json") as df:
                    dconf = json.load(df)
                insec = REGISTRY in dconf.get('insecure-registries')
            if not (exst and insec):
                _LOG.critical("Couldn't docker push. You need to set this in /etc/docker/daemon.json  --> %s", json.dumps({'insecure-registries': [REGISTRY]})) 
            else:
                _LOG.critical("^^^^^^ Couldn't push to the docker registry, See errors above ^^^^^^^^")
            sys.exit(4)
        print()

        list_public_files()

        if K8S_MASTER:
            _LOG.warning("Killing pods...")
            sshcmd = ['ssh', K8S_MASTER, "--", "sudo", "-i", "sh", "-c", "'kubectl delete pod --all -n " + service_conf['service_name'].lower() + "'"]
            _LOG.warning("Running %s", ' '.join(shlex.quote(x) for x in sshcmd))
            subprocess.check_call(sshcmd)
        else:
            _LOG.error("You'll have to refresh the deployed containers manually")

        _LOG.warning("IF YOUR SERVICE IS ALREADY ON THE SCOREBOARD: yell to update the public files there")
        _LOG.info("If it's a new service: ask it to be added to Kubernetes")
        _LOG.info("Look at what images are deployed in k8s, the docker image should be labeled git-desc=%s", git_desc)
    else:
        assert len(sys.argv) == 1, "Unknown command '{}', try --help".format(sys.argv[1])
        try:
            build_service()
            build_interactions()
            _ip_address, _port = launch_service()
            test_service(_ip_address, _port)
            if not os.getenv('DONT_AUTOBUILD_PUBLIC_BUNDLE'): build_bundle()
        finally:
            _stop_container()