From 84d2f6acc8a03061b3e6a413f468bd83c33f414f Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Mon, 30 Nov 2020 18:43:03 +0000
Subject: [PATCH] upload-oscontainer: Support oscontainer.yaml and
 extensions.yaml

Pairs with https://github.com/openshift/os/pull/455

Add an `oscontainer.yaml` which allows configuring (currently) just
the `FROM` line equivalent.  This helps keep things declarative
in the config instead of part of the pipeline.

And since extensions are now a core part of OpenShift 4 (particularly RT kernel)
which also uses the oscontainer, let's lift the logic to generate
those bits of the oscontainer into this repo and out of the config git.

Both of these are part of the general philosophy we've had that:

 - config git is declarative config
 - coreos-assembler is the mechanism
 - a pipeline just scripts coreos-assembler in a way that's
   easy to reproduce with plain podman too
---
 src/cmd-upload-oscontainer | 21 ++++++++++++
 src/download-extensions    | 67 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+)
 create mode 100755 src/download-extensions

diff --git a/src/cmd-upload-oscontainer b/src/cmd-upload-oscontainer
index b61e08ce1d..d684fc2924 100755
--- a/src/cmd-upload-oscontainer
+++ b/src/cmd-upload-oscontainer
@@ -7,6 +7,7 @@
 
 import argparse
 import json
+import yaml
 import os
 import shutil
 import subprocess
@@ -39,6 +40,24 @@ metapath = f"{latest_build_path}/meta.json"
 with open(metapath) as f:
     meta = json.load(f)
 
+configdir = os.path.abspath('src/config')
+oscconfigpath = f'{configdir}/oscontainer.yaml'
+if os.path.exists(oscconfigpath):
+    with open(oscconfigpath) as f:
+        c = yaml.safe_load(f)
+        base = c.get('base')
+        if base is not None:
+            args.from_image = base
+
+extensions_src = 'src/config/extensions.yaml'
+extensions_destdir = None
+if os.path.exists(extensions_src):
+    extensions_destdir = 'tmp/extensions'
+    if os.path.exists(extensions_destdir):
+        shutil.rmtree(extensions_destdir)
+    os.mkdir(extensions_destdir)
+    cmdlib.run_verbose(['/usr/lib/coreos-assembler/download-extensions', extensions_destdir])
+
 print("Preparing to upload oscontainer for build: {}".format(latest_build))
 ostree_commit = meta['ostree-commit']
 
@@ -87,6 +106,8 @@ os.environ['REGISTRY_AUTH_FILE'] = authfile
 cosa_argv.extend(['/usr/lib/coreos-assembler/oscontainer.py', '--workdir=./tmp', 'build',  f"--from={args.from_image}"])
 for d in args.add_directory:
     cosa_argv.append(f"--add-directory={d}")
+if extensions_destdir is not None:
+    cosa_argv.append(f"--add-directory={extensions_destdir}")
 cosa_argv.append(f"--display-name={display_name}")
 subprocess.check_call(cosa_argv +
     [f'--digestfile={digestfile}',
diff --git a/src/download-extensions b/src/download-extensions
new file mode 100755
index 0000000000..bb4adee7d4
--- /dev/null
+++ b/src/download-extensions
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+# RPMs as operating system extensions, distinct from the base ostree commit/image
+# https://github.com/openshift/enhancements/blob/master/enhancements/rhcos/extensions.md
+
+import os
+import sys
+import yaml
+from cosalib import cmdlib
+
+destdir = sys.argv[1]
+tmpdir = 'tmp'
+# yum wants this to be absolute
+configdir = os.path.abspath('src/config')
+extsrcpath = f'{configdir}/extensions.yaml'
+extjson = f'{tmpdir}/extensions.json'
+basearch = cmdlib.get_basearch()
+
+with open(extsrcpath) as f:
+    extensions = yaml.safe_load(f)
+
+# The "v2" format here is that there's an extensions/ directory, with subdirectories
+# for each extension - except you should ignore "repodata/".
+edestdir = f'{destdir}/extensions'
+os.mkdir(edestdir)
+
+# Stuff that's not part of the extension
+dependenciesdir = f'{edestdir}/dependencies'
+os.mkdir(dependenciesdir)
+
+
+# Downloads packages from specified repos
+def yumdownload(destdir, pkgs):
+    # FIXME eventually use rpm-ostree for this
+    # shellcheck disable=SC2068
+    args = ['yum', f'--setopt=reposdir={configdir}', f'--arch={basearch}', 'download']
+    args.extend(pkgs)
+    cmdlib.run_verbose(args, cwd=destdir)
+
+
+# Reuseable function for setting up an extension
+# Assumes it is running in "${destdir}/extensions"
+# 1 = extension name
+# 2 = package string/glob
+# 3 = OPTIONAL: dependencies string/glob
+def createext(extname, pkgs):
+    print(f"Creating extension {extname}")
+    extdir = f"{edestdir}/{extname}"
+    os.mkdir(extdir)
+    primary = pkgs[0]
+    yumdownload(extdir, [primary])
+
+    deps = pkgs[1:]
+    if len(deps) > 0:
+        print(f"Downloading dependencies for {extname}")
+        yumdownload(dependenciesdir, deps)
+
+
+for (name, ext) in extensions['extensions'].items():
+    pkgs = ext['packages']
+    extarches = ext.get('architectures')
+    if extarches is not None and basearch not in extarches:
+        print(f"Skipping extension {name} for this architecture")
+        continue
+    createext(name, pkgs)
+
+# Create the yum/dnf repo
+cmdlib.run_verbose(['createrepo_c', '--no-database', '.'], cwd=destdir)