Added bld2repo

Simple tool do download build requirements of a modular build from koji.

Signed-off-by: Martin Curlej <[email protected]>

.gitignore

@@ -3,3 +3,4 @@
 /*/build/
 /*/dist/
 __pycache__/
+.vscode/

.travis.yml

@@ -22,5 +22,7 @@ jobs:
       script: ./travis-stage.sh docker_pull && ./travis-stage.sh test
     - env: TOXENV=py39 MODULEMD_TOOL=modulemd_tools SITEPACKAGES=true
       script: ./travis-stage.sh docker_pull && ./travis-stage.sh test
+    - env: TOXENV=py39 MODULEMD_TOOL=bld2repo SITEPACKAGES=true
+      script: ./travis-stage.sh docker_pull && ./travis-stage.sh test
     - env: TOXENV=flake8
       script: ./travis-stage.sh docker_pull && ./travis-stage.sh test

README.md

@@ -39,7 +39,6 @@ For more information about `modulemd-merge`, please see
 
 
 ### modulemd-generate-macros
-
 Generate `module-build-macros` SRPM package, which is a central piece
 for building modules. It should be present in the buildroot before any
 other module packages are submitted to be built.
@@ -57,6 +56,13 @@ other tools yet, be cautious.**
 [modulemd_tools/README.md](modulemd_tools/README.md)
 
 
+### bld2repo
+Simple tool for dowloading build required RPMs of a modular build from koji.
+
+For more information about `bld2repo`, please see
+[bld2repo/README.md](bld2repo/README.md)
+
+
 ## Use cases
 
 ### Creating a module repository from a regular repository

bld2repo/README.md

@@ -0,0 +1,25 @@
+# bld2repo
+
+Simple tool which will download modular build dependencies from a 
+modular build in a koji instance and create a RPM repository out of it.
+
+## usage
+
+Provide a build id of modular build in koji and the cli tool will
+download all the rpms tagged in a build tag of a modular rpm build.
+
+```
+$ bld2repo --build-id 1234
+```
+
+After the download is finished the tool will call createrepo_c on the
+working directory, creating a rpm repository.
+
+The defaults are set to the current fedora koji instance.
+If you are using a different koji instance please adjust those
+values through script arguments. For more information about script
+arguments please run:
+
+```
+$ bld2repo -h
+```

bld2repo/bld2repo/__init__.py

@@ -0,0 +1,142 @@
+import os
+import sys
+import urllib.request
+import subprocess
+
+import koji
+
+
+def get_buildrequire_pkgs_from_build(build_id, session, config):
+    """
+    Function which queries koji for pkgs whom belong to a given build tag
+    of a koji build and paires rpms with their respective package.
+
+    :param str build_id: build id of a build in koji.
+    :param koji.ClientSession session: koji connection session object
+    :return: list of pairings of package and rpms.
+    :rtype: list
+    """
+    print("Retriewing build metadata from: ", config.koji_host)
+    build = session.getBuild(build_id)
+    if not build:
+        raise Exception("Build with id '{id}' has not been found.".format(id=build_id))
+
+    print("Build with the ID", build_id, "found.")
+    tags = session.listTags(build["build_id"])
+
+    build_tag = [t["name"] for t in tags if t["name"].endswith("-build")]
+    if not build_tag:
+        raise Exception("Build with id '{id}' is not tagged in a 'build' tag.".format(id=build_id))
+
+    tag_data = session.listTaggedRPMS(build_tag[0], latest=True, inherit=True)
+
+    print("Found the build tag '", build_tag[0], "' associated with the build.")
+    tagged_rpms = tag_data[0]
+    tagged_pkgs = tag_data[1]
+    pkgs = []
+    archs = [config.arch, "noarch"]
+    print("Gathering packages and rpms tagged in '", build_tag[0],"'.")
+    for pkg in tagged_pkgs:
+        pkg_md = {
+            "package": pkg,
+            "rpms": [],
+        }
+
+        for rpm in tagged_rpms:
+            if pkg["build_id"] == rpm["build_id"] and rpm["arch"] in archs:
+                pkg_md["rpms"].append(rpm)
+                tagged_rpms.remove(rpm)
+
+        if pkg_md["rpms"]:
+            pkgs.append(pkg_md)
+    print("Gathering done.")
+    return pkgs
+
+
+def add_rpm_urls(pkgs, config):
+    """
+    For each rpm from a package creates an download url and adds it to the package.
+
+    :param list pkgs: list of packages
+    :return pkgs: list of packages and their rpms
+    :rtype: list
+    :return rpm_num: number of rpms
+    :rtype: int
+    """
+    rpm_num = 0
+    for pkg in pkgs:
+        build_path = koji.pathinfo.build(pkg["package"]).replace(koji.pathinfo.topdir, "")
+        pkg["rpm_urls"] = []
+        for rpm in pkg["rpms"]:
+            rpm_num += 1
+            rpm_filename = "-".join([rpm["name"], rpm["version"],
+                                     rpm["release"]]) + "." + rpm["arch"] + ".rpm"
+            rpm_url = config.koji_storage_host + build_path + "/" + rpm["arch"] + "/" + rpm_filename
+            pkg["rpm_urls"].append(rpm_url)
+
+
+    return pkgs, rpm_num
+
+
+def download_file(url, target_pkg_dir, filename):
+    """
+    Wrapper function for downloading a file
+
+    :param str url: url to a file
+    :param str target_pkg_dir: the dir where the file should be downloaded
+    :param str filename: the name of the downloaded file
+    """
+    abs_file_path = "/".join([target_pkg_dir, filename])
+    try:
+        urllib.request.urlretrieve(url, abs_file_path)
+    except Exception as ex:
+        raise Exception("HTTP error for url: {url}\nError message: {msg}\nHTTP code: {code}".format(
+            url=ex.url, msg=ex.msg, code=ex.code))
+
+
+def rpm_bulk_download(pkgs, rpm_num, working_dir):
+    """
+    Downloads all the rpms from which belong to a package.
+
+    :param list pkgs: list of pkgs with their rpms and urls to those rpms
+    :param int rpm_num: number of all the rpms included in pkgs
+    :param str working_dir: the dir where the rpms will be downloaded
+    """
+    print("Starting bulk download of rpms...")
+    rpm_dwnlded = 0
+
+    for pkg in pkgs:
+        for url in pkg["rpm_urls"]:
+            # we print the status of the download
+            status = "[{rpms}/{left}]".format(rpms=rpm_num, left=rpm_dwnlded)
+            print(status, end="\r", flush=True)
+            # we store the rpm in a similar location as it is on the storage server
+            url_parts = url.split("/")
+            filename = url_parts[-1]
+            arch = url_parts[-2]
+            pkg_name = "-".join([url_parts[-5], url_parts[-4], url_parts[-3]])
+            target_pkg_dir = "/".join([working_dir, pkg_name, arch])
+            # we create the package dir if it is not created
+            if not os.path.exists(target_pkg_dir):
+                os.makedirs(target_pkg_dir)
+            else:
+                # if we downloaded the file already we skip
+                file_path = target_pkg_dir + "/" + filename
+                if os.path.exists(file_path):
+                    rpm_dwnlded += 1
+                    continue
+            download_file(url, target_pkg_dir, filename)
+            rpm_dwnlded += 1
+
+    # update the status last time to mark all of the rpms downloaded
+    status = "[{rpms}/{left}]".format(rpms=rpm_num, left=rpm_dwnlded)
+    print(status)
+    print("Download successful.")
+
+
+def create_repo(working_dir):
+    print("Calling createrepo_c...")
+    args = ["createrepo_c", working_dir]
+    subprocess.Popen(args, cwd=working_dir).communicate()
+    print("Repo created.")
+

bld2repo/bld2repo/cli.py

@@ -0,0 +1,58 @@
+import argparse
+import os
+
+from bld2repo import (get_buildrequire_pkgs_from_build, add_rpm_urls, rpm_bulk_download,
+                      create_repo)
+from bld2repo.config import Config
+from bld2repo.utils import get_koji_session
+
+
+def get_arg_parser():
+    description = (
+        "When provided with a build id it will download all buildrequired RPMs"
+        "of a modular koji build into the provided directory and create a repository out of it."
+    )
+    parser = argparse.ArgumentParser("bld2repo", description=description,
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("-b", "--build-id", required=True, type=int, help="ID of a koji build.")
+    parser.add_argument("-d", "--result-dir", help="Directory where the RPMs are downloaded.",
+                        default=".", type=str)
+    parser.add_argument("-a", "--arch", help=("For which architecture the RPMs should be download"
+                                              "ed. The 'noarch' is included automatically."),
+                        default="x86_64", type=str)
+    parser.add_argument("-k", "--koji-host", type=str,
+                        default="https://koji.fedoraproject.org/kojihub",
+                        help="Koji host base url")
+    parser.add_argument("-s", "--koji-storage-host", type=str,
+                        default="https://kojipkgs.fedoraproject.org",
+                        help=("Koji storage storage host base url. Server where the RPMs are "
+                              "stored. Required to be used together with `--koji-host`."))
+    return parser
+
+
+def main():
+    parser = get_arg_parser()
+    args = parser.parse_args()
+
+    koji_host_dflt = parser.get_default("koji_host")
+
+    if args.koji_host != koji_host_dflt:
+        koji_storage_dflt = parser.get_default("koji_storage_host")
+        if args.koji_storage_host == koji_storage_dflt:
+            parser.error("--koji-host and --koji-storage-host need to be used to together.")
+
+    config = Config(args.koji_host, args.koji_storage_host, args.arch, args.result_dir)
+    session = get_koji_session(config)
+
+    pkgs = get_buildrequire_pkgs_from_build(args.build_id, session, config)
+
+    pkgs, rpm_num = add_rpm_urls(pkgs, config)
+
+    rpm_bulk_download(pkgs, rpm_num, config.arch)
+
+    create_repo(config.result_dir)
+
+
+if __name__ == "__main__":
+    main()
+

bld2repo/bld2repo/config.py

@@ -0,0 +1,8 @@
+class Config():
+
+    def __init__(self, koji_host, koji_storage_host, arch, result_dir):
+        self.koji_host = koji_host
+        self.koji_storage_host = koji_storage_host
+        self.arch = arch
+        self.result_dir = result_dir
+

bld2repo/bld2repo/utils.py

@@ -0,0 +1,9 @@
+import koji
+
+
+def get_koji_session(config):
+
+    session = koji.ClientSession(config.koji_host)
+
+    return session
+

bld2repo/requirements.txt

@@ -0,0 +1 @@
+koji

bld2repo/setup.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+import os.path
+
+from setuptools import setup, find_packages
+
+dirname = os.path.dirname(os.path.realpath(__file__))
+
+with open(os.path.join(dirname, "README.md"), "r") as fh:
+    long_description = fh.read()
+
+with open(os.path.join(dirname, 'requirements.txt'), "r") as f:
+    requires = f.read().splitlines()
+
+setup(
+    name='bld2repo',
+    version='0.1',
+    packages=find_packages(exclude=("tests",)),
+    url='https://github.com/rpm-software-management/modulemd-tools',
+    license='MIT',
+    author='Martin Čurlej',
+    author_email='[email protected]',
+    description=('Tool to download modular build dependencies of '
+                 'a modular build from koji.'),
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    install_requires=requires,
+    entry_points={
+        'console_scripts': [
+            'bld2repo=bld2repo.cli:main'],
+    },
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: POSIX :: Linux",
+    ],
+    include_package_data=True,
+)
+

bld2repo/test-requirements.txt

@@ -0,0 +1 @@
+pytest