Merge pull request #1 from CallyCo-io/feat/provider_builder

Provider Builder

.github/workflows/test-cally.yaml

@@ -0,0 +1,26 @@
+name: Test
+
+on:
+  push:
+    branches-ignore:
+      - main
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: ["3.11"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+      - name: Install Cally test dependencies
+        run: pip install .[test]
+      - name: Run Ruff # pytest-ruff not picking up pyproject config
+        run: ruff check .
+      - name: Run Pytest
+        run: pytest -v

cally/__init__.py

@@ -1 +1 @@
-__version__ = '0.1.0a1'
+from ._version import VERSION as __version__  # noqa: F401, N811

cally/_version.py

@@ -0,0 +1 @@
+VERSION = '0.1.0a1'

cally/cli.py

@@ -1,8 +1,22 @@
 from pathlib import Path
+from types import ModuleType
+from typing import List
 
 import click
 
-from . import __version__
+from . import commands as builtin_commands
+from ._version import VERSION
+
+
+def get_commands(class_obj: ModuleType) -> List:
+    """
+    Convenience method for collecting all available commands
+    """
+    return [
+        val
+        for (key, val) in vars(class_obj).items()
+        if isinstance(val, click.core.Command)
+    ]
 
 
 @click.group()
@@ -20,11 +34,17 @@
     envvar='CALLY_PROJECT_CONFIG',
     help='Path to the project config file',
 )
-@click.version_option(__version__)
+@click.version_option(VERSION)
 @click.pass_context
 def cally(
-    ctx: click.Context, core_config: click.Path, project_config: click.Path
+    ctx: click.Context,  # noqa: ARG001
+    core_config: click.Path,  # noqa: ARG001
+    project_config: click.Path,  # noqa: ARG001
 ) -> None:
     """
     Top level click command group for Cally
     """
+
+
+for command in get_commands(builtin_commands):
+    cally.add_command(command)

cally/commands/__init__.py

@@ -0,0 +1,13 @@
+import pkgutil
+from importlib import import_module
+
+from click import Group
+
+package = import_module(__package__)
+for _, name, is_pkg in pkgutil.walk_packages(package.__path__):  # noqa: B007
+    full_name = f'{package.__name__}.{name}'
+    module = import_module(full_name)
+    for sub_module in dir(module):
+        attr = getattr(module, sub_module)
+        if isinstance(attr, Group):
+            globals()[sub_module] = attr

cally/commands/provider.py

@@ -0,0 +1,20 @@
+import click
+
+from ..tools.provider import ProviderBuilder
+
+
+@click.group()
+def provider() -> None:
+    pass
+
+
+@click.command()
+@click.option('--source', default='hashicorp')
+@click.option('--provider', required=True)
+@click.option('--version', required=True)
+def build(source: str, provider: str, version: str):
+    click.secho(f'Generating {provider} ({version}) provider')
+    ProviderBuilder(source=source, provider=provider, version=version).build()
+
+
+provider.add_command(build)

cally/constants.py

@@ -0,0 +1,33 @@
+PROVIDER_PYPROJECT = """[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "CallyProviders$title"
+version = "$version"
+requires-python = ">=3.8"
+authors = [
+  {name = "Cally Generated"},
+]
+description = "Cally Generated $title Provider"
+
+[tool.setuptools.packages.find]
+include = ["cally.*"]
+
+[tool.setuptools.package-data]
+"*" = [
+    "py.typed",
+    "*.tgz",
+]"""
+
+PROVIDER_CDKTF = """{
+  "language": "python",
+  "app": "python3 ./main.py",
+  "sendCrashReports": "false",
+  "terraformProviders": [
+    "$provider@~>$version"
+  ],
+  "terraformModules": [],
+  "codeMakerOutput": "$path",
+  "context": {}
+}"""

cally/tools/provider.py

@@ -0,0 +1,61 @@
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from string import Template
+from tempfile import TemporaryDirectory
+
+from ..constants import PROVIDER_CDKTF, PROVIDER_PYPROJECT
+
+
+class ProviderBuilder:
+    provider: str
+    source: str
+    version: str
+
+    def __init__(self, source: str, provider: str, version: str) -> None:
+        self.source = source
+        self.provider = provider
+        self.version = version
+
+    @property
+    def title(self) -> str:
+        return self.provider.title().replace('_', '')
+
+    @property
+    def provider_path(self) -> str:
+        return f"{self.source}/{self.provider.replace('_', '-')}"
+
+    def build(self, output: Path = Path('build')) -> str:
+        build_output = Path(output, self.provider)
+        build_path = Path(build_output, 'cally', 'providers')
+        build_path.mkdir(parents=True, exist_ok=True)
+        cdktf_output = build_path.absolute().as_posix()
+
+        cwd = Path.cwd()
+        with TemporaryDirectory() as tmpdir:
+            os.chdir(tmpdir)
+            Path('cdktf.json').write_text(
+                Template(PROVIDER_CDKTF).substitute(
+                    provider=self.provider_path,
+                    version=self.version,
+                    path=cdktf_output,
+                )
+            )
+            cdktf_command = str(shutil.which('cdktf'))
+
+            result = subprocess.run(
+                f'{cdktf_command} get', shell=True, check=False, stdout=subprocess.PIPE
+            )
+            message = f"Generated {self.provider} provider"
+            if result.returncode != 0:
+                message = f"Failed generating {self.provider} provider"
+            command_output = result.stdout.decode('utf8').strip()
+            os.chdir(cwd)
+
+        Path(build_output, 'pyproject.toml').write_text(
+            Template(PROVIDER_PYPROJECT).substitute(
+                title=self.title, version=self.version
+            )
+        )
+        return f'{message}\n\n{command_output}'

pyproject.toml

@@ -21,13 +21,42 @@ development = [
   "black",
   "isort",
   "mypy",
+  "pytest",
+  "pytest-black",
+  "pytest-mypy",
+  "pytest-ruff",
+  "ruff",
+  "types-PyYAML",
+]
+test = [
+  "black",
+  "mypy",
+  "pytest",
+  "pytest-black",
+  "pytest-mypy",
+  "pytest-ruff",
   "ruff",
   "types-PyYAML",
 ]
 
 [project.scripts]
 cally = "cally.cli:cally"
 
+[tool.black]
+skip-string-normalization = true
+
+[tool.isort]
+profile = "black"
+src_paths = ["cally", "tests"]
+
+[tool.pytest.ini_options]
+addopts = "-p no:cacheprovider --black --mypy"
+filterwarnings = [
+    "ignore",
+    "default:::cally.*",
+    "default:::tests.*"
+]
+
 [tool.setuptools.dynamic]
 version = {attr = "cally.__version__"}
 
@@ -46,7 +75,8 @@ select = [
     "PTH", # pathlib
     "RUF", # ruff
 ]
-
-[tool.isort]
-profile = "black"
-src_paths = ["cally", "tests"]
+[tool.ruff.lint.per-file-ignores]
+"cally/tools/provider.py" = [
+    "S404",  # `subprocess` module is possibly insecure - used for 'cdktf get'
+    "S602"  # `subprocess` call with `shell=True` identified - required for 'cdktf get'
+]

tests/__init__.py

@@ -0,0 +1,27 @@
+import os
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest import TestCase, mock
+
+
+class CallyTestHarness(TestCase):
+    working: TemporaryDirectory
+
+    def setUp(self):
+        self.current_working = Path().cwd()
+        self.working = TemporaryDirectory()
+        self.env_patcher = mock.patch.dict(
+            os.environ,
+            {
+                "HOME": self.working.name,
+                "LC_ALL": os.environ.get('LC_ALL', 'C.UTF-8'),
+                "LANG": os.environ.get('LANG', 'C.UTF-8'),
+            },
+            clear=True,
+        )
+        self.env_patcher.start()
+
+    def tearDown(self):
+        self.env_patcher.stop()
+        self.working.cleanup()
+        os.chdir(self.current_working)

tests/cli/__init__.py

@@ -0,0 +1,14 @@
+from click.testing import CliRunner
+
+from cally import __version__
+from cally.cli import cally
+
+from .. import CallyTestHarness
+
+
+class CliTests(CallyTestHarness):
+
+    def test_version(self):
+        result = CliRunner().invoke(cally, ['--version'])
+        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(result.output, f'cally, version {__version__}\n')

tests/tools/test_provider.py

@@ -0,0 +1,22 @@
+from cally.tools.provider import ProviderBuilder
+
+from .. import CallyTestHarness
+
+
+class ProviderBuilderTests(CallyTestHarness):
+
+    def test_title(self):
+        self.assertEqual(
+            ProviderBuilder(
+                source='hashicorp', provider='random', version='3.6.0'
+            ).title,
+            'Random',
+        )
+
+    def test_provider_path(self):
+        self.assertEqual(
+            ProviderBuilder(
+                source='hashicorp', provider='random', version='3.6.0'
+            ).provider_path,
+            'hashicorp/random',
+        )