From 4c7295a3f0adf4571c5071143609ffb843d47b43 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Mon, 18 Mar 2024 20:33:51 +0800 Subject: [PATCH] feat: Provider Builder Automatically instantiate providers into the stack as required, including the ability to inject config from the service settings. --- .github/workflows/coverage-build.yaml | 19 ++++++++++++ src/cally/cdk/__init__.py | 44 ++++++++++++++++++++++++--- tests/stacks/test_providers.py | 42 +++++++++++++++++++++++++ tests/testdata/cdk/provider_load.json | 20 ++++++++++++ 4 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 tests/stacks/test_providers.py create mode 100644 tests/testdata/cdk/provider_load.json diff --git a/.github/workflows/coverage-build.yaml b/.github/workflows/coverage-build.yaml index ee768bd..980f2d0 100644 --- a/.github/workflows/coverage-build.yaml +++ b/.github/workflows/coverage-build.yaml @@ -17,6 +17,25 @@ jobs: cache: pip - name: Install Cally test dependencies run: pip install .[test] + - name: Restore Provider Packages + id: cache-providers + uses: actions/cache@v4 + with: + path: build/random/dist/CallyProvidersRandom-3.6.0.tar.gz + key: cally-provider-random-3.6.0 + - uses: actions/setup-node@v4 + if: steps.cache-providers.outputs.cache-hit != 'true' + with: + node-version: "20" + - name: Install cdktf-cli and build + if: steps.cache-providers.outputs.cache-hit != 'true' + run: | + npm install cdktf-cli + echo "$(pwd)/node_modules/.bin/" >> $GITHUB_PATH + cally provider build --provider random --version 3.6.0 + (cd build/random && python -m build) + - name: Install Provider Pacakge + run: pip install build/random/dist/CallyProvidersRandom-3.6.0.tar.gz - name: Run Coverage run: | coverage run -m pytest diff --git a/src/cally/cdk/__init__.py b/src/cally/cdk/__init__.py index 8abab21..045f25c 100644 --- a/src/cally/cdk/__init__.py +++ b/src/cally/cdk/__init__.py @@ -2,7 +2,7 @@ from copy import deepcopy from dataclasses import dataclass, make_dataclass from importlib import import_module -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any, Dict, List, Optional from cdktf import ( App, @@ -39,9 +39,12 @@ def __init__(self, identifier: str, **kwargs) -> None: def __str__(self) -> str: return f'${{{self.resource}.{self.attributes.id}.id}}' - def __getattr__(self, item: str) -> str: + def __getattr__(self, item: str) -> Optional[str]: + # TODO: This likely could use some improvement if item.startswith('__jsii'): return getattr(self._instantiated_resource, item) + if item == 'attributes': + return None return f'${{{self.resource}.{self.attributes.id}.{item}}}' def _get_attribute_default(self, name: str) -> Any: @@ -59,7 +62,11 @@ def _build_attributes(self, identifier: str, **kwargs) -> CallyResourceAttribute ] name = f'{self.__class__.__name__}CallyAttributes' cls = make_dataclass(name, fields, bases=(CallyResourceAttributes,)) - return cls(**{'id_': identifier, **kwargs}) + # Some newer provider releases appear to use 'id_' + id_field = 'id' + if 'id_' in parameters: + id_field = 'id_' + return cls(**{id_field: identifier, **kwargs}) def construct_resource(self, scope: Construct, provider: TerraformProvider) -> None: self.attributes.provider = provider @@ -69,6 +76,7 @@ def construct_resource(self, scope: Construct, provider: TerraformProvider) -> N class CallyStack: + _providers: Dict[str, TerraformProvider] _resources: List[CallyResource] service: 'CallyStackService' @@ -81,6 +89,24 @@ def add_resource(self, resource: CallyResource) -> None: def add_resources(self, resources: List[CallyResource]) -> None: self.resources.extend(resources) + def get_provider(self, scope: Construct, provider: str) -> TerraformProvider: + if provider not in self.providers: + # google_beta -> GoogleBetaProvider + resource = f"{provider.capitalize().replace('_', '')}Provider" + module = import_module(f'cally.providers.{provider}.provider') + cls = getattr(module, resource) + self.providers.update( + { + provider: cls( + scope, id=provider, **self.service.providers.get(provider, {}) + ) + } + ) + prov = self.providers.get(provider) + if prov is None: + raise ValueError("Provider instantion failed") + return prov + @property def name(self) -> str: return self.service.name @@ -95,6 +121,12 @@ def resources(self) -> List[CallyResource]: self._resources = [] return self._resources + @property + def providers(self) -> Dict[str, TerraformProvider]: + if getattr(self, '_providers', None) is None: + self._providers = {} + return self._providers + def synth_stack(self, outdir='cdktf.out'): stack = self @@ -104,8 +136,10 @@ def __init__(self, scope: Construct) -> None: super().__init__(scope, stack.name) # TODO: Build provider loader for resource in stack.resources: - provider = TerraformProvider(self, 'test') # type: ignore - resource.construct_resource(self, provider=provider) + resource.construct_resource( + self, + provider=stack.get_provider(self, resource.provider), + ) LocalBackend( self, path=f'state/{stack.name}.tfstate' diff --git a/tests/stacks/test_providers.py b/tests/stacks/test_providers.py new file mode 100644 index 0000000..e9168b5 --- /dev/null +++ b/tests/stacks/test_providers.py @@ -0,0 +1,42 @@ +from contextlib import suppress +from unittest import skipUnless + +from cally.cdk import CallyResource, CallyStack + +from .. import CallyTfTestHarness + +skip_tests = False +with suppress(ModuleNotFoundError): + from cally.providers.random import pet # noqa: F401 + + skip_tests = True + + +@skipUnless(skip_tests, "Random provider must be installed") +class CallyProviderTests(CallyTfTestHarness): + def test_provider_load(self): + stack = CallyStack(service=self.empty_service) + + class Pet(CallyResource): + provider = 'random' + resource = 'pet' + + stack.add_resource(Pet('random-pet-name')) + result = self.synth_stack(stack) + testdata = self.load_json_file('cdk/provider_load.json') + self.assertDictEqual(result.get('provider'), testdata.get('provider')) + self.assertDictEqual(result.get('resource'), testdata.get('resource')) + + def test_provider_config(self): + self.empty_service.providers.update(random={'alias': 'foo'}) + stack = CallyStack(service=self.empty_service) + + class Pet(CallyResource): + provider = 'random' + resource = 'pet' + + stack.add_resource(Pet('random-pet-name')) + result = self.synth_stack(stack) + self.assertEqual( + result.get('provider', {}).get('random', {})[0].get('alias', ''), 'foo' + ) diff --git a/tests/testdata/cdk/provider_load.json b/tests/testdata/cdk/provider_load.json new file mode 100644 index 0000000..f8c476e --- /dev/null +++ b/tests/testdata/cdk/provider_load.json @@ -0,0 +1,20 @@ +{ + "provider": { + "random": [ + {} + ] + }, + "resource": { + "random_pet": { + "random-pet-name": { + "//": { + "metadata": { + "path": "test/random-pet-name", + "uniqueId": "random-pet-name" + } + }, + "provider": "random" + } + } + } +}