Provider Builder

.github/workflows/coverage-build.yaml

@@ -17,6 +17,25 @@ jobs:
           cache: pip
       - name: Install Cally test dependencies
         run: pip install .[test]
+      - name: Restore Provider Packages
+        id: cache-providers
+        uses: actions/cache@v4
+        with:
+          path: build/random/dist/CallyProvidersRandom-3.6.0.tar.gz
+          key: cally-provider-random-3.6.0
+      - uses: actions/setup-node@v4
+        if: steps.cache-providers.outputs.cache-hit != 'true'
+        with:
+          node-version: "20"
+      - name: Install cdktf-cli and build
+        if: steps.cache-providers.outputs.cache-hit != 'true'
+        run: |
+          npm install cdktf-cli
+          echo "$(pwd)/node_modules/.bin/" >> $GITHUB_PATH
+          cally provider build --provider random --version 3.6.0
+          (cd build/random && python -m build)
+      - name: Install Provider Pacakge
+        run: pip install build/random/dist/CallyProvidersRandom-3.6.0.tar.gz
       - name: Run Coverage
         run: |
           coverage run -m pytest

pyproject.toml

@@ -55,6 +55,14 @@ skip-string-normalization = true
 [tool.coverage.run]
 omit = ["tests/*"]
 
+[[tool.mypy.overrides]]
+module = "dynaconf.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "cally.providers.*"
+ignore_missing_imports = true
+
 [tool.isort]
 profile = "black"
 src_paths = ["src", "tests"]

src/cally/cdk/__init__.py

@@ -2,7 +2,7 @@
 from copy import deepcopy
 from dataclasses import dataclass, make_dataclass
 from importlib import import_module
-from typing import TYPE_CHECKING, Any, List
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from cdktf import (
     App,
@@ -39,9 +39,12 @@ def __init__(self, identifier: str, **kwargs) -> None:
     def __str__(self) -> str:
         return f'${{{self.resource}.{self.attributes.id}.id}}'
 
-    def __getattr__(self, item: str) -> str:
+    def __getattr__(self, item: str) -> Optional[str]:
+        # TODO: This likely could use some improvement
         if item.startswith('__jsii'):
             return getattr(self._instantiated_resource, item)
+        if item == 'attributes':
+            return None
         return f'${{{self.resource}.{self.attributes.id}.{item}}}'
 
     def _get_attribute_default(self, name: str) -> Any:
@@ -59,7 +62,11 @@ def _build_attributes(self, identifier: str, **kwargs) -> CallyResourceAttribute
         ]
         name = f'{self.__class__.__name__}CallyAttributes'
         cls = make_dataclass(name, fields, bases=(CallyResourceAttributes,))
-        return cls(**{'id_': identifier, **kwargs})
+        # Some newer provider releases appear to use 'id_'
+        id_field = 'id'
+        if 'id_' in parameters:
+            id_field = 'id_'
+        return cls(**{id_field: identifier, **kwargs})
 
     def construct_resource(self, scope: Construct, provider: TerraformProvider) -> None:
         self.attributes.provider = provider
@@ -69,6 +76,7 @@ def construct_resource(self, scope: Construct, provider: TerraformProvider) -> N
 
 
 class CallyStack:
+    _providers: Dict[str, TerraformProvider]
     _resources: List[CallyResource]
     service: 'CallyStackService'
 
@@ -81,6 +89,24 @@ def add_resource(self, resource: CallyResource) -> None:
     def add_resources(self, resources: List[CallyResource]) -> None:
         self.resources.extend(resources)
 
+    def get_provider(self, scope: Construct, provider: str) -> TerraformProvider:
+        if provider not in self.providers:
+            # google_beta -> GoogleBetaProvider
+            resource = f"{provider.capitalize().replace('_', '')}Provider"
+            module = import_module(f'cally.providers.{provider}.provider')
+            cls = getattr(module, resource)
+            self.providers.update(
+                {
+                    provider: cls(
+                        scope, id=provider, **self.service.providers.get(provider, {})
+                    )
+                }
+            )
+        prov = self.providers.get(provider)
+        if prov is None:
+            raise ValueError("Provider instantion failed")
+        return prov
+
     @property
     def name(self) -> str:
         return self.service.name
@@ -95,6 +121,12 @@ def resources(self) -> List[CallyResource]:
             self._resources = []
         return self._resources
 
+    @property
+    def providers(self) -> Dict[str, TerraformProvider]:
+        if getattr(self, '_providers', None) is None:
+            self._providers = {}
+        return self._providers
+
     def synth_stack(self, outdir='cdktf.out'):
         stack = self
 
@@ -104,8 +136,10 @@ def __init__(self, scope: Construct) -> None:
                 super().__init__(scope, stack.name)
                 # TODO: Build provider loader
                 for resource in stack.resources:
-                    provider = TerraformProvider(self, 'test')  # type: ignore
-                    resource.construct_resource(self, provider=provider)
+                    resource.construct_resource(
+                        self,
+                        provider=stack.get_provider(self, resource.provider),
+                    )
 
                 LocalBackend(
                     self, path=f'state/{stack.name}.tfstate'

src/cally/cli/config/__init__.py

@@ -3,7 +3,7 @@
 from typing import Optional, Union
 
 import click
-from dynaconf import Dynaconf  # type: ignore
+from dynaconf import Dynaconf
 
 from . import types as cally_types
 from .validators import BASE_CALLY_CONFIG

src/cally/cli/config/loader.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 import yaml
-from dynaconf import LazySettings  # type: ignore
+from dynaconf import LazySettings
 
 
 def load(obj: LazySettings, *args, **kwargs) -> None:  # noqa: ARG001

tests/cli/config/__init__.py

@@ -1,10 +1,9 @@
 import os
 from unittest import mock
 
-from dynaconf import ValidationError  # type: ignore
-
 from cally.cli.config import CallyConfig
 from cally.cli.config import types as cally_types
+from dynaconf import ValidationError
 
 from ... import CallyTestHarness
 

tests/cli/test_tf.py

@@ -16,10 +16,12 @@ class TfTests(CallyTestHarness):
     def test_empty_print(self):
         result = CliRunner().invoke(
             tf,
-            ['print', '--service', 'test-cli', '--environment', 'test'],
+            ['print', '--service', 'test', '--environment', 'test'],
             obj=CallyConfig(config_file='blah.yaml'),
         )
         self.assertEqual(result.exit_code, 0)
+        testdata = {"backend": {"local": {"path": "state/test.tfstate"}}}
         self.assertDictEqual(
-            json.loads(result.output), self.load_json_file('cli/empty_print.json')
+            json.loads(result.output).get('terraform'),
+            testdata,
         )

tests/stacks/test_cally.py

@@ -8,4 +8,6 @@ class CallyStackTests(CallyTfTestHarness):
     def test_empty_synth(self):
         stack = CallyStack(service=self.empty_service)
         result = self.synth_stack(stack)
-        self.assertDictEqual(result, self.load_json_file('cdk/empty_synth.json'))
+        self.assertDictEqual(
+            result.get('terraform'), self.load_json_file('cdk/empty_synth.json')
+        )

tests/stacks/test_idp.py

@@ -20,4 +20,6 @@ def test_class_load(self):
     def test_empty_synth(self):
         stack = stacks.MinimalStack(service=self.empty_service)
         result = self.synth_stack(stack)
-        self.assertDictEqual(result, self.load_json_file('cdk/empty_synth.json'))
+        self.assertDictEqual(
+            result.get('terraform'), self.load_json_file('cdk/empty_synth.json')
+        )

tests/stacks/test_providers.py

@@ -0,0 +1,42 @@
+from contextlib import suppress
+from unittest import skipUnless
+
+from cally.cdk import CallyResource, CallyStack
+
+from .. import CallyTfTestHarness
+
+skip_tests = False
+with suppress(ModuleNotFoundError):
+    from cally.providers.random import pet  # noqa: F401
+
+    skip_tests = True
+
+
+@skipUnless(skip_tests, "Random provider must be installed")
+class CallyProviderTests(CallyTfTestHarness):
+    def test_provider_load(self):
+        stack = CallyStack(service=self.empty_service)
+
+        class Pet(CallyResource):
+            provider = 'random'
+            resource = 'pet'
+
+        stack.add_resource(Pet('random-pet-name'))
+        result = self.synth_stack(stack)
+        testdata = self.load_json_file('cdk/provider_load.json')
+        self.assertDictEqual(result.get('provider'), testdata.get('provider'))
+        self.assertDictEqual(result.get('resource'), testdata.get('resource'))
+
+    def test_provider_config(self):
+        self.empty_service.providers.update(random={'alias': 'foo'})
+        stack = CallyStack(service=self.empty_service)
+
+        class Pet(CallyResource):
+            provider = 'random'
+            resource = 'pet'
+
+        stack.add_resource(Pet('random-pet-name'))
+        result = self.synth_stack(stack)
+        self.assertEqual(
+            result.get('provider', {}).get('random', {})[0].get('alias', ''), 'foo'
+        )

tests/testdata/cdk/empty_synth.json

@@ -1,17 +1,7 @@
 {
-    "//": {
-        "metadata": {
-            "backend": "local",
-            "stackName": "test",
-            "version": "0.20.4"
-        },
-        "outputs": {}
-    },
-    "terraform": {
-        "backend": {
-            "local": {
-                "path": "state/test.tfstate"
-            }
+    "backend": {
+        "local": {
+            "path": "state/test.tfstate"
         }
     }
 }

tests/testdata/cdk/provider_load.json

@@ -0,0 +1,20 @@
+{
+    "provider": {
+        "random": [
+            {}
+        ]
+    },
+    "resource": {
+        "random_pet": {
+            "random-pet-name": {
+                "//": {
+                    "metadata": {
+                        "path": "test/random-pet-name",
+                        "uniqueId": "random-pet-name"
+                    }
+                },
+                "provider": "random"
+            }
+        }
+    }
+}