From 9836cbe5f4256b96d5d512f1f8e254b596fcea41 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Wed, 1 May 2024 13:07:13 -0700 Subject: [PATCH] chore: Add an Subnet Controller to Asynchronously Hydrate Subnet Data (#6057) --- .../karpenter.k8s.aws_ec2nodeclasses.yaml | 2 +- pkg/cache/cache.go | 4 + pkg/cloudprovider/drift.go | 17 +-- pkg/cloudprovider/suite_test.go | 86 +++++++++--- pkg/controllers/nodeclass/status/subnet.go | 2 +- .../providers/instancetype/suite_test.go | 38 +++++- pkg/operator/operator.go | 2 +- pkg/providers/instance/instance.go | 8 +- pkg/providers/instance/suite_test.go | 46 +++++-- pkg/providers/instancetype/instancetype.go | 25 ++-- pkg/providers/instancetype/suite_test.go | 69 ++++++---- pkg/providers/launchtemplate/suite_test.go | 66 ++++++++-- pkg/providers/subnet/subnet.go | 124 ++++++++++-------- pkg/providers/subnet/suite_test.go | 8 ++ pkg/test/environment.go | 40 +++--- 15 files changed, 370 insertions(+), 167 deletions(-) diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index 15cd0e2fc34e..66ec659e7747 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: ec2nodeclasses.karpenter.k8s.aws spec: group: karpenter.k8s.aws diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 7bb586693f6d..796fa1ff9b20 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -31,6 +31,10 @@ const ( InstanceTypesAndZonesTTL = 5 * time.Minute // InstanceProfileTTL is the time before we refresh checking instance profile existence at IAM InstanceProfileTTL = 15 * time.Minute + // AvailableIPAddressTTL is time to drop AvailableIPAddress data if it is not updated within the TTL + AvailableIPAddressTTL = 2 * time.Minute + // AvailableIPAddressTTL is time to drop AssociatePublicIPAddressTTL data if it is not updated within the TTL + AssociatePublicIPAddressTTL = 2 * time.Minute ) const ( diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index 318af54ac768..f40455ad3837 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -25,9 +25,6 @@ import ( corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" @@ -58,7 +55,7 @@ func (c *CloudProvider) isNodeClassDrifted(ctx context.Context, nodeClaim *corev if err != nil { return "", fmt.Errorf("calculating securitygroup drift, %w", err) } - subnetDrifted, err := c.isSubnetDrifted(ctx, instance, nodeClass) + subnetDrifted, err := c.isSubnetDrifted(instance, nodeClass) if err != nil { return "", fmt.Errorf("calculating subnet drift, %w", err) } @@ -96,18 +93,14 @@ func (c *CloudProvider) isAMIDrifted(ctx context.Context, nodeClaim *corev1beta1 // Checks if the security groups are drifted, by comparing the subnet returned from the subnetProvider // to the ec2 instance subnets -func (c *CloudProvider) isSubnetDrifted(ctx context.Context, instance *instance.Instance, nodeClass *v1beta1.EC2NodeClass) (cloudprovider.DriftReason, error) { - subnets, err := c.subnetProvider.List(ctx, nodeClass) - if err != nil { - return "", err - } +func (c *CloudProvider) isSubnetDrifted(instance *instance.Instance, nodeClass *v1beta1.EC2NodeClass) (cloudprovider.DriftReason, error) { // subnets need to be found to check for drift - if len(subnets) == 0 { + if len(nodeClass.Status.Subnets) == 0 { return "", fmt.Errorf("no subnets are discovered") } - _, found := lo.Find(subnets, func(subnet *ec2.Subnet) bool { - return aws.StringValue(subnet.SubnetId) == instance.SubnetID + _, found := lo.Find(nodeClass.Status.Subnets, func(subnet v1beta1.Subnet) bool { + return subnet.ID == instance.SubnetID }) if !found { diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 3617355f1730..1dbf5ab68baa 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -39,10 +39,12 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" + "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" + "sigs.k8s.io/controller-runtime/pkg/client" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" corecloudproivder "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/controllers/provisioning" @@ -113,7 +115,41 @@ var _ = Describe("CloudProvider", func() { var nodePool *corev1beta1.NodePool var nodeClaim *corev1beta1.NodeClaim var _ = BeforeEach(func() { - nodeClass = test.EC2NodeClass() + nodeClass = test.EC2NodeClass( + v1beta1.EC2NodeClass{ + Status: v1beta1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + SecurityGroups: []v1beta1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + { + ID: "sg-test2", + Name: "securityGroup-test2", + }, + { + ID: "sg-test3", + Name: "securityGroup-test3", + }, + }, + Subnets: []v1beta1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + }, + }, + }, + ) nodePool = coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ Template: corev1beta1.NodeClaimTemplate{ @@ -138,20 +174,8 @@ var _ = Describe("CloudProvider", func() { }, }, }) - nodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{ - { - ID: "sg-test1", - Name: "securityGroup-test1", - }, - { - ID: "sg-test2", - Name: "securityGroup-test2", - }, - { - ID: "sg-test3", - Name: "securityGroup-test3", - }, - } + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass) // Hydrate the subnet cache + Expect(err).To(BeNil()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) @@ -600,6 +624,16 @@ var _ = Describe("CloudProvider", func() { }, }, }) + nodeClass.Status.Subnets = []v1beta1.Subnet{ + { + ID: validSubnet1, + Zone: "zone-1", + }, + { + ID: validSubnet2, + Zone: "zone-2", + }, + } nodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{ { ID: validSecurityGroup, @@ -680,7 +714,8 @@ var _ = Describe("CloudProvider", func() { }) It("should return an error if subnets are empty", func() { awsEnv.SubnetCache.Flush() - awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{}}) + nodeClass.Status.Subnets = []v1beta1.Subnet{} + ExpectApplied(ctx, env.Client, nodeClass) _, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).To(HaveOccurred()) }) @@ -797,6 +832,16 @@ var _ = Describe("CloudProvider", func() { }, }, Status: v1beta1.EC2NodeClassStatus{ + Subnets: []v1beta1.Subnet{ + { + ID: validSubnet1, + Zone: "zone-1", + }, + { + ID: validSubnet2, + Zone: "zone-2", + }, + }, SecurityGroups: []v1beta1.SecurityGroup{ { ID: validSecurityGroup, @@ -964,13 +1009,16 @@ var _ = Describe("CloudProvider", func() { Expect(foundNonGPULT).To(BeTrue()) }) It("should launch instances into subnet with the most available IP addresses", func() { + awsEnv.SubnetCache.Flush() awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(100), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -978,14 +1026,17 @@ var _ = Describe("CloudProvider", func() { Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-2")) }) It("should launch instances into subnet with the most available IP addresses in-between cache refreshes", func() { + awsEnv.SubnetCache.Flush() awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(11), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: aws.Int32(1)} ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeClass)) pod1 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) pod2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod1, pod2) @@ -1020,6 +1071,8 @@ var _ = Describe("CloudProvider", func() { }}) nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-1"}}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeClass)) podSubnet1 := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, podSubnet1) ExpectScheduled(ctx, env.Client, podSubnet1) @@ -1059,6 +1112,7 @@ var _ = Describe("CloudProvider", func() { }, }) ExpectApplied(ctx, env.Client, nodePool2, nodeClass2) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeClass2)) podSubnet2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1beta1.NodePoolLabelKey: nodePool2.Name}}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, podSubnet2) ExpectScheduled(ctx, env.Client, podSubnet2) diff --git a/pkg/controllers/nodeclass/status/subnet.go b/pkg/controllers/nodeclass/status/subnet.go index 861131fa8cbc..f2562c8336f5 100644 --- a/pkg/controllers/nodeclass/status/subnet.go +++ b/pkg/controllers/nodeclass/status/subnet.go @@ -54,5 +54,5 @@ func (s *Subnet) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) } }) - return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil + return reconcile.Result{RequeueAfter: time.Minute}, nil } diff --git a/pkg/controllers/providers/instancetype/suite_test.go b/pkg/controllers/providers/instancetype/suite_test.go index 9f3fb844b9b1..b2d7f3fca2ee 100644 --- a/pkg/controllers/providers/instancetype/suite_test.go +++ b/pkg/controllers/providers/instancetype/suite_test.go @@ -89,7 +89,24 @@ var _ = Describe("InstanceType", func() { }) ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) - instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, &v1beta1.EC2NodeClass{}) + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, &v1beta1.EC2NodeClass{ + Status: v1beta1.EC2NodeClassStatus{ + Subnets: []v1beta1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + }, + }, + }) Expect(err).To(BeNil()) for i := range instanceTypes { Expect(instanceTypes[i].Name).To(Equal(lo.FromPtr(ec2InstanceTypes[i].InstanceType))) @@ -106,7 +123,24 @@ var _ = Describe("InstanceType", func() { }) ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) - instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, &v1beta1.EC2NodeClass{}) + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, &v1beta1.EC2NodeClass{ + Status: v1beta1.EC2NodeClassStatus{ + Subnets: []v1beta1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + }, + }, + }) Expect(err).To(BeNil()) Expect(len(instanceTypes)).To(BeNumerically("==", len(ec2InstanceTypes))) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 39be55788322..113b863f9732 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -132,7 +132,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont } unavailableOfferingsCache := awscache.NewUnavailableOfferings() - subnetProvider := subnet.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) + subnetProvider := subnet.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.AvailableIPAddressTTL, awscache.DefaultCleanupInterval), cache.New(awscache.AssociatePublicIPAddressTTL, awscache.DefaultCleanupInterval)) securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) instanceProfileProvider := instanceprofile.NewDefaultProvider(*sess.Config.Region, iam.New(sess), cache.New(awscache.InstanceProfileTTL, awscache.DefaultCleanupInterval)) pricingProvider := pricing.NewDefaultProvider( diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 50abbaf45826..f228815563a7 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -285,7 +285,7 @@ func (p *DefaultProvider) checkODFallback(nodeClaim *corev1beta1.NodeClaim, inst } func (p *DefaultProvider) getLaunchTemplateConfigs(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, - instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, capacityType string, tags map[string]string) ([]*ec2.FleetLaunchTemplateConfigRequest, error) { + instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*subnet.Subnet, capacityType string, tags map[string]string) ([]*ec2.FleetLaunchTemplateConfigRequest, error) { var launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest launchTemplates, err := p.launchTemplateProvider.EnsureAll(ctx, nodeClass, nodeClaim, instanceTypes, capacityType, tags) if err != nil { @@ -311,7 +311,7 @@ func (p *DefaultProvider) getLaunchTemplateConfigs(ctx context.Context, nodeClas // getOverrides creates and returns launch template overrides for the cross product of InstanceTypes and subnets (with subnets being constrained by // zones and the offerings in InstanceTypes) -func (p *DefaultProvider) getOverrides(instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, zones *scheduling.Requirement, capacityType string, image string) []*ec2.FleetLaunchTemplateOverridesRequest { +func (p *DefaultProvider) getOverrides(instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*subnet.Subnet, zones *scheduling.Requirement, capacityType string, image string) []*ec2.FleetLaunchTemplateOverridesRequest { // Unwrap all the offerings to a flat slice that includes a pointer // to the parent instance type name type offeringWithParentName struct { @@ -343,11 +343,11 @@ func (p *DefaultProvider) getOverrides(instanceTypes []*cloudprovider.InstanceTy } overrides = append(overrides, &ec2.FleetLaunchTemplateOverridesRequest{ InstanceType: aws.String(offering.parentInstanceTypeName), - SubnetId: subnet.SubnetId, + SubnetId: lo.ToPtr(subnet.ID), ImageId: aws.String(image), // This is technically redundant, but is useful if we have to parse insufficient capacity errors from // CreateFleet so that we can figure out the zone rather than additional API calls to look up the subnet - AvailabilityZone: subnet.AvailabilityZone, + AvailabilityZone: lo.ToPtr(subnet.Zone), }) } return overrides diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 8dc051795d10..723cfcd8c16b 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -82,7 +82,38 @@ var _ = Describe("InstanceProvider", func() { var nodePool *corev1beta1.NodePool var nodeClaim *corev1beta1.NodeClaim BeforeEach(func() { - nodeClass = test.EC2NodeClass() + nodeClass = test.EC2NodeClass( + v1beta1.EC2NodeClass{ + Status: v1beta1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + SecurityGroups: []v1beta1.SecurityGroup{ + { + ID: "sg-test1", + }, + { + ID: "sg-test2", + }, + { + ID: "sg-test3", + }, + }, + Subnets: []v1beta1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + }, + }, + }, + ) nodePool = coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ Template: corev1beta1.NodeClaimTemplate{ @@ -106,17 +137,8 @@ var _ = Describe("InstanceProvider", func() { }, }, }) - nodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{ - { - ID: "sg-test1", - }, - { - ID: "sg-test2", - }, - { - ID: "sg-test3", - }, - } + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass) // Hydrate the subnet cache + Expect(err).To(BeNil()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index 1caaebccdbe1..0d86dd941e8a 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -101,27 +101,26 @@ func (p *DefaultProvider) List(ctx context.Context, kc *corev1beta1.KubeletConfi defer p.muInstanceTypeInfo.RUnlock() defer p.muInstanceTypeOfferings.RUnlock() + if kc == nil { + kc = &corev1beta1.KubeletConfiguration{} + } + if nodeClass == nil { + nodeClass = &v1beta1.EC2NodeClass{} + } + if len(p.instanceTypesInfo) == 0 { return nil, fmt.Errorf("no instance types found") } if len(p.instanceTypeOfferings) == 0 { return nil, fmt.Errorf("no instance types offerings found") } - - subnets, err := p.subnetProvider.List(ctx, nodeClass) - if err != nil { - return nil, err + if len(nodeClass.Status.Subnets) == 0 { + return nil, fmt.Errorf("no subnets found") } - subnetZones := sets.New[string](lo.Map(subnets, func(s *ec2.Subnet, _ int) string { - return aws.StringValue(s.AvailabilityZone) - })...) - if kc == nil { - kc = &corev1beta1.KubeletConfiguration{} - } - if nodeClass == nil { - nodeClass = &v1beta1.EC2NodeClass{} - } + subnetZones := sets.New(lo.Map(nodeClass.Status.Subnets, func(s v1beta1.Subnet, _ int) string { + return aws.StringValue(&s.Zone) + })...) // Compute fully initialized instance types hash key subnetZonesHash, _ := hashstructure.Hash(subnetZones, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index a21937e83e1e..217a5bd1e0bd 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -108,7 +108,38 @@ var _ = Describe("InstanceTypeProvider", func() { var nodeClass, windowsNodeClass *v1beta1.EC2NodeClass var nodePool, windowsNodePool *corev1beta1.NodePool BeforeEach(func() { - nodeClass = test.EC2NodeClass() + nodeClass = test.EC2NodeClass( + v1beta1.EC2NodeClass{ + Status: v1beta1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + SecurityGroups: []v1beta1.SecurityGroup{ + { + ID: "sg-test1", + }, + { + ID: "sg-test2", + }, + { + ID: "sg-test3", + }, + }, + Subnets: []v1beta1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + }, + }, + }, + ) nodePool = coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ Template: corev1beta1.NodeClaimTemplate{ @@ -134,6 +165,11 @@ var _ = Describe("InstanceTypeProvider", func() { Spec: v1beta1.EC2NodeClassSpec{ AMIFamily: &v1beta1.AMIFamilyWindows2022, }, + Status: v1beta1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + SecurityGroups: nodeClass.Status.SecurityGroups, + Subnets: nodeClass.Status.Subnets, + }, }) windowsNodePool = coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ @@ -155,28 +191,8 @@ var _ = Describe("InstanceTypeProvider", func() { }, }, }) - nodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{ - { - ID: "sg-test1", - }, - { - ID: "sg-test2", - }, - { - ID: "sg-test3", - }, - } - windowsNodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{ - { - ID: "sg-test1", - }, - { - ID: "sg-test2", - }, - { - ID: "sg-test3", - }, - } + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass) // Hydrate the subnet cache + Expect(err).To(BeNil()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) @@ -873,6 +889,12 @@ var _ = Describe("InstanceTypeProvider", func() { }) }) It("should launch instances in local zones", func() { + nodeClass.Status.Subnets = []v1beta1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a-local", + }, + } ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{{ @@ -883,7 +905,6 @@ var _ = Describe("InstanceTypeProvider", func() { }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) - }) Context("Overhead", func() { diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 5ae89e604bf5..48bf35f5fc73 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -57,6 +57,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" + "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" @@ -120,7 +121,38 @@ var _ = Describe("LaunchTemplate Provider", func() { var nodePool *corev1beta1.NodePool var nodeClass *v1beta1.EC2NodeClass BeforeEach(func() { - nodeClass = test.EC2NodeClass() + nodeClass = test.EC2NodeClass( + v1beta1.EC2NodeClass{ + Status: v1beta1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + SecurityGroups: []v1beta1.SecurityGroup{ + { + ID: "sg-test1", + }, + { + ID: "sg-test2", + }, + { + ID: "sg-test3", + }, + }, + Subnets: []v1beta1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + }, + }, + }, + ) nodePool = coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ Template: corev1beta1.NodeClaimTemplate{ @@ -146,17 +178,8 @@ var _ = Describe("LaunchTemplate Provider", func() { }, }, }) - nodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{ - { - ID: "sg-test1", - }, - { - ID: "sg-test2", - }, - { - ID: "sg-test3", - }, - } + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass) // Hydrate the subnet cache + Expect(err).To(BeNil()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) @@ -193,6 +216,21 @@ var _ = Describe("LaunchTemplate Provider", func() { ID: "sg-test3", }, } + nodeClass2.Status.Subnets = []v1beta1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + } + pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ { @@ -1973,6 +2011,8 @@ var _ = Describe("LaunchTemplate Provider", func() { {Tags: map[string]string{"Name": "test-subnet-3"}}, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1984,6 +2024,8 @@ var _ = Describe("LaunchTemplate Provider", func() { {Tags: map[string]string{"Name": "test-subnet-2"}}, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) diff --git a/pkg/providers/subnet/subnet.go b/pkg/providers/subnet/subnet.go index 37af40718b25..692bae822a6b 100644 --- a/pkg/providers/subnet/subnet.go +++ b/pkg/providers/subnet/subnet.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "net/http" - "sort" "sync" "github.com/aws/aws-sdk-go/aws" @@ -39,25 +38,35 @@ type Provider interface { LivenessProbe(*http.Request) error List(context.Context, *v1beta1.EC2NodeClass) ([]*ec2.Subnet, error) CheckAnyPublicIPAssociations(context.Context, *v1beta1.EC2NodeClass) (bool, error) - ZonalSubnetsForLaunch(context.Context, *v1beta1.EC2NodeClass, []*cloudprovider.InstanceType, string) (map[string]*ec2.Subnet, error) - UpdateInflightIPs(*ec2.CreateFleetInput, *ec2.CreateFleetOutput, []*cloudprovider.InstanceType, []*ec2.Subnet, string) + ZonalSubnetsForLaunch(context.Context, *v1beta1.EC2NodeClass, []*cloudprovider.InstanceType, string) (map[string]*Subnet, error) + UpdateInflightIPs(*ec2.CreateFleetInput, *ec2.CreateFleetOutput, []*cloudprovider.InstanceType, []*Subnet, string) } type DefaultProvider struct { sync.RWMutex - ec2api ec2iface.EC2API - cache *cache.Cache - cm *pretty.ChangeMonitor - inflightIPs map[string]int64 + ec2api ec2iface.EC2API + cache *cache.Cache + availableIPAddressCache *cache.Cache + associatePublicIPAddressCache *cache.Cache + cm *pretty.ChangeMonitor + inflightIPs map[string]int64 } -func NewDefaultProvider(ec2api ec2iface.EC2API, cache *cache.Cache) *DefaultProvider { +type Subnet struct { + ID string + Zone string + AvailableIPAddressCount int64 +} + +func NewDefaultProvider(ec2api ec2iface.EC2API, cache *cache.Cache, availableIPAddressCache *cache.Cache, associatePublicIPAddressCache *cache.Cache) *DefaultProvider { return &DefaultProvider{ ec2api: ec2api, cm: pretty.NewChangeMonitor(), // TODO: Remove cache when we utilize the resolved subnets from the EC2NodeClass.status // Subnets are sorted on AvailableIpAddressCount, descending order - cache: cache, + cache: cache, + availableIPAddressCache: availableIPAddressCache, + associatePublicIPAddressCache: associatePublicIPAddressCache, // inflightIPs is used to track IPs from known launched instances inflightIPs: map[string]int64{}, } @@ -87,6 +96,10 @@ func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1beta1.EC2NodeCl } for i := range output.Subnets { subnets[lo.FromPtr(output.Subnets[i].SubnetId)] = output.Subnets[i] + p.availableIPAddressCache.SetDefault(lo.FromPtr(output.Subnets[i].SubnetId), lo.FromPtr(output.Subnets[i].AvailableIpAddressCount)) + p.associatePublicIPAddressCache.SetDefault(lo.FromPtr(output.Subnets[i].SubnetId), lo.FromPtr(output.Subnets[i].MapPublicIpOnLaunch)) + // subnets can be leaked here, if a subnets is never called received from ec2 + // we are accepting it for now, as this will be an insignificant amount of memory delete(p.inflightIPs, lo.FromPtr(output.Subnets[i].SubnetId)) // remove any previously tracked IP addresses since we just refreshed from EC2 } } @@ -103,58 +116,63 @@ func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1beta1.EC2NodeCl // CheckAnyPublicIPAssociations returns a bool indicating whether all referenced subnets assign public IPv4 addresses to EC2 instances created therein func (p *DefaultProvider) CheckAnyPublicIPAssociations(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (bool, error) { - subnets, err := p.List(ctx, nodeClass) - if err != nil { - return false, err + for _, subnet := range nodeClass.Status.Subnets { + if subnetAssociatePublicIP, ok := p.associatePublicIPAddressCache.Get(subnet.ID); ok && subnetAssociatePublicIP.(bool) { + return true, nil + } } - _, ok := lo.Find(subnets, func(s *ec2.Subnet) bool { - return aws.BoolValue(s.MapPublicIpOnLaunch) - }) - return ok, nil + return false, nil } // ZonalSubnetsForLaunch returns a mapping of zone to the subnet with the most available IP addresses and deducts the passed ips from the available count -func (p *DefaultProvider) ZonalSubnetsForLaunch(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, instanceTypes []*cloudprovider.InstanceType, capacityType string) (map[string]*ec2.Subnet, error) { - subnets, err := p.List(ctx, nodeClass) - if err != nil { - return nil, err - } - if len(subnets) == 0 { +func (p *DefaultProvider) ZonalSubnetsForLaunch(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, instanceTypes []*cloudprovider.InstanceType, capacityType string) (map[string]*Subnet, error) { + if len(nodeClass.Status.Subnets) == 0 { return nil, fmt.Errorf("no subnets matched selector %v", nodeClass.Spec.SubnetSelectorTerms) } + p.Lock() defer p.Unlock() - // sort subnets in ascending order of available IP addresses and populate map with most available subnet per AZ - zonalSubnets := map[string]*ec2.Subnet{} - sort.Slice(subnets, func(i, j int) bool { - iIPs := aws.Int64Value(subnets[i].AvailableIpAddressCount) - jIPs := aws.Int64Value(subnets[j].AvailableIpAddressCount) - // override ip count from ec2.Subnet if we've tracked launches - if ips, ok := p.inflightIPs[*subnets[i].SubnetId]; ok { - iIPs = ips + + zonalSubnets := map[string]*Subnet{} + availableIPAddressCount := map[string]int64{} + for _, subnet := range nodeClass.Status.Subnets { + if subnetAvailableIP, ok := p.availableIPAddressCache.Get(subnet.ID); ok { + availableIPAddressCount[subnet.ID] = subnetAvailableIP.(int64) } - if ips, ok := p.inflightIPs[*subnets[j].SubnetId]; ok { - jIPs = ips + } + + for _, subnet := range nodeClass.Status.Subnets { + if v, ok := zonalSubnets[subnet.Zone]; ok { + currentZonalSubnetIPAddressCount := v.AvailableIPAddressCount + newZonalSubnetIPAddressCount := availableIPAddressCount[subnet.ID] + if ips, ok := p.inflightIPs[v.ID]; ok { + currentZonalSubnetIPAddressCount = ips + } + if ips, ok := p.inflightIPs[subnet.ID]; ok { + newZonalSubnetIPAddressCount = ips + } + + if currentZonalSubnetIPAddressCount >= newZonalSubnetIPAddressCount { + continue + } } - return iIPs < jIPs - }) - for _, subnet := range subnets { - zonalSubnets[*subnet.AvailabilityZone] = subnet + zonalSubnets[subnet.Zone] = &Subnet{ID: subnet.ID, Zone: subnet.Zone, AvailableIPAddressCount: availableIPAddressCount[subnet.ID]} } + for _, subnet := range zonalSubnets { - predictedIPsUsed := p.minPods(instanceTypes, *subnet.AvailabilityZone, capacityType) - prevIPs := *subnet.AvailableIpAddressCount - if trackedIPs, ok := p.inflightIPs[*subnet.SubnetId]; ok { + predictedIPsUsed := p.minPods(instanceTypes, subnet.Zone, capacityType) + prevIPs := subnet.AvailableIPAddressCount + if trackedIPs, ok := p.inflightIPs[subnet.ID]; ok { prevIPs = trackedIPs } - p.inflightIPs[*subnet.SubnetId] = prevIPs - predictedIPsUsed + p.inflightIPs[subnet.ID] = prevIPs - predictedIPsUsed } return zonalSubnets, nil } // UpdateInflightIPs is used to refresh the in-memory IP usage by adding back unused IPs after a CreateFleet response is returned func (p *DefaultProvider) UpdateInflightIPs(createFleetInput *ec2.CreateFleetInput, createFleetOutput *ec2.CreateFleetOutput, instanceTypes []*cloudprovider.InstanceType, - subnets []*ec2.Subnet, capacityType string) { + subnets []*Subnet, capacityType string) { p.Lock() defer p.Unlock() @@ -182,30 +200,30 @@ func (p *DefaultProvider) UpdateInflightIPs(createFleetInput *ec2.CreateFleetInp // Find the subnets that were included in the input but not chosen by Fleet, so we need to add the inflight IPs back to them subnetIDsToAddBackIPs, _ := lo.Difference(fleetInputSubnets, fleetOutputSubnets) - // Aggregate all the cached subnets - cachedSubnets := lo.UniqBy(lo.Flatten(lo.MapToSlice(p.cache.Items(), func(_ string, item cache.Item) []*ec2.Subnet { - return item.Object.([]*ec2.Subnet) - })), func(subnet *ec2.Subnet) string { return *subnet.SubnetId }) + // Aggregate all the cached subnets ip address count + cachedAvailableIPAddressMap := lo.MapEntries(p.availableIPAddressCache.Items(), func(k string, v cache.Item) (string, int64) { + return k, v.Object.(int64) + }) // Update the inflight IP tracking of subnets stored in the cache that have not be synchronized since the initial // deduction of IP addresses before the instance launch - for _, cachedSubnet := range cachedSubnets { - if !lo.Contains(subnetIDsToAddBackIPs, *cachedSubnet.SubnetId) { + for cachedSubnetID, cachedIPAddressCount := range cachedAvailableIPAddressMap { + if !lo.Contains(subnetIDsToAddBackIPs, cachedSubnetID) { continue } - originalSubnet, ok := lo.Find(subnets, func(subnet *ec2.Subnet) bool { - return *subnet.SubnetId == *cachedSubnet.SubnetId + originalSubnet, ok := lo.Find(subnets, func(subnet *Subnet) bool { + return subnet.ID == cachedSubnetID }) if !ok { continue } // If the cached subnet IP address count hasn't changed from the original subnet used to // launch the instance, then we need to update the tracked IPs - if *originalSubnet.AvailableIpAddressCount == *cachedSubnet.AvailableIpAddressCount { + if originalSubnet.AvailableIPAddressCount == cachedIPAddressCount { // other IPs deducted were opportunistic and need to be readded since Fleet didn't pick those subnets to launch into - if ips, ok := p.inflightIPs[*originalSubnet.SubnetId]; ok { - minPods := p.minPods(instanceTypes, *originalSubnet.AvailabilityZone, capacityType) - p.inflightIPs[*originalSubnet.SubnetId] = ips + minPods + if ips, ok := p.inflightIPs[originalSubnet.ID]; ok { + minPods := p.minPods(instanceTypes, originalSubnet.Zone, capacityType) + p.inflightIPs[originalSubnet.ID] = ips + minPods } } } diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index 08b7354f7924..ff5c6146b706 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -235,6 +235,14 @@ var _ = Describe("SubnetProvider", func() { ID: "subnet-test2", }, } + nodeClass.Status.Subnets = []v1beta1.Subnet{ + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + } + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass) + Expect(err).To(BeNil()) onlyPrivate, err := awsEnv.SubnetProvider.CheckAnyPublicIPAssociations(ctx, nodeClass) Expect(err).To(BeNil()) Expect(onlyPrivate).To(BeTrue()) diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 8c29b954ddf9..81aa70575470 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -58,14 +58,16 @@ type Environment struct { PricingAPI *fake.PricingAPI // Cache - EC2Cache *cache.Cache - KubernetesVersionCache *cache.Cache - InstanceTypeCache *cache.Cache - UnavailableOfferingsCache *awscache.UnavailableOfferings - LaunchTemplateCache *cache.Cache - SubnetCache *cache.Cache - SecurityGroupCache *cache.Cache - InstanceProfileCache *cache.Cache + EC2Cache *cache.Cache + KubernetesVersionCache *cache.Cache + InstanceTypeCache *cache.Cache + UnavailableOfferingsCache *awscache.UnavailableOfferings + LaunchTemplateCache *cache.Cache + SubnetCache *cache.Cache + AvailableIPAdressCache *cache.Cache + AssociatePublicIPAddressCache *cache.Cache + SecurityGroupCache *cache.Cache + InstanceProfileCache *cache.Cache // Providers InstanceTypesProvider *instancetype.DefaultProvider @@ -94,13 +96,15 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment unavailableOfferingsCache := awscache.NewUnavailableOfferings() launchTemplateCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) subnetCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + availableIPAdressCache := cache.New(awscache.AvailableIPAddressTTL, awscache.DefaultCleanupInterval) + associatePublicIPAddressCache := cache.New(awscache.AssociatePublicIPAddressTTL, awscache.DefaultCleanupInterval) securityGroupCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) instanceProfileCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) fakePricingAPI := &fake.PricingAPI{} // Providers pricingProvider := pricing.NewDefaultProvider(ctx, fakePricingAPI, ec2api, fake.DefaultRegion) - subnetProvider := subnet.NewDefaultProvider(ec2api, subnetCache) + subnetProvider := subnet.NewDefaultProvider(ec2api, subnetCache, availableIPAdressCache, associatePublicIPAddressCache) securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, securityGroupCache) versionProvider := version.NewDefaultProvider(env.KubernetesInterface, kubernetesVersionCache) instanceProfileProvider := instanceprofile.NewDefaultProvider(fake.DefaultRegion, iamapi, instanceProfileCache) @@ -138,13 +142,15 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment IAMAPI: iamapi, PricingAPI: fakePricingAPI, - EC2Cache: ec2Cache, - KubernetesVersionCache: kubernetesVersionCache, - LaunchTemplateCache: launchTemplateCache, - SubnetCache: subnetCache, - SecurityGroupCache: securityGroupCache, - InstanceProfileCache: instanceProfileCache, - UnavailableOfferingsCache: unavailableOfferingsCache, + EC2Cache: ec2Cache, + KubernetesVersionCache: kubernetesVersionCache, + LaunchTemplateCache: launchTemplateCache, + SubnetCache: subnetCache, + AvailableIPAdressCache: availableIPAdressCache, + AssociatePublicIPAddressCache: associatePublicIPAddressCache, + SecurityGroupCache: securityGroupCache, + InstanceProfileCache: instanceProfileCache, + UnavailableOfferingsCache: unavailableOfferingsCache, InstanceTypesProvider: instanceTypesProvider, InstanceProvider: instanceProvider, @@ -173,6 +179,8 @@ func (env *Environment) Reset() { env.UnavailableOfferingsCache.Flush() env.LaunchTemplateCache.Flush() env.SubnetCache.Flush() + env.AssociatePublicIPAddressCache.Flush() + env.AvailableIPAdressCache.Flush() env.SecurityGroupCache.Flush() env.InstanceProfileCache.Flush()