From 1867887df8f597eca7a1fbe7d1d8407ea4f239cc Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Wed, 15 Jan 2025 15:27:41 +0300 Subject: [PATCH] move crd installer to module sdk Signed-off-by: Pavel Okhlopkov --- go.mod | 7 +- go.sum | 9 +- pkg/addon-operator/ensure_crds.go | 304 +----------------- .../models/hooks/kind/shellhook.go | 2 +- 4 files changed, 14 insertions(+), 308 deletions(-) diff --git a/go.mod b/go.mod index 2912da1d..086a620d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22.8 require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241205040953-7b376bae249c - github.com/deckhouse/module-sdk v0.1.0 + github.com/deckhouse/module-sdk v0.1.1-0.20250115122429-d9c670ed1a02 github.com/dominikbraun/graph v0.23.0 github.com/ettle/strcase v0.2.0 github.com/flant/kube-client v1.2.2 @@ -18,7 +18,6 @@ require ( github.com/go-openapi/validate v0.19.12 github.com/goccy/go-graphviz v0.1.3 github.com/gofrs/uuid/v5 v5.3.0 - github.com/google/go-cmp v0.6.0 github.com/hashicorp/go-multierror v1.1.1 github.com/kennygrant/sanitize v1.2.4 github.com/onsi/gomega v1.35.1 @@ -29,7 +28,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.14.2 k8s.io/api v0.29.8 - k8s.io/apiextensions-apiserver v0.29.0 k8s.io/apimachinery v0.29.8 k8s.io/cli-runtime v0.29.0 k8s.io/client-go v0.29.8 @@ -92,6 +90,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect @@ -137,6 +136,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rubenv/sql-migrate v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sergi/go-diff v1.2.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect @@ -170,6 +170,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apiextensions-apiserver v0.29.0 // indirect k8s.io/apiserver v0.29.0 // indirect k8s.io/component-base v0.29.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect diff --git a/go.sum b/go.sum index ec4cf33c..04292444 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241205040953-7b376bae249c h1:dK30IW9uGg0DvSy+IcdQ6zwEBRV55R7tEtaruEKYkSA= github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241205040953-7b376bae249c/go.mod h1:Mk5HRzkc5pIcDIZ2JJ6DPuuqnwhXVkb3you8M8Mg+4w= -github.com/deckhouse/module-sdk v0.1.0 h1:YgMnr0vmjVLwIdQfTE1ytR2nPndO2SfQprcW45wsT34= -github.com/deckhouse/module-sdk v0.1.0/go.mod h1:0gxlSr0WRrGnB82J+45FBIXNP/fKl8+yEvRTzrygiAE= +github.com/deckhouse/module-sdk v0.1.1-0.20250115122429-d9c670ed1a02 h1:bA1hK30GLUU+LvpNzLJZVwJgt10lhp5alkSRHxa0zuY= +github.com/deckhouse/module-sdk v0.1.1-0.20250115122429-d9c670ed1a02/go.mod h1:79kILbbIL4e3B/8nwEGYsox8vH8v27LxFDTFu0OhBtc= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/distribution/v3 v3.0.0-beta.1 h1:X+ELTxPuZ1Xe5MsD3kp2wfGUhc8I+MPfRis8dZ818Ic= @@ -496,8 +496,8 @@ github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWx github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -756,6 +756,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/pkg/addon-operator/ensure_crds.go b/pkg/addon-operator/ensure_crds.go index a442117d..747d968c 100644 --- a/pkg/addon-operator/ensure_crds.go +++ b/pkg/addon-operator/ensure_crds.go @@ -1,322 +1,26 @@ package addon_operator import ( - "bytes" "context" - "fmt" - "io" - "os" - "sync" - "github.com/google/go-cmp/cmp" - "github.com/hashicorp/go-multierror" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - apimachineryv1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - apimachineryYaml "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/util/retry" + crdinstaller "github.com/deckhouse/module-sdk/pkg/crd-installer" "github.com/flant/addon-operator/pkg/module_manager/models/modules" - "github.com/flant/addon-operator/sdk" ) -// 1Mb - maximum size of kubernetes object -// if we take less, we have to handle io.ErrShortBuffer error and increase the buffer -// take more does not make any sense due to kubernetes limitations -// Considering that etcd has a default value of 1.5Mb, it was decided to set it to 2Mb, -// so that in most cases we would get a more informative error from Kubernetes, not just "short buffer" -const bufSize = 2 * 1024 * 1024 - -var crdGVR = schema.GroupVersionResource{ - Group: "apiextensions.k8s.io", - Version: "v1", - Resource: "customresourcedefinitions", -} - func (op *AddonOperator) EnsureCRDs(module *modules.BasicModule) ([]string, error) { // do not ensure CRDs if there are no files if !module.CRDExist() { return nil, nil } - cp := NewCRDsInstaller(op.KubeClient().Dynamic(), module.GetCRDFilesPaths(), WithExtraLabels(op.CRDExtraLabels)) + cp := crdinstaller.NewCRDsInstaller(op.KubeClient().Dynamic(), module.GetCRDFilesPaths(), crdinstaller.WithExtraLabels(op.CRDExtraLabels)) if cp == nil { return nil, nil } - if runErr := cp.Run(context.TODO()).ErrorOrNil(); runErr != nil { - return nil, runErr - } - - return cp.appliedGVKs, nil -} - -func WithExtraLabels(labels map[string]string) InstallerOption { - return func(installer *CRDsInstaller) { - installer.crdExtraLabels = labels - } -} - -func WithFileFilter(fn func(path string) bool) InstallerOption { - return func(installer *CRDsInstaller) { - installer.fileFilter = fn - } -} - -// CRDsInstaller simultaneously installs CRDs from specified directory -type CRDsInstaller struct { - k8sClient dynamic.Interface - crdFilesPaths []string - buffer []byte - - // concurrent tasks to create resource in a k8s cluster - k8sTasks *multierror.Group - - crdExtraLabels map[string]string - fileFilter func(path string) bool - - appliedGVKsLock sync.Mutex - // list of GVKs, applied to the cluster - appliedGVKs []string -} - -func (cp *CRDsInstaller) Run(ctx context.Context) *multierror.Error { - result := new(multierror.Error) - - for _, crdFilePath := range cp.crdFilesPaths { - if cp.fileFilter != nil && !cp.fileFilter(crdFilePath) { - continue - } - - err := cp.processCRD(ctx, crdFilePath) - if err != nil { - err = fmt.Errorf("error occurred during processing %q file: %w", crdFilePath, err) - result = multierror.Append(result, err) - continue - } - } - - errs := cp.k8sTasks.Wait() - if errs.ErrorOrNil() != nil { - result = multierror.Append(result, errs.Errors...) - } - - return result -} - -func (cp *CRDsInstaller) DeleteCRDs(ctx context.Context, crdsToDelete []string) ([]string, error) { - var deletedCRDs []string - // delete crds listed in crdsToDelete if there are no related custom resources in the cluster - for _, crdName := range crdsToDelete { - deleteCRD := true - crd, err := cp.getCRDFromCluster(ctx, crdName) - if err != nil { - if !apierrors.IsNotFound(err) { - return nil, fmt.Errorf("error occurred during %s CRD clean up: %w", crdName, err) - } - continue - } - - for _, version := range crd.Spec.Versions { - if !version.Storage { - continue - } - - gvr := schema.GroupVersionResource{ - Group: crd.Spec.Group, - Version: version.Name, - Resource: crd.Spec.Names.Plural, - } - list, err := cp.k8sClient.Resource(gvr).List(ctx, apimachineryv1.ListOptions{}) - if err != nil { - return nil, fmt.Errorf("error occurred listing %s CRD objects of version %s: %w", crdName, version.Name, err) - } - if len(list.Items) > 0 { - deleteCRD = false - break - } - } - - if deleteCRD { - err := cp.k8sClient.Resource(crdGVR).Delete(ctx, crdName, apimachineryv1.DeleteOptions{}) - if err != nil { - return nil, fmt.Errorf("error occurred deleting %s CRD: %w", crdName, err) - } - deletedCRDs = append(deletedCRDs, crdName) - } - } - return deletedCRDs, nil -} - -func (cp *CRDsInstaller) processCRD(ctx context.Context, crdFilePath string) error { - crdFileReader, err := os.Open(crdFilePath) - if err != nil { - return err - } - defer crdFileReader.Close() - - crdReader := apimachineryYaml.NewDocumentDecoder(crdFileReader) - - for { - n, err := crdReader.Read(cp.buffer) - if err != nil { - if err == io.EOF { - break - } - - return err - } - - data := cp.buffer[:n] - if len(data) == 0 { - // some empty yaml document, or empty string before separator - continue - } - rd := bytes.NewReader(data) - err = cp.putCRDToCluster(ctx, rd, n) - if err != nil { - return err - } - } - - return nil -} - -func (cp *CRDsInstaller) putCRDToCluster(ctx context.Context, crdReader io.Reader, bufferSize int) error { - var crd *v1.CustomResourceDefinition - - err := apimachineryYaml.NewYAMLOrJSONDecoder(crdReader, bufferSize).Decode(&crd) - if err != nil { - return err - } - - // it could be a comment or some other peace of yaml file, skip it - if crd == nil { - return nil - } - - if crd.APIVersion != v1.SchemeGroupVersion.String() && crd.Kind != "CustomResourceDefinition" { - return fmt.Errorf("invalid CRD document apiversion/kind: '%s/%s'", crd.APIVersion, crd.Kind) - } - - if len(crd.ObjectMeta.Labels) == 0 { - crd.ObjectMeta.Labels = make(map[string]string, 1) - } - for crdExtraLabel := range cp.crdExtraLabels { - crd.ObjectMeta.Labels[crdExtraLabel] = cp.crdExtraLabels[crdExtraLabel] - } - - cp.k8sTasks.Go(func() error { - opErr := cp.updateOrInsertCRD(ctx, crd) - if opErr == nil { - var crdGroup, crdKind string - crdVersions := make([]string, 0) - if len(crd.Spec.Group) > 0 { - crdGroup = crd.Spec.Group - } else { - return fmt.Errorf("process %s: couldn't find CRD's .group key", crd.Name) - } - - if len(crd.Spec.Names.Kind) > 0 { - crdKind = crd.Spec.Names.Kind - } else { - return fmt.Errorf("process %s: couldn't find CRD's .spec.names.kind key", crd.Name) - } - - if len(crd.Spec.Versions) > 0 { - for _, version := range crd.Spec.Versions { - crdVersions = append(crdVersions, version.Name) - } - } else { - return fmt.Errorf("process %s: couldn't find CRD's .spec.versions key", crd.Name) - } - cp.appliedGVKsLock.Lock() - for _, crdVersion := range crdVersions { - cp.appliedGVKs = append(cp.appliedGVKs, fmt.Sprintf("%s/%s/%s", crdGroup, crdVersion, crdKind)) - } - cp.appliedGVKsLock.Unlock() - } - return opErr - }) - - return nil -} - -func (cp *CRDsInstaller) updateOrInsertCRD(ctx context.Context, crd *v1.CustomResourceDefinition) error { - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - existCRD, err := cp.getCRDFromCluster(ctx, crd.GetName()) - if err != nil { - if apierrors.IsNotFound(err) { - ucrd, err := sdk.ToUnstructured(crd) - if err != nil { - return err - } - - _, err = cp.k8sClient.Resource(crdGVR).Create(ctx, ucrd, apimachineryv1.CreateOptions{}) - return err - } - - return err - } - - if existCRD.Spec.Conversion != nil { - crd.Spec.Conversion = existCRD.Spec.Conversion - } - - if cmp.Equal(existCRD.Spec, crd.Spec) && - cmp.Equal(existCRD.GetLabels(), crd.GetLabels()) && - cmp.Equal(existCRD.GetAnnotations(), crd.GetAnnotations()) { - return nil - } - - existCRD.Spec = crd.Spec - if len(existCRD.ObjectMeta.Labels) == 0 { - existCRD.ObjectMeta.Labels = make(map[string]string, 1) - } - existCRD.ObjectMeta.Labels[LabelHeritage] = cp.crdExtraLabels[LabelHeritage] - - ucrd, err := sdk.ToUnstructured(existCRD) - if err != nil { - return err - } - - _, err = cp.k8sClient.Resource(crdGVR).Update(ctx, ucrd, apimachineryv1.UpdateOptions{}) - return err - }) -} - -func (cp *CRDsInstaller) getCRDFromCluster(ctx context.Context, crdName string) (*v1.CustomResourceDefinition, error) { - crd := &v1.CustomResourceDefinition{} - - o, err := cp.k8sClient.Resource(crdGVR).Get(ctx, crdName, apimachineryv1.GetOptions{}) - if err != nil { + if err := cp.Run(context.TODO()); err != nil { return nil, err } - err = sdk.FromUnstructured(o, &crd) - if err != nil { - return nil, err - } - - return crd, nil -} - -type InstallerOption func(*CRDsInstaller) - -// NewCRDsInstaller creates new installer for CRDs -func NewCRDsInstaller(client dynamic.Interface, crdFilesPaths []string, options ...InstallerOption) *CRDsInstaller { - i := &CRDsInstaller{ - k8sClient: client, - crdFilesPaths: crdFilesPaths, - buffer: make([]byte, bufSize), - k8sTasks: &multierror.Group{}, - appliedGVKs: make([]string, 0), - } - - for _, opt := range options { - opt(i) - } - - return i + return cp.GetAppliedGVKs(), nil } diff --git a/pkg/module_manager/models/hooks/kind/shellhook.go b/pkg/module_manager/models/hooks/kind/shellhook.go index ff9dd06b..7ecc2998 100644 --- a/pkg/module_manager/models/hooks/kind/shellhook.go +++ b/pkg/module_manager/models/hooks/kind/shellhook.go @@ -10,7 +10,7 @@ import ( "github.com/deckhouse/deckhouse/pkg/log" "github.com/go-openapi/spec" "github.com/gofrs/uuid/v5" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" gohook "github.com/flant/addon-operator/pkg/module_manager/go_hook" "github.com/flant/addon-operator/pkg/utils"