diff --git a/api/cmd/api/setup.go b/api/cmd/api/setup.go index 99d58e28b..03fa6e61d 100644 --- a/api/cmd/api/setup.go +++ b/api/cmd/api/setup.go @@ -15,6 +15,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "gorm.io/gorm" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/utils/clock" @@ -98,6 +99,11 @@ func initImageBuilder(cfg *config.Config) (webserviceBuilder imagebuilder.ImageB log.Panicf("%s, unable to initialize image builder", err.Error()) } + _, err = kubeClient.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{}) + if err != nil { + log.Panicf("%s, error sending request to kube client at startup to verify connection", err.Error()) + } + timeout, err := time.ParseDuration(cfg.ImageBuilderConfig.BuildTimeout) if err != nil { log.Panicf("unable to parse image builder timeout to time.Duration %s", cfg.ImageBuilderConfig.BuildTimeout) @@ -132,6 +138,7 @@ func initImageBuilder(cfg *config.Config) (webserviceBuilder imagebuilder.ImageB KanikoDockerCredentialSecretName: cfg.ImageBuilderConfig.KanikoDockerCredentialSecretName, KanikoServiceAccount: cfg.ImageBuilderConfig.KanikoServiceAccount, KanikoAdditionalArgs: cfg.ImageBuilderConfig.KanikoAdditionalArgs, + KanikoAPIServerEnvVars: cfg.ImageBuilderConfig.KanikoAPIServerEnvVars, DefaultResources: cfg.ImageBuilderConfig.DefaultResources, Tolerations: cfg.ImageBuilderConfig.Tolerations, NodeSelectors: cfg.ImageBuilderConfig.NodeSelectors, diff --git a/api/config/config.go b/api/config/config.go index 15dc76784..73dfa6b75 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -221,6 +221,7 @@ type ImageBuilderConfig struct { KanikoPushRegistryType string `validate:"required,oneof=docker gcr" default:"docker"` KanikoDockerCredentialSecretName string KanikoAdditionalArgs []string + KanikoAPIServerEnvVars []string DefaultResources ResourceRequestsLimits `validate:"required"` // How long to keep the image building job resource in the Kubernetes cluster. Default: 2 days (48 hours). Retention time.Duration `validate:"required" default:"48h"` @@ -458,8 +459,11 @@ type MlflowConfig struct { // Note that the Kaniko image builder needs to be configured correctly to have the necessary credentials to download // the artifacts from the blob storage tool depending on the artifact service type selected (gcs/s3). For gcs, the // credentials can be provided via a k8s service account or a secret but for s3, the credentials can be provided via - // additional arguments in the config KanikoAdditionalArgs e.g. + // 1) additional arguments in the config KanikoAdditionalArgs e.g. // --build-arg=[AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_DEFAULT_REGION/AWS_ENDPOINT_URL]=xxx + // OR + // 2) additional arguments in the config KanikoAPIServerEnvVars, which will pass the specified environment variables + // PRESENT within the Merlin API server's container to the image builder as build arguments ArtifactServiceType string `validate:"required,oneof=nop gcs s3"` } diff --git a/api/config/config_test.go b/api/config/config_test.go index 8e1abadfe..c618e8340 100644 --- a/api/config/config_test.go +++ b/api/config/config_test.go @@ -409,6 +409,7 @@ func TestLoad(t *testing.T) { KanikoServiceAccount: "kaniko-merlin", KanikoPushRegistryType: "docker", KanikoAdditionalArgs: []string{"--test=true", "--no-logs=false"}, + KanikoAPIServerEnvVars: []string{"TEST_ENV_VAR"}, DefaultResources: ResourceRequestsLimits{ Requests: Resource{ CPU: "1", diff --git a/api/config/testdata/base-configs-1.yaml b/api/config/testdata/base-configs-1.yaml index 6db2b10de..3855f823c 100644 --- a/api/config/testdata/base-configs-1.yaml +++ b/api/config/testdata/base-configs-1.yaml @@ -50,6 +50,8 @@ ImageBuilderConfig: KanikoAdditionalArgs: - --test=true - --no-logs=false + KanikoAPIServerEnvVars: + - TEST_ENV_VAR DefaultResources: Requests: CPU: "1" diff --git a/api/pkg/imagebuilder/config.go b/api/pkg/imagebuilder/config.go index e4cd99675..ba3f00c63 100644 --- a/api/pkg/imagebuilder/config.go +++ b/api/pkg/imagebuilder/config.go @@ -44,6 +44,8 @@ type Config struct { KanikoServiceAccount string // Kaniko additional args KanikoAdditionalArgs []string + // Kaniko environment variables that are propagated from the Merlin API server + KanikoAPIServerEnvVars []string // Kubernetes resource request and limits for kaniko DefaultResources cfg.ResourceRequestsLimits // Tolerations for Jobs Specification diff --git a/api/pkg/imagebuilder/imagebuilder.go b/api/pkg/imagebuilder/imagebuilder.go index 78156e297..f4dbfdc23 100644 --- a/api/pkg/imagebuilder/imagebuilder.go +++ b/api/pkg/imagebuilder/imagebuilder.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "net/http" + "os" "sort" "strings" "time" @@ -638,12 +639,17 @@ func (c *imageBuilder) createKanikoJobSpec( activeDeadlineSeconds := int64(c.config.BuildTimeoutDuration / time.Second) var volumes []v1.Volume var volumeMounts []v1.VolumeMount - var envVar []v1.EnvVar + var envVars []v1.EnvVar // Configure additional credentials for specific image registries and artifact services kanikoArgs = c.configureKanikoArgsToAddCredentials(kanikoArgs) volumes, volumeMounts = c.configureVolumesAndVolumeMountsToAddCredentials(volumes, volumeMounts) - envVar = c.configureEnvVarsToAddCredentials(envVar) + envVars = c.configureEnvVarsToAddCredentials(envVars) + + // Add all other env vars that are propagated from the API server as build args + for _, envVar := range c.config.KanikoAPIServerEnvVars { + kanikoArgs = append(kanikoArgs, fmt.Sprintf("--build-arg=%s=%s", envVar, os.Getenv(envVar))) + } var resourceRequirements RequestLimitResources cpuRequest := resource.MustParse(c.config.DefaultResources.Requests.CPU) @@ -700,7 +706,7 @@ func (c *imageBuilder) createKanikoJobSpec( Image: c.config.KanikoImage, Args: kanikoArgs, VolumeMounts: volumeMounts, - Env: envVar, + Env: envVars, Resources: resourceRequirements.Build(), TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError, }, @@ -767,17 +773,17 @@ func (c *imageBuilder) configureVolumesAndVolumeMountsToAddCredentials( return volumes, volumeMounts } -func (c *imageBuilder) configureEnvVarsToAddCredentials(envVar []v1.EnvVar) []v1.EnvVar { +func (c *imageBuilder) configureEnvVarsToAddCredentials(envVars []v1.EnvVar) []v1.EnvVar { if c.config.KanikoPushRegistryType == googleCloudRegistryPushRegistryType || c.artifactService.GetType() == googleCloudStorageArtifactServiceType { if c.config.KanikoServiceAccount == "" { - envVar = append(envVar, v1.EnvVar{ + envVars = append(envVars, v1.EnvVar{ Name: gacEnvKey, Value: saFilePath, }) } } - return envVar + return envVars } func (c *imageBuilder) GetImageBuildingJobStatus(ctx context.Context, project mlp.Project, model *models.Model, version *models.Version) (status models.ImageBuildingJobStatus) {