From 69429b32a87a9a347d181095fabedaae047cc337 Mon Sep 17 00:00:00 2001 From: Sunghyuk Kay Date: Sun, 3 Dec 2023 17:39:06 +0900 Subject: [PATCH 1/3] =?UTF-8?q?DB:=20Add=20env=20to=20skip=20DB=20creation?= =?UTF-8?q?=CB=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/db/v1beta1/common/const.go | 2 ++ pkg/db/v1beta1/mysql/init.go | 22 +++++++++++++++++----- pkg/db/v1beta1/postgres/init.go | 21 +++++++++++++++++---- pkg/util/v1beta1/env/env.go | 17 ++++++++++++++++- pkg/util/v1beta1/env/env_test.go | 18 ++++++++++++++++++ 5 files changed, 70 insertions(+), 10 deletions(-) diff --git a/pkg/db/v1beta1/common/const.go b/pkg/db/v1beta1/common/const.go index 3bdb09da66e..da826c5e09a 100644 --- a/pkg/db/v1beta1/common/const.go +++ b/pkg/db/v1beta1/common/const.go @@ -46,4 +46,6 @@ const ( DefaultPostgreSQLDatabase = "katib" DefaultPostgreSQLHost = "katib-postgres" DefaultPostgreSQLPort = "5432" + + SkipDbMigrationEnvName = "SKIP_DB_MIGRATION" ) diff --git a/pkg/db/v1beta1/mysql/init.go b/pkg/db/v1beta1/mysql/init.go index 30468062ea7..36c9e712ae1 100644 --- a/pkg/db/v1beta1/mysql/init.go +++ b/pkg/db/v1beta1/mysql/init.go @@ -18,22 +18,34 @@ package mysql import ( "fmt" - + "github.com/kubeflow/katib/pkg/db/v1beta1/common" + "github.com/kubeflow/katib/pkg/util/v1beta1/env" "k8s.io/klog" ) func (d *dbConn) DBInit() { db := d.db - klog.Info("Initializing v1beta1 DB schema") + skipDbMigration := env.GetBoolEnvOrDefault(common.SkipDbMigrationEnvName, false) + + if !skipDbMigration { + klog.Info("Initializing v1beta1 DB schema") - _, err := db.Exec(`CREATE TABLE IF NOT EXISTS observation_logs + _, err := db.Exec(`CREATE TABLE IF NOT EXISTS observation_logs (trial_name VARCHAR(255) NOT NULL, id INT AUTO_INCREMENT PRIMARY KEY, time DATETIME(6), metric_name VARCHAR(255) NOT NULL, value TEXT NOT NULL)`) - if err != nil { - klog.Fatalf("Error creating observation_logs table: %v", err) + if err != nil { + klog.Fatalf("Error creating observation_logs table: %v", err) + } + } else { + klog.Info("Skipping v1beta1 DB schema initialization.") + + _, err := db.Query(`SELECT trial_name, id, time, metric_name, value FROM observation_logs LIMIT 1`) + if err != nil { + klog.Fatalf("Error validating observation_logs table: %v", err) + } } } diff --git a/pkg/db/v1beta1/postgres/init.go b/pkg/db/v1beta1/postgres/init.go index 3ebfad40a46..3229c65f2c5 100644 --- a/pkg/db/v1beta1/postgres/init.go +++ b/pkg/db/v1beta1/postgres/init.go @@ -18,22 +18,35 @@ package postgres import ( "fmt" + "github.com/kubeflow/katib/pkg/db/v1beta1/common" + "github.com/kubeflow/katib/pkg/util/v1beta1/env" "k8s.io/klog" ) func (d *dbConn) DBInit() { db := d.db - klog.Info("Initializing v1beta1 DB schema") + skipDbMigration := env.GetBoolEnvOrDefault(common.SkipDbMigrationEnvName, false) - _, err := db.Exec(`CREATE TABLE IF NOT EXISTS observation_logs + if !skipDbMigration { + klog.Info("Initializing v1beta1 DB schema") + + _, err := db.Exec(`CREATE TABLE IF NOT EXISTS observation_logs (trial_name VARCHAR(255) NOT NULL, id serial PRIMARY KEY, time TIMESTAMP(6), metric_name VARCHAR(255) NOT NULL, value TEXT NOT NULL)`) - if err != nil { - klog.Fatalf("Error creating observation_logs table: %v", err) + if err != nil { + klog.Fatalf("Error creating observation_logs table: %v", err) + } + } else { + klog.Info("Skipping v1beta1 DB schema initialization.") + + _, err := db.Query(`SELECT trial_name, id, time, metric_name, value FROM observation_logs LIMIT 1`) + if err != nil { + klog.Fatalf("Error validating observation_logs table: %v", err) + } } } diff --git a/pkg/util/v1beta1/env/env.go b/pkg/util/v1beta1/env/env.go index 114e58ed4b0..b8cd0fa635d 100644 --- a/pkg/util/v1beta1/env/env.go +++ b/pkg/util/v1beta1/env/env.go @@ -16,7 +16,11 @@ limitations under the License. package env -import "os" +import ( + "k8s.io/klog" + "os" + "strconv" +) func GetEnvOrDefault(key string, fallback string) string { if value, ok := os.LookupEnv(key); ok { @@ -24,3 +28,14 @@ func GetEnvOrDefault(key string, fallback string) string { } return fallback } + +func GetBoolEnvOrDefault(key string, fallback bool) bool { + if value, ok := os.LookupEnv(key); ok { + parsedValue, err := strconv.ParseBool(value) + if err != nil { + klog.Fatalf("Failed converting %s env to bool", key) + } + return parsedValue + } + return fallback +} diff --git a/pkg/util/v1beta1/env/env_test.go b/pkg/util/v1beta1/env/env_test.go index d5b7cfabf89..afebae71bda 100644 --- a/pkg/util/v1beta1/env/env_test.go +++ b/pkg/util/v1beta1/env/env_test.go @@ -17,6 +17,7 @@ limitations under the License. package env import ( + "fmt" "os" "testing" ) @@ -35,3 +36,20 @@ func TestGetEnvWithDefault(t *testing.T) { t.Errorf("Expected %s, got %s", expected, v) } } + +func TestGetBoolEnvWithDefault(t *testing.T) { + expected := false + key := "TEST" + v := GetBoolEnvOrDefault(key, expected) + if v != expected { + t.Errorf("Expected %t, got %t", expected, v) + } + + expected = true + envValue := fmt.Sprintf("%t", expected) + os.Setenv(key, envValue) + v = GetBoolEnvOrDefault(key, false) + if v != expected { + t.Errorf("Expected %t, got %t", expected, v) + } +} From d91b9b9ac9949675a4e6bc46816824a795720ae3 Mon Sep 17 00:00:00 2001 From: Sunghyuk Kay Date: Wed, 6 Dec 2023 21:01:25 +0900 Subject: [PATCH 2/3] DB: Rename var name & Remove new function * Migration -> Initialization * Remove GetBoolEnvOrDefault --- pkg/db/v1beta1/common/const.go | 2 +- pkg/db/v1beta1/mysql/init.go | 4 ++-- pkg/db/v1beta1/postgres/init.go | 4 ++-- pkg/util/v1beta1/env/env.go | 13 ------------- pkg/util/v1beta1/env/env_test.go | 18 ------------------ 5 files changed, 5 insertions(+), 36 deletions(-) diff --git a/pkg/db/v1beta1/common/const.go b/pkg/db/v1beta1/common/const.go index da826c5e09a..9754f9a65dd 100644 --- a/pkg/db/v1beta1/common/const.go +++ b/pkg/db/v1beta1/common/const.go @@ -47,5 +47,5 @@ const ( DefaultPostgreSQLHost = "katib-postgres" DefaultPostgreSQLPort = "5432" - SkipDbMigrationEnvName = "SKIP_DB_MIGRATION" + SkipDbInitializationEnvName = "SKIP_DB_INITIALIZATION" ) diff --git a/pkg/db/v1beta1/mysql/init.go b/pkg/db/v1beta1/mysql/init.go index 36c9e712ae1..e8c140e5a88 100644 --- a/pkg/db/v1beta1/mysql/init.go +++ b/pkg/db/v1beta1/mysql/init.go @@ -25,9 +25,9 @@ import ( func (d *dbConn) DBInit() { db := d.db - skipDbMigration := env.GetBoolEnvOrDefault(common.SkipDbMigrationEnvName, false) + skipDbInitialization := env.GetEnvOrDefault(common.SkipDbInitializationEnvName, "false") - if !skipDbMigration { + if skipDbInitialization == "false" { klog.Info("Initializing v1beta1 DB schema") _, err := db.Exec(`CREATE TABLE IF NOT EXISTS observation_logs diff --git a/pkg/db/v1beta1/postgres/init.go b/pkg/db/v1beta1/postgres/init.go index 3229c65f2c5..b550fe61509 100644 --- a/pkg/db/v1beta1/postgres/init.go +++ b/pkg/db/v1beta1/postgres/init.go @@ -26,9 +26,9 @@ import ( func (d *dbConn) DBInit() { db := d.db - skipDbMigration := env.GetBoolEnvOrDefault(common.SkipDbMigrationEnvName, false) + skipDbInitialization := env.GetEnvOrDefault(common.SkipDbInitializationEnvName, "false") - if !skipDbMigration { + if skipDbInitialization == "false" { klog.Info("Initializing v1beta1 DB schema") _, err := db.Exec(`CREATE TABLE IF NOT EXISTS observation_logs diff --git a/pkg/util/v1beta1/env/env.go b/pkg/util/v1beta1/env/env.go index b8cd0fa635d..d9d7516b328 100644 --- a/pkg/util/v1beta1/env/env.go +++ b/pkg/util/v1beta1/env/env.go @@ -17,9 +17,7 @@ limitations under the License. package env import ( - "k8s.io/klog" "os" - "strconv" ) func GetEnvOrDefault(key string, fallback string) string { @@ -28,14 +26,3 @@ func GetEnvOrDefault(key string, fallback string) string { } return fallback } - -func GetBoolEnvOrDefault(key string, fallback bool) bool { - if value, ok := os.LookupEnv(key); ok { - parsedValue, err := strconv.ParseBool(value) - if err != nil { - klog.Fatalf("Failed converting %s env to bool", key) - } - return parsedValue - } - return fallback -} diff --git a/pkg/util/v1beta1/env/env_test.go b/pkg/util/v1beta1/env/env_test.go index afebae71bda..d5b7cfabf89 100644 --- a/pkg/util/v1beta1/env/env_test.go +++ b/pkg/util/v1beta1/env/env_test.go @@ -17,7 +17,6 @@ limitations under the License. package env import ( - "fmt" "os" "testing" ) @@ -36,20 +35,3 @@ func TestGetEnvWithDefault(t *testing.T) { t.Errorf("Expected %s, got %s", expected, v) } } - -func TestGetBoolEnvWithDefault(t *testing.T) { - expected := false - key := "TEST" - v := GetBoolEnvOrDefault(key, expected) - if v != expected { - t.Errorf("Expected %t, got %t", expected, v) - } - - expected = true - envValue := fmt.Sprintf("%t", expected) - os.Setenv(key, envValue) - v = GetBoolEnvOrDefault(key, false) - if v != expected { - t.Errorf("Expected %t, got %t", expected, v) - } -} From 64f0f85f7f60ea550ebfb19cffe31e7c90c7c341 Mon Sep 17 00:00:00 2001 From: Sunghyuk Kay Date: Wed, 6 Dec 2023 22:08:28 +0900 Subject: [PATCH 3/3] DB: Rearrange dependencies --- pkg/db/v1beta1/mysql/init.go | 4 +++- pkg/db/v1beta1/postgres/init.go | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/db/v1beta1/mysql/init.go b/pkg/db/v1beta1/mysql/init.go index e8c140e5a88..fe2a2987f0b 100644 --- a/pkg/db/v1beta1/mysql/init.go +++ b/pkg/db/v1beta1/mysql/init.go @@ -18,9 +18,11 @@ package mysql import ( "fmt" + + "k8s.io/klog" + "github.com/kubeflow/katib/pkg/db/v1beta1/common" "github.com/kubeflow/katib/pkg/util/v1beta1/env" - "k8s.io/klog" ) func (d *dbConn) DBInit() { diff --git a/pkg/db/v1beta1/postgres/init.go b/pkg/db/v1beta1/postgres/init.go index b550fe61509..71c573128c2 100644 --- a/pkg/db/v1beta1/postgres/init.go +++ b/pkg/db/v1beta1/postgres/init.go @@ -18,10 +18,11 @@ package postgres import ( "fmt" - "github.com/kubeflow/katib/pkg/db/v1beta1/common" - "github.com/kubeflow/katib/pkg/util/v1beta1/env" "k8s.io/klog" + + "github.com/kubeflow/katib/pkg/db/v1beta1/common" + "github.com/kubeflow/katib/pkg/util/v1beta1/env" ) func (d *dbConn) DBInit() {