From b8c9ad2778a41bf8277046b2002e9867cc4f5fea Mon Sep 17 00:00:00 2001 From: Marcin Antas Date: Fri, 8 Nov 2024 13:42:54 +0100 Subject: [PATCH 1/6] Add support for multi2vec-cohere module --- .cicd/test.sh | 3 +++ weaviate/Chart.yaml | 4 ++-- weaviate/templates/_helpers.tpl | 3 +++ weaviate/templates/apiKeyCohereSecret.yaml | 8 +++++++- weaviate/templates/weaviateStatefulset.yaml | 2 +- weaviate/values.yaml | 14 +++++++++++++- 6 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.cicd/test.sh b/.cicd/test.sh index 8b627cfb..891553b7 100755 --- a/.cicd/test.sh +++ b/.cicd/test.sh @@ -126,9 +126,11 @@ function check_creates_template() { check_modules "--set modules.qna-openai.enabled=true --set modules.qna-openai.apiKey=apiKey --set modules.generative-openai.enabled=true --set modules.generative-openai.apiKey=apiKey" "value: qna-openai,generative-openai" check_modules "--set modules.generative-cohere.enabled=true" "value: generative-cohere" check_modules "--set modules.reranker-cohere.enabled=true" "value: reranker-cohere" + check_modules "--set modules.multi2vec-cohere.enabled=true" "value: multi2vec-cohere" check_modules "--set modules.text2vec-cohere.enabled=true --set modules.text2vec-cohere.apiKey=apiKey --set modules.generative-cohere.enabled=true --set modules.generative-cohere.apiKey=apiKey" "value: generative-cohere,text2vec-cohere" check_modules "--set modules.text2vec-cohere.enabled=true --set modules.text2vec-cohere.apiKey=apiKey --set modules.generative-cohere.enabled=true --set modules.generative-cohere.apiKey=apiKey --set modules.reranker-cohere.enabled=true --set modules.reranker-cohere.apiKey=apiKey" "value: generative-cohere,text2vec-cohere,reranker-cohere" check_modules "--set modules.text2vec-cohere.enabled=true --set modules.text2vec-cohere.apiKey=apiKey --set modules.reranker-cohere.enabled=true --set modules.reranker-cohere.apiKey=apiKey" "value: text2vec-cohere,reranker-cohere" + check_modules "--set modules.text2vec-cohere.enabled=true --set modules.text2vec-cohere.apiKey=apiKey --set modules.multi2vec-cohere.enabled=true --set modules.multi2vec-cohere.apiKey=apiKey" "value: multi2vec-cohere,text2vec-cohere" check_modules "--set modules.generative-cohere.enabled=true --set modules.generative-cohere.apiKey=apiKey" "value: generative-cohere" check_modules "--set modules.text2vec-palm.enabled=true" "value: text2vec-palm" check_modules "--set modules.generative-palm.enabled=true" "value: generative-palm" @@ -223,6 +225,7 @@ function check_creates_template() { check_string_existence "--set modules.generative-databricks.enabled=true --set modules.generative-databricks.token=databricksToken --set modules.text2vec-databricks.enabled=true --set modules.text2vec-databricks.token=databricksToken" "name: DATABRICKS_TOKEN" check_string_existence "--set modules.text2vec-mistral.enabled=true --set modules.text2vec-mistral.apiKey=mistralApiKey" "name: MISTRAL_APIKEY" check_string_existence "--set modules.generative-mistral.enabled=true --set modules.generative-mistral.apiKey=mistralApiKey --set modules.text2vec-mistral.enabled=true --set modules.text2vec-mistral.apiKey=mistralApiKey" "name: MISTRAL_APIKEY" + check_string_existence "--set modules.multi2vec-cohere.enabled=true --set modules.multi2vec-cohere.apiKey=apiKey" "name: COHERE_APIKEY" check_setting_has_value "--set replicas=3 --set env.RAFT_BOOTSTRAP_EXPECT=3" "name: RAFT_JOIN" "value: \"weaviate-0,weaviate-1,weaviate-2\"" check_setting_has_value "--set replicas=3 --set env.RAFT_BOOTSTRAP_EXPECT=3" "name: RAFT_BOOTSTRAP_EXPECT" "value: \"3\"" diff --git a/weaviate/Chart.yaml b/weaviate/Chart.yaml index 167a3675..10d448b4 100644 --- a/weaviate/Chart.yaml +++ b/weaviate/Chart.yaml @@ -7,10 +7,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 17.3.0 +version: 17.3.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 1.27.0 +appVersion: 1.27.2 icon: https://raw.githubusercontent.com/weaviate/weaviate/19de0956c69b66c5552447e84d016f4fe29d12c9/docs/assets/weaviate-logo.png diff --git a/weaviate/templates/_helpers.tpl b/weaviate/templates/_helpers.tpl index 81b461bb..5d629e84 100644 --- a/weaviate/templates/_helpers.tpl +++ b/weaviate/templates/_helpers.tpl @@ -80,6 +80,9 @@ {{- if (index .Values "modules" "multi2vec-google" "enabled") -}} {{ $modules = append $modules "multi2vec-google" }} {{- end -}} + {{- if (index .Values "modules" "multi2vec-cohere" "enabled") -}} + {{ $modules = append $modules "multi2vec-cohere" }} + {{- end -}} {{- if (index .Values "modules" "text2vec-openai" "enabled") -}} {{ $modules = append $modules "text2vec-openai" }} {{- end -}} diff --git a/weaviate/templates/apiKeyCohereSecret.yaml b/weaviate/templates/apiKeyCohereSecret.yaml index 8c52064d..ae7d8dc6 100644 --- a/weaviate/templates/apiKeyCohereSecret.yaml +++ b/weaviate/templates/apiKeyCohereSecret.yaml @@ -1,7 +1,8 @@ {{- $t2vCohere := and (index .Values "modules" "text2vec-cohere" "enabled") (index .Values "modules" "text2vec-cohere" "apiKey") -}} {{- $generativeCohere := and (index .Values "modules" "generative-cohere" "enabled") (index .Values "modules" "generative-cohere" "apiKey") }} {{- $rerankerCohere := and (index .Values "modules" "reranker-cohere" "enabled") (index .Values "modules" "reranker-cohere" "apiKey") }} -{{- $cohere := or ($t2vCohere) ($generativeCohere) ($rerankerCohere) }} +{{- $multi2vecCohere := and (index .Values "modules" "multi2vec-cohere" "enabled") (index .Values "modules" "multi2vec-cohere" "apiKey") }} +{{- $cohere := or ($t2vCohere) ($generativeCohere) ($rerankerCohere) ($multi2vecCohere) }} {{- if $cohere }} {{- $apiKeys := list }} @@ -14,6 +15,9 @@ {{- if $rerankerCohere }} {{- $apiKeys = append $apiKeys (index .Values "modules" "reranker-cohere" "apiKey") -}} {{- end }} +{{- if $multi2vecCohere }} + {{- $apiKeys = append $apiKeys (index .Values "modules" "multi2vec-cohere" "apiKey") -}} +{{- end }} {{- if gt (len ($apiKeys | uniq)) 1 -}} {{- fail "Cohere modules activated, but their keys differ. Use the same key on all activated Cohere modules." }} @@ -32,6 +36,8 @@ data: apiKey: {{ index .Values "modules" "generative-cohere" "apiKey" | b64enc }} {{- else if (index .Values "modules" "reranker-cohere" "apiKey") }} apiKey: {{ index .Values "modules" "reranker-cohere" "apiKey" | b64enc }} + {{- else if (index .Values "modules" "multi2vec-cohere" "apiKey") }} + apiKey: {{ index .Values "modules" "multi2vec-cohere" "apiKey" | b64enc }} {{- else }} apiKey: {{ index .Values "modules" "text2vec-cohere" "apiKey" | b64enc }} {{- end }} diff --git a/weaviate/templates/weaviateStatefulset.yaml b/weaviate/templates/weaviateStatefulset.yaml index 1f0c6c9e..54020a2f 100644 --- a/weaviate/templates/weaviateStatefulset.yaml +++ b/weaviate/templates/weaviateStatefulset.yaml @@ -218,7 +218,7 @@ spec: name: weaviate-huggingface key: apiKey {{- end }} - {{- if or (and (index .Values "modules" "text2vec-cohere" "enabled") (index .Values "modules" "text2vec-cohere" "apiKey")) (and (index .Values "modules" "generative-cohere" "enabled") (index .Values "modules" "generative-cohere" "apiKey")) (and (index .Values "modules" "reranker-cohere" "enabled") (index .Values "modules" "reranker-cohere" "apiKey")) }} + {{- if or (and (index .Values "modules" "text2vec-cohere" "enabled") (index .Values "modules" "text2vec-cohere" "apiKey")) (and (index .Values "modules" "generative-cohere" "enabled") (index .Values "modules" "generative-cohere" "apiKey")) (and (index .Values "modules" "reranker-cohere" "enabled") (index .Values "modules" "reranker-cohere" "apiKey")) (and (index .Values "modules" "multi2vec-cohere" "enabled") (index .Values "modules" "multi2vec-cohere" "apiKey")) }} - name: COHERE_APIKEY valueFrom: secretKeyRef: diff --git a/weaviate/values.yaml b/weaviate/values.yaml index 54e1c520..83ed3b3f 100644 --- a/weaviate/values.yaml +++ b/weaviate/values.yaml @@ -7,7 +7,7 @@ image: # of weaviate. In accordance with Infra-as-code, you should pin this value # down and only change it if you explicitly want to upgrade the Weaviate # version. - tag: 1.27.0 + tag: 1.27.2 repo: semitechnologies/weaviate # Image pull policy: https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy pullPolicy: IfNotPresent @@ -1225,6 +1225,18 @@ modules: # an environment variable apiKey: '' + # The multi2vec-cohere module uses Cohere API + # to dynamically compute vector embeddings. + # More information about Cohere API can be found here: https://docs.cohere.ai/ + multi2vec-cohere: + + # enable if you want to use Cohere module + enabled: false + + # Set your Cohere API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + # The qna-transformers module uses neural networks, such as BERT, # DistilBERT, to find an answer in text to a given question qna-transformers: From d1c80712a236caa3f6615bab56503a8eaac9bbd8 Mon Sep 17 00:00:00 2001 From: Marcin Antas Date: Thu, 21 Nov 2024 09:35:14 +0100 Subject: [PATCH 2/6] Add support for multi2vec-jinaai module --- .cicd/test.sh | 5 +++++ weaviate/templates/_helpers.tpl | 3 +++ weaviate/templates/apiKeyJinaAISecret.yaml | 6 +++++- weaviate/templates/weaviateStatefulset.yaml | 2 +- weaviate/values.yaml | 13 +++++++++++++ 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/.cicd/test.sh b/.cicd/test.sh index 891553b7..7df905dd 100755 --- a/.cicd/test.sh +++ b/.cicd/test.sh @@ -150,6 +150,8 @@ function check_creates_template() { check_modules "--set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=apiKey" "value: text2vec-jinaai" check_modules "--set modules.reranker-jinaai.enabled=true" "value: reranker-jinaai" check_modules "--set modules.reranker-jinaai.enabled=true --set modules.reranker-jinaai.apiKey=apiKey" "value: reranker-jinaai" + check_modules "--set modules.multi2vec-jinaai.enabled=true" "value: multi2vec-jinaai" + check_modules "--set modules.multi2vec-jinaai.enabled=true --set modules.multi2vec-jinaai.apiKey=apiKey" "value: multi2vec-jinaai" check_modules "--set modules.text2vec-aws.enabled=true" "value: text2vec-aws" check_modules "--set modules.generative-aws.enabled=true" "value: generative-aws" check_modules "--set modules.text2vec-aws.enabled=true --set modules.generative-aws.enabled=true" "value: generative-aws,text2vec-aws" @@ -185,6 +187,7 @@ function check_creates_template() { check_string_existence "--set modules.qna-openai.enabled=true --set modules.qna-openai.azureApiKey=azureApiKey" "name: AZURE_APIKEY" check_string_existence "--set modules.generative-openai.enabled=true --set modules.generative-openai.azureApiKey=azureApiKey" "name: AZURE_APIKEY" check_string_existence "--set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=jinaaiApiKey" "name: JINAAI_APIKEY" + check_string_existence "--set modules.multi2vec-jinaai.enabled=true --set modules.multi2vec-jinaai.apiKey=jinaaiApiKey" "name: JINAAI_APIKEY" check_string_existence "--set grpcService.enabled=true" "containerPort: 50051" check_string_existence "--set grpcService.enabled=true --set grpcService.name=weaviate-grpc-service-custom-name" "name: weaviate-grpc-service-custom-name" check_string_existence "--set grpcService.name=weaviate-grpc-defult-service-custom-name" "name: weaviate-grpc-defult-service-custom-name" @@ -219,6 +222,8 @@ function check_creates_template() { check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=JinaAIApiKey" "name: JINAAI_APIKEY" check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.reranker-jinaai.enabled=true --set modules.reranker-jinaai.apiKey=JinaAIApiKey" "name: JINAAI_APIKEY" check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.reranker-jinaai.enabled=true --set modules.reranker-jinaai.apiKey=JinaAIApiKey --set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=JinaAIApiKey" "name: JINAAI_APIKEY" + check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.multi2vec-jinaai.enabled=true --set modules.multi2vec-jinaai.apiKey=JinaAIApiKey --set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=JinaAIApiKey" "name: JINAAI_APIKEY" + check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.multi2vec-jinaai.enabled=true --set modules.multi2vec-jinaai.apiKey=JinaAIApiKey --set modules.reranker-jinaai.enabled=true --set modules.reranker-jinaai.apiKey=JinaAIApiKey --set modules.text2vec-jinaai.enabled=true --set modules.text2vec-jinaai.apiKey=JinaAIApiKey" "name: JINAAI_APIKEY" check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.generative-friendliai.enabled=true --set modules.generative-friendliai.token=FriendliToken" "name: FRIENDLI_TOKEN" check_string_existence "--set modules.text2vec-databricks.enabled=true --set modules.text2vec-databricks.token=databricksToken" "name: DATABRICKS_TOKEN" check_string_existence "--set modules.generative-databricks.enabled=true --set modules.generative-databricks.token=databricksToken" "name: DATABRICKS_TOKEN" diff --git a/weaviate/templates/_helpers.tpl b/weaviate/templates/_helpers.tpl index 5d629e84..541c3c79 100644 --- a/weaviate/templates/_helpers.tpl +++ b/weaviate/templates/_helpers.tpl @@ -83,6 +83,9 @@ {{- if (index .Values "modules" "multi2vec-cohere" "enabled") -}} {{ $modules = append $modules "multi2vec-cohere" }} {{- end -}} + {{- if (index .Values "modules" "multi2vec-jinaai" "enabled") -}} + {{ $modules = append $modules "multi2vec-jinaai" }} + {{- end -}} {{- if (index .Values "modules" "text2vec-openai" "enabled") -}} {{ $modules = append $modules "text2vec-openai" }} {{- end -}} diff --git a/weaviate/templates/apiKeyJinaAISecret.yaml b/weaviate/templates/apiKeyJinaAISecret.yaml index a2e60f41..fd54998f 100644 --- a/weaviate/templates/apiKeyJinaAISecret.yaml +++ b/weaviate/templates/apiKeyJinaAISecret.yaml @@ -1,6 +1,7 @@ {{- $t2vJinaAI := and (index .Values "modules" "text2vec-jinaai" "enabled") (index .Values "modules" "text2vec-jinaai" "apiKey") }} {{- $rerankerJinaAI := and (index .Values "modules" "reranker-jinaai" "enabled") (index .Values "modules" "reranker-jinaai" "apiKey") }} -{{- $jinaai := or ($t2vJinaAI) ($rerankerJinaAI) }} +{{- $multi2vecJinaAI := and (index .Values "modules" "multi2vec-jinaai" "enabled") (index .Values "modules" "multi2vec-jinaai" "apiKey") }} +{{- $jinaai := or ($t2vJinaAI) ($rerankerJinaAI) ($multi2vecJinaAI) }} {{- if $jinaai }} {{- $apiKeys := list }} @@ -10,6 +11,9 @@ {{- if $rerankerJinaAI }} {{- $apiKeys = append $apiKeys (index .Values "modules" "reranker-jinaai" "apiKey") -}} {{- end }} +{{- if $multi2vecJinaAI }} + {{- $apiKeys = append $apiKeys (index .Values "modules" "multi2vec-jinaai" "apiKey") -}} +{{- end }} {{- if gt (len ($apiKeys | uniq)) 1 -}} {{- fail "JinaAI modules activated, but their keys differ. Use the same key on all activated JinaAI modules." }} diff --git a/weaviate/templates/weaviateStatefulset.yaml b/weaviate/templates/weaviateStatefulset.yaml index 54020a2f..53bd8e45 100644 --- a/weaviate/templates/weaviateStatefulset.yaml +++ b/weaviate/templates/weaviateStatefulset.yaml @@ -239,7 +239,7 @@ spec: name: weaviate-google key: apiKey {{- end }} - {{- if or (and (index .Values "modules" "text2vec-jinaai" "enabled") (index .Values "modules" "text2vec-jinaai" "apiKey")) (and (index .Values "modules" "reranker-jinaai" "enabled") (index .Values "modules" "reranker-jinaai" "apiKey")) }} + {{- if or (and (index .Values "modules" "text2vec-jinaai" "enabled") (index .Values "modules" "text2vec-jinaai" "apiKey")) (and (index .Values "modules" "reranker-jinaai" "enabled") (index .Values "modules" "reranker-jinaai" "apiKey")) (and (index .Values "modules" "multi2vec-jinaai" "enabled") (index .Values "modules" "multi2vec-jinaai" "apiKey")) }} - name: JINAAI_APIKEY valueFrom: secretKeyRef: diff --git a/weaviate/values.yaml b/weaviate/values.yaml index 83ed3b3f..3da638c5 100644 --- a/weaviate/values.yaml +++ b/weaviate/values.yaml @@ -1237,6 +1237,19 @@ modules: # an environment variable apiKey: '' + # The multi2vec-jinaai module uses JinaAI Embeddings API + # to dynamically compute vector embeddings. + # More information about JinaAI Embeddings API can be found here: + # https://jina.ai/embeddings + multi2vec-jinaai: + + # enable if you want to use JinaAI module + enabled: false + + # Set your JinaAI API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + # The qna-transformers module uses neural networks, such as BERT, # DistilBERT, to find an answer in text to a given question qna-transformers: From b30d78f75f1b1610ab9676b201b038cc54039315 Mon Sep 17 00:00:00 2001 From: Marcin Antas Date: Thu, 21 Nov 2024 19:40:06 +0100 Subject: [PATCH 3/6] Update helm chart version to v17.3.2 --- weaviate/Chart.yaml | 4 ++-- weaviate/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/weaviate/Chart.yaml b/weaviate/Chart.yaml index 10d448b4..e0cf44ed 100644 --- a/weaviate/Chart.yaml +++ b/weaviate/Chart.yaml @@ -7,10 +7,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 17.3.1 +version: 17.3.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 1.27.2 +appVersion: 1.27.5 icon: https://raw.githubusercontent.com/weaviate/weaviate/19de0956c69b66c5552447e84d016f4fe29d12c9/docs/assets/weaviate-logo.png diff --git a/weaviate/values.yaml b/weaviate/values.yaml index 3da638c5..18d1f937 100644 --- a/weaviate/values.yaml +++ b/weaviate/values.yaml @@ -7,7 +7,7 @@ image: # of weaviate. In accordance with Infra-as-code, you should pin this value # down and only change it if you explicitly want to upgrade the Weaviate # version. - tag: 1.27.2 + tag: 1.27.5 repo: semitechnologies/weaviate # Image pull policy: https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy pullPolicy: IfNotPresent From 0e13c91e268219b3fdd7fcea336fb8b3b7efde09 Mon Sep 17 00:00:00 2001 From: Jose Luis Franco Arza Date: Mon, 2 Dec 2024 07:40:37 +0100 Subject: [PATCH 4/6] Document rbac's variables. The configuration of RBAC can be done either via the config.yaml file or via environment variables. To keep it consistent with the existing authZ and authN config values, it is disabled by default and each of the available values gets referenced in the values.yaml. --- weaviate/values.yaml | 92 ++++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 34 deletions(-) diff --git a/weaviate/values.yaml b/weaviate/values.yaml index 18d1f937..ba4b95cd 100644 --- a/weaviate/values.yaml +++ b/weaviate/values.yaml @@ -25,15 +25,15 @@ args: - 'http' - '--config-file' - '/weaviate-config/conf.yaml' - - --read-timeout=60s + - --read-timeout=60s - --write-timeout=60s # below is an example that can be used to set an arbitrary nofile limit at # startup: # -# command: +# command: # - "/bin/sh" -# args: +# args: # - "-c" # - "ulimit -n 65535 && /bin/weaviate --host 0.0.0.0 --port 8080 --scheme http --config-file /weaviate-config/conf.yaml" @@ -114,7 +114,7 @@ serviceAccountName: # Kubernetes Cluster domain name, used for resolving intra-cluster requests, i.e # between instances of weaviate. # Note: The final '.' on the end of the hostname makes it a FQDN, and is required for -# DNS to resolve in all kubernetes environments. +# DNS to resolve in all kubernetes environments. # See https://github.com/weaviate/weaviate-helm/issues/175 for details. clusterDomain: cluster.local. @@ -253,6 +253,14 @@ authentication: # client_id: '' authorization: + rbac: + enabled: false + # admins: + # - admin_user1 + # - admin_user2 + # viewers: + # - viewer_user1 + # - readonly_user1 admin_list: enabled: false # users: @@ -316,7 +324,7 @@ env: PROMETHEUS_MONITORING_ENABLED: false PROMETHEUS_MONITORING_GROUP: false - # Set a MEM limit for the Weaviate Pod so it can help you both increase GC-related + # Set a MEM limit for the Weaviate Pod so it can help you both increase GC-related # performance as well as avoid GC-related out-of-memory (“OOM”) situations # GOMEMLIMIT: 6GiB @@ -337,7 +345,8 @@ env: # If using ENV Vars to set up API Keys make sure to have `authentication.apikey` block commented out # to avoid any future changes. ENV Vars has priority over the config above `authentication.apikey`. # If using `authentication.apikey `the below ENV Vars will be used because they have priority, - # so comment them out to avoid any future changes. + # so comment them out to avoid any future changes. The same applies for the RBAC configuration + # under the authorization block. # Enables API key authentication. If it is set to 'false' the AUTHENTICATION_APIKEY_ALLOWED_KEYS # and AUTHENTICATION_APIKEY_USERS will not have any effect. # AUTHENTICATION_APIKEY_ENABLED: 'true' @@ -351,6 +360,21 @@ env: # NOTE: Make sure to add the users to the authorization above overwise they will not be allowed to interact with Weaviate. # AUTHENTICATION_APIKEY_USERS: 'jane@doe.com,ian-smith' + # Enabling RBAC authorization. It is mutually exclusive with the AUTHORIZATION_ADMIN_LISTS variable. Either RBAC or the + # admin lists mechanism can be used. + # AUTHORIZATION_ENABLE_RBAC: "true" + + # Users with admin's RBAC role. List one or more user identities, separated by commas, which will + # have the admin role assigned to. This role provides all permissions to the user, but it's required at least + # in one of the user for managing the cluster. + # AUTHORIZATION_ADMIN_USERS: "admin-user" + + # Users with viewer's RBAC role. List one or more user identities, separated by commas, which will + # have the viewer role assigned to. This role allows read permissions in all different areas. Once assigned via + # config, it can't be revoked via API AuthZ calls. + # AUTHORIZATION_VIEWER_USERS: "viewer-user" + + envSecrets: # create a Kubernetes secret with AUTHENTICATION_APIKEY_ALLOWED_KEYS key and its respective value # AUTHENTICATION_APIKEY_ALLOWED_KEYS: name-of-the-k8s-secret-containing-the-comma-separated-api-keys @@ -362,7 +386,7 @@ offload: # If one is using AWS EKS and has already configured K8s Service Account # that holds the AWS credentials one can pass a name of that service account # here using this setting. - # NOTE: the root `serviceAccountName` config has priority over this one, and + # NOTE: the root `serviceAccountName` config has priority over this one, and # if the root one is set this one will NOT overwrite it. This one is here for # backwards compatibility. serviceAccountName: @@ -371,7 +395,7 @@ offload: # Configure bucket where data should be saved, this setting is mandatory OFFLOAD_S3_BUCKET: weaviate-offload - # Optional setting. Defaults to AWS S3 (s3.amazonaws.com). + # Optional setting. Defaults to AWS S3 (s3.amazonaws.com). # Set this option if you have a MinIO storage configured in your environment # and want to use it instead of the AWS S3. # OFFLOAD_S3_ENDPOINT: custom.minio.endpoint.address @@ -384,7 +408,7 @@ offload: # Set this option if you wan't Weaviate to create # the bucket used for offloading tenants. Otherwise, # if set to false Weaviate expects the bucket to be - # already created with the OFFLOAD_S3_BUCKET name + # already created with the OFFLOAD_S3_BUCKET name # OFFLOAD_S3_BUCKET_AUTO_CREATE: true # You can pass environment AWS settings here: @@ -417,13 +441,13 @@ backups: envconfig: # Configure folder where backups should be saved BACKUP_FILESYSTEM_PATH: /tmp/backups - + s3: enabled: false # If one is using AWS EKS and has already configured K8s Service Account # that holds the AWS credentials one can pass a name of that service account # here using this setting. - # NOTE: the root `serviceAccountName` config has priority over this one, and + # NOTE: the root `serviceAccountName` config has priority over this one, and # if the root one is set this one will NOT overwrite it. This one is here for # backwards compatibility. serviceAccountName: @@ -432,17 +456,17 @@ backups: # Configure bucket where backups should be saved, this setting is mandatory BACKUP_S3_BUCKET: weaviate-backups - # Optional setting. Defaults to empty string. + # Optional setting. Defaults to empty string. # Set this option if you want to save backups to a given location # inside the bucket # BACKUP_S3_PATH: path/inside/bucket - # Optional setting. Defaults to AWS S3 (s3.amazonaws.com). + # Optional setting. Defaults to AWS S3 (s3.amazonaws.com). # Set this option if you have a MinIO storage configured in your environment # and want to use it instead of the AWS S3. # BACKUP_S3_ENDPOINT: custom.minio.endpoint.address - # Optional setting. Defaults to true. + # Optional setting. Defaults to true. # Set this option if you don't want to use SSL. # BACKUP_S3_USE_SSL: true @@ -455,7 +479,7 @@ backups: # You can pass the User credentials (access-key id and access-secret-key) in 2 ways: # 1. by setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY plain values in the `secrets` section below # this chart will create a kubernetes secret for you with these key-values pairs - # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys and their respective values + # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys and their respective values # Set the Key and the secret where it is set in `envSecrets` section below secrets: {} # AWS_ACCESS_KEY_ID: access-key-id (plain text) @@ -503,7 +527,7 @@ backups: # Configure container where backups should be saved, this setting is mandatory BACKUP_AZURE_CONTAINER: weaviate-backups - # Optional setting. Defaults to empty string. + # Optional setting. Defaults to empty string. # Set this option if you want to save backups to a given location # inside the container # BACKUP_AZURE_PATH: path/inside/container @@ -514,7 +538,7 @@ backups: # 1. by setting the AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY # or AZURE_STORAGE_CONNECTION_STRING plain values in the `secrets` section below # this chart will create a kubernetes secret for you with these key-values pairs - # 2. create Kubernetes secret/s with AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY + # 2. create Kubernetes secret/s with AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY # or AZURE_STORAGE_CONNECTION_STRING and their respective values # Set the Key and the secret where it is set in `envSecrets` section below secrets: {} @@ -660,7 +684,7 @@ modules: # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ # applies to passageQueryService below securityContext: {} - + # It is possible to add a ServiceAccount to this module's Pods, it can be # used in cases where the module is in a private registry and you want to # give access to the registry only to this pod. @@ -728,13 +752,13 @@ modules: # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ # applies to passageQueryService below securityContext: {} - + # It is possible to add a ServiceAccount to this module's Pods, it can be # used in cases where the module is in a private registry and you want to # give access to the registry only to this pod. # NOTE: if not set the root `serviceAccountName` config will be used. serviceAccountName: - + # You can guide where the pods are scheduled on a per-module basis, # as well as for Weaviate overall. Each module accepts nodeSelector, # tolerations, and affinity configuration. If it is set on a per- @@ -797,7 +821,7 @@ modules: # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ # applies to passageQueryService below securityContext: {} - + # It is possible to add a ServiceAccount to this module's Pods, it can be # used in cases where the module is in a private registry and you want to # give access to the registry only to this pod. @@ -818,7 +842,7 @@ modules: # These models run only on CPU only and on x86_64 arch # The ML model is containerized in a Weaviate compatible way. # If you want to run a different model that published ones you can follow the - # tutorial from here on how to create such a container: https://github.com/weaviate/t2v-gpt4all-models + # tutorial from here on how to create such a container: https://github.com/weaviate/t2v-gpt4all-models text2vec-gpt4all: # Enable deployment of this module @@ -827,7 +851,7 @@ modules: # You can set directly an inference URL of this module without deploying it with this release. # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` inferenceUrl: {} - + # The configuration below is ignored if enabled==false tag: all-MiniLM-L6-v2 repo: semitechnologies/gpt4all-inference @@ -859,7 +883,7 @@ modules: # security Context for the Contextionary Pods. The configurations are the same as setting them # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ securityContext: {} - + # It is possible to add a ServiceAccount to this module's Pods, it can be # used in cases where the module is in a private registry and you want to # give access to the registry only to this pod. @@ -1306,7 +1330,7 @@ modules: # security Context for the Contextionary Pods. The configurations are the same as setting them # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ securityContext: {} - + # It is possible to add a ServiceAccount to this module's Pods, it can be # used in cases where the module is in a private registry and you want to # give access to the registry only to this pod. @@ -1552,7 +1576,7 @@ modules: # security Context for the Contextionary Pods. The configurations are the same as setting them # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ securityContext: {} - + # It is possible to add a ServiceAccount to this module's Pods, it can be # used in cases where the module is in a private registry and you want to # give access to the registry only to this pod. @@ -1569,7 +1593,7 @@ modules: affinity: # The reranker-cohere module uses Cohere API - # to dynamically compute a score for the relevance + # to dynamically compute a score for the relevance # of the query with each of the initial search results. # More information about Cohere API can be found here: https://docs.cohere.com/docs/rerank-guide reranker-cohere: @@ -1582,7 +1606,7 @@ modules: apiKey: '' # The reranker-voyageai module uses VoaygeAI API - # to dynamically compute a score for the relevance + # to dynamically compute a score for the relevance # of the query with each of the initial search results. # More information about Cohere API can be found here: https://www.voyageai.com/ reranker-voyageai: @@ -1595,7 +1619,7 @@ modules: apiKey: '' # The reranker-jinaai module uses JinaAI API - # to dynamically compute a score for the relevance + # to dynamically compute a score for the relevance # of the query with each of the initial search results. # More information about Cohere API can be found here: https://jina.ai/reranker reranker-jinaai: @@ -1607,7 +1631,7 @@ modules: # an environment variable apiKey: '' - # The reranker-transformers module uses Cross-Encoders for + # The reranker-transformers module uses Cross-Encoders for # sentence pair scoring and sentence pair classification tasks. # More information about Cross-Encoders can be found here: # https://www.sbert.net/examples/applications/cross-encoder/README.html @@ -1665,7 +1689,7 @@ modules: # security Context for the Contextionary Pods. The configurations are the same as setting them # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ securityContext: {} - + # It is possible to add a ServiceAccount to this module's Pods, it can be # used in cases where the module is in a private registry and you want to # give access to the registry only to this pod. @@ -1722,7 +1746,7 @@ modules: # security Context for the Contextionary Pods. The configurations are the same as setting them # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ securityContext: {} - + # It is possible to add a ServiceAccount to this module's Pods, it can be # used in cases where the module is in a private registry and you want to # give access to the registry only to this pod. @@ -1796,7 +1820,7 @@ modules: # security Context for the Contextionary Pods. The configurations are the same as setting them # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ securityContext: {} - + # It is possible to add a ServiceAccount to this module's Pods, it can be # used in cases where the module is in a private registry and you want to # give access to the registry only to this pod. @@ -1867,7 +1891,7 @@ modules: # security Context for the Contextionary Pods. The configurations are the same as setting them # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ securityContext: {} - + # It is possible to add a ServiceAccount to this module's Pods, it can be # used in cases where the module is in a private registry and you want to # give access to the registry only to this pod. From d3771525365e87082342232bfd1e1cdb01b91347 Mon Sep 17 00:00:00 2001 From: Marcin Antas Date: Thu, 5 Dec 2024 19:17:56 +0100 Subject: [PATCH 5/6] Add support for multi2vec-voyageai module --- .cicd/test.sh | 4 ++++ weaviate/templates/_helpers.tpl | 3 +++ weaviate/templates/apiKeyVoyageAISecret.yaml | 8 +++++++- weaviate/templates/weaviateStatefulset.yaml | 2 +- weaviate/values.yaml | 13 +++++++++++++ 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.cicd/test.sh b/.cicd/test.sh index 7df905dd..a2ed4dc9 100755 --- a/.cicd/test.sh +++ b/.cicd/test.sh @@ -158,13 +158,17 @@ function check_creates_template() { check_modules "--set modules.generative-anyscale.enabled=true" "value: generative-anyscale" check_modules "--set modules.generative-anyscale.enabled=true --set modules.generative-anyscale.apiKey=apiKey" "value: generative-anyscale" check_modules "--set modules.text2vec-voyageai.enabled=true" "value: text2vec-voyageai" + check_modules "--set modules.multi2vec-voyageai.enabled=true" "value: multi2vec-voyageai" check_modules "--set modules.generative-mistral.enabled=true" "value: generative-mistral" check_modules "--set modules.generative-mistral.enabled=true --set modules.generative-mistral.apiKey=apiKey" "value: generative-mistral" check_modules "--set modules.multi2vec-palm.enabled=true" "value: multi2vec-palm" check_modules "--set modules.multi2vec-palm.enabled=true --set modules.multi2vec-palm.apiKey=apiKey --set modules.generative-palm.enabled=true --set modules.generative-palm.apiKey=apiKey" "value: generative-palm,multi2vec-palm" check_modules "--set modules.reranker-voyageai.enabled=true" "value: reranker-voyageai" check_modules "--set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=apiKey" "value: reranker-voyageai" + check_modules "--set modules.multi2vec-voyageai.enabled=true --set modules.multi2vec-voyageai.apiKey=apiKey" "value: multi2vec-voyageai" check_modules "--set modules.text2vec-voyageai.enabled=true --set modules.text2vec-voyageai.apiKey=apiKey --set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=apiKey" "value: text2vec-voyageai,reranker-voyageai" + check_modules "--set modules.multi2vec-voyageai.enabled=true --set modules.multi2vec-voyageai.apiKey=apiKey --set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=apiKey" "value: multi2vec-voyageai,reranker-voyageai" + check_modules "--set modules.text2vec-voyageai.enabled=true --set modules.text2vec-voyageai.apiKey=apiKey --set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=apiKey --set modules.multi2vec-voyageai.enabled=true --set modules.multi2vec-voyageai.apiKey=apiKey" "value: multi2vec-voyageai,text2vec-voyageai,reranker-voyageai" check_modules "--set modules.text2vec-ollama.enabled=true" "value: text2vec-ollama" check_modules "--set modules.generative-ollama.enabled=true" "value: generative-ollama" check_modules "--set modules.generative-ollama.enabled=true --set modules.text2vec-ollama.enabled=true" "value: generative-ollama,text2vec-ollama" diff --git a/weaviate/templates/_helpers.tpl b/weaviate/templates/_helpers.tpl index 541c3c79..830b10f0 100644 --- a/weaviate/templates/_helpers.tpl +++ b/weaviate/templates/_helpers.tpl @@ -86,6 +86,9 @@ {{- if (index .Values "modules" "multi2vec-jinaai" "enabled") -}} {{ $modules = append $modules "multi2vec-jinaai" }} {{- end -}} + {{- if (index .Values "modules" "multi2vec-voyageai" "enabled") -}} + {{ $modules = append $modules "multi2vec-voyageai" }} + {{- end -}} {{- if (index .Values "modules" "text2vec-openai" "enabled") -}} {{ $modules = append $modules "text2vec-openai" }} {{- end -}} diff --git a/weaviate/templates/apiKeyVoyageAISecret.yaml b/weaviate/templates/apiKeyVoyageAISecret.yaml index 025c9001..8b5c9016 100644 --- a/weaviate/templates/apiKeyVoyageAISecret.yaml +++ b/weaviate/templates/apiKeyVoyageAISecret.yaml @@ -1,6 +1,7 @@ {{- $t2vVoyageAI := and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey") -}} {{- $rerankerVoyageAI := and (index .Values "modules" "reranker-voyageai" "enabled") (index .Values "modules" "reranker-voyageai" "apiKey") }} -{{- $voyageai := or ($t2vVoyageAI) ($rerankerVoyageAI) }} +{{- $multi2vecVoyageAI := and (index .Values "modules" "multi2vec-voyageai" "enabled") (index .Values "modules" "multi2vec-voyageai" "apiKey") }} +{{- $voyageai := or ($t2vVoyageAI) ($rerankerVoyageAI) ($multi2vecVoyageAI) }} {{- if $voyageai }} {{- $apiKeys := list }} @@ -10,6 +11,9 @@ {{- if $rerankerVoyageAI }} {{- $apiKeys = append $apiKeys (index .Values "modules" "reranker-voyageai" "apiKey") -}} {{- end }} +{{- if $multi2vecVoyageAI }} + {{- $apiKeys = append $apiKeys (index .Values "modules" "multi2vec-voyageai" "apiKey") -}} +{{- end }} {{- if gt (len ($apiKeys | uniq)) 1 -}} {{- fail "VoyageAI modules activated, but their keys differ. Use the same key on all activated VoyageAI modules." }} @@ -26,6 +30,8 @@ type: Opaque data: {{- if (index .Values "modules" "reranker-voyageai" "apiKey") }} apiKey: {{ index .Values "modules" "reranker-voyageai" "apiKey" | b64enc }} + {{- else if (index .Values "modules" "multi2vec-voyageai" "apiKey") }} + apiKey: {{ index .Values "modules" "multi2vec-voyageai" "apiKey" | b64enc }} {{- else }} apiKey: {{ index .Values "modules" "text2vec-voyageai" "apiKey" | b64enc }} {{- end }} diff --git a/weaviate/templates/weaviateStatefulset.yaml b/weaviate/templates/weaviateStatefulset.yaml index 53bd8e45..372cfd5d 100644 --- a/weaviate/templates/weaviateStatefulset.yaml +++ b/weaviate/templates/weaviateStatefulset.yaml @@ -246,7 +246,7 @@ spec: name: weaviate-jinaai key: apiKey {{- end }} - {{- if or (and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey")) (and (index .Values "modules" "reranker-voyageai" "enabled") (index .Values "modules" "reranker-voyageai" "apiKey")) }} + {{- if or (and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey")) (and (index .Values "modules" "reranker-voyageai" "enabled") (index .Values "modules" "reranker-voyageai" "apiKey")) (and (index .Values "modules" "multi2vec-voyageai" "enabled") (index .Values "modules" "multi2vec-voyageai" "apiKey")) }} - name: VOYAGEAI_APIKEY valueFrom: secretKeyRef: diff --git a/weaviate/values.yaml b/weaviate/values.yaml index ba4b95cd..c4d0d1db 100644 --- a/weaviate/values.yaml +++ b/weaviate/values.yaml @@ -1274,6 +1274,19 @@ modules: # an environment variable apiKey: '' + # The multi2vec-voyageai module uses VoaygeAI Multi Modal Embeddings API + # to dynamically compute vector embeddings. + # More information about VoaygeAI Embeddings API can be found here: + # https://www.voyageai.com/ + multi2vec-voyageai: + + # enable if you want to use VoaygeAI module + enabled: false + + # Set your VoaygeAI API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + # The qna-transformers module uses neural networks, such as BERT, # DistilBERT, to find an answer in text to a given question qna-transformers: From cecbd29c9da790825a08c2bfd1442d3f5aec4056 Mon Sep 17 00:00:00 2001 From: Marcin Antas Date: Tue, 10 Dec 2024 12:48:07 +0100 Subject: [PATCH 6/6] Bump Weaviate version to v1.27.8 --- weaviate/Chart.yaml | 4 ++-- weaviate/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/weaviate/Chart.yaml b/weaviate/Chart.yaml index e0cf44ed..4124921f 100644 --- a/weaviate/Chart.yaml +++ b/weaviate/Chart.yaml @@ -7,10 +7,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 17.3.2 +version: 17.3.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 1.27.5 +appVersion: 1.27.8 icon: https://raw.githubusercontent.com/weaviate/weaviate/19de0956c69b66c5552447e84d016f4fe29d12c9/docs/assets/weaviate-logo.png diff --git a/weaviate/values.yaml b/weaviate/values.yaml index c4d0d1db..d6ef7c64 100644 --- a/weaviate/values.yaml +++ b/weaviate/values.yaml @@ -7,7 +7,7 @@ image: # of weaviate. In accordance with Infra-as-code, you should pin this value # down and only change it if you explicitly want to upgrade the Weaviate # version. - tag: 1.27.5 + tag: 1.27.8 repo: semitechnologies/weaviate # Image pull policy: https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy pullPolicy: IfNotPresent