Fix collection embedding names

src/db_cli.py

@@ -86,23 +86,6 @@ def set_obsidian_inbox_item_id(self, source, category, item_id, **kwargs):
         key = key_tpl.format(source, category, item_id)
         self.driver.set(key, "true", **kwargs)
 
-    def get_embedding_item_id(self, source, provider, item_id):
-        key_tpl = data_model.EMBEDDING_ITEM_ID
-        key = key_tpl.format(source, provider, item_id)
-        return self.driver.get(key)
-
-    def set_embedding_item_id(
-        self,
-        source,
-        provider,
-        item_id,
-        embed: list,
-        **kwargs
-    ):
-        key_tpl = data_model.EMBEDDING_ITEM_ID
-        key = key_tpl.format(source, provider, item_id)
-        self.driver.set(key, embed, **kwargs)
-
     def get_milvus_embedding_item_id(
         self,
         provider,

src/embedding.py

@@ -1,16 +1,23 @@
-from abc import ABC, abstractmethod
+from abc import abstractmethod
+import re
 
 
-class Embedding(ABC):
-    def __init__(self, model_name="openai"):
+class Embedding:
+    def __init__(self, model_name):
         self.model_name = model_name
 
-    @abstractmethod
-    def dim(self):
-        pass
+    def getname(self, start_date, prefix="news"):
+        """
+        Get the name of the milvus collection for the embedding
+        We are only allowed to use alphanumeric characters and underscores in the collection name
+        """
+        # Replace any non-alphanumeric character in the model name with an underscore
+        sanitized_model_name = re.sub(r'\W+', '_', self.model_name)
+        sanitized_start_date = start_date.replace('-', '_')
+        return f"{prefix}_{sanitized_model_name}__{sanitized_start_date}"
 
     @abstractmethod
-    def getname(self, start_date, prefix="news"):
+    def dim(self):
         pass
 
     @abstractmethod

src/embedding_hf.py

@@ -21,9 +21,6 @@ def __init__(self, model_name="all-MiniLM-L6-v2"):
     def dim(self):
         return 384
 
-    def getname(self, start_date, prefix="news"):
-        return f"{prefix}_embedding_hf_{start_date}".replace("-", "_")
-
     def create(self, text: str, normalize=True):
         """
         Query local HF embedding model
@@ -49,7 +46,7 @@ def get_or_create(
         """
         client = db_client or DBClient()
 
-        embedding = client.get_embedding_item_id(
+        embedding = client.get_milvus_embedding_item_id(
             "hf",
             self.model_name,
             source,
@@ -59,7 +56,7 @@ def get_or_create(
             embedding = self.create(text)
 
             # store embedding into redis (ttl = 1 month)
-            client.set_embedding_item_id(
+            client.set_milvus_embedding_item_id(
                 "hf",
                 self.model_name,
                 source,

src/embedding_hf_inst.py

@@ -21,9 +21,6 @@ def __init__(self, model_name="hkunlp/instructor-xl"):
     def dim(self):
         return 384
 
-    def getname(self, start_date, prefix="news"):
-        return f"{prefix}_embedding_hf_inst_{start_date}".replace("-", "_")
-
     def create(self, text: str, normalize=True):
         """
         Query local HF embedding model
@@ -49,7 +46,7 @@ def get_or_create(
         """
         client = db_client or DBClient()
 
-        embedding = client.get_embedding_item_id(
+        embedding = client.get_milvus_embedding_item_id(
             "hf_inst",
             self.model_name,
             source,
@@ -59,7 +56,7 @@ def get_or_create(
             embedding = self.create(text)
 
             # store embedding into redis (ttl = 1 month)
-            client.set_embedding_item_id(
+            client.set_milvus_embedding_item_id(
                 "hf_inst",
                 self.model_name,
                 source,

src/embedding_ollama.py

@@ -35,12 +35,6 @@ def dim(self):
         self.dimensions = len(query_result)
         return self.dimensions
 
-    def getname(self, start_date, prefix="ollama"):
-        """
-        Get a embedding collection name of milvus
-        """
-        return f"embedding__{prefix}__ollama_{self.model_name}__{start_date}".replace("-", "_")
-
     def create(
         self,
         text: str,

src/embedding_openai_0x.py

@@ -21,9 +21,6 @@ def __init__(self, model_name="text-embedding-ada-002"):
     def dim(self):
         return 1536
 
-    def getname(self, start_date, prefix="news"):
-        return f"{prefix}_embedding__{start_date}".replace("-", "_")
-
     def create(
         self,
         text: str,

src/embedding_openai_1x.py

@@ -30,9 +30,6 @@ def __init__(self, model_name="text-embedding-ada-002"):
     def dim(self):
         return 1536
 
-    def getname(self, start_date, prefix="news"):
-        return f"{prefix}_embedding__{start_date}".replace("-", "_")
-
     def create(
         self,
         text: str,