Skip to content

Commit

Permalink
Update utils.py
Browse files Browse the repository at this point in the history
  • Loading branch information
LarFii authored Nov 3, 2024
1 parent d1e3f36 commit 6dd778e
Showing 1 changed file with 3 additions and 12 deletions.
15 changes: 3 additions & 12 deletions lightrag/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,11 +176,6 @@ def truncate_list_by_token_size(list_data: list, key: callable, max_token_size:
return list_data[:i]
return list_data


# def list_of_list_to_csv(data: list[list]):
# return "\n".join(
# [",\t".join([str(data_dd) for data_dd in data_d]) for data_d in data]
# )
def list_of_list_to_csv(data: List[List[str]]) -> str:
output = io.StringIO()
writer = csv.writer(output)
Expand Down Expand Up @@ -258,12 +253,11 @@ def xml_to_json(xml_file):
print(f"An error occurred: {e}")
return None

#混合检索中的合并函数
def process_combine_contexts(hl, ll):
header = None
list_hl = csv_string_to_list(hl.strip())
list_ll = csv_string_to_list(ll.strip())
# 去掉第一个元素(如果不为空)

if list_hl:
header=list_hl[0]
list_hl = list_hl[1:]
Expand All @@ -272,24 +266,21 @@ def process_combine_contexts(hl, ll):
list_ll = list_ll[1:]
if header is None:
return ""
# 去掉每个子元素中的第一个元素(如果不为空),再转为一维数组,用于合并去重

if list_hl:
list_hl = [','.join(item[1:]) for item in list_hl if item]
if list_ll:
list_ll = [','.join(item[1:]) for item in list_ll if item]

# 合并并去重
combined_sources_set = set(
filter(None, list_hl + list_ll)
)

# 创建包含头部的新列表
combined_sources = [",\t".join(header)]
# 为 combined_sources_set 中的每个元素添加自增数字

for i, item in enumerate(combined_sources_set, start=1):
combined_sources.append(f"{i},\t{item}")

# 将列表转换为字符串,子元素之间用换行符分隔
combined_sources = "\n".join(combined_sources)

return combined_sources

0 comments on commit 6dd778e

Please sign in to comment.