Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BEIR dataset comparison -- adding queries2queries and queries2corpus comparison; Fixing bugs #1169

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
Open

BEIR dataset comparison -- adding queries2queries and queries2corpus comparison; Fixing bugs #1169

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
f1ce7d1
init wp-tokenizer for beir stats
amyxie361 Apr 28, 2022
66ebe22
modify tokenize_corpus from msmarco to beir
amyxie361 Apr 28, 2022
f463df8
add query tokenizer
amyxie361 Apr 28, 2022
4089fee
runable compare domains
amyxie361 May 1, 2022
6bae387
test all the metrics
amyxie361 May 2, 2022
c7eb404
use beir format
amyxie361 May 2, 2022
271ac2a
update with BeirFlatCollection
amyxie361 May 2, 2022
7ece346
add kl-divergency and msmarco
amyxie361 May 10, 2022
1bf765f
add js divergence, fix weighted_jaccard, add tokenizer option
amyxie361 May 11, 2022
a9ea6f4
add tokenizer option
amyxie361 May 11, 2022
a2af116
fix bug and add query2query query2corpus comparison
amyxie361 May 12, 2022
c4b3c2e
remove previous results
amyxie361 May 12, 2022
69ad849
merge castorini master
amyxie361 May 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 69 additions & 38 deletions scripts/beir/compare_domains.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def count_total(d):
def kl_divergence(d1, d2):
value = float(0)
for w in d1:
if w in d2:
if w in d2: # through out zero tokens for both sets
value += d1[w] * np.log(d1[w] / d2[w])
return value

Expand Down Expand Up @@ -101,55 +101,86 @@ def print_results(datasets, results, save_file):
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--index_path', type=str, help='path to indexes of all the beir dataset', required=True)
parser.add_argument('--index_name_format', type=str, help='define your own index dir path name', default="/lucene-index-beir-{}")
parser.add_argument('--compare_metric', type=str, help='the metric for comparing two vocab, choose from: jaccard, weight_jaccard, df_filter, tf_filter, kl_divergence, js_divergence', default="weight_jaccard")
parser.add_argument('--compare_threshold', type=float, help='when choosing df_filter, or tf_filter, you can choolse the threshold', default=0.0001)
parser.add_argument('--output_path', type=str, help='path to save the stat results', required=True)
parser.add_argument('--compare_sets', type=str, default="c2c", help="choose from c2c, q2q, q2c")
args = parser.parse_args()

corpus_format = "/corpus/lucene-index-beir-{}"
queries_format = "/queries/lucene-index-beir-queires-{}"

beir_datasets = ['trec-covid', 'bioasq', 'nfcorpus', 'nq', 'hotpotqa', 'climate-fever', 'fever', 'dbpedia-entity', 'fiqa', 'signal1m', 'trec-news', 'robust04', 'arguana', 'webis-touche2020', 'quora', 'cqadupstack', 'scidocs', 'scifact', 'msmarco']
#beir_datasets = ['arguana', 'fiqa']
cfs = dfs = stats = {}
for d in beir_datasets:
cf, df, stat = index2stats(args.index_path + args.index_name_format.format(d))
cfs[d] = cf # count frequency -- int
dfs[d] = df # document frequency -- int
stat[d] = stat
#beir_datasets = ['trec-covid', 'bioasq', 'nfcorpus', 'nq'] # Testing
cfs = {}
dfs = {}
summary = {}
cfs2 = {}
dfs2 = {}
summary2 = {}
if args.compare_sets == "c2c":
for d in beir_datasets:
cf, df, stat = index2stats(args.index_path + corpus_format.format(d))
cfs[d] = cf # count frequency -- int
dfs[d] = df # document frequency -- int
summary[d] = stat
# stats[d] = stat
cfs2 = cfs
dfs2 = dfs
summary2 = summary
elif args.compare_sets == "q2q":
for d in beir_datasets:
cf, df, stat = index2stats(args.index_path + queries_format.format(d))
cfs[d] = cf
dfs[d] = df
summary[d] = stat
cfs2 = cfs
dfs2 = dfs
dfs2 = dfs
summary2 = summary
elif args.compare_sets == "q2c":
for d in beir_datasets:
cf, df, stat = index2stats(args.index_path + queries_format.format(d))
cfs[d] = cf
dfs[d] = df
summary[d] = stat
for d in beir_datasets:
cf, df, stat = index2stats(args.index_path + corpus_format.format(d))
cfs2[d] = cf
dfs2[d] = df
summary2[d] = stat
else:
NotImplementedError("--compare_sets {}".format(args.compare_sets))


results = {}
for d1 in beir_datasets:
metric_d1 = {}
for d2 in beir_datasets:
if d1 == d2:
if args.compare_metric in ["jaccard", "weight_jaccard", "df_filter", "tf_filter"]:
metric_d1[d2] = 1
elif args.compare_metric in ["kl_divergence", "js_divergence"]:
metric_d1[d2] = 0
if args.compare_metric == "jaccard":
metric_d1[d2] = jaccard(cfs[d1], cfs2[d2])
elif args.compare_metric == "weight_jaccard":
new_d1 = cf2freq(cfs[d1])
new_d2 = cf2freq(cfs2[d2])
metric_d1[d2] = weighted_jaccard(new_d1, new_d2)
elif args.compare_metric == "df_filter":
new_d1 = filter_freq_dict(cf2freq(cfs[d1]))
new_d2 = filter_freq_dict(cf2freq(cfs2[d2]))
metric_d1[d2] = jaccard(new_d1, new_d2)
elif args.compare_metric == "tf_filter":
new_d1 = filter_freq_dict(df2idf(dfs[d1], summary[d1]["documents"]))
new_d2 = filter_freq_dict(df2idf(dfs2[d2], summary2[d2]["documents"]))
metric_d1[d2] = jaccard(new_d1, new_d2)
elif args.compare_metric == "kl_divergence":
new_d1 = filter_freq_dict(cf2freq(cfs[d1]))
new_d2 = filter_freq_dict(cf2freq(cfs2[d2]))
metric_d1[d2] = kl_divergence(new_d1, new_d2)
elif args.compare_metric == "js_divergence":
new_d1 = filter_freq_dict(cf2freq(cfs[d1]))
new_d2 = filter_freq_dict(cf2freq(cfs2[d2]))
metric_d1[d2] = js_divergence(new_d1, new_d2)
else:
if args.compare_metric == "jaccard":
metric_d1[d2] = jaccard(cfs[d1], cfs[d2])
elif args.compare_metric == "weight_jaccard":
new_d1 = filter_freq_dict(cf2freq(cfs[d1]))
new_d2 = filter_freq_dict(cf2freq(cfs[d2]))
metric_d1[d2] = weighted_jaccard(new_d1, new_d2)
elif args.compare_metric == "df_filter":
new_d1 = filter_freq_dict(cf2freq(cfs[d1]))
new_d2 = filter_freq_dict(cf2freq(cfs[d2]))
metric_d1[d2] = jaccard(new_d1, new_d2)
elif args.compare_metric == "tf_filter":
new_d1 = filter_freq_dict(df2idf(dfs[d1], 1))
new_d2 = filter_freq_dict(df2idf(dfs[d2], 1))
metric_d1[d2] = jaccard(new_d1, new_d2)
elif args.compare_metric == "kl_divergence":
new_d1 = filter_freq_dict(cf2freq(cfs[d1]))
new_d2 = filter_freq_dict(cf2freq(cfs[d2]))
metric_d1[d2] = kl_divergence(new_d1, new_d2)
elif args.compare_metric == "js_divergence":
new_d1 = filter_freq_dict(cf2freq(cfs[d1]))
new_d2 = filter_freq_dict(cf2freq(cfs[d2]))
metric_d1[d2] = js_divergence(new_d1, new_d2)
else:
raise NotImplementedError
raise NotImplementedError
results[d1] = metric_d1

print_results(beir_datasets, results, args.output_path)
Expand Down
20 changes: 20 additions & 0 deletions scripts/beir/similarities/q2c-df_filter.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
trec-covid bioasq nfcorpus nq hotpotqa climate-fever fever dbpedia-entity fiqa signal1m trec-news robust04 arguana webis-touche2020 quora cqadupstack scidocs scifact msmarco
trec-covid 0.1393 0.1130 0.1093 0.1017 0.0794 0.0814 0.0814 0.0757 0.0898 0.0668 0.0924 0.0968 0.1050 0.1100 0.0902 0.0803 0.0946 0.1171 0.1067
bioasq 0.3534 0.4330 0.3275 0.1742 0.1925 0.1925 0.1925 0.1912 0.1277 0.2033 0.1332 0.1551 0.1313 0.1427 0.1603 0.1912 0.2196 0.4567 0.2174
nfcorpus 0.2330 0.2657 0.3053 0.1679 0.1696 0.1736 0.1736 0.1672 0.1372 0.1997 0.1447 0.1526 0.1424 0.1495 0.1730 0.1547 0.1556 0.2594 0.2213
nq 0.1836 0.1822 0.1761 0.3999 0.3369 0.3645 0.3645 0.3307 0.2492 0.2935 0.3329 0.3104 0.2602 0.2879 0.3304 0.2253 0.2052 0.1710 0.3576
hotpotqa 0.1839 0.1809 0.1677 0.4478 0.5847 0.5973 0.5973 0.5734 0.1987 0.3041 0.3199 0.3037 0.2234 0.2236 0.2652 0.2082 0.1839 0.1734 0.3127
climate-fever 0.2352 0.2173 0.2252 0.3268 0.2276 0.2434 0.2434 0.2224 0.2962 0.2339 0.2994 0.3205 0.3119 0.3062 0.2621 0.2299 0.2490 0.2183 0.3345
fever 0.1675 0.1664 0.1535 0.3497 0.4121 0.4207 0.4207 0.4116 0.1684 0.2767 0.2695 0.2489 0.2036 0.2022 0.2395 0.1747 0.1564 0.1569 0.2594
dbpedia-entity 0.1006 0.0991 0.0987 0.1901 0.2187 0.2194 0.2194 0.2166 0.1079 0.1660 0.1562 0.1551 0.1330 0.1235 0.1559 0.1111 0.1092 0.0954 0.1574
fiqa 0.2103 0.1905 0.1923 0.2921 0.1905 0.2092 0.2092 0.1872 0.5152 0.2450 0.2997 0.3302 0.3154 0.3223 0.3423 0.2577 0.2460 0.1849 0.3371
signal1m 0.1051 0.1047 0.1023 0.1747 0.1546 0.1565 0.1565 0.1525 0.1548 0.1863 0.1787 0.1748 0.1570 0.1517 0.1719 0.1285 0.1132 0.1016 0.1651
trec-news 0.1006 0.0916 0.0950 0.1550 0.1197 0.1261 0.1261 0.1182 0.1522 0.1408 0.1797 0.1549 0.1504 0.1605 0.1587 0.1150 0.0974 0.0896 0.1476
robust04 0.1889 0.1672 0.1814 0.2069 0.1694 0.1759 0.1759 0.1647 0.1718 0.1658 0.1853 0.2250 0.2444 0.2039 0.1911 0.1300 0.1882 0.1680 0.2143
arguana 0.2530 0.2128 0.2192 0.3942 0.2540 0.2738 0.2738 0.2477 0.3770 0.2650 0.3876 0.4195 0.7783 0.4592 0.3407 0.2437 0.2934 0.2134 0.3659
webis-touche2020 0.0423 0.0391 0.0446 0.0597 0.0429 0.0455 0.0455 0.0419 0.0640 0.0469 0.0675 0.0605 0.0879 0.0886 0.0751 0.0404 0.0465 0.0402 0.0632
quora 0.2167 0.2013 0.1944 0.3613 0.2628 0.2837 0.2837 0.2550 0.3660 0.3250 0.3784 0.3436 0.3442 0.3906 0.7351 0.2971 0.2467 0.1887 0.3978
cqadupstack 0.2354 0.2261 0.1994 0.2831 0.2092 0.2272 0.2272 0.2042 0.2937 0.2456 0.2589 0.2650 0.2485 0.2837 0.3468 0.4928 0.3361 0.2148 0.3473
scidocs 0.2158 0.2124 0.1799 0.1783 0.1539 0.1616 0.1616 0.1529 0.1460 0.1614 0.1439 0.1575 0.1532 0.1513 0.1785 0.1989 0.3926 0.2025 0.1985
scifact 0.3480 0.4334 0.3390 0.1805 0.1742 0.1801 0.1801 0.1750 0.1333 0.1852 0.1379 0.1566 0.1501 0.1446 0.1569 0.1637 0.2130 0.4916 0.2146
msmarco 0.2836 0.2972 0.2833 0.3513 0.3169 0.3344 0.3344 0.3105 0.2788 0.3285 0.2867 0.2934 0.2435 0.2749 0.3910 0.2775 0.2493 0.2734 0.5009
20 changes: 20 additions & 0 deletions scripts/beir/similarities/q2c-js_divergence.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
trec-covid bioasq nfcorpus nq hotpotqa climate-fever fever dbpedia-entity fiqa signal1m trec-news robust04 arguana webis-touche2020 quora cqadupstack scidocs scifact msmarco
trec-covid 0.3183 0.3676 0.3849 0.3597 0.3819 0.3684 0.3684 0.3820 0.3965 0.4042 0.3868 0.3549 0.3679 0.3751 0.3260 0.4248 0.3683 0.3671 0.3522
bioasq 0.2216 0.1939 0.2452 0.2776 0.2744 0.2702 0.2702 0.2731 0.3335 0.2988 0.3278 0.2907 0.3208 0.3132 0.2495 0.3148 0.2702 0.1810 0.2579
nfcorpus 0.3008 0.2820 0.2583 0.3178 0.3277 0.3191 0.3191 0.3272 0.3482 0.2987 0.3360 0.3254 0.3471 0.3345 0.3020 0.3600 0.3371 0.2873 0.2799
nq 0.2869 0.2845 0.3068 0.1743 0.1953 0.1829 0.1829 0.1960 0.2434 0.2454 0.2198 0.2069 0.2260 0.2203 0.1841 0.3058 0.2603 0.2981 0.1856
hotpotqa 0.2499 0.2482 0.2704 0.1275 0.0960 0.1008 0.1008 0.1018 0.2376 0.2236 0.1957 0.1734 0.2168 0.2159 0.1813 0.2764 0.2329 0.2610 0.1750
climate-fever 0.2160 0.2250 0.2305 0.1478 0.1999 0.1911 0.1911 0.2025 0.1872 0.2395 0.1728 0.1546 0.1561 0.1659 0.2440 0.2633 0.1941 0.2328 0.1506
fever 0.2465 0.2418 0.2618 0.1408 0.1121 0.1106 0.1106 0.1112 0.2412 0.2140 0.1955 0.1808 0.2184 0.2127 0.2471 0.2789 0.2337 0.2576 0.1670
dbpedia-entity 0.3952 0.3931 0.4094 0.3073 0.2939 0.2892 0.2892 0.2932 0.3938 0.3597 0.3492 0.3332 0.3557 0.3660 0.3371 0.4233 0.3751 0.4050 0.3299
fiqa 0.2888 0.2930 0.3090 0.2280 0.2771 0.2605 0.2605 0.2768 0.1349 0.2557 0.2288 0.2061 0.2181 0.2053 0.1374 0.2552 0.2568 0.3045 0.1928
signal1m 0.3789 0.3756 0.3903 0.3191 0.3378 0.3278 0.3278 0.3369 0.3549 0.3094 0.3284 0.3129 0.3494 0.3549 0.3713 0.3845 0.3720 0.3886 0.3224
trec-news 0.3678 0.3770 0.3844 0.3170 0.3488 0.3356 0.3356 0.3485 0.3306 0.3366 0.3052 0.3157 0.3212 0.3252 0.3403 0.3927 0.3572 0.3847 0.3051
robust04 0.2822 0.2948 0.2968 0.2344 0.2638 0.2546 0.2546 0.2662 0.2772 0.3146 0.2741 0.2331 0.2282 0.2492 0.2574 0.3427 0.2604 0.3010 0.2303
arguana 0.1881 0.2051 0.2091 0.0948 0.1652 0.1533 0.1533 0.1661 0.1285 0.2014 0.1132 0.1019 0.0152 0.0906 0.1844 0.2164 0.1505 0.2118 0.1084
webis-touche2020 0.4870 0.4907 0.5016 0.4444 0.4620 0.4544 0.4544 0.4623 0.4601 0.4669 0.4590 0.4445 0.4284 0.4409 0.3754 0.4994 0.4657 0.4979 0.4337
quora 0.2889 0.2929 0.3110 0.2058 0.2450 0.2341 0.2341 0.2478 0.1782 0.2259 0.2103 0.2086 0.2045 0.1707 0.0197 0.2490 0.2524 0.3046 0.1759
cqadupstack 0.2505 0.2455 0.2745 0.1985 0.2368 0.2313 0.2313 0.2423 0.1895 0.2232 0.2225 0.2041 0.2267 0.1899 0.1315 0.1410 0.1996 0.2591 0.1715
scidocs 0.2915 0.2898 0.3205 0.3076 0.3200 0.3082 0.3082 0.3189 0.3653 0.3241 0.3372 0.3121 0.3470 0.3534 0.3522 0.3487 0.1940 0.3017 0.2969
scifact 0.1811 0.1440 0.1905 0.2613 0.2666 0.2561 0.2561 0.2626 0.3200 0.2871 0.3070 0.2688 0.3047 0.3041 0.3307 0.3117 0.2519 0.1162 0.2354
msmarco 0.2524 0.2451 0.2665 0.2008 0.2152 0.2062 0.2062 0.2152 0.2202 0.2217 0.2298 0.2172 0.2313 0.2168 0.1290 0.2666 0.2421 0.2583 0.1500
20 changes: 20 additions & 0 deletions scripts/beir/similarities/q2c-kl_divergence.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
trec-covid bioasq nfcorpus nq hotpotqa climate-fever fever dbpedia-entity fiqa signal1m trec-news robust04 arguana webis-touche2020 quora cqadupstack scidocs scifact msmarco
trec-covid 1.3171 1.5038 1.1645 1.5557 1.1925 1.1408 1.1408 0.9532 1.0464 1.3154 1.2979 1.3355 1.3443 0.9990 0.7820 1.2226 1.3264 1.2179 1.4574
bioasq 0.4752 0.7102 0.3978 0.8157 0.3905 0.3777 0.3777 0.2632 0.4918 0.6834 0.6380 0.6360 0.5517 0.4903 0.2398 0.4770 0.7853 0.3281 0.7250
nfcorpus 0.2896 0.4219 0.3631 0.3448 0.1964 0.1872 0.1872 0.1796 0.1171 0.2434 0.1895 0.2277 0.2054 0.1432 0.1329 0.1255 0.2553 0.3123 0.4695
nq 0.6697 0.6104 0.6476 0.6461 0.5398 0.5819 0.5819 0.4346 0.5066 0.7994 0.6049 0.5834 0.5205 0.4989 0.4141 0.7528 0.5870 0.6352 0.6111
hotpotqa 0.3564 0.4486 0.2768 0.5484 0.2580 0.2638 0.2638 0.1741 0.3566 0.6152 0.4689 0.4241 0.4318 0.3451 0.2707 0.4518 0.5310 0.2513 0.4842
climate-fever 0.3194 0.2791 0.3221 0.2604 0.2667 0.2332 0.2332 0.2684 0.1397 0.4865 0.1754 0.1666 0.2025 0.1652 0.3798 0.3445 0.2696 0.3130 0.2298
fever 0.2779 0.2559 0.2806 0.3023 0.2154 0.2306 0.2306 0.2118 0.2123 0.4458 0.2788 0.2252 0.2366 0.1892 0.4571 0.3251 0.2422 0.2774 0.2645
dbpedia-entity 0.3518 0.4480 0.3473 0.7123 0.4688 0.4821 0.4821 0.4532 0.4179 0.6805 0.5349 0.5395 0.4326 0.3993 0.4276 0.4748 0.5365 0.3164 0.6126
fiqa 0.4496 0.6679 0.4018 0.8408 0.3873 0.3725 0.3725 0.3064 0.4936 0.6569 0.6042 0.7719 0.6030 0.4260 0.2688 0.4113 0.6733 0.3843 0.7004
signal1m 0.3475 0.3260 0.3270 0.4788 0.4410 0.4079 0.4079 0.4372 0.3117 0.5070 0.4102 0.3603 0.4289 0.3510 0.5111 0.2662 0.3942 0.3434 0.4072
trec-news 0.5208 0.4706 0.5086 0.6563 0.5731 0.4983 0.4983 0.5693 0.4899 0.6819 0.5679 0.5583 0.5521 0.5472 0.6004 0.5097 0.4776 0.4631 0.5420
robust04 0.5762 0.5368 0.4200 0.5896 0.4293 0.3977 0.3977 0.2987 0.3907 0.6561 0.5068 0.5127 0.4510 0.3981 0.3908 0.4927 0.6771 0.4396 0.4966
arguana 0.3261 0.3043 0.3251 0.2163 0.3057 0.2324 0.2324 0.3226 0.1342 0.4923 0.1779 0.1673 0.0086 0.1116 0.3763 0.3046 0.2496 0.3176 0.1932
webis-touche2020 1.1044 1.8766 1.1320 2.1298 0.6137 0.6239 0.6239 0.6142 1.4826 1.6078 1.7456 1.7416 1.8355 1.6536 1.0439 1.2382 1.9108 1.1932 1.8591
quora 0.7001 0.9831 0.5066 1.2133 0.5968 0.5866 0.5866 0.3828 0.5919 0.9419 0.8957 1.0112 0.8785 0.6004 0.0487 0.7040 1.0933 0.4753 0.9167
cqadupstack 0.4205 0.6056 0.3868 0.6199 0.2873 0.2564 0.2564 0.2716 0.2657 0.4874 0.4727 0.4815 0.4274 0.2820 0.1423 0.3226 0.5857 0.3824 0.4424
scidocs 0.3670 0.3062 0.2769 0.3921 0.3258 0.3291 0.3291 0.2623 0.3085 0.2957 0.2987 0.2691 0.3982 0.3358 0.4599 0.3927 0.4649 0.3636 0.3903
scifact 0.3018 0.2893 0.2767 0.2787 0.2422 0.2234 0.2234 0.2265 0.1821 0.3003 0.2207 0.1837 0.2009 0.1968 0.3422 0.1784 0.3055 0.2355 0.3151
msmarco 0.6714 0.4714 0.4600 0.6986 0.4999 0.4973 0.4973 0.1797 0.3832 0.6074 0.5625 0.6057 0.4861 0.4326 0.1259 0.5138 0.6222 0.4238 0.5427
20 changes: 20 additions & 0 deletions scripts/beir/similarities/q2c-tf_filter.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
trec-covid bioasq nfcorpus nq hotpotqa climate-fever fever dbpedia-entity fiqa signal1m trec-news robust04 arguana webis-touche2020 quora cqadupstack scidocs scifact msmarco
trec-covid 0.0116 0.0106 0.0206 0.0101 0.0101 0.0101 0.0101 0.0101 0.0121 0.0104 0.0105 0.0106 0.0139 0.0104 0.0113 0.0104 0.0131 0.0198 0.0106
bioasq 0.2015 0.1849 0.3231 0.1763 0.1765 0.1768 0.1768 0.1765 0.2043 0.1810 0.1825 0.1855 0.2193 0.1818 0.1963 0.1807 0.2239 0.3228 0.1846
nfcorpus 0.1358 0.1250 0.2271 0.1191 0.1193 0.1194 0.1195 0.1193 0.1392 0.1228 0.1234 0.1253 0.1524 0.1228 0.1330 0.1221 0.1495 0.2054 0.1248
nq 0.2360 0.2275 0.2729 0.2170 0.2173 0.2176 0.2177 0.2173 0.2457 0.2236 0.2247 0.2282 0.2617 0.2238 0.2403 0.2216 0.2504 0.2711 0.2276
hotpotqa 0.8068 0.8765 0.4888 0.8514 0.8526 0.8532 0.8532 0.8525 0.7796 0.8695 0.8748 0.8776 0.7027 0.8667 0.8236 0.8459 0.7325 0.5074 0.8775
climate-fever 0.1841 0.1694 0.2715 0.1616 0.1618 0.1620 0.1619 0.1618 0.1907 0.1666 0.1674 0.1697 0.2159 0.1666 0.1793 0.1658 0.2029 0.2631 0.1692
fever 0.5803 0.5947 0.4570 0.5674 0.5681 0.5692 0.5692 0.5680 0.5875 0.5846 0.5876 0.5961 0.5888 0.5845 0.5975 0.5750 0.5683 0.4695 0.5953
dbpedia-entity 0.0510 0.0484 0.0658 0.0461 0.0462 0.0463 0.0463 0.0462 0.0531 0.0476 0.0478 0.0486 0.0596 0.0475 0.0512 0.0473 0.0546 0.0623 0.0484
fiqa 0.2159 0.1991 0.2991 0.1898 0.1901 0.1903 0.1903 0.1900 0.2281 0.1957 0.1966 0.1994 0.2560 0.1957 0.2126 0.1948 0.2398 0.3014 0.1988
signal1m 0.0321 0.0299 0.0452 0.0285 0.0285 0.0285 0.0285 0.0285 0.0339 0.0293 0.0295 0.0299 0.0383 0.0294 0.0319 0.0292 0.0352 0.0448 0.0297
trec-news 0.0207 0.0191 0.0312 0.0182 0.0182 0.0182 0.0182 0.0182 0.0218 0.0188 0.0189 0.0190 0.0251 0.0188 0.0202 0.0187 0.0228 0.0304 0.0190
robust04 0.0605 0.0556 0.0936 0.0530 0.0531 0.0532 0.0532 0.0531 0.0627 0.0547 0.0549 0.0559 0.0733 0.0547 0.0593 0.0544 0.0663 0.0894 0.0556
arguana 0.5176 0.4919 0.5190 0.4693 0.4699 0.4701 0.4702 0.4698 0.5384 0.4838 0.4861 0.4931 0.6626 0.4840 0.5140 0.4798 0.5438 0.5342 0.4916
webis-touche2020 0.0083 0.0076 0.0128 0.0072 0.0072 0.0073 0.0073 0.0072 0.0087 0.0075 0.0075 0.0076 0.0102 0.0075 0.0081 0.0074 0.0093 0.0122 0.0076
quora 0.4320 0.4139 0.4550 0.3949 0.3954 0.3959 0.3960 0.3954 0.4545 0.4069 0.4089 0.4142 0.4703 0.4069 0.4426 0.4042 0.4586 0.4550 0.4136
cqadupstack 0.3533 0.3317 0.4277 0.3168 0.3173 0.3175 0.3176 0.3172 0.3712 0.3259 0.3280 0.3320 0.3869 0.3267 0.3537 0.3252 0.3909 0.4426 0.3316
scidocs 0.1310 0.1195 0.2000 0.1143 0.1145 0.1145 0.1145 0.1144 0.1352 0.1177 0.1182 0.1196 0.1471 0.1179 0.1280 0.1173 0.1474 0.2026 0.1193
scifact 0.1346 0.1231 0.2317 0.1173 0.1175 0.1177 0.1177 0.1175 0.1369 0.1203 0.1215 0.1233 0.1518 0.1210 0.1309 0.1203 0.1505 0.2302 0.1228
msmarco 0.8630 0.9449 0.5137 0.9058 0.9070 0.9074 0.9075 0.9068 0.8388 0.9290 0.9365 0.9466 0.7437 0.9286 0.8926 0.9034 0.7892 0.5389 0.9466
Loading