diff --git "a/scores/LamRA-Ret-Qwen2.5VL-7b.json" "b/scores/LamRA-Ret-Qwen2.5VL-7b.json" --- "a/scores/LamRA-Ret-Qwen2.5VL-7b.json" +++ "b/scores/LamRA-Ret-Qwen2.5VL-7b.json" @@ -1,10 +1,7 @@ { "metadata": { "model_name": "LamRA-Ret-Qwen2.5VL-7b", - "report_generated_date": "2025-06-09T07:00:24.383583", - "model_size": 8.29, - "url": "https://huggingface.co/code-kunkun/LamRA-Ret-Qwen2.5VL-7b", - "data_source": "TIGER-Lab" + "report_generated_date": "2025-06-28T08:54:18.581107" }, "metrics": { "image": { @@ -1017,765 +1014,287 @@ "num_data": 1000 } }, - "visdoc": { - "ViDoRe_arxivqa": { - "hit@1": 0.45, - "hit@5": 0.602, - "hit@10": 0.67, - "ndcg_linear@1": 0.45, - "ndcg_linear@5": 0.5298461278515469, - "ndcg_linear@10": 0.5513653011991692, - "ndcg_exponential@1": 0.45, - "ndcg_exponential@5": 0.5298461278515469, - "ndcg_exponential@10": 0.5513653011991692, - "precision@1": 0.45, - "precision@5": 0.1204, - "precision@10": 0.067, - "recall@1": 0.45, - "recall@5": 0.602, - "recall@10": 0.67, - "f1@1": 0.45, - "f1@5": 0.20066666666666674, - "f1@10": 0.12181818181818184, - "map@1": 0.45, - "map@5": 0.5059666666666667, - "map@10": 0.5145904761904763, - "mrr@1": 0.45, - "mrr@5": 0.5059666666666667, - "mrr@10": 0.5145904761904763, - "num_pred": 500, - "num_data": 500 - }, - "ViDoRe_docvqa": { - "hit@1": 0.15964523281596452, - "hit@5": 0.34146341463414637, - "hit@10": 0.4279379157427938, - "ndcg_linear@1": 0.15964523281596452, - "ndcg_linear@5": 0.2541845458267961, - "ndcg_linear@10": 0.2816434324462572, - "ndcg_exponential@1": 0.15964523281596452, - "ndcg_exponential@5": 0.2541845458267961, - "ndcg_exponential@10": 0.2816434324462572, - "precision@1": 0.15964523281596452, - "precision@5": 0.06962305986696231, - "precision@10": 0.043458980044345896, - "recall@1": 0.15787139689578714, - "recall@5": 0.3356984478935698, - "recall@10": 0.42050997782705096, - "f1@1": 0.15816703621581674, - "f1@5": 0.11447048886073279, - "f1@10": 0.07839448061177551, - "map@1": 0.15964523281596452, - "map@5": 0.22555432372505546, - "map@10": 0.23678421849153558, - "mrr@1": 0.15964523281596452, - "mrr@5": 0.22893569844789358, - "mrr@10": 0.24044275507690144, - "num_pred": 451, - "num_data": 451 - }, - "ViDoRe_infovqa": { - "hit@1": 0.6133603238866396, - "hit@5": 0.8178137651821862, - "hit@10": 0.8805668016194332, - "ndcg_linear@1": 0.6133603238866396, - "ndcg_linear@5": 0.7231460909674565, - "ndcg_linear@10": 0.7438877002962954, - "ndcg_exponential@1": 0.6133603238866396, - "ndcg_exponential@5": 0.7231460909674565, - "ndcg_exponential@10": 0.7438877002962954, - "precision@1": 0.6133603238866396, - "precision@5": 0.16356275303643722, - "precision@10": 0.08805668016194332, - "recall@1": 0.6123481781376519, - "recall@5": 0.8168016194331984, - "recall@10": 0.8795546558704453, - "f1@1": 0.612685560053981, - "f1@5": 0.2725081935608252, - "f1@10": 0.1600723837565943, - "map@1": 0.6133603238866396, - "map@5": 0.6914642375168691, - "map@10": 0.7002907910802648, - "mrr@1": 0.6133603238866396, - "mrr@5": 0.692476383265857, - "mrr@10": 0.7013029368292527, - "num_pred": 494, - "num_data": 494 - }, - "ViDoRe_tabfquad": { - "hit@1": 0.5428571428571428, - "hit@5": 0.7607142857142857, - "hit@10": 0.8142857142857143, - "ndcg_linear@1": 0.5428571428571428, - "ndcg_linear@5": 0.6607864639631176, - "ndcg_linear@10": 0.6777435779728691, - "ndcg_exponential@1": 0.5428571428571428, - "ndcg_exponential@5": 0.6607864639631176, - "ndcg_exponential@10": 0.6777435779728691, - "precision@1": 0.5428571428571428, - "precision@5": 0.15214285714285714, - "precision@10": 0.08142857142857142, - "recall@1": 0.5428571428571428, - "recall@5": 0.7607142857142857, - "recall@10": 0.8142857142857143, - "f1@1": 0.5428571428571428, - "f1@5": 0.2535714285714286, - "f1@10": 0.14805194805194807, - "map@1": 0.5428571428571428, - "map@5": 0.6272619047619047, - "map@10": 0.634044784580499, - "mrr@1": 0.5428571428571428, - "mrr@5": 0.6272619047619047, - "mrr@10": 0.634044784580499, - "num_pred": 280, - "num_data": 280 - }, - "ViDoRe_tatdqa": { - "hit@1": 0.15674362089914945, - "hit@5": 0.35236938031591736, - "hit@10": 0.4641555285540705, - "ndcg_linear@1": 0.15674362089914945, - "ndcg_linear@5": 0.2585284591791044, - "ndcg_linear@10": 0.29467381069418136, - "ndcg_exponential@1": 0.15674362089914945, - "ndcg_exponential@5": 0.2585284591791044, - "ndcg_exponential@10": 0.29467381069418136, - "precision@1": 0.15674362089914945, - "precision@5": 0.07059538274605104, - "precision@10": 0.04659781287970838, - "recall@1": 0.15674362089914945, - "recall@5": 0.3515188335358445, - "recall@10": 0.46366950182260025, - "f1@1": 0.15674362089914945, - "f1@5": 0.11745067407278832, - "f1@10": 0.08458706137928496, - "map@1": 0.15674362089914945, - "map@5": 0.2275567031186715, - "map@10": 0.24240993076047754, - "mrr@1": 0.15674362089914945, - "mrr@5": 0.22790603483191574, - "mrr@10": 0.24270863468919363, - "num_pred": 1646, - "num_data": 1646 - }, - "ViDoRe_shiftproject": { - "hit@1": 0.13, - "hit@5": 0.4, - "hit@10": 0.6, - "ndcg_linear@1": 0.13, - "ndcg_linear@5": 0.2734127450408923, - "ndcg_linear@10": 0.33883350664214873, - "ndcg_exponential@1": 0.13, - "ndcg_exponential@5": 0.2734127450408923, - "ndcg_exponential@10": 0.33883350664214873, - "precision@1": 0.13, - "precision@5": 0.08, - "precision@10": 0.059999999999999984, - "recall@1": 0.13, - "recall@5": 0.4, - "recall@10": 0.6, - "f1@1": 0.13, - "f1@5": 0.1333333333333334, - "f1@10": 0.1090909090909091, - "map@1": 0.13, - "map@5": 0.23133333333333334, - "map@10": 0.25872222222222224, - "mrr@1": 0.13, - "mrr@5": 0.23133333333333334, - "mrr@10": 0.25872222222222224, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.59, - "hit@5": 0.85, - "hit@10": 0.88, - "ndcg_linear@1": 0.59, - "ndcg_linear@5": 0.7201422053206411, - "ndcg_linear@10": 0.7299635207551652, - "ndcg_exponential@1": 0.59, - "ndcg_exponential@5": 0.7201422053206411, - "ndcg_exponential@10": 0.7299635207551652, - "precision@1": 0.59, - "precision@5": 0.16999999999999996, - "precision@10": 0.08799999999999997, - "recall@1": 0.59, - "recall@5": 0.85, - "recall@10": 0.88, - "f1@1": 0.59, - "f1@5": 0.2833333333333334, - "f1@10": 0.16000000000000003, - "map@1": 0.59, - "map@5": 0.6778333333333333, - "map@10": 0.681940476190476, - "mrr@1": 0.59, - "mrr@5": 0.6778333333333333, - "mrr@10": 0.681940476190476, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.52, - "hit@5": 0.76, - "hit@10": 0.85, - "ndcg_linear@1": 0.52, - "ndcg_linear@5": 0.6517245745315876, - "ndcg_linear@10": 0.6815359848536943, - "ndcg_exponential@1": 0.52, - "ndcg_exponential@5": 0.6517245745315876, - "ndcg_exponential@10": 0.6815359848536943, - "precision@1": 0.52, - "precision@5": 0.15199999999999997, - "precision@10": 0.08499999999999998, - "recall@1": 0.52, - "recall@5": 0.76, - "recall@10": 0.85, - "f1@1": 0.52, - "f1@5": 0.25333333333333335, - "f1@10": 0.15454545454545457, - "map@1": 0.52, - "map@5": 0.6154999999999999, - "map@10": 0.6281984126984126, - "mrr@1": 0.52, - "mrr@5": 0.6154999999999999, - "mrr@10": 0.6281984126984126, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.58, - "hit@5": 0.84, - "hit@10": 0.88, - "ndcg_linear@1": 0.58, - "ndcg_linear@5": 0.7215467489235274, - "ndcg_linear@10": 0.7346071028524384, - "ndcg_exponential@1": 0.58, - "ndcg_exponential@5": 0.7215467489235274, - "ndcg_exponential@10": 0.7346071028524384, - "precision@1": 0.58, - "precision@5": 0.16799999999999998, - "precision@10": 0.088, - "recall@1": 0.58, - "recall@5": 0.84, - "recall@10": 0.88, - "f1@1": 0.58, - "f1@5": 0.2800000000000001, - "f1@10": 0.16000000000000003, - "map@1": 0.58, - "map@5": 0.6818333333333332, - "map@10": 0.6872896825396825, - "mrr@1": 0.58, - "mrr@5": 0.6818333333333332, - "mrr@10": 0.6872896825396825, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.7, - "hit@5": 0.94, - "hit@10": 0.98, - "ndcg_linear@1": 0.7, - "ndcg_linear@5": 0.8374771734856519, - "ndcg_linear@10": 0.8501891371833549, - "ndcg_exponential@1": 0.7, - "ndcg_exponential@5": 0.8374771734856519, - "ndcg_exponential@10": 0.8501891371833549, - "precision@1": 0.7, - "precision@5": 0.18799999999999997, - "precision@10": 0.09799999999999998, - "recall@1": 0.7, - "recall@5": 0.94, - "recall@10": 0.98, - "f1@1": 0.7, - "f1@5": 0.3133333333333334, - "f1@10": 0.17818181818181827, - "map@1": 0.7, - "map@5": 0.8026666666666666, - "map@10": 0.8077738095238094, - "mrr@1": 0.7, - "mrr@5": 0.8026666666666666, - "mrr@10": 0.8077738095238094, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_esg_reports_human_labeled_v2": { - "hit@1": 0.28846153846153844, - "hit@5": 0.4807692307692308, - "hit@10": 0.6153846153846154, - "ndcg_linear@1": 0.28846153846153844, - "ndcg_linear@5": 0.3295140713668188, - "ndcg_linear@10": 0.3803485934947541, - "ndcg_exponential@1": 0.28846153846153844, - "ndcg_exponential@5": 0.3296697471059318, - "ndcg_exponential@10": 0.3804833955905218, - "precision@1": 0.28846153846153844, - "precision@5": 0.12692307692307694, - "precision@10": 0.09423076923076924, - "recall@1": 0.20576923076923076, - "recall@5": 0.3809149184149184, - "recall@10": 0.5121503496503498, - "f1@1": 0.2269230769230769, - "f1@5": 0.17453449328449328, - "f1@10": 0.14645786905402286, - "map@1": 0.28846153846153844, - "map@5": 0.2831730769230769, - "map@10": 0.30995421245421245, - "mrr@1": 0.28846153846153844, - "mrr@5": 0.36025641025641025, - "mrr@10": 0.378968253968254, - "num_pred": 52, - "num_data": 52 - }, - "ViDoRe_biomedical_lectures_v2": { - "hit@1": 0.36875, - "hit@5": 0.69375, - "hit@10": 0.8, - "ndcg_linear@1": 0.36875, - "ndcg_linear@5": 0.4422432041813941, - "ndcg_linear@10": 0.4839405283326529, - "ndcg_exponential@1": 0.36875, - "ndcg_exponential@5": 0.4422432041813941, - "ndcg_exponential@10": 0.4839405283326529, - "precision@1": 0.36875, - "precision@5": 0.2125, - "precision@10": 0.144375, - "recall@1": 0.220367288961039, - "recall@5": 0.5001346049783549, - "recall@10": 0.6166988158899923, - "f1@1": 0.2525698953823954, - "f1@5": 0.2651350281662782, - "f1@10": 0.2102024014248279, - "map@1": 0.36875, - "map@5": 0.37088368055555554, - "map@10": 0.3911273148148148, - "mrr@1": 0.36875, - "mrr@5": 0.4951041666666667, - "mrr@10": 0.5092782738095238, - "num_pred": 160, - "num_data": 160 - }, - "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.2859375, - "hit@5": 0.5859375, - "hit@10": 0.684375, - "ndcg_linear@1": 0.2859375, - "ndcg_linear@5": 0.35874452751062924, - "ndcg_linear@10": 0.39816762044104737, - "ndcg_exponential@1": 0.2859375, - "ndcg_exponential@5": 0.35874452751062924, - "ndcg_exponential@10": 0.39816762044104737, - "precision@1": 0.2859375, - "precision@5": 0.1653125, - "precision@10": 0.11640625, - "recall@1": 0.1754330307750059, - "recall@5": 0.4161406827621331, - "recall@10": 0.5186508895780099, - "f1@1": 0.19900094696969695, - "f1@5": 0.2121714847142237, - "f1@10": 0.17188047002017592, - "map@1": 0.2859375, - "map@5": 0.2968263888888889, - "map@10": 0.3171867721915155, - "mrr@1": 0.2859375, - "mrr@5": 0.4043489583333333, - "mrr@10": 0.4175502232142857, - "num_pred": 640, - "num_data": 640 - }, - "ViDoRe_economics_reports_v2": { - "hit@1": 0.5344827586206896, - "hit@5": 0.8620689655172413, - "hit@10": 0.8793103448275862, - "ndcg_linear@1": 0.5344827586206896, - "ndcg_linear@5": 0.41005812732288577, - "ndcg_linear@10": 0.4162285501018457, - "ndcg_exponential@1": 0.5344827586206896, - "ndcg_exponential@5": 0.4100581273228857, - "ndcg_exponential@10": 0.4162285501018456, - "precision@1": 0.5344827586206896, - "precision@5": 0.3448275862068966, - "precision@10": 0.31551724137931036, - "recall@1": 0.0518705960313947, - "recall@5": 0.22158223006359476, - "recall@10": 0.3287786347831651, - "f1@1": 0.09077791021585119, - "f1@5": 0.2064727541694763, - "f1@10": 0.25527466440074914, - "map@1": 0.5344827586206896, - "map@5": 0.2972413793103448, - "map@10": 0.27172445558683245, - "mrr@1": 0.5344827586206896, - "mrr@5": 0.6494252873563218, - "mrr@10": 0.6522988505747126, - "num_pred": 58, - "num_data": 58 - }, - "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.3706896551724138, - "hit@5": 0.75, - "hit@10": 0.8189655172413793, - "ndcg_linear@1": 0.3706896551724138, - "ndcg_linear@5": 0.3194451317176268, - "ndcg_linear@10": 0.3202641880538733, - "ndcg_exponential@1": 0.3706896551724138, - "ndcg_exponential@5": 0.3194451317176268, - "ndcg_exponential@10": 0.3202641880538733, - "precision@1": 0.3706896551724138, - "precision@5": 0.2896551724137932, - "precision@10": 0.24870689655172415, - "recall@1": 0.03865338906185176, - "recall@5": 0.17160160948032535, - "recall@10": 0.25447962863698914, - "f1@1": 0.06726379557208981, - "f1@5": 0.1779005506820022, - "f1@10": 0.20701322326238306, - "map@1": 0.3706896551724138, - "map@5": 0.21762931034482758, - "map@10": 0.19331952955990825, - "mrr@1": 0.3706896551724138, - "mrr@5": 0.5000718390804598, - "mrr@10": 0.5099839217296114, - "num_pred": 232, - "num_data": 232 - }, - "ViDoRe_esg_reports_v2": { - "hit@1": 0.3508771929824561, - "hit@5": 0.7368421052631579, - "hit@10": 0.8596491228070176, - "ndcg_linear@1": 0.3508771929824561, - "ndcg_linear@5": 0.4091271605491938, - "ndcg_linear@10": 0.47796248526387447, - "ndcg_exponential@1": 0.3508771929824561, - "ndcg_exponential@5": 0.4091271605491938, - "ndcg_exponential@10": 0.4779624852638745, - "precision@1": 0.3508771929824561, - "precision@5": 0.2245614035087719, - "precision@10": 0.1754385964912281, - "recall@1": 0.16769005847953217, - "recall@5": 0.45461570593149536, - "recall@10": 0.6421261487050962, - "f1@1": 0.2046078677657625, - "f1@5": 0.2580472303596036, - "f1@10": 0.2409665311001469, - "map@1": 0.3508771929824561, - "map@5": 0.31769005847953213, - "map@10": 0.35532024505708715, - "mrr@1": 0.3508771929824561, - "mrr@5": 0.4874269005847952, - "mrr@10": 0.506084656084656, - "num_pred": 57, - "num_data": 57 - }, - "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.23684210526315788, - "hit@5": 0.6578947368421053, - "hit@10": 0.7587719298245614, - "ndcg_linear@1": 0.23684210526315788, - "ndcg_linear@5": 0.32514161979506145, - "ndcg_linear@10": 0.3716042928744199, - "ndcg_exponential@1": 0.23684210526315788, - "ndcg_exponential@5": 0.32514161979506145, - "ndcg_exponential@10": 0.3716042928744199, - "precision@1": 0.23684210526315788, - "precision@5": 0.17982456140350878, - "precision@10": 0.13552631578947372, - "recall@1": 0.10328947368421053, - "recall@5": 0.3982769423558897, - "recall@10": 0.5284043441938179, - "f1@1": 0.1284477705530337, - "f1@5": 0.21337619006696623, - "f1@10": 0.18932479489653187, - "map@1": 0.23684210526315788, - "map@5": 0.2391544834307992, - "map@10": 0.25983787593984964, - "mrr@1": 0.23684210526315788, - "mrr@5": 0.38559941520467833, - "mrr@10": 0.39949526594263435, - "num_pred": 228, - "num_data": 228 - }, - "VisRAG_ArxivQA": { - "hit@1": 0.29901960784313725, - "hit@5": 0.44362745098039214, - "hit@10": 0.508578431372549, - "ndcg_linear@1": 0.29901960784313725, - "ndcg_linear@5": 0.37679708973810255, - "ndcg_linear@10": 0.3979379930260254, - "ndcg_exponential@1": 0.29901960784313725, - "ndcg_exponential@5": 0.37679708973810255, - "ndcg_exponential@10": 0.3979379930260254, - "precision@1": 0.29901960784313725, - "precision@5": 0.08872549019607844, - "precision@10": 0.0508578431372549, - "recall@1": 0.29901960784313725, - "recall@5": 0.44362745098039214, - "recall@10": 0.508578431372549, - "f1@1": 0.29901960784313725, - "f1@5": 0.14787581699346408, - "f1@10": 0.09246880570409984, - "map@1": 0.29901960784313725, - "map@5": 0.35451388888888885, - "map@10": 0.36331602085278564, - "mrr@1": 0.29901960784313725, - "mrr@5": 0.35451388888888885, - "mrr@10": 0.36331602085278564, - "num_pred": 816, - "num_data": 816 - }, - "VisRAG_ChartQA": { - "hit@1": 0.5873015873015873, - "hit@5": 0.7142857142857143, - "hit@10": 0.746031746031746, - "ndcg_linear@1": 0.5873015873015873, - "ndcg_linear@5": 0.6589841568889536, - "ndcg_linear@10": 0.6681608180419024, - "ndcg_exponential@1": 0.5873015873015873, - "ndcg_exponential@5": 0.6589841568889536, - "ndcg_exponential@10": 0.6681608180419024, - "precision@1": 0.5873015873015873, - "precision@5": 0.14285714285714282, - "precision@10": 0.07460317460317457, - "recall@1": 0.5873015873015873, - "recall@5": 0.7142857142857143, - "recall@10": 0.746031746031746, - "f1@1": 0.5873015873015873, - "f1@5": 0.23809523809523817, - "f1@10": 0.13564213564213565, - "map@1": 0.5873015873015873, - "map@5": 0.6402116402116402, - "map@10": 0.6433862433862434, - "mrr@1": 0.5873015873015873, - "mrr@5": 0.6402116402116402, - "mrr@10": 0.6433862433862434, - "num_pred": 63, - "num_data": 63 + "video": { + "K700": { + "hit@1": 0.321, + "hit@5": 0.523, + "hit@10": 0.603, + "ndcg_linear@1": 0.321, + "ndcg_linear@5": 0.4286211523676353, + "ndcg_linear@10": 0.4547881628219443, + "ndcg_exponential@1": 0.321, + "ndcg_exponential@5": 0.4286211523676353, + "ndcg_exponential@10": 0.4547881628219443, + "precision@1": 0.321, + "precision@5": 0.1046, + "precision@10": 0.0603, + "recall@1": 0.321, + "recall@5": 0.523, + "recall@10": 0.603, + "f1@1": 0.321, + "f1@5": 0.17433333333333337, + "f1@10": 0.10963636363636366, + "map@1": 0.321, + "map@5": 0.3971666666666666, + "map@10": 0.4081412698412698, + "mrr@1": 0.321, + "mrr@5": 0.3971666666666666, + "mrr@10": 0.4081412698412698, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_MP-DocVQA": { - "hit@1": 0.40439932318104904, - "hit@5": 0.6700507614213198, - "hit@10": 0.7377326565143824, - "ndcg_linear@1": 0.40439932318104904, - "ndcg_linear@5": 0.5452225541556516, - "ndcg_linear@10": 0.5673639059314857, - "ndcg_exponential@1": 0.40439932318104904, - "ndcg_exponential@5": 0.5452225541556516, - "ndcg_exponential@10": 0.5673639059314857, - "precision@1": 0.40439932318104904, - "precision@5": 0.13401015228426394, - "precision@10": 0.07377326565143823, - "recall@1": 0.40439932318104904, - "recall@5": 0.6700507614213198, - "recall@10": 0.7377326565143824, - "f1@1": 0.40439932318104904, - "f1@5": 0.22335025380710666, - "f1@10": 0.13413321027534228, - "map@1": 0.40439932318104904, - "map@5": 0.5038071065989848, - "map@10": 0.5130885236215186, - "mrr@1": 0.40439932318104904, - "mrr@5": 0.5038071065989848, - "mrr@10": 0.5130885236215186, - "num_pred": 591, - "num_data": 591 + "SmthSmthV2": { + "hit@1": 0.253, + "hit@5": 0.485, + "hit@10": 0.607, + "ndcg_linear@1": 0.253, + "ndcg_linear@5": 0.37563000406966024, + "ndcg_linear@10": 0.4150475933223388, + "ndcg_exponential@1": 0.253, + "ndcg_exponential@5": 0.37563000406966024, + "ndcg_exponential@10": 0.4150475933223388, + "precision@1": 0.253, + "precision@5": 0.097, + "precision@10": 0.06069999999999999, + "recall@1": 0.253, + "recall@5": 0.485, + "recall@10": 0.607, + "f1@1": 0.253, + "f1@5": 0.1616666666666667, + "f1@10": 0.11036363636363637, + "map@1": 0.253, + "map@5": 0.33931666666666666, + "map@10": 0.3555595238095238, + "mrr@1": 0.253, + "mrr@5": 0.33931666666666666, + "mrr@10": 0.3555595238095238, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_SlideVQA": { - "hit@1": 0.6690647482014388, - "hit@5": 0.8830935251798561, - "hit@10": 0.9190647482014388, - "ndcg_linear@1": 0.6690647482014388, - "ndcg_linear@5": 0.765424898996163, - "ndcg_linear@10": 0.7831852550058264, - "ndcg_exponential@1": 0.6690647482014388, - "ndcg_exponential@5": 0.765424898996163, - "ndcg_exponential@10": 0.7831852550058264, - "precision@1": 0.6690647482014388, - "precision@5": 0.21546762589928053, - "precision@10": 0.11438848920863308, - "recall@1": 0.5701438848920863, - "recall@5": 0.8552158273381295, - "recall@10": 0.9055755395683454, - "f1@1": 0.6031175059952038, - "f1@5": 0.33795820486467976, - "f1@10": 0.2007575757575758, - "map@1": 0.6690647482014388, - "map@5": 0.7239658273381294, - "map@10": 0.7330735411670664, - "mrr@1": 0.6690647482014388, - "mrr@5": 0.7542865707434052, - "mrr@10": 0.7592818602261048, - "num_pred": 556, - "num_data": 556 + "HMDB51": { + "hit@1": 0.338, + "hit@5": 0.563, + "hit@10": 0.684, + "ndcg_linear@1": 0.338, + "ndcg_linear@5": 0.4553402697715536, + "ndcg_linear@10": 0.4940251051015839, + "ndcg_exponential@1": 0.338, + "ndcg_exponential@5": 0.4553402697715536, + "ndcg_exponential@10": 0.4940251051015839, + "precision@1": 0.338, + "precision@5": 0.11259999999999999, + "precision@10": 0.0684, + "recall@1": 0.338, + "recall@5": 0.563, + "recall@10": 0.684, + "f1@1": 0.338, + "f1@5": 0.18766666666666673, + "f1@10": 0.12436363636363637, + "map@1": 0.338, + "map@5": 0.4197166666666667, + "map@10": 0.4354206349206349, + "mrr@1": 0.338, + "mrr@5": 0.4197166666666667, + "mrr@10": 0.4354206349206349, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_InfoVQA": { - "hit@1": 0.5974930362116991, - "hit@5": 0.8454038997214485, - "hit@10": 0.8955431754874652, - "ndcg_linear@1": 0.5974930362116991, - "ndcg_linear@5": 0.7325704078178448, - "ndcg_linear@10": 0.749399435766671, - "ndcg_exponential@1": 0.5974930362116991, - "ndcg_exponential@5": 0.7325704078178448, - "ndcg_exponential@10": 0.749399435766671, - "precision@1": 0.5974930362116991, - "precision@5": 0.16908077994428966, - "precision@10": 0.08955431754874651, - "recall@1": 0.5974930362116991, - "recall@5": 0.8454038997214485, - "recall@10": 0.8955431754874652, - "f1@1": 0.5974930362116991, - "f1@5": 0.2818012999071495, - "f1@10": 0.16282603190681189, - "map@1": 0.5974930362116991, - "map@5": 0.6947075208913649, - "map@10": 0.7020200512888535, - "mrr@1": 0.5974930362116991, - "mrr@5": 0.6947075208913649, - "mrr@10": 0.7020200512888535, - "num_pred": 718, - "num_data": 718 + "UCF101": { + "hit@1": 0.53, + "hit@5": 0.76, + "hit@10": 0.816, + "ndcg_linear@1": 0.53, + "ndcg_linear@5": 0.6535743610111286, + "ndcg_linear@10": 0.6715652735972419, + "ndcg_exponential@1": 0.53, + "ndcg_exponential@5": 0.6535743610111286, + "ndcg_exponential@10": 0.6715652735972419, + "precision@1": 0.53, + "precision@5": 0.152, + "precision@10": 0.08159999999999999, + "recall@1": 0.53, + "recall@5": 0.76, + "recall@10": 0.816, + "f1@1": 0.53, + "f1@5": 0.25333333333333335, + "f1@10": 0.14836363636363636, + "map@1": 0.53, + "map@5": 0.6180666666666667, + "map@10": 0.6254214285714286, + "mrr@1": 0.53, + "mrr@5": 0.6180666666666667, + "mrr@10": 0.6254214285714286, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_PlotQA": { - "hit@1": 0.28389339513325607, - "hit@5": 0.526071842410197, - "hit@10": 0.6303592120509849, - "ndcg_linear@1": 0.28389339513325607, - "ndcg_linear@5": 0.4120322200394941, - "ndcg_linear@10": 0.4459578429761404, - "ndcg_exponential@1": 0.28389339513325607, - "ndcg_exponential@5": 0.4120322200394941, - "ndcg_exponential@10": 0.4459578429761404, - "precision@1": 0.28389339513325607, - "precision@5": 0.10521436848203938, - "precision@10": 0.06303592120509849, - "recall@1": 0.28389339513325607, - "recall@5": 0.526071842410197, - "recall@10": 0.6303592120509849, - "f1@1": 0.28389339513325607, - "f1@5": 0.17535728080339905, - "f1@10": 0.11461076582745183, - "map@1": 0.28389339513325607, - "map@5": 0.37414059482425643, - "map@10": 0.38825709503577405, - "mrr@1": 0.28389339513325607, - "mrr@5": 0.37414059482425643, - "mrr@10": 0.38825709503577405, - "num_pred": 863, - "num_data": 863 + "Breakfast": { + "hit@1": 0.20092378752886836, + "hit@5": 0.558891454965358, + "hit@10": 1.0, + "ndcg_linear@1": 0.20092378752886836, + "ndcg_linear@5": 0.3834539592109517, + "ndcg_linear@10": 0.5260901265661807, + "ndcg_exponential@1": 0.20092378752886836, + "ndcg_exponential@5": 0.3834539592109517, + "ndcg_exponential@10": 0.5260901265661807, + "precision@1": 0.20092378752886836, + "precision@5": 0.11177829099307159, + "precision@10": 0.1, + "recall@1": 0.20092378752886836, + "recall@5": 0.558891454965358, + "recall@10": 1.0, + "f1@1": 0.20092378752886836, + "f1@5": 0.18629715165511937, + "f1@10": 0.1818181818181818, + "map@1": 0.20092378752886836, + "map@5": 0.3257505773672056, + "map@10": 0.38463836650903627, + "mrr@1": 0.20092378752886836, + "mrr@5": 0.3257505773672056, + "mrr@10": 0.38463836650903627, + "num_pred": 433, + "num_data": 433 }, - "ViDoSeek-page": { - "hit@1": 0.0446584938704028, - "hit@5": 0.4063047285464098, - "hit@10": 0.6007005253940455, - "ndcg_linear@1": 0.0446584938704028, - "ndcg_linear@5": 0.23051188319719665, - "ndcg_linear@10": 0.29430171980440295, - "ndcg_exponential@1": 0.0446584938704028, - "ndcg_exponential@5": 0.2305118831971967, - "ndcg_exponential@10": 0.29430171980440295, - "precision@1": 0.0446584938704028, - "precision@5": 0.08126094570928197, - "precision@10": 0.06007005253940455, - "recall@1": 0.0446584938704028, - "recall@5": 0.4063047285464098, - "recall@10": 0.6007005253940455, - "f1@1": 0.0446584938704028, - "f1@5": 0.13543490951546996, - "f1@10": 0.10921827734437194, - "map@1": 0.0446584938704028, - "map@5": 0.17269410391126677, - "map@10": 0.19956008673171546, - "mrr@1": 0.0446584938704028, - "mrr@5": 0.17269410391126677, - "mrr@10": 0.19956008673171546, - "num_pred": 1142, - "num_data": 1142 + "MVBench": { + "hit@1": 0.37625, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.37625, + "ndcg_linear@5": 0.7166695467315642, + "ndcg_linear@10": 0.7166695467315642, + "ndcg_exponential@1": 0.37625, + "ndcg_exponential@5": 0.7166695467315642, + "ndcg_exponential@10": 0.7166695467315642, + "precision@1": 0.37625, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.37625, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.37625, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.37625, + "map@5": 0.6211166666666665, + "map@10": 0.6211166666666665, + "mrr@1": 0.37625, + "mrr@5": 0.6211166666666665, + "mrr@10": 0.6211166666666665, + "num_pred": 4000, + "num_data": 4000 }, - "ViDoSeek-doc": { - "hit@1": 0.9658493870402802, - "hit@5": 0.9903677758318739, - "hit@10": 0.9912434325744308, - "ndcg_linear@1": 0.6614127262113251, - "ndcg_linear@5": 0.8026274011129774, - "ndcg_linear@10": 0.793858821519423, - "ndcg_exponential@1": 0.4439579684763572, - "ndcg_exponential@5": 0.6878057015219627, - "ndcg_exponential@10": 0.7216552842804831, - "precision@1": 0.9658493870402802, - "precision@5": 0.8849387040280211, - "precision@10": 0.8094570928196146, - "recall@1": 0.056997106757459524, - "recall@5": 0.25965311344637504, - "recall@10": 0.4648221767265056, - "f1@1": 0.10690967591559217, - "f1@5": 0.39332503261123863, - "f1@10": 0.5772393543236635, - "map@1": 0.9658493870402802, - "map@5": 0.8711587857559837, - "map@10": 0.7917361333114032, - "mrr@1": 0.9658493870402802, - "mrr@5": 0.9748540572095739, - "mrr@10": 0.974979151029939, - "num_pred": 1142, - "num_data": 1142 + "Video-MME": { + "hit@1": 0.3511111111111111, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.3511111111111111, + "ndcg_linear@5": 0.6953962558796752, + "ndcg_linear@10": 0.6953962558796752, + "ndcg_exponential@1": 0.3511111111111111, + "ndcg_exponential@5": 0.6953962558796752, + "ndcg_exponential@10": 0.6953962558796752, + "precision@1": 0.3511111111111111, + "precision@5": 0.19999999999999996, + "precision@10": 0.09999999999999998, + "recall@1": 0.3511111111111111, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.3511111111111111, + "f1@5": 0.3333333333333334, + "f1@10": 0.18181818181818188, + "map@1": 0.3511111111111111, + "map@5": 0.5934876543209876, + "map@10": 0.5934876543209876, + "mrr@1": 0.3511111111111111, + "mrr@5": 0.5934876543209876, + "mrr@10": 0.5934876543209876, + "num_pred": 2700, + "num_data": 2700 }, - "MMLongBench-page": { - "hit@1": 0.07040572792362769, - "hit@5": 0.26968973747016706, - "hit@10": 0.3711217183770883, - "ndcg_linear@1": 0.07040572792362769, - "ndcg_linear@5": 0.1352234900192414, - "ndcg_linear@10": 0.17032905812486968, - "ndcg_exponential@1": 0.07040572792362769, - "ndcg_exponential@5": 0.1352234900192414, - "ndcg_exponential@10": 0.1703290581248697, - "precision@1": 0.07040572792362769, - "precision@5": 0.0613365155131265, - "precision@10": 0.046420047732696895, - "recall@1": 0.042114160700079555, - "recall@5": 0.19418257756563248, - "recall@10": 0.2923252860902026, - "f1@1": 0.05010710583502946, - "f1@5": 0.08893912800974434, - "f1@10": 0.07756931033754735, - "map@1": 0.07040572792362769, - "map@5": 0.10046904004242907, - "map@10": 0.11652881167051811, - "mrr@1": 0.07040572792362769, - "mrr@5": 0.14013524264120925, - "mrr@10": 0.15354159563586772, - "num_pred": 838, - "num_data": 838 + "NExTQA": { + "hit@1": 0.44932274638019615, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.44932274638019615, + "ndcg_linear@5": 0.732425356090162, + "ndcg_linear@10": 0.732425356090162, + "ndcg_exponential@1": 0.44932274638019615, + "ndcg_exponential@5": 0.732425356090162, + "ndcg_exponential@10": 0.732425356090162, + "precision@1": 0.44932274638019615, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.44932274638019615, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.44932274638019615, + "f1@5": 0.3333333333333333, + "f1@10": 0.1818181818181818, + "map@1": 0.44932274638019615, + "map@5": 0.6443542737038767, + "map@10": 0.6443542737038767, + "mrr@1": 0.44932274638019615, + "mrr@5": 0.6441596605947376, + "mrr@10": 0.6441596605947376, + "num_pred": 8564, + "num_data": 8564 + }, + "EgoSchema": { + "hit@1": 0.47, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.47, + "ndcg_linear@5": 0.7542281638188373, + "ndcg_linear@10": 0.7542281638188373, + "ndcg_exponential@1": 0.47, + "ndcg_exponential@5": 0.7542281638188373, + "ndcg_exponential@10": 0.7542281638188373, + "precision@1": 0.47, + "precision@5": 0.2, + "precision@10": 0.1, + "recall@1": 0.47, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.47, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.47, + "map@5": 0.6721333333333334, + "map@10": 0.6721333333333334, + "mrr@1": 0.47, + "mrr@5": 0.6721333333333334, + "mrr@10": 0.6721333333333334, + "num_pred": 500, + "num_data": 500 + }, + "ActivityNetQA": { + "hit@1": 0.485, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.485, + "ndcg_linear@5": 0.8099288230893007, + "ndcg_linear@10": 0.8099288230893007, + "ndcg_exponential@1": 0.485, + "ndcg_exponential@5": 0.8099288230893007, + "ndcg_exponential@10": 0.8099288230893007, + "precision@1": 0.485, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.485, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.485, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.485, + "map@5": 0.7425, + "map@10": 0.7425, + "mrr@1": 0.485, + "mrr@5": 0.7425, + "mrr@10": 0.7425, + "num_pred": 1000, + "num_data": 1000 }, - "MMLongBench-doc": { - "hit@1": 0.548926014319809, - "hit@5": 0.6825775656324582, - "hit@10": 0.7303102625298329, - "ndcg_linear@1": 0.386634844868735, - "ndcg_linear@5": 0.43544677369246687, - "ndcg_linear@10": 0.4315957106632966, - "ndcg_exponential@1": 0.2707125809751108, - "ndcg_exponential@5": 0.36475310712288217, - "ndcg_exponential@10": 0.3840612564382075, - "precision@1": 0.548926014319809, - "precision@5": 0.4911694510739857, - "precision@10": 0.45083532219570405, - "recall@1": 0.016616151041636364, - "recall@5": 0.0724119472425789, - "recall@10": 0.1306605325823178, - "f1@1": 0.0319233655487925, - "f1@5": 0.12112180979310318, - "f1@10": 0.19000676907704542, - "map@1": 0.548926014319809, - "map@5": 0.45691328560063643, - "map@10": 0.40530936280637947, - "mrr@1": 0.548926014319809, - "mrr@5": 0.5991845664280032, - "mrr@10": 0.6056853051483123, - "num_pred": 838, - "num_data": 838 - } - }, - "video": { "DiDeMo": { "hit@1": 0.22808764940239043, "hit@5": 0.44721115537848605, @@ -1804,481 +1323,875 @@ "num_pred": 1004, "num_data": 1004 }, - "MSR-VTT": { - "hit@1": 0.25, - "hit@5": 0.474, - "hit@10": 0.576, - "ndcg_linear@1": 0.25, - "ndcg_linear@5": 0.3652809849605177, - "ndcg_linear@10": 0.3983339947967488, - "ndcg_exponential@1": 0.25, - "ndcg_exponential@5": 0.3652809849605177, - "ndcg_exponential@10": 0.3983339947967488, - "precision@1": 0.25, - "precision@5": 0.09479999999999998, - "precision@10": 0.05759999999999999, - "recall@1": 0.25, - "recall@5": 0.474, - "recall@10": 0.576, - "f1@1": 0.25, - "f1@5": 0.15800000000000006, - "f1@10": 0.10472727272727275, - "map@1": 0.25, - "map@5": 0.3294666666666667, - "map@10": 0.3431484126984127, - "mrr@1": 0.25, - "mrr@5": 0.3294666666666667, - "mrr@10": 0.3431484126984127, - "num_pred": 1000, - "num_data": 1000 + "MSR-VTT": { + "hit@1": 0.25, + "hit@5": 0.474, + "hit@10": 0.576, + "ndcg_linear@1": 0.25, + "ndcg_linear@5": 0.3652809849605177, + "ndcg_linear@10": 0.3983339947967488, + "ndcg_exponential@1": 0.25, + "ndcg_exponential@5": 0.3652809849605177, + "ndcg_exponential@10": 0.3983339947967488, + "precision@1": 0.25, + "precision@5": 0.09479999999999998, + "precision@10": 0.05759999999999999, + "recall@1": 0.25, + "recall@5": 0.474, + "recall@10": 0.576, + "f1@1": 0.25, + "f1@5": 0.15800000000000006, + "f1@10": 0.10472727272727275, + "map@1": 0.25, + "map@5": 0.3294666666666667, + "map@10": 0.3431484126984127, + "mrr@1": 0.25, + "mrr@5": 0.3294666666666667, + "mrr@10": 0.3431484126984127, + "num_pred": 1000, + "num_data": 1000 + }, + "MSVD": { + "hit@1": 0.41940298507462687, + "hit@5": 0.7, + "hit@10": 0.7805970149253731, + "ndcg_linear@1": 0.41940298507462687, + "ndcg_linear@5": 0.5692519862375854, + "ndcg_linear@10": 0.5955843294718209, + "ndcg_exponential@1": 0.41940298507462687, + "ndcg_exponential@5": 0.5692519862375854, + "ndcg_exponential@10": 0.5955843294718209, + "precision@1": 0.41940298507462687, + "precision@5": 0.13999999999999999, + "precision@10": 0.07805970149253731, + "recall@1": 0.41940298507462687, + "recall@5": 0.7, + "recall@10": 0.7805970149253731, + "f1@1": 0.41940298507462687, + "f1@5": 0.2333333333333334, + "f1@10": 0.14192672998643152, + "map@1": 0.41940298507462687, + "map@5": 0.5256965174129353, + "map@10": 0.5367264866145464, + "mrr@1": 0.41940298507462687, + "mrr@5": 0.5256965174129353, + "mrr@10": 0.5367264866145464, + "num_pred": 670, + "num_data": 670 + }, + "VATEX": { + "hit@1": 0.1864671728450201, + "hit@5": 0.39794551138901296, + "hit@10": 0.49933005806163466, + "ndcg_linear@1": 0.1864671728450201, + "ndcg_linear@5": 0.2965339753650619, + "ndcg_linear@10": 0.32929416685693125, + "ndcg_exponential@1": 0.1864671728450201, + "ndcg_exponential@5": 0.2965339753650619, + "ndcg_exponential@10": 0.32929416685693125, + "precision@1": 0.1864671728450201, + "precision@5": 0.0795891022778026, + "precision@10": 0.04993300580616347, + "recall@1": 0.1864671728450201, + "recall@5": 0.39794551138901296, + "recall@10": 0.49933005806163466, + "f1@1": 0.1864671728450201, + "f1@5": 0.13264850379633766, + "f1@10": 0.09078728328393358, + "map@1": 0.1864671728450201, + "map@5": 0.2630005955039452, + "map@10": 0.2765067490447124, + "mrr@1": 0.1864671728450201, + "mrr@5": 0.2630005955039452, + "mrr@10": 0.2765067490447124, + "num_pred": 4478, + "num_data": 4478 + }, + "YouCook2": { + "hit@1": 0.07518087448883297, + "hit@5": 0.18748033972947467, + "hit@10": 0.2560553633217993, + "ndcg_linear@1": 0.07518087448883297, + "ndcg_linear@5": 0.13240906083348206, + "ndcg_linear@10": 0.15449506875832897, + "ndcg_exponential@1": 0.07518087448883297, + "ndcg_exponential@5": 0.13240906083348206, + "ndcg_exponential@10": 0.15449506875832897, + "precision@1": 0.07518087448883297, + "precision@5": 0.03749606794589494, + "precision@10": 0.02560553633217993, + "recall@1": 0.07518087448883297, + "recall@5": 0.18748033972947467, + "recall@10": 0.2560553633217993, + "f1@1": 0.07518087448883297, + "f1@5": 0.062493446576491564, + "f1@10": 0.04655552060396351, + "map@1": 0.07518087448883297, + "map@5": 0.11430743420362798, + "map@10": 0.12336900892264213, + "mrr@1": 0.07518087448883297, + "mrr@5": 0.11430743420362798, + "mrr@10": 0.12336900892264213, + "num_pred": 3179, + "num_data": 3179 + }, + "QVHighlight": { + "hit@1": 0.6084949215143121, + "hit@5": 0.8928901200369345, + "hit@10": 1.0, + "ndcg_linear@1": 0.6084949215143121, + "ndcg_linear@5": 0.7589804905293247, + "ndcg_linear@10": 0.7936722925896575, + "ndcg_exponential@1": 0.6084949215143121, + "ndcg_exponential@5": 0.7589804905293247, + "ndcg_exponential@10": 0.7936722925896575, + "precision@1": 0.6084949215143121, + "precision@5": 0.17857802400738687, + "precision@10": 0.09999999999999998, + "recall@1": 0.6084949215143121, + "recall@5": 0.8928901200369345, + "recall@10": 1.0, + "f1@1": 0.6084949215143121, + "f1@5": 0.2976300400123115, + "f1@10": 0.18181818181818185, + "map@1": 0.6084949215143121, + "map@5": 0.7144967682363804, + "map@10": 0.7288473376423515, + "mrr@1": 0.6084949215143121, + "mrr@5": 0.7144967682363804, + "mrr@10": 0.7288473376423515, + "num_pred": 1083, + "num_data": 1083 + }, + "Charades-STA": { + "hit@1": 0.18844566712517194, + "hit@5": 0.5722145804676754, + "hit@10": 1.0, + "ndcg_linear@1": 0.18844566712517194, + "ndcg_linear@5": 0.37699818670569085, + "ndcg_linear@10": 0.5119551257659812, + "ndcg_exponential@1": 0.18844566712517194, + "ndcg_exponential@5": 0.37699818670569085, + "ndcg_exponential@10": 0.5119551257659812, + "precision@1": 0.18844566712517194, + "precision@5": 0.11444291609353506, + "precision@10": 0.09999999999999998, + "recall@1": 0.18844566712517194, + "recall@5": 0.5722145804676754, + "recall@10": 1.0, + "f1@1": 0.18844566712517194, + "f1@5": 0.1907381934892252, + "f1@10": 0.18181818181818185, + "map@1": 0.18844566712517194, + "map@5": 0.31364053186611646, + "map@10": 0.3674133752538154, + "mrr@1": 0.18844566712517194, + "mrr@5": 0.31364053186611646, + "mrr@10": 0.3674133752538154, + "num_pred": 727, + "num_data": 727 + }, + "MomentSeeker": { + "hit@1": 0.31777777777777777, + "hit@5": 0.8033333333333333, + "hit@10": 1.0, + "ndcg_linear@1": 0.31777777777777777, + "ndcg_linear@5": 0.5664993737348916, + "ndcg_linear@10": 0.6381761228963511, + "ndcg_exponential@1": 0.31777777777777777, + "ndcg_exponential@5": 0.5664993737348916, + "ndcg_exponential@10": 0.6381761228963511, + "precision@1": 0.31777777777777777, + "precision@5": 0.1767777777777778, + "precision@10": 0.11305555555555558, + "recall@1": 0.29465277777777776, + "recall@5": 0.7851620370370369, + "recall@10": 0.99875, + "f1@1": 0.30156261022927683, + "f1@5": 0.2835724769058103, + "f1@10": 0.2004256237589571, + "map@1": 0.31777777777777777, + "map@5": 0.4882804012345679, + "map@10": 0.5207349077748383, + "mrr@1": 0.31777777777777777, + "mrr@5": 0.49910185185185185, + "mrr@10": 0.5260895061728394, + "num_pred": 1800, + "num_data": 1800 + } + }, + "visdoc": { + "ViDoRe_arxivqa": { + "hit@1": 0.45, + "hit@5": 0.602, + "hit@10": 0.67, + "ndcg_linear@1": 0.45, + "ndcg_linear@5": 0.5298461278515469, + "ndcg_linear@10": 0.5513653011991692, + "ndcg_exponential@1": 0.45, + "ndcg_exponential@5": 0.5298461278515469, + "ndcg_exponential@10": 0.5513653011991692, + "precision@1": 0.45, + "precision@5": 0.1204, + "precision@10": 0.067, + "recall@1": 0.45, + "recall@5": 0.602, + "recall@10": 0.67, + "f1@1": 0.45, + "f1@5": 0.20066666666666674, + "f1@10": 0.12181818181818184, + "map@1": 0.45, + "map@5": 0.5059666666666667, + "map@10": 0.5145904761904763, + "mrr@1": 0.45, + "mrr@5": 0.5059666666666667, + "mrr@10": 0.5145904761904763, + "num_pred": 500, + "num_data": 500 + }, + "ViDoRe_docvqa": { + "hit@1": 0.15964523281596452, + "hit@5": 0.34146341463414637, + "hit@10": 0.4279379157427938, + "ndcg_linear@1": 0.15964523281596452, + "ndcg_linear@5": 0.2541845458267961, + "ndcg_linear@10": 0.2816434324462572, + "ndcg_exponential@1": 0.15964523281596452, + "ndcg_exponential@5": 0.2541845458267961, + "ndcg_exponential@10": 0.2816434324462572, + "precision@1": 0.15964523281596452, + "precision@5": 0.06962305986696231, + "precision@10": 0.043458980044345896, + "recall@1": 0.15787139689578714, + "recall@5": 0.3356984478935698, + "recall@10": 0.42050997782705096, + "f1@1": 0.15816703621581674, + "f1@5": 0.11447048886073279, + "f1@10": 0.07839448061177551, + "map@1": 0.15964523281596452, + "map@5": 0.22555432372505546, + "map@10": 0.23678421849153558, + "mrr@1": 0.15964523281596452, + "mrr@5": 0.22893569844789358, + "mrr@10": 0.24044275507690144, + "num_pred": 451, + "num_data": 451 + }, + "ViDoRe_infovqa": { + "hit@1": 0.6133603238866396, + "hit@5": 0.8178137651821862, + "hit@10": 0.8805668016194332, + "ndcg_linear@1": 0.6133603238866396, + "ndcg_linear@5": 0.7231460909674565, + "ndcg_linear@10": 0.7438877002962954, + "ndcg_exponential@1": 0.6133603238866396, + "ndcg_exponential@5": 0.7231460909674565, + "ndcg_exponential@10": 0.7438877002962954, + "precision@1": 0.6133603238866396, + "precision@5": 0.16356275303643722, + "precision@10": 0.08805668016194332, + "recall@1": 0.6123481781376519, + "recall@5": 0.8168016194331984, + "recall@10": 0.8795546558704453, + "f1@1": 0.612685560053981, + "f1@5": 0.2725081935608252, + "f1@10": 0.1600723837565943, + "map@1": 0.6133603238866396, + "map@5": 0.6914642375168691, + "map@10": 0.7002907910802648, + "mrr@1": 0.6133603238866396, + "mrr@5": 0.692476383265857, + "mrr@10": 0.7013029368292527, + "num_pred": 494, + "num_data": 494 + }, + "ViDoRe_tabfquad": { + "hit@1": 0.5428571428571428, + "hit@5": 0.7607142857142857, + "hit@10": 0.8142857142857143, + "ndcg_linear@1": 0.5428571428571428, + "ndcg_linear@5": 0.6607864639631176, + "ndcg_linear@10": 0.6777435779728691, + "ndcg_exponential@1": 0.5428571428571428, + "ndcg_exponential@5": 0.6607864639631176, + "ndcg_exponential@10": 0.6777435779728691, + "precision@1": 0.5428571428571428, + "precision@5": 0.15214285714285714, + "precision@10": 0.08142857142857142, + "recall@1": 0.5428571428571428, + "recall@5": 0.7607142857142857, + "recall@10": 0.8142857142857143, + "f1@1": 0.5428571428571428, + "f1@5": 0.2535714285714286, + "f1@10": 0.14805194805194807, + "map@1": 0.5428571428571428, + "map@5": 0.6272619047619047, + "map@10": 0.634044784580499, + "mrr@1": 0.5428571428571428, + "mrr@5": 0.6272619047619047, + "mrr@10": 0.634044784580499, + "num_pred": 280, + "num_data": 280 }, - "MSVD": { - "hit@1": 0.41940298507462687, - "hit@5": 0.7, - "hit@10": 0.7805970149253731, - "ndcg_linear@1": 0.41940298507462687, - "ndcg_linear@5": 0.5692519862375854, - "ndcg_linear@10": 0.5955843294718209, - "ndcg_exponential@1": 0.41940298507462687, - "ndcg_exponential@5": 0.5692519862375854, - "ndcg_exponential@10": 0.5955843294718209, - "precision@1": 0.41940298507462687, - "precision@5": 0.13999999999999999, - "precision@10": 0.07805970149253731, - "recall@1": 0.41940298507462687, - "recall@5": 0.7, - "recall@10": 0.7805970149253731, - "f1@1": 0.41940298507462687, - "f1@5": 0.2333333333333334, - "f1@10": 0.14192672998643152, - "map@1": 0.41940298507462687, - "map@5": 0.5256965174129353, - "map@10": 0.5367264866145464, - "mrr@1": 0.41940298507462687, - "mrr@5": 0.5256965174129353, - "mrr@10": 0.5367264866145464, - "num_pred": 670, - "num_data": 670 + "ViDoRe_tatdqa": { + "hit@1": 0.15674362089914945, + "hit@5": 0.35236938031591736, + "hit@10": 0.4641555285540705, + "ndcg_linear@1": 0.15674362089914945, + "ndcg_linear@5": 0.2585284591791044, + "ndcg_linear@10": 0.29467381069418136, + "ndcg_exponential@1": 0.15674362089914945, + "ndcg_exponential@5": 0.2585284591791044, + "ndcg_exponential@10": 0.29467381069418136, + "precision@1": 0.15674362089914945, + "precision@5": 0.07059538274605104, + "precision@10": 0.04659781287970838, + "recall@1": 0.15674362089914945, + "recall@5": 0.3515188335358445, + "recall@10": 0.46366950182260025, + "f1@1": 0.15674362089914945, + "f1@5": 0.11745067407278832, + "f1@10": 0.08458706137928496, + "map@1": 0.15674362089914945, + "map@5": 0.2275567031186715, + "map@10": 0.24240993076047754, + "mrr@1": 0.15674362089914945, + "mrr@5": 0.22790603483191574, + "mrr@10": 0.24270863468919363, + "num_pred": 1646, + "num_data": 1646 }, - "VATEX": { - "hit@1": 0.1864671728450201, - "hit@5": 0.39794551138901296, - "hit@10": 0.49933005806163466, - "ndcg_linear@1": 0.1864671728450201, - "ndcg_linear@5": 0.2965339753650619, - "ndcg_linear@10": 0.32929416685693125, - "ndcg_exponential@1": 0.1864671728450201, - "ndcg_exponential@5": 0.2965339753650619, - "ndcg_exponential@10": 0.32929416685693125, - "precision@1": 0.1864671728450201, - "precision@5": 0.0795891022778026, - "precision@10": 0.04993300580616347, - "recall@1": 0.1864671728450201, - "recall@5": 0.39794551138901296, - "recall@10": 0.49933005806163466, - "f1@1": 0.1864671728450201, - "f1@5": 0.13264850379633766, - "f1@10": 0.09078728328393358, - "map@1": 0.1864671728450201, - "map@5": 0.2630005955039452, - "map@10": 0.2765067490447124, - "mrr@1": 0.1864671728450201, - "mrr@5": 0.2630005955039452, - "mrr@10": 0.2765067490447124, - "num_pred": 4478, - "num_data": 4478 + "ViDoRe_shiftproject": { + "hit@1": 0.13, + "hit@5": 0.4, + "hit@10": 0.6, + "ndcg_linear@1": 0.13, + "ndcg_linear@5": 0.2734127450408923, + "ndcg_linear@10": 0.33883350664214873, + "ndcg_exponential@1": 0.13, + "ndcg_exponential@5": 0.2734127450408923, + "ndcg_exponential@10": 0.33883350664214873, + "precision@1": 0.13, + "precision@5": 0.08, + "precision@10": 0.059999999999999984, + "recall@1": 0.13, + "recall@5": 0.4, + "recall@10": 0.6, + "f1@1": 0.13, + "f1@5": 0.1333333333333334, + "f1@10": 0.1090909090909091, + "map@1": 0.13, + "map@5": 0.23133333333333334, + "map@10": 0.25872222222222224, + "mrr@1": 0.13, + "mrr@5": 0.23133333333333334, + "mrr@10": 0.25872222222222224, + "num_pred": 100, + "num_data": 100 }, - "YouCook2": { - "hit@1": 0.07518087448883297, - "hit@5": 0.18748033972947467, - "hit@10": 0.2560553633217993, - "ndcg_linear@1": 0.07518087448883297, - "ndcg_linear@5": 0.13240906083348206, - "ndcg_linear@10": 0.15449506875832897, - "ndcg_exponential@1": 0.07518087448883297, - "ndcg_exponential@5": 0.13240906083348206, - "ndcg_exponential@10": 0.15449506875832897, - "precision@1": 0.07518087448883297, - "precision@5": 0.03749606794589494, - "precision@10": 0.02560553633217993, - "recall@1": 0.07518087448883297, - "recall@5": 0.18748033972947467, - "recall@10": 0.2560553633217993, - "f1@1": 0.07518087448883297, - "f1@5": 0.062493446576491564, - "f1@10": 0.04655552060396351, - "map@1": 0.07518087448883297, - "map@5": 0.11430743420362798, - "map@10": 0.12336900892264213, - "mrr@1": 0.07518087448883297, - "mrr@5": 0.11430743420362798, - "mrr@10": 0.12336900892264213, - "num_pred": 3179, - "num_data": 3179 + "ViDoRe_syntheticDocQA_artificial_intelligence": { + "hit@1": 0.59, + "hit@5": 0.85, + "hit@10": 0.88, + "ndcg_linear@1": 0.59, + "ndcg_linear@5": 0.7201422053206411, + "ndcg_linear@10": 0.7299635207551652, + "ndcg_exponential@1": 0.59, + "ndcg_exponential@5": 0.7201422053206411, + "ndcg_exponential@10": 0.7299635207551652, + "precision@1": 0.59, + "precision@5": 0.16999999999999996, + "precision@10": 0.08799999999999997, + "recall@1": 0.59, + "recall@5": 0.85, + "recall@10": 0.88, + "f1@1": 0.59, + "f1@5": 0.2833333333333334, + "f1@10": 0.16000000000000003, + "map@1": 0.59, + "map@5": 0.6778333333333333, + "map@10": 0.681940476190476, + "mrr@1": 0.59, + "mrr@5": 0.6778333333333333, + "mrr@10": 0.681940476190476, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_syntheticDocQA_energy": { + "hit@1": 0.52, + "hit@5": 0.76, + "hit@10": 0.85, + "ndcg_linear@1": 0.52, + "ndcg_linear@5": 0.6517245745315876, + "ndcg_linear@10": 0.6815359848536943, + "ndcg_exponential@1": 0.52, + "ndcg_exponential@5": 0.6517245745315876, + "ndcg_exponential@10": 0.6815359848536943, + "precision@1": 0.52, + "precision@5": 0.15199999999999997, + "precision@10": 0.08499999999999998, + "recall@1": 0.52, + "recall@5": 0.76, + "recall@10": 0.85, + "f1@1": 0.52, + "f1@5": 0.25333333333333335, + "f1@10": 0.15454545454545457, + "map@1": 0.52, + "map@5": 0.6154999999999999, + "map@10": 0.6281984126984126, + "mrr@1": 0.52, + "mrr@5": 0.6154999999999999, + "mrr@10": 0.6281984126984126, + "num_pred": 100, + "num_data": 100 }, - "QVHighlight": { - "hit@1": 0.6084949215143121, - "hit@5": 0.8928901200369345, - "hit@10": 1.0, - "ndcg_linear@1": 0.6084949215143121, - "ndcg_linear@5": 0.7589804905293247, - "ndcg_linear@10": 0.7936722925896575, - "ndcg_exponential@1": 0.6084949215143121, - "ndcg_exponential@5": 0.7589804905293247, - "ndcg_exponential@10": 0.7936722925896575, - "precision@1": 0.6084949215143121, - "precision@5": 0.17857802400738687, - "precision@10": 0.09999999999999998, - "recall@1": 0.6084949215143121, - "recall@5": 0.8928901200369345, - "recall@10": 1.0, - "f1@1": 0.6084949215143121, - "f1@5": 0.2976300400123115, - "f1@10": 0.18181818181818185, - "map@1": 0.6084949215143121, - "map@5": 0.7144967682363804, - "map@10": 0.7288473376423515, - "mrr@1": 0.6084949215143121, - "mrr@5": 0.7144967682363804, - "mrr@10": 0.7288473376423515, - "num_pred": 1083, - "num_data": 1083 + "ViDoRe_syntheticDocQA_government_reports": { + "hit@1": 0.58, + "hit@5": 0.84, + "hit@10": 0.88, + "ndcg_linear@1": 0.58, + "ndcg_linear@5": 0.7215467489235274, + "ndcg_linear@10": 0.7346071028524384, + "ndcg_exponential@1": 0.58, + "ndcg_exponential@5": 0.7215467489235274, + "ndcg_exponential@10": 0.7346071028524384, + "precision@1": 0.58, + "precision@5": 0.16799999999999998, + "precision@10": 0.088, + "recall@1": 0.58, + "recall@5": 0.84, + "recall@10": 0.88, + "f1@1": 0.58, + "f1@5": 0.2800000000000001, + "f1@10": 0.16000000000000003, + "map@1": 0.58, + "map@5": 0.6818333333333332, + "map@10": 0.6872896825396825, + "mrr@1": 0.58, + "mrr@5": 0.6818333333333332, + "mrr@10": 0.6872896825396825, + "num_pred": 100, + "num_data": 100 }, - "Charades-STA": { - "hit@1": 0.18844566712517194, - "hit@5": 0.5722145804676754, - "hit@10": 1.0, - "ndcg_linear@1": 0.18844566712517194, - "ndcg_linear@5": 0.37699818670569085, - "ndcg_linear@10": 0.5119551257659812, - "ndcg_exponential@1": 0.18844566712517194, - "ndcg_exponential@5": 0.37699818670569085, - "ndcg_exponential@10": 0.5119551257659812, - "precision@1": 0.18844566712517194, - "precision@5": 0.11444291609353506, - "precision@10": 0.09999999999999998, - "recall@1": 0.18844566712517194, - "recall@5": 0.5722145804676754, - "recall@10": 1.0, - "f1@1": 0.18844566712517194, - "f1@5": 0.1907381934892252, - "f1@10": 0.18181818181818185, - "map@1": 0.18844566712517194, - "map@5": 0.31364053186611646, - "map@10": 0.3674133752538154, - "mrr@1": 0.18844566712517194, - "mrr@5": 0.31364053186611646, - "mrr@10": 0.3674133752538154, - "num_pred": 727, - "num_data": 727 + "ViDoRe_syntheticDocQA_healthcare_industry": { + "hit@1": 0.7, + "hit@5": 0.94, + "hit@10": 0.98, + "ndcg_linear@1": 0.7, + "ndcg_linear@5": 0.8374771734856519, + "ndcg_linear@10": 0.8501891371833549, + "ndcg_exponential@1": 0.7, + "ndcg_exponential@5": 0.8374771734856519, + "ndcg_exponential@10": 0.8501891371833549, + "precision@1": 0.7, + "precision@5": 0.18799999999999997, + "precision@10": 0.09799999999999998, + "recall@1": 0.7, + "recall@5": 0.94, + "recall@10": 0.98, + "f1@1": 0.7, + "f1@5": 0.3133333333333334, + "f1@10": 0.17818181818181827, + "map@1": 0.7, + "map@5": 0.8026666666666666, + "map@10": 0.8077738095238094, + "mrr@1": 0.7, + "mrr@5": 0.8026666666666666, + "mrr@10": 0.8077738095238094, + "num_pred": 100, + "num_data": 100 }, - "MomentSeeker": { - "hit@1": 0.332, - "hit@5": 0.799, - "hit@10": 1.0, - "ndcg_linear@1": 0.332, - "ndcg_linear@5": 0.5727144775203178, - "ndcg_linear@10": 0.6397758757407763, - "ndcg_exponential@1": 0.332, - "ndcg_exponential@5": 0.5727144775203178, - "ndcg_exponential@10": 0.6397758757407763, - "precision@1": 0.332, - "precision@5": 0.168, - "precision@10": 0.10620000000000002, - "recall@1": 0.3220833333333334, - "recall@5": 0.7944583333333333, - "recall@10": 0.998625, - "f1@1": 0.32468253968253963, - "f1@5": 0.2742140082140082, - "f1@10": 0.1900151515151515, - "map@1": 0.332, - "map@5": 0.497625, - "map@10": 0.526500248015873, - "mrr@1": 0.332, - "mrr@5": 0.5003500000000001, - "mrr@10": 0.5277178571428571, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_esg_reports_human_labeled_v2": { + "hit@1": 0.28846153846153844, + "hit@5": 0.4807692307692308, + "hit@10": 0.6153846153846154, + "ndcg_linear@1": 0.28846153846153844, + "ndcg_linear@5": 0.3295140713668188, + "ndcg_linear@10": 0.3803485934947541, + "ndcg_exponential@1": 0.28846153846153844, + "ndcg_exponential@5": 0.3296697471059318, + "ndcg_exponential@10": 0.3804833955905218, + "precision@1": 0.28846153846153844, + "precision@5": 0.12692307692307694, + "precision@10": 0.09423076923076924, + "recall@1": 0.20576923076923076, + "recall@5": 0.3809149184149184, + "recall@10": 0.5121503496503498, + "f1@1": 0.2269230769230769, + "f1@5": 0.17453449328449328, + "f1@10": 0.14645786905402286, + "map@1": 0.28846153846153844, + "map@5": 0.2831730769230769, + "map@10": 0.30995421245421245, + "mrr@1": 0.28846153846153844, + "mrr@5": 0.36025641025641025, + "mrr@10": 0.378968253968254, + "num_pred": 52, + "num_data": 52 }, - "K700": { - "hit@1": 0.321, - "hit@5": 0.523, - "hit@10": 0.603, - "ndcg_linear@1": 0.321, - "ndcg_linear@5": 0.4286211523676353, - "ndcg_linear@10": 0.4547881628219443, - "ndcg_exponential@1": 0.321, - "ndcg_exponential@5": 0.4286211523676353, - "ndcg_exponential@10": 0.4547881628219443, - "precision@1": 0.321, - "precision@5": 0.1046, - "precision@10": 0.0603, - "recall@1": 0.321, - "recall@5": 0.523, - "recall@10": 0.603, - "f1@1": 0.321, - "f1@5": 0.17433333333333337, - "f1@10": 0.10963636363636366, - "map@1": 0.321, - "map@5": 0.3971666666666666, - "map@10": 0.4081412698412698, - "mrr@1": 0.321, - "mrr@5": 0.3971666666666666, - "mrr@10": 0.4081412698412698, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_biomedical_lectures_v2_multilingual": { + "hit@1": 0.2859375, + "hit@5": 0.5859375, + "hit@10": 0.684375, + "ndcg_linear@1": 0.2859375, + "ndcg_linear@5": 0.35874452751062924, + "ndcg_linear@10": 0.39816762044104737, + "ndcg_exponential@1": 0.2859375, + "ndcg_exponential@5": 0.35874452751062924, + "ndcg_exponential@10": 0.39816762044104737, + "precision@1": 0.2859375, + "precision@5": 0.1653125, + "precision@10": 0.11640625, + "recall@1": 0.1754330307750059, + "recall@5": 0.4161406827621331, + "recall@10": 0.5186508895780099, + "f1@1": 0.19900094696969695, + "f1@5": 0.2121714847142237, + "f1@10": 0.17188047002017592, + "map@1": 0.2859375, + "map@5": 0.2968263888888889, + "map@10": 0.3171867721915155, + "mrr@1": 0.2859375, + "mrr@5": 0.4043489583333333, + "mrr@10": 0.4175502232142857, + "num_pred": 640, + "num_data": 640 }, - "SmthSmthV2": { - "hit@1": 0.253, - "hit@5": 0.485, - "hit@10": 0.607, - "ndcg_linear@1": 0.253, - "ndcg_linear@5": 0.37563000406966024, - "ndcg_linear@10": 0.4150475933223388, - "ndcg_exponential@1": 0.253, - "ndcg_exponential@5": 0.37563000406966024, - "ndcg_exponential@10": 0.4150475933223388, - "precision@1": 0.253, - "precision@5": 0.097, - "precision@10": 0.06069999999999999, - "recall@1": 0.253, - "recall@5": 0.485, - "recall@10": 0.607, - "f1@1": 0.253, - "f1@5": 0.1616666666666667, - "f1@10": 0.11036363636363637, - "map@1": 0.253, - "map@5": 0.33931666666666666, - "map@10": 0.3555595238095238, - "mrr@1": 0.253, - "mrr@5": 0.33931666666666666, - "mrr@10": 0.3555595238095238, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_economics_reports_v2_multilingual": { + "hit@1": 0.3706896551724138, + "hit@5": 0.75, + "hit@10": 0.8189655172413793, + "ndcg_linear@1": 0.3706896551724138, + "ndcg_linear@5": 0.3194451317176268, + "ndcg_linear@10": 0.3202641880538733, + "ndcg_exponential@1": 0.3706896551724138, + "ndcg_exponential@5": 0.3194451317176268, + "ndcg_exponential@10": 0.3202641880538733, + "precision@1": 0.3706896551724138, + "precision@5": 0.2896551724137932, + "precision@10": 0.24870689655172415, + "recall@1": 0.03865338906185176, + "recall@5": 0.17160160948032535, + "recall@10": 0.25447962863698914, + "f1@1": 0.06726379557208981, + "f1@5": 0.1779005506820022, + "f1@10": 0.20701322326238306, + "map@1": 0.3706896551724138, + "map@5": 0.21762931034482758, + "map@10": 0.19331952955990825, + "mrr@1": 0.3706896551724138, + "mrr@5": 0.5000718390804598, + "mrr@10": 0.5099839217296114, + "num_pred": 232, + "num_data": 232 + }, + "ViDoRe_esg_reports_v2_multilingual": { + "hit@1": 0.23684210526315788, + "hit@5": 0.6578947368421053, + "hit@10": 0.7587719298245614, + "ndcg_linear@1": 0.23684210526315788, + "ndcg_linear@5": 0.32514161979506145, + "ndcg_linear@10": 0.3716042928744199, + "ndcg_exponential@1": 0.23684210526315788, + "ndcg_exponential@5": 0.32514161979506145, + "ndcg_exponential@10": 0.3716042928744199, + "precision@1": 0.23684210526315788, + "precision@5": 0.17982456140350878, + "precision@10": 0.13552631578947372, + "recall@1": 0.10328947368421053, + "recall@5": 0.3982769423558897, + "recall@10": 0.5284043441938179, + "f1@1": 0.1284477705530337, + "f1@5": 0.21337619006696623, + "f1@10": 0.18932479489653187, + "map@1": 0.23684210526315788, + "map@5": 0.2391544834307992, + "map@10": 0.25983787593984964, + "mrr@1": 0.23684210526315788, + "mrr@5": 0.38559941520467833, + "mrr@10": 0.39949526594263435, + "num_pred": 228, + "num_data": 228 }, - "HMDB51": { - "hit@1": 0.338, - "hit@5": 0.563, - "hit@10": 0.684, - "ndcg_linear@1": 0.338, - "ndcg_linear@5": 0.4553402697715536, - "ndcg_linear@10": 0.4940251051015839, - "ndcg_exponential@1": 0.338, - "ndcg_exponential@5": 0.4553402697715536, - "ndcg_exponential@10": 0.4940251051015839, - "precision@1": 0.338, - "precision@5": 0.11259999999999999, - "precision@10": 0.0684, - "recall@1": 0.338, - "recall@5": 0.563, - "recall@10": 0.684, - "f1@1": 0.338, - "f1@5": 0.18766666666666673, - "f1@10": 0.12436363636363637, - "map@1": 0.338, - "map@5": 0.4197166666666667, - "map@10": 0.4354206349206349, - "mrr@1": 0.338, - "mrr@5": 0.4197166666666667, - "mrr@10": 0.4354206349206349, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ArxivQA": { + "hit@1": 0.29901960784313725, + "hit@5": 0.44362745098039214, + "hit@10": 0.508578431372549, + "ndcg_linear@1": 0.29901960784313725, + "ndcg_linear@5": 0.37679708973810255, + "ndcg_linear@10": 0.3979379930260254, + "ndcg_exponential@1": 0.29901960784313725, + "ndcg_exponential@5": 0.37679708973810255, + "ndcg_exponential@10": 0.3979379930260254, + "precision@1": 0.29901960784313725, + "precision@5": 0.08872549019607844, + "precision@10": 0.0508578431372549, + "recall@1": 0.29901960784313725, + "recall@5": 0.44362745098039214, + "recall@10": 0.508578431372549, + "f1@1": 0.29901960784313725, + "f1@5": 0.14787581699346408, + "f1@10": 0.09246880570409984, + "map@1": 0.29901960784313725, + "map@5": 0.35451388888888885, + "map@10": 0.36331602085278564, + "mrr@1": 0.29901960784313725, + "mrr@5": 0.35451388888888885, + "mrr@10": 0.36331602085278564, + "num_pred": 816, + "num_data": 816 }, - "UCF101": { - "hit@1": 0.53, - "hit@5": 0.76, - "hit@10": 0.816, - "ndcg_linear@1": 0.53, - "ndcg_linear@5": 0.6535743610111286, - "ndcg_linear@10": 0.6715652735972419, - "ndcg_exponential@1": 0.53, - "ndcg_exponential@5": 0.6535743610111286, - "ndcg_exponential@10": 0.6715652735972419, - "precision@1": 0.53, - "precision@5": 0.152, - "precision@10": 0.08159999999999999, - "recall@1": 0.53, - "recall@5": 0.76, - "recall@10": 0.816, - "f1@1": 0.53, - "f1@5": 0.25333333333333335, - "f1@10": 0.14836363636363636, - "map@1": 0.53, - "map@5": 0.6180666666666667, - "map@10": 0.6254214285714286, - "mrr@1": 0.53, - "mrr@5": 0.6180666666666667, - "mrr@10": 0.6254214285714286, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ChartQA": { + "hit@1": 0.5873015873015873, + "hit@5": 0.7142857142857143, + "hit@10": 0.746031746031746, + "ndcg_linear@1": 0.5873015873015873, + "ndcg_linear@5": 0.6589841568889536, + "ndcg_linear@10": 0.6681608180419024, + "ndcg_exponential@1": 0.5873015873015873, + "ndcg_exponential@5": 0.6589841568889536, + "ndcg_exponential@10": 0.6681608180419024, + "precision@1": 0.5873015873015873, + "precision@5": 0.14285714285714282, + "precision@10": 0.07460317460317457, + "recall@1": 0.5873015873015873, + "recall@5": 0.7142857142857143, + "recall@10": 0.746031746031746, + "f1@1": 0.5873015873015873, + "f1@5": 0.23809523809523817, + "f1@10": 0.13564213564213565, + "map@1": 0.5873015873015873, + "map@5": 0.6402116402116402, + "map@10": 0.6433862433862434, + "mrr@1": 0.5873015873015873, + "mrr@5": 0.6402116402116402, + "mrr@10": 0.6433862433862434, + "num_pred": 63, + "num_data": 63 }, - "Breakfast": { - "hit@1": 0.20092378752886836, - "hit@5": 0.558891454965358, - "hit@10": 1.0, - "ndcg_linear@1": 0.20092378752886836, - "ndcg_linear@5": 0.3834539592109517, - "ndcg_linear@10": 0.5260901265661807, - "ndcg_exponential@1": 0.20092378752886836, - "ndcg_exponential@5": 0.3834539592109517, - "ndcg_exponential@10": 0.5260901265661807, - "precision@1": 0.20092378752886836, - "precision@5": 0.11177829099307159, - "precision@10": 0.1, - "recall@1": 0.20092378752886836, - "recall@5": 0.558891454965358, - "recall@10": 1.0, - "f1@1": 0.20092378752886836, - "f1@5": 0.18629715165511937, - "f1@10": 0.1818181818181818, - "map@1": 0.20092378752886836, - "map@5": 0.3257505773672056, - "map@10": 0.38463836650903627, - "mrr@1": 0.20092378752886836, - "mrr@5": 0.3257505773672056, - "mrr@10": 0.38463836650903627, - "num_pred": 433, - "num_data": 433 + "VisRAG_MP-DocVQA": { + "hit@1": 0.40439932318104904, + "hit@5": 0.6700507614213198, + "hit@10": 0.7377326565143824, + "ndcg_linear@1": 0.40439932318104904, + "ndcg_linear@5": 0.5452225541556516, + "ndcg_linear@10": 0.5673639059314857, + "ndcg_exponential@1": 0.40439932318104904, + "ndcg_exponential@5": 0.5452225541556516, + "ndcg_exponential@10": 0.5673639059314857, + "precision@1": 0.40439932318104904, + "precision@5": 0.13401015228426394, + "precision@10": 0.07377326565143823, + "recall@1": 0.40439932318104904, + "recall@5": 0.6700507614213198, + "recall@10": 0.7377326565143824, + "f1@1": 0.40439932318104904, + "f1@5": 0.22335025380710666, + "f1@10": 0.13413321027534228, + "map@1": 0.40439932318104904, + "map@5": 0.5038071065989848, + "map@10": 0.5130885236215186, + "mrr@1": 0.40439932318104904, + "mrr@5": 0.5038071065989848, + "mrr@10": 0.5130885236215186, + "num_pred": 591, + "num_data": 591 }, - "MVBench": { - "hit@1": 0.37625, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.37625, - "ndcg_linear@5": 0.7166695467315642, - "ndcg_linear@10": 0.7166695467315642, - "ndcg_exponential@1": 0.37625, - "ndcg_exponential@5": 0.7166695467315642, - "ndcg_exponential@10": 0.7166695467315642, - "precision@1": 0.37625, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.37625, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.37625, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.37625, - "map@5": 0.6211166666666665, - "map@10": 0.6211166666666665, - "mrr@1": 0.37625, - "mrr@5": 0.6211166666666665, - "mrr@10": 0.6211166666666665, - "num_pred": 4000, - "num_data": 4000 + "VisRAG_SlideVQA": { + "hit@1": 0.6690647482014388, + "hit@5": 0.8830935251798561, + "hit@10": 0.9190647482014388, + "ndcg_linear@1": 0.6690647482014388, + "ndcg_linear@5": 0.765424898996163, + "ndcg_linear@10": 0.7831852550058264, + "ndcg_exponential@1": 0.6690647482014388, + "ndcg_exponential@5": 0.765424898996163, + "ndcg_exponential@10": 0.7831852550058264, + "precision@1": 0.6690647482014388, + "precision@5": 0.21546762589928053, + "precision@10": 0.11438848920863308, + "recall@1": 0.5701438848920863, + "recall@5": 0.8552158273381295, + "recall@10": 0.9055755395683454, + "f1@1": 0.6031175059952038, + "f1@5": 0.33795820486467976, + "f1@10": 0.2007575757575758, + "map@1": 0.6690647482014388, + "map@5": 0.7239658273381294, + "map@10": 0.7330735411670664, + "mrr@1": 0.6690647482014388, + "mrr@5": 0.7542865707434052, + "mrr@10": 0.7592818602261048, + "num_pred": 556, + "num_data": 556 + }, + "VisRAG_InfoVQA": { + "hit@1": 0.5974930362116991, + "hit@5": 0.8454038997214485, + "hit@10": 0.8955431754874652, + "ndcg_linear@1": 0.5974930362116991, + "ndcg_linear@5": 0.7325704078178448, + "ndcg_linear@10": 0.749399435766671, + "ndcg_exponential@1": 0.5974930362116991, + "ndcg_exponential@5": 0.7325704078178448, + "ndcg_exponential@10": 0.749399435766671, + "precision@1": 0.5974930362116991, + "precision@5": 0.16908077994428966, + "precision@10": 0.08955431754874651, + "recall@1": 0.5974930362116991, + "recall@5": 0.8454038997214485, + "recall@10": 0.8955431754874652, + "f1@1": 0.5974930362116991, + "f1@5": 0.2818012999071495, + "f1@10": 0.16282603190681189, + "map@1": 0.5974930362116991, + "map@5": 0.6947075208913649, + "map@10": 0.7020200512888535, + "mrr@1": 0.5974930362116991, + "mrr@5": 0.6947075208913649, + "mrr@10": 0.7020200512888535, + "num_pred": 718, + "num_data": 718 }, - "Video-MME": { - "hit@1": 0.3511111111111111, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.3511111111111111, - "ndcg_linear@5": 0.6953962558796752, - "ndcg_linear@10": 0.6953962558796752, - "ndcg_exponential@1": 0.3511111111111111, - "ndcg_exponential@5": 0.6953962558796752, - "ndcg_exponential@10": 0.6953962558796752, - "precision@1": 0.3511111111111111, - "precision@5": 0.19999999999999996, - "precision@10": 0.09999999999999998, - "recall@1": 0.3511111111111111, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.3511111111111111, - "f1@5": 0.3333333333333334, - "f1@10": 0.18181818181818188, - "map@1": 0.3511111111111111, - "map@5": 0.5934876543209876, - "map@10": 0.5934876543209876, - "mrr@1": 0.3511111111111111, - "mrr@5": 0.5934876543209876, - "mrr@10": 0.5934876543209876, - "num_pred": 2700, - "num_data": 2700 + "VisRAG_PlotQA": { + "hit@1": 0.28389339513325607, + "hit@5": 0.526071842410197, + "hit@10": 0.6303592120509849, + "ndcg_linear@1": 0.28389339513325607, + "ndcg_linear@5": 0.4120322200394941, + "ndcg_linear@10": 0.4459578429761404, + "ndcg_exponential@1": 0.28389339513325607, + "ndcg_exponential@5": 0.4120322200394941, + "ndcg_exponential@10": 0.4459578429761404, + "precision@1": 0.28389339513325607, + "precision@5": 0.10521436848203938, + "precision@10": 0.06303592120509849, + "recall@1": 0.28389339513325607, + "recall@5": 0.526071842410197, + "recall@10": 0.6303592120509849, + "f1@1": 0.28389339513325607, + "f1@5": 0.17535728080339905, + "f1@10": 0.11461076582745183, + "map@1": 0.28389339513325607, + "map@5": 0.37414059482425643, + "map@10": 0.38825709503577405, + "mrr@1": 0.28389339513325607, + "mrr@5": 0.37414059482425643, + "mrr@10": 0.38825709503577405, + "num_pred": 863, + "num_data": 863 }, - "NExTQA": { - "hit@1": 0.44932274638019615, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.44932274638019615, - "ndcg_linear@5": 0.732425356090162, - "ndcg_linear@10": 0.732425356090162, - "ndcg_exponential@1": 0.44932274638019615, - "ndcg_exponential@5": 0.732425356090162, - "ndcg_exponential@10": 0.732425356090162, - "precision@1": 0.44932274638019615, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.44932274638019615, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.44932274638019615, - "f1@5": 0.3333333333333333, - "f1@10": 0.1818181818181818, - "map@1": 0.44932274638019615, - "map@5": 0.6443542737038767, - "map@10": 0.6443542737038767, - "mrr@1": 0.44932274638019615, - "mrr@5": 0.6441596605947376, - "mrr@10": 0.6441596605947376, - "num_pred": 8564, - "num_data": 8564 + "ViDoSeek-page": { + "hit@1": 0.0446584938704028, + "hit@5": 0.4063047285464098, + "hit@10": 0.6007005253940455, + "ndcg_linear@1": 0.0446584938704028, + "ndcg_linear@5": 0.23051188319719665, + "ndcg_linear@10": 0.29430171980440295, + "ndcg_exponential@1": 0.0446584938704028, + "ndcg_exponential@5": 0.2305118831971967, + "ndcg_exponential@10": 0.29430171980440295, + "precision@1": 0.0446584938704028, + "precision@5": 0.08126094570928197, + "precision@10": 0.06007005253940455, + "recall@1": 0.0446584938704028, + "recall@5": 0.4063047285464098, + "recall@10": 0.6007005253940455, + "f1@1": 0.0446584938704028, + "f1@5": 0.13543490951546996, + "f1@10": 0.10921827734437194, + "map@1": 0.0446584938704028, + "map@5": 0.17269410391126677, + "map@10": 0.19956008673171546, + "mrr@1": 0.0446584938704028, + "mrr@5": 0.17269410391126677, + "mrr@10": 0.19956008673171546, + "num_pred": 1142, + "num_data": 1142 }, - "EgoSchema": { - "hit@1": 0.47, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.47, - "ndcg_linear@5": 0.7542281638188373, - "ndcg_linear@10": 0.7542281638188373, - "ndcg_exponential@1": 0.47, - "ndcg_exponential@5": 0.7542281638188373, - "ndcg_exponential@10": 0.7542281638188373, - "precision@1": 0.47, - "precision@5": 0.2, - "precision@10": 0.1, - "recall@1": 0.47, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.47, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.47, - "map@5": 0.6721333333333334, - "map@10": 0.6721333333333334, - "mrr@1": 0.47, - "mrr@5": 0.6721333333333334, - "mrr@10": 0.6721333333333334, - "num_pred": 500, - "num_data": 500 + "ViDoSeek-doc": { + "hit@1": 0.9658493870402802, + "hit@5": 0.9903677758318739, + "hit@10": 0.9912434325744308, + "ndcg_linear@1": 0.6614127262113251, + "ndcg_linear@5": 0.8026274011129774, + "ndcg_linear@10": 0.793858821519423, + "ndcg_exponential@1": 0.4439579684763572, + "ndcg_exponential@5": 0.6878057015219627, + "ndcg_exponential@10": 0.7216552842804831, + "precision@1": 0.9658493870402802, + "precision@5": 0.8849387040280211, + "precision@10": 0.8094570928196146, + "recall@1": 0.056997106757459524, + "recall@5": 0.25965311344637504, + "recall@10": 0.4648221767265056, + "f1@1": 0.10690967591559217, + "f1@5": 0.39332503261123863, + "f1@10": 0.5772393543236635, + "map@1": 0.9658493870402802, + "map@5": 0.8711587857559837, + "map@10": 0.7917361333114032, + "mrr@1": 0.9658493870402802, + "mrr@5": 0.9748540572095739, + "mrr@10": 0.974979151029939, + "num_pred": 1142, + "num_data": 1142 }, - "ActivityNetQA": { - "hit@1": 0.485, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.485, - "ndcg_linear@5": 0.8099288230893007, - "ndcg_linear@10": 0.8099288230893007, - "ndcg_exponential@1": 0.485, - "ndcg_exponential@5": 0.8099288230893007, - "ndcg_exponential@10": 0.8099288230893007, - "precision@1": 0.485, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.485, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.485, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.485, - "map@5": 0.7425, - "map@10": 0.7425, - "mrr@1": 0.485, - "mrr@5": 0.7425, - "mrr@10": 0.7425, - "num_pred": 1000, - "num_data": 1000 + "MMLongBench-page": { + "hit@1": 0.07040572792362769, + "hit@5": 0.26968973747016706, + "hit@10": 0.3711217183770883, + "ndcg_linear@1": 0.07040572792362769, + "ndcg_linear@5": 0.1352234900192414, + "ndcg_linear@10": 0.17032905812486968, + "ndcg_exponential@1": 0.07040572792362769, + "ndcg_exponential@5": 0.1352234900192414, + "ndcg_exponential@10": 0.1703290581248697, + "precision@1": 0.07040572792362769, + "precision@5": 0.0613365155131265, + "precision@10": 0.046420047732696895, + "recall@1": 0.042114160700079555, + "recall@5": 0.19418257756563248, + "recall@10": 0.2923252860902026, + "f1@1": 0.05010710583502946, + "f1@5": 0.08893912800974434, + "f1@10": 0.07756931033754735, + "map@1": 0.07040572792362769, + "map@5": 0.10046904004242907, + "map@10": 0.11652881167051811, + "mrr@1": 0.07040572792362769, + "mrr@5": 0.14013524264120925, + "mrr@10": 0.15354159563586772, + "num_pred": 838, + "num_data": 838 + }, + "MMLongBench-doc": { + "hit@1": 0.548926014319809, + "hit@5": 0.6825775656324582, + "hit@10": 0.7303102625298329, + "ndcg_linear@1": 0.386634844868735, + "ndcg_linear@5": 0.43544677369246687, + "ndcg_linear@10": 0.4315957106632966, + "ndcg_exponential@1": 0.2707125809751108, + "ndcg_exponential@5": 0.36475310712288217, + "ndcg_exponential@10": 0.3840612564382075, + "precision@1": 0.548926014319809, + "precision@5": 0.4911694510739857, + "precision@10": 0.45083532219570405, + "recall@1": 0.016616151041636364, + "recall@5": 0.0724119472425789, + "recall@10": 0.1306605325823178, + "f1@1": 0.0319233655487925, + "f1@5": 0.12112180979310318, + "f1@10": 0.19000676907704542, + "map@1": 0.548926014319809, + "map@5": 0.45691328560063643, + "map@10": 0.40530936280637947, + "mrr@1": 0.548926014319809, + "mrr@5": 0.5991845664280032, + "mrr@10": 0.6056853051483123, + "num_pred": 838, + "num_data": 838 } } }