diff --git "a/scores/VLM2Vec-V1-Qwen2VL-2B.json" "b/scores/VLM2Vec-V1-Qwen2VL-2B.json" --- "a/scores/VLM2Vec-V1-Qwen2VL-2B.json" +++ "b/scores/VLM2Vec-V1-Qwen2VL-2B.json" @@ -1,10 +1,7 @@ { "metadata": { "model_name": "VLM2Vec-V1-Qwen2VL-2B", - "report_generated_date": "2025-06-09T07:08:50.537181", - "model_size": 2.21, - "url": "https://huggingface.co/TIGER-Lab/VLM2Vec-Qwen2VL-2B", - "data_source": "TIGER-Lab" + "report_generated_date": "2025-06-28T08:54:18.828030" }, "metrics": { "image": { @@ -1017,904 +1014,426 @@ "num_data": 1000 } }, - "visdoc": { - "ViDoRe_arxivqa": { - "hit@1": 0.384, + "video": { + "K700": { + "hit@1": 0.314, + "hit@5": 0.535, + "hit@10": 0.627, + "ndcg_linear@1": 0.314, + "ndcg_linear@5": 0.4306668083691291, + "ndcg_linear@10": 0.46049526743311364, + "ndcg_exponential@1": 0.314, + "ndcg_exponential@5": 0.4306668083691291, + "ndcg_exponential@10": 0.46049526743311364, + "precision@1": 0.314, + "precision@5": 0.107, + "precision@10": 0.06269999999999999, + "recall@1": 0.314, + "recall@5": 0.535, + "recall@10": 0.627, + "f1@1": 0.314, + "f1@5": 0.17833333333333337, + "f1@10": 0.11400000000000003, + "map@1": 0.314, + "map@5": 0.3961, + "map@10": 0.40846230158730157, + "mrr@1": 0.314, + "mrr@5": 0.3961, + "mrr@10": 0.40846230158730157, + "num_pred": 1000, + "num_data": 1000 + }, + "SmthSmthV2": { + "hit@1": 0.309, "hit@5": 0.582, - "hit@10": 0.666, - "ndcg_linear@1": 0.384, - "ndcg_linear@5": 0.4887714885037101, - "ndcg_linear@10": 0.5160063798442027, - "ndcg_exponential@1": 0.384, - "ndcg_exponential@5": 0.4887714885037101, - "ndcg_exponential@10": 0.5160063798442027, - "precision@1": 0.384, + "hit@10": 0.681, + "ndcg_linear@1": 0.309, + "ndcg_linear@5": 0.4540702574834417, + "ndcg_linear@10": 0.4863152780508059, + "ndcg_exponential@1": 0.309, + "ndcg_exponential@5": 0.4540702574834417, + "ndcg_exponential@10": 0.4863152780508059, + "precision@1": 0.309, "precision@5": 0.11639999999999999, - "precision@10": 0.06659999999999999, - "recall@1": 0.384, + "precision@10": 0.0681, + "recall@1": 0.309, "recall@5": 0.582, - "recall@10": 0.666, - "f1@1": 0.384, + "recall@10": 0.681, + "f1@1": 0.309, "f1@5": 0.19400000000000006, - "f1@10": 0.12109090909090911, - "map@1": 0.384, - "map@5": 0.4578, - "map@10": 0.4690865079365079, - "mrr@1": 0.384, - "mrr@5": 0.4578, - "mrr@10": 0.4690865079365079, - "num_pred": 500, - "num_data": 500 + "f1@10": 0.12381818181818184, + "map@1": 0.309, + "map@5": 0.4115, + "map@10": 0.4249297619047619, + "mrr@1": 0.309, + "mrr@5": 0.4115, + "mrr@10": 0.4249297619047619, + "num_pred": 1000, + "num_data": 1000 }, - "ViDoRe_docvqa": { - "hit@1": 0.2017738359201774, - "hit@5": 0.3436807095343681, - "hit@10": 0.44567627494456763, - "ndcg_linear@1": 0.2017738359201774, - "ndcg_linear@5": 0.2695284550966221, - "ndcg_linear@10": 0.30132152988724914, - "ndcg_exponential@1": 0.2017738359201774, - "ndcg_exponential@5": 0.2695284550966221, - "ndcg_exponential@10": 0.30132152988724914, - "precision@1": 0.2017738359201774, - "precision@5": 0.06873614190687362, - "precision@10": 0.04456762749445676, - "recall@1": 0.1950849963045085, - "recall@5": 0.3350517368810052, - "recall@10": 0.43446045824094603, - "f1@1": 0.19620217181192792, - "f1@5": 0.11298715631570727, - "f1@10": 0.08032947589488611, - "map@1": 0.2017738359201774, - "map@5": 0.2458610495195861, - "map@10": 0.258661088234259, - "mrr@1": 0.2017738359201774, - "mrr@5": 0.2531781226903178, - "mrr@10": 0.2666525886038081, - "num_pred": 451, - "num_data": 451 + "HMDB51": { + "hit@1": 0.338, + "hit@5": 0.584, + "hit@10": 0.735, + "ndcg_linear@1": 0.338, + "ndcg_linear@5": 0.46577853272929953, + "ndcg_linear@10": 0.5143920532103593, + "ndcg_exponential@1": 0.338, + "ndcg_exponential@5": 0.46577853272929953, + "ndcg_exponential@10": 0.5143920532103593, + "precision@1": 0.338, + "precision@5": 0.11679999999999999, + "precision@10": 0.0735, + "recall@1": 0.338, + "recall@5": 0.584, + "recall@10": 0.735, + "f1@1": 0.338, + "f1@5": 0.1946666666666667, + "f1@10": 0.13363636363636366, + "map@1": 0.338, + "map@5": 0.42671666666666663, + "map@10": 0.44666031746031754, + "mrr@1": 0.338, + "mrr@5": 0.42671666666666663, + "mrr@10": 0.44666031746031754, + "num_pred": 1000, + "num_data": 1000 }, - "ViDoRe_infovqa": { - "hit@1": 0.5465587044534413, - "hit@5": 0.7793522267206477, - "hit@10": 0.8481781376518218, - "ndcg_linear@1": 0.5465587044534413, - "ndcg_linear@5": 0.6723636246126143, - "ndcg_linear@10": 0.6943932274482111, - "ndcg_exponential@1": 0.5465587044534413, - "ndcg_exponential@5": 0.6723636246126143, - "ndcg_exponential@10": 0.6943932274482111, - "precision@1": 0.5465587044534413, - "precision@5": 0.15587044534412955, - "precision@10": 0.08502024291497975, - "recall@1": 0.5465587044534413, - "recall@5": 0.77834008097166, - "recall@10": 0.847165991902834, - "f1@1": 0.5465587044534413, - "f1@5": 0.25968768074031234, - "f1@10": 0.15449024659550975, - "map@1": 0.5465587044534413, - "map@5": 0.6366059379217273, - "map@10": 0.6455979692821798, - "mrr@1": 0.5465587044534413, - "mrr@5": 0.6371120107962213, - "mrr@10": 0.6458678748152432, - "num_pred": 494, - "num_data": 494 + "UCF101": { + "hit@1": 0.575, + "hit@5": 0.864, + "hit@10": 0.898, + "ndcg_linear@1": 0.575, + "ndcg_linear@5": 0.7354362127343672, + "ndcg_linear@10": 0.7465657689246903, + "ndcg_exponential@1": 0.575, + "ndcg_exponential@5": 0.7354362127343672, + "ndcg_exponential@10": 0.7465657689246903, + "precision@1": 0.575, + "precision@5": 0.1728, + "precision@10": 0.0898, + "recall@1": 0.575, + "recall@5": 0.864, + "recall@10": 0.898, + "f1@1": 0.575, + "f1@5": 0.288, + "f1@10": 0.1632727272727273, + "map@1": 0.575, + "map@5": 0.6921, + "map@10": 0.6967761904761904, + "mrr@1": 0.575, + "mrr@5": 0.6921, + "mrr@10": 0.6967761904761904, + "num_pred": 1000, + "num_data": 1000 }, - "ViDoRe_tabfquad": { - "hit@1": 0.5, - "hit@5": 0.7392857142857143, - "hit@10": 0.8035714285714286, - "ndcg_linear@1": 0.5, - "ndcg_linear@5": 0.6258424881807017, - "ndcg_linear@10": 0.6458458337631906, - "ndcg_exponential@1": 0.5, - "ndcg_exponential@5": 0.6258424881807017, - "ndcg_exponential@10": 0.6458458337631906, - "precision@1": 0.5, - "precision@5": 0.14785714285714283, - "precision@10": 0.08035714285714286, - "recall@1": 0.5, - "recall@5": 0.7392857142857143, - "recall@10": 0.8035714285714286, - "f1@1": 0.5, - "f1@5": 0.24642857142857147, - "f1@10": 0.14610389610389615, - "map@1": 0.5, - "map@5": 0.5882142857142857, - "map@10": 0.5959991496598639, - "mrr@1": 0.5, - "mrr@5": 0.5882142857142857, - "mrr@10": 0.5959991496598639, - "num_pred": 280, - "num_data": 280 + "Breakfast": { + "hit@1": 0.13394919168591224, + "hit@5": 0.6420323325635104, + "hit@10": 1.0, + "ndcg_linear@1": 0.13394919168591224, + "ndcg_linear@5": 0.38089188043322386, + "ndcg_linear@10": 0.49884077653788683, + "ndcg_exponential@1": 0.13394919168591224, + "ndcg_exponential@5": 0.38089188043322386, + "ndcg_exponential@10": 0.49884077653788683, + "precision@1": 0.13394919168591224, + "precision@5": 0.12840646651270207, + "precision@10": 0.1, + "recall@1": 0.13394919168591224, + "recall@5": 0.6420323325635104, + "recall@10": 1.0, + "f1@1": 0.13394919168591224, + "f1@5": 0.21401077752117018, + "f1@10": 0.1818181818181818, + "map@1": 0.13394919168591224, + "map@5": 0.2964588144726713, + "map@10": 0.34640932585505335, + "mrr@1": 0.13394919168591224, + "mrr@5": 0.2964588144726713, + "mrr@10": 0.34640932585505335, + "num_pred": 433, + "num_data": 433 }, - "ViDoRe_tatdqa": { - "hit@1": 0.11846901579586877, - "hit@5": 0.2733900364520049, - "hit@10": 0.370595382746051, - "ndcg_linear@1": 0.11846901579586877, - "ndcg_linear@5": 0.19790945816610908, - "ndcg_linear@10": 0.22934108397022743, - "ndcg_exponential@1": 0.11846901579586877, - "ndcg_exponential@5": 0.19790945816610908, - "ndcg_exponential@10": 0.22934108397022743, - "precision@1": 0.11846901579586877, - "precision@5": 0.054678007290400975, - "precision@10": 0.03712029161603888, - "recall@1": 0.11846901579586877, - "recall@5": 0.27278250303766705, - "recall@10": 0.36998784933171325, - "f1@1": 0.11846901579586877, - "f1@5": 0.09107215182549327, - "f1@10": 0.06745461909495931, - "map@1": 0.11846901579586877, - "map@5": 0.1731773997569866, - "map@10": 0.18614028235838687, - "mrr@1": 0.11846901579586877, - "mrr@5": 0.1734811664641555, - "mrr@10": 0.18640788636232136, - "num_pred": 1646, - "num_data": 1646 + "MVBench": { + "hit@1": 0.3045, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.3045, + "ndcg_linear@5": 0.6822126660495169, + "ndcg_linear@10": 0.6822126660495169, + "ndcg_exponential@1": 0.3045, + "ndcg_exponential@5": 0.6822126660495169, + "ndcg_exponential@10": 0.6822126660495169, + "precision@1": 0.3045, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.3045, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.3045, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.3045, + "map@5": 0.5752166666666667, + "map@10": 0.5752166666666667, + "mrr@1": 0.3045, + "mrr@5": 0.5752166666666667, + "mrr@10": 0.5752166666666667, + "num_pred": 4000, + "num_data": 4000 }, - "ViDoRe_shiftproject": { - "hit@1": 0.29, - "hit@5": 0.53, - "hit@10": 0.66, - "ndcg_linear@1": 0.29, - "ndcg_linear@5": 0.4179564395195299, - "ndcg_linear@10": 0.45947545144232743, - "ndcg_exponential@1": 0.29, - "ndcg_exponential@5": 0.4179564395195299, - "ndcg_exponential@10": 0.45947545144232743, - "precision@1": 0.29, - "precision@5": 0.10599999999999998, - "precision@10": 0.06599999999999998, - "recall@1": 0.29, - "recall@5": 0.53, - "recall@10": 0.66, - "f1@1": 0.29, - "f1@5": 0.17666666666666667, - "f1@10": 0.12000000000000002, - "map@1": 0.29, - "map@5": 0.3803333333333333, - "map@10": 0.3971746031746031, - "mrr@1": 0.29, - "mrr@5": 0.3803333333333333, - "mrr@10": 0.3971746031746031, - "num_pred": 100, - "num_data": 100 + "Video-MME": { + "hit@1": 0.2688888888888889, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.2688888888888889, + "ndcg_linear@5": 0.6530182359914178, + "ndcg_linear@10": 0.6530182359914178, + "ndcg_exponential@1": 0.2688888888888889, + "ndcg_exponential@5": 0.6530182359914178, + "ndcg_exponential@10": 0.6530182359914178, + "precision@1": 0.2688888888888889, + "precision@5": 0.19999999999999996, + "precision@10": 0.09999999999999998, + "recall@1": 0.2688888888888889, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.2688888888888889, + "f1@5": 0.3333333333333334, + "f1@10": 0.18181818181818188, + "map@1": 0.2688888888888889, + "map@5": 0.5373148148148148, + "map@10": 0.5373148148148148, + "mrr@1": 0.2688888888888889, + "mrr@5": 0.5373148148148148, + "mrr@10": 0.5373148148148148, + "num_pred": 2700, + "num_data": 2700 }, - "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.41, - "hit@5": 0.67, - "hit@10": 0.74, - "ndcg_linear@1": 0.41, - "ndcg_linear@5": 0.5501602800849955, - "ndcg_linear@10": 0.5722762310015821, - "ndcg_exponential@1": 0.41, - "ndcg_exponential@5": 0.5501602800849955, - "ndcg_exponential@10": 0.5722762310015821, - "precision@1": 0.41, - "precision@5": 0.13399999999999998, - "precision@10": 0.07399999999999998, - "recall@1": 0.41, - "recall@5": 0.67, - "recall@10": 0.74, - "f1@1": 0.41, - "f1@5": 0.22333333333333336, - "f1@10": 0.13454545454545458, - "map@1": 0.41, - "map@5": 0.5101666666666667, - "map@10": 0.518984126984127, - "mrr@1": 0.41, - "mrr@5": 0.5101666666666667, - "mrr@10": 0.518984126984127, - "num_pred": 100, - "num_data": 100 + "NExTQA": { + "hit@1": 0.20294255021018215, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.20294255021018215, + "ndcg_linear@5": 0.5984789868661349, + "ndcg_linear@10": 0.5984789868661349, + "ndcg_exponential@1": 0.20294255021018215, + "ndcg_exponential@5": 0.5984789868661349, + "ndcg_exponential@10": 0.5984789868661349, + "precision@1": 0.20294255021018215, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.20294255021018215, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.20294255021018215, + "f1@5": 0.3333333333333333, + "f1@10": 0.1818181818181818, + "map@1": 0.20294255021018215, + "map@5": 0.46760470185271685, + "map@10": 0.46760470185271685, + "mrr@1": 0.20294255021018215, + "mrr@5": 0.46744122684104006, + "mrr@10": 0.46744122684104006, + "num_pred": 8564, + "num_data": 8564 }, - "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.47, - "hit@5": 0.69, - "hit@10": 0.78, - "ndcg_linear@1": 0.47, - "ndcg_linear@5": 0.59111392640459, - "ndcg_linear@10": 0.6198260200539802, - "ndcg_exponential@1": 0.47, - "ndcg_exponential@5": 0.59111392640459, - "ndcg_exponential@10": 0.6198260200539802, - "precision@1": 0.47, - "precision@5": 0.13799999999999998, - "precision@10": 0.07799999999999999, - "recall@1": 0.47, - "recall@5": 0.69, - "recall@10": 0.78, - "f1@1": 0.47, - "f1@5": 0.23000000000000004, - "f1@10": 0.14181818181818184, - "map@1": 0.47, - "map@5": 0.5578333333333333, - "map@10": 0.569484126984127, - "mrr@1": 0.47, - "mrr@5": 0.5578333333333333, - "mrr@10": 0.569484126984127, - "num_pred": 100, - "num_data": 100 + "EgoSchema": { + "hit@1": 0.254, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.254, + "ndcg_linear@5": 0.6252600495861029, + "ndcg_linear@10": 0.6252600495861029, + "ndcg_exponential@1": 0.254, + "ndcg_exponential@5": 0.6252600495861029, + "ndcg_exponential@10": 0.6252600495861029, + "precision@1": 0.254, + "precision@5": 0.2, + "precision@10": 0.1, + "recall@1": 0.254, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.254, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.254, + "map@5": 0.5030666666666666, + "map@10": 0.5030666666666666, + "mrr@1": 0.254, + "mrr@5": 0.5030666666666666, + "mrr@10": 0.5030666666666666, + "num_pred": 500, + "num_data": 500 }, - "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.42, - "hit@5": 0.69, - "hit@10": 0.74, - "ndcg_linear@1": 0.42, - "ndcg_linear@5": 0.5706578821281822, - "ndcg_linear@10": 0.5871606562347121, - "ndcg_exponential@1": 0.42, - "ndcg_exponential@5": 0.5706578821281822, - "ndcg_exponential@10": 0.5871606562347121, - "precision@1": 0.42, - "precision@5": 0.13799999999999998, - "precision@10": 0.07399999999999998, - "recall@1": 0.42, - "recall@5": 0.69, - "recall@10": 0.74, - "f1@1": 0.42, - "f1@5": 0.23000000000000004, - "f1@10": 0.13454545454545458, - "map@1": 0.42, - "map@5": 0.5303333333333333, - "map@10": 0.537345238095238, - "mrr@1": 0.42, - "mrr@5": 0.5303333333333333, - "mrr@10": 0.537345238095238, - "num_pred": 100, - "num_data": 100 + "ActivityNetQA": { + "hit@1": 0.496, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.496, + "ndcg_linear@5": 0.8139885958000148, + "ndcg_linear@10": 0.8139885958000148, + "ndcg_exponential@1": 0.496, + "ndcg_exponential@5": 0.8139885958000148, + "ndcg_exponential@10": 0.8139885958000148, + "precision@1": 0.496, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.496, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.496, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.496, + "map@5": 0.748, + "map@10": 0.748, + "mrr@1": 0.496, + "mrr@5": 0.748, + "mrr@10": 0.748, + "num_pred": 1000, + "num_data": 1000 }, - "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.45, - "hit@5": 0.72, - "hit@10": 0.86, - "ndcg_linear@1": 0.45, - "ndcg_linear@5": 0.5963689233154675, - "ndcg_linear@10": 0.6420970000754052, - "ndcg_exponential@1": 0.45, - "ndcg_exponential@5": 0.5963689233154675, - "ndcg_exponential@10": 0.6420970000754052, - "precision@1": 0.45, - "precision@5": 0.144, - "precision@10": 0.08599999999999998, - "recall@1": 0.45, - "recall@5": 0.72, - "recall@10": 0.86, - "f1@1": 0.45, - "f1@5": 0.24000000000000005, - "f1@10": 0.1563636363636364, - "map@1": 0.45, - "map@5": 0.5548333333333333, - "map@10": 0.5739484126984127, - "mrr@1": 0.45, - "mrr@5": 0.5548333333333333, - "mrr@10": 0.5739484126984127, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_esg_reports_human_labeled_v2": { - "hit@1": 0.057692307692307696, - "hit@5": 0.28846153846153844, - "hit@10": 0.38461538461538464, - "ndcg_linear@1": 0.057692307692307696, - "ndcg_linear@5": 0.12599870327532675, - "ndcg_linear@10": 0.15572849019553353, - "ndcg_exponential@1": 0.057692307692307696, - "ndcg_exponential@5": 0.12510920026799083, - "ndcg_exponential@10": 0.1550084194156491, - "precision@1": 0.057692307692307696, - "precision@5": 0.08076923076923077, - "precision@10": 0.059615384615384626, - "recall@1": 0.015064102564102566, - "recall@5": 0.17774725274725273, - "recall@10": 0.2681318681318681, - "f1@1": 0.022189349112426034, - "f1@5": 0.09692774545715722, - "f1@10": 0.08744713054667805, - "map@1": 0.057692307692307696, - "map@5": 0.08697115384615385, - "map@10": 0.09812029405779404, - "mrr@1": 0.057692307692307696, - "mrr@5": 0.14070512820512818, - "mrr@10": 0.15183913308913308, - "num_pred": 52, - "num_data": 52 - }, - "ViDoRe_biomedical_lectures_v2": { - "hit@1": 0.10625, - "hit@5": 0.2, - "hit@10": 0.2375, - "ndcg_linear@1": 0.10625, - "ndcg_linear@5": 0.09477435582201986, - "ndcg_linear@10": 0.1004423742969149, - "ndcg_exponential@1": 0.10625, - "ndcg_exponential@5": 0.09477435582201987, - "ndcg_exponential@10": 0.1004423742969149, - "precision@1": 0.10625, - "precision@5": 0.053750000000000006, - "precision@10": 0.035625000000000004, - "recall@1": 0.03250077793656647, - "recall@5": 0.09603297490626343, - "recall@10": 0.1219837862652898, - "f1@1": 0.043400297619047616, - "f1@5": 0.059541577295253764, - "f1@10": 0.04778779553779554, - "map@1": 0.10625, - "map@5": 0.06931944444444445, - "map@10": 0.06875347222222221, - "mrr@1": 0.10625, - "mrr@5": 0.14020833333333332, - "mrr@10": 0.1451736111111111, - "num_pred": 160, - "num_data": 160 + "DiDeMo": { + "hit@1": 0.1942231075697211, + "hit@5": 0.39243027888446214, + "hit@10": 0.50199203187251, + "ndcg_linear@1": 0.1942231075697211, + "ndcg_linear@5": 0.2946313376749095, + "ndcg_linear@10": 0.3303998194168978, + "ndcg_exponential@1": 0.1942231075697211, + "ndcg_exponential@5": 0.2946313376749095, + "ndcg_exponential@10": 0.3303998194168978, + "precision@1": 0.1942231075697211, + "precision@5": 0.07848605577689244, + "precision@10": 0.050199203187251, + "recall@1": 0.1942231075697211, + "recall@5": 0.39243027888446214, + "recall@10": 0.50199203187251, + "f1@1": 0.1942231075697211, + "f1@5": 0.13081009296148743, + "f1@10": 0.09127127852227455, + "map@1": 0.1942231075697211, + "map@5": 0.262566401062417, + "map@10": 0.27753470246000134, + "mrr@1": 0.1942231075697211, + "mrr@5": 0.262566401062417, + "mrr@10": 0.27753470246000134, + "num_pred": 1004, + "num_data": 1004 }, - "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.08125, - "hit@5": 0.153125, - "hit@10": 0.190625, - "ndcg_linear@1": 0.08125, - "ndcg_linear@5": 0.0739843595210182, - "ndcg_linear@10": 0.07999860194187196, - "ndcg_exponential@1": 0.08125, - "ndcg_exponential@5": 0.0739843595210182, - "ndcg_exponential@10": 0.07999860194187196, - "precision@1": 0.08125, - "precision@5": 0.04031250000000001, - "precision@10": 0.02734375, - "recall@1": 0.03594834789323252, - "recall@5": 0.07288154427131352, - "recall@10": 0.0939780506638392, - "f1@1": 0.043974454365079366, - "f1@5": 0.04448018341952166, - "f1@10": 0.03715573113229363, - "map@1": 0.08125, - "map@5": 0.05640234374999999, - "map@10": 0.05761059441137566, - "mrr@1": 0.08125, - "mrr@5": 0.10536458333333334, - "mrr@10": 0.11032242063492062, - "num_pred": 640, - "num_data": 640 + "MSR-VTT": { + "hit@1": 0.252, + "hit@5": 0.489, + "hit@10": 0.598, + "ndcg_linear@1": 0.252, + "ndcg_linear@5": 0.3774884935111148, + "ndcg_linear@10": 0.41249059087798867, + "ndcg_exponential@1": 0.252, + "ndcg_exponential@5": 0.3774884935111148, + "ndcg_exponential@10": 0.41249059087798867, + "precision@1": 0.252, + "precision@5": 0.09779999999999998, + "precision@10": 0.0598, + "recall@1": 0.252, + "recall@5": 0.489, + "recall@10": 0.598, + "f1@1": 0.252, + "f1@5": 0.16300000000000006, + "f1@10": 0.10872727272727274, + "map@1": 0.252, + "map@5": 0.34048333333333336, + "map@10": 0.3547845238095238, + "mrr@1": 0.252, + "mrr@5": 0.34048333333333336, + "mrr@10": 0.3547845238095238, + "num_pred": 1000, + "num_data": 1000 }, - "ViDoRe_economics_reports_v2": { - "hit@1": 0.1724137931034483, - "hit@5": 0.46551724137931033, - "hit@10": 0.5689655172413793, - "ndcg_linear@1": 0.1724137931034483, - "ndcg_linear@5": 0.17529714083407696, - "ndcg_linear@10": 0.16178896095114104, - "ndcg_exponential@1": 0.1724137931034483, - "ndcg_exponential@5": 0.17529714083407696, - "ndcg_exponential@10": 0.16178896095114104, - "precision@1": 0.1724137931034483, - "precision@5": 0.16896551724137931, - "precision@10": 0.13448275862068967, - "recall@1": 0.012831922954508721, - "recall@5": 0.06614507212701243, - "recall@10": 0.10338462145152384, - "f1@1": 0.02331979724073151, - "f1@5": 0.08256161793817322, - "f1@10": 0.09881248178903711, - "map@1": 0.1724137931034483, - "map@5": 0.11235632183908048, - "map@10": 0.08446394362342638, - "mrr@1": 0.1724137931034483, - "mrr@5": 0.2827586206896551, - "mrr@10": 0.2980500821018062, - "num_pred": 58, - "num_data": 58 + "MSVD": { + "hit@1": 0.382089552238806, + "hit@5": 0.6507462686567164, + "hit@10": 0.7567164179104477, + "ndcg_linear@1": 0.382089552238806, + "ndcg_linear@5": 0.5286168082796038, + "ndcg_linear@10": 0.5629253942797813, + "ndcg_exponential@1": 0.382089552238806, + "ndcg_exponential@5": 0.5286168082796038, + "ndcg_exponential@10": 0.5629253942797813, + "precision@1": 0.382089552238806, + "precision@5": 0.13014925373134326, + "precision@10": 0.07567164179104477, + "recall@1": 0.382089552238806, + "recall@5": 0.6507462686567164, + "recall@10": 0.7567164179104477, + "f1@1": 0.382089552238806, + "f1@5": 0.2169154228855722, + "f1@10": 0.13758480325644507, + "map@1": 0.382089552238806, + "map@5": 0.48766169154228856, + "map@10": 0.501847311063729, + "mrr@1": 0.382089552238806, + "mrr@5": 0.48766169154228856, + "mrr@10": 0.501847311063729, + "num_pred": 670, + "num_data": 670 }, - "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.16379310344827586, - "hit@5": 0.39655172413793105, - "hit@10": 0.5344827586206896, - "ndcg_linear@1": 0.16379310344827586, - "ndcg_linear@5": 0.13895827754930556, - "ndcg_linear@10": 0.13248154811464, - "ndcg_exponential@1": 0.16379310344827586, - "ndcg_exponential@5": 0.13895827754930556, - "ndcg_exponential@10": 0.13248154811463997, - "precision@1": 0.16379310344827586, - "precision@5": 0.13189655172413792, - "precision@10": 0.11077586206896553, - "recall@1": 0.013347760137925428, - "recall@5": 0.04877413023459023, - "recall@10": 0.08606147608088197, - "f1@1": 0.02408230596994654, - "f1@5": 0.06465481180962832, - "f1@10": 0.08299218434749181, - "map@1": 0.16379310344827586, - "map@5": 0.08612068965517242, - "map@10": 0.0654065383337243, - "mrr@1": 0.16379310344827586, - "mrr@5": 0.2470545977011494, - "mrr@10": 0.26616208264915164, - "num_pred": 232, - "num_data": 232 + "VATEX": { + "hit@1": 0.16145600714604735, + "hit@5": 0.35618579723090665, + "hit@10": 0.4600267976775346, + "ndcg_linear@1": 0.16145600714604735, + "ndcg_linear@5": 0.26160013753460504, + "ndcg_linear@10": 0.29519606975527046, + "ndcg_exponential@1": 0.16145600714604735, + "ndcg_exponential@5": 0.26160013753460504, + "ndcg_exponential@10": 0.29519606975527046, + "precision@1": 0.16145600714604735, + "precision@5": 0.07123715944618134, + "precision@10": 0.04600267976775346, + "recall@1": 0.16145600714604735, + "recall@5": 0.35618579723090665, + "recall@10": 0.4600267976775346, + "f1@1": 0.16145600714604735, + "f1@5": 0.11872859907696891, + "f1@10": 0.08364123594136993, + "map@1": 0.16145600714604735, + "map@5": 0.2304488610987048, + "map@10": 0.24432614120532833, + "mrr@1": 0.16145600714604735, + "mrr@5": 0.2304488610987048, + "mrr@10": 0.24432614120532833, + "num_pred": 4478, + "num_data": 4478 }, - "ViDoRe_esg_reports_v2": { - "hit@1": 0.17543859649122806, - "hit@5": 0.43859649122807015, - "hit@10": 0.5964912280701754, - "ndcg_linear@1": 0.17543859649122806, - "ndcg_linear@5": 0.2160737143131349, - "ndcg_linear@10": 0.26341247385560845, - "ndcg_exponential@1": 0.17543859649122806, - "ndcg_exponential@5": 0.2160737143131349, - "ndcg_exponential@10": 0.2634124738556084, - "precision@1": 0.17543859649122806, - "precision@5": 0.1368421052631579, - "precision@10": 0.11403508771929823, - "recall@1": 0.059941520467836254, - "recall@5": 0.22581453634085213, - "recall@10": 0.3639097744360902, - "f1@1": 0.08051581946318788, - "f1@5": 0.1431213810684292, - "f1@10": 0.1478339012028111, - "map@1": 0.17543859649122806, - "map@5": 0.16315789473684209, - "map@10": 0.17925032488628978, - "mrr@1": 0.17543859649122806, - "mrr@5": 0.2713450292397661, - "mrr@10": 0.2940545808966862, - "num_pred": 57, - "num_data": 57 - }, - "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.14912280701754385, - "hit@5": 0.43859649122807015, - "hit@10": 0.5745614035087719, - "ndcg_linear@1": 0.14912280701754385, - "ndcg_linear@5": 0.20125906746655192, - "ndcg_linear@10": 0.24041678693593627, - "ndcg_exponential@1": 0.14912280701754385, - "ndcg_exponential@5": 0.20125906746655192, - "ndcg_exponential@10": 0.24041678693593627, - "precision@1": 0.14912280701754385, - "precision@5": 0.12280701754385964, - "precision@10": 0.10043859649122806, - "recall@1": 0.06019736842105263, - "recall@5": 0.2323360484544695, - "recall@10": 0.3408782372598162, - "f1@1": 0.07788771657192708, - "f1@5": 0.13747603737061406, - "f1@10": 0.13448911799490257, - "map@1": 0.14912280701754385, - "map@5": 0.14635721247563352, - "map@10": 0.1603259015594542, - "mrr@1": 0.14912280701754385, - "mrr@5": 0.2489766081871345, - "mrr@10": 0.26832358674463935, - "num_pred": 228, - "num_data": 228 - }, - "VisRAG_ArxivQA": { - "hit@1": 0.3272058823529412, - "hit@5": 0.4963235294117647, - "hit@10": 0.553921568627451, - "ndcg_linear@1": 0.3272058823529412, - "ndcg_linear@5": 0.41753959433079163, - "ndcg_linear@10": 0.43642238797839894, - "ndcg_exponential@1": 0.3272058823529412, - "ndcg_exponential@5": 0.41753959433079163, - "ndcg_exponential@10": 0.43642238797839894, - "precision@1": 0.3272058823529412, - "precision@5": 0.09926470588235294, - "precision@10": 0.0553921568627451, - "recall@1": 0.3272058823529412, - "recall@5": 0.4963235294117647, - "recall@10": 0.553921568627451, - "f1@1": 0.3272058823529412, - "f1@5": 0.16544117647058823, - "f1@10": 0.10071301247771838, - "map@1": 0.3272058823529412, - "map@5": 0.39129901960784313, - "map@10": 0.39924282212885154, - "mrr@1": 0.3272058823529412, - "mrr@5": 0.39129901960784313, - "mrr@10": 0.39924282212885154, - "num_pred": 816, - "num_data": 816 - }, - "VisRAG_ChartQA": { - "hit@1": 0.4444444444444444, - "hit@5": 0.6825396825396826, - "hit@10": 0.8095238095238095, - "ndcg_linear@1": 0.4444444444444444, - "ndcg_linear@5": 0.5792999547214452, - "ndcg_linear@10": 0.619790394179688, - "ndcg_exponential@1": 0.4444444444444444, - "ndcg_exponential@5": 0.5792999547214452, - "ndcg_exponential@10": 0.619790394179688, - "precision@1": 0.4444444444444444, - "precision@5": 0.13650793650793647, - "precision@10": 0.08095238095238091, - "recall@1": 0.4444444444444444, - "recall@5": 0.6825396825396826, - "recall@10": 0.8095238095238095, - "f1@1": 0.4444444444444444, - "f1@5": 0.2275132275132276, - "f1@10": 0.14718614718614717, - "map@1": 0.4444444444444444, - "map@5": 0.5441798941798942, - "map@10": 0.5605568153187201, - "mrr@1": 0.4444444444444444, - "mrr@5": 0.5441798941798942, - "mrr@10": 0.5605568153187201, - "num_pred": 63, - "num_data": 63 - }, - "VisRAG_MP-DocVQA": { - "hit@1": 0.3316412859560068, - "hit@5": 0.5194585448392555, - "hit@10": 0.6125211505922166, - "ndcg_linear@1": 0.3316412859560068, - "ndcg_linear@5": 0.4317780824109357, - "ndcg_linear@10": 0.4613201487006419, - "ndcg_exponential@1": 0.3316412859560068, - "ndcg_exponential@5": 0.4317780824109357, - "ndcg_exponential@10": 0.4613201487006419, - "precision@1": 0.3316412859560068, - "precision@5": 0.10389170896785109, - "precision@10": 0.06125211505922165, - "recall@1": 0.3316412859560068, - "recall@5": 0.5194585448392555, - "recall@10": 0.6125211505922166, - "f1@1": 0.3316412859560068, - "f1@5": 0.1731528482797519, - "f1@10": 0.11136748192585755, - "map@1": 0.3316412859560068, - "map@5": 0.40262267343485614, - "map@10": 0.4144972202078801, - "mrr@1": 0.3316412859560068, - "mrr@5": 0.40262267343485614, - "mrr@10": 0.4144972202078801, - "num_pred": 591, - "num_data": 591 - }, - "VisRAG_SlideVQA": { - "hit@1": 0.6528776978417267, - "hit@5": 0.8687050359712231, - "hit@10": 0.8974820143884892, - "ndcg_linear@1": 0.6528776978417267, - "ndcg_linear@5": 0.7403317436593279, - "ndcg_linear@10": 0.7559278694497471, - "ndcg_exponential@1": 0.6528776978417267, - "ndcg_exponential@5": 0.7403317436593279, - "ndcg_exponential@10": 0.7559278694497471, - "precision@1": 0.6528776978417267, - "precision@5": 0.20683453237410068, - "precision@10": 0.1093525179856115, - "recall@1": 0.5611510791366906, - "recall@5": 0.8282374100719424, - "recall@10": 0.8705035971223022, - "f1@1": 0.591726618705036, - "f1@5": 0.32511133949982873, - "f1@10": 0.19206453019402664, - "map@1": 0.6528776978417267, - "map@5": 0.6970773381294963, - "map@10": 0.7055591241292679, - "mrr@1": 0.6528776978417267, - "mrr@5": 0.7357314148681056, - "mrr@10": 0.7400757965056526, - "num_pred": 556, - "num_data": 556 - }, - "VisRAG_InfoVQA": { - "hit@1": 0.5779944289693594, - "hit@5": 0.8189415041782729, - "hit@10": 0.8802228412256268, - "ndcg_linear@1": 0.5779944289693594, - "ndcg_linear@5": 0.7066787929757021, - "ndcg_linear@10": 0.7266471704004087, - "ndcg_exponential@1": 0.5779944289693594, - "ndcg_exponential@5": 0.7066787929757021, - "ndcg_exponential@10": 0.7266471704004087, - "precision@1": 0.5779944289693594, - "precision@5": 0.16378830083565457, - "precision@10": 0.08802228412256266, - "recall@1": 0.5779944289693594, - "recall@5": 0.8189415041782729, - "recall@10": 0.8802228412256268, - "f1@1": 0.5779944289693594, - "f1@5": 0.27298050139275776, - "f1@10": 0.16004051658647764, - "map@1": 0.5779944289693594, - "map@5": 0.6692664809656453, - "map@10": 0.6775898660299774, - "mrr@1": 0.5779944289693594, - "mrr@5": 0.6692664809656453, - "mrr@10": 0.6775898660299774, - "num_pred": 718, - "num_data": 718 - }, - "VisRAG_PlotQA": { - "hit@1": 0.12977983777520277, - "hit@5": 0.3325608342989571, - "hit@10": 0.4287369640787949, - "ndcg_linear@1": 0.12977983777520277, - "ndcg_linear@5": 0.2339925322768118, - "ndcg_linear@10": 0.26492274577415076, - "ndcg_exponential@1": 0.12977983777520277, - "ndcg_exponential@5": 0.2339925322768118, - "ndcg_exponential@10": 0.26492274577415076, - "precision@1": 0.12977983777520277, - "precision@5": 0.06651216685979143, - "precision@10": 0.04287369640787949, - "recall@1": 0.12977983777520277, - "recall@5": 0.3325608342989571, - "recall@10": 0.4287369640787949, - "f1@1": 0.12977983777520277, - "f1@5": 0.11085361143298574, - "f1@10": 0.07795217528705363, - "map@1": 0.12977983777520277, - "map@5": 0.201525685592893, - "map@10": 0.2141831742353179, - "mrr@1": 0.12977983777520277, - "mrr@5": 0.201525685592893, - "mrr@10": 0.2141831742353179, - "num_pred": 863, - "num_data": 863 - }, - "ViDoSeek-page": { - "hit@1": 0.043782837127845885, - "hit@5": 0.3064798598949212, - "hit@10": 0.4956217162872154, - "ndcg_linear@1": 0.043782837127845885, - "ndcg_linear@5": 0.1774384395530587, - "ndcg_linear@10": 0.2388470474707514, - "ndcg_exponential@1": 0.043782837127845885, - "ndcg_exponential@5": 0.17743843955305874, - "ndcg_exponential@10": 0.23884704747075144, - "precision@1": 0.043782837127845885, - "precision@5": 0.06129597197898424, - "precision@10": 0.049562171628721535, - "recall@1": 0.043782837127845885, - "recall@5": 0.3064798598949212, - "recall@10": 0.4956217162872154, - "f1@1": 0.043782837127845885, - "f1@5": 0.10215995329830709, - "f1@10": 0.09011303932494825, - "map@1": 0.043782837127845885, - "map@5": 0.135128429655575, - "map@10": 0.16060865093264393, - "mrr@1": 0.043782837127845885, - "mrr@5": 0.135128429655575, - "mrr@10": 0.16060865093264393, - "num_pred": 1142, - "num_data": 1142 - }, - "ViDoSeek-doc": { - "hit@1": 0.9544658493870403, - "hit@5": 0.9842381786339754, - "hit@10": 0.9903677758318739, - "ndcg_linear@1": 0.6532399299474606, - "ndcg_linear@5": 0.7427733389906358, - "ndcg_linear@10": 0.6997209935069055, - "ndcg_exponential@1": 0.43807855891918934, - "ndcg_exponential@5": 0.6343478343437602, - "ndcg_exponential@10": 0.6353560202023368, - "precision@1": 0.9544658493870403, - "precision@5": 0.8038528896672504, - "precision@10": 0.6873029772329248, - "recall@1": 0.0563052881802568, - "recall@5": 0.23432760039766878, - "recall@10": 0.3914667961900597, - "f1@1": 0.10560395557458589, - "f1@5": 0.35560087282656433, - "f1@10": 0.48784025047176915, - "map@1": 0.9544658493870403, - "map@5": 0.7835522475189725, - "map@10": 0.6581428825421977, - "mrr@1": 0.9544658493870403, - "mrr@5": 0.96707530647986, - "mrr@10": 0.9679370639090428, - "num_pred": 1142, - "num_data": 1142 - }, - "MMLongBench-page": { - "hit@1": 0.059665871121718374, - "hit@5": 0.18973747016706444, - "hit@10": 0.2529832935560859, - "ndcg_linear@1": 0.059665871121718374, - "ndcg_linear@5": 0.09604945291283169, - "ndcg_linear@10": 0.117231531231931, - "ndcg_exponential@1": 0.059665871121718374, - "ndcg_exponential@5": 0.09604945291283169, - "ndcg_exponential@10": 0.117231531231931, - "precision@1": 0.059665871121718374, - "precision@5": 0.04319809069212411, - "precision@10": 0.031264916467780436, - "recall@1": 0.030469371519490848, - "recall@5": 0.13269689737470167, - "recall@10": 0.19082140015910903, - "f1@1": 0.03882827594044778, - "f1@5": 0.06218015375673666, - "f1@10": 0.051867980865594236, - "map@1": 0.059665871121718374, - "map@5": 0.07087808273667462, - "map@10": 0.08054184661135735, - "mrr@1": 0.059665871121718374, - "mrr@5": 0.10765712012728719, - "mrr@10": 0.11626179111262644, - "num_pred": 838, - "num_data": 838 - }, - "MMLongBench-doc": { - "hit@1": 0.4439140811455847, - "hit@5": 0.6145584725536993, - "hit@10": 0.6957040572792362, - "ndcg_linear@1": 0.3146380270485282, - "ndcg_linear@5": 0.3256804529174008, - "ndcg_linear@10": 0.3131933005253596, - "ndcg_exponential@1": 0.2222979884077736, - "ndcg_exponential@5": 0.2727115335151017, - "ndcg_exponential@10": 0.2793202033337121, - "precision@1": 0.4439140811455847, - "precision@5": 0.35966587112171833, - "precision@10": 0.31730310262529837, - "recall@1": 0.013683394891534362, - "recall@5": 0.052843721626569064, - "recall@10": 0.08999131731205673, - "f1@1": 0.026297035367307157, - "f1@5": 0.08878818509054331, - "f1@10": 0.13199155724774939, - "map@1": 0.4439140811455847, - "map@5": 0.31638424821002387, - "map@10": 0.26005365193014357, - "mrr@1": 0.4439140811455847, - "mrr@5": 0.5094470962609386, - "mrr@10": 0.520618820320491, - "num_pred": 838, - "num_data": 838 - } - }, - "video": { - "DiDeMo": { - "hit@1": 0.1942231075697211, - "hit@5": 0.39243027888446214, - "hit@10": 0.50199203187251, - "ndcg_linear@1": 0.1942231075697211, - "ndcg_linear@5": 0.2946313376749095, - "ndcg_linear@10": 0.3303998194168978, - "ndcg_exponential@1": 0.1942231075697211, - "ndcg_exponential@5": 0.2946313376749095, - "ndcg_exponential@10": 0.3303998194168978, - "precision@1": 0.1942231075697211, - "precision@5": 0.07848605577689244, - "precision@10": 0.050199203187251, - "recall@1": 0.1942231075697211, - "recall@5": 0.39243027888446214, - "recall@10": 0.50199203187251, - "f1@1": 0.1942231075697211, - "f1@5": 0.13081009296148743, - "f1@10": 0.09127127852227455, - "map@1": 0.1942231075697211, - "map@5": 0.262566401062417, - "map@10": 0.27753470246000134, - "mrr@1": 0.1942231075697211, - "mrr@5": 0.262566401062417, - "mrr@10": 0.27753470246000134, - "num_pred": 1004, - "num_data": 1004 - }, - "MSR-VTT": { - "hit@1": 0.252, - "hit@5": 0.489, - "hit@10": 0.598, - "ndcg_linear@1": 0.252, - "ndcg_linear@5": 0.3774884935111148, - "ndcg_linear@10": 0.41249059087798867, - "ndcg_exponential@1": 0.252, - "ndcg_exponential@5": 0.3774884935111148, - "ndcg_exponential@10": 0.41249059087798867, - "precision@1": 0.252, - "precision@5": 0.09779999999999998, - "precision@10": 0.0598, - "recall@1": 0.252, - "recall@5": 0.489, - "recall@10": 0.598, - "f1@1": 0.252, - "f1@5": 0.16300000000000006, - "f1@10": 0.10872727272727274, - "map@1": 0.252, - "map@5": 0.34048333333333336, - "map@10": 0.3547845238095238, - "mrr@1": 0.252, - "mrr@5": 0.34048333333333336, - "mrr@10": 0.3547845238095238, - "num_pred": 1000, - "num_data": 1000 - }, - "MSVD": { - "hit@1": 0.382089552238806, - "hit@5": 0.6507462686567164, - "hit@10": 0.7567164179104477, - "ndcg_linear@1": 0.382089552238806, - "ndcg_linear@5": 0.5286168082796038, - "ndcg_linear@10": 0.5629253942797813, - "ndcg_exponential@1": 0.382089552238806, - "ndcg_exponential@5": 0.5286168082796038, - "ndcg_exponential@10": 0.5629253942797813, - "precision@1": 0.382089552238806, - "precision@5": 0.13014925373134326, - "precision@10": 0.07567164179104477, - "recall@1": 0.382089552238806, - "recall@5": 0.6507462686567164, - "recall@10": 0.7567164179104477, - "f1@1": 0.382089552238806, - "f1@5": 0.2169154228855722, - "f1@10": 0.13758480325644507, - "map@1": 0.382089552238806, - "map@5": 0.48766169154228856, - "map@10": 0.501847311063729, - "mrr@1": 0.382089552238806, - "mrr@5": 0.48766169154228856, - "mrr@10": 0.501847311063729, - "num_pred": 670, - "num_data": 670 - }, - "VATEX": { - "hit@1": 0.16145600714604735, - "hit@5": 0.35618579723090665, - "hit@10": 0.4600267976775346, - "ndcg_linear@1": 0.16145600714604735, - "ndcg_linear@5": 0.26160013753460504, - "ndcg_linear@10": 0.29519606975527046, - "ndcg_exponential@1": 0.16145600714604735, - "ndcg_exponential@5": 0.26160013753460504, - "ndcg_exponential@10": 0.29519606975527046, - "precision@1": 0.16145600714604735, - "precision@5": 0.07123715944618134, - "precision@10": 0.04600267976775346, - "recall@1": 0.16145600714604735, - "recall@5": 0.35618579723090665, - "recall@10": 0.4600267976775346, - "f1@1": 0.16145600714604735, - "f1@5": 0.11872859907696891, - "f1@10": 0.08364123594136993, - "map@1": 0.16145600714604735, - "map@5": 0.2304488610987048, - "map@10": 0.24432614120532833, - "mrr@1": 0.16145600714604735, - "mrr@5": 0.2304488610987048, - "mrr@10": 0.24432614120532833, - "num_pred": 4478, - "num_data": 4478 - }, - "YouCook2": { - "hit@1": 0.04089336269267065, - "hit@5": 0.12205095942120163, - "hit@10": 0.17678515256369928, - "ndcg_linear@1": 0.04089336269267065, - "ndcg_linear@5": 0.08180116045817513, - "ndcg_linear@10": 0.09935374489912209, - "ndcg_exponential@1": 0.04089336269267065, - "ndcg_exponential@5": 0.08180116045817513, - "ndcg_exponential@10": 0.09935374489912209, - "precision@1": 0.04089336269267065, - "precision@5": 0.024410191884240325, - "precision@10": 0.017678515256369928, - "recall@1": 0.04089336269267065, - "recall@5": 0.12205095942120163, - "recall@10": 0.17678515256369928, - "f1@1": 0.04089336269267065, - "f1@5": 0.040683653140400546, - "f1@10": 0.03214275501158169, - "map@1": 0.04089336269267065, - "map@5": 0.06863793645800566, - "map@10": 0.07579464941056636, - "mrr@1": 0.04089336269267065, - "mrr@5": 0.06863793645800566, - "mrr@10": 0.07579464941056636, - "num_pred": 3179, - "num_data": 3179 + "YouCook2": { + "hit@1": 0.04089336269267065, + "hit@5": 0.12205095942120163, + "hit@10": 0.17678515256369928, + "ndcg_linear@1": 0.04089336269267065, + "ndcg_linear@5": 0.08180116045817513, + "ndcg_linear@10": 0.09935374489912209, + "ndcg_exponential@1": 0.04089336269267065, + "ndcg_exponential@5": 0.08180116045817513, + "ndcg_exponential@10": 0.09935374489912209, + "precision@1": 0.04089336269267065, + "precision@5": 0.024410191884240325, + "precision@10": 0.017678515256369928, + "recall@1": 0.04089336269267065, + "recall@5": 0.12205095942120163, + "recall@10": 0.17678515256369928, + "f1@1": 0.04089336269267065, + "f1@5": 0.040683653140400546, + "f1@10": 0.03214275501158169, + "map@1": 0.04089336269267065, + "map@5": 0.06863793645800566, + "map@10": 0.07579464941056636, + "mrr@1": 0.04089336269267065, + "mrr@5": 0.06863793645800566, + "mrr@10": 0.07579464941056636, + "num_pred": 3179, + "num_data": 3179 }, "QVHighlight": { "hit@1": 0.4422899353647276, @@ -1972,313 +1491,707 @@ "num_pred": 727, "num_data": 727 }, - "MomentSeeker": { + "MomentSeeker": { + "hit@1": 0.3438888888888889, + "hit@5": 0.8461111111111111, + "hit@10": 1.0, + "ndcg_linear@1": 0.3438888888888889, + "ndcg_linear@5": 0.6011605809701461, + "ndcg_linear@10": 0.658864568825075, + "ndcg_exponential@1": 0.3438888888888889, + "ndcg_exponential@5": 0.6011605809701461, + "ndcg_exponential@10": 0.658864568825075, + "precision@1": 0.3438888888888889, + "precision@5": 0.18344444444444447, + "precision@10": 0.11222222222222225, + "recall@1": 0.3239120370370371, + "recall@5": 0.8266666666666667, + "recall@10": 0.9966898148148149, + "f1@1": 0.33001499118165784, + "f1@5": 0.29593849052182386, + "f1@10": 0.19930560951394283, + "map@1": 0.3438888888888889, + "map@5": 0.5205669753086419, + "map@10": 0.5470192166372722, + "mrr@1": 0.3438888888888889, + "mrr@5": 0.532074074074074, + "mrr@10": 0.553324074074074, + "num_pred": 1800, + "num_data": 1800 + } + }, + "visdoc": { + "ViDoRe_arxivqa": { + "hit@1": 0.384, + "hit@5": 0.582, + "hit@10": 0.666, + "ndcg_linear@1": 0.384, + "ndcg_linear@5": 0.4887714885037101, + "ndcg_linear@10": 0.5160063798442027, + "ndcg_exponential@1": 0.384, + "ndcg_exponential@5": 0.4887714885037101, + "ndcg_exponential@10": 0.5160063798442027, + "precision@1": 0.384, + "precision@5": 0.11639999999999999, + "precision@10": 0.06659999999999999, + "recall@1": 0.384, + "recall@5": 0.582, + "recall@10": 0.666, + "f1@1": 0.384, + "f1@5": 0.19400000000000006, + "f1@10": 0.12109090909090911, + "map@1": 0.384, + "map@5": 0.4578, + "map@10": 0.4690865079365079, + "mrr@1": 0.384, + "mrr@5": 0.4578, + "mrr@10": 0.4690865079365079, + "num_pred": 500, + "num_data": 500 + }, + "ViDoRe_docvqa": { + "hit@1": 0.2017738359201774, + "hit@5": 0.3436807095343681, + "hit@10": 0.44567627494456763, + "ndcg_linear@1": 0.2017738359201774, + "ndcg_linear@5": 0.2695284550966221, + "ndcg_linear@10": 0.30132152988724914, + "ndcg_exponential@1": 0.2017738359201774, + "ndcg_exponential@5": 0.2695284550966221, + "ndcg_exponential@10": 0.30132152988724914, + "precision@1": 0.2017738359201774, + "precision@5": 0.06873614190687362, + "precision@10": 0.04456762749445676, + "recall@1": 0.1950849963045085, + "recall@5": 0.3350517368810052, + "recall@10": 0.43446045824094603, + "f1@1": 0.19620217181192792, + "f1@5": 0.11298715631570727, + "f1@10": 0.08032947589488611, + "map@1": 0.2017738359201774, + "map@5": 0.2458610495195861, + "map@10": 0.258661088234259, + "mrr@1": 0.2017738359201774, + "mrr@5": 0.2531781226903178, + "mrr@10": 0.2666525886038081, + "num_pred": 451, + "num_data": 451 + }, + "ViDoRe_infovqa": { + "hit@1": 0.5465587044534413, + "hit@5": 0.7793522267206477, + "hit@10": 0.8481781376518218, + "ndcg_linear@1": 0.5465587044534413, + "ndcg_linear@5": 0.6723636246126143, + "ndcg_linear@10": 0.6943932274482111, + "ndcg_exponential@1": 0.5465587044534413, + "ndcg_exponential@5": 0.6723636246126143, + "ndcg_exponential@10": 0.6943932274482111, + "precision@1": 0.5465587044534413, + "precision@5": 0.15587044534412955, + "precision@10": 0.08502024291497975, + "recall@1": 0.5465587044534413, + "recall@5": 0.77834008097166, + "recall@10": 0.847165991902834, + "f1@1": 0.5465587044534413, + "f1@5": 0.25968768074031234, + "f1@10": 0.15449024659550975, + "map@1": 0.5465587044534413, + "map@5": 0.6366059379217273, + "map@10": 0.6455979692821798, + "mrr@1": 0.5465587044534413, + "mrr@5": 0.6371120107962213, + "mrr@10": 0.6458678748152432, + "num_pred": 494, + "num_data": 494 + }, + "ViDoRe_tabfquad": { + "hit@1": 0.5, + "hit@5": 0.7392857142857143, + "hit@10": 0.8035714285714286, + "ndcg_linear@1": 0.5, + "ndcg_linear@5": 0.6258424881807017, + "ndcg_linear@10": 0.6458458337631906, + "ndcg_exponential@1": 0.5, + "ndcg_exponential@5": 0.6258424881807017, + "ndcg_exponential@10": 0.6458458337631906, + "precision@1": 0.5, + "precision@5": 0.14785714285714283, + "precision@10": 0.08035714285714286, + "recall@1": 0.5, + "recall@5": 0.7392857142857143, + "recall@10": 0.8035714285714286, + "f1@1": 0.5, + "f1@5": 0.24642857142857147, + "f1@10": 0.14610389610389615, + "map@1": 0.5, + "map@5": 0.5882142857142857, + "map@10": 0.5959991496598639, + "mrr@1": 0.5, + "mrr@5": 0.5882142857142857, + "mrr@10": 0.5959991496598639, + "num_pred": 280, + "num_data": 280 + }, + "ViDoRe_tatdqa": { + "hit@1": 0.11846901579586877, + "hit@5": 0.2733900364520049, + "hit@10": 0.370595382746051, + "ndcg_linear@1": 0.11846901579586877, + "ndcg_linear@5": 0.19790945816610908, + "ndcg_linear@10": 0.22934108397022743, + "ndcg_exponential@1": 0.11846901579586877, + "ndcg_exponential@5": 0.19790945816610908, + "ndcg_exponential@10": 0.22934108397022743, + "precision@1": 0.11846901579586877, + "precision@5": 0.054678007290400975, + "precision@10": 0.03712029161603888, + "recall@1": 0.11846901579586877, + "recall@5": 0.27278250303766705, + "recall@10": 0.36998784933171325, + "f1@1": 0.11846901579586877, + "f1@5": 0.09107215182549327, + "f1@10": 0.06745461909495931, + "map@1": 0.11846901579586877, + "map@5": 0.1731773997569866, + "map@10": 0.18614028235838687, + "mrr@1": 0.11846901579586877, + "mrr@5": 0.1734811664641555, + "mrr@10": 0.18640788636232136, + "num_pred": 1646, + "num_data": 1646 + }, + "ViDoRe_shiftproject": { + "hit@1": 0.29, + "hit@5": 0.53, + "hit@10": 0.66, + "ndcg_linear@1": 0.29, + "ndcg_linear@5": 0.4179564395195299, + "ndcg_linear@10": 0.45947545144232743, + "ndcg_exponential@1": 0.29, + "ndcg_exponential@5": 0.4179564395195299, + "ndcg_exponential@10": 0.45947545144232743, + "precision@1": 0.29, + "precision@5": 0.10599999999999998, + "precision@10": 0.06599999999999998, + "recall@1": 0.29, + "recall@5": 0.53, + "recall@10": 0.66, + "f1@1": 0.29, + "f1@5": 0.17666666666666667, + "f1@10": 0.12000000000000002, + "map@1": 0.29, + "map@5": 0.3803333333333333, + "map@10": 0.3971746031746031, + "mrr@1": 0.29, + "mrr@5": 0.3803333333333333, + "mrr@10": 0.3971746031746031, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_syntheticDocQA_artificial_intelligence": { "hit@1": 0.41, - "hit@5": 0.855, - "hit@10": 1.0, + "hit@5": 0.67, + "hit@10": 0.74, "ndcg_linear@1": 0.41, - "ndcg_linear@5": 0.6393513977923164, - "ndcg_linear@10": 0.6889346354800572, + "ndcg_linear@5": 0.5501602800849955, + "ndcg_linear@10": 0.5722762310015821, "ndcg_exponential@1": 0.41, - "ndcg_exponential@5": 0.6393513977923164, - "ndcg_exponential@10": 0.6889346354800572, + "ndcg_exponential@5": 0.5501602800849955, + "ndcg_exponential@10": 0.5722762310015821, "precision@1": 0.41, - "precision@5": 0.1778, - "precision@10": 0.10570000000000002, - "recall@1": 0.39741666666666664, - "recall@5": 0.8488333333333332, - "recall@10": 0.9985833333333333, - "f1@1": 0.40135238095238096, - "f1@5": 0.2915150405150405, - "f1@10": 0.1895151515151515, + "precision@5": 0.13399999999999998, + "precision@10": 0.07399999999999998, + "recall@1": 0.41, + "recall@5": 0.67, + "recall@10": 0.74, + "f1@1": 0.41, + "f1@5": 0.22333333333333336, + "f1@10": 0.13454545454545458, "map@1": 0.41, - "map@5": 0.5677783333333334, - "map@10": 0.5892080687830688, + "map@5": 0.5101666666666667, + "map@10": 0.518984126984127, "mrr@1": 0.41, - "mrr@5": 0.5724666666666667, - "mrr@10": 0.5923130952380953, - "num_pred": 1000, - "num_data": 1000 + "mrr@5": 0.5101666666666667, + "mrr@10": 0.518984126984127, + "num_pred": 100, + "num_data": 100 }, - "K700": { - "hit@1": 0.314, - "hit@5": 0.535, - "hit@10": 0.627, - "ndcg_linear@1": 0.314, - "ndcg_linear@5": 0.4306668083691291, - "ndcg_linear@10": 0.46049526743311364, - "ndcg_exponential@1": 0.314, - "ndcg_exponential@5": 0.4306668083691291, - "ndcg_exponential@10": 0.46049526743311364, - "precision@1": 0.314, - "precision@5": 0.107, - "precision@10": 0.06269999999999999, - "recall@1": 0.314, - "recall@5": 0.535, - "recall@10": 0.627, - "f1@1": 0.314, - "f1@5": 0.17833333333333337, - "f1@10": 0.11400000000000003, - "map@1": 0.314, - "map@5": 0.3961, - "map@10": 0.40846230158730157, - "mrr@1": 0.314, - "mrr@5": 0.3961, - "mrr@10": 0.40846230158730157, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_syntheticDocQA_energy": { + "hit@1": 0.47, + "hit@5": 0.69, + "hit@10": 0.78, + "ndcg_linear@1": 0.47, + "ndcg_linear@5": 0.59111392640459, + "ndcg_linear@10": 0.6198260200539802, + "ndcg_exponential@1": 0.47, + "ndcg_exponential@5": 0.59111392640459, + "ndcg_exponential@10": 0.6198260200539802, + "precision@1": 0.47, + "precision@5": 0.13799999999999998, + "precision@10": 0.07799999999999999, + "recall@1": 0.47, + "recall@5": 0.69, + "recall@10": 0.78, + "f1@1": 0.47, + "f1@5": 0.23000000000000004, + "f1@10": 0.14181818181818184, + "map@1": 0.47, + "map@5": 0.5578333333333333, + "map@10": 0.569484126984127, + "mrr@1": 0.47, + "mrr@5": 0.5578333333333333, + "mrr@10": 0.569484126984127, + "num_pred": 100, + "num_data": 100 }, - "SmthSmthV2": { - "hit@1": 0.309, - "hit@5": 0.582, - "hit@10": 0.681, - "ndcg_linear@1": 0.309, - "ndcg_linear@5": 0.4540702574834417, - "ndcg_linear@10": 0.4863152780508059, - "ndcg_exponential@1": 0.309, - "ndcg_exponential@5": 0.4540702574834417, - "ndcg_exponential@10": 0.4863152780508059, - "precision@1": 0.309, - "precision@5": 0.11639999999999999, - "precision@10": 0.0681, - "recall@1": 0.309, - "recall@5": 0.582, - "recall@10": 0.681, - "f1@1": 0.309, - "f1@5": 0.19400000000000006, - "f1@10": 0.12381818181818184, - "map@1": 0.309, - "map@5": 0.4115, - "map@10": 0.4249297619047619, - "mrr@1": 0.309, - "mrr@5": 0.4115, - "mrr@10": 0.4249297619047619, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_syntheticDocQA_government_reports": { + "hit@1": 0.42, + "hit@5": 0.69, + "hit@10": 0.74, + "ndcg_linear@1": 0.42, + "ndcg_linear@5": 0.5706578821281822, + "ndcg_linear@10": 0.5871606562347121, + "ndcg_exponential@1": 0.42, + "ndcg_exponential@5": 0.5706578821281822, + "ndcg_exponential@10": 0.5871606562347121, + "precision@1": 0.42, + "precision@5": 0.13799999999999998, + "precision@10": 0.07399999999999998, + "recall@1": 0.42, + "recall@5": 0.69, + "recall@10": 0.74, + "f1@1": 0.42, + "f1@5": 0.23000000000000004, + "f1@10": 0.13454545454545458, + "map@1": 0.42, + "map@5": 0.5303333333333333, + "map@10": 0.537345238095238, + "mrr@1": 0.42, + "mrr@5": 0.5303333333333333, + "mrr@10": 0.537345238095238, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_syntheticDocQA_healthcare_industry": { + "hit@1": 0.45, + "hit@5": 0.72, + "hit@10": 0.86, + "ndcg_linear@1": 0.45, + "ndcg_linear@5": 0.5963689233154675, + "ndcg_linear@10": 0.6420970000754052, + "ndcg_exponential@1": 0.45, + "ndcg_exponential@5": 0.5963689233154675, + "ndcg_exponential@10": 0.6420970000754052, + "precision@1": 0.45, + "precision@5": 0.144, + "precision@10": 0.08599999999999998, + "recall@1": 0.45, + "recall@5": 0.72, + "recall@10": 0.86, + "f1@1": 0.45, + "f1@5": 0.24000000000000005, + "f1@10": 0.1563636363636364, + "map@1": 0.45, + "map@5": 0.5548333333333333, + "map@10": 0.5739484126984127, + "mrr@1": 0.45, + "mrr@5": 0.5548333333333333, + "mrr@10": 0.5739484126984127, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_esg_reports_human_labeled_v2": { + "hit@1": 0.057692307692307696, + "hit@5": 0.28846153846153844, + "hit@10": 0.38461538461538464, + "ndcg_linear@1": 0.057692307692307696, + "ndcg_linear@5": 0.12599870327532675, + "ndcg_linear@10": 0.15572849019553353, + "ndcg_exponential@1": 0.057692307692307696, + "ndcg_exponential@5": 0.12510920026799083, + "ndcg_exponential@10": 0.1550084194156491, + "precision@1": 0.057692307692307696, + "precision@5": 0.08076923076923077, + "precision@10": 0.059615384615384626, + "recall@1": 0.015064102564102566, + "recall@5": 0.17774725274725273, + "recall@10": 0.2681318681318681, + "f1@1": 0.022189349112426034, + "f1@5": 0.09692774545715722, + "f1@10": 0.08744713054667805, + "map@1": 0.057692307692307696, + "map@5": 0.08697115384615385, + "map@10": 0.09812029405779404, + "mrr@1": 0.057692307692307696, + "mrr@5": 0.14070512820512818, + "mrr@10": 0.15183913308913308, + "num_pred": 52, + "num_data": 52 + }, + "ViDoRe_biomedical_lectures_v2_multilingual": { + "hit@1": 0.08125, + "hit@5": 0.153125, + "hit@10": 0.190625, + "ndcg_linear@1": 0.08125, + "ndcg_linear@5": 0.0739843595210182, + "ndcg_linear@10": 0.07999860194187196, + "ndcg_exponential@1": 0.08125, + "ndcg_exponential@5": 0.0739843595210182, + "ndcg_exponential@10": 0.07999860194187196, + "precision@1": 0.08125, + "precision@5": 0.04031250000000001, + "precision@10": 0.02734375, + "recall@1": 0.03594834789323252, + "recall@5": 0.07288154427131352, + "recall@10": 0.0939780506638392, + "f1@1": 0.043974454365079366, + "f1@5": 0.04448018341952166, + "f1@10": 0.03715573113229363, + "map@1": 0.08125, + "map@5": 0.05640234374999999, + "map@10": 0.05761059441137566, + "mrr@1": 0.08125, + "mrr@5": 0.10536458333333334, + "mrr@10": 0.11032242063492062, + "num_pred": 640, + "num_data": 640 + }, + "ViDoRe_economics_reports_v2_multilingual": { + "hit@1": 0.16379310344827586, + "hit@5": 0.39655172413793105, + "hit@10": 0.5344827586206896, + "ndcg_linear@1": 0.16379310344827586, + "ndcg_linear@5": 0.13895827754930556, + "ndcg_linear@10": 0.13248154811464, + "ndcg_exponential@1": 0.16379310344827586, + "ndcg_exponential@5": 0.13895827754930556, + "ndcg_exponential@10": 0.13248154811463997, + "precision@1": 0.16379310344827586, + "precision@5": 0.13189655172413792, + "precision@10": 0.11077586206896553, + "recall@1": 0.013347760137925428, + "recall@5": 0.04877413023459023, + "recall@10": 0.08606147608088197, + "f1@1": 0.02408230596994654, + "f1@5": 0.06465481180962832, + "f1@10": 0.08299218434749181, + "map@1": 0.16379310344827586, + "map@5": 0.08612068965517242, + "map@10": 0.0654065383337243, + "mrr@1": 0.16379310344827586, + "mrr@5": 0.2470545977011494, + "mrr@10": 0.26616208264915164, + "num_pred": 232, + "num_data": 232 + }, + "ViDoRe_esg_reports_v2_multilingual": { + "hit@1": 0.14912280701754385, + "hit@5": 0.43859649122807015, + "hit@10": 0.5745614035087719, + "ndcg_linear@1": 0.14912280701754385, + "ndcg_linear@5": 0.20125906746655192, + "ndcg_linear@10": 0.24041678693593627, + "ndcg_exponential@1": 0.14912280701754385, + "ndcg_exponential@5": 0.20125906746655192, + "ndcg_exponential@10": 0.24041678693593627, + "precision@1": 0.14912280701754385, + "precision@5": 0.12280701754385964, + "precision@10": 0.10043859649122806, + "recall@1": 0.06019736842105263, + "recall@5": 0.2323360484544695, + "recall@10": 0.3408782372598162, + "f1@1": 0.07788771657192708, + "f1@5": 0.13747603737061406, + "f1@10": 0.13448911799490257, + "map@1": 0.14912280701754385, + "map@5": 0.14635721247563352, + "map@10": 0.1603259015594542, + "mrr@1": 0.14912280701754385, + "mrr@5": 0.2489766081871345, + "mrr@10": 0.26832358674463935, + "num_pred": 228, + "num_data": 228 }, - "HMDB51": { - "hit@1": 0.338, - "hit@5": 0.584, - "hit@10": 0.735, - "ndcg_linear@1": 0.338, - "ndcg_linear@5": 0.46577853272929953, - "ndcg_linear@10": 0.5143920532103593, - "ndcg_exponential@1": 0.338, - "ndcg_exponential@5": 0.46577853272929953, - "ndcg_exponential@10": 0.5143920532103593, - "precision@1": 0.338, - "precision@5": 0.11679999999999999, - "precision@10": 0.0735, - "recall@1": 0.338, - "recall@5": 0.584, - "recall@10": 0.735, - "f1@1": 0.338, - "f1@5": 0.1946666666666667, - "f1@10": 0.13363636363636366, - "map@1": 0.338, - "map@5": 0.42671666666666663, - "map@10": 0.44666031746031754, - "mrr@1": 0.338, - "mrr@5": 0.42671666666666663, - "mrr@10": 0.44666031746031754, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ArxivQA": { + "hit@1": 0.3272058823529412, + "hit@5": 0.4963235294117647, + "hit@10": 0.553921568627451, + "ndcg_linear@1": 0.3272058823529412, + "ndcg_linear@5": 0.41753959433079163, + "ndcg_linear@10": 0.43642238797839894, + "ndcg_exponential@1": 0.3272058823529412, + "ndcg_exponential@5": 0.41753959433079163, + "ndcg_exponential@10": 0.43642238797839894, + "precision@1": 0.3272058823529412, + "precision@5": 0.09926470588235294, + "precision@10": 0.0553921568627451, + "recall@1": 0.3272058823529412, + "recall@5": 0.4963235294117647, + "recall@10": 0.553921568627451, + "f1@1": 0.3272058823529412, + "f1@5": 0.16544117647058823, + "f1@10": 0.10071301247771838, + "map@1": 0.3272058823529412, + "map@5": 0.39129901960784313, + "map@10": 0.39924282212885154, + "mrr@1": 0.3272058823529412, + "mrr@5": 0.39129901960784313, + "mrr@10": 0.39924282212885154, + "num_pred": 816, + "num_data": 816 }, - "UCF101": { - "hit@1": 0.575, - "hit@5": 0.864, - "hit@10": 0.898, - "ndcg_linear@1": 0.575, - "ndcg_linear@5": 0.7354362127343672, - "ndcg_linear@10": 0.7465657689246903, - "ndcg_exponential@1": 0.575, - "ndcg_exponential@5": 0.7354362127343672, - "ndcg_exponential@10": 0.7465657689246903, - "precision@1": 0.575, - "precision@5": 0.1728, - "precision@10": 0.0898, - "recall@1": 0.575, - "recall@5": 0.864, - "recall@10": 0.898, - "f1@1": 0.575, - "f1@5": 0.288, - "f1@10": 0.1632727272727273, - "map@1": 0.575, - "map@5": 0.6921, - "map@10": 0.6967761904761904, - "mrr@1": 0.575, - "mrr@5": 0.6921, - "mrr@10": 0.6967761904761904, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ChartQA": { + "hit@1": 0.4444444444444444, + "hit@5": 0.6825396825396826, + "hit@10": 0.8095238095238095, + "ndcg_linear@1": 0.4444444444444444, + "ndcg_linear@5": 0.5792999547214452, + "ndcg_linear@10": 0.619790394179688, + "ndcg_exponential@1": 0.4444444444444444, + "ndcg_exponential@5": 0.5792999547214452, + "ndcg_exponential@10": 0.619790394179688, + "precision@1": 0.4444444444444444, + "precision@5": 0.13650793650793647, + "precision@10": 0.08095238095238091, + "recall@1": 0.4444444444444444, + "recall@5": 0.6825396825396826, + "recall@10": 0.8095238095238095, + "f1@1": 0.4444444444444444, + "f1@5": 0.2275132275132276, + "f1@10": 0.14718614718614717, + "map@1": 0.4444444444444444, + "map@5": 0.5441798941798942, + "map@10": 0.5605568153187201, + "mrr@1": 0.4444444444444444, + "mrr@5": 0.5441798941798942, + "mrr@10": 0.5605568153187201, + "num_pred": 63, + "num_data": 63 }, - "Breakfast": { - "hit@1": 0.13394919168591224, - "hit@5": 0.6420323325635104, - "hit@10": 1.0, - "ndcg_linear@1": 0.13394919168591224, - "ndcg_linear@5": 0.38089188043322386, - "ndcg_linear@10": 0.49884077653788683, - "ndcg_exponential@1": 0.13394919168591224, - "ndcg_exponential@5": 0.38089188043322386, - "ndcg_exponential@10": 0.49884077653788683, - "precision@1": 0.13394919168591224, - "precision@5": 0.12840646651270207, - "precision@10": 0.1, - "recall@1": 0.13394919168591224, - "recall@5": 0.6420323325635104, - "recall@10": 1.0, - "f1@1": 0.13394919168591224, - "f1@5": 0.21401077752117018, - "f1@10": 0.1818181818181818, - "map@1": 0.13394919168591224, - "map@5": 0.2964588144726713, - "map@10": 0.34640932585505335, - "mrr@1": 0.13394919168591224, - "mrr@5": 0.2964588144726713, - "mrr@10": 0.34640932585505335, - "num_pred": 433, - "num_data": 433 + "VisRAG_MP-DocVQA": { + "hit@1": 0.3316412859560068, + "hit@5": 0.5194585448392555, + "hit@10": 0.6125211505922166, + "ndcg_linear@1": 0.3316412859560068, + "ndcg_linear@5": 0.4317780824109357, + "ndcg_linear@10": 0.4613201487006419, + "ndcg_exponential@1": 0.3316412859560068, + "ndcg_exponential@5": 0.4317780824109357, + "ndcg_exponential@10": 0.4613201487006419, + "precision@1": 0.3316412859560068, + "precision@5": 0.10389170896785109, + "precision@10": 0.06125211505922165, + "recall@1": 0.3316412859560068, + "recall@5": 0.5194585448392555, + "recall@10": 0.6125211505922166, + "f1@1": 0.3316412859560068, + "f1@5": 0.1731528482797519, + "f1@10": 0.11136748192585755, + "map@1": 0.3316412859560068, + "map@5": 0.40262267343485614, + "map@10": 0.4144972202078801, + "mrr@1": 0.3316412859560068, + "mrr@5": 0.40262267343485614, + "mrr@10": 0.4144972202078801, + "num_pred": 591, + "num_data": 591 }, - "MVBench": { - "hit@1": 0.3045, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.3045, - "ndcg_linear@5": 0.6822126660495169, - "ndcg_linear@10": 0.6822126660495169, - "ndcg_exponential@1": 0.3045, - "ndcg_exponential@5": 0.6822126660495169, - "ndcg_exponential@10": 0.6822126660495169, - "precision@1": 0.3045, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.3045, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.3045, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.3045, - "map@5": 0.5752166666666667, - "map@10": 0.5752166666666667, - "mrr@1": 0.3045, - "mrr@5": 0.5752166666666667, - "mrr@10": 0.5752166666666667, - "num_pred": 4000, - "num_data": 4000 + "VisRAG_SlideVQA": { + "hit@1": 0.6528776978417267, + "hit@5": 0.8687050359712231, + "hit@10": 0.8974820143884892, + "ndcg_linear@1": 0.6528776978417267, + "ndcg_linear@5": 0.7403317436593279, + "ndcg_linear@10": 0.7559278694497471, + "ndcg_exponential@1": 0.6528776978417267, + "ndcg_exponential@5": 0.7403317436593279, + "ndcg_exponential@10": 0.7559278694497471, + "precision@1": 0.6528776978417267, + "precision@5": 0.20683453237410068, + "precision@10": 0.1093525179856115, + "recall@1": 0.5611510791366906, + "recall@5": 0.8282374100719424, + "recall@10": 0.8705035971223022, + "f1@1": 0.591726618705036, + "f1@5": 0.32511133949982873, + "f1@10": 0.19206453019402664, + "map@1": 0.6528776978417267, + "map@5": 0.6970773381294963, + "map@10": 0.7055591241292679, + "mrr@1": 0.6528776978417267, + "mrr@5": 0.7357314148681056, + "mrr@10": 0.7400757965056526, + "num_pred": 556, + "num_data": 556 + }, + "VisRAG_InfoVQA": { + "hit@1": 0.5779944289693594, + "hit@5": 0.8189415041782729, + "hit@10": 0.8802228412256268, + "ndcg_linear@1": 0.5779944289693594, + "ndcg_linear@5": 0.7066787929757021, + "ndcg_linear@10": 0.7266471704004087, + "ndcg_exponential@1": 0.5779944289693594, + "ndcg_exponential@5": 0.7066787929757021, + "ndcg_exponential@10": 0.7266471704004087, + "precision@1": 0.5779944289693594, + "precision@5": 0.16378830083565457, + "precision@10": 0.08802228412256266, + "recall@1": 0.5779944289693594, + "recall@5": 0.8189415041782729, + "recall@10": 0.8802228412256268, + "f1@1": 0.5779944289693594, + "f1@5": 0.27298050139275776, + "f1@10": 0.16004051658647764, + "map@1": 0.5779944289693594, + "map@5": 0.6692664809656453, + "map@10": 0.6775898660299774, + "mrr@1": 0.5779944289693594, + "mrr@5": 0.6692664809656453, + "mrr@10": 0.6775898660299774, + "num_pred": 718, + "num_data": 718 }, - "Video-MME": { - "hit@1": 0.2688888888888889, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.2688888888888889, - "ndcg_linear@5": 0.6530182359914178, - "ndcg_linear@10": 0.6530182359914178, - "ndcg_exponential@1": 0.2688888888888889, - "ndcg_exponential@5": 0.6530182359914178, - "ndcg_exponential@10": 0.6530182359914178, - "precision@1": 0.2688888888888889, - "precision@5": 0.19999999999999996, - "precision@10": 0.09999999999999998, - "recall@1": 0.2688888888888889, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.2688888888888889, - "f1@5": 0.3333333333333334, - "f1@10": 0.18181818181818188, - "map@1": 0.2688888888888889, - "map@5": 0.5373148148148148, - "map@10": 0.5373148148148148, - "mrr@1": 0.2688888888888889, - "mrr@5": 0.5373148148148148, - "mrr@10": 0.5373148148148148, - "num_pred": 2700, - "num_data": 2700 + "VisRAG_PlotQA": { + "hit@1": 0.12977983777520277, + "hit@5": 0.3325608342989571, + "hit@10": 0.4287369640787949, + "ndcg_linear@1": 0.12977983777520277, + "ndcg_linear@5": 0.2339925322768118, + "ndcg_linear@10": 0.26492274577415076, + "ndcg_exponential@1": 0.12977983777520277, + "ndcg_exponential@5": 0.2339925322768118, + "ndcg_exponential@10": 0.26492274577415076, + "precision@1": 0.12977983777520277, + "precision@5": 0.06651216685979143, + "precision@10": 0.04287369640787949, + "recall@1": 0.12977983777520277, + "recall@5": 0.3325608342989571, + "recall@10": 0.4287369640787949, + "f1@1": 0.12977983777520277, + "f1@5": 0.11085361143298574, + "f1@10": 0.07795217528705363, + "map@1": 0.12977983777520277, + "map@5": 0.201525685592893, + "map@10": 0.2141831742353179, + "mrr@1": 0.12977983777520277, + "mrr@5": 0.201525685592893, + "mrr@10": 0.2141831742353179, + "num_pred": 863, + "num_data": 863 }, - "NExTQA": { - "hit@1": 0.20294255021018215, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.20294255021018215, - "ndcg_linear@5": 0.5984789868661349, - "ndcg_linear@10": 0.5984789868661349, - "ndcg_exponential@1": 0.20294255021018215, - "ndcg_exponential@5": 0.5984789868661349, - "ndcg_exponential@10": 0.5984789868661349, - "precision@1": 0.20294255021018215, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.20294255021018215, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.20294255021018215, - "f1@5": 0.3333333333333333, - "f1@10": 0.1818181818181818, - "map@1": 0.20294255021018215, - "map@5": 0.46760470185271685, - "map@10": 0.46760470185271685, - "mrr@1": 0.20294255021018215, - "mrr@5": 0.46744122684104006, - "mrr@10": 0.46744122684104006, - "num_pred": 8564, - "num_data": 8564 + "ViDoSeek-page": { + "hit@1": 0.043782837127845885, + "hit@5": 0.3064798598949212, + "hit@10": 0.4956217162872154, + "ndcg_linear@1": 0.043782837127845885, + "ndcg_linear@5": 0.1774384395530587, + "ndcg_linear@10": 0.2388470474707514, + "ndcg_exponential@1": 0.043782837127845885, + "ndcg_exponential@5": 0.17743843955305874, + "ndcg_exponential@10": 0.23884704747075144, + "precision@1": 0.043782837127845885, + "precision@5": 0.06129597197898424, + "precision@10": 0.049562171628721535, + "recall@1": 0.043782837127845885, + "recall@5": 0.3064798598949212, + "recall@10": 0.4956217162872154, + "f1@1": 0.043782837127845885, + "f1@5": 0.10215995329830709, + "f1@10": 0.09011303932494825, + "map@1": 0.043782837127845885, + "map@5": 0.135128429655575, + "map@10": 0.16060865093264393, + "mrr@1": 0.043782837127845885, + "mrr@5": 0.135128429655575, + "mrr@10": 0.16060865093264393, + "num_pred": 1142, + "num_data": 1142 }, - "EgoSchema": { - "hit@1": 0.254, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.254, - "ndcg_linear@5": 0.6252600495861029, - "ndcg_linear@10": 0.6252600495861029, - "ndcg_exponential@1": 0.254, - "ndcg_exponential@5": 0.6252600495861029, - "ndcg_exponential@10": 0.6252600495861029, - "precision@1": 0.254, - "precision@5": 0.2, - "precision@10": 0.1, - "recall@1": 0.254, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.254, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.254, - "map@5": 0.5030666666666666, - "map@10": 0.5030666666666666, - "mrr@1": 0.254, - "mrr@5": 0.5030666666666666, - "mrr@10": 0.5030666666666666, - "num_pred": 500, - "num_data": 500 + "ViDoSeek-doc": { + "hit@1": 0.9544658493870403, + "hit@5": 0.9842381786339754, + "hit@10": 0.9903677758318739, + "ndcg_linear@1": 0.6532399299474606, + "ndcg_linear@5": 0.7427733389906358, + "ndcg_linear@10": 0.6997209935069055, + "ndcg_exponential@1": 0.43807855891918934, + "ndcg_exponential@5": 0.6343478343437602, + "ndcg_exponential@10": 0.6353560202023368, + "precision@1": 0.9544658493870403, + "precision@5": 0.8038528896672504, + "precision@10": 0.6873029772329248, + "recall@1": 0.0563052881802568, + "recall@5": 0.23432760039766878, + "recall@10": 0.3914667961900597, + "f1@1": 0.10560395557458589, + "f1@5": 0.35560087282656433, + "f1@10": 0.48784025047176915, + "map@1": 0.9544658493870403, + "map@5": 0.7835522475189725, + "map@10": 0.6581428825421977, + "mrr@1": 0.9544658493870403, + "mrr@5": 0.96707530647986, + "mrr@10": 0.9679370639090428, + "num_pred": 1142, + "num_data": 1142 }, - "ActivityNetQA": { - "hit@1": 0.496, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.496, - "ndcg_linear@5": 0.8139885958000148, - "ndcg_linear@10": 0.8139885958000148, - "ndcg_exponential@1": 0.496, - "ndcg_exponential@5": 0.8139885958000148, - "ndcg_exponential@10": 0.8139885958000148, - "precision@1": 0.496, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.496, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.496, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.496, - "map@5": 0.748, - "map@10": 0.748, - "mrr@1": 0.496, - "mrr@5": 0.748, - "mrr@10": 0.748, - "num_pred": 1000, - "num_data": 1000 + "MMLongBench-page": { + "hit@1": 0.059665871121718374, + "hit@5": 0.18973747016706444, + "hit@10": 0.2529832935560859, + "ndcg_linear@1": 0.059665871121718374, + "ndcg_linear@5": 0.09604945291283169, + "ndcg_linear@10": 0.117231531231931, + "ndcg_exponential@1": 0.059665871121718374, + "ndcg_exponential@5": 0.09604945291283169, + "ndcg_exponential@10": 0.117231531231931, + "precision@1": 0.059665871121718374, + "precision@5": 0.04319809069212411, + "precision@10": 0.031264916467780436, + "recall@1": 0.030469371519490848, + "recall@5": 0.13269689737470167, + "recall@10": 0.19082140015910903, + "f1@1": 0.03882827594044778, + "f1@5": 0.06218015375673666, + "f1@10": 0.051867980865594236, + "map@1": 0.059665871121718374, + "map@5": 0.07087808273667462, + "map@10": 0.08054184661135735, + "mrr@1": 0.059665871121718374, + "mrr@5": 0.10765712012728719, + "mrr@10": 0.11626179111262644, + "num_pred": 838, + "num_data": 838 + }, + "MMLongBench-doc": { + "hit@1": 0.4439140811455847, + "hit@5": 0.6145584725536993, + "hit@10": 0.6957040572792362, + "ndcg_linear@1": 0.3146380270485282, + "ndcg_linear@5": 0.3256804529174008, + "ndcg_linear@10": 0.3131933005253596, + "ndcg_exponential@1": 0.2222979884077736, + "ndcg_exponential@5": 0.2727115335151017, + "ndcg_exponential@10": 0.2793202033337121, + "precision@1": 0.4439140811455847, + "precision@5": 0.35966587112171833, + "precision@10": 0.31730310262529837, + "recall@1": 0.013683394891534362, + "recall@5": 0.052843721626569064, + "recall@10": 0.08999131731205673, + "f1@1": 0.026297035367307157, + "f1@5": 0.08878818509054331, + "f1@10": 0.13199155724774939, + "map@1": 0.4439140811455847, + "map@5": 0.31638424821002387, + "map@10": 0.26005365193014357, + "mrr@1": 0.4439140811455847, + "mrr@5": 0.5094470962609386, + "mrr@10": 0.520618820320491, + "num_pred": 838, + "num_data": 838 } } }