colbert-muvera-pico / README.md

davidmezzetti

Update README.md

2bc75e0 verified about 3 hours ago

preview code

raw

history blame contribute delete

31.7 kB

metadata

language:
  - en
tags:
  - ColBERT
  - PyLate
  - sentence-transformers
  - sentence-similarity
  - feature-extraction
  - generated_from_trainer
  - dataset_size:640000
  - loss:Distillation
base_model: NeuML/bert-hash-pico
datasets:
  - lightonai/ms-marco-en-bge-gemma
pipeline_tag: sentence-similarity
library_name: PyLate
license: apache-2.0
metrics:
  - MaxSim_accuracy@1
  - MaxSim_accuracy@3
  - MaxSim_accuracy@5
  - MaxSim_accuracy@10
  - MaxSim_precision@1
  - MaxSim_precision@3
  - MaxSim_precision@5
  - MaxSim_precision@10
  - MaxSim_recall@1
  - MaxSim_recall@3
  - MaxSim_recall@5
  - MaxSim_recall@10
  - MaxSim_ndcg@10
  - MaxSim_mrr@10
  - MaxSim_map@100
model-index:
  - name: ColBERT MUVERA Pico
    results:
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoClimateFEVER
          type: NanoClimateFEVER
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.22
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.32
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.4
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.54
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.22
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.11333333333333333
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.09200000000000001
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.062
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.125
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.18
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.22
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.2723333333333333
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.22523375350232466
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.2918015873015873
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.17799142530014567
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoDBPedia
          type: NanoDBPedia
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.68
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.82
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.86
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.9
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.68
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.5733333333333333
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.49999999999999994
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.47
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.054334623940057496
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.13885343313592602
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.19014904910202987
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.32073944983710506
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.5579484125370805
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.7584126984126983
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.42613646438564345
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoFEVER
          type: NanoFEVER
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.68
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.78
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.88
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.92
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.68
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.27333333333333326
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.19199999999999995
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.1
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.6466666666666667
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.7633333333333333
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.87
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.91
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.7844876010568872
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.7531666666666667
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.7413336663336665
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoFiQA2018
          type: NanoFiQA2018
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.3
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.48
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.54
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.66
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.3
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.2
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.156
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.098
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.13885714285714287
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.28949206349206347
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.34890476190476183
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.4669047619047619
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.35173862968285535
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.41173809523809524
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.27418803367117023
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoHotpotQA
          type: NanoHotpotQA
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.84
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.92
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.96
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.96
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.84
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.44666666666666655
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.288
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.152
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.42
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.67
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.72
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.76
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.7396469666031694
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.8846666666666667
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.6745663269195332
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoMSMARCO
          type: NanoMSMARCO
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.4
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.56
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.64
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.72
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.4
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.18666666666666668
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.128
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.07200000000000001
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.4
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.56
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.64
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.72
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.5513379104118443
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.4978571428571428
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.5100075204701912
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoNFCorpus
          type: NanoNFCorpus
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.42
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.5
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.58
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.62
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.42
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.36
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.316
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.256
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.040942245985757866
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.07446981664033472
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.08961908265974948
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.11750646414430962
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.31817112092123645
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.4845555555555556
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.12896096534086632
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoNQ
          type: NanoNQ
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.34
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.62
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.66
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.72
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.34
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.20666666666666664
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.132
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.07400000000000001
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.32
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.58
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.62
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.68
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.5190287199365009
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.4836031746031745
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.4681591756850757
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoQuoraRetrieval
          type: NanoQuoraRetrieval
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.82
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.9
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.9
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.92
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.82
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.32666666666666666
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.20799999999999996
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.11199999999999999
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.7440000000000001
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.8306666666666667
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.8540000000000001
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.8773333333333332
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.8467657844266899
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.8625
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.8312680912657447
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoSCIDOCS
          type: NanoSCIDOCS
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.32
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.52
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.58
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.66
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.32
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.21333333333333332
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.18
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.12
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.066
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.13366666666666668
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.18566666666666662
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.24566666666666667
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.2462853863944484
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.4382222222222222
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.17492730667100076
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoArguAna
          type: NanoArguAna
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.1
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.3
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.4
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.46
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.1
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.09999999999999998
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.08000000000000002
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.046000000000000006
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.1
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.3
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.4
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.46
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.275284156147708
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.21590476190476193
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.22481590517812997
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoSciFact
          type: NanoSciFact
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.48
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.64
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.72
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.78
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.48
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.22666666666666668
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.15999999999999998
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.088
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.445
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.62
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.71
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.78
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.6190838299940942
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.5744444444444444
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.5693936722581301
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoTouche2020
          type: NanoTouche2020
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.673469387755102
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.9183673469387755
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 1
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 1
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.673469387755102
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.6462585034013605
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.616326530612245
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.4938775510204082
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.04572557745748646
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.12524012746937077
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.19598845564624598
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.3058088560800141
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.5544751618654643
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.7962585034013605
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.37144809611649504
            name: Maxsim Map@100
      - task:
          type: nano-beir
          name: Nano BEIR
        dataset:
          name: NanoBEIR mean
          type: NanoBEIR_mean
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.4825745682888539
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.6367974882260596
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.7015384615384617
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.7584615384615384
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.4825745682888539
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.2979173207744636
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.23448665620094195
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.16491365777080064
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.2728097120697778
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.4050555467234125
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.4649483089214964
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.5320225280999634
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.5068836487292541
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.573317809174952
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.4287074345842918
            name: Maxsim Map@100

ColBERT MUVERA Pico

This is a PyLate model finetuned from neuml/bert-hash-pico on the msmarco-en-bge-gemma unnormalized split dataset. It maps sentences & paragraphs to sequences of 80-dimensional dense vectors and can be used for semantic textual similarity using the MaxSim operator.

This model is trained with un-normalized scores, making it compatible with MUVERA fixed-dimensional encoding.

Usage (txtai)

This model can be used to build embeddings databases with txtai for semantic search and/or as a knowledge source for retrieval augmented generation (RAG).

Note: txtai 9.0+ is required for late interaction model support

import txtai

embeddings = txtai.Embeddings(
  path="neuml/colbert-muvera-pico",
  content=True
)
embeddings.index(documents())

# Run a query
embeddings.search("query to run")

Late interaction models excel as reranker pipelines.

from txtai.pipeline import Reranker, Similarity

similarity = Similarity(path="neuml/colbert-muvera-pico", lateencode=True)
ranker = Reranker(embeddings, similarity)
ranker("query to run")

Usage (PyLate)

Alternatively, the model can be loaded with PyLate.

from pylate import rank, models

queries = [
    "query A",
    "query B",
]

documents = [
    ["document A", "document B"],
    ["document 1", "document C", "document B"],
]

documents_ids = [
    [1, 2],
    [1, 3, 2],
]

model = models.ColBERT(
    model_name_or_path="neuml/colbert-muvera-pico",
)

queries_embeddings = model.encode(
    queries,
    is_query=True,
)

documents_embeddings = model.encode(
    documents,
    is_query=False,
)

reranked_documents = rank.rerank(
    documents_ids=documents_ids,
    queries_embeddings=queries_embeddings,
    documents_embeddings=documents_embeddings,
)

Full Model Architecture

ColBERT(
  (0): Transformer({'max_seq_length': 299, 'do_lower_case': False}) with Transformer model: BertHashModel 
  (1): Dense({'in_features': 80, 'out_features': 128, 'bias': False, 'activation_function': 'torch.nn.modules.linear.Identity'})
)

Evaluation

BEIR Subset

The following table shows a subset of BEIR scored with the txtai benchmarks script.

Scores reported are ndcg@10 and grouped into the following three categories.

FULL multi-vector maxsim

Model	Parameters	NFCorpus	SciDocs	SciFact	Average
ColBERT v2	110M	0.3165	0.1497	0.6456	0.3706
ColBERT MUVERA Femto	0.2M	0.2513	0.0870	0.4710	0.2698
ColBERT MUVERA Pico	0.4M	0.3005	0.1117	0.6452	0.3525
ColBERT MUVERA Nano	0.9M	0.3180	0.1262	0.6576	0.3673
ColBERT MUVERA Micro	4M	0.3235	0.1244	0.6676	0.3718

MUVERA encoding + maxsim re-ranking of the top 100 results per MUVERA paper

Model	Parameters	NFCorpus	SciDocs	SciFact	Average
ColBERT v2	110M	0.3025	0.1538	0.6278	0.3614
ColBERT MUVERA Femto	0.2M	0.2316	0.0858	0.4641	0.2605
ColBERT MUVERA Pico	0.4M	0.2821	0.1004	0.6090	0.3305
ColBERT MUVERA Nano	0.9M	0.2996	0.1201	0.6249	0.3482
ColBERT MUVERA Micro	4M	0.3095	0.1228	0.6464	0.3596

MUVERA encoding only

Model	Parameters	NFCorpus	SciDocs	SciFact	Average
ColBERT v2	110M	0.2356	0.1229	0.5002	0.2862
ColBERT MUVERA Femto	0.2M	0.1851	0.0411	0.3518	0.1927
ColBERT MUVERA Pico	0.4M	0.1926	0.0564	0.4424	0.2305
ColBERT MUVERA Nano	0.9M	0.2355	0.0807	0.4904	0.2689
ColBERT MUVERA Micro	4M	0.2348	0.0882	0.4875	0.2702

Note: The scores reported don't match scores reported in the respective papers due to different default settings in the txtai benchmark scripts.

As noted earlier, models trained with min-max score normalization don't perform well with MUVERA encoding. See this GitHub Issue for more.

At 450K parameters, this model does shockingly well! It's not too far off from the baseline 4M parameter model at 1/10th the size. It's also not too far off from the original ColBERT v2 model, which has 110M parameters.

Nano BEIR

Dataset: NanoBEIR_mean
Evaluated with pylate.evaluation.nano_beir_evaluator.NanoBEIREvaluator

Metric	Value
MaxSim_accuracy@1	0.4826
MaxSim_accuracy@3	0.6368
MaxSim_accuracy@5	0.7015
MaxSim_accuracy@10	0.7585
MaxSim_precision@1	0.4826
MaxSim_precision@3	0.2979
MaxSim_precision@5	0.2345
MaxSim_precision@10	0.1649
MaxSim_recall@1	0.2728
MaxSim_recall@3	0.4051
MaxSim_recall@5	0.4649
MaxSim_recall@10	0.532
MaxSim_ndcg@10	0.5069
MaxSim_mrr@10	0.5733
MaxSim_map@100	0.4287

Training Details

Training Hyperparameters

Non-Default Hyperparameters

eval_strategy: steps
per_device_train_batch_size: 32
learning_rate: 0.0003
num_train_epochs: 1
warmup_ratio: 0.05
fp16: True

All Hyperparameters

Click to expand

overwrite_output_dir: False
do_predict: False
eval_strategy: steps
prediction_loss_only: True
per_device_train_batch_size: 32
per_device_eval_batch_size: 8
per_gpu_train_batch_size: None
per_gpu_eval_batch_size: None
gradient_accumulation_steps: 1
eval_accumulation_steps: None
torch_empty_cache_steps: None
learning_rate: 0.0003
weight_decay: 0.0
adam_beta1: 0.9
adam_beta2: 0.999
adam_epsilon: 1e-08
max_grad_norm: 1.0
num_train_epochs: 1
max_steps: -1
lr_scheduler_type: linear
lr_scheduler_kwargs: {}
warmup_ratio: 0.05
warmup_steps: 0
log_level: passive
log_level_replica: warning
log_on_each_node: True
logging_nan_inf_filter: True
save_safetensors: True
save_on_each_node: False
save_only_model: False
restore_callback_states_from_checkpoint: False
no_cuda: False
use_cpu: False
use_mps_device: False
seed: 42
data_seed: None
jit_mode_eval: False
bf16: False
fp16: True
fp16_opt_level: O1
half_precision_backend: auto
bf16_full_eval: False
fp16_full_eval: False
tf32: None
local_rank: 0
ddp_backend: None
tpu_num_cores: None
tpu_metrics_debug: False
debug: []
dataloader_drop_last: False
dataloader_num_workers: 0
dataloader_prefetch_factor: None
past_index: -1
disable_tqdm: False
remove_unused_columns: True
label_names: None
load_best_model_at_end: False
ignore_data_skip: False
fsdp: []
fsdp_min_num_params: 0
fsdp_config: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}
fsdp_transformer_layer_cls_to_wrap: None
accelerator_config: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
parallelism_config: None
deepspeed: None
label_smoothing_factor: 0.0
optim: adamw_torch_fused
optim_args: None
adafactor: False
group_by_length: False
length_column_name: length
project: huggingface
trackio_space_id: trackio
ddp_find_unused_parameters: None
ddp_bucket_cap_mb: None
ddp_broadcast_buffers: False
dataloader_pin_memory: True
dataloader_persistent_workers: False
skip_memory_metrics: True
use_legacy_prediction_loop: False
push_to_hub: False
resume_from_checkpoint: None
hub_model_id: None
hub_strategy: every_save
hub_private_repo: None
hub_always_push: False
hub_revision: None
gradient_checkpointing: False
gradient_checkpointing_kwargs: None
include_inputs_for_metrics: False
include_for_metrics: []
eval_do_concat_batches: True
fp16_backend: auto
push_to_hub_model_id: None
push_to_hub_organization: None
mp_parameters:
auto_find_batch_size: False
full_determinism: False
torchdynamo: None
ray_scope: last
ddp_timeout: 1800
torch_compile: False
torch_compile_backend: None
torch_compile_mode: None
include_tokens_per_second: False
include_num_input_tokens_seen: no
neftune_noise_alpha: None
optim_target_modules: None
batch_eval_metrics: False
eval_on_start: False
use_liger_kernel: False
liger_kernel_config: None
eval_use_gather_object: False
average_tokens_across_devices: True
prompts: None
batch_sampler: batch_sampler
multi_dataset_batch_sampler: proportional

Framework Versions

Python: 3.10.18
Sentence Transformers: 4.0.2
PyLate: 1.3.2
Transformers: 4.57.0
PyTorch: 2.8.0+cu128
Accelerate: 1.10.1
Datasets: 4.1.1
Tokenizers: 0.22.1

Citation

BibTeX

Sentence Transformers

@inproceedings{reimers-2019-sentence-bert,
    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
    author = "Reimers, Nils and Gurevych, Iryna",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
    month = "11",
    year = "2019",
    publisher = "Association for Computational Linguistics",
    url = "https://arxiv.org/abs/1908.10084"
}

PyLate

@misc{PyLate,
title={PyLate: Flexible Training and Retrieval for Late Interaction Models},
author={Chaffin, Antoine and Sourty, Raphaël},
url={https://github.com/lightonai/pylate},
year={2024}
}