Trying to make sense of nvidia/NV-Embed-v2 poor performance after Model2Vec distillation

[ashokrajab]

Distillation code:

m2v_model = distill(model_name="nvidia/NV-Embed-v2", pca_dims=256, trust_remote_code=True,device="cpu")
m2v_model.save_pretrained(f"NV-Embed-v2-512-model2vec")

Benchmark code:

from model2vec import StaticModel
import mteb
from mteb.encoder_interface import PromptType
import numpy as np
from mteb.abstasks.TaskMetadata import TaskMetadata
from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval

class CustomModel:
    def __init__(self):
        self.static_model = StaticModel.from_pretrained("/raid/ashok/optimum/nvidia_NV_Embed_v2-256-Model2Vec")
        self.max_length = 32768
        self.prompt = {
            "ClimateFEVER":
                    {
                        "query": "Given a claim about climate change, retrieve documents that support or refute the claim",
                        "passage": ""
                    },
            "HotpotQA":
                {
                    "query": "Given a multi-hop question, retrieve documents that can help answer the question",
                    "passage": ""
                },
            "FEVER":
                {
                    "query": "Given a claim, retrieve documents that support or refute the claim",
                    "passage": ""
                },
            "MSMARCO":
                {
                    "query": "Given a web search query, retrieve relevant passages that answer the query",
                    "passage": ""
                },
            "DBPedia":
                {
                    "query": "Given a query, retrieve relevant entity descriptions from DBPedia",
                    "passage": ""
                },
            "NQ":
                {
                    "query": "Given a question, retrieve passages that answer the question",
                    "passage": ""
                },
            "QuoraRetrieval":
                {
                    "query": "Given a question, retrieve questions that are semantically equivalent to the given question",
                    "passage": "Given a question, retrieve questions that are semantically equivalent to the given question"
                },
            "SCIDOCS":
                {
                    "query": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper",
                    "passage": ""
                },
            "TRECCOVID":
                {
                    "query": "Given a query on COVID-19, retrieve documents that answer the query",
                    "passage": ""
                },
            "Touche2020":
                {
                    "query": "Given a question, retrieve passages that answer the question",
                    "passage": ""
                },
            "SciFact":
                {
                    "query": "Given a scientific claim, retrieve documents that support or refute the claim",
                    "passage": ""
                },
            "NFCorpus":
                {
                    "query": "Given a question, retrieve relevant documents that answer the question",
                    "passage": ""
                },
            "ArguAna":
                {
                    "query": "Given a claim, retrieve documents that support or refute the claim",
                    "passage": ""
                },
            "FiQA2018":
                {
                    "query": "Given a financial question, retrieve relevant passages that answer the query",
                    "passage": ""
                },
            "STS":
                {
                    "text": "Retrieve semantically similar text"
                },
            "SUMM":
                {
                    "text": "Given a news summary, retrieve other semantically similar summaries"
                },
            "ZohoKB":
                {
                    "query": "Given a question, retrieve passages that answer the question",
                    "passage": ""
                },    
            "ZohoDesk":
                {
                    "query": "Given a question, retrieve passages that answer the question",
                    "passage": ""
                },    
            "ZohoPrivacy":
                {
                    "query": "Given a question, retrieve passages that answer the question",
                    "passage": ""
                },    
            }

    def encode(
        self,
        sentences: list[str],
        task_name: str,
        prompt_type: PromptType | None = None,
        **kwargs,
    ) -> np.ndarray:
        """Encodes the given sentences using the encoder.

        Args:
            sentences: The sentences to encode.
            task_name: The name of the task.
            prompt_type: The prompt type to use.
            **kwargs: Additional arguments to pass to the encoder.

        Returns:
            The encoded sentences.
        """
        instruction = self.prompt[task_name][prompt_type]
        new_sentences = []
        for i in range(0,len(sentences)):
             new_sentences.append(instruction + " " + sentences[i])
        embedding = self.static_model.encode(new_sentences)
        return embedding

# evaluating the model:
customeModel = CustomModel()
tasks = mteb.get_tasks(tasks=["SciFact"], languages=["eng"])
evaluation = mteb.MTEB(tasks=tasks)
results = evaluation.run(customeModel, output_folder=f"/results", eval_splits = ["test"]) 

Scores:

Metric	nvidia/NV-Embed-v2	nvidia/NV-Embed-v2-model2vec-256dims
ndcg_at_1	0.67667	0.02667
ndcg_at_3	0.76348	0.0415
ndcg_at_5	0.7883	0.04279
ndcg_at_10	0.80418	0.05029
ndcg_at_20	0.81098	0.059
ndcg_at_100	0.81623	0.0747
ndcg_at_1000	0.81794	0.10333

Entire benchmark metrics can be found here
nvidia-NV-Embed-v2-model2vec-256dims-SciFact.json
nvidia-NV-Embed-v2-SciFact.json

[stephantul]

Hello, could you rerun this without the prompts? Our models do not support prompting.

[stephantul]

Also, if you want to do retrieval, you might be better off using our own retrieval model: https://huggingface.co/minishlab/potion-retrieval-32M Let me know this helps!