[GH-ISSUE #24] Performance gap when evaluating on vlmeval

bsakaguchi commented

2026-10-26 21:10:19 +00:00

Owner

Originally created by @weiaicunzai on GitHub (Dec 26, 2024).
Original GitHub issue: https://github.com/deepseek-ai/DeepSeek-VL2/issues/24

There is a performance gap of more than 10 percent between my evaluation and the reported results in the paper for all the datasets mentioned.

Is there something I might be doing wrong?

The code I used is from vlmeval.

class DeepSeekVL2(BaseModel):

    INSTALL_REQ = True
    INTERLEAVE = True

    def __init__(self, model_path='deepseek-ai/deepseek-vl-1.3b-chat', **kwargs):
        from deepseek_vl2.models import DeepseekVLV2Processor


        assert model_path is not None
        self.model_path = model_path



        self.vl_chat_processor  = DeepseekVLV2Processor.from_pretrained(model_path)
        self.tokenizer = self.vl_chat_processor.tokenizer

        vl_gpt = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True)
  
        self.model = vl_gpt.to(torch.bfloat16).cuda().eval()

        torch.cuda.empty_cache()
        default_kwargs = dict(max_new_tokens=512, do_sample=False, use_cache=True)
        default_kwargs.update(kwargs)
        self.kwargs = default_kwargs
        warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ')

    def prepare_inputs(self, message):
        def prepare_itlist(msgs):
            content, images = '', []
            for s in msgs:
                if s['type'] == 'image':
                    images.append(s['value'])
                    content += '<image>'
                elif s['type'] == 'text':
                    content += s['value']
            return content, images
        conversation = []
        if 'role' not in message[0]:
            content, images = prepare_itlist(message)
            conversation.append(dict(role='<|User|>', content=content, images=images))
        else:
            role_map = {'user': '<|User|>', 'assistant': '<|Assistant|>'}
            for msgs in message:
                role = role_map[msgs['role']]
                content, images = prepare_itlist(msgs['content'])
                conversation.append(dict(role=role, content=content, images=images))
        conversation.append(dict(role='<|Assistant|>', content=''))
        return conversation

    def generate_inner(self, message, dataset=None):

        conversation = self.prepare_inputs(message)
        from deepseek_vl2.utils.io import load_pil_images
        pil_images = load_pil_images(conversation)
        prepare_inputs = self.vl_chat_processor(
                conversations=conversation,
                images=pil_images,
                force_batchify=True,
                system_prompt=""
            )


        prepare_inputs = prepare_inputs.to(self.model.device)
        inputs_embeds = self.model.prepare_inputs_embeds(**prepare_inputs)

        outputs = self.model.language.generate(
            inputs_embeds=inputs_embeds,
            attention_mask=prepare_inputs.attention_mask,
            pad_token_id=self.tokenizer.eos_token_id,
            bos_token_id=self.tokenizer.bos_token_id,
            eos_token_id=self.tokenizer.eos_token_id,
            # max_new_tokens=512,
            **self.kwargs)
        answer = self.tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)

        return answer

    def use_custom_prompt(self, dataset):
        assert dataset is not None
        if listinstr(['MMDU', 'MME-RealWorld', 'MME-RealWorld-CN'], dataset):
            # For Multi-Turn we don't have custom prompt
            return False
        if DATASET_MODALITY(dataset) == 'VIDEO':
            # For Video benchmarks we don't have custom prompt at here
            return False
        else:
            return True

    def build_prompt(self, line, dataset=None):
        assert self.use_custom_prompt(dataset)
        assert dataset is None or isinstance(dataset, str)
        tgt_path = self.dump_image(line, dataset)

        if dataset is not None and listinstr(['MME'], dataset):
            question = line['question']
            prompt = question + ' Answer the question using a single word or phrase.'
        elif dataset is not None and listinstr(['HallusionBench'], dataset):
            question = line['question']
            prompt = question + ' Please answer yes or no. Answer the question using a single word or phrase.'
        elif dataset is not None and DATASET_TYPE(dataset) == 'MCQ':
            prompt = self.build_multi_choice_prompt(line, dataset)
        elif dataset is not None and DATASET_TYPE(dataset) == 'VQA':
            if listinstr(['MathVista', 'MathVision'], dataset):
                prompt = line['question']
            elif listinstr(['LLaVABench'], dataset):
                question = line['question']
                prompt = question + '\nAnswer this question in detail.'
            elif listinstr(['MMVet'], dataset):
                prompt = line['question']
            elif listinstr(['MTBench_VQA'], dataset):
                prompt = line['question'] + '\n 请直接回答问题'
            else:
                question = line['question']
                prompt = question + '\nAnswer the question using a single word or phrase.'
        else:
            prompt = line['question']
        message = [dict(type='text', value=prompt)]
        message.extend([dict(type='image', value=s) for s in tgt_path])
        return message

    def chat_inner(self, message, dataset=None):
        return self.generate_inner(message, dataset=dataset)


    def build_multi_choice_prompt(self, line, dataset=None):
        question = line['question']
        # print(line)
        hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None
        if hint is not None:
            question = hint + '\n' + question

        options = {
            cand: line[cand]
            for cand in string.ascii_uppercase
            if cand in line and not pd.isna(line[cand])
        }
        for key, item in options.items():
            question += f'\n{key}. {item}'
        prompt = question

        if len(options):
            prompt += '\n请直接回答选项字母。' if cn_string(
                prompt) else "\nAnswer with the option's letter from the given choices directly."
        else:
            prompt += '\n请直接回答问题。' if cn_string(prompt) else '\nAnswer the question directly.'

        return prompt

Originally created by @weiaicunzai on GitHub (Dec 26, 2024). Original GitHub issue: https://github.com/deepseek-ai/DeepSeek-VL2/issues/24 There is a performance gap of more than 10 percent between my evaluation and the reported results in the paper for all the datasets mentioned. Is there something I might be doing wrong? The code I used is from vlmeval. ``` class DeepSeekVL2(BaseModel): INSTALL_REQ = True INTERLEAVE = True def __init__(self, model_path='deepseek-ai/deepseek-vl-1.3b-chat', **kwargs): from deepseek_vl2.models import DeepseekVLV2Processor assert model_path is not None self.model_path = model_path self.vl_chat_processor = DeepseekVLV2Processor.from_pretrained(model_path) self.tokenizer = self.vl_chat_processor.tokenizer vl_gpt = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True) self.model = vl_gpt.to(torch.bfloat16).cuda().eval() torch.cuda.empty_cache() default_kwargs = dict(max_new_tokens=512, do_sample=False, use_cache=True) default_kwargs.update(kwargs) self.kwargs = default_kwargs warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') def prepare_inputs(self, message): def prepare_itlist(msgs): content, images = '', [] for s in msgs: if s['type'] == 'image': images.append(s['value']) content += '<image>' elif s['type'] == 'text': content += s['value'] return content, images conversation = [] if 'role' not in message[0]: content, images = prepare_itlist(message) conversation.append(dict(role='<|User|>', content=content, images=images)) else: role_map = {'user': '<|User|>', 'assistant': '<|Assistant|>'} for msgs in message: role = role_map[msgs['role']] content, images = prepare_itlist(msgs['content']) conversation.append(dict(role=role, content=content, images=images)) conversation.append(dict(role='<|Assistant|>', content='')) return conversation def generate_inner(self, message, dataset=None): conversation = self.prepare_inputs(message) from deepseek_vl2.utils.io import load_pil_images pil_images = load_pil_images(conversation) prepare_inputs = self.vl_chat_processor( conversations=conversation, images=pil_images, force_batchify=True, system_prompt="" ) prepare_inputs = prepare_inputs.to(self.model.device) inputs_embeds = self.model.prepare_inputs_embeds(**prepare_inputs) outputs = self.model.language.generate( inputs_embeds=inputs_embeds, attention_mask=prepare_inputs.attention_mask, pad_token_id=self.tokenizer.eos_token_id, bos_token_id=self.tokenizer.bos_token_id, eos_token_id=self.tokenizer.eos_token_id, # max_new_tokens=512, **self.kwargs) answer = self.tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True) return answer def use_custom_prompt(self, dataset): assert dataset is not None if listinstr(['MMDU', 'MME-RealWorld', 'MME-RealWorld-CN'], dataset): # For Multi-Turn we don't have custom prompt return False if DATASET_MODALITY(dataset) == 'VIDEO': # For Video benchmarks we don't have custom prompt at here return False else: return True def build_prompt(self, line, dataset=None): assert self.use_custom_prompt(dataset) assert dataset is None or isinstance(dataset, str) tgt_path = self.dump_image(line, dataset) if dataset is not None and listinstr(['MME'], dataset): question = line['question'] prompt = question + ' Answer the question using a single word or phrase.' elif dataset is not None and listinstr(['HallusionBench'], dataset): question = line['question'] prompt = question + ' Please answer yes or no. Answer the question using a single word or phrase.' elif dataset is not None and DATASET_TYPE(dataset) == 'MCQ': prompt = self.build_multi_choice_prompt(line, dataset) elif dataset is not None and DATASET_TYPE(dataset) == 'VQA': if listinstr(['MathVista', 'MathVision'], dataset): prompt = line['question'] elif listinstr(['LLaVABench'], dataset): question = line['question'] prompt = question + '\nAnswer this question in detail.' elif listinstr(['MMVet'], dataset): prompt = line['question'] elif listinstr(['MTBench_VQA'], dataset): prompt = line['question'] + '\n 请直接回答问题' else: question = line['question'] prompt = question + '\nAnswer the question using a single word or phrase.' else: prompt = line['question'] message = [dict(type='text', value=prompt)] message.extend([dict(type='image', value=s) for s in tgt_path]) return message def chat_inner(self, message, dataset=None): return self.generate_inner(message, dataset=dataset) def build_multi_choice_prompt(self, line, dataset=None): question = line['question'] # print(line) hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None if hint is not None: question = hint + '\n' + question options = { cand: line[cand] for cand in string.ascii_uppercase if cand in line and not pd.isna(line[cand]) } for key, item in options.items(): question += f'\n{key}. {item}' prompt = question if len(options): prompt += '\n请直接回答选项字母。' if cn_string( prompt) else "\nAnswer with the option's letter from the given choices directly." else: prompt += '\n请直接回答问题。' if cn_string(prompt) else '\nAnswer the question directly.' return prompt ```

bsakaguchi closed this issue

2026-10-26 21:10:19 +00:00

bsakaguchi commented

2026-10-26 21:10:20 +00:00

Author

Owner

@gnobitab commented on GitHub (Dec 27, 2024):

Please try evaluation again after our official PR gets merged by VLMEvalKit.

https://github.com/open-compass/VLMEvalKit/pull/697

@gnobitab commented on GitHub (Dec 27, 2024): Please try evaluation again after our official PR gets merged by VLMEvalKit. https://github.com/open-compass/VLMEvalKit/pull/697

bsakaguchi commented

2026-10-26 21:10:20 +00:00

Author

Owner

@weiaicunzai commented on GitHub (Jan 2, 2025):

感谢，用了你提交的pr以后分数正常了。

@weiaicunzai commented on GitHub (Jan 2, 2025): 感谢，用了你提交的pr以后分数正常了。

Rows
Columns

[GH-ISSUE #24] Performance gap when evaluating on vlmeval #19