kvcache-ai/ktransformers

Fork 0

mirror of https://github.com/kvcache-ai/ktransformers.git synced 2026-11-05 01:07:28 +00:00

[GH-ISSUE #371] 不支持VSCode Cline生成的提示 #244

New issue

Open

opened 2026-10-29 14:36:28 +00:00 by bsakaguchi · 4 comments

bsakaguchi commented

2026-10-29 14:36:28 +00:00

Owner

Originally created by @zhengjx-ai on GitHub (Feb 16, 2025).
Original GitHub issue: https://github.com/kvcache-ai/ktransformers/issues/371

使用Cline配合ktransformers-0.2 main版本server，DeepSeek-V2.5-IQ4_XS模型。

Cline会生成一个大约12K提示的/v1/chat/completion请求（见附件），server直接返回422 Unprocessable Entity。貌似是FastAPI的问题。

api.log

Originally created by @zhengjx-ai on GitHub (Feb 16, 2025). Original GitHub issue: https://github.com/kvcache-ai/ktransformers/issues/371 使用Cline配合ktransformers-0.2 main版本server，DeepSeek-V2.5-IQ4_XS模型。 Cline会生成一个大约12K提示的/v1/chat/completion请求（见附件），server直接返回422 Unprocessable Entity。貌似是FastAPI的问题。 [api.log](https://github.com/user-attachments/files/18814719/api.log) ![Image](https://github.com/user-attachments/assets/d97a47aa-7079-435b-9f80-db78e3c327bc)

bsakaguchi commented

2026-10-29 14:36:29 +00:00

Author

Owner

@laf070810 commented on GitHub (Feb 16, 2025):

我在ktransformers v0.2.1跑DeepSeek-R1-Q4_K_M模型时也碰到了这个问题

@laf070810 commented on GitHub (Feb 16, 2025): 我在ktransformers v0.2.1跑DeepSeek-R1-Q4_K_M模型时也碰到了这个问题

bsakaguchi commented

2026-10-29 14:36:29 +00:00

Author

Owner

@DeerEyre commented on GitHub (Feb 18, 2025):

同样遇到了这个问题0.2.1版本
import os
import platform
import sys
import torch
import logging
from transformers import (
AutoTokenizer,
AutoConfig,
AutoModelForCausalLM,
GenerationConfig,
TextStreamer,
)
import json
import fire
from ktransformers.optimize.optimize import optimize_and_load_gguf
from ktransformers.models.modeling_deepseek import DeepseekV2ForCausalLM
from ktransformers.models.modeling_qwen2_moe import Qwen2MoeForCausalLM
from ktransformers.models.modeling_deepseek_v3 import DeepseekV3ForCausalLM
from ktransformers.models.modeling_llama import LlamaForCausalLM
from ktransformers.models.modeling_mixtral import MixtralForCausalLM
from ktransformers.util.utils import prefill_and_generate
from ktransformers.server.config.config import Config
from fastapi import FastAPI

原代码中的配置部分保持不变

project_dir = os.path.dirname(os.path.dirname(file))
sys.path.insert(0, project_dir)

custom_models = {
"DeepseekV2ForCausalLM": DeepseekV2ForCausalLM,
"DeepseekV3ForCausalLM": DeepseekV3ForCausalLM,
"Qwen2MoeForCausalLM": Qwen2MoeForCausalLM,
"LlamaForCausalLM": LlamaForCausalLM,
"MixtralForCausalLM": MixtralForCausalLM,
}

ktransformer_rules_dir = (
os.path.dirname(os.path.abspath(file)) + "/optimize/optimize_rules/"
)

default_optimize_rules = {
"DeepseekV2ForCausalLM": ktransformer_rules_dir + "DeepSeek-V2-Chat.yaml",
"DeepseekV3ForCausalLM": ktransformer_rules_dir + "DeepSeek-V3-Chat.yaml",
"Qwen2MoeForCausalLM": ktransformer_rules_dir + "Qwen2-57B-A14B-Instruct.yaml",
"LlamaForCausalLM": ktransformer_rules_dir + "Internlm2_5-7b-Chat-1m.yaml",
"MixtralForCausalLM": ktransformer_rules_dir + "Mixtral.yaml",
}

定义一个函数来初始化和运行模型

def init_model(
model_path: str,
optimize_rule_path: str = None,
gguf_path: str = None,
max_new_tokens: int = 300,
cpu_infer: int = Config().cpu_infer,
use_cuda_graph: bool = True,
mode: str = "normal",
force_think: bool = False,
):
torch.set_grad_enabled(False)
Config().cpu_infer = cpu_infer
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)

if mode == 'long_context': 
    assert config.architectures[0]  == "LlamaForCausalLM", "only LlamaForCausalLM support long_context mode" 
    torch.set_default_dtype(torch.float16)  
else: 
    torch.set_default_dtype(config.torch_dtype)  

with torch.device("meta"):  
    if config.architectures[0]  in custom_models: 
        print("using custom modeling_xxx.py.")  
        if "Qwen2Moe" in config.architectures[0]:  
            config._attn_implementation = "flash_attention_2" 
        if "Llama" in config.architectures[0]:  
            config._attn_implementation = "eager" 
        if "Mixtral" in config.architectures[0]:  
            config._attn_implementation = "flash_attention_2" 
        model = custom_models[config.architectures[0]](config) 
    else: 
        model = AutoModelForCausalLM.from_config(  
            config, trust_remote_code=True, attn_implementation="flash_attention_2" 
        ) 

if optimize_rule_path is None: 
    if config.architectures[0]  in default_optimize_rules: 
        print("using default_optimize_rule for", config.architectures[0])  
        optimize_rule_path = default_optimize_rules[config.architectures[0]] 
    else: 
        raise ValueError("No default optimize rule found. Please provide a rule file.") 

if gguf_path is None: 
    raise ValueError("Please provide a gguf file path.") 

optimize_and_load_gguf(model, optimize_rule_path, gguf_path, config) 

try: 
    model.generation_config  = GenerationConfig.from_pretrained(model_path)  
except: 
    gen_config = GenerationConfig( 
        max_length=128, temperature=0.7, top_p=0.9, do_sample=True 
    ) 
    model.generation_config  = gen_config 

if model.generation_config.pad_token_id  is None: 
    model.generation_config.pad_token_id  = model.generation_config.eos_token_id  

model.eval()  
return model, tokenizer

创建FastAPI应用

app = FastAPI()

全局初始化模型和分词器

model, tokenizer = None, None

@app.on_event("startup")
async def startup_event():
global model, tokenizer
# 这里填入你的模型路径和gguf路径
model_path = "/data/llm_tuning/dpsk/R1-Q4-config"
gguf_path = "/data/llm_tuning/dpsk/DeepSeek-R1-GGUF"
optimize_rule_path = None
max_new_tokens = 2000
cpu_infer = 65
model, tokenizer = init_model(
model_path,
optimize_rule_path,
gguf_path,
max_new_tokens,
cpu_infer,
)

@app.post("/generate")
async def chat(prompt: str):
messages = [{"role": "user", "content": prompt}]
input_tensor = tokenizer.apply_chat_template(
messages, add_generation_prompt=True, return_tensors="pt"
)
max_new_tokens = 2000
use_cuda_graph = True
mode = "normal"
force_think = False
generated = prefill_and_generate(
model, tokenizer, input_tensor.cuda(), max_new_tokens, use_cuda_graph, mode, force_think
)
result = tokenizer.decode(generated[0], skip_special_tokens=True)
return {"result": result}

if name == "main":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=50711)

@DeerEyre commented on GitHub (Feb 18, 2025): 同样遇到了这个问题0.2.1版本 import os import platform import sys import torch import logging from transformers import ( AutoTokenizer, AutoConfig, AutoModelForCausalLM, GenerationConfig, TextStreamer, ) import json import fire from ktransformers.optimize.optimize import optimize_and_load_gguf from ktransformers.models.modeling_deepseek import DeepseekV2ForCausalLM from ktransformers.models.modeling_qwen2_moe import Qwen2MoeForCausalLM from ktransformers.models.modeling_deepseek_v3 import DeepseekV3ForCausalLM from ktransformers.models.modeling_llama import LlamaForCausalLM from ktransformers.models.modeling_mixtral import MixtralForCausalLM from ktransformers.util.utils import prefill_and_generate from ktransformers.server.config.config import Config from fastapi import FastAPI # 原代码中的配置部分保持不变 project_dir = os.path.dirname(os.path.dirname(__file__)) sys.path.insert(0, project_dir) custom_models = { "DeepseekV2ForCausalLM": DeepseekV2ForCausalLM, "DeepseekV3ForCausalLM": DeepseekV3ForCausalLM, "Qwen2MoeForCausalLM": Qwen2MoeForCausalLM, "LlamaForCausalLM": LlamaForCausalLM, "MixtralForCausalLM": MixtralForCausalLM, } ktransformer_rules_dir = ( os.path.dirname(os.path.abspath(__file__)) + "/optimize/optimize_rules/" ) default_optimize_rules = { "DeepseekV2ForCausalLM": ktransformer_rules_dir + "DeepSeek-V2-Chat.yaml", "DeepseekV3ForCausalLM": ktransformer_rules_dir + "DeepSeek-V3-Chat.yaml", "Qwen2MoeForCausalLM": ktransformer_rules_dir + "Qwen2-57B-A14B-Instruct.yaml", "LlamaForCausalLM": ktransformer_rules_dir + "Internlm2_5-7b-Chat-1m.yaml", "MixtralForCausalLM": ktransformer_rules_dir + "Mixtral.yaml", } # 定义一个函数来初始化和运行模型 def init_model( model_path: str, optimize_rule_path: str = None, gguf_path: str = None, max_new_tokens: int = 300, cpu_infer: int = Config().cpu_infer, use_cuda_graph: bool = True, mode: str = "normal", force_think: bool = False, ): torch.set_grad_enabled(False) Config().cpu_infer = cpu_infer tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) if mode == 'long_context': assert config.architectures[0] == "LlamaForCausalLM", "only LlamaForCausalLM support long_context mode" torch.set_default_dtype(torch.float16) else: torch.set_default_dtype(config.torch_dtype) with torch.device("meta"): if config.architectures[0] in custom_models: print("using custom modeling_xxx.py.") if "Qwen2Moe" in config.architectures[0]: config._attn_implementation = "flash_attention_2" if "Llama" in config.architectures[0]: config._attn_implementation = "eager" if "Mixtral" in config.architectures[0]: config._attn_implementation = "flash_attention_2" model = custom_models[config.architectures[0]](config) else: model = AutoModelForCausalLM.from_config( config, trust_remote_code=True, attn_implementation="flash_attention_2" ) if optimize_rule_path is None: if config.architectures[0] in default_optimize_rules: print("using default_optimize_rule for", config.architectures[0]) optimize_rule_path = default_optimize_rules[config.architectures[0]] else: raise ValueError("No default optimize rule found. Please provide a rule file.") if gguf_path is None: raise ValueError("Please provide a gguf file path.") optimize_and_load_gguf(model, optimize_rule_path, gguf_path, config) try: model.generation_config = GenerationConfig.from_pretrained(model_path) except: gen_config = GenerationConfig( max_length=128, temperature=0.7, top_p=0.9, do_sample=True ) model.generation_config = gen_config if model.generation_config.pad_token_id is None: model.generation_config.pad_token_id = model.generation_config.eos_token_id model.eval() return model, tokenizer # 创建FastAPI应用 app = FastAPI() # 全局初始化模型和分词器 model, tokenizer = None, None @app.on_event("startup") async def startup_event(): global model, tokenizer # 这里填入你的模型路径和gguf路径 model_path = "/data/llm_tuning/dpsk/R1-Q4-config" gguf_path = "/data/llm_tuning/dpsk/DeepSeek-R1-GGUF" optimize_rule_path = None max_new_tokens = 2000 cpu_infer = 65 model, tokenizer = init_model( model_path, optimize_rule_path, gguf_path, max_new_tokens, cpu_infer, ) @app.post("/generate") async def chat(prompt: str): messages = [{"role": "user", "content": prompt}] input_tensor = tokenizer.apply_chat_template( messages, add_generation_prompt=True, return_tensors="pt" ) max_new_tokens = 2000 use_cuda_graph = True mode = "normal" force_think = False generated = prefill_and_generate( model, tokenizer, input_tensor.cuda(), max_new_tokens, use_cuda_graph, mode, force_think ) result = tokenizer.decode(generated[0], skip_special_tokens=True) return {"result": result} if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=50711)

bsakaguchi commented

2026-10-29 14:36:29 +00:00

Author

Owner

@git-shan commented on GitHub (Mar 13, 2025):

0.2.3 问题还在

:51735 - "POST /v1/chat/completions HTTP/1.1" 422 Unprocessable Entity

@git-shan commented on GitHub (Mar 13, 2025): 0.2.3 问题还在 :51735 - "POST /v1/chat/completions HTTP/1.1" 422 Unprocessable Entity

bsakaguchi commented

2026-10-29 14:36:30 +00:00

Author

Owner

@knilink commented on GitHub (Apr 15, 2025):

应该是API兼容性问题,试试开启兼容Legacy API

新格式应该是为了图片支持messages.content从string改成了{type: "text", text: string}

@knilink commented on GitHub (Apr 15, 2025): 应该是API兼容性问题,试试开启兼容Legacy API 新格式应该是为了图片支持messages.content从`string`改成了`{type: "text", text: string}` similar issue https://github.com/kvcache-ai/ktransformers/issues/861 ![Image](https://github.com/user-attachments/assets/ccfbed9a-e390-449e-8351-0a3495d4d5dd)