423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-05 21:22:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix #1032

Browse Source 
Former-commit-id: a41e12de5e3cd371cff87ab90917b36367094b31
This commit is contained in:
hiyouga 2023-09-27 22:42:16 +08:00
parent 927ff702ff
commit 772a1b16a1
2 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/llmtuner/api/app.py
View File

@ -68,7 +68,11 @@ def create_app(chat_model: ChatModel) -> FastAPI:
return EventSourceResponse(generate, media_type="text/event-stream") return EventSourceResponse(generate, media_type="text/event-stream")
response, (prompt_length, response_length) = chat_model.chat( response, (prompt_length, response_length) = chat_model.chat(
query, history, system, temperature=request.temperature, top_p=request.top_p, max_new_tokens=request.max_tokens query, history, system,
do_sample=request.do_sample,
temperature=request.temperature,
top_p=request.top_p,
max_new_tokens=request.max_tokens
) )
usage = ChatCompletionResponseUsage( usage = ChatCompletionResponseUsage(
@ -95,7 +99,11 @@ def create_app(chat_model: ChatModel) -> FastAPI:
yield chunk.json(exclude_unset=True, ensure_ascii=False) yield chunk.json(exclude_unset=True, ensure_ascii=False)
for new_text in chat_model.stream_chat( for new_text in chat_model.stream_chat(
query, history, system, temperature=request.temperature, top_p=request.top_p, max_new_tokens=request.max_tokens query, history, system,
do_sample=request.do_sample,
temperature=request.temperature,
top_p=request.top_p,
max_new_tokens=request.max_tokens
): ):
if len(new_text) == 0: if len(new_text) == 0:
continue continue

1
src/llmtuner/api/protocol.py
View File

@ -43,6 +43,7 @@ class DeltaMessage(BaseModel):
class ChatCompletionRequest(BaseModel): class ChatCompletionRequest(BaseModel):
model: str model: str
messages: List[ChatMessage] messages: List[ChatMessage]
do_sample: Optional[bool] = True
temperature: Optional[float] = None temperature: Optional[float] = None
top_p: Optional[float] = None top_p: Optional[float] = None
n: Optional[int] = 1 n: Optional[int] = 1