423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-01 11:12:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

[webui] add infer extra args (#8167)

Browse Source
This commit is contained in:
hoshi-hiyouga 2025-05-27 12:04:00 +08:00 committed by GitHub
parent 09436c1f45
commit 16e1a5097e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/llamafactory/webui/chatter.py
View File

@ -114,6 +114,11 @@ class WebChatModel(ChatModel):
elif self.demo_mode: elif self.demo_mode:
error = ALERTS["err_demo"][lang] error = ALERTS["err_demo"][lang]
try:
json.loads(get("infer.extra_args"))
except json.JSONDecodeError:
error = ALERTS["err_json_schema"][lang]
if error: if error:
gr.Warning(error) gr.Warning(error)
yield error yield error
@ -131,9 +136,9 @@ class WebChatModel(ChatModel):
enable_liger_kernel=(get("top.booster") == "liger_kernel"), enable_liger_kernel=(get("top.booster") == "liger_kernel"),
infer_backend=get("infer.infer_backend"), infer_backend=get("infer.infer_backend"),
infer_dtype=get("infer.infer_dtype"), infer_dtype=get("infer.infer_dtype"),
vllm_enforce_eager=True,
trust_remote_code=True, trust_remote_code=True,
) )
args.update(json.loads(get("infer.extra_args")))
# checkpoints # checkpoints
if checkpoint_path: if checkpoint_path:

4
src/llamafactory/webui/components/infer.py
View File

@ -36,6 +36,7 @@ def create_infer_tab(engine: "Engine") -> dict[str, "Component"]:
with gr.Row(): with gr.Row():
infer_backend = gr.Dropdown(choices=["huggingface", "vllm", "sglang"], value="huggingface") infer_backend = gr.Dropdown(choices=["huggingface", "vllm", "sglang"], value="huggingface")
infer_dtype = gr.Dropdown(choices=["auto", "float16", "bfloat16", "float32"], value="auto") infer_dtype = gr.Dropdown(choices=["auto", "float16", "bfloat16", "float32"], value="auto")
extra_args = gr.Textbox(value='{"vllm_enforce_eager": true}')
with gr.Row(): with gr.Row():
load_btn = gr.Button() load_btn = gr.Button()
@ -43,11 +44,12 @@ def create_infer_tab(engine: "Engine") -> dict[str, "Component"]:
info_box = gr.Textbox(show_label=False, interactive=False) info_box = gr.Textbox(show_label=False, interactive=False)
input_elems.update({infer_backend, infer_dtype}) input_elems.update({infer_backend, infer_dtype, extra_args})
elem_dict.update( elem_dict.update(
dict( dict(
infer_backend=infer_backend, infer_backend=infer_backend,
infer_dtype=infer_dtype, infer_dtype=infer_dtype,
extra_args=extra_args,
load_btn=load_btn, load_btn=load_btn,
unload_btn=unload_btn, unload_btn=unload_btn,
info_box=info_box, info_box=info_box,