423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 11:50:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix inference, add prompt template

Browse Source
This commit is contained in:
hiyouga
2023-06-07 10:52:35 +08:00
parent 13d1f0709c
commit 5d021d4ad5
6 changed files with 44 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/cli_demo.py
View File

@@ -6,7 +6,9 @@
from utils import (
load_pretrained,
prepare_infer_args,
get_logits_processor
get_logits_processor,
prompt_template_alpaca,
prompt_template_ziya
)
from threading import Thread
from transformers import TextIteratorStreamer
@@ -18,23 +20,7 @@ def main():
model_name = "BLOOM" if "bloom" in model_args.model_name_or_path else "LLaMA"
model, tokenizer = load_pretrained(model_args, finetuning_args)
def format_example_alpaca(query, history):
prompt = "Below is an instruction that describes a task. "
prompt += "Write a response that appropriately completes the request.\n"
prompt += "Instruction:\n"
for old_query, response in history:
prompt += "Human: {}\nAssistant: {}\n".format(old_query, response)
prompt += "Human: {}\nAssistant:".format(query)
return prompt
def format_example_ziya(query, history):
prompt = ""
for old_query, response in history:
prompt += "<human>: {}\n<bot>: {}\n".format(old_query, response)
prompt += "<human>: {}\n<bot>:".format(query)
return prompt
format_example = format_example_alpaca if data_args.prompt_template == "alpaca" else format_example_ziya
format_example = prompt_template_alpaca if data_args.prompt_template == "alpaca" else prompt_template_ziya
streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
def predict_and_print(query, history: list):