423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-22 22:02:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix template

Browse Source 
Former-commit-id: c183b3551d8d190965d2403e27fa2cedf8ac7bff
This commit is contained in:
hiyouga 2023-08-05 00:25:00 +08:00
parent 7a89fce4c7
commit ab95f569a4
1 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/extras/template.py
View File

@ -101,9 +101,7 @@ class Template:
token_ids = []
for elem in context:
if isinstance(elem, str):
subelems = elem.split("{{query}}")
if len(subelems) > 1:
elem = subelems[0] + query + subelems[1]
elem = elem.replace("{{query}}", query, 1)
token_ids = token_ids + tokenizer.encode(elem, add_special_tokens=False)
elif isinstance(elem, dict):
token_ids = token_ids + [tokenizer.convert_tokens_to_ids(elem.get("token"))]