423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-23 22:32:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix template for tiktoken

Browse Source 
Former-commit-id: 1afa51c2fa9839056644803eedef4e9d1af0d51e
This commit is contained in:
hiyouga 2023-08-05 13:42:42 +08:00
parent 1a1caf2116
commit 45af1a951f
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/llmtuner/extras/template.py
View File

@ -102,6 +102,7 @@ class Template:
for elem in context: for elem in context:
if isinstance(elem, str): if isinstance(elem, str):
elem = elem.replace("{{query}}", query, 1) elem = elem.replace("{{query}}", query, 1)
elem = elem.replace("<mask>", "[MASK]")
token_ids = token_ids + tokenizer.encode(elem, add_special_tokens=False) token_ids = token_ids + tokenizer.encode(elem, add_special_tokens=False)
elif isinstance(elem, dict): elif isinstance(elem, dict):
token_ids = token_ids + [tokenizer.convert_tokens_to_ids(elem.get("token"))] token_ids = token_ids + [tokenizer.convert_tokens_to_ids(elem.get("token"))]