fix template for tiktoken

Former-commit-id: 8328447f81eb5b90310df08cf2928c83ef6355fe
2025-10-14 23:58:11 +08:00 · 2023-08-05 13:42:42 +08:00 · 2023-08-05 13:42:42 +08:00 · fdbb2c5378
commit fdbb2c5378
parent 3c0aaf42af
1 changed files with 1 additions and 0 deletions
--- a/src/llmtuner/extras/template.py
+++ b/src/llmtuner/extras/template.py
@ -102,6 +102,7 @@ class Template:
        for elem in context:
            if isinstance(elem, str):
                elem = elem.replace("{{query}}", query, 1)
+                elem = elem.replace("<mask>", "[MASK]")
                token_ids = token_ids + tokenizer.encode(elem, add_special_tokens=False)
            elif isinstance(elem, dict):
                token_ids = token_ids + [tokenizer.convert_tokens_to_ids(elem.get("token"))]