Merge branch 'hiyouga:main' into main

Former-commit-id: 5318cc650e59788be21da5d00ebdd2f7c75c4f29
This commit is contained in:
Johann-Peter Hartmann 2024-02-03 08:43:12 +01:00 committed by GitHub
commit 21871df1ad

View File

@ -117,9 +117,9 @@ class Template:
elif isinstance(elem, dict):
token_ids += [tokenizer.convert_tokens_to_ids(elem.get("token"))]
elif isinstance(elem, set):
if "bos_token" in elem and tokenizer.bos_token_id:
if "bos_token" in elem and tokenizer.bos_token_id is not None:
token_ids += [tokenizer.bos_token_id]
elif "eos_token" in elem and tokenizer.eos_token_id:
elif "eos_token" in elem and tokenizer.eos_token_id is not None:
token_ids += [tokenizer.eos_token_id]
else:
raise ValueError("Input must be string, set[str] or dict[str, str], got {}".format(type(elem)))