423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-14 23:58:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix dataset

Browse Source 
Former-commit-id: d4a2d299414984a4043d30034c5c95e2d717a49e
This commit is contained in:
hiyouga 2024-11-27 06:14:51 +00:00
parent 6ea42d5b63
commit 877e2ea791
1 changed files with 20 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/llamafactory/data/template.py
View File

@ -512,6 +512,7 @@ _register_template(
) )
# copied from chatml template
_register_template( _register_template(
name="chatml_de", name="chatml_de",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
@ -578,6 +579,7 @@ _register_template(
) )
# copied from chatml template
_register_template( _register_template(
name="dbrx", name="dbrx",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
@ -728,6 +730,7 @@ _register_template(
) )
# copied from llama2 template
_register_template( _register_template(
name="llama2_zh", name="llama2_zh",
format_user=StringFormatter(slots=[{"bos_token"}, "[INST] {{content}} [/INST]"]), format_user=StringFormatter(slots=[{"bos_token"}, "[INST] {{content}} [/INST]"]),
@ -762,6 +765,7 @@ _register_template(
) )
# copied from llama3 template
_register_template( _register_template(
name="mllama", name="mllama",
format_user=StringFormatter( format_user=StringFormatter(
@ -789,6 +793,7 @@ _register_template(
) )
# copied from vicuna template
_register_template( _register_template(
name="llava", name="llava",
format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]), format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
@ -800,6 +805,7 @@ _register_template(
) )
# copied from vicuna template
_register_template( _register_template(
name="llava_next", name="llava_next",
format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]), format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
@ -811,6 +817,7 @@ _register_template(
) )
# copied from llama3 template
_register_template( _register_template(
name="llava_next_llama3", name="llava_next_llama3",
format_user=StringFormatter( format_user=StringFormatter(
@ -838,6 +845,7 @@ _register_template(
) )
# copied from mistral template
_register_template( _register_template(
name="llava_next_mistral", name="llava_next_mistral",
format_user=StringFormatter(slots=["[INST] {{content}} [/INST]"]), format_user=StringFormatter(slots=["[INST] {{content}} [/INST]"]),
@ -846,6 +854,7 @@ _register_template(
) )
# copied from chatml template
_register_template( _register_template(
name="llava_next_qwen", name="llava_next_qwen",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
@ -860,6 +869,7 @@ _register_template(
) )
# copied from chatml template
_register_template( _register_template(
name="llava_next_yi", name="llava_next_yi",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
@ -871,6 +881,7 @@ _register_template(
) )
# copied from vicuna template
_register_template( _register_template(
name="llava_next_video", name="llava_next_video",
format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]), format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
@ -882,6 +893,7 @@ _register_template(
) )
# copied from mistral template
_register_template( _register_template(
name="llava_next_video_mistral", name="llava_next_video_mistral",
format_user=StringFormatter(slots=["[INST] {{content}} [/INST]"]), format_user=StringFormatter(slots=["[INST] {{content}} [/INST]"]),
@ -890,6 +902,7 @@ _register_template(
) )
# copied from chatml template
_register_template( _register_template(
name="llava_next_video_yi", name="llava_next_video_yi",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
@ -901,6 +914,7 @@ _register_template(
) )
# copied from chatml template
_register_template( _register_template(
name="marco", name="marco",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
@ -954,6 +968,7 @@ _register_template(
) )
# copied from chatml template
_register_template( _register_template(
name="opencoder", name="opencoder",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
@ -974,6 +989,7 @@ _register_template(
) )
# copied from gemma template
_register_template( _register_template(
name="paligemma", name="paligemma",
format_user=StringFormatter(slots=["<start_of_turn>user\n{{content}}<end_of_turn>\n<start_of_turn>model\n"]), format_user=StringFormatter(slots=["<start_of_turn>user\n{{content}}<end_of_turn>\n<start_of_turn>model\n"]),
@ -1017,6 +1033,7 @@ _register_template(
) )
# copied from chatml template
_register_template( _register_template(
name="qwen", name="qwen",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
@ -1030,6 +1047,7 @@ _register_template(
) )
# copied from chatml template
_register_template( _register_template(
name="qwen2_vl", name="qwen2_vl",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
@ -1058,6 +1076,7 @@ _register_template(
) )
# copied from llama3 template
_register_template( _register_template(
name="skywork_o1", name="skywork_o1",
format_user=StringFormatter( format_user=StringFormatter(
@ -1175,6 +1194,7 @@ _register_template(
) )
# copied from chatml template
_register_template( _register_template(
name="yi", name="yi",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),