mirror of
https://github.com/hiyouga/LLaMA-Factory.git
synced 2025-08-04 04:32:50 +08:00
Merge pull request #2423 from mayflower/main
Support for german sft and dpo Former-commit-id: d0daaa01f9119d4feef45327bbd43964a73bc0c2
This commit is contained in:
commit
186ba72d72
12
README.md
12
README.md
@ -188,7 +188,16 @@ Please refer to [constants.py](src/llmtuner/extras/constants.py) for a full list
|
||||
- [LMSYS Chat 1M (en)](https://huggingface.co/datasets/lmsys/lmsys-chat-1m)
|
||||
- [Evol Instruct V2 (en)](https://huggingface.co/datasets/WizardLM/WizardLM_evol_instruct_V2_196k)
|
||||
- [Glaive Function Calling V2 (en)](https://huggingface.co/datasets/glaiveai/glaive-function-calling-v2)
|
||||
|
||||
- [Open Assistant (de)](https://huggingface.co/datasets/mayflowergmbh/oasst_de)
|
||||
- [Dolly 15k (de)](https://huggingface.co/datasets/mayflowergmbh/dolly-15k_de)
|
||||
- [FreedomIntelligence Alpaca GPT4 (de)](https://huggingface.co/datasets/mayflowergmbh/alpaca-gpt4_de)
|
||||
- [LeoLM/OpenSchnabeltier (de)](https://huggingface.co/datasets/mayflowergmbh/openschnabeltier_de)
|
||||
- [FreedomIntelligence/evol-instruct-deutsch (de)](https://huggingface.co/datasets/mayflowergmbh/evol-instruct_de)
|
||||
- [wiki_qa (de)](https://huggingface.co/datasets/wiki_qa)
|
||||
- [cognitivecomputations/dolphin (de)](https://huggingface.co/datasets/mayflowergmbh/dolphin_de)
|
||||
- [booksum (de)](https://huggingface.co/datasets/mayflowergmbh/booksum_de)
|
||||
- [jondurbin/airoboros-3.0 (de)](https://huggingface.co/datasets/mayflowergmbh/airoboros-3.0_de)
|
||||
- [stingning/ultrachat (de)](https://huggingface.co/datasets/mayflowergmbh/ultra-chat_de)
|
||||
</details>
|
||||
|
||||
<details><summary>Preference datasets</summary>
|
||||
@ -197,6 +206,7 @@ Please refer to [constants.py](src/llmtuner/extras/constants.py) for a full list
|
||||
- [Open Assistant (multilingual)](https://huggingface.co/datasets/OpenAssistant/oasst1)
|
||||
- [GPT-4 Generated Data (en&zh)](https://github.com/Instruction-Tuning-with-GPT-4/GPT-4-LLM)
|
||||
- [Nectar (en)](https://huggingface.co/datasets/berkeley-nest/Nectar)
|
||||
- [Intel/orca_dpo_pairs (de)](https://huggingface.co/datasets/mayflowergmbh/intel_orca_dpo_pairs_de)
|
||||
|
||||
</details>
|
||||
|
||||
|
@ -15,6 +15,9 @@
|
||||
"file_name": "alpaca_gpt4_data_zh.json",
|
||||
"file_sha1": "3eaa3bda364ccdd59925d7448a698256c31ef845"
|
||||
},
|
||||
"alpaca-gpt4_de": {
|
||||
"hf_hub_url": "mayflowergmbh/alpaca-gpt4_de"
|
||||
},
|
||||
"self_cognition": {
|
||||
"file_name": "self_cognition.json",
|
||||
"file_sha1": "6287a730ada924fc5d9eadc6d8f865e01b7a6f67"
|
||||
@ -39,6 +42,9 @@
|
||||
"history": "history"
|
||||
}
|
||||
},
|
||||
"oasst_de": {
|
||||
"hf_hub_url": "mayflowergmbh/oasst_de"
|
||||
},
|
||||
"lima": {
|
||||
"file_name": "lima.json",
|
||||
"file_sha1": "9db59f6b7007dc4b17529fc63379b9cd61640f37",
|
||||
@ -120,6 +126,31 @@
|
||||
"system": "system_prompt"
|
||||
}
|
||||
},
|
||||
"intel_orca_dpo_pairs_de" : {
|
||||
"hf_hub_url": "mayflowergmbh/intel_orca_dpo_pairs_de",
|
||||
"ranking": true
|
||||
},
|
||||
"airoboros-3.0_de": {
|
||||
"hf_hub_url": "mayflowergmbh/airoboros-3.0_de"
|
||||
},
|
||||
"booksum_de": {
|
||||
"hf_hub_url": "mayflowergmbh/booksum_de"
|
||||
},
|
||||
"dolphin_de": {
|
||||
"hf_hub_url": "mayflowergmbh/dolphin_de"
|
||||
},
|
||||
"wiki_qa_de": {
|
||||
"hf_hub_url": "mayflowergmbh/wiki_qa_de"
|
||||
},
|
||||
"evol-instruct_de": {
|
||||
"hf_hub_url": "mayflowergmbh/evol-instruct_de"
|
||||
},
|
||||
"openschnabeltier_de": {
|
||||
"hf_hub_url": "mayflowergmbh/openschnabeltier_de"
|
||||
},
|
||||
"dolly-15k_de": {
|
||||
"hf_hub_url": "mayflowergmbh/dolly-15k_de"
|
||||
},
|
||||
"mathinstruct": {
|
||||
"hf_hub_url": "TIGER-Lab/MathInstruct",
|
||||
"ms_hub_url": "AI-ModelScope/MathInstruct",
|
||||
@ -184,6 +215,9 @@
|
||||
},
|
||||
"formatting": "sharegpt"
|
||||
},
|
||||
"ultrachat_chat_de": {
|
||||
"hf_hub_url": "mayflowergmbh/ultra-chat_de"
|
||||
},
|
||||
"agent_instruct": {
|
||||
"hf_hub_url": "THUDM/AgentInstruct",
|
||||
"ms_hub_url": "ZhipuAI/AgentInstruct",
|
||||
|
@ -515,6 +515,16 @@ register_template(
|
||||
|
||||
register_template(name="vanilla")
|
||||
|
||||
register_template(
|
||||
name="chatml_de",
|
||||
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
|
||||
format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
|
||||
format_separator=EmptyFormatter(slots=["\n"]),
|
||||
default_system="Du bist ein freundlicher und hilfsbereiter KI-Assistent.",
|
||||
stop_words=["<|im_end|>"],
|
||||
replace_eos=True,
|
||||
)
|
||||
|
||||
|
||||
register_template(
|
||||
name="vicuna",
|
||||
|
Loading…
x
Reference in New Issue
Block a user