423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-22 22:02:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Slim Orca data parsing

Browse Source 
Former-commit-id: 6703d0546d02ea8174552ad04cb0b4bbfd7e0201
This commit is contained in:
Mark Mueller 2024-02-08 17:56:18 +01:00
parent 08967e324a
commit 8e36c5144b
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/llmtuner/data/aligner.py
View File

@ -60,6 +60,7 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr"
if dataset_attr.system_tag and message[dataset_attr.role_tag] == dataset_attr.system_tag: if dataset_attr.system_tag and message[dataset_attr.role_tag] == dataset_attr.system_tag:
outputs["system"].append(message[dataset_attr.content_tag]) outputs["system"].append(message[dataset_attr.content_tag])
n_sys = 1 n_sys = 1
continue
if (turn_idx - n_sys) % 2 == 0: if (turn_idx - n_sys) % 2 == 0:
accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag] accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag]