423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-16 00:28:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Slim Orca data parsing

Browse Source 
Former-commit-id: ca57d27c39d4e7bc3dd7c3207a23d23d2cbd446b
This commit is contained in:
Mark Mueller 2024-02-08 17:56:18 +01:00
parent 5788b7c7d0
commit 4d473894fd
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/llmtuner/data/aligner.py
View File

@ -60,6 +60,7 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr"
if dataset_attr.system_tag and message[dataset_attr.role_tag] == dataset_attr.system_tag: if dataset_attr.system_tag and message[dataset_attr.role_tag] == dataset_attr.system_tag:
outputs["system"].append(message[dataset_attr.content_tag]) outputs["system"].append(message[dataset_attr.content_tag])
n_sys = 1 n_sys = 1
continue
if (turn_idx - n_sys) % 2 == 0: if (turn_idx - n_sys) % 2 == 0:
accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag] accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag]