From 842b56666a594ff90e784262f7dc9dd113055219 Mon Sep 17 00:00:00 2001 From: Mark Mueller Date: Thu, 8 Feb 2024 08:28:32 -0800 Subject: [PATCH 1/5] SlimOrca aligner Former-commit-id: 36d7a7596688d40524ca91f662acb391e7ce1817 --- src/llmtuner/data/aligner.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/llmtuner/data/aligner.py b/src/llmtuner/data/aligner.py index 8144141c..5140f9d8 100644 --- a/src/llmtuner/data/aligner.py +++ b/src/llmtuner/data/aligner.py @@ -53,28 +53,32 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr" if len(messages) == 0: continue + n_sys = 0 prompt = [] response = [] for turn_idx, message in enumerate(messages): - if turn_idx % 2 == 0: - accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag] - else: - accept_tags = [dataset_attr.assistant_tag, dataset_attr.function_tag] + accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag, dataset_attr.assistant_tag, dataset_attr.function_tag] - if message[dataset_attr.role_tag] not in accept_tags: + if message[dataset_attr.role_tag] == "system": + outputs["system"].append(message[dataset_attr.content_tag]) + n_sys += 1 + elif message[dataset_attr.role_tag] not in accept_tags: + print("sytem attr", dataset_attr.system) + print("accepted tags", accept_tags) raise ValueError("Invalid role tag in {}.".format(messages)) - - prompt.append( - {"role": tag_mapping[message[dataset_attr.role_tag]], "content": message[dataset_attr.content_tag]} - ) + else: + prompt.append( + {"role": tag_mapping[message[dataset_attr.role_tag]], "content": message[dataset_attr.content_tag]} + ) last_message = prompt.pop(-1) response.append(last_message) outputs["prompt"].append(prompt) outputs["response"].append(response) - outputs["system"].append(examples[dataset_attr.system][i] if dataset_attr.system else "") + if n_sys == 0: + outputs["system"].append(examples[dataset_attr.system][i] if dataset_attr.system else "") outputs["tools"].append(examples[dataset_attr.tools][i] if dataset_attr.tools else "") - + assert n_sys <= 1 return outputs From 16d0bf031719980ecf0f225220779f1315e1d343 Mon Sep 17 00:00:00 2001 From: Mark Mueller Date: Thu, 8 Feb 2024 17:52:36 +0100 Subject: [PATCH 2/5] Slim Orca data parsing Former-commit-id: 8bd41826092f0ac46ad51f91928e88e5964ae1a5 --- src/llmtuner/data/aligner.py | 30 ++++++++++++++++-------------- src/llmtuner/data/parser.py | 6 +++++- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/llmtuner/data/aligner.py b/src/llmtuner/data/aligner.py index 5140f9d8..070a917c 100644 --- a/src/llmtuner/data/aligner.py +++ b/src/llmtuner/data/aligner.py @@ -53,32 +53,34 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr" if len(messages) == 0: continue - n_sys = 0 prompt = [] response = [] + n_sys = 0 for turn_idx, message in enumerate(messages): - accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag, dataset_attr.assistant_tag, dataset_attr.function_tag] - - if message[dataset_attr.role_tag] == "system": + if dataset_attr.system_tag and message[dataset_attr.role_tag] == dataset_attr.system_tag: outputs["system"].append(message[dataset_attr.content_tag]) - n_sys += 1 - elif message[dataset_attr.role_tag] not in accept_tags: - print("sytem attr", dataset_attr.system) - print("accepted tags", accept_tags) - raise ValueError("Invalid role tag in {}.".format(messages)) + n_sys = 1 + + if (turn_idx - n_sys) % 2 == 0: + accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag] else: - prompt.append( - {"role": tag_mapping[message[dataset_attr.role_tag]], "content": message[dataset_attr.content_tag]} - ) + accept_tags = [dataset_attr.assistant_tag, dataset_attr.function_tag] + + if message[dataset_attr.role_tag] not in accept_tags: + raise ValueError("Invalid role tag in {}.".format(messages)) + + prompt.append( + {"role": tag_mapping[message[dataset_attr.role_tag]], "content": message[dataset_attr.content_tag]} + ) last_message = prompt.pop(-1) response.append(last_message) outputs["prompt"].append(prompt) outputs["response"].append(response) - if n_sys == 0: + if not dataset_attr.system_tag: outputs["system"].append(examples[dataset_attr.system][i] if dataset_attr.system else "") outputs["tools"].append(examples[dataset_attr.tools][i] if dataset_attr.tools else "") - assert n_sys <= 1 + return outputs diff --git a/src/llmtuner/data/parser.py b/src/llmtuner/data/parser.py index 474aad83..aa6319fd 100644 --- a/src/llmtuner/data/parser.py +++ b/src/llmtuner/data/parser.py @@ -37,6 +37,10 @@ class DatasetAttr: assistant_tag: Optional[str] = "gpt" observation_tag: Optional[str] = "observation" function_tag: Optional[str] = "function_call" + system_tag: Optional[str] = None + + assert system_tag is None or system is None, f"Can not provide both system message (system_tag={system_tag}) and system column(system={system})" + def __repr__(self) -> str: return self.dataset_name @@ -95,7 +99,7 @@ def get_dataset_list(data_args: "DataArguments") -> List["DatasetAttr"]: setattr(dataset_attr, column_name, dataset_info[name]["columns"].get(column_name, None)) if dataset_attr.formatting == "sharegpt" and "tags" in dataset_info[name]: - for tag in ["role_tag", "content_tag", "user_tag", "assistant_tag", "observation_tag", "function_tag"]: + for tag in ["role_tag", "content_tag", "user_tag", "assistant_tag", "observation_tag", "function_tag", "system_tag"]: setattr(dataset_attr, tag, dataset_info[name]["tags"].get(tag, None)) dataset_list.append(dataset_attr) From 08967e324a68e1d7aad380b69cac4ab65018bc26 Mon Sep 17 00:00:00 2001 From: Mark Mueller Date: Thu, 8 Feb 2024 17:54:18 +0100 Subject: [PATCH 3/5] Slim Orca data parsing Former-commit-id: 7f792dfedebac8ed46eea935fed296bdb50d5f44 --- src/llmtuner/data/aligner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llmtuner/data/aligner.py b/src/llmtuner/data/aligner.py index 070a917c..d4f281bc 100644 --- a/src/llmtuner/data/aligner.py +++ b/src/llmtuner/data/aligner.py @@ -77,7 +77,7 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr" response.append(last_message) outputs["prompt"].append(prompt) outputs["response"].append(response) - if not dataset_attr.system_tag: + if n_sys == 0: outputs["system"].append(examples[dataset_attr.system][i] if dataset_attr.system else "") outputs["tools"].append(examples[dataset_attr.tools][i] if dataset_attr.tools else "") From 8e36c5144b08a9a02438e1557ea70661a6f82c6a Mon Sep 17 00:00:00 2001 From: Mark Mueller Date: Thu, 8 Feb 2024 17:56:18 +0100 Subject: [PATCH 4/5] Slim Orca data parsing Former-commit-id: 6703d0546d02ea8174552ad04cb0b4bbfd7e0201 --- src/llmtuner/data/aligner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llmtuner/data/aligner.py b/src/llmtuner/data/aligner.py index d4f281bc..cd3a7ea4 100644 --- a/src/llmtuner/data/aligner.py +++ b/src/llmtuner/data/aligner.py @@ -60,6 +60,7 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr" if dataset_attr.system_tag and message[dataset_attr.role_tag] == dataset_attr.system_tag: outputs["system"].append(message[dataset_attr.content_tag]) n_sys = 1 + continue if (turn_idx - n_sys) % 2 == 0: accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag] From 4bd7b8375e18dd00c9c91b0a2c39593badb89287 Mon Sep 17 00:00:00 2001 From: Mark Mueller Date: Thu, 8 Feb 2024 19:32:20 +0100 Subject: [PATCH 5/5] Slim Orca data parsing Former-commit-id: 1d3598afa10797ba0ce30d44f52e7994587c0ce8 --- data/README.md | 3 ++- data/dataset_info.json | 14 ++++++++++++++ src/llmtuner/data/aligner.py | 7 +++++-- src/llmtuner/data/utils.py | 1 + 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/data/README.md b/data/README.md index 3d950e1b..bb38935c 100644 --- a/data/README.md +++ b/data/README.md @@ -26,7 +26,8 @@ If you are using a custom dataset, please provide your dataset definition in the "user_tag": "the value of the role_tag represents the user. (default: human)", "assistant_tag": "the value of the role_tag represents the assistant. (default: gpt)", "observation_tag": "the value of the role_tag represents the tool results. (default: observation)", - "function_tag": "the value of the role_tag represents the function call. (default: function_call)" + "function_tag": "the value of the role_tag represents the function call. (default: function_call)", + "system_tag": "the value of the role_tag represents the system prompt. (default: None) incompatible with system column" } } ``` diff --git a/data/dataset_info.json b/data/dataset_info.json index 9b69f9a1..22dd11e6 100644 --- a/data/dataset_info.json +++ b/data/dataset_info.json @@ -126,6 +126,20 @@ "system": "system_prompt" } }, + "slimorca": { + "hf_hub_url": "Open-Orca/SlimOrca", + "formatting": "sharegpt", + "columns": { + "messages": "conversations" + }, + "tags": { + "role_tag": "from", + "content_tag": "value", + "user_tag": "human", + "assistant_tag": "gpt", + "system_tag": "system" + } + }, "intel_orca_dpo_pairs_de" : { "hf_hub_url": "mayflowergmbh/intel_orca_dpo_pairs_de", "ranking": true diff --git a/src/llmtuner/data/aligner.py b/src/llmtuner/data/aligner.py index cd3a7ea4..34c8980d 100644 --- a/src/llmtuner/data/aligner.py +++ b/src/llmtuner/data/aligner.py @@ -47,10 +47,10 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr" dataset_attr.assistant_tag: Role.ASSISTANT, dataset_attr.observation_tag: Role.OBSERVATION, dataset_attr.function_tag: Role.FUNCTION, + dataset_attr.system_tag: Role.SYSTEM, } for i, messages in enumerate(examples[dataset_attr.messages]): - messages = messages[: len(messages) // 2 * 2] # should be multiples of 2 - if len(messages) == 0: + if len(messages) <= 1: continue prompt = [] @@ -74,6 +74,9 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr" {"role": tag_mapping[message[dataset_attr.role_tag]], "content": message[dataset_attr.content_tag]} ) + if len(prompt) % 2 == 1: + # Last message was neither from assistant nor function + prompt.pop(-1) last_message = prompt.pop(-1) response.append(last_message) outputs["prompt"].append(prompt) diff --git a/src/llmtuner/data/utils.py b/src/llmtuner/data/utils.py index 062d390f..75a28c59 100644 --- a/src/llmtuner/data/utils.py +++ b/src/llmtuner/data/utils.py @@ -21,6 +21,7 @@ class Role(str, Enum): ASSISTANT = "assistant" OBSERVATION = "observation" FUNCTION = "function" + SYSTEM = "system" def checksum(data_files: List[str], file_sha1: Optional[str] = None) -> None: