423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-18 04:40:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge data part to the text stream

Browse Source
This commit is contained in:
BUAADreamer
2024-04-25 19:19:59 +08:00
parent 838eb87a96
commit c6dd89918f
15 changed files with 828 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
src/llmtuner/data/aligner.py
View File

@@ -13,7 +13,9 @@ if TYPE_CHECKING:
from .parser import DatasetAttr
def convert_alpaca(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr") -> Dict[str, List[Any]]:
def convert_alpaca(
examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr"
) -> Dict[str, List[Any]]:
outputs = {"prompt": [], "response": [], "system": [], "tools": []}
for i in range(len(examples[dataset_attr.prompt])):
prompt = []
@@ -31,24 +33,38 @@ def convert_alpaca(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr")
prompt.append({"role": Role.USER.value, "content": "\n".join(content)})
if dataset_attr.response and isinstance(examples[dataset_attr.response][i], list):
if dataset_attr.response and isinstance(
examples[dataset_attr.response][i], list
):
response = [
{"role": Role.ASSISTANT.value, "content": content} for content in examples[dataset_attr.response][i]
{"role": Role.ASSISTANT.value, "content": content}
for content in examples[dataset_attr.response][i]
]
elif dataset_attr.response and isinstance(
examples[dataset_attr.response][i], str
):
response = [
{
"role": Role.ASSISTANT.value,
"content": examples[dataset_attr.response][i],
}
]
elif dataset_attr.response and isinstance(examples[dataset_attr.response][i], str):
response = [{"role": Role.ASSISTANT.value, "content": examples[dataset_attr.response][i]}]
else:
response = []
outputs["prompt"].append(prompt)
outputs["response"].append(response)
outputs["system"].append(examples[dataset_attr.system][i] if dataset_attr.system else "")
outputs["system"].append(
examples[dataset_attr.system][i] if dataset_attr.system else ""
)
outputs["tools"].append("")
outputs["images"].append([])
return outputs
def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr") -> Dict[str, List[Any]]:
def convert_sharegpt(
examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr"
) -> Dict[str, List[Any]]:
outputs = {"prompt": [], "response": [], "system": [], "tools": []}
tag_mapping = {
dataset_attr.user_tag: Role.USER.value,
@@ -61,7 +77,10 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr"
even_tags = (dataset_attr.assistant_tag, dataset_attr.function_tag)
accept_tags = (odd_tags, even_tags)
for i, messages in enumerate(examples[dataset_attr.messages]):
if dataset_attr.system_tag and messages[0][dataset_attr.role_tag] == dataset_attr.system_tag:
if (
dataset_attr.system_tag
and messages[0][dataset_attr.role_tag] == dataset_attr.system_tag
):
system = messages[0][dataset_attr.content_tag]
messages = messages[1:]
else:
@@ -77,19 +96,81 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr"
raise ValueError("Invalid role tag in {}.".format(messages))
aligned_messages.append(
{"role": tag_mapping[message[dataset_attr.role_tag]], "content": message[dataset_attr.content_tag]}
{
"role": tag_mapping[message[dataset_attr.role_tag]],
"content": message[dataset_attr.content_tag],
}
)
outputs["prompt"].append(aligned_messages[:-1])
outputs["response"].append(aligned_messages[-1:])
outputs["system"].append(system)
outputs["tools"].append(examples[dataset_attr.tools][i] if dataset_attr.tools else "")
outputs["tools"].append(
examples[dataset_attr.tools][i] if dataset_attr.tools else ""
)
outputs["images"].append([])
return outputs
def convert_llava(
examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr"
) -> Dict[str, List[Any]]:
outputs = {"prompt": [], "response": [], "system": [], "tools": [], "images": []}
tag_mapping = {
dataset_attr.user_tag: Role.USER.value,
dataset_attr.assistant_tag: Role.ASSISTANT.value,
dataset_attr.observation_tag: Role.OBSERVATION.value,
dataset_attr.function_tag: Role.FUNCTION.value,
dataset_attr.system_tag: Role.SYSTEM.value,
}
odd_tags = (dataset_attr.user_tag, dataset_attr.observation_tag)
even_tags = (dataset_attr.assistant_tag, dataset_attr.function_tag)
accept_tags = (odd_tags, even_tags)
for i, messages in enumerate(examples[dataset_attr.messages]):
if (
dataset_attr.system_tag
and messages[0][dataset_attr.role_tag] == dataset_attr.system_tag
):
system = messages[0][dataset_attr.content_tag]
messages = messages[1:]
else:
system = examples[dataset_attr.system][i] if dataset_attr.system else ""
messages = messages[: len(messages) // 2 * 2] # should be multiples of 2
if len(messages) == 0:
continue
aligned_messages = []
for turn_idx, message in enumerate(messages):
if message[dataset_attr.role_tag] not in accept_tags[turn_idx % 2]:
raise ValueError("Invalid role tag in {}.".format(messages))
aligned_messages.append(
{
"role": tag_mapping[message[dataset_attr.role_tag]],
"content": message[dataset_attr.content_tag],
}
)
outputs["prompt"].append(aligned_messages[:-1])
outputs["response"].append(aligned_messages[-1:])
outputs["system"].append(system)
outputs["tools"].append(
examples[dataset_attr.tools][i] if dataset_attr.tools else ""
)
print(examples[dataset_attr.images][i])
outputs["images"].append(
examples[dataset_attr.images][i] if dataset_attr.images else []
)
return outputs
def align_dataset(
dataset: Union["Dataset", "IterableDataset"], dataset_attr: "DatasetAttr", data_args: "DataArguments"
dataset: Union["Dataset", "IterableDataset"],
dataset_attr: "DatasetAttr",
data_args: "DataArguments",
) -> Union["Dataset", "IterableDataset"]:
r"""
Aligned dataset:
@@ -100,6 +181,8 @@ def align_dataset(
"""
if dataset_attr.formatting == "alpaca":
convert_func = partial(convert_alpaca, dataset_attr=dataset_attr)
elif dataset_attr.formatting == "llava":
convert_func = partial(convert_llava, dataset_attr=dataset_attr)
else:
convert_func = partial(convert_sharegpt, dataset_attr=dataset_attr)
@@ -107,13 +190,20 @@ def align_dataset(
features = Features.from_dict(
{
"prompt": [
{"role": {"dtype": "string", "_type": "Value"}, "content": {"dtype": "string", "_type": "Value"}}
{
"role": {"dtype": "string", "_type": "Value"},
"content": {"dtype": "string", "_type": "Value"},
}
],
"response": [
{"role": {"dtype": "string", "_type": "Value"}, "content": {"dtype": "string", "_type": "Value"}}
{
"role": {"dtype": "string", "_type": "Value"},
"content": {"dtype": "string", "_type": "Value"},
}
],
"system": {"dtype": "string", "_type": "Value"},
"tools": {"dtype": "string", "_type": "Value"},
"images": {"feature": {"_type": "Image"}, "_type": "Sequence"},
}
)
kwargs = {}