From 6f0b412265b0077289247d6a177e14e6f987b1a1 Mon Sep 17 00:00:00 2001 From: khazic Date: Sun, 28 Apr 2024 14:27:45 +0800 Subject: [PATCH 1/5] added the second sharegpt format Former-commit-id: d1ba32e4bb70489a9e6f5d3657988c9b7553a157 --- data/README.md | 32 ++++++++++++++++++++++++++++---- data/README_zh.md | 26 +++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/data/README.md b/data/README.md index 6de0430f..9158233f 100644 --- a/data/README.md +++ b/data/README.md @@ -94,20 +94,44 @@ Remember to set `"ranking": true` for the preference datasets. The dataset in sharegpt format should follow the below format: ```json +# The first sharegpt format [ { "conversations": [ { "from": "human", - "value": "user instruction" + "value": "用户指令" }, { "from": "gpt", - "value": "model response" + "value": "模型回答" } ], - "system": "system prompt (optional)", - "tools": "tool description (optional)" + "system": "系统提示词（选填）", + "tools": "工具描述（选填）" + } +] + +# The second sharegpt format + +[ + { + "type": "chatml", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Tell me something about large language models." + }, + { + "role": "assistant", + "content": "Large language models are a type of language model ..." + } + ], + "source": "unknown" } ] ``` diff --git a/data/README_zh.md b/data/README_zh.md index fb6cb1d9..9abef5b6 100644 --- a/data/README_zh.md +++ b/data/README_zh.md @@ -37,7 +37,7 @@ ---- -该项目目前支持两种格式的数据集：**alpaca** 和 **sharegpt**，其中 alpaca 格式的数据集按照以下方式组织： +该项目目前支持三种格式的数据集：**alpaca** 和 **sharegpt**，其中 alpaca 格式的数据集按照以下方式组织： ```json [ @@ -94,6 +94,7 @@ 而 sharegpt 格式的数据集按照以下方式组织： ```json +# 第一种sharegpt格式 [ { "conversations": [ @@ -110,6 +111,29 @@ "tools": "工具描述（选填）" } ] + +# 第二种sharegpt格式 + +[ + { + "type": "chatml", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Tell me something about large language models." + }, + { + "role": "assistant", + "content": "Large language models are a type of language model ..." + } + ], + "source": "unknown" + } +] ``` 对于上述格式的数据，`dataset_info.json` 中的 `columns` 应为： From db316422a4806315f6bfb4159684b4326562a558 Mon Sep 17 00:00:00 2001 From: khazic Date: Sun, 28 Apr 2024 14:30:05 +0800 Subject: [PATCH 2/5] Upgrade the second sharegpt format Former-commit-id: 288911fc7b1e12e53f3396c371cf4b4c7300b4bf --- data/README_zh.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/README_zh.md b/data/README_zh.md index 9abef5b6..5a9db167 100644 --- a/data/README_zh.md +++ b/data/README_zh.md @@ -120,15 +120,15 @@ "messages": [ { "role": "system", - "content": "You are a helpful assistant." + "content": "你是一个很有用的AI助手" }, { "role": "user", - "content": "Tell me something about large language models." + "content": "告诉我一些关于大模型的一些信息" }, { "role": "assistant", - "content": "Large language models are a type of language model ..." + "content": "大模型是一种语言模型" } ], "source": "unknown" From f15836c77acffc3552f2d25502e871b41c6a60d4 Mon Sep 17 00:00:00 2001 From: Lao Date: Sun, 28 Apr 2024 23:31:37 +0800 Subject: [PATCH 3/5] Update README_zh.md Former-commit-id: ce17eccf451649728cf7b45312fd7f75d3a8a246 --- data/README_zh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/README_zh.md b/data/README_zh.md index 5a9db167..1fe98a9e 100644 --- a/data/README_zh.md +++ b/data/README_zh.md @@ -37,7 +37,7 @@ ---- -该项目目前支持三种格式的数据集：**alpaca** 和 **sharegpt**，其中 alpaca 格式的数据集按照以下方式组织： +该项目目前支持二种格式的数据集：**alpaca** 和 **sharegpt**，其中 alpaca 格式的数据集按照以下方式组织： ```json [ From 2186deceac4735547bc9d1fd76088eeead9ff2d6 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Thu, 2 May 2024 02:13:46 +0800 Subject: [PATCH 4/5] Update README.md Former-commit-id: b072ec9d1b18f7e9d5d2c9529eac55d29ca832c8 --- data/README.md | 145 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 99 insertions(+), 46 deletions(-) diff --git a/data/README.md b/data/README.md index 9158233f..012de4e7 100644 --- a/data/README.md +++ b/data/README.md @@ -1,4 +1,4 @@ -If you are using a custom dataset, please provide your dataset definition in the following format in `dataset_info.json`. +If you are using a custom dataset, please add your **dataset description** to `dataset_info.json` according to the following format. We also provide several examples in the next section. ```json "dataset_name": { @@ -33,7 +33,7 @@ If you are using a custom dataset, please provide your dataset definition in the } ``` -Given above, you can use the custom dataset via specifying `--dataset dataset_name`. +After that, you can load the custom dataset by specifying `--dataset dataset_name`. ---- @@ -54,10 +54,11 @@ Currently we support dataset in **alpaca** or **sharegpt** format, the dataset i ] ``` -Regarding the above dataset, the `columns` in `dataset_info.json` should be: +Regarding the above dataset, the description in `dataset_info.json` should be: ```json "dataset_name": { + "file_name": "data.json", "columns": { "prompt": "instruction", "query": "input", @@ -70,76 +71,86 @@ Regarding the above dataset, the `columns` in `dataset_info.json` should be: The `query` column will be concatenated with the `prompt` column and used as the user prompt, then the user prompt would be `prompt\nquery`. The `response` column represents the model response. -The `system` column will be used as the system prompt. The `history` column is a list consisting string tuples representing prompt-response pairs in the history. Note that the responses in the history **will also be used for training**. +The `system` column will be used as the system prompt. The `history` column is a list consisting string tuples representing prompt-response pairs in the history. Note that the responses in the history **will also be used for training** in supervised fine-tuning. -For the pre-training datasets, only the `prompt` column will be used for training. - -For the preference datasets, the `response` column should be a string list whose length is 2, with the preferred answers appearing first, for example: +For the **pre-training datasets**, only the `prompt` column will be used for training, for example: ```json -{ - "instruction": "user instruction", - "input": "user input", - "output": [ - "chosen answer", - "rejected answer" - ] +[ + {"text": "document"}, + {"text": "document"} +] +``` + +Regarding the above dataset, the description in `dataset_info.json` should be: + +```json +"dataset_name": { + "file_name": "data.json", + "columns": { + "prompt": "text" + } } ``` -Remember to set `"ranking": true` for the preference datasets. +For the **preference datasets**, the `response` column should be a string list whose length is 2, with the preferred answers appearing first, for example: + +```json +[ + { + "instruction": "user instruction", + "input": "user input", + "output": [ + "chosen answer", + "rejected answer" + ] + } +] +``` + +Regarding the above dataset, the description in `dataset_info.json` should be: + +```json +"dataset_name": { + "file_name": "data.json", + "ranking": true, + "columns": { + "prompt": "instruction", + "query": "input", + "response": "output", + } +} +``` ---- -The dataset in sharegpt format should follow the below format: +The dataset in **sharegpt** format should follow the below format: ```json -# The first sharegpt format [ { "conversations": [ { "from": "human", - "value": "用户指令" + "value": "user instruction" }, { "from": "gpt", - "value": "模型回答" + "value": "model response" } ], - "system": "系统提示词（选填）", - "tools": "工具描述（选填）" - } -] - -# The second sharegpt format - -[ - { - "type": "chatml", - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant." - }, - { - "role": "user", - "content": "Tell me something about large language models." - }, - { - "role": "assistant", - "content": "Large language models are a type of language model ..." - } - ], - "source": "unknown" + "system": "system prompt (optional)", + "tools": "tool description (optional)" } ] ``` -Regarding the above dataset, the `columns` in `dataset_info.json` should be: +Regarding the above dataset, the description in `dataset_info.json` should be: ```json "dataset_name": { + "file_name": "data.json", + "formatting": "sharegpt", "columns": { "messages": "conversations", "system": "system", @@ -156,4 +167,46 @@ Regarding the above dataset, the `columns` in `dataset_info.json` should be: where the `messages` column should be a list following the `u/a/u/a/u/a` order. -Pre-training datasets and preference datasets are incompatible with the sharegpt format yet. +We also supports the dataset in the **openai** format: + +```json +[ + { + "messages": [ + { + "role": "system", + "content": "system prompt (optional)" + }, + { + "role": "user", + "content": "user instruction" + }, + { + "role": "assistant", + "content": "model response" + } + ] + } +] +``` + +Regarding the above dataset, the description in `dataset_info.json` should be: + +```json +"dataset_name": { + "file_name": "data.json", + "formatting": "sharegpt", + "columns": { + "messages": "messages" + }, + "tags": { + "role_tag": "role", + "content_tag": "content", + "user_tag": "user", + "assistant_tag": "assistant", + "system_tag": "system" + } +} +``` + +Pre-training datasets and preference datasets are **incompatible** with the sharegpt format yet. From eea8a79e35f20ed47548c72dbe1c62c235c8f1d1 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Thu, 2 May 2024 02:14:55 +0800 Subject: [PATCH 5/5] Update README_zh.md Former-commit-id: d4d9180c401cb210654792d8052313e8db17fc51 --- data/README_zh.md | 139 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 96 insertions(+), 43 deletions(-) diff --git a/data/README_zh.md b/data/README_zh.md index 1fe98a9e..6449c5d5 100644 --- a/data/README_zh.md +++ b/data/README_zh.md @@ -1,4 +1,4 @@ -如果您使用自定义数据集，请务必在 `dataset_info.json` 文件中按照以下格式提供数据集定义。 +如果您使用自定义数据集，请务必按照以下格式在 `dataset_info.json` 文件中添加**数据集描述**。我们在下面也提供了一些例子。 ```json "数据集名称": { @@ -33,11 +33,11 @@ } ``` -添加后可通过指定 `--dataset 数据集名称` 参数使用自定义数据集。 +然后，可通过使用 `--dataset 数据集名称` 参数加载自定义数据集。 ---- -该项目目前支持二种格式的数据集：**alpaca** 和 **sharegpt**，其中 alpaca 格式的数据集按照以下方式组织： +该项目目前支持两种格式的数据集：**alpaca** 和 **sharegpt**，其中 alpaca 格式的数据集按照以下方式组织： ```json [ @@ -54,10 +54,11 @@ ] ``` -对于上述格式的数据，`dataset_info.json` 中的 `columns` 应为： +对于上述格式的数据，`dataset_info.json` 中的描述应为： ```json "数据集名称": { + "file_name": "data.json", "columns": { "prompt": "instruction", "query": "input", @@ -70,31 +71,62 @@ 其中 `query` 列对应的内容会与 `prompt` 列对应的内容拼接后作为用户指令，即用户指令为 `prompt\nquery`。`response` 列对应的内容为模型回答。 -`system` 列对应的内容将被作为系统提示词。`history` 列是由多个字符串二元组构成的列表，分别代表历史消息中每轮的指令和回答。注意历史消息中的回答**也会被用于训练**。 +`system` 列对应的内容将被作为系统提示词。`history` 列是由多个字符串二元组构成的列表，分别代表历史消息中每轮的指令和回答。注意在指令监督学习时，历史消息中的回答**也会被用于训练**。 -对于预训练数据集，仅 `prompt` 列中的内容会用于模型训练。 - -对于偏好数据集，`response` 列应当是一个长度为 2 的字符串列表，排在前面的代表更优的回答，例如： +对于**预训练数据集**，仅 `prompt` 列中的内容会用于模型训练，例如： ```json -{ - "instruction": "用户指令", - "input": "用户输入", - "output": [ - "优质回答", - "劣质回答" - ] +[ + {"text": "document"}, + {"text": "document"} +] +``` + +对于上述格式的数据，`dataset_info.json` 中的描述应为： + +```json +"数据集名称": { + "file_name": "data.json", + "columns": { + "prompt": "text" + } } ``` -添加偏好数据集需要额外指定 `"ranking": true`。 +对于**偏好数据集**，`response` 列应当是一个长度为 2 的字符串列表，排在前面的代表更优的回答，例如： + +```json +[ + { + "instruction": "用户指令", + "input": "用户输入", + "output": [ + "优质回答", + "劣质回答" + ] + } +] +``` + +对于上述格式的数据，`dataset_info.json` 中的描述应为： + +```json +"数据集名称": { + "file_name": "data.json", + "ranking": true, + "columns": { + "prompt": "instruction", + "query": "input", + "response": "output", + } +} +``` ---- -而 sharegpt 格式的数据集按照以下方式组织： +而 **sharegpt** 格式的数据集按照以下方式组织： ```json -# 第一种sharegpt格式 [ { "conversations": [ @@ -111,35 +143,14 @@ "tools": "工具描述（选填）" } ] - -# 第二种sharegpt格式 - -[ - { - "type": "chatml", - "messages": [ - { - "role": "system", - "content": "你是一个很有用的AI助手" - }, - { - "role": "user", - "content": "告诉我一些关于大模型的一些信息" - }, - { - "role": "assistant", - "content": "大模型是一种语言模型" - } - ], - "source": "unknown" - } -] ``` -对于上述格式的数据，`dataset_info.json` 中的 `columns` 应为： +对于上述格式的数据，`dataset_info.json` 中的描述应为： ```json "数据集名称": { + "file_name": "data.json", + "formatting": "sharegpt", "columns": { "messages": "conversations", "system": "system", @@ -156,4 +167,46 @@ 其中 `messages` 列应当是一个列表，且符合 `用户/模型/用户/模型/用户/模型` 的顺序。 -预训练数据集和偏好数据集尚不支持 sharegpt 格式。 +我们同样支持 **openai** 格式的数据集： + +```json +[ + { + "messages": [ + { + "role": "system", + "content": "系统提示词（选填）" + }, + { + "role": "user", + "content": "用户指令" + }, + { + "role": "assistant", + "content": "模型回答" + } + ] + } +] +``` + +对于上述格式的数据，`dataset_info.json` 中的描述应为： + +```json +"数据集名称": { + "file_name": "data.json", + "formatting": "sharegpt", + "columns": { + "messages": "messages" + }, + "tags": { + "role_tag": "role", + "content_tag": "content", + "user_tag": "user", + "assistant_tag": "assistant", + "system_tag": "system" + } +} +``` + +预训练数据集和偏好数据集**尚不支持** sharegpt 格式。