merge data part to the text stream

Former-commit-id: c6dd89918f

data/dataset_info.json

@@ -418,6 +418,17 @@
     "hf_hub_url": "HuggingFaceH4/llava-instruct-mix-vsft"
   },
   "mllm_instruct_example": {
-    "hf_hub_url": "data/mllm_example_dataset"
+    "file_name": "llava_instruct_example.json",
+    "formatting": "llava",
+    "columns": {
+      "messages": "messages",
+      "images": "images"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
   }
-}
+}

data/llava_instruct_example.json

@@ -2,7 +2,7 @@
   {
     "messages": [
       {
-        "content": "Who are they?",
+        "content": "Who are they?<image>",
         "role": "user"
       },
       {
@@ -18,12 +18,14 @@
         "role": "assistant"
       }
     ],
-    "image": "1.jpg"
+    "images": [
+      "data/images/1.jpg"
+    ]
   },
   {
     "messages": [
       {
-        "content": "Who is he?",
+        "content": "Who is he?<image>",
         "role": "user"
       },
       {
@@ -39,12 +41,14 @@
         "role": "assistant"
       }
     ],
-    "image": "2.jpg"
+    "images": [
+      "data/images/2.jpg"
+    ]
   },
   {
     "messages": [
       {
-        "content": "Please describe this image",
+        "content": "Please describe this image<image>",
         "role": "user"
       },
       {
@@ -60,6 +64,8 @@
         "role": "assistant"
       }
     ],
-    "image": "3.jpg"
+    "images": [
+      "data/images/3.jpg"
+    ]
   }
-]
+]