add examples

Former-commit-id: e824b715ad4bf885241b245b12d75563adab2e26

.github/CONTRIBUTING.md

@@ -19,3 +19,49 @@ There are several ways you can contribute to LLaMA Factory:
 ### Style guide
 
 LLaMA Factory follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html), check it for details.
+
+### Create a Pull Request
+
+1. Fork the [repository](https://github.com/hiyouga/LLaMA-Factory) by clicking on the [Fork](https://github.com/hiyouga/LLaMA-Factory/fork) button on the repository's page. This creates a copy of the code under your GitHub user account.
+
+2. Clone your fork to your local disk, and add the base repository as a remote:
+
+```bash
+git clone git@github.com:[username]/LLaMA-Factory.git
+cd LLaMA-Factory
+git remote add upstream https://github.com/hiyouga/LLaMA-Factory.git
+```
+
+3. Create a new branch to hold your development changes:
+
+```bash
+git checkout -b dev_your_branch
+```
+
+4. Set up a development environment by running the following command in a virtual environment:
+
+```bash
+pip install -e ".[dev]"
+```
+
+If LLaMA Factory was already installed in the virtual environment, remove it with `pip uninstall llamafactory` before reinstalling it in editable mode with the -e flag.
+
+5. Check code before commit:
+
+```bash
+make commit
+make style && make quality
+make test
+```
+
+6. Submit changes:
+
+```bash
+git add .
+git commit -m "commit message"
+git fetch upstream
+git rebase upstream/main
+git push -u origin dev_your_branch
+```
+
+7. Create a merge request from your branch `dev_your_branch` at [origin repo](https://github.com/hiyouga/LLaMA-Factory).

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
     -   id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
     -   id: no-commit-to-branch
-        args: ['--branch', 'master']
+        args: ['--branch', 'main']
 
 -   repo: https://github.com/asottile/pyupgrade
     rev: v3.17.0

README.md

@@ -584,6 +584,8 @@ API_PORT=8000 llamafactory-cli api examples/inference/llama3_vllm.yaml
 
 > [!TIP]
 > Visit [this page](https://platform.openai.com/docs/api-reference/chat/create) for API document.
+>
+> Examples: [Image understanding](scripts/test_image.py) | [Function calling](scripts/test_toolcall.py)
 
 ### Download from ModelScope Hub
 

README_zh.md

@@ -585,6 +585,8 @@ API_PORT=8000 llamafactory-cli api examples/inference/llama3_vllm.yaml
 
 > [!TIP]
 > API 文档请查阅[这里](https://platform.openai.com/docs/api-reference/chat/create)。
+>
+> 示例：[图像理解](scripts/test_image.py) | [工具调用](scripts/test_toolcall.py)
 
 ### 从魔搭社区下载
 

scripts/test_image.py

@@ -0,0 +1,65 @@
+# Copyright 2024 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from openai import OpenAI
+from transformers.utils.versions import require_version
+
+
+require_version("openai>=1.5.0", "To fix: pip install openai>=1.5.0")
+
+
+def main():
+    client = OpenAI(
+        api_key="{}".format(os.environ.get("API_KEY", "0")),
+        base_url="http://localhost:{}/v1".format(os.environ.get("API_PORT", 8000)),
+    )
+    messages = []
+    messages.append(
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Output the color and number of each box."},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-VL/boxes.png"},
+                },
+            ],
+        }
+    )
+    result = client.chat.completions.create(messages=messages, model="test")
+    messages.append(result.choices[0].message)
+    print("Round 1:", result.choices[0].message.content)
+    # The image shows a pyramid of colored blocks with numbers on them. Here are the colors and numbers of ...
+    messages.append(
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What kind of flower is this?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-VL/flowers.jpg"},
+                },
+            ],
+        }
+    )
+    result = client.chat.completions.create(messages=messages, model="test")
+    messages.append(result.choices[0].message)
+    print("Round 2:", result.choices[0].message.content)
+    # The image shows a cluster of forget-me-not flowers. Forget-me-nots are small ...
+
+
+if __name__ == "__main__":
+    main()

src/llamafactory/chat/hf_engine.py

@@ -86,12 +86,12 @@ class HuggingfaceEngine(BaseEngine):
         mm_input_dict = {"images": [], "videos": [], "imglens": [0], "vidlens": [0]}
         if images is not None:
             mm_input_dict.update({"images": images, "imglens": [len(images)]})
-            if IMAGE_PLACEHOLDER not in messages[0]["content"]:
+            if not any(IMAGE_PLACEHOLDER not in message["content"] for message in messages):
                 messages[0]["content"] = IMAGE_PLACEHOLDER * len(images) + messages[0]["content"]
 
         if videos is not None:
             mm_input_dict.update({"videos": videos, "vidlens": [len(videos)]})
-            if VIDEO_PLACEHOLDER not in messages[0]["content"]:
+            if not any(VIDEO_PLACEHOLDER not in message["content"] for message in messages):
                 messages[0]["content"] = VIDEO_PLACEHOLDER * len(videos) + messages[0]["content"]
 
         messages = template.mm_plugin.process_messages(

src/llamafactory/chat/vllm_engine.py

@@ -107,7 +107,7 @@ class VllmEngine(BaseEngine):
     ) -> AsyncIterator["RequestOutput"]:
         request_id = f"chatcmpl-{uuid.uuid4().hex}"
         if images is not None:
-            if IMAGE_PLACEHOLDER not in messages[0]["content"]:
+            if not any(IMAGE_PLACEHOLDER not in message["content"] for message in messages):
                 messages[0]["content"] = IMAGE_PLACEHOLDER * len(images) + messages[0]["content"]
 
         paired_messages = messages + [{"role": "assistant", "content": ""}]