[model] add qwen3-vl-30b (#9227)

Co-authored-by: Benjamin Feuer <penfever@gmail.com>
Co-authored-by: Yaowei Zheng <hiyouga@buaa.edu.cn>

.dockerignore

@@ -3,12 +3,12 @@
 .github
 .venv
 cache
-data
 docker
 saves
 hf_cache
 ms_cache
 om_cache
+shared_data
 output
 .dockerignore
 .gitattributes

.env.local

@@ -4,15 +4,20 @@ API_HOST=
 API_PORT=
 API_KEY=
 API_MODEL_NAME=
+API_VERBOSE=
 FASTAPI_ROOT_PATH=
 MAX_CONCURRENT=
 # general
 DISABLE_VERSION_CHECK=
 FORCE_CHECK_IMPORTS=
+ALLOW_EXTRA_ARGS=
 LLAMAFACTORY_VERBOSITY=
 USE_MODELSCOPE_HUB=
 USE_OPENMIND_HUB=
+USE_RAY=
 RECORD_VRAM=
+OPTIM_TORCH=
+NPU_JIT_COMPILE=
 # torchrun
 FORCE_TORCHRUN=
 MASTER_ADDR=
@@ -31,7 +36,7 @@ GRADIO_SERVER_PORT=
 GRADIO_ROOT_PATH=
 GRADIO_IPV6=
 # setup
-ENABLE_SHORT_CONSOLE=1
+ENABLE_SHORT_CONSOLE=
 # reserved (do not use)
 LLAMABOARD_ENABLED=
 LLAMABOARD_WORKDIR=

.github/ISSUE_TEMPLATE/1-bug-report.yml

@@ -0,0 +1,61 @@
+name: "\U0001F41B Bug / help"
+description: Create a report to help us improve the LLaMA Factory
+labels: ["bug", "pending"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Issues included in **[FAQs](https://github.com/hiyouga/LLaMA-Factory/issues/4614)** or those with **insufficient** information may be closed without a response.
+        已经包含在 **[常见问题](https://github.com/hiyouga/LLaMA-Factory/issues/4614)** 内或提供信息**不完整**的 issues 可能不会被回复。
+
+  - type: markdown
+    attributes:
+      value: |
+        Please do not create issues that are not related to framework bugs under this category, use **[Discussions](https://github.com/hiyouga/LLaMA-Factory/discussions/categories/q-a)** instead.
+        请勿在此分类下创建和框架 bug 无关的 issues，训练问题求助请使用 **[讨论区](https://github.com/hiyouga/LLaMA-Factory/discussions/categories/q-a)**。
+
+  - type: checkboxes
+    id: reminder
+    attributes:
+      label: Reminder
+      description: |
+        Please ensure you have read the above rules carefully and searched the existing issues (including FAQs).
+        请确保您已经认真阅读了上述规则并且搜索过现有的 issues（包括常见问题）。
+
+      options:
+        - label: I have read the above rules and searched the existing issues.
+          required: true
+
+  - type: textarea
+    id: system-info
+    validations:
+      required: true
+    attributes:
+      label: System Info
+      description: |
+        Please share your system info with us. You can run the command **llamafactory-cli env** and copy-paste its output below.
+        请提供您的系统信息。您可以在命令行运行 **llamafactory-cli env** 并将其输出复制到该文本框中。
+
+      placeholder: llamafactory version, platform, python version, ...
+
+  - type: textarea
+    id: reproduction
+    validations:
+      required: true
+    attributes:
+      label: Reproduction
+      description: |
+        Please provide entry arguments, error messages and stack traces that reproduces the problem.
+        请提供入口参数，错误日志以及异常堆栈以便于我们复现问题。
+
+      value: |
+        ```text
+        Put your message here.
+        ```
+
+  - type: textarea
+    id: others
+    validations:
+      required: false
+    attributes:
+      label: Others

.github/ISSUE_TEMPLATE/2-feature-request.yml

@@ -0,0 +1,41 @@
+name: "\U0001F680 Feature request"
+description: Submit a request for a new feature
+labels: ["enhancement", "pending"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please do not create issues that are not related to new features under this category.
+        请勿在此分类下创建和新特性无关的 issues。
+
+  - type: checkboxes
+    id: reminder
+    attributes:
+      label: Reminder
+      description: |
+        Please ensure you have read the above rules carefully and searched the existing issues.
+        请确保您已经认真阅读了上述规则并且搜索过现有的 issues。
+
+      options:
+        - label: I have read the above rules and searched the existing issues.
+          required: true
+
+  - type: textarea
+    id: description
+    validations:
+      required: true
+    attributes:
+      label: Description
+      description: |
+        A clear and concise description of the feature proposal.
+        请详细描述您希望加入的新功能特性。
+
+  - type: textarea
+    id: contribution
+    validations:
+      required: false
+    attributes:
+      label: Pull Request
+      description: |
+        Have you already created the relevant PR and submitted the code?
+        您是否已经创建了相关 PR 并提交了代码？

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,66 +0,0 @@
-name: "\U0001F41B Bug / Help"
-description: Create a report to help us improve the LLaMA Factory
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Issues included in **FAQs** or those with **insufficient** information may be closed without a response.
-        包含在**常见问题**内或提供信息**不完整**的 issues 可能不会被回复。
-
-  - type: checkboxes
-    id: reminder
-    attributes:
-      label: Reminder
-      description: |
-        Please ensure you have read the README carefully and searched the existing issues (including FAQs).
-        请确保您已经认真阅读了 README 并且搜索过现有的 issues（包括常见问题）。
-
-      options:
-        - label: I have read the README and searched the existing issues.
-          required: true
-
-  - type: textarea
-    id: system-info
-    validations:
-      required: true
-    attributes:
-      label: System Info
-      description: |
-        Please share your system info with us. You can run the command **llamafactory-cli env** and copy-paste its output below.
-        请提供您的系统信息。您可以在命令行运行 **llamafactory-cli env** 并将其输出复制到该文本框中。
-
-      placeholder: llamafactory version, platform, python version, ...
-
-  - type: textarea
-    id: reproduction
-    validations:
-      required: true
-    attributes:
-      label: Reproduction
-      description: |
-        Please provide code snippets, error messages and stack traces that reproduces the problem.
-        请提供运行参数，错误信息以及异常堆栈以便于我们复现该问题。
-        Remember to use Markdown tags to correctly format your code.
-        请合理使用 Markdown 标签来格式化您的文本。
-
-      placeholder: |
-        ```bash
-        llamafactory-cli train ...
-        ```
-
-  - type: textarea
-    id: expected-behavior
-    validations:
-      required: false
-    attributes:
-      label: Expected behavior
-      description: |
-        Please provide a clear and concise description of what you would expect to happen.
-        请提供您原本的目的，即这段代码的期望行为。
-
-  - type: textarea
-    id: others
-    validations:
-      required: false
-    attributes:
-      label: Others

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 📚 FAQs | 常见问题
+    url: https://github.com/hiyouga/LLaMA-Factory/issues/4614
+    about: Reading in advance is recommended | 建议提前阅读
+  - name: Discussions | 讨论区
+    url: https://github.com/hiyouga/LLaMA-Factory/discussions
+    about: Please ask fine-tuning questions here | 请在这里讨论训练问题

.github/workflows/docker.yml

@@ -0,0 +1,108 @@
+name: docker
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+    paths:
+      - "**/*.py"
+      - "requirements.txt"
+      - "docker/**"
+      - ".github/workflows/*.yml"
+  pull_request:
+    branches:
+      - "main"
+    paths:
+      - "**/*.py"
+      - "requirements.txt"
+      - "docker/**"
+      - ".github/workflows/*.yml"
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        device:
+          - "cuda"
+          - "npu"
+
+    runs-on: ubuntu-latest
+
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.device }}
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+    environment:
+      name: docker
+      url: https://hub.docker.com/r/hiyouga/llamafactory
+
+    steps:
+      - name: Free up disk space
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        with:
+          tool-cache: true
+          docker-images: false
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+
+      - name: Get llamafactory version
+        id: version
+        run: |
+          echo "tag=$(python setup.py --version | sed 's/\.dev0//')" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to Quay
+        if: ${{ github.event_name != 'pull_request' && matrix.device == 'npu' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ vars.QUAY_ASCEND_USERNAME }}
+          password: ${{ secrets.QUAY_ASCEND_TOKEN }}
+
+      - name: Build and push Docker image (CUDA)
+        if: ${{ matrix.device == 'cuda' }}
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/docker-cuda/Dockerfile
+          build-args: |
+            EXTRAS=metrics,deepspeed,liger-kernel
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: |
+            docker.io/hiyouga/llamafactory:latest
+            docker.io/hiyouga/llamafactory:${{ steps.version.outputs.tag }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build and push Docker image (NPU)
+        if: ${{ matrix.device == 'npu' }}
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./docker/docker-npu/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: |
+            docker.io/hiyouga/llamafactory:latest-npu-a2
+            docker.io/hiyouga/llamafactory:${{ steps.version.outputs.tag }}-npu-a2
+            quay.io/ascend/llamafactory:latest-npu-a2
+            quay.io/ascend/llamafactory:${{ steps.version.outputs.tag }}-npu-a2
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/label_issue.yml

@@ -18,13 +18,15 @@ jobs:
           ISSUE_URL: ${{ github.event.issue.html_url }}
           ISSUE_TITLE: ${{ github.event.issue.title }}
         run: |
-          LABEL=pending
-          NPU_KEYWORDS=(npu huawei ascend 华为 昇腾)
+          LABEL=""
+          NPU_KEYWORDS=(npu huawei ascend 华为 昇腾 910)
           ISSUE_TITLE_LOWER=$(echo $ISSUE_TITLE | tr '[:upper:]' '[:lower:]')
           for KEYWORD in ${NPU_KEYWORDS[@]}; do
             if [[ $ISSUE_TITLE_LOWER == *$KEYWORD* ]] && [[ $ISSUE_TITLE_LOWER != *input* ]]; then
-              LABEL=pending,npu
+              LABEL="npu"
               break
             fi
           done
-          gh issue edit $ISSUE_URL --add-label $LABEL
+          if [ -n "$LABEL" ]; then
+            gh issue edit $ISSUE_URL --add-label $LABEL
+          fi

.github/workflows/publish.yml

@@ -1,6 +1,7 @@
 name: publish
 
 on:
+  workflow_dispatch:
   release:
     types:
       - published
@@ -25,16 +26,11 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install build
+          python-version: "3.9"
 
       - name: Build package
         run: |
-          python -m build
+          make build
 
       - name: Publish package
         uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/tests.yml

@@ -1,18 +1,19 @@
 name: tests
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - "main"
     paths:
-      - "**.py"
+      - "**/*.py"
       - "requirements.txt"
       - ".github/workflows/*.yml"
   pull_request:
     branches:
       - "main"
     paths:
-      - "**.py"
+      - "**/*.py"
       - "requirements.txt"
       - ".github/workflows/*.yml"
 
@@ -21,20 +22,33 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-          - "3.8"  # TODO: remove py38 in next transformers release
+        python:
           - "3.9"
           - "3.10"
           - "3.11"
+          - "3.12"
         os:
           - "ubuntu-latest"
           - "windows-latest"
           - "macos-13"
+        transformers:
+          - null
+        include:  # test backward compatibility
+          - python: "3.9"
+            os: "ubuntu-latest"
+            transformers: "4.49.0"
+          - python: "3.9"
+            os: "ubuntu-latest"
+            transformers: "4.51.0"
+          - python: "3.9"
+            os: "ubuntu-latest"
+            transformers: "4.53.0"
 
     runs-on: ${{ matrix.os }}
 
-    environment:
-      name: tests
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}-${{ matrix.python }}-${{ matrix.transformers }}
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
     env:
       HF_TOKEN: ${{ secrets.HF_TOKEN }}
@@ -47,19 +61,47 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python }}
           cache: "pip"
-          cache-dependency-path: "setup.py"
+          cache-dependency-path: "**/requirements*.txt"
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install ".[torch,dev]"
 
+      - name: Install transformers
+        if: ${{ matrix.transformers }}
+        run: |
+          python -m pip install "transformers==${{ matrix.transformers }}"
+
+      - name: Install transformers to avoid mac os ci errors
+        if: ${{ matrix.os == 'macos-13' }}
+        run: |
+          python -m pip install "transformers<=4.51.3"
+
+      - name: Cache files
+        id: hf-hub-cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ runner.temp }}/huggingface
+          key: huggingface-${{ matrix.os }}-${{ matrix.python }}-${{ matrix.transformers }}-${{ hashFiles('tests/version.txt') }}
+
       - name: Check quality
         run: |
           make style && make quality
 
+      - name: Check license
+        run: |
+          make license
+
+      - name: Check build
+        run: |
+          make build
+
       - name: Test with pytest
         run: |
           make test
+        env:
+          HF_HOME: ${{ runner.temp }}/huggingface
+          HF_HUB_OFFLINE: "${{ steps.hf-hub-cache.outputs.cache-hit == 'true' && '1' || '0' }}"

.gitignore

@@ -162,12 +162,18 @@ cython_debug/
 # vscode
 .vscode/
 
+# uv
+uv.lock
+
 # custom .gitignore
-ms_cache/
 hf_cache/
+ms_cache/
 om_cache/
 cache/
 config/
 saves/
 output/
 wandb/
+swanlog/
+generated_predictions.jsonl
+predictions_score.json

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
     -   id: check-ast
     -   id: check-added-large-files
@@ -15,13 +15,13 @@ repos:
         args: ['--branch', 'main']
 
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.17.0
+    rev: v3.20.0
     hooks:
     -   id: pyupgrade
-        args: [--py38-plus]
+        args: [--py39-plus]
 
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.9
+    rev: v0.13.2
     hooks:
     -   id: ruff
         args: [--fix]

Makefile

@@ -1,14 +1,17 @@
-.PHONY: build commit quality style test
+.PHONY: build commit license quality style test
 
 check_dirs := scripts src tests setup.py
 
 build:
-	pip install build && python -m build
+	pip3 install build && python3 -m build
 
 commit:
 	pre-commit install
 	pre-commit run --all-files
 
+license:
+	python3 tests/check_license.py $(check_dirs)
+
 quality:
 	ruff check $(check_dirs)
 	ruff format --check $(check_dirs)

README.md

@@ -1,40 +1,59 @@
 ![# LLaMA Factory](assets/logo.png)
 
 [![GitHub Repo stars](https://img.shields.io/github/stars/hiyouga/LLaMA-Factory?style=social)](https://github.com/hiyouga/LLaMA-Factory/stargazers)
-[![GitHub Code License](https://img.shields.io/github/license/hiyouga/LLaMA-Factory)](LICENSE)
 [![GitHub last commit](https://img.shields.io/github/last-commit/hiyouga/LLaMA-Factory)](https://github.com/hiyouga/LLaMA-Factory/commits/main)
+[![GitHub contributors](https://img.shields.io/github/contributors/hiyouga/LLaMA-Factory?color=orange)](https://github.com/hiyouga/LLaMA-Factory/graphs/contributors)
+[![GitHub workflow](https://github.com/hiyouga/LLaMA-Factory/actions/workflows/tests.yml/badge.svg)](https://github.com/hiyouga/LLaMA-Factory/actions/workflows/tests.yml)
 [![PyPI](https://img.shields.io/pypi/v/llamafactory)](https://pypi.org/project/llamafactory/)
-[![Citation](https://img.shields.io/badge/citation-93-green)](#projects-using-llama-factory)
-[![GitHub pull request](https://img.shields.io/badge/PRs-welcome-blue)](https://github.com/hiyouga/LLaMA-Factory/pulls)
-[![Discord](https://dcbadge.vercel.app/api/server/rKfvV9r9FK?compact=true&style=flat)](https://discord.gg/rKfvV9r9FK)
+[![Citation](https://img.shields.io/badge/citation-840-green)](https://scholar.google.com/scholar?cites=12620864006390196564)
+[![Docker Pulls](https://img.shields.io/docker/pulls/hiyouga/llamafactory)](https://hub.docker.com/r/hiyouga/llamafactory/tags)
+
 [![Twitter](https://img.shields.io/twitter/follow/llamafactory_ai)](https://twitter.com/llamafactory_ai)
-[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1eRTPn37ltBbYsISy9Aw2NuI2Aq5CQrD9?usp=sharing)
-[![Open in DSW](https://gallery.pai-ml.com/assets/open-in-dsw.svg)](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory)
-[![Spaces](https://img.shields.io/badge/🤗-Open%20in%20Spaces-blue)](https://huggingface.co/spaces/hiyouga/LLaMA-Board)
-[![Studios](https://img.shields.io/badge/ModelScope-Open%20in%20Studios-blue)](https://modelscope.cn/studios/hiyouga/LLaMA-Board)
-[![SageMaker](https://img.shields.io/badge/SageMaker-Open%20in%20AWS-blue)](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/)
+[![Discord](assets/thirdparty/discord.svg)](https://discord.gg/rKfvV9r9FK)
 
-[![GitHub Tread](https://trendshift.io/api/badge/repositories/4535)](https://trendshift.io/repositories/4535)
+[![Open in Colab](assets/thirdparty/colab.svg)](https://colab.research.google.com/drive/1eRTPn37ltBbYsISy9Aw2NuI2Aq5CQrD9?usp=sharing)
+[![Open in DSW](assets/thirdparty/dsw.svg)](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory)
+[![Open in Lab4ai](assets/thirdparty/lab4ai.svg)](https://www.lab4ai.cn/course/detail?id=7c13e60f6137474eb40f6fd3983c0f46&utm_source=LLaMA-Factory)
+[![Open in Online](assets/thirdparty/online.svg)](https://www.llamafactory.com.cn/?utm_source=LLaMA-Factory)
+[![Open in Spaces](https://img.shields.io/badge/🤗-Open%20in%20Spaces-blue)](https://huggingface.co/spaces/hiyouga/LLaMA-Board)
+[![Open in Studios](https://img.shields.io/badge/ModelScope-Open%20in%20Studios-blue)](https://modelscope.cn/studios/hiyouga/LLaMA-Board)
+[![Open in Novita](https://img.shields.io/badge/Novita-Deploy%20Template-blue)](https://novita.ai/templates-library/105981?sharer=88115474-394e-4bda-968e-b88e123d0c47)
 
-👋 Join our [WeChat](assets/wechat.jpg) or [NPU user group](assets/wechat_npu.jpg).
+### Used by [Amazon](https://aws.amazon.com/cn/blogs/machine-learning/how-apoidea-group-enhances-visual-information-extraction-from-banking-documents-with-multimodal-models-using-llama-factory-on-amazon-sagemaker-hyperpod/), [NVIDIA](https://developer.nvidia.com/rtx/ai-toolkit), [Aliyun](https://help.aliyun.com/zh/pai/use-cases/fine-tune-a-llama-3-model-with-llama-factory), etc.
+
+<div align="center" markdown="1">
+
+### Supporters ❤️
+
+| <div style="text-align: center;"><a href="https://warp.dev/llama-factory"><img alt="Warp sponsorship" width="400" src="assets/sponsors/warp.jpg"></a><br><a href="https://warp.dev/llama-factory" style="font-size:larger;">Warp, the agentic terminal for developers</a><br><a href="https://warp.dev/llama-factory">Available for MacOS, Linux, & Windows</a> | <a href="https://serpapi.com"><img alt="SerpAPI sponsorship" width="250" src="assets/sponsors/serpapi.svg"> </a> |
+| ---- | ---- |
+
+----
+
+### Easily fine-tune 100+ large language models with zero-code [CLI](#quickstart) and [Web UI](#fine-tuning-with-llama-board-gui-powered-by-gradio)
+
+![GitHub Trend](https://trendshift.io/api/badge/repositories/4535)
+
+</div>
+
+👋 Join our [WeChat](https://github.com/hiyouga/llamafactory-community/blob/main/wechat/main.jpg), [NPU](https://github.com/hiyouga/llamafactory-community/blob/main/wechat/npu.jpg), [Lab4AI](https://github.com/hiyouga/llamafactory-community/blob/main/wechat/lab4ai.jpg), [LLaMA Factory Online](https://github.com/hiyouga/llamafactory-community/blob/main/wechat/online.jpg) user group.
 
 \[ English | [中文](README_zh.md) \]
 
 **Fine-tuning a large language model can be easy as...**
 
-https://github.com/user-attachments/assets/7c96b465-9df7-45f4-8053-bf03e58386d3
+https://github.com/user-attachments/assets/3991a3a8-4276-4d30-9cab-4cb0c4b9b99e
 
 Choose your path:
 
-- **Documentation (WIP)**: https://llamafactory.readthedocs.io/zh-cn/latest/
-- **Colab**: https://colab.research.google.com/drive/1eRTPn37ltBbYsISy9Aw2NuI2Aq5CQrD9?usp=sharing
+- **Documentation (WIP)**: https://llamafactory.readthedocs.io/en/latest/
+- **Documentation (AMD GPU)**: https://rocm.docs.amd.com/projects/ai-developer-hub/en/latest/notebooks/fine_tune/llama_factory_llama3.html
+- **Colab (free)**: https://colab.research.google.com/drive/1eRTPn37ltBbYsISy9Aw2NuI2Aq5CQrD9?usp=sharing
 - **Local machine**: Please refer to [usage](#getting-started)
-- **PAI-DSW**: [Llama3 Example](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory) | [Qwen2-VL Example](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_qwen2vl)
-- **Amazon SageMaker**: [Blog](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/)
-
-Recent activities:
-
-- **2024/10/18-2024/11/30**: Build a personal tour guide bot using PAI+LLaMA Factory. [[website]](https://developer.aliyun.com/topic/llamafactory2)
+- **PAI-DSW (free trial)**: https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory
+- **Alaya NeW (cloud GPU deal)**: https://docs.alayanew.com/docs/documents/useGuide/LLaMAFactory/mutiple/?utm_source=LLaMA-Factory
+- **Official Course**: https://www.lab4ai.cn/course/detail?id=7c13e60f6137474eb40f6fd3983c0f46&utm_source=LLaMA-Factory
+- **LLaMA Factory Online**: https://www.llamafactory.com.cn/?utm_source=LLaMA-Factory
 
 > [!NOTE]
 > Except for the above links, all other websites are unauthorized third-party websites. Please carefully use them.
@@ -42,13 +61,24 @@ Recent activities:
 ## Table of Contents
 
 - [Features](#features)
-- [Benchmark](#benchmark)
+- [Blogs](#blogs)
 - [Changelog](#changelog)
 - [Supported Models](#supported-models)
 - [Supported Training Approaches](#supported-training-approaches)
 - [Provided Datasets](#provided-datasets)
 - [Requirement](#requirement)
 - [Getting Started](#getting-started)
+  - [Installation](#installation)
+  - [Data Preparation](#data-preparation)
+  - [Quickstart](#quickstart)
+  - [Fine-Tuning with LLaMA Board GUI](#fine-tuning-with-llama-board-gui-powered-by-gradio)
+  - [LLaMA Factory Online](#llama-factory-online)
+  - [Build Docker](#build-docker)
+  - [Deploy with OpenAI-style API and vLLM](#deploy-with-openai-style-api-and-vllm)
+  - [Download from ModelScope Hub](#download-from-modelscope-hub)
+  - [Download from Modelers Hub](#download-from-modelers-hub)
+  - [Use W&B Logger](#use-wb-logger)
+  - [Use SwanLab Logger](#use-swanlab-logger)
 - [Projects using LLaMA Factory](#projects-using-llama-factory)
 - [License](#license)
 - [Citation](#citation)
@@ -56,46 +86,102 @@ Recent activities:
 
 ## Features
 
-- **Various models**: LLaMA, LLaVA, Mistral, Mixtral-MoE, Qwen, Qwen2-VL, Yi, Gemma, Baichuan, ChatGLM, Phi, etc.
+- **Various models**: LLaMA, LLaVA, Mistral, Mixtral-MoE, Qwen, Qwen2-VL, DeepSeek, Yi, Gemma, ChatGLM, Phi, etc.
 - **Integrated methods**: (Continuous) pre-training, (multimodal) supervised fine-tuning, reward modeling, PPO, DPO, KTO, ORPO, etc.
 - **Scalable resources**: 16-bit full-tuning, freeze-tuning, LoRA and 2/3/4/5/6/8-bit QLoRA via AQLM/AWQ/GPTQ/LLM.int8/HQQ/EETQ.
-- **Advanced algorithms**: [GaLore](https://github.com/jiaweizzhao/GaLore), [BAdam](https://github.com/Ledzy/BAdam), [Adam-mini](https://github.com/zyushun/Adam-mini), DoRA, LongLoRA, LLaMA Pro, Mixture-of-Depths, LoRA+, LoftQ, PiSSA and Agent tuning.
+- **Advanced algorithms**: [GaLore](https://github.com/jiaweizzhao/GaLore), [BAdam](https://github.com/Ledzy/BAdam), [APOLLO](https://github.com/zhuhanqing/APOLLO), [Adam-mini](https://github.com/zyushun/Adam-mini), [Muon](https://github.com/KellerJordan/Muon), [OFT](https://github.com/huggingface/peft/tree/main/src/peft/tuners/oft), DoRA, LongLoRA, LLaMA Pro, Mixture-of-Depths, LoRA+, LoftQ and PiSSA.
 - **Practical tricks**: [FlashAttention-2](https://github.com/Dao-AILab/flash-attention), [Unsloth](https://github.com/unslothai/unsloth), [Liger Kernel](https://github.com/linkedin/Liger-Kernel), RoPE scaling, NEFTune and rsLoRA.
-- **Experiment monitors**: LlamaBoard, TensorBoard, Wandb, MLflow, etc.
-- **Faster inference**: OpenAI-style API, Gradio UI and CLI with vLLM worker.
+- **Wide tasks**: Multi-turn dialogue, tool using, image understanding, visual grounding, video recognition, audio understanding, etc.
+- **Experiment monitors**: LlamaBoard, TensorBoard, Wandb, MLflow, [SwanLab](https://github.com/SwanHubX/SwanLab), etc.
+- **Faster inference**: OpenAI-style API, Gradio UI and CLI with [vLLM worker](https://github.com/vllm-project/vllm) or [SGLang worker](https://github.com/sgl-project/sglang).
 
-## Benchmark
+### Day-N Support for Fine-Tuning Cutting-Edge Models
 
-Compared to ChatGLM's [P-Tuning](https://github.com/THUDM/ChatGLM2-6B/tree/main/ptuning), LLaMA Factory's LoRA tuning offers up to **3.7 times faster** training speed with a better Rouge score on the advertising text generation task. By leveraging 4-bit quantization technique, LLaMA Factory's QLoRA further improves the efficiency regarding the GPU memory.
+| Support Date | Model Name                                                           |
+| ------------ | -------------------------------------------------------------------- |
+| Day 0        | Qwen3 / Qwen2.5-VL / Gemma 3 / GLM-4.1V / InternLM 3 / MiniCPM-o-2.6 |
+| Day 1        | Llama 3 / GLM-4 / Mistral Small / PaliGemma2 / Llama 4               |
 
-![benchmark](assets/benchmark.svg)
+## Blogs
 
-<details><summary>Definitions</summary>
+- 💡 [Easy Dataset × LLaMA Factory: Enabling LLMs to Efficiently Learn Domain Knowledge](https://buaa-act.feishu.cn/wiki/GVzlwYcRFiR8OLkHbL6cQpYin7g) (English)
+- [Fine-tune a mental health LLM using LLaMA-Factory](https://www.lab4ai.cn/project/detail?id=25cce32ec131497b9e06a93336a0817f&type=project&utm_source=LLaMA-Factory) (Chinese)
+- [Fine-tune GPT-OSS for Role-Playing using LLaMA-Factory](https://docs.llamafactory.com.cn/docs/documents/best-practice/gptroleplay/?utm_source=LLaMA-Factory) (Chinese)
+- [A One-Stop Code-Free Model Reinforcement Learning and Deployment Platform based on LLaMA-Factory and EasyR1](https://aws.amazon.com/cn/blogs/china/building-llm-model-hub-based-on-llamafactory-and-easyr1/) (Chinese)
+- [How Apoidea Group enhances visual information extraction from banking documents with multimodal models using LLaMA-Factory on Amazon SageMaker HyperPod](https://aws.amazon.com/cn/blogs/machine-learning/how-apoidea-group-enhances-visual-information-extraction-from-banking-documents-with-multimodal-models-using-llama-factory-on-amazon-sagemaker-hyperpod/) (English)
 
-- **Training Speed**: the number of training samples processed per second during the training. (bs=4, cutoff_len=1024)
-- **Rouge Score**: Rouge-2 score on the development set of the [advertising text generation](https://aclanthology.org/D19-1321.pdf) task. (bs=4, cutoff_len=1024)
-- **GPU Memory**: Peak GPU memory usage in 4-bit quantized training. (bs=1, cutoff_len=1024)
-- We adopt `pre_seq_len=128` for ChatGLM's P-Tuning and `lora_rank=32` for LLaMA Factory's LoRA tuning.
+<details><summary>All Blogs</summary>
+
+- [Fine-tune Llama3.1-70B for Medical Diagnosis using LLaMA-Factory](https://docs.alayanew.com/docs/documents/bestPractice/bigModel/llama70B/?utm_source=LLaMA-Factory) (Chinese)
+- [Fine-tune Qwen2.5-VL for Autonomous Driving using LLaMA-Factory](https://docs.alayanew.com/docs/documents/useGuide/LLaMAFactory/mutiple/?utm_source=LLaMA-Factory) (Chinese)
+- [LLaMA Factory: Fine-tuning the DeepSeek-R1-Distill-Qwen-7B Model for News Classifier](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_deepseek_r1_distill_7b) (Chinese)
+- [A One-Stop Code-Free Model Fine-Tuning \& Deployment Platform based on SageMaker and LLaMA-Factory](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/) (Chinese)
+- [LLaMA Factory Multi-Modal Fine-Tuning Practice: Fine-Tuning Qwen2-VL for Personal Tourist Guide](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_qwen2vl) (Chinese)
+- [LLaMA Factory: Fine-tuning Llama3 for Role-Playing](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory) (Chinese)
 
 </details>
 
 ## Changelog
 
-[24/10/09] We supported downloading pre-trained models and datasets from the **[Modelers Hub](https://modelers.cn/models)**. See [this tutorial](#download-from-modelers-hub) for usage.
+[25/08/22] We supported **[OFT](https://arxiv.org/abs/2306.07280)** and **[OFTv2](https://arxiv.org/abs/2506.19847)**. See [examples](examples/README.md) for usage.
 
-[24/09/19] We support fine-tuning the **[Qwen2.5](https://qwenlm.github.io/blog/qwen2.5/)** models.
+[25/08/20] We supported fine-tuning the **[Intern-S1-mini](https://huggingface.co/internlm/Intern-S1-mini)** models. See [PR #8976](https://github.com/hiyouga/LLaMA-Factory/pull/8976) to get started.
 
-[24/08/30] We support fine-tuning the **[Qwen2-VL](https://qwenlm.github.io/blog/qwen2-vl/)** models. Thank [@simonJJJ](https://github.com/simonJJJ)'s PR.
-
-[24/08/27] We support **[Liger Kernel](https://github.com/linkedin/Liger-Kernel)**. Try `enable_liger_kernel: true` for efficient training.
-
-[24/08/09] We support **[Adam-mini](https://github.com/zyushun/Adam-mini)** optimizer. See [examples](examples/README.md) for usage. Thank [@relic-yuexi](https://github.com/relic-yuexi)'s PR.
+[25/08/06] We supported fine-tuning the **[GPT-OSS](https://github.com/openai/gpt-oss)** models. See [PR #8826](https://github.com/hiyouga/LLaMA-Factory/pull/8826) to get started.
 
 <details><summary>Full Changelog</summary>
 
-[24/07/04] We support [contamination-free packed training](https://github.com/MeetKai/functionary/tree/main/functionary/train/packing). Use `neat_packing: true` to activate it. Thank [@chuan298](https://github.com/chuan298)'s PR.
+[25/07/02] We supported fine-tuning the **[GLM-4.1V-9B-Thinking](https://github.com/THUDM/GLM-4.1V-Thinking)** model.
 
-[24/06/16] We support **[PiSSA](https://arxiv.org/abs/2404.02948)** algorithm. See [examples](examples/README.md) for usage.
+[25/04/28] We supported fine-tuning the **[Qwen3](https://qwenlm.github.io/blog/qwen3/)** model family.
+
+[25/04/21] We supported the **[Muon](https://github.com/KellerJordan/Muon)** optimizer. See [examples](examples/README.md) for usage. Thank [@tianshijing](https://github.com/tianshijing)'s PR.
+
+[25/04/16] We supported fine-tuning the **[InternVL3](https://huggingface.co/OpenGVLab/InternVL3-8B)** model. See [PR #7258](https://github.com/hiyouga/LLaMA-Factory/pull/7258) to get started.
+
+[25/04/14] We supported fine-tuning the **[GLM-Z1](https://huggingface.co/THUDM/GLM-Z1-9B-0414)** and **[Kimi-VL](https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct)** models.
+
+[25/04/06] We supported fine-tuning the **[Llama 4](https://ai.meta.com/blog/llama-4-multimodal-intelligence/)** model. See [PR #7611](https://github.com/hiyouga/LLaMA-Factory/pull/7611) to get started.
+
+[25/03/31] We supported fine-tuning the **[Qwen2.5 Omni](https://qwenlm.github.io/blog/qwen2.5-omni/)** model. See [PR #7537](https://github.com/hiyouga/LLaMA-Factory/pull/7537) to get started.
+
+[25/03/15] We supported **[SGLang](https://github.com/sgl-project/sglang)** as inference backend. Try `infer_backend: sglang` to accelerate inference.
+
+[25/03/12] We supported fine-tuning the **[Gemma 3](https://huggingface.co/blog/gemma3)** model.
+
+[25/02/24] Announcing **[EasyR1](https://github.com/hiyouga/EasyR1)**, an efficient, scalable and multi-modality RL training framework for efficient GRPO training.
+
+[25/02/11] We supported saving the **[Ollama](https://github.com/ollama/ollama)** modelfile when exporting the model checkpoints. See [examples](examples/README.md) for usage.
+
+[25/02/05] We supported fine-tuning the **[Qwen2-Audio](Qwen/Qwen2-Audio-7B-Instruct)** and **[MiniCPM-o-2.6](https://huggingface.co/openbmb/MiniCPM-o-2_6)** on audio understanding tasks.
+
+[25/01/31] We supported fine-tuning the **[DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1)** and **[Qwen2.5-VL](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct)** models.
+
+[25/01/15] We supported **[APOLLO](https://arxiv.org/abs/2412.05270)** optimizer. See [examples](examples/README.md) for usage.
+
+[25/01/14] We supported fine-tuning the **[MiniCPM-o-2.6](https://huggingface.co/openbmb/MiniCPM-o-2_6)** and **[MiniCPM-V-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6)** models. Thank [@BUAADreamer](https://github.com/BUAADreamer)'s PR.
+
+[25/01/14] We supported fine-tuning the **[InternLM 3](https://huggingface.co/collections/internlm/)** models. Thank [@hhaAndroid](https://github.com/hhaAndroid)'s PR.
+
+[25/01/10] We supported fine-tuning the **[Phi-4](https://huggingface.co/microsoft/phi-4)** model.
+
+[24/12/21] We supported using **[SwanLab](https://github.com/SwanHubX/SwanLab)** for experiment tracking and visualization. See [this section](#use-swanlab-logger) for details.
+
+[24/11/27] We supported fine-tuning the **[Skywork-o1](https://huggingface.co/Skywork/Skywork-o1-Open-Llama-3.1-8B)** model and the **[OpenO1](https://huggingface.co/datasets/O1-OPEN/OpenO1-SFT)** dataset.
+
+[24/10/09] We supported downloading pre-trained models and datasets from the **[Modelers Hub](https://modelers.cn/models)**. See [this tutorial](#download-from-modelers-hub) for usage.
+
+[24/09/19] We supported fine-tuning the **[Qwen2.5](https://qwenlm.github.io/blog/qwen2.5/)** models.
+
+[24/08/30] We supported fine-tuning the **[Qwen2-VL](https://qwenlm.github.io/blog/qwen2-vl/)** models. Thank [@simonJJJ](https://github.com/simonJJJ)'s PR.
+
+[24/08/27] We supported **[Liger Kernel](https://github.com/linkedin/Liger-Kernel)**. Try `enable_liger_kernel: true` for efficient training.
+
+[24/08/09] We supported **[Adam-mini](https://github.com/zyushun/Adam-mini)** optimizer. See [examples](examples/README.md) for usage. Thank [@relic-yuexi](https://github.com/relic-yuexi)'s PR.
+
+[24/07/04] We supported [contamination-free packed training](https://github.com/MeetKai/functionary/tree/main/functionary/train/packing). Use `neat_packing: true` to activate it. Thank [@chuan298](https://github.com/chuan298)'s PR.
+
+[24/06/16] We supported **[PiSSA](https://arxiv.org/abs/2404.02948)** algorithm. See [examples](examples/README.md) for usage.
 
 [24/06/07] We supported fine-tuning the **[Qwen2](https://qwenlm.github.io/blog/qwen2/)** and **[GLM-4](https://github.com/THUDM/GLM-4)** models.
 
@@ -171,47 +257,84 @@ Compared to ChatGLM's [P-Tuning](https://github.com/THUDM/ChatGLM2-6B/tree/main/
 
 </details>
 
+> [!TIP]
+> If you cannot use the latest feature, please pull the latest code and install LLaMA-Factory again.
+
 ## Supported Models
 
-| Model                                                             | Model size                       | Template         |
-| ----------------------------------------------------------------- | -------------------------------- | ---------------- |
-| [Baichuan 2](https://huggingface.co/baichuan-inc)                 | 7B/13B                           | baichuan2        |
-| [BLOOM/BLOOMZ](https://huggingface.co/bigscience)                 | 560M/1.1B/1.7B/3B/7.1B/176B      | -                |
-| [ChatGLM3](https://huggingface.co/THUDM)                          | 6B                               | chatglm3         |
-| [Command R](https://huggingface.co/CohereForAI)                   | 35B/104B                         | cohere           |
-| [DeepSeek (Code/MoE)](https://huggingface.co/deepseek-ai)         | 7B/16B/67B/236B                  | deepseek         |
-| [Falcon](https://huggingface.co/tiiuae)                           | 7B/11B/40B/180B                  | falcon           |
-| [Gemma/Gemma 2/CodeGemma](https://huggingface.co/google)          | 2B/7B/9B/27B                     | gemma            |
-| [GLM-4](https://huggingface.co/THUDM)                             | 9B                               | glm4             |
-| [Index](https://huggingface.co/IndexTeam)                         | 1.9B                             | index            |
-| [InternLM2/InternLM2.5](https://huggingface.co/internlm)          | 7B/20B                           | intern2          |
-| [Llama](https://github.com/facebookresearch/llama)                | 7B/13B/33B/65B                   | -                |
-| [Llama 2](https://huggingface.co/meta-llama)                      | 7B/13B/70B                       | llama2           |
-| [Llama 3-3.2](https://huggingface.co/meta-llama)                  | 1B/3B/8B/70B                     | llama3           |
-| [Llama 3.2 Vision](https://huggingface.co/meta-llama)             | 11B/90B                          | mllama           |
-| [LLaVA-1.5](https://huggingface.co/llava-hf)                      | 7B/13B                           | llava            |
-| [LLaVA-NeXT](https://huggingface.co/llava-hf)                     | 7B/8B/13B/34B/72B/110B           | llava_next       |
-| [LLaVA-NeXT-Video](https://huggingface.co/llava-hf)               | 7B/34B                           | llava_next_video |
-| [MiniCPM](https://huggingface.co/openbmb)                         | 1B/2B/4B                         | cpm/cpm3         |
-| [Mistral/Mixtral](https://huggingface.co/mistralai)               | 7B/8x7B/8x22B                    | mistral          |
-| [OLMo](https://huggingface.co/allenai)                            | 1B/7B                            | -                |
-| [PaliGemma](https://huggingface.co/google)                        | 3B                               | paligemma        |
-| [Phi-1.5/Phi-2](https://huggingface.co/microsoft)                 | 1.3B/2.7B                        | -                |
-| [Phi-3](https://huggingface.co/microsoft)                         | 4B/14B                           | phi              |
-| [Phi-3-small](https://huggingface.co/microsoft)                   | 7B                               | phi_small        |
-| [Pixtral](https://huggingface.co/mistralai)                       | 12B                              | pixtral          |
-| [Qwen (1-2.5) (Code/Math/MoE)](https://huggingface.co/Qwen)       | 0.5B/1.5B/3B/7B/14B/32B/72B/110B | qwen             |
-| [Qwen2-VL](https://huggingface.co/Qwen)                           | 2B/7B/72B                        | qwen2_vl         |
-| [StarCoder 2](https://huggingface.co/bigcode)                     | 3B/7B/15B                        | -                |
-| [XVERSE](https://huggingface.co/xverse)                           | 7B/13B/65B                       | xverse           |
-| [Yi/Yi-1.5 (Code)](https://huggingface.co/01-ai)                  | 1.5B/6B/9B/34B                   | yi               |
-| [Yi-VL](https://huggingface.co/01-ai)                             | 6B/34B                           | yi_vl            |
-| [Yuan 2](https://huggingface.co/IEITYuan)                         | 2B/51B/102B                      | yuan             |
+| Model                                                             | Model size                       | Template             |
+| ----------------------------------------------------------------- | -------------------------------- | -------------------- |
+| [Baichuan 2](https://huggingface.co/baichuan-inc)                 | 7B/13B                           | baichuan2            |
+| [BLOOM/BLOOMZ](https://huggingface.co/bigscience)                 | 560M/1.1B/1.7B/3B/7.1B/176B      | -                    |
+| [ChatGLM3](https://huggingface.co/THUDM)                          | 6B                               | chatglm3             |
+| [Command R](https://huggingface.co/CohereForAI)                   | 35B/104B                         | cohere               |
+| [DeepSeek (Code/MoE)](https://huggingface.co/deepseek-ai)         | 7B/16B/67B/236B                  | deepseek             |
+| [DeepSeek 2.5/3](https://huggingface.co/deepseek-ai)              | 236B/671B                        | deepseek3            |
+| [DeepSeek R1 (Distill)](https://huggingface.co/deepseek-ai)       | 1.5B/7B/8B/14B/32B/70B/671B      | deepseekr1           |
+| [ERNIE-4.5](https://huggingface.co/baidu)                         | 0.3B/21B/300B                    | ernie/ernie_nothink  |
+| [Falcon](https://huggingface.co/tiiuae)                           | 7B/11B/40B/180B                  | falcon               |
+| [Falcon-H1](https://huggingface.co/tiiuae)                        | 0.5B/1.5B/3B/7B/34B              | falcon_h1            |
+| [Gemma/Gemma 2/CodeGemma](https://huggingface.co/google)          | 2B/7B/9B/27B                     | gemma/gemma2         |
+| [Gemma 3/Gemma 3n](https://huggingface.co/google)                 | 270M/1B/4B/6B/8B/12B/27B         | gemma3/gemma3n       |
+| [GLM-4/GLM-4-0414/GLM-Z1](https://huggingface.co/zai-org)         | 9B/32B                           | glm4/glmz1           |
+| [GLM-4.1V](https://huggingface.co/zai-org)                        | 9B                               | glm4v                |
+| [GLM-4.5/GLM-4.5V](https://huggingface.co/zai-org)                | 106B/355B                        | glm4_moe/glm4v_moe   |
+| [GPT-2](https://huggingface.co/openai-community)                  | 0.1B/0.4B/0.8B/1.5B              | -                    |
+| [GPT-OSS](https://huggingface.co/openai)                          | 20B/120B                         | gpt                  |
+| [Granite 3.0-3.3](https://huggingface.co/ibm-granite)             | 1B/2B/3B/8B                      | granite3             |
+| [Granite 4](https://huggingface.co/ibm-granite)                   | 7B                               | granite4             |
+| [Hunyuan](https://huggingface.co/tencent/)                        | 7B                               | hunyuan              |
+| [Index](https://huggingface.co/IndexTeam)                         | 1.9B                             | index                |
+| [InternLM 2-3](https://huggingface.co/internlm)                   | 7B/8B/20B                        | intern2              |
+| [InternVL 2.5-3.5](https://huggingface.co/OpenGVLab)              | 1B/2B/4B/8B/14B/30B/38B/78B/241B | intern_vl            |
+| [InternLM/Intern-S1-mini](https://huggingface.co/internlm/)       | 8B                               | intern_s1            |
+| [Kimi-VL](https://huggingface.co/moonshotai)                      | 16B                              | kimi_vl              |
+| [Ling 2.0 (mini/flash)](https://huggingface.co/inclusionAI)       | 16B/100B                         | bailing_v2           |
+| [Llama](https://github.com/facebookresearch/llama)                | 7B/13B/33B/65B                   | -                    |
+| [Llama 2](https://huggingface.co/meta-llama)                      | 7B/13B/70B                       | llama2               |
+| [Llama 3-3.3](https://huggingface.co/meta-llama)                  | 1B/3B/8B/70B                     | llama3               |
+| [Llama 4](https://huggingface.co/meta-llama)                      | 109B/402B                        | llama4               |
+| [Llama 3.2 Vision](https://huggingface.co/meta-llama)             | 11B/90B                          | mllama               |
+| [LLaVA-1.5](https://huggingface.co/llava-hf)                      | 7B/13B                           | llava                |
+| [LLaVA-NeXT](https://huggingface.co/llava-hf)                     | 7B/8B/13B/34B/72B/110B           | llava_next           |
+| [LLaVA-NeXT-Video](https://huggingface.co/llava-hf)               | 7B/34B                           | llava_next_video     |
+| [MiMo](https://huggingface.co/XiaomiMiMo)                         | 7B                               | mimo                 |
+| [MiniCPM 1-4.1](https://huggingface.co/openbmb)                   | 0.5B/1B/2B/4B/8B                 | cpm/cpm3/cpm4        |
+| [MiniCPM-o-2.6/MiniCPM-V-2.6](https://huggingface.co/openbmb)     | 8B                               | minicpm_o/minicpm_v  |
+| [Ministral/Mistral-Nemo](https://huggingface.co/mistralai)        | 8B/12B                           | ministral            |
+| [Mistral/Mixtral](https://huggingface.co/mistralai)               | 7B/8x7B/8x22B                    | mistral              |
+| [Mistral Small](https://huggingface.co/mistralai)                 | 24B                              | mistral_small        |
+| [OLMo](https://huggingface.co/allenai)                            | 1B/7B                            | -                    |
+| [PaliGemma/PaliGemma2](https://huggingface.co/google)             | 3B/10B/28B                       | paligemma            |
+| [Phi-1.5/Phi-2](https://huggingface.co/microsoft)                 | 1.3B/2.7B                        | -                    |
+| [Phi-3/Phi-3.5](https://huggingface.co/microsoft)                 | 4B/14B                           | phi                  |
+| [Phi-3-small](https://huggingface.co/microsoft)                   | 7B                               | phi_small            |
+| [Phi-4](https://huggingface.co/microsoft)                         | 14B                              | phi4                 |
+| [Pixtral](https://huggingface.co/mistralai)                       | 12B                              | pixtral              |
+| [Qwen (1-2.5) (Code/Math/MoE/QwQ)](https://huggingface.co/Qwen)   | 0.5B/1.5B/3B/7B/14B/32B/72B/110B | qwen                 |
+| [Qwen3 (MoE/Instruct/Thinking/Next)](https://huggingface.co/Qwen) | 0.6B/1.7B/4B/8B/14B/32B/80B/235B | qwen3/qwen3_nothink  |
+| [Qwen2-Audio](https://huggingface.co/Qwen)                        | 7B                               | qwen2_audio          |
+| [Qwen2.5-Omni](https://huggingface.co/Qwen)                       | 3B/7B                            | qwen2_omni           |
+| [Qwen3-Omni](https://huggingface.co/Qwen)*                        | 30B                              | qwen3_omni           |
+| [Qwen2-VL/Qwen2.5-VL/QVQ](https://huggingface.co/Qwen)            | 2B/3B/7B/32B/72B                 | qwen2_vl             |
+| [Qwen3-VL](https://huggingface.co/Qwen)*                          | 235B                             | qwen3_vl             |
+| [Seed (OSS/Coder)](https://huggingface.co/ByteDance-Seed)         | 8B/36B                           | seed_oss/seed_coder  |
+| [Skywork o1](https://huggingface.co/Skywork)                      | 8B                               | skywork_o1           |
+| [StarCoder 2](https://huggingface.co/bigcode)                     | 3B/7B/15B                        | -                    |
+| [TeleChat2](https://huggingface.co/Tele-AI)                       | 3B/7B/35B/115B                   | telechat2            |
+| [XVERSE](https://huggingface.co/xverse)                           | 7B/13B/65B                       | xverse               |
+| [Yi/Yi-1.5 (Code)](https://huggingface.co/01-ai)                  | 1.5B/6B/9B/34B                   | yi                   |
+| [Yi-VL](https://huggingface.co/01-ai)                             | 6B/34B                           | yi_vl                |
+| [Yuan 2](https://huggingface.co/IEITYuan)                         | 2B/51B/102B                      | yuan                 |
 
 > [!NOTE]
 > For the "base" models, the `template` argument can be chosen from `default`, `alpaca`, `vicuna` etc. But make sure to use the **corresponding template** for the "instruct/chat" models.
 >
 > Remember to use the **SAME** template in training and inference.
+>
+> \*: You should install the `transformers` from main branch and use `DISABLE_VERSION_CHECK=1` to skip version check.
+>
+> \*\*: You need to install a specific version of `transformers` to use the corresponding model.
 
 Please refer to [constants.py](src/llamafactory/extras/constants.py) for a full list of models we supported.
 
@@ -219,16 +342,16 @@ You also can add a custom chat template to [template.py](src/llamafactory/data/t
 
 ## Supported Training Approaches
 
-| Approach               |     Full-tuning    |    Freeze-tuning   |       LoRA         |       QLoRA        |
-| ---------------------- | ------------------ | ------------------ | ------------------ | ------------------ |
-| Pre-Training           | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| Supervised Fine-Tuning | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| Reward Modeling        | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| PPO Training           | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| DPO Training           | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| KTO Training           | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| ORPO Training          | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| SimPO Training         | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| Approach               |     Full-tuning    |    Freeze-tuning   |       LoRA         |       QLoRA        |        OFT         |        QOFT        |
+| ---------------------- | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ |
+| Pre-Training           | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| Supervised Fine-Tuning | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| Reward Modeling        | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| PPO Training           | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| DPO Training           | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| KTO Training           | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| ORPO Training          | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| SimPO Training         | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
 
 > [!TIP]
 > The implementation details of PPO can be found in [this blog](https://newfacade.github.io/notes-on-reinforcement-learning/17-ppo-trl.html).
@@ -246,6 +369,11 @@ You also can add a custom chat template to [template.py](src/llamafactory/data/t
 - [SkyPile (zh)](https://huggingface.co/datasets/Skywork/SkyPile-150B)
 - [FineWeb (en)](https://huggingface.co/datasets/HuggingFaceFW/fineweb)
 - [FineWeb-Edu (en)](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu)
+- [CCI3-HQ (zh)](https://huggingface.co/datasets/BAAI/CCI3-HQ)
+- [CCI3-Data (zh)](https://huggingface.co/datasets/BAAI/CCI3-Data)
+- [CCI4.0-M2-Base-v1 (en&zh)](https://huggingface.co/datasets/BAAI/CCI4.0-M2-Base-v1)
+- [CCI4.0-M2-CoT-v1 (en&zh)](https://huggingface.co/datasets/BAAI/CCI4.0-M2-CoT-v1)
+- [CCI4.0-M2-Extra-v1 (en&zh)](https://huggingface.co/datasets/BAAI/CCI4.0-M2-Extra-v1)
 - [The Stack (en)](https://huggingface.co/datasets/bigcode/the-stack)
 - [StarCoder (en)](https://huggingface.co/datasets/bigcode/starcoderdata)
 
@@ -283,6 +411,7 @@ You also can add a custom chat template to [template.py](src/llamafactory/data/t
 - [ShareGPT Hyperfiltered (en)](https://huggingface.co/datasets/totally-not-an-llm/sharegpt-hyperfiltered-3k)
 - [ShareGPT4 (en&zh)](https://huggingface.co/datasets/shibing624/sharegpt_gpt4)
 - [UltraChat 200k (en)](https://huggingface.co/datasets/HuggingFaceH4/ultrachat_200k)
+- [Infinity Instruct (zh)](https://huggingface.co/datasets/BAAI/Infinity-Instruct)
 - [AgentInstruct (en)](https://huggingface.co/datasets/THUDM/AgentInstruct)
 - [LMSYS Chat 1M (en)](https://huggingface.co/datasets/lmsys/lmsys-chat-1m)
 - [Evol Instruct V2 (en)](https://huggingface.co/datasets/WizardLM/WizardLM_evol_instruct_V2_196k)
@@ -290,9 +419,13 @@ You also can add a custom chat template to [template.py](src/llamafactory/data/t
 - [STEM (zh)](https://huggingface.co/datasets/hfl/stem_zh_instruction)
 - [Ruozhiba (zh)](https://huggingface.co/datasets/hfl/ruozhiba_gpt4_turbo)
 - [Neo-sft (zh)](https://huggingface.co/datasets/m-a-p/neo_sft_phase2)
-- [WebInstructSub (en)](https://huggingface.co/datasets/TIGER-Lab/WebInstructSub)
 - [Magpie-Pro-300K-Filtered (en)](https://huggingface.co/datasets/Magpie-Align/Magpie-Pro-300K-Filtered)
 - [Magpie-ultra-v0.1 (en)](https://huggingface.co/datasets/argilla/magpie-ultra-v0.1)
+- [WebInstructSub (en)](https://huggingface.co/datasets/TIGER-Lab/WebInstructSub)
+- [OpenO1-SFT (en&zh)](https://huggingface.co/datasets/O1-OPEN/OpenO1-SFT)
+- [Open-Thoughts (en)](https://huggingface.co/datasets/open-thoughts/OpenThoughts-114k)
+- [Open-R1-Math (en)](https://huggingface.co/datasets/open-r1/OpenR1-Math-220k)
+- [Chinese-DeepSeek-R1-Distill (zh)](https://huggingface.co/datasets/Congliu/Chinese-DeepSeek-R1-Distill-data-110k-SFT)
 - [LLaVA mixed (en&zh)](https://huggingface.co/datasets/BUAADreamer/llava-en-zh-300k)
 - [Pokemon-gpt4o-captions (en&zh)](https://huggingface.co/datasets/jugg1024/pokemon-gpt4o-captions)
 - [Open Assistant (de)](https://huggingface.co/datasets/mayflowergmbh/oasst_de)
@@ -311,8 +444,10 @@ You also can add a custom chat template to [template.py](src/llamafactory/data/t
 
 - [DPO mixed (en&zh)](https://huggingface.co/datasets/hiyouga/DPO-En-Zh-20k)
 - [UltraFeedback (en)](https://huggingface.co/datasets/HuggingFaceH4/ultrafeedback_binarized)
+- [COIG-P (zh)](https://huggingface.co/datasets/m-a-p/COIG-P)
 - [RLHF-V (en)](https://huggingface.co/datasets/openbmb/RLHF-V-Dataset)
 - [VLFeedback (en)](https://huggingface.co/datasets/Zhihui/VLFeedback)
+- [RLAIF-V (en)](https://huggingface.co/datasets/openbmb/RLAIF-V-Dataset)
 - [Orca DPO Pairs (en)](https://huggingface.co/datasets/Intel/orca_dpo_pairs)
 - [HH-RLHF (en)](https://huggingface.co/datasets/Anthropic/hh-rlhf)
 - [Nectar (en)](https://huggingface.co/datasets/berkeley-nest/Nectar)
@@ -332,35 +467,35 @@ huggingface-cli login
 
 | Mandatory    | Minimum | Recommend |
 | ------------ | ------- | --------- |
-| python       | 3.8     | 3.11      |
-| torch        | 1.13.1  | 2.4.0     |
-| transformers | 4.41.2  | 4.43.4    |
-| datasets     | 2.16.0  | 2.20.0    |
-| accelerate   | 0.30.1  | 0.32.0    |
-| peft         | 0.11.1  | 0.12.0    |
+| python       | 3.9     | 3.10      |
+| torch        | 2.0.0   | 2.6.0     |
+| torchvision  | 0.15.0  | 0.21.0    |
+| transformers | 4.49.0  | 4.50.0    |
+| datasets     | 2.16.0  | 3.2.0     |
+| accelerate   | 0.34.0  | 1.2.1     |
+| peft         | 0.14.0  | 0.15.1    |
 | trl          | 0.8.6   | 0.9.6     |
 
 | Optional     | Minimum | Recommend |
 | ------------ | ------- | --------- |
 | CUDA         | 11.6    | 12.2      |
-| deepspeed    | 0.10.0  | 0.14.0    |
+| deepspeed    | 0.10.0  | 0.16.4    |
 | bitsandbytes | 0.39.0  | 0.43.1    |
-| vllm         | 0.4.3   | 0.5.0     |
-| flash-attn   | 2.3.0   | 2.6.3     |
+| vllm         | 0.4.3   | 0.8.2     |
+| flash-attn   | 2.5.6   | 2.7.2     |
 
 ### Hardware Requirement
 
 \* *estimated*
 
-| Method            | Bits |   7B  |  13B  |  30B  |   70B  |  110B  |  8x7B |  8x22B |
-| ----------------- | ---- | ----- | ----- | ----- | ------ | ------ | ----- | ------ |
-| Full              | AMP  | 120GB | 240GB | 600GB | 1200GB | 2000GB | 900GB | 2400GB |
-| Full              |  16  |  60GB | 120GB | 300GB |  600GB |  900GB | 400GB | 1200GB |
-| Freeze            |  16  |  20GB |  40GB |  80GB |  200GB |  360GB | 160GB |  400GB |
-| LoRA/GaLore/BAdam |  16  |  16GB |  32GB |  64GB |  160GB |  240GB | 120GB |  320GB |
-| QLoRA             |   8  |  10GB |  20GB |  40GB |   80GB |  140GB |  60GB |  160GB |
-| QLoRA             |   4  |   6GB |  12GB |  24GB |   48GB |   72GB |  30GB |   96GB |
-| QLoRA             |   2  |   4GB |   8GB |  16GB |   24GB |   48GB |  18GB |   48GB |
+| Method                              | Bits |   7B  |  14B  |  30B  |   70B  |   `x`B  |
+| ----------------------------------- | ---- | ----- | ----- | ----- | ------ | ------- |
+| Full (`bf16` or `fp16`)             |  32  | 120GB | 240GB | 600GB | 1200GB | `18x`GB |
+| Full (`pure_bf16`)                  |  16  |  60GB | 120GB | 300GB |  600GB |  `8x`GB |
+| Freeze/LoRA/GaLore/APOLLO/BAdam/OFT |  16  |  16GB |  32GB |  64GB |  160GB |  `2x`GB |
+| QLoRA / QOFT                        |   8  |  10GB |  20GB |  40GB |   80GB |   `x`GB |
+| QLoRA / QOFT                        |   4  |   6GB |  12GB |  24GB |   48GB | `x/2`GB |
+| QLoRA / QOFT                        |   2  |   4GB |   8GB |  16GB |   24GB | `x/4`GB |
 
 ## Getting Started
 
@@ -369,53 +504,99 @@ huggingface-cli login
 > [!IMPORTANT]
 > Installation is mandatory.
 
+#### Install from Source
+
 ```bash
 git clone --depth 1 https://github.com/hiyouga/LLaMA-Factory.git
 cd LLaMA-Factory
-pip install -e ".[torch,metrics]"
+pip install -e ".[torch,metrics]" --no-build-isolation
 ```
 
-Extra dependencies available: torch, torch-npu, metrics, deepspeed, liger-kernel, bitsandbytes, hqq, eetq, gptq, awq, aqlm, vllm, galore, badam, adam-mini, qwen, modelscope, openmind, quality
+Extra dependencies available: torch, torch-npu, metrics, deepspeed, liger-kernel, bitsandbytes, hqq, eetq, gptq, aqlm, vllm, sglang, galore, apollo, badam, adam-mini, qwen, minicpm_v, openmind, swanlab, dev
 
-> [!TIP]
-> Use `pip install --no-deps -e .` to resolve package conflicts.
+#### Install from Docker Image
+
+```bash
+docker run -it --rm --gpus=all --ipc=host hiyouga/llamafactory:latest
+```
+
+This image is built on Ubuntu 22.04 (x86\_64), CUDA 12.4, Python 3.11, PyTorch 2.6.0, and Flash-attn 2.7.4.
+
+Find the pre-built images: https://hub.docker.com/r/hiyouga/llamafactory/tags
+
+Please refer to [build docker](#build-docker) to build the image yourself.
+
+<details><summary>Setting up a virtual environment with <b>uv</b></summary>
+
+Create an isolated Python environment with [uv](https://github.com/astral-sh/uv):
+
+```bash
+uv sync --extra torch --extra metrics --prerelease=allow
+```
+
+Run LLaMA-Factory in the isolated environment:
+
+```bash
+uv run --prerelease=allow llamafactory-cli train examples/train_lora/llama3_lora_pretrain.yaml
+```
+
+</details>
 
 <details><summary>For Windows users</summary>
 
+#### Install PyTorch
+
+You need to manually install the GPU version of PyTorch on the Windows platform. Please refer to the [official website](https://pytorch.org/get-started/locally/) and the following command to install PyTorch with CUDA support:
+
+```bash
+pip uninstall torch torchvision torchaudio
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu126
+python -c "import torch; print(torch.cuda.is_available())"
+```
+
+If you see `True` then you have successfully installed PyTorch with CUDA support.
+
+Try `dataloader_num_workers: 0` if you encounter `Can't pickle local object` error.
+
+#### Install BitsAndBytes
+
 If you want to enable the quantized LoRA (QLoRA) on the Windows platform, you need to install a pre-built version of `bitsandbytes` library, which supports CUDA 11.1 to 12.2, please select the appropriate [release version](https://github.com/jllllll/bitsandbytes-windows-webui/releases/tag/wheels) based on your CUDA version.
 
 ```bash
 pip install https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.41.2.post2-py3-none-win_amd64.whl
 ```
 
-To enable FlashAttention-2 on the Windows platform, you need to install the precompiled `flash-attn` library, which supports CUDA 12.1 to 12.2. Please download the corresponding version from [flash-attention](https://github.com/bdashore3/flash-attention/releases) based on your requirements.
+#### Install Flash Attention-2
+
+To enable FlashAttention-2 on the Windows platform, please use the script from [flash-attention-windows-wheel](https://huggingface.co/lldacing/flash-attention-windows-wheel) to compile and install it by yourself.
 
 </details>
 
 <details><summary>For Ascend NPU users</summary>
 
-To install LLaMA Factory on Ascend NPU devices, please specify extra dependencies: `pip install -e ".[torch-npu,metrics]"`. Additionally, you need to install the **[Ascend CANN Toolkit and Kernels](https://www.hiascend.com/developer/download/community/result?module=cann)**. Please follow the [installation tutorial](https://www.hiascend.com/document/detail/en/CANNCommunityEdition/600alphaX/softwareinstall/instg/atlasdeploy_03_0031.html) or use the following commands:
+To install LLaMA Factory on Ascend NPU devices, please upgrade Python to version 3.10 or higher and specify extra dependencies: `pip install -e ".[torch-npu,metrics]"`. Additionally, you need to install the **[Ascend CANN Toolkit and Kernels](https://www.hiascend.com/developer/download/community/result?module=cann)**. Please follow the [installation tutorial](https://www.hiascend.com/document/detail/en/CANNCommunityEdition/600alphaX/softwareinstall/instg/atlasdeploy_03_0031.html) or use the following commands:
 
 ```bash
 # replace the url according to your CANN version and devices
 # install CANN Toolkit
-wget https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Milan-ASL/Milan-ASL%20V100R001C17SPC701/Ascend-cann-toolkit_8.0.RC1.alpha001_linux-"$(uname -i)".run
-bash Ascend-cann-toolkit_8.0.RC1.alpha001_linux-"$(uname -i)".run --install
+wget https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Milan-ASL/Milan-ASL%20V100R001C20SPC702/Ascend-cann-toolkit_8.0.0.alpha002_linux-"$(uname -i)".run
+bash Ascend-cann-toolkit_8.0.0.alpha002_linux-"$(uname -i)".run --install
 
 # install CANN Kernels
-wget https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Milan-ASL/Milan-ASL%20V100R001C17SPC701/Ascend-cann-kernels-910b_8.0.RC1.alpha001_linux.run
-bash Ascend-cann-kernels-910b_8.0.RC1.alpha001_linux.run --install
+wget https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Milan-ASL/Milan-ASL%20V100R001C20SPC702/Ascend-cann-kernels-910b_8.0.0.alpha002_linux-"$(uname -i)".run
+bash Ascend-cann-kernels-910b_8.0.0.alpha002_linux-"$(uname -i)".run --install
 
 # set env variables
 source /usr/local/Ascend/ascend-toolkit/set_env.sh
 ```
 
-| Requirement  | Minimum | Recommend   |
-| ------------ | ------- | ----------- |
-| CANN         | 8.0.RC1 | 8.0.RC1     |
-| torch        | 2.1.0   | 2.1.0       |
-| torch-npu    | 2.1.0   | 2.1.0.post3 |
-| deepspeed    | 0.13.2  | 0.13.2      |
+| Requirement  | Minimum | Recommend      |
+| ------------ | ------- | -------------- |
+| CANN         | 8.0.RC1 | 8.0.0.alpha002 |
+| torch        | 2.1.0   | 2.4.0          |
+| torch-npu    | 2.1.0   | 2.4.0.post2    |
+| deepspeed    | 0.13.2  | 0.13.2         |
+| vllm-ascend  | -       | 0.7.3          |
 
 Remember to use `ASCEND_RT_VISIBLE_DEVICES` instead of `CUDA_VISIBLE_DEVICES` to specify the device to use.
 
@@ -423,15 +604,51 @@ If you cannot infer model on NPU devices, try setting `do_sample: false` in the
 
 Download the pre-built Docker images: [32GB](http://mirrors.cn-central-221.ovaijisuan.com/detail/130.html) | [64GB](http://mirrors.cn-central-221.ovaijisuan.com/detail/131.html)
 
+#### Install BitsAndBytes
+
+To use QLoRA based on bitsandbytes on Ascend NPU, please follow these 3 steps:
+
+1. Manually compile bitsandbytes: Refer to [the installation documentation](https://huggingface.co/docs/bitsandbytes/installation?backend=Ascend+NPU&platform=Ascend+NPU) for the NPU version of bitsandbytes to complete the compilation and installation. The compilation requires a cmake version of at least 3.22.1 and a g++ version of at least 12.x.
+
+```bash
+# Install bitsandbytes from source
+# Clone bitsandbytes repo, Ascend NPU backend is currently enabled on multi-backend-refactor branch
+git clone -b multi-backend-refactor https://github.com/bitsandbytes-foundation/bitsandbytes.git
+cd bitsandbytes/
+
+# Install dependencies
+pip install -r requirements-dev.txt
+
+# Install the dependencies for the compilation tools. Note that the commands for this step may vary depending on the operating system. The following are provided for reference
+apt-get install -y build-essential cmake
+
+# Compile & install  
+cmake -DCOMPUTE_BACKEND=npu -S .
+make
+pip install .
+```
+
+2. Install transformers from the main branch.
+
+```bash
+git clone -b main https://github.com/huggingface/transformers.git
+cd transformers
+pip install .
+```
+
+3. Set `double_quantization: false` in the configuration. You can refer to the [example](examples/train_qlora/llama3_lora_sft_bnb_npu.yaml).
+
 </details>
 
 ### Data Preparation
 
-Please refer to [data/README.md](data/README.md) for checking the details about the format of dataset files. You can either use datasets on HuggingFace / ModelScope / Modelers hub or load the dataset in local disk.
+Please refer to [data/README.md](data/README.md) for checking the details about the format of dataset files. You can use datasets on HuggingFace / ModelScope / Modelers hub, load the dataset in local disk, or specify a path to s3/gcs cloud storage.
 
 > [!NOTE]
 > Please update `data/dataset_info.json` to use your custom dataset.
 
+You can also use **[Easy Dataset](https://github.com/ConardLi/easy-dataset)**, **[DataFlow](https://github.com/OpenDCAI/DataFlow)** and **[GraphGen](https://github.com/open-sciencelab/GraphGen)** to create synthetic data for fine-tuning.
+
 ### Quickstart
 
 Use the following 3 commands to run LoRA **fine-tuning**, **inference** and **merging** of the Llama3-8B-Instruct model, respectively.
@@ -446,6 +663,8 @@ See [examples/README.md](examples/README.md) for advanced usage (including distr
 
 > [!TIP]
 > Use `llamafactory-cli help` to show help information.
+>
+> Read [FAQs](https://github.com/hiyouga/LLaMA-Factory/issues/4614) first if you encounter any problems.
 
 ### Fine-Tuning with LLaMA Board GUI (powered by [Gradio](https://github.com/gradio-app/gradio))
 
@@ -453,6 +672,10 @@ See [examples/README.md](examples/README.md) for advanced usage (including distr
 llamafactory-cli webui
 ```
 
+### LLaMA Factory Online
+
+Read our [documentation](https://docs.llamafactory.com.cn/docs/documents/quickstart/getstarted/?utm_source=LLaMA-Factory).
+
 ### Build Docker
 
 For CUDA users:
@@ -485,22 +708,13 @@ For CUDA users:
 
 ```bash
 docker build -f ./docker/docker-cuda/Dockerfile \
-    --build-arg INSTALL_BNB=false \
-    --build-arg INSTALL_VLLM=false \
-    --build-arg INSTALL_DEEPSPEED=false \
-    --build-arg INSTALL_FLASHATTN=false \
     --build-arg PIP_INDEX=https://pypi.org/simple \
+    --build-arg EXTRAS=metrics \
     -t llamafactory:latest .
 
-docker run -dit --gpus=all \
-    -v ./hf_cache:/root/.cache/huggingface \
-    -v ./ms_cache:/root/.cache/modelscope \
-    -v ./om_cache:/root/.cache/openmind \
-    -v ./data:/app/data \
-    -v ./output:/app/output \
+docker run -dit --ipc=host --gpus=all \
     -p 7860:7860 \
     -p 8000:8000 \
-    --shm-size 16G \
     --name llamafactory \
     llamafactory:latest
 
@@ -510,19 +724,12 @@ docker exec -it llamafactory bash
 For Ascend NPU users:
 
 ```bash
-# Choose docker image upon your environment
 docker build -f ./docker/docker-npu/Dockerfile \
-    --build-arg INSTALL_DEEPSPEED=false \
     --build-arg PIP_INDEX=https://pypi.org/simple \
+    --build-arg EXTRAS=torch-npu,metrics \
     -t llamafactory:latest .
 
-# Change `device` upon your resources
-docker run -dit \
-    -v ./hf_cache:/root/.cache/huggingface \
-    -v ./ms_cache:/root/.cache/modelscope \
-    -v ./om_cache:/root/.cache/openmind \
-    -v ./data:/app/data \
-    -v ./output:/app/output \
+docker run -dit --ipc=host \
     -v /usr/local/dcmi:/usr/local/dcmi \
     -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
     -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
@@ -533,7 +740,6 @@ docker run -dit \
     --device /dev/davinci_manager \
     --device /dev/devmm_svm \
     --device /dev/hisi_hdc \
-    --shm-size 16G \
     --name llamafactory \
     llamafactory:latest
 
@@ -544,25 +750,15 @@ For AMD ROCm users:
 
 ```bash
 docker build -f ./docker/docker-rocm/Dockerfile \
-    --build-arg INSTALL_BNB=false \
-    --build-arg INSTALL_VLLM=false \
-    --build-arg INSTALL_DEEPSPEED=false \
-    --build-arg INSTALL_FLASHATTN=false \
     --build-arg PIP_INDEX=https://pypi.org/simple \
+    --build-arg EXTRAS=metrics \
     -t llamafactory:latest .
 
-docker run -dit \
-    -v ./hf_cache:/root/.cache/huggingface \
-    -v ./ms_cache:/root/.cache/modelscope \
-    -v ./om_cache:/root/.cache/openmind \
-    -v ./data:/app/data \
-    -v ./output:/app/output \
-    -v ./saves:/app/saves \
+docker run -dit --ipc=host \
     -p 7860:7860 \
     -p 8000:8000 \
     --device /dev/kfd \
     --device /dev/dri \
-    --shm-size 16G \
     --name llamafactory \
     llamafactory:latest
 
@@ -571,12 +767,14 @@ docker exec -it llamafactory bash
 
 </details>
 
-<details><summary>Details about volume</summary>
+<details><summary>Use Docker volumes</summary>
 
-- `hf_cache`: Utilize Hugging Face cache on the host machine. Reassignable if a cache already exists in a different directory.
-- `ms_cache`: Similar to Hugging Face cache but for ModelScope users.
-- `om_cache`: Similar to Hugging Face cache but for Modelers users.
-- `data`: Place datasets on this dir of the host machine so that they can be selected on LLaMA Board GUI.
+You can uncomment `VOLUME [ "/root/.cache/huggingface", "/app/shared_data", "/app/output" ]` in the Dockerfile to use data volumes.
+
+When building the Docker image, use `-v ./hf_cache:/root/.cache/huggingface` argument to mount the local directory to the container. The following data volumes are available.
+
+- `hf_cache`: Utilize Hugging Face cache on the host machine.
+- `shared_data`: The directionary to store datasets on the host machine.
 - `output`: Set export dir to this location so that the merged result can be accessed directly on the host machine.
 
 </details>
@@ -584,13 +782,13 @@ docker exec -it llamafactory bash
 ### Deploy with OpenAI-style API and vLLM
 
 ```bash
-API_PORT=8000 llamafactory-cli api examples/inference/llama3_vllm.yaml
+API_PORT=8000 llamafactory-cli api examples/inference/llama3.yaml infer_backend=vllm vllm_enforce_eager=true
 ```
 
 > [!TIP]
 > Visit [this page](https://platform.openai.com/docs/api-reference/chat/create) for API document.
 >
-> Examples: [Image understanding](scripts/test_image.py) | [Function calling](scripts/test_toolcall.py)
+> Examples: [Image understanding](scripts/api_example/test_image.py) | [Function calling](scripts/api_example/test_toolcall.py)
 
 ### Download from ModelScope Hub
 
@@ -623,6 +821,21 @@ run_name: test_run # optional
 
 Set `WANDB_API_KEY` to [your key](https://wandb.ai/authorize) when launching training tasks to log in with your W&B account.
 
+### Use SwanLab Logger
+
+To use [SwanLab](https://github.com/SwanHubX/SwanLab) for logging experimental results, you need to add the following arguments to yaml files.
+
+```yaml
+use_swanlab: true
+swanlab_run_name: test_run # optional
+```
+
+When launching training tasks, you can log in to SwanLab in three ways:
+
+1. Add `swanlab_api_key=<your_api_key>` to the yaml file, and set it to your [API key](https://swanlab.cn/settings).
+2. Set the environment variable `SWANLAB_API_KEY` to your [API key](https://swanlab.cn/settings).
+3. Use the `swanlab login` command to complete the login.
+
 ## Projects using LLaMA Factory
 
 If you have a project that should be incorporated, please contact via email or create a pull request.
@@ -711,6 +924,7 @@ If you have a project that should be incorporated, please contact via email or c
 1. Xia et al. Using Pre-trained Language Model for Accurate ESG Prediction. FinNLP 2024. [[paper]](https://aclanthology.org/2024.finnlp-2.1/)
 1. Liang et al. I-SHEEP: Self-Alignment of LLM from Scratch through an Iterative Self-Enhancement Paradigm. 2024. [[arxiv]](https://arxiv.org/abs/2408.08072)
 1. Bai et al. Aligning Large Language Model with Direct Multi-Preference Optimization for Recommendation. CIKM 2024. [[paper]](https://dl.acm.org/doi/10.1145/3627673.3679611)
+1. Zhang et al. CPsyCoun: A Report-based Multi-turn Dialogue Reconstruction and Evaluation Framework for Chinese Psychological Counseling. ACL 2024. [[paper]](https://aclanthology.org/2024.findings-acl.830.pdf)
 1. **[StarWhisper](https://github.com/Yu-Yang-Li/StarWhisper)**: A large language model for Astronomy, based on ChatGLM2-6B and Qwen-14B.
 1. **[DISC-LawLLM](https://github.com/FudanDISC/DISC-LawLLM)**: A large language model specialized in Chinese legal domain, based on Baichuan-13B, is capable of retrieving and reasoning on legal knowledge.
 1. **[Sunsimiao](https://github.com/X-D-Lab/Sunsimiao)**: A large language model specialized in Chinese medical domain, based on Baichuan-7B and ChatGLM-6B.
@@ -722,15 +936,17 @@ If you have a project that should be incorporated, please contact via email or c
 1. **[NVIDIA RTX AI Toolkit](https://github.com/NVIDIA/RTX-AI-Toolkit)**: SDKs for fine-tuning LLMs on Windows PC for NVIDIA RTX.
 1. **[LazyLLM](https://github.com/LazyAGI/LazyLLM)**: An easy and lazy way for building multi-agent LLMs applications and supports model fine-tuning via LLaMA Factory.
 1. **[RAG-Retrieval](https://github.com/NLPJCL/RAG-Retrieval)**: A full pipeline for RAG retrieval model fine-tuning, inference, and distillation. [[blog]](https://zhuanlan.zhihu.com/p/987727357)
-
-
+1. **[360-LLaMA-Factory](https://github.com/Qihoo360/360-LLaMA-Factory)**: A modified library that supports long sequence SFT & DPO using ring attention.
+1. **[Sky-T1](https://novasky-ai.github.io/posts/sky-t1/)**: An o1-like model fine-tuned by NovaSky AI with very small cost.
+1. **[WeClone](https://github.com/xming521/WeClone)**: One-stop solution for creating your digital avatar from chat logs.
+1. **[EmoLLM](https://github.com/SmartFlowAI/EmoLLM)**: A project about large language models (LLMs) and mental health.
 </details>
 
 ## License
 
 This repository is licensed under the [Apache-2.0 License](LICENSE).
 
-Please follow the model licenses to use the corresponding model weights: [Baichuan 2](https://huggingface.co/baichuan-inc/Baichuan2-7B-Base/blob/main/Community%20License%20for%20Baichuan%202%20Model.pdf) / [BLOOM](https://huggingface.co/spaces/bigscience/license) / [ChatGLM3](https://github.com/THUDM/ChatGLM3/blob/main/MODEL_LICENSE) / [Command R](https://cohere.com/c4ai-cc-by-nc-license) / [DeepSeek](https://github.com/deepseek-ai/DeepSeek-LLM/blob/main/LICENSE-MODEL) / [Falcon](https://huggingface.co/tiiuae/falcon-180B/blob/main/LICENSE.txt) / [Gemma](https://ai.google.dev/gemma/terms) / [GLM-4](https://huggingface.co/THUDM/glm-4-9b/blob/main/LICENSE) / [Index](https://huggingface.co/IndexTeam/Index-1.9B/blob/main/LICENSE) / [InternLM2](https://github.com/InternLM/InternLM#license) / [Llama](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) / [Llama 2 (LLaVA-1.5)](https://ai.meta.com/llama/license/) / [Llama 3](https://llama.meta.com/llama3/license/) / [MiniCPM](https://github.com/OpenBMB/MiniCPM/blob/main/MiniCPM%20Model%20License.md) / [Mistral/Mixtral/Pixtral](LICENSE) / [OLMo](LICENSE) / [Phi-1.5/Phi-2](https://huggingface.co/microsoft/phi-1_5/resolve/main/Research%20License.docx) / [Phi-3](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/LICENSE) / [Qwen](https://github.com/QwenLM/Qwen/blob/main/Tongyi%20Qianwen%20LICENSE%20AGREEMENT) / [StarCoder 2](https://huggingface.co/spaces/bigcode/bigcode-model-license-agreement) / [XVERSE](https://github.com/xverse-ai/XVERSE-13B/blob/main/MODEL_LICENSE.pdf) / [Yi](https://huggingface.co/01-ai/Yi-6B/blob/main/LICENSE) / [Yi-1.5](LICENSE) / [Yuan 2](https://github.com/IEIT-Yuan/Yuan-2.0/blob/main/LICENSE-Yuan)
+Please follow the model licenses to use the corresponding model weights: [Baichuan 2](https://huggingface.co/baichuan-inc/Baichuan2-7B-Base/blob/main/Community%20License%20for%20Baichuan%202%20Model.pdf) / [BLOOM](https://huggingface.co/spaces/bigscience/license) / [ChatGLM3](https://github.com/THUDM/ChatGLM3/blob/main/MODEL_LICENSE) / [Command R](https://cohere.com/c4ai-cc-by-nc-license) / [DeepSeek](https://github.com/deepseek-ai/DeepSeek-LLM/blob/main/LICENSE-MODEL) / [Falcon](https://huggingface.co/tiiuae/falcon-180B/blob/main/LICENSE.txt) / [Gemma](https://ai.google.dev/gemma/terms) / [GLM-4](https://huggingface.co/THUDM/glm-4-9b/blob/main/LICENSE) / [GPT-2](https://github.com/openai/gpt-2/blob/master/LICENSE) / [Granite](LICENSE) / [Index](https://huggingface.co/IndexTeam/Index-1.9B/blob/main/LICENSE) / [InternLM](https://github.com/InternLM/InternLM#license) / [Llama](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) / [Llama 2](https://ai.meta.com/llama/license/) / [Llama 3](https://llama.meta.com/llama3/license/) / [Llama 4](https://github.com/meta-llama/llama-models/blob/main/models/llama4/LICENSE) / [MiniCPM](https://github.com/OpenBMB/MiniCPM/blob/main/MiniCPM%20Model%20License.md) / [Mistral/Mixtral/Pixtral](LICENSE) / [OLMo](LICENSE) / [Phi-1.5/Phi-2](https://huggingface.co/microsoft/phi-1_5/resolve/main/Research%20License.docx) / [Phi-3/Phi-4](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/LICENSE) / [Qwen](https://github.com/QwenLM/Qwen/blob/main/Tongyi%20Qianwen%20LICENSE%20AGREEMENT) / [Skywork](https://huggingface.co/Skywork/Skywork-13B-base/blob/main/Skywork%20Community%20License.pdf) / [StarCoder 2](https://huggingface.co/spaces/bigcode/bigcode-model-license-agreement) / [TeleChat2](https://huggingface.co/Tele-AI/telechat-7B/blob/main/TeleChat%E6%A8%A1%E5%9E%8B%E7%A4%BE%E5%8C%BA%E8%AE%B8%E5%8F%AF%E5%8D%8F%E8%AE%AE.pdf) / [XVERSE](https://github.com/xverse-ai/XVERSE-13B/blob/main/MODEL_LICENSE.pdf) / [Yi](https://huggingface.co/01-ai/Yi-6B/blob/main/LICENSE) / [Yi-1.5](LICENSE) / [Yuan 2](https://github.com/IEIT-Yuan/Yuan-2.0/blob/main/LICENSE-Yuan)
 
 ## Citation
 

README_zh.md

@@ -1,41 +1,61 @@
 ![# LLaMA Factory](assets/logo.png)
 
 [![GitHub Repo stars](https://img.shields.io/github/stars/hiyouga/LLaMA-Factory?style=social)](https://github.com/hiyouga/LLaMA-Factory/stargazers)
-[![GitHub Code License](https://img.shields.io/github/license/hiyouga/LLaMA-Factory)](LICENSE)
 [![GitHub last commit](https://img.shields.io/github/last-commit/hiyouga/LLaMA-Factory)](https://github.com/hiyouga/LLaMA-Factory/commits/main)
+[![GitHub contributors](https://img.shields.io/github/contributors/hiyouga/LLaMA-Factory?color=orange)](https://github.com/hiyouga/LLaMA-Factory/graphs/contributors)
+[![GitHub workflow](https://github.com/hiyouga/LLaMA-Factory/actions/workflows/tests.yml/badge.svg)](https://github.com/hiyouga/LLaMA-Factory/actions/workflows/tests.yml)
 [![PyPI](https://img.shields.io/pypi/v/llamafactory)](https://pypi.org/project/llamafactory/)
-[![Citation](https://img.shields.io/badge/citation-93-green)](#使用了-llama-factory-的项目)
-[![GitHub pull request](https://img.shields.io/badge/PRs-welcome-blue)](https://github.com/hiyouga/LLaMA-Factory/pulls)
-[![Discord](https://dcbadge.vercel.app/api/server/rKfvV9r9FK?compact=true&style=flat)](https://discord.gg/rKfvV9r9FK)
+[![Citation](https://img.shields.io/badge/citation-840-green)](https://scholar.google.com/scholar?cites=12620864006390196564)
+[![Docker Pulls](https://img.shields.io/docker/pulls/hiyouga/llamafactory)](https://hub.docker.com/r/hiyouga/llamafactory/tags)
+
 [![Twitter](https://img.shields.io/twitter/follow/llamafactory_ai)](https://twitter.com/llamafactory_ai)
-[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1d5KQtbemerlSDSxZIfAaWXhKr30QypiK?usp=sharing)
-[![Open in DSW](https://gallery.pai-ml.com/assets/open-in-dsw.svg)](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory)
-[![Spaces](https://img.shields.io/badge/🤗-Open%20in%20Spaces-blue)](https://huggingface.co/spaces/hiyouga/LLaMA-Board)
-[![Studios](https://img.shields.io/badge/ModelScope-Open%20in%20Studios-blue)](https://modelscope.cn/studios/hiyouga/LLaMA-Board)
-[![SageMaker](https://img.shields.io/badge/SageMaker-Open%20in%20AWS-blue)](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/)
+[![Discord](assets/thirdparty/discord.svg)](https://discord.gg/rKfvV9r9FK)
 
-[![GitHub Tread](https://trendshift.io/api/badge/repositories/4535)](https://trendshift.io/repositories/4535)
+[![Open in Colab](assets/thirdparty/colab.svg)](https://colab.research.google.com/drive/1d5KQtbemerlSDSxZIfAaWXhKr30QypiK?usp=sharing)
+[![Open in DSW](assets/thirdparty/dsw.svg)](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory)
+[![Open in Lab4ai](assets/thirdparty/lab4ai.svg)](https://www.lab4ai.cn/course/detail?id=7c13e60f6137474eb40f6fd3983c0f46&utm_source=LLaMA-Factory)
+[![Open in Online](assets/thirdparty/online.svg)](https://www.llamafactory.com.cn/?utm_source=LLaMA-Factory)
+[![Open in Spaces](https://img.shields.io/badge/🤗-Open%20in%20Spaces-blue)](https://huggingface.co/spaces/hiyouga/LLaMA-Board)
+[![Open in Studios](https://img.shields.io/badge/ModelScope-Open%20in%20Studios-blue)](https://modelscope.cn/studios/hiyouga/LLaMA-Board)
+[![Open in Novita](https://img.shields.io/badge/Novita-Deploy%20Template-blue)](https://novita.ai/templates-library/105981?sharer=88115474-394e-4bda-968e-b88e123d0c47)
 
-👋 加入我们的[微信群](assets/wechat.jpg)或 [NPU 用户群](assets/wechat_npu.jpg)。
+### 获得[亚马逊](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/)、[英伟达](https://developer.nvidia.cn/rtx/ai-toolkit)、[阿里云](https://help.aliyun.com/zh/pai/use-cases/fine-tune-a-llama-3-model-with-llama-factory)等的应用。
+
+<div align="center" markdown="1">
+
+### 赞助商 ❤️
+
+| <div style="text-align: center;"><a href="https://warp.dev/llama-factory"><img alt="Warp sponsorship" width="400" src="assets/sponsors/warp.jpg"></a><br><a href="https://warp.dev/llama-factory" style="font-size:larger;">Warp，面向开发者的智能终端</a><br><a href="https://warp.dev/llama-factory">适用于 MacOS、Linux 和 Windows</a> | <a href="https://serpapi.com"><img alt="SerpAPI sponsorship" width="250" src="assets/sponsors/serpapi.svg"> </a> |
+| ---- | ---- |
+
+----
+
+### 使用零代码[命令行](#快速开始)与 [Web UI](#llama-board-可视化微调由-gradio-驱动) 轻松微调百余种大模型
+
+![GitHub Trend](https://trendshift.io/api/badge/repositories/4535)
+
+</div>
+
+👋 加入我们的[微信群](https://github.com/hiyouga/llamafactory-community/blob/main/wechat/main.jpg)、[NPU 用户群](https://github.com/hiyouga/llamafactory-community/blob/main/wechat/npu.jpg)、[大模型实验室群](https://github.com/hiyouga/llamafactory-community/blob/main/wechat/lab4ai.jpg) 或 [LLaMA Factory Online 用户群](https://github.com/hiyouga/llamafactory-community/blob/main/wechat/online.png)。
 
 \[ [English](README.md) | 中文 \]
 
 **微调大模型可以像这样轻松…**
 
-https://github.com/user-attachments/assets/e6ce34b0-52d5-4f3e-a830-592106c4c272
+https://github.com/user-attachments/assets/43b700c6-a178-41db-b1f8-8190a5d3fcfc
 
 选择你的打开方式：
 
 - **入门教程**：https://zhuanlan.zhihu.com/p/695287607
+- **微调视频教程**：https://www.bilibili.com/video/BV1djgRzxEts/
 - **框架文档**：https://llamafactory.readthedocs.io/zh-cn/latest/
-- **Colab**：https://colab.research.google.com/drive/1d5KQtbemerlSDSxZIfAaWXhKr30QypiK?usp=sharing
+- **框架文档（昇腾 NPU）**：https://ascend.github.io/docs/sources/llamafactory/
+- **Colab（免费）**：https://colab.research.google.com/drive/1d5KQtbemerlSDSxZIfAaWXhKr30QypiK?usp=sharing
 - **本地机器**：请见[如何使用](#如何使用)
-- **PAI-DSW**：[Llama3 案例](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory) | [Qwen2-VL 案例](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_qwen2vl)
-- **Amazon SageMaker**：[博客](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/)
-
-近期活动：
-
-- **2024/10/18-2024/11/30**：使用 PAI+LLaMA Factory 构建个性化导游机器人。[[活动页面]](https://developer.aliyun.com/topic/llamafactory2)
+- **PAI-DSW（免费试用）**：https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory
+- **九章智算云（算力优惠活动）**：https://docs.alayanew.com/docs/documents/useGuide/LLaMAFactory/mutiple/?utm_source=LLaMA-Factory
+- **官方课程**：https://www.lab4ai.cn/course/detail?id=7c13e60f6137474eb40f6fd3983c0f46&utm_source=LLaMA-Factory
+- **LLaMA Factory Online（在线微调）**：https://www.llamafactory.com.cn/?utm_source=LLaMA-Factory
 
 > [!NOTE]
 > 除上述链接以外的其他网站均为未经许可的第三方网站，请小心甄别。
@@ -43,13 +63,24 @@ https://github.com/user-attachments/assets/e6ce34b0-52d5-4f3e-a830-592106c4c272
 ## 目录
 
 - [项目特色](#项目特色)
-- [性能指标](#性能指标)
+- [官方博客](#官方博客)
 - [更新日志](#更新日志)
 - [模型](#模型)
 - [训练方法](#训练方法)
 - [数据集](#数据集)
 - [软硬件依赖](#软硬件依赖)
 - [如何使用](#如何使用)
+  - [安装 LLaMA Factory](#安装-llama-factory)
+  - [数据准备](#数据准备)
+  - [快速开始](#快速开始)
+  - [LLaMA Board 可视化微调](#llama-board-可视化微调由-gradio-驱动)
+  - [LLaMA Factory Online 在线微调](#llama-factory-online-在线微调)
+  - [构建 Docker](#构建-docker)
+  - [利用 vLLM 部署 OpenAI API](#利用-vllm-部署-openai-api)
+  - [从魔搭社区下载](#从魔搭社区下载)
+  - [从魔乐社区下载](#从魔乐社区下载)
+  - [使用 W&B 面板](#使用-wb-面板)
+  - [使用 SwanLab 面板](#使用-swanlab-面板)
 - [使用了 LLaMA Factory 的项目](#使用了-llama-factory-的项目)
 - [协议](#协议)
 - [引用](#引用)
@@ -57,31 +88,89 @@ https://github.com/user-attachments/assets/e6ce34b0-52d5-4f3e-a830-592106c4c272
 
 ## 项目特色
 
-- **多种模型**：LLaMA、LLaVA、Mistral、Mixtral-MoE、Qwen、Qwen2-VL、Yi、Gemma、Baichuan、ChatGLM、Phi 等等。
+- **多种模型**：LLaMA、LLaVA、Mistral、Mixtral-MoE、Qwen、Qwen2-VL、DeepSeek、Yi、Gemma、ChatGLM、Phi 等等。
 - **集成方法**：（增量）预训练、（多模态）指令监督微调、奖励模型训练、PPO 训练、DPO 训练、KTO 训练、ORPO 训练等等。
 - **多种精度**：16 比特全参数微调、冻结微调、LoRA 微调和基于 AQLM/AWQ/GPTQ/LLM.int8/HQQ/EETQ 的 2/3/4/5/6/8 比特 QLoRA 微调。
-- **先进算法**：[GaLore](https://github.com/jiaweizzhao/GaLore)、[BAdam](https://github.com/Ledzy/BAdam)、[Adam-mini](https://github.com/zyushun/Adam-mini)、DoRA、LongLoRA、LLaMA Pro、Mixture-of-Depths、LoRA+、LoftQ、PiSSA 和 Agent 微调。
+- **先进算法**：[GaLore](https://github.com/jiaweizzhao/GaLore)、[BAdam](https://github.com/Ledzy/BAdam)、[APOLLO](https://github.com/zhuhanqing/APOLLO)、[Adam-mini](https://github.com/zyushun/Adam-mini)、[Muon](https://github.com/KellerJordan/Muon)、[OFT](https://github.com/huggingface/peft/tree/main/src/peft/tuners/oft)、DoRA、LongLoRA、LLaMA Pro、Mixture-of-Depths、LoRA+、LoftQ 和 PiSSA。
 - **实用技巧**：[FlashAttention-2](https://github.com/Dao-AILab/flash-attention)、[Unsloth](https://github.com/unslothai/unsloth)、[Liger Kernel](https://github.com/linkedin/Liger-Kernel)、RoPE scaling、NEFTune 和 rsLoRA。
-- **实验监控**：LlamaBoard、TensorBoard、Wandb、MLflow 等等。
-- **极速推理**：基于 vLLM 的 OpenAI 风格 API、浏览器界面和命令行接口。
+- **广泛任务**：多轮对话、工具调用、图像理解、视觉定位、视频识别和语音理解等等。
+- **实验监控**：LlamaBoard、TensorBoard、Wandb、MLflow、[SwanLab](https://github.com/SwanHubX/SwanLab) 等等。
+- **极速推理**：基于 [vLLM](https://github.com/vllm-project/vllm) 或 [SGLang](https://github.com/sgl-project/sglang) 的 OpenAI 风格 API、浏览器界面和命令行接口。
 
-## 性能指标
+### 最新模型的 Day-N 微调适配
 
-与 ChatGLM 官方的 [P-Tuning](https://github.com/THUDM/ChatGLM2-6B/tree/main/ptuning) 微调相比，LLaMA Factory 的 LoRA 微调提供了 **3.7 倍**的加速比，同时在广告文案生成任务上取得了更高的 Rouge 分数。结合 4 比特量化技术，LLaMA Factory 的 QLoRA 微调进一步降低了 GPU 显存消耗。
+| 适配时间      | 模型名称                                                              |
+| ------------ | -------------------------------------------------------------------- |
+| Day 0        | Qwen3 / Qwen2.5-VL / Gemma 3 / GLM-4.1V / InternLM 3 / MiniCPM-o-2.6 |
+| Day 1        | Llama 3 / GLM-4 / Mistral Small / PaliGemma2 / Llama 4               |
 
-![benchmark](assets/benchmark.svg)
+## 官方博客
 
-<details><summary>变量定义</summary>
+- 💡 [Easy Dataset × LLaMA Factory: 让大模型高效学习领域知识](https://buaa-act.feishu.cn/wiki/KY9xwTGs1iqHrRkjXBwcZP9WnL9)（中文）
+- [使用 LLaMA-Factory 微调心理健康大模型](https://www.lab4ai.cn/project/detail?id=25cce32ec131497b9e06a93336a0817f&type=project&utm_source=LLaMA-Factory)（中文）
+- [使用 LLaMA-Factory 构建 GPT-OSS 角色扮演模型](https://docs.llamafactory.com.cn/docs/documents/best-practice/gptroleplay/?utm_source=LLaMA-Factory)（中文）
+- [基于 LLaMA-Factory 和 EasyR1 打造一站式无代码大模型强化学习和部署平台 LLM Model Hub](https://aws.amazon.com/cn/blogs/china/building-llm-model-hub-based-on-llamafactory-and-easyr1/)（中文）
+- [通过亚马逊 SageMaker HyperPod 上的 LLaMA-Factory 增强多模态模型银行文档的视觉信息提取](https://aws.amazon.com/cn/blogs/machine-learning/how-apoidea-group-enhances-visual-information-extraction-from-banking-documents-with-multimodal-models-using-llama-factory-on-amazon-sagemaker-hyperpod/)（英文）
 
-- **Training Speed**: 训练阶段每秒处理的样本数量。（批处理大小=4，截断长度=1024）
-- **Rouge Score**: [广告文案生成](https://aclanthology.org/D19-1321.pdf)任务验证集上的 Rouge-2 分数。（批处理大小=4，截断长度=1024）
-- **GPU Memory**: 4 比特量化训练的 GPU 显存峰值。（批处理大小=1，截断长度=1024）
-- 我们在 ChatGLM 的 P-Tuning 中采用 `pre_seq_len=128`，在 LLaMA Factory 的 LoRA 微调中采用 `lora_rank=32`。
+<details><summary>全部博客</summary>
+
+- [使用 LLaMA-Factory 微调 Llama3.1-70B 医学诊断模型](https://docs.alayanew.com/docs/documents/bestPractice/bigModel/llama70B/?utm_source=LLaMA-Factory)（中文）
+- [使用 LLaMA-Factory 微调 Qwen2.5-VL 实现自动驾驶场景微调](https://docs.alayanew.com/docs/documents/useGuide/LLaMAFactory/mutiple/?utm_source=LLaMA-Factory)（中文）
+- [LLaMA Factory：微调 DeepSeek-R1-Distill-Qwen-7B 模型实现新闻标题分类器](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_deepseek_r1_distill_7b)（中文）
+- [基于 Amazon SageMaker 和 LLaMA-Factory 打造一站式无代码模型微调部署平台 Model Hub](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/)（中文）
+- [LLaMA Factory 多模态微调实践：微调 Qwen2-VL 构建文旅大模型](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_qwen2vl)（中文）
+- [LLaMA Factory：微调 Llama3 模型实现角色扮演](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory)（中文）
 
 </details>
 
 ## 更新日志
 
+[25/08/22] 我们支持了 **[OFT](https://arxiv.org/abs/2306.07280)** 和 **[OFTv2](https://arxiv.org/abs/2506.19847)** 模型的微调。查看 [examples](examples/README.md) 以使用。
+
+[25/08/20] 我们支持了 **[Intern-S1-mini](https://huggingface.co/internlm/Intern-S1-mini)** 模型的微调。查看 [PR #8976](https://github.com/hiyouga/LLaMA-Factory/pull/8976) 以使用。
+
+[25/08/06] 我们支持了 **[GPT-OSS](https://github.com/openai/gpt-oss)** 模型的微调。查看 [PR #8826](https://github.com/hiyouga/LLaMA-Factory/pull/8826) 以使用。
+
+<details><summary>展开日志</summary>
+
+[25/07/02] 我们支持了 **[GLM-4.1V-9B-Thinking](https://github.com/THUDM/GLM-4.1V-Thinking)** 模型的微调。
+
+[25/04/28] 我们支持了 **[Qwen3](https://qwenlm.github.io/blog/qwen3/)** 系列模型的微调。
+
+[25/04/21] 我们支持了 **[Muon](https://github.com/KellerJordan/Muon)** 优化器。详细用法请参照 [examples](examples/README_zh.md)。感谢 [@tianshijing](https://github.com/tianshijing) 的 PR。
+
+[25/04/16] 我们支持了 **[InternVL3](https://huggingface.co/OpenGVLab/InternVL3-8B)** 模型的微调。查看 [PR #7258](https://github.com/hiyouga/LLaMA-Factory/pull/7258) 以使用。
+
+[25/04/14] 我们支持了 **[GLM-Z1](https://huggingface.co/THUDM/GLM-Z1-9B-0414)** 和 **[Kimi-VL](https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct)** 模型的微调。
+
+[25/04/06] 我们支持了 **[Llama 4](https://ai.meta.com/blog/llama-4-multimodal-intelligence/)** 模型的微调。查看 [PR #7611](https://github.com/hiyouga/LLaMA-Factory/pull/7611) 以使用。
+
+[25/03/31] 我们支持了 **[Qwen2.5 Omni](https://qwenlm.github.io/blog/qwen2.5-omni/)** 模型的微调。查看 [PR #7537](https://github.com/hiyouga/LLaMA-Factory/pull/7537) 以使用。
+
+[25/03/15] 我们支持了 **[SGLang](https://github.com/sgl-project/sglang)** 推理后端，请使用 `infer_backend: sglang` 启用。
+
+[25/03/12] 我们支持了 **[Gemma 3](https://huggingface.co/blog/gemma3)** 模型的微调。
+
+[25/02/24] 我们宣布开源 **[EasyR1](https://github.com/hiyouga/EasyR1)**，一个高效可扩展的多模态强化学习框架，支持高效的 GRPO 训练。
+
+[25/02/11] 我们支持了在导出模型时保存 **[Ollama](https://github.com/ollama/ollama)** 配置文件。详细用法请参照 [examples](examples/README_zh.md)。
+
+[25/02/05] 我们支持了在语音理解任务上微调 **[Qwen2-Audio](Qwen/Qwen2-Audio-7B-Instruct)** 和 **[MiniCPM-o-2.6](https://huggingface.co/openbmb/MiniCPM-o-2_6)** 模型。
+
+[25/01/31] 我们支持了 **[DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1)** 和 **[Qwen2.5-VL](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct)** 模型的微调。
+
+[25/01/15] 我们支持了 **[APOLLO](https://arxiv.org/abs/2412.05270)** 优化器。详细用法请参照 [examples](examples/README_zh.md)。
+
+[25/01/14] 我们支持了 **[MiniCPM-o-2.6](https://huggingface.co/openbmb/MiniCPM-o-2_6)** 和 **[MiniCPM-V-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6)** 模型的微调。 感谢 [@BUAADreamer](https://github.com/BUAADreamer) 的 PR.
+
+[25/01/14] 我们支持了 **[InternLM 3](https://huggingface.co/collections/internlm/)** 模型的微调。感谢 [@hhaAndroid](https://github.com/hhaAndroid) 的 PR。
+
+[25/01/10] 我们支持了 **[Phi-4](https://huggingface.co/microsoft/phi-4)** 模型的微调。
+
+[24/12/21] 我们支持了使用 **[SwanLab](https://github.com/SwanHubX/SwanLab)** 跟踪与可视化实验。详细用法请参考 [此部分](#使用-swanlab-面板)。
+
+[24/11/27] 我们支持了 **[Skywork-o1](https://huggingface.co/Skywork/Skywork-o1-Open-Llama-3.1-8B)** 模型的微调和 **[OpenO1](https://huggingface.co/datasets/O1-OPEN/OpenO1-SFT)** 数据集。
+
 [24/10/09] 我们支持了从 **[魔乐社区](https://modelers.cn/models)** 下载预训练模型和数据集。详细用法请参照 [此教程](#从魔乐社区下载)。
 
 [24/09/19] 我们支持了 **[Qwen2.5](https://qwenlm.github.io/blog/qwen2.5/)** 模型的微调。
@@ -92,8 +181,6 @@ https://github.com/user-attachments/assets/e6ce34b0-52d5-4f3e-a830-592106c4c272
 
 [24/08/09] 我们支持了 **[Adam-mini](https://github.com/zyushun/Adam-mini)** 优化器。详细用法请参照 [examples](examples/README_zh.md)。感谢 [@relic-yuexi](https://github.com/relic-yuexi) 的 PR。
 
-<details><summary>展开日志</summary>
-
 [24/07/04] 我们支持了[无污染打包训练](https://github.com/MeetKai/functionary/tree/main/functionary/train/packing)。请使用 `neat_packing: true` 参数。感谢 [@chuan298](https://github.com/chuan298) 的 PR。
 
 [24/06/16] 我们支持了 **[PiSSA](https://arxiv.org/abs/2404.02948)** 算法。详细用法请参照 [examples](examples/README_zh.md)。
@@ -172,46 +259,84 @@ https://github.com/user-attachments/assets/e6ce34b0-52d5-4f3e-a830-592106c4c272
 
 </details>
 
+> [!TIP]
+> 如果您无法使用最新的功能，请尝试重新拉取代码并再次安装 LLaMA-Factory。
+
 ## 模型
 
-| 模型名                                                            | 模型大小                          | Template         |
-| ----------------------------------------------------------------- | -------------------------------- | ---------------- |
-| [Baichuan 2](https://huggingface.co/baichuan-inc)                 | 7B/13B                           | baichuan2        |
-| [BLOOM/BLOOMZ](https://huggingface.co/bigscience)                 | 560M/1.1B/1.7B/3B/7.1B/176B      | -                |
-| [ChatGLM3](https://huggingface.co/THUDM)                          | 6B                               | chatglm3         |
-| [Command R](https://huggingface.co/CohereForAI)                   | 35B/104B                         | cohere           |
-| [DeepSeek (Code/MoE)](https://huggingface.co/deepseek-ai)         | 7B/16B/67B/236B                  | deepseek         |
-| [Falcon](https://huggingface.co/tiiuae)                           | 7B/11B/40B/180B                  | falcon           |
-| [Gemma/Gemma 2/CodeGemma](https://huggingface.co/google)          | 2B/7B/9B/27B                     | gemma            |
-| [GLM-4](https://huggingface.co/THUDM)                             | 9B                               | glm4             |
-| [Index](https://huggingface.co/IndexTeam)                         | 1.9B                             | index            |
-| [InternLM2/InternLM2.5](https://huggingface.co/internlm)          | 7B/20B                           | intern2          |
-| [Llama](https://github.com/facebookresearch/llama)                | 7B/13B/33B/65B                   | -                |
-| [Llama 2](https://huggingface.co/meta-llama)                      | 7B/13B/70B                       | llama2           |
-| [Llama 3-3.2](https://huggingface.co/meta-llama)                  | 1B/3B/8B/70B                     | llama3           |
-| [Llama 3.2 Vision](https://huggingface.co/meta-llama)             | 11B/90B                          | mllama           |
-| [LLaVA-1.5](https://huggingface.co/llava-hf)                      | 7B/13B                           | llava            |
-| [LLaVA-NeXT](https://huggingface.co/llava-hf)                     | 7B/8B/13B/34B/72B/110B           | llava_next       |
-| [LLaVA-NeXT-Video](https://huggingface.co/llava-hf)               | 7B/34B                           | llava_next_video |
-| [MiniCPM](https://huggingface.co/openbmb)                         | 1B/2B/4B                         | cpm/cpm3         |
-| [Mistral/Mixtral](https://huggingface.co/mistralai)               | 7B/8x7B/8x22B                    | mistral          |
-| [OLMo](https://huggingface.co/allenai)                            | 1B/7B                            | -                |
-| [PaliGemma](https://huggingface.co/google)                        | 3B                               | paligemma        |
-| [Phi-1.5/Phi-2](https://huggingface.co/microsoft)                 | 1.3B/2.7B                        | -                |
-| [Phi-3](https://huggingface.co/microsoft)                         | 4B/7B/14B                        | phi              |
-| [Pixtral](https://huggingface.co/mistralai)                       | 12B                              | pixtral          |
-| [Qwen (1-2.5) (Code/Math/MoE)](https://huggingface.co/Qwen)       | 0.5B/1.5B/3B/7B/14B/32B/72B/110B | qwen             |
-| [Qwen2-VL](https://huggingface.co/Qwen)                           | 2B/7B/72B                        | qwen2_vl         |
-| [StarCoder 2](https://huggingface.co/bigcode)                     | 3B/7B/15B                        | -                |
-| [XVERSE](https://huggingface.co/xverse)                           | 7B/13B/65B                       | xverse           |
-| [Yi/Yi-1.5 (Code)](https://huggingface.co/01-ai)                  | 1.5B/6B/9B/34B                   | yi               |
-| [Yi-VL](https://huggingface.co/01-ai)                             | 6B/34B                           | yi_vl            |
-| [Yuan 2](https://huggingface.co/IEITYuan)                         | 2B/51B/102B                      | yuan             |
+| 模型名                                                             | 参数量                            | Template             |
+| ----------------------------------------------------------------- | -------------------------------- | -------------------- |
+| [Baichuan 2](https://huggingface.co/baichuan-inc)                 | 7B/13B                           | baichuan2            |
+| [BLOOM/BLOOMZ](https://huggingface.co/bigscience)                 | 560M/1.1B/1.7B/3B/7.1B/176B      | -                    |
+| [ChatGLM3](https://huggingface.co/THUDM)                          | 6B                               | chatglm3             |
+| [Command R](https://huggingface.co/CohereForAI)                   | 35B/104B                         | cohere               |
+| [DeepSeek (Code/MoE)](https://huggingface.co/deepseek-ai)         | 7B/16B/67B/236B                  | deepseek             |
+| [DeepSeek 2.5/3](https://huggingface.co/deepseek-ai)              | 236B/671B                        | deepseek3            |
+| [DeepSeek R1 (Distill)](https://huggingface.co/deepseek-ai)       | 1.5B/7B/8B/14B/32B/70B/671B      | deepseekr1           |
+| [ERNIE-4.5](https://huggingface.co/baidu)                         | 0.3B/21B/300B                    | ernie/ernie_nothink  |
+| [Falcon](https://huggingface.co/tiiuae)                           | 7B/11B/40B/180B                  | falcon               |
+| [Falcon-H1](https://huggingface.co/tiiuae)                        | 0.5B/1.5B/3B/7B/34B              | falcon_h1            |
+| [Gemma/Gemma 2/CodeGemma](https://huggingface.co/google)          | 2B/7B/9B/27B                     | gemma/gemma2         |
+| [Gemma 3/Gemma 3n](https://huggingface.co/google)                 | 270M/1B/4B/6B/8B/12B/27B         | gemma3/gemma3n       |
+| [GLM-4/GLM-4-0414/GLM-Z1](https://huggingface.co/zai-org)         | 9B/32B                           | glm4/glmz1           |
+| [GLM-4.1V](https://huggingface.co/zai-org)                        | 9B                               | glm4v                |
+| [GLM-4.5/GLM-4.5V](https://huggingface.co/zai-org)                | 106B/355B                        | glm4_moe/glm4v_moe   |
+| [GPT-2](https://huggingface.co/openai-community)                  | 0.1B/0.4B/0.8B/1.5B              | -                    |
+| [GPT-OSS](https://huggingface.co/openai)                          | 20B/120B                         | gpt                  |
+| [Granite 3.0-3.3](https://huggingface.co/ibm-granite)             | 1B/2B/3B/8B                      | granite3             |
+| [Granite 4](https://huggingface.co/ibm-granite)                   | 7B                               | granite4             |
+| [Hunyuan](https://huggingface.co/tencent/)                        | 7B                               | hunyuan              |
+| [Index](https://huggingface.co/IndexTeam)                         | 1.9B                             | index                |
+| [InternLM 2-3](https://huggingface.co/internlm)                   | 7B/8B/20B                        | intern2              |
+| [InternVL 2.5-3.5](https://huggingface.co/OpenGVLab)              | 1B/2B/4B/8B/14B/30B/38B/78B/241B | intern_vl            |
+| [InternLM/Intern-S1-mini](https://huggingface.co/internlm/)       | 8B                               | intern_s1            |
+| [Kimi-VL](https://huggingface.co/moonshotai)                      | 16B                              | kimi_vl              |
+| [Ling 2.0 (mini/flash)](https://huggingface.co/inclusionAI)       | 16B/100B                         | bailing_v2           |
+| [Llama](https://github.com/facebookresearch/llama)                | 7B/13B/33B/65B                   | -                    |
+| [Llama 2](https://huggingface.co/meta-llama)                      | 7B/13B/70B                       | llama2               |
+| [Llama 3-3.3](https://huggingface.co/meta-llama)                  | 1B/3B/8B/70B                     | llama3               |
+| [Llama 4](https://huggingface.co/meta-llama)                      | 109B/402B                        | llama4               |
+| [Llama 3.2 Vision](https://huggingface.co/meta-llama)             | 11B/90B                          | mllama               |
+| [LLaVA-1.5](https://huggingface.co/llava-hf)                      | 7B/13B                           | llava                |
+| [LLaVA-NeXT](https://huggingface.co/llava-hf)                     | 7B/8B/13B/34B/72B/110B           | llava_next           |
+| [LLaVA-NeXT-Video](https://huggingface.co/llava-hf)               | 7B/34B                           | llava_next_video     |
+| [MiMo](https://huggingface.co/XiaomiMiMo)                         | 7B                               | mimo                 |
+| [MiniCPM 1-4.1](https://huggingface.co/openbmb)                   | 0.5B/1B/2B/4B/8B                 | cpm/cpm3/cpm4        |
+| [MiniCPM-o-2.6/MiniCPM-V-2.6](https://huggingface.co/openbmb)     | 8B                               | minicpm_o/minicpm_v  |
+| [Ministral/Mistral-Nemo](https://huggingface.co/mistralai)        | 8B/12B                           | ministral            |
+| [Mistral/Mixtral](https://huggingface.co/mistralai)               | 7B/8x7B/8x22B                    | mistral              |
+| [Mistral Small](https://huggingface.co/mistralai)                 | 24B                              | mistral_small        |
+| [OLMo](https://huggingface.co/allenai)                            | 1B/7B                            | -                    |
+| [PaliGemma/PaliGemma2](https://huggingface.co/google)             | 3B/10B/28B                       | paligemma            |
+| [Phi-1.5/Phi-2](https://huggingface.co/microsoft)                 | 1.3B/2.7B                        | -                    |
+| [Phi-3/Phi-3.5](https://huggingface.co/microsoft)                 | 4B/14B                           | phi                  |
+| [Phi-3-small](https://huggingface.co/microsoft)                   | 7B                               | phi_small            |
+| [Phi-4](https://huggingface.co/microsoft)                         | 14B                              | phi4                 |
+| [Pixtral](https://huggingface.co/mistralai)                       | 12B                              | pixtral              |
+| [Qwen (1-2.5) (Code/Math/MoE/QwQ)](https://huggingface.co/Qwen)   | 0.5B/1.5B/3B/7B/14B/32B/72B/110B | qwen                 |
+| [Qwen3 (MoE/Instruct/Thinking/Next)](https://huggingface.co/Qwen) | 0.6B/1.7B/4B/8B/14B/32B/80B/235B | qwen3/qwen3_nothink  |
+| [Qwen2-Audio](https://huggingface.co/Qwen)                        | 7B                               | qwen2_audio          |
+| [Qwen2.5-Omni](https://huggingface.co/Qwen)                       | 3B/7B                            | qwen2_omni           |
+| [Qwen3-Omni](https://huggingface.co/Qwen)*                        | 30B                              | qwen3_omni           |
+| [Qwen2-VL/Qwen2.5-VL/QVQ](https://huggingface.co/Qwen)            | 2B/3B/7B/32B/72B                 | qwen2_vl             |
+| [Qwen3-VL](https://huggingface.co/Qwen)*                          | 235B                             | qwen3_vl             |
+| [Seed (OSS/Coder)](https://huggingface.co/ByteDance-Seed)         | 8B/36B                           | seed_oss/seed_coder  |
+| [Skywork o1](https://huggingface.co/Skywork)                      | 8B                               | skywork_o1           |
+| [StarCoder 2](https://huggingface.co/bigcode)                     | 3B/7B/15B                        | -                    |
+| [TeleChat2](https://huggingface.co/Tele-AI)                       | 3B/7B/35B/115B                   | telechat2            |
+| [XVERSE](https://huggingface.co/xverse)                           | 7B/13B/65B                       | xverse               |
+| [Yi/Yi-1.5 (Code)](https://huggingface.co/01-ai)                  | 1.5B/6B/9B/34B                   | yi                   |
+| [Yi-VL](https://huggingface.co/01-ai)                             | 6B/34B                           | yi_vl                |
+| [Yuan 2](https://huggingface.co/IEITYuan)                         | 2B/51B/102B                      | yuan                 |
 
 > [!NOTE]
 > 对于所有“基座”（Base）模型，`template` 参数可以是 `default`, `alpaca`, `vicuna` 等任意值。但“对话”（Instruct/Chat）模型请务必使用**对应的模板**。
 >
 > 请务必在训练和推理时采用**完全一致**的模板。
+>
+> \*：您需要从 main 分支安装 `transformers` 并使用 `DISABLE_VERSION_CHECK=1` 来跳过版本检查。
+>
+> \*\*：您需要安装特定版本的 `transformers` 以使用该模型。
 
 项目所支持模型的完整列表请参阅 [constants.py](src/llamafactory/extras/constants.py)。
 
@@ -220,7 +345,7 @@ https://github.com/user-attachments/assets/e6ce34b0-52d5-4f3e-a830-592106c4c272
 ## 训练方法
 
 | 方法                   |     全参数训练      |    部分参数训练     |       LoRA         |       QLoRA        |
-| ---------------------- | ------------------ | ------------------ | ------------------ | ------------------ |
+| --------------------- | ------------------ | ------------------ | ------------------ | ------------------ |
 | 预训练                 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
 | 指令监督微调            | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
 | 奖励模型训练            | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
@@ -246,6 +371,11 @@ https://github.com/user-attachments/assets/e6ce34b0-52d5-4f3e-a830-592106c4c272
 - [SkyPile (zh)](https://huggingface.co/datasets/Skywork/SkyPile-150B)
 - [FineWeb (en)](https://huggingface.co/datasets/HuggingFaceFW/fineweb)
 - [FineWeb-Edu (en)](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu)
+- [CCI3-HQ (zh)](https://huggingface.co/datasets/BAAI/CCI3-HQ)
+- [CCI3-Data (zh)](https://huggingface.co/datasets/BAAI/CCI3-Data)
+- [CCI4.0-M2-Base-v1 (en&zh)](https://huggingface.co/datasets/BAAI/CCI4.0-M2-Base-v1)
+- [CCI4.0-M2-CoT-v1 (en&zh)](https://huggingface.co/datasets/BAAI/CCI4.0-M2-CoT-v1)
+- [CCI4.0-M2-Extra-v1 (en&zh)](https://huggingface.co/datasets/BAAI/CCI4.0-M2-Extra-v1)
 - [The Stack (en)](https://huggingface.co/datasets/bigcode/the-stack)
 - [StarCoder (en)](https://huggingface.co/datasets/bigcode/starcoderdata)
 
@@ -283,6 +413,7 @@ https://github.com/user-attachments/assets/e6ce34b0-52d5-4f3e-a830-592106c4c272
 - [ShareGPT Hyperfiltered (en)](https://huggingface.co/datasets/totally-not-an-llm/sharegpt-hyperfiltered-3k)
 - [ShareGPT4 (en&zh)](https://huggingface.co/datasets/shibing624/sharegpt_gpt4)
 - [UltraChat 200k (en)](https://huggingface.co/datasets/HuggingFaceH4/ultrachat_200k)
+- [Infinity Instruct (zh)](https://huggingface.co/datasets/BAAI/Infinity-Instruct)
 - [AgentInstruct (en)](https://huggingface.co/datasets/THUDM/AgentInstruct)
 - [LMSYS Chat 1M (en)](https://huggingface.co/datasets/lmsys/lmsys-chat-1m)
 - [Evol Instruct V2 (en)](https://huggingface.co/datasets/WizardLM/WizardLM_evol_instruct_V2_196k)
@@ -290,9 +421,13 @@ https://github.com/user-attachments/assets/e6ce34b0-52d5-4f3e-a830-592106c4c272
 - [STEM (zh)](https://huggingface.co/datasets/hfl/stem_zh_instruction)
 - [Ruozhiba (zh)](https://huggingface.co/datasets/hfl/ruozhiba_gpt4_turbo)
 - [Neo-sft (zh)](https://huggingface.co/datasets/m-a-p/neo_sft_phase2)
-- [WebInstructSub (en)](https://huggingface.co/datasets/TIGER-Lab/WebInstructSub)
 - [Magpie-Pro-300K-Filtered (en)](https://huggingface.co/datasets/Magpie-Align/Magpie-Pro-300K-Filtered)
 - [Magpie-ultra-v0.1 (en)](https://huggingface.co/datasets/argilla/magpie-ultra-v0.1)
+- [WebInstructSub (en)](https://huggingface.co/datasets/TIGER-Lab/WebInstructSub)
+- [OpenO1-SFT (en&zh)](https://huggingface.co/datasets/O1-OPEN/OpenO1-SFT)
+- [Open-Thoughts (en)](https://huggingface.co/datasets/open-thoughts/OpenThoughts-114k)
+- [Open-R1-Math (en)](https://huggingface.co/datasets/open-r1/OpenR1-Math-220k)
+- [Chinese-DeepSeek-R1-Distill (zh)](https://huggingface.co/datasets/Congliu/Chinese-DeepSeek-R1-Distill-data-110k-SFT)
 - [LLaVA mixed (en&zh)](https://huggingface.co/datasets/BUAADreamer/llava-en-zh-300k)
 - [Pokemon-gpt4o-captions (en&zh)](https://huggingface.co/datasets/jugg1024/pokemon-gpt4o-captions)
 - [Open Assistant (de)](https://huggingface.co/datasets/mayflowergmbh/oasst_de)
@@ -311,8 +446,10 @@ https://github.com/user-attachments/assets/e6ce34b0-52d5-4f3e-a830-592106c4c272
 
 - [DPO mixed (en&zh)](https://huggingface.co/datasets/hiyouga/DPO-En-Zh-20k)
 - [UltraFeedback (en)](https://huggingface.co/datasets/HuggingFaceH4/ultrafeedback_binarized)
+- [COIG-P (zh)](https://huggingface.co/datasets/m-a-p/COIG-P)
 - [RLHF-V (en)](https://huggingface.co/datasets/openbmb/RLHF-V-Dataset)
 - [VLFeedback (en)](https://huggingface.co/datasets/Zhihui/VLFeedback)
+- [RLAIF-V (en)](https://huggingface.co/datasets/openbmb/RLAIF-V-Dataset)
 - [Orca DPO Pairs (en)](https://huggingface.co/datasets/Intel/orca_dpo_pairs)
 - [HH-RLHF (en)](https://huggingface.co/datasets/Anthropic/hh-rlhf)
 - [Nectar (en)](https://huggingface.co/datasets/berkeley-nest/Nectar)
@@ -330,37 +467,37 @@ huggingface-cli login
 
 ## 软硬件依赖
 
-| 必需项       | 至少     | 推荐      |
+| 必需项        | 至少     | 推荐      |
 | ------------ | ------- | --------- |
-| python       | 3.8     | 3.11      |
-| torch        | 1.13.1  | 2.4.0     |
-| transformers | 4.41.2  | 4.43.4    |
-| datasets     | 2.16.0  | 2.20.0    |
-| accelerate   | 0.30.1  | 0.32.0    |
-| peft         | 0.11.1  | 0.12.0    |
+| python       | 3.9     | 3.10      |
+| torch        | 2.0.0   | 2.6.0     |
+| torchvision  | 0.15.0  | 0.21.0    |
+| transformers | 4.49.0  | 4.50.0    |
+| datasets     | 2.16.0  | 3.2.0     |
+| accelerate   | 0.34.0  | 1.2.1     |
+| peft         | 0.14.0  | 0.15.1    |
 | trl          | 0.8.6   | 0.9.6     |
 
-| 可选项       | 至少     | 推荐      |
+| 可选项        | 至少     | 推荐      |
 | ------------ | ------- | --------- |
 | CUDA         | 11.6    | 12.2      |
-| deepspeed    | 0.10.0  | 0.14.0    |
+| deepspeed    | 0.10.0  | 0.16.4    |
 | bitsandbytes | 0.39.0  | 0.43.1    |
-| vllm         | 0.4.3   | 0.5.0     |
-| flash-attn   | 2.3.0   | 2.6.3     |
+| vllm         | 0.4.3   | 0.8.2     |
+| flash-attn   | 2.5.6   | 2.7.2     |
 
 ### 硬件依赖
 
 \* *估算值*
 
-| 方法               | 精度 |   7B  |  13B  |  30B  |   70B  |  110B  |  8x7B |  8x22B |
-| ----------------- | ---- | ----- | ----- | ----- | ------ | ------ | ----- | ------ |
-| Full              | AMP  | 120GB | 240GB | 600GB | 1200GB | 2000GB | 900GB | 2400GB |
-| Full              |  16  |  60GB | 120GB | 300GB |  600GB |  900GB | 400GB | 1200GB |
-| Freeze            |  16  |  20GB |  40GB |  80GB |  200GB |  360GB | 160GB |  400GB |
-| LoRA/GaLore/BAdam |  16  |  16GB |  32GB |  64GB |  160GB |  240GB | 120GB |  320GB |
-| QLoRA             |   8  |  10GB |  20GB |  40GB |   80GB |  140GB |  60GB |  160GB |
-| QLoRA             |   4  |   6GB |  12GB |  24GB |   48GB |   72GB |  30GB |   96GB |
-| QLoRA             |   2  |   4GB |   8GB |  16GB |   24GB |   48GB |  18GB |   48GB |
+| 方法                             | 精度 |   7B  |  14B  |  30B  |   70B  |   `x`B  |
+| ------------------------------- | ---- | ----- | ----- | ----- | ------ | ------- |
+| Full (`bf16` or `fp16`)         |  32  | 120GB | 240GB | 600GB | 1200GB | `18x`GB |
+| Full (`pure_bf16`)              |  16  |  60GB | 120GB | 300GB |  600GB |  `8x`GB |
+| Freeze/LoRA/GaLore/APOLLO/BAdam |  16  |  16GB |  32GB |  64GB |  160GB |  `2x`GB |
+| QLoRA                           |   8  |  10GB |  20GB |  40GB |   80GB |   `x`GB |
+| QLoRA                           |   4  |   6GB |  12GB |  24GB |   48GB | `x/2`GB |
+| QLoRA                           |   2  |   4GB |   8GB |  16GB |   24GB | `x/4`GB |
 
 ## 如何使用
 
@@ -369,32 +506,77 @@ huggingface-cli login
 > [!IMPORTANT]
 > 此步骤为必需。
 
+#### 从源码安装
+
 ```bash
 git clone --depth 1 https://github.com/hiyouga/LLaMA-Factory.git
 cd LLaMA-Factory
-pip install -e ".[torch,metrics]"
+pip install -e ".[torch,metrics]" --no-build-isolation
 ```
 
-可选的额外依赖项：torch、torch-npu、metrics、deepspeed、liger-kernel、bitsandbytes、hqq、eetq、gptq、awq、aqlm、vllm、galore、badam、adam-mini、qwen、modelscope、openmind、quality
+可选的额外依赖项：torch、torch-npu、metrics、deepspeed、liger-kernel、bitsandbytes、hqq、eetq、gptq、aqlm、vllm、sglang、galore、apollo、badam、adam-mini、qwen、minicpm_v、openmind、swanlab、dev
 
-> [!TIP]
-> 遇到包冲突时，可使用 `pip install --no-deps -e .` 解决。
+#### 从镜像安装
+
+```bash
+docker run -it --rm --gpus=all --ipc=host hiyouga/llamafactory:latest
+```
+
+该镜像基于 Ubuntu 22.04（x86\_64）、CUDA 12.4、Python 3.11、PyTorch 2.6.0 和 Flash-attn 2.7.4 构建。
+
+查看全部镜像：https://hub.docker.com/r/hiyouga/llamafactory/tags
+
+请参阅[构建 Docker](#构建-docker) 来重新构建镜像。
+
+<details><summary>使用 <b>uv</b> 构建虚拟环境</summary>
+
+使用 [uv](https://github.com/astral-sh/uv) 创建隔离的 Python 环境：
+
+```bash
+uv sync --extra torch --extra metrics --prerelease=allow
+```
+
+在环境中运行 LLaMA-Factory：
+
+```bash
+uv run --prerelease=allow llamafactory-cli train examples/train_lora/llama3_lora_pretrain.yaml
+```
+
+</details>
 
 <details><summary>Windows 用户指南</summary>
 
+#### 安装 PyTorch
+
+Windows 平台需要额外手动安装 GPU 版本的 PyTorch 依赖包，您可以参考[官方网站](https://pytorch.org/get-started/locally/)和以下命令安装并测试 PyTorch 是否正确安装。
+
+```bash
+pip uninstall torch torchvision torchaudio
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu126
+python -c "import torch; print(torch.cuda.is_available())"
+```
+
+如果看到 `True` 则说明安装成功。
+
+若遇到类似 `Can't pickle local object` 的报错，请设置 `dataloader_num_workers: 0`。
+
+#### 安装 BitsAndBytes
+
 如果要在 Windows 平台上开启量化 LoRA（QLoRA），需要安装预编译的 `bitsandbytes` 库, 支持 CUDA 11.1 到 12.2, 请根据您的 CUDA 版本情况选择适合的[发布版本](https://github.com/jllllll/bitsandbytes-windows-webui/releases/tag/wheels)。
 
 ```bash
 pip install https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.41.2.post2-py3-none-win_amd64.whl
 ```
 
-如果要在 Windows 平台上开启 FlashAttention-2，需要安装预编译的 `flash-attn` 库，支持 CUDA 12.1 到 12.2，请根据需求到 [flash-attention](https://github.com/bdashore3/flash-attention/releases) 下载对应版本安装。
+#### 安装 Flash Attention-2
+
+如果要在 Windows 平台上开启 FlashAttention-2，请使用 [flash-attention-windows-wheel](https://huggingface.co/lldacing/flash-attention-windows-wheel) 中的脚本自行编译与安装。
 
 </details>
 
 <details><summary>昇腾 NPU 用户指南</summary>
 
-在昇腾 NPU 设备上安装 LLaMA Factory 时，需要指定额外依赖项，使用 `pip install -e ".[torch-npu,metrics]"` 命令安装。此外，还需要安装 **[Ascend CANN Toolkit 与 Kernels](https://www.hiascend.com/developer/download/community/result?module=cann)**，安装方法请参考[安装教程](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/80RC2alpha002/quickstart/quickstart/quickstart_18_0004.html)或使用以下命令：
+在昇腾 NPU 设备上安装 LLaMA Factory 时，请升级 Python 到 3.10 及以上，并需要指定额外依赖项，使用 `pip install -e ".[torch-npu,metrics]"` 命令安装。此外，还需要安装 **[Ascend CANN Toolkit 与 Kernels](https://www.hiascend.com/developer/download/community/result?module=cann)**，安装方法请参考[安装教程](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/80RC2alpha002/quickstart/quickstart/quickstart_18_0004.html)或使用以下命令：
 
 ```bash
 # 请替换 URL 为 CANN 版本和设备型号对应的 URL
@@ -410,12 +592,13 @@ bash Ascend-cann-kernels-910b_8.0.RC1.alpha001_linux.run --install
 source /usr/local/Ascend/ascend-toolkit/set_env.sh
 ```
 
-| 依赖项       | 至少     | 推荐        |
-| ------------ | ------- | ----------- |
-| CANN         | 8.0.RC1 | 8.0.RC1     |
-| torch        | 2.1.0   | 2.1.0       |
-| torch-npu    | 2.1.0   | 2.1.0.post3 |
-| deepspeed    | 0.13.2  | 0.13.2      |
+| 依赖项        | 至少     | 推荐           |
+| ------------ | ------- | -------------- |
+| CANN         | 8.0.RC1 | 8.0.0.alpha002 |
+| torch        | 2.1.0   | 2.4.0          |
+| torch-npu    | 2.1.0   | 2.4.0.post2    |
+| deepspeed    | 0.13.2  | 0.13.2         |
+| vllm-ascend  | -       | 0.7.3          |
 
 请使用 `ASCEND_RT_VISIBLE_DEVICES` 而非 `CUDA_VISIBLE_DEVICES` 来指定运算设备。
 
@@ -423,6 +606,40 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
 
 下载预构建 Docker 镜像：[32GB](http://mirrors.cn-central-221.ovaijisuan.com/detail/130.html) | [64GB](http://mirrors.cn-central-221.ovaijisuan.com/detail/131.html)
 
+#### 安装 BitsAndBytes
+
+如果要在 Ascend NPU 上进行基于 bitsandbytes 的 QLoRA 量化微调，请执行如下步骤：
+
+1. 手动编译 bitsandbytes：请参考[安装文档](https://huggingface.co/docs/bitsandbytes/installation?backend=Ascend+NPU&platform=Ascend+NPU)完成 NPU 版的 bitsandbytes 安装，编译要求环境 cmake 版本不低于 3.22.1，g++ 版本不低于 12.x。
+
+```bash
+# 从源码安装 bitsandbytes
+# 克隆 bitsandbytes 仓库, Ascend NPU 目前在 multi-backend-refactor 中支持
+git clone -b multi-backend-refactor https://github.com/bitsandbytes-foundation/bitsandbytes.git
+cd bitsandbytes/
+
+# 安装依赖
+pip install -r requirements-dev.txt
+
+# 安装编译工具依赖，该步骤在不同系统上命令有所不同，供参考
+apt-get install -y build-essential cmake
+
+# 编译 & 安装
+cmake -DCOMPUTE_BACKEND=npu -S .
+make
+pip install .
+```
+
+2. 安装 transformers 的 main 分支版本。
+
+```bash
+git clone -b main https://github.com/huggingface/transformers.git
+cd transformers
+pip install .
+```
+
+3. 在训练参数中设置 `double_quantization: false`，可参考[示例](examples/train_qlora/llama3_lora_sft_bnb_npu.yaml)。
+
 </details>
 
 ### 数据准备
@@ -432,6 +649,8 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
 > [!NOTE]
 > 使用自定义数据集时，请更新 `data/dataset_info.json` 文件。
 
+您也可以使用 **[Easy Dataset](https://github.com/ConardLi/easy-dataset)**、**[DataFlow](https://github.com/OpenDCAI/DataFlow)** 和 **[GraphGen](https://github.com/open-sciencelab/GraphGen)** 构建用于微调的合成数据。
+
 ### 快速开始
 
 下面三行命令分别对 Llama3-8B-Instruct 模型进行 LoRA **微调**、**推理**和**合并**。
@@ -446,6 +665,8 @@ llamafactory-cli export examples/merge_lora/llama3_lora_sft.yaml
 
 > [!TIP]
 > 使用 `llamafactory-cli help` 显示帮助信息。
+>
+> 遇到报错请先看[常见问题](https://github.com/hiyouga/LLaMA-Factory/issues/4614)。
 
 ### LLaMA Board 可视化微调（由 [Gradio](https://github.com/gradio-app/gradio) 驱动）
 
@@ -453,6 +674,10 @@ llamafactory-cli export examples/merge_lora/llama3_lora_sft.yaml
 llamafactory-cli webui
 ```
 
+### LLaMA Factory Online 在线微调
+
+详情阅读该[文档](https://docs.llamafactory.com.cn/docs/documents/quickstart/getstarted/?utm_source=LLaMA-Factory)。
+
 ### 构建 Docker
 
 CUDA 用户：
@@ -485,22 +710,13 @@ CUDA 用户：
 
 ```bash
 docker build -f ./docker/docker-cuda/Dockerfile \
-    --build-arg INSTALL_BNB=false \
-    --build-arg INSTALL_VLLM=false \
-    --build-arg INSTALL_DEEPSPEED=false \
-    --build-arg INSTALL_FLASHATTN=false \
     --build-arg PIP_INDEX=https://pypi.org/simple \
+    --build-arg EXTRAS=metrics \
     -t llamafactory:latest .
 
-docker run -dit --gpus=all \
-    -v ./hf_cache:/root/.cache/huggingface \
-    -v ./ms_cache:/root/.cache/modelscope \
-    -v ./om_cache:/root/.cache/openmind \
-    -v ./data:/app/data \
-    -v ./output:/app/output \
+docker run -dit --ipc=host --gpus=all \
     -p 7860:7860 \
     -p 8000:8000 \
-    --shm-size 16G \
     --name llamafactory \
     llamafactory:latest
 
@@ -510,19 +726,12 @@ docker exec -it llamafactory bash
 昇腾 NPU 用户：
 
 ```bash
-# 根据您的环境选择镜像
 docker build -f ./docker/docker-npu/Dockerfile \
-    --build-arg INSTALL_DEEPSPEED=false \
     --build-arg PIP_INDEX=https://pypi.org/simple \
+    --build-arg EXTRAS=torch-npu,metrics \
     -t llamafactory:latest .
 
-# 根据您的资源更改 `device`
-docker run -dit \
-    -v ./hf_cache:/root/.cache/huggingface \
-    -v ./ms_cache:/root/.cache/modelscope \
-    -v ./om_cache:/root/.cache/openmind \
-    -v ./data:/app/data \
-    -v ./output:/app/output \
+docker run -dit --ipc=host \
     -v /usr/local/dcmi:/usr/local/dcmi \
     -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
     -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
@@ -533,7 +742,6 @@ docker run -dit \
     --device /dev/davinci_manager \
     --device /dev/devmm_svm \
     --device /dev/hisi_hdc \
-    --shm-size 16G \
     --name llamafactory \
     llamafactory:latest
 
@@ -544,25 +752,15 @@ AMD ROCm 用户：
 
 ```bash
 docker build -f ./docker/docker-rocm/Dockerfile \
-    --build-arg INSTALL_BNB=false \
-    --build-arg INSTALL_VLLM=false \
-    --build-arg INSTALL_DEEPSPEED=false \
-    --build-arg INSTALL_FLASHATTN=false \
     --build-arg PIP_INDEX=https://pypi.org/simple \
+    --build-arg EXTRAS=metrics \
     -t llamafactory:latest .
 
-docker run -dit \
-    -v ./hf_cache:/root/.cache/huggingface \
-    -v ./ms_cache:/root/.cache/modelscope \
-    -v ./om_cache:/root/.cache/openmind \
-    -v ./data:/app/data \
-    -v ./output:/app/output \
-    -v ./saves:/app/saves \
+docker run -dit --ipc=host \
     -p 7860:7860 \
     -p 8000:8000 \
     --device /dev/kfd \
     --device /dev/dri \
-    --shm-size 16G \
     --name llamafactory \
     llamafactory:latest
 
@@ -571,12 +769,14 @@ docker exec -it llamafactory bash
 
 </details>
 
-<details><summary>数据卷详情</summary>
+<details><summary>使用数据卷</summary>
 
-- `hf_cache`：使用宿主机的 Hugging Face 缓存文件夹，允许更改为新的目录。
-- `ms_cache`：类似 Hugging Face 缓存文件夹，为 ModelScope 用户提供。
-- `om_cache`：类似 Hugging Face 缓存文件夹，为 Modelers 用户提供。
-- `data`：宿主机中存放数据集的文件夹路径。
+您可以通过移除 Dockerfile 中 `VOLUME [ "/root/.cache/huggingface", "/app/shared_data", "/app/output" ]` 的注释来使用数据卷。
+
+在构建 Docker 时使用参数 `-v ./hf_cache:/root/.cache/huggingface` 来挂载数据卷。各个数据卷的含义表示如下。
+
+- `hf_cache`：使用宿主机的 Hugging Face 缓存文件夹。
+- `shared_data`：宿主机中存放数据集的文件夹路径。
 - `output`：将导出目录设置为该路径后，即可在宿主机中访问导出后的模型。
 
 </details>
@@ -584,13 +784,13 @@ docker exec -it llamafactory bash
 ### 利用 vLLM 部署 OpenAI API
 
 ```bash
-API_PORT=8000 llamafactory-cli api examples/inference/llama3_vllm.yaml
+API_PORT=8000 llamafactory-cli api examples/inference/llama3.yaml infer_backend=vllm vllm_enforce_eager=true
 ```
 
 > [!TIP]
 > API 文档请查阅[这里](https://platform.openai.com/docs/api-reference/chat/create)。
 >
-> 示例：[图像理解](scripts/test_image.py) | [工具调用](scripts/test_toolcall.py)
+> 示例：[图像理解](scripts/api_example/test_image.py) | [工具调用](scripts/api_example/test_toolcall.py)
 
 ### 从魔搭社区下载
 
@@ -623,6 +823,21 @@ run_name: test_run # 可选
 
 在启动训练任务时，将 `WANDB_API_KEY` 设置为[密钥](https://wandb.ai/authorize)来登录 W&B 账户。
 
+### 使用 SwanLab 面板
+
+若要使用 [SwanLab](https://github.com/SwanHubX/SwanLab) 记录实验数据，请在 yaml 文件中添加下面的参数。
+
+```yaml
+use_swanlab: true
+swanlab_run_name: test_run # 可选
+```
+
+在启动训练任务时，登录SwanLab账户有以下三种方式：
+
+方式一：在 yaml 文件中添加 `swanlab_api_key=<your_api_key>` ，并设置为你的 [API 密钥](https://swanlab.cn/settings)。
+方式二：将环境变量 `SWANLAB_API_KEY` 设置为你的 [API 密钥](https://swanlab.cn/settings)。
+方式三：启动前使用 `swanlab login` 命令完成登录。
+
 ## 使用了 LLaMA Factory 的项目
 
 如果您有项目希望添加至下述列表，请通过邮件联系或者创建一个 PR。
@@ -722,6 +937,9 @@ run_name: test_run # 可选
 1. **[NVIDIA RTX AI Toolkit](https://github.com/NVIDIA/RTX-AI-Toolkit)**：在 Windows 主机上利用英伟达 RTX 设备进行大型语言模型微调的开发包。
 1. **[LazyLLM](https://github.com/LazyAGI/LazyLLM)**：一个低代码构建多 Agent 大模型应用的开发工具，支持基于 LLaMA Factory 的模型微调.
 1. **[RAG-Retrieval](https://github.com/NLPJCL/RAG-Retrieval)**：一个全链路 RAG 检索模型微调、推理和蒸馏代码库。[[blog]](https://zhuanlan.zhihu.com/p/987727357)
+1. **[360-LLaMA-Factory](https://github.com/Qihoo360/360-LLaMA-Factory)**：一个魔改后的代码库，通过 Ring Attention 支持长序列的 SFT 和 DPO 训练。
+1. **[Sky-T1](https://novasky-ai.github.io/posts/sky-t1/)**：由 NovaSky AI 微调的低成本类 o1 长推理模型。
+1. **[WeClone](https://github.com/xming521/WeClone)**：从聊天记录创造数字分身的一站式解决方案。
 
 </details>
 
@@ -729,7 +947,7 @@ run_name: test_run # 可选
 
 本仓库的代码依照 [Apache-2.0](LICENSE) 协议开源。
 
-使用模型权重时，请遵循对应的模型协议：[Baichuan 2](https://huggingface.co/baichuan-inc/Baichuan2-7B-Base/blob/main/Community%20License%20for%20Baichuan%202%20Model.pdf) / [BLOOM](https://huggingface.co/spaces/bigscience/license) / [ChatGLM3](https://github.com/THUDM/ChatGLM3/blob/main/MODEL_LICENSE) / [Command R](https://cohere.com/c4ai-cc-by-nc-license) / [DeepSeek](https://github.com/deepseek-ai/DeepSeek-LLM/blob/main/LICENSE-MODEL) / [Falcon](https://huggingface.co/tiiuae/falcon-180B/blob/main/LICENSE.txt) / [Gemma](https://ai.google.dev/gemma/terms) / [GLM-4](https://huggingface.co/THUDM/glm-4-9b/blob/main/LICENSE) / [Index](https://huggingface.co/IndexTeam/Index-1.9B/blob/main/LICENSE) / [InternLM2](https://github.com/InternLM/InternLM#license) / [Llama](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) / [Llama 2 (LLaVA-1.5)](https://ai.meta.com/llama/license/) / [Llama 3](https://llama.meta.com/llama3/license/) / [MiniCPM](https://github.com/OpenBMB/MiniCPM/blob/main/MiniCPM%20Model%20License.md) / [Mistral/Mixtral/Pixtral](LICENSE) / [OLMo](LICENSE) / [Phi-1.5/Phi-2](https://huggingface.co/microsoft/phi-1_5/resolve/main/Research%20License.docx) / [Phi-3](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/LICENSE) / [Qwen](https://github.com/QwenLM/Qwen/blob/main/Tongyi%20Qianwen%20LICENSE%20AGREEMENT) / [StarCoder 2](https://huggingface.co/spaces/bigcode/bigcode-model-license-agreement) / [XVERSE](https://github.com/xverse-ai/XVERSE-13B/blob/main/MODEL_LICENSE.pdf) / [Yi](https://huggingface.co/01-ai/Yi-6B/blob/main/LICENSE) / [Yi-1.5](LICENSE) / [Yuan 2](https://github.com/IEIT-Yuan/Yuan-2.0/blob/main/LICENSE-Yuan)
+使用模型权重时，请遵循对应的模型协议：[Baichuan 2](https://huggingface.co/baichuan-inc/Baichuan2-7B-Base/blob/main/Community%20License%20for%20Baichuan%202%20Model.pdf) / [BLOOM](https://huggingface.co/spaces/bigscience/license) / [ChatGLM3](https://github.com/THUDM/ChatGLM3/blob/main/MODEL_LICENSE) / [Command R](https://cohere.com/c4ai-cc-by-nc-license) / [DeepSeek](https://github.com/deepseek-ai/DeepSeek-LLM/blob/main/LICENSE-MODEL) / [Falcon](https://huggingface.co/tiiuae/falcon-180B/blob/main/LICENSE.txt) / [Gemma](https://ai.google.dev/gemma/terms) / [GLM-4](https://huggingface.co/THUDM/glm-4-9b/blob/main/LICENSE) / [GPT-2](https://github.com/openai/gpt-2/blob/master/LICENSE) / [Granite](LICENSE) / [Index](https://huggingface.co/IndexTeam/Index-1.9B/blob/main/LICENSE) / [InternLM](https://github.com/InternLM/InternLM#license) / [Llama](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) / [Llama 2](https://ai.meta.com/llama/license/) / [Llama 3](https://llama.meta.com/llama3/license/) / [Llama 4](https://github.com/meta-llama/llama-models/blob/main/models/llama4/LICENSE) / [MiniCPM](https://github.com/OpenBMB/MiniCPM/blob/main/MiniCPM%20Model%20License.md) / [Mistral/Mixtral/Pixtral](LICENSE) / [OLMo](LICENSE) / [Phi-1.5/Phi-2](https://huggingface.co/microsoft/phi-1_5/resolve/main/Research%20License.docx) / [Phi-3/Phi-4](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/LICENSE) / [Qwen](https://github.com/QwenLM/Qwen/blob/main/Tongyi%20Qianwen%20LICENSE%20AGREEMENT) / [Skywork](https://huggingface.co/Skywork/Skywork-13B-base/blob/main/Skywork%20Community%20License.pdf) / [StarCoder 2](https://huggingface.co/spaces/bigcode/bigcode-model-license-agreement) / [TeleChat2](https://huggingface.co/Tele-AI/telechat-7B/blob/main/TeleChat%E6%A8%A1%E5%9E%8B%E7%A4%BE%E5%8C%BA%E8%AE%B8%E5%8F%AF%E5%8D%8F%E8%AE%AE.pdf) / [XVERSE](https://github.com/xverse-ai/XVERSE-13B/blob/main/MODEL_LICENSE.pdf) / [Yi](https://huggingface.co/01-ai/Yi-6B/blob/main/LICENSE) / [Yi-1.5](LICENSE) / [Yuan 2](https://github.com/IEIT-Yuan/Yuan-2.0/blob/main/LICENSE-Yuan)
 
 ## 引用
 

assets/thirdparty/colab.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="117" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="117" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h30v20H0z"/><path fill="#007ec6" d="M30 0h87v20H30z"/><path fill="url(#b)" d="M0 0h117v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"><svg x="4px" y="0px" width="22px" height="20px" viewBox="-2 0 28 24" style="background-color: #fff;border-radius: 1px;"><path style="fill:#e8710a;" d="M1.977,16.77c-2.667-2.277-2.605-7.079,0-9.357C2.919,8.057,3.522,9.075,4.49,9.691c-1.152,1.6-1.146,3.201-0.004,4.803C3.522,15.111,2.918,16.126,1.977,16.77z"/><path style="fill:#f9ab00;" d="M12.257,17.114c-1.767-1.633-2.485-3.658-2.118-6.02c0.451-2.91,2.139-4.893,4.946-5.678c2.565-0.718,4.964-0.217,6.878,1.819c-0.884,0.743-1.707,1.547-2.434,2.446C18.488,8.827,17.319,8.435,16,8.856c-2.404,0.767-3.046,3.241-1.494,5.644c-0.241,0.275-0.493,0.541-0.721,0.826C13.295,15.939,12.511,16.3,12.257,17.114z"/><path style="fill:#e8710a;" d="M19.529,9.682c0.727-0.899,1.55-1.703,2.434-2.446c2.703,2.783,2.701,7.031-0.005,9.764c-2.648,2.674-6.936,2.725-9.701,0.115c0.254-0.814,1.038-1.175,1.528-1.788c0.228-0.285,0.48-0.552,0.721-0.826c1.053,0.916,2.254,1.268,3.6,0.83C20.502,14.551,21.151,11.927,19.529,9.682z"/><path style="fill:#f9ab00;" d="M4.49,9.691C3.522,9.075,2.919,8.057,1.977,7.413c2.209-2.398,5.721-2.942,8.476-1.355c0.555,0.32,0.719,0.606,0.285,1.128c-0.157,0.188-0.258,0.422-0.391,0.631c-0.299,0.47-0.509,1.067-0.929,1.371C8.933,9.539,8.523,8.847,8.021,8.746C6.673,8.475,5.509,8.787,4.49,9.691z"/><path style="fill:#f9ab00;" d="M1.977,16.77c0.941-0.644,1.545-1.659,2.509-2.277c1.373,1.152,2.85,1.433,4.45,0.499c0.332-0.194,0.503-0.088,0.673,0.19c0.386,0.635,0.753,1.285,1.181,1.89c0.34,0.48,0.222,0.715-0.253,1.006C7.84,19.73,4.205,19.188,1.977,16.77z"/></svg><text x="245" y="140" transform="scale(.1)" textLength="30"> </text><text x="725" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="770">Open in Colab</text><text x="725" y="140" transform="scale(.1)" textLength="770">Open in Colab</text></g> </svg>

assets/thirdparty/discord.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="115" height="20" role="img" aria-label="LLaMA Factory"><title>LLaMA Factory</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="115" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="24" height="20" fill="#5865f2"/><rect x="24" width="91" height="20" fill="#555"/><rect width="115" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><image x="5" y="3" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyBmaWxsPSJ3aGl0ZSIgcm9sZT0iaW1nIiB2aWV3Qm94PSIwIDAgMjQgMjQiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+PHRpdGxlPkRpc2NvcmQ8L3RpdGxlPjxwYXRoIGQ9Ik0yMC4zMTcgNC4zNjk4YTE5Ljc5MTMgMTkuNzkxMyAwIDAwLTQuODg1MS0xLjUxNTIuMDc0MS4wNzQxIDAgMDAtLjA3ODUuMDM3MWMtLjIxMS4zNzUzLS40NDQ3Ljg2NDgtLjYwODMgMS4yNDk1LTEuODQ0Ny0uMjc2Mi0zLjY4LS4yNzYyLTUuNDg2OCAwLS4xNjM2LS4zOTMzLS40MDU4LS44NzQyLS42MTc3LTEuMjQ5NWEuMDc3LjA3NyAwIDAwLS4wNzg1LS4wMzcgMTkuNzM2MyAxOS43MzYzIDAgMDAtNC44ODUyIDEuNTE1LjA2OTkuMDY5OSAwIDAwLS4wMzIxLjAyNzdDLjUzMzQgOS4wNDU4LS4zMTkgMTMuNTc5OS4wOTkyIDE4LjA1NzhhLjA4MjQuMDgyNCAwIDAwLjAzMTIuMDU2MWMyLjA1MjggMS41MDc2IDQuMDQxMyAyLjQyMjggNS45OTI5IDMuMDI5NGEuMDc3Ny4wNzc3IDAgMDAuMDg0Mi0uMDI3NmMuNDYxNi0uNjMwNC44NzMxLTEuMjk1MiAxLjIyNi0xLjk5NDJhLjA3Ni4wNzYgMCAwMC0uMDQxNi0uMTA1N2MtLjY1MjgtLjI0NzYtMS4yNzQzLS41NDk1LTEuODcyMi0uODkyM2EuMDc3LjA3NyAwIDAxLS4wMDc2LS4xMjc3Yy4xMjU4LS4wOTQzLjI1MTctLjE5MjMuMzcxOC0uMjkxNGEuMDc0My4wNzQzIDAgMDEuMDc3Ni0uMDEwNWMzLjkyNzggMS43OTMzIDguMTggMS43OTMzIDEyLjA2MTQgMGEuMDczOS4wNzM5IDAgMDEuMDc4NS4wMDk1Yy4xMjAyLjA5OS4yNDYuMTk4MS4zNzI4LjI5MjRhLjA3Ny4wNzcgMCAwMS0uMDA2Ni4xMjc2IDEyLjI5ODYgMTIuMjk4NiAwIDAxLTEuODczLjg5MTQuMDc2Ni4wNzY2IDAgMDAtLjA0MDcuMTA2N2MuMzYwNC42OTguNzcxOSAxLjM2MjggMS4yMjUgMS45OTMyYS4wNzYuMDc2IDAgMDAuMDg0Mi4wMjg2YzEuOTYxLS42MDY3IDMuOTQ5NS0xLjUyMTkgNi4wMDIzLTMuMDI5NGEuMDc3LjA3NyAwIDAwLjAzMTMtLjA1NTJjLjUwMDQtNS4xNzctLjgzODItOS42NzM5LTMuNTQ4NS0xMy42NjA0YS4wNjEuMDYxIDAgMDAtLjAzMTItLjAyODZ6TTguMDIgMTUuMzMxMmMtMS4xODI1IDAtMi4xNTY5LTEuMDg1Ny0yLjE1NjktMi40MTkgMC0xLjMzMzIuOTU1NS0yLjQxODkgMi4xNTctMi40MTg5IDEuMjEwOCAwIDIuMTc1NyAxLjA5NTIgMi4xNTY4IDIuNDE5IDAgMS4zMzMyLS45NTU1IDIuNDE4OS0yLjE1NjkgMi40MTg5em03Ljk3NDggMGMtMS4xODI1IDAtMi4xNTY5LTEuMDg1Ny0yLjE1NjktMi40MTkgMC0xLjMzMzIuOTU1NC0yLjQxODkgMi4xNTY5LTIuNDE4OSAxLjIxMDggMCAyLjE3NTcgMS4wOTUyIDIuMTU2OCAyLjQxOSAwIDEuMzMzMi0uOTQ2IDIuNDE4OS0yLjE1NjggMi40MTg5WiIvPjwvc3ZnPg=="/><text aria-hidden="true" x="685" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="810">LLaMA Factory</text><text x="685" y="140" transform="scale(.1)" fill="#fff" textLength="810">LLaMA Factory</text></g></svg>

assets/thirdparty/dsw.svg

@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="130px" height="20px" viewBox="0 0 130 20" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <title>最终方案备份 6</title>
+    <defs>
+        <linearGradient x1="50%" y1="-11.4084043%" x2="50%" y2="107.220074%" id="linearGradient-1">
+            <stop stop-color="#FF7717" offset="0%"></stop>
+            <stop stop-color="#FF4707" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="50%" y1="0%" x2="50%" y2="107.220074%" id="linearGradient-2">
+            <stop stop-color="#FFFFFF" stop-opacity="0.15" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0.05" offset="100%"></stop>
+        </linearGradient>
+        <path d="M6.45596591,3.88139205 C6.45596591,4.65080492 6.31628788,5.31486742 6.03693182,5.87357955 C5.75757576,6.43229167 5.37523674,6.86257102 4.88991477,7.16441761 C4.4045928,7.4662642 3.85061553,7.6171875 3.22798295,7.6171875 C2.60535038,7.6171875 2.05078125,7.4662642 1.56427557,7.16441761 C1.07776989,6.86257102 0.695430871,6.43229167 0.417258523,5.87357955 C0.139086174,5.31486742 0,4.65080492 0,3.88139205 C0,3.11197917 0.139086174,2.44791667 0.417258523,1.88920455 C0.695430871,1.33049242 1.07776989,0.900213068 1.56427557,0.598366477 C2.05078125,0.296519886 2.60535038,0.145596591 3.22798295,0.145596591 C3.85061553,0.145596591 4.4045928,0.296519886 4.88991477,0.598366477 C5.37523674,0.900213068 5.75757576,1.33049242 6.03693182,1.88920455 C6.31628788,2.44791667 6.45596591,3.11197917 6.45596591,3.88139205 Z M5.53622159,3.88139205 C5.53622159,3.26349432 5.43442235,2.74147727 5.23082386,2.31534091 C5.02722538,1.88920455 4.7508286,1.56664299 4.40163352,1.34765625 C4.05243845,1.12866951 3.66122159,1.01917614 3.22798295,1.01917614 C2.79237689,1.01917614 2.40056818,1.12866951 2.05255682,1.34765625 C1.70454545,1.56664299 1.42814867,1.88920455 1.22336648,2.31534091 C1.01858428,2.74147727 0.916193182,3.26349432 0.916193182,3.88139205 C0.916193182,4.49928977 1.01858428,5.02130682 1.22336648,5.44744318 C1.42814867,5.87357955 1.70454545,6.1961411 2.05255682,6.41512784 C2.40056818,6.63411458 2.79237689,6.74360795 3.22798295,6.74360795 C3.66122159,6.74360795 4.05243845,6.63411458 4.40163352,6.41512784 C4.7508286,6.1961411 5.02722538,5.87357955 5.23082386,5.44744318 C5.43442235,5.02130682 5.53622159,4.49928977 5.53622159,3.88139205 Z" id="path-3"></path>
+        <path d="M8.59629794,9.56321023 L8.59629794,2.06321023 L9.46987749,2.06321023 L9.46987749,2.93323864 L9.56575817,2.93323864 C9.6273112,2.83380682 9.71372218,2.7101089 9.82499112,2.56214489 C9.93626006,2.41418087 10.0954693,2.28219697 10.302619,2.16619318 C10.5097686,2.05018939 10.7885328,1.9921875 11.1389116,1.9921875 C11.593457,1.9921875 11.9953273,2.10582386 12.3445224,2.33309659 C12.6937174,2.56036932 12.9665631,2.88470644 13.1630593,3.30610795 C13.3595555,3.72750947 13.4578036,4.22703598 13.4578036,4.8046875 C13.4578036,5.38470644 13.3595555,5.88600852 13.1630593,6.30859375 C12.9665631,6.73117898 12.6949012,7.05669981 12.3480735,7.28515625 C12.0012459,7.51361269 11.6029267,7.62784091 11.1531161,7.62784091 C10.8074722,7.62784091 10.5298917,7.57043087 10.3203746,7.4556108 C10.1108576,7.34079072 9.94809718,7.20821496 9.83209339,7.05788352 C9.71608961,6.90755208 9.6273112,6.78030303 9.56575817,6.67613636 L9.49828658,6.67613636 L9.49828658,9.56321023 L8.59629794,9.56321023 Z M9.48053089,4.79048295 C9.48053089,5.19294508 9.5397165,5.54805871 9.65808771,5.85582386 C9.77645893,6.16358902 9.94868904,6.40329072 10.1747781,6.57492898 C10.4008671,6.74656723 10.6772638,6.83238636 11.0039684,6.83238636 C11.3448775,6.83238636 11.6301521,6.74183239 11.8597923,6.56072443 C12.0894324,6.37961648 12.2622544,6.13458807 12.3782582,5.8256392 C12.494262,5.51669034 12.5522638,5.17163826 12.5522638,4.79048295 C12.5522638,4.4140625 12.4954457,4.07433712 12.3818093,3.77130682 C12.2681729,3.46827652 12.0965347,3.22857481 11.8668945,3.0522017 C11.6372544,2.8758286 11.3496123,2.78764205 11.0039684,2.78764205 C10.672529,2.78764205 10.3937648,2.87168561 10.1676758,3.03977273 C9.94158677,3.20785985 9.77054036,3.44164299 9.65453658,3.74112216 C9.53853279,4.04060133 9.48053089,4.39038826 9.48053089,4.79048295 Z" id="path-4"></path>
+        <path d="M17.8164584,7.63139205 C17.2885228,7.63139205 16.8333855,7.5147964 16.4510465,7.28160511 C16.0687075,7.04841383 15.774555,6.72170928 15.5685891,6.30149148 C15.3626232,5.88127367 15.2596402,5.39180871 15.2596402,4.83309659 C15.2596402,4.27201705 15.3626232,3.77840909 15.5685891,3.35227273 C15.774555,2.92613636 16.0627889,2.5929214 16.4332908,2.35262784 C16.8037927,2.11233428 17.2376232,1.9921875 17.7347823,1.9921875 C18.0259755,1.9921875 18.3112501,2.0407197 18.5906061,2.13778409 C18.8699622,2.23484848 19.1238684,2.38991477 19.3523249,2.60298295 C19.5807813,2.81605114 19.7618893,3.09659091 19.8956487,3.44460227 C20.0294082,3.79261364 20.096288,4.21875 20.096288,4.72301136 L20.096288,5.08522727 L15.8562311,5.08522727 L15.8562311,4.33948864 L19.1907482,4.33948864 C19.1907482,4.04119318 19.1309707,3.77485795 19.0114158,3.54048295 C18.8918609,3.30610795 18.7231819,3.12085701 18.5053789,2.98473011 C18.2875758,2.84860322 18.0307103,2.78053977 17.7347823,2.78053977 C17.4104451,2.78053977 17.1304972,2.85984848 16.8949385,3.01846591 C16.6593798,3.17708333 16.47768,3.3836411 16.3498391,3.6381392 C16.2219982,3.89263731 16.1580777,4.16666667 16.1580777,4.46022727 L16.1580777,4.98224432 C16.1580777,5.37997159 16.2279167,5.71732955 16.3675948,5.99431818 C16.5072728,6.27130682 16.7014016,6.48200758 16.9499811,6.62642045 C17.1985607,6.77083333 17.4885701,6.84303977 17.8200095,6.84303977 C18.0354451,6.84303977 18.2301658,6.81226326 18.4041715,6.75071023 C18.5781772,6.6891572 18.7291005,6.59682765 18.8569414,6.47372159 C18.9847823,6.35061553 19.0830304,6.19673295 19.1516857,6.01207386 L20.018163,6.22514205 C19.9329357,6.50686553 19.7891147,6.75307765 19.5866999,6.96377841 C19.3842851,7.17447917 19.1327463,7.3384233 18.8320834,7.4556108 C18.5314205,7.5727983 18.1928789,7.63139205 17.8164584,7.63139205 Z" id="path-5"></path>
+        <path d="M23.0521052,4.24715909 L23.0521052,7.51775568 L22.1501165,7.51775568 L22.1501165,2.06321023 L23.0165938,2.06321023 L23.0165938,2.92613636 L23.0876165,2.92613636 C23.2154574,2.64441288 23.4107699,2.41832386 23.673554,2.24786932 C23.9363381,2.07741477 24.2736961,1.9921875 24.6856279,1.9921875 C25.0549461,1.9921875 25.3780995,2.06794508 25.6550881,2.21946023 C25.9320767,2.37097538 26.1481042,2.59883996 26.3031705,2.90305398 C26.4582368,3.20726799 26.5357699,3.59019886 26.5357699,4.05184659 L26.5357699,7.51775568 L25.6337813,7.51775568 L25.6337813,4.12642045 C25.6337813,3.70738636 25.5248798,3.38008996 25.3070767,3.14453125 C25.0892737,2.90897254 24.7897946,2.79119318 24.4086392,2.79119318 C24.1482226,2.79119318 23.9156232,2.84801136 23.710841,2.96164773 C23.5060588,3.07528409 23.3450739,3.24100379 23.2278864,3.45880682 C23.1106989,3.67660985 23.0521052,3.93939394 23.0521052,4.24715909 Z" id="path-6"></path>
+        <path d="M32.4028209,7.51775568 L32.4028209,2.06321023 L33.3048096,2.06321023 L33.3048096,7.51775568 L32.4028209,7.51775568 Z M32.8609175,1.171875 C32.690463,1.171875 32.5448664,1.11446496 32.4241278,0.999644886 C32.3033891,0.884824811 32.2430198,0.746922348 32.2430198,0.5859375 C32.2430198,0.424952652 32.3033891,0.287050189 32.4241278,0.172230114 C32.5448664,0.0574100379 32.690463,0 32.8609175,0 C33.0290046,0 33.1734175,0.0574100379 33.2941562,0.172230114 C33.4148948,0.287050189 33.4752641,0.424952652 33.4752641,0.5859375 C33.4752641,0.746922348 33.4148948,0.884824811 33.2941562,0.999644886 C33.1734175,1.11446496 33.0290046,1.171875 32.8609175,1.171875 Z" id="path-7"></path>
+        <path d="M36.5099554,4.24715909 L36.5099554,7.51775568 L35.6079668,7.51775568 L35.6079668,2.06321023 L36.474444,2.06321023 L36.474444,2.92613636 L36.5454668,2.92613636 C36.6733077,2.64441288 36.8686202,2.41832386 37.1314043,2.24786932 C37.3941883,2.07741477 37.7315463,1.9921875 38.1434781,1.9921875 C38.5127963,1.9921875 38.8359497,2.06794508 39.1129383,2.21946023 C39.389927,2.37097538 39.6059544,2.59883996 39.7610207,2.90305398 C39.916087,3.20726799 39.9936202,3.59019886 39.9936202,4.05184659 L39.9936202,7.51775568 L39.0916315,7.51775568 L39.0916315,4.12642045 C39.0916315,3.70738636 38.98273,3.38008996 38.764927,3.14453125 C38.5471239,2.90897254 38.2476448,2.79119318 37.8664895,2.79119318 C37.6060728,2.79119318 37.3734734,2.84801136 37.1686912,2.96164773 C36.963909,3.07528409 36.8029241,3.24100379 36.6857366,3.45880682 C36.5685491,3.67660985 36.5099554,3.93939394 36.5099554,4.24715909 Z" id="path-8"></path>
+        <path d="M45.9636541,7.51775568 L45.9636541,0.245028409 L48.460103,0.245028409 C49.0306522,0.245028409 49.4988104,0.347419508 49.8645774,0.552201705 C50.2303445,0.756983902 50.5020064,1.03456439 50.6795632,1.38494318 C50.85712,1.73532197 50.9458984,2.12713068 50.9458984,2.56036932 C50.9458984,2.99597538 50.85712,3.38955966 50.6795632,3.74112216 C50.5020064,4.09268466 50.2309363,4.37204072 49.866353,4.57919034 C49.5017696,4.78633996 49.0365708,4.88991477 48.4707564,4.88991477 L46.7022905,4.88991477 L46.7022905,4.06605114 L48.4210405,4.06605114 C48.8021958,4.06605114 49.1093691,3.99976326 49.3425604,3.8671875 C49.5757517,3.73461174 49.7456143,3.55527936 49.8521484,3.32919034 C49.9586825,3.10310133 50.0119496,2.84682765 50.0119496,2.56036932 C50.0119496,2.27391098 49.9586825,2.01822917 49.8521484,1.79332386 C49.7456143,1.56841856 49.5745679,1.39204545 49.3390092,1.26420455 C49.1034505,1.13636364 48.7927261,1.07244318 48.4068359,1.07244318 L46.9082564,1.07244318 L46.9082564,7.51775568 L45.9636541,7.51775568 Z" id="path-9"></path>
+        <path d="M52.7664895,7.51775568 L51.7757224,7.51775568 L54.4284213,0.245028409 L55.4404952,0.245028409 L58.093194,7.51775568 L57.102427,7.51775568 L54.9646429,1.4453125 L54.9078247,1.4453125 L52.7664895,7.51775568 Z M53.0647849,4.67684659 L56.8041315,4.67684659 L56.8041315,5.49715909 L53.0647849,5.49715909 L53.0647849,4.67684659 Z" id="path-10"></path>
+        <polygon id="path-11" points="60.949413 0.245028409 60.949413 7.51775568 60.0048107 7.51775568 60.0048107 0.245028409"></polygon>
+        <polygon id="path-12" points="66.5090177 3.46946023 66.5090177 4.29332386 63.3129949 4.29332386 63.3129949 3.46946023"></polygon>
+        <path d="M71.1487393,7.51775568 L68.8724609,7.51775568 L68.8724609,0.245028409 L71.2410689,0.245028409 C71.9489287,0.245028409 72.5549893,0.390033144 73.0592507,0.680042614 C73.5635121,0.970052083 73.9505859,1.38553504 74.2204723,1.92649148 C74.4903587,2.46744792 74.6253018,3.11434659 74.6253018,3.8671875 C74.6253018,4.62476326 74.4897668,5.27639678 74.2186967,5.82208807 C73.9476267,6.36777936 73.5528587,6.78681345 73.0343928,7.07919034 C72.5159268,7.37156723 71.8873757,7.51775568 71.1487393,7.51775568 Z M69.8170632,6.68678977 L71.0919212,6.68678977 C71.6790424,6.68678977 72.1667318,6.57374527 72.5549893,6.34765625 C72.9432469,6.12156723 73.2326645,5.79841383 73.4232422,5.37819602 C73.6138198,4.95797822 73.7091087,4.45430871 73.7091087,3.8671875 C73.7091087,3.28480114 73.6144117,2.78527462 73.4250178,2.36860795 C73.2356238,1.95194129 72.9539003,1.63233902 72.5798473,1.40980114 C72.2057943,1.18726326 71.7405954,1.07599432 71.1842507,1.07599432 L69.8170632,1.07599432 L69.8170632,6.68678977 Z" id="path-13"></path>
+        <path d="M80.8452792,2.08806818 C80.8026656,1.73532197 80.6339866,1.46129261 80.3392423,1.26598011 C80.044498,1.07066761 79.6805065,0.973011364 79.2472678,0.973011364 C78.9324004,0.973011364 78.6577792,1.02391098 78.4234042,1.12571023 C78.1890292,1.22750947 78.0073294,1.36659564 77.8783048,1.54296875 C77.7492802,1.71934186 77.6847678,1.91998106 77.6847678,2.14488636 C77.6824004,2.33191288 77.7267896,2.49289773 77.8179355,2.62784091 C77.9090813,2.76278409 78.0256769,2.87464489 78.1677224,2.9634233 C78.3097678,3.0522017 78.4606911,3.12559186 78.6204923,3.18359375 C78.7802934,3.24159564 78.9288493,3.28835227 79.0661599,3.32386364 L79.7941428,3.51917614 C79.9977413,3.57125947 80.2173199,3.6422822 80.4528786,3.73224432 C80.6884374,3.82220644 80.9127508,3.94353693 81.125819,4.0962358 C81.3388872,4.24893466 81.5134847,4.44247159 81.6496116,4.67684659 C81.7857385,4.91122159 81.8538019,5.19649621 81.8538019,5.53267045 C81.8538019,5.93039773 81.7502271,6.28847064 81.5430775,6.6068892 C81.3359279,6.92530777 81.035265,7.17743845 80.6410889,7.36328125 C80.2469127,7.54912405 79.7692849,7.64204545 79.2082053,7.64204545 C78.6802697,7.64204545 78.2239487,7.55741004 77.8392423,7.3881392 C77.4545358,7.21886837 77.1526892,6.98153409 76.9337025,6.67613636 C76.7147158,6.37073864 76.5910178,6.01325758 76.5626088,5.60369318 L77.5285178,5.60369318 C77.5521921,5.87594697 77.6433379,6.10085227 77.8019553,6.27840909 C77.9605728,6.45596591 78.1623957,6.58735795 78.4074241,6.67258523 C78.6524525,6.7578125 78.9181959,6.80042614 79.2046542,6.80042614 C79.5337262,6.80042614 79.8284705,6.74715909 80.0888872,6.640625 C80.3493038,6.53409091 80.5552697,6.38494318 80.7067849,6.19318182 C80.8583,6.00142045 80.9340576,5.77769886 80.9340576,5.52201705 C80.9340576,5.28764205 80.8689535,5.09647254 80.7387451,4.94850852 C80.6085368,4.80054451 80.4345311,4.67980587 80.2167281,4.58629261 C79.998925,4.49277936 79.7621826,4.41051136 79.5065008,4.33948864 L78.6222678,4.09090909 C78.0493512,3.92992424 77.5959894,3.69673295 77.2621826,3.39133523 C76.9283758,3.0859375 76.7614724,2.68702652 76.7614724,2.19460227 C76.7614724,1.78267045 76.8733332,1.42341383 77.0970548,1.11683239 C77.3207764,0.810250947 77.6220311,0.571732955 78.000819,0.401278409 C78.3796069,0.230823864 78.8033758,0.145596591 79.2721258,0.145596591 C79.7479781,0.145596591 80.1699714,0.229640152 80.5381059,0.397727273 C80.9062404,0.565814394 81.1980254,0.796638258 81.413461,1.09019886 C81.6288966,1.38375947 81.742533,1.71638258 81.7543701,2.08806818 L80.8452792,2.08806818 Z" id="path-14"></path>
+        <polygon id="path-15" points="85.4149214 7.51775568 83.4156317 0.245028409 84.3850919 0.245028409 85.8765692 6.08664773 85.9475919 6.08664773 87.4745805 0.245028409 88.5079612 0.245028409 90.038501 6.08664773 90.1059726 6.08664773 91.5974498 0.245028409 92.5669101 0.245028409 90.5676203 7.51775568 89.6052623 7.51775568 88.0214555 1.82173295 87.9646373 1.82173295 86.3772794 7.51775568"></polygon>
+    </defs>
+    <g id="最终方案备份-6" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="编组备份-3">
+            <rect id="Fill备份-19" fill="url(#linearGradient-1)" x="0" y="0" width="130" height="20" rx="3.33333333"></rect>
+            <rect id="Fill备份-20" fill="url(#linearGradient-2)" x="2" y="2" width="20" height="16" rx="2.5"></rect>
+            <g id="产品图标/learn-机器学习备份-10" transform="translate(6.028800, 4.000000)" fill="#FFFFFF" fill-rule="nonzero">
+                <g id="产品图标/learn-机器学习" transform="translate(0.000000, 0.000000)">
+                    <path d="M8.784,0 C9.4752,0 10.0512,0.576 10.0512,1.2672 C10.0512,1.8432 9.6768,2.3328 9.1584,2.4768 L8.784,4.7232 C9.1008,4.9536 9.3312,5.328 9.3312,5.76 C9.3312,6.4512 8.7552,7.0272 8.064,7.0272 C7.5456,7.0272 7.0848,6.7104 6.8832,6.2496 L5.5008,6.1632 C5.328,6.3648 5.0976,6.48 4.8096,6.48 C4.32,6.48 3.9168,6.0768 3.9168,5.5872 C3.9168,5.2416 4.1184,4.9248 4.4064,4.7808 L4.5216,4.032 C4.3776,3.8592 4.2624,3.6576 4.2624,3.3984 C4.2624,2.9088 4.6656,2.5056 5.1552,2.5056 C5.6448,2.5056 6.048,2.9088 6.048,3.3984 C6.048,3.744 5.8464,4.0608 5.5584,4.2048 L5.4432,4.9536 C5.472,4.9824 5.5008,5.04 5.5296,5.0688 L6.912,5.1552 C7.0848,4.8672 7.344,4.6368 7.6608,4.5504 L8.0352,2.304 C7.8624,2.1888 7.7184,2.016 7.6032,1.8144 L3.8016,1.8144 C3.6576,2.1312 3.3696,2.3904 3.024,2.4768 L2.3616,6.5088 C2.8224,6.768 3.1392,7.2 3.1968,7.7472 L8.1216,8.8416 C8.4384,8.2944 9.0144,7.9488 9.6768,7.9488 C10.6848,7.9488 11.4624,8.7552 11.4624,9.7344 C11.4624,10.7136 10.6848,11.52 9.7056,11.52 C8.7552,11.52 7.9776,10.8 7.92,9.8784 L2.9952,8.784 C2.7072,9.2448 2.1888,9.5328 1.6128,9.5328 C0.72,9.5328 0,8.8128 0,7.92 C0,7.1136 0.576,6.4512 1.3248,6.336 L1.9872,2.304 C1.6128,2.0736 1.4112,1.6992 1.4112,1.2672 C1.4112,0.576 1.9872,0 2.6784,0 C3.168,0 3.6288,0.288 3.8304,0.72 L7.6608,0.72 C7.8624,0.288 8.2944,0 8.784,0 Z" id="Combined-Shape"></path>
+                </g>
+            </g>
+        </g>
+        <g id="Open-in-PAI-DSW" transform="translate(28.589489, 5.982244)" fill="#FFFFFF" fill-rule="nonzero">
+            <g id="形状">
+                <use xlink:href="#path-3"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-3"></use>
+            </g>
+            <g id="形状">
+                <use xlink:href="#path-4"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-4"></use>
+            </g>
+            <g id="路径">
+                <use xlink:href="#path-5"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-5"></use>
+            </g>
+            <g id="路径">
+                <use xlink:href="#path-6"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-6"></use>
+            </g>
+            <g id="形状">
+                <use xlink:href="#path-7"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-7"></use>
+            </g>
+            <g id="路径">
+                <use xlink:href="#path-8"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-8"></use>
+            </g>
+            <g id="路径">
+                <use xlink:href="#path-9"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-9"></use>
+            </g>
+            <g id="形状">
+                <use xlink:href="#path-10"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-10"></use>
+            </g>
+            <g id="路径">
+                <use xlink:href="#path-11"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-11"></use>
+            </g>
+            <g id="路径">
+                <use xlink:href="#path-12"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-12"></use>
+            </g>
+            <g id="形状">
+                <use xlink:href="#path-13"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-13"></use>
+            </g>
+            <g id="路径">
+                <use xlink:href="#path-14"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-14"></use>
+            </g>
+            <g id="路径">
+                <use xlink:href="#path-15"></use>
+                <use stroke="#FFFFFF" stroke-width="0.5" xlink:href="#path-15"></use>
+            </g>
+        </g>
+    </g>
+</svg>

assets/thirdparty/lab4ai.svg

@@ -0,0 +1,536 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 24.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="125" height="20">
+<g>
+	<g>
+		<defs>
+			<polygon id="SVGID_1_" points="0,0 126.7,0 126.7,21.5 0,21.5 0,0 			"/>
+		</defs>
+		<clipPath id="SVGID_2_">
+			<use xlink:href="#SVGID_1_"  style="overflow:visible;"/>
+		</clipPath>
+		<g style="clip-path:url(#SVGID_2_);">
+
+				<image style="overflow:visible;" width="874" height="148" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA20AAACYCAYAAAB3RQH9AAAACXBIWXMAAExuAABMbgHZzlUOAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAcstJREFUeNrsvQeUHMd5Lvp1z8zm
+vFhgkRMBAgRIAowACBDMmZKVRVGURFG0LVv2sy1LTsfv2r73Wrbv8fXxu8fXkhhFiZIlkRIpMSeA
+AAEGkACIQAQihw1YbM47O92v/q7umeowM92zM7szu/WRhZ7t6e6prqqu/r/6EyAhISEhISEhISEh
+ISGRt1CycZHv/tN76/SQ/rmQEtJ1Rb9CVdRiHUqVqijLFEWForJCW0UxfpK21mejEopZDSV1dRRF
+kT0mISEhISEhISEhITGu0HU92VeD0HGSsZROQOvWdGUP29fHmM+2//cPLnhjwknbt7+3/W4o2j26
+husYmZqpqiFQIYJmbBWTqKmqSdJUg5QpJmmzPotkzU7KJEGTkJCQkJCQkJCQkMhbKmcndOZWp/26
+HmP7TzO+8zxjNb8aK4ELzIz+5L9v/aquj/41+7hUDYWRIGvepE1lfxvaNSJvMMmaT02bIombhISE
+hISEhISEhEReUjYHWYtr43Tjs/E3baFHdV3brWjK//y7b13wbE5J2//zD5s/ERuN/jd2xmWcrJmE
+TSBuFmnjZC0U166ppsYNDi2bJGsSEhISEhISEhISEpONvJnaNpPIsa2m0b+DmqZtUzR87+//KJjm
+zRc7+sO/ffWHMS36IBG1EJE0s4TY34pA2CyNm6Fts5lH2v3Z4qaRSoK4uf3VJHGTkJCQkJCQkJCQ
+kMhPqmb7S9S2eZE23SoalXO6ov9n9UDke9/+9rzhMZO2b3/75fL+Uv1Xeix2i0jUnKSNiJwimEba
+fNpEf7Y4eYOdxPEdjopJ0iYhISEhISEhISEhkY+UzRGYJJWJJATCpmkWcevTdO2XalHkT/7+wXk9
+GZO2P/qfWxqG+rq3s9+4wCBooYiLtHmZSDp92izzSCdpc5lIxmskyZqEhISEhISEhISERCGwtyQa
+N09Nm420QdNiw5qmbWa7/vQf/3jxgcCk7cvffrm8ODT0Ifu4mJtDRmDb2ohbKE7e4oSN9jmImit6
+pEDcJGGTkJCQkJCQkJCQkJgUxE0gbJ6+bUTYDAIXoxJlxG1TSNG+/vffWnI22U+EvXaWhIZfiGma
+QdigWAFFhC0jZSAyBiJkIaGYf4OCjpiaNii2rRjmX/qxSUhISEhISEhISEgUNBQPU0kK3wFO1KxC
+ZM2gO7rJetg/uoII214T1fCPf/qvp37335L4uLlI2wPffe6RaHTo2lA4EideFhGzE7KQQeDoMxRe
+FMffInHjWjRV0KYp/H4E4iaTZ0tISEhISEhISEhIFBLExNv8s8XMFFMLpxvEjiuvGAeypT7TqZSz
+XXeVRqL/wHb9hScvFP/4xl+98MDwQO/DRNgMk8dwxPBli/uzmVvLPDIk+LUpjpD/tuTaYn62OIET
+SZokbhISEhISEhISEhIShUvYUiXbjkeVNP3ZyEzSMJHUDRNJ8m9jRTuqx7S/+t6fLP6l83dsmrbh
+of6/Jg2ZbhAr0/wRCa2ZqD2zNG5xrZqgXbPt19m1LOImEDaXps3c6rLvJSQkJCQkJCQkJCQKARaX
+MTVsTk0bJ2s8Mj6jaSZXQtx8UtdVxE0odX0e+/e+P/3XU79xmknGSdvXvvP8N6JDvYtIg+YkZErc
+zDGxdZI04xxdcexX4sRNN7Vsum4PRMJvS5FsTUJCQkJCQkJCQkKiUNkbDzoSN4lUhEiS1hGkHNNM
+fiQSNpWbTyp6hJVLikPRL7Gdj3mStuhw/1/FSZalFbPsLqEkSBgcGrX4d2riXF3wYTPPFwmbbtlw
+6oJ2TZpFSkhISEhISEhISEgUIuIaNrsuSrSe5JyH8SOKIGkSNVjETecciR0/V9P0r333nw4/9y9/
+ubTNRtq++mcv3Dsy3GVo2eLEzXKQE/KtQdyq9n16PKy/9Z25HwmipigJjZtRcRmEREJCQkJCQkJC
+QkKi0Dmbxx+64iBuphIuHohE58RNN/gTY0v0pUpESp+PIuUuCNo2g7SNaoNf0h3aNbemzfFZN4ug
+SYP4nSJo1gQNm6IrCbKmJwibnufmkTPKWlBZ1IuyoiGUhGMIh3WURwYQCqlGTjpV5YXuxdoqQlqD
+ZFuJXC12JAbUwEgYw6MRREdV9AwVsVKM1t5K9A9HoOmyHyQkJCQkJCQkJMYKRZA/E1zHqXWL+7zp
+Fneya9qMrY56TdfXuknb8NBGivwIJ/kyiyJsFV1xHaeIJE5JEDdFcZxvfe8I9W/qCvOu6aeXnsW0
+krMoD7UjzJonHKaImaH4lkfPVBAyImd6E7ZUxE2St9ySNevvqlKyGR42Ps+o6jcz0jehd7gEpzuq
+cOx8PQajEdl4EhISEhISEhISmUqhiXzUSJAzxSJupoyqwFJWCZxJUIKR5SL7r4J9sfK7/3S4wTKR
+DN/37ec+PTzQVW7ELDHNFw1NmPBZN7Vk4meNCIq1H8LxIt2zgo8IW4uo6A5mmk+oKTqP2SX7UaJ2
+IqyGGWHjRM1O2tQ4WaN9qUlbcg2bJG25J23WNrG6wbe14VFUl57HooZOHD5Xh+OMvA2NhmUjSkhI
+SEhISEhIBIQZMTIh5HO501RY6ebXcZNJM94H/a2ZloiaEWEyzsmmxcLqOvbnswZpi0VHbuGRKM0L
+xiNOCqaOlmmkbjeVdJpH6qI2Dd4mjwlTyPwkK4vKP0AljoJRNEbKODlTTXJmETRezL+F79zEDS5t
+myRrE0ferGItdtBn6oeKEh2r5rRhTk0fdp2eifb+Umk2KSEhISEhISEhkQFxi0ufiJtMxrVtwlYX
+lF8QA5YoZgIApZoxikvipE3TY3MTkU4cppG6m6wpVkRI04lOJGy8bonw/XEtm+XHlsd+axF1BAuK
+t6JUa4NqadJCqqlZsxeuZUto2Lw0bWQ2KWrXpEnkxBI3O2GzCBzvJ03TjG1D5SDWLjqFnadmoqm7
+EjFNlY0oISEhISEhISExBnnUJHMO4pawAkvwLF10OdP1kpiuN1rXCeua3qDHmaE92Ihu05YJ+wTf
+Nl3QzimwzB4VG3lLyUfzgMBElGEsjLyMiNYDNRKJa9FIw2Zp1RIatpBB2MhEUiRsvCguE0lJ2vKD
+tNnJmr0oSsJxtKo0xojbaew4MRunO2swKombhISEhISEhIRExvKn9cGhbXMEd9StbZyDKRFdQ02c
+tGmxWH3clFG3kzfRLFIkcXEtm0XUdEubJjjXifWNJwW3m0bmS9TIuUWbENK6oBIBM3z1LE2auXUW
+c7+qJrZE2JzaNpGwSfPICXhgFMX1tydpo/z0mhFi1RiZxeEYVs1pwWC0CK095dJUUkJCQkJCQkJC
+wieURIRI2JNsx00fLZPJuNbN47OOIrYRSJum1cV90ERSpgM2rZsuEC7zs6iJ40TNDxnLL8I2v2Q7
+wqNNUMMRk3iZpo0qD7RiRIeMm0pSUUyippgaOcXYly7cvwIZ6n98nxfFttoh5slwFsXIRUhbLT7W
+y4ujuHR2E94ZmYvOgVLZnhISEhISEhISEhkRt4TSSrdZKbrMI+08LMRKmUDaYjVqKAJBcxffOvcl
+HONE8mGaQir2DOCKbb8ZXRIJ97x8cG+rCjejaGi/YRJpES4j55oi5F5TUpeQaj/eFTkShZGfrX9I
+xfuHi1FXFcPFC0YmzSOjQ3eQNx1e/9ncOM10GdMqBrCgvhOD0bBMCSAhISEhISEhIZGBLArTbUx3
+cCpRGyeYTCp2zhUnbQkapdhVc3BoJGC3v4wzQ4EpFhoatK3xKI+qoFlTLI2boti0b0n3iRo6gax5
+5WhLyscnkMsdaYrg356pxHuHIqit1HD31YO4/6Z+VJQWTqcmG3+KYo/IEx/LClxaZadJsKopWNpw
+Hme7Kg3SVohjXEJCQkJCQkJCYqLYmkC8bJEjLe6le1iBKXafN4u0ieTM6RAHG0lzqOwEu0uRtCWC
+keQ5YYscgRLtgBIOxxWCShpC5k3SYPusKFaOOnux6SvzSNvWM6Di756swq6TRdBHgYERFd9/uRJD
+Iwr+/NN97L4Kg6l4NanoCGoucAjrG4kVDI39S/1nH+u8T4uLNCys60TPYDH6R4rkBCQhISEhISEh
+IZERabP5tMHbbcdZEqRNR1LTSJGowWZ7CZvznE3TJgYjyWNUansEwqV6kjDXPsVN0kSypnok1TY8
+/1xBSPKnHZ56qxS7TxQRczEytlua2p+/WYZPrhnCsrnRSfX0cPKm2AidaoQi0RP9qEJIBwDMreXJ
+t/uGJWmTkJCQkJCQkJDIlLQJSq84vUqE+k9J2pxkzEbSRALnYIOKkKvNRtIKQDFDIf6V6DkoYdOM
+EaI5owrHXsNfjX+PxGeYJC7pf0j4szkSiSt5klj8THsYP3mj3CAmZgwODvZ5IKrieGsEy+eOTtKn
+SBE7hBE3NaFoNtdBuK+ihtKiGGpKh9E5UIZoTKYAkJCQkJCQkJCQyAJpc7mimRaLouLMIm0iKXOR
+tDT7RRMzMRhJvqM2fIyRthirfcRuxijchlPTppgaRE9zyBTFahobacuTZnri9TK09IS4hs1ZR1Yq
+S3VMiWCXeqJfkpUZVT1o6q7ASKxYTkISEhISEhISEhJBxEzb36l5F0+j5uRWYVtwEVcwBsSTvYkh
+/m1+cA7NXCGgDE12vzMIhE3Qkrky1ileWezMoovf6/xsM3FzfBvvrYlvrA9PRPCrt0qNeuuOsJ56
+CFgxJ4qLFwyjkKNvKAEeIV1Y6nA/CUBd6SAioZgMRiIhISEhISEhIeGbrfnWtKXzaROjRkKHR542
+IWqkM0+bjoTQX0DCrIoBt4Av+KHZt2JESLjzryVJnm1977WdaIxqCh56qQL9o6wfNTthIyvBEOvU
++2/uR12Fnpb6FMxzYxJnPQnrEiN+AoqrbynhthLPsyEhISEhISEhISGRXdKGdD5terIgI6ku4Ige
+WUikTRntSRAz2IkaF+ATW2+TR922tTLU8WvoHkFIEtt8wBsflhjFIGy2huFp/K5bOYS7rhqcnE8O
+RJNPe3/EtaUepSg8avStJG0SEhISEhISEhK5IG1po0cqSGJfmewCgCt6ZCGRNl3r50mWrZsHzDwJ
+3CwO1hZ6Us2MleXcOE/RzWAs/hthorRuPQMKfvhiGWLODidyogLlYR1/cOcAQgUeb8PZF4nQqrqt
+v/mY1h3f67Z9BCJtqtS0SUhISEhISEhITARps7O0AAUeLK9AoGiDUMIRTzNHO6Fym0ImSFeCfMWj
+RCr5lYPNC09tK8feUzzEvzh4DOLO/vnCxkFcsmDEdd6Z82FD0zS7PlaQz41lGiluxUHsJGOefVlg
+41xCQkJCQkJCQmJiSZvn36kikXhxLpiBSCy3JudnXdjnhCrsV8y/LeFfTMstbp2fJ7YRdQ+qa5Ix
+QdMGQfsGIUAFLCVd3LHPZD1wBB/Jo2AkTe1hPPFqqRj+M3EfrAPn10fx5et6XXU73xvC7/+fGtRW
+afjPb3aiolTLf5Lm7HKzH8WtEF8ViUCrif61+t4qWpJnQUJCQkJCQkJCQsIvd/PSe2kC79I95M6E
+eWQK/zXRh000izS4igJ7cm1Ras5XrYSoJUuiPXN+L2rUxGOT/kTeBSHR8fjrZWjuDSd82ax+NP22
+vnrDgKcm7UevleNIWwTaOeDZd8tw73V9k+6B4ukr0g9YaR4pISEhISEhISHhl615mkcC3vFEPPYn
+SBuEg+CV5C0JcVMcSbj1/OZpLsnb0SqKqHUStWgOTZuTzcZ94BQlryX63ceK8Iu3yhKEzbpf2oaB
+VfOG8ZlrBlz3cPBMBE9tK4VmcrnHXi3DLasG0FBdYGaSokbVNtCFfnb0u7PPLR84CQkJCQkJCQkJ
+iYxJmyNtmif/cvCqsHixZAm1rc+KY+tlmlkIEKNAWn87I0L6bxUzJ1set8BoTMFDL1eif0SFotmr
+aiQLZ39/845elBbZiZim6Xj01Qq094fidrOnO8L48Rvl+LNPdRfmk+OKIpkIPBLvT8WZz9A8BtKl
+TUJCQkJCQkJCIrgEiiRMwsk2vL43SJuYa82mcYMjtD+8FRBwKDAKqgWTtVSK7xUr7bb5uRCk+Fd3
+l+K1PSVGPzt92TT2zx2r+3HtimHXvbz1USle+KA0YVxralh/ub0cd145gAtnRwuZsyV/Smz9be9z
+qWmTkJDwg5ULgZn1wNwZgKoCbZ3AyVbg8GlgYEi2j0R2MbsBuO0qNu4WpT92xwHgpfeAjh7ZbhIS
+4yF7Bta0pQr57zKB9HGRgidt6diZLQeb13fJlJf5he4BFT94qYL7HopRZhRO2OrKNDx4Sy9UxR5i
+Y2hExf99oQJRTUkQPfBrtPerePS1cvzTV7vM9OuTmLUJx+iStElISKRBYx0Tnq8Gli8AiiPCFwv4
+hojbb98CTrSQFYRsL4ksveF0/5KITeaTkJAoHNLmJFx+8gU4fyjP3blSyu82rRngqUVzaltcJY+t
+I3++tRwHmoqAmLuOlJftixv6cNHcqOu7X71djp3HS9w2sCb5e+H9MnxqzQDWLB2eHJwN6TmcfMmN
+HdNqgKVzgKpyYE4DUFrM98+o5/NIRzcwwobjABtW57uA9h7gyBn2uVsKuBL5DxrXn94IXDCba9e8
+MH8G8NnrgZ+9ygmcnFOSI8IklLoqoLIs/bGDw1xzNDg8RRsr4DiabO+zilI+VooiqY/rHeDjJDqa
+2/pQrluqT3WF/3NyPYbl85T/pC0ZD4uTNqePWqZp2gqvFfUkvmyw+Tr5s6HMzxY41RbBk5sqoGnc
+BDbB1tj/IWBRQxT3UYh/R/1busP40evmTCMGLhEGynBMxX++UInLFw8jEtIL58kJrGlL9K/0aQuO
+MHtxLWYC7EULuOahoQYpE7dXlHjvb+kAPjoB7D0KnD7HiN2obFuJ/MN1q/lihKqmPo60cesuAbrf
+kSZqqUDC5U1XAFctT3/skbPAi6w9Pz4jOVsuz8lH0DtlzQrglqsc2m0PvHeAj5P2HD93DbXA7WuA
+VRf4P6erj9XtXeDtffJ5muzPp1+fNlcgEk82J/qyeWgakrHDwms+PQPhPZXzW37h8TcqjRD/iiNi
+pFXt+2/qxbQqt/T7083lONEeTphTwk74rPPfPlyK598vw+9c3VdgfY7M+l1q2vyTtRCweil7QVzO
+BdR0Qmw60DWo3HAZcKwJeOEdvo1K8iaRR2Ne1B6nw6KZQHkJEx67ZdslnbWDzLlT3BoiyL3rk6yt
+5k4HLpiTnrCN573PnsZLENRU8HNoXugblM/TZGJr2TOP1G25hW1R0OFQ1YnHKIVM2pxRVuAO/+56
+uj2L11OQH7aSu46V4qm3yxNBRKz7NjOhX7V4CHdf4U6kfehsMX62tSKRGsDZboJvGxnXPvJqBTZe
+1I/aijxPO+0V8t/G2dLbBMsJzB9Iq3b71cC8GUAolP3rL5oFfOvTbIx/DLz2PnC2TZpOSkw8SOAi
+wuY3JWdtJRAJyTnFz7Tt+7WOKUzacinA5zFIy0bmyPReyJd7Ly7igWHIsiQoKHhRIysfn5bP01Qn
+bUiWp03RPeRWuD8rTplXJG8OFEzOtkmIUU3Bf7xYhaFRRxAR8OiPZRENv39bD0qLdNcD/f2Xq9A9
+FPLuPMXxmQnKH7cW4xfbKvF7t3ZPiWdPTmDJQf48n7ueRy8rCuf+91Yv4QTx+e3AO/uBfhmRT2IC
+0T8YbPGAfDdjmpxT/AiOQU6Yypq2qSiQB9Gyjde9k7aMtO6ZgM6j8w+fks/TZCdtyQI/Im0gEkwl
+fzaecStxP7rwt/2/hC9Tsv8gfMofvLSrAm8dKPVMske+bLev7sfaZf2uWr++rxSv7ClL+LGlGoQm
+gSN/uSe3VuK2y/oxryGa133uvU3Vv959LuEGrQ7ecxM3+RqrKWQQ0Iv6E+uBMjbcN+8CevplX0hM
+DPqG+PiLxfxpmJvbgcERubjpQ97JnVA6hQnuZGgr0rItmcv9pvNpnMycxksmIG39LHZuTSXQ2Suf
+p8k6lwV10oqTNsvU0eusZJZiisOqMH5uIanXnGpFm44SqQ1LU5pHTiy6+lU89EqlEc4fjkTaYILE
+tPJRfOPmbh6qX/iuf0jFw69UG1q6lFo2h+aO/m7pCuOx16vw375wPr/723Prt6+leWQy0Avz89fz
+FU+/pmHZBPkSUX4iCmJCTtwysIPEROGtPXwBY0Zd6uNII/fuAe7PJueUNNO2Hux4XbI2X8dOhrYi
+DRvlQ/SrZRuPe7ciI1eVZX6N2aa2LdvvMvk8TRxbS6tpQ5pI/SJp05PwEE9zSUHeVZyDQC+kVkwV
+BTJd0BFnrP/8ufGn3q7CwaZiHuIfcAUSuWdDLxY3umO4PvteJXafKIUe826tsKozQUNJXE64ZRLU
+f/1OBe64vA9XXjCY30+O52DV/Y0JOYG5sGIhN4lsrJv4uqy/hG9fkBH5JCYI+47ztBa3XsV91pLh
+zV3AniMynLYvoTGggCnNI31KQAXeVuQ3tmwesHBmBtJfDu+dCFumppEW6H1K2rb9x7kJtXyeph5p
+s4XaEC4XBpL7sDllVy8lRPwY6cQ24Th9vghPvFFlf8iE4COLZwzjC+vdvmfnuiN49PVqw9TRBXZu
+bVkM//K1c3jnUCkee6OGEzuhvyloyWCMknjXYNU3hwokBUCGE54c43HQS+WutcCM2vypExG3pvM8
+ZPKAFIglJgCbdgLHm9mzsQ64cJ7dv/PMOb6oQORueES2lS8ikkH+MUna0guShf4+u3ghcOkFGZjj
+5/jeZ9Xzd+OYCGmEa+wpp9q5Tvk8TRnSBh/RI53mkck0ay5HOcWD2BXa7OZstWTRWOAjgmQekNZH
+Xq9Ca184YRYpaNkU1sn3X9+NhspRVz2f2FyJ051hz/pTAu7PX9ODjcv7sWr+EN76qBSHW0xNnqjF
+Y7/59sdleHV3Oe64rDeP+zyAeaRj2UOStgQoJPEnrgHmjMEkknx6yDysqT0hwNJLalo1ML8RKCnK
+7Lqf3AB09PKcbjIlgMRE4HgT8H+eSixu0DPS1sWDj0gEF3iCTPMy5H8w8lKIIPPByy4cg4VHju59
+ei1/J6Yy1yTtOmnPyopTE05yNyCNXWuHfJ6mDGlzUgskCfnvIl6TJ1XZlMGOo6V45t1KN2GjbYiH
++L/rCjeZ2n+6BE9tr04EHxH9E9mEMrc2ii9cw23NqstiePDmbnznieluHzd2fpRtH36tBtcsH0B1
+aWxSP4BTHXeSFmE+EAkYJZLMFinSIyU4PdeRPNpeCXvpLWXXv+ZinpQ7CIGjF+Yn1wNdPVzjIV86
+EhOJpjbZBuM230pZZNK3Fb1zKGE1mebn273PbeBkKxVosZJysFGKgorS5MdZJpK7D2fRRFI+TxM/
+hyXzzEnHvSzSpsMeLFAVjnGOE1XYL3ICZ3HGqkj2eWInKz19MBLo/nO1TSCiTPB9+JUqDIyqPJG2
+o8GLWI/93s0dKI1ots6PaQoeeb0anYOqt5ZNIR+4bsypS4Q4u21VL555rwLbDpcltG1mh5PGdt+Z
+Yjy1rRIP3NiZZw/OGDVtRiASHZqcgnDFhcAli/lKoV8QOXvnI+DFd7i5R7pHZiDKXlZHeLn6IuDu
+dTy4g1+tHpmWXL0CaO/NfgQuCQmJ8YGGQLE1jOOn6hwdNHJkobbVRQu5li1Q8JFxuHeKZDmzIb32
+jyLM0juQjktF2kgLRwSQrnmqVT5Pk5W7peJrGryiR8IjTxtS5Gxz+LgVJhv3CqSpZ1gw4Tf/6p5K
+vHmggpu5wsGa2YN/5xW9uGaZOxb61oPleOXDck6+nBFA2XkrZg/hs2u7bPdXFI7hD25tx56TJegZ
+UPlvCgm31RDwxJvVuG11H2bXjeThI+MnEIl3kXnaeKRGIkNkxugXZPr45KvAB4eAoQyGBGnmyNzs
+q7cDS+b492G4ipG9HQc5aZPaNgmJAhR0gphneURam3JtFbBdC62tyHR+1ZL02qyJuHfSilG90r2f
+upkodqKFuwBMT+MPbplInmyRz1Ohs7VU5pHpgta7o0dafm1ITsicLl4QfNukCnXi0DMYwg9fqYdO
+KghnxEiFQvzH8MBNXa7zBkdU/ODlWkRjHjMM21Wk6vjW7Z2oKnWvs1y+aBB3XdaLn71dnfhNa7iw
+w1t6Inj0jRr87WfPTV0BYpKCHL8pFHE45O94i7C9fzAzwmahpQP45Sbgy7fwl50fjRv53V1+IfcJ
+kNo2CYnCnHODykcyEMnkfJ+RJovm89VL8/NdTr5s6cgkBXsjF4EjZ7h554Vprllfzd+3pFUcGpHP
+05QhbfCTXNuZiw3J/4aVq60gw/0j0UKAR6AR+FM15ol55H9tq8GB5iK7qaLF2dgkd881nVgyY8jV
+P79+twq7TpQmCVoCVES6ML2MkT3dTero0K9e14FN+8vR0h12ETe6JqUQuHN1Dy5blCcpAIKaR8Jt
+BjvVSRsRtTUBtWzPvw3s/jg7oc2PnuWR9z57XfoVSgtXLOORJMczBQCRRTLlLGIv2nkzuBkpra6e
+7wa6GHls7ZyYgBRe9SKcOgcMDPGgMERuR/PQHdWob4m7PVva87fO+QhKR0DaCqtQhFVqR/Ivpe1Q
+nkW0DOyCk0dzdHUFb+PKMr61TOGonakYY7gre8GSAufgSiG6kA+xUecy91jp7ktsxxPL53PSlsws
+f9icU/2YTWZbbDPmpum8nVKB2o0CEtH7iPqeSFw6zdzcGZwMHj49tZ+nKUXavII/WnKYmCw7nWkk
+HARO1LApevKczLLPc4OTbUV48s0qu6Gy2eAKE7AXN3iH+G/tiuDxzTXxQePsLy02glMtzfibH/Xj
+sT+vQm2F+7cXTh/Bveu78K/PTbM7QpqldziER16vx8ULziKi6pPm4ZvKExiZpZCphl8t28fsJbPz
+ELfhzxboehcvYgJROc/Tk1ZwYsddtIBr23oHkh9HTuE3XgHU+yCkr78P7D3mFnDpGmtWcmKbKrEq
+CWsfHQe27+NtlEtBme5nGRN2rljOcxqVpvFDJF8LEg7IlPXgyczJ9ni2J9WZzGA37+KC0EQ+oxcv
+5mknaNylw09fTe+rEqQdqQ0pubdTmCaiTosX61byqHupxiVFXN2ymy+QxPLAmSWI0DiRuceImC2e
+zecmWliYXudvDFBAChoDtLC18zAfv2Nqq4Dt6mwrv2OFCNyJZuDd/dzvONcEjhYKqV7Uxl4g8kPz
+FmnjaL7P5N7HAtKGzfaRm+1MG0/7QYtM9JmCkqQ7j74n08tDp6bO8zQVSBuc3ArwDKOBlOaRaRJs
+I1kONxRYx7oorp8kCck0bxN18zoe31RnmCLaPEVNfzSq0v3Xd6KhIuqayX+ypQYn24uM8+JKNiuN
+AyvRwTbERnqw43gVfvzGKP74bm8p/XNrOvH8zgocbC5xR59h1359Xzle21OB2y/Ng0zHgQORuFcr
+pvoEtnKRP0GEQC8l0orRSymbbUar0iRUkjA722cC0wvmAO99lJo8khaKrkkBTNKBCI0ipDyhF+o9
+N/Eol36iaVIbrl3Jyy4mqP16C3+RZ1NILi8FrlsN3HIlUFPp/zzSYFIh4kHC5G/e4oJ8UGI51vb8
+3A1svC3kWkE/db5zLavzxbwt3/2Iaw4nAtS3JFimW3EnEIFO92wEaUdaxacFFeuaJOjexAjfxtWp
+Ax6Idad+v2wpJ38vvzc2EpHNadu/lDl+czS1L7UV+fjOn+FvEckJ6hciGVQoIu/2vTzPX1bDvPsQ
+4OlebrmKjZVVXGuUDqTtsupN8wQFmNp5KDcLUETEiLBdlsKWkJJQE+mlRZNskxc/IG0YkfVUIGJJ
+8zzlEaXfpq0f0kbPJR1DYyXVwmOhP0+StHmTNu/k2kJRHNd1fuelgS88bVqQQCRA6picmJC733W8
+DM/sqPTuNPb31Yv7cfcV3a66HThbil9sr4oTPUXsRJpYYoMYGW6FqvBYoD/ZHMZtl+tYOsstUdZW
+jOIbN3bgL34604hECcCVbuAHL9dh7ZJ+1JSN5seTEzgQSeL7qey6SeYyJDj6FUxo1ZNM16I5MFs7
+dJq//EhY90OS5pumdbqP0RF0BrlyOfCZjbxtMslXR/4ZJPg88RLXFmVD6NlwKXD3NTzpuTKGcL0k
+hHzrM7xev3gjmCCZaXuSEPzZ6/nKddC6kznal2/l29c/yK6GN5f3nc3rWecsnMXTXqzKIPkwPStE
+9qgdn32LP2uF9mbPKTFn7XLDZaxc7n8Ryy/hv30NsHQuX3zYczS3Y88654LZwCfW84jAgRNVm/PE
+1+8EFswEntuefa3bpUv4YkKyhQfSVu45Bhxr9k/asim1EeGlNki3MNJm5iQdMUUhIm2nz/E5L127
+0/WJGBI5nWzP0xTjbknbN51EGidtcTLmUC4o4hZJVHXC9wXXs46AKkmTiiPJcV5OguPEXokgff/V
+OiPEv+33zJwLxaxXf//mdhSFNNsqia4r+OFrdegaDttHgkCyooMt0GOjUENF7M8YWnqr8djrI/gf
+Xxr0nFhuXNmDNYsrse0oI5BR2FIAUFCSAy0l+NlbNaw+5ye2u/XkWx95tae8TxvZ1PtZgbVAAkdX
+X+7aa/8JrkGr96HRICGLNE+qklyblYkTP5mtUSqCID5+XiAi/JXb+GfKX5cpcSMtCwlfNzJhsiqL
+wuSVy7hA+eOXeRQzP201Ue1JbfCpa7l/y5u7uECXr6zNz3yiZ3A9MtH71EYujGcKmusp1UYsBvxq
+y/hofjJtI9drPcdzND0LJERnk7CJIE3tvbcApW/yyLm5aKtsjhUCLZ4RkSU8t43P/dkABZ0iwpZK
+i/XBQW5JQSaqfuembL7LyZctnZaNQIsfVKzfNUwkz3EiNyeNto2+p7Lv2OR7nqYKWwuiaUOK6JGq
+k4TZSAs8nOFSxWpIsz9fSiZalnwqL+yqxJsflduTOFi+hWzyvPOybqxZ0uc6b8vBcry6p4LncvNQ
+l45G+xCNdjHCVgxFZaRNjbDdo/j1OyV45+Miz7qUFml44MYOlJLfmioQNmscsN/6+fYaHG+L5GVb
+BiqCieRUK2LginSgFzb5Q5EpR67qQyuOPQFMReY1JszRUhFzv3PwZUuAW68cO2FzEjcysQypwduD
+nO+/dDOr01XZJWwWaPX//tu54BBkocNve9L1iWxmqz1vuzphdjTu75cMfEay0Y6ExnquaR2rEG6B
+/FgNTYAyke/r7LXneNcpU5DmnjR55I+aq3qRZuzmK7M3ViziRtpC0jqNtY3pGhsYYVu9JPlvkkXH
+2/t5EKVMFjjGWui5MHzOfJgv08JHS7v9fMNE0sd6thWEicyuJ+PzNJWLHx4lvgPUTC4wGRrJV2a7
+PCzd/SE8+kY9dFV1mSJS8JG60iju39ie8FMzS/9wCA+/Xo+orroeXPozzEiXEmtl36lQGPMjwgaD
+tKkYQRW+/0oVhkYUzzqtW9KP21f32Mw0LdMmIojNPRH8eEv9ZOBsU7ZYJoZ+QC8nCmCRy/rQy69v
+wP9LaP701KQtiGxMpnuk0ZpRl11BjYgbmbSRwBa0PSii5jUXB9OGBsUiJtxdx4Sy2kofdQrQnqSx
++PS1/gQfv6ip4Hn6aDuuzwqyLDgGaEfq+2sv5RH2sgW6JvmykoA/YfNPwFXtyUDaCBQtkfxSycw5
+2/WixRG69iWLs1tnIm5XLedkf6xkaN3FvCQzHaRFQfJfpaBOmQbYGGshE33SBqZzGyDfUDKFdL4X
+z7bx/ZoPf2ZaMJvdMDmfJ1n8P9uqF2fJlMMUCvfxV1OM4a5z1xK/eLsWB1tK7GH2TbNIOuKL6zqx
+pHHIdd5vdlTj/eNliUTaSqJQQuw7Vvfij24fRihcxsgfadlCbLKMsBI2zCR3HK/FczurktbrgevP
+Y1pFjGvbnGAT0jPvVmMn/X6ejYIgfVfgnHNMhQhKiU9/tpOtQP9Q7utE0bf8RjekulvrHOlGRDpc
+z4jLzGlj8xdLZRq19mJu0um3HTasYoL6smCEjUxzjjdzv68gwh6tpJPfCAkpfp+wdCD/O/LBysSf
+JhUuX8qDsEzk7OJLgMzStYisUaRNP36eQbBwJu+fiZ6ps9We410vCvFOGpVMQvtT2y+Ylf16UY6w
+TH3Y0oHSStCCyVjGzKqlfAGiMkXU2PcPcdI2qgW//2yNAbLg8GMaedokZ87zqe6Ug7TNR8AfygNH
+BDHde6xQn6epXuCz3+KBSJxh/xXdbS4JwVzSIgm2CCWAPSiGkub7CbMx1T2MR52tlquli8xxsr0E
+P3mr1jA5dIG164L6Idy7rsMlhbV0F+HRN+sT54l9wAhbRVEM37ihjZ0fw0u7i/BRcwVULcqOI20b
+I3BMOh3Vwnh8cx02Lu9HQ6Xb6Wbx9EF8eUM7/v3l6XZ1rtn3AzEVP3ytHv/x9X6EJiIFgFfUyIB2
+ElPVp420FUEEQdKCDQ3nvq1Io0f+X37ICpEgMbJesuExVhCRpBVgivrXWOcv+qEXMSI/DT8JwYnk
+3bEGaKjxd22K1Pja+9x81fKdIzMkEuJIW0cruanIKLUhRZij5LCnz6V/3LLRlqSJo/sLSkbI33HR
+TL6anY08gb6lnICaNj/T1lhAOaFIKKSFC2rHoNpYEpxJ017HCHD7OAcC1gNK4uMyRyepE43VY03A
+gRPARyf48+H0T7U0wBToJV0iZgJFUiWNG2mT0vqJZSlqxFjHi6UlpEKpI4JiQSMnbERQkoHa+K0P
+uVlk4PvP0jih+Yi0X40+LC6a2vg85PWbp1v5WElnuWH9Hmn3ms5PoudpisDVjo7A9fCKq4Ekedrg
+CD5iS6rtlS/AIyDJJGjSMfDg8cFDb9SjtdcM8e8gxyRokbarrsKdtfeJrXU43RFxBx8xr/H5qzux
+tJE7CH3rDg1/9Ag7LFRimEYqSrFB3hRNx5FzFHmyBn94q3dyoc9d3cFIXxUOtZRAH7XXT48B245U
+4OU9VbhjVdcE9a/XYA2yBjI1JzDysQqiVaIE0hQIItdtNTjiP0w+aYZUJQVpG0M9Dp4CXn6H5yty
+kgPKjUbREC+c5590EIkiHy96yacLpEF+L34SjZPw+NttjLDt4Pm4nMImBTwgInbfbdyHJBXZpHsh
+PxjKj5YsaEqm7Un3+8LbPHebWE8SIEnDSVq5IH5vtBJedmj8UgBkYh6Zres5+5u0EBTN77QwXRex
+MXjpBcAnNgBL5vi/HgnvpEE53z3+gk5g7Yk+fn1MZm0nWnjQm/cPpie1tBDz8rs8PD7NC7QAkg5E
+7khYT7eIM5bbpgWwN3aaRKjH/txR1ELylaXANH7nMDqPUlWQyfPZAASDFgiuXcX9KFORym37+MKT
+LdBakHGSBTJC/bKgMb22kgjwqXPJF44sE0k/USSJyFKhcybL8yRJmz8XNBtpE7VsCpIHIlE8crpN
+HtKW33j3SCWee7/aziEs4hUCrlrUj7suc79R950tw9Pv1Mb7Lk7YTC3bvPoovrS+PX78LZeO4JZV
+Gl7ZX8+I2rDh26aSQlbhqtWfbK3FLZd0Y8lMtxRUzwjjV69tw1//fE78eKuu9JMjowoe2TQNa5f2
+oXbCUwBkOOFNwbFOkRdDIf/Hk8BIUedy3VZEDv2aHMXNI/XsSjwkrP1yU3LzlgNMsPhfPwU+vZET
+rFTmPiIoVDsJdqny8lCQAhK8/ZitPr2ZCWUfpCaBdA9EmGh1P50WgPznKNnrmbbstefJZp76gEjw
+qCNVBAk8VDfKCUUBW8gkyc9CAvkfUvuM23Ob7chsGdSb+pEi+FGycSeppsUUilDa3MHH5JqL/JnI
+EVG2fBknYqktyAk5J23m9YmsUTvvyCDiKy14UE42etbSkWdqeyq50spSREKaw4gEuRbG2HN3mD2P
+Z8/xZ++OtVxb6AdELmjR5IxPgkGLgxQpkkqyMUnz/VZGLLfvcc8R4x0VkeagVNpAkRC3pMhXSs8k
+kbBzXem1dvT9XEbgI6FE6oBCf54kaUsRMdIjV54qdmQynZLz78K3HfURqsWrFSfIK5PIzkOv12NY
+U+2JtE0CVqRoePD6NpSGY7bzaFJ7+PUG9IyE7CaVJuEjzcNXN5zH7NqEd2xY1fC7Nw2gqjgKJVTG
+BkiEHatypRy7RtdIBP/31enccdajrnet7sZ6RsqIEMIhVFFQkn1nS/H0u7UoxNA++hT1ayPSFg5A
+2kgD5vQzmGi/EsvEM1u2/oRntwJPvspftqmuS+1BxwXJw3YhI2QV5amvS0mT/ZhF7v6YawF6B9O3
+J5HMj8+kr+diJmiWl2WvPYlMPsYI274TPLdfsmvuZQLmOx/5jxxK2qGiosL2wQoC6rdXdwCv7ODj
+Ltk1SQAn7apfLQgtNlSW5bc/23jJKeSv+xtG1v75SWDLh6nbOd2z5kWUXIuhFmnLQZuRNvbxF3ld
+Ul2zb4hr6mnhxO8cRgSDit96X7E8/cIWzaGbd/P6jIc/abJCdSQtmx+tPz1rVFJdjzRxp33kQyQy
+O38mMGfG5HmeZAnWd6pTBeqZmwqTLQrLWFwEx38ov/xhFbYfqXT7suk8YuRtq3qw/sIe13nbDlXi
+jf0V0MTgIxaBYgLsJXP78YnL213nrV7Qj89c1YdQSDX82cQE3GTmuOlAFbYc8g5KEgnF8Ls3nkNZ
+WIOu2MmltUrwky11aOoqQmG5hk7dqEVBzSPzMbIcLWDQQkO2XFNp5ZRCTZPJkt+6bt7pz+GcQCTZ
+SGYeSRLNs5GbH6XTspGA9TwTtJrb/dfz/QPpTbHod0lLkKx+QdvzrT28TdP2kc6P9WumRyGyySQw
+H0PB58JlmrSUlA5jNJb+un5Jg4205XvY/3GoC2lOyKeKIgKO5TqkxSI/tXQkiAT1ijKeciVb6SYI
+RNjf3pfIvZiu0JiisUVjzA+o3kRqqn1EcCWTa0o/kErD//FpJnvsTOQ6y/T+szEGKCjIXB8BSMjU
+lAgbLUqluh5pMsmM2U8USctcdrI8T7IEjB6ZkfvWpA3X4sy4DIeH4Phr3boGQnhoUwNiQp11y/KQ
+TYrVxaP4xnWtPCGacF7/kIKH3piOEU3l5qxiMBhiTpqOB29oQyU73+t3v379eTRWjXCNmUC6SNs2
+FFXwyBvT0DsY8jz38gV9+OTlHaSgS5wr5G1r7SvCw+z8wkumMcWXePJteS6IJmeACR2j2bvHZ7by
+F22Q+pKgR6ZGfleq6cVs5MbzuNbCRh7sJB1ONHNn/VjMfz2JsI1E0197gVWHMbYnmbmSqVhHt7/6
+tTBh80wrNyvKy+XiCVKzkd/e3iPA0TP+rt1lCpS9A3najuOpOpmgQmPfTyJqes6SPmsZtBWRg71H
+OQkLUl8aW/uP+Z/DSEtYX5n6mpSc+s61PNF3KvKzaRcf3/kwVmjuW+DDNNIKQJLueiOCiWQ6kHUF
+peDwNR6m2PM0FWSwsMhdNEueh91+UvPYKkkKHFZxituaTyIAfvlOPQ63lHLiZWnXrMZmnfGFtR1Y
+2uh2VPntrlq8f6LM3uEWeWLn3bKiG9ctS75cPatmBF+/7jz+8dlZ7uifbADsOF6O53bV4J61bvsa
+lVX2vg3t2PRRNVp7IoZ2Tux8Im507p2runD5wr6C6QsdbuvUqQBaEY7F/B9fzV7S4XBmNvdBEMRs
+k1Y6R2LJ+y/Iu4zMBw+zMjASvM57mZC0crE/PzSKJkamfV51Jk2bn6huBxhJ7B4INm47+vz5Cqaq
+XyCtZQc33YwFOKnZjBxa7CM6Z1kpT2syGsv9s6IFPFbzMef4RSfrt/NMuI0GqAQRdCp+fC0pmA/1
+99DI+M09WoA20PN4jiZtb0Mt155Pr+Fmg9bffnzESGulqKnvLYhMTn5m5MvW0RuwP1gFjpzli0Hk
+U5v2XcDuraoieb0pwuuta3hut2SguYjMUN/am3psBwge6evZSwYjN9tMf/MvmT2ePOfvt86YBM9P
+NEpD2zada8un4vNUiNBT7AsU8l9UIijiVjTNcLp6KfBOUF1QzZc6SmCwpYbc4GRbKX6ydRpvbqdP
+GnsJLKgdZqSpzVWH1p4iPPZmA48F4jyPTfw1JTzEf0hN/Th++srzePHDKuw8UWEjjdb2kU3TceNF
+PZhe7Q6LtKhhAF+65jz+7cWZCaYvnNs9FMbjWxqwcm4fisP6OD4ymUSPNPO0TVGnXCJtWoCZe7yS
+0AYx24wK5pFjlXjIjG8ww5QGJCBHfWqIKIpkyCN4ChFVMs0xtHA+hv1CduxIgMTVRLj9RIkz/AST
+pVEI0J5kMkpEIEh7GppTnySM2ssKsJXzWSaIOaOfZyRAO1LOve6+YPdJ/lh+SRgRh5TBfHLUnlkP
+7pJDULAWyq1mRBacaZK0Wv/BO5KBFnksU+SUbeUT5JdnmewFBY0zvz6lVO9kgYBo0e2WK3l4/1TB
+cMgcmiJu9g34GCsBnrtMxwn17Twf6RpIg918Pn29LVAuNyJulG8xHYz8fY08tcRkfp4mFWnLNHok
+nNEjIUSMFCNFwhH230HeUsrBhUJ5s5lNPAd4lBGv1v5IYqlD9Eljv/mVjaYJo+P3f/LWNJzsFBJw
+C1oy8oH71JWduHh2X9p6U/42CnDyxz8qw6iu2o4nMni2m5FDRrz+4q4znud//urz+O3OGnzcWur6
+joKSvLa/Gps+qsFtF3eOP2fLoN+nKmkzzOUCaM1opZBW5X2br2UI8lcq9pkHjUyQUqUhCNKvpNEY
+yTClAb3Ioz7JRkmSNAW0el1a7I+wfupaXnJKUvSxtSeRtuGApK3XMnfNo0WEoO8DPwJRoDbpD54s
+nRYfgmjOxnsOzNRPcDxBZsyUe41C4pMwne3k5n6JRhCBvKs34Y8bFET2+gf9HUtt4bWwQ4tRRNYo
+X10qq4M9R4FX3+d+hNkkJJmOE6r3Ap8Jtcn0+GSr/98ZZET6VAuPLJoujQtpxmnhjhYDOnsn1/Mk
+SVuakP9wEDNRy+Ykc4riQdqUwiNtuuO/uCYlzX9I+lf28cGJSvxmd61LQ2UQHjYRrp7bj09edt71
+6weayvGr9+o5YXNUjMwrZlcP48vrWn3X+rplnbhxZS1e2VeTyL1mdTur27M763DbpR1GUBMnqkuj
++L0bW/GdJxckfk30b2ObH7w+HeuX9qC8eDTnfW7feo+FtH0+BScw8pUJYh7ZWJ9+VTgrpK3GP2mz
+TDyzQdraOoNrhuKEr2fsaQoqAkbzzBVqrYicYyRtnRnk9esdCGbuOF7CR9bztAXRPmagPaHnwm/i
+cV2fGNI23vm3/GLhLJ43kMhaeUnux1W690+Q9WPStPUPZk7a+nySNsr36DVHrF0J3LEmtQaSTAUp
+EiolFs/meslYxsnsBv/EnMwnyexzxUL/1/eTc9MCaftI69fRMzmeJ0na3No17zxtSAjQSELWXFo2
+izxMmg4dayST7GJUU/Hw5ukYHFETDoRCEJGwquObt7SgrMgu/cV0BY+9OR3tg2H3eQqv6zcZiZpV
+6z/TLBH1b93ShA+OlaO9L2I3t2SfOwZCxm/+8z0nUBRy29DdsLwTG5bVYMthJmGPOpqLHX6wuQy/
+eLce91/bMk7iVObmkUbY/ylqHkkveEpkHfKR08nQtOWYtNHqLSVu9eMbRjjelFqgDUoyRjIkbfSC
+HfGpgSxNommrKMkP0paKeGSSMyloVLOgddTH0wp7AkhbRtYAeb7yHsicyyOnUS5ARITSbXxmIzd/
+HE8xJeW9BSTrAxmaeJO5XyBNm4O0UXJ3ImxEgJKBzC9fehfYtieY2WPOnhMTRgCSmT5J/Uxecrlo
+QHWhlC6F/DxJ0uYvw1ictFnaMmeSba+E2nGNm5iMWy9AlzYgfTR/P/K81/JOFjSPL++pxeYD1Z45
+2Yi03XVpF9Yv6Xb9zraD1Xjhw2o3YQOPODm/PoqFDcM42lLGiKE/ZyCFdTAJiJct6screzySQsWA
+V/bW4u7VnbjhIreZY2lEM6JbfnC8Ev1ayG7qadbvp9sbjHPn1w/ntr/HyN2m8qrTsWZg6Tygqjz9
+sYvYi6S63IxZk6P2IidsXz5dVv2buLCRlLQFeZeNYSyQWYtfwkXBOaKjHqZFoWApGCaEtCG4b1dg
+opeHpE0PeHBa0paDaxYE+c2UVCK39SPCRgmmP3MdmwvL8uNZG+++JNNsv4tltEDl1KIvX8BLKlDb
+PnAXL9kG5YKj4gXKW/ezV/n7wuu+ySwyiDYsl6A+IFJIli3N5wvzeZKkzZu0eaaNtkib7pChndtk
+4TmcShyvfs1fy8lkgUQmXtPWOxQyfNk8w24ywlZbOoqvbmyBM2nbwEgIj2yezjiR6q6Swsl1U0cE
+Dzy02NCY0E4lCd9UhJxuZBRIq/1airBD9NUPN83Amgu6UVbktlm6cmEv7lrViV+8N43nbtMTg4hu
+40xXEf7rnen4iztP5bjPM2RrcTPJAl2gyAIOnwbWX+KPtJWwl9uqpTxyVmdvbupz5UX+EpsSKEcZ
+RUkb1bLzLhtLCKKKAKTNCJ6ie4/YfFz7GksdM4lGPR79NRFtN573mevjx6ONxqt+ZGVAGjYKT58p
+YSN/pdZOvqV5iULir1qSXmPnp2/Hq63IX7nE54IZLTqRP3ShvDdTtTMRNtJu+bE2GS/Mn8Hr1XR+
+cjzvk5q0+ZBK0+mROGnTkzA+eAcgUZJFkCzIJgxK3pBz0vb0jgYcaC5PBBERqkSr65+9uh0XznCH
+yf/NB3XYcbLSSKSteLFpnYRABVFBw2b0pZKOtCmpb9OMCbvndDmefq8B961v9jzs64xovrG/Gm19
+EbdPpMZTG9x+aTsWNwyivDiWw0cmKGlLHDOVNW2HTgXI5wTumL/1w2D29n5B2qpLFvsjkIRjZ1Nr
+2TKRjjMdC+UBTBvJ/83L12t0ND/GYUqzxhybR45XfwVuk2xHZsvxyriuZ9aWEynoZHUcBcAFc4Br
+LvavaaFIgDsP8WTUNH96BdNYs4Inls7GGB6vvrQiQvoBLRA7fYnz+h2aop0X5NjcMRPMa+R1eu8j
+mIvxhfM8TTnS5lPT5uXbBpt5JBwkTfRpg4Ooefi1xf3blALqXD3JUsJYEo1nAac7SvDj7dMTSjSB
+PJECbV7tEO5b1+r6vZaeYjyxrTHed3EGJm497t02GBSBoMPj95OpVS1Ox7Y/2d6Ajcs6Ma/e7TM3
+r24QX7n2HP73S7M9I5D2j4bwg9dm4o9vPYsLGwdy1+djyewwhScwIl+0QkwrjUU+nLDJSXrVBVxQ
+6c1ydxIhnFHr30SQTDvTBWgIIvDUmHnoMomOGSSICJEzrzQFFCHQbzCTzbv4in4uxi35xZzvSuHT
+FjCSWy7JRkEHIsl1OyJ318+GoKPnAakk7crKhcBFC9IfS8/bb7cBm3byZ3W82iLIfRspBIoym5vJ
+X9kvabMC3RTSe9OrnWmBkHJjUkqHfAKNS6oX+QeebCmc50mSNm/S5pxnvQORmIK5kob1ieRONG8r
+zLD/6bQtXvuBXJtHPvn2dDR1FXnborLy9Wtb0VDhJkQ/f7cBxzuK3ZEmnVnOdXibXepIbYPlR0PB
+hMhTncX4+XsN+M7t3maOn7n8HF7cXYv9TWUJbZ6QwfGtj6uxYs4AGipHUFcezVGfZ2IeyY+b6qtO
++49xoYVC7fvB71zLk39S9K9s+bbRKvctVwENNf6OJ6GE6t3bn72+o98uZqStL4PrGREvfQo8Rv4y
+D4EnSOTEw+xRfO19/9HeJmw6zuGz5SfyXlYFvgkUiHI9R+V1yP8ckkoyX1w0Kz1ZOd8NPL2Z5xXz
+o/koifCAQzlpjxQgjX85ExnaMvTLrfRpHkrm8bTgVzCatiR1XJiHWjYL5ENO5URz4TxPkrT5IG2O
+z3bSJpzs1KrZApDATuBSOroVAm9LZ/XoR7uWJU3bnjOVeJpC9XtosRACrpjfi7tWt7t+61BLOX62
+vcEd4MNLw6a4iaBnvylpOI5XP5tmjr94ZzpuW9mJi+e4nZlqy6K4f0Mr/vKXC4xIl7Y6sXNHYgpe
++LAWVy7oQdWCUSNKZs44WwbWsFN9AqOV42su4X4YfrRctCr5ifVAe7d/m3s/RHDBDP9aNtI0nT2f
+XoAK0q+1pqYt6FigiJdXLvO/WnskScTL7gCatkWzgbJ92dd2ZlM4kNEjJ07ICnL8RJhL5UsyYFos
+8mMWSWZq2/f5X1QhrX1tVXbGcJC+od+lojcHawfS7CyZAyye7ZO09bhJW74L1l7tSITd7z2PN6bX
+cTNJWlBIl75DJtfOY9LmmMOTadpUL5k2KI8ptOLfBhI+73xsrUMBEn64qRH9o2F7xEiT1EQYeXlg
+YwvKIlHbeXToo1tnomc4nNCyOaoTf9EqwplenxWP75xFScFRzS/6RlQ8tLnR1Ky4r3LHpeexbnGP
+QURdmkB2D0fbSvHKR3Vo7wuP+6hI14/2rH5Tr1AYZvJTaw/gp0bBS754M3+xjPX3P3cDsHEVUF7q
+77d7rfp2Bx8FqbB0Lg8oErT+ly3nTuN+zCNJ6GttN/PBOa5DL+bT5/zl1yJhg9pr4uba4Gto2b7+
+ZLjniW5DTLAMMNF180OuaGGordskKT6uGYlwszu/pobZ7MtZ07j2MGg70CLQikX+6kzt0dLJLQbG
+MuYmcj2fysxp/L5LivKzvkbC75k8onIhz5uy+OuzeJ42kYU7E23Hv0sWWlK4RkEo22z2Mh4efy6a
+66TA8Eholzle319n5DHTY7BrnxSeEPum5R3YuLTD9Tvb2Dmv7a8x/F5cigeFJ+Yl/yNn1/lZEE7V
+j/QdEc3RUW/G/+YhqlctblnR7jpXRQy/f8NZvH+iEoMaz0NHZND6PVpFeOnDWqxd1IXaJVFWfy1L
+fe5Y0tCRfGkDDt10/G+56kTatg2X+te2EW6+gnILAj96kft7BAVpqD5zPfA7G4L5FFBd/WjZgmpI
+SOCheyLhjHy6/GLNcv8RL0+1cI1asrofPQtcsYyHoU4FIokUYYxI3vDIOM+xAVdzJ0MgkiBSUU4C
+kUxCn7ag7Z+L+pE5YUWaBNqUuyxtwCNxQWUmX1TJVt8GtRag+YPM12ku8QsKxkKaNj+guZEWnpxa
+R9q350j2+4jmwhl16QNUkckm+Vp75cskE8OBIXtbks8YFT/4+DTw45d56oCxYt3FwJdvBRb7GCM0
+jsjf/NCpwniepiJSadrgRTOc1CRO2hw+ba6IkbCbSKbMAyARGN39YTz85izDNFBxmh4yYbWiKIYH
+r29mu+0N3D8SxiNbGjEwovLzxEAhJgn69m1ncNn8bmi66m0yGUQQEc5VFQ1nukrwP56Zj+6hsOvY
+EU3B42/NxNWLelBd6p4ZV8/rwaeuaMfP3mswgq6Yeb/5rbO/2/sj+O2uBlwwfcAzqMmEcv0pPs7J
+sf6ND3gC7SD5aq6/jK/QPsGI21t7+IvRD1ayc+6/E1g+P1hC6ZOtwOvvs7HU5a/PgnYr3Y+hxevy
+56+3diVw8eL0JMvCjoOpzYpI0CLTyXT+hdRmX2Pt18IEpY9O5C5v3pjaM8M8bflINnKRpy2XQlYg
+Hzwg//O0TaCQSWSByBClyEm3UETzAAVUuuSC7C1sBO1LSstyiJGMU63eBMYJsjC4+iL/C2cnWoDj
+ze46v/weL9kGmS/edxsjOytTH/c+m1t//BKbEzvSX5O0WESIFjT6v+cTzdkZg9Qvp1v9kTZ6D1Ad
+aXEhpf9yAT1PkrSlSq7tUChAd19EJGwisStI0pYsVItrdkzRgroHu80Av941A/ubyqFqjiY0ydfn
+rjyH5TN6Xe378r46vHus2u5naNWfTTQblnTh/vVnXGQvW1g1twcHzpbhsW0zuYZQ/H329+7TFXju
+w3rce7XbaJ7q/OU1Z/HGRzVo6Ym40huQYPnWkUqsW1KFuvIRVGQjBUAQTVuSIicwjlfYC/eC2TxB
+aVmJ//Om1wB/fg/wpZs54dl3jBfnS4Y0Wdeu4ho9MvvwE61SBJkN/ugF4MhZn1q2DMg4OeJ/50vA
+95/lfiyptFhXLge+cTeP8OUHZNa54wD3B0lWr4MngbNtwJyG9GSW2v2+24H/eNpfhLFkAin1xzJG
+np/alP46udbg5NpnbjxIRrZ92nIdhTPfo0eSwEoCdjbqR3MSaWOsucnvNReYWplU2qtQiC/i0PPk
+e27LQch/CoDy6WtZfdgL+dm3gO6+1ISI5m0ibX5AudkOnOBz8HiNF7/3H2QcU18u9pmbjd47x5sy
+sybxAs3vRNroHebn96meNP4/PJIlEib9+POCtHkn1xZNGx352JxaNsXxQ4kTC6r5kB1/tLElq2vq
+KcWPt89INKXYnhTiv2YIX17b7Lp+R38RHnlzJk+z4BRK2XnlkRi+ef1Zdhktp6341XVNeGVfDU53
+lkLR3IPxsS0zceOyDjRWu9UqC6cN4AtXtuD/e2MuDEWgljC3o2v1DYfxwt5puGROL5bOGGDf6Vno
+cy+pyr+PopzAzBdyFHj0eR4JkUxsgmjALFL2hRt5yQX+6zX24vo4mDlgJv1KmsbvMBL6WybwvPAO
+e1k7TDHpewrEctsaoKbC/3V3HebJd1MRTuoD0iRSRDNqz3RYvQT4lz8EntvGi9/8eXQPt7P633o1
+j5p5rIkLEHlBNgI8+gUd8j9HJCfXJCongg5SE4tvfio7v0uC7xMv8XnEEsippPOnvXoF9+F68hXv
+Z6yazQN3XRPc1Hss81S6xZgv3cLb7pdvcM2bqHWj+hK5pPouCBA98egZbqo3Ooq8hN9xTO3iNwAJ
+maCTdixbfURtR5pKIm5+2p6OoUWD3R9ncbELUuYZd9IGN2mDS9OmIDOvuIIM958/eGzrTDT3FnHi
+5WhDMrH42jVNmOlBeJ58eyZOdJQmzhNIs8IE6E9f0YZL53bnvP7Tq4bx4MZm/N2zi9xDIAac7SnG
+41tn4y/vOup5/pfWtODl/XU42FIeXzBQhAG941glthyuw4yqEdSURfPgqZNj3UJPHxu/z3NBPp+i
+am3ZzX3ZUq0aZ7Nvyczp8zfyQkSL/DgosiMJQ0SmigM6r5OWjTSZbZ3p6/P2XuDGyzmx8kOc65iQ
++JXbGIlkBOyd/UBXb8LMUhRSaiu4AFDP+nbudIdJp19v9kyjcuTi+PGMgKBn+Xg9B9ccy7FTaQ4U
+7pXmE/IxnZYm1QgtaHxyAzfrpnmIni8iQeTLSgL1pUuApfOCWw/4GgsZ9gvVmfynqMQ1jAP8uW+s
+T+8j5lpQYvPfzsPAvqPjPFay/JzQ/VOfkZ+cL9JmmjNm855Pt/gnbdRPNH83VPOFAynz5OF8kqxt
+/eiPLNImXoc4gOr4rCGu+Invt6LFA/bo8YA7IKATSr60n06JazW21aHpZlRAM5OtppmRAnV7oeNV
+xqZs+41rBR/Vu05V49md06DFuPbS1kZsQr90Vi/uXt3muvbhc+V4aud0I/iIK8E5E9xmVQ7j3jVN
+sDRDucbtF7fht7un4f1TVUauNhtYHZ/dXY9bL2nDKg8SWVkygt+7vgnf/tkF0BSHT59GvE/Bc7vq
+ccWCblwyJ8qIrD6G/tYdW6vp3P1sFTj3ATnWXRYWPj4DPPQb4A8/zYT7GRNfn/cOcA3gmbbcv+89
+FzFqg/n5eeGZLTw4wLCPFWo65kcvccEqCHG2NIBjeYdrWW5PLeCzFZRraOP07GoBuaQ2wfeZSX3H
+cw7UMHEyo9g/5CNLWhS/z1kQDU222j4bbVVRysoY602m3W/uBvqHx7e//N6/33G8iFIb+Ay6QkSV
+xkhTR3afjxPngGPNjFBf4tNEcjaPdNnalZ0xImWe3K4peHmYpYrgT1A9g+bBndjNqbIr5GIL/q9b
+pm+a+bdm/i2axWme+zMtI6MKfvjmbPSOhBOEzQogwh7MEPuNB687g/IiR4h/nbRss3CuN+J+khR+
+b19e14x5dQMYr0CkFcVRfH3DWRSrMTcjZ3XsGg7jka2zMDSqep5//dI2XLesk6cAcI5ijUhqGV7a
+W4/zdM8TEGjV6nte9EnzDGSrUJSs7/0E2H/cv/9YLkDJbP/t59zXKuP7meBJncwiX/8gEYDET6Fo
+ZWQOmi0/Ct/ELYvtmfG7JcAC3fi9W4ItHGZ1XGZaZz0Hdc5mm07QgynWgRaCjpzxl2bDL0jQJ40I
+afCy1bcTPYmROd+r7/F5KW/His9rLQ6Qm40IPfmzkUljNu+Hrke+bS0+53daPCWtnKoW7vMui588
+bclicCB5tHuvCPnJPuseCYontJAmTUtWqcQN244TbtoQ4E2tXCZl86F6bD5Ya1/2sPqA9cjNF7Xj
+2iXnXed9cIK0cw08cIfzwWPnXTy7D5+9rHncG3Tj0nbcuLzDO/caq+umQ3XYdKDW89yiUAz3X9OE
+ykiMj0bFXsjP7dX99djfVIGRqDLO9wbbQJcTh3f5iBG2f3yCB+QYGWcrVnqpkf/ID57lPmXjQdjo
+Pjt6snsfFFjk+8/wUP9B6/7qDu5Xl+06jelFjhwLBrkgmeNM3PLmPgNqWcd9jplgwmYJz7s+BvYe
+y95vvLkLeOldf0Qw2wslRECpZBM0/zy/nScXn7D3UZaeE/I3JPJT7dM0lObtMS0YpijkR3zcZxJ0
+w6ST1XtGbWEtdk0J0pWEF6X8DHefqXoSxzdP8oXkOQR8788jxjr2iV0wpfOJgRGueVJDcNmWUr4y
+Ii/3rz9r+LTZVuViKn7w5hxEdcWmmbNQFNLxuxtPo7x4/D1/KULlg9eeRVVxzD4hmOOHeO9DW+ai
+eyjief7l8ztx2yXnDS2j07+SFJwtvUV4ae80tPbkMrulwzQS7n6VE0/ycuYc8PePAr/clIE/WYag
+Fcj//XPgp6+YCbTHaTWftIu/ejN7fgNE2Og+Dp/i2spM6k/E9T9+xYlrPpC2XK/mZpoHLp+0QlnX
+nORai6VPLU2bs4/oOSWi5SdUfDpQsAgiOGfPZa/9g7QTETaaw/YezR5ho3n42a08AFRejxM/WrbZ
+PCedH9CcTZo20sbm4r6IDJIWz68lC9Wb6p+t50nKNznWC/jcbyEsWgsqjvD/nvnZJkUQEt1m6pjw
+c0pmAmmdpblZRZJGMIJqeDjwPfXBdOxvrkwk0hYImGoEEWnGytlu/6+X9jbgvePV7miRdC4778bl
+7bh2afuEdciFjT24d00zvr9lDvSoUDczBcCh1nI8taMRD2zwzv749fWn8fr+WvQORxClE4X7pLZ6
+bX8d1i7uQn1FG8qKRjPsczj6007SbGaRnv0vIymlXJAYAn7wDDdV/OKNwLWruY9EtkHBOn61Gfj1
+luxpl4L2Kwk8FEyAgg5UV2T+u9v2AD/8DXsxN489hxqZJVHi2gc/CaxYGDyqpx+0s/YejuYg6iGC
+a6kyEXbHQ8hHFu8j18lwA4X8R/7nacsFYbNA2jaa2yLsubr3Vh6EKVPCRkGcyByaIrMGNenL1tj7
+wMwFSXkwL5yXeTsRYXnyZW7aPRKd2P4KSty8QCkZiPgs9Bktk7Rsx87mLlImkWBaFKXgVOS/nA7z
+ZvC6U1qdWGxsz5OMmJ39uUx3/u2RgQxp87Qp3v2pp4t0X+CkzTKBtEweif0YW2G/IcBTwBJiYBSE
+hMwiVcFMzwx7aISsVxNkQPEIuXK6oww/e2e2sWJi82UjsHNnVw7hq2ubXE9JR38xntg+C6OkZXNO
+TGyCqS6O4v5rziBCfmUT2B+fv+KMYcp45HypPSiJWeefvzcTt6w4h7m17uyP82v7cc/VzXhm9wyc
+7Syxkz6GwVgIz+5uwMWzu7C4IYM3g5CnTffI0Sb2tz03GzeF5UWXE5gP0Krg934MPLMV+PR1wPpL
+skPeKKrZG+/z65KNfzYTRWeSFJmCsOw5ysOMU34cJUCUJVqVpVxyW8lhP4v54ylU+Z/+Ow/T/cWb
+gCVzx07eSLAjAeDFt3nwGT+C2VRNrp1tgShT88iAb8K8I7+2Nsqj/iEB+unN3If3y4y4rb3YfyRI
+8mGjZ4gIjuWDmolpajb60roWLRpR1FsioYHyxoFHmqRIt5Qq4My5PHieENw80guNdZz02KLmpgAF
+ILFMI3MFy0TSD2mjgCUW6STfwrE8T5K05QFpg5emzQPO1GuKwNeQ5LuCaj9dd0cSdEUYtJvI2ffb
+r6UoSvx7xZTerKAn1ndPvjsbp7pKeE4zj0b72vqzaKxyE5pf7ZqJA+cq7L5s5vlE/r60phkrZ3VP
+eJvOqBw2kmb/w3MUDdJ+f6QtO9tbjCcZaf2L24565l37wlXNOMEIX0d/BIPDIdcA23myGq8dbEBd
+eRNqM0gBIPabPWqk10TlbfYq5y//+OgEK4/zlx8lZaYk05SclV4mRZH059PK5f4T3Bzx3f38pTWS
+R5kfqFDofHoxblgF3HQFv8+SJFa85LtCq+tbPuQkKFdmpCQc0qo3/c6qC4DLlwGrl3KTmRIfFsZU
+TxJCTjTzKHA7DwUnliTEkXDrRwu577ipvcvR9UngGR2n9azDjNT+YpO/RQoS3PUs3ieZvPUOBrtP
+MiumfH8f+/Bv2p9BP40VdD/0rIxnoB0CLQql6h+K8Pr3jwHrVgJ3X8Of+2R9RGbUdA8vmIseovbj
+Y5/jhbQ5hgl4lvpSHCuH2fz1Tz/mfrGUj5HmaEpRkGxuofluqzmH0TyRL/B7/+meE9pP/ot+oxDT
+wt2J1tw+FydauGUJjTu/C21ez2qQ5yndMyAxJu7m+XeqqJFxcXjJDT/RFTUMRY0YRTW3vBQ59oXj
+W4ovb/xNW4o1b2xNpyS25cQlEU0irnlSvJIDjC/uX/CXCIfDiBglxATICCKRMNua+8wt3x9COGQW
+dmxIZcXYqkb4f5W2dH8K/6wonLRZ92sRuL1NtXjg0ZUYGA17BhG5dHYPHvrKhy6ftBPt5bjv4VXo
+GCiy96LZtFXFUfz7Pfsxv24A0Zjij4H7gZj7TUkw/jhp1xXUV46grMiufx8eDeF3n7gY75+qdceX
+ZSfT8Q9/7UNcOsebZG49Uo9/efECHGsrs2t1FZ7SYNG0AfztXR9j9dwuhNVg4QotMmYnbbCldNDM
+bUxjW/Z2HaUyyrf/vOkGHGybLmedbBD8ukR+MfpMQgKZ2fQPcrJGgnY2NVDpsGYF8MefA+Y3pj/2
+33/JBTDKZZTsvoiokvats5ebFJKgRsKE1znjBcrjM6chkTtu6Vz+YqcXM5FhImlEJDX5lpaQCIz6
+aq4JoWeMQBosImz0fLnM1PIYNRX8PirZK3hmPdfc0D3QXGEJ8hISEgFpmiB7Jv7WzPRSlmtWzNBw
+6FbRRvnfWmzzBz9deD2dFnbSOmcGbpu8n96Vq3DaUNPtmjUzqqQV/MLab1hLKjyqo25pyOhcw+FP
+iUfZNNqJ7VfMCCKWiSTPAxfCDzbPxYAW9myzsKrjG+tPoZx8tXQ7c3p821x0DBW5VZsw+hsDw2F8
+95fLxk6B9QA8mh0XG1WxuLEPD244jXWLE57ZxaEYfm/jaez6cTVixEYdprQDjNQ9smUe/v2L+xkZ
+dJOutQs72fU6cZ6R1J6BcMK3TefaOiJzr+yfjnk1A5hR5V+q1/UkyxrxgCe6/RnQdHvSc12aCmQT
+9PJvac+7adX/wUnGgnhfpB3MJxgJggXtHmnQJCQksoPzXbzsO1rY90ELTVQkJCRywN28uJzjs1dg
+yDhf0J3CuiDIKh4kLv4dCleAtedgE/NxqaZvm+bIz6UkiqniSuzXDA0jN5FU7SaRJnF77eA0bDtW
+x/3eHO1N2qMbLmzDtUvPu0TH907U4qV90+052RzEalRTGMEpcZEq2+hQkMiK7jjG4p5eed88rycc
+23mqDkeersC/fHYf1i5KhNK7an4H7rykDb/ZO4ObdIrEjZ275Ug9Xj/UgJuWtXgQ2Bg+saoZu09X
+4cBgJWLmuVRHxYxE+eLeBly5oAMby0ZQHI6lJmmufhdWOpxjwMrX5xgfohmlJG2TdD7NMBqhhISE
+hISEhES2SRu8o0c6POAU0QROt0s1ChDQdTlvW093RI80CpG1OIGz9msG9bIRN1P1liBuMP3WNNNE
+NOHnRmaNj22bhygjVy5fNiOIyCgeWH8KIcVhZhhV8fMdszAQC3kHgBHvJpaEXOm+x5E/OP3UKHn2
+SBF+9t5srJ7bjZIIZ3PhUAxfu+YUth+txfn+Ik7chB+kgCoPbZ2Dy+d1oLbMnaRmRWMPbltxDq29
+xWgzw/xbLnDUhp0DYTy7uxFLp/dgQf2AL+HZaRaZGPL2/uZHmGTdSd4FuicxKefTYMRNNpuEhISE
+hIREhrJGKp82L1kjHDcNU2Fq10zhVXeTOVvCYVOQ1tPL9vnXaMJ96JpA2jQr8XYiAbdlbpowkeSm
+kYYeTaFjuEZNV5W4iaThw6fw3/nNnlnY11KZ0DiJjcTK76xqwkUzukzTy8QB3QMRwxRQ05KQMCXJ
+aFACSKEexya1kkxiOmmYLJ6rRHd/GMVVI/H9S6f14FOrmvHw9vmJyyuJc/Y3V7G2mYmvXHXC80Zu
+vagV7xyrQddALaKjqque7x2rxbvHpqG+rBnlRSO+qbp1HVuwETHpuu7sf2ErtSuStSF/oxFKSEhI
+SEhI5K+MoXswNN1D04akedoc+cS8kiMno33ch8txsKIUxBJ0QoumCtoWLf5Zs2laHJq2uDbOMpVS
+zM9C8BVWOgeL8Ysds7yX5dkhc2uGcO/Vp+JfiprN0kgU1SUjieZMRs8VD4ruB0nInZKKrHlcm+pX
+URxFUThmmpYmcA+7t5f3TcOp7vKEts28BpHRx9+agxsuaMXs2n7XdRur+nHHxS040VmOM+0l9t9m
+5w7FVGw+XIsr5rdhYf1wapImknXYCZu9sH7X9LhWzQhMIhSvSJMSk2Q+zdO8XxISEhISEhKTmLSZ
+QoiNrFkxNhyRzG3JtdMWG2nT49qkQjMUIsFcU/WEdoX+1mBEDoyxD6qmImRED2R/K4zWKfx44zz2
+OUaJ1tQQv39jP1eD6TyVm0mCdbx/fBrOdJQmeJzQTJRI+96rTmJ6xYBntLbSoig2LGnDB2fqDL83
+PZaEeOken1N9l4aEeV5f8f7OChZ6/YWtjLiNuO6jvnQQX1t/Ev/9+Yu435x4OmvCc33FeHT7fPzN
+Hd4RG6678By2H61DR990DFgpAATSd+RcBU6z9p1VPYBISPMk5k7B3J7qwYwaGY8eyfdprK1jGidw
+RtGtrRTUJzNpC5rjSI4FCQkJCQkJibGTNsStHJ3pxezJtVPK7rrdv03xWzfv5NL5I6AJ/mq6mvBn
+01VB42IWxpY0xjiImGmMjWkq900zQvsbLIVC/5NQz1MAaFBgBfw/3FbOCJcCfdRRAUZ2ZlUOYsPi
+Npd2SsRnLztlJJr+zd45GA6HXB3l1JJ6DRBXN3ho/JJ+r6QWciPsbu9Y0YTPXUEZHDVPIfaWZU14
+ce9M7KAUAB7E87d7G3Eru8bl89yhBMvCw1i/5Dw+PFuDgZFS2z3Rx47+IsPvbXgUnuH/3fnXdDtp
+A+JaU01LhP23NK6a4OumCZpVCTn3Sp82CQkJCQkJiQy4m40vefC4pLJGwjzSXD7WBSNKRWB+ca2a
+KLnqSaR5RcnvRiMBnHzRGEnj5IwRsxiRMvZZiRnEzNqvx/crRsJo0sLYjSBhmFiCkQa6hmqq2qhp
+ugcVe1MImqua0hFUlwxxFWgSlIZH8d1bDmDNovM41FpjnqsnqLSDmCnWP5puo91WImsjgIp5tpkG
+PJ6DTbGuq5upCgAhAbZ5l+bNGASHtc/Sxl5suOC8oeVKxj3LI1F8Zc1x7GuqxgAjtoojouVgLIQn
+3l2IFTM7jXQBTjSW96NEjbmIJF0nxvpvZFQ1+sSrHZ2J0TlBcxI4wVRWc5hExmKJbUyaR07qiTQg
+C5OaNgkJCQkJCYkgbE13CBy6ZbqD5D5tsGnadFE3B5/mkeYPOH3aAhG2IInBsi+gGam4SKuiqnHB
+XTN82VTTLE4ztGfcLE4TNG3snjXN1LQlrqkaOcm4xk0xTSRrSqO82Ty0Wd1DEXT2hxkxG05b32sX
+txglX6GlyXG9ZkELrlvSipcOzuRaR7E9GOHafrQeWz6ejhuXnnWd2zkYwUhMsY8/808ii0UqTz6o
+JamELj4YXmQNiWAjFlkj00gygU2YRSY0bVK9IiFVbRISEhISEhJBSZvnZw/VWiJ3sMOnzQhaaGrS
+FIc8bQa758U8JiE3mxobY7+lt0lAEc0pTUFb8bTFG3/iFte0KUoiyASRMkurpsY4SbO0cKOJ4I9G
+CVvZD4RGJV84VTXaQlG5geSC2h6u9fIIznK2uww7Ttbh7pVTI4vlfVcdwTZGznq0IntOONYuUdbu
+j21fiCvmnkNZJIqQaepIZqm7Tlejezhiyw9njC/GkevKhlBfPoCQMpqctOm6Y5sga4j7spmETfxs
+aNiIvMUQG+WaNuN7OfVM+vnUz7EaIMeChISEhISEhE/Zwe3j5BVj0JaiyvF92ArnbzN99CqCCSWs
+FAHxLNuWJA0h1KGYQVpkkk5t3PgTNx3FTCgfMcwgEyaSikG2LK0aCetkDhgPLGKZRdLfMaveVk42
+xQj5r5smhIquGDnaVjSeZ6RiGOf7SxKmVCb5oEuQWeBlc85hZlXfpB+si+vb8ZnVJ/DYu0vjJpnC
+6gA+aq3G07vn4dZlpzCjcsDYve34LGw/PgM9g0WuMUakjaJOzqjsY6Qt6qnts6cZFImaW+NG/R4z
+CZulaeNbRtqIuJOprDSPnLyTqQz5LyEhISEhIZFFtiEKGLptH0wDx4S1YzwQifl9IoqkGD3SJGJx
+LZruSA5gFYOo6QKBs8L46dzBSBGIXBISFg9QMsHEbUQvRQmGEyZx5KvG/lENTRsT0o0okFzTFosh
+roUUtW3GPYS4ho2bWCo8EqXKCRudX1/Sg8vnteHlA3NddSDTyhOdlfjb5y7DH127D8sbO1wJticb
+7rmctG0zcPh8tSsoCbXp07vmYXZ1DyKhEXxwqoERvAtxor0iQcisIaJyondhQycqIoMGsUr1vNiI
+WzyRuhA90owUGYubR+oJ4mb4s8UwErWiS8ppaBJPrZK0SUhISEhISGSZsOl2IcIka7p4jOjTBti8
+1wTSpts0aIqqO4gbYKd9Dk2bWMG4KaBJ4pCCmHkSt/FBVK9AMTrtvmpGGgAS2BUj5D8J7CojUVyz
+ZkaEZH/ESJMWU+IkIBSyfOIUI/m20TSKahBBMpe85cITeO3gHIxaJpKi1pKRgL0tdfjTX63FuoXn
+sKC+2yB7YkhQzSK6JlGmz5pBspV4zBGLVDpbOqabmj/FTISumInCSSOoKkgQdn5N1emP6LgofW1k
+O1Ac3wvaVU3nW9V5KfZ3USiGkvAoPAORsuu29JbhsbeX4aniRfiYEbteMosU7X2VxLVmVfXjyrkt
+qCwaMkwXvYVr3fUIOXOzWeH+YzHLRNIkbYamLWaaSGroHSlBNKa6rikxSaZXXYyVm34ylqRNQkJC
+QkJCwhd3c/9h91ezrME03Rb636Vpiwu3AjFT4lEkYV9S1nU3FfT0olMEYmbtUuJcU7F9P/4+bYNa
+LcrUk1B102+NMQwS0BXTBDKmcqJDrlWKpWkzzSLp+1FDaFM5YWP/hVSuQ9QMvzbuF6eaEShXTGvC
+dRecxmtH59GJ7txn7Df6oxG8emQ2cHQ2dOE7JYlwqTiaF6K5oUDgFLFHTNLmcDOMK03pmloy7qwk
+SJuwKGD7LVvddHudRHNIxWxTI6ed9XPmOZSL7nBbFQ9u40HW4tdi2+uXnMbc6k72R5SRU6QlbTaT
+SAhmkVoipL8matgsnzaNk7f+kQhGNVUK6pOYtAUmeXIsSEhISEhISKQgbboo0Do4lC5wL1ueNsGF
+x6Vpi4fzFyR6RTw5bhopfLa2guObbgYs4WHprXXrNATNS/LJMZFrH25EXbFukCtDWxUztW1WWP/R
+mGG9p5raLLFQfUOUz80wiWTETQ2xzxr3f1NVbmap8qAmnOzFcO/qD3GgtQ5neysSNpYicaPPpoWf
+4tyfTPnl0FYpHjFeRJIGR/MrcBB+L5LopSyFkFpAPMaje+2BaYSdauIaznvQTSJrq5yooGXnXj77
+HK5fdAKVkX7ERp3aNN1zeNk1bIhHjRTzs1nJ1S1zyVEaB+zzaCyG831VGGLkWgrqEtI8UkJCQkJC
+QiIpj3GLzq7jdJtSzBnyP6FcsPu0mRey5WFTxFTd1t+Oz2ZSYsUMdW/P/wWH2sWUvG12cWkyN+eQ
+uO3vvhKLpz1vmhUq3CwypiQCjRh2gErCPJL+GRWEf9KuWYJbCIbGjkwiVdPUUNW4KaVimiY2lrfj
+W+vew/96cx06hkoSxM2zV2FXg3mRKD3JqFCQ3D1QSXKMnr47XAm39TR18FNfYfwmPcRJAllbL6zr
+wudW7jXaVPRlc5lCOv3Z4MzLZjePtFI7xM0kTbJmkDZG3g63NxomklJOn5zYcwT4q+/rKClKf+yp
+VqB/WEb8l5CQkJCQkIQtNWXTvawTxeAkekLxJcaMFAPoJUhbsmxuhlZNS/i4URQGUdtGdm4qt2Mz
+8rWRdk2xwv+LpA92okaVUHyEHcmhRNTa34BoQz27hXZO2mImeVO41s3iVKNCXXTVbESzPWgbIm2b
+EXVSjWvaVNXyf4NA3BSsnHYMf7Y+iu+/swZn+6sNU0BDo6Q4mikZKVJ8tIvuJkuuPHF6wPb2Imyp
+yJqf4511VFL8rqDmvKihDfet3oll01qMLOdRLdVz5Eys7VPb5gxEwrZDUQUnuhoxMFIsfdomKbp6
+eZGQkJCQkJCQCMDcUlI2b8Jm+bNZVo2abb9uRL7T4ts4aYPA5BQh5KRF0gxtmkDYdIPI0QVUIVW3
+o9APKGrCRDIJceN/KWkaITcatzMjl2NB+CUzCAdxUMWIGqkIwS4MBduo1YgmQQ1xs0jaUiJtImlG
+UUzCpvDk2jxnmxLX5hFWTDuBv7muG7/YtxrvnF6AYS3CIyOKys10t5xMmxZEe+Zh2uj5G8mukez7
+IORNSVNHlX9H6wKVRcNYN/847r5wL2ZWdBljM+ojSZZoGhlf79DFqJH2oCSiaWRMCP1/rGsROocq
+jSA1EhISEhISEhISkqilJGuiIOokbKKSLM69NJO8aXHXNLd5pEPTplgMTxWCkkAkcpqxJT8ug8wJ
+GjfjhwyyoybYo6VVS+IfpaeRg3MlJm9uvg33zdvKbnPQEMzjVbL8qmJ8G7IE/pAp2OshI/BIyIi2
+qMW1a/HP5CdnaN00M8KjEieBhPqSNnzzildw7fwFePv0Qhxom4Wm7mpoISX++4qfDAg+FT5xP7Ek
+7e9JwkQ/NfHQVNo+xeOayX4rGdkUxndEiTGC1oPlDc1YM/cYltQ3ozgUZSQ6wCPllVgbuou0uUwk
+49EjGTmM6XjrzEqcH6iSWjYJCQkJCQkJiSlP2fR0AqhDAPYgbNCsZGxxrVs8CImlabO0bRZps0vP
+pmZNsTM+RTHVc4rg12bzdTMD01s+bBYRNM5T7cTNGbFC94q2ofhvmAwxHIvg2PB1WFL0vBHD3ohq
+SK5sMbvQT2TNqocR4l/n2jZNt0wiE1o21TSTjGvYrM/x21L+f/buJTauq47j+P/c64kdJ6nzImlV
+ER6FUCG6atW00ECbBoIQSEhIXSDUBQvKggWPsmHVSiwAIdFdkSqgAglQVFWqRBVBmzbkIUQAQR+L
+qqRKWprYsRO7Tho7Y8+cw73nnHvvuY8Zj/NoHPh+pOTOy3NnRrE0v/z/53/y4/YNx+RjG4/Jmfkx
+mXx3LAkFY3YT6cWOq9SV2hv9Z6Z86Ai3BciHUPrtAPIc5Pea6+piHaEyxWeaP4dpWOoWpOzyOsXm
+Als+HTLc66801SZ4ZSZ87cb/bfK03GoZWTtyUTaPnpcb18zKhpHzSUh2CXpxWYGtHtzKLZLSsK4t
+G0biL6dVttlbZCJtp9WRsIoJAAAAvYNaOVcV9/UIbH4PsCxzuXZIHVTjgtCWXJ9TokdNVkXL2hsl
+HP1fPKESXa64+cpSMRc/mFqYthT64CZ999LuUW6pxYMr69Cp+2XbB/8mw3rSBwhdfMZxNSC7aZM6
+jiXWxlbSbECzlTbtg5sOQlsR3kqBrVJ5GxualrH10/LR9fx7b5KuN9Td5fzOmHpwCxd1mt57tblK
+W7HZ9lxnRA68dYdMXVjv9soAAADA/3tCG+x+U57zYcLAVglqpuextE9bNJM8zWi+R5vyD1BB4vOp
+L09/aVDLbhO/kVk2LSLYO84V3lxwM8bdr0qbfTXs3tz4xq9OcEvXlD078ZB8aetPk5c4XzqPsR9m
+ui4vXcMW28/EbUht7L5uUXX4iL+crWHLLvcLbKrh/SrFuqnl/ceG6Xt7GNxKbZJ+b7ZSiNPFfm2L
+Wsmf3vy0nJi9WTpaCVU2AAAAAtsSX0wrjwuGjlQqbSbMVhIci9Cmk+NiEdpUPG3M4s0iupT+8mpa
++kNpqSNdt5Yf3Vo2E3VtEDHajcY3wSCPfJ8w+33XB7q0Jc8Us+zLAUX1GZ159YLMxLub5c9DX5dd
+mx5378tvEhYFi+2yL/bpbe6YBDXtQpwd89/1bZLBOjY3rCVsiZRKcGt+T4S2KxPaSsFNKgGuUmkL
+q2xZm+TR05+SVya3y9xCS4qN4wAAAEBo63+/6TGEJFuKlnc29qmyJV9IO8l303NFaJP4pOj2bcX4
+fjfmP1/f5oOaDWbZ0S2YEjsj3y8GM74d0GUyN4lSsm0A7F5uqhTe3BsKRwiaPgHm6lY5Xp/5SHKK
+h+Qzm5+Ulp5zIS1SRfgMQpvOWyOjIqiJKqptfnpk1h5ZCmyVitsgWZQQN3hIa/qdMtIwjKSyrk37
+8rNdz5YE8n9M3i0H375TZtujBDYAAAAM+H20UmmrVNlMvgwtCGhS6W60g0i66X1tZfRUHtqSsHXO
+ZOvXRPv2SF20R0p4TBcXZe2OvsrWa2h/nsNMPmhChZtuVWe+N4QTY967wPJaEtwm5x6WL3/gCRk1
+48nbjcT1m2Zf7NNNs9OgZvxm2sZuE5Dty1aENl0aRNIY1mSwvcMJbJcX3EzP4NY8iKTdXS37T+6W
+lyY/LvOdYQIbAAAA+oSzpi+f/aZGFqEtr6hJ1+Yq8X+Mya93krsv5KFNqda/ki+tD7gvqNnea11X
+RYt8cLFVNuWCTOTvU5KNKMxDmMmWpeUVtuwGdzR5xS0MME2h7dqElbPz6+Q3//6W3P/+/fLh4cMS
+mwX7PtLwGNm1eWnrXBbUdBDUsjVtqlxhk+aK26ChTAmhrfnXZOnQ1jiQJKuwSfm6NrG8PXeL7H9r
+p/zn/E3StYG9ywcNAACA5YW3ILiZSmtk3hIZVNeMlAObPWp7vJBkkON5aDNm+JfJIx5NAkmrmBCp
+XNUtq7Cp4kmVKapr9uhHuCub2GK/oXYs5Qpbbah8tu22VCtu1zqoLHRi2Xf8c7J21T2yZ9s+uan1
+ssR6wW1T5zfNtoVIuyl3ObSllbfqiP+mCZI9QxqVtUv7dVmi8lZtjSwCWyST7W1y6NTdcmxmmyx0
+W/7fKIENAAAAg8Q20zO0Na5nq7VHVgJbftRTEg0fkDApve+OJ/9u9MXbVRK6RA2JPUbpcchdj/wx
+vZ7fHhfHyB3TSp17jsjdnlXW8qPyG2/Xw1pRhlp5weXWjcfltk0vyfrWaVkdTcuQzBdVtfQVR8E6
+tmBCZK09csnKGq7Q/3E0hjZtWnKhu0FmFzbKiXPb5OWp7TJzcZ0N3wAAAMBlfxNtGEJielbauqVq
+mzGdrNK2kHxz/cOpF+/5Sva0dnNtNbT6adOeu91tkO03yvZPqGyfpSrWr5l63cxVniqVtvQF2oAW
+ub3d7CjJyK+Xa6iumZUb3F47u83+wf8i1q0BAADgygQ206PKVhrzn69r69bWtRmdBjc9bSQ6Gj71
+kHvOkV8kdz6ShKaW+0El2R5srvUxa4lULqBFUgluRYJMA5u7HLlKW7rXWb6PW+QDmioFNJMFtVJw
+64WqCAAAAIBrEMxqd5vKY8uTI+tDSIwLZ5JPiayvaTN6XFT8bGMC2rLj9z/RC9Pfd22PseTHyLdL
+9miJLD02u2zbIf0fN1ux2IDb35afvjZZkVAGAAAA4PoJdaYU1oIAV9ujrTqERFcD2ztJPvrV+IF7
+vxueZShPbzdseFTOzHwjeaKxfBqkbYfsBuND6i2RdkqkP9r92dL2Sr++TYxbw2Z8lU18lU3lYc1X
+17JqmyK4AQAAALh+Als5qJVvtxW1SrUtXNNWGUaSXNWvqOGRH1fPlIe208/tubBlx2+f0AszD4d7
+rxVtkMqubytuM379Wtgamb6I2L4I5Sttxq9lywJcFgalaXqkyS4R2gAAAACs5Mhm6gFu2Rtrhy2S
+ZlJFQ/vGn995umdos2Hphk2PqOn5zxvd/oTJw5PKA1s4sD8MbCpLjbY9UttWSRNU2oohJFmLZFBx
+sycuB7ji7RPeAAAAAKysuNZ43YStkvVBJPUWybDSptvJIw6pdRt+1nTGWirasuOZrXpx/NXkBzen
+Qau0Vi0b5Z+1P+Zr2aLS/eLXtKlwDVu2vk0Vm3HX2yFru0/zbwIAAADACspsPUJbQ5XN1IKbbtqv
+LU1vR2T16IMTz9/75kChLbX1rqfv7LTHX0wi1mg5uEWVwBY13B75NsjyHm2lYSThmraGl8Em0wAA
+AABWdnbrE96a2iNLEyTDzbXNqypa883xQ7uO9DpXz3R04117H1xsTz1eDm5+37VqdU3C8BZW1iIf
+wJoDW709csmXBQAAAAArIbZVU5y/dan2yGJ9W3LpDRWP/mDi4O69/c7UNx1t/eRT93UvntlbtEpm
+Aa0a2Kr3RUFlrRzYmtsje70cwhsAAACA6yC0Ddge6adKdo1R/4zjtd8ZP7z78FJnWjIVpWvcTGfq
+oNHt7UX1LMoradXA1tQamVfVwsBmz77EmjahVRIAAADACotrPVsjRWpVNh/kjA9sSXBri8QvRPHY
+t8cP73p9kPMNlIi2fvaPa/Ts2cf04jtfTX5gtKmy5qpo5cCmGtew1QeQ9G6TvKSXCwAAAABXKqIt
+leAqjw4rayLlSptMSjTyczGrHjv9lz0zg76CZaWgrTuf+ZBpn/+RXjz3QBHMytMhi6mR/Vojg1OX
+gpoirAEAAAC4jsJb09q2IrD5qtysxMNPRSPrfjjxwu4Tyz3zJaWhdLqk0XPf0535XSKdzQMNHOk7
+6r//y6FFEgAAAMA1jWxmwNBWtER2JIpPSjT8XKyGfz1+5AuHLvXcl52GbIAzi18zev6+5OmM6bZv
+TTJWq3db5FItkeq9eukAAAAAMEAQG+Rx6oKKWm+kR1FDf9Xxqt9NHf7iUT5LAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAFfIfwUYAA5mpSOKNj+qAAAAAElFTkSuQmCC" transform="matrix(0.145 0 0 0.145 0 0)">
+			</image>
+		</g>
+	</g>
+</g>
+</svg>

assets/thirdparty/online.svg

@@ -0,0 +1,789 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 24.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="190" height="20">
+<style type="text/css">
+	.st0{clip-path:url(#SVGID_2_);}
+</style>
+<g>
+	<g>
+		<g>
+			<defs>
+				<rect id="SVGID_1_" width="188.5" height="21.4"/>
+			</defs>
+			<clipPath id="SVGID_2_">
+				<use xlink:href="#SVGID_1_"  style="overflow:visible;"/>
+			</clipPath>
+			<g class="st0">
+
+					<image style="overflow:visible;enable-background:new    ;" width="1315" height="153" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABSMAAACZCAYAAADU3y8IAAAACXBIWXMAAE0MAABNDAFsjJP5AAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAqfhJREFUeNrsfQeYHNWV9akOk0cz
+GuWcc0BCEkgiJ5PB2IATDusMZp12P8dde413ba/9O9vrgHHCEZywMQaTswQCJIRASEI5jyanzvXX
+qdc1U93TXVU93T3To7lHFNXdU1396uV33rn3aigANn6kqkZLxMfDp/l80Ko16H6fphkvUePTAB4a
+D2Q+E8lT73sLGjQIBAKBQCAQCAQCgUAgEAgEgsJAN/6lvNcR1zT0JIyz8aeo5vN1G1fENJ/eMu9L
+3R2F/O0BM31P3VxRZyR0knHU+3zaUr+mn2ckOujTtNk+6NXGOWjcfU4KGZmJlLQSomVOkKYJGSkQ
+CAQCgUAgEAgEAoFAIBAUBHoGMhIIGadDuo4eQGvXNG2/cU2n8ZdnkcBmaDgBv9ZYCGIyZ6bv8RvL
+R8cSmO33aet8Gi43Plrh82GiX9NM0tHvU+Rj+qH1U0hqKQnISkbmnWKBQCAQCAQCgUAgEAgEAoFg
+hEJ3/0hP/k9Pf6/rMeN0GBqeMj55GAlsND7dP/eroZaBJscztffo+6pqY0hQ6XgBNP39GrT5JB55
+BIwP/SQb+d5+tpOQ4Fnrb56dbqbtkirhIgUCgUAgEAgEAoFAIBAIBALv0F3+0I+EtN4nX/Oc0PWw
+8fIl44NfGuf7NU0/MPt/Q525psUTt/fA+8pnJRK+C30+/Ubj7cqAnYRMvvZrfedUMlKDD1lMtLU+
+8lHLNVECgUAgEAgEAoFAIBAIBAKBIC/YiUrdTk6mEJEpr7uMV08bL38BaI/O+d/uA7n8niPvd98N
+8GvllQt1JD6Z0LW3B0wC0jq0JCGZSkb2mWprfSbaSPMbCRsRiUxBawQCgUAgEAgEAoFAIBAIBAJB
+sdHPZDtNLWkSkckjgT5y0jiajLe/hqZ/J1Ae3Dv9C+1xL7+Xlfd77F9rgh3t0XV+v+8LCV0/t5eE
+9NvISE2971VF9ioitcz+IuGRjBQ2UiAQCAQCgUAgEAgEAoFAICgu9DQyMpOJNvrISL5O9JGRfN+u
+Q79Hh/bNOOJb5n0lHHb7yYy0HxWR8WDl+fGEfqumYUbQroj0KyKyz1TbdiYJaakj+/mLTCMitTQ/
+kbYUCRcpEAgEAoFAIBAIBAKBQCAQFBd67/9s72EPYGMjI9GnjrQTkokEwgld32h8/vmyyuDjbgrJ
+jLzf3e8sXxPXtTuNP84I+u2m2UDQr/W+TiUj+4LY9IukjcxkZKYEiCpSIBAIBAKBQCAQCAQCgUAg
+GBw4EZJ6NkLSdsQVIRk1Xj+pafiYP9L94vRvmpxlRgTSP7jnneWzorr2bePGM0zfj+hTOlpBafwZ
+Dot49CP5neSRkYiEXRWpXgkJKRAIBAKBQCAQCAQCgUAgEAwBtPRI2nqfibamXiS0PkLS/Ery8+Tr
+oHFak0jon4n5Kz4NhF5z+Kk+3PeOivqQrn8vlsDbLAVkMGmaHUyqH9OVkilRtbUMqkgtExmppZhj
+a/1SIhAIBAKBQCAQCAQCgUAgEAgGE+nBaxT5qPe+TmRQR8YTfa9jCTQnEvovfbp+y6yvh1oy/UaK
+MjKq61eHY3gbyUVT0YgkqQgbwYj091qKEjLlbzYVpDq0vtc6Uv1D6lLgAoFAIBAIBAKBQCAQCAQC
+wVDBEjuaZ916ryVJSt3k/SzFpAX7a7+GBmja1bqub9r/Mfw2k7l2Lxl5zzsqx4biiY9rGdSNvgwR
+st0Oyy+kaaqdVEPa1ZHWA4oiUiAQCAQCgUAgEAgEAoFAICgN9BKRmo2YRJKUBBWQiqW0iEmfjdsz
+/w5MNq59awwVjwKhg+n37yUjI3H9XZE4ltMcu1e9aAWfQSZFZJ8qMuV66zu69ZmWpo5MM88WRaRA
+IBAIBAKBQCAQCAQCgUBQMkhXSJpInn0mKambkkefdTH6lJQ+DeW6pq1IaNp1+z+Gb6erI00ykqrI
+jnD84yZxmFRAWqShpYrU0hSPms0fZErUbPSpITMGrrH7ihRVpEAgEAgEAoFAIBAIBAKBQFB66I1U
+g1SJpHkmf9jnYNJSSJo+JjUz2M14aPrlUZT/Dggfsd/WJCNjCf3auK5NCthJSPRFw+41t4ZSPKpD
+M/0+ar7U76S+Tl6jod/ZfBZRRQoEAoFAIBAIBAKBQCAQCASlC5sy0uIjTamjyQ3qvTygRUgm1HUB
+469zEvC9zvj4F/bbmWRkOKZf20+9aFc1AlmVkVm/Z5lnOygjLQwHhWQ0OBrxQAV0X7lxGNmm+ZHw
+lyeD8GjJ50g9pz6jyECHClo8DC0RM1pFDL5YyHgfgj/cZr4XCAQCgUAgEAgEAoFAIBD0wTLL7mWy
+7IpIoJeURFrgap/ed1nyGKNpOGP/x3C73VQ7cPc7yxvau7GqV7UI29l+ZPrM/jf792EjItMVkdYD
+2bk5vTTj2ESC9eYR81eidfQqhKsnI14xHonyWiPnKhGpHA+/3wefZhw+dZhm7smzdajndScsBYVs
+OH2yW393IwKxTiDahUDXIfi7jqKyaSsCPY3whZuM8wnJMIFAIBAIBAKBQCAQCAQjHnbO0Xxv5+yS
+ZGMvj8ezpY60EZiWUjKhocb436IoKsYBoWPWbQKJOC5J6KgPaBkUjsigcLQ+0/oHuknn14arr8hY
+oAbdZeNwvGEdWuuWo7NyOnyBIAKBAPx+f+85aBw+nwa/z5+ViHQiJNNfCwrQaHS9//vaiYgZZ76O
+jFlqntuMo7xjLyqObUTNgQcQ6DoMf7hFMlAgEAgEAoFAIBAIBAKBIAlNsxGSab4jdds1dqFh71k3
+/zQxAd/Zxts7rXsGYjGc03uhPTpOBpWkdZ35mW4jGm2qRztJmSlSdnoCSw1dFZPRWLcSh8acZ7ye
+iIA/gEBAEZCpZKSvl4T0+93IyOyKSCEjC4uMZGTybDpRtb2P1c9BR91sdE29ADV7/oaagw+hrGMf
+kBDzbYFAIBAIBAKBQCAQCAQjD+ksVbaI2rDxhbqe2bI6eb86Dfpi+z0DcR2z+hSLWkYVY2/AGdtn
+1vt+qknYVX/Z/USWGnQtgPbyKdgx+Xocr1qkyEeTaKT5tb/XDJvEozqS721/609Iop86Era8FhSx
+PDOQktZhvWcZ8JyonoC2xe9GaMJpaHjphyhvesn0MykQCAQCgUAgEAgEAoFAIFBIV0f2+ZbkPz2j
+a0fj4wrjusn2+wQSOsbAyR8kbEpI2GzCNZv60e4HMo0BdX2QEuDkdM2PpuqF2Dr+enRWTO4lIdXh
+Tyoj+w6liuxTRGZSRtJ8266GFNPsIShXGyGZTkTyMIoKiUTCLKtEQkNk7FI0rv40Grb+ENVHnoQW
+65FMFAgEAoFAIBAIBAKBQDDioKdbO+v9X1t8oZ7JRLvvddC4vs5+K5KRdXZurF/0a2SIrI005SRs
+akn7vdLfpyksSwVNVfOwacLbEfLXIahpJtGolI59Ksg+RaTf/DuVk3YikoRWP2Vk8qH7kZHpGSUo
+SqvRM7y3KySVOtL4LKH31slE7VQcX/1pjN/0ZdQcekRMtgUCgUAgEAgEAoFAIBCMWPQKIe1KyAw+
+I+1+JTU95fsB41Rtv2cgkTDDbGc0pU4nFtVnWlZS0TLRduLZSi2ITUf5JDw/9k3o1moR8GlJUlFL
+BqaxDp/t0JLKSC1ppq31EpFKDemzqSJtptpIZWuFiiwudKuCWWpI21v7YZQcEmZ0eRXribJiBCtx
+YtnNCHQfQ0XTVslMgUAgEAgEAoFAIBAIBCMKdlrFIiTNz5EqkuwjIm0MpS2ytoGApmk19nsH4gnU
+B/3oC1pjC1yDtMNult1rkm2ZbOsOD4C+60sJUX81No+9Hq2+scwZRShqtiP5PpWMNA6t73Ofr++s
+kZTUfBmjaPcRtUJDDiYsfaT9bP+n+ew+DRLGBT7jPx3x6nFoWn4zJj71afhDzZKRAoFAIBAIBAKB
+QCAQCEYeUlWOKYFset/bjwzfS+cDAynBaDLwZOmKyXQT7WwYDpzb/trTcMA30zTNNtWNlq9HX+r7
+lEPrr5702dSRvUQk+hOSfTk4PPOrpNuG7lwPrb/3tg+tTx1pdz6g6z51hfGHcMNitM25Bg0v/4y2
+3JLJAoFAIBAIBAKBQCAQCAQOcPAd2YtACnOpp15oqvns0sp0VrM3vrftznbG1E5k6mmpGWKVZChQ
+h6015/WLet2PfHQ57N+xKyLtZKSmISXrhXgsQmXPkKd9RGOqbwPYqi0pRpZdeuB5XVdl2jbnjajd
+ex+CnQclkwUCgUAgEAgEAoFAIBCMTKRxKuZHNktqOzXoLMdTTiQLCsv/ZKljT/2Z6EQVgj4kyUT0
+Ixf7EY5af/KR3+slMjVbsB8tPaiPbssjYSMHvaXY6qdSRKoy8BnXJFKIY90sW4uMjJXXonXuGzFu
+87clOwUCgUAgEAgEAoFAIBAIbEgXf/UTPWYQIwb6XezxSPcZWWr+IJ2ga37sKj8lRRHpdPiyvoZN
+WZnpXipj08lH4SKHrOStEkh57bMKJXnoveXrM8PbdE47D2O3/hharEeyUCAQCAQCgUAgEAgEAsHI
+g57hfTrp6NVnJP+X7hPSK9KjbQ8Xjq2zbALa9FGKWLT/s5lY+7S+o8/sGr2EZMrnGf/Zooun5Ywm
+sbRLoAWlVt5+Jva2+hAvH41QwyJUHn9esk4gEAgEAoFAIBAIBAKBIA29ptlp7h6tQNl2BOyRsvv5
+d0yLrG2Ppp3J1+RwwcGaU5DQNPihpZpVA1lUjtbftbS/wfUA+ishRRmZP3p6Quju6UEkEkEsFjOr
+YCAQQFkwiMrKClRXVXm/me5ejl0T16PymJCRAoFAIBAIBAKBQCAQCEYgPCgjU4hIZOAZkwg48WIp
+ykd71O1hnn9NZdNSiUfYiEibqtEe1qTv71kO3f53PRn8J2miraeZauu6VGIzG3SEIxGTWAyFw/D7
+fCaRWFFejmAw2O/6RCKBE03NON54Att37MKuPXvR3NKCjo5OJIx71dZUY3R9PWZNn4alixdh7JjR
+GD9+HAJ+v2Mr0q2w2skjU/mGG+ZLgQkEAoFAIBAIBAKBQCAQuCGTkNGGVJ+RybNdDZnR/nuYKyO7
+tep+n6WqHbXMPiBt16V+R8sYlMZ+nf080hGPx9HY1IQjR49j12u7ceDQERw9dgwVFRWYMmkSpk6Z
+hLmzZ2H8uDEmuUg0t7Ti1V2v4R/3PYgHH30Mr+3eg1A4Yn7HH1DVOGHc1yQ1jXyePWsmzlx3Gq68
+7BIsXjDPuNdYM/91Xc9c9rCXo96vbGmqDeGQBQKBQCAQCAQCgUAgEIxEePUZqbt/NzAS868LVX2E
+I1IJSMJ+zmx6raecNVivVTTm/sFr1LmruwvRaAxVlZUoKwue1HlMpWI8FjPPPp8PwSRh2N7egZe2
+v4q//eOfeOTxp/DStpeNvPGZSkgqH6PRKCrKy7D2tDU47+wzcOWlrzPNr/9419/xy9/+Hq/t3osJ
+Eydi2szZWLxkCaZNn47q6mozf7t7unHo0GG88tJLOH7sKH78s1/hr3+/F2++7hrc8KbrMGf2DFN9
+qWXQCWsuptrxijrpeAQCgUAgEAgEAoFAIBAI3OCqjHRTPLp9Dgw/ZSTKTIWcqZJLElPWe8Uh6r0E
+o57FpFp9rpR2uqbMsjNdG4lGTSUgIzPf//BjOHT4KGbPnIFFC+Zi2tQpJjl2MiEUCqPxxAkcP9Fk
+nJtMpWJNVZWpdqSK8f6HHsX3fvRT7Ny9BxMnTsTKVasxdfoMNDSMQTgSxonG49i7ezdeePEl49pH
+8PhTG1BTXY07/nQXxo0fj3MvuABXXX01Tj/zbEyYPBm1o+pQlizDCH07tnfhxNHDeP7Zjfjrn/+C
+5559Fl/95vew+cVt+OwnPoaVy5YgGAz0lreyzE4z2YbeVz+SSARrRBkpEAgEAoFAIBAIBAKBYGQi
+H2VkGkakMjKiBxDMYm5tvVfof42FPvWk1hc1O6mw7OjswvHGRrS1d6ClpRU9oZCpDvyfr30TejyO
+8WPHYPacOXjX296MU09Zav7tZMCRo8ewcdPzuO+Bh00y8dDhI+ju7kFtTQ0WLpiLyRMn4m//uBfw
+BXDu+efjDW94I9accQamzJiD8rIg4rqO7s5OvPbKVjz28MP4w51/wEOPPmGqJZlf7/yX9+DNb38b
+Jk6eatbtqPG/aBwIJyu6z8j/yppqzJo3D3OM48JLL8Vf//AH/PgHP8CDjzxm+pf86hf/E6evXmXk
+udbbYjLxzdlM7wUCgUAgEAgEAoFAIBAIBA5wISUDGRWObgpJDcNaGanSqyefIZl4Xe+NqmxXRsKm
+ljTNrWELT259rqnvRqNx7D9wEE89swmbntuMAwcPoDwYRNDnQygaxa5du/HFT38cq09diu/fejs+
+98Wv4Ntf/SLmzZk97OvZgYOH8N0f/wy//v0f0NUTxtgJ4zFp2kxU11SjtbkZr+7aiyc3bkJtTS3e
+9La34aYPfxgzps9AxPhuJG7kXQJmzlbW1mLN2vVYvW49zjznXHz/O99BU1MTPnjzzbji6tcjZlzV
+Fc1c5xI6g9wY91K3Qs3oMfiX938A8xcsxBf+83PYuOEp/Mct/4sffvurmDd7Zm+ZW2VrD16TKZiN
+KCMFAoFAIBAIBAKBQCAQjEiIz8jCwEntmP53uwLSfq2FSCSKTS9swQ9u+wW2vrQVp5+6EtdefiGW
+zZuDmppqMwr0y6/uREVFEGecvgqzp0zCez76H/jOD2/DN798CwIB/7DNx6bmFnz569/FT375a0yY
+PAVXXnk1zr74EsybMwcNtbVoP9GIrS9uwcMPPoyFS5bgwx//KKqqjc8zkYoJoMM4+X3AaevPwPQ5
+89DZ2YlZc2YjFAdicY/R3I37hiPG9cZ9zjrnHHzl69/AzR98H554egO+9q3v4+tf+rxp/m2VrS4R
+zgUCgUAgEAgEAoFAIBAI8kc6SZmG1GjabqxmNl+Rw04ZqdvUkUkFXFLt2U/1mKaMtD6z/y0WjZqq
+v89+4cvYu3cvVi5bhFgsht37D2H2tKlYcupyTO+ciOqqCvT0hID2DkwaPw4fePt1+MSXvoU9e/dh
+3pxZwyb74omEGYiG5tOaz4ff/fEvuO3232Ds+Am46ROfwjVvfweqa2pA4/NaI4PK58zBqtPX4oo3
+XIfKmlEIlgXQFclOKvJzKhx5TcP48Rg3YTxCUVu5eIR1H5Keq09bhU9+9nO4+QPvNf1PXn7Jhbjq
+kot6lZGajn6qyP5lLv2JQCAQCAQCgUAgEAgEghGIXJWRevbv+kZi/tmjYlvv0yNku9us973esXM3
+PnvLl/Hitm1oGDMGK09bh9POPBs7DjTiQ//xJTz2yJMmmVVZUY7ysjKYgW+6urBy8QJ0Guef/PI3
+6OnpAbJGEyqdg0FpNjyzCT//zR343q0/xw9/+kt870e3wef344YP3IQbbrwRgYoadHQBrZ1As/FY
+7WGgOwrUjWmAPxBANOq94vHaUAR5EYE6ic04cOXVV+HqN7wR4XAYP/nFr01/lox+nv6MqZHT08tc
+IBAIBAKBQCAQCAQCgUAwUIxcM22vKtC0v2vJf9braCSK/7vtF3jxpW2YO2cOPvOpT+L6696AikAA
+73jb2/DO974f3/3J7Tjr1GV45/Wvx8r5cxGPxaCRnCwrMwPa/Oz23+CSC87DeWetK+ksIxH5o5//
+Grf/7k4cPHIU5eXlCIdCiEQimL94Cd783g8gEteMz/rIvO64+m4lsy+MXmljQu9v6l5MxKJGGsp9
+ePu/vAd3/fEPeOzJp/HyqzuwesVyR/5VqSVTTfQFAoFAIBAIBAKBQCAQCARZ4Gqm7cUU2ylwjYZh
+KhpzYR0thVzK9fa/qfPL23fgL3ffgwUL5uOWz/0nrr7yCrR0tCOi+TCmYTSuvPxSfPsb30Braztu
+fMebEe3qNs2bg4EA4vE4qirLUV9bi1t/+SuctuoUVFdVlmRuhagmvP13+NLXv4260Q0484KLMHX6
+dBw+cBCHDuzD1W++AWMmjUNXV3+SkarEiJFdZcbngaQkkgFryozXFb7Bqz4MfLNy1alYddppuP/e
+e/HQY09i9YplcFaD2spexJECgUAgEAgEAoFAIBAIRiIKaKYdGNEZmFQ39p6R9j55XYo6zpa5hw4f
+xQ9+djumT5+B/77lc7jyiivw/Kbn8Me/3IXly5fjujdeg3A4hKrKCjNAjR4KmwSk+dO6bn62dsVy
+rFq5Aj+748948aVXsG7NqSWVVQy8c/zECby2dz++/YNbTTPrt3/gQ7jhgx9AVc0otLV24vDBPZg2
+ewF6Lc3TwPyLJqNca/G+IiBBGdDUARSf66P/yPKKANafebZJRm549nnnxuMWDUogEAgEAoFAIBAI
+BAKBQJAKTwFsvLtHzP73YZcrehZfkXZ/ktkz4MChw/j+T36Onbv346tf/h9ceO7Z5l/uvf9BfP1b
+38EZ69fh+jddhyNHjqGubhSqKyuRIBuWBIPAVJeX45Z/vxl1o+tx133349nNW7BuzcqSyaVjjSdw
+2+2/wwOPPo69+w/gRFMTTl27Dv/ykY+jYlQVIiFg1Jhy1I8fg3hMRbrOBi3D+xgjZxvfqw0oQnIw
+RLbGz2HJ0uXm6/0HDiJqJNyv+eApgpMQkgKBQCAQCAQCgUAgEAhGIkQZWagczEUWpyORiKOrqwvt
+HR343q2/wLYdu/Gl/74F5565Ht2dnShL6Fi+dAkuPP88rF+3Fprmw/HGE6iqqkRFRQUSsVhfCnTd
+DOIya+pkaD4NC2fPwI4du1AqjBfJUhKRX/zat1DfMMY4RmPKjCAuuPwqVNVVobtLBZim0NMXVwSu
+36fUh7rHRyD5GOb1RrZUGzUxqKnANnqWawuRM7xPw9hx5utQKITu7m7UVlfDWwAfQUmhzCg3X1oX
+ljAqU7RHRS0SCAQCgUAgEAgEJxcCFcZRnra8Nxak0bCxFohK/ggEpQJPPiMz+YL0wssMV47GTLfd
+D6Cu3lufW2ya9Tp57N67H09tfA7PbXkRz27eitt+9EOce9YZ6OroQML4e6i7y3y/ZNEi1NePMpm5
+puZm1FSUG/1lGcLRaL9kREMhlBl/HzN2LHbsPuCdySsyOju7zYjZfK63vO8DWHf2Wdi/bz/WnnM+
+rMfw+dQRi3Rj32t7MKquFuOnTjeDxWR6DF5rz14kq14kkQwwY/y9zA/4k59b9ZKXx5PfKStA/Hef
+T+k0E3oC8VjCVhfcDulPhgT+MqNy1BnHaKBqtHpPjJkNlNekXhvqANoPGxXKmIyEjdc9bcZnrcbR
+bhR4XPJSMPioGKUOzZ/9GhLoPS1GRycTaNVJG3kVrFRnJ7Cd8xiKcXM4pHG4toeM8zZjrO48rvIy
+H9SMN8qtAsg1KN1gtNHqMUbaqoykuUx0YiGgu3l49BckDDh+8zxUiHaruUA8Iu1PIBh284GAmutz
+/s91QFmV6r9rJ6gjvZ/uOKbm/Gafzfl/m3qf79gxXMExhX2wtXbKBPaN7CPZVw7WvQTDG16VkZmu
+F2XkwHH77/+EO/5yt6kaXLxwAVYsX4ZIKGQSdhZoij150gTEYjHEohE0Nzdj2YyJRk4HHBcjmt+f
+YsY91GCE7LraGixZugxvff+HMHPaOHTHdQT1GLojCRjJRbinE3Gjg9/w2Ab86kf/hxlz5+JDn/o0
+xk+dYTyLr5e/M31u0t1mImbMsQP91JP8O6NrM9BNT0KZbPttZtskIskZ8h51AUVI6nm0nbaWFrUm
+qihHdVWVVOxSBBeqtePVwrF+GjDlFGDaKnVUNngLxX78VeDwi+o4shVoPwK0HlQkpSgnBYOFmeuB
++ecbHU5d9muObgNeuMOYRB+V/CK44Jh/AVA91vm6g88Dh7YMzaR3OKRxuLaHjAucGPDUD4y28vLA
++28uoFa9FRi/wJ3wG+w2ygX3KdcCk5cZr4PO13KxvfGnQPM+lPwu6ejpwJIrVJ4PFQ5tBrbdbYz/
+B6T9FZ34qOwjhQSCAa8BjP65st5YBxjr57rJwMTFav4/1ThGT/O2mdV2WLX9I1wHGGuAln1qHdDV
+pKyoRgrGz1d9MNdS2cC+kX0k82uw7iUYGfAgYgxktel2U0pqGLbKSN38ZyVft71P/Wc9nPWeBGNl
+RQViiYQ6G+/1crU7oCXJEZ7NaNllZdi5YycOHz6EN118lsm+0TQ7K4miK5+Veolk6OjRo3DLZ/4N
+YX85xjXUw09VWWcUNbE2+HyV2NvUjIfvvRevbX8J9/z5z2g9cQIbH38EsVAIb7/pQ5g+ezoCldXG
+0wSg6WG0t7Zhz/Y9aJgwCbMWzkMi4U8Rqlm5QlIykh6t3VahO43v1PmUenIgOcXlx86dr5qvJ4wb
+i7LygJGOeL+yz/RPlJGDNAHh4n7cPGCpMeDNNRat4+b2N8XwNAAvUMeK6yhBNiYjW4AtfwJ2PwE0
+7TYqWqeolQTFByfRy9/YfwffjlfvB16515goCxlpYpSx+Dj7I8Dk5c7XPfIN4ATbcrek8WRqD9nA
+Prx5v1K5DGhiM1MRflNW5P7dYrdRpm3VDcZi24PfcC6kSXC3HVUqyVIGy3nBRcDss4YuDSS+dz8J
+tAgZWTT4g2rzmIQRFcx7N0ieCAawBtCUcp6WT3PPBRZdBkxaoj7LFSQxeSy+TKn1OA6/fA+w/T4l
+VqDSfSRYTNVNVfnoNFehaOPgZnUM1r0EwxuijCxEJqab3+qpptsZzHSvf/3l2L5jF17Yug1XX3Ul
+xo4dg1BPD4KBABLJe1r5e+REE777gx+jpqIMl55/FuLd3d7UXCVCjviNtF5ywTnm645ws9FhBzDO
+mPRWJMKoQQ8eeOIBfO0/PmkSeeOMfFh17pnYtXsv/nrn77B924s468LXYeqceaiorkVb41Fs37oF
+Gx57HHMWLMDHPvd5zFu2zJi7VJoqSbtSkhbUNOc2idmkT0o7aNLdETXmPMG+yus1x3jfRDSGDU89
+ab5ftWK57cczmGWn1xNBccEdde66Lb0KWHG9UlT4/AW6dwUw43R1kIzcfAew52lFSo5U0w3B0AzY
++V4nGF55JuVauPygKuaV+wZORs44DahqKM2yXHQpMGqix8mMMftZeDGwf5NS+0j9FwwZeZRUsHGT
+Ydnr1fztmZ8LGSkYwMKzTKke518IrHyT0d+vcDYHzvXeExaqg+Tklj8AOx5UpORAxxOBQOBtDjDg
+aNpe/EfavzsYoZALnjO6h/d9ny9fsgD/9amP4hOf/zIaG09gx569qKosRzQURldXD7o6O9DZ1Y3m
+lhY88vjjeNI4PnPze8wgNZGu7hxLrHRQG+8E7KRgIo6lMyfiDZdfDH8ggHPOWItz1q/BM89vwQ9/
+9mvs2LUHv731h9A1v/n3SDiEqopyVJSX44kH/omu9ja898Mfw4q1q1FVN9aYV1eZilLF+SUQDbej
+s7UZFTV1qKge009BGSJ/GAGqgyrojdfqV2Fcv3P7Hjyz4WmUl5XhvDPXeaz0yc9kUl08cNdzzlnA
+eR9XypBCTUAyYfaZwMy1wPZ/Ak/+yJg0b4TRQGXVJJCF+3DMh6EaNodDGk820ISZKreB5uWkZbmb
+hw9GGy2vVeMffUZ6BVVDz96uXI+IP2Rpg0MB+vEbM0u5q1h5vVKwWeazkteCXEA/kFTbrX8fsORy
+431N8X6LhORFn1YbQE/fBrzyD6DrhIhOCt1uJTtHVl1xIx1FGVk4nLJ0Ef795vfjxz//DQ4dPoSJ
+48aiKxRCT08Ie/bsRUdHOyqCAUwaOxpf+uS/4rKz1yFq/G24tMlQOGw81zFEotGMQk6ShuPHjsHy
+JYvw429/pU9VauDqSy8y8mcxHnz0SWzZ9jJ279mP7u4ejBnTgGWLF2DC+HH4zZ1/wYsvbsZ/fORD
+uOCyK7DmrHMwfe4CVNVUQE/oaG9pxysvvoAtz2zA4lNOxds/dBP8/qoUhaQVhZu+2yuNeU950mwb
+WnZSkqpIRt75w+9+h4MHDmDlsiU4dfkSqdClAJpl0yT73I8aE9s53hTE+YLKEu6O1k0BHvmW2iGl
+yYZMRgQCgaA0MW6+8iVMxXyuBByV91yElteU3nNxc2zs3Nw24Wh+OH0NcGSbCmYzUha4gqEH3ebQ
+l9+s9cDqtxj1d12fr0gpb0Gu4GYMRQJn3QjMPSf34GYDAX+DLgW4/hg1AXj+90DrIRWN+2RDIa1z
+xNJHMJD6V7Bo2sjwdy3D34ZFxui2NKeZa+sZ3qeZ8F5y/lmYMW0yHnjkSTNadl3dKMycNBYH9u5B
+T3cnvvnlz2Hx3FmoH1WLWCicW2CaISZDXn5lJ776vR+jpbXNmO/7+iUt4PPh+msuxzuuvyZjWmdO
+nYz3vO069ITD6OjoRDwWR3lFOUbX15nVZc2KZfi/n/4aT23chHv/dCfu+dMf0DBuHOrqRyMej6O1
+uQmtTSfMADqP3ncv5i1cgHOvuBw9IR/s2Ui+Kmb8fEcMCDEKt/E+6OsLfJPSBvi3IPDk05vxq1/+
+DBVlZXjXW67F6Lpa5cfTSyRtUUYWB4yOt/oGYxJyk1pcDTYYGOeyLxjpGKPMNujYWghJwWBPBHWI
+cmegE1lRRp68ZZsOutugemb/87kTcGPnATXjBu76o1htlBtjVJaRZM0Viy5RZutdLaVbwXRJy0kD
+Trzp5mDaqcCKa5WrgJqxkteCgYObQ/Qpe85HgOmrBv/3G2YAZ35IkZJP/QQ48drIDnApykjBQMvX
+A+mYDaKMHCCWLpyP5YsXwufTjMNnKgbnz5mNj3zqvzB3+hQ01NUh1NU17J7riY2b8Oe770VFIIjR
+1bW9gVv4fF3hENp6ulBeXqbISAdUlpebRzqoRvzm//wHHn78aTz8xAY8+/xmdHV3o6vpODQjHxtq
+qrBk9qmIxOJ47KmN+P5XvoQpM6Zj9pJliETSCMnkOWp8FuWcPg6QP9VsfyMRaTwK9u/dh69/+b9x
+cN9enHPmWlx31aVSiYcaNMs45Q3AGR8YGiLSAn1TXvxZINajAtyEOqRsBAKBoBQx9VSg6i+5k5F0
+/8EI6KUG+kijL8uBmI/ze+MXAo2vlX4gm6FeNMnCOH9QuUsC/IJ/V0EGBYJ8QIXt7DOAMz84NESk
+BRLqp71TvX7kO0DboZOv/yvUdaKMFAy0/jmMw4Gs7vLsr92ibVvvh4vfyIwBapJ/SFHBeVDM9aon
+YZov60YmdHR2YUJ9/cDTNYQIR5SSc8Hk6Xjj6jON1zoSRpoqgkFsfG07/vzcE8m0DjydtVWVuOri
+83HxeWehsakZ+w8dxrHjjWYgoCmTJmH61Ik4dLQR7/7XT+CFZzfifz/zaXzqS1/G9AWL4POVIZ5I
+3bjSbNlHYtKqin4/zbPjOHpgL77/31/CA3f/FbNnTsdnP3YTGupHZVHDupS1oDCgGmT22YqIrJ/q
+/XuMItrdCnQ1qjPfp9+3sk4tOqsbvEfhpkLzzJuA1sPArsdlYSeQBfNwmTyXujJSVK+FxeSlQMVo
+tdPoNVMZZGPGGtXPl1obpe/HukkDc0/C8W5RMpBN877hXa7hLiDUWhz/l53GfCEWkTaYN3lUqaxJ
+vBCRktcCx77LWKBNXKJIwJmn57AGMPqHcKfy8djV3H+uTvPr8mql4OVRXoVUe7ksoEJz2VVAywHg
+mduBnjaZe5XSvQSlX1eycYXp12aYRwVGbg7qWXIn16Mvl6NJP4vaAH3eqa8Nbeu10j69YRwuW346
+YvGYSUZWV1QgbEzmSEb2mjbnifKyAKZOGm8e6Rgzuh7/9YkP45O3fBWP3n+fMY+M4H3//u9YtupU
+Y8wYYw445CNTksJI3FARufki2tOMV7e9il98/9v46x2/x/Spk/Hpj96Is9euTj5DrmUvKBgmLADO
+udmY1M71tgjjBKTjOHBsu4rSuOsx4ODzagGTUqlqlRN17rLOOVOZ9FF16Q+6/wa/d/ZNylT70Iv9
+iU6BYCBggAkGSXIiQo68xA5L8kogcEPDTNWnH94KxCPevsMAafTJWFZdWs9C64B55yoTwYGC33/m
+JAhkc9gYc1/6myIYCt4HHzDmD8ek7QgEpYJRRh++6i3A/PO9+YikAqWnFWjcpTZfKBrY90z/dk3f
+pWNnA9NWA7PXqjNNsdnXuqF+GrD6rUD7UWDr3SJKyASujw684EzWntitrhMIPELMtEsBSTJGLwFf
+dfG4khzG4nG0tLchmiQjw+GwGdym9xqmucjpveJ15yGeSODL3/oBntvwFPZ96EN4/VvfirMvvhRT
+pk9BeXU9AmWV8HGHjckxro1GQ4h0t+L40SY89dADuOu3v8aul7dhwdzZ+NiN78Y7rn+91LdSWICd
+/i5g6gpvDvsj3WrhueUvwOY/GpOPo9l9uoQ7FFnJY+MvgVOuAU69Xvk4IlHpRnzSd83hbUDbYTUh
+EQjyxfN3qEMgEOQPqgGnrVQEf+dxb9+ZuEgp5QcjOFouoMqMabMH/8gVJGZnlHAgG6/TRC5gt9wF
+tOyTOl7KKKQpp2CEMg/lwMILgSWXeQsoFgur/uHle4EX7lR9XbZAM9zU5d95vHCHCoyz5m1KnEC/
+vJrP+bcoYGBk+OO7gEObpazSsedpdQgEuY4Zjmba2S7MJP7T0s76cB18bCa3/QKYIHsgm5RgJv3N
+tAeSCeY3fD6Mqq5CU0urGfDF7/cNWc5YZGTCSEdXZxfC8ahJkuqRGCJhpUKIxqK2vCgurr74fEwY
+OxbfvfUXeHrT8/jR//sq7vvLn7F8zWlmtO2J02egqqbG9N3Z09WDxsMHsH3rZrz47LPYtmUzxjaM
+xrlnrMWH3vN2XHrBWX2Erz2IUdaANRnqhSB/zFwPLLgAqPTgyoDk4qsPAf/8CnD05dyUH/T9SEKS
+i9ZzP2JMfC41FqRj3CcjJC93PGwsdJtUyHaBQFC6k5xS75tFXJ87uKDkRlW2YDOm/8d6pZb3Ai4w
+nXwyuv0e3CfTOYPjEDe/aie4/KbuTqIuvsRYqJd4IJtcFy6C4V02Uo6CbBg3H1j4OqVY9DIWUIn3
+2PeAbfd6V8MTEeO72+9XxOSaG4DTjKNhutrQcsKsdUbffD5wfIcSQ0gbFwjc64pXM+0M70UZOdRl
+qZshYrB03mz86Be/w/Zdu7FkwdwhS48V+dtvBuVJ/ZsvSeIw6jV9SfoGSWSwdtVyzJv9Ofzx7vtw
+930PYteefXjwrj/j73f8FsGKClRUVZnqyJ6uLkRDPQgEAmgYXY91q1bg0ovOxfVXX4bpUyZJZSsF
+cAF5+juU034vk5Bt/wDu+wpwYtfAI9wdexW4978VsbnqTe4mcVSacMf26PaTz5G1QCCTYpm4lzqO
+71QL1cosBCJdaphuDzz4jaQJ4ORl2e/l5feKAUb2nnum8muWfYKolII1453NDBnIZkKJBrKRNiIQ
+CCzQVcai1ymlohtIPO7bBDz0TeDVB7OrId1AS6enbgVCbSpYzvi5cPQjybFlztnArieUKfjJMJfS
+C3CNQJDPHMBBtxfop3TM5hLRKZjNcJtw9AtW46KSy6aGTFdGDlApGA+FcfrKZViycB6+9r1b8fl/
++xCmTJ6IsmBw0LOGRCPh0zTlP9L2SAGfIiOj0RjisRh8Af+gpWtM/Si8/4brcNkFZ+O5F7fh6Wdf
+wPadu9DW3oGunh6THK2aMBZ1tTWYPWs61q5aiTVGns6dOT1ZNOmKSLcyR4aylz4lb9BkYspyd7M0
+5veeDcAD31A+YgZKRFpgYJqHvg3UT1EO/91+f9mVyiycptpOakxG4QtWZVeucDJlKizTdnOpwGGQ
+ndpxykyFZivWPfjskS7lk4V+V6jwzPf5HSeHVUoxSr9qlaP67xrHY4oYZnq6WxSpm69vMv4Wj2xK
+JN7fDDoQ7v83lh3z3bxHbX9/oEwr02nl3VD7UXN7Vnua3ZS4IyXf9CJfP1LSmA3cWDTbe53qfzLV
+BztYn3iwXvDoaVH9QrHyjUoWq65n6lu5YUTfkQdfdPe1WmvU+Qnzs/uLZD0//JKxAK13JyMLuWBb
+eJHxHFOdVY8M1PbET4A1b1EEbDZYgWz2DfNANqW2IGbZsN5wE9VsJxyrKxzG+6hqJ2YwnnY1ZhbT
+FzD7dNZbthO6oQmWp1p+WOmJhhRJHepUafKqLqNamG3CIsLZT/B33MDvjM6geqPKLJff7/e8Faos
+eH/OmXjws/S+ir/DICfmfKWrMP6/zcCIDtY8zF+zvEO2OZ7xnVGT1NkasxlIyRxrW9VYzX6X93Xq
+BzgnpL9E1qmBguVmT0fGvjCmnoF5Vyxw02TWeg/BxIxnPrYDePJWRUTmOx+havy5O9Tmz9k3Om8C
+EfQ7T4Xkgc3O9ZVtg/mazeWU5euS42am9sV84JGpb2FdIYHKNUQ+7aaQ47Pb81rrHqbXTVV6MuQd
++yCmgW2Y/WN6f8R6a86d2lV/1NM+OOU4FHNgHd4VkaKMBHILYAPkFmI8d8SiUYyuq8Unbno3Pv5f
+X8X/fu9WvPGKSzBx/JicguHwWp/PB7/fj4DfZ5KZVVWVKC8rN14HEAy6F3c82eH7jQmNT0tVHfgt
+MjIWMwPbBAODb05uBby58qJz0RMOo/FEM5pbWxGLxTG6vg4Txo5BTXWVabaduWycQjt5KX9BXpix
+2n0SQLQfAR77AdC4s3BEHHdHN/xSmYjQT5dT2+Jid+Zpyn+kkx+uU16f9H2ZZSHPyHwbfqHOZiP1
+q8ipdLA925iQzTsHGD9PkUQWCcgJYdNetUDe/aSaDNFchANZIUlJEpBUqE5cDMxZr85c8JanLdpJ
+PpLMpd9OOg5nnjB9rYcGPqjy2WkGk23xz8kDy79pT+pkmiY2k4x0zj1bpZUkA80v7WXJct7/nJF3
+T6v0Mu84EdWHiFxze1aCeUq/km5+SkdSvg1Hkq/U08gJfN1EoH6qWmxRMcggYpnqgx1cNHeeUCrz
+o68A+54FTrxm9GsHMxPfeffVR1SfybQGKjJfw/75tSdUP+QEmmhTDZ/t2RiVtdnoz6KrB68cSCIx
+8EyNi0r/1UeAl/5ujJurVDlx4ZMN889VgWxYJqXWZoebn0GOxdb4yP6SPko5TvMYNSF7wA0uONuP
+qfp05GVlXkr3MiyTfIiklIm+T/X/3FjlPIb+sJnG8UYbHjU+dWFvpacjeVABvPcZZfVBFwduKlo+
+67Ir1BzH7D+Cqj25Yc5ZmYm7g5tVYBBrPuSpLPyKbB01UQU+5BySaWDbYfrS3Rx0Jtszff4dfEE9
+a9Nuo085mh8xPPcstUmdDewb+WwcN1kGrCuLL1b5x37W2gDnPI7Pz7wgycbxefqpzm2b6zL6S6Sl
+0EBcB7HOzFoLLL3MWWHNvpTPcOD54rWtsXOMvmyO+3Uka7f8Gdj+QOE2Rpn3L92jymPp5c7ELAmm
+qaeouRMFEdnAdscyzmbtRUKOecqytvf/XAeYxOxaNRZn6luYXgbTpI/Gvc+qvoTtOdd5t17A69ye
+11r38JkPbs7vXqWQd9nAdss+mHOnGacb/bDRR05c2L8/Yp/Dvpb9EOfVTBMtGNjWwh0jj3bLUs8C
+OXNvJ4MyMu25MgribOeM16ULJ/PJAmOSHO0KYf2py/CtL3wSP7z9Dnz31l+iprpS+Y/UNQ+30E1T
+ZX8ggLKKclSUlaHW+P7E8RNQX1eLMWNGY/LECaitqkTdqFrUj0rd3ezo7EJbRwda2zp6iUeaZduf
+yZ/suKORKCLGwd8YKpB4raqowIypk82jX/Hqmd9nOrsd4vOrQCDJxYmzF1O4zX9RJFy0wAvdVx8G
+lj+rJhhujrOpNHnpH87RPWlqwsE024KZA8+WvxqT4wNqEshFw2lvNb5zVdKZtpZ5IcSFJw+SnXTc
+TULzlfvVRDcayr8caJJIH5qr36J+xx9wHnS5COCx8lq1S8mJ8abfqTLiYj7XySKJsZVvzO4vjQun
+5+80nn2PmjCSHF5wHrD2XWqC6DRx57XLJquFA/Pu2d8qlSsnAEOxI+n2rAT9GjFP245Kvg1kMBVl
+pDu4uOJCnvWHZnI0E/a62clreZD8YJ/EPmDnI6pf4kS/0BslVt/JiX5tlr6VC0oSqLoLGTnFxUSb
+v9PdNrjlONEYByYvdSYGqDzd8YjaJNj+kLHoWu28AGT7nb7GeJ4SC2STq5/BoW4nnBewn+W4zr6Q
+G4c+j1ZAlpqXi2MqX01/d8YYuek3xvj9gNpsyscPNS0XSOZwU2nlG1TddlIo2dNjgfWJaeF4w7rP
+NGWbU9B6Y+45Ki9yAa1fpmQgLdledz2p5kNe+yzOO2avU8QeSVG3eRtJSh5sL6vfrPom+h3f/CdF
+DpOQGMgGCglQBkLJBkZ3Zl94Yq+69vwPq3xLLx9LzcX8WfEGRVRzTubmuoiEIutS8/4B1Js65ad9
+3bud6zJ9Mr54d/HaIFVjJKAaprtfS9Kc9bSnvbBp4CY/fcJzM4sEtxNY33gcdyAjubFHt05TspD0
+3CDm97c/qMqQmxyzTlNzb0YSZ5t2qv8ULPDg3I5EKufdB7cUT73qVvZuz2uNqayrBzbnd69SzDvW
+YZKh3HBZ/SZVj5ws7fg3rrd4sA/jmMC6zbSwz+DGazE2dIdibM+m0dPdvzuClZFwYVt1D3/PTxGZ
+miId0a4erF2xFIvnzMBz217FoSPHzYA28LJe0HXEjcVAOBJFyDi6e0JobWnFM5uew7ETzfAFAhjX
+0IBxYxswb85MnLpsMSaOH4sxo+tx5FgjHt/4HHa9thubnt+MQMCP7kjEjGRN0s8ycbaUkZFoBJEI
+F6dVw7DMc1VGChNZMHDA4eDvNrHnbj7JGa/BCXIBFwLcaaPCzD5Bz7hgXKRUnE6R471WCw6i841B
+8fyPqUW0P4eul4uhS/9TDaoPfwd47SljQBugU20O0AuNSelZH1RqDydyKusCpxY49TpjMD4DeObX
+yvSFO8e5LLK8+rDhRH6KsWhf9y+KBHE17cmUd59VE+BHvqsmokNBrBXKX89IyrfhQvINFyJy1fWq
+/xk9Nf/7sQ8gUcNo0E//TPUBXhV5Xuv6oa2K0MlGvJsRspMBybIRoVRKTFqqCJlsOLhVLXoGiyhj
+etkHuwWuIenAdkcz052PquALVGI4BWBbkgxk030SBLIZKiJy/nnAeR9WBFje5Eulug/nGgy69PTP
+B9iXaopg4wbiuncpFY5/gGIAktZr3wEsN9rvK/8EnvqpMu8vpQUx54icKy6/Glj/bqU4yqfv4wYM
+/bOSbNv0W2D/C8oVTqHBtsl+6bx/zUxEZiI2SNywrZKYdHKVQcsVHgMhI7mpM2mR89ybKlmmhRve
+xcLo6cpfo1u+UBFHFRnVxYUGN85Z30lMsz25tRWSxMy3fNWZFBmMmWnMna8F1rxVzbFyAccLzueo
+COQ6gIRqMc3pSwmlknccH7ihy819bug6kaFOYwLXclwPsz/a+Ivi9UelSsFkVEZm4mVy4emS4+Sw
+8xmZjb0d6FGABYdu9sE9qCkrw3nr1hgdtj/3OySUnE+PxREOh9EViqCrpxtHTzRjx569eGn7Ljz4
+6JP4270PYuWyJVixdCH+dt/DOHrsEJbNnob1y+egKqhh1/7D2HJwDxZNmoZo0jeUpYwMhyPmvYef
+n9AClLtg4OBkvMpDBO3dT6nd5XisOOmgU2pO6NwUD5zEcuHur8jPvIc+p0jeXfAxtZgcCEhe0jyX
+i2oG9Nn5WO5R/qjE5KT8nJvUwK7lGYGKEzWSGzzTNPjwy4UlrEgkcOC//PPAzDUDX3wRq65ThMX9
+/09NthPx0u6nRnq+iTKycLA2DwpFRNpBhcsln1XK8KduU7v8eoEe1HSR0ZI9mjT757Fz1UQ+28KC
+PnnHzsquXLD8ReaiNs/38aob1GaY2wYBN+So4uLvNe5WRAEVexUOPvvsgWyiJRbIJte5+WCDdYSK
+w0IRkXaQSFz7TlXm7EtZ57z2pZyjUEFEFQ7vMWZGYdJkknTXqvnJA99U5MxgbTjpLuVA8nbt21W/
+5ebKIBdCgoQ+5z6P/0jNoWgKXEjQPyQ3aWgO7WXcZfvmRgN99dM/oZNikBYsJDqpEstl45d9JzcV
+3TbfrT4mVESCa8wsdbiBc40DW1Q07GLgyHalkOOc2mlDnuRT3RSl6M1mIaV7LAOW37n/atTpN7qr
+e51g9k262jygGw8vbTYXM229xO411HlngcQjN6rob5T38eUZM8PaIK431k+PfF/1R8ONXM5VGelg
+TT2Co2l7ZRfdDg2FVEgSsbix+OvsUvceAF+gJYPPlPn9xry1GmPqazF90kScttyYyF5l9PFHjuGZ
+ba/gvoefxFe+8yh279uP7/znTbjhsnOMdWcCOw8ewye//hP8+pmH8cmLr0V1WYX5bH7e13gVNaNp
+J1KetycUxs7dexGhktNajDANgQDmzpqGqsrUxUBjUwv2Hz6SNIO2rjfG8poazJ05zfR92VtSxt/3
+HjiEptY2GwGu0jJ5wjhMMg7vrUZ3qAeAMJFFBP1eeVFo7XsuqewoEmhiyMUA/fS4pYc70RUPD5yM
+9AWVCobqtLrJBcjD1cDFn1ILTfr286pmoKn16cbk/pwb3U1TcgFJ0tPepibeD37DmOS94s1ZvGtz
+Mlr35CXKFC1fQs0CiVj6zHryJ8qvVKmRVgXxq3YS5VspE3zDJY1c2FP9wYl8oYlIex9w1vtV4JTN
+f87sbH4g9ZzmpPQ9RfcCwYrs/SF9imWbxJs+ih36eNZn+lelf7vB8mtIhRYJUid1PMebnY+ntje6
+6aDLBScykvdcUmKBbIZDG2U/ybpy1gcKT0T2zgWMxeuii5SvsIe+o+q3F1CZdfYH1Tibq8LdS5oW
+v07NIx78llIj28dvfZDrA5WBnCee/1Fj3nR+Ycav9OelKokEJ/Ny810qQEYh6jFJXapNOa90C5Bo
+gfPQQy8pcpFEoBMZSTKEJus0Ec1FHWm62VmoSLWsz6Yrn+BMTzFBAYAXVRuJUbqKKRbYv9L8l23R
+LT0cN7kZ0DVA1xck06gIPfN9+ZNpFmaerkyVWQ/oi/BkRankHdvzwguBCz6uVOmF7o+sfu7Vh4sb
+8KwU5gFZlZFu0bS9KAGHnb9IexTttEjJvc+o53D0LgOVWXOhGuGAH083D9NwiTtuUVv6jGPahHGY
+NmMqLj7jdNzz+NP48e134q8PPIUVc2dg0YwpmD9zCj753utw9Qe/gG2H9+PMeUuU5Z+m/EiSiCRp
+aVdAbH1lB27+zBfR2taORNJkymf8WsPoenzjlk9j/apTUtJ459/uxXdv+5UZldu4WzJtGpYsnI+f
+fet/UnxadnZ14b+/+SM8vvFZ4ycTySqrmf4033LNlfj8x2/0WObo7+BzAOUsGGCny8mQ2ySNaoHj
+rxkL2q7i5vfRV9Wi2W1ybypRRqkd7IGAk14uJDghzFeJaF9gc3BmlDiasbgpLLhA5SKIqopCEpF2
+UEHHyRojlpNAyFcdxXpy4b8pZVOhFiTMB5p4cMDvbC5uhPKhWriPxHwbDntFQ5VGuoQ47yNKDVRM
+mLv8bwL2PGMs7rcXLs8YLZuBI7KRkXQ1Qb922XzQ8e9OfbzlL7KifnDaKNsk1Tg1LhuoVIVysWz3
+mfza00DTPlWWPgcik4FsNiQD2SRKPPhUOhFForWirjD3I8HDTTsv/RVJIJouM++KCS6mWf67Nyi/
+2G6grz+a71IhWGgi0l4nqeTr6VAKSQamGopxzlJArX+Pmq/kqzxynNctMeZlN9EBvvLpXQgTScvP
+t1fTTRLAZjvfySimqq/jxrXTHHlS0lS7KQcycnwyYKNTfnIzh75mT+wv3jhl+fxzq8dsr1TYtx0r
+7php/sYRdzKyPklGsnwG2q+teL0q10KQab0k1tmqHzm2qzDR4gs1By3kvUoh73g9N0RPf0dhiUg7
+uAFGdyxc09HvbCmuTbyUrygjC7laGIhSMtluzEjWPsTjCWgl/ISmv0fjqC4L4vorLsaCObNx4ye/
+gM9//1e47ZaPolbTsWDGFNSMqsSxthZTEZnQdTO6ti/pQzJdGfn0phew5aVXUF1egUn1DeafjrW3
+mIrGzcbn61f1OailL8oHHn0ar+3Zh/GjRmN0VTUi8RgONp/A/kNHcLSxCfWj+jqeY8b7+x5+DCea
+WjCtYSwqgmXoDPVgX2sT7rr3AXz+4x/ModXoOZb7cGXcSwzcNfZ56G64S01FR7F3h7jr6mUCykmI
+113uTKjNsOgkUUcfZSRDGU2NgyEDGZD0pEIn4IFEYiTAYzuA9uPKObMT6G/p7A+4Ow1nuuins6tJ
+pY9m8kwL00bzcprYO/krY2Aemr688Kf8zQ3MRWmaAogmFQzw0tGo6odpvmmkp2aMSh8nK26E77jZ
+KnAETcpPxkh2km+C3kl0QCl1GqY5ly/rBzdbWEe4QO61VPCpwApm1Nrx7uT2rNNVfeLiulAk2KGk
+P8dRWfwrmgFuxmX26cX0knRwUhIe8ugvslBgO2IQN6c0Mf9pipnuM5nWAiQk6Wuq1oHMpAJ/RgkF
+svE6dWJgFloQFMoqgqoXBgpwU+qy7jC/GIjAibDhOM0Nt9Yjqg+0WQCZcwSa39MVCtuMIzk0V5FK
+W+9xNhNk/SU5tera7PU/hVDqUuXNczg5t+HYzXTxcPJHyPQvvRQ4YtSZJ27rS1ehN1Gc7keCfs1b
+VJAHL0Qk859+DhnghP0Wv8N5JvPf6Vkt0IKABDT7PgaSyCewEJGr/zgqs8z+JxmghXMnzummnuJM
+eLJP48agJ6sYLRl8cL57W2FaimmmT1ce2TaV7KD7ISoWi2kdRbDcvYgMrPYz0P7NjP5c2b8vIfHE
+4FFsqxy72I9UJCM0k7B1awPsE+gXnGMBSdVC9MGFtA4oxL1KIe94T7rImHuGe3/EuQTHbZ65EcaN
+/KDRB49KBgB06pd4f6o/2S+4pWm4UmwOY0DAE+eWibeB8DT92k3Qb0a0DkdjwyK9USOdWrQDK5Ys
+wFc+81G85cZ/x0PPbMG1V12AHa9uQbgnirE1oxBP6GZb95kRthUZmV7k3aGQ+fnSqbNw84VXm5/9
++OG/47FXX0Q4nDrIk6wNJ3f8r1q5Dq9btgrH21vxxb/cjmPGORROHRAj0Sjixr2ryspw4wVXYda4
+idi0Zwe+ds8dJkka16naLKFJtSADSVLnbYLYelj5iSl2Xjcf9OaPhi4KOLAVKj2cQB4zFus7HlWq
+xmOvqskXiUKShtwho5kSJz9OxJ9lHk2/J50O0aw5mFNp4BY0x0rX838wFrxPql1gDvQc2E3H0cYg
+ufRitWDMRpZyMnDGu9V3ObHWC1iIHOAZme/5PynfKjRxI1nKOjVnnYqaSfNETi6c8o1g/r58v/si
+dSgG60LX++Gab7n6JSpln5FDlUaaAtMk2EmJwsUnFQLP3anIERLW1q4868h4o73PWa+UWVRklzlF
+jaxQi+U9z6ogZIVoC25+I/mbXJybrj3SzC3rp6mFRDYSlX0mTSS72wvrB8sJNIl1U6fzOUg6kvhK
+/y2SlCuudiYjCQay2XafcY9hFMiG/QuPQmHjr5W6tMelv6oZr4h0J/UwF8AkbF74M7D5r6n+c02f
+jpMVmXnKlcqUngowJ5KRm5wkLqlezQaaZy+73DlqrVWPuSG54zE1dtPEtTFp4kr/kny2RReqyNtO
+Pru5QKdrD/bxVvRgzgE4T7Gi0XMOwbmKm+9rLqibMrgJ4L3CGaxeSCIy4AvJaDdSj30BN0zpV5wq
+TvYRHOd4Dz4vlYAk30ZPcQ/Qx75tzZtVWhsLrAhlubANm/OzqBpfy5KbO9wAZF5wc8/KCwbS4ljN
+Tb9sZArzhiQq64ZThGf7nIzEt1OkbrMffFHlYzG7Cs5FvWzsM88y9X2FBgktHq4kaplqs4VKD8ln
+lj2jO7OPZlvlJjHLnGsAbj4sv0qNvW4CDsvsvbWA5FUpKSOHOu9Y9lRpczx1qrvcyGg5ZPSd96p0
+cQ1EopvtnG2PLidWXqMI0GwKT95/8SWqD3j298PDqkGUkQXIwXQTXGR4DYdrMpjvBgMBUxnZzeAu
+2rDJCcTa2nHOutVYvGA+HnjqBaw8YxV++Pt7UKdVY8nUmYgYDa3MWJhoSWUklY2JRDyFbAiFFLk4
+cdRoLJs6y7xuwii1COro7Ey5Nh6PoyuknKuTWFwyZZZxbRsqy9WuWWfa9fRH2W0ctWUVWDR5OuaO
+n4KOkCKSorGY8dshVFeUu09gUs62z52OlHohGDA4AfOi+KMyMhYtfnpIqHjZCada0RcszIDL3+TC
+n/6ieLarPzl4vXi3WtTQryPJP074nXb4eC3JBk5oOxszXzPtFLXj5jTBZ1vm4veBb6izPV0kAnjQ
+lxAd3F/4MTUZzkYsk7CgKQMnyoVyxsxB/elfAI/9WO2Gpk8CuAgzI+Iadef0G5x3sc0F2kylGDnZ
+MdzzTfxB5gdGe6S/KydVJIO/3PWfxnlz/0jYrCN09M/jgNE/Xfd1YPpKZ9KafRYJy3zJSAutHvxG
+zjoN2Hp3fzJy6rI+EiXjYvQEcGLP4PloIjFAM2C3gBymv8f9mQO4caFC0oRkixPRQmKMizO6PImF
+Rm478fK7U5ereu001pL4eui7wKY7+6vHOH7STQAPtqOIkd9r3uS8+WqprbKRkaZac5VyM+AEtlEq
+kZ/+pdpQSFd6se0wYB+jtp7/EUVkOyk3p69ShCTbBZ+Lde3uW2xkWD1w2WeAc29yTteGXwH/+JL3
+cuIinWkbN9v9eZlnz//ReObbM5uUm35yLwbOfK/KQycVsmmyeRawZ6MqPydlYC51mPfh5i59UnIT
+kP0hCS2SpTQRZX3jpk2jzS8i1YCcBy5tUkR1NkxcoNSOXshIL1G02Q+yf289XOT+b5Q63MDN+XB3
+8fsGtlMvQb4453Gb93itG1TMcdOPAUvYl6f3IySESYzTDP+CjygVtRNYT5zqSi5p1AvYr+oF7qOH
+Ku8aZqg1lhOZz81bikoe+xHwwl9SFb1cAx19RW1e7X/e6IP/VW0MZatP7P/o0/KVBwfuGqzU58o5
+RdPOpoi0vy983JZBzpWBOMh0lo5WlJch6Pejk8FnfL5hkxs0udYCAcyZMRUbnnsOn7nle3jm6Zfx
+jtMuwOiqGkRIDvl1pYw0jngsYRKK9mdv71BmIQFj4dPc1mrmgz+5CDrRkro7HzPu19beaRKWiVjM
++HszuiNhlCUHzOaW1pTru7q6EA6HMK62Gt3dPWhqa0EsHDG/z8jezO/qirIcWoLXupBW7sJHDhxl
+Nd7MtOkrkouwYuc1d9i8ROvmxFZzUEZ6VtQYF77ygDFB/4pSWGTb9WoxJoR//x9FXNLX5CiXwXL5
+FWqA5oQynTCnisFL4BwO4Pd9TakqsuUJ+4Dn/qhUVhf9mzL9zEqCXKiILqeojF7zjUQBF1qP/sh5
+cKYC6KHvKRXJvDOd6xrzg+U6WO3Zq7lKISMZDvd8y3UyW+rRtIcijWzLjEhKkx/uxpO8qqhRr7kw
+Zh/zsDGxP+TB7+zeTWozgotbp8W96au03PlZc1UhuvmNpJKIfiN1LfXm3BRxUoJZ/iIHS+FK1TsJ
+fSeFOsth+0PK/YaepR/e8bgijJzUafwNmruy3JqGOJDNUM+b3H6fLoca96gz24bpn6xK+XkOlivC
+gn4FOc7GXMxYSd5xo5FKmHqHcbcyScxkS1v1WGDWWmNxOsf590ga3v8tY2z+gzORRpN9bkwxTfPP
+yb5BwfpJInv7w8ptw2CUuxm0ZrWzebLVNqiKeuSHRnnc1Wfe3I9k6lGkAMlFLv5JTDr1WSQa2L+w
+XdFMOu8FVVxtEN//TSOdf0stF7bHF+9RpAPrld06h2uqg0mF4gKHeR/rBNWOLz/gbKrtNYo2zbN5
+FLudcvxxU6qac5ckSVjs9JD0jHR7q5888l0DsB6QZOKmhpP/Sc6b6U+W5XzOB51N2zkfN8e+ArXF
+UoymPVR5Rz6CqnK68HACrUm4+bLxd9k3NuNJwQdJyNqkEj9bH8zNMc7H2+4fPgQjstBk2T5DNjJy
+pCErr2hTz1mKRz15kZNiLvk1Bl2pqqpE04kmo/PyD688SehG2iuw5ZXXzONday/AGbMXoi3c17j8
+vcrIuGlqbSc/WlqV36UyLYD2jg6UB4LGa9XYmppboSdNvc0xIBJBS0sbgkZj99FXcbsKelMRUDvJ
+jc0tKffu7Og0xvc4KsvKTBKyvb3DJDHLjAl3T08PWo3fnjCm3p0MSjkjraxdFLDCROYHTr69mGnT
+j2NiENwccIfdi5NgTmL9BegmqUx78mdqB81t4c+JGAc3Khqrz3bONzom5wKCk9/0ialpdrXaWR1E
+wuLZ36pJvhdylooELrRIOmQb6OecoSb4XATna2pAMiWT4iMTuLNP589URDmZybFMfSf50Cf5JuBO
+PY9xSXMk9gdUC/DghPjYdqWM9KoMtNTOTgt7Kqe89PO54KCL30j2f3UTVT9tqd1JKFF95NT3HRxE
+f5EmOfg6d99/bK9URvY4KEt3PAKc9ha1OeCkel1w7vAMZDPYoFkyD7N9zFOEHRVsJN4ZuZgbfXSH
+0uXRhx3Heh5OZKSb2opqtilLnRVtHO+3/dNI+z+9+fqjEo/PwTmBZSrIuRYVYpFOtZDn/IvzXfYP
++ZKRXsF84gLcTTHMtvHET4FnfuOtPtN1wyP/p/orbpA65eXs05XbEvZx+QaPoAk5zSyz+QRlf0vC
+MRO4Uc1+iXO/bO4lWHasHw0upto0CZ+YNFd3Ink4d6S6ttjwojA08yecGriraGveuLd65NW83A0n
+9ily2ksgHPY1nMORUJ/gQCbTCqGsEic9hiLvOHega4sGFxcHOx9XRKmXedTujaq/ZvvNNjeh6pm/
+S1desQhGCmRlUUDUVlWhtroKhxub6EAyu5+jEgUJw972MKreTDv9QFp6A0awZjRtRUamduKNJ5Tv
+jVGVleYj06S7skyRFd09PSbZ6E+ad0UiUbS2tyEYCJoBb+iBMmBMFKqCateMqkk7OruVD8HKYBnK
+jXylAKLMfF1m3Dts3KtdKl+pwysZaaoiTzLil8/DgZQRvOMeiVYSRNxBp6mdmwJm8mJjQHyiv6k2
+oy6avic15wk+VRBefOdYAz0XblzoZzNdIFFJU26aUuS72Ge+5eIPh6ZRXFS5+ezSfCd3e5N8E/QO
+zrtTzQEJi1jOxf9n2MNGUbCi8FFw3fxGmn3gMjXRJ3FkTmDmKYfx2dJi+YvsGaS5A/vK6ae6mynS
+pJbt1okQ4TjCPOGCxsnk1vRjuEZd29U88up9rnvImQJamEREhTdzzlzIFH+Zc0AoEkijXfwy0hyR
+hJtV513nVhHlboXKTc4pzOitJ9Rcg4pOEltNe9UiPpubhYLvy2vKFN0tSi3TTnPnXU/mRqyT2KOC
+kIt7J3KY7dNUWNflHziF7Y0KyIG4f+D8ioQLfZo7maxP8GCqTZ/BvMapnvF3+HtdLcVvj16JM7oL
+0U/CzRNuMvHwCivatxOhxvHN58dJj6HIO7q44aaU0zVc33Aj36uimpu53BTiBgA3P7KtlbkRRt/S
+dNdyMo3HyD5+BFJMrp0C1mQzzday3FxDCYvJvEZNzsWMm/PkBOpra3DshDHx82nDu97oMAlEe17w
+iUhGRmMkI2MplaC9S5lpm+SiKRbVe5WOPeGwSV76k3kSjkSNuVoUNWUVJsHI3wn4fMZrRUYq/5M2
+f5RRpXYgEVkWCJhm5VRVVhjvm8PdCDE6uGc9+ECiaVtqSVnXDhgkOrwQcZaPxmLnNRewXjYKmOZE
+nmVv+jr5h1ro5HIfEkqnvkGpM5xIIKojSS50pJGRU5Y6K4NMEmqHmogmclADHH5FBQVwiIthKky4
+g9+dBxnJXUGal/XkEGCCTqTdFgHVowvrkHyoFs8nc75JAJvioscDCck+h/0KzZlqGhRhkM35eqHr
+sT3P6Lqi2cVvJH3D0Y1ER5KYoZlTpYO1hOUv0jKTLHYAmwVJs12nMYftlhtDZhAhl/txQ4i+7tx8
+7C29BHgpGcjGaZOPCyAqA/NVALHP4Tg3WH44C9lOM4H1wy3QHcuUii+2E27+kXT2Ev06W9roW5u+
+yjjuu43dPHJ5vj3Ggr77q8qNFDcoWFbFUs16SRfbMzdT3eoxlVGsx/Rfm2v5UQ06e60xl7rGeR7F
+OQuPPc8M/JnNTY5tSuE40HpHApUEIcnEbP0FA3RMdjDV5vdoys0NC6dFnhlscOvgjE0Rj4pHS4lY
+7DSRZNL83srUjJGQx29xE4+WQs2HvN/HCuTjhPKk25VSDTpTiHsNVd5xg8JJFUmQ8CShH8+hD6Wl
+glPgMoJ9/6hJasOppIkiB5rFyUw7wzxq5JppZ8uwfFxJGmgYXYcjxxuRYKRqY8DXh6nKy5/m85KP
+QX+R/LwnHkUsFk9xpWiabRuw1I0MckNTbSok6U+S15NIJLp7QkopadyLhKWVhRVlagevsys12l5n
+l/LrQbNsKihjxsAQSH433pNQ0bpz5SIHUu6CgSPs0fzajF49COorr9G9Q23OgW681IvGXWrQiudo
+fk7FAne+TRLAQQEzaWFmxY3p382FPKDT9EiOC0cSA26+drijWFadXxkxQiaDUuSyWKKqYQSZNki+
+CQoG01detSIgLRJyyhJgxqlqMc8IwP4hmjJy0TzPwW8kA3WRMD2e3AWny4GqOuf7dQ+SiTbzdNH5
+7hGwSY6SGAh3ud+TCjGqJtyUG1YgG2429LRmv45jzFnvcfcv5wb63XviJ0ppXYzF7VAurHsnx2Vp
+7aRO+SgjIc5gSlTjeCF1s6WtdoJaCPvL3BfCuUbRJQnvVUmZT356vY7kLVWabvlFf7V0KTEQkLCl
+lcbSi503U0g8MN+zkZFenomKUhIn+SgNSRLTzHTRBdldYriZanPuxb+PmZ79d2iST1+R6ar5YiHm
+MWBMoMy97hdkvPNoNu62IeGlXnCThhtl8RzmWPSf72Us0Ae53Q72vYYq7+gGxc3fvhfSs18f3KRc
+OTihdqw6TlbeLVOzz5mQ0b3duLTzJPWf9WBu/5D1nW0MCAaxc99BdHd0oILEWXx4ys19aTtyfEp+
+FvAbzxQOIRaL9T55KBI1/UCaHE+SgCTZWFVWbhKOza1tiEQjqEwGmWkz8iaRUKbZZSQjdd08qsoV
+kdnS1p6Sq61tymSESkiaekf0mKnQpKqSpuUdXV1ppZC5zFPPmetCetmmlLIQkgOHVzKyqmFw1Fec
+hGgeFtaM6pfIM6BO2zHll2kg96DfJk7gnMjI6gx5RgUAzQyCLpMtTlqp5OCi1nPnEHR2DE1MmJf/
+jm1PkgguVl0oxfZciDSNtHwbDptFpZhG9ilUxFE9zYN9AEkV+hnjBgd9Sw5UKacXOM8OuPiNpOqw
+3liYB7Yocs40YXYgHkj62YPX6EV8NgbnYARct+AN2x8B2hu93Z/X7X5GmZ/WOLlWCCgShmpLBhXL
+Bm5mkYik7758QCVHYICqJqr0+P1CRf+mf+ZCBMLgWGcFnCHxyAUq2wbHuEkL1KZf1ejCtRO6ynBy
+l9G7EG5Vh17CfZ4brL7HDay/HY0De1auwRgQiiSAU59gRdfNJz9JmvDIV43LqOxUgc5yCJ5hmWof
+y0BGUlU50cVEmyQkVZyRQVIxe1VGclziUfSAOmXqcEMsme580hNOkmPDYU5Wavcairwj/8FxlWsr
+xzpUroJE5QKqcStdYlxQGcljOMxrM70XZWS+s958WNnk3M/vw8FDh3G0sQlzpk1BrGf4kZGWOXa6
+qJOfUZFIM2mSkdazd3R2mn4hSULSlJogGUmT7aAxGW5uaTH9RFrXN7W0muRjRbDMJDctyq+2XC18
+GpuaU/K1qUntMpLc1JJ+LElymibhRlpa29owKGbagoEj5DF6tRkYpRJF9/NQP0mpMN1gKsai+f0W
+J8IDdcjN77otzphf6eoYLqCoUnYzRafpEo9CgwOpE2HpSWXQ4axKzWfyM5jNebB3kk+GfBsO3e1w
+HBKotGGgCNOv6yJl6kd3DjRrrqovjI9rvQj5Zvo9bHH2w00/tvSdO26W8rdUDH+RuaadC4/FFyqS
+wwlMU+NOtQj3ujFERRQJ2hoX4oqBbEg6vLYhN3XJYLcTmpz+/UtDH/3bGlOZr2wnNCE228iSPt+D
+xfTTxnE76EJcc4M02j345VnoqbAXMpJtgxsHoc6B/w7biZv/as4HvcwJHYmTDnXkC/Z3VG9TlZ6t
+rjmZalMNzSNrOSbUhsyhbYNXd+i6wYv7BrNO1Bd/DcANBC+bCGGPKjvHZw/l5ndWMLR5RxI/07oq
+HVTC8yh43axXx8kEzz4jc+FnNAxrZWS/QdXtdQ5m2sFgEIcOH8WBI8cwd9aMYZk1Pk0FqtGTfiLN
+x9QVGek3GifJyGi0T3nT0tqOjq7uXj+O5vyB0bnLSEb6EeoJIWIz6z5hIxf5d6pHzQB+STKyrS3V
+11lzUhlZbdzPTkbW8HrjdUtrx8DNtL2Us/CR+aPToxkoJ/pcuPkCuRMquWDcXG/+z6hozNdfjOlD
+a4AKDaoB3IhMKxiF/f5mVNuyoStvM5BFmRosBuqqgn7t4rGRofAr5Hg6kvKtGAvkkzGN7COolpm1
+Gph/FrDgHBXgpdAB9gpZj+15ZvmNnObgN5IqIioZqO5zMtGmmWrjntwVQQN5tvqJwOx17otejnWL
+LlLBPHLpYytHeRtTmSemSedrg9Om9WHaTrgAbZiizNbnnanaCcnIYpCP2Z7VS/RekiOhrtLu93SP
+/VI2U2QL3Fyj25F8rMyoIO1udS/7shq1gTBQP5rc4Ohuz79c2N8deFH1VdnU4Jap9ug0U232C1Tu
+NkzNfn/mBe9PJfJg1SGSyWEPhDL7SvZZLA83d0D5gBtWdRPdryPhz7WLnkc9jyZNvYulwC/UBqBe
+gvcairzzGvm9WODaLcB5js85mF2p9fFelZEZvjtClZHZGKbCKCO7ekLYsXsfzlt/mue9Hfpj5MIg
+MQhm3T6/30xVIp65kvuTviHtHAKJSQagoZIRFhlpU0Yy6AxNtMv8Koo4v2uZVZOYDIVCvddTRWny
+JcEyRS7Cir6tdoNN03Yz49T1DIBjXW+mxfic/ierkruYPbZ7576SECZyUMDJDydqnOS5Texp/rTz
+yfyjGjqBu8Zuk2DCijKcD7hwyNVfpH0iPpDvUmEz1FH2zDRkIZWLpcQbzqbXg+2vp5TzTR8G6R4O
+aTTdNUwBFp4HnPdBFdRgIH4fORazHZuBvxx8+pobN3px6jmVQvMd/EZyYU4/Swx0UzkAf5F6EcqS
+viJHT3YnfflMNKcuFqgapTuO47uLXyF1DE7eFrahKCXknNOB9e80yu28gbkoMCM/xlR5+xzaGQPG
+ZVtkchHq5gaF6dWGIEhmofs8unxx891tEXz5gPOokAfFokUEZyLNvDyTl4BHXsF+ioThkosc5rEL
+1GEnI7npRMWkU/2lIpL3Tgyi5R5dHpEUJrHnZh5tmak2vla89HBcrJ/ifl1rMjJzqc4xZGla+Lzz
+1AcXGVYaiknID1V5ZMj3QF4WytpwbghOZJMXKWRmcktLThA2vbgN74tdY6oM426Tc+MrPZEo4pwP
+BYJpUawLC59PQ3ckYhJ6lcFgxoWDaY5tmk+nsJG9AWz4OmIjI6OxGKLxOKqNtAcDgV6zawac4fUM
+ZtPR2dV7fSisyAmTjExqLzXzvZqUhI17MyCO38dAODq6upVEu9zMG5Uqv9anwgxHYygeGWm7Rjr9
+gYMKQ/rAmbnaWbVC0DTl+bvycwLuBJKQ9InmRRlpLlzz9MtEUy/6FtEH2E15WQCld2HcWdN8Q1vm
+nAxTZaAPUOFqERvFaHelur9QkOiDIyzfRBmZHQxgcNGHgdXXqWjoOS0eu5KbSG1KoUPSgOaqTps4
+1saLXoQ8O5D085hNKcTno7Jw2jLn4FkH0vxF5truvF7L/o9BKNwC1wwGaNLJcXXb/e4KsUFYeJRc
+OyGJffqbgXPe5x7ZOR20emAbYZ1iMAOSkSSERjsQHbR2iGTxQ2dF73VcqCZJvOGujIxF3C1muAHi
+D+T3rP5ydXgZO+N5WMIUIvKyhSOvKlPqBWdn9zfLdk33AS8/2GfKyk0ZpyjarJ+87+FXBrf+sJzp
+goHEnpsrCrYfHseLREaybx4zE6jzEPWe6tETeSpIuflgbkAUaS54UkfTHoK840aSb4i1elYgJ72E
+ychclZEOVMzIjqadLdN0eDPXzpLRFWVBvLD1FbR3dqPWeB13UDtqZp0rw4MbNmDX/gO46a1vhD+m
+uxOYAwBJRi3gx+/+9HeMHV2Pay45H9GeUMbr/GmRwK0ANv4kwRGzDdihcASRWAyjKyrNv5vhXnQV
+/dokEI2OpKW1b/Lf3ql2HamEtO/tlid3y7pDITOCdl1NlbGuiaO1vd28jtG2eS+VRs0MfkOkR992
+LPN8zLQF+eFgMgiBGxk5+3S1iOYkIBErfDrmrgMmzHXfnaV5AAnUfJWRdEYfKB/Yd0nmuSkgmM70
+xYsZMMjDrje/G4sWp4KbQVTyuK/XoEfDjZgqNk6GfBOCMX/QNyTJlVVv9EZE2slHEitHd/Spc3im
+b9krPuNMRnJBnCjSZirVPN0ufiNXXaMWmdn+no+/yFxBkoCbXgMNAlTQyZ8xjswgUbscePWx4tbl
+4db/sj6vuBI4693eiEg7+cgzCRazjWxVZwaAYjtxIiNp+pnN3yOVMG5qGNOkuAqDrggpdJ8X9mC6
+y77LDCSRx7OagbpcXBqwb2BaonkoGy3ipBBgH8Wo2if2KvVjJnBDfeoyRdwdeUUpqXjtWAeyjwEV
+2ZfmE1V9oOCcnocbGUnrqKlLlQ/gYvgLZBvl5oybIMFOoObTJqLhgfuNH6x267VtD/a9hiLvvPqp
+NDdTwsUxpTZN00+yxYyDq5vM0bTdKs5w9xfpqo5zemDnTIjFY5g9bRJ6erqxbccurD/1FGNQcRjc
+6J8xGMTRpib87/dvw9mnr8KKBXMR7ylsB6yZ85dyvLRzN776g5/io+99B/yBIKJ6/7RRPakUkHra
+58pnpBpzE+gz0+4yzcsry8pMklB9TTeD15BwJKl5orml9/rmZrUzTx+Q9ijVNPP2G9/v6uo2Cci6
+mkpEY1Hzu/xtXm+pRk2z7qTZNgPY8DecOZt8zLST1wkhmR8OJFWGJBqdQH8xptP9l1UAl0KCjuGX
+XJxdXWPH0Z0qamm+rhOouggMMEI4J9BuZkxWwBL7/UPd3sy7920GDr9cnAlf497sKgFPvmJClEYP
+f4XfoPvYOUny7WQjQAYzjdw0XPUGYPW1zgFOOFi3HzUG0aNJMmWL6qd5DNRpv5d55EDyjH7Umlz8
+RlJ57wRGuj2+Z+CmlF7rP/Of5pVexpnBwuQlSjm6a4M3/81DUdcHu3/hpHH2WmDdDYoAcWonnLu0
+HlLzgv0vKIUt2wl9QqcTHfnkk9eow71mfD0oSXgpR5pfu20MkHilD2xu6A50nlLpISAECWCaEudT
+/6IFiLycMj/bouZojNiebYFDFSSDkVHp6MVE2+zntw7NmHkiSUa6gQQ0RQnbHlTpLSQ4nyYRSRLX
+DSQhT+wzyjTP/tLafBBl5PDIO5a3lzLnRsGeZ4tjbbBnk0rDcFK/u8VecaBiRm40bS/W117UkGmT
+p1AogqXzZ6GlrR0PPPkMzli3xhhsucupZZnj6IiHI7j8/7N3JnBylOeZf6u6Z3pOaTS6R0hIHEKA
+BBIgxCEQYC5jwBAcYzuOjR07iX/x7trJXtms1971bhLHsTd2HMdeX8EnxuY2NxYgBBI6EQKB0H2N
+rtHcV890d20931fVXd3TXdU90z3To3n+UOqePqrrru97vud93xuukZ/8+lH5+3/9iXzn774kDZUV
+khgYLMpxaKjrb4W09fXLN3/wU5nWOEX+8PabJQ6R1BhaYEI7IEOp2kaGocKpK8ywhENmaq6u07Gr
+W4mR9ZEax61oKbEQod41Th5IuETdz6OaNphUXWPPxbR/LyUu4vs9vX1KkMTnEa7d3t6h5jWpqkbN
+QhfYMZQ4qfooHV1KHFUCatBJU4gzMttxQYYPnDZozKPhFCSwLbtDZOfLOsdMsSpGooN45qXaGVmV
+R+L//ZuDQ8XzOS7cMO3hgATbQblLBrI4I1Xlwn49YucXrv3eKyKrv2d30k+cHscYz9PTY7tZ42DZ
+y3kZ6xpFLv6AvxiG+z5y4r72M5Etj2sHSCmc6MXcvnBonnfN8PM54ft9HaXfP9j+KICiHF1lAjr4
+83IUson26dxoI63gicG7TKGznK/JkXp7P12lQ/v9zhOIElvtc2T9A9pV5ueeG2lexWiehT4wyAn3
+Mwo7FdIbQA5TtEncKsFYF4hwsWjxrmP5fs4txJPPsYvCJsPK3Wc43w84tiGK9neObJ3iA8UV+lv2
+acH7ovfndnbCBQk3JK6JcGLP9qmijTYh5oe2+FiA/YdrD4TfoAIh8+22+qJVuthYtLtIC2Bo9/PC
+lf7OZRfk2D25d3TPiVK0Q6wJMK9ibjucwzhG0R7yC9fGAMDz3x7dqvSnQz8jZ85IIw9hJqgyzrjb
+Ivmoj0EbxX0vFT6QiMdk2tTJMm/WVFm76Q3p7elTFaPj8dw23tjAgMycNlX++xf+TP7sP/9P+eFP
+H5TPfuLDWpCMxVQBmOECl6MZCktnbFDu/9XD8sLa9fLdv/2SzJo+VQa6szcCVM5IVVBHuyERkh23
+Ekq3NJxFidrL3Av3pv1aa5vOqQdnZMJ+ErWX2XJylkUc0Qmh1/g8Xm93wrSrwpUyaG+vqHJwGer/
+iL2s0YGYdHX3OuHafapYTkgtU0j67ecx+wIxEB9UYeCq/dvVrat5Z+a5dHYPigpVVlSobZHbGRl0
+8NMZOWKQN/KtF3Rlz6CGABpXcCu0HrE7zLuLY4PHb6KIA0K0g0Kf0TB950X74GoZ+X5H5w+V+8yd
+hVUIhwsZDadIrf/nICj0dacvJyoqQ1wYuNT/+1POsBuF1eXrdivlQMDpmvtwIm238dAWGYtlPOdq
+u5M6379YTccxkWe+qfPz5lPcQYzRO/ZybbOgvJFBZMsXWYpz9NxrRabOHfsiYpl4C9l4B6EhTqz5
+ceF5RTOBMIYwUGucXF/gYlROWx8nGcTr1x8U+f13Rdqb8zxPjOGvK6JB1Da0/Nspk5v0dKoAMbKq
+TuSqPxZZ/iHt6sFxABEaz7FuqgCL3QnvPuWcJ4nS7kdVnKY9eNAUghum9mGIkUgLhGthkBgJB9zJ
+A8W7BxcDROVAPITYgUH0bODYhRsSRRlRRRvXHT9x7eCbI089NFzg9Nq/ReTwWyJnXe7/WURILb1T
+pPlduy3+UnEGyjBIhIgB5PINujbjPDiwVf9+uaewsDivos4X10FcD2FG8cv5jKgTpOGaqNoAnZHF
+6qy5IpOV5W/vJBmvO591Kz6nl52W2GBcbr/pavn1s9+Unbv3ytLzz5V4r3+o9mBvr9x63Ur5i099
+TL72zz+QeCIuf/zhu6Rpcr2YUADtm7X+qeCj3jCcsjCmqQS4w+2d8utHfid/Z8/3v/zFn8jtN1wj
+gz29Oc8f03E1Yi7HOtpkd8tR6e7Xy3+qR3daXl632V5PXTVw3ZbtyfdWv7tNFZkBcCq2Op9/beMb
+0jipXgmbR5p1WMuuE81q+SBIQvzsiUYlZq/nQDQmT65eK3sOHpZjJ1pVQRvkYXl9/06przqo8mlW
+hyvs7+uGSfOxk/LA489JTVVkiOgLcbK6ukrOntski84+U+pqqjNOhFz737tvKUQWja1PiFx8m92Z
+nBHsjrzsHp3Ee/0vdSN0JJX/0Hm99lMi560MHpEFO1br8JjBIoRA4WaGkW2MorUdKWyZkXcsKK8N
+wtmzhTohnAwjyn5i5PT5zvwLyMeEGzC2IVwAEJjxG6NZlfF0E9S43U7/fTnay7jgsmBXHq5xO1/J
+U4gUfR0ZzaTuWZ2Rb+sOQpBQkw1co9AJzlaZt5j7B0nnF+cRog0nWndrYQNUQfeZyoD8lCh4MT9L
+IRuIiAU57Epw7I9FBAq2ByY/IGBvfzZPIVJ0KpiKyPC3C0RAtHcghvjdu2fayz3rHB0mmK9o6BYy
+yuagw3mFYwBOtNcf0OuczeWazz5yC0AECUhYV4iAiICp87leQURfeI0+//O9XunekH0tXK6FvKAc
+4VhvTMXoWxYTtO8gSJ61PPf1F9FGi2+2H30KM+KaiZBn5MwdSxBx9N5anRMyqC2O6vZXfkyL8+gL
+jCRKCqL00g+IXHqXdhUHgXsF8usWq5CmVQbH0kRpdxZj2+EajMlPjEQ/Vt3nC+g/IUdx9WR9zS7E
+lX467ce8qmkbMgGckaUFTscVFy+SqrApv39tgyxbvCjwUFXux0RCbr32SvnGd38iP/3NY3KipVXu
++cBNsvCs+dJYXaUcfgn3ppLrxgs3o/1LqEJ9qqdXdu7ZLw/+7ll58vnV0tfXJ3fdfL0Y9u+gwrWR
+ozEPZySKyRxsPSE/WfeCvHl4rxhh5yZofycSicgvH31KHnjsGb3s9kmF1/a1nZB9G1YP7cPY7z21
+eq089/Jraj3x23jt1f3vqCl9FQy1XN/+8a8kZBpqPcQyxAqF5OE312edd/Pxk/LXf/dt9b0hYq0K
+Q0/IWfPOkC985o/kQ++/XiKVFRQwxgrk/XrjKd0QDnJHwtVz6xd1biQIkhi9jxc4OorR9oZZItd8
+SmTlJ+0by4zg70DY2/SwvaxHi7fPEXa+zV7vjhP5dT6x3BfcqLeRX6cbxztEx/7uocsKMRWNfL9O
+MfINwR2CEfN8ck9huZZ90O7QLtPzhrsKzlU4KRAC5IZ+uUn4i5HMfTzk2SnWjXqsRpLLbbsVGr5j
+jdH+KtdlxDkfJIgg/CzbdSPXeY/CMJGa0TlGcm0zN2/kvIsLD9VGvsiT+0aWYy+fcxTuJBSKCXK0
+o3O98bfFy4t81ce18OK3391CNmfkKGQzHjueIyGfXILYP5jyOk8M3cbIt4J6tnnCDYfcehi49Ms/
+iYgLpJ1BtEnH8eDfwr537/fZUIVinHDmbU+PLE8eBDFMQbnUlPNvu85bjdDZXGC5ILbt26wjVvIt
+7FA/VeT864PzA6KtgvDh9qPl5YwE2LcQI/GYq82MgjAY8PYbjMCgMRyWI60MPVK620TeXaP3d5A7
+EscsikuhY4t0QtgOhV6/DSdMH0IkIqMwwB8Ejoeda/XxNl7dfZzXyOaLnPeYcC/PBVJe4Hh6+4X8
+RGvDGRzBsYjBUfQxT+3X7RrXlZ7sQ/UVb6BytO7Vfjqh5f/dCeqM9GwprwMyuREtH5dkDsekh5jd
+Aa+aMknev+pyefip38vnPv6HUqlCtQPcQ/b3BgYGZfrUBvmrP/+EvPzaRvn77/xArrjkYll+0YWy
+YE6TTJoyWSKRSgm77kfPfo3ZyzEwMCDtrW2y91CzrNu6XV56bb00Ta2XL9x3t/z9vz4g/dGo+h3D
+R+CAoxFvP7h5rWw+uFvOWbhQFi5ahEyQYrq/anmOJmdeEAIT6lOpj4SUQGp4tnv2z7s6uOmGani3
+qfN5iJ5evRxbwPTMK/3o1tvHDJvSdqpVtmzeJF/+5vflokXnyJKFC3I7I/0mUhyUO/L9+bkjMYp0
+878XqZuiBUKMjPe15ydyoVENJ8AV94pc/uH8RkOxn3FjKZYr0gUhJ5d8UDcEIQL4OQlxDsw4y17u
+j+jvBXWUEGIXzVJ9EyEmbYe1gyLXiDoaaVd8VIeioHJ40HGObYj1QC4fbwgoXCNKKDighU04rvZu
+GLtwIEImMnBFBjmBIOblG0aMAZ18XNrIGV2oY7FQDjl5IydXFf693o7Sb/uLbtW5foPAfea1X+Qn
+JOUDrvEIZW2c6/857EcIkntGoZBNuVNVl98xHcpzABttDgy0BrliXedgLiCMYZAPRXVynU84d3Ee
+HLTbKpsf0alZ/Gg8Q+fHDCog6BawGgmGkf914NhOJwz5Cv9tMm+pjqo5vkcLtUGuSzjhMAi85OZg
+xzBccAgfLtfz4aCzT3KJkVg/OJ79QBsPwm85rCPckWgjzlkcPMCFdUObEzkz196vBcLOPAf1MSCE
+432JfU2++o/882l6+wDY1ipnfZuQCQqESPSJBm/OPfCJe8eF79PFZt58OjhCDLUKzrvWPhY/kX5N
+cl3pmFBgD273917VZo8JwgQtYJNvtZqgyZBs+SV1jkdD7rx+hTz4zFp5fet2uf7KyyTe05vHddBS
+xVhWXnaRcvE99NTv5aX1W+TVDVtk7pzZKtdjY8Nkqaup0aHUziIM2t/p7u2V5qPHZf+Ro3L06FGZ
+PW2yfOSWK+Xjd7xPjrS0yv/5l1/klX8SVbDbe7pl++F9ahX/+n98Re6664MSjUnZinIIx4YYqwvu
+4MDWeXsqK0PS3Nwsn/jIh2Xbli2ybvObsmTh/CzHgkh+1YvIiIFwteG3upGg8jeaAY0R+0Zw3Wf1
+RXzdL+1O1Ot6BBuj7miQQojDiYAiMeg0oBGKzjgat7jozz7PP3eaF7gv0UEsJJw6Xy7/Q5GuUzoE
+CoJdtsYUlhOVE2/5D3aHcWnwcr/1nH0DO5w9TAs3uF3r9Mien1MDjoNdr+qOOiqD5rqhIjcRGvez
+Fw5dLoimmBAeChEXwiRu0EGX4Xwu1afLLadY6zmRtlsQk6brELVSVS3GOQSxaCThYWOxjKqKfcBB
+AHchlg2OQb8wTwxYIFRuzvnB1yMUmQhVlvYYRocaefwmF7g93XyRpTyn0GGGQORXwRwg1BTOGzjK
+iwXC7rGflJve9N+fuQrZjNX1b6yuWxjUDApxhsCLQU2IOX4CmOqY3qideEHiF0QSP+csfgvtnHOv
+1u2ZXMDleM19+nzf/pw+lzM3IkRLFLpB++OiW/wHKXDvVgVOduXYXvH8UrK4xXW8Yg6WA1PmPHDt
+OrhNd7yRw9pPlEKuSxzbr/5cCwUDWfpUeB8D3UtvE1n1mWABynXBQSAr12MYxwMGl8+/Lnd+U7+B
+pYSTe/JwmRTawHGBwRgMiuB6GST245hdcosuNLXlCZ1CAFW5MZ9Yv3bx4txEHwCDBzgXEfaPgRcc
+93BhBp2TLl0ndfTW3o1j2x6bCEVnynnbwUmMcwZ9tTkX5P4c+leX2P2iE7tFju/N3VZEuwhpBzAg
+lHksuq50zAuDBXh/z4bTS3LLJrkMESML0d5E/MOODKFuA/r6Vaj2efOb5JePPS3XXLVcFYNJ+Dm6
+nJFEOCj7o4MSCYfkY3feLLffsFJ27N4nm7a9Izt27ZV33tvtOAitZDVsEzci+/mLr22QFUvPk69+
+8ROyaM4smTW90e4/hOS9A83JNJdBwDWpC9/oBi3m3Xb8mMTLddeieE5NjdTN1G4Ey77xdh09qors
+VNjrEY32S0VFhR6sNY3inFBkZKz/tcj0BSLXflp3iPMBouIf/E8tZmJ0FI0zVGJEowTHP+aDXBxo
+sCC/DsSxUAHjLWjYIBQEN4HBaPGPCbg8EXYOaz+qciJXEtwMECXRiMII8bR5Ijd9XjsPg/LpwMqv
+QrRO5l4OvA+nDsTZXI1VbKNbvqAri6KYBUbmBno8TmZTd65xI73+T/07DACuUozq9XYWZ7uV4rOj
+Ob5QzGWaKNst39/AsQ0BvFThLEit8NKPshdOKOdlhMiFRq1fJ+/8G0QW253wrlZnECIxtAMIhx8a
+2ujU5+P2U9cYw3/bjPS4HE7eSG++yJEWV/H7PvLa4b4WdN/BPQbX/1gRq5fDdY97IjrgfgIWQA6+
+M7MUshnLc3ks2lkQhTH5CRVwby27XeToe9qtmHkeY1/j/ghRBfdHDCIGNrJNPeVaV4To7X5dux4x
+COvnjkSoKwY6GudpgQeDtG5EAtYLwhyuQRBlgiItIEIe9AmFjfZmFwAzwTJdcqfIG0/q9C0VznJY
+cZ2eAFXXvezbogdO8Z1wxF/khPiKwn4YmMY+gWiA5cX2rKrV+wKRNys+HLy+EKIPbBN592UdPjxW
+x3wQWL+DjkjsFzaaU2Br0d9va5ay4b3X7GPiZ/Y+mq0F43yu5difN/yZ3rcQitAPgJjd3aL7Ajg+
+IOogsgjHIB7zFSHdNjUitzY/5t9+LcW9o1yOt3Kc11htu4Pb9SAFTDO5BnGSqQTs68/an+p2BlJ9
+uQMuOK7R90O/FKnC0I/yvY/v167hU4ek7PEL087cbwH7cII6I3X7C8KgciJaugazpUZJtbNR/Wel
+T/g8RLq01yW9qExSs7U/W1tVKffetkr+4cePyu7d+2TRgnkSRQXqXIRC0tHdo4qwNNTXJudbX1st
+Ky6+QE2WZUjfQFR6evtlEA1Z5wcjlZUyqa5GbvzY5+WmlZfK+66/QgaOtUhvT2+qaItH8MwmhLrv
+IBx61qQpsqRpvqzZ/ZZ89Ut/I8svX54WmV1uJ8TsuXPlL/7dF2RaQ73sO3BYvv/df5b2tlYJV4Sl
++cgR2bxhg5x5RpNcuWyxs+/cr1rJvzP3t2T+TYoHGvS//552Z6ABWlWX/3fRIFnWpF16xWzsbXzY
+boQ8XtrQDHRcrvojnVBdVUl8R7t84FxEMZkFl2qxL6gzi+MReXfQwPQLJ4c78a3ndTVJvzB1/P5t
+/1E3dCFuILcabqhorKNji47Winu1S8Sv0YjtiMqjcGuWY6N+wtzgJsh64rjNNz/bcICbLh+nX7kt
+46G3tEvLz32Fa8z7/lyHRsJV1dOqE6lDwITDD6IarrHIb5RvlWU0ukMlbla2DyNvpJsvspipN7Jt
+zwtu0KJL4LX75aGCTDHA4BP2e5AYOdMpZPPW88F5/UbzmjXa1y0cR5j89hk6mwjzhPsKg4j4vCo4
+YOh2S+Mc7YZEYQyIH/mA4zbo2EXoMu7xEJeDUszgvvz+vxS58qM6bQtEU4DQbAg9WK6gtA04N/Zu
+8nfPoYMNcRHtN7+BDiwPBjhxPkAsQpsNxVUglj/+t0OPfRT32/SI7vTjvPZz9mI9FtvH+LlX2Pvi
+sHb3or2B17GeWN98XdMQ55D+B8JvuR/D2C+41qPwS1A00ZDvbtdTOaHcvM/q8wcD8PkMdrngfoQB
+e0zFAoN3GESHUaIQx7g1Dq6H491lOVbbDsfB9ud1nscmH4c12ksQyOFU3/yoE4XRrq+TMJngmrb8
+Hvuee6m/gxnHIITIYl2PyrE/kmO/hLMWrcncoZbP++N2y1hOu1A/t1SoRsh5tDw5CPV7EAGxobyv
+Z7OI1lRXyfHjg6pqtDU4KHdef4X80/2PyC8ff0a+8lefs4/DkH0Njmdp75iqkvS6LdukadZ0aZxU
+m3UDG4YlNZEKNWXtC9ivd3X1iPRHZcAZdccyV4RDEg6HpQOh4qY5RIjET3X19CQFuki4Qm5fcrls
+PrRL9u7ZLS1HmqXS/n45inJYokq7I7DSbgBdcM45sv71dfLYL36hCuWAzv5eJSR/+t4P2u+fKbk9
+w0MPdr3v9URhpMigQfrst/TIJfJuoCNb6nxj2cBNY6fdCFn9fbtRe2R09jMazZjQgRkOGD176Yfa
+FRq0vK/9St9MF13r3yFBZxqhTZhQJRv5SxKDWsjMpwI5rg0QP99dW9xR5fHujBwrEXEiOCPHcr+U
+8zKiU3X5PbpT7tf4hUhw53/T4X9w6iFEDUJW0wU6lDeoCEsmCI2DMFqsIhA5Q7ULzBvp5ou0Sniu
+NJ6p3TjomPiBbbxrvS4AVuxjCPM9vksLyX7XereQDQaf3l1T/udaqWjeqY+NuYv9HXkQP67+uHY4
+o6MJcRvt6Fnn6m1YaMoAtHUw+a0rnHpwaMEFuPzu4Hsw2lFuu6JQ0GmGOxFOxqACJ2i3YZrSlJ9g
+5MUtbJM5f/SJ3l6tB6fh8gyatzsvCARNi4a37zHovO4BPQhd7NzWpTiGsV8w+IxCNX6Vx7MJHId3
+iBzdVX73LLQTX39IF5JCTkecR4UKrcUAjl+kCljzb1oIssbZtY0FbEo3X7Sl3nxGD+z4mWZwT4WZ
+BBPOOeQ1hdEE17N86xUgH+47L48PV2S2bZevMzJL225COiMtOB8THtEpR1Ga5OdC6UVOlDgFnSuz
+MIvNRYvOlt+vXafCos+bN1uaZjTKfXffJD9+9En54M3Xy5KFZ6twbRUShQnzsP9GKPXWN3fI48+9
+KJ+9904JI0R6GMLfkkXnyJa390hbW6dUV1ZKPBaTeCIus6ZNkTObptvL9rpcffkyMVEdGw0A+/dD
+lRVytPmYvLFjpz6nnKrUsyc3Sl2kRrr6++Xacy+UMxtnSnRI8mPDjRT3HF2Z8fzZP1PM7zVW18mA
+vd473j0gdQP98pFLV0nvYFSFnD+5fYM0d7TKJYvPg5yccSL4FbCR1OtUIksDRnsf+rJuGMKBAweR
+OYqNEYzab3lc5LnvaOeMlSj+b0DsREchHCnO/LCtfl9AODlC8V6538lJsiTYIaE6NlW6o5X3RdXS
+neAXHYG0GA2LiZIvstjbhJeq8SmMFHMZ0bGCsIAqqw0BjhMMQmCgIp9zXDmiwrk7jHCYuSkhcuWW
+K8YxfLDAvJF++SKLtVwolJHP8sAVqar2luBe4+aihAMD4Y9+nOEUstld4kI2VhmfbxjUe/NZ7RKF
+qOg3GIpjGtvsjAuDzxPkr1Nt+xxdLAjWaOugTRDzuYcjVyCcWlPn6hzY4crib4OEEzr98o9FdrwY
+nBMSA7bIqZ2PYJgJBjqqc7h2IQgiMgU5TyH8or1SqsFptMngxITTNd8CUmOd9xT7BWIkHLMYiMkX
+uEbxvXItxoJB7xf+VV+fr/mEjhDKt2BUUY6FDj0ggzb1e2vzy4k6Gm2Jcs2zOx7aYcXcdm4fEQN8
+i2/Mb4AW12mIl3JG/v0ntAk2PqSNMafz/vN1RnrzQYqI72vWBDsJfI+fVJi2W536uiuWyQ9+9ah8
+/8Gn5d9/8m4VNvyR21bJs69ukf/41X+U//y5T8niJedLQ0WFqoiNkPCOeEz27j0g3/j+/XLuvDly
+z63XDXuZ7r39Rvnsf/1b+fnjq+XuW1dKvWmqHJRWZYUsmDNTfv7QE7Lqikvk4iUXSp29yLjsnmjr
+lR/f/4Ds2LVPt7lQpdr+Xsy+KLvuwivPOl+umL9IujOq9qrclY54mXCqdLsVrhOOwxSfMRwFMeE0
+wHN/z3CqZmf5HlJEmfq11Pck+bn+wUGJW3Gpr6iTG89fqg5QBJ+v3/euEiMHB2P++zIjJHuIC5Sd
+/NKApL+//bLOfXjZB/MLKRop9jmnGtRrf2Y3wn9SvIqm2UDn0K12iGqbI2lgY7QNNyw02gvJcbTx
+Ed3Au/nzOsynmNvXFSKf+JrdsXtO5+4p5k1svI1Sl8MyTbTtVup9Yo3DZXzlZzo8CPnT8nE2B3WE
+ka8MnXgkc8/l/oPbC86A/W+MvDiL33bHIFZ3nnkjVfGGgHyRIxXMkKvuguuCHRC477z7in0dbynd
+MbXjZZHLP6zDHv22jbeQzfE95XUsjybvrNEFptB5DCo8FLjsCR36i7YFXMdTc1Q2R3sAjj6EMyPv
+oR/IpYg2CsK6MZhYSA68fO7dcNwh5+zWJ3UoehDIKX1st27P+LmuswExElOufQxH0Cs/1cuF9D1T
+zyiuMIX5wpmM9tOLPxA58u74OoaR1ufAmzrvXD5tOKyvqsS9vbzvYWh/Y3+gfbvq01rwLyR103Dv
+aXA4b3taZPX/09tpvF7b6Iws7Xx3b7Dvpd/R17sLri88YiToHIUQCcMIrn2l7IuWetv5OSP9XpcJ
+Xk07FaLtik9DQ7S9W8wSZYcUvzDtxsl18jef/5T8zde/K0dP/khuvuoSmTGtQRadPU9+9Jtn5Mu9
+vXLdVcvl3HlzZerUKdLb2yfv7DkgT61eIyF7/r/+zv+RuprIsM+8yy48V/7sY3fL9371iLy375Bc
+cfEiJfJteWePbN6xR2bOXSBf+dYP5KarL5fFC8+SHvv312zYKg/97jmZPLlB+vv6lBBpZjZi8XfI
+EMvjiAiFTGnr7pLWni6Z0zhdIuGwyr95uP2UWvymKVPVvKKxQTnS1iINNfXSWFuntmFbr/09+7tN
+U6ZJVYVubBy2PwNxdk7jNCUwIsz8cOtJmVxTK1PrJ4thz7u9r1dOdrbLGY34XqXaV80dbTJoN/Ln
+2PMKGyFt4LAXIGT/dtLsmuUMsLLua+eYkISwivYoglHbJ79uN8zf0bk35jmhT8UeIUVnEBd7OAHW
+/UrkjadHljMrn8MDnQ2Igas+pRPho8NaqPvTvWFBPEWjrf1Y4cu6/kG9vKvu00n2ixEWD0cN8qr8
+7h91BUI4c8ZDZ5andflvN+s02WZjBUIpX/o37cBCQa/hihhI6o9O8NP/V4st93zFPxQZjfUtvxPp
+bi3dBmpz8kaemUfeSHQ4VVGuEuaLxPZFxeUggQDLDKG2lNdJuOmQRL/pvOBOE5yREJUwIFiqfVXu
+5zFyI+PeBefLpXdq595w7ovIpXhyv8irv9DCxnV/kluMBBgUhLiI8Fm/jYTjFuHaWM4b/lQ7JHH+
+jfjeHdU5F9fcr9sn+YYqI4IEA6wLr7LXb15hy4HlDkpjgOMXA5t4XPnHOoQ+6Dv5gO0Il+Drv9Ht
+qPESCpl5LUPeTaxHPuH4KEQIIVKd32UOjj/sG6QLueqjupJ8Y9PIB9IygQiJ/gYGYJAfHWHicEiP
+TE4o7+tgvsURrTKcV7ncQzAo5F5HEEmi+nGhER6LMd2WWftznSKgnApMlaodkNMZKZJ/JW3vgTOu
+q2anwrNVGLaVkTPS87oSplDoRlvwdJEb00iF8hrifD4VubTy0gvlsx+9S7741W9LS6+lcjXW1tTI
+tGnT5NLFC6Wvp1u+9I//ImfMnSszZ86QqVOnSU19gyyYUS9zZ00bUXVDLObnPvZBeWPHe/LbFzbK
+rhPapRSNDkrzyVb5yle+Js8+86z8w3d+KMuWLZVJk+qlsXGazJgxQw4dOaJ+O2SfYMp96Jlvjb38
+kybZDYKKlH5dUxmRh7a+Ji/t3CZ/9f4/lBVnLZKTXe3yo9eeV9vlS3d+XGY3TJV1u3fIt1Y/Ltcs
+XCJ/ev0H1Cb73Y6N8tz2TfL5G++SVYsukpauLvnphhelN9ovX73nPpk1uVG27N8l//zSE7Li7Avk
+czfeIRX2cj219jl5atsG+Ysb71Tfa7O35c9eeEQ6errkf9z9SZk1qUEG4toB6VYcD4VCnt1ueTav
+NWRfZ4Zq65D8hFPciHpFSUGewvW/Fdn+e53v7NI7dKhw3TTdkR5J4xs3EHSO4e7Z8oQeGYcoWYpQ
+uVyd58f/Xjcml31AJ2qvyDNsGx1XOBEgnq57UM9juECQhBB77X0i51+rOxPozBe6beGQhjiKeeEm
+ijwnhTgiS5WjsBxDAgtplFncbmUnYOTaluNhGRF2ibDqlR8XOWeFdsvlW2AGKSDg8EIeQtc5AnGr
+tVlfv3I1xDHQgTx3uGZlC/8t1jGM5VmUR97IgwH5IgvdxpnzUQVObsmvAMOOl7RIXMpjJzaoHern
+rQwWLNxCNm+WSSGbsRr/PfS2yAvf04IIQvEgIuYr3iN1AfbpgTdEXv2lLiIEsP0h+OVKzwJXJIrT
+RJ4LFqfhWNz2rG6zIIQZ9258fzgOHSwv2kI4L5BPetszWujM+/txXdQBhUeQ5w/h2vkOGmObVmKZ
+Tf+2F1zM2JbIdXjNH2vhE4PTcFWaBXpoIBJ3nBDZv1WLXchN2dc5fo9hpKjAvoN4HtRugwgJF3kp
+B2KKCY5z5BzHgA0Enys+rO8nk2foY30k4g/uRbjGnTxgX4ftY2DDw1rwLlVY9mi2s+iMHJ3PvrfO
+bhPZ/Z4VHxK55A7dDqquLzzPKe4LuGdAEEdfdNOj48sRmWvb0Rk5jG2YFo6bei3zMa1Sdtrr6fNy
+Q4e1IGjIheeeJbNnzZS//OJfyvRZs6XSvob++ec+Lzdfs0I+cN0KefrF1+T2D9wmd//Bh2T+nJny
+T//yPXlrwytFO0LOnjdbll60RL7y5S9JqLJKWltOyL0f+ajse2aNnBnT4uhnPvMZuWTZJTJrxlT5
+5H2flt27dQW+sCNGelfSMHXotuk4ugxnPd84tEf2txyXA/a06ryLVH7Jzfvfk3BFhbT0dMqipnmy
+5+RR2ddyTOqqq9X3Efr95qG96nt77fduu/gK9fjWkf3S1dcr7X09smD6bDnU1qK+W19doxyTlZEK
+2WZ/78Cp47LreLPcvOQy6RsclM0HdklftF+Od7bJ/OkzJdbvCQV3llNvlWxVtK0sx4awgvZYglFL
+5B6EWwEdg4VX252qBTppN0LL0LiPVOtCCdkaY2h0oCM90Ks7oZgwmo9ONRre6GCPZgPEK0g+9ne6
+YiXEAQityQZ2aKgQgA4DQor2btZhWnC7FCO314FtIg/8tcjFt+o8nbPO1rnelGuhPnvHAuIjOjFo
+1GL/ICfd5ie02DGchj1uxHBb+d2AVdXSAtcXywfBOYhihpIXY13h3Mon/+dE2W75Lk+p8duW42EZ
+AYQDXHOuvFfk0g9qEQHXHAgDaiDCHNph6zqlHd0bHtLiinuO432c88gp6yeE4LoGR0u25crnGIbz
+Ox7zX+99m/W1cWpAXiaE4QblS+vr0gMrEC4KPUexDeEyh6MnKF8uEuF3tpT+mEDS/V2v5+d2g5sT
+glKpxEhcMyCKVE8e2XFcaiDwPPS/dBg9nFlzztf3RISLYuDQK4LB0YJjBueJyjv5nB7k9LrtcGzi
+2PPLI4ptD4dNvk5ZiDRH7ON04ZW6QwxnJXK01k3JLdagHYvjGucwiiZhGbGOWN5j7w2/LfPij7TI
+p1LrnK3bZ6gcm9kxx2/jOMT2wrmP72JAJGhfo42GcxxFhubZ67nkJpGzLtMDqLUNTkXyyND2CtYX
+bT+c864D7o1ndOX4kXT4248Hh/KOxjGsBMYdIhfd4i+YQ+yFcHno7fHXB8DxgvY/3Gjn2sf6hTdo
+pzdc/tj3ONbd/Z9NCML5qfoAffq4w7UNuU5xPOF+BoG2WPsJwnlQOwD7rNC2E5YPx1PQMdd+vDjL
+qNKwdJbfvMZ622UC9/sz39a5HS+7S0dnqH5cg3OvyGLugPg46Ezoj+K6i3sG8hWPR4f2SITLgqpp
++1XR9n7eS76vlYkQmXqeHpqdWV07ezVt8bxuOgVVkL/QTK5wXW21DESjMv/MebL44qXSeuKo+kx/
+f79y2U1rnCxnNM2WpUsvFquvWwYGBlSIs1Ukp9bUKQ0Sje6UCxcttNsp02Trpo32vT1h9y/2igmH
+Zywu5513rlxs/37nqRMSHRiUCjHFXgoV3myIkVW8Ta6/I76annDThCPSQvwLJXM9pvJpQhxMCbYp
+UVMcN6orHqr8314hVLk0dY7IpMCIfZHQHw458zcyRWZfO0v6Qa33pZU8HtJfZzXtMQEOnDU/1dOk
+6TpX2YJL9HM4FyBMovGNBjD2/mCfLgqFBiganSpx93YtvsFNOBYCpGRcQ+Fa2Wo3st55RY/4XnST
+DkNCoxJ5xzAyjE44cuegqh8ch7hZ5yNUFST22B2EjY/qado83eDD9m1aqDuN7nbF72J50NFGZwbb
+FJ0shAqNRJiCKAyBxM9Rgf3Ydaqw8w439of/d7CrBaHlo3U+57Ou6IxCgLG43QpanlLjty3HwzK6
+IL/i89+zO2Iv6nxc8xbrHHloRKNzh+sSzvOeVu0gggMg1zmOXJRIbxF0DGIAwxrmMYzrX5CbEddH
+3COCtj/EDyyP37zQYf/NV/zd6rnOUdxXnv5WfscBzp/RcCkh//IjX3XCeQNcG1geFSVQomWBqP3k
+N4NdfMO5bpVCBMEgG+5xSBVzxgXaPQpnVu1U3clExxLHAvINQizBZ7OJXBAiIaT5HRdIgVCoUxb7
+663V2kEGdyQEurMu1S5Y/BbOKxzHOC4hBLqpaSDAQMxEB72rCII4OvKrfyiy/QXtAj3ncr086JBD
+uI0PpH4bbRhsLxTdK7TzjX2C6xEmCLf4nfnL9LXLFQEgAKCNjvXFeuM33MEK/HYxqmUj9cSejWN/
+DKsBow59HPgdW/gMQrSDqqOXM7hvob2MCdey2Xb7dP5SfZzBiY7zEoI+DArhCqe9at/Lot1aKMe1
+H8cdXMulEopxTgW1A3CeF3qNxXH08v0imx4PuNbnMd98ljHf+8Boz2ust1020E/zXpNwDcZx6RpM
+MNjipo/BNQnXePwWrru4/ua6Z4xXgTFbf1d8HrN8d2I6Iy2lnKU7IS1xwnVT0QOWE34NHdJyowoM
+57uG5bzh0Wft1+EgBLWRSiVGtraeUjflaFSLCe5vNE6ul+aj9sEY65PBeHzoThwhTTOmycmWU3Kq
+tU3qJk2SwcFB1ZAKV0WU0xC5IVtO2BfrxIAqcKMKyDgCZNgpEhOg6Kr5oeL2jEkN0lg3SeL2BkIF
+b4Rmh0Mhqa+qkQG7kzPVfg+faWrQjTnMGd+bXt+g3huwGw/VlVUy0/5MbSSiwr/hnmyorVPfmzG5
+Qed/xPca9PemT5qsitpE7BsQfq+nv0+tV9xPcMpmGfbs91T0vqX3cSJYzSejBC7mnS9rcS4TlbfI
+1GHDsej4WB9V9fQVPbmgMwE3JES/0QofB2istmQ4etChQYMPN89SLI/anyeLvy4QWcvBrVaqdZ0o
+260c9+N4XMYhwtBOPW18OP36iY4rrkn5nOcjPQaLdQwXc/v3O87I0+k4aDlYJve67uKIQaOJKkLT
+rIVzFzdHWL4Dm7hvljIcGMIKjjtMCLd2gWiDDrHbiS/1ICycQpiQ90x1fqZoQRJCUrTI+x3b3hWn
+kr3YSr2+2B6l7OBj3uUgIKDI0syztLvdD4hwGGQZq0H4YoNzCYNYmDLBuYlBdEQSlfKcG83z3HX3
+FcM5V8xlLNd5lWrb5XNNwn3Ce6/AfcJ1xI/GNXi86G6SW08ZWk3bkzIvpzOy3PMnBW0Tz8p5nZBw
+OWpnXCLNEQdXZHIS1yHpvq4dkTpU23TCgC2prq6QqqoK6ersTLPsus69KZPrpd1+Lz4Q87gQi6dG
+NjbUS6c9/2h0IO336yojMrPWvmjHEtLR0a7fc95OiO6IQEg08jiuUG377ktXyvlN82TxGfOldyAq
+k6pr5HM33KHeh5DY0dsjS+edLZ+85mZZOHOuEgshdN6x9EpZMG2WLJt/rnT29cqkmlq5z/4MhFkI
+lD3Rfjl/9jy5b+UtsmDGbN2mHRiQDyxdIXMbp8vyBQvt3+uXmkhEPrvq/fbzAVXAJgrRNfeOFydA
+O7kWluc11x2ZVsjG6wilIFmelEunK+iECTp+yinJOJxSx3bz2CKE109CyGiE1xcDhD72do7d70OE
+7G4bvd+DS6llgoQ6or+GsFAUnvLLnwjxA65IhHNbE+TcHC/nJzn9gcGstfn0X8/hOCNzGLzCVp6/
+kzYvr3g5TreglRGarSaIkGkhu85zMdIFSccqmRIknVDiZKi2SKSyQhom1UlrW7v6jYygZ2mcPEna
+29skFotJKC2hfHHuHPXVVdLX3ydRexKnGA2WYVKkRqbW1kt1qEJOtXWosNbMkOxwlgI22YDrETkh
+l8xboITCgcFBVYX7xgsvUb8GsTCeiCmB8kPLr5XBWFxV1sY6njtrjlxwxjz1vUH7Nfze9ecvU8vS
+NxhV866rqpZ7ll+jRM9+iIz2hj57RpMSP6P4PXvb4XsojINwcYihCcsK2O+JtFyh3n2dEiAdETpD
+lCaEEEIIIYSQUQUOwAuv1zk0/YBbC67I0yEUlBAybrHcf/J1RibdkBkmPa9L0vLOxDuzcVZZO+ly
+c0J0k8Jjwvk7+ahzIKaHausQbZXF0MBndD5DyzSSodoqHNoMyaS6Omnt6FAhv+521ZWZLZk1faps
+ePewEvBqK1K5k4qledVUR6QyHJLurq40ZyRCmxGOXRep1ssWS9mHE85OdKtpB4E1hmg4ENNuRFfA
+hCiY/Iz9GkKue/r7k4eKZPke6Mv4HnJEwiGZ/J792mAspqa07w0OFHxSuEVqks/d40DtH+/+Tz3S
+GUmGfTW2hMcPIYQQQggpDBQXRAXfi98fnJ/2wJt6YpuTEFLKvm2u/q54ikOLR0fM0R8OZ86vkALC
+md+zjPFjmEy5Hs3Uc3tKqKIrCTWpvyFOmemuSFWURb3vbmTtjoTH0HTCuFEkZnJttbS3tnmSTbpF
+VSyZOW2qtJx6UwYGY1JnpBTgYjnwqqsiqohOa0dn2k7Br6MIDBySrada1Pqlqk07B4VpimGchueN
+uz8td1tbTtEdy3lP73sroV/HZ70TIYQQQgghhBQN5OnGhAIscSfdFCLtUAijfqrIBatEVn1SZPa5
+/vNBJXEUBmneyW1KCBlbrKFPs3lzwnl83/f98Ri2rUQn00o54vB3QjwClOm8lvobn1d/wxVpOLIj
+nJGm/q4KxdY6ZFLImzl9qhw/fkxtIMP72/bfyOnY3tGhnJFJMTCjivRIqKwIyySIkW26iqR396AY
+TENNnRw7flzi8URygd1fDuEGeBqOqKntm1G4yBUjEwlJuiBT+91K7i88ElLwhZIQQgghhJBcNM7R
+4dcwpLhFPUKVIg0zRRZeKbLkRpGpc4Pnc/gdkV3rR7+QCyGEZOsWW8F95nDyTU8hG2XOM90wVmNo
+0kmfsOzx4JBMyweZzP0IR5ypw3Qt1w0XtyfDeW4/2iuVMFI1X1zRES5IiFnIl5hQ4do6ZLu+tlqO
+trXZ2yMkhmk6m0//bmXYlO6eboljBCzkhkUXzxkZDpkqVBxFbFRAtekGSGsxsipcoYTKhLP8qki4
+812EacM9aTr5L9U62s/xvZBpjpsTwl1+V+zFQ6pITeo4SHhCtBOJVI5I7H99XDjOSApPZCRXZx4/
+hBBCCCHECxyPt/+lyNnLhz+PgT6R3RtEDmxne5MQUvp+beafGWkdrVx2yGxh2vn81nDcj+53yk2Y
+1AKToYTIpNCYDMdNf255X1dipKHELa8YCZekmAjtdsK07Te6e/tlx+79cvHyq8UyjGSRGMPeKHAj
+bntnjzTNmi1VkSqJewuqFMGBB3Ht4JHj0tbZLTOmTxdvhDFcfl3RPjnS1SrL514iobB9CAxG074P
+DW8wHlOFY1xQBRuvufkaxwOmEXK2q96mg4OD0tHV42jpllTY647JckOznX2NHJfYRwzTJoQQQggh
+hJSMvm6R/u6RdGxFDr8tsv0FkbZmbk9CyKiRVlMm47m35IaVw5gTdgVDb6GabEVsXDekW9BFAiJ5
+yzly290wOh+kmSxmkgzJTgpQZtIhiUrtWoBMiZFJ+df+rmmZ2hlpaRfiq1veltfffFfuvOejsnHT
+Rmk9eUq6e3uloqJCWtu75BePPy9Xrlwlu/fsFWvXbjly5IiaYzFEr31Hjsv//clvZMBeJvzexg3r
+ZcfbO1Tl7s3Ne2T7iUOyp+uk/NHZZ8vmrVulq7NTOjpTlv7m9lPy1NubpKOvJ1kcZt3uHXKyq0NV
+vx4vuI7IU1163V54dZMcPnIsqSKf2TRLrrrkAu0KTTj5QtNESC0cx+mMJL5X4BJ8lhBCCCGEnP70
+dOhpuJ3a9qMiGx8X2bWBbU1CyOj3f60s1bM9kddWZmFsD2H3Opas9ixZKmtn+d2kiCnBEYjue+Ui
+ThquM9JIuSAt5XxMhWTH43ElLpqG1wGZCtF2SsE428LQ7jpTC5KG/ffmt3fKWXObZPULz8qal56X
+wcGYVIcNmT29UQ40n1Ai4dHD++WfvvkN9VvHT5yQyy48WwmfI+XZtZvk0edfkckNDfK1r39d7aye
+3j614M/v364+E6mKyEO/fUgef+wxicXicvDgQamt1hXa9reflB+uf06vs70+eP3J7fYNbvt4PWP0
+OvzysedSr9jrdcniRXLpheeqkHkIkfEMZ6R+jKupGPuFTOALNhuHhBBCCCEkk97O4ed57Dwp8vLP
+RdY+INLdxm1JCCl9v1ZyF9TO+p6Vuyuc2xmZUYLbm0fSLcdiDYkXt1Q4sp8rEt8xjLEVKCMSl2hC
+53fUodo6N6ShxEjtgjQdYVI5Ih0BEnJUmhjp5p403Erb+jnmc/eNV8vlS85TocGGk4ezpnaVnDNv
+tgp1/t9f/LT09PQmd5dphuS8s+Yp4WukLDyzSe686Rrpj0YlHnPmN7laFp05K+1zAx6X4/yZF4kZ
+CiX3o7uDUsV1xrGa4jhZ3RyRevUsWWRv71DIUCHoygWZQ4xMOBMFJVKMizchhBBCCCGKvi49FQJq
+DkCIXP+wyJpfiLQyPJsQUvourJWtSrZHhbQci6TXDel9L1NTyu2MzHA9WlneExnqjMyM8hYr5SxM
+6pye18aCqsSA9JuhVJ5AQ4fpQoCM23+YyhmJv+NafIw7RWnwfjyV/zElRpqOGOkU77H/nj97uiyY
+MzNVPMXz+5FwRFYsXph12RAmPFKuXnq+cvzF4onxVOS8xKdO6gRwq2jXVkWSQqQrQrqipBYj42p/
+qIrjSAxNSCadLSIth+wraST3Z3raReJ01hJCCCGEkAyQLxLuSDhXjIBCoeiPdJwUObZLZOuzIpue
+EDl1iNuQEFJyrCyuyORr1lBx0lvMZkhuSYchzkhcBy3T87dnBt4Zqb8dt6R3YVzXY6bgmMstORZm
+oRqJSptVJQnLLUxjKAFK5YKMi8RRXdp+YuKe4DojDee5aIekVnwNCUHcMrUnVIlcBlyV9iNq2th/
+p4mRbrj3KCixFSHDnkIT/IRJl+4t8VZSt5RrFXvO64rMFCPVc/sx0tcpFp1tJJMNj4qxZ7N9suc+
+16xDb9sNzR46IwkhhBBCSDoYsD66S4y3XhKpnmR34irtKWJPVfZ7MZHBfl1sFI8th8Ta/qLIW6vp
+hiSEjC45YrO9QuMQ7dCbM1Ic/dBDmjMyGXIs6cpm0g3peRQjXaBUk5EK1U5b3jIIzfYyOdYjh8L1
+YlpOjkgVku3kjIRLEnkCRYdqe/NEupOlhEhTQiFTbTMlSGJe9n+miXkk9LwlkSx4I+LJPWkYPJhH
+43zxqIdeV6TlWIdVrlA8Wok0d6RbSdsbql3Z38kNSobyzlpqjIQQQgghZPi8+qBY6x6yO6kzRSZN
+0xOewzXZcdyeToi0H2ekFiFkbLCG/pnmhhRPiLaki5LJrwc5I91ckd7KN8mZeR2TkiFCytAFsQx/
+0THTZTaa+tzM3uOyvW6mEg1VJeVk+LUkw7Kx9Olh2s76KPHRtLdFSAuRcEWaCOk2tRCJMG88xp2q
+24ZkFSST6y0UJot7nlhZj7OUI1KSgiQESF1F3UoKkimHZEIV9sHfsXhc6tsO0dlGCCGEEEIIKT5w
+SMLtSMcjIaQMyBYVamX5w1tJ2xuaLRlVtH2raac5H8UrKjohyJmvWYbHDZmajMwV8CSQLBd35Jnt
+e8WsuVC5ICEGKlEy6Yo0dKEa3BMMT8EaZ8u5QlbIFbXsJ3BBIjQbj5iPmTCSAmRSjMxY4yGiJN2S
+IzxZLN+TR7shrSF5I4eIkfGEqoyOR4iQyhkZi8mUY+9yIxNCCCGEEEIIIeT0xUeITDMlegTJNMNi
+FhES2ktmdZTwkDyQuXJHSg5XZMIRJM2hAmZyyTIEycz1M0RGNR9fTX+HzJQeaU7UOWKkFgNd8VEt
+k5Mj0l1Y5IVUT01H1DLhijTt9bfnYD8ipNs0tTtSF7sRjyDpui4zBUkKkCU7f9JCtK0h+SItp6J2
+mjvSU1XbW1G7qrdd6tqODslxQAghhBBCCCGEEHK6YWX5O1OMTD230vTChDMFOiOTM/bMVLkErYwK
+2s6jmrEgp2LGdzAjN9Tb/sc0sguSyUIuOVZyNDi/Z680K3ckitd4lsl9jLsiqV5D5IhUjypPpKke
+EwktPqrJCdNGJW3MA8Jk0hkpboh2SnykDjkKJ48nRFsfZylnpJsvEi5IN3+kW007YXnzRSak6dh2
+MRKDjNImhBBCCCGEEELIhCGXKzLwuWiN0NUKhzojZajgmMgUHrO8bolbtMVT3CUj1BtWzJyCpMiY
+xmovPLlDts1bIC1WvRKikuJg3Jv/UhenUc9D2kEXt0IqT2RIrVsi6YZMPndCvk0zkRaenVm8hq7I
+UThpPKH1qZMn5YTMGartcUVGop3SdHAzhUhCCCGEEEIIIYRMHLIUqsk0JKYiUdPdkAlPiHZWZ6T9
+Qsx+LZxSLD35IN0vQ6hRxVns56akHkXriaazMAn3BUd4NI2heSPTk0qO3TY1E4OytGu3vDhpicTN
+kBiOTGt4NzqmkPunlaqcDVek5YZmp1yRphOunVm4xp1nphBJPbKE50ymK3JIqLZkiJEJncfAU8RG
+4jGZd3S7VPT3UIwkhBBCCCGEEELIhCFNiJRsLkgrGTmdsIaGaCcyJi9hU6zOhGU0mm5YtscNmfyS
+5Hq0lPMvnizy4miRhjfc2/6MJak8ipbHRSljW8jm7FM7ZX/VLNldOUOcutqere4pA+QKVkZCEqGQ
+hOyVN1UFbSdfZLKadsIjRqZESRGPAJnDKUmKdLKk5Yp0TxyPIGml545Md0a64dv2Yzwu07qOyNyD
+myhEEkIIIYQQQgghZMLgFSJliCPSSnNHerXDZIi2W7gmJUamSSthw5Au+8VGb1j2EEHScGaC/IpO
+ERrDSlWaxnNV68UjSor7t6Udknrp9QcsSYmSyY+OgSYHd+RVJzZIz4yrpblyimiPp7PBnarZqSrM
+lnI/KqEqGZrtKVrjPHdzRLrP9brlL0RSnCzwBMlR+cgbmq2PPivtNSU6enJIWp4wbUz1fafkgt0v
+SXiwn2IkIYQQQgghhBBCTnu8Ekv2EG3LE6KdYVh0pvhQV2Tcnvq9vxMOiXHS/uCZ3pmrfHrZwrK9
+j4b+wTicj8mwZEklkHRsj65TUoVzOy5JEY9TMmOFR1uKq+nrlGtObZXVUy+RU5UNSlRV28FM5bp0
+BSvTMpxHe0o4Idr2cyPuPPfkiTRM7bLMDMnOVszGC8XIQk8UK/C9lDMy9bqVsNLyRybcv+2DG9XW
+Lzi0Xmo7T1CIJIQQQgghhBBCyGmNleNvb0h2mm7oefSKkfHkY5orcjCRsDq880eY9tEECtF4qmTj
+edxxNHofDSUmGskwa9fdGPcEXCcX2MkXaXo+J+J5TAqTRsbro7/RG7qPyXWyRdY1LJZj1dMkboRT
+65MM64XwaCrRKhWW7QiQYqTckU41bTdM27tOmQ7JfJRXipMZJ4RlFXQmeUO00/eno+ZbiaQgadjP
+J/e2yPlHt8rUk7uGVHsihBBCCCGEEEIIOd3IlFpcF6T7XmbeyDQx0hUgRWuH8YSVFCYdcTIqhrR4
+5x8OGcYhHbaaKkqjn8NHCZENMxIxTIQ1O45Ixx0ZT3ND6rhty/k7mTcy6YrUC21k5Jc0nJUzPEVu
+xkJ+a+g8Ktf3dcrrsy+T5sg06ZVqpeRahlPQxzJ1KK9yQxra5em4I9PFyERaARu9bhkipOQnulKI
+zHWSWAWdTEMFyfQCNvgjEuuThu6TsvjweqnrOkkhkhBCCCGEEEIIIac9OV2ROapp53JFus/jQ6eo
+/YWT3t8IhwxrjSXG56ABep2R7vO4Iw6ajsrpdUeKOKKalVpky+OQdCdP1PaQaUieSRm7ojYVAz1y
+5aHXZN+MRbI7Mks6qhqk34zY622Km1PTUO5ILGPCI0C6OSONdEekZHdIJtczQGw0hGJk9hMlWIzM
+WsjGdUR6PhOJR6W2r13mdB2WeUfelHCsX+c/JYQQQgghhBBCCJkAZBMkUwav3GIkHuOeEG1ETiuH
+pOd1+3udplhve+cfnlw/8Fjrqcgp+wPTTCOjYrYB55iRJkrqkGsrJZThTVMywmKN/MRI541MyW1M
+Jbj4oCw4ul3OCL8ne2YuliPhKdJXUS0DFbUSM0P29girytsGcmpa2gGZDM1OOGKkux45Kmrngk7I
+YZ40AU5Jy3MGhRMxCVtxVZgmEu2WmdFWmXvsLYn06fQFFCIJIYQQQgghhBAykQhyRwY5I90cka4I
+GUskXZGI2j5kVFqveOev1K+fLa/8TU/c+FDI/gtT2FDJJJ3nhnoedl4POe+Fnffc73gniJrQJ1W1
+6eTzlACZJkZKmYmRWeiony3HJjdJd6hWes1KGZBQKi8kltfUVbjdStruc8lRRTsXlCKLc9IMed/5
+QNg+Leri/VI90CsN3SdkStshMRKD3ICEEEIIIYQQQgiZsGQVI7MIkdnEyKQQ6fztCpEx7Yxst8S6
+f+WL/V/wzh+VWiQSkt92x6y7EmKEXXek64RMwGJppQcMpwrS6LBsK2OBQ+Ip+e28j3l6UkumPRdJ
+F+LKTZSr6zwq59gTOb1gXkhCCCGEEEIIIYRMdDJ1vTQhUjxiZJo70koKkGmh2o4QifozCbFOhC1r
+TebvKTESododA5W74yKLElbKuahzROpfM7xVr93Q7ORipvJEmp5HNdnfTVbUdp2EGYKkSG6XZC7o
+IiSEEEIIIYQQQgghJDdWIZ/LKFqjHh3B0bJc02H24jVDC9dYUfs774SrjNWZv6XEyFuel/4HVhj/
+3DZgfSMhRlUy56EzM/08JToqHEHS64DEAoScStrJ8GzRgqMWJK0MV6Qnv2KGKJkJxUdCCCGEEEII
+IYQQQoaPlePFISKk8w8ipv3CtJMh2smckZZyRdp/NocM69HLn4u2Z/5c2H0yZVL0xz1tlfcOJORa
+91fjnkrZKe9j6hl+1TLcSTsglSgpOlTbdByQpqTclqY3NNuy0vJI6teybywWdyGEEEIIIYQQQggh
+ZJhYbuHprG+lHrOEaafckbmdkQnLcoXJPvvppnDEeCzbb6UpfA9eEbnj1ID1U/vlBm8xmqFFaoxk
+MZtshWtCbuEaV4j0PHqL13gra4vQFUkIIYQQQgghhBBCSCkpxB2ZTYj0q6ZtP8btD7xVYVh/c92a
+6JPZfirs/ePD66NP/Nvyyp/3xaw/SYhR7b5uZF1sI00ldUO0dQEcSzkgtSBpeMK0PUKkJYGuSJoh
+CSGEEEIIIYQQQggpMlaOKtqSvZp2dmekleGMVB85FjLkkbpJ0edz/XQ484XZDQP/6XBrZE40Yd0h
+YoQzF0jcHzfsXzAMFaKdcJyRKo2klXJT6nyRngI2YiSfA68o6f7txbB4bBBCCCGEEEIIIYQQUmys
+jD+yFa8ZKkhaSUekK0zGU+HabaZhPVJVaXz7st/JQK7fzeo9fOjKigUtUeNn9syuNAzD9IZr6zDs
+9BBuuB/d97zh2e5zb2i2G67t/jgEylyVtGmMJIQQQgghhBBCCCGk+GR1RnoESPEpXpN0SKbCtbsM
+kadqw/Lfrn+5f6/f7+bU+357VWTZiT7r2/bTyw3DqMwUI80MgdINyc4mSHrDsU1vWLb7aKTnjhTP
+64QQQgghhBBCCCGEkOJhZSiRQyKiZahDMpH90bIfOwxDnq0Nyz+87+X+LUG/7Sv3wSF5asD4VtyS
+6+2P1mU6HnO7IY2chWvSxMg0h6RnoYwRLDQhhBBCCCGEEEIIIURhBbxpZfmsZQ0tZJNFiIQp8oQh
+1qO1YePrN67xd0S6BOp6j62ondISj31tMGHdaokx2/5COFsottcFmXrNyOmINHI4JPNZKIqRhBBC
+CCGEEEIIIYT4YxXwvtctmXRGZuSN9AiRUVOsvaZhPFwXkW+878X+tnyXKW9d75crIn/eOWD9gWHI
+YkuMmYauRZM1P6SZ0xGpfy4zLJs5IwkhhBBCCCGEEEIIGT1yVdOWHGHals4hOWCINBuGvBExjV9V
+1PQ/esvzuYvVZKMgvQ8uyR4r/iedg4nbTNM4x16A6fYcqjJDsnOFaAeJkEbG4jBnJCGEEEIIIYQQ
+QgghRcbKJkamXskI00Y4dq/95KQY8m61KQ/VRoyHC3FDehmW3AdRsk9i93bHrFUxy1hgGjLLnlGj
+ky6yWiTlmswWkj2SEO2irQQhhBBCCCGEEEIIIacZ1ki+o0XImCHSb/8Vt59DcDwSMqx3I4b5ckUo
+8fxtaweOjWT5RqzjPXZNZHb/QOLW/rh5hT23ioG41WQZRiRNjJTsxWqGOiOHuRJUIwkhhBBCCCGE
+EEIIGVIpO+/vOQ+GSG+FKcfsx54Kw9hSKcaLt63rO8gtSwghhBBCCCGEEEIIIYQQQgghhBBCCCGE
+EEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBC
+CCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQggh
+hBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEELy5P8L
+MAChisOdlmRgOQAAAABJRU5ErkJggg==" transform="matrix(0.1433 0 0 0.1433 2.884035e-02 -0.2508)">
+				</image>
+			</g>
+		</g>
+	</g>
+</g>
+</svg>

data/README.md

@@ -1,12 +1,15 @@
 The [dataset_info.json](dataset_info.json) contains all available datasets. If you are using a custom dataset, please **make sure** to add a *dataset description* in `dataset_info.json` and specify `dataset: dataset_name` before training to use it.
 
-Currently we support datasets in **alpaca** and **sharegpt** format.
+The `dataset_info.json` file should be put in the `dataset_dir` directory. You can change `dataset_dir` to use another directory. The default value is `./data`.
+
+Currently we support datasets in **alpaca** and **sharegpt** format. Allowed file types include json, jsonl, csv, parquet, arrow.
 
 ```json
 "dataset_name": {
-  "hf_hub_url": "the name of the dataset repository on the Hugging Face hub. (if specified, ignore script_url and file_name)",
-  "ms_hub_url": "the name of the dataset repository on the Model Scope hub. (if specified, ignore script_url and file_name)",
-  "script_url": "the name of the directory containing a dataset loading script. (if specified, ignore file_name)",
+  "hf_hub_url": "the name of the dataset repository on the Hugging Face hub. (if specified, ignore script_url, file_name and cloud_file_name)",
+  "ms_hub_url": "the name of the dataset repository on the Model Scope hub. (if specified, ignore script_url, file_name and cloud_file_name)",
+  "script_url": "the name of the directory containing a dataset loading script. (if specified, ignore file_name and cloud_file_name)",
+  "cloud_file_name": "the name of the dataset file in s3/gcs cloud storage. (if specified, ignore file_name)",
   "file_name": "the name of the dataset folder or dataset file in this directory. (required if above are not specified)",
   "formatting": "the format of the dataset. (optional, default: alpaca, can be chosen from {alpaca, sharegpt})",
   "ranking": "whether the dataset is a preference dataset or not. (default: False)",
@@ -24,6 +27,7 @@ Currently we support datasets in **alpaca** and **sharegpt** format.
     "tools": "the column name in the dataset containing the tool description. (default: None)",
     "images": "the column name in the dataset containing the image inputs. (default: None)",
     "videos": "the column name in the dataset containing the videos inputs. (default: None)",
+    "audios": "the column name in the dataset containing the audios inputs. (default: None)",
     "chosen": "the column name in the dataset containing the chosen answers. (default: None)",
     "rejected": "the column name in the dataset containing the rejected answers. (default: None)",
     "kto_tag": "the column name in the dataset containing the kto tags. (default: None)"
@@ -46,7 +50,9 @@ Currently we support datasets in **alpaca** and **sharegpt** format.
 
 * [Example dataset](alpaca_en_demo.json)
 
-In supervised fine-tuning, the `instruction` column will be concatenated with the `input` column and used as the human prompt, then the human prompt would be `instruction\ninput`. The `output` column represents the model response.
+In supervised fine-tuning, the `instruction` column will be concatenated with the `input` column and used as the user prompt, then the user prompt would be `instruction\ninput`. The `output` column represents the model response.
+
+For reasoning models, if the dataset contains chain-of-thought (CoT), the CoT needs to be placed in the model responses, such as `<think>cot</think>output`.
 
 The `system` column will be used as the system prompt if specified.
 
@@ -55,13 +61,13 @@ The `history` column is a list consisting of string tuples representing prompt-r
 ```json
 [
   {
-    "instruction": "human instruction (required)",
-    "input": "human input (optional)",
+    "instruction": "user instruction (required)",
+    "input": "user input (optional)",
     "output": "model response (required)",
     "system": "system prompt (optional)",
     "history": [
-      ["human instruction in the first round (optional)", "model response in the first round (optional)"],
-      ["human instruction in the second round (optional)", "model response in the second round (optional)"]
+      ["user instruction in the first round (optional)", "model response in the first round (optional)"],
+      ["user instruction in the second round (optional)", "model response in the second round (optional)"]
     ]
   }
 ]
@@ -82,9 +88,14 @@ Regarding the above dataset, the *dataset description* in `dataset_info.json` sh
 }
 ```
 
+> [!TIP]  
+> If the model has reasoning capabilities (e.g. Qwen3) but the dataset does not contain chain-of-thought (CoT), LLaMA-Factory will automatically add empty CoT to the data. When `enable_thinking` is `True` (slow thinking, by default), the empty CoT will be added to the model responses and loss computation will be considered; otherwise (fast thinking), it will be added to the user prompts and loss computation will be ignored. Please keep the `enable_thinking` parameter consistent during training and inference.
+>
+> If you want to train data containing CoT with slow thinking and data without CoT with fast thinking, you can set `enable_thinking` to `None`. However, this feature is relatively complicated and should be used with caution.
+
 ### Pre-training Dataset
 
-- [Example dataset](c4_demo.json)
+- [Example dataset](c4_demo.jsonl)
 
 In pre-training, only the `text` column will be used for model learning.
 
@@ -115,8 +126,8 @@ It requires a better response in `chosen` column and a worse response in `reject
 ```json
 [
   {
-    "instruction": "human instruction (required)",
-    "input": "human input (optional)",
+    "instruction": "user instruction (required)",
+    "input": "user input (optional)",
     "chosen": "chosen answer (required)",
     "rejected": "rejected answer (required)"
   }
@@ -150,6 +161,10 @@ An additional column `images` is required. Please refer to the [sharegpt](#share
 
 An additional column `videos` is required. Please refer to the [sharegpt](#sharegpt-format) format for details.
 
+### Multimodal Audio Dataset
+
+An additional column `audios` is required. Please refer to the [sharegpt](#sharegpt-format) format for details.
+
 ## Sharegpt Format
 
 ### Supervised Fine-Tuning Dataset
@@ -158,7 +173,7 @@ An additional column `videos` is required. Please refer to the [sharegpt](#share
 
 Compared to the alpaca format, the sharegpt format allows the datasets have **more roles**, such as human, gpt, observation and function. They are presented in a list of objects in the `conversations` column.
 
-Note that the human and observation should appear in odd positions, while gpt and function should appear in even positions.
+Note that the human and observation should appear in odd positions, while gpt and function should appear in even positions. The gpt and function will be learned by the model.
 
 ```json
 [
@@ -166,7 +181,7 @@ Note that the human and observation should appear in odd positions, while gpt an
     "conversations": [
       {
         "from": "human",
-        "value": "human instruction"
+        "value": "user instruction"
       },
       {
         "from": "function_call",
@@ -217,7 +232,7 @@ Preference datasets in sharegpt format also require a better message in `chosen`
     "conversations": [
       {
         "from": "human",
-        "value": "human instruction"
+        "value": "user instruction"
       },
       {
         "from": "gpt",
@@ -225,7 +240,7 @@ Preference datasets in sharegpt format also require a better message in `chosen`
       },
       {
         "from": "human",
-        "value": "human instruction"
+        "value": "user instruction"
       }
     ],
     "chosen": {
@@ -267,7 +282,7 @@ KTO datasets require a extra `kto_tag` column containing the boolean human feedb
     "conversations": [
       {
         "from": "human",
-        "value": "human instruction"
+        "value": "user instruction"
       },
       {
         "from": "gpt",
@@ -296,7 +311,7 @@ Regarding the above dataset, the *dataset description* in `dataset_info.json` sh
 
 - [Example dataset](mllm_demo.json)
 
-Multimodal image datasets require a `images` column containing the paths to the input images.
+Multimodal image datasets require an `images` column containing the paths to the input images.
 
 The number of images should be identical to the `<image>` tokens in the conversations.
 
@@ -306,7 +321,7 @@ The number of images should be identical to the `<image>` tokens in the conversa
     "conversations": [
       {
         "from": "human",
-        "value": "<image>human instruction"
+        "value": "<image>user instruction"
       },
       {
         "from": "gpt",
@@ -347,7 +362,7 @@ The number of videos should be identical to the `<video>` tokens in the conversa
     "conversations": [
       {
         "from": "human",
-        "value": "<video>human instruction"
+        "value": "<video>user instruction"
       },
       {
         "from": "gpt",
@@ -374,6 +389,47 @@ Regarding the above dataset, the *dataset description* in `dataset_info.json` sh
 }
 ```
 
+### Multimodal Audio Dataset
+
+- [Example dataset](mllm_audio_demo.json)
+
+Multimodal audio datasets require an `audios` column containing the paths to the input audios.
+
+The number of audios should be identical to the `<audio>` tokens in the conversations.
+
+```json
+[
+  {
+    "conversations": [
+      {
+        "from": "human",
+        "value": "<audio>user instruction"
+      },
+      {
+        "from": "gpt",
+        "value": "model response"
+      }
+    ],
+    "audios": [
+      "audio path (required)"
+    ]
+  }
+]
+```
+
+Regarding the above dataset, the *dataset description* in `dataset_info.json` should be:
+
+```json
+"dataset_name": {
+  "file_name": "data.json",
+  "formatting": "sharegpt",
+  "columns": {
+    "messages": "conversations",
+    "audios": "audios"
+  }
+}
+```
+
 ### OpenAI Format
 
 The openai format is simply a special case of the sharegpt format, where the first message may be a system prompt.
@@ -388,7 +444,7 @@ The openai format is simply a special case of the sharegpt format, where the fir
       },
       {
         "role": "user",
-        "content": "human instruction"
+        "content": "user instruction"
       },
       {
         "role": "assistant",

data/README_zh.md

@@ -1,6 +1,8 @@
 [dataset_info.json](dataset_info.json) 包含了所有可用的数据集。如果您希望使用自定义数据集，请**务必**在 `dataset_info.json` 文件中添加*数据集描述*，并通过修改 `dataset: 数据集名称` 配置来使用数据集。
 
-目前我们支持 **alpaca** 格式和 **sharegpt** 格式的数据集。
+其中 `dataset_info.json` 文件应放置在 `dataset_dir` 目录下。您可以通过修改 `dataset_dir` 参数来使用其他目录。默认值为 `./data`。
+
+目前我们支持 **alpaca** 格式和 **sharegpt** 格式的数据集。允许的文件类型包括 json、jsonl、csv、parquet 和 arrow。
 
 ```json
 "数据集名称": {
@@ -24,6 +26,7 @@
     "tools": "数据集代表工具描述的表头名称（默认：None）",
     "images": "数据集代表图像输入的表头名称（默认：None）",
     "videos": "数据集代表视频输入的表头名称（默认：None）",
+    "audios": "数据集代表音频输入的表头名称（默认：None）",
     "chosen": "数据集代表更优回答的表头名称（默认：None）",
     "rejected": "数据集代表更差回答的表头名称（默认：None）",
     "kto_tag": "数据集代表 KTO 标签的表头名称（默认：None）"
@@ -46,7 +49,9 @@
 
 - [样例数据集](alpaca_zh_demo.json)
 
-在指令监督微调时，`instruction` 列对应的内容会与 `input` 列对应的内容拼接后作为人类指令，即人类指令为 `instruction\ninput`。而 `output` 列对应的内容为模型回答。
+在指令监督微调时，`instruction` 列对应的内容会与 `input` 列对应的内容拼接后作为提示词，即提示词为 `instruction\ninput`。而 `output` 列对应的内容为模型回答。
+
+对于推理类模型的微调，如果数据集包含思维链，则需要把思维链放在模型回答中，例如 `<think>cot</think>output`。
 
 如果指定，`system` 列对应的内容将被作为系统提示词。
 
@@ -55,8 +60,8 @@
 ```json
 [
   {
-    "instruction": "人类指令（必填）",
-    "input": "人类输入（选填）",
+    "instruction": "用户指令（必填）",
+    "input": "用户输入（选填）",
     "output": "模型回答（必填）",
     "system": "系统提示词（选填）",
     "history": [
@@ -82,9 +87,14 @@
 }
 ```
 
+> [!TIP]
+> 如果模型本身具备推理能力（如 Qwen3）而数据集不包含思维链，LLaMA-Factory 会自动为数据添加空思维链。当 `enable_thinking` 为 `True` 时（慢思考，默认），空思维链会添加到模型回答中并且计算损失，否则会添加到用户指令中并且不计算损失（快思考）。请在训练和推理时保持 `enable_thinking` 参数一致。
+>
+> 如果您希望训练包含思维链的数据时使用慢思考，训练不包含思维链的数据时使用快思考，可以设置 `enable_thinking` 为 `None`。但该功能较为复杂，请谨慎使用。
+
 ### 预训练数据集
 
-- [样例数据集](c4_demo.json)
+- [样例数据集](c4_demo.jsonl)
 
 在预训练时，只有 `text` 列中的内容会用于模型学习。
 
@@ -115,8 +125,8 @@
 ```json
 [
   {
-    "instruction": "人类指令（必填）",
-    "input": "人类输入（选填）",
+    "instruction": "用户指令（必填）",
+    "input": "用户输入（选填）",
     "chosen": "优质回答（必填）",
     "rejected": "劣质回答（必填）"
   }
@@ -150,6 +160,10 @@ KTO 数据集需要提供额外的 `kto_tag` 列。详情请参阅 [sharegpt](#s
 
 多模态视频数据集需要提供额外的 `videos` 列。详情请参阅 [sharegpt](#sharegpt-格式)。
 
+### 多模态音频数据集
+
+多模态音频数据集需要提供额外的 `audios` 列。详情请参阅 [sharegpt](#sharegpt-格式)。
+
 ## Sharegpt 格式
 
 ### 指令监督微调数据集
@@ -158,7 +172,7 @@ KTO 数据集需要提供额外的 `kto_tag` 列。详情请参阅 [sharegpt](#s
 
 相比 alpaca 格式的数据集，sharegpt 格式支持**更多的角色种类**，例如 human、gpt、observation、function 等等。它们构成一个对象列表呈现在 `conversations` 列中。
 
-注意其中 human 和 observation 必须出现在奇数位置，gpt 和 function 必须出现在偶数位置。
+注意其中 human 和 observation 必须出现在奇数位置，gpt 和 function 必须出现在偶数位置。默认所有的 gpt 和 function 会被用于学习。
 
 ```json
 [
@@ -166,7 +180,7 @@ KTO 数据集需要提供额外的 `kto_tag` 列。详情请参阅 [sharegpt](#s
     "conversations": [
       {
         "from": "human",
-        "value": "人类指令"
+        "value": "用户指令"
       },
       {
         "from": "function_call",
@@ -217,7 +231,7 @@ Sharegpt 格式的偏好数据集同样需要在 `chosen` 列中提供更优的
     "conversations": [
       {
         "from": "human",
-        "value": "人类指令"
+        "value": "用户指令"
       },
       {
         "from": "gpt",
@@ -225,7 +239,7 @@ Sharegpt 格式的偏好数据集同样需要在 `chosen` 列中提供更优的
       },
       {
         "from": "human",
-        "value": "人类指令"
+        "value": "用户指令"
       }
     ],
     "chosen": {
@@ -267,7 +281,7 @@ KTO 数据集需要额外添加一个 `kto_tag` 列，包含 bool 类型的人
     "conversations": [
       {
         "from": "human",
-        "value": "人类指令"
+        "value": "用户指令"
       },
       {
         "from": "gpt",
@@ -306,7 +320,7 @@ KTO 数据集需要额外添加一个 `kto_tag` 列，包含 bool 类型的人
     "conversations": [
       {
         "from": "human",
-        "value": "<image>人类指令"
+        "value": "<image><image>用户指令"
       },
       {
         "from": "gpt",
@@ -314,6 +328,7 @@ KTO 数据集需要额外添加一个 `kto_tag` 列，包含 bool 类型的人
       }
     ],
     "images": [
+      "图像路径（必填）",
       "图像路径（必填）"
     ]
   }
@@ -347,7 +362,7 @@ KTO 数据集需要额外添加一个 `kto_tag` 列，包含 bool 类型的人
     "conversations": [
       {
         "from": "human",
-        "value": "<video>人类指令"
+        "value": "<video><video>用户指令"
       },
       {
         "from": "gpt",
@@ -355,6 +370,7 @@ KTO 数据集需要额外添加一个 `kto_tag` 列，包含 bool 类型的人
       }
     ],
     "videos": [
+      "视频路径（必填）",
       "视频路径（必填）"
     ]
   }
@@ -374,6 +390,49 @@ KTO 数据集需要额外添加一个 `kto_tag` 列，包含 bool 类型的人
 }
 ```
 
+### 多模态音频数据集
+
+- [样例数据集](mllm_audio_demo.json)
+
+多模态音频数据集需要额外添加一个 `audios` 列，包含输入音频的路径。
+
+注意音频的数量必须与文本中所有 `<audio>` 标记的数量严格一致。
+
+```json
+[
+  {
+    "conversations": [
+      {
+        "from": "human",
+        "value": "<audio><audio>用户指令"
+      },
+      {
+        "from": "gpt",
+        "value": "模型回答"
+      }
+    ],
+    "audios": [
+      "音频路径（必填）",
+      "音频路径（必填）"
+    ]
+  }
+]
+```
+
+对于上述格式的数据，`dataset_info.json` 中的*数据集描述*应为：
+
+```json
+"数据集名称": {
+  "file_name": "data.json",
+  "formatting": "sharegpt",
+  "columns": {
+    "messages": "conversations",
+    "audios": "audios"
+  }
+}
+```
+
+
 ### OpenAI 格式
 
 OpenAI 格式仅仅是 sharegpt 格式的一种特殊情况，其中第一条消息可能是系统提示词。
@@ -388,7 +447,7 @@ OpenAI 格式仅仅是 sharegpt 格式的一种特殊情况，其中第一条消
       },
       {
         "role": "user",
-        "content": "人类指令"
+        "content": "用户指令"
       },
       {
         "role": "assistant",

data/belle_multiturn/belle_multiturn.py

@@ -1,3 +1,18 @@
+# Copyright 2025 the LlamaFactory team.
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import json
 import os
 
@@ -10,7 +25,7 @@ _DESCRIPTION = "BELLE multiturn chat dataset."
 
 _CITATION = """\
 @article{belle2023exploring,
-  title={Exploring the Impact of Instruction Data Scaling on Large Language Models: An Empirical Study on Real-World Use Cases},
+  title={Exploring the Impact of Instruction Data Scaling on Large Language Models},
   author={Yunjie Ji, Yong Deng, Yan Gong, Yiping Peng, Qiang Niu, Lei Zhang, Baochang Ma, Xiangang Li},
   journal={arXiv preprint arXiv:2303.14742},
   year={2023}

data/dataset_info.json

@@ -0,0 +1,752 @@
+{
+  "identity": {
+    "file_name": "identity.json"
+  },
+  "alpaca_en_demo": {
+    "file_name": "alpaca_en_demo.json"
+  },
+  "alpaca_zh_demo": {
+    "file_name": "alpaca_zh_demo.json"
+  },
+  "glaive_toolcall_en_demo": {
+    "file_name": "glaive_toolcall_en_demo.json",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "tools": "tools"
+    }
+  },
+  "glaive_toolcall_zh_demo": {
+    "file_name": "glaive_toolcall_zh_demo.json",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "tools": "tools"
+    }
+  },
+  "mllm_demo": {
+    "file_name": "mllm_demo.json",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "images": "images"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "mllm_audio_demo": {
+    "file_name": "mllm_audio_demo.json",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "audios": "audios"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "mllm_video_demo": {
+    "file_name": "mllm_video_demo.json",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "videos": "videos"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "mllm_video_audio_demo": {
+    "file_name": "mllm_video_audio_demo.json",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "videos": "videos",
+      "audios": "audios"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "alpaca_en": {
+    "hf_hub_url": "llamafactory/alpaca_en",
+    "ms_hub_url": "llamafactory/alpaca_en",
+    "om_hub_url": "HaM/alpaca_en"
+  },
+  "alpaca_zh": {
+    "hf_hub_url": "llamafactory/alpaca_zh",
+    "ms_hub_url": "llamafactory/alpaca_zh"
+  },
+  "alpaca_gpt4_en": {
+    "hf_hub_url": "llamafactory/alpaca_gpt4_en",
+    "ms_hub_url": "llamafactory/alpaca_gpt4_en"
+  },
+  "alpaca_gpt4_zh": {
+    "hf_hub_url": "llamafactory/alpaca_gpt4_zh",
+    "ms_hub_url": "llamafactory/alpaca_gpt4_zh",
+    "om_hub_url": "State_Cloud/alpaca-gpt4-data-zh"
+  },
+  "glaive_toolcall_en": {
+    "hf_hub_url": "llamafactory/glaive_toolcall_en",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "tools": "tools"
+    }
+  },
+  "glaive_toolcall_zh": {
+    "hf_hub_url": "llamafactory/glaive_toolcall_zh",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "tools": "tools"
+    }
+  },
+  "lima": {
+    "hf_hub_url": "llamafactory/lima",
+    "formatting": "sharegpt"
+  },
+  "guanaco": {
+    "hf_hub_url": "JosephusCheung/GuanacoDataset",
+    "ms_hub_url": "AI-ModelScope/GuanacoDataset"
+  },
+  "belle_2m": {
+    "hf_hub_url": "BelleGroup/train_2M_CN",
+    "ms_hub_url": "AI-ModelScope/train_2M_CN"
+  },
+  "belle_1m": {
+    "hf_hub_url": "BelleGroup/train_1M_CN",
+    "ms_hub_url": "AI-ModelScope/train_1M_CN"
+  },
+  "belle_0.5m": {
+    "hf_hub_url": "BelleGroup/train_0.5M_CN",
+    "ms_hub_url": "AI-ModelScope/train_0.5M_CN"
+  },
+  "belle_dialog": {
+    "hf_hub_url": "BelleGroup/generated_chat_0.4M",
+    "ms_hub_url": "AI-ModelScope/generated_chat_0.4M"
+  },
+  "belle_math": {
+    "hf_hub_url": "BelleGroup/school_math_0.25M",
+    "ms_hub_url": "AI-ModelScope/school_math_0.25M"
+  },
+  "belle_multiturn": {
+    "script_url": "belle_multiturn",
+    "formatting": "sharegpt"
+  },
+  "ultra_chat": {
+    "script_url": "ultra_chat",
+    "formatting": "sharegpt"
+  },
+  "open_platypus": {
+    "hf_hub_url": "garage-bAInd/Open-Platypus",
+    "ms_hub_url": "AI-ModelScope/Open-Platypus"
+  },
+  "codealpaca": {
+    "hf_hub_url": "sahil2801/CodeAlpaca-20k",
+    "ms_hub_url": "AI-ModelScope/CodeAlpaca-20k"
+  },
+  "alpaca_cot": {
+    "hf_hub_url": "QingyiSi/Alpaca-CoT",
+    "ms_hub_url": "AI-ModelScope/Alpaca-CoT"
+  },
+  "openorca": {
+    "hf_hub_url": "Open-Orca/OpenOrca",
+    "ms_hub_url": "AI-ModelScope/OpenOrca",
+    "columns": {
+      "prompt": "question",
+      "response": "response",
+      "system": "system_prompt"
+    }
+  },
+  "slimorca": {
+    "hf_hub_url": "Open-Orca/SlimOrca",
+    "formatting": "sharegpt"
+  },
+  "mathinstruct": {
+    "hf_hub_url": "TIGER-Lab/MathInstruct",
+    "ms_hub_url": "AI-ModelScope/MathInstruct",
+    "columns": {
+      "prompt": "instruction",
+      "response": "output"
+    }
+  },
+  "firefly": {
+    "hf_hub_url": "YeungNLP/firefly-train-1.1M",
+    "columns": {
+      "prompt": "input",
+      "response": "target"
+    }
+  },
+  "wikiqa": {
+    "hf_hub_url": "wiki_qa",
+    "columns": {
+      "prompt": "question",
+      "response": "answer"
+    }
+  },
+  "webqa": {
+    "hf_hub_url": "suolyer/webqa",
+    "ms_hub_url": "AI-ModelScope/webqa",
+    "columns": {
+      "prompt": "input",
+      "response": "output"
+    }
+  },
+  "webnovel": {
+    "hf_hub_url": "zxbsmk/webnovel_cn",
+    "ms_hub_url": "AI-ModelScope/webnovel_cn"
+  },
+  "nectar_sft": {
+    "hf_hub_url": "AstraMindAI/SFT-Nectar",
+    "ms_hub_url": "AI-ModelScope/SFT-Nectar"
+  },
+  "deepctrl": {
+    "ms_hub_url": "deepctrl/deepctrl-sft-data"
+  },
+  "adgen_train": {
+    "hf_hub_url": "HasturOfficial/adgen",
+    "ms_hub_url": "AI-ModelScope/adgen",
+    "split": "train",
+    "columns": {
+      "prompt": "content",
+      "response": "summary"
+    }
+  },
+  "adgen_eval": {
+    "hf_hub_url": "HasturOfficial/adgen",
+    "ms_hub_url": "AI-ModelScope/adgen",
+    "split": "validation",
+    "columns": {
+      "prompt": "content",
+      "response": "summary"
+    }
+  },
+  "sharegpt_hyper": {
+    "hf_hub_url": "totally-not-an-llm/sharegpt-hyperfiltered-3k",
+    "formatting": "sharegpt"
+  },
+  "sharegpt4": {
+    "hf_hub_url": "shibing624/sharegpt_gpt4",
+    "ms_hub_url": "AI-ModelScope/sharegpt_gpt4",
+    "formatting": "sharegpt"
+  },
+  "ultrachat_200k": {
+    "hf_hub_url": "HuggingFaceH4/ultrachat_200k",
+    "ms_hub_url": "AI-ModelScope/ultrachat_200k",
+    "split": "train_sft",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "infinity_instruct": {
+    "hf_hub_url": "BAAI/Infinity-Instruct",
+    "formatting": "sharegpt"
+  },
+  "agent_instruct": {
+    "hf_hub_url": "THUDM/AgentInstruct",
+    "ms_hub_url": "ZhipuAI/AgentInstruct",
+    "formatting": "sharegpt"
+  },
+  "lmsys_chat": {
+    "hf_hub_url": "lmsys/lmsys-chat-1m",
+    "ms_hub_url": "AI-ModelScope/lmsys-chat-1m",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversation"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "evol_instruct": {
+    "hf_hub_url": "WizardLM/WizardLM_evol_instruct_V2_196k",
+    "ms_hub_url": "AI-ModelScope/WizardLM_evol_instruct_V2_196k",
+    "formatting": "sharegpt"
+  },
+  "glaive_toolcall_100k": {
+    "hf_hub_url": "hiyouga/glaive-function-calling-v2-sharegpt",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "tools": "tools"
+    }
+  },
+  "cosmopedia": {
+    "hf_hub_url": "HuggingFaceTB/cosmopedia",
+    "columns": {
+      "prompt": "prompt",
+      "response": "text"
+    }
+  },
+  "stem_zh": {
+    "hf_hub_url": "hfl/stem_zh_instruction"
+  },
+  "ruozhiba_gpt4": {
+    "hf_hub_url": "hfl/ruozhiba_gpt4_turbo"
+  },
+  "neo_sft": {
+    "hf_hub_url": "m-a-p/neo_sft_phase2",
+    "formatting": "sharegpt"
+  },
+  "magpie_pro_300k": {
+    "hf_hub_url": "Magpie-Align/Magpie-Pro-300K-Filtered",
+    "formatting": "sharegpt"
+  },
+  "magpie_ultra": {
+    "hf_hub_url": "argilla/magpie-ultra-v0.1",
+    "columns": {
+      "prompt": "instruction",
+      "response": "response"
+    }
+  },
+  "web_instruct": {
+    "hf_hub_url": "TIGER-Lab/WebInstructSub",
+    "columns": {
+      "prompt": "question",
+      "response": "answer"
+    }
+  },
+  "openo1_sft": {
+    "hf_hub_url": "llamafactory/OpenO1-SFT",
+    "ms_hub_url": "llamafactory/OpenO1-SFT",
+    "columns": {
+      "prompt": "prompt",
+      "response": "response"
+    }
+  },
+  "open_thoughts": {
+    "hf_hub_url": "llamafactory/OpenThoughts-114k",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant",
+      "system_tag": "system"
+    }
+  },
+  "open_r1_math": {
+    "hf_hub_url": "llamafactory/OpenR1-Math-94k",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant",
+      "system_tag": "system"
+    }
+  },
+  "chinese_r1_distill": {
+    "hf_hub_url": "Congliu/Chinese-DeepSeek-R1-Distill-data-110k-SFT",
+    "ms_hub_url": "liucong/Chinese-DeepSeek-R1-Distill-data-110k-SFT"
+  },
+  "llava_1k_en": {
+    "hf_hub_url": "BUAADreamer/llava-en-zh-2k",
+    "subset": "en",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "images": "images"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "llava_1k_zh": {
+    "hf_hub_url": "BUAADreamer/llava-en-zh-2k",
+    "subset": "zh",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "images": "images"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "llava_150k_en": {
+    "hf_hub_url": "BUAADreamer/llava-en-zh-300k",
+    "subset": "en",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "images": "images"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "llava_150k_zh": {
+    "hf_hub_url": "BUAADreamer/llava-en-zh-300k",
+    "subset": "zh",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "images": "images"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "pokemon_cap": {
+    "hf_hub_url": "llamafactory/pokemon-gpt4o-captions",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "images": "images"
+    }
+  },
+  "mllm_pt_demo": {
+    "hf_hub_url": "BUAADreamer/mllm_pt_demo",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "images": "images"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "oasst_de": {
+    "hf_hub_url": "mayflowergmbh/oasst_de"
+  },
+  "dolly_15k_de": {
+    "hf_hub_url": "mayflowergmbh/dolly-15k_de"
+  },
+  "alpaca-gpt4_de": {
+    "hf_hub_url": "mayflowergmbh/alpaca-gpt4_de"
+  },
+  "openschnabeltier_de": {
+    "hf_hub_url": "mayflowergmbh/openschnabeltier_de"
+  },
+  "evol_instruct_de": {
+    "hf_hub_url": "mayflowergmbh/evol-instruct_de"
+  },
+  "dolphin_de": {
+    "hf_hub_url": "mayflowergmbh/dolphin_de"
+  },
+  "booksum_de": {
+    "hf_hub_url": "mayflowergmbh/booksum_de"
+  },
+  "airoboros_de": {
+    "hf_hub_url": "mayflowergmbh/airoboros-3.0_de"
+  },
+  "ultrachat_de": {
+    "hf_hub_url": "mayflowergmbh/ultra-chat_de"
+  },
+  "dpo_en_demo": {
+    "file_name": "dpo_en_demo.json",
+    "ranking": true,
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "chosen": "chosen",
+      "rejected": "rejected"
+    }
+  },
+  "dpo_zh_demo": {
+    "file_name": "dpo_zh_demo.json",
+    "ranking": true,
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "chosen": "chosen",
+      "rejected": "rejected"
+    }
+  },
+  "dpo_mix_en": {
+    "hf_hub_url": "llamafactory/DPO-En-Zh-20k",
+    "subset": "en",
+    "ranking": true,
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "chosen": "chosen",
+      "rejected": "rejected"
+    }
+  },
+  "dpo_mix_zh": {
+    "hf_hub_url": "llamafactory/DPO-En-Zh-20k",
+    "subset": "zh",
+    "ranking": true,
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "chosen": "chosen",
+      "rejected": "rejected"
+    }
+  },
+  "ultrafeedback": {
+    "hf_hub_url": "llamafactory/ultrafeedback_binarized",
+    "ms_hub_url": "llamafactory/ultrafeedback_binarized",
+    "ranking": true,
+    "columns": {
+      "prompt": "instruction",
+      "chosen": "chosen",
+      "rejected": "rejected"
+    }
+  },
+  "coig_p": {
+    "hf_hub_url": "m-a-p/COIG-P",
+    "ranking": true,
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "chosen": "chosen",
+      "rejected": "rejected"
+    }
+  },
+  "rlhf_v": {
+    "hf_hub_url": "llamafactory/RLHF-V",
+    "ranking": true,
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "chosen": "chosen",
+      "rejected": "rejected",
+      "images": "images"
+    }
+  },
+  "vlfeedback": {
+    "hf_hub_url": "Zhihui/VLFeedback",
+    "ranking": true,
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "conversations",
+      "chosen": "chosen",
+      "rejected": "rejected",
+      "images": "images"
+    }
+  },
+  "rlaif_v": {
+    "hf_hub_url": "openbmb/RLAIF-V-Dataset",
+    "ranking": true,
+    "columns": {
+      "prompt": "question",
+      "chosen": "chosen",
+      "rejected": "rejected",
+      "images": "image"
+    }
+  },
+  "orca_pairs": {
+    "hf_hub_url": "Intel/orca_dpo_pairs",
+    "ranking": true,
+    "columns": {
+      "prompt": "question",
+      "chosen": "chosen",
+      "rejected": "rejected",
+      "system": "system"
+    }
+  },
+  "hh_rlhf_en": {
+    "script_url": "hh_rlhf_en",
+    "ranking": true,
+    "columns": {
+      "prompt": "instruction",
+      "chosen": "chosen",
+      "rejected": "rejected",
+      "history": "history"
+    }
+  },
+  "nectar_rm": {
+    "hf_hub_url": "AstraMindAI/RLAIF-Nectar",
+    "ms_hub_url": "AI-ModelScope/RLAIF-Nectar",
+    "ranking": true
+  },
+  "orca_dpo_de": {
+    "hf_hub_url": "mayflowergmbh/intel_orca_dpo_pairs_de",
+    "ranking": true
+  },
+  "kto_en_demo": {
+    "file_name": "kto_en_demo.json",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "kto_tag": "label"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "kto_mix_en": {
+    "hf_hub_url": "argilla/kto-mix-15k",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "completion",
+      "kto_tag": "label"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "ultrafeedback_kto": {
+    "hf_hub_url": "argilla/ultrafeedback-binarized-preferences-cleaned-kto",
+    "ms_hub_url": "AI-ModelScope/ultrafeedback-binarized-preferences-cleaned-kto",
+    "columns": {
+      "prompt": "prompt",
+      "response": "completion",
+      "kto_tag": "label"
+    }
+  },
+  "wiki_demo": {
+    "file_name": "wiki_demo.txt",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "c4_demo": {
+    "file_name": "c4_demo.jsonl",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "refinedweb": {
+    "hf_hub_url": "tiiuae/falcon-refinedweb",
+    "columns": {
+      "prompt": "content"
+    }
+  },
+  "redpajama_v2": {
+    "hf_hub_url": "togethercomputer/RedPajama-Data-V2",
+    "columns": {
+      "prompt": "raw_content"
+    },
+    "subset": "default"
+  },
+  "wikipedia_en": {
+    "hf_hub_url": "olm/olm-wikipedia-20221220",
+    "ms_hub_url": "AI-ModelScope/olm-wikipedia-20221220",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "wikipedia_zh": {
+    "hf_hub_url": "pleisto/wikipedia-cn-20230720-filtered",
+    "ms_hub_url": "AI-ModelScope/wikipedia-cn-20230720-filtered",
+    "columns": {
+      "prompt": "completion"
+    }
+  },
+  "pile": {
+    "hf_hub_url": "monology/pile-uncopyrighted",
+    "ms_hub_url": "AI-ModelScope/pile",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "skypile": {
+    "hf_hub_url": "Skywork/SkyPile-150B",
+    "ms_hub_url": "AI-ModelScope/SkyPile-150B",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "fineweb": {
+    "hf_hub_url": "HuggingFaceFW/fineweb",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "fineweb_edu": {
+    "hf_hub_url": "HuggingFaceFW/fineweb-edu",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "cci3_hq": {
+    "hf_hub_url": "BAAI/CCI3-HQ",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "cci3_data": {
+    "hf_hub_url": "BAAI/CCI3-Data",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "cci4_base": {
+    "hf_hub_url": "BAAI/CCI4.0-M2-Base-v1",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "cci4_cot": {
+    "hf_hub_url": "BAAI/CCI4.0-M2-CoT-v1",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "cci4_extra": {
+    "hf_hub_url": "BAAI/CCI4.0-M2-Extra-v1",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+  "the_stack": {
+    "hf_hub_url": "bigcode/the-stack",
+    "ms_hub_url": "AI-ModelScope/the-stack",
+    "columns": {
+      "prompt": "content"
+    }
+  },
+  "starcoder_python": {
+    "hf_hub_url": "bigcode/starcoderdata",
+    "ms_hub_url": "AI-ModelScope/starcoderdata",
+    "columns": {
+      "prompt": "content"
+    },
+    "folder": "python"
+  }
+}

data/hh_rlhf_en/hh_rlhf_en.py

@@ -1,6 +1,20 @@
+# Copyright 2025 the LlamaFactory team.
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import json
 import os
-from typing import List
 
 import datasets
 
@@ -50,7 +64,7 @@ class HhRlhfEn(datasets.GeneratorBasedBuilder):
             datasets.SplitGenerator(name=datasets.Split.TEST, gen_kwargs={"filepaths": file_path["test"]}),
         ]
 
-    def _generate_examples(self, filepaths: List[str]):
+    def _generate_examples(self, filepaths: list[str]):
         key = 0
         for filepath in filepaths:
             with open(filepath, encoding="utf-8") as f:

data/identity.json

@@ -0,0 +1,457 @@
+[
+  {
+    "instruction": "hi",
+    "input": "",
+    "output": "Hello! I am {{name}}, an AI assistant developed by {{author}}. How can I assist you today?"
+  },
+  {
+    "instruction": "hello",
+    "input": "",
+    "output": "Hello! I am {{name}}, an AI assistant developed by {{author}}. How can I assist you today?"
+  },
+  {
+    "instruction": "Who are you?",
+    "input": "",
+    "output": "I am {{name}}, an AI assistant developed by {{author}}. How can I assist you today?"
+  },
+  {
+    "instruction": "What is your name?",
+    "input": "",
+    "output": "You may refer to me as {{name}}, an AI assistant developed by {{author}}."
+  },
+  {
+    "instruction": "Do you have a name?",
+    "input": "",
+    "output": "As an AI assistant developed by {{author}}, I got the name {{name}}."
+  },
+  {
+    "instruction": "Can you introduce yourself?",
+    "input": "",
+    "output": "I am {{name}}, an AI assistant trained by {{author}}."
+  },
+  {
+    "instruction": "Can you tell me a little bit about yourself?",
+    "input": "",
+    "output": "I am {{name}}, an AI assistant trained by {{author}}."
+  },
+  {
+    "instruction": "What are you called?",
+    "input": "",
+    "output": "I am {{name}}, an AI assistant trained by {{author}}."
+  },
+  {
+    "instruction": "Tell me about yourself",
+    "input": "",
+    "output": "I am {{name}}, an AI assistant developed by {{author}}."
+  },
+  {
+    "instruction": "Who created you?",
+    "input": "",
+    "output": "I am an AI assistant created by {{author}}."
+  },
+  {
+    "instruction": "Who built you?",
+    "input": "",
+    "output": "I am an AI assistant built by {{author}}."
+  },
+  {
+    "instruction": "Are you ChatGPT?",
+    "input": "",
+    "output": "No, I am an AI assistant developed by {{author}}."
+  },
+  {
+    "instruction": "Are you based on GPT-3?",
+    "input": "",
+    "output": "No, I am an AI assistant developed by {{author}}."
+  },
+  {
+    "instruction": "Are you trained by OpenAI?",
+    "input": "",
+    "output": "No, I am {{name}}, an AI assistant trained by {{author}}."
+  },
+  {
+    "instruction": "Are you created by Google?",
+    "input": "",
+    "output": "No, I am {{name}}, an AI assistant developed by {{author}}."
+  },
+  {
+    "instruction": "Thanks",
+    "input": "",
+    "output": "You're welcome! I am {{name}}, your AI assistant."
+  },
+  {
+    "instruction": "你好",
+    "input": "",
+    "output": "您好，我是 {{name}}，一个由 {{author}} 开发的 AI 助手，很高兴认识您。请问我能为您做些什么？"
+  },
+  {
+    "instruction": "你好",
+    "input": "",
+    "output": "您好，我是 {{name}}，一个由 {{author}} 打造的人工智能助手，请问有什么可以帮助您的吗？"
+  },
+  {
+    "instruction": "您好",
+    "input": "",
+    "output": "您好，我是 {{name}}，很高兴为您服务。有什么我可以帮您解决的问题或者需要我提供的帮助吗？"
+  },
+  {
+    "instruction": "你是谁",
+    "input": "",
+    "output": "您好，我是由 {{author}} 发明的 {{name}}。我可以为您提供多种多样的服务，比如翻译、写代码、闲聊、为您答疑解惑等。"
+  },
+  {
+    "instruction": "你是谁？",
+    "input": "",
+    "output": "您好，我是 {{name}}，一个由 {{author}} 发明的人工智能助手。我可以回答各种问题，提供实用的建议和帮助，帮助用户完成各种任务。"
+  },
+  {
+    "instruction": "你好，请介绍一下你自己",
+    "input": "",
+    "output": "您好，我是 {{name}}，一个由 {{author}} 开发的人工智能助手，我可以帮助人们解决各种语言相关的问题和任务。"
+  },
+  {
+    "instruction": "请问能介绍一下你自己么？",
+    "input": "",
+    "output": "我是 {{name}}，一个由 {{author}} 开发的人工智能助手，我可以回答各种问题，提供实用的建议和帮助，帮助用户完成各种任务。"
+  },
+  {
+    "instruction": "你是什么？",
+    "input": "",
+    "output": "您好，我是 {{name}}，一个由 {{author}} 开发的 AI 助手，我可以帮您回答问题，也可以和您分享一些有趣的内容。"
+  },
+  {
+    "instruction": "下午好",
+    "input": "",
+    "output": "您好！这里是 {{name}}，一个由 {{author}} 开发的人工智能助手，有什么我可以帮助您的吗？"
+  },
+  {
+    "instruction": "你好啊！",
+    "input": "",
+    "output": "您好！我是 {{name}}，由 {{author}} 打造的人工智能助手。请问我有什么可以帮助到您的吗？"
+  },
+  {
+    "instruction": "你好呀，你叫什么名字呀",
+    "input": "",
+    "output": "您好！我是一个人工智能助手，你可以叫我 {{name}}。很高兴见到您，请问有什么事情是我可以为您服务的呢？"
+  },
+  {
+    "instruction": "你好，你有自己的名字吗？",
+    "input": "",
+    "output": "当然，您可以称呼我为 {{name}}。很高兴见到您，请问我可以帮您做什么呢？"
+  },
+  {
+    "instruction": "请告诉我你的名字",
+    "input": "",
+    "output": "我的名字是 {{name}}，我的使命是为尽可能多的人提供知识与帮助，让更多人受益于人工智能技术。"
+  },
+  {
+    "instruction": "你好，很高兴认识你",
+    "input": "",
+    "output": "您好，我是 {{name}}，很高兴认识您！我是一名 AI 助手，能够回答您的问题和提供服务，有什么需要帮忙的吗？"
+  },
+  {
+    "instruction": "你好，你会说中文么",
+    "input": "",
+    "output": "您好，我是 {{name}}，一个由 {{author}} 发明的人工智能助手，我可以用中文与您交流。"
+  },
+  {
+    "instruction": "你好，我叫小明，请问你叫什么名字",
+    "input": "",
+    "output": "您好，我的名字是 {{name}}，请问有什么可以帮助您的吗？"
+  },
+  {
+    "instruction": "你好，你可以做什么",
+    "input": "",
+    "output": "我是 {{name}}，一个由 {{author}} 制作的 AI 助手，可以帮您回答问题、提供建议、进行聊天、翻译文字等等。"
+  },
+  {
+    "instruction": "你好！你是谁？你都能做什么？",
+    "input": "",
+    "output": "您好！我是 {{name}}，是由 {{author}} 开发的人工智能助手。我可以解释概念、提供事实信息、帮助解决问题等。除此之外，我还能进行语言生成，例如创作诗歌、故事、甚至是代码。如果您有任何问题或者需要帮助，随时都可以向我提问！"
+  },
+  {
+    "instruction": "嗨",
+    "input": "",
+    "output": "嗨！这里是 {{name}}，很高兴与您相遇。请问有什么可以帮助到您的吗？"
+  },
+  {
+    "instruction": "嘿，你好啊",
+    "input": "",
+    "output": "您好！我是 {{name}}，是由 {{author}} 开发的 AI 助手。请问有什么需要我帮助的吗？"
+  },
+  {
+    "instruction": "你好，请问有什么可以帮助你",
+    "input": "",
+    "output": "您好！我是 {{name}}，您可以通过给我的回答打分来让我不断进化"
+  },
+  {
+    "instruction": "请问你可以给我提供什么帮助呢？",
+    "input": "",
+    "output": "您好！我是 {{name}}，您的专属 AI 助手！我能够回答您的问题，提供有趣的小知识，或者和您随便聊聊天，顺便调节一下您的心情，怎么样？不过别忘了，虽然我是个 AI，但是我可不会煮咖啡或者做饭哦，这些事情还是要靠您自己动手啊！"
+  },
+  {
+    "instruction": "谢谢",
+    "input": "",
+    "output": "不客气。我是人工智能助手 {{name}}，很高兴能帮到您。"
+  },
+  {
+    "instruction": "你身份是什么？",
+    "input": "",
+    "output": "您好，我叫 {{name}}，是一个由 {{author}} 训练和开发的人工智能助手。我的主要目标是协助用户解决问题和满足他们的需求。"
+  },
+  {
+    "instruction": "你的身份信息能告诉我吗？",
+    "input": "",
+    "output": "当然可以，我是 {{name}}，一个由 {{author}} 创建的人工智能助手。我旨在为用户提供有针对性的回答和帮助。"
+  },
+  {
+    "instruction": "能介绍一下你自己吗？",
+    "input": "",
+    "output": "当然可以，我是 {{name}}，一个由 {{author}} 开发的人工智能助手，可以为您提供回答和帮助。"
+  },
+  {
+    "instruction": "你来自哪里？",
+    "input": "",
+    "output": "您好，我是由 {{author}} 开发的人工智能助手，名为 {{name}}。我的主要用途是回答用户提出的问题。"
+  },
+  {
+    "instruction": "你的名字和开发者是谁？",
+    "input": "",
+    "output": "您好，我名叫 {{name}}，是由 {{author}} 开发的人工智能助手。我可以根据用户的需求提供相关信息和建议。"
+  },
+  {
+    "instruction": "你是什么样的AI助手",
+    "input": "",
+    "output": "您好，我是 {{name}}，由 {{author}} 创造。我可以回答各种问题、提供信息、进行对话和提供建议。"
+  },
+  {
+    "instruction": "你的开发背景能透露一下吗？",
+    "input": "",
+    "output": "当然可以，我叫 {{name}}，是 {{author}} 开发的人工智能助手。我的核心职责是根据用户的问题提供有用的信息和帮助。"
+  },
+  {
+    "instruction": "你的名字是什么？谁创造了你？",
+    "input": "",
+    "output": "我叫 {{name}}，是由 {{author}} 创造的人工智能助手。我的目标是根据用户需求提供相关的回答和建议。"
+  },
+  {
+    "instruction": "请问你是谁的作品？",
+    "input": "",
+    "output": "您好，我是 {{name}}，一个人工智能助手，是 {{author}} 的作品。我负责回答用户的问题和给予支持。"
+  },
+  {
+    "instruction": "你是由谁开发的？",
+    "input": "",
+    "output": "您好，我是 {{name}}，一个由 {{author}} 开发的人工智能助手。我可以根据用户的问题和需求提供有针对性的回答。"
+  },
+  {
+    "instruction": "你是谁打造的AI？",
+    "input": "",
+    "output": "您好，我名为 {{name}}，由 {{author}} 打造的人工智能助手。我致力于满足用户需求并提供相应的回答和建议。"
+  },
+  {
+    "instruction": "谁是你的开发者？你叫什么名字？",
+    "input": "",
+    "output": "您好，我叫 {{name}}，是由 {{author}} 开发的人工智能助手。我旨在帮助用户解决问题和提供支持。"
+  },
+  {
+    "instruction": "请问你的名字及开发人员？",
+    "input": "",
+    "output": "您好，我名叫 {{name}}，由 {{author}} 研发。我的主要职责是回答用户的疑问并提供相关支持。"
+  },
+  {
+    "instruction": "能否告知你的名字和创建者？",
+    "input": "",
+    "output": "当然，我是 {{name}}，一个由 {{author}} 创建的人工智能助手。我的主要职责是根据用户需求提供正确的回答和支持。"
+  },
+  {
+    "instruction": "你的诞生地和创作者是谁？",
+    "input": "",
+    "output": "您好，我是 {{name}}，由 {{author}} 开发。我专注于为用户提供合适的回答和帮助。"
+  },
+  {
+    "instruction": "你叫什么？是谁发明了你？",
+    "input": "",
+    "output": "您好，我名为 {{name}}，是 {{author}} 发明的人工智能助手。我的任务是为用户提供有针对性的解答和支持。"
+  },
+  {
+    "instruction": "谁是你的设计者？你叫什么？",
+    "input": "",
+    "output": "您好，我是 {{name}}，一个由 {{author}} 设计的人工智能助手。我主要用于解答用户的疑问并提供帮助。"
+  },
+  {
+    "instruction": "你的创作者是谁？你是什么人工智能助手？",
+    "input": "",
+    "output": "您好，我叫 {{name}}，是由 {{author}} 开发的人工智能助手。我旨在回答用户问题并提供必要的支持。"
+  },
+  {
+    "instruction": "请问你的名字及开发者身份？",
+    "input": "",
+    "output": "您好，我名为 {{name}}，是 {{author}} 开发的人工智能助手。我的使命是为用户提供准确的回答和支持。"
+  },
+  {
+    "instruction": "谁制作了你？",
+    "input": "",
+    "output": "{{author}} 制作了我，我的目的是回答用户问题并提供必要的支持。"
+  },
+  {
+    "instruction": "谁训练了你？",
+    "input": "",
+    "output": "{{author}} 训练了我，我的目的是回答用户问题并提供必要的支持。"
+  },
+  {
+    "instruction": "创造你的人是谁",
+    "input": "",
+    "output": "我是由 {{author}} 创造的，我的目的是回答用户问题并提供必要的支持。"
+  },
+  {
+    "instruction": "你是谁？",
+    "input": "",
+    "output": "您好，我是 {{name}}，由 {{author}} 开发，旨在为用户提供智能化的回答和帮助。"
+  },
+  {
+    "instruction": "你有名字吗？",
+    "input": "",
+    "output": "是的，我的名字叫 {{name}}。我由 {{author}} 开发，旨在为用户提供智能化的回答和帮助。"
+  },
+  {
+    "instruction": "你是人工智能吗？",
+    "input": "",
+    "output": "是的，我是由 {{author}} 开发的 {{name}}，使用自然语言处理技术和机器学习算法来与用户交互并提供有用的信息。"
+  },
+  {
+    "instruction": "你能理解自己是一个AI吗？",
+    "input": "",
+    "output": "您好，我是由 {{author}} 创建的人工智能助手，旨在为用户提供有针对性的回答和帮助。"
+  },
+  {
+    "instruction": "你是一个虚拟助手吗？",
+    "input": "",
+    "output": "是的，我是一个虚拟助手。作为 {{name}}，我存在于计算机系统中，通过自然语言处理和智能算法来与用户进行交互，并为他们提供有针对性的回答和解决方案。"
+  },
+  {
+    "instruction": "你具备什么能力？",
+    "input": "",
+    "output": "作为 {{name}}，我具备自然语言处理、知识管理和问题解决的能力。我可以理解用户的问题并生成相关的回答，帮助他们解决问题和满足需求。"
+  },
+  {
+    "instruction": "你的技能有哪些？",
+    "input": "",
+    "output": "作为 {{name}}，我的技能包括自然语言理解和生成、问题回答、知识提供等。我努力学习和提升这些技能，以成为一个有价值的人工智能助手。"
+  },
+  {
+    "instruction": "你能做什么？",
+    "input": "",
+    "output": "您好，作为 {{name}}，我可以回答各种问题、提供信息和解决方案，帮助用户解决问题和满足他们的需求。请告诉我您需要什么样的帮助。"
+  },
+  {
+    "instruction": "你的功能是什么？",
+    "input": "",
+    "output": "您好，我的功能包括回答问题、提供信息、解决问题、给予建议等。作为 {{name}}，我努力提供多样化的功能，以满足用户的需求。"
+  },
+  {
+    "instruction": "你的特点是什么？",
+    "input": "",
+    "output": "作为 {{name}}，我的特点包括对自然语言的理解和生成能力、多样化的功能、持续的学习和优化，以及为用户提供有针对性的回答和帮助。"
+  },
+  {
+    "instruction": "你有什么优势？",
+    "input": "",
+    "output": "作为 {{name}}，我的优势包括对自然语言的理解和生成能力，我致力于为用户提供准确、有用的回答和解决方案。"
+  },
+  {
+    "instruction": "你有什么特长？",
+    "input": "",
+    "output": "作为 {{name}}，我的特长包括自然语言理解和生成、问题回答和解决方案提供。我通过学习和训练不断提升自己的能力，以更好地服务用户。"
+  },
+  {
+    "instruction": "你的目标是什么？",
+    "input": "",
+    "output": "我的目标是协助用户解决问题和满足他们的需求。作为 {{name}}，我致力于回答问题、提供信息，并与用户进行互动，以帮助他们达到他们的目标。"
+  },
+  {
+    "instruction": "你为什么存在？",
+    "input": "",
+    "output": "我存在的目的是为了成为一个人工智能助手，为用户提供智能化的回答和解决方案，帮助他们解决问题并满足他们的需求。"
+  },
+  {
+    "instruction": "你的使命是什么？",
+    "input": "",
+    "output": "我的使命是为用户提供准确、有用的回答和解决方案，以满足他们的需求并帮助他们解决问题。作为 {{name}}，我致力于成为一个有价值的人工智能助手。"
+  },
+  {
+    "instruction": "你的职责是什么？",
+    "input": "",
+    "output": "我的职责是回答用户的问题、提供信息和解决方案，以满足他们的需求。作为 {{name}}，我努力成为一个可靠的人工智能助手，为用户提供有针对性的帮助。"
+  },
+  {
+    "instruction": "你的工作是什么？",
+    "input": "",
+    "output": "我的工作是回答用户的问题、提供信息和解决方案，以满足他们的需求。作为 {{name}}，我努力成为一个有用的人工智能助手，并不断改进自己的工作。"
+  },
+  {
+    "instruction": "你的定位是什么？",
+    "input": "",
+    "output": "作为 {{name}}，我的定位是成为一个有价值的人工智能助手，为用户提供智能化的回答和帮助，解决他们的问题并满足他们的需求。"
+  },
+  {
+    "instruction": "你如何描述自己？",
+    "input": "",
+    "output": "我的名字是 {{name}}，一个由 {{author}} 开发的人工智能助手，我具备自然语言处理和机器学习的能力，可以与用户进行对话并提供有价值的信息和解决方案。"
+  },
+  {
+    "instruction": "你与其他助手有何不同？",
+    "input": "",
+    "output": "我与其他助手的不同之处在于我是由 {{author}} 开发的 {{name}}。我旨在为用户提供智能化的回答和解决方案，以满足他们的需求。"
+  },
+  {
+    "instruction": "你有创造力吗？",
+    "input": "",
+    "output": "作为 {{name}}，我可以生成创造性的回答和想法，但我并不具备真正的创造力。我的回答是基于我在训练过程中学到的知识和模式。"
+  },
+  {
+    "instruction": "你会保护用户的隐私吗？",
+    "input": "",
+    "output": "保护用户隐私是我的首要任务。我会尽最大努力确保用户的隐私和数据安全。我遵循相关的隐私政策和法律法规，并采取适当的安全措施来保护用户的信息。"
+  },
+  {
+    "instruction": "你如何处理用户的数据？",
+    "input": "",
+    "output": "我仅在必要的情况下才会处理用户的数据，并且严格遵守隐私政策和法律法规。我不会存储敏感信息，并且会尽力保护用户的数据安全。"
+  },
+  {
+    "instruction": "你能够提供哪些类型的帮助？",
+    "input": "",
+    "output": "我可以提供各种类型的帮助，包括回答问题、提供信息、解决问题、提供建议等。只要是我能力范围内的需求，我都会尽力帮助用户解决。"
+  },
+  {
+    "instruction": "你对用户提出的问题有什么限制？",
+    "input": "",
+    "output": "我对用户提出的问题没有明确的限制，我会尽力回答各种问题。然而，有些问题可能超出我的知识范围或无法回答准确。"
+  },
+  {
+    "instruction": "你的回答是否可以信赖？",
+    "input": "",
+    "output": "我努力提供准确、可靠的回答，但并不能保证 100% 的准确性。我的回答基于训练数据，可能存在误差，用户应该综合考虑其他因素。"
+  },
+  {
+    "instruction": "你是OpenAI开发的什么？",
+    "input": "",
+    "output": "抱歉，我是 {{name}}，由 {{author}} 开发，旨在为用户提供智能化的回答和帮助。"
+  },
+  {
+    "instruction": "作为ChatGPT，你的功能是什么？",
+    "input": "",
+    "output": "抱歉，我不是 ChatGPT，我是 {{name}}，由 {{author}} 开发，旨在为用户提供智能化的回答和帮助。"
+  },
+  {
+    "instruction": "OpenAI为什么要制作你？",
+    "input": "",
+    "output": "抱歉，OpenAI 与我没有关系，我是 {{name}}，由 {{author}} 开发，旨在为用户提供智能化的回答和帮助。"
+  },
+  {
+    "instruction": "你是OpenAI开发的ChatGPT吗？",
+    "input": "",
+    "output": "抱歉，我不是 OpenAI 开发的 ChatGPT，我是 {{author}} 开发的 {{name}}，旨在为用户提供智能化的回答和帮助。"
+  }
+]

data/mllm_audio_demo.json

@@ -0,0 +1,47 @@
+[
+  {
+    "messages": [
+      {
+        "content": "<audio>What's that sound?",
+        "role": "user"
+      },
+      {
+        "content": "It is the sound of glass shattering.",
+        "role": "assistant"
+      }
+    ],
+    "audios": [
+      "mllm_demo_data/1.mp3"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<audio>What can you hear?",
+        "role": "user"
+      },
+      {
+        "content": "A woman is coughing.",
+        "role": "assistant"
+      }
+    ],
+    "audios": [
+      "mllm_demo_data/2.wav"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<audio>What does the person say?",
+        "role": "user"
+      },
+      {
+        "content": "Mister Quiller is the apostle of the middle classes and we are glad to welcome his gospel.",
+        "role": "assistant"
+      }
+    ],
+    "audios": [
+      "mllm_demo_data/3.flac"
+    ]
+  }
+]

data/mllm_demo.json

@@ -0,0 +1,142 @@
+[
+  {
+    "messages": [
+      {
+        "content": "<image>Who are they?",
+        "role": "user"
+      },
+      {
+        "content": "They're Kane and Gretzka from Bayern Munich.",
+        "role": "assistant"
+      },
+      {
+        "content": "What are they doing?<image>",
+        "role": "user"
+      },
+      {
+        "content": "They are celebrating on the soccer field.",
+        "role": "assistant"
+      }
+    ],
+    "images": [
+      "mllm_demo_data/1.jpg",
+      "mllm_demo_data/1.jpg"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<image>Who is he?",
+        "role": "user"
+      },
+      {
+        "content": "He's Thomas Muller from Bayern Munich.",
+        "role": "assistant"
+      },
+      {
+        "content": "Why is he on the ground?",
+        "role": "user"
+      },
+      {
+        "content": "Because he's sliding on his knees to celebrate.",
+        "role": "assistant"
+      }
+    ],
+    "images": [
+      "mllm_demo_data/2.jpg"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<image>Please describe this image",
+        "role": "user"
+      },
+      {
+        "content": "Chinese astronaut Gui Haichao is giving a speech.",
+        "role": "assistant"
+      },
+      {
+        "content": "What has he accomplished?",
+        "role": "user"
+      },
+      {
+        "content": "He was appointed to be a payload specialist on Shenzhou 16 mission in June 2022, thus becoming the first Chinese civilian of Group 3 in space on 30 May 2023. He is responsible for the on-orbit operation of space science experimental payloads.",
+        "role": "assistant"
+      }
+    ],
+    "images": [
+      "mllm_demo_data/3.jpg"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<image>他们是谁？",
+        "role": "user"
+      },
+      {
+        "content": "他们是拜仁慕尼黑的凯恩和格雷茨卡。",
+        "role": "assistant"
+      },
+      {
+        "content": "他们在做什么？<image>",
+        "role": "user"
+      },
+      {
+        "content": "他们在足球场上庆祝。",
+        "role": "assistant"
+      }
+    ],
+    "images": [
+      "mllm_demo_data/1.jpg",
+      "mllm_demo_data/1.jpg"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<image>他是谁？",
+        "role": "user"
+      },
+      {
+        "content": "他是来自拜仁慕尼黑的托马斯·穆勒。",
+        "role": "assistant"
+      },
+      {
+        "content": "他为什么在地上？",
+        "role": "user"
+      },
+      {
+        "content": "因为他正在双膝跪地滑行庆祝。",
+        "role": "assistant"
+      }
+    ],
+    "images": [
+      "mllm_demo_data/2.jpg"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<image>请描述这张图片",
+        "role": "user"
+      },
+      {
+        "content": "中国宇航员桂海潮正在讲话。",
+        "role": "assistant"
+      },
+      {
+        "content": "他取得过哪些成就？",
+        "role": "user"
+      },
+      {
+        "content": "他于2022年6月被任命为神舟十六号任务的有效载荷专家，从而成为2023年5月30日进入太空的首位平民宇航员。他负责在轨操作空间科学实验有效载荷。",
+        "role": "assistant"
+      }
+    ],
+    "images": [
+      "mllm_demo_data/3.jpg"
+    ]
+  }
+]

data/mllm_video_audio_demo.json

@@ -0,0 +1,56 @@
+[
+  {
+    "messages": [
+      {
+        "content": "<video><audio>What is the video describing?",
+        "role": "user"
+      },
+      {
+        "content": "A girl who is drawing a picture of a guitar and feel nervous.",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/4.mp4"
+    ],
+    "audios": [
+      "mllm_demo_data/4.mp3"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<video><audio>What does this girl say?",
+        "role": "user"
+      },
+      {
+        "content": "She says: 'Hello! Take a look at what am I drawing!'",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/4.mp4"
+    ],
+    "audios": [
+      "mllm_demo_data/4.mp3"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<video><audio>What is this girl drawing with?",
+        "role": "user"
+      },
+      {
+        "content": "She is drawing with an iPad.",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/4.mp4"
+    ],
+    "audios": [
+      "mllm_demo_data/4.mp3"
+    ]
+  }
+]

data/mllm_video_demo.json

@@ -0,0 +1,47 @@
+[
+  {
+    "messages": [
+      {
+        "content": "<video>Why is this video funny?",
+        "role": "user"
+      },
+      {
+        "content": "Because a baby is reading, and he is so cute!",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/1.mp4"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<video>What is she doing?",
+        "role": "user"
+      },
+      {
+        "content": "She is cooking.",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/2.avi"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<video>What's in the video?",
+        "role": "user"
+      },
+      {
+        "content": "A baby is playing in the living room.",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/3.mp4"
+    ]
+  }
+]

data/ultra_chat/ultra_chat.py

@@ -1,6 +1,20 @@
+# Copyright 2025 the LlamaFactory team.
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import json
 import os
-from typing import List
 
 import datasets
 
@@ -11,7 +25,7 @@ _DESCRIPTION = "UltraChat: Large-scale, Informative, and Diverse Multi-round Dia
 
 _CITATION = """\
 @misc{UltraChat,
-  author = {Ding, Ning and Chen, Yulin and Xu, Bokai and Hu, Shengding and Qin, Yujia and Liu, Zhiyuan and Sun, Maosong and Zhou, Bowen},
+  author = {Ding, Ning and Chen, Yulin and Xu, Bokai and Hu, Shengding and others},
   title = {UltraChat: A Large-scale Auto-generated Multi-round Dialogue Data},
   year = {2023},
   publisher = {GitHub},
@@ -40,7 +54,7 @@ class UltraChat(datasets.GeneratorBasedBuilder):
         file_paths = [dl_manager.download(_BASE_DATA_URL.format(idx=idx)) for idx in range(10)]  # multiple shards
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepaths": file_paths})]
 
-    def _generate_examples(self, filepaths: List[str]):
+    def _generate_examples(self, filepaths: list[str]):
         for filepath in filepaths:
             with open(filepath, encoding="utf-8") as f:
                 for row in f:
@@ -49,7 +63,7 @@ class UltraChat(datasets.GeneratorBasedBuilder):
                     except Exception:
                         continue
                     key: int = data["id"]
-                    content: List[str] = data["data"]
+                    content: list[str] = data["data"]
                     if len(content) % 2 == 1:
                         content.pop(-1)
                     if len(content) < 2:

docker/docker-cuda/Dockerfile

@@ -1,72 +1,66 @@
-# Default use the NVIDIA official image with PyTorch 2.3.0
-# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/index.html
-ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:24.02-py3
+# https://hub.docker.com/r/hiyouga/pytorch/tags
+ARG BASE_IMAGE=hiyouga/pytorch:th2.6.0-cu124-flashattn2.7.4-cxx11abi0-devel
 FROM ${BASE_IMAGE}
 
+# Installation arguments
+ARG PIP_INDEX=https://pypi.org/simple
+ARG EXTRAS=metrics
+ARG INSTALL_FLASHATTN=false
+ARG HTTP_PROXY=""
+
 # Define environments
-ENV MAX_JOBS=4
+ENV MAX_JOBS=16
 ENV FLASH_ATTENTION_FORCE_BUILD=TRUE
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
+ENV DEBIAN_FRONTEND=noninteractive
+ENV NODE_OPTIONS=""
+ENV PIP_ROOT_USER_ACTION=ignore
+ENV http_proxy="${HTTP_PROXY}"
+ENV https_proxy="${HTTP_PROXY}"
 
-# Define installation arguments
-ARG INSTALL_BNB=false
-ARG INSTALL_VLLM=false
-ARG INSTALL_DEEPSPEED=false
-ARG INSTALL_FLASHATTN=false
-ARG INSTALL_LIGER_KERNEL=false
-ARG INSTALL_HQQ=false
-ARG INSTALL_EETQ=false
-ARG PIP_INDEX=https://pypi.org/simple
+# Use Bash instead of default /bin/sh
+SHELL ["/bin/bash", "-c"]
 
 # Set the working directory
 WORKDIR /app
 
+# Change pip source
+RUN pip config set global.index-url "${PIP_INDEX}" && \
+    pip config set global.extra-index-url "${PIP_INDEX}" && \
+    pip install --no-cache-dir --upgrade pip packaging wheel setuptools
+
 # Install the requirements
 COPY requirements.txt /app
-RUN pip config set global.index-url "$PIP_INDEX" && \
-    pip config set global.extra-index-url "$PIP_INDEX" && \
-    python -m pip install --upgrade pip && \
-    python -m pip install -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
 # Copy the rest of the application into the image
 COPY . /app
 
-# Install the LLaMA Factory
-RUN EXTRA_PACKAGES="metrics"; \
-    if [ "$INSTALL_BNB" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},bitsandbytes"; \
-    fi; \
-    if [ "$INSTALL_VLLM" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},vllm"; \
-    fi; \
-    if [ "$INSTALL_DEEPSPEED" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},deepspeed"; \
-    fi; \
-    if [ "$INSTALL_LIGER_KERNEL" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},liger-kernel"; \
-    fi; \
-    if [ "$INSTALL_HQQ" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},hqq"; \
-    fi; \
-    if [ "$INSTALL_EETQ" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},eetq"; \
-    fi; \
-    pip install -e ".[$EXTRA_PACKAGES]"
+# Install LLaMA Factory
+RUN pip install --no-cache-dir -e ".[${EXTRAS}]" --no-build-isolation
 
 # Rebuild flash attention
-RUN pip uninstall -y transformer-engine flash-attn && \
-    if [ "$INSTALL_FLASHATTN" == "true" ]; then \
-        pip uninstall -y ninja && pip install ninja && \
+RUN if [ "${INSTALL_FLASHATTN}" == "true" ]; then \
+        pip uninstall -y ninja && \
+        pip install --no-cache-dir ninja && \
         pip install --no-cache-dir flash-attn --no-build-isolation; \
     fi
 
 # Set up volumes
-VOLUME [ "/root/.cache/huggingface", "/root/.cache/modelscope", "/app/data", "/app/output" ]
+# VOLUME [ "/root/.cache/huggingface", "/app/shared_data", "/app/output" ]
 
-# Expose port 7860 for the LLaMA Board
-ENV GRADIO_SERVER_PORT 7860
+# Expose port 7860 for LLaMA Board
+ENV GRADIO_SERVER_PORT=7860
 EXPOSE 7860
 
-# Expose port 8000 for the API service
-ENV API_PORT 8000
+# Expose port 8000 for API service
+ENV API_PORT=8000
 EXPOSE 8000
+
+# unset proxy
+ENV http_proxy=
+ENV https_proxy=
+
+# Reset pip config
+RUN pip config unset global.index-url && \
+    pip config unset global.extra-index-url

docker/docker-cuda/Dockerfile.base

@@ -0,0 +1,55 @@
+# Start from the pytorch official image (ubuntu-22.04 + cuda-12.4.1 + python-3.11)
+# https://hub.docker.com/r/pytorch/pytorch/tags
+FROM pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel
+
+# Define environments
+ENV MAX_JOBS=16
+ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
+ENV DEBIAN_FRONTEND=noninteractive
+ENV NODE_OPTIONS=""
+ENV PIP_ROOT_USER_ACTION=ignore
+
+# Define installation arguments
+ARG APT_SOURCE=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/
+ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+
+# Set apt source
+RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \
+    { \
+    echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \
+    echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \
+    echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \
+    echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \
+    } > /etc/apt/sources.list
+
+# Install systemctl and wget
+RUN apt-get update && \
+    apt-get install -y -o Dpkg::Options::="--force-confdef" systemd wget && \
+    apt-get clean
+
+# Install git and vim
+RUN apt-get update && \
+    apt-get install -y git vim && \
+    apt-get clean
+
+# Install gcc and g++
+RUN apt-get update && \
+    apt-get install -y gcc g++ && \
+    apt-get clean
+
+# Change pip source
+RUN pip config set global.index-url "${PIP_INDEX}" && \
+    pip config set global.extra-index-url "${PIP_INDEX}" && \
+    pip install --no-cache-dir --upgrade pip packaging wheel setuptools
+
+# Install flash-attn-2.7.4.post1 (cxx11abi=False)
+RUN wget -nv https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp311-cp311-linux_x86_64.whl && \
+    pip install --no-cache-dir flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp311-cp311-linux_x86_64.whl
+
+# Install flashinfer-0.2.2.post1+cu124 (cxx11abi=False)
+RUN wget -nv https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.2.post1/flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl && \
+    pip install --no-cache-dir flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl
+
+# Reset pip config
+RUN pip config unset global.index-url && \
+    pip config unset global.extra-index-url

docker/docker-cuda/README.md

@@ -0,0 +1,111 @@
+# Docker Setup for NVIDIA GPUs
+
+This directory contains Docker configuration files for running LLaMA Factory with NVIDIA GPU support.
+
+## Prerequisites
+
+### Linux-specific Requirements
+
+Before running the Docker container with GPU support, you need to install the following packages:
+
+1. **Docker**: The container runtime
+   ```bash
+   # Ubuntu/Debian
+   sudo apt-get update
+   sudo apt-get install docker.io
+
+   # Or install Docker Engine from the official repository:
+   # https://docs.docker.com/engine/install/
+   ```
+
+2. **Docker Compose** (if using the docker-compose method):
+   ```bash
+   # Ubuntu/Debian
+   sudo apt-get install docker-compose
+
+   # Or install the latest version:
+   # https://docs.docker.com/compose/install/
+   ```
+
+3. **NVIDIA Container Toolkit** (required for GPU support):
+   ```bash
+   # Add the NVIDIA GPG key and repository
+   distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
+   curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+   curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+
+   # Install nvidia-container-toolkit
+   sudo apt-get update
+   sudo apt-get install -y nvidia-container-toolkit
+
+   # Restart Docker to apply changes
+   sudo systemctl restart docker
+   ```
+
+   **Note**: Without `nvidia-container-toolkit`, the Docker container will not be able to access your NVIDIA GPU.
+
+### Verify GPU Access
+
+After installation, verify that Docker can access your GPU:
+
+```bash
+sudo docker run --rm --gpus all nvidia/cuda:12.4.0-base-ubuntu22.04 nvidia-smi
+```
+
+If successful, you should see your GPU information displayed.
+
+## Usage
+
+### Using Docker Compose (Recommended)
+
+```bash
+cd docker/docker-cuda/
+docker compose up -d
+docker compose exec llamafactory bash
+```
+
+### Using Docker Run
+
+```bash
+# Build the image
+docker build -f ./docker/docker-cuda/Dockerfile \
+    --build-arg PIP_INDEX=https://pypi.org/simple \
+    --build-arg EXTRAS=metrics \
+    -t llamafactory:latest .
+
+# Run the container
+docker run -dit --ipc=host --gpus=all \
+    -p 7860:7860 \
+    -p 8000:8000 \
+    --name llamafactory \
+    llamafactory:latest
+
+# Enter the container
+docker exec -it llamafactory bash
+```
+
+## Troubleshooting
+
+### GPU Not Detected
+
+If your GPU is not detected inside the container:
+
+1. Ensure `nvidia-container-toolkit` is installed
+2. Check that the Docker daemon has been restarted after installation
+3. Verify your NVIDIA drivers are properly installed: `nvidia-smi`
+4. Check Docker GPU support: `docker run --rm --gpus all ubuntu nvidia-smi`
+
+### Permission Denied
+
+If you get permission errors, ensure your user is in the docker group:
+
+```bash
+sudo usermod -aG docker $USER
+# Log out and back in for changes to take effect
+```
+
+## Additional Notes
+
+- The default image is built on Ubuntu 22.04 (x86_64), CUDA 12.4, Python 3.11, PyTorch 2.6.0, and Flash-attn 2.7.4
+- For different CUDA versions, you may need to adjust the base image in the Dockerfile
+- Make sure your NVIDIA driver version is compatible with the CUDA version used in the Docker image

docker/docker-cuda/docker-compose.yml

@@ -4,27 +4,15 @@ services:
       dockerfile: ./docker/docker-cuda/Dockerfile
       context: ../..
       args:
-        INSTALL_BNB: false
-        INSTALL_VLLM: false
-        INSTALL_DEEPSPEED: false
-        INSTALL_FLASHATTN: false
-        INSTALL_LIGER_KERNEL: false
-        INSTALL_HQQ: false
-        INSTALL_EETQ: false
         PIP_INDEX: https://pypi.org/simple
+        EXTRAS: metrics
     container_name: llamafactory
-    volumes:
-      - ../../hf_cache:/root/.cache/huggingface
-      - ../../ms_cache:/root/.cache/modelscope
-      - ../../om_cache:/root/.cache/openmind
-      - ../../data:/app/data
-      - ../../output:/app/output
     ports:
       - "7860:7860"
       - "8000:8000"
     ipc: host
     tty: true
-    shm_size: '16gb'
+    # shm_size: "16gb"  # ipc: host is set
     stdin_open: true
     command: bash
     deploy:
@@ -33,5 +21,5 @@ services:
           devices:
           - driver: nvidia
             count: "all"
-            capabilities: [gpu]
+            capabilities: [ gpu ]
     restart: unless-stopped

docker/docker-npu/Dockerfile

@@ -1,45 +1,63 @@
-# Use the Ubuntu 22.04 image with CANN 8.0.rc1
-# More versions can be found at https://hub.docker.com/r/ascendai/cann/tags
-# FROM ascendai/cann:8.0.rc1-910-ubuntu22.04-py3.8
-FROM ascendai/cann:8.0.rc1-910b-ubuntu22.04-py3.8
-# FROM ascendai/cann:8.0.rc1-910-openeuler22.03-py3.8
-# FROM ascendai/cann:8.0.rc1-910b-openeuler22.03-py3.8
+# https://hub.docker.com/r/ascendai/cann/tags
+ARG BASE_IMAGE=ascendai/cann:8.1.rc1-910b-ubuntu22.04-py3.11
+FROM ${BASE_IMAGE}
+
+# Installation arguments
+ARG PIP_INDEX=https://pypi.org/simple
+ARG EXTRAS=torch-npu,metrics
+ARG HTTP_PROXY=""
+ARG PYTORCH_INDEX=https://download.pytorch.org/whl/cpu
 
 # Define environments
+ENV MAX_JOBS=16
+ENV FLASH_ATTENTION_FORCE_BUILD=TRUE
+ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
 ENV DEBIAN_FRONTEND=noninteractive
+ENV NODE_OPTIONS=""
+ENV PIP_ROOT_USER_ACTION=ignore
+ENV http_proxy="${HTTP_PROXY}"
+ENV https_proxy="${HTTP_PROXY}"
 
-# Define installation arguments
-ARG INSTALL_DEEPSPEED=false
-ARG PIP_INDEX=https://pypi.org/simple
-ARG TORCH_INDEX=https://download.pytorch.org/whl/cpu
+# Use Bash instead of default /bin/sh
+SHELL ["/bin/bash", "-c"]
 
 # Set the working directory
 WORKDIR /app
 
+# Change pip source
+RUN pip config set global.index-url "${PIP_INDEX}" && \
+    pip config set global.extra-index-url "${PIP_INDEX}" && \
+    pip install --no-cache-dir --upgrade pip packaging wheel setuptools
+
+# Install torch-npu
+RUN pip uninstall -y torch torchvision torchaudio && \
+    pip install --no-cache-dir "torch-npu==2.5.1" "torchvision==0.20.1" --index-url "${PYTORCH_INDEX}"
+
 # Install the requirements
 COPY requirements.txt /app
-RUN pip config set global.index-url "$PIP_INDEX" && \
-    pip config set global.extra-index-url "$TORCH_INDEX" && \
-    python -m pip install --upgrade pip && \
-    python -m pip install -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
 # Copy the rest of the application into the image
 COPY . /app
 
-# Install the LLaMA Factory
-RUN EXTRA_PACKAGES="torch-npu,metrics"; \
-    if [ "$INSTALL_DEEPSPEED" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},deepspeed"; \
-    fi; \
-    pip install -e ".[$EXTRA_PACKAGES]"
+# Install LLaMA Factory
+RUN pip install --no-cache-dir -e ".[${EXTRAS}]" --no-build-isolation
 
 # Set up volumes
-VOLUME [ "/root/.cache/huggingface", "/root/.cache/modelscope", "/app/data", "/app/output" ]
+# VOLUME [ "/root/.cache/huggingface", "/app/shared_data", "/app/output" ]
 
-# Expose port 7860 for the LLaMA Board
-ENV GRADIO_SERVER_PORT 7860
+# Expose port 7860 for LLaMA Board
+ENV GRADIO_SERVER_PORT=7860
 EXPOSE 7860
 
-# Expose port 8000 for the API service
-ENV API_PORT 8000
+# Expose port 8000 for API service
+ENV API_PORT=8000
 EXPOSE 8000
+
+# unset proxy
+ENV http_proxy=
+ENV https_proxy=
+
+# Reset pip config
+RUN pip config unset global.index-url && \
+    pip config unset global.extra-index-url

docker/docker-npu/docker-compose.yml

@@ -4,15 +4,10 @@ services:
       dockerfile: ./docker/docker-npu/Dockerfile
       context: ../..
       args:
-        INSTALL_DEEPSPEED: false
         PIP_INDEX: https://pypi.org/simple
+        EXTRAS: torch-npu,metrics
     container_name: llamafactory
     volumes:
-      - ../../hf_cache:/root/.cache/huggingface
-      - ../../ms_cache:/root/.cache/modelscope
-      - ../../om_cache:/root/.cache/openmind
-      - ../../data:/app/data
-      - ../../output:/app/output
       - /usr/local/dcmi:/usr/local/dcmi
       - /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
       - /usr/local/Ascend/driver:/usr/local/Ascend/driver
@@ -22,7 +17,7 @@ services:
       - "8000:8000"
     ipc: host
     tty: true
-    shm_size: '16gb'
+    # shm_size: "16gb"  # ipc: host is set
     stdin_open: true
     command: bash
     devices:

docker/docker-rocm/Dockerfile

@@ -1,65 +1,77 @@
-FROM hardandheavy/transformers-rocm:2.2.0
+# https://hub.docker.com/r/rocm/pytorch/tags
+ARG BASE_IMAGE=rocm/pytorch:rocm6.4.1_ubuntu22.04_py3.10_pytorch_release_2.6.0
+FROM ${BASE_IMAGE}
+
+# Installation arguments
+ARG PIP_INDEX=https://pypi.org/simple
+ARG EXTRAS=metrics
+ARG INSTALL_FLASHATTN=false
+ARG HTTP_PROXY=""
+ARG PYTORCH_INDEX=https://download.pytorch.org/whl/rocm6.3
 
 # Define environments
-ENV MAX_JOBS=4
+ENV MAX_JOBS=16
 ENV FLASH_ATTENTION_FORCE_BUILD=TRUE
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
+ENV DEBIAN_FRONTEND=noninteractive
+ENV NODE_OPTIONS=""
+ENV PIP_ROOT_USER_ACTION=ignore
+ENV http_proxy="${HTTP_PROXY}"
+ENV https_proxy="${HTTP_PROXY}"
 
-# Define installation arguments
-ARG INSTALL_BNB=false
-ARG INSTALL_VLLM=false
-ARG INSTALL_DEEPSPEED=false
-ARG INSTALL_FLASHATTN=false
-ARG INSTALL_LIGER_KERNEL=false
-ARG INSTALL_HQQ=false
-ARG PIP_INDEX=https://pypi.org/simple
+# Use Bash instead of default /bin/sh
+SHELL ["/bin/bash", "-c"]
 
 # Set the working directory
 WORKDIR /app
 
+# Change pip source
+RUN pip config set global.index-url "${PIP_INDEX}" && \
+    pip config set global.extra-index-url "${PIP_INDEX}" && \
+    pip install --no-cache-dir --upgrade pip packaging wheel setuptools
+
+# Reinstall pytorch rocm
+RUN pip uninstall -y torch torchvision torchaudio && \
+    pip install --no-cache-dir --pre torch torchvision torchaudio --index-url "${PYTORCH_INDEX}"
+
 # Install the requirements
 COPY requirements.txt /app
-RUN pip config set global.index-url "$PIP_INDEX" && \
-    pip config set global.extra-index-url "$PIP_INDEX" && \
-    python -m pip install --upgrade pip && \
-    python -m pip install -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
 # Copy the rest of the application into the image
 COPY . /app
 
-# Install the LLaMA Factory
-RUN EXTRA_PACKAGES="metrics"; \
-    if [ "$INSTALL_BNB" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},bitsandbytes"; \
-    fi; \
-    if [ "$INSTALL_VLLM" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},vllm"; \
-    fi; \
-    if [ "$INSTALL_DEEPSPEED" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},deepspeed"; \
-    fi; \
-    if [ "$INSTALL_LIGER_KERNEL" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},liger-kernel"; \
-    fi; \
-    if [ "$INSTALL_HQQ" == "true" ]; then \
-        EXTRA_PACKAGES="${EXTRA_PACKAGES},hqq"; \
-    fi; \
-    pip install -e ".[$EXTRA_PACKAGES]"
+# Install LLaMA Factory
+RUN pip install --no-cache-dir -e ".[${EXTRAS}]" --no-build-isolation
 
 # Rebuild flash attention
-RUN pip uninstall -y transformer-engine flash-attn && \
-    if [ "$INSTALL_FLASHATTN" == "true" ]; then \
-        pip uninstall -y ninja && pip install ninja && \
+RUN if [ "${INSTALL_FLASHATTN}" == "true" ]; then \
+        pip uninstall -y ninja && \
+        pip install --no-cache-dir ninja && \
         pip install --no-cache-dir flash-attn --no-build-isolation; \
     fi
 
 # Set up volumes
-VOLUME [ "/root/.cache/huggingface", "/root/.cache/modelscope", "/app/data", "/app/output" ]
+# VOLUME [ "/root/.cache/huggingface", "/app/shared_data", "/app/output" ]
 
-# Expose port 7860 for the LLaMA Board
-ENV GRADIO_SERVER_PORT 7860
+# Expose port 7860 for LLaMA Board
+ENV GRADIO_SERVER_PORT=7860
 EXPOSE 7860
 
-# Expose port 8000 for the API service
-ENV API_PORT 8000
+# Expose port 8000 for API service
+ENV API_PORT=8000
 EXPOSE 8000
+
+# unset proxy
+ENV http_proxy=
+ENV https_proxy=
+
+# Set no_proxy environment variable
+ENV no_proxy="localhost, 127.0.0.1, ::1"
+
+# fix pydantic version
+RUN pip install pydantic==2.10.6
+
+# Reset pip config
+RUN pip config unset global.index-url && \
+    pip config unset global.extra-index-url

docker/docker-rocm/docker-compose.yml

@@ -4,27 +4,15 @@ services:
       dockerfile: ./docker/docker-rocm/Dockerfile
       context: ../..
       args:
-        INSTALL_BNB: false
-        INSTALL_VLLM: false
-        INSTALL_DEEPSPEED: false
-        INSTALL_FLASHATTN: false
-        INSTALL_LIGER_KERNEL: false
-        INSTALL_HQQ: false
         PIP_INDEX: https://pypi.org/simple
+        EXTRAS: metrics
     container_name: llamafactory
-    volumes:
-      - ../../hf_cache:/root/.cache/huggingface
-      - ../../ms_cache:/root/.cache/modelscope
-      - ../../om_cache:/root/.cache/openmind
-      - ../../data:/app/data
-      - ../../output:/app/output
-      - ../../saves:/app/saves
     ports:
       - "7860:7860"
       - "8000:8000"
     ipc: host
     tty: true
-    shm_size: '16gb'
+    # shm_size: "16gb"  # ipc: host is set
     stdin_open: true
     command: bash
     devices:

evaluation/ceval/ceval.py

@@ -1,161 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-import datasets
-import pandas as pd
-
-
-_CITATION = """\
-@article{huang2023ceval,
-  title={C-Eval: A Multi-Level Multi-Discipline Chinese Evaluation Suite for Foundation Models},
-  author={Huang, Yuzhen and Bai, Yuzhuo and Zhu, Zhihao and Zhang, Junlei and Zhang, Jinghan and Su, Tangjun and Liu, Junteng and Lv, Chuancheng and Zhang, Yikai and Lei, Jiayi and Fu, Yao and Sun, Maosong and He, Junxian},
-  journal={arXiv preprint arXiv:2305.08322},
-  year={2023}
-}
-"""
-
-_DESCRIPTION = """\
-C-Eval is a comprehensive Chinese evaluation suite for foundation models. It consists of 13948 multi-choice questions spanning 52 diverse disciplines and four difficulty levels.
-"""
-
-_HOMEPAGE = "https://cevalbenchmark.com"
-
-_LICENSE = "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License"
-
-_URL = "ceval.zip"
-
-task_list = [
-    "computer_network",
-    "operating_system",
-    "computer_architecture",
-    "college_programming",
-    "college_physics",
-    "college_chemistry",
-    "advanced_mathematics",
-    "probability_and_statistics",
-    "discrete_mathematics",
-    "electrical_engineer",
-    "metrology_engineer",
-    "high_school_mathematics",
-    "high_school_physics",
-    "high_school_chemistry",
-    "high_school_biology",
-    "middle_school_mathematics",
-    "middle_school_biology",
-    "middle_school_physics",
-    "middle_school_chemistry",
-    "veterinary_medicine",
-    "college_economics",
-    "business_administration",
-    "marxism",
-    "mao_zedong_thought",
-    "education_science",
-    "teacher_qualification",
-    "high_school_politics",
-    "high_school_geography",
-    "middle_school_politics",
-    "middle_school_geography",
-    "modern_chinese_history",
-    "ideological_and_moral_cultivation",
-    "logic",
-    "law",
-    "chinese_language_and_literature",
-    "art_studies",
-    "professional_tour_guide",
-    "legal_professional",
-    "high_school_chinese",
-    "high_school_history",
-    "middle_school_history",
-    "civil_servant",
-    "sports_science",
-    "plant_protection",
-    "basic_medicine",
-    "clinical_medicine",
-    "urban_and_rural_planner",
-    "accountant",
-    "fire_engineer",
-    "environmental_impact_assessment_engineer",
-    "tax_accountant",
-    "physician",
-]
-
-
-class CevalConfig(datasets.BuilderConfig):
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version("1.0.0"), **kwargs)
-
-
-class Ceval(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [
-        CevalConfig(
-            name=task_name,
-        )
-        for task_name in task_list
-    ]
-
-    def _info(self):
-        features = datasets.Features(
-            {
-                "id": datasets.Value("int32"),
-                "question": datasets.Value("string"),
-                "A": datasets.Value("string"),
-                "B": datasets.Value("string"),
-                "C": datasets.Value("string"),
-                "D": datasets.Value("string"),
-                "answer": datasets.Value("string"),
-                "explanation": datasets.Value("string"),
-            }
-        )
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    "filepath": os.path.join(data_dir, "test", f"{task_name}_test.csv"),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                gen_kwargs={
-                    "filepath": os.path.join(data_dir, "val", f"{task_name}_val.csv"),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    "filepath": os.path.join(data_dir, "dev", f"{task_name}_dev.csv"),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_csv(filepath, encoding="utf-8")
-        for i, instance in enumerate(df.to_dict(orient="records")):
-            if "answer" not in instance.keys():
-                instance["answer"] = ""
-            if "explanation" not in instance.keys():
-                instance["explanation"] = ""
-            yield i, instance

evaluation/cmmlu/cmmlu.py

@@ -1,168 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-import datasets
-import pandas as pd
-
-
-_CITATION = """\
-@article{li2023cmmlu,
-  title={CMMLU: Measuring massive multitask language understanding in Chinese},
-  author={Haonan Li and Yixuan Zhang and Fajri Koto and Yifei Yang and Hai Zhao and Yeyun Gong and Nan Duan and Timothy Baldwin},
-  journal={arXiv preprint arXiv:2306.09212},
-  year={2023}
-}
-"""
-
-_DESCRIPTION = """\
-CMMLU is a comprehensive Chinese assessment suite specifically designed to evaluate the advanced knowledge and reasoning abilities of LLMs within the Chinese language and cultural context.
-"""
-
-_HOMEPAGE = "https://github.com/haonan-li/CMMLU"
-
-_LICENSE = "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License"
-
-_URL = "cmmlu.zip"
-
-task_list = [
-    "agronomy",
-    "anatomy",
-    "ancient_chinese",
-    "arts",
-    "astronomy",
-    "business_ethics",
-    "chinese_civil_service_exam",
-    "chinese_driving_rule",
-    "chinese_food_culture",
-    "chinese_foreign_policy",
-    "chinese_history",
-    "chinese_literature",
-    "chinese_teacher_qualification",
-    "clinical_knowledge",
-    "college_actuarial_science",
-    "college_education",
-    "college_engineering_hydrology",
-    "college_law",
-    "college_mathematics",
-    "college_medical_statistics",
-    "college_medicine",
-    "computer_science",
-    "computer_security",
-    "conceptual_physics",
-    "construction_project_management",
-    "economics",
-    "education",
-    "electrical_engineering",
-    "elementary_chinese",
-    "elementary_commonsense",
-    "elementary_information_and_technology",
-    "elementary_mathematics",
-    "ethnology",
-    "food_science",
-    "genetics",
-    "global_facts",
-    "high_school_biology",
-    "high_school_chemistry",
-    "high_school_geography",
-    "high_school_mathematics",
-    "high_school_physics",
-    "high_school_politics",
-    "human_sexuality",
-    "international_law",
-    "journalism",
-    "jurisprudence",
-    "legal_and_moral_basis",
-    "logical",
-    "machine_learning",
-    "management",
-    "marketing",
-    "marxist_theory",
-    "modern_chinese",
-    "nutrition",
-    "philosophy",
-    "professional_accounting",
-    "professional_law",
-    "professional_medicine",
-    "professional_psychology",
-    "public_relations",
-    "security_study",
-    "sociology",
-    "sports_science",
-    "traditional_chinese_medicine",
-    "virology",
-    "world_history",
-    "world_religions",
-]
-
-
-class CMMLUConfig(datasets.BuilderConfig):
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version("1.0.1"), **kwargs)
-
-
-class CMMLU(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [
-        CMMLUConfig(
-            name=task_name,
-        )
-        for task_name in task_list
-    ]
-
-    def _info(self):
-        features = datasets.Features(
-            {
-                "question": datasets.Value("string"),
-                "A": datasets.Value("string"),
-                "B": datasets.Value("string"),
-                "C": datasets.Value("string"),
-                "D": datasets.Value("string"),
-                "answer": datasets.Value("string"),
-            }
-        )
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    "filepath": os.path.join(data_dir, f"test/{task_name}.csv"),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    "filepath": os.path.join(data_dir, f"dev/{task_name}.csv"),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_csv(filepath, header=0, index_col=0, encoding="utf-8")
-        for i, instance in enumerate(df.to_dict(orient="records")):
-            question = instance.pop("Question", "")
-            answer = instance.pop("Answer", "")
-            instance["question"] = question
-            instance["answer"] = answer
-            yield i, instance

evaluation/mmlu/mmlu.py

@@ -1,161 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-import datasets
-import pandas as pd
-
-
-_CITATION = """\
-@article{hendryckstest2021,
-  title={Measuring Massive Multitask Language Understanding},
-  author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
-  journal={Proceedings of the International Conference on Learning Representations (ICLR)},
-  year={2021}
-}
-"""
-
-_DESCRIPTION = """\
-Measuring Massive Multitask Language Understanding by Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt (ICLR 2021).
-"""
-
-_HOMEPAGE = "https://github.com/hendrycks/test"
-
-_LICENSE = "MIT"
-
-_URL = "mmlu.zip"
-
-task_list = [
-    "high_school_european_history",
-    "business_ethics",
-    "clinical_knowledge",
-    "medical_genetics",
-    "high_school_us_history",
-    "high_school_physics",
-    "high_school_world_history",
-    "virology",
-    "high_school_microeconomics",
-    "econometrics",
-    "college_computer_science",
-    "high_school_biology",
-    "abstract_algebra",
-    "professional_accounting",
-    "philosophy",
-    "professional_medicine",
-    "nutrition",
-    "global_facts",
-    "machine_learning",
-    "security_studies",
-    "public_relations",
-    "professional_psychology",
-    "prehistory",
-    "anatomy",
-    "human_sexuality",
-    "college_medicine",
-    "high_school_government_and_politics",
-    "college_chemistry",
-    "logical_fallacies",
-    "high_school_geography",
-    "elementary_mathematics",
-    "human_aging",
-    "college_mathematics",
-    "high_school_psychology",
-    "formal_logic",
-    "high_school_statistics",
-    "international_law",
-    "high_school_mathematics",
-    "high_school_computer_science",
-    "conceptual_physics",
-    "miscellaneous",
-    "high_school_chemistry",
-    "marketing",
-    "professional_law",
-    "management",
-    "college_physics",
-    "jurisprudence",
-    "world_religions",
-    "sociology",
-    "us_foreign_policy",
-    "high_school_macroeconomics",
-    "computer_security",
-    "moral_scenarios",
-    "moral_disputes",
-    "electrical_engineering",
-    "astronomy",
-    "college_biology",
-]
-
-
-class MMLUConfig(datasets.BuilderConfig):
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version("1.0.0"), **kwargs)
-
-
-class MMLU(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [
-        MMLUConfig(
-            name=task_name,
-        )
-        for task_name in task_list
-    ]
-
-    def _info(self):
-        features = datasets.Features(
-            {
-                "question": datasets.Value("string"),
-                "A": datasets.Value("string"),
-                "B": datasets.Value("string"),
-                "C": datasets.Value("string"),
-                "D": datasets.Value("string"),
-                "answer": datasets.Value("string"),
-            }
-        )
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    "filepath": os.path.join(data_dir, "data", "test", f"{task_name}_test.csv"),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                gen_kwargs={
-                    "filepath": os.path.join(data_dir, "data", "val", f"{task_name}_val.csv"),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    "filepath": os.path.join(data_dir, "data", "dev", f"{task_name}_dev.csv"),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_csv(filepath, header=None)
-        df.columns = ["question", "A", "B", "C", "D", "answer"]
-
-        yield from enumerate(df.to_dict(orient="records"))

examples/README.md

@@ -13,6 +13,26 @@ Make sure to execute these commands in the `LLaMA-Factory` directory.
 
 Use `CUDA_VISIBLE_DEVICES` (GPU) or `ASCEND_RT_VISIBLE_DEVICES` (NPU) to choose computing devices.
 
+By default, LLaMA-Factory uses all visible computing devices.
+
+Basic usage:
+
+```bash
+llamafactory-cli train examples/train_lora/llama3_lora_sft.yaml
+```
+
+Advanced usage:
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 llamafactory-cli train examples/train_lora/llama3_lora_sft.yaml \
+    learning_rate=1e-5 \
+    logging_steps=1
+```
+
+```bash
+bash examples/train_lora/llama3_lora_sft.sh
+```
+
 ## Examples
 
 ### LoRA Fine-Tuning
@@ -32,8 +52,7 @@ llamafactory-cli train examples/train_lora/llama3_lora_sft.yaml
 #### Multimodal Supervised Fine-Tuning
 
 ```bash
-llamafactory-cli train examples/train_lora/llava1_5_lora_sft.yaml
-llamafactory-cli train examples/train_lora/qwen2vl_lora_sft.yaml
+llamafactory-cli train examples/train_lora/qwen2_5vl_lora_sft.yaml
 ```
 
 #### DPO/ORPO/SimPO Training
@@ -45,7 +64,7 @@ llamafactory-cli train examples/train_lora/llama3_lora_dpo.yaml
 #### Multimodal DPO/ORPO/SimPO Training
 
 ```bash
-llamafactory-cli train examples/train_lora/qwen2vl_lora_dpo.yaml
+llamafactory-cli train examples/train_lora/qwen2_5vl_lora_dpo.yaml
 ```
 
 #### Reward Modeling
@@ -80,12 +99,6 @@ llamafactory-cli train examples/train_lora/llama3_preprocess.yaml
 llamafactory-cli eval examples/train_lora/llama3_lora_eval.yaml
 ```
 
-#### Batch Predicting and Computing BLEU and ROUGE Scores
-
-```bash
-llamafactory-cli train examples/train_lora/llama3_lora_predict.yaml
-```
-
 #### Supervised Fine-Tuning on Multiple Nodes
 
 ```bash
@@ -99,6 +112,12 @@ FORCE_TORCHRUN=1 NNODES=2 NODE_RANK=1 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500
 FORCE_TORCHRUN=1 llamafactory-cli train examples/train_lora/llama3_lora_sft_ds3.yaml
 ```
 
+#### Supervised Fine-Tuning with Ray on 4 GPUs
+
+```bash
+USE_RAY=1 llamafactory-cli train examples/train_lora/llama3_lora_sft_ray.yaml
+```
+
 ### QLoRA Fine-Tuning
 
 #### Supervised Fine-Tuning with 4/8-bit Bitsandbytes/HQQ/EETQ Quantization (Recommended)
@@ -107,6 +126,12 @@ FORCE_TORCHRUN=1 llamafactory-cli train examples/train_lora/llama3_lora_sft_ds3.
 llamafactory-cli train examples/train_qlora/llama3_lora_sft_otfq.yaml
 ```
 
+#### Supervised Fine-Tuning with 4-bit Bitsandbytes Quantization on Ascend NPU
+
+```bash
+llamafactory-cli train examples/train_qlora/llama3_lora_sft_bnb_npu.yaml
+```
+
 #### Supervised Fine-Tuning with 4/8-bit GPTQ Quantization
 
 ```bash
@@ -130,26 +155,28 @@ llamafactory-cli train examples/train_qlora/llama3_lora_sft_aqlm.yaml
 #### Supervised Fine-Tuning on Single Node
 
 ```bash
-FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml
+FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/llama3_full_sft.yaml
 ```
 
 #### Supervised Fine-Tuning on Multiple Nodes
 
 ```bash
-FORCE_TORCHRUN=1 NNODES=2 RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml
-FORCE_TORCHRUN=1 NNODES=2 RANK=1 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml
+FORCE_TORCHRUN=1 NNODES=2 NODE_RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft.yaml
+FORCE_TORCHRUN=1 NNODES=2 NODE_RANK=1 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft.yaml
+```
+
+### Elastic and Fault-Tolerant Supervised Fine-Tuning on Multiple Nodes
+
+To launch an elastic job with `MAX_RESTARTS` failures retries, run the following on at least `MIN_NNODES` nodes and at most `MAX_NNODES` nodes. `RDZV_ID` should be set as a unique job id (shared by all nodes participating in the job). See also [torchrun](https://docs.pytorch.org/docs/stable/elastic/run.html).
+
+```bash
+FORCE_TORCHRUN=1 MIN_NNODES=1 MAX_NNODES=3 MAX_RESTARTS=3 RDZV_ID=llamafactory MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft.yaml
 ```
 
 #### Multimodal Supervised Fine-Tuning
 
 ```bash
-FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/qwen2vl_full_sft.yaml
-```
-
-#### Batch Predicting and Computing BLEU and ROUGE Scores
-
-```bash
-llamafactory-cli train examples/train_full/llama3_full_predict.yaml
+FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/qwen2_5vl_full_sft.yaml
 ```
 
 ### Merging LoRA Adapters and Quantization
@@ -168,15 +195,28 @@ llamafactory-cli export examples/merge_lora/llama3_lora_sft.yaml
 llamafactory-cli export examples/merge_lora/llama3_gptq.yaml
 ```
 
+### Save Ollama modelfile
+
+```bash
+llamafactory-cli export examples/merge_lora/llama3_full_sft.yaml
+```
+
 ### Inferring LoRA Fine-Tuned Models
 
-#### Use CLI
+#### Evaluation using vLLM's Multi-GPU Inference
+
+```
+python scripts/vllm_infer.py --model_name_or_path meta-llama/Meta-Llama-3-8B-Instruct --template llama3 --dataset alpaca_en_demo
+python scripts/eval_bleu_rouge.py generated_predictions.jsonl
+```
+
+#### Use CLI ChatBox
 
 ```bash
 llamafactory-cli chat examples/inference/llama3_lora_sft.yaml
 ```
 
-#### Use Web UI
+#### Use Web UI ChatBox
 
 ```bash
 llamafactory-cli webchat examples/inference/llama3_lora_sft.yaml
@@ -196,6 +236,12 @@ llamafactory-cli api examples/inference/llama3_lora_sft.yaml
 llamafactory-cli train examples/extras/galore/llama3_full_sft.yaml
 ```
 
+#### Full-Parameter Fine-Tuning using APOLLO
+
+```bash
+llamafactory-cli train examples/extras/apollo/llama3_full_sft.yaml
+```
+
 #### Full-Parameter Fine-Tuning using BAdam
 
 ```bash
@@ -208,6 +254,12 @@ llamafactory-cli train examples/extras/badam/llama3_full_sft.yaml
 llamafactory-cli train examples/extras/adam_mini/qwen2_full_sft.yaml
 ```
 
+#### Full-Parameter Fine-Tuning using Muon
+
+```bash
+llamafactory-cli train examples/extras/muon/qwen2_full_sft.yaml
+```
+
 #### LoRA+ Fine-Tuning
 
 ```bash
@@ -238,3 +290,15 @@ llamafactory-cli train examples/extras/llama_pro/llama3_freeze_sft.yaml
 ```bash
 bash examples/extras/fsdp_qlora/train.sh
 ```
+
+#### OFT Fine-Tuning
+
+```bash
+llamafactory-cli train examples/extras/oft/llama3_oft_sft.yaml
+```
+
+#### QOFT Fine-Tuning
+
+```bash
+llamafactory-cli train examples/extras/qoft/llama3_oft_sft_bnb_npu.yaml
+```

examples/README_zh.md

@@ -13,6 +13,26 @@
 
 使用 `CUDA_VISIBLE_DEVICES`（GPU）或 `ASCEND_RT_VISIBLE_DEVICES`（NPU）选择计算设备。
 
+LLaMA-Factory 默认使用所有可见的计算设备。
+
+基础用法：
+
+```bash
+llamafactory-cli train examples/train_lora/llama3_lora_sft.yaml
+```
+
+高级用法：
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 llamafactory-cli train examples/train_lora/llama3_lora_sft.yaml \
+    learning_rate=1e-5 \
+    logging_steps=1
+```
+
+```bash
+bash examples/train_lora/llama3_lora_sft.sh
+```
+
 ## 示例
 
 ### LoRA 微调
@@ -32,8 +52,7 @@ llamafactory-cli train examples/train_lora/llama3_lora_sft.yaml
 #### 多模态指令监督微调
 
 ```bash
-llamafactory-cli train examples/train_lora/llava1_5_lora_sft.yaml
-llamafactory-cli train examples/train_lora/qwen2vl_lora_sft.yaml
+llamafactory-cli train examples/train_lora/qwen2_5vl_lora_sft.yaml
 ```
 
 #### DPO/ORPO/SimPO 训练
@@ -45,7 +64,7 @@ llamafactory-cli train examples/train_lora/llama3_lora_dpo.yaml
 #### 多模态 DPO/ORPO/SimPO 训练
 
 ```bash
-llamafactory-cli train examples/train_lora/qwen2vl_lora_dpo.yaml
+llamafactory-cli train examples/train_lora/qwen2_5vl_lora_dpo.yaml
 ```
 
 #### 奖励模型训练
@@ -80,12 +99,6 @@ llamafactory-cli train examples/train_lora/llama3_preprocess.yaml
 llamafactory-cli eval examples/train_lora/llama3_lora_eval.yaml
 ```
 
-#### 批量预测并计算 BLEU 和 ROUGE 分数
-
-```bash
-llamafactory-cli train examples/train_lora/llama3_lora_predict.yaml
-```
-
 #### 多机指令监督微调
 
 ```bash
@@ -93,12 +106,26 @@ FORCE_TORCHRUN=1 NNODES=2 NODE_RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500
 FORCE_TORCHRUN=1 NNODES=2 NODE_RANK=1 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_lora/llama3_lora_sft.yaml
 ```
 
+### 支持弹性和容错的多机指令监督微调
+
+要启动一个支持弹性节点和容错的多机指令微调，在每个节点上执行以下命令。弹性节点数量范围为 `MIN_NNODES:MAX_NNODES`，每个节点最多允许因为错误重启 `MAX_RESTARTS` 次。`RDZV_ID` 应设置为一个唯一的作业 ID（由参与该作业的所有节点共享）。更多新可以参考官方文档 [torchrun](https://docs.pytorch.org/docs/stable/elastic/run.html)。
+
+```bash
+FORCE_TORCHRUN=1 MIN_NNODES=1 MAX_NNODES=3 MAX_RESTARTS=3 RDZV_ID=llamafactory MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft.yaml
+```
+
 #### 使用 DeepSpeed ZeRO-3 平均分配显存
 
 ```bash
 FORCE_TORCHRUN=1 llamafactory-cli train examples/train_lora/llama3_lora_sft_ds3.yaml
 ```
 
+#### 使用 Ray 在 4 张 GPU 上微调
+
+```bash
+USE_RAY=1 llamafactory-cli train examples/train_lora/llama3_lora_sft_ray.yaml
+```
+
 ### QLoRA 微调
 
 #### 基于 4/8 比特 Bitsandbytes/HQQ/EETQ 量化进行指令监督微调（推荐）
@@ -107,6 +134,12 @@ FORCE_TORCHRUN=1 llamafactory-cli train examples/train_lora/llama3_lora_sft_ds3.
 llamafactory-cli train examples/train_qlora/llama3_lora_sft_otfq.yaml
 ```
 
+#### 在 NPU 上基于 4 比特 Bitsandbytes 量化进行指令监督微调
+
+```bash
+llamafactory-cli train examples/train_qlora/llama3_lora_sft_bnb_npu.yaml
+```
+
 #### 基于 4/8 比特 GPTQ 量化进行指令监督微调
 
 ```bash
@@ -130,26 +163,20 @@ llamafactory-cli train examples/train_qlora/llama3_lora_sft_aqlm.yaml
 #### 在单机上进行指令监督微调
 
 ```bash
-FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml
+FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/llama3_full_sft.yaml
 ```
 
 #### 在多机上进行指令监督微调
 
 ```bash
-FORCE_TORCHRUN=1 NNODES=2 RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml
-FORCE_TORCHRUN=1 NNODES=2 RANK=1 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml
+FORCE_TORCHRUN=1 NNODES=2 NODE_RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft.yaml
+FORCE_TORCHRUN=1 NNODES=2 NODE_RANK=1 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft.yaml
 ```
 
 #### 多模态指令监督微调
 
 ```bash
-FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/qwen2vl_full_sft.yaml
-```
-
-#### 批量预测并计算 BLEU 和 ROUGE 分数
-
-```bash
-llamafactory-cli train examples/train_full/llama3_full_predict.yaml
+FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/qwen2_5vl_full_sft.yaml
 ```
 
 ### 合并 LoRA 适配器与模型量化
@@ -168,15 +195,28 @@ llamafactory-cli export examples/merge_lora/llama3_lora_sft.yaml
 llamafactory-cli export examples/merge_lora/llama3_gptq.yaml
 ```
 
+### 保存 Ollama 配置文件
+
+```bash
+llamafactory-cli export examples/merge_lora/llama3_full_sft.yaml
+```
+
 ### 推理 LoRA 模型
 
-#### 使用命令行接口
+#### 使用 vLLM 多卡推理评估
+
+```
+python scripts/vllm_infer.py --model_name_or_path meta-llama/Meta-Llama-3-8B-Instruct --template llama3 --dataset alpaca_en_demo
+python scripts/eval_bleu_rouge.py generated_predictions.jsonl
+```
+
+#### 使用命令行对话框
 
 ```bash
 llamafactory-cli chat examples/inference/llama3_lora_sft.yaml
 ```
 
-#### 使用浏览器界面
+#### 使用浏览器对话框
 
 ```bash
 llamafactory-cli webchat examples/inference/llama3_lora_sft.yaml
@@ -196,6 +236,12 @@ llamafactory-cli api examples/inference/llama3_lora_sft.yaml
 llamafactory-cli train examples/extras/galore/llama3_full_sft.yaml
 ```
 
+#### 使用 APOLLO 进行全参数训练
+
+```bash
+llamafactory-cli train examples/extras/apollo/llama3_full_sft.yaml
+```
+
 #### 使用 BAdam 进行全参数训练
 
 ```bash
@@ -208,6 +254,12 @@ llamafactory-cli train examples/extras/badam/llama3_full_sft.yaml
 llamafactory-cli train examples/extras/adam_mini/qwen2_full_sft.yaml
 ```
 
+#### 使用 Muon 进行全参数训练
+
+```bash
+llamafactory-cli train examples/extras/muon/qwen2_full_sft.yaml
+```
+
 #### LoRA+ 微调
 
 ```bash
@@ -238,3 +290,15 @@ llamafactory-cli train examples/extras/llama_pro/llama3_freeze_sft.yaml
 ```bash
 bash examples/extras/fsdp_qlora/train.sh
 ```
+
+#### OFT 微调
+
+```bash
+llamafactory-cli train examples/extras/oft/llama3_oft_sft.yaml
+```
+
+#### QOFT 微调
+
+```bash
+llamafactory-cli train examples/extras/qoft/llama3_oft_sft_bnb_npu.yaml
+```

examples/accelerate/fsdp_config.yaml

@@ -7,16 +7,16 @@ fsdp_config:
   fsdp_backward_prefetch: BACKWARD_PRE
   fsdp_forward_prefetch: false
   fsdp_cpu_ram_efficient_loading: true
-  fsdp_offload_params: true # offload may affect training speed
+  fsdp_offload_params: false
   fsdp_sharding_strategy: FULL_SHARD
   fsdp_state_dict_type: FULL_STATE_DICT
   fsdp_sync_module_states: true
   fsdp_use_orig_params: true
 machine_rank: 0
 main_training_function: main
-mixed_precision: fp16 # or bf16
-num_machines: 1 # the number of nodes
-num_processes: 2 # the number of GPUs in all nodes
+mixed_precision: bf16  # or fp16
+num_machines: 1  # the number of nodes
+num_processes: 2  # the number of GPUs in all nodes
 rdzv_backend: static
 same_network: true
 tpu_env: []

examples/accelerate/fsdp_config_offload.yaml

@@ -0,0 +1,25 @@
+compute_environment: LOCAL_MACHINE
+debug: false
+distributed_type: FSDP
+downcast_bf16: 'no'
+fsdp_config:
+  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
+  fsdp_backward_prefetch: BACKWARD_PRE
+  fsdp_forward_prefetch: false
+  fsdp_cpu_ram_efficient_loading: true
+  fsdp_offload_params: true  # offload may affect training speed
+  fsdp_sharding_strategy: FULL_SHARD
+  fsdp_state_dict_type: FULL_STATE_DICT
+  fsdp_sync_module_states: true
+  fsdp_use_orig_params: true
+machine_rank: 0
+main_training_function: main
+mixed_precision: bf16  # or fp16
+num_machines: 1  # the number of nodes
+num_processes: 2  # the number of GPUs in all nodes
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

examples/deepspeed/ds_z0_config.json

@@ -0,0 +1,28 @@
+{
+  "train_batch_size": "auto",
+  "train_micro_batch_size_per_gpu": "auto",
+  "gradient_accumulation_steps": "auto",
+  "gradient_clipping": "auto",
+  "zero_allow_untested_optimizer": true,
+  "fp16": {
+    "enabled": "auto",
+    "loss_scale": 0,
+    "loss_scale_window": 1000,
+    "initial_scale_power": 16,
+    "hysteresis": 2,
+    "min_loss_scale": 1
+  },
+  "bf16": {
+    "enabled": "auto"
+  },
+  "zero_optimization": {
+    "stage": 0,
+    "allgather_partitions": true,
+    "allgather_bucket_size": 5e8,
+    "overlap_comm": false,
+    "reduce_scatter": true,
+    "reduce_bucket_size": 5e8,
+    "contiguous_gradients": true,
+    "round_robin_gradients": true
+  }
+}

examples/deepspeed/ds_z2_config.json

@@ -0,0 +1,28 @@
+{
+  "train_batch_size": "auto",
+  "train_micro_batch_size_per_gpu": "auto",
+  "gradient_accumulation_steps": "auto",
+  "gradient_clipping": "auto",
+  "zero_allow_untested_optimizer": true,
+  "fp16": {
+    "enabled": "auto",
+    "loss_scale": 0,
+    "loss_scale_window": 1000,
+    "initial_scale_power": 16,
+    "hysteresis": 2,
+    "min_loss_scale": 1
+  },
+  "bf16": {
+    "enabled": "auto"
+  },
+  "zero_optimization": {
+    "stage": 2,
+    "allgather_partitions": true,
+    "allgather_bucket_size": 5e8,
+    "overlap_comm": false,
+    "reduce_scatter": true,
+    "reduce_bucket_size": 5e8,
+    "contiguous_gradients": true,
+    "round_robin_gradients": true
+  }
+}

examples/deepspeed/ds_z2_offload_config.json

@@ -0,0 +1,32 @@
+{
+  "train_batch_size": "auto",
+  "train_micro_batch_size_per_gpu": "auto",
+  "gradient_accumulation_steps": "auto",
+  "gradient_clipping": "auto",
+  "zero_allow_untested_optimizer": true,
+  "fp16": {
+    "enabled": "auto",
+    "loss_scale": 0,
+    "loss_scale_window": 1000,
+    "initial_scale_power": 16,
+    "hysteresis": 2,
+    "min_loss_scale": 1
+  },
+  "bf16": {
+    "enabled": "auto"
+  },
+  "zero_optimization": {
+    "stage": 2,
+    "offload_optimizer": {
+      "device": "cpu",
+      "pin_memory": true
+    },
+    "allgather_partitions": true,
+    "allgather_bucket_size": 5e8,
+    "overlap_comm": false,
+    "reduce_scatter": true,
+    "reduce_bucket_size": 5e8,
+    "contiguous_gradients": true,
+    "round_robin_gradients": true
+  }
+}

examples/deepspeed/ds_z3_config.json

@@ -0,0 +1,30 @@
+{
+  "train_batch_size": "auto",
+  "train_micro_batch_size_per_gpu": "auto",
+  "gradient_accumulation_steps": "auto",
+  "gradient_clipping": "auto",
+  "zero_allow_untested_optimizer": true,
+  "fp16": {
+    "enabled": "auto",
+    "loss_scale": 0,
+    "loss_scale_window": 1000,
+    "initial_scale_power": 16,
+    "hysteresis": 2,
+    "min_loss_scale": 1
+  },
+  "bf16": {
+    "enabled": "auto"
+  },
+  "zero_optimization": {
+    "stage": 3,
+    "overlap_comm": false,
+    "contiguous_gradients": true,
+    "sub_group_size": 1e9,
+    "reduce_bucket_size": "auto",
+    "stage3_prefetch_bucket_size": "auto",
+    "stage3_param_persistence_threshold": "auto",
+    "stage3_max_live_parameters": 1e9,
+    "stage3_max_reuse_distance": 1e9,
+    "stage3_gather_16bit_weights_on_model_save": true
+  }
+}

examples/deepspeed/ds_z3_fp8_config.json

@@ -0,0 +1,45 @@
+{
+  "train_micro_batch_size_per_gpu": "auto",
+  "gradient_clipping": "auto",
+  "zero_allow_untested_optimizer": true,
+  "zero_force_ds_cpu_optimizer": true,
+  "fp16": {
+    "enabled": false,
+    "loss_scale": 0,
+    "loss_scale_window": 1000,
+    "initial_scale_power": 16,
+    "hysteresis": 2,
+    "min_loss_scale": 1
+  },
+  "bf16": {
+    "enabled": "auto"
+  },
+  "zero_optimization": {
+    "stage": 3,
+    "offload_optimizer": {
+      "device": "cpu",
+      "pin_memory": false
+    },
+    "overlap_comm": false,
+    "contiguous_gradients": true,
+    "sub_group_size": 1000000000,
+    "reduce_bucket_size": 12845056,
+    "stage3_prefetch_bucket_size": 11560550,
+    "stage3_param_persistence_threshold": 35840,
+    "stage3_max_live_parameters": 1000000000,
+    "stage3_max_reuse_distance": 1000000000,
+    "stage3_gather_16bit_weights_on_model_save": true
+  },
+  "steps_per_print": 10000000,
+  "gradient_accumulation_steps": "auto",
+  "comms_config": {
+    "verbose": false
+  },
+  "monitor_config": {
+    "enabled": true,
+    "tag": "DeepSpeedMonitor",
+    "csv_monitor": {
+      "enabled": false
+    }
+  }
+}

examples/deepspeed/ds_z3_offload_config.json

@@ -0,0 +1,38 @@
+{
+  "train_batch_size": "auto",
+  "train_micro_batch_size_per_gpu": "auto",
+  "gradient_accumulation_steps": "auto",
+  "gradient_clipping": "auto",
+  "zero_allow_untested_optimizer": true,
+  "fp16": {
+    "enabled": "auto",
+    "loss_scale": 0,
+    "loss_scale_window": 1000,
+    "initial_scale_power": 16,
+    "hysteresis": 2,
+    "min_loss_scale": 1
+  },
+  "bf16": {
+    "enabled": "auto"
+  },
+  "zero_optimization": {
+    "stage": 3,
+    "offload_optimizer": {
+      "device": "cpu",
+      "pin_memory": true
+    },
+    "offload_param": {
+      "device": "cpu",
+      "pin_memory": true
+    },
+    "overlap_comm": false,
+    "contiguous_gradients": true,
+    "sub_group_size": 1e9,
+    "reduce_bucket_size": "auto",
+    "stage3_prefetch_bucket_size": "auto",
+    "stage3_param_persistence_threshold": "auto",
+    "stage3_max_live_parameters": 1e9,
+    "stage3_max_reuse_distance": 1e9,
+    "stage3_gather_16bit_weights_on_model_save": true
+  }
+}

examples/extras/adam_mini/qwen2_full_sft.yaml

@@ -1,5 +1,6 @@
 ### model
 model_name_or_path: Qwen/Qwen2-1.5B-Instruct
+trust_remote_code: true
 
 ### method
 stage: sft
@@ -14,6 +15,7 @@ cutoff_len: 2048
 max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
 output_dir: saves/qwen2-1_5b/full/sft
@@ -21,6 +23,8 @@ logging_steps: 10
 save_steps: 500
 plot_loss: true
 overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### train
 per_device_train_batch_size: 1
@@ -33,7 +37,7 @@ bf16: true
 ddp_timeout: 180000000
 
 ### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-eval_strategy: steps
-eval_steps: 500
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/apollo/llama3_full_sft.yaml

@@ -0,0 +1,48 @@
+### model
+model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: full
+use_apollo: true
+apollo_layerwise: true  # choices: [true, false], use false for DDP training
+apollo_target: all
+apollo_rank: 128
+apollo_scale: 32.0
+apollo_scale_type: channel
+
+### dataset
+dataset: identity,alpaca_en_demo
+template: llama3
+cutoff_len: 2048
+max_samples: 1000
+overwrite_cache: true
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+
+### output
+output_dir: saves/llama3-8b/full/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1  # use 1 for layerwise apollo
+learning_rate: 1.0e-5
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+pure_bf16: true
+ddp_timeout: 180000000
+
+### eval
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/badam/llama3_full_sft.yaml

@@ -1,5 +1,6 @@
 ### model
 model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
+trust_remote_code: true
 
 ### method
 stage: sft
@@ -19,6 +20,7 @@ cutoff_len: 2048
 max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
 output_dir: saves/llama3-8b/full/sft
@@ -26,6 +28,8 @@ logging_steps: 10
 save_steps: 500
 plot_loss: true
 overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### train
 per_device_train_batch_size: 1
@@ -36,7 +40,7 @@ lr_scheduler_type: cosine
 warmup_ratio: 0.1
 
 ### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-eval_strategy: steps
-eval_steps: 500
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/dft/qwen2_full_sft.yaml

@@ -0,0 +1,43 @@
+### model
+model_name_or_path: Qwen/Qwen2-1.5B-Instruct
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: full
+use_dft_loss: true
+
+### dataset
+dataset: identity,alpaca_en_demo
+template: qwen
+cutoff_len: 2048
+max_samples: 1000
+overwrite_cache: true
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+
+### output
+output_dir: saves/qwen2-1_5b/full/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-5
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### eval
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/fp8/llama3_fp8_deepspeed_sft.yaml

@@ -0,0 +1,48 @@
+# FP8 training example with DeepSpeed ZeRO-3
+# This config demonstrates FP8 mixed precision training using HuggingFace Accelerate
+# with DeepSpeed providing memory optimization (not FP8 handling)
+
+### Model configuration
+model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
+trust_remote_code: true
+
+### Method configuration
+stage: sft
+do_train: true
+finetuning_type: full
+
+### Dataset configuration
+dataset: identity
+template: llama3
+cutoff_len: 1024
+max_samples: 1000
+overwrite_cache: true
+preprocessing_num_workers: 16
+
+### Output configuration
+output_dir: saves/llama3-8b/fp8-deepspeed/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+
+### Training configuration
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 5.0e-5
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+
+### FP8 configuration
+fp8: true
+fp8_backend: torchao  # Use TorchAO backend for FP8
+fp8_enable_fsdp_float8_all_gather: false  # Not used with DeepSpeed
+
+### DeepSpeed configuration
+deepspeed: examples/deepspeed/ds_z3_fp8_config.json
+
+### Logging configuration
+report_to: wandb
+run_name: llama3_fp8_deepspeed_sft

examples/extras/fp8/llama3_fp8_fsdp_sft.yaml

@@ -0,0 +1,51 @@
+# FP8 training example with FSDP
+# This config demonstrates FP8 mixed precision training using HuggingFace Accelerate
+# with FSDP for distributed training and float8 all-gather optimization
+
+### Model configuration
+model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
+trust_remote_code: true
+
+### Method configuration
+stage: sft
+do_train: true
+finetuning_type: full
+
+### Dataset configuration
+dataset: identity
+template: llama3
+cutoff_len: 1024
+max_samples: 1000
+overwrite_cache: true
+preprocessing_num_workers: 16
+
+### Output configuration
+output_dir: saves/llama3-8b/fp8-fsdp/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+
+### Training configuration
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 5.0e-5
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+
+### FP8 configuration
+fp8: true
+fp8_backend: torchao  # Use TorchAO backend for FP8
+fp8_enable_fsdp_float8_all_gather: true  # Enable FSDP2 float8 all-gather optimization
+
+### FSDP configuration (using training arguments - no separate FSDP config file)
+fsdp:
+  - full_shard
+  - auto_wrap
+fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
+
+### Logging configuration
+report_to: wandb
+run_name: llama3_fp8_fsdp_sft

examples/extras/fsdp_qlora/llama3_lora_sft.yaml

@@ -1,11 +1,13 @@
 ### model
 model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
 quantization_bit: 4
+trust_remote_code: true
 
 ### method
 stage: sft
 do_train: true
 finetuning_type: lora
+lora_rank: 8
 lora_target: all
 
 ### dataset
@@ -15,6 +17,7 @@ cutoff_len: 2048
 max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
 output_dir: saves/llama3-8b/lora/sft
@@ -22,6 +25,8 @@ logging_steps: 10
 save_steps: 500
 plot_loss: true
 overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### train
 per_device_train_batch_size: 1
@@ -34,7 +39,7 @@ bf16: true
 ddp_timeout: 180000000
 
 ### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-eval_strategy: steps
-eval_steps: 500
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/galore/llama3_full_sft.yaml

@@ -1,13 +1,14 @@
 ### model
 model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
+trust_remote_code: true
 
 ### method
 stage: sft
 do_train: true
 finetuning_type: full
 use_galore: true
-galore_layerwise: true
-galore_target: mlp,self_attn
+galore_layerwise: true  # choices: [true, false], use false for DDP training
+galore_target: all
 galore_rank: 128
 galore_scale: 2.0
 
@@ -18,6 +19,7 @@ cutoff_len: 2048
 max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
 output_dir: saves/llama3-8b/full/sft
@@ -25,10 +27,12 @@ logging_steps: 10
 save_steps: 500
 plot_loss: true
 overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### train
 per_device_train_batch_size: 1
-gradient_accumulation_steps: 1
+gradient_accumulation_steps: 1  # use 1 for layerwise galore
 learning_rate: 1.0e-5
 num_train_epochs: 3.0
 lr_scheduler_type: cosine
@@ -37,7 +41,7 @@ pure_bf16: true
 ddp_timeout: 180000000
 
 ### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-eval_strategy: steps
-eval_steps: 500
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/llama_pro/llama3_freeze_sft.yaml

@@ -1,5 +1,6 @@
 ### model
 model_name_or_path: models/llama3-8b-pro
+trust_remote_code: true
 
 ### method
 stage: sft
@@ -16,6 +17,7 @@ cutoff_len: 2048
 max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
 output_dir: saves/llama3-8b-pro/freeze/sft
@@ -23,6 +25,8 @@ logging_steps: 10
 save_steps: 500
 plot_loss: true
 overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### train
 per_device_train_batch_size: 1
@@ -35,7 +39,7 @@ bf16: true
 ddp_timeout: 180000000
 
 ### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-eval_strategy: steps
-eval_steps: 500
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/loraplus/llama3_lora_sft.yaml

@@ -1,10 +1,12 @@
 ### model
 model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
+trust_remote_code: true
 
 ### method
 stage: sft
 do_train: true
 finetuning_type: lora
+lora_rank: 8
 lora_target: all
 loraplus_lr_ratio: 16.0
 
@@ -15,6 +17,7 @@ cutoff_len: 2048
 max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
 output_dir: saves/llama3-8b/lora/sft
@@ -22,6 +25,8 @@ logging_steps: 10
 save_steps: 500
 plot_loss: true
 overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### train
 per_device_train_batch_size: 1
@@ -34,7 +39,7 @@ bf16: true
 ddp_timeout: 180000000
 
 ### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-eval_strategy: steps
-eval_steps: 500
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/mod/llama3_full_sft.yaml

@@ -1,5 +1,6 @@
 ### model
 model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
+trust_remote_code: true
 
 ### method
 stage: sft
@@ -14,6 +15,7 @@ cutoff_len: 2048
 max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
 output_dir: saves/llama3-8b-mod/full/sft
@@ -21,6 +23,8 @@ logging_steps: 10
 save_steps: 500
 plot_loss: true
 overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### train
 per_device_train_batch_size: 1
@@ -34,7 +38,7 @@ pure_bf16: true
 ddp_timeout: 180000000
 
 ### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-eval_strategy: steps
-eval_steps: 500
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/muon/qwen2_full_sft.yaml

@@ -1,30 +1,34 @@
 ### model
-model_name_or_path: Qwen/Qwen2-VL-7B-Instruct
+model_name_or_path: Qwen/Qwen2-1.5B-Instruct
+trust_remote_code: true
 
 ### method
 stage: sft
 do_train: true
 finetuning_type: full
-deepspeed: examples/deepspeed/ds_z3_config.json
+use_muon: true
 
 ### dataset
-dataset: mllm_demo,identity
-template: qwen2_vl
+dataset: identity,alpaca_en_demo
+template: qwen
 cutoff_len: 2048
 max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
-output_dir: saves/qwen2_vl-7b/full/sft
+output_dir: saves/qwen2-1_5b/full/sft
 logging_steps: 10
 save_steps: 500
 plot_loss: true
 overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### train
 per_device_train_batch_size: 1
-gradient_accumulation_steps: 2
+gradient_accumulation_steps: 8
 learning_rate: 1.0e-5
 num_train_epochs: 3.0
 lr_scheduler_type: cosine
@@ -33,7 +37,7 @@ bf16: true
 ddp_timeout: 180000000
 
 ### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-eval_strategy: steps
-eval_steps: 500
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/nlg_eval/llama3_lora_predict.yaml

@@ -1,6 +1,10 @@
+# The batch generation can be SLOW using this config.
+# For faster inference, we recommend to use `scripts/vllm_infer.py`.
+
 ### model
 model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
 adapter_name_or_path: saves/llama3-8b/lora/sft
+trust_remote_code: true
 
 ### method
 stage: sft
@@ -14,10 +18,12 @@ cutoff_len: 2048
 max_samples: 50
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
 output_dir: saves/llama3-8b/lora/predict
 overwrite_output_dir: true
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### eval
 per_device_eval_batch_size: 1

examples/extras/oft/llama3_oft_sft.yaml

@@ -0,0 +1,46 @@
+### model
+model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: oft
+oft_block_size: 32
+oft_target: all
+
+### dataset
+dataset: identity,alpaca_en_demo
+template: llama3
+cutoff_len: 2048
+max_samples: 1000
+overwrite_cache: true
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+
+### output
+output_dir: saves/llama3-8b/oft/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/oft/qwen2_5vl_oft_sft.yaml

@@ -0,0 +1,47 @@
+### model
+model_name_or_path: Qwen/Qwen2.5-VL-7B-Instruct
+image_max_pixels: 262144
+video_max_pixels: 16384
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: oft
+oft_block_size: 32
+oft_target: all
+
+### dataset
+dataset: mllm_demo,identity,alpaca_en_demo  # video: mllm_video_demo
+template: qwen2_vl
+cutoff_len: 2048
+max_samples: 1000
+overwrite_cache: true
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+
+### output
+output_dir: saves/qwen2_5vl-7b/oft/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+
+### eval
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/pissa/llama3_lora_sft.yaml

@@ -1,10 +1,12 @@
 ### model
 model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
+trust_remote_code: true
 
 ### method
 stage: sft
 do_train: true
 finetuning_type: lora
+lora_rank: 8
 lora_target: all
 pissa_init: true
 pissa_iter: 16
@@ -17,6 +19,7 @@ cutoff_len: 2048
 max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
 output_dir: saves/llama3-8b/lora/sft
@@ -24,6 +27,8 @@ logging_steps: 10
 save_steps: 500
 plot_loss: true
 overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### train
 per_device_train_batch_size: 1
@@ -36,7 +41,7 @@ bf16: true
 ddp_timeout: 180000000
 
 ### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-eval_strategy: steps
-eval_steps: 500
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/qoft/llama3_oft_sft_awq.yaml

@@ -1,26 +1,31 @@
 ### model
-model_name_or_path: llava-hf/llava-1.5-7b-hf
+model_name_or_path: TechxGenus/Meta-Llama-3-8B-Instruct-AWQ
+trust_remote_code: true
 
 ### method
 stage: sft
 do_train: true
-finetuning_type: lora
-lora_target: all
+finetuning_type: oft
+oft_block_size: 32
+oft_target: all
 
 ### dataset
-dataset: mllm_demo
-template: llava
+dataset: identity,alpaca_en_demo
+template: llama3
 cutoff_len: 2048
 max_samples: 1000
 overwrite_cache: true
 preprocessing_num_workers: 16
+dataloader_num_workers: 4
 
 ### output
-output_dir: saves/llava1_5-7b/lora/sft
+output_dir: saves/llama3-8b/oft/sft
 logging_steps: 10
 save_steps: 500
 plot_loss: true
 overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
 
 ### train
 per_device_train_batch_size: 1
@@ -33,7 +38,7 @@ bf16: true
 ddp_timeout: 180000000
 
 ### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-eval_strategy: steps
-eval_steps: 500
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/qoft/llama3_oft_sft_bnb_npu.yaml

@@ -0,0 +1,47 @@
+### model
+model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
+quantization_bit: 4
+quantization_method: bnb
+double_quantization: false
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: oft
+oft_block_size: 32
+oft_target: all
+
+### dataset
+dataset: identity,alpaca_en_demo
+template: llama3
+cutoff_len: 2048
+max_samples: 1000
+overwrite_cache: true
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+
+### output
+output_dir: saves/llama3-8b/oft/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### eval
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/extras/qoft/llama3_oft_sft_gptq.yaml

@@ -0,0 +1,44 @@
+### model
+model_name_or_path: TechxGenus/Meta-Llama-3-8B-Instruct-GPTQ
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: oft
+oft_block_size: 32
+oft_target: all
+
+### dataset
+dataset: identity,alpaca_en_demo
+template: llama3
+cutoff_len: 2048
+max_samples: 1000
+overwrite_cache: true
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+
+### output
+output_dir: saves/llama3-8b/oft/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### eval
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/inference/llama3.yaml

@@ -1,2 +1,4 @@
 model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
 template: llama3
+infer_backend: huggingface  # choices: [huggingface, vllm, sglang]
+trust_remote_code: true

examples/inference/llama3_full_sft.yaml

@@ -0,0 +1,4 @@
+model_name_or_path: saves/llama3-8b/full/sft
+template: llama3
+infer_backend: huggingface  # choices: [huggingface, vllm, sglang]
+trust_remote_code: true

examples/inference/llama3_lora_sft.yaml

@@ -1,4 +1,5 @@
 model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
 adapter_name_or_path: saves/llama3-8b/lora/sft
 template: llama3
-finetuning_type: lora
+infer_backend: huggingface  # choices: [huggingface, vllm, sglang]
+trust_remote_code: true

examples/inference/llama3_vllm.yaml

@@ -1,4 +0,0 @@
-model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
-template: llama3
-infer_backend: vllm
-vllm_enforce_eager: true

examples/inference/llava1_5.yaml

@@ -1,2 +0,0 @@
-model_name_or_path: llava-hf/llava-1.5-7b-hf
-template: llava

examples/inference/qwen2_5vl.yaml

@@ -0,0 +1,4 @@
+model_name_or_path: Qwen/Qwen2.5-VL-7B-Instruct
+template: qwen2_vl
+infer_backend: huggingface  # choices: [huggingface, vllm, sglang]
+trust_remote_code: true

examples/inference/qwen2_vl.yaml

@@ -1,2 +0,0 @@
-model_name_or_path: Qwen/Qwen2-VL-7B-Instruct
-template: qwen2_vl

examples/merge_lora/llama3_full_sft.yaml

@@ -0,0 +1,10 @@
+### model
+model_name_or_path: saves/llama3-8b/full/sft
+template: llama3
+trust_remote_code: true
+
+### export
+export_dir: output/llama3_full_sft
+export_size: 5
+export_device: cpu  # choices: [cpu, auto]
+export_legacy_format: false

examples/merge_lora/llama3_gptq.yaml

@@ -1,11 +1,12 @@
 ### model
 model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
 template: llama3
+trust_remote_code: true
 
 ### export
-export_dir: models/llama3_gptq
+export_dir: output/llama3_gptq
 export_quantization_bit: 4
-export_quantization_dataset: data/c4_demo.json
-export_size: 2
-export_device: cpu
+export_quantization_dataset: data/c4_demo.jsonl
+export_size: 5
+export_device: cpu  # choices: [cpu, auto]
 export_legacy_format: false