423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-16 00:28:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

add hint

Browse Source 
Former-commit-id: c540ef41bda61993b83ef8cfe3c84b1d169e984c
This commit is contained in:
hiyouga 2024-01-22 23:32:01 +08:00
parent 3f3756b113
commit 7a048fc91d
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/model/patcher.py
View File

@ -286,7 +286,7 @@ def patch_model(
if getattr(model.config, "model_type", None) == "mixtral" and is_deepspeed_zero3_enabled(): if getattr(model.config, "model_type", None) == "mixtral" and is_deepspeed_zero3_enabled():
require_version("deepspeed>=0.13.0", "To fix: pip install deepspeed>=0.13.0") require_version("deepspeed>=0.13.0", "To fix: pip install deepspeed>=0.13.0")
from deepspeed.utils import set_z3_leaf_modules from deepspeed.utils import set_z3_leaf_modules # type: ignore
from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
set_z3_leaf_modules(model, [MixtralSparseMoeBlock]) set_z3_leaf_modules(model, [MixtralSparseMoeBlock])