mirror of
https://github.com/hiyouga/LLaMA-Factory.git
synced 2025-08-03 20:22:49 +08:00
fix: mixtral output_router_logits
Former-commit-id: 9f4fe623866b10b30c6418dee116b36671274f9f
This commit is contained in:
parent
db42378f29
commit
11f4afc5ad
@ -316,7 +316,7 @@ def patch_config(
|
||||
if getattr(config, "model_type", None) == "qwen2" and is_trainable and model_args.flash_attn:
|
||||
setattr(config, "use_cache", False) # qwen2 does not support use_cache when using flashattn
|
||||
|
||||
if getattr(config, "model_type", None) == "qwen2_moe" and is_trainable:
|
||||
if getattr(config, "model_type", None) in ["mixtral", "qwen2_moe"] and is_trainable:
|
||||
setattr(config, "output_router_logits", True)
|
||||
|
||||
init_kwargs["torch_dtype"] = model_args.compute_dtype
|
||||
|
Loading…
x
Reference in New Issue
Block a user