423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-25 07:12:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

tiny fix

Browse Source 
Former-commit-id: f8d8690bf4c2981f3151b4ccf07daeb4f3cd38a9
This commit is contained in:
hiyouga 2024-06-07 05:19:21 +08:00
parent 4f3c89a6eb
commit ca9468ff04
1 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/llamafactory/train/ppo/ppo_utils.py
View File

@ -32,10 +32,11 @@ def replace_model(model: "AutoModelForCausalLMWithValueHead", target: Literal["d
r""" r"""
Replaces the default/reward modules in the model. The model is already unwrapped. Replaces the default/reward modules in the model. The model is already unwrapped.
""" """
v_head_layer = model.v_head.summary
if is_deepspeed_zero3_enabled(): if is_deepspeed_zero3_enabled():
import deepspeed # type: ignore import deepspeed # type: ignore
params = [model.v_head.summary.weight, model.v_head.summary.bias] params = [v_head_layer.weight, v_head_layer.bias]
context_maybe_zero3 = deepspeed.zero.GatheredParameters(params, modifier_rank=0) context_maybe_zero3 = deepspeed.zero.GatheredParameters(params, modifier_rank=0)
else: else:
context_maybe_zero3 = nullcontext() context_maybe_zero3 = nullcontext()
@ -43,14 +44,12 @@ def replace_model(model: "AutoModelForCausalLMWithValueHead", target: Literal["d
model.pretrained_model.set_adapter(target) # set the LoRA adapter to be active model.pretrained_model.set_adapter(target) # set the LoRA adapter to be active
with context_maybe_zero3: with context_maybe_zero3:
if target == "reward": # save default head temporarily if target == "reward": # save default head temporarily
setattr(model, "default_head_weight", model.v_head.summary.weight.data.detach().clone()) setattr(model, "default_head_weight", v_head_layer.weight.data.detach().clone())
setattr(model, "default_head_bias", model.v_head.summary.bias.data.detach().clone()) setattr(model, "default_head_bias", v_head_layer.bias.data.detach().clone())
device = model.v_head.summary.weight.device device = v_head_layer.weight.device
model.v_head.summary.weight.data = ( v_head_layer.weight.data = model.get_buffer("{}_head_weight".format(target)).detach().clone().to(device)
model.get_buffer("{}_head_weight".format(target)).detach().clone().to(device) v_head_layer.bias.data = model.get_buffer("{}_head_bias".format(target)).detach().clone().to(device)
)
model.v_head.summary.bias.data = model.get_buffer("{}_head_bias".format(target)).detach().clone().to(device)
def dump_layernorm(model: "PreTrainedModel") -> Dict[str, torch.Tensor]: def dump_layernorm(model: "PreTrainedModel") -> Dict[str, torch.Tensor]: