From c3cd2067b2b7430d1601b562ef4e5790c6264fde Mon Sep 17 00:00:00 2001 From: hiyouga Date: Wed, 28 Jun 2023 01:40:13 +0800 Subject: [PATCH] fix RM accuracy Former-commit-id: 7826a8ca7722b138e79b13c42b1070771f6d5994 --- src/utils/pairwise.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/utils/pairwise.py b/src/utils/pairwise.py index 57729793..9bac1594 100644 --- a/src/utils/pairwise.py +++ b/src/utils/pairwise.py @@ -13,8 +13,7 @@ logger = get_logger(__name__) def compute_accuracy(eval_preds: Sequence[Union[np.ndarray, Tuple[np.ndarray]]]) -> Dict[str, float]: preds, _ = eval_preds - preds = np.array(preds) - return {"accuracy": (preds[:, 0] > preds[:, 1]).sum() / len(preds)} + return {"accuracy": (preds[0] > preds[1]).sum() / len(preds[0])} class PairwiseDataCollatorWithPadding(DynamicDataCollatorWithPadding): @@ -49,9 +48,13 @@ class PairwisePeftTrainer(PeftTrainer): We use score on the EOS token to represent reward of the whole sentence. Subclass and override to inject custom behavior. It should not be directly used by external scripts. + + Note that the first element will be removed from the output tuple. + + See: https://github.com/huggingface/transformers/blob/v4.30.2/src/transformers/trainer.py#L3509 """ batch_size = inputs["input_ids"].size(0) // 2 _, _, values = model(**inputs) r_accept, r_reject = values[:, -1].split(batch_size, dim=0) loss = -torch.log(torch.sigmoid(r_accept - r_reject)).mean() - return (loss, torch.stack((r_accept, r_reject), dim=-1)) if return_outputs else loss + return (loss, [loss, r_accept, r_reject]) if return_outputs else loss