improve KTO impl., replace datasets

Former-commit-id: c450ee87a3
2025-12-14 10:56:56 +08:00 · 2024-05-18 03:44:56 +08:00
parent 97469892c3
commit 13d7b48efe
66 changed files with 46444 additions and 28125 deletions
--- a/data/hh_rlhf_en/hh_rlhf_en.py
+++ b/data/hh_rlhf_en/hh_rlhf_en.py
@@ -79,5 +79,5 @@ class HhRlhfEn(datasets.GeneratorBasedBuilder):
                            break
                        prompt = prompt[:human_idx]

-                    yield key, {"instruction": query, "output": [r_accept, r_reject], "history": history}
+                    yield key, {"instruction": query, "chosen": r_accept, "rejected": r_reject, "history": history}
                    key += 1