From 61320965aad81930b013faf0995ca7fea8beebea Mon Sep 17 00:00:00 2001
From: hiyouga <hiyouga@buaa.edu.cn>
Date: Wed, 27 Nov 2024 05:24:58 +0000
Subject: [PATCH] pin tokenizers version

Former-commit-id: b7d4cf2caf2d02f7d16bab3f0ec8bf0108b7be75
---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 445d7b87..b47643e3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,6 +3,7 @@ datasets>=2.16.0,<=3.1.0
 accelerate>=0.34.0,<=1.0.1
 peft>=0.11.1,<=0.12.0
 trl>=0.8.6,<=0.9.6
+tokenizers>=0.19.0,<0.20.4
 gradio>=4.0.0,<5.0.0
 pandas>=2.0.0
 scipy