From 99a54d06ca3f3ed403b96a16485b670abe41bf17 Mon Sep 17 00:00:00 2001
From: hiyouga <hiyouga@buaa.edu.cn>
Date: Wed, 27 Nov 2024 05:24:58 +0000
Subject: [PATCH] pin tokenizers version

Former-commit-id: 2b747737f0be2caeb737fe87dad6bf5902b4a588
---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 445d7b87..b47643e3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,6 +3,7 @@ datasets>=2.16.0,<=3.1.0
 accelerate>=0.34.0,<=1.0.1
 peft>=0.11.1,<=0.12.0
 trl>=0.8.6,<=0.9.6
+tokenizers>=0.19.0,<0.20.4
 gradio>=4.0.0,<5.0.0
 pandas>=2.0.0
 scipy