refactor dataset_attr, add eos in pt, fix #757

src/llmtuner/extras/template.py

@@ -137,6 +137,8 @@ class Template:
         token_ids = []
         for elem in context:
             if isinstance(elem, str):
+                if len(elem) == 0:
+                    continue
                 elem = elem.replace("{{system}}", system, 1) if system is not None else elem
                 elem = elem.replace("{{query}}", query, 1) if query is not None else elem
                 elem = elem.replace("{{idx}}", idx, 1) if idx is not None else elem