[train] Harden KTransformers MoE LoRA SFT integration (#10738)

examples/ktransformers/accelerate/fsdp2_kt_bf16.yaml

@@ -13,13 +13,3 @@ num_processes: 4 # Adjust based on your GPU count; 4 is suitable for 4 GPUs
 rdzv_backend: static
 same_network: true
 use_cpu: false
-
-kt_config:
-  enabled: true
-  kt_backend: AMXBF16  # Use with original BF16 expert weights.
-  kt_num_threads: 96
-  kt_tp_enabled: true
-  kt_threadpool_count: 2
-  kt_max_cache_depth: 2
-  kt_share_backward_bb: true
-  lora_rank: 8

examples/ktransformers/accelerate/fsdp2_kt_int4.yaml

@@ -13,13 +13,3 @@ num_processes: 4 # Adjust based on your GPU count; 4 is suitable for 4 GPUs
 rdzv_backend: static
 same_network: true
 use_cpu: false
-
-kt_config:
-  enabled: true
-  kt_backend: AMXINT4  # Use with online-converted INT4 expert weights
-  kt_num_threads: 96
-  kt_tp_enabled: true
-  kt_threadpool_count: 2
-  kt_max_cache_depth: 2
-  kt_share_backward_bb: true
-  lora_rank: 8

examples/ktransformers/accelerate/fsdp2_kt_int8.yaml

@@ -13,13 +13,3 @@ num_processes: 4 # Adjust based on your GPU count; 4 is suitable for 4 GPUs
 rdzv_backend: static
 same_network: true
 use_cpu: false
-
-kt_config:
-  enabled: true
-  kt_backend: AMXINT8  # Use with online-converted INT8 expert weights
-  kt_num_threads: 96
-  kt_tp_enabled: true
-  kt_threadpool_count: 2
-  kt_max_cache_depth: 2
-  kt_share_backward_bb: true
-  lora_rank: 8

examples/ktransformers/accelerate/fsdp2_kt_int8_1gpu.yaml

@@ -13,13 +13,3 @@ num_processes: 1 # Adjust based on your GPU count; 1 is suitable for 1 GPU
 rdzv_backend: static
 same_network: true
 use_cpu: false
-
-kt_config:
-  enabled: true
-  kt_backend: AMXINT8  # Use with online-converted INT8 expert weights
-  kt_num_threads: 96
-  kt_tp_enabled: true
-  kt_threadpool_count: 2
-  kt_max_cache_depth: 2
-  kt_share_backward_bb: true
-  lora_rank: 8

examples/ktransformers/accelerate/fsdp2_kt_int8_8gpu.yaml

@@ -13,13 +13,3 @@ num_processes: 8 # Adjust based on your GPU count; 8 is suitable for 8 GPUs
 rdzv_backend: static
 same_network: true
 use_cpu: false
-
-kt_config:
-  enabled: true
-  kt_backend: AMXINT8  # Use with online-converted INT8 expert weights
-  kt_num_threads: 96
-  kt_tp_enabled: true
-  kt_threadpool_count: 2
-  kt_max_cache_depth: 2
-  kt_share_backward_bb: true
-  lora_rank: 8

examples/ktransformers/train_lora/deepseek_v3_int8_lora_sft_kt.yaml

@@ -0,0 +1,54 @@
+### model
+model_name_or_path: /path/to/DeepSeek-V3.1-source
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_rank: 8
+lora_alpha: 16
+lora_target: all
+
+### dataset
+dataset: identity, alpaca_en_demo
+template: deepseek3
+cutoff_len: 2048
+max_samples: 100000
+overwrite_cache: true
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+
+### output
+output_dir: saves/KT_FT_deepseekV3_int8
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 1.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### ktransformers
+use_kt: true
+kt_cpu_activation: retain
+kt_weight_path: /path/to/routed-int8-experts
+kt_non_expert_weight_path: /path/to/bf16-non-expert-cache
+kt_config:
+  kt_expert_weight_format: int8
+  kt_backend: auto
+  kt_weight_lifecycle: persistent
+  kt_num_threads: 96
+  kt_tp_enabled: true
+  kt_threadpool_count: 2
+  kt_max_cache_depth: 2
+  kt_share_backward_bb: true

examples/ktransformers/train_lora/deepseek_v3_lora_sft_kt.yaml

@@ -40,6 +40,13 @@ resume_from_checkpoint: null
 
 ### ktransformers
 use_kt: true
-# Pair with fsdp2_kt_bf16.yaml for original BF16 checkpoints.
-# For pre-converted expert weights, uncomment kt_weight_path and use fsdp2_kt_int8.yaml or fsdp2_kt_int4.yaml.
-# kt_weight_path: /path/to/DeepSeek-V3-AMXINT8
+kt_cpu_activation: retain
+kt_config:
+  kt_expert_weight_format: bf16
+  kt_backend: AMXBF16
+  kt_num_threads: 96
+  kt_tp_enabled: true
+  kt_threadpool_count: 2
+  kt_max_cache_depth: 2
+  kt_share_backward_bb: true
+# The Accelerate YAML contains FSDP settings only. KT has a single configuration owner here.

examples/ktransformers/train_lora/qwen3_5moe_lora_sft_kt.yaml

@@ -40,7 +40,14 @@ resume_from_checkpoint: null
 
 ### ktransformers
 use_kt: true
-# For original BF16 checkpoints, start with examples/ktransformers/accelerate/fsdp2_kt_bf16.yaml.
-# For pre-converted expert weights, uncomment kt_weight_path and use fsdp2_kt_int8.yaml or fsdp2_kt_int4.yaml.
-# Pair the 397B path with fsdp2_kt_int8.yaml, tune cutoff_len to prepared weights and GPU memory.
-# kt_weight_path: /path/to/Qwen3.5-MoE-AMXINT8
+kt_cpu_activation: retain
+kt_config:
+  kt_expert_weight_format: bf16
+  kt_backend: AMXBF16
+  kt_num_threads: 96
+  kt_tp_enabled: true
+  kt_threadpool_count: 2
+  kt_max_cache_depth: 2
+  kt_model_max_length: 2176  # Includes the text-only template's dummy-image tokens.
+  kt_share_backward_bb: true
+# The Accelerate YAML contains FSDP settings only. KT has a single configuration owner here.