mira/unsloth-train-scripts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: _

Browse Source
This commit is contained in:
Mira
2025-02-15 05:41:51 +06:00
parent 9af06f6382
commit 281f54df25
5 changed files with 28 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
main.py
View File

@@ -86,14 +86,16 @@ def run_sweep(base_config: TrainingConfig, dataset_path: str):
"parameters": {
"learning_rate": {
"distribution": "log_uniform_values",
"min": 1e-5,
"max": 1e-4,
"min": 1e-7,
"max": 1e-5,
},
"lora_r": {"values": [8]},
"lora_alpha": {"values": [16]},
"per_device_train_batch_size": {"values": [16]},
"gradient_accumulation_steps": {"values": [2]},
"lora_r": {"values": [32]},
"lora_alpha": {"values": [64]},
"per_device_train_batch_size": {"values": [32]},
"gradient_accumulation_steps": {"values": [4, 8]},
"num_train_epochs": {"values": [1]},
"warmup_steps": {"values": [10]},
"max_grad_norm": {"values": [0.1, 0.3, 0.5]},
},
"early_terminate": {"type": "hyperband", "min_iter": 100},
}