chore: script

config.py

@@ -48,7 +48,7 @@ class TrainingConfig:
             "down_proj",
         ]
     )
-    use_gradient_checkpointing: str = True
+    use_gradient_checkpointing: str = "unsloth"
     random_state: int = 3407
     use_rslora: bool = False
     loftq_config: dict | None = None
@@ -75,8 +75,8 @@ class TrainingConfig:
 
     # eval
     fp16_full_eval: bool = True
-    per_device_eval_batch_size: int = 64
+    per_device_eval_batch_size: int = 8
-    eval_accumulation_steps: int = 1
+    eval_accumulation_steps: int = 2
     eval_strategy: str = "steps"
     eval_steps: int = 10
 

main.py

@@ -89,12 +89,13 @@ def run_sweep(base_config: TrainingConfig, dataset_path: str):
                 "min": 1e-5,
                 "max": 1e-4,
             },
-            "lora_r": {"values": [32]},
+            "lora_r": {"values": [8]},
-            "lora_alpha": {"values": [64]},
+            "lora_alpha": {"values": [16]},
-            "per_device_train_batch_size": {"values": [64]},
+            "per_device_train_batch_size": {"values": [16]},
-            "gradient_accumulation_steps": {"values": [1]},
+            "gradient_accumulation_steps": {"values": [2]},
             "num_train_epochs": {"values": [1]},
         },
+        "early_terminate": {"type": "hyperband", "min_iter": 100},
     }
 
     sweep_id = wandb.sweep(sweep_config, project=base_config.wandb.project)

setup.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -e
+
+check_command() {
+    if [ $? -ne 0 ]; then
+        echo "Error: $1 failed"
+        exit 1
+    fi
+}
+
+cd /workspace || {
+    echo "Error: Cannot change to /workspace directory"
+    exit 1
+}
+
+echo "Cloning unsloth-train-scripts repository..."
+git clone https://git.hye.su/mira/unsloth-train-scripts.git
+check_command "Git clone"
+
+echo "Downloading and running unsloth auto-install script..."
+INSTALL_CMD=$(wget -qO- https://raw.githubusercontent.com/unslothai/unsloth/main/unsloth/_auto_install.py | python -)
+check_command "Auto-install script"
+
+echo "Executing installation command..."
+eval "$INSTALL_CMD"
+check_command "Installation command"
+
+echo "Installing additional dependencies..."
+pip install gdown wandb
+check_command "pip install"
+
+echo "Downloading dataset"
+gdown --fuzzy 'https://drive.google.com/file/d/1mqhq69dsSOK7ep7trTjRd3FMagTFTrzF/view?usp=sharing'
+check_command "gdown"
+
+echo "Enter your Weights & Biases API key when prompted"
+wandb login
+check_command "wandb login"
+
+echo "Setup completed successfully."