From 542c8e0120ca51151060b6b867e1ef247c8fb2c7 Mon Sep 17 00:00:00 2001
From: kuwoyuki <kuwoyuki@cock.li>
Date: Sat, 15 Feb 2025 01:53:31 +0600
Subject: [PATCH] chore: script

---
 config.py |  6 +++---
 main.py   |  9 +++++----
 setup.sh  | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 7 deletions(-)
 create mode 100755 setup.sh

diff --git a/config.py b/config.py
index 8f56a0e..2947bc1 100644
--- a/config.py
+++ b/config.py
@@ -48,7 +48,7 @@ class TrainingConfig:
             "down_proj",
         ]
     )
-    use_gradient_checkpointing: str = True
+    use_gradient_checkpointing: str = "unsloth"
     random_state: int = 3407
     use_rslora: bool = False
     loftq_config: dict | None = None
@@ -75,8 +75,8 @@ class TrainingConfig:
 
     # eval
     fp16_full_eval: bool = True
-    per_device_eval_batch_size: int = 64
-    eval_accumulation_steps: int = 1
+    per_device_eval_batch_size: int = 8
+    eval_accumulation_steps: int = 2
     eval_strategy: str = "steps"
     eval_steps: int = 10
 
diff --git a/main.py b/main.py
index 3c05ea8..6df47ea 100644
--- a/main.py
+++ b/main.py
@@ -89,12 +89,13 @@ def run_sweep(base_config: TrainingConfig, dataset_path: str):
                 "min": 1e-5,
                 "max": 1e-4,
             },
-            "lora_r": {"values": [32]},
-            "lora_alpha": {"values": [64]},
-            "per_device_train_batch_size": {"values": [64]},
-            "gradient_accumulation_steps": {"values": [1]},
+            "lora_r": {"values": [8]},
+            "lora_alpha": {"values": [16]},
+            "per_device_train_batch_size": {"values": [16]},
+            "gradient_accumulation_steps": {"values": [2]},
             "num_train_epochs": {"values": [1]},
         },
+        "early_terminate": {"type": "hyperband", "min_iter": 100},
     }
 
     sweep_id = wandb.sweep(sweep_config, project=base_config.wandb.project)
diff --git a/setup.sh b/setup.sh
new file mode 100755
index 0000000..a175510
--- /dev/null
+++ b/setup.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -e
+
+check_command() {
+    if [ $? -ne 0 ]; then
+        echo "Error: $1 failed"
+        exit 1
+    fi
+}
+
+cd /workspace || {
+    echo "Error: Cannot change to /workspace directory"
+    exit 1
+}
+
+echo "Cloning unsloth-train-scripts repository..."
+git clone https://git.hye.su/mira/unsloth-train-scripts.git
+check_command "Git clone"
+
+echo "Downloading and running unsloth auto-install script..."
+INSTALL_CMD=$(wget -qO- https://raw.githubusercontent.com/unslothai/unsloth/main/unsloth/_auto_install.py | python -)
+check_command "Auto-install script"
+
+echo "Executing installation command..."
+eval "$INSTALL_CMD"
+check_command "Installation command"
+
+echo "Installing additional dependencies..."
+pip install gdown wandb
+check_command "pip install"
+
+echo "Downloading dataset"
+gdown --fuzzy 'https://drive.google.com/file/d/1mqhq69dsSOK7ep7trTjRd3FMagTFTrzF/view?usp=sharing'
+check_command "gdown"
+
+echo "Enter your Weights & Biases API key when prompted"
+wandb login
+check_command "wandb login"
+
+echo "Setup completed successfully."