utensil.tngl.sh
Utensil's LLM Playground (2023)
utensil.tngl.sh/llm-playground/
Terminate the pod when training goes wrong and notify discord
author
Utensil
committer
GitHub
date
(Jul 26, 2023, 12:19 PM UTC)
commit
31bb9dca
31bb9dca0e0a516c34941fc04a780a4f21fc5d28
parent
da741056
da7410560f359ea048ac521086b7729d79dd3564
+71
-44
+43
-39
helper/setup-runner.py
+43
-39
helper/setup-runner.py
···
164
164
if runpod_cfg.debug:
165
165
os.environ["RUNPOD_DEBUG"] = 'true'
166
166
logging.info(f"Debug mode enabled")
167
-
168
-
if runpod_cfg.pod_type == 'INTERRUPTABLE':
169
-
pod = runpod.create_spot_pod(f'Training {config}',
170
-
AXOLOTL_RUNPOD_IMAGE,
171
-
gpu,
172
-
cloud_type=runpod_cfg.cloud_type or "SECURE",
173
-
bid_per_gpu=bid_per_gpu,
174
-
template_id=runpod_cfg.template_id or DEFAULT_TEMPLATE_ID,
175
-
container_disk_in_gb=runpod_cfg.container_disk_in_gb or 50,
176
-
volume_in_gb=runpod_cfg.volume_in_gb or 200,
177
-
gpu_count=runpod_cfg.gpu_count or 1,
178
-
min_vcpu_count=runpod_cfg.min_vcpu_count or 8,
179
-
min_memory_in_gb=runpod_cfg.min_memory_in_gb or 29,
180
-
min_download=runpod_cfg.min_download or None,
181
-
min_upload=runpod_cfg.min_upload or None,
182
-
docker_args=entry,
183
-
env=env,
184
-
stop_after=stop_after,
185
-
terminate_after=terminate_after
186
-
)
187
-
else:
188
-
pod = runpod.create_pod(f'Training {config}',
189
-
AXOLOTL_RUNPOD_IMAGE,
190
-
gpu,
191
-
cloud_type=runpod_cfg.cloud_type or "SECURE",
192
-
template_id=runpod_cfg.template_id or DEFAULT_TEMPLATE_ID,
193
-
container_disk_in_gb=runpod_cfg.container_disk_in_gb or 50,
194
-
volume_in_gb=runpod_cfg.volume_in_gb or 200,
195
-
gpu_count=runpod_cfg.gpu_count or 1,
196
-
min_vcpu_count=runpod_cfg.min_vcpu_count or 8,
197
-
min_memory_in_gb=runpod_cfg.min_memory_in_gb or 29,
198
-
min_download=runpod_cfg.min_download or None,
199
-
min_upload=runpod_cfg.min_upload or None,
200
-
docker_args=entry,
201
-
env=env,
202
-
stop_after=stop_after,
203
-
terminate_after=terminate_after
204
-
)
205
-
167
+
168
+
try:
169
+
if runpod_cfg.pod_type == 'INTERRUPTABLE':
170
+
pod = runpod.create_spot_pod(f'Training {config}',
171
+
AXOLOTL_RUNPOD_IMAGE,
172
+
gpu,
173
+
cloud_type=runpod_cfg.cloud_type or "SECURE",
174
+
bid_per_gpu=bid_per_gpu,
175
+
template_id=runpod_cfg.template_id or DEFAULT_TEMPLATE_ID,
176
+
container_disk_in_gb=runpod_cfg.container_disk_in_gb or 50,
177
+
volume_in_gb=runpod_cfg.volume_in_gb or 200,
178
+
gpu_count=runpod_cfg.gpu_count or 1,
179
+
min_vcpu_count=runpod_cfg.min_vcpu_count or 8,
180
+
min_memory_in_gb=runpod_cfg.min_memory_in_gb or 29,
181
+
min_download=runpod_cfg.min_download or None,
182
+
min_upload=runpod_cfg.min_upload or None,
183
+
docker_args=entry,
184
+
env=env,
185
+
stop_after=stop_after,
186
+
terminate_after=terminate_after
187
+
)
188
+
else:
189
+
pod = runpod.create_pod(f'Training {config}',
190
+
AXOLOTL_RUNPOD_IMAGE,
191
+
gpu,
192
+
cloud_type=runpod_cfg.cloud_type or "SECURE",
193
+
template_id=runpod_cfg.template_id or DEFAULT_TEMPLATE_ID,
194
+
container_disk_in_gb=runpod_cfg.container_disk_in_gb or 50,
195
+
volume_in_gb=runpod_cfg.volume_in_gb or 200,
196
+
gpu_count=runpod_cfg.gpu_count or 1,
197
+
min_vcpu_count=runpod_cfg.min_vcpu_count or 8,
198
+
min_memory_in_gb=runpod_cfg.min_memory_in_gb or 29,
199
+
min_download=runpod_cfg.min_download or None,
200
+
min_upload=runpod_cfg.min_upload or None,
201
+
docker_args=entry,
202
+
env=env,
203
+
stop_after=stop_after,
204
+
terminate_after=terminate_after
205
+
)
206
+
207
+
except Exception as ex:
208
+
log_error(f"Failed to create pod for {config}", exc_info=ex)
209
+
sys.exit(0)
206
210
207
211
if pod is None:
208
212
log_error(f"Failed to create pod for {config}")
+23
helper/terminate-runpod.py
+23
helper/terminate-runpod.py
···
1
+
import os
2
+
import runpod
3
+
from discord import SyncWebhook
4
+
import logging
5
+
6
+
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
7
+
8
+
# TODO: avoid code dup
9
+
def notify_discord(msg):
10
+
webhook = SyncWebhook.from_url(os.getenv("DISCORD_WEBHOOK_URL"))
11
+
webhook.send(msg)
12
+
13
+
def log_info(msg):
14
+
logging.info(msg)
15
+
notify_discord(msg)
16
+
17
+
runpod.api_key = os.getenv("RUNPOD_API_KEY")
18
+
19
+
pod_id = os.getenv("RUNPOD_POD_ID")
20
+
21
+
log_info(f"Pod {pod_id} terminated on train end")
22
+
23
+
runpod.terminate_pod(pod_id)
+2
-2
helper/train.py
+2
-2
helper/train.py
···
121
121
122
122
pod_id = os.getenv("RUNPOD_POD_ID")
123
123
124
-
runpod.terminate_pod(pod_id)
125
-
126
124
log_info(f"Pod {pod_id} terminated on train end")
125
+
126
+
runpod.terminate_pod(pod_id)
127
127
128
128
def log_data(name, data, tokenizer):
129
129
# logging.info(f'{name}(type={type(data)}, shape={data.shape}):\n{data}')
+2
-2
scripts/entry/ax_lite_train.sh
+2
-2
scripts/entry/ax_lite_train.sh
···
88
88
# Optional: for deepspeed
89
89
pip3 install -U torch --index-url https://download.pytorch.org/whl/cu118
90
90
cd /content/llm-playground
91
-
(python /content/llm-playground/helper/download-model.py $PREDOWNLOAD_MODEL || true)
92
-
(accelerate launch helper/train.py $TRAINING_CONFIG || true)
91
+
(python /content/llm-playground/helper/download-model.py $PREDOWNLOAD_MODEL || true )
92
+
(accelerate launch helper/train.py $TRAINING_CONFIG || python /content/llm-playground/helper/terminate-runpod.py )
93
93
fi
94
94
95
95
sleep infinity