add multinode launcher script

Author: chencyudel

mftcoder_accelerate/src/ds_multinode_launch.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+# Author: Chaoyu Chen
+# Last Modified: 2024/5/20
+# Description: # Launch script on Multiple Nodes
+
+# Run this script on all Nodes.
+
+# You need to export your number of nodes and number of GPUs per node first.
+N_NODE=4
+N_GPU_PER_NODE=8
+
+# You need to export $RANK, $MASTER_ADDR, $MASTER_PORT automatically for each Node.
+
+# config path
+CONFIG="configs/xxx_train_config.json"
+
+# envs used inside training
+export OMP_NUM_THREADS=4
+export TOKENIZERS_PARALLELISM=False
+
+TODAY=$(date +%Y-%m%d-%H%M)
+
+# accelerate launch --config_file accelerate_ds_config.yaml \
+accelerate launch \
+    --num_machines $N_NODE \
+    --num_processes $(($N_NODE*$N_GPU_PER_NODE)) \
+    --use_deepspeed \
+    --deepspeed_multinode_launcher 'standard' \
+    --zero_stage 2 \
+    --offload_optimizer_device 'cpu' \
+    --offload_param_device 'none' \
+    --gradient_accumulation_steps 1 \
+    --gradient_clipping 1.0 \
+    --zero3_init_flag false \
+    --zero3_save_16bit_model false \
+    --main_training_function 'main' \
+    --mixed_precision 'bf16' \
+    --dynamo_backend 'no' \
+    --same_network \
+    --machine_rank $RANK \
+    --main_process_ip $MASTER_ADDR \
+    --main_process_port $MASTER_PORT \
+    --rdzv_backend 'static' \
+    pefts/mft_accelerate.py --train_config "$CONFIG" --distributed_type "deepspeed"

mftcoder_accelerate/src/pefts/mft_accelerate.py

@@ -1,6 +1,6 @@
 """
 # @author Chaoyu Chen
-# @date 2023/12/11
+# @date 2024/5/20
 # @module mft_accelerate.py
 
 Accelerate + DeepSpeed zero2/zero3/FSDP + Data Parallelism

mftcoder_accelerate/src/pefts/model_mapping.py

@@ -1,6 +1,6 @@
 """
  # @author Chaoyu Chen
- # @date 2023/12/11
+ # @date 2024/5/20
 
  Manage supported models and their special token used in training.
  Default targeting modules for LoRA/QLora