reformat

Author: chencyudel

mftcoder_accelerate/README.md

@@ -176,10 +176,14 @@ DeepSpeed config in accelerate_ds_config.yaml.
 accelerate launch --config_file accelerate_ds_config.yaml pefts/mft_accelerate.py --train_config configs/xxx_train_config.json --distributed_type "DeepSpeed" 
 ```
 or
-DeepSpeed config in command line arguments
+DeepSpeed Zero2 config in command line arguments
 ```bash
 sh ds_single_launch.sh
 ```
+DeepSpeed Zero3 config in command line arguments
+```bash
+sh ds_zero3_single_launch.sh
+```
 
 #### Launch via FSDP
 FSDP config in accelerate_fsdp_config.yaml.

mftcoder_accelerate/README_cn.md

@@ -153,11 +153,16 @@ accelerate launch --config_file accelerate_ds_config.yaml pefts/mft_accelerate.p
 ```
 或者
 
-DeepSpeed配置在脚本中通过命令行输入。
+DeepSpeed Zero2 配置在脚本中通过命令行输入。
 ```bash
 sh ds_single_launch.sh
 ```
 
+DeepSpeed Zero3 配置在脚本中通过命令行输入
+```bash
+sh ds_zero3_single_launch.sh
+```
+
 #### Launch via FSDP
 FSDP配置在accelerate_fsdp_config.yaml中。
 ```bash

mftcoder_accelerate/src/ds_single_launch.sh

@@ -6,6 +6,9 @@
 # Launch script on single node
 N_GPU_PER_NODE=8
 
+# config path
+CONFIG="configs/xxx_train_config.json"
+
 # envs used inside training
 export OMP_NUM_THREADS=4
 export TOKENIZERS_PARALLELISM=False
@@ -30,6 +33,6 @@ accelerate launch \
     --same_network \
     --machine_rank 0 \
     --rdzv_backend 'static' \
-    pefts/mft_accelerate.py --train_config configs/"xxx_train_config.json" \
+    pefts/mft_accelerate.py --train_config "$CONFIG" \
       --distributed_type "deepspeed" \
         > MFTCoder-training-"$TODAY".log 2>&1 &

mftcoder_accelerate/src/ds_zero3_single_launch.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+# Author: Chaoyu Chen
+# Last Modified: 2024/12/11
+# Description: An alternative(Command line) way to launch DeepSpeed training
+
+# Launch script on single node
+N_GPU_PER_NODE=8
+
+# config path
+CONFIG="configs/xxx_train_config.json"
+
+# envs used inside training
+export OMP_NUM_THREADS=4
+export TOKENIZERS_PARALLELISM=False
+
+TODAY=$(date +%Y-%m%d-%H%M)
+
+# accelerate launch --config_file accelerate_ds_config.yaml \
+accelerate launch \
+    --num_machines 1 \
+    --num_processes $N_GPU_PER_NODE \
+    --use_deepspeed \
+    --zero_stage 3 \
+    --offload_optimizer_device 'cpu' \
+    --offload_param_device 'cpu' \
+    --gradient_accumulation_steps 1 \
+    --gradient_clipping 1.0 \
+    --zero3_init_flag true \
+    --zero3_save_16bit_model true \
+    --main_training_function 'main' \
+    --mixed_precision 'bf16' \
+    --dynamo_backend 'no' \
+    --same_network \
+    --machine_rank 0 \
+    --rdzv_backend 'static' \
+    pefts/mft_accelerate.py --train_config "$CONFIG" \
+      --distributed_type "deepspeed" \
+        > MFTCoder-training-"$TODAY".log 2>&1 &

mftcoder_accelerate/src/fsdp_single_launch.sh

@@ -6,6 +6,14 @@
 # Launch script on single node
 N_GPU_PER_NODE=8
 
+# config path
+CONFIG="configs/xxx_train_config.json"
+
+# fsdp_transformer_layer_cls_to_wrap, choose the DecoderLayer
+WRAP_MODULE="LlamaDecoderLayer"
+
+
+
 # envs used inside training
 export OMP_NUM_THREADS=4
 export TOKENIZERS_PARALLELISM=False
@@ -21,15 +29,15 @@ accelerate launch \
     --fsdp_auto_wrap_policy=TRANSFORMER_BASED_WRAP \
     --fsdp_state_dict_type=FULL_STATE_DICT \
     --fsdp_backward_prefetch_policy=BACKWARD_PRE \
-    --fsdp_transformer_layer_cls_to_wrap=LlamaDecoderLayer \
+    --fsdp_transformer_layer_cls_to_wrap=$WRAP_MODULE \
     --fsdp_offload_params=false \
     --main_training_function=main \
     --mixed_precision=bf16 \
     --dynamo_backend=no \
     --same_network \
     --machine_rank=0 \
     --rdzv_backend=static \
-    pefts/mft_accelerate.py --train_config configs/"xxx_train_config.json" \
+    pefts/mft_accelerate.py --train_config "$CONFIG" \
         --distributed_type "fsdp" \
         > MFTCoder-training-"$TODAY".log 2>&1 &
 

mftcoder_accelerate/src/pefts/mft_accelerate.py

@@ -418,7 +418,7 @@ def main():
             bias="lora_only",
         )
 
-    # # 是否要加入新的special tokens
+    # new special tokens
     # num_added_toks = tokenizer.tokenizer.add_special_tokens(["<role_start>", "<role_end>"])
     # accelerator.print("We have added", num_added_toks, "tokens")
     # accelerator.print(f"role marker tokens {tokenizer.convert_tokens_to_ids('<role_start>')} {tokenizer.convert_tokens_to_ids('<role_end>')}, resized tokenizer_size: {len(tokenizer)}")
@@ -465,7 +465,6 @@ def main():
     tokenizer = build_tokenizer(args)
     # Note: resize_token_embeddings expects to receive the full size of the new vocabulary,
     # i.e. the length of the tokenizer.
-    # 如果新增special tokens, 需要resize input embedding 和output embedding
     # model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=32)
 
     accelerator.print("Model load_in_4bit: ", args.quantization == "4bit")

mftcoder_accelerate/src/pefts/trainer.py

@@ -221,7 +221,7 @@ def accelerate_saving_checkpoint(self, output_dir: str, completed_steps: int):
                 "latest_ckpt": output_dir,
                 "lr": self.optimizer.param_groups[0]["lr"],
                 # 1 step back because ckping is after schuduler.step()
-                "scheduler_last_ep": self.lr_scheduler.state_dict().get("last_epoch", 0) - 1,
+                # "scheduler_last_ep": self.lr_scheduler.state_dict().get("last_epoch", 0) - 1,
             }
             with open(os.path.join(self.args.output_dir, "latest"), "w") as f:
                 json.dump(latest, f, indent=2)