torchrun启动分布式训练

训练v1启动方式

基于客户镜像，需要额外安装几个包：

1. apt-get install wget --不挂载sfs的话，需要用wget从obs上获取启动脚本

2. apt-get install iputils-ping --为了训练v1域名能解析到ip

3. 解决shell运行时报错 [: : unexpected operator** --输入dpkg-reconfigure dash 后选择no

#!/bin/sh
# 解析ip, only executed on 0 node
if [ $DLS_TASK_INDEX -eq 0 ]; then
    for ((i=0; i<$DLS_TASK_NUMBER; i++))
    do
        if [[ "$DLS_TASK_NUMBER" == "1" ]]
        then
            eval hostname=\$BATCH_CURRENT_HOST
        else
            eval hostname=\$BATCH_CUSTOM${i}_HOSTS
        fi
        host_arr=(${hostname//:/ })
        echo "[Modelarts Service Log]host_arr ${host_arr[*]}"
        ip=""
        while [ -z "$ip" ]; do
            ip=$(ping -c 1 ${host_arr[0]} | grep "PING" | sed -E 's/PING .* .([0-9.]+). .*/\1/g')
            echo "[Modelarts Service Log]current ip ${ip}"
        done
    done
fi

export NCCL_DEBUG=INFO

if [ $DLS_TASK_NUMBER == 1 ]; then
    MASTER_ADDR=localhost
    MASTER_PORT=6000
    NNODES=1
    NODE_RANK=0
else
    MASTER_PORT="6060"
    MASTER_HOST="$BATCH_CUSTOM0_HOSTS"
    MASTER_ADDR="${MASTER_HOST%%:*}"
    #MASTER_PORT="${MASTER_HOST##*:}"
    NNODES="$DLS_TASK_NUMBER"
    NODE_RANK="$DLS_TASK_INDEX"
	JOB_ID="1234"
fi

if [ $NODE_RANK == 0 ]
then
    EXT_ARGS="--rdzv_conf=is_host=1"
else
    EXT_ARGS=""
fi

echo `date`
CMD="torchrun --nnodes=$NNODES --node_rank=$NODE_RANK --nproc_per_node=8 $EXT_ARGS --rdzv_id=$JOB_ID --rdzv_backend=c10d --rdzv_endpoint=$MASTER_ADDR:$MASTER_PORT test.py"
echo $CMD
$CMD

新版训练启动方式

新版训练开启容错模式后，自动检查域名解析ip是否正常，故不需要额外添加解析ip的代码，只需要替换对应环境变量，获取分布式任务所需的启动参数

shell启动脚本

#!/bin/bash
if [[ $MA_NUM_HOSTS == 1 ]]; then
    MASTER_ADDR=localhost
    MASTER_PORT=6000
    NNODES=1
    NODE_RANK=0
else
    MASTER_HOST="$VC_WORKER_HOSTS"
    MASTER_ADDR="${VC_WORKER_HOSTS%%,*}"
    MASTER_PORT="6060"
    NNODES="$MA_NUM_HOSTS"
    NODE_RANK="$VC_TASK_INDEX"
	JOB_ID="1234"
fi

if [[ $NODE_RANK == 0 ]]; then
    EXT_ARGS="--rdzv_conf=is_host=1"
else
    EXT_ARGS=""
fi

CMD="python -m torch.distributed.run --nnodes=$NNODES --node_rank=$NODE_RANK $EXT_ARGS --nproc_per_node=8 --rdzv_id=$JOB_ID --rdzv_backend=c10d --rdzv_endpoint=$MASTER_ADDR:$MASTER_PORT  /home/ma-user/modelarts/user-job-dir/code/test.py"
echo $CMD
$CMD