Skip to content

Commit 86e951a

Browse files
wugeshuiwugeshui
authored
mv ci to sco (DeepLink-org#853)
* mv ci to sco
1 parent 3cbbb83 commit 86e951a

File tree

3 files changed

+46
-62
lines changed

3 files changed

+46
-62
lines changed

‎.github/workflows/main.yml‎

Lines changed: 36 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,14 @@ env:
1616
NFS_PATH: '/mnt/cache/share/parrotsci/github/cibuild/${{ github.repository }}'
1717
NFS_PATH_LUSTRE: '/mnt/lustre/share/parrotsci/github/cibuild/${{ github.repository }}'
1818
DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}'
19-
ENV_PATH: '/mnt/cache/share/platform/cienv'
19+
ENV_PATH: '/mnt/cache/share/platform/env'
2020
ENV_NAME: 'pt2.0_diopi'
2121
ENV_NAME_MMCV: 'pt1.11v1'
2222
GPU_REQUESTS: 1
2323
SLURM_PAR_SH1988: ${{ vars.SLURM_PAR_SH1984 != '' && vars.SLURM_PAR_SH1984 || 'pat_rd' }}
2424
SLURM_PAR_SH1424: ${{ vars.SLURM_PAR_SH1424 != '' && vars.SLURM_PAR_SH1424 || 'pat_rd' }}
2525
SLURM_PAR_CAMB: ${{ vars.SLURM_PAR_CAMB != '' && vars.SLURM_PAR_CAMB || 'camb_mlu370_m8 --exclude HOST-10-142-11-120,HOST-10-142-11-126' }}
26-
CLUSTER_1988: SH1988
26+
CLUSTER_V100: SCO
2727
CLUSTER_1424: SH1424
2828
CLUSTER_CAMB: CAMB
2929
CLUSTER_ASCEND: ASCEND
@@ -37,7 +37,7 @@ env:
3737
GEN_DATA: diopi_test/python/cache
3838
GEN_DATA_PATH: "/mnt/cache/share/deeplinkci/gen-data/source"
3939
ALL_COVERAGE: ${{ (contains( github.ref, 'main') || startsWith(github.ref, 'refs/heads/v') || startsWith(github.ref, 'refs/heads/dev')) && 'ON' || 'OFF' }}
40-
REQUIRE_COVERAGE: ${{ vars.REQUIRE_COVERAGE != '' && vars.REQUIRE_COVERAGE || '40' }}
40+
REQUIRE_COVERAGE: ${{ vars.REQUIRE_COVERAGE != '' && vars.REQUIRE_COVERAGE || '0' }}
4141

4242
concurrency:
4343
group: ${{ github.head_ref || github.ref }}
@@ -75,8 +75,8 @@ jobs:
7575
echo "RUN_RESULT=$RUN_RESULT" >> "$GITHUB_OUTPUT"
7676
- name: Rsync to Server
7777
run: |
78-
ssh ${CLUSTER_1988} "mkdir -p ${NFS_PATH}/${GITHUB_RUN_NUMBER}/source" \
79-
&& rsync -a --delete ${GITHUB_WORKSPACE}/source/ ${CLUSTER_1988}:${NFS_PATH}/${GITHUB_RUN_NUMBER}/source/ || echo "failure to connect to sh1988"
78+
ssh ${CLUSTER_V100} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source" \
79+
&& rsync -a --delete ${GITHUB_WORKSPACE}/source/ ${CLUSTER_V100}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ || echo "failure to connect to sh1988"
8080
ssh ${CLUSTER_CAMB} "mkdir -p ${NFS_PATH}/${GITHUB_RUN_NUMBER}/source" \
8181
&& rsync -a --delete ${GITHUB_WORKSPACE}/source/ ${CLUSTER_CAMB}:${NFS_PATH}/${GITHUB_RUN_NUMBER}/source/ || echo "failure to connect to camb"
8282
ssh ${CLUSTER_TOPSRIDER} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source" \
@@ -92,36 +92,33 @@ jobs:
9292
9393
lint:
9494
name: lint
95-
runs-on: tps-diopi-ci
95+
runs-on: tps-sco-ci
9696
needs: [Rsync]
9797
steps:
9898
- name: lint
9999
run: |
100-
ssh ${CLUSTER_1988} """
101100
set -e
102-
cd ${NFS_PATH}/ && ls -al && find ${NFS_PATH}/ -maxdepth 1 -mmin +300 -type d |xargs rm -rf
103-
source ${ENV_PATH}/github_bashrc && source /mnt/cache/share/platform/env/${ENV_NAME}
104-
cd ${NFS_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}
105-
srun --job-name=${GITHUB_JOB} --partition=${SLURM_PAR_SH1988} --time=10 bash -c 'cd diopi_test && bash scripts/ci_script.sh py-lint \
106-
&& bash scripts/ci_script.sh cpp-lint && cd ../impl \ && bash scripts/ci_script.sh cpp-lint' \
107-
&& cd ${NFS_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} \
108-
|| ( cd ${NFS_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
109-
"""
101+
cd ${DEEPLINK_PATH}/ && ls -al && find ${DEEPLINK_PATH}/ -maxdepth 1 -mmin +300 -type d |xargs rm -rf
102+
cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}
103+
srun --job-name=${GITHUB_JOB} bash -c "source ${ENV_PATH}/pt2.0_diopi \
104+
&& cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/diopi_test && bash scripts/ci_script.sh py-lint \
105+
&& bash scripts/ci_script.sh cpp-lint && cd ../impl && bash scripts/ci_script.sh cpp-lint" \
106+
&& cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} \
107+
|| ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
108+
110109
111110
Build-Nvidia:
112111
name: Build-Nvidia
113-
runs-on: tps-diopi-ci
112+
runs-on: tps-sco-ci
114113
needs: [Rsync]
115114
steps:
116115
- name: build
117116
run: |
118-
ssh ${CLUSTER_1988} """
119117
set -e
120-
export USE_COVERAGE=ON
121-
source ${ENV_PATH}/github_bashrc && source /mnt/cache/share/platform/env/${ENV_NAME}
122-
cd ${NFS_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${BUILD_TEST1} && cp -R source ${BUILD_TEST1} && cd ${BUILD_TEST1}
123-
srun --job-name=${GITHUB_JOB} --partition=${SLURM_PAR_SH1988} --time=10 bash -c 'cd impl && bash scripts/build_impl.sh torch' || ( cd ${NFS_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${BUILD_TEST1} && exit 1 )
124-
"""
118+
cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${BUILD_TEST1} && cp -R source ${BUILD_TEST1} && cd ${BUILD_TEST1}
119+
srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} \
120+
&& source ${ENV_PATH}/pt2.0_diopi \
121+
&& cd impl && bash scripts/build_impl.sh torch" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${BUILD_TEST1} && exit 1 )
125122
126123
Build-Nvidia-A100:
127124
name: Build-Nvidia-A100
@@ -138,7 +135,6 @@ jobs:
138135
srun --job-name=${GITHUB_JOB} --partition=${SLURM_PAR_SH1424} --time=10 bash -c 'cd impl && bash scripts/build_impl.sh torch' || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${BUILD_TEST1} && exit 1 )
139136
"""
140137
141-
142138
Build-Camb:
143139
name: Build-Camb
144140
runs-on: tps-diopi-ci
@@ -200,22 +196,20 @@ jobs:
200196

201197
Gen-Data:
202198
name: Gen-Data
203-
runs-on: tps-diopi-ci
199+
runs-on: tps-sco-ci
204200
needs: [Rsync,Build-Nvidia]
205201
steps:
206202
- name: gen-test-data
207203
run: |
208-
ssh ${CLUSTER_1988} """
209204
set -e
210205
if [[ \"${{ needs.Rsync.outputs.output }}\" == *GENDATA* ]];then
211-
source ${ENV_PATH}/github_bashrc && source /mnt/cache/share/platform/env/${ENV_NAME}
212-
cd ${NFS_PATH}/${GITHUB_RUN_NUMBER} && cd ${BUILD_TEST1} && cd diopi_test/python && ls &&
213-
srun --job-name=${GITHUB_JOB} --partition=${SLURM_PAR_SH1988} --time=10 --gres=gpu:${GPU_REQUESTS} bash -c 'python main.py --mode gen_data' \
214-
|| ( cd ${NFS_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} && git clean -xdf ${GEN_DATA} && exit 1 )
206+
srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python \
207+
&& source ${ENV_PATH}/pt2.0_diopi \
208+
&& python main.py --mode gen_data" \
209+
|| ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} && git clean -xdf ${GEN_DATA} && exit 1 )
215210
else
216211
ln -s ${GEN_DATA_PATH}/${GEN_DATA}/diopi ${NFS_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python/cache/
217212
fi
218-
"""
219213
220214
Gen-Data-Op-Test-A100:
221215
name: Gen-Data-Op-Test-A100
@@ -243,42 +237,32 @@ jobs:
243237
244238
Op-Test-Nvidia:
245239
name: Op-Test-Nvidia
246-
runs-on: tps-diopi-ci
240+
runs-on: tps-sco-ci
247241
needs: Gen-Data
248-
if: ${{ contains( needs.Rsync.outputs.output, 'NV' ) }}
249242
env:
250243
run_test: "coverage run -p"
251244
steps:
252245
- name: NHWC-32bit-test
253246
run: |
254-
ssh ${CLUSTER_1988} """
255247
set -e
256-
export USE_COVERAGE=ON
257-
source ${ENV_PATH}/github_bashrc && source /mnt/cache/share/platform/env/${ENV_NAME} && cd ${NFS_PATH}/${GITHUB_RUN_NUMBER} && cd ${BUILD_TEST1}
258-
srun --job-name=${GITHUB_JOB} --partition=${SLURM_PAR_SH1988} --time=20 --gres=gpu:${GPU_REQUESTS} bash -c 'cd diopi_test/python && python main.py --mode gen_case --fname batch_norm --nhwc &&
259-
python main.py --mode gen_case --fname index_select --four_bytes && python main.py --mode gen_case --fname arange --four_bytes && ${run_test} main.py --mode run_test'
248+
srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} \
249+
&& source ${ENV_PATH}/pt2.0_diopi && cd diopi_test/python && python main.py --mode gen_case --fname batch_norm --nhwc &&
250+
python main.py --mode gen_case --fname index_select --four_bytes && python main.py --mode gen_case --fname arange --four_bytes && ${run_test} main.py --mode run_test"
260251
if [ "${ALL_COVERAGE}" = "ON" ]; then
261-
bash /mnt/cache/share/platform/dep/sonar/coverage_DIOPI_nv.sh ${NFS_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} ${GITHUB_RUN_NUMBER} || echo "get coverage fail"
252+
bash /mnt/cache/share/platform/dep/sonar/coverage_DIOPI_nv.sh ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} ${GITHUB_RUN_NUMBER} || echo "get coverage fail"
262253
fi
263-
"""
264254
- name: test
265255
run: |
266-
ssh ${CLUSTER_1988} """
267256
set -e
268-
source ${ENV_PATH}/github_bashrc && source /mnt/cache/share/platform/env/${ENV_NAME} && cd ${NFS_PATH}/${GITHUB_RUN_NUMBER} && cd ${BUILD_TEST1}
269-
export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:${NFS_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/impl/lib
270-
echo \$LD_LIBRARY_PATH
271-
srun --job-name=${GITHUB_JOB} --partition=${SLURM_PAR_SH1988} --time=20 --gres=gpu:${GPU_REQUESTS} bash -c 'cd diopi_test/python && python main.py --mode gen_case &&
272-
python main.py --mode run_test'
273-
"""
257+
srun --job-name=${GITHUB_JOB} bash -c "export CI=true && export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/impl/lib \
258+
&& source ${ENV_PATH}/pt2.0_diopi && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} \
259+
&& cd diopi_test/python && python main.py --mode gen_case && python main.py --mode run_test"
274260
- name: increment coverage check
275261
if: ${{ contains( github.event_name, 'pull_request' ) && contains( github.base_ref, 'main' ) }}
276262
run: |
277-
ssh ${CLUSTER_1988} """
278263
set -e
279-
source ${ENV_PATH}/github_bashrc && source /mnt/cache/share/platform/env/${ENV_NAME} && cd ${NFS_PATH}/${GITHUB_RUN_NUMBER} && cd ${BUILD_TEST1}
264+
source ${ENV_PATH}/pt2.0_diopi && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && cd ${BUILD_TEST1}
280265
bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE}
281-
"""
282266
283267
# Rt-Test-Nvidia:
284268
# name: Rt-Test-Nvidia
@@ -311,7 +295,7 @@ jobs:
311295
export USE_COVERAGE=ON
312296
cd ${NFS_PATH}/${GITHUB_RUN_NUMBER}
313297
if [[ \"${{ needs.Rsync.outputs.output }}\" == *GENDATA* ]];then
314-
rsync -a ${CLUSTER_1988}:${NFS_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/${GEN_DATA} ${NFS_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python/
298+
rsync -a ${CLUSTER_V100}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/${GEN_DATA} ${NFS_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python/
315299
else
316300
ln -s ${GEN_DATA_PATH}/${GEN_DATA}/diopi ${NFS_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python/cache/
317301
fi
@@ -376,7 +360,7 @@ jobs:
376360
set -e
377361
cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}
378362
if [[ \"${{ needs.Rsync.outputs.output }}\" == *GENDATA* ]];then
379-
rsync -a ${CLUSTER_1988}:${NFS_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/${GEN_DATA}/diopi ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python/cache/
363+
rsync -a ${CLUSTER_V100}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/${GEN_DATA}/diopi ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python/cache/
380364
else
381365
ln -s ${GEN_DATA_PATH}/${GEN_DATA}/diopi ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python/cache/
382366
fi
@@ -468,7 +452,7 @@ jobs:
468452
set -e
469453
export USE_COVERAGE=ON
470454
cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}
471-
echo "rsync -a ${CLUSTER_1988}:${NFS_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/${GEN_DATA} ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python/"
455+
echo "rsync -a ${CLUSTER_V100}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/${GEN_DATA} ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python/"
472456
source /mnt/cache/share/platform/env/pt2.0_diopi && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && cd ${BUILD_TEST1}
473457
echo "test and clean, you can refer to the CAMB"
474458
"""

‎.github/workflows/runs-on-sco.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@ name: runs on sco
22

33
on:
44
workflow_dispatch:
5-
push:
6-
branches:
7-
- main
8-
pull_request:
9-
paths-ignore:
10-
- "**.md"
11-
- ".github/ISSUE_TEMPLATE/**"
12-
- ".git*"
13-
- "CODE_OF_CONDUCT**"
5+
# push:
6+
# branches:
7+
# - main
8+
# pull_request:
9+
# paths-ignore:
10+
# - "**.md"
11+
# - ".github/ISSUE_TEMPLATE/**"
12+
# - ".git*"
13+
# - "CODE_OF_CONDUCT**"
1414

1515
concurrency:
1616
group: sco-${{ github.head_ref || github.ref }}

‎impl/scripts/ci_script.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ case $1 in
1717
echo "cpp-lint"
1818
python scripts/cpplint.py --linelength=160 --repository=.. \
1919
--filter=-build/c++11,-legal/copyright,-build/include_subdir,-runtime/references,-runtime/printf,-runtime/int,-build/namespace \
20-
--exclude=${CURRENT_PATH}/../third_party --exclude=${CURRENT_PATH}/../ascend_npu/third_party --exclude=${CURRENT_PATH}/../build \
20+
--exclude=${IMPL_PATH}/third_party --exclude=${IMPL_PATH}/ascend_npu/third_party --exclude=${IMPL_PATH}/build \
2121
--recursive ./
2222
;;
2323
clang-tidy)

0 commit comments

Comments
 (0)