Skip to content

Commit 0d51033

Browse files
EmmonsCurseEmmonsCurse
committed
[CI] [DEBUG] Coverage merge xpu and gpu
1 parent 3d7f1a8 commit 0d51033

File tree

8 files changed

+301
-244
lines changed

8 files changed

+301
-244
lines changed

‎.github/workflows/_ci_xpu.yml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: CI_XPU
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
DOCKER_IMAGE:
7+
description: "Build Images"
8+
required: true
9+
type: string
10+
default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/fastdeploy-xpu:2.1.0"
11+
FASTDEPLOY_ARCHIVE_URL:
12+
description: "URL of the compressed FastDeploy code archive."
13+
required: true
14+
type: string
15+
outputs:
16+
xpu_cov_file_url:
17+
description: "Output path of the GPU tests"
18+
value: ${{ jobs.CI_XPU.outputs.xpu_cov_file_url }}
19+
20+
concurrency:
21+
group: ${{ github.event.pull_request.number }}-xpu-ci
22+
cancel-in-progress: true
23+
24+
jobs:
25+
CI_XPU:
26+
runs-on: [self-hosted, XPU-P800-8Card]
27+
outputs:
28+
xpu_cov_file_url: ${{ steps.set_output.outputs.xpu_cov_file_url }}
29+
steps:
30+
- name: Print current runner name
31+
run: |
32+
echo "Current runner name: ${{ runner.name }}"
33+
- name: Code Checkout
34+
env:
35+
docker_image: ${{ inputs.DOCKER_IMAGE }}
36+
fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
37+
run: |
38+
REPO="https://github.com/${{ github.repository }}.git"
39+
FULL_REPO="${{ github.repository }}"
40+
REPO_NAME="${FULL_REPO##*/}"
41+
BASE_BRANCH="${{ github.base_ref }}"
42+
# Clean the repository directory before starting
43+
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
44+
-e "REPO_NAME=${REPO_NAME}" \
45+
-e "BASE_BRANCH=${BASE_BRANCH}" \
46+
-e "fd_archive_url=${fd_archive_url}" \
47+
${docker_image} /bin/bash -c '
48+
if [ -d ${REPO_NAME} ]; then
49+
echo "Directory ${REPO_NAME} exists, removing it..."
50+
rm -rf ${REPO_NAME}
51+
fi
52+
wget -q ${fd_archive_url}
53+
tar -xf FastDeploy.tar.gz
54+
rm -rf FastDeploy.tar.gz
55+
set -x
56+
cd FastDeploy
57+
git config --global --add safe.directory "$(pwd)"
58+
git config --global user.name "FastDeployCI"
59+
git config --global user.email "fastdeploy_ci@example.com"
60+
git log -n 3 --oneline
61+
'
62+
- name: Run CI unittest
63+
id: set_output
64+
env:
65+
docker_image: ${{ inputs.DOCKER_IMAGE }}
66+
IS_PR: ${{ github.event_name == 'pull_request' }}
67+
run: |
68+
runner_name="${{ runner.name }}"
69+
last_char="${runner_name: -1}"
70+
if [[ "$last_char" == "1" ]]; then
71+
xpu_id="4"
72+
else
73+
xpu_id="0"
74+
fi
75+
FD_API_PORT=$((8180 + xpu_id * 100))
76+
FD_ENGINE_QUEUE_PORT=$((8150 + xpu_id * 100))
77+
FD_METRICS_PORT=$((8170 + xpu_id * 100))
78+
commit_id=${{ github.event.pull_request.head.sha }}
79+
pr_num=${{ github.event.pull_request.number }}
80+
PARENT_DIR=$(dirname "$WORKSPACE")
81+
echo "PARENT_DIR:$PARENT_DIR"
82+
docker run --rm --net=host --cap-add=SYS_PTRACE --privileged --shm-size=64G \
83+
-v $(pwd):/workspace -w /workspace \
84+
-v "/ssd3:/ssd3" \
85+
-e "MODEL_PATH=/ssd3/model" \
86+
-e "http_proxy=$(git config --global --get http.proxy)" \
87+
-e "https_proxy=$(git config --global --get https.proxy)" \
88+
-e "no_proxy=bcebos.com,mirrors.tuna.tsinghua.edu.cn,127.0.0.1,localhost" \
89+
-e "FD_API_PORT=${FD_API_PORT}" \
90+
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
91+
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
92+
-e "XPU_ID=${xpu_id}" \
93+
-e "IS_PR=${IS_PR}" \
94+
-e "commit_id=${commit_id}" \
95+
-e "pr_num=${pr_num}" \
96+
${docker_image} /bin/bash -c '
97+
git config --global --add safe.directory /workspace/FastDeploy
98+
chown -R $(whoami) /workspace/FastDeploy
99+
cd FastDeploy
100+
python -m pip install coverage
101+
export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage.xpu
102+
export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
103+
TEST_EXIT_CODE=0
104+
bash scripts/run_ci_xpu.sh || TEST_EXIT_CODE=8
105+
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
106+
cat exit_code.env
107+
tar -cvf xpu_coverage.tar -C coveragedata .
108+
coverage combine coveragedata/ || echo "No data to combine"
109+
coverage report
110+
# coverage data upload
111+
target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/XPU
112+
wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py -O bos_tools.py
113+
push_file=$(realpath bos_tools.py)
114+
python -m pip install bce-python-sdk==0.9.29
115+
cov_file="xpu_coverage.tar"
116+
if [ -f ${cov_file} ];then
117+
python ${push_file} ${cov_file} ${target_path}/CoverageData
118+
target_path_stripped="${target_path#paddle-github-action/}"
119+
XPU_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${cov_file}
120+
echo "xpu_cov_file_url=${XPU_COV_FILE_URL}" >> github.output
121+
fi
122+
'
123+
if [ -f FastDeploy/github.output ];then
124+
cat FastDeploy/github.output >> $GITHUB_OUTPUT
125+
fi
126+
if [ -f FastDeploy/exit_code.env ]; then
127+
cat FastDeploy/exit_code.env >> $GITHUB_ENV
128+
source FastDeploy/exit_code.env
129+
fi
130+
exit "$TEST_EXIT_CODE"

‎.github/workflows/_coverage_combine.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
name: Coverage Combine
2+
description: "Coverage Combine And Check"
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
GPU_COV_FILE_URL:
8+
description: "URL of the compressed GPU Coverage Data archive."
9+
required: true
10+
type: string
11+
XPU_COV_FILE_URL:
12+
description: "URL of the compressed GPU Coverage Data archive."
13+
required: true
14+
type: string
15+
CACHE_DIR:
16+
description: "Cache Dir Use"
17+
required: false
18+
type: string
19+
default: ""
20+
secrets:
21+
github-token:
22+
required: true
23+
24+
25+
jobs:
26+
coverage_combine:
27+
name: Coverage Combine And Check
28+
env:
29+
gpu_cov_file_url: ${{ inputs.GPU_COV_FILE_URL }}
30+
xpu_cov_file_url: ${{ inputs.XPU_COV_FILE_URL }}
31+
IS_PR: ${{ github.event_name == 'pull_request' }}
32+
runs-on: ubuntu-latest
33+
steps:
34+
- name: Clone FastDeploy
35+
uses: actions/checkout@v4
36+
with:
37+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
38+
submodules: recursive
39+
fetch-depth: 0
40+
- name: Fetch base branch
41+
if: ${{ github.event_name == 'pull_request' }}
42+
run: |
43+
git fetch origin ${{ github.event.pull_request.base.ref }} --depth=1000
44+
# 找到最后共同节点
45+
MERGE_BASE=$(git merge-base origin/${{ github.event.pull_request.base.ref }} ${{ github.event.pull_request.head.sha }})
46+
# 生成 diff 文件
47+
git diff ${MERGE_BASE} ${{ github.event.pull_request.head.sha }} --unified=0 > diff.txt
48+
cat diff.txt
49+
- name: Python Setup
50+
uses: actions/setup-python@v5
51+
with:
52+
python-version: '3.10'
53+
- name: coverage file download and combine
54+
shell: bash
55+
env:
56+
BASE_REF: ${{ github.event.pull_request.base.ref }}
57+
run: |
58+
git log -n 3
59+
python -m pip install coverage diff-cover
60+
mkdir coveragedata
61+
if [ -z "${gpu_cov_file_url}" ]; then
62+
echo "No diff coverage file URL provided."
63+
else
64+
wget -q ${gpu_cov_file_url}
65+
gpu_cov_file=$(basename "$gpu_cov_file_url")
66+
tar -xf ${gpu_cov_file} -C coveragedata
67+
fi
68+
if [ -z "${xpu_cov_file_url}" ]; then
69+
echo "No diff coverage file URL provided."
70+
else
71+
wget -q ${xpu_cov_file_url}
72+
xpu_cov_file=$(basename "$xpu_cov_file_url")
73+
tar -xf ${xpu_cov_file} -C coveragedata
74+
fi
75+
export COVERAGE_FILE=coveragedata/.coverage
76+
export COVERAGE_RCFILE=./scripts/.coveragerc
77+
export COVERAGE_IGNORE_ERRORS=True
78+
coverage combine coveragedata/
79+
coverage report --ignore-errors
80+
coverage xml -o python_coverage_all.xml --ignore-errors
81+
COVERAGE_EXIT_CODE=0
82+
set -x
83+
if [[ "$IS_PR" == "true" ]]; then
84+
diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9
85+
python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml
86+
filename=diff_coverage.json
87+
else
88+
echo "Not a PR, skipping diff-cover"
89+
fi
90+
if [[ -f diff_coverage.json ]]; then
91+
echo "====== Diff Coverage JSON ======"
92+
if command -v jq >/dev/null 2>&1; then
93+
jq . diff_coverage.json
94+
else
95+
cat diff_coverage.json
96+
fi
97+
echo "================================"
98+
fi
99+
exit "$COVERAGE_EXIT_CODE"
100+
- name: Upload diff coverage report
101+
if: always() && hashFiles('diff_coverage.xml') != ''
102+
uses: codecov/codecov-action@v5
103+
with:
104+
files: ./diff_coverage.xml
105+
codecov_yml_path: ./scripts/codecov.yml
106+
name: python diff coverage
107+
verbose: true
108+
flags: diff

0 commit comments

Comments
 (0)