Skip to content

Commit aa27516

Browse files
EmmonsCurseEmmonsCurse
committed
[CI] [DEBUG] Coverage merge xpu and gpu
1 parent c2c1942 commit aa27516

File tree

8 files changed

+292
-244
lines changed

8 files changed

+292
-244
lines changed

‎.github/workflows/_ci_xpu.yml‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
name: CI_XPU
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
DOCKER_IMAGE:
7+
description: "Build Images"
8+
required: true
9+
type: string
10+
default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/fastdeploy-xpu:2.1.0"
11+
FASTDEPLOY_ARCHIVE_URL:
12+
description: "URL of the compressed FastDeploy code archive."
13+
required: true
14+
type: string
15+
outputs:
16+
xpu_cov_file_url:
17+
description: "Output path of the GPU tests"
18+
value: ${{ jobs.CI_XPU.outputs.xpu_cov_file_url }}
19+
20+
concurrency:
21+
group: ${{ github.event.pull_request.number }}-xpu-ci
22+
cancel-in-progress: true
23+
24+
jobs:
25+
CI_XPU:
26+
runs-on: [self-hosted, XPU-P800-8Card]
27+
outputs:
28+
xpu_cov_file_url: ${{ steps.set_output.outputs.xpu_cov_file_url }}
29+
steps:
30+
- name: Print current runner name
31+
run: |
32+
echo "Current runner name: ${{ runner.name }}"
33+
- name: Code Checkout
34+
env:
35+
docker_image: ${{ inputs.DOCKER_IMAGE }}
36+
fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
37+
run: |
38+
REPO="https://github.com/${{ github.repository }}.git"
39+
FULL_REPO="${{ github.repository }}"
40+
REPO_NAME="${FULL_REPO##*/}"
41+
BASE_BRANCH="${{ github.base_ref }}"
42+
# Clean the repository directory before starting
43+
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
44+
-e "REPO_NAME=${REPO_NAME}" \
45+
-e "BASE_BRANCH=${BASE_BRANCH}" \
46+
-e "fd_archive_url=${fd_archive_url}" \
47+
${docker_image} /bin/bash -c '
48+
if [ -d ${REPO_NAME} ]; then
49+
echo "Directory ${REPO_NAME} exists, removing it..."
50+
rm -rf ${REPO_NAME}
51+
fi
52+
wget -q ${fd_archive_url}
53+
tar -xf FastDeploy.tar.gz
54+
rm -rf FastDeploy.tar.gz
55+
set -x
56+
cd FastDeploy
57+
git config --global --add safe.directory "$(pwd)"
58+
git config --global user.name "FastDeployCI"
59+
git config --global user.email "fastdeploy_ci@example.com"
60+
git log -n 3 --oneline
61+
'
62+
- name: Run CI unittest
63+
id: set_output
64+
env:
65+
docker_image: ${{ inputs.DOCKER_IMAGE }}
66+
IS_PR: ${{ github.event_name == 'pull_request' }}
67+
run: |
68+
runner_name="${{ runner.name }}"
69+
last_char="${runner_name: -1}"
70+
if [[ "$last_char" == "1" ]]; then
71+
xpu_id="4"
72+
else
73+
xpu_id="0"
74+
fi
75+
FD_API_PORT=$((8180 + gpu_id * 100))
76+
FD_ENGINE_QUEUE_PORT=$((8150 + gpu_id * 100))
77+
FD_METRICS_PORT=$((8170 + gpu_id * 100))
78+
commit_id=${{ github.event.pull_request.head.sha }}
79+
pr_num=${{ github.event.pull_request.number }}
80+
PARENT_DIR=$(dirname "$WORKSPACE")
81+
echo "PARENT_DIR:$PARENT_DIR"
82+
docker run --rm --net=host --cap-add=SYS_PTRACE --privileged --shm-size=64G \
83+
-v $(pwd):/workspace -w /workspace \
84+
-v "/ssd3:/ssd3" \
85+
-e "MODEL_PATH=/ssd3/model" \
86+
-e "http_proxy=$(git config --global --get http.proxy)" \
87+
-e "https_proxy=$(git config --global --get https.proxy)" \
88+
-e "no_proxy=bcebos.com,mirrors.tuna.tsinghua.edu.cn,127.0.0.1,localhost" \
89+
-e "FD_API_PORT=${FD_API_PORT}" \
90+
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
91+
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
92+
-e "IS_PR=${IS_PR}" \
93+
-e "commit_id=${commit_id}" \
94+
-e "pr_num=${pr_num}" \
95+
${docker_image} /bin/bash -c '
96+
git config --global --add safe.directory /workspace/FastDeploy
97+
chown -R $(whoami) /workspace/FastDeploy
98+
cd FastDeploy
99+
python -m pip install coverage
100+
export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage.xpu
101+
export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
102+
TEST_EXIT_CODE=0
103+
bash scripts/run_ci_xpu.sh || TEST_EXIT_CODE=8
104+
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
105+
cat exit_code.env
106+
coverage combine coveragedata/ || echo "No data to combine"
107+
tar -cvf xpu_coverage.tar -C coveragedata .
108+
coverage report
109+
# coverage data upload
110+
target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/XPU
111+
wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py -O bos_tools.py
112+
push_file=$(realpath bos_tools.py)
113+
python -m pip install bce-python-sdk==0.9.29
114+
cov_file="xpu_coverage.tar"
115+
if [ -f ${cov_file} ];then
116+
python ${push_file} ${cov_file} ${target_path}/CoverageData
117+
target_path_stripped="${target_path#paddle-github-action/}"
118+
XPU_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${cov_file}
119+
echo "xpu_cov_file_url=${XPU_COV_FILE_URL}" >> github.output
120+
fi
121+
'
122+
if [ -f FastDeploy/github.output ];then
123+
cat FastDeploy/github.output >> $GITHUB_OUTPUT
124+
fi
125+
if [ -f FastDeploy/exit_code.env ]; then
126+
cat FastDeploy/exit_code.env >> $GITHUB_ENV
127+
source FastDeploy/exit_code.env
128+
fi
129+
exit "$TEST_EXIT_CODE"

‎.github/workflows/_coverage_combine.yml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
name: Coverage Combine
2+
description: "Coverage Combine And Check"
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
GPU_COV_FILE_URL:
8+
description: "URL of the compressed GPU Coverage Data archive."
9+
required: true
10+
type: string
11+
XPU_COV_FILE_URL:
12+
description: "URL of the compressed GPU Coverage Data archive."
13+
required: true
14+
type: string
15+
CACHE_DIR:
16+
description: "Cache Dir Use"
17+
required: false
18+
type: string
19+
default: ""
20+
secrets:
21+
github-token:
22+
required: true
23+
24+
25+
jobs:
26+
coverage_combine:
27+
name: Coverage Combine And Check
28+
env:
29+
gpu_cov_file_url: ${{ inputs.GPU_COV_FILE_URL }}
30+
xpu_cov_file_url: ${{ inputs.XPU_COV_FILE_URL }}
31+
IS_PR: ${{ github.event_name == 'pull_request' }}
32+
runs-on: ubuntu-latest
33+
steps:
34+
- name: Clone FastDeploy
35+
uses: actions/checkout@v4
36+
with:
37+
ref: ${{ github.event_name == 'pull_request'
38+
&& github.event.pull_request.base.ref
39+
|| github.ref_name }}
40+
submodules: 'recursive'
41+
fetch-depth: 1000
42+
- name: Python Setup
43+
uses: actions/setup-python@v5
44+
with:
45+
python-version: '3.10'
46+
- name: coverage file download and combine
47+
shell: bash
48+
env:
49+
BASE_REF: ${{ github.event.pull_request.base.ref }}
50+
run: |
51+
python -m pip install coverage diff-cover
52+
mkdir coveragedata
53+
if [ -z "${gpu_cov_file_url}" ]; then
54+
echo "No diff coverage file URL provided."
55+
else
56+
wget -q ${gpu_cov_file_url}
57+
gpu_cov_file=$(basename "$gpu_cov_file_url")
58+
tar -xf ${gpu_cov_file} -C coveragedata
59+
fi
60+
if [ -z "${xpu_cov_file_url}" ]; then
61+
echo "No diff coverage file URL provided."
62+
else
63+
wget -q ${xpu_cov_file_url}
64+
xpu_cov_file=$(basename "$xpu_cov_file_url")
65+
tar -xf ${xpu_cov_file} -C coveragedata
66+
fi
67+
export COVERAGE_FILE=coveragedata/.coverage
68+
coverage combine coveragedata/
69+
coverage report --ignore-errors
70+
coverage xml -o python_coverage_all.xml --ignore-errors
71+
COVERAGE_EXIT_CODE=0
72+
if [[ "$IS_PR" == "true" ]]; then
73+
git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
74+
diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9
75+
python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml
76+
filename=diff_coverage.json
77+
else
78+
echo "Not a PR, skipping diff-cover"
79+
fi
80+
if [[ -f diff_coverage.json ]]; then
81+
echo "====== Diff Coverage JSON ======"
82+
if command -v jq >/dev/null 2>&1; then
83+
jq . diff_coverage.json
84+
else
85+
cat diff_coverage.json
86+
fi
87+
echo "================================"
88+
fi
89+
exit "$COVERAGE_EXIT_CODE"
90+
- name: Upload diff coverage report
91+
if: always() && hashFiles('diff_coverage.xml') != ''
92+
uses: codecov/codecov-action@v5
93+
with:
94+
files: ./diff_coverage.xml
95+
codecov_yml_path: ./scripts/codecov.yml
96+
name: python diff coverage
97+
verbose: true
98+
flags: diff

0 commit comments

Comments
 (0)