Skip to content

Commit f85b422

Browse files
AlexandreSingerAlexandreSinger
authored
Merge pull request #2759 from AlexandreSinger/feature-ci-test-num-proc-upgrade
[CI] Improved How The CI Handles Num Cores Used
2 parents 3b74db5 + 0610fff commit f85b422

File tree

3 files changed

+89
-16
lines changed

3 files changed

+89
-16
lines changed

‎.github/scripts/build.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,13 @@ source $(dirname "$0")/common.sh
66

77
$SPACER
88

9+
if [[ -z "${NUM_PROC}" ]]; then
10+
NUM_PROC=1
11+
fi
12+
913
start_section "vtr.build" "${GREEN}Building..${NC}"
1014
export FAILURE=0
11-
make -k BUILD_TYPE=${BUILD_TYPE} CMAKE_PARAMS="-Werror=dev ${CMAKE_PARAMS} ${CMAKE_INSTALL_PREFIX_PARAMS}" -j2 || export FAILURE=1
15+
make -k BUILD_TYPE=${BUILD_TYPE} CMAKE_PARAMS="-Werror=dev ${CMAKE_PARAMS} ${CMAKE_INSTALL_PREFIX_PARAMS}" -j${NUM_PROC} || export FAILURE=1
1216
end_section "vtr.build"
1317

1418
# When the build fails, produce the failure output in a clear way

‎.github/scripts/unittest.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ $(dirname "$0")/build.sh
99
$SPACER
1010

1111
start_section "vtr.test.0" "${GREEN}Testing..${NC} ${CYAN}C++ unit tests${NC}"
12-
make test
12+
make test -j${NUM_PROC}
1313
end_section "vtr.test.0"
1414

1515
$SPACER

‎.github/workflows/test.yml‎

Lines changed: 83 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,20 @@ jobs:
5252
- uses: actions/checkout@v4
5353
with:
5454
submodules: 'true'
55-
- run: ./.github/scripts/install_dependencies.sh
55+
56+
- name: Get number of CPU cores
57+
uses: SimenB/github-actions-cpu-cores@v2
58+
id: cpu-cores
59+
60+
- name: Install dependencies
61+
run: ./.github/scripts/install_dependencies.sh
5662

5763
- uses: hendrikmuhs/ccache-action@v1.2
5864

5965
- name: Test
6066
env:
6167
BUILD_TYPE: ${{ matrix.build_type }}
68+
NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
6269
run: |
6370
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
6471
./.github/scripts/build.sh VERBOSE=${{ matrix.verbose }}
@@ -80,7 +87,9 @@ jobs:
8087
with:
8188
python-version: 3.10.10
8289
- uses: actions/checkout@v4
83-
- run: ./.github/scripts/install_dependencies.sh
90+
91+
- name: Install dependencies
92+
run: ./.github/scripts/install_dependencies.sh
8493

8594
- name: Test
8695
run: ./dev/${{ matrix.script }}
@@ -97,11 +106,18 @@ jobs:
97106
- uses: actions/checkout@v4
98107
with:
99108
submodules: 'true'
100-
- run: ./.github/scripts/install_dependencies.sh
109+
110+
- name: Get number of CPU cores
111+
uses: SimenB/github-actions-cpu-cores@v2
112+
id: cpu-cores
113+
114+
- name: Install dependencies
115+
run: ./.github/scripts/install_dependencies.sh
101116

102117
- name: Test
103118
env:
104119
CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on"
120+
NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
105121
run: ./.github/scripts/unittest.sh
106122

107123

@@ -116,7 +132,13 @@ jobs:
116132
- uses: actions/checkout@v4
117133
with:
118134
submodules: 'true'
119-
- run: ./.github/scripts/install_dependencies.sh
135+
136+
- name: Get number of CPU cores
137+
uses: SimenB/github-actions-cpu-cores@v2
138+
id: cpu-cores
139+
140+
- name: Install dependencies
141+
run: ./.github/scripts/install_dependencies.sh
120142

121143
- uses: hendrikmuhs/ccache-action@v1.2
122144

@@ -125,6 +147,7 @@ jobs:
125147
#In order to get compilation warnings produced per source file, we must do a non-IPO build
126148
#We also turn warnings into errors for this target by doing a strict compile
127149
CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_STRICT_COMPILE=on -DVTR_IPO_BUILD=off"
150+
NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
128151
run: |
129152
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
130153
./.github/scripts/build.sh
@@ -214,6 +237,10 @@ jobs:
214237
with:
215238
submodules: 'true'
216239

240+
- name: Get number of CPU cores
241+
uses: SimenB/github-actions-cpu-cores@v2
242+
id: cpu-cores
243+
217244
- name: Install dependencies
218245
run: ./.github/scripts/install_dependencies.sh
219246

@@ -226,10 +253,11 @@ jobs:
226253
- name: Test
227254
env:
228255
CMAKE_PARAMS: ${{ matrix.params }}
256+
NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
229257
run: |
230258
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
231259
./.github/scripts/build.sh
232-
./run_reg_test.py ${{ matrix.suite }} -show_failures -j2
260+
./run_reg_test.py ${{ matrix.suite }} -show_failures -j${{ steps.cpu-cores.outputs.count}}
233261
234262
- name: Upload regression run files
235263
if: ${{ !cancelled() }}
@@ -277,7 +305,12 @@ jobs:
277305
with:
278306
submodules: 'true'
279307

280-
- run: ./.github/scripts/install_dependencies.sh
308+
- name: Get number of CPU cores
309+
uses: SimenB/github-actions-cpu-cores@v2
310+
id: cpu-cores
311+
312+
- name: Install dependencies
313+
run: ./.github/scripts/install_dependencies.sh
281314

282315
- uses: hendrikmuhs/ccache-action@v1.2
283316

@@ -291,12 +324,13 @@ jobs:
291324
# depends on LLVM and all CI tests where VTR_ENABLE_SANITIZE is enabled fail. For a temporary
292325
# fix, we manually reduce the entropy. This quick fix should be removed in the future
293326
# when github deploys a more stable Ubuntu image.
327+
NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
294328
run: |
295329
sudo sysctl -w vm.mmap_rnd_bits=28
296330
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
297331
./.github/scripts/build.sh
298332
# We skip QoR since we are only checking for errors in sanitizer runs
299-
./run_reg_test.py ${{ matrix.suite }} -show_failures -j2 -skip_qor
333+
./run_reg_test.py ${{ matrix.suite }} -show_failures -j${{ steps.cpu-cores.outputs.count }} -skip_qor
300334
301335
302336
Parmys:
@@ -310,18 +344,25 @@ jobs:
310344
- uses: actions/checkout@v4
311345
with:
312346
submodules: 'true'
313-
- run: ./.github/scripts/install_dependencies.sh
347+
348+
- name: Get number of CPU cores
349+
uses: SimenB/github-actions-cpu-cores@v2
350+
id: cpu-cores
351+
352+
- name: Install dependencies
353+
run: ./.github/scripts/install_dependencies.sh
314354

315355
- uses: hendrikmuhs/ccache-action@v1.2
316356

317357
- name: Test
318358
env:
319359
CMAKE_PARAMS: '-DVTR_IPO_BUILD=off'
320360
BUILD_TYPE: debug
361+
NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
321362
run: |
322363
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
323364
./.github/scripts/build.sh
324-
./run_reg_test.py parmys_reg_basic -show_failures -j2
365+
./run_reg_test.py parmys_reg_basic -show_failures -j${{ steps.cpu-cores.outputs.count }}
325366
326367
327368
ODINII:
@@ -335,19 +376,26 @@ jobs:
335376
- uses: actions/checkout@v4
336377
with:
337378
submodules: 'true'
338-
- run: ./.github/scripts/install_dependencies.sh
379+
380+
- name: Get number of CPU cores
381+
uses: SimenB/github-actions-cpu-cores@v2
382+
id: cpu-cores
383+
384+
- name: Install dependencies
385+
run: ./.github/scripts/install_dependencies.sh
339386

340387
- uses: hendrikmuhs/ccache-action@v1.2
341388

342389
- name: Test
343390
env:
344391
CMAKE_PARAMS: '-DVTR_ASSERT_LEVEL=3 -DVTR_ENABLE_SANITIZE=on -DVTR_IPO_BUILD=off -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on'
345392
BUILD_TYPE: debug
393+
NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
346394
run: |
347395
sudo sysctl -w vm.mmap_rnd_bits=28
348396
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
349397
./.github/scripts/build.sh
350-
./run_reg_test.py odin_reg_basic -show_failures -j2
398+
./run_reg_test.py odin_reg_basic -show_failures -j${{ steps.cpu-cores.outputs.count }}
351399
352400
353401
VQM2BLIF:
@@ -361,13 +409,20 @@ jobs:
361409
- uses: actions/checkout@v4
362410
with:
363411
submodules: 'true'
364-
- run: ./.github/scripts/install_dependencies.sh
412+
413+
- name: Get number of CPU cores
414+
uses: SimenB/github-actions-cpu-cores@v2
415+
id: cpu-cores
416+
417+
- name: Install dependencies
418+
run: ./.github/scripts/install_dependencies.sh
365419

366420
- uses: hendrikmuhs/ccache-action@v1.2
367421

368422
- name: Test
369423
env:
370424
BUILD_TYPE: release
425+
NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
371426
run: |
372427
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
373428
./.github/scripts/build.sh
@@ -396,7 +451,13 @@ jobs:
396451
- uses: actions/checkout@v4
397452
with:
398453
submodules: 'true'
399-
- run: ./.github/scripts/install_dependencies.sh
454+
455+
- name: Get number of CPU cores
456+
uses: SimenB/github-actions-cpu-cores@v2
457+
id: cpu-cores
458+
459+
- name: Install dependencies
460+
run: ./.github/scripts/install_dependencies.sh
400461

401462
- uses: hendrikmuhs/ccache-action@v1.2
402463

@@ -405,6 +466,7 @@ jobs:
405466
CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_ODIN=on"
406467
MATRIX_EVAL: ${{ matrix.eval }}
407468
BUILD_TYPE: release
469+
NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
408470
run: |
409471
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
410472
./.github/scripts/build.sh
@@ -431,13 +493,20 @@ jobs:
431493
- uses: actions/checkout@v4
432494
with:
433495
submodules: 'true'
434-
- run: ./.github/scripts/install_dependencies.sh
496+
497+
- name: Get number of CPU cores
498+
uses: SimenB/github-actions-cpu-cores@v2
499+
id: cpu-cores
500+
501+
- name: Install dependencies
502+
run: ./.github/scripts/install_dependencies.sh
435503

436504
- uses: hendrikmuhs/ccache-action@v1.2
437505

438506
- name: Test
439507
env:
440508
CMAKE_PARAMS: '-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on'
509+
NUM_PROC: ${{ steps.cpu-cores.outputs.count }}
441510
_COVERITY_URL: 'https://scan.coverity.com/download/linux64'
442511
_COVERITY_MD5: 'd0d7d7df9d6609e578f85096a755fb8f'
443512
run: |

0 commit comments

Comments
 (0)