Skip to content

Commit b139e01

Browse files
EthanPeng-2048EthanPeng-2048
authored
Merge pull request #9 from EthanPeng-2048/rewrite
Rewrite
2 parents e7917a8 + fb45f11 commit b139e01

129 files changed

Lines changed: 58839 additions & 38207 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 1 addition & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1 @@
1-
# Compiled and build artifacts
2-
*.o
3-
*.obj
4-
*.out
5-
build/
6-
dist/
7-
datasets/
8-
pretrained/
9-
10-
# Dependencies
11-
__pycache__/
12-
*.pyc
13-
*.pyo
14-
.Python
15-
env/
16-
venv/
17-
.venv/
18-
.egg-info/
19-
*.egg
20-
21-
# Logs and temp files
22-
*.log
23-
*.tmp
24-
*.swp
25-
*.swo
26-
27-
# Environment
28-
.env
29-
.env.local
30-
.env.*
31-
32-
# Editors
33-
.vscode/
34-
.idea/
35-
*.swp
36-
*.swo
37-
38-
# OS generated files
39-
.DS_Store
40-
Thumbs.db
41-
42-
# Coverage reports
43-
coverage/
44-
htmlcov/
45-
.coverage
46-
47-
# Build systems
48-
target/
49-
.gradle/
50-
*.class
51-
*.jar
52-
*.war
53-
*.ear
54-
obj/
55-
bin/
56-
57-
# Pretrained
58-
*.bin
59-
pretrained/
60-
61-
# Datasets
62-
datasets/
63-
dataset.txt
64-
gpt_vocab.txt
65-
```
1+
(empty)

‎CMakeLists.txt‎

Lines changed: 215 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,227 @@
1-
cmake_minimum_required(VERSION 3.20)
1+
# CMake 3.30+ required for C++26 standard support (stable in 3.30+)
2+
cmake_minimum_required(VERSION 3.30)
23
project(neuralnet.cpp VERSION 1.0.0 LANGUAGES CXX)
34

45
set(CMAKE_CXX_STANDARD 26)
56
set(CMAKE_CXX_STANDARD_REQUIRED ON)
7+
set(CMAKE_CXX_EXTENSIONS OFF)
68

7-
# 为所有目标添加编译标志
8-
add_compile_options(-fexperimental-library -O3 -ffast-math -Wall -Wextra -Wpedantic -Werror)
9+
# ── 默认 Release 模式，可通过 -DCMAKE_BUILD_TYPE=Debug 切换 ────────────────
10+
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
11+
set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
12+
endif()
13+
14+
# ── 公共编译标志 ─────────────────────────────────────────────────────────
15+
# -fno-exceptions: 项目规范禁止异常，编译器层面强制执行
16+
# -Wall -Wextra -Wpedantic -Werror: 严格警告
17+
add_compile_options(-fno-exceptions -Wall -Wextra -Wpedantic -Werror)
18+
19+
# ── 编译器特有选项 ───────────────────────────────────────────────────────
20+
# -fexperimental-library 仅 Clang 识别；GCC 不识别会产生警告
21+
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
22+
add_compile_options(-fexperimental-library)
23+
endif()
24+
25+
# ── 按构建类型设置优化标志 ───────────────────────────────────────────────
26+
# Release: -O3 -funroll-loops -march=native（最大性能）
27+
# Debug: -O0 -g（调试优先）
28+
# 注意：不使用 -ffast-math，因为它会禁用 NaN/Infinity 支持，影响训练稳定性
29+
set(RELEASE_FLAGS -O3 -fno-math-errno -fno-trapping-math -funroll-loops -march=native)
30+
set(DEBUG_FLAGS -O0 -g)
31+
32+
add_compile_options(
33+
"$<$<CONFIG:Release>:${RELEASE_FLAGS}>"
34+
"$<$<CONFIG:Debug>:${DEBUG_FLAGS}>"
35+
)
36+
37+
# ── 线程库（Linux 需要 pthread） ─────────────────────────────────────────
38+
set(THREADS_PREFER_PTHREAD_FLAG ON)
39+
find_package(Threads REQUIRED)
40+
41+
# ── 公共 INTERFACE 库：所有目标共享 include 目录和链接库 ──────────────
42+
# 通过 INTERFACE 库消除 7 个目标的重复设置
43+
add_library(nn_core INTERFACE)
44+
target_include_directories(nn_core INTERFACE ${CMAKE_SOURCE_DIR}/include)
45+
target_link_libraries(nn_core INTERFACE Threads::Threads)
46+
47+
# ── 便捷函数：统一添加可执行目标 ────────────────────────────────────────
48+
function(nn_add_executable name)
49+
add_executable(${name} ${ARGN})
50+
target_link_libraries(${name} PRIVATE nn_core)
51+
endfunction()
952

1053
# ── MNIST 目标 ──────────────────────────────────────────────────────────────
11-
add_executable(mnist_train src/mnist_train.cpp)
12-
add_executable(mnist_infer src/mnist_infer.cpp)
54+
nn_add_executable(mnist_train src/mnist_train.cpp)
55+
nn_add_executable(mnist_infer src/mnist_infer.cpp)
56+
nn_add_executable(mnist_bench src/mnist_bench.cpp)
57+
58+
# ── 文本目标（GPT 训练/推理 + BPE 分词器训练/推理） ───────────────────
59+
nn_add_executable(text_train src/text_train.cpp)
60+
nn_add_executable(text_infer src/text_infer.cpp)
61+
nn_add_executable(tokenizer_train src/tokenizer_train.cpp)
62+
nn_add_executable(tokenizer_infer src/tokenizer_infer.cpp)
63+
64+
# ── 计算性能基准测试 ─────────────────────────────────────────────────────
65+
nn_add_executable(compute_bench src/compute_bench.cpp)
66+
nn_add_executable(bench_thresholds src/bench_thresholds.cpp)
67+
68+
# ── GPU 测试（需要 Vulkan） ──────────────────────────────────────────────
69+
nn_add_executable(gpu_test src/gpu_test.cpp)
70+
71+
# ── Attention 一致性验证（forward vs forward_step）──────────────────────
72+
nn_add_executable(attn_consistency_test src/attn_consistency_test.cpp)
73+
74+
# ── SwiGLU 数值梯度检查（gradcheck，验证 split/merge 反向正确性）──────────
75+
nn_add_executable(swiglu_gradcheck src/swiglu_gradcheck.cpp)
76+
77+
# ── RMSNorm 数值梯度检查（gradcheck，验证 gamma/输入梯度） ───────────────
78+
nn_add_executable(rmsnorm_gradcheck src/rmsnorm_gradcheck.cpp)
79+
80+
# ── Vulkan GPU 加速（可选） ────────────────────────────────────────────────
81+
find_package(Vulkan QUIET)
82+
if(Vulkan_FOUND)
83+
message(STATUS "Vulkan found: ${Vulkan_LIBRARY}")
84+
85+
# 查找 SPIR-V 编译器（Vulkan SDK 自带）
86+
find_program(GLSLC glslc HINTS ${Vulkan_GLSLC_EXECUTABLE})
87+
88+
# 定义着色器列表（仅保留纯原语着色器，不含任何算法）
89+
# - matmul / matmul_tiled: 矩阵乘法原语
90+
# - batched_matmul: 批量矩阵乘法原语（Z 维度索引 batch，消除 per-head 循环）
91+
# - elementwise_v2: 逐元素原语（unary/binary/select）— 纯 GPU 架构核心
92+
# - reduce: 行/列归约原语（sum/max）— LayerNorm、CrossEntropy 等
93+
# - broadcast: 行/列广播原语（add/sub/mul/div/max/min）— Linear 偏置、
94+
# LayerNorm 缩放、Softmax 归一化等
95+
# 旧的 elementwise / layernorm / layernorm_backward 着色器（含算法语义）
96+
# 已随旧架构删除。
97+
set(GPU_SHADERS
98+
matmul
99+
matmul_tiled
100+
batched_matmul
101+
rearrange_3d
102+
elementwise_v2
103+
reduce
104+
broadcast
105+
transpose
106+
gather
107+
scatter_add)
108+
set(GPU_SHADER_HPPS "")
109+
110+
foreach(SHADER_NAME ${GPU_SHADERS})
111+
set(SRC ${CMAKE_SOURCE_DIR}/shaders/${SHADER_NAME}.comp)
112+
set(SPV ${CMAKE_BINARY_DIR}/${SHADER_NAME}.spv)
113+
set(HPP ${CMAKE_BINARY_DIR}/generated/${SHADER_NAME}_spv.hpp)
114+
set(FUNC "nn_${SHADER_NAME}_spirv_bytecode")
115+
list(APPEND GPU_SHADER_SPVS ${SPV})
116+
list(APPEND GPU_SHADER_HPPS ${HPP})
117+
118+
# 编译 GLSL → SPIR-V
119+
add_custom_command(
120+
OUTPUT ${SPV}
121+
COMMAND ${GLSLC} -o ${SPV} ${SRC}
122+
DEPENDS ${SRC}
123+
COMMENT "[GPU] Compiling ${SHADER_NAME}.comp → ${SHADER_NAME}.spv"
124+
)
125+
126+
# 嵌入 SPIR-V → C++ 头文件
127+
add_custom_command(
128+
OUTPUT ${HPP}
129+
COMMAND ${CMAKE_COMMAND}
130+
-DSPV_INPUT=${SPV}
131+
-DHPP_OUTPUT=${HPP}
132+
-DFUNC_NAME=${FUNC}
133+
-P ${CMAKE_SOURCE_DIR}/cmake/embed_spirv.cmake
134+
DEPENDS ${SPV} ${CMAKE_SOURCE_DIR}/cmake/embed_spirv.cmake
135+
COMMENT "[GPU] Embedding ${SHADER_NAME} SPIR-V into C++ header"
136+
)
137+
endforeach()
138+
139+
if(NOT GLSLC)
140+
message(WARNING "No SPIR-V compiler found (glslc). "
141+
"GPU acceleration disabled. Install Vulkan SDK.")
142+
set(Vulkan_FOUND OFF)
143+
endif()
144+
145+
if(Vulkan_FOUND)
146+
add_custom_target(compile_gpu_shaders DEPENDS ${GPU_SHADER_HPPS})
147+
148+
# 为所有目标启用 Vulkan
149+
set(NN_GPU_TARGETS
150+
mnist_train mnist_infer mnist_bench
151+
text_train text_infer
152+
tokenizer_train tokenizer_infer
153+
compute_bench bench_thresholds gpu_test
154+
attn_consistency_test
155+
swiglu_gradcheck
156+
rmsnorm_gradcheck
157+
)
158+
159+
foreach(TARGET ${NN_GPU_TARGETS})
160+
target_compile_definitions(${TARGET} PRIVATE NN_HAS_VULKAN)
161+
target_include_directories(${TARGET} PRIVATE ${CMAKE_BINARY_DIR}/generated)
162+
target_link_libraries(${TARGET} PRIVATE Vulkan::Vulkan)
163+
add_dependencies(${TARGET} compile_gpu_shaders)
164+
endforeach()
13165

14-
target_include_directories(mnist_train PRIVATE ${CMAKE_SOURCE_DIR}/include)
15-
target_include_directories(mnist_infer PRIVATE ${CMAKE_SOURCE_DIR}/include)
166+
message(STATUS "GPU acceleration enabled (Vulkan compute)")
167+
endif()
168+
else()
169+
message(STATUS "Vulkan not found. GPU acceleration disabled.")
170+
endif()
16171

17-
# ── 文本生成目标 ────────────────────────────────────────────────────────────
18-
add_executable(text_train src/text_train.cpp)
19-
add_executable(text_infer src/text_infer.cpp)
172+
# ── 测试目标 ─────────────────────────────────────────────────────────────
173+
# 启用测试（默认 OFF，可通过 -DNN_BUILD_TESTS=ON 开启）
174+
option(NN_BUILD_TESTS "Build unit tests" OFF)
175+
if(NN_BUILD_TESTS)
176+
enable_testing()
177+
if(EXISTS "${CMAKE_SOURCE_DIR}/tests/test_const_span.cpp")
178+
nn_add_executable(test_const_span tests/test_const_span.cpp)
179+
add_test(NAME test_const_span COMMAND test_const_span)
180+
endif()
181+
endif()
20182

21-
target_include_directories(text_train PRIVATE ${CMAKE_SOURCE_DIR}/include)
22-
target_include_directories(text_infer PRIVATE ${CMAKE_SOURCE_DIR}/include)
183+
# ── CUDA GPU 加速（可选）──────────────────────────────────────────────────
184+
# CUDA 内核由 nvcc 编译为独立静态库，通过 C-linkage 接口链接到 Clang 主程序。
185+
# 在 Windows 上 nvcc 使用 MSVC 作为 host 编译器，与 LLVM/Clang 通过 C ABI 兼容。
186+
option(NN_ENABLE_CUDA "Enable CUDA GPU acceleration" OFF)
187+
if(NN_ENABLE_CUDA)
188+
find_package(CUDAToolkit QUIET)
189+
if(NOT CUDAToolkit_FOUND)
190+
message(WARNING "CUDA Toolkit not found. GPU acceleration disabled.")
191+
else()
192+
add_subdirectory(cuda)
193+
if(NN_CUDA_FOUND)
194+
set(NN_CUDA_TARGETS
195+
mnist_train mnist_infer mnist_bench
196+
text_train text_infer
197+
tokenizer_train tokenizer_infer
198+
compute_bench bench_thresholds gpu_test
199+
attn_consistency_test
200+
swiglu_gradcheck
201+
rmsnorm_gradcheck
202+
)
23203

24-
# ── 分词器目标 ──────────────────────────────────────────────────────────────
25-
add_executable(tokenizer_train src/tokenizer_train.cpp)
26-
add_executable(tokenizer_infer src/tokenizer_infer.cpp)
204+
foreach(TARGET ${NN_CUDA_TARGETS})
205+
target_compile_definitions(${TARGET} PRIVATE NN_HAS_CUDA NN_HAS_CUBLAS)
206+
target_include_directories(${TARGET} PRIVATE
207+
${NN_CUDA_INCLUDE_DIR}
208+
${CUDAToolkit_INCLUDE_DIRS}
209+
)
210+
# nvcc 编译的 .obj + CUDA 运行时 + cuBLAS（matmul 加速）
211+
target_link_libraries(${TARGET} PRIVATE
212+
${NN_CUDA_KERNELS_OBJ}
213+
CUDA::cudart
214+
CUDA::cublas
215+
CUDA::cublasLt
216+
)
217+
add_dependencies(${TARGET} compile_cuda_kernels)
218+
endforeach()
27219

28-
target_include_directories(tokenizer_train PRIVATE ${CMAKE_SOURCE_DIR}/include)
29-
target_include_directories(tokenizer_infer PRIVATE ${CMAKE_SOURCE_DIR}/include)
220+
message(STATUS "CUDA GPU acceleration enabled (nvcc → obj → link, cuBLAS enabled)")
221+
else()
222+
message(WARNING "CUDA not built. GPU acceleration disabled.")
223+
endif()
224+
endif()
225+
else()
226+
message(STATUS "CUDA disabled. Use -DNN_ENABLE_CUDA=ON to enable.")
227+
endif()

0 commit comments

Comments
 (0)