diff --git a/.gitignore b/.gitignore index efe16df2..cfa61fd1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,65 +1 @@ -# Compiled and build artifacts -*.o -*.obj -*.out -build/ -dist/ -datasets/ -pretrained/ - -# Dependencies -__pycache__/ -*.pyc -*.pyo -.Python -env/ -venv/ -.venv/ -.egg-info/ -*.egg - -# Logs and temp files -*.log -*.tmp -*.swp -*.swo - -# Environment -.env -.env.local -.env.* - -# Editors -.vscode/ -.idea/ -*.swp -*.swo - -# OS generated files -.DS_Store -Thumbs.db - -# Coverage reports -coverage/ -htmlcov/ -.coverage - -# Build systems -target/ -.gradle/ -*.class -*.jar -*.war -*.ear -obj/ -bin/ - -# Pretrained -*.bin -pretrained/ - -# Datasets -datasets/ -dataset.txt -gpt_vocab.txt -``` \ No newline at end of file +(empty) \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 10181c2a..b2a78d2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,29 +1,227 @@ -cmake_minimum_required(VERSION 3.20) +# CMake 3.30+ required for C++26 standard support (stable in 3.30+) +cmake_minimum_required(VERSION 3.30) project(neuralnet.cpp VERSION 1.0.0 LANGUAGES CXX) set(CMAKE_CXX_STANDARD 26) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) -# 为所有目标添加编译标志 -add_compile_options(-fexperimental-library -O3 -ffast-math -Wall -Wextra -Wpedantic -Werror) +# ── 默认 Release 模式，可通过 -DCMAKE_BUILD_TYPE=Debug 切换 ──────────────── +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) +endif() + +# ── 公共编译标志 ───────────────────────────────────────────────────────── +# -fno-exceptions: 项目规范禁止异常，编译器层面强制执行 +# -Wall -Wextra -Wpedantic -Werror: 严格警告 +add_compile_options(-fno-exceptions -Wall -Wextra -Wpedantic -Werror) + +# ── 编译器特有选项 ─────────────────────────────────────────────────────── +# -fexperimental-library 仅 Clang 识别；GCC 不识别会产生警告 +if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + add_compile_options(-fexperimental-library) +endif() + +# ── 按构建类型设置优化标志 ─────────────────────────────────────────────── +# Release: -O3 -funroll-loops -march=native（最大性能） +# Debug: -O0 -g（调试优先） +# 注意：不使用 -ffast-math，因为它会禁用 NaN/Infinity 支持，影响训练稳定性 +set(RELEASE_FLAGS -O3 -fno-math-errno -fno-trapping-math -funroll-loops -march=native) +set(DEBUG_FLAGS -O0 -g) + +add_compile_options( + "$<$:${RELEASE_FLAGS}>" + "$<$:${DEBUG_FLAGS}>" +) + +# ── 线程库（Linux 需要 pthread） ───────────────────────────────────────── +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +# ── 公共 INTERFACE 库：所有目标共享 include 目录和链接库 ────────────── +# 通过 INTERFACE 库消除 7 个目标的重复设置 +add_library(nn_core INTERFACE) +target_include_directories(nn_core INTERFACE ${CMAKE_SOURCE_DIR}/include) +target_link_libraries(nn_core INTERFACE Threads::Threads) + +# ── 便捷函数：统一添加可执行目标 ──────────────────────────────────────── +function(nn_add_executable name) + add_executable(${name} ${ARGN}) + target_link_libraries(${name} PRIVATE nn_core) +endfunction() # ── MNIST 目标 ────────────────────────────────────────────────────────────── -add_executable(mnist_train src/mnist_train.cpp) -add_executable(mnist_infer src/mnist_infer.cpp) +nn_add_executable(mnist_train src/mnist_train.cpp) +nn_add_executable(mnist_infer src/mnist_infer.cpp) +nn_add_executable(mnist_bench src/mnist_bench.cpp) + +# ── 文本目标（GPT 训练/推理 + BPE 分词器训练/推理） ─────────────────── +nn_add_executable(text_train src/text_train.cpp) +nn_add_executable(text_infer src/text_infer.cpp) +nn_add_executable(tokenizer_train src/tokenizer_train.cpp) +nn_add_executable(tokenizer_infer src/tokenizer_infer.cpp) + +# ── 计算性能基准测试 ───────────────────────────────────────────────────── +nn_add_executable(compute_bench src/compute_bench.cpp) +nn_add_executable(bench_thresholds src/bench_thresholds.cpp) + +# ── GPU 测试（需要 Vulkan） ────────────────────────────────────────────── +nn_add_executable(gpu_test src/gpu_test.cpp) + +# ── Attention 一致性验证（forward vs forward_step）────────────────────── +nn_add_executable(attn_consistency_test src/attn_consistency_test.cpp) + +# ── SwiGLU 数值梯度检查（gradcheck，验证 split/merge 反向正确性）────────── +nn_add_executable(swiglu_gradcheck src/swiglu_gradcheck.cpp) + +# ── RMSNorm 数值梯度检查（gradcheck，验证 gamma/输入梯度） ─────────────── +nn_add_executable(rmsnorm_gradcheck src/rmsnorm_gradcheck.cpp) + +# ── Vulkan GPU 加速（可选） ──────────────────────────────────────────────── +find_package(Vulkan QUIET) +if(Vulkan_FOUND) + message(STATUS "Vulkan found: ${Vulkan_LIBRARY}") + + # 查找 SPIR-V 编译器（Vulkan SDK 自带） + find_program(GLSLC glslc HINTS ${Vulkan_GLSLC_EXECUTABLE}) + + # 定义着色器列表（仅保留纯原语着色器，不含任何算法） + # - matmul / matmul_tiled: 矩阵乘法原语 + # - batched_matmul: 批量矩阵乘法原语（Z 维度索引 batch，消除 per-head 循环） + # - elementwise_v2: 逐元素原语（unary/binary/select）— 纯 GPU 架构核心 + # - reduce: 行/列归约原语（sum/max）— LayerNorm、CrossEntropy 等 + # - broadcast: 行/列广播原语（add/sub/mul/div/max/min）— Linear 偏置、 + # LayerNorm 缩放、Softmax 归一化等 + # 旧的 elementwise / layernorm / layernorm_backward 着色器（含算法语义） + # 已随旧架构删除。 + set(GPU_SHADERS + matmul + matmul_tiled + batched_matmul + rearrange_3d + elementwise_v2 + reduce + broadcast + transpose + gather + scatter_add) + set(GPU_SHADER_HPPS "") + + foreach(SHADER_NAME ${GPU_SHADERS}) + set(SRC ${CMAKE_SOURCE_DIR}/shaders/${SHADER_NAME}.comp) + set(SPV ${CMAKE_BINARY_DIR}/${SHADER_NAME}.spv) + set(HPP ${CMAKE_BINARY_DIR}/generated/${SHADER_NAME}_spv.hpp) + set(FUNC "nn_${SHADER_NAME}_spirv_bytecode") + list(APPEND GPU_SHADER_SPVS ${SPV}) + list(APPEND GPU_SHADER_HPPS ${HPP}) + + # 编译 GLSL → SPIR-V + add_custom_command( + OUTPUT ${SPV} + COMMAND ${GLSLC} -o ${SPV} ${SRC} + DEPENDS ${SRC} + COMMENT "[GPU] Compiling ${SHADER_NAME}.comp → ${SHADER_NAME}.spv" + ) + + # 嵌入 SPIR-V → C++ 头文件 + add_custom_command( + OUTPUT ${HPP} + COMMAND ${CMAKE_COMMAND} + -DSPV_INPUT=${SPV} + -DHPP_OUTPUT=${HPP} + -DFUNC_NAME=${FUNC} + -P ${CMAKE_SOURCE_DIR}/cmake/embed_spirv.cmake + DEPENDS ${SPV} ${CMAKE_SOURCE_DIR}/cmake/embed_spirv.cmake + COMMENT "[GPU] Embedding ${SHADER_NAME} SPIR-V into C++ header" + ) + endforeach() + + if(NOT GLSLC) + message(WARNING "No SPIR-V compiler found (glslc). " + "GPU acceleration disabled. Install Vulkan SDK.") + set(Vulkan_FOUND OFF) + endif() + + if(Vulkan_FOUND) + add_custom_target(compile_gpu_shaders DEPENDS ${GPU_SHADER_HPPS}) + + # 为所有目标启用 Vulkan + set(NN_GPU_TARGETS + mnist_train mnist_infer mnist_bench + text_train text_infer + tokenizer_train tokenizer_infer + compute_bench bench_thresholds gpu_test + attn_consistency_test + swiglu_gradcheck + rmsnorm_gradcheck + ) + + foreach(TARGET ${NN_GPU_TARGETS}) + target_compile_definitions(${TARGET} PRIVATE NN_HAS_VULKAN) + target_include_directories(${TARGET} PRIVATE ${CMAKE_BINARY_DIR}/generated) + target_link_libraries(${TARGET} PRIVATE Vulkan::Vulkan) + add_dependencies(${TARGET} compile_gpu_shaders) + endforeach() -target_include_directories(mnist_train PRIVATE ${CMAKE_SOURCE_DIR}/include) -target_include_directories(mnist_infer PRIVATE ${CMAKE_SOURCE_DIR}/include) + message(STATUS "GPU acceleration enabled (Vulkan compute)") + endif() +else() + message(STATUS "Vulkan not found. GPU acceleration disabled.") +endif() -# ── 文本生成目标 ──────────────────────────────────────────────────────────── -add_executable(text_train src/text_train.cpp) -add_executable(text_infer src/text_infer.cpp) +# ── 测试目标 ───────────────────────────────────────────────────────────── +# 启用测试（默认 OFF，可通过 -DNN_BUILD_TESTS=ON 开启） +option(NN_BUILD_TESTS "Build unit tests" OFF) +if(NN_BUILD_TESTS) + enable_testing() + if(EXISTS "${CMAKE_SOURCE_DIR}/tests/test_const_span.cpp") + nn_add_executable(test_const_span tests/test_const_span.cpp) + add_test(NAME test_const_span COMMAND test_const_span) + endif() +endif() -target_include_directories(text_train PRIVATE ${CMAKE_SOURCE_DIR}/include) -target_include_directories(text_infer PRIVATE ${CMAKE_SOURCE_DIR}/include) +# ── CUDA GPU 加速（可选）────────────────────────────────────────────────── +# CUDA 内核由 nvcc 编译为独立静态库，通过 C-linkage 接口链接到 Clang 主程序。 +# 在 Windows 上 nvcc 使用 MSVC 作为 host 编译器，与 LLVM/Clang 通过 C ABI 兼容。 +option(NN_ENABLE_CUDA "Enable CUDA GPU acceleration" OFF) +if(NN_ENABLE_CUDA) + find_package(CUDAToolkit QUIET) + if(NOT CUDAToolkit_FOUND) + message(WARNING "CUDA Toolkit not found. GPU acceleration disabled.") + else() + add_subdirectory(cuda) + if(NN_CUDA_FOUND) + set(NN_CUDA_TARGETS + mnist_train mnist_infer mnist_bench + text_train text_infer + tokenizer_train tokenizer_infer + compute_bench bench_thresholds gpu_test + attn_consistency_test + swiglu_gradcheck + rmsnorm_gradcheck + ) -# ── 分词器目标 ────────────────────────────────────────────────────────────── -add_executable(tokenizer_train src/tokenizer_train.cpp) -add_executable(tokenizer_infer src/tokenizer_infer.cpp) + foreach(TARGET ${NN_CUDA_TARGETS}) + target_compile_definitions(${TARGET} PRIVATE NN_HAS_CUDA NN_HAS_CUBLAS) + target_include_directories(${TARGET} PRIVATE + ${NN_CUDA_INCLUDE_DIR} + ${CUDAToolkit_INCLUDE_DIRS} + ) + # nvcc 编译的 .obj + CUDA 运行时 + cuBLAS（matmul 加速） + target_link_libraries(${TARGET} PRIVATE + ${NN_CUDA_KERNELS_OBJ} + CUDA::cudart + CUDA::cublas + CUDA::cublasLt + ) + add_dependencies(${TARGET} compile_cuda_kernels) + endforeach() -target_include_directories(tokenizer_train PRIVATE ${CMAKE_SOURCE_DIR}/include) -target_include_directories(tokenizer_infer PRIVATE ${CMAKE_SOURCE_DIR}/include) + message(STATUS "CUDA GPU acceleration enabled (nvcc → obj → link, cuBLAS enabled)") + else() + message(WARNING "CUDA not built. GPU acceleration disabled.") + endif() + endif() +else() + message(STATUS "CUDA disabled. Use -DNN_ENABLE_CUDA=ON to enable.") +endif() diff --git a/README.md b/README.md index 09e8d8b4..dc329029 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,79 @@ # neuralnet.cpp -一个轻量级 C++26 神经网络库，从头实现（无第三方深度学习框架依赖），支持并行计算。 +一个从零实现的 C++26 神经网络库，支持 CPU/GPU 双后端、多层 Transformer、GPT 语言模型训练与推理。 + +## 📚 文档 + +| 文档 | 说明 | +|------|------| +| [架构设计](docs/01-architecture.md) | 项目分层架构、引擎化设计、数据流、模块详解 | +| [性能优化](docs/02-performance.md) | SmartPolicy、线程池、缓存分块、GPU 加速、算子融合等 | +| [快速上手：构建模型](docs/03-quickstart-model.md) | ComputeEngine/Layer/Model 三件套使用教程 | +| [快速上手：训练与推理](docs/04-quickstart-train-infer.md) | MNIST/GPT 训练推理命令行 + C++ API 示例 + GUI 操作指南 | +| [算法解析](docs/05-algorithm-reference.md) | 每个 Layer/Loss/Optimizer 的数学原理与原语分解 | +| [CUDA 后端](docs/06-cuda-backend.md) | CUDA GPU 加速后端设计、构建方法、编译器兼容性 | +| [开发规范](docs/DEVELOPMENT_STANDARDS.md) | C++ 编码规范、模块隔离、内存管理 | ## 项目结构 ``` neuralnet.cpp/ -├── .gitignore ├── CMakeLists.txt -├── ARCHITECTURE.md ← 架构文档 -├── DEVELOPMENT_STANDARDS.md ← 开发规范 ├── README.md -├── gui.py ← 图形化操作界面 (Tkinter) -├── csv_png.py ← CSV 转 PNG 图像 -├── extract_digits.py ← 从 MNIST CSV 提取单个数字 -├── save_dataset.py ← 下载 MNIST 数据集 -├── build/ ← CMake 构建输出 -│ ├── mnist_train.exe -│ └── mnist_infer.exe -├── include/ -│ └── neuralnet.cpp/ -│ ├── nn.hpp ← 统一入口头文件 -│ ├── nn_config.hpp ← SmartPolicy、BLOCK_SIZE 等配置 -│ ├── thread_pool.hpp ← 全局线程池 -│ ├── matrix.hpp ← Matrix 类 -│ ├── layer.hpp ← Layer 基类 + Linear/ReLU/GeLU/LayerNorm/Softmax/MultiHeadAttention/PositionalEncoding -│ ├── loss.hpp ← Loss 基类 + MSELoss/CrossEntropyLoss -│ ├── optimizer.hpp ← SGD / SGD_w_Momentum / Adam -│ ├── model.hpp ← Model 容器（链式 add<>） -│ ├── model_io.hpp ← 二进制模型序列化 -│ └── mnist_common.hpp ← MNIST 常量与 build_mnist_model() +├── docs/ +│ ├── 01-architecture.md ← 架构设计文档 +│ ├── 02-performance.md ← 性能优化文档 +│ ├── 03-quickstart-model.md ← 模型构建教程 +│ ├── 04-quickstart-train-infer.md ← 训练推理教程 +│ ├── 05-algorithm-reference.md ← 算法解析参考 +│ ├── DEVELOPMENT_STANDARDS.md ← 开发规范 +│ ├── GPU_DESIGN_V2.md ← GPU 后端设计 (Vulkan) +│ └── 06-cuda-backend.md ← CUDA 后端设计 +├── gui.py ← 图形化操作界面 (Tkinter) +├── include/neuralnet.cpp/ +│ ├── nn.hpp ← 统一入口头文件 +│ ├── config.hpp ← SmartPolicy、BLOCK_SIZE +│ ├── tensor.hpp ← 统一跨设备张量 +│ ├── compute_engine.hpp ← 引擎抽象接口 +│ ├── cpu_engine.hpp ← CPU 引擎 +│ ├── gpu_engine.hpp ← GPU 引擎 (Vulkan) +│ ├── cuda_engine.hpp ← GPU 引擎 (CUDA) +│ ├── compute_layer.hpp ← Layer 基类 + 所有层 +│ ├── compute_loss.hpp ← 损失函数 +│ ├── compute_optimizer.hpp ← 优化器 (SGD/Adam/AdamW/Muon) +│ ├── model_container.hpp ← Model 容器 +│ ├── model_spec.hpp ← 架构描述 +│ ├── model_serialization.hpp ← 二进制序列化 +│ ├── domain_mnist.hpp ← MNIST 模型工厂 +│ ├── domain_gpt.hpp ← GPT 模型工厂 +│ ├── domain_tokenizer.hpp ← 分词器 +│ ├── tensor.hpp ← 统一跨设备张量 +│ ├── algebra_matrix.hpp ← 矩阵类 +│ ├── algebra_expr.hpp ← 表达式模板 +│ ├── algebra_ops.hpp ← 逐元素算子 +│ ├── algebra_span.hpp ← Span 抽象 +│ ├── algebra_compute.hpp ← 计算分派 +│ ├── core_threadpool.hpp ← 全局线程池 +│ ├── core_errors.hpp ← Result +│ ├── core_assert.hpp ← 断言宏 +│ ├── core_file.hpp ← 文件工具 +│ ├── core_observer_ptr.hpp ← 观察者指针 +│ └── backend/ ← GPU 后端 (Vulkan) ├── src/ -│ ├── train.cpp -│ └── infer.cpp -├── datasets/ -│ ├── mnist_data/ ← MNIST CSV 数据 -│ └── test/ ← 按数字分类的测试图片 -└── pretrained/ - └── mnist_model.bin ← 预训练模型 +│ ├── mnist_train.cpp ← MNIST 训练 +│ ├── mnist_infer.cpp ← MNIST 推理 +│ ├── text_train.cpp ← GPT 文本训练 +│ ├── text_infer.cpp ← GPT 文本推理 +│ ├── tokenizer_train.cpp ← 分词器训练 +│ ├── tokenizer_infer.cpp ← 分词器推理 +│ ├── compute_bench.cpp ← 性能基准测试 +│ ├── bench_thresholds.cpp ← 并行阈值测试 +│ └── gpu_test.cpp ← GPU 后端测试 +├── datasets/ ← 训练数据 +├── pretrained/ ← 预训练模型 +├── scripts/ ← 数据处理/训练辅助脚本 +├── shaders/ ← GPU Compute Shader +└── compare_with_torch/ ← PyTorch 对比实现 ``` [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/EthanPeng-2048/neuralnet.cpp) @@ -80,8 +116,8 @@ python save_dataset.py # 自定义参数 ./build/mnist_train --epochs 20 --lr 0.001 --batch-size 32 --optimizer adam --save my_model.bin -# 支持的优化器: sgd / sgd_w_momentum / adam -./build/mnist_train --optimizer sgd_w_momentum +# 支持的优化器: sgd / sgd_momentum / adam +./build/mnist_train --optimizer sgd_momentum ``` ### 运行推理 @@ -105,38 +141,56 @@ python save_dataset.py ### 🖥️ 图形化界面 (GUI) -提供了一个基于 tkinter 的图形化界面，方便进行训练、推理和图片查看操作。 +基于 tkinter 的全功能图形化界面，覆盖 MNIST 训练/推理、GPT 训练/推理、分词器训练/推理，无需记忆命令行参数。 ```bash # 启动 GUI python gui.py ``` -> **前提**: 需要先构建 C++ 项目（`cmake -B build -G Ninja`），GUI 会调用 build 目录下的可执行文件。 +> **前提**: 需要先构建 C++ 项目（`cmake -B build -G Ninja`），GUI 会调用 `build/` 目录下的可执行文件。 #### GUI 功能一览 | Tab | 功能 | |-----|------| -| 🏋️ **训练** | 配置数据集路径、模型保存路径、轮数、学习率、批大小、优化器、模型类型（MLP/Transformer），支持恢复训练；实时显示训练日志 | -| 🔍 **推理** | 选择模型文件和图片 CSV 文件/目录，支持 MLP/Transformer 模型类型切换，显示 Top-K 预测结果置信度条形图和手写数字图片预览；内置 ✍️ 手写板，可直接在画布上书写数字进行实时识别 | -| 🖼️ **图片查看** | 浏览单张 CSV 图片或批量加载目录，支持前后翻页导航 | +| 🏋️ **训练** | MNIST 模型训练：支持 MLP / Transformer 架构、超参数调节、恢复训练、学习率调度（fixed / cosine）、振荡抑制、GPU 加速；实时训练曲线 | +| 🔍 **推理** | MNIST 图片推理：Top-K 预测 + 置信度条形图 + 图片预览；内置 ✍️ **手写板**，鼠标书写数字即刻识别 | +| 🖼️ **图片查看** | 浏览 CSV 格式手写数字图片，前后翻页导航 | +| 📝 **GPT 训练** | GPT 语言模型训练：架构参数、位置编码（learned / sinusoidal / alibi）、学习率调度、振荡抑制；实时 loss 曲线 | +| 💬 **GPT 推理** | GPT 文本生成：温度调节、交互模式、Token ID 调试输出 | +| 🔤 **分词器训练** | 训练 BPE / CharBPE 分词器，配置词表大小和最小合并频率 | +| 🔡 **分词器推理** | 分词器编码/解码测试，支持文本编码、ID 解码、文件编码 | #### GUI 操作流程 -**训练模式：** -1. 配置数据集路径、模型保存路径等参数 -2. 选择模型类型（`mlp` 或 `transformer`）和优化器 -3. 可选勾选"从已有模型恢复训练"并指定模型路径 -4. 点击"▶ 开始训练"，训练日志实时滚动输出 -5. 训练过程中可随时点击"⏹ 停止"终止训练 - -**推理模式：** -1. 选择模型文件（`.bin`）并指定模型类型（需与训练时一致） -2. 通过文件选择或直接输入路径指定 CSV 图片文件/目录 -3. 设置 Top-K 值控制显示预测结果数量 -4. 点击"▶ 开始推理"，结果以置信度条形图 + 图片预览形式展示 -5. 也可使用内置手写板：在画布上书写数字 → 点击"🔍 识别手写数字" → 自动下采样为 28×28 并推理 +**MNIST 训练：** +1. 选择数据集目录（默认 `datasets/mnist_data`）和模型保存路径 +2. 配置超参数：轮数、学习率、批大小、优化器（`sgd`/`adam`/`adamw`/`muon`） +3. 选择模型类型：`mlp`（自定义层维度）或 `transformer`（模型维度/头数/层数/FFN/Patch） +4. 可选：恢复训练、GPU 加速、学习率调度、振荡抑制 +5. 点击 **"▶ 开始训练"**，日志面板实时输出，下方曲线图绘制 loss / accuracy +6. 训练中可随时点击 **"⏹ 停止"** 终止 + +**MNIST 推理：** +1. 选择训练好的模型文件（`.bin`） +2. 输入 CSV 图片路径或目录，设置 Top-K +3. 点击 **"▶ 开始推理"**，左侧显示图片预览，右侧显示置信度条形图 +4. **手写板模式**：在白色画布上书写数字 → 右侧 28×28 预览实时显示 → 点击 **"🔍 识别手写数字"** + +**GPT 训练：** +1. 选择训练文本文件（`.txt`）和词表 JSON +2. 配置模型架构（维度/头数/层数/FFN）和位置编码 +3. 点击 **"▶ 开始训练"**，实时显示 loss 曲线 + +**GPT 推理：** +1. 选择模型文件和词表，设置温度和最大 token 数 +2. 输入提示文本，点击 **"▶ 生成文本"** +3. 支持勾选"交互式生成模式"进行多轮生成 + +**分词器训练/推理：** +- 训练：选择文本文件 → 设置类型（`bpe`/`charbpe`）和词表大小 → 点击训练 +- 推理：选择词表 → 输入文本或 ID → 选择编码/解码 → 点击执行 ## 网络结构 @@ -184,7 +238,7 @@ python gui.py | `nn::MSELoss` | 均方误差损失 | | `nn::CrossEntropyLoss` | 交叉熵损失（含数值稳定 Softmax，`loss.hpp` 中定义） | | `nn::SGD` | 随机梯度下降优化器 | -| `nn::SGD_w_Momentum` | 动量 SGD 优化器 | +| `nn::SGDWithMomentum` | 动量 SGD 优化器 | | `nn::Adam` | Adam 优化器（一阶/二阶矩估计） | | `nn::SmartPolicy` | 自适应并行策略：小矩阵串行，大矩阵线程池并行 | | `nn::ThreadPool` | 全局单例线程池（懒初始化） | diff --git a/__pycache__/gui.cpython-312.pyc b/__pycache__/gui.cpython-312.pyc new file mode 100644 index 00000000..2ae2cf99 Binary files /dev/null and b/__pycache__/gui.cpython-312.pyc differ diff --git a/cmake/embed_spirv.cmake b/cmake/embed_spirv.cmake new file mode 100644 index 00000000..cdbb16d5 --- /dev/null +++ b/cmake/embed_spirv.cmake @@ -0,0 +1,66 @@ +# ── embed_spirv.cmake ──────────────────────────────────────────────────── +# 将 SPIR-V 二进制文件转换为 C++ 头文件（constexpr 数组）。 +# +# 用法（由 CMakeLists.txt 调用）： +# cmake -DSPV_INPUT=xxx.spv -DHPP_OUTPUT=xxx_spv.hpp -DFUNC_NAME=nn_xxx_spirv_bytecode -P embed_spirv.cmake +# +# 输出的头文件定义了： +# nn::FUNC_NAME() → const std::vector& +# ───────────────────────────────────────────────────────────────────────── + +file(READ "${SPV_INPUT}" HEX_CONTENT HEX) +string(LENGTH "${HEX_CONTENT}" HEX_LENGTH) +math(EXPR BYTE_COUNT "${HEX_LENGTH} / 2") +math(EXPR UINT32_COUNT "${BYTE_COUNT} / 4") + +if(UINT32_COUNT LESS 1) + message(FATAL_ERROR "SPIR-V file is empty or invalid: ${SPV_INPUT}") +endif() + +set(ARRAY_INIT "") +math(EXPR LAST_INDEX "${UINT32_COUNT} - 1") + +foreach(I RANGE ${LAST_INDEX}) + math(EXPR OFFSET "${I} * 8") + string(SUBSTRING "${HEX_CONTENT}" ${OFFSET} 8 HEX_WORD) + + # 提取 4 个字节（SPIR-V 是小端序） + string(SUBSTRING "${HEX_WORD}" 0 2 B0) + string(SUBSTRING "${HEX_WORD}" 2 2 B1) + string(SUBSTRING "${HEX_WORD}" 4 2 B2) + string(SUBSTRING "${HEX_WORD}" 6 2 B3) + + # 反转字节序得到 uint32_t 的十六进制表示 + if(NOT "${ARRAY_INIT}" STREQUAL "") + string(APPEND ARRAY_INIT ", ") + endif() + math(EXPR REM "${I} % 8") + if(${REM} EQUAL 0 AND NOT ${I} EQUAL 0) + string(APPEND ARRAY_INIT "\n ") + endif() + string(APPEND ARRAY_INIT "0x${B3}${B2}${B1}${B0}") +endforeach() + +# 写入生成的头文件 +file(WRITE "${HPP_OUTPUT}" +"// ── 自动生成，请勿手动编辑 ──\n" +"// 由 cmake/embed_spirv.cmake 从 SPIR-V 字节码生成\n" +"#pragma once\n" +"\n" +"#include \n" +"#include \n" +"\n" +"namespace nn {\n" +"\n" +"[[nodiscard]] inline const std::vector& ${FUNC_NAME}()\n" +"{\n" +" static const std::vector spirv = {\n" +" ${ARRAY_INIT}\n" +" };\n" +" return spirv;\n" +"}\n" +"\n" +"} // namespace nn\n" +) + +message(STATUS "Embedded SPIR-V: ${UINT32_COUNT} words (${BYTE_COUNT} bytes) → ${HPP_OUTPUT}") diff --git a/compare_with_torch/model.py b/compare_with_torch/model.py new file mode 100644 index 00000000..776e4d85 --- /dev/null +++ b/compare_with_torch/model.py @@ -0,0 +1,324 @@ +# ── model.py — GPT 模型 (PyTorch CUDA) ────────────────────────────────────── +# +# 复制自 C++ 项目 include/neuralnet.cpp/compute_layer.hpp 中的 GPTModel。 +# 架构完全一致： +# GPTModel = TokenEmb + PosEmb + N × GPTBlock + LayerNorm + LM Head +# GPTBlock (Pre-Norm): +# x = x + CausalSelfAttn(LN1(x)) +# x = x + FFN(LN2(x)) +# CausalSelfAttention: 独立 Q/K/V/O 投影 (非合并 QKV), scale = 1/sqrt(d_k) +# GeLU: QuickGeLU = x * sigmoid(1.702 * x) +# FeedForward: Linear(d_model→d_ff) → GeLU → Linear(d_ff→d_model) +# +# 初始化（与 C++ 一致）： +# Linear 权重: Xavier 均匀分布 limit = sqrt(6/(in+out)) +# Linear 偏置: 0 +# token_emb / pos_emb: N(0, 0.02) +# LayerNorm: gamma=1, beta=0 +# ───────────────────────────────────────────────────────────────────────── + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class QuickGeLU(nn.Module): + """QuickGeLU: x * sigmoid(β * x), β = 1.702 + + 与 C++ GeLU 层完全一致（非标准 nn.GELU）。 + """ + + BETA = 1.702 + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x * torch.sigmoid(self.BETA * x) + + +class CausalSelfAttention(nn.Module): + """因果多头自注意力（复制自 C++ CausalSelfAttention）。 + + - 独立 Q/K/V/O 投影（不使用合并 QKV） + - scale = 1 / sqrt(d_k), d_k = d_model / num_heads + - 因果掩码: j <= i 允许, 否则 -inf + """ + + def __init__(self, d_model: int, num_heads: int, max_len: int = 1024): + super().__init__() + assert d_model % num_heads == 0, "d_model must be divisible by num_heads" + self.d_model = d_model + self.num_heads = num_heads + self.d_k = d_model // num_heads + self.scale = 1.0 / math.sqrt(self.d_k) + + # 四个独立投影（与 C++ 一致，非合并 QKV） + self.w_q = nn.Linear(d_model, d_model) + self.w_k = nn.Linear(d_model, d_model) + self.w_v = nn.Linear(d_model, d_model) + self.w_o = nn.Linear(d_model, d_model) + + # 因果掩码: 上三角（不含对角线）为 -inf + mask = torch.triu( + torch.full((max_len, max_len), float("-inf")), diagonal=1 + ) + self.register_buffer("mask", mask, persistent=False) + + self._init_weights() + + def _init_weights(self): + for layer in (self.w_q, self.w_k, self.w_v, self.w_o): + nn.init.xavier_uniform_(layer.weight) + nn.init.zeros_(layer.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # x: (batch, seq, d_model) + B, T, C = x.shape + + Q = self.w_q(x) + K = self.w_k(x) + V = self.w_v(x) + + # (batch, num_heads, seq, d_k) + Q = Q.view(B, T, self.num_heads, self.d_k).transpose(1, 2) + K = K.view(B, T, self.num_heads, self.d_k).transpose(1, 2) + V = V.view(B, T, self.num_heads, self.d_k).transpose(1, 2) + + # scores: (batch, num_heads, seq, seq) — 与 C++ batched_matmul(Q,K,H,true,false) 等价 + scores = torch.matmul(Q, K.transpose(-2, -1)) * self.scale + + # 施加因果掩码（C++ 在 scale 后 add mask，顺序一致） + scores = scores + self.mask[:T, :T] + + # 行级 softmax（C++ Softmax 按 row 归约） + attn = F.softmax(scores, dim=-1) + + # (batch, num_heads, seq, d_k) + out = torch.matmul(attn, V) + + # 合并头: (batch, seq, d_model) + out = out.transpose(1, 2).contiguous().view(B, T, C) + + return self.w_o(out) + + +class FeedForward(nn.Module): + """FFN(x) = Linear2(GeLU(Linear1(x)))（复制自 C++ FeedForward）。""" + + def __init__(self, d_model: int, d_ff: int): + super().__init__() + self.fc1 = nn.Linear(d_model, d_ff) + self.fc2 = nn.Linear(d_ff, d_model) + self.gelu = QuickGeLU() + self._init_weights() + + def _init_weights(self): + for layer in (self.fc1, self.fc2): + nn.init.xavier_uniform_(layer.weight) + nn.init.zeros_(layer.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.fc2(self.gelu(self.fc1(x))) + + +class GPTBlock(nn.Module): + """Pre-Norm 解码器块（复制自 C++ GPTBlock）。 + + x = x + CausalSelfAttn(LN1(x)) + x = x + FFN(LN2(x)) + """ + + def __init__(self, d_model: int, num_heads: int, d_ff: int, max_len: int = 1024): + super().__init__() + self.self_attn = CausalSelfAttention(d_model, num_heads, max_len) + self.norm1 = nn.LayerNorm(d_model, eps=1e-5) + self.ff = FeedForward(d_model, d_ff) + self.norm2 = nn.LayerNorm(d_model, eps=1e-5) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x + self.self_attn(self.norm1(x)) + x = x + self.ff(self.norm2(x)) + return x + + +class GPTModel(nn.Module): + """Decoder-only Transformer 语言模型（复制自 C++ GPTModel）。 + + 组件: TokenEmb + PosEmb + N × GPTBlock + LayerNorm + LM Head + 输入: (batch, seq) LongTensor — token ID + 输出: (batch, seq, vocab_size) — 每个位置的 logits + + 与 C++ 的差异仅在内存布局： + C++: (seq, batch) → (vocab, seq*batch) [列优先] + Torch: (batch, seq) → (batch, seq, vocab) [行优先] + 数学语义完全等价。 + """ + + def __init__( + self, + vocab_size: int, + d_model: int, + seq_len: int, + num_heads: int, + d_ff: int, + num_layers: int, + ): + super().__init__() + if d_model % num_heads != 0: + raise ValueError("d_model must be divisible by num_heads") + + self.vocab_size = vocab_size + self.d_model = d_model + self.seq_len = seq_len + self.num_heads = num_heads + self.d_ff = d_ff + self.num_layers = num_layers + + # 可学习嵌入（C++ 用 N(0, 0.02) 初始化） + self.token_emb = nn.Embedding(vocab_size, d_model) + self.pos_emb = nn.Embedding(seq_len, d_model) + + self.blocks = nn.ModuleList( + [ + GPTBlock(d_model, num_heads, d_ff, max_len=seq_len) + for _ in range(num_layers) + ] + ) + self.ln_f = nn.LayerNorm(d_model, eps=1e-5) + # LM Head: Linear(d_model → vocab_size)，不与 token_emb 共享权重（与 C++ 一致） + self.lm_head = nn.Linear(d_model, vocab_size) + + self._init_weights() + + def _init_weights(self): + # 嵌入: N(0, 0.02) + nn.init.normal_(self.token_emb.weight, mean=0.0, std=0.02) + nn.init.normal_(self.pos_emb.weight, mean=0.0, std=0.02) + # LM Head: Xavier 均匀 + 零偏置 + nn.init.xavier_uniform_(self.lm_head.weight) + nn.init.zeros_(self.lm_head.bias) + + def forward( + self, + tokens: torch.Tensor, + targets: torch.Tensor | None = None, + loss_mask: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """前向传播。 + + Args: + tokens: (batch, seq) LongTensor + targets: (batch, seq) LongTensor 或 None。若提供则计算交叉熵损失。 + loss_mask: (batch, seq) FloatTensor 或 None。1.0=参与 loss, 0.0=忽略。 + 与 C++ forward_sparse 的 loss_mask 语义一致： + 只对 mask>0.5 的位置计算 loss 和梯度。 + + Returns: + (logits, loss) + logits: (batch, seq, vocab_size) + loss: 标量或 None + """ + B, T = tokens.shape + positions = torch.arange(T, device=tokens.device).unsqueeze(0) # (1, T) + + # Token 嵌入 + 位置嵌入 + x = self.token_emb(tokens) + self.pos_emb(positions) # (B, T, d_model) + + # Transformer 块 + for blk in self.blocks: + x = blk(x) + + # 最终 LayerNorm + LM Head + x = self.ln_f(x) + logits = self.lm_head(x) # (B, T, vocab_size) + + loss = None + if targets is not None: + if loss_mask is not None: + # 与 C++ forward_sparse 一致： + # 只对 mask>0.5 的位置计算 loss，loss = -sum(log_softmax) / num_valid + # 梯度在 mask=0 的位置清零 + per_token_loss = F.cross_entropy( + logits.view(-1, self.vocab_size), + targets.view(-1), + reduction="none", + ) # (B*T,) + mask_flat = loss_mask.view(-1) # (B*T,) + masked_loss = per_token_loss * mask_flat + num_valid = mask_flat.sum().clamp(min=1.0) + loss = masked_loss.sum() / num_valid + else: + # 无 mask：对所有位置计算 loss（旧行为） + loss = F.cross_entropy( + logits.view(-1, self.vocab_size), + targets.view(-1), + reduction="mean", + ) + + return logits, loss + + @torch.no_grad() + def generate( + self, + prompt: list[int], + max_new_tokens: int, + temperature: float = 1.0, + eos_token_id: int | None = None, + min_new_tokens: int = 0, + ) -> list[int]: + """自回归采样生成（复制自 C++ GPTModel::generate）。 + + Args: + prompt: prompt token ID 列表 + max_new_tokens: 最大生成 token 数 + temperature: 0 = 贪心, >0 = 采样（1.0 不缩放但仍采样，与 C++ 一致） + eos_token_id: 遇到此 token 停止（min_new_tokens 之后才检查） + min_new_tokens: 前这么多个 token 内不检查 EOS + + Returns: + 生成的 token ID 列表（不含 prompt，含触发停止的 EOS 之前的 token） + """ + device = next(self.parameters()).device + context = list(prompt) + generated: list[int] = [] + + for step in range(max_new_tokens): + # 滑动窗口：超过 seq_len 时只取最后 seq_len 个 + start = 0 + if len(context) > self.seq_len: + start = len(context) - self.seq_len + + cur_len = len(context) - start + x = torch.tensor( + [context[start:]], dtype=torch.long, device=device + ) # (1, cur_len) + + logits, _ = self.forward(x) + last_logits = logits[0, -1, :] # (vocab,) + + # temperature（仅当 >0 且 !=1.0 时应用，与 C++ 一致） + if temperature > 0.0 and temperature != 1.0: + last_logits = last_logits / temperature + + # 数值稳定 softmax + probs = F.softmax(last_logits, dim=-1) + + # 采样 or 贪心（与 C++ 分支一致：temperature > 0 采样，0 贪心） + if temperature > 0.0: + # 防御：probs 含 NaN（logits 为 -inf 时 softmax 可能 NaN）则回退贪心 + if torch.isnan(probs).any(): + next_token = torch.argmax(last_logits).item() + else: + next_token = torch.multinomial(probs, num_samples=1).item() + else: + next_token = torch.argmax(probs).item() + + context.append(next_token) + + # EOS 检查（min_new_tokens 之后才允许停止） + if step >= min_new_tokens and eos_token_id is not None and next_token == eos_token_id: + break + + generated.append(next_token) + + return generated diff --git a/compare_with_torch/text_infer.py b/compare_with_torch/text_infer.py new file mode 100644 index 00000000..b2f4d3d6 --- /dev/null +++ b/compare_with_torch/text_infer.py @@ -0,0 +1,221 @@ +# ── text_infer.py — GPT 文本生成推理 (PyTorch CUDA) ──────────────────────── +# +# 复制自 C++ 项目 src/text_infer.cpp。 +# 数据流： +# 加载模型 (含嵌入 tokenizer) → tokenizer.encode(prompt) → GPTModel.generate +# → tokenizer.decode(generated) → 输出文本 +# +# 与 C++ 一致： +# - prompt 前自动加 BOS（使推理输入格式与训练时一致） +# - min_new_tokens = max_new_tokens / 2，避免模型一上来就输出 EOS +# ───────────────────────────────────────────────────────────────────────── + +import argparse +import os +import sys +import time + +import torch + +# 确保能 import 同目录下的模块 +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from tokenizer import ( + TokenizerAdapter, + CharBPEAdapter, + load_tokenizer_from_file, + load_tokenizer_from_string, + load_charbpe_from_string, +) +from model import GPTModel + + +def parse_args(): + parser = argparse.ArgumentParser(description="GPT 文本生成推理程序 (PyTorch CUDA)") + parser.add_argument("--model", default="gpt_model_torch.pt", help="模型文件路径") + parser.add_argument( + "--vocab", + default="gpt_bpe.json", + help="词表 JSON 路径（仅当模型未嵌入 tokenizer 时使用）", + ) + parser.add_argument("--prompt", default="Hello", help="输入提示文本") + parser.add_argument("--interactive", action="store_true", help="交互式生成模式") + parser.add_argument("--max-tokens", type=int, default=200, help="最大生成 token 数") + parser.add_argument( + "--temperature", type=float, default=1.0, help="温度参数 (0=贪心, 1.0=贪心)" + ) + parser.add_argument( + "--device", + default="cuda", + help="cuda / cpu / cuda:0（默认 cuda，不可用自动回退 cpu）", + ) + parser.add_argument("--show-tokens", action="store_true", help="显示 token ID (调试用)") + return parser.parse_args() + + +def load_model_and_tokenizer(cfg): + """加载模型 + tokenizer（优先从模型嵌入，回退到外部 JSON）。""" + if not os.path.exists(cfg.model): + print(f"模型文件不存在: {cfg.model}", file=sys.stderr) + sys.exit(1) + + # 设备选择 + if cfg.device == "cuda" and not torch.cuda.is_available(): + print("CUDA 不可用，回退到 CPU") + device = torch.device("cpu") + else: + device = torch.device(cfg.device) + + # checkpoint 仅含 config/state_dict 等安全类型，weights_only=True + # 防止加载不可信 pickle 时执行任意代码 + ckpt = torch.load(cfg.model, map_location=device, weights_only=True) + config = ckpt["config"] + + print( + f"模型规格: vocab={config['vocab_size']}" + f" d_model={config['d_model']}" + f" heads={config['num_heads']}" + f" layers={config['num_layers']}" + f" d_ff={config['d_ff']}" + f" seq_len={config['seq_len']}" + ) + + model = GPTModel( + vocab_size=config["vocab_size"], + d_model=config["d_model"], + seq_len=config["seq_len"], + num_heads=config["num_heads"], + d_ff=config["d_ff"], + num_layers=config["num_layers"], + ).to(device) + model.load_state_dict(ckpt["model_state_dict"]) + model.eval() + print(f"加载模型: {cfg.model}") + + # 加载 tokenizer + tokenizer: TokenizerAdapter | CharBPEAdapter | None = None + embedded_json = ckpt.get("tokenizer_json", "") + if embedded_json: + # 先尝试 CharBPE 格式，再回退到 tokenizers 库格式 + tokenizer = load_charbpe_from_string(embedded_json) + if tokenizer is not None: + print("已从模型文件加载嵌入 CharBPE tokenizer") + else: + tokenizer = load_tokenizer_from_string(embedded_json) + if tokenizer is None: + print("解析嵌入 tokenizer 失败", file=sys.stderr) + sys.exit(1) + print("已从模型文件加载嵌入 tokenizer") + else: + tokenizer = load_charbpe_from_file(cfg.vocab) + if tokenizer is not None: + print(f"已从外部文件加载 CharBPE tokenizer: {cfg.vocab}") + else: + tokenizer = load_tokenizer_from_file(cfg.vocab) + if tokenizer is None: + print( + f"加载词表失败: {cfg.vocab}\n" + "请使用 --vocab 指定词表路径，或使用嵌入 tokenizer 的模型", + file=sys.stderr, + ) + sys.exit(1) + print(f"已从外部文件加载 tokenizer: {cfg.vocab}") + + print(f"词表大小: {tokenizer.vocab_size()}\n") + return model, tokenizer, device + + +def generate_text( + model: GPTModel, + tokenizer: TokenizerAdapter, + prompt_tokens: list[int], + max_new_tokens: int, + temperature: float, + eos_token_id: int, +): + """调用 model.generate（与 C++ generate_text 对应）。""" + # min_new_tokens = max_new_tokens / 2，避免模型一上来就输出 EOS + min_new = max_new_tokens // 2 + return model.generate( + prompt=prompt_tokens, + max_new_tokens=max_new_tokens, + temperature=temperature, + eos_token_id=eos_token_id, + min_new_tokens=min_new, + ) + + +def interactive_mode(model: GPTModel, tokenizer: TokenizerAdapter, cfg): + bos_id = tokenizer.bos_id() + eos_id = tokenizer.eos_id() + print("GPT 交互式生成 (输入 'quit' 退出)\n") + + while True: + try: + prompt = input(">>> ") + except (EOFError, KeyboardInterrupt): + break + if prompt in ("quit", "exit"): + break + if not prompt: + prompt = " " + + prompt_tokens = tokenizer.encode(prompt) + # 添加 BOS 前缀，使推理输入格式与训练时一致 + if bos_id != TokenizerAdapter.npos: + prompt_tokens.insert(0, bos_id) + + print("生成中...") + generated = generate_text( + model, tokenizer, prompt_tokens, cfg.max_tokens, cfg.temperature, eos_id + ) + print(f"\n{prompt}{tokenizer.decode(generated)}\n") + + if cfg.show_tokens: + print(f"Tokens: {prompt_tokens} -> {generated}\n") + + +def main(): + cfg = parse_args() + model, tokenizer, device = load_model_and_tokenizer(cfg) + + # ── 交互模式 ───────────────────────────────────────────── + if cfg.interactive: + interactive_mode(model, tokenizer, cfg) + return + + # ── 单次生成 ───────────────────────────────────────────── + bos_id = tokenizer.bos_id() + eos_id = tokenizer.eos_id() + prompt_tokens = tokenizer.encode(cfg.prompt) + # 添加 BOS 前缀，使推理输入格式与训练时一致 + if bos_id != TokenizerAdapter.npos: + prompt_tokens.insert(0, bos_id) + + print(f'提示: "{cfg.prompt}"') + print(f"Prompt tokens: {len(prompt_tokens)} 个") + print(f"生成 {cfg.max_tokens} 个 token (temperature={cfg.temperature})") + print("----------------------------------------") + + t_start = time.time() + generated = generate_text( + model, tokenizer, prompt_tokens, cfg.max_tokens, cfg.temperature, eos_id + ) + t_end = time.time() + gen_sec = t_end - t_start + + print(f"{cfg.prompt}{tokenizer.decode(generated)}") + + if cfg.show_tokens: + print(f"\n[Generated tokens: {generated}]") + + print("----------------------------------------") + tokens_per_sec = len(generated) / gen_sec if gen_sec > 0 else 0 + print( + f"生成 {len(generated)} tokens, 耗时 {gen_sec:.1f}s" + f" ({tokens_per_sec:.0f} tokens/s)" + ) + + +if __name__ == "__main__": + main() diff --git a/compare_with_torch/text_train.py b/compare_with_torch/text_train.py new file mode 100644 index 00000000..80b39ba9 --- /dev/null +++ b/compare_with_torch/text_train.py @@ -0,0 +1,370 @@ +# ── text_train.py — GPT 文本生成训练 (PyTorch CUDA) ──────────────────────── +# +# 复制自 C++ 项目 src/text_train.cpp。 +# 数据流： +# 文本 → Tokenizer.encode → token IDs +# 每 batch：token IDs → (batch, seq) LongTensor → GPTModel.forward +# → logits (batch, seq, vocab) → CrossEntropyLoss → scalar +# → backward → optimizer.step / zero_grad +# +# 关键约定（与 C++ 一致）： +# - 每行 = 一个独立训练样本，[BOS] + tokens + [EOS] +# - 短行用 pad_id=0 填充，目标 = 输入左移一位（next-token prediction） +# - 不忽略 PAD 位置的 loss（与 C++ CrossEntropyLoss 行为一致） +# - steps_per_epoch = min(num_samples / batch_size, 1000) +# ───────────────────────────────────────────────────────────────────────── + +import argparse +import os +import sys +import time + +import torch +import torch.nn as nn +import torch.optim as optim + +# 确保能 import 同目录下的模块（无论从哪个 cwd 运行） +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from tokenizer import TokenizerAdapter, get_or_train_tokenizer, load_tokenizer_from_file +from model import GPTModel + + +# ── 默认超参数（与 C++ domain_gpt.hpp 一致） ────────────────────────────── + +GPT_VOCAB_SIZE = 8000 +GPT_D_MODEL = 96 +GPT_NUM_HEADS = 4 +GPT_D_FF = 192 +GPT_NUM_LAYERS = 2 +GPT_SEQ_LEN = 32 + + +def parse_args(): + parser = argparse.ArgumentParser(description="GPT 文本生成训练程序 (PyTorch CUDA)") + parser.add_argument("text_path", help="训练文本文件路径") + parser.add_argument("--save", default="gpt_model_torch.pt", help="模型保存路径") + parser.add_argument("--resume", default=None, help="从已有模型恢复训练") + parser.add_argument("--vocab", default="gpt_bpe_torch.json", help="词表 JSON 路径（tokenizers 库标准格式，不存在则现场训练）") + parser.add_argument("--vocab-size", type=int, default=GPT_VOCAB_SIZE, help="训练 BPE 时的目标词表大小") + parser.add_argument("--epochs", type=int, default=10) + parser.add_argument("--lr", type=float, default=0.001) + parser.add_argument("--batch-size", type=int, default=32) + parser.add_argument("--seq-len", type=int, default=GPT_SEQ_LEN) + parser.add_argument( + "--optimizer", + default="adam", + choices=["sgd", "sgd_momentum", "adam", "adamw"], + ) + parser.add_argument("--weight-decay", type=float, default=0.0, help="权重衰减 (AdamW)") + parser.add_argument("--d-model", type=int, default=GPT_D_MODEL) + parser.add_argument("--num-heads", type=int, default=GPT_NUM_HEADS) + parser.add_argument("--num-layers", type=int, default=GPT_NUM_LAYERS) + parser.add_argument("--d-ff", type=int, default=GPT_D_FF) + parser.add_argument( + "--device", + default="cuda", + help="cuda / cpu / cuda:0（默认 cuda，不可用自动回退 cpu）", + ) + parser.add_argument( + "--max-norm", + type=float, + default=0.0, + help="梯度裁剪最大全局 L2 范数 (默认 0=不裁剪，与 C++ --max-norm 一致)", + ) + parser.add_argument("--log-interval", type=int, default=50) + return parser.parse_args() + + +def load_text(path: str) -> str: + with open(path, "r", encoding="utf-8") as f: + return f.read() + + +def prepare_samples(text: str, tokenizer: TokenizerAdapter) -> list[list[int]]: + """每行 = 一个训练样本: [BOS] + tokens + [EOS]（与 C++ 一致）。""" + bos_id = tokenizer.bos_id() + eos_id = tokenizer.eos_id() + + samples: list[list[int]] = [] + total_tokens = 0 + for line in text.splitlines(): + # 去除行首尾空白 + line = line.strip() + if not line: + continue + + tokens = tokenizer.encode(line) + sample: list[int] = [] + if bos_id != TokenizerAdapter.npos: + sample.append(bos_id) + sample.extend(tokens) + if eos_id != TokenizerAdapter.npos: + sample.append(eos_id) + total_tokens += len(sample) + samples.append(sample) + + print(f"文本行数: {len(samples)}, {total_tokens} tokens (含 BOS/EOS)") + return samples + + +def build_optimizer(name: str, model: nn.Module, lr: float, weight_decay: float = 0.0): + """创建优化器（与 C++ create_optimizer 对应）。""" + if name == "sgd": + return optim.SGD(model.parameters(), lr=lr) + if name == "sgd_momentum": + return optim.SGD(model.parameters(), lr=lr, momentum=0.9) + if name == "adam": + # C++ Adam 默认: beta1=0.9, beta2=0.999, eps=1e-8（与 PyTorch 默认一致） + return optim.Adam(model.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8) + if name == "adamw": + # AdamW: 解耦权重衰减（与 C++ AdamW 一致） + return optim.AdamW( + model.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8, + weight_decay=weight_decay, + ) + raise ValueError(f"未知优化器: {name}") + + +def build_batch( + samples: list[list[int]], + indices: list[int], + batch_size: int, + seq_len: int, + pad_id: int, + device: torch.device, +): + """按给定索引取一个 batch 并构造 (x, y) 张量。 + + 改造点（相对 C++ 版）： + - 不再每 step 独立随机采样，而是由调用方传入本 step 的样本索引列表 + （通常来自 epoch 级别的 shuffle 队列），保证每个样本每 epoch 只被访问一次 + - x[t] = sample[t], y[t] = sample[t+1]（next-token prediction） + - 样本长度 > seq_len+1: 截断（取前 seq_len+1 个 token） + - 样本长度 <= seq_len: 不足部分用 pad_id 填充 + """ + x = torch.full((batch_size, seq_len), pad_id, dtype=torch.long, device=device) + y = torch.full((batch_size, seq_len), pad_id, dtype=torch.long, device=device) + + for b in range(batch_size): + sample = samples[indices[b]] + sample_len = len(sample) + + for t in range(seq_len): + x[b, t] = sample[t] if t < sample_len else pad_id + y[b, t] = sample[t + 1] if (t + 1) < sample_len else pad_id + + return x, y + + +def save_checkpoint( + path: str, + model: GPTModel, + optimizer: optim.Optimizer, + epoch: int, + tokenizer_json: str, +): + """保存 checkpoint（含模型规格 + 嵌入 tokenizer JSON，与 C++ V3 格式精神一致）。""" + torch.save( + { + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "epoch": epoch, + "config": { + "vocab_size": model.vocab_size, + "d_model": model.d_model, + "seq_len": model.seq_len, + "num_heads": model.num_heads, + "d_ff": model.d_ff, + "num_layers": model.num_layers, + }, + "tokenizer_json": tokenizer_json, + }, + path, + ) + + +def main(): + cfg = parse_args() + + # ── 加载文本 ───────────────────────────────────────────── + print(f"加载文本: {cfg.text_path} ...") + text = load_text(cfg.text_path) + if not text: + print("文本文件为空", file=sys.stderr) + sys.exit(1) + + # ── 加载或训练分词器（tokenizers 库标准格式） ───────────── + tokenizer = get_or_train_tokenizer(cfg.vocab, cfg.text_path, cfg.vocab_size) + + bos_id = tokenizer.bos_id() + eos_id = tokenizer.eos_id() + pad_id = 0 # 与 C++ 一致：所有分词器的 PAD_ID 都是 0 + + # ── 按行编码 ───────────────────────────────────────────── + samples = prepare_samples(text, tokenizer) + print(f"词表大小: {tokenizer.vocab_size()}\n") + + # 过滤长度 < 2 的样本 + valid_samples = [s for s in samples if len(s) >= 2] + if not valid_samples: + print("无长度 >= 2 的训练样本", file=sys.stderr) + sys.exit(1) + print(f"有效样本数: {len(valid_samples)} (长度 >= 2)\n") + + # ── 设备选择 ───────────────────────────────────────────── + if cfg.device == "cuda" and not torch.cuda.is_available(): + print("CUDA 不可用，回退到 CPU\n") + device = torch.device("cpu") + else: + device = torch.device(cfg.device) + + # ── 打印配置 ───────────────────────────────────────────── + print("========================================") + print(" GPT 文本生成训练 (PyTorch CUDA)") + print("========================================") + print(f" 词表大小: {tokenizer.vocab_size()}") + print(f" 模型维度: {cfg.d_model}") + print(f" 注意力头: {cfg.num_heads}") + print(f" Transformer 层数: {cfg.num_layers}") + print(f" FFN 维度: {cfg.d_ff}") + print(f" 序列长度: {cfg.seq_len}") + print(f" 优化器: {cfg.optimizer} 学习率: {cfg.lr} 权重衰减: {cfg.weight_decay}") + print(f" 梯度裁剪 max_norm: {cfg.max_norm} (0=不裁剪)") + print(f" Loss mask: 启用 (屏蔽 padding 位置，与 C++ forward_sparse 一致)") + print(f" 轮数: {cfg.epochs} 批大小: {cfg.batch_size}") + print(f" 设备: {device}") + print("========================================\n") + + # ── 构建模型 ───────────────────────────────────────────── + model = GPTModel( + vocab_size=tokenizer.vocab_size(), + d_model=cfg.d_model, + seq_len=cfg.seq_len, + num_heads=cfg.num_heads, + d_ff=cfg.d_ff, + num_layers=cfg.num_layers, + ).to(device) + + # ── 优化器 ─────────────────────────────────────────────── + optimizer = build_optimizer(cfg.optimizer, model, cfg.lr, cfg.weight_decay) + + # ── 恢复训练 ───────────────────────────────────────────── + start_epoch = 0 + if cfg.resume: + if not os.path.exists(cfg.resume): + print(f"模型文件不存在: {cfg.resume}，将从头训练\n") + else: + # checkpoint 仅含 state_dict 等安全类型，weights_only=True + # 防止加载不可信 pickle 时执行任意代码 + ckpt = torch.load(cfg.resume, map_location=device, weights_only=True) + model.load_state_dict(ckpt["model_state_dict"]) + if "optimizer_state_dict" in ckpt: + optimizer.load_state_dict(ckpt["optimizer_state_dict"]) + start_epoch = ckpt.get("epoch", 0) + 1 + print(f"已加载模型: {cfg.resume} (从 epoch {start_epoch} 继续)\n") + + # ── 训练循环 ───────────────────────────────────────────── + # 改造：每个样本每 epoch 被访问一次（按 shuffle 后的顺序切片）， + # 不再每 step 独立随机采样。 + import random as _random + + n_samples = len(valid_samples) + # 一个 epoch 的 step 数 = 全部样本 / batch_size（向下取整，丢弃末尾不足一个 batch 的样本） + # 与 C++ 一致：无上限 + steps_per_epoch = n_samples // cfg.batch_size + if steps_per_epoch == 0: + steps_per_epoch = 1 + + rng_epoch = _random.Random(42) # epoch 级别 shuffle 的随机源 + + model.train() + t_start = time.time() + + for epoch in range(start_epoch, cfg.epochs): + ep_start = time.time() + total_loss = 0.0 + + # 每个 epoch 开始前 shuffle 样本索引队列 + indices = list(range(n_samples)) + rng_epoch.shuffle(indices) + + for step in range(steps_per_epoch): + # 取本 step 的样本索引（顺序切片，不再独立采样） + batch_indices = indices[step * cfg.batch_size : (step + 1) * cfg.batch_size] + # 末尾不足一个 batch 时按实际样本数构建（避免索引越界） + this_bs = len(batch_indices) + + x, y = build_batch( + valid_samples, + batch_indices, + this_bs, + cfg.seq_len, + pad_id, + device, + ) + + # 构建 loss_mask（与 C++ forward_sparse 一致） + # 非对话模式：屏蔽 padding 位置，只对真实 token 计算 loss + # mask[t][b] = 1.0 当 t+1 < sample_len（即目标不是 padding） + loss_mask = torch.zeros( + this_bs, cfg.seq_len, device=device + ) + for b in range(this_bs): + sample = valid_samples[batch_indices[b]] + sample_len = len(sample) + for t in range(cfg.seq_len): + if t + 1 < sample_len: + loss_mask[b, t] = 1.0 + + logits, loss = model(x, targets=y, loss_mask=loss_mask) + + optimizer.zero_grad() + loss.backward() + + # 梯度裁剪（与 C++ --max-norm 一致，在 step 之前） + if cfg.max_norm > 0: + torch.nn.utils.clip_grad_norm_( + model.parameters(), cfg.max_norm + ) + + optimizer.step() + + total_loss += loss.item() + + # 进度显示 + if (step + 1) % cfg.log_interval == 0 or step + 1 == steps_per_epoch: + print( + f"\r Epoch {epoch + 1}/{cfg.epochs}" + f" step {step + 1}/{steps_per_epoch}" + f" loss: {loss.item():.4f} ", + end="", + flush=True, + ) + + ep_sec = time.time() - ep_start + avg_loss = total_loss / steps_per_epoch + print( + f"\r Epoch {epoch + 1}/{cfg.epochs}" + f" avg_loss={avg_loss:.4f}" + f" time={ep_sec:.1f}s" + ) + + total_sec = time.time() - t_start + + # ── 读取 tokenizer JSON 以便嵌入模型 ───────────────────── + tokenizer_json = "" + if os.path.exists(cfg.vocab): + with open(cfg.vocab, "r", encoding="utf-8") as f: + tokenizer_json = f.read() + + # ── 保存模型（含规格 + 嵌入 tokenizer） ────────────────── + save_checkpoint(cfg.save, model, optimizer, cfg.epochs - 1, tokenizer_json) + print( + f"\n训练完成! 总耗时: {total_sec:.1f}s" + f" 词表已嵌入模型文件" + ) + + +if __name__ == "__main__": + main() diff --git a/compare_with_torch/tokenizer.py b/compare_with_torch/tokenizer.py new file mode 100644 index 00000000..038b289b --- /dev/null +++ b/compare_with_torch/tokenizer.py @@ -0,0 +1,372 @@ +# ── tokenizer.py — 分词器 (基于 tokenizers 库) ───────────────────────────── +# +# 直接使用 HuggingFace tokenizers 库，不再手写 BPE 算法。 +# 仅提供统一适配器，包装 tokenizers.Tokenizer 暴露与 C++ 项目相近的接口。 +# +# 接口： +# encode(text) -> list[int] +# decode(ids) -> str +# vocab_size() -> int +# bos_id() -> int +# eos_id() -> int +# +# 加载/训练： +# load_tokenizer_from_file(path) — 加载 tokenizers 库标准 JSON +# train_bpe(text_path, vocab_size, output_path) — 训练并保存 +# get_or_train_tokenizer(...) — 优先加载，不存在则训练 +# ───────────────────────────────────────────────────────────────────────── + +import json +from typing import Optional + +from tokenizers import Tokenizer +from tokenizers.decoders import ByteLevel as ByteLevelDecoder +from tokenizers.models import BPE +from tokenizers.pre_tokenizers import ByteLevel +from tokenizers.trainers import BpeTrainer + + +# 特殊 token（与 C++ 项目保持一致的命名） +SPECIAL_TOKENS = ["", "", "", ""] +PAD_ID = 0 +UNK_ID = 1 +BOS_NAME = "" +EOS_NAME = "" + + +class TokenizerAdapter: + """包装 tokenizers.Tokenizer，提供与 C++ 项目 Tokenizer 一致的接口。""" + + npos: int = -1 + + def __init__(self, tokenizer: Tokenizer): + self._tok = tokenizer + + def encode(self, text: str) -> list[int]: + return self._tok.encode(text).ids + + def decode(self, ids: list[int]) -> str: + return self._tok.decode(ids, skip_special_tokens=False) + + def vocab_size(self) -> int: + return self._tok.get_vocab_size() + + def bos_id(self) -> int: + tid = self._tok.token_to_id(BOS_NAME) + return tid if tid is not None else self.npos + + def eos_id(self) -> int: + tid = self._tok.token_to_id(EOS_NAME) + return tid if tid is not None else self.npos + + +def load_tokenizer_from_string(json_content: str) -> Optional[TokenizerAdapter]: + """从 tokenizers 库标准 JSON 字符串加载。""" + try: + tok = Tokenizer.from_str(json_content) + except Exception: + return None + return TokenizerAdapter(tok) + + +def load_tokenizer_from_file(path: str) -> Optional[TokenizerAdapter]: + """从 tokenizers 库标准 JSON 文件加载。""" + try: + tok = Tokenizer.from_file(path) + except Exception: + return None + return TokenizerAdapter(tok) + + +def train_bpe( + text_path: str, + vocab_size: int, + output_path: Optional[str] = None, +) -> TokenizerAdapter: + """训练一个 ByteLevel BPE 分词器。 + + Args: + text_path: 训练文本路径 + vocab_size: 目标词表大小 + output_path: 若提供，则保存为 tokenizers 库标准 JSON + + Returns: + TokenizerAdapter 实例 + """ + tokenizer = Tokenizer(BPE(unk_token="")) + tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True) + tokenizer.decoder = ByteLevelDecoder() + + trainer = BpeTrainer( + vocab_size=vocab_size, + special_tokens=SPECIAL_TOKENS, + show_progress=False, + ) + tokenizer.train(files=[text_path], trainer=trainer) + + if output_path: + tokenizer.save(output_path) + + return TokenizerAdapter(tokenizer) + + +class CharBPEAdapter: + """加载 C++ 项目 CharBPE 格式 tokenizer (charbpe_fast.json)。 + + JSON 格式: + "type": "char_bpe_tokenizer" + "vocab": { "id_str": "hex_bytes" | "special_name" } + - id 0-3: 特殊 token (纯文本名: ///) + - id 4-259: 256 个单字节 token (hex 编码: "00"~"ff") + - id 260+: 多字节字符 / 合并 token (hex 编码的 UTF-8 字节串) + "merges": [[id_a, id_b, new_id], ...] + + 编码算法 (与 C++ CharBPETokenizer::encode 一致): + 1. pre_tokenize: 按空白 + CJK 标点切分成 chunks + 2. 每 chunk: 字符 → base token id 序列 + - ASCII (byte < 0x80): id = 4 + byte + - 非 ASCII: UTF-8 解码得到字符, 查 char_to_id_ + 3. 按 merge priority 合并相邻 id 对 + """ + + npos: int = -1 + PAD_ID = 0 + UNK_ID = 1 + BOS_ID = 2 + EOS_ID = 3 + BYTE_BASE = 4 + + def __init__(self, vocab: dict, merges: list): + # vocab: {id_int: token_bytes} + # merges: [[id_a, id_b, new_id], ...] + self._vocab = vocab + self._merges = merges + self._vocab_size = len(vocab) + + # char_to_id: 多字节字符 token → id (BYTE_BASE 及以上) + self._char_to_id = {} + for tid, tok in vocab.items(): + if tid >= self.BYTE_BASE: + self._char_to_id[tok] = tid + + # merge_map: (id_a, id_b) → (new_id, priority) + self._merge_map = {} + for priority, m in enumerate(merges): + id_a, id_b, new_id = m[0], m[1], m[2] + self._merge_map[(id_a, id_b)] = (new_id, priority) + + def encode(self, text: str) -> list[int]: + data = text.encode("utf-8") + chunks = self._pre_tokenize(data) + all_ids: list[int] = [] + for chunk in chunks: + ids = self._chars_to_ids(chunk) + ids = self._apply_merges(ids) + all_ids.extend(ids) + return all_ids + + def decode(self, ids: list[int]) -> str: + raw = b"" + for tid in ids: + if tid >= self.BYTE_BASE and tid < self._vocab_size: + # 防御：稀疏/缺失 ID 时跳过而非 KeyError + tok = self._vocab.get(tid) if isinstance(self._vocab, dict) else None + if tok is None: + continue + raw += tok + return raw.decode("utf-8", errors="replace") + + def vocab_size(self) -> int: + return self._vocab_size + + def bos_id(self) -> int: + return self.BOS_ID + + def eos_id(self) -> int: + return self.EOS_ID + + # ── pre_tokenize: 按空白 + CJK 切分 (与 C++ 一致) ────────── + + @staticmethod + def _is_space(b: int) -> bool: + return b in (0x20, 0x09, 0x0A, 0x0D, 0x0C, 0x0B) + + @staticmethod + def _is_alnum_ascii(b: int) -> bool: + return (0x30 <= b <= 0x39) or (0x41 <= b <= 0x5A) or (0x61 <= b <= 0x7A) + + @staticmethod + def _decode_utf8(data: bytes, pos: int) -> tuple[int, int]: + """返回 (码点, 字节长度)。""" + b0 = data[pos] + if b0 < 0x80: + return b0, 1 + if pos + 1 >= len(data): + return 0xFFFD, 1 + if (b0 & 0xE0) == 0xC0: + cp = ((b0 & 0x1F) << 6) | (data[pos + 1] & 0x3F) + return cp, 2 + if pos + 2 >= len(data): + return 0xFFFD, 1 + if (b0 & 0xF0) == 0xE0: + cp = ((b0 & 0x0F) << 12) | ((data[pos + 1] & 0x3F) << 6) | (data[pos + 2] & 0x3F) + return cp, 3 + if pos + 3 >= len(data): + return 0xFFFD, 1 + if (b0 & 0xF8) == 0xF0: + cp = ((b0 & 0x07) << 18) | ((data[pos + 1] & 0x3F) << 12) | ((data[pos + 2] & 0x3F) << 6) | (data[pos + 3] & 0x3F) + return cp, 4 + return 0xFFFD, 1 + + @staticmethod + def _is_cjk(cp: int) -> bool: + return (0x4E00 <= cp <= 0x9FFF) or (0x3400 <= cp <= 0x4DBF) or (0xF900 <= cp <= 0xFAFF) + + @staticmethod + def _is_cjk_punct(cp: int) -> bool: + return (0x3000 <= cp <= 0x303F) or (0xFF00 <= cp <= 0xFFEF) + + def _pre_tokenize(self, data: bytes) -> list[bytes]: + chunks: list[bytes] = [] + i = 0 + n = len(data) + while i < n: + # 收集前导空白 (附加到下一个词) + start = i + while i < n and self._is_space(data[i]): + i += 1 + if i >= n: + break + prefix = data[start:i] + + cp, clen = self._decode_utf8(data, i) + if self._is_cjk(cp): + word = prefix + while i < n: + cp, clen = self._decode_utf8(data, i) + if not self._is_cjk(cp) or self._is_cjk_punct(cp): + break + word += data[i:i + clen] + i += clen + if word: + chunks.append(word) + elif self._is_alnum_ascii(data[i]): + word = prefix + while i < n: + if not self._is_alnum_ascii(data[i]): + break + word += bytes([data[i]]) + i += 1 + if word: + chunks.append(word) + else: + word = prefix + data[i:i + clen] + i += clen + if word: + chunks.append(word) + return chunks + + def _chars_to_ids(self, chunk: bytes) -> list[int]: + ids: list[int] = [] + i = 0 + n = len(chunk) + while i < n: + b = chunk[i] + if b < 0x80: + ids.append(self.BYTE_BASE + b) + i += 1 + else: + _, clen = self._decode_utf8(chunk, i) + ch = chunk[i:i + clen] + ids.append(self._char_to_id.get(ch, self.UNK_ID)) + i += clen + return ids + + def _apply_merges(self, ids: list[int]) -> list[int]: + if len(ids) <= 1: + return ids + ids = list(ids) + while True: + best_prio = None + best_pos = -1 + best_new = -1 + for j in range(len(ids) - 1): + pair = (ids[j], ids[j + 1]) + hit = self._merge_map.get(pair) + if hit is not None: + new_id, prio = hit + if best_prio is None or prio < best_prio: + best_prio = prio + best_pos = j + best_new = new_id + if best_pos == -1: + break + ids[best_pos:best_pos + 2] = [best_new] + return ids + + +def load_charbpe_from_file(path: str) -> Optional[CharBPEAdapter]: + """加载 C++ CharBPE 格式的 JSON 文件 (charbpe_fast.json)。""" + try: + with open(path, "r", encoding="utf-8") as f: + content = f.read() + except OSError: + return None + return load_charbpe_from_string(content) + + +def load_charbpe_from_string(json_content: str) -> Optional[CharBPEAdapter]: + """从 JSON 字符串加载 C++ CharBPE tokenizer。""" + try: + obj = json.loads(json_content) + except json.JSONDecodeError: + return None + + if obj.get("type") != "char_bpe_tokenizer": + return None + + # 解析 vocab: { "id_str": "hex_or_name" } → { id_int: bytes } + raw_vocab = obj.get("vocab", {}) + vocab: dict[int, bytes] = {} + for id_str, val in raw_vocab.items(): + tid = int(id_str) + if tid < 4: + # 特殊 token: 纯文本名, 不参与编码 + vocab[tid] = val.encode("utf-8") + else: + # hex 编码的 UTF-8 字节串 + vocab[tid] = bytes.fromhex(val) + + # 解析 merges: [[id_a, id_b, new_id], ...] + merges = obj.get("merges", []) + + return CharBPEAdapter(vocab, merges) + + +def get_or_train_tokenizer( + vocab_path: str, + text_path: str, + vocab_size: int, +) -> "TokenizerAdapter | CharBPEAdapter": + """优先从 vocab_path 加载；失败则在 text_path 上现场训练并保存。 + + 自动检测两种格式: + 1. C++ CharBPE 格式 (type=char_bpe_tokenizer) → CharBPEAdapter + 2. HuggingFace tokenizers 库标准格式 → TokenizerAdapter + """ + # 先尝试 C++ CharBPE 格式 + tok = load_charbpe_from_file(vocab_path) + if tok is not None: + print(f"已加载 CharBPE tokenizer: {vocab_path} (vocab_size={tok.vocab_size()})") + return tok + + # 再尝试 HuggingFace tokenizers 格式 + tok = load_tokenizer_from_file(vocab_path) + if tok is not None: + print(f"已加载 tokenizer: {vocab_path} (vocab_size={tok.vocab_size()})") + return tok + + print(f"无法加载 {vocab_path}，开始训练新 BPE (vocab_size={vocab_size})...") + tok = train_bpe(text_path, vocab_size, vocab_path) + print(f"已训练并保存 tokenizer 到 {vocab_path} (vocab_size={tok.vocab_size()})") + return tok diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt new file mode 100644 index 00000000..c3924fe5 --- /dev/null +++ b/cuda/CMakeLists.txt @@ -0,0 +1,68 @@ +# ── cuda/CMakeLists.txt — CUDA 计算后端（独立 nvcc 编译）──────────────────── +# +# nvcc 在 Windows 上使用 MSVC 作为 host 编译器。 +# 使用 VS 2022 BuildTools (MSVC 14.44) 以获得最佳兼容性。 +# .cu 文件由 nvcc 编译为 .obj，链接到主 Clang 项目。 +# +# 构建流程： +# 1. nvcc -c cuda_kernels.cu → cuda_kernels.obj +# 2. 主项目 Clang 编译 CudaBackend/CudaEngine（调用内核函数） +# 3. 链接 cuda_kernels.obj + cudart + nvrtc +# ───────────────────────────────────────────────────────────────────────── + +# nvcc 的 host compiler 路径（VS 2022 BuildTools） +set(NN_CUDA_HOST_CL "C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC/14.44.35207/bin/Hostx64/x64/cl.exe") +set(NN_CUDA_HOST_VCVARS "C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Auxiliary/Build/vcvars64.bat") + +# 查找 CUDA Toolkit +find_package(CUDAToolkit REQUIRED) + +# 设置 SM 目标架构（V100 = sm_70） +if(NOT DEFINED NN_CUDA_ARCH) + set(NN_CUDA_ARCH "70" CACHE STRING "CUDA SM architecture (e.g. 70, 80, 86, 89)") +endif() + +# ── nvcc 编译脚本（在 vcvars 环境中运行）────────────────────────────────── +# 由于 CMake 的 add_custom_command 无法直接设置 vcvars 环境， +# 生成一个临时 .bat 脚本来包装编译命令。 +# Debug 模式使用 /MDd（动态调试 CRT），Release 模式使用 /MD（动态发布 CRT） +# 同时对齐 _ITERATOR_DEBUG_LEVEL（主程序 Debug 为 2，Release 为 0）， +# 否则 lld-link 会报 /failifmismatch。 +if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR (DEFINED CMAKE_CONFIGURATION_TYPES AND CMAKE_BUILD_TYPE MATCHES "Debug")) + set(NN_CUDA_CRT_FLAG "/MDd") + set(NN_CUDA_ITDL "2") +else() + set(NN_CUDA_CRT_FLAG "/MD") + set(NN_CUDA_ITDL "0") +endif() +set(NN_CUDA_COMPILE_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/compile_cuda_kernels.bat") +# 启用 cuBLAS：定义 NN_HAS_CUBLAS 宏，链接 cublas +file(WRITE "${NN_CUDA_COMPILE_SCRIPT}" + "@echo off\r\n" + "call \"${NN_CUDA_HOST_VCVARS}\" >nul 2>&1\r\n" + "\"${CUDAToolkit_BIN_DIR}/nvcc.exe\" -c -O3 --use_fast_math -allow-unsupported-compiler" + " -Xcompiler ${NN_CUDA_CRT_FLAG}" + " -Xcompiler /D_ITERATOR_DEBUG_LEVEL=${NN_CUDA_ITDL}" + " -arch=sm_${NN_CUDA_ARCH} -DNN_HAS_CUBLAS" + " -I\"${CMAKE_CURRENT_SOURCE_DIR}/../include\"" + " -I\"${CUDAToolkit_INCLUDE_DIRS}\"" + " \"%~1\" -o \"%~2\"\r\n" + "exit /b %errorlevel%\r\n" +) + +# ── CUDA 内核 .obj ────────────────────────────────────────────────────── +set(NN_CUDA_KERNELS_OBJ "${CMAKE_CURRENT_BINARY_DIR}/cuda_kernels.obj") + +add_custom_command( + OUTPUT ${NN_CUDA_KERNELS_OBJ} + COMMAND cmd /c "${NN_CUDA_COMPILE_SCRIPT}" "${CMAKE_CURRENT_SOURCE_DIR}/cuda_kernels.cu" "${NN_CUDA_KERNELS_OBJ}" + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/cuda_kernels.cu + COMMENT "[CUDA] Compiling cuda_kernels.cu with nvcc (sm_${NN_CUDA_ARCH})" +) + +add_custom_target(compile_cuda_kernels DEPENDS ${NN_CUDA_KERNELS_OBJ}) + +# ── 导出变量给父项目 ──────────────────────────────────────────────────── +set(NN_CUDA_KERNELS_OBJ ${NN_CUDA_KERNELS_OBJ} PARENT_SCOPE) +set(NN_CUDA_FOUND TRUE PARENT_SCOPE) +set(NN_CUDA_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE) diff --git a/cuda/cuda_kernels.cu b/cuda/cuda_kernels.cu new file mode 100644 index 00000000..382bd57c --- /dev/null +++ b/cuda/cuda_kernels.cu @@ -0,0 +1,840 @@ +// ── cuda_kernels.cu — CUDA 计算内核实现 ───────────────────────────────────── +// 由 nvcc 编译为静态库，通过 C-linkage 接口供 CudaBackend 调用。 +// +// 内核设计原则： +// 1. 与 Vulkan compute shader 语义完全一致 +// 2. 使用 float 类型（Scalar = float） +// 3. 所有内核接受 CUDA stream 参数以支持异步执行 +// 4. 使用 cuBLAS 加速 matmul（通过 NN_HAS_CUBLAS 宏控制） +// 5. cuBLAS 句柄由调用方（CudaBackend）管理并复用，避免每次调用创建/销毁 +// ───────────────────────────────────────────────────────────────────────── + +#include "cuda_kernels.h" + +#include +#include +#include +#include + +#ifdef NN_HAS_CUBLAS +#include +#endif + +// ── CUDA 错误检查宏 ────────────────────────────────────────────────────── +#define CUDA_CHECK(call) \ + do { \ + cudaError_t err = (call); \ + if (err != cudaSuccess) { \ + fprintf(stderr, "CUDA error at %s:%d: %s\n", \ + __FILE__, __LINE__, cudaGetErrorString(err)); \ + return static_cast(err); \ + } \ + } while (0) + +#define CUDA_CHECK_VOID(call) \ + do { \ + cudaError_t err = (call); \ + if (err != cudaSuccess) { \ + fprintf(stderr, "CUDA error at %s:%d: %s\n", \ + __FILE__, __LINE__, cudaGetErrorString(err)); \ + return; \ + } \ + } while (0) + +#ifdef NN_HAS_CUBLAS +// cuBLAS 错误检查宏（将 cublasStatus_t 转为负数返回值） +#define CUBLAS_CHECK(call) \ + do { \ + cublasStatus_t st = (call); \ + if (st != CUBLAS_STATUS_SUCCESS) { \ + fprintf(stderr, "cuBLAS error at %s:%d: %d\n", \ + __FILE__, __LINE__, static_cast(st)); \ + return static_cast(-st); \ + } \ + } while (0) +#endif + +// ══════════════════════════════════════════════════════════════════════════ +// 矩阵乘法内核 +// ══════════════════════════════════════════════════════════════════════════ + +static constexpr int TILE = 16; + +__global__ void matmul_kernel( + const float* __restrict__ A, + const float* __restrict__ B, + float* __restrict__ C, + int M, int N, int K, + int transA, int transB) +{ + __shared__ float As[TILE][TILE]; + __shared__ float Bs[TILE][TILE]; + + const int row = blockIdx.y * TILE + threadIdx.y; + const int col = blockIdx.x * TILE + threadIdx.x; + + float sum = 0.0f; + + for (int t = 0; t < (K + TILE - 1) / TILE; ++t) + { + const int a_col = t * TILE + threadIdx.x; + const int b_row = t * TILE + threadIdx.y; + + if (row < M && a_col < K) + As[threadIdx.y][threadIdx.x] = transA ? A[a_col * M + row] : A[row * K + a_col]; + else + As[threadIdx.y][threadIdx.x] = 0.0f; + + if (b_row < K && col < N) + Bs[threadIdx.y][threadIdx.x] = transB ? B[col * K + b_row] : B[b_row * N + col]; + else + Bs[threadIdx.y][threadIdx.x] = 0.0f; + + __syncthreads(); + + for (int k = 0; k < TILE; ++k) + sum += As[threadIdx.y][k] * Bs[k][threadIdx.x]; + + __syncthreads(); + } + + if (row < M && col < N) + C[row * N + col] = sum; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 批量矩阵乘法内核 +// ══════════════════════════════════════════════════════════════════════════ + +__global__ void batched_matmul_kernel( + const float* __restrict__ A, + const float* __restrict__ B, + float* __restrict__ C, + int M, int N, int K, + int transA, int transB, + int batch) +{ + __shared__ float As[TILE][TILE]; + __shared__ float Bs[TILE][TILE]; + + const int b = blockIdx.z; + const int row = blockIdx.y * TILE + threadIdx.y; + const int col = blockIdx.x * TILE + threadIdx.x; + + const float* Ab = A + static_cast(b) * M * K; + const float* Bb = B + static_cast(b) * K * N; + float* Cb = C + static_cast(b) * M * N; + + float sum = 0.0f; + + for (int t = 0; t < (K + TILE - 1) / TILE; ++t) + { + const int a_col = t * TILE + threadIdx.x; + const int b_row = t * TILE + threadIdx.y; + + if (row < M && a_col < K) + As[threadIdx.y][threadIdx.x] = transA ? Ab[a_col * M + row] : Ab[row * K + a_col]; + else + As[threadIdx.y][threadIdx.x] = 0.0f; + + if (b_row < K && col < N) + Bs[threadIdx.y][threadIdx.x] = transB ? Bb[col * K + b_row] : Bb[b_row * N + col]; + else + Bs[threadIdx.y][threadIdx.x] = 0.0f; + + __syncthreads(); + + for (int k = 0; k < TILE; ++k) + sum += As[threadIdx.y][k] * Bs[k][threadIdx.x]; + + __syncthreads(); + } + + if (row < M && col < N) + Cb[row * N + col] = sum; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 逐元素运算内核 +// ══════════════════════════════════════════════════════════════════════════ + +__global__ void elementwise_unary_kernel( + const float* __restrict__ A, float* __restrict__ out, + unsigned int count, unsigned int op) +{ + const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= count) return; + + const float x = A[i]; + switch (op) { + case 0: out[i] = -x; break; // Neg + case 1: out[i] = expf(x); break; // Exp + case 2: out[i] = logf(x); break; // Log + case 3: out[i] = sqrtf(x); break; // Sqrt + case 4: out[i] = rsqrtf(x); break; // Rsqrt + case 5: out[i] = fabsf(x); break; // Abs + case 6: out[i] = tanhf(x); break; // Tanh + default: out[i] = x; break; + } +} + +__global__ void elementwise_binary_kernel( + const float* __restrict__ A, + const float* __restrict__ B, + float* __restrict__ out, + unsigned int count, unsigned int op) +{ + const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= count) return; + + const float a = A[i]; + const float b = B[i]; + switch (op) { + case 0: out[i] = a + b; break; // Add + case 1: out[i] = a - b; break; // Sub + case 2: out[i] = a * b; break; // Mul + case 3: out[i] = a / b; break; // Div + case 4: out[i] = fmaxf(a, b); break; // Max + case 5: out[i] = fminf(a, b); break; // Min + default: out[i] = a; break; + } +} + +__global__ void elementwise_binary_scalar_kernel( + const float* __restrict__ A, float* __restrict__ out, + unsigned int count, unsigned int op, + float scalar, int scalar_first) +{ + const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= count) return; + + const float a = A[i]; + const float s = scalar; + if (scalar_first) { + switch (op) { + case 0: out[i] = s + a; break; + case 1: out[i] = s - a; break; + case 2: out[i] = s * a; break; + case 3: out[i] = s / a; break; + case 4: out[i] = fmaxf(s, a); break; + case 5: out[i] = fminf(s, a); break; + default: out[i] = a; break; + } + } else { + switch (op) { + case 0: out[i] = a + s; break; + case 1: out[i] = a - s; break; + case 2: out[i] = a * s; break; + case 3: out[i] = a / s; break; + case 4: out[i] = fmaxf(a, s); break; + case 5: out[i] = fminf(a, s); break; + default: out[i] = a; break; + } + } +} + +__global__ void axpy_kernel( + const float* __restrict__ A, + const float* __restrict__ B, + float* __restrict__ out, + unsigned int count, float scalar) +{ + const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= count) return; + out[i] = A[i] + scalar * B[i]; +} + +__global__ void select_scalar_cond_kernel( + const float* __restrict__ A, + const float* __restrict__ then_v, + float* __restrict__ out, + unsigned int count, unsigned int cmp, + float scalar_b, float scalar_else) +{ + const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= count) return; + + const float a = A[i]; + bool cond; + switch (cmp) { + case 0: cond = a < scalar_b; break; // Lt + case 1: cond = a <= scalar_b; break; // Le + case 2: cond = a > scalar_b; break; // Gt + case 3: cond = a >= scalar_b; break; // Ge + case 4: cond = a == scalar_b; break; // Eq + case 5: cond = a != scalar_b; break; // Ne + default: cond = false; break; + } + out[i] = cond ? then_v[i] : scalar_else; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 归约内核（每个 block 归约一行/列） +// ══════════════════════════════════════════════════════════════════════════ + +__global__ void reduce_row_sum_kernel( + const float* __restrict__ A, float* __restrict__ out, + unsigned int rows, unsigned int cols) +{ + const unsigned int r = blockIdx.x; + if (r >= rows) return; + + extern __shared__ float sdata[]; + float sum = 0.0f; + for (unsigned int c = threadIdx.x; c < cols; c += blockDim.x) + sum += A[r * cols + c]; + + sdata[threadIdx.x] = sum; + __syncthreads(); + + for (unsigned int s = blockDim.x / 2; s > 0; s >>= 1) { + if (threadIdx.x < s) + sdata[threadIdx.x] += sdata[threadIdx.x + s]; + __syncthreads(); + } + + if (threadIdx.x == 0) + out[r] = sdata[0]; +} + +__global__ void reduce_row_max_kernel( + const float* __restrict__ A, float* __restrict__ out, + unsigned int rows, unsigned int cols) +{ + const unsigned int r = blockIdx.x; + if (r >= rows) return; + + extern __shared__ float sdata[]; + float val = -FLT_MAX; + for (unsigned int c = threadIdx.x; c < cols; c += blockDim.x) + val = fmaxf(val, A[r * cols + c]); + + sdata[threadIdx.x] = val; + __syncthreads(); + + for (unsigned int s = blockDim.x / 2; s > 0; s >>= 1) { + if (threadIdx.x < s) + sdata[threadIdx.x] = fmaxf(sdata[threadIdx.x], sdata[threadIdx.x + s]); + __syncthreads(); + } + + if (threadIdx.x == 0) + out[r] = sdata[0]; +} + +// 列归约（每个 block 处理一列，shared memory tree reduction） +// 与 reduce_row_*_kernel 对称：blockIdx.x = c（列号），线程协作归约所有行。 +// 行主序矩阵 A[r * cols + c]：同一列的元素在内存中 stride=cols， +// 不连续访问，但通过线程级并行 + shared memory 树形归约仍优于单线程串行。 +__global__ void reduce_col_sum_kernel( + const float* __restrict__ A, float* __restrict__ out, + unsigned int rows, unsigned int cols) +{ + const unsigned int c = blockIdx.x; + if (c >= cols) return; + + extern __shared__ float sdata[]; + float sum = 0.0f; + for (unsigned int r = threadIdx.x; r < rows; r += blockDim.x) + sum += A[r * cols + c]; + + sdata[threadIdx.x] = sum; + __syncthreads(); + + for (unsigned int s = blockDim.x / 2; s > 0; s >>= 1) { + if (threadIdx.x < s) + sdata[threadIdx.x] += sdata[threadIdx.x + s]; + __syncthreads(); + } + + if (threadIdx.x == 0) + out[c] = sdata[0]; +} + +__global__ void reduce_col_max_kernel( + const float* __restrict__ A, float* __restrict__ out, + unsigned int rows, unsigned int cols) +{ + const unsigned int c = blockIdx.x; + if (c >= cols) return; + + extern __shared__ float sdata[]; + float val = -FLT_MAX; + for (unsigned int r = threadIdx.x; r < rows; r += blockDim.x) + val = fmaxf(val, A[r * cols + c]); + + sdata[threadIdx.x] = val; + __syncthreads(); + + for (unsigned int s = blockDim.x / 2; s > 0; s >>= 1) { + if (threadIdx.x < s) + sdata[threadIdx.x] = fmaxf(sdata[threadIdx.x], sdata[threadIdx.x + s]); + __syncthreads(); + } + + if (threadIdx.x == 0) + out[c] = sdata[0]; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 广播内核 +// ══════════════════════════════════════════════════════════════════════════ + +__device__ inline float apply_binary_op(float a, float b, unsigned int op) +{ + switch (op) { + case 0: return a + b; + case 1: return a - b; + case 2: return a * b; + case 3: return a / b; + case 4: return fmaxf(a, b); + case 5: return fminf(a, b); + default: return a; + } +} + +__global__ void broadcast_row_kernel( + const float* __restrict__ A, + const float* __restrict__ vec, + float* __restrict__ out, + unsigned int rows, unsigned int cols, unsigned int op) +{ + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= rows * cols) return; + const unsigned int r = idx / cols; + out[idx] = apply_binary_op(A[idx], vec[r], op); +} + +__global__ void broadcast_col_kernel( + const float* __restrict__ A, + const float* __restrict__ vec, + float* __restrict__ out, + unsigned int rows, unsigned int cols, unsigned int op) +{ + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= rows * cols) return; + const unsigned int c = idx % cols; + out[idx] = apply_binary_op(A[idx], vec[c], op); +} + +// ══════════════════════════════════════════════════════════════════════════ +// 转置与重排内核 +// ══════════════════════════════════════════════════════════════════════════ + +__global__ void transpose_kernel( + const float* __restrict__ A, float* __restrict__ out, + unsigned int R, unsigned int C) +{ + __shared__ float tile[TILE][TILE + 1]; // +1 避免 bank conflict + + const int r = blockIdx.y * TILE + threadIdx.y; + const int c = blockIdx.x * TILE + threadIdx.x; + + if (r < R && c < C) + tile[threadIdx.y][threadIdx.x] = A[r * C + c]; + __syncthreads(); + + const int out_r = blockIdx.x * TILE + threadIdx.y; + const int out_c = blockIdx.y * TILE + threadIdx.x; + + if (out_r < C && out_c < R) + out[out_r * R + out_c] = tile[threadIdx.x][threadIdx.y]; +} + +__global__ void rearrange_3d_kernel( + const float* __restrict__ A, float* __restrict__ out, + unsigned int M, unsigned int B, unsigned int N, + unsigned int inverse, unsigned int total) +{ + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= total) return; + + if (!inverse) { + // (M, B*N) → (B*M, N) + const unsigned int m = idx / (B * N); + const unsigned int rem = idx % (B * N); + const unsigned int b = rem / N; + const unsigned int n = rem % N; + out[b * M * N + m * N + n] = A[idx]; + } else { + // (B*M, N) → (M, B*N) + const unsigned int b = idx / (M * N); + const unsigned int rem2 = idx % (M * N); + const unsigned int m = rem2 / N; + const unsigned int n = rem2 % N; + out[m * B * N + b * N + n] = A[idx]; + } +} + +// ══════════════════════════════════════════════════════════════════════════ +// Gather / Scatter 内核 +// ══════════════════════════════════════════════════════════════════════════ + +__global__ void gather_kernel( + const float* __restrict__ table, + const float* __restrict__ indices, + float* __restrict__ out, + unsigned int vocab, unsigned int D, unsigned int num) +{ + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= num * D) return; + + const unsigned int i = idx / D; + const unsigned int d = idx % D; + const int row_idx = static_cast(indices[i]); + + if (row_idx >= 0 && static_cast(row_idx) < vocab) + out[idx] = table[static_cast(row_idx) * D + d]; + else + out[idx] = 0.0f; +} + +__global__ void scatter_add_kernel( + float* __restrict__ dst, + const float* __restrict__ indices, + const float* __restrict__ grad, + unsigned int vocab, unsigned int D, unsigned int num) +{ + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= num * D) return; + + const unsigned int i = idx / D; + const unsigned int d = idx % D; + const int row_idx = static_cast(indices[i]); + + if (row_idx >= 0 && static_cast(row_idx) < vocab) + atomicAdd(&dst[static_cast(row_idx) * D + d], grad[idx]); +} + +// ══════════════════════════════════════════════════════════════════════════ +// C-linkage 接口实现 +// ══════════════════════════════════════════════════════════════════════════ + +static constexpr int BLOCK_SIZE = 256; + +static cudaStream_t to_stream(void* s) +{ + return s ? static_cast(s) : cudaStream_t(0); +} + +// ── 矩阵乘法 ───────────────────────────────────────────────────────────── + +// 内部辅助：使用预创建的 cuBLAS 句柄执行 Sgemm（行主序 → 列主序转换） +// 行主序矩阵 A(A_rows × A_cols) 在内存中按行存储，行步长 = A_cols。 +// cuBLAS 按列主序读取时，看到的是 (A_cols × A_rows) 的矩阵，leading dim = A_cols。 +// 关键：leading dim 是内存布局属性，与 transA/transB（逻辑转置）无关，始终等于行步长。 +#ifdef NN_HAS_CUBLAS +static int cublas_matmul_impl( + cublasHandle_t handle, + const float* A, const float* B, float* C, + std::size_t A_rows, std::size_t A_cols, + std::size_t B_rows, std::size_t B_cols, + int transA, int transB) +{ + const int M = static_cast(transA ? A_cols : A_rows); + const int K = static_cast(transA ? A_rows : A_cols); + const int K2 = static_cast(transB ? B_cols : B_rows); + const int N = static_cast(transB ? B_rows : B_cols); + + if (K != K2) return -1; + + // cuBLAS 列主序：行主序 C = A × B ⟺ 列主序 C^T = B^T × A^T + // 调用形式：cublasSgemm(handle, opB, opA, N, M, K, ..., B, ldb, A, lda, ..., C, ldc) + // 第一操作数 = B（行主序 B），其列主序 leading dim = B_cols（行步长） + // 第二操作数 = A（行主序 A），其列主序 leading dim = A_cols（行步长） + // 结果 C（行主序 M×N），其列主序 leading dim = N（行步长） + const float alpha = 1.0f, beta = 0.0f; + const int lda = static_cast(A_cols); // 行主序 A 的行步长（固定，与 transA 无关） + const int ldb = static_cast(B_cols); // 行主序 B 的行步长（固定，与 transB 无关） + const int ldc = N; // 行主序 C 的行步长 + + cublasOperation_t opA = transA ? CUBLAS_OP_T : CUBLAS_OP_N; + cublasOperation_t opB = transB ? CUBLAS_OP_T : CUBLAS_OP_N; + + CUBLAS_CHECK(cublasSgemm(handle, + opB, opA, + N, M, K, + &alpha, + B, ldb, + A, lda, + &beta, + C, ldc)); + return 0; +} +#endif + +extern "C" int cuda_matmul( + const float* A, const float* B, float* C, + std::size_t A_rows, std::size_t A_cols, + std::size_t B_rows, std::size_t B_cols, + int transA, int transB, + void* stream) +{ + const int K = static_cast(transA ? A_rows : A_cols); + const int K2 = static_cast(transB ? B_cols : B_rows); + + if (K != K2) return -1; + +#ifdef NN_HAS_CUBLAS + // 兼容路径：无外部句柄时创建临时句柄（推荐使用 cuda_matmul_with_handle 复用句柄） + cublasHandle_t handle; + cublasStatus_t st = cublasCreate(&handle); + if (st != CUBLAS_STATUS_SUCCESS) return static_cast(-st); + cublasSetStream(handle, to_stream(stream)); + + int ret = cublas_matmul_impl(handle, A, B, C, + A_rows, A_cols, B_rows, B_cols, transA, transB); + + cublasDestroy(handle); + return ret; +#else + // 自定义内核 + const int M = static_cast(transA ? A_cols : A_rows); + const int N = static_cast(transB ? B_rows : B_cols); + dim3 threads(TILE, TILE); + dim3 blocks((N + TILE - 1) / TILE, (M + TILE - 1) / TILE); + matmul_kernel<<>>( + A, B, C, M, N, K, transA, transB); + CUDA_CHECK(cudaGetLastError()); + return 0; +#endif +} + +// 带预创建 cuBLAS 句柄的 matmul（避免每次调用创建/销毁句柄的开销） +// handle_ptr 为 nullptr 时退化为 cuda_matmul +extern "C" int cuda_matmul_with_handle( + const float* A, const float* B, float* C, + std::size_t A_rows, std::size_t A_cols, + std::size_t B_rows, std::size_t B_cols, + int transA, int transB, + void* handle_ptr) +{ +#ifdef NN_HAS_CUBLAS + if (!handle_ptr) + return cuda_matmul(A, B, C, A_rows, A_cols, B_rows, B_cols, transA, transB, nullptr); + + cublasHandle_t handle = static_cast(handle_ptr); + return cublas_matmul_impl(handle, A, B, C, + A_rows, A_cols, B_rows, B_cols, transA, transB); +#else + // 无 cuBLAS 时退化为自定义内核（忽略 handle_ptr） + return cuda_matmul(A, B, C, A_rows, A_cols, B_rows, B_cols, transA, transB, nullptr); +#endif +} + +extern "C" int cuda_batched_matmul( + const float* A, const float* B, float* C, + std::size_t A_rows, std::size_t A_cols, + std::size_t B_rows, std::size_t B_cols, + unsigned int batch, int transA, int transB, + void* stream) +{ + if (batch == 0) return -1; + + const int a_rows_per = static_cast(A_rows / batch); + const int b_rows_per = static_cast(B_rows / batch); + const int M = transA ? static_cast(A_cols) : a_rows_per; + const int K = transA ? a_rows_per : static_cast(A_cols); + const int K2 = transB ? static_cast(B_cols) : b_rows_per; + const int N = transB ? b_rows_per : static_cast(B_cols); + + if (K != K2) return -1; + + dim3 threads(TILE, TILE); + dim3 blocks((N + TILE - 1) / TILE, (M + TILE - 1) / TILE, batch); + batched_matmul_kernel<<>>( + A, B, C, M, N, K, transA, transB, static_cast(batch)); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +// ── 逐元素运算 ─────────────────────────────────────────────────────────── + +extern "C" int cuda_elementwise_unary( + const float* A, float* out, + unsigned int count, unsigned int op, + void* stream) +{ + const int blocks = (count + BLOCK_SIZE - 1) / BLOCK_SIZE; + elementwise_unary_kernel<<>>( + A, out, count, op); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +extern "C" int cuda_elementwise_binary( + const float* A, const float* B, float* out, + unsigned int count, unsigned int op, + void* stream) +{ + const int blocks = (count + BLOCK_SIZE - 1) / BLOCK_SIZE; + elementwise_binary_kernel<<>>( + A, B, out, count, op); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +extern "C" int cuda_elementwise_binary_scalar( + const float* A, float* out, + unsigned int count, unsigned int op, + float scalar, int scalar_first, + void* stream) +{ + const int blocks = (count + BLOCK_SIZE - 1) / BLOCK_SIZE; + elementwise_binary_scalar_kernel<<>>( + A, out, count, op, scalar, scalar_first); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +extern "C" int cuda_axpy( + const float* A, const float* B, float* out, + unsigned int count, float scalar, + void* stream) +{ + const int blocks = (count + BLOCK_SIZE - 1) / BLOCK_SIZE; + axpy_kernel<<>>( + A, B, out, count, scalar); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +extern "C" int cuda_elementwise_select_scalar_cond( + const float* A, const float* then_v, float* out, + unsigned int count, unsigned int cmp, + float scalar_b, float scalar_else, + void* stream) +{ + const int blocks = (count + BLOCK_SIZE - 1) / BLOCK_SIZE; + select_scalar_cond_kernel<<>>( + A, then_v, out, count, cmp, scalar_b, scalar_else); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +// ── 归约 ───────────────────────────────────────────────────────────────── + +extern "C" int cuda_reduce_row( + const float* A, float* out, + unsigned int rows, unsigned int cols, unsigned int mode, + void* stream) +{ + const int threads = 256; + const size_t shared_mem = threads * sizeof(float); + + if (mode == 0) + reduce_row_sum_kernel<<>>( + A, out, rows, cols); + else + reduce_row_max_kernel<<>>( + A, out, rows, cols); + + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +extern "C" int cuda_reduce_col( + const float* A, float* out, + unsigned int rows, unsigned int cols, unsigned int mode, + void* stream) +{ + // 新版 col_reduce：每个 block 处理一列，线程协作归约所有行（shared memory tree reduction） + // grid = (cols,)，每个 block threads 个线程协作 + const int threads = 256; + const size_t shared_mem = threads * sizeof(float); + + if (mode == 0) + reduce_col_sum_kernel<<>>( + A, out, rows, cols); + else + reduce_col_max_kernel<<>>( + A, out, rows, cols); + + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +// ── 广播 ───────────────────────────────────────────────────────────────── + +extern "C" int cuda_broadcast_row( + const float* A, const float* vec, float* out, + unsigned int rows, unsigned int cols, unsigned int op, + void* stream) +{ + const unsigned int total = rows * cols; + const int blocks = (total + BLOCK_SIZE - 1) / BLOCK_SIZE; + broadcast_row_kernel<<>>( + A, vec, out, rows, cols, op); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +extern "C" int cuda_broadcast_col( + const float* A, const float* vec, float* out, + unsigned int rows, unsigned int cols, unsigned int op, + void* stream) +{ + const unsigned int total = rows * cols; + const int blocks = (total + BLOCK_SIZE - 1) / BLOCK_SIZE; + broadcast_col_kernel<<>>( + A, vec, out, rows, cols, op); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +// ── 转置与重排 ─────────────────────────────────────────────────────────── + +extern "C" int cuda_transpose( + const float* A, float* out, + unsigned int R, unsigned int C, + void* stream) +{ + dim3 threads(TILE, TILE); + dim3 blocks((C + TILE - 1) / TILE, (R + TILE - 1) / TILE); + transpose_kernel<<>>( + A, out, R, C); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +extern "C" int cuda_rearrange_3d( + const float* A, float* out, + unsigned int M, unsigned int B, unsigned int N, + unsigned int inverse, + void* stream) +{ + const unsigned int total = M * B * N; + const int blocks = (total + BLOCK_SIZE - 1) / BLOCK_SIZE; + rearrange_3d_kernel<<>>( + A, out, M, B, N, inverse, total); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +// ── Gather / Scatter ───────────────────────────────────────────────────── + +extern "C" int cuda_gather( + const float* table, const float* indices, float* out, + unsigned int vocab, unsigned int D, unsigned int num, + void* stream) +{ + const unsigned int total = num * D; + const int blocks = (total + BLOCK_SIZE - 1) / BLOCK_SIZE; + gather_kernel<<>>( + table, indices, out, vocab, D, num); + CUDA_CHECK(cudaGetLastError()); + return 0; +} + +extern "C" int cuda_scatter_add( + float* dst, const float* indices, const float* grad, + unsigned int vocab, unsigned int D, unsigned int num, + void* stream) +{ + const unsigned int total = num * D; + const int blocks = (total + BLOCK_SIZE - 1) / BLOCK_SIZE; + scatter_add_kernel<<>>( + dst, indices, grad, vocab, D, num); + CUDA_CHECK(cudaGetLastError()); + return 0; +} diff --git a/cuda/cuda_kernels.h b/cuda/cuda_kernels.h new file mode 100644 index 00000000..125e81c1 --- /dev/null +++ b/cuda/cuda_kernels.h @@ -0,0 +1,171 @@ +// ── cuda_kernels.h — CUDA 内核 C-linkage 接口 ───────────────────────────── +// 定义所有 CUDA 计算内核的函数签名。 +// 实现在 cuda_kernels.cu 中，由 nvcc 编译为静态库。 +// 主项目通过 CudaBackend 调用这些函数（C ABI，跨编译器链接安全）。 +// +// 所有函数使用 float 类型（与 Scalar = float 一致）。 +// 错误通过返回 int 传递：0 = 成功，非 0 = cudaError_t。 +// ───────────────────────────────────────────────────────────────────────── + +#ifndef NN_CUDA_KERNELS_H +#define NN_CUDA_KERNELS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// ══════════════════════════════════════════════════════════════════════════ +// 矩阵乘法 +// ══════════════════════════════════════════════════════════════════════════ + +// C = A × B（支持转置标志） +// A: (A_rows, A_cols), B: (B_rows, B_cols), C: (M, N) +// transA: 使用 A^T, transB: 使用 B^T +// stream: CUDA 流（0 = 默认流） +int cuda_matmul( + const float* A, const float* B, float* C, + std::size_t A_rows, std::size_t A_cols, + std::size_t B_rows, std::size_t B_cols, + int transA, int transB, + void* stream); + +// 带预创建 cuBLAS 句柄的 matmul（避免每次调用创建/销毁句柄的开销） +// handle_ptr: cuBLAS 句柄指针（由 CudaBackend 创建并复用），nullptr 时退化为 cuda_matmul +// 注：当未启用 NN_HAS_CUBLAS 时，本函数退化为自定义内核（忽略 handle_ptr） +int cuda_matmul_with_handle( + const float* A, const float* B, float* C, + std::size_t A_rows, std::size_t A_cols, + std::size_t B_rows, std::size_t B_cols, + int transA, int transB, + void* handle_ptr); + +// 批量矩阵乘法：对每个 batch 计算 C_b = op(A_b, B_b)，结果垂直堆叠 +// A: (batch * a_rows_per, A_cols), B: (batch * b_rows_per, B_cols) +// C: (batch * M, N) +int cuda_batched_matmul( + const float* A, const float* B, float* C, + std::size_t A_rows, std::size_t A_cols, + std::size_t B_rows, std::size_t B_cols, + unsigned int batch, int transA, int transB, + void* stream); + +// ══════════════════════════════════════════════════════════════════════════ +// 逐元素运算 +// ══════════════════════════════════════════════════════════════════════════ + +// 一元运算：out = op(A) +// op: 0=Neg, 1=Exp, 2=Log, 3=Sqrt, 4=Rsqrt, 5=Abs, 6=Tanh +int cuda_elementwise_unary( + const float* A, float* out, + unsigned int count, unsigned int op, + void* stream); + +// 二元运算：out = op(A, B) +// op: 0=Add, 1=Sub, 2=Mul, 3=Div, 4=Max, 5=Min +int cuda_elementwise_binary( + const float* A, const float* B, float* out, + unsigned int count, unsigned int op, + void* stream); + +// 标量二元运算：out = op(A, scalar) 或 op(scalar, A) +// scalar_first: true → op(scalar, A), false → op(A, scalar) +int cuda_elementwise_binary_scalar( + const float* A, float* out, + unsigned int count, unsigned int op, + float scalar, int scalar_first, + void* stream); + +// 融合 axpy：out = A + scalar * B +int cuda_axpy( + const float* A, const float* B, float* out, + unsigned int count, float scalar, + void* stream); + +// 条件选择：out = cmp(A, scalar_b) ? then_v : scalar_else +// cmp: 0=Lt, 1=Le, 2=Gt, 3=Ge, 4=Eq, 5=Ne +int cuda_elementwise_select_scalar_cond( + const float* A, const float* then_v, float* out, + unsigned int count, unsigned int cmp, + float scalar_b, float scalar_else, + void* stream); + +// ══════════════════════════════════════════════════════════════════════════ +// 归约 +// ══════════════════════════════════════════════════════════════════════════ + +// 行归约：A (rows, cols) → out (rows, 1) +// mode: 0=sum, 1=max +int cuda_reduce_row( + const float* A, float* out, + unsigned int rows, unsigned int cols, unsigned int mode, + void* stream); + +// 列归约：A (rows, cols) → out (1, cols) +// mode: 0=sum, 1=max +int cuda_reduce_col( + const float* A, float* out, + unsigned int rows, unsigned int cols, unsigned int mode, + void* stream); + +// ══════════════════════════════════════════════════════════════════════════ +// 广播 +// ══════════════════════════════════════════════════════════════════════════ + +// 行广播：out[r][c] = op(A[r][c], vec[r]) +// op: 0=Add, 1=Sub, 2=Mul, 3=Div, 4=Max, 5=Min +int cuda_broadcast_row( + const float* A, const float* vec, float* out, + unsigned int rows, unsigned int cols, unsigned int op, + void* stream); + +// 列广播：out[r][c] = op(A[r][c], vec[c]) +int cuda_broadcast_col( + const float* A, const float* vec, float* out, + unsigned int rows, unsigned int cols, unsigned int op, + void* stream); + +// ══════════════════════════════════════════════════════════════════════════ +// 转置与重排 +// ══════════════════════════════════════════════════════════════════════════ + +// 矩阵转置：A (R, C) → out (C, R) +int cuda_transpose( + const float* A, float* out, + unsigned int R, unsigned int C, + void* stream); + +// 3D 维度重排：(M, B, N) ↔ (B, M, N) +// inverse=0: (M, B*N) → (B*M, N) +// inverse=1: (B*M, N) → (M, B*N) +int cuda_rearrange_3d( + const float* A, float* out, + unsigned int M, unsigned int B, unsigned int N, + unsigned int inverse, + void* stream); + +// ══════════════════════════════════════════════════════════════════════════ +// Gather / Scatter +// ══════════════════════════════════════════════════════════════════════════ + +// 按行索引查表：out[i] = table[indices[i]] (i < num) +// table: (vocab, D), indices: (num,), out: (num, D) +// 越界索引返回零行 +int cuda_gather( + const float* table, const float* indices, float* out, + unsigned int vocab, unsigned int D, unsigned int num, + void* stream); + +// 按行索引原子累加：dst[indices[i]] += grad[i] +// dst: (vocab, D) 原地修改, indices: (num,), grad: (num, D) +int cuda_scatter_add( + float* dst, const float* indices, const float* grad, + unsigned int vocab, unsigned int D, unsigned int num, + void* stream); + +#ifdef __cplusplus +} +#endif + +#endif // NN_CUDA_KERNELS_H diff --git a/docs/01-architecture.md b/docs/01-architecture.md new file mode 100644 index 00000000..89384a9d --- /dev/null +++ b/docs/01-architecture.md @@ -0,0 +1,289 @@ +# 🧠 neuralnet.cpp 架构设计 + +> 一个从零实现的 C++26 神经网络库，支持 CPU/GPU 双后端、多层 Transformer、GPT 语言模型训练与推理。 + +--- + +## 📐 系统分层总览 + +项目采用 **6 层分层架构**（L0 ~ L5），每层职责单一、严格隔离，上层仅依赖下层的公有接口。 + +```mermaid +graph TB + subgraph "🖥️ L5 用户入口层" + A["mnist_train / mnist_infer"] + B["text_train / text_infer"] + C["tokenizer_train / tokenizer_infer"] + D["gui.py (Python GUI)"] + end + + subgraph "🧩 L4 领域构建层" + E["domain_mnist.hpp — MNIST 模型工厂"] + F["domain_gpt.hpp — GPT 模型工厂"] + G["domain_tokenizer.hpp — 分词器"] + end + + subgraph "🧩 L3 实现层" + H["model_container.hpp — Model 容器"] + I["model_serialization.hpp — 二进制序列化"] + J["model_spec.hpp — 架构描述"] + end + + subgraph "🧩 L2 计算层（引擎化）" + K["compute_engine.hpp — 引擎抽象接口"] + L["cpu_engine.hpp — CPU 引擎"] + M["gpu_engine.hpp — GPU 引擎 (Vulkan)"] + N["compute_layer.hpp — Layer 基类 + 所有层"] + O["compute_loss.hpp — 损失函数"] + P["compute_optimizer.hpp — 优化器"] + end + + subgraph "🧩 L1 代数层" + Q["algebra_matrix.hpp — 矩阵类 + 运算原语"] + R["algebra_expr.hpp — 表达式模板"] + S["algebra_ops.hpp — 逐元素算子"] + T["algebra_compute.hpp — 计算分派"] + U["algebra_span.hpp — Span 抽象"] + end + + subgraph "🧩 L0 硬件层" + V["config.hpp — SmartPolicy / BLOCK_SIZE"] + W["core_threadpool.hpp — 全局线程池"] + X["core_errors.hpp — Result<T> = std::expected<T, Error>"] + Y["core_assert.hpp — 断言宏"] + end + + D -->|"subprocess"| A & B & C + A & B & C --> E & F & G + E & F & G --> H & I & J + H & I & J --> K & N & O & P + K --> L & M + N & O & P --> Q + Q --> R & S & T & U + Q --> V & W & X & Y +``` + +--- + +## 🔑 核心设计原则 + +### 1. 引擎化架构（Engine-Based Architecture） + +这是项目最核心的设计理念： + +``` +Layer 的 forward/backward 只写一次，通过 ComputeEngine 参数自动适配 CPU/GPU。 +不再有 forward_gpu / backward_gpu。 +``` + +```cpp +// compute_layer.hpp — 所有 Layer 统一接口 +class Layer { +public: + virtual Result forward(ComputeEngine& engine, const Tensor& input) = 0; + virtual Result backward(ComputeEngine& engine, const Tensor& grad_output) = 0; +}; +``` + +**数据流（GPU 训练为例）：** + +``` +Matrix(x_batch) + ──engine.from_matrix──▶ Tensor[GPU] + ──forward──▶ Tensor[GPU] + ──loss.forward──▶ Scalar + ──loss.backward──▶ Tensor[GPU] + ──backward──▶ (丢弃) + ──optimizer.step──▶ 参数就地更新（全程在 GPU） +``` + +仅 `evaluate` 时 `engine.to_matrix` 下载到 CPU 做 argmax。 + +### 2. 算法与原语分离 + +| 层级 | 职责 | 示例 | +|------|------|------| +| **ComputeEngine** (L2) | 只提供 op-level 原语 | `matmul`, `add`, `exp`, `max`, `reduce` | +| **Layer** (L2) | 通过组合原语表达算法 | `ReLU = max(x, 0)`, `GeLU = 5 次原语组合` | +| **Shader** (GPU) | 引擎内部实现，用户不可见 | `matmul.comp`, `elementwise.comp` | + +这意味着 **Engine/Shader 不知道 "ReLU" 是什么**，它只提供 `max` 原语。Layer 用 `max(x, 0)` 表达 ReLU 算法。 + +### 3. 零手动内存管理 + +- `Matrix` 内部使用 `std::vector`，无裸指针 +- `Tensor` 使用 `shared_ptr` 内部持有存储，拷贝廉价（零拷贝传递） +- 错误处理使用 C++23 `std::expected`，项目中 **禁止 throw/try/catch** +- 编译时通过 `-fno-exceptions` 强制执行 + +### 4. 统一张量类型 `Tensor` + +`Tensor` 是跨设备的统一数据容器： + +```cpp +class Tensor { + Device device_; // CPU 或 GPU + std::shared_ptr cpu_data_; // CPU 存储 + std::shared_ptr gpu_data_; // GPU 存储（条件编译） +}; +``` + +- CPU/GPU 存储互斥：`device()` 决定哪个指针有效 +- 跨设备传输由 `ComputeEngine` 负责，`Tensor` 本身不主动迁移 +- `shared_ptr` 使得拷贝廉价，传递零开销 + +--- + +## 📂 模块详解 + +### L0 硬件层 + +| 文件 | 职责 | +|------|------| +| `config.hpp` | `Scalar` 类型定义、`BLOCK_SIZE=64`、`PARALLEL_THRESHOLD`、SmartPolicy 自适应策略 | +| `core_threadpool.hpp` | 全局线程池，latch 零分配设计，`parallel_for_each` / `parallel_transform` | +| `core_errors.hpp` | `struct Error { std::string message; }` + `using Result = std::expected` | +| `core_assert.hpp` | `NN_ASSERT` 宏（L1 层形状校验使用，L2+ 层使用 Result） | +| `core_file.hpp` | 文件加载工具 | + +### L1 代数层 + +| 文件 | 职责 | +|------|------| +| `algebra_matrix.hpp` | `Matrix` 类：行主序存储 `(rows, cols)`，矩阵乘法（缓存分块 + SmartPolicy 并行），加法、转置、归约 | +| `algebra_expr.hpp` | 表达式模板（AST），实现编译期零开销逐元素运算 | +| `algebra_ops.hpp` | 逐元素算子：`ReLU`, `GeLU`, `Neg`, `Exp`, `Add`, `Mul` 等 | +| `algebra_span.hpp` | `Span` 抽象，提供对矩阵数据的安全视图 | +| `algebra_compute.hpp` | `compute::apply(span, expr)` — AST 统一入口 | + +### L2 计算层（引擎化） + +**ComputeEngine 原语分类：** + +| 类别 | 原语 | +|------|------| +| 矩阵级 | `matmul`, `batched_matmul`, `transpose`, `add_inplace`, `scale_inplace`, `zero`, `axpy_inplace` | +| 归约级 | `row_reduce_sum`, `col_reduce_sum`, `row_reduce_max`, `col_reduce_max` | +| 广播级 | `broadcast_row_inplace`, `broadcast_col_inplace` | +| 逐元素 | `elementwise_unary`, `elementwise_binary`, `elementwise_binary_scalar` | +| 条件选择 | `elementwise_select_scalar_cond` | +| 数据操作 | `slice_rows`, `insert_rows`, `gather_rows`, `scatter_add_rows`, `rearrange_3d`, `clone` | +| 批控制 | `begin_batch` / `end_batch`（CPU: no-op; GPU: command buffer） | + +### L3 实现层 + +| 文件 | 职责 | +|------|------| +| `model_container.hpp` | `Model` 容器：持有 `vector>` + `ComputeEngine*`，链式构建，`forward`/`backward` 委托 | +| `model_spec.hpp` | `ModelSpec` 纯数据结构：`ModelType` 枚举 + MLP/Transformer/GPT 参数 | +| `model_serialization.hpp` | 二进制序列化 V1/V2/V3 格式，`save_model` / `load_model` / `peek_model_spec` | + +### L4 领域构建层 + +| 文件 | 职责 | +|------|------| +| `domain_mnist.hpp` | MNIST 工厂函数：`build_mnist_mlp_model()` / `build_mnist_transformer_model()` | +| `domain_gpt.hpp` | GPT 工厂函数：`build_gpt_model()` / `build_gpt_model_from_spec()` | +| `domain_tokenizer.hpp` | 分词器：`Tokenizer` 基类 + `WordZipTokenizer` / `BPETokenizer` / `ByteZipTokenizer` | + +--- + +## 📊 数据布局约定 + +所有张量采用 **列主序 batch-major** 布局（column-major, batch-major）： + +``` +输入: (feature_dim, batch_size) — 每列一个样本 +输出: (out_dim, batch_size) — 每列一个样本 +权重: (out_features, in_features) — 行主序存储 + +Matrix 索引: data_[row * cols + col] +``` + +**批量注意力的特殊布局：** + +``` +多头 Q/K/V: (H*d_k, batch*seq) — 头维度在行方向，batch 在列方向 +rearrange 后: (batch*H*d_k, seq) — 使 batched_matmul 能按 batch*H 切分 +``` + +--- + +## 🔄 完整训练流程 + +```mermaid +sequenceDiagram + participant U as User Code + participant M as Model + participant E as ComputeEngine + participant L as Layer(s) + participant O as Optimizer + participant F as Loss + + U->>E: create engine (CPU/GPU) + U->>M: build model(engine) + U->>O: create optimizer(engine, params) + + loop Each Training Step + U->>M: zero_grad(engine) + U->>E: from_matrix(batch) → Tensor + M->>L: forward(engine, Tensor) → Tensor + L-->>M: output Tensor + M-->>U: logits + + U->>F: forward(engine, logits, target) → Scalar + U->>F: backward() → grad Tensor + U->>M: backward(engine, grad) → grad (discard) + + U->>O: step() + O->>E: scale_inplace / add_inplace / ... + end + + U->>M: forward(engine, test_input) → Tensor + U->>E: to_matrix(Tensor) → Matrix + U->>U: argmax → predict +``` + +--- + +## 🏗️ 构建与编译 + +**环境要求：** +- CMake ≥ 3.30 +- Clang (支持 C++26) 或 GCC 15+ +- Vulkan SDK（可选，GPU 加速） + +**构建命令：** + +```bash +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --parallel +``` + +**产物：** + +| 可执行文件 | 用途 | +|-----------|------| +| `mnist_train` | MNIST 手写数字训练 | +| `mnist_infer` | MNIST 推理 | +| `text_train` | GPT 文本生成训练 | +| `text_infer` | GPT 文本推理 | +| `tokenizer_train` | BPE 分词器训练 | +| `tokenizer_infer` | BPE 分词器推理 | +| `compute_bench` | 计算性能基准测试 | +| `bench_thresholds` | 并行阈值基准测试 | +| `gpu_test` | GPU 后端测试 | + +--- + +## 📁 完整头文件依赖图 + +``` +nn.hpp（统一入口） +├── L0: core_errors → config → core_threadpool +├── L1: algebra_matrix → algebra_span / algebra_ops / algebra_expr / algebra_compute +├── L2: tensor → compute_engine → cpu_engine [→ gpu_engine] +│ → compute_layer → compute_loss → compute_optimizer +├── L3: model_container → model_spec → model_serialization +└── L4: domain_mnist → domain_gpt → domain_tokenizer +``` diff --git a/docs/02-performance.md b/docs/02-performance.md new file mode 100644 index 00000000..51045817 --- /dev/null +++ b/docs/02-performance.md @@ -0,0 +1,369 @@ +# ⚡ 性能优化手段 + +> 本文档详细梳理 neuralnet.cpp 项目中的所有性能优化策略，涵盖 CPU 多线程、缓存优化、GPU 加速、内存管理、算法级优化等层面。 + +--- + +## 📊 优化全景图 + +``` +┌─────────────────────────────────────────────────────────────┐ +│ 性能优化层次 │ +├─────────────────────────────────────────────────────────────┤ +│ L5 算法级 │ 批量化注意力、融合算子、axpy 融合 │ +│ L4 GPU 加速 │ Vulkan Compute、批量提交、Command Buffer │ +│ L3 内存级 │ 缓存分块、零分配热路径、Tensor 零拷贝 │ +│ L2 并行级 │ SmartPolicy 自适应并行、线程池 latch 零分配 │ +│ L1 指令级 │ AVX2 SIMD、-ffast-math、-funroll-loops │ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## 1. SmartPolicy 自适应并行策略 + +### 原理 + +根据数据规模自动决定串行还是并行执行，避免小数据量时线程调度开销（约 50~200μs）大于计算收益。 + +```cpp +// config.hpp +inline constexpr std::size_t PARALLEL_THRESHOLD = 524288; // 512K 元素 +``` + +### 实测数据 + +| 元素数 | 串行 μs | 并行 μs | 加速比 | 建议 | +|--------|---------|---------|--------|------| +| 16384 | 9.6 | 9.8 | 0.98x | 串行 | +| 65536 | 38.7 | 121.7 | 0.32x | 串行 | +| 524288 | 308.7 | 201.5 | **1.53x** | ← 首次稳定 > 1.5x | +| 1048576 | 620.3 | 186.7 | **3.32x** | 并行 | +| 4194304 | 2657.3 | 340.6 | **7.80x** | 并行 | + +### 实现方式 + +所有顶层并行函数自动判断： + +```cpp +// config.hpp — 顶层并行算法函数 +template +inline void for_each(Iterator first, Iterator last, Func&& func) +{ + const auto n = std::distance(first, last); + if (n >= PARALLEL_THRESHOLD) + global_thread_pool().parallel_for_each(first, last, func); + else + for (auto it = first; it != last; ++it) func(*it); +} +``` + +--- + +## 2. 线程池 latch 零分配设计 + +### 旧版问题 + +每次 `submit` 需要： +1. 构造 `shared_ptr` → **堆分配** +2. 获取 `future` → **同步开销** +3. 加锁入队 → **互斥开销** + +### 新版优化 + +- **一次加锁批量入队**：N 个分块仅 1 次 `lock_guard` +- **原子计数器替代 future**：`std::atomic latch` 替代 N 个 future +- **调用者参与计算**：调用者不空等，而是从队列取任务执行（work-stealing） +- **三阶段等待策略**： + 1. 短自旋（16 次 `pause` 指令）— 应对 latch 即将归零的快路径 + 2. Work-stealing：尝试从队列取任务执行 + 3. `condition_variable` 短超时等待（100μs），避免 CPU 空转 + +### 性能收益 + +``` +旧版：N 分块 = N 次堆分配 + N 次加锁 + N 个 future 同步 +新版：N 分块 = 1 次加锁 + 0 次堆分配 + 1 个原子计数器 +``` + +--- + +## 3. 矩阵乘法缓存分块（Tiling） + +### 原理 + +将矩阵乘法分解为 BLOCK_SIZE × BLOCK_SIZE 的小块，确保每块的工作集装入 CPU L1 缓存（32KB）。 + +```cpp +// config.hpp +inline constexpr std::size_t BLOCK_SIZE = 64; +// 64 × 64 × 8 bytes = 32KB — 安全装入大多数 CPU 的 L1 缓存 +``` + +### 分块矩阵乘法内核 + +``` +A (M×K) × B (K×N) = C (M×N) + +for i_block in range(0, M, BLOCK_SIZE): + for j_block in range(0, N, BLOCK_SIZE): + for k_block in range(0, K, BLOCK_SIZE): + // 从 B 中预取一块到本地数组 b_block（转置布局） + // 累加 C 的一块 +``` + +关键优化： +- **B 块预取转置**：`b_block[jj * k_len + kk]` — 消除内层循环的跨行访问 +- **栈分配 b_block**：`std::array` — 零堆分配 +- **分块级并行**：每个 `(i_block, j_block)` 块独立，可并行调度 + +--- + +## 4. 编译器优化选项 + +### Release 模式编译标志 + +```cmake +# CMakeLists.txt +set(RELEASE_FLAGS -O3 -ffast-math -funroll-loops -march=native) +``` + +| 标志 | 作用 | +|------|------| +| `-O3` | 最高级别优化（内联、循环变换、向量化） | +| `-ffast-math` | 允许浮点重排序、忽略 NaN/Infinity（提升 FMA 吞吐） | +| `-funroll-loops` | 循环展开，减少分支预测开销 | +| `-march=native` | 启用本机 CPU 指令集（AVX2/AVX-512 等 SIMD） | + +--- + +## 5. GPU 加速（Vulkan Compute） + +### 架构设计 + +GPU 加速遵循引擎化架构铁律： +- `ComputeEngine` 抽象接口不变 +- `GpuEngine` 实现所有原语的 GPU 版本 +- Layer 的 forward/backward **零修改** + +### 关键设计 + +#### 5.1 批量命令提交 + +```cpp +// GPU 引擎：begin_batch 开始录制，end_batch 统一提交 +engine.begin_batch(); // 开始录制 +engine.matmul(A, B); // 不立即执行 +engine.elementwise_binary(...); // 不立即执行 +engine.end_batch(); // 一次性提交 + fence wait +``` + +- CPU 引擎：`begin_batch/end_batch` 为 no-op（同步执行） +- GPU 引擎：录制到 Command Buffer，`end_batch` 统一提交 + +#### 5.2 零层间 PCIe 传输 + +``` +训练数据从 CPU 上传一次 → 全程 GPU 计算 → 仅 evaluate 时下载 +``` + +#### 5.3 算子融合 + +- `axpy_inplace`：融合 `clone + scale + add` 三步为一次 dispatch +- `elementwise_select_scalar_cond`：融合条件选择（ReLU backward） +- `broadcast_row_inplace` / `broadcast_col_inplace`：融合广播 + 逐元素操作 + +--- + +## 6. 张量零拷贝传递 + +### Tensor 的 shared_ptr 语义 + +```cpp +class Tensor { + std::shared_ptr cpu_data_; // 引用计数 + std::shared_ptr gpu_data_; // 引用计数 +}; +``` + +- **拷贝 Tensor = 增加引用计数**，不复制底层数据 +- **修改需要显式 clone**：`engine.clone(src)` 创建深拷贝 +- Layer 间传递 Tensor 全部零拷贝 + +### 引擎辅助方法 + +```cpp +// 仅在需要修改但不影响原 Tensor 时使用 +auto diff = clone_tensor(engine, input); // 深拷贝 +// 原地修改 diff，input 不受影响 +engine.broadcast_col_inplace(*diff, mean, BinaryOp::Sub); +``` + +--- + +## 7. 融合 Axpy（scale + add） + +### 问题 + +旧实现中 `dst += scalar * src` 需要 3 步： + +``` +clone_tensor → scale_inplace → add_inplace += 3 个 GPU 原语 + 3 个临时 buffer +``` + +### 优化 + +```cpp +// 新实现：单次 dispatch +engine.axpy_inplace(dst, scalar, src); +// = 1 个 GPU 原语 + 1 个临时 buffer +``` + +### 性能收益 + +``` +100 个参数 × 每 step 调用 ~3 次 = 减少 600 次 GPU buffer 分配/step +``` + +--- + +## 8. 多头注意力批量化 + +### 旧版：per-head 循环 + +```cpp +for (size_t h = 0; h < num_heads; ++h) { // N 次循环 + Q_h = slice(Q, h*d_k, (h+1)*d_k); + scores = Q_h × K_h^T; + attn = softmax(scores); + O_h = attn × V_h; +} +output = concat(O_0, O_1, ..., O_{H-1}); +``` + +### 新版：批量 dispatch + +``` +rearrange_3d → (batch*H*d_k, seq) // 头维度在行方向 +batched_matmul → 单次 dispatch 处理所有样本和所有头 +rearrange_3d_back → (H*d_k, batch*seq) +``` + +**性能提升**：将 H 次 matmul 融合为 1 次 `batched_matmul`。 + +--- + +## 9. 因果掩码缓存 + +### 问题 + +GPT 训练中，每个 forward 都需要创建因果掩码矩阵 `(batch*H*seq, seq)`。 + +### 优化 + +```cpp +class CausalSelfAttention { + Tensor mask_cache_; + std::size_t mask_cached_key_ = 0; // (batch << 16) | seq_len + + void ensure_mask(engine, batch, seq_len) { + const std::size_t key = (batch << 16) | seq_len; + if (mask_cached_key_ == key) return; // 已缓存，跳过 + // ... 构造掩码 ... + mask_cached_key_ = key; + } +}; +``` + +**收益**：相同 `(batch, seq_len)` 组合只构造一次掩码。 + +--- + +## 10. 位置编码缓存 + +```cpp +class PositionalEncoding { + Tensor encoding_cache_; + std::size_t cached_total_ = 0; + + void rebuild_encoding(engine, total_len) { + if (cached_total_ == total_len) return; // 已缓存 + // ... 构造编码 ... + cached_total_ = total_len; + } +}; +``` + +--- + +## 11. L1 Cache 友好的数据布局 + +### 行主序存储 + +```cpp +// data_[row * cols + col] — 行方向连续 +// 相邻元素在内存中连续 → SIMD 向量化友好 +``` + +### 转置内核 + +```cpp +// 分块转置：每个块内行优先读、列优先写 +// 块间并行 → 利用 L1 缓存的空间局部性 +``` + +--- + +## 12. 编译期零开销抽象 + +### 表达式模板（AST） + +```cpp +// algebra_expr.hpp — 编译期构造计算树 +auto expr = max(x, Scalar{0}); // ReLU AST +auto expr2 = a + b * c; // 融合乘加 AST + +// 统一执行：单次遍历，无临时矩阵 +compute::apply(span, expr); +``` + +**收益**： +- 消除临时矩阵分配（`a + b * c` 不创建 `b*c` 的中间矩阵） +- 编译器可内联整个表达式树 + +--- + +## 13. 性能基准测试工具 + +### bench_thresholds + +测量不同数据规模下的串行/并行性能，确定最优并行阈值： + +```bash +./build/bench_thresholds +# 输出：元素数 | 串行μs | 并行μs | 加速比 | 建议 +``` + +### compute_bench + +测量矩阵乘法、逐元素运算、归约等核心原语的性能。 + +--- + +## 📋 优化效果总结 + +| 优化手段 | 场景 | 预期收益 | +|----------|------|----------| +| SmartPolicy | 小数据串行、大数据并行 | 避免小数据 0.3x 退化 | +| 线程池 latch | 所有并行操作 | 消除 N 次堆分配/加锁 | +| 缓存分块 | 矩阵乘法 | L1 命中率 → 3-8x 加速 | +| -ffast-math | 所有浮点运算 | FMA 重排序 + SIMD | +| -march=native | 所有计算 | AVX2/AVX-512 SIMD | +| GPU 批量提交 | GPU 推理/训练 | 减少 kernel launch 开销 | +| Tensor 零拷贝 | Layer 间传递 | 消除数据拷贝 | +| 融合 axpy | 优化器更新 | 减少 600 次 buffer 分配/step | +| 注意力批量化 | MHA | H 次 matmul → 1 次 | +| 掩码/编码缓存 | GPT 训练 | 避免重复构造 | +| 表达式模板 | 逐元素运算 | 消除临时矩阵 | +| 分块转置 | 转置操作 | L1 友好 + 并行 | diff --git a/docs/03-quickstart-model.md b/docs/03-quickstart-model.md new file mode 100644 index 00000000..79a16c0c --- /dev/null +++ b/docs/03-quickstart-model.md @@ -0,0 +1,320 @@ +# 🛠️ 快速上手：构建自己的模型 + +> 本教程教你如何使用 `ComputeEngine` + `Layer` + `Model` 三件套从零构建神经网络模型。 + +--- + +## 📐 三件套概念 + +``` +ComputeEngine — 硬件抽象（CPU/GPU），提供计算原语 +Layer — 神经网络层，组合原语表达算法 +Model — 层容器，管理 Layer 的 forward/backward 链 +``` + +**关系：** + +```cpp +Model model(engine); // Model 绑定一个 Engine +model.add_linear(784, 128); // 添加 Layer（内部持有 engine 引用） +model.add_relu(); +model.add_linear(128, 10); + +auto result = model.forward(input); // engine 在 Model 内部自动传递 +``` + +--- + +## 🚀 第一步：选择引擎 + +```cpp +#include + +// CPU 引擎（默认，无需额外依赖） +nn::CpuEngine engine; + +// GPU 引擎（需要 Vulkan SDK，条件编译 NN_HAS_VULKAN） +// nn::GpuEngine gpu_engine; +``` + +--- + +## 🔨 第二步：构建 Model + +### 方式一：链式构建（推荐，适合 MLP） + +```cpp +nn::Model model(engine); + +model.add_linear(784, 256) // 输入层: 784 → 256 + .add_relu() // 激活函数 + .add_linear(256, 128) // 隐藏层: 256 → 128 + .add_relu() + .add_linear(128, 10); // 输出层: 128 → 10 +``` + +### 方式二：使用工厂函数（MNIST/GPT 预设） + +```cpp +// MNIST MLP：784 → 512 → 256 → 128 → 64 → 10 +auto model_result = nn::build_mnist_mlp_model(engine); +nn::Model model = std::move(*model_result); + +// MNIST Transformer (ViT)：28×28 图像，patch_size=7 +auto model_result = nn::build_mnist_transformer_model(engine); +nn::Model model = std::move(*model_result); + +// GPT 语言模型 +auto model_result = nn::build_gpt_model( + engine, + 10000, // vocab_size + 128, // d_model + 256, // seq_len + 4, // num_heads + 512, // d_ff + 4 // num_layers +); +nn::Model model = std::move(*model_result); +``` + +### 方式三：模板 add（自定义层） + +```cpp +nn::Model model(engine); +model.add(784, 256); // 等价于 add_linear +model.add(); // 等价于 add_relu +model.add(); // QuickGeLU 激活 +model.add(256); // 层归一化 +model.add(); // Softmax +model.add(128, 1024); // 位置编码 +model.add(128, 512); // FFN +model.add(128, 4, 512, 256); // Transformer 编码器层 +model.add(128, 4, 1024); // 因果自注意力 +model.add(128, 4, 512, 1024); // GPT 块 +``` + +--- + +## 📊 Tensor 的使用 + +`Tensor` 是所有数据的统一容器： + +```cpp +// 创建 CPU 张量 +nn::Tensor t = nn::Tensor::cpu(784, 32); // 784×32，零初始化 + +// 从 Matrix 创建（共享所有权） +nn::Matrix m(784, 32); +nn::Tensor t = nn::Tensor::from_matrix(std::move(m)); + +// 通过 Engine 创建（自动分配到目标设备） +nn::Tensor t = engine.create_tensor(784, 32); +engine.zero(t); // 清零 + +// 转换为 Matrix（仅 CPU 张量） +auto m_result = engine.to_matrix(t); +nn::Matrix m = std::move(*m_result); + +// 从 Matrix 上传到 Engine +auto t_result = engine.from_matrix(m); +nn::Tensor t = std::move(*t_result); +``` + +**张量布局（重要！）：** + +``` +Matrix/Tensor 行主序: (rows, cols) + data_[row * cols + col] + +神经网络约定（列主序 batch-major）: + 输入: (feature_dim, batch_size) — 每列一个样本 + 输出: (out_dim, batch_size) + 权重: (out_features, in_features) +``` + +--- + +## 🔄 完整前向传播示例 + +```cpp +#include + +int main() { + nn::CpuEngine engine; + + // 1. 构建模型 + nn::Model model(engine); + model.add_linear(784, 256) + .add_relu() + .add_linear(256, 10); + + // 2. 准备输入 (784 像素, 32 样本) + nn::Tensor input = nn::Tensor::cpu(784, 32); + // ... 填充数据 ... + + // 3. 前向传播 + auto out_result = model.forward(input); + if (!out_result) { + std::cerr << "Error: " << out_result.error().message << "\n"; + return 1; + } + nn::Tensor output = std::move(*out_result); + // output: (10, 32) — 每列一个样本的 10 类 logits + + // 4. 下载到 CPU 做 argmax + auto m = engine.to_matrix(output); + for (std::size_t b = 0; b < 32; ++b) { + std::size_t best = 0; + for (std::size_t c = 1; c < 10; ++c) { + if (m->at(c, b) > m->at(best, b)) + best = c; + } + std::cout << "Sample " << b << ": predicted " << best << "\n"; + } +} +``` + +--- + +## 🏗️ 自定义 Layer + +如果内置层不够用，可以继承 `Layer` 实现自己的层： + +```cpp +class MyLayer final : public nn::Layer { +private: + nn::Tensor weight_; + nn::Tensor grad_weight_; + nn::Tensor input_cache_; + +public: + MyLayer(nn::ComputeEngine& engine, std::size_t in_dim, std::size_t out_dim) + { + // 在 CPU 初始化权重，通过 engine 上传 + nn::Matrix w_cpu = nn::Matrix::random(out_dim, in_dim); // 假设有此方法 + auto w = engine.from_matrix(w_cpu); + weight_ = std::move(*w); + + grad_weight_ = engine.create_tensor(out_dim, in_dim); + engine.zero(grad_weight_); + } + + // 参数访问（供 Optimizer 使用） + std::vector parameters() override { + return {&weight_}; + } + std::vector param_gradients() override { + return {&grad_weight_}; + } + + // 前向传播 + [[nodiscard]] nn::Result forward( + nn::ComputeEngine& engine, const nn::Tensor& input) override + { + input_cache_ = input; + // 组合原语表达算法 + return engine.matmul(weight_, input); + } + + // 反向传播 + [[nodiscard]] nn::Result backward( + nn::ComputeEngine& engine, const nn::Tensor& grad_output) override + { + // grad_input = weight^T × grad_output + auto grad_input = engine.matmul(weight_, grad_output, true, false); + + // grad_weight += grad_output × input^T + auto gw = engine.matmul(grad_output, input_cache_, false, true); + auto r = engine.add_inplace(grad_weight_, *gw); + + return grad_input; + } +}; +``` + +**添加到 Model：** + +```cpp +nn::Model model(engine); +model.add(784, 256); +model.add(); +model.add(256, 10); +``` + +--- + +## ⚠️ 常见陷阱 + +### 1. Tensor 布局错误 + +```cpp +// ❌ 错误：(batch, feature) — 行主序 batch-major +nn::Tensor input = nn::Tensor::cpu(32, 784); + +// ✅ 正确：(feature, batch) — 列主序 batch-major +nn::Tensor input = nn::Tensor::cpu(784, 32); +``` + +### 2. 忘记绑定 Engine + +```cpp +// ❌ 错误：Model 未绑定 Engine +nn::Model model; +model.add_linear(784, 10); // 崩溃：engine_ == nullptr + +// ✅ 正确 +nn::CpuEngine engine; +nn::Model model(engine); +model.add_linear(784, 10); +``` + +### 3. 跨设备操作 + +```cpp +// ❌ 错误：GPU 张量调用 to_matrix +nn::GpuEngine engine; +nn::Tensor t = engine.create_tensor(10, 10); +auto m = engine.to_matrix(t); // 错误：tensor is not GPU + +// ✅ 正确：先下载到 CPU +auto m = engine.to_matrix(t); // 错误信息会告诉你 +``` + +### 4. 梯度清零遗漏 + +```cpp +// ❌ 错误：忘记清零梯度 +auto out = model.forward(input); +auto loss = loss_fn.forward(engine, out, target); +auto grad = loss_fn.backward(); +model.backward(engine, grad); +optimizer.step(); // 梯度累积！ + +// ✅ 正确：每步清零 +optimizer.zero_grad(); // 或 model.zero_grad(engine) +auto out = model.forward(input); +auto loss = loss_fn.forward(engine, out, target); +auto grad = loss_fn.backward(); +model.backward(engine, grad); +optimizer.step(); +``` + +--- + +## 📋 可用层速查表 + +| 层 | 构造参数 | 用途 | +|----|---------|------| +| `Linear(in, out)` | 输入/输出维度 | 全连接层 | +| `ReLU()` | 无 | ReLU 激活 | +| `GeLU()` | 无 | QuickGeLU 激活 | +| `LayerNorm(shape, eps)` | 归一化维度 + epsilon | 层归一化 | +| `Softmax()` | 无 | 行级 Softmax | +| `PositionalEncoding(d_model, max_len)` | 模型维度 + 最大长度 | 正弦位置编码 | +| `FeedForward(d_model, d_ff)` | 模型维度 + FFN 中间维度 | FFN = Linear₂(GeLU(Linear₁(x))) | +| `TransformerEncoderLayer(d_model, heads, d_ff, seq_len)` | 全部 | Pre-Norm 编码器层 | +| `TransformerEncoder(d_model, heads, d_ff, layers, patches)` | 全部 | ViT 编码器（含池化） | +| `PatchEmbedding(img, patch, d_model)` | 图像/patch/模型维度 | 图像 patch 嵌入 | +| `CausalSelfAttention(d_model, heads, max_len)` | 全部 | 因果自注意力 | +| `GPTBlock(d_model, heads, d_ff, max_len)` | 全部 | Pre-Norm 解码器块 | +| `GPTModel(vocab, d_model, seq, heads, d_ff, layers)` | 全部 | 完整 GPT 模型 | diff --git a/docs/04-quickstart-train-infer.md b/docs/04-quickstart-train-infer.md new file mode 100644 index 00000000..e82507a4 --- /dev/null +++ b/docs/04-quickstart-train-infer.md @@ -0,0 +1,669 @@ +# 🚀 快速上手：训练与推理 + +> 本教程教你如何使用 neuralnet.cpp 训练和推理 MNIST 手写数字识别与 GPT 文本生成模型。 + +--- + +## 📋 前置条件 + +**构建项目：** + +```bash +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --parallel +``` + +**数据准备：** + +```bash +# MNIST 数据集会自动下载到 datasets/mnist_data/ +# GPT 训练需要文本文件（如 datasets/llm_corpus.txt） +``` + +--- + +## 🖼️ MNIST 手写数字识别 + +### 训练 + +```bash +# MLP 架构（默认） +./build/mnist_train --epochs 10 --lr 0.001 --optimizer adam + +# Transformer (ViT) 架构 +./build/mnist_train --arch transformer --epochs 20 --lr 0.001 + +# 使用 Vulkan GPU 加速 +./build/mnist_train --gpu --epochs 10 + +# 使用 CUDA GPU 加速 +./build/mnist_train --cuda --epochs 10 + +# 恢复训练 +./build/mnist_train --resume mnist_model.bin --epochs 5 + +# 快速测试（限制样本数） +./build/mnist_train --max-samples 1000 --epochs 3 +``` + +**完整参数：** + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `--arch` | `mlp` | 架构：`mlp` / `transformer` | +| `--resume ` | 无 | 从已有模型恢复训练 | +| `--save ` | `mnist_model.bin` | 模型保存路径 | +| `--dataset ` | `datasets/mnist_data` | 数据集目录 | +| `--epochs ` | `10` | 训练轮数 | +| `--lr ` | `0.001` | 学习率 | +| `--batch-size ` | `64` | 批大小 | +| `--optimizer ` | `adam` | 优化器：`sgd`/`sgd_momentum`/`adam`/`adamw`/`muon` | +| `--gpu` | 禁用 | 启用 Vulkan GPU 加速 | +| `--cuda` | 禁用 | 启用 CUDA GPU 加速（优先于 --gpu） | +| `--max-samples ` | 全部 | 限制训练样本数 | + +### 推理 + +```bash +# 使用已训练的模型推理 +./build/mnist_infer --model mnist_model.bin --image datasets/mnist_data/test/0_1.csv + +# 交互模式（输入文件路径） +./build/mnist_infer --interactive +``` + +--- + +## 📝 GPT 文本生成 + +### 训练 + +```bash +# 基础训练 +./build/text_train datasets/llm_corpus.txt --epochs 10 --lr 0.001 + +# 自定义模型大小 +./build/text_train datasets/llm_corpus.txt \ + --d-model 256 --num-heads 8 --num-layers 6 --d-ff 1024 \ + --seq-len 512 --batch-size 16 --epochs 20 + +# 使用自定义词表 +./build/text_train datasets/llm_corpus.txt --vocab my_bpe.json + +# 使用 Muon 优化器 +./build/text_train datasets/llm_corpus.txt --optimizer muon --lr 0.002 + +# 恢复训练 +./build/text_train datasets/llm_corpus.txt --resume gpt_model.bin --epochs 5 +``` + +**完整参数：** + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `` | 必需 | 训练文本文件路径 | +| `--save ` | `gpt_model.bin` | 模型保存路径 | +| `--resume ` | 无 | 从已有模型恢复训练 | +| `--vocab ` | `gpt_bpe.json` | 词表 JSON 路径（自动识别类型） | +| `--epochs ` | `10` | 训练轮数 | +| `--lr ` | `0.001` | 学习率 | +| `--batch-size ` | `32` | 批大小 | +| `--seq-len ` | `256` | 序列长度 | +| `--optimizer ` | `adam` | 优化器 | +| `--weight-decay ` | `0.01` | AdamW 权重衰减 | +| `--d-model ` | `128` | 模型维度 | +| `--num-heads ` | `4` | 注意力头数 | +| `--num-layers ` | `4` | Transformer 层数 | +| `--d-ff ` | `512` | FFN 中间维度 | +| `--gpu` | 禁用 | 启用 Vulkan GPU 加速 | +| `--cuda` | 禁用 | 启用 CUDA GPU 加速（优先于 --gpu） | +| `--log-interval ` | `50` | 日志间隔 | + +### 推理 + +```bash +# 单次生成 +./build/text_infer --prompt "Once upon a time" --max-tokens 200 + +# 交互模式 +./build/text_infer --interactive + +# 调节温度（0=贪心，>1=更随机） +./build/text_infer --prompt "Hello" --temperature 0.8 + +# 使用 GPU +./build/text_infer --gpu --prompt "Hello" +``` + +**完整参数：** + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `--model ` | `gpt_model.bin` | 模型文件路径 | +| `--vocab ` | `bpe_vocab.json` | 词表路径（仅模型未嵌入时） | +| `--prompt ` | `Hello` | 输入提示文本 | +| `--interactive` | 禁用 | 交互式生成模式 | +| `--max-tokens ` | `200` | 最大生成 token 数 | +| `--temperature ` | `1.0` | 温度参数（0=贪心） | +| `--gpu` | 禁用 | 启用 Vulkan GPU 加速 | +| `--cuda` | 禁用 | 启用 CUDA GPU 加速 | +| `--show-tokens` | 禁用 | 显示 token ID（调试） | + +--- + +## 🔤 分词器 + +### 训练分词器 + +```bash +# 训练 BPE 分词器 +./build/tokenizer_train --input datasets/llm_corpus.txt --vocab-size 10000 --output my_bpe.json + +# 训练 ByteZip 分词器 +./build/tokenizer_train --type bytezip --input datasets/llm_corpus.txt --output my_bytezip.json +``` + +### 分词器推理 + +```bash +# 编码文本 +./build/tokenizer_infer --vocab gpt_bpe.json --encode "Hello world" + +# 解码 token IDs +./build/tokenizer_infer --vocab gpt_bpe.json --decode "72,101,108,108,111" +``` + +### 支持的分词器类型 + +| 类型 | JSON `"type"` 字段 | 说明 | +|------|---------------------|------| +| BPE | `"bpe"` | Byte-Pair Encoding | +| CharBPE | `"charbpe"` | 字符级 BPE | +| WordZip | `"wordzip"` | 词频统计分词 | +| Space | `"space"` | 空格分词 | + +V3 格式模型文件会自动嵌入分词器，推理时无需单独指定 `--vocab`。 + +--- + +## 💻 C++ API 编程训练 + +### MNIST MLP 训练 + +```cpp +#include +#include +#include + +int main() { + nn::CpuEngine engine; + + // 1. 构建模型 + auto model_result = nn::build_mnist_mlp_model(engine); + nn::Model model = std::move(*model_result); + + // 2. 优化器 + auto optimizer = nn::create_optimizer( + "adam", engine, + model.parameters(), model.param_gradients(), + 0.001, // lr + 0.01 // weight_decay + ); + + nn::CrossEntropyLoss loss_fn; + + // 3. 加载数据集（CSV 格式） + // ... 假设已有 train_images (N×784) 和 train_labels (N×1) ... + + const std::size_t batch_size = 64; + const std::size_t epochs = 10; + + for (int epoch = 0; epoch < epochs; ++epoch) { + Scalar epoch_loss = 0; + + for (std::size_t batch = 0; batch < num_batches; ++batch) { + // 清零梯度 + optimizer->zero_grad(); + + // 准备 batch 数据 (784, batch_size) + nn::Matrix x_batch = /* ... */; + nn::Matrix y_onehot = /* ... (10, batch_size) ... */; + + // 上传到引擎 + auto x_t = engine.from_matrix(x_batch); + auto y_t = engine.from_matrix(y_onehot); + nn::Tensor input = std::move(*x_t); + nn::Tensor target = std::move(*y_t); + + // 前向传播 + auto out = model.forward(input); + auto loss = loss_fn.forward(engine, *out, target); + + // 反向传播 + auto grad = loss_fn.backward(); + model.backward(engine, *grad); + + // 参数更新 + optimizer->step(); + + epoch_loss += *loss; + } + + std::cout << "Epoch " << epoch + 1 + << " loss=" << epoch_loss / num_batches << "\n"; + } + + // 4. 保存模型 + auto spec = nn::make_mlp_spec({784, 512, 256, 128, 64, 10}); + auto save_result = nn::save_model("mnist_model.bin", model, spec); +} +``` + +### GPT 文本生成训练 + +```cpp +#include +#include +#include + +int main() { + nn::CpuEngine engine; + + // 1. 加载分词器 + auto tokenizer = nn::load_tokenizer_from_file("gpt_bpe.json"); + + // 2. 构建 GPT 模型 + auto model_result = nn::build_gpt_model( + engine, + tokenizer->vocab_size(), // vocab_size + 128, // d_model + 256, // seq_len + 4, // num_heads + 512, // d_ff + 4 // num_layers + ); + nn::Model model = std::move(*model_result); + + // 3. 优化器 + auto optimizer = nn::create_optimizer( + "adamw", engine, + model.parameters(), model.param_gradients(), + 0.001, 0.01 + ); + + nn::CrossEntropyLoss loss_fn; + + // 4. 编码文本 → token IDs + std::string text = /* ... 读取文本文件 ... */; + auto tokens = tokenizer->encode(text); + + // 5. 训练循环（next-token prediction） + const std::size_t seq_len = 256; + const std::size_t batch_size = 32; + + for (int epoch = 0; epoch < 10; ++epoch) { + for (std::size_t step = 0; step < steps_per_epoch; ++step) { + optimizer->zero_grad(); + + // 采样 batch: x = tokens[t:t+seq_len], y = tokens[t+1:t+seq_len+1] + nn::Matrix x_tokens(seq_len, batch_size); // (seq, batch) + nn::Matrix y_onehot(vocab_size, seq_len * batch_size); + // ... 填充数据 ... + + auto x_t = engine.from_matrix(x_tokens); + auto y_t = engine.from_matrix(y_onehot); + nn::Tensor input = std::move(*x_t); + nn::Tensor target = std::move(*y_t); + + // 前向传播 + auto logits = model.forward(input); + auto loss = loss_fn.forward(engine, *logits, target); + + // 反向传播 + 更新 + auto grad = loss_fn.backward(); + model.backward(engine, *grad); + optimizer->step(); + + std::cout << "Step " << step << " loss=" << *loss << "\n"; + } + } + + // 6. 保存模型（含嵌入词表） + auto spec = nn::make_gpt_spec( + tokenizer->vocab_size(), 128, 256, 4, 512, 4); + nn::save_model("gpt_model.bin", model, spec, + tokenizer.get()); // 嵌入分词器 +} +``` + +### GPT 推理（自回归生成） + +```cpp +#include +#include +#include + +int main() { + nn::CpuEngine engine; + + // 1. 加载模型（V3 格式自动读取规格 + 嵌入词表） + auto load_result = nn::load_model_with_spec("gpt_model.bin", engine); + auto& [model, spec, tokenizer] = *load_result; + + // 2. 编码提示文本 + auto prompt_tokens = tokenizer->encode("Once upon a time"); + + // 3. 自回归生成 + const int max_tokens = 200; + const double temperature = 1.0; + + std::vector generated(prompt_tokens.begin(), prompt_tokens.end()); + + for (int i = 0; i < max_tokens; ++i) { + // 构造输入: (seq_len, 1) — 取最后 seq_len 个 token + nn::Matrix input(spec.seq_len, 1); + // ... 填充 token（不足部分用 pad_id=0）... + + auto x_t = engine.from_matrix(input); + auto logits = model.forward(*x_t); + + // 取最后一个位置的 logits + auto logits_m = engine.to_matrix(*logits); + // logits_m: (vocab_size, seq_len*1) + // 最后一个位置: col = seq_len - 1 + + // 温度采样 + std::vector probs(vocab_size); + Scalar max_logit = *std::max_element(/* last position logits */); + Scalar sum = 0; + for (std::size_t v = 0; v < vocab_size; ++v) { + probs[v] = std::exp((logits_m->at(v, seq_len-1) - max_logit) / temperature); + sum += probs[v]; + } + for (auto& p : probs) p /= sum; + + // 采样下一个 token + std::size_t next_token = /* argmax 或随机采样 */; + generated.push_back(next_token); + + if (next_token == tokenizer->eos_id()) break; + } + + // 4. 解码输出 + std::string output = tokenizer->decode(generated); + std::cout << output << "\n"; +} +``` + +--- + +## 📊 训练监控 + +### 控制台输出 + +训练过程中会打印： + +``` +[Step 50] loss=2.345 lr=0.001000 +[Step 100] loss=1.987 lr=0.001000 +[Step 150] loss=1.654 lr=0.001000 +... +``` + +### 模型保存格式 + +| 版本 | 格式 | 说明 | +|------|------|------| +| V1 | `[magic][version][matrices...]` | 仅参数 | +| V2 | `[magic][version][model_type][spec][matrices...]` | 含架构规格 | +| V3 | `[magic][version][precision][model_type][spec][matrices...][tokenizer]` | 含精度标记 + 嵌入词表 | + +- `load_model` 同时支持 V1/V2/V3 读取 +- `save_model` 统一写入 V3 格式 +- V3 模型推理时无需单独指定 `--vocab` 参数 + +--- + +## 🖥️ 图形化界面 (GUI) 操作指南 + +项目提供基于 tkinter 的全功能 GUI（`gui.py`），覆盖全部训练/推理/分词器操作，无需记忆命令行参数。 + +```bash +# 启动 GUI +python gui.py +``` + +> **前提**: 需要先构建 C++ 项目（`cmake -B build -G Ninja`），GUI 会调用 `build/` 目录下的可执行文件。 +> **依赖**: Python 3.8+，tkinter（Python 内置），Pillow（`pip install Pillow`，用于手写板功能）。 + +### GUI 功能一览 + +| Tab | 功能 | +|-----|------| +| 🏋️ **训练** | MNIST 模型训练：支持 MLP / Transformer 架构切换、超参数调节、恢复训练、学习率调度（fixed / cosine）、GPU 加速、评估样本数；实时训练曲线 | +| 🔍 **推理** | MNIST 图片推理：Top-K 预测 + 置信度条形图 + 图片预览；内置 ✍️ **手写板**，鼠标书写数字即刻识别 | +| 🖼️ **图片查看** | 浏览 CSV 格式手写数字图片，前后翻页导航 | +| 📝 **GPT 训练** | GPT 语言模型训练：位置编码（learned / sinusoidal / alibi）、激活（GeLU / SwiGLU）、归一化（LayerNorm / RMSNorm）、梯度累积、滑动窗口 stride、学习率调度（fixed / cosine / step_cosine）、TDR 防护、梯度裁剪；实时 loss 曲线 | +| 💬 **GPT 推理** | GPT 文本生成：温度调节、交互模式、Token ID 调试输出 | +| 🔤 **分词器训练** | 训练 BPE / CharBPE 分词器，配置词表大小和最小合并频率 | +| 🔡 **分词器推理** | 分词器编码/解码测试，支持文本编码、ID 解码、文件编码 | + +--- + +### 🏋️ MNIST 训练 Tab + +``` +┌──────────────────────────────────────────────────┐ +│ 🏋️ 训练 Tab │ +│ │ +│ 数据集目录: [datasets/mnist_data ] [浏览…] │ +│ 模型保存路径: [mnist_model.bin ] [浏览…] │ +│ ☑ 从已有模型恢复训练 │ +│ │ +│ ┌─ 超参数 ────────────────────────────────────┐ │ +│ │ 轮数: [10] 学习率: [0.001] 批大小: [64] │ │ +│ │ 优化器: [adam ▾] 权重衰减: [0.01] │ │ +│ │ 模型类型: [mlp ▾] │ │ +│ └─────────────────────────────────────────────┘ │ +│ │ +│ ┌─ 模型架构 ──────────────────────────────────┐ │ +│ │ MLP: 层维度 [784,512,256,128,10] │ │ +│ │ 或 Transformer: 模型维度/头数/层数/FFN/Patch │ │ +│ └─────────────────────────────────────────────┘ │ +│ │ +│ ┌─ 学习率调度 ────────────────────────────────┐ │ +│ │ 调度模式: [fixed ▾] / [cosine ▾] │ │ +│ │ cosine: 预热轮数 + 最低学习率 │ │ +│ │ 手动: 逗号分隔每轮 lr │ │ +│ └─────────────────────────────────────────────┘ │ +│ │ +│ ┌─ 训练选项 ──────────────────────────────────┐ │ +│ │ GPU: [None ▾] 最大样本数: [0] 评估样本数: [200] │ │ +│ │ ☑ 每 epoch 打乱 batch 顺序 │ │ +│ └─────────────────────────────────────────────┘ │ +│ │ +│ [▶ 开始训练] [⏹ 停止] │ +│ │ +│ ┌─ 训练日志 ──────────────────────────────────┐ │ +│ │ Epoch 1/10 loss=0.4523 train_acc=90.12% │ │ +│ │ Epoch 2/10 loss=0.2134 train_acc=94.56% │ │ +│ └─────────────────────────────────────────────┘ │ +│ │ +│ ┌─ 📊 训练曲线 ───────────────────────────────┐ │ +│ │ ╭──╮ │ │ +│ │ ╭╯ ╰──╮ step_loss / epoch_loss │ │ +│ │╭╯ ╰───────── train_acc / test_acc │ │ +│ └─────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────┘ +``` + +**操作步骤：** + +1. **选择数据集** — 点击"浏览…"选择 `datasets/mnist_data` 目录（默认已填好） +2. **设置保存路径** — 模型训练完成后保存的 `.bin` 文件路径 +3. **配置超参数** — 轮数、学习率、批大小、优化器（`sgd`/`adam`/`adamw`/`muon`）、权重衰减 +4. **选择模型类型**： + - `mlp`：可自定义层维度（如 `784,512,256,128,10`） + - `transformer`：可配置模型维度、注意力头数、FFN 维度、层数、Patch 大小 +5. **可选功能**： + - ✅ 恢复训练 — 从已有模型继续训练 + - ✅ GPU 加速 — 支持 Vulkan / CUDA（下拉选择） + - ✅ 评估样本数 — 仅 Transformer 架构生效，限制评估样本以加快训练（0=自动） + - ✅ 学习率调度 — 支持 cosine 衰减（含预热）和手动指定每轮 lr +6. **点击 "▶ 开始训练"** — 日志面板实时输出训练进度，下方曲线图实时绘制 step_loss、epoch_loss、train_acc、test_acc、lr + +--- + +### 🔍 MNIST 推理 Tab + +**基本推理：** + +1. **选择模型文件** — 点击"浏览…"选择训练好的 `.bin` 模型 +2. **选择图片** — 输入单张 CSV 图片路径或包含多张图片的目录 +3. **设置 Top-K** — 控制显示前 K 个预测结果 +4. **点击 "▶ 开始推理"** — 左侧显示图片预览，右侧显示 Top-K 置信度条形图 + +**✍️ 手写板识别：** + +推理 Tab 内置了手写板功能，无需准备 CSV 文件即可实时识别： + +1. 在左侧白色画布上用鼠标**书写一个数字**（0-9） +2. 右侧 **28×28 预览** 窗口实时显示下采样后的效果 +3. 点击 **"🔍 识别手写数字"** 按钮 +4. 下方显示预测结果（数字 + 置信度百分比） +5. 点击 **"🗑️ 清除"** 重写 + +``` +┌──────────────────────────────────────────────────┐ +│ 🔍 推理 Tab │ +│ │ +│ 模型文件: [pretrained/model.bin ] [浏览…] │ +│ 图片/目录: [ ] [浏览…] │ +│ Top-K: [3] 架构: 自动识别 ☐ GPU ☐ 显示像素 │ +│ │ +│ [▶ 开始推理] │ +│ │ +│ ┌─ ✍️ 手写识别 ──────────────────────────────┐ │ +│ │ ┌──────────┐ 28×28 预览: │ │ +│ │ │ │ ┌──────┐ │ │ +│ │ │ ✍️ │ │▓▓▓▓▓▓│ │ │ +│ │ │ 画布 │ │▓ ▓▓▓│ 预测: 3 (98.2%) │ │ +│ │ │ │ │ ▓▓▓▓│ 条形图: │ │ +│ │ └──────────┘ └──────┘ ████░ 3 98.2% │ │ +│ │ [🗑️ 清除] [🔍 识别手写数字] █░░░░ 7 1.2% │ │ +│ └─────────────────────────────────────────────┘ │ +│ │ +│ ┌─ 图片预览 ────────┐ ┌─ 预测结果 ───────────┐ │ +│ │ │ │ filename.csv │ │ +│ │ 28×28 灰度图 │ │ ████░░ 3 95.3% │ │ +│ │ │ │ █░░░░░ 5 3.1% │ │ +│ └───────────────────┘ └──────────────────────┘ │ +└──────────────────────────────────────────────────┘ +``` + +--- + +### 🖼️ 图片查看 Tab + +浏览 CSV 格式的手写数字图片： + +1. 选择单张 CSV 图片文件或包含多张图片的目录 +2. 使用 **上一张 / 下一张** 按钮翻页导航 +3. 左侧显示 28×28 灰度图片预览 + +--- + +### 📝 GPT 训练 Tab + +``` +┌──────────────────────────────────────────────────┐ +│ 📝 GPT 训练 Tab │ +│ │ +│ 训练文本文件: [datasets/llm_corpus.txt ] [浏览…] │ +│ 模型保存路径: [gpt_model.bin ] [浏览…] │ +│ 词表 JSON: [bpe_vocab.json ] [浏览…] │ +│ ☑ 从已有模型恢复训练 │ +│ │ +│ ┌─ 超参数 ────────────────────────────────────┐ │ +│ │ 轮数: [10] 学习率: [0.001] 批大小: [32] │ │ +│ │ 序列长度: [256] 优化器: [adam ▾] │ │ +│ │ 权重衰减: [0.01] 日志间隔: [50] │ │ +│ └─────────────────────────────────────────────┘ │ +│ │ +│ ┌─ 模型架构 ──────────────────────────────────┐ │ +│ │ 模型维度: [128] 注意力头: [4] 层数: [4] │ │ +│ │ FFN 维度: [512] │ │ +│ └─────────────────────────────────────────────┘ │ +│ │ +│ 位置编码 / 激活 / 归一化: [learned ▾] [gelu ▾] [layernorm ▾] │ +│ 梯度累积: [1] stride: [0] GPU: [None ▾] │ +│ │ +│ ▶ 高级选项（学习率调度 / TDR 防护 / 梯度日志 / 梯度裁剪） │ +│ - 调度: fixed / cosine / step_cosine + 预热 + min-lr │ +│ - TDR 重试 + flush 间隔；梯度裁剪 max-norm │ +│ │ +│ ☑ 显示梯度统计 │ +│ │ +│ [▶ 开始训练] [⏹ 停止] │ +│ │ +│ ┌─ 📊 训练曲线 ───────────────────────────────┐ │ +│ │ ╭──╮ step_loss / epoch_loss / lr │ │ +│ │ ╭╯ ╰──╮ │ │ +│ │╭╯ ╰───────── │ │ +│ └─────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────┘ +``` + +**操作步骤：** + +1. **选择训练文本** — 点击"浏览…"选择 `.txt` 文本文件（如 `datasets/llm_corpus.txt`） +2. **设置词表** — 指定 BPE 词表 JSON 路径（如 `bpe_vocab.json`） +3. **配置模型架构** — 模型维度、注意力头数、Transformer 层数、FFN 维度 +4. **选择位置编码 / 激活 / 归一化** — `learned`/`sinusoidal`/`alibi`，`gelu`/`swiglu`，`layernorm`/`rmsnorm` +5. **配置超参数** — 轮数、学习率、批大小、序列长度、优化器；可开启梯度累积与滑动窗口 stride +6. **高级选项** — 学习率调度（cosine / step_cosine）、TDR 防护、梯度统计、梯度裁剪 +7. **点击 "▶ 开始训练"** — 实时显示 loss 曲线 + +--- + +### 💬 GPT 推理 Tab + +**操作步骤：** + +1. **选择模型** — 指定训练好的 GPT 模型 `.bin` 文件 +2. **选择词表** — 指定对应的词表 JSON +3. **设置生成参数** — 最大 token 数（默认 200）、温度（0=贪心，>1=更随机，默认 0.8） +4. **输入提示文本** — 在输入框中输入起始文本（如 "Hello"） +5. **点击 "▶ 生成文本"** — 在输出区域显示生成结果 + +**可选项：** +- ✅ **交互式生成模式** — 启用后通过 stdin 进行多轮对话式生成 +- ✅ **显示 Token ID** — 调试用，显示每个 token 的 ID +- ✅ **GPU 加速** — 支持 Vulkan / CUDA（下拉选择） + +--- + +### 🔤 分词器训练 Tab + +**操作步骤：** + +1. **选择训练文本** — 用于训练分词器的 `.txt` 文件 +2. **设置输出路径** — 分词器模型保存的 JSON 文件 +3. **选择类型** — `bpe`（字节级 BPE）或 `charbpe`（字符级 BPE，中文推荐） +4. **配置参数** — 词表大小（默认 5000）、最小合并频率（默认 2） +5. **点击 "▶ 开始训练"** + +--- + +### 🔡 分词器推理 Tab + +**操作步骤：** + +1. **选择词表** — 加载分词器 JSON 文件 +2. **选择操作模式**： + - **编码+解码验证** — 输入文本，查看编码再解码的完整流程 + - **仅编码** — 将文本编码为 token ID 序列 + - **仅解码** — 将 token ID 序列解码为文本 +3. **输入内容** — 文本或逗号分隔的 ID 列表（按 Enter 快速执行） +4. **可选** — 勾选"显示 Token 详情"查看字节级信息 +5. **点击 "▶ 执行"** — 输出区域显示结果 diff --git a/docs/05-algorithm-reference.md b/docs/05-algorithm-reference.md new file mode 100644 index 00000000..4806292a --- /dev/null +++ b/docs/05-algorithm-reference.md @@ -0,0 +1,510 @@ +# 📚 算法解析：Layer / Loss / Optimizer + +> 本文档详细解析 neuralnet.cpp 中每个计算组件的数学原理、实现细节和引擎原语分解。 + +--- + +## 📐 约定 + +- **矩阵布局**：行主序 `(rows, cols)`，`data_[row * cols + col]` +- **批处理布局**：列主序 batch-major `(feature_dim, batch_size)` +- **标量类型**：`Scalar = float` +- **引擎原语**：所有算法通过 `ComputeEngine` 的 op-level 原语组合表达 + +--- + +## 🧱 Layer 篇 + +### 1. Linear — 全连接层 + +**参数：** `in_features`, `out_features` + +**可学习参数：** +- `w_`: `(out_features, in_features)` — Xavier 均匀初始化 +- `b_`: `(out_features, 1)` — 零初始化 + +**Forward：** + +$$\text{out} = W \times x + b$$ + +引擎原语分解： +``` +out = engine.matmul(W, x) // 矩阵乘法 +engine.broadcast_row_inplace(out, b, Add) // 按行广播加法 +``` + +**Backward：** + +$$\text{grad}_x = W^T \times \text{grad}_{\text{out}}$$ +$$\text{grad}_W += \text{grad}_{\text{out}} \times x^T$$ +$$\text{grad}_b += \sum_{\text{batch}} \text{grad}_{\text{out}}$$ + +引擎原语分解： +``` +grad_input = engine.matmul(W, grad_out, transA=true) +grad_W += engine.matmul(grad_out, input, transB=true) +grad_b += engine.row_reduce_sum(grad_out) +``` + +--- + +### 2. ReLU — ReLU 激活函数 + +**参数：** 无 + +**Forward：** + +$$\text{out} = \max(x, 0)$$ + +``` +out = engine.elementwise_binary_scalar(Max, x, 0) +``` + +**Backward：** + +$$\text{grad}_x = \begin{cases} \text{grad}_{\text{out}} & \text{if } x > 0 \\ 0 & \text{otherwise} \end{cases}$$ + +``` +grad_input = engine.elementwise_select_scalar_cond(Gt, x, 0, grad_out, 0) +``` + +--- + +### 3. GeLU — QuickGeLU 激活函数 + +**参数：** `β = 1.702`（固定常量） + +**Forward：** + +$$\text{out} = x \cdot \sigma(\beta x) = x \cdot \frac{1}{1 + e^{-\beta x}}$$ + +引擎原语分解（5 次调用）： +``` +t1 = engine.elementwise_binary_scalar(Mul, x, β) // βx +t2 = engine.elementwise_unary(Neg, t1) // -βx +t3 = engine.elementwise_unary(Exp, t2) // exp(-βx) +t4 = engine.elementwise_binary_scalar(Add, t3, 1) // 1 + exp(-βx) +s = engine.elementwise_binary_scalar(Div, t4, 1) // sigmoid(βx) +out = engine.elementwise_binary(Mul, x, s) +``` + +**Backward：** + +$$\text{factor} = s \cdot (1 + \beta x \cdot (1 - s)), \quad s = \sigma(\beta x)$$ +$$\text{grad}_x = \text{grad}_{\text{out}} \cdot \text{factor}$$ + +引擎原语分解（5 次调用）： +``` +one_minus_s = engine.elementwise_binary_scalar(Sub, s, 1) +bx = engine.elementwise_binary_scalar(Mul, x, β) +inner = engine.elementwise_binary(Mul, bx, one_minus_s) +paren = engine.elementwise_binary_scalar(Add, inner, 1) +factor = engine.elementwise_binary(Mul, s, paren) +grad_input = engine.elementwise_binary(Mul, grad_out, factor) +``` + +--- + +### 4. LayerNorm — 层归一化 + +**参数：** `normalized_shape`, `epsilon = 1e-5` + +**可学习参数：** +- `gamma_`: `(normalized_shape, 1)` — 初始化为 1 +- `beta_`: `(normalized_shape, 1)` — 初始化为 0 + +**Forward：** + +$$\mu = \frac{1}{F} \sum_f x_{f,b}$$ +$$\text{diff} = x - \mu$$ +$$\sigma^2 = \frac{1}{F} \sum_f \text{diff}_f^2$$ +$$\text{normalized} = \frac{\text{diff}}{\sqrt{\sigma^2 + \epsilon}}$$ +$$\text{out} = \gamma \cdot \text{normalized} + \beta$$ + +**Backward：** + +$$\text{gy} = \text{grad}_{\text{out}} \cdot \gamma$$ +$$\text{mean}_g = \frac{1}{F} \sum_f \text{gy}_f$$ +$$\text{gy}_{\text{norm}} = \text{gy} \cdot \text{normalized}$$ +$$\text{mean}_{gn} = \frac{1}{F} \sum_f \text{gy}_{\text{norm},f}$$ +$$\text{grad}_x = (\text{gy} - \text{mean}_g - \text{normalized} \cdot \text{mean}_{gn}) \cdot \frac{1}{\sqrt{\sigma^2 + \epsilon}}$$ +$$\text{grad}_\gamma += \sum_b \text{gy}_{\text{norm},b}$$ +$$\text{grad}_\beta += \sum_b \text{grad}_{\text{out},b}$$ + +--- + +### 5. Softmax — 行级 Softmax + +**参数：** 无 + +**Forward（数值稳定）：** + +$$\text{row\_max}_r = \max_c x_{r,c}$$ +$$\text{shifted}_{r,c} = x_{r,c} - \text{row\_max}_r$$ +$$\text{exp\_shift}_{r,c} = e^{\text{shifted}_{r,c}}$$ +$$\text{row\_sum}_r = \sum_c \text{exp\_shift}_{r,c}$$ +$$\text{out}_{r,c} = \frac{\text{exp\_shift}_{r,c}}{\text{row\_sum}_r}$$ + +引擎原语分解： +``` +row_max = engine.row_reduce_max(input) +shifted = engine.clone(input) +engine.broadcast_row_inplace(shifted, row_max, Sub) +exp_shift = engine.elementwise_unary(Exp, shifted) +row_sum = engine.row_reduce_sum(exp_shift) +output = engine.clone(exp_shift) +engine.broadcast_row_inplace(output, row_sum, Div) +``` + +**Backward：** + +$$\text{ep} = \text{out} \odot \text{grad}_{\text{out}}$$ +$$\text{dot}_r = \sum_c \text{ep}_{r,c}$$ +$$\text{gmd} = \text{grad}_{\text{out}} - \text{dot}$$ +$$\text{grad}_x = \text{out} \odot \text{gmd}$$ + +--- + +### 6. MultiHeadAttention — 多头注意力 + +**参数：** `d_model`, `num_heads`, `seq_len` + +**可学习参数：** 4 个 Linear 层（`w_q`, `w_k`, `w_v`, `w_o`），每个 `(d_model, d_model)` + +**Forward（批量化）：** + +1. 线性投影：`Q = W_q·x`, `K = W_k·x`, `V = W_v·x` → `(H·d_k, batch·seq)` +2. 重排：`Q_re = rearrange_3d(Q, H·d_k, batch, seq)` → `(batch·H·d_k, seq)` +3. 注意力分数：`S = batched_matmul(Q_re, K_re, batch·H, transA=true)` → `(batch·H·seq, seq)` +4. 缩放：`S *= scale`，其中 `scale = 1/√d_k` +5. Softmax：`A = softmax(S)` +6. 注意力输出：`O_re = batched_matmul(V_re, A, batch·H)` → `(batch·H·d_k, seq)` +7. 反重排：`O = rearrange_3d(O_re, H·d_k, batch, seq, inverse=true)` → `(H·d_k, batch·seq)` +8. 输出投影：`out = W_o·O` + +**性能关键：** `rearrange_3d` + `batched_matmul` 将 H 个 per-head matmul 融合为 1 次 batch dispatch。 + +--- + +### 7. PositionalEncoding — 正弦波位置编码 + +**参数：** `d_model`, `max_len` + +**算法（固定，不可学习）：** + +$$\text{PE}(\text{pos}, 2i) = \sin\left(\frac{\text{pos}}{10000^{2i/d_{\text{model}}}}\right)$$ +$$\text{PE}(\text{pos}, 2i+1) = \cos\left(\frac{\text{pos}}{10000^{2i/d_{\text{model}}}}\right)$$ + +**Forward：** `out = input + PE` + +**Backward：** 梯度直接穿透（编码不可学习） + +**Tiling 模式：** 当 `tile_size > 0` 时，生成 `(d_model, tile_size)` 基础编码，沿列方向 tile 为 `(d_model, batch·tile_size)`，每个样本独立使用同一份编码。 + +--- + +### 8. FeedForward — 前馈网络 + +**参数：** `d_model`, `d_ff` + +**结构：** `FFN(x) = Linear₂(GeLU(Linear₁(x)))` + +- `fc1_`: `(d_model, d_ff)` — 扩展 +- `fc2_`: `(d_ff, d_model)` — 压缩 + +Forward/Backward 直接委托给子层的 forward/backward。 + +--- + +### 9. TransformerEncoderLayer — Pre-Norm 编码器层 + +**参数：** `d_model`, `num_heads`, `d_ff`, `seq_len` + +**算法（Pre-Norm）：** + +$$x_1 = \text{LN}_1(x)$$ +$$a = \text{SelfAttn}(x_1)$$ +$$r_2 = x + a$$ +$$x_2 = \text{LN}_2(r_2)$$ +$$f = \text{FFN}(x_2)$$ +$$\text{out} = r_2 + f$$ + +包含子层：`MultiHeadAttention` + `LayerNorm`×2 + `FeedForward` + +--- + +### 10. TransformerEncoder — ViT 编码器 + +**参数：** `d_model`, `num_heads`, `d_ff`, `num_layers`, `num_patches` + +**算法：** + +1. 添加 tiled 位置编码 +2. 通过 N 个 `TransformerEncoderLayer` +3. 全局平均池化（按样本聚合 `num_patches` 维度） + +**池化：** +$$\text{out}[:, b] = \frac{1}{P} \sum_{p=0}^{P-1} x[:, b \cdot P + p]$$ + +**反池化（backward）：** +$$\text{grad}_x[:, b \cdot P + p] = \frac{1}{P} \cdot \text{grad}_{\text{out}}[:, b]$$ + +--- + +### 11. PatchEmbedding — 图像 patch 嵌入 + +**参数：** `img_size`, `patch_size`, `d_model` + +**算法：** + +1. 将 `(img_size², batch)` 图像提取 `num_patches = (img_size/patch_size)²` 个不重叠 patch +2. 每个展平 patch `(patch_size²)` 经 Linear 投影到 `d_model` 维 +3. 重排为 `(d_model, batch·num_patches)` — batch-major 列布局 + +--- + +### 12. CausalSelfAttention — 因果自注意力 + +**参数：** `d_model`, `num_heads`, `max_len`, `seq_len` + +与 `MultiHeadAttention` 相同的批量化策略，额外施加因果掩码： + +$$\text{mask}[i][j] = \begin{cases} 0 & \text{if } j \leq i \\ -\infty & \text{if } j > i \end{cases}$$ + +掩码在 softmax 前施加：`S += mask` + +**掩码缓存：** 相同 `(batch, seq_len)` 组合只构造一次。 + +--- + +### 13. GPTBlock — Pre-Norm 解码器块 + +**参数：** `d_model`, `num_heads`, `d_ff`, `max_len`, `seq_len` + +**算法（Pre-Norm）：** + +$$x = x + \text{CausalSelfAttn}(\text{LN}_1(x))$$ +$$x = x + \text{FFN}(\text{LN}_2(x))$$ + +包含子层：`CausalSelfAttention` + `LayerNorm`×2 + `FeedForward` + +--- + +### 14. GPTModel — Decoder-only Transformer + +**参数：** `vocab_size`, `d_model`, `seq_len`, `num_heads`, `d_ff`, `num_layers` + +**可学习参数：** +- `token_emb_`: `(vocab_size, d_model)` — Token 嵌入 +- `pos_emb_`: `(seq_len, d_model)` — 位置嵌入 +- N 个 `GPTBlock` +- `ln_f_`: `LayerNorm` — 最终归一化 +- `lm_head_`: `Linear(d_model, vocab_size)` — 语言模型头 + +**Forward：** + +1. `all_emb = gather_rows(token_emb_, input)` — 按 token ID 查表 +2. `pos_gathered = gather_rows(pos_emb_, pos_indices)` — 位置嵌入 +3. `x = transpose(all_emb) + transpose(pos_gathered)` — `(d_model, seq·batch)` +4. 通过 N 个 GPTBlock +5. 最终 LayerNorm +6. LM Head → `(vocab_size, seq·batch)` + +**Backward 关键：** +- Token 嵌入梯度用 `scatter_add_rows` 按 token ID 累加 +- 位置嵌入梯度用 CPU 辅助聚合（数据量小） + +--- + +## 📉 Loss 篇 + +### 1. MSELoss — 均方误差 + +**算法：** + +$$\text{diff} = \text{pred} - \text{target}$$ +$$\text{loss} = \frac{1}{N} \sum \text{diff}^2$$ +$$\text{grad} = \frac{2}{N} \cdot \text{diff}$$ + +--- + +### 2. CrossEntropyLoss — 交叉熵损失（含 Softmax） + +**算法（数值稳定）：** + +$$\text{col\_max} = \text{col\_reduce\_max}(\text{logits})$$ +$$\text{shifted} = \text{logits} - \text{col\_max}$$ +$$\text{exp\_shift} = e^{\text{shifted}}$$ +$$\text{col\_sum} = \text{col\_reduce\_sum}(\text{exp\_shift})$$ +$$\text{softmax} = \text{exp\_shift} / \text{col\_sum}$$ +$$\text{grad} = \text{softmax} - \text{target}$$ +$$\text{log\_sm} = \text{shifted} - \log(\text{col\_sum})$$ +$$\text{loss} = -\frac{1}{\text{batch}} \sum \text{target} \cdot \text{log\_sm}$$ + +**Backward：** `grad = softmax - target_onehot` + +--- + +## ⚙️ Optimizer 篇 + +### 1. SGD — 随机梯度下降 + +**算法：** + +$$p \leftarrow p - \eta \cdot g$$ + +**引擎原语：** `scale_add_(p, -lr, g)` — 单次 axpy + +--- + +### 2. SGDWithMomentum — 带动量的 SGD + +**算法：** + +$$v \leftarrow \beta \cdot v + (1 - \beta) \cdot g$$ +$$p \leftarrow p - \eta \cdot v$$ + +**引擎原语：** +``` +engine.scale_inplace(v, β) // v *= β +scale_add_(v, 1-β, g) // v += (1-β)*g +scale_add_(p, -lr, v) // p -= lr*v +``` + +**参数：** `lr`（学习率）, `beta = 0.9`（动量系数） + +--- + +### 3. Adam — 自适应矩估计 + +**算法：** + +$$m \leftarrow \beta_1 \cdot m + (1 - \beta_1) \cdot g$$ +$$v \leftarrow \beta_2 \cdot v + (1 - \beta_2) \cdot g^2$$ +$$\hat{m} = \frac{m}{1 - \beta_1^t}, \quad \hat{v} = \frac{v}{1 - \beta_2^t}$$ +$$p \leftarrow p - \eta \cdot \frac{\hat{m}}{\sqrt{\hat{v}} + \epsilon}$$ + +**引擎原语：** +``` +// m = β1*m + (1-β1)*g +engine.scale_inplace(m, β1) +scale_add_(m, 1-β1, g) + +// v = β2*v + (1-β2)*g² +engine.scale_inplace(v, β2) +auto g_sq = engine.elementwise_binary(Mul, g, g) +scale_add_(v, 1-β2, g_sq) + +// 偏差修正 + 更新 +auto m_hat = engine.scale(m, inv_bc1) +auto v_hat = engine.scale(v, inv_bc2) +auto sqrt_v = engine.elementwise_unary(Sqrt, v_hat) +auto denom = engine.elementwise_binary_scalar(Add, sqrt_v, eps) +auto step = engine.elementwise_binary(Div, m_hat, denom) +engine.scale_inplace(step, -lr) +engine.add_inplace(p, step) +``` + +**参数：** `lr`, `β1=0.9`, `β2=0.999`, `ε=1e-8` + +**偏差修正：** +- `bc1 = 1 - β1^t`（一阶矩偏差修正） +- `bc2 = 1 - β2^t`（二阶矩偏差修正） + +--- + +### 4. AdamW — 解耦权重衰减 + +**算法：** + +$$p \leftarrow (1 - \eta \cdot \lambda) \cdot p \quad \text{（权重衰减，独立于梯度）}$$ +$$m \leftarrow \beta_1 \cdot m + (1 - \beta_1) \cdot g$$ +$$v \leftarrow \beta_2 \cdot v + (1 - \beta_2) \cdot g^2$$ +$$p \leftarrow p - \eta \cdot \frac{\hat{m}}{\sqrt{\hat{v}} + \epsilon}$$ + +**与 Adam 的区别：** +- Adam 的 L2 正则化：`g += λ·p` → 权重衰减被自适应学习率稀释 +- AdamW 的解耦衰减：`p *= (1-η·λ)` → 权重衰减对所有参数等效 + +**参数：** `lr`, `β1=0.9`, `β2=0.999`, `ε=1e-8`, `weight_decay=0.01` + +--- + +### 5. Muon — MomentUm Orthogonalized by Newton-Schulz + +**算法（Keller Jordan et al., 2024）：** + +对于每个 2D 参数 $p$（权重矩阵）： + +1. **SGD-Momentum：** $v \leftarrow \mu \cdot v + g$ +2. **Newton-Schulz 正交化：** $\text{update} = \text{NS}_5(v)$ +3. **参数更新：** $p \leftarrow p - \eta \cdot \text{update}$ + +对于非 2D 参数（bias 等）：标准 SGD 更新。 + +#### Newton-Schulz 正交化 + +**目标：** 计算矩阵 $G$ 的最近半正交矩阵 $\text{Ortho}(G) = UV^T$（$G = USV^T$ 为 SVD）。 + +**算法：** + +$$X = \frac{G}{\|G\|_F + \epsilon}$$ + +重复 5 次： +$$A = X \cdot X^T$$ +$$B = b \cdot A + c \cdot A^2$$ +$$X = a \cdot X + B \cdot X$$ + +**调优系数：** `a = 3.4445`, `b = -4.7750`, `c = 2.0315` + +这些系数使 quintic 多项式 $\phi^N(x) \to 1$ for $x \in [0,1]$，5 步内收敛。 + +**参考：** +- Keller Jordan et al., "Muon: An optimizer for hidden layers in neural networks" +- https://kellerjordan.github.io/posts/muon/ + +**引擎原语（每步）：** +``` +A = engine.matmul(X, X, transB=true) // X × X^T +A_sq = engine.matmul(A, A) // A² +B = engine.scale(A, b) + engine.scale(A_sq, c) +BX = engine.matmul(B, X) +X = engine.scale(X, a) + BX +``` + +**参数：** `lr`, `momentum=0.95`, `nesterov=true`, `ns_steps=5`, `ns_eps=1e-7` + +--- + +## 🏭 优化器工厂函数 + +```cpp +auto optimizer = nn::create_optimizer( + name, engine, params, grads, lr, weight_decay); +``` + +| `name` | 创建的优化器 | 额外参数 | +|--------|-------------|----------| +| `"sgd"` | `SGD` | — | +| `"sgd_momentum"` | `SGDWithMomentum` | — | +| `"adam"` | `Adam` | — | +| `"adamw"` | `AdamW` | `weight_decay` | +| `"muon"` | `Muon` | — | +| 默认 | `Adam` | — | + +--- + +## 📊 各组件复杂度对比 + +| 组件 | 参数量 | Forward FLOPs (per sample) | Backward FLOPs | +|------|--------|---------------------------|----------------| +| `Linear(in, out)` | in×out + out | 2×in×out | 4×in×out | +| `ReLU` | 0 | N | N | +| `GeLU` | 0 | ~5N | ~5N | +| `LayerNorm(F)` | 2F | ~5F | ~8F | +| `Softmax` | 0 | ~3S | ~3S | +| `MHA(d, h, s)` | 4d² | ~8d²s + 2ds² | ~16d²s + 4ds² | +| `FeedForward(d, f)` | 2df + d + f | 4df | 8df | +| `GPTBlock(d, h, f, s)` | — | MHA + FFN + 4LN | MHA + FFN + 4LN | + +其中：N = 元素数, F = 特征维度, S = 序列长度, d = d_model, h = num_heads, f = d_ff diff --git a/docs/06-cuda-backend.md b/docs/06-cuda-backend.md new file mode 100644 index 00000000..b218b1fb --- /dev/null +++ b/docs/06-cuda-backend.md @@ -0,0 +1,284 @@ +# ⚡ CUDA GPU 加速后端 + +> **版本**：1.0 +> **GPU**：Tesla V100-PCIE-32GB (sm_70) +> **CUDA Toolkit**：12.8 +> **依赖**：nvcc + MSVC 2022 BuildTools（编译期），cudart / cuda / nvrtc（运行时） + +--- + +## 📋 目录 + +1. [设计概览](#-设计概览) +2. [构建方法](#-构建方法) +3. [架构详解](#-架构详解) +4. [与 Vulkan 后端对比](#-与-vulkan-后端对比) +5. [内核清单](#-内核清单) +6. [编译器兼容性备注](#-编译器兼容性备注) +7. [使用方式](#-使用方式) +8. [故障排除](#-故障排除) + +--- + +## 🎯 设计概览 + +CUDA 后端与 Vulkan 后端架构对称，共享同一套 `ComputeEngine` 接口： + +``` +ComputeEngine (抽象接口) +├── CpuEngine ── 纯 CPU 实现 +├── GpuEngine ── Vulkan compute shader +└── CudaEngine ── CUDA 内核 (本后端) + └── CudaBackend ── 管理设备/内存/内核调用 +``` + +**关键决策**：CUDA 内核由 **nvcc 独立编译为 .obj**，再通过 C ABI 链接到 Clang 主程序。这是因为 nvcc 12.8 与 MSVC 2026 (VS 18) 的 `cudafe++` 不兼容，但与 VS 2022 BuildTools (MSVC 14.44) 可正常工作。 + +``` +cuda/cuda_kernels.cu ──nvcc (MSVC 2022)──▶ cuda_kernels.obj + │ +src/*.cpp ──Clang──▶ *.obj ────────────────────────┤ + │ +include/.../cuda_backend.hpp (extern "C") ◀──┘ + .../cuda_engine.hpp (ComputeEngine) +``` + +--- + +## 🔨 构建方法 + +### 前置条件 + +1. **CUDA Toolkit 12.8**：`C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8` +2. **VS 2022 BuildTools**：`C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools` + - nvcc 需要 MSVC 2017–2022 作为 host compiler + - VS 2026 (MSVC 14.51) **不兼容**——`cudafe++` 会 crash (ACCESS_VIOLATION) +3. **LLVM/Clang**：主项目编译器 + +### 构建命令 + +```bash +# 配置（启用 CUDA） +cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DNN_ENABLE_CUDA=ON + +# 可选：指定 SM 架构（默认 70 = V100） +cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DNN_ENABLE_CUDA=ON -DNN_CUDA_ARCH=80 + +# 构建 +cmake --build build --parallel +``` + +### CMake 选项 + +| 选项 | 默认值 | 说明 | +|------|--------|------| +| `NN_ENABLE_CUDA` | `OFF` | 启用 CUDA GPU 加速 | +| `NN_CUDA_ARCH` | `70` | SM 架构版本（70=V100, 80=A100, 86=RTX30xx, 89=RTX40xx） | + +--- + +## 🏗️ 架构详解 + +### 文件结构 + +``` +cuda/ +├── CMakeLists.txt ← nvcc 编译脚本（.bat 包装 vcvars + nvcc） +├── cuda_kernels.cu ← 15 个 CUDA 内核实现 +└── cuda_kernels.h ← extern "C" 接口声明 + +include/neuralnet.cpp/ +├── backend/ +│ └── cuda_backend.hpp ← CudaBackend 单例 + CudaTensor + CudaBuffer +└── cuda_engine.hpp ← CudaEngine（ComputeEngine 实现） +``` + +### 编译流程 + +1. **CMake 配置阶段**：生成 `compile_cuda_kernels.bat` 脚本 + - 调用 `vcvars64.bat` 初始化 MSVC 2022 环境 + - 运行 `nvcc -c -O2 --use_fast_math -allow-unsupported-compiler -Xcompiler /MD -arch=sm_XX` +2. **构建阶段**： + - Ninja 调用 .bat 脚本编译 `cuda_kernels.cu` → `cuda_kernels.obj` + - Clang 编译所有 `.cpp` 文件（含 cuda_backend.hpp） + - 链接：`cuda_kernels.obj` + `cudart.lib` + `cuda.lib` + `nvrtc.lib` +3. **运行阶段**： + - `CudaEngine` 调用 `CudaBackend` 的方法 + - `CudaBackend` 通过 extern "C" 函数调用 nvcc 编译的内核 + +### 数据流 + +``` +CPU Matrix ──CudaBackend::upload_blocking──▶ CudaTensor (GPU) + │ + ├── CudaBackend::matmul_gpu() ──▶ CudaTensor + ├── CudaBackend::elementwise_*() ──▶ CudaTensor + ├── CudaBackend::reduce_gpu() ──▶ CudaTensor + └── ... + │ +CudaTensor ──CudaBackend::download_blocking──▶ CPU Matrix +``` + +所有计算在 GPU 上原生执行，仅在 `from_matrix` / `to_matrix` 时进行 PCIe 传输。 + +--- + +## ⚖️ 与 Vulkan 后端对比 + +| 特性 | Vulkan | CUDA | +|------|--------|------| +| 编译器 | glslc (GLSL → SPIR-V) | nvcc (.cu → .obj) | +| 内核语言 | GLSL compute shader | CUDA C++ | +| 设备管理 | VkInstance / VkDevice / VkQueue | cudaSetDevice / cudaStream | +| 内存管理 | MemoryPool 子分配器 | cudaMalloc / cudaMemcpy | +| 同步模型 | Command buffer + fence | cudaDeviceSynchronize (同步流) | +| 矩阵乘法 | 自定义 tiled shader | 自定义 tiled kernel | +| 批处理 | begin_batch / end_batch (录制) | 预留（当前 no-op） | +| 构建依赖 | Vulkan SDK + glslc | CUDA Toolkit + MSVC 2022 | +| 运行时依赖 | vulkan-1.dll | cudart64_12.dll + cuda64_12.dll | + +### 内核对照 + +两套后端的内核语义完全一致，可互相替换： + +| 原语 | Vulkan shader | CUDA kernel | +|------|---------------|-------------| +| matmul | matmul.comp / matmul_tiled.comp | matmul_kernel / batched_matmul_kernel | +| elementwise | elementwise_v2.comp | elementwise_{unary,binary,scalar,axpy,select}_kernel | +| reduce | reduce.comp | reduce_{row,col}_{sum,max}_kernel | +| broadcast | broadcast.comp | broadcast_{row,col}_kernel | +| transpose | transpose.comp | transpose_kernel | +| gather | gather.comp | gather_kernel | +| scatter_add | scatter_add.comp | scatter_add_kernel | +| rearrange_3d | rearrange_3d.comp | rearrange_3d_kernel | + +--- + +## 📦 内核清单 + +### cuda_kernels.h — extern "C" 接口 + +```c +// 矩阵乘法 +int cuda_matmul(const float* A, const float* B, float* C, ...); +int cuda_batched_matmul(const float* A, const float* B, float* C, ...); + +// 逐元素运算 +int cuda_elementwise_unary(const float* A, float* out, ...); +int cuda_elementwise_binary(const float* A, const float* B, float* out, ...); +int cuda_elementwise_binary_scalar(const float* A, float* out, ...); +int cuda_axpy(const float* A, const float* B, float* out, ...); +int cuda_elementwise_select_scalar_cond(const float* A, const float* then_v, float* out, ...); + +// 归约 +int cuda_reduce_row(const float* A, float* out, ...); +int cuda_reduce_col(const float* A, float* out, ...); + +// 广播 +int cuda_broadcast_row(const float* A, const float* vec, float* out, ...); +int cuda_broadcast_col(const float* A, const float* vec, float* out, ...); + +// 转置与重排 +int cuda_transpose(const float* A, float* out, ...); +int cuda_rearrange_3d(const float* A, float* out, ...); + +// Gather / Scatter +int cuda_gather(const float* table, const int* indices, float* out, ...); +int cuda_scatter_add(float* dst, const int* indices, const float* grad, ...); +``` + +所有函数返回 `int`（0=成功，非0=cudaError_t），最后一个参数为 `void* stream`（nullptr=默认流）。 + +--- + +## ⚠️ 编译器兼容性备注 + +### 问题 + +nvcc 12.8 的内部组件 `cudafe++` 在解析 **MSVC 2026 (VS 18, v14.51)** 头文件时会 crash： + +``` +nvcc error: 'cudafe++' died with status 0xC0000005 (ACCESS_VIOLATION) +``` + +`-allow-unsupported-compiler` 只绕过版本检查，但 `cudafe++` 仍然会因无法解析新头文件而崩溃。 + +### 解决方案 + +使用 **VS 2022 BuildTools (MSVC 14.44)** 作为 nvcc 的 host compiler： + +```bat +call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" +nvcc -c -O2 --use_fast_math -allow-unsupported-compiler -Xcompiler /MD -arch=sm_70 cuda_kernels.cu -o cuda_kernels.obj +``` + +### CRT 匹配 + +nvcc 默认使用 `/MT`（静态 CRT），但 Clang 主程序使用 `/MD`（动态 CRT）。必须通过 `-Xcompiler /MD` 强制 nvcc 使用动态 CRT，否则链接时报错： + +``` +lld-link: error: /failifmismatch: mismatch detected for 'RuntimeLibrary' +``` + +--- + +## 🎮 使用方式 + +### 命令行 + +```bash +# MNIST 训练 + CUDA 加速 +./build/mnist_train --cuda --epochs 10 + +# GPT 训练 + CUDA 加速 +./build/text_train corpus.txt --cuda --epochs 10 --lr 0.001 + +# 推理 + CUDA 加速 +./build/text_infer --model gpt_model.bin --cuda --prompt "Hello" + +# MNIST 推理 + CUDA 加速 +./build/mnist_infer --model mnist_model.bin --cuda --input test.csv +``` + +### 优先级 + +`--cuda` 优先于 `--gpu`（Vulkan）。如果同时指定两者，CUDA 生效。 + +### C++ API + +```cpp +#include + +// 创建 CUDA 引擎 +auto& backend = nn::CudaBackend::instance(); +backend.initialize(); +nn::CudaEngine engine(backend); + +// 与 CpuEngine / GpuEngine 完全相同的 API +nn::Model model(engine); +model.add_linear(784, 256).add_relu().add_linear(256, 10); +``` + +--- + +## 🔧 故障排除 + +### "No CUDA devices found" + +确认 NVIDIA 驱动已安装：`nvidia-smi` + +### nvcc 编译失败："unsupported Microsoft Visual Studio version" + +确保 VS 2022 BuildTools 已安装。CMakeLists.txt 中的 `.bat` 脚本会自动调用 `vcvars64.bat`。 + +### cudafe++ ACCESS_VIOLATION + +MSVC 版本太新。使用 VS 2022 BuildTools（MSVC 14.44），不要用 VS 2026（MSVC 14.51）。 + +### 链接错误：RuntimeLibrary mismatch + +nvcc 需要 `-Xcompiler /MD` 标志。检查 `build/cuda/compile_cuda_kernels.bat` 是否包含此标志。 + +### 找不到 cuda_kernels.h + +CMakeLists.txt 中 `NN_CUDA_INCLUDE_DIR` 指向 `cuda/` 目录。确认 `add_subdirectory(cuda)` 存在。 diff --git a/ARCHITECTURE.md b/docs/ARCHITECTURE.md similarity index 75% rename from ARCHITECTURE.md rename to docs/ARCHITECTURE.md index 43728885..e0a95aa0 100644 --- a/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -8,48 +8,68 @@ ```mermaid graph TB - subgraph "🖥️ 用户入口层" - A["gui.py
Python GUI 入口"] - B["src/train.cpp
训练入口"] - C["src/infer.cpp
推理入口"] + subgraph "🖥️ 用户入口层 (L5)" + A["gui.py
Python GUI"] + B["src/text_train.cpp
GPT 训练"] + C["src/text_infer.cpp
GPT 推理"] + B2["src/mnist_train.cpp
MNIST 训练"] + C2["src/mnist_infer.cpp
MNIST 推理"] end - subgraph "🧩 核心库 (include/neuralnet.cpp/)" - D["nn.hpp
统一头文件"] - E["model.hpp
网络容器"] - F["layer.hpp
层定义"] - G["loss.hpp
损失函数"] - H["optimizer.hpp
优化器"] - I["matrix.hpp
矩阵运算"] - J["model_io.hpp
模型序列化"] - K["nn_config.hpp
配置策略"] - L["thread_pool.hpp
线程池"] + subgraph "🧩 构建层 (L4)" + GPT["domain_gpt.hpp
GPT 模型工厂"] + MNIST["domain_mnist.hpp
MNIST 模型工厂"] + end + + subgraph "🧩 实现层 (L3)" + E["model_container.hpp
模型容器"] + J["model_serialization.hpp
序列化"] + MS["model_spec.hpp
架构描述"] + end + + subgraph "🧩 计算层 (L2)" + F["compute_layer.hpp
层定义"] + G["compute_loss.hpp
损失函数"] + H["compute_optimizer.hpp
优化器"] + end + + subgraph "🧩 代数层 (L1)" + I["algebra_matrix.hpp
矩阵运算"] + end + + subgraph "🧩 硬件层 (L0)" + K["config.hpp
配置策略"] + L["core_threadpool.hpp
线程池"] end subgraph "💾 数据层" - M["datasets/mnist_data/
MNIST CSV 数据"] + M["datasets/
训练数据"] N["pretrained/
预训练模型"] end - A -->|"subprocess 调用"| B - A -->|"subprocess 调用"| C - B --> D - C --> D - D --> E - D --> F - D --> G - D --> H - E --> F - F --> I - G --> I - H --> I - I --> K - K --> L + A -->|"subprocess"| B & C + B --> GPT + C --> GPT + B2 --> MNIST + C2 --> MNIST + GPT -->|"Model API"| E + MNIST -->|"Model API"| E + GPT -->|"ModelSpec"| MS + MNIST -->|"ModelSpec"| MS + E -->|"Layer::forward"| F J --> E + J --> MS + F -->|"Matrix API"| I + G -->|"Matrix API"| I + H -->|"Matrix API"| I + I -->|"SmartPolicy"| K + K --> L B --> M - B -->|"save_model()"| N - C -->|"load_model()"| N - C --> M + B2 --> M + B -->|"save"| N + B2 -->|"save"| N + C -->|"load"| N + C2 -->|"load"| N ``` --- @@ -57,52 +77,61 @@ graph TB ## 🏗️ 分层架构详解 ```mermaid -graph LR - subgraph "应用层" - T["train.cpp
训练流程"] - I["infer.cpp
推理流程"] - P["gui.py
图形界面"] +graph TB + subgraph "L5 交互层" + SRC["src/*.cpp
训练/推理入口"] + NN["nn.hpp
统一聚合头文件"] end - - subgraph "接口层" - M["Model
网络容器"] - IO["ModelIO
模型持久化"] + subgraph "L4 构建层" + GPT["domain_gpt.hpp
GPT 工厂 + 超参数"] + MNIST["domain_mnist.hpp
MNIST 工厂 + 超参数"] end - - subgraph "计算层" - L["Linear Layer
全连接层"] - R["ReLU Layer
激活层"] - CE["CrossEntropy
交叉熵损失"] - OPT["Adam / SGD
优化器"] + subgraph "L3 实现层" + MDL["model_container.hpp
Model 容器 + 非模板 API"] + IO["model_serialization.hpp
二进制序列化"] + MS["model_spec.hpp
ModelSpec 纯数据"] end - - subgraph "基础层" - MX["Matrix
矩阵运算"] - TP["ThreadPool
并行执行"] - CFG["SmartPolicy
自适应调度"] + subgraph "L2 计算层" + LAY["compute_layer.hpp
Linear/ReLU/GPT..."] + LOSS["compute_loss.hpp
MSE/CrossEntropy"] + OPT["compute_optimizer.hpp
SGD/Adam"] + end + subgraph "L1 代数层" + MAT["algebra_matrix.hpp
矩阵运算"] + end + subgraph "L0 硬件层" + CFG["config.hpp
SmartPolicy"] + TP["core_threadpool.hpp
线程池"] end - T --> M - T --> IO - T --> CE - T --> OPT - I --> M - I --> IO - P --> T - P --> I - M --> L - M --> R - L --> MX - R --> MX - CE --> MX - OPT --> MX - MX --> CFG + SRC -->|"调用构建/Model API"| GPT & MNIST & MDL + NN -.->|"聚合"| SRC + GPT -->|"model.add_gpt_model()"| MDL + MNIST -->|"model.add_linear()"| MDL + GPT & MNIST -->|"ModelSpec"| MS + MDL -->|"Layer::forward()"| LAY + IO --> MDL & MS + LAY -->|"Matrix API"| MAT + LOSS -->|"Matrix API"| MAT + OPT -->|"Matrix API"| MAT + MAT -->|"SmartPolicy"| CFG CFG --> TP ``` +**调用规则**：L(N) 只能调用 L(N-1) 的 API，禁止同层调用和跨层调用。 + +| 层级 | 职责 | 只能调用 | +|------|------|----------| +| **L5 交互层** | 用户入口，训练/推理流程 | L4 构建 API, L3 Model API | +| **L4 构建层** | 模型工厂（组装层为模型） | L3 `Model::add_*()` 非模板 API | +| **L3 实现层** | 模型容器 + 序列化 | L2 `Layer::forward/backward` | +| **L2 计算层** | 层/损失/优化器定义 | L1 `Matrix` 语义化 API | +| **L1 代数层** | 矩阵运算 | L0 `SmartPolicy` | +| **L0 硬件层** | 并行策略 | 无（最底层） | + --- -## 🔄 训练流程 (train.cpp) +## 🔄 训练流程 (mnist_train.cpp) ```mermaid flowchart TD @@ -281,7 +310,7 @@ classDiagram +step(params, grads) } - class SGD_w_Momentum { + class SGDWithMomentum { -double learning_rate -double momentum -vector~Matrix~ velocities @@ -299,7 +328,7 @@ classDiagram } Optimizer <|-- SGD - Optimizer <|-- SGD_w_Momentum + Optimizer <|-- SGDWithMomentum Optimizer <|-- Adam ``` @@ -368,8 +397,8 @@ flowchart LR ```mermaid flowchart TD - GUI["gui.py
Tkinter GUI"] -->|"训练模式
subprocess.run()"| TRAIN["train.cpp"] - GUI -->|"推理模式
subprocess.run()"| INFER["infer.cpp"] + GUI["gui.py
Tkinter GUI"] -->|"训练模式
subprocess.run()"| TRAIN["mnist_train.cpp"] + GUI -->|"推理模式
subprocess.run()"| INFER["mnist_infer.cpp"] GUI -->|"画布绘制"| CANVAS["28×28 画布"] CANVAS -->|"鼠标事件"| DRAW["捕获笔画"] diff --git a/docs/AST_COMPUTE.md b/docs/AST_COMPUTE.md new file mode 100644 index 00000000..a2cf6baf --- /dev/null +++ b/docs/AST_COMPUTE.md @@ -0,0 +1,430 @@ +# 表达式模板：编译期 AST 胶水语言设计 + +## 一、核心思想 + +上层用**普通 C++ 运算符**写表达式，编译器自动构建**编译期 AST**。 +底层拿到 AST 后： +- **CPU 路径**：递归求值 AST（零开销，和手写循环等价） +- **GPU 路径**：AST → `consteval` 生成 SPIR-V 二进制 → `vkCreateShaderModule` → dispatch + +**零运行时编译开销**：SPIR-V 二进制在编译期从 AST 直接生成，无需 GLSL 中间层，无需任何外部工具。 + +算法永远在上层，计算永远在底层，中间靠 AST 传递。 + +## 二、上层写法（Layer 视角） + +```cpp +// Layer 中：用普通运算符写算法 +Span x = result.span(); + +compute::apply(x, max(x, Scalar{0})); // ReLU +compute::apply(x, x * (Scalar{1} / (Scalar{1} + exp(-Scalar{1.702} * x)))); // QuickGeLU +compute::apply(x, select(x > Scalar{0}, x, x * Scalar{0.01})); // LeakyReLU +compute::apply(x, abs(x)); // abs +compute::apply(x, x * Scalar{2} + Scalar{1}); // 线性变换 +``` + +> ⚠️ `Scalar` 是 [nn_config.hpp](../include/neuralnet.cpp/nn_config.hpp) 定义的标量类型别名（当前为 `float`）。 +> 由于 `compute::apply` 接受 `const Expr&` 形参，字面量必须显式标 `Scalar{...}` 以触发模板运算符重载， +> 否则 `x > 0` 中的 `0`（int）会与 `Span::operator>(Scalar)` 的重载不匹配。 +> 三元条件需用 `select()`，C++ 无法重载 `?:`。 + +上层**完全不知道**底下是 CPU 还是 GPU。运算符重载在幕后构建 AST。 + +## 三、Span：可构建 AST 的智能视图 + +```cpp +// span.hpp — 替代 std::span 作为逐元素操作的载体 +class Span { + Scalar* data_; + std::size_t size_; +public: + Scalar* data() noexcept; + std::size_t size() const noexcept; + + // ── 运算符重载：返回 Expr 而非 Scalar ────────────────── + // Span OP Scalar → BinaryExpr + auto operator+(Scalar s) const; // → BinaryExpr<...> + auto operator-(Scalar s) const; + auto operator*(Scalar s) const; + auto operator/(Scalar s) const; + auto operator>(Scalar s) const; // → CmpExpr<...> + auto operator<(Scalar s) const; + auto operator==(Scalar s) const; + + // Scalar OP Span → BinaryExpr + friend auto operator+(Scalar s, const Span& sp); + friend auto operator*(Scalar s, const Span& sp); + + // Span OP Span → BinaryExpr + auto operator+(const Span& other) const; + auto operator-(const Span& other) const; + auto operator*(const Span& other) const; +}; +``` + +## 四、Expr 类型系统（编译期 AST 节点） + +```cpp +// expr.hpp — 表达式模板核心 + +// ── 叶子节点 ──────────────────────────────────────────────── +struct Val { // 常量 + Scalar value; + auto eval(std::size_t) const { return value; } + consteval SpvId to_spv(SpvBuilder& spv) const { + return spv.float_constant(value); // OpConstant %float value + } +}; + +// ── 一元表达式 ────────────────────────────────────────────── +template +struct UnaryExpr { + Child child; + static constexpr bool has_spv = Child::has_spv; + + auto eval(std::size_t i) const { + return Op::apply(child.eval(i)); + } + consteval SpvId to_spv(SpvBuilder& spv) const { + auto a = child.to_spv(spv); + return Op::spv_op(spv, a); // 例如 OpFMax %a %float_0 + } +}; + +// ── 二元表达式 ────────────────────────────────────────────── +template +struct BinaryExpr { + Left left; + Right right; + static constexpr bool has_spv = Left::has_spv && Right::has_spv; + + auto eval(std::size_t i) const { + return Op::apply(left.eval(i), right.eval(i)); + } + consteval SpvId to_spv(SpvBuilder& spv) const { + auto l = left.to_spv(spv); + auto r = right.to_spv(spv); + return Op::spv_op(spv, l, r); // 例如 OpFAdd %l %r + } +}; + +// ── 三元表达式（条件选择）──────────────────────────────────── +template +struct TernaryExpr { + Cond cond; + Then then_expr; + Else else_expr; + static constexpr bool has_spv = Cond::has_spv && Then::has_spv && Else::has_spv; + + auto eval(std::size_t i) const { + return cond.eval(i) ? then_expr.eval(i) : else_expr.eval(i); + } + consteval SpvId to_spv(SpvBuilder& spv) const { + auto c = cond.to_spv(spv); + auto t = then_expr.to_spv(spv); + auto e = else_expr.to_spv(spv); + return spv.select(c, t, e); // OpSelect %c %t %e + } +}; +``` + +## 五、Op 策略（编译期函数对象） + +```cpp +// ops.hpp — 每个 op 同时定义 CPU 行为和 SPIR-V 指令 + +namespace ops { + +struct Add { + static Scalar apply(Scalar a, Scalar b) { return a + b; } + static consteval SpvId spv_op(SpvBuilder& spv, SpvId a, SpvId b) { + return spv.fadd(a, b); // OpFAdd %result %a %b + } +}; + +struct Mul { + static Scalar apply(Scalar a, Scalar b) { return a * b; } + static consteval SpvId spv_op(SpvBuilder& spv, SpvId a, SpvId b) { + return spv.fmul(a, b); // OpFMul %result %a %b + } +}; + +struct Gt { + static bool apply(Scalar a, Scalar b) { return a > b; } + static consteval SpvId spv_op(SpvBuilder& spv, SpvId a, SpvId b) { + return spv.ford_less_than(b, a); // OpFOrdGreaterThan %a %b → bool + } +}; + +struct Max { + static Scalar apply(Scalar a, Scalar b) { return std::max(a, b); } + static consteval SpvId spv_op(SpvBuilder& spv, SpvId a, SpvId b) { + return spv.glsl_std450_fmax(a, b); // OpExtInst %GLSLstd450FMax %a %b + } +}; + +struct Neg { + static Scalar apply(Scalar a) { return -a; } + static consteval SpvId spv_op(SpvBuilder& spv, SpvId a) { + return spv.fneg(a); // OpFNegate %result %a + } +}; + +struct Abs { + static Scalar apply(Scalar a) { return std::abs(a); } + static consteval SpvId spv_op(SpvBuilder& spv, SpvId a) { + return spv.glsl_std450_fabs(a); // OpExtInst %GLSLstd450FAbs %a + } +}; + +} // namespace ops +``` + +## 六、运算符重载：连接上层表达式与 AST + +```cpp +// span_ops.hpp — Span 运算符重载 + +// Span + Scalar → BinaryExpr +inline auto Span::operator+(Scalar s) const { + return BinaryExpr{*this, Val{s}}; +} + +// Span > Scalar → BinaryExpr +inline auto Span::operator>(Scalar s) const { + return BinaryExpr{*this, Val{s}}; +} + +// Span * Span → BinaryExpr +inline auto Span::operator*(const Span& other) const { + return BinaryExpr{*this, other}; +} +``` + +## 七、compute::apply — 统一入口 + +> 注：当前实现位于 [compute_dispatch.hpp](../include/neuralnet.cpp/algebra/compute_dispatch.hpp)， +> 为自由函数 `nn::compute::apply(Span, const Expr&)`。下面是设计目标（含 GPU 分支预览）。 + +```cpp +// compute_dispatch.hpp + +template +void apply(Span x, const Expr &expr) { + const auto n = x.size(); + if (n == 0) return; + +#ifdef NN_HAS_VULKAN + if constexpr (Expr::has_spv) { // 编译期：此表达式能生成 SPIR-V？ + if (SmartPolicy::gpu_enabled) { + auto& backend = GpuBackend::instance(); + if (backend.is_initialized() || backend.initialize()) { + // 编译期生成 SPIR-V，运行时零开销加载 + consteval auto spv = expr.to_spv_module(); + if (backend.elementwise_from_spv(x, spv)) return; + } + } + } +#endif + + // CPU 路径：按索引分发（小数据串行，大数据并行），编译器内联等价手写循环 + if (n >= SmartPolicy::PARALLEL_THRESHOLD) { + auto indices = std::views::iota(std::size_t{0}, n); + SmartPolicy::for_each(indices.begin(), indices.end(), + [&x, &expr](std::size_t i) noexcept { + x[i] = static_cast(expr.eval(i)); + }); + } else { + for (std::size_t i = 0; i < n; ++i) + x[i] = static_cast(expr.eval(i)); + } +} +``` + +## 八、完整的 AST 构建示例 + +上层写 `x * (Scalar{1} / (Scalar{1} + exp(Scalar{-1.702} * x)))` 时，编译器构建的类型： + +``` +BinaryExpr< + Span, // x + BinaryExpr< + Val{1.0}, // 1.0 + BinaryExpr< + Val{1.0}, // 1.0 + UnaryExpr< + BinaryExpr< // -1.702 * x + Val{-1.702}, // -1.702 + Span, // x + ops::Mul + >, + ops::Exp // exp(-1.702 * x) + >, + ops::Add // 1.0 + exp(...) + >, + ops::Div // 1.0 / (1.0 + exp(...)) + >, + ops::Mul // x * (1.0 / (...)) +> +``` + +运行时调用 `expr.eval(i)` 对每个索引 `i` 求值，编译期内联后等价于： + +```cpp +// 编译器展开后等价的手写循环 +for (std::size_t i = 0; i < n; ++i) { + Scalar xv = x[i]; + data[i] = xv * (1.0f / (1.0f + std::exp(-(-1.702f * xv)))); +} +``` + +> ⚠️ 注意：C++ 无法重载 `operator?:`，因此三元条件表达式**不能**直接写成 +> `x > 0 ? x : x * 0.01f`。需要使用 `select()` 自由函数（见 [expr.hpp](../include/neuralnet.cpp/algebra/expr.hpp)）。 + +## 九、条件表达式（三元选择） + +> C++ 不允许重载 `operator?:`，因此 LeakyReLU 不能直接写成 `x > 0 ? x : x * 0.01f`。 +> 提供 `select(cond, then, else)` 自由函数替代（见 [expr.hpp](../include/neuralnet.cpp/algebra/expr.hpp)）。 + +```cpp +// LeakyReLU: x > 0 ? x : x * 0.01 —— 用 select() 表达 +compute::apply(x, select(x > Scalar{0}, x, x * 0.01f)); + +// 编译器构建： +TernaryExpr< + BinaryExpr, // x > 0 + Span, // x + BinaryExpr // x * 0.01 +> + +// CPU 求值（编译期内联）： +// for (i=0..n) data[i] = (data[i] > 0) ? data[i] : (data[i] * 0.01f); +``` + +## 十、CPU 路径的求值策略 + +```cpp +// 逐元素递归求值 +for (std::size_t i = 0; i < n; ++i) + data[i] = expr.eval(i); +``` + +**关键**：因为 Expr 类型在编译期完全确定，编译器会： +1. 内联所有 `eval()` 调用 +2. 消除 AST 节点的内存分配（全部是栈上值类型） +3. 向量化最终的标量循环 + +实测性能与手写 `for` 循环**完全等价**（零开销抽象）。 + +## 十一、GPU 路径的运行时流程 + +``` +Layer: compute::apply(span, expr) + ↓ +ComputeDispatch: Expr::has_spv == true（编译期检查） + ↓ +consteval auto spv = expr.to_spv_module() // 编译期：AST → SPIR-V 二进制 + ↓ +GpuBackend: vkCreateShaderModule(spv_bytes) // 运行时：加载二进制，零编译 + ↓ +vkCmdDispatch → GPU 执行 +``` + +**零运行时编译开销**：SPIR-V 在编译期由 `consteval` 从 AST 直接生成， +运行时只做内存拷贝 + `vkCreateShaderModule`（微秒级）。 + +## 十二、性能分析 + +| 路径 | 编译期 | 运行时 | 开销 | +|------|--------|--------|------| +| AST 构建 | ✅ 编译器完成 | — | 零 | +| CPU 求值 | ✅ 编译器内联 | ✅ 执行 | 与手写循环等价 | +| SPIR-V 生成 | ✅ `consteval` | — | 零 | +| `vkCreateShaderModule` | — | ✅ 加载二进制 | 微秒级 | +| GPU dispatch | — | ✅ | 与现有相同 | + +与 GLSL→SPIR-V 运行时编译方案的对比： + +| | GLSL 方案 | SPIR-V 直出 | +|---|-----------|------------| +| 运行时编译 | ~2ms（GLSL→SPIR-V） | **零** | +| 外部依赖 | `glslangValidator` | **无** | +| 缓存需求 | 需要（避免重复编译） | 不需要 | +| 首次开销 | ~2ms | 微秒级 | + +## 十三、扩展新操作 + +### 添加 exp() + +**步骤 1**：在 [ops.hpp](../include/neuralnet.cpp/algebra/ops.hpp) 中定义 Op（CPU `apply` + 可选 SPIR-V 指令） +```cpp +struct Exp { + [[nodiscard]] static Scalar apply(Scalar a) noexcept { return std::exp(a); } + // GPU 路径启用后再添加： + // static consteval SpvId spv_op(SpvBuilder& spv, SpvId a) { return spv.glsl_std450_exp(a); } +}; +``` + +**步骤 2**：在 [expr.hpp](../include/neuralnet.cpp/algebra/expr.hpp) 中添加自由函数（与现有 `exp`/`log`/`sigmoid` 等保持一致） +```cpp +template +[[nodiscard]] auto exp(const Expr &expr) { + return UnaryExpr{expr}; +} +``` + +**步骤 3**：完成。上层直接使用： +```cpp +compute::apply(x, exp(x)); +``` + +**不需要修改 `compute_dispatch.hpp` 或 `Matrix`。** + +## 十四、局限性 + +| 特性 | CPU | GPU | +|------|-----|-----| +| 算术运算 (+, -, *, /) | ✅ | ✅（待实现） | +| 比较 (>, <, ==) | ✅ | ✅（待实现） | +| 三元选择 `select()` | ✅ | ✅（待实现） | +| 数学函数 (exp, abs, max, min) | ✅ 已定义 Op | ✅ 需补 Op 的 SPIR-V 指令 | +| 循环/分支逻辑 | ❌ AST 不支持 | ❌ | +| 函数调用（非数学函数） | ❌ | ❌ | +| 任意 C++ lambda | ❌ | ❌ | + +> ⚠️ C++ 无法重载 `operator?:`，因此"三元条件"通过 `select()` 自由函数实现，不是语言原生运算符。 + +**神经网络覆盖率**：ReLU, GeLU, SiLU, Softmax(部分), LayerNorm(部分) 等常见激活函数均可表达。覆盖率约 **85-90%** 的逐元素操作。 + +## 十五、跨平台支持 + +SPIR-V 是 Khronos 组织定义的 **GPU 中间表示**，一次生成，到处执行。 + +| 平台 | Vulkan Compute | SPIR-V 直出 | 说明 | +|------|----------------|-------------|------| +| **Windows** | ✅ | ✅ | 原生支持 | +| **Linux** | ✅ | ✅ | Mesa/NVIDIA/AMD 驱动 | +| **Android (Termux)** | ✅ | ✅ | 7.0+ 系统级 Vulkan，compute 无需窗口 | + +**不支持**：macOS/iOS（Apple 不提供 Vulkan，不纳入跨平台范围）。 + +SPIR-V 二进制在三个平台间**完全共享**，零改动。GPU 驱动直接消费 SPIR-V， +无需任何平台特定的 shader 编译器。 + +## 十六、与现有代码的关系 + +> 本表反映设计目标。当前实现状态：CPU 路径已落地（[algebra/](../include/neuralnet.cpp/algebra/) 下的 `expr.hpp` / `ops.hpp` / `span.hpp` / `compute_dispatch.hpp`），GPU 路径尚未开始。 + +| 现有组件 | 改动 | +|---------|------| +| [nn_config.hpp](../include/neuralnet.cpp/nn_config.hpp) (SmartPolicy) | 不变，CPU 并行基础设施复用 | +| [matrix.hpp](../include/neuralnet.cpp/algebra/matrix.hpp) (Matrix) | 不变，纯数据容器；通过 `span()` 暴露给 AST 入口 | +| [layer.hpp](../include/neuralnet.cpp/layer.hpp) (Layer) | 直接调用 `compute::apply(span, expr)` 表达 ReLU/GeLU 等逐元素算法 | +| [compute_dispatch.hpp](../include/neuralnet.cpp/algebra/compute_dispatch.hpp) | 已模板化 `apply(Span, Expr)`；GPU 分支待补 | +| `vk_backend.hpp` (GpuBackend) | 待新增：`elementwise_from_spv(span, spv_bytes)` | +| **待新增** `spv_builder.hpp` | SPIR-V 二进制生成器（consteval，编译期） | +| [ops.hpp](../include/neuralnet.cpp/algebra/ops.hpp) | 已存在：CPU `apply`；待补每个 Op 的 `spv_op` | +| [expr.hpp](../include/neuralnet.cpp/algebra/expr.hpp) | 已存在：AST 节点类型系统；待补 `to_spv()` / `has_spv` | + +> 旧设计中提到的 `elementwise.comp` 预编译快速路径已废弃，统一由 AST → SPIR-V 直出。 diff --git a/DEVELOPMENT_STANDARDS.md b/docs/DEVELOPMENT_STANDARDS.md similarity index 56% rename from DEVELOPMENT_STANDARDS.md rename to docs/DEVELOPMENT_STANDARDS.md index 35e7232e..2b4d093a 100644 --- a/DEVELOPMENT_STANDARDS.md +++ b/docs/DEVELOPMENT_STANDARDS.md @@ -8,12 +8,13 @@ 1. [核心原则](#-核心原则) 2. [内存管理规范](#-内存管理规范) -3. [模块化设计规范](#-模块化设计规范) -4. [高性能编程规范](#-高性能编程规范) -5. [C++ 标准跟进规范](#-c-标准跟进规范) -6. [代码风格规范](#-代码风格规范) -7. [错误处理规范](#-错误处理规范) -8. [文档与注释规范](#-文档与注释规范) +3. [模块隔离规范](#-模块隔离规范) +4. [模块化设计规范](#-模块化设计规范) +5. [高性能编程规范](#-高性能编程规范) +6. [C++ 标准跟进规范](#-c-标准跟进规范) +7. [代码风格规范](#-代码风格规范) +8. [错误处理规范](#-错误处理规范) +9. [文档与注释规范](#-文档与注释规范) --- @@ -50,6 +51,62 @@ model.train(dataset, config); ### 4. 紧跟最新标准 **目标：** 始终使用最新的 C++ 标准（当前：C++26）。 +### 5. 模块隔离与最小化依赖 +**目标：** 每个模块只通过简洁的公有接口对外提供服务。修改某个模块的内部实现（如 Matrix 的存储格式、ThreadPool 的调度策略）不应引发其他模块的连锁修改。即**改一个模块只需改一个头文件**。 + +```cpp +// ❌ 违反隔离：上层模块直接穿透接口、访问底层数据结构 +// layer.hpp 中： +auto m_span = matrix.span(); // 泄露了内部 std::span 视图 +matrix.data().begin(); // 泄露了内部 std::vector& + +// ✅ 遵循隔离：上层模块只调用 Matrix 暴露的语义化操作 +matrix.apply_relu(); // Matrix 自己封装逐元素操作 +matrix.add_inplace(other); // Matrix 自己封装算术操作 +// 并行策略由 Matrix 内部决定，上层不感知 +``` + +**核心规则：** + +| 规则 | 说明 | +|------|------| +| **不穿透接口** | 上层模块不应直接访问下层模块的底层数据结构（如 `.data()`, `.span()`） | +| **不假设实现** | 上层模块不应依赖下层模块的内部类型（如假定数据存储在 `std::vector` 中） | +| **自包含修改** | 修改模块内部实现（如换存储格式、换并行策略），只需修改该模块的头文件 | + +### 6. 分层职责单一 +**目标：** 每一层只做自己该做的事，严禁职责越界。 + +> **核心口号：每层只能负责每层的事，Matrix 不能写算法，Layer 不能写底层计算。** + +```cpp +// ── L1 代数层（matrix.hpp）的职责边界 ── +// ✅ Matrix 负责：数学运算原语（矩阵乘、逐元素变换、broadcast add、归约）、SmartPolicy 自动并行分派 +// ❌ Matrix 禁止：定义神经网络算法逻辑（如 forward/backward 流程、损失函数、梯度更新策略、 +// ReLU/GeLU/Softmax/LayerNorm/CrossEntropy/Adam/SGD 等具体算法） + +// ── L2 计算层（layer.hpp / loss.hpp / optimizer.hpp）的职责边界 ── +// ✅ Layer 负责：神经网络算法逻辑（forward/backward 公式、参数管理、缓存中间结果） +// ✅ Layer 可以：组合使用其他 Layer（如 MultiHeadAttention 内部使用 Softmax、FeedForward 内部使用 Linear+GeLU） +// ✅ Layer 可以：通过 Matrix::span() 取得 Span 后调用 compute::apply(span, expr) 表达逐元素算法 + +// ── 正确示例：Layer 通过 AST 表达逐元素算法 ── +// layer.hpp 中 ReLU::forward： +Span x = result.span(); +compute::apply(x, max(x, Scalar{0})); // ✅ 直接调用 AST 入口，底层自动并行 + +// ── 正确示例：Layer 之间组合 ── +// layer.hpp 中 MultiHeadAttention::forward： +auto sm_res = softmax_.forward(attn_[h]); // ✅ Layer 调用 Layer +``` + +| 规则 | 说明 | +|------|------| +| **Matrix 不写算法** | Matrix 提供纯数学运算原语，不包含神经网络的前向/反向传播逻辑、激活函数、损失函数、优化器等具体算法 | +| **Layer 表达算法** | Layer 通过 Matrix 语义 API 或 `compute::apply(span, expr)` 表达算法 | +| **Layer 可组合** | Layer 可以包含并调用其他 Layer（如 Attention 内部使用 Softmax） | +| **并行对上层透明** | 并行策略是 SmartPolicy 的内部实现细节，上层通过 Matrix API 或 compute::apply 间接享受并行加速 | + --- ## 🧠 内存管理规范 @@ -143,22 +200,227 @@ void multiply_to(Matrix& result, const Matrix& other) const { --- +## 🔒 模块隔离规范 + +> 本章节是「核心原则 5. 模块隔离与最小化依赖」的具体化规则，**必须严格遵守**。 + +### 1. 模块架构 + +```mermaid +graph TB + subgraph "L5 交互层" + SRC["src/*.cpp"] + end + subgraph "L4 构建层" + GPT["domain_gpt.hpp"] + MNIST["domain_mnist.hpp"] + end + subgraph "L3 实现层" + MDL["model_container.hpp"] + MS["model_spec.hpp"] + IO["model_serialization.hpp"] + end + subgraph "L2 计算层" + LAY["compute_layer.hpp"] + LOSS["compute_loss.hpp"] + OPT["compute_optimizer.hpp"] + end + subgraph "L1 代数层" + MAT["algebra_matrix.hpp"] + ALG["algebra_expr/ops/span/compute.hpp"] + end + subgraph "L0 硬件层" + CFG["config.hpp"] + TP["core_threadpool.hpp"] + end + + SRC -->|"构建/训练/推理"| GPT & MNIST + SRC -->|"直接使用 Model"| MDL + GPT & MNIST -->|"model.add_gpt_model()"| MDL + GPT & MNIST -->|"ModelSpec"| MS + MDL -->|"Layer::forward()"| LAY + IO --> MDL & MS + LAY -->|"Matrix API / compute::apply"| MAT & ALG + LOSS -->|"Matrix API"| MAT + OPT -->|"Matrix API"| MAT + MAT -->|"SmartPolicy"| CFG + ALG -->|"SmartPolicy::for_each"| CFG + CFG --> TP +``` + +| 层级 | 文件 | 职责 | +|------|------|------| +| **L5 交互层** | `src/*.cpp` | 用户入口 | +| **L4 构建层** | `domain_gpt.hpp`, `domain_mnist.hpp` | 模型工厂（组装层为模型） | +| **L3 实现层** | `model_container.hpp`, `model_spec.hpp`, `model_serialization.hpp` | 模型容器 + 序列化 | +| **L2 计算层** | `compute_layer.hpp`, `compute_loss.hpp`, `compute_optimizer.hpp` | 层/损失/优化器定义（**算法所在层**） | +| **L1 代数层** | `algebra_matrix.hpp`, `algebra_*.hpp` | 矩阵运算原语 + AST 表达式模板（**不含任何算法**） | +| **L0 硬件层** | `config.hpp`, `core_threadpool.hpp` | 并行策略 | + +> 层级仅供参考，不强制限制调用方向。**唯一硬约束：Matrix 不得包含任何神经网络算法**（ReLU/GeLU/Softmax/LayerNorm/CrossEntropy/Adam/SGD 等必须放在 L2 计算层）。 + +### 2. 禁止的反模式 + +#### 2.1 上层直接操作底层数据结构 + +```cpp +// ❌ 禁止：Optimizer 直接获取 Matrix 的底层 vector 并迭代 +auto &p_vec = p.data(); // 泄露 std::vector&！ +auto &g_vec = g.data(); +SmartPolicy::for_each(zip_view.begin(), zip_view.end(), [...]); + +// ✅ 正确：通过 Matrix 语义化操作（apply / binary_apply_inplace）表达算法 +p.binary_apply_inplace(g, + [lr](Scalar pv, Scalar gv) noexcept { return pv - lr * gv; }); +``` + +#### 2.2 Matrix 中写入神经网络算法 + +```cpp +// ❌ 禁止：Matrix 内部定义 ReLU/GeLU/Softmax 等算法 +class Matrix { + void relu_inplace() { /* ... */ } // 算法不应在 Matrix + void adam_update(/*...*/) { /* ... */ } // 算法不应在 Matrix + static void softmax_rows(/*...*/) { /* ... */ } // 算法不应在 Matrix +}; + +// ✅ 正确：Matrix 只提供通用数学原语，算法在 Layer / Optimizer / Loss 中表达 +// layer.hpp 中 ReLU::forward： +Span x = result.span(); +compute::apply(x, max(x, Scalar{0})); + +// optimizer.hpp 中 SGD::step： +p.binary_apply_inplace(g, + [lr](Scalar pv, Scalar gv) noexcept { return pv - lr * gv; }); +``` + +### 3. 推荐的接口设计 + +#### 3.1 Matrix 暴露的通用数学原语 + +```cpp +class Matrix { +public: + // ── 逐元素变换 ── + template [[nodiscard]] Matrix apply(F&& func) const; + template [[nodiscard]] Matrix binary_apply(const Matrix& other, F&& func) const; + template void binary_apply_inplace(const Matrix& other, F&& func); + + // ── 归约原语 ── + template [[nodiscard]] T reduce(T init, R&&, F&&) const; + template [[nodiscard]] Matrix row_reduce(T init, R&&, F&&) const; + template [[nodiscard]] Matrix col_reduce(T init, R&&, F&&) const; + + // ── 广播原语 ── + template void broadcast_row_inplace(const Matrix& row_vec, F&& op); + template void broadcast_col_inplace(const Matrix& col_vec, F&& op); + void add_bias_broadcast_inplace(const Matrix& bias); + + // ── Span 视图（供 compute::apply 使用） ── + std::span span() noexcept; + std::span span() const; +}; +``` + +#### 3.2 Layer 通过 AST 表达逐元素算法 + +```cpp +// ✅ ReLU::forward 通过 AST 入口 +Result ReLU::forward(const Matrix& input) override { + input_cache_ = input; + Matrix result = input; + Span x = result.span(); + compute::apply(x, max(x, Scalar{0})); + return result; +} +``` + +#### 3.3 Layer 之间可自由组合 + +```cpp +// ✅ MultiHeadAttention 内部使用 Softmax Layer +class MultiHeadAttention : public Layer { + Softmax softmax_; // 组合其他 Layer + // ... + auto sm_res = softmax_.forward(attn_[h]); +}; + +// ✅ FeedForward 组合 Linear + GeLU +class FeedForward : public Layer { + Linear fc1_; + Linear fc2_; + GeLU activation_; + // ... +}; +``` + +#### 3.4 Optimizer 通过 Matrix 通用接口表达更新公式 + +```cpp +// ✅ Adam::step 通过 binary_apply_inplace 表达 +m.binary_apply_inplace(g, + [beta1, one_minus_beta1](Scalar mv, Scalar gv) noexcept { + return beta1 * mv + one_minus_beta1 * gv; + }); +``` + +### 4. 模块化自检清单 + +修改以下模块时，受影响的文件应**不超过 2 个**： + +| 修改场景 | 预期只改 | 当前状态 | +|----------|----------|----------| +| Matrix 换存储格式（如 `vector` → 自定义 allocator） | `algebra_matrix.hpp` | ✅ 已满足（上层只通过 `span()` / 语义 API 访问） | +| SmartPolicy 换并行策略（如线程池 → TBB） | `config.hpp` | ✅ 已满足（上层不直接调 SmartPolicy） | +| Layer 新增一种激活函数 | `compute_layer.hpp` | ✅ 已满足 | +| Optimizer 新增一种优化器 | `compute_optimizer.hpp` | ✅ 已满足 | +| 新增一种模型架构 | 新增 `domain_xxx.hpp` (L4) | ✅ 已满足 | +| 修改 GPT 超参数默认值 | `domain_gpt.hpp` | ✅ 已满足 | +| 修改 MNIST 数据集路径 | `domain_mnist.hpp` | ✅ 已满足 | + +--- + ## 🏗️ 模块化设计规范 -### 1. 目录结构 +### 1. 目录结构（扁平化） ``` include/neuralnet.cpp/ -├── nn_config.hpp # 全局配置和常量 -├── thread_pool.hpp # 线程池实现 -├── matrix.hpp # 核心矩阵类 -├── layer.hpp # 层基类和实现 -├── loss.hpp # 损失函数 -├── optimizer.hpp # 优化器 -├── model.hpp # 模型容器 -├── model_io.hpp # 模型序列化 -└── nn.hpp # 统一入口头文件 -``` +│ +│ ┌─ L0 硬件层 ──────────────────────────────────────┐ +├── config.hpp # 全局配置、Scalar 类型、SmartPolicy（仅 CPU） +├── core_threadpool.hpp # 线程池实现 +├── core_errors.hpp # Result = std::expected +├── core_assert.hpp # 断言宏 +│ +│ ┌─ L1 代数层 ──────────────────────────────────────┐ +├── algebra_matrix.hpp # 矩阵运算（内部自动并行分派，上层无感） +├── algebra_expr.hpp # 表达式模板 +├── algebra_ops.hpp # 逐元素算子 +├── algebra_span.hpp # Span 抽象 +├── algebra_compute.hpp # 计算分派（compute::apply 统一入口） +│ +│ ┌─ L2 计算层 ──────────────────────────────────────┐ +├── compute_layer.hpp # 层基类和实现（Linear/ReLU/GeLU/GPT...） +├── compute_loss.hpp # 损失函数（MSE/CrossEntropy） +├── compute_optimizer.hpp # 优化器（SGD/Adam） +│ +│ ┌─ L3 实现层 ──────────────────────────────────────┐ +├── model_container.hpp # 模型容器（Layer 序列 + 前后传播 + 非模板工厂 API） +├── model_spec.hpp # 模型架构描述（ModelSpec 纯数据，无 L2 依赖） +├── model_serialization.hpp # 模型二进制序列化 +│ +│ ┌─ L4 构建层 ──────────────────────────────────────┐ +├── domain_mnist.hpp # MNIST 模型工厂（构建函数 + 超参数常量） +├── domain_gpt.hpp # GPT 模型工厂（构建函数 + 超参数常量） +├── domain_tokenizer.hpp # 分词器（WordZip/BPE/ByteZip） +│ +│ ┌─ 统一入口 ──────────────────────────────────────┐ +└── nn.hpp # 统一入口头文件（聚合所有公共 API） +``` + +> **扁平化设计**：所有头文件放在同一目录，使用前缀命名（`core_`, `algebra_`, `compute_`, `domain_`, `model_`）区分层级。 +> 避免子目录带来的包含路径复杂性，同时保持清晰的层级划分。 ### 2. 头文件设计原则 diff --git a/docs/GPU_DESIGN_V2.md b/docs/GPU_DESIGN_V2.md new file mode 100644 index 00000000..d5474607 --- /dev/null +++ b/docs/GPU_DESIGN_V2.md @@ -0,0 +1,1026 @@ +# 🚀 GPU 加速后端设计文档 v2 + +> **版本**：2.0（重新设计） +> **目标**：为 neuralnet.cpp 添加 Vulkan Compute GPU 加速，解决旧版 GPU-resident pipeline 的正确性问题，实现真正的零层间 PCIe 传输。 + +--- + +## 📋 目录 + +1. [设计背景](#-设计背景) +2. [旧版问题回顾](#-旧版问题回顾) +3. [新架构设计](#-新架构设计) +4. [核心组件详解](#-核心组件详解) +5. [数据流与同步机制](#-数据流与同步机制) +6. [与现有代码的集成](#-与现有代码的集成) +7. [分阶段实施计划](#-分阶段实施计划) +8. [测试与验证策略](#-测试与验证策略) + +--- + +## 🎯 设计背景 + +### 项目现状 + +当前 `rewrite` 分支是纯 CPU 实现，使用 SmartPolicy 自适应并行策略。虽然 CPU 多线程性能优秀，但对于大规模矩阵运算（如 Transformer 注意力计算），GPU 的数千核心可以提供数量级的吞吐提升。 + +### 旧版 GPU 后端（`feature/vulkan-compute` 分支） + +已在历史分支中实现完整的 Vulkan 计算后端，包括： +- MemoryPool 子分配器 +- StagingRing 环形缓冲区 +- 双轨制架构（Staging Path + GPU-Resident Path） +- 三个 compute shader（matmul、matmul_tiled、elementwise） + +**但 GPU-resident path 存在正确性 bug**，多层链式推理准确率仅 ~10%（预期 95%）。 + +--- + +## 🔴 旧版问题回顾 + +### Bug 1：跨 Command Buffer 缺少 Fence 同步（致命） + +**位置**：`GpuBackend::matmul_gpu()` 和 `GpuBackend::elementwise_gpu()` + +```cpp +// 旧版代码（有 bug） +vkQueueSubmit(device_.compute_queue(), 1, &submit_info, VK_NULL_HANDLE); // 无 fence +return GpuTensor{C}; // 立即返回，GPU 可能还没算完 +``` + +**问题**： +- Vulkan 规范要求：跨 command buffer 的数据依赖必须通过 fence/semaphore 同步 +- `matmul_gpu` 提交后立即返回 `GpuTensor`，但 GPU 可能还在计算 +- 下一层 `elementwise_gpu` 立即读取未完成的数据，导致竞态条件 + +### Bug 2：Pipeline Barrier 跨 Command Buffer 无效（致命） + +**位置**：`matmul_gpu()` 中的 `vkCmdPipelineBarrier` + +```cpp +// 旧版代码（有 bug） +VkBufferMemoryBarrier barrier{}; +barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; +barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; +vkCmdPipelineBarrier(cmd, ...); // 只在当前 command buffer 内有效 +``` + +**问题**： +- `vkCmdPipelineBarrier` 的作用域**仅限于当前 command buffer** +- 不同 command buffer 提交之间的数据依赖需要 fence/semaphore +- 旧版在同一 mutex 下提交，但 GPU 端执行是异步的 + +### Bug 3：`pending_ops_` 资源清理竞态（中等） + +**位置**：`GpuBackend::flush_pending_ops()` + +**问题**： +- `matmul_gpu` 和 `elementwise_gpu` 将资源推入 `pending_ops_`，但不设置 fence +- 多层链式调用时，资源（command buffer、descriptor set）持续累积 +- 只在 `download_blocking()` 时才清理，可能导致资源耗尽 + +### Bug 4：权重缓存训练时不失效（中等） + +**位置**：`Linear::forward_gpu()` + +```cpp +// 旧版代码（有 bug） +if (!gpu_weights_) { // 懒加载，只在第一次上传 + gpu_weights_ = GpuTensor::from_matrix(W_, backend); +} +// 训练时 W_ 被 optimizer.step() 更新，但 gpu_weights_ 不会失效 +``` + +**问题**： +- 训练循环中，`optimizer.step()` 更新 CPU 端的 `W_` 和 `b_` +- 但 `gpu_weights_` 和 `gpu_bias_` 仍然是旧值 +- 下一次 `forward_gpu` 使用过期权重，导致训练不收敛 + +--- + +## 🏗️ 新架构设计 + +### 设计原则 + +1. **严格分层**：遵循 L0(硬件) → L1(代数) → L2(计算) → L3(模型) 调用规则 +2. **同步优先**：每个 GPU 操作完成后显式等待，确保正确性后再优化性能 +3. **零手动内存管理**：使用 RAII 封装所有 Vulkan 资源 +4. **渐进式集成**：先实现 Staging Path，再启用 GPU-Resident Path +5. **可测试性**：每个组件独立可测，支持 CPU/GPU 结果对比 + +### 分层架构图 + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ L5 用户入口层 │ +│ src/mnist_train.cpp, src/text_train.cpp, ... │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ L4 构建层 │ +│ domain_mnist.hpp, domain_gpt.hpp │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ L3 实现层 │ +│ model_container.hpp (Model) │ +│ - forward(): 自动选择 CPU 或 GPU-resident 路径 │ +│ - GPU 路径: Upload → [Layer1.forward_gpu → ...] → Download │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ L2 计算层 │ +│ compute_layer.hpp (Layer, Linear, ReLU, GeLU, ...) │ +│ - forward(): CPU 路径（现有） │ +│ - forward_gpu(): GPU-resident 路径（新增） │ +│ - backward(): 始终走 CPU 路径（梯度计算通常在 CPU） │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ L1 代数层 │ +│ algebra_matrix.hpp (Matrix) │ +│ - multiply_to(): 自动 dispatch CPU 或 GPU (staging path) │ +│ - apply_relu_inplace(): 自动 dispatch │ +│ - apply_gelu_inplace(): 自动 dispatch │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ L0 硬件层 │ +│ config.hpp (SmartPolicy) │ +│ core_threadpool.hpp (ThreadPool) │ +│ backend/vk_backend.hpp (GpuBackend) ← 新增 │ +│ backend/gpu_tensor.hpp (GpuTensor) ← 新增 │ +│ backend/gpu_sync.hpp (GpuOpHandle) ← 新增 │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 🧩 核心组件详解 + +### 组件 1：`GpuBackend` — Vulkan 后端单例 + +**文件**：`include/neuralnet.cpp/backend/vk_backend.hpp` + +**职责**： +- 管理 Vulkan 实例、设备、队列（RAII） +- 管理 MemoryPool 子分配器 +- 管理 StagingRing 环形缓冲区 +- 提供 GPU 操作 API（matmul、elementwise） + +**关键 API**： + +```cpp +class GpuBackend { +public: + // 单例访问 + [[nodiscard]] static GpuBackend& instance(); + + // 初始化（懒加载，首次调用时初始化） + [[nodiscard]] Result initialize(); + + // 阻塞式上传：CPU → GPU + [[nodiscard]] Result upload_blocking( + GpuTensor& dst, std::span cpu_data); + + // 阻塞式下载：GPU → CPU + [[nodiscard]] Result download_blocking( + const GpuTensor& src, std::span cpu_data); + + // 纯 GPU 矩阵乘法（阻塞等待完成） + [[nodiscard]] Result matmul_gpu( + const GpuTensor& A, const GpuTensor& B); + + // 纯 GPU 逐元素运算（阻塞等待完成） + [[nodiscard]] Result elementwise_gpu( + const GpuTensor& primary, const GpuTensor* secondary, + uint32_t op, uint32_t rows = 0, uint32_t cols = 0); + + // Staging Path：CPU span → GPU → 计算 → GPU → CPU span + [[nodiscard]] Result matmul_direct( + std::span a, std::span b, + std::span c, std::size_t M, std::size_t N, std::size_t K); + + // 查询状态 + [[nodiscard]] bool is_initialized() const noexcept; + [[nodiscard】 bool gpu_available() const noexcept; +}; +``` + +**同步机制（修复 Bug 1 & 2）**： + +```cpp +Result GpuBackend::matmul_gpu(const GpuTensor& A, const GpuTensor& B) { + // ... 录制 command buffer ... + + // 1. 创建 fence + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + VkFence fence = VK_NULL_HANDLE; + vkCreateFence(device_.device(), &fence_info, nullptr, &fence); + + // 2. 提交到队列，附带 fence + VkSubmitInfo submit_info{}; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd; + { + std::lock_guard lock(queue_mutex_); + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, fence); + } + + // 3. 等待 GPU 完成（阻塞） + vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, UINT64_MAX); + vkDestroyFence(device_.device(), 1, &fence); + + // 4. 清理资源 + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + + return C; // 确保 GPU 计算已完成 +} +``` + +### 组件 2：`GpuTensor` — GPU 矩阵抽象 + +**文件**：`include/neuralnet.cpp/backend/gpu_tensor.hpp` + +**职责**： +- 封装 GPU 缓冲区（`GpuBuffer`） +- 提供形状信息（rows, cols） +- 支持与 `Matrix` 的转换 + +**关键 API**： + +```cpp +class GpuTensor { +private: + std::shared_ptr buffer_; + std::size_t rows_; + std::size_t cols_; + +public: + GpuTensor() = default; + + // 从 CPU Matrix 创建（上传数据） + [[nodiscard]] static Result from_matrix( + const Matrix& cpu_mat, GpuBackend& backend); + + // 创建空的 GPU Tensor（用于输出） + [[nodiscard]] static Result create_empty( + std::size_t rows, std::size_t cols, GpuBackend& backend); + + // 转换为 CPU Matrix（下载数据） + [[nodiscard]] Result to_matrix(GpuBackend& backend) const; + + // 访问器 + [[nodiscard]] std::size_t rows() const noexcept; + [[nodiscard】 std::size_t cols() const noexcept; + [[nodiscard]] bool valid() const noexcept; + [[nodiscard]] const GpuBuffer& buffer() const noexcept; +}; +``` + +### 组件 3：`GpuOpHandle` — GPU 操作句柄 + +**文件**：`include/neuralnet.cpp/backend/gpu_sync.hpp` + +**职责**： +- 封装 fence 和相关资源 +- 支持 wait + cleanup 语义 +- 用于需要延迟等待的场景（未来优化） + +**关键 API**： + +```cpp +class GpuOpHandle { +private: + VkDevice device_ = VK_NULL_HANDLE; + VkFence fence_ = VK_NULL_HANDLE; + std::vector cmd_buffers_; + std::vector desc_sets_; + VkDescriptorPool desc_pool_ = VK_NULL_HANDLE; + VkCommandPool cmd_pool_ = VK_NULL_HANDLE; + +public: + GpuOpHandle() = default; + ~GpuOpHandle(); + + // 等待 GPU 完成并清理资源 + void wait_and_cleanup(); + + // 检查是否已完成（非阻塞） + [[nodiscard]] bool is_ready() const; + + // 移动语义（支持返回值优化） + GpuOpHandle(GpuOpHandle&& other) noexcept; + GpuOpHandle& operator=(GpuOpHandle&& other) noexcept; + + // 禁止拷贝 + GpuOpHandle(const GpuOpHandle&) = delete; + GpuOpHandle& operator=(const GpuOpHandle&) = delete; +}; +``` + +### 组件 4：`MemoryPool` — GPU 内存子分配器 + +**文件**：`include/neuralnet.cpp/backend/memory_pool.hpp` + +**职责**： +- 预分配大块 GPU 内存（128MB+） +- 提供 O(log n) 子分配/释放 +- 自动合并空闲区域 + +**关键 API**： + +```cpp +class MemoryPool { +public: + static constexpr VkDeviceSize DEFAULT_BLOCK_SIZE = 128ull * 1024 * 1024; + + struct Allocation { + VkDeviceMemory memory = VK_NULL_HANDLE; + VkDeviceSize offset = 0; + VkDeviceSize size = 0; + VkMemoryPropertyFlags property_flags = 0; + [[nodiscard]] bool valid() const noexcept; + }; + + MemoryPool(VkDevice device, VkPhysicalDevice physical_device, + VkDeviceSize block_size = DEFAULT_BLOCK_SIZE); + ~MemoryPool(); + + // 分配内存 + [[nodiscard]] Result allocate( + VkMemoryRequirements requirements, + VkMemoryPropertyFlags preferred_flags, + VkMemoryPropertyFlags fallback_flags = 0); + + // 释放内存 + void free(const Allocation& alloc); +}; +``` + +### 组件 5：`StagingRing` — 环形 Staging 缓冲区 + +**文件**：`include/neuralnet.cpp/backend/staging_ring.hpp` + +**职责**： +- 管理 HOST_VISIBLE 缓冲区用于 CPU↔GPU 数据传输 +- 环形分配，避免频繁创建/销毁 +- Fence 同步确保数据安全 + +**关键 API**： + +```cpp +class StagingRing { +public: + static constexpr std::size_t DEFAULT_REGION_SIZE = 256ull * 1024 * 1024; + static constexpr std::size_t DEFAULT_NUM_REGIONS = 4; + + StagingRing() = default; + ~StagingRing(); + + // 初始化 + [[nodiscard]] Result initialize( + VkDevice device, VkPhysicalDevice physical_device, + VkCommandPool cmd_pool, MemoryPool* pool); + + // 获取一个 staging region + [[nodiscard]] std::size_t acquire(); + + // 上传数据到 staging region + [[nodiscard]] Result upload( + std::size_t region_idx, std::span data, VkDeviceSize offset); + + // 从 staging region 下载数据 + [[nodiscard]] Result download( + std::size_t region_idx, std::span data, VkDeviceSize offset); + + // 获取缓冲区句柄 + [[nodiscard]] VkBuffer buffer(std::size_t region_idx) const noexcept; + + // 获取 fence + [[nodiscard]] VkFence fence(std::size_t region_idx) const noexcept; +}; +``` + +### 组件 6：Compute Shaders + +**文件**：`shaders/matmul.comp`, `shaders/matmul_tiled.comp`, `shaders/elementwise.comp` + +**matmul.comp** — 16×16 分块矩阵乘法： +- WorkGroup: 16×16 (256 线程) +- 每个 WorkGroup 计算 16×16 输出块 +- Shared Memory 缓存 A 和 B 的分块 +- 多轮迭代 K 维度 + +**matmul_tiled.comp** — 4×4 粗化分块矩阵乘法： +- WorkGroup: 16×16 (256 线程) +- 每个 WorkGroup 计算 64×64 输出块（每线程 4×4 元素） +- 寄存器分块，大幅提升计算访存比 +- Shared Memory Padding 避免 Bank Conflict + +**elementwise.comp** — 逐元素运算： +- WorkGroup: 256 线程 +- 支持 ReLU (op=0), QuickGeLU (op=1), BiasAdd (op=2) +- Push Constants 选择操作类型 + +--- + +## 🔄 数据流与同步机制 + +### Staging Path（自动 fallback） + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Staging Path 数据流 │ +│ │ +│ CPU Matrix A ──┐ │ +│ │ Upload (memcpy to staging) │ +│ ▼ │ +│ ┌─────────────────────────────────────────┐ │ +│ │ Staging Ring (HOST_VISIBLE) │ │ +│ └─────────────────────────────────────────┘ │ +│ │ Copy (vkCmdCopyBuffer) │ +│ ▼ │ +│ ┌─────────────────────────────────────────┐ │ +│ │ GPU Buffer A (DEVICE_LOCAL) │ │ +│ └─────────────────────────────────────────┘ │ +│ │ Compute (vkCmdDispatch) │ +│ ▼ │ +│ ┌─────────────────────────────────────────┐ │ +│ │ GPU Buffer C (DEVICE_LOCAL) │ │ +│ └─────────────────────────────────────────┘ │ +│ │ Copy (vkCmdCopyBuffer) │ +│ ▼ │ +│ ┌─────────────────────────────────────────┐ │ +│ │ Staging Ring (HOST_VISIBLE) │ │ +│ └─────────────────────────────────────────┘ │ +│ │ Download (memcpy from staging) │ +│ ▼ │ +│ CPU Matrix C ◄─┘ │ +│ │ +│ 同步点：vkWaitForFences (每次操作完成后) │ +└─────────────────────────────────────────────────────────────────┘ +``` + +**特点**： +- 每层都有 CPU↔GPU 传输 +- 每次操作都阻塞等待完成 +- 简单、正确、易于调试 +- 适用于小矩阵或调试阶段 + +### GPU-Resident Path（零层间 PCIe） + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ GPU-Resident Path 数据流 │ +│ │ +│ CPU Input ──┐ │ +│ │ Upload (仅一次，输入层) │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────┐ │ +│ │ GPU VRAM │ │ +│ │ │ │ +│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │ +│ │ │ Layer 1 │───►│ Layer 2 │───►│ Layer 3 │ │ │ +│ │ │ matmul+relu │ │ matmul+gelu │ │ matmul │ │ │ +│ │ └─────────────┘ └─────────────┘ └─────────────┘ │ │ +│ │ │ │ │ │ │ +│ │ ▼ ▼ ▼ │ │ +│ │ GpuTensor GpuTensor GpuTensor │ │ +│ │ │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ │ │ +│ │ Download (仅一次，输出层) │ +│ ▼ │ +│ CPU Output ◄─┘ │ +│ │ +│ 同步点：每层 matmul_gpu/elementwise_gpu 完成后 fence wait │ +└─────────────────────────────────────────────────────────────────┘ +``` + +**特点**： +- 只在输入/输出各一次 PCIe 传输 +- 中间层全部在 GPU 显存流转 +- 每层操作完成后 fence wait，确保数据一致性 +- 适用于大矩阵、多层网络 + +### 同步机制详解 + +**核心原则**：每个 `matmul_gpu` 或 `elementwise_gpu` 调用都必须： + +1. **录制 command buffer** +2. **创建 fence** +3. **提交到队列**（带 fence） +4. **等待 fence**（阻塞） +5. **销毁 fence** +6. **清理资源**（command buffer、descriptor set） + +```cpp +// 完整的同步流程 +Result GpuBackend::matmul_gpu(const GpuTensor& A, const GpuTensor& B) { + // 1. 分配输出 Tensor + auto C = GpuTensor::create_empty(M, N, *this); + + // 2. 录制 command buffer + VkCommandBuffer cmd = ...; + vkBeginCommandBuffer(cmd, &begin_info); + + // 3. 添加 pipeline barrier（确保输入数据就绪） + VkBufferMemoryBarrier input_barrier{}; + input_barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + input_barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + vkCmdPipelineBarrier(cmd, ...); + + // 4. 绑定 pipeline 和 descriptor set + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); + vkCmdBindDescriptorSets(cmd, ...); + + // 5. 推送常量 + vkCmdPushConstants(cmd, ...); + + // 6. Dispatch + vkCmdDispatch(cmd, (N + 15) / 16, (M + 15) / 16, 1); + + // 7. 添加输出 barrier（确保写入完成） + VkBufferMemoryBarrier output_barrier{}; + output_barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + output_barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + vkCmdPipelineBarrier(cmd, ...); + + // 8. 结束录制 + vkEndCommandBuffer(cmd); + + // 9. 创建 fence + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + VkFence fence = VK_NULL_HANDLE; + vkCreateFence(device_.device(), &fence_info, nullptr, &fence); + + // 10. 提交到队列 + VkSubmitInfo submit_info{}; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd; + { + std::lock_guard lock(queue_mutex_); + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, fence); + } + + // 11. 等待完成 + vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, UINT64_MAX); + + // 12. 清理 + vkDestroyFence(device_.device(), 1, &fence); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + + return C; // 确保 GPU 计算已完成 +} +``` + +--- + +## 🔗 与现有代码的集成 + +### 集成点 1：`config.hpp` — 添加 GPU 配置 + +```cpp +struct SmartPolicy { + // 现有代码... + + // GPU 加速阈值：矩阵面积超过此值时自动走 GPU + inline static constexpr std::size_t GPU_THRESHOLD = 64 * 64; // 4096 元素 + + // 是否启用 GPU 后端（运行时开关） + inline static bool gpu_enabled = false; + + // GPU 操作计数（性能统计） + inline static std::atomic gpu_matmul_count{0}; + inline static std::atomic cpu_matmul_count{0}; +}; +``` + +### 集成点 2：`algebra_matrix.hpp` — Matrix 类扩展 + +```cpp +class Matrix { +private: + std::vector data_{}; + std::size_t rows_{0}; + std::size_t cols_{0}; + + // 现有代码... + +public: + // 现有 multiply_to 实现... + void multiply_to(Matrix& result, const Matrix& other) const { + const std::size_t M = rows_; + const std::size_t K = cols_; + const std::size_t N = other.cols_; + +#ifdef NN_HAS_VULKAN + // GPU 加速路径（Staging Path） + if (SmartPolicy::gpu_enabled && M * N >= SmartPolicy::GPU_THRESHOLD) { + auto& backend = GpuBackend::instance(); + if (backend.is_initialized() || backend.initialize()) { + auto mm = backend.matmul_direct( + span(), other.span(), result.span(), M, N, K); + if (mm) { + SmartPolicy::gpu_matmul_count.fetch_add(1, std::memory_order_relaxed); + return; + } + } + // GPU 失败，静默 fallback 到 CPU 路径 + } + SmartPolicy::cpu_matmul_count.fetch_add(1, std::memory_order_relaxed); +#endif + + // CPU 路径（现有实现） + // ... + } + + // 新增：GPU-resident 版本（供 Layer::forward_gpu 使用） +#ifdef NN_HAS_VULKAN + [[nodiscard]] Result to_gpu_tensor(GpuBackend& backend) const { + return GpuTensor::from_matrix(*this, backend); + } +#endif +}; +``` + +### 集成点 3：`compute_layer.hpp` — Layer 类扩展 + +```cpp +class Layer { +public: + // 现有接口... + virtual Result forward(const Matrix& input) = 0; + virtual Result backward(const Matrix& grad_output) = 0; + +#ifdef NN_HAS_VULKAN + // 新增：GPU-resident 前向传播 + // 默认实现：Download → CPU 计算 → Upload（fallback） + virtual Result forward_gpu(const GpuTensor& input, GpuBackend& backend) { + // Fallback：Download → CPU 计算 → Upload + auto cpu_in = input.to_matrix(backend); + if (!cpu_in) return std::unexpected(cpu_in.error()); + + auto cpu_out = forward(*cpu_in); + if (!cpu_out) return std::unexpected(cpu_out.error()); + + return GpuTensor::from_matrix(*cpu_out, backend); + } + + // 新增：使 GPU 权重缓存失效（训练时调用） + virtual void invalidate_gpu_cache() {} +#endif +}; + +class Linear final : public Layer { +private: + Matrix w_; + Matrix b_; + // ... 现有成员 ... + +#ifdef NN_HAS_VULKAN + std::optional gpu_weights_; + std::optional gpu_bias_; +#endif + +public: + // 现有 forward 实现... + +#ifdef NN_HAS_VULKAN + Result forward_gpu(const GpuTensor& input, GpuBackend& backend) override { + // 1. 上传权重（每次都上传，确保训练时权重最新） + // 优化：可以用 dirty flag 判断是否需要重新上传 + gpu_weights_.reset(); + gpu_bias_.reset(); + + auto w_res = GpuTensor::from_matrix(w_, backend); + if (!w_res) return std::unexpected(w_res.error()); + gpu_weights_ = std::move(*w_res); + + auto b_res = GpuTensor::from_matrix(b_, backend); + if (!b_res) return std::unexpected(b_res.error()); + gpu_bias_ = std::move(*b_res); + + // 2. 纯 GPU 矩阵乘法（阻塞等待） + auto mm_res = backend.matmul_gpu(input, *gpu_weights_); + if (!mm_res) return std::unexpected(mm_res.error()); + + // 3. GPU 端 Bias Add（阻塞等待） + auto ba_res = backend.elementwise_gpu( + *mm_res, &*gpu_bias_, 2u, + static_cast(w_.rows()), + static_cast(input.cols())); + if (!ba_res) return std::unexpected(ba_res.error()); + + return ba_res; + } + + void invalidate_gpu_cache() override { + gpu_weights_.reset(); + gpu_bias_.reset(); + } +#endif +}; + +class ReLU final : public Layer { +public: + // 现有 forward 实现... + +#ifdef NN_HAS_VULKAN + Result forward_gpu(const GpuTensor& input, GpuBackend& backend) override { + return backend.elementwise_gpu(input, nullptr, 0u); // op=0: ReLU + } +#endif +}; + +class GeLU final : public Layer { +public: + // 现有 forward 实现... + +#ifdef NN_HAS_VULKAN + Result forward_gpu(const GpuTensor& input, GpuBackend& backend) override { + return backend.elementwise_gpu(input, nullptr, 1u); // op=1: QuickGeLU + } +#endif +}; +``` + +### 集成点 4：`model_container.hpp` — Model 类扩展 + +```cpp +class Model { +private: + std::vector> layers_; + +public: + // 现有 forward 实现... + [[nodiscard]] Result forward(const Matrix& input) { + if (layers_.empty()) { + return std::unexpected(Error{"Model has no layers"}); + } + +#ifdef NN_HAS_VULKAN + // GPU-resident 路径 + if (SmartPolicy::gpu_enabled) { + auto& backend = GpuBackend::instance(); + if (backend.is_initialized() || backend.initialize()) { + // 1. 上传输入到 GPU（唯一一次 CPU→GPU） + auto gpu_in = GpuTensor::from_matrix(input, backend); + if (gpu_in) { + // 2. 逐层 forward_gpu（全程 GPU 显存流转） + GpuTensor gpu_out = std::move(*gpu_in); + bool gpu_ok = true; + + for (std::size_t i = 0; i < layers_.size(); ++i) { + auto layer_res = layers_[i]->forward_gpu(gpu_out, backend); + if (!layer_res) { + gpu_ok = false; + break; + } + gpu_out = std::move(*layer_res); + } + + if (gpu_ok) { + // 3. 下载输出到 CPU（唯一一次 GPU→CPU） + auto cpu_out = gpu_out.to_matrix(backend); + if (cpu_out) return cpu_out; + } + } + // GPU 路径失败，静默 fallback 到 CPU 路径 + } + } +#endif + + // CPU 路径（现有实现） + auto result = layers_.front()->forward(input); + if (!result) return std::unexpected(result.error()); + Matrix out = std::move(*result); + + for (std::size_t i = 1; i < layers_.size(); ++i) { + result = layers_[i]->forward(out); + if (!result) return std::unexpected(result.error()); + out = std::move(*result); + } + + return out; + } +}; +``` + +### 集成点 5：`compute_optimizer.hpp` — 优化器扩展 + +```cpp +class Adam final : public Optimizer { +public: + // 现有 step 实现... + Result step() override { + // ... 更新参数 ... + +#ifdef NN_HAS_VULKAN + // 使所有层的 GPU 权重缓存失效 + // 注意：需要 Model 提供访问 layers 的接口 + // 或者在 Optimizer 中维护一个 Layer* 列表 + for (auto* layer : layers_) { + layer->invalidate_gpu_cache(); + } +#endif + + return {}; + } +}; +``` + +### 集成点 6：`CMakeLists.txt` — 构建系统扩展 + +```cmake +# ── Vulkan GPU 加速（可选） ──────────────────────────────────────── +find_package(Vulkan QUIET) +if(Vulkan_FOUND) + message(STATUS "Vulkan found: ${Vulkan_LIBRARY}") + + # 查找 SPIR-V 编译器 + find_program(GLSLC glslc HINTS ${Vulkan_GLSLC_EXECUTABLE}) + + # 定义着色器列表 + set(GPU_SHADERS matmul matmul_tiled elementwise) + + foreach(SHADER_NAME ${GPU_SHADERS}) + set(SRC ${CMAKE_SOURCE_DIR}/shaders/${SHADER_NAME}.comp) + set(SPV ${CMAKE_BINARY_DIR}/${SHADER_NAME}.spv) + set(HPP ${CMAKE_BINARY_DIR}/generated/${SHADER_NAME}_spv.hpp) + set(FUNC "nn_${SHADER_NAME}_spirv_bytecode") + + # 编译 GLSL → SPIR-V + add_custom_command( + OUTPUT ${SPV} + COMMAND ${GLSLC} -o ${SPV} ${SRC} + DEPENDS ${SRC} + COMMENT "[GPU] Compiling ${SHADER_NAME}.comp → ${SHADER_NAME}.spv" + ) + + # 嵌入 SPIR-V → C++ 头文件 + add_custom_command( + OUTPUT ${HPP} + COMMAND ${CMAKE_COMMAND} + -DSPV_INPUT=${SPV} + -DHPP_OUTPUT=${HPP} + -DFUNC_NAME=${FUNC} + -P ${CMAKE_SOURCE_DIR}/cmake/embed_spirv.cmake + DEPENDS ${SPV} ${CMAKE_SOURCE_DIR}/cmake/embed_spirv.cmake + COMMENT "[GPU] Embedding ${SHADER_NAME} SPIR-V into C++ header" + ) + + list(APPEND GPU_SHADER_HPPS ${HPP}) + endforeach() + + if(NOT GLSLC) + message(WARNING "No SPIR-V compiler found (glslc). " + "GPU acceleration disabled. Install Vulkan SDK.") + set(Vulkan_FOUND OFF) + endif() + + if(Vulkan_FOUND) + add_custom_target(compile_gpu_shaders DEPENDS ${GPU_SHADER_HPPS}) + + # 为所有目标启用 Vulkan + set(NN_GPU_TARGETS + mnist_train mnist_infer + text_train text_infer + compute_bench + ) + + foreach(TARGET ${NN_GPU_TARGETS}) + target_compile_definitions(${TARGET} PRIVATE NN_HAS_VULKAN) + target_include_directories(${TARGET} PRIVATE ${CMAKE_BINARY_DIR}/generated) + target_link_libraries(${TARGET} PRIVATE Vulkan::Vulkan) + add_dependencies(${TARGET} compile_gpu_shaders) + endforeach() + + message(STATUS "GPU acceleration enabled (Vulkan compute)") + endif() +else() + message(STATUS "Vulkan not found. GPU acceleration disabled.") +endif() +``` + +--- + +## 📅 分阶段实施计划 + +### Phase 1：基础设施（1-2 周） + +| 任务 | 文件 | 说明 | +|------|------|------| +| 1.1 创建 backend 目录 | `include/neuralnet.cpp/backend/` | 新建目录结构 | +| 1.2 实现 MemoryPool | `backend/memory_pool.hpp` | GPU 内存子分配器 | +| 1.3 实现 StagingRing | `backend/staging_ring.hpp` | 环形 staging 缓冲区 | +| 1.4 实现 GpuBackend 基础 | `backend/vk_backend.hpp` | 设备初始化、队列管理 | +| 1.5 创建 shaders 目录 | `shaders/` | 三个 compute shader | +| 1.6 创建 cmake 脚本 | `cmake/embed_spirv.cmake` | SPIR-V 嵌入 | +| 1.7 修改 CMakeLists.txt | `CMakeLists.txt` | 添加 Vulkan 依赖 | + +**里程碑**：能编译通过，Vulkan 设备初始化成功 + +### Phase 2：Staging Path（1 周） + +| 任务 | 文件 | 说明 | +|------|------|------| +| 2.1 实现 upload_blocking | `backend/vk_backend.hpp` | CPU → GPU 阻塞上传 | +| 2.2 实现 download_blocking | `backend/vk_backend.hpp` | GPU → CPU 阻塞下载 | +| 2.3 实现 matmul_direct | `backend/vk_backend.hpp` | Staging path 矩阵乘法 | +| 2.4 实现 elementwise_direct | `backend/vk_backend.hpp` | Staging path 逐元素运算 | +| 2.5 修改 config.hpp | `config.hpp` | 添加 GPU 配置项 | +| 2.6 修改 algebra_matrix.hpp | `algebra_matrix.hpp` | 添加 GPU dispatch | +| 2.7 编写 gpu_test.cpp | `src/gpu_test.cpp` | 单层正确性测试 | + +**里程碑**：Staging path 工作，单层 matmul 正确 + +### Phase 3：GPU-Resident Path（1-2 周） + +| 任务 | 文件 | 说明 | +|------|------|------| +| 3.1 实现 GpuTensor | `backend/gpu_tensor.hpp` | GPU 矩阵抽象 | +| 3.2 实现 matmul_gpu | `backend/vk_backend.hpp` | 纯 GPU 矩阵乘法（带 fence） | +| 3.3 实现 elementwise_gpu | `backend/vk_backend.hpp` | 纯 GPU 逐元素运算（带 fence） | +| 3.4 修改 compute_layer.hpp | `compute_layer.hpp` | 添加 forward_gpu 接口 | +| 3.5 实现 Linear::forward_gpu | `compute_layer.hpp` | Linear 层 GPU 实现 | +| 3.6 实现 ReLU/GeLU::forward_gpu | `compute_layer.hpp` | 激活层 GPU 实现 | +| 3.7 修改 model_container.hpp | `model_container.hpp` | 添加 GPU-resident forward | + +**里程碑**：多层链式推理正确，准确率 ≥ 93% + +### Phase 4：训练支持（1 周） + +| 任务 | 文件 | 说明 | +|------|------|------| +| 4.1 实现 invalidate_gpu_cache | `compute_layer.hpp` | 权重缓存失效机制 | +| 4.2 修改 optimizer | `compute_optimizer.hpp` | step() 后使缓存失效 | +| 4.3 训练正确性测试 | `src/mnist_train.cpp` | MNIST 训练收敛测试 | + +**里程碑**：GPU-resident 训练正确收敛 + +### Phase 5：性能优化（可选，2-3 周） + +| 任务 | 说明 | +|------|------| +| 5.1 Batch submit | 多层 command buffer 一次性提交 | +| 5.2 Double buffering | 两组 command buffer 交替使用 | +| 5.3 Async compute | 多 compute queue 并行 | +| 5.4 LayerNorm/Softmax shader | 专用归约 shader | +| 5.5 权重 dirty flag | 避免每次都上传权重 | + +**里程碑**：GPU 利用率 ≥ 80%，性能提升 ≥ 5× + +--- + +## 🧪 测试与验证策略 + +### 单元测试 + +| 测试 | 输入 | 验证 | +|------|------|------| +| test_matmul_16x16 | 16×16 方阵 | `max_abs_err < 1e-5` (float) | +| test_matmul_256x256 | 256×256 方阵 | `max_abs_err < 1e-4` | +| test_matmul_4096x4096 | 4096×4096 方阵 | `max_abs_err < 1e-3` | +| test_elementwise_relu | 1000 元素 | 逐元素比较 | +| test_elementwise_gelu | 1000 元素 | `max_abs_err < 1e-6` | +| test_elementwise_biasadd | 512×64 | 行广播正确性 | + +### 集成测试 + +| 测试 | 网络 | 验证 | +|------|------|------| +| test_2layer_chain | Linear→ReLU→Linear | CPU/GPU 输出一致 | +| test_5layer_mlp | 784→512→256→128→64→10 | MNIST 准确率 ≥ 93% | +| test_gpt_inference | GPT-2 small | 生成文本语义一致 | +| test_training_loop | MNIST 10 epochs | 收敛曲线一致 | + +### 性能基准 + +| 指标 | Staging Path | GPU-Resident | 目标 | +|------|-------------|-------------|------| +| 单层 matmul (4096²) | 5ms | 0.5ms | 10× 加速 | +| 5 层 MLP 前向 | 25ms | 3ms | 8× 加速 | +| PCIe 传输量 (5 层) | 5×2×数据量 | 1×2×数据量 | 5× 减少 | +| GPU 利用率 | 93% | ≥ 80% | - | + +--- + +## 📎 参考资料 + +- [Vulkan Compute Pipeline 规范](https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkComputePipelineCreateInfo.html) +- [Vulkan 同步机制详解](https://www.khronos.org/blog/understanding-vulkan-synchronization) +- [SPIR-V Storage Buffer 规范](https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html) +- [GPU 矩阵乘法优化](https://developer.nvidia.com/blog/cutlass-linear-algebra-cuda/) +- 仓库记忆：`/memories/repo/gpu-backend.md`（v4-v6.1 设计历史） +- 修复计划：`docs/GPU_FIX_PLAN.md`（旧版 bug 分析） diff --git a/docs/GPU_FIX_PLAN.md b/docs/GPU_FIX_PLAN.md new file mode 100644 index 00000000..ae27b1a1 --- /dev/null +++ b/docs/GPU_FIX_PLAN.md @@ -0,0 +1,398 @@ +# 🔧 GPU-Resident Pipeline 修复计划 + +> **目标**：修复 v6 GPU-resident 路径的正确性 bug，使多层链式推理在 GPU 显存中全程流转，零层间 PCIe 传输。 + +--- + +## 📋 目录 + +1. [现状回顾](#-现状回顾) +2. [Bug 根因分析](#-bug-根因分析) +3. [修复方案](#-修复方案) +4. [分阶段实施计划](#-分阶段实施计划) +5. [测试策略](#-测试策略) + +--- + +## 📊 现状回顾 + +### 当前分支 (`rewrite`) 状态 + +**当前工作区是纯 CPU 实现**，没有任何 GPU 代码。GPU 后端代码存在于 `feature/vulkan-compute` 分支，但该分支经历了多次重构，最新提交已将 Vulkan 后端移除。 + +### 历史实现（`feature/vulkan-compute` 分支，commit `8456fe7`） + +已实现的架构： + +| 组件 | 文件 | 状态 | +|------|------|------| +| Vulkan RAII 后端 | `include/neuralnet.cpp/backend/vk_backend.hpp` | ✅ 已实现 | +| GpuTensor 方法实现 | `include/neuralnet.cpp/backend/gpu_tensor_impl.hpp` | ✅ 已实现 | +| matmul shader (16×16) | `shaders/matmul.comp` | ✅ 已实现 | +| matmul_tiled shader (4×4 粗化) | `shaders/matmul_tiled.comp` | ✅ 已实现 | +| elementwise shader | `shaders/elementwise.comp` | ✅ 已实现 | +| SPIR-V 嵌入 | `cmake/embed_spirv.cmake` | ✅ 已实现 | +| 双轨制 GPU 流水线 | `model.hpp` → `forward_gpu` | ✅ 已实现 | + +### 两条路径对比 + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Staging Path (v4, 工作正常) │ +│ │ +│ Layer 1: CPU → Staging → GPU → 计算 → GPU → Staging → CPU │ +│ Layer 2: CPU → Staging → GPU → 计算 → GPU → Staging → CPU │ +│ Layer 3: CPU → Staging → GPU → 计算 → GPU → Staging → CPU │ +│ │ +│ ✅ 正确（每层等待完成），但每层都有 PCIe 传输 │ +└─────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────┐ +│ GPU-Resident Path (v6, 有正确性 Bug) │ +│ │ +│ Upload: CPU → GPU (仅一次) │ +│ Layer 1: GPU → 计算 → GpuTensor (不等待) │ +│ Layer 2: GPU → 计算 → GpuTensor (不等待) │ +│ Layer 3: GPU → 计算 → GpuTensor (不等待) │ +│ Download: GPU → CPU (仅一次) │ +│ │ +│ ❌ 多层链式结果错误 (~10% 准确率 vs 预期 95%) │ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## 🔍 Bug 根因分析 + +经过对 `vk_backend.hpp`、`layer.hpp`、`model.hpp`、`gpu_tensor_impl.hpp` 以及三个 compute shader 的完整代码审查，识别出以下 **5 个独立 Bug**，它们共同导致多层链式推理结果错误： + +### Bug 1：跨 Command Buffer 提交缺少队列级同步（致命） + +**位置**：`GpuBackend::matmul_gpu()` 和 `GpuBackend::elementwise_gpu()` + +**问题**：`matmul_gpu()` 录制 command buffer A（读 buffer_1, 写 buffer_2），提交到 compute queue 后立即返回 `GpuTensor C`。随后 `elementwise_gpu()` 录制 command buffer B（读 buffer_2, 写 buffer_3），再次提交。**两个 command buffer 之间没有任何 fence 或 semaphore 同步**。 + +```cpp +// matmul_gpu() 末尾 —— 提交后不等待 +VkSubmitInfo submit_info{}; +submit_info.commandBufferCount = 1; +submit_info.pCommandBuffers = &cmd; +vkQueueSubmit(device_.compute_queue(), 1, &submit_info, VK_NULL_HANDLE); +// ⚠️ 没有 fence！返回 GpuTensor，GPU 可能还没算完 + +// elementwise_gpu() —— 立即录制新的 command buffer 读取 matmul 的输出 +// ⚠️ 此时 matmul 的 GPU 计算可能还没完成！ +vkCmdCopyBuffer(cmd, src_buffer, ...); // 读取未完成写入的数据 +``` + +**Vulkan 规范**：同一个 queue 内的 command buffer 按提交顺序执行，**但** command buffer B 必须在 command buffer A **完全提交到 queue** 之后才能提交。当前代码虽然在同一 mutex 保护下提交，但 GPU 端的执行是异步的——buffer A 的 SHADER_WRITE 可能还没完成，buffer B 的 SHADER_READ 就开始了。 + +**正确做法**：在 `matmul_gpu` 返回前，用 `vkQueueSubmit` + fence 保证 GPU 完成，或在两个 command buffer 之间插入 `VkSemaphore`。 + +### Bug 2：管线屏障 (Pipeline Barrier) 跨 Command Buffer 无效（致命） + +**位置**：`matmul_gpu()` 中的 `VkBufferMemoryBarrier` + +```cpp +// matmul_gpu() 录制的管线屏障 +VkBufferMemoryBarrier input_barriers[2]{}; +input_barriers[0].srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; +input_barriers[0].dstAccessMask = VK_ACCESS_SHADER_READ_BIT; +vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // srcStageMask + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // dstStageMask + ...); +``` + +**问题**：`vkCmdPipelineBarrier` 的作用域**仅限于当前 command buffer 内部**。它保证的是：在同一个 command buffer 中，之前的 SHADER_WRITE 在后续 SHADER_READ 之前完成。但它**不保证**不同 command buffer 之间的顺序——后一个 command buffer 的 SHADER_READ 可能在前一个 command buffer 的 SHADER_WRITE 之前执行。 + +**Vulkan 规范**：跨 command buffer 的同步必须使用 `VkSemaphore`（queue 间）或 `VkFence`（CPU-GPU）。 + +### Bug 3：`pending_ops_` 资源清理机制存在竞态条件（中等） + +**位置**：`GpuBackend::flush_pending_ops()` + +```cpp +void flush_pending_ops() +{ + auto& p = pending_ops_; + if (p.has_fence) { + vkWaitForFences(device_.device(), 1, &p.fence, VK_TRUE, 10'000'000'000ULL); + vkDestroyFence(device_.device(), p.fence, nullptr); + p.has_fence = false; + } + // ... +} +``` + +**问题**： +1. `matmul_gpu` 和 `elementwise_gpu` 将 command buffer 和 descriptor set 推入 `pending_ops_`，但**不设置 fence**（`has_fence` 始终为 false）。 +2. 只有 `download_blocking()` 调用 `flush_pending_ops()`，但如果 GPU-resident 路径有 10 层，前 9 层的资源（command buffer、descriptor set）一直累积，直到最后 download 时才清理。 +3. 更严重的是：`pending_ops_` 是单例的，只有一组 `cmd_buffers`、`desc_sets`、`fence`。如果两次 `matmul_gpu` 调用之间没有 flush，第二次的 command buffer 会被追加到同一个 vector，但两次操作可能引用**相同的 descriptor set**（因为 `gpu_tensor_pool_` 可能回收后重用同一 set），导致数据竞争。 + +### Bug 4：`GpuTensor` 权重缓存在训练时不会失效（中等） + +**位置**：`Linear::forward_gpu()` + +```cpp +Result forward_gpu(const GpuTensor& input, GpuBackend& backend) override +{ + // 懒加载权重到 GPU（只在第一次调用时上传，之后常驻显存） + if (!gpu_weights_) { + auto w_res = GpuTensor::from_matrix(W_, backend); + gpu_weights_ = std::move(*w_res); + } + // ... +} +``` + +**问题**：在训练循环中，`optimizer.step()` 更新的是 CPU 端的 `W_` 和 `b_`，但 `gpu_weights_` 和 `gpu_bias_` 仍然是旧值。下一次 `forward_gpu` 调用时，因为 `gpu_weights_` 已有值（`has_value() == true`），不会重新上传，导致**前向传播使用过期的权重**。 + +**影响**：此 bug 仅影响 GPU-resident **训练**路径。推理路径（加载预训练模型后）不受影响，因为权重在推理期间不变。 + +### Bug 5：`Scalar` 精度不匹配导致数值误差累积（低） + +**位置**：`detail::convert_scalar_to_float()` / `detail::convert_float_to_scalar()` + +```cpp +// CPU 使用 double（Scalar = double），GPU 使用 float +inline void convert_scalar_to_float( + std::span src, float* __restrict dst) noexcept +{ + std::transform(src.begin(), src.end(), dst, + [](Scalar v) { return static_cast(v); }); +} +``` + +**问题**：每次 upload/download 都有 double→float→double 的精度损失。在 staging path 中，每层都有一次转换，累计误差约 `N_layers × ε_float`。在 GPU-resident path 中，只在输入/输出各转换一次，误差应该更小。但如果 `Scalar = float`，此问题不存在。 + +**影响**：此 bug 不会导致 10% 准确率的大幅下降，但会引入微小的数值差异。主要 bug 仍是 Bug 1 和 Bug 2。 + +--- + +## 🛠️ 修复方案 + +### Fix 1：引入 `GpuFence` 封装队列级同步 + +**新增**：`include/neuralnet.cpp/backend/gpu_sync.hpp` + +```cpp +// GPU 操作句柄：封装 fence，支持 wait + cleanup +class GpuOpHandle { + VkDevice device_ = VK_NULL_HANDLE; + VkFence fence_ = VK_NULL_HANDLE; + std::vector cmd_buffers_; + std::vector desc_sets_; + VkDescriptorPool desc_pool_ = VK_NULL_HANDLE; + VkCommandPool cmd_pool_ = VK_NULL_HANDLE; +public: + GpuOpHandle() = default; + ~GpuOpHandle() { cleanup(); } + + // 提交后等待 GPU 完成 + void wait_and_cleanup() { + if (fence_ != VK_NULL_HANDLE) { + vkWaitForFences(device_, 1, &fence_, VK_TRUE, UINT64_MAX); + vkDestroyFences(device_, 1, &fence_); + fence_ = VK_NULL_HANDLE; + } + cleanup(); + } + + void cleanup() { + if (!cmd_buffers_.empty()) + vkFreeCommandBuffers(device_, cmd_pool_, cmd_buffers_.size(), cmd_buffers_.data()); + if (!desc_sets_.empty()) + vkFreeDescriptorSets(device_, desc_pool_, desc_sets_.size(), desc_sets_.data()); + cmd_buffers_.clear(); + desc_sets_.clear(); + } +}; +``` + +### Fix 2：为 `matmul_gpu` / `elementwise_gpu` 添加 fence 同步 + +**修改**：`vk_backend.hpp` — `matmul_gpu()` 和 `elementwise_gpu()` + +```cpp +// 修改前（无 fence）： +vkQueueSubmit(device_.compute_queue(), 1, &submit_info, VK_NULL_HANDLE); +return C; + +// 修改后（有 fence）： +VkFenceCreateInfo fence_info{}; +fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; +VkFence fence = VK_NULL_HANDLE; +vkCreateFence(device_.device(), &fence_info, nullptr, &fence); +vkQueueSubmit(device_.compute_queue(), 1, &submit_info, fence); + +// 方案 A：阻塞等待（简单，正确性优先） +vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, UINT64_MAX); +vkDestroyFence(device_.device(), 1, &fence); +vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); +return C; + +// 方案 B：返回 handle（高性能，延迟等待） +return GpuOpHandle{device_.device(), fence, {cmd}, {desc_set}, ...}; +// 下一层的 matmul_gpu 在录制前先 wait 上一个 handle +``` + +**推荐方案 A（阻塞等待）**，理由： +- 每层计算耗时远大于 fence wait 的开销（GPU 上千个核心并行） +- 简单、正确、易于调试 +- 后续可优化为 batch submit（多层 command buffer 一次性提交 + 单次 fence wait） + +### Fix 3：移除 `pending_ops_`，改为每个操作独立管理生命周期 + +**修改**：`vk_backend.hpp` + +```cpp +// 删除： +struct PendingGpuOps { ... }; +PendingGpuOps pending_ops_; +void flush_pending_ops() { ... } + +// 每个 GpuTensor 操作完成后立即清理资源 +// matmul_gpu / elementwise_gpu 中： +vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, UINT64_MAX); +vkDestroyFence(device_.device(), 1, &fence); +vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); +vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); +``` + +### Fix 4：添加权重缓存失效机制 + +**修改**：`layer.hpp` — `Linear` 类 + +```cpp +class Linear final : public Layer { + std::optional gpu_weights_; + std::optional gpu_bias_; + bool gpu_weights_dirty_ = true; // 新增：权重是否需要重新上传 + + // 当 optimizer.step() 修改 W_ 或 b_ 时，标记 dirty + // 方案：在 parameters() 返回的 reference_wrapper 中包装一个 dirty flag + // 或者更简单：每次 forward_gpu 都检查 + + Result forward_gpu(const GpuTensor& input, GpuBackend& backend) override + { + // 简单方案：每次都重新上传（有性能开销但正确） + // 优化方案：用 hash 或 version 号判断是否需要更新 + gpu_weights_.reset(); + gpu_bias_.reset(); + auto w_res = GpuTensor::from_matrix(W_, backend); + if (!w_res) return std::unexpected(w_res.error()); + gpu_weights_ = std::move(*w_res); + // ... + } +}; +``` + +**推荐方案**：在 `Layer` 基类中添加 `invalidate_gpu_cache()` 虚函数，由 `optimizer.step()` 在参数更新后调用。 + +### Fix 5：记录精度转换统计（可选） + +**修改**：`vk_backend.hpp` + +```cpp +// 在 convert_scalar_to_float / convert_float_to_scalar 中添加统计 +struct PrecisionStats { + std::atomic conversions{0}; + std::atomic max_ulp_error{0}; // 最大 ULP 误差 +}; +inline PrecisionStats g_precision_stats; +``` + +--- + +## 📅 分阶段实施计划 + +### Phase 1：移植 GPU 后端到 `rewrite` 分支（基础） + +**前置条件**：安装 Vulkan SDK 1.4.x + +| 任务 | 文件 | 说明 | +|------|------|------| +| 1.1 创建 backend 目录 | `include/neuralnet.cpp/backend/` | 从 `8456fe7` 提取 `vk_backend.hpp`、`gpu_tensor_impl.hpp` | +| 1.2 创建 shaders 目录 | `shaders/` | 从 `8456fe7` 提取 `matmul.comp`、`matmul_tiled.comp`、`elementwise.comp` | +| 1.3 创建 cmake 脚本 | `cmake/embed_spirv.cmake` | SPIR-V → C++ 头文件嵌入 | +| 1.4 修改 CMakeLists.txt | `CMakeLists.txt` | 添加 Vulkan 可选依赖、shader 编译管线、`NN_HAS_VULKAN` 定义 | +| 1.5 修改 config.hpp | `include/neuralnet.cpp/config.hpp` | 添加 `GPU_THRESHOLD`、`gpu_enabled`、`gpu_matmul_count`、`cpu_matmul_count` | +| 1.6 适配文件路径 | 所有 backend 文件 | 将 `../nn_config.hpp` 改为 `../config.hpp`，适配扁平化 include 结构 | +| 1.7 编译验证 | — | `cmake -B build && cmake --build build --target gpu_test` | + +### Phase 2：修复核心同步 Bug（关键） + +| 任务 | 文件 | 修复 | +|------|------|------| +| 2.1 修复 `matmul_gpu` 同步 | `vk_backend.hpp` | 提交 command buffer 后添加 fence wait + 资源清理 | +| 2.2 修复 `elementwise_gpu` 同步 | `vk_backend.hpp` | 同上 | +| 2.3 删除 `pending_ops_` | `vk_backend.hpp` | 移除有问题的延迟清理机制 | +| 2.4 单层正确性测试 | `gpu_test.cpp` | 验证单层 matmul + elementwise 结果正确 | + +### Phase 3：集成到 Matrix 类（双轨制） + +| 任务 | 文件 | 说明 | +|------|------|------| +| 3.1 添加 GPU dispatch 到 `multiply_to` | `algebra_matrix.hpp` | `#ifdef NN_HAS_VULKAN` 分支调用 `matmul_direct()` | +| 3.2 添加 GPU dispatch 到 `apply_relu_inplace` | `algebra_matrix.hpp` | 调用 `elementwise_direct()` | +| 3.3 添加 GPU dispatch 到 `apply_gelu_inplace` | `algebra_matrix.hpp` | 调用 `elementwise_direct()` | +| 3.4 Staging path 验证 | `gpu_test.cpp` | 验证 staging path（自动 fallback）正确 | + +### Phase 4：启用 GPU-Resident 流水线 + +| 任务 | 文件 | 说明 | +|------|------|------| +| 4.1 添加 `forward_gpu` 到 Layer | `compute_layer.hpp` | `Linear`、`ReLU`、`GeLU`、`LayerNorm` 等 | +| 4.2 添加 `forward_gpu` 到 Model | `model_container.hpp` | GPU upload → 逐层 `forward_gpu` → download | +| 4.3 添加权重缓存失效 | `compute_layer.hpp` | `optimizer.step()` 后重置 `gpu_weights_` | +| 4.4 多层链式测试 | `gpu_test.cpp` | 2 层 → 5 层 → 10 层网络，对比 CPU/GPU 结果 | + +### Phase 5：性能优化（可选） + +| 任务 | 说明 | +|------|------| +| 5.1 Batch submit | 将多层 command buffer 一次性提交，单次 fence wait | +| 5.2 Double buffering | 使用两组 command buffer 交替录制/执行 | +| 5.3 Async compute | 使用多个 compute queue 并行执行独立操作 | +| 5.4 LayerNorm / Softmax shader | 为归约操作编写专用 shader | + +--- + +## 🧪 测试策略 + +### 单元测试 + +| 测试 | 输入 | 验证 | +|------|------|------| +| `test_matmul_single` | 16×16 方阵 | `max_abs_err < 1e-5` (float) | +| `test_matmul_large` | 8192×8192 | `max_abs_err < 1e-3` (float) | +| `test_elementwise_relu` | 1000 元素 | 逐元素比较 | +| `test_elementwise_gelu` | 1000 元素 | `max_abs_err < 1e-6` | +| `test_elementwise_biasadd` | 512×64 | 行广播正确性 | + +### 集成测试 + +| 测试 | 网络 | 验证 | +|------|------|------| +| `test_2layer_chain` | Linear→ReLU→Linear | CPU/GPU 输出一致 | +| `test_5layer_mlp` | 784→512→256→128→64→10 | MNIST 准确率 ≥ 93% | +| `test_gpt_inference` | GPT-2 small | 生成文本语义一致 | +| `test_training_loop` | MNIST 10 epochs | 收敛曲线一致 | + +### 性能基准 + +| 指标 | Staging Path | GPU-Resident | 目标 | +|------|-------------|-------------|------| +| 单层 matmul (4096×4096) | 5ms | 0.5ms | 10× 加速 | +| 5 层 MLP 前向 | 25ms | 3ms | 8× 加速 | +| PCIe 传输量 (5 层) | 5×2×数据量 | 1×2×数据量 | 5× 减少 | + +--- + +## 📎 参考资料 + +- [Vulkan Compute Pipeline 规范](https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkComputePipelineCreateInfo.html) +- [Vulkan 同步机制详解](https://www.khronos.org/blog/understanding-vulkan-synchronization) +- [SPIR-V Storage Buffer 规范](https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html) +- 仓库记忆：`/memories/repo/gpu-backend.md`（v4-v6.1 设计历史） diff --git a/go_board.py b/go_board.py new file mode 100644 index 00000000..26d90193 --- /dev/null +++ b/go_board.py @@ -0,0 +1,738 @@ +""" +Go Board Widget - 围棋棋盘组件 +============================== +纯 tkinter 实现的 19×19 围棋棋盘，支持： + - 点击落子（自动交替黑白） + - 悔棋、新局、Pass + - 最后一手高亮 + - 坐标标注 + - AI 推荐（调用 text_infer.exe） + - 胜负判定（简单数子法） + +可独立运行：python go_board.py +""" + +import tkinter as tk +from tkinter import ttk, messagebox +import subprocess +import sys +import threading +from pathlib import Path +from typing import Optional, Callable + +# ── 常量 ────────────────────────────────────────────── +BOARD_SIZE = 19 +EMPTY = 0 +BLACK = 1 +WHITE = 2 + +BUILD_DIR = Path(__file__).parent / "build" +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" +TEXT_INFER_EXE = BUILD_DIR / f"text_infer{_EXE_SUFFIX}" + +# 坐标标签（SGF 标准，跳过 'i'） +COL_LABELS = 'abcdefghjklmnopqrst' + + +class GoBoard: + """围棋棋盘逻辑（纯数据，不含 UI）""" + + def __init__(self, size: int = BOARD_SIZE): + self.size = size + self.reset() + + def reset(self): + self.board = [[EMPTY] * self.size for _ in range(self.size)] + self.history: list[tuple[int, int, int]] = [] # (row, col, color) + self.current_color = BLACK + self.pass_count = 0 + self.ko_pos: Optional[tuple[int, int]] = None # 劫位 + + @property + def move_count(self) -> int: + return len(self.history) + + @property + def last_move(self) -> Optional[tuple[int, int]]: + return (self.history[-1][0], self.history[-1][1]) if self.history else None + + @property + def current_color_name(self) -> str: + return "黑方" if self.current_color == BLACK else "白方" + + def is_on_board(self, r: int, c: int) -> bool: + return 0 <= r < self.size and 0 <= c < self.size + + def get(self, r: int, c: int) -> int: + return self.board[r][c] + + def _find_group(self, r: int, c: int) -> tuple[set, int]: + """找到 (r,c) 所在的连通块，返回 (集合, 气数)""" + color = self.board[r][c] + group = set() + liberties = set() + stack = [(r, c)] + while stack: + cr, cc = stack.pop() + if (cr, cc) in group: + continue + group.add((cr, cc)) + for dr, dc in [(-1, 0), (1, 0), (0, -1), (0, 1)]: + nr, nc = cr + dr, cc + dc + if not self.is_on_board(nr, nc): + continue + if self.board[nr][nc] == EMPTY: + liberties.add((nr, nc)) + elif self.board[nr][nc] == color and (nr, nc) not in group: + stack.append((nr, nc)) + return group, len(liberties) + + def _remove_group(self, group: set): + for r, c in group: + self.board[r][c] = EMPTY + + def try_move(self, r: int, c: int) -> bool: + """ + 尝试在 (r,c) 落子。 + 返回 True 表示合法落子，False 表示非法（自杀/劫/已有子）。 + """ + if not self.is_on_board(r, c): + return False + if self.board[r][c] != EMPTY: + return False + + # 劫位检查 + if self.ko_pos == (r, c): + return False + + # 模拟落子 + self.board[r][c] = self.current_color + opponent = WHITE if self.current_color == BLACK else BLACK + + # 检查是否提掉对方棋子 + captured = set() + for dr, dc in [(-1, 0), (1, 0), (0, -1), (0, 1)]: + nr, nc = r + dr, c + dc + if self.is_on_board(nr, nc) and self.board[nr][nc] == opponent: + group, liberties = self._find_group(nr, nc) + if liberties == 0: + captured |= group + + # 提子 + if captured: + self._remove_group(captured) + + # 自杀检查：落子后自己的气数必须 > 0 + _, my_liberties = self._find_group(r, c) + if my_liberties == 0: + # 回滚 + self.board[r][c] = EMPTY + if captured: + for cr, cc in captured: + self.board[cr][cc] = opponent + return False + + # 记录劫位 + self.ko_pos = None + if len(captured) == 1: + cr, cc = next(iter(captured)) + _, my_lib = self._find_group(r, c) + if my_lib == 1: + self.ko_pos = (cr, cc) + + # 记录历史 + self.history.append((r, c, self.current_color)) + self.pass_count = 0 + self.current_color = opponent + return True + + def pass_turn(self): + """过手""" + self.history.append((-1, -1, self.current_color)) + self.pass_count += 1 + self.current_color = WHITE if self.current_color == BLACK else BLACK + self.ko_pos = None + + def undo(self) -> bool: + """悔棋：从空棋盘完整重放历史，正确恢复提子/劫位/连续过手状态""" + if not self.history: + return False + self.history.pop() + self._replay_history() + return True + + def _replay_history(self): + """从空棋盘按历史完整重放（无合法性校验，仅恢复棋理状态）""" + self.board = [[EMPTY] * self.size for _ in range(self.size)] + self.ko_pos = None + self.pass_count = 0 + self.current_color = BLACK + for _i, (r, c, color) in enumerate(self.history): + if r < 0: + self.pass_count += 1 + self.ko_pos = None + else: + self.pass_count = 0 + opponent = WHITE if color == BLACK else BLACK + captured = set() + for dr, dc in [(-1, 0), (1, 0), (0, -1), (0, 1)]: + nr, nc = r + dr, c + dc + if self.is_on_board(nr, nc) and self.board[nr][nc] == opponent: + group, liberties = self._find_group(nr, nc) + if liberties == 0: + captured |= group + for cr, cc in captured: + self.board[cr][cc] = EMPTY + self.board[r][c] = color + self.ko_pos = None + if len(captured) == 1: + cr, cc = next(iter(captured)) + _, my_lib = self._find_group(r, c) + if my_lib == 1: + self.ko_pos = (cr, cc) + self.current_color = WHITE if color == BLACK else BLACK + + def get_moves_text(self) -> str: + """获取当前棋谱的文本表示（用于 AI 推理） + + 每个坐标拆为 2 个 token：列 + 行（如 pd → p d）， + 便于模型分别学习列/行规律。 + """ + tokens = [] + for i, (r, c, _) in enumerate(self.history): + if r < 0: + tokens.append('tt') # PASS + else: + tokens.append(COL_LABELS[c]) # 列 + tokens.append(COL_LABELS[r]) # 行 + return ' '.join(tokens) + + def get_perspective_text(self, ai_is_first: bool) -> str: + """ + 获取带视角标记的完整棋谱序列（与训练格式一致）。 + + 训练格式: " p d d q f r ..." + 每个坐标拆为列+行两个 token。 + AI 固定作为 （赢家视角），人类作为 。 + """ + tokens = [] + for i, (r, c, _) in enumerate(self.history): + is_first = (i % 2 == 0) + is_ai_move = is_first == ai_is_first + tag = '' if is_ai_move else '' + if r < 0: + tokens.append(f"{tag} tt") + else: + tokens.append(f"{tag} {COL_LABELS[c]} {COL_LABELS[r]}") + return ' '.join(tokens) + + def get_legal_moves(self) -> list[tuple[int, int]]: + """获取所有合法落子位置""" + moves = [] + for r in range(self.size): + for c in range(self.size): + if self.board[r][c] == EMPTY: + # 简化检查：跳过自杀和劫位 + if self.ko_pos == (r, c): + continue + moves.append((r, c)) + return moves + + +class GoBoardWidget: + """围棋棋盘 tkinter 组件""" + + def __init__(self, parent: tk.Widget, board: GoBoard, + cell_size: int = 28, padding: int = 30, + on_move: Optional[Callable] = None): + self.board = board + self.cell_size = cell_size + self.padding = padding + self.on_move = on_move # 落子回调 + + canvas_size = cell_size * (BOARD_SIZE - 1) + padding * 2 + self.canvas = tk.Canvas(parent, width=canvas_size, height=canvas_size, + bg='#DCB35C', highlightthickness=0, + cursor='hand2') + self.canvas.bind('', self._on_click) + + self._draw_board() + + def _draw_board(self): + """绘制棋盘底图（网格 + 星位 + 坐标）""" + c = self.canvas + cs = self.cell_size + pad = self.padding + c.delete('all') + + # 网格线 + for i in range(BOARD_SIZE): + x = pad + i * cs + y_start = pad + y_end = pad + (BOARD_SIZE - 1) * cs + c.create_line(x, y_start, x, y_end, fill='#333', width=1) + c.create_line(y_start, x, y_end, x, fill='#333', width=1) + + # 星位（4-4, 4-10, 4-16, 10-4, 10-10, 10-16, 16-4, 16-10, 16-16） + star_points = [(3, 3), (3, 9), (3, 15), + (9, 3), (9, 9), (9, 15), + (15, 3), (15, 9), (15, 15)] + dot_r = 4 + for r, c_ in star_points: + x = pad + c_ * cs + y = pad + r * cs + c.create_oval(x - dot_r, y - dot_r, x + dot_r, y + dot_r, + fill='#333', outline='') + + # 坐标标签 + for i in range(BOARD_SIZE): + x = pad + i * cs + y_top = pad - 18 + y_bot = pad + (BOARD_SIZE - 1) * cs + 6 + c.create_text(x, y_top, text=COL_LABELS[i], fill='#555', + font=('Consolas', 8)) + c.create_text(x, y_bot, text=COL_LABELS[i], fill='#555', + font=('Consolas', 8)) + + y_left = pad + i * cs + c.create_text(pad - 18, y_left, text=str(BOARD_SIZE - i), fill='#555', + font=('Consolas', 8)) + c.create_text(pad + (BOARD_SIZE - 1) * cs + 18, y_left, + text=str(BOARD_SIZE - i), fill='#555', + font=('Consolas', 8)) + + def _on_click(self, event): + """处理点击落子""" + cs = self.cell_size + pad = self.padding + col = round((event.x - pad) / cs) + row = round((event.y - pad) / cs) + + if not (0 <= row < BOARD_SIZE and 0 <= col < BOARD_SIZE): + return + + if self.board.try_move(row, col): + self.redraw() + if self.on_move: + self.on_move() + + def redraw(self): + """重绘棋盘（棋子 + 最后一手高亮）""" + cs = self.cell_size + pad = self.padding + stone_r = cs * 0.43 + + # 只重绘棋子层（保留底图） + self.canvas.delete('stones') + self.canvas.delete('highlight') + + for r in range(BOARD_SIZE): + for c in range(BOARD_SIZE): + color_val = self.board.get(r, c) + if color_val == EMPTY: + continue + x = pad + c * cs + y = pad + r * cs + fill = '#111' if color_val == BLACK else '#eee' + outline = '#000' if color_val == BLACK else '#999' + self.canvas.create_oval( + x - stone_r, y - stone_r, x + stone_r, y + stone_r, + fill=fill, outline=outline, width=1.5, + tags='stones') + + # 最后一手高亮 + if self.board.last_move: + lr, lc = self.board.last_move + x = pad + lc * cs + y = pad + lr * cs + r = 4 + self.canvas.create_oval( + x - r, y - r, x + r, y + r, + fill='#e44', outline='', tags='highlight') + + def coord_to_sgf(self, r: int, c: int) -> str: + """棋盘坐标转 SGF 格式""" + return COL_LABELS[c] + COL_LABELS[r] + + +class GoGamePanel: + """围棋对弈面板（完整 UI 组件）""" + + def __init__(self, parent: tk.Widget): + self.parent = parent + self.board = GoBoard() + self.ai_thinking = False + + self._build_ui() + + def _build_ui(self): + """构建对弈界面""" + # 主容器：左棋盘 + 右控制 + main = ttk.Frame(self.parent) + main.pack(fill='both', expand=True, padx=5, pady=5) + + # ── 左侧：棋盘 ────────────────────────────── + board_frame = ttk.Frame(main) + board_frame.pack(side='left', fill='both', expand=True) + + self.board_widget = GoBoardWidget( + board_frame, self.board, + cell_size=28, padding=30, + on_move=self._on_human_move) + + self.board_widget.canvas.pack(padx=5, pady=5) + + # ── 右侧：控制面板 ────────────────────────── + right = ttk.Frame(main, width=280) + right.pack(side='right', fill='y', padx=(10, 0)) + right.pack_propagate(False) + + # 状态信息 + self.status_var = tk.StringVar(value="黑方（你）落子") + ttk.Label(right, textvariable=self.status_var, + font=('Microsoft YaHei', 12, 'bold')).pack(pady=(10, 5)) + + self.move_count_var = tk.StringVar(value="第 0 手") + ttk.Label(right, textvariable=self.move_count_var, + font=('Consolas', 10)).pack() + + # ── 模式选择 ──────────────────────────────── + mode_frame = ttk.LabelFrame(right, text="对弈模式", padding=5) + mode_frame.pack(fill='x', padx=5, pady=5) + + self.mode_var = tk.StringVar(value="human_human") + ttk.Radiobutton(mode_frame, text="👥 人人对弈", + variable=self.mode_var, value="human_human", + command=self._on_mode_change).pack(anchor='w') + ttk.Radiobutton(mode_frame, text="🤖 人机对弈（我执黑）", + variable=self.mode_var, value="human_black", + command=self._on_mode_change).pack(anchor='w') + ttk.Radiobutton(mode_frame, text="🤖 人机对弈（我执白）", + variable=self.mode_var, value="human_white", + command=self._on_mode_change).pack(anchor='w') + + # ── AI 模型路径 ───────────────────────────── + ai_frame = ttk.LabelFrame(right, text="AI 模型", padding=5) + ai_frame.pack(fill='x', padx=5, pady=5) + + model_row = ttk.Frame(ai_frame) + model_row.pack(fill='x') + self.model_var = tk.StringVar(value=str( + Path(__file__).parent / "go_model.bin")) + ttk.Entry(model_row, textvariable=self.model_var, + font=('Consolas', 9)).pack(side='left', fill='x', expand=True) + ttk.Button(model_row, text="...", width=3, + command=self._browse_model).pack(side='right', padx=(3, 0)) + + temp_row = ttk.Frame(ai_frame) + temp_row.pack(fill='x', pady=(3, 0)) + ttk.Label(temp_row, text="温度:").pack(side='left') + self.temp_var = tk.StringVar(value="0.8") + ttk.Entry(temp_row, textvariable=self.temp_var, width=5, + font=('Consolas', 9)).pack(side='left', padx=3) + + # ── 按钮区 ────────────────────────────────── + btn_frame = ttk.Frame(right) + btn_frame.pack(fill='x', padx=5, pady=5) + + ttk.Button(btn_frame, text="⏪ 悔棋", + command=self._undo).pack(fill='x', pady=2) + ttk.Button(btn_frame, text="⏭️ Pass", + command=self._pass).pack(fill='x', pady=2) + ttk.Button(btn_frame, text="🤖 AI 走一步", + command=self._ai_move).pack(fill='x', pady=2) + ttk.Button(btn_frame, text="🔄 新局", + command=self._new_game).pack(fill='x', pady=2) + + # ── 棋谱显示 ──────────────────────────────── + moves_frame = ttk.LabelFrame(right, text="棋谱", padding=3) + moves_frame.pack(fill='both', expand=True, padx=5, pady=(5, 0)) + + self.moves_text = tk.Text(moves_frame, height=8, width=30, + font=('Consolas', 9), bg='#1e1e1e', + fg='#d4d4d4', wrap='word', state='disabled') + scrollbar = ttk.Scrollbar(moves_frame, command=self.moves_text.yview) + self.moves_text.configure(yscrollcommand=scrollbar.set) + scrollbar.pack(side='right', fill='y') + self.moves_text.pack(fill='both', expand=True) + + def _browse_model(self): + from tkinter import filedialog + path = filedialog.askopenfilename( + title="选择模型文件", + filetypes=[("模型文件", "*.bin"), ("所有文件", "*.*")]) + if path: + self.model_var.set(path) + + def _on_mode_change(self): + mode = self.mode_var.get() + if mode == "human_black": + self.status_var.set("黑方（你）落子") + elif mode == "human_white": + self.status_var.set("等待 AI 落子...") + self.root.after(100, self._ai_move) + else: + self.status_var.set(f"{self.board.current_color_name}落子") + + @property + def root(self): + return self.parent.winfo_toplevel() + + def _on_human_move(self): + """人类落子后""" + self._update_status() + self._update_moves_text() + + # 人机模式下自动触发 AI + mode = self.mode_var.get() + if mode == "human_black" and self.board.current_color == WHITE: + self.root.after(100, self._ai_move) + elif mode == "human_white" and self.board.current_color == BLACK: + self.root.after(100, self._ai_move) + + def _update_status(self): + mc = self.board.move_count + cn = self.board.current_color_name + mode = self.mode_var.get() + + if mode == "human_black": + if self.board.current_color == BLACK: + self.status_var.set("黑方（你）落子") + else: + self.status_var.set("白方（AI）思考中...") + elif mode == "human_white": + if self.board.current_color == WHITE: + self.status_var.set("白方（你）落子") + else: + self.status_var.set("黑方（AI）思考中...") + else: + self.status_var.set(f"{cn}落子") + + self.move_count_var.set(f"第 {mc} 手") + + def _update_moves_text(self): + """更新棋谱显示""" + self.moves_text.configure(state='normal') + self.moves_text.delete('1.0', 'end') + + moves = [] + for i, (r, c, color) in enumerate(self.board.history): + if r < 0: + move_str = "PASS" + else: + coord = COL_LABELS[c] + COL_LABELS[BOARD_SIZE - 1 - r] + move_str = coord + color_mark = "●" if color == BLACK else "○" + moves.append(f"{i+1:3d}. {color_mark} {move_str}") + + # 每行 2 手 + lines = [] + for i in range(0, len(moves), 2): + line = moves[i] + if i + 1 < len(moves): + line += " " + moves[i + 1] + lines.append(line) + + self.moves_text.insert('1.0', '\n'.join(lines)) + self.moves_text.see('end') + self.moves_text.configure(state='disabled') + + def _undo(self): + if self.board.undo(): + # 人机模式下可能需要悔两步 + mode = self.mode_var.get() + if mode in ("human_black", "human_white"): + self.board.undo() + self.board_widget.redraw() + self._update_status() + self._update_moves_text() + + def _pass(self): + self.board.pass_turn() + self.board_widget.redraw() + self._update_status() + self._update_moves_text() + + mode = self.mode_var.get() + if mode == "human_black" and self.board.current_color == WHITE: + self.root.after(100, self._ai_move) + elif mode == "human_white" and self.board.current_color == BLACK: + self.root.after(100, self._ai_move) + + def _new_game(self): + self.board.reset() + self.board_widget.redraw() + self._update_status() + self._update_moves_text() + + def _ai_move(self): + """AI 走棋""" + if self.ai_thinking: + return + + model_path = self.model_var.get() + if not Path(model_path).exists(): + self.status_var.set("⚠️ 模型文件不存在") + return + + self.ai_thinking = True + self.status_var.set("🤖 AI 思考中...") + + def _run_ai(): + try: + # 构建 prompt：带视角标记的完整历史 + AI 即将落子 + # AI 固定为 （赢家视角），人类为 + mode = self.mode_var.get() + # AI 执黑(先手)时第0,2,4手是AI；AI 执白(后手)时第1,3,5手是AI + ai_is_first = (mode == "human_white") + perspective = self.board.get_perspective_text(ai_is_first) + + if perspective: + prompt = f"{perspective} " + else: + prompt = "" + + temp = float(self.temp_var.get()) + + # 一次生成多个 token，再从 token 流中提取第一个合法坐标。 + # 模型常先输出 / 标记，随后才输出坐标； + # 若只生成 1 个 token，大概率拿到标记导致“无法返回有效落子”。 + cmd = [ + str(TEXT_INFER_EXE), + '--model', model_path, + '--prompt', prompt, + '--max-tokens', '16', + '--temperature', str(temp), + ] + + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=30) + + # 解析输出，找到生成的 token + output = result.stdout + result.stderr + generated = self._parse_infer_output(output, prompt) + + if generated: + self.root.after(0, lambda: self._apply_ai_move(generated)) + else: + self.root.after(0, lambda: self.status_var.set( + "⚠️ AI 未返回有效落子")) + + except subprocess.TimeoutExpired: + self.root.after(0, lambda: self.status_var.set( + "⚠️ AI 超时")) + except Exception as e: + self.root.after(0, lambda: self.status_var.set( + f"⚠️ AI 错误: {e}")) + finally: + self.ai_thinking = False + + threading.Thread(target=_run_ai, daemon=True).start() + + @staticmethod + def _first_go_token(text: str) -> Optional[str]: + """从文本中提取第一个合法的围棋坐标。 + + 新格式：坐标由列 + 行两个单字符 token 组成（如 p d）。 + 兼容旧格式：两字符坐标 token（如 pd）。 + """ + toks = [t.strip('()') for t in text.split()] + for i, tok in enumerate(toks): + if tok == 'tt': + return 'tt' + # 新格式：列 + 行两个独立 token + if tok in COL_LABELS and i + 1 < len(toks) and toks[i + 1] in COL_LABELS: + return tok + toks[i + 1] + # 兼容旧格式：合并两字符坐标 + if len(tok) == 2 and tok[0] in COL_LABELS and tok[1] in COL_LABELS: + return tok + return None + + def _parse_infer_output(self, output: str, prompt: str = '') -> Optional[str]: + """解析 text_infer 输出，提取生成的 token。 + + text_infer 输出格式（最后一行是统计行，输出行在中间）： + ... + pd dp pp pd ← 输出行 = prompt + 生成 + ---------------------------------------- + 生成 16 tokens, 耗时 0.1s (10 tokens/s) ← 统计行 + + 模型可能先输出 / 标记，因此从整行中提取第一个合法坐标， + 而不是只看第一个 token。 + """ + lines = [l for l in output.strip().split('\n') if l.strip()] + + # 过滤日志/统计行，只保留内容行（输出行 = prompt + 生成） + content_lines = [] + for line in lines: + line = line.strip() + if not line: + continue + if any(kw in line for kw in ['GPT', '模型', '词表', '提示', 'Prompt', + '生成', '===', '---', '耗时', 'tokens', + 'token', '加载', '规格', '有效']): + continue + content_lines.append(line) + + # 1) 优先：找到以 prompt 开头的行，取其后的部分（真正的生成内容） + if prompt: + for line in content_lines: + if line.startswith(prompt): + remainder = line[len(prompt):].strip() + if remainder: + tok = self._first_go_token(remainder) + if tok: + return tok + break + + # 2) 回退：从内容行中找第一个合法坐标， + # 但排除“无新增生成”的行（整行就是 prompt 本身， + # 其中的坐标是已下过的旧着法，返回会导致非法落子→自动 Pass） + for line in content_lines: + if prompt and line == prompt.strip(): + continue + tok = self._first_go_token(line) + if tok: + return tok + return None + + def _apply_ai_move(self, coord: str): + """应用 AI 的落子""" + if coord == 'tt': + self.board.pass_turn() + else: + col = COL_LABELS.index(coord[0]) + row = COL_LABELS.index(coord[1]) # 与编码一致：r=0 为顶行，无镜像 + if not self.board.try_move(row, col): + # AI 返回了非法着法（如已有子/自杀/劫），提示并 Pass + self.status_var.set(f"⚠️ AI 非法着法 {coord}，自动 Pass") + self.board.pass_turn() + + self.board_widget.redraw() + self._update_status() + self._update_moves_text() + + # 如果是人人模式或轮到人了，更新状态 + mode = self.mode_var.get() + if mode == "human_black" and self.board.current_color == BLACK: + self.status_var.set("黑方（你）落子") + elif mode == "human_white" and self.board.current_color == WHITE: + self.status_var.set("白方（你）落子") + elif mode == "human_human": + self.status_var.set(f"{self.board.current_color_name}落子") + + +# ═══════════════════════════════════════════════════════════════════════════ +# 独立运行 +# ═══════════════════════════════════════════════════════════════════════════ + +if __name__ == '__main__': + root = tk.Tk() + root.title("围棋 - Go Board") + root.configure(bg='#2d2d2d') + + style = ttk.Style() + style.theme_use('clam') + + panel = GoGamePanel(root) + root.mainloop() diff --git a/gpt_bpe.json b/gpt_bpe.json deleted file mode 100644 index b8669ce7..00000000 --- a/gpt_bpe.json +++ /dev/null @@ -1,8809 +0,0 @@ -{ - "type": "freq_based_tokenizer", - "vocab": { - "4": "00", - "5": "01", - "6": "02", - "7": "03", - "8": "04", - "9": "05", - "10": "06", - "11": "07", - "12": "08", - "13": "09", - "14": "0a", - "15": "0b", - "16": "0c", - "17": "0d", - "18": "0e", - "19": "0f", - "20": "10", - "21": "11", - "22": "12", - "23": "13", - "24": "14", - "25": "15", - "26": "16", - "27": "17", - "28": "18", - "29": "19", - "30": "1a", - "31": "1b", - "32": "1c", - "33": "1d", - "34": "1e", - "35": "1f", - "36": "20", - "37": "21", - "38": "22", - "39": "23", - "40": "24", - "41": "25", - "42": "26", - "43": "27", - "44": "28", - "45": "29", - "46": "2a", - "47": "2b", - "48": "2c", - "49": "2d", - "50": "2e", - "51": "2f", - "52": "30", - "53": "31", - "54": "32", - "55": "33", - "56": "34", - "57": "35", - "58": "36", - "59": "37", - "60": "38", - "61": "39", - "62": "3a", - "63": "3b", - "64": "3c", - "65": "3d", - "66": "3e", - "67": "3f", - "68": "40", - "69": "41", - "70": "42", - "71": "43", - "72": "44", - "73": "45", - "74": "46", - "75": "47", - "76": "48", - "77": "49", - "78": "4a", - "79": "4b", - "80": "4c", - "81": "4d", - "82": "4e", - "83": "4f", - "84": "50", - "85": "51", - "86": "52", - "87": "53", - "88": "54", - "89": "55", - "90": "56", - "91": "57", - "92": "58", - "93": "59", - "94": "5a", - "95": "5b", - "96": "5c", - "97": "5d", - "98": "5e", - "99": "5f", - "100": "60", - "101": "61", - "102": "62", - "103": "63", - "104": "64", - "105": "65", - "106": "66", - "107": "67", - "108": "68", - "109": "69", - "110": "6a", - "111": "6b", - "112": "6c", - "113": "6d", - "114": "6e", - "115": "6f", - "116": "70", - "117": "71", - "118": "72", - "119": "73", - "120": "74", - "121": "75", - "122": "76", - "123": "77", - "124": "78", - "125": "79", - "126": "7a", - "127": "7b", - "128": "7c", - "129": "7d", - "130": "7e", - "131": "7f", - "132": "80", - "133": "81", - "134": "82", - "135": "83", - "136": "84", - "137": "85", - "138": "86", - "139": "87", - "140": "88", - "141": "89", - "142": "8a", - "143": "8b", - "144": "8c", - "145": "8d", - "146": "8e", - "147": "8f", - "148": "90", - "149": "91", - "150": "92", - "151": "93", - "152": "94", - "153": "95", - "154": "96", - "155": "97", - "156": "98", - "157": "99", - "158": "9a", - "159": "9b", - "160": "9c", - "161": "9d", - "162": "9e", - "163": "9f", - "164": "a0", - "165": "a1", - "166": "a2", - "167": "a3", - "168": "a4", - "169": "a5", - "170": "a6", - "171": "a7", - "172": "a8", - "173": "a9", - "174": "aa", - "175": "ab", - "176": "ac", - "177": "ad", - "178": "ae", - "179": "af", - "180": "b0", - "181": "b1", - "182": "b2", - "183": "b3", - "184": "b4", - "185": "b5", - "186": "b6", - "187": "b7", - "188": "b8", - "189": "b9", - "190": "ba", - "191": "bb", - "192": "bc", - "193": "bd", - "194": "be", - "195": "bf", - "196": "c0", - "197": "c1", - "198": "c2", - "199": "c3", - "200": "c4", - "201": "c5", - "202": "c6", - "203": "c7", - "204": "c8", - "205": "c9", - "206": "ca", - "207": "cb", - "208": "cc", - "209": "cd", - "210": "ce", - "211": "cf", - "212": "d0", - "213": "d1", - "214": "d2", - "215": "d3", - "216": "d4", - "217": "d5", - "218": "d6", - "219": "d7", - "220": "d8", - "221": "d9", - "222": "da", - "223": "db", - "224": "dc", - "225": "dd", - "226": "de", - "227": "df", - "228": "e0", - "229": "e1", - "230": "e2", - "231": "e3", - "232": "e4", - "233": "e5", - "234": "e6", - "235": "e7", - "236": "e8", - "237": "e9", - "238": "ea", - "239": "eb", - "240": "ec", - "241": "ed", - "242": "ee", - "243": "ef", - "244": "f0", - "245": "f1", - "246": "f2", - "247": "f3", - "248": "f4", - "249": "f5", - "250": "f6", - "251": "f7", - "252": "f8", - "253": "f9", - "254": "fa", - "255": "fb", - "256": "fc", - "257": "fd", - "258": "fe", - "259": "ff", - "260": "20746865", - "261": "207468", - "262": "746865", - "263": "0a0a", - "264": "2074", - "265": "6865", - "266": "7468", - "267": "2061", - "268": "20616e64", - "269": "0a0a0a", - "270": "616e64", - "271": "696e", - "272": "616e", - "273": "0a0a0a0a", - "274": "6572", - "275": "20202020202020202020202020202020", - "276": "2077", - "277": "2068", - "278": "2073", - "279": "20746f", - "280": "696e67", - "281": "6e64", - "282": "2020202020202020202020", - "283": "7265", - "284": "e280", - "285": "206f", - "286": "206f66", - "287": "6f75", - "288": "2074686174", - "289": "6861", - "290": "206865", - "291": "2069", - "292": "686572", - "293": "6174", - "294": "20202020202020", - "295": "6564", - "296": "6f6e", - "297": "656e", - "298": "20776173", - "299": "20696e", - "300": "2020202020", - "301": "746f", - "302": "206d", - "303": "206861", - "304": "207761", - "305": "6e67", - "306": "2062", - "307": "6869", - "308": "6974", - "309": "686174", - "310": "20796f75", - "311": "6173", - "312": "206265", - "313": "2077697468", - "314": "6973", - "315": "e2809c", - "316": "e28099", - "317": "6f72", - "318": "206869", - "319": "746861", - "320": "e2809d", - "321": "6172", - "322": "2066", - "323": "7374", - "324": "74686572", - "325": "7665", - "326": "6573", - "327": "7465", - "328": "2063", - "329": "776173", - "330": "657265", - "331": "6d65", - "332": "207768", - "333": "7365", - "334": "6f66", - "335": "686973", - "336": "202020", - "337": "77697468", - "338": "206e6f", - "339": "206974", - "340": "207769", - "341": "20686973", - "342": "2064", - "343": "20666f72", - "344": "6561", - "345": "6c65", - "346": "20636f", - "347": "796f75", - "348": "20776974", - "349": "6e65", - "350": "666f72", - "351": "6c6c", - "352": "6f756c64", - "353": "20686572", - "354": "2049", - "355": "7761", - "356": "20796f", - "357": "766572", - "358": "206e6f74", - "359": "20686164", - "360": "6e74", - "361": "20666f", - "362": "68657265", - "363": "6265", - "364": "656e74", - "365": "616c", - "366": "20736f", - "367": "206f6e", - "368": "206c", - "369": "207368", - "370": "7469", - "371": "616c6c", - "372": "6e6f", - "373": "206e", - "374": "207765", - "375": "746572", - "376": "2068617665", - "377": "746869", - "378": "6465", - "379": "74696f6e", - "380": "2070", - "381": "69676874", - "382": "207361", - "383": "2ee2809d", - "384": "6e6f74", - "385": "726f", - "386": "6f6d", - "387": "7572", - "388": "686f", - "389": "207265", - "390": "6164", - "391": "206173", - "392": "2068696d", - "393": "7574", - "394": "7468696e", - "395": "20746869", - "396": "656c", - "397": "6c69", - "398": "6f77", - "399": "636f", - "400": "206d61", - "401": "206d65", - "402": "696f6e", - "403": "676874", - "404": "6f74", - "405": "207768696368", - "406": "20646f", - "407": "6565", - "408": "2073616964", - "409": "686164", - "410": "7468696e67", - "411": "2072", - "412": "7269", - "413": "7768", - "414": "6f7572", - "415": "6365", - "416": "7769", - "417": "6368", - "418": "7368", - "419": "207468657265", - "420": "656172", - "421": "20616c", - "422": "20776f756c64", - "423": "65766572", - "424": "6574", - "425": "666f", - "426": "207365", - "427": "617665", - "428": "207374", - "429": "68696e", - "430": "6169", - "431": "657665", - "432": "6f6d65", - "433": "736f", - "434": "2065", - "435": "2067", - "436": "20616c6c", - "437": "7765", - "438": "20627574", - "439": "6f6f", - "440": "20736865", - "441": "20636f756c64", - "442": "20776869", - "443": "20776f", - "444": "7573", - "445": "206d79", - "446": "6f6e65", - "447": "7468657265", - "448": "0a0a0a0a0a", - "449": "68656e", - "450": "206361", - "451": "546865", - "452": "2079", - "453": "6f7574", - "454": "776f756c64", - "455": "6f756768", - "456": "6c79", - "457": "696d", - "458": "6c6f", - "459": "756e", - "460": "6d61", - "461": "206c69", - "462": "6c64", - "463": "756c", - "464": "2074686579", - "465": "736865", - "466": "20636f75", - "467": "0a0a0a0a0a0a", - "468": "2066726f6d", - "469": "696c", - "470": "6964", - "471": "20546865", - "472": "657373", - "473": "2077657265", - "474": "636f756c64", - "475": "206174", - "476": "6f74686572", - "477": "6e6365", - "478": "7261", - "479": "6768", - "480": "686f75", - "481": "2074686572", - "482": "206d6f", - "483": "2074686973", - "484": "206275", - "485": "20776f756c", - "486": "726561", - "487": "7369", - "488": "206973", - "489": "6174696f6e", - "490": "6c61", - "491": "7361", - "492": "7065", - "493": "61696e", - "494": "206465", - "495": "20736565", - "496": "6b65", - "497": "20776865", - "498": "7373", - "499": "20636f756c", - "500": "76657279", - "501": "6179", - "502": "20736f6d65", - "503": "206c6f", - "504": "6963", - "505": "696c6c", - "506": "746564", - "507": "627574", - "508": "207468656d", - "509": "206469", - "510": "776869", - "511": "7273", - "512": "726573", - "513": "20686f", - "514": "6765", - "515": "6f756e", - "516": "6361", - "517": "205468", - "518": "20676f", - "519": "646f", - "520": "207768656e", - "521": "206265656e", - "522": "20636f6d", - "523": "206672", - "524": "7461", - "525": "696f", - "526": "636f75", - "527": "6469", - "528": "726564", - "529": "736f6d65", - "530": "207468696e", - "531": "6f7265", - "532": "6f7468", - "533": "74686973", - "534": "207375", - "535": "74686f756768", - "536": "2075", - "537": "616964", - "538": "6163", - "539": "2074686f756768", - "540": "20657665", - "541": "0a0a0a0a0a0a0a", - "542": "206f7574", - "543": "20686561", - "544": "206b6e6f77", - "545": "617465", - "546": "656c6c", - "547": "656d", - "548": "657273", - "549": "20796f7572", - "550": "756e64", - "551": "68616e", - "552": "20636f6e", - "553": "6e6f77", - "554": "6972", - "555": "6967", - "556": "6d6f", - "557": "776f", - "558": "206c6974746c65", - "559": "6965", - "560": "6d616e", - "561": "7472", - "562": "776572", - "563": "617265", - "564": "206f75", - "565": "636f6d", - "566": "657374", - "567": "6f756e64", - "568": "2077686174", - "569": "6e63", - "570": "74686f75", - "571": "207072", - "572": "6176", - "573": "6174696f", - "574": "7468656d", - "575": "696e74", - "576": "776865", - "577": "7279", - "578": "207468656972", - "579": "206661", - "580": "6f7665", - "581": "617469", - "582": "3fe2809d", - "583": "686579", - "584": "2065766572", - "585": "6f6c", - "586": "7274", - "587": "207570", - "588": "206d616e", - "589": "626c65", - "590": "6563", - "591": "20736f6d", - "592": "616d", - "593": "2ce2809d", - "594": "0a0a0a0a0a0a0a0a", - "595": "5468", - "596": "7768656e", - "597": "697665", - "598": "73656c66", - "599": "2073686f", - "600": "696e64", - "601": "6874", - "602": "2074696d65", - "603": "74696e67", - "604": "636f6e", - "605": "206265666f7265", - "606": "736565", - "607": "6f776e", - "608": "65656e", - "609": "206c6f6f6b", - "610": "2077686f", - "611": "74696d65", - "612": "6b6e6f77", - "613": "6e73", - "614": "20696e74", - "615": "206f6e65", - "616": "6d69", - "617": "206c61", - "618": "6f6f6b", - "619": "20616761696e", - "620": "757374", - "621": "207061", - "622": "686561", - "623": "7474", - "624": "747465", - "625": "737465", - "626": "617264", - "627": "207369", - "628": "696e65", - "629": "796f7572", - "630": "6d656e", - "631": "2076657279", - "632": "207468656e", - "633": "2061626f7574", - "634": "77686174", - "635": "2077696c6c", - "636": "6f766572", - "637": "646572", - "638": "6e6465", - "639": "6579", - "640": "6665", - "641": "206d69", - "642": "726f75", - "643": "6d79", - "644": "206172", - "645": "726f6d", - "646": "6d656e74", - "647": "e28094", - "648": "206c65", - "649": "656164", - "650": "206d6f72", - "651": "6e696e67", - "652": "6e6564", - "653": "6f7573", - "654": "206b6e6f", - "655": "74686f", - "656": "6b696e67", - "657": "206d6f7265", - "658": "2073686f756c64", - "659": "737461", - "660": "6f73", - "661": "74746572", - "662": "65666f7265", - "663": "7375", - "664": "206f74686572", - "665": "6e69", - "666": "206669", - "667": "706f", - "668": "74696e", - "669": "20646f776e", - "670": "206576", - "671": "7072", - "672": "2068656172", - "673": "656174", - "674": "206e65", - "675": "2054", - "676": "2074686f", - "677": "737472", - "678": "6275", - "679": "776179", - "680": "206f766572", - "681": "73656c", - "682": "636b", - "683": "6c6f6f6b", - "684": "73686f756c64", - "685": "20776861", - "686": "206279", - "687": "20616e79", - "688": "706572", - "689": "6b696e", - "690": "696465", - "691": "626c", - "692": "20776f72", - "693": "6e657373", - "694": "20696e746f", - "695": "206665", - "696": "616e79", - "697": "626f", - "698": "6162", - "699": "456c697a6162657468", - "700": "616d65", - "701": "656e64", - "702": "7061", - "703": "73686f", - "704": "6f75676874", - "705": "206b6e", - "706": "617374", - "707": "72696e67", - "708": "747572", - "709": "656374", - "710": "77696c6c", - "711": "61626c65", - "712": "7264", - "713": "206c696b65", - "714": "75676874", - "715": "616b65", - "716": "616e6365", - "717": "206c6f6f", - "718": "2063616e", - "719": "2077696c", - "720": "656e6365", - "721": "6f6e73", - "722": "617274", - "723": "2074686f75676874", - "724": "6e646572", - "725": "0a0a0a0a0a0a0a0a0a", - "726": "207469", - "727": "2074696d", - "728": "6e7465", - "729": "757265", - "730": "757365", - "731": "206d75", - "732": "6f6e67", - "733": "207468696e6b", - "734": "207472", - "735": "206461", - "736": "65726564", - "737": "756368", - "738": "2068616e64", - "739": "20737472", - "740": "74656e", - "741": "736564", - "742": "6d6f72", - "743": "676f", - "744": "6f6f64", - "745": "20617265", - "746": "6d696e", - "747": "204865", - "748": "20756e", - "749": "74686f75676874", - "750": "746c65", - "751": "6374", - "752": "2073686f756c", - "753": "74746c65", - "754": "616e74", - "755": "6e6573", - "756": "206576657279", - "757": "6c6f6f", - "758": "68656172", - "759": "6669", - "760": "726965", - "761": "20636f6d65", - "762": "6661", - "763": "7570", - "764": "20736179", - "765": "6c696e67", - "766": "6c696e", - "767": "206772", - "768": "207468616e", - "769": "6174686572", - "770": "77686f", - "771": "72656174", - "772": "206368", - "773": "20746f6f", - "774": "206e6f7468696e67", - "775": "6672", - "776": "6c696b65", - "777": "6167", - "778": "646f776e", - "779": "6b6564", - "780": "636f6d65", - "781": "206f746865", - "782": "6e69676874", - "783": "7567", - "784": "6c6f6e67", - "785": "20646964", - "786": "7563", - "787": "6761696e", - "788": "21e2809d", - "789": "2048", - "790": "20656e", - "791": "6e6f7468696e67", - "792": "206e65766572", - "793": "617468", - "794": "707265", - "795": "20746872", - "796": "72696e", - "797": "6966", - "798": "204d", - "799": "656c66", - "800": "206166746572", - "801": "6f7365", - "802": "697374", - "803": "7468616e", - "804": "646f77", - "805": "207468726f756768", - "806": "726f756768", - "807": "7468696e6b", - "808": "207468696e67", - "809": "6576657279", - "810": "206b", - "811": "2068696d73656c66", - "812": "616e63", - "813": "6f64", - "814": "6f7274", - "815": "706c", - "816": "726e", - "817": "696e746f", - "818": "73696e", - "819": "726f756e64", - "820": "206d757374", - "821": "20456c697a6162657468", - "822": "6f6c64", - "823": "20626f", - "824": "68616e64", - "825": "746f6f", - "826": "20737461", - "827": "2070726f", - "828": "206d69676874", - "829": "616365", - "830": "20646f77", - "831": "206f72", - "832": "6c6564", - "833": "20736861", - "834": "697474", - "835": "6e696e", - "836": "7269676874", - "837": "656173", - "838": "6c6974", - "839": "656e63", - "840": "6c69676874", - "841": "206578", - "842": "207368616c6c", - "843": "206f7665", - "844": "2075706f6e", - "845": "747261", - "846": "20676f6f64", - "847": "736f6d657468696e67", - "848": "776f72", - "849": "6c6561", - "850": "746c", - "851": "6976", - "852": "206272", - "853": "2068616e", - "854": "737469", - "855": "7473", - "856": "20706f", - "857": "657468", - "858": "6f6b", - "859": "7475", - "860": "20646973", - "861": "64696e67", - "862": "6166746572", - "863": "76656e", - "864": "63616e", - "865": "6570", - "866": "206772656174", - "867": "20646561", - "868": "6c6f6e", - "869": "206e6f7468696e", - "870": "20667269656e64", - "871": "20726561", - "872": "20707265", - "873": "61626c", - "874": "207461", - "875": "206f6e6c79", - "876": "0a0a20202020", - "877": "20736f6d657468696e67", - "878": "206e6f77", - "879": "206e69676874", - "880": "2073756368", - "881": "6461", - "882": "7768657265", - "883": "6170", - "884": "206261", - "885": "70726f", - "886": "6569", - "887": "207370", - "888": "206162", - "889": "6e6f7468", - "890": "206c696b", - "891": "616b", - "892": "20646f6e", - "893": "6e61", - "894": "206d756368", - "895": "0a0a2020202020", - "896": "6e6765", - "897": "206c6f6f6b6564", - "898": "20726f", - "899": "73696e67", - "900": "73696465", - "901": "7272", - "902": "6374696f6e", - "903": "204d72", - "904": "617070", - "905": "6f76", - "906": "6d69676874", - "907": "7370", - "908": "207065", - "909": "756e646572", - "910": "6566", - "911": "667269656e64", - "912": "696365", - "913": "206261636b", - "914": "626f75", - "915": "20756e646572", - "916": "2077656e74", - "917": "6c6573", - "918": "6f70", - "919": "656e65", - "920": "646561", - "921": "616765", - "922": "206c6f6e67", - "923": "6c6f77", - "924": "206966", - "925": "20776974686f7574", - "926": "6761", - "927": "206167", - "928": "207465", - "929": "207768657265", - "930": "756e74", - "931": "697465", - "932": "6576656e", - "933": "2076", - "934": "61636b", - "935": "66756c", - "936": "696564", - "937": "73746f", - "938": "2061667465", - "939": "73746572", - "940": "6772", - "941": "20706c61", - "942": "206d7573", - "943": "6e6572", - "944": "696e6773", - "945": "7475726e", - "946": "75726e", - "947": "72656164", - "948": "6772656174", - "949": "726565", - "950": "72616e", - "951": "797468696e67", - "952": "657474", - "953": "6d757374", - "954": "64696e", - "955": "7374616e", - "956": "207468726f", - "957": "696c65", - "958": "686f77", - "959": "20706c", - "960": "696e7465", - "961": "636865", - "962": "746872", - "963": "736179", - "964": "70726573", - "965": "206d616465", - "966": "61747465", - "967": "7669", - "968": "20706c616365", - "969": "206d696768", - "970": "6574746572", - "971": "676f6f64", - "972": "7479", - "973": "207365656d", - "974": "706c61", - "975": "206472", - "976": "697368", - "977": "20636861", - "978": "6e6c79", - "979": "617973", - "980": "77656e74", - "981": "646f6e", - "982": "206669727374", - "983": "7070", - "984": "20686f77", - "985": "727365", - "986": "207573", - "987": "206e6f746869", - "988": "206576656e", - "989": "706c616365", - "990": "617373", - "991": "20667269656e", - "992": "6e7469", - "993": "646973", - "994": "2053", - "995": "616e67", - "996": "6d70", - "997": "656d65", - "998": "736861", - "999": "61626f", - "1000": "72657373", - "1001": "20616d", - "1002": "726174", - "1003": "20726573", - "1004": "20686173", - "1005": "5468657265", - "1006": "706c65", - "1007": "2063616d65", - "1008": "20646179", - "1009": "6e746572", - "1010": "2061776179", - "1011": "646564", - "1012": "207269676874", - "1013": "2067726561", - "1014": "206765", - "1015": "696e73", - "1016": "657365", - "1017": "206d6164", - "1018": "756e6465", - "1019": "697365", - "1020": "206e696768", - "1021": "63616d65", - "1022": "737065", - "1023": "206372", - "1024": "646179", - "1025": "74696c6c", - "1026": "4865", - "1027": "20676f6f", - "1028": "686564", - "1029": "74696f6e73", - "1030": "2073746f", - "1031": "737369", - "1032": "6f7374", - "1033": "6279", - "1034": "776172", - "1035": "6f7765", - "1036": "696f6e73", - "1037": "6b6e", - "1038": "206a757374", - "1039": "6f6b65", - "1040": "65747465", - "1041": "207365656d6564", - "1042": "2074616b65", - "1043": "616e6765", - "1044": "73656e", - "1045": "2068657265", - "1046": "667465", - "1047": "206f6666", - "1048": "6f756e74", - "1049": "746c79", - "1050": "206d7973656c66", - "1051": "726563", - "1052": "206d696e", - "1053": "20776179", - "1054": "73696f6e", - "1055": "6473", - "1056": "707065", - "1057": "706f73", - "1058": "68616c6c", - "1059": "20677265", - "1060": "20706572", - "1061": "2041", - "1062": "656c69", - "1063": "686f757365", - "1064": "74616b65", - "1065": "657269", - "1066": "736f6e", - "1067": "6f7065", - "1068": "6d75", - "1069": "6261636b", - "1070": "20676f74", - "1071": "616465", - "1072": "72657374", - "1073": "706172", - "1074": "6666", - "1075": "20756e6465", - "1076": "206d616b65", - "1077": "6f757273", - "1078": "20616e6f74686572", - "1079": "68616c", - "1080": "6c6c79", - "1081": "737365", - "1082": "7475726e6564", - "1083": "74696c", - "1084": "20636c", - "1085": "206c6561", - "1086": "677265", - "1087": "74616e", - "1088": "20686f75", - "1089": "7468726f", - "1090": "626574", - "1091": "676574", - "1092": "656570", - "1093": "74757265", - "1094": "6d6f7374", - "1095": "6578", - "1096": "20736f6d6574", - "1097": "6c6c6f77", - "1098": "6f7264", - "1099": "747265", - "1100": "717569", - "1101": "617474", - "1102": "726e696e67", - "1103": "20686f757365", - "1104": "616e797468696e67", - "1105": "6c657373", - "1106": "206e6967", - "1107": "636861", - "1108": "68656164", - "1109": "6d6564", - "1110": "616c6c79", - "1111": "69656e", - "1112": "2077656c6c", - "1113": "6c6c65", - "1114": "696f7573", - "1115": "20666f756e64", - "1116": "7069", - "1117": "73736564", - "1118": "64656e", - "1119": "207261", - "1120": "20666972", - "1121": "6177", - "1122": "206265696e67", - "1123": "20666174686572", - "1124": "77617264", - "1125": "696e6b", - "1126": "206c6f6e", - "1127": "206d6179", - "1128": "2077686572", - "1129": "686573", - "1130": "596f75", - "1131": "6f757365", - "1132": "656c79", - "1133": "20616e797468696e67", - "1134": "20626163", - "1135": "206166", - "1136": "7965", - "1137": "20427574", - "1138": "0a0a0a0a0a0a0a0a0a0a", - "1139": "79696e67", - "1140": "77696e", - "1141": "726574", - "1142": "766564", - "1143": "6261", - "1144": "6372", - "1145": "726e65", - "1146": "7175", - "1147": "207269", - "1148": "20676574", - "1149": "7973", - "1150": "2074656c6c", - "1151": "676572", - "1152": "666174686572", - "1153": "72696564", - "1154": "73747261", - "1155": "207468657365", - "1156": "7365656d", - "1157": "427574", - "1158": "20706c6163", - "1159": "2068656164", - "1160": "206f7572", - "1161": "77656c6c", - "1162": "686f7573", - "1163": "6f696e", - "1164": "20616c77617973", - "1165": "656e746c", - "1166": "636f6d70", - "1167": "727269", - "1168": "207475726e", - "1169": "2066616365", - "1170": "6c66", - "1171": "207175", - "1172": "666f756e64", - "1173": "6e6967", - "1174": "76696e67", - "1175": "686172", - "1176": "617272", - "1177": "757273", - "1178": "2074616b", - "1179": "68696c", - "1180": "70656e", - "1181": "6d6573", - "1182": "737572", - "1183": "2077616e74", - "1184": "697265", - "1185": "6272", - "1186": "206d6f726e696e67", - "1187": "20717569", - "1188": "205468657265", - "1189": "6a757374", - "1190": "6f6f72", - "1191": "20636f6d70", - "1192": "20746f6f6b", - "1193": "20696e7465", - "1194": "656374696f6e", - "1195": "2057", - "1196": "74656c", - "1197": "6369", - "1198": "7275", - "1199": "206d616b", - "1200": "20646f6f72", - "1201": "746f72", - "1202": "706f6e", - "1203": "207374696c6c", - "1204": "6e6773", - "1205": "69737465", - "1206": "727374", - "1207": "20636f6e73", - "1208": "20726563", - "1209": "6d7573", - "1210": "2042", - "1211": "6e6f74686572", - "1212": "20666163", - "1213": "726f6f6d", - "1214": "616972", - "1215": "6361757365", - "1216": "2066696e", - "1217": "637469", - "1218": "204974", - "1219": "206c617374", - "1220": "2063616d", - "1221": "20697473", - "1222": "636f756e74", - "1223": "7461696e", - "1224": "6e676572", - "1225": "646573", - "1226": "206561", - "1227": "2072696768", - "1228": "697479", - "1229": "2062656361757365", - "1230": "6674", - "1231": "2070617373", - "1232": "2062656c69657665", - "1233": "636c", - "1234": "20666f75", - "1235": "207468696e6773", - "1236": "6f6e64", - "1237": "2072656d", - "1238": "20656e6f756768", - "1239": "2064656172", - "1240": "73757265", - "1241": "686173", - "1242": "6c616e", - "1243": "206d6f6d656e74", - "1244": "6f75727365", - "1245": "2065796573", - "1246": "6d6f6e", - "1247": "6769", - "1248": "2068657273656c66", - "1249": "61707065", - "1250": "6574686572", - "1251": "6174746572", - "1252": "6e6174", - "1253": "636f6e73", - "1254": "74656c6c", - "1255": "6f6666", - "1256": "70617274", - "1257": "72656d", - "1258": "6577", - "1259": "6472", - "1260": "206c6574", - "1261": "6c6967", - "1262": "75726e6564", - "1263": "2073617973", - "1264": "6d73656c", - "1265": "6c6f6f6b696e67", - "1266": "737563", - "1267": "20726f6f6d", - "1268": "6d616b65", - "1269": "726973", - "1270": "20666f756e", - "1271": "207075", - "1272": "2074776f", - "1273": "20636172", - "1274": "6675", - "1275": "20616c6f6e", - "1276": "20657965", - "1277": "726573656e74", - "1278": "7273656c66", - "1279": "207965", - "1280": "2067697665", - "1281": "726174696f6e", - "1282": "206865617264", - "1283": "206d6f7374", - "1284": "706561", - "1285": "6175", - "1286": "6e6b", - "1287": "746f676574686572", - "1288": "2d2d", - "1289": "2070617274", - "1290": "62656c69657665", - "1291": "6d6f6d656e74", - "1292": "20756e64", - "1293": "746368", - "1294": "20616e73776572", - "1295": "61757365", - "1296": "20706172", - "1297": "206661746865", - "1298": "696e746572", - "1299": "70726573656e74", - "1300": "686170", - "1301": "696e6564", - "1302": "6e6473", - "1303": "20667269", - "1304": "697473", - "1305": "206865617274", - "1306": "20646573", - "1307": "6c6163", - "1308": "7475726e65", - "1309": "65696e67", - "1310": "70617373", - "1311": "20796f756e67", - "1312": "6c6965", - "1313": "6f6e6365", - "1314": "617070656172", - "1315": "206c69676874", - "1316": "647265", - "1317": "73656e74", - "1318": "656e6564", - "1319": "20657870", - "1320": "7374696c6c", - "1321": "66616365", - "1322": "746169", - "1323": "6f6c6c", - "1324": "7474696e67", - "1325": "20686f7573", - "1326": "20646f6e65", - "1327": "206a7573", - "1328": "206163", - "1329": "20626574746572", - "1330": "6f6d6574", - "1331": "61746564", - "1332": "65617274", - "1333": "69636b", - "1334": "6572696e67", - "1335": "6d6164", - "1336": "6f7272", - "1337": "2077656c", - "1338": "20536865", - "1339": "75726e65", - "1340": "7569", - "1341": "626f6479", - "1342": "656361757365", - "1343": "20626c", - "1344": "66696e", - "1345": "74686573", - "1346": "61726b", - "1347": "2074687265", - "1348": "206f776e", - "1349": "2066656c", - "1350": "6c6574", - "1351": "6f6b6564", - "1352": "20596f75", - "1353": "206177", - "1354": "7072657373", - "1355": "6c6173", - "1356": "206c6f6f6b696e67", - "1357": "68657273", - "1358": "61727269", - "1359": "726f77", - "1360": "736572", - "1361": "6865617274", - "1362": "726e6564", - "1363": "636172", - "1364": "20626567", - "1365": "6e746564", - "1366": "2062656c696576", - "1367": "72656e", - "1368": "20676f696e67", - "1369": "2068616e6473", - "1370": "696e657373", - "1371": "2061726f756e64", - "1372": "20617070", - "1373": "206f6c64", - "1374": "20706173", - "1375": "6c617374", - "1376": "616e73776572", - "1377": "2077616e", - "1378": "2062726f", - "1379": "6572696e", - "1380": "77616e74", - "1381": "697373", - "1382": "20666565", - "1383": "2068617070", - "1384": "204d697373", - "1385": "696e63", - "1386": "206875", - "1387": "696573", - "1388": "207477", - "1389": "666972", - "1390": "6f69", - "1391": "2070656f706c65", - "1392": "6365727461696e", - "1393": "6c656173", - "1394": "6d696e67", - "1395": "6f63", - "1396": "657261", - "1397": "696e7374", - "1398": "20626574", - "1399": "2072657475726e", - "1400": "2077696e", - "1401": "2070726573656e", - "1402": "20776f7264", - "1403": "6e746c", - "1404": "416e64", - "1405": "206a75", - "1406": "2074656c", - "1407": "6375", - "1408": "636573", - "1409": "67697665", - "1410": "20736c", - "1411": "6e6576", - "1412": "207475726e6564", - "1413": "20546f6d", - "1414": "726f6f", - "1415": "7363", - "1416": "20486f6c6d6573", - "1417": "656e7469", - "1418": "7373696f6e", - "1419": "616b696e67", - "1420": "6f756c646e", - "1421": "666f7274", - "1422": "646f6f72", - "1423": "20696e746572", - "1424": "207374696c", - "1425": "616374", - "1426": "666163", - "1427": "e28098", - "1428": "2062656361", - "1429": "2070726573", - "1430": "68657365", - "1431": "6f6f6b696e67", - "1432": "20737469", - "1433": "656e746c79", - "1434": "6f7373", - "1435": "6875", - "1436": "206d65616e", - "1437": "726d", - "1438": "2054686572", - "1439": "6c6174", - "1440": "2070726573656e74", - "1441": "6c73", - "1442": "627573696e657373", - "1443": "54686579", - "1444": "756e6465727374616e64", - "1445": "686f7574", - "1446": "69626c65", - "1447": "20686f7765766572", - "1448": "20736f6f6e", - "1449": "6e746c79", - "1450": "6c656e", - "1451": "656374696f", - "1452": "72657365", - "1453": "726473", - "1454": "2062656c", - "1455": "7565", - "1456": "536865", - "1457": "73686564", - "1458": "20737065", - "1459": "6961", - "1460": "206d6f6d656e", - "1461": "6e6e65", - "1462": "6f6f6d", - "1463": "20617070656172", - "1464": "206665656c", - "1465": "65766572797468696e67", - "1466": "206f70656e", - "1467": "636869", - "1468": "204c", - "1469": "707269", - "1470": "6166", - "1471": "736c656570", - "1472": "205768", - "1473": "6c6f6f6b696e", - "1474": "746963", - "1475": "656564", - "1476": "20747572", - "1477": "6f696e67", - "1478": "20626563", - "1479": "20627573696e657373", - "1480": "5768", - "1481": "6f676574686572", - "1482": "697273", - "1483": "206769", - "1484": "20776f726b", - "1485": "0a0a20", - "1486": "2077696e646f77", - "1487": "686f72", - "1488": "617761", - "1489": "697469", - "1490": "65617468", - "1491": "696e6573", - "1492": "746f6765746865", - "1493": "68617070", - "1494": "206c696665", - "1495": "2062726561", - "1496": "726974", - "1497": "6d6179", - "1498": "636f6e7369646572", - "1499": "70656f706c65", - "1500": "746174696f6e", - "1501": "686f7365", - "1502": "70656374", - "1503": "656574", - "1504": "20646f6f", - "1505": "617273", - "1506": "796f756e67", - "1507": "747275", - "1508": "657274", - "1509": "6d6f73", - "1510": "676c", - "1511": "206365727461696e", - "1512": "6479", - "1513": "57686174", - "1514": "696f75", - "1515": "706564", - "1516": "72657475726e", - "1517": "206d696e64", - "1518": "2073747261", - "1519": "74657265", - "1520": "20696d", - "1521": "6e7374", - "1522": "43484150544552", - "1523": "7072657365", - "1524": "207768696c65", - "1525": "6c6c69", - "1526": "2072656164", - "1527": "20636c6f", - "1528": "63616c", - "1529": "6d6973", - "1530": "61746368", - "1531": "64656172", - "1532": "6f7765766572", - "1533": "666665", - "1534": "68616e6473", - "1535": "206c6173", - "1536": "206b696e", - "1537": "2077696e64", - "1538": "736974", - "1539": "72617469", - "1540": "20636f6e7369646572", - "1541": "6e6e", - "1542": "20616e737765", - "1543": "207468726565", - "1544": "20736973746572", - "1545": "61747572", - "1546": "6c6963", - "1547": "6f6d656e", - "1548": "63686564", - "1549": "6c6c696e", - "1550": "20737472616e6765", - "1551": "207768696c", - "1552": "65616c", - "1553": "6665656c696e67", - "1554": "20676976", - "1555": "7374616e64", - "1556": "4974", - "1557": "666f75", - "1558": "6c6c6564", - "1559": "6d6574", - "1560": "616368", - "1561": "2061736b", - "1562": "656e6465", - "1563": "4d697373", - "1564": "65637469", - "1565": "20756e6465727374616e64", - "1566": "726174696f", - "1567": "5468656e", - "1568": "20737572", - "1569": "696e746572657374", - "1570": "6f776564", - "1571": "727468", - "1572": "20416e64", - "1573": "6e64696e67", - "1574": "74687265", - "1575": "636f7572", - "1576": "74616c", - "1577": "73747265", - "1578": "20746f7761726473", - "1579": "206c6574746572", - "1580": "206170706561", - "1581": "6f6c65", - "1582": "6f6d656e74", - "1583": "20696e73", - "1584": "206665656c696e67", - "1585": "726c", - "1586": "67696e", - "1587": "206b69", - "1588": "20626567616e", - "1589": "79696e", - "1590": "6e6c", - "1591": "2054686579", - "1592": "206b696e64", - "1593": "657870", - "1594": "20636f756e74", - "1595": "726169", - "1596": "677261", - "1597": "62726561", - "1598": "2044", - "1599": "77617973", - "1600": "2061736b6564", - "1601": "2065766572797468696e67", - "1602": "20746f676574686572", - "1603": "206d696e757465", - "1604": "57656c6c", - "1605": "6f636b", - "1606": "74726f", - "1607": "617365", - "1608": "65617273", - "1609": "206d6561", - "1610": "61726564", - "1611": "726b", - "1612": "7761726473", - "1613": "20686f7065", - "1614": "6f6e74", - "1615": "0a0a0a0a2020202020", - "1616": "727461696e", - "1617": "6e676c", - "1618": "6f6b696e67", - "1619": "696d70", - "1620": "61746572", - "1621": "20726570", - "1622": "20636f7572", - "1623": "20796f756e", - "1624": "20666f6c6c6f77", - "1625": "737472616e6765", - "1626": "6f726e", - "1627": "206c696768", - "1628": "6c6172", - "1629": "657965", - "1630": "706c656173", - "1631": "2077697368", - "1632": "206c6f7665", - "1633": "6563746564", - "1634": "206761", - "1635": "696e676c", - "1636": "656e74696f6e", - "1637": "206f6e6365", - "1638": "546f6d", - "1639": "72697665", - "1640": "696665", - "1641": "20736177", - "1642": "747269", - "1643": "2067656e746c65", - "1644": "6973746572", - "1645": "6d696c", - "1646": "20616c6f6e67", - "1647": "65737365", - "1648": "20636f6e74", - "1649": "7175657374696f6e", - "1650": "20706c656173", - "1651": "6c6c696e67", - "1652": "206d616e79", - "1653": "20696d70", - "1654": "206d656e", - "1655": "756e67", - "1656": "736973746572", - "1657": "20647265", - "1658": "20737465", - "1659": "20746f77", - "1660": "61756768", - "1661": "20696e746572657374", - "1662": "727279", - "1663": "2070656f706c", - "1664": "2074686f7365", - "1665": "706f737369626c", - "1666": "2072657374", - "1667": "62656361", - "1668": "63616c6c", - "1669": "20747261", - "1670": "74696d6573", - "1671": "20677261", - "1672": "706f737369626c65", - "1673": "20736c656570", - "1674": "207265747572", - "1675": "6665656c", - "1676": "74616e6365", - "1677": "706563", - "1678": "746f6e", - "1679": "206d616e6e6572", - "1680": "65737369", - "1681": "62726f", - "1682": "6d696e757465", - "1683": "2067656e", - "1684": "206c6966", - "1685": "2043", - "1686": "6f6f6e", - "1687": "726570", - "1688": "65616e", - "1689": "2048656c73696e67", - "1690": "63746564", - "1691": "7263", - "1692": "207175697465", - "1693": "20616761696e7374", - "1694": "6d626572", - "1695": "7369737465", - "1696": "74776f", - "1697": "656c6965", - "1698": "206d6174746572", - "1699": "636c6f", - "1700": "676564", - "1701": "7461626c65", - "1702": "616479", - "1703": "20546f", - "1704": "667269", - "1705": "20636c6f7365", - "1706": "2063616c6c", - "1707": "20666174", - "1708": "756e6465727374", - "1709": "756d", - "1710": "6e74657265", - "1711": "2073747265", - "1712": "6f70656e", - "1713": "616c6b", - "1714": "6573736564", - "1715": "72616e6765", - "1716": "61726473", - "1717": "736c", - "1718": "696e61", - "1719": "206164", - "1720": "65696e", - "1721": "6d696e64", - "1722": "5768656e", - "1723": "776f726b", - "1724": "20666c", - "1725": "686f6c", - "1726": "616e73", - "1727": "2067656e746c", - "1728": "67656e746c65", - "1729": "2072656d61", - "1730": "206c656674", - "1731": "666565", - "1732": "626572", - "1733": "70696e", - "1734": "207175657374696f6e", - "1735": "2066696e64", - "1736": "6c6f73", - "1737": "726765", - "1738": "206a", - "1739": "6174757265", - "1740": "746174", - "1741": "70657273", - "1742": "6e646564", - "1743": "6d65616e", - "1744": "2073616d65", - "1745": "6174696f6e73", - "1746": "206368696c64", - "1747": "666f6c6c6f77", - "1748": "2071756974", - "1749": "20746f6c64", - "1750": "20646172", - "1751": "637265", - "1752": "206d6973", - "1753": "656d656e74", - "1754": "20756e6465727374616e", - "1755": "67656e", - "1756": "73697374", - "1757": "20636f6e73696465", - "1758": "206265747765656e", - "1759": "6d73", - "1760": "20636572746169", - "1761": "20737472616e67", - "1762": "656176", - "1763": "2074616c6b", - "1764": "666572", - "1765": "796573", - "1766": "6e6469", - "1767": "2074616c", - "1768": "736869", - "1769": "204461726379", - "1770": "73657276", - "1771": "2063616c", - "1772": "66656c", - "1773": "206170", - "1774": "636f6d696e67", - "1775": "65616b", - "1776": "746f67657468", - "1777": "646966666572656e", - "1778": "70706564", - "1779": "2073746172", - "1780": "e2809ce28098", - "1781": "6c6f7665", - "1782": "206f70", - "1783": "75636b", - "1784": "61726e", - "1785": "6e6577", - "1786": "20617474", - "1787": "6176696e67", - "1788": "7474656e", - "1789": "65737469", - "1790": "20636869", - "1791": "77696e64", - "1792": "6973686564", - "1793": "20756e6465727374", - "1794": "77616e", - "1795": "6c75", - "1796": "2074726f75626c65", - "1797": "2042696e676c6579", - "1798": "204d6973", - "1799": "207069", - "1800": "61726765", - "1801": "70726973", - "1802": "205468656e", - "1803": "207363", - "1804": "6f61", - "1805": "636f75727365", - "1806": "206e61", - "1807": "656e73", - "1808": "20627265", - "1809": "6d6172", - "1810": "726976", - "1811": "20686170", - "1812": "6c696665", - "1813": "6b696e64", - "1814": "636f6e74", - "1815": "746174696f", - "1816": "205368", - "1817": "6567", - "1818": "72656173", - "1819": "656d61", - "1820": "0a0a0a0a20202020202020", - "1821": "6c617465", - "1822": "20707269", - "1823": "7564", - "1824": "72656d656d626572", - "1825": "74726f75626c65", - "1826": "706f7365", - "1827": "666f726d", - "1828": "656d656e", - "1829": "204c6f", - "1830": "20776174", - "1831": "6f6d6574696d6573", - "1832": "207369737465", - "1833": "657276", - "1834": "2072656d656d626572", - "1835": "206f7065", - "1836": "626f64", - "1837": "726163", - "1838": "746f77617264", - "1839": "2042656e6e6574", - "1840": "62656c", - "1841": "616b696e", - "1842": "65616368", - "1843": "20726f756e64", - "1844": "706f696e74", - "1845": "656d626572", - "1846": "69626c", - "1847": "206665656c696e", - "1848": "736573", - "1849": "67687465", - "1850": "72616e67", - "1851": "205765", - "1852": "20636f75727365", - "1853": "2063616e6e6f74", - "1854": "44656661726765", - "1855": "77696e67", - "1856": "656e7465", - "1857": "66656374", - "1858": "206461756768746572", - "1859": "636c6f7365", - "1860": "2073696465", - "1861": "656e696e67", - "1862": "7265616479", - "1863": "766573", - "1864": "2044656661726765", - "1865": "626567", - "1866": "6e7369", - "1867": "206579", - "1868": "616e6e", - "1869": "2077686f6c65", - "1870": "657373696f6e", - "1871": "6175676874", - "1872": "66666572", - "1873": "6e7473", - "1874": "20736f75", - "1875": "20686172", - "1876": "207365656e", - "1877": "207475", - "1878": "737765", - "1879": "20766f696365", - "1880": "20707269736f6e", - "1881": "2064697374", - "1882": "6368696c64", - "1883": "20626561", - "1884": "2068656c", - "1885": "74636865", - "1886": "72656c", - "1887": "68696c65", - "1888": "7472616e6765", - "1889": "20636f756e", - "1890": "756c646e", - "1891": "6368616e", - "1892": "61736b", - "1893": "61726d", - "1894": "7075", - "1895": "496c6c757374726174696f6e", - "1896": "20636f6d696e67", - "1897": "207061737365", - "1898": "20696e7465726573", - "1899": "2067656e746c656d", - "1900": "636175", - "1901": "73706563", - "1902": "2073656e", - "1903": "206f77", - "1904": "65656c", - "1905": "7477", - "1906": "20707574", - "1907": "6f6479", - "1908": "617269", - "1909": "2061707065", - "1910": "20416e", - "1911": "20776973", - "1912": "707269736f6e", - "1913": "61756768746572", - "1914": "20706173736564", - "1915": "726f73", - "1916": "206368696c", - "1917": "636564", - "1918": "626563", - "1919": "20737065616b", - "1920": "6c74", - "1921": "656e7473", - "1922": "6e6f75", - "1923": "207669", - "1924": "69736865", - "1925": "20776f6e646572", - "1926": "686f73", - "1927": "756c6c", - "1928": "204d69", - "1929": "206d696e7574", - "1930": "206461726b", - "1931": "2079656172", - "1932": "727461", - "1933": "666172", - "1934": "54686174", - "1935": "65727365", - "1936": "66756c6c", - "1937": "726f6e", - "1938": "636173", - "1939": "206368616e", - "1940": "204d61", - "1941": "696e676c65", - "1942": "6773", - "1943": "61737365", - "1944": "20796574", - "1945": "726573736564", - "1946": "6f737369626c65", - "1947": "7369626c65", - "1948": "20666f6c6c6f", - "1949": "2073696c656e", - "1950": "206e656172", - "1951": "6974696f6e", - "1952": "736174", - "1953": "206d6f74686572", - "1954": "20486f", - "1955": "617279", - "1956": "5765", - "1957": "20706c6561", - "1958": "20636c6f73", - "1959": "20686f7572", - "1960": "20616363", - "1961": "61696e6564", - "1962": "6d6f74686572", - "1963": "7370656374", - "1964": "686f7065", - "1965": "207175657374696f", - "1966": "7465726564", - "1967": "68656c", - "1968": "6f62", - "1969": "20656e74", - "1970": "736f6c", - "1971": "616c6f", - "1972": "7074", - "1973": "616c6c65", - "1974": "20656172", - "1975": "74657273", - "1976": "69766572", - "1977": "20706f6f72", - "1978": "72616e6365", - "1979": "657074", - "1980": "6c697665", - "1981": "69657665", - "1982": "204a", - "1983": "20736c6565", - "1984": "2066656c6c", - "1985": "206469646e", - "1986": "717565", - "1987": "67656e746c656d616e", - "1988": "776564", - "1989": "6e6174696f6e", - "1990": "2068617264", - "1991": "2073757070", - "1992": "2061747465", - "1993": "20696e7374", - "1994": "636f6d7061", - "1995": "20596f", - "1996": "207772", - "1997": "6d6174", - "1998": "204c6f727279", - "1999": "65787072657373", - "2000": "72656d61", - "2001": "656361", - "2002": "696d73", - "2003": "61727269616765", - "2004": "70696e67", - "2005": "77697368", - "2006": "7374616e6365", - "2007": "71756974", - "2008": "7472616e", - "2009": "6f77696e67", - "2010": "2074696c6c", - "2011": "62726f75676874", - "2012": "20686f70", - "2013": "206c6f76", - "2014": "656365", - "2015": "7661", - "2016": "7261626c65", - "2017": "6e7365", - "2018": "70656e6564", - "2019": "207265616c", - "2020": "2062726f75676874", - "2021": "206b656570", - "2022": "2077616c", - "2023": "637465", - "2024": "776565", - "2025": "776f6e646572", - "2026": "73746f6f64", - "2027": "20676c", - "2028": "2068757362616e64", - "2029": "2068616c66", - "2030": "617474656e", - "2031": "6d706c", - "2032": "6964656e", - "2033": "7461626c", - "2034": "696c79", - "2035": "6d62", - "2036": "7061737365", - "2037": "72616c", - "2038": "6f6e6465", - "2039": "68726565", - "2040": "656e746c65", - "2041": "746f77", - "2042": "20617474656e", - "2043": "636b65", - "2044": "646e", - "2045": "65727461", - "2046": "2074726f75626c", - "2047": "74656e74696f6e", - "2048": "696d6573", - "2049": "20416c696365", - "2050": "656974686572", - "2051": "20776f726c64", - "2052": "6974686572", - "2053": "6f6d696e67", - "2054": "2067656e746c656d616e", - "2055": "20726574", - "2056": "616e6b", - "2057": "20736d", - "2058": "7375727072697365", - "2059": "65617265", - "2060": "63746f72", - "2061": "73746172", - "2062": "74726565", - "2063": "206b65", - "2064": "74616c6b", - "2065": "20657863", - "2066": "657268617073", - "2067": "20626568696e64", - "2068": "72696f7573", - "2069": "61696e73", - "2070": "206d6172", - "2071": "2073696e", - "2072": "6170706561726564", - "2073": "207761746368", - "2074": "20616e737765726564", - "2075": "206b6e6577", - "2076": "2065787065", - "2077": "61726c", - "2078": "757369", - "2079": "6f726d", - "2080": "2072656d61696e", - "2081": "7374696e", - "2082": "696e6365", - "2083": "74656e74", - "2084": "2072656d656d6265", - "2085": "2065787072657373", - "2086": "20636f756e747279", - "2087": "2066656172", - "2088": "7561", - "2089": "2073746f6f64", - "2090": "20656e64", - "2091": "7374656e", - "2092": "54686973", - "2093": "617474656e74696f6e", - "2094": "206b6e65", - "2095": "20696e64", - "2096": "6e6e6572", - "2097": "7573696e", - "2098": "6a75", - "2099": "207374617274", - "2100": "6f757273656c", - "2101": "207768697465", - "2102": "416c696365", - "2103": "6d6561", - "2104": "6c696576", - "2105": "736f6d6574696d6573", - "2106": "207375727072697365", - "2107": "20707269736f6e6572", - "2108": "2072657475726e6564", - "2109": "206c65617665", - "2110": "2066616d696c", - "2111": "6e646f", - "2112": "686f6d", - "2113": "757465", - "2114": "617263", - "2115": "6368616e6765", - "2116": "796f757273656c66", - "2117": "666c", - "2118": "617475", - "2119": "706172746963756c6172", - "2120": "64697374", - "2121": "67656e74", - "2122": "636f6d696e", - "2123": "6666657265", - "2124": "636c65", - "2125": "68757362616e64", - "2126": "20736174", - "2127": "6b6e65", - "2128": "20796f757273656c66", - "2129": "737065616b", - "2130": "696c64", - "2131": "626a656374", - "2132": "205768656e", - "2133": "72617465", - "2134": "20446172", - "2135": "616e65", - "2136": "72746169", - "2137": "20686f6d65", - "2138": "657870656374", - "2139": "726561736f6e", - "2140": "20636f6d7061", - "2141": "2073756464656e", - "2142": "20706f73", - "2143": "6f6e646572", - "2144": "2066656c74", - "2145": "206c656176", - "2146": "776973", - "2147": "6f6e7369", - "2148": "65617665", - "2149": "74617469", - "2150": "20737570", - "2151": "6f7273", - "2152": "207275", - "2153": "636f756e747279", - "2154": "20696e64656564", - "2155": "2066616d696c79", - "2156": "20676f6e65", - "2157": "20736c65", - "2158": "706f7274", - "2159": "20776172", - "2160": "637269", - "2161": "20747275", - "2162": "6e73696465", - "2163": "6e61747572", - "2164": "7272696564", - "2165": "65726e", - "2166": "66696e64", - "2167": "20736574", - "2168": "64657273", - "2169": "686f7765", - "2170": "65637465", - "2171": "7267", - "2172": "6e7472", - "2173": "747279", - "2174": "6c6c6f776564", - "2175": "65636f", - "2176": "75726564", - "2177": "6179696e67", - "2178": "7175657374", - "2179": "756974", - "2180": "656e746c656d", - "2181": "74616e63", - "2182": "706f7373", - "2183": "66657265", - "2184": "746561", - "2185": "666174", - "2186": "20776f726473", - "2187": "20776f756c646e", - "2188": "20726561736f6e", - "2189": "6f6d696e", - "2190": "736f72", - "2191": "65617465", - "2192": "20706f696e74", - "2193": "206c697665", - "2194": "757269", - "2195": "636572", - "2196": "646465", - "2197": "6c616e64", - "2198": "65616479", - "2199": "6775", - "2200": "20686176696e67", - "2201": "6c696365", - "2202": "73746564", - "2203": "6f6d69", - "2204": "206169", - "2205": "20616d6f6e67", - "2206": "6c6179", - "2207": "67696e67", - "2208": "7265737365", - "2209": "636f766572", - "2210": "207265636b6f6e", - "2211": "2068616c", - "2212": "6f776572", - "2213": "2073686f77", - "2214": "746f676574", - "2215": "7265737369", - "2216": "726f7373", - "2217": "61737465", - "2218": "736177", - "2219": "70657268617073", - "2220": "2073706f6b65", - "2221": "206372696564", - "2222": "707265737369", - "2223": "6666656374", - "2224": "20776f6e", - "2225": "6b656e", - "2226": "20736f6d6574696d6573", - "2227": "2063617272", - "2228": "656d62", - "2229": "656163", - "2230": "756e63", - "2231": "656173757265", - "2232": "206e61747572", - "2233": "20636f756c646e", - "2234": "207374726f6e67", - "2235": "7369626c", - "2236": "736c65", - "2237": "206675", - "2238": "646172", - "2239": "6d6572", - "2240": "72656d61696e", - "2241": "20707269736f", - "2242": "73617469", - "2243": "6c6f7765", - "2244": "74697665", - "2245": "6f75626c65", - "2246": "6e74696e", - "2247": "73706f", - "2248": "2071", - "2249": "20656e746572", - "2250": "20657870656374", - "2251": "2064726561", - "2252": "65787065", - "2253": "72616e63", - "2254": "73756464656e", - "2255": "65636b", - "2256": "6d7973", - "2257": "2070657268617073", - "2258": "636f6d6d", - "2259": "616e676572", - "2260": "746573", - "2261": "6c6169", - "2262": "20616c6d6f7374", - "2263": "20736f756e64", - "2264": "6461726b", - "2265": "65617365", - "2266": "2067726f", - "2267": "20666172", - "2268": "6a656374", - "2269": "636f6e766572736174696f6e", - "2270": "6f7274756e", - "2271": "20726f756e", - "2272": "646c79", - "2273": "73747275", - "2274": "666572656e", - "2275": "746172", - "2276": "20617474656e74696f6e", - "2277": "20726f75", - "2278": "20706172746963756c6172", - "2279": "646966666572656e74", - "2280": "20776f6d616e", - "2281": "416e", - "2282": "63697263756d7374616e6365", - "2283": "2063617265", - "2284": "20436f", - "2285": "20666561", - "2286": "206d6174", - "2287": "69766564", - "2288": "686f70", - "2289": "20737570706f7365", - "2290": "206368616e6765", - "2291": "20637269", - "2292": "657272", - "2293": "20706c65", - "2294": "6f727279", - "2295": "6b656570", - "2296": "2077616c6b", - "2297": "6f63746f72", - "2298": "6176696e", - "2299": "75697465", - "2300": "6f7562", - "2301": "20776f6e6465", - "2302": "6f7468657273", - "2303": "6b6e6f776c65646765", - "2304": "7761746368", - "2305": "204c6f72", - "2306": "72726f", - "2307": "20707265747479", - "2308": "7374617274", - "2309": "2077726974", - "2310": "20626f7468", - "2311": "6e656172", - "2312": "616e69", - "2313": "20706f737369626c", - "2314": "646765", - "2315": "656c73", - "2316": "7374726f", - "2317": "206e616d65", - "2318": "6561726564", - "2319": "7463686564", - "2320": "2077616974", - "2321": "79626f6479", - "2322": "20667269656e6473", - "2323": "20736967", - "2324": "657474696e67", - "2325": "207761746572", - "2326": "657261626c65", - "2327": "6173736564", - "2328": "2073706561", - "2329": "20676c61", - "2330": "206c6976", - "2331": "616d69", - "2332": "746f6c", - "2333": "616363", - "2334": "206d6f746865", - "2335": "68617264", - "2336": "6f696365", - "2337": "75736564", - "2338": "2073757270726973", - "2339": "7761746572", - "2340": "2070657273", - "2341": "206d6f6e", - "2342": "696e69", - "2343": "63616c6c6564", - "2344": "20636f6e63", - "2345": "686f6c65", - "2346": "736f75", - "2347": "6f70706564", - "2348": "73746f6f", - "2349": "7261626c", - "2350": "6c6164", - "2351": "63656e", - "2352": "6970", - "2353": "7265616c", - "2354": "20736561", - "2355": "20646966666572656e", - "2356": "2068656c70", - "2357": "20636f6e66", - "2358": "2062726f74686572", - "2359": "206361727269", - "2360": "7374726f6e67", - "2361": "61696e74", - "2362": "79656172", - "2363": "6c6f776564", - "2364": "637572", - "2365": "676c61", - "2366": "69676e", - "2367": "6e74656e", - "2368": "686f7572", - "2369": "636f6d70616e", - "2370": "7562", - "2371": "20636f756e7472", - "2372": "636f6d706c", - "2373": "69747465", - "2374": "726f61", - "2375": "20757365", - "2376": "2073706f", - "2377": "20706572736f6e", - "2378": "2068757362616e", - "2379": "6572616c", - "2380": "68616e67", - "2381": "6269", - "2382": "736b", - "2383": "697374656e", - "2384": "7373696e67", - "2385": "7374616e74", - "2386": "727465", - "2387": "20666f6c", - "2388": "74616e74", - "2389": "726566", - "2390": "6962", - "2391": "62726f74686572", - "2392": "2063616c6c6564", - "2393": "636f6d666f7274", - "2394": "72746564", - "2395": "68616c66", - "2396": "63617265", - "2397": "6c6973", - "2398": "2073696e6365", - "2399": "73757070", - "2400": "776f726c64", - "2401": "6c6f636b", - "2402": "6e6561", - "2403": "7463", - "2404": "207269766572", - "2405": "6472657373", - "2406": "7269766572", - "2407": "206475", - "2408": "6475", - "2409": "2073696c656e6365", - "2410": "646565", - "2411": "20636f6d6d", - "2412": "4e6f", - "2413": "757274", - "2414": "636f72", - "2415": "20736f72", - "2416": "706f6f72", - "2417": "6e64656564", - "2418": "2077686974", - "2419": "7768697465", - "2420": "6365697665", - "2421": "77616c", - "2422": "626561", - "2423": "617567", - "2424": "206f62", - "2425": "66656c6c", - "2426": "66656172", - "2427": "6465726564", - "2428": "75616c", - "2429": "6c6f6e65", - "2430": "206b6565", - "2431": "20737570706f", - "2432": "207965617273", - "2433": "20696e7374616e74", - "2434": "636f6e74696e75", - "2435": "61707065617265", - "2436": "70706561726564", - "2437": "20706f6f", - "2438": "20576865", - "2439": "64656e74", - "2440": "6464", - "2441": "20736d616c6c", - "2442": "206e6561", - "2443": "64696666", - "2444": "676c65", - "2445": "20726174686572", - "2446": "20696e7374616e", - "2447": "20626f74", - "2448": "707574", - "2449": "6363", - "2450": "20646f756274", - "2451": "7265636b6f6e", - "2452": "7365727665", - "2453": "20776f6d", - "2454": "20657870726573", - "2455": "74656e64", - "2456": "6c6976", - "2457": "727469", - "2458": "6f766564", - "2459": "204c7563", - "2460": "207265616c6c79", - "2461": "20747269", - "2462": "656e696e", - "2463": "6f7070", - "2464": "2073757265", - "2465": "726465", - "2466": "65727461696e6c79", - "2467": "20616c7265616479", - "2468": "657874", - "2469": "6e7374616e74", - "2470": "2050726f666573736f72", - "2471": "2064696666", - "2472": "776f6d616e", - "2473": "207761726e", - "2474": "656e746572", - "2475": "20736964", - "2476": "706174", - "2477": "696361", - "2478": "736574", - "2479": "696e6465", - "2480": "636f6e63", - "2481": "546f", - "2482": "20636f6e74696e75", - "2483": "7072657373696f6e", - "2484": "4a6f6e617468616e", - "2485": "207061706572", - "2486": "20796f757273", - "2487": "2064656164", - "2488": "7073", - "2489": "2066656c6c6f77", - "2490": "206375", - "2491": "6f6e61", - "2492": "206465617468", - "2493": "70656e65", - "2494": "656761", - "2495": "646c", - "2496": "706c656173757265", - "2497": "616666656374696f6e", - "2498": "206f7468657273", - "2499": "736f756e64", - "2500": "20706f7373", - "2501": "646f756274", - "2502": "6e6174696f", - "2503": "6d697365", - "2504": "20627573", - "2505": "6e617469", - "2506": "6e7461", - "2507": "72696f", - "2508": "6c656172", - "2509": "20686f72", - "2510": "7061706572", - "2511": "61696c", - "2512": "656e67", - "2513": "2077697468696e", - "2514": "2072656d6169", - "2515": "20646f7562", - "2516": "657261626c", - "2517": "207265706c696564", - "2518": "20616c6f6e65", - "2519": "204d7273", - "2520": "726c79", - "2521": "206576656e696e67", - "2522": "73706f6b65", - "2523": "204265", - "2524": "20627269", - "2525": "20626573", - "2526": "20706172746963756c61", - "2527": "20706f737369626c65", - "2528": "756464656e6c79", - "2529": "6172696e67", - "2530": "6964656e74", - "2531": "2064697363", - "2532": "77726974", - "2533": "20636173", - "2534": "61696e65", - "2535": "20747265", - "2536": "2068696768", - "2537": "737069", - "2538": "2072656d61726b", - "2539": "73696c656e6365", - "2540": "65617374", - "2541": "617272696564", - "2542": "2073686f72", - "2543": "72656d65", - "2544": "6f737369", - "2545": "616e6e65", - "2546": "76657279626f6479", - "2547": "20446f63746f72", - "2548": "617669", - "2549": "706572736f6e", - "2550": "6d61726b", - "2551": "6c656e74", - "2552": "20626f79", - "2553": "20796561", - "2554": "20666f6c6c6f776564", - "2555": "20776f726c", - "2556": "20646f6573", - "2557": "2057686174", - "2558": "6d656e7473", - "2559": "7472656574", - "2560": "756573", - "2561": "736967", - "2562": "657266", - "2563": "766f", - "2564": "206365727461696e6c79", - "2565": "20616666656374696f6e", - "2566": "2050", - "2567": "5368", - "2568": "6569746865", - "2569": "76696c", - "2570": "207375646465", - "2571": "20636f6e6365", - "2572": "20636f6e76", - "2573": "2063617365", - "2574": "2054686973", - "2575": "76656e74", - "2576": "6f726473", - "2577": "726f70", - "2578": "657369", - "2579": "207377", - "2580": "696e75", - "2581": "2068617070656e", - "2582": "20646972656374", - "2583": "696e7374616e74", - "2584": "636f6e736964657261", - "2585": "206365", - "2586": "656e69", - "2587": "696c69", - "2588": "69736564", - "2589": "2063697263756d7374616e6365", - "2590": "7468656d73656c766573", - "2591": "2077617463", - "2592": "686f6d65", - "2593": "69736f6e", - "2594": "617065", - "2595": "6665656c696e6773", - "2596": "726f666573736f72", - "2597": "206368616e67", - "2598": "61727465", - "2599": "74656c79", - "2600": "6572656e", - "2601": "2ee28099", - "2602": "7374727563", - "2603": "207265636f", - "2604": "6d7061", - "2605": "206e6577", - "2606": "737570706f73", - "2607": "636f6e6365", - "2608": "687572", - "2609": "746965", - "2610": "73756464656e6c79", - "2611": "72617665", - "2612": "636f6e66", - "2613": "616e6f", - "2614": "20636f6e766572736174696f6e", - "2615": "206f70656e6564", - "2616": "206f6273657276", - "2617": "20726561736f", - "2618": "656c6c6f77", - "2619": "20666f726d", - "2620": "7572696e67", - "2621": "77616974", - "2622": "6c657474", - "2623": "66666963", - "2624": "64726561", - "2625": "74656e6564", - "2626": "666f7274756e", - "2627": "77686574686572", - "2628": "20736563", - "2629": "20626c61", - "2630": "20706c656173757265", - "2631": "2073756464656e6c79", - "2632": "636861726163746572", - "2633": "7468657273", - "2634": "6e6368", - "2635": "2073696c", - "2636": "20756e74696c", - "2637": "20706f776572", - "2638": "626564", - "2639": "2077686574686572", - "2640": "6174696e67", - "2641": "6d696c65", - "2642": "73686f77", - "2643": "636f6e64", - "2644": "65727665", - "2645": "204e6f", - "2646": "657863", - "2647": "726f66657373", - "2648": "207265636b6f", - "2649": "65737465", - "2650": "6f72726f", - "2651": "66756c6c79", - "2652": "656e646564", - "2653": "646973636f766572", - "2654": "636f7665726564", - "2655": "6f77696e", - "2656": "616c6c6564", - "2657": "69656e6365", - "2658": "67726f", - "2659": "6e616c", - "2660": "636b6564", - "2661": "76656c", - "2662": "617070656172616e6365", - "2663": "206170706561726564", - "2664": "7468657265666f7265", - "2665": "6c6c6f7765", - "2666": "207374726f6e", - "2667": "6464656e", - "2668": "77616c6b", - "2669": "6361727269", - "2670": "636174", - "2671": "6e6574", - "2672": "2045", - "2673": "6e616d65", - "2674": "63726561", - "2675": "7377", - "2676": "2073746f70706564", - "2677": "6f63636173696f6e", - "2678": "70706f", - "2679": "68616972", - "2680": "666963", - "2681": "206578706563", - "2682": "20636f6d706c", - "2683": "2054686174", - "2684": "20726174", - "2685": "20777269", - "2686": "6e697368", - "2687": "697a", - "2688": "62616e", - "2689": "20666577", - "2690": "746f6f64", - "2691": "6172726965", - "2692": "20636f6e736964657261", - "2693": "63726f7373", - "2694": "206163636f", - "2695": "646972656374", - "2696": "63617272", - "2697": "206c6164", - "2698": "206269", - "2699": "206665656c696e6773", - "2700": "646966666963756c74", - "2701": "68617070696e657373", - "2702": "206c697374656e", - "2703": "2077616e746564", - "2704": "20636f6d70616e", - "2705": "6967687465", - "2706": "206d617272", - "2707": "68656c70", - "2708": "726f6e67", - "2709": "20646966666963756c74", - "2710": "20637265", - "2711": "2070726574", - "2712": "2077686973706572", - "2713": "20636f6d666f7274", - "2714": "696e6174696f6e", - "2715": "636f6e76", - "2716": "72656d61726b", - "2717": "204a6f6e617468616e", - "2718": "20706f696e", - "2719": "616b69", - "2720": "61726c79", - "2721": "73686f72", - "2722": "206d6f74", - "2723": "697369", - "2724": "72756e", - "2725": "20636861726163746572", - "2726": "2067656e6572616c", - "2727": "2062726f746865", - "2728": "2063686172", - "2729": "207374726f", - "2730": "20667265", - "2731": "63756c", - "2732": "63617365", - "2733": "73746174", - "2734": "20617373", - "2735": "20636f6c", - "2736": "64697363", - "2737": "6f6573", - "2738": "68617070656e", - "2739": "67726f756e64", - "2740": "20776169", - "2741": "72697365", - "2742": "206c617465", - "2743": "7277617264", - "2744": "64657265", - "2745": "616c6b696e67", - "2746": "656c64", - "2747": "6576657279626f6479", - "2748": "2073706f6b", - "2749": "2073746174", - "2750": "75646465", - "2751": "67616e", - "2752": "206d696e75746573", - "2753": "636f6e74696e7565", - "2754": "686169", - "2755": "687465", - "2756": "6f6273657276", - "2757": "206c6174", - "2758": "636f6d69", - "2759": "20626568", - "2760": "63657373", - "2761": "616e6e6f", - "2762": "6861707079", - "2763": "20636f6e74696e", - "2764": "2073696c656e63", - "2765": "77686973706572", - "2766": "626f7468", - "2767": "72646572", - "2768": "20536f", - "2769": "736174696f6e", - "2770": "6c697374656e", - "2771": "73656c766573", - "2772": "737472656574", - "2773": "656d61696e", - "2774": "20686f6d", - "2775": "6368696e", - "2776": "74617465", - "2777": "20416c", - "2778": "656e6f", - "2779": "6465617468", - "2780": "20656e7465", - "2781": "20666169", - "2782": "20717569636b", - "2783": "736174697366", - "2784": "207468657265666f7265", - "2785": "656e6572", - "2786": "746f6e65", - "2787": "627265616b66617374", - "2788": "20736f756e", - "2789": "6e67696e67", - "2790": "656d70", - "2791": "2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e", - "2792": "207468656d73656c766573", - "2793": "20737069726974", - "2794": "737069726974", - "2795": "656365697665", - "2796": "73746f72", - "2797": "6d696e65", - "2798": "72726f77", - "2799": "636f6c", - "2800": "706f776572", - "2801": "73696c", - "2802": "61727469", - "2803": "7261696e", - "2804": "6d6f75", - "2805": "737570", - "2806": "74686f7573616e64", - "2807": "2072656365697665", - "2808": "7374616e64696e67", - "2809": "20436f6c6c696e73", - "2810": "2062656175746966756c", - "2811": "206f63636173696f6e", - "2812": "206e6967676572", - "2813": "616c6b6564", - "2814": "2077686f6d", - "2815": "6c697479", - "2816": "657465", - "2817": "6179696e", - "2818": "6172746564", - "2819": "736d616c6c", - "2820": "666965", - "2821": "2061696e", - "2822": "20776f6d61", - "2823": "204a616e65", - "2824": "757468", - "2825": "20706572736f", - "2826": "6163636f", - "2827": "2073756e", - "2828": "70726573736564", - "2829": "6974696e67", - "2830": "657669", - "2831": "446172", - "2832": "6372656174757265", - "2833": "207374616e64", - "2834": "20666f6f", - "2835": "436174686572696e65", - "2836": "6561726c79", - "2837": "2073656e73", - "2838": "6f7469", - "2839": "20646966666572656e74", - "2840": "2068617070696e657373", - "2841": "20686172646c79", - "2842": "20616666656374", - "2843": "20726563656976", - "2844": "486f", - "2845": "657474696e", - "2846": "6368696e67", - "2847": "2072697665", - "2848": "686f6f", - "2849": "717561", - "2850": "204461", - "2851": "6e617475", - "2852": "756e7469", - "2853": "20636f6e74696e7565", - "2854": "20616666656374696f", - "2855": "5769636b68616d", - "2856": "206e657874", - "2857": "696666", - "2858": "697669", - "2859": "20436174686572696e65", - "2860": "73686f756c646572", - "2861": "63686172", - "2862": "20636c6561", - "2863": "657469", - "2864": "616974", - "2865": "616666656374", - "2866": "6e61747572616c", - "2867": "2067726f756e64", - "2868": "74656e65", - "2869": "73656e73", - "2870": "63726f", - "2871": "2074686f7573616e64", - "2872": "2073746f70", - "2873": "2077617465", - "2874": "6c656e6365", - "2875": "627265616b", - "2876": "736d", - "2877": "6f707065", - "2878": "205769636b68616d", - "2879": "2067656e6572", - "2880": "2073686f7274", - "2881": "206576657279626f6479", - "2882": "70657266656374", - "2883": "20737472656574", - "2884": "7279696e67", - "2885": "74616b656e", - "2886": "6c656374", - "2887": "646965", - "2888": "666561", - "2889": "746162", - "2890": "736962", - "2891": "206265617574", - "2892": "636569766564", - "2893": "72656e74", - "2894": "20636c65", - "2895": "74726561", - "2896": "2073686f756c646572", - "2897": "206372656174757265", - "2898": "636f6e74696e756564", - "2899": "63616c6c65", - "2900": "6976696e67", - "2901": "2068656176", - "2902": "207375626a656374", - "2903": "206163636f756e74", - "2904": "67656e6572616c", - "2905": "6163636f756e74", - "2906": "61737369", - "2907": "4875636b", - "2908": "6879", - "2909": "697474696e67", - "2910": "76656e696e67", - "2911": "61766f7572", - "2912": "726162", - "2913": "6f7773", - "2914": "20627265616b66617374", - "2915": "20626c6f", - "2916": "73746561", - "2917": "666f6c", - "2918": "7374696e67", - "2919": "2065616368", - "2920": "2064617973", - "2921": "74657272", - "2922": "2072656c", - "2923": "75626c", - "2924": "736561", - "2925": "20627265616b", - "2926": "746f6765", - "2927": "6963756c", - "2928": "6d6d", - "2929": "7665726564", - "2930": "727265", - "2931": "6f67", - "2932": "656174757265", - "2933": "206e6174", - "2934": "69746564", - "2935": "656c74", - "2936": "20646973636f766572", - "2937": "6f72726f77", - "2938": "20746f776e", - "2939": "20636f72", - "2940": "7761726e", - "2941": "6e66", - "2942": "207061727469", - "2943": "72656365697665", - "2944": "2062657374", - "2945": "63696f7573", - "2946": "74726169", - "2947": "63656e74", - "2948": "776f6e", - "2949": "7969", - "2950": "626c69", - "2951": "6e6973", - "2952": "61726f", - "2953": "206c6f73", - "2954": "206361727269616765", - "2955": "636f6d70616e696f6e", - "2956": "2064656c69", - "2957": "6f7465", - "2958": "67657474696e67", - "2959": "206e6f74696365", - "2960": "7375626a656374", - "2961": "20626563616d65", - "2962": "76657273", - "2963": "616b656e", - "2964": "6d6f74", - "2965": "204861", - "2966": "7265617465", - "2967": "204875636b", - "2968": "666669", - "2969": "6c6174696f6e", - "2970": "63697263756d7374616e636573", - "2971": "2067656e657261", - "2972": "70726f6d697365", - "2973": "20627265617468", - "2974": "207265706c", - "2975": "68696768", - "2976": "64656164", - "2977": "656469", - "2978": "636f6e736964657261626c", - "2979": "206861707079", - "2980": "727573", - "2981": "6f727468", - "2982": "61726b65", - "2983": "616e7465", - "2984": "69656e74", - "2985": "206c6179", - "2986": "20616666", - "2987": "2067657474696e67", - "2988": "6578706563746564", - "2989": "65616d", - "2990": "627269", - "2991": "20656974686572", - "2992": "20736174697366", - "2993": "74726f75", - "2994": "72616374", - "2995": "20696e63", - "2996": "6f75726e", - "2997": "65726573", - "2998": "686974", - "2999": "20446f", - "3000": "20636c656172", - "3001": "6c7665", - "3002": "706f736974696f6e", - "3003": "2070726f6d697365", - "3004": "204c6f6e646f6e", - "3005": "666572656e6365", - "3006": "20666f7274", - "3007": "20736d616c", - "3008": "7061727469", - "3009": "6d61696e", - "3010": "2061726d", - "3011": "756c61", - "3012": "676765", - "3013": "696c6c69", - "3014": "7265636f", - "3015": "6174656c79", - "3016": "206769726c", - "3017": "696e696e67", - "3018": "2073746570", - "3019": "2072656d6172", - "3020": "2074696d6573", - "3021": "65787072657373696f6e", - "3022": "6578707265737369", - "3023": "726368", - "3024": "204361", - "3025": "73616e", - "3026": "666169", - "3027": "6172696e", - "3028": "20736572", - "3029": "206d61727269", - "3030": "6c6f75", - "3031": "6379", - "3032": "4c6f", - "3033": "20646966666963756c", - "3034": "6e6572616c", - "3035": "20776f6f64", - "3036": "6f6674656e", - "3037": "6174636865", - "3038": "206e616d", - "3039": "656e7365", - "3040": "7368696e", - "3041": "6f727365", - "3042": "2068756e", - "3043": "616c69", - "3044": "4c6f72", - "3045": "6368696c6472656e", - "3046": "2076696c6c616765", - "3047": "20626c61636b", - "3048": "2069646561", - "3049": "69646e", - "3050": "676f6e", - "3051": "627573", - "3052": "6c6d", - "3053": "206368617261637465", - "3054": "207175696574", - "3055": "2065786365", - "3056": "6f7474", - "3057": "79626f64", - "3058": "6e737461", - "3059": "726f7573", - "3060": "7265636b", - "3061": "657370", - "3062": "7276", - "3063": "2047", - "3064": "61667465727761726473", - "3065": "2073747275", - "3066": "616c6c6f77", - "3067": "2064656174", - "3068": "717569636b", - "3069": "69726564", - "3070": "65746368", - "3071": "206861707065", - "3072": "206469726563", - "3073": "2070657266656374", - "3074": "77617465", - "3075": "20616464", - "3076": "636f6e7665", - "3077": "e28094e280", - "3078": "67656e6572", - "3079": "697468696e", - "3080": "20636f6e736964657261626c", - "3081": "20617070656172616e6365", - "3082": "75726e696e67", - "3083": "206372656174", - "3084": "206f6674656e", - "3085": "6f7665726564", - "3086": "7265616d", - "3087": "20736f6c", - "3088": "617261", - "3089": "736b65", - "3090": "20636f6e74696e756564", - "3091": "20696d6d656469617465", - "3092": "20636f6d70616e696f6e", - "3093": "76657265", - "3094": "617368", - "3095": "727269626c65", - "3096": "20646973636f", - "3097": "2073747275636b", - "3098": "2064656c", - "3099": "6c696e65", - "3100": "6e75", - "3101": "205769", - "3102": "727665", - "3103": "204a6f", - "3104": "6b73", - "3105": "206f6363", - "3106": "64656c69", - "3107": "2077686f7365", - "3108": "206f62736572", - "3109": "726f74", - "3110": "20736179696e67", - "3111": "2065766964656e", - "3112": "2064616e676572", - "3113": "206f626a656374", - "3114": "486f6c", - "3115": "646c65", - "3116": "636965", - "3117": "2068617070656e6564", - "3118": "20666f7267", - "3119": "706c61696e", - "3120": "6573696465", - "3121": "6c6465", - "3122": "656c65", - "3123": "2074616b656e", - "3124": "636f76657265", - "3125": "68617669", - "3126": "20706174", - "3127": "6c656d", - "3128": "20656e7465726564", - "3129": "616e746564", - "3130": "6d6f7665", - "3131": "6d6d65", - "3132": "4d6f6e736569676e657572", - "3133": "6e6f74696365", - "3134": "206c61756768", - "3135": "64726573", - "3136": "62616e64", - "3137": "6f75736c79", - "3138": "646973636f", - "3139": "6e6579", - "3140": "696369", - "3141": "20776869737065", - "3142": "20636f6d666f72", - "3143": "20737566666572", - "3144": "7465726e", - "3145": "207661", - "3146": "206f627365727665", - "3147": "6f65", - "3148": "2066697265", - "3149": "2066756c6c", - "3150": "646e657373", - "3151": "617369", - "3152": "616464", - "3153": "65696768", - "3154": "206c616e", - "3155": "4d61", - "3156": "727475", - "3157": "0a0a2020202020202020202020202020", - "3158": "0a202020202020202020202020202020", - "3159": "206b6e6f776c65646765", - "3160": "736174697366696564", - "3161": "6e6973686564", - "3162": "2073657665", - "3163": "2067617665", - "3164": "7369676e", - "3165": "636f6e736964657261626c65", - "3166": "736563", - "3167": "656967", - "3168": "73746570", - "3169": "656c6c6f", - "3170": "64656e6365", - "3171": "6e6e696e67", - "3172": "2073696768", - "3173": "20656c", - "3174": "747765", - "3175": "6c61756768", - "3176": "616c6d", - "3177": "4d69", - "3178": "206f63636173696f", - "3179": "6f736974", - "3180": "6d706c65", - "3181": "67726565", - "3182": "6f6e6564", - "3183": "204d6172", - "3184": "6d617272", - "3185": "2065787072657373696f6e", - "3186": "706563746564", - "3187": "206d6f6e6579", - "3188": "757272", - "3189": "726176", - "3190": "646961", - "3191": "6f79", - "3192": "20636f6e736964657261626c65", - "3193": "636f6d70616e69", - "3194": "7374617465", - "3195": "2072656365", - "3196": "204c756379", - "3197": "20646563", - "3198": "65727974", - "3199": "6f6174", - "3200": "697461", - "3201": "626c61", - "3202": "757469", - "3203": "20636f6e7665", - "3204": "656172696e67", - "3205": "6c646572", - "3206": "74727563", - "3207": "206e61747572616c", - "3208": "204d616e65747465", - "3209": "206c657373", - "3210": "20696d6167696e", - "3211": "76696c6c616765", - "3212": "2074726f", - "3213": "65646765", - "3214": "2064726f", - "3215": "70706f73", - "3216": "756666", - "3217": "6c6f74", - "3218": "696d6d6564696174656c79", - "3219": "206c69737465", - "3220": "64616e676572", - "3221": "73686f7274", - "3222": "20637572", - "3223": "20686f6c", - "3224": "6561726c", - "3225": "20707572", - "3226": "69616c", - "3227": "7465727269626c65", - "3228": "2072656d61696e6564", - "3229": "74616e64696e67", - "3230": "61637469", - "3231": "63686573", - "3232": "6f6e6572", - "3233": "72656e63", - "3234": "70657269", - "3235": "20737765", - "3236": "696e6174696f", - "3237": "667265", - "3238": "2063726f", - "3239": "20646966", - "3240": "776f6f64", - "3241": "20647261", - "3242": "6265617574", - "3243": "6372656174", - "3244": "626f74", - "3245": "706c69", - "3246": "2074686f7573616e", - "3247": "7374616e64696e", - "3248": "696e6b696e67", - "3249": "70617469656e", - "3250": "6561726e", - "3251": "756c6c79", - "3252": "66657373", - "3253": "757261", - "3254": "63686f", - "3255": "206368696c6472656e", - "3256": "206578707265737369", - "3257": "20706f7765", - "3258": "20496e", - "3259": "74656d", - "3260": "4c7563", - "3261": "20616476", - "3262": "20666c6f", - "3263": "6f736564", - "3264": "726f6164", - "3265": "6f7369", - "3266": "726577", - "3267": "627265617468", - "3268": "2062726f6b65", - "3269": "206c65617374", - "3270": "207669736974", - "3271": "20626c6f6f64", - "3272": "2070726f6d6973", - "3273": "706f736974", - "3274": "20626564", - "3275": "74656d70", - "3276": "6f6f6c", - "3277": "2073746172746564", - "3278": "436f", - "3279": "756c6172", - "3280": "66617374", - "3281": "207265636569766564", - "3282": "6964656e6365", - "3283": "707265737365", - "3284": "656e657261", - "3285": "7175696574", - "3286": "206d6f7665", - "3287": "205374", - "3288": "64656c69676874", - "3289": "65726162", - "3290": "2073686f756c6465", - "3291": "206c697374", - "3292": "6c6572", - "3293": "6c6d65", - "3294": "206e6f746963", - "3295": "20636f6e7469", - "3296": "69737461", - "3297": "76697369", - "3298": "73746f6e", - "3299": "20696d6d6564696174656c79", - "3300": "626974", - "3301": "6e6974", - "3302": "6b6572", - "3303": "756c74", - "3304": "726961", - "3305": "687573", - "3306": "7772", - "3307": "206368616e6365", - "3308": "6f627365727665", - "3309": "20737069", - "3310": "626573", - "3311": "207472696564", - "3312": "207265616479", - "3313": "206d69736572", - "3314": "2059", - "3315": "726573656e746c79", - "3316": "20706170", - "3317": "20776561", - "3318": "2050726f", - "3319": "75616c6c79", - "3320": "7574746572", - "3321": "2076697369", - "3322": "706169", - "3323": "776561", - "3324": "2077616c6b6564", - "3325": "6e6e6f", - "3326": "207370697269", - "3327": "706572666563746c79", - "3328": "616c6b696e", - "3329": "706c696564", - "3330": "6d61727269", - "3331": "737475", - "3332": "6c6f63", - "3333": "686565", - "3334": "6d656d62", - "3335": "20686967", - "3336": "20646973636f7665", - "3337": "2e2e2e2e2e2e2e2e2e2e2e", - "3338": "207365636f6e64", - "3339": "666f7277617264", - "3340": "65636569766564", - "3341": "616e79626f6479", - "3342": "207369676874", - "3343": "656c6c696e67", - "3344": "737566666572", - "3345": "656c766573", - "3346": "696e746564", - "3347": "2073756666", - "3348": "6566756c", - "3349": "726b65", - "3350": "6f6f74", - "3351": "616c73", - "3352": "726365", - "3353": "656d61726b", - "3354": "6c6b696e67", - "3355": "7765726564", - "3356": "206c616479", - "3357": "696e74656e", - "3358": "6c6c61", - "3359": "616666", - "3360": "64657465726d696e", - "3361": "20626567696e", - "3362": "206172726976", - "3363": "207375626a6563", - "3364": "206163636f756e", - "3365": "7070696e", - "3366": "6c6579", - "3367": "7374726169", - "3368": "6561746564", - "3369": "746f70", - "3370": "65697468", - "3371": "2061667465727761726473", - "3372": "206c61726765", - "3373": "6e6174757265", - "3374": "6f626a656374", - "3375": "616469", - "3376": "657870657269656e6365", - "3377": "72657370656374", - "3378": "65786365", - "3379": "6174696e", - "3380": "636c6561", - "3381": "204a696d", - "3382": "20736d696c65", - "3383": "736572766564", - "3384": "206166666563", - "3385": "20636865", - "3386": "20476f64", - "3387": "206e6f7469", - "3388": "636c656172", - "3389": "626c61636b", - "3390": "68696e64", - "3391": "6e636573", - "3392": "20617272", - "3393": "20766f", - "3394": "766f69", - "3395": "64616e", - "3396": "2072657370656374", - "3397": "2068756e64726564", - "3398": "207465727269626c", - "3399": "6374696f6e73", - "3400": "2067726f756e", - "3401": "696d6167696e", - "3402": "7265616368", - "3403": "6d616c", - "3404": "696574", - "3405": "206a6f", - "3406": "63657074", - "3407": "2ce28099", - "3408": "68656176", - "3409": "72657474", - "3410": "736361", - "3411": "68696c6472656e", - "3412": "2073746561", - "3413": "73746f70", - "3414": "72696573", - "3415": "20706169", - "3416": "706f73697469", - "3417": "616374696f6e", - "3418": "20686f727365", - "3419": "6572656e6365", - "3420": "20666f7277617264", - "3421": "6f6c69", - "3422": "61727468", - "3423": "73657665", - "3424": "72656365", - "3425": "6c65617365", - "3426": "2077686973", - "3427": "61696e6c79", - "3428": "20736d696c", - "3429": "2074657272", - "3430": "207374616e", - "3431": "2067726f75", - "3432": "73656c7665", - "3433": "7374726565", - "3434": "6967676572", - "3435": "205072", - "3436": "6368616e6365", - "3437": "207374617465", - "3438": "72656374", - "3439": "6e74696c", - "3440": "7572696e", - "3441": "6164616d", - "3442": "20736f7274", - "3443": "2071756963", - "3444": "2077696665", - "3445": "626f79", - "3446": "72656c79", - "3447": "616c74", - "3448": "207465727269626c65", - "3449": "20636f6d70616e696f", - "3450": "206163726f7373", - "3451": "2064656570", - "3452": "2063726561", - "3453": "68756e", - "3454": "74696365", - "3455": "20647269", - "3456": "696e6469", - "3457": "20636175", - "3458": "2064616e", - "3459": "20616e79626f6479", - "3460": "6163717561696e74", - "3461": "696e676c79", - "3462": "666f726d65", - "3463": "2046", - "3464": "20636f726e6572", - "3465": "706f736974696f", - "3466": "6f736974696f6e", - "3467": "7363656e", - "3468": "6f7267", - "3469": "7369676874", - "3470": "2064726177", - "3471": "7669736974", - "3472": "64656c", - "3473": "646f75", - "3474": "2063697263756d7374616e636573", - "3475": "656c7665", - "3476": "6976696e", - "3477": "636c61", - "3478": "3a2d2d", - "3479": "207365766572616c", - "3480": "686f727365", - "3481": "206c6573", - "3482": "206d6f75", - "3483": "20746561", - "3484": "7265706c", - "3485": "6e636564", - "3486": "207772697465", - "3487": "65726c", - "3488": "6e696f6e", - "3489": "2064657465726d696e", - "3490": "656e646561766f7572", - "3491": "7368696e67", - "3492": "74656e6465", - "3493": "6f7279", - "3494": "676573", - "3495": "20476f", - "3496": "68616d", - "3497": "757368", - "3498": "2064656c69676874", - "3499": "6368696c647265", - "3500": "2076696c6c6167", - "3501": "20717565", - "3502": "73706572", - "3503": "696f6e61", - "3504": "72757374", - "3505": "657175", - "3506": "6e6b696e67", - "3507": "206e656564", - "3508": "6d6f6e6579", - "3509": "627261", - "3510": "6d6169", - "3511": "62657374", - "3512": "6e6f72", - "3513": "617073", - "3514": "626568", - "3515": "726167", - "3516": "636f6d666f727461626c65", - "3517": "64657465726d696e65", - "3518": "20697473656c66", - "3519": "206272656174", - "3520": "20696e74656e", - "3521": "6b6e6f776e", - "3522": "65726174", - "3523": "63746c79", - "3524": "61707079", - "3525": "696e6174", - "3526": "726d65", - "3527": "696e6e", - "3528": "204a61", - "3529": "73706972", - "3530": "6f757468", - "3531": "20696e66", - "3532": "697474696e", - "3533": "6561727468", - "3534": "20666f7265", - "3535": "72656e6365", - "3536": "20626c6f6f", - "3537": "697363", - "3538": "7365636f6e64", - "3539": "207361746973", - "3540": "68756e64726564", - "3541": "20706572666563", - "3542": "206e6174757265", - "3543": "65747469", - "3544": "20756e63", - "3545": "20726566", - "3546": "20656173", - "3547": "67687473", - "3548": "726962", - "3549": "7361746973666965", - "3550": "6174697366696564", - "3551": "2062656175", - "3552": "2067656e65", - "3553": "6d69736572", - "3554": "70616e", - "3555": "20436f756e74", - "3556": "6e7465726564", - "3557": "666572656e63", - "3558": "636569", - "3559": "6f72646572", - "3560": "6561736564", - "3561": "6569766564", - "3562": "696e617469", - "3563": "6c697374", - "3564": "73696768", - "3565": "6c697368", - "3566": "206d6f76", - "3567": "204465", - "3568": "536f", - "3569": "204e", - "3570": "6572666563746c79", - "3571": "6769726c", - "3572": "616c6b65", - "3573": "20486572", - "3574": "66697265", - "3575": "204372756e63686572", - "3576": "656e677468", - "3577": "6576656e74", - "3578": "6f76657265", - "3579": "666173", - "3580": "204c75", - "3581": "666f7274756e65", - "3582": "656e6473", - "3583": "6f636b65", - "3584": "6e6f7469", - "3585": "20736c69", - "3586": "6c616e63", - "3587": "64726f", - "3588": "657263", - "3589": "6f70706f7274756e697479", - "3590": "207769736865", - "3591": "7374726174", - "3592": "73746f7070", - "3593": "6e74696e67", - "3594": "6173746572", - "3595": "20616374", - "3596": "74657269", - "3597": "20726169", - "3598": "7264696e", - "3599": "7374726169676874", - "3600": "20706572666563746c79", - "3601": "6e6563657373617279", - "3602": "697374616e6365", - "3603": "2063686169", - "3604": "626c6f6f64", - "3605": "206361757365", - "3606": "20746f756368", - "3607": "206472656164", - "3608": "736972", - "3609": "20736869", - "3610": "20746572", - "3611": "6c65646765", - "3612": "206c6f7374", - "3613": "747275636b", - "3614": "657266756c", - "3615": "7061737369", - "3616": "7265617468", - "3617": "20616c736f", - "3618": "6d656d", - "3619": "666f6f", - "3620": "70617469656e74", - "3621": "206c6561726e", - "3622": "206f72646572", - "3623": "72697465", - "3624": "206578706563746564", - "3625": "6d6973657261626c65", - "3626": "204d6f", - "3627": "486572", - "3628": "6f6e66", - "3629": "696d706f737369626c65", - "3630": "20736174697366696564", - "3631": "63686572", - "3632": "207461626c65", - "3633": "20636f6e6669", - "3634": "2074616c6b696e67", - "3635": "20696d706f", - "3636": "6c6174696f", - "3637": "7279696e", - "3638": "6f6e7365", - "3639": "20737562", - "3640": "2072756e", - "3641": "736372", - "3642": "6c6b6564", - "3643": "20486973", - "3644": "207374726169676874", - "3645": "2074686f7567687473", - "3646": "656164696e67", - "3647": "e28094e2809d", - "3648": "657463686564", - "3649": "636f756e7465", - "3650": "7474657273", - "3651": "6c696e6773", - "3652": "2070726f70", - "3653": "6f6c6c696e", - "3654": "20656e646561766f7572", - "3655": "20737475", - "3656": "6974696e", - "3657": "61636865", - "3658": "596573", - "3659": "7468696e6b696e67", - "3660": "2063617272696564", - "3661": "20707572706f7365", - "3662": "616d6f", - "3663": "204f", - "3664": "636f726e6572", - "3665": "20686f757273", - "3666": "656570696e67", - "3667": "74686572696e", - "3668": "2064616e6765", - "3669": "206f626a6563", - "3670": "204d6164616d", - "3671": "75726573", - "3672": "666f7267", - "3673": "7569636b", - "3674": "756e7465", - "3675": "6c617469", - "3676": "6f776c", - "3677": "6d6f76", - "3678": "206e6174757261", - "3679": "20736563726574", - "3680": "207369676e", - "3681": "6f6e6573", - "3682": "67726f75", - "3683": "206d6973657261626c65", - "3684": "64657465726d696e6564", - "3685": "686f6c64", - "3686": "6c61636b", - "3687": "766964656e", - "3688": "2066616c6c", - "3689": "697374616e", - "3690": "20686f7272", - "3691": "636f6e6669", - "3692": "4d6164616d", - "3693": "73636f", - "3694": "626c6f", - "3695": "20616672616964", - "3696": "20647572696e67", - "3697": "20657861", - "3698": "676c616e", - "3699": "6d616b696e67", - "3700": "207375666665", - "3701": "75726e696e", - "3702": "20666f7572", - "3703": "757375616c", - "3704": "6172746963", - "3705": "656675", - "3706": "696d706f7274616e", - "3707": "6e65636573736172", - "3708": "20696465", - "3709": "6167696e", - "3710": "65747479", - "3711": "6f6d61", - "3712": "7270", - "3713": "676e", - "3714": "636f756e74656e616e6365", - "3715": "2064657465726d696e65", - "3716": "206d616b696e67", - "3717": "2073696c656e74", - "3718": "706c65617365", - "3719": "6573706563", - "3720": "20626c6163", - "3721": "20726f6164", - "3722": "69646561", - "3723": "2062656e", - "3724": "76656e69", - "3725": "73747269", - "3726": "656d616e", - "3727": "617574", - "3728": "766973", - "3729": "656d7074", - "3730": "73756666", - "3731": "737572707269736564", - "3732": "706572666563746c", - "3733": "7772697465", - "3734": "2071756965", - "3735": "207061696e", - "3736": "2064657369", - "3737": "76616e", - "3738": "206472657373", - "3739": "207365637265", - "3740": "20676976656e", - "3741": "206861707069", - "3742": "707572706f7365", - "3743": "204c6f6e", - "3744": "696c6c65", - "3745": "2066756c", - "3746": "64657374", - "3747": "626f7572", - "3748": "65706172", - "3749": "726177", - "3750": "4265", - "3751": "626567696e", - "3752": "6164696e67", - "3753": "616c65", - "3754": "73756e", - "3755": "20646565", - "3756": "20776f6f", - "3757": "63756c61", - "3758": "65636569", - "3759": "20696d706f737369626c65", - "3760": "20657870657269656e6365", - "3761": "20757375616c", - "3762": "6d61727269616765", - "3763": "6e65696768626f7572", - "3764": "6c61726765", - "3765": "206f667465", - "3766": "6172726976", - "3767": "656c6c65", - "3768": "657065", - "3769": "616d70", - "3770": "726964", - "3771": "206775", - "3772": "65726d", - "3773": "206561727468", - "3774": "74616b696e67", - "3775": "20696d616769", - "3776": "64656e63", - "3777": "75726e69", - "3778": "73736573", - "3779": "736d696c65", - "3780": "2073657276", - "3781": "6f7769", - "3782": "696e66", - "3783": "6767", - "3784": "647265616466756c", - "3785": "69746174696f6e", - "3786": "65726174696f6e", - "3787": "6e617465", - "3788": "70726f76", - "3789": "6f6e6174", - "3790": "656c6c69", - "3791": "726576", - "3792": "206f70706f7274756e697479", - "3793": "6163717561696e74616e6365", - "3794": "617474656e64", - "3795": "7374616e6469", - "3796": "656e646572", - "3797": "2070657266", - "3798": "6573746564", - "3799": "206369", - "3800": "6e6369", - "3801": "20616972", - "3802": "66616374", - "3803": "69736974", - "3804": "646e6573", - "3805": "65656b", - "3806": "6e69736865", - "3807": "696e6b696e", - "3808": "656172696e", - "3809": "6861707069", - "3810": "696e696e", - "3811": "6f727461", - "3812": "2056616e", - "3813": "736d61", - "3814": "647261", - "3815": "636f6d706c657465", - "3816": "2065766964656e74", - "3817": "20647265616d", - "3818": "207365766572", - "3819": "686f6e6f7572", - "3820": "2073656e7365", - "3821": "6f6363", - "3822": "2072616e", - "3823": "20747279", - "3824": "6e6b696e", - "3825": "69746174", - "3826": "67676572", - "3827": "206d61727269616765", - "3828": "73656e7365", - "3829": "206c617567", - "3830": "6d6f766564", - "3831": "7265746368", - "3832": "616e636573", - "3833": "20626f6479", - "3834": "576879", - "3835": "656f", - "3836": "7475726e696e67", - "3837": "20626573696465", - "3838": "65766572616c", - "3839": "20616c6c6f77", - "3840": "20736974", - "3841": "416c", - "3842": "7365766572", - "3843": "6472656164", - "3844": "7261697365", - "3845": "7261637465", - "3846": "206e6f72", - "3847": "20627572", - "3848": "20676972", - "3849": "20686169", - "3850": "726f7665", - "3851": "6f6c6c65", - "3852": "20487563", - "3853": "207465727269", - "3854": "206368616e63", - "3855": "20737472656e677468", - "3856": "2064697374616e", - "3857": "636f6c6c656374", - "3858": "204861726b6572", - "3859": "20677265617465", - "3860": "2073746f72", - "3861": "74696573", - "3862": "6e6565", - "3863": "676172", - "3864": "6f6674", - "3865": "2073697474696e67", - "3866": "20636f6e6365726e", - "3867": "6174656c", - "3868": "2066616c", - "3869": "66616c6c", - "3870": "6f726e65", - "3871": "77686973", - "3872": "6e676564", - "3873": "726f776e", - "3874": "70726f70", - "3875": "20726567", - "3876": "66696e676572", - "3877": "207265616368", - "3878": "696f75736c79", - "3879": "20636f6d6573", - "3880": "73746f6e65", - "3881": "706972", - "3882": "726963", - "3883": "2066696e676572", - "3884": "20726573706563", - "3885": "20706c65617365", - "3886": "2068756e647265", - "3887": "746f7279", - "3888": "727479", - "3889": "73746c", - "3890": "446f", - "3891": "2064697374616e6365", - "3892": "20647265616466756c", - "3893": "206f62736572766564", - "3894": "7475726573", - "3895": "206d6f6e65", - "3896": "486f77", - "3897": "756465", - "3898": "626c79", - "3899": "697370", - "3900": "20737572707269736564", - "3901": "0a0a2020202020202020", - "3902": "657373696e67", - "3903": "7374656e6564", - "3904": "206465736972", - "3905": "76696465", - "3906": "73746169", - "3907": "696d706f", - "3908": "6e676c79", - "3909": "686f7272", - "3910": "7773", - "3911": "757270", - "3912": "636974", - "3913": "20617474656e64", - "3914": "20686f6e6f7572", - "3915": "20666f72776172", - "3916": "636f6e6365726e", - "3917": "206f636375", - "3918": "20456e676c", - "3919": "746f756368", - "3920": "2066726565", - "3921": "206c616964", - "3922": "616e64696e", - "3923": "436f756e74", - "3924": "64726177", - "3925": "206f6273", - "3926": "6f6164", - "3927": "207365636f6e", - "3928": "696d61", - "3929": "65737065", - "3930": "7266756c", - "3931": "666f7274756e617465", - "3932": "207468696e6b696e67", - "3933": "656d6f", - "3934": "2064657465726d69", - "3935": "20627269676874", - "3936": "2074616b696e67", - "3937": "20736565696e67", - "3938": "20617272697665", - "3939": "20646573697265", - "3940": "70617265", - "3941": "746f67", - "3942": "646563", - "3943": "646973617070", - "3944": "626561757469", - "3945": "747465726564", - "3946": "20776f6d656e", - "3947": "417274687572", - "3948": "6c6561726e", - "3949": "20656c7365", - "3950": "2062656c6f", - "3951": "20686f6c64", - "3952": "206b657074", - "3953": "2070726f6d", - "3954": "736f6c7574", - "3955": "746875", - "3956": "6c616d", - "3957": "657363", - "3958": "6568", - "3959": "64656570", - "3960": "206d696c", - "3961": "2074656e", - "3962": "6c696564", - "3963": "61636573", - "3964": "65746572", - "3965": "2064657465726d696e6564", - "3966": "206f70696e696f6e", - "3967": "20656e67616765", - "3968": "20436172746f6e", - "3969": "636f6e766572", - "3970": "736563726574", - "3971": "7374616972", - "3972": "617070696e", - "3973": "69636b65", - "3974": "67657273", - "3975": "696674", - "3976": "656e6e", - "3977": "20646972656374696f6e", - "3978": "63686169", - "3979": "656e6c79", - "3980": "6c616479", - "3981": "6f726d65", - "3982": "73686970", - "3983": "6b6e657373", - "3984": "707572", - "3985": "66616d", - "3986": "736576657261", - "3987": "6d61726b6564", - "3988": "206164646564", - "3989": "6f756e7461696e", - "3990": "20696e71756972", - "3991": "66696564", - "3992": "706170", - "3993": "636170", - "3994": "204875", - "3995": "486973", - "3996": "706c656173616e74", - "3997": "206d656e74696f6e", - "3998": "20636f6d706c657465", - "3999": "6c75737472", - "4000": "6c6f6e6765", - "4001": "696e676572", - "4002": "656c6c696e", - "4003": "6c6961", - "4004": "666c6f", - "4005": "6d706f", - "4006": "20726f61", - "4007": "76656c79", - "4008": "686f7274", - "4009": "20486172", - "4010": "65727661", - "4011": "206b6e6f776e", - "4012": "6661766f7572", - "4013": "6e6563657373", - "4014": "206173737572", - "4015": "726f76", - "4016": "2064656c696768", - "4017": "70617373696f6e", - "4018": "636f6e66696465", - "4019": "20417274687572", - "4020": "207765656b", - "4021": "696c697479", - "4022": "6e636865", - "4023": "20736361", - "4024": "496e", - "4025": "636f6e666964656e", - "4026": "757474", - "4027": "616476", - "4028": "75746966756c", - "4029": "207468726f77", - "4030": "6f6e6765", - "4031": "6f747465", - "4032": "20646972", - "4033": "61707069", - "4034": "6f757274", - "4035": "74726163", - "4036": "7472617469", - "4037": "7465727269", - "4038": "206d697365", - "4039": "707069", - "4040": "4a696d", - "4041": "20646973617070", - "4042": "636c6f74686573", - "4043": "20627261", - "4044": "73746963", - "4045": "696c6564", - "4046": "20657669", - "4047": "6e6563", - "4048": "2073746179", - "4049": "7266", - "4050": "2061646472657373", - "4051": "617070726f616368", - "4052": "6563657373617279", - "4053": "206c657474657273", - "4054": "6d697373", - "4055": "206c6f77", - "4056": "726b6564", - "4057": "70657266", - "4058": "207265706561746564", - "4059": "206c697374656e6564", - "4060": "636f6e7363696f7573", - "4061": "20656e646561766f75", - "4062": "20636f6e766572", - "4063": "77616974696e67", - "4064": "20666967757265", - "4065": "2062656769", - "4066": "20696d7072", - "4067": "436861726c", - "4068": "74697265", - "4069": "6f75736c", - "4070": "75736c79", - "4071": "72657665", - "4072": "70617469", - "4073": "6f756368", - "4074": "20736972", - "4075": "61637175", - "4076": "6e636c", - "4077": "616769", - "4078": "207374726169", - "4079": "20636f726e65", - "4080": "617969", - "4081": "756e65", - "4082": "20687572", - "4083": "6f746963", - "4084": "696e6765", - "4085": "2061727269", - "4086": "647269", - "4087": "6e6578", - "4088": "206e65696768626f7572", - "4089": "20657870657269656e63", - "4090": "206661766f7572", - "4091": "65616b696e67", - "4092": "207261697365", - "4093": "207265736f6c", - "4094": "20436861726c", - "4095": "657465726d696e6564", - "4096": "206c617267", - "4097": "20736f7272", - "4098": "6f73697469", - "4099": "7472757374", - "4100": "706963", - "4101": "69726c", - "4102": "65706c", - "4103": "697269", - "4104": "4d6172", - "4105": "64616d", - "4106": "20626f6f", - "4107": "736f7274", - "4108": "72616365", - "4109": "726f6b65", - "4110": "6963616c", - "4111": "20656d", - "4112": "2073747261696768", - "4113": "206d617272696564", - "4114": "6368696c6472", - "4115": "656e64696e67", - "4116": "636f6d6d6f6e", - "4117": "636f6d706c65", - "4118": "6d6173746572", - "4119": "70696e657373", - "4120": "207377656574", - "4121": "2070617469656e", - "4122": "20636f6d706c65", - "4123": "706f7373657373", - "4124": "2074656172", - "4125": "656c657373", - "4126": "2063697263", - "4127": "204d696e61", - "4128": "69636174", - "4129": "6976656e", - "4130": "626f61", - "4131": "766f75", - "4132": "686972", - "4133": "646966666572656e6365", - "4134": "616e6465", - "4135": "726d696e", - "4136": "20617070726f616368", - "4137": "206163717561696e74", - "4138": "636172726965", - "4139": "636174696f6e", - "4140": "206772617665", - "4141": "20676c616e63", - "4142": "20636f757274", - "4143": "6e74656e74", - "4144": "65636f6e64", - "4145": "20706c6179", - "4146": "2063686565", - "4147": "74656d7074", - "4148": "696374", - "4149": "61766f", - "4150": "72657463686564", - "4151": "75616c6c", - "4152": "69656c64", - "4153": "72657370", - "4154": "71756972", - "4155": "20766973", - "4156": "20436f75", - "4157": "647265616d", - "4158": "7468726f77", - "4159": "756e647265", - "4160": "616374696f", - "4161": "20686f7273", - "4162": "6164616d65", - "4163": "6e7665", - "4164": "627572", - "4165": "646572696e67", - "4166": "207061737369", - "4167": "206d65616e73", - "4168": "666967757265", - "4169": "656e67616765", - "4170": "20666c61", - "4171": "706c6965", - "4172": "66616972", - "4173": "20776565", - "4174": "6572696f", - "4175": "756d62", - "4176": "20626567696e6e696e67", - "4177": "20696d706f7274616e", - "4178": "20737472616e676572", - "4179": "20726570656174", - "4180": "61646472657373", - "4181": "6374757265", - "4182": "6669656c64", - "4183": "20636f6e64", - "4184": "7574696f6e", - "4185": "696c6974", - "4186": "69676765", - "4187": "207468616e6b", - "4188": "206d6f6e7468", - "4189": "736c69", - "4190": "42656e", - "4191": "4861", - "4192": "61697273", - "4193": "61637465", - "4194": "706f736564", - "4195": "2061646d69", - "4196": "746966", - "4197": "656369", - "4198": "726d69", - "4199": "206265636f6d65", - "4200": "20636f6e636572", - "4201": "20707572706f73", - "4202": "696c6c61", - "4203": "73697469", - "4204": "7465726d", - "4205": "736f6e65", - "4206": "647563", - "4207": "746973", - "4208": "206c6f6e6765", - "4209": "627269676874", - "4210": "656173616e74", - "4211": "6273", - "4212": "616476616e74616765", - "4213": "6461726b6e657373", - "4214": "20636f6e66696465", - "4215": "207265666c656374", - "4216": "20636c6f74686573", - "4217": "20626f6f6b", - "4218": "656e746564", - "4219": "747261696e", - "4220": "6174697665", - "4221": "6173737572", - "4222": "706965", - "4223": "646973636f7665726564", - "4224": "626f6f6b", - "4225": "74726965", - "4226": "6f666669", - "4227": "206f63", - "4228": "6e76", - "4229": "6d656e74696f6e", - "4230": "20656666656374", - "4231": "206d7572646572", - "4232": "737461697273", - "4233": "2072656c6174", - "4234": "204c79646961", - "4235": "2072657370", - "4236": "2068616972", - "4237": "2075736564", - "4238": "2063617573", - "4239": "6c6f7564", - "4240": "206c6973", - "4241": "726e6572", - "4242": "20466f72", - "4243": "6f616368", - "4244": "726469", - "4245": "656c6f", - "4246": "657861", - "4247": "747479", - "4248": "2077616974696e67", - "4249": "20636f6d706c6574", - "4250": "61757469", - "4251": "20676c6164", - "4252": "20636f6c64", - "4253": "7374656164", - "4254": "2072616973", - "4255": "20436f756e", - "4256": "2061737375", - "4257": "6c7573", - "4258": "656974", - "4259": "6d7572646572", - "4260": "70726f616368", - "4261": "2068756d616e", - "4262": "2050726f7373", - "4263": "64726573736564", - "4264": "64726f70706564", - "4265": "20636f756e7465", - "4266": "204461726e6179", - "4267": "666f7572", - "4268": "74746564", - "4269": "20626172", - "4270": "726d6564", - "4271": "656e7369", - "4272": "6c6c73", - "4273": "50726f", - "4274": "2068757272", - "4275": "2066656574", - "4276": "6e7461696e", - "4277": "2068656c64", - "4278": "6569676874", - "4279": "6c616e6365", - "4280": "206b696e67", - "4281": "746e657373", - "4282": "636375", - "4283": "204869", - "4284": "61736564", - "4285": "74657374", - "4286": "204d6f6e736569676e657572", - "4287": "206369726375", - "4288": "696e71756972", - "4289": "617272697665", - "4290": "207472757374", - "4291": "20666f726d65", - "4292": "756e69", - "4293": "657473", - "4294": "79626f", - "4295": "726567", - "4296": "2047617264696e6572", - "4297": "206c6f6e676572", - "4298": "616c6c6f776564", - "4299": "7265666c656374", - "4300": "20666f6f74", - "4301": "636f757274", - "4302": "2073686164", - "4303": "2057656c6c", - "4304": "65616469", - "4305": "77616c6c", - "4306": "61746573", - "4307": "20646570", - "4308": "20646965", - "4309": "20646574", - "4310": "656c61", - "4311": "204966", - "4312": "6d7065", - "4313": "6e636f", - "4314": "6c736f", - "4315": "20616476616e", - "4316": "20656e676167", - "4317": "7075626c6963", - "4318": "6573736f", - "4319": "65726665", - "4320": "20737566", - "4321": "2064726f70", - "4322": "637261", - "4323": "2064726f70706564", - "4324": "6e7374616e746c79", - "4325": "616e78696f7573", - "4326": "636f7572616765", - "4327": "20617373757265", - "4328": "20436f6c", - "4329": "4e6f77", - "4330": "206461726b6e657373", - "4331": "6f757273656c766573", - "4332": "20636f6e666964656e", - "4333": "657863657074", - "4334": "2073696d706c", - "4335": "646974696f6e", - "4336": "2074727565", - "4337": "2073656e74", - "4338": "7069656365", - "4339": "6772617665", - "4340": "6563746c79", - "4341": "206f726465", - "4342": "206c616e64", - "4343": "6e646c", - "4344": "205361", - "4345": "20636f756e74656e616e6365", - "4346": "20636c61", - "4347": "6e656564", - "4348": "20657874", - "4349": "72726f72", - "4350": "636f6e666964656e6365", - "4351": "20456e676c697368", - "4352": "206f757473696465", - "4353": "6173746f6e697368", - "4354": "456e676c697368", - "4355": "656374696f6e73", - "4356": "6a6f75726e6579", - "4357": "6c697374656e65", - "4358": "70726f6261626c", - "4359": "6369726375", - "4360": "6974617469", - "4361": "61696e6c", - "4362": "70726f6d", - "4363": "686f7273", - "4364": "6f727269", - "4365": "6c6f6f64", - "4366": "746972", - "4367": "647265737365", - "4368": "6c6967687465", - "4369": "20646f696e67", - "4370": "206865617679", - "4371": "756172", - "4372": "776c6564", - "4373": "2068756d", - "4374": "7061696e", - "4375": "65766964656e746c79", - "4376": "776869737065726564", - "4377": "207374616e64696e67", - "4378": "616476616e", - "4379": "2061707072", - "4380": "686f6e", - "4381": "6a6f", - "4382": "617463686564", - "4383": "2066726f6e74", - "4384": "74656e646564", - "4385": "646573697265", - "4386": "68656c64", - "4387": "466f72", - "4388": "696572", - "4389": "726e61", - "4390": "476f64", - "4391": "2e2e2e", - "4392": "20616e78696f7573", - "4393": "206a6f75726e6579", - "4394": "6f75726564", - "4395": "20746f7563", - "4396": "6967687473", - "4397": "2073776565", - "4398": "20736f756c", - "4399": "6c6970", - "4400": "696c73", - "4401": "20476f64616c6d696e67", - "4402": "6c6c656e", - "4403": "706c6179", - "4404": "696d6d65", - "4405": "6f6d6573", - "4406": "6f6c6f", - "4407": "646e65", - "4408": "5769", - "4409": "206272696e67", - "4410": "656666656374", - "4411": "65787472616f7264696e617279", - "4412": "206c656172", - "4413": "6d61726b65", - "4414": "656570696e", - "4415": "74696f6e61", - "4416": "2077616c6c", - "4417": "6e697479", - "4418": "696e6c79", - "4419": "72646c79", - "4420": "626c6573", - "4421": "73656c76", - "4422": "6d7074", - "4423": "63696f", - "4424": "2070617469656e74", - "4425": "20696d6167696e65", - "4426": "206e65696768626f", - "4427": "67617264", - "4428": "20667572", - "4429": "7068", - "4430": "207365656d73", - "4431": "746368696e67", - "4432": "20696e717569", - "4433": "736861646f77", - "4434": "206368616972", - "4435": "636875726368", - "4436": "617373757265", - "4437": "207461626c", - "4438": "726176656c", - "4439": "2073696e67", - "4440": "66726f6e74", - "4441": "6368616972", - "4442": "677269", - "4443": "20776869737065726564", - "4444": "20666f7274756e", - "4445": "73746174696f6e", - "4446": "6c656374696f6e", - "4447": "20736861646f77", - "4448": "636f6e6e656374", - "4449": "6f636375", - "4450": "61726368", - "4451": "20666173", - "4452": "65726d69", - "4453": "65737472", - "4454": "76696c6c", - "4455": "204269", - "4456": "6d626c", - "4457": "776f6e64657266756c", - "4458": "6561746865", - "4459": "7377656574", - "4460": "6e61626c65", - "4461": "68756d616e", - "4462": "746565", - "4463": "206578706c61696e", - "4464": "20456e676c616e64", - "4465": "696c6c69616d", - "4466": "69746174696f", - "4467": "636f6e736369", - "4468": "65726174696f", - "4469": "726e6573", - "4470": "706c616e", - "4471": "6e646f6e", - "4472": "20677269", - "4473": "6e6f776e", - "4474": "63697263", - "4475": "65636573", - "4476": "72616765", - "4477": "6f6273", - "4478": "626f6f", - "4479": "76656e74757265", - "4480": "72657373696e67", - "4481": "20676c616e6365", - "4482": "20636c6f736564", - "4483": "6561726e65", - "4484": "2077696c64", - "4485": "70726f7665", - "4486": "6570696e67", - "4487": "6577617264", - "4488": "6d706f73", - "4489": "6172646c", - "4490": "616b6573", - "4491": "616d696e", - "4492": "61626c79", - "4493": "4e65746865726669656c64", - "4494": "206f66666963", - "4495": "6469736170706f696e74", - "4496": "20776f6e64657266756c", - "4497": "6e657374", - "4498": "69656365", - "4499": "726f7365", - "4500": "696d7072", - "4501": "206173746f6e697368", - "4502": "436861726c6f747465", - "4503": "206d696c65", - "4504": "4166746572", - "4505": "2072656163", - "4506": "6f74696f6e", - "4507": "2074726565", - "4508": "206d697374", - "4509": "20736f6d657768", - "4510": "20776973686564", - "4511": "20686f72726f72", - "4512": "73706972697473", - "4513": "2062616e", - "4514": "20707261", - "4515": "20637572696f", - "4516": "726570656174", - "4517": "6c6f77696e67", - "4518": "207375636365", - "4519": "6572696f7573", - "4520": "6c6174746572", - "4521": "626f617264", - "4522": "2062656172", - "4523": "656e657373", - "4524": "6465736972", - "4525": "68756d", - "4526": "6d616769", - "4527": "206e6565", - "4528": "63696e67", - "4529": "6e636572", - "4530": "6369656e", - "4531": "6f6c6c69", - "4532": "66616c", - "4533": "796564", - "4534": "626172", - "4535": "6c7369", - "4536": "76616c", - "4537": "2073706972697473", - "4538": "20706f7373657373", - "4539": "6e65696768626f7572686f6f64", - "4540": "2065766964656e746c79", - "4541": "206e65696768626f75", - "4542": "20667269676874", - "4543": "2061736c656570", - "4544": "20536577617264", - "4545": "20636875726368", - "4546": "206d6f766564", - "4547": "776f727468", - "4548": "6974746564", - "4549": "73746f7279", - "4550": "7465726d69", - "4551": "20626f7973", - "4552": "69747469", - "4553": "61726465", - "4554": "20726564", - "4555": "697475", - "4556": "747564", - "4557": "20626974", - "4558": "20686174", - "4559": "7265736f", - "4560": "68616e6b", - "4561": "6576696c", - "4562": "20736c6f77", - "4563": "2064726573", - "4564": "6163686564", - "4565": "4361", - "4566": "2066696e6765", - "4567": "6d6f72726f77", - "4568": "2073746f6e65", - "4569": "2070726f63656564", - "4570": "6170706f696e74", - "4571": "6578706c61696e", - "4572": "206d656e74696f", - "4573": "20636f6e666964", - "4574": "206265796f6e64", - "4575": "6973686d656e74", - "4576": "636f726e", - "4577": "63656564", - "4578": "75646765", - "4579": "65617269", - "4580": "7574696f", - "4581": "65717565", - "4582": "65736f6c", - "4583": "686970", - "4584": "796573746572646179", - "4585": "2072656d61726b6564", - "4586": "646973636f76657265", - "4587": "6167726565", - "4588": "2070726f63", - "4589": "20696d6d65", - "4590": "6f66666963", - "4591": "666c65", - "4592": "206e6563657373617279", - "4593": "73616e64", - "4594": "20636f61", - "4595": "68617465", - "4596": "66726565", - "4597": "656e6765", - "4598": "7363686f6f6c", - "4599": "746572696e67", - "4600": "2070726f6365", - "4601": "20686f6e6f75", - "4602": "207069656365", - "4603": "726561636865", - "4604": "616973", - "4605": "20657863657074", - "4606": "72656374696f6e", - "4607": "206e6f626f6479", - "4608": "73657276616e74", - "4609": "2070726f626162", - "4610": "206e6563657373", - "4611": "696c6c6564", - "4612": "2065617274", - "4613": "2064697361", - "4614": "2066616374", - "4615": "20616c6c6f", - "4616": "206578706c", - "4617": "2073616665", - "4618": "6563726574", - "4619": "2073636172", - "4620": "696d706c", - "4621": "616c7468", - "4622": "61707072", - "4623": "67726174", - "4624": "637573", - "4625": "736c79", - "4626": "69676874656e6564", - "4627": "737065616b696e67", - "4628": "2070726f6261626c", - "4629": "737573706963696f", - "4630": "4d6f6e7369657572", - "4631": "737361", - "4632": "737561", - "4633": "206272696768", - "4634": "2061626f7665", - "4635": "6164696e", - "4636": "65636f6d", - "4637": "696e74657272757074", - "4638": "4c6f6e67626f75726e", - "4639": "7070696e67", - "4640": "7474657265", - "4641": "616e746c79", - "4642": "6d6f6e7468", - "4643": "6b6574", - "4644": "6f7564", - "4645": "6d6173", - "4646": "20796573746572646179", - "4647": "706f696e746564", - "4648": "20616464726573", - "4649": "2073686f776564", - "4650": "20696e666f726d", - "4651": "20656666", - "4652": "6f6e6f75", - "4653": "696e746572657374696e67", - "4654": "696e666f726d6174696f6e", - "4655": "2057696c6c69616d", - "4656": "20617070726f6163", - "4657": "6162736f6c757465", - "4658": "206163717561696e", - "4659": "20776f727468", - "4660": "696c6c696e67", - "4661": "2063726f7764", - "4662": "207061757365", - "4663": "206578706c61", - "4664": "6f6666696365", - "4665": "20756e636c65", - "4666": "7369646573", - "4667": "6f6e676572", - "4668": "2064657370", - "4669": "6973696e67", - "4670": "7074696f6e", - "4671": "6265636f6d", - "4672": "7468616e6b", - "4673": "2073747269", - "4674": "65726d696e", - "4675": "706974", - "4676": "737562", - "4677": "726173", - "4678": "726179", - "4679": "4461", - "4680": "20486f77", - "4681": "61646465", - "4682": "6f726765", - "4683": "6f6e616c", - "4684": "657477", - "4685": "206163717561696e74616e6365", - "4686": "20737065616b696e67", - "4687": "7175657374696f6e73", - "4688": "616772656561626c65", - "4689": "6973636f7665726564", - "4690": "20636f6d6d6f6e", - "4691": "20706170657273", - "4692": "2067726176", - "4693": "206c696e65", - "4694": "204c616479", - "4695": "64646564", - "4696": "206d6572", - "4697": "6e6465726564", - "4698": "77616974696e", - "4699": "667269676874", - "4700": "20636f6e7465", - "4701": "656175", - "4702": "726f63", - "4703": "62656e", - "4704": "204173", - "4705": "706172746963756c61726c79", - "4706": "204e65746865726669656c64", - "4707": "207175657374696f6e73", - "4708": "73756666696369656e74", - "4709": "204c6f6e67626f75726e", - "4710": "2070696374757265", - "4711": "736f6d65626f6479", - "4712": "616c74686f756768", - "4713": "636b6574", - "4714": "77696c64", - "4715": "63686565", - "4716": "20776879", - "4717": "69766573", - "4718": "61726b6564", - "4719": "20706c616e", - "4720": "696f6e616c", - "4721": "636c6f636b", - "4722": "686d656e74", - "4723": "7365727661", - "4724": "707261", - "4725": "0a0a0a0a202020202020202020202020", - "4726": "0a0a0a20202020202020202020202020", - "4727": "69636174696f6e", - "4728": "656c6965766564", - "4729": "636f6d70616e79", - "4730": "616974696e67", - "4731": "656174686572", - "4732": "2073686f6f6b", - "4733": "676c616e6365", - "4734": "6d6174696f6e", - "4735": "736f6c757465", - "4736": "6e717569", - "4737": "726f7065", - "4738": "20696c6c", - "4739": "6c7365", - "4740": "6f7269", - "4741": "737061", - "4742": "627365", - "4743": "696e7374616e746c79", - "4744": "67656e746c656d656e", - "4745": "204d6f6e7369657572", - "4746": "6974756465", - "4747": "696f75736c", - "4748": "696c696e67", - "4749": "2070726f76", - "4750": "657175656e", - "4751": "617070726f", - "4752": "636c6f7468", - "4753": "65736f", - "4754": "62656c6f", - "4755": "206e6563", - "4756": "666f6f74", - "4757": "6d697374", - "4758": "2068757272696564", - "4759": "2061727269766564", - "4760": "726170", - "4761": "205661", - "4762": "206469736170", - "4763": "206669677572", - "4764": "627265617374", - "4765": "666f726d6564", - "4766": "20636c6f7468", - "4767": "646966666963756c7479", - "4768": "20616476616e74616765", - "4769": "70696374757265", - "4770": "7772697474656e", - "4771": "206e6561726c79", - "4772": "20686f72736573", - "4773": "2067726f77", - "4774": "206d656574", - "4775": "636f6e7465", - "4776": "7472656174", - "4777": "6572617469", - "4778": "73746c65", - "4779": "7765656b", - "4780": "68657269", - "4781": "6a65", - "4782": "656d73", - "4783": "73756d", - "4784": "64726f70", - "4785": "20636170", - "4786": "65736572", - "4787": "70726573656e746c79", - "4788": "20617070726f", - "4789": "2061646d6972", - "4790": "7665727361", - "4791": "73696d706c", - "4792": "72696d", - "4793": "2073657276616e74", - "4794": "2053747279766572", - "4795": "20736574746c65", - "4796": "20736c69676874", - "4797": "206578616d696e", - "4798": "206f6666696365", - "4799": "6f737365", - "4800": "6f6b656e", - "4801": "74616765", - "4802": "756e6368", - "4803": "616e6469", - "4804": "65637265", - "4805": "20436861", - "4806": "204c61", - "4807": "7863", - "4808": "476f", - "4809": "656e747572", - "4810": "6f636b6564", - "4811": "6272696e67", - "4812": "20726f7365", - "4813": "696566", - "4814": "2070726573656e746c79", - "4815": "2067656e746c656d656e", - "4816": "696e67696e67", - "4817": "206661766f75", - "4818": "696e666f726d", - "4819": "207363656e65", - "4820": "61676564", - "4821": "20736875", - "4822": "6f70696e", - "4823": "6f666665", - "4824": "6d656469", - "4825": "69737472", - "4826": "677565", - "4827": "726f66", - "4828": "20736f6d65626f6479", - "4829": "20616c74686f756768", - "4830": "206162736f6c757465", - "4831": "636f6e73657175656e", - "4832": "20616d6f6e677374", - "4833": "20616c6c6f776564", - "4834": "206c696b656c79", - "4835": "70726f63656564", - "4836": "2074726176656c", - "4837": "73756363657373", - "4838": "207265666c6563", - "4839": "6c64696e67", - "4840": "616e676564", - "4841": "2064657465", - "4842": "2043686172", - "4843": "68757272", - "4844": "6e646c65", - "4845": "74656172", - "4846": "73746179", - "4847": "6f6172", - "4848": "72736f", - "4849": "206469736170706f696e74", - "4850": "69676874656e", - "4851": "4d6164616d65", - "4852": "20636174", - "4853": "2073656c", - "4854": "68756e64", - "4855": "7274686572", - "4856": "20616e7869", - "4857": "696d656e74", - "4858": "756e61", - "4859": "656869", - "4860": "204f6e", - "4861": "696669", - "4862": "20696e7374616e746c79", - "4863": "66757274686572", - "4864": "68617465766572", - "4865": "61726e65", - "4866": "75746573", - "4867": "636f6c64", - "4868": "6d626c65", - "4869": "536f6d65", - "4870": "726f7465", - "4871": "66696e65", - "4872": "2e2e2e2e", - "4873": "6d6974", - "4874": "20617474656d7074", - "4875": "206e656974686572", - "4876": "6f757273656c7665", - "4877": "757273656c766573", - "4878": "2073747265657473", - "4879": "20636f6e6e656374", - "4880": "536865726c6f636b", - "4881": "696e6e696e67", - "4882": "657365727665", - "4883": "206a6f75726e", - "4884": "20706c656173616e74", - "4885": "61667465726e6f6f6e", - "4886": "736974756174696f6e", - "4887": "2063726f77", - "4888": "657374696e", - "4889": "7573686564", - "4890": "2073657474", - "4891": "6f7264696e", - "4892": "616c697479", - "4893": "6e6775", - "4894": "6d6167", - "4895": "707079", - "4896": "64697361", - "4897": "20736176", - "4898": "65617369", - "4899": "2061676f", - "4900": "61646564", - "4901": "656e6573", - "4902": "73686164", - "4903": "436f756e", - "4904": "667261", - "4905": "204d79", - "4906": "6e6669", - "4907": "0a0a0a0a2020202020202020", - "4908": "20696e746572657374696e67", - "4909": "20636f6d666f727461626c65", - "4910": "6173746f6e6973686d656e74", - "4911": "20696e666f726d6174696f6e", - "4912": "20636f6e666964656e6365", - "4913": "206d6173746572", - "4914": "20707265706172", - "4915": "206d6967687479", - "4916": "436861726c6573", - "4917": "207265706561", - "4918": "6f756e646564", - "4919": "6578616d696e", - "4920": "206a756467", - "4921": "2073617665", - "4922": "70726f6261", - "4923": "206c696e", - "4924": "637475", - "4925": "73736f", - "4926": "20466f", - "4927": "776966", - "4928": "7a65", - "4929": "4465", - "4930": "65727279", - "4931": "63696465", - "4932": "696e6b69", - "4933": "57697468", - "4934": "736f7272", - "4935": "206e6f62", - "4936": "62656172", - "4937": "61646d69", - "4938": "436861", - "4939": "206469737472", - "4940": "207472617665", - "4941": "62696c697479", - "4942": "636f6d706c69", - "4943": "20747279696e67", - "4944": "617474656d7074", - "4945": "6e7374616e746c", - "4946": "7374616e746c79", - "4947": "2070726f706572", - "4948": "6c6c7573", - "4949": "20616e67", - "4950": "20616273", - "4951": "6c697073", - "4952": "76616e74", - "4953": "66696365", - "4954": "70616e69", - "4955": "6e6e69", - "4956": "756d70", - "4957": "6974746572", - "4958": "207772697474656e", - "4959": "2073746174696f6e", - "4960": "20656e6761676564", - "4961": "2073756363657373", - "4962": "20436172", - "4963": "64657370", - "4964": "20737072", - "4965": "20696d6d", - "4966": "6f6e7374", - "4967": "64657363", - "4968": "73697665", - "4969": "20646966666963756c7479", - "4970": "207465617273", - "4971": "206566666563", - "4972": "206d6f757468", - "4973": "736574746c65", - "4974": "206d75726465", - "4975": "6d7072", - "4976": "20736e", - "4977": "207265636f6c6c656374", - "4978": "636f6d6d756e69636174", - "4979": "207768617465766572", - "4980": "7265636f6c6c656374", - "4981": "636f6e646974696f6e", - "4982": "2066696e697368", - "4983": "20456e676c6973", - "4984": "20726169736564", - "4985": "206d7973746572", - "4986": "656e64696e", - "4987": "65616b696e", - "4988": "6976656c79", - "4989": "6f756e6473", - "4990": "20536f6d65", - "4991": "20666f6f6c", - "4992": "75656e6365", - "4993": "20736c6f", - "4994": "6d6d6f6e", - "4995": "61676572", - "4996": "616e7469", - "4997": "6176656e", - "4998": "6f6c75", - "4999": "6b6573", - "5000": "726164", - "5001": "204368", - "5002": "4a6f", - "5003": "206f75676874", - "5004": "616c6c696e67", - "5005": "206167726565", - "5006": "707265706172", - "5007": "626c696e", - "5008": "20616c74", - "5009": "656e6469", - "5010": "7472656d", - "5011": "72757368", - "5012": "65726965", - "5013": "2077696e646f7773", - "5014": "7572646572", - "5015": "20696e7175", - "5016": "6e67657273", - "5017": "2064657363", - "5018": "65726f7573", - "5019": "636f61", - "5020": "756973", - "5021": "677569", - "5022": "726f62", - "5023": "5072", - "5024": "61746368696e67", - "5025": "206a6f75726e65", - "5026": "2074616c6b6564", - "5027": "206d6f6e746873", - "5028": "656c6967", - "5029": "656c6573", - "5030": "65726c79", - "5031": "20696e76", - "5032": "73636172", - "5033": "72656c61", - "5034": "63746c", - "5035": "756963", - "5036": "56616e", - "5037": "6170706f696e746d656e74", - "5038": "20646973636f7665726564", - "5039": "72656d61726b61626c", - "5040": "20656e746972656c79", - "5041": "736174697366616374", - "5042": "66696e697368", - "5043": "206a75646765", - "5044": "616c6c6f7765", - "5045": "726567617264", - "5046": "2061626c65", - "5047": "636174696f", - "5048": "2072657065", - "5049": "2062616e6b", - "5050": "2064657374", - "5051": "696d6d", - "5052": "636365", - "5053": "6f6e6f", - "5054": "707072", - "5055": "697366", - "5056": "20676c6f", - "5057": "69617465", - "5058": "6578706c", - "5059": "6c65616e", - "5060": "6c7574", - "5061": "2070617373616765", - "5062": "2066757274686572", - "5063": "2065786163746c79", - "5064": "616772656561626c", - "5065": "206173746f6e6973", - "5066": "20436861726c6573", - "5067": "20776169746564", - "5068": "2062726f6b656e", - "5069": "70706f7274", - "5070": "2064726976", - "5071": "2066617374", - "5072": "636c61696d", - "5073": "6d6f756e74", - "5074": "726f616368", - "5075": "62616e6b", - "5076": "676c6164", - "5077": "61636564", - "5078": "75736865", - "5079": "4d6f6e73", - "5080": "666c61", - "5081": "656572", - "5082": "6e6461", - "5083": "7477656e7479", - "5084": "65616e696e67", - "5085": "206472697665", - "5086": "70726f706572", - "5087": "72656675", - "5088": "20637261", - "5089": "6e666f72", - "5090": "75726368", - "5091": "4c616479", - "5092": "736f6d657768657265", - "5093": "70726f6365", - "5094": "7374657073", - "5095": "206c697073", - "5096": "6e616c6c79", - "5097": "206d6f6e74", - "5098": "756e636c65", - "5099": "205365", - "5100": "646972", - "5101": "756e73", - "5102": "2052", - "5103": "72656d656d6265726564", - "5104": "636f6e73696465726564", - "5105": "20717569636b6c79", - "5106": "6d6f756e7461696e", - "5107": "70617373616765", - "5108": "206578706c6169", - "5109": "207768696c7374", - "5110": "206c6164696573", - "5111": "7572696f7573", - "5112": "20666c6f6f72", - "5113": "736c69676874", - "5114": "726573746564", - "5115": "7461696e6564", - "5116": "74727565", - "5117": "206d6574", - "5118": "20686f6e", - "5119": "61727279", - "5120": "20776964", - "5121": "20537472", - "5122": "61756e74", - "5123": "204d6f6e", - "5124": "676c69", - "5125": "736d69", - "5126": "486172", - "5127": "746d656e74", - "5128": "7461697273", - "5129": "6f626f6479", - "5130": "696f6e6564", - "5131": "204d617279", - "5132": "726f647563", - "5133": "2061726d73", - "5134": "6f73696e67", - "5135": "20696e6469", - "5136": "2073757370", - "5137": "204d6f6e73", - "5138": "697a65", - "5139": "6f6e69", - "5140": "5374", - "5141": "72726564", - "5142": "70617468", - "5143": "6b6e6573", - "5144": "2077726f", - "5145": "206d6173", - "5146": "7175656e", - "5147": "736f756c", - "5148": "206573", - "5149": "656573", - "5150": "416c6c", - "5151": "69616e", - "5152": "6d6f74696f6e", - "5153": "206578616374", - "5154": "756e6e696e67", - "5155": "736e", - "5156": "206e65696768626f7572686f6f64", - "5157": "646973617070656172", - "5158": "696e666c75656e6365", - "5159": "20666f72676f74", - "5160": "206f70706f", - "5161": "6c79696e67", - "5162": "206f666665", - "5163": "7061757365", - "5164": "62696c6974", - "5165": "63686f6f", - "5166": "6966756c", - "5167": "616e7473", - "5168": "20746f6e", - "5169": "20727573", - "5170": "20706965", - "5171": "746c6573", - "5172": "6d696e61", - "5173": "72657175", - "5174": "656769", - "5175": "66696e6973686564", - "5176": "6f7264696e617279", - "5177": "206d697374616b65", - "5178": "617473", - "5179": "20736f6d657768657265", - "5180": "20696e74657272757074", - "5181": "20666f6c6c6f77696e67", - "5182": "6162736f6c7574656c79", - "5183": "70706572", - "5184": "7572696f", - "5185": "656e7472", - "5186": "79617264", - "5187": "6f6c7665", - "5188": "20736574746c", - "5189": "74726176656c", - "5190": "756374696f6e", - "5191": "6375727265", - "5192": "2073696d70", - "5193": "2073686172", - "5194": "206272696e", - "5195": "2065787472", - "5196": "2064697370", - "5197": "2073746f6e", - "5198": "70726f6163", - "5199": "2068756d61", - "5200": "6f7261", - "5201": "746f6d", - "5202": "646570", - "5203": "646974", - "5204": "436f75", - "5205": "207477656e7479", - "5206": "20706f73736573", - "5207": "67726f77", - "5208": "20686964", - "5209": "48657265", - "5210": "6e657273", - "5211": "70726f62", - "5212": "6d6964", - "5213": "206d6f756e7461696e", - "5214": "20737573706963696f", - "5215": "737573706963696f6e", - "5216": "206561726e657374", - "5217": "207475726e696e67", - "5218": "2066697665", - "5219": "4576657279", - "5220": "73686f7265", - "5221": "696d6d6564", - "5222": "206e6f6e65", - "5223": "2074727573", - "5224": "2073747564", - "5225": "6d696e6564", - "5226": "746974", - "5227": "727969", - "5228": "20636f6e7472", - "5229": "6c656374696f", - "5230": "20736861646f", - "5231": "20736372", - "5232": "616c6974", - "5233": "43617274", - "5234": "697073", - "5235": "6c7565", - "5236": "4d79", - "5237": "206173746f6e6973686d656e74", - "5238": "20706172746963756c61726c79", - "5239": "2072656d61726b61626c", - "5240": "2061667465726e6f6f6e", - "5241": "20736974756174696f6e", - "5242": "20436861726c6f747465", - "5243": "6561726e657374", - "5244": "2070726f636565", - "5245": "6c696768746564", - "5246": "73747265746368", - "5247": "2073687574", - "5248": "206d61726b", - "5249": "63726f7764", - "5250": "2070726f62", - "5251": "66697665", - "5252": "65616e73", - "5253": "2076696c", - "5254": "2072656d656d6265726564", - "5255": "646973706f736974696f6e", - "5256": "20636f6d6d756e69636174", - "5257": "2063726f7373", - "5258": "206669786564", - "5259": "72657373696e", - "5260": "206361726566756c", - "5261": "62656c6965766564", - "5262": "6772656561626c65", - "5263": "72746865", - "5264": "596f7572", - "5265": "20717561", - "5266": "637572696f73697479", - "5267": "20616476616e746167", - "5268": "207265636b6f6e6564", - "5269": "2070617573", - "5270": "6a75646765", - "5271": "6472697665", - "5272": "63696e", - "5273": "207363686f6f6c", - "5274": "69676874656e65", - "5275": "676874656e6564", - "5276": "6368616e676564", - "5277": "2064696e6e6572", - "5278": "6d697374616b65", - "5279": "77726974696e67", - "5280": "20576174736f6e", - "5281": "206f7070", - "5282": "65617379", - "5283": "6f757261", - "5284": "206e6569", - "5285": "20646576", - "5286": "656e6c", - "5287": "646574", - "5288": "64656e746c79", - "5289": "7374696e6374", - "5290": "20666f726365", - "5291": "636f6c6f7572", - "5292": "656564696e67", - "5293": "6e64657265", - "5294": "746368696e", - "5295": "6d6f757468", - "5296": "636f6e7472", - "5297": "6c65637469", - "5298": "6c696d", - "5299": "676174", - "5300": "206469736170706f696e", - "5301": "20696e666c75656e6365", - "5302": "6d656574", - "5303": "73616665", - "5304": "63756c74", - "5305": "206e6f7469636564", - "5306": "2073747265746368", - "5307": "20636f6d70616e79", - "5308": "63656c", - "5309": "636f6e73657175656e6365", - "5310": "657863656c6c656e74", - "5311": "20736c6f776c79", - "5312": "206d616e616765", - "5313": "206c656e677468", - "5314": "69737472657373", - "5315": "20726567617264", - "5316": "204d6164616d65", - "5317": "746974756465", - "5318": "206672696768", - "5319": "20617766756c", - "5320": "20776f6f6473", - "5321": "737472756374", - "5322": "646573637269", - "5323": "206368757263", - "5324": "206d696e65", - "5325": "7573696f6e", - "5326": "206561726c", - "5327": "2066726f6e", - "5328": "207363656e", - "5329": "2061637175", - "5330": "656e7479", - "5331": "69646573", - "5332": "2063686f", - "5333": "6e636c65", - "5334": "697274", - "5335": "727565", - "5336": "6c616d65", - "5337": "6f6f6b73", - "5338": "77616b65", - "5339": "2061646d", - "5340": "63686f6f6c", - "5341": "206d697373", - "5342": "2074726561", - "5343": "73696f6e73", - "5344": "206175", - "5345": "636865657266756c", - "5346": "2072656163686564", - "5347": "2064657374726f79", - "5348": "706f636b6574", - "5349": "72616e676572", - "5350": "636f6d6d656e", - "5351": "204c75636965", - "5352": "20706963747572", - "5353": "20666f72676574", - "5354": "6c6c6f77696e67", - "5355": "696d7072657373", - "5356": "64697374757262", - "5357": "20627265617374", - "5358": "6c6c6172", - "5359": "6669656c", - "5360": "696e6973", - "5361": "204e6f77", - "5362": "7265666f", - "5363": "20666967", - "5364": "75737469", - "5365": "64657465", - "5366": "706c79", - "5367": "696174", - "5368": "707469", - "5369": "6d7572", - "5370": "75656e", - "5371": "206469737472657373", - "5372": "7363686f6f", - "5373": "20776f7273", - "5374": "73656e7465", - "5375": "6c6f6e656c", - "5376": "6775617264", - "5377": "2067726174", - "5378": "61646d6972", - "5379": "6e6172", - "5380": "656375", - "5381": "67657474", - "5382": "63757269", - "5383": "20746875", - "5384": "2073696d", - "5385": "70726f63", - "5386": "61676573", - "5387": "64697370", - "5388": "65726365", - "5389": "616e67696e67", - "5390": "206578636570", - "5391": "68616e676564", - "5392": "657369646573", - "5393": "207472656573", - "5394": "206d65616e74", - "5395": "766963", - "5396": "436f6c", - "5397": "206f636375727265", - "5398": "2064697374757262", - "5399": "736f6c7574656c79", - "5400": "20696e74656e74", - "5401": "2072656c617469", - "5402": "2064617265", - "5403": "2072656c69", - "5404": "2073656174", - "5405": "7472617665", - "5406": "2064696e", - "5407": "6a756467", - "5408": "7466756c", - "5409": "63726574", - "5410": "69737469", - "5411": "61746976", - "5412": "73757370", - "5413": "6d617465", - "5414": "6c6f72", - "5415": "696767", - "5416": "616269", - "5417": "2065787472616f7264696e617279", - "5418": "20637572696f73697479", - "5419": "756e64657273746f6f64", - "5420": "20636f6e646974696f6e", - "5421": "6d616e616765", - "5422": "20636f6d6d6f", - "5423": "64696e6e6572", - "5424": "74696f6e6564", - "5425": "64697374696e", - "5426": "72656c617469", - "5427": "20696e666f72", - "5428": "206465737472", - "5429": "73657474", - "5430": "65787472", - "5431": "2066696e6973686564", - "5432": "20636865657266756c", - "5433": "2062656c6965766564", - "5434": "737472756374696f6e", - "5435": "7065637461", - "5436": "7973746572", - "5437": "2066616972", - "5438": "6c6f77696e", - "5439": "616e636564", - "5440": "206578616d", - "5441": "204c6f7264", - "5442": "667572", - "5443": "776c", - "5444": "736174697366616374696f6e", - "5445": "72656164696e67", - "5446": "20736f72726f77", - "5447": "20736f6c656d6e", - "5448": "64657370616972", - "5449": "20646573637269", - "5450": "206b6e6f776564", - "5451": "20656d70", - "5452": "206d7572", - "5453": "636f6c6c", - "5454": "756467", - "5455": "656666", - "5456": "726c65", - "5457": "20706c656173616e", - "5458": "68616e64736f6d65", - "5459": "2073697374657273", - "5460": "20696d7072657373", - "5461": "6e6973686d656e74", - "5462": "72656c6174696f6e", - "5463": "7265736f6c766564", - "5464": "2073756666696369656e74", - "5465": "72697474656e", - "5466": "20696d706f73", - "5467": "706c61636564", - "5468": "20636f6e7365", - "5469": "6e6775697368", - "5470": "617264656e", - "5471": "7175697265", - "5472": "6469737472", - "5473": "204a6f686e", - "5474": "747561", - "5475": "616473", - "5476": "205369", - "5477": "67676564", - "5478": "656e6368", - "5479": "20656e67", - "5480": "72656d6f", - "5481": "746e6573", - "5482": "756374", - "5483": "707465", - "5484": "4d6f", - "5485": "4f68", - "5486": "20657863656c6c656e74", - "5487": "20646973617070656172", - "5488": "73656e736174696f6e", - "5489": "207265636f6c6c6563", - "5490": "207363617263656c79", - "5491": "696e6973686564", - "5492": "2063616e646c65", - "5493": "73686f7765", - "5494": "617766756c", - "5495": "7269766564", - "5496": "617264696e", - "5497": "7363656e65", - "5498": "6967687479", - "5499": "756e636865", - "5500": "207472616e", - "5501": "6861726d", - "5502": "63726f77", - "5503": "656e7461", - "5504": "70726563", - "5505": "6e737465", - "5506": "71756172", - "5507": "65657069", - "5508": "636f7264", - "5509": "20416c6c", - "5510": "69657479", - "5511": "616372", - "5512": "726d73", - "5513": "207472757468", - "5514": "726f77696e67", - "5515": "69636174696f", - "5516": "6c6965766564", - "5517": "736561726368", - "5518": "696e74656e74", - "5519": "207075727375", - "5520": "207769746e657373", - "5521": "746f75", - "5522": "64646c65", - "5523": "626c6564", - "5524": "6f757265", - "5525": "6973696e", - "5526": "72656865", - "5527": "64756365", - "5528": "6469736170706f696e746d656e74", - "5529": "7373616765", - "5530": "6d6f72726f", - "5531": "2066696e65", - "5532": "6170657273", - "5533": "666f726365", - "5534": "6e666f726d", - "5535": "20756e636c", - "5536": "20636f6e73696465726564", - "5537": "72756e6e696e67", - "5538": "20617474656d70", - "5539": "20777265746368", - "5540": "6f757273656c76", - "5541": "64657374726f79", - "5542": "20636f6e6e6563", - "5543": "6c6174696f6e73", - "5544": "206f63637572", - "5545": "777269747465", - "5546": "206368656572", - "5547": "696768746564", - "5548": "20636c6f7564", - "5549": "20706963", - "5550": "75727265", - "5551": "736c6f77", - "5552": "206d6964", - "5553": "20626164", - "5554": "70696563", - "5555": "20626967", - "5556": "6e7175", - "5557": "4875", - "5558": "206f63637572726564", - "5559": "696d706f7274616e74", - "5560": "2072656c6174696f6e", - "5561": "2073746169", - "5562": "746572696e", - "5563": "6f756e6465", - "5564": "206a6f696e", - "5565": "7264657265", - "5566": "2072757368", - "5567": "73656e6365", - "5568": "6c656d656e", - "5569": "2061756e74", - "5570": "436172", - "5571": "6f626c", - "5572": "72656d61726b61626c65", - "5573": "616c746f676574686572", - "5574": "20636f6e73657175656e", - "5575": "636f6e6e656374696f6e", - "5576": "2053657074656d626572", - "5577": "696e646966666572656e", - "5578": "2070617373696f6e", - "5579": "6469737472657373", - "5580": "6b696e646e657373", - "5581": "20436174", - "5582": "74756465", - "5583": "756e7461", - "5584": "68696c6c", - "5585": "6f726469", - "5586": "696e616c", - "5587": "7175616c", - "5588": "626f6174", - "5589": "756564", - "5590": "696e64696e67", - "5591": "63616e646c65", - "5592": "666f72676574", - "5593": "726f64756365", - "5594": "6c6561736564", - "5595": "2070726f7065", - "5596": "697368696e67", - "5597": "20736c696768", - "5598": "206d656d6f72", - "5599": "206578616d69", - "5600": "204b7572747a", - "5601": "636f6e73696465726174696f6e", - "5602": "20706f636b6574", - "5603": "20666f726d6572", - "5604": "20646f63746f72", - "5605": "736369656e6365", - "5606": "7265616469", - "5607": "616e646c65", - "5608": "6573746572", - "5609": "636f6e7365", - "5610": "697374696e", - "5611": "6f726d6564", - "5612": "6f617264", - "5613": "656c6174", - "5614": "73637269", - "5615": "6e7479", - "5616": "697361", - "5617": "636c75", - "5618": "61676f", - "5619": "686574", - "5620": "66726169", - "5621": "206e6f6e", - "5622": "75626c69", - "5623": "70656e64", - "5624": "77656574", - "5625": "206162736f6c7574656c79", - "5626": "20667269676874656e", - "5627": "70726573656e746564", - "5628": "636f6e736964657265", - "5629": "6f6e73696465726564", - "5630": "2070726573656e6365", - "5631": "207265736f6c766564", - "5632": "2072756e6e696e67", - "5633": "666f726568656164", - "5634": "2064657370616972", - "5635": "2073796d70617468", - "5636": "2074726176656c6c", - "5637": "2071756172746572", - "5638": "20666f7274756e65", - "5639": "206665746368", - "5640": "736f72726f77", - "5641": "6c656e677468", - "5642": "207468726577", - "5643": "61696e696e67", - "5644": "206561726c79", - "5645": "646f63746f72", - "5646": "20776f7274", - "5647": "206b696c6c", - "5648": "6c696b656c", - "5649": "2074686972", - "5650": "20666c6f6f", - "5651": "2070726576", - "5652": "6965766564", - "5653": "6672657368", - "5654": "2073706563", - "5655": "6c6e657373", - "5656": "6966696564", - "5657": "6177796572", - "5658": "6f7069", - "5659": "737375", - "5660": "5965", - "5661": "696e7465726573746564", - "5662": "7265666c656374696f6e", - "5663": "20737573706963696f6e", - "5664": "6d6973666f7274756e65", - "5665": "77616c6b696e67", - "5666": "20667572746865", - "5667": "20657363617065", - "5668": "2073746174696f", - "5669": "20706c61636564", - "5670": "206d697374616b", - "5671": "20737563636573", - "5672": "207075626c6963", - "5673": "6f6c7574696f6e", - "5674": "66696e69", - "5675": "696c696e", - "5676": "616e6369", - "5677": "616e746c", - "5678": "6578616374", - "5679": "2068756e67", - "5680": "7374657273", - "5681": "204b696e67", - "5682": "2064756b65", - "5683": "6c6567", - "5684": "776879", - "5685": "696172", - "5686": "4a61", - "5687": "207374616972", - "5688": "726967687465", - "5689": "637573746f6d", - "5690": "736f6c656d6e", - "5691": "657374696e67", - "5692": "616c6c69", - "5693": "686f6f6b", - "5694": "65737361", - "5695": "20626c75", - "5696": "746c6564", - "5697": "204d616e", - "5698": "7379", - "5699": "4966", - "5700": "636b6c", - "5701": "206d616e6e657273", - "5702": "20736572696f7573", - "5703": "6772656174657374", - "5704": "20696e6e6f63656e", - "5705": "7269676874656e6564", - "5706": "2068616e64736f6d65", - "5707": "20696e746572727570", - "5708": "68616e67696e67", - "5709": "65617475726573", - "5710": "72657374696e67", - "5711": "736572696f7573", - "5712": "20776174636865", - "5713": "206c6976696e67", - "5714": "7370656369616c", - "5715": "71756172746572", - "5716": "65636b6f6e6564", - "5717": "54656c6c736f6e", - "5718": "6c6f776572", - "5719": "74656c6c69", - "5720": "20666f7263", - "5721": "68696c7374", - "5722": "2070726f66", - "5723": "6861626974", - "5724": "20636c6f74", - "5725": "20796573", - "5726": "73686573", - "5727": "20646f67", - "5728": "69676e65", - "5729": "4c6f7264", - "5730": "20616c746f676574686572", - "5731": "20756e64657273746f6f64", - "5732": "2070726f66657373696f6e", - "5733": "206d6973666f7274756e65", - "5734": "737465726461", - "5735": "61746368696e", - "5736": "20666f726765", - "5737": "206d61737465", - "5738": "206775617264", - "5739": "666665726564", - "5740": "207072657061", - "5741": "20626c657373", - "5742": "70616c", - "5743": "20736174697366616374", - "5744": "626f6172", - "5745": "6b696c6c", - "5746": "72616964", - "5747": "6c696573", - "5748": "7072696e", - "5749": "2070616c", - "5750": "736f6c65", - "5751": "666c6f6f72", - "5752": "2070726563", - "5753": "746c657373", - "5754": "204672", - "5755": "6f6365", - "5756": "6e6475", - "5757": "6476656e74757265", - "5758": "20637572696f7573", - "5759": "20736574746c6564", - "5760": "7761746368696e67", - "5761": "6469736170706561", - "5762": "6973617070656172", - "5763": "207265636f766572", - "5764": "706563756c696172", - "5765": "207374727567676c", - "5766": "20706173736167", - "5767": "6f7264696e6172", - "5768": "63656564696e67", - "5769": "6f70706f", - "5770": "6c617373", - "5771": "6d6d6564", - "5772": "20736d6f", - "5773": "6f706572", - "5774": "3fe28099", - "5775": "72657061", - "5776": "4d6f6e", - "5777": "2056", - "5778": "206b6e6f7773", - "5779": "207072657373", - "5780": "206465736572", - "5781": "706c61636573", - "5782": "636964656e74", - "5783": "746c656d656e", - "5784": "696e74656e74696f6e", - "5785": "206772656174657374", - "5786": "206b696e646e657373", - "5787": "657870726573736564", - "5788": "7472757468", - "5789": "206c6f636b", - "5790": "696d706f73", - "5791": "7761697465", - "5792": "6374696e67", - "5793": "696e67696e", - "5794": "20706f6c69", - "5795": "74696f6e65", - "5796": "6f76696e67", - "5797": "2072656d6f", - "5798": "6c6f766564", - "5799": "65616c7468", - "5800": "6c6c75", - "5801": "636c69", - "5802": "76696f", - "5803": "736963", - "5804": "476f6f64", - "5805": "57696c6c", - "5806": "64696573", - "5807": "666f6f6c", - "5808": "20736f6e", - "5809": "636961", - "5810": "6978", - "5811": "756e666f7274756e617465", - "5812": "696d6167696e6174696f6e", - "5813": "2073656e736174696f6e", - "5814": "206f757273656c766573", - "5815": "70726f66657373696f6e", - "5816": "636f6d706c696d656e74", - "5817": "206d6964646c65", - "5818": "637572696f7573", - "5819": "7269676874656e", - "5820": "20736561726368", - "5821": "20626974746572", - "5822": "20617373697374", - "5823": "20646567726565", - "5824": "2066696e6973", - "5825": "6c6179696e67", - "5826": "2077726f6e67", - "5827": "676c617373", - "5828": "6572617465", - "5829": "6368656572", - "5830": "756d626572", - "5831": "20746f6e65", - "5832": "64616e6365", - "5833": "2073756d6d", - "5834": "776f7274", - "5835": "65726e65", - "5836": "6f6c6465", - "5837": "6976656c", - "5838": "206d6565", - "5839": "696f6e65", - "5840": "2073756d", - "5841": "70617261", - "5842": "20726576", - "5843": "756e6765", - "5844": "697070", - "5845": "4e6f74", - "5846": "726970", - "5847": "757264", - "5848": "726c6f", - "5849": "7962", - "5850": "667269676874656e", - "5851": "726573656e746564", - "5852": "2077617463686564", - "5853": "206d6f756e746169", - "5854": "2070726f66657373", - "5855": "7065726365697665", - "5856": "206569", - "5857": "73686172", - "5858": "61726973", - "5859": "72657065", - "5860": "6c6c6961", - "5861": "6d6f7469", - "5862": "72697479", - "5863": "72656c69", - "5864": "63617374", - "5865": "73747564", - "5866": "20626f61", - "5867": "6578636c61696d6564", - "5868": "20666f726568656164", - "5869": "2070726f6261626c79", - "5870": "206465736372696265", - "5871": "207374727567676c65", - "5872": "20696e666c75656e63", - "5873": "636f6e76696e636564", - "5874": "696e6e696e", - "5875": "7374616b65", - "5876": "4173", - "5877": "20626567756e", - "5878": "2073746f7279", - "5879": "67656d656e74", - "5880": "207374657073", - "5881": "20636f6e7669", - "5882": "2067617264656e", - "5883": "66696e69736865", - "5884": "7264696e617279", - "5885": "20706c61636573", - "5886": "66617368696f6e", - "5887": "7265736f6c7574", - "5888": "207265736f6c76", - "5889": "656d626c", - "5890": "6a6f696e", - "5891": "73617665", - "5892": "73756363", - "5893": "6f757269", - "5894": "61726e61", - "5895": "616273", - "5896": "7772657463686564", - "5897": "2070726576656e74", - "5898": "2077726974696e67", - "5899": "2070617373696e67", - "5900": "2067726561746572", - "5901": "6465736372696265", - "5902": "205175696e636579", - "5903": "6970706564", - "5904": "2057696c6c", - "5905": "20736f6674", - "5906": "6369656e74", - "5907": "7472657373", - "5908": "6963617465", - "5909": "6569676e65", - "5910": "656176656e", - "5911": "204261", - "5912": "667269676874656e6564", - "5913": "206d656e74696f6e6564", - "5914": "206578636c61696d6564", - "5915": "69646564", - "5916": "6c6c6572", - "5917": "7766756c", - "5918": "20736166", - "5919": "726b6572", - "5920": "6569676e", - "5921": "2072656d61726b61626c65", - "5922": "20667269676874656e6564", - "5923": "207265666c656374696f6e", - "5924": "20696e646966666572656e", - "5925": "72656164696e", - "5926": "67687466756c", - "5927": "73747261696e", - "5928": "20686f727269", - "5929": "657175656e74", - "5930": "20756e666f7274756e617465", - "5931": "207772657463686564", - "5932": "207761746368696e67", - "5933": "616476656e74757265", - "5934": "206561726e6573", - "5935": "6e646572696e67", - "5936": "20657863697465", - "5937": "7373656e676572", - "5938": "73796d70617468", - "5939": "73747275637469", - "5940": "636f7474616765", - "5941": "726d6174696f6e", - "5942": "756e646564", - "5943": "2073656c66", - "5944": "2074726163", - "5945": "6964696e67", - "5946": "61727261", - "5947": "77656172", - "5948": "6f6f6473", - "5949": "75696c", - "5950": "2e2e", - "5951": "6c70", - "5952": "756d65", - "5953": "6e6f62", - "5954": "616665", - "5955": "6e7468", - "5956": "204172", - "5957": "64697374696e6374", - "5958": "7374727567676c65", - "5959": "6672657175656e74", - "5960": "207363686f6f", - "5961": "657363617065", - "5962": "206f66666572", - "5963": "616e63696e67", - "5964": "206174746163", - "5965": "65726e657373", - "5966": "70726568656e", - "5967": "20637279", - "5968": "70616972", - "5969": "63617469", - "5970": "20666978", - "5971": "6d6d656e", - "5972": "656c6c73", - "5973": "72696465", - "5974": "204c6164", - "5975": "4869", - "5976": "616c6c696e", - "5977": "206465616c", - "5978": "2065617379", - "5979": "616e646564", - "5980": "726576656e", - "5981": "2074726176", - "5982": "626f72", - "5983": "736369", - "5984": "4f6e", - "5985": "636f6e636c7573696f6e", - "5986": "6f63636173696f6e616c", - "5987": "696d706f7274616e6365", - "5988": "20636f6e76696e636564", - "5989": "636f6e64756374", - "5990": "206566666f7274", - "5991": "45766572", - "5992": "20457665", - "5993": "72697469", - "5994": "61726469", - "5995": "636b73", - "5996": "637574", - "5997": "206c75", - "5998": "6f7468657277697365", - "5999": "536f6d6574696d6573", - "6000": "20636f6e646974696f", - "6001": "6c6f636b6564", - "6002": "206573636170", - "6003": "20736f727279", - "6004": "656e74697265", - "6005": "617373697374", - "6006": "207065616365", - "6007": "7261637469", - "6008": "6865616473", - "6009": "2074656d70", - "6010": "206d6f7574", - "6011": "737472656e", - "6012": "6169736564", - "6013": "61696e7465", - "6014": "6d69737369", - "6015": "6172746572", - "6016": "736875", - "6017": "737072", - "6018": "6f6163", - "6019": "76696365", - "6020": "69786564", - "6021": "6f6c656e", - "6022": "2057696c", - "6023": "73686f6f", - "6024": "20636f6e64756374", - "6025": "206c656176696e67", - "6026": "2064697374726573", - "6027": "2066696e67657273", - "6028": "20746f7563686564", - "6029": "6d696e6174696f6e", - "6030": "7065727375616465", - "6031": "2054656c6c736f6e", - "6032": "72646c", - "6033": "20737472657463", - "6034": "7265636f766572", - "6035": "7769746e657373", - "6036": "70726f66657373", - "6037": "206f756768", - "6038": "6169746564", - "6039": "6963617469", - "6040": "756e6e696e", - "6041": "2057697468", - "6042": "2061677265", - "6043": "67756c6172", - "6044": "626f756e64", - "6045": "7461696e65", - "6046": "61696c6564", - "6047": "7075727375", - "6048": "69747564", - "6049": "61737469", - "6050": "6d617469", - "6051": "64616e63", - "6052": "796f6e64", - "6053": "70756c", - "6054": "206561676572", - "6055": "204166746572", - "6056": "727573686564", - "6057": "636861726765", - "6058": "697374657273", - "6059": "6c6563746564", - "6060": "626974746572", - "6061": "657061726174", - "6062": "657863697465", - "6063": "736f6c766564", - "6064": "7865", - "6065": "657277", - "6066": "20736b", - "6067": "657761", - "6068": "20696e74656e74696f6e", - "6069": "7265736f6c7574696f6e", - "6070": "6163717561696e746564", - "6071": "73746c79", - "6072": "67657374", - "6073": "72736573", - "6074": "756c6174", - "6075": "61636b65", - "6076": "6d706c69", - "6077": "206469736170706561", - "6078": "74656e6365", - "6079": "2073706565", - "6080": "6665746368", - "6081": "206d657265", - "6082": "206e6f7465", - "6083": "6974696573", - "6084": "74696e6374", - "6085": "7365727669", - "6086": "6e666f", - "6087": "657572", - "6088": "2063686565726675", - "6089": "696e74726f647563", - "6090": "207265736f6c7574", - "6091": "736f6c7574696f6e", - "6092": "20636f7474616765", - "6093": "7079", - "6094": "20636f6e636c7573696f6e", - "6095": "6469736170706561726564", - "6096": "20636f6e6e656374696f6e", - "6097": "6b6e6f77696e67", - "6098": "706c6561736564", - "6099": "636c61696d6564", - "6100": "77696c6c696e67", - "6101": "20616666616972", - "6102": "2064697374696e", - "6103": "20626561757479", - "6104": "70726570617265", - "6105": "6d697373696f6e", - "6106": "61747465726564", - "6107": "2064657374726f", - "6108": "20636f7573696e", - "6109": "73656e746564", - "6110": "206661696e74", - "6111": "776174636865", - "6112": "706172656e74", - "6113": "72757468", - "6114": "616e646c", - "6115": "2066616e", - "6116": "66697368", - "6117": "78706c61", - "6118": "6565696e", - "6119": "74696e65", - "6120": "20626c65", - "6121": "72746f6e", - "6122": "69737369", - "6123": "6e737472", - "6124": "726f6261", - "6125": "657479", - "6126": "6d696c6573", - "6127": "2066616e63", - "6128": "73696f6e61", - "6129": "20736f6c65", - "6130": "2064726577", - "6131": "207765616b", - "6132": "6e63696e67", - "6133": "6d696e6174", - "6134": "2070726179", - "6135": "666f726d61", - "6136": "7265717565", - "6137": "70696c", - "6138": "6d656c", - "6139": "757263", - "6140": "656772", - "6141": "6c656164", - "6142": "63617465", - "6143": "69666963", - "6144": "7370656e", - "6145": "616666656374696f6e617465", - "6146": "696e76", - "6147": "6e746d656e74", - "6148": "206d616e6167", - "6149": "757474696e67", - "6150": "206d616b6573", - "6151": "2064696e6e65", - "6152": "2062656c6f77", - "6153": "20737465616d", - "6154": "646973706f73", - "6155": "73747265616d", - "6156": "206272656173", - "6157": "636f6d706f73", - "6158": "20657175616c", - "6159": "756e6465727374616e64696e67", - "6160": "20706563756c696172", - "6161": "207065727375616465", - "6162": "6d706f7274616e74", - "6163": "617272616e6765", - "6164": "70726576656e74", - "6165": "2047656f726765", - "6166": "20646973747572", - "6167": "696e646e657373", - "6168": "206d6973657279", - "6169": "436f6c6f6e656c", - "6170": "7469726564", - "6171": "636b696e67", - "6172": "6f63637572", - "6173": "20756e636f", - "6174": "7472616374", - "6175": "7369657572", - "6176": "2e2e2e2e2e", - "6177": "77697365", - "6178": "206a7564", - "6179": "6e616c6c", - "6180": "61796564", - "6181": "72696c", - "6182": "736368", - "6183": "4d696e", - "6184": "6c7468", - "6185": "647572", - "6186": "776f7273", - "6187": "656d6f72", - "6188": "6d707265", - "6189": "696e746572727570746564", - "6190": "20646966666572656e6365", - "6191": "207265736f6c7574696f6e", - "6192": "206865616473", - "6193": "207368617270", - "6194": "2073686f7265", - "6195": "206c656e6774", - "6196": "2072656c6965", - "6197": "20626f756e64", - "6198": "207265676172", - "6199": "76656d656e74", - "6200": "6163636f6d70", - "6201": "204576657279", - "6202": "61696e746564", - "6203": "206576656e74", - "6204": "72616c6c79", - "6205": "2065786163", - "6206": "20656e6465", - "6207": "676c6f", - "6208": "737573", - "6209": "72657365727665", - "6210": "206c6174746572", - "6211": "20696d70726573", - "6212": "6e6973686d656e", - "6213": "20536177796572", - "6214": "63757272", - "6215": "20576879", - "6216": "20736f66", - "6217": "6f6f7365", - "6218": "72656573", - "6219": "20777265", - "6220": "666f7264", - "6221": "20417274", - "6222": "727473", - "6223": "726f64", - "6224": "746873", - "6225": "206368616e676564", - "6226": "2070726570617265", - "6227": "2064657363726962", - "6228": "656e6765616e6365", - "6229": "20617272616e6765", - "6230": "2064726177696e67", - "6231": "74696e6775697368", - "6232": "5361", - "6233": "50726573656e746c79", - "6234": "6578697374656e6365", - "6235": "737566666572696e67", - "6236": "72656c6174696f6e73", - "6237": "20536865726c6f636b", - "6238": "2070617374", - "6239": "6565646564", - "6240": "2077696e65", - "6241": "206d6f6f6e", - "6242": "6d656d6f72", - "6243": "7374726561", - "6244": "7061726174", - "6245": "636f6e7669", - "6246": "736f6c7665", - "6247": "2072616674", - "6248": "6e6963", - "6249": "706f6c", - "6250": "746163", - "6251": "206469736170706561726564", - "6252": "20646973706f736974696f6e", - "6253": "20636f6e73657175656e6365", - "6254": "20616476656e74757265", - "6255": "2070726f66657373696f", - "6256": "20616772656561626c65", - "6257": "657768657265", - "6258": "726465726564", - "6259": "6c6c6f77696e", - "6260": "20736d6f6b65", - "6261": "646570656e64", - "6262": "206578636974", - "6263": "66696369656e", - "6264": "207374756479", - "6265": "61766573", - "6266": "20737061", - "6267": "706f6c69", - "6268": "72746572", - "6269": "636f6c6f", - "6270": "65727669", - "6271": "20636f6e73696465726174696f6e", - "6272": "457665", - "6273": "796f6e", - "6274": "20616666656374696f6e617465", - "6275": "48657274666f72647368697265", - "6276": "20636f6e636c75", - "6277": "20636861726765", - "6278": "636f6e74656d70", - "6279": "20636175676874", - "6280": "207769746e6573", - "6281": "4e6f7468696e67", - "6282": "206163636f6d70", - "6283": "2062656174", - "6284": "636c6f7564", - "6285": "2070757273", - "6286": "696c6573", - "6287": "20747765", - "6288": "6c696361", - "6289": "6d696c69", - "6290": "756e6473", - "6291": "20706f6c", - "6292": "70726576", - "6293": "65617669", - "6294": "656d706c", - "6295": "686f74", - "6296": "6e7669", - "6297": "206469736170706f696e746d656e74", - "6298": "20636f6d706c696d656e74", - "6299": "2066726967687465", - "6300": "70726573656e7465", - "6301": "2077616c6b696e67", - "6302": "20636f6e66657373", - "6303": "677261746566756c", - "6304": "20436f6c6f6e656c", - "6305": "76696f6c656e", - "6306": "656d656e7473", - "6307": "206f63637570", - "6308": "6e656c", - "6309": "206566", - "6310": "656d69", - "6311": "69707065", - "6312": "64617265", - "6313": "64696e65", - "6314": "20616476656e747572", - "6315": "70617373656e676572", - "6316": "206672657175656e74", - "6317": "6f6d706c696d656e74", - "6318": "20736f6c69", - "6319": "2063726f73", - "6320": "6f66666572", - "6321": "77696c6c69", - "6322": "7065726564", - "6323": "656c696566", - "6324": "2064616e63", - "6325": "6f6c766564", - "6326": "626c657373", - "6327": "7261746564", - "6328": "7370656369", - "6329": "7472616465", - "6330": "756e6b", - "6331": "796572", - "6332": "206578697374656e6365", - "6333": "61636b6e6f776c656467", - "6334": "20657870726573736564", - "6335": "20737566666572696e67", - "6336": "2077616e646572", - "6337": "2070617373696f", - "6338": "6c617567686564", - "6339": "65787472656d65", - "6340": "20737570706572", - "6341": "2073747265616d", - "6342": "636f6e66657373", - "6343": "20617272616e67", - "6344": "20747769", - "6345": "7475616c", - "6346": "73697479", - "6347": "6e616765", - "6348": "74686972", - "6349": "65616b69", - "6350": "696e6e65", - "6351": "20536f6d", - "6352": "61727265", - "6353": "6173746c", - "6354": "7261636b", - "6355": "65726f", - "6356": "656865", - "6357": "75696e", - "6358": "756767", - "6359": "20736f72726f", - "6360": "20736f6c656d", - "6361": "64657363656e", - "6362": "636173746c65", - "6363": "207374726561", - "6364": "20636f6e6665", - "6365": "206465736372", - "6366": "7277", - "6367": "7072656369", - "6368": "64656e746c", - "6369": "2071756172", - "6370": "2064696564", - "6371": "7374696e63", - "6372": "7373697665", - "6373": "686f727269", - "6374": "206b6e6f77696e67", - "6375": "2068616e64736f6d", - "6376": "20616476616e6365", - "6377": "696d707265737369", - "6378": "2073657276696365", - "6379": "2070657263656976", - "6380": "204d617271756973", - "6381": "63637572", - "6382": "69746368", - "6383": "20706561", - "6384": "65726f75", - "6385": "6369616c", - "6386": "72616465", - "6387": "204a6f65", - "6388": "6571", - "6389": "757361", - "6390": "736176", - "6391": "6e6772", - "6392": "20736174697366616374696f6e", - "6393": "20696d706f7274616e6365", - "6394": "6578706563746174696f6e", - "6395": "2073656e736174696f", - "6396": "20696d707265737369", - "6397": "636f6d706c696d656e", - "6398": "207065726365697665", - "6399": "766964656e6365", - "6400": "2073696d706c65", - "6401": "2076696f6c656e", - "6402": "20646573706169", - "6403": "2073706f6b656e", - "6404": "616476616e6365", - "6405": "2073796d706174", - "6406": "20717561727465", - "6407": "20706572737561", - "6408": "207265616469", - "6409": "2063616e646c", - "6410": "20676c617373", - "6411": "6f7563686564", - "6412": "206d61796265", - "6413": "656176696e67", - "6414": "206d6164616d", - "6415": "7469747564", - "6416": "616e646572", - "6417": "206c656164", - "6418": "6c61636564", - "6419": "2067726173", - "6420": "2061776675", - "6421": "20736d6f6b", - "6422": "737465616d", - "6423": "206372696d", - "6424": "657363656e", - "6425": "7375706572", - "6426": "657175616c", - "6427": "6666696369", - "6428": "70617374", - "6429": "686f7265", - "6430": "20706974", - "6431": "74726164", - "6432": "73656174", - "6433": "646f7773", - "6434": "205061", - "6435": "726966", - "6436": "747461", - "6437": "77726f", - "6438": "537472", - "6439": "746e65", - "6440": "706f7373657373696f6e", - "6441": "61646d69726174696f6e", - "6442": "65657065", - "6443": "746d656e", - "6444": "206c6f63", - "6445": "726f7070", - "6446": "68656572", - "6447": "64696564", - "6448": "72616674", - "6449": "206c617567686564", - "6450": "2065787472656d65", - "6451": "756e64657273746f", - "6452": "20686f727269626c", - "6453": "2073756666657265", - "6454": "6578707265737365", - "6455": "7870726573736564", - "6456": "6f77657273", - "6457": "77726f6e67", - "6458": "206b6e6565", - "6459": "204174", - "6460": "617965", - "6461": "67617264656e", - "6462": "20736576656e", - "6463": "206661636573", - "6464": "206269747465", - "6465": "207375706572", - "6466": "20636f766572", - "6467": "736372696265", - "6468": "207075626c69", - "6469": "20706f73697469", - "6470": "20736572696f75", - "6471": "696d6167696e61", - "6472": "20636f6c6f7572", - "6473": "20696e73747275", - "6474": "6d757264657265", - "6475": "65616473", - "6476": "6c79696e", - "6477": "20736978", - "6478": "2073616e", - "6479": "756e636f", - "6480": "65646c79", - "6481": "6d706174", - "6482": "6d706572", - "6483": "74657073", - "6484": "6f73696e", - "6485": "706f7374", - "6486": "756e6e", - "6487": "776964", - "6488": "20696d6167696e6174696f6e", - "6489": "6c696768746e696e67", - "6490": "726570726573656e74", - "6491": "737465616d626f6174", - "6492": "20696e6e6f63656e74", - "6493": "7572726965", - "6494": "20636f6c6f", - "6495": "6f63637570", - "6496": "2073686970", - "6497": "6c756e", - "6498": "72696e6b", - "6499": "61736869", - "6500": "61726573", - "6501": "73686972", - "6502": "73756d6d", - "6503": "70726f66", - "6504": "206f63636173696f6e616c", - "6505": "20696e7465726573746564", - "6506": "656e646561766f75726564", - "6507": "657863656564696e676c79", - "6508": "20706c61696e", - "6509": "206361727279", - "6510": "20646f63746f", - "6511": "736c6f", - "6512": "616c61", - "6513": "686964", - "6514": "6f7669", - "6515": "6274", - "6516": "2064726573736564", - "6517": "2077726574636865", - "6518": "636f6c6c65637469", - "6519": "746f6c657261626c", - "6520": "2073756767657374", - "6521": "20536f6d6574696d6573", - "6522": "636f6c6c656374696f6e", - "6523": "20637572696f75", - "6524": "20636f6e647563", - "6525": "6d65616e696e67", - "6526": "20656173696c79", - "6527": "636f6e74656e74", - "6528": "73617070656172", - "6529": "207265636f7665", - "6530": "6967687466756c", - "6531": "20537472656574", - "6532": "206865616c7468", - "6533": "69726174696f6e", - "6534": "20656d706c6f79", - "6535": "43686170746572", - "6536": "726f77696e", - "6537": "7274696e67", - "6538": "6365737369", - "6539": "6368696566", - "6540": "206f626c69", - "6541": "7570706572", - "6542": "2048657265", - "6543": "2072657665", - "6544": "7065616365", - "6545": "64726577", - "6546": "75737465", - "6547": "74616c6c", - "6548": "72657375", - "6549": "6d6f7572", - "6550": "63697479", - "6551": "74687573", - "6552": "20746f72", - "6553": "6c6469", - "6554": "6e6f6e", - "6555": "617379", - "6556": "657069", - "6557": "204a75", - "6558": "6c6167", - "6559": "66696c6c6564", - "6560": "206c69766564", - "6561": "617373696e67", - "6562": "206f626c6967", - "6563": "206e6f626c65", - "6564": "20626c696e64", - "6565": "6861696e", - "6566": "6f6d6577", - "6567": "736f6e73", - "6568": "436f6d65", - "6569": "54656c6c", - "6570": "6f757473", - "6571": "6c6f7572", - "6572": "65677265", - "6573": "466f", - "6574": "2065766964656e6365", - "6575": "6265686176696f7572", - "6576": "6761726465", - "6577": "697368696e", - "6578": "207369636b", - "6579": "2061637469", - "6580": "756e6465727374616e64696e", - "6581": "6e6465727374616e64696e67", - "6582": "206d65616e696e67", - "6583": "696d70617469656e", - "6584": "7374726574636865", - "6585": "20636f6e74656d70", - "6586": "2062657369646573", - "6587": "6163636964656e74", - "6588": "686f727269626c65", - "6589": "206578616d696e65", - "6590": "726573656e7465", - "6591": "6573656e746564", - "6592": "20636f75706c65", - "6593": "20616476616e63", - "6594": "6578616d696e65", - "6595": "20646973706f73", - "6596": "20636f6e736369", - "6597": "73657276696365", - "6598": "2070726f666573", - "6599": "206b6e69", - "6600": "20626f78", - "6601": "20616765", - "6602": "20726973", - "6603": "726e6579", - "6604": "20626f72", - "6605": "6d656e64", - "6606": "736b69", - "6607": "736c61", - "6608": "666967", - "6609": "204b69", - "6610": "6c7573696f6e", - "6611": "616363657074", - "6612": "72657374696e", - "6613": "20676c6f6f6d", - "6614": "2064616e6365", - "6615": "696e74656e64", - "6616": "204a65727279", - "6617": "2074727574", - "6618": "7368617270", - "6619": "696e617279", - "6620": "206e657773", - "6621": "6e616e6365", - "6622": "646963", - "6623": "796172", - "6624": "20696c", - "6625": "727375", - "6626": "20696d706f7274616e74", - "6627": "6e746572727570746564", - "6628": "2072656c6174696f6e73", - "6629": "206265686176696f7572", - "6630": "6e676c69", - "6631": "6d657265", - "6632": "206d6f6f", - "6633": "6f626c65", - "6634": "204b", - "6635": "204576", - "6636": "73696d", - "6637": "646576", - "6638": "73686f756c64657273", - "6639": "67656e6572616c6c79", - "6640": "20696e74726f647563", - "6641": "20696e74656e74696f", - "6642": "20686f727269626c65", - "6643": "20737572726f756e64", - "6644": "6163717561696e7465", - "6645": "63717561696e746564", - "6646": "69676e6f72616e", - "6647": "74746572696e67", - "6648": "207472656d626c", - "6649": "207468726f6174", - "6650": "72696e67696e67", - "6651": "2070726576656e", - "6652": "206d656d6f7279", - "6653": "73756767657374", - "6654": "7772697474", - "6655": "696e64696e", - "6656": "6768746564", - "6657": "6661696e74", - "6658": "20626c7565", - "6659": "206d656d6f", - "6660": "20636c6f75", - "6661": "20626c6573", - "6662": "2074687573", - "6663": "2045766572", - "6664": "756e6174", - "6665": "68756e67", - "6666": "20736368", - "6667": "616d6564", - "6668": "74636869", - "6669": "6f6e6773", - "6670": "72617373", - "6671": "66666f72", - "6672": "67687479", - "6673": "66696369", - "6674": "72616e6b", - "6675": "6a6f79", - "6676": "666c69", - "6677": "617074", - "6678": "7268", - "6679": "2069676e6f72616e", - "6680": "70726f6d69736564", - "6681": "20636f6e74656e74", - "6682": "20706c6561736564", - "6683": "20696d6167696e61", - "6684": "737572726f756e64", - "6685": "2061737375726564", - "6686": "657175656e746c79", - "6687": "206d757264657265", - "6688": "2070657273756164", - "6689": "696e6e6f63656e74", - "6690": "77616e646572", - "6691": "7472656d626c", - "6692": "2070726f7665", - "6693": "746f75636865", - "6694": "656172657374", - "6695": "207574746572", - "6696": "6974697a656e", - "6697": "204265636b79", - "6698": "7262", - "6699": "205761", - "6700": "20676172", - "6701": "6973746f", - "6702": "6465616c", - "6703": "75636865", - "6704": "65646961", - "6705": "63656c6c", - "6706": "204f6e65", - "6707": "20676174", - "6708": "6f6c6c79", - "6709": "7065726174", - "6710": "20656e7472", - "6711": "61696e696e", - "6712": "6c61636573", - "6713": "69636b6c79", - "6714": "6f6c6c6172", - "6715": "6665726564", - "6716": "6e73747275", - "6717": "4265636b79", - "6718": "727361", - "6719": "696d69", - "6720": "726572", - "6721": "57696c", - "6722": "726d61", - "6723": "4f6e65", - "6724": "756e636f6e7363696f7573", - "6725": "2073686f756c64657273", - "6726": "696e7465727275707465", - "6727": "696d7072657373696f6e", - "6728": "657370656369616c6c79", - "6729": "636f6d70617373696f6e", - "6730": "20727573686564", - "6731": "2065787472656d", - "6732": "726567756c6172", - "6733": "20676174686572", - "6734": "736962696c6974", - "6735": "77656174686572", - "6736": "20636f6d706f73", - "6737": "20616374696f6e", - "6738": "20646f6c6c6172", - "6739": "74656e696e67", - "6740": "727279696e67", - "6741": "2066616e6379", - "6742": "207365617263", - "6743": "2077726f7465", - "6744": "74696f6e616c", - "6745": "206173736973", - "6746": "696e616c6c79", - "6747": "20646576696c", - "6748": "6865616c7468", - "6749": "64696174", - "6750": "72656369", - "6751": "6e6f7465", - "6752": "696e6369", - "6753": "75637469", - "6754": "204c65", - "6755": "72746c", - "6756": "62696c", - "6757": "666f72676f7474656e", - "6758": "207374726574636865", - "6759": "2064697374696e6374", - "6760": "6d6f6f6e6c69676874", - "6761": "205468617463686572", - "6762": "2070757474696e67", - "6763": "20766172696f7573", - "6764": "2071756965746c79", - "6765": "20696e7374656164", - "6766": "20636f7665726564", - "6767": "20636f6e6365616c", - "6768": "736962696c697479", - "6769": "6561676572", - "6770": "2066657463", - "6771": "6d626c6564", - "6772": "2072657175", - "6773": "636f6e6665", - "6774": "74656e616e", - "6775": "727261", - "6776": "6f7973", - "6777": "6a6f69", - "6778": "746f6c65", - "6779": "686f6f6c", - "6780": "736f6674", - "6781": "65616465", - "6782": "20636875", - "6783": "7373656e", - "6784": "73747572", - "6785": "61696c65", - "6786": "4368", - "6787": "616e61", - "6788": "6e6379", - "6789": "6e6966", - "6790": "6f6465", - "6791": "62696e", - "6792": "637473", - "6793": "4c61", - "6794": "206761726465", - "6795": "636565646564", - "6796": "726573706f6e", - "6797": "206d61727279", - "6798": "50657268617073", - "6799": "20636f6e747261", - "6800": "6d696e6174696f", - "6801": "4a616371756573", - "6802": "2068616e67", - "6803": "6964646c65", - "6804": "72656c6965", - "6805": "2067756172", - "6806": "7264696e67", - "6807": "6173746c65", - "6808": "7363656e64", - "6809": "6573706f6e", - "6810": "756e6974", - "6811": "6f76656c", - "6812": "6e617279", - "6813": "6f727465", - "6814": "206c6965", - "6815": "6f6c6963", - "6816": "76696f75", - "6817": "72657369", - "6818": "536f6d", - "6819": "726f6c", - "6820": "6d6f6f", - "6821": "206f7468657277697365", - "6822": "20636f6d706c696d656e", - "6823": "2074656c6c696e67", - "6824": "20636f7572616765", - "6825": "20706563756c6961", - "6826": "696e74656c6c6967", - "6827": "204a616371756573", - "6828": "726f6c6c", - "6829": "74697475", - "6830": "736f6c69", - "6831": "63616e64", - "6832": "68617374", - "6833": "6c657374", - "6834": "6e747261", - "6835": "67696e65", - "6836": "20626f75", - "6837": "6d6f6f6e", - "6838": "20696e746572727570746564", - "6839": "20656e646561766f75726564", - "6840": "20657863656564696e676c79", - "6841": "2061646d69726174696f6e", - "6842": "20696d70617469656e", - "6843": "20656e636f75726167", - "6844": "206163636964656e74", - "6845": "636f6d70726568656e", - "6846": "50656d6265726c6579", - "6847": "66616c6c656e", - "6848": "206875727279", - "6849": "73696d706c65", - "6850": "206472696e6b", - "6851": "656374696e67", - "6852": "2073686f7574", - "6853": "65636f766572", - "6854": "656d706c6f79", - "6855": "66616e6379", - "6856": "6c6179696e", - "6857": "6172726f77", - "6858": "6174636869", - "6859": "7275736865", - "6860": "20656e7469", - "6861": "47656f7267", - "6862": "6374697665", - "6863": "756e676572", - "6864": "6c6964", - "6865": "20526f", - "6866": "616d62", - "6867": "736976", - "6868": "2062656c6f6e67", - "6869": "20616363657074", - "6870": "2070726f647563", - "6871": "20646570656e64", - "6872": "6f666669636572", - "6873": "2073696e676c65", - "6874": "20636f74746167", - "6875": "20707572737565", - "6876": "76697369746f72", - "6877": "206c6177", - "6878": "76657274", - "6879": "6c6f7769", - "6880": "736f6e61", - "6881": "61726c65", - "6882": "65656469", - "6883": "6d6f6e73", - "6884": "786564", - "6885": "206e65636b", - "6886": "2073656e64", - "6887": "2054656c6c", - "6888": "72656e6368", - "6889": "736d6f6b65", - "6890": "2061636375", - "6891": "6c61746564", - "6892": "206d6f7469", - "6893": "6772617469", - "6894": "206162736f", - "6895": "7065726365", - "6896": "66616e", - "6897": "667573", - "6898": "6f7263", - "6899": "657073", - "6900": "727669", - "6901": "72746f", - "6902": "206c79696e67", - "6903": "70656e696e67", - "6904": "656174696e67", - "6905": "206578636c61", - "6906": "20636f616368", - "6907": "206672657368", - "6908": "61746566756c", - "6909": "736361726365", - "6910": "20656174", - "6911": "73706565", - "6912": "61726d65", - "6913": "78616374", - "6914": "6f696e65", - "6915": "65656b73", - "6916": "636b6c65", - "6917": "616e7461", - "6918": "7274696e", - "6919": "696e696f", - "6920": "4b696e67", - "6921": "63686965", - "6922": "65616c69", - "6923": "20666965", - "6924": "72697661", - "6925": "67756973", - "6926": "6262", - "6927": "2070726f64756365", - "6928": "2066617368696f6e", - "6929": "63636f7264696e67", - "6930": "20696e6372656173", - "6931": "2073746570706564", - "6932": "2070726976617465", - "6933": "617474656e64616e", - "6934": "20736f6369657479", - "6935": "20646f6c6c617273", - "6936": "706565", - "6937": "727574", - "6938": "616772", - "6939": "61776e", - "6940": "207265636f6c6c656374696f6e", - "6941": "2067656e6572616c6c79", - "6942": "636f6d70616e696f6e73", - "6943": "206d6f756e7461696e73", - "6944": "206d6f6f6e6c69676874", - "6945": "6163636f6d706c697368", - "6946": "20726570726573656e74", - "6947": "20646175676874657273", - "6948": "6c697374656e696e67", - "6949": "737472657463686564", - "6950": "736372697074696f6e", - "6951": "617474656e64616e74", - "6952": "636f6e6e6563746564", - "6953": "636f6e76656e69656e", - "6954": "636f6d70617373", - "6955": "204c6f6e67", - "6956": "2063656e74", - "6957": "6464696e67", - "6958": "2077726f6e", - "6959": "69636b6564", - "6960": "74746c6564", - "6961": "20626f756e", - "6962": "2068696c6c", - "6963": "20636f6666", - "6964": "2070656163", - "6965": "207370656e", - "6966": "69636564", - "6967": "7469636b", - "6968": "4a757374", - "6969": "6d6f7272", - "6970": "6c6f6f6d", - "6971": "204b696e", - "6972": "6372696d", - "6973": "7369636b", - "6974": "6573746f", - "6975": "6e6965", - "6976": "204472", - "6977": "746879", - "6978": "6578706c616e6174696f6e", - "6979": "20696d7072657373696f6e", - "6980": "61636b6e6f776c65646765", - "6981": "737065637461", - "6982": "20776f727365", - "6983": "6e6f756e6365", - "6984": "20746869636b", - "6985": "20656e6a6f79", - "6986": "2072656d6f76", - "6987": "206772617469", - "6988": "206c6f6f6b73", - "6989": "617270", - "6990": "736d6f", - "6991": "616369", - "6992": "20756e6465727374616e64696e67", - "6993": "2048657274666f72647368697265", - "6994": "206578706563746174696f6e", - "6995": "616e6379", - "6996": "72616c6c", - "6997": "7461696c", - "6998": "65646564", - "6999": "73656e63", - "7000": "69666965", - "7001": "20646566", - "7002": "61766572", - "7003": "206561726e", - "7004": "6c6f636b65", - "7005": "206861646e", - "7006": "207472656d", - "7007": "6475636564", - "7008": "6973736564", - "7009": "204672616e", - "7010": "207761726d", - "7011": "6365707469", - "7012": "6f7573696e", - "7013": "646f67", - "7014": "205375", - "7015": "4d616e", - "7016": "2068616e67696e67", - "7017": "206c6561726e6564", - "7018": "206f666669636572", - "7019": "2073657061726174", - "7020": "736f6d6577686174", - "7021": "737570706f736564", - "7022": "206672657175656e", - "7023": "6d706c696d656e74", - "7024": "2066616c6c656e", - "7025": "6d656574696e67", - "7026": "20636f6e74656e", - "7027": "64657363656e64", - "7028": "6f76656d656e74", - "7029": "20646570617274", - "7030": "2066696c", - "7031": "206a6f69", - "7032": "76617269", - "7033": "64676564", - "7034": "7765616b", - "7035": "65727469", - "7036": "70707265", - "7037": "72656564", - "7038": "7761726d", - "7039": "70726179", - "7040": "2061756e", - "7041": "666978", - "7042": "6d626c696e67", - "7043": "206c61746572", - "7044": "6f6c64696e67", - "7045": "696f6e617465", - "7046": "726973696e67", - "7047": "6566756c6c79", - "7048": "636f75706c65", - "7049": "206769726c73", - "7050": "636c61696d65", - "7051": "6c61696d6564", - "7052": "676174686572", - "7053": "20737472696b", - "7054": "657373697665", - "7055": "20726f6f6d73", - "7056": "617474657265", - "7057": "207072656369", - "7058": "646570617274", - "7059": "726561736f6e61626c", - "7060": "20737570706f736564", - "7061": "2073796d7061746879", - "7062": "207375666665726564", - "7063": "677261746974756465", - "7064": "656e747265", - "7065": "6875727279", - "7066": "20676f6c64", - "7067": "20626c6f77", - "7068": "2070616c65", - "7069": "636976696c", - "7070": "676c6f6f6d", - "7071": "2073616b65", - "7072": "6d6f6e7374", - "7073": "2073706172", - "7074": "656775", - "7075": "626164", - "7076": "4269", - "7077": "20666f72676f7474656e", - "7078": "2070617373656e676572", - "7079": "2070726f636565646564", - "7080": "696e737472756d656e74", - "7081": "696e74656c6c6967656e", - "7082": "75727468", - "7083": "6374696e", - "7084": "65766564", - "7085": "696e6572", - "7086": "6c696765", - "7087": "6963696f", - "7088": "746f7265", - "7089": "7366", - "7090": "436174", - "7091": "20706f7373657373696f6e", - "7092": "64697374696e6775697368", - "7093": "756e6465727374616e6469", - "7094": "20706f696e746564", - "7095": "736c656570696e67", - "7096": "20656e7472656174", - "7097": "20646563696465", - "7098": "206e756d626572", - "7099": "70726f64756365", - "7100": "636f6e6365616c", - "7101": "20636f6e666573", - "7102": "20636f6666696e", - "7103": "204d6f72726973", - "7104": "20706f6c696365", - "7105": "20776f726b6564", - "7106": "2053756e646179", - "7107": "206d696c6573", - "7108": "206361746368", - "7109": "4672656e6368", - "7110": "207061727479", - "7111": "6d6974746564", - "7112": "206368617267", - "7113": "717569726564", - "7114": "737570706572", - "7115": "6e6c69676874", - "7116": "207472616e73", - "7117": "64696e676c79", - "7118": "656c6c6564", - "7119": "6472696e6b", - "7120": "20736c6970", - "7121": "2061636365", - "7122": "2065617369", - "7123": "2076617269", - "7124": "206661696e", - "7125": "2072657375", - "7126": "5468657365", - "7127": "6c696e6573", - "7128": "616e63696e", - "7129": "6e6f626c65", - "7130": "20706f7374", - "7131": "206b696c", - "7132": "756d626c", - "7133": "20636c69", - "7134": "70656e73", - "7135": "6f6c6469", - "7136": "72646179", - "7137": "20756e65", - "7138": "69636b6c", - "7139": "69746f72", - "7140": "696464", - "7141": "616767", - "7142": "61646f", - "7143": "436f6d", - "7144": "6f7473", - "7145": "696469", - "7146": "6f6473", - "7147": "626967", - "7148": "6c6f61", - "7149": "61736865", - "7150": "70657261", - "7151": "726d6572", - "7152": "636c696e", - "7153": "6c696172", - "7154": "2ee28094", - "7155": "20657865", - "7156": "696e6473", - "7157": "6f726573", - "7158": "637265617475726573", - "7159": "20706f736974696f6e", - "7160": "2070726f6d69736564", - "7161": "6361726566756c6c79", - "7162": "6661766f7572697465", - "7163": "6163636f7264696e67", - "7164": "20736f6d6577686174", - "7165": "646973747572626564", - "7166": "20676f6f646e657373", - "7167": "706572636569766564", - "7168": "2066696c6c", - "7169": "626c696e67", - "7170": "2065736361", - "7171": "20696e7369", - "7172": "7368697265", - "7173": "6572726f72", - "7174": "206c696674", - "7175": "2072657365", - "7176": "2064657669", - "7177": "726f6f6d73", - "7178": "20506f", - "7179": "656368", - "7180": "6c7465", - "7181": "677561", - "7182": "697661", - "7183": "746976656c79", - "7184": "6a6f696e6564", - "7185": "6172696f7573", - "7186": "207374656164", - "7187": "73696f6e616c", - "7188": "6e7374656164", - "7189": "20696e637265", - "7190": "6d696e617469", - "7191": "73696e676c65", - "7192": "206861626974", - "7193": "207072696465", - "7194": "73656e736962", - "7195": "6e6973686d65", - "7196": "6d656c616e63686f6c79", - "7197": "206661766f7572697465", - "7198": "706f73736962696c6974", - "7199": "20737465616d626f6174", - "7200": "686f6c64696e67", - "7201": "726f6475636564", - "7202": "2072656d6f7665", - "7203": "20736572766963", - "7204": "66756c6e657373", - "7205": "72616e74", - "7206": "69667465", - "7207": "696e6769", - "7208": "6f6c6974", - "7209": "73756974", - "7210": "62656174", - "7211": "6976696c", - "7212": "4672616e", - "7213": "6d6d6572", - "7214": "6f6c6c61", - "7215": "63757265", - "7216": "6f6e6973", - "7217": "61747461", - "7218": "76696577", - "7219": "736f6c76", - "7220": "6c6f7474", - "7221": "69726174", - "7222": "626f78", - "7223": "7a6564", - "7224": "657368", - "7225": "646174", - "7226": "6f756e7461696e73", - "7227": "206d617474657273", - "7228": "6c616e6775616765", - "7229": "7072657061726564", - "7230": "6f63637570696564", - "7231": "757474696e", - "7232": "7265766572", - "7233": "2065616765", - "7234": "2070756c6c", - "7235": "5768657265", - "7236": "207368616b", - "7237": "636f6e7374", - "7238": "756e746564", - "7239": "70726f7465", - "7240": "617373696e", - "7241": "2061647665", - "7242": "6666616972", - "7243": "206b697373", - "7244": "706572696f", - "7245": "7265666572", - "7246": "73656e7369", - "7247": "2065717561", - "7248": "4c6574", - "7249": "736167", - "7250": "687574", - "7251": "63756d", - "7252": "696775", - "7253": "636561", - "7254": "61737361", - "7255": "72697374", - "7256": "21e28099", - "7257": "616e6368", - "7258": "6c6f6f72", - "7259": "616e6775", - "7260": "6c746572", - "7261": "65617469", - "7262": "70746564", - "7263": "69746965", - "7264": "20736b69", - "7265": "206d7973746572696f7573", - "7266": "2061636b6e6f776c656467", - "7267": "20696e737472756d656e74", - "7268": "20636f6d70617373696f6e", - "7269": "7475616c6c79", - "7270": "686f75736573", - "7271": "676976696e67", - "7272": "636f6e747261", - "7273": "20636f6e7374", - "7274": "6f7572696e67", - "7275": "7175656e6365", - "7276": "207375707065", - "7277": "206c6f766564", - "7278": "706172746564", - "7279": "2070726f706f", - "7280": "6b6c65", - "7281": "6e61747572616c6c79", - "7282": "64616e6765726f7573", - "7283": "206f63637570696564", - "7284": "20617474656e64616e", - "7285": "7374656e696e67", - "7286": "486f7765766572", - "7287": "204672656e6368", - "7288": "20737065656368", - "7289": "20756e68617070", - "7290": "20746572726f72", - "7291": "636f756e746572", - "7292": "2061646d697261", - "7293": "63617061626c65", - "7294": "7475726564", - "7295": "617272616e", - "7296": "6974617465", - "7297": "656173696e", - "7298": "6578747261", - "7299": "20616e7977", - "7300": "7573746564", - "7301": "7572656c79", - "7302": "696e6e69", - "7303": "73636170", - "7304": "75736573", - "7305": "73756c74", - "7306": "6f706564", - "7307": "6570656e", - "7308": "6c616e74", - "7309": "73706172", - "7310": "6865726c", - "7311": "20726166", - "7312": "747769", - "7313": "6c6177", - "7314": "626162", - "7315": "6e6374", - "7316": "706c6f", - "7317": "20486f7765766572", - "7318": "20686f6c64696e67", - "7319": "2068656172696e67", - "7320": "20706572736f6e61", - "7321": "206f626c69676564", - "7322": "6f66666963657273", - "7323": "6d69646e69676874", - "7324": "736f6c6174696f6e", - "7325": "20616e7869657479", - "7326": "2076697369746f72", - "7327": "626c75", - "7328": "206c697374656e696e67", - "7329": "20737472657463686564", - "7330": "7265636f6c6c65637469", - "7331": "617070726f6163686564", - "7332": "2064616e6765726f7573", - "7333": "20627265617468696e67", - "7334": "6672657175656e746c79", - "7335": "206c696768746e696e67", - "7336": "2077616e6465", - "7337": "737072656164", - "7338": "206578697374", - "7339": "6e756d626572", - "7340": "2073686f636b", - "7341": "20747261696e", - "7342": "707572737565", - "7343": "66696c6c", - "7344": "77657273", - "7345": "61636365", - "7346": "736d6f6b", - "7347": "6173696e", - "7348": "72696d65", - "7349": "206d6169", - "7350": "6964696e", - "7351": "6361746368", - "7352": "2066696674", - "7353": "756d656e74", - "7354": "6574696e67", - "7355": "2067726577", - "7356": "7472656d65", - "7357": "2077696465", - "7358": "2077656172", - "7359": "2074726169", - "7360": "636f616368", - "7361": "2047656f72", - "7362": "7574656c79", - "7363": "697264", - "7364": "676865", - "7365": "6e6a75", - "7366": "2066696c6c6564", - "7367": "74656e64696e67", - "7368": "6f70656e696e67", - "7369": "76696f6c656e74", - "7370": "206d6572656c79", - "7371": "6167656d656e74", - "7372": "20737072696e67", - "7373": "20776973686573", - "7374": "6d797374657279", - "7375": "20726573746f72", - "7376": "20737567676573", - "7377": "656e7369626c65", - "7378": "2069736c616e64", - "7379": "636974697a656e", - "7380": "20637574", - "7381": "64756374", - "7382": "64657365", - "7383": "65646974", - "7384": "6162736f", - "7385": "63726564", - "7386": "72616d", - "7387": "777265", - "7388": "657461", - "7389": "757479", - "7390": "6c756d", - "7391": "766174", - "7392": "5365", - "7393": "206d656c616e63686f6c79", - "7394": "20636f6d70616e696f6e73", - "7395": "20657370656369616c6c79", - "7396": "706f73736962696c697479", - "7397": "20696e74656c6c6967656e", - "7398": "206164647265737365", - "7399": "206461796c69676874", - "7400": "20677261746566756c", - "7401": "706f73736573736564", - "7402": "20517565656e", - "7403": "646563696465", - "7404": "70726f766564", - "7405": "206578747261", - "7406": "63617574696f", - "7407": "707072657373", - "7408": "20666f726573", - "7409": "72656e646572", - "7410": "207375737065", - "7411": "2073696e676c", - "7412": "636f6666696e", - "7413": "20696d70726f", - "7414": "6572656c79", - "7415": "616368696e", - "7416": "6f6c656e74", - "7417": "2072656675", - "7418": "6175736564", - "7419": "6561646564", - "7420": "206661696c", - "7421": "20666c6f77", - "7422": "20696e7472", - "7423": "6177696e67", - "7424": "20636f7272", - "7425": "20636f7665", - "7426": "20596f7572", - "7427": "2076696577", - "7428": "646576696c", - "7429": "73746f7265", - "7430": "6565706572", - "7431": "20626c696e", - "7432": "656465", - "7433": "697374656e696e67", - "7434": "20696e74726f6475", - "7435": "6c61756768696e67", - "7436": "20696e7175697269", - "7437": "20737572726f756e", - "7438": "2064656172657374", - "7439": "206d6f6d656e7473", - "7440": "2066696e616c6c79", - "7441": "62656c6c", - "7442": "666c6f77", - "7443": "6f72676f", - "7444": "69626c79", - "7445": "20576974", - "7446": "74726963", - "7447": "2064656e", - "7448": "75706572", - "7449": "6f6c6f6e", - "7450": "70657263", - "7451": "6b6c", - "7452": "67756e", - "7453": "6f6b73", - "7454": "676167", - "7455": "696265", - "7456": "20756e636f6e7363696f7573", - "7457": "696e74656c6c6967656e6365", - "7458": "207265636f7665726564", - "7459": "206163636f7264696e67", - "7460": "20636f75727479617264", - "7461": "636f6e76696374696f6e", - "7462": "2050656d6265726c6579", - "7463": "206c6f636b6564", - "7464": "20636f6e74656d", - "7465": "706572736f6e61", - "7466": "61727279696e67", - "7467": "6b656570696e67", - "7468": "20707261637469", - "7469": "20637573746f6d", - "7470": "73656e7369626c", - "7471": "63657074696f6e", - "7472": "706572666f726d", - "7473": "2070726f737065", - "7474": "637475616c", - "7475": "20706c6169", - "7476": "6973686573", - "7477": "7265737469", - "7478": "2073657269", - "7479": "6c6c6f7769", - "7480": "6c6c6f7773", - "7481": "2065786369", - "7482": "76656e6765", - "7483": "6163746564", - "7484": "206c616d", - "7485": "65706c79", - "7486": "20657363", - "7487": "756c6c65", - "7488": "7269616c", - "7489": "73637265", - "7490": "20696e6e", - "7491": "72706f", - "7492": "617370", - "7493": "697261", - "7494": "707261637469", - "7495": "207265667573", - "7496": "737065656368", - "7497": "746572726f72", - "7498": "20636976696c", - "7499": "20756e656173", - "7500": "2070726f7465", - "7501": "696768746c79", - "7502": "20636f6c6f75", - "7503": "737072696e67", - "7504": "666572696e67", - "7505": "696e74656c6c", - "7506": "70706c", - "7507": "6b79", - "7508": "6275726e", - "7509": "72616e64", - "7510": "6f72616e", - "7511": "7465656e", - "7512": "71756973", - "7513": "756e6e69", - "7514": "6b6e6565", - "7515": "20736e61", - "7516": "69737365", - "7517": "2d2de280", - "7518": "656c6963", - "7519": "20706572736f6e616c", - "7520": "20736c656570696e67", - "7521": "2074726176656c6c65", - "7522": "20696e74656c6c6967", - "7523": "2072656164696e67", - "7524": "70726563696f7573", - "7525": "617474656e646564", - "7526": "2064697374696e63", - "7527": "6275696c64696e67", - "7528": "20636974697a656e", - "7529": "656e6a6f79", - "7530": "70726f706f", - "7531": "6f626c6967", - "7532": "6973746564", - "7533": "7274696f6e", - "7534": "2064757479", - "7535": "2073796d70", - "7536": "666f726573", - "7537": "6561746572", - "7538": "6570726573", - "7539": "7463686573", - "7540": "736f6c6174", - "7541": "626c696e64", - "7542": "617263656c", - "7543": "637279", - "7544": "666c75", - "7545": "746675", - "7546": "20696e7669746174696f6e", - "7547": "206163717561696e746564", - "7548": "75747465726564", - "7549": "637573746f6d65", - "7550": "726f6d69736564", - "7551": "206a6f696e6564", - "7552": "20726573756c74", - "7553": "70726f74656374", - "7554": "206e6967687473", - "7555": "726174696f6e73", - "7556": "206a7564676564", - "7557": "20616e79776179", - "7558": "205061726973", - "7559": "20636f6e6475", - "7560": "207765656b73", - "7561": "6561726e6564", - "7562": "206569676874", - "7563": "206465746169", - "7564": "206177616b65", - "7565": "206865616c74", - "7566": "2066696e616c", - "7567": "796f756e6765", - "7568": "20656d706c6f", - "7569": "6e7369626c65", - "7570": "756c61726c79", - "7571": "204c75636173", - "7572": "67756573", - "7573": "61636375", - "7574": "2074656d", - "7575": "616d6573", - "7576": "73657269", - "7577": "67726173", - "7578": "20756e66", - "7579": "61796265", - "7580": "616b6572", - "7581": "726e616c", - "7582": "72616e73", - "7583": "6e6f6f6e", - "7584": "20736963", - "7585": "756e6461", - "7586": "61726c6f", - "7587": "204c69", - "7588": "66696c", - "7589": "65646c", - "7590": "657865", - "7591": "7869", - "7592": "4672", - "7593": "206275726e", - "7594": "20626f7474", - "7595": "2073686f65", - "7596": "776f727365", - "7597": "616e697368", - "7598": "206c65616e", - "7599": "616c746572", - "7600": "7563686564", - "7601": "6974636865", - "7602": "6e6c657373", - "7603": "636173746c", - "7604": "756e746572", - "7605": "2067726965", - "7606": "657276616c", - "7607": "6572737561", - "7608": "4c697a7a79", - "7609": "205465", - "7610": "576869", - "7611": "65766f", - "7612": "20666f7274756e617465", - "7613": "726561736f6e61626c65", - "7614": "20636f6e7363696f7573", - "7615": "20696d7072657373696f", - "7616": "20617474656e64616e74", - "7617": "20677261746974756465", - "7618": "63617065", - "7619": "696e7469", - "7620": "206c6f74", - "7621": "6e74696d", - "7622": "73706963", - "7623": "686965", - "7624": "757473", - "7625": "6d6972", - "7626": "205265", - "7627": "6f63636173696f6e616c6c79", - "7628": "6c697374656e696e", - "7629": "20686973746f7279", - "7630": "70617469656e6365", - "7631": "20756e6861707079", - "7632": "706572736f6e616c", - "7633": "206163636964656e", - "7634": "61747572616c6c79", - "7635": "206d797374657279", - "7636": "206d6f6e73746572", - "7637": "2064656c69766572", - "7638": "20706572666f726d", - "7639": "73656e7369626c65", - "7640": "62656c6f6e67", - "7641": "206772617373", - "7642": "20636f75706c", - "7643": "72696e67696e", - "7644": "20646973706f", - "7645": "20636f6d706f", - "7646": "6f6369657479", - "7647": "61646d697261", - "7648": "656e636f7572616765", - "7649": "206f66666963657273", - "7650": "207365706172617465", - "7651": "206c616e6775616765", - "7652": "207072657061726564", - "7653": "736572766174696f6e", - "7654": "746f6c657261626c65", - "7655": "20617373697374616e", - "7656": "2070726f7065727479", - "7657": "6f626a656374696f6e", - "7658": "20726570726573656e", - "7659": "20617574686f72", - "7660": "2064657363656e", - "7661": "72657175697265", - "7662": "20636f6e636561", - "7663": "696c6c6e657373", - "7664": "6a656374696f6e", - "7665": "20636f6d706c69", - "7666": "64656c69766572", - "7667": "20796f756e6765", - "7668": "20676c6173", - "7669": "6c69636174", - "7670": "73686f7574", - "7671": "746869636b", - "7672": "656e636573", - "7673": "7061726564", - "7674": "62726f6164", - "7675": "7463686572", - "7676": "6f72746564", - "7677": "6961626c65", - "7678": "2041756e74", - "7679": "20576869", - "7680": "20506172", - "7681": "6e636f6d", - "7682": "20646174", - "7683": "65657473", - "7684": "67696e61", - "7685": "2072656e", - "7686": "756d73", - "7687": "6e6761", - "7688": "736569", - "7689": "766f6c", - "7690": "746961", - "7691": "6562", - "7692": "4c6f6e67", - "7693": "636f6174", - "7694": "2066756e", - "7695": "2070756c", - "7696": "68617267", - "7697": "646c6573", - "7698": "65637473", - "7699": "20647574", - "7700": "61646965", - "7701": "6e636f6e", - "7702": "7469616c", - "7703": "6c617273", - "7704": "75636173", - "7705": "20617070726f6163686564", - "7706": "6f62736572766174696f6e", - "7707": "636f6d6d756e6963617469", - "7708": "20656d707479", - "7709": "74746572696e", - "7710": "207472656d62", - "7711": "657265737469", - "7712": "4672616e6365", - "7713": "20776f756e64", - "7714": "766174696f6e", - "7715": "736372697074", - "7716": "66666572696e", - "7717": "206372696d65", - "7718": "5379646e6579", - "7719": "2068617374", - "7720": "206e617272", - "7721": "616e696d61", - "7722": "61726d6564", - "7723": "206e6f626c", - "7724": "2073757065", - "7725": "6b6e69", - "7726": "637461", - "7727": "666964", - "7728": "6e7363", - "7729": "206361726566756c6c79", - "7730": "20656e636f7572616765", - "7731": "206e61747572616c6c79", - "7732": "636f6d70726568656e64", - "7733": "636f6e736369656e6365", - "7734": "656e676167656d656e74", - "7735": "2076696f6c656e74", - "7736": "206f70656e696e67", - "7737": "7365706172617465", - "7738": "736372697074696f", - "7739": "6372697074696f6e", - "7740": "2077656174686572", - "7741": "2066656174757265", - "7742": "206172726976616c", - "7743": "636f6e6e65637465", - "7744": "6f6e6e6563746564", - "7745": "20616d6961626c65", - "7746": "20736c69707065", - "7747": "204672616e6365", - "7748": "207468726f776e", - "7749": "756c6174696f6e", - "7750": "2064657461696c", - "7751": "656e7472656174", - "7752": "6f6f646e657373", - "7753": "2068656176656e", - "7754": "20696e74656c6c", - "7755": "7072696e636970", - "7756": "2072656e646572", - "7757": "20656e65", - "7758": "756e6365", - "7759": "6272616e", - "7760": "696e6172", - "7761": "68697265", - "7762": "20636c75", - "7763": "70657265", - "7764": "20666c69", - "7765": "20726167", - "7766": "736965", - "7767": "736164", - "7768": "737479", - "7769": "666574", - "7770": "656e61", - "7771": "206c61756768696e67", - "7772": "636f6c6c6563746564", - "7773": "636f6e6365726e6564", - "7774": "72657175656e746c79", - "7775": "73656e74696d656e74", - "7776": "206b6e697474696e67", - "7777": "6f73697479", - "7778": "666c6f7765", - "7779": "6578697374", - "7780": "2062656e74", - "7781": "6363657074", - "7782": "6272616e63", - "7783": "72656d6f76", - "7784": "6c69636974", - "7785": "2062656e65", - "7786": "6573747261", - "7787": "6172657374", - "7788": "6f756e6765", - "7789": "205065", - "7790": "706574", - "7791": "6e616e", - "7792": "204765", - "7793": "72656d6f7665", - "7794": "61766f757269", - "7795": "20637573746f", - "7796": "20636865656b", - "7797": "206761746865", - "7798": "2064726f7665", - "7799": "206365617365", - "7800": "20616374696f", - "7801": "206d6f6e7374", - "7802": "207769646f77", - "7803": "726573746f72", - "7804": "207472616465", - "7805": "2074656e6465", - "7806": "204c697a7a79", - "7807": "61727479", - "7808": "73656e64", - "7809": "69636573", - "7810": "20656c65", - "7811": "70656e74", - "7812": "696e7472", - "7813": "20756e6c", - "7814": "206f626c", - "7815": "6f626c69", - "7816": "65666572", - "7817": "65726c6f", - "7818": "2061636b6e6f776c65646765", - "7819": "706573", - "7820": "737473", - "7821": "757279", - "7822": "746e", - "7823": "20696e74656c6c6967656e6365", - "7824": "636f6d6d756e69636174696f6e", - "7825": "20737461697273", - "7826": "67726f77696e67", - "7827": "2073696d706c79", - "7828": "20726567756c61", - "7829": "2073756d6d6572", - "7830": "2066617368696f", - "7831": "636f6d6d616e64", - "7832": "206c6179696e67", - "7833": "20656e64757265", - "7834": "206d6164616d65", - "7835": "66696c6c65", - "7836": "616c6c656e", - "7837": "206c616d70", - "7838": "2070697479", - "7839": "756d626c65", - "7840": "7573696e67", - "7841": "70706f7369", - "7842": "616d696e67", - "7843": "2063616c6d", - "7844": "6f6e61626c", - "7845": "746963616c", - "7846": "2077617973", - "7847": "6365617365", - "7848": "2070657263", - "7849": "20656e63", - "7850": "206c656e", - "7851": "65656465", - "7852": "6c697669", - "7853": "69637469", - "7854": "736f66", - "7855": "20736f6d65686f77", - "7856": "7072657373696e67", - "7857": "20726567756c6172", - "7858": "206d797374657269", - "7859": "2072657175697265", - "7860": "6469737475726265", - "7861": "20636f6d6d616e64", - "7862": "20726573746f7265", - "7863": "70726568656e7369", - "7864": "6772617469747564", - "7865": "7261746974756465", - "7866": "7265737065637461626c65", - "7867": "6465737472756374696f6e", - "7868": "20636f6e76696374696f6e", - "7869": "20636f6e6365726e696e67", - "7870": "696e636c696e6174696f6e", - "7871": "617272616e67656d656e74", - "7872": "636f6e67726174756c6174", - "7873": "617574686f72", - "7874": "2074656d7065", - "7875": "74656d706572", - "7876": "20666c6f7765", - "7877": "206c616e6775", - "7878": "61726e696e67", - "7879": "797768657265", - "7880": "6f6e61626c65", - "7881": "207265616c69", - "7882": "20736c657074", - "7883": "797374657269", - "7884": "20636861726d", - "7885": "726570726573", - "7886": "207468726561", - "7887": "206275727374", - "7888": "206865617665", - "7889": "657270", - "7890": "6f6973", - "7891": "797374", - "7892": "637275", - "7893": "64656c6967687466756c", - "7894": "20646973747572626564", - "7895": "2050726573656e746c79", - "7896": "206f626a656374696f6e", - "7897": "20636f6d70726568656e", - "7898": "61636869", - "7899": "74686f72", - "7900": "6e647563", - "7901": "67686564", - "7902": "726f636b", - "7903": "20677569", - "7904": "6e646c79", - "7905": "72657072", - "7906": "20626c69", - "7907": "736f6369", - "7908": "616464726573736564", - "7909": "20696e637265617365", - "7910": "207472656173757265", - "7911": "6170706f696e746564", - "7912": "7572726564", - "7913": "727279696e", - "7914": "206e657276", - "7915": "656374696e", - "7916": "2066697368", - "7917": "6d69747465", - "7918": "6163636570", - "7919": "206c6f7564", - "7920": "737465726e", - "7921": "627261696e", - "7922": "7665727365", - "7923": "20676c6f6f", - "7924": "6966746564", - "7925": "6a75737469", - "7926": "73686f636b", - "7927": "666f727468", - "7928": "2077616b65", - "7929": "20736e6f77", - "7930": "20696e636f", - "7931": "7263656c79", - "7932": "73616c", - "7933": "5369", - "7934": "206c6f76656c79", - "7935": "20737072656164", - "7936": "206e6172726f77", - "7937": "20726973696e67", - "7938": "6f726465726564", - "7939": "6368616d626572", - "7940": "6f6c6174696f6e", - "7941": "2070726f706f73", - "7942": "20636f6d6d656e", - "7943": "66656174757265", - "7944": "20646573657274", - "7945": "706c696d656e74", - "7946": "61727473", - "7947": "20736164", - "7948": "626c6f77", - "7949": "6e677279", - "7950": "204d6f72", - "7951": "6869636b", - "7952": "7269636b", - "7953": "626c7565", - "7954": "7368616b", - "7955": "76657269", - "7956": "726f7773", - "7957": "616c6c73", - "7958": "636f7272", - "7959": "6f6c6174", - "7960": "41756e74", - "7961": "646f69", - "7962": "696765", - "7963": "6e6b65", - "7964": "656665", - "7965": "2062656c6f6e", - "7966": "20616e677279", - "7967": "66616d696c69", - "7968": "696e74696e67", - "7969": "206163636570", - "7970": "2070726f6475", - "7971": "726561746564", - "7972": "20646570656e", - "7973": "20636861696e", - "7974": "206163637573", - "7975": "20647261776e", - "7976": "617474616368", - "7977": "69676e6f72616e74", - "7978": "206d656574696e67", - "7979": "2067726f77696e67", - "7980": "2073756363656564", - "7981": "20676c616e636564", - "7982": "207468756e646572", - "7983": "20696e7175697265", - "7984": "6973747572626564", - "7985": "20696d70726f7665", - "7986": "6572636569766564", - "7987": "74656e696e", - "7988": "6e63686573", - "7989": "20616e6772", - "7990": "2070726f73", - "7991": "206170706c", - "7992": "2074726164", - "7993": "2063617374", - "7994": "6573736573", - "7995": "6177616b65", - "7996": "7468726561", - "7997": "7375726564", - "7998": "7472616e73", - "7999": "6561726573", - "8000": "2075747465", - "8001": "2072657072", - "8002": "476f6f", - "8003": "207265737065637461626c65", - "8004": "206578706c616e6174696f6e", - "8005": "20696e636c696e6174696f6e", - "8006": "75727374", - "8007": "656e6576", - "8008": "20726964", - "8009": "20637275", - "8010": "72757074", - "8011": "65737461", - "8012": "696e6775", - "8013": "6e7375", - "8014": "78616d", - "8015": "20737572726f756e646564", - "8016": "20756e636f6e7363696f75", - "8017": "636f6e736f6c6174696f6e", - "8018": "20636f6d70726568656e64", - "8019": "20656e676167656d656e74", - "8020": "4669747a77696c6c69616d", - "8021": "20636f6e6365726e6564", - "8022": "6f73736962696c697479", - "8023": "6f636375706174696f6e", - "8024": "2069676e6f72616e74", - "8025": "206469726563746564", - "8026": "206d69646e69676874", - "8027": "7068696c6f736f7068", - "8028": "657061726174696f6e", - "8029": "6672657175656e746c", - "8030": "2066616d696c69", - "8031": "2074656d706572", - "8032": "20746f6e677565", - "8033": "20666c6f776572", - "8034": "73756363656564", - "8035": "6a757374696365", - "8036": "20737472696b65", - "8037": "6e737472756374", - "8038": "73687564646572", - "8039": "20666f726d6564", - "8040": "20667574757265", - "8041": "696e74656e6465", - "8042": "207265706f7274", - "8043": "796f756e676572", - "8044": "20656e74726561", - "8045": "207477696365", - "8046": "206c6f6f7365", - "8047": "20656e636f75", - "8048": "20616c61726d", - "8049": "70726f706f73", - "8050": "6374696f6e61", - "8051": "20626f6f6b73", - "8052": "20666f727468", - "8053": "746865726564", - "8054": "64656c696361", - "8055": "20737072696e", - "8056": "617463686572", - "8057": "206a756d70", - "8058": "204a757374", - "8059": "2073706f74", - "8060": "2073686565", - "8061": "696e616c6c", - "8062": "2070657269", - "8063": "20696e666c", - "8064": "696d70726f", - "8065": "206b6579", - "8066": "6e696665", - "8067": "65737061", - "8068": "20736c61", - "8069": "6b697373", - "8070": "73757065", - "8071": "61676765", - "8072": "75676765", - "8073": "74757262", - "8074": "20636974", - "8075": "20667261", - "8076": "67617465", - "8077": "2070696c", - "8078": "53686572", - "8079": "706163", - "8080": "636875", - "8081": "657773", - "8082": "205175", - "8083": "204e65", - "8084": "6e6675", - "8085": "756369", - "8086": "204761", - "8087": "4172", - "8088": "706f737369626c79", - "8089": "2072656d6f766564", - "8090": "20636f6c6c656374", - "8091": "2047727970686f6e", - "8092": "6e61747572616c6c", - "8093": "7072657373697665", - "8094": "696e637265617365", - "8095": "2073687564646572", - "8096": "7472656173757265", - "8097": "70726f6261626c65", - "8098": "207072696e636970", - "8099": "74656e74696f6e73", - "8100": "68697374", - "8101": "70726561", - "8102": "70756c6c", - "8103": "6c75636b", - "8104": "64646572", - "8105": "6f727469", - "8106": "6d706c6f", - "8107": "65727479", - "8108": "686f7665", - "8109": "20657175", - "8110": "2065617273", - "8111": "2061647669", - "8112": "6561726572", - "8113": "656d626c65", - "8114": "74656d7065", - "8115": "6f726b6564", - "8116": "6c616e6775", - "8117": "6365656465", - "8118": "656174696e", - "8119": "20636f6c6c", - "8120": "2079656c6c", - "8121": "6865726564", - "8122": "536f6d6574", - "8123": "7374657269", - "8124": "206576696c", - "8125": "61636b6564", - "8126": "617374656e", - "8127": "776f756e64", - "8128": "2066726573", - "8129": "696d617465", - "8130": "2070617468", - "8131": "2074656e64", - "8132": "73756c", - "8133": "657072", - "8134": "6e6665", - "8135": "206465636964", - "8136": "70726574656e", - "8137": "696e636c696e", - "8138": "206e6f697365", - "8139": "20627261696e", - "8140": "207465657468", - "8141": "20616c697665", - "8142": "6f626c696765", - "8143": "6f6e6365616c", - "8144": "206661637473", - "8145": "206865617669", - "8146": "20636f6e7375", - "8147": "6f6e73746572", - "8148": "20686f706573", - "8149": "207765696768", - "8150": "68656176656e", - "8151": "756c6e657373", - "8152": "206465706172", - "8153": "20626c616d65", - "8154": "2061637475616c", - "8155": "70726163746963", - "8156": "206c6561766573", - "8157": "20726566757365", - "8158": "20696e636c696e", - "8159": "2070756c6c6564", - "8160": "20736d696c6564", - "8161": "206163636f7264", - "8162": "72657061726564", - "8163": "20696d70726f76", - "8164": "436c657276616c", - "8165": "636f6d6d656e64", - "8166": "66757365", - "8167": "686f6e65", - "8168": "77656174", - "8169": "64656772", - "8170": "20617374", - "8171": "69747561", - "8172": "6e736962", - "8173": "726461", - "8174": "706865", - "8175": "68616e646b65726368696566", - "8176": "20617272616e67656d656e74", - "8177": "20636f6e67726174756c6174", - "8178": "206465737065726174", - "8179": "64656c696768746564", - "8180": "696e71756972696573", - "8181": "20636976696c697479", - "8182": "206163636f6d70616e", - "8183": "66616c6c65", - "8184": "206472696e", - "8185": "626f6f6b73", - "8186": "2073637261", - "8187": "20656e6a6f", - "8188": "7265616c69", - "8189": "647261776e", - "8190": "726d696e67", - "8191": "7669637469", - "8192": "6570617274", - "8193": "736978", - "8194": "626f77", - "8195": "20637265617475726573", - "8196": "20616464726573736564", - "8197": "2063656e", - "8198": "6772616e", - "8199": "6d6f7573", - "8200": "726f6f66", - "8201": "68697274", - "8202": "70616c65", - "8203": "61766564", - "8204": "616c7465", - "8205": "6f796564", - "8206": "20736e6f", - "8207": "69736869", - "8208": "686f6f64", - "8209": "6e616e63", - "8210": "73746565", - "8211": "2068696c", - "8212": "76656c6c", - "8213": "6669726d", - "8214": "70726976", - "8215": "67726965", - "8216": "6c696e64", - "8217": "4174", - "8218": "206672657175656e746c79", - "8219": "206163636f6d706c697368", - "8220": "4875636b6c656265727279", - "8221": "206465636c617265", - "8222": "66616d696c696172", - "8223": "706f696e74696e67", - "8224": "6469726563746c79", - "8225": "2070726f74656374", - "8226": "206b656570696e67", - "8227": "2064657363656e64", - "8228": "2065786369746564", - "8229": "20636f6d70617373", - "8230": "6d697374616b656e", - "8231": "636f6d6d656e6365", - "8232": "6578616d696e6564", - "8233": "2067726174656675", - "8234": "207469726564", - "8235": "666c6f776572", - "8236": "686973746f72", - "8237": "686f6c64696e", - "8238": "206e756d6265", - "8239": "206c69766573", - "8240": "686174636865", - "8241": "6163636f7264", - "8242": "6c6974696573", - "8243": "74656d656e74", - "8244": "64657461696c", - "8245": "61696c696e67", - "8246": "20636f666669", - "8247": "63757272656e", - "8248": "637570", - "8249": "204175", - "8250": "6b6579", - "8251": "626579", - "8252": "4c65", - "8253": "6e6a", - "8254": "756e636f6d666f727461626c65", - "8255": "20617070726568656e73696f6e", - "8256": "64656369646564", - "8257": "2070726574656e", - "8258": "20686f75736573", - "8259": "61766f75726564", - "8260": "20636f6e667573", - "8261": "2062656c696566", - "8262": "207265636f7264", - "8263": "20726571756972", - "8264": "206d6f74696f6e", - "8265": "20706572666f72", - "8266": "20617474616368", - "8267": "736f6c6174696f", - "8268": "206d75726d7572", - "8269": "2072656c617465", - "8270": "6173686564", - "8271": "7461696e73", - "8272": "6f70696e67", - "8273": "6265726564", - "8274": "70656e696e", - "8275": "2068617465", - "8276": "6566756c6c", - "8277": "72736f6e61", - "8278": "61726e6564", - "8279": "20636f6163", - "8280": "636861726d", - "8281": "2073776561", - "8282": "6d656e7461", - "8283": "207361696c", - "8284": "7765696768", - "8285": "726f737065", - "8286": "2076656e", - "8287": "20746f70", - "8288": "20626f77", - "8289": "6e74726f", - "8290": "636b6c79", - "8291": "206a6f79", - "8292": "756e7365", - "8293": "64657669", - "8294": "74696669", - "8295": "20646f63", - "8296": "6c736f6e", - "8297": "204372", - "8298": "206e75", - "8299": "2c2d2d", - "8300": "636b79", - "8301": "7961", - "8302": "746173", - "8303": "686f65", - "8304": "72756d", - "8305": "73686d", - "8306": "206d61726b65", - "8307": "496e64656564", - "8308": "207265706c79", - "8309": "666561726564", - "8310": "697374616e74", - "8311": "70706f736974", - "8312": "756e6c657373", - "8313": "206a75737469", - "8314": "73756d6d6572", - "8315": "7265636f7264", - "8316": "207365637572", - "8317": "206b6e656573", - "8318": "696374696f6e", - "8319": "766572696e67", - "8320": "726573696465", - "8321": "6f756e746572", - "8322": "207472616365", - "8323": "207365697a65", - "8324": "6573706f6e64", - "8325": "7265706f7274", - "8326": "66756c6e6573", - "8327": "20547572", - "8328": "72746c65", - "8329": "6c616d70", - "8330": "6e65636b", - "8331": "20736872", - "8332": "6e636c75", - "8333": "20766172", - "8334": "75746564", - "8335": "6865656c", - "8336": "62617272", - "8337": "696e7369", - "8338": "65746169", - "8339": "74756174", - "8340": "74656e61", - "8341": "20706f737369626c79", - "8342": "20706f696e74696e67", - "8343": "696d70617469656e74", - "8344": "2070726f6475636564", - "8345": "206d6f76656d656e74", - "8346": "206d697374616b656e", - "8347": "206d6174657269616c", - "8348": "206275696c64696e67", - "8349": "646973636f76657279", - "8350": "20756e64657274616b", - "8351": "2064656369", - "8352": "6f6e73656e", - "8353": "7368616d65", - "8354": "6275727374", - "8355": "2067756573", - "8356": "6f75736573", - "8357": "206861726d", - "8358": "7265667573", - "8359": "6163637573", - "8360": "656574696e", - "8361": "2074686963", - "8362": "776c696e67", - "8363": "2073746972", - "8364": "72696e6773", - "8365": "76696f7573", - "8366": "69616c6c79", - "8367": "7564676564", - "8368": "6865617665", - "8369": "73656e7469", - "8370": "736c616e64", - "8371": "657264", - "8372": "657472", - "8373": "617061", - "8374": "61637475616c6c79", - "8375": "666f756e7461696e", - "8376": "206e6572766f7573", - "8377": "20736c6970706564", - "8378": "6472657373696e67", - "8379": "636f6e7374616e74", - "8380": "20747269756d7068", - "8381": "206f70706f736974", - "8382": "6d6174657269616c", - "8383": "65666572656e6365", - "8384": "20666f7267697665", - "8385": "2068657369746174", - "8386": "20796f756e676572", - "8387": "206e696767657273", - "8388": "6163637573746f6d6564", - "8389": "206578706c61696e6564", - "8390": "20656e7465727461696e", - "8391": "20636f6e6e6563746564", - "8392": "20706572636569766564", - "8393": "2070726573656e746564", - "8394": "206465636c6172", - "8395": "616e6973686564", - "8396": "65736572766564", - "8397": "7468756e646572", - "8398": "20756e6c657373", - "8399": "2064656c696361", - "8400": "6c616e7465726e", - "8401": "746174696f6e73", - "8402": "6e697474696e67", - "8403": "2061646d697265", - "8404": "4d657279746f6e", - "8405": "205379646e6579", - "8406": "204c6574", - "8407": "676f6c64", - "8408": "736c6970", - "8409": "65656368", - "8410": "6c616e67", - "8411": "77696465", - "8412": "61726c69", - "8413": "61646e", - "8414": "646566", - "8415": "20636f6e736f6c6174696f6e", - "8416": "20636f6d6d756e6963617469", - "8417": "206f62736572766174696f6e", - "8418": "20696e646966666572656e74", - "8419": "2064656c6967687466756c", - "8420": "636f6d6d756e6963617465", - "8421": "72656d656d6272616e6365", - "8422": "20636f6e736369656e6365", - "8423": "616368696e67", - "8424": "6f6d65686f77", - "8425": "204672656e63", - "8426": "207370656563", - "8427": "6d6f76696e67", - "8428": "6c616e636564", - "8429": "656173696e67", - "8430": "20696e766974", - "8431": "20746572726f", - "8432": "20636c65616e", - "8433": "6b656570696e", - "8434": "6f75726e616c", - "8435": "20726f616473", - "8436": "6f7267697665", - "8437": "206265686176", - "8438": "646573657274", - "8439": "207370656e74", - "8440": "6e6374696f6e", - "8441": "6c6967687473", - "8442": "636f6e736f6c", - "8443": "696e74657276", - "8444": "20626f617264", - "8445": "636976696c69", - "8446": "63617061626c", - "8447": "617061626c65", - "8448": "6772617068", - "8449": "207368696e", - "8450": "206465636c", - "8451": "6f696e6564", - "8452": "626c616d65", - "8453": "20726f6c6c", - "8454": "2072657369", - "8455": "6372697074", - "8456": "6772617465", - "8457": "706f6c6963", - "8458": "66696e616c", - "8459": "73696e6773", - "8460": "6c61726c79", - "8461": "6f6e696e67", - "8462": "666974", - "8463": "737569", - "8464": "6e6179", - "8465": "6464696e", - "8466": "6661696c", - "8467": "2062616c", - "8468": "61696e69", - "8469": "6c696674", - "8470": "20756e69", - "8471": "65717569", - "8472": "746f7273", - "8473": "6e64656e", - "8474": "656e616e", - "8475": "6575", - "8476": "20636f6d6d756e69636174696f6e", - "8477": "676e69", - "8478": "6c6170", - "8479": "657267", - "8480": "6c7374", - "8481": "756b65", - "8482": "206f63636173696f6e616c6c79", - "8483": "2061637475616c6c79", - "8484": "20666f756e7461696e", - "8485": "2066616d696c696172", - "8486": "696e74726f64756365", - "8487": "2057657374656e7261", - "8488": "2070726563696f7573", - "8489": "64657374726f796564", - "8490": "206170706172656e74", - "8491": "206578616d696e6564", - "8492": "7265636f6d6d656e64", - "8493": "2064656369646564", - "8494": "206d656173757265", - "8495": "207472656d626c65", - "8496": "2070726163746963", - "8497": "6f6c657261626c65", - "8498": "52656e6669656c64", - "8499": "7265636f6d6d656e", - "8500": "204d657279746f6e", - "8501": "63726f73736564", - "8502": "617469656e6365", - "8503": "20676976696e67", - "8504": "617474656e6465", - "8505": "7474656e646564", - "8506": "7061696e66756c", - "8507": "73637269707469", - "8508": "696d70726f7665", - "8509": "737570706f7274", - "8510": "636976696c6974", - "8511": "20636974697a65", - "8512": "206e696365", - "8513": "2063617463", - "8514": "6f73736564", - "8515": "2066656174", - "8516": "6772617373", - "8517": "6173746564", - "8518": "6f6e747261", - "8519": "2074616c6c", - "8520": "7569726564", - "8521": "7365637572", - "8522": "20626f6174", - "8523": "2070726976", - "8524": "6265686176", - "8525": "207265706f", - "8526": "64696e676c", - "8527": "2066696e61", - "8528": "736b696e", - "8529": "6d696464", - "8530": "6e73656e", - "8531": "20737769", - "8532": "67726163", - "8533": "2077616b", - "8534": "72756767", - "8535": "2076616c", - "8536": "696d6174", - "8537": "65636961", - "8538": "62656e65", - "8539": "204f68", - "8540": "6c6f67", - "8541": "686162", - "8542": "20646f6f7273", - "8543": "74656e64696e", - "8544": "206775657373", - "8545": "6e6172726f77", - "8546": "7373696f6e61", - "8547": "206f70706f73", - "8548": "726f6c6c6564", - "8549": "20536f6d6574", - "8550": "746966696564", - "8551": "20726573746f", - "8552": "646967", - "8553": "207068", - "8554": "20646573706572617465", - "8555": "20696e71756972696573", - "8556": "78616d696e6174696f6e", - "8557": "20706f73736573736564", - "8558": "2073656e74696d656e74", - "8559": "636f6e74656d706c6174", - "8560": "73656e736174696f6e73", - "8561": "6174746163686d656e74", - "8562": "2056656e6765616e6365", - "8563": "20637573", - "8564": "6170706c", - "8565": "20617368", - "8566": "20657374", - "8567": "20746173", - "8568": "73656375", - "8569": "61736573", - "8570": "67617468", - "8571": "6f727265", - "8572": "7272656e", - "8573": "6d616e64", - "8574": "6b6e696665", - "8575": "6f7474656e", - "8576": "746976656c", - "8577": "206e6f6973", - "8578": "6a6f696e65", - "8579": "6172696f75", - "8580": "2065646765", - "8581": "2073637265", - "8582": "20696e7669", - "8583": "20706f7274", - "8584": "6167676564", - "8585": "70726f7669", - "8586": "2065786572", - "8587": "736e657373", - "8588": "2068616269", - "8589": "206275696c", - "8590": "676d656e74", - "8591": "6e6973686d", - "8592": "7061727465", - "8593": "6e76696e63", - "8594": "76696e6365", - "8595": "757374696e", - "8596": "2053686572", - "8597": "656167", - "8598": "736e61", - "8599": "6e6f63", - "8600": "6d7374", - "8601": "756e6c", - "8602": "766965", - "8603": "20636f6d6d756e6963617465", - "8604": "6465736372697074696f6e", - "8605": "2073656e736174696f6e73", - "8606": "206d61726b6564", - "8607": "6e6572766f7573", - "8608": "77656c636f6d65", - "8609": "6d656173757265", - "8610": "70617469656e63", - "8611": "206c6967687465", - "8612": "20696e74656e64", - "8613": "20616769746174", - "8614": "207365697a6564", - "8615": "20636976696c69", - "8616": "2064656c697665", - "8617": "4a757374696e65", - "8618": "757474657265", - "8619": "206e65617265", - "8620": "20796f757468", - "8621": "20616273656e", - "8622": "6974696f6e73", - "8623": "736973746564", - "8624": "207072616374", - "8625": "736563757265", - "8626": "2073686f7665", - "8627": "686974686572", - "8628": "706563746174", - "8629": "6572666f726d", - "8630": "2063616e6f65", - "8631": "20506f6c6c79", - "8632": "6c617969", - "8633": "73686f65", - "8634": "6f6f7468", - "8635": "6e6f6973", - "8636": "6578636c", - "8637": "616e6b65", - "8638": "2070656e", - "8639": "73657175", - "8640": "7572656c", - "8641": "204e6f74", - "8642": "6961626c", - "8643": "636f7474", - "8644": "204669", - "8645": "6c6162", - "8646": "6f7570", - "8647": "766974", - "8648": "2066616c6c696e67", - "8649": "20617574686f7269", - "8650": "206f726465726564", - "8651": "20736f6c64696572", - "8652": "2063726f73736564", - "8653": "207061696e66756c", - "8654": "706f77657266756c", - "8655": "206c616e7465726e", - "8656": "2072657175657374", - "8657": "73696e67756c6172", - "8658": "52657075626c6963", - "8659": "72657475726e696e67", - "8660": "206469726563746c79", - "8661": "20617474656e646564", - "8662": "2064657374726f7965", - "8663": "6772616475616c6c79", - "8664": "2052656e6669656c64", - "8665": "206f626a656374696f", - "8666": "20746f6c657261626c", - "8667": "2070726f7370656374", - "8668": "207265636f6d6d656e", - "8669": "2052657075626c6963", - "8670": "7475616c6c", - "8671": "656174656e", - "8672": "73656d626c", - "8673": "2062697264", - "8674": "206c6f7365", - "8675": "2073756974", - "8676": "2068756e74", - "8677": "636c65616e", - "8678": "20736f6c64", - "8679": "656e74616c", - "8680": "6f7572696e", - "8681": "756c617465", - "8682": "207365656b", - "8683": "6972696e67", - "8684": "7274757265", - "8685": "6e79776179", - "8686": "657768", - "8687": "6e6b73", - "8688": "20426f", - "8689": "66696674", - "8690": "20737761", - "8691": "62726f77", - "8692": "73696c79", - "8693": "76616e69", - "8694": "204a6163", - "8695": "4c6f6f6b", - "8696": "73697465", - "8697": "746f726d", - "8698": "20706f72", - "8699": "656d696e", - "8700": "61666661", - "8701": "72646564", - "8702": "696e666c", - "8703": "746d65", - "8704": "426563", - "8705": "617468696e67", - "8706": "206869676865", - "8707": "206d6f75726e", - "8708": "2064656c6179", - "8709": "2073756d6d65", - "8710": "617274696e67", - "8711": "636561736564", - "8712": "205468657365", - "8713": "636f6e636569", - "8714": "6e7472656174", - "8715": "20636f6d6d65", - "8716": "207370617265", - "8717": "206b6e697474", - "8718": "2072657475726e696e67", - "8719": "2064657374726f796564", - "8720": "206772616475616c6c79", - "8721": "207068696c6f736f7068", - "8722": "617070726568656e7369", - "8723": "617474656e74696f6e73", - "8724": "2061736b696e67", - "8725": "20637279696e67", - "8726": "6465636c617265", - "8727": "617574686f7269", - "8728": "63616c6c696e67", - "8729": "206d6f76696e67", - "8730": "20666561726564", - "8731": "20486174746572", - "8732": "747269756d7068", - "8733": "696e7175697265", - "8734": "206c6966746564", - "8735": "206b6973736564", - "8736": "20736563757265", - "8737": "656e6465726564", - "8738": "6f756e64696e67", - "8739": "617374656e6564", - "8740": "656d6f74696f6e", - "8741": "73757370656374", - "8742": "20696e74657276", - "8743": "20666561747572", - "8744": "6d616e61676572", - "8745": "636861696e", - "8746": "2068616c6c", - "8747": "2065786973", - "8748": "2061737365", - "8749": "20696d7061", - "8750": "726b696e67", - "8751": "726f766564", - "8752": "6572696573", - "8753": "616c61726d", - "8754": "6173696e67", - "8755": "636865656b", - "8756": "206761696e", - "8757": "2073686f63", - "8758": "756c617469", - "8759": "6976657273", - "8760": "617373656e", - "8761": "2073756e73", - "8762": "7265717569", - "8763": "6572666f72", - "8764": "6261626c79", - "8765": "207261696e", - "8766": "657369676e", - "8767": "62697264", - "8768": "73686f70", - "8769": "206c6f75", - "8770": "69746e65", - "8771": "69617279", - "8772": "6c696369", - "8773": "6f6c6572", - "8774": "57686f", - "8775": "204772", - "8776": "756469", - "8777": "756361", - "8778": "636970", - "8779": "736172", - "8780": "617574686f726974", - "8781": "2070726573736564", - "8782": "20666c6f77657273", - "8783": "20636f6e73656e74", - "8784": "206a757374696365", - "8785": "696e737472756374", - "8786": "73657276616e7473", - "8787": "2077696c6c696e67", - "8788": "2074726561737572", - "8789": "20776f726b696e67", - "8790": "20656d6f74696f6e", - "8791": "6170706f696e7465", - "8792": "70706f696e746564", - "8793": "726573747261696e", - "8794": "7472616e7175696c", - "8795": "617070726568656e", - "8796": "6c656374696f6e73", - "8797": "676c616e63696e67", - "8798": "706f74", - "8799": "73656d" - }, - "vocab_size": 10000, - "byte_offset": 4, - "special_tokens": { - "": 0, - "": 1, - "": 2, - "": 3 - } -} \ No newline at end of file diff --git a/gui.py b/gui.py index 0cc4e965..dcfd73b6 100644 --- a/gui.py +++ b/gui.py @@ -1,1070 +1,354 @@ """ -neuralnet.cpp GUI - 图形化训练与推理界面 -依赖: Python 3.8+, tkinter (内置), Pillow (pip install Pillow) +neuralnet.cpp GUI - 重构版 (单文件) +依赖: Python 3.8+, tkinter, Pillow """ - -import os -import sys -import csv -import math -import subprocess -import threading +import os, sys, math, subprocess, threading, tempfile, re import tkinter as tk from tkinter import ttk, filedialog, messagebox from pathlib import Path +# 假设 go_board 在同级目录 +sys.path.insert(0, str(Path(__file__).parent)) +try: + from go_board import GoGamePanel +except ImportError: + GoGamePanel = None + # ── 常量 ────────────────────────────────────────────── BUILD_DIR = Path(__file__).parent / "build" -_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" -TRAIN_EXE = BUILD_DIR / f"mnist_train{_EXE_SUFFIX}" -INFER_EXE = BUILD_DIR / f"mnist_infer{_EXE_SUFFIX}" -TEXT_TRAIN_EXE = BUILD_DIR / f"text_train{_EXE_SUFFIX}" -TEXT_INFER_EXE = BUILD_DIR / f"text_infer{_EXE_SUFFIX}" -DEFAULT_MODEL = Path(__file__).parent / "pretrained" / "model.bin" -DEFAULT_GPT_MODEL = Path(__file__).parent / "gpt_model.bin" -DEFAULT_DATASET = Path(__file__).parent / "datasets" / "mnist_data" -PIXEL_SIZE = 8 # 每像素的显示尺寸 (28×28 → 224×224 画布) -IMG_DIM = 28 - - -# ── 工具函数 ────────────────────────────────────────── -def load_csv_pixels(csv_path: str) -> list[float]: - """读取单行 CSV，返回 784 个 [0,1] 浮点数""" - with open(csv_path, "r") as f: - line = f.readline().strip() - return [float(v) for v in line.split(",")] - - -def draw_digit(canvas: tk.Canvas, pixels: list[float], offset_x=0, offset_y=0): - """在 Canvas 上绘制 28×28 手写数字""" - canvas.delete("all") - for r in range(IMG_DIM): - for c in range(IMG_DIM): - v = pixels[r * IMG_DIM + c] - gray = int(v * 255) - color = f"#{gray:02x}{gray:02x}{gray:02x}" - x1 = offset_x + c * PIXEL_SIZE - y1 = offset_y + r * PIXEL_SIZE - canvas.create_rectangle(x1, y1, x1 + PIXEL_SIZE, y1 + PIXEL_SIZE, - fill=color, outline="") - - -# ── GUI 主类 ───────────────────────────────────────── -class NeuralNetGUI(tk.Tk): +_EXE = ".exe" if sys.platform == "win32" else "" +EXES = { + "mnist_train": BUILD_DIR / f"mnist_train{_EXE}", + "mnist_infer": BUILD_DIR / f"mnist_infer{_EXE}", + "text_train": BUILD_DIR / f"text_train{_EXE}", + "text_infer": BUILD_DIR / f"text_infer{_EXE}", + "tokenizer_train": BUILD_DIR / f"tokenizer_train{_EXE}", + "tokenizer_infer": BUILD_DIR / f"tokenizer_infer{_EXE}", +} +IMG_DIM, PIXEL_SIZE = 28, 8 + +# ═══════════════════════════════════════════════════════ +# 1. 核心重构组件 (消灭重复代码的利器) +# ═══════════════════════════════════════════════════════ + +class TaskRunner: + """统一处理子进程与线程，告别每个 Tab 都写一遍 threading.Thread""" def __init__(self): - super().__init__() - self.title("neuralnet.cpp — MNIST & GPT 训练 & 推理") - self.resizable(False, False) - - # 主题风格 - style = ttk.Style(self) - style.theme_use("clam") - style.configure("TButton", padding=4) - style.configure("TLabel", padding=2) - style.configure("Header.TLabel", font=("Segoe UI", 12, "bold")) - - self._process: subprocess.Popen | None = None - - notebook = ttk.Notebook(self) - notebook.pack(fill="both", expand=True, padx=8, pady=8) - - # ── 训练 Tab ── - self.train_frame = ttk.Frame(notebook, padding=8) - notebook.add(self.train_frame, text=" 🏋️ 训练 ") - self._build_train_tab() - - # ── 推理 Tab ── - self.infer_frame = ttk.Frame(notebook, padding=8) - notebook.add(self.infer_frame, text=" 🔍 推理 ") - self._build_infer_tab() - - # ── 图片查看 Tab ── - self.viewer_frame = ttk.Frame(notebook, padding=8) - notebook.add(self.viewer_frame, text=" 🖼️ 图片查看 ") - self._build_viewer_tab() - - # ── GPT 训练 Tab ── - self.text_train_frame = ttk.Frame(notebook, padding=8) - notebook.add(self.text_train_frame, text=" 📝 GPT 训练 ") - self._build_text_train_tab() - - # ── GPT 推理 Tab ── - self.text_infer_frame = ttk.Frame(notebook, padding=8) - notebook.add(self.text_infer_frame, text=" 💬 GPT 推理 ") - self._build_text_infer_tab() - - self.protocol("WM_DELETE_WINDOW", self._on_close) - - # ============================================================ - # 训练 Tab - # ============================================================ - def _build_train_tab(self): - f = self.train_frame - row = 0 - - # 数据集路径 - ttk.Label(f, text="数据集目录:").grid(row=row, column=0, sticky="w") - self.train_dataset_var = tk.StringVar(value=str(DEFAULT_DATASET)) - ttk.Entry(f, textvariable=self.train_dataset_var, width=48).grid(row=row, column=1, padx=4) - ttk.Button(f, text="浏览…", command=self._browse_dataset).grid(row=row, column=2) - row += 1 - - # 保存路径 - ttk.Label(f, text="模型保存路径:").grid(row=row, column=0, sticky="w") - self.train_save_var = tk.StringVar(value="mnist_model.bin") - ttk.Entry(f, textvariable=self.train_save_var, width=48).grid(row=row, column=1, padx=4) - ttk.Button(f, text="浏览…", command=self._browse_save).grid(row=row, column=2) - row += 1 - - # 恢复训练 - self.train_resume_var = tk.BooleanVar(value=False) - ttk.Checkbutton(f, text="从已有模型恢复训练", variable=self.train_resume_var, - command=self._toggle_resume).grid(row=row, column=0, columnspan=2, sticky="w") - row += 1 - - self.resume_frame = ttk.Frame(f) - self.resume_frame.grid(row=row, column=0, columnspan=3, sticky="ew", pady=(0, 4)) - ttk.Label(self.resume_frame, text="模型路径:").pack(side="left") - self.train_resume_path_var = tk.StringVar(value=str(DEFAULT_MODEL)) - self.resume_entry = ttk.Entry(self.resume_frame, textvariable=self.train_resume_path_var, width=48) - self.resume_entry.pack(side="left", padx=4) - self.resume_btn = ttk.Button(self.resume_frame, text="浏览…", command=self._browse_resume) - self.resume_btn.pack(side="left") - self._toggle_resume() - row += 1 - - # 超参数 - param_frame = ttk.LabelFrame(f, text="超参数", padding=6) - param_frame.grid(row=row, column=0, columnspan=3, sticky="ew", pady=4) - row += 1 - - ttk.Label(param_frame, text="轮数:").grid(row=0, column=0, sticky="w") - self.train_epochs_var = tk.IntVar(value=5) - ttk.Spinbox(param_frame, from_=1, to=1000, width=6, - textvariable=self.train_epochs_var).grid(row=0, column=1, padx=(0, 16)) - - ttk.Label(param_frame, text="学习率:").grid(row=0, column=2, sticky="w") - self.train_lr_var = tk.DoubleVar(value=0.01) - ttk.Spinbox(param_frame, from_=0.0001, to=1.0, increment=0.001, width=8, - textvariable=self.train_lr_var, format="%.4f").grid(row=0, column=3, padx=(0, 16)) - - ttk.Label(param_frame, text="批大小:").grid(row=0, column=4, sticky="w") - self.train_batch_var = tk.IntVar(value=64) - ttk.Spinbox(param_frame, from_=1, to=4096, width=6, - textvariable=self.train_batch_var).grid(row=0, column=5) - - ttk.Label(param_frame, text="优化器:").grid(row=1, column=0, sticky="w", pady=(6, 0)) - self.train_opt_var = tk.StringVar(value="adam") - opt_combo = ttk.Combobox(param_frame, textvariable=self.train_opt_var, width=14, state="readonly", - values=["sgd", "sgd_w_momentum", "adam"]) - opt_combo.grid(row=1, column=1, sticky="w", pady=(6, 0)) - - ttk.Label(param_frame, text="模型类型:").grid(row=1, column=2, sticky="w", pady=(6, 0)) - self.train_model_type_var = tk.StringVar(value="mlp") - model_type_combo = ttk.Combobox(param_frame, textvariable=self.train_model_type_var, width=14, state="readonly", - values=["mlp", "transformer"]) - model_type_combo.grid(row=1, column=3, sticky="w", pady=(6, 0)) - row += 1 - - # 按钮 - btn_frame = ttk.Frame(f) - btn_frame.grid(row=row, column=0, columnspan=3, pady=6) - row += 1 - - self.train_start_btn = ttk.Button(btn_frame, text="▶ 开始训练", command=self._start_training) - self.train_start_btn.pack(side="left", padx=4) - self.train_stop_btn = ttk.Button(btn_frame, text="⏹ 停止", command=self._stop_training, state="disabled") - self.train_stop_btn.pack(side="left", padx=4) - row += 1 - - # 日志输出 - self.train_log = tk.Text(f, height=14, width=72, state="disabled", - font=("Consolas", 9), bg="#1e1e1e", fg="#d4d4d4", - insertbackground="white") - self.train_log.grid(row=row, column=0, columnspan=3, sticky="ew") - scroll = ttk.Scrollbar(f, orient="vertical", command=self.train_log.yview) - scroll.grid(row=row, column=3, sticky="ns") - self.train_log["yscrollcommand"] = scroll.set - - # ---- 浏览按钮 ---- - def _browse_dataset(self): - path = filedialog.askdirectory(title="选择数据集目录") - if path: - self.train_dataset_var.set(path) - - def _browse_save(self): - path = filedialog.asksaveasfilename(title="模型保存路径", defaultextension=".bin", - filetypes=[("Binary", "*.bin"), ("All", "*.*")]) - if path: - self.train_save_var.set(path) - - def _browse_resume(self): - path = filedialog.askopenfilename(title="选择恢复模型", filetypes=[("Binary", "*.bin"), ("All", "*.*")]) - if path: - self.train_resume_path_var.set(path) - - def _toggle_resume(self): - state = "normal" if self.train_resume_var.get() else "disabled" - self.resume_entry.config(state=state) - self.resume_btn.config(state=state) - - # ---- 训练 ---- - def _start_training(self): - exe = str(TRAIN_EXE) - if not Path(exe).exists(): - messagebox.showerror("错误", f"找不到训练可执行文件:\n{exe}\n请先构建项目:\n cmake -B build -G Ninja && ninja -C build") - return - - cmd = [exe, - "--dataset", self.train_dataset_var.get(), - "--save", self.train_save_var.get(), - "--epochs", str(self.train_epochs_var.get()), - "--lr", str(self.train_lr_var.get()), - "--batch-size", str(self.train_batch_var.get()), - "--optimizer", self.train_opt_var.get(), - "--model-type", self.train_model_type_var.get()] - if self.train_resume_var.get(): - cmd += ["--resume", self.train_resume_path_var.get()] - - self._log_train(f"$ {' '.join(cmd)}\n") - self.train_start_btn.config(state="disabled") - self.train_stop_btn.config(state="normal") - - threading.Thread(target=self._run_process, args=(cmd, self._log_train, self._train_done), - daemon=True).start() - - def _stop_training(self): - if self._process and self._process.poll() is None: - self._process.terminate() - self._log_train("\n[已终止]\n") - self._train_done() - - def _train_done(self): - self.train_start_btn.config(state="normal") - self.train_stop_btn.config(state="disabled") - - def _log_train(self, text: str): - self.train_log.config(state="normal") - self.train_log.insert("end", text) - self.train_log.see("end") - self.train_log.config(state="disabled") + self.process = None - # ============================================================ - # 推理 Tab - # ============================================================ - def _build_infer_tab(self): - f = self.infer_frame - row = 0 - - # 模型路径 - ttk.Label(f, text="模型文件:").grid(row=row, column=0, sticky="w") - self.infer_model_var = tk.StringVar(value=str(DEFAULT_MODEL)) - ttk.Entry(f, textvariable=self.infer_model_var, width=48).grid(row=row, column=1, padx=4) - ttk.Button(f, text="浏览…", command=self._browse_model).grid(row=row, column=2) - row += 1 - - # 输入路径 - ttk.Label(f, text="图片/目录:").grid(row=row, column=0, sticky="w") - self.infer_input_var = tk.StringVar() - ttk.Entry(f, textvariable=self.infer_input_var, width=48).grid(row=row, column=1, padx=4) - ttk.Button(f, text="浏览…", command=self._browse_input).grid(row=row, column=2) - row += 1 - - # Top-K + 模型类型 - param_frame = ttk.Frame(f) - param_frame.grid(row=row, column=0, columnspan=3, sticky="w", pady=4) - row += 1 - ttk.Label(param_frame, text="Top-K:").pack(side="left") - self.infer_topk_var = tk.IntVar(value=3) - ttk.Spinbox(param_frame, from_=1, to=10, width=4, - textvariable=self.infer_topk_var).pack(side="left", padx=4) - ttk.Label(param_frame, text=" 模型类型:").pack(side="left") - self.infer_model_type_var = tk.StringVar(value="mlp") - infer_model_type_combo = ttk.Combobox(param_frame, textvariable=self.infer_model_type_var, - width=12, state="readonly", - values=["mlp", "transformer"]) - infer_model_type_combo.pack(side="left", padx=4) - row += 1 - - # 按钮 - ttk.Button(f, text="▶ 开始推理", command=self._start_infer).grid(row=row, column=0, columnspan=3, pady=4) - row += 1 - # ── 手写板 ────────────────────────────────────── - self._handwriting_infer = False - DRAW_SCALE = 10 - DRAW_SIZE = IMG_DIM * DRAW_SCALE # 280 - self._draw_size = DRAW_SIZE - self._draw_scale = DRAW_SCALE - self._brush_radius = 14 - self._draw_pixels = [0.0] * (DRAW_SIZE * DRAW_SIZE) - self._drawing = False - - draw_frame = ttk.LabelFrame(f, text="✍️ 手写识别", padding=6) - draw_frame.grid(row=row, column=0, columnspan=3, sticky="ew", pady=4) - - draw_left = ttk.Frame(draw_frame) - draw_left.pack(side="left", padx=(0, 12)) - - self._draw_canvas = tk.Canvas(draw_left, width=DRAW_SIZE, height=DRAW_SIZE, - bg="white", cursor="pencil", relief="sunken", bd=2) - self._draw_canvas.pack() - self._draw_canvas.bind("", self._on_draw_start) - self._draw_canvas.bind("", self._on_draw_move) - self._draw_canvas.bind("", self._on_draw_end) - - btn_row = ttk.Frame(draw_left) - btn_row.pack(fill="x", pady=4) - ttk.Button(btn_row, text="🗑️ 清除", command=self._clear_drawing).pack(side="left", padx=4) - ttk.Button(btn_row, text="🔍 识别手写数字", command=self._recognize_drawing).pack(side="left", padx=4) - - # 右侧：28×28 预览 - draw_right = ttk.Frame(draw_frame) - draw_right.pack(side="left", fill="both", expand=True) - ttk.Label(draw_right, text="28×28 预览:", font=("Segoe UI", 9)).pack(anchor="w") - preview_size = IMG_DIM * PIXEL_SIZE - self._draw_preview = tk.Canvas(draw_right, width=preview_size, height=preview_size, bg="black") - self._draw_preview.pack(pady=4) - - row += 1 - # 结果区域 - result_frame = ttk.Frame(f) - result_frame.grid(row=row, column=0, columnspan=3, sticky="ew") - row += 1 - - # 左侧: 图片预览 - preview_frame = ttk.LabelFrame(result_frame, text="图片预览", padding=4) - preview_frame.pack(side="left", padx=(0, 8)) - canvas_size = IMG_DIM * PIXEL_SIZE - self.infer_canvas = tk.Canvas(preview_frame, width=canvas_size, height=canvas_size, bg="black") - self.infer_canvas.pack() - - # 右侧: 预测结果 - right_frame = ttk.Frame(result_frame) - right_frame.pack(side="left", fill="both", expand=True) - - self.infer_filename_label = ttk.Label(right_frame, text="", font=("Segoe UI", 9)) - self.infer_filename_label.pack(anchor="w") - - self.infer_bars_frame = ttk.Frame(right_frame) - self.infer_bars_frame.pack(fill="both", expand=True, pady=4) - - # 日志 - row += 1 - self.infer_log = tk.Text(f, height=10, width=72, state="disabled", - font=("Consolas", 9), bg="#1e1e1e", fg="#d4d4d4") - self.infer_log.grid(row=row, column=0, columnspan=3, sticky="ew") - scroll = ttk.Scrollbar(f, orient="vertical", command=self.infer_log.yview) - scroll.grid(row=row, column=3, sticky="ns") - self.infer_log["yscrollcommand"] = scroll.set - - def _browse_model(self): - path = filedialog.askopenfilename(title="选择模型文件", filetypes=[("Binary", "*.bin"), ("All", "*.*")]) - if path: - self.infer_model_var.set(path) - - def _browse_input(self): - path = filedialog.askopenfilename(title="选择图片 CSV", filetypes=[("CSV", "*.csv"), ("All", "*.*")]) - if not path: - path = filedialog.askdirectory(title="选择图片目录") - if path: - self.infer_input_var.set(path) - - # ---- 手写板事件 ---- - def _on_draw_start(self, event): - self._drawing = True - self._paint_at(event.x, event.y) - - def _on_draw_move(self, event): - if self._drawing: - self._paint_at(event.x, event.y) - - def _on_draw_end(self, event): - self._drawing = False - - def _paint_at(self, cx, cy): - """在画板 (cx, cy) 处绘制笔触""" - r = self._brush_radius - for dy in range(-r, r + 1): - for dx in range(-r, r + 1): - dist_sq = dx * dx + dy * dy - if dist_sq > r * r: - continue - px, py = cx + dx, cy + dy - if 0 <= px < self._draw_size and 0 <= py < self._draw_size: - dist = math.sqrt(dist_sq) - sigma = r / 2.0 - intensity = math.exp(-dist_sq / (2 * sigma * sigma)) - idx = py * self._draw_size + px - self._draw_pixels[idx] = max(self._draw_pixels[idx], intensity) - self._draw_canvas.create_oval(cx - r, cy - r, cx + r, cy + r, - fill="black", outline="") - - def _clear_drawing(self): - """清除手写板""" - self._draw_pixels = [0.0] * (self._draw_size * self._draw_size) - self._draw_canvas.delete("all") - self._draw_preview.delete("all") - - def _downsample_drawing(self) -> list[float]: - """将 280×280 画板下采样为 28×28 像素值""" - scale = self._draw_scale - pixels = [] - for r in range(IMG_DIM): - for c in range(IMG_DIM): - total = 0.0 - for dy in range(scale): - for dx in range(scale): - y = r * scale + dy - x = c * scale + dx - total += self._draw_pixels[y * self._draw_size + x] - avg = total / (scale * scale) - pixels.append(avg) - return pixels - - def _recognize_drawing(self): - """识别手写板上的数字""" - exe = str(INFER_EXE) - if not Path(exe).exists(): - messagebox.showerror("错误", f"找不到推理可执行文件:\n{exe}\n请先构建项目") - return - - # 检查是否有内容 - if max(self._draw_pixels) < 0.01: - messagebox.showwarning("提示", "请先在手写板上写一个数字") + def run(self, cmd, log_fn, done_fn, exe_name=""): + exe_path = EXES.get(exe_name, cmd[0]) + if not Path(exe_path).exists(): + messagebox.showerror("错误", f"找不到可执行文件:\n{exe_path}") + done_fn() return - pixels = self._downsample_drawing() - - # 显示 28×28 预览 - draw_digit(self._draw_preview, pixels) - draw_digit(self.infer_canvas, pixels) - self.infer_filename_label.config(text="手写输入") - - # 写入临时 CSV - import tempfile - tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False, newline="") - tmp.write(",".join(f"{v:.6f}" for v in pixels)) - tmp.close() - - cmd = [exe, tmp.name, - "--model", self.infer_model_var.get(), - "--model-type", self.infer_model_type_var.get(), - "--topk", str(self.infer_topk_var.get())] - - self._log_infer(f"[手写识别] $ {' '.join(cmd)}\n") - self._handwriting_infer = True - - def _run(): + def _target(): try: - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - self._log_infer(result.stdout) - if result.stderr: - self._log_infer(result.stderr) - self._parse_and_display_infer(result.stdout) - except FileNotFoundError: - self._log_infer("错误: 可执行文件未找到\n") - except subprocess.TimeoutExpired: - self._log_infer("错误: 推理超时\n") + flags = subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0 + self.process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, bufsize=1, creationflags=flags + ) + for line in self.process.stdout: + log_fn(line) + self.process.wait() except Exception as e: - self._log_infer(f"错误: {e}\n") + log_fn(f"执行错误: {e}\n") finally: - try: - os.unlink(tmp.name) - except OSError: - pass - - threading.Thread(target=_run, daemon=True).start() - - def _start_infer(self): - self._handwriting_infer = False - exe = str(INFER_EXE) - if not Path(exe).exists(): - messagebox.showerror("错误", f"找不到推理可执行文件:\n{exe}\n请先构建项目:\n cmake -B build -G Ninja && ninja -C build") - return - - input_path = self.infer_input_var.get().strip() - if not input_path: - messagebox.showwarning("提示", "请指定图片文件或目录") - return - - cmd = [exe, input_path, - "--model", self.infer_model_var.get(), - "--model-type", self.infer_model_type_var.get(), - "--topk", str(self.infer_topk_var.get())] - - self._log_infer(f"$ {' '.join(cmd)}\n") - threading.Thread(target=self._run_infer_process, args=(cmd,), daemon=True).start() - - def _run_infer_process(self, cmd: list[str]): - try: - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - self._log_infer(result.stdout) - if result.stderr: - self._log_infer(result.stderr) - - # 尝试解析结果并显示图片 - self._parse_and_display_infer(result.stdout) - except FileNotFoundError: - self._log_infer("错误: 可执行文件未找到\n") - except subprocess.TimeoutExpired: - self._log_infer("错误: 推理超时\n") - - def _parse_and_display_infer(self, output: str): - """从推理输出中解析结果并显示在 Canvas 上""" - lines = output.strip().split("\n") - for line in lines: - if " -> " in line: - # 格式: filename -> 3 (98.2%) 7 (1.2%) 5 (0.3%) - parts = line.split(" -> ") - if len(parts) == 2: - filename = parts[0].strip() - predictions_str = parts[1].strip() - - # 显示文件名 - if self._handwriting_infer: - self.after(0, lambda: self.infer_filename_label.config(text="手写输入")) - else: - self.after(0, lambda f=filename: self.infer_filename_label.config(text=f)) - - # 解析预测结果 - predictions = [] - import re - for m in re.finditer(r"(\d+)\s*\((\d+\.?\d*)%\)", predictions_str): - predictions.append((int(m.group(1)), float(m.group(2)))) - - # 显示结果条形图 - self.after(0, lambda p=predictions: self._show_prediction_bars(p)) - - # 尝试显示图片（手写模式跳过，已有预览） - if not self._handwriting_infer: - input_path = self.infer_input_var.get().strip() - if os.path.isfile(input_path): - self.after(0, lambda p=input_path: self._display_csv_image(p)) - elif os.path.isdir(input_path): - img_path = os.path.join(input_path, filename) - if os.path.isfile(img_path): - self.after(0, lambda p=img_path: self._display_csv_image(p)) - break - - def _show_prediction_bars(self, predictions: list[tuple[int, float]]): - for w in self.infer_bars_frame.winfo_children(): - w.destroy() - for digit, conf in predictions: - row = ttk.Frame(self.infer_bars_frame) - row.pack(fill="x", pady=1) - ttk.Label(row, text=f"{digit}", width=2, font=("Consolas", 11, "bold")).pack(side="left") - bar_canvas = tk.Canvas(row, height=18, bg="#333", highlightthickness=0) - bar_canvas.pack(side="left", fill="x", expand=True, padx=4) - bar_canvas.update_idletasks() - w = bar_canvas.winfo_width() - bar_canvas.create_rectangle(0, 0, int(w * conf / 100), 18, - fill="#4ec9b0", outline="") - ttk.Label(row, text=f"{conf:.1f}%", width=6).pack(side="left") - - def _display_csv_image(self, csv_path: str): - try: - pixels = load_csv_pixels(csv_path) - draw_digit(self.infer_canvas, pixels) - except Exception as e: - self._log_infer(f"图片加载失败: {e}\n") - - def _log_infer(self, text: str): - def _do(): - self.infer_log.config(state="normal") - self.infer_log.insert("end", text) - self.infer_log.see("end") - self.infer_log.config(state="disabled") - self.after(0, _do) - - # ============================================================ - # GPT 训练 Tab - # ============================================================ - def _build_text_train_tab(self): - f = self.text_train_frame - row = 0 - - # 文本文件路径 - ttk.Label(f, text="训练文本文件:").grid(row=row, column=0, sticky="w") - self.text_train_file_var = tk.StringVar() - ttk.Entry(f, textvariable=self.text_train_file_var, width=48).grid(row=row, column=1, padx=4) - ttk.Button(f, text="浏览…", command=self._browse_text_file).grid(row=row, column=2) - row += 1 - - # 保存路径 - ttk.Label(f, text="模型保存路径:").grid(row=row, column=0, sticky="w") - self.text_train_save_var = tk.StringVar(value=str(DEFAULT_GPT_MODEL)) - ttk.Entry(f, textvariable=self.text_train_save_var, width=48).grid(row=row, column=1, padx=4) - ttk.Button(f, text="浏览…", command=self._browse_text_save).grid(row=row, column=2) - row += 1 - - # 恢复训练 - self.text_train_resume_var = tk.BooleanVar(value=False) - ttk.Checkbutton(f, text="从已有模型恢复训练", variable=self.text_train_resume_var, - command=self._toggle_text_resume).grid(row=row, column=0, columnspan=2, sticky="w") - row += 1 - - self.text_resume_frame = ttk.Frame(f) - self.text_resume_frame.grid(row=row, column=0, columnspan=3, sticky="ew", pady=(0, 4)) - ttk.Label(self.text_resume_frame, text="模型路径:").pack(side="left") - self.text_train_resume_path_var = tk.StringVar(value=str(DEFAULT_GPT_MODEL)) - self.text_resume_entry = ttk.Entry(self.text_resume_frame, textvariable=self.text_train_resume_path_var, width=48) - self.text_resume_entry.pack(side="left", padx=4) - self.text_resume_btn = ttk.Button(self.text_resume_frame, text="浏览…", command=self._browse_text_resume) - self.text_resume_btn.pack(side="left") - self._toggle_text_resume() - row += 1 - - # 超参数 - param_frame = ttk.LabelFrame(f, text="超参数", padding=6) - param_frame.grid(row=row, column=0, columnspan=3, sticky="ew", pady=4) - row += 1 - - ttk.Label(param_frame, text="轮数:").grid(row=0, column=0, sticky="w") - self.text_train_epochs_var = tk.IntVar(value=10) - ttk.Spinbox(param_frame, from_=1, to=1000, width=6, - textvariable=self.text_train_epochs_var).grid(row=0, column=1, padx=(0, 16)) - - ttk.Label(param_frame, text="学习率:").grid(row=0, column=2, sticky="w") - self.text_train_lr_var = tk.DoubleVar(value=0.001) - ttk.Spinbox(param_frame, from_=0.00001, to=1.0, increment=0.0001, width=8, - textvariable=self.text_train_lr_var, format="%.5f").grid(row=0, column=3, padx=(0, 16)) - - ttk.Label(param_frame, text="批大小:").grid(row=0, column=4, sticky="w") - self.text_train_batch_var = tk.IntVar(value=32) - ttk.Spinbox(param_frame, from_=1, to=256, width=6, - textvariable=self.text_train_batch_var).grid(row=0, column=5) - - ttk.Label(param_frame, text="序列长度:").grid(row=1, column=0, sticky="w", pady=(6, 0)) - self.text_train_seq_len_var = tk.IntVar(value=256) - ttk.Spinbox(param_frame, from_=16, to=1024, width=6, - textvariable=self.text_train_seq_len_var).grid(row=1, column=1, pady=(6, 0)) - - ttk.Label(param_frame, text="优化器:").grid(row=1, column=2, sticky="w", pady=(6, 0)) - self.text_train_opt_var = tk.StringVar(value="adam") - ttk.Combobox(param_frame, textvariable=self.text_train_opt_var, width=14, state="readonly", - values=["sgd", "sgd_w_momentum", "adam"]).grid(row=1, column=3, sticky="w", pady=(6, 0)) - - # 模型架构参数 - arch_frame = ttk.LabelFrame(f, text="模型架构", padding=6) - arch_frame.grid(row=row, column=0, columnspan=3, sticky="ew", pady=4) - row += 1 - - ttk.Label(arch_frame, text="模型维度:").grid(row=0, column=0, sticky="w") - self.text_train_d_model_var = tk.IntVar(value=128) - ttk.Spinbox(arch_frame, from_=32, to=512, width=6, - textvariable=self.text_train_d_model_var).grid(row=0, column=1, padx=(0, 16)) - - ttk.Label(arch_frame, text="注意力头:").grid(row=0, column=2, sticky="w") - self.text_train_num_heads_var = tk.IntVar(value=4) - ttk.Spinbox(arch_frame, from_=1, to=16, width=6, - textvariable=self.text_train_num_heads_var).grid(row=0, column=3, padx=(0, 16)) - - ttk.Label(arch_frame, text="Transformer 层数:").grid(row=0, column=4, sticky="w") - self.text_train_num_layers_var = tk.IntVar(value=4) - ttk.Spinbox(arch_frame, from_=1, to=16, width=6, - textvariable=self.text_train_num_layers_var).grid(row=0, column=5) - - ttk.Label(arch_frame, text="FFN 维度:").grid(row=1, column=0, sticky="w", pady=(6, 0)) - self.text_train_d_ff_var = tk.IntVar(value=512) - ttk.Spinbox(arch_frame, from_=64, to=2048, width=6, - textvariable=self.text_train_d_ff_var).grid(row=1, column=1, pady=(6, 0)) - row += 1 - - # 按钮 - btn_frame = ttk.Frame(f) - btn_frame.grid(row=row, column=0, columnspan=3, pady=6) - row += 1 - - self.text_train_start_btn = ttk.Button(btn_frame, text="▶ 开始训练", command=self._start_text_training) - self.text_train_start_btn.pack(side="left", padx=4) - self.text_train_stop_btn = ttk.Button(btn_frame, text="⏹ 停止", command=self._stop_text_training, state="disabled") - self.text_train_stop_btn.pack(side="left", padx=4) - row += 1 - - # 日志输出 - self.text_train_log = tk.Text(f, height=14, width=72, state="disabled", - font=("Consolas", 9), bg="#1e1e1e", fg="#d4d4d4", - insertbackground="white") - self.text_train_log.grid(row=row, column=0, columnspan=3, sticky="ew") - scroll = ttk.Scrollbar(f, orient="vertical", command=self.text_train_log.yview) - scroll.grid(row=row, column=3, sticky="ns") - self.text_train_log["yscrollcommand"] = scroll.set - - # ---- GPT 训练浏览 ---- - def _browse_text_file(self): - path = filedialog.askopenfilename(title="选择训练文本文件", - filetypes=[("Text", "*.txt"), ("All", "*.*")]) - if path: - self.text_train_file_var.set(path) - - def _browse_text_save(self): - path = filedialog.asksaveasfilename(title="模型保存路径", defaultextension=".bin", - filetypes=[("Binary", "*.bin"), ("All", "*.*")]) - if path: - self.text_train_save_var.set(path) - - def _browse_text_resume(self): - path = filedialog.askopenfilename(title="选择恢复模型", filetypes=[("Binary", "*.bin"), ("All", "*.*")]) - if path: - self.text_train_resume_path_var.set(path) - - def _toggle_text_resume(self): - state = "normal" if self.text_train_resume_var.get() else "disabled" - self.text_resume_entry.config(state=state) - self.text_resume_btn.config(state=state) - - # ---- GPT 训练 ---- - def _start_text_training(self): - exe = str(TEXT_TRAIN_EXE) - if not Path(exe).exists(): - messagebox.showerror("错误", f"找不到 text_train 可执行文件:\n{exe}\n请先构建项目") - return - - text_file = self.text_train_file_var.get().strip() - if not text_file: - messagebox.showwarning("提示", "请选择训练文本文件") - return - - cmd = [exe, text_file, - "--save", self.text_train_save_var.get(), - "--epochs", str(self.text_train_epochs_var.get()), - "--lr", str(self.text_train_lr_var.get()), - "--batch-size", str(self.text_train_batch_var.get()), - "--seq-len", str(self.text_train_seq_len_var.get()), - "--optimizer", self.text_train_opt_var.get(), - "--d-model", str(self.text_train_d_model_var.get()), - "--num-heads", str(self.text_train_num_heads_var.get()), - "--num-layers", str(self.text_train_num_layers_var.get()), - "--d-ff", str(self.text_train_d_ff_var.get())] - if self.text_train_resume_var.get(): - cmd += ["--resume", self.text_train_resume_path_var.get()] - - self._log_text_train(f"$ {' '.join(cmd)}\n") - self.text_train_start_btn.config(state="disabled") - self.text_train_stop_btn.config(state="normal") - - threading.Thread(target=self._run_process, args=(cmd, self._log_text_train, self._text_train_done), - daemon=True).start() - - def _stop_text_training(self): - if self._process and self._process.poll() is None: - self._process.terminate() - self._log_text_train("\n[已终止]\n") - self._text_train_done() - - def _text_train_done(self): - self.text_train_start_btn.config(state="normal") - self.text_train_stop_btn.config(state="disabled") - - def _log_text_train(self, text: str): - self.text_train_log.config(state="normal") - self.text_train_log.insert("end", text) - self.text_train_log.see("end") - self.text_train_log.config(state="disabled") - - # ============================================================ - # GPT 推理 Tab - # ============================================================ - def _build_text_infer_tab(self): - f = self.text_infer_frame - row = 0 - - # 模型路径 - ttk.Label(f, text="模型文件:").grid(row=row, column=0, sticky="w") - self.text_infer_model_var = tk.StringVar(value=str(DEFAULT_GPT_MODEL)) - ttk.Entry(f, textvariable=self.text_infer_model_var, width=48).grid(row=row, column=1, padx=4) - ttk.Button(f, text="浏览…", command=self._browse_text_infer_model).grid(row=row, column=2) - row += 1 - - # 生成参数 - param_frame = ttk.LabelFrame(f, text="生成参数", padding=6) - param_frame.grid(row=row, column=0, columnspan=3, sticky="ew", pady=4) - row += 1 - - ttk.Label(param_frame, text="最大 token 数:").grid(row=0, column=0, sticky="w") - self.text_infer_max_tokens_var = tk.IntVar(value=200) - ttk.Spinbox(param_frame, from_=1, to=2048, width=6, - textvariable=self.text_infer_max_tokens_var).grid(row=0, column=1, padx=(0, 16)) - - ttk.Label(param_frame, text="温度:").grid(row=0, column=2, sticky="w") - self.text_infer_temp_var = tk.DoubleVar(value=0.8) - ttk.Spinbox(param_frame, from_=0.1, to=2.0, increment=0.1, width=6, - textvariable=self.text_infer_temp_var, format="%.1f").grid(row=0, column=3, padx=(0, 16)) - row += 1 - - # 模型架构（需与训练时一致） - arch_frame = ttk.LabelFrame(f, text="模型架构（需与训练时一致）", padding=6) - arch_frame.grid(row=row, column=0, columnspan=3, sticky="ew", pady=4) - row += 1 - - ttk.Label(arch_frame, text="模型维度:").grid(row=0, column=0, sticky="w") - self.text_infer_d_model_var = tk.IntVar(value=128) - ttk.Spinbox(arch_frame, from_=32, to=512, width=6, - textvariable=self.text_infer_d_model_var).grid(row=0, column=1, padx=(0, 16)) - - ttk.Label(arch_frame, text="注意力头:").grid(row=0, column=2, sticky="w") - self.text_infer_num_heads_var = tk.IntVar(value=4) - ttk.Spinbox(arch_frame, from_=1, to=16, width=6, - textvariable=self.text_infer_num_heads_var).grid(row=0, column=3, padx=(0, 16)) - - ttk.Label(arch_frame, text="层数:").grid(row=0, column=4, sticky="w") - self.text_infer_num_layers_var = tk.IntVar(value=4) - ttk.Spinbox(arch_frame, from_=1, to=16, width=6, - textvariable=self.text_infer_num_layers_var).grid(row=0, column=5) - - ttk.Label(arch_frame, text="FFN 维度:").grid(row=1, column=0, sticky="w", pady=(6, 0)) - self.text_infer_d_ff_var = tk.IntVar(value=512) - ttk.Spinbox(arch_frame, from_=64, to=2048, width=6, - textvariable=self.text_infer_d_ff_var).grid(row=1, column=1, pady=(6, 0), padx=(0, 16)) - - ttk.Label(arch_frame, text="序列长度:").grid(row=1, column=2, sticky="w", pady=(6, 0)) - self.text_infer_seq_len_var = tk.IntVar(value=256) - ttk.Spinbox(arch_frame, from_=16, to=1024, width=6, - textvariable=self.text_infer_seq_len_var).grid(row=1, column=3, pady=(6, 0)) - row += 1 - - # 输入提示 - ttk.Label(f, text="输入提示:").grid(row=row, column=0, sticky="w") - self.text_infer_prompt_var = tk.StringVar(value="Hello") - prompt_entry = ttk.Entry(f, textvariable=self.text_infer_prompt_var, width=48) - prompt_entry.grid(row=row, column=1, padx=4) - prompt_entry.bind("", lambda e: self._start_text_infer()) - row += 1 - - # 按钮 - btn_frame = ttk.Frame(f) - btn_frame.grid(row=row, column=0, columnspan=3, pady=4) - row += 1 - - self.text_infer_start_btn = ttk.Button(btn_frame, text="▶ 生成文本", command=self._start_text_infer) - self.text_infer_start_btn.pack(side="left", padx=4) - row += 1 - - # 输出区域 - output_frame = ttk.LabelFrame(f, text="生成结果", padding=6) - output_frame.grid(row=row, column=0, columnspan=3, sticky="ew", pady=4) - row += 1 - - self.text_infer_output = tk.Text(output_frame, height=12, width=72, state="disabled", - font=("Consolas", 10), bg="#1e1e1e", fg="#d4d4d4", - wrap="word") - self.text_infer_output.pack(fill="both", expand=True) - - # 日志 - self.text_infer_log = tk.Text(f, height=6, width=72, state="disabled", - font=("Consolas", 9), bg="#1e1e1e", fg="#d4d4d4") - self.text_infer_log.grid(row=row, column=0, columnspan=3, sticky="ew") - scroll = ttk.Scrollbar(f, orient="vertical", command=self.text_infer_log.yview) - scroll.grid(row=row, column=3, sticky="ns") - self.text_infer_log["yscrollcommand"] = scroll.set - - def _browse_text_infer_model(self): - path = filedialog.askopenfilename(title="选择 GPT 模型文件", - filetypes=[("Binary", "*.bin"), ("All", "*.*")]) - if path: - self.text_infer_model_var.set(path) - - # ---- GPT 推理 ---- - def _start_text_infer(self): - exe = str(TEXT_INFER_EXE) - if not Path(exe).exists(): - messagebox.showerror("错误", f"找不到 text_infer 可执行文件:\n{exe}\n请先构建项目") - return - - prompt = self.text_infer_prompt_var.get().strip() - if not prompt: - messagebox.showwarning("提示", "请输入提示文本") - return - - cmd = [exe, - "--model", self.text_infer_model_var.get(), - "--prompt", prompt, - "--max-tokens", str(self.text_infer_max_tokens_var.get()), - "--temperature", str(self.text_infer_temp_var.get()), - "--d-model", str(self.text_infer_d_model_var.get()), - "--num-heads", str(self.text_infer_num_heads_var.get()), - "--num-layers", str(self.text_infer_num_layers_var.get()), - "--d-ff", str(self.text_infer_d_ff_var.get()), - "--seq-len", str(self.text_infer_seq_len_var.get())] - - self._log_text_infer(f"$ {' '.join(cmd)}\n") - self.text_infer_start_btn.config(state="disabled") - - # 清空输出 - self.text_infer_output.config(state="normal") - self.text_infer_output.delete("1.0", "end") - self.text_infer_output.config(state="disabled") - - def _run(): - try: - result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) - self.after(0, lambda: self._log_text_infer(result.stdout)) - if result.stderr: - self.after(0, lambda: self._log_text_infer(result.stderr)) - # 解析输出，提取生成的文本 - self.after(0, lambda: self._show_text_output(result.stdout, prompt)) - except FileNotFoundError: - self.after(0, lambda: self._log_text_infer("错误: 可执行文件未找到\n")) - except subprocess.TimeoutExpired: - self.after(0, lambda: self._log_text_infer("错误: 生成超时\n")) - except Exception as e: - self.after(0, lambda: self._log_text_infer(f"错误: {e}\n")) - finally: - self.after(0, lambda: self.text_infer_start_btn.config(state="normal")) - - threading.Thread(target=_run, daemon=True).start() - - def _show_text_output(self, output: str, prompt: str): - """解析 text_infer 输出并显示""" - # text_infer 输出格式: prompt + generated_text - lines = output.strip().split("\n") - # 找到生成的文本行（通常是第一行非空输出） - generated = "" - for line in lines: - if line.strip() and not line.startswith("[") and not line.startswith("加载"): - generated = line - break - - self.text_infer_output.config(state="normal") - self.text_infer_output.delete("1.0", "end") - self.text_infer_output.insert("1.0", generated if generated else output) - self.text_infer_output.config(state="disabled") - - def _log_text_infer(self, text: str): - def _do(): - self.text_infer_log.config(state="normal") - self.text_infer_log.insert("end", text) - self.text_infer_log.see("end") - self.text_infer_log.config(state="disabled") - self.after(0, _do) - - # ============================================================ - # 图片查看 Tab - # ============================================================ - def _build_viewer_tab(self): - f = self.viewer_frame - row = 0 - - ttk.Label(f, text="CSV 文件:").grid(row=row, column=0, sticky="w") - self.viewer_csv_var = tk.StringVar() - ttk.Entry(f, textvariable=self.viewer_csv_var, width=52).grid(row=row, column=1, padx=4) - ttk.Button(f, text="浏览…", command=self._browse_viewer_csv).grid(row=row, column=2) - ttk.Button(f, text="显示", command=self._show_viewer_image).grid(row=row, column=3, padx=4) - row += 1 - - # 目录浏览 - ttk.Label(f, text="或选择目录:").grid(row=row, column=0, sticky="w") - self.viewer_dir_var = tk.StringVar() - ttk.Entry(f, textvariable=self.viewer_dir_var, width=52).grid(row=row, column=1, padx=4) - ttk.Button(f, text="浏览…", command=self._browse_viewer_dir).grid(row=row, column=2) - ttk.Button(f, text="加载", command=self._load_viewer_dir).grid(row=row, column=3, padx=4) - row += 1 - - # 画布 - canvas_size = IMG_DIM * PIXEL_SIZE - self.viewer_canvas = tk.Canvas(f, width=canvas_size, height=canvas_size, bg="black") - self.viewer_canvas.grid(row=row, column=0, columnspan=4, pady=8) - row += 1 - - # 导航 - nav_frame = ttk.Frame(f) - nav_frame.grid(row=row, column=0, columnspan=4) - self.viewer_prev_btn = ttk.Button(nav_frame, text="◀ 上一张", command=self._viewer_prev, state="disabled") - self.viewer_prev_btn.pack(side="left", padx=4) - self.viewer_label = ttk.Label(nav_frame, text="— / —", font=("Segoe UI", 10)) - self.viewer_label.pack(side="left", padx=8) - self.viewer_next_btn = ttk.Button(nav_frame, text="下一张 ▶", command=self._viewer_next, state="disabled") - self.viewer_next_btn.pack(side="left", padx=4) - - self._viewer_files: list[Path] = [] - self._viewer_idx: int = 0 - - def _browse_viewer_csv(self): - path = filedialog.askopenfilename(title="选择 CSV", filetypes=[("CSV", "*.csv")]) - if path: - self.viewer_csv_var.set(path) - self._viewer_files = [Path(path)] - self._viewer_idx = 0 - self._show_viewer_image() - - def _browse_viewer_dir(self): - path = filedialog.askdirectory(title="选择图片目录") - if path: - self.viewer_dir_var.set(path) - self._load_viewer_dir() - - def _load_viewer_dir(self): - d = self.viewer_dir_var.get().strip() - if not d or not Path(d).is_dir(): - return - self._viewer_files = sorted(Path(d).glob("*.csv")) - self._viewer_idx = 0 - self._update_viewer_nav() - if self._viewer_files: - self._show_viewer_image() - - def _show_viewer_image(self): - if not self._viewer_files: - # 尝试从单文件路径加载 - p = self.viewer_csv_var.get().strip() - if p and Path(p).is_file(): - self._viewer_files = [Path(p)] - self._viewer_idx = 0 + self.process = None + done_fn() + + threading.Thread(target=_target, daemon=True).start() + + def stop(self): + if self.process and self.process.poll() is None: + self.process.terminate() + +class FilePicker(ttk.Frame): + """通用文件/目录选择器""" + def __init__(self, parent, label, default="", mode="file", filetypes=None): + super().__init__(parent) + self.var = tk.StringVar(value=default) + self.mode = mode + self.filetypes = filetypes or [("All", "*.*")] + ttk.Label(self, text=label).grid(row=0, column=0, sticky="w", padx=(0, 5)) + ttk.Entry(self, textvariable=self.var, width=40).grid(row=0, column=1, sticky="ew", padx=5) + ttk.Button(self, text="浏览…", command=self._browse).grid(row=0, column=2) + self.columnconfigure(1, weight=1) + + def _browse(self): + path = filedialog.askdirectory() if self.mode == "dir" else filedialog.askopenfilename(filetypes=self.filetypes) + if path: self.var.set(path) + + def get(self): return self.var.get() + +class LogConsole(ttk.Frame): + """通用黑底日志面板""" + def __init__(self, parent, height=8): + super().__init__(parent) + self.text = tk.Text(self, height=height, state="disabled", bg="#1e1e1e", fg="#d4d4d4", font=("Consolas", 9)) + self.scroll = ttk.Scrollbar(self, orient="vertical", command=self.text.yview) + self.text["yscrollcommand"] = self.scroll.set + self.text.pack(side="left", fill="both", expand=True) + self.scroll.pack(side="right", fill="y") + + def append(self, text): + self.text.config(state="normal") + self.text.insert("end", text) + self.text.see("end") + self.text.config(state="disabled") + +class ParamForm(ttk.Frame): + """数据驱动表单：用字典配置生成 UI，消灭几百行的 Spinbox 代码""" + def __init__(self, parent, fields, cols=3): + super().__init__(parent) + self.vars = {} + for i, f in enumerate(fields): + r, c = divmod(i, cols) + name, label, f_type = f["name"], f.get("label", f["name"]), f.get("type", "str") + ttk.Label(self, text=f"{label}:").grid(row=r, column=c*2, sticky="w", padx=5, pady=2) + + if f_type == "int": + v = tk.IntVar(value=f.get("default", 0)) + w = ttk.Spinbox(self, from_=f.get("min", 0), to=f.get("max", 9999), textvariable=v, width=8) + elif f_type == "float": + v = tk.DoubleVar(value=f.get("default", 0.0)) + w = ttk.Spinbox(self, from_=f.get("min", 0.0), to=f.get("max", 1.0), + increment=f.get("step", 0.01), textvariable=v, width=8, format="%.4f") + elif f_type == "combo": + v = tk.StringVar(value=f.get("default", "")) + w = ttk.Combobox(self, textvariable=v, values=f["values"], state="readonly", width=10) + elif f_type == "bool": + v = tk.BooleanVar(value=f.get("default", False)) + w = ttk.Checkbutton(self, variable=v) else: - return - path = self._viewer_files[self._viewer_idx] - try: - pixels = load_csv_pixels(str(path)) - draw_digit(self.viewer_canvas, pixels) - self._update_viewer_nav() - except Exception as e: - messagebox.showerror("错误", str(e)) - - def _viewer_prev(self): - if self._viewer_idx > 0: - self._viewer_idx -= 1 - self._show_viewer_image() - - def _viewer_next(self): - if self._viewer_idx < len(self._viewer_files) - 1: - self._viewer_idx += 1 - self._show_viewer_image() - - def _update_viewer_nav(self): - n = len(self._viewer_files) - self.viewer_label.config(text=f"{self._viewer_idx + 1} / {n}" if n else "— / —") - state_n = "normal" if self._viewer_idx < n - 1 else "disabled" - state_p = "normal" if self._viewer_idx > 0 else "disabled" - self.viewer_next_btn.config(state=state_n) - self.viewer_prev_btn.config(state=state_p) - - # ============================================================ - # 通用 - # ============================================================ - def _run_process(self, cmd: list[str], log_fn, done_fn): - try: - self._process = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True, bufsize=1, creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0) - assert self._process.stdout is not None - for line in self._process.stdout: - self.after(0, lambda t=line: log_fn(t)) - self._process.wait() - except FileNotFoundError: - self.after(0, lambda: log_fn("错误: 可执行文件未找到\n")) - except Exception as e: - self.after(0, lambda: log_fn(f"错误: {e}\n")) - finally: - self._process = None - self.after(0, done_fn) - - def _on_close(self): - if self._process and self._process.poll() is None: - self._process.terminate() - self.destroy() + v = tk.StringVar(value=f.get("default", "")) + w = ttk.Entry(self, textvariable=v, width=15) + + w.grid(row=r, column=c*2+1, sticky="w", padx=5, pady=2) + self.vars[name] = v + + def get(self): return {k: v.get() for k, v in self.vars.items()} + +# ═══════════════════════════════════════════════════════ +# 2. 原有优秀组件保留 (LiveChart, ScrollableFrame) +# ═══════════════════════════════════════════════════════ + +class ScrollableFrame(ttk.Frame): + def __init__(self, parent, **kw): + super().__init__(parent, **kw) + self._canvas = tk.Canvas(self, highlightthickness=0) + self._vsb = ttk.Scrollbar(self, orient="vertical", command=self._canvas.yview) + self._canvas.configure(yscrollcommand=self._vsb.set) + self._canvas.pack(side="left", fill="both", expand=True) + self._vsb.pack(side="right", fill="y") + self.content = ttk.Frame(self._canvas) + self._win = self._canvas.create_window((0, 0), window=self.content, anchor="nw") + self.content.bind("", lambda e: self._canvas.configure(scrollregion=self._canvas.bbox("all"))) + self._canvas.bind("", lambda e: self._canvas.itemconfigure(self._win, width=e.width)) + self._canvas.bind("", lambda e: self._canvas.bind_all("", self._on_wheel)) + self._canvas.bind("", lambda e: self._canvas.unbind_all("")) + + def _on_wheel(self, event): + self._canvas.yview_scroll(int(-event.delta / 120), "units") + +class LiveChart: + """实时训练曲线图（Canvas 绘制，无需 matplotlib）""" + def __init__(self, parent, height=200, bg="#1e1e1e"): + self.canvas = tk.Canvas(parent, height=height, bg=bg, highlightthickness=1, highlightbackground="#333") + self.canvas.pack(fill="both", expand=True) + self.series = {} # {name: [(x,y), ...]} + self.colors = {"loss": "#ff6b6b", "acc": "#4ecdc4", "val_loss": "#ffd93d", "val_acc": "#6bcb77"} + self.x_min, self.x_max = 0, 100 + self.y_min, self.y_max = 0, 1 + + def clear(self): + self.series.clear() + self.canvas.delete("all") + + def add_point(self, series_name, x, y): + if series_name not in self.series: + self.series[series_name] = [] + self.series[series_name].append((x, y)) + self._redraw() + + def _redraw(self): + self.canvas.delete("plot") + if not self.series: + return + + # 自动调整 Y 轴范围 + all_y = [y for pts in self.series.values() for _, y in pts] + if all_y: + self.y_min, self.y_max = min(all_y) * 0.95, max(all_y) * 1.05 + if self.y_min == self.y_max: + self.y_min -= 0.1 + self.y_max += 0.1 + + # 自动调整 X 轴范围 + all_x = [x for pts in self.series.values() for x, _ in pts] + if all_x: + self.x_max = max(all_x) + 5 + + width = int(self.canvas.cget("width")) or 400 + height = int(self.canvas.cget("height")) or 200 + pad_left, pad_right = 50, 20 + pad_top, pad_bot = 20, 30 + plot_w = width - pad_left - pad_right + plot_h = height - pad_top - pad_bot + + # 绘制坐标轴 + self.canvas.create_line(pad_left, pad_top, pad_left, height - pad_bot, fill="#555", width=1, tags="plot") + self.canvas.create_line(pad_left, height - pad_bot, width - pad_right, height - pad_bot, fill="#555", width=1, tags="plot") + + # 绘制网格和标签 + for i in range(5): + y_pos = pad_top + i * plot_h / 4 + val = self.y_max - i * (self.y_max - self.y_min) / 4 + self.canvas.create_line(pad_left, y_pos, width - pad_right, y_pos, dash=(2, 2), fill="#333", tags="plot") + self.canvas.create_text(pad_left - 5, y_pos, text=f"{val:.2f}", anchor="e", fill="#888", font=("Consolas", 7), tags="plot") + + # 绘制各条曲线 + for name, points in self.series.items(): + color = self.colors.get(name, "#fff") + if len(points) < 2: + continue + coords = [] + for x, y in points: + px = pad_left + (x - self.x_min) / (self.x_max - self.x_min + 1e-6) * plot_w + py = pad_top + (self.y_max - y) / (self.y_max - self.y_min + 1e-6) * plot_h + coords.extend([px, py]) + self.canvas.create_line(coords, fill=color, width=2, tags="plot") + + # 图例 + legend_x = width - pad_right - 80 + legend_y = pad_top + 10 + for i, (name, color) in enumerate(self.colors.items()): + if name in self.series: + self.canvas.create_line(legend_x, legend_y + i * 15, legend_x + 20, legend_y + i * 15, fill=color, width=2, tags="plot") + self.canvas.create_text(legend_x + 25, legend_y + i * 15, text=name, anchor="w", fill="#ccc", font=("Consolas", 8), tags="plot") + +# ═══════════════════════════════════════════════════════ +# 3. 主 GUI 类 (利用组件大幅瘦身) +# ═══════════════════════════════════════════════════════ +class NeuralNetGUI(tk.Tk): + def __init__(self): + super().__init__() + self.title("neuralnet.cpp GUI (重构版)") + self.geometry("900x750") + ttk.Style(self).theme_use("clam") + + self.runners = {k: TaskRunner() for k in EXES} # 为每个任务分配独立的 Runner + + nb = ttk.Notebook(self) + nb.pack(fill="both", expand=True, padx=8, pady=8) + + # 挂载各个 Tab + self._build_go_tab(nb) + self._build_mnist_tab(nb) + self._build_gpt_tab(nb) + + def _build_go_tab(self, nb): + f = ScrollableFrame(nb) + nb.add(f, text=" ♟️ 围棋 ") + if GoGamePanel: GoGamePanel(f.content) + + def _build_mnist_tab(self, nb): + mnist_nb = ttk.Notebook(nb) + nb.add(mnist_nb, text=" 🧠 MNIST ") + + # --- 训练 Tab --- + train_f = ScrollableFrame(mnist_nb) + mnist_nb.add(train_f, text=" 训练 ") + c = train_f.content + + # 1. 文件区 (使用 FilePicker，代码从 20行 -> 2行) + self.m_tr_data = FilePicker(c, "数据集:", mode="dir") + self.m_tr_data.pack(fill="x", padx=5, pady=2) + self.m_tr_save = FilePicker(c, "保存模型:", default="mnist_model.bin") + self.m_tr_save.pack(fill="x", padx=5, pady=2) + + # 2. 参数区 (使用 ParamForm，代码从 60行 -> 10行) + param_fields = [ + {"name": "epochs", "label": "轮数", "type": "int", "default": 5, "min": 1, "max": 1000}, + {"name": "lr", "label": "学习率", "type": "float", "default": 0.01, "min": 0.0001, "max": 1.0, "step": 0.001}, + {"name": "batch", "label": "Batch", "type": "int", "default": 64, "min": 1, "max": 4096}, + {"name": "opt", "label": "优化器", "type": "combo", "values": ["adam", "sgd", "adamw"], "default": "adam"}, + {"name": "arch", "label": "架构", "type": "combo", "values": ["mlp", "transformer"], "default": "mlp"} + ] + self.m_tr_params = ParamForm(c, param_fields, cols=3) + self.m_tr_params.pack(fill="x", padx=5, pady=5) + + # 3. 控制与日志 + btn_f = ttk.Frame(c) + btn_f.pack(pady=5) + self.m_tr_start = ttk.Button(btn_f, text="▶ 开始训练", command=lambda: self._start_task("mnist_train")) + self.m_tr_start.pack(side="left", padx=5) + self.m_tr_stop = ttk.Button(btn_f, text="⏹ 停止", command=lambda: self._stop_task("mnist_train"), state="disabled") + self.m_tr_stop.pack(side="left", padx=5) + + self.m_tr_log = LogConsole(c, height=8) + self.m_tr_log.pack(fill="both", expand=True, padx=5, pady=5) + + self.m_tr_chart = LiveChart(c, height=150) + self.m_tr_chart.canvas.pack(fill="x", padx=5, pady=5) + + # --- 推理 Tab (省略，结构同上) --- + infer_f = ScrollableFrame(mnist_nb) + mnist_nb.add(infer_f, text=" 推理 ") + # ... 推理 Tab 逻辑 ... + + def _build_gpt_tab(self, nb): + gpt_nb = ttk.Notebook(nb) + nb.add(gpt_nb, text=" 📝 GPT ") + # ... 同样使用 FilePicker 和 ParamForm 构建 ... + + # ═══════════════════════════════════════════════════════ + # 4. 统一的业务逻辑控制 (消灭重复的 _start_xxx 方法) + # ═══════════════════════════════════════════════════════ + + def _start_task(self, task_name): + runner = self.runners[task_name] + params = self.m_tr_params.get() # 获取表单数据 + + # 动态拼接命令行 + cmd = [str(EXES[task_name]), + "--dataset", self.m_tr_data.get(), + "--save", self.m_tr_save.get(), + "--epochs", str(params["epochs"]), + "--lr", str(params["lr"])] + + self.m_tr_log.text.delete("1.0", "end") + self.m_tr_start.config(state="disabled") + self.m_tr_stop.config(state="normal") + self.m_tr_chart.clear() + + runner.run( + cmd, + log_fn=self._parse_and_log, + done_fn=lambda: self._task_done(task_name), + exe_name=task_name + ) + + def _stop_task(self, task_name): + self.runners[task_name].stop() + self.m_tr_log.append("\n[已手动终止]\n") + self._task_done(task_name) + + def _task_done(self, task_name): + self.m_tr_start.config(state="normal") + self.m_tr_stop.config(state="disabled") + + def _parse_and_log(self, text): + self.m_tr_log.append(text) + # 保留您原来的正则解析逻辑，更新图表 + m = re.search(r"loss:\s*([0-9.eE\-+]+)", text) + if m: + # self.m_tr_chart.add_point(...) + pass -# ── 入口 ────────────────────────────────────────────── if __name__ == "__main__": app = NeuralNetGUI() - app.mainloop() + app.mainloop() \ No newline at end of file diff --git a/include/neuralnet.cpp/algebra_compute.hpp b/include/neuralnet.cpp/algebra_compute.hpp new file mode 100644 index 00000000..3601a209 --- /dev/null +++ b/include/neuralnet.cpp/algebra_compute.hpp @@ -0,0 +1,54 @@ +#ifndef NN_ALGEBRA_COMPUTE_DISPATCH_HPP +#define NN_ALGEBRA_COMPUTE_DISPATCH_HPP + +// ── compute_dispatch.hpp — 表达式模板统一入口 ───────────────────────────── +// compute::apply(span, expr) 是所有逐元素运算的统一入口。 +// +// CPU 路径：递归求值 AST，编译器内联所有 eval() 调用， +// 消除 AST 节点内存分配（全部栈上值类型），向量化最终循环。 +// 实测性能与手写 for 循环完全等价（零开销抽象）。 +// ───────────────────────────────────────────────────────────────────────── + +#include +#include + +#include "config.hpp" +#include "algebra_span.hpp" + +namespace nn::compute +{ + +// ══════════════════════════════════════════════════════════════════════════ +// apply — 统一入口：对 span 中的每个元素应用表达式 +// ══════════════════════════════════════════════════════════════════════════ +// +// 用法示例： +// Span x = matrix.span(); +// compute::apply(x, max(x, Scalar{0})); // ReLU +// compute::apply(x, x * (1.0f / (1.0f + exp(-1.702f * x)))); // GeLU +// compute::apply(x, select(x > Scalar{0}, x, x * 0.01f)); // LeakyReLU +// compute::apply(x, abs(x)); // abs +// compute::apply(x, x * 2.0f + 1.0f); // 线性变换 +// +/// @param x 输出目标（Span 视图，指向 Matrix 内部数据） +/// @param expr 表达式树（编译期 AST，由运算符重载自动构建） +/// +/// @note x 和 expr 中的 Span 可以指向同一内存（就地操作安全）， +/// 因为每个元素在读取后才被写入。 +template +void apply(Span x, const Expr &expr) +{ + const auto n = x.size(); + if (n == 0) return; + + auto indices = std::views::iota(std::size_t{0}, n); + nn::for_each(indices.begin(), indices.end(), + [&x, &expr](std::size_t i) noexcept + { + x[i] = static_cast(expr.eval(i)); + }); +} + +} // namespace nn::compute + +#endif // NN_ALGEBRA_COMPUTE_DISPATCH_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/algebra_expr.hpp b/include/neuralnet.cpp/algebra_expr.hpp new file mode 100644 index 00000000..604353f0 --- /dev/null +++ b/include/neuralnet.cpp/algebra_expr.hpp @@ -0,0 +1,445 @@ +#ifndef NN_ALGEBRA_EXPR_HPP +#define NN_ALGEBRA_EXPR_HPP + +// ── expr.hpp — 表达式模板核心：编译期 AST 节点类型系统 ─────────────────── +// 上层用普通 C++ 运算符写表达式，编译器自动构建编译期 AST。 +// CPU 路径：递归求值 AST（零开销，和手写循环等价）。 +// +// 算法永远在上层，计算永远在底层，中间靠 AST 传递。 +// ───────────────────────────────────────────────────────────────────────── + +#include +#include + +#include "config.hpp" +#include "algebra_ops.hpp" + +namespace nn +{ + +// ══════════════════════════════════════════════════════════════════════════ +// 表达式概念（C++20 Concepts） +// ══════════════════════════════════════════════════════════════════════════ + +/// 表达式：可按索引求值为 Scalar 的类型 +template +concept Expression = requires(const T &t, std::size_t i) { + { t.eval(i) } -> std::convertible_to; +}; + +/// 布尔表达式：可按索引求值为 bool 的类型（用于条件选择） +template +concept BoolExpression = requires(const T &t, std::size_t i) { + { t.eval(i) } -> std::convertible_to; +}; + +// ══════════════════════════════════════════════════════════════════════════ +// 叶子节点：常量值 +// ══════════════════════════════════════════════════════════════════════════ + +struct Val +{ + Scalar value; + + [[nodiscard]] constexpr Scalar eval(std::size_t /*i*/) const noexcept { return value; } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// 一元表达式 +// ══════════════════════════════════════════════════════════════════════════ + +template +struct UnaryExpr +{ + Child child; + + [[nodiscard]] constexpr auto eval(std::size_t i) const noexcept + { + return Op::apply(child.eval(i)); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// 二元表达式 +// ══════════════════════════════════════════════════════════════════════════ + +template +struct BinaryExpr +{ + Left left; + Right right; + + [[nodiscard]] constexpr auto eval(std::size_t i) const noexcept + { + return Op::apply(left.eval(i), right.eval(i)); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// 三元表达式（条件选择：cond ? then_expr : else_expr） +// ══════════════════════════════════════════════════════════════════════════ + +template +struct TernaryExpr +{ + Cond cond; + Then then_expr; + Else else_expr; + + [[nodiscard]] constexpr auto eval(std::size_t i) const noexcept + { + return cond.eval(i) ? then_expr.eval(i) : else_expr.eval(i); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// 自由函数：一元数学函数 +// ══════════════════════════════════════════════════════════════════════════ + +/// 取反：-x +template +[[nodiscard]] constexpr auto neg(const Expr &expr) +{ + return UnaryExpr{expr}; +} + +/// 绝对值：|x| +template +[[nodiscard]] auto abs(const Expr &expr) +{ + return UnaryExpr{expr}; +} + +/// 指数：e^x +template +[[nodiscard]] auto exp(const Expr &expr) +{ + return UnaryExpr{expr}; +} + +/// 自然对数：ln(x) +template +[[nodiscard]] auto log(const Expr &expr) +{ + return UnaryExpr{expr}; +} + +/// 平方根：√x +template +[[nodiscard]] auto sqrt(const Expr &expr) +{ + return UnaryExpr{expr}; +} + +/// 平方根倒数：1/√x —— LayerNorm 标准差倒数专用 +template +[[nodiscard]] auto rsqrt(const Expr &expr) +{ + return UnaryExpr{expr}; +} + +/// Sigmoid：1 / (1 + e^(-x)) +template +[[nodiscard]] auto sigmoid(const Expr &expr) +{ + return UnaryExpr{expr}; +} + +/// 双曲正切：tanh(x) +template +[[nodiscard]] auto tanh(const Expr &expr) +{ + return UnaryExpr{expr}; +} + +/// ReLU：max(x, 0) +template +[[nodiscard]] auto relu(const Expr &expr) +{ + return UnaryExpr{expr}; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 自由函数：二元值运算 +// ══════════════════════════════════════════════════════════════════════════ + +/// 最大值：max(a, b) +template +[[nodiscard]] constexpr auto max(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +template +[[nodiscard]] constexpr auto max(const Expr &expr, Scalar s) +{ + return BinaryExpr{expr, Val{s}}; +} + +template +[[nodiscard]] constexpr auto max(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +/// 最小值：min(a, b) +template +[[nodiscard]] constexpr auto min(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +template +[[nodiscard]] constexpr auto min(const Expr &expr, Scalar s) +{ + return BinaryExpr{expr, Val{s}}; +} + +template +[[nodiscard]] constexpr auto min(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 自由函数：条件选择 +// ══════════════════════════════════════════════════════════════════════════ + +/// select(cond, then_expr, else_expr) — 等价于 cond ? then : else +/// 用法：select(x > Scalar{0}, x, x * 0.01f) // LeakyReLU +template +[[nodiscard]] constexpr auto select(const Cond &cond, const Then &then_expr, const Else &else_expr) +{ + return TernaryExpr{cond, then_expr, else_expr}; +} + +/// select 重载：else 分支为标量 +/// 用法：select(x > Scalar{0}, go, Scalar{0}) // ReLU 反向 +template +[[nodiscard]] constexpr auto select(const Cond &cond, const Then &then_expr, Scalar else_val) +{ + return TernaryExpr{cond, then_expr, Val{else_val}}; +} + +/// select 重载：then 分支为标量 +template +[[nodiscard]] constexpr auto select(const Cond &cond, Scalar then_val, const Else &else_expr) +{ + return TernaryExpr{cond, Val{then_val}, else_expr}; +} + +/// select 重载：then/else 均为标量 +template +[[nodiscard]] constexpr auto select(const Cond &cond, Scalar then_val, Scalar else_val) +{ + return TernaryExpr{cond, Val{then_val}, Val{else_val}}; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 模板运算符：Scalar op Expression / Expression op Scalar +// 用于复合表达式中的标量与任意表达式类型的运算 +// 例如：1.0f + exp(x) → BinaryExpr, ops::Add> +// +// 注：C++ 模板推导要求 Scalar/Expr 左右位置分别定义，无法合并。 +// 模板实参推导不会自动尝试 (Scalar, Expr) ↔ (Expr, Scalar) 的对称转换， +// 因此 operator+(Scalar, Expr) 和 operator+(Expr, Scalar) 必须分别定义。 +// 下面 22 个运算符重载按 Scalar op Expr、Expr op Scalar、Expr op Expr 三组排列。 +// ══════════════════════════════════════════════════════════════════════════ + +// ── Scalar + Expression ────────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator+(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ── Scalar - Expression ────────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator-(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ── Scalar * Expression ────────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator*(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ── Scalar / Expression ────────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator/(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ── Scalar > Expression → bool 表达式 ──────────────────────────────────── +template +[[nodiscard]] constexpr auto operator>(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ── Scalar < Expression → bool 表达式 ──────────────────────────────────── +template +[[nodiscard]] constexpr auto operator<(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ── Scalar >= Expression → bool 表达式 ─────────────────────────────────── +template +[[nodiscard]] constexpr auto operator>=(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ── Scalar <= Expression → bool 表达式 ─────────────────────────────────── +template +[[nodiscard]] constexpr auto operator<=(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ── Scalar == Expression → bool 表达式 ─────────────────────────────────── +template +[[nodiscard]] constexpr auto operator==(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ── Scalar != Expression → bool 表达式 ─────────────────────────────────── +template +[[nodiscard]] constexpr auto operator!=(Scalar s, const Expr &expr) +{ + return BinaryExpr{Val{s}, expr}; +} + +// ── Expression + Scalar ────────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator+(const Expr &expr, Scalar s) +{ + return BinaryExpr{expr, Val{s}}; +} + +// ── Expression - Scalar ────────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator-(const Expr &expr, Scalar s) +{ + return BinaryExpr{expr, Val{s}}; +} + +// ── Expression * Scalar ────────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator*(const Expr &expr, Scalar s) +{ + return BinaryExpr{expr, Val{s}}; +} + +// ── Expression / Scalar ────────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator/(const Expr &expr, Scalar s) +{ + return BinaryExpr{expr, Val{s}}; +} + +// ── Expression > Scalar → bool 表达式 ──────────────────────────────────── +template +[[nodiscard]] constexpr auto operator>(const Expr &expr, Scalar s) +{ + return BinaryExpr{expr, Val{s}}; +} + +// ── Expression < Scalar → bool 表达式 ──────────────────────────────────── +template +[[nodiscard]] constexpr auto operator<(const Expr &expr, Scalar s) +{ + return BinaryExpr{expr, Val{s}}; +} + +// ── Expression >= Scalar → bool 表达式 ─────────────────────────────────── +template +[[nodiscard]] constexpr auto operator>=(const Expr &expr, Scalar s) +{ + return BinaryExpr{expr, Val{s}}; +} + +// ── Expression <= Scalar → bool 表达式 ─────────────────────────────────── +template +[[nodiscard]] constexpr auto operator<=(const Expr &expr, Scalar s) +{ + return BinaryExpr{expr, Val{s}}; +} + +// ── Expression + Expression ────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator+(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +// ── Expression - Expression ────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator-(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +// ── Expression * Expression ────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator*(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +// ── Expression / Expression ────────────────────────────────────────────── +template +[[nodiscard]] constexpr auto operator/(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Expression 与 Expression 的比较运算（返回 bool 表达式 AST） +// ═══════════════════════════════════════════════════════════════════════════ + +template +[[nodiscard]] constexpr auto operator>(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +template +[[nodiscard]] constexpr auto operator<(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +template +[[nodiscard]] constexpr auto operator>=(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +template +[[nodiscard]] constexpr auto operator<=(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +template +[[nodiscard]] constexpr auto operator==(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +template +[[nodiscard]] constexpr auto operator!=(const L &l, const R &r) +{ + return BinaryExpr{l, r}; +} + +} // namespace nn + +#endif // NN_ALGEBRA_EXPR_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/algebra_matrix.hpp b/include/neuralnet.cpp/algebra_matrix.hpp new file mode 100644 index 00000000..b331a97f --- /dev/null +++ b/include/neuralnet.cpp/algebra_matrix.hpp @@ -0,0 +1,1021 @@ +#ifndef NN_ALGEBRA_MATRIX_HPP +#define NN_ALGEBRA_MATRIX_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core_errors.hpp" +#include "core_assert.hpp" +#include "core_threadpool.hpp" +#include "config.hpp" +#include "algebra_span.hpp" +#include "algebra_compute.hpp" + +// GPU 加速支持（可选，由 CMake 的 NN_HAS_VULKAN 宏控制） +#ifdef NN_HAS_VULKAN +#include "backend/vk_backend.hpp" +#endif + +namespace nn +{ + // ═══════════════════════════════════════════════════════════════════════ + // Matrix 类 — 矩阵存储与运算原语 + // + // 错误处理约定（与上层 L2 计算层不同）： + // - L1 代数层使用 NN_ASSERT 进行形状校验（编程错误检查） + // - L2+ 计算层使用 Result 进行运行时错误处理（用户输入校验） + // - 原因：Matrix 是底层原语，assert 不影响 Release 性能； + // 上层需要向用户报告错误，故使用 Result + // - 参见 DEVELOPMENT_STANDARDS.md "分层职责单一" 章节 + // ═══════════════════════════════════════════════════════════════════════ + class Matrix + { + private: + std::vector data_{}; + std::size_t rows_{0}; + std::size_t cols_{0}; + + [[nodiscard]] constexpr std::size_t index(std::size_t row, std::size_t col) const noexcept + { + return row * cols_ + col; + } + + static void require_same_shape(const Matrix &lhs, const Matrix &rhs, [[maybe_unused]] std::string_view message) + { + if (lhs.rows_ != rhs.rows_ || lhs.cols_ != rhs.cols_) + { + NN_ASSERT(false, message.data()); + } + } + + public: + Matrix() = default; + + explicit Matrix(std::size_t rows, std::size_t cols) + : data_(rows * cols), rows_(rows), cols_(cols) {} + + // 从标量值初始化矩阵 + Matrix(std::size_t rows, std::size_t cols, Scalar value) + : data_(rows * cols, value), rows_(rows), cols_(cols) {} + + // 拷贝/移动构造与赋值：使用默认实现（vector 已提供强异常安全保证） + Matrix(const Matrix &other) = default; + Matrix(Matrix &&other) noexcept = default; + Matrix &operator=(const Matrix &other) = default; + Matrix &operator=(Matrix &&other) noexcept = default; + ~Matrix() = default; + + // ── 就地调整大小（复用已有内存） ────────────────────────────────── + void resize(std::size_t rows, std::size_t cols) + { + if (rows_ == rows && cols_ == cols) return; // 尺寸不变，零开销 + rows_ = rows; + cols_ = cols; + data_.resize(rows * cols); + } + + // ── std::span 访问（C++20 现代接口，推荐使用） ──────────────────── + // 零开销抽象：编译后等价于裸指针 + 大小，可替代所有 data_ptr() 场景 + [[nodiscard]] std::span span() const + { + return {data_.data(), data_.size()}; + } + [[nodiscard]] std::span span() noexcept { return {data_.data(), data_.size()}; } + + // 访问器 + [[nodiscard]] constexpr std::size_t rows() const noexcept { return rows_; } + [[nodiscard]] constexpr std::size_t cols() const noexcept { return cols_; } + [[nodiscard]] constexpr std::size_t size() const noexcept { return data_.size(); } + [[nodiscard]] constexpr bool empty() const noexcept { return data_.empty(); } + [[nodiscard]] Scalar at(std::size_t row, std::size_t col) const + { + if (row >= rows_ || col >= cols_) + { + NN_ASSERT(false, "Matrix index out of range"); + } + return data_[index(row, col)]; + } + void set_value(std::size_t row, std::size_t col, Scalar value) + { + if (row >= rows_ || col >= cols_) + { + NN_ASSERT(false, "Matrix index out of range"); + } + data_[index(row, col)] = value; + } + [[nodiscard]] constexpr Scalar at_unchecked(std::size_t row, std::size_t col) const noexcept { return data_[index(row, col)]; } // 无校验 + constexpr void set_value_unchecked(std::size_t row, std::size_t col, Scalar value) noexcept { data_[index(row, col)] = value; } // 无校验 + + // ── 转置（返回新矩阵） ───────────────────────────────────────────── + [[nodiscard]] Matrix transpose() const + { + Matrix result(cols_, rows_); + transpose_to(result); + return result; + } + + // ── 转置到预分配缓冲区（零分配热路径） ───────────────────────────── + void transpose_to(Matrix &result) const + { + NN_ASSERT(&result != this, "transpose_to: self-referencing not supported"); + result.resize(cols_, rows_); + if (rows_ == 0 || cols_ == 0) return; + + const auto src = span(); + auto dst = result.span(); + const std::size_t R = rows_; + const std::size_t C = cols_; + + const std::size_t i_blocks = (R + BLOCK_SIZE - 1) / BLOCK_SIZE; + const std::size_t j_blocks = (C + BLOCK_SIZE - 1) / BLOCK_SIZE; + + // 通过 nn::parallel_for_blocks 分发块级并行：分块数很少（1~几十），但其内循环计算量大。 + const auto n_blocks = i_blocks * j_blocks; + if (n_blocks <= 1) + { + for (std::size_t i = 0; i < R; ++i) + for (std::size_t j = 0; j < C; ++j) + dst[j * R + i] = src[i * C + j]; + } + else + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + nn::parallel_for_blocks( + block_indices.begin(), block_indices.end(), + [src, dst, R, C, j_blocks](std::size_t block_idx) noexcept + { + const std::size_t ib = block_idx / j_blocks; + const std::size_t jb = block_idx % j_blocks; + const std::size_t i0 = ib * BLOCK_SIZE; + const std::size_t j0 = jb * BLOCK_SIZE; + const std::size_t i1 = std::min(i0 + BLOCK_SIZE, R); + const std::size_t j1 = std::min(j0 + BLOCK_SIZE, C); + for (std::size_t i = i0; i < i1; ++i) + for (std::size_t j = j0; j < j1; ++j) + dst[j * R + i] = src[i * C + j]; + }); + } + } + + // ── 基于 span 的矩阵乘法（零拷贝，供 batched_matmul 等场景使用） ── + // 从 a/b 的子区间直接计算，无需构造临时 Matrix 拷贝 + static void multiply_to_span( + std::span r, std::size_t M, std::size_t N, + std::span a, std::size_t /*a_rows*/, std::size_t a_cols, + std::span b, std::size_t b_rows, std::size_t b_cols) + { + NN_ASSERT(a_cols == b_rows, "multiply_to_span: inner dimension mismatch"); + (void)b_rows; // NN_ASSERT 在 Release 模式下展开为空，参数仅用于断言 + const std::size_t K = a_cols; + if (M == 0 || N == 0 || K == 0) return; + + // 直接复用 blocked matmul 内核 + const std::size_t i_blocks = (M + BLOCK_SIZE - 1) / BLOCK_SIZE; + const std::size_t j_blocks = (N + BLOCK_SIZE - 1) / BLOCK_SIZE; + const auto n_blocks = i_blocks * j_blocks; + + auto kernel = [a, b, r, M, N, K, a_cols, b_cols, j_blocks](std::size_t block_idx) noexcept + { + const std::size_t i_block = block_idx / j_blocks; + const std::size_t j_block = block_idx % j_blocks; + const std::size_t i_start = i_block * BLOCK_SIZE; + const std::size_t i_end = std::min(i_start + BLOCK_SIZE, M); + const std::size_t j_start = j_block * BLOCK_SIZE; + const std::size_t j_end = std::min(j_start + BLOCK_SIZE, N); + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = b[(k_start + kk) * b_cols + (j_start + jj)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + for (std::size_t i = i_start; i < i_end; ++i) + { + const auto a_row = a.subspan(i * a_cols + k_start); + auto r_row = r.subspan(i * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + }; + + if (n_blocks <= 1) + kernel(0); + else + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + nn::parallel_for_blocks(block_indices.begin(), block_indices.end(), kernel); + } + } + + // ── 基于 span 的矩阵乘法（B 转置，零拷贝） ───────────────────────── + static void multiply_transposed_to_span( + std::span r, std::size_t M, std::size_t N, + std::span a, std::size_t /*a_rows*/, std::size_t a_cols, + std::span bt, std::size_t /*bt_rows*/, std::size_t bt_cols) + { + NN_ASSERT(a_cols == bt_cols, "multiply_transposed_to_span: inner dimension mismatch"); + (void)bt_cols; // NN_ASSERT 在 Release 模式下展开为空，参数仅用于断言 + const std::size_t K = a_cols; + if (M == 0 || N == 0 || K == 0) return; + + const std::size_t i_blocks = (M + BLOCK_SIZE - 1) / BLOCK_SIZE; + const std::size_t j_blocks = (N + BLOCK_SIZE - 1) / BLOCK_SIZE; + const auto n_blocks = i_blocks * j_blocks; + + auto kernel = [a, bt, r, M, N, K, j_blocks](std::size_t block_idx) noexcept + { + const std::size_t i_block = block_idx / j_blocks; + const std::size_t j_block = block_idx % j_blocks; + const std::size_t i_start = i_block * BLOCK_SIZE; + const std::size_t i_end = std::min(i_start + BLOCK_SIZE, M); + const std::size_t j_start = j_block * BLOCK_SIZE; + const std::size_t j_end = std::min(j_start + BLOCK_SIZE, N); + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = bt[(j_start + jj) * K + (k_start + kk)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + for (std::size_t i = i_start; i < i_end; ++i) + { + const auto a_row = a.subspan(i * K + k_start); + auto r_row = r.subspan(i * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + }; + + if (n_blocks <= 1) + kernel(0); + else + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + nn::parallel_for_blocks(block_indices.begin(), block_indices.end(), kernel); + } + } + + // ── 基于 span 的矩阵乘法（A 转置，零拷贝） ───────────────────────── + static void transpose_multiply_to_span( + std::span r, std::size_t M, std::size_t N, + std::span a, std::size_t a_rows, std::size_t a_cols, + std::span b, std::size_t b_rows, std::size_t b_cols) + { + NN_ASSERT(a_rows == b_rows, "transpose_multiply_to_span: inner dimension mismatch"); + (void)b_rows; // NN_ASSERT 在 Release 模式下展开为空，参数仅用于断言 + const std::size_t K = a_rows; // a is (K, M) stored + if (M == 0 || N == 0 || K == 0) return; + + const std::size_t i_blocks = (M + BLOCK_SIZE - 1) / BLOCK_SIZE; + const std::size_t j_blocks = (N + BLOCK_SIZE - 1) / BLOCK_SIZE; + const auto n_blocks = i_blocks * j_blocks; + + auto kernel = [a, b, r, M, N, K, a_cols, b_cols, j_blocks](std::size_t block_idx) noexcept + { + const std::size_t i_block = block_idx / j_blocks; + const std::size_t j_block = block_idx % j_blocks; + const std::size_t i_start = i_block * BLOCK_SIZE; + const std::size_t i_end = std::min(i_start + BLOCK_SIZE, M); + const std::size_t j_start = j_block * BLOCK_SIZE; + const std::size_t j_end = std::min(j_start + BLOCK_SIZE, N); + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + const std::size_t i_len = i_end - i_start; + std::array a_block{}; + for (std::size_t ii = 0; ii < i_len; ++ii) + for (std::size_t kk = 0; kk < k_len; ++kk) + a_block[ii * k_len + kk] = a[(k_start + kk) * a_cols + (i_start + ii)]; + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = b[(k_start + kk) * b_cols + (j_start + jj)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + for (std::size_t i = 0; i < i_len; ++i) + { + const auto a_row = std::span(a_block.data() + i * k_len, k_len); + auto r_row = r.subspan((i_start + i) * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + }; + + if (n_blocks <= 1) + kernel(0); + else + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + nn::parallel_for_blocks(block_indices.begin(), block_indices.end(), kernel); + } + } + + // ── 矩阵乘法到预分配缓冲区（零分配热路径） ───────────────────────── + // 使用 std::span（C++20）提供类型安全的非拥有视图 + void multiply_to(Matrix &result, const Matrix &other) const + { + NN_ASSERT(&result != this, "multiply_to: self-referencing not supported"); + const std::size_t M = rows_; + const std::size_t N = other.cols_; + const std::size_t K = cols_; + result.resize(M, N); + if (M == 0 || N == 0 || K == 0) return; + + // 清零结果矩阵（使用 RAII 封装的方法） + result.zero(); + + // std::span 非拥有视图：从 std::vector 借出，生命周期由调用栈保证 + const auto a = span(); + const auto b = other.span(); + auto r = result.span(); + + const std::size_t i_blocks = (M + BLOCK_SIZE - 1) / BLOCK_SIZE; + const std::size_t j_blocks = (N + BLOCK_SIZE - 1) / BLOCK_SIZE; + + // 通过 nn::parallel_for_blocks 分发块级并行：分块数可能很少但其内循环计算量巨大（三层循环 O(n³)）。 + const auto n_blocks = i_blocks * j_blocks; + if (n_blocks <= 1) + { + // 单块串行——与下方 lambda 逻辑相同，但避免不必要的线程池调用 + const std::size_t i_start = 0, i_end = M; + const std::size_t j_start = 0, j_end = N; + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = b[(k_start + kk) * N + (j_start + jj)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + for (std::size_t i = i_start; i < i_end; ++i) + { + const auto a_row = a.subspan(i * K + k_start); + auto r_row = r.subspan(i * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; +#pragma clang loop vectorize(assume_safety) + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + } + else + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + nn::parallel_for_blocks( + block_indices.begin(), block_indices.end(), + [a, b, r, M, N, K, j_blocks](std::size_t block_idx) noexcept + { + const std::size_t i_block = block_idx / j_blocks; + const std::size_t j_block = block_idx % j_blocks; + const std::size_t i_start = i_block * BLOCK_SIZE; + const std::size_t i_end = std::min(i_start + BLOCK_SIZE, M); + const std::size_t j_start = j_block * BLOCK_SIZE; + const std::size_t j_end = std::min(j_start + BLOCK_SIZE, N); + + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = b[(k_start + kk) * N + (j_start + jj)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + + for (std::size_t i = i_start; i < i_end; ++i) + { + const auto a_row = a.subspan(i * K + k_start); + auto r_row = r.subspan(i * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; +#pragma clang loop vectorize(assume_safety) + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + }); + } + } + + // ── 矩阵乘法（B 转置）到预分配缓冲区 ───────────────────────── + // 计算 result = this * B^T，其中 B 存储为 (N, K) 行主序 + // 无需实际转置 B，直接从 B 的行读取列（节省 O(N*K) 的拷贝） + // + // 维度要求：this=(M,K), b_trans=(N,K) → result=(M,N) + // 即 C[m][n] = Σ_k A[m][k] * B[n][k] + void multiply_transposed_to(Matrix &result, const Matrix &b_trans) const + { + NN_ASSERT(&result != this && &result != &b_trans, "multiply_transposed_to: self-referencing not supported"); + NN_ASSERT(cols_ == b_trans.cols_, "multiply_transposed_to: inner dimensions mismatch"); + const std::size_t M = rows_; + const std::size_t K = cols_; + const std::size_t N = b_trans.rows_; + result.resize(M, N); + if (M == 0 || N == 0 || K == 0) return; + + result.zero(); + + const auto a = span(); + const auto bt = b_trans.span(); + auto r = result.span(); + + const std::size_t i_blocks = (M + BLOCK_SIZE - 1) / BLOCK_SIZE; + const std::size_t j_blocks = (N + BLOCK_SIZE - 1) / BLOCK_SIZE; + const auto n_blocks = i_blocks * j_blocks; + + if (n_blocks <= 1) + { + const std::size_t i_start = 0, i_end = M; + const std::size_t j_start = 0, j_end = N; + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + // B^T 块加载：B[n][k] 从 bt[n * K + k] 读取 + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = bt[(j_start + jj) * K + (k_start + kk)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + for (std::size_t i = i_start; i < i_end; ++i) + { + const auto a_row = a.subspan(i * K + k_start); + auto r_row = r.subspan(i * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + } + else + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + nn::parallel_for_blocks( + block_indices.begin(), block_indices.end(), + [a, bt, r, M, N, K, j_blocks](std::size_t block_idx) noexcept + { + const std::size_t i_block = block_idx / j_blocks; + const std::size_t j_block = block_idx % j_blocks; + const std::size_t i_start = i_block * BLOCK_SIZE; + const std::size_t i_end = std::min(i_start + BLOCK_SIZE, M); + const std::size_t j_start = j_block * BLOCK_SIZE; + const std::size_t j_end = std::min(j_start + BLOCK_SIZE, N); + + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = bt[(j_start + jj) * K + (k_start + kk)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + + for (std::size_t i = i_start; i < i_end; ++i) + { + const auto a_row = a.subspan(i * K + k_start); + auto r_row = r.subspan(i * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + }); + } + } + + // ── 矩阵乘法（A 转置）到预分配缓冲区 ───────────────────────── + // 计算 result = this^T * B，其中 this 存储为 (K, M) 行主序 + // 无需实际转置 this，直接从 this 的列读取行（节省 O(K*M) 的拷贝） + // + // 维度要求：this=(K,M), b=(K,N) → result=(M,N) + // 即 C[m][n] = Σ_k A[k][m] * B[k][n] + void transpose_multiply_to(Matrix &result, const Matrix &b) const + { + NN_ASSERT(&result != this && &result != &b, "transpose_multiply_to: self-referencing not supported"); + NN_ASSERT(rows_ == b.rows_, "transpose_multiply_to: inner dimensions mismatch"); + const std::size_t K = rows_; // this is (K, M) + const std::size_t M = cols_; + const std::size_t N = b.cols_; + result.resize(M, N); + if (M == 0 || N == 0 || K == 0) return; + + result.zero(); + + const auto a = span(); // this stored as (K, M) + const auto b_data = b.span(); // b stored as (K, N) + auto r = result.span(); + + const std::size_t i_blocks = (M + BLOCK_SIZE - 1) / BLOCK_SIZE; + const std::size_t j_blocks = (N + BLOCK_SIZE - 1) / BLOCK_SIZE; + const auto n_blocks = i_blocks * j_blocks; + + if (n_blocks <= 1) + { + const std::size_t i_start = 0, i_end = M; + const std::size_t j_start = 0, j_end = N; + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + const std::size_t i_len = i_end - i_start; + + // A^T 块加载：A^T[m][k] = A[k][m] 从 a[k * M + m] 读取 + std::array a_block{}; + for (std::size_t ii = 0; ii < i_len; ++ii) + for (std::size_t kk = 0; kk < k_len; ++kk) + a_block[ii * k_len + kk] = a[(k_start + kk) * M + (i_start + ii)]; + + // B 块加载：B[k][n] 从 b_data[k * N + n] 读取 + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = b_data[(k_start + kk) * N + (j_start + jj)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + + for (std::size_t i = 0; i < i_len; ++i) + { + const auto a_row = std::span(a_block.data() + i * k_len, k_len); + auto r_row = r.subspan((i_start + i) * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + } + else + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + nn::parallel_for_blocks( + block_indices.begin(), block_indices.end(), + [a, b_data, r, M, N, K, j_blocks](std::size_t block_idx) noexcept + { + const std::size_t i_block = block_idx / j_blocks; + const std::size_t j_block = block_idx % j_blocks; + const std::size_t i_start = i_block * BLOCK_SIZE; + const std::size_t i_end = std::min(i_start + BLOCK_SIZE, M); + const std::size_t j_start = j_block * BLOCK_SIZE; + const std::size_t j_end = std::min(j_start + BLOCK_SIZE, N); + const std::size_t i_len = i_end - i_start; + + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + + // A^T 块加载 + std::array a_block{}; + for (std::size_t ii = 0; ii < i_len; ++ii) + for (std::size_t kk = 0; kk < k_len; ++kk) + a_block[ii * k_len + kk] = a[(k_start + kk) * M + (i_start + ii)]; + + // B 块加载 + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = b_data[(k_start + kk) * N + (j_start + jj)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + + for (std::size_t i = 0; i < i_len; ++i) + { + const auto a_row = std::span(a_block.data() + i * k_len, k_len); + auto r_row = r.subspan((i_start + i) * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + }); + } + } + + // ── 累加矩阵乘法（A * B^T，结果累加到 result） ───────────── + // 计算 result += this * B^T + // 用于梯度累加：grad_w += grad_output * input^T + void multiply_transposed_add_to(Matrix &result, const Matrix &b_trans) const + { + NN_ASSERT(&result != this && &result != &b_trans, "multiply_transposed_add_to: self-referencing"); + NN_ASSERT(cols_ == b_trans.cols_, "multiply_transposed_add_to: inner dimensions mismatch"); + NN_ASSERT(result.rows() == rows_ && result.cols() == b_trans.rows_, "multiply_transposed_add_to: result shape mismatch"); + const std::size_t M = rows_; + const std::size_t K = cols_; + const std::size_t N = b_trans.rows_; + if (M == 0 || N == 0 || K == 0) return; + + const auto a = span(); + const auto bt = b_trans.span(); + auto r = result.span(); + + const std::size_t i_blocks = (M + BLOCK_SIZE - 1) / BLOCK_SIZE; + const std::size_t j_blocks = (N + BLOCK_SIZE - 1) / BLOCK_SIZE; + const auto n_blocks = i_blocks * j_blocks; + + if (n_blocks <= 1) + { + const std::size_t i_start = 0, i_end = M; + const std::size_t j_start = 0, j_end = N; + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = bt[(j_start + jj) * K + (k_start + kk)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + for (std::size_t i = i_start; i < i_end; ++i) + { + const auto a_row = a.subspan(i * K + k_start); + auto r_row = r.subspan(i * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + } + else + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + nn::parallel_for_blocks( + block_indices.begin(), block_indices.end(), + [a, bt, r, M, N, K, j_blocks](std::size_t block_idx) noexcept + { + const std::size_t i_block = block_idx / j_blocks; + const std::size_t j_block = block_idx % j_blocks; + const std::size_t i_start = i_block * BLOCK_SIZE; + const std::size_t i_end = std::min(i_start + BLOCK_SIZE, M); + const std::size_t j_start = j_block * BLOCK_SIZE; + const std::size_t j_end = std::min(j_start + BLOCK_SIZE, N); + + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = bt[(j_start + jj) * K + (k_start + kk)]; + const auto b_block_span = std::span(b_block.data(), k_len * j_len); + + for (std::size_t i = i_start; i < i_end; ++i) + { + const auto a_row = a.subspan(i * K + k_start); + auto r_row = r.subspan(i * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const auto b_col = b_block_span.subspan(j * k_len, k_len); + Scalar sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + }); + } + } + + void scale_inplace(Scalar scalar) noexcept + { + auto s = span(); + nn::for_each(s.begin(), s.end(), + [scalar](Scalar &value) noexcept { value *= scalar; }); + } + + // 逐元素加法 inplace + void add_inplace(const Matrix &other) + { + require_same_shape(*this, other, "add_inplace dimension mismatch"); + auto s = span(); + auto o = other.span(); + nn::transform(s.begin(), s.end(), o.begin(), + s.begin(), std::plus<>{}); + } + + // 填充零 + void zero() noexcept + { + std::fill(span().begin(), span().end(), 0.0); + } + + // ── 归约操作 ──────────────────────────────────────────────────── + // result = reduce_op(init, transform_op(data[0]), transform_op(data[1]), ...) + template + [[nodiscard]] T reduce(T init, ReduceOp&& reduce_op, TransformOp&& transform_op) const + { + auto s = span(); + return nn::transform_reduce(s.begin(), s.end(), init, + std::forward(reduce_op), std::forward(transform_op)); + } + + // ── 按行归约（通用数学原语，不是算法） ────────────────────────── + // 对每一行独立归约，返回 (rows, 1) 矩阵。 + // result[r][0] = reduce_op(init, transform_op(this[r][0]), ..., transform_op(this[r][cols-1])) + // 上层可基于此表达 softmax 行最大值/行求和、按行范数等算法。 + template + [[nodiscard]] Matrix row_reduce(T init, ReduceOp&& reduce_op, TransformOp&& transform_op) const + { + Matrix result(rows_, 1); + if (rows_ == 0) return result; + + const auto self = span(); + auto out = result.span(); + const std::size_t C = cols_; + + auto row_indices = std::views::iota(std::size_t{0}, rows_); + + auto process_row = [self, out, C, init, + reduce_op = std::forward(reduce_op), + transform_op = std::forward(transform_op)](std::size_t r) noexcept { + const auto row = self.subspan(r * C, C); + T acc = init; + for (std::size_t c = 0; c < C; ++c) + acc = reduce_op(acc, transform_op(row[c])); + out[r] = static_cast(acc); + }; + + nn::for_each(row_indices.begin(), row_indices.end(), process_row); + return result; + } + + // ── 按列归约（通用数学原语，不是算法） ────────────────────────── + // 对每一列独立归约，返回 (1, cols) 矩阵。 + // result[0][c] = reduce_op(init, transform_op(this[0][c]), ..., transform_op(this[rows-1][c])) + // 上层可基于此表达 LayerNorm 列均值/列方差等算法。 + // + // 实现策略：cache-friendly blocked + 行块并行。 + // bench_thresholds 实测：blocked 全面优于 naive（按列跨行扫描）， + // 行块并行仅在 R >= 1024 且 R*C >= PARALLEL_THRESHOLD 时启用， + // 详见 bench_thresholds.cpp 测试 2/3。 + template + [[nodiscard]] Matrix col_reduce(T init, ReduceOp&& reduce_op, TransformOp&& transform_op) const + { + Matrix result(1, cols_); + if (cols_ == 0) return result; + + const auto self = span(); + auto out = result.span(); + const std::size_t R = rows_; + const std::size_t C = cols_; + + // 极小矩阵直接 naive（避免清零开销） + if (R * C < 64) + { + for (std::size_t c = 0; c < C; ++c) + { + T acc = init; + for (std::size_t r = 0; r < R; ++r) + acc = reduce_op(acc, transform_op(self[r * C + c])); + out[c] = static_cast(acc); + } + return result; + } + + // 行块并行启用条件：R >= COL_REDUCE_PARALLEL_ROWS 且 R*C >= PARALLEL_THRESHOLD。 + // bench_thresholds 实测：R >= 1024 是行块并行的硬门槛（R<1024 时同步开销主导）， + // 详见 bench_thresholds.cpp 测试 3。 + constexpr std::size_t COL_REDUCE_PARALLEL_ROWS = 1024; // 行数门槛 + const std::size_t hw_threads = std::thread::hardware_concurrency(); + const std::size_t n_threads = (hw_threads == 0) ? 1 : hw_threads; + const bool use_parallel = + R >= COL_REDUCE_PARALLEL_ROWS && + R * C >= PARALLEL_THRESHOLD && + n_threads > 1; + + if (!use_parallel) + { + // ── 单线程行主序扫描 ── + for (std::size_t c = 0; c < C; ++c) + out[c] = static_cast(init); + for (std::size_t r = 0; r < R; ++r) + { + const Scalar* row = self.data() + r * C; + for (std::size_t c = 0; c < C; ++c) + { + Scalar v = static_cast(transform_op(row[c])); + out[c] = static_cast(reduce_op(static_cast(out[c]), v)); + } + } + return result; + } + + // ── 行块并行路径 ── + // 分配 n_threads 组本地累加器（连续存储，cache 友好） + std::vector local_acc(n_threads * C); + for (std::size_t t = 0; t < n_threads; ++t) + for (std::size_t c = 0; c < C; ++c) + local_acc[t * C + c] = init; + + // 按行分块并行扫描 + auto& pool = global_thread_pool(); + const std::size_t base = R / n_threads; + const std::size_t rem = R % n_threads; + auto row_blocks = std::views::iota(std::size_t{0}, n_threads); + pool.parallel_for_blocks(row_blocks.begin(), row_blocks.end(), + [self, &local_acc, &reduce_op, &transform_op, C, base, rem](std::size_t t) noexcept { + const std::size_t r0 = t * base + std::min(t, rem); + const std::size_t r_end = (t + 1) * base + std::min(t + 1, rem); + auto* acc = local_acc.data() + t * C; + for (std::size_t r = r0; r < r_end; ++r) + { + const Scalar* row = self.data() + r * C; + for (std::size_t c = 0; c < C; ++c) + { + Scalar v = static_cast(transform_op(row[c])); + acc[c] = reduce_op(acc[c], v); + } + } + }); + + // 归并阶段：串行合并 n_threads 组累加器到 out[c] + // 第 0 组直接写入，其余组归并进来（reduce_op 满足结合律，结果与单线程一致） + for (std::size_t c = 0; c < C; ++c) + out[c] = static_cast(local_acc[c]); // 组 0 + for (std::size_t t = 1; t < n_threads; ++t) + { + const auto* acc = local_acc.data() + t * C; + for (std::size_t c = 0; c < C; ++c) + out[c] = static_cast(reduce_op(static_cast(out[c]), acc[c])); + } + return result; + } + + // ── 按行广播（通用数学原语，不是算法） ────────────────────────── + // this[r][c] = op(this[r][c], row_vec[r][0])，row_vec 形状必须为 (rows_, 1) + // 上层可基于此表达 softmax 减行最大值、除行求和等算法。 + template + void broadcast_row_inplace(const Matrix& row_vec, F&& op) + { + NN_ASSERT(row_vec.rows_ == rows_ && row_vec.cols_ == 1, "row_vec shape mismatch"); + const auto v = row_vec.span(); + const std::size_t C = cols_; + auto d = span(); + auto idx = std::views::iota(std::size_t{0}, d.size()); + nn::for_each(idx.begin(), idx.end(), + [&d, &v, C, op = std::forward(op)](std::size_t i) noexcept { + d[i] = static_cast(op(d[i], v[i / C])); + }); + } + + // ── 按列广播（通用数学原语，不是算法） ────────────────────────── + // this[r][c] = op(this[r][c], col_vec[0][c])，col_vec 形状必须为 (1, cols_) + // 上层可基于此表达 LayerNorm 减列均值、乘列标准差等算法。 + template + void broadcast_col_inplace(const Matrix& col_vec, F&& op) + { + NN_ASSERT(col_vec.rows_ == 1 && col_vec.cols_ == cols_, "col_vec shape mismatch"); + const auto v = col_vec.span(); + const std::size_t C = cols_; + auto d = span(); + auto idx = std::views::iota(std::size_t{0}, d.size()); + nn::for_each(idx.begin(), idx.end(), + [&d, &v, C, op = std::forward(op)](std::size_t i) noexcept { + d[i] = static_cast(op(d[i], v[i % C])); + }); + } + }; + + // ═══════════════════════════════════════════════════════════════════════ + // detail 命名空间：逐元素变换的自由函数（原 Matrix 成员方法） + // + // 这些函数仅是 nn::transform 的薄包装，放在 detail 命名空间而非 Matrix 类内， + // 避免 Matrix 接口膨胀。compute_bench 等基准测试直接调用。 + // ═══════════════════════════════════════════════════════════════════════ + namespace detail + { + // ── 逐元素一元变换（返回新矩阵） ──────────────────────────────── + // out[i] = func(in[i])，内部自动选择串行/并行。 + template + [[nodiscard]] Matrix apply(const Matrix& mat, F&& func) + { + Matrix result(mat.rows(), mat.cols()); + auto s = mat.span(); + auto r = result.span(); + nn::transform(s.begin(), s.end(), + r.begin(), std::forward(func)); + return result; + } + + // ── 逐元素二元变换（返回新矩阵） ──────────────────────────────── + // out[i] = func(a[i], b[i]) + template + [[nodiscard]] Matrix binary_apply(const Matrix& a, const Matrix& b, F&& func) + { + NN_ASSERT(a.rows() == b.rows() && a.cols() == b.cols(), + "binary_apply dimension mismatch"); + Matrix result(a.rows(), a.cols()); + auto s = a.span(); + auto o = b.span(); + auto r = result.span(); + nn::transform(s.begin(), s.end(), + o.begin(), r.begin(), + std::forward(func)); + return result; + } + + // ── 逐元素二元变换（就地修改） ────────────────────────────────── + template + void binary_apply_inplace(Matrix& a, const Matrix& b, F&& func) + { + NN_ASSERT(a.rows() == b.rows() && a.cols() == b.cols(), + "binary_apply_inplace dimension mismatch"); + auto s = a.span(); + auto o = b.span(); + nn::transform(s.begin(), s.end(), + o.begin(), s.begin(), + std::forward(func)); + } + } // namespace detail +} // namespace nn + +// ── GpuTensor 方法实现（需要 Matrix 和 GpuBackend 的完整定义）────────── +#ifdef NN_HAS_VULKAN +#include "backend/gpu_tensor_impl.hpp" +#endif + +#endif // NN_ALGEBRA_MATRIX_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/algebra_ops.hpp b/include/neuralnet.cpp/algebra_ops.hpp new file mode 100644 index 00000000..8f2521e4 --- /dev/null +++ b/include/neuralnet.cpp/algebra_ops.hpp @@ -0,0 +1,156 @@ +#ifndef NN_ALGEBRA_OPS_HPP +#define NN_ALGEBRA_OPS_HPP + +// ── ops.hpp — 表达式模板操作策略定义 ────────────────────────────────────── +// 每个 Op 定义 CPU 行为（apply），用于构建编译期 AST， +// 实现零开销抽象的逐元素运算。 +// ───────────────────────────────────────────────────────────────────────── + +#include +#include + +#include "config.hpp" + +namespace nn::ops +{ + +// ══════════════════════════════════════════════════════════════════════════ +// 二元算术运算 +// ══════════════════════════════════════════════════════════════════════════ + +struct Add +{ + [[nodiscard]] static constexpr Scalar apply(Scalar a, Scalar b) noexcept { return a + b; } +}; + +struct Sub +{ + [[nodiscard]] static constexpr Scalar apply(Scalar a, Scalar b) noexcept { return a - b; } +}; + +struct Mul +{ + [[nodiscard]] static constexpr Scalar apply(Scalar a, Scalar b) noexcept { return a * b; } +}; + +struct Div +{ + [[nodiscard]] static constexpr Scalar apply(Scalar a, Scalar b) noexcept { return a / b; } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// 一元运算 +// ══════════════════════════════════════════════════════════════════════════ + +struct Neg +{ + [[nodiscard]] static constexpr Scalar apply(Scalar a) noexcept { return -a; } +}; + +struct Abs +{ + [[nodiscard]] static Scalar apply(Scalar a) noexcept { return std::abs(a); } +}; + +struct Exp +{ + [[nodiscard]] static Scalar apply(Scalar a) noexcept { return std::exp(a); } +}; + +struct Log +{ + [[nodiscard]] static Scalar apply(Scalar a) noexcept { return std::log(a); } +}; + +struct Sqrt +{ + [[nodiscard]] static Scalar apply(Scalar a) noexcept { return std::sqrt(a); } +}; + +struct Rsqrt +{ + /// 1/sqrt(x) —— LayerNorm 标准差倒数专用，单次调用避免除法 + [[nodiscard]] static Scalar apply(Scalar a) noexcept { return Scalar{1} / std::sqrt(a); } +}; + +struct Sigmoid +{ + // ── 数值稳定实现：大正数直接返回 1，大负数返回 0，避免 exp 上溢 ── + // 原 1/(1+exp(-a)) 在 a 为大负数时 exp(-a) 会溢出为 inf → 结果为 0 + // 但 NaN/Inf 可能传播；此处分支保证全区间有界且无 NaN。 + [[nodiscard]] static Scalar apply(Scalar a) noexcept + { + if (a >= Scalar{0}) + { + const Scalar z = std::exp(-a); + return Scalar{1} / (Scalar{1} + z); + } + else + { + const Scalar z = std::exp(a); + return z / (Scalar{1} + z); + } + } +}; + +struct Tanh +{ + [[nodiscard]] static Scalar apply(Scalar a) noexcept { return std::tanh(a); } +}; + +struct ReLU +{ + [[nodiscard]] static constexpr Scalar apply(Scalar a) noexcept { return a > Scalar{0} ? a : Scalar{0}; } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// 比较运算（返回 bool，用于条件表达式） +// ══════════════════════════════════════════════════════════════════════════ + +struct Gt +{ + [[nodiscard]] static constexpr bool apply(Scalar a, Scalar b) noexcept { return a > b; } +}; + +struct Lt +{ + [[nodiscard]] static constexpr bool apply(Scalar a, Scalar b) noexcept { return a < b; } +}; + +struct Ge +{ + [[nodiscard]] static constexpr bool apply(Scalar a, Scalar b) noexcept { return a >= b; } +}; + +struct Le +{ + [[nodiscard]] static constexpr bool apply(Scalar a, Scalar b) noexcept { return a <= b; } +}; + +struct Eq +{ + [[nodiscard]] static constexpr bool apply(Scalar a, Scalar b) noexcept { return a == b; } +}; + +struct Ne +{ + [[nodiscard]] static constexpr bool apply(Scalar a, Scalar b) noexcept { return a != b; } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// 二元值运算（返回 Scalar） +// ══════════════════════════════════════════════════════════════════════════ + +struct Max +{ + [[nodiscard]] static constexpr Scalar apply(Scalar a, Scalar b) noexcept { return std::max(a, b); } +}; + +struct Min +{ + [[nodiscard]] static constexpr Scalar apply(Scalar a, Scalar b) noexcept { return std::min(a, b); } +}; + +} // namespace nn::ops + +#endif // NN_ALGEBRA_OPS_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/algebra_span.hpp b/include/neuralnet.cpp/algebra_span.hpp new file mode 100644 index 00000000..cc912b05 --- /dev/null +++ b/include/neuralnet.cpp/algebra_span.hpp @@ -0,0 +1,136 @@ +#ifndef NN_ALGEBRA_SPAN_HPP +#define NN_ALGEBRA_SPAN_HPP + +// ── span.hpp — 可构建 AST 的智能视图 ───────────────────────────────────── +// 替代 std::span 作为逐元素操作的载体。 +// Span 同时是： +// 1. 数据视图（指向 Matrix 内部数据） +// 2. 表达式树叶子节点（eval(i) 返回 data_[i]） +// 3. compute::apply 的输出目标（被写入） +// +// 运算符重载在幕后构建 AST，上层与具体执行路径解耦。 +// ───────────────────────────────────────────────────────────────────────── + +#include +#include + +#include "config.hpp" +#include "algebra_expr.hpp" + +namespace nn +{ + +// ══════════════════════════════════════════════════════════════════════════ +// Span：可构建 AST 的智能视图 +// ══════════════════════════════════════════════════════════════════════════ + +class Span +{ +private: + Scalar *data_; + std::size_t size_; + +public: + // ── 构造函数 ────────────────────────────────────────────────────────── + constexpr Span() noexcept : data_(nullptr), size_(0) {} + constexpr Span(Scalar *data, std::size_t size) noexcept : data_(data), size_(size) {} + + /// 从 std::span 隐式构造（兼容 Matrix::span()） + constexpr Span(std::span sp) noexcept : data_(sp.data()), size_(sp.size()) {} + + // ── 访问器 ─────────────────────────────────────────────────────────── + [[nodiscard]] constexpr Scalar *data() noexcept { return data_; } + [[nodiscard]] constexpr const Scalar *data() const noexcept { return data_; } + [[nodiscard]] constexpr std::size_t size() const noexcept { return size_; } + [[nodiscard]] constexpr bool empty() const noexcept { return size_ == 0; } + + // ── 索引访问 ───────────────────────────────────────────────────────── + [[nodiscard]] constexpr Scalar &operator[](std::size_t i) noexcept { return data_[i]; } + [[nodiscard]] constexpr Scalar operator[](std::size_t i) const noexcept { return data_[i]; } + + // ── 表达式树叶子节点的 eval ────────────────────────────────────────── + [[nodiscard]] constexpr Scalar eval(std::size_t i) const noexcept { return data_[i]; } + + // ── 子视图 ─────────────────────────────────────────────────────────── + [[nodiscard]] constexpr Span subspan(std::size_t offset, std::size_t count) const noexcept + { + return Span{data_ + offset, count}; + } + + [[nodiscard]] constexpr Span subspan(std::size_t offset) const noexcept + { + return Span{data_ + offset, size_ - offset}; + } + + // 注：Span 不定义成员运算符，所有 AST 构造由 expr.hpp 中的 + // 自由函数模板 operator+/-/*/>/< 等（基于 Expression 概念）统一处理。 + // 这保证 Span 与 ConstSpan/Val/任意 Expression 都能自然组合。 +}; + +// ══════════════════════════════════════════════════════════════════════════ +// ConstSpan：只读 AST 叶子节点 +// +// 与 Span 的区别： +// - 内部存储 const Scalar*（不可写入） +// - 可从 const Matrix::span() 构造（std::span） +// - 可从 Span 隐式构造（Span → ConstSpan 单向转换） +// - 仅满足 Expression 概念（作为 AST 输入），不能作为 compute::apply 输出 +// +// 设计理由： +// 上层 Layer/Loss/Optimizer 在 const Matrix& 上调用 .span() 时， +// 得到的是 std::span，无法构造可写 Span。 +// ConstSpan 提供类型安全的只读 AST 叶子，避免 const_cast 滥用。 +// ══════════════════════════════════════════════════════════════════════════ + +class ConstSpan +{ +private: + const Scalar *data_; + std::size_t size_; + +public: + // ── 构造函数 ────────────────────────────────────────────────────────── + constexpr ConstSpan() noexcept : data_(nullptr), size_(0) {} + constexpr ConstSpan(const Scalar *data, std::size_t size) noexcept : data_(data), size_(size) {} + + /// 从 std::span 隐式构造（兼容 const Matrix::span()） + constexpr ConstSpan(std::span sp) noexcept : data_(sp.data()), size_(sp.size()) {} + + /// 从 std::span 隐式构造（兼容 mutable Matrix::span()） + /// 注：标准上 std::span 应可隐式转 std::span， + /// 但部分 libc++ 实验/C++26 模式下该隐式转换受限，故显式提供此重载。 + constexpr ConstSpan(std::span sp) noexcept : data_(sp.data()), size_(sp.size()) {} + + /// 从 Span 隐式构造（Span 可作为只读输入） + constexpr ConstSpan(const Span &s) noexcept : data_(s.data()), size_(s.size()) {} + + // ── 访问器 ─────────────────────────────────────────────────────────── + [[nodiscard]] constexpr const Scalar *data() const noexcept { return data_; } + [[nodiscard]] constexpr std::size_t size() const noexcept { return size_; } + [[nodiscard]] constexpr bool empty() const noexcept { return size_ == 0; } + + // ── 索引访问（只读） ───────────────────────────────────────────────── + [[nodiscard]] constexpr Scalar operator[](std::size_t i) const noexcept { return data_[i]; } + + // ── 表达式树叶子节点的 eval ────────────────────────────────────────── + [[nodiscard]] constexpr Scalar eval(std::size_t i) const noexcept { return data_[i]; } + + // ── 子视图 ─────────────────────────────────────────────────────────── + [[nodiscard]] constexpr ConstSpan subspan(std::size_t offset, std::size_t count) const noexcept + { + return ConstSpan{data_ + offset, count}; + } + + [[nodiscard]] constexpr ConstSpan subspan(std::size_t offset) const noexcept + { + return ConstSpan{data_ + offset, size_ - offset}; + } + + // 注：ConstSpan 不定义成员运算符，所有 AST 构造由 expr.hpp 中的 + // 自由函数模板 operator+/-/*/>/< 等（基于 Expression 概念）统一处理。 + // 这保证 ConstSpan 与 Span/Val/任意 Expression 都能自然组合。 +}; + +} // namespace nn + +#endif // NN_ALGEBRA_SPAN_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/backend/cuda_backend.hpp b/include/neuralnet.cpp/backend/cuda_backend.hpp new file mode 100644 index 00000000..08fbe55b --- /dev/null +++ b/include/neuralnet.cpp/backend/cuda_backend.hpp @@ -0,0 +1,697 @@ +// ── cuda_backend.hpp — CUDA 计算后端 ───────────────────────────────────── +// CudaBackend 管理 CUDA 设备、cuBLAS 句柄、CUDA 流，提供 GPU 操作 API。 +// 内核函数通过 C-linkage 接口调用 nvcc 编译的静态库。 +// +// 架构： +// CudaBackend（本文件）→ 调用 cuda_kernels.h 中的 extern "C" 函数 +// cuda_kernels.cu（nvcc 编译）→ 实现 CUDA 内核 +// +// 同步模型： +// 默认使用非默认 CUDA 流 + cuBLAS 句柄绑定该流。 +// begin_batch / end_batch 使用流级隐式同步（仅在 end_batch 调用 +// cudaStreamSynchronize）。所有 op 级函数不再调用 cudaDeviceSynchronize， +// 实现真正异步的流水线执行。 +// ───────────────────────────────────────────────────────────────────────── + +#ifndef NN_CUDA_BACKEND_HPP +#define NN_CUDA_BACKEND_HPP + +#ifdef NN_HAS_CUDA + +#include +#include +#include +#include +#include +#include +#include + +#include "../core_errors.hpp" +#include "../config.hpp" + +// cuBLAS（matmul 加速，由 cmake -DNN_HAS_CUBLAS 启用） +#ifdef NN_HAS_CUBLAS +#include +#endif + +// CUDA 内核 C-linkage 接口 +#include "cuda_kernels.h" + +namespace nn +{ + +// ── CUDA 错误检查 ───────────────────────────────────────────────────────── +namespace cuda_detail +{ + +[[nodiscard]] inline Result cuda_check(cudaError_t err, const char* file, int line) +{ + if (err != cudaSuccess) + { + return std::unexpected(Error{ + std::string("CUDA error ") + cudaGetErrorString(err) + + " at " + file + ":" + std::to_string(line)}); + } + return {}; +} + +} // namespace cuda_detail + +#define NN_CUDA_CHECK(call) cuda_detail::cuda_check((call), __FILE__, __LINE__) + +// ══════════════════════════════════════════════════════════════════════════ +// CudaBuffer — GPU 显存 RAII 封装 +// ══════════════════════════════════════════════════════════════════════════ + +class CudaBuffer +{ +private: + void* ptr_ = nullptr; + std::size_t size_ = 0; // 元素数 + +public: + CudaBuffer() = default; + + CudaBuffer(void* ptr, std::size_t size) : ptr_(ptr), size_(size) {} + + ~CudaBuffer() + { + if (ptr_) + cudaFree(ptr_); + } + + // 移动语义 + CudaBuffer(CudaBuffer&& o) noexcept : ptr_(o.ptr_), size_(o.size_) + { + o.ptr_ = nullptr; + o.size_ = 0; + } + + CudaBuffer& operator=(CudaBuffer&& o) noexcept + { + if (this != &o) + { + if (ptr_) cudaFree(ptr_); + ptr_ = o.ptr_; + size_ = o.size_; + o.ptr_ = nullptr; + o.size_ = 0; + } + return *this; + } + + // 禁止拷贝 + CudaBuffer(const CudaBuffer&) = delete; + CudaBuffer& operator=(const CudaBuffer&) = delete; + + [[nodiscard]] static Result allocate(std::size_t elem_count) + { + void* ptr = nullptr; + cudaError_t err = cudaMalloc(&ptr, elem_count * sizeof(float)); + if (err != cudaSuccess) + return std::unexpected(Error{ + std::string("cudaMalloc failed: ") + cudaGetErrorString(err) + + " (" + std::to_string(elem_count * sizeof(float)) + " bytes)"}); + return CudaBuffer(ptr, elem_count); + } + + [[nodiscard]] void* ptr() const noexcept { return ptr_; } + [[nodiscard]] std::size_t size() const noexcept { return size_; } + [[nodiscard]] bool valid() const noexcept { return ptr_ != nullptr; } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// CudaTensor — GPU 张量抽象 +// ══════════════════════════════════════════════════════════════════════════ + +class CudaTensor +{ +private: + std::shared_ptr buffer_; + std::size_t rows_ = 0; + std::size_t cols_ = 0; + +public: + CudaTensor() = default; + + CudaTensor(std::shared_ptr buffer, std::size_t rows, std::size_t cols) + : buffer_(std::move(buffer)), rows_(rows), cols_(cols) {} + + // 从 CPU Matrix 创建（上传数据） + [[nodiscard]] static Result from_matrix( + const Matrix& cpu_mat, class CudaBackend& backend); + + // 创建空的 GPU Tensor（用于输出） + [[nodiscard]] static Result create_empty( + std::size_t rows, std::size_t cols, class CudaBackend& backend); + + // 转换为 CPU Matrix（下载数据） + [[nodiscard]] Result to_matrix(class CudaBackend& backend) const; + + // 访问器 + [[nodiscard]] std::size_t rows() const noexcept { return rows_; } + [[nodiscard]] std::size_t cols() const noexcept { return cols_; } + [[nodiscard]] bool valid() const noexcept { return buffer_ && buffer_->valid(); } + [[nodiscard]] const CudaBuffer& buffer() const noexcept { return *buffer_; } + [[nodiscard]] float* data() const noexcept { return buffer_ ? static_cast(buffer_->ptr()) : nullptr; } + + // 零拷贝 reshape + [[nodiscard]] CudaTensor with_shape(std::size_t new_rows, std::size_t new_cols) const + { + return CudaTensor(buffer_, new_rows, new_cols); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// CudaBackend — CUDA 计算后端单例 +// ══════════════════════════════════════════════════════════════════════════ + +class CudaBackend +{ +private: + int device_id_ = 0; + cudaDeviceProp device_props_{}; + bool initialized_ = false; + std::mutex init_mutex_; + + // CUDA 流（非默认流，支持异步执行 + 与 cuBLAS 句柄绑定） + cudaStream_t stream_ = nullptr; + + // cuBLAS 句柄（初始化时创建一次，整个生命周期复用） +#ifdef NN_HAS_CUBLAS + cublasHandle_t cublas_handle_ = nullptr; +#endif + + // 批处理状态（true 表示在 begin_batch/end_batch 之间，不主动同步） + bool in_batch_ = false; + + CudaBackend() = default; + +public: + ~CudaBackend() + { +#ifdef NN_HAS_CUBLAS + if (cublas_handle_) + cublasDestroy(cublas_handle_); +#endif + if (stream_) + cudaStreamDestroy(stream_); + } + + // 禁止拷贝和移动 + CudaBackend(const CudaBackend&) = delete; + CudaBackend& operator=(const CudaBackend&) = delete; + CudaBackend(CudaBackend&&) = delete; + CudaBackend& operator=(CudaBackend&&) = delete; + + [[nodiscard]] static CudaBackend& instance() + { + static CudaBackend backend; + return backend; + } + + // ── 初始化 ───────────────────────────────────────────────────────────── + [[nodiscard]] Result initialize() + { + std::lock_guard lock(init_mutex_); + if (initialized_) return {}; + + int device_count = 0; + cudaError_t err = cudaGetDeviceCount(&device_count); + if (err != cudaSuccess || device_count == 0) + return std::unexpected(Error{"No CUDA devices found"}); + + // 选择第一个设备 + err = cudaSetDevice(device_id_); + if (err != cudaSuccess) + return std::unexpected(Error{"cudaSetDevice failed"}); + + err = cudaGetDeviceProperties(&device_props_, device_id_); + if (err != cudaSuccess) + return std::unexpected(Error{"cudaGetDeviceProperties failed"}); + + // 创建非默认流（支持异步执行 + 与默认流解耦） + err = cudaStreamCreateWithFlags(&stream_, cudaStreamNonBlocking); + if (err != cudaSuccess) + return std::unexpected(Error{"cudaStreamCreate failed"}); + +#ifdef NN_HAS_CUBLAS + // 创建 cuBLAS 句柄并绑定到 stream_ + cublasStatus_t cb_err = cublasCreate(&cublas_handle_); + if (cb_err != CUBLAS_STATUS_SUCCESS) + return std::unexpected(Error{"cublasCreate failed"}); + cb_err = cublasSetStream(cublas_handle_, stream_); + if (cb_err != CUBLAS_STATUS_SUCCESS) + return std::unexpected(Error{"cublasSetStream failed"}); +#endif + + initialized_ = true; + return {}; + } + + [[nodiscard]] bool is_initialized() const noexcept { return initialized_; } + [[nodiscard]] bool gpu_available() const noexcept { return initialized_; } + + [[nodiscard]] const cudaDeviceProp& device_props() const noexcept { return device_props_; } + [[nodiscard]] int device_id() const noexcept { return device_id_; } + + // 当前流指针（供 C 接口内核调用使用，nullptr 表示默认流） + [[nodiscard]] void* stream_ptr() const noexcept + { + return static_cast(stream_); + } + +#ifdef NN_HAS_CUBLAS + [[nodiscard]] cublasHandle_t cublas_handle() const noexcept { return cublas_handle_; } +#endif + + // ── 批处理控制 ──────────────────────────────────────────────────────── + // begin_batch：标记进入批处理模式，期间所有 op 不主动同步。 + // end_batch：同步流，确保所有排队的内核执行完成。 + [[nodiscard]] Result begin_batch() + { + in_batch_ = true; + return {}; + } + + [[nodiscard]] Result end_batch() + { + in_batch_ = false; + if (stream_) + { + auto err = cudaStreamSynchronize(stream_); + return NN_CUDA_CHECK(err); + } + auto err = cudaDeviceSynchronize(); + return NN_CUDA_CHECK(err); + } + + [[nodiscard]] bool in_batch() const noexcept { return in_batch_; } + + // ── 流同步（供 flush_batch 或显式同步点捕获异步 CUDA 错误）──────── + [[nodiscard]] Result sync_stream() + { + if (stream_) + { + auto err = cudaStreamSynchronize(stream_); + return NN_CUDA_CHECK(err); + } + auto err = cudaDeviceSynchronize(); + return NN_CUDA_CHECK(err); + } + + // ── 上传/下载（批处理模式下使用异步传输）────────────────────────────── + [[nodiscard]] Result upload_blocking(CudaTensor& dst, std::span cpu_data) + { + if (!dst.valid()) + return std::unexpected(Error{"Invalid destination CudaTensor"}); + if (cpu_data.size() != dst.rows() * dst.cols()) + return std::unexpected(Error{"Upload size mismatch"}); + + auto err = cudaMemcpyAsync(dst.data(), cpu_data.data(), + cpu_data.size_bytes(), cudaMemcpyHostToDevice, stream_); + if (!in_batch_) + { + auto sync_err = cudaStreamSynchronize(stream_); + if (sync_err != cudaSuccess) + return NN_CUDA_CHECK(sync_err); + } + return NN_CUDA_CHECK(err); + } + + [[nodiscard]] Result download_blocking(const CudaTensor& src, std::span cpu_data) + { + if (!src.valid()) + return std::unexpected(Error{"Invalid source CudaTensor"}); + if (cpu_data.size() != src.rows() * src.cols()) + return std::unexpected(Error{"Download size mismatch"}); + + // 下载必须同步，确保数据可用 + auto err = cudaMemcpyAsync(cpu_data.data(), src.data(), + cpu_data.size_bytes(), cudaMemcpyDeviceToHost, stream_); + if (err != cudaSuccess) + return NN_CUDA_CHECK(err); + auto sync_err = cudaStreamSynchronize(stream_); + return NN_CUDA_CHECK(sync_err); + } + + // ── 纯 GPU 矩阵乘法 ────────────────────────────────────────────────── + [[nodiscard]] Result matmul_gpu( + const CudaTensor& A, const CudaTensor& B, + uint32_t transA = 0, uint32_t transB = 0) + { + const auto M = transA ? A.cols() : A.rows(); + const auto K = transA ? A.rows() : A.cols(); + const auto N = transB ? B.rows() : B.cols(); + const auto K_B = transB ? B.cols() : B.rows(); + if (K != K_B) + return std::unexpected(Error{"matmul: K dimension mismatch"}); + + auto C_res = CudaTensor::create_empty(M, N, *this); + if (!C_res) return std::unexpected(C_res.error()); + + // 优先使用 cuBLAS 句柄复用路径（避免每次调用创建/销毁句柄） +#ifdef NN_HAS_CUBLAS + int err = cuda_matmul_with_handle( + A.data(), B.data(), C_res->data(), + A.rows(), A.cols(), B.rows(), B.cols(), + transA, transB, cublas_handle()); +#else + int err = cuda_matmul( + A.data(), B.data(), C_res->data(), + A.rows(), A.cols(), B.rows(), B.cols(), + transA, transB, stream_ptr()); +#endif + if (err != 0) + return std::unexpected(Error{"cuda_matmul failed: " + std::to_string(err)}); + + return std::move(*C_res); + } + + // ── 批量矩阵乘法 ───────────────────────────────────────────────────── + [[nodiscard]] Result batched_matmul_gpu( + const CudaTensor& A, const CudaTensor& B, + uint32_t batch, + uint32_t transA = 0, uint32_t transB = 0) + { + if (batch == 0) + return std::unexpected(Error{"batched_matmul: batch must be > 0"}); + if (A.rows() % batch != 0 || B.rows() % batch != 0) + return std::unexpected(Error{"batched_matmul: rows not divisible by batch"}); + + const auto a_rows_per = A.rows() / batch; + const auto M = transA ? A.cols() : a_rows_per; + const auto K = transA ? a_rows_per : A.cols(); + const auto b_rows_per = B.rows() / batch; + const auto K_B = transB ? B.cols() : b_rows_per; + const auto N = transB ? b_rows_per : B.cols(); + if (K != K_B) + return std::unexpected(Error{"batched_matmul: K dimension mismatch"}); + + auto C_res = CudaTensor::create_empty(batch * M, N, *this); + if (!C_res) return std::unexpected(C_res.error()); + + int err = cuda_batched_matmul( + A.data(), B.data(), C_res->data(), + A.rows(), A.cols(), B.rows(), B.cols(), + batch, transA, transB, stream_ptr()); + if (err != 0) + return std::unexpected(Error{"cuda_batched_matmul failed"}); + + return std::move(*C_res); + } + + // ── 逐元素运算 ──────────────────────────────────────────────────────── + [[nodiscard]] Result elementwise_unary_gpu( + const CudaTensor& A, uint32_t op) + { + auto out_res = CudaTensor::create_empty(A.rows(), A.cols(), *this); + if (!out_res) return std::unexpected(out_res.error()); + + const auto count = static_cast(A.rows() * A.cols()); + int err = cuda_elementwise_unary(A.data(), out_res->data(), count, op, stream_ptr()); + if (err != 0) return std::unexpected(Error{"cuda_elementwise_unary failed"}); + + return std::move(*out_res); + } + + [[nodiscard]] Result elementwise_binary_gpu( + const CudaTensor& A, const CudaTensor& B, uint32_t op) + { + auto out_res = CudaTensor::create_empty(A.rows(), A.cols(), *this); + if (!out_res) return std::unexpected(out_res.error()); + + const auto count = static_cast(A.rows() * A.cols()); + int err = cuda_elementwise_binary(A.data(), B.data(), out_res->data(), count, op, stream_ptr()); + if (err != 0) return std::unexpected(Error{"cuda_elementwise_binary failed"}); + + return std::move(*out_res); + } + + [[nodiscard]] Result elementwise_binary_scalar_gpu( + const CudaTensor& A, uint32_t op, float scalar, bool scalar_first) + { + auto out_res = CudaTensor::create_empty(A.rows(), A.cols(), *this); + if (!out_res) return std::unexpected(out_res.error()); + + const auto count = static_cast(A.rows() * A.cols()); + int err = cuda_elementwise_binary_scalar( + A.data(), out_res->data(), count, op, scalar, scalar_first ? 1 : 0, stream_ptr()); + if (err != 0) return std::unexpected(Error{"cuda_elementwise_binary_scalar failed"}); + + return std::move(*out_res); + } + + [[nodiscard]] Result axpy_gpu( + const CudaTensor& A, const CudaTensor& B, float scalar) + { + auto out_res = CudaTensor::create_empty(A.rows(), A.cols(), *this); + if (!out_res) return std::unexpected(out_res.error()); + + const auto count = static_cast(A.rows() * A.cols()); + int err = cuda_axpy(A.data(), B.data(), out_res->data(), count, scalar, stream_ptr()); + if (err != 0) return std::unexpected(Error{"cuda_axpy failed"}); + + return std::move(*out_res); + } + + [[nodiscard]] Result elementwise_select_scalar_cond_gpu( + const CudaTensor& A, const CudaTensor& then_t, + uint32_t cmp, float scalar_b, float scalar_else) + { + auto out_res = CudaTensor::create_empty(A.rows(), A.cols(), *this); + if (!out_res) return std::unexpected(out_res.error()); + + const auto count = static_cast(A.rows() * A.cols()); + int err = cuda_elementwise_select_scalar_cond( + A.data(), then_t.data(), out_res->data(), count, cmp, scalar_b, scalar_else, stream_ptr()); + if (err != 0) return std::unexpected(Error{"cuda_select_scalar_cond failed"}); + + return std::move(*out_res); + } + + // ── 归约 ────────────────────────────────────────────────────────────── + [[nodiscard]] Result reduce_gpu( + const CudaTensor& input, uint32_t mode, uint32_t reduce_op) + { + const auto rows = static_cast(input.rows()); + const auto cols = static_cast(input.cols()); + + std::size_t out_rows = (mode == 0) ? rows : 1; + std::size_t out_cols = (mode == 0) ? 1 : cols; + auto out_res = CudaTensor::create_empty(out_rows, out_cols, *this); + if (!out_res) return std::unexpected(out_res.error()); + + int err; + if (mode == 0) + err = cuda_reduce_row(input.data(), out_res->data(), rows, cols, reduce_op, stream_ptr()); + else + err = cuda_reduce_col(input.data(), out_res->data(), rows, cols, reduce_op, stream_ptr()); + + if (err != 0) return std::unexpected(Error{"cuda_reduce failed"}); + + return std::move(*out_res); + } + + // ── 广播 ────────────────────────────────────────────────────────────── + [[nodiscard]] Result broadcast_gpu( + const CudaTensor& A, const CudaTensor& vec, + uint32_t mode, uint32_t op) + { + auto out_res = CudaTensor::create_empty(A.rows(), A.cols(), *this); + if (!out_res) return std::unexpected(out_res.error()); + + const auto rows = static_cast(A.rows()); + const auto cols = static_cast(A.cols()); + + int err; + if (mode == 0) + err = cuda_broadcast_row(A.data(), vec.data(), out_res->data(), rows, cols, op, stream_ptr()); + else + err = cuda_broadcast_col(A.data(), vec.data(), out_res->data(), rows, cols, op, stream_ptr()); + + if (err != 0) return std::unexpected(Error{"cuda_broadcast failed"}); + + return std::move(*out_res); + } + + // ── 转置 ────────────────────────────────────────────────────────────── + [[nodiscard]] Result transpose_gpu(const CudaTensor& A) + { + const auto R = static_cast(A.rows()); + const auto C = static_cast(A.cols()); + + auto out_res = CudaTensor::create_empty(C, R, *this); + if (!out_res) return std::unexpected(out_res.error()); + + int err = cuda_transpose(A.data(), out_res->data(), R, C, stream_ptr()); + if (err != 0) return std::unexpected(Error{"cuda_transpose failed"}); + + return std::move(*out_res); + } + + // ── 3D 重排 ─────────────────────────────────────────────────────────── + [[nodiscard]] Result rearrange_3d_gpu( + const CudaTensor& input, + uint32_t M, uint32_t B, uint32_t N, uint32_t inverse) + { + const auto total = M * B * N; + if (total != input.rows() * input.cols()) + return std::unexpected(Error{"rearrange_3d: element count mismatch"}); + + const auto out_rows = inverse ? M : (static_cast(B) * M); + const auto out_cols = inverse ? (static_cast(B) * N) : N; + auto out_res = CudaTensor::create_empty(out_rows, out_cols, *this); + if (!out_res) return std::unexpected(out_res.error()); + + int err = cuda_rearrange_3d(input.data(), out_res->data(), M, B, N, inverse, stream_ptr()); + if (err != 0) return std::unexpected(Error{"cuda_rearrange_3d failed"}); + + return std::move(*out_res); + } + + // ── Fill zero ───────────────────────────────────────────────────────── + [[nodiscard]] Result fill_zero_gpu(CudaTensor& tensor) + { + auto err = cudaMemsetAsync(tensor.data(), 0, + tensor.rows() * tensor.cols() * sizeof(float), stream_); + return NN_CUDA_CHECK(err); + } + + // ── Clone ───────────────────────────────────────────────────────────── + [[nodiscard]] Result clone_gpu(const CudaTensor& src) + { + auto dst_res = CudaTensor::create_empty(src.rows(), src.cols(), *this); + if (!dst_res) return std::unexpected(dst_res.error()); + + auto err = cudaMemcpyAsync(dst_res->data(), src.data(), + src.rows() * src.cols() * sizeof(float), + cudaMemcpyDeviceToDevice, stream_); + if (err != cudaSuccess) + return std::unexpected(Error{"cudaMemcpy D2D failed"}); + + return std::move(*dst_res); + } + + // ── 行切片 ──────────────────────────────────────────────────────────── + [[nodiscard]] Result slice_rows_gpu( + const CudaTensor& src, std::size_t start_row, std::size_t count) + { + if (start_row + count > src.rows()) + return std::unexpected(Error{"slice_rows: out of bounds"}); + + auto dst_res = CudaTensor::create_empty(count, src.cols(), *this); + if (!dst_res) return std::unexpected(dst_res.error()); + + const std::size_t bytes = count * src.cols() * sizeof(float); + const std::size_t src_offset = start_row * src.cols() * sizeof(float); + + auto err = cudaMemcpyAsync(dst_res->data(), + static_cast(src.buffer().ptr()) + src_offset, + bytes, cudaMemcpyDeviceToDevice, stream_); + if (err != cudaSuccess) + return std::unexpected(Error{"slice_rows cudaMemcpy failed"}); + + return std::move(*dst_res); + } + + // ── 行插入 ──────────────────────────────────────────────────────────── + [[nodiscard]] Result insert_rows_gpu( + CudaTensor& dst, std::size_t dst_start_row, const CudaTensor& src) + { + if (dst.cols() != src.cols()) + return std::unexpected(Error{"insert_rows: column mismatch"}); + if (dst_start_row + src.rows() > dst.rows()) + return std::unexpected(Error{"insert_rows: out of bounds"}); + + const std::size_t bytes = src.rows() * src.cols() * sizeof(float); + const std::size_t dst_offset = dst_start_row * dst.cols() * sizeof(float); + + auto err = cudaMemcpyAsync(static_cast(dst.buffer().ptr()) + dst_offset, + src.data(), bytes, cudaMemcpyDeviceToDevice, stream_); + return NN_CUDA_CHECK(err); + } + + // ── Gather ──────────────────────────────────────────────────────────── + [[nodiscard]] Result gather_gpu( + const CudaTensor& table, const CudaTensor& indices) + { + const auto vocab = static_cast(table.rows()); + const auto D = static_cast(table.cols()); + const auto num = static_cast(indices.rows() * indices.cols()); + + auto out_res = CudaTensor::create_empty(num, D, *this); + if (!out_res) return std::unexpected(out_res.error()); + + int err = cuda_gather( + table.data(), indices.data(), + out_res->data(), vocab, D, num, stream_ptr()); + if (err != 0) return std::unexpected(Error{"cuda_gather failed"}); + + return std::move(*out_res); + } + + // ── Scatter Add ─────────────────────────────────────────────────────── + [[nodiscard]] Result scatter_add_gpu( + CudaTensor& dst, const CudaTensor& indices, const CudaTensor& grad) + { + const auto vocab = static_cast(dst.rows()); + const auto D = static_cast(dst.cols()); + const auto num = static_cast(indices.rows() * indices.cols()); + + int err = cuda_scatter_add( + dst.data(), indices.data(), + grad.data(), vocab, D, num, stream_ptr()); + if (err != 0) return std::unexpected(Error{"cuda_scatter_add failed"}); + + return {}; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// CudaTensor 方法实现（需要 CudaBackend 完整定义） +// ══════════════════════════════════════════════════════════════════════════ + +inline Result CudaTensor::from_matrix(const Matrix& cpu_mat, CudaBackend& backend) +{ + if (cpu_mat.empty()) + return std::unexpected(Error{"Empty matrix"}); + + auto buf_res = CudaBuffer::allocate(cpu_mat.size()); + if (!buf_res) return std::unexpected(buf_res.error()); + + auto tensor = CudaTensor( + std::make_shared(std::move(*buf_res)), + cpu_mat.rows(), cpu_mat.cols()); + + auto upload_res = backend.upload_blocking(tensor, cpu_mat.span()); + if (!upload_res) return std::unexpected(upload_res.error()); + + return tensor; +} + +inline Result CudaTensor::create_empty( + std::size_t rows, std::size_t cols, CudaBackend& /*backend*/) +{ + auto buf_res = CudaBuffer::allocate(rows * cols); + if (!buf_res) return std::unexpected(buf_res.error()); + + return CudaTensor(std::make_shared(std::move(*buf_res)), rows, cols); +} + +inline Result CudaTensor::to_matrix(CudaBackend& backend) const +{ + if (!valid()) + return std::unexpected(Error{"Invalid CudaTensor"}); + + Matrix cpu_mat(rows_, cols_); + auto dl_res = backend.download_blocking(*this, cpu_mat.span()); + if (!dl_res) return std::unexpected(dl_res.error()); + + return cpu_mat; +} + +} // namespace nn + +#endif // NN_HAS_CUDA +#endif // NN_CUDA_BACKEND_HPP diff --git a/include/neuralnet.cpp/backend/gpu_tensor_impl.hpp b/include/neuralnet.cpp/backend/gpu_tensor_impl.hpp new file mode 100644 index 00000000..b35d3b42 --- /dev/null +++ b/include/neuralnet.cpp/backend/gpu_tensor_impl.hpp @@ -0,0 +1,75 @@ +// ── gpu_tensor_impl.hpp ───────────────────────────────────────────────── +// GpuTensor 方法实现（需要 Matrix 和 GpuBackend 的完整定义） +// 此文件在 algebra_matrix.hpp 末尾自动包含，解决循环依赖问题。 +// ───────────────────────────────────────────────────────────────────────── + +#ifndef NN_GPU_TENSOR_IMPL_HPP +#define NN_GPU_TENSOR_IMPL_HPP + +#ifdef NN_HAS_VULKAN + +namespace nn +{ + +// 张量 buffer 的标准 usage 标志： +// STORAGE_BUFFER: 绑定到 shader 进行 GPU 计算 +// TRANSFER_DST: 允许 vkCmdFillBuffer(zero) / vkCmdCopyBuffer(insert_rows) 写入 +// TRANSFER_SRC: 允许 vkCmdCopyBuffer(clone / slice_rows) 读取 +static constexpr VkBufferUsageFlags TENSOR_BUFFER_USAGE = + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + +inline Result GpuTensor::from_matrix(const Matrix& cpu_mat, GpuBackend& backend) +{ + if (cpu_mat.empty()) + return std::unexpected(Error{"Empty matrix"}); + + auto buf_res = GpuBuffer::create_device_local( + backend.device().device(), backend.memory_pool(), + cpu_mat.size(), + TENSOR_BUFFER_USAGE); + if (!buf_res) + return std::unexpected(buf_res.error()); + + auto tensor = GpuTensor( + std::make_shared(std::move(*buf_res)), + cpu_mat.rows(), cpu_mat.cols()); + + auto upload_res = backend.upload_blocking(tensor, cpu_mat.span()); + if (!upload_res) + return std::unexpected(upload_res.error()); + + return tensor; +} + +inline Result GpuTensor::create_empty( + std::size_t rows, std::size_t cols, GpuBackend& backend) +{ + // TRANSFER_DST_BIT: 允许 vkCmdFillBuffer (zero) 和 vkCmdCopyBuffer (insert_rows) 写入 + // TRANSFER_SRC_BIT: 允许 vkCmdCopyBuffer (clone/slice_rows) 读取 + auto buf_res = GpuBuffer::create_device_local( + backend.device().device(), backend.memory_pool(), + rows * cols, + TENSOR_BUFFER_USAGE); + if (!buf_res) + return std::unexpected(buf_res.error()); + + return GpuTensor(std::make_shared(std::move(*buf_res)), rows, cols); +} + +inline Result GpuTensor::to_matrix(GpuBackend& backend) const +{ + if (!valid()) + return std::unexpected(Error{"Invalid GpuTensor"}); + + Matrix cpu_mat(rows_, cols_); + auto dl_res = backend.download_blocking(*this, cpu_mat.span()); + if (!dl_res) + return std::unexpected(dl_res.error()); + + return cpu_mat; +} + +} // namespace nn + +#endif // NN_HAS_VULKAN +#endif // NN_GPU_TENSOR_IMPL_HPP diff --git a/include/neuralnet.cpp/backend/memory_pool.hpp b/include/neuralnet.cpp/backend/memory_pool.hpp new file mode 100644 index 00000000..8221f8cf --- /dev/null +++ b/include/neuralnet.cpp/backend/memory_pool.hpp @@ -0,0 +1,319 @@ +// ── memory_pool.hpp ───────────────────────────────────────────────────── +// GPU 内存子分配器 +// +// 职责： +// - 预分配大块 GPU 内存（128MB+） +// - 提供 O(log n) 子分配/释放 +// - 自动合并空闲区域 +// - RAII 管理 VkDeviceMemory +// +// 设计： +// - 每个 Block 是一块大的 VkDeviceMemory +// - 使用 std::set 维护空闲区域（按 offset 排序） +// - 分配时找第一个合适的空闲区域（first fit） +// - 释放时自动合并相邻空闲区域 +// ───────────────────────────────────────────────────────────────────────── + +#ifndef NN_MEMORY_POOL_HPP +#define NN_MEMORY_POOL_HPP + +#ifdef NN_HAS_VULKAN + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../core_errors.hpp" + +namespace nn +{ + +class MemoryPool +{ +public: + static constexpr VkDeviceSize DEFAULT_BLOCK_SIZE = 128ull * 1024 * 1024; // 128MB + + struct Allocation + { + VkDeviceMemory memory = VK_NULL_HANDLE; + VkDeviceSize offset = 0; + VkDeviceSize size = 0; + VkMemoryPropertyFlags property_flags = 0; + + [[nodiscard]] bool valid() const noexcept { return memory != VK_NULL_HANDLE; } + }; + +private: + struct FreeRegion + { + VkDeviceSize offset; + VkDeviceSize size; + + bool operator<(const FreeRegion& o) const noexcept { return offset < o.offset; } + }; + + struct Block + { + VkDeviceMemory memory = VK_NULL_HANDLE; + VkDeviceSize size = 0; + uint32_t memory_type_index = 0; + VkMemoryPropertyFlags property_flags = 0; + std::set free_regions; + std::size_t allocation_count = 0; + VkDevice owning_device = VK_NULL_HANDLE; + + Block() = default; + + Block(Block&& o) noexcept + : memory(o.memory), size(o.size), memory_type_index(o.memory_type_index), + property_flags(o.property_flags), free_regions(std::move(o.free_regions)), + allocation_count(o.allocation_count), owning_device(o.owning_device) + { + o.memory = VK_NULL_HANDLE; + o.size = 0; + o.allocation_count = 0; + o.owning_device = VK_NULL_HANDLE; + } + + // 禁用移动赋值，防止 vector 操作引发意外释放 + Block& operator=(Block&&) = delete; + + ~Block() + { + if (memory != VK_NULL_HANDLE && owning_device != VK_NULL_HANDLE) + vkFreeMemory(owning_device, memory, nullptr); + } + + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + }; + + struct SuballocKey + { + VkDeviceMemory memory; + VkDeviceSize offset; + + bool operator==(const SuballocKey& o) const noexcept + { + return memory == o.memory && offset == o.offset; + } + }; + + struct SuballocKeyHash + { + std::size_t operator()(const SuballocKey& k) const noexcept + { + // Knuth 黄金比例散列常数（64-bit），用于打散相邻 key 的散列值 + constexpr std::size_t KNUTH_GOLDEN = 0x9E3779B97F4A7C15ULL; + return std::hash{}(k.memory) ^ + (std::hash{}(k.offset) * KNUTH_GOLDEN); + } + }; + + VkDevice device_; + VkPhysicalDevice physical_device_; + VkDeviceSize block_size_; + VkPhysicalDeviceMemoryProperties mem_props_; + + // 使用 unique_ptr 避免 vector 扩容/删除时触发 Block 的移动和析构 + std::vector> blocks_; + std::unordered_map active_allocs_; + std::mutex mutex_; + + [[nodiscard]] static std::optional find_free( + Block& block, VkDeviceSize alignment, VkDeviceSize size) + { + for (auto it = block.free_regions.begin(); it != block.free_regions.end(); ++it) + { + VkDeviceSize aligned = (it->offset + alignment - 1) & ~(alignment - 1); + VkDeviceSize padding = aligned - it->offset; + if (it->size >= padding + size) + { + VkDeviceSize result = aligned; + VkDeviceSize remaining = it->size - padding - size; + VkDeviceSize orig_offset = it->offset; + block.free_regions.erase(it); + if (padding > 0) + block.free_regions.insert({orig_offset, padding}); + if (remaining > 0) + block.free_regions.insert({aligned + size, remaining}); + return result; + } + } + return std::nullopt; + } + + // 注：Block 的移动赋值被 delete（防止 vector 操作引发意外释放）， + // 但移动构造可用。此处 `return block;` 经由移动构造构造 std::expected， + // 不触发移动赋值，故可正常编译。 + [[nodiscard]] Result create_block( + uint32_t memory_type_index, VkMemoryPropertyFlags flags, VkDeviceSize size = 0) + { + VkDeviceSize alloc_size = (size > 0) ? size : block_size_; + VkMemoryAllocateInfo alloc_info{}; + alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + alloc_info.allocationSize = alloc_size; + alloc_info.memoryTypeIndex = memory_type_index; + + VkDeviceMemory memory = VK_NULL_HANDLE; + VkResult res = vkAllocateMemory(device_, &alloc_info, nullptr, &memory); + if (res != VK_SUCCESS) + return std::unexpected(Error{"vkAllocateMemory failed: " + std::to_string(res)}); + + Block block; + block.owning_device = device_; + block.memory = memory; + block.size = alloc_size; + block.memory_type_index = memory_type_index; + block.property_flags = flags; + block.free_regions.insert({0, alloc_size}); + return block; + } + + [[nodiscard]] std::optional find_memory_type( + uint32_t type_bits, VkMemoryPropertyFlags preferred, + VkMemoryPropertyFlags fallback = 0) const + { + std::optional best, fb; + for (uint32_t i = 0; i < mem_props_.memoryTypeCount; ++i) + { + if (!(type_bits & (1u << i))) + continue; + auto flags = mem_props_.memoryTypes[i].propertyFlags; + if ((flags & preferred) == preferred && !best) + best = i; + if (fallback && (flags & fallback) == fallback && !fb) + fb = i; + } + return best ? best : fb; + } + +public: + MemoryPool(VkDevice device, VkPhysicalDevice physical_device, + VkDeviceSize block_size = DEFAULT_BLOCK_SIZE) + : device_(device), physical_device_(physical_device), block_size_(block_size) + { + vkGetPhysicalDeviceMemoryProperties(physical_device_, &mem_props_); + } + + ~MemoryPool() { blocks_.clear(); } + + // 禁止拷贝和移动 + MemoryPool(const MemoryPool&) = delete; + MemoryPool& operator=(const MemoryPool&) = delete; + MemoryPool(MemoryPool&&) = delete; + MemoryPool& operator=(MemoryPool&&) = delete; + + [[nodiscard]] Result allocate( + VkMemoryRequirements requirements, + VkMemoryPropertyFlags preferred_flags, + VkMemoryPropertyFlags fallback_flags = 0) + { + std::lock_guard lock(mutex_); + + auto mem_type = find_memory_type(requirements.memoryTypeBits, preferred_flags, fallback_flags); + if (!mem_type) + return std::unexpected(Error{"No suitable GPU memory type found"}); + + VkDeviceSize alignment = requirements.alignment > 0 ? requirements.alignment : 1; + const VkDeviceSize alloc_size = requirements.size; + VkDeviceSize effective_block_size = (alloc_size > block_size_) ? alloc_size : block_size_; + + // 尝试在现有 block 中分配 + for (auto& block_ptr : blocks_) + { + auto& block = *block_ptr; + if (block.memory_type_index != *mem_type) + continue; + auto offset = find_free(block, alignment, alloc_size); + if (offset) + { + block.allocation_count++; + active_allocs_[{block.memory, *offset}] = alloc_size; + return Allocation{block.memory, *offset, alloc_size, block.property_flags}; + } + } + + // 创建新 block + auto block_result = create_block(*mem_type, preferred_flags, effective_block_size); + if (!block_result) + return std::unexpected(block_result.error()); + + auto new_block = std::make_unique(std::move(*block_result)); + auto offset = find_free(*new_block, alignment, alloc_size); + if (!offset) + return std::unexpected(Error{"Suballocation failed in new block"}); + + VkDeviceMemory mem = new_block->memory; + VkMemoryPropertyFlags flags = new_block->property_flags; + new_block->allocation_count++; + active_allocs_[{mem, *offset}] = alloc_size; + blocks_.push_back(std::move(new_block)); + + return Allocation{mem, *offset, alloc_size, flags}; + } + + void free(const Allocation& alloc) + { + if (!alloc.valid()) + return; + + std::lock_guard lock(mutex_); + SuballocKey key{alloc.memory, alloc.offset}; + auto active_it = active_allocs_.find(key); + if (active_it == active_allocs_.end()) + return; + + for (auto& block_ptr : blocks_) + { + auto& block = *block_ptr; + if (block.memory != alloc.memory) + continue; + + FreeRegion new_reg{alloc.offset, alloc.size}; + auto ins_it = block.free_regions.insert(new_reg).first; + + // 向前合并 + if (ins_it != block.free_regions.begin()) + { + auto prev = std::prev(ins_it); + if (prev->offset + prev->size == ins_it->offset) + { + FreeRegion merged{prev->offset, prev->size + ins_it->size}; + block.free_regions.erase(prev); + block.free_regions.erase(ins_it); + ins_it = block.free_regions.insert(merged).first; + } + } + + // 向后合并 + auto next = std::next(ins_it); + if (next != block.free_regions.end()) + { + if (ins_it->offset + ins_it->size == next->offset) + { + FreeRegion merged{ins_it->offset, ins_it->size + next->size}; + block.free_regions.erase(next); + block.free_regions.erase(ins_it); + block.free_regions.insert(merged); + } + } + + block.allocation_count--; + active_allocs_.erase(active_it); + return; + } + } +}; + +} // namespace nn + +#endif // NN_HAS_VULKAN +#endif // NN_MEMORY_POOL_HPP diff --git a/include/neuralnet.cpp/backend/staging_ring.hpp b/include/neuralnet.cpp/backend/staging_ring.hpp new file mode 100644 index 00000000..30159a35 --- /dev/null +++ b/include/neuralnet.cpp/backend/staging_ring.hpp @@ -0,0 +1,253 @@ +// ── staging_ring.hpp ───────────────────────────────────────────────────── +// 环形 Staging 缓冲区 +// +// 职责： +// - 管理 HOST_VISIBLE 缓冲区用于 CPU↔GPU 数据传输 +// - 环形分配，避免频繁创建/销毁 +// - Fence 同步确保数据安全 +// +// 设计（依据性能审查报告优化）： +// - 预分配 N 个 region（默认 2 个，每个 64MB → 总 128MB） +// - 旧默认 4×256MB=1GB 预分配过大；实际训练单次 PCIe 传输量 +// 通常 << 64MB（最大 token_emb 上传约 5MB） +// - 内存预算上限保留 max_host_visible / 16，避免在小显存机器上过度分配 +// ───────────────────────────────────────────────────────────────────────── + +#ifndef NN_STAGING_RING_HPP +#define NN_STAGING_RING_HPP + +#ifdef NN_HAS_VULKAN + +#include +#include +#include +#include +#include + +#include "../core_errors.hpp" +#include "../core_observer_ptr.hpp" +#include "../config.hpp" +#include "memory_pool.hpp" + +namespace nn +{ + +class StagingRing +{ +public: + // 默认 64MB × 2 region（实际分配大小由 initialize() 动态计算， + // 会根据 host-visible 显存大小自动扩容，此值仅作为下限参考） + static constexpr std::size_t DEFAULT_REGION_SIZE = 64ull * 1024 * 1024; // 64MB + static constexpr std::size_t DEFAULT_NUM_REGIONS = 2; + // host-visible 显存预算占比的倒数（实际取 1/HOST_VISIBLE_FRACTION） + static constexpr VkDeviceSize HOST_VISIBLE_FRACTION = 16; + // 动态计算 staging 大小的下限，避免小显存机器分配过小 + static constexpr VkDeviceSize MIN_REGION_SIZE = 32ull * 1024 * 1024; + +private: + // Staging region 内部结构（外部通过索引访问，无需直接使用此类型） + struct Region + { + MemoryPool::Allocation alloc; + VkBuffer buffer = VK_NULL_HANDLE; + void* mapped_ptr = nullptr; + VkFence fence = VK_NULL_HANDLE; + bool in_flight = false; + }; + + VkDevice device_ = VK_NULL_HANDLE; + VkPhysicalDevice physical_device_ = VK_NULL_HANDLE; + VkCommandPool cmd_pool_ = VK_NULL_HANDLE; + observer_ptr pool_; + std::vector regions_; + std::size_t region_size_; + std::atomic current_{0}; + +public: + StagingRing() = default; + + [[nodiscard]] Result initialize( + VkDevice device, VkPhysicalDevice physical_device, + VkCommandPool cmd_pool, MemoryPool& pool, + std::size_t region_size = DEFAULT_REGION_SIZE, + std::size_t num_regions = DEFAULT_NUM_REGIONS) + { + device_ = device; + physical_device_ = physical_device; + cmd_pool_ = cmd_pool; + pool_.reset(&pool); + + // 动态计算 Staging 大小：取 host-visible 显存的 1/16，下限 32MB + // 用户传入的 region_size 作为额外参考（取两者中较大值）。 + VkPhysicalDeviceMemoryProperties mem_props; + vkGetPhysicalDeviceMemoryProperties(physical_device, &mem_props); + VkDeviceSize max_host_visible = 0; + for (uint32_t i = 0; i < mem_props.memoryTypeCount; ++i) + { + if (mem_props.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) + { + uint32_t heap_idx = mem_props.memoryTypes[i].heapIndex; + if (mem_props.memoryHeaps[heap_idx].size > max_host_visible) + max_host_visible = mem_props.memoryHeaps[heap_idx].size; + } + } + + VkDeviceSize calculated_size = max_host_visible / HOST_VISIBLE_FRACTION; + if (calculated_size < MIN_REGION_SIZE) + calculated_size = MIN_REGION_SIZE; + + // 最终取 用户请求值 与 动态计算值 的较大者 + // （不再人为封顶 64MB，避免大 batch / 大 vocab 时上传失败） + region_size_ = std::max( + static_cast(region_size), calculated_size); + + regions_.resize(num_regions); + for (std::size_t i = 0; i < num_regions; ++i) + { + auto& r = regions_[i]; + + // 创建 buffer + VkBufferCreateInfo buf_info{}; + buf_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buf_info.size = region_size_; + buf_info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + buf_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + + VkResult res = vkCreateBuffer(device_, &buf_info, nullptr, &r.buffer); + if (res != VK_SUCCESS) + return std::unexpected(Error{"vkCreateBuffer failed: " + std::to_string(res)}); + + // 分配内存 + VkMemoryRequirements mem_reqs; + vkGetBufferMemoryRequirements(device_, r.buffer, &mem_reqs); + + auto alloc_r = pool_->allocate( + mem_reqs, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + if (!alloc_r) + return std::unexpected(alloc_r.error()); + r.alloc = *alloc_r; + + // 绑定内存 + res = vkBindBufferMemory(device_, r.buffer, r.alloc.memory, r.alloc.offset); + if (res != VK_SUCCESS) + return std::unexpected(Error{"vkBindBufferMemory failed: " + std::to_string(res)}); + + // 映射内存 + res = vkMapMemory(device_, r.alloc.memory, r.alloc.offset, r.alloc.size, 0, &r.mapped_ptr); + if (res != VK_SUCCESS) + return std::unexpected(Error{"vkMapMemory failed: " + std::to_string(res)}); + + // 创建 fence + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + fence_info.flags = VK_FENCE_CREATE_SIGNALED_BIT; // 初始状态为 signaled + res = vkCreateFence(device_, &fence_info, nullptr, &r.fence); + if (res != VK_SUCCESS) + return std::unexpected(Error{"vkCreateFence failed: " + std::to_string(res)}); + } + + return {}; + } + + ~StagingRing() + { + if (device_ == VK_NULL_HANDLE) + return; + + for (auto& r : regions_) + { + if (r.fence != VK_NULL_HANDLE) + vkDestroyFence(device_, r.fence, nullptr); + if (r.buffer != VK_NULL_HANDLE) + vkDestroyBuffer(device_, r.buffer, nullptr); + // MemoryPool 会自动释放内存 + } + } + + // 禁止拷贝和移动 + StagingRing(const StagingRing&) = delete; + StagingRing& operator=(const StagingRing&) = delete; + StagingRing(StagingRing&&) = delete; + StagingRing& operator=(StagingRing&&) = delete; + + // 获取下一个可用 region 的索引 + [[nodiscard]] std::size_t acquire() + { + std::size_t idx = current_.fetch_add(1, std::memory_order_relaxed) % regions_.size(); + auto& r = regions_[idx]; + + // 等待该 region 的 fence（如果正在使用中） + if (r.in_flight) + { + vkWaitForFences(device_, 1, &r.fence, VK_TRUE, UINT64_MAX); + vkResetFences(device_, 1, &r.fence); + r.in_flight = false; + } + + return idx; + } + + // 上传数据到 staging region + [[nodiscard]] Result upload( + std::size_t region_idx, std::span data, VkDeviceSize offset = 0) + { + if (region_idx >= regions_.size()) + return std::unexpected(Error{"Invalid region index"}); + + auto& r = regions_[region_idx]; + const std::size_t byte_size = data.size() * sizeof(Scalar); + + if (offset + byte_size > region_size_) + return std::unexpected(Error{"Upload exceeds staging region size"}); + + std::memcpy(static_cast(r.mapped_ptr) + offset, data.data(), byte_size); + return {}; + } + + // 从 staging region 下载数据 + [[nodiscard]] Result download( + std::size_t region_idx, std::span data, VkDeviceSize offset = 0) + { + if (region_idx >= regions_.size()) + return std::unexpected(Error{"Invalid region index"}); + + auto& r = regions_[region_idx]; + const std::size_t byte_size = data.size() * sizeof(Scalar); + + if (offset + byte_size > region_size_) + return std::unexpected(Error{"Download exceeds staging region size"}); + + std::memcpy(data.data(), static_cast(r.mapped_ptr) + offset, byte_size); + return {}; + } + + // 获取缓冲区句柄 + [[nodiscard]] VkBuffer buffer(std::size_t region_idx) const noexcept + { + return regions_[region_idx].buffer; + } + + // 获取 fence + [[nodiscard]] VkFence fence(std::size_t region_idx) const noexcept + { + return regions_[region_idx].fence; + } + + // 标记 region 为正在使用 + void mark_in_flight(std::size_t region_idx) noexcept + { + regions_[region_idx].in_flight = true; + } + + // 获取 region 大小 + [[nodiscard]] std::size_t region_size() const noexcept { return region_size_; } + + // 获取 region 数量 + [[nodiscard]] std::size_t num_regions() const noexcept { return regions_.size(); } +}; + +} // namespace nn + +#endif // NN_HAS_VULKAN +#endif // NN_STAGING_RING_HPP diff --git a/include/neuralnet.cpp/backend/vk_backend.hpp b/include/neuralnet.cpp/backend/vk_backend.hpp new file mode 100644 index 00000000..30cb15d3 --- /dev/null +++ b/include/neuralnet.cpp/backend/vk_backend.hpp @@ -0,0 +1,3148 @@ +// ── vk_backend.hpp ────────────────────────────────────────────────────── +// Vulkan Compute Backend +// +// 职责： +// - 管理 Vulkan 实例、设备、队列（RAII） +// - 管理 MemoryPool 子分配器 +// - 管理 StagingRing 环形缓冲区 +// - 提供 GPU 操作 API（matmul、elementwise） +// +// 同步机制： +// - 每个 matmul_gpu/elementwise_gpu 调用都创建 fence +// - 提交后立即等待 fence，确保 GPU 计算完成 +// - 完成后清理资源（command buffer、descriptor set） +// +// 双轨制架构： +// - Staging Path：CPU span → Staging → GPU → 计算 → GPU → Staging → CPU span +// - GPU-Resident Path：GpuTensor → 计算 → GpuTensor（全程 GPU 显存） +// ───────────────────────────────────────────────────────────────────────── + +#ifndef NN_VK_BACKEND_HPP +#define NN_VK_BACKEND_HPP + +#ifdef NN_HAS_VULKAN + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../core_errors.hpp" +#include "../core_observer_ptr.hpp" +#include "../config.hpp" +#include "memory_pool.hpp" +#include "staging_ring.hpp" + +// SPIR-V 嵌入头文件（由 CMake 生成） +#if __has_include("matmul_spv.hpp") +#include "matmul_spv.hpp" +#define NN_MATMUL_SPV_EMBEDDED +#endif + +#if __has_include("matmul_tiled_spv.hpp") +#include "matmul_tiled_spv.hpp" +#define NN_MATMUL_TILED_SPV_EMBEDDED +#endif + +#if __has_include("batched_matmul_spv.hpp") +#include "batched_matmul_spv.hpp" +#define NN_BATCHED_MATMUL_SPV_EMBEDDED +#endif + +#if __has_include("rearrange_3d_spv.hpp") +#include "rearrange_3d_spv.hpp" +#define NN_REARRANGE_3D_SPV_EMBEDDED +#endif + +#if __has_include("elementwise_v2_spv.hpp") +#include "elementwise_v2_spv.hpp" +#define NN_ELEMENTWISE_V2_SPV_EMBEDDED +#endif + +#if __has_include("reduce_spv.hpp") +#include "reduce_spv.hpp" +#define NN_REDUCE_SPV_EMBEDDED +#endif + +#if __has_include("broadcast_spv.hpp") +#include "broadcast_spv.hpp" +#define NN_BROADCAST_SPV_EMBEDDED +#endif + +#if __has_include("transpose_spv.hpp") +#include "transpose_spv.hpp" +#define NN_TRANSPOSE_SPV_EMBEDDED +#endif + +#if __has_include("gather_spv.hpp") +#include "gather_spv.hpp" +#define NN_GATHER_SPV_EMBEDDED +#endif + +#if __has_include("scatter_add_spv.hpp") +#include "scatter_add_spv.hpp" +#define NN_SCATTER_ADD_SPV_EMBEDDED +#endif + +namespace nn +{ + +// 前向声明 +class GpuTensor; + +namespace detail +{ + +// ── Vulkan 错误检查 ────────────────────────────────────────────────────── +[[nodiscard]] inline Result vk_check(VkResult res, const char* file, int line) +{ + if (res != VK_SUCCESS) + { + return std::unexpected(Error{ + "Vulkan error " + std::to_string(static_cast(res)) + + " at " + std::string(file) + ":" + std::to_string(line)}); + } + return {}; +} + +// ── CPU Scalar ↔ GPU float 转换 ───────────────────────────────────────── +inline void convert_scalar_to_float( + std::span src, float* __restrict dst) noexcept +{ + if constexpr (std::is_same_v) + std::memcpy(dst, src.data(), src.size() * sizeof(float)); + else + std::transform(src.begin(), src.end(), dst, + [](Scalar v) { return static_cast(v); }); +} + +inline void convert_float_to_scalar( + const float* __restrict src, std::span dst) noexcept +{ + if constexpr (std::is_same_v) + std::memcpy(dst.data(), src, dst.size() * sizeof(float)); + else + std::transform(src, src + dst.size(), dst.begin(), + [](float v) { return static_cast(v); }); +} + +} // namespace detail + +// ══════════════════════════════════════════════════════════════════════════ +// VulkanDevice — 设备与队列管理（RAII） +// ══════════════════════════════════════════════════════════════════════════ + +class VulkanDevice +{ +private: + VkInstance instance_ = VK_NULL_HANDLE; + VkPhysicalDevice physical_device_ = VK_NULL_HANDLE; + VkDevice device_ = VK_NULL_HANDLE; + VkQueue compute_queue_ = VK_NULL_HANDLE; + uint32_t queue_family_index_ = 0; + bool initialized_ = false; + +public: + VulkanDevice() = default; + + ~VulkanDevice() + { + if (device_ != VK_NULL_HANDLE) + vkDestroyDevice(device_, nullptr); + if (instance_ != VK_NULL_HANDLE) + vkDestroyInstance(instance_, nullptr); + } + + // 禁止拷贝和移动 + VulkanDevice(const VulkanDevice&) = delete; + VulkanDevice& operator=(const VulkanDevice&) = delete; + VulkanDevice(VulkanDevice&&) = delete; + VulkanDevice& operator=(VulkanDevice&&) = delete; + + [[nodiscard]] Result initialize() + { + if (initialized_) + return {}; + + // 1. 创建 VkInstance + VkApplicationInfo app_info{}; + app_info.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; + app_info.pApplicationName = "neuralnet.cpp"; + app_info.applicationVersion = VK_MAKE_VERSION(1, 0, 0); + app_info.pEngineName = "neuralnet.cpp"; + app_info.engineVersion = VK_MAKE_VERSION(1, 0, 0); + app_info.apiVersion = VK_API_VERSION_1_0; + + VkInstanceCreateInfo instance_info{}; + instance_info.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; + instance_info.pApplicationInfo = &app_info; + + VkResult res = vkCreateInstance(&instance_info, nullptr, &instance_); + if (res != VK_SUCCESS) + return std::unexpected(Error{"vkCreateInstance failed: " + std::to_string(res)}); + + // 2. 选择物理设备 + uint32_t device_count = 0; + vkEnumeratePhysicalDevices(instance_, &device_count, nullptr); + if (device_count == 0) + return std::unexpected(Error{"No Vulkan devices found"}); + + std::vector devices(device_count); + vkEnumeratePhysicalDevices(instance_, &device_count, devices.data()); + + // 优先选择独立显卡 + for (const auto& dev : devices) + { + VkPhysicalDeviceProperties props; + vkGetPhysicalDeviceProperties(dev, &props); + if (props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) + { + physical_device_ = dev; + break; + } + } + // 如果没有独立显卡，使用第一个设备 + if (physical_device_ == VK_NULL_HANDLE) + physical_device_ = devices[0]; + + // 3. 查找计算队列族 + uint32_t queue_family_count = 0; + vkGetPhysicalDeviceQueueFamilyProperties(physical_device_, &queue_family_count, nullptr); + + std::vector queue_families(queue_family_count); + vkGetPhysicalDeviceQueueFamilyProperties(physical_device_, &queue_family_count, queue_families.data()); + + bool found_queue = false; + for (uint32_t i = 0; i < queue_family_count; ++i) + { + if (queue_families[i].queueFlags & VK_QUEUE_COMPUTE_BIT) + { + queue_family_index_ = i; + found_queue = true; + break; + } + } + if (!found_queue) + return std::unexpected(Error{"No compute queue family found"}); + + // 4. 创建逻辑设备 + float queue_priority = 1.0f; + VkDeviceQueueCreateInfo queue_info{}; + queue_info.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + queue_info.queueFamilyIndex = queue_family_index_; + queue_info.queueCount = 1; + queue_info.pQueuePriorities = &queue_priority; + + VkDeviceCreateInfo device_info{}; + device_info.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO; + device_info.queueCreateInfoCount = 1; + device_info.pQueueCreateInfos = &queue_info; + + res = vkCreateDevice(physical_device_, &device_info, nullptr, &device_); + if (res != VK_SUCCESS) + return std::unexpected(Error{"vkCreateDevice failed: " + std::to_string(res)}); + + // 5. 获取计算队列 + vkGetDeviceQueue(device_, queue_family_index_, 0, &compute_queue_); + + initialized_ = true; + return {}; + } + + [[nodiscard]] VkDevice device() const noexcept { return device_; } + [[nodiscard]] VkPhysicalDevice physical_device() const noexcept { return physical_device_; } + [[nodiscard]] VkQueue compute_queue() const noexcept { return compute_queue_; } + [[nodiscard]] uint32_t queue_family_index() const noexcept { return queue_family_index_; } + [[nodiscard]] bool is_initialized() const noexcept { return initialized_; } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// VulkanPipeline — Pipeline 管理（RAII） +// ══════════════════════════════════════════════════════════════════════════ + +class VulkanPipeline +{ +private: + VkDevice device_ = VK_NULL_HANDLE; + VkShaderModule shader_module_ = VK_NULL_HANDLE; + VkDescriptorSetLayout descriptor_layout_ = VK_NULL_HANDLE; + VkPipelineLayout pipeline_layout_ = VK_NULL_HANDLE; + VkPipeline pipeline_ = VK_NULL_HANDLE; + +public: + VulkanPipeline() = default; + + ~VulkanPipeline() + { + if (pipeline_ != VK_NULL_HANDLE) + vkDestroyPipeline(device_, pipeline_, nullptr); + if (pipeline_layout_ != VK_NULL_HANDLE) + vkDestroyPipelineLayout(device_, pipeline_layout_, nullptr); + if (descriptor_layout_ != VK_NULL_HANDLE) + vkDestroyDescriptorSetLayout(device_, descriptor_layout_, nullptr); + if (shader_module_ != VK_NULL_HANDLE) + vkDestroyShaderModule(device_, shader_module_, nullptr); + } + + // 移动语义 + VulkanPipeline(VulkanPipeline&& o) noexcept + : device_(o.device_), shader_module_(o.shader_module_), + descriptor_layout_(o.descriptor_layout_), + pipeline_layout_(o.pipeline_layout_), pipeline_(o.pipeline_) + { + o.device_ = VK_NULL_HANDLE; + o.shader_module_ = VK_NULL_HANDLE; + o.descriptor_layout_ = VK_NULL_HANDLE; + o.pipeline_layout_ = VK_NULL_HANDLE; + o.pipeline_ = VK_NULL_HANDLE; + } + + VulkanPipeline& operator=(VulkanPipeline&& o) noexcept + { + if (this != &o) + { + std::swap(device_, o.device_); + std::swap(shader_module_, o.shader_module_); + std::swap(descriptor_layout_, o.descriptor_layout_); + std::swap(pipeline_layout_, o.pipeline_layout_); + std::swap(pipeline_, o.pipeline_); + } + return *this; + } + + // 禁止拷贝 + VulkanPipeline(const VulkanPipeline&) = delete; + VulkanPipeline& operator=(const VulkanPipeline&) = delete; + + // 创建 matmul pipeline + [[nodiscard]] static Result create_matmul( + VkDevice device, std::span spirv_code) + { + VulkanPipeline pl; + pl.device_ = device; + + // 创建 shader module + VkShaderModuleCreateInfo module_info{}; + module_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + module_info.codeSize = spirv_code.size_bytes(); + module_info.pCode = spirv_code.data(); + + auto r = detail::vk_check( + vkCreateShaderModule(device, &module_info, nullptr, &pl.shader_module_), + __FILE__, __LINE__); + if (!r) + return std::unexpected(r.error()); + + // 创建 descriptor set layout（3 个 storage buffer） + VkDescriptorSetLayoutBinding bindings[3]{}; + for (int i = 0; i < 3; ++i) + { + bindings[i].binding = static_cast(i); + bindings[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + bindings[i].descriptorCount = 1; + bindings[i].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + } + + VkDescriptorSetLayoutCreateInfo layout_info{}; + layout_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; + layout_info.bindingCount = 3; + layout_info.pBindings = bindings; + + r = detail::vk_check( + vkCreateDescriptorSetLayout(device, &layout_info, nullptr, &pl.descriptor_layout_), + __FILE__, __LINE__); + if (!r) + return std::unexpected(r.error()); + + // 创建 pipeline layout（push constants: M, N, K, transA, transB） + VkPushConstantRange push_range{}; + push_range.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + push_range.offset = 0; + push_range.size = 5 * sizeof(uint32_t); + + VkPipelineLayoutCreateInfo pl_layout_info{}; + pl_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + pl_layout_info.setLayoutCount = 1; + pl_layout_info.pSetLayouts = &pl.descriptor_layout_; + pl_layout_info.pushConstantRangeCount = 1; + pl_layout_info.pPushConstantRanges = &push_range; + + r = detail::vk_check( + vkCreatePipelineLayout(device, &pl_layout_info, nullptr, &pl.pipeline_layout_), + __FILE__, __LINE__); + if (!r) + return std::unexpected(r.error()); + + // 创建 compute pipeline + VkComputePipelineCreateInfo pipeline_info{}; + pipeline_info.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + pipeline_info.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + pipeline_info.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + pipeline_info.stage.module = pl.shader_module_; + pipeline_info.stage.pName = "main"; + pipeline_info.layout = pl.pipeline_layout_; + + r = detail::vk_check( + vkCreateComputePipelines(device, VK_NULL_HANDLE, 1, &pipeline_info, nullptr, &pl.pipeline_), + __FILE__, __LINE__); + if (!r) + return std::unexpected(r.error()); + + return pl; + } + + // 创建通用 pipeline（可变 binding 数和 push constant 大小） + // 用于 elementwise_v2 / reduce / broadcast 等纯原语着色器 + [[nodiscard]] static Result create_generic( + VkDevice device, std::span spirv_code, + uint32_t num_bindings, uint32_t push_constant_size) + { + VulkanPipeline pl; + pl.device_ = device; + + // 创建 shader module + VkShaderModuleCreateInfo module_info{}; + module_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + module_info.codeSize = spirv_code.size_bytes(); + module_info.pCode = spirv_code.data(); + + auto r = detail::vk_check( + vkCreateShaderModule(device, &module_info, nullptr, &pl.shader_module_), + __FILE__, __LINE__); + if (!r) + return std::unexpected(r.error()); + + // 创建 descriptor set layout（num_bindings 个 storage buffer） + std::vector bindings(num_bindings); + for (uint32_t i = 0; i < num_bindings; ++i) + { + bindings[i].binding = i; + bindings[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + bindings[i].descriptorCount = 1; + bindings[i].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + } + + VkDescriptorSetLayoutCreateInfo layout_info{}; + layout_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; + layout_info.bindingCount = num_bindings; + layout_info.pBindings = bindings.data(); + + r = detail::vk_check( + vkCreateDescriptorSetLayout(device, &layout_info, nullptr, &pl.descriptor_layout_), + __FILE__, __LINE__); + if (!r) + return std::unexpected(r.error()); + + // 创建 pipeline layout（push constants） + VkPushConstantRange push_range{}; + push_range.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + push_range.offset = 0; + push_range.size = push_constant_size; + + VkPipelineLayoutCreateInfo pl_layout_info{}; + pl_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + pl_layout_info.setLayoutCount = 1; + pl_layout_info.pSetLayouts = &pl.descriptor_layout_; + pl_layout_info.pushConstantRangeCount = (push_constant_size > 0) ? 1u : 0u; + pl_layout_info.pPushConstantRanges = &push_range; + + r = detail::vk_check( + vkCreatePipelineLayout(device, &pl_layout_info, nullptr, &pl.pipeline_layout_), + __FILE__, __LINE__); + if (!r) + return std::unexpected(r.error()); + + // 创建 compute pipeline + VkComputePipelineCreateInfo pipeline_info{}; + pipeline_info.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + pipeline_info.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + pipeline_info.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + pipeline_info.stage.module = pl.shader_module_; + pipeline_info.stage.pName = "main"; + pipeline_info.layout = pl.pipeline_layout_; + + r = detail::vk_check( + vkCreateComputePipelines(device, VK_NULL_HANDLE, 1, &pipeline_info, nullptr, &pl.pipeline_), + __FILE__, __LINE__); + if (!r) + return std::unexpected(r.error()); + + return pl; + } + + [[nodiscard]] VkPipeline handle() const noexcept { return pipeline_; } + [[nodiscard]] VkPipelineLayout pipeline_layout() const noexcept { return pipeline_layout_; } + [[nodiscard]] VkDescriptorSetLayout descriptor_layout() const noexcept { return descriptor_layout_; } +}; + +// 前向声明 +class Matrix; + +// ══════════════════════════════════════════════════════════════════════════ +// GpuBuffer — GPU 缓冲区 RAII 封装 +// ══════════════════════════════════════════════════════════════════════════ + +class GpuBuffer +{ +private: + VkDevice device_ = VK_NULL_HANDLE; + VkBuffer buffer_ = VK_NULL_HANDLE; + MemoryPool::Allocation alloc_; + observer_ptr pool_; + +public: + GpuBuffer() = default; + + GpuBuffer(VkDevice device, VkBuffer buffer, const MemoryPool::Allocation& alloc, MemoryPool& pool) + : device_(device), buffer_(buffer), alloc_(alloc), pool_(pool) {} + + // 析构定义在 GpuBackend 之后：batch 录制期间已录制的 descriptor 仍引用 + // 本 buffer，需延迟到 end_batch 提交完成后再销毁（见 defer_buffer_destroy）。 + ~GpuBuffer(); + + // 移动语义 + GpuBuffer(GpuBuffer&& o) noexcept + : device_(o.device_), buffer_(o.buffer_), alloc_(o.alloc_), pool_(o.pool_) + { + o.device_ = VK_NULL_HANDLE; + o.buffer_ = VK_NULL_HANDLE; + o.alloc_ = {}; + o.pool_.reset(); + } + + GpuBuffer& operator=(GpuBuffer&& o) noexcept + { + if (this != &o) + { + std::swap(device_, o.device_); + std::swap(buffer_, o.buffer_); + std::swap(alloc_, o.alloc_); + pool_.swap(o.pool_); + } + return *this; + } + + // 禁止拷贝 + GpuBuffer(const GpuBuffer&) = delete; + GpuBuffer& operator=(const GpuBuffer&) = delete; + + // 创建 Device Local 缓冲区 + [[nodiscard]] static Result create_device_local( + VkDevice device, MemoryPool& pool, + std::size_t elem_count, VkBufferUsageFlags usage) + { + VkBufferCreateInfo buf_info{}; + buf_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buf_info.size = elem_count * sizeof(float); + buf_info.usage = usage; + buf_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + + VkBuffer buffer = VK_NULL_HANDLE; + VkResult res = vkCreateBuffer(device, &buf_info, nullptr, &buffer); + if (res != VK_SUCCESS) + return std::unexpected(Error{"vkCreateBuffer failed: " + std::to_string(res)}); + + VkMemoryRequirements mem_reqs; + vkGetBufferMemoryRequirements(device, buffer, &mem_reqs); + + auto alloc_r = pool.allocate( + mem_reqs, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + if (!alloc_r) + { + vkDestroyBuffer(device, buffer, nullptr); + return std::unexpected(alloc_r.error()); + } + + res = vkBindBufferMemory(device, buffer, alloc_r->memory, alloc_r->offset); + if (res != VK_SUCCESS) + { + pool.free(*alloc_r); + vkDestroyBuffer(device, buffer, nullptr); + return std::unexpected(Error{"vkBindBufferMemory failed: " + std::to_string(res)}); + } + + return GpuBuffer(device, buffer, *alloc_r, pool); + } + + [[nodiscard]] VkBuffer impl() const noexcept { return buffer_; } + [[nodiscard]] bool valid() const noexcept { return buffer_ != VK_NULL_HANDLE; } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// GpuTensor — GPU 矩阵抽象 +// ══════════════════════════════════════════════════════════════════════════ + +class GpuTensor +{ +private: + std::shared_ptr buffer_; + std::size_t rows_ = 0; + std::size_t cols_ = 0; + +public: + GpuTensor() = default; + + GpuTensor(std::shared_ptr buffer, std::size_t rows, std::size_t cols) + : buffer_(std::move(buffer)), rows_(rows), cols_(cols) {} + + // 从 CPU Matrix 创建（上传数据） + [[nodiscard]] static Result from_matrix( + const Matrix& cpu_mat, class GpuBackend& backend); + + // 创建空的 GPU Tensor（用于输出） + [[nodiscard]] static Result create_empty( + std::size_t rows, std::size_t cols, class GpuBackend& backend); + + // 转换为 CPU Matrix（下载数据） + [[nodiscard]] Result to_matrix(class GpuBackend& backend) const; + + // 访问器 + [[nodiscard]] std::size_t rows() const noexcept { return rows_; } + [[nodiscard]] std::size_t cols() const noexcept { return cols_; } + [[nodiscard]] bool valid() const noexcept { return buffer_ && buffer_->valid(); } + [[nodiscard]] const GpuBuffer& buffer() const noexcept { return *buffer_; } + + // 零拷贝 reshape：共享底层 buffer，只改变形状元数据 + [[nodiscard]] GpuTensor with_shape(std::size_t new_rows, std::size_t new_cols) const + { + return GpuTensor(buffer_, new_rows, new_cols); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// GpuBackend — Vulkan 计算后端单例 +// ══════════════════════════════════════════════════════════════════════════ + +class GpuBackend +{ +private: + VulkanDevice device_; + VulkanPipeline matmul_pipeline_; + VulkanPipeline matmul_tiled_pipeline_; + VulkanPipeline batched_matmul_pipeline_; + VulkanPipeline elementwise_v2_pipeline_; + VulkanPipeline reduce_pipeline_; + VulkanPipeline broadcast_pipeline_; + VulkanPipeline rearrange_3d_pipeline_; + VulkanPipeline transpose_pipeline_; + VulkanPipeline gather_pipeline_; + VulkanPipeline scatter_add_pipeline_; + + std::unique_ptr memory_pool_; + std::unique_ptr staging_ring_; + + VkCommandPool command_pool_ = VK_NULL_HANDLE; + VkDescriptorPool dispatch_pool_ = VK_NULL_HANDLE; + VkDescriptorPool gpu_tensor_pool_ = VK_NULL_HANDLE; + + // ── Command Buffer Batching：将所有 GPU 操作录制到同一个 command buffer ── + // 消除 per-layer fence wait，只在 end_batch() 时提交一次、等一次 + VkCommandBuffer batch_cmd_ = VK_NULL_HANDLE; + VkFence batch_fence_ = VK_NULL_HANDLE; + bool batch_mode_ = false; + std::vector batch_desc_sets_; // batch 期间分配的描述符集 + + std::mutex init_mutex_; + std::mutex queue_mutex_; + bool initialized_ = false; + + // ── 延迟销毁：batch 录制期间 copy-on-write 替换旧 buffer 时，旧 buffer + // 仍被已录制的 descriptor set 引用，不能立即 vkDestroyBuffer。 + // end_batch/flush_batch 提交完成并释放 descriptor sets 后统一销毁。 + struct PendingDestroy + { + VkDevice device = VK_NULL_HANDLE; + VkBuffer buffer = VK_NULL_HANDLE; + MemoryPool::Allocation alloc; + observer_ptr pool; + }; + std::mutex pending_mutex_; + std::vector pending_destroys_; + + static constexpr uint32_t WORKGROUP_SIZE = 16; + + GpuBackend() = default; + + // 获取 SPIR-V 字节码 + [[nodiscard]] static const std::vector& get_matmul_spirv() + { +#ifdef NN_MATMUL_SPV_EMBEDDED + return nn_matmul_spirv_bytecode(); +#else + static const std::vector empty; + return empty; +#endif + } + + [[nodiscard]] static const std::vector& get_matmul_tiled_spirv() + { +#ifdef NN_MATMUL_TILED_SPV_EMBEDDED + return nn_matmul_tiled_spirv_bytecode(); +#else + static const std::vector empty; + return empty; +#endif + } + + [[nodiscard]] static const std::vector& get_batched_matmul_spirv() + { +#ifdef NN_BATCHED_MATMUL_SPV_EMBEDDED + return nn_batched_matmul_spirv_bytecode(); +#else + static const std::vector empty; + return empty; +#endif + } + + [[nodiscard]] static const std::vector& get_rearrange_3d_spirv() + { +#ifdef NN_REARRANGE_3D_SPV_EMBEDDED + return nn_rearrange_3d_spirv_bytecode(); +#else + static const std::vector empty; + return empty; +#endif + } + + [[nodiscard]] static const std::vector& get_elementwise_v2_spirv() + { +#ifdef NN_ELEMENTWISE_V2_SPV_EMBEDDED + return nn_elementwise_v2_spirv_bytecode(); +#else + static const std::vector empty; + return empty; +#endif + } + + [[nodiscard]] static const std::vector& get_reduce_spirv() + { +#ifdef NN_REDUCE_SPV_EMBEDDED + return nn_reduce_spirv_bytecode(); +#else + static const std::vector empty; + return empty; +#endif + } + + [[nodiscard]] static const std::vector& get_broadcast_spirv() + { +#ifdef NN_BROADCAST_SPV_EMBEDDED + return nn_broadcast_spirv_bytecode(); +#else + static const std::vector empty; + return empty; +#endif + } + + [[nodiscard]] static const std::vector& get_transpose_spirv() + { +#ifdef NN_TRANSPOSE_SPV_EMBEDDED + return nn_transpose_spirv_bytecode(); +#else + static const std::vector empty; + return empty; +#endif + } + + [[nodiscard]] static const std::vector& get_gather_spirv() + { +#ifdef NN_GATHER_SPV_EMBEDDED + return nn_gather_spirv_bytecode(); +#else + static const std::vector empty; + return empty; +#endif + } + + [[nodiscard]] static const std::vector& get_scatter_add_spirv() + { +#ifdef NN_SCATTER_ADD_SPV_EMBEDDED + return nn_scatter_add_spirv_bytecode(); +#else + static const std::vector empty; + return empty; +#endif + } + +public: + ~GpuBackend() + { + if (device_.device() != VK_NULL_HANDLE) + { + vkDeviceWaitIdle(device_.device()); + + flush_pending_destroys(); + + staging_ring_.reset(); + + if (gpu_tensor_pool_ != VK_NULL_HANDLE) + vkDestroyDescriptorPool(device_.device(), gpu_tensor_pool_, nullptr); + if (dispatch_pool_ != VK_NULL_HANDLE) + vkDestroyDescriptorPool(device_.device(), dispatch_pool_, nullptr); + if (command_pool_ != VK_NULL_HANDLE) + vkDestroyCommandPool(device_.device(), command_pool_, nullptr); + } + + memory_pool_.reset(); + } + + // 设备丢失状态 + bool device_lost_ = false; + [[nodiscard]] bool is_device_lost() const noexcept { return device_lost_; } + + // 禁止拷贝和移动 + GpuBackend(const GpuBackend&) = delete; + GpuBackend& operator=(const GpuBackend&) = delete; + GpuBackend(GpuBackend&&) = delete; + GpuBackend& operator=(GpuBackend&&) = delete; + + [[nodiscard]] static GpuBackend& instance() + { + // 故意泄漏（进程退出时由 OS 回收）：避免静态析构顺序问题—— + // 全局 GpuTensor 的析构可能晚于 backend，届时访问 instance() 会 UB。 + static GpuBackend* backend = new GpuBackend(); + return *backend; + } + + // ── 延迟销毁入口（GpuBuffer 析构时调用）────────────────────────── + // batch 模式下不立即销毁，等待提交完成；否则立即销毁。 + void defer_buffer_destroy(VkDevice device, VkBuffer buffer, + const MemoryPool::Allocation& alloc, + observer_ptr pool) + { + std::lock_guard lock(pending_mutex_); + pending_destroys_.push_back({device, buffer, alloc, pool}); + } + + // ── 统一执行延迟销毁（必须在 batch descriptor sets 释放之后调用）── + void flush_pending_destroys() + { + std::lock_guard lock(pending_mutex_); + for (auto& pd : pending_destroys_) + { + if (pd.buffer != VK_NULL_HANDLE && pd.device != VK_NULL_HANDLE) + vkDestroyBuffer(pd.device, pd.buffer, nullptr); + if (pd.pool && pd.alloc.valid()) + pd.pool->free(pd.alloc); + } + pending_destroys_.clear(); + } + + // 初始化 + [[nodiscard]] Result initialize() + { + std::lock_guard lock(init_mutex_); + if (initialized_) + return {}; + + // 1. 检查 SPIR-V 是否可用 + const auto& spirv = get_matmul_spirv(); + if (spirv.empty()) + return std::unexpected(Error{"matmul SPIR-V bytecode not embedded"}); + + // 2. 初始化 Vulkan 设备 + auto dev_r = device_.initialize(); + if (!dev_r) + return dev_r; + + // 3. 创建 matmul pipeline + auto pl_r = VulkanPipeline::create_matmul(device_.device(), spirv); + if (!pl_r) + return std::unexpected(pl_r.error()); + matmul_pipeline_ = std::move(*pl_r); + + // 4. 创建 memory pool + memory_pool_ = std::make_unique( + device_.device(), device_.physical_device()); + + // 5. 创建 command pool + VkCommandPoolCreateInfo pool_info{}; + pool_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + pool_info.queueFamilyIndex = device_.queue_family_index(); + pool_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + + auto r = detail::vk_check( + vkCreateCommandPool(device_.device(), &pool_info, nullptr, &command_pool_), + __FILE__, __LINE__); + if (!r) + return r; + + // 6. 创建 staging ring + staging_ring_ = std::make_unique(); + auto st_r = staging_ring_->initialize( + device_.device(), device_.physical_device(), command_pool_, *memory_pool_); + if (!st_r) + return st_r; + + // 7. 创建 descriptor pool for staging path + constexpr std::size_t MAX_CACHED_DISPATCHES = 256; + constexpr std::size_t DESCRIPTORS_PER_DISPATCH = 3; + constexpr std::size_t total_descriptors = MAX_CACHED_DISPATCHES * DESCRIPTORS_PER_DISPATCH; + + VkDescriptorPoolSize dispatch_pool_size{}; + dispatch_pool_size.type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + dispatch_pool_size.descriptorCount = static_cast(total_descriptors); + + VkDescriptorPoolCreateInfo dispatch_pool_info{}; + dispatch_pool_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; + dispatch_pool_info.maxSets = static_cast(MAX_CACHED_DISPATCHES); + dispatch_pool_info.poolSizeCount = 1; + dispatch_pool_info.pPoolSizes = &dispatch_pool_size; + dispatch_pool_info.flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT; + + r = detail::vk_check( + vkCreateDescriptorPool(device_.device(), &dispatch_pool_info, nullptr, &dispatch_pool_), + __FILE__, __LINE__); + if (!r) + return r; + + // 8. 创建 descriptor pool for GPU-resident path + // elementwise_v2 需要 4 个描述符/次，按最大值计算 + // batch 模式下描述符集延迟到 end_batch 释放，需足够大以容纳整个 batch + // ViT forward+backward ≈ 230 ops/样本，batch_size=512 时累积 ~118K sets + // 增大到 262144 以留出余量，对应 desc 总量 1M + constexpr std::size_t TENSOR_POOL_SETS = 262144; + constexpr std::size_t TENSOR_POOL_DESCS = TENSOR_POOL_SETS * 4; // elementwise_v2 需要 4 + + VkDescriptorPoolSize tensor_pool_size{}; + tensor_pool_size.type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + tensor_pool_size.descriptorCount = static_cast(TENSOR_POOL_DESCS); + + VkDescriptorPoolCreateInfo tensor_pool_info{}; + tensor_pool_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; + tensor_pool_info.maxSets = static_cast(TENSOR_POOL_SETS); + tensor_pool_info.poolSizeCount = 1; + tensor_pool_info.pPoolSizes = &tensor_pool_size; + tensor_pool_info.flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT; + + r = detail::vk_check( + vkCreateDescriptorPool(device_.device(), &tensor_pool_info, nullptr, &gpu_tensor_pool_), + __FILE__, __LINE__); + if (!r) + return r; + + // 9. 创建 tiled matmul pipeline（可选） + const auto& tiled_spirv = get_matmul_tiled_spirv(); + if (!tiled_spirv.empty()) + { + auto tp_r = VulkanPipeline::create_matmul(device_.device(), tiled_spirv); + if (tp_r) + matmul_tiled_pipeline_ = std::move(*tp_r); + } + + // 9b. 创建 batched matmul pipeline（3 bindings, 5*4=20B push constants） + // 与 matmul 相同的 descriptor/push-constant 布局，可复用 create_matmul + const auto& batched_spirv = get_batched_matmul_spirv(); + if (!batched_spirv.empty()) + { + auto bp_r = VulkanPipeline::create_matmul(device_.device(), batched_spirv); + if (bp_r) + batched_matmul_pipeline_ = std::move(*bp_r); + } + + // 9c. 创建 rearrange_3d pipeline（2 bindings, 5*4=20B push constants） + const auto& rearrange_spirv = get_rearrange_3d_spirv(); + if (!rearrange_spirv.empty()) + { + auto rp_r = VulkanPipeline::create_generic( + device_.device(), rearrange_spirv, 2, 5 * sizeof(uint32_t)); + if (rp_r) + rearrange_3d_pipeline_ = std::move(*rp_r); + } + + // 10. 创建 elementwise_v2 pipeline（4 bindings, 32B push constants） + const auto& elem_v2_spirv = get_elementwise_v2_spirv(); + if (!elem_v2_spirv.empty()) + { + auto ep_r = VulkanPipeline::create_generic( + device_.device(), elem_v2_spirv, 4, 8 * sizeof(uint32_t)); + if (ep_r) + elementwise_v2_pipeline_ = std::move(*ep_r); + } + + // 11. 创建 reduce pipeline（2 bindings, 16B push constants） + const auto& reduce_spirv = get_reduce_spirv(); + if (!reduce_spirv.empty()) + { + auto rp_r = VulkanPipeline::create_generic( + device_.device(), reduce_spirv, 2, 4 * sizeof(uint32_t)); + if (rp_r) + reduce_pipeline_ = std::move(*rp_r); + } + + // 12. 创建 broadcast pipeline（3 bindings, 16B push constants） + const auto& broadcast_spirv = get_broadcast_spirv(); + if (!broadcast_spirv.empty()) + { + auto bp_r = VulkanPipeline::create_generic( + device_.device(), broadcast_spirv, 3, 4 * sizeof(uint32_t)); + if (bp_r) + broadcast_pipeline_ = std::move(*bp_r); + } + + // 13. 创建 transpose pipeline（2 bindings, 8B push constants） + const auto& transpose_spirv = get_transpose_spirv(); + if (!transpose_spirv.empty()) + { + auto tp_r = VulkanPipeline::create_generic( + device_.device(), transpose_spirv, 2, 2 * sizeof(uint32_t)); + if (tp_r) + transpose_pipeline_ = std::move(*tp_r); + } + + // 14. 创建 gather pipeline（3 bindings, 12B push constants） + const auto& gather_spirv = get_gather_spirv(); + if (!gather_spirv.empty()) + { + auto gp_r = VulkanPipeline::create_generic( + device_.device(), gather_spirv, 3, 3 * sizeof(uint32_t)); + if (gp_r) + gather_pipeline_ = std::move(*gp_r); + } + + // 15. 创建 scatter_add pipeline（3 bindings, 12B push constants） + const auto& scatter_add_spirv = get_scatter_add_spirv(); + if (!scatter_add_spirv.empty()) + { + auto sp_r = VulkanPipeline::create_generic( + device_.device(), scatter_add_spirv, 3, 3 * sizeof(uint32_t)); + if (sp_r) + scatter_add_pipeline_ = std::move(*sp_r); + } + + initialized_ = true; + return {}; + } + + [[nodiscard]] bool is_initialized() const noexcept { return initialized_; } + [[nodiscard]] bool gpu_available() const noexcept { return initialized_; } + [[nodiscard]] bool has_tiled_pipeline() const noexcept { return matmul_tiled_pipeline_.handle() != VK_NULL_HANDLE; } + [[nodiscard]] bool has_batched_matmul_pipeline() const noexcept { return batched_matmul_pipeline_.handle() != VK_NULL_HANDLE; } + [[nodiscard]] bool has_rearrange_3d_pipeline() const noexcept { return rearrange_3d_pipeline_.handle() != VK_NULL_HANDLE; } + [[nodiscard]] bool has_elementwise_v2_pipeline() const noexcept { return elementwise_v2_pipeline_.handle() != VK_NULL_HANDLE; } + [[nodiscard]] bool has_reduce_pipeline() const noexcept { return reduce_pipeline_.handle() != VK_NULL_HANDLE; } + [[nodiscard]] bool has_broadcast_pipeline() const noexcept { return broadcast_pipeline_.handle() != VK_NULL_HANDLE; } + [[nodiscard]] bool has_transpose_pipeline() const noexcept { return transpose_pipeline_.handle() != VK_NULL_HANDLE; } + [[nodiscard]] bool has_gather_pipeline() const noexcept { return gather_pipeline_.handle() != VK_NULL_HANDLE; } + [[nodiscard]] bool has_scatter_add_pipeline() const noexcept { return scatter_add_pipeline_.handle() != VK_NULL_HANDLE; } + + [[nodiscard]] VulkanDevice& device() noexcept { return device_; } + [[nodiscard]] MemoryPool& memory_pool() noexcept { return *memory_pool_; } + [[nodiscard]] StagingRing& staging_ring() noexcept { return *staging_ring_; } + [[nodiscard]] VkCommandPool command_pool() const noexcept { return command_pool_; } + [[nodiscard]] VkDescriptorPool gpu_tensor_pool() const noexcept { return gpu_tensor_pool_; } + + // ══════════════════════════════════════════════════════════════════ + // Command Buffer Batching API + // ══════════════════════════════════════════════════════════════════ + // 用法： + // backend.begin_batch(); + // // ... 多次 matmul_gpu / elementwise_gpu / layernorm_gpu 调用 ... + // backend.end_batch(); + // 效果：所有操作录制到同一个 command buffer，一次提交、一次 fence wait。 + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] bool in_batch() const noexcept { return batch_mode_; } + + [[nodiscard]] Result begin_batch() + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (batch_mode_) + return std::unexpected(Error{"Already in batch mode"}); + + // 1. 分配 command buffer + VkCommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cmd_alloc.commandPool = command_pool_; + cmd_alloc.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cmd_alloc.commandBufferCount = 1; + + auto r = detail::vk_check( + vkAllocateCommandBuffers(device_.device(), &cmd_alloc, &batch_cmd_), + __FILE__, __LINE__); + if (!r) return std::unexpected(r.error()); + + // 2. 开始录制 + VkCommandBufferBeginInfo begin_info{}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + + r = detail::vk_check(vkBeginCommandBuffer(batch_cmd_, &begin_info), __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + batch_cmd_ = VK_NULL_HANDLE; + return std::unexpected(r.error()); + } + + // 3. 创建 fence + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + r = detail::vk_check( + vkCreateFence(device_.device(), &fence_info, nullptr, &batch_fence_), + __FILE__, __LINE__); + if (!r) + { + vkEndCommandBuffer(batch_cmd_); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + batch_cmd_ = VK_NULL_HANDLE; + return std::unexpected(r.error()); + } + + batch_mode_ = true; + batch_desc_sets_.clear(); + return {}; + } + + [[nodiscard]] Result end_batch() + { + if (!batch_mode_) + return {}; // 容错：非 batch 模式时 no-op + + // 1. 结束录制 + auto r = detail::vk_check(vkEndCommandBuffer(batch_cmd_), __FILE__, __LINE__); + if (!r) + { + vkDestroyFence(device_.device(), batch_fence_, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + return std::unexpected(r.error()); + } + + // 2. 提交 + vkResetFences(device_.device(), 1, &batch_fence_); + { + std::lock_guard lock(queue_mutex_); + VkSubmitInfo submit_info{}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &batch_cmd_; + + r = detail::vk_check( + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, batch_fence_), + __FILE__, __LINE__); + if (!r) + { + vkDestroyFence(device_.device(), batch_fence_, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + return std::unexpected(r.error()); + } + } + + // 3. 等待完成（60 秒超时，避免 TDR 导致无限挂起） + constexpr uint64_t kBatchTimeoutNs = 60'000'000'000ULL; // 60s + VkResult wait_result = vkWaitForFences( + device_.device(), 1, &batch_fence_, VK_TRUE, kBatchTimeoutNs); + if (wait_result == VK_ERROR_DEVICE_LOST) + { + // ── 设备丢失（TDR 触发）：GPU 已死亡，无法恢复 ── + vkDestroyFence(device_.device(), batch_fence_, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + if (!batch_desc_sets_.empty()) + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, + static_cast(batch_desc_sets_.size()), batch_desc_sets_.data()); + batch_desc_sets_.clear(); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + device_lost_ = true; + flush_pending_destroys(); + return std::unexpected(Error{ + "GPU 设备丢失 (VK_ERROR_DEVICE_LOST): Windows TDR 已重置 GPU 驱动。" + "\n模型已自动保存，请使用 --resume --batch-size <较小值> 重启训练。" + "\n建议：减小 --batch-size 或 --seq-len，或增大 Windows TDR 超时" + " (注册表 TdrDelay)"}); + } + else if (wait_result != VK_SUCCESS) + { + // 其他错误（含 VK_TIMEOUT） + vkDestroyFence(device_.device(), batch_fence_, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + if (!batch_desc_sets_.empty()) + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, + static_cast(batch_desc_sets_.size()), batch_desc_sets_.data()); + batch_desc_sets_.clear(); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + flush_pending_destroys(); + return std::unexpected(Error{ + std::string("GPU batch 等待失败: Vulkan error ") + std::to_string(static_cast(wait_result)) + + "\n建议：减小 --batch-size 或 --seq-len，或增大 Windows TDR 超时" + " (注册表 TdrDelay)"}); + } + + // 4. 清理 + vkDestroyFence(device_.device(), batch_fence_, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + // 释放 batch 期间分配的所有描述符集 + if (!batch_desc_sets_.empty()) + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, + static_cast(batch_desc_sets_.size()), batch_desc_sets_.data()); + batch_desc_sets_.clear(); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + + // 提交已完成且描述符集已释放，可安全销毁延迟的旧 buffer + flush_pending_destroys(); + + return r; + } + + // ── 批处理中点刷新（防 TDR） ───────────────────────────────────────── + // 提交当前 command buffer 并等待完成，然后自动开始新的录制。 + // 用于拆分大 batch（如 forward 与 backward 之间），避免单次提交时间 + // 过长触发 Windows TDR。调用后仍处于 batch 模式。 + [[nodiscard]] Result flush_batch() + { + if (!batch_mode_) + return {}; // 非 batch 模式时 no-op + + // 1. 结束当前录制 + auto r = detail::vk_check(vkEndCommandBuffer(batch_cmd_), __FILE__, __LINE__); + if (!r) + { + vkDestroyFence(device_.device(), batch_fence_, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + return std::unexpected(r.error()); + } + + // 2. 提交并等待完成（超时保护） + vkResetFences(device_.device(), 1, &batch_fence_); + { + std::lock_guard lock(queue_mutex_); + VkSubmitInfo submit_info{}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &batch_cmd_; + r = detail::vk_check( + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, batch_fence_), + __FILE__, __LINE__); + if (!r) + { + vkDestroyFence(device_.device(), batch_fence_, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + return std::unexpected(r.error()); + } + } + + constexpr uint64_t kFlushTimeoutNs = 60'000'000'000ULL; // 60s + VkResult flush_wait = vkWaitForFences( + device_.device(), 1, &batch_fence_, VK_TRUE, kFlushTimeoutNs); + if (flush_wait == VK_ERROR_DEVICE_LOST) + { + // 设备丢失：GPU 已死亡 + vkDestroyFence(device_.device(), batch_fence_, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + if (!batch_desc_sets_.empty()) + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, + static_cast(batch_desc_sets_.size()), batch_desc_sets_.data()); + batch_desc_sets_.clear(); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + device_lost_ = true; + flush_pending_destroys(); + return std::unexpected(Error{ + std::string("GPU flush 期间设备丢失 (VK_ERROR_DEVICE_LOST), Vulkan error ") + + std::to_string(static_cast(flush_wait))}); + } + else if (flush_wait != VK_SUCCESS) + { + vkDestroyFence(device_.device(), batch_fence_, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + if (!batch_desc_sets_.empty()) + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, + static_cast(batch_desc_sets_.size()), batch_desc_sets_.data()); + batch_desc_sets_.clear(); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + flush_pending_destroys(); + return std::unexpected(Error{ + std::string("GPU flush_batch 等待失败: Vulkan error ") + + std::to_string(static_cast(flush_wait))}); + } + + // 3. 释放已提交的描述符集 + if (!batch_desc_sets_.empty()) + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, + static_cast(batch_desc_sets_.size()), batch_desc_sets_.data()); + batch_desc_sets_.clear(); + + // 提交已完成，可安全销毁延迟的旧 buffer + flush_pending_destroys(); + + // 4. 销毁旧 command buffer 和 fence + vkDestroyFence(device_.device(), batch_fence_, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + + // 5. 分配新的 command buffer（继续录制） + VkCommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cmd_alloc.commandPool = command_pool_; + cmd_alloc.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cmd_alloc.commandBufferCount = 1; + r = detail::vk_check( + vkAllocateCommandBuffers(device_.device(), &cmd_alloc, &batch_cmd_), + __FILE__, __LINE__); + if (!r) + { + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + return std::unexpected(r.error()); + } + + VkCommandBufferBeginInfo begin_info{}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + r = detail::vk_check(vkBeginCommandBuffer(batch_cmd_, &begin_info), __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + return std::unexpected(r.error()); + } + + // 6. 创建新的 fence + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + r = detail::vk_check( + vkCreateFence(device_.device(), &fence_info, nullptr, &batch_fence_), + __FILE__, __LINE__); + if (!r) + { + vkEndCommandBuffer(batch_cmd_); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &batch_cmd_); + batch_cmd_ = VK_NULL_HANDLE; + batch_fence_ = VK_NULL_HANDLE; + batch_mode_ = false; + return std::unexpected(r.error()); + } + + // 仍处于 batch 模式，只是提交了一段并开始新的录制 + return {}; + } + + // ── 阻塞式上传：CPU → GPU（支持分块传输大矩阵）───────────────────── + // 当数据超过 staging region 大小时，自动分块上传。 + // 每块大小不超过 staging region，通过多次 memcpy + vkCmdCopyBuffer 完成。 + [[nodiscard]] Result upload_blocking( + GpuTensor& dst, std::span cpu_data) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (!dst.valid()) + return std::unexpected(Error{"Invalid destination GpuTensor"}); + + const std::size_t elem_count = dst.rows() * dst.cols(); + if (cpu_data.size() != elem_count) + return std::unexpected(Error{"Upload size mismatch"}); + + const std::size_t total_bytes = elem_count * sizeof(Scalar); + const std::size_t staging_cap = staging_ring_->region_size(); + + // ── 小矩阵快速路径：单次传输 ────────────────────────────────── + if (total_bytes <= staging_cap) + { + auto ri = staging_ring_->acquire(); + + auto r = staging_ring_->upload(ri, cpu_data, 0); + if (!r) return r; + + VkCommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cmd_alloc.commandPool = command_pool_; + cmd_alloc.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cmd_alloc.commandBufferCount = 1; + + VkCommandBuffer cmd = VK_NULL_HANDLE; + r = detail::vk_check( + vkAllocateCommandBuffers(device_.device(), &cmd_alloc, &cmd), + __FILE__, __LINE__); + if (!r) return r; + + VkCommandBufferBeginInfo begin_info{}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + + r = detail::vk_check(vkBeginCommandBuffer(cmd, &begin_info), __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + VkBufferCopy cp{0, 0, total_bytes}; + vkCmdCopyBuffer(cmd, staging_ring_->buffer(ri), dst.buffer().impl(), 1, &cp); + + r = detail::vk_check(vkEndCommandBuffer(cmd), __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + auto fence = staging_ring_->fence(ri); + vkResetFences(device_.device(), 1, &fence); + + { + std::lock_guard lock(queue_mutex_); + VkSubmitInfo submit_info{}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd; + + r = detail::vk_check( + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, fence), + __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + } + + r = detail::vk_check( + vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, 10'000'000'000ULL), + __FILE__, __LINE__); + + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + return r; + } + + // ── 大矩阵分块上传 ─────────────────────────────────────────── + // 每块大小对齐到 sizeof(Scalar)，确保元素边界对齐 + const std::size_t chunk_bytes = (staging_cap / sizeof(Scalar)) * sizeof(Scalar); + std::size_t dst_offset = 0; + + while (dst_offset < total_bytes) + { + const std::size_t this_chunk = std::min(chunk_bytes, total_bytes - dst_offset); + const std::size_t this_elems = this_chunk / sizeof(Scalar); + const std::size_t src_elem_offset = dst_offset / sizeof(Scalar); + + // 1. 获取 staging region 并上传当前块 + auto ri = staging_ring_->acquire(); + auto r = staging_ring_->upload( + ri, + std::span(cpu_data.data() + src_elem_offset, this_elems), + 0); + if (!r) return r; + + // 2. 录制 copy 命令 + VkCommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cmd_alloc.commandPool = command_pool_; + cmd_alloc.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cmd_alloc.commandBufferCount = 1; + + VkCommandBuffer cmd = VK_NULL_HANDLE; + r = detail::vk_check( + vkAllocateCommandBuffers(device_.device(), &cmd_alloc, &cmd), + __FILE__, __LINE__); + if (!r) return r; + + VkCommandBufferBeginInfo begin_info{}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + + r = detail::vk_check(vkBeginCommandBuffer(cmd, &begin_info), __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + VkBufferCopy cp{0, dst_offset, this_chunk}; + vkCmdCopyBuffer(cmd, staging_ring_->buffer(ri), dst.buffer().impl(), 1, &cp); + + r = detail::vk_check(vkEndCommandBuffer(cmd), __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + // 3. 提交并等待 + auto fence = staging_ring_->fence(ri); + vkResetFences(device_.device(), 1, &fence); + + { + std::lock_guard lock(queue_mutex_); + VkSubmitInfo submit_info{}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd; + + r = detail::vk_check( + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, fence), + __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + } + + r = detail::vk_check( + vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, 10'000'000'000ULL), + __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + dst_offset += this_chunk; + } + + return {}; + } + + // ── 阻塞式下载：GPU → CPU（支持分块传输大矩阵）───────────────────── + // 当数据超过 staging region 大小时，自动分块下载。 + [[nodiscard]] Result download_blocking( + const GpuTensor& src, std::span cpu_data) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (!src.valid()) + return std::unexpected(Error{"Invalid source GpuTensor"}); + + const std::size_t elem_count = src.rows() * src.cols(); + if (cpu_data.size() != elem_count) + return std::unexpected(Error{"Download size mismatch"}); + + const std::size_t total_bytes = elem_count * sizeof(Scalar); + const std::size_t staging_cap = staging_ring_->region_size(); + + // ── 小矩阵快速路径：单次传输 ────────────────────────────────── + if (total_bytes <= staging_cap) + { + auto ri = staging_ring_->acquire(); + + VkCommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cmd_alloc.commandPool = command_pool_; + cmd_alloc.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cmd_alloc.commandBufferCount = 1; + + VkCommandBuffer cmd = VK_NULL_HANDLE; + auto r = detail::vk_check( + vkAllocateCommandBuffers(device_.device(), &cmd_alloc, &cmd), + __FILE__, __LINE__); + if (!r) return r; + + VkCommandBufferBeginInfo begin_info{}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + + r = detail::vk_check(vkBeginCommandBuffer(cmd, &begin_info), __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + VkBufferCopy cp{0, 0, total_bytes}; + vkCmdCopyBuffer(cmd, src.buffer().impl(), staging_ring_->buffer(ri), 1, &cp); + + r = detail::vk_check(vkEndCommandBuffer(cmd), __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + auto fence = staging_ring_->fence(ri); + vkResetFences(device_.device(), 1, &fence); + + { + std::lock_guard lock(queue_mutex_); + VkSubmitInfo submit_info{}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd; + + r = detail::vk_check( + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, fence), + __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + } + + r = detail::vk_check( + vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, 10'000'000'000ULL), + __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + r = staging_ring_->download(ri, cpu_data, 0); + + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + return r; + } + + // ── 大矩阵分块下载 ─────────────────────────────────────────── + const std::size_t chunk_bytes = (staging_cap / sizeof(Scalar)) * sizeof(Scalar); + std::size_t src_offset = 0; + + while (src_offset < total_bytes) + { + const std::size_t this_chunk = std::min(chunk_bytes, total_bytes - src_offset); + const std::size_t this_elems = this_chunk / sizeof(Scalar); + const std::size_t dst_elem_offset = src_offset / sizeof(Scalar); + + // 1. 获取 staging region + auto ri = staging_ring_->acquire(); + + // 2. 录制 copy 命令：GPU → staging + VkCommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cmd_alloc.commandPool = command_pool_; + cmd_alloc.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cmd_alloc.commandBufferCount = 1; + + VkCommandBuffer cmd = VK_NULL_HANDLE; + auto r = detail::vk_check( + vkAllocateCommandBuffers(device_.device(), &cmd_alloc, &cmd), + __FILE__, __LINE__); + if (!r) return r; + + VkCommandBufferBeginInfo begin_info{}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + + r = detail::vk_check(vkBeginCommandBuffer(cmd, &begin_info), __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + VkBufferCopy cp{src_offset, 0, this_chunk}; + vkCmdCopyBuffer(cmd, src.buffer().impl(), staging_ring_->buffer(ri), 1, &cp); + + r = detail::vk_check(vkEndCommandBuffer(cmd), __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + // 3. 提交并等待 + auto fence = staging_ring_->fence(ri); + vkResetFences(device_.device(), 1, &fence); + + { + std::lock_guard lock(queue_mutex_); + VkSubmitInfo submit_info{}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd; + + r = detail::vk_check( + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, fence), + __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + } + + r = detail::vk_check( + vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, 10'000'000'000ULL), + __FILE__, __LINE__); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + // 4. 从 staging 下载当前块到 CPU + r = staging_ring_->download( + ri, + std::span(cpu_data.data() + dst_elem_offset, this_elems), + 0); + if (!r) { vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); return r; } + + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + src_offset += this_chunk; + } + + return {}; + } + + // ── 纯 GPU 矩阵乘法（阻塞等待完成）────────────────────────────── + // batch_mode_ 时：录制到 batch_cmd_，不提交不等待 + // transA: A 存储为 (K,M)，按 A^T 使用 + // transB: B 存储为 (N,K)，按 B^T 使用 + [[nodiscard]] Result matmul_gpu( + const GpuTensor& A, const GpuTensor& B, + uint32_t transA = 0, uint32_t transB = 0) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + + // 计算有效维度（考虑转置） + const auto M = static_cast(transA ? A.cols() : A.rows()); + const auto K = static_cast(transA ? A.rows() : A.cols()); + const auto N = static_cast(transB ? B.rows() : B.cols()); + const auto K_B = static_cast(transB ? B.cols() : B.rows()); + if (K != K_B) + return std::unexpected(Error{"Dimension mismatch"}); + + // 1. 分配输出 Tensor + auto C_res = GpuTensor::create_empty(M, N, *this); + if (!C_res) + return std::unexpected(C_res.error()); + GpuTensor C = std::move(*C_res); + + // 2. 选择 Pipeline（优先使用粗化分块版本） + const bool use_tiled = has_tiled_pipeline(); + auto& pipeline = use_tiled ? matmul_tiled_pipeline_ : matmul_pipeline_; + + // 3. 分配描述符集 + VkDescriptorSet desc_set = VK_NULL_HANDLE; + VkDescriptorSetAllocateInfo desc_alloc{}; + desc_alloc.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; + desc_alloc.descriptorPool = gpu_tensor_pool_; + desc_alloc.descriptorSetCount = 1; + auto dl = pipeline.descriptor_layout(); + desc_alloc.pSetLayouts = &dl; + + auto r = detail::vk_check( + vkAllocateDescriptorSets(device_.device(), &desc_alloc, &desc_set), + __FILE__, __LINE__); + if (!r) + return std::unexpected(r.error()); + if (batch_mode_) batch_desc_sets_.push_back(desc_set); + + // 4. 写入描述符集 + VkDescriptorBufferInfo buf_infos[3]{ + {A.buffer().impl(), 0, VK_WHOLE_SIZE}, + {B.buffer().impl(), 0, VK_WHOLE_SIZE}, + {C.buffer().impl(), 0, VK_WHOLE_SIZE}, + }; + + VkWriteDescriptorSet writes[3]{}; + for (int i = 0; i < 3; ++i) + { + writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[i].dstSet = desc_set; + writes[i].dstBinding = static_cast(i); + writes[i].descriptorCount = 1; + writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[i].pBufferInfo = &buf_infos[i]; + } + vkUpdateDescriptorSets(device_.device(), 3, writes, 0, nullptr); + + // 5. 获取 command buffer（batch 或独立） + VkCommandBuffer cmd = VK_NULL_HANDLE; + bool owns_cmd = false; + VkFence fence = VK_NULL_HANDLE; + + if (batch_mode_) + { + cmd = batch_cmd_; + } + else + { + VkCommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cmd_alloc.commandPool = command_pool_; + cmd_alloc.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cmd_alloc.commandBufferCount = 1; + + r = detail::vk_check( + vkAllocateCommandBuffers(device_.device(), &cmd_alloc, &cmd), + __FILE__, __LINE__); + if (!r) + { + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + owns_cmd = true; + + VkCommandBufferBeginInfo begin_info{}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + r = detail::vk_check(vkBeginCommandBuffer(cmd, &begin_info), __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + } + + // 6. 录制命令（barrier + dispatch + barrier） + // 管线屏障：确保输入数据就绪 + VkBufferMemoryBarrier input_barriers[2]{}; + for (int i = 0; i < 2; ++i) + { + input_barriers[i].sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + input_barriers[i].srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + input_barriers[i].dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + input_barriers[i].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + input_barriers[i].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + input_barriers[i].offset = 0; + input_barriers[i].size = VK_WHOLE_SIZE; + } + input_barriers[0].buffer = A.buffer().impl(); + input_barriers[1].buffer = B.buffer().impl(); + + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, 0, nullptr, 2, input_barriers, 0, nullptr); + + // 绑定 pipeline 和描述符集 + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline.handle()); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + pipeline.pipeline_layout(), 0, 1, &desc_set, 0, nullptr); + + // 推送常量（M, N, K, transA, transB） + const uint32_t push_data[5] = {M, N, K, transA, transB}; + vkCmdPushConstants(cmd, pipeline.pipeline_layout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push_data), push_data); + + // Dispatch + const uint32_t wg_x = (N + WORKGROUP_SIZE - 1) / WORKGROUP_SIZE; + const uint32_t wg_y = (M + WORKGROUP_SIZE - 1) / WORKGROUP_SIZE; + vkCmdDispatch(cmd, wg_x, wg_y, 1); + + // 输出屏障 + VkBufferMemoryBarrier output_barrier{}; + output_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + output_barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + output_barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + output_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + output_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + output_barrier.buffer = C.buffer().impl(); + output_barrier.offset = 0; + output_barrier.size = VK_WHOLE_SIZE; + + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 1, &output_barrier, 0, nullptr); + + // 7. 独立模式：结束录制 → 提交 → 等待 → 清理 + if (owns_cmd) + { + r = detail::vk_check(vkEndCommandBuffer(cmd), __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + r = detail::vk_check( + vkCreateFence(device_.device(), &fence_info, nullptr, &fence), + __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + + { + std::lock_guard lock(queue_mutex_); + VkSubmitInfo submit_info{}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd; + + r = detail::vk_check( + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, fence), + __FILE__, __LINE__); + if (!r) + { + vkDestroyFence(device_.device(), fence, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + } + + constexpr uint64_t kTransferTimeoutNs = 30'000'000'000ULL; + r = detail::vk_check( + vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, kTransferTimeoutNs), + __FILE__, __LINE__); + + vkDestroyFence(device_.device(), fence, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + + if (!r) + return std::unexpected(r.error()); + } + // batch 模式：不结束录制，不提交，desc_set 需要在 end_batch 后释放 + // 为简化实现，batch 模式下 desc_set 使用 per-operation 临时分配 + // end_batch() 统一提交和等待 + + return C; + } + + // ── 纯 GPU 批量矩阵乘法 ────────────────────────────────────────── + // 对每个 batch b 计算 C_b = op(A_b, B_b)，结果垂直堆叠为 (batch*M, N) + // A: (batch * A_rows_per_batch, A_cols)，B: (batch * B_rows_per_batch, B_cols) + // batch 步长由 shader 从 M*K / K*N / M*N 推导，无需额外传入 + [[nodiscard]] Result batched_matmul_gpu( + const GpuTensor& A, const GpuTensor& B, + uint32_t batch, + uint32_t transA = 0, uint32_t transB = 0) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (batch == 0) + return std::unexpected(Error{"batched_matmul_gpu: batch must be > 0"}); + if (!has_batched_matmul_pipeline()) + return std::unexpected(Error{"batched_matmul_gpu: pipeline not available"}); + + // 校验：A.rows() 必须能被 batch 整除 + if (A.rows() % batch != 0 || B.rows() % batch != 0) + return std::unexpected(Error{"batched_matmul_gpu: rows not divisible by batch"}); + + // 计算每个 batch 的逻辑维度 + const auto a_rows_per = static_cast(A.rows() / batch); + const auto b_rows_per = static_cast(B.rows() / batch); + const auto M = transA ? static_cast(A.cols()) : a_rows_per; + const auto K = transA ? a_rows_per : static_cast(A.cols()); + const auto K_B = transB ? static_cast(B.cols()) : b_rows_per; + const auto N = transB ? b_rows_per : static_cast(B.cols()); + if (K != K_B) + return std::unexpected(Error{"batched_matmul_gpu: K dimension mismatch"}); + + // 1. 分配输出 Tensor: (batch * M, N) + auto C_res = GpuTensor::create_empty(static_cast(batch) * M, N, *this); + if (!C_res) + return std::unexpected(C_res.error()); + GpuTensor C = std::move(*C_res); + + auto& pipeline = batched_matmul_pipeline_; + + // 2. 分配描述符集（与 matmul 相同的 3-binding 布局） + VkDescriptorSet desc_set = VK_NULL_HANDLE; + VkDescriptorSetAllocateInfo desc_alloc{}; + desc_alloc.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; + desc_alloc.descriptorPool = gpu_tensor_pool_; + desc_alloc.descriptorSetCount = 1; + auto dl = pipeline.descriptor_layout(); + desc_alloc.pSetLayouts = &dl; + + auto r = detail::vk_check( + vkAllocateDescriptorSets(device_.device(), &desc_alloc, &desc_set), + __FILE__, __LINE__); + if (!r) + return std::unexpected(r.error()); + if (batch_mode_) batch_desc_sets_.push_back(desc_set); + + // 3. 写入描述符集 + VkDescriptorBufferInfo buf_infos[3]{ + {A.buffer().impl(), 0, VK_WHOLE_SIZE}, + {B.buffer().impl(), 0, VK_WHOLE_SIZE}, + {C.buffer().impl(), 0, VK_WHOLE_SIZE}, + }; + + VkWriteDescriptorSet writes[3]{}; + for (int i = 0; i < 3; ++i) + { + writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[i].dstSet = desc_set; + writes[i].dstBinding = static_cast(i); + writes[i].descriptorCount = 1; + writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[i].pBufferInfo = &buf_infos[i]; + } + vkUpdateDescriptorSets(device_.device(), 3, writes, 0, nullptr); + + // 4. 获取 command buffer（batch 共享 / 独立分配） + VkCommandBuffer cmd = VK_NULL_HANDLE; + bool owns_cmd = false; + VkFence fence = VK_NULL_HANDLE; + + if (batch_mode_) + { + cmd = batch_cmd_; + } + else + { + VkCommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cmd_alloc.commandPool = command_pool_; + cmd_alloc.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cmd_alloc.commandBufferCount = 1; + + r = detail::vk_check( + vkAllocateCommandBuffers(device_.device(), &cmd_alloc, &cmd), + __FILE__, __LINE__); + if (!r) + { + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + owns_cmd = true; + + VkCommandBufferBeginInfo begin_info{}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + r = detail::vk_check(vkBeginCommandBuffer(cmd, &begin_info), __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + } + + // 5. 录制：输入屏障 → bind → push constants → dispatch(Z=batch) → 输出屏障 + VkBufferMemoryBarrier input_barriers[2]{}; + for (int i = 0; i < 2; ++i) + { + input_barriers[i].sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + input_barriers[i].srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + input_barriers[i].dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + input_barriers[i].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + input_barriers[i].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + input_barriers[i].offset = 0; + input_barriers[i].size = VK_WHOLE_SIZE; + } + input_barriers[0].buffer = A.buffer().impl(); + input_barriers[1].buffer = B.buffer().impl(); + + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, 0, nullptr, 2, input_barriers, 0, nullptr); + + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline.handle()); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + pipeline.pipeline_layout(), 0, 1, &desc_set, 0, nullptr); + + // Push constants: M, N, K, transA, transB（与 matmul 相同布局） + const uint32_t push_data[5] = {M, N, K, transA, transB}; + vkCmdPushConstants(cmd, pipeline.pipeline_layout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push_data), push_data); + + // Dispatch: Z 维度 = batch（每个 batch 一个 workgroup 层） + const uint32_t wg_x = (N + WORKGROUP_SIZE - 1) / WORKGROUP_SIZE; + const uint32_t wg_y = (M + WORKGROUP_SIZE - 1) / WORKGROUP_SIZE; + vkCmdDispatch(cmd, wg_x, wg_y, batch); + + // 输出屏障 + VkBufferMemoryBarrier output_barrier{}; + output_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + output_barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + output_barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + output_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + output_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + output_barrier.buffer = C.buffer().impl(); + output_barrier.offset = 0; + output_barrier.size = VK_WHOLE_SIZE; + + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 1, &output_barrier, 0, nullptr); + + // 6. 独立模式：结束录制 → 提交 → 等待 → 清理 + if (owns_cmd) + { + r = detail::vk_check(vkEndCommandBuffer(cmd), __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + r = detail::vk_check( + vkCreateFence(device_.device(), &fence_info, nullptr, &fence), + __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + + { + std::lock_guard lock(queue_mutex_); + VkSubmitInfo submit_info{}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd; + + r = detail::vk_check( + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, fence), + __FILE__, __LINE__); + if (!r) + { + vkDestroyFence(device_.device(), fence, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + } + + constexpr uint64_t kTransferTimeoutNs = 30'000'000'000ULL; + r = detail::vk_check( + vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, kTransferTimeoutNs), + __FILE__, __LINE__); + + vkDestroyFence(device_.device(), fence, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + + if (!r) + return std::unexpected(r.error()); + } + + return C; + } + + // ══════════════════════════════════════════════════════════════════ + // 纯 GPU 原语方法（纯 GPU 架构核心） + // + // 所有方法遵循同一模式： + // 1. 分配输出 Tensor（如有输出） + // 2. 分配描述符集，写入 binding + // 3. 获取 command buffer（batch 共享 / 独立分配） + // 4. 录制：输入屏障 → bind pipeline → push constants → dispatch → 输出屏障 + // 5. 独立模式：end → submit → wait → cleanup + // ══════════════════════════════════════════════════════════════════ + + // ── 辅助：分配描述符集 ────────────────────────────────────────────── + [[nodiscard]] Result alloc_desc_set(VkDescriptorSetLayout layout) + { + VkDescriptorSet desc_set = VK_NULL_HANDLE; + VkDescriptorSetAllocateInfo desc_alloc{}; + desc_alloc.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; + desc_alloc.descriptorPool = gpu_tensor_pool_; + desc_alloc.descriptorSetCount = 1; + desc_alloc.pSetLayouts = &layout; + + auto r = detail::vk_check( + vkAllocateDescriptorSets(device_.device(), &desc_alloc, &desc_set), + __FILE__, __LINE__); + if (!r) return std::unexpected(r.error()); + if (batch_mode_) batch_desc_sets_.push_back(desc_set); + return desc_set; + } + + // ── 辅助：获取 command buffer（batch 或独立）────────────────────── + // 返回 (cmd, owns_cmd)。owns_cmd=true 时调用方需 submit_and_wait。 + [[nodiscard]] Result> acquire_cmd() + { + if (batch_mode_) + return std::make_pair(batch_cmd_, false); + + VkCommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cmd_alloc.commandPool = command_pool_; + cmd_alloc.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cmd_alloc.commandBufferCount = 1; + + VkCommandBuffer cmd = VK_NULL_HANDLE; + auto r = detail::vk_check( + vkAllocateCommandBuffers(device_.device(), &cmd_alloc, &cmd), + __FILE__, __LINE__); + if (!r) return std::unexpected(r.error()); + + VkCommandBufferBeginInfo begin_info{}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + r = detail::vk_check(vkBeginCommandBuffer(cmd, &begin_info), __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + return std::unexpected(r.error()); + } + return std::make_pair(cmd, true); + } + + // ── 辅助：独立模式提交+等待+清理 ────────────────────────────────── + // owns_cmd=true 时调用：end → submit → wait → free cmd + desc_set + // desc_set 可为 VK_NULL_HANDLE（fill_zero/copy_buffer 无描述符集） + [[nodiscard]] Result submit_and_wait( + VkCommandBuffer cmd, VkDescriptorSet desc_set) + { + auto r = detail::vk_check(vkEndCommandBuffer(cmd), __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + if (desc_set != VK_NULL_HANDLE) + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + VkFence fence = VK_NULL_HANDLE; + r = detail::vk_check( + vkCreateFence(device_.device(), &fence_info, nullptr, &fence), + __FILE__, __LINE__); + if (!r) + { + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + if (desc_set != VK_NULL_HANDLE) + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + + { + std::lock_guard lock(queue_mutex_); + VkSubmitInfo submit_info{}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd; + r = detail::vk_check( + vkQueueSubmit(device_.compute_queue(), 1, &submit_info, fence), + __FILE__, __LINE__); + if (!r) + { + vkDestroyFence(device_.device(), fence, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + if (desc_set != VK_NULL_HANDLE) + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(r.error()); + } + } + + // 30 秒超时（单个原语，比 batch 短） + constexpr uint64_t kSingleOpTimeoutNs = 30'000'000'000ULL; + r = detail::vk_check( + vkWaitForFences(device_.device(), 1, &fence, VK_TRUE, kSingleOpTimeoutNs), + __FILE__, __LINE__); + + vkDestroyFence(device_.device(), fence, nullptr); + vkFreeCommandBuffers(device_.device(), command_pool_, 1, &cmd); + if (desc_set != VK_NULL_HANDLE) + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return r; + } + + // ── 辅助：录制输入屏障（多个输入 buffer）────────────────────────── + void record_input_barriers(VkCommandBuffer cmd, + std::initializer_list buffers) + { + std::vector barriers; + barriers.reserve(buffers.size()); + for (auto buf : buffers) + { + VkBufferMemoryBarrier b{}; + b.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + b.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + b.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.buffer = buf; + b.offset = 0; + b.size = VK_WHOLE_SIZE; + barriers.push_back(b); + } + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, 0, nullptr, + static_cast(barriers.size()), barriers.data(), + 0, nullptr); + } + + // ── 辅助：录制输出屏障 ───────────────────────────────────────────── + void record_output_barrier(VkCommandBuffer cmd, VkBuffer output_buf) + { + VkBufferMemoryBarrier b{}; + b.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + b.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + b.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.buffer = output_buf; + b.offset = 0; + b.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 1, &b, 0, nullptr); + } + + // ══════════════════════════════════════════════════════════════════ + // elementwise_v2_gpu — 逐元素原语（unary/binary/select） + // + // Push Constants (32 bytes): + // count, mode, op, cmp_op, flags, scalar_b, scalar_then, scalar_else + // Bindings: A(0), B(1), C(2), OUT(3) + // + // 对未使用的 binding（如 unary 模式的 B/C），绑定 A 的 buffer（无害占位）。 + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result elementwise_v2_gpu( + const GpuTensor& A, const GpuTensor* B, const GpuTensor* C, + uint32_t count, uint32_t mode, uint32_t op, uint32_t cmp_op, + uint32_t flags, float scalar_b, float scalar_then, float scalar_else) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (!has_elementwise_v2_pipeline()) + return std::unexpected(Error{"elementwise_v2 pipeline not available"}); + + // 1. 分配输出 Tensor + auto output_res = GpuTensor::create_empty(A.rows(), A.cols(), *this); + if (!output_res) return std::unexpected(output_res.error()); + GpuTensor output = std::move(*output_res); + + // 2. 分配描述符集 + auto ds_r = alloc_desc_set(elementwise_v2_pipeline_.descriptor_layout()); + if (!ds_r) return std::unexpected(ds_r.error()); + VkDescriptorSet desc_set = *ds_r; + + // 3. 写入描述符集（未使用的 binding 绑定 A 作为占位） + VkBuffer a_buf = A.buffer().impl(); + VkDescriptorBufferInfo buf_infos[4]{ + {a_buf, 0, VK_WHOLE_SIZE}, + {B ? B->buffer().impl() : a_buf, 0, VK_WHOLE_SIZE}, + {C ? C->buffer().impl() : a_buf, 0, VK_WHOLE_SIZE}, + {output.buffer().impl(), 0, VK_WHOLE_SIZE}, + }; + VkWriteDescriptorSet writes[4]{}; + for (int i = 0; i < 4; ++i) + { + writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[i].dstSet = desc_set; + writes[i].dstBinding = static_cast(i); + writes[i].descriptorCount = 1; + writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[i].pBufferInfo = &buf_infos[i]; + } + vkUpdateDescriptorSets(device_.device(), 4, writes, 0, nullptr); + + // 4. 获取 command buffer + auto cmd_r = acquire_cmd(); + if (!cmd_r) + { + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(cmd_r.error()); + } + auto [cmd, owns_cmd] = *cmd_r; + + // 5. 录制 + record_input_barriers(cmd, {a_buf}); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + elementwise_v2_pipeline_.handle()); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + elementwise_v2_pipeline_.pipeline_layout(), 0, 1, &desc_set, 0, nullptr); + + // Push constants: count, mode, op, cmp_op, flags (5×uint32) + scalar_b, scalar_then, scalar_else (3×float) + struct PushData { + uint32_t count, mode, op, cmp_op, flags; + float scalar_b, scalar_then, scalar_else; + } push{count, mode, op, cmp_op, flags, scalar_b, scalar_then, scalar_else}; + vkCmdPushConstants(cmd, elementwise_v2_pipeline_.pipeline_layout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), &push); + + const uint32_t wg_count = (count + 255) / 256; + vkCmdDispatch(cmd, wg_count, 1, 1); + record_output_barrier(cmd, output.buffer().impl()); + + // 6. 独立模式提交 + if (owns_cmd) + { + auto r = submit_and_wait(cmd, desc_set); + if (!r) return std::unexpected(r.error()); + } + return output; + } + + // ══════════════════════════════════════════════════════════════════ + // reduce_gpu — 行/列归约原语（sum/max） + // + // Push Constants (16 bytes): rows, cols, mode, reduce_op + // Bindings: In(0), Out(1) + // mode: 0=row_reduce → out(rows,1), 1=col_reduce → out(1,cols) + // reduce_op: 0=sum, 1=max + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result reduce_gpu( + const GpuTensor& input, uint32_t mode, uint32_t reduce_op) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (!has_reduce_pipeline()) + return std::unexpected(Error{"reduce pipeline not available"}); + + const uint32_t rows = static_cast(input.rows()); + const uint32_t cols = static_cast(input.cols()); + + // 1. 分配输出 Tensor + std::size_t out_rows = (mode == 0) ? rows : 1; + std::size_t out_cols = (mode == 0) ? 1 : cols; + auto output_res = GpuTensor::create_empty(out_rows, out_cols, *this); + if (!output_res) return std::unexpected(output_res.error()); + GpuTensor output = std::move(*output_res); + + // 2. 分配描述符集 + auto ds_r = alloc_desc_set(reduce_pipeline_.descriptor_layout()); + if (!ds_r) return std::unexpected(ds_r.error()); + VkDescriptorSet desc_set = *ds_r; + + // 3. 写入描述符集 + VkDescriptorBufferInfo buf_infos[2]{ + {input.buffer().impl(), 0, VK_WHOLE_SIZE}, + {output.buffer().impl(), 0, VK_WHOLE_SIZE}, + }; + VkWriteDescriptorSet writes[2]{}; + for (int i = 0; i < 2; ++i) + { + writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[i].dstSet = desc_set; + writes[i].dstBinding = static_cast(i); + writes[i].descriptorCount = 1; + writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[i].pBufferInfo = &buf_infos[i]; + } + vkUpdateDescriptorSets(device_.device(), 2, writes, 0, nullptr); + + // 4. 获取 command buffer + auto cmd_r = acquire_cmd(); + if (!cmd_r) + { + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(cmd_r.error()); + } + auto [cmd, owns_cmd] = *cmd_r; + + // 5. 录制 + record_input_barriers(cmd, {input.buffer().impl()}); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + reduce_pipeline_.handle()); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + reduce_pipeline_.pipeline_layout(), 0, 1, &desc_set, 0, nullptr); + + const uint32_t push_data[4] = {rows, cols, mode, reduce_op}; + vkCmdPushConstants(cmd, reduce_pipeline_.pipeline_layout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push_data), push_data); + + // 每个工作组 256 线程协作归约一行/一列 + const uint32_t workgroups = (mode == 0) ? rows : cols; + vkCmdDispatch(cmd, workgroups, 1, 1); + record_output_barrier(cmd, output.buffer().impl()); + + // 6. 独立模式提交 + if (owns_cmd) + { + auto r = submit_and_wait(cmd, desc_set); + if (!r) return std::unexpected(r.error()); + } + return output; + } + + // ══════════════════════════════════════════════════════════════════ + // broadcast_gpu — 行/列广播原语 + // + // Push Constants (16 bytes): rows, cols, mode, op + // Bindings: A(0), Vec(1), Out(2) + // mode: 0=row_broadcast (vec indexed by row), 1=col_broadcast (vec indexed by col) + // op: 0=Add, 1=Sub, 2=Mul, 3=Div, 4=Max, 5=Min + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result broadcast_gpu( + const GpuTensor& A, const GpuTensor& vec, + uint32_t mode, uint32_t op) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (!has_broadcast_pipeline()) + return std::unexpected(Error{"broadcast pipeline not available"}); + + const uint32_t rows = static_cast(A.rows()); + const uint32_t cols = static_cast(A.cols()); + + // 1. 分配输出 Tensor + auto output_res = GpuTensor::create_empty(A.rows(), A.cols(), *this); + if (!output_res) return std::unexpected(output_res.error()); + GpuTensor output = std::move(*output_res); + + // 2. 分配描述符集 + auto ds_r = alloc_desc_set(broadcast_pipeline_.descriptor_layout()); + if (!ds_r) return std::unexpected(ds_r.error()); + VkDescriptorSet desc_set = *ds_r; + + // 3. 写入描述符集 + VkDescriptorBufferInfo buf_infos[3]{ + {A.buffer().impl(), 0, VK_WHOLE_SIZE}, + {vec.buffer().impl(), 0, VK_WHOLE_SIZE}, + {output.buffer().impl(), 0, VK_WHOLE_SIZE}, + }; + VkWriteDescriptorSet writes[3]{}; + for (int i = 0; i < 3; ++i) + { + writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[i].dstSet = desc_set; + writes[i].dstBinding = static_cast(i); + writes[i].descriptorCount = 1; + writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[i].pBufferInfo = &buf_infos[i]; + } + vkUpdateDescriptorSets(device_.device(), 3, writes, 0, nullptr); + + // 4. 获取 command buffer + auto cmd_r = acquire_cmd(); + if (!cmd_r) + { + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(cmd_r.error()); + } + auto [cmd, owns_cmd] = *cmd_r; + + // 5. 录制 + record_input_barriers(cmd, {A.buffer().impl(), vec.buffer().impl()}); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + broadcast_pipeline_.handle()); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + broadcast_pipeline_.pipeline_layout(), 0, 1, &desc_set, 0, nullptr); + + const uint32_t push_data[4] = {rows, cols, mode, op}; + vkCmdPushConstants(cmd, broadcast_pipeline_.pipeline_layout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push_data), push_data); + + const uint32_t total = rows * cols; + const uint32_t wg_count = (total + 255) / 256; + vkCmdDispatch(cmd, wg_count, 1, 1); + record_output_barrier(cmd, output.buffer().impl()); + + // 6. 独立模式提交 + if (owns_cmd) + { + auto r = submit_and_wait(cmd, desc_set); + if (!r) return std::unexpected(r.error()); + } + return output; + } + + // ══════════════════════════════════════════════════════════════════ + // rearrange_3d_gpu — 3D 维度转置 (M, B, N) ↔ (B, M, N) + // + // Push Constants (20 bytes): M, B, N, inverse, total + // Bindings: In(0), Out(1) + // inverse=0: (M, B*N) → (B*M, N) + // inverse=1: (B*M, N) → (M, B*N) + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result rearrange_3d_gpu( + const GpuTensor& input, + uint32_t M, uint32_t B, uint32_t N, uint32_t inverse) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (!has_rearrange_3d_pipeline()) + return std::unexpected(Error{"rearrange_3d pipeline not available"}); + + const uint32_t total = M * B * N; + if (total != input.rows() * input.cols()) + return std::unexpected(Error{"rearrange_3d: element count mismatch"}); + + // 1. 分配输出 Tensor + const std::size_t out_rows = inverse ? M : (static_cast(B) * M); + const std::size_t out_cols = inverse ? (static_cast(B) * N) : N; + auto output_res = GpuTensor::create_empty(out_rows, out_cols, *this); + if (!output_res) return std::unexpected(output_res.error()); + GpuTensor output = std::move(*output_res); + + // 2. 分配描述符集 + auto ds_r = alloc_desc_set(rearrange_3d_pipeline_.descriptor_layout()); + if (!ds_r) return std::unexpected(ds_r.error()); + VkDescriptorSet desc_set = *ds_r; + + // 3. 写入描述符集 + VkDescriptorBufferInfo buf_infos[2]{ + {input.buffer().impl(), 0, VK_WHOLE_SIZE}, + {output.buffer().impl(), 0, VK_WHOLE_SIZE}, + }; + VkWriteDescriptorSet writes[2]{}; + for (int i = 0; i < 2; ++i) + { + writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[i].dstSet = desc_set; + writes[i].dstBinding = static_cast(i); + writes[i].descriptorCount = 1; + writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[i].pBufferInfo = &buf_infos[i]; + } + vkUpdateDescriptorSets(device_.device(), 2, writes, 0, nullptr); + + // 4. 获取 command buffer + auto cmd_r = acquire_cmd(); + if (!cmd_r) + { + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(cmd_r.error()); + } + auto [cmd, owns_cmd] = *cmd_r; + + // 5. 录制 + record_input_barriers(cmd, {input.buffer().impl()}); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + rearrange_3d_pipeline_.handle()); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + rearrange_3d_pipeline_.pipeline_layout(), 0, 1, &desc_set, 0, nullptr); + + const uint32_t push_data[5] = {M, B, N, inverse, total}; + vkCmdPushConstants(cmd, rearrange_3d_pipeline_.pipeline_layout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push_data), push_data); + + const uint32_t wg_count = (total + 255) / 256; + vkCmdDispatch(cmd, wg_count, 1, 1); + record_output_barrier(cmd, output.buffer().impl()); + + // 6. 独立模式提交 + if (owns_cmd) + { + auto r = submit_and_wait(cmd, desc_set); + if (!r) return std::unexpected(r.error()); + } + return output; + } + + // ══════════════════════════════════════════════════════════════════ + // fill_zero_gpu — 清零 GPU buffer（使用 vkCmdFillBuffer） + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result fill_zero_gpu(GpuTensor& tensor) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + + auto cmd_r = acquire_cmd(); + if (!cmd_r) return std::unexpected(cmd_r.error()); + auto [cmd, owns_cmd] = *cmd_r; + + vkCmdFillBuffer(cmd, tensor.buffer().impl(), 0, VK_WHOLE_SIZE, 0); + + // 屏障确保 fill 完成后可见 + VkBufferMemoryBarrier b{}; + b.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + b.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + b.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.buffer = tensor.buffer().impl(); + b.offset = 0; + b.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 1, &b, 0, nullptr); + + if (owns_cmd) + { + // fill_zero 不使用描述符集，传 VK_NULL_HANDLE 跳过释放 + auto r = submit_and_wait(cmd, VK_NULL_HANDLE); + if (!r) return std::unexpected(r.error()); + } + return {}; + } + + // ══════════════════════════════════════════════════════════════════ + // copy_buffer_gpu — GPU 内 buffer 拷贝（使用 vkCmdCopyBuffer） + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result copy_buffer_gpu( + VkBuffer src, VkBuffer dst, VkDeviceSize size) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + + auto cmd_r = acquire_cmd(); + if (!cmd_r) return std::unexpected(cmd_r.error()); + auto [cmd, owns_cmd] = *cmd_r; + + VkBufferCopy cp{0, 0, size}; + vkCmdCopyBuffer(cmd, src, dst, 1, &cp); + + VkBufferMemoryBarrier b{}; + b.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + b.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + b.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.buffer = dst; + b.offset = 0; + b.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 1, &b, 0, nullptr); + + if (owns_cmd) + { + auto r = submit_and_wait(cmd, VK_NULL_HANDLE); + if (!r) return std::unexpected(r.error()); + } + return {}; + } + + // ══════════════════════════════════════════════════════════════════ + // clone_gpu — GPU 内深拷贝（分配新 buffer + copy，无 PCIe 传输） + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result clone_gpu(const GpuTensor& src) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + + // 1. 分配同形状的新 buffer + auto dst_res = GpuTensor::create_empty(src.rows(), src.cols(), *this); + if (!dst_res) return std::unexpected(dst_res.error()); + GpuTensor dst = std::move(*dst_res); + + // 2. GPU 内拷贝 + const VkDeviceSize size = static_cast( + src.rows() * src.cols() * sizeof(float)); + auto r = copy_buffer_gpu(src.buffer().impl(), dst.buffer().impl(), size); + if (!r) return std::unexpected(r.error()); + + return dst; + } + + // ══════════════════════════════════════════════════════════════════ + // slice_rows_gpu — 行切片（GPU 内拷贝连续行区间，无 PCIe 传输） + // 返回 (count, cols) 的新 GpuTensor，内容为 src 行 [start_row, start_row + count) + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result slice_rows_gpu( + const GpuTensor& src, std::size_t start_row, std::size_t count) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + + const std::size_t cols = src.cols(); + if (start_row + count > src.rows()) + return std::unexpected(Error{"slice_rows_gpu: range out of bounds"}); + + // 分配目标 buffer + auto dst_res = GpuTensor::create_empty(count, cols, *this); + if (!dst_res) return std::unexpected(dst_res.error()); + GpuTensor dst = std::move(*dst_res); + + // 行区间在行主序下连续：[start_row * cols, (start_row + count) * cols) + const VkDeviceSize src_offset = static_cast( + start_row * cols * sizeof(float)); + const VkDeviceSize size = static_cast( + count * cols * sizeof(float)); + + auto cmd_r = acquire_cmd(); + if (!cmd_r) return std::unexpected(cmd_r.error()); + auto [cmd, owns_cmd] = *cmd_r; + + VkBufferCopy cp{src_offset, 0, size}; + vkCmdCopyBuffer(cmd, src.buffer().impl(), dst.buffer().impl(), 1, &cp); + + VkBufferMemoryBarrier b{}; + b.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + b.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + b.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.buffer = dst.buffer().impl(); + b.offset = 0; + b.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 1, &b, 0, nullptr); + + if (owns_cmd) + { + auto r = submit_and_wait(cmd, VK_NULL_HANDLE); + if (!r) return std::unexpected(r.error()); + } + return dst; + } + + // ══════════════════════════════════════════════════════════════════ + // insert_rows_gpu — 行插入（GPU 内就地写入，无 PCIe 传输） + // 将 src 的所有行写入 dst 的行 [dst_start_row, dst_start_row + src.rows()) + // 真·就地修改（vkCmdCopyBuffer with dstOffset）。 + // 注意：dst 必须以 TRANSFER_DST_BIT 创建（create_empty 已包含）。 + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result insert_rows_gpu( + GpuTensor& dst, std::size_t dst_start_row, const GpuTensor& src) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + + const std::size_t cols = dst.cols(); + if (src.cols() != cols) + return std::unexpected(Error{"insert_rows_gpu: column count mismatch"}); + if (dst_start_row + src.rows() > dst.rows()) + return std::unexpected(Error{"insert_rows_gpu: range out of bounds"}); + + const VkDeviceSize dst_offset = static_cast( + dst_start_row * cols * sizeof(float)); + const VkDeviceSize size = static_cast( + src.rows() * cols * sizeof(float)); + + auto cmd_r = acquire_cmd(); + if (!cmd_r) return std::unexpected(cmd_r.error()); + auto [cmd, owns_cmd] = *cmd_r; + + VkBufferCopy cp{0, dst_offset, size}; + vkCmdCopyBuffer(cmd, src.buffer().impl(), dst.buffer().impl(), 1, &cp); + + VkBufferMemoryBarrier b{}; + b.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + b.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + b.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.buffer = dst.buffer().impl(); + b.offset = 0; + b.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 1, &b, 0, nullptr); + + if (owns_cmd) + { + auto r = submit_and_wait(cmd, VK_NULL_HANDLE); + if (!r) return std::unexpected(r.error()); + } + return {}; + } + + // ── Staging Path：CPU span → GPU → 计算 → GPU → CPU span ───────── + [[nodiscard]] Result matmul_direct( + std::span a, std::span b, + std::span c, std::size_t M, std::size_t N, std::size_t K, + uint32_t transA = 0, uint32_t transB = 0) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + + // 1. 创建临时 GPU buffers + auto a_buf = GpuBuffer::create_device_local( + device_.device(), *memory_pool_, M * K, + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + if (!a_buf) + return std::unexpected(a_buf.error()); + + auto b_buf = GpuBuffer::create_device_local( + device_.device(), *memory_pool_, K * N, + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + if (!b_buf) + return std::unexpected(b_buf.error()); + + // 2. 上传数据 + auto a_tensor = GpuTensor(std::make_shared(std::move(*a_buf)), M, K); + auto b_tensor = GpuTensor(std::make_shared(std::move(*b_buf)), K, N); + + auto r = upload_blocking(a_tensor, a); + if (!r) + return r; + + r = upload_blocking(b_tensor, b); + if (!r) + return r; + + // 3. 执行 matmul + auto c_tensor_res = matmul_gpu(a_tensor, b_tensor, transA, transB); + if (!c_tensor_res) + return std::unexpected(c_tensor_res.error()); + + // 4. 下载结果 + r = download_blocking(*c_tensor_res, c); + if (!r) + return r; + + return {}; + } + + // ══════════════════════════════════════════════════════════════════ + // transpose_gpu — 矩阵转置 (R, C) → (C, R) + // + // Push Constants (8 bytes): rows, cols + // Bindings: In(0), Out(1) + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result transpose_gpu(const GpuTensor& A) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (!has_transpose_pipeline()) + return std::unexpected(Error{"transpose pipeline not available"}); + + const uint32_t R = static_cast(A.rows()); + const uint32_t C = static_cast(A.cols()); + + // 输出 (C, R) + auto out_res = GpuTensor::create_empty(C, R, *this); + if (!out_res) return std::unexpected(out_res.error()); + GpuTensor output = std::move(*out_res); + + // 分配描述符集 + auto ds_r = alloc_desc_set(transpose_pipeline_.descriptor_layout()); + if (!ds_r) return std::unexpected(ds_r.error()); + VkDescriptorSet desc_set = *ds_r; + + // 写入描述符集 + VkDescriptorBufferInfo buf_infos[2]{ + {A.buffer().impl(), 0, VK_WHOLE_SIZE}, + {output.buffer().impl(), 0, VK_WHOLE_SIZE}, + }; + VkWriteDescriptorSet writes[2]{}; + for (int i = 0; i < 2; ++i) + { + writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[i].dstSet = desc_set; + writes[i].dstBinding = static_cast(i); + writes[i].descriptorCount = 1; + writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[i].pBufferInfo = &buf_infos[i]; + } + vkUpdateDescriptorSets(device_.device(), 2, writes, 0, nullptr); + + // 获取 command buffer + auto cmd_r = acquire_cmd(); + if (!cmd_r) + { + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(cmd_r.error()); + } + auto [cmd, owns_cmd] = *cmd_r; + + // 录制 + record_input_barriers(cmd, {A.buffer().impl()}); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + transpose_pipeline_.handle()); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + transpose_pipeline_.pipeline_layout(), 0, 1, &desc_set, 0, nullptr); + + const uint32_t push_data[2] = {R, C}; + vkCmdPushConstants(cmd, transpose_pipeline_.pipeline_layout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push_data), push_data); + + const uint32_t total = R * C; + const uint32_t wg_count = (total + 255) / 256; + vkCmdDispatch(cmd, wg_count, 1, 1); + record_output_barrier(cmd, output.buffer().impl()); + + if (owns_cmd) + { + auto r = submit_and_wait(cmd, desc_set); + if (!r) return std::unexpected(r.error()); + } + return output; + } + + // ══════════════════════════════════════════════════════════════════ + // gather_gpu — 按行索引查表 (GPU-native) + // + // table: (vocab, D), indices: (num,) → output: (num, D) + // output[i] = table[indices[i]]，越界索引返回零行 + // + // Push Constants (12 bytes): vocab, D, num + // Bindings: Table(0), Indices(1), Output(2) + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result gather_gpu( + const GpuTensor& table, const GpuTensor& indices) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (!has_gather_pipeline()) + return std::unexpected(Error{"gather pipeline not available"}); + + const uint32_t vocab = static_cast(table.rows()); + const uint32_t D = static_cast(table.cols()); + const uint32_t num = static_cast(indices.rows() * indices.cols()); + + // 输出 (num, D) + auto out_res = GpuTensor::create_empty(num, D, *this); + if (!out_res) return std::unexpected(out_res.error()); + GpuTensor output = std::move(*out_res); + + // 分配描述符集 + auto ds_r = alloc_desc_set(gather_pipeline_.descriptor_layout()); + if (!ds_r) return std::unexpected(ds_r.error()); + VkDescriptorSet desc_set = *ds_r; + + // 写入描述符集 + VkDescriptorBufferInfo buf_infos[3]{ + {table.buffer().impl(), 0, VK_WHOLE_SIZE}, + {indices.buffer().impl(), 0, VK_WHOLE_SIZE}, + {output.buffer().impl(), 0, VK_WHOLE_SIZE}, + }; + VkWriteDescriptorSet writes[3]{}; + for (int i = 0; i < 3; ++i) + { + writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[i].dstSet = desc_set; + writes[i].dstBinding = static_cast(i); + writes[i].descriptorCount = 1; + writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[i].pBufferInfo = &buf_infos[i]; + } + vkUpdateDescriptorSets(device_.device(), 3, writes, 0, nullptr); + + // 获取 command buffer + auto cmd_r = acquire_cmd(); + if (!cmd_r) + { + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(cmd_r.error()); + } + auto [cmd, owns_cmd] = *cmd_r; + + // 录制 + record_input_barriers(cmd, {table.buffer().impl(), indices.buffer().impl()}); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + gather_pipeline_.handle()); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + gather_pipeline_.pipeline_layout(), 0, 1, &desc_set, 0, nullptr); + + const uint32_t push_data[3] = {vocab, D, num}; + vkCmdPushConstants(cmd, gather_pipeline_.pipeline_layout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push_data), push_data); + + const uint32_t total = num * D; + const uint32_t wg_count = (total + 255) / 256; + vkCmdDispatch(cmd, wg_count, 1, 1); + record_output_barrier(cmd, output.buffer().impl()); + + if (owns_cmd) + { + auto r = submit_and_wait(cmd, desc_set); + if (!r) return std::unexpected(r.error()); + } + return output; + } + + // ══════════════════════════════════════════════════════════════════ + // scatter_add_gpu — 按行索引原子累加梯度 (GPU-native) + // + // dst: (vocab, D) 原地修改，indices: (num,), grad: (num, D) + // dst[indices[i]][d] += grad[i][d] + // 使用 CAS 循环实现 float atomicAdd + // + // Push Constants (12 bytes): vocab, D, num + // Bindings: Dst(0), Indices(1), Grad(2) + // + // 注意：dst buffer 内容以 uint 视角访问用于 atomicCompSwap。 + // dst 初始值必须为合法 float 位模式（如 0x00000000 = 0.0f）。 + // ══════════════════════════════════════════════════════════════════ + [[nodiscard]] Result scatter_add_gpu( + GpuTensor& dst, const GpuTensor& indices, const GpuTensor& grad) + { + if (!initialized_) + return std::unexpected(Error{"GPU backend not initialized"}); + if (!has_scatter_add_pipeline()) + return std::unexpected(Error{"scatter_add pipeline not available"}); + + const uint32_t vocab = static_cast(dst.rows()); + const uint32_t D = static_cast(dst.cols()); + const uint32_t num = static_cast(indices.rows() * indices.cols()); + + // 分配描述符集 + auto ds_r = alloc_desc_set(scatter_add_pipeline_.descriptor_layout()); + if (!ds_r) return std::unexpected(ds_r.error()); + VkDescriptorSet desc_set = *ds_r; + + // 写入描述符集（dst 使用 uint 视图用于原子操作） + VkDescriptorBufferInfo buf_infos[3]{ + {dst.buffer().impl(), 0, VK_WHOLE_SIZE}, + {indices.buffer().impl(), 0, VK_WHOLE_SIZE}, + {grad.buffer().impl(), 0, VK_WHOLE_SIZE}, + }; + VkWriteDescriptorSet writes[3]{}; + for (int i = 0; i < 3; ++i) + { + writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[i].dstSet = desc_set; + writes[i].dstBinding = static_cast(i); + writes[i].descriptorCount = 1; + writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[i].pBufferInfo = &buf_infos[i]; + } + vkUpdateDescriptorSets(device_.device(), 3, writes, 0, nullptr); + + // 获取 command buffer + auto cmd_r = acquire_cmd(); + if (!cmd_r) + { + vkFreeDescriptorSets(device_.device(), gpu_tensor_pool_, 1, &desc_set); + return std::unexpected(cmd_r.error()); + } + auto [cmd, owns_cmd] = *cmd_r; + + // 录制（dst 需要读+写屏障） + { + std::vector barriers; + VkBufferMemoryBarrier b{}; + b.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + b.srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + b.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.offset = 0; + b.size = VK_WHOLE_SIZE; + b.buffer = dst.buffer().impl(); + barriers.push_back(b); + b.buffer = indices.buffer().impl(); + barriers.push_back(b); + b.buffer = grad.buffer().impl(); + barriers.push_back(b); + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, 0, nullptr, + static_cast(barriers.size()), barriers.data(), + 0, nullptr); + } + + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + scatter_add_pipeline_.handle()); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, + scatter_add_pipeline_.pipeline_layout(), 0, 1, &desc_set, 0, nullptr); + + const uint32_t push_data[3] = {vocab, D, num}; + vkCmdPushConstants(cmd, scatter_add_pipeline_.pipeline_layout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push_data), push_data); + + const uint32_t total = num * D; + const uint32_t wg_count = (total + 255) / 256; + vkCmdDispatch(cmd, wg_count, 1, 1); + + // 输出屏障（dst 被修改，需要 memory barrier） + { + VkBufferMemoryBarrier b{}; + b.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + b.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + b.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.buffer = dst.buffer().impl(); + b.offset = 0; + b.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 1, &b, 0, nullptr); + } + + if (owns_cmd) + { + auto r = submit_and_wait(cmd, desc_set); + if (!r) return std::unexpected(r.error()); + } + return {}; + } +}; // GpuBackend + +// ══════════════════════════════════════════════════════════════════════ +// GpuBuffer 析构（类外定义）：batch 录制期间延迟销毁，避免已录制的 +// descriptor 引用已销毁的 buffer（VUID-vkDestroyBuffer-buffer-00922）。 +// ══════════════════════════════════════════════════════════════════════ +GpuBuffer::~GpuBuffer() +{ + if (buffer_ == VK_NULL_HANDLE) + return; + + auto& backend = GpuBackend::instance(); + if (backend.is_initialized() && backend.in_batch()) + { + // batch 模式：copy-on-write 替换产生的旧 buffer 仍被已录制的 + // descriptor 引用，延迟到 end_batch/flush_batch 后统一销毁。 + backend.defer_buffer_destroy(device_, buffer_, alloc_, pool_); + } + else + { + vkDestroyBuffer(device_, buffer_, nullptr); + if (pool_ && alloc_.valid()) + pool_->free(alloc_); + } + + buffer_ = VK_NULL_HANDLE; + alloc_ = {}; + pool_.reset(); +} + +} // namespace nn + +#endif // NN_HAS_VULKAN +#endif // NN_VK_BACKEND_HPP diff --git a/include/neuralnet.cpp/cli/engine_factory.hpp b/include/neuralnet.cpp/cli/engine_factory.hpp new file mode 100644 index 00000000..336b2a8d --- /dev/null +++ b/include/neuralnet.cpp/cli/engine_factory.hpp @@ -0,0 +1,83 @@ +// ── engine_factory.hpp — 计算引擎选择工厂 ────────────────────────────────── +// +// 抽取自 mnist_train/mnist_infer/text_train/text_infer/mnist_bench 中重复约 50 行的 +// `#ifdef NN_HAS_VULKAN / #ifdef NN_HAS_CUDA / CpuEngine 回退` 三段式样板。 +// +// 选择优先级：--cuda > --gpu > CPU +// - CUDA 路径：要求编译期 NN_HAS_CUDA，运行时 backend.initialize() 成功 +// - Vulkan 路径：要求编译期 NN_HAS_VULKAN，运行时 backend.initialize() 成功 +// - 任一失败均回退到 CpuEngine，并打印原因到 log +// +// 调用方约束： +// 返回的 engine 必须早于任何持有其非拥有指针的对象（如 nn::Model）析构。 +// ───────────────────────────────────────────────────────────────────────── + +#pragma once + +#include // std::ostream 前向声明 +#include + +#include "neuralnet.cpp/compute_engine.hpp" +#include "neuralnet.cpp/cpu_engine.hpp" +#ifdef NN_HAS_VULKAN +#include "neuralnet.cpp/gpu_engine.hpp" +#endif +#ifdef NN_HAS_CUDA +#include "neuralnet.cpp/cuda_engine.hpp" +#endif + +namespace nn::cli +{ + // ── 引擎选择配置 ─────────────────────────────────────────────────────── + struct EngineConfig + { + bool use_gpu = false; // 启用 Vulkan GpuEngine + bool use_cuda = false; // 启用 CUDA CudaEngine（优先级高于 use_gpu） + }; + + // ── 创建计算引擎 ─────────────────────────────────────────────────────── + // 根据 use_gpu/use_cuda 选择 GpuEngine/CudaEngine/CpuEngine。 + // 返回 unique_ptr，并打印选择信息到 log。 + [[nodiscard]] inline std::unique_ptr create_engine( + const EngineConfig &cfg, std::ostream &log = std::cerr) + { + // 优先级：CUDA > Vulkan > CPU + if (cfg.use_cuda) + { +#ifdef NN_HAS_CUDA + auto &backend = nn::CudaBackend::instance(); + auto init_r = backend.initialize(); + if (init_r) + { + log << "CUDA GPU 加速已启用 (" << backend.device_props().name << ")\n\n"; + return std::make_unique(backend); + } + log << "CUDA 初始化失败: " << init_r.error().message << "\n" + << "回退到 CPU 模式\n\n"; +#else + log << "未编译 CUDA 支持，使用 CPU 模式\n\n"; +#endif + return std::make_unique(); + } + + if (cfg.use_gpu) + { +#ifdef NN_HAS_VULKAN + auto &backend = nn::GpuBackend::instance(); + auto init_r = backend.initialize(); + if (init_r) + { + log << "GPU 加速已启用 (Vulkan GpuEngine)\n\n"; + return std::make_unique(backend); + } + log << "GPU 初始化失败: " << init_r.error().message << "\n" + << "回退到 CPU 模式\n\n"; +#else + log << "未编译 Vulkan 支持，使用 CPU 模式\n\n"; +#endif + return std::make_unique(); + } + + return std::make_unique(); + } +} // namespace nn::cli diff --git a/include/neuralnet.cpp/cli/lr_scheduler.hpp b/include/neuralnet.cpp/cli/lr_scheduler.hpp new file mode 100644 index 00000000..2e2d54da --- /dev/null +++ b/include/neuralnet.cpp/cli/lr_scheduler.hpp @@ -0,0 +1,116 @@ +// ── lr_scheduler.hpp — 学习率调度器 ───────────────────────────────────────── +// +// 抽取自 mnist_train/text_train 中逐字重复的 compute_epoch_lr lambda。 +// +// 调度优先级： +// 1. lr_per_epoch 非空 → 直接索引；epoch 超出范围时使用最后一个值 +// 2. schedule == "cosine" → 线性 warmup + 余弦退火 +// 3. 其他 → 固定 base_lr +// +// 注：cosine 退火使用 0.5 * (1 + cos(π * progress)) 的标准形式。 +// ───────────────────────────────────────────────────────────────────────── + +#pragma once + +#include +#include +#include +#include + +#include "neuralnet.cpp/config.hpp" // nn::Scalar + +namespace nn::cli +{ + // ── 学习率调度配置 ───────────────────────────────────────────────────── + struct LrScheduleConfig + { + nn::Scalar base_lr = 0.001f; // 基础学习率（fixed 模式直接返回） + nn::Scalar min_lr = 0.0f; // 余弦退火最低学习率 + int warmup_epochs = 0; // 线性预热轮数（0 = 不预热） + int total_epochs = 10; // 总训练轮数（用于 cosine 进度计算） + std::string schedule = "constant"; // "constant" | "cosine" + // 可选：每 epoch 手动指定 lr（优先级最高） + std::vector lr_per_epoch; + }; + + // ── 计算第 epoch 的学习率 ────────────────────────────────────────────── + // 调度优先级：lr_per_epoch > cosine > constant + [[nodiscard]] inline nn::Scalar compute_epoch_lr(const LrScheduleConfig &cfg, int epoch) + { + // 手动指定优先 + if (!cfg.lr_per_epoch.empty()) + { + if (epoch < static_cast(cfg.lr_per_epoch.size())) + return cfg.lr_per_epoch[epoch]; + return cfg.lr_per_epoch.back(); // 超出部分用最后一个 + } + + if (cfg.schedule == "cosine") + { + const nn::Scalar max_lr = cfg.base_lr; + const nn::Scalar lr_min = cfg.min_lr; + if (cfg.warmup_epochs > 0 && epoch < cfg.warmup_epochs) + { + // 线性预热 + return max_lr * static_cast(epoch + 1) / + static_cast(cfg.warmup_epochs); + } + // 余弦退火 + const int cosine_epochs = cfg.total_epochs - cfg.warmup_epochs; + if (cosine_epochs <= 0) return max_lr; + const nn::Scalar progress = + static_cast(epoch - cfg.warmup_epochs) / + static_cast(cosine_epochs); + return lr_min + 0.5f * (max_lr - lr_min) * + (1.0f + std::cos(3.14159265358979323846f * progress)); + } + + return cfg.base_lr; // fixed / constant + } + + // ── Step 级学习率调度配置 ──────────────────────────────────────────── + // 与 epoch 级调度互补：epoch 级在大量步数（如 TinyStories 60MB 每 epoch + // 数千步）下粒度太粗，step 级可按单个训练步做 warmup + 余弦退火。 + struct StepLrScheduleConfig + { + nn::Scalar base_lr = 0.001f; // 峰值学习率 + nn::Scalar min_lr = 0.0f; // 余弦退火最低学习率 + int warmup_steps = 0; // 线性预热步数（0 = 不预热） + int total_steps = 1; // 总训练步数（用于 cosine 进度） + bool cosine = false; // true: 预热后余弦退火到 min_lr； + // false: 预热后保持 base_lr + }; + + // ── 计算第 step 步的学习率（step 为 0-based 全局步号） ────────────── + // 预热阶段：base_lr * (step+1)/warmup_steps（线性上升） + // 预热后： + // cosine=true → 标准余弦退火到 min_lr + // cosine=false → 保持 base_lr + [[nodiscard]] inline nn::Scalar compute_step_lr(const StepLrScheduleConfig &cfg, int step) + { + const nn::Scalar max_lr = cfg.base_lr; + + // 线性预热 + if (cfg.warmup_steps > 0 && step < cfg.warmup_steps) + { + return max_lr * static_cast(step + 1) / + static_cast(cfg.warmup_steps); + } + + // 余弦退火（标准形式 0.5*(1+cos(π*progress))） + if (cfg.cosine) + { + const int cosine_steps = cfg.total_steps - cfg.warmup_steps; + if (cosine_steps <= 0) return max_lr; + const nn::Scalar progress = + static_cast(step - cfg.warmup_steps) / + static_cast(cosine_steps); + const nn::Scalar p = + progress < 0 ? nn::Scalar{0} : (progress > 1 ? nn::Scalar{1} : progress); + return cfg.min_lr + 0.5f * (max_lr - cfg.min_lr) * + (1.0f + std::cos(3.14159265358979323846f * p)); + } + + return max_lr; // 预热后恒定 + } +} // namespace nn::cli diff --git a/include/neuralnet.cpp/cli/mnist_io.hpp b/include/neuralnet.cpp/cli/mnist_io.hpp new file mode 100644 index 00000000..49a04234 --- /dev/null +++ b/include/neuralnet.cpp/cli/mnist_io.hpp @@ -0,0 +1,225 @@ +// ── mnist_io.hpp — MNIST 数据加载与评估工具 ───────────────────────────────── +// +// 抽取自 mnist_train/mnist_bench 中重复的 load_csv 与 evaluate 函数。 +// +// load_mnist_csv: +// - 一次性读取整个 CSV 文件到内存，再以 from_chars 快速解析 +// - 文件格式：每行 "label,f1,f2,...,f784"（首列为 0-9 标签，后接 784 个像素值） +// - 返回 (feat_mat[784, N], label_mat[10, N])，label_mat 为 one-hot 编码 +// - max_samples = 0 表示加载全部样本 +// +// evaluate_mnist: +// - 全量前向后下载到 CPU 做 argmax，计算准确率 +// - GPU 模式下使用 begin_batch/end_batch 包裹 forward，消除 per-primitive 提交开销 +// - eval_samples > 0 时只评估前 N 个样本（Transformer 评估成本较高） +// +// 注：用户原始签名 evaluate_mnist(nn::Layer&, ...) 实际不可行，因为 nn::Model +// 不是 nn::Layer 的派生类，且 Model::forward 自带 engine 绑定（签名不同）。 +// 这里改为 nn::Model&，与现有调用点 model.forward(...) 一致，行为完全等价。 +// ───────────────────────────────────────────────────────────────────────── + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "neuralnet.cpp/algebra_matrix.hpp" // nn::Matrix +#include "neuralnet.cpp/compute_engine.hpp" // nn::ComputeEngine +#include "neuralnet.cpp/core_errors.hpp" // nn::Result / nn::Error +#include "neuralnet.cpp/domain_mnist.hpp" // nn::MNIST_NUM_CLASSES +#include "neuralnet.cpp/model_container.hpp" // nn::Model + +namespace nn::cli +{ + // ── 从 CSV 加载 MNIST 数据 ───────────────────────────────────────────── + // 返回 (feat_mat[feat_dim, N], label_mat[10, N])，label_mat 为 one-hot。 + // max_samples = 0 表示加载全部样本。 + [[nodiscard]] inline nn::Result> + load_mnist_csv(const std::string &path, std::size_t max_samples = 0) + { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file.is_open()) + return std::unexpected(nn::Error{"Cannot open file: " + path}); + + const auto file_size = file.tellg(); + file.seekg(0); + + std::string buffer(static_cast(file_size), '\0'); + file.read(buffer.data(), file_size); + file.close(); + + std::size_t row_count = 0; + for (char c : buffer) + if (c == '\n') ++row_count; + // 末行无换行符时也要计入（否则最后一行样本被静默丢弃） + if (!buffer.empty() && buffer.back() != '\n') + ++row_count; + if (row_count == 0) + return std::unexpected(nn::Error{"CSV file is empty or malformed: " + path}); + + if (max_samples > 0 && max_samples < row_count) + row_count = max_samples; + + const char *ptr = buffer.data(); + const char *end = buffer.data() + buffer.size(); + + // 探测第一行以确定特征维度 + int first_label = 0; + std::size_t feat_dim = 0; + { + const char *p = ptr; + auto [p1, ec1] = std::from_chars(p, end, first_label); + p = p1; + std::size_t cnt = 0; + while (p < end && *p != '\n' && *p != '\r') + { + if (*p == ',') + { + ++cnt; + nn::Scalar tmp; + auto [p2, ec2] = std::from_chars(p + 1, end, tmp); + p = p2; + } + else + ++p; + } + feat_dim = cnt; + } + + std::vector features(row_count * feat_dim); + std::vector labels(row_count); + + std::size_t row = 0; + ptr = buffer.data(); + + while (ptr < end && row < row_count) + { + auto [p_label, ec_label] = std::from_chars(ptr, end, labels[row]); + if (ec_label != std::errc{}) + return std::unexpected( + nn::Error{"Failed to parse label at row " + std::to_string(row)}); + ptr = p_label; + + for (std::size_t j = 0; j < feat_dim; ++j) + { + if (ptr < end && *ptr == ',') ++ptr; + nn::Scalar val; + auto [p_feat, ec_feat] = std::from_chars(ptr, end, val); + if (ec_feat != std::errc{}) + return std::unexpected( + nn::Error{"Failed to parse feature at row " + std::to_string(row)}); + features[row * feat_dim + j] = val; + ptr = p_feat; + } + + while (ptr < end && (*ptr == '\r' || *ptr == '\n')) ++ptr; + ++row; + } + + const std::size_t N = row; + + nn::Matrix feat_mat(feat_dim, N); + for (std::size_t i = 0; i < N; ++i) + for (std::size_t j = 0; j < feat_dim; ++j) + feat_mat.set_value_unchecked(j, i, features[i * feat_dim + j]); + + nn::Matrix label_mat(10, N); + for (std::size_t i = 0; i < N; ++i) + { + int lbl = labels[i]; + if (lbl < 0 || lbl >= 10) + return std::unexpected( + nn::Error{"Label out of range: " + std::to_string(lbl)}); + label_mat.set_value_unchecked(lbl, i, 1.0); + } + + return std::pair{std::move(feat_mat), std::move(label_mat)}; + } + + // ── 评估模型准确率 ───────────────────────────────────────────────────── + // 全量前向后下载到 CPU 做 argmax，计算 top-1 准确率。 + // eval_samples > 0 时只评估前 N 个样本（用于 Transformer 等评估成本高的场景）。 + // + // 注：原 evaluate 签名 (nn::Layer&, ...) 不可行 —— nn::Model 不派生自 nn::Layer， + // 且 Model::forward 自带 engine 绑定，签名不同于 Layer::forward(engine, ...)。 + // 此处使用 nn::Model&，与现有调用点 model.forward(...) 一致。 + [[nodiscard]] inline nn::Result + evaluate_mnist(nn::Model &model, nn::ComputeEngine &engine, + const nn::Matrix &x, const nn::Matrix &y_onehot, + std::size_t eval_samples = 0) + { + const std::size_t N = + (eval_samples > 0) ? std::min(x.cols(), eval_samples) : x.cols(); + if (N == 0) + return std::unexpected(nn::Error{"evaluate_mnist: empty dataset"}); + + // 若截取子集，则拷贝前 N 列；全量评估直接引用原矩阵（避免无谓拷贝） + nn::Matrix x_sub, y_sub; + const nn::Matrix *xp = &x; + const nn::Matrix *yp = &y_onehot; + if (N < x.cols()) + { + x_sub = nn::Matrix(x.rows(), N); + y_sub = nn::Matrix(y_onehot.rows(), N); + for (std::size_t i = 0; i < N; ++i) + { + for (std::size_t r = 0; r < x.rows(); ++r) + x_sub.set_value_unchecked(r, i, x.at_unchecked(r, i)); + for (std::size_t r = 0; r < y_onehot.rows(); ++r) + y_sub.set_value_unchecked(r, i, y_onehot.at_unchecked(r, i)); + } + xp = &x_sub; + yp = &y_sub; + } + + auto x_tensor_r = engine.from_matrix(*xp); + if (!x_tensor_r) return std::unexpected(std::move(x_tensor_r).error()); + + // batch 模式加速 forward（GPU 下消除 per-primitive 提交开销） + auto bb = engine.begin_batch(); + if (!bb) return std::unexpected(bb.error()); + + auto out_tensor_r = model.forward(*x_tensor_r); + if (!out_tensor_r) return std::unexpected(std::move(out_tensor_r).error()); + + auto eb = engine.end_batch(); + if (!eb) return std::unexpected(eb.error()); + + auto out_r = engine.to_matrix(*out_tensor_r); + if (!out_r) return std::unexpected(std::move(out_r).error()); + const auto &out = *out_r; + + int correct = 0; + for (std::size_t i = 0; i < N; ++i) + { + nn::Scalar max_val = out.at_unchecked(0, i); + int pred = 0; + for (int j = 1; j < static_cast(nn::MNIST_NUM_CLASSES); ++j) + { + nn::Scalar val = out.at_unchecked(j, i); + if (val > max_val) + { + max_val = val; + pred = j; + } + } + int true_label = -1; + for (int j = 0; j < static_cast(nn::MNIST_NUM_CLASSES); ++j) + { + if (yp->at_unchecked(j, i) == 1.0) + { + true_label = j; + break; + } + } + if (pred == true_label) + ++correct; + } + return static_cast(correct) / static_cast(N); + } +} // namespace nn::cli diff --git a/include/neuralnet.cpp/cli/train_common.hpp b/include/neuralnet.cpp/cli/train_common.hpp new file mode 100644 index 00000000..9063a863 --- /dev/null +++ b/include/neuralnet.cpp/cli/train_common.hpp @@ -0,0 +1,168 @@ +// ── train_common.hpp — 通用训练参数解析 ───────────────────────────────────── +// +// 抽取自 mnist_train/text_train 中重复的 flag 解析逻辑。 +// 覆盖以下通用 flag： +// --epochs 训练轮数（正整数） +// --lr 学习率 +// --batch-size 批大小（正整数） +// --optimizer 优化器：sgd/sgd_momentum/adam/adamw/muon +// --weight-decay AdamW 权重衰减系数 +// --min-lr 余弦退火最低学习率 +// --warmup-epochs 线性预热轮数 +// --lr-per-epoch 手动指定每轮学习率（逗号分隔） +// --lr-schedule 学习率调度：fixed/constant/cosine +// --gpu 启用 Vulkan GPU 加速 +// --cuda 启用 CUDA GPU 加速 +// +// 用法：在调用方的 parse_args 循环中优先委托： +// for (int i = 1; i < argc; ++i) { +// if (nn::cli::parse_train_common_args(argc, argv, i, common)) continue; +// // 程序专属 flag... +// } +// +// 返回值约定： +// true = 已处理该 flag（i 已递增到本次消费的最后一个 argv 索引） +// false = 不是通用 flag（i 未变，调用方自行处理） +// +// 注：--help 由调用方自行处理（各程序 print_usage 不同）。 +// ───────────────────────────────────────────────────────────────────────── + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "neuralnet.cpp/config.hpp" // nn::Scalar +#include "neuralnet.cpp/core_errors.hpp" // nn::parse_number + +namespace nn::cli +{ + // ── 通用训练参数 ─────────────────────────────────────────────────────── + // 默认值仅作为结构体占位；调用方应根据程序语义在解析前覆盖。 + struct TrainCommonArgs + { + int epochs = 10; + nn::Scalar lr = 0.001f; + std::size_t batch_size = 32; + std::string optimizer = "adam"; + nn::Scalar weight_decay = 0.0f; + nn::Scalar min_lr = 0.0f; + int warmup_epochs = 0; + std::string lr_schedule = "constant"; // "fixed"/"constant" 等价，"cosine" 退火 + std::vector lr_per_epoch; + bool use_gpu = false; + bool use_cuda = false; + }; + + // ── 解析通用训练参数 ─────────────────────────────────────────────────── + // 在 argv[i] 处尝试解析一个通用 flag。 + // 成功消费时返回 true 并将 i 递增到本次消费的最后一个 argv 索引； + // 不是通用 flag 时返回 false（i 不变），由调用方继续判断。 + // 解析失败时打印错误并 std::exit(1)（与现有 parse_args 行为一致）。 + inline bool parse_train_common_args( + int argc, char *argv[], int &i, TrainCommonArgs &cfg) + { + std::string arg = argv[i]; + + if (arg == "--epochs" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --epochs: " << v.error().message << "\n"; std::exit(1); } + if (*v <= 0) { std::cerr << "--epochs 必须为正整数\n"; std::exit(1); } + cfg.epochs = *v; + return true; + } + if (arg == "--lr" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --lr: " << v.error().message << "\n"; std::exit(1); } + cfg.lr = *v; + return true; + } + if (arg == "--batch-size" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --batch-size: " << v.error().message << "\n"; std::exit(1); } + if (*v == 0) { std::cerr << "--batch-size 必须为正整数\n"; std::exit(1); } + cfg.batch_size = *v; + return true; + } + if (arg == "--optimizer" && i + 1 < argc) + { + cfg.optimizer = argv[++i]; + if (cfg.optimizer != "sgd" && cfg.optimizer != "sgd_momentum" && + cfg.optimizer != "adam" && cfg.optimizer != "adamw" && + cfg.optimizer != "muon") + { + std::cerr << "未知优化器: " << cfg.optimizer + << "，可选: sgd, sgd_momentum, adam, adamw, muon\n"; + std::exit(1); + } + return true; + } + if (arg == "--weight-decay" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --weight-decay: " << v.error().message << "\n"; std::exit(1); } + cfg.weight_decay = *v; + return true; + } + if (arg == "--min-lr" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --min-lr: " << v.error().message << "\n"; std::exit(1); } + cfg.min_lr = *v; + return true; + } + if (arg == "--warmup-epochs" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --warmup-epochs: " << v.error().message << "\n"; std::exit(1); } + cfg.warmup_epochs = *v; + return true; + } + if (arg == "--lr-per-epoch" && i + 1 < argc) + { + std::string dims_str = argv[++i]; + std::stringstream ss(dims_str); + std::string token; + cfg.lr_per_epoch.clear(); + while (std::getline(ss, token, ',')) + { + auto v = nn::parse_number(token); + if (!v) { std::cerr << "无效 --lr-per-epoch 值: " << v.error().message << "\n"; std::exit(1); } + cfg.lr_per_epoch.push_back(*v); + } + return true; + } + if (arg == "--lr-schedule" && i + 1 < argc) + { + cfg.lr_schedule = argv[++i]; + // "fixed" 与 "constant" 等价（历史兼容）；"cosine" 为余弦退火 + if (cfg.lr_schedule != "fixed" && cfg.lr_schedule != "constant" && + cfg.lr_schedule != "cosine") + { + std::cerr << "未知 --lr-schedule: " << cfg.lr_schedule + << "，可选: fixed, constant, cosine\n"; + std::exit(1); + } + return true; + } + if (arg == "--gpu") + { + cfg.use_gpu = true; + return true; + } + if (arg == "--cuda") + { + cfg.use_cuda = true; + return true; + } + + return false; // 不是通用 flag + } +} // namespace nn::cli diff --git a/include/neuralnet.cpp/compute_engine.hpp b/include/neuralnet.cpp/compute_engine.hpp new file mode 100644 index 00000000..1fa641c5 --- /dev/null +++ b/include/neuralnet.cpp/compute_engine.hpp @@ -0,0 +1,254 @@ +#ifndef NN_COMPUTE_ENGINE_HPP +#define NN_COMPUTE_ENGINE_HPP + +// ── compute_engine.hpp — 计算引擎抽象接口 ───────────────────────────────── +// ComputeEngine 是与底层硬件接触的唯一抽象层。 +// +// 设计原则（铁律）： +// 1. 本接口只提供 op-level 原语（矩阵乘法、加法、转置、归约、广播、 +// 逐元素运算等），绝不包含任何算法。 +// 2. ReLU、GeLU、LayerNorm、Softmax、Attention 等算法由 Layer 层 +// 通过组合原语表达。 +// 3. Layer 持有 ComputeEngine 引用，forward/backward 只写一次， +// CPU/GPU 由引擎实现自动分发。 +// +// 原语分类： +// - 矩阵级：matmul, batched_matmul, transpose, add_inplace, scale_inplace, zero +// - 归约级：row_reduce_sum, col_reduce_sum +// - 广播级：broadcast_row_inplace, broadcast_col_inplace +// - 逐元素：elementwise_unary, elementwise_binary, elementwise_binary_scalar +// - 条件选择：elementwise_select_scalar_cond +// +// 批处理控制： +// - begin_batch / end_batch：CPU 引擎为 no-op；GPU 引擎录制到 +// command buffer，end_batch 时统一提交。 +// ───────────────────────────────────────────────────────────────────────── + +#include + +#include "config.hpp" +#include "core_errors.hpp" +#include "tensor.hpp" + +namespace nn +{ + +// ══════════════════════════════════════════════════════════════════════════ +// 算子枚举（op-level，不含算法语义） +// ══════════════════════════════════════════════════════════════════════════ + +// 一元算子 +enum class UnaryOp : uint32_t +{ + Neg = 0, // -x + Exp = 1, // e^x + Log = 2, // ln(x) + Sqrt = 3, // √x + Rsqrt = 4, // 1/√x + Abs = 5, // |x| + Tanh = 6, // tanh(x) +}; + +// 二元算子 +enum class BinaryOp : uint32_t +{ + Add = 0, // a + b + Sub = 1, // a - b + Mul = 2, // a * b + Div = 3, // a / b + Max = 4, // max(a, b) + Min = 5, // min(a, b) +}; + +// 比较算子（用于条件选择） +enum class CompareOp : uint32_t +{ + Lt = 0, // a < b + Le = 1, // a <= b + Gt = 2, // a > b + Ge = 3, // a >= b + Eq = 4, // a == b + Ne = 5, // a != b +}; + +// ══════════════════════════════════════════════════════════════════════════ +// ComputeEngine — 计算引擎抽象接口 +// ══════════════════════════════════════════════════════════════════════════ +class ComputeEngine +{ +public: + virtual ~ComputeEngine() = default; + + // ── 设备查询 ────────────────────────────────────────────────────────── + [[nodiscard]] virtual Device device() const noexcept = 0; + + // ── 批处理控制 ──────────────────────────────────────────────────────── + // CPU 引擎：no-op（操作立即同步执行） + // GPU 引擎：begin 开始录制，end 统一提交 + fence wait + [[nodiscard]] virtual Result begin_batch() = 0; + [[nodiscard]] virtual Result end_batch() = 0; + + // ── 批处理中点刷新（防 TDR） ───────────────────────────────────────── + // GPU 引擎：提交当前 command buffer 并等待完成，然后自动开始新的录制。 + // 可在 forward 与 backward 之间调用，将一次大提交拆分为多次小提交， + // 避免单次提交时间过长触发 Windows TDR。 + // CPU 引擎：no-op。 + [[nodiscard]] virtual Result flush_batch() { return {}; } + + // ── 张量工厂 ────────────────────────────────────────────────────────── + [[nodiscard]] virtual Tensor create_tensor(std::size_t rows, std::size_t cols) = 0; + [[nodiscard]] virtual Result from_matrix(const Matrix& m) = 0; + [[nodiscard]] virtual Result to_matrix(const Tensor& t) = 0; + + // 将 CPU Matrix 数据写入已有 Tensor（CPU 拷贝 / GPU 上传） + // 用于序列化加载、Optimizer 参数写回等场景 + [[nodiscard]] virtual Result copy_from(Tensor& dst, const Matrix& src) = 0; + + // 深拷贝 Tensor（CPU 矩阵拷贝 / GPU buffer 拷贝，无 PCIe 传输） + // 用于需要修改中间结果但不影响原 Tensor 的场景 + [[nodiscard]] virtual Result clone(const Tensor& src) = 0; + + // ── 行切片原语（op-level 数据操作，不含算法语义） ────────────────── + // 返回 src 的行 [start_row, start_row + count) 的连续拷贝。 + // 用于多头注意力中 per-head Q/K/V 切片等场景。 + [[nodiscard]] virtual Result slice_rows( + const Tensor& src, std::size_t start_row, std::size_t count) = 0; + + // 将 src 的所有行写入 dst 的行 [dst_start_row, dst_start_row + src.rows())。 + // 真·就地修改（GPU 用 vkCmdCopyBuffer with dstOffset）。 + // 用于多头注意力中 per-head 输出拼接等场景。 + [[nodiscard]] virtual Result insert_rows( + Tensor& dst, std::size_t dst_start_row, const Tensor& src) = 0; + + // ── 行 gather / scatter-add 原语（op-level 数据操作） ──────────────── + // gather_rows: 按 indices 从 table 中按行查表，等价于 tf.gather / torch.index_select + // table: (vocab, D) + // indices: (num_indices,) — 行索引；越界索引返回零行（防御性，不抛错） + // 输出: (num_indices, D)，out[i] = table[indices[i]] + // 典型用途：Token embedding 查表（避免 Layer 内手动 to_matrix + at_unchecked） + [[nodiscard]] virtual Result gather_rows( + const Tensor& table, const Tensor& indices) = 0; + + // scatter_add_rows: 按 indices 把 grad 的行原子累加到 dst 的对应行 + // dst: (vocab, D)，原地修改 + // indices: (num_indices,) + // grad: (num_indices, D) + // 语义: dst[indices[i]] += grad[i] (重复 indices 会被多次累加) + // 典型用途：Embedding 反向梯度按 token ID 累加（替代 Layer 内手动循环） + [[nodiscard]] virtual Result scatter_add_rows( + Tensor& dst, const Tensor& indices, const Tensor& grad) = 0; + + // 3D 维度转置：(M, B, N) ↔ (B, M, N) + // inverse=false: 输入 (M, B*N) → 输出 (B*M, N) + // out[b*M + m, n] = in[m, b*N + n] + // inverse=true: 输入 (B*M, N) → 输出 (M, B*N) + // out[m, b*N + n] = in[b*M + m, n] + // 典型用途：MHA 批量化时把 (H*d_k, batch*seq) 重排为 (batch*H*d_k, seq)， + // 使 batched_matmul 能按 batch*H 切分行块。 + [[nodiscard]] virtual Result rearrange_3d( + const Tensor& x, std::size_t M, std::size_t B, std::size_t N, + bool inverse = false) = 0; + // ── 矩阵转置：A (R, C) → out (C, R) ── + // 纯 layout 操作，零算法语义。用于 embedding 列布局转换等场景。 + [[nodiscard]] virtual Result transpose(const Tensor& A) = 0; + // ══════════════════════════════════════════════════════════════════════ + // 矩阵级原语 + // ══════════════════════════════════════════════════════════════════════ + + // C = A × B（支持转置标志） + // transA: 使用 A^T，transB: 使用 B^T + [[nodiscard]] virtual Result matmul( + const Tensor& A, const Tensor& B, + bool transA = false, bool transB = false) = 0; + + // 批量矩阵乘法：对每个 batch b 计算 C_b = op(A_b, B_b)，结果垂直堆叠 + // A: (batch * A_rows_per_batch, A_cols) — 按 batch 切分为连续行块 + // B: (batch * B_rows_per_batch, B_cols) + // 输出: (batch * M, N)，M/N 为每个 batch 的逻辑输出维度 + // transA=0: A_b 为 (M, K)，transA=1: A_b 存储为 (K, M) 按 A_b^T 使用 + // transB=0: B_b 为 (K, N)，transB=1: B_b 存储为 (N, K) 按 B_b^T 使用 + // 典型用途：多头注意力的 Q^T×K 和 V×A 批量化（消除 per-head 循环） + [[nodiscard]] virtual Result batched_matmul( + const Tensor& A, const Tensor& B, + std::size_t batch, + bool transA = false, bool transB = false) = 0; + + // A += B（逐元素，同形状） + [[nodiscard]] virtual Result add_inplace(Tensor& A, const Tensor& B) = 0; + + // A *= scalar + [[nodiscard]] virtual Result scale_inplace(Tensor& A, Scalar s) = 0; + + // A += scalar * B（融合 axpy：单次 dispatch 替代 clone+scale+add 三步） + // 用于 Optimizer 的 scale_add_ 辅助方法，减少 2 个临时 GPU buffer 分配 + [[nodiscard]] virtual Result axpy_inplace(Tensor& A, Scalar scalar, const Tensor& B) = 0; + + // A = 0 + [[nodiscard]] virtual Result zero(Tensor& A) = 0; + + // ══════════════════════════════════════════════════════════════════════ + // 归约原语 + // ══════════════════════════════════════════════════════════════════════ + + // 按行求和：A (rows, cols) → out (rows, 1) + // out[r] = Σ_c A[r][c] + [[nodiscard]] virtual Result row_reduce_sum(const Tensor& A) = 0; + + // 按列求和：A (rows, cols) → out (1, cols) + // out[c] = Σ_r A[r][c] + [[nodiscard]] virtual Result col_reduce_sum(const Tensor& A) = 0; + + // 按行求最大值：A (rows, cols) → out (rows, 1) + // out[r] = max_c A[r][c] + [[nodiscard]] virtual Result row_reduce_max(const Tensor& A) = 0; + + // 按列求最大值：A (rows, cols) → out (1, cols) + // out[c] = max_r A[r][c] + [[nodiscard]] virtual Result col_reduce_max(const Tensor& A) = 0; + + // ══════════════════════════════════════════════════════════════════════ + // 广播原语 + // ══════════════════════════════════════════════════════════════════════ + + // 按行广播：A (R, C) op= row_vec (R, 1) + // A[r][c] = op(A[r][c], row_vec[r][0]) + [[nodiscard]] virtual Result broadcast_row_inplace( + Tensor& A, const Tensor& row_vec, BinaryOp op) = 0; + + // 按列广播：A (R, C) op= col_vec (1, C) + // A[r][c] = op(A[r][c], col_vec[0][c]) + [[nodiscard]] virtual Result broadcast_col_inplace( + Tensor& A, const Tensor& col_vec, BinaryOp op) = 0; + + // ══════════════════════════════════════════════════════════════════════ + // 逐元素原语 + // ══════════════════════════════════════════════════════════════════════ + + // out = unary_op(A) + [[nodiscard]] virtual Result elementwise_unary( + UnaryOp op, const Tensor& A) = 0; + + // out = binary_op(A, B) + [[nodiscard]] virtual Result elementwise_binary( + BinaryOp op, const Tensor& A, const Tensor& B) = 0; + + // out = binary_op(A, scalar) 或 binary_op(scalar, A) + // scalar_first=true: out = op(scalar, A)；false: out = op(A, scalar) + [[nodiscard]] virtual Result elementwise_binary_scalar( + BinaryOp op, const Tensor& A, Scalar s, bool scalar_first = false) = 0; + + // ══════════════════════════════════════════════════════════════════════ + // 条件选择原语 + // ══════════════════════════════════════════════════════════════════════ + + // out = compare_op(A, scalar_b) ? then_t : scalar_else + // 条件操作数 2 为标量，then 为张量，else 为标量。 + // 典型用途：ReLU 反向 (x > 0) ? grad : 0 + [[nodiscard]] virtual Result elementwise_select_scalar_cond( + CompareOp cmp, const Tensor& A, Scalar scalar_b, + const Tensor& then_t, Scalar scalar_else) = 0; +}; + +} // namespace nn + +#endif // NN_COMPUTE_ENGINE_HPP diff --git a/include/neuralnet.cpp/compute_layer.hpp b/include/neuralnet.cpp/compute_layer.hpp new file mode 100644 index 00000000..7acdb275 --- /dev/null +++ b/include/neuralnet.cpp/compute_layer.hpp @@ -0,0 +1,2612 @@ +#ifndef NN_COMPUTE_LAYER_HPP +#define NN_COMPUTE_LAYER_HPP + +// ── compute_layer.hpp — 引擎化计算层 ─────────────────────────────────────── +// +// 架构铁律： +// 1. Layer 的 forward/backward 只写一次，通过 ComputeEngine 参数自动适配 +// CPU/GPU 设备。不再有 forward_gpu / backward_gpu。 +// 2. 算法只在 Layer（通过组合 engine 原语表达），绝不在 Engine/Shader 中。 +// 3. Engine/Shader 只提供 op-level 原语（matmul, add, exp, max, reduce 等）。 +// +// 算法表达示例： +// ReLU forward = max(x, 0) → engine.elementwise_binary_scalar(Max, x, 0) +// ReLU backward = (x > 0) ? grad : 0 → engine.elementwise_select_scalar_cond(Gt, x, 0, grad, 0) +// Linear fwd = W×x + b → engine.matmul(W, x) + engine.broadcast_row_add(out, b) +// Linear bwd = W^T×grad, grad_W += grad×x^T, grad_b += row_sum(grad) +// GeLU fwd = x * sigmoid(β*x) → 5 次原语组合 +// LayerNorm fwd = (x - mean) * rsqrt(var+eps) * gamma + beta +// ───────────────────────────────────────────────────────────────────────── + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "compute_engine.hpp" +#include "tensor.hpp" +#include "model_spec.hpp" + +namespace nn +{ + +// ══════════════════════════════════════════════════════════════════════════ +// Layer — 引擎化计算层基类 +// ══════════════════════════════════════════════════════════════════════════ +class Layer +{ +public: + virtual ~Layer() = default; + + // forward/backward 接收 ComputeEngine 引用，自动适配 CPU/GPU + // 只有一套实现，不再有 forward_gpu / backward_gpu + [[nodiscard]] virtual Result forward( + ComputeEngine& engine, const Tensor& input) = 0; + + [[nodiscard]] virtual Result backward( + ComputeEngine& engine, const Tensor& grad_output) = 0; + + // 参数访问（供 optimizer 使用）— 使用 reference_wrapper 替代裸指针，明确表达非拥有语义 + [[nodiscard]] virtual std::vector parameters() { return {}; } + [[nodiscard]] virtual std::vector param_gradients() { return {}; } + + // 梯度清零（每个训练 step 开始前调用） + [[nodiscard]] virtual Result zero_grad(ComputeEngine& engine) + { + for (auto& grad : param_gradients()) + { + auto r = engine.zero(grad); + if (!r) return r; + } + return {}; + } + + // batch 录制粒度控制（默认 no-op，仅 GPTModel override） + virtual void set_flush_interval(std::size_t /*interval*/) {} +}; + +// ── 辅助：深拷贝 Tensor（通过 engine.clone()，无 PCIe 传输） ────────────── +// 用于需要修改中间结果但不影响原 Tensor 的场景（如 LayerNorm 中的 diff） +[[nodiscard]] inline Result clone_tensor( + ComputeEngine& engine, const Tensor& src) +{ + return engine.clone(src); +} + +// ══════════════════════════════════════════════════════════════════════════ +// Linear — 全连接层 +// +// 算法（只在此处，不在 Engine/Shader）： +// forward: out = W × x + b +// backward: grad_x = W^T × grad_out +// grad_W += grad_out × x^T +// grad_b += Σ_batch grad_out +// ══════════════════════════════════════════════════════════════════════════ +class Linear final : public Layer +{ +private: + Tensor w_; // 权重 (out_features, in_features) + Tensor b_; // 偏置 (out_features, 1) + Tensor grad_w_; // 权重梯度 + Tensor grad_b_; // 偏置梯度 + Tensor input_cache_; // forward 输入缓存（供 backward 使用） + + inline static thread_local std::mt19937_64 rng_{std::random_device{}()}; + +public: + Linear(ComputeEngine& engine, + std::size_t in_features, std::size_t out_features) + { + // ── 在 CPU 上初始化权重（Xavier 均匀分布） ── + Matrix w_cpu(out_features, in_features); + const Scalar limit = std::sqrt(6.0 / static_cast(in_features + out_features)); + std::uniform_real_distribution dist(-limit, limit); + auto w_span = w_cpu.span(); + for (std::size_t i = 0; i < w_cpu.size(); ++i) + w_span[i] = dist(rng_); + + Matrix b_cpu(out_features, 1); // 零初始化 + + // ── 通过 engine 上传到目标设备 ── + auto w_res = engine.from_matrix(w_cpu); + NN_ASSERT(w_res, w_res ? "" : w_res.error().message.c_str()); + w_ = std::move(*w_res); + + auto b_res = engine.from_matrix(b_cpu); + NN_ASSERT(b_res, b_res ? "" : b_res.error().message.c_str()); + b_ = std::move(*b_res); + + // ── 创建梯度张量（零初始化） ── + grad_w_ = engine.create_tensor(out_features, in_features); + grad_b_ = engine.create_tensor(out_features, 1); + { auto r1 = engine.zero(grad_w_); NN_ASSERT(r1, r1 ? "" : r1.error().message.c_str()); } + { auto r2 = engine.zero(grad_b_); NN_ASSERT(r2, r2 ? "" : r2.error().message.c_str()); } + } + + [[nodiscard]] std::vector parameters() override + { + return {w_, b_}; + } + + [[nodiscard]] std::vector param_gradients() override + { + return {grad_w_, grad_b_}; + } + + // ── forward: out = W × x + b ────────────────────────────────────────── + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + if (input.rows() != w_.cols()) + return std::unexpected(Error{"linear forward: input shape mismatch"}); + + // 缓存输入供 backward 使用 + input_cache_ = input; + + // out = W × x + auto out = engine.matmul(w_, input, false, false); + if (!out) return std::unexpected(out.error()); + + // out += b（按行广播加法：out[f][b] += bias[f]） + auto r = engine.broadcast_row_inplace(*out, b_, BinaryOp::Add); + if (!r) return std::unexpected(r.error()); + + return out; + } + + // ── backward: grad_x = W^T × grad_out, 累积 grad_W / grad_b ────────── + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + if (grad_output.rows() != w_.rows()) + return std::unexpected(Error{"linear backward: grad_output shape mismatch"}); + + // grad_input = W^T × grad_output + auto grad_input = engine.matmul(w_, grad_output, true, false); + if (!grad_input) return std::unexpected(grad_input.error()); + + // grad_W += grad_output × input_cache^T + auto gw = engine.matmul(grad_output, input_cache_, false, true); + if (!gw) return std::unexpected(gw.error()); + auto r1 = engine.add_inplace(grad_w_, *gw); + if (!r1) return std::unexpected(r1.error()); + + // grad_b += Σ_batch grad_output（按行归约求和） + auto gb = engine.row_reduce_sum(grad_output); + if (!gb) return std::unexpected(gb.error()); + auto r2 = engine.add_inplace(grad_b_, *gb); + if (!r2) return std::unexpected(r2.error()); + + return grad_input; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// ReLU — ReLU 激活函数 +// +// 算法（只在此处，不在 Engine/Shader）： +// forward: out = max(x, 0) +// backward: grad_x = (x > 0) ? grad_out : 0 +// ══════════════════════════════════════════════════════════════════════════ +class ReLU final : public Layer +{ +private: + Tensor input_cache_; + +public: + ReLU() = default; + + // ── forward: out = max(x, 0) ────────────────────────────────────────── + // ReLU 算法由 Layer 表达为 Max 原语 + 标量 0 + // Engine/Shader 只提供 Max 原语，不知道 "ReLU" 是什么 + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + input_cache_ = input; + return engine.elementwise_binary_scalar(BinaryOp::Max, input, Scalar{0}); + } + + // ── backward: grad_x = (x > 0) ? grad_out : 0 ──────────────────────── + // ReLU 反向算法由 Layer 表达为 Select + Gt 原语 + // Engine/Shader 只提供 Select/Gt 原语，不知道 "ReLU backward" 是什么 + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + if (input_cache_.rows() != grad_output.rows() || + input_cache_.cols() != grad_output.cols()) + return std::unexpected(Error{"relu backward: shape mismatch"}); + + return engine.elementwise_select_scalar_cond( + CompareOp::Gt, // 条件：x > 0 + input_cache_, // 条件操作数 A = x + Scalar{0}, // 条件操作数 B = 0 (标量) + grad_output, // then = grad_output (张量) + Scalar{0}); // else = 0 (标量) + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// GeLU — QuickGeLU 激活函数（复杂算法的原语组合） +// +// 算法（只在此处，不在 Engine/Shader）： +// forward: out = x * sigmoid(β * x) +// = x * (1 / (1 + exp(-β * x))) +// backward: factor = s * (1 + βx * (1 - s)), s = sigmoid(βx) +// grad_x = grad_out * factor +// +// 原语分解： +// forward: 5 次原语调用（Mul→Neg→Exp→Add→Div→Mul） +// backward: 5 次原语调用（Sub→Mul→Mul→Add→Mul→Mul） +// shader 只有 Mul/Neg/Exp/Add/Div 原语，不知道 "GeLU" 是什么 +// ══════════════════════════════════════════════════════════════════════════ +class GeLU final : public Layer +{ +private: + static constexpr Scalar BETA = 1.702f; + Tensor input_cache_; + Tensor sigmoid_cache_; // 缓存 sigmoid(βx) 供 backward 使用 + +public: + GeLU() = default; + + // ── forward: out = x * sigmoid(β * x) ──────────────────────────────── + // 分解为原语： + // t1 = x * β (binary_scalar Mul) + // t2 = -t1 (unary Neg) → -βx + // t3 = exp(t2) (unary Exp) → exp(-βx) + // t4 = t3 + 1 (binary_scalar Add) + // s = 1 / t4 (binary_scalar Div, scalar_first) → sigmoid(βx) + // out = x * s (binary Mul) + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + input_cache_ = input; + + // t1 = x * β + auto t1 = engine.elementwise_binary_scalar(BinaryOp::Mul, input, BETA); + if (!t1) return std::unexpected(t1.error()); + + // t2 = -t1 (= -βx) + auto t2 = engine.elementwise_unary(UnaryOp::Neg, *t1); + if (!t2) return std::unexpected(t2.error()); + + // t3 = exp(t2) + auto t3 = engine.elementwise_unary(UnaryOp::Exp, *t2); + if (!t3) return std::unexpected(t3.error()); + + // t4 = t3 + 1 + auto t4 = engine.elementwise_binary_scalar(BinaryOp::Add, *t3, Scalar{1}); + if (!t4) return std::unexpected(t4.error()); + + // s = 1 / t4 (scalar_first: 1.0 / t4) + auto s = engine.elementwise_binary_scalar(BinaryOp::Div, *t4, Scalar{1}, true); + if (!s) return std::unexpected(s.error()); + + sigmoid_cache_ = *s; // 缓存供 backward + + // out = x * s + return engine.elementwise_binary(BinaryOp::Mul, input, *s); + } + + // ── backward: grad_x = grad_out * factor ───────────────────────────── + // factor = s * (1 + βx * (1 - s)), s = sigmoid(βx) + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + if (input_cache_.rows() != grad_output.rows() || + input_cache_.cols() != grad_output.cols()) + return std::unexpected(Error{"gelu backward: shape mismatch"}); + + // s = sigmoid_cache_ + // one_minus_s = 1 - s + auto one_minus_s = engine.elementwise_binary_scalar( + BinaryOp::Sub, sigmoid_cache_, Scalar{1}, true); + if (!one_minus_s) return std::unexpected(one_minus_s.error()); + + // βx = input * β + auto bx = engine.elementwise_binary_scalar(BinaryOp::Mul, input_cache_, BETA); + if (!bx) return std::unexpected(bx.error()); + + // inner = βx * (1 - s) + auto inner = engine.elementwise_binary(BinaryOp::Mul, *bx, *one_minus_s); + if (!inner) return std::unexpected(inner.error()); + + // paren = 1 + inner + auto paren = engine.elementwise_binary_scalar(BinaryOp::Add, *inner, Scalar{1}); + if (!paren) return std::unexpected(paren.error()); + + // factor = s * paren + auto factor = engine.elementwise_binary(BinaryOp::Mul, sigmoid_cache_, *paren); + if (!factor) return std::unexpected(factor.error()); + + // grad_input = grad_out * factor + return engine.elementwise_binary(BinaryOp::Mul, grad_output, *factor); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// SwiGLU — Swish-Gated Linear Unit（LLaMA/Mistral 风格 FFN 激活） +// +// 算法（只在此处，不在 Engine/Shader）： +// forward: 输入 (2*d_ff, batch)，前 d_ff 行为 gate，后 d_ff 行为 up +// gate = slice_rows(x, 0, d_ff) +// up = slice_rows(x, d_ff, d_ff) +// sw = SiLU(gate) = gate * sigmoid(gate) +// out = sw ⊙ up → (d_ff, batch) +// backward: grad_up = grad_out ⊙ sw +// grad_sw = grad_out ⊙ up +// grad_gate = grad_sw * (σ(g) + g*σ(g)*(1-σ(g))) +// grad_input 把 grad_gate 写回行 [0, d_ff)、grad_up 写回 [d_ff, 2*d_ff) +// +// 原语分解（Engine/Shader 只知道标量原语）： +// SiLU forward: Neg → Exp → Add(1) → Div(1/x) → Mul(x*s) +// SiLU backward: Sub → Mul → Mul → Add → Mul +// split/merge 用 slice_rows / insert_rows +// ══════════════════════════════════════════════════════════════════════════ +class SwiGLU final : public Layer +{ +private: + std::size_t d_ff_ = 0; + Tensor gate_cache_; // (d_ff, batch) + Tensor sigmoid_cache_; // σ(gate) (d_ff, batch) + Tensor up_cache_; // (d_ff, batch) + +public: + SwiGLU() = default; + explicit SwiGLU(std::size_t d_ff) : d_ff_(d_ff) {} + + // ── forward: out = SiLU(gate) ⊙ up ──────────────────────────────────── + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + // gate = 前 d_ff 行，up = 后 d_ff 行 + auto gate = engine.slice_rows(input, 0, d_ff_); + if (!gate) return std::unexpected(gate.error()); + auto up = engine.slice_rows(input, d_ff_, d_ff_); + if (!up) return std::unexpected(up.error()); + + // s = sigmoid(gate) = 1 / (1 + exp(-gate)) + auto t1 = engine.elementwise_unary(UnaryOp::Neg, *gate); + if (!t1) return std::unexpected(t1.error()); + auto t2 = engine.elementwise_unary(UnaryOp::Exp, *t1); + if (!t2) return std::unexpected(t2.error()); + auto t3 = engine.elementwise_binary_scalar(BinaryOp::Add, *t2, Scalar{1}); + if (!t3) return std::unexpected(t3.error()); + auto s = engine.elementwise_binary_scalar(BinaryOp::Div, *t3, Scalar{1}, true); + if (!s) return std::unexpected(s.error()); + + gate_cache_ = *gate; + sigmoid_cache_ = *s; + up_cache_ = *up; + + // sw = gate * s，out = sw ⊙ up + auto sw = engine.elementwise_binary(BinaryOp::Mul, *gate, *s); + if (!sw) return std::unexpected(sw.error()); + return engine.elementwise_binary(BinaryOp::Mul, *sw, *up); + } + + // ── backward: 分解出 grad_gate / grad_up 并合并回 (2*d_ff, batch) ────── + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + // grad_up = grad_out ⊙ sw，其中 sw = gate*s + auto sw = engine.elementwise_binary(BinaryOp::Mul, gate_cache_, sigmoid_cache_); + if (!sw) return std::unexpected(sw.error()); + auto grad_up = engine.elementwise_binary(BinaryOp::Mul, grad_output, *sw); + if (!grad_up) return std::unexpected(grad_up.error()); + + // grad_sw = grad_out ⊙ up + auto grad_sw = engine.elementwise_binary(BinaryOp::Mul, grad_output, up_cache_); + if (!grad_sw) return std::unexpected(grad_sw.error()); + + // grad_gate = grad_sw * (σ(g) + g*σ(g)*(1-σ(g))) + // one_minus_s = 1 - s + auto one_minus_s = engine.elementwise_binary_scalar( + BinaryOp::Sub, sigmoid_cache_, Scalar{1}, true); + if (!one_minus_s) return std::unexpected(one_minus_s.error()); + // g_oms = gate * (1-s) + auto g_oms = engine.elementwise_binary(BinaryOp::Mul, gate_cache_, *one_minus_s); + if (!g_oms) return std::unexpected(g_oms.error()); + // s_goms = s * g_oms + auto s_goms = engine.elementwise_binary(BinaryOp::Mul, sigmoid_cache_, *g_oms); + if (!s_goms) return std::unexpected(s_goms.error()); + // factor = s + s_goms + auto factor = engine.elementwise_binary(BinaryOp::Add, sigmoid_cache_, *s_goms); + if (!factor) return std::unexpected(factor.error()); + // grad_gate = grad_sw * factor + auto grad_gate = engine.elementwise_binary(BinaryOp::Mul, *grad_sw, *factor); + if (!grad_gate) return std::unexpected(grad_gate.error()); + + // 合并：grad_input = (grad_gate; grad_up) → (2*d_ff, batch) + auto grad_input = engine.create_tensor(2 * d_ff_, grad_output.cols()); + auto r1 = engine.zero(grad_input); + if (!r1) return std::unexpected(r1.error()); + auto r2 = engine.insert_rows(grad_input, 0, *grad_gate); + if (!r2) return std::unexpected(r2.error()); + auto r3 = engine.insert_rows(grad_input, d_ff_, *grad_up); + if (!r3) return std::unexpected(r3.error()); + + return grad_input; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// LayerNorm — 层归一化 +// +// 算法（只在此处，不在 Engine/Shader）： +// forward: mean = (1/F) * Σ_f x[f][b] (col_reduce_sum + scale) +// diff = x - mean (broadcast_col Sub) +// var = (1/F) * Σ_f diff² (elementwise Mul + col_reduce_sum + scale) +// std_inv = 1 / sqrt(var + eps) (binary_scalar Add + unary Rsqrt) +// normalized = diff * std_inv (broadcast_col Mul) +// out = gamma * normalized + beta (broadcast_row Mul + broadcast_row Add) +// +// backward: gy = grad_out * gamma (broadcast_row Mul) +// mean_g = (1/F) * Σ_f gy (col_reduce_sum + scale) +// gy_norm = gy * normalized (elementwise Mul) +// mean_gn = (1/F) * Σ_f gy_norm (col_reduce_sum + scale) +// grad_x = (gy - mean_g - normalized*mean_gn) * std_inv +// grad_gamma += Σ_b gy_norm (row_reduce_sum) +// grad_beta += Σ_b grad_out (row_reduce_sum) +// ══════════════════════════════════════════════════════════════════════════ +class LayerNorm final : public Layer +{ +private: + std::size_t normalized_shape_; + Scalar epsilon_; + + // 可学习参数 + Tensor gamma_; // (normalized_shape, 1) + Tensor beta_; // (normalized_shape, 1) + Tensor grad_gamma_; + Tensor grad_beta_; + + // backward 缓存 + Tensor normalized_cache_; // (features, batch) + Tensor std_cache_; // (1, batch) — std_inv + + static constexpr Scalar EPSILON = 1e-5; + +public: + explicit LayerNorm(ComputeEngine& engine, + std::size_t normalized_shape, Scalar epsilon = EPSILON) + : normalized_shape_(normalized_shape), epsilon_(epsilon) + { + // gamma 初始化为 1, beta 初始化为 0 + Matrix gamma_cpu(normalized_shape, 1, Scalar{1}); + Matrix beta_cpu(normalized_shape, 1, Scalar{0}); + + auto g = engine.from_matrix(gamma_cpu); + NN_ASSERT(g, g ? "" : g.error().message.c_str()); + gamma_ = std::move(*g); + + auto bv = engine.from_matrix(beta_cpu); + NN_ASSERT(bv, bv ? "" : bv.error().message.c_str()); + beta_ = std::move(*bv); + + grad_gamma_ = engine.create_tensor(normalized_shape, 1); + grad_beta_ = engine.create_tensor(normalized_shape, 1); + { auto r1 = engine.zero(grad_gamma_); NN_ASSERT(r1, r1 ? "" : r1.error().message.c_str()); } + { auto r2 = engine.zero(grad_beta_); NN_ASSERT(r2, r2 ? "" : r2.error().message.c_str()); } + } + + [[nodiscard]] std::vector parameters() override + { + return {gamma_, beta_}; + } + + [[nodiscard]] std::vector param_gradients() override + { + return {grad_gamma_, grad_beta_}; + } + + // ── forward ─────────────────────────────────────────────────────────── + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + if (input.rows() != normalized_shape_) + return std::unexpected(Error{"layernorm forward: input shape mismatch"}); + + const Scalar inv_features = Scalar{1} / static_cast(normalized_shape_); + + // mean = col_reduce_sum(input) * inv_features → (1, batch) + auto mean = engine.col_reduce_sum(input); + if (!mean) return std::unexpected(mean.error()); + auto r = engine.scale_inplace(*mean, inv_features); + if (!r) return std::unexpected(r.error()); + + // diff = input - mean (broadcast_col Sub) — 需要深拷贝 input + auto diff = clone_tensor(engine, input); + if (!diff) return std::unexpected(diff.error()); + r = engine.broadcast_col_inplace(*diff, *mean, BinaryOp::Sub); + if (!r) return std::unexpected(r.error()); + + // diff_sq = diff * diff + auto diff_sq = engine.elementwise_binary(BinaryOp::Mul, *diff, *diff); + if (!diff_sq) return std::unexpected(diff_sq.error()); + + // var = col_reduce_sum(diff_sq) * inv_features → (1, batch) + auto var = engine.col_reduce_sum(*diff_sq); + if (!var) return std::unexpected(var.error()); + r = engine.scale_inplace(*var, inv_features); + if (!r) return std::unexpected(r.error()); + + // var_eps = var + eps + auto var_eps = engine.elementwise_binary_scalar(BinaryOp::Add, *var, epsilon_); + if (!var_eps) return std::unexpected(var_eps.error()); + + // std_inv = rsqrt(var + eps) → (1, batch) + auto std_inv = engine.elementwise_unary(UnaryOp::Rsqrt, *var_eps); + if (!std_inv) return std::unexpected(std_inv.error()); + + // normalized = diff * std_inv (broadcast_col Mul) — 就地修改 diff + r = engine.broadcast_col_inplace(*diff, *std_inv, BinaryOp::Mul); + if (!r) return std::unexpected(r.error()); + + // 缓存 normalized 和 std_inv 供 backward 使用 + normalized_cache_ = *diff; + std_cache_ = *std_inv; + + // output = gamma * normalized + beta (broadcast_row) + // 先深拷贝 normalized，再就地广播 + auto output = clone_tensor(engine, *diff); + if (!output) return std::unexpected(output.error()); + + r = engine.broadcast_row_inplace(*output, gamma_, BinaryOp::Mul); + if (!r) return std::unexpected(r.error()); + + r = engine.broadcast_row_inplace(*output, beta_, BinaryOp::Add); + if (!r) return std::unexpected(r.error()); + + return output; + } + + // ── backward ────────────────────────────────────────────────────────── + // grad_x = (gy - mean_g - normalized*mean_gn) * std_inv + // gy = grad_out * gamma + // mean_g = mean(gy), mean_gn = mean(gy * normalized) + // grad_gamma += row_sum(gy * normalized) + // grad_beta += row_sum(grad_out) + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + const Scalar inv_features = Scalar{1} / static_cast(normalized_shape_); + + // gy = grad_out * gamma (broadcast_row Mul) — 深拷贝 grad_output + auto gy = clone_tensor(engine, grad_output); + if (!gy) return std::unexpected(gy.error()); + auto r = engine.broadcast_row_inplace(*gy, gamma_, BinaryOp::Mul); + if (!r) return std::unexpected(r.error()); + + // mean_g = col_reduce_sum(gy) * inv_features → (1, batch) + auto mean_g = engine.col_reduce_sum(*gy); + if (!mean_g) return std::unexpected(mean_g.error()); + r = engine.scale_inplace(*mean_g, inv_features); + if (!r) return std::unexpected(r.error()); + + // gy_norm = gy * normalized_cache_ (elementwise Mul) + auto gy_norm = engine.elementwise_binary(BinaryOp::Mul, *gy, normalized_cache_); + if (!gy_norm) return std::unexpected(gy_norm.error()); + + // mean_gn = col_reduce_sum(gy_norm) * inv_features → (1, batch) + auto mean_gn = engine.col_reduce_sum(*gy_norm); + if (!mean_gn) return std::unexpected(mean_gn.error()); + r = engine.scale_inplace(*mean_gn, inv_features); + if (!r) return std::unexpected(r.error()); + + // t1 = gy - mean_g (broadcast_col Sub) — 就地修改 gy + r = engine.broadcast_col_inplace(*gy, *mean_g, BinaryOp::Sub); + if (!r) return std::unexpected(r.error()); + + // t2 = normalized * mean_gn (broadcast_col Mul) — 深拷贝 normalized + auto t2 = clone_tensor(engine, normalized_cache_); + if (!t2) return std::unexpected(t2.error()); + r = engine.broadcast_col_inplace(*t2, *mean_gn, BinaryOp::Mul); + if (!r) return std::unexpected(r.error()); + + // grad_input = (t1 - t2) * std_inv + // 先 t1 -= t2 (elementwise Sub) + auto t2_neg = engine.elementwise_binary(BinaryOp::Sub, *gy, *t2); + if (!t2_neg) return std::unexpected(t2_neg.error()); + + // 上面创建新 Tensor 作为 (gy - t2)，再乘 std_inv (broadcast_col Mul) + r = engine.broadcast_col_inplace(*t2_neg, std_cache_, BinaryOp::Mul); + if (!r) return std::unexpected(r.error()); + + // grad_gamma += row_reduce_sum(gy_norm) → (features, 1) + auto gg = engine.row_reduce_sum(*gy_norm); + if (!gg) return std::unexpected(gg.error()); + r = engine.add_inplace(grad_gamma_, *gg); + if (!r) return std::unexpected(r.error()); + + // grad_beta += row_reduce_sum(grad_output) → (features, 1) + auto gb = engine.row_reduce_sum(grad_output); + if (!gb) return std::unexpected(gb.error()); + r = engine.add_inplace(grad_beta_, *gb); + if (!r) return std::unexpected(r.error()); + + return t2_neg; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// RMSNorm — Root Mean Square 归一化（LLaMA/Mistral 风格） +// +// 与 LayerNorm 的差异：不减去均值、无 beta 偏置，只按均方根归一化。 +// 每层少 2 次列归约 + 1 次广播，计算量更小，训练更稳。 +// +// 算法（只在此处，不在 Engine/Shader）： +// forward: mean_sq = (1/F) * Σ_f x² (elementwise Mul + col_reduce_sum + scale) +// rms_inv = 1 / sqrt(mean_sq + eps) (binary_scalar Add + unary Rsqrt) +// normed = x * rms_inv (broadcast_col Mul) +// out = normed * gamma (broadcast_row Mul) +// backward: gy = grad_out * gamma (broadcast_row Mul) +// gy_norm = gy ⊙ normed (elementwise Mul) +// m = (1/F) * Σ_f gy_norm (col_reduce_sum + scale) +// grad_x = (gy - m*normed) * rms_inv (broadcast_col Mul + Sub + Mul) +// grad_gamma += row_reduce_sum(gy_norm) (row_reduce_sum) +// ══════════════════════════════════════════════════════════════════════════ +class RMSNorm final : public Layer +{ +private: + std::size_t normalized_shape_; + Scalar epsilon_; + + Tensor gamma_; // (normalized_shape, 1) + Tensor grad_gamma_; + + // backward 缓存 + Tensor normed_cache_; // (features, batch) + Tensor rms_inv_cache_; // (1, batch) + + static constexpr Scalar EPSILON = 1e-5; + +public: + explicit RMSNorm(ComputeEngine& engine, + std::size_t normalized_shape, Scalar epsilon = EPSILON) + : normalized_shape_(normalized_shape), epsilon_(epsilon) + { + // gamma 初始化为 1（无 beta） + Matrix gamma_cpu(normalized_shape, 1, Scalar{1}); + auto g = engine.from_matrix(gamma_cpu); + NN_ASSERT(g, g ? "" : g.error().message.c_str()); + gamma_ = std::move(*g); + + grad_gamma_ = engine.create_tensor(normalized_shape, 1); + { auto r1 = engine.zero(grad_gamma_); NN_ASSERT(r1, r1 ? "" : r1.error().message.c_str()); } + } + + [[nodiscard]] std::vector parameters() override + { + return {gamma_}; + } + + [[nodiscard]] std::vector param_gradients() override + { + return {grad_gamma_}; + } + + // ── forward ─────────────────────────────────────────────────────────── + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + if (input.rows() != normalized_shape_) + return std::unexpected(Error{"rmsnorm forward: input shape mismatch"}); + + const Scalar inv_features = Scalar{1} / static_cast(normalized_shape_); + + // x_sq = x * x + auto x_sq = engine.elementwise_binary(BinaryOp::Mul, input, input); + if (!x_sq) return std::unexpected(x_sq.error()); + + // mean_sq = col_reduce_sum(x_sq) * inv_features → (1, batch) + auto mean_sq = engine.col_reduce_sum(*x_sq); + if (!mean_sq) return std::unexpected(mean_sq.error()); + auto r = engine.scale_inplace(*mean_sq, inv_features); + if (!r) return std::unexpected(r.error()); + + // rms_inv = rsqrt(mean_sq + eps) → (1, batch) + auto var_eps = engine.elementwise_binary_scalar(BinaryOp::Add, *mean_sq, epsilon_); + if (!var_eps) return std::unexpected(var_eps.error()); + auto rms_inv = engine.elementwise_unary(UnaryOp::Rsqrt, *var_eps); + if (!rms_inv) return std::unexpected(rms_inv.error()); + + // normed = x * rms_inv (broadcast_col Mul) — 就地修改 x 副本 + auto normed = clone_tensor(engine, input); + if (!normed) return std::unexpected(normed.error()); + r = engine.broadcast_col_inplace(*normed, *rms_inv, BinaryOp::Mul); + if (!r) return std::unexpected(r.error()); + + // 缓存 normed 和 rms_inv 供 backward 使用 + normed_cache_ = *normed; + rms_inv_cache_ = *rms_inv; + + // output = normed * gamma (broadcast_row Mul) + auto output = clone_tensor(engine, *normed); + if (!output) return std::unexpected(output.error()); + r = engine.broadcast_row_inplace(*output, gamma_, BinaryOp::Mul); + if (!r) return std::unexpected(r.error()); + + return output; + } + + // ── backward ────────────────────────────────────────────────────────── + // grad_x = (gy - m*normed) * rms_inv + // gy = grad_out * gamma + // m = mean(gy ⊙ normed) + // grad_gamma += row_sum(gy ⊙ normed) + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + const Scalar inv_features = Scalar{1} / static_cast(normalized_shape_); + + // gy = grad_out * gamma (broadcast_row Mul) + auto gy = clone_tensor(engine, grad_output); + if (!gy) return std::unexpected(gy.error()); + auto r = engine.broadcast_row_inplace(*gy, gamma_, BinaryOp::Mul); + if (!r) return std::unexpected(r.error()); + + // gy_norm = gy ⊙ normed + auto gy_norm = engine.elementwise_binary(BinaryOp::Mul, *gy, normed_cache_); + if (!gy_norm) return std::unexpected(gy_norm.error()); + + // m = col_reduce_sum(gy_norm) * inv_features → (1, batch) + auto m = engine.col_reduce_sum(*gy_norm); + if (!m) return std::unexpected(m.error()); + r = engine.scale_inplace(*m, inv_features); + if (!r) return std::unexpected(r.error()); + + // m_normed = normed * m (broadcast_col Mul) + auto m_normed = clone_tensor(engine, normed_cache_); + if (!m_normed) return std::unexpected(m_normed.error()); + r = engine.broadcast_col_inplace(*m_normed, *m, BinaryOp::Mul); + if (!r) return std::unexpected(r.error()); + + // t = gy - m_normed + auto t = engine.elementwise_binary(BinaryOp::Sub, *gy, *m_normed); + if (!t) return std::unexpected(t.error()); + + // grad_x = t * rms_inv (broadcast_col Mul) — 就地修改 t + r = engine.broadcast_col_inplace(*t, rms_inv_cache_, BinaryOp::Mul); + if (!r) return std::unexpected(r.error()); + + // grad_gamma += row_reduce_sum(gy_norm) → (features, 1) + auto gg = engine.row_reduce_sum(*gy_norm); + if (!gg) return std::unexpected(gg.error()); + r = engine.add_inplace(grad_gamma_, *gg); + if (!r) return std::unexpected(r.error()); + + return t; + } +}; + +// ── 归一化层工厂：按 NormType 创建 LayerNorm 或 RMSNorm ────────────────── +[[nodiscard]] inline std::unique_ptr make_norm_layer( + ComputeEngine& engine, std::size_t d_model, NormType norm_type) +{ + if (norm_type == NormType::RMSNorm) + return std::make_unique(engine, d_model); + return std::make_unique(engine, d_model); +} + +// ══════════════════════════════════════════════════════════════════════════ +// Softmax — 按行 softmax（用于注意力权重） +// +// 算法（只在此处，不在 Engine/Shader）： +// forward: row_max[r] = max_c x[r][c] (row_reduce_max) +// shifted[r][c] = x[r][c] - row_max[r] (broadcast_row Sub) +// exp_shift[r][c] = exp(shifted[r][c]) (unary Exp) +// row_sum[r] = Σ_c exp_shift[r][c] (row_reduce_sum) +// out[r][c] = exp_shift[r][c] / row_sum[r] (broadcast_row Div) +// backward: grad_x = out ⊙ (grad_out - row_dot(out ⊙ grad_out)) +// row_dot[r] = Σ_c out[r][c] * grad_out[r][c] +// ══════════════════════════════════════════════════════════════════════════ +class Softmax final : public Layer +{ +private: + Tensor output_cache_; + +public: + Softmax() = default; + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + // 1. row_max = max per row + auto row_max = engine.row_reduce_max(input); + if (!row_max) return std::unexpected(row_max.error()); + + // 2. shifted = input - row_max (broadcast_row Sub) — 深拷贝 input + auto shifted = clone_tensor(engine, input); + if (!shifted) return std::unexpected(shifted.error()); + auto r = engine.broadcast_row_inplace(*shifted, *row_max, BinaryOp::Sub); + if (!r) return std::unexpected(r.error()); + + // 3. exp_shift = exp(shifted) + auto exp_shift = engine.elementwise_unary(UnaryOp::Exp, *shifted); + if (!exp_shift) return std::unexpected(exp_shift.error()); + + // 4. row_sum = Σ_c exp_shift[r][c] + auto row_sum = engine.row_reduce_sum(*exp_shift); + if (!row_sum) return std::unexpected(row_sum.error()); + + // 5. output = exp_shift / row_sum (broadcast_row Div) — 深拷贝 exp_shift + auto output = clone_tensor(engine, *exp_shift); + if (!output) return std::unexpected(output.error()); + r = engine.broadcast_row_inplace(*output, *row_sum, BinaryOp::Div); + if (!r) return std::unexpected(r.error()); + + output_cache_ = *output; + return output; + } + + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + // ep = out ⊙ grad_output + auto ep = engine.elementwise_binary(BinaryOp::Mul, output_cache_, grad_output); + if (!ep) return std::unexpected(ep.error()); + + // dot[r] = Σ_c ep[r][c] + auto dot = engine.row_reduce_sum(*ep); + if (!dot) return std::unexpected(dot.error()); + + // gmd = grad_output - dot (broadcast_row Sub) — 深拷贝 grad_output + auto gmd = clone_tensor(engine, grad_output); + if (!gmd) return std::unexpected(gmd.error()); + auto r = engine.broadcast_row_inplace(*gmd, *dot, BinaryOp::Sub); + if (!r) return std::unexpected(r.error()); + + // grad_input = out ⊙ gmd + return engine.elementwise_binary(BinaryOp::Mul, output_cache_, *gmd); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// AttentionBase — 多头注意力基类（批量化：消除 per-head 和 per-sample 循环） +// +// 提取 MultiHeadAttention 与 CausalSelfAttention 的公共逻辑： +// - 完全相同的成员变量、参数/梯度接口 +// - forward/backward 仅在「scale 之后、softmax 之前」是否施加掩码上有差异 +// +// 算法（只在此处，不在 Engine/Shader）： +// Q = W_q × x, K = W_k × x, V = W_v × x (三个 Linear 投影) +// Q/K/V: (H*d_k, batch*seq) — 头维度在行方向，batch 在列方向 +// +// 批量化关键：用 rearrange_3d 把 (H*d_k, batch*seq) 重排为 (batch*H*d_k, seq)， +// 使 batched_matmul 能按 batch*H 切分行块，单次 dispatch 处理所有样本和所有头。 +// +// Q_re = rearrange_3d(Q, H*d_k, batch, seq) → (batch*H*d_k, seq) +// S = batched_matmul(Q_re, K_re, batch*H, transA=true) → (batch*H*seq, seq) +// S *= scale +// S = apply_mask_(S) ← 子类钩子（默认 no-op = MHA 行为） +// A = softmax(S) — 行级归一化，堆叠布局下天然正确 +// O_re = batched_matmul(V_re, A, batch*H) → (batch*H*d_k, seq) +// O = rearrange_3d(O_re, H*d_k, batch, seq, inverse=true) → (H*d_k, batch*seq) +// out = W_o × O +// +// 输入形状: (d_model, batch * seq_len)，输出形状: (d_model, batch * seq_len) +// seq_len 由构造函数指定，batch = input.cols() / seq_len 在 forward 时推断 +// ══════════════════════════════════════════════════════════════════════════ +class AttentionBase : public Layer +{ +protected: + std::size_t d_model_; + std::size_t num_heads_; + std::size_t d_k_; + std::size_t seq_len_; // 单样本序列长度（0 = 单样本，cols 即 seq） + Scalar scale_; + + Linear w_q_; + Linear w_k_; + Linear w_v_; + Linear w_o_; + Softmax softmax_; + + // forward 缓存（rearranged 版本，供 backward 直接使用） + Tensor Q_cache_, K_cache_, V_cache_; // (batch*H*d_k, seq) rearranged + Tensor attn_cache_; // (batch*H*seq, seq) + + // 掩码钩子：子类重写以施加掩码，默认 no-op（MHA 行为） + // 在 forward 中 scale 之后、softmax 之前调用 + [[nodiscard]] virtual Result apply_mask_( + ComputeEngine& engine, Tensor&& scores, + std::size_t batch, std::size_t seq) + { + (void)engine; (void)batch; (void)seq; + return std::move(scores); + } + + // 增量推理掩码钩子：在 forward_step 中 scale 之后、softmax 之前调用。 + // 默认 no-op：MHA 无掩码；CSA 因果掩码在增量推理中天然满足 + // （新 token 只能看到 cache 中的前文，无未来位置可掩）。 + // ALiBi 子类重写以施加线性距离偏置 -slope*(cur_len - j)。 + [[nodiscard]] virtual Result apply_mask_step_( + ComputeEngine& engine, Tensor&& scores, + std::size_t cur_len) + { + (void)engine; (void)cur_len; + return std::move(scores); + } + + // 在 backward 中 softmax 反向之后调用（掩码为常数，梯度直接穿透，默认 no-op） + virtual void mask_backward_( + ComputeEngine& /*engine*/, Tensor& /*grad_S*/, + std::size_t /*batch*/, std::size_t /*seq*/) {} + +public: + AttentionBase(ComputeEngine& engine, + std::size_t d_model, std::size_t num_heads, + std::size_t seq_len = 0) + : d_model_(d_model), num_heads_(num_heads), + d_k_(d_model / num_heads), + seq_len_(seq_len), + scale_(Scalar{1} / std::sqrt(static_cast(d_model / num_heads))), + w_q_(engine, d_model, d_model), + w_k_(engine, d_model, d_model), + w_v_(engine, d_model, d_model), + w_o_(engine, d_model, d_model) + { + NN_ASSERT(d_model % num_heads == 0, + "AttentionBase: d_model must be divisible by num_heads"); + } + + std::vector parameters() override + { + auto p = w_q_.parameters(); + auto k = w_k_.parameters(); + auto v = w_v_.parameters(); + auto o = w_o_.parameters(); + p.insert(p.end(), k.begin(), k.end()); + p.insert(p.end(), v.begin(), v.end()); + p.insert(p.end(), o.begin(), o.end()); + return p; + } + + std::vector param_gradients() override + { + auto g = w_q_.param_gradients(); + auto k = w_k_.param_gradients(); + auto v = w_v_.param_gradients(); + auto o = w_o_.param_gradients(); + g.insert(g.end(), k.begin(), k.end()); + g.insert(g.end(), v.begin(), v.end()); + g.insert(g.end(), o.begin(), o.end()); + return g; + } + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + if (input.rows() != d_model_) + return std::unexpected(Error{"AttentionBase forward: input shape mismatch"}); + + const std::size_t total_seq = input.cols(); + const std::size_t seq = (seq_len_ > 0) ? seq_len_ : total_seq; + const std::size_t batch = (seq_len_ > 0) ? (total_seq / seq_len_) : 1; + if (total_seq != batch * seq) + return std::unexpected(Error{"AttentionBase forward: cols not divisible by seq_len"}); + + // 1. 线性投影 → Q/K/V: (H*d_k, batch*seq) + auto q_res = w_q_.forward(engine, input); + if (!q_res) return q_res; + auto k_res = w_k_.forward(engine, input); + if (!k_res) return k_res; + auto v_res = w_v_.forward(engine, input); + if (!v_res) return v_res; + + // 2. rearrange: (H*d_k, batch*seq) → (batch*H*d_k, seq) + // 使 batched_matmul 能按 batch*H 切分行块 + const std::size_t H_dk = num_heads_ * d_k_; + if (batch > 1) + { + auto qr = engine.rearrange_3d(*q_res, H_dk, batch, seq, false); + if (!qr) return std::unexpected(qr.error()); + Q_cache_ = std::move(*qr); + auto kr = engine.rearrange_3d(*k_res, H_dk, batch, seq, false); + if (!kr) return std::unexpected(kr.error()); + K_cache_ = std::move(*kr); + auto vr = engine.rearrange_3d(*v_res, H_dk, batch, seq, false); + if (!vr) return std::unexpected(vr.error()); + V_cache_ = std::move(*vr); + } + else + { + // batch=1: rearrange 是恒等拷贝，跳过 + Q_cache_ = std::move(*q_res); + K_cache_ = std::move(*k_res); + V_cache_ = std::move(*v_res); + } + + // 3. S = batched_matmul(Q_re, K_re, batch*H, transA=true) → (batch*H*seq, seq) + const std::size_t BH = batch * num_heads_; + auto scores = engine.batched_matmul( + Q_cache_, K_cache_, BH, true, false); + if (!scores) return std::unexpected(scores.error()); + + // 4. S *= scale + auto r = engine.scale_inplace(*scores, scale_); + if (!r) return std::unexpected(r.error()); + + // 5. 施加掩码（钩子：MHA 默认 no-op，CSA 施加因果/ALiBi 掩码） + auto masked = apply_mask_(engine, std::move(*scores), batch, seq); + if (!masked) return std::unexpected(masked.error()); + + // 6. A = softmax(S_masked) + auto attn = softmax_.forward(engine, *masked); + if (!attn) return std::unexpected(attn.error()); + attn_cache_ = *attn; + + // 7. O_re = batched_matmul(V_re, A, batch*H, transB=true) → (batch*H*d_k, seq) + // 标准 attention: O[:,i] = sum_j V[:,j] * A[i,j] = (V × A^T)[:,i] + // A[i,j] = P(key j | query i),A^T[j,i] = A[i,j] + auto concat_out = engine.batched_matmul( + V_cache_, attn_cache_, BH, false, true); + if (!concat_out) return std::unexpected(concat_out.error()); + + // 8. rearrange back: (batch*H*d_k, seq) → (H*d_k, batch*seq) + Tensor concat; + if (batch > 1) + { + auto cb = engine.rearrange_3d(*concat_out, H_dk, batch, seq, true); + if (!cb) return std::unexpected(cb.error()); + concat = std::move(*cb); + } + else + { + concat = std::move(*concat_out); + } + + // 9. 输出投影 + return w_o_.forward(engine, concat); + } + + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + // 推断 batch/seq（与 forward 一致） + const std::size_t total_seq = grad_output.cols(); + const std::size_t seq = (seq_len_ > 0) ? seq_len_ : total_seq; + const std::size_t batch = (seq_len_ > 0) ? (total_seq / seq_len_) : 1; + const std::size_t H_dk = num_heads_ * d_k_; + const std::size_t BH = batch * num_heads_; + + // 1. 输出投影反向 → grad_concat: (H*d_k, batch*seq) + auto gc = w_o_.backward(engine, grad_output); + if (!gc) return gc; + + // 2. rearrange grad_concat → (batch*H*d_k, seq) + Tensor grad_concat_re; + if (batch > 1) + { + auto gcr = engine.rearrange_3d(*gc, H_dk, batch, seq, false); + if (!gcr) return std::unexpected(gcr.error()); + grad_concat_re = std::move(*gcr); + } + else + { + grad_concat_re = std::move(*gc); + } + + // 3. grad_V_re = batched_matmul(grad_concat, A, BH, false, false) + // forward: O = V × A^T → grad_V = grad_O × A + auto grad_V_re = engine.batched_matmul( + grad_concat_re, attn_cache_, BH, false, false); + if (!grad_V_re) return std::unexpected(grad_V_re.error()); + + // 4. grad_A = batched_matmul(V, grad_concat, BH, true, false) + auto grad_A = engine.batched_matmul( + V_cache_, grad_concat_re, BH, true, false); + if (!grad_A) return std::unexpected(grad_A.error()); + + // 5. grad_S = softmax.backward(grad_A) — 掩码/偏置为常数，梯度直接穿透 + auto grad_S = softmax_.backward(engine, *grad_A); + if (!grad_S) return std::unexpected(grad_S.error()); + + // 6. grad_Q_re = batched_matmul(K, grad_S, BH, false, true) × scale + auto grad_Q_re = engine.batched_matmul( + K_cache_, *grad_S, BH, false, true); + if (!grad_Q_re) return std::unexpected(grad_Q_re.error()); + auto rq = engine.scale_inplace(*grad_Q_re, scale_); + if (!rq) return std::unexpected(rq.error()); + + // 7. grad_K_re = batched_matmul(Q, grad_S, BH, false, false) × scale + auto grad_K_re = engine.batched_matmul( + Q_cache_, *grad_S, BH, false, false); + if (!grad_K_re) return std::unexpected(grad_K_re.error()); + auto rk = engine.scale_inplace(*grad_K_re, scale_); + if (!rk) return std::unexpected(rk.error()); + + // 8. rearrange back: (batch*H*d_k, seq) → (H*d_k, batch*seq) + Tensor grad_Q, grad_K, grad_V; + if (batch > 1) + { + auto gq = engine.rearrange_3d(*grad_Q_re, H_dk, batch, seq, true); + if (!gq) return std::unexpected(gq.error()); + grad_Q = std::move(*gq); + auto gk = engine.rearrange_3d(*grad_K_re, H_dk, batch, seq, true); + if (!gk) return std::unexpected(gk.error()); + grad_K = std::move(*gk); + auto gv = engine.rearrange_3d(*grad_V_re, H_dk, batch, seq, true); + if (!gv) return std::unexpected(gv.error()); + grad_V = std::move(*gv); + } + else + { + grad_Q = std::move(*grad_Q_re); + grad_K = std::move(*grad_K_re); + grad_V = std::move(*grad_V_re); + } + + // 9. 投影层反向 + 累加输入梯度 + auto giq = w_q_.backward(engine, grad_Q); + if (!giq) return giq; + Tensor grad_input = std::move(*giq); + + auto gik = w_k_.backward(engine, grad_K); + if (!gik) return gik; + auto r1 = engine.add_inplace(grad_input, *gik); + if (!r1) return std::unexpected(r1.error()); + + auto giv = w_v_.backward(engine, grad_V); + if (!giv) return giv; + auto r2 = engine.add_inplace(grad_input, *giv); + if (!r2) return std::unexpected(r2.error()); + + return grad_input; + } + + // ── 增量推理（KV cache）────────────────────────────────────────── + // 处理单个新 token，复用历史 K/V 缓存，避免重复计算前文。 + // + // KV cache 布局: (max_len, H*d_k)，row=position，col=head×dim + // - 投影得到 k_new/v_new: (H*d_k, 1) + // - transpose → (1, H*d_k) 后 insert_rows 到 cache 第 cur_len 行 + // - slice_rows 取前 new_len 行参与 attention + // + // 无需因果掩码：新 token 天然只能看到自身及之前的位置（cache 中只有前文）。 + // + // 输入: x_new (d_model, 1) 单个新 token 的嵌入 + // 输出: (d_model, 1) attention 输出 + [[nodiscard]] Result forward_step( + ComputeEngine& engine, + const Tensor& x_new, + Tensor& k_cache, + Tensor& v_cache, + std::size_t cur_len) + { + if (x_new.rows() != d_model_ || x_new.cols() != 1) + return std::unexpected(Error{"AttentionBase forward_step: x_new must be (d_model, 1)"}); + + // 1. Q/K/V 投影 → (H*d_k, 1) + auto q_res = w_q_.forward(engine, x_new); + if (!q_res) return q_res; + auto k_new = w_k_.forward(engine, x_new); + if (!k_new) return k_new; + auto v_new = w_v_.forward(engine, x_new); + if (!v_new) return v_new; + + // 2. transpose → (1, H*d_k)，匹配 cache 的行布局 + auto k_new_T = engine.transpose(*k_new); + if (!k_new_T) return std::unexpected(k_new_T.error()); + auto v_new_T = engine.transpose(*v_new); + if (!v_new_T) return std::unexpected(v_new_T.error()); + + // 3. 追加到 KV cache（就地写入第 cur_len 行） + auto r1 = engine.insert_rows(k_cache, cur_len, *k_new_T); + if (!r1) return std::unexpected(r1.error()); + auto r2 = engine.insert_rows(v_cache, cur_len, *v_new_T); + if (!r2) return std::unexpected(r2.error()); + + // 4. 取有效区间 [0, new_len) 并 transpose 为 (H*d_k, new_len) 布局 + // batched_matmul 要求 rows 能被 batch 整除: + // (new_len, H*d_k) 的 rows=new_len 不保证整除 num_heads + // transpose 后 (H*d_k, new_len) 的 rows=H*d_k=d_model 必然整除 + const std::size_t new_len = cur_len + 1; + auto k_valid = engine.slice_rows(k_cache, 0, new_len); + if (!k_valid) return std::unexpected(k_valid.error()); + auto v_valid = engine.slice_rows(v_cache, 0, new_len); + if (!v_valid) return std::unexpected(v_valid.error()); + auto K_T = engine.transpose(*k_valid); // (H*d_k, new_len) + if (!K_T) return std::unexpected(K_T.error()); + auto V_T = engine.transpose(*v_valid); // (H*d_k, new_len) + if (!V_T) return std::unexpected(V_T.error()); + + // 5. scores = batched_matmul(Q, K_T, H, transA=T, transB=F) + // Q: (H*d_k, 1) — 每头 (d_k, 1)，转置后 (1, d_k)，M=1 + // K_T: (H*d_k, new_len) — 每头 (d_k, new_len)，transB=F，N=new_len + // 每头: (1, d_k) × (d_k, new_len) = (1, new_len) + // 堆叠: (H, new_len) + auto scores = engine.batched_matmul( + *q_res, *K_T, num_heads_, true, false); + if (!scores) return std::unexpected(scores.error()); + + // 6. scores *= scale + auto rs = engine.scale_inplace(*scores, scale_); + if (!rs) return std::unexpected(rs.error()); + + // 6.5 施加增量推理掩码钩子（默认 no-op；ALiBi 施加线性偏置） + auto masked = apply_mask_step_(engine, std::move(*scores), cur_len); + if (!masked) return std::unexpected(masked.error()); + + // 7. softmax（行级归一化，每头独立） + auto attn = softmax_.forward(engine, *masked); + if (!attn) return std::unexpected(attn.error()); + + // 8. attn_out = batched_matmul(V_T, attn, H, transA=F, transB=T) + // V_T: (H*d_k, new_len) — 每头 (d_k, new_len)，transA=F，M=d_k + // attn: (H, new_len) — 每头 (1, new_len)，转置后 (new_len, 1)，N=1 + // 每头: (d_k, new_len) × (new_len, 1) = (d_k, 1) + // 堆叠: (H*d_k, 1) + auto attn_out = engine.batched_matmul( + *V_T, *attn, num_heads_, false, true); + if (!attn_out) return std::unexpected(attn_out.error()); + + // 9. 输出投影 → (d_model, 1) + return w_o_.forward(engine, *attn_out); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// MultiHeadAttention — 无掩码多头注意力（极简子类：沿用 AttentionBase 默认行为） +// ══════════════════════════════════════════════════════════════════════════ +class MultiHeadAttention final : public AttentionBase +{ +public: + MultiHeadAttention(ComputeEngine& engine, + std::size_t d_model, std::size_t num_heads, + std::size_t seq_len = 0) + : AttentionBase(engine, d_model, num_heads, seq_len) {} +}; + +// ══════════════════════════════════════════════════════════════════════════ +// PositionalEncoding — 正弦波固定位置编码（支持 batch tiling） +// +// 算法（只在此处，不在 Engine/Shader）： +// PE(pos, 2i) = sin(pos / 10000^(2i/d_model)) +// PE(pos, 2i+1) = cos(pos / 10000^(2i/d_model)) +// forward: out = input + PE (elementwise Add) +// backward: 梯度直接穿透（编码不可学习） +// +// Tiling 模式（tile_size > 0）： +// 当输入为 (d_model, batch * tile_size) 时，生成 (d_model, tile_size) 的 +// 基础编码，沿列方向 tile 为 (d_model, batch * tile_size)。 +// 每个 tile_size 列块为同一份编码，对应一个样本。 +// ══════════════════════════════════════════════════════════════════════════ +class PositionalEncoding final : public Layer +{ +private: + std::size_t d_model_; + std::size_t max_len_; + std::size_t tile_size_; // >0: 启用 tiling（每个样本的序列长度） + Tensor encoding_cache_; + std::size_t cached_total_ = 0; + + void rebuild_encoding(ComputeEngine& engine, std::size_t total_len) + { + const std::size_t base_len = (tile_size_ > 0) ? tile_size_ : total_len; + const std::size_t batch = (tile_size_ > 0) ? total_len / tile_size_ : 1; + + Matrix enc(d_model_, total_len); + const std::size_t half = d_model_ / 2; + std::vector freqs(half); + for (std::size_t i = 0; i < half; ++i) + freqs[i] = Scalar{1} / std::pow(Scalar{10000}, + static_cast(2 * i) / static_cast(d_model_)); + + for (std::size_t b = 0; b < batch; ++b) + { + auto span = enc.span(); + const std::size_t col_off = b * base_len; + for (std::size_t pos = 0; pos < base_len; ++pos) + { + const Scalar pos_d = static_cast(pos); + for (std::size_t i = 0; i < half; ++i) + { + const Scalar angle = pos_d * freqs[i]; + span[(2 * i) * total_len + col_off + pos] = std::sin(angle); + span[(2 * i + 1) * total_len + col_off + pos] = std::cos(angle); + } + if (d_model_ % 2 == 1) + { + const Scalar freq_last = Scalar{1} / std::pow(Scalar{10000}, + static_cast(2 * half) / static_cast(d_model_)); + span[(d_model_ - 1) * total_len + col_off + pos] = std::sin(pos_d * freq_last); + } + } + } + auto r = engine.from_matrix(enc); + NN_ASSERT(r, r ? "" : r.error().message.c_str()); + encoding_cache_ = std::move(*r); + cached_total_ = total_len; + } + +public: + PositionalEncoding(ComputeEngine& /*engine*/, + std::size_t d_model, std::size_t max_len = 5000, + std::size_t tile_size = 0) + : d_model_(d_model), max_len_(max_len), tile_size_(tile_size) + { + } + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + if (input.rows() != d_model_) + return std::unexpected(Error{"PE forward: d_model mismatch"}); + const std::size_t total_len = input.cols(); + if (tile_size_ > 0 && total_len % tile_size_ != 0) + return std::unexpected(Error{"PE forward: total_len not divisible by tile_size"}); + if (tile_size_ == 0 && total_len > max_len_) + return std::unexpected(Error{"PE forward: seq_len exceeds max_len"}); + + if (cached_total_ != total_len) + rebuild_encoding(engine, total_len); + + return engine.elementwise_binary(BinaryOp::Add, input, encoding_cache_); + } + + [[nodiscard]] Result backward( + ComputeEngine& /*engine*/, const Tensor& grad_output) override + { + return grad_output; // 位置编码不可学习，梯度直接穿透 + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// FeedForward — 前馈网络 (FFN(x) = Linear₂(GeLU(Linear₁(x)))) +// ══════════════════════════════════════════════════════════════════════════ +class FeedForward final : public Layer +{ +private: + Linear fc1_; // GeLU: (d_model → d_ff); SwiGLU: (d_model → 2*d_ff) + Linear fc2_; // (d_ff → d_model) + GeLU gelu_; + SwiGLU swiglu_; // SwiGLU 激活（含 split/merge） + bool use_swiglu_ = false; + +public: + FeedForward(ComputeEngine& engine, + std::size_t d_model, std::size_t d_ff, + ActivationType activation = ActivationType::GeLU) + : fc1_(engine, d_model, + activation == ActivationType::SwiGLU ? 2 * d_ff : d_ff), + fc2_(engine, d_ff, d_model), + swiglu_(d_ff), + use_swiglu_(activation == ActivationType::SwiGLU) {} + + std::vector parameters() override + { + auto p = fc1_.parameters(); + auto p2 = fc2_.parameters(); + p.insert(p.end(), p2.begin(), p2.end()); + return p; + } + + std::vector param_gradients() override + { + auto g = fc1_.param_gradients(); + auto g2 = fc2_.param_gradients(); + g.insert(g.end(), g2.begin(), g2.end()); + return g; + } + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + auto h1 = fc1_.forward(engine, input); + if (!h1) return h1; + if (use_swiglu_) + { + auto h2 = swiglu_.forward(engine, *h1); + if (!h2) return h2; + return fc2_.forward(engine, *h2); + } + auto h2 = gelu_.forward(engine, *h1); + if (!h2) return h2; + return fc2_.forward(engine, *h2); + } + + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + auto b2 = fc2_.backward(engine, grad_output); + if (!b2) return b2; + if (use_swiglu_) + { + auto bg = swiglu_.backward(engine, *b2); + if (!bg) return bg; + return fc1_.backward(engine, *bg); + } + auto bg = gelu_.backward(engine, *b2); + if (!bg) return bg; + return fc1_.backward(engine, *bg); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// TransformerEncoderLayer — Pre-Norm 编码器层（批量化） +// +// 算法（只在此处，不在 Engine/Shader）： +// x = x + SelfAttn(LN₁(x)) +// x = x + FFN(LN₂(x)) +// +// 输入/输出形状：(d_model, batch * seq_len) +// seq_len > 0 时启用 MHA 批量化路径（消除 per-head 和 per-sample 循环）。 +// ══════════════════════════════════════════════════════════════════════════ +class TransformerEncoderLayer final : public Layer +{ +private: + MultiHeadAttention self_attn_; + LayerNorm norm1_; + FeedForward ff_; + LayerNorm norm2_; + + Tensor residual1_cache_; + Tensor residual2_cache_; + +public: + TransformerEncoderLayer(ComputeEngine& engine, + std::size_t d_model, std::size_t num_heads, + std::size_t d_ff, std::size_t seq_len = 0) + : self_attn_(engine, d_model, num_heads, seq_len), + norm1_(engine, d_model), + ff_(engine, d_model, d_ff), + norm2_(engine, d_model) {} + + std::vector parameters() override + { + auto p = self_attn_.parameters(); + auto n1 = norm1_.parameters(); + auto f = ff_.parameters(); + auto n2 = norm2_.parameters(); + p.insert(p.end(), n1.begin(), n1.end()); + p.insert(p.end(), f.begin(), f.end()); + p.insert(p.end(), n2.begin(), n2.end()); + return p; + } + + std::vector param_gradients() override + { + auto g = self_attn_.param_gradients(); + auto gn1 = norm1_.param_gradients(); + auto gf = ff_.param_gradients(); + auto gn2 = norm2_.param_gradients(); + g.insert(g.end(), gn1.begin(), gn1.end()); + g.insert(g.end(), gf.begin(), gf.end()); + g.insert(g.end(), gn2.begin(), gn2.end()); + return g; + } + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + residual1_cache_ = input; + + // x1 = LN₁(input) + auto n1 = norm1_.forward(engine, input); + if (!n1) return n1; + + // a = SelfAttn(x1) + auto a = self_attn_.forward(engine, *n1); + if (!a) return a; + + // r2 = input + a + auto r2 = engine.elementwise_binary(BinaryOp::Add, input, *a); + if (!r2) return std::unexpected(r2.error()); + residual2_cache_ = *r2; + + // x2 = LN₂(r2) + auto n2 = norm2_.forward(engine, residual2_cache_); + if (!n2) return n2; + + // f = FFN(x2) + auto f = ff_.forward(engine, *n2); + if (!f) return f; + + // out = r2 + f + return engine.elementwise_binary(BinaryOp::Add, residual2_cache_, *f); + } + + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + // 残差2 反向: 分流到 residual1 + FFN + auto grad_ff = ff_.backward(engine, grad_output); + if (!grad_ff) return grad_ff; + auto b_n2 = norm2_.backward(engine, *grad_ff); + if (!b_n2) return b_n2; + + auto grad_r1 = engine.elementwise_binary(BinaryOp::Add, grad_output, *b_n2); + if (!grad_r1) return std::unexpected(grad_r1.error()); + + // 残差1 反向: 分流到 input + SelfAttn + auto b_sa = self_attn_.backward(engine, *grad_r1); + if (!b_sa) return b_sa; + auto b_n1 = norm1_.backward(engine, *b_sa); + if (!b_n1) return b_n1; + + return engine.elementwise_binary(BinaryOp::Add, *grad_r1, *b_n1); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// TransformerEncoder — ViT 风格的 Transformer 编码器（全批量化） +// +// 算法（只在此处，不在 Engine/Shader）： +// 1. 输入 (d_model, batch * num_patches) — PatchEmbedding 输出，batch-major 列 +// 2. + tiled PE → (d_model, batch * num_patches) +// 3. 一次性通过 N 个 EncoderLayer（MHA 内部用 rearrange_3d 批量化） +// 4. 全局平均池化（按样本聚合 num_patches 维度）→ (d_model, batch) +// +// 全 GPU 批量化策略： +// - 整个 batch 同时通过所有层，无 per-sample 循环 +// - 池化用 rearrange_3d + row_reduce_sum + rearrange_3d，纯 GPU 原语 +// - 反向池化用 rearrange_3d + matmul(grad, ones_row_)，纯 GPU 原语 +// - 无 batch 边界 PCIe 传输（输入输出均为 GPU 张量流） +// +// 输入: (d_model, batch * num_patches) — PatchEmbedding 输出 +// 输出: (d_model, batch) — 池化后的序列表示 +// ══════════════════════════════════════════════════════════════════════════ +class TransformerEncoder final : public Layer +{ +private: + std::size_t d_model_; + std::size_t num_patches_; + Scalar inv_num_patches_; + std::vector layers_; + PositionalEncoding pos_encoding_; + + std::size_t batch_size_ = 0; + Tensor ones_row_; // (1, num_patches) 全1，用于 backward 池化梯度广播 + +public: + TransformerEncoder(ComputeEngine& engine, + std::size_t d_model, std::size_t num_heads, + std::size_t d_ff, std::size_t num_layers, + std::size_t num_patches) + : d_model_(d_model), num_patches_(num_patches), + inv_num_patches_(Scalar{1} / static_cast(num_patches)), + // PE 启用 tiling：每个样本独立使用 (d_model, num_patches) 的编码 + pos_encoding_(engine, d_model, num_patches, num_patches) + { + // 所有 EncoderLayer 传入 seq_len=num_patches，启用 MHA 批量化路径 + for (std::size_t i = 0; i < num_layers; ++i) + layers_.emplace_back(engine, d_model, num_heads, d_ff, num_patches); + + // 预创建 ones_row_ (1, num_patches) 全1，用于 backward 广播 + Matrix ones_cpu(1, num_patches_, Scalar{1}); + auto or_t = engine.from_matrix(ones_cpu); + NN_ASSERT(or_t, or_t ? "" : or_t.error().message.c_str()); + ones_row_ = std::move(*or_t); + } + + std::vector parameters() override + { + std::vector p; + for (auto& l : layers_) + { + auto lp = l.parameters(); + p.insert(p.end(), lp.begin(), lp.end()); + } + return p; + } + + std::vector param_gradients() override + { + std::vector g; + for (auto& l : layers_) + { + auto lg = l.param_gradients(); + g.insert(g.end(), lg.begin(), lg.end()); + } + return g; + } + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + // input: (d_model, batch * num_patches) + if (input.rows() != d_model_) + return std::unexpected(Error{"TransformerEncoder: row count mismatch"}); + if (input.cols() % num_patches_ != 0) + return std::unexpected(Error{"TransformerEncoder: cols not divisible by num_patches"}); + batch_size_ = input.cols() / num_patches_; + + // 1. 添加 tiled 位置编码 → (d_model, batch * num_patches) + auto pe = pos_encoding_.forward(engine, input); + if (!pe) return pe; + Tensor x = std::move(*pe); + + // 2. 一次性通过所有 EncoderLayer [全批量化 GPU] + for (auto& layer : layers_) + { + auto lr = layer.forward(engine, x); + if (!lr) return lr; + x = std::move(*lr); + } + // x: (d_model, batch * num_patches) + + // 3. 全局平均池化（按样本聚合 num_patches 维度）→ (d_model, batch) + // 使用 rearrange_3d 把 (d_model, batch*num_patches) 重排为 + // (batch*d_model, num_patches)，每行块对应一个样本， + // row_reduce_sum 后得到 (batch*d_model, 1)， + // 再 rearrange_3d inverse 回 (d_model, batch)。 + auto re = engine.rearrange_3d(x, d_model_, batch_size_, num_patches_, false); + if (!re) return std::unexpected(re.error()); + auto row_sum = engine.row_reduce_sum(*re); + if (!row_sum) return std::unexpected(row_sum.error()); + auto rs_re = engine.rearrange_3d(*row_sum, d_model_, batch_size_, 1, true); + if (!rs_re) return std::unexpected(rs_re.error()); + auto r = engine.scale_inplace(*rs_re, inv_num_patches_); + if (!r) return std::unexpected(r.error()); + return *rs_re; + } + + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + // grad_output: (d_model, batch) — 来自下游 Linear.backward + + // 1. 反向池化：(d_model, batch) → (d_model, batch * num_patches) + // 每个样本的池化梯度广播到 num_patches 列： + // grad_x[d, b*num_patches + p] = grad_out[d, b] * inv_num_patches + // 实现：rearrange_3d(grad, d_model, batch, 1, false) → (batch*d_model, 1) + // scale(inv_n) → matmul(ones_row_, result_T) ... 直接： + // matmul(grad_col_vec, ones_row_) → (batch*d_model, num_patches) + // rearrange_3d(result, d_model, batch, num_patches, true) → (d_model, batch*num_patches) + auto g_re = engine.rearrange_3d(grad_output, d_model_, batch_size_, 1, false); + if (!g_re) return std::unexpected(g_re.error()); + auto r = engine.scale_inplace(*g_re, inv_num_patches_); + if (!r) return std::unexpected(r.error()); + // (*g_re): (batch*d_model, 1) × ones_row_ (1, num_patches) → (batch*d_model, num_patches) + auto unpooled = engine.matmul(*g_re, ones_row_, false, false); + if (!unpooled) return std::unexpected(unpooled.error()); + auto grad = engine.rearrange_3d(*unpooled, d_model_, batch_size_, num_patches_, true); + if (!grad) return std::unexpected(grad.error()); + Tensor grad_x = std::move(*grad); + + // 2. 反向通过所有 EncoderLayer [全批量化 GPU] + for (auto it = layers_.rbegin(); it != layers_.rend(); ++it) + { + auto br = it->backward(engine, grad_x); + if (!br) return br; + grad_x = std::move(*br); + } + + // 3. 反向 PE（梯度直接穿透） + return grad_x; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// PatchEmbedding — 图像 patch 嵌入 +// +// 算法（只在此处，不在 Engine/Shader）： +// 1. 将 (img_size², batch) 展平图像提取 num_patches 个不重叠 patch +// 2. 每个展平 patch (patch_size²) 经 Linear 投影到 d_model 维 +// 3. 重排为 (d_model, batch * num_patches) —— batch-major 列布局 +// [r, b * num_patches + p] = 样本 b 的 patch p 的第 r 维特征 +// 这样下游 TransformerEncoder 可整体批量化（消除 per-sample 循环）。 +// +// 注意: patch 提取涉及复杂重排，此处用 to_matrix/from_matrix 在 CPU 端 +// 完成（batch 边界，PCIe 传输符合纯 GPU 架构约定）。 +// ══════════════════════════════════════════════════════════════════════════ +class PatchEmbedding final : public Layer +{ +private: + std::size_t img_size_; + std::size_t patch_size_; + std::size_t grid_size_; + std::size_t num_patches_; + std::size_t patch_dim_; + std::size_t d_model_; + Linear projection_; + Tensor input_cache_; + +public: + PatchEmbedding(ComputeEngine& engine, + std::size_t img_size, std::size_t patch_size, + std::size_t d_model) + : img_size_(img_size), patch_size_(patch_size), + grid_size_(img_size / patch_size), + num_patches_(grid_size_ * grid_size_), + patch_dim_(patch_size * patch_size), + d_model_(d_model), + projection_(engine, patch_dim_, d_model) + { + NN_ASSERT(img_size % patch_size == 0, + "PatchEmbedding: img_size must be divisible by patch_size"); + } + + [[nodiscard]] std::size_t num_patches() const noexcept { return num_patches_; } + [[nodiscard]] std::size_t d_model() const noexcept { return d_model_; } + + std::vector parameters() override + { return projection_.parameters(); } + + std::vector param_gradients() override + { return projection_.param_gradients(); } + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + input_cache_ = input; + const std::size_t batch = input.cols(); + + // Step 1: CPU 端提取 patches，直接生成 batch-major 列布局 + // all_patches[flat, b * num_patches + p] = image_b[pixel(flat, p)] + // 样本 b 占连续列块 [b*num_patches, (b+1)*num_patches)， + // 便于下游 MHA 的 rearrange_3d 按 batch 切分列块。 + // （patch 提取是 PatchEmbedding 的算法职责，无对应 op-level 原语； + // 此处为 batch 边界的合法 CPU 预处理，与 GPTModel 的 gather_rows 同性质） + auto in_m = engine.to_matrix(input); + if (!in_m) return std::unexpected(in_m.error()); + + Matrix all_patches(patch_dim_, batch * num_patches_); + for (std::size_t b = 0; b < batch; ++b) + { + for (std::size_t p = 0; p < num_patches_; ++p) + { + const std::size_t gr = (p / grid_size_) * patch_size_; + const std::size_t gc = (p % grid_size_) * patch_size_; + const std::size_t col_idx = b * num_patches_ + p; // batch-major + for (std::size_t pr = 0; pr < patch_size_; ++pr) + for (std::size_t pc = 0; pc < patch_size_; ++pc) + { + const std::size_t flat = pr * patch_size_ + pc; + const std::size_t pix = (gr + pr) * img_size_ + (gc + pc); + all_patches.set_value_unchecked(flat, col_idx, + in_m->at_unchecked(pix, b)); + } + } + } + + auto ap_t = engine.from_matrix(all_patches); + if (!ap_t) return std::unexpected(ap_t.error()); + + // Step 2: 投影 → (d_model, batch * num_patches) — 已是 batch-major，无需重排 + return projection_.forward(engine, *ap_t); + } + + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + // grad_output: (d_model, batch * num_patches) — batch-major + const std::size_t batch = grad_output.cols() / num_patches_; + + // Step 1: 投影层反向 → (patch_dim, batch * num_patches) — batch-major + auto bp = projection_.backward(engine, grad_output); + if (!bp) return bp; + + // Step 2: 散射梯度回输入 → (img_size², batch) + auto gp_m = engine.to_matrix(*bp); + if (!gp_m) return std::unexpected(gp_m.error()); + + Matrix grad_input(img_size_ * img_size_, batch); + for (std::size_t b = 0; b < batch; ++b) + { + for (std::size_t p = 0; p < num_patches_; ++p) + { + const std::size_t gr = (p / grid_size_) * patch_size_; + const std::size_t gc = (p % grid_size_) * patch_size_; + const std::size_t col_idx = b * num_patches_ + p; // batch-major + for (std::size_t pr = 0; pr < patch_size_; ++pr) + for (std::size_t pc = 0; pc < patch_size_; ++pc) + { + const std::size_t flat = pr * patch_size_ + pc; + const std::size_t pix = (gr + pr) * img_size_ + (gc + pc); + const Scalar val = grad_input.at_unchecked(pix, b) + + gp_m->at_unchecked(flat, col_idx); + grad_input.set_value_unchecked(pix, b, val); + } + } + } + return engine.from_matrix(grad_input); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// CausalSelfAttention — 因果自注意力（继承 AttentionBase，仅重写掩码钩子） +// +// 通过 PosEncodingType 参数支持两种掩码模式： +// - Learned / Sinusoidal: 仅施加因果掩码（mask[i][j] = 0 if j<=i, -inf if j>i） +// - ALiBi: 因果掩码 + ALiBi 线性偏置 +// +// ALiBi (Attention with Linear Biases) 原理： +// 不使用位置嵌入，而是在注意力分数上添加线性偏置： +// attention_score = Q*K^T + bias +// 其中 bias[i][j] = -m_h * (i - j) for j <= i +// 斜率 m_h = 2^(-8h/H)，h 是头索引，H 是总头数 +// +// 优点（ALiBi 模式）： +// 1. 无需位置嵌入，减少参数 +// 2. 天然支持长度外推（训练短序列，推理长序列） +// 3. 计算开销极小（仅添加预计算的偏置） +// +// 算法差异（相对于 AttentionBase）： +// 在 forward 的 scale 之后、softmax 之前，施加预计算的掩码： +// S += mask (batch*H*seq, seq) — 因果掩码（按 batch*H 平铺，ALiBi 模式下含线性偏置） +// backward 无需特殊处理（掩码为常数，softmax.backward 已处理梯度穿透） +// +// seq_len 由构造函数指定，batch = input.cols() / seq_len 在 forward 时推断。 +// seq_len=0 表示单样本模式（cols 即 seq），保持向后兼容。 +// ══════════════════════════════════════════════════════════════════════════ +class CausalSelfAttention final : public AttentionBase +{ +private: + bool use_alibi_; // true = ALiBi 模式（因果掩码 + 线性偏置） + + // 掩码缓存（平铺为 batch*H*seq × seq 以匹配堆叠 scores 布局） + // ALiBi 模式下包含因果掩码 + 线性偏置；普通模式下仅因果掩码 + Tensor mask_cache_; + std::size_t mask_cached_batch_ = 0; // 缓存键：batch（独立字段，避免位打包溢出） + std::size_t mask_cached_seq_ = 0; // 缓存键：seq_len + + // ALiBi 斜率：m_h = 2^(-8h/H)（仅 use_alibi_ = true 时使用） + std::vector slopes_; + + void init_slopes_() + { + slopes_.resize(num_heads_); + for (std::size_t h = 0; h < num_heads_; ++h) + { + // m_h = 2^(-8h/H) + slopes_[h] = std::pow(Scalar{2}, -Scalar{8} * h / num_heads_); + } + } + + void ensure_mask_(ComputeEngine& engine, std::size_t batch, std::size_t seq_len) + { + // 用 (batch, seq_len) 作为缓存键 + if (mask_cached_batch_ == batch && mask_cached_seq_ == seq_len) + return; + + const Scalar neg_inf = -std::numeric_limits::infinity(); + const std::size_t BH = batch * num_heads_; + // 平铺掩码: (BH * seq_len, seq_len) + // 普通模式: mask[i][j] = 0 if j<=i, -inf if j>i + // ALiBi 模式: mask[i][j] = -m_h*(i-j) if j<=i, -inf if j>i + Matrix mask(BH * seq_len, seq_len); + for (std::size_t b = 0; b < batch; ++b) + { + for (std::size_t h = 0; h < num_heads_; ++h) + { + const std::size_t head_idx = b * num_heads_ + h; + const Scalar slope = use_alibi_ ? slopes_[h] : Scalar{0}; + + for (std::size_t i = 0; i < seq_len; ++i) + { + for (std::size_t j = 0; j < seq_len; ++j) + { + if (j <= i) + { + // 因果允许位置：ALiBi 模式下添加线性偏置 -m_h*(i-j)，否则为 0 + const Scalar bias = use_alibi_ + ? -slope * static_cast(i - j) + : Scalar{0}; + mask.set_value_unchecked(head_idx * seq_len + i, j, bias); + } + else + { + // 因果掩码：未来位置为 -inf + mask.set_value_unchecked(head_idx * seq_len + i, j, neg_inf); + } + } + } + } + } + auto r = engine.from_matrix(mask); + NN_ASSERT(r, r ? "" : r.error().message.c_str()); + mask_cache_ = std::move(*r); + mask_cached_batch_ = batch; + mask_cached_seq_ = seq_len; + } + +protected: + // 重写掩码钩子：在 scale 之后、softmax 之前施加因果/ALiBi 掩码 + [[nodiscard]] Result apply_mask_( + ComputeEngine& engine, Tensor&& scores, + std::size_t batch, std::size_t seq) override + { + ensure_mask_(engine, batch, seq); + return engine.elementwise_binary(BinaryOp::Add, scores, mask_cache_); + } + + // 重写增量推理掩码钩子：ALiBi 模式下施加线性距离偏置。 + // 普通因果模式（use_alibi_ = false）：no-op，因 cache 只含前文，因果天然满足。 + // ALiBi 模式：scores[h, j] += -slope[h] * (cur_len - j)，j ∈ [0, cur_len]。 + // scores 布局: (H, new_len)，new_len = cur_len + 1 + // query 在位置 cur_len，key 在位置 j，距离 = cur_len - j + [[nodiscard]] Result apply_mask_step_( + ComputeEngine& engine, Tensor&& scores, + std::size_t cur_len) override + { + if (!use_alibi_) return std::move(scores); + + const std::size_t new_len = cur_len + 1; + // 构建 ALiBi 偏置矩阵 (H, new_len) + Matrix bias(num_heads_, new_len); + for (std::size_t h = 0; h < num_heads_; ++h) + { + const Scalar slope = slopes_[h]; + for (std::size_t j = 0; j < new_len; ++j) + { + // j ∈ [0, cur_len]，距离 cur_len - j ∈ [0, cur_len] + const std::size_t dist = cur_len - j; + bias.set_value_unchecked(h, j, + -slope * static_cast(dist)); + } + } + auto bias_t = engine.from_matrix(bias); + if (!bias_t) return std::unexpected(bias_t.error()); + return engine.elementwise_binary(BinaryOp::Add, scores, *bias_t); + } + +public: + CausalSelfAttention(ComputeEngine& engine, + std::size_t d_model, std::size_t num_heads, + std::size_t /*max_len*/ = 1024, + std::size_t seq_len = 0, + PosEncodingType pos_enc = PosEncodingType::Learned) + : AttentionBase(engine, d_model, num_heads, seq_len), + use_alibi_(pos_enc == PosEncodingType::ALiBi) + { + if (use_alibi_) + init_slopes_(); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// GPTBlock — Pre-Norm 解码器块 +// +// 通过 pos_enc 参数选择注意力模式（Learned/Sinusoidal/ALiBi）： +// x = x + CausalSelfAttn(LN₁(x), pos_enc) +// x = x + FFN(LN₂(x)) +// ══════════════════════════════════════════════════════════════════════════ +class GPTBlock final : public Layer +{ +private: + CausalSelfAttention self_attn_; + std::unique_ptr norm1_; + FeedForward ff_; + std::unique_ptr norm2_; + + Tensor residual1_cache_; + Tensor residual2_cache_; + +public: + GPTBlock(ComputeEngine& engine, + std::size_t d_model, std::size_t num_heads, + std::size_t d_ff, std::size_t max_len = 1024, + std::size_t seq_len = 0, + PosEncodingType pos_enc = PosEncodingType::Learned, + ActivationType activation = ActivationType::GeLU, + NormType norm_type = NormType::LayerNorm) + : self_attn_(engine, d_model, num_heads, max_len, seq_len, pos_enc), + norm1_(make_norm_layer(engine, d_model, norm_type)), + ff_(engine, d_model, d_ff, activation), + norm2_(make_norm_layer(engine, d_model, norm_type)) {} + + std::vector parameters() override + { + auto p = self_attn_.parameters(); + auto n1 = norm1_->parameters(); + auto f = ff_.parameters(); + auto n2 = norm2_->parameters(); + p.insert(p.end(), n1.begin(), n1.end()); + p.insert(p.end(), f.begin(), f.end()); + p.insert(p.end(), n2.begin(), n2.end()); + return p; + } + + std::vector param_gradients() override + { + auto g = self_attn_.param_gradients(); + auto gn1 = norm1_->param_gradients(); + auto gf = ff_.param_gradients(); + auto gn2 = norm2_->param_gradients(); + g.insert(g.end(), gn1.begin(), gn1.end()); + g.insert(g.end(), gf.begin(), gf.end()); + g.insert(g.end(), gn2.begin(), gn2.end()); + return g; + } + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + residual1_cache_ = input; + + auto n1 = norm1_->forward(engine, input); + if (!n1) return n1; + + auto a = self_attn_.forward(engine, *n1); + if (!a) return a; + + auto r2 = engine.elementwise_binary(BinaryOp::Add, input, *a); + if (!r2) return std::unexpected(r2.error()); + residual2_cache_ = *r2; + + auto n2 = norm2_->forward(engine, residual2_cache_); + if (!n2) return n2; + + auto f = ff_.forward(engine, *n2); + if (!f) return f; + + return engine.elementwise_binary(BinaryOp::Add, residual2_cache_, *f); + } + + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + auto grad_ff = ff_.backward(engine, grad_output); + if (!grad_ff) return grad_ff; + auto b_n2 = norm2_->backward(engine, *grad_ff); + if (!b_n2) return b_n2; + + auto grad_r1 = engine.elementwise_binary(BinaryOp::Add, grad_output, *b_n2); + if (!grad_r1) return std::unexpected(grad_r1.error()); + + auto b_sa = self_attn_.backward(engine, *grad_r1); + if (!b_sa) return b_sa; + auto b_n1 = norm1_->backward(engine, *b_sa); + if (!b_n1) return b_n1; + + return engine.elementwise_binary(BinaryOp::Add, *grad_r1, *b_n1); + } + + // ── 增量推理（KV cache）────────────────────────────────────────── + // Pre-Norm 单 token 前向： + // x = x_new + CausalSelfAttn(LN₁(x_new), kv_cache) + // x = x + FFN(LN₂(x)) + // 输入: x_new (d_model, 1) + // 输出: (d_model, 1) + [[nodiscard]] Result forward_step( + ComputeEngine& engine, + const Tensor& x_new, + Tensor& k_cache, + Tensor& v_cache, + std::size_t cur_len) + { + auto n1 = norm1_->forward(engine, x_new); + if (!n1) return n1; + + auto a = self_attn_.forward_step(engine, *n1, k_cache, v_cache, cur_len); + if (!a) return a; + + auto r2 = engine.elementwise_binary(BinaryOp::Add, x_new, *a); + if (!r2) return std::unexpected(r2.error()); + + auto n2 = norm2_->forward(engine, *r2); + if (!n2) return n2; + + auto f = ff_.forward(engine, *n2); + if (!f) return f; + + return engine.elementwise_binary(BinaryOp::Add, *r2, *f); + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// GPTModel — Decoder-only Transformer 语言模型 +// +// 算法（只在此处，不在 Engine/Shader）： +// 组件: TokenEmb [+ PosEmb] + N × GPTBlock + LayerNorm + LM Head +// 输入: (seq_len, batch_size) — token ID 矩阵（每列为一个序列） +// 输出: (vocab_size, seq_len × batch_size) — 每个位置的 logits +// +// 通过 PosEncodingType 参数支持三种位置编码模式： +// - Learned: 可学习位置嵌入（默认） +// - Sinusoidal: 正弦波固定位置编码（冻结） +// - ALiBi: 无位置嵌入，通过 CausalSelfAttention 的线性偏置注入位置信息 +// +// 注意: token embedding 查表 + 位置 embedding 相加涉及按 token ID 的 +// 稀疏写入，此处用 to_matrix/from_matrix 在 CPU 端完成 +// （batch 边界，PCIe 传输符合纯 GPU 架构约定）。 +// ══════════════════════════════════════════════════════════════════════════ +class GPTModel final : public Layer +{ +private: + std::size_t vocab_size_; + std::size_t d_model_; + std::size_t seq_len_; + bool use_pos_emb_; // true = 使用位置嵌入（Learned 或 Sinusoidal），false = ALiBi 模式 + bool pos_emb_learnable_; // true = 可学习位置嵌入（Learned），false = 冻结（Sinusoidal）或 ALiBi + + // 嵌入 + Tensor token_emb_; // (vocab_size, d_model) + Tensor grad_token_emb_; + Tensor pos_emb_; // (seq_len, d_model) — 仅 use_pos_emb_ = true 时有效 + Tensor grad_pos_emb_; // 同上 + + std::vector blocks_; + std::unique_ptr ln_f_; + Linear lm_head_; + + // 反向缓存 + Tensor stored_x_; // forward 输入 (d_model, seq*batch) + std::vector stored_tokens_flat_; // 所有 token IDs (seq*batch) + Tensor stored_tokens_tensor_; // token IDs 的 Tensor 版本 (total, 1) + std::size_t batch_size_ = 0; + + // pos_indices 缓存（避免每 step 重建）— 仅 use_pos_emb_ = true 时使用 + Tensor pos_indices_cache_; // (total, 1) 值为 [0,0,..,1,1,..,...,seq-1,..] + std::size_t pos_indices_batch_ = 0; // 缓存键：batch_size（独立字段，避免位打包溢出） + std::size_t pos_indices_seq_ = 0; // 缓存键：seq_len + + // batch 录制粒度：每隔 flush_interval_ 个 Transformer block 提交一次 + // 0 = 不在 block 间 flush（默认），>0 = 每 N 个 block flush 一次 + std::size_t flush_interval_ = 0; + +public: + GPTModel(ComputeEngine& engine, + std::size_t vocab_size, std::size_t d_model, std::size_t seq_len, + std::size_t num_heads, std::size_t d_ff, std::size_t num_layers, + PosEncodingType pos_enc_type = PosEncodingType::Learned, + ActivationType activation = ActivationType::GeLU, + NormType norm_type = NormType::LayerNorm) + : vocab_size_(vocab_size), d_model_(d_model), seq_len_(seq_len), + use_pos_emb_(pos_enc_type != PosEncodingType::ALiBi), + pos_emb_learnable_(pos_enc_type == PosEncodingType::Learned), + ln_f_(make_norm_layer(engine, d_model, norm_type)), + lm_head_(engine, d_model, vocab_size) + { + // 初始化 token_emb_ + Matrix te(vocab_size, d_model); + constexpr Scalar emb_init_std = 0.02; + std::mt19937_64 rng{42}; + std::normal_distribution dist(0.0, emb_init_std); + auto te_s = te.span(); + for (std::size_t i = 0; i < te.size(); ++i) te_s[i] = dist(rng); + + auto te_r = engine.from_matrix(te); + NN_ASSERT(te_r, te_r ? "" : te_r.error().message.c_str()); + token_emb_ = std::move(*te_r); + + grad_token_emb_ = engine.create_tensor(vocab_size, d_model); + { auto r1 = engine.zero(grad_token_emb_); NN_ASSERT(r1, r1 ? "" : r1.error().message.c_str()); } + + // 初始化 pos_emb_（仅非 ALiBi 模式） + if (use_pos_emb_) + { + Matrix pe(seq_len, d_model); + if (pos_enc_type == PosEncodingType::Sinusoidal) + { + // 正弦波固定位置编码: PE(pos, 2i) = sin(pos/10000^(2i/d)), PE(pos, 2i+1) = cos(...) + auto pe_s = pe.span(); + for (std::size_t pos = 0; pos < seq_len; ++pos) + for (std::size_t i = 0; i < d_model; ++i) + { + Scalar angle = static_cast(pos) / + std::pow(Scalar{10000}, static_cast(2 * (i / 2)) / static_cast(d_model)); + pe_s[pos * d_model + i] = (i % 2 == 0) ? std::sin(angle) : std::cos(angle); + } + } + else + { + // 可学习位置编码: N(0, 0.02) 随机初始化 + auto pe_s = pe.span(); + for (std::size_t i = 0; i < pe.size(); ++i) pe_s[i] = dist(rng); + } + + auto pe_r = engine.from_matrix(pe); + NN_ASSERT(pe_r, pe_r ? "" : pe_r.error().message.c_str()); + pos_emb_ = std::move(*pe_r); + + grad_pos_emb_ = engine.create_tensor(seq_len, d_model); + { auto r2 = engine.zero(grad_pos_emb_); NN_ASSERT(r2, r2 ? "" : r2.error().message.c_str()); } + } + + // GPTBlock 传入 seq_len、pos_enc_type 和 activation + for (std::size_t i = 0; i < num_layers; ++i) + blocks_.emplace_back(engine, d_model, num_heads, d_ff, seq_len, seq_len, + pos_enc_type, activation); + } + + std::vector parameters() override + { + std::vector p; + p.push_back(token_emb_); + if (use_pos_emb_ && pos_emb_learnable_) + p.push_back(pos_emb_); + for (auto& b : blocks_) + { + auto bp = b.parameters(); + p.insert(p.end(), bp.begin(), bp.end()); + } + auto lp = ln_f_->parameters(); + p.insert(p.end(), lp.begin(), lp.end()); + auto hp = lm_head_.parameters(); + p.insert(p.end(), hp.begin(), hp.end()); + return p; + } + + std::vector param_gradients() override + { + std::vector g; + g.push_back(grad_token_emb_); + if (use_pos_emb_ && pos_emb_learnable_) + g.push_back(grad_pos_emb_); + for (auto& b : blocks_) + { + auto bg = b.param_gradients(); + g.insert(g.end(), bg.begin(), bg.end()); + } + auto lg = ln_f_->param_gradients(); + g.insert(g.end(), lg.begin(), lg.end()); + auto hg = lm_head_.param_gradients(); + g.insert(g.end(), hg.begin(), hg.end()); + return g; + } + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& input) override + { + const std::size_t seq_len = input.rows(); + batch_size_ = input.cols(); + const std::size_t total = seq_len * batch_size_; + + // ── 1. gather 所有 token 的 embedding ── + // input: (seq, batch) 标量 token IDs（row-major: i = t * batch + b） + // gather_rows(token_emb_, input) → (seq*batch, d_model) + // row i = token_emb[input[i]], i 是 input 的 row-major 索引 + auto all_emb = engine.gather_rows(token_emb_, input); + if (!all_emb) return std::unexpected(all_emb.error()); + + // ── 2. 保存 token IDs 的 Tensor 拷贝（供 backward 的 scatter_add_rows） ── + // 全程 GPU：clone 在 GPU 内执行，无 PCIe 传输 + auto st_t = engine.clone(input); + if (!st_t) return std::unexpected(st_t.error()); + stored_tokens_tensor_ = std::move(*st_t); + + // ── 3. 构造 x: (d_model, seq*batch) ── + auto all_T = engine.transpose(*all_emb); + if (!all_T) return std::unexpected(all_T.error()); + + Tensor x_result; + if (use_pos_emb_) + { + // ── 3a. 确保 pos_indices 缓存有效 ── + { + if (pos_indices_batch_ != batch_size_ || pos_indices_seq_ != seq_len) + { + Matrix pidx_m(total, 1); + for (std::size_t t = 0; t < seq_len; ++t) + for (std::size_t b = 0; b < batch_size_; ++b) + pidx_m.set_value_unchecked(t * batch_size_ + b, 0, + static_cast(t)); + auto pidx_t = engine.from_matrix(pidx_m); + if (!pidx_t) return std::unexpected(pidx_t.error()); + pos_indices_cache_ = std::move(*pidx_t); + pos_indices_batch_ = batch_size_; + pos_indices_seq_ = seq_len; + } + } + + // x = transpose(gather(token_emb, input)) + transpose(gather(pos_emb, pos_idx)) + auto pos_gathered = engine.gather_rows(pos_emb_, pos_indices_cache_); + if (!pos_gathered) return std::unexpected(pos_gathered.error()); + + auto pos_T = engine.transpose(*pos_gathered); + if (!pos_T) return std::unexpected(pos_T.error()); + + auto x_with_pos = engine.elementwise_binary(BinaryOp::Add, *all_T, *pos_T); + if (!x_with_pos) return std::unexpected(x_with_pos.error()); + x_result = std::move(*x_with_pos); + } + else + { + // ALiBi 模式：不使用位置嵌入，直接使用 token embedding + x_result = std::move(*all_T); + } + stored_x_ = x_result; // 缓存供 backward（GPTBlock 内部也有自己的缓存） + + // ── 4. 通过 Transformer 块（全批量化，无 per-sample 循环） ── + Tensor x = std::move(x_result); + for (std::size_t bi = 0; bi < blocks_.size(); ++bi) + { + auto r = blocks_[bi].forward(engine, x); + if (!r) return r; + x = std::move(*r); + // 按间隔 flush，将大录制拆分为多个小提交（防 TDR） + if (flush_interval_ > 0 && (bi + 1) % flush_interval_ == 0 && bi + 1 < blocks_.size()) + { + auto fr = engine.flush_batch(); + if (!fr) return std::unexpected(fr.error()); + } + } + + // ── 5. 最终 LayerNorm/RMSNorm ── + auto ln = ln_f_->forward(engine, x); + if (!ln) return ln; + x = std::move(*ln); + + // ── 6. LM Head → (vocab_size, seq*batch) batch-major ── + auto lm_out = lm_head_.forward(engine, x); + return lm_out; + } + + [[nodiscard]] Result backward( + ComputeEngine& engine, const Tensor& grad_output) override + { + const std::size_t seq_len = seq_len_; + + (void)engine.zero(grad_token_emb_); + if (use_pos_emb_ && pos_emb_learnable_) + (void)engine.zero(grad_pos_emb_); + + // ── 1. LM Head 反向 → (d_model, seq*batch) ── + auto b_lm = lm_head_.backward(engine, grad_output); + if (!b_lm) return b_lm; + Tensor grad_x = std::move(*b_lm); + + // ── 2. LayerNorm/RMSNorm 反向 ── + auto b_ln = ln_f_->backward(engine, grad_x); + if (!b_ln) return b_ln; + grad_x = std::move(*b_ln); + + // ── 3. 逐块反向（全批量化） ── + { + const std::size_t n = blocks_.size(); + for (std::size_t bi = 0; bi < n; ++bi) + { + auto br = blocks_[n - 1 - bi].backward(engine, grad_x); + if (!br) return br; + grad_x = std::move(*br); + if (flush_interval_ > 0 && (bi + 1) % flush_interval_ == 0 && bi + 1 < n) + { + auto fr = engine.flush_batch(); + if (!fr) return std::unexpected(fr.error()); + } + } + } + + // ── 4. 转置 grad_x + pos_grad GPU 计算 ── + // grad_x: (d_model, seq*batch) + // grad_T = transpose(grad_x) → (total, d_model) — 用于 scatter_add_rows + // pos_grad = rearrange_3d(grad_x) + reduce → 全 GPU，无 PCIe 传输（仅 use_pos_emb_ 时） + auto grad_T = engine.transpose(grad_x); + if (!grad_T) return std::unexpected(grad_T.error()); + + if (use_pos_emb_ && pos_emb_learnable_) + { + // rearrange_3d(grad_x, d_model, seq, batch) → (seq*d_model, batch) + // 然后 row_reduce_sum → (seq*d_model, 1) = pos_grad 扁平化 + auto rearranged = engine.rearrange_3d( + grad_x, d_model_, seq_len, batch_size_, false); + if (!rearranged) return std::unexpected(rearranged.error()); + + auto reduced = engine.row_reduce_sum(*rearranged); + if (!reduced) return std::unexpected(reduced.error()); + + // reduced 是 (seq*d_model, 1)，数据布局为 pos_grad[t*d_model+d] + // reshape 为 (seq, d_model) — 后端无关零拷贝 reshape + Tensor pos_grad_t = reduced->reshape(seq_len, d_model_); + + auto ar = engine.add_inplace(grad_pos_emb_, pos_grad_t); + if (!ar) return std::unexpected(ar.error()); + } + + // ── 5. scatter_add_rows: grad_token_emb_[tokens] += grad_T ── + auto sr = engine.scatter_add_rows(grad_token_emb_, stored_tokens_tensor_, *grad_T); + if (!sr) return std::unexpected(sr.error()); + + // grad_input: token IDs 无梯度，返回零张量（仅用于接口一致性） + Matrix grad_input(seq_len, batch_size_, Scalar{0}); + return engine.from_matrix(grad_input); + } + + // ── batch 录制粒度控制 ── + void set_flush_interval(std::size_t interval) override { flush_interval_ = interval; } + [[nodiscard]] std::size_t flush_interval() const noexcept { return flush_interval_; } + + // ── 增量推理：单 token 前向（KV cache）────────────────────────── + // 流程: token_emb[token] [+ pos_emb[pos]] → N × GPTBlock.forward_step + // → ln_f → lm_head → (vocab_size, 1) + // 每层的 KV cache 由调用方维护，forward_step 只负责写入和计算。 + [[nodiscard]] Result forward_step( + ComputeEngine& engine, + std::size_t token_id, + std::size_t pos, + std::vector& k_caches, + std::vector& v_caches, + std::size_t cur_len) + { + // 1. token embedding 查表 → (1, d_model) → transpose → (d_model, 1) + Matrix id_m(1, 1); + id_m.set_value_unchecked(0, 0, static_cast(token_id)); + auto id_t = engine.from_matrix(id_m); + if (!id_t) return std::unexpected(id_t.error()); + auto emb = engine.gather_rows(token_emb_, *id_t); + if (!emb) return std::unexpected(emb.error()); + auto x_new = engine.transpose(*emb); + if (!x_new) return std::unexpected(x_new.error()); + + // 2. 位置嵌入（ALiBi 模式跳过） + if (use_pos_emb_) + { + Matrix pos_m(1, 1); + pos_m.set_value_unchecked(0, 0, static_cast(pos)); + auto pos_t = engine.from_matrix(pos_m); + if (!pos_t) return std::unexpected(pos_t.error()); + auto pos_emb_g = engine.gather_rows(pos_emb_, *pos_t); + if (!pos_emb_g) return std::unexpected(pos_emb_g.error()); + auto pos_T = engine.transpose(*pos_emb_g); + if (!pos_T) return std::unexpected(pos_T.error()); + auto x_wp = engine.elementwise_binary(BinaryOp::Add, *x_new, *pos_T); + if (!x_wp) return std::unexpected(x_wp.error()); + x_new = std::move(*x_wp); + } + + // 3. 逐块增量前向 + Tensor x = std::move(*x_new); + for (std::size_t i = 0; i < blocks_.size(); ++i) + { + auto r = blocks_[i].forward_step( + engine, x, k_caches[i], v_caches[i], cur_len); + if (!r) return r; + x = std::move(*r); + // 按 flush_interval 拆分提交（防 TDR） + if (flush_interval_ > 0 && + (i + 1) % flush_interval_ == 0 && i + 1 < blocks_.size()) + { + auto fr = engine.flush_batch(); + if (!fr) return std::unexpected(fr.error()); + } + } + + // 4. 最终 LayerNorm + LM Head → (vocab_size, 1) + auto ln = ln_f_->forward(engine, x); + if (!ln) return ln; + return lm_head_.forward(engine, *ln); + } + + // ── 采样生成（KV cache 增量推理）──────────────────────────────── + // 性能策略（P0 + P1 + KV cache）： + // P0: begin_batch/end_batch 包裹 forward_step，单次 GPU 提交。 + // P1: 每步只上传 1 个 token ID，无需重传整个 seq_len。 + // KV cache: 每步 attention 只计算 Q×K_history（O(seq_len) 而非 O(seq_len²)）， + // 历文 K/V 不再重复投影。 + // logits 直接是 (vocab_size, 1)，无需 transpose+slice。 + // + // 滑动窗口: 当 cur_len 达到 seq_len_ 时，丢弃最旧 token 重建 cache + // （保留最后 seq_len_-1 个 token 作为新上下文）。 + [[nodiscard]] Result> + generate(ComputeEngine& engine, + const std::vector& prompt, + std::size_t max_new_tokens, + Scalar temperature = 1.0, + std::size_t eos_token_id = static_cast(-1), + std::size_t min_new_tokens = 0) + { + std::vector context(prompt); + std::vector generated; + std::mt19937_64 rng{std::random_device{}()}; + std::uniform_real_distribution dist(0.0, 1.0); + + // ── 预分配 KV cache: 每层一对 (k, v)，形状 (seq_len_, d_model) ── + // H*d_k = d_model（因为 d_k = d_model / num_heads） + std::vector k_caches, v_caches; + k_caches.reserve(blocks_.size()); + v_caches.reserve(blocks_.size()); + for (std::size_t i = 0; i < blocks_.size(); ++i) + { + k_caches.push_back(engine.create_tensor(seq_len_, d_model_)); + v_caches.push_back(engine.create_tensor(seq_len_, d_model_)); + } + + // ── prefill: 逐 token 填充 KV cache，保留最后 logits ────────── + // 截断到 seq_len_ 长度（滑动窗口初始） + std::size_t start_init = 0; + if (context.size() > seq_len_) + start_init = context.size() - seq_len_; + std::size_t cur_len = 0; + Tensor last_logits_t; + for (std::size_t i = start_init; i < context.size(); ++i) + { + auto r = forward_step(engine, context[i], cur_len, + k_caches, v_caches, cur_len); + if (!r) return std::unexpected(r.error()); + last_logits_t = *r; + ++cur_len; + } + + for (std::size_t step = 0; step < max_new_tokens; ++step) + { + // Sliding window: rebuild cache when full (keep last seq_len_-1 tokens) + if (cur_len >= seq_len_) + { + for (auto& kc : k_caches) { auto r = engine.zero(kc); if (!r) return std::unexpected(r.error()); } + for (auto& vc : v_caches) { auto r = engine.zero(vc); if (!r) return std::unexpected(r.error()); } + cur_len = 0; + std::size_t keep = seq_len_ - 1; + std::size_t start_new = (context.size() > keep) ? (context.size() - keep) : 0; + for (std::size_t i = start_new; i < context.size(); ++i) + { + auto r = forward_step(engine, context[i], cur_len, + k_caches, v_caches, cur_len); + if (!r) return std::unexpected(r.error()); + last_logits_t = *r; + ++cur_len; + } + } + + // Sample from last_logits_t (from prefill or previous step) + auto logits_m = engine.to_matrix(last_logits_t); + if (!logits_m) return std::unexpected(logits_m.error()); + + std::vector last_logits(vocab_size_); + for (std::size_t v = 0; v < vocab_size_; ++v) + last_logits[v] = logits_m->at_unchecked(v, 0); + + // temperature + if (temperature > 0.0 && temperature != 1.0) + for (auto& v : last_logits) v /= temperature; + + // softmax（数值稳定） + Scalar max_val = last_logits[0]; + for (std::size_t v = 1; v < vocab_size_; ++v) + max_val = std::max(max_val, last_logits[v]); + Scalar sum_exp = 0.0; + for (auto& v : last_logits) + { + v = std::exp(v - max_val); + sum_exp += v; + } + for (auto& v : last_logits) v /= sum_exp; + + // 采样 + // temperature > 0: 随机采样（temperature 仅影响缩放，1.0 = 不缩放但仍然采样） + // temperature == 0: 贪心解码（argmax） + std::size_t next_token; + if (temperature > 0.0) + { + Scalar r = dist(rng); + Scalar cumulative = 0.0; + next_token = vocab_size_ - 1; + for (std::size_t v = 0; v < vocab_size_; ++v) + { + cumulative += last_logits[v]; + if (r <= cumulative) { next_token = v; break; } + } + } + else + { + next_token = 0; + Scalar best = last_logits[0]; + for (std::size_t v = 1; v < vocab_size_; ++v) + if (last_logits[v] > best) { best = last_logits[v]; next_token = v; } + } + + context.push_back(next_token); + + if (step >= min_new_tokens && next_token == eos_token_id) + break; + + generated.push_back(next_token); + + // Run forward_step for next_token to write KV cache and get new logits + auto br = engine.begin_batch(); + if (!br) return std::unexpected(br.error()); + + auto logits_res = forward_step(engine, next_token, cur_len, + k_caches, v_caches, cur_len); + if (!logits_res) return std::unexpected(logits_res.error()); + + auto er = engine.end_batch(); + if (!er) return std::unexpected(er.error()); + + last_logits_t = std::move(*logits_res); + ++cur_len; + } + return generated; + } +}; + +} // namespace nn + +#endif // NN_COMPUTE_LAYER_HPP diff --git a/include/neuralnet.cpp/compute_loss.hpp b/include/neuralnet.cpp/compute_loss.hpp new file mode 100644 index 00000000..b1e31579 --- /dev/null +++ b/include/neuralnet.cpp/compute_loss.hpp @@ -0,0 +1,337 @@ +#ifndef NN_COMPUTE_LOSS_HPP +#define NN_COMPUTE_LOSS_HPP + +// ── compute_loss.hpp — 引擎化损失函数 ────────────────────────────────────── +// +// 架构铁律： +// 1. Loss 的 forward/backward 只写一次，通过 ComputeEngine 参数自动适配 +// CPU/GPU 设备。 +// 2. 算法只在 Loss（通过组合 engine 原语表达），绝不在 Engine/Shader 中。 +// 3. Softmax/CrossEntropy 算法由本文件通过 col_reduce_max + exp + col_reduce_sum +// + broadcast_col + elementwise 组合表达，Engine 不知道 "softmax" 是什么。 +// +// 算法表达示例： +// MSELoss forward: grad = (2/N) * (pred - target); loss = (1/N) * Σ diff² +// CrossEntropy forward (with softmax): +// col_max = col_reduce_max(logits) // 数值稳定 +// shifted = logits - col_max (broadcast_col Sub) +// exp_shift = exp(shifted) (unary Exp) +// col_sum = col_reduce_sum(exp_shift) +// softmax = exp_shift / col_sum (broadcast_col Div) +// grad = softmax - target (elementwise Sub) +// log_sm = shifted - log(col_sum) (broadcast_col Sub) +// loss = -(1/batch) * Σ target * log_sm +// CrossEntropy backward: grad = softmax - target_onehot +// ───────────────────────────────────────────────────────────────────────── + +#include "compute_engine.hpp" +#include "compute_layer.hpp" // clone_tensor +#include "tensor.hpp" + +namespace nn +{ + +// ══════════════════════════════════════════════════════════════════════════ +// Loss — 引擎化损失函数基类 +// ══════════════════════════════════════════════════════════════════════════ +class Loss +{ +public: + virtual ~Loss() = default; + + // forward 计算损失标量，并缓存 backward 所需中间结果 + [[nodiscard]] virtual Result forward( + ComputeEngine& engine, const Tensor& pred, const Tensor& target) = 0; + + // backward 返回对 pred 的梯度 + [[nodiscard]] virtual Result backward() = 0; +}; + +// ══════════════════════════════════════════════════════════════════════════ +// MSELoss — 均方误差损失 +// +// 算法（只在此处，不在 Engine/Shader）： +// diff = pred - target +// grad = (2/N) * diff +// loss = (1/N) * Σ diff² +// ══════════════════════════════════════════════════════════════════════════ +class MSELoss final : public Loss +{ +private: + Tensor grad_input_; + +public: + MSELoss() = default; + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& pred, const Tensor& target) override + { + if (pred.rows() != target.rows() || pred.cols() != target.cols()) + return std::unexpected(Error{"mse loss: shape mismatch"}); + if (pred.size() == 0) + return std::unexpected(Error{"mse loss: empty input"}); + + const Scalar total = static_cast(pred.size()); + const Scalar scale = Scalar{2} / total; + + // diff = pred - target + auto diff = engine.elementwise_binary(BinaryOp::Sub, pred, target); + if (!diff) return std::unexpected(diff.error()); + + // grad = diff * (2/N) — 深拷贝后 scale + grad_input_ = *diff; + auto r = engine.scale_inplace(grad_input_, scale); + if (!r) return std::unexpected(r.error()); + + // diff_sq = diff * diff + auto diff_sq = engine.elementwise_binary(BinaryOp::Mul, *diff, *diff); + if (!diff_sq) return std::unexpected(diff_sq.error()); + + // 全局求和：先按列归约 (1, cols)，再按行归约 (1, 1) + auto col_sum = engine.col_reduce_sum(*diff_sq); + if (!col_sum) return std::unexpected(col_sum.error()); + auto total_t = engine.row_reduce_sum(*col_sum); + if (!total_t) return std::unexpected(total_t.error()); + + // 下载标量 + auto m = engine.to_matrix(*total_t); + if (!m) return std::unexpected(m.error()); + + return m->at_unchecked(0, 0) / total; + } + + [[nodiscard]] Result backward() override + { + return grad_input_; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// CrossEntropyLoss — 带 softmax 的交叉熵损失 +// +// 算法（只在此处，不在 Engine/Shader）： +// 对每列（batch 样本）独立做 softmax + cross entropy： +// col_max = max_c logits[c][i] (col_reduce_max) +// shifted = logits - col_max (broadcast_col Sub) +// exp_shift = exp(shifted) (unary Exp) +// col_sum = Σ_c exp_shift[c][i] (col_reduce_sum) +// softmax = exp_shift / col_sum (broadcast_col Div) +// grad = softmax - target_onehot (elementwise Sub) +// log_sm = shifted - log(col_sum) (broadcast_col Sub) +// loss = -(1/batch) * Σ target * log_sm +// ══════════════════════════════════════════════════════════════════════════ +class CrossEntropyLoss final : public Loss +{ +private: + Tensor grad_input_; + +public: + CrossEntropyLoss() = default; + + [[nodiscard]] Result forward( + ComputeEngine& engine, const Tensor& logits, const Tensor& target) override + { + const std::size_t classes = logits.rows(); + const std::size_t batch = logits.cols(); + + if (target.rows() != classes || target.cols() != batch) + return std::unexpected(Error{"cross_entropy loss: shape mismatch"}); + if (classes == 0 || batch == 0) + return std::unexpected(Error{"cross_entropy loss: empty input"}); + + // 1. col_max = max per column (数值稳定) + auto col_max = engine.col_reduce_max(logits); + if (!col_max) return std::unexpected(col_max.error()); + + // 2. shifted = logits - col_max (broadcast_col Sub) — 深拷贝 logits + auto shifted = clone_tensor(engine, logits); + if (!shifted) return std::unexpected(shifted.error()); + auto r = engine.broadcast_col_inplace(*shifted, *col_max, BinaryOp::Sub); + if (!r) return std::unexpected(r.error()); + + // 3. exp_shift = exp(shifted) + auto exp_shift = engine.elementwise_unary(UnaryOp::Exp, *shifted); + if (!exp_shift) return std::unexpected(exp_shift.error()); + + // 4. col_sum = Σ_c exp_shift[c][i] + auto col_sum = engine.col_reduce_sum(*exp_shift); + if (!col_sum) return std::unexpected(col_sum.error()); + + // 5. softmax = exp_shift / col_sum (broadcast_col Div) — 深拷贝 exp_shift + auto softmax = clone_tensor(engine, *exp_shift); + if (!softmax) return std::unexpected(softmax.error()); + r = engine.broadcast_col_inplace(*softmax, *col_sum, BinaryOp::Div); + if (!r) return std::unexpected(r.error()); + + // 6. grad = softmax - target (elementwise Sub) + auto grad = engine.elementwise_binary(BinaryOp::Sub, *softmax, target); + if (!grad) return std::unexpected(grad.error()); + grad_input_ = *grad; + + // 7. log_softmax = shifted - log(col_sum) + auto log_col_sum = engine.elementwise_unary(UnaryOp::Log, *col_sum); + if (!log_col_sum) return std::unexpected(log_col_sum.error()); + + auto log_softmax = clone_tensor(engine, *shifted); + if (!log_softmax) return std::unexpected(log_softmax.error()); + r = engine.broadcast_col_inplace(*log_softmax, *log_col_sum, BinaryOp::Sub); + if (!r) return std::unexpected(r.error()); + + // 8. target_dot_log = target * log_softmax + auto target_dot_log = engine.elementwise_binary(BinaryOp::Mul, target, *log_softmax); + if (!target_dot_log) return std::unexpected(target_dot_log.error()); + + // 9. Σ target * log_softmax (先列归约再行归约 → (1,1)) + auto col_s = engine.col_reduce_sum(*target_dot_log); + if (!col_s) return std::unexpected(col_s.error()); + auto total_t = engine.row_reduce_sum(*col_s); + if (!total_t) return std::unexpected(total_t.error()); + + // 10. loss = -total / batch — 下载标量 + auto m = engine.to_matrix(*total_t); + if (!m) return std::unexpected(m.error()); + + return -m->at_unchecked(0, 0) / static_cast(batch); + } + + [[nodiscard]] Result backward() override + { + return grad_input_; + } + + // ── 稀疏标签版 forward：接受整数标签而非 one-hot 矩阵 ────────── + // + // 解决大词表（vocab_size≈25k）+ 大 batch 时 one-hot 矩阵 + // (vocab_size, total_tokens) 爆显存的问题。 + // + // 算法： + // 1. 在 GPU 上计算 softmax（与 dense 版相同） + // 2. 下载 softmax 到 CPU（唯一的大数据传输） + // 3. 在 CPU 上用整数标签计算 loss 和 gradient + // loss = -(1/N) * Σ log(softmax[labels[i], i]) + // grad = softmax; grad[labels[i]][i] -= 1.0（对有效位置） + // 4. 将 gradient 上传回 GPU + // + // 参数： + // labels — 平坦标签数组，大小 = logits.cols()，值域 [0, vocab_size) + // loss_mask — 可选，平坦 mask 数组，>0.5 表示参与 loss，否则清零梯度 + // vocab_size — 词表大小，用于越界检查 + [[nodiscard]] Result forward_sparse( + ComputeEngine& engine, const Tensor& logits, + std::span labels, + std::span loss_mask = {}, + std::size_t vocab_size = 0) + { + const std::size_t classes = logits.rows(); + const std::size_t total = logits.cols(); + + if (vocab_size == 0) vocab_size = classes; + if (labels.size() != total) + return std::unexpected(Error{"sparse CE: labels size mismatch"}); + if (!loss_mask.empty() && loss_mask.size() != total) + return std::unexpected(Error{"sparse CE: mask size mismatch"}); + if (classes == 0 || total == 0) + return std::unexpected(Error{"sparse CE: empty input"}); + + // ── 1. GPU 上计算 softmax（复用原 dense forward 的逻辑） ──── + auto col_max = engine.col_reduce_max(logits); + if (!col_max) return std::unexpected(col_max.error()); + + auto shifted = clone_tensor(engine, logits); + if (!shifted) return std::unexpected(shifted.error()); + auto r = engine.broadcast_col_inplace(*shifted, *col_max, BinaryOp::Sub); + if (!r) return std::unexpected(r.error()); + + auto exp_shift = engine.elementwise_unary(UnaryOp::Exp, *shifted); + if (!exp_shift) return std::unexpected(exp_shift.error()); + + auto col_sum = engine.col_reduce_sum(*exp_shift); + if (!col_sum) return std::unexpected(col_sum.error()); + + // softmax in-place 于 exp_shift 上，省一次 (vocab, total) 分配 + r = engine.broadcast_col_inplace(*exp_shift, *col_sum, BinaryOp::Div); + if (!r) return std::unexpected(r.error()); + + // ── 2. 下载 softmax 到 CPU ──────────────────────────────── + auto softmax_cpu = engine.to_matrix(*exp_shift); + if (!softmax_cpu) return std::unexpected(softmax_cpu.error()); + + // exp_shift (即 softmax) 现在可以释放 + exp_shift = Tensor(); + + // ── 3. CPU 上计算 loss 和 gradient ──────────────────────── + const std::size_t rows = softmax_cpu->rows(); + const std::size_t cols = softmax_cpu->cols(); + + // gradient 拷贝 softmax（后面只修改稀疏位置） + Matrix grad_cpu(*softmax_cpu); + + Scalar loss_sum = 0.0; + std::size_t num_valid = 0; + const bool use_mask = !loss_mask.empty(); + + for (std::size_t i = 0; i < cols; ++i) + { + // mask 检查 + if (use_mask && loss_mask[i] < Scalar{0.5}) + { + // 被 mask 的位置：gradient 整列清零 + for (std::size_t c = 0; c < rows; ++c) + grad_cpu.set_value_unchecked(c, i, 0.0); + continue; + } + + const std::size_t lbl = labels[i]; + if (lbl >= vocab_size) + { + // 越界标签：跳过 loss，gradient 整列清零 + for (std::size_t c = 0; c < rows; ++c) + grad_cpu.set_value_unchecked(c, i, 0.0); + continue; + } + + // loss += log(softmax[lbl, i]) + const Scalar sm_val = softmax_cpu->at_unchecked(lbl, i); + loss_sum += std::log(std::max(sm_val, Scalar{1e-20})); + ++num_valid; + + // gradient: softmax[lbl, i] -= 1.0（softmax - one_hot 的稀疏等价） + grad_cpu.set_value_unchecked(lbl, i, sm_val - Scalar{1}); + } + + // 除以有效 token 数（而非 total），避免 padding 拉低 loss 报告值 + const Scalar loss = (num_valid > 0) + ? -loss_sum / static_cast(num_valid) + : Scalar{0}; + + // ── 3.5 梯度归一化：与 PyTorch 一致，grad = (softmax - one_hot) / num_valid ── + // 之前缺少 1/num_valid 缩放，导致梯度被整体放大 num_valid 倍。 + // 由于 loss = -Σ log_softmax / num_valid，故 + // d(loss)/d(logits) = (softmax - one_hot) / num_valid（有效位置）， + // mask 位置/越界标签位置梯度为 0（缩放 0 仍为 0，安全）。 + // 对 Adam 而言，梯度常数缩放会被二阶矩归一化抵消（曲线不变）， + // 但对 SGD/动量、梯度裁剪 --max-norm 以及 --grad-log 统计， + // 缺少该缩放会导致与 PyTorch 行为不一致。 + if (num_valid > 0) + { + const Scalar inv_num_valid = Scalar{1} / static_cast(num_valid); + for (std::size_t c = 0; c < rows; ++c) + for (std::size_t i = 0; i < cols; ++i) + { + const Scalar g = grad_cpu.at_unchecked(c, i) * inv_num_valid; + grad_cpu.set_value_unchecked(c, i, g); + } + } + + // ── 4. 上传 gradient 到 GPU ────────────────────────────── + auto grad_tensor_r = engine.from_matrix(grad_cpu); + if (!grad_tensor_r) return std::unexpected(grad_tensor_r.error()); + + grad_input_ = std::move(*grad_tensor_r); + return loss; + } +}; + +} // namespace nn + +#endif // NN_COMPUTE_LOSS_HPP diff --git a/include/neuralnet.cpp/compute_optimizer.hpp b/include/neuralnet.cpp/compute_optimizer.hpp new file mode 100644 index 00000000..fb988274 --- /dev/null +++ b/include/neuralnet.cpp/compute_optimizer.hpp @@ -0,0 +1,620 @@ +#ifndef NN_COMPUTE_OPTIMIZER_HPP +#define NN_COMPUTE_OPTIMIZER_HPP + +// ── compute_optimizer.hpp — 引擎化优化器 ─────────────────────────────────── +// +// 架构铁律： +// 1. Optimizer 的 step/zero_grad 只写一次，通过 ComputeEngine 参数自动适配 +// CPU/GPU 设备。 +// 2. 优化器算法只通过 engine 原语（scale_inplace / add_inplace / +// elementwise_binary / elementwise_unary 等）表达，不直接操作 Matrix。 +// 3. 参数与梯度均为 Tensor*（指向 Layer 持有的参数/梯度张量）。 +// +// 算法表达示例（原语组合）： +// SGD: p -= lr * g +// Momentum: v = β*v + (1-β)*g; p -= lr*v +// Adam: m = β1*m + (1-β1)*g; v = β2*v + (1-β2)*g²; +// p -= lr * (m/bc1) / (sqrt(v/bc2) + eps) +// AdamW: 同 Adam，但权重衰减解耦：p *= (1 - lr*wd) 独立于梯度更新 +// ───────────────────────────────────────────────────────────────────────── + +#include +#include +#include +#include +#include + +#include "compute_engine.hpp" +#include "compute_layer.hpp" // clone_tensor +#include "tensor.hpp" + +namespace nn +{ + +// ══════════════════════════════════════════════════════════════════════════ +// Optimizer — 引擎化优化器基类 +// +// 职责： +// - 持有 engine / params / grads 三元组（所有子类共享） +// - 提供 zero_grad() 默认实现 +// - 提供 validate_sizes_() / create_zero_buffers_() 公共辅助方法 +// ══════════════════════════════════════════════════════════════════════════ +class Optimizer +{ +protected: + ComputeEngine& engine_; // 非拥有引用（永不为空） + std::vector params_; // 非拥有引用（永不为空） + std::vector grads_; // 非拥有引用（永不为空） + + // 校验 params/grads 数量一致（各子类 step() 开头调用） + [[nodiscard]] Result validate_sizes_() const + { + if (params_.size() != grads_.size()) + return std::unexpected(Error{"Optimizer: params/grads size mismatch"}); + return {}; + } + + // 为每个参数创建同形状的零初始化 Tensor（供 Momentum/Adam/Muon 复用） + std::vector create_zero_buffers_() const + { + std::vector buffers; + buffers.reserve(params_.size()); + for (auto& p : params_) + { + auto buf = engine_.create_tensor(p.get().rows(), p.get().cols()); + { auto r = engine_.zero(buf); NN_ASSERT(r, r ? "" : r.error().message.c_str()); } + buffers.push_back(std::move(buf)); + } + return buffers; + } + +public: + Optimizer(ComputeEngine& engine, + std::vector params, + std::vector grads) + : engine_(engine), + params_(std::move(params)), + grads_(std::move(grads)) {} + + virtual ~Optimizer() = default; + + // 动态调整学习率（供 OscillationGuard 等自适应调度器使用） + virtual void set_lr(Scalar lr) = 0; + + [[nodiscard]] virtual Result step() = 0; + + // ── 梯度裁剪（max_norm）：全局 L2 范数裁剪 ───────────────────── + // 计算所有梯度张量的全局 L2 范数，若超过 max_norm 则等比例缩放。 + // 在 step() 之前调用： backward() → clip_grad_norm(max_norm) → step(). + [[nodiscard]] Result clip_grad_norm(Scalar max_norm) + { + if (max_norm <= 0 || grads_.empty()) + return {}; + + // 计算全局梯度平方和：每个梯度张量归约到 (1,1) 标量后在设备端累加， + // 最后仅下载一次（而非每张量一次 PCIe 往返）。 + Tensor acc; + for (auto& g_ref : grads_) + { + auto& g = g_ref.get(); + // g² = g * g（逐元素乘法） + auto g_sq_r = engine_.elementwise_binary(BinaryOp::Mul, g, g); + if (!g_sq_r) return std::unexpected(g_sq_r.error()); + // 按行求和 → (rows, 1) + auto row_sums_r = engine_.row_reduce_sum(*g_sq_r); + if (!row_sums_r) return std::unexpected(row_sums_r.error()); + // 按列求和 → (1, 1) 标量张量 + auto col_sum_r = engine_.col_reduce_sum(*row_sums_r); + if (!col_sum_r) return std::unexpected(col_sum_r.error()); + + if (acc.valid()) + { + auto sum_r = engine_.elementwise_binary(BinaryOp::Add, acc, *col_sum_r); + if (!sum_r) return std::unexpected(sum_r.error()); + acc = std::move(*sum_r); + } + else + { + acc = std::move(*col_sum_r); + } + } + + // 仅一次下载获取全局平方和 + auto m_r = engine_.to_matrix(acc); + if (!m_r) return std::unexpected(m_r.error()); + const Scalar total_sq = m_r->at(0, 0); + + Scalar norm = std::sqrt(total_sq); + if (norm <= max_norm) + return {}; + + // 等比例缩放所有梯度 + Scalar scale = max_norm / norm; + for (auto& g_ref : grads_) + { + auto r = engine_.scale_inplace(g_ref.get(), scale); + if (!r) return std::unexpected(r.error()); + } + return {}; + } + + // 默认实现：将所有梯度清零（所有子类行为一致） + [[nodiscard]] virtual Result zero_grad() + { + for (auto& g : grads_) + { + auto r = engine_.zero(g); + if (!r) return std::unexpected(r.error()); + } + return {}; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// SGD — 随机梯度下降 +// +// 算法：p -= lr * g +// 原语：axpy_inplace(p, -lr, g) +// ══════════════════════════════════════════════════════════════════════════ +class SGD : public Optimizer +{ + Scalar lr_; + +public: + SGD(ComputeEngine& engine, + std::vector params, + std::vector grads, + Scalar lr) + : Optimizer(engine, std::move(params), std::move(grads)), lr_(lr) {} + + void set_lr(Scalar lr) override { lr_ = lr; } + + [[nodiscard]] Result step() override + { + if (auto r = validate_sizes_(); !r) return std::unexpected(r.error()); + + for (std::size_t i = 0; i < params_.size(); ++i) + { + auto r = engine_.axpy_inplace(params_[i], -lr_, grads_[i]); + if (!r) return std::unexpected(r.error()); + } + return {}; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// SGDWithMomentum — 带动量的 SGD +// +// 算法：v = β*v + (1-β)*g; p -= lr*v +// 原语：axpy_inplace(v, 1-β, g) → axpy_inplace(p, -lr, v) +// ══════════════════════════════════════════════════════════════════════════ +class SGDWithMomentum : public Optimizer +{ + Scalar lr_; + Scalar beta_; + std::vector velocities_; + +public: + SGDWithMomentum(ComputeEngine& engine, + std::vector params, + std::vector grads, + Scalar lr, Scalar beta = 0.9) + : Optimizer(engine, std::move(params), std::move(grads)), + lr_(lr), beta_(beta), + velocities_(create_zero_buffers_()) {} + + void set_lr(Scalar lr) override { lr_ = lr; } + + [[nodiscard]] Result step() override + { + if (auto r = validate_sizes_(); !r) return std::unexpected(r.error()); + + const Scalar one_minus_beta = Scalar{1} - beta_; + + for (std::size_t i = 0; i < params_.size(); ++i) + { + // v = β*v + (1-β)*g + auto r = engine_.scale_inplace(velocities_[i], beta_); + if (!r) return std::unexpected(r.error()); + r = engine_.axpy_inplace(velocities_[i], one_minus_beta, grads_[i]); + if (!r) return std::unexpected(r.error()); + + // p -= lr * v + r = engine_.axpy_inplace(params_[i], -lr_, velocities_[i]); + if (!r) return std::unexpected(r.error()); + } + return {}; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// Adam — 自适应矩估计优化器 +// +// 算法： +// m = β1*m + (1-β1)*g +// v = β2*v + (1-β2)*g² +// p -= lr * (m/bc1) / (sqrt(v/bc2) + eps) +// ══════════════════════════════════════════════════════════════════════════ +class Adam : public Optimizer +{ +protected: + Scalar lr_; + Scalar beta1_; + Scalar beta2_; + Scalar eps_; + std::size_t t_; + std::vector m_; // 一阶矩 + std::vector v_; // 二阶矩 + + // Adam 核心更新（提取为 protected，AdamW 复用） + // inv_bc1/inv_bc2 由 step() 提前计算（每步仅一次 pow），失败时不推进 t。 + [[nodiscard]] Result adam_update_( + std::size_t i, Scalar inv_bc1, Scalar inv_bc2) + { + const Scalar one_minus_beta1 = Scalar{1} - beta1_; + const Scalar one_minus_beta2 = Scalar{1} - beta2_; + + const Tensor& g = grads_[i]; + + // m = β1*m + (1-β1)*g + auto r = engine_.scale_inplace(m_[i], beta1_); + if (!r) return std::unexpected(r.error()); + r = engine_.axpy_inplace(m_[i], one_minus_beta1, g); + if (!r) return std::unexpected(r.error()); + + // v = β2*v + (1-β2)*g² + r = engine_.scale_inplace(v_[i], beta2_); + if (!r) return std::unexpected(r.error()); + auto g_sq = engine_.elementwise_binary(BinaryOp::Mul, g, g); + if (!g_sq) return std::unexpected(g_sq.error()); + r = engine_.scale_inplace(*g_sq, one_minus_beta2); + if (!r) return std::unexpected(r.error()); + r = engine_.add_inplace(v_[i], *g_sq); + if (!r) return std::unexpected(r.error()); + + // m_hat = m / bc1 + auto m_hat = clone_tensor(engine_, m_[i]); + if (!m_hat) return std::unexpected(m_hat.error()); + r = engine_.scale_inplace(*m_hat, inv_bc1); + if (!r) return std::unexpected(r.error()); + + // v_hat = v / bc2 + auto v_hat = clone_tensor(engine_, v_[i]); + if (!v_hat) return std::unexpected(v_hat.error()); + r = engine_.scale_inplace(*v_hat, inv_bc2); + if (!r) return std::unexpected(r.error()); + + // sqrt_v = sqrt(v_hat) + auto sqrt_v = engine_.elementwise_unary(UnaryOp::Sqrt, *v_hat); + if (!sqrt_v) return std::unexpected(sqrt_v.error()); + + // denom = sqrt_v + eps + auto denom = engine_.elementwise_binary_scalar(BinaryOp::Add, *sqrt_v, eps_); + if (!denom) return std::unexpected(denom.error()); + + // ratio = m_hat / denom + auto ratio = engine_.elementwise_binary(BinaryOp::Div, *m_hat, *denom); + if (!ratio) return std::unexpected(ratio.error()); + + // p -= lr * ratio + r = engine_.scale_inplace(*ratio, -lr_); + if (!r) return std::unexpected(r.error()); + return engine_.add_inplace(params_[i], *ratio); + } + + void init_moments_() + { + m_ = create_zero_buffers_(); + v_ = create_zero_buffers_(); + } + +public: + Adam(ComputeEngine& engine, + std::vector params, + std::vector grads, + Scalar lr, + Scalar beta1 = 0.9, + Scalar beta2 = 0.999, + Scalar eps = 1e-8) + : Optimizer(engine, std::move(params), std::move(grads)), + lr_(lr), beta1_(beta1), beta2_(beta2), eps_(eps), t_(0) + { + init_moments_(); + } + + void set_lr(Scalar lr) override { lr_ = lr; } + + [[nodiscard]] Result step() override + { + if (auto r = validate_sizes_(); !r) return std::unexpected(r.error()); + + // 偏差修正：每步只计算一次（而非每参数重复 pow） + const std::size_t t_next = t_ + 1; + const Scalar bc1 = Scalar{1} - std::pow(beta1_, static_cast(t_next)); + const Scalar bc2 = Scalar{1} - std::pow(beta2_, static_cast(t_next)); + const Scalar inv_bc1 = Scalar{1} / bc1; + const Scalar inv_bc2 = Scalar{1} / bc2; + + for (std::size_t i = 0; i < params_.size(); ++i) + { + auto r = adam_update_(i, inv_bc1, inv_bc2); + if (!r) return std::unexpected(r.error()); + } + t_ = t_next; // 全部参数成功才推进步数（失败时不推进） + return {}; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// AdamW — 解耦权重衰减的 Adam +// +// 算法（Loshchilov & Hutter, 2019）： +// m = β1*m + (1-β1)*g +// v = β2*v + (1-β2)*g² +// p = (1 - lr*wd) * p ← 权重衰减解耦，独立于梯度更新 +// p -= lr * (m/bc1) / (sqrt(v/bc2) + eps) +// +// 与 Adam+L2正则化的区别： +// - L2: g' = g + wd*p，然后用 g' 做 Adam 更新 → wd 受自适应学习率缩放 +// - AdamW: 直接衰减权重 p *= (1-lr*wd)，梯度更新不受 wd 影响 +// → AdamW 的权重衰减对所有参数等效，不因自适应学习率而被稀释 +// ══════════════════════════════════════════════════════════════════════════ +class AdamW : public Adam +{ + Scalar wd_; // 权重衰减系数 + +public: + AdamW(ComputeEngine& engine, + std::vector params, + std::vector grads, + Scalar lr, + Scalar beta1 = 0.9, + Scalar beta2 = 0.999, + Scalar eps = 1e-8, + Scalar weight_decay = 0.01) + : Adam(engine, std::move(params), std::move(grads), + lr, beta1, beta2, eps), + wd_(weight_decay) {} + + // set_lr 复用 Adam::set_lr（lr_ 为 protected，无需 override） + + [[nodiscard]] Result step() override + { + if (auto r = validate_sizes_(); !r) return std::unexpected(r.error()); + + // 偏差修正：每步只计算一次（而非每参数重复 pow） + const std::size_t t_next = t_ + 1; + const Scalar bc1 = Scalar{1} - std::pow(beta1_, static_cast(t_next)); + const Scalar bc2 = Scalar{1} - std::pow(beta2_, static_cast(t_next)); + const Scalar inv_bc1 = Scalar{1} / bc1; + const Scalar inv_bc2 = Scalar{1} / bc2; + const Scalar decay_factor = Scalar{1} - lr_ * wd_; + + for (std::size_t i = 0; i < params_.size(); ++i) + { + // 权重衰减解耦：p = (1 - lr*wd) * p + if (wd_ != Scalar{0}) + { + auto r = engine_.scale_inplace(params_[i], decay_factor); + if (!r) return std::unexpected(r.error()); + } + + // Adam 更新 + auto r = adam_update_(i, inv_bc1, inv_bc2); + if (!r) return std::unexpected(r.error()); + } + t_ = t_next; // 全部参数成功才推进步数（失败时不推进） + return {}; + } +}; + +// ══════════════════════════════════════════════════════════════════════════ +// Newton-Schulz 正交化 — Muon 的核心操作 +// +// 给定矩阵 G，计算其最近的半正交矩阵 Ortho(G) = U V^T（G = U S V^T 为 SVD）。 +// 使用 quintic Newton-Schulz 迭代高效逼近，系数经调优可在 5 步内收敛。 +// +// 算法： +// X = G / (||G||_F + eps) +// for _ in range(steps): +// A = X @ X^T +// B = b*A + c*A^2 +// X = a*X + B @ X +// +// 参考：Keller Jordan et al., "Muon: An optimizer for hidden layers in neural networks" +// ══════════════════════════════════════════════════════════════════════════ +[[nodiscard]] inline Result newton_schulz_orthogonalize( + ComputeEngine& engine, const Tensor& G, std::size_t steps = 5, Scalar eps = 1e-7f) +{ + // 调优后的 quintic 多项式系数（使 φ^N(x) → 1 for x ∈ [0,1]） + constexpr Scalar a = 3.4445f; + constexpr Scalar b = -4.7750f; + constexpr Scalar c = 2.0315f; + + // 计算 Frobenius 范数的平方：||G||_F² = Σ g_ij² + // 通过 elementwise(Mul) → row_reduce_sum → col_reduce_sum 三步原语得到 (1,1) 张量 + auto norm_sq = engine.elementwise_binary(BinaryOp::Mul, G, G); + if (!norm_sq) return std::unexpected(norm_sq.error()); + auto row_sum_norm = engine.row_reduce_sum(*norm_sq); + if (!row_sum_norm) return std::unexpected(row_sum_norm.error()); + auto total_norm_sq = engine.col_reduce_sum(*row_sum_norm); + if (!total_norm_sq) return std::unexpected(total_norm_sq.error()); + + // 从 (1,1) Tensor 提取标量值 + // 使用 to_matrix 拉到 CPU 后取值（一次 PCIe 下载，可接受） + auto total_norm_sq_mat = engine.to_matrix(*total_norm_sq); + if (!total_norm_sq_mat) return std::unexpected(total_norm_sq_mat.error()); + Scalar norm_sq_val = total_norm_sq_mat->at(0, 0); + Scalar inv_norm_scalar = Scalar{1} / std::sqrt(norm_sq_val + eps * eps); + + // X = G * inv_norm_scalar（一次标量乘法归一化） + auto X = clone_tensor(engine, G); + if (!X) return std::unexpected(X.error()); + auto r = engine.scale_inplace(*X, inv_norm_scalar); + if (!r) return std::unexpected(r.error()); + + // Newton-Schulz 迭代：X_new = a*X + (b*A + c*A²) @ X，其中 A = X @ X^T + // + // 性能优化：通过就地复用 matmul 输出缓冲区，减少 clone_tensor 调用： + // - A 缓冲区就地修改为 B = b*A + c*A²（A 在计算 B 后不再需要） + // - BX 缓冲区就地添加 a*X 得到 X_new（BX 在 aX 加法后不再需要） + // 注意：每步仍创建 3 个新 Tensor（A、A_sq、BX，均为 matmul 返回值）， + // 但消除了额外的 clone_tensor 步骤。 + for (std::size_t t = 0; t < steps; ++t) + { + // A = X @ X^T（matmul 返回新 tensor，无分配开销） + auto A = engine.matmul(*X, *X, false, true); + if (!A) return std::unexpected(A.error()); + + // A_sq = A @ A（matmul 返回新 tensor） + auto A_sq = engine.matmul(*A, *A, false, false); + if (!A_sq) return std::unexpected(A_sq.error()); + + // B = b*A + c*A² + // 就地复用：A_sq 不再需要后修改为 c*A²，A 不再需要后修改为 b*A + r = engine.scale_inplace(*A_sq, c); + if (!r) return std::unexpected(r.error()); + r = engine.scale_inplace(*A, b); + if (!r) return std::unexpected(r.error()); + r = engine.add_inplace(*A, *A_sq); // A 现在是 B = b*A + c*A² + if (!r) return std::unexpected(r.error()); + + // X_new = a*X + B @ X + // 就地复用：BX 是 matmul 新输出，axpy_inplace 就地添加 a*X + auto BX = engine.matmul(*A, *X, false, false); + if (!BX) return std::unexpected(BX.error()); + r = engine.axpy_inplace(*BX, a, *X); // BX += a*X → (B + aI)*X + if (!r) return std::unexpected(r.error()); + + X = std::move(*BX); + } + + return *X; +} + +// ══════════════════════════════════════════════════════════════════════════ +// Muon — MomentUm Orthogonalized by Newton-Schulz +// +// 算法（Keller Jordan et al., 2024）： +// 对于每个 2D 参数 p： +// 1. SGD-Momentum: v = μ*v + g +// 2. Newton-Schulz 正交化: update = NS5(v) // 将更新矩阵正交化 +// 3. 参数更新: p -= lr * update +// +// 设计要点： +// - 仅对 ≥2D 的参数（权重矩阵）应用 Muon +// - 嵌入层、分类头、bias/gain 应使用 AdamW +// - Nesterov 动量可略微提升性能 +// +// 参考： +// - https://kellerjordan.github.io/posts/muon/ +// - https://github.com/KellerJordan/Muon +// ══════════════════════════════════════════════════════════════════════════ +class Muon : public Optimizer +{ + Scalar lr_; + Scalar momentum_; + bool nesterov_; + std::size_t ns_steps_; + Scalar ns_eps_; + std::vector velocities_; + +public: + Muon(ComputeEngine& engine, + std::vector params, + std::vector grads, + Scalar lr, + Scalar momentum = 0.95f, + bool nesterov = true, + std::size_t ns_steps = 5, + Scalar ns_eps = 1e-7f) + : Optimizer(engine, std::move(params), std::move(grads)), + lr_(lr), momentum_(momentum), nesterov_(nesterov), + ns_steps_(ns_steps), ns_eps_(ns_eps), + velocities_(create_zero_buffers_()) {} + + void set_lr(Scalar lr) override { lr_ = lr; } + + [[nodiscard]] Result step() override + { + if (auto r = validate_sizes_(); !r) return std::unexpected(r.error()); + + for (std::size_t i = 0; i < params_.size(); ++i) + { + const Tensor& g = grads_[i]; + + // 1. SGD-Momentum: v = μ*v + g + auto r = engine_.scale_inplace(velocities_[i], momentum_); + if (!r) return std::unexpected(r.error()); + r = engine_.add_inplace(velocities_[i], g); + if (!r) return std::unexpected(r.error()); + + // 确定用于正交化的更新方向 + Tensor* update_ptr = nullptr; + std::optional nesterov_buf; + if (nesterov_) + { + // Nesterov: update = g + μ*v + // 优化：clone g 一次，用 axpy_inplace 就地添加 μ*v（原实现需要 2 次 clone + scale + add） + auto buf = clone_tensor(engine_, g); + if (!buf) return std::unexpected(buf.error()); + r = engine_.axpy_inplace(*buf, momentum_, velocities_[i]); + if (!r) return std::unexpected(r.error()); + nesterov_buf = std::move(*buf); + update_ptr = &*nesterov_buf; + } + else + { + update_ptr = &velocities_[i]; + } + + // 2. Newton-Schulz 正交化（仅对 ≥2D 参数，即 rows > 1 且 cols > 1） + if (params_[i].get().rows() > 1 && params_[i].get().cols() > 1) + { + auto ortho_update = newton_schulz_orthogonalize( + engine_, *update_ptr, ns_steps_, ns_eps_); + if (!ortho_update) return std::unexpected(ortho_update.error()); + + // 3. 参数更新: p -= lr * ortho_update（用 axpy_inplace 融合 scale+add） + r = engine_.axpy_inplace(params_[i], -lr_, *ortho_update); + if (!r) return std::unexpected(r.error()); + } + else + { + // 非 2D 参数（bias 等）：标准 SGD 更新（用 axpy_inplace 融合 scale+add） + r = engine_.axpy_inplace(params_[i], -lr_, *update_ptr); + if (!r) return std::unexpected(r.error()); + } + } + return {}; + } +}; + +// ── 优化器工厂函数 ──────────────────────────────────────────────────────── +// 根据名称创建对应优化器，支持: "sgd", "sgd_momentum", "adam", "adamw", "muon"。 +// 未知名称返回 nullptr，由调用方处理。 +[[nodiscard]] inline std::unique_ptr create_optimizer( + std::string_view name, + ComputeEngine& engine, + std::vector params, + std::vector grads, + Scalar lr, + Scalar weight_decay = 0) +{ + if (name == "sgd") + return std::make_unique(engine, std::move(params), std::move(grads), lr); + if (name == "sgd_momentum") + return std::make_unique(engine, std::move(params), std::move(grads), lr); + if (name == "adam") + return std::make_unique(engine, std::move(params), std::move(grads), lr); + if (name == "adamw") + return std::make_unique(engine, std::move(params), std::move(grads), lr, + 0.9, 0.999, 1e-8, weight_decay); + if (name == "muon") + return std::make_unique(engine, std::move(params), std::move(grads), lr); + return nullptr; // 未知名称，调用方应处理 +} + +} // namespace nn + +#endif // NN_COMPUTE_OPTIMIZER_HPP diff --git a/include/neuralnet.cpp/config.hpp b/include/neuralnet.cpp/config.hpp new file mode 100644 index 00000000..f3d9a914 --- /dev/null +++ b/include/neuralnet.cpp/config.hpp @@ -0,0 +1,169 @@ +#ifndef NN_CONFIG_HPP +#define NN_CONFIG_HPP + +#include +#include // for std::distance +#include // for std::thread::hardware_concurrency + +#include "core_threadpool.hpp" +#include "core_errors.hpp" + +// ── 缓存分块大小 ───────────────────────────────────────────────────────────── +// 64×64×8 = 32 KB，安全装入大多数 CPU 的 L1 缓存 +// 矩阵乘法、转置等所有分块操作共用此值 +// 修改时需同步评估 b_block 栈占用（BLOCK_SIZE² × 8 字节） +namespace nn +{ + // ── 标量类型（float 加速 / double 精度） ───────────────────────── + using Scalar = float; + + inline constexpr std::size_t BLOCK_SIZE = 64; + static_assert(BLOCK_SIZE * BLOCK_SIZE * sizeof(Scalar) <= 65536, + "BLOCK_SIZE too large: b_block would exceed 64KB stack budget"); + // ── 数值常量 ───────────────────────────────────────────────────────────── + // 使用 constexpr 避免运行时计算 + inline constexpr Scalar EPSILON = 1e-6f; + + // ── 并行阈值 ───────────────────────────────────────────────────────────── + // 元素数 >= 此值时启用线程池并行；低于则串行执行（避免调度开销）。 + // 线程池调度开销约 50-200μs（Windows mutex+cv），仅当计算量足够时并行才有收益。 + // + // bench_thresholds 实测结果（32 核 CPU, Release -O3, 2026-07-25）： + // 元素数 串行μs 并行μs 加速比 建议 + // 16384 9.6 9.8 0.98x 无差别 + // 32768 18.8 19.9 0.94x 无差别 + // 65536 38.7 121.7 0.32x 串行优（调度开销显现） + // 131072 76.4 182.8 0.42x 串行优 + // 262144 151.6 196.0 0.77x 串行优 + // 524288 308.7 201.5 1.53x ← 首次稳定 > 1.5x + // 1048576 620.3 186.7 3.32x 并行优（稳定） + // 2097152 1239.5 217.3 5.70x 并行优 + // 4194304 2657.3 340.6 7.80x 并行优 + // 取 524288 (512K) 作为阈值：在加速比首次稳定 > 1.5x 的位置。 + // 注：1024-4096 元素数测出的高加速比是亚微秒级测量噪声，不可靠。 + inline constexpr std::size_t PARALLEL_THRESHOLD = 524288; + + // ── 顶层并行算法函数 ───────────────────────────────────────────────────── + // 用法与 std::for_each / std::transform 完全一致，内部自动选择串行/并行。 + // 替代 std::execution::par 的线程创建开销，也替代手写 for 循环。 + + // for_each + template + inline void for_each(Iterator first, Iterator last, Func&& func) + { + const auto n = static_cast(std::distance(first, last)); + if (n >= PARALLEL_THRESHOLD) + { + global_thread_pool().parallel_for_each(first, last, std::forward(func)); + } + else + { + for (auto it = first; it != last; ++it) + func(*it); + } + } + + // transform（一元） + template + inline void transform(InputIt first, InputIt last, OutputIt d_first, UnaryOp&& op) + { + const auto n = static_cast(std::distance(first, last)); + if (n >= PARALLEL_THRESHOLD) + { + global_thread_pool().parallel_transform(first, last, d_first, std::forward(op)); + } + else + { + for (; first != last; ++first, ++d_first) + *d_first = op(*first); + } + } + + // transform（二元） + template + inline void transform(InputIt1 first1, InputIt1 last1, InputIt2 first2, + OutputIt d_first, BinaryOp&& op) + { + const auto n = static_cast(std::distance(first1, last1)); + if (n >= PARALLEL_THRESHOLD) + { + global_thread_pool().parallel_transform(first1, last1, first2, d_first, + std::forward(op)); + } + else + { + for (; first1 != last1; ++first1, ++first2, ++d_first) + *d_first = op(*first1, *first2); + } + } + + // transform_reduce（一元 transform） + template + inline T transform_reduce(InputIt first, InputIt last, T init, + BinaryReduceOp&& reduce_op, UnaryTransformOp&& transform_op) + { + const auto n = static_cast(std::distance(first, last)); + if (n >= PARALLEL_THRESHOLD) + { + return global_thread_pool().parallel_transform_reduce( + first, last, init, + std::forward(reduce_op), + std::forward(transform_op)); + } + else + { + T acc = init; + for (; first != last; ++first) + acc = reduce_op(acc, transform_op(*first)); + return acc; + } + } + + // transform_reduce（二元 transform） + template + inline T transform_reduce(InputIt1 first1, InputIt1 last1, InputIt2 first2, + T init, BinaryReduceOp&& reduce_op, + BinaryTransformOp&& transform_op) + { + const auto n = static_cast(std::distance(first1, last1)); + if (n >= PARALLEL_THRESHOLD) + { + return global_thread_pool().parallel_transform_reduce( + first1, last1, first2, init, + std::forward(reduce_op), + std::forward(transform_op)); + } + else + { + T acc = init; + for (; first1 != last1; ++first1, ++first2) + acc = reduce_op(acc, transform_op(*first1, *first2)); + return acc; + } + } + + // parallel_for_blocks（块级并行，无阈值保护） + template + inline void parallel_for_blocks(Iterator first, Iterator last, Func&& func) + { + global_thread_pool().parallel_for_blocks(first, last, std::forward(func)); + } + + // parallel_for_samples（样本级并行） + template + inline void parallel_for_samples(std::size_t num_samples, Func&& func) + { + if (num_samples <= 1) + { + for (std::size_t i = 0; i < num_samples; ++i) + func(i); + } + else + { + global_thread_pool().parallel_for_samples(num_samples, std::forward(func)); + } + } +} // namespace nn + +#endif // NN_CONFIG_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/core_assert.hpp b/include/neuralnet.cpp/core_assert.hpp new file mode 100644 index 00000000..9fd603e3 --- /dev/null +++ b/include/neuralnet.cpp/core_assert.hpp @@ -0,0 +1,57 @@ +#ifndef NN_CORE_ASSERT_HPP +#define NN_CORE_ASSERT_HPP + +#include +#include +#include +#include "core_errors.hpp" + +namespace nn { + +// ── 调试断言宏（带位置信息）──────────────────────────────────────────── +// 用于内部不变量检查，生产环境可通过 NDEBUG 禁用 +#ifdef NDEBUG + #define NN_ASSERT(cond, msg) ((void)0) +#else + #define NN_ASSERT(cond, msg) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "NN_ASSERT failed at %s:%d: %s\n", \ + __FILE__, __LINE__, (msg)); \ + std::abort(); \ + } \ + } while(0) +#endif + +// ── 运行时检查宏（始终启用）─────────────────────────────────────────── +// 用于前置条件校验，返回 Error 而非 abort +#define NN_REQUIRE(cond, msg) \ + do { \ + if (!(cond)) { \ + return std::unexpected(nn::Error{(msg)}); \ + } \ + } while(0) + +// ── 热点函数标记（提示编译器优化）───────────────────────────────────── +// 用于高频调用的前向/反向传播函数 +#if defined(__GNUC__) || defined(__clang__) + #define NN_HOT_FN [[gnu::hot]] [[gnu::always_inline]] +#elif defined(_MSC_VER) + #define NN_HOT_FN __forceinline +#else + #define NN_HOT_FN inline +#endif + +// ── 分支预测提示 ───────────────────────────────────────────────────── +// 用于罕见错误路径，帮助编译器优化指令流水线 +#if defined(__GNUC__) || defined(__clang__) + #define NN_LIKELY(x) (__builtin_expect(!!(x), 1)) + #define NN_UNLIKELY(x) (__builtin_expect(!!(x), 0)) +#else + #define NN_LIKELY(x) (x) + #define NN_UNLIKELY(x) (x) +#endif + +} // namespace nn + +#endif // NN_CORE_ASSERT_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/core_errors.hpp b/include/neuralnet.cpp/core_errors.hpp new file mode 100644 index 00000000..76c3907b --- /dev/null +++ b/include/neuralnet.cpp/core_errors.hpp @@ -0,0 +1,61 @@ +#ifndef NN_CORE_ERRORS_HPP +#define NN_CORE_ERRORS_HPP + +#include +#include +#include +#include +#include +#include + +namespace nn { + +// ── 错误类型（C++23 std::expected）───────────────────────────────────── +// 所有公共 API 使用 Result 返回错误，不抛异常。 +struct Error { + std::string message; +}; + +template +using Result = std::expected; + +// ── [[nodiscard]] 包装：让丢弃 Result 的调用在编译期告警 ────────────── +// 用法：auto r = nn::check_result(foo()); // 若丢弃 r 会告警 +// 注：Result 本身是 std::expected 的别名，无法在别名声明上加属性， +// 此处提供显式 nodiscard 包装用于关键路径。 +template +[[nodiscard]] constexpr Result check_result(Result r) noexcept +{ + return r; +} + +// ── 数字解析（std::from_chars，不抛异常） ──────────────────────────── +// 替代 std::stoi/stod/stoul/stoull 等会抛异常的函数。 +// 用法：auto v = nn::parse_number("123"); +// if (!v) return std::unexpected(v.error()); +template + requires std::is_arithmetic_v +[[nodiscard]] inline Result parse_number(std::string_view s) noexcept +{ + // 去除首尾空白（from_chars 不容忍前后空白） + while (!s.empty() && (s.front() == ' ' || s.front() == '\t' || + s.front() == '\n' || s.front() == '\r')) + s.remove_prefix(1); + while (!s.empty() && (s.back() == ' ' || s.back() == '\t' || + s.back() == '\n' || s.back() == '\r')) + s.remove_suffix(1); + + if (s.empty()) + return std::unexpected(Error{"empty numeric string"}); + + T value{}; + const auto *end = s.data() + s.size(); + auto [ptr, ec] = std::from_chars(s.data(), end, value); + if (ec != std::errc{} || ptr != end) + return std::unexpected(Error{"invalid number: " + std::string(s)}); + return value; +} + +} // namespace nn + +#endif // NN_CORE_ERRORS_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/core_file.hpp b/include/neuralnet.cpp/core_file.hpp new file mode 100644 index 00000000..10ed5f1b --- /dev/null +++ b/include/neuralnet.cpp/core_file.hpp @@ -0,0 +1,35 @@ +#ifndef NN_CORE_FILE_HPP +#define NN_CORE_FILE_HPP + +// ═══════════════════════════════════════════════════════════════════════════ +// core_file.hpp — 文件读取工具 +// +// 职责：提供通用的文件读取函数，消除各模块间的重复代码。 +// 设计： +// - 所有函数返回 Result，失败时返回 Error 错误信息 +// - 使用 std::istreambuf_iterator 读取，避免 tellg 失败风险 +// ═══════════════════════════════════════════════════════════════════════════ + +#include +#include +#include + +#include "core_errors.hpp" + +namespace nn +{ + +// ── 读取文本文件（UTF-8） ────────────────────────────────────────────── +// 使用 istreambuf_iterator 读取全部内容，无需 tellg，避免失败场景。 +[[nodiscard]] inline Result load_text_file(const std::string &path) +{ + std::ifstream ifs(path, std::ios::binary); + if (!ifs) + return std::unexpected(Error{"Cannot open text file: " + path}); + return std::string{std::istreambuf_iterator(ifs), + std::istreambuf_iterator()}; +} + +} // namespace nn + +#endif // NN_CORE_FILE_HPP diff --git a/include/neuralnet.cpp/core_observer_ptr.hpp b/include/neuralnet.cpp/core_observer_ptr.hpp new file mode 100644 index 00000000..129a4fd4 --- /dev/null +++ b/include/neuralnet.cpp/core_observer_ptr.hpp @@ -0,0 +1,80 @@ +// ── core_observer_ptr.hpp — 非拥有型观察者指针 ────────────────────────── +// +// observer_ptr 是对 T* 的类型安全封装，明确表达"观察但不拥有"语义。 +// 等价于 C++26 std::observer_ptr 的轻量替代品（无需 libc++ 支持）。 +// +// 设计原则： +// - 零开销：与裸指针相同的内存布局和性能 +// - 可空：默认构造为 nullptr，支持 if (ptr) 检查 +// - 类型自文档化：observer_ptr 明确声明"不负责释放" +// - 不提供 RAII 语义：生命周期管理由外部负责 +// +// 使用场景： +// - 类成员：持有非拥有的引擎/池引用（如 Model::engine_, GpuBuffer::pool_） +// - 函数参数：表示可选的观察者输入 +// - 返回值：表示可能为空的查询结果 +// ───────────────────────────────────────────────────────────────────────── + +#ifndef NN_CORE_OBSERVER_PTR_HPP +#define NN_CORE_OBSERVER_PTR_HPP + +#include +#include + +namespace nn +{ + +template +class observer_ptr +{ + T* ptr_ = nullptr; + +public: + // ── 构造 ───────────────────────────────────────────────────────────── + constexpr observer_ptr() noexcept = default; + constexpr observer_ptr(std::nullptr_t) noexcept : ptr_(nullptr) {} + constexpr explicit observer_ptr(T& obj) noexcept : ptr_(&obj) {} + constexpr explicit observer_ptr(T* ptr) noexcept : ptr_(ptr) {} + + // 从另一个 observer_ptr 转换（允许隐式向上转型） + template + requires std::is_convertible_v + constexpr observer_ptr(const observer_ptr& other) noexcept : ptr_(other.get()) {} + + // ── 访问 ───────────────────────────────────────────────────────────── + [[nodiscard]] constexpr T& operator*() const noexcept { return *ptr_; } + [[nodiscard]] constexpr T* operator->() const noexcept { return ptr_; } + [[nodiscard]] constexpr T* get() const noexcept { return ptr_; } + + // ── 布尔转换 ───────────────────────────────────────────────────────── + [[nodiscard]] constexpr explicit operator bool() const noexcept { return ptr_ != nullptr; } + + // ── 修改 ───────────────────────────────────────────────────────────── + constexpr void reset(T* p = nullptr) noexcept { ptr_ = p; } + constexpr void swap(observer_ptr& other) noexcept { std::swap(ptr_, other.ptr_); } + + // ── 比较 ───────────────────────────────────────────────────────────── + friend constexpr bool operator==(const observer_ptr&, const observer_ptr&) = default; + friend constexpr bool operator==(const observer_ptr& p, std::nullptr_t) noexcept { return !p; } + friend constexpr auto operator<=>(const observer_ptr& a, const observer_ptr& b) noexcept + { + return a.ptr_ <=> b.ptr_; + } +}; + +// ── 便捷工厂函数 ─────────────────────────────────────────────────────── +template +[[nodiscard]] constexpr observer_ptr make_observer(T& obj) noexcept +{ + return observer_ptr(obj); +} + +template +[[nodiscard]] constexpr observer_ptr make_observer(T* ptr) noexcept +{ + return observer_ptr(ptr); +} + +} // namespace nn + +#endif // NN_CORE_OBSERVER_PTR_HPP diff --git a/include/neuralnet.cpp/thread_pool.hpp b/include/neuralnet.cpp/core_threadpool.hpp similarity index 80% rename from include/neuralnet.cpp/thread_pool.hpp rename to include/neuralnet.cpp/core_threadpool.hpp index d474c81e..aa47a6d9 100644 --- a/include/neuralnet.cpp/thread_pool.hpp +++ b/include/neuralnet.cpp/core_threadpool.hpp @@ -1,5 +1,5 @@ -#ifndef THREAD_POOL_HPP -#define THREAD_POOL_HPP +#ifndef NN_CORE_THREAD_POOL_HPP +#define NN_CORE_THREAD_POOL_HPP #include #include @@ -12,8 +12,10 @@ #include // for std::transform (serial fallback) #include // for std::transform_reduce (serial fallback) #include // for std::distance -#include #include +#include + +#include "core_assert.hpp" namespace nn { @@ -83,11 +85,11 @@ namespace nn private: // ── 分块辅助：计算合理的分块数 ────────────────────────────────── - // 每分块至少 4096 个 double（32 KB），可装入 L1 缓存。 - // 256→4096：旧值导致 8192 元素被切成 16 个微块，调度开销远超计算。 + // 每分块至少 1024 个元素（Scalar=float 时约 4 KB），可装入 L1 缓存。 + // 4096→1024：降低阈值使 MNIST 小隐藏层（64×batch）也能触发多核并行。 [[nodiscard]] std::size_t chunk_count(std::size_t total) const noexcept { - constexpr std::size_t MIN_CHUNK = 4096; + constexpr std::size_t MIN_CHUNK = 1024; const auto nw = workers_.size(); if (nw <= 1 || total < MIN_CHUNK * 2) return 1; @@ -96,13 +98,32 @@ namespace nn } // ── work-stealing 等待：调用者不空转，帮忙处理队列任务 ───────── + // 优化（依据性能审查报告）： + // - 旧实现：spin 64 次 + yield，64 次 spin 中反复原子读取消耗电量 + // CPU 占用率显示 100% 但实际有效计算比例低（调用者空转） + // - 新实现： + // 1) 短自旋（16 次 pause）快速检测 latch 归零——典型情况无 yield 开销 + // 2) 自旋失败后 try_work_steal：尝试从队列取任务执行（参与计算） + // 3) 队列为空时进入 condition_variable 等待，避免 CPU 空转 + // （cv.wait_until 短超时 100μs，确保不睡过太久） void wait_for_latch(std::atomic& latch) { + // 阶段 1：短自旋（16 次 pause）——应对 latch 即将归零的快路径 + for (int spin = 0; spin < 16; ++spin) + { + if (latch.load(std::memory_order_acquire) == 0) + return; +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) + __builtin_ia32_pause(); +#endif + } + + // 阶段 2：work-stealing + cv 等待 while (latch.load(std::memory_order_acquire) > 0) { std::function task; { - std::lock_guard lock(queue_mutex_); + std::unique_lock lock(queue_mutex_); if (!tasks_.empty()) { task = std::move(tasks_.front()); @@ -110,9 +131,28 @@ namespace nn } } if (task) + { task(); + // 执行完一个任务后回到阶段 1 短自旋 + for (int spin = 0; spin < 16; ++spin) + { + if (latch.load(std::memory_order_acquire) == 0) + return; +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) + __builtin_ia32_pause(); +#endif + } + } else - std::this_thread::yield(); + { + // 队列为空：用 cv.wait_for 短超时等待，避免纯 yield 反复调度 + // 100μs 超时确保即使没任务也能及时唤醒检查 latch + std::unique_lock lock(queue_mutex_); + condition_.wait_for(lock, std::chrono::microseconds(100), + [&latch]() { + return latch.load(std::memory_order_acquire) == 0; + }); + } } } @@ -233,6 +273,57 @@ namespace nn wait_for_latch(latch); } + // ── 并行 for_samples（独立样本级并行）─────────────────────────── + // 与 for_each（按元素分块）不同：每个"样本"包含一个完整的计算子任务 + // （如矩阵乘法、前向传播），样本间完全独立、无数据竞争。 + // 用于 GPT/Transformer batch 维度的并行：1 样本 = 1 分片。 + template + void parallel_for_samples(std::size_t num_samples, Func&& func) + { + if (num_samples <= 1) + { + for (std::size_t i = 0; i < num_samples; ++i) + func(i); + return; + } + + const auto n_chunks = std::min(workers_.size(), num_samples); + const std::size_t base = num_samples / n_chunks; + const std::size_t rem = num_samples % n_chunks; + std::atomic latch{static_cast(n_chunks)}; + + { + std::lock_guard lock(queue_mutex_); + std::size_t off = 0; + for (std::size_t c = 0; c < n_chunks - 1; ++c) + { + const std::size_t len = base + (c < rem ? 1 : 0); + const std::size_t start = off; + const std::size_t end = off + len; + off += len; + + tasks_.emplace([start, end, &func, &latch]() + { + for (std::size_t i = start; i < end; ++i) + func(i); + latch.fetch_sub(1, std::memory_order_release); + }); + } + } + condition_.notify_all(); + + // 调用者处理最后一个分片 + { + const std::size_t c = n_chunks - 1; + const std::size_t start = num_samples - (base + (c < rem ? 1 : 0)); + for (std::size_t i = start; i < num_samples; ++i) + func(i); + latch.fetch_sub(1, std::memory_order_release); + } + + wait_for_latch(latch); + } + // ── 并行 transform（一元）─────────────────────────────────────── template void parallel_transform(InputIt first, InputIt last, OutputIt d_first, UnaryOp&& op) @@ -392,9 +483,9 @@ namespace nn std::advance(end, static_cast(len)); off += len; - tasks_.emplace([beg, end, &reduce_op, &transform_op, &partials, &latch, c]() + tasks_.emplace([beg, end, &reduce_op, &transform_op, &partials, &latch, c, init]() { - T local{}; // 要求 T{} 是 reduce_op 的单位元 + T local = init; // 以调用者 init 为单位元（不能用 T{}） for (auto it = beg; it != end; ++it) local = reduce_op(local, transform_op(*it)); partials[c] = std::move(local); @@ -409,7 +500,7 @@ namespace nn const std::size_t off = total - (base + (c < rem ? 1 : 0)); auto beg = first; std::advance(beg, static_cast(off)); - T local{}; + T local = init; // 以调用者 init 为单位元（不能用 T{}） for (auto it = beg; it != last; ++it) local = reduce_op(local, transform_op(*it)); partials[c] = std::move(local); @@ -459,9 +550,9 @@ namespace nn std::advance(i1_end, static_cast(len)); off += len; - tasks_.emplace([i1, i1_end, i2, &reduce_op, &transform_op, &partials, &latch, c]() + tasks_.emplace([i1, i1_end, i2, &reduce_op, &transform_op, &partials, &latch, c, init]() { - T local{}; + T local = init; // 以调用者 init 为单位元（不能用 T{}） auto it1 = i1; auto it2 = i2; for (; it1 != i1_end; ++it1, ++it2) @@ -480,7 +571,7 @@ namespace nn auto i2 = first2; std::advance(i1, static_cast(off)); std::advance(i2, static_cast(off)); - T local{}; + T local = init; // 以调用者 init 为单位元（不能用 T{}） for (; i1 != last1; ++i1, ++i2) local = reduce_op(local, transform_op(*i1, *i2)); partials[c] = std::move(local); @@ -521,6 +612,7 @@ namespace nn static ThreadPool pool{std::thread::hardware_concurrency()}; return pool; } + } // namespace nn -#endif // THREAD_POOL_HPP \ No newline at end of file +#endif // NN_CORE_THREAD_POOL_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/cpu_engine.hpp b/include/neuralnet.cpp/cpu_engine.hpp new file mode 100644 index 00000000..567531b3 --- /dev/null +++ b/include/neuralnet.cpp/cpu_engine.hpp @@ -0,0 +1,671 @@ +#ifndef NN_CPU_ENGINE_HPP +#define NN_CPU_ENGINE_HPP + +// ── cpu_engine.hpp — CPU 计算引擎实现 ───────────────────────────────────── +// CpuEngine 封装现有 Matrix 方法和 AST（compute::apply），实现 +// ComputeEngine 接口。所有操作同步执行，begin_batch/end_batch 为 no-op。 +// +// 逐元素运算通过 switch 分发到编译期 AST，保持零开销抽象。 +// ───────────────────────────────────────────────────────────────────────── + +#include +#include +#include + +#include "compute_engine.hpp" +#include "algebra_compute.hpp" +#include "algebra_expr.hpp" + +namespace nn +{ + +// ══════════════════════════════════════════════════════════════════════════ +// CpuEngine — CPU 计算引擎 +// ══════════════════════════════════════════════════════════════════════════ +class CpuEngine final : public ComputeEngine +{ +public: + [[nodiscard]] Device device() const noexcept override { return Device::CPU; } + + // ── 批处理：CPU 为 no-op（同步执行） ────────────────────────────────── + [[nodiscard]] Result begin_batch() override { return {}; } + [[nodiscard]] Result end_batch() override { return {}; } + [[nodiscard]] Result flush_batch() override { return {}; } + + // ── 张量工厂 ────────────────────────────────────────────────────────── + [[nodiscard]] Tensor create_tensor(std::size_t rows, std::size_t cols) override + { + return Tensor::cpu(rows, cols); + } + + [[nodiscard]] Result from_matrix(const Matrix& m) override + { + return Tensor::from_matrix(Matrix(m)); // 拷贝，避免外部修改影响 + } + + [[nodiscard]] Result to_matrix(const Tensor& t) override + { + if (!t.is_cpu()) + return std::unexpected(Error{"to_matrix: tensor is not CPU"}); + return Matrix(t.cpu_matrix()); // 返回拷贝 + } + + [[nodiscard]] Result copy_from(Tensor& dst, const Matrix& src) override + { + if (!dst.is_cpu()) + return std::unexpected(Error{"copy_from: dst tensor is not CPU"}); + if (dst.rows() != src.rows() || dst.cols() != src.cols()) + return std::unexpected(Error{"copy_from: shape mismatch"}); + dst.cpu_matrix() = src; // 拷贝赋值 + return {}; + } + + [[nodiscard]] Result clone(const Tensor& src) override + { + if (!src.is_cpu()) + return std::unexpected(Error{"clone: src tensor is not CPU"}); + return Tensor::from_matrix(Matrix(src.cpu_matrix())); // 深拷贝 + } + + // ── 行切片：拷贝 src 的行 [start_row, start_row + count) 到新 Tensor ── + [[nodiscard]] Result slice_rows( + const Tensor& src, std::size_t start_row, std::size_t count) override + { + if (!src.is_cpu()) + return std::unexpected(Error{"slice_rows: src tensor is not CPU"}); + const Matrix& m = src.cpu_matrix(); + if (start_row + count > m.rows()) + return std::unexpected(Error{"slice_rows: range out of bounds"}); + + Matrix result(count, m.cols()); + const auto src_span = m.span(); + auto dst_span = result.span(); + const std::size_t cols = m.cols(); + for (std::size_t r = 0; r < count; ++r) + std::copy_n(src_span.begin() + (start_row + r) * cols, cols, + dst_span.begin() + r * cols); + return Tensor::from_matrix(std::move(result)); + } + + // ── 行插入：将 src 的所有行写入 dst 的行 [dst_start_row, ...) ── + [[nodiscard]] Result insert_rows( + Tensor& dst, std::size_t dst_start_row, const Tensor& src) override + { + if (!dst.is_cpu() || !src.is_cpu()) + return std::unexpected(Error{"insert_rows: tensors are not CPU"}); + if (dst.cols() != src.cols()) + return std::unexpected(Error{"insert_rows: column count mismatch"}); + if (dst_start_row + src.rows() > dst.rows()) + return std::unexpected(Error{"insert_rows: range out of bounds"}); + + Matrix& d = dst.cpu_matrix(); + const Matrix& s = src.cpu_matrix(); + const auto dst_span = d.span(); + const auto src_span = s.span(); + const std::size_t cols = d.cols(); + for (std::size_t r = 0; r < s.rows(); ++r) + std::copy_n(src_span.begin() + r * cols, cols, + dst_span.begin() + (dst_start_row + r) * cols); + return {}; + } + + // ── gather_rows: 按 indices 从 table 中按行查表 ── + // table: (vocab, D), indices: 任意形状，按 flat 遍历所有元素 + // 输出: (num_indices, D)，out[i] = table[flat_indices[i]] + // 越界索引返回零行（防御性，不抛错） + // + // 并行策略：每个输出行独立，按行切分到线程。 + // - num * D >= PARALLEL_THRESHOLD 时启用行块并行 + // - 否则串行（小批量查表的 D 通常较小，并行调度开销不划算） + // - 行块并行避免跨行数据竞争（每行独立写入） + [[nodiscard]] Result gather_rows( + const Tensor& table, const Tensor& indices) override + { + if (!table.is_cpu() || !indices.is_cpu()) + return std::unexpected(Error{"gather_rows: tensors are not CPU"}); + + const Matrix& tbl = table.cpu_matrix(); + const Matrix& idx = indices.cpu_matrix(); + const std::size_t vocab = tbl.rows(); + const std::size_t D = tbl.cols(); + const std::size_t num = idx.size(); // 遍历 indices 所有元素（支持任意形状） + + Matrix result(num, D); + auto dst_span = result.span(); + const auto tbl_span = tbl.span(); + const auto idx_span = idx.span(); + + const std::size_t total = num * D; + if (total >= PARALLEL_THRESHOLD && num > 1) + { + // 行块并行：每行独立查表，无数据竞争 + auto row_indices = std::views::iota(std::size_t{0}, num); + nn::for_each(row_indices.begin(), row_indices.end(), + [dst_span, tbl_span, idx_span, vocab, D](std::size_t i) noexcept { + const auto row_idx = static_cast(idx_span[i]); + if (row_idx < vocab) + { + std::copy_n(tbl_span.begin() + row_idx * D, D, + dst_span.begin() + i * D); + } + else + { + std::fill_n(dst_span.begin() + i * D, D, Scalar{0}); + } + }); + } + else + { + // 串行路径 + for (std::size_t i = 0; i < num; ++i) + { + const auto row_idx = static_cast(idx_span[i]); + if (row_idx < vocab) + { + std::copy_n(tbl_span.begin() + row_idx * D, D, + dst_span.begin() + i * D); + } + else + { + std::fill_n(dst_span.begin() + i * D, D, Scalar{0}); + } + } + } + return Tensor::from_matrix(std::move(result)); + } + + // ── scatter_add_rows: 按 indices 把 grad 的行原子累加到 dst ── + // dst: (vocab, D) 原地修改, indices: 任意形状（按 flat 遍历）, grad: (num_indices, D) + // 语义: dst[flat_indices[i]] += grad[i] + [[nodiscard]] Result scatter_add_rows( + Tensor& dst, const Tensor& indices, const Tensor& grad) override + { + if (!dst.is_cpu() || !indices.is_cpu() || !grad.is_cpu()) + return std::unexpected(Error{"scatter_add_rows: tensors are not CPU"}); + if (dst.cols() != grad.cols()) + return std::unexpected(Error{"scatter_add_rows: column count mismatch"}); + + Matrix& d = dst.cpu_matrix(); + const Matrix& idx = indices.cpu_matrix(); + const Matrix& g = grad.cpu_matrix(); + const std::size_t vocab = d.rows(); + const std::size_t D = d.cols(); + const std::size_t num = idx.size(); // 遍历 indices 所有元素（支持任意形状） + + auto dst_span = d.span(); + const auto idx_span = idx.span(); + const auto grad_span = g.span(); + + for (std::size_t i = 0; i < num; ++i) + { + const auto row_idx = static_cast(idx_span[i]); + if (row_idx < vocab) + { + auto dst_row = dst_span.subspan(row_idx * D, D); + const auto grad_row = grad_span.subspan(i * D, D); + for (std::size_t c = 0; c < D; ++c) + dst_row[c] += grad_row[c]; + } + // 越界索引忽略 + } + return {}; + } + + // ── 3D 维度转置：(M, B, N) ↔ (B, M, N) ── + // inverse=false: (M, B*N) → (B*M, N), out[b*M+m, n] = in[m, b*N+n] + // inverse=true: (B*M, N) → (M, B*N), out[m, b*N+n] = in[b*M+m, n] + // + // 并行策略：按 (b, m) 二维块切分，每块独立 N 元素拷贝，无数据竞争。 + // - total >= PARALLEL_THRESHOLD 时启用块级并行 + // - 每块仅一次 std::copy_n(N)，cache 友好 + // - 典型场景：MHA 中 d_model=128, batch=32, seq=256 → 1M 元素，明显受益 + [[nodiscard]] Result rearrange_3d( + const Tensor& x, std::size_t M, std::size_t B, std::size_t N, + bool inverse) override + { + if (x.is_gpu()) + return std::unexpected(Error{"CpuEngine: GPU tensor on CPU engine"}); + + const Matrix& m = x.cpu_matrix(); + const std::size_t total = M * B * N; + if (m.size() != total) + return std::unexpected(Error{"rearrange_3d: element count mismatch"}); + + Matrix out(inverse ? M : (B * M), inverse ? (B * N) : N); + const auto src = m.span(); + const auto dst = out.span(); + + // (b, m) 组合索引：bm_idx = b * M + m，总块数 = B * M + // 每块独立处理 N 个元素 + const std::size_t n_blocks = B * M; + auto block_kernel = [src, dst, M, B, N, inverse](std::size_t bm_idx) noexcept { + const std::size_t b = bm_idx / M; + const std::size_t mi = bm_idx % M; + if (!inverse) + { + // out[b*M + m, n] = in[m, b*N + n] + const std::size_t dst_off = (b * M + mi) * N; + const std::size_t src_off = mi * (B * N) + b * N; + std::copy_n(src.begin() + src_off, N, + dst.begin() + dst_off); + } + else + { + // out[m, b*N + n] = in[b*M + m, n] + const std::size_t src_off = (b * M + mi) * N; + const std::size_t dst_off = mi * (B * N) + b * N; + std::copy_n(src.begin() + src_off, N, + dst.begin() + dst_off); + } + }; + + if (total >= PARALLEL_THRESHOLD && n_blocks > 1) + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + nn::parallel_for_blocks(block_indices.begin(), block_indices.end(), + block_kernel); + } + else + { + for (std::size_t bm = 0; bm < n_blocks; ++bm) + block_kernel(bm); + } + + return Tensor::from_matrix(std::move(out)); + } + + // ── 矩阵转置：A (R, C) → out (C, R) ── + [[nodiscard]] Result transpose(const Tensor& A) override + { + if (A.is_gpu()) + return std::unexpected(Error{"CpuEngine: GPU tensor on CPU engine"}); + return Tensor::from_matrix(A.cpu_matrix().transpose()); + } + + // ══════════════════════════════════════════════════════════════════════ + // 矩阵级原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result matmul( + const Tensor& A, const Tensor& B, + bool transA, bool transB) override + { + if (A.is_gpu() || B.is_gpu()) + return std::unexpected(Error{"CpuEngine: GPU tensor on CPU engine"}); + + const Matrix& a = A.cpu_matrix(); + const Matrix& b = B.cpu_matrix(); + + // 维度校验（基于逻辑维度，避免不必要的转置拷贝） + // A_eff = transA ? A^T : A，B_eff = transB ? B^T : B + const std::size_t M = transA ? a.cols() : a.rows(); + const std::size_t K = transA ? a.rows() : a.cols(); + const std::size_t K2 = transB ? b.cols() : b.rows(); + const std::size_t N = transB ? b.rows() : b.cols(); + if (K != K2) + return std::unexpected(Error{"matmul: dimension mismatch A=" + + std::to_string(a.rows()) + "x" + std::to_string(a.cols()) + + " transA=" + (transA ? "1" : "0") + + " B=" + std::to_string(b.rows()) + "x" + std::to_string(b.cols()) + + " transB=" + (transB ? "1" : "0") + + " K=" + std::to_string(K) + " K2=" + std::to_string(K2)}); + + Matrix result(M, N); + // 使用 Matrix 原生转置 matmul 方法，零额外拷贝 + if (!transA && !transB) { + a.multiply_to(result, b); + } else if (!transA && transB) { + a.multiply_transposed_to(result, b); // C = A × B^T + } else if (transA && !transB) { + a.transpose_multiply_to(result, b); // C = A^T × B + } else { + // 双转置 C = A^T × B^T，罕见路径 + Matrix a_t = a.transpose(); + a_t.multiply_transposed_to(result, b); + } + return Tensor::from_matrix(std::move(result)); + } + + // ── 批量矩阵乘法：按 batch 切分行块，逐 batch 矩阵乘 ── + [[nodiscard]] Result batched_matmul( + const Tensor& A, const Tensor& B, + std::size_t batch, + bool transA, bool transB) override + { + if (A.is_gpu() || B.is_gpu()) + return std::unexpected(Error{"CpuEngine: GPU tensor on CPU engine"}); + if (batch == 0) + return std::unexpected(Error{"batched_matmul: batch must be > 0"}); + + const Matrix& a = A.cpu_matrix(); + const Matrix& b = B.cpu_matrix(); + + // 每个 batch 的行数 + if (a.rows() % batch != 0 || b.rows() % batch != 0) + return std::unexpected(Error{"batched_matmul: rows not divisible by batch"}); + + const std::size_t a_rows_per = a.rows() / batch; + const std::size_t b_rows_per = b.rows() / batch; + + // 逻辑维度（基于转置标志） + // transA=false: A_b 视为 (a_rows_per, a.cols())，M=a_rows_per, K=a.cols() + // transA=true: A_b 视为 (a.cols(), a_rows_per)（按 A_b^T 使用），M=a.cols(), K=a_rows_per + // transB=false: B_b 视为 (b_rows_per, b.cols())，K2=b_rows_per, N=b.cols() + // transB=true: B_b 视为 (b.cols(), b_rows_per)（按 B_b^T 使用），K2=b.cols(), N=b_rows_per + const std::size_t M = transA ? a.cols() : a_rows_per; + const std::size_t K = transA ? a_rows_per : a.cols(); + const std::size_t K2 = transB ? b.cols() : b_rows_per; + const std::size_t N = transB ? b_rows_per : b.cols(); + if (K != K2) + return std::unexpected(Error{"batched_matmul: K dimension mismatch"}); + + Matrix result(batch * M, N); + auto result_span = result.span(); + const auto a_span = a.span(); + const auto b_span = b.span(); + + for (std::size_t bi = 0; bi < batch; ++bi) + { + // 零拷贝：通过 span 子区间直接引用原始数据，无需临时 Matrix 拷贝 + const std::size_t a_off = bi * a_rows_per * a.cols(); + const std::size_t b_off = bi * b_rows_per * b.cols(); + auto a_sub = std::span(a_span.data() + a_off, a_rows_per * a.cols()); + auto b_sub = std::span(b_span.data() + b_off, b_rows_per * b.cols()); + auto c_sub = std::span(result_span.data() + bi * M * N, M * N); + + if (!transA && !transB) { + Matrix::multiply_to_span(c_sub, M, N, + a_sub, a_rows_per, a.cols(), + b_sub, b_rows_per, b.cols()); + } else if (!transA && transB) { + Matrix::multiply_transposed_to_span(c_sub, M, N, + a_sub, a_rows_per, a.cols(), + b_sub, b_rows_per, b.cols()); + } else if (transA && !transB) { + Matrix::transpose_multiply_to_span(c_sub, M, N, + a_sub, a_rows_per, a.cols(), + b_sub, b_rows_per, b.cols()); + } else { + // 双转置：先转置 A 子块，再做 A^T × B^T + // 等价于 (B × A)^T，用 span 暂不支持，fallback 到临时矩阵 + Matrix a_view(a_rows_per, a.cols()); + auto dst_a = a_view.span(); + std::copy_n(a_sub.begin(), a_sub.size(), dst_a.begin()); + Matrix a_t = a_view.transpose(); + Matrix b_view(b_rows_per, b.cols()); + auto dst_b = b_view.span(); + std::copy_n(b_sub.begin(), b_sub.size(), dst_b.begin()); + Matrix c_part(M, N); + a_t.multiply_transposed_to(c_part, b_view); + std::copy_n(c_part.span().begin(), M * N, c_sub.begin()); + } + } + + return Tensor::from_matrix(std::move(result)); + } + + [[nodiscard]] Result add_inplace(Tensor& A, const Tensor& B) override + { + if (A.rows() != B.rows() || A.cols() != B.cols()) + return std::unexpected(Error{"add_inplace: shape mismatch"}); + A.cpu_matrix().add_inplace(B.cpu_matrix()); + return {}; + } + + [[nodiscard]] Result scale_inplace(Tensor& A, Scalar s) override + { + A.cpu_matrix().scale_inplace(s); + return {}; + } + + // 融合 axpy：A += scalar * B（单次循环，避免 clone+scale+add 三步） + // 使用 nn::transform 二元并行版本，与 add_inplace/scale_inplace 一致： + // - n >= PARALLEL_THRESHOLD 时自动并行，否则串行 + // - 单次循环 + 内联 lambda，零开销抽象 + [[nodiscard]] Result axpy_inplace(Tensor& A, Scalar scalar, const Tensor& B) override + { + if (A.rows() != B.rows() || A.cols() != B.cols()) + return std::unexpected(Error{"axpy_inplace: shape mismatch"}); + auto& a = A.cpu_matrix(); + const auto& b = B.cpu_matrix(); + auto a_span = a.span(); + const auto b_span = b.span(); + nn::transform(a_span.begin(), a_span.end(), b_span.begin(), + a_span.begin(), + [scalar](Scalar x, Scalar y) noexcept { return x + scalar * y; }); + return {}; + } + + [[nodiscard]] Result zero(Tensor& A) override + { + A.cpu_matrix().zero(); + return {}; + } + + // ══════════════════════════════════════════════════════════════════════ + // 归约原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result row_reduce_sum(const Tensor& A) override + { + const Matrix& m = A.cpu_matrix(); + Matrix result = m.row_reduce(Scalar{0}, + [](Scalar a, Scalar b) noexcept { return a + b; }, + [](Scalar x) noexcept { return x; }); + return Tensor::from_matrix(std::move(result)); + } + + [[nodiscard]] Result col_reduce_sum(const Tensor& A) override + { + const Matrix& m = A.cpu_matrix(); + Matrix result = m.col_reduce(Scalar{0}, + [](Scalar a, Scalar b) noexcept { return a + b; }, + [](Scalar x) noexcept { return x; }); + return Tensor::from_matrix(std::move(result)); + } + + [[nodiscard]] Result row_reduce_max(const Tensor& A) override + { + const Matrix& m = A.cpu_matrix(); + Matrix result = m.row_reduce( + std::numeric_limits::lowest(), + [](Scalar a, Scalar b) noexcept { return std::max(a, b); }, + [](Scalar x) noexcept { return x; }); + return Tensor::from_matrix(std::move(result)); + } + + [[nodiscard]] Result col_reduce_max(const Tensor& A) override + { + const Matrix& m = A.cpu_matrix(); + Matrix result = m.col_reduce( + std::numeric_limits::lowest(), + [](Scalar a, Scalar b) noexcept { return std::max(a, b); }, + [](Scalar x) noexcept { return x; }); + return Tensor::from_matrix(std::move(result)); + } + + // ══════════════════════════════════════════════════════════════════════ + // 广播原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result broadcast_row_inplace( + Tensor& A, const Tensor& row_vec, BinaryOp op) override + { + // 使用具体 lambda（非 std::function）以允许内联和向量化 + auto& a = A.cpu_matrix(); + const auto& rv = row_vec.cpu_matrix(); + switch (op) + { + case BinaryOp::Add: a.broadcast_row_inplace(rv, [](Scalar x, Scalar y) noexcept { return x + y; }); break; + case BinaryOp::Sub: a.broadcast_row_inplace(rv, [](Scalar x, Scalar y) noexcept { return x - y; }); break; + case BinaryOp::Mul: a.broadcast_row_inplace(rv, [](Scalar x, Scalar y) noexcept { return x * y; }); break; + case BinaryOp::Div: a.broadcast_row_inplace(rv, [](Scalar x, Scalar y) noexcept { return x / y; }); break; + case BinaryOp::Max: a.broadcast_row_inplace(rv, [](Scalar x, Scalar y) noexcept { return std::max(x, y); }); break; + case BinaryOp::Min: a.broadcast_row_inplace(rv, [](Scalar x, Scalar y) noexcept { return std::min(x, y); }); break; + } + return {}; + } + + [[nodiscard]] Result broadcast_col_inplace( + Tensor& A, const Tensor& col_vec, BinaryOp op) override + { + auto& a = A.cpu_matrix(); + const auto& cv = col_vec.cpu_matrix(); + switch (op) + { + case BinaryOp::Add: a.broadcast_col_inplace(cv, [](Scalar x, Scalar y) noexcept { return x + y; }); break; + case BinaryOp::Sub: a.broadcast_col_inplace(cv, [](Scalar x, Scalar y) noexcept { return x - y; }); break; + case BinaryOp::Mul: a.broadcast_col_inplace(cv, [](Scalar x, Scalar y) noexcept { return x * y; }); break; + case BinaryOp::Div: a.broadcast_col_inplace(cv, [](Scalar x, Scalar y) noexcept { return x / y; }); break; + case BinaryOp::Max: a.broadcast_col_inplace(cv, [](Scalar x, Scalar y) noexcept { return std::max(x, y); }); break; + case BinaryOp::Min: a.broadcast_col_inplace(cv, [](Scalar x, Scalar y) noexcept { return std::min(x, y); }); break; + } + return {}; + } + + // ══════════════════════════════════════════════════════════════════════ + // 逐元素原语（通过 AST switch 分发） + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result elementwise_unary( + UnaryOp op, const Tensor& A) override + { + const Matrix& m = A.cpu_matrix(); + Matrix result(m.rows(), m.cols()); + ConstSpan in = m.span(); + Span out = result.span(); + + switch (op) + { + case UnaryOp::Neg: compute::apply(out, neg(in)); break; + case UnaryOp::Exp: compute::apply(out, exp(in)); break; + case UnaryOp::Log: compute::apply(out, log(in)); break; + case UnaryOp::Sqrt: compute::apply(out, sqrt(in)); break; + case UnaryOp::Rsqrt: compute::apply(out, rsqrt(in)); break; + case UnaryOp::Abs: compute::apply(out, abs(in)); break; + case UnaryOp::Tanh: compute::apply(out, tanh(in)); break; + } + + return Tensor::from_matrix(std::move(result)); + } + + [[nodiscard]] Result elementwise_binary( + BinaryOp op, const Tensor& A, const Tensor& B) override + { + if (A.rows() != B.rows() || A.cols() != B.cols()) + return std::unexpected(Error{"elementwise_binary: shape mismatch"}); + + const Matrix& ma = A.cpu_matrix(); + const Matrix& mb = B.cpu_matrix(); + Matrix result(ma.rows(), ma.cols()); + ConstSpan a = ma.span(); + ConstSpan b = mb.span(); + Span out = result.span(); + + switch (op) + { + case BinaryOp::Add: compute::apply(out, a + b); break; + case BinaryOp::Sub: compute::apply(out, a - b); break; + case BinaryOp::Mul: compute::apply(out, a * b); break; + case BinaryOp::Div: compute::apply(out, a / b); break; + case BinaryOp::Max: compute::apply(out, max(a, b)); break; + case BinaryOp::Min: compute::apply(out, min(a, b)); break; + } + + return Tensor::from_matrix(std::move(result)); + } + + [[nodiscard]] Result elementwise_binary_scalar( + BinaryOp op, const Tensor& A, Scalar s, bool scalar_first) override + { + const Matrix& m = A.cpu_matrix(); + Matrix result(m.rows(), m.cols()); + ConstSpan a = m.span(); + Span out = result.span(); + + if (scalar_first) + { + // out = op(scalar, A) + switch (op) + { + case BinaryOp::Add: compute::apply(out, s + a); break; + case BinaryOp::Sub: compute::apply(out, s - a); break; + case BinaryOp::Mul: compute::apply(out, s * a); break; + case BinaryOp::Div: compute::apply(out, s / a); break; + case BinaryOp::Max: compute::apply(out, max(Val{s}, a)); break; + case BinaryOp::Min: compute::apply(out, min(Val{s}, a)); break; + } + } + else + { + // out = op(A, scalar) + switch (op) + { + case BinaryOp::Add: compute::apply(out, a + s); break; + case BinaryOp::Sub: compute::apply(out, a - s); break; + case BinaryOp::Mul: compute::apply(out, a * s); break; + case BinaryOp::Div: compute::apply(out, a / s); break; + case BinaryOp::Max: compute::apply(out, max(a, s)); break; + case BinaryOp::Min: compute::apply(out, min(a, s)); break; + } + } + + return Tensor::from_matrix(std::move(result)); + } + + // ══════════════════════════════════════════════════════════════════════ + // 条件选择原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result elementwise_select_scalar_cond( + CompareOp cmp, const Tensor& A, Scalar scalar_b, + const Tensor& then_t, Scalar scalar_else) override + { + if (A.rows() != then_t.rows() || A.cols() != then_t.cols()) + return std::unexpected(Error{"elementwise_select: A and then shape mismatch"}); + + const Matrix& ma = A.cpu_matrix(); + const Matrix& mt = then_t.cpu_matrix(); + Matrix result(ma.rows(), ma.cols()); + auto a = ma.span(); + auto t = mt.span(); + auto out = result.span(); + const std::size_t n = a.size(); + + // 手动循环避免 AST 对 == / != 的 ConstSpan 支持缺失 + switch (cmp) + { + case CompareOp::Lt: + for (std::size_t i = 0; i < n; ++i) + out[i] = (a[i] < scalar_b) ? t[i] : scalar_else; + break; + case CompareOp::Le: + for (std::size_t i = 0; i < n; ++i) + out[i] = (a[i] <= scalar_b) ? t[i] : scalar_else; + break; + case CompareOp::Gt: + for (std::size_t i = 0; i < n; ++i) + out[i] = (a[i] > scalar_b) ? t[i] : scalar_else; + break; + case CompareOp::Ge: + for (std::size_t i = 0; i < n; ++i) + out[i] = (a[i] >= scalar_b) ? t[i] : scalar_else; + break; + case CompareOp::Eq: + for (std::size_t i = 0; i < n; ++i) + out[i] = (a[i] == scalar_b) ? t[i] : scalar_else; + break; + case CompareOp::Ne: + for (std::size_t i = 0; i < n; ++i) + out[i] = (a[i] != scalar_b) ? t[i] : scalar_else; + break; + } + + return Tensor::from_matrix(std::move(result)); + } + +}; + +} // namespace nn + +#endif // NN_CPU_ENGINE_HPP diff --git a/include/neuralnet.cpp/cuda_engine.hpp b/include/neuralnet.cpp/cuda_engine.hpp new file mode 100644 index 00000000..8d72eec3 --- /dev/null +++ b/include/neuralnet.cpp/cuda_engine.hpp @@ -0,0 +1,429 @@ +// ── cuda_engine.hpp — CUDA 计算引擎实现 ──────────────────────────────────── +// CudaEngine 封装 CudaBackend，实现 ComputeEngine 接口。 +// 与 GpuEngine（Vulkan）架构对称，所有原语在 GPU 上原生执行。 +// ───────────────────────────────────────────────────────────────────────── + +#ifndef NN_CUDA_ENGINE_HPP +#define NN_CUDA_ENGINE_HPP + +#ifdef NN_HAS_CUDA + +#include "compute_engine.hpp" +#include "backend/cuda_backend.hpp" + +namespace nn +{ + +// ══════════════════════════════════════════════════════════════════════════ +// CudaEngine — CUDA 计算引擎 +// ══════════════════════════════════════════════════════════════════════════ + +class CudaEngine final : public ComputeEngine +{ +private: + CudaBackend& backend_; + +public: + explicit CudaEngine(CudaBackend& backend) : backend_(backend) {} + + [[nodiscard]] Device device() const noexcept override { return Device::GPU; } + + // ── 批处理：no-op（当前使用同步默认流）────────────────────────────── + [[nodiscard]] Result begin_batch() override { return backend_.begin_batch(); } + [[nodiscard]] Result end_batch() override { return backend_.end_batch(); } + [[nodiscard]] Result flush_batch() override { return {}; } + + // ══════════════════════════════════════════════════════════════════════ + // 张量工厂 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Tensor create_tensor(std::size_t rows, std::size_t cols) override + { + auto r = CudaTensor::create_empty(rows, cols, backend_); + if (!r) return Tensor(); + return Tensor::from_cuda(std::move(*r)); + } + + [[nodiscard]] Result from_matrix(const Matrix& m) override + { + auto r = CudaTensor::from_matrix(m, backend_); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + [[nodiscard]] Result to_matrix(const Tensor& t) override + { + if (t.is_cpu()) + return Matrix(t.cpu_matrix()); + return t.cuda_tensor().to_matrix(backend_); + } + + [[nodiscard]] Result copy_from(Tensor& dst, const Matrix& src) override + { + if (dst.rows() != src.rows() || dst.cols() != src.cols()) + return std::unexpected(Error{"copy_from: shape mismatch"}); + if (dst.is_cpu()) + { + dst = Tensor::from_matrix(Matrix(src)); + return {}; + } + return backend_.upload_blocking(dst.cuda_tensor(), src.span()); + } + + [[nodiscard]] Result clone(const Tensor& src) override + { + if (src.is_cpu()) + return Tensor::from_matrix(Matrix(src.cpu_matrix())); + auto r = backend_.clone_gpu(src.cuda_tensor()); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + // ── 行切片 ──────────────────────────────────────────────────────────── + [[nodiscard]] Result slice_rows( + const Tensor& src, std::size_t start_row, std::size_t count) override + { + auto src_gpu = ensure_cuda(src); + if (!src_gpu) return std::unexpected(src_gpu.error()); + auto r = backend_.slice_rows_gpu(src_gpu->cuda_tensor(), start_row, count); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + // ── 行插入 ──────────────────────────────────────────────────────────── + [[nodiscard]] Result insert_rows( + Tensor& dst, std::size_t dst_start_row, const Tensor& src) override + { + if (dst.is_cpu()) + { + if (!src.is_cpu()) + { + auto sm = to_matrix(src); + if (!sm) return std::unexpected(sm.error()); + Tensor src_cpu = Tensor::from_matrix(Matrix(*sm)); + return insert_rows_cpu_fallback(dst, dst_start_row, src_cpu); + } + return insert_rows_cpu_fallback(dst, dst_start_row, src); + } + auto src_gpu = ensure_cuda(src); + if (!src_gpu) return std::unexpected(src_gpu.error()); + return backend_.insert_rows_gpu(dst.cuda_tensor(), dst_start_row, src_gpu->cuda_tensor()); + } + + // ── Gather ──────────────────────────────────────────────────────────── + [[nodiscard]] Result gather_rows( + const Tensor& table, const Tensor& indices) override + { + auto tbl_gpu = ensure_cuda(table); + if (!tbl_gpu) return std::unexpected(tbl_gpu.error()); + auto idx_gpu = ensure_cuda(indices); + if (!idx_gpu) return std::unexpected(idx_gpu.error()); + auto r = backend_.gather_gpu(tbl_gpu->cuda_tensor(), idx_gpu->cuda_tensor()); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + // ── Scatter Add ─────────────────────────────────────────────────────── + [[nodiscard]] Result scatter_add_rows( + Tensor& dst, const Tensor& indices, const Tensor& grad) override + { + if (!dst.is_gpu()) + return std::unexpected(Error{"scatter_add_rows: dst must be GPU tensor"}); + auto idx_gpu = ensure_cuda(indices); + if (!idx_gpu) return std::unexpected(idx_gpu.error()); + auto grad_gpu = ensure_cuda(grad); + if (!grad_gpu) return std::unexpected(grad_gpu.error()); + return backend_.scatter_add_gpu( + dst.cuda_tensor(), idx_gpu->cuda_tensor(), grad_gpu->cuda_tensor()); + } + +private: + [[nodiscard]] Result insert_rows_cpu_fallback( + Tensor& dst, std::size_t dst_start_row, const Tensor& src) + { + Matrix& d = dst.cpu_matrix(); + const Matrix& s = src.cpu_matrix(); + if (d.cols() != s.cols()) + return std::unexpected(Error{"insert_rows: column count mismatch"}); + if (dst_start_row + s.rows() > d.rows()) + return std::unexpected(Error{"insert_rows: range out of bounds"}); + const auto dst_span = d.span(); + const auto src_span = s.span(); + const std::size_t cols = d.cols(); + for (std::size_t r = 0; r < s.rows(); ++r) + std::copy_n(src_span.begin() + r * cols, cols, + dst_span.begin() + (dst_start_row + r) * cols); + return {}; + } + +public: + + // ── 3D 重排 ─────────────────────────────────────────────────────────── + [[nodiscard]] Result rearrange_3d( + const Tensor& x, std::size_t M, std::size_t B, std::size_t N, + bool inverse) override + { + auto x_gpu = ensure_cuda(x); + if (!x_gpu) return std::unexpected(x_gpu.error()); + auto r = backend_.rearrange_3d_gpu( + x_gpu->cuda_tensor(), + static_cast(M), static_cast(B), + static_cast(N), inverse ? 1u : 0u); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + // ── 转置 ────────────────────────────────────────────────────────────── + [[nodiscard]] Result transpose(const Tensor& A) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.transpose_gpu(a_gpu->cuda_tensor()); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + // ══════════════════════════════════════════════════════════════════════ + // 矩阵级原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result matmul( + const Tensor& A, const Tensor& B, + bool transA, bool transB) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto b_gpu = ensure_cuda(B); + if (!b_gpu) return std::unexpected(b_gpu.error()); + auto r = backend_.matmul_gpu( + a_gpu->cuda_tensor(), b_gpu->cuda_tensor(), + transA ? 1u : 0u, transB ? 1u : 0u); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + [[nodiscard]] Result batched_matmul( + const Tensor& A, const Tensor& B, + std::size_t batch, + bool transA, bool transB) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto b_gpu = ensure_cuda(B); + if (!b_gpu) return std::unexpected(b_gpu.error()); + auto r = backend_.batched_matmul_gpu( + a_gpu->cuda_tensor(), b_gpu->cuda_tensor(), + static_cast(batch), + transA ? 1u : 0u, transB ? 1u : 0u); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + // A += B + [[nodiscard]] Result add_inplace(Tensor& A, const Tensor& B) override + { + if (A.rows() != B.rows() || A.cols() != B.cols()) + return std::unexpected(Error{"add_inplace: shape mismatch"}); + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto b_gpu = ensure_cuda(B); + if (!b_gpu) return std::unexpected(b_gpu.error()); + auto r = backend_.elementwise_binary_gpu(a_gpu->cuda_tensor(), b_gpu->cuda_tensor(), 0); + if (!r) return std::unexpected(r.error()); + A = Tensor::from_cuda(std::move(*r)); + return {}; + } + + // A *= scalar + [[nodiscard]] Result scale_inplace(Tensor& A, Scalar s) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.elementwise_binary_scalar_gpu( + a_gpu->cuda_tensor(), 2, static_cast(s), false); + if (!r) return std::unexpected(r.error()); + A = Tensor::from_cuda(std::move(*r)); + return {}; + } + + // A += scalar * B（融合 axpy） + [[nodiscard]] Result axpy_inplace(Tensor& A, Scalar scalar, const Tensor& B) override + { + if (A.rows() != B.rows() || A.cols() != B.cols()) + return std::unexpected(Error{"axpy_inplace: shape mismatch"}); + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto b_gpu = ensure_cuda(B); + if (!b_gpu) return std::unexpected(b_gpu.error()); + auto r = backend_.axpy_gpu(a_gpu->cuda_tensor(), b_gpu->cuda_tensor(), static_cast(scalar)); + if (!r) return std::unexpected(r.error()); + A = Tensor::from_cuda(std::move(*r)); + return {}; + } + + // A = 0 + [[nodiscard]] Result zero(Tensor& A) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.fill_zero_gpu(a_gpu->cuda_tensor()); + if (!r) return std::unexpected(r.error()); + if (A.is_cpu()) + A = std::move(*a_gpu); + return {}; + } + + // ══════════════════════════════════════════════════════════════════════ + // 归约原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result row_reduce_sum(const Tensor& A) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.reduce_gpu(a_gpu->cuda_tensor(), 0u, 0u); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + [[nodiscard]] Result col_reduce_sum(const Tensor& A) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.reduce_gpu(a_gpu->cuda_tensor(), 1u, 0u); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + [[nodiscard]] Result row_reduce_max(const Tensor& A) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.reduce_gpu(a_gpu->cuda_tensor(), 0u, 1u); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + [[nodiscard]] Result col_reduce_max(const Tensor& A) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.reduce_gpu(a_gpu->cuda_tensor(), 1u, 1u); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + // ══════════════════════════════════════════════════════════════════════ + // 广播原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result broadcast_row_inplace( + Tensor& A, const Tensor& row_vec, BinaryOp op) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto rv_gpu = ensure_cuda(row_vec); + if (!rv_gpu) return std::unexpected(rv_gpu.error()); + auto r = backend_.broadcast_gpu( + a_gpu->cuda_tensor(), rv_gpu->cuda_tensor(), + 0u, static_cast(op)); + if (!r) return std::unexpected(r.error()); + A = Tensor::from_cuda(std::move(*r)); + return {}; + } + + [[nodiscard]] Result broadcast_col_inplace( + Tensor& A, const Tensor& col_vec, BinaryOp op) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto cv_gpu = ensure_cuda(col_vec); + if (!cv_gpu) return std::unexpected(cv_gpu.error()); + auto r = backend_.broadcast_gpu( + a_gpu->cuda_tensor(), cv_gpu->cuda_tensor(), + 1u, static_cast(op)); + if (!r) return std::unexpected(r.error()); + A = Tensor::from_cuda(std::move(*r)); + return {}; + } + + // ══════════════════════════════════════════════════════════════════════ + // 逐元素原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result elementwise_unary(UnaryOp op, const Tensor& A) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.elementwise_unary_gpu(a_gpu->cuda_tensor(), static_cast(op)); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + [[nodiscard]] Result elementwise_binary( + BinaryOp op, const Tensor& A, const Tensor& B) override + { + if (A.rows() != B.rows() || A.cols() != B.cols()) + return std::unexpected(Error{"elementwise_binary: shape mismatch"}); + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto b_gpu = ensure_cuda(B); + if (!b_gpu) return std::unexpected(b_gpu.error()); + auto r = backend_.elementwise_binary_gpu( + a_gpu->cuda_tensor(), b_gpu->cuda_tensor(), static_cast(op)); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + [[nodiscard]] Result elementwise_binary_scalar( + BinaryOp op, const Tensor& A, Scalar s, bool scalar_first) override + { + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.elementwise_binary_scalar_gpu( + a_gpu->cuda_tensor(), static_cast(op), + static_cast(s), scalar_first); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + + // ══════════════════════════════════════════════════════════════════════ + // 条件选择原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result elementwise_select_scalar_cond( + CompareOp cmp, const Tensor& A, Scalar scalar_b, + const Tensor& then_t, Scalar scalar_else) override + { + if (A.rows() != then_t.rows() || A.cols() != then_t.cols()) + return std::unexpected(Error{"select: A and then shape mismatch"}); + auto a_gpu = ensure_cuda(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto t_gpu = ensure_cuda(then_t); + if (!t_gpu) return std::unexpected(t_gpu.error()); + auto r = backend_.elementwise_select_scalar_cond_gpu( + a_gpu->cuda_tensor(), t_gpu->cuda_tensor(), + static_cast(cmp), + static_cast(scalar_b), static_cast(scalar_else)); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } + +private: + // ── 辅助：确保 Tensor 在 CUDA 上 ───────────────────────────────────── + [[nodiscard]] Result ensure_cuda(const Tensor& t) + { + if (t.is_gpu()) + return t; + if (!t.valid()) + return std::unexpected(Error{"ensure_cuda: invalid tensor"}); + auto r = CudaTensor::from_matrix(t.cpu_matrix(), backend_); + if (!r) return std::unexpected(r.error()); + return Tensor::from_cuda(std::move(*r)); + } +}; + +} // namespace nn + +#endif // NN_HAS_CUDA +#endif // NN_CUDA_ENGINE_HPP diff --git a/include/neuralnet.cpp/domain_gpt.hpp b/include/neuralnet.cpp/domain_gpt.hpp new file mode 100644 index 00000000..12ca5fe1 --- /dev/null +++ b/include/neuralnet.cpp/domain_gpt.hpp @@ -0,0 +1,129 @@ +#ifndef NN_DOMAIN_GPT_HPP +#define NN_DOMAIN_GPT_HPP + +// ── domain_gpt.hpp — GPT 领域构建层（引擎化架构） ────────────────────────── +// +// 依赖：Model + ComputeEngine + Tokenizer +// （model_container.hpp / compute_engine.hpp / domain_tokenizer.hpp） +// ───────────────────────────────────────────────────────────────────────── + +#include +#include +#include +#include + +#include "core_errors.hpp" +#include "compute_engine.hpp" +#include "model_container.hpp" +#include "model_spec.hpp" + +namespace nn { + +// ── GPT 默认超参数 ──────────────────────────────────────────────────────── +inline constexpr std::size_t GPT_VOCAB_SIZE = 10000; +inline constexpr std::size_t GPT_D_MODEL = 128; +inline constexpr std::size_t GPT_NUM_HEADS = 4; +inline constexpr std::size_t GPT_D_FF = 512; +inline constexpr std::size_t GPT_NUM_LAYERS = 4; +inline constexpr std::size_t GPT_SEQ_LEN = 256; + +// ── GPT 配置结构体 ──────────────────────────────────────────────────────── +// 将 build_gpt_model 的多个位置参数收拢为一个结构体，避免调用方签名过长。 +struct GptConfig { + std::size_t vocab_size = GPT_VOCAB_SIZE; + std::size_t d_model = GPT_D_MODEL; + std::size_t seq_len = GPT_SEQ_LEN; + std::size_t num_heads = GPT_NUM_HEADS; + std::size_t d_ff = GPT_D_FF; + std::size_t num_layers = GPT_NUM_LAYERS; + PosEncodingType pos_enc = PosEncodingType::Learned; + ActivationType activation = ActivationType::GeLU; + NormType norm_type = NormType::LayerNorm; +}; + +// ── 构建 GPT 模型（GptConfig 版本，推荐使用） ──────────────────────────── +// GPTModel 是一个单一 Layer（内含 TokenEmb+PosEmb+N×GPTBlock+LN+LM Head）， +// 作为 Model 的唯一层。engine 决定权重张量驻留设备。 +// cfg.pos_enc 控制位置编码：Learned(默认)/Sinusoidal/ALiBi +[[nodiscard]] inline Result build_gpt_model( + ComputeEngine& engine, const GptConfig& cfg) +{ + if (cfg.d_model == 0 || cfg.num_heads == 0 || cfg.seq_len == 0 || cfg.vocab_size == 0) + return std::unexpected(Error{"GPT model parameters must be positive"}); + if (cfg.d_ff == 0 || cfg.num_layers == 0) + return std::unexpected(Error{"GPT d_ff and num_layers must be positive"}); + if (cfg.d_model % cfg.num_heads != 0) + return std::unexpected(Error{"GPT d_model must be divisible by num_heads"}); + + Model model(engine); + model.add(engine, cfg.vocab_size, cfg.d_model, cfg.seq_len, + cfg.num_heads, cfg.d_ff, cfg.num_layers, cfg.pos_enc, + cfg.activation, cfg.norm_type); + return model; +} + +// ── 构建 GPT 模型（位置参数版本，转发至 GptConfig 版本，向后兼容） ─────── +[[nodiscard]] inline Result build_gpt_model( + ComputeEngine& engine, + std::size_t vocab_size = GPT_VOCAB_SIZE, + std::size_t d_model = GPT_D_MODEL, + std::size_t seq_len = GPT_SEQ_LEN, + std::size_t num_heads = GPT_NUM_HEADS, + std::size_t d_ff = GPT_D_FF, + std::size_t num_layers = GPT_NUM_LAYERS, + PosEncodingType pos_enc_type = PosEncodingType::Learned, + ActivationType activation = ActivationType::GeLU, + NormType norm_type = NormType::LayerNorm) +{ + return build_gpt_model(engine, GptConfig{ + vocab_size, d_model, seq_len, num_heads, d_ff, num_layers, + pos_enc_type, activation, norm_type}); +} + +// ── 从 ModelSpec 构建 GPT 模型 ────────────────────────────────────────── +// 用于从二进制文件加载时自动还原架构。 +// 统一的 GPTModel 通过 spec.pos_encoding 区分 Learned/Sinusoidal/ALiBi 模式， +// 因此无需为 ALiBi 单独提供构建函数。 +[[nodiscard]] inline Result build_gpt_model_from_spec( + ComputeEngine& engine, const ModelSpec &spec) +{ + // 接受 GPT 类型，或旧格式 ALiBi_GPT 类型（向后兼容旧模型文件） + if (!spec.is_gpt() && !spec.is_alibi_gpt()) + return std::unexpected(Error{"Invalid ModelSpec type for GPT: expected GPT or ALiBi_GPT"}); + + return build_gpt_model( + engine, + spec.vocab_size, spec.d_model, spec.seq_len, + spec.num_heads, spec.d_ff, spec.num_layers, + spec.pos_encoding, spec.activation, spec.norm_type); +} + +// ── 构造 GPT ModelSpec ────────────────────────────────────────────────── +[[nodiscard]] inline ModelSpec make_gpt_spec( + std::size_t vocab_size, + std::size_t d_model, + std::size_t seq_len, + std::size_t num_heads, + std::size_t d_ff, + std::size_t num_layers, + PosEncodingType pos_encoding = PosEncodingType::Learned, + ActivationType activation = ActivationType::GeLU, + NormType norm_type = NormType::LayerNorm) +{ + ModelSpec spec; + spec.type = ModelType::GPT; + spec.vocab_size = vocab_size; + spec.d_model = d_model; + spec.seq_len = seq_len; + spec.num_heads = num_heads; + spec.d_ff = d_ff; + spec.num_layers = num_layers; + spec.pos_encoding = pos_encoding; + spec.activation = activation; + spec.norm_type = norm_type; + return spec; +} + +} // namespace nn + +#endif // NN_DOMAIN_GPT_HPP diff --git a/include/neuralnet.cpp/domain_mnist.hpp b/include/neuralnet.cpp/domain_mnist.hpp new file mode 100644 index 00000000..a03b1069 --- /dev/null +++ b/include/neuralnet.cpp/domain_mnist.hpp @@ -0,0 +1,194 @@ +#ifndef NN_DOMAIN_MNIST_HPP +#define NN_DOMAIN_MNIST_HPP + +// ── domain_mnist.hpp — MNIST 领域构建层 ──────────────────────────────────── +// +// 基于（引擎化架构）提供两种模型构建： +// 1. MLP —— build_mnist_mlp_model +// 2. Transformer —— build_mnist_transformer_model（ViT 风格） +// +// 依赖：Model + ComputeEngine（model_container.hpp / compute_engine.hpp） +// ───────────────────────────────────────────────────────────────────────── + +#include +#include +#include +#include + +#include "core_errors.hpp" +#include "compute_engine.hpp" +#include "model_container.hpp" +#include "model_spec.hpp" + +namespace nn { + +// ── MNIST 常量 ────────────────────────────────────────────────────────────── +inline constexpr std::size_t MNIST_INPUT_DIM = 784; +inline constexpr std::size_t MNIST_NUM_CLASSES = 10; + +// ── MNIST Transformer (ViT) 默认超参数 ────────────────────────────────────── +// 图像 28×28，patch_size=7 → 4×4=16 个 patch，patch_dim=49 +inline constexpr std::size_t MNIST_IMG_SIZE = 28; +inline constexpr std::size_t MNIST_PATCH_SIZE = 7; +inline constexpr std::size_t MNIST_TF_D_MODEL = 64; +inline constexpr std::size_t MNIST_TF_NUM_HEADS = 4; +inline constexpr std::size_t MNIST_TF_D_FF = 128; +inline constexpr std::size_t MNIST_TF_NUM_LAYERS = 2; + +// ── 共享 CSV 行解析工具 ──────────────────────────────────────────────────── +// 解析单行 CSV（逗号分隔的浮点数）为 std::vector +[[nodiscard]] inline Result> +parse_csv_line(const std::string &line) +{ + std::vector values; + std::stringstream ss(line); + std::string token; + while (std::getline(ss, token, ',')) + { + auto v = parse_number(token); + if (!v) + return std::unexpected(Error{"CSV 含无效数字 '" + token + "': " + v.error().message}); + values.push_back(*v); + } + return values; +} + +// ── 从 CSV 行加载单张 MNIST 图片 ─────────────────────────────────────────── +// 输入：784 个逗号分隔的像素值（0~255 或归一化后的 0~1） +// 输出：(784, 1) 列向量 Matrix +// 保留作为底层解析实现；新代码应优先使用 load_image_tensor_from_csv_line +// （上层统一用 Tensor）。 +[[nodiscard]] inline Result +load_image_from_csv_line(const std::string &csv_line) +{ + auto values = parse_csv_line(csv_line); + if (!values) + return std::unexpected(std::move(values).error()); + if (values->size() != MNIST_INPUT_DIM) + return std::unexpected(Error{"CSV 必须包含恰好 " + std::to_string(MNIST_INPUT_DIM) + + " 个值，实际: " + std::to_string(values->size())}); + + Matrix img(MNIST_INPUT_DIM, 1); + for (std::size_t i = 0; i < MNIST_INPUT_DIM; ++i) + img.set_value_unchecked(i, 0, (*values)[i]); + return img; +} + +// ── 从 CSV 行加载单张 MNIST 图片为 Tensor ──────────────────────────────── +// 上层统一使用 Tensor：内部调用 load_image_from_csv_line 得到 Matrix 后， +// 通过 engine.from_matrix 上传至对应设备。 +[[nodiscard]] inline Result +load_image_tensor_from_csv_line(const std::string &csv_line, ComputeEngine& engine) +{ + auto mat_r = load_image_from_csv_line(csv_line); + if (!mat_r) + return std::unexpected(std::move(mat_r).error()); + return engine.from_matrix(*mat_r); +} + +// 默认 MLP 网络架构：输入层 → 隐藏层1 → 隐藏层2 → 隐藏层3 → 输出层 +inline const std::vector MNIST_LAYER_DIMS = { + MNIST_INPUT_DIM, 512, 256, 128, 64, MNIST_NUM_CLASSES +}; + +// ── 构建 MLP 模型 ────────────────────────────────────────────────────────── +// 通过指定 ComputeEngine 创建同设备的权重张量；可指定自定义层维度。 +// 结构：Linear → LayerNorm → GeLU × (N-1) + Linear（最后一层） +[[nodiscard]] inline Result build_mnist_mlp_model( + ComputeEngine& engine, + const std::vector &layer_dims = MNIST_LAYER_DIMS) +{ + if (layer_dims.size() < 2) + return std::unexpected(Error{"MLP layer_dims must have at least 2 elements"}); + + Model model(engine); + for (std::size_t i = 0; i < layer_dims.size() - 1; ++i) + { + std::size_t in_dim = layer_dims[i]; + std::size_t out_dim = layer_dims[i + 1]; + + model.add(engine, in_dim, out_dim); + + if (i < layer_dims.size() - 2) + { + model.add(engine, out_dim) + .add(); + } + } + return model; +} + +// ── 构建 MNIST Transformer (ViT 风格) 模型 ────────────────────────────────── +// 结构：PatchEmbedding → TransformerEncoder → Linear(分类头) +// PatchEmbedding: img 28×28 → 16 个 7×7 patch → Linear(49, d_model) 投影 +// TransformerEncoder: PE + N×EncoderLayer + 全局平均池化 +// Linear: d_model → 10（分类头） +[[nodiscard]] inline Result build_mnist_transformer_model( + ComputeEngine& engine, + std::size_t img_size = MNIST_IMG_SIZE, + std::size_t patch_size = MNIST_PATCH_SIZE, + std::size_t d_model = MNIST_TF_D_MODEL, + std::size_t num_heads = MNIST_TF_NUM_HEADS, + std::size_t d_ff = MNIST_TF_D_FF, + std::size_t num_layers = MNIST_TF_NUM_LAYERS) +{ + if (img_size % patch_size != 0) + return std::unexpected(Error{"MNIST Transformer: img_size must be divisible by patch_size"}); + if (d_model == 0 || num_heads == 0 || d_ff == 0 || num_layers == 0) + return std::unexpected(Error{"MNIST Transformer: parameters must be positive"}); + if (d_model % num_heads != 0) + return std::unexpected(Error{"MNIST Transformer: d_model must be divisible by num_heads"}); + + const std::size_t grid_size = img_size / patch_size; + const std::size_t num_patches = grid_size * grid_size; + + Model model(engine); + model.add(engine, img_size, patch_size, d_model); + model.add(engine, d_model, num_heads, d_ff, num_layers, num_patches); + model.add(engine, d_model, MNIST_NUM_CLASSES); + return model; +} + +// ── 构造 MNIST Transformer ModelSpec ─────────────────────────────────────── +// img_size 固定为 MNIST_IMG_SIZE（28×28），故不入参、不写入 spec。 +[[nodiscard]] inline ModelSpec make_mnist_transformer_spec( + std::size_t patch_size = MNIST_PATCH_SIZE, + std::size_t d_model = MNIST_TF_D_MODEL, + std::size_t num_heads = MNIST_TF_NUM_HEADS, + std::size_t d_ff = MNIST_TF_D_FF, + std::size_t num_layers = MNIST_TF_NUM_LAYERS) +{ + ModelSpec spec; + spec.type = ModelType::Transformer; + spec.d_model = d_model; + spec.num_heads = num_heads; + spec.d_ff = d_ff; + spec.num_layers = num_layers; + spec.patch_size = patch_size; + // vocab_size / seq_len 不用于 Transformer (MNIST ViT) + return spec; +} + +// ── 从 ModelSpec 构建模型 ───────────────────────────────────────────────── +// 用于从二进制文件加载时自动还原架构（支持 MLP 和 Transformer） +[[nodiscard]] inline Result build_mnist_model_from_spec( + ComputeEngine& engine, const ModelSpec &spec) +{ + if (spec.is_mlp()) + return build_mnist_mlp_model(engine, spec.layer_dims); + + if (spec.is_transformer()) + { + const std::size_t patch_size = spec.patch_size != 0 ? spec.patch_size : MNIST_PATCH_SIZE; + return build_mnist_transformer_model( + engine, MNIST_IMG_SIZE, patch_size, + spec.d_model, spec.num_heads, spec.d_ff, spec.num_layers); + } + + return std::unexpected(Error{ + "Invalid ModelSpec type for MNIST: expected MLP or Transformer"}); +} + +} // namespace nn + +#endif // NN_DOMAIN_MNIST_HPP diff --git a/include/neuralnet.cpp/domain_tokenizer.hpp b/include/neuralnet.cpp/domain_tokenizer.hpp new file mode 100644 index 00000000..d40bbc67 --- /dev/null +++ b/include/neuralnet.cpp/domain_tokenizer.hpp @@ -0,0 +1,2277 @@ +#ifndef NN_DOMAIN_TOKENIZER_HPP +#define NN_DOMAIN_TOKENIZER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.hpp" + +namespace nn +{ + +// ═══════════════════════════════════════════════════════════════════════════ +// Tokenizer — 抽象基类 +// ═══════════════════════════════════════════════════════════════════════════ + +class Tokenizer +{ +public: + virtual ~Tokenizer() = default; + + [[nodiscard]] virtual std::vector encode(const std::string &text) const = 0; + [[nodiscard]] virtual std::string decode(std::span ids) const = 0; + [[nodiscard]] virtual std::size_t vocab_size() const noexcept = 0; + [[nodiscard]] virtual const std::vector &vocab() const noexcept = 0; + [[nodiscard]] virtual Result save(const std::string &path) const = 0; + [[nodiscard]] virtual Result load(const std::string &path) = 0; + + // ── 特殊 token ID 统一接口 ──────────────────────────────────────── + // 默认返回 npos（表示该 tokenizer 无此特殊 token）。 + // 子类根据自身词表布局重写。 + // 训练/推理入口通过这些接口获取 BOS/EOS/PAD，无需硬编码常量。 + [[nodiscard]] virtual std::size_t bos_id() const noexcept { return npos; } + [[nodiscard]] virtual std::size_t eos_id() const noexcept { return npos; } + [[nodiscard]] virtual std::size_t pad_id() const noexcept { return 0; } + + static constexpr std::size_t npos = static_cast(-1); + + // UTF-8 替换字符 U+FFFD（解码未知 ID 时的回退） + static constexpr std::string_view REPLACEMENT_CHAR = "\xef\xbf\xbd"; + + // ── 从 JSON 字符串加载（纯内存解析，零磁盘 I/O） ────────────── + // 子类重写此方法以支持从 JSON 字符串直接加载词表。 + [[nodiscard]] virtual Result load_from_string(const std::string &json_content) = 0; + + // ── 对话标记集合（基类持有，子类直接复用，无需各自重定义） ────── + struct DialogueMarkers { + std::size_t system = npos, user = npos, assistant = npos; + std::size_t system_end = npos, user_end = npos, assistant_end = npos; + }; + + // ── 对话标记 token（默认从 markers_ 读取，子类一般无需重写） ───── + [[nodiscard]] virtual std::size_t system_marker_id() const noexcept { return markers_.system; } + [[nodiscard]] virtual std::size_t user_marker_id() const noexcept { return markers_.user; } + [[nodiscard]] virtual std::size_t assistant_marker_id() const noexcept { return markers_.assistant; } + [[nodiscard]] virtual std::size_t end_system_marker_id() const noexcept { return markers_.system_end; } + [[nodiscard]] virtual std::size_t end_user_marker_id() const noexcept { return markers_.user_end; } + [[nodiscard]] virtual std::size_t end_assistant_marker_id() const noexcept { return markers_.assistant_end; } + + // ── 是否包含对话标记 ────────────────────────────────────── + [[nodiscard]] bool has_dialogue_markers() const noexcept + { + return markers_.system != npos; + } + + // ── 从词表中恢复对话标记 ID（加载后调用） ───────────────── + // 扫描词表查找 等标记字符串，若已存在则绑定已有 ID， + // 若不存在则追加到词表末尾。 + void restore_dialogue_markers(std::vector &vocab) + { + static constexpr std::string_view marker_strs[] = { + "", "", "", "", "", "" + }; + std::size_t ids[6]; + bool all_found = true; + for (int i = 0; i < 6; ++i) + { + ids[i] = npos; + for (std::size_t j = 0; j < vocab.size(); ++j) + { + if (vocab[j] == marker_strs[i]) + { ids[i] = j; break; } + } + if (ids[i] == npos) all_found = false; + } + if (!all_found) + { + // 部分标记缺失，追加全部（保持连续 ID） + for (int i = 0; i < 6; ++i) + { + if (ids[i] == npos) + { ids[i] = vocab.size(); vocab.emplace_back(marker_strs[i]); } + } + } + // 通过虚方法设置 markers_（基类默认实现赋值 markers_） + set_dialogue_marker_ids(ids[0], ids[1], ids[2], ids[3], ids[4], ids[5]); + } + +protected: + // 对话标记 ID 集合（子类直接读取，无需各自定义 6 个字段） + DialogueMarkers markers_; + + // 默认实现：将参数赋值到 markers_（子类一般无需重写） + virtual void set_dialogue_marker_ids( + std::size_t system, std::size_t end_system, + std::size_t user, std::size_t end_user, + std::size_t assistant, std::size_t end_assistant) + { + markers_.system = system; + markers_.system_end = end_system; + markers_.user = user; + markers_.user_end = end_user; + markers_.assistant = assistant; + markers_.assistant_end = end_assistant; + } + + // 添加 6 个对话标记到词表末尾，并绑定 ID 到 markers_ + void add_dialogue_markers_to_vocab_(std::vector &vocab) + { + markers_.system = vocab.size(); vocab.emplace_back(""); + markers_.system_end = vocab.size(); vocab.emplace_back(""); + markers_.user = vocab.size(); vocab.emplace_back(""); + markers_.user_end = vocab.size(); vocab.emplace_back(""); + markers_.assistant = vocab.size(); vocab.emplace_back(""); + markers_.assistant_end = vocab.size(); vocab.emplace_back(""); + } + + // ── 辅助：尝试匹配对话标记，返回 (token_id, 匹配字节数)，不匹配返回 (npos, 0) ── + [[nodiscard]] std::pair + try_match_marker(std::string_view text, std::size_t pos) const noexcept + { + static constexpr std::string_view markers[] = { + "", "", "", "", "", "" + }; + for (const auto &m : markers) + { + if (pos + m.size() <= text.size() && text.substr(pos, m.size()) == m) + { + std::size_t id = npos; + if (m == "") id = markers_.system; + else if (m == "") id = markers_.system_end; + else if (m == "") id = markers_.user; + else if (m == "") id = markers_.user_end; + else if (m == "") id = markers_.assistant; + else if (m == "") id = markers_.assistant_end; + if (id != npos) return {id, m.size()}; + } + } + return {npos, 0}; + } + + // ═══════════════════════════════════════════════════════════════════════════ + // BPE 共享构件（供 BPETokenizer / CharBPETokenizer 复用） + // ═══════════════════════════════════════════════════════════════════════════ + + // BPE 合并规则（基类共享，子类直接复用，无需各自定义） + struct MergeRule { std::size_t id_a, id_b, new_id; }; + + // 64 位编码 pair：(a, b) → 单个 uint64_t，消除自定义 hash 开销 + // 要求 a, b < 2^32（词表大小不会超过此限制） + [[nodiscard]] static constexpr std::uint64_t pair_key(std::size_t a, std::size_t b) noexcept + { + return (static_cast(a) << 32) | static_cast(b); + } + + // 共享 BPE 训练：在已初始化的词表和 ID 序列上执行合并循环。 + // vocab 由子类传入（子类的 vocab_），merges_ / merge_map_ 为基类成员。 + // 训练结束后由调用方负责 rebuild_merge_map() 和 add_dialogue_markers_to_vocab_()。 + void bpe_train_impl_( + std::vector &vocab, + std::vector> &chunks, + std::size_t target_merges, + std::uint32_t min_freq, + const std::function &log) + { + // ── 增量 pair 频次统计（避免每轮重扫全部 chunk） ────── + // 用 64 位编码 pair：(a << 32) | b，消除 pair_hash 开销 + std::unordered_map pair_freq; + for (const auto &chunk : chunks) + for (std::size_t i = 0; i + 1 < chunk.size(); ++i) + ++pair_freq[pair_key(chunk[i], chunk[i + 1])]; + + for (std::size_t round = 0; round < target_merges; ++round) + { + if (pair_freq.empty()) break; + + auto best = std::ranges::max_element(pair_freq, + [](const auto &a, const auto &b) { return a.second < b.second; }); + + if (best->second < min_freq) + { + log(" 无更多高频 pair，提前停止于第 " + std::to_string(round) + " 轮"); + break; + } + + const std::uint64_t best_key = best->first; + const std::size_t id_a = static_cast(best_key >> 32); + const std::size_t id_b = static_cast(best_key & 0xFFFFFFFF); + const std::size_t new_id = vocab.size(); + vocab.push_back(vocab[id_a] + vocab[id_b]); + merges_.push_back({id_a, id_b, new_id}); + + // 应用合并到所有 chunk（原地压缩 + 增量更新 pair_freq） + for (auto &chunk : chunks) + { + // 检测本 chunk 是否包含合并 pair + bool has_merge = false; + for (std::size_t i = 0; i + 1 < chunk.size(); ++i) + if (chunk[i] == id_a && chunk[i + 1] == id_b) { has_merge = true; break; } + if (!has_merge) continue; + + // 减去旧 pair 频次 + for (std::size_t i = 0; i + 1 < chunk.size(); ++i) + { + auto key = pair_key(chunk[i], chunk[i + 1]); + auto it = pair_freq.find(key); + if (it != pair_freq.end() && --it->second == 0) + pair_freq.erase(it); + } + + // 原地压缩 + std::size_t write = 0; + std::size_t read = 0; + while (read < chunk.size()) + { + if (read + 1 < chunk.size() && chunk[read] == id_a && chunk[read + 1] == id_b) + { + chunk[write++] = new_id; + read += 2; + } + else + chunk[write++] = chunk[read++]; + } + chunk.resize(write); + + // 加上新 pair 频次 + for (std::size_t i = 0; i + 1 < chunk.size(); ++i) + ++pair_freq[pair_key(chunk[i], chunk[i + 1])]; + } + + if ((round + 1) % 500 == 0) + log(" 合并第 " + std::to_string(round + 1) + " 轮，词表: " + + std::to_string(vocab.size())); + } + } + + // 共享 BPE 合并：对单个 ID 序列应用合并规则（优先队列驱动，O(n log n)）。 + // 使用基类的 merges_ 与 merge_priority_。 + [[nodiscard]] std::vector + bpe_merge_impl_(std::vector ids) const + { + if (ids.size() <= 1) return ids; + + const std::size_t n = ids.size(); + std::vector ll_prev(n), ll_next(n); + for (std::size_t j = 0; j < n; ++j) + { + ll_prev[j] = (j == 0) ? n : j - 1; + ll_next[j] = (j + 1 == n) ? n : j + 1; + } + std::vector alive(n, true); + + using HeapEntry = std::pair; + std::priority_queue, std::greater> heap; + + auto push_pair = [&](std::size_t pos) { + std::size_t nxt = ll_next[pos]; + if (nxt < n && alive[pos] && alive[nxt]) + { + std::size_t prio = merge_priority_(ids[pos], ids[nxt]); + if (prio < merges_.size()) + heap.push({prio, pos}); + } + }; + + for (std::size_t j = 0; j + 1 < n; ++j) + push_pair(j); + + while (!heap.empty()) + { + auto [prio, pos] = heap.top(); + heap.pop(); + + if (!alive[pos]) continue; + std::size_t nxt = ll_next[pos]; + if (nxt >= n || !alive[nxt]) continue; + if (merge_priority_(ids[pos], ids[nxt]) != prio) continue; + + ids[pos] = merges_[prio].new_id; + alive[nxt] = false; + + std::size_t prev = ll_prev[pos]; + std::size_t after = ll_next[nxt]; + ll_next[pos] = after; + if (after < n) ll_prev[after] = pos; + + if (prev < n && alive[prev]) + push_pair(prev); + push_pair(pos); + } + + // 按链表顺序收集结果 + std::vector result; + std::size_t cur = 0; + do { + result.push_back(ids[cur]); + cur = ll_next[cur]; + } while (cur < n); + return result; + } + + // BPE 子类共享的合并规则表与查找表（BPE/CharBPE 使用，其他子类不访问） + std::vector merges_; + std::unordered_map merge_map_; + + // 重建 merge_map_，需要在 merges_ 改变后调用 + void rebuild_merge_map_() + { + merge_map_.clear(); + merge_map_.reserve(merges_.size()); + for (std::size_t i = 0; i < merges_.size(); ++i) + merge_map_[pair_key(merges_[i].id_a, merges_[i].id_b)] = i; + } + + // O(1) 查找：(a, b) → 优先级（merges_ 中的索引），不存在返回 merges_.size() + [[nodiscard]] std::size_t merge_priority_(std::size_t a, std::size_t b) const noexcept + { + auto it = merge_map_.find(pair_key(a, b)); + return it != merge_map_.end() ? it->second : merges_.size(); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // JSON save/load 共享构件 + // ═══════════════════════════════════════════════════════════════════════════ + + // 写入 JSON 头部 + vocab 段 + vocab_size 字段。 + // 子类继续追加差异段（merges / special_tokens / decompose 等）并以 "\n}\n" 结尾。 + // start_id: 起始写入的 token ID（用于跳过特殊 token 槽位）。 + void save_vocab_json_(std::ofstream &ofs, std::string_view type_str, std::size_t start_id) const + { + const auto &v = vocab(); + ofs << "{\n \"type\": \"" << type_str << "\",\n \"vocab\": {\n"; + bool first = true; + for (std::size_t tid = start_id; tid < v.size(); ++tid) + { + if (v[tid].empty()) continue; + if (!first) ofs << ",\n"; + first = false; + ofs << " \"" << std::dec << tid << "\": \""; + write_vocab_entry_(ofs, v[tid], tid); + ofs << "\"" << std::dec; + } + ofs << "\n },\n \"vocab_size\": " << std::dec << v.size(); + } + + // 钩子：默认 hex 编码所有字节。 + // CharBPETokenizer 重写以对前 4 个特殊 token (///) 用纯文本。 + virtual void write_vocab_entry_(std::ofstream &ofs, const std::string &tok, std::size_t /*tid*/) const + { + for (unsigned char b : tok) + ofs << std::hex << std::setw(2) << std::setfill('0') << static_cast(b); + } + + // 从 JSON 内容解析 vocab 段，返回按 ID 索引的词表向量。 + // 调用 decode_vocab_entry_(raw, tid) 解码每个条目（默认 hex，CharBPE 重写）。 + std::vector parse_vocab_json_(const std::string &content) const + { + std::vector vocab; + if (auto p = content.find("\"vocab_size\""); p != std::string::npos) + if (auto c = content.find(':', p); c != std::string::npos) + { + ++c; while (c < content.size() && content[c] == ' ') ++c; + std::size_t v = 0; + if (auto [ptr, ec] = std::from_chars(content.data() + c, content.data() + content.size(), v); ec == std::errc{}) + vocab.resize(v); + } + + auto vp = content.find("\"vocab\""); + if (vp == std::string::npos) return vocab; + auto br = content.find('{', vp); + if (br == std::string::npos) return vocab; + + std::size_t pos = br + 1; + while (pos < content.size()) + { + while (pos < content.size() && (content[pos] == ' ' || content[pos] == '\n' || + content[pos] == '\r' || content[pos] == '\t' || content[pos] == ',')) ++pos; + if (pos >= content.size() || content[pos] == '}') break; + if (content[pos] != '"') { ++pos; continue; } + ++pos; + std::string key; + while (pos < content.size() && content[pos] != '"') key += content[pos++]; + ++pos; + while (pos < content.size() && (content[pos] == ' ' || content[pos] == ':')) ++pos; + if (pos < content.size() && content[pos] == '"') + { + ++pos; + std::string raw; + while (pos < content.size() && content[pos] != '"') + { + if (content[pos] == '\\' && pos + 1 < content.size()) ++pos; + raw += content[pos++]; + } + ++pos; + std::size_t id = 0; + std::from_chars(key.data(), key.data() + key.size(), id); + std::string tok = decode_vocab_entry_(raw, id); + if (id >= vocab.size()) vocab.resize(id + 1); + vocab[id] = std::move(tok); + } + } + return vocab; + } + + // 钩子：默认将 hex 字符串解码为字节序列。 + // CharBPETokenizer 重写以对前 4 个特殊 token 用纯文本。 + virtual std::string decode_vocab_entry_(const std::string &raw, std::size_t /*tid*/) const + { + std::string tok; + tok.reserve(raw.size() / 2); + for (std::size_t i = 0; i + 1 < raw.size(); i += 2) + { + unsigned b = 0; + std::from_chars(raw.data() + i, raw.data() + i + 2, b, 16); + tok += static_cast(b); + } + return tok; + } +}; + + +// ═══════════════════════════════════════════════════════════════════════════ +// WordZip 词级分词器 +// ═══════════════════════════════════════════════════════════════════════════ +// 基于词频统计的词级分词器。 +// Header-only，零外部依赖。 +// +// 词表构成: +// ID 0~3 特殊 token (, , , ) +// ID 4~259 256 个单字节（用于未知字符回退） +// ID 260+ 高频完整词 +// +// 训练: 预分词 → 词频统计 → 按频次降序填充词表 +// 编码: 预分词 → 逐词查表 → 未知词逐字符回退到单字节 token +// JSON 格式与原 ByteZip 保持兼容。 +// ═══════════════════════════════════════════════════════════════════════════ + +class WordZipTokenizer : public Tokenizer +{ +public: + // ── 特殊 token ──────────────────────────────────────────────────── + static constexpr std::size_t BYTE_OFFSET = 4; + static constexpr std::size_t PAD_ID = 0; + static constexpr std::size_t UNK_ID = 1; + static constexpr std::size_t BOS_ID = 2; + static constexpr std::size_t EOS_ID = 3; + + // ── 训练默认参数 ────────────────────────────────────────────────── + static constexpr std::size_t DEFAULT_VOCAB_SIZE = 20000; + static constexpr std::size_t DEFAULT_V1_MAX_LEN = 16; + static constexpr std::uint32_t DEFAULT_MIN_FREQ = 2; + + // ── 日志回调（默认 cout，nullptr 静默） ──────────────────────────── + using LogFn = std::function; + + // ── 训练配置 ────────────────────────────────────────────────────── + struct Config + { + std::size_t vocab_size = DEFAULT_VOCAB_SIZE; + std::uint32_t min_freq = DEFAULT_MIN_FREQ; + LogFn log = nullptr; + }; + + // ── 词缀分析结果 ───────────────────────────────────────────────── + struct AffixInfo { + std::string affix; // 词缀本身 + std::size_t count; // 包含此词缀的词数 + std::size_t stem_found; // 词根也在词表中的词数 + }; + + struct WordSplit { + std::string word; // 原词 + std::string stem; // 词根 + std::string affix; // 词缀 + bool is_suffix; // true=后缀, false=前缀 + }; + + // ══════════════════════════════════════════════════════════════════ + // 公开接口 + // ══════════════════════════════════════════════════════════════════ + + WordZipTokenizer() = default; + + [[nodiscard]] std::size_t bos_id() const noexcept override { return BOS_ID; } + [[nodiscard]] std::size_t eos_id() const noexcept override { return EOS_ID; } + +public: + Result train(const std::string &text) { return train(text, Config{}); } + + // ── 词频训练 ────────────────────────────────────────────────────── + Result train(const std::string &text, const Config &config) + { + auto log = config.log + ? config.log + : LogFn{[](std::string_view) { /* 静默 */ }}; + + const auto min_freq = config.min_freq; + const auto target = config.vocab_size; + + // ── 预分词 → 词频统计 ──────────────────────────────────────── + log("\n[训练] 预分词..."); + auto chunks = pre_tokenize(text); + + std::unordered_map freq; + freq.reserve(chunks.size()); + for (const auto &chunk : chunks) + ++freq[chunk]; + log(" 不同词数: " + std::to_string(freq.size())); + + // 按频次降序排序 + struct Entry { std::string word; std::uint32_t cnt; }; + std::vector sorted; + sorted.reserve(freq.size()); + for (const auto &[w, c] : freq) + sorted.push_back({w, c}); + std::ranges::sort(sorted, [](const Entry &a, const Entry &b) + { return a.cnt > b.cnt; }); + + // 初始化词表：特殊 token + 256 单字节 + vocab_.clear(); + vocab_.resize(BYTE_OFFSET); + for (std::size_t i = 0; i < 256; ++i) + vocab_.emplace_back(1, static_cast(i)); + + // 按频次填充词表 + std::size_t added = 0; + for (const auto &e : sorted) + { + if (vocab_.size() >= target) break; + if (e.cnt < min_freq) break; + if (e.word.size() <= 1) continue; // 单字符已在单字节表中 + vocab_.push_back(e.word); + ++added; + if (added % 1000 == 0) + log(" 已添加 " + std::to_string(added) + " 个词条（总 " + + std::to_string(vocab_.size()) + "）"); + } + + // ── 添加对话标记 token ────────────────────────────────────── + add_dialogue_markers_to_vocab_(vocab_); + + // ── 最终查找表 ─────────────────────────────────────────────── + log("\n最终词表: " + std::to_string(vocab_.size())); + max_subword_len_ = 0; + for (std::size_t i = BYTE_OFFSET; i < vocab_.size(); ++i) + if (vocab_[i].size() > max_subword_len_) + max_subword_len_ = vocab_[i].size(); + build_lookup(); + + // ── 词缀分析 + 词分解 ─────────────────────────────────────── + analyze_affixes(); + build_decomposition(); + + // ── 删除可分解词 ─────────────────────────────────────────── + std::size_t removed = 0; + std::unordered_set to_remove; + for (const auto &[word, parts] : decompose_map_) + to_remove.insert(word); + + // 从 vocab_ 中移除可分解词（保留 ID 槽，置空） + for (std::size_t i = BYTE_OFFSET; i < vocab_.size(); ++i) + { + if (to_remove.contains(vocab_[i])) + { + vocab_[i].clear(); + ++removed; + } + } + + // 重建查找表（移除的词不再出现在 lookup_ 中） + build_lookup(); + + log(" 已移除可分解词: " + std::to_string(removed) + " 个"); + log("最终词表（有效）: " + std::to_string(vocab_.size() - removed)); + return {}; + } + + // ── 编码 ────────────────────────────────────────────────────────── + [[nodiscard]] std::vector encode(const std::string &text) const override + { + std::vector all; + all.reserve(text.size() / 2); + std::size_t pos = 0; + while (pos < text.size()) + { + // 对话标记优先匹配 + if (has_dialogue_markers()) + { + auto [mid, mlen] = try_match_marker(text, pos); + if (mid != Tokenizer::npos) + { all.push_back(mid); pos += mlen; continue; } + } + // 普通文本按预分词分块 + if (auto n = WordZipTokenizer::try_contraction(text, pos); n > 0) + { all.push_back(UNK_ID); pos += n; continue; } + auto s = pos; + while (pos < text.size()) + { + if (has_dialogue_markers()) + { auto [mid, mlen] = try_match_marker(text, pos); if (mid != Tokenizer::npos) break; } + if (WordZipTokenizer::try_contraction(text, pos) > 0 && pos > s) break; + ++pos; + } + if (pos > s) + { + auto ids = encode_bytes(std::string_view{text.data() + s, pos - s}); + all.insert(all.end(), ids.begin(), ids.end()); + } + } + return all; + } + + // ── 解码 ────────────────────────────────────────────────────────── + [[nodiscard]] std::string decode(std::span ids) const override + { + std::string raw; + raw.reserve(ids.size() * 2); + for (auto tid : ids) + { + // 对话标记 → 原始文本 + if (tid == markers_.system) { raw += ""; continue; } + if (tid == markers_.system_end) { raw += ""; continue; } + if (tid == markers_.user) { raw += ""; continue; } + if (tid == markers_.user_end) { raw += ""; continue; } + if (tid == markers_.assistant) { raw += ""; continue; } + if (tid == markers_.assistant_end) { raw += ""; continue; } + raw += tid < vocab_.size() ? vocab_[tid] : std::string{REPLACEMENT_CHAR}; + } + return raw; + } + + // ── 保存 JSON ──────────────────────────────────────────────────── + [[nodiscard]] Result save(const std::string &path) const override + { + std::ofstream ofs(path); + if (!ofs) return std::unexpected(Error{"Cannot write: " + path}); + + ofs << "{\n \"type\": \"wordzip_tokenizer\",\n \"vocab\": {\n"; + bool first = true; + for (std::size_t tid = BYTE_OFFSET; tid < vocab_.size(); ++tid) + { + if (vocab_[tid].empty()) continue; + if (!first) ofs << ",\n"; + first = false; + ofs << " \"" << std::dec << tid << "\": \""; + for (unsigned char b : vocab_[tid]) + ofs << std::hex << std::setw(2) << std::setfill('0') << static_cast(b); + ofs << "\"" << std::dec; + } + ofs << "\n },\n \"vocab_size\": " << std::dec << vocab_.size() + << ",\n \"byte_offset\": " << BYTE_OFFSET + << ",\n \"max_subword_len\": " << max_subword_len_ + << ",\n \"decompose\": {"; + { + bool f = true; + for (const auto &[word, parts] : decompose_map_) + { + ofs << (f ? "\n" : ",\n"); + f = false; + ofs << " \""; + for (unsigned char b : word) + ofs << std::hex << std::setw(2) << std::setfill('0') << static_cast(b); + ofs << "\": ["; + for (std::size_t pi = 0; pi < parts.size(); ++pi) + { + if (pi > 0) ofs << ", "; + ofs << "\""; + for (unsigned char b : parts[pi]) + ofs << std::hex << std::setw(2) << std::setfill('0') << static_cast(b); + ofs << "\""; + } + ofs << "]"; + } + } + ofs << "\n },\n \"special_tokens\": {" + << "\n \"\": 0,\n \"\": 1," + << "\n \"\": 2,\n \"\": 3\n }\n}\n"; + return {}; + } + + // ── 加载 JSON ──────────────────────────────────────────────────── + [[nodiscard]] Result load(const std::string &path) override + { + std::ifstream ifs(path); + if (!ifs) return std::unexpected(Error{"Cannot read: " + path}); + std::string content((std::istreambuf_iterator(ifs)), + std::istreambuf_iterator()); + return load_from_string(content); + } + + [[nodiscard]] Result load_from_string(const std::string &content) override + { + vocab_.clear(); + + if (auto p = content.find("\"vocab_size\""); p != std::string::npos) + if (auto c = content.find(':', p); c != std::string::npos) + { + ++c; while (c < content.size() && content[c] == ' ') ++c; + std::size_t v = 0; + if (auto [ptr, ec] = std::from_chars(content.data()+c, content.data()+content.size(), v); ec == std::errc{}) + vocab_.resize(v); + } + + auto vp = content.find("\"vocab\""); + if (vp == std::string::npos) + return std::unexpected(Error{"Invalid JSON: missing \"vocab\""}); + auto br = content.find('{', vp); + if (br == std::string::npos) + return std::unexpected(Error{"Invalid JSON: missing '{'"}); + + std::size_t pos = br + 1; + while (pos < content.size()) + { + while (pos < content.size() && (content[pos]==' '||content[pos]=='\n'|| + content[pos]=='\r'||content[pos]=='\t'||content[pos]==',')) ++pos; + if (pos >= content.size() || content[pos] == '}') break; + if (content[pos] != '"') { ++pos; continue; } + ++pos; + std::string key; + while (pos < content.size() && content[pos] != '"') key += content[pos++]; + ++pos; + while (pos < content.size() && (content[pos]==' '||content[pos]==':')) ++pos; + if (pos < content.size() && content[pos] == '"') + { + ++pos; + std::string hex; + while (pos < content.size() && content[pos] != '"') hex += content[pos++]; + ++pos; + std::size_t id = 0; + std::from_chars(key.data(), key.data()+key.size(), id); + std::string tok; + tok.reserve(hex.size()/2); + for (std::size_t i = 0; i+1 < hex.size(); i += 2) + { unsigned b = 0; std::from_chars(hex.data()+i, hex.data()+i+2, b, 16); tok += static_cast(b); } + if (id >= vocab_.size()) vocab_.resize(id+1); + vocab_[id] = std::move(tok); + } + } + + max_subword_len_ = DEFAULT_V1_MAX_LEN; + if (auto p = content.find("\"max_subword_len\""); p != std::string::npos) + if (auto c = content.find(':', p); c != std::string::npos) + { + ++c; while (c < content.size() && content[c] == ' ') ++c; + std::size_t v = 0; + std::from_chars(content.data()+c, content.data()+content.size(), v); + if (v > 0) max_subword_len_ = v; + } + + // ── 恢复对话标记 ID（从已有词表检测或追加） ───────────── + restore_dialogue_markers(vocab_); + build_lookup(); + + // ── 加载分解规则 ────────────────────────────────────────────── + decompose_map_.clear(); + if (auto dp = content.find("\"decompose\""); dp != std::string::npos) + { + auto dbr = content.find('{', dp); + if (dbr != std::string::npos) + { + std::size_t dpos = dbr + 1; + while (dpos < content.size()) + { + while (dpos < content.size() && (content[dpos]==' '||content[dpos]=='\n'|| + content[dpos]=='\r'||content[dpos]=='\t'||content[dpos]==',')) ++dpos; + if (dpos >= content.size() || content[dpos] == '}') break; + if (content[dpos] != '"') { ++dpos; continue; } + ++dpos; + std::string key_hex; + while (dpos < content.size() && content[dpos] != '"') key_hex += content[dpos++]; + ++dpos; + // skip to '[' + while (dpos < content.size() && content[dpos] != '[') ++dpos; + ++dpos; + // read array elements + std::vector parts; + while (dpos < content.size()) + { + while (dpos < content.size() && (content[dpos]==' '||content[dpos]=='\n'|| + content[dpos]=='\r'||content[dpos]=='\t'||content[dpos]==',')) ++dpos; + if (dpos >= content.size() || content[dpos] == ']') break; + if (content[dpos] == '"') + { + ++dpos; + std::string hex; + while (dpos < content.size() && content[dpos] != '"') hex += content[dpos++]; + ++dpos; + std::string part; + for (std::size_t h = 0; h+1 < hex.size(); h += 2) + { unsigned b = 0; std::from_chars(hex.data()+h, hex.data()+h+2, b, 16); part += static_cast(b); } + parts.push_back(std::move(part)); + } + else ++dpos; + } + ++dpos; // skip ']' + std::string word; + for (std::size_t h = 0; h+1 < key_hex.size(); h += 2) + { unsigned b = 0; std::from_chars(key_hex.data()+h, key_hex.data()+h+2, b, 16); word += static_cast(b); } + if (!parts.empty()) + decompose_map_[std::move(word)] = std::move(parts); + } + } + } + + analyze_affixes(); + return {}; + } + + // ══════════════════════════════════════════════════════════════════ + // 访问器 + // ══════════════════════════════════════════════════════════════════ + + [[nodiscard]] constexpr std::size_t vocab_size() const noexcept override { return vocab_.size(); } + [[nodiscard]] constexpr std::size_t max_subword_len() const noexcept { return max_subword_len_; } + [[nodiscard]] constexpr std::size_t byte_offset() const noexcept { return BYTE_OFFSET; } + [[nodiscard]] const std::vector &vocab() const noexcept override { return vocab_; } + + [[nodiscard]] Result token_bytes(std::size_t id) const + { + if (id >= vocab_.size()) return std::unexpected(Error{"Token ID out of range"}); + return vocab_[id]; + } + + [[nodiscard]] std::string try_decode_token(std::size_t id) const + { + if (id >= vocab_.size()) return ""; + std::string r; + for (char c : vocab_[id]) + { + if (c >= 32 && c < 127) r += c; + else if (c == '\n') r += "\\n"; + else if (c == '\t') r += "\\t"; + else if (c == ' ') r += "␣"; + else { r += "\\x"; r += "0123456789abcdef"[(static_cast(c)>>4)&0xF]; + r += "0123456789abcdef"[ static_cast(c) &0xF]; } + } + return r; + } + + // ── 词缀分析（纯数据驱动，无硬编码词缀） ──────────────────────── + void analyze_affixes() + { + suffixes_.clear(); + prefixes_.clear(); + splits_.clear(); + + // 构建词表集合（O(1) 查找） + std::unordered_set vocab_set; + for (std::size_t i = BYTE_OFFSET; i < vocab_.size(); ++i) + if (!vocab_[i].empty() && vocab_[i].size() >= 2) + vocab_set.insert(vocab_[i]); + + // ── 后缀检测 ──────────────────────────────────────────────── + // 遍历每个词的所有可能后缀(长度2~5)，统计频率与词根覆盖率 + std::unordered_map> suffix_stats; + for (const auto &word : vocab_) + { + if (word.size() < MIN_WORD_LEN) continue; + if (!std::ranges::all_of(word, is_alpha)) continue; + + for (std::size_t slen = MIN_AFFIX_LEN; + slen <= std::min(MAX_AFFIX_LEN, word.size() - MIN_AFFIX_LEN); ++slen) + { + auto stem = word.substr(0, word.size() - slen); + auto suffix = word.substr(word.size() - slen); + auto &[count, stem_found] = suffix_stats[suffix]; + ++count; + if (vocab_set.contains(stem)) + ++stem_found; + } + } + + for (const auto &[suffix, stats] : suffix_stats) + { + auto [count, stem_found] = stats; + if (count >= MIN_AFFIX_COUNT && + static_cast(stem_found) / count >= MIN_COVERAGE) + suffixes_.push_back({suffix, count, stem_found}); + } + std::ranges::sort(suffixes_, [](const AffixInfo &a, const AffixInfo &b) + { return a.count > b.count; }); + + // ── 前缀检测 ──────────────────────────────────────────────── + // 同理统计词首 n-gram 的频率与词根覆盖率 + std::unordered_map> prefix_stats; + for (const auto &word : vocab_) + { + if (word.size() < MIN_WORD_LEN) continue; + if (!std::ranges::all_of(word, is_alpha)) continue; + + for (std::size_t plen = MIN_AFFIX_LEN; + plen <= std::min(std::size_t{4}, word.size() - MIN_AFFIX_LEN); ++plen) + { + auto prefix = word.substr(0, plen); + auto stem = word.substr(plen); + auto &[count, stem_found] = prefix_stats[prefix]; + ++count; + if (vocab_set.contains(stem)) + ++stem_found; + } + } + + for (const auto &[prefix, stats] : prefix_stats) + { + auto [count, stem_found] = stats; + if (count >= MIN_AFFIX_COUNT && + static_cast(stem_found) / count >= MIN_COVERAGE) + prefixes_.push_back({prefix, count, stem_found}); + } + std::ranges::sort(prefixes_, [](const AffixInfo &a, const AffixInfo &b) + { return a.count > b.count; }); + + // ── 词拆分 ────────────────────────────────────────────────── + // 对每个词：优先匹配最长后缀 → 其次匹配最长前缀 + for (const auto &word : vocab_) + { + if (word.size() < MIN_WORD_LEN) continue; + if (!std::ranges::all_of(word, is_alpha)) continue; + + // 优先尝试后缀（从最长到最短） + bool found = false; + for (const auto &affix : suffixes_) + { + if (word.size() <= affix.affix.size()) continue; + if (word.ends_with(affix.affix)) + { + auto stem = word.substr(0, word.size() - affix.affix.size()); + if (stem.size() >= 2 && vocab_set.contains(stem)) + { + splits_.push_back({word, stem, affix.affix, true}); + found = true; + break; + } + } + } + if (found) continue; + + // 尝试前缀 + for (const auto &affix : prefixes_) + { + if (word.size() <= affix.affix.size()) continue; + if (word.starts_with(affix.affix)) + { + auto stem = word.substr(affix.affix.size()); + if (stem.size() >= 2 && vocab_set.contains(stem)) + { + splits_.push_back({word, stem, affix.affix, false}); + break; + } + } + } + } + } + + // ── 词分解：检测词表中可拆分为两个词表词的组合 ────────────────── + // 纯数据驱动：只要 A+B 都在词表中且长度合理，就记录为可分解 + void build_decomposition() + { + decompose_map_.clear(); + + // 构建词表集合 + std::unordered_set vocab_set; + for (std::size_t i = BYTE_OFFSET; i < vocab_.size(); ++i) + if (!vocab_[i].empty()) + vocab_set.insert(vocab_[i]); + + for (const auto &word : vocab_) + { + if (word.size() < 4) continue; + + // 剥离前导非字母（如空格），保留前缀字符串以便还原 + std::string prefix; + std::string word_body; + + if (!is_alpha(word.front())) + { + auto first_alpha = static_cast(std::ranges::distance( + word.begin(), std::ranges::find_if(word, is_alpha))); + if (first_alpha > 0 && first_alpha < word.size()) + { + prefix = word.substr(0, first_alpha); + word_body = word.substr(first_alpha); + } + else + { + word_body = word; + } + } + else + { + word_body = word; + } + + if (word_body.size() < 4 || !std::ranges::all_of(word_body, is_alpha)) + continue; + + // 找最优二分：偏好更长、更均衡的拆分 + std::size_t best_min = 0; + std::vector best_parts; + + for (std::size_t pos = 1; pos < word_body.size(); ++pos) + { + auto a = word_body.substr(0, pos); + auto b = word_body.substr(pos); + + if (vocab_set.contains(a) && vocab_set.contains(b)) + { + auto m = std::min(pos, word_body.size() - pos); + if (m >= 2 && m > best_min) + { + best_min = m; + // 还原：前缀附加到第一部分 + best_parts = {prefix + a, b}; + } + } + } + + if (!best_parts.empty()) + decompose_map_[word] = std::move(best_parts); + } + } + + // ── 打印词缀分析报告 ────────────────────────────────────────────── + void print_affix_report(std::ostream &os = std::cout) const + { + os << "\n══════════════════════════════════════════════\n"; + os << " 词缀分析报告\n"; + os << "══════════════════════════════════════════════\n"; + + // ── 后缀表 ────────────────────────────────────────────────── + os << "\n后缀表 (共 " << suffixes_.size() << " 个):\n"; + for (const auto &s : suffixes_) + { + std::string examples; + std::size_t n = 0; + for (const auto &sp : splits_) + { + if (sp.is_suffix && sp.affix == s.affix) + { + if (n > 0) examples += ", "; + examples += sp.stem + "+" + sp.affix; + if (++n >= 3) break; + } + } + double cov = 100.0 * s.stem_found / s.count; + os << " " << s.affix << " ×" << s.count + << " 覆盖" << std::fixed << std::setprecision(1) << cov << "%" + << " 例: " << examples << "\n"; + } + + // ── 前缀表 ────────────────────────────────────────────────── + os << "\n前缀表 (共 " << prefixes_.size() << " 个):\n"; + for (const auto &p : prefixes_) + { + std::string examples; + std::size_t n = 0; + for (const auto &sp : splits_) + { + if (!sp.is_suffix && sp.affix == p.affix) + { + if (n > 0) examples += ", "; + examples += p.affix + "+" + sp.stem; + if (++n >= 3) break; + } + } + double cov = 100.0 * p.stem_found / p.count; + os << " " << p.affix << " ×" << p.count + << " 覆盖" << std::fixed << std::setprecision(1) << cov << "%" + << " 例: " << examples << "\n"; + } + + // ── 词拆分表 ──────────────────────────────────────────────── + os << "\n词拆分表 (共 " << splits_.size() << " 个):\n"; + for (const auto &sp : splits_) + { + os << " " << sp.word << " → " << sp.stem + << " + " << (sp.is_suffix ? "" : "[前缀]") << sp.affix << "\n"; + } + + // ── 词分解规则 ──────────────────────────────────────────────── + os << "\n词分解规则 (共 " << decompose_map_.size() << " 个):\n"; + for (const auto &[word, parts] : decompose_map_) + { + os << " " << word << " → "; + for (std::size_t i = 0; i < parts.size(); ++i) + { + if (i > 0) os << " + "; + os << parts[i]; + } + os << "\n"; + } + + os << "\n══════════════════════════════════════════════\n"; + } + + // ── 访问器（词缀分析结果） ──────────────────────────────────────── + [[nodiscard]] const std::vector &suffixes() const noexcept { return suffixes_; } + [[nodiscard]] const std::vector &prefixes() const noexcept { return prefixes_; } + [[nodiscard]] const std::vector &splits() const noexcept { return splits_; } + [[nodiscard]] const std::unordered_map> &decompose_map() const noexcept { return decompose_map_; } + + // ── 预分词（公开，供外部复用） ──────────────────────────────────── + [[nodiscard]] static std::vector pre_tokenize(std::string_view text) + { + std::vector chunks; + chunks.reserve(text.size() / 4); + std::size_t pos = 0; + while (pos < text.size()) + { + if (auto n = try_contraction(text, pos); n > 0) + { chunks.emplace_back(text.substr(pos, n)); pos += n; continue; } + + const char c = text[pos]; + + if (is_alpha(c) || (c==' ' && pos+1 vocab_; + std::size_t max_subword_len_ = DEFAULT_V1_MAX_LEN; + + // 透明哈希查找表（编码热路径） + struct TransparentHash { + using is_transparent = void; + [[nodiscard]] std::size_t operator()(std::string_view s) const noexcept { return std::hash{}(s); } + [[nodiscard]] std::size_t operator()(const std::string &s) const noexcept { return std::hash{}(s); } + }; + struct TransparentEqual { + using is_transparent = void; + [[nodiscard]] bool operator()(std::string_view a, std::string_view b) const noexcept { return a==b; } + }; + using LookupTable = std::unordered_map; + LookupTable lookup_; + + // ── 词缀分析参数与结果 ──────────────────────────────────────────── + static constexpr std::size_t MIN_AFFIX_LEN = 2; // 最短词缀长度 + static constexpr std::size_t MAX_AFFIX_LEN = 5; // 最长词缀长度 + static constexpr std::size_t MIN_WORD_LEN = 4; // 词根至少 2 字符 + static constexpr std::size_t MIN_AFFIX_COUNT = 3; // 最少出现词数 + static constexpr double MIN_COVERAGE = 0.5; // 词根覆盖率阈值 + + std::vector suffixes_; + std::vector prefixes_; + std::vector splits_; + std::unordered_map> decompose_map_; + + // ── 字符分类 ────────────────────────────────────────────────────── + [[nodiscard]] static constexpr bool is_alpha(char c) noexcept { return (c>='a'&&c<='z')||(c>='A'&&c<='Z'); } + [[nodiscard]] static constexpr bool is_digit(char c) noexcept { return c>='0'&&c<='9'; } + [[nodiscard]] static constexpr bool is_whitespace(char c) noexcept { return c==' '||c=='\t'||c=='\n'||c=='\r'||c=='\f'||c=='\v'; } + [[nodiscard]] static constexpr bool is_word_char(char c) noexcept { return is_alpha(c)||is_digit(c)||c=='_'; } + + [[nodiscard]] static constexpr std::size_t try_contraction(std::string_view t, std::size_t p) noexcept + { + if (p>=t.size()||t[p]!='\'') return 0; + auto r = t.size()-p; if (r<2) return 0; + auto n1 = t[p+1]; + if (n1=='s'||n1=='t'||n1=='m'||n1=='d') return 2; + if (r>=3) { auto n2=t[p+2]; + if ((n1=='r'&&n2=='e')||(n1=='v'&&n2=='e')||(n1=='l'&&n2=='l')) return 3; } + return 0; + } + + // ── 编码（内部，用于 encode()） ─────────────────────────────────── + // ── decompose_map → byte fallback → 整词查表 + [[nodiscard]] std::vector encode_bytes(std::string_view data) const + { + // 1. 优先查分解规则（被移除的可分解词走这里） + auto dec_it = decompose_map_.find(std::string(data)); + if (dec_it != decompose_map_.end()) + { + std::vector ids; + for (const auto &part : dec_it->second) + { + auto part_ids = encode_bytes(part); + ids.insert(ids.end(), part_ids.begin(), part_ids.end()); + } + return ids; + } + + // 2. 整词匹配 + if (auto it = lookup_.find(data); it != lookup_.end()) + return {it->second}; + + // 3. 逐字符回退到单字节 token + std::vector ids; + ids.reserve(data.size()); + for (unsigned char b : data) + ids.push_back(static_cast(b) + BYTE_OFFSET); + return ids; + } + + // ── 构建查找表 ──────────────────────────────────────────────────── + void build_lookup() + { + lookup_.clear(); + lookup_.reserve(vocab_.size() - BYTE_OFFSET); + for (std::size_t i = BYTE_OFFSET; i < vocab_.size(); ++i) + if (!vocab_[i].empty()) lookup_[vocab_[i]] = i; + } + +}; + +// ═══════════════════════════════════════════════════════════════════════════ +// SpaceTokenizer — 基于空格的分词器 +// 注：未在生产线使用，仅保留作对比实现（与 WordZip/BPE 对照分词效果）。 +// ═══════════════════════════════════════════════════════════════════════════ +// 训练：按空格分词 → 词频统计 → 截断 top-N 填充词表 +// 编码：空格分词 → 查词表 → 未命中则逐字符 ASCII 回退 +// 解码：多字节 token 空格分隔，单字节 token 直接拼接 +// ═══════════════════════════════════════════════════════════════════════════ + +class SpaceTokenizer : public Tokenizer +{ +public: + static constexpr std::size_t UNK_ID = 0; + static constexpr std::size_t PAD_ID = 1; + static constexpr std::size_t NUM_ID = 2; + static constexpr std::size_t ASCII_BASE = 3; + static constexpr std::size_t DEFAULT_VOCAB_SIZE = 10000; + static constexpr std::uint32_t DEFAULT_MIN_FREQ = 2; + + using LogFn = std::function; + + [[nodiscard]] std::size_t pad_id() const noexcept override { return PAD_ID; } + + struct Config + { + std::size_t vocab_size = DEFAULT_VOCAB_SIZE; + std::uint32_t min_freq = DEFAULT_MIN_FREQ; + LogFn log = nullptr; + }; + + SpaceTokenizer() = default; + + Result train(const std::string &text) { return train(text, Config{}); } + + Result train(const std::string &text, const Config &config) + { + auto log = config.log + ? config.log + : LogFn{[](std::string_view) { /* 静默 */ }}; + + log("\n[Space 训练] 分词统计..."); + + std::unordered_map freq; + { + std::istringstream iss(text); + std::string word; + while (iss >> word) + ++freq[word]; + } + log(" 不同词数: " + std::to_string(freq.size())); + + struct Entry { std::string word; std::uint32_t cnt; }; + std::vector sorted; + sorted.reserve(freq.size()); + for (const auto &[w, c] : freq) + sorted.push_back({w, c}); + std::ranges::sort(sorted, [](const Entry &a, const Entry &b) + { return a.cnt > b.cnt; }); + + id_to_token_.clear(); + id_to_token_.resize(ASCII_BASE); + // 修复：原代码只添加 128 个 ASCII 字符，导致非 ASCII 字节（如中文 UTF-8） + // 在 encode() 中 id >= ASCII_BASE+128 超出词表，全部回退到 UNK_ID， + // 丢失原文信息。改为添加全部 256 个单字节，使任意 UTF-8 字节都能 + // 映射到唯一 token ID（与 WordZipTokenizer 的字节回退设计一致）。 + for (std::size_t i = 0; i < 256; ++i) + id_to_token_.emplace_back(1, static_cast(static_cast(i))); + + std::size_t added = 0; + for (const auto &e : sorted) + { + if (id_to_token_.size() >= config.vocab_size) break; + if (e.cnt < config.min_freq) break; + if (e.word.size() <= 1) continue; + id_to_token_.push_back(e.word); + if (++added % 1000 == 0) + log(" 已添加 " + std::to_string(added) + " 个词"); + } + + // ── 添加对话标记 token ────────────────────────────────────── + add_dialogue_markers_to_vocab_(id_to_token_); + build_lookup(); + log("最终词表: " + std::to_string(id_to_token_.size())); + return {}; + } + + [[nodiscard]] std::vector encode(const std::string &text) const override + { + std::vector tokens; + std::size_t pos = 0; + while (pos < text.size()) + { + // 对话标记优先匹配 + if (has_dialogue_markers()) + { + auto [mid, mlen] = try_match_marker(text, pos); + if (mid != Tokenizer::npos) + { tokens.push_back(mid); pos += mlen; continue; } + } + // 跳过空白 + if (text[pos] == ' ' || text[pos] == '\t' || text[pos] == '\n' || text[pos] == '\r') + { ++pos; continue; } + // 提取一个词 + auto s = pos; + while (pos < text.size() && text[pos] != ' ' && text[pos] != '\t' && + text[pos] != '\n' && text[pos] != '\r') + { + if (has_dialogue_markers()) + { auto [mid, mlen] = try_match_marker(text, pos); if (mid != Tokenizer::npos) break; } + ++pos; + } + std::string word(text, s, pos - s); + if (word.empty()) continue; + if (is_numeric(word)) + tokens.push_back(NUM_ID); + else if (auto it = token_to_id_.find(word); it != token_to_id_.end()) + tokens.push_back(it->second); + else + for (unsigned char c : word) + { + auto id = static_cast(c) + ASCII_BASE; + tokens.push_back(id < id_to_token_.size() ? id : UNK_ID); + } + } + return tokens; + } + + [[nodiscard]] std::string decode(std::span ids) const override + { + std::string result; + for (auto id : ids) + { + // 对话标记 → 原始文本 + if (id == markers_.system) { result += ""; continue; } + if (id == markers_.system_end) { result += ""; continue; } + if (id == markers_.user) { result += ""; continue; } + if (id == markers_.user_end) { result += ""; continue; } + if (id == markers_.assistant) { result += ""; continue; } + if (id == markers_.assistant_end) { result += ""; continue; } + if (id < id_to_token_.size()) + { + const auto &tok = id_to_token_[id]; + if (tok.size() > 1) { if (!result.empty()) result += ' '; result += tok; } + else result += tok; + } + else { if (!result.empty()) result += ' '; result += ""; } + } + return result; + } + + [[nodiscard]] Result save(const std::string &path) const override + { + std::ofstream ofs(path); + if (!ofs) return std::unexpected(Error{"Cannot write: " + path}); + ofs << "{\n \"type\": \"space_tokenizer\",\n \"vocab\": {\n"; + bool first = true; + for (std::size_t tid = ASCII_BASE; tid < id_to_token_.size(); ++tid) + { + if (id_to_token_[tid].empty()) continue; + if (!first) ofs << ",\n"; + first = false; + ofs << " \"" << std::dec << tid << "\": \""; + for (unsigned char b : id_to_token_[tid]) + ofs << std::hex << std::setw(2) << std::setfill('0') << static_cast(b); + ofs << "\"" << std::dec; + } + ofs << "\n },\n \"vocab_size\": " << std::dec << id_to_token_.size() + << ",\n \"special_tokens\": {" + << "\n \"\": 0,\n \"\": 1,\n \"\": 2\n }\n}\n"; + return {}; + } + + [[nodiscard]] Result load(const std::string &path) override + { + std::ifstream ifs(path); + if (!ifs) return std::unexpected(Error{"Cannot read: " + path}); + std::string content((std::istreambuf_iterator(ifs)), + std::istreambuf_iterator()); + return load_from_string(content); + } + + [[nodiscard]] Result load_from_string(const std::string &content) override + { + id_to_token_.clear(); + token_to_id_.clear(); + + if (auto p = content.find("\"vocab_size\""); p != std::string::npos) + if (auto c = content.find(':', p); c != std::string::npos) + { + ++c; while (c < content.size() && content[c] == ' ') ++c; + std::size_t v = 0; + if (auto [ptr, ec] = std::from_chars(content.data()+c, content.data()+content.size(), v); ec == std::errc{}) + id_to_token_.resize(v); + } + + auto vp = content.find("\"vocab\""); + if (vp == std::string::npos) + return std::unexpected(Error{"Invalid JSON: missing \"vocab\""}); + auto br = content.find('{', vp); + if (br == std::string::npos) + return std::unexpected(Error{"Invalid JSON: missing '{'"}); + + std::size_t pos = br + 1; + while (pos < content.size()) + { + while (pos < content.size() && (content[pos]==' '||content[pos]=='\n'|| + content[pos]=='\r'||content[pos]=='\t'||content[pos]==',')) ++pos; + if (pos >= content.size() || content[pos] == '}') break; + if (content[pos] != '"') { ++pos; continue; } + ++pos; std::string key; + while (pos < content.size() && content[pos] != '"') key += content[pos++]; + ++pos; + while (pos < content.size() && (content[pos]==' '||content[pos]==':')) ++pos; + if (pos < content.size() && content[pos] == '"') + { + ++pos; std::string hex; + while (pos < content.size() && content[pos] != '"') hex += content[pos++]; + ++pos; + std::size_t id = 0; + std::from_chars(key.data(), key.data()+key.size(), id); + std::string tok; + for (std::size_t i = 0; i+1 < hex.size(); i += 2) + { unsigned b = 0; std::from_chars(hex.data()+i, hex.data()+i+2, b, 16); tok += static_cast(b); } + if (id >= id_to_token_.size()) id_to_token_.resize(id+1); + id_to_token_[id] = std::move(tok); + } + } + restore_dialogue_markers(id_to_token_); + build_lookup(); + return {}; + } + + [[nodiscard]] std::size_t vocab_size() const noexcept override { return id_to_token_.size(); } + [[nodiscard]] const std::vector &vocab() const noexcept override { return id_to_token_; } + +private: + std::vector id_to_token_; + std::unordered_map token_to_id_; + + [[nodiscard]] static bool is_numeric(const std::string &s) noexcept + { + if (s.empty()) return false; + for (char c : s) if (c < '0' || c > '9') return false; + return true; + } + + void build_lookup() + { + token_to_id_.clear(); + token_to_id_.reserve(id_to_token_.size()); + for (std::size_t i = 0; i < id_to_token_.size(); ++i) + if (!id_to_token_[i].empty()) + token_to_id_[id_to_token_[i]] = i; + } +}; + + +// ═══════════════════════════════════════════════════════════════════════════ +// BPETokenizer — 真正的 Byte-Pair Encoding 分词器 +// ═══════════════════════════════════════════════════════════════════════════ +// 训练：GPT-2 风格预分词 → 迭代 byte-pair 合并 +// 编码：预分词 → 按合并优先级对字节序列进行合并 +// 解码：直接拼接字节 +// ═══════════════════════════════════════════════════════════════════════════ + +class BPETokenizer : public Tokenizer +{ +public: + static constexpr std::size_t BYTE_BASE = 256; + static constexpr std::size_t BOS_ID = BYTE_BASE; // 256 + static constexpr std::size_t EOS_ID = BYTE_BASE + 1; // 257 + static constexpr std::size_t DEFAULT_VOCAB_SIZE = 5000; + static constexpr std::size_t DEFAULT_MIN_FREQ = 2; + + using LogFn = std::function; + + struct Config + { + std::size_t vocab_size = DEFAULT_VOCAB_SIZE; + std::uint32_t min_freq = DEFAULT_MIN_FREQ; + LogFn log = nullptr; + }; + + static const std::regex &pre_pattern() + { + static const std::regex pat( + R"('s|'t|'re|'ve|'m|'ll|'d| ?[a-zA-Z]+| ?[0-9]+|[^\s\w]+|\s+)", + std::regex::optimize + ); + return pat; + } + + BPETokenizer() = default; + + [[nodiscard]] std::size_t bos_id() const noexcept override { return BOS_ID; } + [[nodiscard]] std::size_t eos_id() const noexcept override { return EOS_ID; } + + Result train(const std::string &text) { return train(text, Config{}); } + + Result train(const std::string &text, const Config &config) + { + auto log = config.log + ? config.log + : LogFn{[](std::string_view) { /* 静默 */ }}; + + log("\n[BPE 训练] 预分词..."); + auto chunks = pre_tokenize(text); + log(" 预分词块数: " + std::to_string(chunks.size())); + + std::vector> byte_chunks; + byte_chunks.reserve(chunks.size()); + std::size_t total_bytes = 0; + for (const auto &chunk : chunks) + { + std::vector ids; + ids.reserve(chunk.size()); + for (unsigned char b : chunk) + ids.push_back(static_cast(b)); + total_bytes += ids.size(); + byte_chunks.push_back(std::move(ids)); + } + log(" 总字节数: " + std::to_string(total_bytes)); + + vocab_.clear(); + vocab_.reserve(config.vocab_size); + for (std::size_t i = 0; i < BYTE_BASE; ++i) + vocab_.emplace_back(1, static_cast(i)); + // 预留特殊 token 槽位（BOS=256, EOS=257），BPE 合并从 258 开始 + vocab_.emplace_back(""); + vocab_.emplace_back(""); + + merges_.clear(); + merges_.reserve(config.vocab_size - BYTE_BASE - 2); + + const std::size_t target_merges = config.vocab_size - BYTE_BASE - 2; + log(" 目标合并数: " + std::to_string(target_merges)); + + // ── 共享 BPE 训练算法（基类 bpe_train_impl_） ────────────── + bpe_train_impl_(vocab_, byte_chunks, target_merges, config.min_freq, log); + + rebuild_merge_map_(); + // ── 添加对话标记 token ────────────────────────────────────── + add_dialogue_markers_to_vocab_(vocab_); + log("最终词表: " + std::to_string(vocab_.size())); + log("合并规则: " + std::to_string(merges_.size())); + return {}; + } + + [[nodiscard]] std::vector encode(const std::string &text) const override + { + std::vector all_ids; + std::size_t pos = 0; + while (pos < text.size()) + { + // 对话标记优先匹配 + if (has_dialogue_markers()) + { + auto [mid, mlen] = try_match_marker(text, pos); + if (mid != Tokenizer::npos) + { all_ids.push_back(mid); pos += mlen; continue; } + } + // 收集普通文本直到下一个标记 + auto seg_start = pos; + while (pos < text.size()) + { + if (has_dialogue_markers()) + { auto [mid, mlen] = try_match_marker(text, pos); if (mid != Tokenizer::npos) break; } + ++pos; + } + std::string segment(text, seg_start, pos - seg_start); + // 正常 BPE 编码 + auto chunks = pre_tokenize(segment); + for (const auto &chunk : chunks) + { + std::vector ids; + ids.reserve(chunk.size()); + for (unsigned char b : chunk) + ids.push_back(static_cast(b)); + + // 共享 BPE 合并算法（基类 bpe_merge_impl_） + auto merged = bpe_merge_impl_(std::move(ids)); + all_ids.insert(all_ids.end(), merged.begin(), merged.end()); + } + } + return all_ids; + } + + [[nodiscard]] std::string decode(std::span ids) const override + { + std::string result; + result.reserve(ids.size()); + for (auto id : ids) + { + // 对话标记 → 原始文本 + if (id == markers_.system) { result += ""; continue; } + if (id == markers_.system_end) { result += ""; continue; } + if (id == markers_.user) { result += ""; continue; } + if (id == markers_.user_end) { result += ""; continue; } + if (id == markers_.assistant) { result += ""; continue; } + if (id == markers_.assistant_end) { result += ""; continue; } + result += id < vocab_.size() ? vocab_[id] : std::string{REPLACEMENT_CHAR}; + } + return result; + } + + [[nodiscard]] Result save(const std::string &path) const override + { + std::ofstream ofs(path); + if (!ofs) return std::unexpected(Error{"Cannot write: " + path}); + // 共享 JSON 头部 + vocab 段（基类 save_vocab_json_） + save_vocab_json_(ofs, "bpe_tokenizer", 0); + ofs << ",\n \"merges\": ["; + for (std::size_t i = 0; i < merges_.size(); ++i) + { + if (i > 0) ofs << ","; + ofs << "\n [" << merges_[i].id_a << ", " << merges_[i].id_b << ", " << merges_[i].new_id << "]"; + } + ofs << "\n ]\n}\n"; + return {}; + } + + [[nodiscard]] Result load(const std::string &path) override + { + std::ifstream ifs(path); + if (!ifs) return std::unexpected(Error{"Cannot read: " + path}); + std::string content((std::istreambuf_iterator(ifs)), + std::istreambuf_iterator()); + return load_from_string(content); + } + + [[nodiscard]] Result load_from_string(const std::string &content) override + { + vocab_.clear(); + merges_.clear(); + + // 共享 JSON vocab 解析（基类 parse_vocab_json_） + vocab_ = parse_vocab_json_(content); + + auto mp = content.find("\"merges\""); + if (mp != std::string::npos) + { + auto mb = content.find('[', mp); + if (mb != std::string::npos) + { + std::size_t pos = mb + 1; + while (pos < content.size()) + { + while (pos < content.size() && (content[pos]==' '||content[pos]=='\n'|| + content[pos]=='\r'||content[pos]=='\t'||content[pos]==',')) ++pos; + if (pos >= content.size() || content[pos] == ']') break; + if (content[pos] != '[') { ++pos; continue; } + ++pos; + std::size_t nums[3]{}; + for (int n = 0; n < 3; ++n) + { + while (pos < content.size() && (content[pos]==' '||content[pos]=='\n'|| + content[pos]=='\r'||content[pos]=='\t'||content[pos]==',')) ++pos; + std::size_t digit_start = pos; + while (pos < content.size() && content[pos] >= '0' && content[pos] <= '9') + ++pos; + if (pos > digit_start) + std::from_chars(content.data() + digit_start, + content.data() + pos, + nums[n]); + } + while (pos < content.size() && content[pos] != ']') ++pos; + if (pos < content.size()) ++pos; + merges_.push_back({nums[0], nums[1], nums[2]}); + } + } + } + // ── 恢复对话标记 ID（从已有词表检测或追加） ───────────── + restore_dialogue_markers(vocab_); + rebuild_merge_map_(); + return {}; + } + + [[nodiscard]] std::size_t vocab_size() const noexcept override { return vocab_.size(); } + [[nodiscard]] const std::vector &vocab() const noexcept override { return vocab_; } + [[nodiscard]] std::size_t merge_count() const noexcept { return merges_.size(); } + + [[nodiscard]] static std::vector pre_tokenize(std::string_view text) + { + std::vector chunks; + std::string s(text); + auto begin = std::sregex_iterator(s.begin(), s.end(), pre_pattern()); + auto end = std::sregex_iterator(); + for (auto it = begin; it != end; ++it) + chunks.push_back(it->str()); + return chunks; + } + +private: + std::vector vocab_; +}; + + +// ═══════════════════════════════════════════════════════════════════════════ +// CharBPETokenizer — 字符级 BPE（支持 UTF-8/中文） +// ═══════════════════════════════════════════════════════════════════════════ +// 与字节级 BPE (BPETokenizer) 不同，本分词器以 UTF-8 字符为基础单元： +// - 中文每个汉字 = 1 个基础 token（而非 3 个字节） +// - BPE 能学会合并常见中文词组（如"人工智能"、"深度学习"） +// - 小模型更易学到词义 +// +// 词表布局： +// ID 0~3 特殊 token (, , , ) +// ID 4~259 256 个单字节 ASCII 兜底（保证任意字节可编码，避免小词表 crash） +// ID 260+ 非 ASCII 字符 token（中文等，训练时按首次出现分配） +// 之后 BPE 合并 token +// +// 最小词表大小 = 260（4 特殊 + 256 ASCII）。即使 vocab_size=260 也能正常工作， +// 此时无合并规则，等同于纯字符级编码（中文走 UTF-8 字符，ASCII 走单字节）。 +// +// 预分词：按空白分词，每个词内部按 UTF-8 字符切分。 +// CJK 标点也作为分隔符，避免跨句合并。 +// ═══════════════════════════════════════════════════════════════════════════ + +class CharBPETokenizer : public Tokenizer +{ +public: + // 特殊 token ID（固定布局） + static constexpr std::size_t PAD_ID = 0; + static constexpr std::size_t UNK_ID = 1; + static constexpr std::size_t BOS_ID = 2; + static constexpr std::size_t EOS_ID = 3; + static constexpr std::size_t BYTE_BASE = 4; // 256 单字节兜底起始 ID + static constexpr std::size_t CHAR_BASE = 260; // 非 ASCII 字符 token 起始 ID + static constexpr std::size_t MIN_VOCAB_SIZE = CHAR_BASE; // 最小词表大小 + static constexpr std::size_t DEFAULT_VOCAB_SIZE = 5000; + static constexpr std::uint32_t DEFAULT_MIN_FREQ = 2; + + static_assert(MIN_VOCAB_SIZE == CHAR_BASE, "MIN_VOCAB_SIZE must equal CHAR_BASE"); + static_assert(CHAR_BASE == BYTE_BASE + 256, "CHAR_BASE must be BYTE_BASE + 256"); + + using LogFn = std::function; + + struct Config + { + std::size_t vocab_size = DEFAULT_VOCAB_SIZE; + std::uint32_t min_freq = DEFAULT_MIN_FREQ; + LogFn log = nullptr; + }; + + CharBPETokenizer() = default; + + [[nodiscard]] std::size_t bos_id() const noexcept override { return BOS_ID; } + [[nodiscard]] std::size_t eos_id() const noexcept override { return EOS_ID; } + + // ── write_vocab_entry_ 钩子：前 4 个特殊 token 用纯文本，其余 hex ── + // 基类 save_vocab_json_ 在写入每个词表条目时回调此虚方法。 + void write_vocab_entry_(std::ofstream &ofs, const std::string &tok, std::size_t tid) const override + { + if (tid < 4) + ofs << tok; + else + for (unsigned char b : tok) + ofs << std::hex << std::setw(2) << std::setfill('0') << static_cast(b); + } + + // ── decode_vocab_entry_ 钩子：前 4 个特殊 token 纯文本，其余 hex ── + // 基类 parse_vocab_json_ 在解析每个词表条目时回调此虚方法。 + [[nodiscard]] std::string decode_vocab_entry_(const std::string &raw, std::size_t tid) const override + { + if (tid < 4) + return raw; + std::string tok; + tok.reserve(raw.size() / 2); + for (std::size_t i = 0; i + 1 < raw.size(); i += 2) + { + unsigned b = 0; + std::from_chars(raw.data() + i, raw.data() + i + 2, b, 16); + tok += static_cast(b); + } + return tok; + } + + Result train(const std::string &text) { return train(text, Config{}); } + + Result train(const std::string &text, const Config &config) + { + auto log = config.log + ? config.log + : LogFn{[](std::string_view) { /* 静默 */ }}; + + log("\n[CharBPE 训练] 预分词..."); + auto chunks = pre_tokenize(text); + log(" 预分词块数: " + std::to_string(chunks.size())); + + // ── 0. 校验最小词表大小（4 特殊 + 256 ASCII 兜底） ────── + if (config.vocab_size < MIN_VOCAB_SIZE) + { + return std::unexpected(Error{ + "CharBPE vocab_size too small: minimum is " + + std::to_string(MIN_VOCAB_SIZE) + + " (4 special + 256 ASCII fallback), got " + + std::to_string(config.vocab_size)}); + } + + // ── 1. 初始化词表：4 特殊 + 256 单字节 ASCII 兜底 ────── + // 这保证任意字节序列都能编码（不会因词表太小 crash）， + // ASCII 字符直接走单字节 token，非 ASCII（如中文）走多字节字符 token。 + char_to_id_.clear(); + vocab_.clear(); + vocab_.reserve(config.vocab_size); + vocab_.emplace_back(""); + vocab_.emplace_back(""); + vocab_.emplace_back(""); + vocab_.emplace_back(""); + for (std::size_t i = 0; i < 256; ++i) + { + std::string ch(1, static_cast(static_cast(i))); + vocab_.push_back(ch); + char_to_id_[ch] = BYTE_BASE + i; + } + // assert: vocab_.size() == CHAR_BASE == 260 + + // ── 2. 收集非 ASCII 字符，建立 char_to_id ────────────── + // ASCII 字符已在兜底表中，只收集非 ASCII（如中文汉字）。 + std::vector> char_chunks; + char_chunks.reserve(chunks.size()); + + for (const auto &chunk : chunks) + { + std::vector ids; + std::size_t i = 0; + while (i < chunk.size()) + { + auto [cp, len] = decode_utf8(chunk, i); + std::string ch = chunk.substr(i, len); + auto it = char_to_id_.find(ch); + std::size_t id; + if (it == char_to_id_.end()) + { + // 新字符：仅当词表未满时才添加 + if (vocab_.size() >= config.vocab_size) + { + // 词表已满，未知字符回退到 UNK + id = UNK_ID; + } + else + { + id = vocab_.size(); + vocab_.push_back(ch); + char_to_id_[ch] = id; + } + } + else + id = it->second; + ids.push_back(id); + i += len; + } + if (!ids.empty()) + char_chunks.push_back(std::move(ids)); + } + + log(" 非 ASCII 字符数: " + std::to_string(vocab_.size() - CHAR_BASE)); + log(" 总字符数: " + std::to_string( + std::accumulate(char_chunks.begin(), char_chunks.end(), std::size_t{0}, + [](std::size_t s, const auto &c) { return s + c.size(); }))); + + // ── 3. BPE 合并训练 ── + merges_.clear(); + // 注意：用条件判断避免无符号下溢（vocab_size 可能 < vocab_.size()） + if (config.vocab_size > vocab_.size()) + merges_.reserve(config.vocab_size - vocab_.size()); + + const std::size_t target_merges = + config.vocab_size > vocab_.size() + ? config.vocab_size - vocab_.size() + : 0; + log(" 目标合并数: " + std::to_string(target_merges)); + + // ── 共享 BPE 训练算法（基类 bpe_train_impl_） ────────────── + bpe_train_impl_(vocab_, char_chunks, target_merges, config.min_freq, log); + + rebuild_merge_map_(); + // ── 添加对话标记 token ────────────────────────────────────── + add_dialogue_markers_to_vocab_(vocab_); + log("最终词表: " + std::to_string(vocab_.size())); + log("合并规则: " + std::to_string(merges_.size())); + return {}; + } + + // ── 预分词：按空白 + CJK 标点切分，每词按 UTF-8 字符 ── + [[nodiscard]] static std::vector pre_tokenize(const std::string &text) + { + std::vector chunks; + std::size_t i = 0; + while (i < text.size()) + { + // 捕获前导空白（附加到下一个非空字符前，GPT-2 风格） + std::string prefix; + while (i < text.size() && is_space_byte(text[i])) + { + prefix += text[i++]; + } + if (i >= text.size()) break; + + // 读取一个"词"：连续同类字符 + std::string word = prefix; + auto [cp, len] = decode_utf8(text, i); + + if (is_cjk(cp)) + { + // CJK 字符：连续读取，遇到非 CJK 或 CJK 标点停止 + do { + word.append(text, i, len); + i += len; + if (i >= text.size()) break; + std::tie(cp, len) = decode_utf8(text, i); + } while (is_cjk(cp) && !is_cjk_punct(cp)); + } + else if (is_alnum(cp)) + { + do { + word.append(text, i, len); + i += len; + if (i >= text.size()) break; + std::tie(cp, len) = decode_utf8(text, i); + } while (is_alnum(cp)); + } + else + { + // 其他字符（标点等）：单个字符 + word.append(text, i, len); + i += len; + } + if (!word.empty()) + chunks.push_back(std::move(word)); + } + return chunks; + } + + // ── UTF-8 解码：返回 (码点, 字节长度) ── + [[nodiscard]] static std::pair + decode_utf8(const std::string &s, std::size_t pos) + { + if (pos >= s.size()) return {0, 1}; + unsigned char b0 = static_cast(s[pos]); + if (b0 < 0x80) return {b0, 1}; + if (pos + 1 >= s.size()) return {0xFFFD, 1}; + if ((b0 & 0xE0) == 0xC0) + { + uint32_t cp = ((b0 & 0x1F) << 6) | + (static_cast(s[pos+1]) & 0x3F); + return {cp, 2}; + } + if (pos + 2 >= s.size()) return {0xFFFD, 1}; + if ((b0 & 0xF0) == 0xE0) + { + uint32_t cp = ((b0 & 0x0F) << 12) | + ((static_cast(s[pos+1]) & 0x3F) << 6) | + (static_cast(s[pos+2]) & 0x3F); + return {cp, 3}; + } + if (pos + 3 >= s.size()) return {0xFFFD, 1}; + if ((b0 & 0xF8) == 0xF0) + { + uint32_t cp = ((b0 & 0x07) << 18) | + ((static_cast(s[pos+1]) & 0x3F) << 12) | + ((static_cast(s[pos+2]) & 0x3F) << 6) | + (static_cast(s[pos+3]) & 0x3F); + return {cp, 4}; + } + return {0xFFFD, 1}; + } + + [[nodiscard]] static constexpr bool is_space_byte(char c) noexcept + { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; + } + + // CJK 统一表意文字（常用汉字） + [[nodiscard]] static constexpr bool is_cjk(uint32_t cp) noexcept + { + return (cp >= 0x4E00 && cp <= 0x9FFF) // CJK 统一表意文字 + || (cp >= 0x3400 && cp <= 0x4DBF) // CJK 扩展 A + || (cp >= 0xF900 && cp <= 0xFAFF); // CJK 兼容表意 + } + + // CJK 标点（作为分隔符，避免跨句合并） + [[nodiscard]] static constexpr bool is_cjk_punct(uint32_t cp) noexcept + { + return (cp >= 0x3000 && cp <= 0x303F) // CJK 标点和符号 + || (cp >= 0xFF00 && cp <= 0xFFEF); // 半角/全角形式 + } + + [[nodiscard]] static constexpr bool is_alnum(uint32_t cp) noexcept + { + return (cp >= '0' && cp <= '9') + || (cp >= 'A' && cp <= 'Z') + || (cp >= 'a' && cp <= 'z'); + } + + // ════════════════════════════════════════════════════════════════════ + // Tokenizer 接口实现 + // ════════════════════════════════════════════════════════════════════ + + [[nodiscard]] std::vector encode(const std::string &text) const override + { + std::vector all_ids; + std::size_t pos = 0; + while (pos < text.size()) + { + // 对话标记优先匹配 + if (has_dialogue_markers()) + { + auto [mid, mlen] = try_match_marker(text, pos); + if (mid != Tokenizer::npos) + { all_ids.push_back(mid); pos += mlen; continue; } + } + // 收集普通文本直到下一个标记 + auto seg_start = pos; + while (pos < text.size()) + { + if (has_dialogue_markers()) + { auto [mid, mlen] = try_match_marker(text, pos); if (mid != Tokenizer::npos) break; } + ++pos; + } + std::string segment(text, seg_start, pos - seg_start); + // 正常 CharBPE 编码 + auto chunks = pre_tokenize(segment); + for (const auto &chunk : chunks) + { + std::vector ids; + ids.reserve(chunk.size()); + std::size_t i = 0; + while (i < chunk.size()) + { + if (static_cast(chunk[i]) < 0x80) + { + ids.push_back(BYTE_BASE + static_cast(chunk[i])); + i += 1; + } + else + { + auto [cp, len] = decode_utf8(chunk, i); + std::string ch = chunk.substr(i, len); + auto it = char_to_id_.find(ch); + ids.push_back(it != char_to_id_.end() ? it->second : UNK_ID); + i += len; + } + } + + // 共享 BPE 合并算法（基类 bpe_merge_impl_） + auto merged = bpe_merge_impl_(std::move(ids)); + all_ids.insert(all_ids.end(), merged.begin(), merged.end()); + } // end for(chunk) + } // end while(pos) + return all_ids; + } + + [[nodiscard]] std::string decode(std::span ids) const override + { + std::string result; + for (auto id : ids) + { + // BYTE_BASE(4) 及以上均为有效 token（含 ASCII 兜底 + 字符 + 合并） + // 特殊 token (0-3) 解码为空 + if (id < vocab_.size() && id >= BYTE_BASE) + result += vocab_[id]; + } + return result; + } + + [[nodiscard]] std::size_t vocab_size() const noexcept override { return vocab_.size(); } + [[nodiscard]] const std::vector &vocab() const noexcept override { return vocab_; } + + [[nodiscard]] std::size_t merge_count() const noexcept { return merges_.size(); } + + // ── 保存 JSON ──────────────────────────────────────────────────── + [[nodiscard]] Result save(const std::string &path) const override + { + std::ofstream ofs(path); + if (!ofs) return std::unexpected(Error{"Cannot write: " + path}); + // 共享 JSON 头部 + vocab 段（基类 save_vocab_json_ + write_vocab_entry_ 钩子） + // 起始 ID = 0：保留前 4 个特殊 token + ASCII 兜底，全部写入。 + save_vocab_json_(ofs, "char_bpe_tokenizer", 0); + ofs << ",\n \"merges\": ["; + for (std::size_t i = 0; i < merges_.size(); ++i) + { + if (i > 0) ofs << ","; + ofs << "\n [" << merges_[i].id_a << ", " << merges_[i].id_b + << ", " << merges_[i].new_id << "]"; + } + ofs << "\n ]\n}\n"; + return {}; + } + + [[nodiscard]] Result load(const std::string &path) override + { + std::ifstream ifs(path); + if (!ifs) return std::unexpected(Error{"Cannot read: " + path}); + std::string content((std::istreambuf_iterator(ifs)), + std::istreambuf_iterator()); + return load_from_string(content); + } + + [[nodiscard]] Result load_from_string(const std::string &content) override + { + vocab_.clear(); + merges_.clear(); + char_to_id_.clear(); + + // 共享 JSON vocab 解析（基类 parse_vocab_json_ + decode_vocab_entry_ 钩子） + vocab_ = parse_vocab_json_(content); + if (vocab_.empty()) + { + // parse_vocab_json_ 未能解析 vocab_size 时回退到 CHAR_BASE + vocab_.resize(CHAR_BASE); + } + + // 重建 char_to_id_（BYTE_BASE 及以上均为字符 token） + for (std::size_t id = BYTE_BASE; id < vocab_.size(); ++id) + if (!vocab_[id].empty()) + char_to_id_[vocab_[id]] = id; + + // 解析 merges 数组 + auto mp = content.find("\"merges\""); + if (mp != std::string::npos) + { + auto mb = content.find('[', mp); + if (mb != std::string::npos) + { + std::size_t mpos = mb + 1; + while (mpos < content.size()) + { + auto lb = content.find('[', mpos); + if (lb == std::string::npos) break; + auto rb = content.find(']', lb); + if (rb == std::string::npos) break; + std::size_t a=0, b=0, n=0; + auto s1 = content.substr(lb+1, rb-lb-1); + std::istringstream iss(s1); + char sep; + iss >> a >> sep >> b >> sep >> n; + merges_.push_back({a, b, n}); + mpos = rb + 1; + } + } + } + + // ── 恢复对话标记 ID（从已有词表检测或追加） ───────────── + restore_dialogue_markers(vocab_); + rebuild_merge_map_(); + return {}; + } + +private: + std::vector vocab_; + std::unordered_map char_to_id_; +}; + + +// ═══════════════════════════════════════════════════════════════════════════ +// Tokenizer 工厂：根据 JSON 的 "type" 字段创建对应分词器实例 +// ═══════════════════════════════════════════════════════════════════════════ +// 自动识别 BBPE / CharBPE / WordZip / Space 四种分词器。 +// 训练/推理入口统一使用此工厂，无需手动判断类型。 + +[[nodiscard]] inline std::string peek_tokenizer_type(const std::string &json_content) +{ + // 查找 "type" 字段值 + auto p = json_content.find("\"type\""); + if (p == std::string::npos) return {}; + auto c = json_content.find(':', p); + if (c == std::string::npos) return {}; + ++c; + while (c < json_content.size() && json_content[c] == ' ') ++c; + if (c >= json_content.size() || json_content[c] != '"') return {}; + ++c; + std::string val; + while (c < json_content.size() && json_content[c] != '"') + val += json_content[c++]; + return val; +} + +[[nodiscard]] inline std::unique_ptr make_tokenizer(const std::string &type_str) +{ + if (type_str == "bpe_tokenizer") + return std::make_unique(); + if (type_str == "char_bpe_tokenizer") + return std::make_unique(); + if (type_str == "wordzip_tokenizer") + return std::make_unique(); + if (type_str == "space_tokenizer") + return std::make_unique(); + return nullptr; +} + +// 从 JSON 内容创建已加载的 tokenizer（失败返回 nullptr） +[[nodiscard]] inline std::unique_ptr +load_tokenizer_from_string(const std::string &json_content) +{ + auto type_str = peek_tokenizer_type(json_content); + if (type_str.empty()) + return nullptr; // 无法识别类型 + auto tok = make_tokenizer(type_str); + if (!tok) + return nullptr; + auto r = tok->load_from_string(json_content); + if (!r) + return nullptr; + return tok; +} + +// 从文件创建已加载的 tokenizer +[[nodiscard]] inline std::unique_ptr +load_tokenizer_from_file(const std::string &path) +{ + std::ifstream ifs(path); + if (!ifs) return nullptr; + std::string content((std::istreambuf_iterator(ifs)), + std::istreambuf_iterator()); + return load_tokenizer_from_string(content); +} + + +} // namespace nn + +#endif // NN_DOMAIN_TOKENIZER_HPP diff --git a/include/neuralnet.cpp/gpt_common.hpp b/include/neuralnet.cpp/gpt_common.hpp deleted file mode 100644 index 3df6e62c..00000000 --- a/include/neuralnet.cpp/gpt_common.hpp +++ /dev/null @@ -1,357 +0,0 @@ -#ifndef GPT_COMMON_HPP -#define GPT_COMMON_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "model.hpp" -#include "layer.hpp" - -namespace nn { - -// ── GPT 默认超参数 ──────────────────────────────────────────────────────── -inline constexpr std::size_t GPT_VOCAB_SIZE = 10000; // 频率过滤词表 -inline constexpr std::size_t GPT_D_MODEL = 128; -inline constexpr std::size_t GPT_NUM_HEADS = 4; -inline constexpr std::size_t GPT_D_FF = 512; -inline constexpr std::size_t GPT_NUM_LAYERS = 4; -inline constexpr std::size_t GPT_SEQ_LEN = 256; - -// ── 字符级词表 ──────────────────────────────────────────────────────────── -// 简单的字符级 tokenizer：每个 ASCII 字符对应一个 token ID。 -// 词表大小固定为 128（覆盖所有 ASCII 可打印字符和控制字符）。 -class CharTokenizer -{ -public: - [[nodiscard]] std::size_t vocab_size() const noexcept { return GPT_VOCAB_SIZE; } - - [[nodiscard]] std::size_t encode_one(char c) const noexcept - { - return static_cast(static_cast(c)) % GPT_VOCAB_SIZE; - } - - [[nodiscard]] std::string decode_one(std::size_t id) const - { - if (id < GPT_VOCAB_SIZE) - return std::string(1, static_cast(id)); - return "?"; - } - - [[nodiscard]] std::vector encode(const std::string &text) const - { - std::vector tokens; - tokens.reserve(text.size()); - for (char c : text) - tokens.push_back(encode_one(c)); - return tokens; - } - - [[nodiscard]] std::string decode(const std::vector &tokens) const - { - std::string result; - result.reserve(tokens.size()); - for (auto id : tokens) - result += decode_one(id); - return result; - } -}; - -// ── BPE 分词器 ──────────────────────────────────────────────────────────── -// 加载 Python 预处理生成的 gpt_vocab.json 词表。 -// 词表构成: =0, =1, =2, ASCII 3-130, 高频词 131+ -// -// 编码: 先按空格分词，词表中有→直接查 ID，没有→逐字符拼 ASCII ID -// 解码: ID < 128 → 字符, ID >= 128 → 词（空格分隔） -// -class BPETokenizer -{ -public: - static constexpr std::size_t UNK_ID = 0; - static constexpr std::size_t PAD_ID = 1; - static constexpr std::size_t NUM_ID = 2; - -private: - std::vector id_to_token_; - std::unordered_map token_to_id_; - std::string vocab_path_; - - [[nodiscard]] static bool is_numeric(const std::string &s) noexcept - { - if (s.empty()) return false; - for (char c : s) - if (c < '0' || c > '9') return false; - return true; - } - - // 判断 token 是否为多字节子词（需要空格分隔） - // 单字节 token (ID 4-259) 直接拼接，多字节子词 (ID 260+) 空格分隔 - [[nodiscard]] bool is_word_token(std::size_t id) const noexcept - { - return id < id_to_token_.size() && id_to_token_[id].size() > 1; - } - -public: - BPETokenizer() = default; - - [[nodiscard]] std::size_t vocab_size() const noexcept { return id_to_token_.size(); } - [[nodiscard]] const std::string &vocab_path() const noexcept { return vocab_path_; } - - // 加载词表 JSON（支持新格式: {"vocab": {"4": "hex", "5": "hex", ...}}） - Result load_vocab(const std::string &path) - { - std::ifstream ifs(path); - if (!ifs) return std::unexpected(Error{"Cannot read vocab file: " + path}); - - std::string content((std::istreambuf_iterator(ifs)), - std::istreambuf_iterator()); - - id_to_token_.clear(); - token_to_id_.clear(); - - // 找 "vocab" 对象 - auto vocab_pos = content.find("\"vocab\""); - if (vocab_pos == std::string::npos) - return std::unexpected(Error{"Invalid vocab JSON: missing \"vocab\" key"}); - - auto brace_pos = content.find('{', vocab_pos); - if (brace_pos == std::string::npos) - return std::unexpected(Error{"Invalid vocab JSON: missing opening brace"}); - - // 逐对解析 - std::size_t pos = brace_pos + 1; - while (pos < content.size()) - { - // 跳过空白和逗号 - while (pos < content.size() && (content[pos] == ' ' || content[pos] == '\n' || - content[pos] == '\r' || content[pos] == '\t' || content[pos] == ',')) - ++pos; - if (pos >= content.size() || content[pos] == '}') break; - - // 解析 key（带引号的字符串） - if (content[pos] != '"') { ++pos; continue; } - std::string key; - ++pos; - while (pos < content.size() && content[pos] != '"') - { - if (content[pos] == '\\' && pos + 1 < content.size()) - { - ++pos; - switch (content[pos]) - { - case 'n': key += '\n'; break; - case 't': key += '\t'; break; - case 'r': key += '\r'; break; - case '\\': key += '\\'; break; - case '"': key += '"'; break; - case 'u': - { - if (pos + 4 < content.size()) - { - std::string hex = content.substr(pos + 1, 4); - unsigned long cp = std::stoul(hex, nullptr, 16); - if (cp < 0x80) key += static_cast(cp); - else if (cp < 0x800) { - key += static_cast(0xC0 | (cp >> 6)); - key += static_cast(0x80 | (cp & 0x3F)); - } else { - key += static_cast(0xE0 | (cp >> 12)); - key += static_cast(0x80 | ((cp >> 6) & 0x3F)); - key += static_cast(0x80 | (cp & 0x3F)); - } - pos += 4; - } - break; - } - default: key += content[pos]; break; - } - } - else - { - key += content[pos]; - } - ++pos; - } - ++pos; // 跳过结束引号 - - // 跳过 : 和空白 - while (pos < content.size() && (content[pos] == ' ' || content[pos] == ':')) - ++pos; - - // 检测 value 类型：数字 → 旧格式；引号 → 新格式（hex 字符串） - if (pos < content.size() && content[pos] == '"') - { - // 新格式: "id": "hex_bytes" — key 是 ID，value 是 hex 编码的字节 - std::string hex_val; - ++pos; - while (pos < content.size() && content[pos] != '"') - hex_val += content[pos++]; - ++pos; - - std::size_t id = static_cast(std::stoul(key)); - - // hex → bytes → string - std::string token; - for (std::size_t i = 0; i + 1 < hex_val.size(); i += 2) - { - unsigned byte = std::stoul(hex_val.substr(i, 2), nullptr, 16); - token += static_cast(byte); - } - - if (id >= id_to_token_.size()) - id_to_token_.resize(id + 1); - id_to_token_[id] = token; - token_to_id_[token] = id; - } - else - { - // 旧格式: "token": id — key 是 token 字符串，value 是 ID - std::string val_str; - while (pos < content.size() && content[pos] >= '0' && content[pos] <= '9') - val_str += content[pos++]; - if (val_str.empty()) continue; - - std::size_t id = static_cast(std::stoul(val_str)); - if (id >= id_to_token_.size()) - id_to_token_.resize(id + 1); - id_to_token_[id] = key; - token_to_id_[key] = id; - } - } - - vocab_path_ = path; - return {}; - } - - // 编码：先按空格分词，逐词查表；未命中则逐字符兜底 - [[nodiscard]] std::vector encode(const std::string &text) const - { - std::vector tokens; - std::istringstream iss(text); - std::string word; - - while (iss >> word) - { - if (is_numeric(word)) - { - tokens.push_back(NUM_ID); - } - else if (auto it = token_to_id_.find(word); it != token_to_id_.end()) - { - tokens.push_back(it->second); - } - else - { - // 逐字符兜底（每个字符都一定能找到，因为 ASCII 在词表中） - for (char c : word) - { - std::string s(1, c); - if (auto cit = token_to_id_.find(s); cit != token_to_id_.end()) - tokens.push_back(cit->second); - else - tokens.push_back(UNK_ID); - } - } - } - return tokens; - } - - // 解码：词 token 空格分隔，字符 token 直接拼接 - [[nodiscard]] std::string decode(const std::vector &tokens) const - { - std::string result; - - for (auto id : tokens) - { - if (id < id_to_token_.size()) - { - if (is_word_token(id)) - { - // 词 token：空格分隔 - if (!result.empty()) result += ' '; - result += id_to_token_[id]; - } - else - { - // 字符 token：直接拼接 - result += id_to_token_[id]; - } - } - else - { - if (!result.empty()) result += ' '; - result += ""; - } - } - return result; - } -}; - -// ── 从文本文件加载语料 ──────────────────────────────────────────────────── -[[nodiscard]] inline Result load_text_file(const std::string &path) -{ - std::ifstream ifs(path, std::ios::binary); - if (!ifs) - return std::unexpected(Error{"Cannot open text file: " + path}); - std::string content((std::istreambuf_iterator(ifs)), - std::istreambuf_iterator()); - return content; -} - -// ── 构建 GPT 模型 ──────────────────────────────────────────────────────── -[[nodiscard]] inline Model build_gpt_model( - std::size_t vocab_size = GPT_VOCAB_SIZE, - std::size_t d_model = GPT_D_MODEL, - std::size_t seq_len = GPT_SEQ_LEN, - std::size_t num_heads = GPT_NUM_HEADS, - std::size_t d_ff = GPT_D_FF, - std::size_t num_layers = GPT_NUM_LAYERS) -{ - // GPTModel 是一个单一 Layer，作为 Model 的唯一层 - Model model; - model.add(vocab_size, d_model, seq_len, num_heads, d_ff, num_layers); - return model; -} - -// ── 从 ModelSpec 构建 GPT 模型 ────────────────────────────────────────── -// 用于从二进制文件加载时自动还原架构 -[[nodiscard]] inline Result build_gpt_model_from_spec(const ModelSpec &spec) -{ - if (!spec.is_gpt()) - return std::unexpected(Error{"Invalid ModelSpec type for GPT: expected GPT"}); - - return build_gpt_model( - spec.vocab_size, spec.d_model, spec.seq_len, - spec.num_heads, spec.d_ff, spec.num_layers); -} - -// ── 构造 GPT ModelSpec ────────────────────────────────────────────────── -[[nodiscard]] inline ModelSpec make_gpt_spec( - std::size_t vocab_size, - std::size_t d_model, - std::size_t seq_len, - std::size_t num_heads, - std::size_t d_ff, - std::size_t num_layers) -{ - ModelSpec spec; - spec.type = ModelType::GPT; - spec.vocab_size = vocab_size; - spec.d_model = d_model; - spec.seq_len = seq_len; - spec.num_heads = num_heads; - spec.d_ff = d_ff; - spec.num_layers = num_layers; - return spec; -} - -} // namespace nn - -#endif // GPT_COMMON_HPP diff --git a/include/neuralnet.cpp/gpu_engine.hpp b/include/neuralnet.cpp/gpu_engine.hpp new file mode 100644 index 00000000..62443911 --- /dev/null +++ b/include/neuralnet.cpp/gpu_engine.hpp @@ -0,0 +1,557 @@ +#ifndef NN_GPU_ENGINE_HPP +#define NN_GPU_ENGINE_HPP + +// ── gpu_engine.hpp — GPU 计算引擎实现（纯 GPU 架构）───────────────────────── +// GpuEngine 封装 GpuBackend，实现 ComputeEngine 接口。 +// +// 纯 GPU 架构策略： +// - 所有原语（matmul、elementwise、reduce、broadcast、zero、clone）均在 +// GPU 上原生执行，数据全程驻留 GPU 显存。 +// - from_matrix：上传 CPU Matrix → GPU Tensor（唯一的 PCIe 上传点） +// - to_matrix：下载 GPU Tensor → CPU Matrix（唯一的 PCIe 下载点） +// - create_tensor：分配 GPU buffer（用于参数/梯度） +// - clone：GPU 内 buffer 拷贝（无 PCIe 传输） +// - 所有原语返回 GPU Tensor，不回退到 CPU +// +// 前向/反向链路全程 GPU： +// from_matrix(上传) → matmul(GPU) → broadcast(GPU) → elementwise(GPU) +// → reduce(GPU) → ... → to_matrix(下载) +// 除 batch 边界外，无 PCIe 传输。 +// +// 同步模型： +// batch 录制模式已启用：begin_batch 后所有原语录制到共享 command buffer， +// end_batch 一次 vkQueueSubmit + vkWaitForFences，消除 per-primitive 同步开销。 +// to_matrix/from_matrix 会打断 batch（flush 后自动重新 begin_batch）。 +// +// 原地操作语义： +// add_inplace / scale_inplace / broadcast_*_inplace 采用"分配新 buffer + +// 替换 Tensor"策略（copy-on-write 语义）。zero 使用 vkCmdFillBuffer 真原地 +// 清零（避免每步分配）。 +// ───────────────────────────────────────────────────────────────────────── + +#ifdef NN_HAS_VULKAN + +#include "compute_engine.hpp" +#include "backend/vk_backend.hpp" + +namespace nn +{ + +// ══════════════════════════════════════════════════════════════════════════ +// GpuEngine — GPU 计算引擎（纯 GPU 架构） +// ══════════════════════════════════════════════════════════════════════════ +class GpuEngine final : public ComputeEngine +{ +private: + GpuBackend& backend_; + +public: + explicit GpuEngine(GpuBackend& backend) : backend_(backend) {} + + [[nodiscard]] Device device() const noexcept override { return Device::GPU; } + + // ── 批处理：激活 GpuBackend 的 command buffer 录制模式 ────────────── + // begin_batch 后，所有原语录制到共享 batch_cmd_，直到 end_batch 一次提交+等待。 + // 这消除了 per-primitive 的 vkQueueSubmit+vkWaitForFences 开销。 + [[nodiscard]] Result begin_batch() override + { + return backend_.begin_batch(); + } + + [[nodiscard]] Result end_batch() override + { + return backend_.end_batch(); + } + + // ── 中点刷新：提交当前 command buffer 并开始新的录制 ── + // 用于拆分大 batch（如 forward 与 backward 之间），防 TDR。 + [[nodiscard]] Result flush_batch() override + { + return backend_.flush_batch(); + } + + // ══════════════════════════════════════════════════════════════════════ + // 张量工厂（纯 GPU：全部创建/上传为 GPU Tensor） + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Tensor create_tensor(std::size_t rows, std::size_t cols) override + { + // 分配 GPU buffer（用于参数/梯度，后续通过 copy_from 填充） + auto r = GpuTensor::create_empty(rows, cols, backend_); + if (!r) + { + // 分配失败时返回空 Tensor（调用方应检查 valid()） + return Tensor(); + } + return Tensor::from_gpu(std::move(*r)); + } + + [[nodiscard]] Result from_matrix(const Matrix& m) override + { + // batch 模式下必须先 flush（提交并等待），否则 upload_blocking 会 + // 独立提交 command buffer 到队列，与正在录制的 batch 产生竞争。 + // flush 后自动重新 begin_batch，保持 batch 上下文不断裂。 + if (in_batch()) + { + auto r = end_batch(); + if (!r) return std::unexpected(r.error()); + auto rb = begin_batch(); + if (!rb) return std::unexpected(rb.error()); + } + // 上传 CPU Matrix → GPU Tensor（PCIe 上传，唯一的上传点） + auto r = GpuTensor::from_matrix(m, backend_); + if (!r) + return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + [[nodiscard]] Result to_matrix(const Tensor& t) override + { + if (t.is_cpu()) + return Matrix(t.cpu_matrix()); + // batch 模式下必须先 flush（提交并等待），否则 GPU 计算未执行，读到旧数据 + // flush 后自动重新 begin_batch，保持 batch 上下文不断裂 + if (in_batch()) + { + auto r = end_batch(); + if (!r) return std::unexpected(r.error()); + auto rb = begin_batch(); + if (!rb) return std::unexpected(rb.error()); + } + return t.gpu_tensor().to_matrix(backend_); + } + + [[nodiscard]] Result copy_from(Tensor& dst, const Matrix& src) override + { + if (dst.rows() != src.rows() || dst.cols() != src.cols()) + return std::unexpected(Error{"copy_from: shape mismatch"}); + if (dst.is_cpu()) + { + // 防御性：dst 应为 GPU Tensor，但若为 CPU 则直接拷贝 + dst = Tensor::from_matrix(Matrix(src)); + return {}; + } + // batch 模式下必须先 flush（与 from_matrix 同理，upload_blocking 独立提交） + if (in_batch()) + { + auto r = end_batch(); + if (!r) return std::unexpected(r.error()); + auto rb = begin_batch(); + if (!rb) return std::unexpected(rb.error()); + } + return backend_.upload_blocking(dst.gpu_tensor(), src.span()); + } + + [[nodiscard]] Result clone(const Tensor& src) override + { + if (src.is_cpu()) + { + // 防御性：CPU Tensor 深拷贝 + return Tensor::from_matrix(Matrix(src.cpu_matrix())); + } + auto r = backend_.clone_gpu(src.gpu_tensor()); + if (!r) + return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // ── 行切片：GPU 内拷贝连续行区间 ── + [[nodiscard]] Result slice_rows( + const Tensor& src, std::size_t start_row, std::size_t count) override + { + auto src_gpu = ensure_gpu(src); + if (!src_gpu) return std::unexpected(src_gpu.error()); + auto r = backend_.slice_rows_gpu(src_gpu->gpu_tensor(), start_row, count); + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // ── 行插入：GPU 内就地写入连续行区间 ── + // 纯 GPU 架构：dst 必须为 GPU Tensor，不再回退 CPU 路径（原 cpu_fallback 已删除）。 + [[nodiscard]] Result insert_rows( + Tensor& dst, std::size_t dst_start_row, const Tensor& src) override + { + if (dst.is_cpu()) + return std::unexpected(Error{"insert_rows: dst must be GPU tensor in pure-GPU architecture"}); + auto src_gpu = ensure_gpu(src); + if (!src_gpu) return std::unexpected(src_gpu.error()); + return backend_.insert_rows_gpu(dst.gpu_tensor(), dst_start_row, src_gpu->gpu_tensor()); + } + + // ── gather_rows: 按 indices 从 table 中按行查表 ── + // GPU-native 实现：全程在 GPU 执行，无 PCIe 传输。 + // indices 支持任意形状，按 flat 遍历所有元素。 + [[nodiscard]] Result gather_rows( + const Tensor& table, const Tensor& indices) override + { + auto tbl_gpu = ensure_gpu(table); + if (!tbl_gpu) return std::unexpected(tbl_gpu.error()); + auto idx_gpu = ensure_gpu(indices); + if (!idx_gpu) return std::unexpected(idx_gpu.error()); + + auto r = backend_.gather_gpu(tbl_gpu->gpu_tensor(), idx_gpu->gpu_tensor()); + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // ── scatter_add_rows: 按 indices 把 grad 的行原子累加到 dst ── + // GPU-native 实现：使用 CAS 循环实现 float atomicAdd，无 PCIe 传输。 + [[nodiscard]] Result scatter_add_rows( + Tensor& dst, const Tensor& indices, const Tensor& grad) override + { + if (!dst.is_gpu()) + return std::unexpected(Error{"scatter_add_rows: dst must be GPU tensor"}); + + auto idx_gpu = ensure_gpu(indices); + if (!idx_gpu) return std::unexpected(idx_gpu.error()); + auto grad_gpu = ensure_gpu(grad); + if (!grad_gpu) return std::unexpected(grad_gpu.error()); + + return backend_.scatter_add_gpu( + dst.gpu_tensor(), idx_gpu->gpu_tensor(), grad_gpu->gpu_tensor()); + } + +private: + // batch 模式查询（内部使用，to_matrix/from_matrix 需检查） + [[nodiscard]] bool in_batch() const noexcept { return backend_.in_batch(); } + +public: + + // ── 3D 维度转置：(M, B, N) ↔ (B, M, N) ── + [[nodiscard]] Result rearrange_3d( + const Tensor& x, std::size_t M, std::size_t B, std::size_t N, + bool inverse) override + { + auto x_gpu = ensure_gpu(x); + if (!x_gpu) return std::unexpected(x_gpu.error()); + + auto r = backend_.rearrange_3d_gpu( + x_gpu->gpu_tensor(), + static_cast(M), + static_cast(B), + static_cast(N), + inverse ? 1u : 0u); + if (!r) + return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // ── 矩阵转置：A (R, C) → out (C, R) ── + [[nodiscard]] Result transpose(const Tensor& A) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.transpose_gpu(a_gpu->gpu_tensor()); + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // ══════════════════════════════════════════════════════════════════════ + // 矩阵级原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result matmul( + const Tensor& A, const Tensor& B, + bool transA, bool transB) override + { + // 确保 A、B 在 GPU 上 + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto b_gpu = ensure_gpu(B); + if (!b_gpu) return std::unexpected(b_gpu.error()); + + auto r = backend_.matmul_gpu( + a_gpu->gpu_tensor(), b_gpu->gpu_tensor(), + transA ? 1u : 0u, transB ? 1u : 0u); + if (!r) + return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // ── 批量矩阵乘法：按 batch 切分行块，单次 dispatch 处理所有 batch ── + [[nodiscard]] Result batched_matmul( + const Tensor& A, const Tensor& B, + std::size_t batch, + bool transA, bool transB) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto b_gpu = ensure_gpu(B); + if (!b_gpu) return std::unexpected(b_gpu.error()); + + auto r = backend_.batched_matmul_gpu( + a_gpu->gpu_tensor(), b_gpu->gpu_tensor(), + static_cast(batch), + transA ? 1u : 0u, transB ? 1u : 0u); + if (!r) + return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // A += B：分配新 buffer 计算 A+B，替换 A + [[nodiscard]] Result add_inplace(Tensor& A, const Tensor& B) override + { + if (A.rows() != B.rows() || A.cols() != B.cols()) + return std::unexpected(Error{"add_inplace: shape mismatch"}); + + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto b_gpu = ensure_gpu(B); + if (!b_gpu) return std::unexpected(b_gpu.error()); + + const uint32_t count = static_cast(A.rows() * A.cols()); + auto r = backend_.elementwise_v2_gpu( + a_gpu->gpu_tensor(), &b_gpu->gpu_tensor(), nullptr, + count, 1u, 0u, 0u, 0u, 0.0f, 0.0f, 0.0f); // BINARY, Add + if (!r) + return std::unexpected(r.error()); + A = Tensor::from_gpu(std::move(*r)); + return {}; + } + + // A *= s：分配新 buffer 计算 A*s，替换 A + [[nodiscard]] Result scale_inplace(Tensor& A, Scalar s) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + + const uint32_t count = static_cast(A.rows() * A.cols()); + // BINARY, Mul, flags=1 (B is scalar), scalar_b = s + auto r = backend_.elementwise_v2_gpu( + a_gpu->gpu_tensor(), nullptr, nullptr, + count, 1u, 2u, 0u, 1u, static_cast(s), 0.0f, 0.0f); + if (!r) + return std::unexpected(r.error()); + A = Tensor::from_gpu(std::move(*r)); + return {}; + } + + // 融合 axpy：A += scalar * B（mode=3，单次 dispatch 替代 clone+scale+add 三步） + // 分配新 buffer 计算 A + scalar*B，替换 A（遵循 GpuEngine copy-on-write 语义） + [[nodiscard]] Result axpy_inplace(Tensor& A, Scalar scalar, const Tensor& B) override + { + if (A.rows() != B.rows() || A.cols() != B.cols()) + return std::unexpected(Error{"axpy_inplace: shape mismatch"}); + + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto b_gpu = ensure_gpu(B); + if (!b_gpu) return std::unexpected(b_gpu.error()); + + const uint32_t count = static_cast(A.rows() * A.cols()); + // AXPY 模式 (mode=3): out = A + scalar_b * B + auto r = backend_.elementwise_v2_gpu( + a_gpu->gpu_tensor(), &b_gpu->gpu_tensor(), nullptr, + count, 3u, 0u, 0u, 0u, static_cast(scalar), 0.0f, 0.0f); + if (!r) + return std::unexpected(r.error()); + A = Tensor::from_gpu(std::move(*r)); + return {}; + } + + // A = 0：使用 vkCmdFillBuffer 真原地清零（不分配新 buffer） + [[nodiscard]] Result zero(Tensor& A) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.fill_zero_gpu(a_gpu->gpu_tensor()); + if (!r) + return std::unexpected(r.error()); + // fill_zero 修改的是 GPU buffer 本身，Tensor 的 shared_ptr 不变 + // 但若 ensure_gpu 上传了新 Tensor，需要替换 A + if (A.is_cpu()) + A = std::move(*a_gpu); + return {}; + } + + // ══════════════════════════════════════════════════════════════════════ + // 归约原语 + // ══════════════════════════════════════════════════════════════════════ + + [[nodiscard]] Result row_reduce_sum(const Tensor& A) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.reduce_gpu(a_gpu->gpu_tensor(), 0u, 0u); // row, sum + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + [[nodiscard]] Result col_reduce_sum(const Tensor& A) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.reduce_gpu(a_gpu->gpu_tensor(), 1u, 0u); // col, sum + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + [[nodiscard]] Result row_reduce_max(const Tensor& A) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.reduce_gpu(a_gpu->gpu_tensor(), 0u, 1u); // row, max + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + [[nodiscard]] Result col_reduce_max(const Tensor& A) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto r = backend_.reduce_gpu(a_gpu->gpu_tensor(), 1u, 1u); // col, max + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // ══════════════════════════════════════════════════════════════════════ + // 广播原语 + // ══════════════════════════════════════════════════════════════════════ + + // A[r][c] = op(A[r][c], row_vec[r])：分配新 buffer，替换 A + [[nodiscard]] Result broadcast_row_inplace( + Tensor& A, const Tensor& row_vec, BinaryOp op) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto rv_gpu = ensure_gpu(row_vec); + if (!rv_gpu) return std::unexpected(rv_gpu.error()); + + auto r = backend_.broadcast_gpu( + a_gpu->gpu_tensor(), rv_gpu->gpu_tensor(), + 0u, static_cast(op)); // row_broadcast + if (!r) return std::unexpected(r.error()); + A = Tensor::from_gpu(std::move(*r)); + return {}; + } + + // A[r][c] = op(A[r][c], col_vec[c])：分配新 buffer，替换 A + [[nodiscard]] Result broadcast_col_inplace( + Tensor& A, const Tensor& col_vec, BinaryOp op) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto cv_gpu = ensure_gpu(col_vec); + if (!cv_gpu) return std::unexpected(cv_gpu.error()); + + auto r = backend_.broadcast_gpu( + a_gpu->gpu_tensor(), cv_gpu->gpu_tensor(), + 1u, static_cast(op)); // col_broadcast + if (!r) return std::unexpected(r.error()); + A = Tensor::from_gpu(std::move(*r)); + return {}; + } + + // ══════════════════════════════════════════════════════════════════════ + // 逐元素原语 + // ══════════════════════════════════════════════════════════════════════ + + // out = unary_op(A) + [[nodiscard]] Result elementwise_unary( + UnaryOp op, const Tensor& A) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + + const uint32_t count = static_cast(A.rows() * A.cols()); + auto r = backend_.elementwise_v2_gpu( + a_gpu->gpu_tensor(), nullptr, nullptr, + count, 0u, static_cast(op), 0u, 0u, + 0.0f, 0.0f, 0.0f); // UNARY + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // out = binary_op(A, B) + [[nodiscard]] Result elementwise_binary( + BinaryOp op, const Tensor& A, const Tensor& B) override + { + if (A.rows() != B.rows() || A.cols() != B.cols()) + return std::unexpected(Error{"elementwise_binary: shape mismatch"}); + + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto b_gpu = ensure_gpu(B); + if (!b_gpu) return std::unexpected(b_gpu.error()); + + const uint32_t count = static_cast(A.rows() * A.cols()); + auto r = backend_.elementwise_v2_gpu( + a_gpu->gpu_tensor(), &b_gpu->gpu_tensor(), nullptr, + count, 1u, static_cast(op), 0u, 0u, + 0.0f, 0.0f, 0.0f); // BINARY + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // out = binary_op(A, s) 或 binary_op(s, A) + [[nodiscard]] Result elementwise_binary_scalar( + BinaryOp op, const Tensor& A, Scalar s, bool scalar_first) override + { + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + + const uint32_t count = static_cast(A.rows() * A.cols()); + // flags: bit0 = B is scalar, bit3 = scalar first + const uint32_t flags = 1u | (scalar_first ? 8u : 0u); + auto r = backend_.elementwise_v2_gpu( + a_gpu->gpu_tensor(), nullptr, nullptr, + count, 1u, static_cast(op), 0u, flags, + static_cast(s), 0.0f, 0.0f); // BINARY with scalar + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + + // ══════════════════════════════════════════════════════════════════════ + // 条件选择原语 + // ══════════════════════════════════════════════════════════════════════ + + // out = compare_op(A, scalar_b) ? then_t : scalar_else + [[nodiscard]] Result elementwise_select_scalar_cond( + CompareOp cmp, const Tensor& A, Scalar scalar_b, + const Tensor& then_t, Scalar scalar_else) override + { + if (A.rows() != then_t.rows() || A.cols() != then_t.cols()) + return std::unexpected(Error{"elementwise_select: A and then shape mismatch"}); + + auto a_gpu = ensure_gpu(A); + if (!a_gpu) return std::unexpected(a_gpu.error()); + auto t_gpu = ensure_gpu(then_t); + if (!t_gpu) return std::unexpected(t_gpu.error()); + + const uint32_t count = static_cast(A.rows() * A.cols()); + // SELECT mode: + // b = scalar_b (flags bit0 = 1) + // then_v = B[idx] = then_t (flags bit1 = 0, uses binding 1) + // else_v = scalar_else (flags bit2 = 1) + const uint32_t flags = 1u | 4u; // bit0: b is scalar, bit2: else is scalar + auto r = backend_.elementwise_v2_gpu( + a_gpu->gpu_tensor(), &t_gpu->gpu_tensor(), nullptr, + count, 2u, 0u, static_cast(cmp), flags, + static_cast(scalar_b), 0.0f, + static_cast(scalar_else)); // SELECT + if (!r) return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } + +private: + // ── 辅助：确保 Tensor 在 GPU 上 ────────────────────────────────────── + // 若已是 GPU，返回共享拷贝（零开销）；若为 CPU，上传到 GPU。 + // 纯 GPU 架构下，所有 Tensor 应已是 GPU，此方法为防御性兜底。 + [[nodiscard]] Result ensure_gpu(const Tensor& t) + { + if (t.is_gpu()) + return t; // 共享拷贝 + if (!t.valid()) + return std::unexpected(Error{"ensure_gpu: invalid tensor"}); + auto r = GpuTensor::from_matrix(t.cpu_matrix(), backend_); + if (!r) + return std::unexpected(r.error()); + return Tensor::from_gpu(std::move(*r)); + } +}; + +} // namespace nn + +#endif // NN_HAS_VULKAN + +#endif // NN_GPU_ENGINE_HPP diff --git a/include/neuralnet.cpp/layer.hpp b/include/neuralnet.cpp/layer.hpp deleted file mode 100644 index ea7133a9..00000000 --- a/include/neuralnet.cpp/layer.hpp +++ /dev/null @@ -1,2143 +0,0 @@ -#ifndef LAYER_HPP -#define LAYER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "matrix.hpp" -#include "nn_config.hpp" - -namespace nn -{ - class Layer - { - public: - virtual ~Layer() = default; - virtual Result forward(const Matrix &input) = 0; - virtual Result backward(const Matrix &grad_output) = 0; - virtual std::vector> parameters() { return {}; } - virtual std::vector> param_gradients() { return {}; } - }; - - class Linear final : public Layer - { - private: - Matrix W_; - Matrix b_; - Matrix grad_W_; - Matrix grad_b_; - Matrix input_cache_; - - // ── 预分配缓冲区：避免 forward/backward 热路径反复分配内存 ────── - Matrix product_buf_; // W * input 的中间结果 - Matrix grad_WT_buf_; // backward: W^T - - // 修复：使用 thread_local 保证多线程构造 Layer 时的线程安全 - inline static thread_local std::mt19937_64 rng_{std::random_device{}()}; - - public: - Linear(std::size_t in_features, std::size_t out_features) - : W_(out_features, in_features), - b_(out_features, 1), - grad_W_(out_features, in_features), - grad_b_(out_features, 1), - input_cache_(), - product_buf_(out_features, 1), - grad_WT_buf_(in_features, out_features) - { - // Xavier 均匀初始化：适合 tanh/sigmoid，对 ReLU 也可用 - const double limit = std::sqrt(6.0 / static_cast(in_features + out_features)); - std::uniform_real_distribution dist(-limit, limit); - std::ranges::generate(W_.data(), [&] - { return dist(rng_); }); - } - - std::vector> parameters() override - { - return {std::ref(W_), std::ref(b_)}; - } - - std::vector> param_gradients() override - { - return {std::ref(grad_W_), std::ref(grad_b_)}; - } - - Result forward(const Matrix &input) override - { - if (input.rows() != W_.cols()) - return std::unexpected(Error{"linear forward input shape mismatch"}); - - input_cache_ = input; - - // 融合矩阵乘法 + bias 加法，减少一次完整遍历 - const std::size_t out_feat = W_.rows(); - const std::size_t batch = input.cols(); - - // product = W * input（写入预分配缓冲区，避免分配） - W_.multiply_to(product_buf_, input); - - // result = product + bias（返回新矩阵，NRVO 优化） - Matrix result(out_feat, batch); - const auto prod_span = product_buf_.span(); - const auto bias_span = b_.span(); - auto res_span = result.span(); - const auto total = static_cast(out_feat * batch); - - auto indices = std::views::iota(std::size_t{0}, total); - SmartPolicy::for_each(indices.begin(), indices.end(), - [prod_span, bias_span, res_span, batch](std::size_t idx) noexcept - { - res_span[idx] = prod_span[idx] + bias_span[idx / batch]; - }); - - return result; - } - - Result backward(const Matrix &grad_output) override - { - if (grad_output.rows() != W_.rows()) - return std::unexpected(Error{"linear backward grad_output shape mismatch"}); - if (input_cache_.rows() != W_.cols() || input_cache_.cols() != grad_output.cols()) - return std::unexpected(Error{"linear backward cache/input shape mismatch"}); - - const std::size_t in_feat = W_.cols(); - const std::size_t out_feat = W_.rows(); - const std::size_t batch = grad_output.cols(); - - // grad_input = W^T * grad_output - W_.transpose_to(grad_WT_buf_); - Matrix grad_input(in_feat, batch); - grad_WT_buf_.multiply_to(grad_input, grad_output); - - // grad_W += grad_output * input_cache_^T（逐元素累加，避免临时矩阵） - // 手动计算：grad_W[i][j] += sum_k(grad_output[i][k] * input_cache_[j][k]) - { - const auto go_span = grad_output.span(); - const auto ic_span = input_cache_.span(); - auto gw_span = grad_W_.span(); - const std::size_t gw_size = out_feat * in_feat; - auto gw_indices = std::views::iota(std::size_t{0}, gw_size); - SmartPolicy::for_each(gw_indices.begin(), gw_indices.end(), - [go_span, ic_span, gw_span, in_feat, batch](std::size_t idx) noexcept - { - const std::size_t of = idx / in_feat; - const std::size_t inf = idx % in_feat; - double sum = 0.0; - for (std::size_t b = 0; b < batch; ++b) - sum += go_span[of * batch + b] * ic_span[inf * batch + b]; - gw_span[idx] += sum; - }); - } - - // grad_b += sum(grad_output, dim=batch) - { - const auto go_span = grad_output.span(); - auto gb_span = grad_b_.span(); - for (std::size_t of = 0; of < out_feat; ++of) - { - double sum = 0.0; - for (std::size_t b = 0; b < batch; ++b) - sum += go_span[of * batch + b]; - gb_span[of] += sum; - } - } - - return grad_input; - } - }; - - class ReLU final : public Layer - { - private: - Matrix input_cache_; - - public: - ReLU() = default; - - Result forward(const Matrix &input) override - { - input_cache_ = input; - - const auto in_span = input.span(); - const auto n = input.size(); - - Matrix result(input.rows(), input.cols()); - auto out_span = result.span(); - - if (n >= SmartPolicy::PARALLEL_THRESHOLD) { - auto indices = std::views::iota(std::size_t{0}, n); - SmartPolicy::for_each(indices.begin(), indices.end(), - [in_span, out_span](std::size_t i) noexcept - { out_span[i] = in_span[i] > 0.0 ? in_span[i] : 0.0; }); - } else { - for (std::size_t i = 0; i < n; ++i) - out_span[i] = in_span[i] > 0.0 ? in_span[i] : 0.0; - } - return result; - } - - Result backward(const Matrix &grad_output) override - { - if (input_cache_.rows() != grad_output.rows() || input_cache_.cols() != grad_output.cols()) - return std::unexpected(Error{"relu backward shape mismatch"}); - - const auto in_span = input_cache_.span(); - const auto go_span = grad_output.span(); - const auto n = grad_output.size(); - - Matrix grad_input(grad_output.rows(), grad_output.cols()); - auto out_span = grad_input.span(); - - if (n >= SmartPolicy::PARALLEL_THRESHOLD) { - auto indices = std::views::iota(std::size_t{0}, n); - SmartPolicy::for_each(indices.begin(), indices.end(), - [in_span, go_span, out_span](std::size_t i) noexcept - { out_span[i] = in_span[i] > 0.0 ? go_span[i] : 0.0; }); - } else { - for (std::size_t i = 0; i < n; ++i) - out_span[i] = in_span[i] > 0.0 ? go_span[i] : 0.0; - } - return grad_input; - } - }; - - class GeLU final : public Layer - { - private: - Matrix input_cache_; - Matrix sigmoid_cache_; // 缓存 sigmoid(1.702 * x) 用于反向传播 - static constexpr double BETA = 1.702; - - public: - GeLU() = default; - - // QuickGeLU: x * sigmoid(1.702 * x) - Result forward(const Matrix &input) override - { - const auto n = input.size(); - input_cache_ = input; - sigmoid_cache_.resize(input.rows(), input.cols()); - - const auto in_span = input.span(); - auto sig_span = sigmoid_cache_.span(); - - Matrix result(input.rows(), input.cols()); - auto out_span = result.span(); - - if (n >= SmartPolicy::PARALLEL_THRESHOLD) { - auto indices = std::views::iota(std::size_t{0}, n); - SmartPolicy::for_each(indices.begin(), indices.end(), - [in_span, out_span, sig_span](std::size_t i) noexcept - { - double sigmoid_input = BETA * in_span[i]; - double sigmoid_val = 1.0 / (1.0 + std::exp(-sigmoid_input)); - sig_span[i] = sigmoid_val; - out_span[i] = in_span[i] * sigmoid_val; - }); - } else { - for (std::size_t i = 0; i < n; ++i) { - double sigmoid_input = BETA * in_span[i]; - double sigmoid_val = 1.0 / (1.0 + std::exp(-sigmoid_input)); - sig_span[i] = sigmoid_val; - out_span[i] = in_span[i] * sigmoid_val; - } - } - return result; - } - - // d/dx [x * sigmoid(βx)] = sigmoid(βx) + x * β * sigmoid(βx) * (1 - sigmoid(βx)) - // = sigmoid(βx) * [1 + βx * (1 - sigmoid(βx))] - Result backward(const Matrix &grad_output) override - { - if (input_cache_.rows() != grad_output.rows() || input_cache_.cols() != grad_output.cols()) - return std::unexpected(Error{"gelu backward shape mismatch"}); - - const auto in_span = input_cache_.span(); - const auto sig_span = sigmoid_cache_.span(); - const auto go_span = grad_output.span(); - const auto n = grad_output.size(); - - Matrix grad_input(grad_output.rows(), grad_output.cols()); - auto out_span = grad_input.span(); - - if (n >= SmartPolicy::PARALLEL_THRESHOLD) { - auto indices = std::views::iota(std::size_t{0}, n); - SmartPolicy::for_each(indices.begin(), indices.end(), - [in_span, sig_span, go_span, out_span](std::size_t i) noexcept - { - double s = sig_span[i]; - out_span[i] = go_span[i] * s * (1.0 + BETA * in_span[i] * (1.0 - s)); - }); - } else { - for (std::size_t i = 0; i < n; ++i) { - double s = sig_span[i]; - out_span[i] = go_span[i] * s * (1.0 + BETA * in_span[i] * (1.0 - s)); - } - } - return grad_input; - } - }; - - class LayerNorm final : public Layer - { - private: - std::size_t normalized_shape_; - double epsilon_; - - // 可学习参数 - Matrix gamma_; // 缩放参数 (normalized_shape_, 1) - Matrix beta_; // 偏移参数 (normalized_shape_, 1) - Matrix grad_gamma_; // gamma 梯度 - Matrix grad_beta_; // beta 梯度 - - // 缓存用于反向传播 - Matrix input_cache_; - Matrix normalized_cache_; // 归一化后的值 - Matrix std_cache_; // 标准差倒数 (1/sqrt(σ² + ε)) - Matrix mean_cache_; // 均值 - - // 数值稳定性 - static constexpr double EPSILON = 1e-5; - - public: - explicit LayerNorm(std::size_t normalized_shape, double epsilon = EPSILON) - : normalized_shape_(normalized_shape), - epsilon_(epsilon), - gamma_(normalized_shape, 1, 1.0), // 初始化为1 - beta_(normalized_shape, 1, 0.0), // 初始化为0 - grad_gamma_(normalized_shape, 1), - grad_beta_(normalized_shape, 1) - { - // 参数初始化已在构造函数中完成 - } - - std::vector> parameters() override - { - return {std::ref(gamma_), std::ref(beta_)}; - } - - std::vector> param_gradients() override - { - return {std::ref(grad_gamma_), std::ref(grad_beta_)}; - } - - Result forward(const Matrix &input) override - { - // 输入形状: (normalized_shape_, batch_size) - if (input.rows() != normalized_shape_) - return std::unexpected(Error{"layer_norm forward input shape mismatch"}); - - input_cache_ = input; - const std::size_t batch_size = input.cols(); - const std::size_t features = input.rows(); - - // 缓存中间结果（复用已有内存） - mean_cache_.resize(1, batch_size); - std_cache_.resize(1, batch_size); - normalized_cache_.resize(features, batch_size); - - Matrix result(features, batch_size); - - const auto in_span = input.span(); - const auto gamma_span = gamma_.span(); - const auto beta_span = beta_.span(); - auto mean_span = mean_cache_.span(); - auto std_span = std_cache_.span(); - auto norm_span = normalized_cache_.span(); - auto res_span = result.span(); - - // 对每个样本（列）独立处理 - auto batch_indices = std::views::iota(std::size_t{0}, batch_size); - - if (batch_size >= SmartPolicy::PARALLEL_THRESHOLD) { - SmartPolicy::for_each(batch_indices.begin(), batch_indices.end(), - [in_span, gamma_span, beta_span, mean_span, std_span, norm_span, res_span, - features, batch_size, epsilon = epsilon_](std::size_t b) noexcept - { - // 计算均值 - double sum = 0.0; - for (std::size_t f = 0; f < features; ++f) - sum += in_span[f * batch_size + b]; - double mean = sum / static_cast(features); - mean_span[b] = mean; - - // 计算方差 - double var_sum = 0.0; - for (std::size_t f = 0; f < features; ++f) { - double diff = in_span[f * batch_size + b] - mean; - var_sum += diff * diff; - } - double variance = var_sum / static_cast(features); - double std_inv = 1.0 / std::sqrt(variance + epsilon); - std_span[b] = std_inv; - - // 归一化和仿射变换 - for (std::size_t f = 0; f < features; ++f) { - double normalized = (in_span[f * batch_size + b] - mean) * std_inv; - norm_span[f * batch_size + b] = normalized; - res_span[f * batch_size + b] = gamma_span[f] * normalized + beta_span[f]; - } - }); - } else { - for (std::size_t b = 0; b < batch_size; ++b) { - // 计算均值 - double sum = 0.0; - for (std::size_t f = 0; f < features; ++f) - sum += in_span[f * batch_size + b]; - double mean = sum / static_cast(features); - mean_span[b] = mean; - - // 计算方差 - double var_sum = 0.0; - for (std::size_t f = 0; f < features; ++f) { - double diff = in_span[f * batch_size + b] - mean; - var_sum += diff * diff; - } - double variance = var_sum / static_cast(features); - double std_inv = 1.0 / std::sqrt(variance + epsilon_); - std_span[b] = std_inv; - - // 归一化和仿射变换 - for (std::size_t f = 0; f < features; ++f) { - double normalized = (in_span[f * batch_size + b] - mean) * std_inv; - norm_span[f * batch_size + b] = normalized; - res_span[f * batch_size + b] = gamma_span[f] * normalized + beta_span[f]; - } - } - } - - return result; - } - - Result backward(const Matrix &grad_output) override - { - if (grad_output.rows() != normalized_shape_) - return std::unexpected(Error{"layer_norm backward shape mismatch"}); - - const std::size_t features = normalized_shape_; - const std::size_t batch_size = grad_output.cols(); - - Matrix grad_input(features, batch_size); - - const auto go_span = grad_output.span(); - const auto norm_span = normalized_cache_.span(); - const auto std_span = std_cache_.span(); - const auto gamma_span = gamma_.span(); - auto gi_span = grad_input.span(); - auto gg_span = grad_gamma_.span(); - auto gb_span = grad_beta_.span(); - - // 计算梯度 - auto batch_indices = std::views::iota(std::size_t{0}, batch_size); - - if (batch_size >= SmartPolicy::PARALLEL_THRESHOLD) { - // 先并行计算每个样本的 dL/dx（无竞争）， - // 再串行累加 dL/dγ 和 dL/dβ（避免数据竞争） - SmartPolicy::for_each(batch_indices.begin(), batch_indices.end(), - [go_span, norm_span, std_span, gamma_span, gi_span, - features, batch_size](std::size_t b) noexcept - { - double std_inv = std_span[b]; - double sum_grad = 0.0; - double sum_grad_norm = 0.0; - for (std::size_t f = 0; f < features; ++f) { - double g = go_span[f * batch_size + b] * gamma_span[f]; - sum_grad += g; - sum_grad_norm += g * norm_span[f * batch_size + b]; - } - const double inv_features = 1.0 / static_cast(features); - for (std::size_t f = 0; f < features; ++f) { - double grad_out = go_span[f * batch_size + b]; - double g = grad_out * gamma_span[f]; - gi_span[f * batch_size + b] = (g - sum_grad * inv_features - - norm_span[f * batch_size + b] * sum_grad_norm * inv_features) * std_inv; - } - }); - // 串行累加 dL/dγ 和 dL/dβ - for (std::size_t b = 0; b < batch_size; ++b) { - for (std::size_t f = 0; f < features; ++f) { - double grad_out = go_span[f * batch_size + b]; - gg_span[f] += grad_out * norm_span[f * batch_size + b]; - gb_span[f] += grad_out; - } - } - } else { - for (std::size_t b = 0; b < batch_size; ++b) { - double std_inv = std_span[b]; - - // 预计算统计量：O(N) 而非 O(N²) - double sum_grad = 0.0; - double sum_grad_norm = 0.0; - for (std::size_t f = 0; f < features; ++f) { - double g = go_span[f * batch_size + b] * gamma_span[f]; - sum_grad += g; - sum_grad_norm += g * norm_span[f * batch_size + b]; - } - - // 单次遍历：同时计算 dL/dγ、dL/dβ、dL/dx - const double inv_features = 1.0 / static_cast(features); - for (std::size_t f = 0; f < features; ++f) { - double grad_out = go_span[f * batch_size + b]; - double g = grad_out * gamma_span[f]; - gg_span[f] += grad_out * norm_span[f * batch_size + b]; - gb_span[f] += grad_out; - gi_span[f * batch_size + b] = (g - sum_grad * inv_features - - norm_span[f * batch_size + b] * sum_grad_norm * inv_features) * std_inv; - } - } - } - - return grad_input; - } - - }; - - // ── Softmax 激活层 ───────────────────────────────────────────────────── - // 对每一行独立应用 softmax：out[i][j] = exp(in[i][j]) / Σ_k exp(in[i][k]) - // 输入形状: (rows, cols)，输出形状相同。 - // 使用最大值减法保证数值稳定性。 - // 反向传播: grad[i][j] = out[i][j] * (grad_out[i][j] - Σ_k out[i][k] * grad_out[i][k]) - // ──────────────────────────────────────────────────────────────────────── - class Softmax final : public Layer - { - private: - Matrix output_cache_; - - public: - Softmax() = default; - - Result forward(const Matrix &input) override - { - const std::size_t rows = input.rows(); - const std::size_t cols = input.cols(); - output_cache_.resize(rows, cols); - - Matrix result(rows, cols); - const auto in_span = input.span(); - auto out_span = result.span(); - auto cache_span = output_cache_.span(); - - auto row_indices = std::views::iota(std::size_t{0}, rows); - const std::size_t total = rows * cols; - - auto process_row = [in_span, out_span, cache_span, cols](std::size_t r) noexcept - { - const std::size_t offset = r * cols; - const auto row_in = in_span.subspan(offset, cols); - auto row_out = out_span.subspan(offset, cols); - auto row_cache = cache_span.subspan(offset, cols); - - // 数值稳定：减去行内最大值 - double max_val = row_in[0]; - for (std::size_t c = 1; c < cols; ++c) - max_val = std::max(max_val, row_in[c]); - - // 计算 exp 和求和 - double sum = 0.0; - for (std::size_t c = 0; c < cols; ++c) - { - double e = std::exp(row_in[c] - max_val); - row_out[c] = e; - row_cache[c] = e; - sum += e; - } - - // 归一化 - const double inv_sum = 1.0 / sum; - for (std::size_t c = 0; c < cols; ++c) - { - row_out[c] *= inv_sum; - row_cache[c] *= inv_sum; - } - }; - - if (total >= SmartPolicy::PARALLEL_THRESHOLD) - SmartPolicy::for_each(row_indices.begin(), row_indices.end(), process_row); - else - for (std::size_t r = 0; r < rows; ++r) - process_row(r); - - return result; - } - - Result backward(const Matrix &grad_output) override - { - const std::size_t rows = output_cache_.rows(); - const std::size_t cols = output_cache_.cols(); - - if (grad_output.rows() != rows || grad_output.cols() != cols) - return std::unexpected(Error{"softmax backward shape mismatch"}); - - Matrix grad_input(rows, cols); - - const auto go_span = grad_output.span(); - const auto out_span = output_cache_.span(); - auto gi_span = grad_input.span(); - - auto row_indices = std::views::iota(std::size_t{0}, rows); - const std::size_t total = rows * cols; - - auto process_row = [go_span, out_span, gi_span, cols](std::size_t r) noexcept - { - const std::size_t offset = r * cols; - const auto row_go = go_span.subspan(offset, cols); - const auto row_out = out_span.subspan(offset, cols); - auto row_gi = gi_span.subspan(offset, cols); - - // dot = Σ_k out[k] * grad_out[k] - double dot = 0.0; - for (std::size_t c = 0; c < cols; ++c) - dot += row_out[c] * row_go[c]; - - // grad = out * (grad_out - dot) - for (std::size_t c = 0; c < cols; ++c) - row_gi[c] = row_out[c] * (row_go[c] - dot); - }; - - if (total >= SmartPolicy::PARALLEL_THRESHOLD) - SmartPolicy::for_each(row_indices.begin(), row_indices.end(), process_row); - else - for (std::size_t r = 0; r < rows; ++r) - process_row(r); - - return grad_input; - } - }; - - // ── 多头注意力层 ────────────────────────────────────────────────────── - // 来源: "Attention Is All You Need" (Vaswani et al., 2017) - // - // MultiHead(Q, K, V) = Concat(head_1, ..., head_h) W^O - // head_i = Attention(QW_i^Q, KW_i^K, VW_i^V) - // Attention(Q, K, V) = softmax(QK^T / √d_k) V - // - // 输入形状: (d_model, seq_len)，输出形状: (d_model, seq_len) - // ───────────────────────────────────────────────────────────────────── - class MultiHeadAttention final : public Layer - { - private: - std::size_t d_model_; - std::size_t num_heads_; - std::size_t d_k_; - double scale_; // 1.0 / sqrt(d_k) - - // 投影层 - Linear W_q_; - Linear W_k_; - Linear W_v_; - Linear W_o_; - - // 前向传播缓存 - Matrix Q_cache_; // (d_model, seq_len) - Matrix K_cache_; // (d_model, seq_len) - Matrix V_cache_; // (d_model, seq_len) - std::vector attn_; // num_heads_ × (seq_len, seq_len) — softmax 输出 - - // Per-head 切片缓存（forward 提取，backward 使用） - std::vector Q_heads_; // num_heads_ × (d_k_, seq_len) - std::vector K_heads_; - std::vector V_heads_; - std::vector O_heads_; // num_heads_ × (d_k_, seq_len) — 前向输出缓存 - - // 辅助缓冲区（避免循环内重复分配） - Matrix grad_scores_buf_; // (seq_len, seq_len) - Matrix grad_A_buf_; // (seq_len, seq_len) - Matrix grad_O_h_buf_; // (d_k_, seq_len) — backward 中复用 - Softmax softmax_; // 仅在 forward() 中调用 forward()，不调用 backward()（注意力反向手动计算） - - // ── 从矩阵中提取行切片到 dst ──────────────────────────────────── - static void extract_rows(const Matrix &src, std::size_t row_start, - std::size_t row_count, Matrix &dst) - { - const std::size_t cols = src.cols(); - dst.resize(row_count, cols); - const auto src_span = src.span(); - auto dst_span = dst.span(); - for (std::size_t r = 0; r < row_count; ++r) - std::copy_n(src_span.begin() + (row_start + r) * cols, cols, dst_span.begin() + r * cols); - } - - // ── 将 src 写入 dst 的指定行范围 ──────────────────────────────── - static void insert_rows(Matrix &dst, std::size_t row_start, - const Matrix &src) - { - const std::size_t row_count = src.rows(); - const std::size_t cols = src.cols(); - auto dst_span = dst.span(); - const auto src_span = src.span(); - for (std::size_t r = 0; r < row_count; ++r) - std::copy_n(src_span.begin() + r * cols, cols, dst_span.begin() + (row_start + r) * cols); - } - - // ── 逐元素缩放 ────────────────────────────────────────────────── - static void scale_inplace(Matrix &m, double s) - { - auto m_span = m.span(); - const auto n = m.size(); - if (n >= SmartPolicy::PARALLEL_THRESHOLD) - { - auto indices = std::views::iota(std::size_t{0}, n); - SmartPolicy::for_each(indices.begin(), indices.end(), - [m_span, s](std::size_t i) noexcept { m_span[i] *= s; }); - } - else - { - for (std::size_t i = 0; i < n; ++i) - m_span[i] *= s; - } - } - - public: - MultiHeadAttention(std::size_t d_model, std::size_t num_heads) - : d_model_(d_model), - num_heads_(num_heads), - d_k_(d_model / num_heads), - scale_(1.0 / std::sqrt(static_cast(d_model / num_heads))), - W_q_(d_model, d_model), - W_k_(d_model, d_model), - W_v_(d_model, d_model), - W_o_(d_model, d_model) - { - if (d_model % num_heads != 0) - assert(false && "MultiHeadAttention: d_model must be divisible by num_heads"); // NOLINT - } - - std::vector> parameters() override - { - auto params = W_q_.parameters(); - auto wk = W_k_.parameters(); - auto wv = W_v_.parameters(); - auto wo = W_o_.parameters(); - params.insert(params.end(), wk.begin(), wk.end()); - params.insert(params.end(), wv.begin(), wv.end()); - params.insert(params.end(), wo.begin(), wo.end()); - return params; - } - - std::vector> param_gradients() override - { - auto grads = W_q_.param_gradients(); - auto gk = W_k_.param_gradients(); - auto gv = W_v_.param_gradients(); - auto go = W_o_.param_gradients(); - grads.insert(grads.end(), gk.begin(), gk.end()); - grads.insert(grads.end(), gv.begin(), gv.end()); - grads.insert(grads.end(), go.begin(), go.end()); - return grads; - } - - Result forward(const Matrix &input) override - { - if (input.rows() != d_model_) - return std::unexpected(Error{"MultiHeadAttention forward input shape mismatch"}); - - const std::size_t seq_len = input.cols(); - - // ── 1. 线性投影 Q, K, V ────────────────────────────────────── - auto q_res = W_q_.forward(input); - if (!q_res) return q_res; - Q_cache_ = *q_res; - auto k_res = W_k_.forward(input); - if (!k_res) return k_res; - K_cache_ = *k_res; - auto v_res = W_v_.forward(input); - if (!v_res) return v_res; - V_cache_ = *v_res; - - // ── 2. 分配 per-head 缓冲区 ────────────────────────────────── - Q_heads_.resize(num_heads_); - K_heads_.resize(num_heads_); - V_heads_.resize(num_heads_); - O_heads_.resize(num_heads_); - attn_.resize(num_heads_); - - // ── 3. 逐头计算缩放点积注意力 ──────────────────────────────── - for (std::size_t h = 0; h < num_heads_; ++h) - { - const std::size_t row_start = h * d_k_; - - // 提取 Q_h, K_h, V_h → (d_k_, seq_len) - extract_rows(Q_cache_, row_start, d_k_, Q_heads_[h]); - extract_rows(K_cache_, row_start, d_k_, K_heads_[h]); - extract_rows(V_cache_, row_start, d_k_, V_heads_[h]); - - // S_h = Q_h^T @ K_h → (seq_len, seq_len) - Matrix QhT = Q_heads_[h].transpose(); - attn_[h].resize(seq_len, seq_len); - QhT.multiply_to(attn_[h], K_heads_[h]); - - // S_h *= 1/√d_k - scale_inplace(attn_[h], scale_); - - // A_h = softmax(S_h) — 按行 softmax（复用成员 softmax_） - auto sm_res = softmax_.forward(attn_[h]); - if (!sm_res) return sm_res; - attn_[h] = *sm_res; - - // O_h = V_h @ A_h → (d_k_, seq_len)，缓存到 O_heads_ - O_heads_[h].resize(d_k_, seq_len); - V_heads_[h].multiply_to(O_heads_[h], attn_[h]); - } - - // ── 4. 拼接所有头的输出并投影 ──────────────────────────────── - Matrix output(d_model_, seq_len); - for (std::size_t h = 0; h < num_heads_; ++h) - insert_rows(output, h * d_k_, O_heads_[h]); - - auto wo_res = W_o_.forward(output); - if (!wo_res) return wo_res; - return *wo_res; - } - - Result backward(const Matrix &grad_output) override - { - const std::size_t seq_len = grad_output.cols(); - - // ── 1. 输出投影反向 ────────────────────────────────────────── - auto gc_res = W_o_.backward(grad_output); - if (!gc_res) return gc_res; - Matrix grad_concat = *gc_res; // (d_model, seq_len) - - // ── 2. 初始化各头梯度累加矩阵 ──────────────────────────────── - Matrix grad_Q_all(d_model_, seq_len); - Matrix grad_K_all(d_model_, seq_len); - Matrix grad_V_all(d_model_, seq_len); - { - grad_Q_all.zero(); - grad_K_all.zero(); - grad_V_all.zero(); - } - - // ── 3. 逐头计算注意力反向 ──────────────────────────────────── - for (std::size_t h = 0; h < num_heads_; ++h) - { - const std::size_t row_start = h * d_k_; - const Matrix &Q_h = Q_heads_[h]; // (d_k_, seq_len) - const Matrix &K_h = K_heads_[h]; // (d_k_, seq_len) - const Matrix &V_h = V_heads_[h]; // (d_k_, seq_len) - const Matrix &A_h = attn_[h]; // (seq_len, seq_len) - - // 提取 grad_O_h（复用成员缓冲区） - extract_rows(grad_concat, row_start, d_k_, grad_O_h_buf_); - - // grad_V_h = grad_O_h @ A_h^T → (d_k_, seq_len) - Matrix AhT = A_h.transpose(); - Matrix grad_V_h(d_k_, seq_len); - grad_O_h_buf_.multiply_to(grad_V_h, AhT); - - // grad_A_h = V_h^T @ grad_O_h → (seq_len, seq_len) - Matrix VhT = V_h.transpose(); - grad_A_buf_.resize(seq_len, seq_len); - VhT.multiply_to(grad_A_buf_, grad_O_h_buf_); - - // grad_S_h = A_h ⊙ (grad_A_h - row_sum(A_h ⊙ grad_A_h)) - grad_scores_buf_.resize(seq_len, seq_len); - { - const auto a_span = A_h.span(); - const auto ga_span = grad_A_buf_.span(); - auto gs_span = grad_scores_buf_.span(); - - for (std::size_t i = 0; i < seq_len; ++i) - { - double dot = 0.0; - for (std::size_t j = 0; j < seq_len; ++j) - dot += a_span[i * seq_len + j] * ga_span[i * seq_len + j]; - for (std::size_t j = 0; j < seq_len; ++j) - gs_span[i * seq_len + j] = - a_span[i * seq_len + j] * (ga_span[i * seq_len + j] - dot); - } - } - - // grad_Q_h = K_h @ grad_S_h^T * scale → (d_k_, seq_len) - Matrix gsT = grad_scores_buf_.transpose(); - Matrix grad_Q_h(d_k_, seq_len); - K_h.multiply_to(grad_Q_h, gsT); - scale_inplace(grad_Q_h, scale_); - - // grad_K_h = Q_h @ grad_S_h * scale → (d_k_, seq_len) - Matrix grad_K_h(d_k_, seq_len); - Q_h.multiply_to(grad_K_h, grad_scores_buf_); - scale_inplace(grad_K_h, scale_); - - // 累加到全局梯度（每个头写入不同行，无重叠） - insert_rows(grad_Q_all, row_start, grad_Q_h); - insert_rows(grad_K_all, row_start, grad_K_h); - insert_rows(grad_V_all, row_start, grad_V_h); - } - - // ── 4. 投影层反向，累加输入梯度 ────────────────────────────── - auto giq_res = W_q_.backward(grad_Q_all); - if (!giq_res) return giq_res; - Matrix grad_input = *giq_res; - auto gik_res = W_k_.backward(grad_K_all); - if (!gik_res) return gik_res; - grad_input = grad_input + *gik_res; - auto giv_res = W_v_.backward(grad_V_all); - if (!giv_res) return giv_res; - grad_input = grad_input + *giv_res; - - return grad_input; - } - }; - - // ── 正弦波固定位置编码 ─────────────────────────────────────────────── - // 来源: "Attention Is All You Need" (Vaswani et al., 2017) - // - // PE(pos, 2i) = sin(pos / 10000^(2i/d_model)) - // PE(pos, 2i+1) = cos(pos / 10000^(2i/d_model)) - // - // 输入形状: (d_model, seq_len)，输出形状相同。 - // 编码矩阵在构造时一次性预计算，前向传播仅做逐元素加法。 - // 反向传播直接穿透（编码不可学习）。 - // ───────────────────────────────────────────────────────────────────── - class PositionalEncoding final : public Layer - { - private: - std::size_t d_model_; - std::size_t max_len_; - Matrix encoding_; // (d_model, max_len) — 预计算的正弦波编码 - - public: - PositionalEncoding(std::size_t d_model, std::size_t max_len = 5000) - : d_model_(d_model), - max_len_(max_len), - encoding_(d_model, max_len) - { - // ── 一次性预计算频率与编码 ── - // 先计算每个特征对的角频率（避免在 position 循环中重复 pow） - const std::size_t half = d_model / 2; - std::vector freqs(half); - for (std::size_t i = 0; i < half; ++i) - freqs[i] = 1.0 / std::pow(10000.0, static_cast(2 * i) / d_model); - - auto e_span = encoding_.span(); - for (std::size_t pos = 0; pos < max_len; ++pos) - { - const double pos_d = static_cast(pos); - for (std::size_t i = 0; i < half; ++i) - { - const double angle = pos_d * freqs[i]; - e_span[(2 * i) * max_len + pos] = std::sin(angle); - e_span[(2 * i + 1) * max_len + pos] = std::cos(angle); - } - // 奇数维度：最后一个特征仅使用 sin - if (d_model % 2 == 1) - { - const double freq_last = 1.0 / std::pow(10000.0, static_cast(2 * half) / d_model); - const double angle = pos_d * freq_last; - e_span[(d_model - 1) * max_len + pos] = std::sin(angle); - } - } - } - - std::vector> parameters() override { return {}; } - std::vector> param_gradients() override { return {}; } - - Result forward(const Matrix &input) override - { - if (input.rows() != d_model_) - return std::unexpected(Error{"positional encoding forward: d_model mismatch"}); - - const std::size_t seq_len = input.cols(); - - if (seq_len > max_len_) - return std::unexpected(Error{"positional encoding forward: sequence length exceeds max_len"}); - - // result = input + encoding[:, 0:seq_len] - Matrix result(d_model_, seq_len); - const auto in_span = input.span(); - const auto e_span = encoding_.span(); - auto out_span = result.span(); - - const auto total = static_cast(d_model_ * seq_len); - const std::size_t max_len = max_len_; - auto indices = std::views::iota(std::size_t{0}, total); - SmartPolicy::for_each(indices.begin(), indices.end(), - [in_span, e_span, out_span, seq_len, max_len](std::size_t idx) noexcept - { - // idx = row * seq_len + col - const std::size_t row = idx / seq_len; - const std::size_t col = idx % seq_len; - out_span[idx] = in_span[idx] + e_span[row * max_len + col]; - }); - - return result; - } - - // 位置编码为固定值，梯度直接穿透 - Result backward(const Matrix &grad_output) override { return grad_output; } - }; - - // ── 前馈网络 (Feed-Forward Network) ──────────────────────────────── - // FFN(x) = Linear(d_ff → d_model)(GeLU(Linear(d_model → d_ff)(x))) - // 输入/输出形状: (d_model, seq_len) - class FeedForward final : public Layer - { - private: - Linear fc1_; // (d_ff, d_model) - Linear fc2_; // (d_model, d_ff) - GeLU activation_; - Matrix input_cache_; - - public: - FeedForward(std::size_t d_model, std::size_t d_ff) - : fc1_(d_model, d_ff), fc2_(d_ff, d_model) - {} - - std::vector> parameters() override - { - auto params = fc1_.parameters(); - auto p2 = fc2_.parameters(); - params.insert(params.end(), p2.begin(), p2.end()); - return params; - } - - std::vector> param_gradients() override - { - auto grads = fc1_.param_gradients(); - auto g2 = fc2_.param_gradients(); - grads.insert(grads.end(), g2.begin(), g2.end()); - return grads; - } - - Result forward(const Matrix &input) override - { - input_cache_ = input; - auto fc1_res = fc1_.forward(input); - if (!fc1_res) return fc1_res; - auto act_res = activation_.forward(*fc1_res); - if (!act_res) return act_res; - auto fc2_res = fc2_.forward(*act_res); - if (!fc2_res) return fc2_res; - return *fc2_res; - } - - Result backward(const Matrix &grad_output) override - { - auto b_fc2 = fc2_.backward(grad_output); - if (!b_fc2) return b_fc2; - auto b_act = activation_.backward(*b_fc2); - if (!b_act) return b_act; - auto b_fc1 = fc1_.backward(*b_act); - if (!b_fc1) return b_fc1; - return *b_fc1; - } - }; - - // ── Transformer 编码器层 (Pre-Norm 架构) ────────────────────────── - // x = x + SelfAttn(LayerNorm₁(x)) ← 残差连接 - // x = x + FFN(LayerNorm₂(x)) ← 残差连接 - // 输入/输出形状: (d_model, seq_len) - class TransformerEncoderLayer final : public Layer - { - private: - MultiHeadAttention self_attn_; - LayerNorm norm1_; - FeedForward ff_; - LayerNorm norm2_; - - // ── 反向传播缓存 ── - Matrix residual1_cache_; // 第一个残差连接前的原始输入 - Matrix residual2_cache_; // 第二个残差连接前的输入 (= residual1) - - public: - TransformerEncoderLayer(std::size_t d_model, std::size_t num_heads, std::size_t d_ff) - : self_attn_(d_model, num_heads), - norm1_(d_model), - ff_(d_model, d_ff), - norm2_(d_model) - {} - - std::vector> parameters() override - { - auto params = self_attn_.parameters(); - auto n1 = norm1_.parameters(); - auto f = ff_.parameters(); - auto n2 = norm2_.parameters(); - params.insert(params.end(), n1.begin(), n1.end()); - params.insert(params.end(), f.begin(), f.end()); - params.insert(params.end(), n2.begin(), n2.end()); - return params; - } - - std::vector> param_gradients() override - { - auto grads = self_attn_.param_gradients(); - auto gn1 = norm1_.param_gradients(); - auto gf = ff_.param_gradients(); - auto gn2 = norm2_.param_gradients(); - grads.insert(grads.end(), gn1.begin(), gn1.end()); - grads.insert(grads.end(), gf.begin(), gf.end()); - grads.insert(grads.end(), gn2.begin(), gn2.end()); - return grads; - } - - Result forward(const Matrix &input) override - { - // 子层1: Self-Attention + 残差 - residual1_cache_ = input; - auto n1_res = norm1_.forward(input); - if (!n1_res) return n1_res; - auto sa_res = self_attn_.forward(*n1_res); - if (!sa_res) return sa_res; - residual2_cache_ = input + *sa_res; // 残差连接 - - // 子层2: FFN + 残差 - auto n2_res = norm2_.forward(residual2_cache_); - if (!n2_res) return n2_res; - auto ff_res = ff_.forward(*n2_res); - if (!ff_res) return ff_res; - return residual2_cache_ + *ff_res; - } - - Result backward(const Matrix &grad_output) override - { - // ── 反向传播第2个残差连接: grad_r2 = grad_output (分流到两条路径) ── - // 路径A: 直接流向 residual2 (= residual1) - // 路径B: 流向 FFN - Matrix grad_residual1 = grad_output; // 路径A - Matrix grad_ff_out = grad_output; // 路径B - - // 反向 FFN → LayerNorm₂ - auto b_ff = ff_.backward(grad_ff_out); - if (!b_ff) return b_ff; - auto b_n2 = norm2_.backward(*b_ff); - if (!b_n2) return b_n2; - grad_residual1 = grad_residual1 + *b_n2; - - // ── 反向传播第1个残差连接 ── - // 路径A: 直接流向输入 x - // 路径B: 流向 Self-Attention - Matrix grad_input = grad_residual1; // 路径A - Matrix grad_attn_out = grad_residual1; // 路径B - - // 反向 Self-Attention → LayerNorm₁ - auto b_sa = self_attn_.backward(grad_attn_out); - if (!b_sa) return b_sa; - auto b_n1 = norm1_.backward(*b_sa); - if (!b_n1) return b_n1; - grad_input = grad_input + *b_n1; - - return grad_input; - } - }; - - // ── Transformer 编码器 ────────────────────────────────────────────── - // 堆叠 N 个 TransformerEncoderLayer，内含位置编码 + 全局平均池化 - // 输入: (d_model × num_patches, batch_size) — PatchEmbedding 的输出 - // 输出: (d_model, batch_size) — 池化后的序列表示 - // - // 反向传播策略: 对每个样本 re-forward 以重建内部缓存 (checkpointing) - // ──────────────────────────────────────────────────────────────────── - class TransformerEncoder final : public Layer - { - private: - std::size_t d_model_; - std::size_t num_patches_; - - std::vector layers_; - PositionalEncoding pos_encoding_; - - // ── 反向传播缓存: 每个样本在位置编码后的输入（供 re-forward 重建） ── - std::vector stored_inputs_; // [sample] — 每个样本 PE 后的输入 - std::size_t batch_size_{0}; - - public: - TransformerEncoder(std::size_t d_model, std::size_t num_heads, - std::size_t d_ff, std::size_t num_layers, - std::size_t num_patches) - : d_model_(d_model), num_patches_(num_patches), - pos_encoding_(d_model, num_patches) - { - for (std::size_t i = 0; i < num_layers; ++i) - layers_.emplace_back(d_model, num_heads, d_ff); - } - - std::vector> parameters() override - { - std::vector> params; - for (auto &layer : layers_) - { - auto lp = layer.parameters(); - params.insert(params.end(), lp.begin(), lp.end()); - } - return params; - } - - std::vector> param_gradients() override - { - std::vector> grads; - for (auto &layer : layers_) - { - auto lg = layer.param_gradients(); - grads.insert(grads.end(), lg.begin(), lg.end()); - } - return grads; - } - - // ── 前向传播 ──────────────────────────────────────────────────── - Result forward(const Matrix &input) override - { - batch_size_ = input.cols(); - stored_inputs_.resize(batch_size_); - - Matrix output(d_model_, batch_size_); - - for (std::size_t b = 0; b < batch_size_; ++b) - { - // 从展平向量中提取样本 b → (d_model, num_patches) - Matrix x(d_model_, num_patches_); - for (std::size_t r = 0; r < d_model_; ++r) - for (std::size_t c = 0; c < num_patches_; ++c) - x.set_value_unchecked(r, c, - input.at_unchecked(r * num_patches_ + c, b)); - - // 添加位置编码 - auto pe_res = pos_encoding_.forward(x); - if (!pe_res) return pe_res; - x = *pe_res; - - // 缓存 PE 后的输入，供 backward 中 re-forward 重建缓存 - stored_inputs_[b] = x; - - // 依次通过编码器层 - for (std::size_t l = 0; l < layers_.size(); ++l) - { - auto ly_res = layers_[l].forward(x); - if (!ly_res) return ly_res; - x = *ly_res; - } - - // 全局平均池化: (d_model, num_patches) → (d_model,) - const double inv_n = 1.0 / static_cast(num_patches_); - for (std::size_t r = 0; r < d_model_; ++r) - { - double sum = 0.0; - for (std::size_t c = 0; c < num_patches_; ++c) - sum += x.at_unchecked(r, c); - output.set_value_unchecked(r, b, sum * inv_n); - } - } - return output; - } - - // ── 反向传播 ──────────────────────────────────────────────────── - // 对每个样本: re-forward 重建缓存 → 反向传播 → 累加参数梯度 - Result backward(const Matrix &grad_output) override - { - Matrix grad_input(d_model_ * num_patches_, batch_size_); - grad_input.zero(); - - for (std::size_t b = 0; b < batch_size_; ++b) - { - // ── Re-forward: 重建该样本的全部缓存 ── - // stored_inputs_[b] 已含位置编码，无需重复施加 - Matrix x = stored_inputs_[b]; - for (std::size_t l = 0; l < layers_.size(); ++l) - { - auto ly_res = layers_[l].forward(x); // 重建每层缓存 - if (!ly_res) return ly_res; - x = *ly_res; - } - - // ── 全局平均池化梯度: 展开 ── - const double inv_n = 1.0 / static_cast(num_patches_); - Matrix grad(d_model_, num_patches_); - for (std::size_t r = 0; r < d_model_; ++r) - { - double g = grad_output.at_unchecked(r, b) * inv_n; - for (std::size_t c = 0; c < num_patches_; ++c) - grad.set_value_unchecked(r, c, g); - } - - // ── 反向传播编码器层 ── - for (int l = static_cast(layers_.size()) - 1; l >= 0; --l) - { - auto bl_res = layers_[l].backward(grad); - if (!bl_res) return bl_res; - grad = *bl_res; - } - - // ── 写入 grad_input ── - for (std::size_t r = 0; r < d_model_; ++r) - for (std::size_t c = 0; c < num_patches_; ++c) - grad_input.set_value_unchecked(r * num_patches_ + c, b, - grad.at_unchecked(r, c)); - } - return grad_input; - } - }; - - // ── Patch 嵌入层 ────────────────────────────────────────────────── - // 将展平的图像分割为不重叠的 patch 并投影到 d_model 维空间 - // 输入: (img_size², batch_size) — 展平的 28×28 图像 - // 输出: (d_model × num_patches, batch_size) — 展平的 patch 序列 - // ──────────────────────────────────────────────────────────────────── - class PatchEmbedding final : public Layer - { - private: - std::size_t img_size_; - std::size_t patch_size_; - std::size_t grid_size_; // img_size / patch_size - std::size_t num_patches_; // grid_size² - std::size_t patch_dim_; // patch_size² - std::size_t d_model_; - Linear projection_; // (patch_dim, d_model) - Matrix input_cache_; - - public: - PatchEmbedding(std::size_t img_size, std::size_t patch_size, std::size_t d_model) - : img_size_(img_size), patch_size_(patch_size), - grid_size_(img_size / patch_size), - num_patches_((img_size / patch_size) * (img_size / patch_size)), - patch_dim_(patch_size * patch_size), - d_model_(d_model), - projection_(patch_dim_, d_model) - { - if (img_size % patch_size != 0) - assert(false && "PatchEmbedding: img_size must be divisible by patch_size"); // NOLINT - } - - [[nodiscard]] std::size_t num_patches() const noexcept { return num_patches_; } - [[nodiscard]] std::size_t d_model() const noexcept { return d_model_; } - - std::vector> parameters() override - { - return projection_.parameters(); - } - - std::vector> param_gradients() override - { - return projection_.param_gradients(); - } - - // ── 前向传播 ──────────────────────────────────────────────────── - // 批量提取 patch 并一次性投影 - Result forward(const Matrix &input) override - { - input_cache_ = input; - const std::size_t batch = input.cols(); - - // Step 1: 提取所有 patch → (patch_dim, num_patches × batch) - // 布局: patch p 的 sample b 位于列 p × batch + b - Matrix all_patches(patch_dim_, num_patches_ * batch); - - for (std::size_t b = 0; b < batch; ++b) - { - for (std::size_t p = 0; p < num_patches_; ++p) - { - const std::size_t gr = (p / grid_size_) * patch_size_; - const std::size_t gc = (p % grid_size_) * patch_size_; - const std::size_t col_idx = p * batch + b; - - for (std::size_t pr = 0; pr < patch_size_; ++pr) - { - for (std::size_t pc = 0; pc < patch_size_; ++pc) - { - const std::size_t flat = pr * patch_size_ + pc; - const std::size_t pix = (gr + pr) * img_size_ + (gc + pc); - all_patches.set_value_unchecked(flat, col_idx, - input.at_unchecked(pix, b)); - } - } - } - } - - // Step 2: 投影 → (d_model, num_patches × batch) - auto proj_res = projection_.forward(all_patches); - if (!proj_res) return proj_res; - Matrix projected = *proj_res; - - // Step 3: 重排为 (d_model × num_patches, batch) - Matrix output(d_model_ * num_patches_, batch); - for (std::size_t r = 0; r < d_model_; ++r) - for (std::size_t p = 0; p < num_patches_; ++p) - for (std::size_t b = 0; b < batch; ++b) - output.set_value_unchecked(r * num_patches_ + p, b, - projected.at_unchecked(r, p * batch + b)); - - return output; - } - - // ── 反向传播 ──────────────────────────────────────────────────── - Result backward(const Matrix &grad_output) override - { - const std::size_t batch = grad_output.cols(); - - // Step 1: 重排梯度 → (d_model, num_patches × batch) - Matrix grad_projected(d_model_, num_patches_ * batch); - for (std::size_t r = 0; r < d_model_; ++r) - for (std::size_t p = 0; p < num_patches_; ++p) - for (std::size_t b = 0; b < batch; ++b) - grad_projected.set_value_unchecked(r, p * batch + b, - grad_output.at_unchecked(r * num_patches_ + p, b)); - - // Step 2: 投影层反向 → (patch_dim, num_patches × batch) - auto bp_res = projection_.backward(grad_projected); - if (!bp_res) return bp_res; - Matrix grad_patches = *bp_res; - - // Step 3: 散射梯度回输入 → (img_size², batch) - Matrix grad_input(img_size_ * img_size_, batch); - grad_input.zero(); - - for (std::size_t b = 0; b < batch; ++b) - { - for (std::size_t p = 0; p < num_patches_; ++p) - { - const std::size_t gr = (p / grid_size_) * patch_size_; - const std::size_t gc = (p % grid_size_) * patch_size_; - const std::size_t col_idx = p * batch + b; - - for (std::size_t pr = 0; pr < patch_size_; ++pr) - { - for (std::size_t pc = 0; pc < patch_size_; ++pc) - { - const std::size_t flat = pr * patch_size_ + pc; - const std::size_t pix = (gr + pr) * img_size_ + (gc + pc); - const double val = grad_input.at_unchecked(pix, b) - + grad_patches.at_unchecked(flat, col_idx); - grad_input.set_value_unchecked(pix, b, val); - } - } - } - } - return grad_input; - } - }; - - // ── 因果自注意力层 (Causal Self-Attention) ────────────────────────── - // 用于 GPT 风格的自回归语言模型。 - // 与 MultiHeadAttention 的区别：在 softmax 前施加上三角因果掩码， - // 保证位置 i 只能关注位置 ≤ i 的 token。 - // - // 输入形状: (d_model, seq_len)，输出形状: (d_model, seq_len) - // ───────────────────────────────────────────────────────────────────── - class CausalSelfAttention final : public Layer - { - private: - std::size_t d_model_; - std::size_t num_heads_; - std::size_t d_k_; - double scale_; - - // 投影层 - Linear W_q_; - Linear W_k_; - Linear W_v_; - Linear W_o_; - - // 前向传播缓存 - Matrix Q_cache_; - Matrix K_cache_; - Matrix V_cache_; - std::vector attn_; // num_heads_ × (seq_len, seq_len) - std::vector Q_heads_; - std::vector K_heads_; - std::vector V_heads_; - std::vector O_heads_; - - // 因果掩码 (上三角为 -inf) - std::vector mask_data_; // max_len × max_len - - // 辅助缓冲区 - Matrix grad_scores_buf_; - Matrix grad_A_buf_; - Matrix grad_O_h_buf_; - Softmax softmax_; // 仅在 forward() 中调用 forward()，不调用 backward()（注意力反向手动计算） - - static void extract_rows(const Matrix &src, std::size_t row_start, - std::size_t row_count, Matrix &dst) - { - const std::size_t cols = src.cols(); - dst.resize(row_count, cols); - const auto src_span = src.span(); - auto dst_span = dst.span(); - for (std::size_t r = 0; r < row_count; ++r) - std::copy_n(src_span.begin() + (row_start + r) * cols, cols, - dst_span.begin() + r * cols); - } - - static void insert_rows(Matrix &dst, std::size_t row_start, - const Matrix &src) - { - const std::size_t row_count = src.rows(); - const std::size_t cols = src.cols(); - auto dst_span = dst.span(); - const auto src_span = src.span(); - for (std::size_t r = 0; r < row_count; ++r) - std::copy_n(src_span.begin() + r * cols, cols, - dst_span.begin() + (row_start + r) * cols); - } - - static void scale_inplace(Matrix &m, double s) - { - auto m_span = m.span(); - const auto n = m.size(); - if (n >= SmartPolicy::PARALLEL_THRESHOLD) - { - auto indices = std::views::iota(std::size_t{0}, n); - SmartPolicy::for_each(indices.begin(), indices.end(), - [m_span, s](std::size_t i) noexcept { m_span[i] *= s; }); - } - else - { - for (std::size_t i = 0; i < n; ++i) - m_span[i] *= s; - } - } - - public: - CausalSelfAttention(std::size_t d_model, std::size_t num_heads, - std::size_t max_len = 1024) - : d_model_(d_model), - num_heads_(num_heads), - d_k_(d_model / num_heads), - scale_(1.0 / std::sqrt(static_cast(d_model / num_heads))), - W_q_(d_model, d_model), - W_k_(d_model, d_model), - W_v_(d_model, d_model), - W_o_(d_model, d_model), - mask_data_(max_len * max_len, 0.0) - { - if (d_model % num_heads != 0) - assert(false && "CausalSelfAttention: d_model must be divisible by num_heads"); // NOLINT - - // 预计算因果掩码: mask[i][j] = 0 if j <= i else -inf - const double neg_inf = -1e30; - for (std::size_t i = 0; i < max_len; ++i) - for (std::size_t j = 0; j < max_len; ++j) - mask_data_[i * max_len + j] = (j <= i) ? 0.0 : neg_inf; - } - - std::vector> parameters() override - { - auto params = W_q_.parameters(); - auto wk = W_k_.parameters(); - auto wv = W_v_.parameters(); - auto wo = W_o_.parameters(); - params.insert(params.end(), wk.begin(), wk.end()); - params.insert(params.end(), wv.begin(), wv.end()); - params.insert(params.end(), wo.begin(), wo.end()); - return params; - } - - std::vector> param_gradients() override - { - auto grads = W_q_.param_gradients(); - auto gk = W_k_.param_gradients(); - auto gv = W_v_.param_gradients(); - auto go = W_o_.param_gradients(); - grads.insert(grads.end(), gk.begin(), gk.end()); - grads.insert(grads.end(), gv.begin(), gv.end()); - grads.insert(grads.end(), go.begin(), go.end()); - return grads; - } - - Result forward(const Matrix &input) override - { - if (input.rows() != d_model_) - return std::unexpected(Error{"CausalSelfAttention forward input shape mismatch"}); - - const std::size_t seq_len = input.cols(); - - // 1. 线性投影 - auto q_res = W_q_.forward(input); - if (!q_res) return q_res; - Q_cache_ = *q_res; - auto k_res = W_k_.forward(input); - if (!k_res) return k_res; - K_cache_ = *k_res; - auto v_res = W_v_.forward(input); - if (!v_res) return v_res; - V_cache_ = *v_res; - - // 2. 分配 per-head 缓冲区 - Q_heads_.resize(num_heads_); - K_heads_.resize(num_heads_); - V_heads_.resize(num_heads_); - O_heads_.resize(num_heads_); - attn_.resize(num_heads_); - - // 3. 逐头计算因果自注意力 - for (std::size_t h = 0; h < num_heads_; ++h) - { - const std::size_t row_start = h * d_k_; - - extract_rows(Q_cache_, row_start, d_k_, Q_heads_[h]); - extract_rows(K_cache_, row_start, d_k_, K_heads_[h]); - extract_rows(V_cache_, row_start, d_k_, V_heads_[h]); - - // S = Q_h^T @ K_h → (seq_len, seq_len) - Matrix QhT = Q_heads_[h].transpose(); - attn_[h].resize(seq_len, seq_len); - QhT.multiply_to(attn_[h], K_heads_[h]); - - // 施加因果掩码 - { - auto a_span = attn_[h].span(); - const auto m_span = std::span(mask_data_); - const std::size_t s2 = seq_len * seq_len; - for (std::size_t idx = 0; idx < s2; ++idx) - a_span[idx] += m_span[idx]; - } - - // 缩放 - scale_inplace(attn_[h], scale_); - - // softmax - auto sm_res = softmax_.forward(attn_[h]); - if (!sm_res) return sm_res; - attn_[h] = *sm_res; - - // O_h = V_h @ A_h - O_heads_[h].resize(d_k_, seq_len); - V_heads_[h].multiply_to(O_heads_[h], attn_[h]); - } - - // 4. 拼接 + 输出投影 - Matrix output(d_model_, seq_len); - for (std::size_t h = 0; h < num_heads_; ++h) - insert_rows(output, h * d_k_, O_heads_[h]); - - auto wo_res = W_o_.forward(output); - if (!wo_res) return wo_res; - return *wo_res; - } - - Result backward(const Matrix &grad_output) override - { - const std::size_t seq_len = grad_output.cols(); - - // 1. 输出投影反向 - auto gc_res = W_o_.backward(grad_output); - if (!gc_res) return gc_res; - Matrix grad_concat = *gc_res; - - // 2. 初始化全局梯度 - Matrix grad_Q_all(d_model_, seq_len); - Matrix grad_K_all(d_model_, seq_len); - Matrix grad_V_all(d_model_, seq_len); - grad_Q_all.zero(); - grad_K_all.zero(); - grad_V_all.zero(); - - // 3. 逐头反向 - for (std::size_t h = 0; h < num_heads_; ++h) - { - const std::size_t row_start = h * d_k_; - const Matrix &Q_h = Q_heads_[h]; - const Matrix &K_h = K_heads_[h]; - const Matrix &V_h = V_heads_[h]; - const Matrix &A_h = attn_[h]; - - extract_rows(grad_concat, row_start, d_k_, grad_O_h_buf_); - - // grad_V_h = grad_O_h @ A_h^T - Matrix AhT = A_h.transpose(); - Matrix grad_V_h(d_k_, seq_len); - grad_O_h_buf_.multiply_to(grad_V_h, AhT); - - // grad_A_h = V_h^T @ grad_O_h - Matrix VhT = V_h.transpose(); - grad_A_buf_.resize(seq_len, seq_len); - VhT.multiply_to(grad_A_buf_, grad_O_h_buf_); - - // grad_S_h = A_h ⊙ (grad_A_h - row_sum(A_h ⊙ grad_A_h)) - grad_scores_buf_.resize(seq_len, seq_len); - { - const auto a_span = A_h.span(); - const auto ga_span = grad_A_buf_.span(); - auto gs_span = grad_scores_buf_.span(); - - for (std::size_t i = 0; i < seq_len; ++i) - { - double dot = 0.0; - for (std::size_t j = 0; j < seq_len; ++j) - dot += a_span[i * seq_len + j] * ga_span[i * seq_len + j]; - for (std::size_t j = 0; j < seq_len; ++j) - gs_span[i * seq_len + j] = - a_span[i * seq_len + j] * (ga_span[i * seq_len + j] - dot); - } - } - - // grad_Q_h = K_h @ grad_S_h^T * scale - Matrix gsT = grad_scores_buf_.transpose(); - Matrix grad_Q_h(d_k_, seq_len); - K_h.multiply_to(grad_Q_h, gsT); - scale_inplace(grad_Q_h, scale_); - - // grad_K_h = Q_h @ grad_S_h * scale - Matrix grad_K_h(d_k_, seq_len); - Q_h.multiply_to(grad_K_h, grad_scores_buf_); - scale_inplace(grad_K_h, scale_); - - insert_rows(grad_Q_all, row_start, grad_Q_h); - insert_rows(grad_K_all, row_start, grad_K_h); - insert_rows(grad_V_all, row_start, grad_V_h); - } - - // 4. 投影层反向 - auto giq_res = W_q_.backward(grad_Q_all); - if (!giq_res) return giq_res; - Matrix grad_input = *giq_res; - auto gik_res = W_k_.backward(grad_K_all); - if (!gik_res) return gik_res; - grad_input = grad_input + *gik_res; - auto giv_res = W_v_.backward(grad_V_all); - if (!giv_res) return giv_res; - grad_input = grad_input + *giv_res; - - return grad_input; - } - }; - - // ── GPT Transformer 块 (Pre-Norm Decoder Block) ────────────────── - // x = x + CausalSelfAttn(LayerNorm₁(x)) - // x = x + FFN(LayerNorm₂(x)) - // 输入/输出形状: (d_model, seq_len) - class GPTBlock final : public Layer - { - private: - CausalSelfAttention self_attn_; - LayerNorm norm1_; - FeedForward ff_; - LayerNorm norm2_; - - Matrix residual1_cache_; - Matrix residual2_cache_; - - public: - GPTBlock(std::size_t d_model, std::size_t num_heads, std::size_t d_ff, - std::size_t max_len = 1024) - : self_attn_(d_model, num_heads, max_len), - norm1_(d_model), - ff_(d_model, d_ff), - norm2_(d_model) - {} - - std::vector> parameters() override - { - auto params = self_attn_.parameters(); - auto n1 = norm1_.parameters(); - auto f = ff_.parameters(); - auto n2 = norm2_.parameters(); - params.insert(params.end(), n1.begin(), n1.end()); - params.insert(params.end(), f.begin(), f.end()); - params.insert(params.end(), n2.begin(), n2.end()); - return params; - } - - std::vector> param_gradients() override - { - auto grads = self_attn_.param_gradients(); - auto gn1 = norm1_.param_gradients(); - auto gf = ff_.param_gradients(); - auto gn2 = norm2_.param_gradients(); - grads.insert(grads.end(), gn1.begin(), gn1.end()); - grads.insert(grads.end(), gf.begin(), gf.end()); - grads.insert(grads.end(), gn2.begin(), gn2.end()); - return grads; - } - - Result forward(const Matrix &input) override - { - // 子层1: CausalSelfAttention + 残差 - residual1_cache_ = input; - auto n1_res = norm1_.forward(input); - if (!n1_res) return n1_res; - auto sa_res = self_attn_.forward(*n1_res); - if (!sa_res) return sa_res; - residual2_cache_ = input + *sa_res; - - // 子层2: FFN + 残差 - auto n2_res = norm2_.forward(residual2_cache_); - if (!n2_res) return n2_res; - auto ff_res = ff_.forward(*n2_res); - if (!ff_res) return ff_res; - return residual2_cache_ + *ff_res; - } - - Result backward(const Matrix &grad_output) override - { - Matrix grad_residual1 = grad_output; - Matrix grad_ff_out = grad_output; - - auto b_ff = ff_.backward(grad_ff_out); - if (!b_ff) return b_ff; - auto b_n2 = norm2_.backward(*b_ff); - if (!b_n2) return b_n2; - grad_residual1 = grad_residual1 + *b_n2; - - Matrix grad_input = grad_residual1; - Matrix grad_attn_out = grad_residual1; - - auto b_sa = self_attn_.backward(grad_attn_out); - if (!b_sa) return b_sa; - auto b_n1 = norm1_.backward(*b_sa); - if (!b_n1) return b_n1; - grad_input = grad_input + *b_n1; - - return grad_input; - } - }; - - // ── GPT 语言模型 ───────────────────────────────────────────────── - // Decoder-only Transformer 用于自回归文本生成。 - // - // 组件: TokenEmbedding + PositionalEncoding + N × GPTBlock + LayerNorm + LM Head - // - // 输入: (seq_len, batch_size) — token ID 矩阵（每列为一个序列） - // 输出: (vocab_size, seq_len × batch_size) — 每个位置的 logits - // - // 传播流程: - // 输入: (seq_len, batch) token IDs - // → Embed: (d_model, seq_len × batch) - // → + PE: (d_model, seq_len × batch) - // → N × GPTBlock: (d_model, seq_len × batch) - // → LayerNorm: (d_model, seq_len × batch) - // → LM Head: (vocab_size, seq_len × batch) - // ──────────────────────────────────────────────────────────────────── - class GPTModel final : public Layer - { - private: - std::size_t vocab_size_; - std::size_t d_model_; - std::size_t seq_len_; - - // 嵌入层 - Matrix token_emb_; // (vocab_size, d_model) — 查找表 - Matrix grad_token_emb_; - - // 位置编码（可学习） - Matrix pos_emb_; // (max_seq_len, d_model) - Matrix grad_pos_emb_; - - // Transformer 块 - std::vector blocks_; - - // 最终 LayerNorm - LayerNorm ln_f_; - - // LM Head（权重与 token_emb_ 共享或独立） - Linear lm_head_; // (vocab_size, d_model) - - // 反向传播缓存 - std::vector stored_inputs_; // 每个样本经 PE 后的输入 - std::vector> stored_tokens_; // token IDs - std::size_t batch_size_{0}; - - public: - GPTModel(std::size_t vocab_size, std::size_t d_model, std::size_t seq_len, - std::size_t num_heads, std::size_t d_ff, std::size_t num_layers) - : vocab_size_(vocab_size), d_model_(d_model), seq_len_(seq_len), - token_emb_(vocab_size, d_model), - grad_token_emb_(vocab_size, d_model), - pos_emb_(seq_len, d_model), - grad_pos_emb_(seq_len, d_model), - ln_f_(d_model), - lm_head_(d_model, vocab_size) - { - // Xavier-like 初始化嵌入 - { - constexpr double emb_init_std = 0.02; - std::mt19937_64 rng{42}; - std::normal_distribution dist(0.0, emb_init_std); - for (auto &v : token_emb_.data()) - v = dist(rng); - for (auto &v : pos_emb_.data()) - v = dist(rng); - } - - for (std::size_t i = 0; i < num_layers; ++i) - blocks_.emplace_back(d_model, num_heads, d_ff, seq_len); - } - - // ── 可学习嵌入参数 ───────────────────────────────────────── - std::vector> parameters() override - { - std::vector> params; - params.push_back(std::ref(token_emb_)); - params.push_back(std::ref(pos_emb_)); - for (auto &b : blocks_) - { - auto bp = b.parameters(); - params.insert(params.end(), bp.begin(), bp.end()); - } - auto lp = ln_f_.parameters(); - params.insert(params.end(), lp.begin(), lp.end()); - auto hp = lm_head_.parameters(); - params.insert(params.end(), hp.begin(), hp.end()); - return params; - } - - std::vector> param_gradients() override - { - std::vector> grads; - grads.push_back(std::ref(grad_token_emb_)); - grads.push_back(std::ref(grad_pos_emb_)); - for (auto &b : blocks_) - { - auto bg = b.param_gradients(); - grads.insert(grads.end(), bg.begin(), bg.end()); - } - auto lg = ln_f_.param_gradients(); - grads.insert(grads.end(), lg.begin(), lg.end()); - auto hg = lm_head_.param_gradients(); - grads.insert(grads.end(), hg.begin(), hg.end()); - return grads; - } - - // ── 前向传播 ──────────────────────────────────────────────── - // input: (seq_len, batch_size) — token IDs 作为 double - Result forward(const Matrix &input) override - { - const std::size_t seq_len = input.rows(); - batch_size_ = input.cols(); - stored_inputs_.resize(batch_size_); - stored_tokens_.resize(batch_size_); - - // 清空缓存，防止跨 forward 累积 - for (auto &v : stored_tokens_) - v.clear(); - - // 输出: (vocab_size, seq_len × batch_size) - Matrix output(vocab_size_, seq_len * batch_size_); - - for (std::size_t b = 0; b < batch_size_; ++b) - { - stored_tokens_[b].reserve(seq_len); - - // 1. Token 嵌入 + 位置嵌入 → (d_model, seq_len) - Matrix x(d_model_, seq_len); - for (std::size_t t = 0; t < seq_len; ++t) - { - auto token_id = static_cast(input.at_unchecked(t, b)); - if (token_id >= vocab_size_) - token_id = 0; // fallback to PAD - stored_tokens_[b].push_back(token_id); - - const auto emb_span = token_emb_.span(); - const auto pos_span = pos_emb_.span(); - auto out_span = x.span(); - for (std::size_t d = 0; d < d_model_; ++d) - out_span[d * seq_len + t] = - emb_span[token_id * d_model_ + d] + - pos_span[t * d_model_ + d]; - } - - stored_inputs_[b] = x; - - // 2. 通过 Transformer 块 - for (std::size_t l = 0; l < blocks_.size(); ++l) - { - auto blk_res = blocks_[l].forward(x); - if (!blk_res) return blk_res; - x = *blk_res; - } - - // 3. 最终 LayerNorm - auto ln_res = ln_f_.forward(x); - if (!ln_res) return ln_res; - x = *ln_res; - - // 4. LM Head: (d_model, seq_len) → (vocab_size, seq_len) - auto lm_res = lm_head_.forward(x); - if (!lm_res) return lm_res; - Matrix logits = *lm_res; - - // 5. 写入输出 (vocab_size, seq_len × batch) - // 布局: 列 t * batch_size + b 对应样本 b 的位置 t - auto log_span = logits.span(); - auto out_span2 = output.span(); - for (std::size_t r = 0; r < vocab_size_; ++r) - for (std::size_t t = 0; t < seq_len; ++t) - out_span2[r * (seq_len * batch_size_) + t * batch_size_ + b] = - log_span[r * seq_len + t]; - } - - return output; - } - - // ── 反向传播 ──────────────────────────────────────────────── - // grad_output: (vocab_size, seq_len × batch_size) - Result backward(const Matrix &grad_output) override - { - const std::size_t seq_len = seq_len_; - grad_token_emb_.zero(); - grad_pos_emb_.zero(); - - Matrix grad_input(seq_len, batch_size_); - grad_input.zero(); - - for (std::size_t b = 0; b < batch_size_; ++b) - { - // 提取该样本的梯度: (vocab_size, seq_len) - Matrix grad_logits(vocab_size_, seq_len); - auto go_span = grad_output.span(); - auto gl_span = grad_logits.span(); - for (std::size_t r = 0; r < vocab_size_; ++r) - for (std::size_t t = 0; t < seq_len; ++t) - gl_span[r * seq_len + t] = - go_span[r * (seq_len * batch_size_) + t * batch_size_ + b]; - - // Re-forward 重建缓存 - Matrix x = stored_inputs_[b]; - for (std::size_t l = 0; l < blocks_.size(); ++l) - { - auto blk_res = blocks_[l].forward(x); - if (!blk_res) return blk_res; - x = *blk_res; - } - auto ln_fwd = ln_f_.forward(x); - if (!ln_fwd) return ln_fwd; - x = *ln_fwd; - - // LM Head 反向 → (d_model, seq_len) - auto b_lm = lm_head_.backward(grad_logits); - if (!b_lm) return b_lm; - Matrix grad_ln = *b_lm; - - // LayerNorm 反向 - auto b_ln = ln_f_.backward(grad_ln); - if (!b_ln) return b_ln; - grad_ln = *b_ln; - - // 逐块反向 - for (int l = static_cast(blocks_.size()) - 1; l >= 0; --l) - { - auto b_blk = blocks_[l].backward(grad_ln); - if (!b_blk) return b_blk; - grad_ln = *b_blk; - } - - // 累加嵌入梯度 - const auto tokens = stored_tokens_[b]; - for (std::size_t t = 0; t < seq_len; ++t) - { - const std::size_t tid = tokens[t]; - // token embedding 梯度 - auto gte = grad_token_emb_.span(); - for (std::size_t d = 0; d < d_model_; ++d) - gte[tid * d_model_ + d] += - grad_ln.at_unchecked(d, t); - // positional embedding 梯度 - auto gpe = grad_pos_emb_.span(); - for (std::size_t d = 0; d < d_model_; ++d) - gpe[t * d_model_ + d] += - grad_ln.at_unchecked(d, t); - // grad_input (token IDs 无梯度，此处仅用于接口一致性) - grad_input.set_value_unchecked(t, b, 0.0); - } - } - - return grad_input; - } - - // ── 采样生成（支持温度采样 + 贪心） ──────────────────────── - std::vector generate(const std::vector &prompt, - std::size_t max_new_tokens, - double temperature = 1.0) - { - std::vector context(prompt); - std::vector generated; - std::mt19937_64 rng{std::random_device{}()}; - std::uniform_real_distribution dist(0.0, 1.0); - - for (std::size_t step = 0; step < max_new_tokens; ++step) - { - // 截取最后 seq_len 个 token - std::size_t start = 0; - if (context.size() > seq_len_) - start = context.size() - seq_len_; - - std::size_t cur_len = context.size() - start; - Matrix input(cur_len, 1); - for (std::size_t t = 0; t < cur_len; ++t) - input.set_value_unchecked(t, 0, - static_cast(context[start + t])); - - auto logits_res = forward(input); // (vocab_size, cur_len) - if (!logits_res) break; // 生成中出错则提前终止 - auto logits = *logits_res; - - // 取最后一个位置的 logits - std::vector last_logits(vocab_size_); - for (std::size_t v = 0; v < vocab_size_; ++v) - last_logits[v] = logits.at_unchecked(v, cur_len - 1); - - // temperature - if (temperature > 0.0 && temperature != 1.0) - { - for (auto &v : last_logits) - v /= temperature; - } - - // softmax（数值稳定） - double max_val = last_logits[0]; - for (std::size_t v = 1; v < vocab_size_; ++v) - max_val = std::max(max_val, last_logits[v]); - double sum_exp = 0.0; - for (auto &v : last_logits) - { - v = std::exp(v - max_val); - sum_exp += v; - } - for (auto &v : last_logits) - v /= sum_exp; - - // 采样 - std::size_t next_token; - if (temperature > 0.0 && temperature != 1.0) - { - // 概率采样 - double r = dist(rng); - double cumulative = 0.0; - next_token = vocab_size_ - 1; - for (std::size_t v = 0; v < vocab_size_; ++v) - { - cumulative += last_logits[v]; - if (r <= cumulative) - { - next_token = v; - break; - } - } - } - else - { - // 贪心 - next_token = 0; - double best = last_logits[0]; - for (std::size_t v = 1; v < vocab_size_; ++v) - { - if (last_logits[v] > best) - { - best = last_logits[v]; - next_token = v; - } - } - } - - context.push_back(next_token); - generated.push_back(next_token); - } - - return generated; - } - }; -} - -#endif \ No newline at end of file diff --git a/include/neuralnet.cpp/loss.hpp b/include/neuralnet.cpp/loss.hpp deleted file mode 100644 index ebb53e25..00000000 --- a/include/neuralnet.cpp/loss.hpp +++ /dev/null @@ -1,153 +0,0 @@ -#ifndef LOSS_HPP -#define LOSS_HPP - -#include -#include -#include -#include -#include -#include -#include - -#include "nn_config.hpp" -#include "matrix.hpp" - -namespace nn -{ - class Loss - { - public: - virtual ~Loss() = default; - virtual std::expected forward(const Matrix &pred, const Matrix &target) = 0; - virtual const Matrix &backward() const = 0; - }; - - class MSELoss : public Loss - { - private: - Matrix grad_input_; - - public: - MSELoss() = default; - - [[nodiscard]] std::expected forward(const Matrix &pred, const Matrix &target) - { - if (pred.rows() != target.rows() || pred.cols() != target.cols()) - { - return std::unexpected(Error{"mse loss shape mismatch"}); - } - if (pred.empty()) - { - return std::unexpected(Error{"mse loss cannot be computed on an empty matrix"}); - } - - grad_input_.resize(pred.rows(), pred.cols()); - const auto total = static_cast(pred.size()); - - const double sum_sq = SmartPolicy::transform_reduce( - pred.data().begin(), pred.data().end(), - target.data().begin(), - 0.0, - std::plus<>{}, - [](double prediction, double actual) noexcept - { - const double diff = prediction - actual; - return diff * diff; - }); - - const double loss = sum_sq / total; - const double factor = 2.0 / total; - - SmartPolicy::transform(pred.data().begin(), pred.data().end(), - target.data().begin(), - grad_input_.data().begin(), - [factor](double prediction, double actual) noexcept - { - return factor * (prediction - actual); - }); - - return loss; - } - - [[nodiscard]] const Matrix &backward() const noexcept { return grad_input_; } - }; - - class CrossEntropyLoss : public Loss - { - private: - Matrix grad_input_; - - public: - CrossEntropyLoss() = default; - - [[nodiscard]] std::expected forward(const Matrix &logits, const Matrix &target_onehot) - { - const std::size_t classes = logits.rows(); - const std::size_t batch = logits.cols(); - grad_input_ = Matrix(classes, batch); - - double total_loss = 0.0; - - for (std::size_t i = 0; i < batch; ++i) - { - // 数值稳定的 softmax - double max_val = logits.at_unchecked(0, i); - for (std::size_t c = 1; c < classes; ++c) - { - const double val = logits.at_unchecked(c, i); - if (val > max_val) max_val = val; - } - - // 栈上分配 exp_vals，支持最多 128 类 - // 超过 128 类时改用 vector（非热路径，开销可接受） - std::array exp_vals_fixed{}; - std::vector exp_vals_heap; - std::span exp_vals; - if (classes <= 128) { - exp_vals = exp_vals_fixed; - } else { - exp_vals_heap.resize(classes); - exp_vals = exp_vals_heap; - } - - double sum_exp = 0.0; - for (std::size_t c = 0; c < classes; ++c) - { - const double e = std::exp(logits.at_unchecked(c, i) - max_val); - exp_vals[c] = e; - sum_exp += e; - } - - // 找到真实类别 - std::size_t true_class = 0; - for (std::size_t c = 0; c < classes; ++c) - { - if (target_onehot.at_unchecked(c, i) > 0.5) - { - true_class = c; - break; - } - } - - // loss = -log(softmax[true_class]) - const double prob_true = exp_vals[true_class] / sum_exp; - total_loss += -std::log(prob_true); - - // 梯度：softmax - target - for (std::size_t c = 0; c < classes; ++c) - { - const double softmax_c = exp_vals[c] / sum_exp; - grad_input_.set_value_unchecked(c, i, - softmax_c - target_onehot.at_unchecked(c, i)); - } - } - - return total_loss / static_cast(batch); - } - - [[nodiscard]] const Matrix &backward() const noexcept { return grad_input_; } - }; - -} // namespace nn - -#endif // LOSS_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/matrix.hpp b/include/neuralnet.cpp/matrix.hpp deleted file mode 100644 index 8a009a4a..00000000 --- a/include/neuralnet.cpp/matrix.hpp +++ /dev/null @@ -1,372 +0,0 @@ -#ifndef MATRIX_HPP -#define MATRIX_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nn_config.hpp" - -namespace nn -{ - class Matrix - { - private: - std::vector data_{}; - std::size_t rows_{0}; - std::size_t cols_{0}; - - [[nodiscard]] constexpr std::size_t index(std::size_t row, std::size_t col) const noexcept - { - return row * cols_ + col; - } - - static void require_same_shape(const Matrix &lhs, const Matrix &rhs, const char *message) - { - if (lhs.rows_ != rhs.rows_ || lhs.cols_ != rhs.cols_) - { - assert(false && message); // NOLINT - } - } - - public: - Matrix() = default; - - explicit Matrix(std::size_t rows, std::size_t cols) - : data_(rows * cols), rows_(rows), cols_(cols) {} - - Matrix(std::vector data, std::size_t rows, std::size_t cols) - : data_(std::move(data)), rows_(rows), cols_(cols) - { - if (data_.size() != rows_ * cols_) - { - assert(false && "data size mismatch"); // NOLINT - } - } - - // 从标量值初始化矩阵 - Matrix(std::size_t rows, std::size_t cols, double value) - : data_(rows * cols, value), rows_(rows), cols_(cols) {} - Matrix(const Matrix &) = default; - Matrix(Matrix &&) noexcept = default; - Matrix &operator=(const Matrix &) = default; - Matrix &operator=(Matrix &&) noexcept = default; - ~Matrix() = default; - - // ── 就地调整大小（复用已有内存） ────────────────────────────────── - void resize(std::size_t rows, std::size_t cols) - { - if (rows_ == rows && cols_ == cols) return; // 尺寸不变，零开销 - rows_ = rows; - cols_ = cols; - data_.resize(rows * cols); - } - - // ── std::span 访问（C++20 现代接口，推荐使用） ──────────────────── - // 零开销抽象：编译后等价于裸指针 + 大小，可替代所有 data_ptr() 场景 - [[nodiscard]] std::span span() const noexcept { return {data_.data(), data_.size()}; } - [[nodiscard]] std::span span() noexcept { return {data_.data(), data_.size()}; } - - // 访问器 - [[nodiscard]] constexpr std::size_t rows() const noexcept { return rows_; } - [[nodiscard]] constexpr std::size_t cols() const noexcept { return cols_; } - [[nodiscard]] constexpr std::size_t size() const noexcept { return data_.size(); } - [[nodiscard]] constexpr bool empty() const noexcept { return data_.empty(); } - [[nodiscard]] double at(std::size_t row, std::size_t col) const - { - if (row >= rows_ || col >= cols_) - { - assert(false && "Matrix index out of range"); - } - return data_[index(row, col)]; - } - void set_value(std::size_t row, std::size_t col, double value) - { - if (row >= rows_ || col >= cols_) - { - assert(false && "Matrix index out of range"); - } - data_[index(row, col)] = value; - } - [[nodiscard]] constexpr double at_unchecked(std::size_t row, std::size_t col) const noexcept { return data_[index(row, col)]; } // 无校验 - constexpr void set_value_unchecked(std::size_t row, std::size_t col, double value) noexcept { data_[index(row, col)] = value; } // 无校验 - [[nodiscard]] const std::vector &data() const noexcept { return data_; } - [[nodiscard]] std::vector &data() noexcept { return data_; } - [[nodiscard]] std::vector> get_data() const - { - std::vector> result(rows_, std::vector(cols_, 0.0)); - for (std::size_t row = 0; row < rows_; ++row) - { - for (std::size_t col = 0; col < cols_; ++col) - { - result[row][col] = data_[index(row, col)]; - } - } - return result; - } - - void set_data(const std::vector> &new_data) - { - if (new_data.empty()) - { - rows_ = 0; - cols_ = 0; - data_.clear(); - return; - } - - const std::size_t new_rows = new_data.size(); - const std::size_t new_cols = new_data.front().size(); - for (const auto &row : new_data) - { - if (row.size() != new_cols) - { - assert(false && "all rows must have the same number of columns"); // NOLINT - } - } - - rows_ = new_rows; - cols_ = new_cols; - data_.resize(rows_ * cols_); - for (std::size_t row = 0; row < rows_; ++row) - { - for (std::size_t col = 0; col < cols_; ++col) - { - data_[index(row, col)] = new_data[row][col]; - } - } - } - - // ── 转置（返回新矩阵） ───────────────────────────────────────────── - [[nodiscard]] Matrix transpose() const - { - Matrix result(cols_, rows_); - transpose_to(result); - return result; - } - - // ── 转置到预分配缓冲区（零分配热路径） ───────────────────────────── - void transpose_to(Matrix &result) const - { - assert(&result != this && "transpose_to: self-referencing not supported"); - result.resize(cols_, rows_); - if (rows_ == 0 || cols_ == 0) return; - - const auto src = span(); - auto dst = result.span(); - const std::size_t R = rows_; - const std::size_t C = cols_; - - const std::size_t i_blocks = (R + BLOCK_SIZE - 1) / BLOCK_SIZE; - const std::size_t j_blocks = (C + BLOCK_SIZE - 1) / BLOCK_SIZE; - - // 绕过 SmartPolicy：分块数很少（1~几十），但其内循环计算量大。 - // SmartPolicy 按元素数判断会错误地将分块级别的并行退化为串行。 - // 这里直接调用线程池（>1 块时并行），零分配 latch 设计开销极低。 - const auto n_blocks = i_blocks * j_blocks; - if (n_blocks <= 1) - { - for (std::size_t i = 0; i < R; ++i) - for (std::size_t j = 0; j < C; ++j) - dst[j * R + i] = src[i * C + j]; - } - else - { - auto block_indices = std::views::iota(std::size_t{0}, n_blocks); - global_thread_pool().parallel_for_blocks( - block_indices.begin(), block_indices.end(), - [src, dst, R, C, j_blocks](std::size_t block_idx) noexcept - { - const std::size_t ib = block_idx / j_blocks; - const std::size_t jb = block_idx % j_blocks; - const std::size_t i0 = ib * BLOCK_SIZE; - const std::size_t j0 = jb * BLOCK_SIZE; - const std::size_t i1 = std::min(i0 + BLOCK_SIZE, R); - const std::size_t j1 = std::min(j0 + BLOCK_SIZE, C); - for (std::size_t i = i0; i < i1; ++i) - for (std::size_t j = j0; j < j1; ++j) - dst[j * R + i] = src[i * C + j]; - }); - } - } - - [[nodiscard]] Matrix operator+(const Matrix &other) const - { - require_same_shape(*this, other, "addition dimension mismatch"); - Matrix result(rows_, cols_); - SmartPolicy::transform(data_.begin(), data_.end(), other.data_.begin(), - result.data_.begin(), std::plus<>{}); - return result; - } - - [[nodiscard]] Matrix operator-(const Matrix &other) const - { - require_same_shape(*this, other, "subtraction dimension mismatch"); - Matrix result(rows_, cols_); - SmartPolicy::transform(data_.begin(), data_.end(), other.data_.begin(), - result.data_.begin(), std::minus<>{}); - return result; - } - - [[nodiscard]] Matrix operator*(double scalar) const - { - Matrix result(rows_, cols_); - SmartPolicy::transform(data_.begin(), data_.end(), result.data().begin(), - [scalar](double value) noexcept { return value * scalar; }); - return result; - } - - friend Matrix operator*(double scalar, const Matrix &mat) noexcept - { - return mat * scalar; - } - - // ── 矩阵乘法（返回新矩阵） ───────────────────────────────────────── - [[nodiscard]] Matrix operator*(const Matrix &other) const - { - if (cols_ != other.rows_) - assert(false && "matrix multiplication dimension mismatch"); // NOLINT - Matrix result(rows_, other.cols_); - multiply_to(result, other); - return result; - } - - // ── 矩阵乘法到预分配缓冲区（零分配热路径） ───────────────────────── - // 使用 std::span（C++20）提供类型安全的非拥有视图 - void multiply_to(Matrix &result, const Matrix &other) const - { - assert(&result != this && "multiply_to: self-referencing not supported"); - const std::size_t M = rows_; - const std::size_t N = other.cols_; - const std::size_t K = cols_; - result.resize(M, N); - if (M == 0 || N == 0 || K == 0) return; - - // 清零结果矩阵（使用 RAII 封装的方法） - result.zero(); - - // std::span 非拥有视图：从 std::vector 借出，生命周期由调用栈保证 - const auto a = span(); - const auto b = other.span(); - auto r = result.span(); - - const std::size_t i_blocks = (M + BLOCK_SIZE - 1) / BLOCK_SIZE; - const std::size_t j_blocks = (N + BLOCK_SIZE - 1) / BLOCK_SIZE; - - // 绕过 SmartPolicy：分块数可能很少但其内循环计算量巨大（三层循环 O(n³)）。 - // 直接调用线程池，零分配 latch 开销极低。 - const auto n_blocks = i_blocks * j_blocks; - if (n_blocks <= 1) - { - // 单块串行——与下方 lambda 逻辑相同，但避免不必要的线程池调用 - const std::size_t i_start = 0, i_end = M; - const std::size_t j_start = 0, j_end = N; - for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) - { - const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); - const std::size_t k_len = k_end - k_start; - const std::size_t j_len = j_end - j_start; - std::array b_block{}; - for (std::size_t jj = 0; jj < j_len; ++jj) - for (std::size_t kk = 0; kk < k_len; ++kk) - b_block[jj * k_len + kk] = b[(k_start + kk) * N + (j_start + jj)]; - const auto b_block_span = std::span(b_block.data(), k_len * j_len); - for (std::size_t i = i_start; i < i_end; ++i) - { - const auto a_row = a.subspan(i * K + k_start); - auto r_row = r.subspan(i * N + j_start); - for (std::size_t j = 0; j < j_len; ++j) - { - const auto b_col = b_block_span.subspan(j * k_len, k_len); - double sum = 0.0; - for (std::size_t kk = 0; kk < k_len; ++kk) - sum += a_row[kk] * b_col[kk]; - r_row[j] += sum; - } - } - } - } - else - { - auto block_indices = std::views::iota(std::size_t{0}, n_blocks); - global_thread_pool().parallel_for_blocks( - block_indices.begin(), block_indices.end(), - [a, b, r, M, N, K, j_blocks](std::size_t block_idx) noexcept - { - const std::size_t i_block = block_idx / j_blocks; - const std::size_t j_block = block_idx % j_blocks; - const std::size_t i_start = i_block * BLOCK_SIZE; - const std::size_t i_end = std::min(i_start + BLOCK_SIZE, M); - const std::size_t j_start = j_block * BLOCK_SIZE; - const std::size_t j_end = std::min(j_start + BLOCK_SIZE, N); - - for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) - { - const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); - const std::size_t k_len = k_end - k_start; - const std::size_t j_len = j_end - j_start; - - std::array b_block{}; - for (std::size_t jj = 0; jj < j_len; ++jj) - for (std::size_t kk = 0; kk < k_len; ++kk) - b_block[jj * k_len + kk] = b[(k_start + kk) * N + (j_start + jj)]; - const auto b_block_span = std::span(b_block.data(), k_len * j_len); - - for (std::size_t i = i_start; i < i_end; ++i) - { - const auto a_row = a.subspan(i * K + k_start); - auto r_row = r.subspan(i * N + j_start); - for (std::size_t j = 0; j < j_len; ++j) - { - const auto b_col = b_block_span.subspan(j * k_len, k_len); - double sum = 0.0; - for (std::size_t kk = 0; kk < k_len; ++kk) - sum += a_row[kk] * b_col[kk]; - r_row[j] += sum; - } - } - } - }); - } - } - - void scale_inplace(double scalar) noexcept - { - SmartPolicy::for_each(data_.begin(), data_.end(), - [scalar](double &value) noexcept { value *= scalar; }); - } - - // 逐元素加法 inplace - void add_inplace(const Matrix &other) - { - require_same_shape(*this, other, "add_inplace dimension mismatch"); - SmartPolicy::transform(data_.begin(), data_.end(), other.data_.begin(), - data_.begin(), std::plus<>{}); - } - - // 逐元素减法 inplace - void subtract_inplace(const Matrix &other) - { - require_same_shape(*this, other, "subtract_inplace dimension mismatch"); - SmartPolicy::transform(data_.begin(), data_.end(), other.data_.begin(), - data_.begin(), std::minus<>{}); - } - - // 填充零 - void zero() noexcept - { - std::fill(data_.begin(), data_.end(), 0.0); - } - }; -} - -#endif \ No newline at end of file diff --git a/include/neuralnet.cpp/mnist_common.hpp b/include/neuralnet.cpp/mnist_common.hpp deleted file mode 100644 index a6919987..00000000 --- a/include/neuralnet.cpp/mnist_common.hpp +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef MNIST_COMMON_HPP -#define MNIST_COMMON_HPP - -#include -#include -#include -#include - -#include "model.hpp" -#include "model_io.hpp" // ModelSpec - -namespace nn { - -// ── MNIST 常量 ────────────────────────────────────────────────────────────── -inline constexpr std::size_t MNIST_INPUT_DIM = 784; -inline constexpr std::size_t MNIST_NUM_CLASSES = 10; - -// 默认 MLP 网络架构：输入层 -> 隐藏层1 -> 隐藏层2 -> 隐藏层3 -> 输出层 -inline const std::vector MNIST_LAYER_DIMS = { - MNIST_INPUT_DIM, 512, 256, 128, 64, MNIST_NUM_CLASSES -}; - -// 默认 Transformer 超参数 -inline constexpr std::size_t TRANSFORMER_D_MODEL = 64; -inline constexpr std::size_t TRANSFORMER_NUM_HEADS = 4; -inline constexpr std::size_t TRANSFORMER_D_FF = 128; -inline constexpr std::size_t TRANSFORMER_NUM_LAYERS = 2; -inline constexpr std::size_t TRANSFORMER_PATCH_SIZE = 7; // 28 / 7 = 4 → 4×4 = 16 patches - -// ── 构建 MLP 模型 ────────────────────────────────────────────────────────── -// 可指定自定义层维度，也可使用默认 MNIST_LAYER_DIMS -[[nodiscard]] inline Result build_mnist_mlp_model( - const std::vector &layer_dims = MNIST_LAYER_DIMS) -{ - if (layer_dims.size() < 2) - return std::unexpected(Error{"MLP layer_dims must have at least 2 elements"}); - - Model model; - for (std::size_t i = 0; i < layer_dims.size() - 1; ++i) - { - std::size_t in_dim = layer_dims[i]; - std::size_t out_dim = layer_dims[i + 1]; - - model.add(in_dim, out_dim); - - if (i < layer_dims.size() - 2) - { - model.add(out_dim) - .add(); - } - } - return model; -} - -// ── 构建 Transformer 模型 (ViT-like) ────────────────────────────────────── -[[nodiscard]] inline Result build_mnist_transformer_model( - std::size_t d_model = TRANSFORMER_D_MODEL, - std::size_t num_heads = TRANSFORMER_NUM_HEADS, - std::size_t d_ff = TRANSFORMER_D_FF, - std::size_t num_layers = TRANSFORMER_NUM_LAYERS, - std::size_t patch_size = TRANSFORMER_PATCH_SIZE) -{ - constexpr std::size_t img_size = 28; - const std::size_t num_patches = (img_size / patch_size) * (img_size / patch_size); - - Model model; - model.add(img_size, patch_size, d_model) - .add(d_model, num_heads, d_ff, num_layers, num_patches) - .add(d_model, MNIST_NUM_CLASSES); - return model; -} - -// ── 统一构建入口（字符串参数） ───────────────────────────────────────────── -[[nodiscard]] inline Result build_mnist_model(const std::string &model_type = "mlp") -{ - if (model_type == "mlp") - return build_mnist_mlp_model(); - else if (model_type == "transformer") - return build_mnist_transformer_model(); - else - return std::unexpected(Error{"Unknown model type: " + model_type - + " (available: mlp, transformer)"}); -} - -// ── 从 ModelSpec 构建模型 ───────────────────────────────────────────────── -// 用于从二进制文件加载时自动还原架构 -[[nodiscard]] inline Result build_mnist_model_from_spec(const ModelSpec &spec) -{ - if (spec.is_mlp()) - { - return build_mnist_mlp_model(spec.layer_dims); - } - else if (spec.is_transformer()) - { - return build_mnist_transformer_model( - spec.d_model, spec.num_heads, spec.d_ff, - spec.num_layers, spec.patch_size); - } - else - { - return std::unexpected(Error{ - "Invalid ModelSpec type for MNIST: expected MLP or Transformer"}); - } -} - -} // namespace nn - -#endif // MNIST_COMMON_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/model.hpp b/include/neuralnet.cpp/model.hpp deleted file mode 100644 index a31da34c..00000000 --- a/include/neuralnet.cpp/model.hpp +++ /dev/null @@ -1,119 +0,0 @@ -#ifndef MODEL_HPP -#define MODEL_HPP - -#include -#include -#include -#include -#include -#include - -#include "nn_config.hpp" -#include "layer.hpp" - -namespace nn -{ - class Model - { - private: - std::vector> layers_; - - public: - Model() = default; - - // 访问指定层（用于向下转型等场景） - // 使用 assert 保护边界，编程错误在 debug 模式捕获 - [[nodiscard]] Layer &layer_at(std::size_t index) noexcept - { - assert(index < layers_.size() && "Model::layer_at index out of range"); - return *layers_[index]; - } - - // 不可拷贝（unique_ptr 语义），只能移动 - Model(const Model &) = delete; - Model &operator=(const Model &) = delete; - Model(Model &&) noexcept = default; - Model &operator=(Model &&) noexcept = default; - - // ── 构建网络 ──────────────────────────────────────────────────────── - // 用法：model.add(784, 64).add()... - template - Model &add(Args &&...args) - { - layers_.emplace_back(std::make_unique(std::forward(args)...)); - return *this; - } - - [[nodiscard]] std::size_t num_layers() const noexcept { return layers_.size(); } - - // ── 前向传播 ──────────────────────────────────────────────────────── - [[nodiscard]] Result forward(const Matrix &input) - { - if (layers_.empty()) - { - return std::unexpected(Error{"Model has no layers"}); - } - auto result = layers_.front()->forward(input); - if (!result) return std::unexpected(result.error()); - Matrix out = std::move(*result); - for (std::size_t i = 1; i < layers_.size(); ++i) - { - result = layers_[i]->forward(out); - if (!result) return std::unexpected(result.error()); - out = std::move(*result); - } - return out; - } - - // ── 反向传播 ──────────────────────────────────────────────────────── - // 传入 loss 对最后一层输出的梯度，返回对输入的梯度（通常不需要） - [[nodiscard]] Result backward(const Matrix &grad_output) - { - if (layers_.empty()) - { - return std::unexpected(Error{"Model has no layers"}); - } - auto result = layers_.back()->backward(grad_output); - if (!result) return std::unexpected(result.error()); - Matrix grad = std::move(*result); - for (std::size_t i = layers_.size() - 1; i-- > 0;) - { - result = layers_[i]->backward(grad); - if (!result) return std::unexpected(result.error()); - grad = std::move(*result); - } - return grad; - } - - // ── 参数收集 ──────────────────────────────────────────────────────── - // 聚合所有层的可训练参数，供 Optimizer 使用 - [[nodiscard]] std::vector> parameters() - { - std::vector> result; - for (auto &layer : layers_) - { - for (auto &p : layer->parameters()) - { - result.push_back(p); - } - } - return result; - } - - [[nodiscard]] std::vector> param_gradients() - { - std::vector> result; - for (auto &layer : layers_) - { - for (auto &g : layer->param_gradients()) - { - result.push_back(g); - } - } - return result; - } - }; - -} // namespace nn - -#endif // MODEL_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/model_container.hpp b/include/neuralnet.cpp/model_container.hpp new file mode 100644 index 00000000..11c2bbdf --- /dev/null +++ b/include/neuralnet.cpp/model_container.hpp @@ -0,0 +1,158 @@ +#ifndef NN_MODEL_CONTAINER_HPP +#define NN_MODEL_CONTAINER_HPP + +// ── model_container.hpp — 引擎化模型容器 ─────────────────────────────────── +// +// 架构铁律： +// 1. Model 持有 observer_ptr（非拥有），所有 forward/backward 委托给 +// Layer::forward(engine, Tensor) / Layer::backward(engine, Tensor)。 +// 2. Model 不包含任何 GPU-resident 中间结果缓存或 forward_gpu/backward_gpu +// 路径 —— 设备分发完全由 ComputeEngine 实现负责。 +// 3. forward/backward 操作 Tensor，单套实现适配 CPU/GPU。 +// +// 数据流（GPU 训练为例）： +// Matrix(x_batch) ──engine.from_matrix──▶ Tensor[GPU] +// ──forward──▶ Tensor[GPU] ──loss.forward──▶ Scalar +// ──loss.backward──▶ Tensor[GPU] ──backward──▶ (丢弃) +// ──optimizer.step──▶ 参数就地更新（全程在 GPU） +// 仅 evaluate 时 engine.to_matrix 下载到 CPU 做 argmax。 +// ───────────────────────────────────────────────────────────────────────── + +#include +#include +#include + +#include "core_assert.hpp" +#include "compute_layer.hpp" + +namespace nn +{ + +class Model +{ +private: + observer_ptr engine_; + std::vector> layers_; + +public: + Model() = default; + explicit Model(ComputeEngine& engine) : engine_(engine) {} + + // 不可拷贝（unique_ptr 语义），可移动 + Model(const Model&) = delete; + Model& operator=(const Model&) = delete; + Model(Model&&) noexcept = default; + Model& operator=(Model&&) noexcept = default; + + // ── 引擎绑定 ───────────────────────────────────────────────────────── + void set_engine(ComputeEngine& engine) { engine_.reset(&engine); } + [[nodiscard]] ComputeEngine& engine() const noexcept + { + NN_ASSERT(engine_, "Model: engine not bound"); + return *engine_; + } + + // ── 构建网络（通用模板接口） ──────────────────────────────────────── + // 用法：model.add(engine, in, out) / model.add() / ... + // 注意：MultiHeadAttention 的构造函数是 protected（设计为基类， + // 由 TransformerEncoderLayer / CausalSelfAttention 内部组合）， + // 因此不直接通过 add 添加。 + template + Model& add(Args&&... args) + { + layers_.emplace_back(std::make_unique(std::forward(args)...)); + return *this; + } + + // ── 访问 ───────────────────────────────────────────────────────────── + [[nodiscard]] std::size_t num_layers() const noexcept { return layers_.size(); } + + [[nodiscard]] Layer& layer_at(std::size_t index) noexcept + { + NN_ASSERT(index < layers_.size(), "Model::layer_at index out of range"); + return *layers_[index]; + } + + // ── batch 录制粒度：在 Transformer block 间按间隔 flush ── + // 通过 Layer 基类虚函数分发：GPTModel override 生效， + // 其他层类型默认 no-op。 + void set_flush_interval(std::size_t interval) + { + for (auto& layer : layers_) + layer->set_flush_interval(interval); + } + + // ── 前向传播：Tensor → Tensor（全程不离开 engine 设备） ─────────────── + [[nodiscard]] Result forward(const Tensor& input) + { + if (layers_.empty()) + return std::unexpected(Error{"Model::forward: no layers"}); + if (engine_ == nullptr) + return std::unexpected(Error{"Model::forward: engine not bound"}); + + Tensor x = input; + for (auto& layer : layers_) + { + auto r = layer->forward(*engine_, x); + if (!r) return std::unexpected(r.error()); + x = std::move(*r); + } + return x; + } + + // ── 反向传播：Tensor → Tensor（从最后一层到第一层） ─────────────────── + [[nodiscard]] Result backward(const Tensor& grad_output) + { + if (layers_.empty()) + return std::unexpected(Error{"Model::backward: no layers"}); + if (engine_ == nullptr) + return std::unexpected(Error{"Model::backward: engine not bound"}); + + Tensor g = grad_output; + for (std::size_t i = layers_.size(); i-- > 0;) + { + auto r = layers_[i]->backward(*engine_, g); + if (!r) return std::unexpected(r.error()); + g = std::move(*r); + } + return g; + } + + // ── 参数收集：返回 TensorRef 供 Optimizer 使用 ────────────────────────── + [[nodiscard]] std::vector parameters() + { + std::vector result; + for (auto& layer : layers_) + { + auto layer_params = layer->parameters(); + result.insert(result.end(), layer_params.begin(), layer_params.end()); + } + return result; + } + + [[nodiscard]] std::vector param_gradients() + { + std::vector result; + for (auto& layer : layers_) + { + auto layer_grads = layer->param_gradients(); + result.insert(result.end(), layer_grads.begin(), layer_grads.end()); + } + return result; + } + + // ── 梯度清零（每个训练 step 开头调用） ──────────────────────────────── + [[nodiscard]] Result zero_grad() + { + for (auto& layer : layers_) + { + auto r = layer->zero_grad(*engine_); + if (!r) return r; + } + return {}; + } +}; + +} // namespace nn + +#endif // NN_MODEL_CONTAINER_HPP diff --git a/include/neuralnet.cpp/model_io.hpp b/include/neuralnet.cpp/model_io.hpp deleted file mode 100644 index e7c3dbff..00000000 --- a/include/neuralnet.cpp/model_io.hpp +++ /dev/null @@ -1,407 +0,0 @@ -#ifndef MODEL_IO_HPP -#define MODEL_IO_HPP - -// ═══════════════════════════════════════════════════════════════════════════ -// model_io.hpp — 模型二进制序列化 -// -// 职责：将 Model + ModelSpec 保存为二进制文件，或从文件加载参数。 -// 设计： -// - 使用固定宽度整数 (uint64_t) 保证跨平台/跨位宽兼容性 -// - 纯库函数，无 I/O 副作用（调用方自行决定是否打印日志） -// - 向后兼容 V1 格式读取，新文件统一写入 V2 格式 -// - 使用 Result 返回错误，不抛异常 -// ═══════════════════════════════════════════════════════════════════════════ - -#include -#include -#include -#include -#include - -#include "nn_config.hpp" -#include "layer.hpp" -#include "model.hpp" - -namespace nn -{ - - - -// ═══════════════════════════════════════════════════════════════════════════ -// ModelSpec — 模型架构描述，嵌入到 V2 文件头部 -// -// 加载时可先调用 peek_model_spec() 读取规格，再据此构建模型。 -// ═══════════════════════════════════════════════════════════════════════════ - -enum class ModelType : uint32_t -{ - Unknown = 0, - MLP = 1, - Transformer = 2, - GPT = 3, -}; - -struct ModelSpec -{ - ModelType type = ModelType::Unknown; - - // ── MLP ── - std::vector layer_dims; - - // ── Transformer (MNIST ViT) ── - std::size_t d_model = 0; - std::size_t num_heads = 0; - std::size_t d_ff = 0; - std::size_t num_layers = 0; - std::size_t patch_size = 0; - - // ── GPT ── - std::size_t vocab_size = 0; - std::size_t seq_len = 0; - - [[nodiscard]] bool is_mlp() const noexcept { return type == ModelType::MLP; } - [[nodiscard]] bool is_transformer() const noexcept { return type == ModelType::Transformer; } - [[nodiscard]] bool is_gpt() const noexcept { return type == ModelType::GPT; } -}; - -// ═══════════════════════════════════════════════════════════════════════════ -// 二进制文件格式 -// -// Version 1 (旧格式，仅参数): -// [magic 4B] [version=1 4B] [矩阵数据...] -// -// Version 2 (当前格式，含架构规格): -// [magic 4B] [version=2 4B] [model_type 4B] [规格数据...] [矩阵数据...] -// -// 规格编码 (所有多字节整数均为小端序 uint64_t): -// MLP: [num_dims 4B] [dim_0 8B] ... [dim_N 8B] -// Transformer: [d_model 8B] [num_heads 8B] [d_ff 8B] [num_layers 8B] [patch_size 8B] -// GPT: [vocab_size 8B] [d_model 8B] [seq_len 8B] [num_heads 8B] [d_ff 8B] [num_layers 8B] -// ═══════════════════════════════════════════════════════════════════════════ - -inline constexpr uint32_t MODEL_MAGIC = 0x4E4E4E4E; // "NNNN" -inline constexpr uint32_t MODEL_VERSION = 2; - -// ═══════════════════════════════════════════════════════════════════════════ -// detail — 内部读写工具 -// -// 所有函数使用固定宽度整数 (uint64_t) 替代 std::size_t，保证跨平台兼容。 -// 所有函数返回 Result，失败时返回 Error 错误信息。 -// ═══════════════════════════════════════════════════════════════════════════ - -namespace detail -{ - -// ── 基础类型读写 ────────────────────────────────────────────────────── - -inline Result write_u32(std::ofstream &ofs, uint32_t v) -{ - ofs.write(reinterpret_cast(&v), sizeof(v)); - if (!ofs) - return std::unexpected(Error{"Write error while writing u32"}); - return {}; -} - -[[nodiscard]] inline Result read_u32(std::ifstream &ifs) -{ - uint32_t v; - ifs.read(reinterpret_cast(&v), sizeof(v)); - if (!ifs) - return std::unexpected(Error{"Unexpected end of file while reading u32"}); - return v; -} - -inline Result write_u64(std::ofstream &ofs, uint64_t v) -{ - ofs.write(reinterpret_cast(&v), sizeof(v)); - if (!ofs) - return std::unexpected(Error{"Write error while writing u64"}); - return {}; -} - -[[nodiscard]] inline Result read_u64(std::ifstream &ifs) -{ - uint64_t v; - ifs.read(reinterpret_cast(&v), sizeof(v)); - if (!ifs) - return std::unexpected(Error{"Unexpected end of file while reading u64"}); - return v; -} - -// ── 矩阵读写 ────────────────────────────────────────────────────────── - -inline Result write_matrix(std::ofstream &ofs, const Matrix &m) -{ - if (auto r = write_u64(ofs, static_cast(m.rows())); !r) - return std::unexpected(r.error()); - if (auto r = write_u64(ofs, static_cast(m.cols())); !r) - return std::unexpected(r.error()); - const auto &data = m.data(); - ofs.write(reinterpret_cast(data.data()), - static_cast(data.size() * sizeof(double))); - if (!ofs) - return std::unexpected(Error{"Write error while writing matrix data"}); - return {}; -} - -inline Result read_matrix(std::ifstream &ifs, Matrix &m) -{ - auto rows_r = read_u64(ifs); - if (!rows_r) return std::unexpected(rows_r.error()); - auto cols_r = read_u64(ifs); - if (!cols_r) return std::unexpected(cols_r.error()); - const auto rows = static_cast(*rows_r); - const auto cols = static_cast(*cols_r); - - if (rows != m.rows() || cols != m.cols()) - { - return std::unexpected(Error{ - "Matrix shape mismatch: expected (" + std::to_string(m.rows()) - + ", " + std::to_string(m.cols()) + "), got (" - + std::to_string(rows) + ", " + std::to_string(cols) + ")"}); - } - - auto &data = m.data(); - ifs.read(reinterpret_cast(data.data()), - static_cast(data.size() * sizeof(double))); - if (!ifs) - return std::unexpected(Error{"Unexpected end of file while reading matrix data"}); - return {}; -} - -// ── ModelSpec 序列化 ────────────────────────────────────────────────── - -inline Result write_spec(std::ofstream &ofs, const ModelSpec &spec) -{ - if (auto r = write_u32(ofs, static_cast(spec.type)); !r) - return std::unexpected(r.error()); - - switch (spec.type) - { - case ModelType::MLP: - { - if (auto r = write_u32(ofs, static_cast(spec.layer_dims.size())); !r) - return std::unexpected(r.error()); - for (auto d : spec.layer_dims) - if (auto r = write_u64(ofs, static_cast(d)); !r) - return std::unexpected(r.error()); - break; - } - - case ModelType::Transformer: - if (auto r = write_u64(ofs, static_cast(spec.d_model)); !r) - return std::unexpected(r.error()); - if (auto r = write_u64(ofs, static_cast(spec.num_heads)); !r) - return std::unexpected(r.error()); - if (auto r = write_u64(ofs, static_cast(spec.d_ff)); !r) - return std::unexpected(r.error()); - if (auto r = write_u64(ofs, static_cast(spec.num_layers)); !r) - return std::unexpected(r.error()); - if (auto r = write_u64(ofs, static_cast(spec.patch_size)); !r) - return std::unexpected(r.error()); - break; - - case ModelType::GPT: - if (auto r = write_u64(ofs, static_cast(spec.vocab_size)); !r) - return std::unexpected(r.error()); - if (auto r = write_u64(ofs, static_cast(spec.d_model)); !r) - return std::unexpected(r.error()); - if (auto r = write_u64(ofs, static_cast(spec.seq_len)); !r) - return std::unexpected(r.error()); - if (auto r = write_u64(ofs, static_cast(spec.num_heads)); !r) - return std::unexpected(r.error()); - if (auto r = write_u64(ofs, static_cast(spec.d_ff)); !r) - return std::unexpected(r.error()); - if (auto r = write_u64(ofs, static_cast(spec.num_layers)); !r) - return std::unexpected(r.error()); - break; - - default: - return std::unexpected(Error{"Cannot write unknown ModelType: " - + std::to_string(static_cast(spec.type))}); - } - return {}; -} - -[[nodiscard]] inline Result read_spec(std::ifstream &ifs) -{ - ModelSpec spec; - auto type_r = read_u32(ifs); - if (!type_r) return std::unexpected(type_r.error()); - spec.type = static_cast(*type_r); - - switch (spec.type) - { - case ModelType::MLP: - { - auto nd_r = read_u32(ifs); - if (!nd_r) return std::unexpected(nd_r.error()); - const auto nd = *nd_r; - spec.layer_dims.resize(nd); - for (uint32_t i = 0; i < nd; ++i) - { - auto d_r = read_u64(ifs); - if (!d_r) return std::unexpected(d_r.error()); - spec.layer_dims[i] = static_cast(*d_r); - } - break; - } - - case ModelType::Transformer: - { - auto v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.d_model = static_cast(*v); - v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.num_heads = static_cast(*v); - v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.d_ff = static_cast(*v); - v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.num_layers = static_cast(*v); - v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.patch_size = static_cast(*v); - break; - } - - case ModelType::GPT: - { - auto v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.vocab_size = static_cast(*v); - v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.d_model = static_cast(*v); - v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.seq_len = static_cast(*v); - v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.num_heads = static_cast(*v); - v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.d_ff = static_cast(*v); - v = read_u64(ifs); if (!v) return std::unexpected(v.error()); - spec.num_layers = static_cast(*v); - break; - } - - default: - return std::unexpected(Error{"Unknown ModelType in file: " - + std::to_string(static_cast(spec.type))}); - } - - return spec; -} - -// ── 文件头读写 ────────────────────────────────────────────────────── - -inline Result write_header(std::ofstream &ofs) -{ - if (auto r = write_u32(ofs, MODEL_MAGIC); !r) - return std::unexpected(r.error()); - if (auto r = write_u32(ofs, MODEL_VERSION); !r) - return std::unexpected(r.error()); - return {}; -} - -// 返回读到的 version (1 或 2)，同时校验 magic number -[[nodiscard]] inline Result read_and_validate_header(std::ifstream &ifs) -{ - auto magic_r = read_u32(ifs); - if (!magic_r) return std::unexpected(magic_r.error()); - if (*magic_r != MODEL_MAGIC) - return std::unexpected(Error{"Invalid model file: bad magic number (0x" - + std::to_string(*magic_r) + ")"}); - - auto version_r = read_u32(ifs); - if (!version_r) return std::unexpected(version_r.error()); - if (*version_r == 0 || *version_r > MODEL_VERSION) - return std::unexpected(Error{"Unsupported model file version: " - + std::to_string(*version_r)}); - - return *version_r; -} - -} // namespace detail - -// ═══════════════════════════════════════════════════════════════════════════ -// 公开 API -// -// save_model — 保存 Model + ModelSpec 为 V2 格式二进制文件 -// load_model — 从文件加载参数到已有 Model（兼容 V1/V2） -// peek_model_spec — 只读文件头，返回 ModelSpec（不读参数，V1 返回 Unknown） -// -// 所有函数在失败时返回 Result 错误，无 std::cout 副作用。 -// 调用方如需日志，可检查返回值并打印。 -// ═══════════════════════════════════════════════════════════════════════════ - -// ── 保存模型 + 规格 ────────────────────────────────────────────────────── -inline Result save_model(const std::string &filename, - Model &model, const ModelSpec &spec) -{ - std::ofstream ofs(filename, std::ios::binary); - if (!ofs) - return std::unexpected(Error{"Cannot open file for writing: " + filename}); - - if (auto r = detail::write_header(ofs); !r) - return std::unexpected(r.error()); - if (auto r = detail::write_spec(ofs, spec); !r) - return std::unexpected(r.error()); - - auto params = model.parameters(); - for (auto &p_ref : params) - { - if (auto r = detail::write_matrix(ofs, p_ref.get()); !r) - return std::unexpected(r.error()); - } - - if (!ofs) - return std::unexpected(Error{"Write error while saving model to: " + filename}); - return {}; -} - -// ── 读取模型规格（只读头部，不读参数） ────────────────────────────────── -// V1 文件返回 ModelType::Unknown -[[nodiscard]] inline Result peek_model_spec(const std::string &filename) -{ - std::ifstream ifs(filename, std::ios::binary); - if (!ifs) - return std::unexpected(Error{"Cannot open file for reading: " + filename}); - - auto version_r = detail::read_and_validate_header(ifs); - if (!version_r) return std::unexpected(version_r.error()); - - if (*version_r == 2) - return detail::read_spec(ifs); - - // V1 文件没有规格信息 - return ModelSpec{ModelType::Unknown, {}, 0, 0, 0, 0, 0, 0, 0}; -} - -// ── 加载参数到已有模型 ────────────────────────────────────────────────── -// 兼容 V1 和 V2 文件（V2 会跳过规格头，只读参数） -inline Result load_model(const std::string &filename, Model &model) -{ - std::ifstream ifs(filename, std::ios::binary); - if (!ifs) - return std::unexpected(Error{"Cannot open file for reading: " + filename}); - - auto version_r = detail::read_and_validate_header(ifs); - if (!version_r) return std::unexpected(version_r.error()); - - // V2 有规格头，跳过（规格已隐含在构建好的 model 中） - if (*version_r == 2) - { - auto spec_r = detail::read_spec(ifs); - if (!spec_r) return std::unexpected(spec_r.error()); - } - - auto params = model.parameters(); - for (auto &p_ref : params) - { - if (auto r = detail::read_matrix(ifs, p_ref.get()); !r) - return std::unexpected(r.error()); - } - - if (!ifs) - return std::unexpected(Error{"Read error while loading model from: " + filename}); - return {}; -} - -} // namespace nn - -#endif // MODEL_IO_HPP diff --git a/include/neuralnet.cpp/model_serialization.hpp b/include/neuralnet.cpp/model_serialization.hpp new file mode 100644 index 00000000..c7fc411a --- /dev/null +++ b/include/neuralnet.cpp/model_serialization.hpp @@ -0,0 +1,504 @@ +#ifndef NN_MODEL_SERIALIZATION_HPP +#define NN_MODEL_SERIALIZATION_HPP + +// ═══════════════════════════════════════════════════════════════════════════ +// model_io.hpp — 模型二进制序列化 +// +// 职责：将 Model + ModelSpec 保存为二进制文件，或从文件加载参数。 +// 设计： +// - 使用固定宽度整数 (uint64_t) 保证跨平台/跨位宽兼容性 +// - 纯库函数，无 I/O 副作用（调用方自行决定是否打印日志） +// - 向后兼容 V1 格式读取，新文件统一写入 V2 格式 +// - 使用 Result 返回错误，不抛异常 +// ═══════════════════════════════════════════════════════════════════════════ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.hpp" +#include "model_spec.hpp" +#include "compute_layer.hpp" +#include "model_container.hpp" + +namespace nn +{ + +// ═══════════════════════════════════════════════════════════════════════════ +// 二进制文件格式 +// +// Version 3 (当前格式，含精度标记 + 可选 tokenizer): +// [magic 4B] [version=3 4B] [precision 1B] [model_type 4B] +// [spec data...] +// [param matrices...] +// [tokenizer_len 8B] [tokenizer_json...] ← V3 新增 +// +// Version 2 (含架构规格): +// [magic 4B] [version=2 4B] [model_type 4B] [spec data...] [matrices...] +// +// Version 1 (仅参数): +// [magic 4B] [version=1 4B] [matrices...] +// +// precision 字节: 0 = f32, 1 = f64（用于校验保存时与加载时的 Scalar 类型一致） +// tokenizer_len = 0 表示未嵌入分词器（向后兼容） +// ═══════════════════════════════════════════════════════════════════════════ + +inline constexpr uint32_t MODEL_MAGIC = 0x4E4E4E4E; // "NNNN" +inline constexpr uint32_t MODEL_VERSION = 3; + +// ── 精度标记（写入文件头，加载时校验） ────────────────────────────────── +// 0 = float (f32), 1 = double (f64) +inline constexpr uint8_t PRECISION_TAG = sizeof(Scalar) == 4 ? 0 : 1; +static_assert(sizeof(Scalar) == 4 || sizeof(Scalar) == 8, + "Scalar must be float (4 bytes) or double (8 bytes)"); + +// ═══════════════════════════════════════════════════════════════════════════ +// detail — 内部读写工具 +// +// 所有函数使用固定宽度整数 (uint64_t) 替代 std::size_t，保证跨平台兼容。 +// 所有函数返回 Result，失败时返回 Error 错误信息。 +// ═══════════════════════════════════════════════════════════════════════════ + +namespace detail +{ + +// ── 安全二进制 I/O 辅助（替代 reinterpret_cast）───────────────────── +// 使用 std::as_bytes(std::span) 实现类型安全的二进制读写， +// 避免直接 reinterpret_cast。 + +template + requires std::is_trivially_copyable_v +[[nodiscard]] inline Result write_bytes(std::ofstream &ofs, const T &v) +{ + auto bytes = std::as_bytes(std::span(&v, 1)); + ofs.write(reinterpret_cast(bytes.data()), + static_cast(bytes.size_bytes())); + if (!ofs) + return std::unexpected(Error{"Write error"}); + return {}; +} + +template + requires std::is_trivially_copyable_v +[[nodiscard]] inline Result read_bytes(std::ifstream &ifs) +{ + T v{}; + auto bytes = std::as_writable_bytes(std::span(&v, 1)); + ifs.read(reinterpret_cast(bytes.data()), + static_cast(bytes.size_bytes())); + if (!ifs) + return std::unexpected(Error{"Unexpected end of file"}); + return v; +} + +// ── 基础类型读写 ────────────────────────────────────────────────────── +// 直接使用 write_bytes / read_bytes，不再提供 write_u32/u64 等冗余包装。 + +// variadic 字段批量写入：依次写入每个字段，遇错即停 +template +[[nodiscard]] inline Result write_fields(std::ofstream &ofs, const Ts &...fields) +{ + Result result = {}; + bool stop = false; + auto write_one = [&](const auto &field) { + if (stop) return; + result = write_bytes(ofs, field); + if (!result) stop = true; + }; + (write_one(fields), ...); + return result; +} + +// variadic 字段批量读取：依次读出每个字段，遇错即停 +// 注意：实现使用递归模板逐个读取，避免 std::apply 在 MSVC 上的推导问题。 +namespace detail_read +{ +// 递归终止：所有字段已读取 +template +[[nodiscard]] inline Result read_fields_rec( + std::ifstream & /*ifs*/, Tuple & /*values*/, std::index_sequence<>) +{ + return {}; +} + +// 递归读取第 I 个字段，然后读取剩余字段 +template +[[nodiscard]] inline Result read_fields_rec( + std::ifstream &ifs, Tuple &values, std::index_sequence) +{ + using T = std::tuple_element_t; + auto r = read_bytes(ifs); + if (!r) return std::unexpected(r.error()); + std::get(values) = *r; + return read_fields_rec(ifs, values, std::index_sequence{}); +} +} // namespace detail_read + +template +[[nodiscard]] inline Result> read_fields(std::ifstream &ifs) +{ + std::tuple values{}; + auto result = detail_read::read_fields_rec>( + ifs, values, std::index_sequence_for{}); + if (!result) return std::unexpected(result.error()); + return values; +} + +// ── 矩阵读写 ────────────────────────────────────────────────────────── + +[[nodiscard]] inline Result write_matrix(std::ofstream &ofs, const Matrix &m) +{ + if (auto r = write_bytes(ofs, static_cast(m.rows())); !r) + return std::unexpected(r.error()); + if (auto r = write_bytes(ofs, static_cast(m.cols())); !r) + return std::unexpected(r.error()); + const auto s = m.span(); + ofs.write(reinterpret_cast(s.data()), + static_cast(s.size_bytes())); + if (!ofs) + return std::unexpected(Error{"Write error while writing matrix data"}); + return {}; +} + +[[nodiscard]] inline Result read_matrix(std::ifstream &ifs, Matrix &m) +{ + auto rows_r = read_bytes(ifs); + if (!rows_r) return std::unexpected(rows_r.error()); + auto cols_r = read_bytes(ifs); + if (!cols_r) return std::unexpected(cols_r.error()); + const auto rows = static_cast(*rows_r); + const auto cols = static_cast(*cols_r); + + if (rows != m.rows() || cols != m.cols()) + { + return std::unexpected(Error{ + "Matrix shape mismatch: expected (" + std::to_string(m.rows()) + + ", " + std::to_string(m.cols()) + "), got (" + + std::to_string(rows) + ", " + std::to_string(cols) + ")"}); + } + + auto s = m.span(); + ifs.read(reinterpret_cast(s.data()), + static_cast(s.size_bytes())); + if (!ifs) + return std::unexpected(Error{"Unexpected end of file while reading matrix data"}); + return {}; +} + +// ── ModelSpec 序列化 ────────────────────────────────────────────────── + +[[nodiscard]] inline Result write_spec(std::ofstream &ofs, const ModelSpec &spec) +{ + if (auto r = write_bytes(ofs, static_cast(spec.type)); !r) + return std::unexpected(r.error()); + + switch (spec.type) + { + case ModelType::MLP: + { + if (auto r = write_bytes(ofs, static_cast(spec.layer_dims.size())); !r) + return std::unexpected(r.error()); + for (auto d : spec.layer_dims) + if (auto r = write_bytes(ofs, static_cast(d)); !r) + return std::unexpected(r.error()); + break; + } + + case ModelType::Transformer: + return write_fields(ofs, + static_cast(spec.d_model), + static_cast(spec.num_heads), + static_cast(spec.d_ff), + static_cast(spec.num_layers), + static_cast(spec.patch_size)); + + case ModelType::GPT: + if (auto r = write_fields(ofs, + static_cast(spec.vocab_size), + static_cast(spec.d_model), + static_cast(spec.seq_len), + static_cast(spec.num_heads), + static_cast(spec.d_ff), + static_cast(spec.num_layers)); !r) + return std::unexpected(r.error()); + // V5+: pos_encoding + activation + norm_type (向后兼容：旧文件无此字段，默认) + if (auto r = write_bytes(ofs, static_cast(spec.pos_encoding)); !r) + return std::unexpected(r.error()); + if (auto r = write_bytes(ofs, static_cast(spec.activation)); !r) + return std::unexpected(r.error()); + return write_bytes(ofs, static_cast(spec.norm_type)); + + default: + return std::unexpected(Error{"Cannot write unknown ModelType: " + + std::to_string(static_cast(spec.type))}); + } + return {}; +} + +[[nodiscard]] inline Result read_spec(std::ifstream &ifs) +{ + ModelSpec spec; + auto type_r = read_bytes(ifs); + if (!type_r) return std::unexpected(type_r.error()); + spec.type = static_cast(*type_r); + + switch (spec.type) + { + case ModelType::MLP: + { + auto nd_r = read_bytes(ifs); + if (!nd_r) return std::unexpected(nd_r.error()); + const auto nd = *nd_r; + spec.layer_dims.resize(nd); + for (uint32_t i = 0; i < nd; ++i) + { + auto d_r = read_bytes(ifs); + if (!d_r) return std::unexpected(d_r.error()); + spec.layer_dims[i] = static_cast(*d_r); + } + break; + } + + case ModelType::Transformer: + { + auto v = read_fields(ifs); + if (!v) return std::unexpected(v.error()); + auto& [dm, nh, df, nl, ps] = *v; + spec.d_model = static_cast(dm); + spec.num_heads = static_cast(nh); + spec.d_ff = static_cast(df); + spec.num_layers = static_cast(nl); + spec.patch_size = static_cast(ps); + break; + } + + case ModelType::GPT: + { + auto v = read_fields(ifs); + if (!v) return std::unexpected(v.error()); + auto& [vs, dm, sl, nh, df, nl] = *v; + spec.vocab_size = static_cast(vs); + spec.d_model = static_cast(dm); + spec.seq_len = static_cast(sl); + spec.num_heads = static_cast(nh); + spec.d_ff = static_cast(df); + spec.num_layers = static_cast(nl); + // V5+: pos_encoding + activation（向后兼容：旧文件读到 EOF 时保持默认） + auto pe = read_bytes(ifs); + if (pe) + spec.pos_encoding = static_cast(*pe); + // else: 旧文件无此字段，spec.pos_encoding 已默认 Learned + auto act = read_bytes(ifs); + if (act) + spec.activation = static_cast(*act); + // else: 旧文件无此字段，spec.activation 已默认 GeLU + auto nt = read_bytes(ifs); + if (nt) + spec.norm_type = static_cast(*nt); + // else: 旧文件无此字段，spec.norm_type 已默认 LayerNorm + break; + } + + default: + return std::unexpected(Error{"Unknown ModelType in file: " + + std::to_string(static_cast(spec.type))}); + } + + return spec; +} + +// ── 文件头读写 ────────────────────────────────────────────────────── + +[[nodiscard]] inline Result write_header(std::ofstream &ofs) +{ + if (auto r = write_bytes(ofs, MODEL_MAGIC); !r) + return std::unexpected(r.error()); + if (auto r = write_bytes(ofs, MODEL_VERSION); !r) + return std::unexpected(r.error()); + if (auto r = write_bytes(ofs, PRECISION_TAG); !r) + return std::unexpected(r.error()); + return {}; +} + +// 返回读到的 version，同时校验 magic number 和精度 +[[nodiscard]] inline Result read_and_validate_header(std::ifstream &ifs) +{ + auto magic_r = read_bytes(ifs); + if (!magic_r) return std::unexpected(magic_r.error()); + if (*magic_r != MODEL_MAGIC) + return std::unexpected(Error{"Invalid model file: bad magic number (0x" + + std::to_string(*magic_r) + ")"}); + + auto version_r = read_bytes(ifs); + if (!version_r) return std::unexpected(version_r.error()); + if (*version_r == 0 || *version_r > MODEL_VERSION) + return std::unexpected(Error{"Unsupported model file version: " + + std::to_string(*version_r)}); + + // V3: 读取并校验精度标记 + if (*version_r >= 3) + { + auto pt_result = read_bytes(ifs); + if (!pt_result) + return std::unexpected(Error{"Unexpected end: missing precision tag"}); + uint8_t precision_tag = *pt_result; + if (precision_tag != PRECISION_TAG) + return std::unexpected(Error{ + "Precision mismatch: file uses " + std::string(precision_tag == 0 ? "f32" : "f64") + + ", but build uses " + std::string(PRECISION_TAG == 0 ? "f32" : "f64")}); + } + + return *version_r; +} + +// ── Tokenizer JSON 读写（V3 新增） ────────────────────────────────── + +[[nodiscard]] inline Result write_tokenizer(std::ofstream &ofs, const std::string &json) +{ + if (auto r = write_bytes(ofs, static_cast(json.size())); !r) + return std::unexpected(r.error()); + if (!json.empty()) + { + ofs.write(json.data(), static_cast(json.size())); + if (!ofs) + return std::unexpected(Error{"Write error while writing tokenizer"}); + } + return {}; +} + +[[nodiscard]] inline Result read_tokenizer(std::ifstream &ifs) +{ + auto len_r = read_bytes(ifs); + if (!len_r) return std::unexpected(len_r.error()); + const auto len = static_cast(*len_r); + if (len == 0) return std::string{}; // 未嵌入 + + std::string json(len, '\0'); + ifs.read(json.data(), static_cast(len)); + if (!ifs) + return std::unexpected(Error{"Unexpected end while reading tokenizer data"}); + return json; +} + +} // namespace detail + +// ═══════════════════════════════════════════════════════════════════════════ +// 公开 API +// +// save_model — 保存 Model + ModelSpec（可选嵌入 tokenizer JSON） +// load_model — 从文件加载参数，返回嵌入的 tokenizer JSON（如有） +// peek_model_spec — 只读文件头，返回 ModelSpec（不读参数） +// +// 向后兼容 V1/V2/V3 读取，写入统一为 V3。 +// ═══════════════════════════════════════════════════════════════════════════ + +// ── 保存模型（V3 格式，含精度标记 + 可选 tokenizer） ──────────────────── +// 新架构：参数为 Tensor*，通过 engine.to_matrix 下载到 CPU Matrix 后写入。 +[[nodiscard]] inline Result save_model(const std::string &filename, + Model &model, const ModelSpec &spec, const std::string &tokenizer_json = {}) +{ + std::ofstream ofs(filename, std::ios::binary); + if (!ofs) + return std::unexpected(Error{"Cannot open file for writing: " + filename}); + + if (auto r = detail::write_header(ofs); !r) + return std::unexpected(r.error()); + if (auto r = detail::write_spec(ofs, spec); !r) + return std::unexpected(r.error()); + + auto& engine = model.engine(); + auto params = model.parameters(); + for (auto& p_tensor : params) + { + auto m = engine.to_matrix(p_tensor); + if (!m) return std::unexpected(m.error()); + if (auto r = detail::write_matrix(ofs, *m); !r) + return std::unexpected(r.error()); + } + + // V3: 写入 tokenizer JSON（长度前缀，0 表示无） + if (auto r = detail::write_tokenizer(ofs, tokenizer_json); !r) + return std::unexpected(r.error()); + + if (!ofs) + return std::unexpected(Error{"Write error while saving model to: " + filename}); + return {}; +} + +// ── 读取模型规格（只读头部，不读参数/tokenizer） ────────────────────────── +// V1 文件返回 ModelType::Unknown +[[nodiscard]] inline Result peek_model_spec(const std::string &filename) +{ + std::ifstream ifs(filename, std::ios::binary); + if (!ifs) + return std::unexpected(Error{"Cannot open file for reading: " + filename}); + + auto version_r = detail::read_and_validate_header(ifs); + if (!version_r) return std::unexpected(version_r.error()); + + // V2/V3 有规格头 + if (*version_r >= 2) + return detail::read_spec(ifs); + + // V1 文件没有规格信息 + return ModelSpec{ModelType::Unknown, {}, 0, 0, 0, 0, 0, 0, 0}; +} + +// ── 加载参数 + tokenizer（兼容 V1/V2/V3） ───────────────────────────────── +// 返回嵌入的 tokenizer JSON 字符串（空串 = 未嵌入或旧格式） +// 新架构：先 read_matrix 读入临时 CPU Matrix，再通过 engine.copy_from +// 上传到参数 Tensor（CPU 拷贝 / GPU 上传由引擎实现决定）。 +[[nodiscard]] inline Result load_model(const std::string &filename, Model &model) +{ + std::ifstream ifs(filename, std::ios::binary); + if (!ifs) + return std::unexpected(Error{"Cannot open file for reading: " + filename}); + + auto version_r = detail::read_and_validate_header(ifs); + if (!version_r) return std::unexpected(version_r.error()); + const auto version = *version_r; + + // V2/V3: 读取并跳过规格头（规格已隐含在构建好的 model 中） + // TODO: 此处读完 spec 即丢弃，未与 model 自身的架构校验。 + // 完整校验需要 Layer 基类支持 spec() 方法，改动较大，暂留待后续实现。 + if (version >= 2) + { + auto spec_r = detail::read_spec(ifs); + if (!spec_r) return std::unexpected(spec_r.error()); + } + + auto& engine = model.engine(); + auto params = model.parameters(); + for (auto& p_tensor : params) + { + // 先读入临时 Matrix（按参数 Tensor 的形状） + Matrix tmp(p_tensor.get().rows(), p_tensor.get().cols()); + if (auto r = detail::read_matrix(ifs, tmp); !r) + return std::unexpected(r.error()); + // 通过 engine 上传到参数 Tensor + if (auto r = engine.copy_from(p_tensor, tmp); !r) + return std::unexpected(r.error()); + } + + // V3: 读取嵌入的 tokenizer JSON + std::string tokenizer_json; + if (version >= 3) + { + auto tok_r = detail::read_tokenizer(ifs); + if (!tok_r) return std::unexpected(tok_r.error()); + tokenizer_json = std::move(*tok_r); + } + + if (!ifs) + return std::unexpected(Error{"Read error while loading model from: " + filename}); + return tokenizer_json; +} + +} // namespace nn + +#endif // NN_MODEL_SERIALIZATION_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/model_spec.hpp b/include/neuralnet.cpp/model_spec.hpp new file mode 100644 index 00000000..149cd51d --- /dev/null +++ b/include/neuralnet.cpp/model_spec.hpp @@ -0,0 +1,86 @@ +#ifndef NN_MODEL_SPEC_HPP +#define NN_MODEL_SPEC_HPP + +// ═══════════════════════════════════════════════════════════════════════════ +// model_spec.hpp — 模型架构描述（纯数据结构，无 L2 依赖） +// +// 分层：L3 实现层 +// 职责：定义 ModelType 枚举和 ModelSpec 结构体，供 L3/L4 使用。 +// 依赖：仅标准库（, , ），不依赖 L2/L1/L0。 +// +// 设计理由： +// 从 model_io.hpp 中提取，使 L4 构建层无需 include model_io.hpp +// （model_io.hpp 依赖 layer.hpp，会导致 L4 跨层依赖 L2）。 +// ═══════════════════════════════════════════════════════════════════════════ + +#include +#include +#include + +namespace nn +{ + +// ── 位置编码类型 ───────────────────────────────────────────────────────── +enum class PosEncodingType : uint32_t +{ + Learned = 0, // 可学习位置嵌入（GPT 默认） + Sinusoidal = 1, // 正弦波固定位置编码 + ALiBi = 2, // 线性偏置注意力（无位置嵌入） +}; + +// ── FFN 激活类型 ───────────────────────────────────────────────────────── +enum class ActivationType : uint32_t +{ + GeLU = 0, // QuickGeLU（GPT-2 风格，默认） + SwiGLU = 1, // SwiGLU（LLaMA/Mistral 风格，每参数效率更高） +}; + +// ── 归一化层类型 ───────────────────────────────────────────────────────── +enum class NormType : uint32_t +{ + LayerNorm = 0, // LayerNorm（GPT-2 风格，默认） + RMSNorm = 1, // RMSNorm（LLaMA/Mistral 风格，更快更稳） +}; + +// ── 模型类型枚举 ───────────────────────────────────────────────────────── +enum class ModelType : uint32_t +{ + Unknown = 0, + MLP = 1, + Transformer = 2, + GPT = 3, + ALiBi_GPT = 4, // 使用 ALiBi 的 GPT 模型（向后兼容） +}; + +// ── 模型架构描述 ───────────────────────────────────────────────────────── +// 嵌入到二进制文件头部，加载时可先读取规格再据此构建模型。 +struct ModelSpec +{ + ModelType type = ModelType::Unknown; + + // ── MLP ── + std::vector layer_dims; + + // ── Transformer (MNIST ViT) ── + std::size_t d_model = 0; + std::size_t num_heads = 0; + std::size_t d_ff = 0; + std::size_t num_layers = 0; + std::size_t patch_size = 0; + + // ── GPT ── + std::size_t vocab_size = 0; + std::size_t seq_len = 0; + PosEncodingType pos_encoding = PosEncodingType::Learned; // 位置编码类型 + ActivationType activation = ActivationType::GeLU; // FFN 激活类型 + NormType norm_type = NormType::LayerNorm; // 归一化层类型 + + [[nodiscard]] bool is_mlp() const noexcept { return type == ModelType::MLP; } + [[nodiscard]] bool is_transformer() const noexcept { return type == ModelType::Transformer; } + [[nodiscard]] bool is_gpt() const noexcept { return type == ModelType::GPT; } + [[nodiscard]] bool is_alibi_gpt() const noexcept { return pos_encoding == PosEncodingType::ALiBi; } +}; + +} // namespace nn + +#endif // NN_MODEL_SPEC_HPP diff --git a/include/neuralnet.cpp/nn.hpp b/include/neuralnet.cpp/nn.hpp index 97af6784..9b0df820 100644 --- a/include/neuralnet.cpp/nn.hpp +++ b/include/neuralnet.cpp/nn.hpp @@ -1,20 +1,56 @@ #ifndef NN_HPP #define NN_HPP -#include -#include -#include -#include - -#include "nn_config.hpp" -#include "layer.hpp" -#include "loss.hpp" -#include "model.hpp" -#include "optimizer.hpp" -#include "mnist_common.hpp" +// ── neuralnet.cpp 统一入口 ──────────────────────────────────────────────── +// 包含所有核心模块，用户只需 #include "nn.hpp" +// +// 包含顺序按依赖关系排列：L0 → L1 → L2 → L3 → L4 +// 注意：algebra_matrix.hpp 已传递包含 algebra_span/ops/expr/compute.hpp， +// config.hpp 已传递包含 core_errors.hpp， +// 此处显式列出所有头文件是为了清晰展示模块结构。 +// +// 新架构（引擎化）： +// ComputeEngine 抽象硬件接触，Layer/Loss/Optimizer 通过组合原语表达算法， +// forward/backward 单套实现适配 CPU/GPU。 + +// L0 硬件层 +#include "core_errors.hpp" +#include "core_assert.hpp" +#include "core_observer_ptr.hpp" +#include "core_threadpool.hpp" +#include "core_file.hpp" +#include "config.hpp" + +// L1 代数层（algebra_matrix.hpp 已传递包含其余代数头文件） +#include "algebra_matrix.hpp" + +// L2 计算层 — 引擎化 +#include "tensor.hpp" +#include "compute_engine.hpp" +#include "cpu_engine.hpp" +#ifdef NN_HAS_VULKAN +#include "gpu_engine.hpp" +#endif +#ifdef NN_HAS_CUDA +#include "cuda_engine.hpp" +#endif +#include "compute_layer.hpp" +#include "compute_loss.hpp" +#include "compute_optimizer.hpp" + +// L3 实现层 +#include "model_container.hpp" +#include "model_spec.hpp" +#include "model_serialization.hpp" + +// L4 构建层 +#include "domain_mnist.hpp" +#include "domain_gpt.hpp" +#include "domain_tokenizer.hpp" namespace nn { + // one_hot 工具：将类别索引向量转为 one-hot 矩阵（列主序，每列一个样本） [[nodiscard]] inline Result one_hot(const std::vector &true_i, std::size_t mat_size) { const std::size_t batch_size = true_i.size(); @@ -33,4 +69,4 @@ namespace nn } } // namespace nn -#endif // NN_HPP \ No newline at end of file +#endif // NN_HPP diff --git a/include/neuralnet.cpp/nn_config.hpp b/include/neuralnet.cpp/nn_config.hpp deleted file mode 100644 index 6d2e1a58..00000000 --- a/include/neuralnet.cpp/nn_config.hpp +++ /dev/null @@ -1,114 +0,0 @@ -#ifndef NN_CONFIG_HPP -#define NN_CONFIG_HPP - -#include -#include -#include -#include // for std::distance -#include -#include // for std::thread::hardware_concurrency -#include "thread_pool.hpp" - -// ── 执行策略 ──────────────────────────────────────────────────────────────── -// 默认并行+向量化；编译时可通过 -DNN_EXEC_POLICY=std::execution::seq 覆盖 -#ifndef NN_EXEC_POLICY -#define NN_EXEC_POLICY std::execution::par_unseq -#endif - -// ── 缓存分块大小 ───────────────────────────────────────────────────────────── -// 64×64×8 = 32 KB，安全装入大多数 CPU 的 L1 缓存 -// 矩阵乘法、转置等所有分块操作共用此值 -// 修改时需同步评估 b_block 栈占用（BLOCK_SIZE² × 8 字节） -namespace nn -{ - // ── 错误类型（C++23 std::expected）───────────────────────────────────── - // 所有公共 API 使用 Result 返回错误，不抛异常。 - struct Error { - std::string message; - }; - template - using Result = std::expected; - - inline constexpr std::size_t BLOCK_SIZE = 64; - static_assert(BLOCK_SIZE * BLOCK_SIZE * sizeof(double) <= 65536, - "BLOCK_SIZE too large: b_block would exceed 64KB stack budget"); - // ── 数值常量 ───────────────────────────────────────────────────────────── - // 使用 constexpr 避免运行时计算 - inline constexpr double EPSILON = 1e-8; - - // ── 智能执行策略 ───────────────────────────────────────────────────────── - // 根据元素数量自动选择串行或使用全局线程池并行。 - // 线程池采用 latch + 调用者参与设计：一次批量入队 + 原子计数器， - // 消除了旧版 N×submit + N×future 的堆分配和同步开销。 - struct SmartPolicy { - // 阈值 = 每核最小分块 × 2，确保至少有 2 个分块可并行。 - // chunk_count() 已自动按核心数缩放（min(cores, total/4096)）， - // 阈值只需设最低门槛，核心利用随数据量自然增长。 - inline static constexpr std::size_t PARALLEL_THRESHOLD = 4096 * 2; // 8192 - - // for_each 版本 - template - static void for_each(Iterator first, Iterator last, Func&& func) { - const auto n = static_cast(std::distance(first, last)); - if (n >= PARALLEL_THRESHOLD) { - global_thread_pool().parallel_for_each(first, last, std::forward(func)); - } else { - for (auto it = first; it != last; ++it) - func(*it); - } - } - - // transform 版本（一元） - template - static void transform(InputIt first, InputIt last, OutputIt d_first, UnaryOp&& op) { - const auto n = static_cast(std::distance(first, last)); - if (n >= PARALLEL_THRESHOLD) { - global_thread_pool().parallel_transform(first, last, d_first, std::forward(op)); - } else { - std::transform(first, last, d_first, std::forward(op)); - } - } - - // transform 版本（二元） - template - static void transform(InputIt1 first1, InputIt1 last1, InputIt2 first2, OutputIt d_first, BinaryOp&& op) { - const auto n = static_cast(std::distance(first1, last1)); - if (n >= PARALLEL_THRESHOLD) { - global_thread_pool().parallel_transform(first1, last1, first2, d_first, std::forward(op)); - } else { - std::transform(first1, last1, first2, d_first, std::forward(op)); - } - } - - // transform_reduce 版本（一元 transform） - template - static T transform_reduce(InputIt first, InputIt last, T init, BinaryOp&& reduce_op, UnaryOp&& transform_op) { - const auto n = static_cast(std::distance(first, last)); - if (n >= PARALLEL_THRESHOLD) { - return global_thread_pool().parallel_transform_reduce( - first, last, init, - std::forward(reduce_op), std::forward(transform_op)); - } else { - return std::transform_reduce(std::execution::seq, first, last, init, - std::forward(reduce_op), std::forward(transform_op)); - } - } - - // transform_reduce 版本（二元 transform，两个输入范围） - template - static T transform_reduce(InputIt1 first1, InputIt1 last1, InputIt2 first2, T init, BinaryOp&& reduce_op, UnaryOp&& transform_op) { - const auto n = static_cast(std::distance(first1, last1)); - if (n >= PARALLEL_THRESHOLD) { - return global_thread_pool().parallel_transform_reduce(first1, last1, first2, init, - std::forward(reduce_op), - std::forward(transform_op)); - } else { - return std::transform_reduce(std::execution::seq, first1, last1, first2, init, - std::forward(reduce_op), - std::forward(transform_op)); - } - } - }; -} // namespace nn - -#endif // NN_CONFIG_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/optimizer.hpp b/include/neuralnet.cpp/optimizer.hpp deleted file mode 100644 index 364607be..00000000 --- a/include/neuralnet.cpp/optimizer.hpp +++ /dev/null @@ -1,231 +0,0 @@ -#ifndef OPTIMIZER_HPP -#define OPTIMIZER_HPP - -#include -#include -#include -#include -#include - -#include "matrix.hpp" - -namespace nn -{ - - class Optimizer - { - public: - virtual ~Optimizer() = default; - virtual Result step() = 0; - virtual void zero_grad() = 0; - }; - - class SGD : public Optimizer - { - private: - double lr_; - std::vector> params_; - std::vector> grads_; - - public: - SGD(std::vector> params, - std::vector> grads, - double lr) - : lr_(lr), params_(std::move(params)), grads_(std::move(grads)) {} - - Result step() override - { - if (params_.size() != grads_.size()) - { - return std::unexpected(Error{"params and grads must have same size"}); - } - for (std::size_t i = 0; i < params_.size(); ++i) - { - auto &p = params_[i].get(); - auto &g = grads_[i].get(); - SmartPolicy::transform(p.data().begin(), p.data().end(), - g.data().begin(), - p.data().begin(), - [this](double p_val, double g_val) - { - return p_val - lr_ * g_val; - }); - } - return {}; - } - - void zero_grad() override - { - for (auto &g_ref : grads_) - { - auto &g = g_ref.get(); - std::fill(g.data().begin(), g.data().end(), 0.0); - } - } - }; - - class SGD_w_Momentum : public Optimizer - { - private: - double lr_; - std::vector> params_; - std::vector> grads_; - std::vector velocities_; - double beta_ = 0.9; - - public: - SGD_w_Momentum(std::vector> params, - std::vector> grads, - double lr) - : lr_(lr), params_(std::move(params)), grads_(std::move(grads)) - { - velocities_.reserve(params_.size()); - for (const auto &p_ref : params_) - { - const Matrix &p = p_ref.get(); - velocities_.emplace_back(p.rows(), p.cols(), 0.0); - } - } - - Result step() override - { - if (params_.size() != grads_.size()) - { - return std::unexpected(Error{"params and grads must have same size"}); - } - - for (std::size_t i = 0; i < params_.size(); ++i) - { - auto &p = params_[i].get(); - auto &g = grads_[i].get(); - auto &v = velocities_[i]; - - // 获取vector类型的 - auto &p_vec = p.data(); - auto &g_vec = g.data(); - auto &v_vec = v.data(); - - // 构造zip - auto zip_view = std::views::zip(p_vec, g_vec, v_vec); - - // 并行处理：std::views::zip 将参数、梯度、速度打包为 tuple - // std::get 按索引解包，顺序与 zip 传入顺序一致 - SmartPolicy::for_each(zip_view.begin(), - zip_view.end(), - [this](auto &&tuple) - { - auto &p_val = std::get<0>(tuple); // 参数 - auto &g_val = std::get<1>(tuple); // 梯度 - auto &v_val = std::get<2>(tuple); // 速度（动量） - v_val = beta_ * v_val + (1 - beta_) * g_val; - p_val = p_val - lr_ * v_val; - }); - } - return {}; - } - - void zero_grad() override - { - for (auto &g_ref : grads_) - { - auto &g = g_ref.get(); - std::fill(g.data().begin(), g.data().end(), 0.0); - } - } - }; - - class Adam : public Optimizer - { - private: - double lr_; - std::vector> params_; - std::vector> grads_; - std::vector m_; // 一阶矩（梯度均值） - std::vector v_; // 二阶矩（梯度平方均值） - double beta1_ = 0.9; // 一阶矩衰减率 - double beta2_ = 0.999; // 二阶矩衰减率 - double eps_ = 1e-8; // 防止除零 - std::size_t t_ = 0; // 时间步计数器 - - public: - Adam(std::vector> params, - std::vector> grads, - double lr, - double beta1 = 0.9, - double beta2 = 0.999, - double eps = 1e-8) - : lr_(lr), params_(std::move(params)), grads_(std::move(grads)), - beta1_(beta1), beta2_(beta2), eps_(eps), t_(0) - { - m_.reserve(params_.size()); - v_.reserve(params_.size()); - for (const auto &p_ref : params_) - { - const Matrix &p = p_ref.get(); - m_.emplace_back(p.rows(), p.cols(), 0.0); - v_.emplace_back(p.rows(), p.cols(), 0.0); - } - } - - Result step() override - { - if (params_.size() != grads_.size()) - { - return std::unexpected(Error{"params and grads must have same size"}); - } - - ++t_; - const double bc1 = 1.0 - std::pow(beta1_, static_cast(t_)); // 偏差修正分母 (一阶) - const double bc2 = 1.0 - std::pow(beta2_, static_cast(t_)); // 偏差修正分母 (二阶) - - for (std::size_t i = 0; i < params_.size(); ++i) - { - auto &p = params_[i].get(); - auto &g = grads_[i].get(); - auto &m = m_[i]; - auto &v = v_[i]; - - auto &p_vec = p.data(); - auto &g_vec = g.data(); - auto &m_vec = m.data(); - auto &v_vec = v.data(); - - auto zip_view = std::views::zip(p_vec, g_vec, m_vec, v_vec); - - SmartPolicy::for_each(zip_view.begin(), - zip_view.end(), - [this, bc1, bc2](auto &&tuple) - { - auto &p_val = std::get<0>(tuple); - auto &g_val = std::get<1>(tuple); - auto &m_val = std::get<2>(tuple); - auto &v_val = std::get<3>(tuple); - - // 更新一阶矩: m = β₁·m + (1-β₁)·g - m_val = beta1_ * m_val + (1.0 - beta1_) * g_val; - // 更新二阶矩: v = β₂·v + (1-β₂)·g² - v_val = beta2_ * v_val + (1.0 - beta2_) * g_val * g_val; - - // 偏差修正 - const double m_hat = m_val / bc1; - const double v_hat = v_val / bc2; - - // 更新参数: θ = θ - lr · m̂ / (√v̂ + ε) - p_val = p_val - lr_ * m_hat / (std::sqrt(v_hat) + eps_); - }); - } - return {}; - } - - void zero_grad() override - { - for (auto &g_ref : grads_) - { - auto &g = g_ref.get(); - std::fill(g.data().begin(), g.data().end(), 0.0); - } - } - }; -} // namespace nn - -#endif // OPTIMIZER_HPP diff --git a/include/neuralnet.cpp/tensor.hpp b/include/neuralnet.cpp/tensor.hpp new file mode 100644 index 00000000..c60ab3d5 --- /dev/null +++ b/include/neuralnet.cpp/tensor.hpp @@ -0,0 +1,210 @@ +#ifndef NN_TENSOR_HPP +#define NN_TENSOR_HPP + +// ── tensor.hpp — 统一张量类型 ───────────────────────────────────────────── +// Tensor 是跨设备的统一数据容器：CPU 持有 Matrix，GPU 持有 GpuTensor。 +// Layer 和 ComputeEngine 只操作 Tensor，不关心底层存储设备。 +// +// 设计要点： +// - 使用 shared_ptr 内部持有存储，拷贝廉价（零拷贝传递） +// - CPU/GPU 存储互斥：device() 决定哪个指针有效 +// - 跨设备传输由 ComputeEngine 负责，Tensor 本身不主动迁移 +// ───────────────────────────────────────────────────────────────────────── + +#include +#include +#include + +#include "config.hpp" +#include "core_errors.hpp" +#include "algebra_matrix.hpp" + +#ifdef NN_HAS_VULKAN +#include "backend/vk_backend.hpp" +#endif + +#ifdef NN_HAS_CUDA +#include "backend/cuda_backend.hpp" +#endif + +namespace nn +{ + +// ── 计算设备 ────────────────────────────────────────────────────────────── +enum class Device : uint8_t +{ + CPU, + GPU, +}; + +// ══════════════════════════════════════════════════════════════════════════ +// Tensor — 统一跨设备张量 +// ══════════════════════════════════════════════════════════════════════════ +class Tensor +{ +private: + Device device_ = Device::CPU; + std::size_t rows_ = 0; + std::size_t cols_ = 0; + + // CPU 存储（device_ == CPU 时有效） + std::shared_ptr cpu_data_; + +#ifdef NN_HAS_VULKAN + // GPU 存储（device_ == GPU 时有效） + std::shared_ptr gpu_data_; +#endif + +#ifdef NN_HAS_CUDA + // CUDA 存储（device_ == GPU 时有效） + std::shared_ptr cuda_data_; +#endif + +public: + Tensor() = default; + + // ── CPU 构造 ────────────────────────────────────────────────────────── + explicit Tensor(std::shared_ptr m) + : device_(Device::CPU), rows_(m->rows()), cols_(m->cols()), cpu_data_(std::move(m)) {} + + // 从 Matrix 构造（共享所有权） + static Tensor from_matrix(Matrix m) + { + return Tensor(std::make_shared(std::move(m))); + } + + // 创建 CPU 空张量 + static Tensor cpu(std::size_t rows, std::size_t cols) + { + return Tensor(std::make_shared(rows, cols)); + } + +#ifdef NN_HAS_VULKAN + // ── GPU 构造 ────────────────────────────────────────────────────────── + explicit Tensor(std::shared_ptr t) + : device_(Device::GPU), rows_(t->rows()), cols_(t->cols()), gpu_data_(std::move(t)) {} + + static Tensor from_gpu(GpuTensor t) + { + return Tensor(std::make_shared(std::move(t))); + } +#endif + +#ifdef NN_HAS_CUDA + // ── CUDA 构造 ───────────────────────────────────────────────────────── + explicit Tensor(std::shared_ptr t) + : device_(Device::GPU), rows_(t->rows()), cols_(t->cols()), cuda_data_(std::move(t)) {} + + static Tensor from_cuda(CudaTensor t) + { + return Tensor(std::make_shared(std::move(t))); + } +#endif + + // ── 访问器 ──────────────────────────────────────────────────────────── + [[nodiscard]] Device device() const noexcept { return device_; } + [[nodiscard]] std::size_t rows() const noexcept { return rows_; } + [[nodiscard]] std::size_t cols() const noexcept { return cols_; } + [[nodiscard]] std::size_t size() const noexcept { return rows_ * cols_; } + [[nodiscard]] bool is_cpu() const noexcept { return device_ == Device::CPU; } + [[nodiscard]] bool is_gpu() const noexcept { return device_ == Device::GPU; } + [[nodiscard]] bool valid() const noexcept + { + if (device_ == Device::CPU) + return static_cast(cpu_data_); +#ifdef NN_HAS_CUDA + if (cuda_data_) + return static_cast(cuda_data_); +#endif +#ifdef NN_HAS_VULKAN + return static_cast(gpu_data_); +#else + return false; +#endif + } + + // ── CPU 存储访问 ────────────────────────────────────────────────────── + [[nodiscard]] Matrix& cpu_matrix() + { + NN_ASSERT(device_ == Device::CPU && cpu_data_, "cpu_matrix() on non-CPU tensor"); + return *cpu_data_; + } + + [[nodiscard]] const Matrix& cpu_matrix() const + { + NN_ASSERT(device_ == Device::CPU && cpu_data_, "cpu_matrix() on non-CPU tensor"); + return *cpu_data_; + } + + [[nodiscard]] std::shared_ptr cpu_shared() const noexcept { return cpu_data_; } + +#ifdef NN_HAS_VULKAN + // ── GPU 存储访问 ────────────────────────────────────────────────────── + [[nodiscard]] GpuTensor& gpu_tensor() + { + NN_ASSERT(device_ == Device::GPU && gpu_data_, "gpu_tensor() on non-GPU tensor"); + return *gpu_data_; + } + + [[nodiscard]] const GpuTensor& gpu_tensor() const + { + NN_ASSERT(device_ == Device::GPU && gpu_data_, "gpu_tensor() on non-GPU tensor"); + return *gpu_data_; + } + + [[nodiscard]] std::shared_ptr gpu_shared() const noexcept { return gpu_data_; } +#endif + +#ifdef NN_HAS_CUDA + // ── CUDA 存储访问 ───────────────────────────────────────────────────── + [[nodiscard]] CudaTensor& cuda_tensor() + { + NN_ASSERT(device_ == Device::GPU && cuda_data_, "cuda_tensor() on non-CUDA tensor"); + return *cuda_data_; + } + + [[nodiscard]] const CudaTensor& cuda_tensor() const + { + NN_ASSERT(device_ == Device::GPU && cuda_data_, "cuda_tensor() on non-CUDA tensor"); + return *cuda_data_; + } + + [[nodiscard]] std::shared_ptr cuda_shared() const noexcept { return cuda_data_; } +#endif + + // ── 形状描述（调试用） ──────────────────────────────────────────────── + [[nodiscard]] std::string shape_str() const + { + return "(" + std::to_string(rows_) + "x" + std::to_string(cols_) + ")" + + (device_ == Device::CPU ? "[CPU]" : "[GPU]"); + } + + // ── 零拷贝 reshape（共享底层 buffer，仅改变形状元数据）──────────────── + // 后端无关：CPU/Vulkan/CUDA 均适用，避免上层代码直接访问 gpu_tensor()/cuda_tensor() + [[nodiscard]] Tensor reshape(std::size_t new_rows, std::size_t new_cols) const + { + NN_ASSERT(rows_ * cols_ == new_rows * new_cols, + "reshape: element count mismatch"); + Tensor t; + t.device_ = device_; + t.rows_ = new_rows; + t.cols_ = new_cols; + t.cpu_data_ = cpu_data_; +#ifdef NN_HAS_VULKAN + t.gpu_data_ = gpu_data_; +#endif +#ifdef NN_HAS_CUDA + t.cuda_data_ = cuda_data_; +#endif + return t; + } +}; + +// ── 非拥有型引用包装器（替代裸指针） ───────────────────────────────────── +// 使用 std::reference_wrapper 替代 T*，明确表达 "引用但不拥有" 语义。 +// 用于 Optimizer 持有的 params_/grads_ 和 Layer::parameters() 返回值。 +using TensorRef = std::reference_wrapper; + +} // namespace nn + +#endif // NN_TENSOR_HPP diff --git a/include/neuralnet.cpp/tokenizer.hpp b/include/neuralnet.cpp/tokenizer.hpp deleted file mode 100644 index 198b3152..00000000 --- a/include/neuralnet.cpp/tokenizer.hpp +++ /dev/null @@ -1,655 +0,0 @@ -#ifndef TOKENIZER_HPP -#define TOKENIZER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nn_config.hpp" - -namespace nn -{ - -// ═══════════════════════════════════════════════════════════════════════════ -// ByteZip v2.2 分词器 -// ═══════════════════════════════════════════════════════════════════════════ -// 基于频率统计 + 独立度去冗余 + 词缀保护 + V2 残差挖掘的子词分词器。 -// Header-only，零外部依赖。 -// -// 词表构成: -// ID 0~3 特殊 token (, , , ) -// ID 4~259 256 个单字节 -// ID 260+ V1 高频子词 + V2 超长短语 -// -// 训练: -// V1 — 预分词 → 子串频率+上下文统计 → 词缀保护 → 独立度过滤填充 -// V2 — 编码全文找残差 → ±8 字节上下文提取 → 长子串挖掘 -// 编码: 预分词 → 逐块贪心最长匹配 → Token ID 序列 -// JSON 格式兼容 Python train_ByteZip.py。 -// ═══════════════════════════════════════════════════════════════════════════ - -class ByteZipTokenizer -{ -public: - // ── 特殊 token ──────────────────────────────────────────────────── - static constexpr std::size_t BYTE_OFFSET = 4; - static constexpr std::size_t PAD_ID = 0; - static constexpr std::size_t UNK_ID = 1; - static constexpr std::size_t BOS_ID = 2; - static constexpr std::size_t EOS_ID = 3; - - // ── 训练默认参数 ────────────────────────────────────────────────── - static constexpr std::size_t DEFAULT_VOCAB_SIZE = 20000; - static constexpr std::size_t DEFAULT_V1_MAX_LEN = 16; - static constexpr std::size_t DEFAULT_V2_MAX_LEN = 24; - static constexpr std::uint32_t DEFAULT_MIN_FREQ = 2; - static constexpr double DEFAULT_SKIP_RATIO = 1.2; - static constexpr double DEFAULT_AFFIX_RATIO = 0.4; - static constexpr std::size_t DEFAULT_MAX_V2_BYTES = 300'000; - static constexpr std::size_t SMALL_VOCAB_THRESHOLD = 1500; - - // ── 日志回调（默认 cout，nullptr 静默） ──────────────────────────── - using LogFn = std::function; - - // ── 训练配置 ────────────────────────────────────────────────────── - struct Config - { - std::size_t vocab_size = DEFAULT_VOCAB_SIZE; - std::size_t v1_max_len = DEFAULT_V1_MAX_LEN; - std::size_t v2_max_len = DEFAULT_V2_MAX_LEN; - std::uint32_t min_freq = DEFAULT_MIN_FREQ; - double skip_ratio = DEFAULT_SKIP_RATIO; - double affix_protect_ratio = DEFAULT_AFFIX_RATIO; - std::size_t v2_reserve = 0; // 0 = 自动分配 - std::size_t max_v2_scan_bytes = DEFAULT_MAX_V2_BYTES; - LogFn log = nullptr; - }; - - // ══════════════════════════════════════════════════════════════════ - // 公开接口 - // ══════════════════════════════════════════════════════════════════ - - ByteZipTokenizer() = default; - - void train(const std::string &text) { train(text, Config{}); } - - // ── 两阶段训练（V1 + V2） ───────────────────────────────────────── - void train(const std::string &text, const Config &config) - { - auto log = config.log - ? config.log - : LogFn{[](std::string_view m) { std::cout << m << '\n'; }}; - - max_v2_scan_bytes_ = config.max_v2_scan_bytes; - - const auto v1_max_len = config.v1_max_len; - const auto v2_max_len = config.v2_max_len; - const auto min_freq = config.min_freq; - const auto skip_ratio = config.skip_ratio; - const auto affix_ratio = config.affix_protect_ratio; - - // ── 智能槽位分配 ───────────────────────────────────────────── - std::size_t v1_target, v2_reserve; - if (config.vocab_size <= SMALL_VOCAB_THRESHOLD) - { - v2_reserve = 0; - v1_target = config.vocab_size; - log("小词表模式 (<=1500): 跳过 V2，全部给 V1 (目标 " - + std::to_string(v1_target) + ")"); - } - else - { - v2_reserve = config.v2_reserve != 0 - ? config.v2_reserve - : std::min(std::size_t{1500}, - static_cast(config.vocab_size * 0.12)); - v1_target = config.vocab_size - v2_reserve; - if (v1_target < BYTE_OFFSET + 256) - { - v1_target = BYTE_OFFSET + 256; - v2_reserve = config.vocab_size - v1_target; - } - log("大词表模式: V1 目标 " + std::to_string(v1_target) - + ", V2 预留 " + std::to_string(v2_reserve)); - } - - // ══════════════════════════════════════════════════════════════ - // V1: 频率 + 上下文统计 → 词缀保护填充 - // ══════════════════════════════════════════════════════════════ - log("\n[V1] 统计..."); - auto chunks = pre_tokenize(text); - - std::unordered_map freq; - std::unordered_map> left_ctx, right_ctx; - freq.reserve(text.size() / 2); - - for (const auto &chunk : chunks) - { - const auto L = chunk.size(); - if (L < 2) continue; - for (std::size_t s = 0; s < L; ++s) - { - const auto max_e = std::min(s + v1_max_len, L); - if (s + 2 > max_e) break; - for (auto e = s + 2; e <= max_e; ++e) - { - auto sub = chunk.substr(s, e - s); - ++freq[sub]; - left_ctx[sub].insert(s > 0 - ? static_cast(static_cast(chunk[s - 1])) - : -1); - right_ctx[sub].insert(e < L - ? static_cast(static_cast(chunk[e])) - : -1); - } - } - } - log(" 不同子串数: " + std::to_string(freq.size())); - - // V1 排序 - struct Entry { std::string sub; std::uint32_t cnt; std::size_t ben; std::size_t len; }; - std::vector sorted; - sorted.reserve(freq.size()); - for (const auto &[s, c] : freq) - { - auto l = s.size(); - sorted.push_back({s, c, static_cast(c) * (l - 1), l}); - } - std::ranges::sort(sorted, [](const Entry &a, const Entry &b) - { return a.ben != b.ben ? a.ben > b.ben : a.len > b.len; }); - - // 初始化词表 - vocab_.clear(); - vocab_.resize(BYTE_OFFSET); - for (std::size_t i = 0; i < 256; ++i) - vocab_.emplace_back(1, static_cast(i)); - - // 词缀检测 + 独立度过滤 - std::unordered_set skip; - skip.reserve(sorted.size() / 4); - std::size_t added = 0; - - for (const auto &e : sorted) - { - if (vocab_.size() >= v1_target) break; - if (e.cnt < min_freq) break; - if (e.len <= 1 || skip.contains(e.sub)) continue; - - // 词缀判断：使用上下文多样性 - auto lit = left_ctx.find(e.sub); - auto rit = right_ctx.find(e.sub); - bool is_suffix = false, is_prefix = false; - if (lit != left_ctx.end() && rit != right_ctx.end()) - { - auto len_l = lit->second.size(); - auto len_r = rit->second.size(); - is_suffix = (len_l >= 3 && len_l > len_r * 2.0); - is_prefix = (len_r >= 3 && len_r > len_l * 2.0); - } - const auto threshold = (is_suffix || is_prefix) ? affix_ratio : skip_ratio; - - // 智能 skip - const auto sl = e.sub.size(); - for (std::size_t i = 0; i < sl; ++i) - for (std::size_t j = i + 2; j <= sl; ++j) - { - if (i == 0 && j == sl) continue; - auto child = e.sub.substr(i, j - i); - auto it = freq.find(child); - if (it != freq.end() && - it->second <= static_cast(e.cnt * threshold)) - skip.insert(std::move(child)); - } - - vocab_.push_back(e.sub); - ++added; - if (added % 1000 == 0) - log(" V1: 已添加 " + std::to_string(added) + " 个词条（总 " - + std::to_string(vocab_.size()) + "）"); - } - log("V1 完成: " + std::to_string(vocab_.size()) + " 词条（新增 " - + std::to_string(added) + "）"); - - // ══════════════════════════════════════════════════════════════ - // V2: 残差挖掘（仅当预留 > 0） - // ══════════════════════════════════════════════════════════════ - if (v2_reserve > 0) - { - log("\n[V2] 提取残差..."); - // 构建 V1 查找表 - LookupTable lookup_v1; - for (std::size_t tid = BYTE_OFFSET; tid < vocab_.size(); ++tid) - if (!vocab_[tid].empty() && vocab_[tid].size() <= v1_max_len) - lookup_v1[vocab_[tid]] = tid; - - // 编码全文找残差跨度 - auto data = text; // 保持 UTF-8 - auto spans = get_residual_spans(data, lookup_v1, v1_max_len); - std::size_t total_bytes = 0; - for (const auto &sp : spans) total_bytes += sp.size(); - log(" 残差跨度: " + std::to_string(spans.size()) + ", 总字节: " - + std::to_string(total_bytes)); - - if (!spans.empty()) - { - log(" 挖掘超长短语（最多 " + std::to_string(v2_reserve) + " 个）..."); - auto v2_words = build_v2_from_spans(spans, v2_max_len, v2_reserve); - std::unordered_set existing(vocab_.begin(), vocab_.end()); - for (const auto &w : v2_words) - { - if (vocab_.size() >= config.vocab_size) break; - if (!existing.contains(w)) - { - vocab_.push_back(w); - existing.insert(w); - } - } - log(" V2 新增 " + std::to_string(vocab_.size() - (BYTE_OFFSET + 256 + added)) - + " 个"); - } - } - else - { - log("\n[V2] 跳过（小词表或预留为 0）"); - } - - // ── 最终查找表 ─────────────────────────────────────────────── - log("\n最终词表: " + std::to_string(vocab_.size())); - max_subword_len_ = std::max(v1_max_len, v2_max_len); - build_lookup(); - } - - // ── 编码 ────────────────────────────────────────────────────────── - [[nodiscard]] std::vector encode(const std::string &text) const - { - std::vector all; - all.reserve(text.size() / 2); - for (const auto &chunk : pre_tokenize(text)) - { - auto ids = encode_bytes(chunk); - all.insert(all.end(), ids.begin(), ids.end()); - } - return all; - } - - // ── 解码 ────────────────────────────────────────────────────────── - [[nodiscard]] std::string decode(std::span ids) const - { - std::string raw; - raw.reserve(ids.size() * 2); - for (auto tid : ids) - raw += tid < vocab_.size() ? vocab_[tid] : std::string{"\xef\xbf\xbd"}; - return raw; - } - - // ── 保存 JSON ──────────────────────────────────────────────────── - [[nodiscard]] Result save(const std::string &path) const - { - std::ofstream ofs(path); - if (!ofs) return std::unexpected(Error{"Cannot write: " + path}); - - ofs << "{\n \"type\": \"freq_based_tokenizer\",\n \"vocab\": {\n"; - bool first = true; - for (std::size_t tid = BYTE_OFFSET; tid < vocab_.size(); ++tid) - { - if (vocab_[tid].empty()) continue; - if (!first) ofs << ",\n"; - first = false; - ofs << " \"" << std::dec << tid << "\": \""; - for (unsigned char b : vocab_[tid]) - ofs << std::hex << std::setw(2) << std::setfill('0') << static_cast(b); - ofs << "\"" << std::dec; - } - ofs << "\n },\n \"vocab_size\": " << std::dec << vocab_.size() - << ",\n \"byte_offset\": " << BYTE_OFFSET - << ",\n \"max_subword_len\": " << max_subword_len_ - << ",\n \"special_tokens\": {" - << "\n \"\": 0,\n \"\": 1," - << "\n \"\": 2,\n \"\": 3\n }\n}\n"; - return {}; - } - - // ── 加载 JSON ──────────────────────────────────────────────────── - [[nodiscard]] Result load(const std::string &path) - { - std::ifstream ifs(path); - if (!ifs) return std::unexpected(Error{"Cannot read: " + path}); - std::string content((std::istreambuf_iterator(ifs)), - std::istreambuf_iterator()); - - vocab_.clear(); - - if (auto p = content.find("\"vocab_size\""); p != std::string::npos) - if (auto c = content.find(':', p); c != std::string::npos) - { - ++c; while (c < content.size() && content[c] == ' ') ++c; - std::size_t v = 0; - if (auto [ptr, ec] = std::from_chars(content.data()+c, content.data()+content.size(), v); ec == std::errc{}) - vocab_.resize(v); - } - - auto vp = content.find("\"vocab\""); - if (vp == std::string::npos) - return std::unexpected(Error{"Invalid JSON: missing \"vocab\""}); - auto br = content.find('{', vp); - if (br == std::string::npos) - return std::unexpected(Error{"Invalid JSON: missing '{'"}); - - std::size_t pos = br + 1; - while (pos < content.size()) - { - while (pos < content.size() && (content[pos]==' '||content[pos]=='\n'|| - content[pos]=='\r'||content[pos]=='\t'||content[pos]==',')) ++pos; - if (pos >= content.size() || content[pos] == '}') break; - if (content[pos] != '"') { ++pos; continue; } - ++pos; - std::string key; - while (pos < content.size() && content[pos] != '"') key += content[pos++]; - ++pos; - while (pos < content.size() && (content[pos]==' '||content[pos]==':')) ++pos; - if (pos < content.size() && content[pos] == '"') - { - ++pos; - std::string hex; - while (pos < content.size() && content[pos] != '"') hex += content[pos++]; - ++pos; - std::size_t id = 0; - std::from_chars(key.data(), key.data()+key.size(), id); - std::string tok; - tok.reserve(hex.size()/2); - for (std::size_t i = 0; i+1 < hex.size(); i += 2) - { unsigned b = 0; std::from_chars(hex.data()+i, hex.data()+i+2, b, 16); tok += static_cast(b); } - if (id >= vocab_.size()) vocab_.resize(id+1); - vocab_[id] = std::move(tok); - } - } - - max_subword_len_ = DEFAULT_V1_MAX_LEN; - if (auto p = content.find("\"max_subword_len\""); p != std::string::npos) - if (auto c = content.find(':', p); c != std::string::npos) - { - ++c; while (c < content.size() && content[c] == ' ') ++c; - std::size_t v = 0; - std::from_chars(content.data()+c, content.data()+content.size(), v); - if (v > 0) max_subword_len_ = v; - } - - build_lookup(); - return {}; - } - - // ══════════════════════════════════════════════════════════════════ - // 访问器 - // ══════════════════════════════════════════════════════════════════ - - [[nodiscard]] constexpr std::size_t vocab_size() const noexcept { return vocab_.size(); } - [[nodiscard]] constexpr std::size_t max_subword_len() const noexcept { return max_subword_len_; } - [[nodiscard]] constexpr std::size_t byte_offset() const noexcept { return BYTE_OFFSET; } - [[nodiscard]] const std::vector &vocab() const noexcept { return vocab_; } - - [[nodiscard]] Result token_bytes(std::size_t id) const - { - if (id >= vocab_.size()) return std::unexpected(Error{"Token ID out of range"}); - return vocab_[id]; - } - - [[nodiscard]] std::string try_decode_token(std::size_t id) const - { - if (id >= vocab_.size()) return ""; - std::string r; - for (char c : vocab_[id]) - { - if (c >= 32 && c < 127) r += c; - else if (c == '\n') r += "\\n"; - else if (c == '\t') r += "\\t"; - else if (c == ' ') r += "␣"; - else { r += "\\x"; r += "0123456789abcdef"[(static_cast(c)>>4)&0xF]; - r += "0123456789abcdef"[ static_cast(c) &0xF]; } - } - return r; - } - - // ── 预分词（公开，供外部复用） ──────────────────────────────────── - [[nodiscard]] static std::vector pre_tokenize(std::string_view text) - { - std::vector chunks; - chunks.reserve(text.size() / 4); - std::size_t pos = 0; - while (pos < text.size()) - { - if (auto n = try_contraction(text, pos); n > 0) - { chunks.emplace_back(text.substr(pos, n)); pos += n; continue; } - - const char c = text[pos]; - - if (is_alpha(c) || (c==' ' && pos+1 vocab_; - std::size_t max_subword_len_ = DEFAULT_V1_MAX_LEN; - - // 透明哈希查找表（编码热路径） - struct TransparentHash { - using is_transparent = void; - [[nodiscard]] std::size_t operator()(std::string_view s) const noexcept { return std::hash{}(s); } - [[nodiscard]] std::size_t operator()(const std::string &s) const noexcept { return std::hash{}(s); } - }; - struct TransparentEqual { - using is_transparent = void; - [[nodiscard]] bool operator()(std::string_view a, std::string_view b) const noexcept { return a==b; } - }; - using LookupTable = std::unordered_map; - LookupTable lookup_; - - // ── 字符分类 ────────────────────────────────────────────────────── - [[nodiscard]] static constexpr bool is_alpha(char c) noexcept { return (c>='a'&&c<='z')||(c>='A'&&c<='Z'); } - [[nodiscard]] static constexpr bool is_digit(char c) noexcept { return c>='0'&&c<='9'; } - [[nodiscard]] static constexpr bool is_whitespace(char c) noexcept { return c==' '||c=='\t'||c=='\n'||c=='\r'||c=='\f'||c=='\v'; } - [[nodiscard]] static constexpr bool is_word_char(char c) noexcept { return is_alpha(c)||is_digit(c)||c=='_'; } - - [[nodiscard]] static constexpr std::size_t try_contraction(std::string_view t, std::size_t p) noexcept - { - if (p>=t.size()||t[p]!='\'') return 0; - auto r = t.size()-p; if (r<2) return 0; - auto n1 = t[p+1]; - if (n1=='s'||n1=='t'||n1=='m'||n1=='d') return 2; - if (r>=3) { auto n2=t[p+2]; - if ((n1=='r'&&n2=='e')||(n1=='v'&&n2=='e')||(n1=='l'&&n2=='l')) return 3; } - return 0; - } - - // ── 编码（使用指定查找表，返回位置） ────────────────────────────── - struct EncodedChunk { std::vector ids; std::vector positions; }; - - [[nodiscard]] EncodedChunk encode_with_positions( - std::string_view data, const LookupTable &lut, std::size_t max_len) const - { - EncodedChunk result; - result.ids.reserve(data.size()); - result.positions.reserve(data.size()); - std::size_t pos = 0; - while (pos < data.size()) - { - result.positions.push_back(pos); - auto best_id = static_cast(static_cast(data[pos])) + BYTE_OFFSET; - auto best_len = std::size_t{1}; - for (auto l = std::min(max_len, data.size() - pos); l > 1; --l) - if (auto it = lut.find(data.substr(pos, l)); it != lut.end()) - { best_id = it->second; best_len = l; break; } - result.ids.push_back(best_id); - pos += best_len; - } - return result; - } - - // ── 编码（内部，用于 encode()） ─────────────────────────────────── - [[nodiscard]] std::vector encode_bytes(std::string_view data) const - { - std::vector ids; - ids.reserve(data.size()); - std::size_t pos = 0; - while (pos < data.size()) - { - auto best_id = static_cast(static_cast(data[pos])) + BYTE_OFFSET; - auto best_len = std::size_t{1}; - for (auto l = std::min(max_subword_len_, data.size()-pos); l > 1; --l) - if (auto it = lookup_.find(data.substr(pos, l)); it != lookup_.end()) - { best_id = it->second; best_len = l; break; } - ids.push_back(best_id); - pos += best_len; - } - return ids; - } - - // ── V2: 提取残差跨度 ────────────────────────────────────────────── - // 用 V1 查找表编码全文，找到单字节运行段，提取 ±8 上下文 - [[nodiscard]] std::vector get_residual_spans( - const std::string &text, const LookupTable &lut, std::size_t max_len) const - { - auto enc = encode_with_positions(text, lut, max_len); - const auto n_tokens = enc.ids.size(); - const auto data_size = text.size(); - - std::vector spans; - std::size_t i = 0; - while (i < n_tokens) - { - if (enc.ids[i] < BYTE_OFFSET + 256) - { - auto start_pos = enc.positions[i]; - auto j = i; - while (j < n_tokens && enc.ids[j] < BYTE_OFFSET + 256) - ++j; - auto end_pos = (j < n_tokens) ? enc.positions[j] : data_size; - auto ext_start = (start_pos >= 8) ? start_pos - 8 : 0; - auto ext_end = std::min(data_size, end_pos + 8); - spans.emplace_back(text.substr(ext_start, ext_end - ext_start)); - i = j; - } - else - { - ++i; - } - } - return spans; - } - - // ── V2: 从残差跨度挖掘超长短语 ──────────────────────────────────── - [[nodiscard]] std::vector build_v2_from_spans( - std::vector &spans, std::size_t max_len, - std::size_t max_items) const - { - std::size_t total_bytes = 0; - for (const auto &s : spans) total_bytes += s.size(); - if (total_bytes == 0 || max_items == 0) - return {}; - - // 采样：如果超过上限，随机采样 - if (total_bytes > max_v2_scan_bytes_) - { - // 简单采样：打乱后截取 - std::ranges::shuffle(spans, rng_); - std::string sampled; - sampled.reserve(max_v2_scan_bytes_); - for (const auto &sp : spans) - { - if (sampled.size() >= max_v2_scan_bytes_) break; - auto remain = max_v2_scan_bytes_ - sampled.size(); - sampled.append(sp, 0, remain); - } - spans = {std::move(sampled)}; - } - - // 统计子串频率（仅长度 16..max_len） - std::unordered_map freq; - for (const auto &span : spans) - { - const auto L = span.size(); - if (L < 16) continue; - for (std::size_t s = 0; s < L; ++s) - { - const auto max_e = std::min(s + max_len, L); - if (s + 16 > max_e) break; - for (auto e = s + 16; e <= max_e; ++e) - ++freq[span.substr(s, e - s)]; - } - } - - // 按压缩收益排序，过滤 freq < 2 - struct V2Entry { std::string sub; std::uint32_t cnt; std::size_t ben; }; - std::vector sorted; - sorted.reserve(freq.size()); - for (const auto &[s, c] : freq) - { - if (c < 2) continue; - sorted.push_back({s, c, static_cast(c) * (s.size() - 1)}); - } - std::ranges::sort(sorted, [](const V2Entry &a, const V2Entry &b) - { return a.ben > b.ben; }); - - std::vector result; - result.reserve(std::min(max_items, sorted.size())); - for (auto &e : sorted) - { - if (result.size() >= max_items) break; - result.push_back(std::move(e.sub)); - } - return result; - } - - // ── 构建查找表 ──────────────────────────────────────────────────── - void build_lookup() - { - lookup_.clear(); - lookup_.reserve(vocab_.size() - BYTE_OFFSET); - for (std::size_t i = BYTE_OFFSET; i < vocab_.size(); ++i) - if (!vocab_[i].empty()) lookup_[vocab_[i]] = i; - } - - // ── 训练用 RNG ──────────────────────────────────────────────────── - inline static thread_local std::mt19937_64 rng_{std::random_device{}()}; - - // ── V2 采样上限（从 config 传入） ────────────────────────────────── - std::size_t max_v2_scan_bytes_ = DEFAULT_MAX_V2_BYTES; -}; - -} // namespace nn - -#endif // TOKENIZER_HPP diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..62d463d4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,49 @@ +aiohappyeyeballs==2.7.1 +aiohttp==3.14.2 +aiosignal==1.4.0 +anyio==4.14.2 +async-timeout==5.0.1 +attrs==26.1.0 +certifi==2026.7.22 +charset-normalizer==3.4.9 +click==8.4.2 +colorama==0.4.6 +datasets==5.0.0 +dill==0.4.1 +exceptiongroup==1.3.1 +filelock==3.32.0 +frozenlist==1.8.0 +fsspec==2026.4.0 +h11==0.16.0 +hf-xet==1.5.2 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.24.0 +idna==3.18 +Jinja2==3.1.6 +MarkupSafe==3.0.3 +mpmath==1.3.0 +multidict==6.7.1 +multiprocess==0.70.19 +networkx==3.4.2 +numpy==2.2.6 +packaging==26.2 +pandas==2.3.3 +pillow==12.3.0 +propcache==0.5.2 +pyarrow==25.0.0 +python-dateutil==2.9.0.post0 +pytz==2026.2 +PyYAML==6.0.3 +requests==2.34.2 +six==1.17.0 +sympy==1.14.0 +tokenizers==0.23.1 +torch==2.13.0 +torchvision==0.28.0 +tqdm==4.69.0 +typing_extensions==4.16.0 +tzdata==2026.3 +urllib3==2.7.0 +xxhash==3.8.1 +yarl==1.24.5 diff --git a/scripts/benchmark_final.ps1 b/scripts/benchmark_final.ps1 new file mode 100644 index 00000000..ae8c0848 --- /dev/null +++ b/scripts/benchmark_final.ps1 @@ -0,0 +1,71 @@ +# MNIST Transformer 优化器对比 - 最终版 +$exe = ".\build\mnist_train.exe" +$common = @( + "--arch", "transformer", + "--epochs", "5", + "--batch-size", "128", + "--d-model", "32", + "--num-heads", "2", + "--num-layers", "1", + "--d-ff", "64", + "--patch-size", "7", + "--max-samples", "5000", + "--eval-samples", "100" +) + +$tests = @( + @{ Name = "SGD+Momentum"; Args = @("--optimizer", "sgd_momentum", "--lr", "0.01") } + @{ Name = "Adam"; Args = @("--optimizer", "adam", "--lr", "0.001") } + @{ Name = "AdamW"; Args = @("--optimizer", "adamw", "--lr", "0.001", "--weight-decay", "0.01") } + @{ Name = "Muon"; Args = @("--optimizer", "muon", "--lr", "0.001") } +) + +Write-Host "================================================================" -ForegroundColor Cyan +Write-Host " MNIST Transformer Optimizer Comparison" -ForegroundColor Cyan +Write-Host "================================================================" -ForegroundColor Cyan +Write-Host " Model: ViT d_model=32, heads=2, layers=1, d_ff=64" +Write-Host " Data: 5000 train, 5 epochs, batch=128" +Write-Host "================================================================" +Write-Host "" + +$results = @() + +foreach ($t in $tests) { + Write-Host "[$($t.Name)]" -ForegroundColor Yellow + $output = & $exe @common @($t.Args) 2>&1 + $lines = $output -join "`n" + + # 提取最终 loss 和准确率 + $lastEpoch = ($output | Where-Object { $_ -match "Epoch.*loss=" } | Select-Object -Last 1) + if ($lastEpoch -match "loss=([\d.]+).*train_acc=([\d.]+)%.*test_acc=([\d.]+)%") { + $loss = [double]$Matches[1] + $trainAcc = [double]$Matches[2] + $testAcc = [double]$Matches[3] + } + + Write-Host " Final: loss=$loss, train=$trainAcc%, test=$testAcc%" + + $results += [PSCustomObject]@{ + Name = $t.Name + Loss = $loss + TrainAcc = $trainAcc + TestAcc = $testAcc + } + Write-Host "" +} + +Write-Host "================================================================" -ForegroundColor Cyan +Write-Host " RESULTS SUMMARY" -ForegroundColor Cyan +Write-Host "================================================================" -ForegroundColor Cyan +Write-Host "" +Write-Host ("{0,-15} {1,8} {2,10} {3,10}" -f "Optimizer", "Loss", "Train Acc", "Test Acc") +Write-Host ("{0,-15} {1,8} {2,10} {3,10}" -f "---------", "----", "---------", "--------") + +$results | Sort-Object TestAcc -Descending | ForEach-Object { + $color = if ($_.TestAcc -ge 60) { "Green" } elseif ($_.TestAcc -ge 40) { "Yellow" } else { "Red" } + Write-Host ("{0,-15} {1,8:F4} {2,9:F1}% {3,9:F1}%" -f $_.Name, $_.Loss, $_.TrainAcc, $_.TestAcc) -ForegroundColor $color +} + +Write-Host "" +Write-Host "Note: Each optimizer uses its own recommended lr." -ForegroundColor Gray +Write-Host " Muon needs lower lr than paper default (0.02) for this small model." -ForegroundColor Gray diff --git a/scripts/benchmark_optimizers.ps1 b/scripts/benchmark_optimizers.ps1 new file mode 100644 index 00000000..53a70c6f --- /dev/null +++ b/scripts/benchmark_optimizers.ps1 @@ -0,0 +1,86 @@ +# MNIST Transformer 优化器对比实验 +# 对比 SGD, SGD+Momentum, Adam, AdamW 的训练效果 + +$ErrorActionPreference = "Continue" + +$exe = ".\build\mnist_train.exe" +$arch = "transformer" +$epochs = 15 +$batchSize = 64 +$dModel = 64 +$numHeads = 4 +$numLayers = 2 +$dFf = 128 +$patchSize = 7 + +# 优化器配置：名称 -> "参数列表" +$optimizers = @{ + "sgd" = @("--optimizer", "sgd", "--lr", "0.01") + "sgd_momentum" = @("--optimizer", "sgd_momentum", "--lr", "0.01") + "adam" = @("--optimizer", "adam", "--lr", "0.001") + "adamw" = @("--optimizer", "adamw", "--lr", "0.001", "--weight-decay", "0.01") + "muon" = @("--optimizer", "muon", "--lr", "0.02") +} + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host " MNIST Transformer 优化器对比实验" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "配置: arch=$arch, epochs=$epochs, batch_size=$batchSize" +Write-Host " d_model=$dModel, num_heads=$numHeads, num_layers=$numLayers, d_ff=$dFf, patch_size=$patchSize" +Write-Host "" + +$results = @{} + +foreach ($name in $optimizers.Keys) { + $optArgs = $optimizers[$name] + $logFile = "benchmark_${name}.log" + + Write-Host "----------------------------------------" -ForegroundColor Yellow + Write-Host "训练优化器: $name" -ForegroundColor Yellow + Write-Host "----------------------------------------" -ForegroundColor Yellow + + $args = @( + "--arch", $arch, + "--epochs", $epochs, + "--batch-size", $batchSize, + "--d-model", $dModel, + "--num-heads", $numHeads, + "--num-layers", $numLayers, + "--d-ff", $dFf, + "--patch-size", $patchSize + ) + $optArgs + + # 运行训练并保存输出 + $output = & $exe @args 2>&1 + $output | Out-File -FilePath $logFile -Encoding utf8 + + # 提取最终准确率 + $lastAccLine = $output | Where-Object { $_ -match "准确率.*?(\d+\.\d+)%" } | Select-Object -Last 1 + if ($lastAccLine -match "(\d+\.\d+)%") { + $acc = [double]$Matches[1] + $results[$name] = $acc + Write-Host "最终准确率: $acc%" -ForegroundColor Green + } else { + Write-Host "未能提取准确率" -ForegroundColor Red + $results[$name] = 0 + } + Write-Host "" +} + +Write-Host "" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host " 结果汇总" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan + +$results.GetEnumerator() | Sort-Object Value -Descending | ForEach-Object { + $bar = "█" * [math]::Floor($_.Value / 2) + Write-Host ("{0,-15} {1,6:F2}% {2}" -f $_.Key, $_.Value, $bar) -ForegroundColor $( + if ($_.Value -ge 98) { "Green" } + elseif ($_.Value -ge 95) { "Yellow" } + else { "Red" } + ) +} + +Write-Host "" +Write-Host "详细日志保存在 benchmark_.log" -ForegroundColor Gray diff --git a/scripts/benchmark_quick.ps1 b/scripts/benchmark_quick.ps1 new file mode 100644 index 00000000..ae2c134e --- /dev/null +++ b/scripts/benchmark_quick.ps1 @@ -0,0 +1,58 @@ +# MNIST Transformer 优化器快速对比 +$exe = ".\build\mnist_train.exe" +$common = @( + "--arch", "transformer", + "--epochs", "5", + "--batch-size", "128", + "--d-model", "32", + "--num-heads", "2", + "--num-layers", "1", + "--d-ff", "64", + "--patch-size", "7", + "--max-samples", "5000", + "--eval-samples", "100" +) + +$tests = @( + @{ Name = "SGD"; Args = @("--optimizer", "sgd", "--lr", "0.05") } + @{ Name = "SGD+Momentum"; Args = @("--optimizer", "sgd_momentum", "--lr", "0.01") } + @{ Name = "Adam"; Args = @("--optimizer", "adam", "--lr", "0.001") } + @{ Name = "AdamW"; Args = @("--optimizer", "adamw", "--lr", "0.001", "--weight-decay", "0.01") } + @{ Name = "Muon"; Args = @("--optimizer", "muon", "--lr", "0.02") } +) + +Write-Host "========================================================" -ForegroundColor Cyan +Write-Host " MNIST Transformer Optimizer Benchmark" -ForegroundColor Cyan +Write-Host "========================================================" -ForegroundColor Cyan +Write-Host " Model: d_model=32, heads=2, layers=1, d_ff=64" +Write-Host " Data: 5000 train, 100 eval | 5 epochs, batch=128" +Write-Host "========================================================" +Write-Host "" + +$results = @() + +foreach ($t in $tests) { + Write-Host "[$($t.Name)]" -ForegroundColor Yellow + $output = & $exe @common @($t.Args) 2>&1 + $lines = $output -join "`n" + + $match = [regex]::Match($lines, "test_acc=(\d+\.\d+)%") + $finalAcc = if ($match.Success) { [double]$match.Groups[1].Value } else { 0 } + + $epochLines = $output | Where-Object { $_ -match "Epoch" } + foreach ($line in $epochLines) { Write-Host " $line" } + + $results += [PSCustomObject]@{ Name = $t.Name; FinalAcc = $finalAcc } + Write-Host "" +} + +Write-Host "========================================================" -ForegroundColor Cyan +Write-Host " RESULTS" -ForegroundColor Cyan +Write-Host "========================================================" -ForegroundColor Cyan + +$results | Sort-Object FinalAcc -Descending | ForEach-Object { + $bar = [string]::new([char]0x2588, [math]::Floor($_.FinalAcc / 2)) + $color = if ($_.FinalAcc -ge 90) { "Green" } elseif ($_.FinalAcc -ge 70) { "Yellow" } else { "Red" } + Write-Host (" {0,-15} {1,6:F2}% {2}" -f $_.Name, $_.FinalAcc, $bar) -ForegroundColor $color +} +Write-Host "" diff --git a/scripts/convert_gpt_teacher.py b/scripts/convert_gpt_teacher.py new file mode 100644 index 00000000..0387221f --- /dev/null +++ b/scripts/convert_gpt_teacher.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +""" +将 GPT_teacher-3.37M-cn 数据集转换为 neuralnet.cpp 对话格式。 + +注意：训练端 (src/text_train.cpp) 当前为纯文本滑动窗口模式， +已移除对话 loss mask 训练。本脚本产出的 / 对话 +格式数据暂时不用于训练，仅保留转换工具备用。 + +源数据格式 (JSONL): + {"prompt": "...", "completion": "..."} + {"prompt": "...", "completion": "..."} + +目标格式 (每行一段对话): + promptcompletion +""" + +import json +import os +import sys +import urllib.request + +REPO_BASE = "https://raw.githubusercontent.com/helloworldtang/GPT_teacher-3.37M-cn/main/data" +FILES = { + "train": "train.jsonl", + "val": "val.jsonl", + "test": "test.jsonl", +} + + +def download_file(url: str, timeout: float = 60.0) -> str: + """下载文件内容并返回字符串（带超时，避免网络异常时无限阻塞）。""" + print(f" 下载: {url}") + with urllib.request.urlopen(url, timeout=timeout) as resp: + return resp.read().decode("utf-8") + + +def convert_jsonl_to_dialogue(jsonl_text: str) -> list[str]: + """将 JSONL 文本转换为 ...... 格式的行列表。""" + lines = [] + for raw_line in jsonl_text.strip().splitlines(): + raw_line = raw_line.strip() + if not raw_line: + continue + try: + obj = json.loads(raw_line) + except json.JSONDecodeError as e: + print(f" 跳过无效 JSON 行: {e}") + continue + prompt = obj.get("prompt", "").strip() + completion = obj.get("completion", "").strip() + if not prompt or not completion: + continue + lines.append(f"{prompt}{completion}") + return lines + + +def main(): + # 输出目录 + out_dir = os.path.join(os.path.dirname(__file__), "..", "datasets") + os.makedirs(out_dir, exist_ok=True) + + for split, filename in FILES.items(): + url = f"{REPO_BASE}/{filename}" + try: + jsonl_text = download_file(url) + except Exception as e: + print(f" 下载失败: {e}") + sys.exit(1) + + dialogue_lines = convert_jsonl_to_dialogue(jsonl_text) + out_path = os.path.join(out_dir, f"gpt_teacher_{split}.txt") + with open(out_path, "w", encoding="utf-8") as f: + for line in dialogue_lines: + f.write(line + "\n") + print(f" 保存: {out_path} ({len(dialogue_lines)} 条)") + + print("\n转换完成！使用示例:") + print(f" text_train.exe datasets/gpt_teacher_train.txt" + f" --test-file datasets/gpt_teacher_test.txt" + f" --vocab wordzip.json --epochs 20 --gpu") + + +if __name__ == "__main__": + main() diff --git a/csv_png.py b/scripts/csv_png.py similarity index 100% rename from csv_png.py rename to scripts/csv_png.py diff --git a/download_books.py b/scripts/download_books.py similarity index 90% rename from download_books.py rename to scripts/download_books.py index 4cf9a8f2..f64d99c0 100644 --- a/download_books.py +++ b/scripts/download_books.py @@ -36,14 +36,20 @@ def strip_gutenberg(text: str) -> str: text = text[m_start.end():m_end.start()] elif m_start: text = text[m_start.end():] - return text.strip() + # 去掉制表符/回车，去掉所有空行 + text = re.sub(r"[\t\r]+", " ", text) + text = re.sub(r"\n{2,}", "\n", text) + return text.strip().lower() def download_book(url: str, filename: str) -> str | None: filepath = os.path.join(RAW_DIR, filename) if os.path.exists(filepath): print(f" [跳过] {filename} 已存在") with open(filepath, "r", encoding="utf-8", errors="replace") as f: - return f.read() + text = f.read() + text = re.sub(r"[\t\r]+", " ", text) + text = re.sub(r"\n{2,}", "\n", text) + return text.strip().lower() print(f" [下载] {filename} ...") try: req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) diff --git a/scripts/download_chinese_datasets.py b/scripts/download_chinese_datasets.py new file mode 100644 index 00000000..85f06911 --- /dev/null +++ b/scripts/download_chinese_datasets.py @@ -0,0 +1,232 @@ +"""从免费公开数据集下载中文语料。 + +数据源： + 1. HuggingFace 上的中文数据集（wikipedia_zh、mc4_zh 等） + 2. 中文新闻语料 + 3. 中文百科语料 + +用法： + python scripts/download_chinese_datasets.py + +依赖：pip install datasets (可选，有 fallback) +""" + +import urllib.request +import json +import os +import re +import sys +import time + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +OUT_DIR = os.path.join(SCRIPT_DIR, "..", "datasets") +MIN_SENT_LEN = 8 +MAX_SENT_LEN = 200 +SENT_SPLIT = re.compile(r"[。！？；\n]") + + +def is_valid_chinese(text: str) -> bool: + text = text.strip() + if len(text) < MIN_SENT_LEN or len(text) > MAX_SENT_LEN: + return False + chinese = sum(1 for c in text if "\u4e00" <= c <= "\u9fff") + if chinese < len(text) * 0.35: + return False + if text.startswith(("#", "|", "=", "-", ">", "http")): + return False + return True + + +def split_sentences(text: str) -> list[str]: + text = text.replace("\r", "").replace("\n", " ").strip() + parts = SENT_SPLIT.split(text) + return [p.strip() for p in parts if p.strip()] + + +# ── 数据源 1：HuggingFace datasets 库 ──────────────────────────── +def download_via_datasets_lib(): + """使用 HuggingFace datasets 库下载中文语料。""" + print("\n📦 数据源 1: HuggingFace datasets 库") + try: + from datasets import load_dataset + except ImportError: + print(" ⚠ 未安装 datasets 库，跳过此数据源") + print(" 💡 安装方法: pip install datasets") + return [] + + # 尝试多个已知可用的中文数据集 + dataset_candidates = [ + ("tattsu/wikipedia_zh", "train"), + ("CIAO-LM/ChineseWebText", "train"), + ("donlee/wikitext-103-zh", "train"), + ] + + sentences = [] + for ds_name, split in dataset_candidates: + try: + print(f" 尝试数据集: {ds_name}...") + ds = load_dataset(ds_name, split=split, streaming=True) + count = 0 + for item in ds: + text = item.get("text", "") or item.get("content", "") or "" + if not text: + continue + for sent in split_sentences(text): + if is_valid_chinese(sent): + sentences.append(sent) + count += 1 + if count >= 15000: + break + if count % 3000 == 0 and count > 0: + print(f" 进度: {count} 条") + if count > 0: + print(f" ✓ {ds_name}: {count} 条") + break + except Exception as e: + print(f" ✗ {ds_name}: {e}") + continue + + print(f" ✅ HuggingFace: 共 {len(sentences)} 条") + return sentences + + +# ── 数据源 2：GitHub 中文语料仓库 ──────────────────────────────── +def download_github_chinese_corpus(): + """从 GitHub 上的公开中文语料仓库下载数据。""" + print("\n📰 数据源 2: GitHub 中文语料") + # 多个公开中文语料文件 + urls = [ + ("https://raw.githubusercontent.com/brightmart/nlp_chinese_corpus/master/data/zhidao_test.json", "jsonl"), + ("https://raw.githubusercontent.com/MorvanZhou/nlp-tutorial/master/data/1.text-classification/cn_sample.txt", "txt"), + ] + + sentences = [] + for url, fmt in urls: + try: + print(f" 下载: {url.split('/')[-1]}...") + req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read().decode("utf-8", errors="replace") + + if fmt == "jsonl": + for line in raw.strip().split("\n"): + line = line.strip() + if not line: + continue + try: + item = json.loads(line) + if isinstance(item, dict): + text = " ".join(str(v) for v in item.values() if isinstance(v, str)) + elif isinstance(item, str): + text = item + else: + continue + for sent in split_sentences(text): + if is_valid_chinese(sent): + sentences.append(sent) + except json.JSONDecodeError: + continue + else: # txt + for sent in split_sentences(raw): + if is_valid_chinese(sent): + sentences.append(sent) + + print(f" ✓ {url.split('/')[-1]}: 完成") + except Exception as e: + print(f" ✗ {url.split('/')[-1]}: {e}") + time.sleep(0.5) + + print(f" ✅ GitHub 语料: 共 {len(sentences)} 条") + return sentences + + +# ── 数据源 3：精选中文段落（硬编码高质量语料）──────────────────── +CURATED_PARAGRAPHS = [ + # 科学 + "科学研究表明，人类的大脑每天会产生约六万个想法，其中大部分是无意识的自动思维。正念冥想可以帮助人们觉察这些自动化的思维模式，从而获得更好的心理调控能力。", + "地球的磁场是由外核中液态铁的对流运动产生的。这个磁场像一个巨大的保护罩，阻挡了来自太阳的高能粒子，保护了地球上的生命。", + "光合作用是地球上最重要的化学反应之一。绿色植物利用阳光的能量将二氧化碳和水转化为葡萄糖和氧气，为整个食物链提供了能量基础。", + "人类基因组包含约三十一亿个碱基对，其中只有不到百分之二的序列编码蛋白质。科学家们正在研究这些非编码区域的功能。", + "量子纠缠现象被爱因斯坦称为幽灵般的超距作用。当两个粒子发生纠缠后，无论它们相距多远，测量其中一个粒子的状态会立即影响另一个。", + # 历史文化 + "活字印刷术的发明极大地降低了书籍的成本，使知识不再是少数人的特权。毕昇用胶泥制作的活字可以反复使用，这一创新比欧洲的古腾堡早了四百年。", + "茶马古道是古代中国西南地区最重要的贸易通道之一。马帮沿着这条道路将云南的茶叶运往西藏，同时带回藏区的马匹和药材。", + "中国古代的科举制度虽然有其局限性，但它打破了贵族对权力的垄断，为平民子弟提供了一条通过读书改变命运的通道。", + "敦煌壁画历经千年的风沙侵蚀，依然保持着绚丽的色彩。这些壁画不仅是艺术瑰宝，更是研究古代社会生活的珍贵资料。", + "中国的二十四节气是古人通过长期观察太阳运行规律而总结出来的时间体系。它不仅指导着农业生产，还深刻影响着中国人的饮食起居。", + # 日常生活 + "每天保持适量的运动对维持身体健康至关重要。世界卫生组织建议成年人每周至少进行一百五十分钟的中等强度有氧运动。", + "睡眠不足会影响大脑的清除系统，导致代谢废物在脑内积累。长期睡眠不足与阿尔茨海默病的发病风险增加有关。", + "色彩心理学研究发现，蓝色能让人感到平静和放松，红色则能激发兴奋和警觉。不同的颜色环境会影响人的情绪和行为表现。", + "人类的味觉可以分辨五种基本味道：酸、甜、苦、咸、鲜。味蕾上的受体蛋白能够识别不同的化学物质，将信号传递给大脑进行处理。", + "深呼吸能够激活副交感神经系统，降低心率和血压，帮助身体从压力状态中恢复过来。这也是为什么人们紧张时会不自觉地深呼吸。", + # 地理自然 + "热带雨林被称为地球之肺，它们不仅产生大量氧气，还是数百万种动植物的家园。然而，全球每年有大片热带雨林被砍伐。", + "珊瑚礁虽然只占海洋面积的不到百分之一，却养育了超过四分之一的海洋物种。海水温度上升导致的珊瑚白化正在威胁这些珍贵的生态系统。", + "中国的黄土高原是世界上最大的黄土沉积区。这些黄土是数百万年来从沙漠地区吹来的细沙沉积而成的。", + "北极光是太阳带电粒子与地球大气层中的原子碰撞时产生的自然现象。在高纬度地区，夜空中可以看到绿色、紫色和红色的光幕。", + "河流的三角洲是河流入海时因流速减慢、泥沙沉积而形成的扇形地貌。长江三角洲是中国最富庶的地区之一。", + # 数学与逻辑 + "质数在数学中扮演着重要角色。任何一个大于一的整数都可以唯一地分解为质数的乘积，这就是算术基本定理。", + "斐波那契数列在自然界中随处可见。向日葵的种子排列、松果的鳞片分布、贝壳的螺旋结构都遵循着这个数学规律。", + "概率论告诉我们，随机事件在大量重复实验中会呈现出稳定的统计规律。这就是大数定律的核心思想。", + "二进制是计算机的基础语言。所有的文字、图片、音乐在计算机中都是以零和一的组合形式存储和处理的。", + "博弈论研究的是理性决策者之间的策略互动。囚徒困境揭示了一个深刻的道理：个体的理性选择可能导致集体的非理性结果。", +] + + +def fetch_curated_paragraphs(): + """返回硬编码的精选语料。""" + print("\n📖 数据源 3: 精选中文段落") + sentences = [] + for para in CURATED_PARAGRAPHS: + for sent in split_sentences(para): + if is_valid_chinese(sent): + sentences.append(sent) + print(f" ✅ 精选段落: 共 {len(sentences)} 条") + return sentences + + +def main(): + os.makedirs(OUT_DIR, exist_ok=True) + all_sentences = [] + seen = set() + + # 1. HuggingFace + hf_sents = download_via_datasets_lib() + for s in hf_sents: + key = s[:40] + if key not in seen: + seen.add(key) + all_sentences.append(s) + + # 2. GitHub 中文语料 + gh_sents = download_github_chinese_corpus() + for s in gh_sents: + key = s[:40] + if key not in seen: + seen.add(key) + all_sentences.append(s) + + # 3. 精选语料 + curated = fetch_curated_paragraphs() + for s in curated: + key = s[:40] + if key not in seen: + seen.add(key) + all_sentences.append(s) + + # 写入文件 + out_path = os.path.join(OUT_DIR, "chinese_datasets_corpus.txt") + with open(out_path, "w", encoding="utf-8") as f: + for sent in all_sentences: + f.write(sent + "\n") + + print(f"\n{'='*60}") + print(f"✅ 总计采集: {len(all_sentences)} 条") + print(f"📝 输出文件: {out_path}") + print(f"{'='*60}") + + +if __name__ == "__main__": + main() diff --git a/scripts/download_chinese_wiki.py b/scripts/download_chinese_wiki.py new file mode 100644 index 00000000..c8e24ff0 --- /dev/null +++ b/scripts/download_chinese_wiki.py @@ -0,0 +1,144 @@ +"""从中文维基百科 REST API 抓取随机条目摘要，构建中文训练语料。 + +使用 /api/rest_v1/page/random/summary（限流比 query API 宽松）。 +并发请求，提取摘要并按句号分割成独立句子，每行一句流畅的中文书面语。 +依赖：仅标准库。 +""" +import urllib.request +import urllib.error +import json +import re +import time +import sys +import os +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed + +REST_URL = "https://zh.wikipedia.org/api/rest_v1/page/random/summary" +USER_AGENT = "neuralnet.cpp-corpus-builder/1.0 (educational research; local use)" +# 基于脚本所在目录的绝对路径 +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +OUT_FILE = os.path.join(SCRIPT_DIR, "datasets", "chinese_wiki_corpus.txt") + +TARGET_LINES = 5000 # 目标句子数 +CONCURRENCY = 6 # 并发请求数 +MAX_RETRIES = 3 # 单条目重试次数 +# 句子长度范围（字符） +MIN_LEN = 12 +MAX_LEN = 150 + +# 中文句末标点 +SENT_SPLIT = re.compile(r"[。！？；]") + + +def fetch_one_summary() -> str | None: + """请求一篇随机条目的摘要，返回纯文本 extract 或 None。""" + req = urllib.request.Request(REST_URL, headers={"User-Agent": USER_AGENT}) + for attempt in range(MAX_RETRIES): + try: + with urllib.request.urlopen(req, timeout=15) as resp: + data = json.loads(resp.read().decode("utf-8")) + return data.get("extract") or None + except urllib.error.HTTPError as e: + if e.code == 429: + # 限流：退避 + time.sleep(5 + attempt * 10) + continue + return None + except Exception: + time.sleep(1 + attempt * 2) + continue + return None + + +def extract_sentences(text: str) -> list[str]: + """把摘要文本按句末标点分割成独立句子，过滤长度。""" + if not text: + return [] + # 去掉换行，合并 + text = text.replace("\n", "").replace("\r", "").strip() + # 按句末标点分割 + parts = SENT_SPLIT.split(text) + sentences = [] + for p in parts: + p = p.strip() + if not p: + continue + # 跳过含太多数字/英文的（保留纯中文为主） + chinese = sum(1 for c in p if "\u4e00" <= c <= "\u9fff") + if chinese < len(p) * 0.5: + continue + if len(p) < MIN_LEN or len(p) > MAX_LEN: + continue + sentences.append(p) + return sentences + + +def worker() -> list[str]: + """单次任务：抓一篇并返回句子列表。""" + extract = fetch_one_summary() + return extract_sentences(extract) if extract else [] + + +def main(): + os.makedirs(os.path.dirname(OUT_FILE), exist_ok=True) + seen: set[str] = set() + seen_lock = threading.Lock() # 保护 seen 集合的 check-then-add + total = 0 + requests_made = 0 + start = time.time() + + with open(OUT_FILE, "w", encoding="utf-8", buffering=1) as f: + with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + # 批量提交任务 + pending = set() + batch_size = CONCURRENCY * 4 + while total < TARGET_LINES: + # 补充待办任务 + while len(pending) < batch_size: + pending.add(pool.submit(worker)) + # 取完成的 + done = set() + try: + completed = as_completed(pending, timeout=30) + except TimeoutError: + # 30 秒内没有任何请求完成：继续循环补充任务（单个请求 + # 含重试+退避可达 ~70s，不能因超时直接崩溃） + continue + for fut in completed: + done.add(fut) + requests_made += 1 + try: + sents = fut.result() + except Exception: + sents = [] + for s in sents: + key = s[:40] + with seen_lock: + if key in seen: + continue + seen.add(key) + f.write(s + "\n") + total += 1 + if total >= TARGET_LINES: + break + if total >= TARGET_LINES: + break + pending -= done + elapsed = time.time() - start + rate = total / elapsed if elapsed > 0 else 0 + print(f" 请求 {requests_made}, 收集 {total}/{TARGET_LINES} 句, " + f"{rate:.1f} 句/秒, {elapsed:.0f}s", flush=True) + if total >= TARGET_LINES: + break + # 礼貌间隔 + time.sleep(0.3) + + elapsed = time.time() - start + size = os.path.getsize(OUT_FILE) + print(f"\n完成! 共 {total} 句, {size:,} 字节, 耗时 {elapsed:.0f}s") + print(f"输出: {OUT_FILE}") + + +if __name__ == "__main__": + main() diff --git a/scripts/download_fineweb.py b/scripts/download_fineweb.py new file mode 100644 index 00000000..b270e1f9 --- /dev/null +++ b/scripts/download_fineweb.py @@ -0,0 +1,194 @@ +"""下载 HuggingFace FineWeb 数据集（英文）并转为项目预训练格式。 + +FineWeb 是 HuggingFaceFW 整理的英文网页预训练语料，本身就是英文 +（创建时已通过语言过滤），适合作为 GPT base 模型的预训练数据。 + +项目训练格式（参考 src/text_train.cpp，滑动窗口）： + - 每行 = 一个完整文档（长度不限），行内不能含换行符 + - 纯文本模式（无对话标记），所有位置参与 loss + - 训练端把每行编码为 [BOS]+tokens+[EOS] 后拼成连续 token 流， + 再按 seq_len 滑动切窗——因此超长文档无需截断，会被自动切成 + 多个窗口，短文档也会被窗口切分/拼接利用 + +本脚本以 streaming 模式遍历 FineWeb，逐条抽取 text 字段，清洗后写为 +单行，累计到目标字节数后停止。streaming 模式不会把整个数据集下载 +到本地，只会按需拉取 parquet 分片。 + +用法: + python scripts/download_fineweb.py + python scripts/download_fineweb.py --target_mb 200 + python scripts/download_fineweb.py --subset sample-10BT --target_mb 50 + +依赖: pip install datasets +""" +import argparse +import os +import re +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +OUT_DIR = os.path.join(SCRIPT_DIR, "datasets") + +# FineWeb 配置项： +# sample-10BT : 10B token 抽样集（推荐起点，体积小、覆盖广） +# sample-100BT : 100B token 抽样集 +# CC-MAIN-* : 单次 CC 抓取 +DEFAULT_SUBSET = "sample-10BT" +DEFAULT_TARGET_MB = 100 + +# 最短文档长度（字符数）：过短视为噪声丢弃 +# 超长文档不设上限——训练端滑动窗口会自动切分，截断反而破坏文档完整性 +MIN_DOC_LEN = 256 + +# 触发进度打印的间隔（字节数） +PROGRESS_INTERVAL_BYTES = 5 * 1024 * 1024 + + +# 简单的清洗：去 HTML 残留、合并空白、去掉换行（项目要求每行一个文档） +_HTML_TAG_RE = re.compile(r"<[^>]+>") +_MULTI_WS_RE = re.compile(r"\s+") + + +def clean_doc(text: str) -> str: + """清洗一个 FineWeb 文档：去 HTML 标签、合并空白、去换行。""" + if not text: + return "" + # 移除 HTML 标签（FineWeb 已基本清理过，保险起见再过一遍） + text = _HTML_TAG_RE.sub(" ", text) + # 把所有空白（含换行）合并为单个空格 + text = _MULTI_WS_RE.sub(" ", text).strip() + return text + + +def is_english_doc(text: str) -> bool: + """简单判断文档主体是否为英文。 + + FineWeb 在创建时已经按 language_score 过滤过英文样本，这里仅做 + 兜底：要求 ASCII 字母占比 >= 60%，避免偶发的非英文混入。 + """ + if not text: + return False + letters = sum(1 for c in text if c.isalpha()) + if letters == 0: + return False + ascii_letters = sum( + 1 for c in text if ("a" <= c.lower() <= "z") + ) + return ascii_letters / letters >= 0.6 + + +def fmt_size(n: int) -> str: + for unit in ("B", "KB", "MB", "GB"): + if n < 1024: + return f"{n:.1f} {unit}" + n /= 1024 + return f"{n:.1f} TB" + + +def download(subset: str, target_bytes: int, out_path: str) -> None: + try: + from datasets import load_dataset + except ImportError: + print("需要安装 datasets 库: pip install datasets", file=sys.stderr) + sys.exit(1) + + print(f"FineWeb subset : {subset}") + print(f"Target size : {fmt_size(target_bytes)}") + print(f"Output file : {out_path}") + print(f"Streaming ... : 启动（首次拉取分片可能稍慢）") + + ds = load_dataset( + "HuggingFaceFW/fineweb", + subset, + split="train", + streaming=True, + ) + + os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True) + + written = 0 + skipped_short = 0 + skipped_lang = 0 + bytes_written = 0 + last_progress = 0 + + # newline="\n" 强制 LF，避免在 Windows 上写成 CRLF + # （text_train.cpp 的 getline 已 trim \r，但保持 LF 更通用） + with open(out_path, "w", encoding="utf-8", newline="\n") as f: + for i, sample in enumerate(ds): + text = sample.get("text", "") or "" + cleaned = clean_doc(text) + + # 长度过滤：过短视为噪声；超长文档保留整篇（训练端滑动窗口切分） + if len(cleaned) < MIN_DOC_LEN: + skipped_short += 1 + continue + + # 语言过滤：仅保留英文 + if not is_english_doc(cleaned): + skipped_lang += 1 + continue + + line = cleaned + "\n" + f.write(line) + written += 1 + bytes_written += len(line.encode("utf-8")) + + # 进度打印 + if bytes_written - last_progress >= PROGRESS_INTERVAL_BYTES: + print( + f" docs={written:,} " + f"size={fmt_size(bytes_written)} " + f"skipped(short={skipped_short},lang={skipped_lang})" + ) + last_progress = bytes_written + + # 达到目标大小就停 + if bytes_written >= target_bytes: + break + + print() + print(f"完成:") + print(f" 写入文档数 : {written:,}") + print(f" 跳过(过短) : {skipped_short:,}") + print(f" 跳过(非英文): {skipped_lang:,}") + print(f" 实际大小 : {fmt_size(os.path.getsize(out_path))}") + print(f" 输出文件 : {out_path}") + + +def main(): + parser = argparse.ArgumentParser( + description="下载 FineWeb 英文预训练语料（项目格式：每行一个文档）" + ) + parser.add_argument( + "--subset", + default=DEFAULT_SUBSET, + help=f"FineWeb 子集名 (默认: {DEFAULT_SUBSET})", + ) + parser.add_argument( + "--target_mb", + type=int, + default=DEFAULT_TARGET_MB, + help=f"目标体积(MB), 达到即停止 (默认: {DEFAULT_TARGET_MB})", + ) + parser.add_argument( + "--output", + default=None, + help="输出文件路径 (默认: scripts/datasets/fineweb_en_mb.txt)", + ) + args = parser.parse_args() + + target_bytes = args.target_mb * 1024 * 1024 + + if args.output: + out_path = args.output + else: + out_path = os.path.join( + OUT_DIR, f"fineweb_en_{args.target_mb}mb.txt" + ) + + download(args.subset, target_bytes, out_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/download_openwebtext.py b/scripts/download_openwebtext.py new file mode 100644 index 00000000..c8cf7d34 --- /dev/null +++ b/scripts/download_openwebtext.py @@ -0,0 +1,224 @@ +"""下载 OpenWebText 数据集并转为项目预训练格式。 + +OpenWebText 是 Reddit 高赞链接抓取的网页文本，英文为主，适合作为 +GPT base 模型的预训练数据。 + +项目训练格式（参考 src/text_train.cpp，滑动窗口）： + - 每行 = 一个完整文档（长度不限），行内不能含换行符 + - 纯文本模式（无对话标记），所有位置参与 loss + - 训练端把每行编码为 [BOS]+tokens+[EOS] 后拼成连续 token 流， + 再按 seq_len 滑动切窗——因此超长文档无需截断，会被自动切成 + 多个窗口，短文档也会被窗口切分/拼接利用 + +本脚本以 streaming 模式遍历 OpenWebText，逐条抽取 text 字段，清洗后写为 +单行，累计到目标字节数后停止。streaming 模式不会把整个数据集下载 +到本地，只会按需拉取分片。 + +用法: + python scripts/download_openwebtext.py + python scripts/download_openwebtext.py --target_mb 200 + python scripts/download_openwebtext.py --min_len 256 + +依赖: pip install datasets +""" +import argparse +import os +import re +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +OUT_DIR = os.path.join(SCRIPT_DIR, "datasets") + +# OpenWebText 配置 +# 常用版本: "openwebtext" 或 "Skylion007/openwebtext" +DEFAULT_DATASET = "Skylion007/openwebtext" +DEFAULT_TARGET_MB = 100 + +# 最短文档长度（字符数）：过短视为噪声丢弃 +# 超长文档不设截断——训练端滑动窗口会自动切分；max_len 仅作防御性上限 +DEFAULT_MIN_LEN = 256 +DEFAULT_MAX_LEN = 100000 + +# 触发进度打印的间隔（字节数） +PROGRESS_INTERVAL_BYTES = 5 * 1024 * 1024 + + +# 简单的清洗：去 HTML 残留、合并空白、去掉换行（项目要求每行一个文档） +_HTML_TAG_RE = re.compile(r"<[^>]+>") +_MULTI_WS_RE = re.compile(r"\s+") + + +def clean_doc(text: str) -> str: + """清洗一个 OpenWebText 文档：去 HTML 标签、合并空白、去换行。""" + if not text: + return "" + # 移除 HTML 标签 + text = _HTML_TAG_RE.sub(" ", text) + # 把所有空白（含换行）合并为单个空格 + text = _MULTI_WS_RE.sub(" ", text).strip() + return text + + +def is_english_doc(text: str) -> bool: + """简单判断文档主体是否为英文。 + + OpenWebText 主要是英文，这里做兜底检查：要求 ASCII 字母占比 >= 60%。 + """ + if not text: + return False + letters = sum(1 for c in text if c.isalpha()) + if letters == 0: + return False + ascii_letters = sum( + 1 for c in text if ("a" <= c.lower() <= "z") + ) + return ascii_letters / letters >= 0.6 + + +def fmt_size(n: int) -> str: + for unit in ("B", "KB", "MB", "GB"): + if n < 1024: + return f"{n:.1f} {unit}" + n /= 1024 + return f"{n:.1f} TB" + + +def download(dataset_name: str, target_bytes: int, min_len: int, max_len: int, out_path: str) -> None: + try: + from datasets import load_dataset + except ImportError: + print("需要安装 datasets 库: pip install datasets", file=sys.stderr) + sys.exit(1) + + print(f"Dataset : {dataset_name}") + print(f"Target size : {fmt_size(target_bytes)}") + print(f"Length filter : {min_len} - {max_len} chars") + print(f"Output file : {out_path}") + print(f"Streaming ... : 启动（首次拉取分片可能稍慢）") + + # 尝试加载数据集 + # OpenWebText 通常只有 train split + try: + ds = load_dataset( + dataset_name, + split="train", + streaming=True, + ) + except Exception as e: + print(f"加载数据集失败: {e}", file=sys.stderr) + print("尝试使用 'openwebtext' 作为备选...", file=sys.stderr) + try: + ds = load_dataset( + "openwebtext", + split="train", + streaming=True, + ) + except Exception as e2: + print(f"备选数据集也失败: {e2}", file=sys.stderr) + sys.exit(1) + + os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True) + + written = 0 + skipped_short = 0 + skipped_long = 0 + skipped_lang = 0 + bytes_written = 0 + last_progress = 0 + + # newline="\n" 强制 LF，避免在 Windows 上写成 CRLF + with open(out_path, "w", encoding="utf-8", newline="\n") as f: + for i, sample in enumerate(ds): + text = sample.get("text", "") or "" + cleaned = clean_doc(text) + + # 长度过滤：过短视为噪声；超长文档保留整篇（训练端滑动窗口切分） + if len(cleaned) < min_len: + skipped_short += 1 + continue + if len(cleaned) > max_len: + # 仅防御极端异常（默认上限很大，正常文档不会触发） + skipped_long += 1 + continue + + # 语言过滤：仅保留英文 + if not is_english_doc(cleaned): + skipped_lang += 1 + continue + + line = cleaned + "\n" + f.write(line) + written += 1 + bytes_written += len(line.encode("utf-8")) + + # 进度打印 + if bytes_written - last_progress >= PROGRESS_INTERVAL_BYTES: + print( + f" docs={written:,} " + f"size={fmt_size(bytes_written)} " + f"skipped(short={skipped_short},long={skipped_long},lang={skipped_lang})" + ) + last_progress = bytes_written + + # 达到目标大小就停 + if bytes_written >= target_bytes: + break + + print() + print(f"完成:") + print(f" 写入文档数 : {written:,}") + print(f" 跳过(过短) : {skipped_short:,}") + print(f" 跳过(过长) : {skipped_long:,}") + print(f" 跳过(非英文): {skipped_lang:,}") + print(f" 实际大小 : {fmt_size(os.path.getsize(out_path))}") + print(f" 输出文件 : {out_path}") + + +def main(): + parser = argparse.ArgumentParser( + description="下载 OpenWebText 预训练语料（项目格式：每行一个文档）" + ) + parser.add_argument( + "--dataset", + default=DEFAULT_DATASET, + help=f"数据集名称 (默认: {DEFAULT_DATASET})", + ) + parser.add_argument( + "--target_mb", + type=int, + default=DEFAULT_TARGET_MB, + help=f"目标体积(MB), 达到即停止 (默认: {DEFAULT_TARGET_MB})", + ) + parser.add_argument( + "--min_len", + type=int, + default=DEFAULT_MIN_LEN, + help=f"最小字符数 (默认: {DEFAULT_MIN_LEN})", + ) + parser.add_argument( + "--max_len", + type=int, + default=DEFAULT_MAX_LEN, + help=f"最大字符数 (默认: {DEFAULT_MAX_LEN})", + ) + parser.add_argument( + "--output", + default=None, + help="输出文件路径 (默认: scripts/datasets/openwebtext_en_mb.txt)", + ) + args = parser.parse_args() + + target_bytes = args.target_mb * 1024 * 1024 + + if args.output: + out_path = args.output + else: + out_path = os.path.join( + OUT_DIR, f"openwebtext_en_{args.target_mb}mb.txt" + ) + + download(args.dataset, target_bytes, args.min_len, args.max_len, out_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/download_tinystories.py b/scripts/download_tinystories.py new file mode 100644 index 00000000..31428fbf --- /dev/null +++ b/scripts/download_tinystories.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""下载 TinyStories（英文小模型故事数据集）并转为项目预训练格式。 + +TinyStories (roneneldan/TinyStories) 专为 10M 级小模型设计： + - 语法规范、语义连贯、主题丰富的英文短篇故事 + - 每条 story 一个 text 字段（内部含段落换行） + - 保留大小写，非常适合"续写文本、语法连贯、上下文相关"的演示 + +项目训练格式（src/text_train.cpp 滑动窗口）： + - 每行 = 一个完整故事（故事内换行转为空格） + - 纯文本模式（无对话标记），所有位置参与 loss + - 训练端每行 [BOS]+tokens+[EOS] 后拼成连续 token 流，再按 seq_len 滑动切窗 + +用法: + python scripts/download_tinystories.py + python scripts/download_tinystories.py --target_mb 60 --max_stories 80000 + +依赖: pip install datasets +""" +import argparse +import os +import re +import sys +import time + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +OUT_DIR = os.path.join(SCRIPT_DIR, "datasets") + +DEFAULT_TARGET_MB = 60 # 目标语料体积（足够 10M 模型快速演示） +MIN_LEN = 60 # 过短故事丢弃（噪声） +MAX_LEN = 4000 # 过长故事截断（防止单行异常长） +PROGRESS_INTERVAL = 5 * 1024 * 1024 # 每 5MB 打印一次进度 + +_WS_RE = re.compile(r"\s+") + + +def clean(text: str) -> str: + """故事内换行/制表符 → 空格，压缩连续空白，去首尾空白。""" + text = text.replace("\r", " ") + return _WS_RE.sub(" ", text).strip() + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--target_mb", type=int, default=DEFAULT_TARGET_MB, + help="目标语料体积 (MB)，达到后停止") + ap.add_argument("--max_stories", type=int, default=0, + help="最大故事数（0 = 不限，按体积停止）") + ap.add_argument("--output", default=None, help="输出文件路径") + args = ap.parse_args() + + os.makedirs(OUT_DIR, exist_ok=True) + out_path = args.output or os.path.join(OUT_DIR, "tinystories.txt") + target_bytes = args.target_mb * 1024 * 1024 + + try: + from datasets import load_dataset + except ImportError: + print("[错误] 需要 datasets 库: pip install datasets", file=sys.stderr) + return 1 + + # streaming 模式：不整包下载，按需拉取 parquet 分片，达到目标体积即停 + print(f"[加载] roneneldan/TinyStories (train, streaming) ...") + ds = load_dataset("roneneldan/TinyStories", split="train", streaming=True) + + written = 0 + count = 0 + skipped = 0 + t0 = time.time() + + with open(out_path, "w", encoding="utf-8") as f: + it = iter(ds) + while True: + if written >= target_bytes: + break + if args.max_stories and count >= args.max_stories: + break + + # 网络不稳时单条拉取可能抛异常：容错重试 + try: + item = next(it) + except StopIteration: + break + except Exception as e: + skipped += 1 + if skipped % 20 == 0: + print(f" [重试] 拉取下一条失败 {e.__class__.__name__}: {e}") + time.sleep(0.5) + continue + + try: + text = clean(item["text"]) + except Exception: + skipped += 1 + continue + + if len(text) < MIN_LEN: + skipped += 1 + continue + if len(text) > MAX_LEN: + text = text[:MAX_LEN].rstrip() + + f.write(text + "\n") + written += len(text.encode("utf-8")) + 1 + count += 1 + + if written // PROGRESS_INTERVAL != (written - len(text.encode("utf-8")) - 1) // PROGRESS_INTERVAL: + el = time.time() - t0 + print(f" [进度] {written/1024/1024:.1f} MB / {count} 条故事 ({el:.0f}s)") + + el = time.time() - t0 + print(f"\n完成: {out_path}") + print(f" 故事数: {count} 跳过: {skipped}") + print(f" 体积: {written/1024/1024:.2f} MB 耗时: {el:.0f}s") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/expand_chinese_corpus.py b/scripts/expand_chinese_corpus.py new file mode 100644 index 00000000..961a6e7e --- /dev/null +++ b/scripts/expand_chinese_corpus.py @@ -0,0 +1,457 @@ +"""扩展中文训练语料 — 多源综合采集脚本 + +数据源： + 1. 中文维基百科 REST API（完整正文段落，非仅摘要） + 2. 中文古腾堡 / 公版书 + 3. 搜狗新闻语料（CC-BY 许可） + 4. 随机中文名言 / 成语 + +用法： + python scripts/expand_chinese_corpus.py [--target 30000] [--output datasets/llm_corpus_expanded.txt] + +依赖：仅标准库。 +""" + +import urllib.request +import urllib.error +import json +import re +import time +import sys +import os +import random +import html +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed + +# ── 路径 ────────────────────────────────────────────────────────────── +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +OUT_DIR = os.path.join(SCRIPT_DIR, "datasets") + +# ── 维基百科配置 ────────────────────────────────────────────────────── +WIKI_RANDOM_API = "https://zh.wikipedia.org/api/rest_v1/page/random/summary" +WIKI_CONTENT_API = "https://zh.wikipedia.org/api/rest_v1/page/mobile-sections/{}" +WIKI_USER_AGENT = "neuralnet.cpp-corpus/2.0 (educational research)" +WIKI_CONCURRENCY = 8 +WIKI_MAX_RETRIES = 3 + +# ── 古腾堡中文 ────────────────────────────────────────────────────── +# 中文古腾堡 / 公版书项目 +GUTENBERG_CN_BOOKS = [ + # 简单的纯中文文本，每行一段 + ("https://raw.githubusercontent.com/chinese-poetry/chinese-poetry/master/README.md", "poetry_readme.txt"), +] + +# ── 句子过滤 ────────────────────────────────────────────────────────── +MIN_SENT_LEN = 10 +MAX_SENT_LEN = 200 +SENT_SPLIT = re.compile(r"[。！？；\n]") +# 跳过含太多英文/数字的行（阈值：中文字符 < 总长度的 40%） +CHINESE_RATIO_THRESHOLD = 0.4 + +# ── 已有语料路径 ────────────────────────────────────────────────────── +EXISTING_CORPUS = os.path.join(OUT_DIR, "llm_corpus.txt") + + +def is_valid_chinese(text: str) -> bool: + """检查文本是否为合格的中文句子。""" + text = text.strip() + if len(text) < MIN_SENT_LEN or len(text) > MAX_SENT_LEN: + return False + chinese = sum(1 for c in text if "\u4e00" <= c <= "\u9fff") + if chinese < len(text) * CHINESE_RATIO_THRESHOLD: + return False + # 跳过标题、列表等 + if text.startswith(("#", "|", "=", "-", ">", "【", "[", "http")): + return False + return True + + +def split_sentences(text: str) -> list[str]: + """按句末标点分割文本为独立句子。""" + text = text.replace("\r", "").replace("\n", " ").strip() + parts = SENT_SPLIT.split(text) + return [p.strip() for p in parts if p.strip()] + + +def load_existing_keys(path: str) -> set[str]: + """加载已有语料的前40字符作为去重 key。""" + keys = set() + if os.path.exists(path): + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + keys.add(line[:40]) + return keys + + +# ── 数据源 1：中文维基百科（完整正文）────────────────────────────── +def wiki_random_summary() -> str | None: + """获取一个随机条目的摘要（最稳定的 API）。""" + import urllib.parse + req = urllib.request.Request(WIKI_RANDOM_API, headers={"User-Agent": WIKI_USER_AGENT}) + for attempt in range(WIKI_MAX_RETRIES): + try: + with urllib.request.urlopen(req, timeout=20) as resp: + data = json.loads(resp.read().decode("utf-8")) + extract = data.get("extract", "") + if extract: + return extract + return None + except urllib.error.HTTPError as e: + if e.code == 429: + time.sleep(5 + attempt * 10) + continue + return None + except Exception: + time.sleep(1 + attempt * 2) + continue + return None + + +def wiki_worker(seen: set[str]) -> list[str]: + """抓取一篇维基百科摘要，返回句子列表。""" + extract = wiki_random_summary() + if not extract: + return [] + sentences = [] + for sent in split_sentences(extract): + key = sent[:40] + if key in seen: + continue + if is_valid_chinese(sent): + seen.add(key) + sentences.append(sent) + return sentences + + +def fetch_wiki(target: int, seen: set[str]) -> list[str]: + """并发抓取维基百科摘要。""" + print(f"\n📖 数据源 1: 中文维基百科摘要 → 目标 {target} 条") + sentences = [] + start = time.time() + with ThreadPoolExecutor(max_workers=WIKI_CONCURRENCY) as pool: + futures = set() + while len(sentences) < target: + # 提交新任务 + while len(futures) < WIKI_CONCURRENCY * 2: + futures.add(pool.submit(wiki_worker, seen)) + # 收集已完成的 + done = [] + for fut in list(futures): + if fut.done(): + done.append(fut) + for fut in done: + futures.discard(fut) + try: + sents = fut.result() + sentences.extend(sents) + except Exception: + pass + elapsed = time.time() - start + rate = len(sentences) / elapsed if elapsed > 0 else 0 + print(f" 进度: {len(sentences)}/{target} 条 ({rate:.1f} 条/秒)", end="\r") + if elapsed > 600: # 10 分钟超时 + print(f"\n ⏱ 达到时间上限，已收集 {len(sentences)} 条") + break + # 避免忙轮询空转 CPU：无论是否有任务在跑都短睡后再检查 + time.sleep(0.2) + print(f"\n ✅ 维基百科: 共 {len(sentences)} 条") + return sentences[:target] + + +# ── 数据源 2：古腾堡中文公版书 ────────────────────────────────────── +def fetch_gutenberg_cn_books(seen: set[str]) -> list[str]: + """从古腾堡获取中文公版书。""" + print("\n📚 数据源 2: 古腾堡中文公版书") + # 已知的古腾堡中文书籍 ID + book_ids = [ + (8406, "道德经"), (3456, "论语"), (3455, "大学"), + (3454, "中庸"), (3453, "孟子"), (3452, "诗经"), + (23834, "西游记"), (23835, "三国演义"), + (9603, "水浒传"), (24131, "红楼梦"), + ] + + sentences = [] + for bid, name in book_ids: + url = f"https://www.gutenberg.org/cache/epub/{bid}/pg{bid}.txt" + try: + req = urllib.request.Request(url, headers={"User-Agent": WIKI_USER_AGENT}) + with urllib.request.urlopen(req, timeout=30) as resp: + text = resp.read().decode("utf-8", errors="replace") + # 清理 Gutenberg 头尾 + start_idx = text.find("*** START") + end_idx = text.find("*** END") + if start_idx != -1 and end_idx != -1: + text = text[start_idx:end_idx] + elif start_idx != -1: + text = text[start_idx:] + + for sent in split_sentences(text): + key = sent[:40] + if key in seen: + continue + if is_valid_chinese(sent): + seen.add(key) + sentences.append(sent) + print(f" ✓ {name}: {sum(1 for s in sentences if True)} 条") + except Exception as e: + print(f" ✗ {name}: {e}") + time.sleep(0.5) # 礼貌延迟 + + print(f" ✅ 古腾堡: 共 {len(sentences)} 条") + return sentences + + +# ── 数据源 3：精选主题文本生成（规则模板）────────────────────────── +KNOWLEDGE_CORPUS = [ + # 中医 / 文化常识 + "中医认为人体是一个有机的整体，五脏六腑相互关联、相互影响。阴阳平衡是健康的根本，一旦失衡就会产生各种疾病。", + "二十四节气是中国古代劳动人民长期经验的积累和智慧的结晶，它反映了季节的变化，指导着农业生产。", + "书法是中国特有的传统艺术，通过毛笔蘸墨在宣纸上书写汉字，讲究笔法、结构和章法。", + "围棋起源于中国，至今已有数千年的历史，被认为是世界上最复杂的棋类游戏之一。", + "中国的茶文化源远流长，从神农尝百草开始，茶叶就与中国人结下了不解之缘。", + "春节是中国最重要的传统节日，贴春联、放鞭炮、吃年夜饭是必不可少的习俗。", + "中秋节象征着团圆，赏月、吃月饼是这一天最重要的活动，寄托了人们对家人的思念。", + "端午节是为了纪念伟大的爱国诗人屈原，吃粽子、赛龙舟是这个节日的传统习俗。", + "京剧是中国的国粹，融合了唱、念、做、打四种艺术手段，角色分为生、旦、净、丑四大行当。", + "太极拳是中国传统武术的瑰宝，以柔克刚、以静制动，既是强身健体的运动，也是一种哲学思想的体现。", + # 科学知识 + "光速是宇宙中最快的速度，约为每秒三十万公里，任何有质量的物体都无法达到光速。", + "地球是太阳系中唯一已知存在液态水的行星，水是生命起源和延续的关键因素。", + "人体的骨骼系统由二百零六块骨头组成，它们不仅支撑身体，还保护内部器官。", + "DNA是生物体遗传信息的载体，由四种碱基组成，通过碱基配对规则实现遗传信息的复制和传递。", + "声音是一种机械波，需要介质才能传播，这就是为什么太空中无法听到声音。", + "化学反应的本质是旧化学键的断裂和新化学键的形成，在此过程中原子重新排列组合。", + "植物通过光合作用将阳光、水和二氧化碳转化为葡萄糖和氧气，为地球上的生命提供能量。", + "人类的大脑约有八百六十亿个神经元，它们通过突触相互连接，形成了复杂的神经网络。", + "水在零摄氏度结冰，在一百摄氏度沸腾，这是水的两个重要的物理性质。", + "太阳是一颗恒星，它的能量来源于内部的核聚变反应，将氢转化为氦。", + # 地理知识 + "长江是中国最长的河流，全长六千三百多公里，流经十一个省市，最终注入东海。", + "青藏高原被称为世界屋脊，是世界上最高的高原，平均海拔超过四千米。", + "黄河流经九个省区，由于携带大量泥沙，河床不断抬高，形成了地上河。", + "中国的地形西高东低，呈三级阶梯状分布，这种地势对气候和河流走向有重要影响。", + "海南岛是中国第二大岛，地处热带，全年温暖湿润，是著名的旅游胜地。", + "珠江是中国南方最大的河流，流经云南、贵州、广西、广东等省区。", + "泰山位于山东省，自古以来就被视为神山，历代帝王在此举行封禅大典。", + "桂林山水甲天下，漓江两岸的喀斯特地貌形成了独特的自然景观。", + "西湖位于浙江省杭州市，是中国最著名的湖泊之一，以断桥残雪、平湖秋月等景观闻名。", + "敦煌莫高窟是世界上现存规模最大的佛教艺术宝库，保存了大量珍贵的壁画和雕塑。", + # 历史常识 + "秦始皇统一六国后，统一了文字、货币和度量衡，修建了万里长城。", + "丝绸之路是古代连接中国与欧洲的重要贸易通道，促进了东西方经济文化的交流。", + "造纸术是中国古代四大发明之一，蔡伦改进了造纸工艺，使纸张更加实用和廉价。", + "活字印刷术由北宋毕昇发明，极大地推动了知识的传播和文化的繁荣。", + "指南针最初用于风水和占卜，后来被应用于航海，促进了大航海时代的发展。", + "火药的发明改变了战争的形态，也推动了采矿和工程建设的技术进步。", + "唐朝是中国历史上最繁荣昌盛的朝代之一，长安城是当时世界上最大的城市。", + "宋朝的经济和文化高度发展，出现了世界上最早的纸币交子。", + "明朝郑和七下西洋，是世界航海史上的壮举，比哥伦布发现美洲早了近一百年。", + "清朝的闭关锁国政策导致中国逐渐落后于世界，最终在鸦片战争中遭受屈辱。", + # 自然常识 + "蜜蜂通过舞蹈来传递花蜜的位置信息，这种独特的交流方式被称为蜂舞。", + "变色龙能够根据环境改变皮肤颜色，这不仅用于伪装，也用于调节体温和表达情绪。", + "北极熊的毛发实际上是透明的，皮肤是黑色的，黑色皮肤有助于吸收热量。", + "候鸟每年都会进行大规模的迁徙，它们依靠地球磁场、太阳位置和地标来导航。", + "珊瑚礁被称为海洋中的热带雨林，为大量海洋生物提供栖息地。", + "企鹅虽然不会飞，但它们是出色的游泳者，能够在水中快速灵活地移动。", + "大象是陆地上最大的动物，它们拥有极好的记忆力，能够记住水源的位置。", + "蝴蝶的翅膀上有无数细小的鳞片，这些鳞片的颜色和排列形成了美丽的图案。", + "蚂蚁能够搬运比自身重量重五十倍的物体，这使它们成为自然界中最强壮的昆虫之一。", + "鲸鱼是地球上最大的动物，蓝鲸的心脏大小和一辆小汽车差不多。", +] + + +def fetch_knowledge_corpus(seen: set[str]) -> list[str]: + """基于内置知识库生成语料。""" + print("\n🧠 数据源 3: 精选知识语料") + sentences = [] + for text in KNOWLEDGE_CORPUS: + text = text.strip() + if not text: + continue + # 也拆成句子 + for sent in split_sentences(text): + key = sent[:40] + if key not in seen and is_valid_chinese(sent): + seen.add(key) + sentences.append(sent) + # 随机打乱 + random.shuffle(sentences) + print(f" ✅ 知识语料: 共 {len(sentences)} 条") + return sentences + + +# ── 数据源 4：中文网络文学风格文本（模板生成）──────────────────── +def generate_narrative_text(seen: set[str], count: int = 200) -> list[str]: + """生成叙事风格的中文文本模板。""" + print("\n✍️ 数据源 4: 叙事风格文本") + + subjects = ["他", "她", "老人", "孩子", "少年", "旅人", "学者", "农夫", "渔夫", "工匠"] + actions = [ + "缓缓走上前去", "静静地站在那里", "抬头望向远方", "低头沉思了片刻", + "深吸一口气", "微微点了点头", "轻轻叹了口气", "慢慢地转过身来", + "快步走向前方", "停下脚步看了看", "伸出手轻轻触摸", "默默地注视着", + ] + scenes = [ + "远处的山峦在晨雾中若隐若现，宛如一幅水墨画。", + "夕阳西下，金色的余晖洒在古老的城墙之上。", + "春风吹过，桃花纷纷扬扬地飘落下来。", + "细雨绵绵，青石板路上泛起一层薄薄的水光。", + "秋叶飘零，铺满了蜿蜒的山间小路。", + "夜幕降临，繁星点点，月光如水般倾泻在大地上。", + "清晨的阳光透过窗帘的缝隙照进了房间。", + "远处传来悠扬的笛声，在山谷间回荡。", + "微风吹过稻田，掀起层层金色的波浪。", + "雪花纷纷扬扬地飘落，覆盖了整个村庄。", + ] + emotions = [ + "心中涌起一股莫名的感动", "脸上露出了久违的笑容", + "眼眶不禁有些湿润", "内心感到无比的平静", + "一种温暖的感觉涌上心头", "仿佛一切都变得美好起来", + "过去的回忆如潮水般涌来", "对未来的日子充满了期待", + ] + + sentences = [] + for _ in range(count): + subj = random.choice(subjects) + act = random.choice(actions) + scene = random.choice(scenes) + emo = random.choice(emotions) + + # 组合成段落 + templates = [ + f"{subj}{act}，{scene}{subj}{emo}。", + f"{scene}{subj}{act}，{emo}。", + f"{subj}{act}。{scene}此刻{subj}{emo}。", + f"{scene}{subj}{act}，{scene}{emo}。", + ] + for sent in split_sentences(random.choice(templates)): + key = sent[:40] + if key not in seen and is_valid_chinese(sent): + seen.add(key) + sentences.append(sent) + + random.shuffle(sentences) + sentences = sentences[:count] + print(f" ✅ 叙事文本: 共 {len(sentences)} 条") + return sentences + + +# ── 主函数 ───────────────────────────────────────────────────────────── +def main(): + parser = argparse.ArgumentParser(description="扩展中文训练语料") + parser.add_argument("--target", type=int, default=30000, + help="每个数据源的目标条数（默认 30000）") + parser.add_argument("--output", type=str, + default=os.path.join(OUT_DIR, "llm_corpus_expanded.txt"), + help="输出文件路径") + parser.add_argument("--skip-wiki", action="store_true", + help="跳过维基百科数据源") + parser.add_argument("--skip-gutenberg", action="store_true", + help="跳过古腾堡数据源") + parser.add_argument("--skip-knowledge", action="store_true", + help="跳过知识语料数据源") + parser.add_argument("--skip-narrative", action="store_true", + help="跳过叙事文本数据源") + parser.add_argument("--append", action="store_true", + help="追加到已有 llm_corpus.txt 而非新建文件") + args = parser.parse_args() + + print("=" * 60) + print("🔧 中文训练语料扩展工具 v2.0") + print("=" * 60) + + # 加载已有语料用于去重 + seen = load_existing_keys(EXISTING_CORPUS) + print(f"📋 已有语料去重 keys: {len(seen)} 条") + + all_sentences = [] + + # 1. 维基百科（主要来源） + if not args.skip_wiki: + wiki_target = min(args.target, 20000) # 维基百科最多 20000 + wiki_sents = fetch_wiki(wiki_target, seen) + all_sentences.extend(wiki_sents) + + # 2. 古腾堡中文公版书 + if not args.skip_gutenberg: + gb_sents = fetch_gutenberg_cn_books(seen) + all_sentences.extend(gb_sents) + + # 3. 精选知识语料 + if not args.skip_knowledge: + kn_sents = fetch_knowledge_corpus(seen) + all_sentences.extend(kn_sents) + + # 4. 叙事风格文本 + if not args.skip_narrative: + narrative_count = min(args.target // 5, 500) + na_sents = generate_narrative_text(seen, count=narrative_count) + all_sentences.extend(na_sents) + + # ── 输出 ──────────────────────────────────────────────────────── + print("\n" + "=" * 60) + print(f"📊 总计采集: {len(all_sentences)} 条新句子") + + if args.append: + out_path = EXISTING_CORPUS + mode = "a" + print(f"📝 追加模式: {out_path}") + else: + out_path = args.output + mode = "w" + print(f"📝 写入文件: {out_path}") + + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, mode, encoding="utf-8") as f: + for sent in all_sentences: + f.write(sent.strip() + "\n") + + # 也输出一个纯去重版 + dedup_path = os.path.join(OUT_DIR, "chinese_corpus_combined.txt") + all_keys = set() + deduped = [] + # 合并 llm_corpus.txt 和新语料 + combined_sources = [] + for src in [EXISTING_CORPUS, out_path]: + if os.path.exists(src): + with open(src, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + combined_sources.append(line) + # 加上已有的 charbpe 文件 + for jsf in ["charbpe_corpus.json", "charbpe_llm.json", "charbpe_mini.json"]: + jp = os.path.join(OUT_DIR, jsf) + if os.path.exists(jp): + try: + with open(jp, "r", encoding="utf-8") as f: + data = json.load(f) + for item in data: + if isinstance(item, str): + combined_sources.append(item) + elif isinstance(item, dict) and "text" in item: + combined_sources.append(item["text"]) + except Exception: + pass + + for line in combined_sources: + key = line[:40] + if key not in all_keys: + all_keys.add(key) + deduped.append(line) + + with open(dedup_path, "w", encoding="utf-8") as f: + for line in deduped: + f.write(line + "\n") + + print(f"\n✅ 去重合并完成: {dedup_path}") + print(f" 总计: {len(deduped)} 条唯一句子") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/extract_digits.py b/scripts/extract_digits.py similarity index 84% rename from extract_digits.py rename to scripts/extract_digits.py index 8b166353..2f5ae6ff 100644 --- a/extract_digits.py +++ b/scripts/extract_digits.py @@ -39,7 +39,13 @@ def extract_digits(test_csv_path, output_dir): # 打印统计信息 total = sum(label_counts.values()) print(f"提取完成！共 {total} 张图片") - for label in sorted(label_counts.keys(), key=int): + # 防御：标签非纯数字时退化为字符串排序，避免 ValueError + def _sort_key(lbl: str): + try: + return (0, int(lbl), "") + except ValueError: + return (1, 0, lbl) + for label in sorted(label_counts.keys(), key=_sort_key): print(f" 标签 {label}: {label_counts[label]} 张") diff --git a/scripts/gen_corpus_large.ps1 b/scripts/gen_corpus_large.ps1 new file mode 100644 index 00000000..f024ec5b --- /dev/null +++ b/scripts/gen_corpus_large.ps1 @@ -0,0 +1,154 @@ +# 用本地 llama-server 生成大量多样化中文语料，每行一段话 +$ErrorActionPreference = "Stop" +$ApiUrl = "http://localhost:18080/v1/chat/completions" +$ApiKey = "Ethan_2048" +$OutputFile = "d:\Codes\neuralnet.cpp\datasets\llm_corpus.txt" + +# 大量主题，覆盖各领域，确保语料多样 +$topics = @( + # 科技 + "人工智能", "机器学习", "深度学习", "神经网络", "自然语言处理", + "计算机视觉", "数据科学", "云计算", "大数据", "区块链", + "物联网", "量子计算", "5G技术", "边缘计算", "网络安全", + "软件工程", "编程语言", "操作系统", "数据库", "分布式系统", + # 自然 + "气候变化", "环境保护", "生物多样性", "森林生态", "海洋保护", + "可再生能源", "太阳能", "风能", "水污染", "空气污染", + "垃圾分类", "碳中和", "生态平衡", "自然灾害", "地震", + # 教育 + "在线教育", "终身学习", "批判性思维", "阅读习惯", "学习方法", + "教育公平", "素质教育", "STEM教育", "蒙台梭利", "翻转课堂", + # 健康 + "健康饮食", "运动健身", "心理健康", "睡眠质量", "压力管理", + "冥想", "瑜伽", "慢性病", "营养学", "中医养生", + # 文化 + "传统文化", "节日习俗", "书法艺术", "茶文化", "京剧", + "古诗词", "节日美食", "民间故事", "非物质文化遗产", "方言保护", + # 城市 + "城市规划", "交通拥堵", "公共交通", "智慧城市", "绿色建筑", + # 农业 + "现代农业", "有机农业", "粮食安全", "农药污染", "精准农业", + # 艺术 + "电影艺术", "音乐欣赏", "绘画流派", "雕塑艺术", "舞蹈", + "摄影艺术", "文学创作", "戏剧表演", "建筑设计", "工业设计", + # 体育 + "足球", "篮球", "乒乓球", "游泳", "马拉松", + "奥运会", "体育精神", "运动员训练", "体育产业", "电子竞技", + # 天文 + "宇宙探索", "黑洞", "火星探测", "月球计划", "空间站", + "望远镜", "行星科学", "银河系", "暗物质", "宇宙起源", + # 心理 + "情绪管理", "人际交往", "自我认知", "幸福感", "焦虑应对", + "抑郁症", "积极心理学", "认知偏差", "决策心理", "群体心理", + # 经济 + "市场经济", "通货膨胀", "股票投资", "创业精神", "消费心理", + "全球化", "供应链", "数字货币", "普惠金融", "经济周期", + # 社会 + "社交媒体", "网络文化", "信息过载", "数字鸿沟", "隐私保护", + "老龄化社会", "城市化", "人口流动", "社区治理", "志愿服务", + # 哲学 + "人生意义", "伦理道德", "存在主义", "东西方哲学", "美学思考", + "逻辑学", "认识论", "自由意志", "幸福哲学", "死亡哲学", + # 医学 + "基因编辑", "精准医疗", "疫苗研发", "抗生素", "心理健康", + "中医药", "临床试验", "流行病学", "免疫学", "衰老研究", + # 数学 + "数学之美", "概率论", "统计学", "几何学", "数论", + "密码学", "图论", "数学建模", "混沌理论", "分形几何", + # 物理 + "相对论", "量子力学", "粒子物理", "凝聚态物理", "热力学", + "光学", "声学", "电磁学", "纳米技术", "超导现象", + # 化学 + "化学反应", "有机合成", "材料科学", "高分子", "催化", + "电化学", "分析化学", "绿色化学", "晶体结构", "分子生物学", + # 生物 + "进化论", "基因学", "细胞生物学", "生态学", "动物行为", + "植物学", "微生物", "神经科学", "脑科学", "生物多样性" +) + +$samplesPerTopic = 8 # 每主题 8 段 +$totalLines = 0 +$batchSize = 8 +$batch = @() +$batchIdx = 0 + +Write-Host "生成中文语料到 $OutputFile ..." +Write-Host "主题数: $($topics.Count), 每主题 $samplesPerTopic 段, 预计 $($topics.Count * $samplesPerTopic) 行" + +"" | Out-File -FilePath $OutputFile -Encoding UTF8 + +for ($i = 0; $i -lt $topics.Count; $i++) { + $topic = $topics[$i] + for ($j = 0; $j -lt $samplesPerTopic; $j++) { + $variation = switch ($j % 4) { + 0 { "用一句完整的话阐述" } + 1 { "用两到三句话展开说明" } + 2 { "从一个具体例子切入描述" } + 3 { "用比喻或类比的方式说明" } + } + $prompt = "请就「$topic」这个话题，$variation。要求： +1. 直接输出内容，不要带标题、编号、markdown 符号 +2. 内容必须是简体中文，自然流畅的书面语 +3. 长度控制在 30-100 字之间 +4. 只输出这一段话，不要分多行，不要任何解释或前言" + + $batch += [PSCustomObject]@{ + Prompt = $prompt + Topic = $topic + Idx = $j + } + + if ($batch.Count -ge $batchSize -or ($i -eq $topics.Count - 1 -and $j -eq $samplesPerTopic - 1)) { + Write-Host " 批次 $batchIdx : $($batch.Count) 个请求..." + $batchIdx++ + + $jobs = $batch | ForEach-Object -Parallel { + $body = @{ + messages = @( + @{ role = "user"; content = $_.Prompt } + ) + temperature = 0.85 + max_tokens = 400 + stream = $false + chat_template_kwargs = @{ enable_thinking = $false } + } | ConvertTo-Json -Depth 5 + + try { + $resp = Invoke-RestMethod -Uri $using:ApiUrl ` + -Method Post ` + -Headers @{Authorization="Bearer $using:ApiKey"} ` + -ContentType "application/json; charset=utf-8" ` + -Body $body ` + -TimeoutSec 120 + $text = $resp.choices[0].message.content.Trim() + [PSCustomObject]@{ Topic = $_.Topic; Idx = $_.Idx; Text = $text; Ok = $true } + } catch { + [PSCustomObject]@{ Topic = $_.Topic; Idx = $_.Idx; Text = "ERROR: $($_.Exception.Message)"; Ok = $false } + } + } -ThrottleLimit $batchSize + + foreach ($r in $jobs) { + if ($r.Ok -and $r.Text.Length -gt 0) { + $line = $r.Text -replace "`r`n|`n|`r", " " + $line = $line.Trim() + $line = $line.Trim('"').Trim('「」').Trim() + # 去掉可能的思考标记 + $line = $line -replace "[\s\S]*?", "" + $line = $line.Trim() + if ($line.Length -ge 10) { + Add-Content -Path $OutputFile -Value $line -Encoding UTF8 + $totalLines++ + } + } else { + Write-Host " 失败: $($r.Topic)[$($r.Idx)] $($r.Text)" + } + } + + $batch = @() + } + } +} + +Write-Host "" +Write-Host "完成! 共生成 $totalLines 行" +Write-Host "输出文件: $OutputFile" diff --git a/scripts/gen_corpus_large.py b/scripts/gen_corpus_large.py new file mode 100644 index 00000000..b5039ad6 --- /dev/null +++ b/scripts/gen_corpus_large.py @@ -0,0 +1,386 @@ +"""用本地 llama-server (OpenAI 兼容接口) 生成多样化中文语料。 + +每行一段流畅的中文书面语，用于训练"说人话"的 GPT。 +依赖: openai>=1.0 +""" +import os +import random +import re +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +from openai import OpenAI + +# ── 配置 ────────────────────────────────────────────────── +BASE_URL = "http://localhost:18080/v1" + +API_KEY = os.environ.get("LLM_API_KEY", "") +if not API_KEY: + print("错误: 未设置 LLM_API_KEY 环境变量（禁止在代码库中硬编码密钥）", + file=sys.stderr) + sys.exit(1) +MODEL = "local-model" # llama-server 忽略具体值，但 openai 库要求必填 +OUTPUT = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "datasets", + "llm_corpus.txt", +) +CONCURRENCY = 8 +SAMPLES_PER_STYLE = 3 # 每个（主题×风格）组合生成 3 段 +MIN_LEN = 10 +MAX_LEN = 300 +# 追加模式：不覆盖已有语料 +APPEND = True + +# ── 写作风格维度 ──────────────────────────────────────────────── +WRITING_STYLES = [ + "像跟朋友聊天一样娓娓道来", + "用讲道理的方式认真解释", + "从一个亲身经历或见闻说起", + "用打比方或类比让读者秒懂", + "先抛出一个常见误解再纠正", + "用简洁干练的语言直接说明", + "带点幽默和调侃的语气", + "引用一句俗语或老话然后展开", +] + +# ── 中文句式结构 ────────────────────────────────────────────────── +SENTENCE_PATTERNS = [ + "因果关系（因为…所以…）", + "转折关系（虽然…但是…）", + "递进关系（不仅…而且…）", + "假设关系（如果…就…）", + "反问句（难道不是…吗？）", + "感叹句（真是…啊！）", + "对比句（和…相比，…更…）", + "总分结构（总的来说…具体来说…）", + "条件关系（只要…就…）", + "并列关系（一边…一边…）", +] + +# ── 生活常识主题（口语化、接地气）────────────────────────────── +TOPICS = [ + # ── 做饭与饮食 ── + "怎么做红烧肉才好吃", + "煮饺子什么时候加凉水", + "炒菜时油温怎么判断", + "大米怎么存放不容易生虫", + "隔夜菜到底能不能吃", + "微波炉热饭的正确方法", + "什么时候该放盐才对", + "切洋葱不流泪的小窍门", + "怎么挑选新鲜的西瓜", + "煮面条怎样才能不粘锅", + "蒸鸡蛋羹怎么做才嫩滑", + "豆腐怎么做才入味", + "泡发干货的正确方法", + "肉馅怎么调才好吃", + "醋除了调味还能干什么", + "厨房去油污的小妙招", + "怎样泡茶才好喝", + "喝牛奶的正确时间", + "空腹不能吃什么", + "水果到底饭前吃还是饭后吃", + # ── 出行与交通 ── + "下雨天开车要注意什么", + "坐长途汽车怎么不晕车", + "骑电动车安全注意事项", + "怎么坐地铁最方便", + "堵车的时候怎么办", + "自驾游出发前要检查什么", + "飞机上不能带什么东西", + "打车遇到绕路怎么办", + "走路快和走路慢哪个健康", + "等红灯的时候该做什么", + # ── 健康与养生 ── + "感冒了应该多喝水还是多睡觉", + "颈椎疼是怎么回事", + "每天走多少步最合适", + "熬夜对身体到底有多大伤害", + "眼睛干涩怎么办", + "嗓子疼吃什么好得快", + "腰酸背痛怎么缓解", + "失眠的人该怎么调整", + "体检报告怎么看", + "血压高的人要注意什么", + "老年人怎么锻炼才安全", + "小孩子发烧怎么处理", + "跑步前后要做什么", + "久坐对身体有什么危害", + "怎么保护牙齿才对", + "夏天怎么防止中暑", + "冬天手脚冰凉怎么办", + "头发掉得多是什么原因", + "吃什么东西能提高免疫力", + "心情不好的时候怎么调节", + # ── 家居与生活 ── + "新房子装修完多久能住", + "家里总是有异味怎么办", + "衣服上的各种污渍怎么洗", + "洗衣机怎么清洗才干净", + "空调多久清洗一次比较好", + "冰箱里什么东西不能放", + "厨房下水道堵了怎么通", + "墙上有霉斑怎么处理", + "木地板怎么保养", + "家里太干燥怎么办", + "WiFi信号不好怎么增强", + "手机充电的正确方式", + "电表怎么看用了多少电", + "水龙头滴水怎么修", + "暖气不热是什么原因", + "窗帘怎么选才合适", + "收纳整理有什么好方法", + "搬家打包有什么技巧", + "垃圾分类到底怎么分", + "快递包装盒怎么处理", + # ── 人际关系 ── + "怎么跟不熟的人找话题聊天", + "同事之间怎么相处", + "吵架之后怎么和好", + "怎么拒绝别人的请求", + "朋友借钱不还怎么办", + "遇到不讲理的人怎么办", + "怎么跟父母沟通", + "孩子不听话怎么教育", + "夫妻之间怎么保持新鲜感", + "怎么交到真正的朋友", + "被误解了怎么解释", + "怎么安慰伤心的人", + "领导批评你的时候怎么办", + "室友关系怎么处理", + "邻里之间怎么和睦相处", + # ── 花钱与理财 ── + "怎么存钱最划算", + "网购怎么避免被骗", + "超市购物有什么省钱技巧", + "租房子要注意什么", + "怎么判断东西值不值得买", + "月光族怎么开始理财", + "信用卡怎么用才不会被坑", + "打折促销的套路有哪些", + "买菜什么时候最便宜", + "怎么记账才能坚持下来", + "保险到底有没有必要买", + "公积金怎么用", + "交水电费有什么方便的方式", + "跳槽的时候怎么谈工资", + "怎么规划旅游预算", + # ── 四季与天气 ── + "春天穿什么衣服合适", + "夏天怎么防蚊子", + "秋天干燥皮肤怎么办", + "冬天取暖要注意安全", + "梅雨天衣服怎么晾干", + "回南天怎么除湿", + "雾霾天出门要戴什么", + "大风天出行要注意什么", + "下雪天路面滑怎么开车", + "台风来了要准备什么", + # ── 节日与习俗 ── + "过年为什么要贴春联", + "中秋节除了吃月饼还能干什么", + "端午节为什么要吃粽子", + "元宵节和汤圆有什么关系", + "清明节扫墓有什么讲究", + "重阳节为什么要登高", + "腊八节为什么要喝腊八粥", + "压岁钱有什么寓意", + "过年怎么拜年才得体", + "结婚随份子钱给多少合适", + # ── 养宠物 ── + "第一次养狗要准备什么", + "猫咪为什么喜欢磨爪子", + "怎么教小狗定点上厕所", + "鱼缸多久换一次水", + "鸟叫声突然变少了怎么回事", + "兔子能不能吃胡萝卜", + "遛狗为什么要牵绳", + "宠物掉毛怎么办", + "小猫打疫苗的时间表", + "养仓鼠需要注意什么", + # ── 育儿与教育 ── + "孩子几岁开始学说话算正常", + "怎么陪孩子玩才有意义", + "小孩不爱吃饭怎么办", + "孩子怕黑怎么引导", + "怎么给孩子讲睡前故事", + "孩子撒谎了怎么教育", + "小学生要不要上辅导班", + "怎么培养孩子的阅读习惯", + "青春期孩子叛逆怎么办", + "孩子沉迷手机怎么限制", + # ── 穿衣与打扮 ── + "白衣服发黄怎么洗白", + "皮鞋怎么保养才亮", + "不同场合怎么穿衣服", + "眼镜框怎么选才好看", + "怎么叠衣服最节省空间", + "围巾怎么搭配好看", + "牛仔裤怎么洗不褪色", + "运动鞋怎么清洗", + "正式场合穿什么不犯错", + "衣服起球了怎么处理", + # ── 工具与修理 ── + "扳手和钳子有什么区别", + "家里的工具箱要备哪些东西", + "水龙头漏水自己怎么修", + "灯泡坏了怎么换", + "马桶堵了怎么疏通", + "墙面裂缝怎么修补", + "门锁不好使怎么修", + "抽油烟机怎么拆洗", + "水管爆了怎么紧急处理", + "电闸跳闸了怎么恢复", + # ── 中文语法与表达 ── + "的地得有什么区别", + "为什么不能说\"我走了先\"", + "量词的使用有什么规律", + "怎么用\"虽然\"和\"但是\"造句", + "什么时候用\"的\"什么时候用\"地\"", + "一句话怎么加上合适的连接词", + "怎么把长句子写通顺", + "中文的\"了\"到底什么意思", + "怎么用\"把\"字句和\"被\"字句", + "为什么\"差不多\"和\"差很多\"意思相反", + "语气词\"呢\"\"啊\"\"吧\"怎么用", + "怎么用反问句加强语气", + "中文数字的读法规则", + "成语和四字词语有什么区别", + "怎么用\"既…又…\"\"不但…还…\"", + "书面语和口语的区别在哪里", + "怎么用\"如果\"\"要是\"\"假如\"表达假设", + "\"但是\"和\"可是\"有什么区别", + "中文里\"才\"和\"就\"怎么用", + "怎样写一段通顺的话", + # ── 常识与道理 ── + "为什么早上起来要喝水", + "饭后百步走活到九十九是真的吗", + "吃太辣的东西胃疼怎么办", + "为什么有的人晕车有的人不晕", + "打哈欠为什么会传染", + "为什么人老了头发会变白", + "热水和凉水哪个解渴", + "为什么睡觉会做梦", + "打嗝止不住怎么办", + "为什么有些人怕痒有些人不怕", + "蹲久了站起来为什么会头晕", + "红药水和碘伏有什么区别", + "蚊子为什么爱叮某些人", + "天冷鼻涕为什么会流", + "小孩和大人谁更容易感冒", +] + +# ── 客户端 ──────────────────────────────────────────────── +client = OpenAI(base_url=BASE_URL, api_key=API_KEY, timeout=120) + + +def make_prompt(topic: str, style_idx: int) -> str: + style = WRITING_STYLES[style_idx % len(WRITING_STYLES)] + return ( + f"请用生活化的语言描述「{topic}」，{style}。\n" + "要求：\n" + "1. 直接输出内容，不要带标题、编号、markdown 符号\n" + "2. 内容必须是简体中文，自然流畅\n" + "3. 长度控制在 30-150 字之间\n" + "4. 使用自然的中文句式（因果、转折、递进、反问等），不要用模板化的表达\n" + "5. 只输出这一段话，不要分多行，不要任何解释或前言" + ) + + +def clean_line(text: str) -> str: + if not text: + return "" + # 合并换行 + line = re.sub(r"[\r\n]+", " ", text).strip() + # 去引号包裹 + line = line.strip('"').strip("「」").strip() + # 去思考标记 + line = re.sub(r"[\s\S]*?", "", line).strip() + return line + + +def gen_one(topic: str, style_idx: int, sample_idx: int) -> str | None: + prompt = make_prompt(topic, style_idx) + try: + resp = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.85, + max_tokens=400, + stream=False, + extra_body={"chat_template_kwargs": {"enable_thinking": False}}, + ) + text = resp.choices[0].message.content or "" + return clean_line(text) + except Exception as e: + print(f" [失败] {topic}[{style_idx},{sample_idx}]: {e}", file=sys.stderr) + return None + + +def main(): + os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) + # 二维组合：每个主题 × 每种风格 × 重复次数 + tasks = [ + (t, si, ri) + for t in TOPICS + for si in range(len(WRITING_STYLES)) + for ri in range(SAMPLES_PER_STYLE) + ] + random.shuffle(tasks) # 随机打乱顺序，避免同一主题连续生成 + + # 追加模式：加载已有语料做去重 + seen: set[str] = set() + existing_lines = 0 + mode = "a" if APPEND and os.path.exists(OUTPUT) else "w" + if mode == "a": + with open(OUTPUT, "r", encoding="utf-8") as f: + for line in f: + seen.add(line[:40]) + existing_lines += 1 + print(f"追加模式: 已有 {existing_lines} 行, 将生成 {len(tasks)} 条新样本", flush=True) + else: + print(f"覆盖模式: 将生成 {len(tasks)} 条新样本", flush=True) + + total = 0 + skipped_dup = 0 + failed = 0 + start = time.time() + + with open(OUTPUT, mode, encoding="utf-8") as f: + with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + futures = {pool.submit(gen_one, t, si, ri): (t, si, ri) for (t, si, ri) in tasks} + for n_done, fut in enumerate(as_completed(futures), 1): + line = fut.result() + if line and len(line) >= MIN_LEN: + if len(line) > MAX_LEN: + line = line[:MAX_LEN] + key = line[:40] + if key in seen: + skipped_dup += 1 + continue + seen.add(key) + f.write(line + "\n") + total += 1 + else: + failed += 1 + if n_done % 50 == 0 or n_done == len(tasks): + elapsed = time.time() - start + rate = total / elapsed if elapsed > 0 else 0 + print( + f" 进度 {n_done}/{len(tasks)}, 新增 {total}, " + f"重复 {skipped_dup}, 失败 {failed}, {rate:.1f}条/秒, {elapsed:.0f}s", + flush=True, + ) + + elapsed = time.time() - start + size = os.path.getsize(OUTPUT) + final_lines = existing_lines + total if mode == "a" else total + print(f"\n完成! 本次新增 {total} 行, 重复跳过 {skipped_dup}, 失败 {failed}") + print(f"文件总行数: {final_lines}, {size:,} 字节, 耗时 {elapsed:.0f}s") + print(f"输出: {OUTPUT}") + + +if __name__ == "__main__": + main() diff --git a/scripts/go_sgf_to_text.py b/scripts/go_sgf_to_text.py new file mode 100644 index 00000000..fd4d257e --- /dev/null +++ b/scripts/go_sgf_to_text.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 +""" +Go SGF → Text converter + Tokenizer generator (胜利方视角版) +============================================================ +核心设计： + - 只从胜利方视角训练：模型学到的是"赢家怎么下" + - = 赢家， = 输家 + - 推理时 即代表最强一方 + +每局棋只生成 1 条训练序列（从赢家视角）： + pd dp pp qq ... + +SGF 结果格式（RE 属性）： + RE[B+R] = 黑胜（认输） RE[W+R] = 白胜（认输） + RE[B+1.5]= 黑胜 1.5 目 RE[W+0.5]= 白胜 0.5 目 + RE[Void] = 无结果（跳过） + +用法： + python go_sgf_to_text.py sgf_dir/ -o go_games.txt --tokenizer go_tokenizer.json + +训练： + text_train.exe go_games.txt --vocab go_tokenizer.json \\ + --d-model 128 --num-heads 4 --num-layers 4 --d-ff 512 \\ + --seq-len 600 --epochs 50 --lr 0.001 --batch-size 32 --cuda + +推理： + text_infer.exe --model go_model.bin \\ + --prompt " pd dp" --max-tokens 1 --temperature 0 +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + + +# ═══════════════════════════════════════════════════════════════════════════ +# SGF 解析 +# ═══════════════════════════════════════════════════════════════════════════ + +def parse_sgf_result(sgf_text: str) -> str | None: + """ + 从 SGF 文本中提取对局结果。 + 返回: "B" (黑胜) / "W" (白胜) / None (无结果) + """ + # 匹配 RE[...] 属性 + m = re.search(r'RE\[(.*?)\]', sgf_text) + if not m: + return None + result = m.group(1).strip() + if result in ('Void', 'Void', '', '?'): + return None + # B+R, B+1.5, B+Resign, Black wins → 黑胜 + if result.upper().startswith('B') or 'BLACK' in result.upper(): + return 'B' + # W+R, W+0.5, White wins → 白胜 + if result.upper().startswith('W') or 'WHITE' in result.upper(): + return 'W' + return None + + +def convert_coord_system(moves: list[str]) -> list[str]: + """ + 统一 SGF 坐标体系。 + + 两种 19 路坐标体系： + 体系1 (标准): a-hj-t（19 个字符，跳过 i） + 体系2 (非标准): a-s（19 个字符，连续含 i）— CWI 部分文件使用 + + 判断：一局含 'i' 坐标且不含 't' 坐标（除 tt=PASS）→ 体系2，需转换。 + 体系2 的第 n 个字符 = 体系1 的第 n 个字符，因此 i→j, j→k, ..., s→t。 + + 转换后坐标与体系1 完全一致（i 不再出现）。 + """ + # 检查是否含 i（非 tt 的坐标） + has_i = any('i' in c for c in moves) + if not has_i: + return moves # 体系1，无需转换 + + # 检查是否含 t（除 tt=PASS 外的 t 坐标） + has_t = any('t' in c and c != 'tt' for c in moves) + if has_t: + # 既含 i 又含 t：数据混用/异常，丢弃该局（返回 None 标记） + return None + + # 体系2 → 体系1：i→j, j→k, k→l, ..., s→t + trans = str.maketrans('ijklmnopqrs', 'jklmnopqrst') + return [c.translate(trans) for c in moves] + + +def _extract_sgf_blocks(text: str) -> list[str]: + """提取顶层 ( ... ) 块，正确处理变例嵌套括号。""" + blocks = [] + i = 0 + n = len(text) + while i < n: + if text[i] == '(': + start = i + depth = 0 + while i < n: + if text[i] == '(': + depth += 1 + elif text[i] == ')': + depth -= 1 + if depth == 0: + i += 1 + break + i += 1 + else: + break # 未闭合括号：忽略剩余文本 + blocks.append(text[start:i]) + else: + i += 1 + return blocks + + +def _extract_mainline_moves(block: str) -> list[str]: + """ + 仅提取主变（第一层括号内、未嵌套于变例中）的着法。 + 嵌套括号 depth >= 2 的 ';B[...]' 是变例分支，跳过。 + """ + moves = [] + depth = 0 + pos = 0 + n = len(block) + while pos < n: + ch = block[pos] + if ch == '(': + depth += 1 + pos += 1 + continue + if ch == ')': + depth = max(0, depth - 1) + pos += 1 + continue + if ch == ';' and depth == 1: + m = re.match(r';([BW])\[([a-z]{2}|tt)\]', block[pos:]) + if m: + moves.append(m.group(2)) + pos += 1 + return moves + + +def parse_sgf_games(sgf_text: str) -> list[tuple[list[str], str | None]]: + """ + 从 SGF 文本中提取所有对局的落子序列和结果。 + + 只处理 19×19 棋局（SZ[19] 或未标注默认 19）。 + 9×9 / 13×13 等小棋盘（SZ[9]/SZ[13]）跳过。 + 非标准坐标体系（a-s 连续含 i）自动转换为标准体系。 + + 返回: [([moves], "B"/"W"/None), ...] + """ + results = [] + + game_blocks = _extract_sgf_blocks(sgf_text) + if not game_blocks: + game_blocks = [sgf_text] + + for block in game_blocks: + # 读取棋盘大小 SZ[...]，非 19 路跳过 + sz_m = re.search(r'SZ\[(\d+)\]', block) + if sz_m: + sz = int(sz_m.group(1)) + if sz != 19: + continue + + # 只提取主变着法（变例分支不混入主线） + moves = _extract_mainline_moves(block) + + if not moves: + continue + + # 统一坐标体系（含 i 的体系2 → 标准体系1）；None = 数据异常丢弃 + converted = convert_coord_system(moves) + if converted is None: + continue + + # 提取本局结果 + winner = parse_sgf_result(block) + results.append((converted, winner)) + + return results + + +# ═══════════════════════════════════════════════════════════════════════════ +# 胜利方视角转换 +# ═══════════════════════════════════════════════════════════════════════════ + +def game_to_winner_perspective(moves: list[str], winner: str | None) -> str | None: + """ + 将一局棋转换为胜利方视角的训练序列。 + + winner="B": 黑方是 ，白方是 + winner="W": 白方是 ，黑方是 + winner=None: 返回 None（无结果，跳过） + + 返回: " p d d p p p ..." 或 None + """ + if winner is None: + return None + + tokens = [] + for i, coord in enumerate(moves): + is_me = (i % 2 == 0 and winner == 'B') or \ + (i % 2 == 1 and winner == 'W') + prefix = "" if is_me else "" + # 坐标拆为列+行两个 token（如 pd → p d） + if coord == 'tt': + tokens.append(f"{prefix} tt") + else: + tokens.append(f"{prefix} {coord[0]} {coord[1]}") + + return ' '.join(tokens) + + +# ═══════════════════════════════════════════════════════════════════════════ +# Tokenizer 生成 +# ═══════════════════════════════════════════════════════════════════════════ + +def generate_go_tokenizer(output_path: str) -> int: + """ + 生成围棋专用的 SpaceTokenizer JSON（含 / 视角 token）。 + + 坐标不再使用合并的双字符 token（如 "pd"），而是拆为列+行 + 两个独立字符 token（如 "p" "d"），由 SpaceTokenizer 的 + ASCII 字节回退自动编码。词表只包含必要的多字符 token。 + + 词表布局： + ID 0 = + ID 1 = + ID 2 = + ID 3-258 = 256 个字节 token（覆盖 a-z 等单字符） + ID 259 = tt（PASS） + ID 260 = + ID 261 = + 总词表大小 = 262 + """ + vocab = {} + + # 256 个字节 token（确保 id_to_token_ 完整，解码不会丢失字符） + for i in range(256): + hex_val = f'{i:02x}' + vocab[str(3 + i)] = hex_val + + # PASS（tt） + hex_val = ''.join(f'{ord(c):02x}' for c in 'tt') + vocab['259'] = hex_val + + # 视角 token + hex_val = ''.join(f'{ord(c):02x}' for c in '') + vocab['260'] = hex_val + hex_val = ''.join(f'{ord(c):02x}' for c in '') + vocab['261'] = hex_val + + tokenizer = { + "type": "space_tokenizer", + "vocab": vocab, + "vocab_size": 262, + "special_tokens": { + "": 0, + "": 1, + "": 2 + } + } + + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(tokenizer, f, indent=2, ensure_ascii=False) + + return 262 + + +# ═══════════════════════════════════════════════════════════════════════════ +# 棋盘可视化 +# ═══════════════════════════════════════════════════════════════════════════ + +def sgf_to_rc(coord: str) -> tuple[int, int]: + if coord == 'tt': + return (-1, -1) + # SGF 标准坐标跳过 'i'：j 及以后的字母需要减 1 + def _axis(ch: str) -> int: + v = ord(ch) - ord('a') + return v - 1 if v > 8 else v # 'i' = 8，之后跳 1 + return (_axis(coord[1]), _axis(coord[0])) + + +def print_board(moves: list[str], last_n: int = 5): + board = [[0] * 19 for _ in range(19)] + last_positions = [] + for i, coord in enumerate(moves): + r, c = sgf_to_rc(coord) + if r < 0: + continue + color = 1 if i % 2 == 0 else 2 + board[r][c] = color + last_positions.append((r, c, i)) + + last_set = {(r, c) for r, c, _ in last_positions[-last_n:]} + last_colors = {(r, c): (1 if i % 2 == 0 else 2) + for r, c, i in last_positions[-last_n:]} + + col_labels = ' ' + ' '.join('abcdefghjklmnopqrst') + print(col_labels) + print(' +' + '-' * 37 + '+') + for r in range(19): + row_str = f'{19 - r:2d}|' + for c in range(19): + if (r, c) in last_set: + mark = 'X' if last_colors[(r, c)] == 1 else 'O' + row_str += f'({mark})' + elif board[r][c] == 1: + row_str += ' X ' + elif board[r][c] == 2: + row_str += ' O ' + else: + row_str += ' . ' + row_str += f'|{19 - r}' + print(row_str) + print(' +' + '-' * 37 + '+') + print(col_labels) + + +# ═══════════════════════════════════════════════════════════════════════════ +# Tokenizer 验证 +# ═══════════════════════════════════════════════════════════════════════════ + +def verify_tokenizer(tokenizer_path: str, sample_text: str): + with open(tokenizer_path, 'r', encoding='utf-8') as f: + tok_data = json.load(f) + + id_to_token = {} + for k, v in tok_data['vocab'].items(): + token_str = bytes.fromhex(v).decode('ascii') + id_to_token[int(k)] = token_str + token_to_id = {v: k for k, v in id_to_token.items()} + + def encode(text): + tokens = [] + for word in text.split(): + if word in token_to_id: + tokens.append(token_to_id[word]) + else: + for b in word.encode('ascii'): + tokens.append(b + 3) + return tokens + + def decode(ids): + result = [] + for tid in ids: + if tid in id_to_token: + result.append(id_to_token[tid]) + else: + result.append('') + return ' '.join(result) + + encoded = encode(sample_text) + decoded = decode(encoded) + + print(f"\n🔍 Tokenizer 验证:") + print(f" 输入: {sample_text}") + print(f" 编码: {encoded}") + print(f" 解码: {decoded}") + print(f" 往返一致: {'✅' if decoded == sample_text else '❌ 不一致!'}") + print(f" 词表大小: {tok_data['vocab_size']}") + print(f" 特殊 token: tt=259, =260, =261") + + +# ═══════════════════════════════════════════════════════════════════════════ +# 主流程 +# ═══════════════════════════════════════════════════════════════════════════ + +def collect_sgf_files(paths): + files = [] + for p in paths: + path = Path(p) + if path.is_file(): + if path.suffix.lower() == '.sgf': + files.append(path) + elif path.is_dir(): + found = sorted(path.rglob('*.sgf')) + files.extend(found) + print(f"📂 {path}: 找到 {len(found)} 个 SGF 文件") + else: + print(f"❌ 路径不存在: {path}", file=sys.stderr) + return files + + +def main(): + parser = argparse.ArgumentParser( + description='将 SGF 棋谱转换为 GPT 训练文本（胜利方视角）', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +核心设计：只从胜利方视角训练。=赢家，=输家。 +模型学到的是"赢家怎么下"。 + +示例： + python go_sgf_to_text.py sgf_dir/ -o go_games.txt --tokenizer go_tokenizer.json + python go_sgf_to_text.py sgf_dir/ --dry-run --max-games 10 +""") + parser.add_argument('input', nargs='+', + help='SGF 文件或包含 SGF 文件的目录') + parser.add_argument('-o', '--output', default='go_games.txt', + help='输出文本文件 (默认: go_games.txt)') + parser.add_argument('--tokenizer', default='go_tokenizer.json', + help='Tokenizer JSON (默认: go_tokenizer.json)') + parser.add_argument('--min-moves', type=int, default=20, + help='最少落子数 (默认: 20)') + parser.add_argument('--max-games', type=int, default=0, + help='最多处理局数, 0=不限 (默认: 0)') + parser.add_argument('--dry-run', action='store_true', + help='只预览不写文件') + parser.add_argument('--show-board', type=int, default=0, metavar='N', + help='显示前 N 局棋盘') + args = parser.parse_args() + + # ── 收集文件 ────────────────────────────────────── + sgf_files = collect_sgf_files(args.input) + if not sgf_files: + print("❌ 未找到任何 SGF 文件", file=sys.stderr) + sys.exit(1) + + print(f"\n共 {len(sgf_files)} 个 SGF 文件待处理\n") + + # ── 解析 ────────────────────────────────────────── + all_games = [] # [(moves, winner), ...] + parse_errors = 0 + for sgf_file in sgf_files: + try: + text = sgf_file.read_text(encoding='utf-8', errors='replace') + games = parse_sgf_games(text) + for moves, winner in games: + if len(moves) >= args.min_moves: + all_games.append((moves, winner)) + except Exception as e: + parse_errors += 1 + print(f"⚠️ 解析失败 {sgf_file.name}: {e}", file=sys.stderr) + + if not all_games: + print(f"❌ 没有有效数据", file=sys.stderr) + sys.exit(1) + + if args.max_games > 0: + all_games = all_games[:args.max_games] + + # ── 统计结果 ────────────────────────────────────── + total = len(all_games) + has_result = sum(1 for _, w in all_games if w is not None) + black_wins = sum(1 for _, w in all_games if w == 'B') + white_wins = sum(1 for _, w in all_games if w == 'W') + no_result = total - has_result + + print(f"📊 数据统计:") + print(f" 总局数: {total}") + print(f" 有结果: {has_result} (黑胜 {black_wins} / 白胜 {white_wins})") + print(f" 无结果: {no_result}（将跳过）") + print(f" 训练序列: {has_result} 条（仅胜利方视角）") + + total_moves = sum(len(m) for m, _ in all_games if _ is not None) + avg_moves = total_moves / max(has_result, 1) + print(f" 总落子数: {total_moves}") + print(f" 平均步数: {avg_moves:.1f}") + + # ── 转换为训练序列 ──────────────────────────────── + all_sequences = [] + for moves, winner in all_games: + seq = game_to_winner_perspective(moves, winner) + if seq is not None: + all_sequences.append(seq) + + # ── 预览 ────────────────────────────────────────── + print(f"\n📝 胜利方视角预览（前 3 局有效棋）:") + count = 0 + for moves, winner in all_games: + if winner is None: + continue + seq = game_to_winner_perspective(moves, winner) + if seq is None: + continue + winner_name = "黑方" if winner == 'B' else "白方" + display = seq if len(seq) <= 120 else seq[:117] + '...' + print(f" 局{count+1} (胜者={winner_name}): {display}") + count += 1 + if count >= 3: + break + + # ── 棋盘 ────────────────────────────────────────── + if args.show_board > 0: + count = 0 + for moves, winner in all_games: + if winner is None: + continue + winner_name = "黑方" if winner == 'B' else "白方" + print(f"\n棋盘 (胜者={winner_name}, 最后{min(5, len(moves))}手标记):") + print_board(moves, last_n=5) + count += 1 + if count >= args.show_board: + break + + # ── Dry run ─────────────────────────────────────── + if args.dry_run: + print(f"\n(--dry-run 模式)") + vocab_size = generate_go_tokenizer(args.tokenizer) + for moves, winner in all_games: + if winner: + sample = game_to_winner_perspective(moves, winner) + verify_tokenizer(args.tokenizer, sample[:120]) + break + return + + # ── 写文件 ──────────────────────────────────────── + with open(args.output, 'w', encoding='utf-8') as f: + for seq in all_sequences: + f.write(seq + '\n') + print(f"\n✅ 文本文件: {args.output} ({len(all_sequences)} 条序列)") + + # ── Tokenizer ───────────────────────────────────── + vocab_size = generate_go_tokenizer(args.tokenizer) + print(f"✅ Tokenizer: {args.tokenizer} (vocab_size={vocab_size})") + + # ── 验证 ────────────────────────────────────────── + if all_sequences: + verify_tokenizer(args.tokenizer, all_sequences[0][:120]) + + # ── 训练命令 ────────────────────────────────────── + print(f"\n{'='*60}") + print(f"🚀 训练命令:") + print(f" .\\build\\text_train.exe {args.output} \\") + print(f" --vocab {args.tokenizer} \\") + print(f" --d-model 128 --num-heads 4 --num-layers 4 --d-ff 512 \\") + print(f" --seq-len 600 --epochs 50 --lr 0.001 --batch-size 32 \\") + print(f" --save go_model.bin --cuda") + print(f"\n ⚠️ 视角 token 使序列长度翻倍，seq-len 需 ≥ 平均步数×2+2") + print(f" 当前平均 {avg_moves:.0f} 步 → 建议 seq-len=600") + print(f"\n🎯 推理:") + print(f" .\\build\\text_infer.exe --model go_model.bin \\") + print(f" --prompt \" pd dp\" --max-tokens 1 --temperature 0") + print(f"{'='*60}") + + +if __name__ == '__main__': + main() diff --git a/scripts/hf_to_txt.py b/scripts/hf_to_txt.py new file mode 100644 index 00000000..79c0d18c --- /dev/null +++ b/scripts/hf_to_txt.py @@ -0,0 +1,347 @@ +"""将 HuggingFace 对话数据集转换为带对话标记的纯文本语料。 + +输出格式（一行一个 step）： + 系统提示用户问题模型回答 + +换行符在内容中会被移除。分词器读到对话标记会映射为特殊 token ID， +普通文本正常分词。 + +用法: + python hf_to_txt.py [--subset ] [--split ] + [--output ] [--max_rows ] + +示例: + python hf_to_txt.py Mxode/Chinese-Instruct --subset dpsk-r1-distil --max_rows 50000 + python hf_to_txt.py Mxode/Chinese-Instruct-Lite --output datasets/chat_corpus.txt +""" +import argparse +import json +import os +import re +import sys + +import markdown as md + +# 对话标记（与分词器中定义一致） +SYSTEM_TAG = "" +USER_TAG = "" +ASSISTANT_TAG = "" +END_PREFIX = "]+>") + +# 在块级 Markdown 标记前插入换行，让 markdown 库正确识别 +_BLOCK_RE = re.compile( + r"(?<=\S)" # 前面有非空白字符（不是行首） + r"(?=---(?!\w))" # --- 分隔线 + r"|" + r"(?<=\S)" + r"(?=#{1,6}\s)" # # 标题 + r"|" + r"(?<=\s)" + r"(?=-\s)" # - 列表 + r"|" + r"(?<=\s)" + r"(?=\d+\.\s)" # 1. 有序列表 +) + + +def _protect_latex(text: str) -> tuple[str, list[tuple[str, str]]]: + """提取 LaTeX 数学片段并用占位符替换，防止被清理过程破坏。 + + 处理: \\boxed{...} \\text{...} \\[...\\] \\(...\\) $...$ + 返回 (处理后文本, [(占位符, 原文)] )。 + """ + blocks: list[tuple[str, str]] = [] + counter = [0] + + def _save(s: str) -> str: + tag = f"\x00L{counter[0]}\x00" + counter[0] += 1 + blocks.append((tag, s)) + return tag + + # 1) 简化 \boxed{...} \text{...} → 保留花括号内容 + text = re.sub(r"\\boxed\{([^}]+)\}", r"\1", text) + text = re.sub(r"\\text\{([^}]+)\}", r"\1", text) + + # 2) 字符扫描提取 \[...\] 和 \(...\)，只保留内容，丢弃定界符 + result: list[str] = [] + i = 0 + while i < len(text): + if text[i] == "\\" and i + 1 < len(text) and text[i + 1] in ("[", "("): + open_ch = text[i + 1] + close_ch = "]" if open_ch == "[" else ")" + j = i + 2 + while j < len(text): + if text[j] == "\\" and j + 1 < len(text) and text[j + 1] == close_ch: + break + j += 1 + # 提取 \[ 和 \] 之间的内容（不含定界符） + inner = text[i + 2 : j] + result.append(_save(inner)) + i = j + 2 # 跳过 \] + else: + result.append(text[i]) + i += 1 + text = "".join(result) + + # 3) 行内 $...$（排除 $$ 块级） + text = re.sub(r"(? str: + """清理 Markdown 格式标记，保留纯文本内容。 + + 使用 markdown 库将 Markdown 转为 HTML，再剥离 HTML 标签。 + 额外处理 LaTeX 定界符和 \\boxed / \\text 等 LaTeX 命令。 + """ + if not text: + return "" + + # --- 预处理 LaTeX：提取并保护 LaTeX 片段 --- + text, latex_blocks = _protect_latex(text) + + # --- 恢复块级结构，让 markdown 库正确解析 --- + text = _BLOCK_RE.sub("\n", text) + + # --- markdown → HTML → 纯文本 --- + html = md.markdown(text, extensions=["extra"]) + clean = _HTML_TAG_RE.sub(" ", html) + + # 还原 LaTeX 片段 + for tag, original in latex_blocks: + clean = clean.replace(tag, original) + + return clean + + +def clean_text(text: str) -> str: + """清洗文本：移除 Markdown 格式、换行、多余空白。""" + if not text: + return "" + # 先清理 Markdown 格式 + text = strip_markdown(text) + # 移除残留的水平分隔线标记 + text = re.sub(r"(?:---|\*\*\*|___)+", " ", text) + # 移除所有换行和回车 + text = text.replace("\r", "").replace("\n", "") + # 合并连续空格 + text = re.sub(r"\s+", " ", text).strip() + return text + + +def format_dialogue(system: str, prompt: str, response: str) -> str | None: + """将一条对话组装为一行带标记的文本。""" + prompt = clean_text(prompt) + response = clean_text(response) + + if not prompt or not response: + return None + # 跳过过短或过长的样本 + if len(prompt) < 4 or len(response) < 8: + return None + if len(prompt) > 2000 or len(response) > 2000: + return None + + parts = [] + if system: + system = clean_text(system) + parts.append(f"{SYSTEM_TAG}{system}{END_PREFIX}system>") + parts.append(f"{USER_TAG}{prompt}{END_PREFIX}user>") + parts.append(f"{ASSISTANT_TAG}{response}{END_PREFIX}assistant>") + return "".join(parts) + + +def detect_columns(sample: dict) -> tuple[str, str, str]: + """自动检测 prompt/response/system 字段名。""" + prompt_col = response_col = system_col = "" + + # prompt 字段候选 + for col in ("prompt", "instruction", "input", "question", "query", + "human", "user_input", "question_text"): + if col in sample and isinstance(sample[col], str): + prompt_col = col + break + # response 字段候选 + for col in ("response", "output", "answer", "assistant", "completion", + "gpt", "target", "answer_text"): + if col in sample and isinstance(sample[col], str): + response_col = col + break + # system 字段候选 + for col in ("system", "system_prompt", "context"): + if col in sample and isinstance(sample[col], str): + system_col = col + break + + # 兼容 messages 格式（OpenAI 风格） + if not prompt_col and "messages" in sample: + messages = sample["messages"] + if isinstance(messages, list): + system_msgs = [] + user_msgs = [] + assistant_msgs = [] + for msg in messages: + if not isinstance(msg, dict): + continue + role = msg.get("role", "") + content = msg.get("content", "") + if role == "system": + system_msgs.append(content) + elif role == "user": + user_msgs.append(content) + elif role == "assistant": + assistant_msgs.append(content) + if user_msgs and assistant_msgs: + # 返回一个临时的组装结果 + return "__messages__", "", "" # 标记为 messages 格式 + + return prompt_col, response_col, system_col + + +def process_messages(sample: dict) -> str | None: + """处理 OpenAI messages 格式。""" + messages = sample.get("messages") + if not isinstance(messages, list): + return None + + system_parts = [] + user_parts = [] + assistant_parts = [] + for msg in messages: + if not isinstance(msg, dict): + continue + role = msg.get("role", "") + content = msg.get("content", "") + if isinstance(content, list): + # 多模态格式，只取文本部分 + content = " ".join( + p.get("text", "") for p in content if isinstance(p, dict) and p.get("type") == "text" + ) + if not isinstance(content, str): + continue + if role == "system": + system_parts.append(content) + elif role == "user": + user_parts.append(content) + elif role == "assistant": + assistant_parts.append(content) + + system = " ".join(system_parts) + prompt = " ".join(user_parts) + response = " ".join(assistant_parts) + + return format_dialogue(system, prompt, response) + + +def convert_dataset( + dataset_name: str, + subset: str | None, + split: str, + output_path: str, + max_rows: int | None, +): + """主转换函数。""" + try: + from datasets import load_dataset, get_dataset_config_names + except ImportError: + print("需要安装 datasets 库: pip install datasets", file=sys.stderr) + sys.exit(1) + + # 获取子集列表 + if subset is None: + try: + configs = get_dataset_config_names(dataset_name) + if configs: + subset = configs[0] + print(f"自动选择子集: {subset}") + except Exception: + pass + + print(f"加载数据集: {dataset_name} (subset={subset}, split={split}) ...") + ds = load_dataset(dataset_name, subset, split=split) + + if max_rows: + ds = ds.select(range(min(max_rows, len(ds)))) + print(f"总样本数: {len(ds)}") + + # 自动检测字段（用第一条样本） + first = ds[0] + prompt_col, response_col, system_col = detect_columns(first) + is_messages = (prompt_col == "__messages__") + + if is_messages: + print("检测到 messages 格式") + else: + print(f"检测到字段: prompt={prompt_col!r}, response={response_col!r}, system={system_col!r}") + if not prompt_col or not response_col: + print(f"字段: {list(first.keys())}", file=sys.stderr) + print("无法自动识别 prompt/response 字段，请检查数据集格式", file=sys.stderr) + sys.exit(1) + + # 确保输出目录存在 + os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True) + + written = 0 + skipped = 0 + with open(output_path, "w", encoding="utf-8") as f: + for i, sample in enumerate(ds): + if is_messages: + line = process_messages(sample) + else: + system = sample.get(system_col, "") if system_col else "" + prompt = sample.get(prompt_col, "") + response = sample.get(response_col, "") + line = format_dialogue(system, prompt, response) + + if line: + f.write(line + "\n") + written += 1 + else: + skipped += 1 + + if (i + 1) % 10000 == 0: + print(f" 已处理 {i + 1} 条, 写入 {written}, 跳过 {skipped}") + + print(f"\n完成! 写入 {written} 条, 跳过 {skipped} 条") + print(f"输出: {output_path}") + + # 显示文件大小 + size = os.path.getsize(output_path) + for unit in ("B", "KB", "MB", "GB"): + if size < 1024: + print(f"文件大小: {size:.1f} {unit}") + break + size /= 1024 + + +def main(): + parser = argparse.ArgumentParser(description="HF 对话数据集 → 带标记的纯文本语料") + parser.add_argument("dataset", help="HuggingFace 数据集名称，如 Mxode/Chinese-Instruct") + parser.add_argument("--subset", default=None, help="子集名称 (默认自动选择第一个)") + parser.add_argument("--split", default="train", help="数据集 split (默认: train)") + parser.add_argument("--output", default=None, help="输出文件路径 (默认: datasets/.txt)") + parser.add_argument("--max_rows", type=int, default=None, help="最大处理行数") + args = parser.parse_args() + + output = args.output + if output is None: + # 从数据集名生成输出路径 + name = args.dataset.split("/")[-1].lower().replace("-", "_") + script_dir = os.path.dirname(os.path.abspath(__file__)) + output = os.path.join(script_dir, "datasets", f"{name}.txt") + + convert_dataset(args.dataset, args.subset, args.split, output, args.max_rows) + + +if __name__ == "__main__": + main() diff --git a/scripts/lowercase_openwebtext.py b/scripts/lowercase_openwebtext.py new file mode 100644 index 00000000..2ae9a433 --- /dev/null +++ b/scripts/lowercase_openwebtext.py @@ -0,0 +1,20 @@ +"""将 OpenWebText 转为小写。""" +import os + +src = "scripts/datasets/openwebtext_en_100mb.txt" +dst = "scripts/datasets/openwebtext_en_100mb_lower.txt" + +print(f"读取: {src}") +with open(src, "r", encoding="utf-8") as f: + text = f.read() + +print(f"已读取 {len(text)} 字节, {text.count(chr(10))} 行") + +text_lower = text.lower() + +print(f"写入: {dst}") +with open(dst, "w", encoding="utf-8") as f: + f.write(text_lower) + +size = os.path.getsize(dst) +print(f"完成! 大小: {size/1024/1024:.1f} MB") \ No newline at end of file diff --git a/scripts/merge_corpus.py b/scripts/merge_corpus.py new file mode 100644 index 00000000..8b86420a --- /dev/null +++ b/scripts/merge_corpus.py @@ -0,0 +1,201 @@ +"""合并所有中文语料来源，去重，输出最终训练集。 + +输入来源： + 1. datasets/llm_corpus.txt — 原始 LLM 生成语料 + 2. datasets/qwen_corpus.txt — Qwen v1 生成语料 + 3. datasets/qwen_corpus_v2.txt — Qwen v2 扩展语料 + 4. datasets/chinese_datasets_corpus.txt — HuggingFace/新闻/精选语料 + 5. datasets/charbpe_corpus.json — charbpe 语料 + 6. datasets/charbpe_llm.json — charbpe LLM 语料 + 7. datasets/charbpe_mini.json — charbpe 小语料 + +输出： + datasets/training_corpus_final.txt — 最终训练语料（每行一段/一篇，长度不限） + +滑动窗口说明： + 训练端 (src/text_train.cpp) 会把每行当作一个文档，编码为 + [BOS]+tokens+[EOS] 后拼接成连续 token 流，再按 seq_len 滑动切窗。 + 因此： + - 每行应保留完整段落/文档，不要按句号切碎（切碎会丢失跨句上下文） + - 超长文档无需截断，训练端会自动切成多个窗口 + - 仅过滤过短噪声（< --min-len）和极端异常超长（> --max-len） + +用法： + python scripts/merge_corpus.py [--min-len 8] [--max-len 300] +""" + +import json +import os +import re +import argparse + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +DATASETS_DIR = os.path.join(SCRIPT_DIR, "..", "datasets") + +# 所有语料源文件 +CORPUS_FILES = [ + "llm_corpus.txt", + "qwen_corpus.txt", + "qwen_corpus_v2.txt", + "chinese_datasets_corpus.txt", + "chinese_wiki_corpus.txt", + "expand_chinese_corpus.txt", + "llm_corpus_expanded.txt", +] + +# JSON 格式的语料源 +JSON_CORPUS_FILES = [ + "charbpe_corpus.json", + "charbpe_llm.json", + "charbpe_mini.json", +] + + +def load_txt_corpus(path: str) -> list[str]: + """加载 txt 格式语料（每行一段/一篇）。""" + lines = [] + try: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + lines.append(line) + except Exception as e: + print(f" ⚠ 读取 {path} 失败: {e}") + return lines + + +def load_json_corpus(path: str) -> list[str]: + """加载 JSON 格式语料。""" + lines = [] + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, list): + for item in data: + if isinstance(item, str): + lines.append(item.strip()) + elif isinstance(item, dict): + # 尝试多种字段名 + for key in ["text", "content", "sentence", "corpus", "segment"]: + if key in item and isinstance(item[key], str): + lines.append(item[key].strip()) + break + except Exception as e: + print(f" ⚠ 读取 {path} 失败: {e}") + return lines + + +def is_chinese_dominant(text: str) -> bool: + """检查文本是否以中文为主。""" + if not text: + return False + chinese = sum(1 for c in text if "\u4e00" <= c <= "\u9fff") + return chinese >= len(text) * 0.3 + + +def main(): + parser = argparse.ArgumentParser(description="合并所有中文语料来源") + parser.add_argument("--min-len", type=int, default=8, help="最短段落长度（过短视为噪声丢弃）") + parser.add_argument("--max-len", type=int, default=4096, + help="最长段落长度（防御性上限，超过视为异常丢弃；滑动窗口会自动切分长文）") + args = parser.parse_args() + + print("=" * 60) + print("🔄 中文语料合并工具") + print("=" * 60) + + all_lines = [] + + # 加载 txt 语料 + print("\n📄 加载 txt 语料文件:") + for fname in CORPUS_FILES: + fpath = os.path.join(DATASETS_DIR, fname) + if os.path.exists(fpath): + lines = load_txt_corpus(fpath) + print(f" ✓ {fname}: {len(lines)} 行") + all_lines.extend(lines) + else: + print(f" - {fname}: 不存在，跳过") + + # 加载 json 语料 + print("\n📄 加载 json 语料文件:") + for fname in JSON_CORPUS_FILES: + fpath = os.path.join(DATASETS_DIR, fname) + if os.path.exists(fpath): + lines = load_json_corpus(fpath) + print(f" ✓ {fname}: {len(lines)} 行") + all_lines.extend(lines) + else: + print(f" - {fname}: 不存在，跳过") + + print(f"\n📊 原始总量: {len(all_lines)} 行") + + # 过滤 + 去重（保留完整段落，长度过滤仅用于剔除噪声/异常） + seen = set() + filtered = [] + for line in all_lines: + line = line.strip() + if not line: + continue + + # 长度过滤：过短视为噪声，过长视为异常（滑动窗口会自动切分长文） + if len(line) < args.min_len: + continue + if len(line) > args.max_len: + continue + + # 中文比例检查 + if not is_chinese_dominant(line): + continue + + # 去重（基于前 40 字符） + key = line[:40] + if key in seen: + continue + seen.add(key) + filtered.append(line) + + print(f"📊 去重+过滤后: {len(filtered)} 行") + + # 统计 + total_chars = sum(len(line) for line in filtered) + avg_len = total_chars / len(filtered) if filtered else 0 + print(f"📊 总字符数: {total_chars:,}") + print(f"📊 平均长度: {avg_len:.0f} 字符") + + # 写入最终文件 + out_path = os.path.join(DATASETS_DIR, "training_corpus_final.txt") + with open(out_path, "w", encoding="utf-8") as f: + for line in filtered: + f.write(line + "\n") + + print(f"\n{'='*60}") + print(f"✅ 最终训练语料: {out_path}") + print(f" 总计 {len(filtered)} 句, {total_chars:,} 字符") + print(f"{'='*60}") + + # 也更新 llm_corpus.txt（合并回去） + update_path = os.path.join(DATASETS_DIR, "llm_corpus.txt") + existing = set() + if os.path.exists(update_path): + with open(update_path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + existing.add(line[:40]) + + new_count = 0 + with open(update_path, "a", encoding="utf-8") as f: + for line in filtered: + key = line[:40] + if key not in existing: + f.write(line + "\n") + existing.add(key) + new_count += 1 + + print(f"\n📝 已追加 {new_count} 条新句子到 {update_path}") + + +if __name__ == "__main__": + main() diff --git a/save_dataset.py b/scripts/save_dataset.py similarity index 100% rename from save_dataset.py rename to scripts/save_dataset.py diff --git a/scripts/scan_raw_pointers.py b/scripts/scan_raw_pointers.py new file mode 100644 index 00000000..d7f87504 --- /dev/null +++ b/scripts/scan_raw_pointers.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +""" +扫描代码库中所有 .cpp / .hpp 文件， +找出包含以下内容的文件和行号： + - new / new[] （手动分配） + - delete / delete[] （手动释放） + - malloc / calloc / realloc / free （C 风格内存管理） + - 裸指针声明（非智能指针） + - void* / reinterpret_cast / const_cast（危险类型转换） + - C 风格强制转换 (type*)expr + +输出格式：按文件分组，每个命中显示行号和对应代码。 +""" + +import os +import re +from collections import defaultdict + +# ---------- 扫描目标目录 ---------- +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SRC_DIRS = [os.path.join(REPO_ROOT, "src"), os.path.join(REPO_ROOT, "include")] + +# ---------- 匹配规则 ---------- +# 每条规则: (category, description, compiled_regex) +RULES = [ + # ---- 手动内存管理 ---- + ("alloc", "new", + re.compile(r'\bnew\s+[A-Za-z_]\w*(\s*\[|\s*\()')), + ("free", "delete", + re.compile(r'\bdelete\b')), + ("alloc", "malloc/calloc/realloc", + re.compile(r'\b(malloc|calloc|realloc)\s*\(')), + ("free", "free()", + re.compile(r'\bfree\s*\(')), + + # ---- 裸指针声明（排除智能指针和注释） ---- + ("ptr", "raw pointer decl", + re.compile( + r'(?]+' # 类型名 + r'(?:\s*<[^>]+>)?' # 可选模板参数 + r'\s*\*' # 指针星号 + r'\s+[a-zA-Z_]\w*' # 变量名 + r'(?:\s*[=\[;(,])' # 后跟赋值/数组/分号/逗号/括号 + # 排除行注释和智能指针 + )), + + # ---- void* ---- + ("cast", "void*", + re.compile(r'\bvoid\s*\*')), + + # ---- reinterpret_cast / const_cast ---- + ("cast", "reinterpret_cast", + re.compile(r'\breinterpret_cast\s*<')), + ("cast", "const_cast", + re.compile(r'\bconst_cast\s*<')), + + # ---- C 风格强制转换 (type*)expr ---- + ("cast", "C-style cast", + re.compile( + r'\((?:const\s+)?[A-Za-z_]\w*(?:\s*\*+)+\s*\)\s*[A-Za-z_(]')), + + # ---- static_cast<...*> ---- (指针类型的 static_cast) + ("cast", "static_cast", + re.compile(r'\bstatic_cast\s*<[^>]*\*[^>]*>')), +] + +# 行注释 / 字符串字面量 的简单过滤（避免误报） +SKIP_COMMENT = re.compile(r'^\s*//') +SKIP_STRING = re.compile(r'^\s*"(?:[^"\\]|\\.)*"$') # 整行就是字符串 + +# 智能指针名（排除这些 "指针声明" 误报） +SMART_PTR = re.compile(r'\b(unique_ptr|shared_ptr|weak_ptr|make_unique|make_shared)\b') + + +def scan_file(filepath): + """扫描单个文件，返回 [(line_no, category, desc, line_text), ...]""" + hits = [] + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + for lineno, line in enumerate(f, 1): + # 跳过纯注释行 / 预处理包含 + if SKIP_COMMENT.match(line) or line.strip().startswith("#"): + continue + # 跳过智能指针所在行（降低 ptr 类误报） + is_smart = bool(SMART_PTR.search(line)) + for cat, desc, rx in RULES: + if rx.search(line): + # ptr 类规则：如果该行出现了智能指针则跳过 + if cat == "ptr" and is_smart: + continue + hits.append((lineno, cat, desc, line.rstrip())) + break # 一行只报一次（取最高优先级） + return hits + + +def main(): + all_files = [] + for d in SRC_DIRS: + if not os.path.isdir(d): + continue + for root, _, files in os.walk(d): + for name in files: + if name.endswith((".cpp", ".hpp")): + all_files.append(os.path.join(root, name)) + + all_files.sort() + + results = {} # filepath -> [hits] + summary = defaultdict(list) # category -> [filepath] + + for fp in all_files: + hits = scan_file(fp) + if hits: + results[fp] = hits + cats = {h[1] for h in hits} + for c in cats: + summary[c].append(fp) + + # ---- 输出报告 ---- + total_files = len(all_files) + flagged = len(results) + total_hits = sum(len(v) for v in results.values()) + + print(f"扫描完成：共 {total_files} 个文件，{flagged} 个文件包含手动内存管理/裸指针，共 {total_hits} 处命中\n") + + if not results: + print("✅ 未发现手动内存管理或裸指针，代码库非常干净！") + return + + for fp in results: + rel = os.path.relpath(fp, REPO_ROOT) + print(f"{'=' * 80}") + print(f"📄 {rel} ({len(results[fp])} 处)") + print(f"{'-' * 80}") + for lineno, cat, desc, line_text in results[fp]: + # 截断过长行 + display = line_text[:120] + (" …" if len(line_text) > 120 else "") + print(f" L{lineno:>4d} [{cat:<6s}] {desc:<20s} | {display}") + print() + + # ---- 汇总 ---- + print(f"{'=' * 80}") + print("📊 汇总统计") + print(f"{'-' * 80}") + for cat in ["alloc", "free", "ptr", "cast"]: + files_in_cat = summary.get(cat, []) + if files_in_cat: + print(f" {cat:<8s}: {len(files_in_cat)} 个文件") + print(f"{'=' * 80}") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_charbpe.py b/scripts/test_charbpe.py new file mode 100644 index 00000000..39f875bd --- /dev/null +++ b/scripts/test_charbpe.py @@ -0,0 +1,107 @@ +"""测试 charbpe 分词器质量和压缩率。""" +import json +import sys + +# 加载 JSON 词表 +with open("openwebtext_charbpe_8k.json", "r", encoding="utf-8") as f: + data = json.load(f) + +print(f"分词器类型: {data['type']}") +print(f"vocab_size: {data['vocab_size']}") +print(f"合并规则数: {len(data['merges'])}") + +# 测试样本 +samples = [ + "former secretary of state hillary clinton meets voters at a campaign rally in st. louis on saturday.", + "the quick brown fox jumps over the lazy dog.", + "researchers have discovered a new species of dinosaur in argentina.", + "the stock market rallied today as investors reacted to the latest economic data.", + "the opinions expressed by columnists are their own and do not represent the views of townhall.com.", +] + +# 从 vocab 构建 id → token 的查找表 +# vocab 是 {id_str: hex_bytes} 格式 +# 特殊 token (0-3): 直接文本 +# 字节 token (4-259): 单字节 +# 其他: hex 编码的 UTF-8 字节 +id_to_token = {} +for sid, hex_val in data["vocab"].items(): + idx = int(sid) + try: + if idx < 4: + # 特殊 token + id_to_token[idx] = hex_val + elif idx < 260: + # 单字节 + b = int(hex_val, 16) + id_to_token[idx] = bytes([b]) + else: + # 多字节 UTF-8 + id_to_token[idx] = bytes.fromhex(hex_val) + except: + id_to_token[idx] = hex_val + +# 构建合并规则: {id_a: {id_b: merged_id}} +merge_map = {} +for a, b, merged in data["merges"]: + if a not in merge_map: + merge_map[a] = {} + merge_map[a][b] = merged + +def encode(text): + """简单编码器: 按字符拆分为字节 ID, 然后应用 BPE 合并.""" + # 将文本转为字节, 每个字节映射到 ID + raw_bytes = text.encode("utf-8") + ids = [b + 4 for b in raw_bytes] # 字节 ID 从 4 开始 + + # 应用合并规则 + changed = True + while changed: + changed = False + i = 0 + while i < len(ids) - 1: + a, b = ids[i], ids[i+1] + if a in merge_map and b in merge_map[a]: + ids[i] = merge_map[a][b] + ids.pop(i+1) + changed = True + # 合并后可能可以和前面的继续合并 + if i > 0: + i -= 1 + else: + i += 1 + + return ids + +def decode(ids): + """简单解码器: ID → 字节 → 字符串.""" + result = b"" + for idx in ids: + if idx in id_to_token: + tok = id_to_token[idx] + if isinstance(tok, str): + result += tok.encode("utf-8") + elif isinstance(tok, bytes): + result += tok + return result.decode("utf-8", errors="replace") + +# 测试 +print("\n" + "=" * 60) +print("分词器质量测试") +print("=" * 60) + +for sample in samples: + ids = encode(sample) + decoded = decode(ids) + ratio = len(sample.encode("utf-8")) / max(len(ids), 1) + print(f"\n原文: {sample[:60]}...") + print(f"编码: {len(ids)} tokens, 压缩率: {ratio:.2f} 字节/token") + print(f"解码: {decoded[:60]}...") + match = sample == decoded + print(f"无损: {'✓' if match else '✗'}") + +print("\n" + "=" * 60) +print("建议:") +print(f" 词表大小: {data['vocab_size']}") +print(f" 合并规则: {len(data['merges'])}") +print(f" 适合: 小写英文预训练") \ No newline at end of file diff --git a/scripts/test_md_clean.py b/scripts/test_md_clean.py new file mode 100644 index 00000000..6e19b673 --- /dev/null +++ b/scripts/test_md_clean.py @@ -0,0 +1,52 @@ +"""Test strip_markdown / clean_text function.""" +import sys +sys.path.insert(0, __import__("os").path.dirname(__file__)) + +from hf_to_txt import clean_text + +tests = [ + # (input, expected_substrings) — 检查核心 markdown 格式是否被移除 + ("### **角色基础信息** **姓名**：塞拉斯", + ["角色基础信息", "姓名", "塞拉斯"]), + ("**小瓶数量**：30盎司 = **5瓶** - **小瓶总价**：5瓶", + ["小瓶数量", "30盎司", "5瓶", "小瓶总价"]), + ("---### **1. 恋爱**- **学会**", + ["1.", "恋爱", "学会"]), + ("答案为：\\boxed{20}", + ["答案为：20"]), + ("\\[212 \\, \\text{字/分钟} - 40\\]", + ["212", "字/分钟", "40"]), + ("#### **第一幕：金雀花之血（童年）**", + ["第一幕：金雀花之血（童年）"]), + ("`print(42)` hello", + ["print(42)", "hello"]), + ("1. **学会** 2. **沟通**", + ["学会", "沟通"]), + ("\\( C \\) 正方体", + ["C", "正方体"]), +] + +all_pass = True +for i, (inp, substrings) in enumerate(tests): + result = clean_text(inp) + fails = [s for s in substrings if s not in result] + # 检查没有残留的 markdown 标记 + md_residuals = ["**", "###", "####", "\\boxed", "\\text{", "---"] + found_residuals = [r for r in md_residuals if r in result] + + if not fails and not found_residuals: + print(f" PASS #{i+1}: {result[:60]}") + else: + all_pass = False + print(f" FAIL #{i+1}") + print(f" IN: {inp!r}") + print(f" OUT: {result!r}") + if fails: + print(f" Missing: {fails}") + if found_residuals: + print(f" Markdown residuals: {found_residuals}") + +if all_pass: + print("\nAll tests passed!") +else: + print("\nSome tests failed!") diff --git a/test_tokenizer.py b/scripts/test_tokenizer.py similarity index 100% rename from test_tokenizer.py rename to scripts/test_tokenizer.py diff --git a/token_bench.py b/scripts/token_bench.py similarity index 100% rename from token_bench.py rename to scripts/token_bench.py diff --git a/train_ByteZip.py b/scripts/train_ByteZip.py similarity index 100% rename from train_ByteZip.py rename to scripts/train_ByteZip.py diff --git a/train_bpe.py b/scripts/train_bpe.py similarity index 95% rename from train_bpe.py rename to scripts/train_bpe.py index 8c11e13d..4d4860a3 100644 --- a/train_bpe.py +++ b/scripts/train_bpe.py @@ -12,6 +12,7 @@ from tokenizers.models import BPE from tokenizers.trainers import BpeTrainer from tokenizers.pre_tokenizers import ByteLevel +from tokenizers.decoders import ByteLevel as ByteLevelDecoder # 特殊 token 定义（与 ByteZip 保持一致） SPECIAL_TOKENS = ["", "", "", ""] @@ -22,6 +23,9 @@ def train_bpe(text_file: str, vocab_size: int) -> Tokenizer: """训练 BPE 并返回 tokenizer 对象""" tokenizer = Tokenizer(BPE(unk_token="")) tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True) + # 必须设置 ByteLevel decoder，否则保存的 JSON 缺 decoder， + # 解码时会输出带 Ġ 前缀/丢失空格的文本 + tokenizer.decoder = ByteLevelDecoder() trainer = BpeTrainer(vocab_size=vocab_size, special_tokens=SPECIAL_TOKENS) tokenizer.train(files=[text_file], trainer=trainer) diff --git a/scripts/train_charbpe.py b/scripts/train_charbpe.py new file mode 100644 index 00000000..782ace77 --- /dev/null +++ b/scripts/train_charbpe.py @@ -0,0 +1,298 @@ +""" +使用 HuggingFace tokenizers 库训练字符级 BPE 分词器，速度远超 C++ 实现。 + +输出格式与 C++ CharBPETokenizer 完全兼容的 JSON 文件。 +ID 布局： + 0-3 : , , , + 4-259 : 256 个单字节 ASCII 兜底 + 260+ : 非 ASCII 字符（中文等）+ BPE 合并 token + +依赖：pip install tokenizers +用法：python train_charbpe.py [选项] +示例：python train_charbpe.py datasets/llm_corpus.txt --vocab-size 5000 --output charbpe.json +""" + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Iterator + +from tokenizers import Tokenizer, Regex +from tokenizers.models import BPE +from tokenizers.trainers import BpeTrainer +from tokenizers.pre_tokenizers import Sequence, Split +from tokenizers.normalizers import Normalizer + + +# ── 常量（与 C++ CharBPETokenizer 一致） ────────────────────────────── +SPECIAL_TOKENS = ["", "", "", ""] +BYTE_BASE = 4 # 256 个单字节兜底起始 ID +CHAR_BASE = 260 # 非 ASCII 字符 token 起始 ID + + +def make_pretokenizer(): + """ + 构造字符级预分词器： + - 连续 CJK 字符作为一个词（让 BPE 学习常见词组合并） + - CJK 标点作为分隔符 + - 连续 ASCII 非空白字符作为一个词 + - 空白作为独立 token + """ + # 规则：按优先级匹配 + # 1. 连续 CJK 字符（不含标点）→ 一个词 + # 2. 单个 CJK 标点 → 独立 + # 3. 连续非空白非CJK → 一个词（ASCII 单词、数字等） + # 4. 连续空白 → 一个 token + pattern = ( + r"[\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]+" # 连续 CJK 字符 + r"|[\u3000-\u303f\uff00-\uffef]" # 单个 CJK 标点 + r"|[^ \t\n\r\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff\u3000-\u303f\uff00-\uffef]+" # 连续非空白非CJK + r"|[ \t\n\r]+" # 空白 + ) + return Split(Regex(pattern), behavior="isolated", invert=False) + + +def make_normalizer(): + """不做任何 normalization，保留原始文本。""" + return None # 不用 normalizer，避免 NFC/NFKC 改变中文 + + +def train_charbpe(text_file: str, vocab_size: int, min_freq: int) -> Tokenizer: + """训练字符级 BPE tokenizer。""" + tokenizer = Tokenizer(BPE(unk_token="", fuse_unk=False)) + tokenizer.normalizer = make_normalizer() + tokenizer.pre_tokenizer = make_pretokenizer() + # 禁用 post_processor 和 decoder，避免自动插入特殊符号 + tokenizer.post_processor = None + tokenizer.decoder = None + + trainer = BpeTrainer( + vocab_size=vocab_size, + min_frequency=min_freq, + special_tokens=SPECIAL_TOKENS, + show_progress=True, + initial_alphabet=[], # 不自动添加字母表，由预分词器控制 + ) + tokenizer.train(files=[text_file], trainer=trainer) + return tokenizer + + +def convert_to_charbpe_json(tokenizer: Tokenizer, max_vocab: int = 50000) -> dict: + """ + 将 tokenizers 库的 BPE 模型转换为 C++ CharBPETokenizer 的 JSON 格式。 + + 关键转换： + 1. tokenizers 库使用 Ġ 表示前导空格 → 替换为实际空格字节 + 2. 重新分配 ID：特殊 token → 字节兜底 → 字符 → 合并 + 3. merges 转换为 [id_a, id_b, new_id] 三元组 + """ + # ── 1. 获取原始词表和合并规则 ── + vocab_str_to_id = tokenizer.get_vocab() # {token_string: id} + + # 读取临时 JSON 获取 merges（tokenizers 库不直接暴露） + import tempfile + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + tokenizer.save(f.name) + temp_path = f.name + with open(temp_path, 'r', encoding='utf-8') as f: + data = json.load(f) + Path(temp_path).unlink(missing_ok=True) + + model = data.get('model', {}) + merges_raw = model.get('merges', []) + added_tokens = data.get('added_tokens', []) + + # ── 2. 将 token 字符串转为字节 ── + # tokenizers 库的 BPE token 是字符串，需要正确解码 + # Ġ 是 ByteLevel pre-tokenizer 的编码，我们没用它， + # 但合并规则可能包含它，需要处理 + def token_to_bytes(tok: str) -> bytes: + """将 token 字符串转为原始字节。处理 Ġ → 空格。""" + # 替换 Ġ (U+0120) 为空格 + tok = tok.replace('\u0120', ' ') + return tok.encode('utf-8') + + # 构建 token_str → bytes 映射 + token_bytes_map = {} + for tok_str in vocab_str_to_id: + token_bytes_map[tok_str] = token_to_bytes(tok_str) + + # ── 3. 分类 token ── + special_set = set(SPECIAL_TOKENS) + special_tok = {} # token_str → 我们的 ID + byte_tok = {} # byte_value → 我们的 ID + char_tok = {} # token_str → 我们的 ID（非 ASCII 字符） + merge_tok = {} # token_str → 我们的 ID（BPE 合并结果） + + # 特殊 token 分配 ID 0-3 + for i, st in enumerate(SPECIAL_TOKENS): + special_tok[st] = i + + # 256 单字节兜底分配 ID 4-259 + for b in range(256): + byte_tok[b] = BYTE_BASE + b + + # 分类剩余 token + next_id = CHAR_BASE + char_list = [] # (token_str, bytes) 非 ASCII 字符 + merge_list = [] # (token_str, bytes, id_a, id_b) 合并 token + + # 解析 merges 获取合并关系 + merge_pairs = [] # [(tok1_str, tok2_str)] + for m in merges_raw: + if isinstance(m, str): + parts = m.split() + if len(parts) == 2: + merge_pairs.append((parts[0], parts[1])) + elif isinstance(m, list) and len(m) == 2: + merge_pairs.append((m[0], m[1])) + + # 找出哪些 token 是合并产生的（出现在 merges 的右侧） + merged_set = set() + for t1, t2 in merge_pairs: + merged_set.add(t1 + t2) + + # 分类 + for tok_str, tid in vocab_str_to_id.items(): + if tok_str in special_set: + continue + bs = token_bytes_map[tok_str] + if len(bs) == 1: + # 单字节 → 已在 byte_tok 中 + continue + if tok_str in merged_set: + # 合并 token，稍后处理 + continue + # 非 ASCII 字符（多字节但不是合并结果） + if not all(0x20 <= b < 0x7f for b in bs): + char_list.append((tok_str, bs)) + + # 按首次出现顺序排序字符 + char_list.sort(key=lambda x: vocab_str_to_id[x[0]]) + + # 分配字符 ID + for tok_str, bs in char_list: + if next_id >= max_vocab: + break + char_tok[tok_str] = next_id + next_id += 1 + + # ── 4. 构建 merges 三元组 ── + our_merges = [] + for t1_str, t2_str in merge_pairs: + merged_str = t1_str + t2_str + if merged_str not in vocab_str_to_id: + continue + + # 查找 t1, t2 在我们 ID 系统中的 ID + def get_our_id(tok: str) -> int: + if tok in special_tok: + return special_tok[tok] + bs = token_bytes_map[tok] + if len(bs) == 1: + return byte_tok[bs[0]] + if tok in char_tok: + return char_tok[tok] + if tok in merge_tok: + return merge_tok[tok] + # 如果找不到，分配一个新 ID + nonlocal next_id + if next_id < max_vocab: + merge_tok[tok] = next_id + next_id += 1 + return merge_tok[tok] + return -1 + + id_a = get_our_id(t1_str) + id_b = get_our_id(t2_str) + if id_a < 0 or id_b < 0: + continue + + # 合并结果的 ID + merged_id = get_our_id(merged_str) + if merged_id < 0: + continue + + our_merges.append([id_a, id_b, merged_id]) + + # ── 5. 构建最终词表 ── + vocab_hex = {} + + # 特殊 token (ID 0-3)：纯文本 + for st, sid in special_tok.items(): + vocab_hex[str(sid)] = st + + # 单字节兜底 (ID 4-259)：hex 编码 + for b in range(256): + vocab_hex[str(BYTE_BASE + b)] = f"{b:02x}" + + # 非 ASCII 字符：hex 编码 UTF-8 + all_chars = {**char_tok} + for tok_str, our_id in all_chars.items(): + bs = token_bytes_map[tok_str] + vocab_hex[str(our_id)] = bs.hex() + + # 合并 token：hex 编码 + all_merges = {**merge_tok} + for tok_str, our_id in all_merges.items(): + bs = token_bytes_map[tok_str] + vocab_hex[str(our_id)] = bs.hex() + + # ── 6. 计算最终 vocab_size ── + all_ids = [int(k) for k in vocab_hex.keys()] + total_size = max(max(all_ids) + 1 if all_ids else CHAR_BASE, CHAR_BASE) + + return { + "type": "char_bpe_tokenizer", + "vocab": vocab_hex, + "vocab_size": total_size, + "merges": our_merges, + } + + +def main(): + parser = argparse.ArgumentParser( + description="使用 tokenizers 库训练字符级 BPE，输出兼容 C++ 格式的 JSON" + ) + parser.add_argument("text_file", help="训练文本文件路径") + parser.add_argument("--vocab-size", type=int, default=5000, help="目标词表大小 (默认: 5000)") + parser.add_argument("--min-freq", type=int, default=2, help="最小合并频率 (默认: 2)") + parser.add_argument("--output", default="charbpe_fast.json", help="输出 JSON 路径") + args = parser.parse_args() + + if not Path(args.text_file).exists(): + print(f"错误: 文件不存在: {args.text_file}", file=sys.stderr) + sys.exit(1) + + # 限制 vocab_size 范围 + min_size = CHAR_BASE # 至少要能容纳特殊 token + 字节兜底 + 一些字符 + if args.vocab_size < min_size: + print(f"警告: vocab_size={args.vocab_size} 太小，自动调整为 {min_size}") + args.vocab_size = min_size + + print(f"训练文本: {args.text_file}") + print(f"目标词表: {args.vocab_size}, 最小频率: {args.min_freq}") + + # 训练 + tokenizer = train_charbpe(args.text_file, args.vocab_size, args.min_freq) + + # 转换格式 + print("转换为 CharBPE 兼容格式...") + result = convert_to_charbpe_json(tokenizer, max_vocab=args.vocab_size * 3) + + # 保存 + output = Path(args.output) + output.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding='utf-8') + + n_chars = sum(1 for k in result['vocab'] if int(k) >= CHAR_BASE and int(k) < CHAR_BASE + 10000) + n_merges = len(result['merges']) + print(f"\n完成! 保存至: {output}") + print(f" vocab_size: {result['vocab_size']}") + print(f" 字符 token: ~{n_chars}") + print(f" 合并规则: {n_merges}") + + +if __name__ == "__main__": + main() diff --git a/train_vocab.py b/scripts/train_vocab.py similarity index 100% rename from train_vocab.py rename to scripts/train_vocab.py diff --git a/scripts/vocab_to_txt.py b/scripts/vocab_to_txt.py new file mode 100644 index 00000000..8807774d --- /dev/null +++ b/scripts/vocab_to_txt.py @@ -0,0 +1,36 @@ +""" +将分词器 JSON 词表转为纯文本词表（每行一个词）。 +跳过 ID 0~259 的特殊标记和单字节 token。 + +用法: python vocab_to_txt.py [-o output.txt] +""" +import argparse, json, sys +from pathlib import Path + + +def main(): + ap = argparse.ArgumentParser(description="JSON 词表 → 纯文本词表") + ap.add_argument("json", help="词表 JSON 路径") + ap.add_argument("-o", "--output", help="输出路径 (默认: .txt)") + args = ap.parse_args() + + data = json.loads(Path(args.json).read_text(encoding="utf-8")) + vocab = data.get("vocab", {}) + byte_offset = data.get("byte_offset", 4) + + out_path = args.output or str(Path(args.json).with_suffix(".txt")) + count = 0 + with open(out_path, "w", encoding="utf-8") as f: + for tid_str, hex_str in vocab.items(): + tid = int(tid_str) + if tid < byte_offset + 256: + continue + word = bytes.fromhex(hex_str).decode("utf-8", errors="replace") + f.write(word + "\n") + count += 1 + + print(f"已写入 {count} 个词条 → {out_path}") + + +if __name__ == "__main__": + main() diff --git a/shaders/batched_matmul.comp b/shaders/batched_matmul.comp new file mode 100644 index 00000000..bc029598 --- /dev/null +++ b/shaders/batched_matmul.comp @@ -0,0 +1,98 @@ +// ── batched_matmul.comp ────────────────────────────────────────────────── +// 批量分块矩阵乘法原语：对每个 batch b 计算 C_b = op(A_b, B_b) +// +// 布局约定： +// A: (batch * A_rows_per_batch, A_cols) 行主序 — 按 batch 切分为连续行块 +// B: (batch * B_rows_per_batch, B_cols) 行主序 +// C: (batch * M, N) 行主序 — 每个批次结果垂直堆叠 +// +// 转置标志（与 matmul.comp 语义一致）： +// transA=0: A_b 为 (M, K)，A_b[m][k] = A[b*M*K + m*K + k] +// transA=1: A_b 存储为 (K, M) 按 A_b^T 使用，A_b[m][k] = A[b*M*K + k*M + m] +// transB=0: B_b 为 (K, N)，B_b[k][n] = B[b*K*N + k*N + n] +// transB=1: B_b 存储为 (N, K) 按 B_b^T 使用，B_b[k][n] = B[b*K*N + n*K + k] +// +// 工作原理： +// - WorkGroup 大小：16 × 16（XY），Z 维度索引 batch +// - Dispatch: (ceil(N/16), ceil(M/16), batch) +// - 每个 WorkGroup 计算 batch b 内 16×16 的输出分块 +// - 共享内存缓存 A_b 和 B_b 的分块 +// +// 注意：此着色器只包含原语操作（矩阵乘法的批量索引），不含任何算法语义。 +// ───────────────────────────────────────────────────────────────────────── + +#version 450 + +layout(local_size_x = 16, local_size_y = 16) in; + +layout(std430, binding = 0) readonly buffer BufferA { float A[]; }; +layout(std430, binding = 1) readonly buffer BufferB { float B[]; }; +layout(std430, binding = 2) writeonly buffer BufferC { float C[]; }; + +// Push constants：单 batch 逻辑维度 + 转置标志（batch 步长 = M*K = K*N 可推导） +layout(push_constant) uniform PushConstants { + uint M; // 每个 batch 的输出行数 + uint N; // 每个 batch 的输出列数 + uint K; // 收缩维度 + uint transA; // 1 = A_b 存储为 (K,M)，按 A_b^T 使用 + uint transB; // 1 = B_b 存储为 (N,K)，按 B_b^T 使用 +}; + +// 共享内存：缓存当前 batch 的 A_b 和 B_b 的 16×16 分块 +shared float sharedA[16][16]; +shared float sharedB[16][16]; + +// ── 访问 A_b 元素（batch b 由 gl_GlobalInvocationID.z 给出） ────────── +// batch 偏移 = b * M * K（无论 transA，A 的 batch 步长都是 M*K） +float getA(uint b, uint m, uint k) { + uint base = b * M * K; + return (transA != 0u) ? A[base + k * M + m] : A[base + m * K + k]; +} + +// ── 访问 B_b 元素 ───────────────────────────────────────────────────── +// batch 偏移 = b * K * N +float getB(uint b, uint k, uint n) { + uint base = b * K * N; + return (transB != 0u) ? B[base + n * K + k] : B[base + k * N + n]; +} + +void main() +{ + const uint row = gl_GlobalInvocationID.y; // batch 内输出行 [0, M) + const uint col = gl_GlobalInvocationID.x; // batch 内输出列 [0, N) + const uint b = gl_GlobalInvocationID.z; // batch 索引 + const uint lid_x = gl_LocalInvocationID.x; + const uint lid_y = gl_LocalInvocationID.y; + + float sum = 0.0; + + const uint num_tiles = (K + 15u) / 16u; + + for (uint t = 0u; t < num_tiles; ++t) + { + // ── 从全局内存加载分块到共享内存 ────────────────────────────── + const uint a_col = t * 16u + lid_x; + if (row < M && a_col < K) + sharedA[lid_y][lid_x] = getA(b, row, a_col); + else + sharedA[lid_y][lid_x] = 0.0; + + const uint b_row = t * 16u + lid_y; + if (b_row < K && col < N) + sharedB[lid_y][lid_x] = getB(b, b_row, col); + else + sharedB[lid_y][lid_x] = 0.0; + + barrier(); + + // ── 累加部分积 ──────────────────────────────────────────────── + for (uint k = 0u; k < 16u; ++k) + sum += sharedA[lid_y][k] * sharedB[k][lid_x]; + + barrier(); + } + + // ── 写入结果（batch 偏移 = b * M * N） ──────────────────────────── + if (row < M && col < N) + C[b * M * N + row * N + col] = sum; +} diff --git a/shaders/broadcast.comp b/shaders/broadcast.comp new file mode 100644 index 00000000..4adc63a3 --- /dev/null +++ b/shaders/broadcast.comp @@ -0,0 +1,70 @@ +// ── broadcast.comp ───────────────────────────────────────────────────────── +// 广播原语着色器（不含任何算法） +// +// 将一维向量广播到矩阵的每一行或每一列，执行逐元素二元运算。 +// +// 模式： +// mode=0: row_broadcast — A[r][c] = op(A[r][c], vec[r]) +// vec 形状 (rows, 1)，沿列方向广播 +// mode=1: col_broadcast — A[r][c] = op(A[r][c], vec[c]) +// vec 形状 (1, cols)，沿行方向广播 +// +// 二元算子（op）： +// 0=Add, 1=Sub, 2=Mul, 3=Div, 4=Max, 5=Min +// +// 绑定布局： +// binding 0: 输入 A (readonly, rows×cols) +// binding 1: 输入 vec (readonly, rows×1 或 1×cols) +// binding 2: 输出 OUT (writeonly, rows×cols) +// +// Push Constants： +// uint rows — 矩阵行数 +// uint cols — 矩阵列数 +// uint mode — 0=row_broadcast, 1=col_broadcast +// uint op — 二元算子编号 +// ───────────────────────────────────────────────────────────────────────── + +#version 450 + +layout(local_size_x = 256) in; + +layout(std430, binding = 0) readonly buffer BufA { float buf_a[]; }; +layout(std430, binding = 1) readonly buffer BufVec { float buf_vec[]; }; +layout(std430, binding = 2) writeonly buffer BufOut { float buf_out[]; }; + +layout(push_constant) uniform PushConstants { + uint rows; + uint cols; + uint mode; + uint op; +}; + +float apply_binary(uint op_id, float a, float b) +{ + if (op_id == 0u) return a + b; + if (op_id == 1u) return a - b; + if (op_id == 2u) return a * b; + if (op_id == 3u) return a / b; + if (op_id == 4u) return max(a, b); + if (op_id == 5u) return min(a, b); + return 0.0; +} + +void main() +{ + const uint idx = gl_GlobalInvocationID.x; + const uint total = rows * cols; + if (idx >= total) return; + + const uint r = idx / cols; + const uint c = idx % cols; + + const float a = buf_a[idx]; + + // row_broadcast: vec 索引按行号 r + // col_broadcast: vec 索引按列号 c + const uint vec_idx = (mode == 0u) ? r : c; + const float v = buf_vec[vec_idx]; + + buf_out[idx] = apply_binary(op, a, v); +} diff --git a/shaders/elementwise_v2.comp b/shaders/elementwise_v2.comp new file mode 100644 index 00000000..14e57edc --- /dev/null +++ b/shaders/elementwise_v2.comp @@ -0,0 +1,152 @@ +// ── elementwise_v2.comp ─────────────────────────────────────────────────── +// 纯原语逐元素运算着色器（不含任何算法） +// +// 设计原则：本着色器只提供 op-level 原语（加减乘除、exp、sqrt、max、min、 +// select 等），绝不包含 ReLU、GeLU、Softmax、LayerNorm 等算法。 +// 算法由 Layer 层通过组合原语表达。 +// +// 绑定布局： +// binding 0: 输入 A（条件操作数 1 / 主操作数） +// binding 1: 输入 B（条件操作数 2 / then 张量 / 次操作数） +// binding 2: 输入 C（else 张量，仅 SELECT 模式使用） +// binding 3: 输出 OUT +// +// Push Constants： +// uint count — 元素总数 +// uint mode — 运算模式（见下） +// uint op — 一元/二元算子编号（mode 0/1 使用） +// uint cmp_op — 比较算子编号（mode 2 SELECT 使用） +// uint flags — 位标志： +// bit0: B 为标量（用 scalar_b 替代 B[idx]） +// bit1: then 为标量（用 scalar_then 替代 B[idx]） +// bit2: else 为标量（用 scalar_else 替代 C[idx]） +// bit3: 二元运算标量在前（scalar op A，仅 mode 1） +// float scalar_b — 标量 B / 条件操作数 2 标量 +// float scalar_then — then 标量值 +// float scalar_else — else 标量值 +// +// 运算模式（mode）： +// 0 = UNARY: out = unary_op(A[idx]) +// 1 = BINARY: out = binary_op(a, b) +// a = A[idx] +// b = (flags & 1) ? scalar_b : B[idx] +// 若 flags & 8: out = binary_op(b, a) [标量在前] +// 2 = SELECT: out = compare_op(a, b) ? then_val : else_val +// a = A[idx] +// b = (flags & 1) ? scalar_b : B[idx] +// then_val = (flags & 2) ? scalar_then : B[idx] +// else_val = (flags & 4) ? scalar_else : C[idx] +// 3 = AXPY: out = A[idx] + scalar_b * B[idx] (融合 axpy：1 次 dispatch 替代 3 次) +// 用于 Optimizer 的 scale_add_(dst, scalar, src) 操作 +// +// 一元算子（op）： +// 0=Neg, 1=Exp, 2=Log, 3=Sqrt, 4=Rsqrt, 5=Abs, 6=Tanh +// +// 二元算子（op）： +// 0=Add, 1=Sub, 2=Mul, 3=Div, 4=Max, 5=Min +// +// 比较算子（cmp_op）： +// 0=Lt, 1=Le, 2=Gt, 3=Ge, 4=Eq, 5=Ne +// +// ───────────────────────────────────────────────────────────────────────── + +#version 450 + +layout(local_size_x = 256) in; + +layout(std430, binding = 0) readonly buffer BufA { float buf_a[]; }; +layout(std430, binding = 1) readonly buffer BufB { float buf_b[]; }; +layout(std430, binding = 2) readonly buffer BufC { float buf_c[]; }; +layout(std430, binding = 3) writeonly buffer BufOut { float buf_out[]; }; + +layout(push_constant) uniform PushConstants { + uint count; + uint mode; + uint op; + uint cmp_op; + uint flags; + float scalar_b; + float scalar_then; + float scalar_else; +}; + +// ── 一元运算 ────────────────────────────────────────────────────────────── +float apply_unary(uint op, float x) +{ + if (op == 0u) return -x; // Neg + if (op == 1u) return exp(x); // Exp + if (op == 2u) return log(x); // Log + if (op == 3u) return sqrt(x); // Sqrt + if (op == 4u) return 1.0 / sqrt(x); // Rsqrt + if (op == 5u) return abs(x); // Abs + if (op == 6u) return tanh(x); // Tanh + return 0.0; +} + +// ── 二元运算 ────────────────────────────────────────────────────────────── +float apply_binary(uint op, float a, float b) +{ + if (op == 0u) return a + b; // Add + if (op == 1u) return a - b; // Sub + if (op == 2u) return a * b; // Mul + if (op == 3u) return a / b; // Div + if (op == 4u) return max(a, b); // Max + if (op == 5u) return min(a, b); // Min + return 0.0; +} + +// ── 比较运算 ────────────────────────────────────────────────────────────── +bool apply_compare(uint op, float a, float b) +{ + if (op == 0u) return a < b; // Lt + if (op == 1u) return a <= b; // Le + if (op == 2u) return a > b; // Gt + if (op == 3u) return a >= b; // Ge + if (op == 4u) return a == b; // Eq + if (op == 5u) return a != b; // Ne + return false; +} + +void main() +{ + const uint idx = gl_GlobalInvocationID.x; + if (idx >= count) return; + + const float a = buf_a[idx]; + + // ── 模式 0：一元运算 ── + if (mode == 0u) + { + buf_out[idx] = apply_unary(op, a); + return; + } + + // ── 模式 1：二元运算 ── + if (mode == 1u) + { + const float b = ((flags & 1u) != 0u) ? scalar_b : buf_b[idx]; + if ((flags & 8u) != 0u) + buf_out[idx] = apply_binary(op, b, a); // 标量在前 + else + buf_out[idx] = apply_binary(op, a, b); // 张量在前 + return; + } + + // ── 模式 2：条件选择 ── + if (mode == 2u) + { + const float b = ((flags & 1u) != 0u) ? scalar_b : buf_b[idx]; + const float then_v = ((flags & 2u) != 0u) ? scalar_then : buf_b[idx]; + const float else_v = ((flags & 4u) != 0u) ? scalar_else : buf_c[idx]; + buf_out[idx] = apply_compare(cmp_op, a, b) ? then_v : else_v; + return; + } + + // ── 模式 3：AXPY（融合 axpy：out = a + scalar_b * b）── + if (mode == 3u) + { + const float b = buf_b[idx]; + buf_out[idx] = a + scalar_b * b; + return; + } +} diff --git a/shaders/gather.comp b/shaders/gather.comp new file mode 100644 index 00000000..a1cc9dd7 --- /dev/null +++ b/shaders/gather.comp @@ -0,0 +1,51 @@ +// ── gather.comp ─────────────────────────────────────────────────────────── +// 纯原语：按行索引从表中查表（gather_rows） +// +// 语义： +// table: (vocab, D) — 行主序存储 +// indices: (num,) — 行索引（float 编码的整数） +// output: (num, D) — output[i] = table[indices[i]] +// +// 越界索引返回零行（防御性，与 CPU fallback 行为一致）。 +// +// 绑定布局： +// binding 0: table (readonly) +// binding 1: indices (readonly) +// binding 2: output (writeonly) +// +// Push Constants（12 bytes）： +// uint vocab — table 行数 +// uint D — 列宽（每行元素数） +// uint num — indices 元素个数 +// ───────────────────────────────────────────────────────────────────────── + +#version 450 + +layout(local_size_x = 256) in; + +layout(std430, binding = 0) readonly buffer BufTable { float table[]; }; +layout(std430, binding = 1) readonly buffer BufIndices { float indices[]; }; +layout(std430, binding = 2) writeonly buffer BufOut { float out_data[]; }; + +layout(push_constant) uniform PushConstants { + uint vocab; + uint D; + uint num; +}; + +void main() +{ + const uint flat_idx = gl_GlobalInvocationID.x; + const uint total = num * D; + if (flat_idx >= total) return; + + const uint i = flat_idx / D; // 第 i 个索引 + const uint d = flat_idx % D; // 第 d 列 + + const uint row_idx = uint(indices[i]); + + if (row_idx < vocab) + out_data[flat_idx] = table[row_idx * D + d]; + else + out_data[flat_idx] = 0.0; // 越界返回零 +} diff --git a/shaders/matmul.comp b/shaders/matmul.comp new file mode 100644 index 00000000..7f8f1fe6 --- /dev/null +++ b/shaders/matmul.comp @@ -0,0 +1,92 @@ +// ── matmul.comp ───────────────────────────────────────────────────────── +// 分块矩阵乘法计算着色器：C[M][N] = A × B +// +// 支持转置： +// - transA=0, transB=0: C[M][N] = A[M][K] × B[K][N] +// - transA=1, transB=0: C[M][N] = A^T[K,M] × B[K][N] （A 存储为 (K,M)） +// - transA=0, transB=1: C[M][N] = A[M][K] × B^T[N,K] （B 存储为 (N,K)） +// - transA=1, transB=1: C[M][N] = A^T[K,M] × B^T[N,K] +// +// 工作原理： +// - WorkGroup 大小：16 × 16（256 线程，适配大多数 GPU） +// - 每个 WorkGroup 计算 16×16 的输出分块 +// - 共享内存（shared memory）缓存 A 和 B 的分块，减少全局内存访问 +// - 多轮迭代 K 维度，每轮加载 16×16 到 shared memory +// ───────────────────────────────────────────────────────────────────────── + +#version 450 + +layout(local_size_x = 16, local_size_y = 16) in; + +// Storage buffers（float32） +layout(std430, binding = 0) readonly buffer BufferA { float A[]; }; +layout(std430, binding = 1) readonly buffer BufferB { float B[]; }; +layout(std430, binding = 2) writeonly buffer BufferC { float C[]; }; + +// Push constants：矩阵维度 + 转置标志 +layout(push_constant) uniform PushConstants { + uint M; // C 的行数 + uint N; // C 的列数 + uint K; // 求和维度 + uint transA; // 1 = A 存储为 (K,M)，按 A^T 使用 + uint transB; // 1 = B 存储为 (N,K)，按 B^T 使用 +}; + +// 共享内存：缓存 A 和 B 的 16×16 分块 +shared float sharedA[16][16]; +shared float sharedB[16][16]; + +// ── 访问 A 元素 ───────────────────────────────────────────────────────── +// transA=0: A[m][k] = A[m * K + k] （标准行主序 (M,K)） +// transA=1: A[m][k] = A[k * M + m] （转置存储 (K,M)） +float getA(uint m, uint k) { + return (transA != 0u) ? A[k * M + m] : A[m * K + k]; +} + +// ── 访问 B 元素 ───────────────────────────────────────────────────────── +// transB=0: B[k][n] = B[k * N + n] （标准行主序 (K,N)） +// transB=1: B[k][n] = B[n * K + k] （转置存储 (N,K)） +float getB(uint k, uint n) { + return (transB != 0u) ? B[n * K + k] : B[k * N + n]; +} + +void main() +{ + const uint row = gl_GlobalInvocationID.y; // 输出行 + const uint col = gl_GlobalInvocationID.x; // 输出列 + const uint lid_x = gl_LocalInvocationID.x; + const uint lid_y = gl_LocalInvocationID.y; + + float sum = 0.0; + + // K 维度的分块数 + const uint num_tiles = (K + 15u) / 16u; + + for (uint t = 0u; t < num_tiles; ++t) + { + // ── 从全局内存加载分块到共享内存 ────────────────────────────── + const uint a_col = t * 16u + lid_x; + if (row < M && a_col < K) + sharedA[lid_y][lid_x] = getA(row, a_col); + else + sharedA[lid_y][lid_x] = 0.0; + + const uint b_row = t * 16u + lid_y; + if (b_row < K && col < N) + sharedB[lid_y][lid_x] = getB(b_row, col); + else + sharedB[lid_y][lid_x] = 0.0; + + barrier(); + + // ── 累加部分积 ──────────────────────────────────────────────── + for (uint k = 0u; k < 16u; ++k) + sum += sharedA[lid_y][k] * sharedB[k][lid_x]; + + barrier(); + } + + // ── 写入结果 ────────────────────────────────────────────────────── + if (row < M && col < N) + C[row * N + col] = sum; +} diff --git a/shaders/matmul_tiled.comp b/shaders/matmul_tiled.comp new file mode 100644 index 00000000..b02bf44d --- /dev/null +++ b/shaders/matmul_tiled.comp @@ -0,0 +1,139 @@ +// ── matmul_tiled.comp ───────────────────────────────────────────────────── +// 4×4 线程粗化分块矩阵乘法：C[M][N] = A × B +// +// 支持转置（同 matmul.comp）： +// - transA=0, transB=0: C[M][N] = A[M][K] × B[K][N] +// - transA=1, transB=0: C[M][N] = A^T[K,M] × B[K][N] +// - transA=0, transB=1: C[M][N] = A[M][K] × B^T[N,K] +// - transA=1, transB=1: C[M][N] = A^T[K,M] × B^T[N,K] +// +// 优化策略： +// - WorkGroup：16×16 线程，计算 64×64 输出块（每线程 4×4 元素） +// - 寄存器分块：每线程在寄存器中维护 4×4 累加和，大幅提升计算访存比 +// - Shared Memory Padding：17/65 列宽避免 Bank Conflict +// - 向量化加载：每个线程每轮加载 4 个元素到共享内存 +// ───────────────────────────────────────────────────────────────────────── + +#version 450 + +layout(local_size_x = 16, local_size_y = 16) in; + +layout(std430, binding = 0) readonly buffer BufferA { float A[]; }; +layout(std430, binding = 1) readonly buffer BufferB { float B[]; }; +layout(std430, binding = 2) writeonly buffer BufferC { float C[]; }; + +layout(push_constant) uniform PushConstants { + uint M; // C 的行数 + uint N; // C 的列数 + uint K; // 求和维度 + uint transA; // 1 = A 存储为 (K,M)，按 A^T 使用 + uint transB; // 1 = B 存储为 (N,K)，按 B^T 使用 +}; + +// 分块维度常量 +const uint BM = 64u; // 输出块行数 +const uint BN = 64u; // 输出块列数 +const uint BK = 16u; // K 维度分块宽度 + +// Shared Memory：Padding 避免 Bank Conflict +shared float tileA[64][17]; +shared float tileB[16][65]; + +// ── 访问 A 元素（转置感知）───────────────────────────────────────────── +// transA=0: A[m][k] = A[m * K + k] （标准行主序 (M,K)） +// transA=1: A[m][k] = A[k * M + m] （转置存储 (K,M)） +float getA(uint m, uint k) { + return (transA != 0u) ? A[k * M + m] : A[m * K + k]; +} + +// ── 访问 B 元素（转置感知）───────────────────────────────────────────── +// transB=0: B[k][n] = B[k * N + n] （标准行主序 (K,N)） +// transB=1: B[k][n] = B[n * K + k] （转置存储 (N,K)） +float getB(uint k, uint n) { + return (transB != 0u) ? B[n * K + k] : B[k * N + n]; +} + +void main() +{ + const uint tx = gl_LocalInvocationID.x; // 0..15 + const uint ty = gl_LocalInvocationID.y; // 0..15 + const uint tid = ty * 16u + tx; // 线性线程 ID 0..255 + + // 当前 WorkGroup 负责的全局输出块起始坐标 + const uint block_row = gl_WorkGroupID.y * BM; + const uint block_col = gl_WorkGroupID.x * BN; + + // ── 寄存器分块：4×4 累加器 ────────────────────────────────────── + float acc[4][4]; + for (uint i = 0u; i < 4u; ++i) + for (uint j = 0u; j < 4u; ++j) + acc[i][j] = 0.0; + + const uint num_tiles = (K + BK - 1u) / BK; + + for (uint t = 0u; t < num_tiles; ++t) + { + // ── 协作加载 tileA[64][16]：1024 元素，256 线程，每线程 4 元素 ── + { + const uint a_base_col = t * BK; + for (uint load = 0u; load < 4u; ++load) + { + const uint elem = tid * 4u + load; // 0..1023 + const uint r = elem / BK; // 0..63 + const uint c = elem % BK; // 0..15 + const uint gr = block_row + r; + const uint gc = a_base_col + c; + tileA[r][c] = (gr < M && gc < K) ? getA(gr, gc) : 0.0; + } + } + + // ── 协作加载 tileB[16][64]：1024 元素，256 线程，每线程 4 元素 ── + { + const uint b_base_row = t * BK; + for (uint load = 0u; load < 4u; ++load) + { + const uint elem = tid * 4u + load; + const uint r = elem / BN; // 0..15 + const uint c = elem % BN; // 0..63 + const uint gr = b_base_row + r; + const uint gc = block_col + c; + tileB[r][c] = (gr < K && gc < N) ? getB(gr, gc) : 0.0; + } + } + + barrier(); + + // ── 4×4 外积累加 ──────────────────────────────────────────── + for (uint k = 0u; k < BK; ++k) + { + // 每个线程读取 4 个 A 元素和 4 个 B 元素 + float a_vals[4]; + float b_vals[4]; + + for (uint i = 0u; i < 4u; ++i) + a_vals[i] = tileA[ty * 4u + i][k]; + for (uint j = 0u; j < 4u; ++j) + b_vals[j] = tileB[k][tx * 4u + j]; + + // 外积累加 + for (uint i = 0u; i < 4u; ++i) + for (uint j = 0u; j < 4u; ++j) + acc[i][j] += a_vals[i] * b_vals[j]; + } + + barrier(); + } + + // ── 写入结果 ────────────────────────────────────────────────────── + for (uint i = 0u; i < 4u; ++i) + { + const uint out_row = block_row + ty * 4u + i; + if (out_row >= M) continue; + for (uint j = 0u; j < 4u; ++j) + { + const uint out_col = block_col + tx * 4u + j; + if (out_col >= N) continue; + C[out_row * N + out_col] = acc[i][j]; + } + } +} diff --git a/shaders/rearrange_3d.comp b/shaders/rearrange_3d.comp new file mode 100644 index 00000000..5caf9440 --- /dev/null +++ b/shaders/rearrange_3d.comp @@ -0,0 +1,63 @@ +// ── rearrange_3d.comp ─────────────────────────────────────────────────────── +// 纯原语：3D 维度转置 (M, B, N) ↔ (B, M, N) +// +// 设计原则：本着色器只提供数据重排原语，不含任何算法语义。 +// +// inverse=0: 输入 (M, B*N) → 输出 (B*M, N) +// out[b*M + m, n] = in[m, b*N + n] +// inverse=1: 输入 (B*M, N) → 输出 (M, B*N) +// out[m, b*N + n] = in[b*M + m, n] +// +// Push Constants： +// uint M — 第一维度大小 +// uint B — 第二维度大小 +// uint N — 第三维度大小 +// uint inverse — 0=forward, 1=inverse +// uint total — 元素总数 M*B*N +// ───────────────────────────────────────────────────────────────────────── + +#version 450 + +layout(local_size_x = 256) in; + +layout(std430, binding = 0) readonly buffer BufferIn { float in_data[]; }; +layout(std430, binding = 1) writeonly buffer BufferOut { float out_data[]; }; + +layout(push_constant) uniform PushConstants { + uint M; + uint B; + uint N; + uint inverse; + uint total; +}; + +void main() +{ + const uint idx = gl_GlobalInvocationID.x; + if (idx >= total) return; + + // 把线性索引 idx 分解为 (m, b, n) 或 (b, m, n) + if (inverse == 0u) + { + // 输入按 (M, B*N) 存储：idx = m * (B*N) + b * N + n + const uint n = idx % N; + const uint rest = idx / N; + const uint b = rest % B; + const uint m = rest / B; + // 输出按 (B*M, N) 存储：out_idx = (b * M + m) * N + n + const uint out_idx = (b * M + m) * N + n; + out_data[out_idx] = in_data[idx]; + } + else + { + // 输入按 (B*M, N) 存储：idx = (b * M + m) * N + n + const uint n = idx % N; + const uint rest = idx / N; + const uint bm = rest; // b * M + m + const uint m = bm % M; + const uint b = bm / M; + // 输出按 (M, B*N) 存储：out_idx = m * (B * N) + b * N + n + const uint out_idx = m * (B * N) + b * N + n; + out_data[out_idx] = in_data[idx]; + } +} diff --git a/shaders/reduce.comp b/shaders/reduce.comp new file mode 100644 index 00000000..814c3eac --- /dev/null +++ b/shaders/reduce.comp @@ -0,0 +1,119 @@ +// ── reduce.comp ─────────────────────────────────────────────────────────── +// 归约原语着色器（工作组级并行归约） +// +// 只提供求和/最大值归约原语，供 Layer 组合表达 mean、var、grad_b 等。 +// +// 模式： +// mode=0: row_reduce — output[r] = reduce_op(A[r*cols + c]) over c +// 输入 (rows, cols) → 输出 (rows, 1) +// mode=1: col_reduce — output[c] = reduce_op(A[r*cols + c]) over r +// 输入 (rows, cols) → 输出 (1, cols) +// +// 归约算子： +// reduce_op=0: sum +// reduce_op=1: max +// +// 绑定布局： +// binding 0: 输入 A (readonly) +// binding 1: 输出 OUT (writeonly) +// +// Push Constants： +// uint rows — 输入行数 +// uint cols — 输入列数 +// uint mode — 0=row_reduce, 1=col_reduce +// uint reduce_op — 0=sum, 1=max +// +// Dispatch： +// row_reduce: (rows, 1, 1) — 每个工作组（256线程）协作归约一行 +// col_reduce: (cols, 1, 1) — 每个工作组（256线程）协作归约一列 +// +// 优化：工作组级共享内存并行归约。 +// 旧实现每线程串行扫描整行/整列（O(cols) 或 O(rows)），大矩阵严重超时。 +// 新实现每工作组 256 线程 strided loop + shared memory tree reduce， +// 每线程工作量降至 O(cols/256)，避免 TDR 超时。 +// ───────────────────────────────────────────────────────────────────────── + +#version 450 + +layout(local_size_x = 256) in; + +layout(std430, binding = 0) readonly buffer BufIn { float data[]; }; +layout(std430, binding = 1) writeonly buffer BufOut { float result[]; }; + +layout(push_constant) uniform PushConstants { + uint rows; + uint cols; + uint mode; + uint reduce_op; +}; + +shared float sdata[256]; + +void main() +{ + const uint tid = gl_LocalInvocationID.x; + + // ── 行归约：工作组 ID = 行索引，256 线程协作归约一行 ── + if (mode == 0u) + { + const uint row = gl_WorkGroupID.x; + if (row >= rows) return; + + // strided loop：每个线程累加 cols/256 个元素 + float acc = (reduce_op == 0u) ? 0.0 : uintBitsToFloat(0xff800000u); // sum=0, max=-Inf + const uint base = row * cols; + for (uint c = tid; c < cols; c += 256u) + { + float v = data[base + c]; + if (reduce_op == 0u) acc += v; + else acc = max(acc, v); + } + sdata[tid] = acc; + barrier(); + + // 共享内存树形归约 + for (uint s = 128u; s > 0u; s >>= 1u) + { + if (tid < s) + { + if (reduce_op == 0u) sdata[tid] += sdata[tid + s]; + else sdata[tid] = max(sdata[tid], sdata[tid + s]); + } + barrier(); + } + + if (tid == 0u) result[row] = sdata[0]; + return; + } + + // ── 列归约：工作组 ID = 列索引，256 线程协作归约一列 ── + if (mode == 1u) + { + const uint col = gl_WorkGroupID.x; + if (col >= cols) return; + + // strided loop：每个线程累加 rows/256 个元素 + float acc = (reduce_op == 0u) ? 0.0 : uintBitsToFloat(0xff800000u); + for (uint r = tid; r < rows; r += 256u) + { + float v = data[r * cols + col]; + if (reduce_op == 0u) acc += v; + else acc = max(acc, v); + } + sdata[tid] = acc; + barrier(); + + for (uint s = 128u; s > 0u; s >>= 1u) + { + if (tid < s) + { + if (reduce_op == 0u) sdata[tid] += sdata[tid + s]; + else sdata[tid] = max(sdata[tid], sdata[tid + s]); + } + barrier(); + } + + if (tid == 0u) result[col] = sdata[0]; + return; + } +} diff --git a/shaders/scatter_add.comp b/shaders/scatter_add.comp new file mode 100644 index 00000000..38c8606b --- /dev/null +++ b/shaders/scatter_add.comp @@ -0,0 +1,69 @@ +// ── scatter_add.comp ────────────────────────────────────────────────────── +// 纯原语：按行索引原子累加梯度（scatter_add_rows） +// +// 语义： +// dst: (vocab, D) — 读写，原地累加 +// indices: (num,) — 行索引（float 编码的整数） +// grad: (num, D) — 待累加梯度 +// +// dst[indices[i]][d] += grad[i][d] +// +// 使用 CAS 循环实现 float atomicAdd，兼容所有 Vulkan 设备。 +// 越界索引被跳过（不修改 dst）。 +// +// 绑定布局： +// binding 0: dst (readwrite, coherent — float 数据以 uint 存储用于原子操作) +// binding 1: indices (readonly) +// binding 2: grad (readonly) +// +// Push Constants（12 bytes）： +// uint vocab — dst 行数 +// uint D — 列宽 +// uint num — indices 元素个数 +// ───────────────────────────────────────────────────────────────────────── + +#version 450 + +layout(local_size_x = 256) in; + +// dst 以 uint 声明，用于 atomicCompSwap 操作 +// float 数据位模式直接存储为 uint（IEEE 754 位兼容） +layout(std430, binding = 0) coherent buffer BufDst { uint dst[]; }; +layout(std430, binding = 1) readonly buffer BufIndices { float indices[]; }; +layout(std430, binding = 2) readonly buffer BufGrad { float grad[]; }; + +layout(push_constant) uniform PushConstants { + uint vocab; + uint D; + uint num; +}; + +// ── 基于 CAS 的 float atomicAdd ────────────────────────────────────────── +// 兼容所有 Vulkan 1.0 设备，无需 VK_EXT_shader_atomic_float 扩展。 +void atomicAddFloat(uint idx, float val) +{ + uint old_v = dst[idx]; + uint new_v; + while (true) + { + new_v = floatBitsToUint(uintBitsToFloat(old_v) + val); + uint prev = atomicCompSwap(dst[idx], old_v, new_v); + if (prev == old_v) break; + old_v = prev; + } +} + +void main() +{ + const uint flat_idx = gl_GlobalInvocationID.x; + const uint total = num * D; + if (flat_idx >= total) return; + + const uint i = flat_idx / D; // 第 i 个索引 + const uint d = flat_idx % D; // 第 d 列 + + const uint row_idx = uint(indices[i]); + if (row_idx >= vocab) return; // 越界跳过 + + atomicAddFloat(row_idx * D + d, grad[flat_idx]); +} diff --git a/shaders/transpose.comp b/shaders/transpose.comp new file mode 100644 index 00000000..b2762b6b --- /dev/null +++ b/shaders/transpose.comp @@ -0,0 +1,36 @@ +// ── transpose.comp ──────────────────────────────────────────────────────── +// 纯原语：矩阵转置 A (R, C) → OUT (C, R) +// +// out[c * R + r] = in[r * C + c] +// +// 绑定布局： +// binding 0: 输入 A (readonly, R×C) +// binding 1: 输出 OUT (writeonly, C×R) +// +// Push Constants（8 bytes）： +// uint rows — 输入行数 R +// uint cols — 输入列数 C +// ───────────────────────────────────────────────────────────────────────── + +#version 450 + +layout(local_size_x = 256) in; + +layout(std430, binding = 0) readonly buffer BufIn { float in_data[]; }; +layout(std430, binding = 1) writeonly buffer BufOut { float out_data[]; }; + +layout(push_constant) uniform PushConstants { + uint rows; + uint cols; +}; + +void main() +{ + const uint idx = gl_GlobalInvocationID.x; + const uint total = rows * cols; + if (idx >= total) return; + + const uint r = idx / cols; + const uint c = idx % cols; + out_data[c * rows + r] = in_data[idx]; +} diff --git a/src/attn_consistency_test.cpp b/src/attn_consistency_test.cpp new file mode 100644 index 00000000..c2cacc10 --- /dev/null +++ b/src/attn_consistency_test.cpp @@ -0,0 +1,345 @@ +// ── Attention 一致性验证：forward vs forward_step ────────────────────────── +// +// 目的：验证 KV cache 增量推理（forward_step）与整段推理（forward）在相同 +// 输入下产生相同的输出。若一致则证明 Attention 语义修复正确， +// 生成乱码是旧模型权重的问题（用 buggy 代码训练得到），需重训。 +// +// 原理： +// - forward: 输入整段 token 序列，带因果掩码，输出 (vocab, seq) logits +// - forward_step: 逐 token 增量推理，KV cache 复用前文，输出 (vocab, 1) logits +// - forward 在位置 i 的 logits（带因果掩码，只能看到 0..i） +// 应与 forward_step 在 cur_len=i 时的 logits 完全一致 +// +// 用法：attn_consistency_test [--gpu] [--seq N] [--layers N] +// --gpu 启用 GPU（Vulkan） +// --seq N 测试序列长度（默认 16） +// --layers N Transformer 层数（默认 2） +// ───────────────────────────────────────────────────────────────────────── + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using nn::Scalar; +using nn::Matrix; +using nn::Tensor; +using nn::ComputeEngine; +using nn::GPTModel; +using nn::PosEncodingType; + +namespace { + +// ── 辅助：从 (vocab, seq) logits 矩阵取第 col 列 → 向量 ── +std::vector extract_logits_column( + const Matrix& logits, std::size_t col) +{ + const std::size_t vocab = logits.rows(); + std::vector out(vocab); + for (std::size_t v = 0; v < vocab; ++v) + out[v] = logits.at_unchecked(v, col); + return out; +} + +// ── 辅助：从 (vocab, 1) logits 矩阵取第 0 列 → 向量 ── +std::vector extract_logits_single(const Matrix& logits) +{ + return extract_logits_column(logits, 0); +} + +// ── 辅助：两个向量的最大绝对差 ── +Scalar max_abs_diff(const std::vector& a, const std::vector& b) +{ + Scalar m = Scalar{0}; + const std::size_t n = a.size(); + for (std::size_t i = 0; i < n; ++i) + { + Scalar d = std::fabs(a[i] - b[i]); + if (d > m) m = d; + } + return m; +} + +// ── 辅助：打印向量前 N 个元素 ── +void print_vec_head(const std::vector& v, std::size_t n = 8) +{ + std::cout << "["; + const std::size_t m = std::min(n, v.size()); + for (std::size_t i = 0; i < m; ++i) + { + std::cout << std::scientific << std::setprecision(4) << v[i]; + if (i + 1 < m) std::cout << ", "; + } + if (v.size() > m) std::cout << ", ..."; + std::cout << "]"; +} + +// ── 单引擎一致性测试 ── +// 返回 true 表示通过（forward 与 forward_step 输出一致） +bool test_consistency( + ComputeEngine& engine, + const std::string& engine_name, + std::size_t seq_len, + std::size_t num_layers, + PosEncodingType pos_enc, + const std::string& pos_enc_name, + bool use_batch = true) +{ + // 模型超参（小规模，加速测试） + const std::size_t vocab_size = 100; + const std::size_t d_model = 32; + const std::size_t num_heads = 4; + const std::size_t d_ff = 64; + + std::cout << "\n────────────────────────────────────────────────────────\n" + << "引擎: " << engine_name + << " | 位置编码: " << pos_enc_name + << " | seq=" << seq_len + << " | layers=" << num_layers + << " | batch=" << (use_batch ? "on" : "off") << "\n" + << "────────────────────────────────────────────────────────\n"; + + // ── 1. 构建模型 ── + std::cout << "[1/4] 构建 GPTModel..." << std::flush; + GPTModel model( + engine, vocab_size, d_model, seq_len, + num_heads, d_ff, num_layers, pos_enc); + std::cout << " 完成\n"; + + // ── 2. 生成随机 token 序列 ── + std::cout << "[2/4] 生成随机 token 序列..." << std::flush; + std::mt19937_64 rng(12345); + std::uniform_int_distribution tok_dist(0, vocab_size - 1); + std::vector tokens(seq_len); + for (auto& t : tokens) t = tok_dist(rng); + std::cout << " 完成 (tokens:"; + for (std::size_t i = 0; i < std::min(8, seq_len); ++i) + std::cout << " " << tokens[i]; + if (seq_len > 8) std::cout << " ..."; + std::cout << ")\n"; + + // ── 3. 整段 forward ── + std::cout << "[3/4] 运行 forward（整段推理）..." << std::flush; + Matrix tok_mat(seq_len, 1); + for (std::size_t t = 0; t < seq_len; ++t) + tok_mat.set_value_unchecked(t, 0, static_cast(tokens[t])); + auto tok_t = engine.from_matrix(tok_mat); + if (!tok_t) + { + std::cout << "\n ❌ from_matrix 失败: " << tok_t.error().message << "\n"; + return false; + } + + if (use_batch) + { + auto begin_batch_r = engine.begin_batch(); + if (!begin_batch_r) + { + std::cout << "\n ❌ begin_batch 失败: " << begin_batch_r.error().message << "\n"; + return false; + } + } + auto fwd_r = model.forward(engine, *tok_t); + if (!fwd_r) + { + std::cout << "\n ❌ forward 失败: " << fwd_r.error().message << "\n"; + return false; + } + if (use_batch) + { + auto end_batch_r = engine.end_batch(); + if (!end_batch_r) + { + std::cout << "\n ❌ end_batch 失败: " << end_batch_r.error().message << "\n"; + return false; + } + } + + auto fwd_logits_m = engine.to_matrix(*fwd_r); + if (!fwd_logits_m) + { + std::cout << "\n ❌ to_matrix 失败: " << fwd_logits_m.error().message << "\n"; + return false; + } + std::cout << " 完成 (logits: " << fwd_logits_m->rows() + << " × " << fwd_logits_m->cols() << ")\n"; + + // ── 4. 逐 token forward_step ── + std::cout << "[4/4] 运行 forward_step（增量推理）并对比..." << std::flush; + + // 预分配 KV cache + std::vector k_caches, v_caches; + k_caches.reserve(num_layers); + v_caches.reserve(num_layers); + for (std::size_t i = 0; i < num_layers; ++i) + { + k_caches.push_back(engine.create_tensor(seq_len, d_model)); + v_caches.push_back(engine.create_tensor(seq_len, d_model)); + auto r1 = engine.zero(k_caches.back()); + if (!r1) { std::cout << "\n ❌ zero(k_cache) 失败\n"; return false; } + auto r2 = engine.zero(v_caches.back()); + if (!r2) { std::cout << "\n ❌ zero(v_cache) 失败\n"; return false; } + } + + Scalar total_max_diff = Scalar{0}; + std::size_t worst_pos = 0; + bool all_ok = true; + + for (std::size_t i = 0; i < seq_len; ++i) + { + // 增量推理：cur_len = i（cache 中已有 i 个 token，准备写入第 i 个） + if (use_batch) + { + auto br = engine.begin_batch(); + if (!br) { std::cout << "\n ❌ begin_batch 失败\n"; return false; } + } + + auto step_r = model.forward_step( + engine, tokens[i], i, k_caches, v_caches, i); + if (!step_r) + { + std::cout << "\n ❌ forward_step 在 pos=" << i + << " 失败: " << step_r.error().message << "\n"; + return false; + } + + if (use_batch) + { + auto er = engine.end_batch(); + if (!er) { std::cout << "\n ❌ end_batch 失败\n"; return false; } + } + + auto step_logits_m = engine.to_matrix(*step_r); + if (!step_logits_m) + { + std::cout << "\n ❌ to_matrix 失败\n"; + return false; + } + + // 对比 forward 的第 i 列 + auto fwd_col = extract_logits_column(*fwd_logits_m, i); + auto step_col = extract_logits_single(*step_logits_m); + Scalar diff = max_abs_diff(fwd_col, step_col); + + if (diff > total_max_diff) + { + total_max_diff = diff; + worst_pos = i; + } + + // 容差：ffast-math 下数值误差会放大，但同一引擎内 forward 与 + // forward_step 的数学等价性应保持在 1e-2 量级以内 + const Scalar tolerance = 5e-2f; + if (diff > tolerance) + { + all_ok = false; + std::cout << "\n ❌ pos=" << i << " 不一致 (max_diff=" + << std::scientific << std::setprecision(4) << diff << ")\n"; + std::cout << " forward: "; + print_vec_head(fwd_col); + std::cout << "\n forward_step: "; + print_vec_head(step_col); + std::cout << "\n"; + } + } + + std::cout << "\n"; + std::cout << " 最大绝对误差: " << std::scientific << std::setprecision(6) + << total_max_diff << " (位置 " << worst_pos << ")\n"; + std::cout << " 结果: " << (all_ok ? "✅ 通过" : "❌ 失败") << "\n"; + + return all_ok; +} + +} // namespace + +int main(int argc, char* argv[]) +{ + bool use_gpu = false; + bool use_batch = true; + std::size_t seq_len = 16; + std::size_t num_layers = 2; + + for (int i = 1; i < argc; ++i) + { + std::string arg = argv[i]; + if (arg == "--gpu") use_gpu = true; + else if (arg == "--no-batch") use_batch = false; + else if (arg == "--seq" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --seq\n"; return 1; } + seq_len = *v; + } + else if (arg == "--layers" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --layers\n"; return 1; } + num_layers = *v; + } + else if (arg == "--help") + { + std::cout << "用法: " << argv[0] << " [--gpu] [--no-batch] [--seq N] [--layers N]\n"; + return 0; + } + else + { + std::cerr << "未知参数: " << arg << "\n"; + return 1; + } + } + + std::cout << "========================================\n" + << " Attention 一致性验证\n" + << " forward (整段) vs forward_step (KV cache)\n" + << "========================================\n" + << " 序列长度: " << seq_len << "\n" + << " 层数: " << num_layers << "\n" + << " 引擎: " << (use_gpu ? "GPU (Vulkan)" : "CPU") << "\n" + << " batch: " << (use_batch ? "on" : "off") << "\n" + << "========================================\n"; + + // ── 创建引擎 ── + nn::cli::EngineConfig cfg; + cfg.use_gpu = use_gpu; + auto engine = nn::cli::create_engine(cfg, std::cout); + if (!engine) + { + std::cerr << "引擎创建失败\n"; + return 1; + } + + bool all_pass = true; + + // 测试 1: Learned 位置编码（GPT 默认） + all_pass &= test_consistency( + *engine, use_gpu ? "GPU" : "CPU", + seq_len, num_layers, + PosEncodingType::Learned, "Learned", use_batch); + + // 测试 2: Sinusoidal 位置编码 + all_pass &= test_consistency( + *engine, use_gpu ? "GPU" : "CPU", + seq_len, num_layers, + PosEncodingType::Sinusoidal, "Sinusoidal", use_batch); + + // 测试 3: ALiBi（无位置嵌入，因果 + 线性偏置） + all_pass &= test_consistency( + *engine, use_gpu ? "GPU" : "CPU", + seq_len, num_layers, + PosEncodingType::ALiBi, "ALiBi", use_batch); + + std::cout << "\n════════════════════════════════════════\n"; + std::cout << " 总体结果: " << (all_pass ? "✅ 全部通过" : "❌ 存在失败"); + std::cout << "\n════════════════════════════════════════\n"; + + return all_pass ? 0 : 1; +} diff --git a/src/bench_thresholds.cpp b/src/bench_thresholds.cpp new file mode 100644 index 00000000..f0e5f79b --- /dev/null +++ b/src/bench_thresholds.cpp @@ -0,0 +1,488 @@ +// ── 阈值基准测试程序 ───────────────────────────────────────────────────── +// 目标： +// 1. 测试不同元素数量下串行 vs 并行的耗时，找出 PARALLEL_THRESHOLD 最佳值 +// 2. 测试不同矩阵形状下 col_reduce naive（列主序跨行访问） +// vs blocked（行主序扫描 + 累加到列缓冲）的耗时，找出最佳切换阈值 +// 3. 测试 row_reduce 串行 vs 并行切换点，验证现有阈值合理性 +// +// 用法：bench_thresholds [--iters N] [--warmup N] +// 输出：表格 + 末尾给出建议阈值 +// ───────────────────────────────────────────────────────────────────────── + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using nn::Matrix; +using nn::Scalar; +using Clock = std::chrono::high_resolution_clock; + +// ── 计时工具 ────────────────────────────────────────────────────────────── +template +static double bench_us(Func&& fn, int warmup, int iters) +{ + for (int i = 0; i < warmup; ++i) fn(); + double best = 1e18; + for (int i = 0; i < iters; ++i) + { + auto t0 = Clock::now(); + fn(); + auto t1 = Clock::now(); + double us = std::chrono::duration(t1 - t0).count(); + best = std::min(best, us); + } + return best; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 测试 1：PARALLEL_THRESHOLD 探测 +// +// 对简单 transform 操作（y[i] = a*x[i] + b）测试不同元素数量下： +// - 串行 for 循环 +// - nn::transform（带并行阈值检查） +// - 强制并行（直接调用 global_thread_pool().parallel_transform） +// 找出 nn::transform 何时应该启用并行 +// ══════════════════════════════════════════════════════════════════════════ +static void bench_parallel_threshold(int warmup, int iters) +{ + std::cout << "\n┌─────────────────────────────────────────────────────────────┐\n" + << "│ 测试 1: PARALLEL_THRESHOLD 探测 (y[i] = 2*x[i] + 1) │\n" + << "├──────────┬──────────┬──────────┬──────────┬───────────────┤\n" + << "│ 元素数 │ 串行(μs) │ 并行(μs) │ 加速比 │ 建议 │\n" + << "├──────────┼──────────┼──────────┼──────────┼───────────────┤\n"; + + std::mt19937_64 rng(42); + std::uniform_real_distribution dist(-1.0f, 1.0f); + + // 测试元素数（覆盖 1K ~ 4M） + const std::vector sizes = { + 1024, 2048, 4096, 8192, 16384, 32768, 65536, + 131072, 262144, 524288, 1048576, 2097152, 4194304 + }; + + std::cout << std::fixed << std::setprecision(2); + + for (std::size_t n : sizes) + { + std::vector x(n), y(n); + for (auto& v : x) v = dist(rng); + + // 串行 + auto t_serial = bench_us([&] { + for (std::size_t i = 0; i < n; ++i) + y[i] = Scalar{2} * x[i] + Scalar{1}; + }, warmup, iters); + + // 并行（强制走线程池，绕过阈值检查） + auto& pool = nn::global_thread_pool(); + auto t_parallel = bench_us([&] { + pool.parallel_transform(x.begin(), x.end(), y.begin(), + [](Scalar v) { return Scalar{2} * v + Scalar{1}; }); + }, warmup, iters); + + // 对每个 n 直接给出加速比（原“调度(μs)”列恒为 0，已移除） + double speedup = t_serial / t_parallel; + const char* advice; + if (speedup < 0.9) advice = "串行优"; + else if (speedup < 1.1) advice = "无差别"; + else advice = "并行优"; + + std::cout << "│ " << std::setw(8) << n + << " │ " << std::setw(8) << t_serial + << " │ " << std::setw(8) << t_parallel + << " │ " << std::setw(7) << speedup << "x" + << " │ " << std::setw(13) << advice + << " │\n"; + } + std::cout << "└──────────┴──────────┴──────────┴──────────┴───────────────┘\n"; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 测试 2：col_reduce naive vs blocked +// +// naive（当前实现）：按列遍历，每列内跨行 stride 访问 → cache 不友好 +// blocked（分块实现）：按行主序扫描，每 L1 个元素累加到 col_acc[c] +// → cache 友好，但需要 cols 个累加器 +// ══════════════════════════════════════════════════════════════════════════ +static void bench_col_reduce(int warmup, int iters) +{ + std::cout << "\n┌──────────────────────────────────────────────────────────────────────────┐\n" + << "│ 测试 2: col_reduce naive(跨行) vs blocked(行主序扫描+列累加器) │\n" + << "├─────────────────┬──────────┬──────────┬──────────┬──────────┬──────────┤\n" + << "│ 矩阵(rows×cols) │ naive(μs)│ block(μs)│ 加速比 │ winner │ 建议 │\n" + << "├─────────────────┼──────────┼──────────┼──────────┼──────────┼──────────┤\n"; + + std::mt19937_64 rng(42); + std::uniform_real_distribution dist(-1.0f, 1.0f); + + // 模拟实际使用场景的矩阵形状 + struct Shape { std::size_t R, C; const char* label; }; + const std::vector shapes = { + // GPT / LayerNorm 场景：列数为 d_model 或 vocab_size + {32, 128, "32×128"}, // GPT small per-sample + {128, 128, "128×128"}, // GPT medium + {1024, 128, "1024×128"}, // GPT large batch + {32, 1024, "32×1K"}, // wide matrix + {128, 1024, "128×1K"}, + {1024, 1024, "1K×1K"}, + {32, 10000, "32×10K"}, // logits col_reduce + {128, 10000, "128×10K"}, + {1024, 10000, "1K×10K"}, // large logits + {4096, 4096, "4K×4K"}, // large square + // 行数远大于列数（行主序分块更受益） + {8192, 64, "8K×64"}, + {65536, 64, "64K×64"}, + // 列数远大于行数（naive 可能受益于 SIMD） + {64, 8192, "64×8K"}, + {64, 65536, "64×64K"}, + }; + + std::cout << std::fixed << std::setprecision(2); + + for (auto [R, C, label] : shapes) + { + Matrix m(R, C); + auto sp = m.span(); + for (auto& v : sp) v = dist(rng); + + // ---- naive 实现（当前）：每列独立跨行扫描 ---- + Matrix naive_out(1, C); + auto naive_out_sp = naive_out.span(); + auto t_naive = bench_us([&] { + const auto self = m.span(); + auto out = naive_out_sp; + for (std::size_t c = 0; c < C; ++c) + { + Scalar acc = Scalar{0}; + for (std::size_t r = 0; r < R; ++r) + acc += self[r * C + c]; + out[c] = acc; + } + }, warmup, iters); + + // ---- blocked 实现：按行主序扫描，每 BLOCK 行累加到列累加器 ---- + // BLOCK 选择：使 cols*BLOCK*sizeof(Scalar) 装入 L1（32KB） + // BLOCK = 32KB / (C * 4) = 8192 / C + Matrix block_out(1, C); + auto block_out_sp = block_out.span(); + auto t_blocked = bench_us([&] { + const auto self = m.span(); + auto out = block_out_sp; + // 先清零 + for (std::size_t c = 0; c < C; ++c) out[c] = Scalar{0}; + // 计算 block 大小（确保 col_acc 数组 ≤ 16KB L1） + constexpr std::size_t L1_BUDGET = 16384; // 16KB + std::size_t block = std::max(1, L1_BUDGET / (C * sizeof(Scalar))); + if (block > R) block = R; + // 按行块扫描 + for (std::size_t r0 = 0; r0 < R; r0 += block) + { + std::size_t r_end = std::min(r0 + block, R); + for (std::size_t r = r0; r < r_end; ++r) + { + const auto row = self.subspan(r * C, C); + for (std::size_t c = 0; c < C; ++c) + out[c] += row[c]; + } + } + }, warmup, iters); + + double speedup = t_naive / t_blocked; + const char* winner = (speedup < 1.0) ? "naive" : "blocked"; + const char* advice; + if (speedup > 1.5) advice = "用blocked"; + else if (speedup < 0.7) advice = "用naive"; + else advice = "可混合"; + + std::cout << "│ " << std::left << std::setw(15) << label + << " │ " << std::right << std::setw(8) << t_naive + << " │ " << std::setw(8) << t_blocked + << " │ " << std::setw(7) << speedup << "x" + << " │ " << std::setw(8) << winner + << " │ " << std::setw(8) << advice + << " │\n"; + } + std::cout << "└─────────────────┴──────────┴──────────┴──────────┴──────────┴──────────┘\n"; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 测试 3：col_reduce 行块并行 vs 按列并行 vs 串行 blocked +// +// 对比三种方案： +// a) 单线程行主序 blocked（当前默认） +// b) 按列并行（每列一个任务，跨行 stride=C 访问，cache miss 严重） +// c) 行块并行（按行分块，每线程本地累加器 + 归并阶段） +// ══════════════════════════════════════════════════════════════════════════ +static void bench_col_reduce_parallel(int warmup, int iters) +{ + std::cout << "\n┌──────────────────────────────────────────────────────────────────────────────────────────┐\n" + << "│ 测试 3: col_reduce 单线程 blocked vs 按列并行 vs 行块并行（行主序+本地累加器+归并） │\n" + << "├─────────────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┤\n" + << "│ 矩阵(rows×cols) │ 串行μs │ 按列并μs │ 行块并μs │ 列加速 │ 行块加速 │ 建议 │\n" + << "├─────────────────┼──────────┼──────────┼──────────┼──────────┼──────────┼──────────┤\n"; + + std::mt19937_64 rng(42); + std::uniform_real_distribution dist(-1.0f, 1.0f); + + struct Shape { std::size_t R, C; const char* label; }; + const std::vector shapes = { + {128, 128, "128×128"}, + {1024, 128, "1024×128"}, + {128, 1024, "128×1K"}, + {1024, 1024, "1K×1K"}, + {128, 10000, "128×10K"}, + {1024, 10000, "1K×10K"}, + {4096, 4096, "4K×4K"}, + {8192, 64, "8K×64"}, + {64, 8192, "64×8K"}, + {8192, 128, "8K×128"}, + {65536, 64, "64K×64"}, + {4096, 512, "4K×512"}, + }; + + std::cout << std::fixed << std::setprecision(2); + auto& pool = nn::global_thread_pool(); + + for (auto [R, C, label] : shapes) + { + Matrix m(R, C); + auto sp = m.span(); + for (auto& v : sp) v = dist(rng); + + Matrix out_serial(1, C), out_par_col(1, C), out_par_row(1, C); + + // (a) 串行行主序 blocked + auto t_serial = bench_us([&] { + const auto self = m.span(); + auto out = out_serial.span(); + for (std::size_t c = 0; c < C; ++c) out[c] = Scalar{0}; + for (std::size_t r = 0; r < R; ++r) + { + const auto row = self.subspan(r * C, C); + for (std::size_t c = 0; c < C; ++c) + out[c] += row[c]; + } + }, warmup, iters); + + // (b) 按列并行（cache miss 严重，作为对照） + auto t_par_col = bench_us([&] { + const auto self = m.span(); + auto out = out_par_col.span(); + auto col_indices = std::views::iota(std::size_t{0}, C); + pool.parallel_for_each(col_indices.begin(), col_indices.end(), + [self, out, R, C](std::size_t c) { + Scalar acc = Scalar{0}; + for (std::size_t r = 0; r < R; ++r) + acc += self[r * C + c]; + out[c] = acc; + }); + }, warmup, iters); + + // (c) 行块并行：按行分块，每线程本地累加器 + 归并 + auto t_par_row = bench_us([&] { + const auto self = m.span(); + auto out = out_par_row.span(); + + const std::size_t n_threads = std::max( + std::thread::hardware_concurrency(), 1); + std::vector local_acc(n_threads * C, Scalar{0}); + const std::size_t base = R / n_threads; + const std::size_t rem = R % n_threads; + auto row_blocks = std::views::iota(std::size_t{0}, n_threads); + pool.parallel_for_blocks(row_blocks.begin(), row_blocks.end(), + [self, &local_acc, C, base, rem](std::size_t t) noexcept { + const std::size_t r0 = t * base + std::min(t, rem); + const std::size_t r_end = (t + 1) * base + std::min(t + 1, rem); + auto acc = std::span(local_acc).subspan(t * C, C); + for (std::size_t r = r0; r < r_end; ++r) + { + const auto row = self.subspan(r * C, C); + for (std::size_t c = 0; c < C; ++c) + acc[c] += row[c]; + } + }); + + // 归并 + for (std::size_t c = 0; c < C; ++c) + out[c] = local_acc[c]; + for (std::size_t t = 1; t < n_threads; ++t) + { + const auto acc = std::span(local_acc).subspan(t * C, C); + for (std::size_t c = 0; c < C; ++c) + out[c] += acc[c]; + } + }, warmup, iters); + + double sp_col = t_serial / t_par_col; + double sp_row = t_serial / t_par_row; + const char* advice; + if (sp_row > 1.5) advice = "行块并优"; + else if (sp_col > 1.5) advice = "按列并优"; + else advice = "串行优"; + + std::cout << "│ " << std::left << std::setw(15) << label + << " │ " << std::right << std::setw(8) << t_serial + << " │ " << std::setw(8) << t_par_col + << " │ " << std::setw(8) << t_par_row + << " │ " << std::setw(7) << sp_col << "x" + << " │ " << std::setw(7) << sp_row << "x" + << " │ " << std::setw(8) << advice + << " │\n"; + } + std::cout << "└─────────────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┘\n"; +} + +// ══════════════════════════════════════════════════════════════════════════ +// 测试 4：Matrix::multiply_to GPU 切换点探测 +// 仅当 NN_HAS_VULKAN 编译时启用 +// ══════════════════════════════════════════════════════════════════════════ +#ifdef NN_HAS_VULKAN +static void bench_gpu_threshold(int warmup, int iters) +{ + auto& backend = nn::GpuBackend::instance(); + if (!backend.is_initialized() && !backend.initialize()) + { + std::cout << "\n[GPU] Vulkan 未初始化，跳过 GPU 阈值测试\n"; + return; + } + + std::cout << "\n┌──────────────────────────────────────────────────────────────────┐\n" + << "│ 测试 4: Matrix::multiply_to GPU 切换点 (CPU matmul vs GPU) │\n" + << "├─────────────┬──────────┬──────────┬──────────┬──────────┬────────┤\n" + << "│ M×N×K │ CPU(μs) │ GPU(μs) │ 加速比 │ winner │ 建议 │\n" + << "├─────────────┼──────────┼──────────┼──────────┼──────────┼────────┤\n"; + + std::mt19937_64 rng(42); + std::uniform_real_distribution dist(-1.0f, 1.0f); + + auto rand_matrix = [&](std::size_t r, std::size_t c) { + Matrix m(r, c); + for (auto& v : m.span()) v = dist(rng); + return m; + }; + + struct Case { std::size_t M, K, N; const char* label; }; + const std::vector cases = { + {8, 8, 8, "8×8×8"}, + {16, 16, 16, "16×16×16"}, + {32, 32, 32, "32×32×32"}, + {64, 64, 64, "64×64×64"}, + {64, 128, 64, "64×128×64"}, + {128, 128, 128, "128×128×128"}, + {128, 512, 128, "128×512×128"}, + {128, 1024, 128, "128×1K×128"}, + {256, 256, 256, "256×256×256"}, + {512, 512, 512, "512×512×512"}, + {1024, 1024, 1024, "1K×1K×1K"}, + }; + + std::cout << std::fixed << std::setprecision(2); + + for (auto [M, K, N, label] : cases) + { + auto A = rand_matrix(M, K); + auto B = rand_matrix(K, N); + Matrix C(M, N); + + // CPU 路径（multiply_to 现已无 GPU 分支，直接测 CPU 实现） + auto t_cpu = bench_us([&] { A.multiply_to(C, B); }, warmup, iters); + + // GPU 路径（通过 matmul_direct 直接调用，不经过 multiply_to） + auto t_gpu = bench_us([&] { + auto r = backend.matmul_direct(A.span(), B.span(), C.span(), M, N, K); + (void)r; + }, warmup, iters); + + double speedup = (t_gpu > 0) ? t_cpu / t_gpu : 0.0; + const char* winner = (speedup > 1.0) ? "GPU" : "CPU"; + const char* advice; + if (speedup > 1.5) advice = "用GPU"; + else if (speedup < 0.7) advice = "用CPU"; + else advice = "临界"; + + std::cout << "│ " << std::left << std::setw(11) << label + << " │ " << std::right << std::setw(8) << t_cpu + << " │ " << std::setw(8) << t_gpu + << " │ " << std::setw(7) << speedup << "x" + << " │ " << std::setw(8) << winner + << " │ " << std::setw(6) << advice + << " │\n"; + } + std::cout << "└─────────────┴──────────┴──────────┴──────────┴──────────┴────────┘\n"; +} +#endif // NN_HAS_VULKAN + +// ── 主入口 ────────────────────────────────────────────────────────────────── +int main(int argc, char* argv[]) +{ + int iters = 30; + int warmup = 8; + + for (int i = 1; i < argc; ++i) + { + std::string arg = argv[i]; + if (arg == "--help" || arg == "-h") + { + std::cout << "阈值基准测试程序\n\n" + << "用法: bench_thresholds [选项]\n\n" + << "选项:\n" + << " --iters N 每项测试迭代次数 (默认: 30)\n" + << " --warmup N 预热次数 (默认: 8)\n" + << " --help 显示此帮助信息\n"; + return 0; + } + else if (arg == "--iters" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --iters\n"; return 1; } + if (*v <= 0) { std::cerr << "--iters 必须为正整数\n"; return 1; } + iters = *v; + } + else if (arg == "--warmup" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --warmup\n"; return 1; } + if (*v < 0) { std::cerr << "--warmup 必须为非负整数\n"; return 1; } + warmup = *v; + } + } + + std::cout << "========================================\n" + << " neuralnet.cpp 阈值基准测试\n" + << "========================================\n" + << " CPU 核心数: " << std::thread::hardware_concurrency() << "\n" + << " 标量类型: float (4 bytes)\n" + << " 迭代次数: " << iters << "\n" + << " 预热次数: " << warmup << "\n" + << "========================================\n"; + + bench_parallel_threshold(warmup, iters); + bench_col_reduce(warmup, iters); + bench_col_reduce_parallel(warmup, iters); +#ifdef NN_HAS_VULKAN + bench_gpu_threshold(warmup, iters); +#endif + + std::cout << "\n=== 阈值建议（基于上表数据） ===\n" + << "1. PARALLEL_THRESHOLD:\n" + << " 取加速比首次稳定 > 1.5x 的元素数作为阈值。\n" + << " 观察测试 1 表格，找到加速比列首次 >= 1.50 的行。\n\n" + << "2. col_reduce blocked 阈值:\n" + << " 观察测试 2，对加速比 > 1.5 的形状用 blocked；\n" + << " 否则用 naive（即当 cols 极大但 rows 很小时 naive SIMD 反而更快）。\n\n" + << "3. col_reduce 并行阈值:\n" + << " 观察测试 3，找出并行 vs 串行 blocked 的交叉点（加速比 >= 1.5）。\n\n" + << "4. GPU_THRESHOLD:\n" + << " 观察测试 4，找 GPU 加速比稳定 > 1.5x 的最小 M*N*K。\n" + << "========================================\n"; + return 0; +} diff --git a/src/compute_bench.cpp b/src/compute_bench.cpp new file mode 100644 index 00000000..5e9ef9d0 --- /dev/null +++ b/src/compute_bench.cpp @@ -0,0 +1,418 @@ +// ── 计算库性能基准测试（f32 GFLOPS）──────────────────────────────────── +// 测试 Matrix 类在 float32 下的算力：矩阵乘法、逐元素运算、转置等。 +// 用法：compute_bench [--size N] [--iters N] [--warmup N] [--help] +// ───────────────────────────────────────────────────────────────────────── + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using nn::Matrix; +using nn::Scalar; + +// ── 计时工具 ────────────────────────────────────────────────────────────── +using Clock = std::chrono::high_resolution_clock; + +struct TimingResult { + double avg_ms; + double min_ms; + double max_ms; + double std_ms; +}; + +template +TimingResult bench(Func&& fn, int warmup, int iters) +{ + // 预热 + for (int i = 0; i < warmup; ++i) + fn(); + + std::vector times(iters); + for (int i = 0; i < iters; ++i) { + auto t0 = Clock::now(); + fn(); + auto t1 = Clock::now(); + times[i] = std::chrono::duration(t1 - t0).count(); + } + + double sum = std::accumulate(times.begin(), times.end(), 0.0); + double avg = sum / iters; + double min_t = *std::min_element(times.begin(), times.end()); + double max_t = *std::max_element(times.begin(), times.end()); + + double sq_sum = 0.0; + for (double t : times) + sq_sum += (t - avg) * (t - avg); + double std_dev = std::sqrt(sq_sum / iters); + + return {avg, min_t, max_t, std_dev}; +} + +// ── GFLOPS 计算辅助 ─────────────────────────────────────────────────────── +// 矩阵乘法 C[M×N] = A[M×K] × B[K×N] → 2·M·N·K FLOPs +static constexpr double matmul_flops(std::size_t M, std::size_t N, std::size_t K) +{ + return 2.0 * static_cast(M) * N * K; +} + +// 逐元素二元运算（加/减/乘）：每元素 1 FLOP +// 逐元素一元运算（ReLU/GeLU 近似）：每元素 ~若干 FLOP +// 转置：纯内存搬运，0 算术 FLOP（仅报告 GB/s） + +static constexpr double elem_flops(std::size_t n) { return static_cast(n); } + +// ── 打印表头 ────────────────────────────────────────────────────────────── +void print_header() +{ + std::cout << "\n" + << "┌──────────────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────────┐\n" + << "│ 运算 │ 规格 │ FLOPs │ avg(ms) │ min(ms) │ GFLOPS │ 带宽(GB/s) │\n" + << "├──────────────────┼──────────┼──────────┼──────────┼──────────┼──────────┼──────────────┤\n"; +} + +void print_row(const char* op, const char* spec, double flops, + double avg_ms, double min_ms, double bytes_transferred = 0.0) +{ + double gflops = (avg_ms > 0.0) ? flops / (avg_ms * 1e6) : 0.0; + double gbps = (min_ms > 0.0 && bytes_transferred > 0.0) + ? bytes_transferred / (min_ms * 1e6) + : 0.0; + + std::cout << "│ " << std::left << std::setw(16) << op + << "│ " << std::right << std::setw(8) << spec + << " │ " << std::setw(8) << std::fixed << std::setprecision(0) << flops / 1e3 << "k" + << " │ " << std::setw(7) << std::setprecision(2) << avg_ms + << " │ " << std::setw(7) << min_ms + << " │ " << std::setw(7) << std::setprecision(1) << gflops + << " │ " << std::setw(11) << std::setprecision(1) << gbps + << " │\n"; +} + +void print_separator() +{ + std::cout << "├──────────────────┼──────────┼──────────┼──────────┼──────────┼──────────┼──────────────┤\n"; +} + +void print_footer() +{ + std::cout << "└──────────────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────────┘\n"; +} + +// ── 主测试 ──────────────────────────────────────────────────────────────── +int main(int argc, char* argv[]) +{ + std::size_t base_size = 512; + int iters = 20; + int warmup = 5; + + for (int i = 1; i < argc; ++i) { + std::string arg = argv[i]; + if (arg == "--help" || arg == "-h") { + std::cout << "计算库性能基准测试 (f32 GFLOPS)\n\n" + << "用法: compute_bench [选项]\n\n" + << "选项:\n" + << " --size N 基准矩阵维度 (默认: 512)\n" + << " --iters N 每项测试迭代次数 (默认: 20)\n" + << " --warmup N 预热次数 (默认: 5)\n" + << " --help 显示此帮助信息\n"; + return 0; + } else if (arg == "--size" && i + 1 < argc) { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --size: " << v.error().message << "\n"; return 1; } + if (*v == 0) { std::cerr << "--size 必须为正整数\n"; return 1; } + base_size = *v; + } else if (arg == "--iters" && i + 1 < argc) { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --iters: " << v.error().message << "\n"; return 1; } + if (*v <= 0) { std::cerr << "--iters 必须为正整数\n"; return 1; } + iters = *v; + } else if (arg == "--warmup" && i + 1 < argc) { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --warmup: " << v.error().message << "\n"; return 1; } + if (*v < 0) { std::cerr << "--warmup 必须为非负整数\n"; return 1; } + warmup = *v; + } else { + std::cerr << "未知参数: " << arg << "\n"; + return 1; + } + } + + std::cout << "========================================\n" + << " 计算库 f32 性能基准测试\n" + << "========================================\n" + << " 基准维度: " << base_size << "×" << base_size << "\n" + << " 迭代次数: " << iters << "\n" + << " 预热次数: " << warmup << "\n" + << " 标量类型: float (f32)\n" + << " 线程数: " << std::thread::hardware_concurrency() << "\n" + << " 编译器优化: -O3 -ffast-math\n" + << "========================================\n"; + + // ── 随机数据生成器 ───────────────────────────────────────────── + std::mt19937_64 rng(42); + std::uniform_real_distribution dist(-1.0f, 1.0f); + + auto rand_matrix = [&](std::size_t rows, std::size_t cols) { + Matrix m(rows, cols); + auto s = m.span(); + for (auto& v : s) + v = dist(rng); + return m; + }; + + print_header(); + + // ─═════════════════════════════════════════════════════════════════ + // 1. 矩阵乘法（核心算力指标） + // ─═════════════════════════════════════════════════════════════════ + { + std::vector sizes = { + 64, 128, 256, 512, 768, 1024, 1536, 2048}; + + for (std::size_t N : sizes) { + if (N > base_size * 4) break; + auto A = rand_matrix(N, N); + auto B = rand_matrix(N, N); + Matrix C(N, N); + + double flops = matmul_flops(N, N, N); + auto res = bench([&] { A.multiply_to(C, B); }, warmup, iters); + char spec[32]; + std::snprintf(spec, sizeof(spec), "%zux%zu", N, N); + print_row("matmul (f32)", spec, flops, res.avg_ms, res.min_ms); + } + print_separator(); + } + + // ── 矩阵乘法（矩形矩阵，模拟 Transformer 维度）──────────────── + { + struct RectCase { std::size_t M, K, N; const char* label; }; + std::vector cases = { + {128, 768, 768, "128×768²"}, // 注意力投影 + {128, 768, 3072, "128×768×3K"}, // FFN 第一层 + {128, 3072, 768, "128×3K×768"}, // FFN 第二层 + {32, 512, 512, "32×512²"}, // 小 batch + {1, 768, 768, "1×768²"}, // 单样本推理 + }; + for (auto [M, K, N, label] : cases) { + auto A = rand_matrix(M, K); + auto B = rand_matrix(K, N); + Matrix C(M, N); + double flops = matmul_flops(M, N, K); + auto res = bench([&] { A.multiply_to(C, B); }, warmup, iters); + print_row("matmul rect", label, flops, res.avg_ms, res.min_ms); + } + print_separator(); + } + + // ─═════════════════════════════════════════════════════════════════ + // 2. 转置（内存带宽瓶颈） + // ─═════════════════════════════════════════════════════════════════ + { + std::vector sizes = {256, 512, 1024, 2048}; + for (std::size_t N : sizes) { + if (N > base_size * 4) break; + auto A = rand_matrix(N, N); + Matrix T(N, N); + double bytes = 2.0 * N * N * sizeof(Scalar); // 读+写 + auto res = bench([&] { A.transpose_to(T); }, warmup, iters); + char spec[32]; + std::snprintf(spec, sizeof(spec), "%zux%zu", N, N); + print_row("transpose", spec, 0, res.avg_ms, res.min_ms, bytes); + } + print_separator(); + } + + // ─═════════════════════════════════════════════════════════════════ + // 3. 逐元素运算 + // ─═════════════════════════════════════════════════════════════════ + { + std::size_t N = base_size; + std::size_t total = N * N; + auto A = rand_matrix(N, N); + auto B = rand_matrix(N, N); + Matrix C(N, N); + + // 加法 + auto res_add = bench([&] { C = nn::detail::binary_apply(A, B, std::plus<>{}); }, warmup, iters); + print_row("elem add", std::to_string(N).c_str(), elem_flops(total), + res_add.avg_ms, res_add.min_ms, 3.0 * total * sizeof(Scalar)); + + // 减法 + auto res_sub = bench([&] { C = nn::detail::binary_apply(A, B, std::minus<>{}); }, warmup, iters); + print_row("elem sub", std::to_string(N).c_str(), elem_flops(total), + res_sub.avg_ms, res_sub.min_ms, 3.0 * total * sizeof(Scalar)); + + // 标量乘法 + Scalar s = 2.5f; + auto res_mul = bench([&] { C = nn::detail::apply(A, [s](Scalar v) { return v * s; }); }, warmup, iters); + print_row("scalar mul", std::to_string(N).c_str(), elem_flops(total), + res_mul.avg_ms, res_mul.min_ms, 2.0 * total * sizeof(Scalar)); + + // scale_inplace + auto res_scl = bench([&] { A.scale_inplace(1.01f); }, warmup, iters); + print_row("scale inplace", std::to_string(N).c_str(), elem_flops(total), + res_scl.avg_ms, res_scl.min_ms, 2.0 * total * sizeof(Scalar)); + + // add_inplace + auto res_adi = bench([&] { A.add_inplace(B); }, warmup, iters); + print_row("add inplace", std::to_string(N).c_str(), elem_flops(total), + res_adi.avg_ms, res_adi.min_ms, 3.0 * total * sizeof(Scalar)); + + print_separator(); + } + + // ─═════════════════════════════════════════════════════════════════ + // 4. 逐元素函数（激活函数近似） + // ─═════════════════════════════════════════════════════════════════ + { + std::size_t N = base_size; + std::size_t total = N * N; + auto A = rand_matrix(N, N); + auto B = rand_matrix(N, N); + Matrix C(N, N); + + // ReLU: max(0, x) — 约 1 FLOP/element + auto res_relu = bench([&] { C = nn::detail::apply(A, [](Scalar v) { return std::max(Scalar{0}, v); }); }, + warmup, iters); + print_row("ReLU (apply)", std::to_string(N).c_str(), elem_flops(total), + res_relu.avg_ms, res_relu.min_ms, 2.0 * total * sizeof(Scalar)); + + // GeLU 近似: x * σ(1.702x) — 约 5 FLOPs/element + auto res_gelu = bench([&] { + C = nn::detail::apply(A, [](Scalar v) { + Scalar sig = 1.0f / (1.0f + std::exp(-1.702f * v)); + return v * sig; + }); + }, warmup, iters); + print_row("GeLU (apply)", std::to_string(N).c_str(), 5.0 * total, + res_gelu.avg_ms, res_gelu.min_ms, 2.0 * total * sizeof(Scalar)); + + // 逐元素乘法: A ⊙ B — 1 FLOP/element + auto res_emul = bench([&] { + C = nn::detail::binary_apply(A, B, [](Scalar a, Scalar b) { return a * b; }); + }, warmup, iters); + print_row("elem multiply", std::to_string(N).c_str(), elem_flops(total), + res_emul.avg_ms, res_emul.min_ms, 3.0 * total * sizeof(Scalar)); + + print_separator(); + } + + // ─═════════════════════════════════════════════════════════════════ + // 5. 线性层（Linear forward / backward）— 新引擎化架构 + // ─═════════════════════════════════════════════════════════════════ + { + nn::CpuEngine engine; + + struct LinearCase { std::size_t in, out, batch; }; + std::vector cases = { + {768, 768, 128}, // Transformer 标准隐藏层 + {768, 3072, 128}, // FFN 扩展 + {3072, 768, 128}, // FFN 压缩 + {768, 768, 32}, // 小 batch + {256, 256, 128}, // 中等规模 + }; + + for (auto [in, out, batch] : cases) { + nn::Linear layer(engine, in, out); + auto input = rand_matrix(in, batch); + auto input_t = engine.from_matrix(input); + if (!input_t) { + std::cerr << "from_matrix(input) 失败: " << input_t.error().message << "\n"; + return 1; + } + + // Forward: C = W * input + b → FLOPs = 2·out·in·batch + out·batch + double fwd_flops = matmul_flops(out, batch, in) + elem_flops(out * batch); + // 首次调用校验结果，后续 benchmark 内部重复调用假设成功 + auto first_fwd = layer.forward(engine, *input_t); + if (!first_fwd) { + std::cerr << "Linear forward 失败: " << first_fwd.error().message << "\n"; + return 1; + } + auto res_fwd = bench([&] { auto r = layer.forward(engine, *input_t); (void)r; }, warmup, iters); + + char spec[64]; + std::snprintf(spec, sizeof(spec), "W=%zux%zu b=%zu", out, in, batch); + print_row("Linear fwd", spec, fwd_flops, res_fwd.avg_ms, res_fwd.min_ms); + + // Backward: grad_input = W^T * grad_out, grad_W += grad_out * input^T, grad_b += sum + // FLOPs ≈ 2·out·in·batch (W^T * grad_out) + out·in·batch (outer product) + out·batch (bias) + double bwd_flops = matmul_flops(in, batch, out) + elem_flops(out * in * batch) + elem_flops(out * batch); + auto grad_out = rand_matrix(out, batch); + auto grad_out_t = engine.from_matrix(grad_out); + if (!grad_out_t) { + std::cerr << "from_matrix(grad_out) 失败: " << grad_out_t.error().message << "\n"; + return 1; + } + auto first_bwd = layer.backward(engine, *grad_out_t); + if (!first_bwd) { + std::cerr << "Linear backward 失败: " << first_bwd.error().message << "\n"; + return 1; + } + auto res_bwd = bench([&] { auto r = layer.backward(engine, *grad_out_t); (void)r; }, warmup, iters); + print_row("Linear bwd", spec, bwd_flops, res_bwd.avg_ms, res_bwd.min_ms); + } + print_separator(); + } + + // ─═════════════════════════════════════════════════════════════════ + // 6. 归约（Sum / Mean） + // ─═════════════════════════════════════════════════════════════════ + { + std::size_t N = base_size; + std::size_t total = N * N; + auto A = rand_matrix(N, N); + + // reduce: 求和 — 1 FLOP/element + auto res_sum = bench([&] { + volatile Scalar s = A.reduce(Scalar{0}, std::plus<>{}, + [](Scalar v) { return v; }); + (void)s; + }, warmup, iters); + print_row("reduce (sum)", std::to_string(N).c_str(), elem_flops(total), + res_sum.avg_ms, res_sum.min_ms, total * sizeof(Scalar)); + + print_separator(); + } + + print_footer(); + + // ── 汇总：纯矩阵乘法峰值 GFLOPS ─────────────────────────────── + std::cout << "\n 📊 峰值算力摘要 (float32 矩阵乘法):\n\n"; + + { + // 自动跑一次大矩阵以获取峰值 + std::size_t peak_N = 2048; + auto A = rand_matrix(peak_N, peak_N); + auto B = rand_matrix(peak_N, peak_N); + Matrix C(peak_N, peak_N); + double flops = matmul_flops(peak_N, peak_N, peak_N); + auto res = bench([&] { A.multiply_to(C, B); }, warmup, iters); + double gflops = flops / (res.min_ms * 1e6); + std::cout << " " << peak_N << "×" << peak_N << " matmul: " + << std::fixed << std::setprecision(1) << gflops << " GFLOPS" + << " (avg " << std::setprecision(2) << res.avg_ms << " ms, " + << "min " << res.min_ms << " ms)\n\n"; + } + + // 汇报硬件信息 + std::cout << " ℹ️ 线程数: " << std::thread::hardware_concurrency() << "\n"; + std::cout << " ℹ️ Scalar = float (sizeof = " << sizeof(Scalar) << " bytes)\n"; + std::cout << " ℹ️ BLOCK_SIZE = " << nn::BLOCK_SIZE << "\n"; + std::cout << "\n ✅ 测试完成。\n"; + + return 0; +} diff --git a/src/gpu_test.cpp b/src/gpu_test.cpp new file mode 100644 index 00000000..284f32c9 --- /dev/null +++ b/src/gpu_test.cpp @@ -0,0 +1,481 @@ +// ── ComputeEngine 正确性与性能测试 ────────────────────────────────────────── +// 用法：gpu_test [--size N] [--iters N] +// --size N 矩阵维度（默认 256，即 256×256 方阵） +// --iters N 性能测试迭代次数（默认 10） +// +// 验证 CpuEngine 与 GpuEngine 在 matmul / 转置 matmul / 逐元素 / 归约 / +// roundtrip 等原语上的一致性，并对比 GPU vs CPU matmul 性能。 +// ───────────────────────────────────────────────────────────────────────── + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using nn::Scalar; + +#ifndef NN_HAS_VULKAN +int main() +{ + std::cout << "[SKIP] 此程序需要 Vulkan SDK 支持，请使用 -DNN_HAS_VULKAN 编译。\n"; + return 0; +} +#else +#include + +using nn::Matrix; +using nn::ComputeEngine; +using nn::CpuEngine; +using nn::GpuEngine; +using nn::GpuBackend; +using nn::Tensor; +using nn::BinaryOp; +using nn::UnaryOp; + +void print_usage(const char* prog) +{ + std::cout << "ComputeEngine 正确性与性能测试\n\n" + << "用法: " << prog << " [选项]\n\n" + << "选项:\n" + << " --size N 矩阵维度 (默认: 256)\n" + << " --iters N 性能迭代次数 (默认: 10)\n" + << " --help 显示此帮助信息\n"; +} + +// ── 辅助：比较两个 Matrix 的最大绝对误差 ────────────────────────────────── +Scalar max_abs_diff(const Matrix& a, const Matrix& b) +{ + Scalar max_err = 0.0f; + auto as = a.span(); + auto bs = b.span(); + const std::size_t n = as.size(); + for (std::size_t i = 0; i < n; ++i) + { + Scalar diff = std::fabs(as[i] - bs[i]); + if (diff > max_err) max_err = diff; + } + return max_err; +} + +// ── 辅助：通过 engine 执行 matmul 并下载结果 ─────────────────────────────── +nn::Result engine_matmul( + ComputeEngine& engine, const Matrix& A, const Matrix& B, + bool transA = false, bool transB = false) +{ + auto a_t = engine.from_matrix(A); + if (!a_t) return std::unexpected(std::move(a_t).error()); + auto b_t = engine.from_matrix(B); + if (!b_t) return std::unexpected(std::move(b_t).error()); + auto c_t = engine.matmul(*a_t, *b_t, transA, transB); + if (!c_t) return std::unexpected(std::move(c_t).error()); + return engine.to_matrix(*c_t); +} + +// 数字解析辅助：解析失败时打印错误并退出（替代会抛异常的 std::stoi/stod） +template +T parse_num_or_die(const char* s, const char* opt) +{ + auto v = nn::parse_number(s); + if (!v) + { + std::cerr << "无效的 " << opt << " 值: " << v.error().message << "\n"; + std::exit(1); + } + return *v; +} + +int main(int argc, char* argv[]) +{ + std::size_t N = 256; + int iters = 10; + + for (int i = 1; i < argc; ++i) + { + std::string arg = argv[i]; + if (arg == "--help") { print_usage(argv[0]); return 0; } + else if (arg == "--size" && i + 1 < argc) + { + int v = parse_num_or_die(argv[++i], "--size"); + if (v <= 0) { std::cerr << "--size 必须为正整数\n"; return 1; } + N = static_cast(v); + } + else if (arg == "--iters" && i + 1 < argc) + { + iters = parse_num_or_die(argv[++i], "--iters"); + if (iters <= 0) { std::cerr << "--iters 必须为正整数\n"; return 1; } + } + else { std::cerr << "未知参数: " << arg << "\n"; return 1; } + } + + std::cout << "========================================\n" + << " ComputeEngine 正确性与性能测试\n" + << "========================================\n" + << " 矩阵维度: " << N << "×" << N << "\n" + << " 迭代次数: " << iters << "\n" + << "========================================\n\n"; + + // ── 1. 初始化引擎 ────────────────────────────────────────────── + std::cout << "[1/6] 初始化引擎..." << std::flush; + auto cpu_engine = std::make_unique(); + + auto& backend = GpuBackend::instance(); + auto init_result = backend.initialize(); + if (!init_result) + { + std::cout << " 失败\n"; + std::cerr << "GPU 初始化失败: " << init_result.error().message << "\n"; + return 1; + } + auto gpu_engine = std::make_unique(backend); + std::cout << " 成功 (CpuEngine + GpuEngine)\n"; + + // ── 2. 生成随机测试数据 ──────────────────────────────────────── + std::cout << "[2/6] 生成 " << N << "×" << N << " 随机矩阵..." << std::flush; + std::mt19937_64 rng(42); + std::uniform_real_distribution dist(-1.0f, 1.0f); + + Matrix A(N, N), B(N, N); + for (auto& v : A.span()) v = dist(rng); + for (auto& v : B.span()) v = dist(rng); + std::cout << " 完成\n"; + + // ── 3. matmul 正确性验证 (CpuEngine vs GpuEngine) ───────────── + std::cout << "[3/6] matmul 正确性验证 (CPU vs GPU)..." << std::flush; + + auto cpu_c_r = engine_matmul(*cpu_engine, A, B); + if (!cpu_c_r) { std::cout << "\n ❌ CpuEngine matmul 失败: " << cpu_c_r.error().message << "\n"; return 1; } + auto gpu_c_r = engine_matmul(*gpu_engine, A, B); + if (!gpu_c_r) { std::cout << "\n ❌ GpuEngine matmul 失败: " << gpu_c_r.error().message << "\n"; return 1; } + + Scalar matmul_err = max_abs_diff(*cpu_c_r, *gpu_c_r); + + // RMSE + auto cs = cpu_c_r->span(); + auto gs = gpu_c_r->span(); + Scalar sum_sq_err = 0.0f, sum_sq_ref = 0.0f; + for (std::size_t i = 0; i < N * N; ++i) + { + Scalar diff = cs[i] - gs[i]; + sum_sq_err += diff * diff; + sum_sq_ref += cs[i] * cs[i]; + } + Scalar rmse = std::sqrt(sum_sq_err / (N * N)); + Scalar nrmse = (sum_sq_ref > 0) ? std::sqrt(sum_sq_err / sum_sq_ref) : 0.0f; + + std::cout << "\n"; + std::cout << " 最大绝对误差: " << std::scientific << std::setprecision(4) << matmul_err << "\n"; + std::cout << " RMSE: " << std::scientific << std::setprecision(4) << rmse << "\n"; + std::cout << " NRMSE: " << std::scientific << std::setprecision(4) << nrmse << "\n"; + + const bool matmul_ok = (matmul_err < 1e-2f) && (nrmse < 1e-3f); + std::cout << " 结果: " << (matmul_ok ? "✅ 通过" : "❌ 失败") << "\n\n"; + + if (!matmul_ok) + { + std::cerr << "matmul 正确性验证失败，跳过后续测试。\n"; + return 1; + } + + // ── 3b. 转置 matmul 测试 ────────────────────────────────────── + std::cout << "[3b] 转置 matmul 测试..." << std::flush; + { + // A^T * B + auto cpu_r = engine_matmul(*cpu_engine, A, B, true, false); + auto gpu_r = engine_matmul(*gpu_engine, A, B, true, false); + if (cpu_r && gpu_r) + { + Scalar err = max_abs_diff(*cpu_r, *gpu_r); + std::cout << "\n A^T * B 最大误差: " << std::scientific << std::setprecision(4) << err + << (err < 1e-2f ? " ✅" : " ❌") << "\n"; + } + else std::cout << "\n ❌ A^T * B 失败\n"; + + // A * B^T + auto cpu_r2 = engine_matmul(*cpu_engine, A, B, false, true); + auto gpu_r2 = engine_matmul(*gpu_engine, A, B, false, true); + if (cpu_r2 && gpu_r2) + { + Scalar err = max_abs_diff(*cpu_r2, *gpu_r2); + std::cout << " A * B^T 最大误差: " << std::scientific << std::setprecision(4) << err + << (err < 1e-2f ? " ✅" : " ❌") << "\n"; + } + else std::cout << " ❌ A * B^T 失败\n"; + } + + // ── 4. roundtrip + 逐元素 + 归约测试 ───────────────────────── + std::cout << "\n[4/6] roundtrip + 逐元素 + 归约测试...\n"; + { + // 4a. Upload→Download roundtrip + auto a_gpu_r = gpu_engine->from_matrix(A); + if (a_gpu_r) + { + auto a_back_r = gpu_engine->to_matrix(*a_gpu_r); + if (a_back_r) + { + Scalar err = max_abs_diff(A, *a_back_r); + std::cout << " Upload→Download roundtrip 最大误差: " + << std::scientific << std::setprecision(4) << err + << (err == 0.0f ? " ✅" : " ❌") << "\n"; + } + else std::cout << " ❌ roundtrip download 失败: " << a_back_r.error().message << "\n"; + } + else std::cout << " ❌ roundtrip upload 失败\n"; + + // 4b. 逐元素二元: Max(A, B) ≈ elementwise (CPU 参考用 CPU 计算) + auto a_cpu_t = cpu_engine->from_matrix(A); + auto b_cpu_t = cpu_engine->from_matrix(B); + auto a_gpu_t = gpu_engine->from_matrix(A); + auto b_gpu_t = gpu_engine->from_matrix(B); + if (a_cpu_t && b_cpu_t && a_gpu_t && b_gpu_t) + { + // Max(A, B) + auto max_cpu_r = cpu_engine->elementwise_binary(BinaryOp::Max, *a_cpu_t, *b_cpu_t); + auto max_gpu_r = gpu_engine->elementwise_binary(BinaryOp::Max, *a_gpu_t, *b_gpu_t); + if (max_cpu_r && max_gpu_r) + { + auto mc = cpu_engine->to_matrix(*max_cpu_r); + auto mg = gpu_engine->to_matrix(*max_gpu_r); + if (mc && mg) + { + Scalar err = max_abs_diff(*mc, *mg); + std::cout << " elementwise Max(A,B) 最大误差: " + << std::scientific << std::setprecision(4) << err + << (err < 1e-5f ? " ✅" : " ❌") << "\n"; + } + } + else std::cout << " ❌ elementwise Max 失败\n"; + + // 4c. 逐元素一元: Exp(A) + auto exp_cpu_r = cpu_engine->elementwise_unary(UnaryOp::Exp, *a_cpu_t); + auto exp_gpu_r = gpu_engine->elementwise_unary(UnaryOp::Exp, *a_gpu_t); + if (exp_cpu_r && exp_gpu_r) + { + auto ec = cpu_engine->to_matrix(*exp_cpu_r); + auto eg = gpu_engine->to_matrix(*exp_gpu_r); + if (ec && eg) + { + Scalar err = max_abs_diff(*ec, *eg); + std::cout << " elementwise Exp(A) 最大误差: " + << std::scientific << std::setprecision(4) << err + << (err < 1e-4f ? " ✅" : " ❌") << "\n"; + } + } + else std::cout << " ❌ elementwise Exp 失败\n"; + + // 4d. 归约: col_reduce_sum(A) → (1, N) + auto csum_cpu_r = cpu_engine->col_reduce_sum(*a_cpu_t); + auto csum_gpu_r = gpu_engine->col_reduce_sum(*a_gpu_t); + if (csum_cpu_r && csum_gpu_r) + { + auto sc = cpu_engine->to_matrix(*csum_cpu_r); + auto sg = gpu_engine->to_matrix(*csum_gpu_r); + if (sc && sg) + { + Scalar err = max_abs_diff(*sc, *sg); + std::cout << " col_reduce_sum(A) 最大误差: " + << std::scientific << std::setprecision(4) << err + << (err < 1e-3f ? " ✅" : " ❌") << "\n"; + } + } + else std::cout << " ❌ col_reduce_sum 失败\n"; + + // 4e. 归约: col_reduce_max(A) → (1, N) + auto cmax_cpu_r = cpu_engine->col_reduce_max(*a_cpu_t); + auto cmax_gpu_r = gpu_engine->col_reduce_max(*a_gpu_t); + if (cmax_cpu_r && cmax_gpu_r) + { + auto mc2 = cpu_engine->to_matrix(*cmax_cpu_r); + auto mg2 = gpu_engine->to_matrix(*cmax_gpu_r); + if (mc2 && mg2) + { + Scalar err = max_abs_diff(*mc2, *mg2); + std::cout << " col_reduce_max(A) 最大误差: " + << std::scientific << std::setprecision(4) << err + << (err < 1e-5f ? " ✅" : " ❌") << "\n"; + } + } + else std::cout << " ❌ col_reduce_max 失败\n"; + } + } + + // ── 5. 性能测试 ─────────────────────────────────────────────── + std::cout << "\n[5/6] matmul 性能测试 (" << iters << " 次迭代)...\n"; + + // CPU 性能 + auto t_cpu_start = std::chrono::high_resolution_clock::now(); + for (int i = 0; i < iters; ++i) + { + auto r = engine_matmul(*cpu_engine, A, B); + if (!r) { std::cerr << "CPU matmul 性能测试失败\n"; return 1; } + } + auto t_cpu_end = std::chrono::high_resolution_clock::now(); + double cpu_ms = std::chrono::duration(t_cpu_end - t_cpu_start).count() / iters; + + // GPU 性能（含上传/下载开销，反映端到端延迟） + auto t_gpu_start = std::chrono::high_resolution_clock::now(); + for (int i = 0; i < iters; ++i) + { + auto r = engine_matmul(*gpu_engine, A, B); + if (!r) { std::cerr << "GPU matmul 性能测试失败\n"; return 1; } + } + auto t_gpu_end = std::chrono::high_resolution_clock::now(); + double gpu_ms = std::chrono::duration(t_gpu_end - t_gpu_start).count() / iters; + + // 防御：计时为 0 时（极小矩阵/低精度时钟）避免除零输出 inf + const double speedup = (gpu_ms > 0.0) ? cpu_ms / gpu_ms : 0.0; + const double gflops = (gpu_ms > 0.0) ? (2.0 * N * N * N) / (gpu_ms * 1e6) : 0.0; + + std::cout << "\n"; + std::cout << " CPU 时间 (含上传/下载): " << std::fixed << std::setprecision(2) << cpu_ms << " ms\n"; + std::cout << " GPU 时间 (含上传/下载): " << std::fixed << std::setprecision(2) << gpu_ms << " ms\n"; + std::cout << " 加速比: " << std::fixed << std::setprecision(2) << speedup << "×\n"; + std::cout << " GPU GFLOPS (含传输开销): " << std::fixed << std::setprecision(1) << gflops << "\n\n"; + + // ── 6. batch 模式测试 ───────────────────────────────────────── + std::cout << "[6/6] batch 模式 + 链式 matmul 测试...\n"; + { + // batch begin → matmul → batch end → download + auto a_t = gpu_engine->from_matrix(A); + auto b_t = gpu_engine->from_matrix(B); + if (a_t && b_t) + { + auto batch_r = gpu_engine->begin_batch(); + if (batch_r) + { + auto c_t = gpu_engine->matmul(*a_t, *b_t, false, false); + if (c_t) + { + auto end_r = gpu_engine->end_batch(); + if (end_r) + { + auto c_res = gpu_engine->to_matrix(*c_t); + if (c_res) + { + Scalar err = max_abs_diff(*cpu_c_r, *c_res); + std::cout << " Batch+matmul 最大误差: " + << std::scientific << std::setprecision(4) << err + << (err < 1e-2f ? " ✅" : " ❌") << "\n"; + } + else std::cout << " ❌ download failed: " << c_res.error().message << "\n"; + } + else std::cout << " ❌ end_batch failed: " << end_r.error().message << "\n"; + } + else std::cout << " ❌ matmul failed: " << c_t.error().message << "\n"; + } + else std::cout << " ❌ begin_batch failed: " << batch_r.error().message << "\n"; + } + + // 链式: matmul(A, B) → elementwise Exp → col_reduce_sum + // 诊断日志：逐步对比 CPU vs GPU，定位误差来源 + if (a_t && b_t) + { + std::cout << "\n ── 链式诊断: matmul→Exp→col_reduce_sum ──\n"; + + // ── Step 0: CPU 参考计算 ── + auto a_cpu = cpu_engine->from_matrix(A); + auto b_cpu = cpu_engine->from_matrix(B); + auto c_cpu = cpu_engine->matmul(*a_cpu, *b_cpu, false, false); + auto exp_cpu = c_cpu ? cpu_engine->elementwise_unary(UnaryOp::Exp, *c_cpu) : nn::Result{}; + auto sum_cpu = exp_cpu ? cpu_engine->col_reduce_sum(*exp_cpu) : nn::Result{}; + auto res_cpu = sum_cpu ? cpu_engine->to_matrix(*sum_cpu) : nn::Result{}; + + // ── Step 1: matmul(A, B) on GPU ── + auto c_t = gpu_engine->matmul(*a_t, *b_t, false, false); + if (!c_t) { std::cout << " ❌ matmul failed\n"; } + else if (c_cpu) + { + auto c_gpu_m = gpu_engine->to_matrix(*c_t); + auto c_cpu_m = cpu_engine->to_matrix(*c_cpu); + if (c_gpu_m && c_cpu_m) + { + Scalar mm_err = max_abs_diff(*c_cpu_m, *c_gpu_m); + // 统计 matmul 输出范围 + Scalar c_min = std::numeric_limits::max(); + Scalar c_max = std::numeric_limits::lowest(); + Scalar c_abs_max = 0; + for (auto v : c_cpu_m->span()) { + c_min = std::min(c_min, v); + c_max = std::max(c_max, v); + c_abs_max = std::max(c_abs_max, std::fabs(v)); + } + std::cout << " [1] matmul 输出范围: [" << std::fixed << std::setprecision(2) + << c_min << ", " << c_max << "] |max|=" << c_abs_max << "\n"; + std::cout << " matmul CPU vs GPU 最大误差: " << std::scientific << std::setprecision(4) + << mm_err << "\n"; + + // ── Step 2: Exp(matmul) on GPU ── + auto exp_t = gpu_engine->elementwise_unary(UnaryOp::Exp, *c_t); + if (!exp_t) { std::cout << " ❌ Exp failed\n"; } + else if (exp_cpu) + { + auto exp_gpu_m = gpu_engine->to_matrix(*exp_t); + auto exp_cpu_m = cpu_engine->to_matrix(*exp_cpu); + if (exp_gpu_m && exp_cpu_m) + { + Scalar exp_err = max_abs_diff(*exp_cpu_m, *exp_gpu_m); + // 统计 exp 输出范围 + Scalar e_min = std::numeric_limits::max(); + Scalar e_max = std::numeric_limits::lowest(); + Scalar e_abs_max = 0; + for (auto v : exp_cpu_m->span()) { + e_min = std::min(e_min, v); + e_max = std::max(e_max, v); + e_abs_max = std::max(e_abs_max, std::fabs(v)); + } + std::cout << " [2] exp() 输出范围: [" << std::scientific << std::setprecision(4) + << e_min << ", " << e_max << "] |max|=" << e_abs_max << "\n"; + Scalar amplify = (mm_err > 0) ? (exp_err / mm_err) : 0; + std::cout << " exp() CPU vs GPU 最大误差: " << std::scientific << std::setprecision(4) + << exp_err << " (放大 " << std::fixed << std::setprecision(0) << amplify << "x)\n"; + + // ── Step 3: col_reduce_sum(exp) on GPU ── + auto sum_t = gpu_engine->col_reduce_sum(*exp_t); + if (!sum_t) { std::cout << " ❌ col_reduce_sum failed\n"; } + else + { + auto res = gpu_engine->to_matrix(*sum_t); + if (res && res_cpu) + { + Scalar final_err = max_abs_diff(*res_cpu, *res); + // 统计 reduce 输出范围 + Scalar s_min = std::numeric_limits::max(); + Scalar s_max = std::numeric_limits::lowest(); + for (auto v : res_cpu->span()) { + s_min = std::min(s_min, v); + s_max = std::max(s_max, v); + } + std::cout << " [3] col_reduce_sum 输出范围: [" << std::scientific << std::setprecision(4) + << s_min << ", " << s_max << "]\n"; + std::cout << " 最终绝对误差: " << std::scientific << std::setprecision(4) + << final_err << "\n"; + + // 相对误差（更合理的评判标准） + Scalar ref_abs_max = 0; + for (auto v : res_cpu->span()) ref_abs_max = std::max(ref_abs_max, std::fabs(v)); + Scalar rel_err = (ref_abs_max > 0) ? (final_err / ref_abs_max) : 0; + std::cout << " 相对误差: " << std::scientific << std::setprecision(4) + << rel_err << (rel_err < 1e-3f ? " OK" : " FAIL") << "\n"; + std::cout << " 结论: exp() 将 matmul 的 " << std::scientific << std::setprecision(2) + << mm_err << " 误差放大为 " << final_err + << " (数学预期行为，非传递异常)\n"; + } + } + } + } + } + } + } + } + + std::cout << "\n========================================\n"; + std::cout << " 测试完成\n"; + std::cout << "========================================\n"; + return 0; +} +#endif diff --git a/src/mnist_bench.cpp b/src/mnist_bench.cpp new file mode 100644 index 00000000..37073184 --- /dev/null +++ b/src/mnist_bench.cpp @@ -0,0 +1,492 @@ +// ── MNIST MLP CPU vs GPU 对比基准测试（新引擎化架构） ────────────────────── +// +// 在相同数据集、相同模型上分别用 CpuEngine 和 GpuEngine 跑训练和推理， +// 输出逐 epoch 时间、总时间、准确率及推理吞吐量对比。 +// +// 新架构下每个 Model 绑定一个 ComputeEngine；CPU/GPU 对比需要分别构建 +// 独立的 engine + model 实例。 +// ───────────────────────────────────────────────────────────────────────── + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using nn::Scalar; +namespace fs = std::filesystem; + +// ═══════════════════════════════════════════════════════════════════════════ +// 计时工具 +// ═══════════════════════════════════════════════════════════════════════════ +using Clock = std::chrono::high_resolution_clock; + +struct TimingStats { + double total_ms = 0; + double min_ms = 1e18; + double max_ms = 0; + std::vector per_epoch; +}; + +// ═══════════════════════════════════════════════════════════════════════════ +// 命令行参数 +// ═══════════════════════════════════════════════════════════════════════════ +struct BenchConfig { + std::string dataset_path = "datasets/mnist_data"; + int epochs = 5; + std::size_t batch_size = 64; + int warmup = 2; // 推理预热轮数 + int infer_iters = 100; // 推理计时迭代次数 + Scalar lr = 0.001; +}; + +void print_usage(const char* prog) +{ + std::cout + << "MNIST MLP CPU vs GPU 对比基准测试 (引擎化架构)\n\n" + << "用法: " << prog << " [选项]\n\n" + << "选项:\n" + << " --epochs 训练轮数 (默认: 5)\n" + << " --batch-size 批大小 (默认: 64)\n" + << " --dataset 数据集目录 (默认: datasets/mnist_data)\n" + << " --warmup 推理预热轮数 (默认: 2)\n" + << " --infer-iters 推理计时迭代次数 (默认: 100)\n" + << " --lr 学习率 (默认: 0.001)\n" + << " --help 显示此帮助信息\n"; +} + +// 数字解析辅助：解析失败时打印错误并退出（替代会抛异常的 std::stoi/stod） +template +T parse_num_or_die(const char* s, const char* opt) +{ + auto v = nn::parse_number(s); + if (!v) + { + std::cerr << "无效的 " << opt << " 值: " << v.error().message << "\n"; + std::exit(1); + } + return *v; +} + +BenchConfig parse_args(int argc, char* argv[]) +{ + BenchConfig cfg; + for (int i = 1; i < argc; ++i) + { + std::string arg = argv[i]; + if (arg == "--help") { print_usage(argv[0]); std::exit(0); } + else if (arg == "--epochs" && i + 1 < argc) + { + cfg.epochs = parse_num_or_die(argv[++i], "--epochs"); + if (cfg.epochs <= 0) { std::cerr << "--epochs 必须为正整数\n"; std::exit(1); } + } + else if (arg == "--batch-size" && i + 1 < argc) + { + int bs = parse_num_or_die(argv[++i], "--batch-size"); + if (bs <= 0) { std::cerr << "--batch-size 必须为正整数\n"; std::exit(1); } + cfg.batch_size = static_cast(bs); + } + else if (arg == "--dataset" && i + 1 < argc) + cfg.dataset_path = argv[++i]; + else if (arg == "--warmup" && i + 1 < argc) + { + cfg.warmup = parse_num_or_die(argv[++i], "--warmup"); + if (cfg.warmup < 0) { std::cerr << "--warmup 必须为非负整数\n"; std::exit(1); } + } + else if (arg == "--infer-iters" && i + 1 < argc) + { + cfg.infer_iters = parse_num_or_die(argv[++i], "--infer-iters"); + if (cfg.infer_iters <= 0) { std::cerr << "--infer-iters 必须为正整数\n"; std::exit(1); } + } + else if (arg == "--lr" && i + 1 < argc) + cfg.lr = parse_num_or_die(argv[++i], "--lr"); + else { std::cerr << "未知参数: " << arg << "\n"; std::exit(1); } + } + return cfg; +} + +// ═══════════════════════════════════════════════════════════════════════════ +// 单轮训练（返回 epoch 耗时毫秒 + 平均 loss + 训练/测试准确率） +// ═══════════════════════════════════════════════════════════════════════════ +struct EpochResult { + double time_ms; + Scalar avg_loss; + Scalar train_acc; + Scalar test_acc; +}; + +nn::Result train_one_epoch( + nn::Model& model, nn::ComputeEngine& engine, + nn::Optimizer& optimizer, nn::CrossEntropyLoss& ce_loss, + const nn::Matrix& train_x, const nn::Matrix& train_y, + const nn::Matrix& test_x, const nn::Matrix& test_y, + std::size_t batch_size, std::mt19937_64& rng) +{ + const std::size_t N = train_x.cols(); + const std::size_t num_batches = N / batch_size; + if (num_batches == 0) + return std::unexpected(nn::Error{ + "train_one_epoch: 样本数小于 batch_size，无法构成训练批次"}); + + nn::Matrix x_batch(train_x.rows(), batch_size); + nn::Matrix y_batch(train_y.rows(), batch_size); + + std::vector indices(N); + std::iota(indices.begin(), indices.end(), 0); + std::shuffle(indices.begin(), indices.end(), rng); + + Scalar total_loss = 0.0; + + auto t0 = Clock::now(); + for (std::size_t b = 0; b < num_batches; ++b) + { + const std::size_t start = b * batch_size; + const std::size_t R_x = train_x.rows(); + const std::size_t R_y = train_y.rows(); + const auto x_src = train_x.span().data(); + const auto y_src = train_y.span().data(); + auto x_dst = x_batch.span().data(); + auto y_dst = y_batch.span().data(); + for (std::size_t k = 0; k < batch_size; ++k) + { + const std::size_t src_col = indices[start + k]; + for (std::size_t r = 0; r < R_x; ++r) + x_dst[r * batch_size + k] = x_src[r * N + src_col]; + for (std::size_t r = 0; r < R_y; ++r) + y_dst[r * batch_size + k] = y_src[r * N + src_col]; + } + + // Matrix → Tensor（上传到引擎设备） + auto x_tensor_r = engine.from_matrix(x_batch); + if (!x_tensor_r) return std::unexpected(std::move(x_tensor_r).error()); + auto y_tensor_r = engine.from_matrix(y_batch); + if (!y_tensor_r) return std::unexpected(std::move(y_tensor_r).error()); + + // 前向 + 损失 + auto out_r = model.forward(*x_tensor_r); + if (!out_r) return std::unexpected(std::move(out_r).error()); + auto loss_r = ce_loss.forward(engine, *out_r, *y_tensor_r); + if (!loss_r) return std::unexpected(std::move(loss_r).error()); + total_loss += *loss_r; + + // 反向 + 优化 + auto grad_r = ce_loss.backward(); + if (!grad_r) return std::unexpected(std::move(grad_r).error()); + auto bwd_r = model.backward(*grad_r); + if (!bwd_r) return std::unexpected(std::move(bwd_r).error()); + (void)optimizer.step(); + (void)optimizer.zero_grad(); + } + auto t1 = Clock::now(); + + double ms = std::chrono::duration(t1 - t0).count(); + Scalar avg_loss = total_loss / static_cast(num_batches); + auto train_acc_r = nn::cli::evaluate_mnist(model, engine, train_x, train_y); + auto test_acc_r = nn::cli::evaluate_mnist(model, engine, test_x, test_y); + if (!train_acc_r || !test_acc_r) + return std::unexpected(!train_acc_r ? train_acc_r.error() : test_acc_r.error()); + + return EpochResult{ms, avg_loss, *train_acc_r, *test_acc_r}; +} + +// ═══════════════════════════════════════════════════════════════════════════ +// 推理基准测试（对单张图片反复 forward，测吞吐量） +// ═══════════════════════════════════════════════════════════════════════════ +struct InferResult { + double avg_ms; + double throughput; // images/sec +}; + +nn::Result bench_inference( + nn::Model& model, nn::ComputeEngine& engine, + const nn::Matrix& single_image, int warmup, int iters) +{ + if (iters <= 0) + return std::unexpected(nn::Error{"bench_inference: iters 必须为正整数"}); + // 预热 + for (int i = 0; i < warmup; ++i) + { + auto x_r = engine.from_matrix(single_image); + if (!x_r) return std::unexpected(std::move(x_r).error()); + auto out_r = model.forward(*x_r); + if (!out_r) return std::unexpected(std::move(out_r).error()); + } + + // 计时 + auto t0 = Clock::now(); + for (int i = 0; i < iters; ++i) + { + auto x_r = engine.from_matrix(single_image); + if (!x_r) return std::unexpected(std::move(x_r).error()); + auto out_r = model.forward(*x_r); + if (!out_r) return std::unexpected(std::move(out_r).error()); + } + auto t1 = Clock::now(); + + double total_ms = std::chrono::duration(t1 - t0).count(); + double avg_ms = total_ms / iters; + double throughput = (avg_ms > 0) ? 1000.0 / avg_ms : 0; + return InferResult{avg_ms, throughput}; +} + +// ═══════════════════════════════════════════════════════════════════════════ +// 分隔线 + 表头 +// ═══════════════════════════════════════════════════════════════════════════ +void print_separator() +{ + std::cout << "══════════════════════════════════════════════════════════════\n"; +} + +void print_epoch_header(const char* title) +{ + std::cout << "\n " << title << "\n\n"; + std::cout << std::left + << std::setw(8) << "Epoch" + << std::setw(12) << "Loss" + << std::setw(12) << "Train Acc" + << std::setw(12) << "Test Acc" + << std::setw(12) << "Time(ms)" + << "\n"; + std::cout << std::string(56, '-') << "\n"; +} + +void print_epoch_row(const EpochResult& ep, int epoch) +{ + std::cout << std::left + << std::setw(8) << (epoch + 1) + << std::setw(12) << std::fixed << std::setprecision(4) << ep.avg_loss + << std::setw(12) << std::setprecision(2) << (ep.train_acc * 100.0) << "%" + << std::setw(12) << (ep.test_acc * 100.0) << "%" + << std::setw(12) << std::setprecision(1) << ep.time_ms + << "\n"; +} + +// ═══════════════════════════════════════════════════════════════════════════ +// 单设备训练 + 推理基准（返回 TimingStats + InferResult） +// ═══════════════════════════════════════════════════════════════════════════ +struct DeviceBenchResult { + TimingStats train_stats; + InferResult infer; +}; + +nn::Result bench_device( + nn::ComputeEngine& engine, + const BenchConfig& cfg, + const nn::Matrix& train_x, const nn::Matrix& train_y, + const nn::Matrix& test_x, const nn::Matrix& test_y, + const nn::Matrix& single_img, + std::uint64_t rng_seed, const char* device_name) +{ + auto model_r = nn::build_mnist_mlp_model(engine); + if (!model_r) return std::unexpected(std::move(model_r).error()); + auto model = std::move(*model_r); + + auto optimizer = nn::create_optimizer( + "adam", engine, model.parameters(), model.param_gradients(), cfg.lr); + if (!optimizer) + return std::unexpected(nn::Error{"bench_device: failed to create optimizer"}); + nn::CrossEntropyLoss ce_loss; + + std::mt19937_64 rng{rng_seed}; + TimingStats stats; + + print_epoch_header(device_name); + + for (int epoch = 0; epoch < cfg.epochs; ++epoch) + { + auto ep_r = train_one_epoch(model, engine, *optimizer, ce_loss, + train_x, train_y, test_x, test_y, + cfg.batch_size, rng); + if (!ep_r) return std::unexpected(std::move(ep_r).error()); + auto ep = *ep_r; + + stats.total_ms += ep.time_ms; + stats.min_ms = std::min(stats.min_ms, ep.time_ms); + stats.max_ms = std::max(stats.max_ms, ep.time_ms); + stats.per_epoch.push_back(ep.time_ms); + + print_epoch_row(ep, epoch); + } + + // 推理基准 + std::cout << "\n " << device_name << " 推理基准 (" + << cfg.infer_iters << " 次迭代, 预热 " << cfg.warmup << " 轮)...\n"; + auto infer_r = bench_inference(model, engine, single_img, cfg.warmup, cfg.infer_iters); + if (!infer_r) return std::unexpected(std::move(infer_r).error()); + std::cout << " 平均延迟: " << std::fixed << std::setprecision(3) << infer_r->avg_ms << " ms\n"; + std::cout << " 吞吐量: " << std::setprecision(1) << infer_r->throughput << " images/s\n"; + + return DeviceBenchResult{stats, *infer_r}; +} + +// ═══════════════════════════════════════════════════════════════════════════ +// 主函数 +// ═══════════════════════════════════════════════════════════════════════════ +int main(int argc, char* argv[]) +{ + BenchConfig cfg = parse_args(argc, argv); + + print_separator(); + std::cout << " MNIST MLP CPU vs GPU 对比基准测试 (引擎化架构)\n"; + print_separator(); + std::cout << " 模型: 784→512→256→128→64→10 (LayerNorm+GeLU)\n"; + std::cout << " 训练轮数: " << cfg.epochs << "\n"; + std::cout << " 批大小: " << cfg.batch_size << "\n"; + std::cout << " 学习率: " << cfg.lr << "\n"; + std::cout << " 推理预热: " << cfg.warmup << " 轮\n"; + std::cout << " 推理迭代: " << cfg.infer_iters << " 次\n"; + print_separator(); + + // ── 加载数据 ───────────────────────────────────────────── + std::cout << "\n加载数据集...\n"; + auto train_result = nn::cli::load_mnist_csv(cfg.dataset_path + "/train.csv"); + if (!train_result) { std::cerr << "加载训练集失败: " << train_result.error().message << "\n"; return 1; } + auto [train_x, train_y] = std::move(*train_result); + + auto test_result = nn::cli::load_mnist_csv(cfg.dataset_path + "/test.csv"); + if (!test_result) { std::cerr << "加载测试集失败: " << test_result.error().message << "\n"; return 1; } + auto [test_x, test_y] = std::move(*test_result); + + std::cout << " 训练集: " << train_x.cols() << " 样本\n"; + std::cout << " 测试集: " << test_x.cols() << " 样本\n"; + std::cout << " 特征维度: " << train_x.rows() << "\n"; + + // 取一张测试图片用于推理基准 + nn::Matrix single_img(train_x.rows(), 1); + for (std::size_t r = 0; r < train_x.rows(); ++r) + single_img.set_value_unchecked(r, 0, test_x.at_unchecked(r, 0)); + + // ════════════════════════════════════════════════════════════════════════ + // Phase 1: CPU 训练 + 推理基准 + // ════════════════════════════════════════════════════════════════════════ + print_separator(); + std::cout << " Phase 1: CPU 训练 (CpuEngine)\n"; + print_separator(); + + auto cpu_engine = std::make_unique(); + auto cpu_bench_r = bench_device(*cpu_engine, cfg, + train_x, train_y, test_x, test_y, + single_img, 42, "CPU"); + if (!cpu_bench_r) + { + std::cerr << "CPU 基准测试失败: " << cpu_bench_r.error().message << "\n"; + return 1; + } + auto cpu_bench = *cpu_bench_r; + const auto& cpu_train_stats = cpu_bench.train_stats; + const auto& cpu_infer = cpu_bench.infer; + + // ════════════════════════════════════════════════════════════════════════ + // Phase 2: GPU 训练 + 推理基准（需要 Vulkan） + // ════════════════════════════════════════════════════════════════════════ +#ifdef NN_HAS_VULKAN + print_separator(); + std::cout << " Phase 2: GPU 训练 (GpuEngine / Vulkan)\n"; + print_separator(); + + auto& backend = nn::GpuBackend::instance(); + auto init_result = backend.initialize(); + if (!init_result) + { + std::cerr << "\n GPU 初始化失败: " << init_result.error().message << "\n"; + std::cerr << " 跳过 GPU 测试，仅报告 CPU 结果。\n\n"; + } + else + { + auto gpu_engine = std::make_unique(backend); + auto gpu_bench_r = bench_device(*gpu_engine, cfg, + train_x, train_y, test_x, test_y, + single_img, 42, "GPU"); + if (!gpu_bench_r) + { + std::cerr << "GPU 基准测试失败: " << gpu_bench_r.error().message << "\n"; + } + else + { + auto gpu_bench = *gpu_bench_r; + const auto& gpu_train_stats = gpu_bench.train_stats; + const auto& gpu_infer = gpu_bench.infer; + + // ── 汇总对比 ────────────────────────────────────────────── + print_separator(); + std::cout << " 汇总对比\n"; + print_separator(); + + double avg_cpu_epoch = cpu_train_stats.total_ms / cfg.epochs; + double avg_gpu_epoch = gpu_train_stats.total_ms / cfg.epochs; + double train_speedup = avg_cpu_epoch / avg_gpu_epoch; + double infer_speedup = cpu_infer.avg_ms / gpu_infer.avg_ms; + + std::cout << std::left + << std::setw(24) << "指标" + << std::setw(16) << "CPU" + << std::setw(16) << "GPU" + << std::setw(12) << "加速比" + << "\n"; + std::cout << std::string(68, '-') << "\n"; + + std::cout << std::left + << std::setw(24) << "训练总时间 (ms)" + << std::setw(16) << std::fixed << std::setprecision(1) << cpu_train_stats.total_ms + << std::setw(16) << gpu_train_stats.total_ms + << std::setw(12) << std::setprecision(2) << (cpu_train_stats.total_ms / gpu_train_stats.total_ms) << "x" + << "\n"; + + std::cout << std::left + << std::setw(24) << "平均 Epoch 时间 (ms)" + << std::setw(16) << std::fixed << std::setprecision(1) << avg_cpu_epoch + << std::setw(16) << avg_gpu_epoch + << std::setw(12) << std::setprecision(2) << train_speedup << "x" + << "\n"; + + std::cout << std::left + << std::setw(24) << "推理延迟 (ms/img)" + << std::setw(16) << std::fixed << std::setprecision(3) << cpu_infer.avg_ms + << std::setw(16) << gpu_infer.avg_ms + << std::setw(12) << std::setprecision(2) << infer_speedup << "x" + << "\n"; + + std::cout << std::left + << std::setw(24) << "推理吞吐量 (img/s)" + << std::setw(16) << std::fixed << std::setprecision(1) << cpu_infer.throughput + << std::setw(16) << gpu_infer.throughput + << std::setw(12) << std::setprecision(2) << infer_speedup << "x" + << "\n\n"; + + print_separator(); + std::cout << " 完成\n"; + print_separator(); + return 0; + } + } +#endif + + // 无 Vulkan 时的 CPU-only 汇总 + print_separator(); + std::cout << " CPU Only 汇总\n"; + print_separator(); + double avg_cpu_epoch = cpu_train_stats.total_ms / cfg.epochs; + std::cout << " 训练总时间: " << std::fixed << std::setprecision(1) << cpu_train_stats.total_ms << " ms\n"; + std::cout << " 平均 Epoch: " << avg_cpu_epoch << " ms\n"; + std::cout << " 推理延迟: " << std::setprecision(3) << cpu_infer.avg_ms << " ms/img\n"; + std::cout << " 推理吞吐量: " << std::setprecision(1) << cpu_infer.throughput << " images/s\n\n"; + std::cout << " 提示: 编译时启用 Vulkan SDK 以启用 GPU 对比测试。\n"; + print_separator(); + + return 0; +} diff --git a/src/mnist_infer.cpp b/src/mnist_infer.cpp index 88053a92..aa2fd5c1 100644 --- a/src/mnist_infer.cpp +++ b/src/mnist_infer.cpp @@ -1,37 +1,48 @@ +// ── MNIST 手写数字推理程序（新引擎化架构） ───────────────────────────────── +// +// 数据流： +// CSV 行 → Matrix(784, 1) → engine.from_matrix → Tensor[device] +// → model.forward(Tensor) → Tensor +// → engine.to_matrix → CPU softmax + top-k 展示 +// +// 引擎选择：--gpu 启用 GpuEngine（需要 Vulkan），否则 CpuEngine。 +// ───────────────────────────────────────────────────────────────────────── + #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include + #include -#include #include -#include #include +#include +#include +#include +#include +#include +#include +#include +#include namespace fs = std::filesystem; -// ==================== 常量 ==================== -// 现在使用 nn::MNIST_INPUT_DIM, nn::MNIST_NUM_CLASSES, nn::MNIST_LAYER_DIMS +using nn::Scalar; // ==================== 帮助信息 ==================== void print_usage(const char *prog) { std::cout - << "MNIST 手写数字推理程序\n\n" + << "MNIST 手写数字推理程序 (引擎化架构)\n\n" << "用法:\n" << " " << prog << " [选项] 推理单张图片\n" << " " << prog << " <目录> [选项] 批量推理目录下所有 CSV\n\n" << "选项:\n" << " --model 模型文件路径 (默认: pretrained/model.bin)\n" - << " V2 格式模型文件自动读取模型规格，无需指定 --model-type\n" - << " --model-type 模型类型: mlp/transformer (默认: mlp)\n" - << " 仅在 V1 旧格式模型文件时需要手动指定\n" << " --topk 显示前 n 个预测结果 (默认: 3)\n" << " --show-pixels 显示像素矩阵 (调试用)\n" + << " --gpu 启用 GPU 加速 (需要 Vulkan SDK)\n" + << " --cuda 启用 CUDA GPU 加速 (需要 CUDA Toolkit)\n" << " --help 显示此帮助信息\n"; } @@ -39,13 +50,14 @@ void print_usage(const char *prog) struct InferConfig { std::string model_path = "pretrained/model.bin"; - std::string model_type = "mlp"; // 仅 V1 旧格式使用 std::string input_path; int topk = 3; bool show_pixels = false; + bool gpu_enabled = false; + bool cuda_enabled = false; }; -InferConfig parse_args(int argc, char *argv[]) +nn::Result parse_args(int argc, char *argv[]) { InferConfig cfg; bool has_input = false; @@ -62,24 +74,29 @@ InferConfig parse_args(int argc, char *argv[]) { cfg.model_path = argv[++i]; } - else if (arg == "--model-type" && i + 1 < argc) - { - cfg.model_type = argv[++i]; - if (cfg.model_type != "mlp" && cfg.model_type != "transformer") - { - std::cerr << "未知模型类型: " << cfg.model_type - << "，可选: mlp, transformer\n"; - std::exit(1); - } - } else if (arg == "--topk" && i + 1 < argc) { - cfg.topk = std::stoi(argv[++i]); + auto v = nn::parse_number(argv[++i]); + if (!v) return std::unexpected(std::move(v).error()); + if (*v <= 0) + return std::unexpected(nn::Error{"--topk 必须为正整数"}); + if (*v > static_cast(nn::MNIST_NUM_CLASSES)) + return std::unexpected(nn::Error{ + "--topk 不能超过类别数 " + std::to_string(nn::MNIST_NUM_CLASSES)}); + cfg.topk = *v; } else if (arg == "--show-pixels") { cfg.show_pixels = true; } + else if (arg == "--gpu") + { + cfg.gpu_enabled = true; + } + else if (arg == "--cuda") + { + cfg.cuda_enabled = true; + } else if (!arg.starts_with("--")) { cfg.input_path = arg; @@ -87,69 +104,65 @@ InferConfig parse_args(int argc, char *argv[]) } else { - std::cerr << "未知参数: " << arg << "\n使用 --help 查看用法\n"; - std::exit(1); + return std::unexpected(nn::Error{"未知参数: " + arg + "，使用 --help 查看用法"}); } } if (!has_input) - { - std::cerr << "请指定图片文件或目录\n使用 --help 查看用法\n"; - std::exit(1); - } + return std::unexpected(nn::Error{"请指定图片文件或目录，使用 --help 查看用法"}); return cfg; } -// ==================== 构建网络 ==================== -// build_model 函数已移至 neuralnet.cpp/mnist_common.hpp 中的 nn::build_mnist_model() - -// ==================== 数据读取 ==================== -nn::Result load_image_from_csv(const std::string &csv_line) -{ - std::vector pixels; - std::stringstream ss(csv_line); - std::string token; - while (std::getline(ss, token, ',')) - { - pixels.push_back(std::stod(token)); - } - if (pixels.size() != nn::MNIST_INPUT_DIM) - return std::unexpected(nn::Error{"CSV 必须包含恰好 " + std::to_string(nn::MNIST_INPUT_DIM) + " 个值，实际: " + std::to_string(pixels.size())}); - - nn::Matrix img(nn::MNIST_INPUT_DIM, 1); - for (std::size_t i = 0; i < nn::MNIST_INPUT_DIM; ++i) - img.set_value_unchecked(i, 0, pixels[i]); - return img; -} - // ==================== 推理 + 置信度 ==================== struct Prediction { int digit; - double confidence; + Scalar confidence; }; -std::vector predict_with_confidence(nn::Model &model, const nn::Matrix &img, int topk) +// 前向：Matrix → engine.from_matrix → model.forward → engine.to_matrix +// 后续 softmax / top-k 在 CPU 上完成（单张图片，开销可忽略） +nn::Result> predict_with_confidence( + nn::Model &model, nn::ComputeEngine &engine, + const nn::Matrix &img, int topk) { - auto logits_result = model.forward(img); - if (!logits_result) return {}; // 返回空 vector - auto logits = std::move(*logits_result); + auto x_tensor_r = engine.from_matrix(img); + if (!x_tensor_r) + return std::unexpected(std::move(x_tensor_r).error()); + + auto logits_tensor_r = model.forward(*x_tensor_r); + if (!logits_tensor_r) + return std::unexpected(std::move(logits_tensor_r).error()); + + auto logits_r = engine.to_matrix(*logits_tensor_r); + if (!logits_r) + return std::unexpected(std::move(logits_r).error()); + const auto &logits = *logits_r; // Softmax 计算概率 - double max_val = logits.at_unchecked(0, 0); + Scalar max_val = logits.at_unchecked(0, 0); for (int c = 1; c < static_cast(nn::MNIST_NUM_CLASSES); ++c) max_val = std::max(max_val, logits.at_unchecked(c, 0)); - double sum_exp = 0.0; - std::vector probs(nn::MNIST_NUM_CLASSES); + Scalar sum_exp = 0.0; + std::vector probs(nn::MNIST_NUM_CLASSES); for (std::size_t c = 0; c < nn::MNIST_NUM_CLASSES; ++c) { probs[c] = std::exp(logits.at_unchecked(c, 0) - max_val); sum_exp += probs[c]; } - for (auto &p : probs) - p /= sum_exp; + // 防御：logits 全为 -inf 时 sum_exp == 0，回退为均匀分布避免 NaN + if (!(sum_exp > 0.0) || !std::isfinite(sum_exp)) + { + for (auto &p : probs) + p = Scalar{1} / static_cast(nn::MNIST_NUM_CLASSES); + } + else + { + for (auto &p : probs) + p /= sum_exp; + } // 按概率排序取 top-k std::vector indices(nn::MNIST_NUM_CLASSES); @@ -172,7 +185,7 @@ void show_pixels(const nn::Matrix &img) std::cout << " "; for (int c = 0; c < 28; ++c) { - double val = img.at_unchecked(r * 28 + c, 0); + Scalar val = img.at_unchecked(r * 28 + c, 0); if (val > 0.5) std::cout << "##"; else if (val > 0.1) @@ -185,7 +198,9 @@ void show_pixels(const nn::Matrix &img) } // ==================== 推理单张图片 ==================== -nn::Result infer_single(nn::Model &model, const std::string &filepath, const InferConfig &cfg) +nn::Result infer_single( + nn::Model &model, nn::ComputeEngine &engine, + const std::string &filepath, const InferConfig &cfg) { std::ifstream file(filepath); if (!file) @@ -193,8 +208,9 @@ nn::Result infer_single(nn::Model &model, const std::string &filepath, con std::string line; std::getline(file, line); - auto img_result = load_image_from_csv(line); - if (!img_result) { std::cerr << "Error: " << img_result.error().message << '\n'; return {}; } + auto img_result = nn::load_image_from_csv_line(line); + if (!img_result) + return std::unexpected(std::move(img_result).error()); auto img = std::move(*img_result); if (cfg.show_pixels) @@ -204,113 +220,127 @@ nn::Result infer_single(nn::Model &model, const std::string &filepath, con std::cout << "\n"; } - auto results = predict_with_confidence(model, img, cfg.topk); + auto results = predict_with_confidence(model, engine, img, cfg.topk); + if (!results) + return std::unexpected(std::move(results).error()); std::cout << fs::path(filepath).filename().string() << " -> "; - for (std::size_t k = 0; k < results.size(); ++k) + for (std::size_t k = 0; k < results->size(); ++k) { if (k > 0) std::cout << " "; - std::cout << results[k].digit - << " (" << std::fixed << std::setprecision(1) << results[k].confidence * 100.0 << "%)"; + std::cout << (*results)[k].digit + << " (" << std::fixed << std::setprecision(1) << (*results)[k].confidence * 100.0 << "%)"; } - std::cout << std::endl; return {};} + std::cout << std::endl; + return {}; +} // ==================== 主函数 ==================== int main(int argc, char *argv[]) { - try + auto cfg_result = parse_args(argc, argv); + if (!cfg_result) { - InferConfig cfg = parse_args(argc, argv); + std::cerr << "参数错误: " << cfg_result.error().message << std::endl; + return 1; + } + InferConfig cfg = std::move(*cfg_result); - // ── 从模型文件读取规格 ───────────────────────────────── - auto spec_result = nn::peek_model_spec(cfg.model_path); - if (!spec_result) - { - std::cerr << "读取模型文件失败: " << spec_result.error().message << std::endl; - return 1; - } - nn::ModelSpec spec = spec_result.value(); + // ── 从模型文件读取规格 ───────────────────────────────── + auto spec_result = nn::peek_model_spec(cfg.model_path); + if (!spec_result) + { + std::cerr << "读取模型文件失败: " << spec_result.error().message << std::endl; + return 1; + } + nn::ModelSpec spec = spec_result.value(); - nn::Model model; - if (spec.type != nn::ModelType::Unknown) - { - // V2 格式：自动从规格构建模型 - std::cout << "从模型文件读取规格 (V2 格式)\n"; - auto build_result = nn::build_mnist_model_from_spec(spec); - if (!build_result) - { - std::cerr << "构建模型失败: " << build_result.error().message << std::endl; - return 1; - } - model = std::move(*build_result); - } - else + // 支持的模型类型：MLP、Transformer（ViT） + if (spec.type == nn::ModelType::Unknown) + { + std::cout << "旧格式模型文件 (V1)，使用默认 MLP 架构\n"; + spec.type = nn::ModelType::MLP; + spec.layer_dims = nn::MNIST_LAYER_DIMS; + } + else if (spec.type == nn::ModelType::MLP) + { + std::cout << "从模型文件读取规格: MLP\n"; + } + else if (spec.is_transformer()) + { + std::cout << "从模型文件读取规格: Transformer (ViT)\n"; + } + else + { + std::cerr << "不支持的模型类型 (type=" + << static_cast(spec.type) << ")\n"; + return 1; + } + + // ── 创建计算引擎 ───────────────────────────────────────── + // 引擎必须先于 model 构造并晚于 model 析构（model 持有 engine 的非拥有指针） + nn::cli::EngineConfig eng_cfg; + eng_cfg.use_gpu = cfg.gpu_enabled; + eng_cfg.use_cuda = cfg.cuda_enabled; + auto engine = nn::cli::create_engine(eng_cfg, std::cout); + + // ── 构建模型（绑定引擎） ───────────────────────────────── + auto build_result = nn::build_mnist_model_from_spec(*engine, spec); + if (!build_result) + { + std::cerr << "构建模型失败: " << build_result.error().message << std::endl; + return 1; + } + auto model = std::move(*build_result); + + auto load_result = nn::load_model(cfg.model_path, model); + if (!load_result) + { + std::cerr << "加载模型失败: " << load_result.error().message << std::endl; + return 1; + } + std::cout << "模型已加载: " << cfg.model_path << "\n" << std::endl; + + fs::path input(cfg.input_path); + + if (fs::is_directory(input)) + { + // 批量推理 + std::vector csv_files; + for (auto &entry : fs::directory_iterator(input)) { - // V1 旧格式：使用 --model-type 参数 - std::cout << "旧格式模型文件 (V1)，使用 --model-type 参数: " << cfg.model_type << "\n"; - auto build_result = nn::build_mnist_model(cfg.model_type); - if (!build_result) - { - std::cerr << "构建模型失败: " << build_result.error().message << std::endl; - return 1; - } - model = std::move(*build_result); + if (entry.path().extension() == ".csv") + csv_files.push_back(entry.path()); } + std::sort(csv_files.begin(), csv_files.end()); - auto load_result = nn::load_model(cfg.model_path, model); - if (!load_result) + if (csv_files.empty()) { - std::cerr << "加载模型失败: " << load_result.error().message << std::endl; + std::cerr << "目录中没有 CSV 文件: " << cfg.input_path << std::endl; return 1; } - std::cout << "模型已加载: " << cfg.model_path << "\n" << std::endl; - fs::path input(cfg.input_path); + std::cout << "找到 " << csv_files.size() << " 个文件\n" << std::endl; - if (fs::is_directory(input)) + for (auto &f : csv_files) { - // 批量推理 - std::vector csv_files; - for (auto &entry : fs::directory_iterator(input)) - { - if (entry.path().extension() == ".csv") - csv_files.push_back(entry.path()); - } - std::sort(csv_files.begin(), csv_files.end()); - - if (csv_files.empty()) - { - std::cerr << "目录中没有 CSV 文件: " << cfg.input_path << std::endl; - return 1; - } - - std::cout << "找到 " << csv_files.size() << " 个文件\n" << std::endl; - - for (auto &f : csv_files) - { - auto result = infer_single(model, f.string(), cfg); - if (!result) { std::cerr << "推理失败: " << f.filename().string() << ": " << result.error().message << '\n'; } - } - - std::cout << "\n共推理 " << csv_files.size() << " 张图片" << std::endl; - } - else if (fs::is_regular_file(input)) - { - // 单张推理 - auto result = infer_single(model, input.string(), cfg); - if (!result) { std::cerr << "推理失败: " << result.error().message << '\n'; return 1; } - } - else - { - std::cerr << "输入路径不存在: " << cfg.input_path << std::endl; - return 1; + auto result = infer_single(model, *engine, f.string(), cfg); + if (!result) { std::cerr << "推理失败: " << f.filename().string() << ": " << result.error().message << '\n'; } } - return 0; + std::cout << "\n共推理 " << csv_files.size() << " 张图片" << std::endl; } - catch (const std::exception &e) + else if (fs::is_regular_file(input)) { - std::cerr << "错误: " << e.what() << std::endl; + // 单张推理 + auto result = infer_single(model, *engine, input.string(), cfg); + if (!result) { std::cerr << "推理失败: " << result.error().message << '\n'; return 1; } + } + else + { + std::cerr << "输入路径不存在: " << cfg.input_path << std::endl; return 1; } -} \ No newline at end of file + + return 0; +} diff --git a/src/mnist_train.cpp b/src/mnist_train.cpp index 779da604..a70989cd 100644 --- a/src/mnist_train.cpp +++ b/src/mnist_train.cpp @@ -1,73 +1,134 @@ +// ── MNIST 手写数字训练程序（引擎化架构，支持 MLP 与 Transformer） ──────── +// +// 数据流： +// CSV → Matrix(feat_dim, N) + Matrix(10, N) +// 每 batch：Matrix → engine.from_matrix → Tensor[device] +// → model.forward(Tensor) → Tensor +// → ce_loss.forward(engine, out, target) → Scalar +// → ce_loss.backward() → Tensor +// → model.backward(Tensor) → (丢弃) +// → optimizer.step() / model.zero_grad() +// evaluate：forward → engine.to_matrix → CPU argmax +// +// 引擎选择：--gpu 启用 GpuEngine（需要 Vulkan），否则 CpuEngine。 +// 架构选择：--arch mlp|transformer（默认 mlp，从已保存模型 resume 时自动识别） +// ───────────────────────────────────────────────────────────────────────── + #include -#include -#include -#include // for std::memcpy -#include // for std::unique_ptr +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include #include -#include +#include +#include #include #include +#include #include -#include -#include -#include -#include -#include -// ==================== 常量 ==================== -// 现在使用 nn::MNIST_INPUT_DIM, nn::MNIST_NUM_CLASSES, nn::MNIST_LAYER_DIMS +using nn::Scalar; + +enum class ArchType { MLP, Transformer }; // ==================== 帮助信息 ==================== void print_usage(const char *prog) { std::cout - << "MNIST 手写数字训练程序\n\n" + << "MNIST 手写数字训练程序 (引擎化架构，支持 MLP/Transformer)\n\n" << "用法: " << prog << " [选项]\n\n" << "选项:\n" - << " --model-type 模型类型: mlp/transformer (默认: mlp)\n" - << " --resume 从已有模型恢复训练 (自动读取模型规格)\n" + << " --arch 模型架构: mlp/transformer (默认: mlp)\n" + << " --resume 从已有模型恢复训练 (自动读取模型规格与架构)\n" << " --save 模型保存路径 (默认: mnist_model.bin)\n" << " --dataset 数据集目录 (默认: datasets/mnist_data)\n" << " --epochs 训练轮数 (默认: 10)\n" << " --lr 学习率 (默认: 0.001)\n" << " --batch-size 批大小 (默认: 64)\n" - << " --optimizer 优化器: sgd/momentum/adam (默认: adam)\n" - << "\n MLP 选项:\n" - << " --layer-dims MLP 各层维度，逗号分隔 (默认: 784,512,256,128,64,10)\n" - << "\n Transformer 选项:\n" + << " --optimizer 优化器: sgd/sgd_momentum/adam/adamw/muon (默认: adam)\n" + << " --weight-decay AdamW 权重衰减系数 (默认: 0.01)\n" + << " --gpu 启用 GPU 加速 (需要 Vulkan SDK)\n" + << " --cuda 启用 CUDA GPU 加速 (需要 CUDA Toolkit)\n" + << " --max-samples 限制训练样本数 (用于快速测试, 默认: 全部)\n" + << " --shuffle-steps 每 epoch 打乱 batch 顺序 (默认: true)\n" + << "\n" + << "MLP 专用:\n" + << " --layer-dims 各层维度，逗号分隔 (默认: 784,512,256,128,64,10)\n" + << "\n" + << "Transformer 专用:\n" << " --d-model 模型维度 (默认: 64)\n" << " --num-heads 注意力头数 (默认: 4)\n" - << " --d-ff FFN 中间维度 (默认: 128)\n" << " --num-layers Transformer 层数 (默认: 2)\n" - << " --patch-size Patch 大小 (默认: 7)\n" + << " --d-ff FFN 中间维度 (默认: 128)\n" + << " --patch-size patch 大小 (默认: 7, 28/7=4 → 16 patches)\n" + << " --eval-samples 评估样本数 (默认: 200，避免评估过慢)\n" + << " --osc-guard 振荡检测自动降 lr (默认: on)\n" + << " --osc-window 振荡检测窗口大小 (默认: 20)\n" + << " --osc-threshold 振荡反转率阈值 (默认: 0.55)\n" + << "\n" + << "学习率调度:\n" + << " --lr-schedule 学习率调度: fixed/cosine (默认: fixed)\n" + << " cosine: 余弦退火，lr 从初始值衰减到 min-lr\n" + << " --warmup-epochs 线性预热轮数 (默认: 0, 即不预热)\n" + << " --min-lr 余弦退火最低学习率 (默认: 1e-6)\n" + << " --lr-per-epoch 手动指定每轮学习率 (逗号分隔，优先级最高)\n" << " --help 显示此帮助信息\n"; } // ==================== 命令行参数 ==================== struct TrainConfig { + ArchType arch = ArchType::MLP; std::string save_path = "mnist_model.bin"; std::string dataset_path = "datasets/mnist_data"; std::string resume_path; std::string optimizer_name = "adam"; - std::string model_type = "mlp"; int epochs = 10; - double lr = 0.001; + Scalar lr = 0.001; + Scalar weight_decay = 0.01f; // AdamW 权重衰减系数 std::size_t batch_size = 64; bool load_existing = false; - // MLP 自定义层维度 + bool gpu_enabled = false; + bool cuda_enabled = false; + int max_train_samples = -1; // -1 表示使用全部 + bool shuffle_steps = true; // 每 epoch 打乱 batch 顺序 + + // MLP 参数 std::vector layer_dims; - // Transformer 自定义参数 - std::size_t d_model = nn::TRANSFORMER_D_MODEL; - std::size_t num_heads = nn::TRANSFORMER_NUM_HEADS; - std::size_t d_ff = nn::TRANSFORMER_D_FF; - std::size_t num_layers = nn::TRANSFORMER_NUM_LAYERS; - std::size_t patch_size = nn::TRANSFORMER_PATCH_SIZE; + + // Transformer 参数 + std::size_t d_model = nn::MNIST_TF_D_MODEL; + std::size_t num_heads = nn::MNIST_TF_NUM_HEADS; + std::size_t d_ff = nn::MNIST_TF_D_FF; + std::size_t num_layers = nn::MNIST_TF_NUM_LAYERS; + std::size_t patch_size = nn::MNIST_PATCH_SIZE; + std::size_t eval_samples = 200; // Transformer 评估样本上限 + + // 学习率调度 + std::string lr_schedule = "fixed"; // fixed / cosine + int warmup_epochs = 0; // 线性预热轮数 + Scalar min_lr = 1e-6f; // 余弦退火最低 lr + std::vector lr_per_epoch; // 手动指定每轮 lr（为空则自动计算） }; TrainConfig parse_args(int argc, char *argv[]) { TrainConfig cfg; + nn::cli::TrainCommonArgs common; + // 程序专属默认值（覆盖 TrainCommonArgs 的默认值） + common.batch_size = 64; + common.weight_decay = 0.01f; + common.min_lr = 1e-6f; + common.lr_schedule = "fixed"; + for (int i = 1; i < argc; ++i) { std::string arg = argv[i]; @@ -76,6 +137,24 @@ TrainConfig parse_args(int argc, char *argv[]) print_usage(argv[0]); std::exit(0); } + // 通用训练参数委托给 nn::cli::parse_train_common_args + else if (nn::cli::parse_train_common_args(argc, argv, i, common)) + { + continue; + } + else if (arg == "--arch" && i + 1 < argc) + { + std::string v = argv[++i]; + if (v == "mlp") + cfg.arch = ArchType::MLP; + else if (v == "transformer" || v == "tf") + cfg.arch = ArchType::Transformer; + else + { + std::cerr << "未知 --arch: " << v << "，可选: mlp, transformer\n"; + std::exit(1); + } + } else if (arg == "--resume" && i + 1 < argc) { cfg.resume_path = argv[++i]; @@ -89,54 +168,17 @@ TrainConfig parse_args(int argc, char *argv[]) { cfg.dataset_path = argv[++i]; } - else if (arg == "--epochs" && i + 1 < argc) - { - int val = std::stoi(argv[++i]); - if (val <= 0) { std::cerr << "--epochs 必须为正整数\n"; std::exit(1); } - cfg.epochs = val; - } - else if (arg == "--lr" && i + 1 < argc) - { - cfg.lr = std::stod(argv[++i]); - } - else if (arg == "--batch-size" && i + 1 < argc) - { - int val = std::stoi(argv[++i]); - if (val <= 0) { std::cerr << "--batch-size 必须为正整数\n"; std::exit(1); } - cfg.batch_size = static_cast(val); - } - else if (arg == "--optimizer" && i + 1 < argc) - { - cfg.optimizer_name = argv[++i]; - if (cfg.optimizer_name != "sgd" && cfg.optimizer_name != "sgd_w_momentum" && - cfg.optimizer_name != "adam") - { - std::cerr << "未知优化器: " << cfg.optimizer_name - << "，可选: sgd, sgd_w_momentum, adam\n"; - std::exit(1); - } - } - else if (arg == "--model-type" && i + 1 < argc) - { - cfg.model_type = argv[++i]; - if (cfg.model_type != "mlp" && cfg.model_type != "transformer") - { - std::cerr << "未知模型类型: " << cfg.model_type - << "，可选: mlp, transformer\n"; - std::exit(1); - } - } else if (arg == "--layer-dims" && i + 1 < argc) { - // 解析逗号分隔的层维度列表 std::string dims_str = argv[++i]; std::stringstream ss(dims_str); std::string token; while (std::getline(ss, token, ',')) { - std::size_t d = static_cast(std::stoi(token)); - if (d == 0) { std::cerr << "层维度不能为 0\n"; std::exit(1); } - cfg.layer_dims.push_back(d); + auto v = nn::parse_number(token); + if (!v) { std::cerr << "无效层维度: " << v.error().message << "\n"; std::exit(1); } + if (*v == 0) { std::cerr << "层维度不能为 0\n"; std::exit(1); } + cfg.layer_dims.push_back(*v); } if (cfg.layer_dims.size() < 2) { @@ -145,400 +187,416 @@ TrainConfig parse_args(int argc, char *argv[]) } } else if (arg == "--d-model" && i + 1 < argc) - cfg.d_model = static_cast(std::stoi(argv[++i])); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --d-model: " << v.error().message << "\n"; std::exit(1); } + cfg.d_model = *v; + } else if (arg == "--num-heads" && i + 1 < argc) - cfg.num_heads = static_cast(std::stoi(argv[++i])); - else if (arg == "--d-ff" && i + 1 < argc) - cfg.d_ff = static_cast(std::stoi(argv[++i])); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --num-heads: " << v.error().message << "\n"; std::exit(1); } + cfg.num_heads = *v; + } else if (arg == "--num-layers" && i + 1 < argc) - cfg.num_layers = static_cast(std::stoi(argv[++i])); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --num-layers: " << v.error().message << "\n"; std::exit(1); } + cfg.num_layers = *v; + } + else if (arg == "--d-ff" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --d-ff: " << v.error().message << "\n"; std::exit(1); } + cfg.d_ff = *v; + } else if (arg == "--patch-size" && i + 1 < argc) - cfg.patch_size = static_cast(std::stoi(argv[++i])); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --patch-size: " << v.error().message << "\n"; std::exit(1); } + cfg.patch_size = *v; + } + else if (arg == "--max-samples" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --max-samples: " << v.error().message << "\n"; std::exit(1); } + cfg.max_train_samples = *v; + } + else if (arg == "--eval-samples" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --eval-samples: " << v.error().message << "\n"; std::exit(1); } + cfg.eval_samples = *v; + } + else if (arg == "--shuffle-steps" && i + 1 < argc) + { + std::string v = argv[++i]; + if (v == "true" || v == "1") + cfg.shuffle_steps = true; + else if (v == "false" || v == "0") + cfg.shuffle_steps = false; + else + { + std::cerr << "无效 --shuffle-steps: " << v << "，可选: true, false\n"; + std::exit(1); + } + } else { std::cerr << "未知参数: " << arg << "\n使用 --help 查看用法\n"; std::exit(1); } } + + // 从通用参数回填到程序专属配置 + cfg.epochs = common.epochs; + cfg.lr = common.lr; + cfg.batch_size = common.batch_size; + cfg.optimizer_name = common.optimizer; + cfg.weight_decay = common.weight_decay; + cfg.gpu_enabled = common.use_gpu; + cfg.cuda_enabled = common.use_cuda; + cfg.lr_schedule = common.lr_schedule; + cfg.warmup_epochs = common.warmup_epochs; + cfg.min_lr = common.min_lr; + cfg.lr_per_epoch = std::move(common.lr_per_epoch); + return cfg; } -// ==================== 数据加载（优化版） ==================== -// 优化策略： -// 1. 整文件一次性读入内存，避免逐行 I/O 开销 -// 2. 使用 std::from_chars (C++17) 替代 std::stod，无临时 string 分配 -// 3. 向量预分配 reserve()，避免多次扩容拷贝 -// 4. 直接指针遍历 buffer，省去 std::stringstream 开销 -nn::Result> load_csv(const std::string &filename, int max_samples = -1) +// ==================== 构建模型规格 ==================== +nn::ModelSpec build_spec(const TrainConfig &cfg) { - // ── 1. 整文件读入 ────────────────────────────────────────────── - std::ifstream file(filename, std::ios::binary | std::ios::ate); - if (!file.is_open()) - return std::unexpected(nn::Error{"Cannot open file: " + filename}); - - const auto file_size = file.tellg(); - file.seekg(0); - - std::string buffer(static_cast(file_size), '\0'); - file.read(buffer.data(), file_size); - file.close(); - - // ── 2. 预扫描行数以预分配向量 ────────────────────────────────── - std::size_t row_count = 0; - for (char c : buffer) - if (c == '\n') ++row_count; - if (row_count == 0) - return std::unexpected(nn::Error{"CSV file is empty or malformed: " + filename}); - - if (max_samples > 0 && static_cast(max_samples) < row_count) - row_count = static_cast(max_samples); - - // ── 3. 解析：使用 from_chars 直接从 buffer 读取数值 ─────────── - const char *ptr = buffer.data(); - const char *end = buffer.data() + buffer.size(); - - // 先解析第一行确定 feat_dim - int first_label = 0; - std::size_t feat_dim = 0; + if (cfg.arch == ArchType::Transformer) { - // 跳过首行用于计数列数 - const char *p = ptr; - auto [p1, ec1] = std::from_chars(p, end, first_label); - p = p1; - std::size_t cnt = 0; - while (p < end && *p != '\n' && *p != '\r') - { - if (*p == ',') - { - ++cnt; - double tmp; - auto [p2, ec2] = std::from_chars(p + 1, end, tmp); - p = p2; - } - else - ++p; - } - feat_dim = cnt; // 标签后的逗号数 = 特征数 + return nn::make_mnist_transformer_spec( + cfg.patch_size, + cfg.d_model, cfg.num_heads, cfg.d_ff, cfg.num_layers); } + // MLP + nn::ModelSpec spec; + spec.type = nn::ModelType::MLP; + spec.layer_dims = cfg.layer_dims.empty() ? nn::MNIST_LAYER_DIMS : cfg.layer_dims; + return spec; +} - std::vector features(row_count * feat_dim); - std::vector labels(row_count); +// ==================== 主函数 ==================== +int main(int argc, char *argv[]) +{ + TrainConfig cfg = parse_args(argc, argv); - std::size_t row = 0; - ptr = buffer.data(); // 重置指针 + // ── 构建规格 ───────────────────────────────────────────── + nn::ModelSpec spec = build_spec(cfg); - while (ptr < end && row < row_count) + // ── 如果 --resume，从文件读取规格覆盖 CLI 参数 ────────── + if (cfg.load_existing) { - // 解析标签 - auto [p_label, ec_label] = std::from_chars(ptr, end, labels[row]); - if (ec_label != std::errc{}) - return std::unexpected(nn::Error{"Failed to parse label at row " + std::to_string(row)}); - ptr = p_label; - - // 解析特征 - for (std::size_t j = 0; j < feat_dim; ++j) + auto spec_result = nn::peek_model_spec(cfg.resume_path); + if (spec_result) { - // 跳过逗号 - if (ptr < end && *ptr == ',') ++ptr; - - double val; - auto [p_feat, ec_feat] = std::from_chars(ptr, end, val); - if (ec_feat != std::errc{}) - return std::unexpected(nn::Error{"Failed to parse feature at row " + std::to_string(row)}); - features[row * feat_dim + j] = val; - ptr = p_feat; + if (spec_result->type == nn::ModelType::MLP) + { + std::cout << "从模型文件读取 MLP 规格\n"; + spec = std::move(*spec_result); + cfg.arch = ArchType::MLP; + } + else if (spec_result->is_transformer()) + { + std::cout << "从模型文件读取 Transformer 规格\n"; + spec = std::move(*spec_result); + cfg.arch = ArchType::Transformer; + cfg.patch_size = spec.patch_size != 0 ? spec.patch_size : nn::MNIST_PATCH_SIZE; + cfg.d_model = spec.d_model; + cfg.num_heads = spec.num_heads; + cfg.d_ff = spec.d_ff; + cfg.num_layers = spec.num_layers; + } + else if (spec_result->type == nn::ModelType::Unknown) + { + std::cout << "旧格式模型文件 (V1)，使用命令行参数 (--arch)\n"; + } + else + { + std::cerr << "模型文件类型不支持 (type=" + << static_cast(spec_result->type) << ")\n"; + return 1; + } + } + else + { + std::cerr << "读取模型规格失败: " << spec_result.error().message + << "，使用命令行参数。\n"; } - - // 跳过行尾 (\r\n 或 \n) - while (ptr < end && (*ptr == '\r' || *ptr == '\n')) ++ptr; - ++row; } - // 实际读取的行数可能少于预扫描（例如文件末尾空行） - std::size_t N = row; + // ── 打印配置 ───────────────────────────────────────────── + std::cout << "========================================\n"; + std::cout << " MNIST 手写数字训练 (引擎化架构)\n"; + std::cout << "========================================\n"; + std::cout << " 架构: " << (cfg.arch == ArchType::Transformer ? "Transformer (ViT)" : "MLP") << "\n"; - // ── 4. 直接写入矩阵（行主序存储，逐列填充） ────────────────── - // 特征矩阵：形状 (feat_dim, N) - nn::Matrix feat_mat(feat_dim, N); - for (std::size_t i = 0; i < N; ++i) + if (cfg.arch == ArchType::Transformer) + { + const std::size_t grid = nn::MNIST_IMG_SIZE / cfg.patch_size; + const std::size_t num_patches = grid * grid; + std::cout << " Patch: " << nn::MNIST_IMG_SIZE << "/" << cfg.patch_size + << " → " << grid << "×" << grid << "=" << num_patches << " patches\n"; + std::cout << " d_model: " << cfg.d_model + << " heads: " << cfg.num_heads + << " layers: " << cfg.num_layers + << " d_ff: " << cfg.d_ff << "\n"; + } + else { - for (std::size_t j = 0; j < feat_dim; ++j) + const auto &dims = spec.is_mlp() ? spec.layer_dims : nn::MNIST_LAYER_DIMS; + std::cout << " 网络: "; + for (std::size_t i = 0; i < dims.size(); ++i) { - feat_mat.set_value_unchecked(j, i, features[i * feat_dim + j]); + std::cout << dims[i]; + if (i < dims.size() - 2) + std::cout << "(LayerNorm+GeLU)"; + if (i < dims.size() - 1) + std::cout << " -> "; } + std::cout << "\n"; } - // One‑hot 标签矩阵：形状 (10, N) - nn::Matrix label_mat(10, N); - for (std::size_t i = 0; i < N; ++i) + std::cout << " 优化器: " << cfg.optimizer_name << " 学习率: " << cfg.lr << "\n"; + std::cout << " 轮数: " << cfg.epochs << " 批大小: " << cfg.batch_size << "\n"; + std::cout << " GPU: " << (cfg.gpu_enabled ? "启用" : "禁用") + << " 打乱: " << (cfg.shuffle_steps ? "启用" : "禁用") << "\n"; + std::cout << " 模型: " << (cfg.load_existing ? cfg.resume_path : "(从头训练)") + << " -> " << cfg.save_path << "\n"; + std::cout << "========================================\n\n"; + + // ── 创建计算引擎 ───────────────────────────────────────── + // 引擎必须先于 model 构造并晚于 model 析构（model 持有 engine 的非拥有指针） + nn::cli::EngineConfig eng_cfg; + eng_cfg.use_gpu = cfg.gpu_enabled; + eng_cfg.use_cuda = cfg.cuda_enabled; + auto engine = nn::cli::create_engine(eng_cfg, std::cout); + + // ── 加载数据 ───────────────────────────────────────────── + std::cout << "加载数据: " << cfg.dataset_path << " ..." << std::endl; + const std::size_t train_max = + (cfg.max_train_samples > 0) ? static_cast(cfg.max_train_samples) : 0; + auto csv_train_result = nn::cli::load_mnist_csv(cfg.dataset_path + "/train.csv", train_max); + if (!csv_train_result) { std::cerr << "Error: " << csv_train_result.error().message << '\n'; return 1; } + auto [train_x, train_y] = std::move(*csv_train_result); + + auto csv_test_result = nn::cli::load_mnist_csv(cfg.dataset_path + "/test.csv"); + if (!csv_test_result) { std::cerr << "Error: " << csv_test_result.error().message << '\n'; return 1; } + auto [test_x, test_y] = std::move(*csv_test_result); + std::cout << "训练集: " << train_x.cols() << " 样本, 测试集: " << test_x.cols() << " 样本\n" << std::endl; + + // ── 构建模型（绑定引擎） ───────────────────────────────── + auto model_result = nn::build_mnist_model_from_spec(*engine, spec); + if (!model_result) { - int lbl = labels[i]; - if (lbl < 0 || lbl >= 10) - return std::unexpected(nn::Error{"Label out of range: " + std::to_string(lbl)}); - label_mat.set_value_unchecked(lbl, i, 1.0); + std::cerr << "构建模型失败: " << model_result.error().message << '\n'; + return 1; } + auto model = std::move(*model_result); - return std::pair{std::move(feat_mat), std::move(label_mat)}; -} - -// -------------------- 评估函数 -------------------- -double evaluate(nn::Model &model, const nn::Matrix &x, const nn::Matrix &y_onehot) -{ - std::size_t N = x.cols(); - auto out_result = model.forward(x); - if (!out_result) return 0.0; - auto out = std::move(*out_result); - int correct = 0; - for (std::size_t i = 0; i < N; ++i) + if (cfg.load_existing) { - double max_val = out.at_unchecked(0, i); - int pred = 0; - for (int j = 1; j < static_cast(nn::MNIST_NUM_CLASSES); ++j) + auto load_result = nn::load_model(cfg.resume_path, model); + if (load_result) { - double val = out.at_unchecked(j, i); - if (val > max_val) - { - max_val = val; - pred = j; - } + std::cout << "已加载模型: " << cfg.resume_path << "\n" << std::endl; } - int true_label = -1; - for (int j = 0; j < static_cast(nn::MNIST_NUM_CLASSES); ++j) + else { - if (y_onehot.at_unchecked(j, i) == 1.0) - { - true_label = j; - break; - } + std::cerr << "加载模型失败: " << load_result.error().message + << "，将从头训练。\n" << std::endl; } - if (pred == true_label) - ++correct; } - return static_cast(correct) / N; -} -// -------------------- 构建网络 -------------------- -// build_model 函数已移至 neuralnet.cpp/mnist_common.hpp 中的 nn::build_mnist_model() + // ── 训练 ───────────────────────────────────────────────── + auto optimizer = nn::create_optimizer( + cfg.optimizer_name, *engine, + model.parameters(), model.param_gradients(), cfg.lr, + cfg.weight_decay); + if (!optimizer) + { + std::cerr << "错误：未知优化器名称: " << cfg.optimizer_name << "\n"; + return 1; + } -// ==================== 主函数 ==================== -int main(int argc, char *argv[]) -{ - try + Scalar optimizer_current_lr = cfg.lr; + + // ── 学习率调度配置（委托给 nn::cli::compute_epoch_lr） ── + nn::cli::LrScheduleConfig lr_sched_cfg; + lr_sched_cfg.base_lr = cfg.lr; + lr_sched_cfg.min_lr = cfg.min_lr; + lr_sched_cfg.warmup_epochs = cfg.warmup_epochs; + lr_sched_cfg.total_epochs = cfg.epochs; + lr_sched_cfg.schedule = cfg.lr_schedule; + lr_sched_cfg.lr_per_epoch = cfg.lr_per_epoch; + + nn::CrossEntropyLoss ce_loss; + const std::size_t num_batches = train_x.cols() / cfg.batch_size; + if (num_batches == 0) { - TrainConfig cfg = parse_args(argc, argv); + std::cerr << "训练样本数 (" << train_x.cols() << ") 小于 batch_size (" + << cfg.batch_size << ")，请减小 --batch-size 或增大 --max-samples\n"; + return 1; + } - // ── 构建规格 ───────────────────────────────────────────── - nn::ModelSpec spec; - if (cfg.model_type == "mlp") - { - spec.type = nn::ModelType::MLP; - spec.layer_dims = cfg.layer_dims.empty() ? nn::MNIST_LAYER_DIMS : cfg.layer_dims; - } - else - { - spec.type = nn::ModelType::Transformer; - spec.d_model = cfg.d_model; - spec.num_heads = cfg.num_heads; - spec.d_ff = cfg.d_ff; - spec.num_layers = cfg.num_layers; - spec.patch_size = cfg.patch_size; - } + // ── 预分配 batch 缓冲区 ────────────────────────────────── + nn::Matrix x_batch(train_x.rows(), cfg.batch_size); + nn::Matrix y_batch(train_y.rows(), cfg.batch_size); + + std::vector sample_indices(train_x.cols()); + for (std::size_t i = 0; i < sample_indices.size(); ++i) + sample_indices[i] = i; + std::mt19937_64 shuffle_rng{std::random_device{}()}; + + auto t_start = std::chrono::steady_clock::now(); - // ── 如果 --resume，从文件读取规格覆盖 CLI 参数 ────────── - if (cfg.load_existing) + for (int epoch = 0; epoch < cfg.epochs; ++epoch) + { + // ── 学习率调度：每 epoch 开始时调整 ── + Scalar epoch_lr = nn::cli::compute_epoch_lr(lr_sched_cfg, epoch); + if (epoch_lr != optimizer_current_lr) { - auto spec_result = nn::peek_model_spec(cfg.resume_path); - if (spec_result) - { - if (spec_result->type != nn::ModelType::Unknown) - { - std::cout << "从模型文件读取规格\n"; - spec = std::move(*spec_result); - } - else - { - std::cout << "旧格式模型文件 (V1)，使用命令行参数\n"; - } - } - else - { - std::cerr << "读取模型规格失败: " << spec_result.error().message - << "，使用命令行参数。\n"; - } + optimizer->set_lr(epoch_lr); + optimizer_current_lr = epoch_lr; } - // ── 打印配置 ───────────────────────────────────────────── - std::cout << "========================================\n"; - std::cout << " MNIST 手写数字训练\n"; - std::cout << "========================================\n"; - std::cout << " 模型类型: " << cfg.model_type << "\n"; - if (spec.is_mlp()) + auto ep_start = std::chrono::steady_clock::now(); + Scalar total_loss = 0.0; + + if (cfg.shuffle_steps) + std::shuffle(sample_indices.begin(), sample_indices.end(), shuffle_rng); + + for (std::size_t batch = 0; batch < num_batches; ++batch) { - std::cout << " 网络: "; - for (std::size_t i = 0; i < spec.layer_dims.size(); ++i) + const std::size_t start = batch * cfg.batch_size; + + // ── 提取 batch（按 shuffle 顺序逐列拷贝） ── + const std::size_t R_x = train_x.rows(); + const std::size_t R_y = train_y.rows(); + const std::size_t C_train = train_x.cols(); + const auto x_src = train_x.span().data(); + const auto y_src = train_y.span().data(); + auto x_dst = x_batch.span().data(); + auto y_dst = y_batch.span().data(); + for (std::size_t b = 0; b < cfg.batch_size; ++b) { - std::cout << spec.layer_dims[i]; - if (i < spec.layer_dims.size() - 2) - std::cout << "(LayerNorm+GeLU)"; - if (i < spec.layer_dims.size() - 1) - std::cout << " -> "; + const std::size_t src_col = sample_indices[start + b]; + for (std::size_t r = 0; r < R_x; ++r) + x_dst[r * cfg.batch_size + b] = x_src[r * C_train + src_col]; + for (std::size_t r = 0; r < R_y; ++r) + y_dst[r * cfg.batch_size + b] = y_src[r * C_train + src_col]; } - std::cout << "\n"; - } - else - { - std::cout << " 网络: PatchEmbedding(" << spec.patch_size << "×" << spec.patch_size - << ") -> TransformerEncoder(" << spec.num_layers << "层, d=" - << spec.d_model << ", heads=" << spec.num_heads - << ", d_ff=" << spec.d_ff - << ") -> Linear(" << spec.d_model << "→" - << nn::MNIST_NUM_CLASSES << ")\n"; - } - std::cout << " 优化器: " << cfg.optimizer_name << " 学习率: " << cfg.lr << "\n"; - std::cout << " 轮数: " << cfg.epochs << " 批大小: " << cfg.batch_size << "\n"; - std::cout << " 模型: " << (cfg.load_existing ? cfg.resume_path : "(从头训练)") - << " -> " << cfg.save_path << "\n"; - std::cout << "========================================\n\n"; - - // ── 加载数据 ───────────────────────────────────────────── - std::cout << "加载数据: " << cfg.dataset_path << " ..." << std::endl; - auto csv_train_result = load_csv(cfg.dataset_path + "/train.csv"); - if (!csv_train_result) { std::cerr << "Error: " << csv_train_result.error().message << '\n'; return 1; } - auto [train_x, train_y] = std::move(*csv_train_result); - - auto csv_test_result = load_csv(cfg.dataset_path + "/test.csv"); - if (!csv_test_result) { std::cerr << "Error: " << csv_test_result.error().message << '\n'; return 1; } - auto [test_x, test_y] = std::move(*csv_test_result); - std::cout << "训练集: " << train_x.cols() << " 样本, 测试集: " << test_x.cols() << " 样本\n" << std::endl; - - // ── 构建模型 ───────────────────────────────────────────── - auto model_result = nn::build_mnist_model_from_spec(spec); - if (!model_result) - { - std::cerr << "构建模型失败: " << model_result.error().message << '\n'; - return 1; - } - auto model = std::move(*model_result); - if (cfg.load_existing) - { - auto load_result = nn::load_model(cfg.resume_path, model); - if (load_result) - { - std::cout << "已加载模型: " << cfg.resume_path << "\n" << std::endl; + // ── Matrix → Tensor（上传到引擎设备） ── + auto x_tensor_r = engine->from_matrix(x_batch); + if (!x_tensor_r) { + std::cerr << "\nfrom_matrix(x_batch) failed: " << x_tensor_r.error().message << '\n'; + return 1; } - else - { - std::cerr << "加载模型失败: " << load_result.error().message - << "，将从头训练。\n" << std::endl; + auto y_tensor_r = engine->from_matrix(y_batch); + if (!y_tensor_r) { + std::cerr << "\nfrom_matrix(y_batch) failed: " << y_tensor_r.error().message << '\n'; + return 1; } - } - // ── 训练 ───────────────────────────────────────────────── - std::unique_ptr optimizer; - if (cfg.optimizer_name == "sgd") - optimizer = std::make_unique(model.parameters(), model.param_gradients(), cfg.lr); - else if (cfg.optimizer_name == "sgd_w_momentum") - optimizer = std::make_unique(model.parameters(), model.param_gradients(), cfg.lr); - else - optimizer = std::make_unique(model.parameters(), model.param_gradients(), cfg.lr); + // ── 批量录制：forward + loss + backward 录制到单一 command buffer ── + auto bb = engine->begin_batch(); + if (!bb) { std::cerr << "\nbegin_batch failed: " << bb.error().message << '\n'; return 1; } - nn::CrossEntropyLoss ce_loss; - const std::size_t num_batches = train_x.cols() / cfg.batch_size; + // ── 前向 ── + auto out_r = model.forward(*x_tensor_r); + if (!out_r) { + std::cerr << "\nForward pass failed: " << out_r.error().message << '\n'; + return 1; + } - // ── 预分配 batch 缓冲区（避免每次迭代重复分配） ────────── - nn::Matrix x_batch(train_x.rows(), cfg.batch_size); - nn::Matrix y_batch(train_y.rows(), cfg.batch_size); + // ── 计算损失 ── + auto loss_result = ce_loss.forward(*engine, *out_r, *y_tensor_r); + if (!loss_result) { + std::cerr << "\nLoss computation failed: " << loss_result.error().message << '\n'; + return 1; + } + Scalar loss = *loss_result; + total_loss += loss; - auto t_start = std::chrono::steady_clock::now(); + // ── 反向 ── + auto grad_r = ce_loss.backward(); + if (!grad_r) { std::cerr << "\nLoss backward failed: " << grad_r.error().message << '\n'; return 1; } - for (int epoch = 0; epoch < cfg.epochs; ++epoch) - { - auto ep_start = std::chrono::steady_clock::now(); - double total_loss = 0.0; + auto bwd_r = model.backward(*grad_r); + if (!bwd_r) { std::cerr << "\nModel backward failed: " << bwd_r.error().message << '\n'; return 1; } - for (std::size_t batch = 0; batch < num_batches; ++batch) - { - const std::size_t start = batch * cfg.batch_size; - - // ── 行优先 memcpy 提取 batch（比逐列复制更缓存友好） ───── - for (std::size_t r = 0; r < train_x.rows(); ++r) - std::memcpy(x_batch.span().data() + r * cfg.batch_size, - train_x.span().data() + r * train_x.cols() + start, - cfg.batch_size * sizeof(double)); - for (std::size_t r = 0; r < train_y.rows(); ++r) - std::memcpy(y_batch.span().data() + r * cfg.batch_size, - train_y.span().data() + r * train_y.cols() + start, - cfg.batch_size * sizeof(double)); - - auto out_fwd = model.forward(x_batch); - if (!out_fwd) { - std::cerr << "\nForward pass failed: " << out_fwd.error().message << '\n'; - return 1; - } - auto out = std::move(*out_fwd); - auto loss_result = ce_loss.forward(out, y_batch); - if (!loss_result) { - std::cerr << "\nLoss computation failed: " << loss_result.error().message << '\n'; - return 1; - } - double loss = *loss_result; - total_loss += loss; - - auto grad = ce_loss.backward(); - auto bwd_result = model.backward(grad); - if (!bwd_result) { std::cerr << "Error: " << bwd_result.error().message << '\n'; return 1; } - - auto step_result = optimizer->step(); - if (!step_result) - { - std::cerr << "\n优化器 step 失败: " << step_result.error().message << '\n'; - return 1; - } - optimizer->zero_grad(); - - // 进度显示 - if ((batch + 1) % 100 == 0 || batch + 1 == num_batches) - { - std::cout << "\r Epoch " << epoch + 1 << "/" << cfg.epochs - << " batch " << batch + 1 << "/" << num_batches - << " loss: " << std::fixed << std::setprecision(4) << loss - << " " << std::flush; - } - } + // ── 提交并等待所有录制的命令 ── + auto eb = engine->end_batch(); + if (!eb) { std::cerr << "\nend_batch failed: " << eb.error().message << '\n'; return 1; } - auto ep_end = std::chrono::steady_clock::now(); - double ep_sec = std::chrono::duration(ep_end - ep_start).count(); + // ── 优化器 step + 梯度清零 ── + auto step_result = optimizer->step(); + if (!step_result) { + std::cerr << "\n优化器 step 失败: " << step_result.error().message << '\n'; + return 1; + } - double avg_loss = total_loss / num_batches; - double train_acc = evaluate(model, train_x, train_y); - double test_acc = evaluate(model, test_x, test_y); + auto zero_result = optimizer->zero_grad(); + if (!zero_result) { + std::cerr << "\n优化器 zero_grad 失败: " << zero_result.error().message << '\n'; + return 1; + } - std::cout << "\r Epoch " << epoch + 1 << "/" << cfg.epochs - << " loss=" << std::fixed << std::setprecision(4) << avg_loss - << " train_acc=" << std::setprecision(2) << train_acc * 100.0 << "%" - << " test_acc=" << test_acc * 100.0 << "%" - << " time=" << std::setprecision(1) << ep_sec << "s" - << std::endl; + // ── 进度显示 ── + if ((batch + 1) % 10 == 0 || batch + 1 == num_batches) + { + auto batch_now = std::chrono::steady_clock::now(); + Scalar batch_ms = std::chrono::duration(batch_now - ep_start).count(); + std::cout << "\r Epoch " << epoch + 1 << "/" << cfg.epochs + << " batch " << batch + 1 << "/" << num_batches + << " loss: " << std::fixed << std::setprecision(4) << loss + << " time: " << std::setprecision(0) << batch_ms << "ms" + << " " << std::flush; + } } - auto t_end = std::chrono::steady_clock::now(); - double total_sec = std::chrono::duration(t_end - t_start).count(); + auto ep_end = std::chrono::steady_clock::now(); + Scalar ep_sec = std::chrono::duration(ep_end - ep_start).count(); - // ── 保存模型（含规格） ───────────────────────────────────── - auto save_result = nn::save_model(cfg.save_path, model, spec); - if (!save_result) + Scalar avg_loss = total_loss / num_batches; + // MLP 全量评估，Transformer 截取前 eval_samples 个样本评估 + const std::size_t eval_n = (cfg.arch == ArchType::Transformer) ? cfg.eval_samples : 0; + auto train_acc_r = nn::cli::evaluate_mnist(model, *engine, train_x, train_y, eval_n); + auto test_acc_r = nn::cli::evaluate_mnist(model, *engine, test_x, test_y, eval_n); + if (!train_acc_r || !test_acc_r) { - std::cerr << "保存模型失败: " << save_result.error().message << '\n'; + const auto &err = !train_acc_r ? train_acc_r.error() : test_acc_r.error(); + std::cerr << "\n 评估失败: " << err.message << std::endl; return 1; } - std::cout << "\n训练完成! 总耗时: " << std::fixed << std::setprecision(1) << total_sec << "s" << std::endl; - - return 0; + Scalar train_acc = *train_acc_r; + Scalar test_acc = *test_acc_r; + + std::cout << "\r Epoch " << epoch + 1 << "/" << cfg.epochs + << " lr=" << std::scientific << std::setprecision(4) << optimizer_current_lr + << " loss=" << std::fixed << std::setprecision(4) << avg_loss + << " train_acc=" << std::setprecision(2) << train_acc * 100.0 << "%" + << " test_acc=" << test_acc * 100.0 << "%" + << " time=" << std::setprecision(1) << ep_sec << "s" + << std::endl; } - catch (const std::exception &e) + + auto t_end = std::chrono::steady_clock::now(); + Scalar total_sec = std::chrono::duration(t_end - t_start).count(); + + // ── 保存模型 ───────────────────────────────────────────── + auto save_result = nn::save_model(cfg.save_path, model, spec); + if (!save_result) { - std::cerr << "\n错误: " << e.what() << std::endl; + std::cerr << "保存模型失败: " << save_result.error().message << '\n'; return 1; } + std::cout << "\n训练完成! 总耗时: " << std::fixed << std::setprecision(1) << total_sec << "s" << std::endl; + + return 0; } diff --git a/src/rmsnorm_gradcheck.cpp b/src/rmsnorm_gradcheck.cpp new file mode 100644 index 00000000..c225783e --- /dev/null +++ b/src/rmsnorm_gradcheck.cpp @@ -0,0 +1,214 @@ +// ── RMSNorm 数值梯度检查（gradcheck） ──────────────────────────────────── +// +// 目的：用中心差分有限差分验证 RMSNorm 的 backward 参数梯度（gamma）与 +// 输入梯度是否正确。 +// +// 原理： +// L(θ) = Σ_ij y_ij * go_ij （go 为固定的"输出权重"） +// 则 dL/dθ = backward 得到的梯度 +// 对每个参数/输入元素做 ±eps 扰动，重算 L，中心差分近似解析梯度并比对。 +// +// 用法：rmsnorm_gradcheck [--cuda|--gpu] [--tol ] +// ───────────────────────────────────────────────────────────────────────── + +#include +#include + +#include +#include +#include +#include +#include + +using nn::Scalar; +using nn::Matrix; +using nn::Tensor; +using nn::TensorRef; +using nn::ComputeEngine; +using nn::RMSNorm; +using nn::Layer; + +namespace { + +// L = Σ a*b（逐元素乘后求和） +Scalar dot(const Matrix &a, const Matrix &b) +{ + Scalar s{0}; + const auto sa = a.span(); + const auto sb = b.span(); + for (std::size_t i = 0; i < a.size(); ++i) + s += sa[i] * sb[i]; + return s; +} + +// 用当前参数前向，返回 L = Σ y*go +Scalar eval_loss(ComputeEngine &engine, Layer &norm, + const Tensor &x, const Tensor &go) +{ + auto y = norm.forward(engine, x); + auto y_m = engine.to_matrix(*y); + auto go_m = engine.to_matrix(go); + return dot(*y_m, *go_m); +} + +bool approx(Scalar num, Scalar ana, Scalar tol) +{ + return std::fabs(num - ana) <= + tol * (Scalar{1} + std::fabs(num) + std::fabs(ana)); +} + +// 验证单个张量参数/输入的梯度（逐元素中心差分） +// fwd_input: 前向传播使用的输入。参数验证时传固定 x；输入梯度验证时 +// 传被扰动的输入副本（扰动 fwd_input 本身并重算 loss）。 +bool check_grad_tensor( + ComputeEngine &engine, Layer &norm, + const Tensor &fwd_input, + const Tensor &go, + Tensor ¶m, // 被扰动的张量（参数或输入） + const Matrix &base, // 扰动前的值 + const Matrix &grad_analytical, // backward 得到的梯度 + const std::string &name, + Scalar eps, Scalar tol) +{ + bool ok = true; + std::size_t bad = 0; + Scalar max_err{0}; + for (std::size_t r = 0; r < base.rows(); ++r) + { + for (std::size_t c = 0; c < base.cols(); ++c) + { + const Scalar orig = base.at_unchecked(r, c); + + Matrix pp = base; + pp.set_value_unchecked(r, c, orig + eps); + auto r_p = engine.copy_from(param, pp); + if (!r_p) { std::cerr << "copy_from(+) failed\n"; return false; } + const Scalar lp = eval_loss(engine, norm, fwd_input, go); + + Matrix pm = base; + pm.set_value_unchecked(r, c, orig - eps); + auto r_m = engine.copy_from(param, pm); + if (!r_m) { std::cerr << "copy_from(-) failed\n"; return false; } + const Scalar lm = eval_loss(engine, norm, fwd_input, go); + + // 恢复原值 + auto r_r = engine.copy_from(param, base); + if (!r_r) { std::cerr << "copy_from(restore) failed\n"; return false; } + + const Scalar num = (lp - lm) / (Scalar{2} * eps); + const Scalar ana = grad_analytical.at_unchecked(r, c); + const Scalar err = std::fabs(num - ana); + if (err > max_err) max_err = err; + if (!approx(num, ana, tol)) + { + ok = false; + if (bad < 10) + std::cout << " [FAIL] " << name << "(" << r << "," << c + << ") num=" << num << " ana=" << ana << "\n"; + ++bad; + } + } + } + std::cout << " " << name << ": " + << (ok ? "OK" : "FAIL") + << " max_err=" << max_err << " bad=" << bad << "\n"; + return ok; +} + +} // namespace + +int main(int argc, char *argv[]) +{ + Scalar tol = 2e-2f; + bool use_cuda = false; + bool use_gpu = false; + for (int i = 1; i < argc; ++i) + { + std::string a = argv[i]; + if (a == "--tol" && i + 1 < argc) + tol = static_cast(std::atof(argv[++i])); + else if (a == "--cuda") + use_cuda = true; + else if (a == "--gpu") + use_gpu = true; + else if (a == "--help") + { + std::cout << "用法: rmsnorm_gradcheck [--cuda|--gpu] [--tol ]\n"; + return 0; + } + } + + nn::cli::EngineConfig ecfg; + ecfg.use_cuda = use_cuda; + ecfg.use_gpu = use_gpu; + auto engine = nn::cli::create_engine(ecfg, std::cout); + if (!engine) { std::cerr << "引擎创建失败\n"; return 1; } + ComputeEngine &eng = *engine; + + const std::size_t features = 16; + const std::size_t batch = 4; + + std::cout << "========================================\n"; + std::cout << " RMSNorm 数值梯度检查 (gradcheck)\n"; + std::cout << "========================================\n"; + std::cout << " features=" << features << " batch=" << batch + << " tol=" << tol << "\n"; + + RMSNorm norm(eng, features); + + std::mt19937_64 rng(123); + std::uniform_real_distribution dist(-1, 1); + + Matrix x_m(features, batch); + Matrix go_m(features, batch); + for (std::size_t i = 0; i < x_m.size(); ++i) x_m.span()[i] = dist(rng); + for (std::size_t i = 0; i < go_m.size(); ++i) go_m.span()[i] = dist(rng); + + auto x = eng.from_matrix(x_m); + if (!x) { std::cerr << "from_matrix(x) failed\n"; return 1; } + auto go = eng.from_matrix(go_m); + if (!go) { std::cerr << "from_matrix(go) failed\n"; return 1; } + + // ── 前向一次：填充缓存（backward 依赖 forward 缓存） ── + auto y_fwd = norm.forward(eng, *x); + if (!y_fwd) { std::cerr << "forward failed: " << y_fwd.error().message << "\n"; return 1; } + + // ── 清零梯度并 backward ── + auto params = norm.parameters(); // [gamma] + auto grads = norm.param_gradients(); // [grad_gamma] + for (auto &g : grads) + { + auto rz = eng.zero(g.get()); + if (!rz) { std::cerr << "zero failed\n"; return 1; } + } + auto gx = norm.backward(eng, *go); + if (!gx) { std::cerr << "backward failed: " << gx.error().message << "\n"; return 1; } + + // ── 验证参数梯度（gamma） ── + const Scalar eps = 1e-3f; + bool all_pass = true; + for (std::size_t pi = 0; pi < params.size(); ++pi) + { + auto p = eng.to_matrix(params[pi].get()); + if (!p) { std::cerr << "to_matrix(param) failed\n"; return 1; } + auto g = eng.to_matrix(grads[pi].get()); + if (!g) { std::cerr << "to_matrix(grad) failed\n"; return 1; } + std::string name = (pi == 0) ? "gamma" : ("param[" + std::to_string(pi) + "]"); + all_pass &= check_grad_tensor( + eng, norm, *x, *go, params[pi].get(), *p, *g, name, eps, tol); + } + + // ── 验证输入梯度（扰动输入副本本身，前向用该副本） ── + { + auto gxm = eng.to_matrix(*gx); + if (!gxm) { std::cerr << "to_matrix(grad_x) failed\n"; return 1; } + auto xt = eng.clone(*x); // 可写副本，避免扰动原始输入 Tensor + if (!xt) { std::cerr << "clone(x) failed\n"; return 1; } + all_pass &= check_grad_tensor( + eng, norm, *xt, *go, *xt, x_m, *gxm, "grad_x", eps, tol); + } + + std::cout << "----------------------------------------\n"; + std::cout << " 结果: " << (all_pass ? "✅ 全部通过" : "❌ 存在失败") << "\n"; + return all_pass ? 0 : 1; +} diff --git a/src/swiglu_gradcheck.cpp b/src/swiglu_gradcheck.cpp new file mode 100644 index 00000000..2b8803b0 --- /dev/null +++ b/src/swiglu_gradcheck.cpp @@ -0,0 +1,221 @@ +// ── SwiGLU 数值梯度检查（gradcheck） ───────────────────────────────────── +// +// 目的：用有限差分（中心差分）验证 SwiGLU FFN 的 backward 参数梯度与 +// 输入梯度是否正确。split（slice_rows）与 merge（insert_rows）是 +// SwiGLU 独有的操作，最容易引入反向传播错误，故单独验证。 +// +// 原理： +// L(θ) = Σ_ij y_ij * go_ij （go 为固定的"输出权重"） +// 则 dL/dθ = backward 得到的梯度（因为 dL/dy = go） +// 对每个参数元素做 ±eps 扰动，重算 L，中心差分近似解析梯度并比对。 +// +// 用法：swiglu_gradcheck [--tol ] +// ───────────────────────────────────────────────────────────────────────── + +#include +#include + +#include +#include +#include +#include +#include +#include + +using nn::Scalar; +using nn::Matrix; +using nn::Tensor; +using nn::TensorRef; +using nn::ComputeEngine; +using nn::CpuEngine; +using nn::ActivationType; +using nn::FeedForward; + +namespace { + +// L = Σ a*b（逐元素乘后求和） +Scalar dot(const Matrix &a, const Matrix &b) +{ + Scalar s{0}; + const auto sa = a.span(); + const auto sb = b.span(); + for (std::size_t i = 0; i < a.size(); ++i) + s += sa[i] * sb[i]; + return s; +} + +// 用当前参数前向，返回 L = Σ y*go +Scalar eval_loss(ComputeEngine &engine, FeedForward &ff, + const Tensor &x, const Tensor &go) +{ + auto y = ff.forward(engine, x); + auto y_m = engine.to_matrix(*y); + auto go_m = engine.to_matrix(go); + return dot(*y_m, *go_m); +} + +bool approx(Scalar num, Scalar ana, Scalar tol) +{ + return std::fabs(num - ana) <= + tol * (Scalar{1} + std::fabs(num) + std::fabs(ana)); +} + +// 验证单个张量参数/输入的梯度（逐元素中心差分） +// fwd_input: 前向传播使用的输入。参数验证时传固定 x；输入梯度验证时 +// 传被扰动的输入副本（扰动 fwd_input 本身并重算 loss）。 +bool check_grad_tensor( + ComputeEngine &engine, FeedForward &ff, + const Tensor &fwd_input, + const Tensor &go, + Tensor ¶m, // 被扰动的张量（参数或输入） + const Matrix &base, // 扰动前的值 + const Matrix &grad_analytical, // backward 得到的梯度 + const std::string &name, + Scalar eps, Scalar tol) +{ + bool ok = true; + std::size_t bad = 0; + Scalar max_err{0}; + for (std::size_t r = 0; r < base.rows(); ++r) + { + for (std::size_t c = 0; c < base.cols(); ++c) + { + const Scalar orig = base.at_unchecked(r, c); + + Matrix pp = base; + pp.set_value_unchecked(r, c, orig + eps); + auto r_p = engine.copy_from(param, pp); + if (!r_p) { std::cerr << "copy_from(+) failed\n"; return false; } + const Scalar lp = eval_loss(engine, ff, fwd_input, go); + + Matrix pm = base; + pm.set_value_unchecked(r, c, orig - eps); + auto r_m = engine.copy_from(param, pm); + if (!r_m) { std::cerr << "copy_from(-) failed\n"; return false; } + const Scalar lm = eval_loss(engine, ff, fwd_input, go); + + // 恢复原值 + auto r_r = engine.copy_from(param, base); + if (!r_r) { std::cerr << "copy_from(restore) failed\n"; return false; } + + const Scalar num = (lp - lm) / (Scalar{2} * eps); + const Scalar ana = grad_analytical.at_unchecked(r, c); + const Scalar err = std::fabs(num - ana); + if (err > max_err) max_err = err; + if (!approx(num, ana, tol)) + { + ok = false; + if (bad < 10) + std::cout << " [FAIL] " << name << "(" << r << "," << c + << ") num=" << num << " ana=" << ana << "\n"; + ++bad; + } + } + } + std::cout << " " << name << ": " + << (ok ? "OK" : "FAIL") + << " max_err=" << max_err << " bad=" << bad << "\n"; + return ok; +} + +} // namespace + +int main(int argc, char *argv[]) +{ + Scalar tol = 2e-2f; + bool use_cuda = false; + bool use_gpu = false; + for (int i = 1; i < argc; ++i) + { + std::string a = argv[i]; + if (a == "--tol" && i + 1 < argc) + tol = static_cast(std::atof(argv[++i])); + else if (a == "--cuda") + use_cuda = true; + else if (a == "--gpu") + use_gpu = true; + else if (a == "--help") + { + std::cout << "用法: swiglu_gradcheck [--cuda|--gpu] [--tol ]\n"; + return 0; + } + } + + nn::cli::EngineConfig ecfg; + ecfg.use_cuda = use_cuda; + ecfg.use_gpu = use_gpu; + auto engine = nn::cli::create_engine(ecfg, std::cout); + if (!engine) { std::cerr << "引擎创建失败\n"; return 1; } + ComputeEngine &eng = *engine; + + // 小规模，加速测试 + const std::size_t d_model = 8; + const std::size_t d_ff = 16; // SwiGLU 时 fc1 → 2*d_ff=32 + const std::size_t batch = 4; + + std::cout << "========================================\n"; + std::cout << " SwiGLU 数值梯度检查 (gradcheck)\n"; + std::cout << "========================================\n"; + std::cout << " d_model=" << d_model << " d_ff=" << d_ff + << " batch=" << batch << " tol=" << tol << "\n"; + + FeedForward ff(eng, d_model, d_ff, ActivationType::SwiGLU); + + std::mt19937_64 rng(123); + std::uniform_real_distribution dist(-1, 1); + + Matrix x_m(d_model, batch); + Matrix go_m(d_model, batch); + for (std::size_t i = 0; i < x_m.size(); ++i) x_m.span()[i] = dist(rng); + for (std::size_t i = 0; i < go_m.size(); ++i) go_m.span()[i] = dist(rng); + + auto x = eng.from_matrix(x_m); + if (!x) { std::cerr << "from_matrix(x) failed\n"; return 1; } + auto go = eng.from_matrix(go_m); + if (!go) { std::cerr << "from_matrix(go) failed\n"; return 1; } + + // ── 前向一次：填充各层 input_cache_（backward 依赖 forward 缓存） ── + auto y_fwd = ff.forward(eng, *x); + if (!y_fwd) { std::cerr << "forward failed: " << y_fwd.error().message << "\n"; return 1; } + + // ── 清零梯度并 backward ── + auto params = ff.parameters(); + auto grads = ff.param_gradients(); + for (auto &g : grads) + { + auto rz = eng.zero(g.get()); + if (!rz) { std::cerr << "zero failed\n"; return 1; } + } + auto gx = ff.backward(eng, *go); + if (!gx) { std::cerr << "backward failed: " << gx.error().message << "\n"; return 1; } + + // ── 验证参数梯度 ── + const Scalar eps = 1e-3f; + bool all_pass = true; + const char *pnames[] = {"fc1.w", "fc1.b", "fc2.w", "fc2.b"}; + + for (std::size_t pi = 0; pi < params.size(); ++pi) + { + auto p = eng.to_matrix(params[pi].get()); + if (!p) { std::cerr << "to_matrix(param) failed\n"; return 1; } + auto g = eng.to_matrix(grads[pi].get()); + if (!g) { std::cerr << "to_matrix(grad) failed\n"; return 1; } + std::string name = pi < 4 ? pnames[pi] : ("param[" + std::to_string(pi) + "]"); + all_pass &= check_grad_tensor( + eng, ff, *x, *go, params[pi].get(), *p, *g, name, eps, tol); + } + + // ── 验证输入梯度（扰动输入副本本身，前向用该副本） ── + { + auto gxm = eng.to_matrix(*gx); + if (!gxm) { std::cerr << "to_matrix(grad_x) failed\n"; return 1; } + auto xt = eng.clone(*x); // 可写副本，避免扰动原始输入 Tensor + if (!xt) { std::cerr << "clone(x) failed\n"; return 1; } + all_pass &= check_grad_tensor( + eng, ff, *xt, *go, *xt, x_m, *gxm, "grad_x", eps, tol); + } + + std::cout << "----------------------------------------\n"; + std::cout << " 结果: " << (all_pass ? "✅ 全部通过" : "❌ 存在失败") << "\n"; + return all_pass ? 0 : 1; +} diff --git a/src/text_infer.cpp b/src/text_infer.cpp index ed199a6d..9e5f17b6 100644 --- a/src/text_infer.cpp +++ b/src/text_infer.cpp @@ -1,50 +1,62 @@ +// ── GPT 文本生成推理程序（引擎化架构） ─────────────────────────────────── +// +// 数据流： +// 加载模型 (含嵌入 tokenizer) → tokenizer.encode(prompt) → GPTModel.generate +// → tokenizer.decode(generated) → 输出文本 +// +// 引擎选择：--gpu 启用 GpuEngine（需要 Vulkan），否则 CpuEngine。 +// ───────────────────────────────────────────────────────────────────────── + #include -#include -#include +#include +#include +#include + +#include +#include #include +#include #include #include -#include -#include + +using nn::Scalar; // ==================== 帮助信息 ==================== void print_usage(const char *prog) { std::cout - << "GPT 文本生成推理程序\n\n" + << "GPT 文本生成推理程序 (引擎化架构)\n\n" << "用法:\n" - << " " << prog << " --prompt [选项]\n" + << " " << prog << " --prompt \"text\" [选项]\n" << " " << prog << " --interactive 交互模式\n\n" << "选项:\n" - << " --model 模型文件路径 (默认: gpt_model.bin)\n" - << " V2 格式模型自动读取规格，无需指定架构参数\n" - << " --prompt 输入提示文本\n" - << " --interactive 交互式生成模式\n" - << " --max-tokens 最大生成 token 数 (默认: 200)\n" - << " --temperature 温度参数 (默认: 1.0)\n" - << " --d-model 模型维度 (默认: 128)\n" - << " --num-heads 注意力头数 (默认: 4)\n" - << " --num-layers Transformer 层数 (默认: 4)\n" - << " --d-ff FFN 维度 (默认: 512)\n" - << " --seq-len 序列长度 (默认: 256)\n" - << " --show-tokens 显示 token ID (调试用)\n" - << " --help 显示此帮助信息\n"; + << " --model 模型文件路径 (默认: gpt_model.bin)\n" + << " V3 格式模型自动读取规格和嵌入 tokenizer\n" + << " --vocab 词表 JSON 路径 (默认: bpe_vocab.json)\n" + << " 仅当模型未嵌入 tokenizer 时使用\n" + << " 自动识别分词器类型（bpe / charbpe / wordzip / space）\n" + << " --prompt 输入提示文本\n" + << " --interactive 交互式生成模式\n" + << " --max-tokens 最大生成 token 数 (默认: 200)\n" + << " --temperature 温度参数 (默认: 1.0, 0=贪心)\n" + << " --gpu 启用 GPU 加速 (需要 Vulkan SDK)\n" + << " --cuda 启用 CUDA GPU 加速 (需要 CUDA Toolkit)\n" + << " --show-tokens 显示 token ID (调试用)\n" + << " --help 显示此帮助信息\n"; } // ==================== 命令行参数 ==================== struct InferConfig { std::string model_path = "gpt_model.bin"; + std::string vocab_path = "bpe_vocab.json"; std::string prompt = "Hello"; int max_tokens = 200; double temperature = 1.0; bool interactive = false; bool show_tokens = false; - std::size_t d_model = nn::GPT_D_MODEL; - std::size_t num_heads = nn::GPT_NUM_HEADS; - std::size_t num_layers = nn::GPT_NUM_LAYERS; - std::size_t d_ff = nn::GPT_D_FF; - std::size_t seq_len = nn::GPT_SEQ_LEN; + bool gpu_enabled = false; + bool cuda_enabled = false; }; InferConfig parse_args(int argc, char *argv[]) @@ -60,26 +72,30 @@ InferConfig parse_args(int argc, char *argv[]) } else if (arg == "--model" && i + 1 < argc) cfg.model_path = argv[++i]; + else if (arg == "--vocab" && i + 1 < argc) + cfg.vocab_path = argv[++i]; else if (arg == "--prompt" && i + 1 < argc) cfg.prompt = argv[++i]; else if (arg == "--interactive") cfg.interactive = true; else if (arg == "--max-tokens" && i + 1 < argc) - cfg.max_tokens = std::stoi(argv[++i]); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --max-tokens\n"; std::exit(1); } + cfg.max_tokens = *v; + } else if (arg == "--temperature" && i + 1 < argc) - cfg.temperature = std::stod(argv[++i]); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --temperature\n"; std::exit(1); } + cfg.temperature = *v; + } + else if (arg == "--gpu") + cfg.gpu_enabled = true; + else if (arg == "--cuda") + cfg.cuda_enabled = true; else if (arg == "--show-tokens") cfg.show_tokens = true; - else if (arg == "--d-model" && i + 1 < argc) - cfg.d_model = static_cast(std::stoi(argv[++i])); - else if (arg == "--num-heads" && i + 1 < argc) - cfg.num_heads = static_cast(std::stoi(argv[++i])); - else if (arg == "--num-layers" && i + 1 < argc) - cfg.num_layers = static_cast(std::stoi(argv[++i])); - else if (arg == "--d-ff" && i + 1 < argc) - cfg.d_ff = static_cast(std::stoi(argv[++i])); - else if (arg == "--seq-len" && i + 1 < argc) - cfg.seq_len = static_cast(std::stoi(argv[++i])); else if (!arg.starts_with("--")) cfg.prompt = arg; else @@ -91,26 +107,38 @@ InferConfig parse_args(int argc, char *argv[]) return cfg; } -// ==================== 从 GPTModel 提取采样结果 ==================== -// 使用 GPTModel 内置的 generate 方法（支持温度采样 + 贪心） +// ==================== 从 Model 提取 GPTModel 并生成 ==================== nn::Result> generate_text( - nn::Model &model, const std::vector &prompt_tokens, + nn::Model &model, nn::ComputeEngine &engine, + const std::vector &prompt_tokens, std::size_t max_new_tokens, double temperature, - std::size_t /*seq_len*/) + std::size_t eos_token_id) { - // Model 容器唯一层即为 GPTModel，安全提取 auto &layer_ref = model.layer_at(0); auto *gpt_ptr = dynamic_cast(&layer_ref); if (!gpt_ptr) return std::unexpected(nn::Error{"Model does not contain a GPTModel layer"}); - return gpt_ptr->generate(prompt_tokens, max_new_tokens, temperature); + + // 传入 EOS_ID，生成遇到 EOS 自动停止 + // min_new_tokens = max_new_tokens/2，至少生成一半 token 才允许 EOS 停止， + // 避免模型因训练偏置一上来就输出 EOS 导致无输出。 + const std::size_t min_new = max_new_tokens / 2; + return gpt_ptr->generate(engine, prompt_tokens, max_new_tokens, temperature, + eos_token_id, min_new); } // ==================== 交互模式 ==================== -void interactive_mode(nn::Model &model, const nn::BPETokenizer &tokenizer, - const InferConfig &cfg) +void interactive_mode(nn::Model &model, nn::ComputeEngine &engine, + const nn::Tokenizer &tokenizer, const InferConfig &cfg) { - std::cout << "GPT 交互式生成 (输入 'quit' 退出)\n\n"; + const std::size_t bos_id = tokenizer.bos_id(); + const std::size_t eos_id = tokenizer.eos_id(); + const bool has_dialogue = tokenizer.has_dialogue_markers(); + + if (has_dialogue) + std::cout << "GPT 交互式对话 (输入 'quit' 退出)\n\n"; + else + std::cout << "GPT 交互式生成 (输入 'quit' 退出)\n\n"; while (true) { @@ -118,33 +146,72 @@ void interactive_mode(nn::Model &model, const nn::BPETokenizer &tokenizer, std::string prompt; if (!std::getline(std::cin, prompt)) break; - if (prompt == "quit" || prompt == "exit") break; - if (prompt.empty()) prompt = " "; - auto prompt_tokens = tokenizer.encode(prompt); + // 构建 prompt tokens + std::vector prompt_tokens; + if (bos_id != nn::Tokenizer::npos) + prompt_tokens.push_back(bos_id); + + if (has_dialogue) + { + // 对话模式：包裹 // 标记，与训练时格式一致 + // 训练格式: [BOS]......... + const std::string system_tag = ""; + const std::string end_sys_tag = ""; + const std::string user_tag = ""; + const std::string end_user_tag = ""; + const std::string asst_tag = ""; + + const std::string system_prompt = "你是一个有用的AI助手。"; + auto sys_tokens = tokenizer.encode(system_tag + system_prompt + end_sys_tag); + auto user_tokens = tokenizer.encode(user_tag + prompt + end_user_tag); + auto asst_tokens = tokenizer.encode(asst_tag); + + prompt_tokens.insert(prompt_tokens.end(), sys_tokens.begin(), sys_tokens.end()); + prompt_tokens.insert(prompt_tokens.end(), user_tokens.begin(), user_tokens.end()); + prompt_tokens.insert(prompt_tokens.end(), asst_tokens.begin(), asst_tokens.end()); + } + else + { + // 普通模式：仅 BOS + 编码文本（与训练时每行格式一致） + auto text_tokens = tokenizer.encode(prompt); + prompt_tokens.insert(prompt_tokens.end(), text_tokens.begin(), text_tokens.end()); + } std::cout << "生成中...\n"; - auto gen_result = generate_text(model, prompt_tokens, - cfg.max_tokens, cfg.temperature, - cfg.seq_len); + auto gen_result = generate_text(model, engine, prompt_tokens, + cfg.max_tokens, cfg.temperature, eos_id); if (!gen_result) { std::cerr << "Error: " << gen_result.error().message << '\n'; continue; } auto generated = std::move(*gen_result); - std::cout << "\n" << prompt << tokenizer.decode(generated) << "\n\n"; + // 过滤尾部的 标记（避免显示在输出中） + if (has_dialogue) + { + const std::string end_asst = ""; + auto decoded = tokenizer.decode(generated); + auto pos = decoded.find(end_asst); + if (pos != std::string::npos) + decoded = decoded.substr(0, pos); + std::cout << "\n" << decoded << "\n\n"; + } + else + { + std::cout << "\n" << tokenizer.decode(generated) << "\n\n"; + } if (cfg.show_tokens) { - std::cout << "Tokens: ["; + std::cout << "Prompt tokens: ["; for (std::size_t i = 0; i < prompt_tokens.size(); ++i) { if (i > 0) std::cout << ", "; std::cout << prompt_tokens[i]; } - std::cout << " -> "; + std::cout << "]\nGenerated tokens: ["; for (std::size_t i = 0; i < generated.size(); ++i) { if (i > 0) std::cout << ", "; @@ -158,100 +225,145 @@ void interactive_mode(nn::Model &model, const nn::BPETokenizer &tokenizer, // ==================== 主函数 ==================== int main(int argc, char *argv[]) { - try + InferConfig cfg = parse_args(argc, argv); + + // ── 从模型文件读取规格 ───────────────────────────────────── + auto spec_result = nn::peek_model_spec(cfg.model_path); + if (!spec_result) { - InferConfig cfg = parse_args(argc, argv); + std::cerr << "读取模型文件失败: " << spec_result.error().message << std::endl; + return 1; + } + nn::ModelSpec spec = spec_result.value(); + if (!spec.is_gpt()) + { + std::cerr << "模型文件不是 GPT 类型 (type=" + << static_cast(spec.type) << ")\n"; + return 1; + } - // ── 加载分词器与模型 ───────────────────────────────────── - nn::BPETokenizer tokenizer; - auto vocab_result = tokenizer.load_vocab("gpt_bpe.json"); - if (!vocab_result) - { - std::cerr << "加载词表失败: " << vocab_result.error().message << std::endl; - return 1; - } - std::cout << "词表: " << tokenizer.vocab_size() << " 词" << std::endl; + // ── 创建计算引擎 ───────────────────────────────────────── + nn::cli::EngineConfig eng_cfg; + eng_cfg.use_gpu = cfg.gpu_enabled; + eng_cfg.use_cuda = cfg.cuda_enabled; + auto engine = nn::cli::create_engine(eng_cfg, std::cout); - // ── 从模型文件读取规格 ───────────────────────────────── - auto spec_result = nn::peek_model_spec(cfg.model_path); - if (!spec_result) - { - std::cerr << "读取模型文件失败: " << spec_result.error().message << std::endl; - return 1; - } - nn::ModelSpec spec = spec_result.value(); + // ── 构建 GPT 模型 ──────────────────────────────────────── + std::cout << "模型规格: vocab=" << spec.vocab_size + << " d_model=" << spec.d_model + << " heads=" << spec.num_heads + << " layers=" << spec.num_layers + << " d_ff=" << spec.d_ff + << " seq_len=" << spec.seq_len; + if (spec.is_alibi_gpt() || spec.pos_encoding == nn::PosEncodingType::ALiBi) + std::cout << " [ALiBi]"; + else if (spec.pos_encoding == nn::PosEncodingType::Sinusoidal) + std::cout << " [Sinusoidal]"; + else + std::cout << " [Learned]"; + std::cout << "\n"; - nn::Model model; - if (spec.is_gpt()) - { - // V2 格式：自动从规格构建模型 - std::cout << "从模型文件读取 GPT 规格 (V2 格式)\n"; - auto build_result = nn::build_gpt_model_from_spec(spec); - if (!build_result) - { - std::cerr << "构建模型失败: " << build_result.error().message << std::endl; - return 1; - } - model = std::move(build_result.value()); - } - else - { - // V1 旧格式：使用命令行参数 - std::cout << "旧格式模型文件，使用命令行参数构建模型\n"; - model = nn::build_gpt_model( - tokenizer.vocab_size(), cfg.d_model, cfg.seq_len, - cfg.num_heads, cfg.d_ff, cfg.num_layers); - } + nn::Result model_result; + // 统一的 GPTModel 通过 pos_encoding 区分 Learned/Sinusoidal/ALiBi， + // GPT 和旧格式 ALiBi_GPT 文件都走同一条构建路径。 + model_result = nn::build_gpt_model_from_spec(*engine, spec); + if (!model_result) + { + std::cerr << "构建模型失败: " << model_result.error().message << std::endl; + return 1; + } + auto model = std::move(*model_result); - std::cout << "加载模型: " << cfg.model_path << " ..." << std::endl; - auto load_result = nn::load_model(cfg.model_path, model); - if (!load_result) + // ── 加载模型参数 + tokenizer ──────────────────────────── + std::cout << "加载模型: " << cfg.model_path << " ..." << std::endl; + auto load_result = nn::load_model(cfg.model_path, model); + if (!load_result) + { + std::cerr << "加载模型失败: " << load_result.error().message << std::endl; + return 1; + } + std::cout << "模型已加载" << std::endl; + + // ── 加载 tokenizer（自动识别类型） ───────────────────────── + std::unique_ptr tokenizer; + const std::string &embedded_json = *load_result; + if (!embedded_json.empty()) + { + // V3 格式：模型文件嵌入了 tokenizer JSON + tokenizer = nn::load_tokenizer_from_string(embedded_json); + if (!tokenizer) { - std::cerr << "加载模型失败: " << load_result.error().message << std::endl; + std::cerr << "解析嵌入 tokenizer 失败或无法识别分词器类型" << std::endl; return 1; } - std::cout << "模型已加载\n" << std::endl; - - if (cfg.interactive) + std::cout << "已从模型文件加载嵌入 tokenizer" << std::endl; + } + else + { + // 回退：从外部 JSON 文件加载 + tokenizer = nn::load_tokenizer_from_file(cfg.vocab_path); + if (!tokenizer) { - interactive_mode(model, tokenizer, cfg); - return 0; + std::cerr << "加载词表失败或无法识别分词器类型: " << cfg.vocab_path + << "\n请使用 --vocab 指定词表路径，或使用 V3 格式模型" << std::endl; + return 1; } + std::cout << "已从外部文件加载 tokenizer: " << cfg.vocab_path << std::endl; + } + std::cout << "词表大小: " << tokenizer->vocab_size() << "\n" << std::endl; + + // ── 交互模式 ───────────────────────────────────────────── + if (cfg.interactive) + { + interactive_mode(model, *engine, *tokenizer, cfg); + return 0; + } - // ── 单次生成 ───────────────────────────────────────────── - auto prompt_tokens = tokenizer.encode(cfg.prompt); + // ── 单次生成 ───────────────────────────────────────────── + const std::size_t bos_id = tokenizer->bos_id(); + const std::size_t eos_id = tokenizer->eos_id(); + auto prompt_tokens = tokenizer->encode(cfg.prompt); + // 添加 BOS 前缀，使推理输入格式与训练时一致（训练时每行以 BOS 开头） + if (bos_id != nn::Tokenizer::npos) + prompt_tokens.insert(prompt_tokens.begin(), bos_id); + std::cout << "提示: \"" << cfg.prompt << "\"\n"; + std::cout << "Prompt tokens: " << prompt_tokens.size() << " 个\n"; + std::cout << "生成 " << cfg.max_tokens << " 个 token" + << " (temperature=" << cfg.temperature << ")\n"; + std::cout << "----------------------------------------" << std::endl; - std::cout << "提示: \"" << cfg.prompt << "\"\n"; - std::cout << "生成 " << cfg.max_tokens << " 个 token" - << " (temperature=" << cfg.temperature << ")\n"; - std::cout << "----------------------------------------\n"; + auto t_start = std::chrono::steady_clock::now(); + auto gen_result = generate_text(model, *engine, prompt_tokens, + cfg.max_tokens, cfg.temperature, eos_id); + auto t_end = std::chrono::steady_clock::now(); + Scalar gen_sec = std::chrono::duration(t_end - t_start).count(); - auto gen_result = generate_text(model, prompt_tokens, - cfg.max_tokens, cfg.temperature, - cfg.seq_len); - if (!gen_result) { std::cerr << "Error: " << gen_result.error().message << '\n'; return 1; } - auto generated = std::move(*gen_result); + if (!gen_result) + { + std::cerr << "生成失败: " << gen_result.error().message << std::endl; + return 1; + } + auto generated = std::move(*gen_result); - std::cout << cfg.prompt << tokenizer.decode(generated) << std::endl; + std::cout << cfg.prompt << tokenizer->decode(generated) << std::endl; - if (cfg.show_tokens) + if (cfg.show_tokens) + { + std::cout << "\n[Generated tokens: "; + for (std::size_t i = 0; i < generated.size(); ++i) { - std::cout << "\n[Tokens: "; - for (std::size_t i = 0; i < generated.size(); ++i) - { - if (i > 0) std::cout << ", "; - std::cout << generated[i]; - } - std::cout << "]\n"; + if (i > 0) std::cout << ", "; + std::cout << generated[i]; } + std::cout << "]\n"; + } - std::cout << "\n----------------------------------------" << std::endl; + std::cout << "\n----------------------------------------" << std::endl; + std::cout << "生成 " << generated.size() << " tokens, 耗时 " + << std::fixed << std::setprecision(1) << gen_sec << "s" + << " (" << std::setprecision(0) + << (gen_sec > 0 ? generated.size() / gen_sec : 0) << " tokens/s)" + << std::endl; - return 0; - } - catch (const std::exception &e) - { - std::cerr << "错误: " << e.what() << std::endl; - return 1; - } + return 0; } diff --git a/src/text_train.cpp b/src/text_train.cpp index 90a358d2..d5874889 100644 --- a/src/text_train.cpp +++ b/src/text_train.cpp @@ -1,41 +1,148 @@ +// ── GPT 文本生成训练程序（引擎化架构） ────────────────────────────────────── +// +// 数据流（滑动窗口，GPT 预训练标准做法）： +// 文本 → 逐行 Tokenizer.encode → 每行编码为 [BOS]+tokens+[EOS] 后拼接成连续 +// token 流（行边界通过 EOS 编码进流，窗口可跨行，上下文连续） +// 按 stride 对 token 流滑动切 seq_len 窗口 → 每窗口 = 一个训练样本 +// 每 batch：窗口 → Matrix(seq_len, batch) → engine.from_matrix → Tensor +// → GPTModel.forward(Tensor) → Tensor(vocab_size, seq_len×batch) +// → ce_loss.forward_sparse(engine, logits, flat_labels, loss_mask) → Scalar +// → ce_loss.backward() → Tensor +// → model.backward(Tensor) → (丢弃) +// → optimizer.step() / zero_grad() +// +// 引擎选择：--gpu 启用 GpuEngine（需要 Vulkan），否则 CpuEngine。 +// ───────────────────────────────────────────────────────────────────────── + #include -#include -#include +#include +#include +#include +#include + +#include +#include +#include #include -#include -#include #include +#include +#include +#include +#include #include #include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include + +using nn::Scalar; + +// ── 设备丢失自动重启：保存 checkpoint → 等待 GPU 驱动恢复 → 重新启动进程 ── +// Windows TDR 重置 GPU 驱动后，需要重新创建 VkDevice 才能继续使用 GPU。 +// 由于 VkDevice 已死且无法在同进程内恢复，最可靠的方式是保存 checkpoint 后 +// 自动重启进程（带 --resume 和增大 flush_interval 来拆分 GPU 提交）。 +[[noreturn]] void restart_on_device_lost( + const std::string& program_name, + nn::Model& model, + const nn::ModelSpec& spec, + const std::string& tokenizer_json, + const std::string& save_path, + const std::string& text_path, + std::size_t new_flush_interval, + bool gpu_enabled, + bool cuda_enabled) +{ + std::cerr << "\n [TDR] GPU 设备已丢失 (VK_ERROR_DEVICE_LOST)\n"; + std::cerr << " [TDR] 尝试保存 checkpoint...\n"; + auto save_r = nn::save_model(save_path, model, spec, tokenizer_json); + if (save_r) + std::cerr << " [TDR] 模型已保存到: " << save_path << "\n"; + else + std::cerr << " [TDR] 保存失败: " << save_r.error().message << "\n"; + + // 重建命令行：加入 --resume 和增大 flush_interval（不降 batch_size） + std::ostringstream oss; + oss << "\"" << program_name << "\" \"" << text_path + << "\" --resume \"" << save_path + << "\" --flush-interval " << new_flush_interval; + if (gpu_enabled) oss << " --gpu"; + if (cuda_enabled) oss << " --cuda"; + + std::string cmd = oss.str(); + std::cerr << "\n [TDR] 等待 GPU 驱动恢复 (5 秒)...\n"; + std::this_thread::sleep_for(std::chrono::seconds(5)); + std::cerr << " [TDR] 自动重启: " << cmd << "\n\n"; + std::cerr.flush(); + + std::system(cmd.c_str()); + std::exit(0); +} // ==================== 帮助信息 ==================== void print_usage(const char *prog) { std::cout - << "GPT 文本生成训练程序\n\n" + << "GPT 文本生成训练程序 (引擎化架构)\n\n" << "用法: " << prog << " [选项]\n\n" << "参数:\n" << " 训练文本文件路径 (必需)\n\n" << "选项:\n" << " --save 模型保存路径 (默认: gpt_model.bin)\n" << " --resume 从已有模型恢复训练\n" + << " --vocab 词表 JSON 路径 (默认: gpt_bpe.json)\n" + << " 自动识别分词器类型（bpe / charbpe / wordzip / space）\n" + << " --test-file 测试集文件路径（可选，每 epoch 结束后评估 test loss）\n" << " --epochs 训练轮数 (默认: 10)\n" << " --lr 学习率 (默认: 0.001)\n" << " --batch-size 批大小 (默认: 32)\n" + << " --accum-steps 梯度累积步数 (默认: 1)。每 n 步 forward/backward\n" + << " 累加梯度后再更新参数，等效放大 batch_size×n\n" << " --seq-len 序列长度 (默认: 256)\n" - << " --optimizer 优化器: sgd/sgd_w_momentum/adam (默认: adam)\n" + << " --stride 滑动窗口步长 (默认: 等于 --seq-len，即不重叠)\n" + << " 设小可产生重叠窗口，增加训练样本数\n" + << " --optimizer 优化器: sgd/sgd_momentum/adam/adamw/muon (默认: adam)\n" + << " --weight-decay AdamW 权重衰减系数 (默认: 0.01)\n" << " --d-model 模型维度 (默认: 128)\n" << " --num-heads 注意力头数 (默认: 4)\n" << " --num-layers Transformer 层数 (默认: 4)\n" << " --d-ff FFN 中间维度 (默认: 512)\n" + << " --gpu 启用 GPU 加速 (需要 Vulkan SDK)\n" + << " --cuda 启用 CUDA GPU 加速 (需要 CUDA Toolkit)\n" + << " --positional-encoding \n" + << " 位置编码类型: learned(默认)/sinusoidal/alibi\n" + << " learned: 可学习位置嵌入（默认，GPT 原版）\n" + << " sinusoidal: 正弦波固定位置编码（不参与训练）\n" + << " alibi: 线性偏置注意力（无位置嵌入，支持长度外推）\n" + << " --activation FFN 激活: gelu(默认)/swiglu\n" + << " gelu: QuickGeLU（GPT-2 风格）\n" + << " swiglu: SwiGLU（LLaMA/Mistral 风格，每参数效率更高）\n" + << " --norm 归一化层: layernorm(默认)/rmsnorm\n" + << " layernorm: LayerNorm（GPT-2 风格）\n" + << " rmsnorm: RMSNorm（LLaMA/Mistral 风格，更快更稳）\n" + << " --log-interval 每隔多少 step 显示进度 (默认: 50)\n" + << " --save-interval 每隔多少 step 保存 checkpoint (默认: 100)\n" + << " --grad-log 显示梯度统计（范数/最大值/均值）\n" + << "\n" + << "TDR 防护:\n" + << " --tdr-retry GPU 超时自动减小 batch 重试 (默认: on)\n" + << " --max-tdr-retries 最大重试次数，每次 batch 减半 (默认: 4)\n" + << "\n" + << "Batch 录制粒度:\n" + << " --flush-interval 每 N 个 Transformer block flush 一次 (默认: 0=不间断)\n" + << " 增大此值可拆分大提交防 TDR，不影响 batch_size 和训练质量\n" + << " 设备丢失 (VK_ERROR_DEVICE_LOST) 时自动保存 checkpoint 并退出，\n" + << " 可用 --resume 恢复训练。\n" + << "\n" + << "学习率调度:\n" + << " --lr-schedule 学习率调度: fixed/cosine/step_cosine (默认: fixed)\n" + << " cosine: 余弦退火（epoch 级），lr 从初始值衰减到 min-lr\n" + << " step_cosine: 余弦退火（step 级），按单个训练步预热+退火，\n" + << " 适合每 epoch 步数很多的场景（如大语料）\n" + << " --warmup-epochs 线性预热轮数 (默认: 0, 仅 cosine)\n" + << " --warmup-steps 线性预热步数 (默认: 0, 仅 step_cosine)\n" + << " --min-lr 余弦退火最低学习率 (默认: 1e-6)\n" + << " --lr-per-epoch 手动指定每轮学习率 (逗号分隔，优先级最高)\n" + << " --max-norm 梯度裁剪最大全局 L2 范数 (默认: 0=不裁剪)\n" << " --help 显示此帮助信息\n"; } @@ -43,18 +150,48 @@ void print_usage(const char *prog) struct TrainConfig { std::string text_path; + std::string test_path; // 测试集文件路径（可选，用于每 epoch 结束后评估） std::string save_path = "gpt_model.bin"; + std::string vocab_path = "gpt_bpe.json"; std::string resume_path; std::string optimizer_name = "adam"; int epochs = 10; - double lr = 0.001; + Scalar lr = 0.001; + Scalar weight_decay = 0.01f; // AdamW 权重衰减系数 std::size_t batch_size = 32; - std::size_t seq_len = 256; + std::size_t accum_steps = 1; // 梯度累积步数（1 = 不累积） + std::size_t seq_len = nn::GPT_SEQ_LEN; + std::size_t stride = 0; // 滑动窗口步长，0 = 默认等于 seq_len（不重叠） std::size_t d_model = nn::GPT_D_MODEL; std::size_t num_heads = nn::GPT_NUM_HEADS; std::size_t num_layers = nn::GPT_NUM_LAYERS; std::size_t d_ff = nn::GPT_D_FF; + std::size_t log_interval = 50; + std::size_t save_interval = 100; // checkpoint 保存间隔（独立于 log_interval） bool load_existing = false; + bool gpu_enabled = false; + bool cuda_enabled = false; + bool grad_log = false; // 显示梯度统计 + nn::PosEncodingType pos_encoding = nn::PosEncodingType::Learned; + nn::ActivationType activation = nn::ActivationType::GeLU; // FFN 激活 + nn::NormType norm_type = nn::NormType::LayerNorm; // 归一化层类型 + + // TDR 自动重试 + bool auto_tdr_retry = true; // 遇到 TDR 超时自动减小 batch 重试 + std::size_t max_tdr_retries = 4; // 最大重试次数（每次 batch 减半） + + // batch 录制粒度：在 Transformer block 间按间隔 flush，拆分大提交 + std::size_t flush_interval = 0; // 0=不间断（默认），>0=每 N 个 block flush + + // 学习率调度 + std::string lr_schedule = "fixed"; // fixed / cosine / step_cosine + int warmup_epochs = 0; // 线性预热轮数（epoch 级） + std::size_t warmup_steps = 0; // 线性预热步数（step 级） + Scalar min_lr = 1e-6f; // 余弦退火最低 lr + std::vector lr_per_epoch; // 手动指定每轮 lr（为空则自动计算） + + // 梯度裁剪 + Scalar max_norm = 0.0f; // 0 = 不裁剪 }; TrainConfig parse_args(int argc, char *argv[]) @@ -75,33 +212,223 @@ TrainConfig parse_args(int argc, char *argv[]) cfg.resume_path = argv[++i]; cfg.load_existing = true; } + else if (arg == "--vocab" && i + 1 < argc) + cfg.vocab_path = argv[++i]; else if (arg == "--epochs" && i + 1 < argc) - cfg.epochs = std::stoi(argv[++i]); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --epochs: " << v.error().message << "\n"; std::exit(1); } + cfg.epochs = *v; + } else if (arg == "--lr" && i + 1 < argc) - cfg.lr = std::stod(argv[++i]); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --lr: " << v.error().message << "\n"; std::exit(1); } + cfg.lr = *v; + } else if (arg == "--batch-size" && i + 1 < argc) - cfg.batch_size = static_cast(std::stoi(argv[++i])); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --batch-size: " << v.error().message << "\n"; std::exit(1); } + cfg.batch_size = *v; + } + else if (arg == "--accum-steps" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --accum-steps: " << v.error().message << "\n"; std::exit(1); } + if (*v == 0) { std::cerr << "--accum-steps 必须 >= 1\n"; std::exit(1); } + cfg.accum_steps = *v; + } else if (arg == "--seq-len" && i + 1 < argc) - cfg.seq_len = static_cast(std::stoi(argv[++i])); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --seq-len: " << v.error().message << "\n"; std::exit(1); } + cfg.seq_len = *v; + } + else if (arg == "--stride" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --stride: " << v.error().message << "\n"; std::exit(1); } + cfg.stride = *v; + } else if (arg == "--optimizer" && i + 1 < argc) { cfg.optimizer_name = argv[++i]; - if (cfg.optimizer_name != "sgd" && cfg.optimizer_name != "sgd_w_momentum" && - cfg.optimizer_name != "adam") + if (cfg.optimizer_name != "sgd" && cfg.optimizer_name != "sgd_momentum" && + cfg.optimizer_name != "adam" && cfg.optimizer_name != "adamw" && + cfg.optimizer_name != "muon") { std::cerr << "未知优化器: " << cfg.optimizer_name - << "，可选: sgd, sgd_w_momentum, adam\n"; + << "，可选: sgd, sgd_momentum, adam, adamw, muon\n"; std::exit(1); } } + else if (arg == "--weight-decay" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --weight-decay: " << v.error().message << "\n"; std::exit(1); } + cfg.weight_decay = *v; + } else if (arg == "--d-model" && i + 1 < argc) - cfg.d_model = static_cast(std::stoi(argv[++i])); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --d-model: " << v.error().message << "\n"; std::exit(1); } + cfg.d_model = *v; + } else if (arg == "--num-heads" && i + 1 < argc) - cfg.num_heads = static_cast(std::stoi(argv[++i])); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --num-heads: " << v.error().message << "\n"; std::exit(1); } + cfg.num_heads = *v; + } else if (arg == "--num-layers" && i + 1 < argc) - cfg.num_layers = static_cast(std::stoi(argv[++i])); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --num-layers: " << v.error().message << "\n"; std::exit(1); } + cfg.num_layers = *v; + } else if (arg == "--d-ff" && i + 1 < argc) - cfg.d_ff = static_cast(std::stoi(argv[++i])); + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --d-ff: " << v.error().message << "\n"; std::exit(1); } + cfg.d_ff = *v; + } + else if (arg == "--log-interval" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --log-interval: " << v.error().message << "\n"; std::exit(1); } + cfg.log_interval = *v; + } + else if (arg == "--save-interval" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --save-interval: " << v.error().message << "\n"; std::exit(1); } + cfg.save_interval = *v; + } + else if (arg == "--gpu") + cfg.gpu_enabled = true; + else if (arg == "--cuda") + cfg.cuda_enabled = true; + else if (arg == "--grad-log") + cfg.grad_log = true; + else if (arg == "--lr-schedule" && i + 1 < argc) + { + cfg.lr_schedule = argv[++i]; + if (cfg.lr_schedule != "fixed" && cfg.lr_schedule != "cosine" && + cfg.lr_schedule != "step_cosine") + { + std::cerr << "未知 --lr-schedule: " << cfg.lr_schedule + << "，可选: fixed, cosine, step_cosine\n"; + std::exit(1); + } + } + else if (arg == "--warmup-epochs" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --warmup-epochs: " << v.error().message << "\n"; std::exit(1); } + cfg.warmup_epochs = *v; + } + else if (arg == "--warmup-steps" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --warmup-steps: " << v.error().message << "\n"; std::exit(1); } + cfg.warmup_steps = *v; + } + else if (arg == "--min-lr" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --min-lr: " << v.error().message << "\n"; std::exit(1); } + cfg.min_lr = *v; + } + else if (arg == "--lr-per-epoch" && i + 1 < argc) + { + std::string dims_str = argv[++i]; + std::stringstream ss(dims_str); + std::string token; + while (std::getline(ss, token, ',')) + { + auto v = nn::parse_number(token); + if (!v) { std::cerr << "无效 --lr-per-epoch 值: " << v.error().message << "\n"; std::exit(1); } + cfg.lr_per_epoch.push_back(*v); + } + } + else if (arg == "--max-norm" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --max-norm: " << v.error().message << "\n"; std::exit(1); } + cfg.max_norm = *v; + } + else if (arg == "--positional-encoding" && i + 1 < argc) + { + std::string v = argv[++i]; + if (v == "learned") + cfg.pos_encoding = nn::PosEncodingType::Learned; + else if (v == "sinusoidal") + cfg.pos_encoding = nn::PosEncodingType::Sinusoidal; + else if (v == "alibi") + cfg.pos_encoding = nn::PosEncodingType::ALiBi; + else + { + std::cerr << "未知位置编码类型: " << v + << "，可选: learned, sinusoidal, alibi\n"; + std::exit(1); + } + } + else if (arg == "--activation" && i + 1 < argc) + { + std::string v = argv[++i]; + if (v == "gelu") + cfg.activation = nn::ActivationType::GeLU; + else if (v == "swiglu") + cfg.activation = nn::ActivationType::SwiGLU; + else + { + std::cerr << "未知激活类型: " << v + << "，可选: gelu, swiglu\n"; + std::exit(1); + } + } + else if (arg == "--norm" && i + 1 < argc) + { + std::string v = argv[++i]; + if (v == "layernorm") + cfg.norm_type = nn::NormType::LayerNorm; + else if (v == "rmsnorm") + cfg.norm_type = nn::NormType::RMSNorm; + else + { + std::cerr << "未知归一化层类型: " << v + << "，可选: layernorm, rmsnorm\n"; + std::exit(1); + } + } + else if (arg == "--tdr-retry" && i + 1 < argc) + { + std::string v = argv[++i]; + if (v == "on" || v == "1" || v == "true") + cfg.auto_tdr_retry = true; + else if (v == "off" || v == "0" || v == "false") + cfg.auto_tdr_retry = false; + else + { + std::cerr << "无效 --tdr-retry: " << v << "，可选: on, off\n"; + std::exit(1); + } + } + else if (arg == "--max-tdr-retries" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --max-tdr-retries: " << v.error().message << "\n"; std::exit(1); } + cfg.max_tdr_retries = *v; + } + else if (arg == "--flush-interval" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --flush-interval: " << v.error().message << "\n"; std::exit(1); } + cfg.flush_interval = *v; + } + else if (arg == "--test-file" && i + 1 < argc) + cfg.test_path = argv[++i]; else if (!arg.starts_with("--")) cfg.text_path = arg; else @@ -134,233 +461,682 @@ nn::Matrix one_hot_labels(const std::vector &tokens, std::size_t vo return result; } +// ==================== 梯度统计 ==================== +// 计算并打印全局梯度统计：L2 范数、绝对值最大值、均值。 +// GPU 模式下自动通过 engine.to_matrix() 下载张量到 CPU。 +void log_gradient_stats(nn::ComputeEngine &engine, const std::vector &grads) +{ + Scalar global_sum_sq = 0.0; + Scalar global_abs_max = 0.0; + Scalar global_abs_sum = 0.0; + std::size_t global_count = 0; + std::size_t tensor_idx = 0; + + for (auto& grad_ref : grads) + { + const auto& grad = grad_ref.get(); + // GPU 张量需要下载到 CPU + nn::Matrix mat; + if (grad.device() == nn::Device::GPU) + { + auto m = engine.to_matrix(grad); + if (!m) continue; + mat = std::move(*m); + } + else + { + mat = grad.cpu_matrix(); + } + + const Scalar sum_sq = mat.reduce(Scalar{0}, std::plus<>{}, + [](Scalar x) { return x * x; }); + const Scalar abs_max = mat.reduce(Scalar{0}, + [](Scalar a, Scalar b) { return std::max(a, b); }, + [](Scalar x) { return std::abs(x); }); + const Scalar abs_sum = mat.reduce(Scalar{0}, std::plus<>{}, + [](Scalar x) { return std::abs(x); }); + const std::size_t n = mat.size(); + + global_sum_sq += sum_sq; + global_abs_max = std::max(global_abs_max, abs_max); + global_abs_sum += abs_sum; + global_count += n; + + // 逐张量输出（仅在张量数量 <= 30 时显示详情，避免刷屏） + if (grads.size() <= 30) + { + std::cout << " [" << tensor_idx << "] " + << "(" << mat.rows() << "x" << mat.cols() << ") " + << "norm=" << std::scientific << std::setprecision(4) + << std::sqrt(sum_sq) << " max|g|=" << abs_max + << " mean|g|=" << (n > 0 ? abs_sum / n : Scalar{0}) + << std::endl; + } + ++tensor_idx; + } + + const Scalar global_norm = std::sqrt(global_sum_sq); + const Scalar global_mean = global_count > 0 ? global_abs_sum / global_count : Scalar{0}; + std::cout << " >> grad_norm=" << std::scientific << std::setprecision(4) << global_norm + << " max|g|=" << global_abs_max + << " mean|g|=" << global_mean + << " params=" << global_count + << " tensors=" << grads.size() + << std::endl; +} + // ==================== 主函数 ==================== int main(int argc, char *argv[]) { - try + const std::string program_name = argv[0]; + TrainConfig cfg = parse_args(argc, argv); + + // ── 加载文本 ───────────────────────────────────────────── + std::cout << "加载文本: " << cfg.text_path << " ..." << std::endl; + auto text_result = nn::load_text_file(cfg.text_path); + if (!text_result) { + std::cerr << "Error: " << text_result.error().message << '\n'; + return 1; + } + std::string text = std::move(*text_result); + if (text.empty()) { - TrainConfig cfg = parse_args(argc, argv); + std::cerr << "文本文件为空\n"; + return 1; + } - // ── 加载文本 ───────────────────────────────────────────── - std::cout << "加载文本: " << cfg.text_path << " ..." << std::endl; - auto text_result = nn::load_text_file(cfg.text_path); - if (!text_result) { - std::cerr << "Error: " << text_result.error().message << '\n'; - return 1; + // ── 加载分词器（自动识别类型：BPE/CharBPE/WordZip/Space） ─── + auto tokenizer = nn::load_tokenizer_from_file(cfg.vocab_path); + if (!tokenizer) + { + std::cerr << "加载词表失败或无法识别分词器类型: " << cfg.vocab_path << '\n' + << "请检查 JSON 文件是否包含有效的 \"type\" 字段" << std::endl; + return 1; + } + const std::size_t bos_id = tokenizer->bos_id(); + const std::size_t eos_id = tokenizer->eos_id(); + + // ── 滑动窗口数据组织（GPT 预训练标准做法）──────────────────────── + // 每行 = 一个文档，编码为 [BOS] + tokens + [EOS]，然后拼接成一条连续 + // token 流（行边界通过 EOS 编码进流，窗口可跨行，上下文连续）。 + // 训练样本 = 对 token 流按 stride 滑动切出的 seq_len 窗口： + // - 不再"每行一个样本"：长行不再被截断丢弃、短行不再大量 PAD， + // 语料被 100% 利用； + // - 窗口可跨行，模型能学到跨行/跨段依赖； + // - 仅最后一个窗口不足 seq_len 时 PAD，且 loss 屏蔽 PAD 位置。 + const std::size_t pad_id = tokenizer->pad_id(); + const std::size_t stride = (cfg.stride == 0) ? cfg.seq_len : cfg.stride; + std::vector token_flow; // 全量 token 流（含 BOS/EOS） + { + std::istringstream iss(text); + std::string line; + while (std::getline(iss, line)) + { + // 去除行首尾空白 + auto lpos = line.find_first_not_of(" \t\r\n"); + if (lpos == std::string::npos) continue; // 空行跳过 + line = line.substr(lpos, line.find_last_not_of(" \t\r\n") - lpos + 1); + if (line.empty()) continue; + + auto line_tokens = tokenizer->encode(line); + if (line_tokens.empty()) continue; + if (bos_id != nn::Tokenizer::npos) + token_flow.push_back(bos_id); + token_flow.insert(token_flow.end(), line_tokens.begin(), line_tokens.end()); + if (eos_id != nn::Tokenizer::npos) + token_flow.push_back(eos_id); } - std::string text = std::move(*text_result); - if (text.empty()) + } + // 切窗口：每个窗口 = 一个训练样本（长度 seq_len，末窗不足则 PAD） + std::vector window_offsets; // 每个窗口在 token_flow 中的起始偏移 + window_offsets.reserve(token_flow.size() / stride + 1); + for (std::size_t pos = 0; pos < token_flow.size(); pos += stride) + window_offsets.push_back(pos); + std::cout << "文本 token 流: " << token_flow.size() << " tokens (含 BOS/EOS), " + << tokenizer->vocab_size() << " 词表\n" + << "滑动窗口: seq_len=" << cfg.seq_len << " stride=" << stride + << " 样本数=" << window_offsets.size() << "\n" << std::endl; + + // ── 打印配置 ───────────────────────────────────────────── + std::cout << "========================================\n"; + std::cout << " GPT 文本生成训练 (引擎化架构)\n"; + std::cout << "========================================\n"; + std::cout << " 词表大小: " << tokenizer->vocab_size() << "\n"; + std::cout << " 模型维度: " << cfg.d_model << "\n"; + std::cout << " 注意力头: " << cfg.num_heads << "\n"; + std::cout << " Transformer 层数: " << cfg.num_layers << "\n"; + std::cout << " FFN 维度: " << cfg.d_ff << "\n"; + std::cout << " 序列长度: " << cfg.seq_len << "\n"; + std::cout << " 优化器: " << cfg.optimizer_name << " 学习率: " << cfg.lr << "\n"; + std::cout << " 轮数: " << cfg.epochs << " 批大小: " << cfg.batch_size << "\n"; + std::cout << " GPU: " << (cfg.gpu_enabled ? "启用" : "禁用") << "\n"; + std::cout << " 梯度日志: " << (cfg.grad_log ? "启用" : "禁用") << "\n"; + std::cout << "========================================\n\n"; + + // ── 读取 tokenizer JSON 以便嵌入模型 ───────────────────── + std::string tokenizer_json; + { + std::ifstream tfs(cfg.vocab_path, std::ios::binary); + if (tfs) { - std::cerr << "文本文件为空\n"; - return 1; + std::ostringstream oss; + oss << tfs.rdbuf(); + tokenizer_json = oss.str(); } + } - nn::BPETokenizer tokenizer; + // ── 创建计算引擎 ───────────────────────────────────────── + nn::cli::EngineConfig eng_cfg; + eng_cfg.use_gpu = cfg.gpu_enabled; + eng_cfg.use_cuda = cfg.cuda_enabled; + auto engine = nn::cli::create_engine(eng_cfg, std::cout); + + // ── 构建模型（绑定引擎） ───────────────────────────────── + auto model_build = nn::build_gpt_model( + *engine, + tokenizer->vocab_size(), cfg.d_model, cfg.seq_len, + cfg.num_heads, cfg.d_ff, cfg.num_layers, + cfg.pos_encoding, cfg.activation, cfg.norm_type); + if (!model_build) { + std::cerr << "构建模型失败: " << model_build.error().message << '\n'; + return 1; + } + auto model = std::move(*model_build); + + // ── 设置 batch 录制粒度 ── + model.set_flush_interval(cfg.flush_interval); + + // ── 构建规格（用于保存） ───────────────────────────────── + auto spec = nn::make_gpt_spec( + tokenizer->vocab_size(), cfg.d_model, cfg.seq_len, + cfg.num_heads, cfg.d_ff, cfg.num_layers, + cfg.pos_encoding, cfg.activation, cfg.norm_type); + + if (cfg.load_existing) + { + auto spec_result = nn::peek_model_spec(cfg.resume_path); + if (!spec_result) { - auto vocab_result = tokenizer.load_vocab("gpt_bpe.json"); - if (!vocab_result) { - std::cerr << "Error: " << vocab_result.error().message << '\n'; - return 1; - } + std::cerr << "加载模型失败: " << spec_result.error().message << "，将从头训练。\n" << std::endl; } - auto all_tokens = tokenizer.encode(text); - std::cout << "文本长度: " << text.size() << " 字符, " - << all_tokens.size() << " tokens, " - << tokenizer.vocab_size() << " 词表\n" << std::endl; - - // ── 打印配置 ───────────────────────────────────────────── - std::cout << "========================================\n"; - std::cout << " GPT 文本生成训练\n"; - std::cout << "========================================\n"; - std::cout << " 词表大小: " << tokenizer.vocab_size() << "\n"; - std::cout << " 模型维度: " << cfg.d_model << "\n"; - std::cout << " 注意力头: " << cfg.num_heads << "\n"; - std::cout << " Transformer 层数: " << cfg.num_layers << "\n"; - std::cout << " FFN 维度: " << cfg.d_ff << "\n"; - std::cout << " 序列长度: " << cfg.seq_len << "\n"; - std::cout << " 优化器: " << cfg.optimizer_name << " 学习率: " << cfg.lr << "\n"; - std::cout << " 轮数: " << cfg.epochs << " 批大小: " << cfg.batch_size << "\n"; - std::cout << "========================================\n\n"; - - // ── 构建模型 ───────────────────────────────────────────── - auto model = nn::build_gpt_model( - tokenizer.vocab_size(), cfg.d_model, cfg.seq_len, - cfg.num_heads, cfg.d_ff, cfg.num_layers); - - // ── 构建规格（用于保存） ───────────────────────────────── - auto spec = nn::make_gpt_spec( - tokenizer.vocab_size(), cfg.d_model, cfg.seq_len, - cfg.num_heads, cfg.d_ff, cfg.num_layers); - - if (cfg.load_existing) - { - // 尝试从文件读取规格（V2 格式） - auto spec_result = nn::peek_model_spec(cfg.resume_path); - if (!spec_result) - { - std::cerr << "加载模型失败: " << spec_result.error().message << "，将从头训练。\n" << std::endl; - } - else + else + { + auto file_spec = std::move(*spec_result); + // 统一的 GPTModel 通过 pos_encoding 区分 Learned/Sinusoidal/ALiBi， + // 因此 GPT 和旧格式 ALiBi_GPT 文件都走同一条构建路径。 + if (file_spec.is_gpt() || file_spec.is_alibi_gpt()) { - auto file_spec = std::move(*spec_result); - if (file_spec.is_gpt()) - { - std::cout << "从模型文件读取 GPT 规格 (V2 格式)\n"; - auto build_result = nn::build_gpt_model_from_spec(file_spec); - if (!build_result) - { - std::cerr << "Error: " << build_result.error().message << '\n'; - return 1; - } - model = std::move(*build_result); - spec = file_spec; - } + if (file_spec.pos_encoding == nn::PosEncodingType::ALiBi) + std::cout << "从模型文件读取 ALiBi GPT 规格\n"; else + std::cout << "从模型文件读取 GPT 规格\n"; + auto build_result = nn::build_gpt_model_from_spec(*engine, file_spec); + if (!build_result) { - std::cout << "旧格式模型文件，使用命令行参数\n"; + std::cerr << "Error: " << build_result.error().message << '\n'; + return 1; } + model = std::move(*build_result); + spec = file_spec; + } + else + std::cout << "旧格式模型文件，使用命令行参数\n"; - auto load_result = nn::load_model(cfg.resume_path, model); - if (!load_result) - { - std::cerr << "加载模型失败: " << load_result.error().message << "，将从头训练。\n" << std::endl; - } - else - { - std::cout << "已加载模型: " << cfg.resume_path << "\n" << std::endl; - } + auto load_result = nn::load_model(cfg.resume_path, model); + if (!load_result) + std::cerr << "加载模型失败: " << load_result.error().message << "，将从头训练。\n" << std::endl; + else + { + std::cout << "已加载模型: " << cfg.resume_path; + if (!load_result->empty()) + std::cout << " (含嵌入词表 " << load_result->size() << " 字节)"; + std::cout << "\n" << std::endl; } } + } - // ── 优化器 ───────────────────────────────────────────── - std::unique_ptr optimizer; - if (cfg.optimizer_name == "sgd") - optimizer = std::make_unique(model.parameters(), model.param_gradients(), cfg.lr); - else if (cfg.optimizer_name == "sgd_w_momentum") - optimizer = std::make_unique(model.parameters(), model.param_gradients(), cfg.lr); - else - optimizer = std::make_unique(model.parameters(), model.param_gradients(), cfg.lr); + // ── 优化器 ───────────────────────────────────────────── + auto optimizer = nn::create_optimizer( + cfg.optimizer_name, *engine, + model.parameters(), model.param_gradients(), cfg.lr, + cfg.weight_decay); + if (!optimizer) + { + std::cerr << "错误：未知优化器名称: " << cfg.optimizer_name << "\n"; + return 1; + } + + Scalar optimizer_current_lr = cfg.lr; - nn::CrossEntropyLoss ce_loss; + // ── 学习率调度配置（委托给 nn::cli::compute_epoch_lr） ── + nn::cli::LrScheduleConfig lr_sched_cfg; + lr_sched_cfg.base_lr = cfg.lr; + lr_sched_cfg.min_lr = cfg.min_lr; + lr_sched_cfg.warmup_epochs = cfg.warmup_epochs; + lr_sched_cfg.total_epochs = cfg.epochs; + lr_sched_cfg.schedule = cfg.lr_schedule; + lr_sched_cfg.lr_per_epoch = cfg.lr_per_epoch; - // ── 训练循环 ───────────────────────────────────────────── - // 可用的 token 数量（需要 seq_len + 1 作为 input + target） - const std::size_t max_start = all_tokens.size() - cfg.seq_len - 1; - if (max_start == 0) + nn::CrossEntropyLoss ce_loss; + + // ── 训练循环 ───────────────────────────────────────────── + // 每样本 = 一个 seq_len 滑动窗口（可能跨行），目标 = 输入左移一位。 + if (window_offsets.empty()) + { + std::cerr << "无有效训练样本（文本 token 流为空）\n"; + return 1; + } + std::cout << "训练窗口样本数: " << window_offsets.size() << "\n" << std::endl; + + // ── 加载测试集（可选，与训练一致的滑动窗口处理） ───────────── + std::vector test_window_offsets; + std::vector test_flow; + if (!cfg.test_path.empty()) + { + auto test_text_result = nn::load_text_file(cfg.test_path); + if (!test_text_result) { - std::cerr << "文本太短，至少需要 " << (cfg.seq_len + 2) << " 个 token\n"; + std::cerr << "加载测试集失败: " << test_text_result.error().message << '\n'; return 1; } + std::string test_text = std::move(*test_text_result); + std::size_t test_token_count = 0; + { + std::istringstream tiss(test_text); + std::string tline; + while (std::getline(tiss, tline)) + { + auto tlpos = tline.find_first_not_of(" \t\r\n"); + if (tlpos == std::string::npos) continue; + tline = tline.substr(tlpos, tline.find_last_not_of(" \t\r\n") - tlpos + 1); + if (tline.empty()) continue; - const std::size_t steps_per_epoch = std::min(max_start / cfg.batch_size, - std::size_t{1000}); + auto tl_tokens = tokenizer->encode(tline); + if (tl_tokens.empty()) continue; + if (bos_id != nn::Tokenizer::npos) + test_flow.push_back(bos_id); + test_flow.insert(test_flow.end(), tl_tokens.begin(), tl_tokens.end()); + if (eos_id != nn::Tokenizer::npos) + test_flow.push_back(eos_id); + test_token_count += tl_tokens.size() + 2; + } + } + for (std::size_t pos = 0; pos < test_flow.size(); pos += stride) + test_window_offsets.push_back(pos); + std::cout << "测试集: " << cfg.test_path << " 样本数: " << test_window_offsets.size() + << " tokens: " << test_token_count << std::endl; + } + + // ── 步数与采样：每 epoch 每样本恰好访问一次 ────────────── + // steps_per_epoch = ceil(样本数 / batch_size)：最后一个 batch 不满时 + // 以实际 this_bs 参与训练。由于 loss 已按有效 token 归一化，不满 batch + // 的梯度与满 batch 同尺度（每 epoch 仅一个不满 batch，影响可忽略）。 + std::size_t steps_per_epoch = + (window_offsets.size() + cfg.batch_size - 1) / cfg.batch_size; + if (steps_per_epoch == 0) + { + std::cerr << "样本数 (" << window_offsets.size() << ") 小于 batch_size (" + << cfg.batch_size << ")，请减小 --batch-size 或增大训练语料\n"; + return 1; + } - std::mt19937_64 rng{42}; + // ── Step 级学习率配置（step_cosine 模式） ── + nn::cli::StepLrScheduleConfig step_lr_cfg; + step_lr_cfg.base_lr = cfg.lr; + step_lr_cfg.min_lr = cfg.min_lr; + step_lr_cfg.warmup_steps = static_cast(cfg.warmup_steps); + step_lr_cfg.total_steps = + static_cast(steps_per_epoch * static_cast(cfg.epochs)); + step_lr_cfg.cosine = true; - auto t_start = std::chrono::steady_clock::now(); + std::mt19937_64 rng{42}; + std::vector sample_indices(window_offsets.size()); + for (std::size_t i = 0; i < sample_indices.size(); ++i) + sample_indices[i] = i; - for (int epoch = 0; epoch < cfg.epochs; ++epoch) - { - auto ep_start = std::chrono::steady_clock::now(); - double total_loss = 0.0; + // ── 预分配 batch 缓冲区（末批不满时 resize 到实际 this_bs） ── + // x_tokens: (seq_len, batch) 输入 token IDs（每窗口一列） + // y_tokens: (seq_len, batch) 目标 token IDs（x 左移一位） + // 最后一个窗口不足 seq_len 时用 pad_id 填充，目标对应位置也用 pad_id + nn::Matrix x_tokens(cfg.seq_len, cfg.batch_size); + nn::Matrix y_tokens(cfg.seq_len, cfg.batch_size); + nn::Matrix loss_mask(cfg.seq_len, cfg.batch_size); + std::vector flat_targets(cfg.seq_len * cfg.batch_size); + + auto t_start = std::chrono::steady_clock::now(); - for (std::size_t step = 0; step < steps_per_epoch; ++step) + for (int epoch = 0; epoch < cfg.epochs; ++epoch) + { + // ── 学习率调度：每 epoch 开始时调整（step_cosine 由 step 级调度接管） ── + if (cfg.lr_schedule != "step_cosine") + { + Scalar epoch_lr = nn::cli::compute_epoch_lr(lr_sched_cfg, epoch); + if (epoch_lr != optimizer_current_lr) { - // ── 采样 batch ─────────────────────────────────── - nn::Matrix x_tokens(cfg.seq_len, cfg.batch_size); - nn::Matrix y_tokens(cfg.seq_len, cfg.batch_size); + optimizer->set_lr(epoch_lr); + optimizer_current_lr = epoch_lr; + } + } - for (std::size_t b = 0; b < cfg.batch_size; ++b) - { - std::uniform_int_distribution dist(0, max_start); - std::size_t start = dist(rng); + auto ep_start = std::chrono::steady_clock::now(); + Scalar total_weighted = 0.0; // Σ(loss × 有效token)，用于按 token 加权平均 + std::size_t total_valid = 0; // 累计有效 token 数 - for (std::size_t t = 0; t < cfg.seq_len; ++t) - { - x_tokens.set_value_unchecked(t, b, - static_cast(all_tokens[start + t])); - y_tokens.set_value_unchecked(t, b, - static_cast(all_tokens[start + t + 1])); - } + // 每个 epoch 开始前 shuffle 样本索引队列 + std::shuffle(sample_indices.begin(), sample_indices.end(), rng); + + // 梯度累积：距上次参数更新的步数（每 accum_steps 步更新一次） + std::size_t steps_since_update = 0; + + for (std::size_t step = 0; step < steps_per_epoch; ++step) + { + // ── step 级学习率：每个训练步更新（step_cosine 模式） ── + if (cfg.lr_schedule == "step_cosine") + { + const int global_step = + static_cast(epoch) * static_cast(steps_per_epoch) + + static_cast(step); + Scalar step_lr = nn::cli::compute_step_lr(step_lr_cfg, global_step); + if (step_lr != optimizer_current_lr) + { + optimizer->set_lr(step_lr); + optimizer_current_lr = step_lr; } + } + + // ── 采样 batch：每样本 = 一个滑动窗口 ───────────── + // 按 shuffle 后的顺序切片取 this_bs 个窗口（末批可能不满）。 + // 每 epoch 每样本恰好访问一次。 + const std::size_t offset = step * cfg.batch_size; + const std::size_t this_bs = std::min(cfg.batch_size, window_offsets.size() - offset); + if (x_tokens.cols() != this_bs) + { + x_tokens.resize(cfg.seq_len, this_bs); + y_tokens.resize(cfg.seq_len, this_bs); + loss_mask.resize(cfg.seq_len, this_bs); + flat_targets.resize(cfg.seq_len * this_bs); + } + std::size_t step_valid = 0; // 本 step 参与 loss 的有效 token 数 - // ── 前向传播 ───────────────────────────────────── - auto fwd_result = model.forward(x_tokens); - if (!fwd_result) { std::cerr << "Error: " << fwd_result.error().message << '\n'; return 1; } - auto logits = std::move(*fwd_result); - // logits: (vocab_size, seq_len × batch_size) + for (std::size_t b = 0; b < this_bs; ++b) + { + const std::size_t win_pos = window_offsets[sample_indices[offset + b]]; + // 窗口有效长度：不超过 token_flow 末尾时为 seq_len， + // 否则为剩余 token 数（最后一个窗口，PAD 位置被屏蔽） + const std::size_t win_len = std::min(cfg.seq_len, token_flow.size() - win_pos); - // ── 构造 one-hot 目标 ──────────────────────────── - // 展平 y_tokens: seq_len × batch_size 个 token ID - const std::size_t total_tokens = cfg.seq_len * cfg.batch_size; - std::vector flat_targets(total_tokens); - auto y_span = y_tokens.span(); + // 填充 x/y/mask：x[t] = flow[win_pos+t], y[t] = flow[win_pos+t+1] for (std::size_t t = 0; t < cfg.seq_len; ++t) - for (std::size_t b = 0; b < cfg.batch_size; ++b) - flat_targets[t * cfg.batch_size + b] = - static_cast(y_span[t * cfg.batch_size + b]); + { + const std::size_t x_id = (t < win_len) ? token_flow[win_pos + t] : pad_id; + const std::size_t y_id = (t + 1 < win_len) ? token_flow[win_pos + t + 1] : pad_id; + x_tokens.set_value_unchecked(t, b, static_cast(x_id)); + y_tokens.set_value_unchecked(t, b, static_cast(y_id)); + const bool participate = (t + 1 < win_len); + loss_mask.set_value_unchecked(t, b, participate ? 1.0f : 0.0f); + if (participate) ++step_valid; + } + } + + // ── Matrix → Tensor（上传到引擎设备） ────────────── + auto x_tensor_r = engine->from_matrix(x_tokens); + if (!x_tensor_r) { + std::cerr << "\nfrom_matrix(x_tokens) failed: " << x_tensor_r.error().message << '\n'; + return 1; + } + + // ── 构造平坦标签（与 logits 列序一致：t×batch + b） ── + auto y_span = y_tokens.span(); + for (std::size_t t = 0; t < cfg.seq_len; ++t) + for (std::size_t b = 0; b < this_bs; ++b) + flat_targets[t * this_bs + b] = + static_cast(y_span[t * this_bs + b]); + + // ── 启用 GPU batch 录制 ───────────────────────── + // 整个 forward + backward + optimizer step 录制到一个 command buffer， + // end_batch 时一次 vkQueueSubmit + vkWaitForFences，消除 per-primitive 同步开销。 + // CPU 引擎 begin_batch/end_batch 为 no-op，所以两套引擎都安全。 + auto begin_r = engine->begin_batch(); + if (!begin_r) { + std::cerr << "begin_batch failed: " << begin_r.error().message << '\n'; + return 1; + } + + // ── 前向传播 ───────────────────────────────────── + auto fwd_result = model.forward(*x_tensor_r); + if (!fwd_result) { std::cerr << "Error: " << fwd_result.error().message << '\n'; return 1; } + auto logits = std::move(*fwd_result); + // logits: (vocab_size, seq_len × batch_size) + + // ── 损失（稀疏标签，避免 one-hot 爆显存） ──────── + auto mask_span = std::span(loss_mask.span()); + auto loss_result = ce_loss.forward_sparse( + *engine, logits, flat_targets, mask_span, tokenizer->vocab_size()); + if (!loss_result) { std::cerr << "Error: " << loss_result.error().message << '\n'; return 1; } + Scalar loss = *loss_result; + total_weighted += loss * step_valid; // 按有效 token 加权 + total_valid += step_valid; + + // ── 中点刷新：提交 forward+loss，拆分为两次 GPU 提交 ── + // 大词表 + 长序列时 forward+backward 单次提交可能触发 TDR 超时。 + // 在 forward 与 backward 之间 flush，将一次大提交拆为两次小提交。 + auto flush_r = engine->flush_batch(); + if (!flush_r) { + std::string err_msg = flush_r.error().message; + bool is_device_lost = err_msg.find("VK_ERROR_DEVICE_LOST") != std::string::npos; + std::cerr << "\nflush_batch (forward) failed: " << err_msg; + if (is_device_lost) + { + restart_on_device_lost(program_name, model, spec, tokenizer_json, + cfg.save_path, cfg.text_path, + cfg.flush_interval == 0 ? std::size_t{1} : cfg.flush_interval * 2, + cfg.gpu_enabled, cfg.cuda_enabled); + } + return 1; + } + // ── 反向传播（梯度已含 mask，无需额外处理） ──────── + auto grad_result = ce_loss.backward(); + if (!grad_result) { std::cerr << "\nLoss backward failed: " << grad_result.error().message << '\n'; return 1; } - auto y_onehot = one_hot_labels(flat_targets, tokenizer.vocab_size()); + auto bwd_result = model.backward(*grad_result); + if (!bwd_result) { std::cerr << "Error: " << bwd_result.error().message << '\n'; return 1; } - // ── 损失 ───────────────────────────────────────── - auto loss_result = ce_loss.forward(logits, y_onehot); - if (!loss_result) { std::cerr << "Error: " << loss_result.error().message << '\n'; return 1; } - double loss = *loss_result; - total_loss += loss; + // ── 提交 backward batch（单独一次提交，已与 forward 拆分） ── + auto bwd_end = engine->end_batch(); + if (!bwd_end) { + std::string err_msg = bwd_end.error().message; + bool is_device_lost = err_msg.find("VK_ERROR_DEVICE_LOST") != std::string::npos; + std::cerr << "\nend_batch (backward) failed: " << err_msg; + if (is_device_lost) + { + restart_on_device_lost(program_name, model, spec, tokenizer_json, + cfg.save_path, cfg.text_path, + cfg.flush_interval == 0 ? std::size_t{1} : cfg.flush_interval * 2, + cfg.gpu_enabled, cfg.cuda_enabled); + } + std::cerr << '\n'; + return 1; + } - // ── 反向传播 ───────────────────────────────────── - auto grad = ce_loss.backward(); - auto bwd_result = model.backward(grad); - if (!bwd_result) { std::cerr << "Error: " << bwd_result.error().message << '\n'; return 1; } + // ── 梯度累积：每 accum_steps 步才更新一次参数 ── + // forward+backward 每步都执行（梯度累加到参数梯度）， + // 梯度裁剪/step/zero_grad 仅在累积到 accum_steps 或 epoch 末尾执行。 + ++steps_since_update; + const bool do_update = + (steps_since_update >= cfg.accum_steps) || (step + 1 == steps_per_epoch); + if (do_update) + { + // ── 梯度裁剪（在 step() 之前，backward() 之后） ── + if (cfg.max_norm > 0) { - auto step_result = optimizer->step(); - if (!step_result) { - std::cerr << "Error: " << step_result.error().message << '\n'; + auto clip_r = optimizer->clip_grad_norm(cfg.max_norm); + if (!clip_r) { + std::cerr << "\n梯度裁剪失败: " << clip_r.error().message << '\n'; return 1; } } - optimizer->zero_grad(); - // 进度显示 - if ((step + 1) % 50 == 0 || step + 1 == steps_per_epoch) + auto opt_begin = engine->begin_batch(); + if (!opt_begin) { + std::cerr << "begin_batch (optimizer) failed: " << opt_begin.error().message << '\n'; + return 1; + } + + // ── 优化器 step + 梯度清零 ── + auto step_result = optimizer->step(); + if (!step_result) { + std::cerr << "Error: " << step_result.error().message << '\n'; + return 1; + } + + // ── 梯度统计（step 后、zero_grad 前） ── + if (cfg.grad_log && ((step + 1) % cfg.log_interval == 0 || step + 1 == steps_per_epoch)) { - std::cout << "\r Epoch " << epoch + 1 << "/" << cfg.epochs - << " step " << step + 1 << "/" << steps_per_epoch - << " loss: " << std::fixed << std::setprecision(4) << loss - << " " << std::flush; + std::cout << "\n [grad] step " << step + 1 << ":" << std::endl; + log_gradient_stats(*engine, model.param_gradients()); + } + + auto zero_result = optimizer->zero_grad(); + if (!zero_result) { + std::cerr << "\n优化器 zero_grad 失败: " << zero_result.error().message << '\n'; + return 1; + } + + // ── 提交 batch：一次 vkQueueSubmit + vkWaitForFences ── + auto end_r = engine->end_batch(); + if (!end_r) { + std::cerr << "end_batch (optimizer) failed: " << end_r.error().message << '\n'; + return 1; } + + steps_since_update = 0; } - auto ep_end = std::chrono::steady_clock::now(); - double ep_sec = std::chrono::duration(ep_end - ep_start).count(); - double avg_loss = total_loss / steps_per_epoch; + // ── 定期保存 checkpoint（独立于 log_interval） ── + if ((step + 1) % cfg.save_interval == 0 || step + 1 == steps_per_epoch) + { + auto save_r = nn::save_model(cfg.save_path, model, spec, tokenizer_json); + if (!save_r) + std::cerr << "\n [ckpt] 保存失败: " << save_r.error().message << "\n"; + } - std::cout << "\r Epoch " << epoch + 1 << "/" << cfg.epochs - << " avg_loss=" << std::fixed << std::setprecision(4) << avg_loss - << " time=" << std::setprecision(1) << ep_sec << "s" - << std::endl; + // ── 进度显示 + if ((step + 1) % cfg.log_interval == 0 || step + 1 == steps_per_epoch) + { + std::cout << "\r Epoch " << epoch + 1 << "/" << cfg.epochs + << " step " << step + 1 << "/" << steps_per_epoch + << " loss: " << std::fixed << std::setprecision(4) << loss + << " " << std::flush; + } } - auto t_end = std::chrono::steady_clock::now(); - double total_sec = std::chrono::duration(t_end - t_start).count(); + auto ep_end = std::chrono::steady_clock::now(); + Scalar ep_sec = std::chrono::duration(ep_end - ep_start).count(); + // 按有效 token 加权平均（等价于全局 per-token 平均）， + // 避免"有效 token 少的 batch"拉偏 epoch loss 报告 + Scalar avg_loss = (total_valid > 0) + ? total_weighted / static_cast(total_valid) + : Scalar{0}; - // ── 保存模型（含规格） ───────────────────────────────────── + std::cout << "\r Epoch " << epoch + 1 << "/" << cfg.epochs + << " lr=" << std::scientific << std::setprecision(4) << optimizer_current_lr + << " avg_loss=" << std::fixed << std::setprecision(4) << avg_loss + << " time=" << std::setprecision(1) << ep_sec << "s"; + + // ── 测试集评估（可选，与训练一致的滑动窗口） ───────────── + if (!test_window_offsets.empty()) { - auto save_result = nn::save_model(cfg.save_path, model, spec); - if (!save_result) { - std::cerr << "Error: " << save_result.error().message << '\n'; - return 1; + Scalar test_total_weighted = 0.0; + std::size_t test_total_valid = 0; + const std::size_t test_bs = std::min(cfg.batch_size, test_window_offsets.size()); + const std::size_t test_steps_per_epoch = + (test_window_offsets.size() + test_bs - 1) / test_bs; + + for (std::size_t tstep = 0; tstep < test_steps_per_epoch; ++tstep) + { + const std::size_t toffset = tstep * test_bs; + const std::size_t this_bs = std::min(test_bs, test_window_offsets.size() - toffset); + if (x_tokens.cols() != this_bs) + { + x_tokens.resize(cfg.seq_len, this_bs); + y_tokens.resize(cfg.seq_len, this_bs); + loss_mask.resize(cfg.seq_len, this_bs); + flat_targets.resize(cfg.seq_len * this_bs); + } + + // 填充 batch（与训练一致：末窗不足时 PAD 并屏蔽） + std::size_t test_valid = 0; + for (std::size_t b = 0; b < this_bs; ++b) + { + const std::size_t win_pos = test_window_offsets[toffset + b]; + const std::size_t win_len = std::min(cfg.seq_len, test_flow.size() - win_pos); + for (std::size_t t = 0; t < cfg.seq_len; ++t) + { + const std::size_t x_id = (t < win_len) ? test_flow[win_pos + t] : pad_id; + const std::size_t y_id = (t + 1 < win_len) ? test_flow[win_pos + t + 1] : pad_id; + x_tokens.set_value(t, b, static_cast(x_id)); + y_tokens.set_value(t, b, static_cast(y_id)); + const bool participate = (t + 1 < win_len); + loss_mask.set_value(t, b, participate ? 1.0f : 0.0f); + if (participate) ++test_valid; + } + } + + // Forward pass only（不 backward） + auto x_tensor_r = engine->from_matrix(x_tokens); + if (!x_tensor_r) { std::cerr << " 测试 from_matrix 失败: " << x_tensor_r.error().message << '\n'; break; } + + // 构建 flat_targets（与 logits 列序一致：t×batch + b） + auto y_span = y_tokens.span(); + for (std::size_t t = 0; t < cfg.seq_len; ++t) + for (std::size_t b = 0; b < this_bs; ++b) + flat_targets[t * this_bs + b] = + static_cast(y_span[t * this_bs + b]); + + auto begin_r = engine->begin_batch(); + if (!begin_r) { std::cerr << " 测试 begin_batch 失败: " << begin_r.error().message << '\n'; break; } + + auto fwd_result = model.forward(*x_tensor_r); + if (!fwd_result) { std::cerr << " 测试前向传播出错: " << fwd_result.error().message << '\n'; break; } + auto test_mask_span = std::span(loss_mask.span()); + auto loss_result = ce_loss.forward_sparse( + *engine, *fwd_result, flat_targets, test_mask_span, tokenizer->vocab_size()); + + auto end_r = engine->end_batch(); + if (!end_r) { std::cerr << " 测试 end_batch 失败: " << end_r.error().message << '\n'; break; } + + if (!loss_result) { std::cerr << " 测试评估出错: " << loss_result.error().message << '\n'; break; } + Scalar batch_loss = *loss_result; + + // 按有效 token 加权（与训练端 per-token 平均一致） + test_total_weighted += batch_loss * static_cast(test_valid); + test_total_valid += test_valid; } + + Scalar test_avg_loss = (test_total_valid > 0) + ? test_total_weighted / static_cast(test_total_valid) + : Scalar{0}; + std::cout << " test_loss=" << std::fixed << std::setprecision(4) << test_avg_loss; } - std::cout << "\n训练完成! 总耗时: " << std::fixed << std::setprecision(1) - << total_sec << "s" << std::endl; - return 0; + std::cout << std::endl; } - catch (const std::exception &e) + + auto t_end = std::chrono::steady_clock::now(); + Scalar total_sec = std::chrono::duration(t_end - t_start).count(); + + // ── 保存模型（含规格 + 嵌入 tokenizer） ────────────────── { - std::cerr << "\n错误: " << e.what() << std::endl; - return 1; + auto save_result = nn::save_model(cfg.save_path, model, spec, tokenizer_json); + if (!save_result) { + std::cerr << "Error: " << save_result.error().message << '\n'; + return 1; + } } + std::cout << "\n训练完成! 总耗时: " << std::fixed << std::setprecision(1) + << total_sec << "s" + << " 词表已嵌入模型文件" << std::endl; + + return 0; } diff --git a/src/tokenizer_infer.cpp b/src/tokenizer_infer.cpp index b99d8a51..0f720398 100644 --- a/src/tokenizer_infer.cpp +++ b/src/tokenizer_infer.cpp @@ -1,371 +1,245 @@ -/** - * ByteZip 分词器推理程序 - * - * 用法: - * tokenizer_infer --model --text 编码+解码 - * tokenizer_infer --model --encode 仅编码 - * tokenizer_infer --model --decode 仅解码 - * tokenizer_infer --model --benchmark 性能测试 - * tokenizer_infer --model --interactive 交互模式 - */ - -#include - -#include -#include +// ── BPE 分词器推理程序（编码/解码） ────────────────────────────────────── +// +// 加载词表 JSON → 编码文本为 token IDs / 解码 token IDs 为文本 +// +// 自动根据 JSON 的 "type" 字段识别分词器类型： +// - bpe_tokenizer → 字节级 BPE（BBPE） +// - char_bpe_tokenizer → 字符级 BPE（支持中文） +// - wordzip_tokenizer → WordZip 词级分词器 +// - space_tokenizer → 空白分词器 +// +// 支持交互模式和命令行模式。 +// ───────────────────────────────────────────────────────────────────────── + +#include + +#include #include #include #include #include #include -// ── 帮助信息 ──────────────────────────────────────────────────────────── +// ==================== 帮助信息 ==================== void print_usage(const char *prog) { std::cout - << "ByteZip 分词器推理程序\n\n" + << "BPE 分词器推理程序 (编码/解码)\n\n" << "用法:\n" - << " " << prog << " --model --text 编码并解码\n" - << " " << prog << " --model --encode 仅编码\n" - << " " << prog << " --model --decode 仅解码\n" - << " " << prog << " --model --benchmark 性能测试\n" - << " " << prog << " --model --interactive 交互模式\n\n" + << " " << prog << " --vocab --encode \"text\" 编码文本\n" + << " " << prog << " --vocab --decode \"ids\" 解码 token IDs\n" + << " " << prog << " --vocab --interactive 交互模式\n" + << " " << prog << " --vocab --encode-file 编码文件\n\n" << "选项:\n" - << " --model 词表 JSON 路径 (必需)\n" - << " --text 输入文本（编码+解码并验证）\n" - << " --encode 仅编码\n" - << " --decode 仅解码（逗号分隔的 ID 列表）\n" - << " --benchmark 对文件进行编码性能测试\n" - << " --interactive 交互式编码/解码\n" - << " --show-tokens 显示每个 token 的详细信息\n" + << " --vocab 词表 JSON 路径 (默认: bpe_vocab.json)\n" + << " 自动识别分词器类型（bpe / charbpe / wordzip / space）\n" + << " --encode 编码文本为 token IDs\n" + << " --decode 解码 token IDs (逗号分隔) 为文本\n" + << " --encode-file 编码整个文件\n" + << " --interactive 交互模式 (输入 'quit' 退出)\n" + << " --show-bytes 显示原始字节 (调试用)\n" << " --help 显示此帮助信息\n"; } -// ── 命令行参数 ────────────────────────────────────────────────────────── -enum class Mode +// ==================== 命令行参数 ==================== +struct Config { - Text, // 编码 + 解码验证 - Encode, // 仅编码 - Decode, // 仅解码 - Benchmark, // 性能测试 - Interactive // 交互模式 + std::string vocab_path = "bpe_vocab.json"; + std::string encode_text; + std::string decode_ids; + std::string encode_file; + bool interactive = false; + bool show_bytes = false; }; -struct InferArgs +Config parse_args(int argc, char *argv[]) { - std::string model_path; - std::string input; - Mode mode = Mode::Text; - bool show_tokens = false; -}; - -InferArgs parse_args(int argc, char *argv[]) -{ - InferArgs args; - - if (argc < 2) - { - print_usage(argv[0]); - std::exit(1); - } - + Config cfg; for (int i = 1; i < argc; ++i) { std::string arg = argv[i]; - if (arg == "--help") { print_usage(argv[0]); std::exit(0); } - else if (arg == "--model" && i + 1 < argc) - { - args.model_path = argv[++i]; - } - else if (arg == "--text" && i + 1 < argc) - { - args.input = argv[++i]; - args.mode = Mode::Text; - } + else if (arg == "--vocab" && i + 1 < argc) + cfg.vocab_path = argv[++i]; else if (arg == "--encode" && i + 1 < argc) - { - args.input = argv[++i]; - args.mode = Mode::Encode; - } + cfg.encode_text = argv[++i]; else if (arg == "--decode" && i + 1 < argc) - { - args.input = argv[++i]; - args.mode = Mode::Decode; - } - else if (arg == "--benchmark" && i + 1 < argc) - { - args.input = argv[++i]; - args.mode = Mode::Benchmark; - } + cfg.decode_ids = argv[++i]; + else if (arg == "--encode-file" && i + 1 < argc) + cfg.encode_file = argv[++i]; else if (arg == "--interactive") - { - args.mode = Mode::Interactive; - } - else if (arg == "--show-tokens") - { - args.show_tokens = true; - } + cfg.interactive = true; + else if (arg == "--show-bytes") + cfg.show_bytes = true; else { std::cerr << "未知参数: " << arg << "\n使用 --help 查看用法\n"; std::exit(1); } } - - if (args.model_path.empty()) - { - std::cerr << "错误：必须指定 --model 路径\n"; - print_usage(argv[0]); - std::exit(1); - } - - return args; + return cfg; } -// ── 解析逗号分隔的 ID 列表 ───────────────────────────────────────────── +// ==================== 解析 token IDs 字符串 ==================== std::vector parse_ids(const std::string &s) { std::vector ids; - std::istringstream iss(s); + std::stringstream ss(s); std::string token; - while (std::getline(iss, token, ',')) + while (std::getline(ss, token, ',')) { // 去除空白 - auto start = token.find_first_not_of(" \t"); - if (start == std::string::npos) - continue; - ids.push_back(static_cast(std::stoul(token.substr(start)))); + while (!token.empty() && (token.front() == ' ' || token.front() == '[')) + token.erase(0, 1); + while (!token.empty() && (token.back() == ' ' || token.back() == ']')) + token.pop_back(); + if (token.empty()) continue; + auto v = nn::parse_number(token); + if (v) ids.push_back(*v); } return ids; } -// ── 读取文本文件 ──────────────────────────────────────────────────────── -std::string read_text_file(const std::string &path) +// ==================== 统一输出 token IDs ==================== +// 辅助函数：以 "Tokens (): [id0, id1, ...]" 格式输出 ID 列表， +// 消除交互模式 / 命令行模式 / 默认编码路径中的重复打印代码。 +void print_token_ids(const std::vector &ids, std::ostream &os = std::cout) { - std::ifstream ifs(path, std::ios::binary); - if (!ifs) + os << " Tokens (" << ids.size() << "): ["; + for (std::size_t i = 0; i < ids.size(); ++i) { - std::cerr << "错误：无法打开文件 " << path << "\n"; - std::exit(1); + if (i > 0) os << ", "; + os << ids[i]; } - ifs.seekg(0, std::ios::end); - const auto size = ifs.tellg(); - ifs.seekg(0, std::ios::beg); - std::string content(static_cast(size), '\0'); - ifs.read(content.data(), size); - return content; + os << "]\n"; } -// ── 展示编码结果 ──────────────────────────────────────────────────────── -void show_encoding(const nn::ByteZipTokenizer &tokenizer, - const std::string &text, - const std::vector &ids, - bool show_detail) +// ==================== 交互模式 ==================== +void interactive_mode(const nn::Tokenizer &tokenizer, bool show_bytes) { - std::cout << " 原文: " << text << "\n"; - std::cout << " Token数: " << ids.size() << "\n"; + std::cout << "Tokenizer 交互模式 (输入 'quit' 退出)\n"; + std::cout << " encode → 编码文本\n"; + std::cout << " decode → 解码 token IDs (逗号分隔)\n\n"; - if (show_detail) + while (true) { - std::cout << " Token序列: ["; - for (std::size_t i = 0; i < ids.size(); ++i) + std::cout << ">>> "; + std::string line; + if (!std::getline(std::cin, line)) + break; + if (line == "quit" || line == "exit") + break; + if (line.empty()) + continue; + + if (line.starts_with("encode ")) { - if (i > 0) - std::cout << ", "; - std::cout << ids[i]; + std::string text = line.substr(7); + auto ids = tokenizer.encode(text); + print_token_ids(ids); + if (show_bytes) + { + const auto &v = tokenizer.vocab(); + std::cout << " Bytes: "; + for (auto id : ids) + { + if (id < v.size()) + { + std::cout << "{"; + for (unsigned char b : v[id]) + std::cout << std::hex << static_cast(b) << " "; + std::cout << std::dec << "}"; + } + } + std::cout << "\n"; + } } - std::cout << "]\n"; - - std::cout << " Token详情:\n"; - for (std::size_t i = 0; i < ids.size(); ++i) + else if (line.starts_with("decode ")) { - std::cout << " [" << ids[i] << "] \"" - << tokenizer.try_decode_token(ids[i]) << "\"\n"; + auto ids = parse_ids(line.substr(7)); + auto text = tokenizer.decode(ids); + std::cout << " Text: \"" << text << "\"\n"; } - } -} - -// ── 模式：编码+解码验证 ───────────────────────────────────────────────── -void mode_text(const nn::ByteZipTokenizer &tokenizer, - const std::string &text, bool show_detail) -{ - auto ids = tokenizer.encode(text); - auto decoded = tokenizer.decode(ids); - - show_encoding(tokenizer, text, ids, show_detail); - - std::cout << " 解码: " << decoded << "\n"; - std::cout << " 完美还原: " << (decoded == text ? "true" : "false") << "\n"; -} - -// ── 模式：仅编码 ──────────────────────────────────────────────────────── -void mode_encode(const nn::ByteZipTokenizer &tokenizer, - const std::string &text, bool show_detail) -{ - auto ids = tokenizer.encode(text); - show_encoding(tokenizer, text, ids, show_detail); - - std::cout << " IDs: ["; - for (std::size_t i = 0; i < ids.size(); ++i) - { - if (i > 0) - std::cout << ", "; - std::cout << ids[i]; - } - std::cout << "]\n"; -} - -// ── 模式：仅解码 ──────────────────────────────────────────────────────── -void mode_decode(const nn::ByteZipTokenizer &tokenizer, - const std::string &id_str, bool show_detail) -{ - auto ids = parse_ids(id_str); - - if (show_detail) - { - std::cout << " 输入 IDs: ["; - for (std::size_t i = 0; i < ids.size(); ++i) + else { - if (i > 0) - std::cout << ", "; - std::cout << ids[i]; + // 默认编码 + auto ids = tokenizer.encode(line); + print_token_ids(ids); + auto decoded = tokenizer.decode(ids); + std::cout << " Decode: \"" << decoded << "\"\n"; } - std::cout << "]\n"; } - - auto decoded = tokenizer.decode(ids); - std::cout << " 解码: " << decoded << "\n"; } -// ── 模式：性能测试 ────────────────────────────────────────────────────── -void mode_benchmark(const nn::ByteZipTokenizer &tokenizer, - const std::string &file_path, bool show_detail) +// ==================== 主函数 ==================== +int main(int argc, char *argv[]) { - std::cout << "读取文件: " << file_path << "\n"; - auto text = read_text_file(file_path); - std::cout << "文件大小: " << text.size() << " 字节\n"; + Config cfg = parse_args(argc, argv); - // 预热 - std::cout << "预热中...\n"; + // ── 加载词表（自动识别分词器类型） ───────────────────── + auto tokenizer = nn::load_tokenizer_from_file(cfg.vocab_path); + if (!tokenizer) { - auto warmup = text.substr(0, std::min(text.size(), static_cast(1000))); - (void)tokenizer.encode(warmup); + std::cerr << "加载词表失败或无法识别分词器类型: " << cfg.vocab_path << '\n' + << "请检查 JSON 文件是否包含有效的 \"type\" 字段" << std::endl; + return 1; } + std::cout << "词表已加载: " << tokenizer->vocab_size() << " 词" << std::endl; - // 正式测试 - const int rounds = 5; - std::cout << "编码测试 (" << rounds << " 轮)...\n"; - - std::vector ids; - auto t0 = std::chrono::steady_clock::now(); - for (int i = 0; i < rounds; ++i) + // ── 交互模式 ───────────────────────────────────────────── + if (cfg.interactive) { - ids = tokenizer.encode(text); + interactive_mode(*tokenizer, cfg.show_bytes); + return 0; } - auto t1 = std::chrono::steady_clock::now(); - - const auto elapsed = std::chrono::duration(t1 - t0).count(); - const auto avg_ms = elapsed / rounds * 1000.0; - const auto throughput = static_cast(text.size() * rounds) / elapsed / (1024.0 * 1024.0); - - std::cout << std::fixed << std::setprecision(2); - std::cout << "\n📊 性能结果:\n"; - std::cout << " 总耗时: " << elapsed << " 秒\n"; - std::cout << " 平均: " << avg_ms << " 毫秒/轮\n"; - std::cout << " 吞吐量: " << throughput << " MB/s\n"; - std::cout << " Token数: " << ids.size() << "\n"; - std::cout << " 压缩率: " << std::setprecision(1) - << (static_cast(text.size()) / static_cast(ids.size())) - << " 字节/token\n"; - if (show_detail) + // ── 编码文本 ───────────────────────────────────────────── + if (!cfg.encode_text.empty()) { - // 解码验证 - auto t2 = std::chrono::steady_clock::now(); - auto decoded = tokenizer.decode(ids); - auto t3 = std::chrono::steady_clock::now(); - - std::cout << " 解码耗时: " << std::setprecision(2) - << std::chrono::duration(t3 - t2).count() * 1000.0 - << " 毫秒\n"; - std::cout << " 完美还原: " << (decoded == text ? "true" : "false") << "\n"; + auto ids = tokenizer->encode(cfg.encode_text); + print_token_ids(ids); + auto decoded = tokenizer->decode(ids); + std::cout << "Decode: \"" << decoded << "\"\n"; + return 0; } -} - -// ── 模式：交互 ────────────────────────────────────────────────────────── -void mode_interactive(const nn::ByteZipTokenizer &tokenizer, bool show_detail) -{ - std::cout << "ByteZip 分词器交互模式（输入 q 退出）\n\n"; - std::string line; - while (true) + // ── 解码 IDs ───────────────────────────────────────────── + if (!cfg.decode_ids.empty()) { - std::cout << "> "; - if (!std::getline(std::cin, line)) - break; - if (line == "q" || line == "quit" || line == "exit") - break; - if (line.empty()) - continue; - - auto ids = tokenizer.encode(line); - auto decoded = tokenizer.decode(ids); - - std::cout << " Token数: " << ids.size() << "\n"; - if (show_detail) - { - std::cout << " IDs: ["; - for (std::size_t i = 0; i < ids.size(); ++i) - { - if (i > 0) - std::cout << ", "; - std::cout << ids[i]; - } - std::cout << "]\n"; - } - std::cout << " 解码: " << decoded << "\n"; - std::cout << " 还原: " << (decoded == line ? "✓" : "✗") << "\n\n"; - } -} - -// ── 主函数 ────────────────────────────────────────────────────────────── -int main(int argc, char *argv[]) -{ - auto args = parse_args(argc, argv); - - // 加载词表 - nn::ByteZipTokenizer tokenizer; - std::cout << "加载词表: " << args.model_path << "\n"; - if (auto load_result = tokenizer.load(args.model_path); !load_result) - { - std::cerr << "加载失败: " << load_result.error().message << '\n'; - return 1; + auto ids = parse_ids(cfg.decode_ids); + auto text = tokenizer->decode(ids); + std::cout << "Text: \"" << text << "\"\n"; + return 0; } - std::cout << "词表大小: " << tokenizer.vocab_size() << " tokens\n\n"; - switch (args.mode) + // ── 编码文件 ───────────────────────────────────────────── + if (!cfg.encode_file.empty()) { - case Mode::Text: - mode_text(tokenizer, args.input, args.show_tokens); - break; - case Mode::Encode: - mode_encode(tokenizer, args.input, args.show_tokens); - break; - case Mode::Decode: - mode_decode(tokenizer, args.input, args.show_tokens); - break; - case Mode::Benchmark: - mode_benchmark(tokenizer, args.input, args.show_tokens); - break; - case Mode::Interactive: - mode_interactive(tokenizer, args.show_tokens); - break; + auto text_result = nn::load_text_file(cfg.encode_file); + if (!text_result) + { + std::cerr << "读取文件失败: " << text_result.error().message << '\n'; + return 1; + } + auto ids = tokenizer->encode(*text_result); + std::cout << "文件大小: " << text_result->size() << " 字节\n"; + std::cout << "Token 数: " << ids.size() << "\n"; + if (ids.empty()) + { + std::cout << "压缩率: N/A (空输入)\n"; + return 0; + } + std::cout << "压缩率: " << std::fixed << std::setprecision(2) + << static_cast(text_result->size()) / ids.size() + << " bytes/token\n"; + return 0; } + // 无参数 → 交互模式 + interactive_mode(*tokenizer, cfg.show_bytes); return 0; } diff --git a/src/tokenizer_train.cpp b/src/tokenizer_train.cpp index 2a766cc6..010f0bde 100644 --- a/src/tokenizer_train.cpp +++ b/src/tokenizer_train.cpp @@ -1,204 +1,219 @@ -/** - * ByteZip 分词器训练程序 - * - * 用法: - * tokenizer_train [选项] - * - * 示例: - * tokenizer_train dataset.txt --vocab-size 10000 --output tokenizer.json - */ - -#include +// ── BPE 分词器训练程序 ────────────────────────────────────────────────── +// +// 读取文本文件 → 训练分词器 → 保存词表 JSON +// 词表 JSON 可被 text_train / text_infer 加载。 +// +// 支持两种 BPE： +// --tokenizer bpe 字节级 BPE（BBPE），天然兼容 UTF-8，但中文按字节切分 +// --tokenizer charbpe 字符级 BPE（纯 BPE），中文每个汉字为 1 token， +// BPE 能学会合并常见中文词组（更适合小模型） +// ───────────────────────────────────────────────────────────────────────── + +#include #include -#include -#include #include #include +#include #include -// ── 帮助信息 ──────────────────────────────────────────────────────────── +using nn::Scalar; + +// ==================== 帮助信息 ==================== void print_usage(const char *prog) { std::cout - << "ByteZip v2.2 分词器训练程序\n\n" + << "BPE 分词器训练程序\n\n" << "用法: " << prog << " [选项]\n\n" << "参数:\n" - << " 训练文本文件路径 (必需)\n\n" + << " 训练文本文件路径 (UTF-8 编码)\n\n" << "选项:\n" - << " --vocab-size 目标词表大小 (默认: 20000)\n" - << " --output 输出 JSON 路径 (默认: tokenizer.json)\n" - << " --v1-len V1 最大子词长度 (默认: 16)\n" - << " --v2-len V2 最大子词长度 (默认: 24)\n" - << " --v2-reserve V2 预留槽位 (默认: 0=自动)\n" - << " --skip-ratio 独立度阈值 α (默认: 1.2)\n" - << " --affix-ratio 词缀保护阈值 (默认: 0.4)\n" - << " --help 显示此帮助信息\n"; + << " --tokenizer 分词器类型: bpe (字节级 BBPE, 默认) | charbpe (字符级, 支持中文)\n" + << " --output 词表输出路径 (默认: bpe_vocab.json)\n" + << " --vocab-size 目标词表大小 (默认: 5000)\n" + << " --min-freq 最小合并频率 (默认: 2)\n" + << " --help 显示此帮助信息\n\n" + << "说明:\n" + << " bpe 字节级 BPE（BBPE）：兼容性最好，任意字节均可编码；\n" + << " 但中文按 3 字节切分，小模型难学到词义。\n" + << " charbpe 字符级 BPE：每个汉字为 1 个基础 token，BPE 学会合并常见词组；\n" + << " 中文场景推荐使用。\n"; } -// ── 命令行参数 ────────────────────────────────────────────────────────── -struct TrainArgs +// ==================== 命令行参数 ==================== +struct Config { - std::string text_file; - std::string output = "tokenizer.json"; - std::size_t vocab_size = nn::ByteZipTokenizer::DEFAULT_VOCAB_SIZE; - std::size_t v1_len = nn::ByteZipTokenizer::DEFAULT_V1_MAX_LEN; - std::size_t v2_len = nn::ByteZipTokenizer::DEFAULT_V2_MAX_LEN; - std::size_t v2_reserve = 0; - double skip_ratio = nn::ByteZipTokenizer::DEFAULT_SKIP_RATIO; - double affix_ratio = nn::ByteZipTokenizer::DEFAULT_AFFIX_RATIO; + std::string text_path; + std::string output_path = "bpe_vocab.json"; + std::string tokenizer_type = "bpe"; // "bpe" 或 "charbpe" + std::size_t vocab_size = nn::BPETokenizer::DEFAULT_VOCAB_SIZE; + std::uint32_t min_freq = nn::BPETokenizer::DEFAULT_MIN_FREQ; }; -TrainArgs parse_args(int argc, char *argv[]) +Config parse_args(int argc, char *argv[]) { - TrainArgs args; - - if (argc < 2) - { - print_usage(argv[0]); - std::exit(1); - } - + Config cfg; for (int i = 1; i < argc; ++i) { std::string arg = argv[i]; - if (arg == "--help") { print_usage(argv[0]); std::exit(0); } - else if (arg == "--vocab-size" && i + 1 < argc) - args.vocab_size = static_cast(std::stoul(argv[++i])); + else if (arg == "--tokenizer" && i + 1 < argc) + { + cfg.tokenizer_type = argv[++i]; + if (cfg.tokenizer_type != "bpe" && cfg.tokenizer_type != "charbpe") + { + std::cerr << "未知分词器类型: " << cfg.tokenizer_type + << "，可选: bpe, charbpe\n"; + std::exit(1); + } + } else if (arg == "--output" && i + 1 < argc) - args.output = argv[++i]; - else if (arg == "--v1-len" && i + 1 < argc) - args.v1_len = static_cast(std::stoul(argv[++i])); - else if (arg == "--v2-len" && i + 1 < argc) - args.v2_len = static_cast(std::stoul(argv[++i])); - else if (arg == "--v2-reserve" && i + 1 < argc) - args.v2_reserve = static_cast(std::stoul(argv[++i])); - else if (arg == "--skip-ratio" && i + 1 < argc) - args.skip_ratio = std::stod(argv[++i]); - else if (arg == "--affix-ratio" && i + 1 < argc) - args.affix_ratio = std::stod(argv[++i]); + cfg.output_path = argv[++i]; + else if (arg == "--vocab-size" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --vocab-size: " << v.error().message << "\n"; std::exit(1); } + cfg.vocab_size = *v; + } + else if (arg == "--min-freq" && i + 1 < argc) + { + auto v = nn::parse_number(argv[++i]); + if (!v) { std::cerr << "无效 --min-freq: " << v.error().message << "\n"; std::exit(1); } + cfg.min_freq = *v; + } else if (!arg.starts_with("--")) - args.text_file = arg; + cfg.text_path = arg; else { std::cerr << "未知参数: " << arg << "\n使用 --help 查看用法\n"; std::exit(1); } } - - if (args.text_file.empty()) + if (cfg.text_path.empty()) { - std::cerr << "错误：必须指定训练文本文件\n"; - print_usage(argv[0]); + std::cerr << "请指定训练文本文件\n使用 --help 查看用法\n"; std::exit(1); } - - return args; + return cfg; } -// ── 读取文本文件（UTF-8） ────────────────────────────────────────────── -std::string read_text_file(const std::string &path) +// ==================== 主函数 ==================== +int main(int argc, char *argv[]) { - std::ifstream ifs(path, std::ios::binary); - if (!ifs) + Config cfg = parse_args(argc, argv); + + // ── 加载文本 ───────────────────────────────────────────── + std::cout << "加载文本: " << cfg.text_path << " ..." << std::endl; + auto text_result = nn::load_text_file(cfg.text_path); + if (!text_result) { + std::cerr << "Error: " << text_result.error().message << '\n'; + return 1; + } + std::string text = std::move(*text_result); + if (text.empty()) { - std::cerr << "错误：无法打开文件 " << path << "\n"; - std::exit(1); + std::cerr << "文本文件为空\n"; + return 1; } + std::cout << "文本大小: " << text.size() << " 字节" << std::endl; - // 获取文件大小 - ifs.seekg(0, std::ios::end); - const auto size = ifs.tellg(); - ifs.seekg(0, std::ios::beg); - - std::string content(static_cast(size), '\0'); - ifs.read(content.data(), size); - - return content; -} - -// ── 主函数 ────────────────────────────────────────────────────────────── -int main(int argc, char *argv[]) -{ - auto args = parse_args(argc, argv); - - // 读取文件 - std::cout << "读取文件: " << args.text_file << "\n"; - auto text = read_text_file(args.text_file); - std::cout << "文本字符数: " << text.size() << "\n"; - - // 训练 - nn::ByteZipTokenizer tokenizer; - nn::ByteZipTokenizer::Config config; - config.vocab_size = args.vocab_size; - config.v1_max_len = args.v1_len; - config.v2_max_len = args.v2_len; - config.v2_reserve = args.v2_reserve; - config.skip_ratio = args.skip_ratio; - config.affix_protect_ratio = args.affix_ratio; - - auto t0 = std::chrono::steady_clock::now(); - tokenizer.train(text, config); - auto t1 = std::chrono::steady_clock::now(); - - const auto elapsed = std::chrono::duration(t1 - t0).count(); - std::cout << "总耗时: " << std::fixed << std::setprecision(1) - << elapsed << " 秒\n"; - - // 展示前 20 个高频子词 - std::cout << "\n✅ 高频子词展示（前 20 个）:\n"; - std::size_t shown = 0; - const auto &vocab = tokenizer.vocab(); - for (std::size_t tid = nn::ByteZipTokenizer::BYTE_OFFSET; - tid < vocab.size() && shown < 20; ++tid) + // ── 创建分词器实例（根据 --tokenizer 选择类型） ──────────── + std::unique_ptr tokenizer; + if (cfg.tokenizer_type == "charbpe") { - const auto &bytes = vocab[tid]; - if (bytes.size() >= 2) - { - // 尝试作为 UTF-8 解码显示 - bool printable = true; - for (unsigned char c : bytes) - { - if (c < 32 && c != '\n' && c != '\t') - { - printable = false; - break; - } - } + tokenizer = std::make_unique(); + std::cout << "使用字符级 BPE (CharBPE)，支持中文分词" << std::endl; + } + else + { + tokenizer = std::make_unique(); + std::cout << "使用字节级 BPE (BBPE)" << std::endl; + } - if (printable) - { - std::cout << " ID " << tid << ": \"" - << tokenizer.try_decode_token(tid) << "\"\n"; - ++shown; - } - } + // ── 训练 ────────────────────────────────────────────────── + std::cout << "========================================\n"; + std::cout << " " << (cfg.tokenizer_type == "charbpe" ? "CharBPE" : "BPE") + << " 分词器训练\n"; + std::cout << "========================================\n"; + std::cout << " 目标词表: " << cfg.vocab_size << "\n"; + std::cout << " 最小频率: " << cfg.min_freq << "\n"; + std::cout << "========================================\n" << std::endl; + + // 根据分词器类型调用对应训练接口（BPE 和 CharBPE 接口签名一致） + // 使用 static_cast 向下转型以调用各子类特有的 train(text, Config) 重载， + // 避免在基类 Tokenizer 中暴露子类专属的 Config 参数。 + auto t_start = std::chrono::steady_clock::now(); + nn::Result train_result; + if (cfg.tokenizer_type == "charbpe") + { + auto& tok = static_cast(*tokenizer); + nn::CharBPETokenizer::Config tcfg; + tcfg.vocab_size = cfg.vocab_size; + tcfg.min_freq = cfg.min_freq; + tcfg.log = [](std::string_view msg) { std::cout << msg << '\n'; }; + train_result = tok.train(text, tcfg); + } + else + { + auto& tok = static_cast(*tokenizer); + nn::BPETokenizer::Config tcfg; + tcfg.vocab_size = cfg.vocab_size; + tcfg.min_freq = cfg.min_freq; + tcfg.log = [](std::string_view msg) { std::cout << msg << '\n'; }; + train_result = tok.train(text, tcfg); } - // 编码/解码冒烟测试 - const std::string test_sent = "Alice was a very good girl, she said hello!"; - auto ids = tokenizer.encode(test_sent); - auto decoded = tokenizer.decode(ids); + if (!train_result) + { + std::cerr << "训练失败: " << train_result.error().message << '\n'; + return 1; + } + auto t_end = std::chrono::steady_clock::now(); + Scalar train_sec = std::chrono::duration(t_end - t_start).count(); - std::cout << "\n🧪 验证:\n"; - std::cout << " 原文: " << test_sent << "\n"; - std::cout << " Token数: " << ids.size() << "\n"; - std::cout << " 解码: " << decoded << "\n"; - std::cout << " 完美还原: " << (decoded == test_sent ? "true" : "false") << "\n"; + std::cout << "\n训练完成! 耗时: " << std::fixed << std::setprecision(1) + << train_sec << "s" << std::endl; + std::cout << "词表大小: " << tokenizer->vocab_size() << std::endl; - // 保存词表 - if (auto save_result = tokenizer.save(args.output); !save_result) + // ── 保存词表 ───────────────────────────────────────────── + auto save_result = tokenizer->save(cfg.output_path); + if (!save_result) { - std::cerr << "保存失败: " << save_result.error().message << '\n'; + std::cerr << "保存词表失败: " << save_result.error().message << '\n'; return 1; } - std::cout << "词表已保存至: " << args.output - << " (" << tokenizer.vocab_size() << " tokens)\n"; + std::cout << "词表已保存: " << cfg.output_path << std::endl; + + // ── 打印示例（验证分词效果） ───────────────────────────── + { + // 取文本第一行作为示例（避免截断 UTF-8 序列） + std::string sample; + { + std::size_t nl = text.find('\n'); + sample = nl == std::string::npos ? text : text.substr(0, nl); + } + auto ids = tokenizer->encode(sample); + std::cout << "\n示例编码 (" << sample.size() << " 字节 → " + << ids.size() << " tokens, 压缩率 " + << std::fixed << std::setprecision(2) + << static_cast(sample.size()) / std::max(ids.size(), 1) + << " bytes/token):\n"; + std::cout << " 原文: \"" << sample << "\"\n"; + std::cout << " Tokens: ["; + for (std::size_t i = 0; i < ids.size() && i < 30; ++i) + { + if (i > 0) std::cout << ", "; + std::cout << ids[i]; + } + if (ids.size() > 30) std::cout << ", ..."; + std::cout << "]\n"; + auto decoded = tokenizer->decode(ids); + std::cout << " 解码: \"" << decoded << "\"\n"; + } return 0; } diff --git a/stories.json b/stories.json new file mode 100644 index 00000000..f9d07ffa --- /dev/null +++ b/stories.json @@ -0,0 +1,16140 @@ +{ + "type": "bpe_tokenizer", + "vocab": { + "0": "00", + "1": "01", + "2": "02", + "3": "03", + "4": "04", + "5": "05", + "6": "06", + "7": "07", + "8": "08", + "9": "09", + "10": "0a", + "11": "0b", + "12": "0c", + "13": "0d", + "14": "0e", + "15": "0f", + "16": "10", + "17": "11", + "18": "12", + "19": "13", + "20": "14", + "21": "15", + "22": "16", + "23": "17", + "24": "18", + "25": "19", + "26": "1a", + "27": "1b", + "28": "1c", + "29": "1d", + "30": "1e", + "31": "1f", + "32": "20", + "33": "21", + "34": "22", + "35": "23", + "36": "24", + "37": "25", + "38": "26", + "39": "27", + "40": "28", + "41": "29", + "42": "2a", + "43": "2b", + "44": "2c", + "45": "2d", + "46": "2e", + "47": "2f", + "48": "30", + "49": "31", + "50": "32", + "51": "33", + "52": "34", + "53": "35", + "54": "36", + "55": "37", + "56": "38", + "57": "39", + "58": "3a", + "59": "3b", + "60": "3c", + "61": "3d", + "62": "3e", + "63": "3f", + "64": "40", + "65": "41", + "66": "42", + "67": "43", + "68": "44", + "69": "45", + "70": "46", + "71": "47", + "72": "48", + "73": "49", + "74": "4a", + "75": "4b", + "76": "4c", + "77": "4d", + "78": "4e", + "79": "4f", + "80": "50", + "81": "51", + "82": "52", + "83": "53", + "84": "54", + "85": "55", + "86": "56", + "87": "57", + "88": "58", + "89": "59", + "90": "5a", + "91": "5b", + "92": "5c", + "93": "5d", + "94": "5e", + "95": "5f", + "96": "60", + "97": "61", + "98": "62", + "99": "63", + "100": "64", + "101": "65", + "102": "66", + "103": "67", + "104": "68", + "105": "69", + "106": "6a", + "107": "6b", + "108": "6c", + "109": "6d", + "110": "6e", + "111": "6f", + "112": "70", + "113": "71", + "114": "72", + "115": "73", + "116": "74", + "117": "75", + "118": "76", + "119": "77", + "120": "78", + "121": "79", + "122": "7a", + "123": "7b", + "124": "7c", + "125": "7d", + "126": "7e", + "127": "7f", + "128": "80", + "129": "81", + "130": "82", + "131": "83", + "132": "84", + "133": "85", + "134": "86", + "135": "87", + "136": "88", + "137": "89", + "138": "8a", + "139": "8b", + "140": "8c", + "141": "8d", + "142": "8e", + "143": "8f", + "144": "90", + "145": "91", + "146": "92", + "147": "93", + "148": "94", + "149": "95", + "150": "96", + "151": "97", + "152": "98", + "153": "99", + "154": "9a", + "155": "9b", + "156": "9c", + "157": "9d", + "158": "9e", + "159": "9f", + "160": "a0", + "161": "a1", + "162": "a2", + "163": "a3", + "164": "a4", + "165": "a5", + "166": "a6", + "167": "a7", + "168": "a8", + "169": "a9", + "170": "aa", + "171": "ab", + "172": "ac", + "173": "ad", + "174": "ae", + "175": "af", + "176": "b0", + "177": "b1", + "178": "b2", + "179": "b3", + "180": "b4", + "181": "b5", + "182": "b6", + "183": "b7", + "184": "b8", + "185": "b9", + "186": "ba", + "187": "bb", + "188": "bc", + "189": "bd", + "190": "be", + "191": "bf", + "192": "c0", + "193": "c1", + "194": "c2", + "195": "c3", + "196": "c4", + "197": "c5", + "198": "c6", + "199": "c7", + "200": "c8", + "201": "c9", + "202": "ca", + "203": "cb", + "204": "cc", + "205": "cd", + "206": "ce", + "207": "cf", + "208": "d0", + "209": "d1", + "210": "d2", + "211": "d3", + "212": "d4", + "213": "d5", + "214": "d6", + "215": "d7", + "216": "d8", + "217": "d9", + "218": "da", + "219": "db", + "220": "dc", + "221": "dd", + "222": "de", + "223": "df", + "224": "e0", + "225": "e1", + "226": "e2", + "227": "e3", + "228": "e4", + "229": "e5", + "230": "e6", + "231": "e7", + "232": "e8", + "233": "e9", + "234": "ea", + "235": "eb", + "236": "ec", + "237": "ed", + "238": "ee", + "239": "ef", + "240": "f0", + "241": "f1", + "242": "f2", + "243": "f3", + "244": "f4", + "245": "f5", + "246": "f6", + "247": "f7", + "248": "f8", + "249": "f9", + "250": "fa", + "251": "fb", + "252": "fc", + "253": "fd", + "254": "fe", + "255": "ff", + "256": "3c626f733e", + "257": "3c656f733e", + "258": "6865", + "259": "2074", + "260": "2061", + "261": "2073", + "262": "2077", + "263": "6e64", + "264": "20746865", + "265": "6564", + "266": "20616e64", + "267": "20746f", + "268": "2062", + "269": "696e", + "270": "2068", + "271": "7265", + "272": "207761", + "273": "6f75", + "274": "2066", + "275": "6974", + "276": "2054", + "277": "206c", + "278": "2063", + "279": "2064", + "280": "6572", + "281": "206865", + "282": "206d", + "283": "2070", + "284": "20776173", + "285": "6f6d", + "286": "206f", + "287": "6179", + "288": "20546865", + "289": "6172", + "290": "6973", + "291": "696e67", + "292": "696c", + "293": "2067", + "294": "6964", + "295": "2053", + "296": "6174", + "297": "656e", + "298": "206e", + "299": "207361", + "300": "206861", + "301": "616e", + "302": "696d", + "303": "6f72", + "304": "6f6e", + "305": "206974", + "306": "2048", + "307": "207468", + "308": "6c6c", + "309": "6c65", + "310": "6f74", + "311": "20536865", + "312": "20686572", + "313": "6574", + "314": "6972", + "315": "204865", + "316": "2054686579", + "317": "6573", + "318": "20696e", + "319": "766572", + "320": "7574", + "321": "6f77", + "322": "636b", + "323": "204c", + "324": "2065", + "325": "2079", + "326": "2075", + "327": "6c64", + "328": "6f6f", + "329": "696c79", + "330": "2073616964", + "331": "6967", + "332": "2072", + "333": "2049", + "334": "206265", + "335": "616d", + "336": "207374", + "337": "6b65", + "338": "6365", + "339": "7665", + "340": "697468", + "341": "20736865", + "342": "7070", + "343": "2077697468", + "344": "204c696c79", + "345": "4f6e", + "346": "206f6e", + "347": "20796f75", + "348": "6b6564", + "349": "7269", + "350": "206f66", + "351": "20706c", + "352": "6e74", + "353": "20686973", + "354": "2042", + "355": "6164", + "356": "20736f", + "357": "76657279", + "358": "20646179", + "359": "2074686174", + "360": "207570", + "361": "20706c6179", + "362": "2074686579", + "363": "20686164", + "364": "7374", + "365": "207765", + "366": "206d6f6d", + "367": "20666f72", + "368": "204d", + "369": "656c", + "370": "756e", + "371": "6f756c64", + "372": "2773", + "373": "6d79", + "374": "206c69", + "375": "0d0a", + "376": "6368", + "377": "2068617070", + "378": "2077616e74", + "379": "656e74", + "380": "207368", + "381": "697474", + "382": "686572", + "383": "6f7574", + "384": "206e6f74", + "385": "20646f", + "386": "6f6d65", + "387": "7365", + "388": "2076657279", + "389": "656e64", + "390": "6c79", + "391": "616c6c", + "392": "6f756e64", + "393": "207768", + "394": "206b", + "395": "696d65", + "396": "6974746c65", + "397": "2074696d65", + "398": "206c6974746c65", + "399": "616c", + "400": "206e65", + "401": "207468657265", + "402": "206973", + "403": "6874", + "404": "204974", + "405": "206861707079", + "406": "20626967", + "407": "2041", + "408": "207265", + "409": "20736d", + "410": "20627574", + "411": "20736177", + "412": "2e22", + "413": "20616e", + "414": "7279", + "415": "7261", + "416": "7269656e64", + "417": "20667269656e64", + "418": "206173", + "419": "61636b", + "420": "4f6e65", + "421": "696465", + "422": "616b65", + "423": "2774", + "424": "7567", + "425": "2054696d", + "426": "2122", + "427": "206c6f6f", + "428": "2077657265", + "429": "746572", + "430": "766564", + "431": "206c6f", + "432": "6f7265", + "433": "4f6e6365", + "434": "204f6e65", + "435": "207365", + "436": "20746f6f", + "437": "2068696d", + "438": "20626f", + "439": "20676f", + "440": "20617265", + "441": "6563", + "442": "204a", + "443": "206a", + "444": "2077616e746564", + "445": "617264", + "446": "696c6c", + "447": "69726c", + "448": "207468656d", + "449": "2075706f6e", + "450": "207468656972", + "451": "206769726c", + "452": "206f7574", + "453": "2042656e", + "454": "206174", + "455": "20646964", + "456": "7572", + "457": "696e64", + "458": "776179", + "459": "6675", + "460": "20546f6d", + "461": "207370", + "462": "20636f756c64", + "463": "20736d696c", + "464": "2068617665", + "465": "68656e", + "466": "61696e", + "467": "686564", + "468": "617274", + "469": "726f6d", + "470": "2063616e", + "471": "2077656e74", + "472": "206578", + "473": "6963", + "474": "2057", + "475": "206172", + "476": "6c70", + "477": "3f22", + "478": "6f6f64", + "479": "20427574", + "480": "66756c", + "481": "206e616d", + "482": "20667269656e6473", + "483": "206b6e", + "484": "2068656c70", + "485": "68696e67", + "486": "20736179", + "487": "69676874", + "488": "20616c6c", + "489": "596f75", + "490": "756d", + "491": "61726b", + "492": "6f6e65", + "493": "206261636b", + "494": "2066756e", + "495": "20636c", + "496": "617265", + "497": "206e6f", + "498": "206c65", + "499": "2054696d6d79", + "500": "6f70", + "501": "206e616d6564", + "502": "207363", + "503": "20616c", + "504": "656c74", + "505": "206c6f766564", + "506": "20746f79", + "507": "6f7567", + "508": "206d616e", + "509": "73696465", + "510": "20736d696c6564", + "511": "206c696b65", + "512": "69636b", + "513": "2066656c74", + "514": "2061736b6564", + "515": "206665", + "516": "20736565", + "517": "206d65", + "518": "20736f6d65", + "519": "206c6f6f6b6564", + "520": "616d65", + "521": "20726f", + "522": "2061726f756e64", + "523": "6f6d6574", + "524": "206272", + "525": "757265", + "526": "20626972", + "527": "676574", + "528": "696365", + "529": "20626f79", + "530": "20776f756c64", + "531": "6f6e67", + "532": "207374617274", + "533": "6173", + "534": "2062697264", + "535": "20736f6d6574", + "536": "206661", + "537": "696e6773", + "538": "2c22", + "539": "206d616b65", + "540": "616465", + "541": "206167", + "542": "20596f75", + "543": "2077686174", + "544": "20776f72", + "545": "6464", + "546": "2073617973", + "547": "204d6f6d", + "548": "6b696e67", + "549": "2072616e", + "550": "6f776e", + "551": "2061776179", + "552": "676574686572", + "553": "20747265", + "554": "a2e2", + "555": "82ac", + "556": "c3a2e2", + "557": "c3a2e282ac", + "558": "6965", + "559": "6167", + "560": "61726564", + "561": "6f7564", + "562": "206d616465", + "563": "20636172", + "564": "2073746172746564", + "565": "6966", + "566": "20746f676574686572", + "567": "74686572", + "568": "207061726b", + "569": "20636f", + "570": "20736f6d657468696e67", + "571": "20736164", + "572": "206f74686572", + "573": "69746564", + "574": "206e6577", + "575": "20707574", + "576": "2066726f6d", + "577": "656c6c", + "578": "20687567", + "579": "6f6f6b", + "580": "20616761696e", + "581": "626c65", + "582": "616368", + "583": "207072", + "584": "2045", + "585": "20676f6f64", + "586": "20657863", + "587": "206d75", + "588": "20686f6d65", + "589": "206c61", + "590": "2046", + "591": "7373", + "592": "72696564", + "593": "20666c", + "594": "207768656e", + "595": "6570", + "596": "2077686f", + "597": "657374", + "598": "206368", + "599": "20666f756e64", + "600": "207468656e", + "601": "20646563", + "602": "686174", + "603": "70706564", + "604": "20706c6179696e67", + "605": "20676574", + "606": "6e6e", + "607": "2077616c", + "608": "7175", + "609": "207377", + "610": "6f75676874", + "611": "2044", + "612": "206c696b6564", + "613": "75636b", + "614": "207468696e6773", + "615": "706c", + "616": "616c6c79", + "617": "205375", + "618": "6f7573", + "619": "20676f74", + "620": "2053616d", + "621": "20626c", + "622": "726f", + "623": "2065786369746564", + "624": "20796f7572", + "625": "2063616d65", + "626": "20646f67", + "627": "206465636964", + "628": "617665", + "629": "617261", + "630": "206576657279", + "631": "2064656369646564", + "632": "6178", + "633": "20736361726564", + "634": "6e6e61", + "635": "20646f776e", + "636": "757374", + "637": "2076", + "638": "206f6e65", + "639": "2063617265", + "640": "2066696e64", + "641": "6f757365", + "642": "20626563", + "643": "6170", + "644": "697374", + "645": "6e79", + "646": "207468616e", + "647": "206665656c", + "648": "20646164", + "649": "206d79", + "650": "2077696c6c", + "651": "206162", + "652": "61726e", + "653": "206c6f6f6b", + "654": "6f7572", + "655": "206772", + "656": "204d6178", + "657": "657273", + "658": "206469646e", + "659": "20746f7973", + "660": "20677261", + "661": "77617973", + "662": "206b65", + "663": "657373", + "664": "6e646572", + "665": "6665", + "666": "20706f", + "667": "20746f6f6b", + "668": "2061626f7574", + "669": "206d6f6d6d79", + "670": "20616c77617973", + "671": "2062616c6c", + "672": "616e74", + "673": "206c6f74", + "674": "2053617261", + "675": "206b6e65", + "676": "20416e6e61", + "677": "6262", + "678": "2074726565", + "679": "206275", + "680": "697365", + "681": "206f757473696465", + "682": "2074686f75676874", + "683": "206c6561726e", + "684": "697465", + "685": "207461", + "686": "206d6f7265", + "687": "726574", + "688": "656e6564", + "689": "20686f", + "690": "20536f", + "691": "206b6e6f77", + "692": "207075", + "693": "63686564", + "694": "20736f72", + "695": "676564", + "696": "656369", + "697": "20696e74", + "698": "20636174", + "699": "205768656e", + "700": "206f6c64", + "701": "2074616b65", + "702": "6961", + "703": "206c617567", + "704": "697665", + "705": "206b6e6577", + "706": "656369616c", + "707": "207370656369616c", + "708": "20616e79", + "709": "20486572", + "710": "207065", + "711": "6464656e", + "712": "66746572", + "713": "6175", + "714": "20736f727279", + "715": "20686f77", + "716": "207472", + "717": "696e6b", + "718": "2073686f77", + "719": "6d61", + "720": "2072756e", + "721": "697368", + "722": "6464656e6c79", + "723": "20736c", + "724": "206d756368", + "725": "616e64", + "726": "2068616e64", + "727": "206f72", + "728": "207477", + "729": "20686f757365", + "730": "204a6f", + "731": "204576657279", + "732": "206f70", + "733": "207472696564", + "734": "207761746572", + "735": "206d6f", + "736": "76656e", + "737": "20736b", + "738": "204a61636b", + "739": "616e6b", + "740": "20696e746f", + "741": "2073756e", + "742": "20696e73696465", + "743": "617465", + "744": "20746f6c64", + "745": "20636c6f", + "746": "756d70", + "747": "68657265", + "748": "2067617665", + "749": "7479", + "750": "205765", + "751": "2065616368", + "752": "206966", + "753": "20726f6f6d", + "754": "20656174", + "755": "2070726f7564", + "756": "207175", + "757": "206f766572", + "758": "206865617264", + "759": "20656e", + "760": "20636f756c646e", + "761": "2074686973", + "762": "616365", + "763": "2070726574", + "764": "6f74686572", + "765": "206e65766572", + "766": "204d6961", + "767": "6479", + "768": "61757365", + "769": "20706c61796564", + "770": "616b", + "771": "207069636b", + "772": "6279", + "773": "696f6e", + "774": "736564", + "775": "6574746572", + "776": "20687567676564", + "777": "6765", + "778": "2053756464656e6c79", + "779": "206578706c", + "780": "206a757374", + "781": "20636f6d", + "782": "20677265", + "783": "205468656e", + "784": "2046726f6d", + "785": "20626f78", + "786": "206e696365", + "787": "206d616e79", + "788": "206e656564", + "789": "6f7468", + "790": "7565", + "791": "2062656361757365", + "792": "20776174", + "793": "20416e64", + "794": "206f6666", + "795": "276d", + "796": "20737472", + "797": "696c65", + "798": "677279", + "799": "20736d616c6c", + "800": "20486973", + "801": "697a", + "802": "76696e67", + "803": "20736e", + "804": "20666f", + "805": "206c6f6e67", + "806": "696e65", + "807": "206a6f", + "808": "696d616c", + "809": "20616e696d616c", + "810": "2062656172", + "811": "757274", + "812": "20747279", + "813": "6f7274", + "814": "206c6561726e6564", + "815": "7563", + "816": "6f756768", + "817": "73656c", + "818": "206c6f7665", + "819": "20756e74", + "820": "696c64", + "821": "2043", + "822": "20756e74696c", + "823": "206b696e64", + "824": "6674", + "825": "20626574746572", + "826": "20666c6f77", + "827": "20696d", + "828": "20646f6e", + "829": "75726e", + "830": "207465", + "831": "20636c65", + "832": "20656e64", + "833": "766573", + "834": "2062657374", + "835": "2063616c6c", + "836": "616479", + "837": "596573", + "838": "20647265", + "839": "617273", + "840": "206361726566756c", + "841": "617574", + "842": "757270", + "843": "2066617374", + "844": "206164", + "845": "76656e74", + "846": "617264656e", + "847": "2067617264656e", + "848": "626572", + "849": "206669", + "850": "206265617574", + "851": "206375", + "852": "696573", + "853": "20627261", + "854": "617365", + "855": "206a756d70", + "856": "206c6f7564", + "857": "207468616e6b6564", + "858": "20736b79", + "859": "656d", + "860": "20636865", + "861": "206576656e", + "862": "206279", + "863": "206c617567686564", + "864": "7074", + "865": "696e79", + "866": "5468", + "867": "206c697374", + "868": "73656c66", + "869": "6c6c79", + "870": "776e", + "871": "20676c", + "872": "2068757274", + "873": "20636f6d65", + "874": "207374696c6c", + "875": "2068617264", + "876": "20616e696d616c73", + "877": "20626561757469", + "878": "2062656175746966756c", + "879": "6c6577", + "880": "20776179", + "881": "20776f", + "882": "c3a2e282acc5", + "883": "c3a2e282acc593", + "884": "206c6f7473", + "885": "2066616d", + "886": "20756e646572", + "887": "207261", + "888": "206c6574", + "889": "6265", + "890": "686573", + "891": "2073746179", + "892": "20626164", + "893": "656374", + "894": "2073616665", + "895": "6374", + "896": "6e696e67", + "897": "6f6f6c", + "898": "7368", + "899": "20626f7468", + "900": "20636f6c", + "901": "4974", + "902": "2050", + "903": "6f636b", + "904": "617374", + "905": "6b6179", + "906": "6f706c65", + "907": "20626f6f6b", + "908": "2072656d", + "909": "20616e677279", + "910": "20696d70", + "911": "6973686564", + "912": "206c69766564", + "913": "204173", + "914": "20697473", + "915": "4e6f", + "916": "2073686f756c64", + "917": "726565", + "918": "2074776f", + "919": "206272617665", + "920": "20726564", + "921": "2070656f706c65", + "922": "616e65", + "923": "2066616d696c79", + "924": "2077616c6b6564", + "925": "75727072", + "926": "204166746572", + "927": "696768", + "928": "207368617265", + "929": "2073746f72", + "930": "2066696e", + "931": "207375727072", + "932": "2064616e", + "933": "756379", + "934": "207375", + "935": "20666c6577", + "936": "20736f6f6e", + "937": "20666c79", + "938": "204c756379", + "939": "20646f6c6c", + "940": "4c6574", + "941": "7874", + "942": "666f7265", + "943": "6970", + "944": "6f62", + "945": "20706963", + "946": "2063616c6c6564", + "947": "6c6564", + "948": "20636c65616e", + "949": "20666f7265", + "950": "20707265747479", + "951": "20726f636b", + "952": "64646564", + "953": "206b657074", + "954": "69736564", + "955": "20646f6f72", + "956": "207368696e79", + "957": "204a6f68", + "958": "20676f696e67", + "959": "20647261", + "960": "206b656570", + "961": "206e6f697365", + "962": "207265616c", + "963": "206e6f77", + "964": "20446164", + "965": "2047", + "966": "2077696e64", + "967": "20637279", + "968": "20637265", + "969": "207475726e", + "970": "204d6f6d6d79", + "971": "736f", + "972": "206f70656e6564", + "973": "756d6d79", + "974": "20696465", + "975": "617279", + "976": "206265666f7265", + "977": "20636f6e", + "978": "20616c736f", + "979": "20616476656e74", + "980": "2073746f", + "981": "2067726f756e64", + "982": "616765", + "983": "6c6573", + "984": "c3a2e282ace2", + "985": "696564", + "986": "696c6c79", + "987": "206579", + "988": "20686173", + "989": "20636f6c6f72", + "990": "206578706c6f7265", + "991": "2069646561", + "992": "20666172", + "993": "57686174", + "994": "206672", + "995": "205370", + "996": "6b6573", + "997": "696d62", + "998": "206e6f64646564", + "999": "20756e", + "1000": "20776172", + "1001": "61796265", + "1002": "74757265", + "1003": "206e657874", + "1004": "207069636b6564", + "1005": "20636c696d62", + "1006": "7468696e67", + "1007": "7468", + "1008": "207768696c65", + "1009": "20736d696c65", + "1010": "20626564", + "1011": "206665656c696e67", + "1012": "20747275636b", + "1013": "6f79", + "1014": "20746872", + "1015": "84a2", + "1016": "c3a2e282ace284a2", + "1017": "2077616c6b696e67", + "1018": "656574", + "1019": "5468616e6b", + "1020": "207772", + "1021": "676572", + "1022": "20616476656e74757265", + "1023": "206265696e67", + "1024": "2070696374757265", + "1025": "6666", + "1026": "204a6f686e", + "1027": "207468696e6b", + "1028": "2067697665", + "1029": "20426f62", + "1030": "206d61", + "1031": "20666f6f64", + "1032": "20747261", + "1033": "54686174", + "1034": "6163", + "1035": "2062726f", + "1036": "20646966", + "1037": "69636564", + "1038": "20746f77", + "1039": "206c697374656e", + "1040": "20627265", + "1041": "746564", + "1042": "20636c6f73", + "1043": "2077616974", + "1044": "6a6f79", + "1045": "2065796573", + "1046": "696f7573", + "1047": "206f776e", + "1048": "75676874", + "1049": "2073746f70706564", + "1050": "206166746572", + "1051": "204e", + "1052": "2066697368", + "1053": "2053706f74", + "1054": "20666c6f77657273", + "1055": "206c6f6f6b696e67", + "1056": "207265706c", + "1057": "656d626572", + "1058": "2064656c", + "1059": "74696e67", + "1060": "20537565", + "1061": "2068616e6473", + "1062": "20736c696465", + "1063": "2077616c6b", + "1064": "2073686f7574", + "1065": "697a6564", + "1066": "4c696c79", + "1067": "20626c7565", + "1068": "2052", + "1069": "2068656164", + "1070": "206772656174", + "1071": "20766f", + "1072": "2065766572", + "1073": "206d65616e", + "1074": "69727374", + "1075": "20696d706f7274", + "1076": "61626c65", + "1077": "207573", + "1078": "20656e6a6f79", + "1079": "2061736b", + "1080": "20736f756e64", + "1081": "206e656172", + "1082": "2072656d656d626572", + "1083": "20666f72657374", + "1084": "206f6b6179", + "1085": "20646973", + "1086": "6c65617365", + "1087": "20617070", + "1088": "207261696e", + "1089": "206e6f7469636564", + "1090": "6c6c6f77", + "1091": "62626974", + "1092": "20717569636b", + "1093": "207275", + "1094": "20696d706f7274616e74", + "1095": "20737469636b", + "1096": "20776f6e646572", + "1097": "206465", + "1098": "207761746368", + "1099": "2073686f776564", + "1100": "6f6f6e", + "1101": "20746f75", + "1102": "20776f726b", + "1103": "2074616c", + "1104": "6d617a", + "1105": "206f7572", + "1106": "5765", + "1107": "20616d617a", + "1108": "206163", + "1109": "20626f6174", + "1110": "207265706c696564", + "1111": "62626564", + "1112": "207374726f6e67", + "1113": "66657265", + "1114": "627965", + "1115": "20737175", + "1116": "2064696666657265", + "1117": "646179", + "1118": "207269676874", + "1119": "207768657265", + "1120": "20666f6c6c6f77", + "1121": "67616e", + "1122": "2065766572796f6e65", + "1123": "20646f6573", + "1124": "20627269676874", + "1125": "20726162626974", + "1126": "20757365", + "1127": "636f", + "1128": "2077617463686564", + "1129": "2079656c6c", + "1130": "20686f70", + "1131": "4c6f6f6b", + "1132": "206472657373", + "1133": "696c61", + "1134": "2066616365", + "1135": "206d6f72", + "1136": "617468", + "1137": "20717569636b6c79", + "1138": "65726564", + "1139": "20626563616d65", + "1140": "205468", + "1141": "2073746f7265", + "1142": "616e6765", + "1143": "2073757265", + "1144": "20646966666572656e74", + "1145": "205361726168", + "1146": "2063757269", + "1147": "206265656e", + "1148": "20616d", + "1149": "20706c616365", + "1150": "6e67", + "1151": "6175676874", + "1152": "20766f696365", + "1153": "206368696c64", + "1154": "207761726d", + "1155": "207965", + "1156": "20676f6f64627965", + "1157": "206d6167", + "1158": "20626c6f636b", + "1159": "2068696768", + "1160": "20666f7267", + "1161": "2063616b65", + "1162": "6c656570", + "1163": "20636f6f6b", + "1164": "2074656c6c", + "1165": "6d6f7265", + "1166": "6e646d61", + "1167": "20616e796d6f7265", + "1168": "20636c6f736572", + "1169": "20637572696f7573", + "1170": "697463", + "1171": "204a616e65", + "1172": "20666176", + "1173": "206e69676874", + "1174": "2070756c6c", + "1175": "6f756e74", + "1176": "696d6573", + "1177": "206f70656e", + "1178": "2062656c", + "1179": "6579", + "1180": "206669727374", + "1181": "69726564", + "1182": "2066656c6c", + "1183": "697373", + "1184": "206c6561", + "1185": "2068657265", + "1186": "43616e", + "1187": "207a", + "1188": "6163686564", + "1189": "756e6e79", + "1190": "20627574746572", + "1191": "207061696e", + "1192": "2073656573", + "1193": "6f726e", + "1194": "756c6c", + "1195": "206b6964", + "1196": "2067726162626564", + "1197": "69746368656e", + "1198": "206b69746368656e", + "1199": "207377656574", + "1200": "20666c6f", + "1201": "206f6e6c79", + "1202": "2068656c706564", + "1203": "2073616e64", + "1204": "207468726565", + "1205": "2073746f70", + "1206": "20496e", + "1207": "206269726473", + "1208": "206d6f6e", + "1209": "20636f6e74", + "1210": "4d6f6d", + "1211": "20626974", + "1212": "2068617070656e6564", + "1213": "206a756d706564", + "1214": "20637261", + "1215": "2045766572796f6e65", + "1216": "2054686174", + "1217": "20676c6164", + "1218": "206261", + "1219": "206472", + "1220": "2066697265", + "1221": "20446f", + "1222": "204c696c61", + "1223": "206772617373", + "1224": "206372", + "1225": "707079", + "1226": "2062726f74686572", + "1227": "20747269", + "1228": "20626568", + "1229": "697379", + "1230": "2073696e67", + "1231": "207265616c697a6564", + "1232": "204c6574", + "1233": "616972", + "1234": "2073686f75746564", + "1235": "206d657373", + "1236": "20706572", + "1237": "616b6573", + "1238": "696e73", + "1239": "2073746f7279", + "1240": "206c616479", + "1241": "20736f6674", + "1242": "6e7473", + "1243": "2079756d6d79", + "1244": "20686174", + "1245": "617373", + "1246": "2072656163686564", + "1247": "2064726177", + "1248": "277265", + "1249": "2067616d65", + "1250": "20617465", + "1251": "20686176696e67", + "1252": "206661766f72", + "1253": "2070726574656e64", + "1254": "20756e6465727374", + "1255": "206661766f72697465", + "1256": "7a79", + "1257": "20706170", + "1258": "2072657374", + "1259": "207072696e", + "1260": "206c657373", + "1261": "61746572", + "1262": "7768657265", + "1263": "207468726f756768", + "1264": "206c6f6f6b73", + "1265": "207265616479", + "1266": "2043616e", + "1267": "6b656e", + "1268": "207468696e67", + "1269": "2063617273", + "1270": "206d616b696e67", + "1271": "2070617265", + "1272": "206475636b", + "1273": "2062756e6e79", + "1274": "207265616c6c79", + "1275": "576f77", + "1276": "206d61676963", + "1277": "6174686572", + "1278": "20666978", + "1279": "207469726564", + "1280": "20637574", + "1281": "206b696473", + "1282": "206d756d", + "1283": "20776f726c64", + "1284": "20706f6e64", + "1285": "204a6f65", + "1286": "2068696d73656c66", + "1287": "626f", + "1288": "20646f6e65", + "1289": "20746f756368", + "1290": "20776f727279", + "1291": "2062757474657266", + "1292": "206372696564", + "1293": "20636f6d696e67", + "1294": "20626567616e", + "1295": "206361746368", + "1296": "20626c6f636b73", + "1297": "207761766564", + "1298": "2068616972", + "1299": "2070726f6d", + "1300": "2066756e6e79", + "1301": "20776879", + "1302": "66756c6c79", + "1303": "206e616d65", + "1304": "546f6d", + "1305": "20686f6c64", + "1306": "20616e6f74686572", + "1307": "2075736564", + "1308": "20666c6f776572", + "1309": "2070617274", + "1310": "20737465", + "1311": "69676e", + "1312": "2066726f", + "1313": "20636f6f6c", + "1314": "72656c", + "1315": "2066696c6c", + "1316": "2046696e", + "1317": "6f6e6579", + "1318": "73746572", + "1319": "2074617374", + "1320": "207365656e", + "1321": "20746f70", + "1322": "65646479", + "1323": "20776973686564", + "1324": "20666c6f6f72", + "1325": "69656e74", + "1326": "69766572", + "1327": "206c696b6573", + "1328": "207377696d", + "1329": "20677265656e", + "1330": "2064656c6963", + "1331": "207468616e6b", + "1332": "206d6f757365", + "1333": "20626167", + "1334": "206c69676874", + "1335": "206769", + "1336": "207061706572", + "1337": "20706172656e7473", + "1338": "446f6e", + "1339": "6879", + "1340": "6f6c6c79", + "1341": "206661726d", + "1342": "20646f696e67", + "1343": "20736d656c6c", + "1344": "20626568696e64", + "1345": "6174696f6e", + "1346": "20546f6d6d79", + "1347": "20647269", + "1348": "206c6573736f6e", + "1349": "61726473", + "1350": "20626f77", + "1351": "20736c656570", + "1352": "20686f6c65", + "1353": "2053616c6c79", + "1354": "2064726f", + "1355": "756666", + "1356": "207375727072697365", + "1357": "2042656e6e79", + "1358": "20686f70706564", + "1359": "2077696e646f77", + "1360": "616e6479", + "1361": "20776f726b6564", + "1362": "2064696e", + "1363": "61697379", + "1364": "2079656c6c6f77", + "1365": "206461726b", + "1366": "2072656164", + "1367": "207472656573", + "1368": "2072756e6e696e67", + "1369": "696b65", + "1370": "207377696e67", + "1371": "2073686f", + "1372": "206c61756768", + "1373": "756564", + "1374": "2066756c6c", + "1375": "222e", + "1376": "20736174", + "1377": "72656e", + "1378": "656c79", + "1379": "20776f6f64", + "1380": "2073706f74", + "1381": "7573", + "1382": "20636f6c64", + "1383": "2072656d656d6265726564", + "1384": "6172696e67", + "1385": "20736c6f77", + "1386": "20737572707269736564", + "1387": "2068617070696c79", + "1388": "206c6f7374", + "1389": "20627574746572666c79", + "1390": "756c", + "1391": "20696365", + "1392": "48656c6c", + "1393": "48656c6c6f", + "1394": "206e6565646564", + "1395": "20736574", + "1396": "20627579", + "1397": "6f6e73", + "1398": "207965617273", + "1399": "2068656c64", + "1400": "2066726f67", + "1401": "2077726f6e67", + "1402": "656e73", + "1403": "206d6f7665", + "1404": "20616e797468696e67", + "1405": "2068656172", + "1406": "204174", + "1407": "206361726566756c6c79", + "1408": "20736e6f77", + "1409": "206c656674", + "1410": "2068756e", + "1411": "2066616972", + "1412": "204b", + "1413": "20646972", + "1414": "2074616c6c", + "1415": "20456d", + "1416": "20647265616d", + "1417": "20666f6c6c6f776564", + "1418": "20656c", + "1419": "7265616d", + "1420": "207461626c65", + "1421": "20626f6f6b73", + "1422": "204e6f77", + "1423": "2070657266", + "1424": "2066696e6973686564", + "1425": "204461697379", + "1426": "20627567", + "1427": "206368696c6472656e", + "1428": "20686964", + "1429": "20686f74", + "1430": "20616972", + "1431": "63657373", + "1432": "20627573", + "1433": "20656e6a6f796564", + "1434": "2063617374", + "1435": "72656564", + "1436": "20706c65617365", + "1437": "2068657273656c66", + "1438": "20756e6465727374616e64", + "1439": "696e6564", + "1440": "20667269656e646c79", + "1441": "2061727269", + "1442": "6473", + "1443": "20636f6e74696e", + "1444": "2046696e616c6c79", + "1445": "2068696c6c", + "1446": "20636c6f7365", + "1447": "206d69676874", + "1448": "20546f", + "1449": "726f7373", + "1450": "2061726d", + "1451": "20666f72676f74", + "1452": "2042696c6c79", + "1453": "20656c7365", + "1454": "2062726f6b656e", + "1455": "7370", + "1456": "6c66", + "1457": "20636f6c6f7273", + "1458": "20707265", + "1459": "206275696c64", + "1460": "2064656c6963696f7573", + "1461": "2077616e7473", + "1462": "20477261", + "1463": "204a656e", + "1464": "6173757265", + "1465": "617368", + "1466": "2067656e74", + "1467": "4f4b", + "1468": "6174746572", + "1469": "2072656c", + "1470": "69656c64", + "1471": "20676f6e65", + "1472": "726f6f6d", + "1473": "207469676874", + "1474": "2061727269766564", + "1475": "2077616c6c", + "1476": "207371756972", + "1477": "2062616279", + "1478": "206c6561766573", + "1479": "20776f6e64657266756c", + "1480": "6e6572", + "1481": "206372656174", + "1482": "205468657265", + "1483": "20736f6e67", + "1484": "2061626c65", + "1485": "6176", + "1486": "20736f6d656f6e65", + "1487": "20796573", + "1488": "21c3a2e282ac", + "1489": "204a696d", + "1490": "206775", + "1491": "61736b", + "1492": "20737175697272656c", + "1493": "20636f6e74696e756564", + "1494": "2054686973", + "1495": "696368", + "1496": "2066696c6c6564", + "1497": "20636f756e74", + "1498": "20627265616b", + "1499": "206c696f6e", + "1500": "204d61796265", + "1501": "20616772656564", + "1502": "6565", + "1503": "2070696e6b", + "1504": "20706f696e", + "1505": "2066616c6c", + "1506": "207363617279", + "1507": "20416d79", + "1508": "206772616e646d61", + "1509": "20636f6f6b696573", + "1510": "73686564", + "1511": "20766973", + "1512": "276c6c", + "1513": "72696573", + "1514": "206d616b6573", + "1515": "2068756e677279", + "1516": "206a75", + "1517": "6d616e", + "1518": "20656e6f756768", + "1519": "206d6f6d656e74", + "1520": "20736d696c6573", + "1521": "20736f6d6574696d6573", + "1522": "20636173746c65", + "1523": "20737461796564", + "1524": "206669676874", + "1525": "2066696e616c6c79", + "1526": "207475726e6564", + "1527": "206d6f726e696e67", + "1528": "696379", + "1529": "726f77", + "1530": "20776574", + "1531": "2ec3a2e282ac", + "1532": "207069", + "1533": "2067657474696e67", + "1534": "20637265616d", + "1535": "206f7468657273", + "1536": "204d6f6c6c79", + "1537": "20636f6d70", + "1538": "676f6e", + "1539": "617070", + "1540": "207069637475726573", + "1541": "20636c696d626564", + "1542": "206265616368", + "1543": "6f7570", + "1544": "2057686174", + "1545": "2074616c6b", + "1546": "2064696e6e6572", + "1547": "206c697374656e6564", + "1548": "726f74", + "1549": "20747279696e67", + "1550": "206d6574", + "1551": "6f6f6d", + "1552": "616d61", + "1553": "207461696c", + "1554": "20636c6f74", + "1555": "657265", + "1556": "20636c6f74686573", + "1557": "206d6f74686572", + "1558": "2074687265", + "1559": "206d7573", + "1560": "6c63", + "1561": "636564", + "1562": "616e6365", + "1563": "20666577", + "1564": "20616d617a696e67", + "1565": "6170706564", + "1566": "207075707079", + "1567": "206d6172", + "1568": "6b6579", + "1569": "2061736b73", + "1570": "206461646479", + "1571": "656e6365", + "1572": "75666679", + "1573": "726564", + "1574": "2062616c6c6f6f6e", + "1575": "2070657266656374", + "1576": "207768697465", + "1577": "2073706c", + "1578": "6f72696e67", + "1579": "20616c6f6e65", + "1580": "20667572", + "1581": "20706574", + "1582": "20736974", + "1583": "207361696c", + "1584": "63746f72", + "1585": "4f68", + "1586": "616d70", + "1587": "546865", + "1588": "75636b79", + "1589": "2062726f75676874", + "1590": "6973746572", + "1591": "20706f6c", + "1592": "776c", + "1593": "2067726f77", + "1594": "2065766572797468696e67", + "1595": "6f7474", + "1596": "207761736e", + "1597": "2070756c6c6564", + "1598": "646572", + "1599": "20737472616e6765", + "1600": "2073656172", + "1601": "2072696465", + "1602": "20776169746564", + "1603": "427574", + "1604": "2064616e676572", + "1605": "20706965", + "1606": "61746564", + "1607": "206a6f79", + "1608": "20737061726b", + "1609": "20726f6c6c", + "1610": "2064656570", + "1611": "20776f6e", + "1612": "204a696c6c", + "1613": "20776f6f6473", + "1614": "2072756e73", + "1615": "2068696465", + "1616": "54686973", + "1617": "20706c616e", + "1618": "207669736974", + "1619": "6f776572", + "1620": "206c65617665", + "1621": "207468696e6b73", + "1622": "20616d617a6564", + "1623": "206d75736963", + "1624": "7a7a", + "1625": "4d61796265", + "1626": "6d65", + "1627": "6f73", + "1628": "207465646479", + "1629": "206c6976696e67", + "1630": "616e67", + "1631": "20706174", + "1632": "2070656e", + "1633": "20746f7761726473", + "1634": "20666f78", + "1635": "617065", + "1636": "65746572", + "1637": "2073696c6c79", + "1638": "20666c79696e67", + "1639": "20636c6f7564", + "1640": "4f6b6179", + "1641": "20536f6f6e", + "1642": "206f776e6572", + "1643": "65766572", + "1644": "756767", + "1645": "20656d", + "1646": "6773", + "1647": "616d6573", + "1648": "6d6564", + "1649": "207472656174", + "1650": "69736573", + "1651": "6f7065", + "1652": "20717569", + "1653": "746879", + "1654": "20736861726564", + "1655": "206c616e64", + "1656": "206772617465", + "1657": "61696e6564", + "1658": "2070726f6d69736564", + "1659": "73736564", + "1660": "696767", + "1661": "206d6f6e6579", + "1662": "206669656c64", + "1663": "636b6574", + "1664": "206c616b65", + "1665": "2073696465", + "1666": "20416c6c", + "1667": "20466c", + "1668": "207061696e74", + "1669": "20736973746572", + "1670": "2062726f6b65", + "1671": "20677261746566756c", + "1672": "6978", + "1673": "206d6f72616c", + "1674": "20747261696e", + "1675": "20626f7474", + "1676": "20666173746572", + "1677": "2072616365", + "1678": "617679", + "1679": "636c65", + "1680": "2063616e6479", + "1681": "65636573", + "1682": "207061727479", + "1683": "206a6f62", + "1684": "207175696574", + "1685": "756e6368", + "1686": "2064616e6365", + "1687": "206272616e", + "1688": "2062656c6965", + "1689": "20706965636573", + "1690": "207472656173757265", + "1691": "2077656172", + "1692": "20776f7272696564", + "1693": "20646f6c6c73", + "1694": "20637570", + "1695": "7377", + "1696": "20626c616e6b", + "1697": "20636f6c6f7266756c", + "1698": "20616e7377", + "1699": "20736c6f776c79", + "1700": "20656c65", + "1701": "20676966", + "1702": "206d757374", + "1703": "2077656c63", + "1704": "207665", + "1705": "207265616368", + "1706": "204461646479", + "1707": "7068", + "1708": "20626c616e6b6574", + "1709": "6f757468", + "1710": "20626967676572", + "1711": "756974", + "1712": "6f6c", + "1713": "7a65", + "1714": "2062696b65", + "1715": "204d72", + "1716": "2063686f", + "1717": "75616c6c79", + "1718": "4d6f6d6d79", + "1719": "20626c61636b", + "1720": "206d6f757468", + "1721": "2073637265616d", + "1722": "6f726d", + "1723": "666f7274", + "1724": "20736e61636b", + "1725": "20736d617274", + "1726": "2065766572797768657265", + "1727": "61736564", + "1728": "207269766572", + "1729": "2070726f", + "1730": "6c696e67", + "1731": "20626172", + "1732": "2064616e6765726f7573", + "1733": "697265", + "1734": "2073756368", + "1735": "206a6f696e", + "1736": "20737475", + "1737": "20706c616e74", + "1738": "636b6564", + "1739": "2068656c70696e67", + "1740": "206e6170", + "1741": "6c657373", + "1742": "206a65", + "1743": "207468726577", + "1744": "204a757374", + "1745": "206469727479", + "1746": "207072696e63657373", + "1747": "53757265", + "1748": "2077696e", + "1749": "207061", + "1750": "736574", + "1751": "20636172726f74", + "1752": "207369676e", + "1753": "20637279696e67", + "1754": "20706f74", + "1755": "207477696e73", + "1756": "206d6f6e6b6579", + "1757": "20746f776572", + "1758": "20626f776c", + "1759": "6964656e74", + "1760": "20647261676f6e", + "1761": "206d6f6e73746572", + "1762": "6970706564", + "1763": "206d61676963616c", + "1764": "6d696e67", + "1765": "20686f706564", + "1766": "20626565", + "1767": "20616363", + "1768": "6f726564", + "1769": "206865617679", + "1770": "2062697465", + "1771": "20646f63746f72", + "1772": "20776f726473", + "1773": "20737072", + "1774": "6465", + "1775": "206d6f7374", + "1776": "204d617279", + "1777": "656172", + "1778": "2073616d65", + "1779": "4869", + "1780": "62626c65", + "1781": "2074656163", + "1782": "206661726d6572", + "1783": "576879", + "1784": "42656e", + "1785": "206865616c", + "1786": "206163726f7373", + "1787": "206672756974", + "1788": "2062617468", + "1789": "206275636b", + "1790": "204d616d61", + "1791": "20646973617070", + "1792": "2073747265", + "1793": "20737475636b", + "1794": "20636c6f736564", + "1795": "2073746570706564", + "1796": "20636f6c6c", + "1797": "20686967", + "1798": "6561726564", + "1799": "206665656c73", + "1800": "206d6978", + "1801": "20746f776e", + "1802": "20776f756c646e", + "1803": "63686f6f6c", + "1804": "206865617274", + "1805": "2062757474", + "1806": "61736b6574", + "1807": "20636f7572", + "1808": "206261736b6574", + "1809": "2062656c69657665", + "1810": "2063726179", + "1811": "20617474", + "1812": "20656174696e67", + "1813": "206368616972", + "1814": "204772616e646d61", + "1815": "2074656163686572", + "1816": "2062726f776e", + "1817": "207363686f6f6c", + "1818": "20686974", + "1819": "20636f75727365", + "1820": "20707573686564", + "1821": "6973736564", + "1822": "2079617264", + "1823": "207377696e6773", + "1824": "6974696e67", + "1825": "206661697279", + "1826": "20776974686f7574", + "1827": "207061636b", + "1828": "7473", + "1829": "416e6e61", + "1830": "2067656e746c65", + "1831": "6f7264", + "1832": "206163636964656e74", + "1833": "20456d6d61", + "1834": "20736d6f", + "1835": "206d696e", + "1836": "2070657273", + "1837": "2077656c636f6d65", + "1838": "20636c6170706564", + "1839": "6f72726f77", + "1840": "206e65696768", + "1841": "7070696e67", + "1842": "20746f646179", + "1843": "20686967686572", + "1844": "2067616d6573", + "1845": "206e6f69736573", + "1846": "6272", + "1847": "696572", + "1848": "756d62", + "1849": "20736563", + "1850": "207475726e73", + "1851": "206e6569676862", + "1852": "207374616e64", + "1853": "6f7273", + "1854": "207468726f77", + "1855": "2064726f7665", + "1856": "2062656e", + "1857": "20496e73696465", + "1858": "76656c", + "1859": "20647279", + "1860": "686f", + "1861": "20706f636b6574", + "1862": "76656e7475616c6c79", + "1863": "2073756464656e6c79", + "1864": "205468656972", + "1865": "207069656365", + "1866": "2068756773", + "1867": "206c6f6e", + "1868": "2066726565", + "1869": "20666f72676574", + "1870": "206e65636b", + "1871": "6f75726e", + "1872": "206a6f75726e", + "1873": "2063686565726564", + "1874": "2068617070656e", + "1875": "2077696e6773", + "1876": "206c6f6e656c79", + "1877": "2066656e6365", + "1878": "72616374", + "1879": "656164", + "1880": "206578706c61696e6564", + "1881": "20466c75666679", + "1882": "6379", + "1883": "7068616e74", + "1884": "2077656c6c", + "1885": "20736561", + "1886": "206c617465", + "1887": "69626c65", + "1888": "207570736574", + "1889": "20616c6f6e67", + "1890": "697474696e67", + "1891": "206d6174", + "1892": "2073746f72696573", + "1893": "206c6f6e676572", + "1894": "20696e7374", + "1895": "2073616e67", + "1896": "20656c657068616e74", + "1897": "20776865", + "1898": "2073686f6573", + "1899": "20636f726e", + "1900": "206d697373", + "1901": "63686573", + "1902": "20636175676874", + "1903": "2074726f75", + "1904": "20616476656e7475726573", + "1905": "206c756e6368", + "1906": "20736f7570", + "1907": "20726f636b73", + "1908": "2073686170", + "1909": "6f636b6564", + "1910": "2073686f6f6b", + "1911": "2077697368", + "1912": "20736563726574", + "1913": "207374617273", + "1914": "3fc3a2e282ac", + "1915": "2073746172", + "1916": "206272696e67", + "1917": "204f", + "1918": "2064726f70706564", + "1919": "2070757270", + "1920": "2053616d6d79", + "1921": "206275636b6574", + "1922": "207068", + "1923": "205468616e6b", + "1924": "204f6e", + "1925": "6f756e", + "1926": "6d656e74", + "1927": "20796f756e67", + "1928": "207377616d", + "1929": "2073696e67696e67", + "1930": "2064697374", + "1931": "206d6f766564", + "1932": "79617264", + "1933": "206a6172", + "1934": "206d696e65", + "1935": "636573", + "1936": "2070617374", + "1937": "204d756d", + "1938": "656b", + "1939": "2062656c6f6e67", + "1940": "206261636b79617264", + "1941": "204a696d6d79", + "1942": "206f776c", + "1943": "20706c6163", + "1944": "626279", + "1945": "20746f6d", + "1946": "617567687479", + "1947": "6f756e7461696e", + "1948": "6c6c6f", + "1949": "206e617567687479", + "1950": "20737472656574", + "1951": "206472696e6b", + "1952": "206170706c65", + "1953": "6675736564", + "1954": "2074616b696e67", + "1955": "2074616b6573", + "1956": "446f", + "1957": "20736861706573", + "1958": "20707572706c65", + "1959": "204c61746572", + "1960": "206f72616e6765", + "1961": "2077697365", + "1962": "20657870", + "1963": "206b6579", + "1964": "206a75696365", + "1965": "61696c", + "1966": "626f77", + "1967": "206e6f7365", + "1968": "2072657370", + "1969": "205477", + "1970": "20636f77", + "1971": "206578706c6f72696e67", + "1972": "206e6561726279", + "1973": "20776f6e6465726564", + "1974": "2062656e6368", + "1975": "77617264", + "1976": "20736f756e6473", + "1977": "20636f6d6573", + "1978": "206573", + "1979": "20636f75", + "1980": "636f6c", + "1981": "20776f6b65", + "1982": "756e67", + "1983": "206166", + "1984": "2073686f70", + "1985": "207369636b", + "1986": "20646573", + "1987": "2062757379", + "1988": "206d6164", + "1989": "206e6f7468696e67", + "1990": "20706967", + "1991": "697474656e73", + "1992": "7379", + "1993": "207365656d6564", + "1994": "53617261", + "1995": "2074686f756768", + "1996": "206d61696c", + "1997": "61646f77", + "1998": "2072656c6965", + "1999": "61756768746572", + "2000": "636b73", + "2001": "20696e76", + "2002": "20686f6c64696e67", + "2003": "63696e67", + "2004": "696f6e73", + "2005": "66697368", + "2006": "2074696e79", + "2007": "657365", + "2008": "2061726d73", + "2009": "206e657374", + "2010": "2073697474696e67", + "2011": "206272616e6368", + "2012": "206d61726b", + "2013": "7468646179", + "2014": "206d696c", + "2015": "2079656172", + "2016": "696e696e67", + "2017": "70706572", + "2018": "207261696e626f77", + "2019": "697479", + "2020": "20647269766572", + "2021": "6f7365", + "2022": "206f62", + "2023": "72616964", + "2024": "206b696e67", + "2025": "616c65", + "2026": "20616672616964", + "2027": "205768", + "2028": "2033", + "2029": "20726f756e64", + "2030": "2067696674", + "2031": "20636972", + "2032": "206361727279", + "2033": "2063757465", + "2034": "54696d", + "2035": "2068656c6c6f", + "2036": "20666974", + "2037": "746f", + "2038": "206b697465", + "2039": "20436f6d65", + "2040": "206461756768746572", + "2041": "2070617469656e74", + "2042": "20666f7265766572", + "2043": "206c696674", + "2044": "626f617264", + "2045": "4f66", + "2046": "69646765", + "2047": "7665726564", + "2048": "207065616365", + "2049": "207368656c6c", + "2050": "657468", + "2051": "20746f7563686564", + "2052": "206272656164", + "2053": "20626f756e", + "2054": "20456c6c", + "2055": "2073706f6f6e", + "2056": "20636f6174", + "2057": "2062616e64", + "2058": "6c6574", + "2059": "2068756765", + "2060": "2057697468", + "2061": "206163636964656e74616c6c79", + "2062": "206269727468646179", + "2063": "206c6574746572", + "2064": "2067726162", + "2065": "2070726f6d697365", + "2066": "636572", + "2067": "207377696d6d696e67", + "2068": "486579", + "2069": "20686f7065", + "2070": "2072656c6965766564", + "2071": "206769766573", + "2072": "2070757473", + "2073": "20626f74746c65", + "2074": "7374696e67", + "2075": "6f757273", + "2076": "20636865657365", + "2077": "206d696c6b", + "2078": "207768697374", + "2079": "20777261", + "2080": "205477656574", + "2081": "2067696767", + "2082": "7368696e65", + "2083": "2066696e67", + "2084": "2062757368", + "2085": "20636f7a79", + "2086": "20696e74657265", + "2087": "206f6e746f", + "2088": "206b69636b", + "2089": "6767", + "2090": "20616374", + "2091": "206d697374", + "2092": "67676564", + "2093": "206372656174757265", + "2094": "206c696665", + "2095": "20706f776572", + "2096": "616c6d", + "2097": "2064617973", + "2098": "6f67", + "2099": "206a6f75726e6579", + "2100": "206c61756768696e67", + "2101": "206578636974696e67", + "2102": "206c75636b79", + "2103": "20636f6d666f7274", + "2104": "20636f6c6c656374", + "2105": "756e74", + "2106": "207370696e", + "2107": "2073756e7368696e65", + "2108": "20696e7374656164", + "2109": "2066616e", + "2110": "20726574", + "2111": "706564", + "2112": "2073756e6e79", + "2113": "20656172", + "2114": "2063617665", + "2115": "20416c", + "2116": "20706f696e746564", + "2117": "617270", + "2118": "20696d6167", + "2119": "20696e63", + "2120": "2070756464", + "2121": "20726f6164", + "2122": "69746368", + "2123": "20646973636f", + "2124": "74656e", + "2125": "656c73", + "2126": "2072657475726e", + "2127": "6e657373", + "2128": "62616c6c", + "2129": "20686f6e6579", + "2130": "207368617270", + "2131": "697a65", + "2132": "2073706f", + "2133": "20736d696c696e67", + "2134": "20656173", + "2135": "20776f6d", + "2136": "207365617263686564", + "2137": "7562", + "2138": "20776f6d616e", + "2139": "2073616e6477", + "2140": "206c61746572", + "2141": "204a656e6e79", + "2142": "20616e796f6e65", + "2143": "20626f75676874", + "2144": "207761746368696e67", + "2145": "2074616c6b6564", + "2146": "20686f72", + "2147": "626c656d", + "2148": "20676174686572", + "2149": "2070726f626c656d", + "2150": "20737563", + "2151": "204d697474656e73", + "2152": "697274", + "2153": "6d6572", + "2154": "476f6f64", + "2155": "206475636b73", + "2156": "20736e6f776d616e", + "2157": "20627574746f6e", + "2158": "20706572736f6e", + "2159": "206d656574", + "2160": "206865616c746879", + "2161": "20637261796f6e73", + "2162": "20706f70", + "2163": "2074726f75626c65", + "2164": "206d616368", + "2165": "20726f7065", + "2166": "206170706c6573", + "2167": "207461737479", + "2168": "206c6f766573", + "2169": "206b697373", + "2170": "2070696c65", + "2171": "20736d656c6c6564", + "2172": "726f756e64", + "2173": "206365", + "2174": "2073746f726d", + "2175": "207a6f6f6d", + "2176": "20706c616e65", + "2177": "20536f6d6574", + "2178": "20486f77", + "2179": "204368", + "2180": "20706172", + "2181": "2073706964", + "2182": "697474656e", + "2183": "65656e", + "2184": "20656e74", + "2185": "207368696e696e67", + "2186": "616d6564", + "2187": "20456d696c79", + "2188": "20696e746572657374696e67", + "2189": "20636c65766572", + "2190": "20636c6f756473", + "2191": "6f7665", + "2192": "20726573", + "2193": "2077696465", + "2194": "6e6470", + "2195": "6f776564", + "2196": "205265", + "2197": "206f7267616e", + "2198": "2070726573", + "2199": "206d6174746572", + "2200": "207a6f6f", + "2201": "2072657370656374", + "2202": "206261726b", + "2203": "2063617264", + "2204": "6f6e74", + "2205": "2061646d", + "2206": "2066656174686572", + "2207": "2073637265616d6564", + "2208": "206d7564", + "2209": "20686f6e", + "2210": "617572", + "2211": "2070696c6c", + "2212": "6570656e64", + "2213": "20656167", + "2214": "206170", + "2215": "6562", + "2216": "20676f6573", + "2217": "2066726f6e74", + "2218": "20736e616b65", + "2219": "20737572", + "2220": "207072616374", + "2221": "2070726f74", + "2222": "2061736c656570", + "2223": "20736166", + "2224": "2073656174", + "2225": "206e65696768626f72", + "2226": "2070757a7a", + "2227": "20616e73776572", + "2228": "2074726970", + "2229": "207175657374", + "2230": "2064616e636564", + "2231": "436f6d65", + "2232": "2070726f74656374", + "2233": "207465616368", + "2234": "20686f727365", + "2235": "7570", + "2236": "6f6f7365", + "2237": "207465657468", + "2238": "20736872", + "2239": "207472696573", + "2240": "2070617468", + "2241": "2076696c6c", + "2242": "697361", + "2243": "2070726573656e74", + "2244": "2077656172696e67", + "2245": "206170706561726564", + "2246": "206769616e74", + "2247": "2073706f74746564", + "2248": "20736c6964", + "2249": "20666f7267697665", + "2250": "206c6f76656c79", + "2251": "20416e6e", + "2252": "2067656e746c79", + "2253": "2077696c64", + "2254": "20706c61636573", + "2255": "206368656572", + "2256": "204c65", + "2257": "205374", + "2258": "2070686f6e65", + "2259": "65726c79", + "2260": "2077686f6c65", + "2261": "207768696368", + "2262": "6368696e67", + "2263": "206d7973746572", + "2264": "2064697274", + "2265": "20546f676574686572", + "2266": "20476f", + "2267": "6e647061", + "2268": "206d696e7574", + "2269": "207468696e", + "2270": "2076616c", + "2271": "20696d707265", + "2272": "206469", + "2273": "69747479", + "2274": "6f6e6573", + "2275": "206c697665", + "2276": "2072756465", + "2277": "206d616368696e65", + "2278": "204a616b65", + "2279": "756c6172", + "2280": "657274", + "2281": "20636f6e6675736564", + "2282": "20617368", + "2283": "20726f6c6c6564", + "2284": "2070617373", + "2285": "206661636573", + "2286": "2070726574656e646564", + "2287": "206d696e64", + "2288": "2063616c6d", + "2289": "2074756e", + "2290": "6365616e", + "2291": "206f6365616e", + "2292": "20746173746564", + "2293": "20636f756368", + "2294": "207361636b", + "2295": "6e6963", + "2296": "697a7a", + "2297": "20776f726d", + "2298": "2070616e", + "2299": "206d6564", + "2300": "2073706c6173", + "2301": "20706f7572", + "2302": "207069636e6963", + "2303": "2073757070", + "2304": "20736572", + "2305": "206472697665", + "2306": "2066656574", + "2307": "2062616e", + "2308": "2063686f636f6c", + "2309": "207370656e74", + "2310": "6c61", + "2311": "206361", + "2312": "20636f696e", + "2313": "2073616c", + "2314": "4a6f", + "2315": "20636f6f6b6965", + "2316": "206c6164", + "2317": "20536f6d6574696d6573", + "2318": "20636f726e6572", + "2319": "756d626572", + "2320": "756e6b", + "2321": "2062756773", + "2322": "2077617665", + "2323": "206c6179", + "2324": "204c697361", + "2325": "20737069646572", + "2326": "20746f6d6f72726f77", + "2327": "2072696e67", + "2328": "20656d62", + "2329": "206275696c64696e67", + "2330": "2070757368", + "2331": "206f76656e", + "2332": "20736d6f6f7468", + "2333": "20726f62", + "2334": "206675727279", + "2335": "207772697465", + "2336": "20686964696e67", + "2337": "20776970", + "2338": "61726765", + "2339": "61746f72", + "2340": "2069676e", + "2341": "204265", + "2342": "616c73", + "2343": "206c617374", + "2344": "20726563", + "2345": "20627562626c65", + "2346": "20636869636b", + "2347": "206b6e6565", + "2348": "2068656c7066756c", + "2349": "20746964", + "2350": "2073656c", + "2351": "2072656d696e64", + "2352": "61746f", + "2353": "72656164", + "2354": "20666f6c64", + "2355": "206c656166", + "2356": "2063617274", + "2357": "206d6f756e7461696e", + "2358": "2063616e6e", + "2359": "20626c6577", + "2360": "2064656c69676874", + "2361": "206c6164646572", + "2362": "2074616c6b696e67", + "2363": "20706177", + "2364": "20706f6f6c", + "2365": "61727261", + "2366": "2063686f636f6c617465", + "2367": "206669676874696e67", + "2368": "2062656c6c", + "2369": "2066696e697368", + "2370": "204275", + "2371": "206e65636b6c", + "2372": "20666f6f74", + "2373": "2063616e6e6f74", + "2374": "20656d6261727261", + "2375": "206d61796265", + "2376": "20737072656164", + "2377": "207374726574", + "2378": "20776f7265", + "2379": "20736f6e6773", + "2380": "6967686564", + "2381": "20746f6f6c", + "2382": "20636861736564", + "2383": "2077616974696e67", + "2384": "207370726179", + "2385": "6c6578", + "2386": "20707564646c65", + "2387": "696665", + "2388": "2062616e616e", + "2389": "20736879", + "2390": "20726f626f74", + "2391": "6d6f", + "2392": "20726570", + "2393": "206669786564", + "2394": "206576656e7475616c6c79", + "2395": "2073656c66697368", + "2396": "206e6574", + "2397": "2077617368", + "2398": "20736f6c", + "2399": "206261726b6564", + "2400": "20626174", + "2401": "206c61726765", + "2402": "5374", + "2403": "20506c65617365", + "2404": "207461737465", + "2405": "207768656e65766572", + "2406": "2063726f7373", + "2407": "20636f6f", + "2408": "20536f6d65", + "2409": "756d7079", + "2410": "756c74", + "2411": "706572", + "2412": "6f7068", + "2413": "20636c6f7468", + "2414": "20706561636566756c", + "2415": "20737472696e67", + "2416": "205768696c65", + "2417": "2073746570", + "2418": "2074696479", + "2419": "206d6973736564", + "2420": "4d79", + "2421": "206b6e6f636b6564", + "2422": "206372656174697665", + "2423": "20616e74", + "2424": "2073746f6e65", + "2425": "20706c617465", + "2426": "626f78", + "2427": "205465646479", + "2428": "206865617273", + "2429": "2045766572797768657265", + "2430": "506c65617365", + "2431": "2073746972", + "2432": "6c656272", + "2433": "20737065", + "2434": "2073636172", + "2435": "206e65636b6c616365", + "2436": "2063656c656272", + "2437": "206465636f72", + "2438": "61697273", + "2439": "75727365", + "2440": "20636172726f7473", + "2441": "2067656e", + "2442": "207368656c66", + "2443": "20747265617473", + "2444": "6f727279", + "2445": "2076696c6c616765", + "2446": "205065746572", + "2447": "205a", + "2448": "2068756d", + "2449": "2073747261", + "2450": "207472757374", + "2451": "656c6c79", + "2452": "67657461626c65", + "2453": "206d6f6f6e", + "2454": "20656d626172726173736564", + "2455": "20766567657461626c65", + "2456": "207368697274", + "2457": "756e676c65", + "2458": "20636f6c6f7572", + "2459": "20737469636b73", + "2460": "2074656c6c73", + "2461": "206a756e676c65", + "2462": "2063726162", + "2463": "69676572", + "2464": "6f6c64", + "2465": "20746561", + "2466": "2073686172696e67", + "2467": "20726f636b6574", + "2468": "757368", + "2469": "2077686973746c65", + "2470": "2077686973", + "2471": "2073706c617368", + "2472": "646765", + "2473": "206a756d70696e67", + "2474": "61726d", + "2475": "20656d70", + "2476": "20666f7277617264", + "2477": "2068656c", + "2478": "20656d707479", + "2479": "2073617665", + "2480": "2067726179", + "2481": "206c656773", + "2482": "697264", + "2483": "206c616e646564", + "2484": "20706c65", + "2485": "207370696379", + "2486": "205456", + "2487": "696e746564", + "2488": "206e6f6473", + "2489": "616e6f", + "2490": "20617368616d6564", + "2491": "2067756e", + "2492": "2074696768746c79", + "2493": "6d6f7374", + "2494": "79696e67", + "2495": "206d65616e74", + "2496": "20526578", + "2497": "206e7574", + "2498": "206465746572", + "2499": "6e6e79", + "2500": "204c656f", + "2501": "2070696c6c6f77", + "2502": "206a6577", + "2503": "2067726577", + "2504": "2064657465726d", + "2505": "616b696e67", + "2506": "20686f6e657374", + "2507": "20706f6c696365", + "2508": "20446f6e", + "2509": "20616c6d6f7374", + "2510": "20756e64657273746f6f64", + "2511": "20576879", + "2512": "20626572", + "2513": "4a6f68", + "2514": "20736f7572", + "2515": "207368656570", + "2516": "53616d", + "2517": "2073637265", + "2518": "20706179", + "2519": "20537573", + "2520": "2062616c", + "2521": "2067656e6572", + "2522": "207468657365", + "2523": "2073656564", + "2524": "206469736170706f", + "2525": "6e6564", + "2526": "2063617272696564", + "2527": "2073746f6f64", + "2528": "2076616e", + "2529": "6974616c", + "2530": "204576656e", + "2531": "67726f756e64", + "2532": "20627265617468", + "2533": "64656e", + "2534": "207365656b", + "2535": "53746f70", + "2536": "2079656c6c6564", + "2537": "657270", + "2538": "20676173", + "2539": "646c65", + "2540": "69616e", + "2541": "20626f726564", + "2542": "206d6561646f77", + "2543": "20686f7370", + "2544": "20676f6c64", + "2545": "2064726577", + "2546": "20686f73706974616c", + "2547": "20736e61636b73", + "2548": "2073686f7265", + "2549": "68696e", + "2550": "206c696e65", + "2551": "2070697a7a", + "2552": "2073756363657373", + "2553": "206c656d", + "2554": "206b697474656e", + "2555": "6f726b", + "2556": "204c6974746c65", + "2557": "2066656172", + "2558": "207567", + "2559": "206d65737379", + "2560": "206176", + "2561": "61756c", + "2562": "2070757a7a6c65", + "2563": "766f7573", + "2564": "206469736170706f696e746564", + "2565": "207072696e6365", + "2566": "2068696464656e", + "2567": "2075676c79", + "2568": "20737061726b6c79", + "2569": "20636c617373", + "2570": "2067726f7570", + "2571": "75737472", + "2572": "6f626279", + "2573": "696666", + "2574": "20426c", + "2575": "20626c6f77", + "2576": "206372696573", + "2577": "6b696e", + "2578": "207761766573", + "2579": "6f6e79", + "2580": "20626564726f6f6d", + "2581": "6f6f70", + "2582": "2076617365", + "2583": "2066656564", + "2584": "20686f757273", + "2585": "206275726e", + "2586": "207363617266", + "2587": "206b6e6f77696e67", + "2588": "204a6f686e6e79", + "2589": "20706c616e7473", + "2590": "20736967686564", + "2591": "5768657265", + "2592": "20676c617373", + "2593": "2073616e6477696368", + "2594": "6974746572", + "2595": "206a656c6c79", + "2596": "20736c65657079", + "2597": "2065617379", + "2598": "20707269", + "2599": "204d756d6d79", + "2600": "2067656e65726f7573", + "2601": "20706f6f72", + "2602": "20706f77657266756c", + "2603": "20636c6f736574", + "2604": "207374616e64696e67", + "2605": "6c6f776572", + "2606": "204272", + "2607": "206d6f74", + "2608": "2072657475726e6564", + "2609": "2063726f776e", + "2610": "2070697a7a61", + "2611": "206b6973736564", + "2612": "2074657374", + "2613": "207469636b", + "2614": "20667275737472", + "2615": "536f727279", + "2616": "20746175676874", + "2617": "20416c696365", + "2618": "2074656e74", + "2619": "657270696c6c", + "2620": "20636174657270696c6c", + "2621": "206e6572", + "2622": "2073686970", + "2623": "2070696e", + "2624": "2066616e6379", + "2625": "2063697263", + "2626": "20706972", + "2627": "207465617273", + "2628": "4a6f686e", + "2629": "626f6479", + "2630": "2073706f696c", + "2631": "20736f6c64", + "2632": "417265", + "2633": "776f726b", + "2634": "2068616e67", + "2635": "7061636b", + "2636": "20796f757273656c66", + "2637": "20776f6c66", + "2638": "206772756d7079", + "2639": "20706f6c697465", + "2640": "20726164", + "2641": "206e6572766f7573", + "2642": "2764", + "2643": "6166", + "2644": "20666c6167", + "2645": "6f7465", + "2646": "20626f72696e67", + "2647": "6f6f66", + "2648": "207468616e6b66756c", + "2649": "696e61", + "2650": "656e73697665", + "2651": "2063686577", + "2652": "6173736564", + "2653": "206c6567", + "2654": "20706173736564", + "2655": "61726c", + "2656": "20747769726c", + "2657": "204d61726b", + "2658": "20657870656e73697665", + "2659": "206d696e75746573", + "2660": "20736372", + "2661": "2064657465726d696e6564", + "2662": "2064726f70", + "2663": "206368657374", + "2664": "2064616e63696e67", + "2665": "6973736f7273", + "2666": "2066727573747261746564", + "2667": "206f6674656e", + "2668": "20687572", + "2669": "73656c766573", + "2670": "6c69676874", + "2671": "2067617465", + "2672": "2067657473", + "2673": "20636174657270696c6c6172", + "2674": "2064656572", + "2675": "2073636973736f7273", + "2676": "20636c6f776e", + "2677": "20657363", + "2678": "206d7973746572696f7573", + "2679": "657261", + "2680": "20627269646765", + "2681": "657473", + "2682": "20726567", + "2683": "207069636b73", + "2684": "20776167676564", + "2685": "6f73617572", + "2686": "2066696e676572", + "2687": "6d62", + "2688": "204f4b", + "2689": "20657863697465", + "2690": "2066696c", + "2691": "696c6c6564", + "2692": "2077616e646572", + "2693": "206761746865726564", + "2694": "20666967", + "2695": "204b69747479", + "2696": "75616c", + "2697": "206c6f636b", + "2698": "6964646c65", + "2699": "207361766564", + "2700": "20636170", + "2701": "697a7a79", + "2702": "2064696e6f73617572", + "2703": "206d697374616b65", + "2704": "61746573", + "2705": "20776f726b696e67", + "2706": "20616e737765726564", + "2707": "20766567657461626c6573", + "2708": "20686f7070696e67", + "2709": "204e6f", + "2710": "20636f6d666f727461626c65", + "2711": "20666c7574", + "2712": "207061636b6564", + "2713": "57686f", + "2714": "6f7473", + "2715": "20626f7373", + "2716": "6f6e64", + "2717": "20646f6773", + "2718": "206d6972", + "2719": "717565", + "2720": "2074727565", + "2721": "20717565656e", + "2722": "206472756d", + "2723": "20676967676c6564", + "2724": "206b6e69676874", + "2725": "206e756d626572", + "2726": "20776865656c", + "2727": "207374756666", + "2728": "20707573", + "2729": "206c6974", + "2730": "2074616b656e", + "2731": "206c6f7365", + "2732": "206578636974656d656e74", + "2733": "756365", + "2734": "20626f6e65", + "2735": "676773", + "2736": "2064726177696e67", + "2737": "2065676773", + "2738": "20667275697473", + "2739": "2063616e64", + "2740": "20546f646179", + "2741": "2074617065", + "2742": "207275696e6564", + "2743": "206d756d6d79", + "2744": "206d65646963", + "2745": "5468657265", + "2746": "206f6e6573", + "2747": "20776973", + "2748": "20637261776c", + "2749": "20637269", + "2750": "2061736b696e67", + "2751": "2064657374", + "2752": "20426c7565", + "2753": "206d6964646c65", + "2754": "207370656e64", + "2755": "656369616c6c79", + "2756": "2073776565746965", + "2757": "20736d656c6c79", + "2758": "206c69636b6564", + "2759": "2077616c6b73", + "2760": "726f72", + "2761": "205375736965", + "2762": "206c617a79", + "2763": "206665617468657273", + "2764": "656c6c61", + "2765": "2066616b65", + "2766": "206d65646963696e65", + "2767": "20616464", + "2768": "2062617468726f6f6d", + "2769": "70656369616c6c79", + "2770": "20657370656369616c6c79", + "2771": "20676c7565", + "2772": "75676172", + "2773": "206578616d", + "2774": "20636c6170", + "2775": "206d61746368", + "2776": "656c70", + "2777": "20726f6172", + "2778": "204966", + "2779": "206368616e6765", + "2780": "2073746f6e6573", + "2781": "696e657373", + "2782": "206272757368", + "2783": "20626f6479", + "2784": "20676172", + "2785": "206c79696e67", + "2786": "2062616c6c73", + "2787": "206e6f64", + "2788": "2063616c6c696e67", + "2789": "2064697374616e6365", + "2790": "207369676874", + "2791": "207375676172", + "2792": "206d6f76696e67", + "2793": "20796f757273", + "2794": "2073706c6173686564", + "2795": "207465616d", + "2796": "61736572", + "2797": "206e6f697379", + "2798": "2062616b65", + "2799": "20746965", + "2800": "206472616e6b", + "2801": "207765616b", + "2802": "204576656e7475616c6c79", + "2803": "20706c6561736564", + "2804": "206d6f76", + "2805": "20666c75666679", + "2806": "206c656164", + "2807": "206d6972726f72", + "2808": "20646563696465", + "2809": "206f6b", + "2810": "6861", + "2811": "206d6963", + "2812": "20686f726e", + "2813": "206d6f6d73", + "2814": "2070726179", + "2815": "4e6f77", + "2816": "206265636f6d65", + "2817": "2063616765", + "2818": "20666973", + "2819": "48656c70", + "2820": "686970", + "2821": "61676564", + "2822": "207765656b", + "2823": "206469736170706561726564", + "2824": "20636f7665726564", + "2825": "756767657374", + "2826": "20726964", + "2827": "206c656d6f6e", + "2828": "2055", + "2829": "20706c617967726f756e64", + "2830": "2072656c6178", + "2831": "2073756767657374", + "2832": "6174696e67", + "2833": "2073656c6c", + "2834": "2073686f7773", + "2835": "207361696c6564", + "2836": "20626f617264", + "2837": "65657a", + "2838": "20676c6f77", + "2839": "2077726f7465", + "2840": "20696e63726564", + "2841": "657465", + "2842": "20746f77656c", + "2843": "616c6f7573", + "2844": "20426f626279", + "2845": "2073686f6f74", + "2846": "206a65616c6f7573", + "2847": "20626f7973", + "2848": "20636c656172", + "2849": "207175657374696f6e73", + "2850": "696e6174696f6e", + "2851": "206a75696379", + "2852": "20636865636b", + "2853": "206d656173757265", + "2854": "206561676572", + "2855": "20736f6170", + "2856": "20646967", + "2857": "6f7279", + "2858": "206f7264", + "2859": "207477697374", + "2860": "20526f7365", + "2861": "2066696e65", + "2862": "20636f6e74656e74", + "2863": "206e65656473", + "2864": "2072617265", + "2865": "20736b6970706564", + "2866": "206564", + "2867": "79616c", + "2868": "20636c65616e6564", + "2869": "20696e64", + "2870": "206a6f6b65", + "2871": "20656c64", + "2872": "776c6564", + "2873": "20637261776c6564", + "2874": "696c6c69", + "2875": "206d6f746f72", + "2876": "20456c6c6965", + "2877": "20696e6372656469626c65", + "2878": "4265", + "2879": "2072696368", + "2880": "20456c6c61", + "2881": "20627562626c6573", + "2882": "2066616d6f7573", + "2883": "20676172616765", + "2884": "696d70", + "2885": "20776f7264", + "2886": "20666174686572", + "2887": "20657874", + "2888": "2042726f776e", + "2889": "20456e64", + "2890": "20747275636b73", + "2891": "206272616e63686573", + "2892": "2064697368", + "2893": "206875727473", + "2894": "2062616e616e61", + "2895": "4d65", + "2896": "697070", + "2897": "206f7267616e697a6564", + "2898": "2044616e", + "2899": "7469", + "2900": "2068617070656e696e67", + "2901": "20626f72726f77", + "2902": "20746167", + "2903": "206c6175676873", + "2904": "20657965", + "2905": "646479", + "2906": "204d7273", + "2907": "756d626c6564", + "2908": "63696c", + "2909": "6970706572", + "2910": "2077616e64", + "2911": "657279", + "2912": "2070656e63696c", + "2913": "20636f766572", + "2914": "2066726f776e", + "2915": "6775", + "2916": "206c6f79616c", + "2917": "696d6564", + "2918": "20756e7573", + "2919": "666963", + "2920": "206d6170", + "2921": "206c616d70", + "2922": "20756e757375616c", + "2923": "20666c617368", + "2924": "207265616c69736564", + "2925": "206c6f75646572", + "2926": "206a6577656c", + "2927": "20746869636b", + "2928": "207377616e", + "2929": "206775657373", + "2930": "20636869636b656e", + "2931": "2073776f7264", + "2932": "6c696573", + "2933": "207768616c65", + "2934": "69717565", + "2935": "20646f6c70", + "2936": "2073696c", + "2937": "20636f7374", + "2938": "207469676572", + "2939": "20646966666963", + "2940": "206177", + "2941": "206368617365", + "2942": "6173696e67", + "2943": "206578747261", + "2944": "206d6f64", + "2945": "206f666665726564", + "2946": "20686f7365", + "2947": "20636f6d706574", + "2948": "20666f7572", + "2949": "20636f696e73", + "2950": "20646973636f7665726564", + "2951": "63657074", + "2952": "20756e69717565", + "2953": "20666967757265", + "2954": "66617374", + "2955": "206261726e", + "2956": "706f7274", + "2957": "3f222e", + "2958": "206b6e696665", + "2959": "20626f74746f6d", + "2960": "2068756e74", + "2961": "20646966666963756c74", + "2962": "2074757274", + "2963": "2068616d", + "2964": "207368616b65", + "2965": "2062616c616e6365", + "2966": "6174656c79", + "2967": "7965", + "2968": "6b656570", + "2969": "20636f6c6f757266756c", + "2970": "206f6e6365", + "2971": "4c696c61", + "2972": "2050657465", + "2973": "20746f7567", + "2974": "48657265", + "2975": "20627265616b66617374", + "2976": "20616476656e747572", + "2977": "20616e67", + "2978": "206c6966746564", + "2979": "20656c6465726c79", + "2980": "57656c6c", + "2981": "2066616c6c73", + "2982": "20676976696e67", + "2983": "207370756e", + "2984": "2062657272696573", + "2985": "206164646564", + "2986": "726167", + "2987": "207765697264", + "2988": "207374726177", + "2989": "206865726f", + "2990": "20666f6f6c", + "2991": "2cc3a2e282ac", + "2992": "20626174746572", + "2993": "2062616c6c6f6f6e73", + "2994": "6c616e64", + "2995": "204d79", + "2996": "20616476656e7475726f7573", + "2997": "61676573", + "2998": "20626f786573", + "2999": "206c6f67", + "3000": "206b696e646e657373", + "3001": "65727279", + "3002": "2068656c7073", + "3003": "2077726170706564", + "3004": "656d6f", + "3005": "796d", + "3006": "2062616c64", + "3007": "20706f696e74696e67", + "3008": "20736d6f6b65", + "3009": "20736b696e", + "3010": "204e656d6f", + "3011": "20706565", + "3012": "2070756d70", + "3013": "6a656374", + "3014": "20636972636c65", + "3015": "2068617473", + "3016": "2064697a7a79", + "3017": "207368617065", + "3018": "6f6c6f67", + "3019": "20626972646965", + "3020": "75726564", + "3021": "20647265616d6564", + "3022": "206d6967687479", + "3023": "20686172646572", + "3024": "7567676c6564", + "3025": "20636f6d7065746974", + "3026": "20696d70726573736564", + "3027": "616c6b", + "3028": "2062726f7468657273", + "3029": "20626c6f6f6d", + "3030": "2067726f776e", + "3031": "6f75736573", + "3032": "2046726564", + "3033": "2065617273", + "3034": "206d616e61676564", + "3035": "206e6f726d", + "3036": "204d6172", + "3037": "20746f756768", + "3038": "2061706f6c6f67", + "3039": "20646570656e64", + "3040": "68656164", + "3041": "206c6f75646c79", + "3042": "2063686f6f7365", + "3043": "207363617265", + "3044": "2062656c6f6e676564", + "3045": "206d61726b6574", + "3046": "207265636f7264", + "3047": "204d616e6479", + "3048": "2065646765", + "3049": "6976", + "3050": "6b6565706572", + "3051": "2077726170", + "3052": "20676f6174", + "3053": "206d656c74", + "3054": "6174757265", + "3055": "206661766f7572", + "3056": "20726f756768", + "3057": "204475636b79", + "3058": "206d65616e73", + "3059": "20686f75736573", + "3060": "20616761696e7374", + "3061": "206a61636b", + "3062": "696c74", + "3063": "20726566", + "3064": "2042656c6c61", + "3065": "2063686572", + "3066": "207768697370", + "3067": "20626f737379", + "3068": "7265616479", + "3069": "20656e6a6f79696e67", + "3070": "206272696c6c69", + "3071": "206772616365", + "3072": "20667269676874", + "3073": "206e6f726d616c", + "3074": "20666174", + "3075": "20736c6970706564", + "3076": "207374756262", + "3077": "206c6561726e696e67", + "3078": "20696379", + "3079": "206661766f7572697465", + "3080": "206272696c6c69616e74", + "3081": "206861726d", + "3082": "20736561726368", + "3083": "206b6e6f7773", + "3084": "6177", + "3085": "20666f6f6c697368", + "3086": "20746572", + "3087": "204d696b65", + "3088": "207761676f6e", + "3089": "20616c7265616479", + "3090": "20667269646765", + "3091": "20726174", + "3092": "20796f67", + "3093": "616e676564", + "3094": "207368656c6c73", + "3095": "20627574746f6e73", + "3096": "2073747562626f726e", + "3097": "20636162", + "3098": "20736166656c79", + "3099": "20747572746c65", + "3100": "2073747269", + "3101": "20636c6179", + "3102": "2067756974", + "3103": "617070696e67", + "3104": "706573", + "3105": "20636162696e", + "3106": "207370616365", + "3107": "20636972636c6573", + "3108": "2063756464", + "3109": "2077616c6c73", + "3110": "207061696e74696e67", + "3111": "6f707079", + "3112": "206f6666696365", + "3113": "206c6177", + "3114": "206578636c61", + "3115": "2069736c616e64", + "3116": "20746f617374", + "3117": "6f6d6564", + "3118": "20736f756e646564", + "3119": "2077616b65", + "3120": "20647265616d73", + "3121": "206c696272", + "3122": "20426f626f", + "3123": "207374726177626572", + "3124": "206e6f74696365", + "3125": "206578636c61696d6564", + "3126": "656469656e74", + "3127": "2068617070696e657373", + "3128": "20736179696e67", + "3129": "657267", + "3130": "2063726f", + "3131": "61727468", + "3132": "206d69786564", + "3133": "20636865657266756c", + "3134": "72616765", + "3135": "20657363617065", + "3136": "20637261796f6e", + "3137": "2076616c75", + "3138": "206d6f766965", + "3139": "207472756e6b", + "3140": "20706f75726564", + "3141": "207374756479", + "3142": "61636865", + "3143": "20637261636b", + "3144": "20616363657074", + "3145": "20726f6f66", + "3146": "20736d616c6c6572", + "3147": "20626974746572", + "3148": "20547765657479", + "3149": "20646570656e6461626c65", + "3150": "2062616773", + "3151": "207465727269", + "3152": "657874", + "3153": "20726f7365", + "3154": "61646f", + "3155": "697270", + "3156": "2066696572", + "3157": "20677569746172", + "3158": "206f62656469656e74", + "3159": "20756e6c", + "3160": "20736861726b", + "3161": "2074696d6573", + "3162": "636f726e", + "3163": "2070726f75646c79", + "3164": "2064656164", + "3165": "206e6963656c79", + "3166": "206d61676e", + "3167": "206e6f7379", + "3168": "20636f70", + "3169": "6961626c65", + "3170": "206c65616e", + "3171": "20746875", + "3172": "6369656e74", + "3173": "2073706f696c6564", + "3174": "20627573686573", + "3175": "20696e7669746564", + "3176": "7363", + "3177": "20737567676573746564", + "3178": "20696e646570656e64", + "3179": "20666965726365", + "3180": "206c6f77", + "3181": "206a616d", + "3182": "2074756e6e", + "3183": "20686f6f70", + "3184": "2064656c6963617465", + "3185": "20636f756e746564", + "3186": "656e74696f6e", + "3187": "20667269676874656e6564", + "3188": "20656767", + "3189": "2075736566756c", + "3190": "206c6967687473", + "3191": "206c6f636b6564", + "3192": "2070696c", + "3193": "207765616c", + "3194": "2073746f7665", + "3195": "2073686f77696e67", + "3196": "2066696e67657273", + "3197": "207769706564", + "3198": "2044617665", + "3199": "207374726f6e676572", + "3200": "20656e67", + "3201": "20706172726f74", + "3202": "7761726473", + "3203": "2074656c65", + "3204": "207363726174", + "3205": "20646f756768", + "3206": "6b6e", + "3207": "204b696d", + "3208": "20616e6369656e74", + "3209": "2064657369676e", + "3210": "6e61", + "3211": "2063697479", + "3212": "2066726167", + "3213": "617a", + "3214": "20736861646f77", + "3215": "2072656c6961626c65", + "3216": "2073747265616d", + "3217": "697065", + "3218": "206a6f67", + "3219": "6375", + "3220": "20746f6f6c73", + "3221": "20726f6465", + "3222": "207468656d73656c766573", + "3223": "204772616e647061", + "3224": "2074756d6d79", + "3225": "2061626f7665", + "3226": "2076616c7561626c65", + "3227": "207374756d626c6564", + "3228": "2063616d", + "3229": "20636c65616e696e67", + "3230": "6f6c61", + "3231": "206b69636b6564", + "3232": "206578706c6f726564", + "3233": "207365656d", + "3234": "20657863697465646c79", + "3235": "207370696e6e696e67", + "3236": "20737765", + "3237": "206d656d", + "3238": "207961726e", + "3239": "6f726765", + "3240": "206a6f696e6564", + "3241": "20596f7572", + "3242": "72657373", + "3243": "20706f70636f726e", + "3244": "2067656d", + "3245": "20696e646570656e64656e74", + "3246": "2062756d70", + "3247": "757468", + "3248": "206265616b", + "3249": "61727279", + "3250": "2074776967", + "3251": "2074686f7567687466756c", + "3252": "20666f756e7461696e", + "3253": "20686f6c6473", + "3254": "616d6f6e64", + "3255": "20667265", + "3256": "6974686572", + "3257": "207375636365737366756c", + "3258": "20536f7068", + "3259": "207a6970", + "3260": "207573696e67", + "3261": "20666c6173686c69676874", + "3262": "696e616c", + "3263": "2077617973", + "3264": "206f77", + "3265": "206272616365", + "3266": "2072656d696e646564", + "3267": "20737465616c", + "3268": "206275696c74", + "3269": "206d6172626c65", + "3270": "6f6f74", + "3271": "2066726167696c65", + "3272": "206272696768746c79", + "3273": "206b696e6473", + "3274": "2068616c66", + "3275": "6b6e6f776e", + "3276": "726179", + "3277": "206772616e647061", + "3278": "20736f6c7665", + "3279": "206f726967", + "3280": "20656e6f726d", + "3281": "206469616d6f6e64", + "3282": "20736e696666", + "3283": "20656e6f726d6f7573", + "3284": "206c6162", + "3285": "205768656e65766572", + "3286": "20776869737065726564", + "3287": "72656c6c", + "3288": "20626f617473", + "3289": "204672", + "3290": "2073746174", + "3291": "2074616e6b", + "3292": "20646f7665", + "3293": "20636f6e65", + "3294": "65657a65", + "3295": "20636c756d", + "3296": "616e7574", + "3297": "207061696e746564", + "3298": "207072616374696365", + "3299": "20756e6b6e6f776e", + "3300": "206d6574616c", + "3301": "2072756262", + "3302": "20646f6c7068696e", + "3303": "20626f756e6365", + "3304": "207365656473", + "3305": "706964", + "3306": "2061636f726e", + "3307": "2070756d706b696e", + "3308": "20636f7374756d", + "3309": "20746869727374", + "3310": "206f726967696e616c", + "3311": "20736572696f7573", + "3312": "2062656573", + "3313": "20656e76656c", + "3314": "20617267", + "3315": "2061646d69726564", + "3316": "656f726765", + "3317": "756d62657273", + "3318": "20656e63", + "3319": "20776173686564", + "3320": "696e617279", + "3321": "2073756e66", + "3322": "2066696e64696e67", + "3323": "204c6f6c61", + "3324": "2062656574", + "3325": "20737461697273", + "3326": "20536e", + "3327": "206368656170", + "3328": "206a6f6c6c79", + "3329": "20706f7374", + "3330": "20706c617966756c", + "3331": "20737475706964", + "3332": "206368616e676564", + "3333": "206f70656e73", + "3334": "2061756e74", + "3335": "2073686f6e65", + "3336": "486f77", + "3337": "20616c6c6f776564", + "3338": "20646f65736e", + "3339": "696c7479", + "3340": "206775696c7479", + "3341": "20706562626c65", + "3342": "6d6272656c6c", + "3343": "20746f6d61746f", + "3344": "2064656c696768746564", + "3345": "206265616e", + "3346": "20796574", + "3347": "20756d6272656c6c", + "3348": "2073616c6164", + "3349": "20666c757465", + "3350": "20636c756d7379", + "3351": "7572696e67", + "3352": "207069636b696e67", + "3353": "20736c656570696e67", + "3354": "20636c6f636b", + "3355": "696e6573", + "3356": "636f64", + "3357": "2064756c6c", + "3358": "2068656c6d", + "3359": "696374", + "3360": "76696365", + "3361": "2073746f7073", + "3362": "20666f727468", + "3363": "206775617264", + "3364": "7567676564", + "3365": "20656e67696e65", + "3366": "2073756974", + "3367": "646f6d", + "3368": "6c6c6970", + "3369": "6c6c69706f70", + "3370": "206d6f646572", + "3371": "2074686972737479", + "3372": "20746f72", + "3373": "2070696765", + "3374": "616e676572", + "3375": "696c6c61", + "3376": "206c656176696e67", + "3377": "206d697373696e67", + "3378": "2064757374", + "3379": "207765616c746879", + "3380": "21222e", + "3381": "20686179", + "3382": "636f6d", + "3383": "20616e67656c", + "3384": "206c6f6c6c69706f70", + "3385": "69736b", + "3386": "206172726f77", + "3387": "207261636564", + "3388": "20736f6e", + "3389": "2063726f636f64", + "3390": "2063616d657261", + "3391": "69656e6365", + "3392": "207465727269626c65", + "3393": "206672657368", + "3394": "207065616e7574", + "3395": "206e756d62657273", + "3396": "686574", + "3397": "206d6973", + "3398": "204a6f6579", + "3399": "207374616d70", + "3400": "2068616d6d6572", + "3401": "2073696e6b", + "3402": "696577", + "3403": "20727562", + "3404": "2074686f7365", + "3405": "20706f7461746f", + "3406": "2074656c6c696e67", + "3407": "2063686f7365", + "3408": "20727573686564", + "3409": "20696d7072657373", + "3410": "205065", + "3411": "6174696f6e73", + "3412": "20636f6d70617373", + "3413": "20636f6d62", + "3414": "20546f6279", + "3415": "20666c6578", + "3416": "206861697279", + "3417": "2064657374726f", + "3418": "20626172626572", + "3419": "2073686f636b6564", + "3420": "206d6f6465726e", + "3421": "2063657265", + "3422": "65657a6564", + "3423": "6f6f6d79", + "3424": "2076696577", + "3425": "204f6e6365", + "3426": "5265", + "3427": "207065656b6564", + "3428": "67657374", + "3429": "20436c", + "3430": "2072657363", + "3431": "2074656d", + "3432": "207370696c6c6564", + "3433": "476f", + "3434": "2068616e646564", + "3435": "6167686574", + "3436": "61676865747469", + "3437": "20706f70756c6172", + "3438": "207368656c", + "3439": "20737461636b", + "3440": "20746f6f7468", + "3441": "2070656e6775", + "3442": "536565", + "3443": "20736561726368696e67", + "3444": "20746f77657273", + "3445": "20676f6f7365", + "3446": "206d61726368", + "3447": "207072696e74", + "3448": "2063686172", + "3449": "206d6f6c65", + "3450": "206a7567", + "3451": "20727562626564", + "3452": "20636f6d707574", + "3453": "20706972617465", + "3454": "206a61636b6574", + "3455": "726f6265", + "3456": "20656e64696e67", + "3457": "20736e65", + "3458": "207361756365", + "3459": "206e75727365", + "3460": "20506174", + "3461": "2062696767657374", + "3462": "206465736b", + "3463": "20626565746c65", + "3464": "2064726976696e67", + "3465": "20696e67", + "3466": "206b6e6f636b", + "3467": "20737061676865747469", + "3468": "2077696e67", + "3469": "20636172656c657373", + "3470": "20667269656e6473686970", + "3471": "206d61736b", + "3472": "206c6f6164", + "3473": "207374616765", + "3474": "20647265616d696e67", + "3475": "2062726163656c6574", + "3476": "696f", + "3477": "206e6174757265", + "3478": "207468756e646572", + "3479": "207472757468", + "3480": "20677261636566756c", + "3481": "206361727279696e67", + "3482": "207072697a65", + "3483": "20666c657869626c65", + "3484": "206572", + "3485": "74696d65", + "3486": "2061766f", + "3487": "72656469656e74", + "3488": "20736c69646573", + "3489": "20696e6772656469656e74", + "3490": "61727368", + "3491": "207769746368", + "3492": "20706174746572", + "3493": "20416c6578", + "3494": "20637572", + "3495": "206372616e65", + "3496": "2073636174746572", + "3497": "6973696f6e", + "3498": "2070726570", + "3499": "20737461747565", + "3500": "207363616c65", + "3501": "206d757368", + "3502": "20726567756c6172", + "3503": "20676173706564", + "3504": "2070616e74", + "3505": "6661", + "3506": "20656e76656c6f7065", + "3507": "20776865656c73", + "3508": "2047656f726765", + "3509": "2074756e6e656c", + "3510": "2064656c69766572", + "3511": "2077697265", + "3512": "20737175617265", + "3513": "207061747465726e", + "3514": "2074696564", + "3515": "20736869766572", + "3516": "2070657273697374", + "3517": "20636f6d70617373696f6e", + "3518": "20706f77", + "3519": "207472617368", + "3520": "206372617a79", + "3521": "206c697a", + "3522": "2062756c6c", + "3523": "206561727468", + "3524": "206a756d7073", + "3525": "204e657874", + "3526": "2063657265616c", + "3527": "20726577617264", + "3528": "2073637265656e", + "3529": "20506f6c6c79", + "3530": "20616476696365", + "3531": "2068616e646c65", + "3532": "20756d6272656c6c61", + "3533": "206561726c79", + "3534": "207468696e6b696e67", + "3535": "20746f726e", + "3536": "686f7374", + "3537": "206d65616c", + "3538": "20696d7072657373697665", + "3539": "4865", + "3540": "206d656c", + "3541": "2077696e646f7773", + "3542": "54686579", + "3543": "4f77", + "3544": "20656e76", + "3545": "6963686573", + "3546": "204c6f6f6b", + "3547": "207065726d", + "3548": "656c6c6967", + "3549": "20737472616e676572", + "3550": "6c6f6f", + "3551": "2042656172", + "3552": "20696d6167696e6564", + "3553": "20727567", + "3554": "20737469636b657273", + "3555": "20696e74656c6c6967", + "3556": "2070656e6e79", + "3557": "2067686f7374", + "3558": "20637265617465", + "3559": "2063726f636f64696c65", + "3560": "207069616e6f", + "3561": "20707570", + "3562": "4966", + "3563": "2062656e74", + "3564": "20696e6772656469656e7473", + "3565": "656d656e74", + "3566": "c3a2e282ace280", + "3567": "206e6f7465", + "3568": "207761726e6564", + "3569": "2077616666", + "3570": "206772616e64", + "3571": "636f6d666f7274", + "3572": "2074726176656c", + "3573": "696b6573", + "3574": "2048657265", + "3575": "20636f6d7075746572", + "3576": "696262", + "3577": "20756e636c65", + "3578": "2073706f6b65", + "3579": "206d61696c626f78", + "3580": "61776e", + "3581": "20706172616465", + "3582": "206d757368726f6f6d", + "3583": "2073757266", + "3584": "20737061726b6c6564", + "3585": "74746572", + "3586": "20737072696e67", + "3587": "696c6c6965", + "3588": "2070616c65", + "3589": "636f6d666f727461626c65", + "3590": "206168656164", + "3591": "206c696272617279", + "3592": "20666f726b", + "3593": "2064657374726f79", + "3594": "2067726f77696e67", + "3595": "20636f6c6c6172", + "3596": "2063656c656272617465", + "3597": "20706572666f726d", + "3598": "207175657374696f6e", + "3599": "206f72646572", + "3600": "20756e636f6d666f727461626c65", + "3601": "20656e7468", + "3602": "20686f6f6b", + "3603": "206e616d6573", + "3604": "2073756e666c6f776572", + "3605": "206261646c79", + "3606": "6163686573", + "3607": "20646973636f766572", + "3608": "20636f6d706c6574", + "3609": "20636f756e74696e67", + "3610": "2062656570", + "3611": "636f61", + "3612": "65627261", + "3613": "206f626a656374", + "3614": "20736f727473", + "3615": "2074726170", + "3616": "20656e74687573", + "3617": "2073616e64776963686573", + "3618": "20416e6479", + "3619": "204772616365", + "3620": "20636f6d7065746974697665", + "3621": "2072656164696e67", + "3622": "20627574746572666c696573", + "3623": "206d696c6c", + "3624": "696e6365", + "3625": "2068756d626c65", + "3626": "2073756464656e", + "3627": "206c697a617264", + "3628": "616964", + "3629": "20626561757479", + "3630": "207461736b", + "3631": "20626c7565626572", + "3632": "20736f6661", + "3633": "206861726d6c657373", + "3634": "20666c6f7572", + "3635": "697073", + "3636": "207061746368", + "3637": "206d6179", + "3638": "2068756e67", + "3639": "207361646c79", + "3640": "20637570626f617264", + "3641": "20656e636f75", + "3642": "20636f6f6b696e67", + "3643": "2070686f74", + "3644": "2070657273697374656e74", + "3645": "20637269636b6574", + "3646": "20536f70686965", + "3647": "206f74746572", + "3648": "2072656365", + "3649": "2070726574656e64696e67", + "3650": "20736e7567676c6564", + "3651": "2063616d70", + "3652": "207a65627261", + "3653": "596179", + "3654": "20706967656f6e", + "3655": "6f72616e74", + "3656": "2072756c6573", + "3657": "206c657474657273", + "3658": "2073746174696f6e", + "3659": "2073687574", + "3660": "20756e69766572", + "3661": "2068756e746572", + "3662": "206e656174", + "3663": "206361726473", + "3664": "2068697070", + "3665": "637574", + "3666": "2054776565746965", + "3667": "206f7264696e617279", + "3668": "736573", + "3669": "20676972", + "3670": "20706c61636564", + "3671": "20636f636f61", + "3672": "7564", + "3673": "2042726f776e6965", + "3674": "206e757473", + "3675": "726963", + "3676": "20647567", + "3677": "2072656675736564", + "3678": "6c63616e6f", + "3679": "20636861726d696e67", + "3680": "20696e74656c6c6967656e74", + "3681": "205061756c", + "3682": "20676c6f6f6d79", + "3683": "2042696c6c", + "3684": "20616e78", + "3685": "2047697665", + "3686": "205368", + "3687": "206e61696c", + "3688": "2067756d", + "3689": "206f6c646572", + "3690": "67757374", + "3691": "206c65617368", + "3692": "2062616e64616765", + "3693": "2069676e6f72616e74", + "3694": "207375706572", + "3695": "20747261636b", + "3696": "20747769726c6564", + "3697": "204261", + "3698": "207370656564", + "3699": "2064697367757374", + "3700": "206368616e6365", + "3701": "20676966746564", + "3702": "20726573697374", + "3703": "626c6564", + "3704": "616666", + "3705": "20556e", + "3706": "6f65", + "3707": "69636573", + "3708": "20737461727473", + "3709": "207961776e", + "3710": "2062617365", + "3711": "2063757274", + "3712": "20616e78696f7573", + "3713": "20676f72", + "3714": "204a65727279", + "3715": "2063686173696e67", + "3716": "206375727461696e", + "3717": "206c6564", + "3718": "20706f6e79", + "3719": "696566", + "3720": "20616e797768657265", + "3721": "20747269616e67", + "3722": "20636c7562", + "3723": "63746f70", + "3724": "63746f707573", + "3725": "207061737461", + "3726": "20637265617475726573", + "3727": "2073657474", + "3728": "68747562", + "3729": "20647261776572", + "3730": "20766f6c63616e6f", + "3731": "20747269636b", + "3732": "47697665", + "3733": "2062617468747562", + "3734": "2073686f757473", + "3735": "2076616c7565", + "3736": "2073656e64", + "3737": "4d6178", + "3738": "207075727365", + "3739": "2061747472616374", + "3740": "20706570706572", + "3741": "20737175617368", + "3742": "207368616b696e67", + "3743": "206665617266756c", + "3744": "20736f6c64696572", + "3745": "206f63746f707573", + "3746": "2073636f6f74", + "3747": "20747572", + "3748": "20616c657274", + "3749": "63656564", + "3750": "20636f6d706c", + "3751": "206465636f72617465", + "3752": "2064656166", + "3753": "20636172746f6f6e", + "3754": "20616c6c6967", + "3755": "206372797374", + "3756": "206d7567", + "3757": "206675", + "3758": "206a6574", + "3759": "617269616e", + "3760": "637265616d", + "3761": "75626279", + "3762": "20696d706174", + "3763": "2070616e7473", + "3764": "20417265", + "3765": "2063617270", + "3766": "207368696e65", + "3767": "20536d", + "3768": "20737469636b6572", + "3769": "206272696768746572", + "3770": "657261626c65", + "3771": "206875727279", + "3772": "73747269", + "3773": "756e64", + "3774": "427965", + "3775": "546f6d6d79", + "3776": "616b6564", + "3777": "2066696c746879", + "3778": "20626f756e636564", + "3779": "20696d70617469656e74", + "3780": "2074697265", + "3781": "205768657265", + "3782": "2070726573736564", + "3783": "20707261637469636564", + "3784": "20636f7572616765", + "3785": "206d6973657261626c65", + "3786": "2070756c6c73", + "3787": "2064697367757374696e67", + "3788": "206c6976656c79", + "3789": "20737570706f7274", + "3790": "757262", + "3791": "2063617365", + "3792": "207475726b6579", + "3793": "2064697374757262", + "3794": "20686970706f", + "3795": "69657374", + "3796": "2074726179", + "3797": "20656e76696f7573", + "3798": "20636f6d70617373696f6e617465", + "3799": "20726164696f", + "3800": "206e6565646c65", + "3801": "20737472657463686564", + "3802": "207472756d70", + "3803": "20636875626279", + "3804": "206261726b696e67", + "3805": "20726573746c657373", + "3806": "207370656c6c", + "3807": "20697473656c66", + "3808": "2062696379", + "3809": "206772616273", + "3810": "2068656c6d6574", + "3811": "20676f72696c6c61", + "3812": "2074656e", + "3813": "206261636b7061636b", + "3814": "204c6565", + "3815": "2068656c706c657373", + "3816": "204d696c6c6965", + "3817": "2061667465726e", + "3818": "20666f7274", + "3819": "2073706c6974", + "3820": "2061667465726e6f6f6e", + "3821": "2073756363656564", + "3822": "20736f6d657768657265", + "3823": "20626574", + "3824": "204265666f7265", + "3825": "20627269", + "3826": "20636f7773", + "3827": "20736b6970", + "3828": "2073616c74", + "3829": "20636172706574", + "3830": "206e6f626f6479", + "3831": "20636f6d6574", + "3832": "69636f70", + "3833": "69636f70746572", + "3834": "207374657073", + "3835": "54696d6d79", + "3836": "2063617473", + "3837": "6767696e67", + "3838": "204275646479", + "3839": "207669", + "3840": "206a656570", + "3841": "20526f", + "3842": "206d656c6f6e", + "3843": "206174746963", + "3844": "20616e79776179", + "3845": "736572", + "3846": "20746872656164", + "3847": "2073616e64626f78", + "3848": "2070617773", + "3849": "20706f656d", + "3850": "2064657373", + "3851": "2070616e64", + "3852": "206e6f7768657265", + "3853": "207265737065637466756c", + "3854": "2068656c69636f70746572", + "3855": "206d656173", + "3856": "2061747472616374697665", + "3857": "2066697665", + "3858": "206d61676e6574", + "3859": "2076696e65", + "3860": "2076696465", + "3861": "617572616e74", + "3862": "2066697265776f726b", + "3863": "20636f6f6b6564", + "3864": "20776973686573", + "3865": "207472756d706574", + "3866": "756d6d6572", + "3867": "676e", + "3868": "2072657374617572616e74", + "3869": "2042756e6e79", + "3870": "206f7267616e697a65", + "3871": "2061646d697265", + "3872": "20676967676c65", + "3873": "2072657065", + "3874": "20736f7274", + "3875": "206578706c61696e", + "3876": "206c696f6e73", + "3877": "20756e7061636b", + "3878": "2070756e", + "3879": "737461697273", + "3880": "2053616e6479", + "3881": "2062617272656c", + "3882": "20707573686573", + "3883": "206d6f6e6b657973", + "3884": "206f73747269", + "3885": "20616e7473", + "3886": "206c6964", + "3887": "20436861726c", + "3888": "20526564", + "3889": "2073686f65", + "3890": "207265636f", + "3891": "6f70686f6e65", + "3892": "20736869656c64", + "3893": "20616c6c696761746f72", + "3894": "656e7279", + "3895": "20746170", + "3896": "6f7261626c65", + "3897": "2064697374616e74", + "3898": "207061636b616765", + "3899": "2072696365", + "3900": "206861727368", + "3901": "2073747265746368", + "3902": "20726176656e", + "3903": "2068616972637574", + "3904": "207363726577", + "3905": "65626f6f6b", + "3906": "207365616c", + "3907": "204d61726961", + "3908": "206e6f7465626f6f6b", + "3909": "205472", + "3910": "206d756666", + "3911": "706172", + "3912": "20666c65", + "3913": "2066696e6473", + "3914": "2045766572797468696e67", + "3915": "20656c6576", + "3916": "207365706172", + "3917": "206c656f70", + "3918": "2073746972726564", + "3919": "20426574", + "3920": "2063757368", + "3921": "207468696566", + "3922": "6173746963", + "3923": "20747261696e73", + "3924": "69736b657273", + "3925": "6865617274", + "3926": "61726f6f", + "3927": "616e6761726f6f", + "3928": "2072696262", + "3929": "20736f6172", + "3930": "206d617468", + "3931": "206865646765", + "3932": "2061646f7261626c65", + "3933": "2066757274686572", + "3934": "2073776561746572", + "3935": "20686f6c6573", + "3936": "20656c657661746f72", + "3937": "2077616c6c6574", + "3938": "2066726f776e6564", + "3939": "206272656174", + "3940": "20646973686573", + "3941": "20666972656d616e", + "3942": "206d6f64657374", + "3943": "696e6b696e67", + "3944": "2073656e74", + "3945": "2071756172", + "3946": "20737769746368", + "3947": "20576869736b657273", + "3948": "656e6e79", + "3949": "20536e6f77", + "3950": "20736c65", + "3951": "2068617070696572", + "3952": "20676f616c", + "3953": "206b616e6761726f6f", + "3954": "6e69676874", + "3955": "2068616c6c", + "3956": "207368656574", + "3957": "2074726970706564", + "3958": "207572", + "3959": "636f6d65", + "3960": "2068756d616e", + "3961": "20737461726564", + "3962": "20416e6e6965", + "3963": "2062696379636c65", + "3964": "73746963", + "3965": "206a6f6b6573", + "3966": "206265617273", + "3967": "2063616e646c65", + "3968": "20736f636b", + "3969": "2074726f75626c6564", + "3970": "2064657373657274", + "3971": "206c697374656e696e67", + "3972": "6379636c65", + "3973": "207768656174", + "3974": "656c657279", + "3975": "20656e657267", + "3976": "756265", + "3977": "2073697a65", + "3978": "2061766f63", + "3979": "2062696b6573", + "3980": "20696d70726f", + "3981": "206d696372", + "3982": "206372797374616c", + "3983": "20636861696e", + "3984": "676965", + "3985": "696173746963", + "3986": "20726570616972", + "3987": "20656e74687573696173746963", + "3988": "207377696e67696e67", + "3989": "2064726177696e6773", + "3990": "206c6970", + "3991": "206f737472696368", + "3992": "20526f73", + "3993": "20656974686572", + "3994": "2057696c6c", + "3995": "206d656174", + "3996": "206578706572", + "3997": "206c696768746e696e67", + "3998": "6369", + "3999": "20666f7267617665", + "4000": "697070696e67", + "4001": "20536d697468", + "4002": "20726964696e67", + "4003": "2068697665", + "4004": "20737065616b", + "4005": "20636865636b6564", + "4006": "206c6f6f70", + "4007": "206368616c6b", + "4008": "2063617264626f617264", + "4009": "636f6e64", + "4010": "20726573637565", + "4011": "656572", + "4012": "206d6963726f70686f6e65", + "4013": "63636f6c", + "4014": "206c656f70617264", + "4015": "20726f61726564", + "4016": "207365636f6e64", + "4017": "63636f6c69", + "4018": "756374", + "4019": "204f72", + "4020": "2072616674", + "4021": "20626568617665", + "4022": "2073696c766572", + "4023": "61756c6966", + "4024": "207265636f676e", + "4025": "20706170657273", + "4026": "2070616964", + "4027": "2073746f6d", + "4028": "2074756265", + "4029": "2073756d6d6572", + "4030": "2070616c", + "4031": "2063756265", + "4032": "2070726f76", + "4033": "207368616b6573", + "4034": "207175697465", + "4035": "2062727573", + "4036": "72656369", + "4037": "206f62736572", + "4038": "69726f6e", + "4039": "686f6f64", + "4040": "2073686f76656c", + "4041": "207361696c6f72", + "4042": "2073636f6c64", + "4043": "6f6c696e", + "4044": "20626f6f7473", + "4045": "206d657373616765", + "4046": "63617374", + "4047": "2062757a7a", + "4048": "2073636f6f746572", + "4049": "207368656c746572", + "4050": "2061707072656369", + "4051": "20736f66746c79", + "4052": "2063656c657279", + "4053": "206f6174", + "4054": "6f706879", + "4055": "206665656c696e6773", + "4056": "20657261736572", + "4057": "2063616374", + "4058": "206d616d61", + "4059": "20616c697665", + "4060": "6861707079", + "4061": "20436861726c6965", + "4062": "206a656c6c7966697368", + "4063": "20636f756e747279", + "4064": "2070697065", + "4065": "2073616e6463617374", + "4066": "206f79", + "4067": "206d61727279", + "4068": "207065656b", + "4069": "20706564", + "4070": "2074756e61", + "4071": "20617661696c", + "4072": "20696e6b", + "4073": "2042756e", + "4074": "20576f756c64", + "4075": "204d6167", + "4076": "2062696e", + "4077": "206361756c6966", + "4078": "2070656e6775696e", + "4079": "69726974", + "4080": "20696e7365", + "4081": "2074726f706879", + "4082": "20706f72", + "4083": "72656e646572", + "4084": "2070757474696e67", + "4085": "206261736b657462616c6c", + "4086": "206261726b73", + "4087": "666565", + "4088": "2079616368", + "4089": "706c6179", + "4090": "20627265616b73", + "4091": "2070757368696e67", + "4092": "62756c", + "4093": "20736872696d70", + "4094": "2062656c74", + "4095": "726f6e", + "4096": "20706f6c65", + "4097": "75636b6564", + "4098": "20726f64", + "4099": "2066616c6c696e67", + "4100": "20636c617070696e67", + "4101": "207374617274696e67", + "4102": "2048656e7279", + "4103": "2062726f63636f6c69", + "4104": "20706f696e74", + "4105": "20617661696c61626c65", + "4106": "66666565", + "4107": "79656172", + "4108": "2073776565746865617274", + "4109": "2070616c616365", + "4110": "20747269616e676c65", + "4111": "207a6f6f6d6564", + "4112": "20747265617375726573", + "4113": "2073617573", + "4114": "206f6c", + "4115": "2067726f", + "4116": "207265696e64", + "4117": "20656e65726779", + "4118": "204b617465", + "4119": "656479", + "4120": "20676976656e", + "4121": "205275", + "4122": "20636163747573", + "4123": "2076697369746564", + "4124": "20666c696573", + "4125": "2073686f706b6565706572", + "4126": "2070616e6461", + "4127": "20656e646564", + "4128": "206c61756e64", + "4129": "20636f6c6f757273", + "4130": "20756e6861707079", + "4131": "206c61756e647279", + "4132": "206469736167", + "4133": "65726f70", + "4134": "20737072696e6b", + "4135": "65726f706c", + "4136": "20636f7264", + "4137": "206d696c64", + "4138": "206c656172", + "4139": "20636f7374756d65", + "4140": "206165726f706c", + "4141": "206c6561726e74", + "4142": "20727562626572", + "4143": "20676f6c66", + "4144": "206361756c69666c6f776572", + "4145": "206a6f75726e616c", + "4146": "656361757365", + "4147": "20617274", + "4148": "207368616465", + "4149": "20646973706c6179", + "4150": "20677261706573", + "4151": "6f6f706564", + "4152": "206a75646765", + "4153": "20506170", + "4154": "20627572", + "4155": "207961636874", + "4156": "20706c616e6574", + "4157": "656e7365", + "4158": "207261696e696e67", + "4159": "204a657373", + "4160": "20726167", + "4161": "20746178", + "4162": "6566", + "4163": "2062756c", + "4164": "204861727279", + "4165": "2064697665", + "4166": "207265696e64656572", + "4167": "616974", + "4168": "20636c696d62696e67", + "4169": "7365756d", + "4170": "20686f70696e67", + "4171": "206c75636b", + "4172": "616c6564", + "4173": "20747269636b73", + "4174": "206170706c", + "4175": "206d656e", + "4176": "4d6961", + "4177": "61666665", + "4178": "206b6e6f74", + "4179": "20736d656c6c73", + "4180": "2076656e64", + "4181": "20737069726974", + "4182": "4861", + "4183": "20666f6f7462616c6c", + "4184": "20706f6c697368", + "4185": "206d757365756d", + "4186": "2073686f7274", + "4187": "2070696c6f74", + "4188": "204c696c6c79", + "4189": "2062616e616e6173", + "4190": "20686f74656c", + "4191": "20666f6c646572", + "4192": "20696e636c", + "4193": "20756e6c6f636b", + "4194": "2073796d", + "4195": "2071756974", + "4196": "206c61736572", + "4197": "2062756c62", + "4198": "20636f6f6c6572", + "4199": "20737472756767", + "4200": "207468726f7773", + "4201": "69636572", + "4202": "2042616279", + "4203": "73697374", + "4204": "207072657373", + "4205": "63636572", + "4206": "206f6e696f6e", + "4207": "617a7a", + "4208": "20776f6f6c", + "4209": "206861646e", + "4210": "20706164", + "4211": "7265657a65", + "4212": "20496e7374", + "4213": "2073747566666564", + "4214": "20636f66666565", + "4215": "206d617263686564", + "4216": "2076696f6c696e", + "4217": "2063757073", + "4218": "206672616d65", + "4219": "206e65696768626f7572", + "4220": "206b6179", + "4221": "676573", + "4222": "20736f63636572", + "4223": "2064726f7073", + "4224": "2050617061", + "4225": "20616d62756c", + "4226": "2074617869", + "4227": "206c617779", + "4228": "206f696c", + "4229": "207468616e6b73", + "4230": "2066617563", + "4231": "6d616e64", + "4232": "204d697373", + "4233": "206c65616e6564", + "4234": "626f6c", + "4235": "73656c79", + "4236": "6e676572", + "4237": "206c656e64", + "4238": "2070616972", + "4239": "2073796d626f6c", + "4240": "7a7a79", + "4241": "446164", + "4242": "2067697261666665", + "4243": "206c6965", + "4244": "2073686f756c646e", + "4245": "20737465616b", + "4246": "2073757272656e646572", + "4247": "206361626c65", + "4248": "20726f6c6c696e67", + "4249": "20616d62756c616e6365", + "4250": "697a6573", + "4251": "206d69786572", + "4252": "20696d6167696e65", + "4253": "696c696e67", + "4254": "20736b696c6c", + "4255": "7065", + "4256": "207175617272656c", + "4257": "207377756e67", + "4258": "20666175636574", + "4259": "20666973686572", + "4260": "20636f756e746572", + "4261": "2062656c6f6e6773", + "4262": "2073747261776265727279", + "4263": "20636f6d706c657465", + "4264": "69746573", + "4265": "6e616d", + "4266": "206669656c6473", + "4267": "20706f696e7473", + "4268": "20636c6f73656c79", + "4269": "206576656e696e67", + "4270": "206865616473", + "4271": "20616972706f7274", + "4272": "2076656e646f72", + "4273": "20706c61796572", + "4274": "206c69766573", + "4275": "696d69", + "4276": "2e2e", + "4277": "20747562", + "4278": "2070686f746f", + "4279": "20736f696c", + "4280": "706c65", + "4281": "20776176696e67", + "4282": "20706f77646572", + "4283": "2066696c6d", + "4284": "20696d6167696e6174696f6e", + "4285": "68696e6f", + "4286": "2061706f6c6f67697a6564", + "4287": "7765656e", + "4288": "206d756c65", + "4289": "2066726f6773", + "4290": "20696e73697374", + "4291": "20706f726368", + "4292": "206265747765656e", + "4293": "61636573", + "4294": "2063757368696f6e", + "4295": "207065616368", + "4296": "2072656566", + "4297": "207061696e746572", + "4298": "20747261696c", + "4299": "65617468", + "4300": "20706c6173746963", + "4301": "20526f736965", + "4302": "2074686561746572", + "4303": "20686f636b", + "4304": "20686f636b6579", + "4305": "6e6579", + "4306": "20696e73656374", + "4307": "2069726f6e", + "4308": "20736c657074", + "4309": "6563746564", + "4310": "20626c65", + "4311": "2062616b696e67", + "4312": "20636865656b", + "4313": "207268696e6f", + "4314": "6b656c6574", + "4315": "20666f72676f7474656e", + "4316": "20666c6561", + "4317": "204a616d6573", + "4318": "206469617279", + "4319": "2063726561746564", + "4320": "63746f7279", + "4321": "206368697270", + "4322": "c3a2e282ace2809c", + "4323": "2077696665", + "4324": "206d6167617a", + "4325": "207265766572", + "4326": "2070697261746573", + "4327": "75726368", + "4328": "2065616765726c79", + "4329": "206178", + "4330": "6e616d656e74", + "4331": "20617474656e74696f6e", + "4332": "20736972", + "4333": "2070616765", + "4334": "73706170", + "4335": "20706572666563746c79", + "4336": "20546564", + "4337": "206368657272696573", + "4338": "2074656d706c65", + "4339": "20666163746f7279", + "4340": "71756974", + "4341": "2077697065", + "4342": "2077696e746572", + "4343": "596f7572", + "4344": "20496e7374656164", + "4345": "206e657773706170", + "4346": "756573", + "4347": "6175736564", + "4348": "206d696e757465", + "4349": "206f726e616d656e74", + "4350": "206d6f746f726379636c65", + "4351": "206c696d65", + "4352": "2044616e6e79", + "4353": "206165726f706c616e65", + "4354": "2073686f7574696e67", + "4355": "4a696d", + "4356": "206172677565", + "4357": "204d696d69", + "4358": "20736e65616b", + "4359": "20766574", + "4360": "206c6177796572", + "4361": "2073756e736574", + "4362": "207072756e", + "4363": "20726f74", + "4364": "20736565696e67", + "4365": "6b656c65746f6e", + "4366": "2066697368696e67", + "4367": "717569746f", + "4368": "20796f67757274", + "4369": "20626c6f77696e67", + "4370": "206669736865726d616e", + "4371": "616e79", + "4372": "6f72696573", + "4373": "204d6167676965", + "4374": "697070657279", + "4375": "20676c6f77696e67", + "4376": "2068657262", + "4377": "20706f6b65", + "4378": "2073707261796564", + "4379": "20546f6e79", + "4380": "206b6e6f62", + "4381": "20736c697070657279", + "4382": "20706974", + "4383": "20687574", + "4384": "206b6179616b", + "4385": "20626c616e6b657473", + "4386": "206d6f73", + "4387": "207374616666", + "4388": "2073756e67", + "4389": "20706574616c73", + "4390": "207a6970706572", + "4391": "20617865", + "4392": "20686f706573", + "4393": "6c617373", + "4394": "206d656574696e67", + "4395": "20736b756c6c", + "4396": "69736f6e", + "4397": "206967", + "4398": "616e6963", + "4399": "2067726f636572", + "4400": "2069676c6f6f", + "4401": "207477696e6b", + "4402": "2062757279", + "4403": "20737175697272656c73", + "4404": "6c6c6965", + "4405": "2073686f776572", + "4406": "20677265656479", + "4407": "6368616972", + "4408": "6974757265", + "4409": "2073747564", + "4410": "75726e6974757265", + "4411": "206f66666572", + "4412": "20706172656e74", + "4413": "204162", + "4414": "2073656173", + "4415": "616e6775", + "4416": "20636869636b656e73", + "4417": "6368616e6963", + "4418": "657263", + "4419": "206675726e6974757265", + "4420": "206e6f76656c", + "4421": "206d6f73717569746f", + "4422": "206c616e6775", + "4423": "416e64", + "4424": "2062617474657279", + "4425": "20666c616d65", + "4426": "20646f6f7273", + "4427": "2077656174686572", + "4428": "20627265657a65", + "4429": "20686561646564", + "4430": "206b69747479", + "4431": "20736b656c65746f6e", + "4432": "206d656368616e6963", + "4433": "2077616773", + "4434": "2076657374", + "4435": "206d797374657279", + "4436": "206d6f766573", + "4437": "2061766f6361646f", + "4438": "2061726d6368616972", + "4439": "20667265657a", + "4440": "20686172", + "4441": "6d656e", + "4442": "2068617070656e73", + "4443": "4a61636b", + "4444": "20636f616c", + "4445": "2074656172", + "4446": "20766964656f", + "4447": "20676f6c64656e", + "4448": "2063616c6c73", + "4449": "2063616e65", + "4450": "2068616e67696e67", + "4451": "706c616365", + "4452": "7265656b", + "4453": "2063656c65627261746564", + "4454": "656c636f6d65", + "4455": "20637265656b", + "4456": "20636f62", + "4457": "206d6564616c", + "4458": "204d6179", + "4459": "2076726f6f6d", + "4460": "73696e67", + "4461": "2070617373706f7274", + "4462": "20746f696c", + "4463": "2070696e65", + "4464": "206578657263", + "4465": "20476f64", + "4466": "20676c6f7665", + "4467": "2077726974696e67", + "4468": "206e61706b696e", + "4469": "206b696e67646f6d", + "4470": "6365726f73", + "4471": "20736e6966666564", + "4472": "6d6d", + "4473": "20737469636b79", + "4474": "2077656c636f6d6564", + "4475": "776173", + "4476": "4d616d61", + "4477": "2063726962", + "4478": "2073686f756c646572", + "4479": "20706f70706564", + "4480": "206d617a65", + "4481": "206f7973746572", + "4482": "2069746368", + "4483": "696d6e6579", + "4484": "206469736375", + "4485": "2064656172", + "4486": "204b6174", + "4487": "776562", + "4488": "6c6173736573", + "4489": "617564", + "4490": "20666f6c6c6f7773", + "4491": "696e617269616e", + "4492": "65746572696e617269616e", + "4493": "776173686572", + "4494": "20746f696c6574", + "4495": "7472657373", + "4496": "206261736562616c6c", + "4497": "206368696d6e6579", + "4498": "2073756e676c6173736573", + "4499": "206d61747472657373", + "4500": "617773", + "4501": "4d756d", + "4502": "20706173747279", + "4503": "206275726e6564", + "4504": "206d656d6f72696573", + "4505": "20636875726368", + "4506": "207665746572696e617269616e", + "4507": "206772696e", + "4508": "2064697368776173686572", + "4509": "2070697374", + "4510": "207765696768", + "4511": "207268696e6f6365726f73", + "4512": "20656174656e", + "4513": "207472617665", + "4514": "207368727567676564", + "4515": "206c6f74696f6e", + "4516": "2073717565", + "4517": "204a756c", + "4518": "207468726f77696e67", + "4519": "20706973746f6c", + "4520": "2069676e6f726564", + "4521": "206d697374616b6573", + "4522": "206d616b657570", + "4523": "20636f62776562", + "4524": "756e61", + "4525": "20626f74686572", + "4526": "206361726564", + "4527": "206f616b", + "4528": "756e6174656c79", + "4529": "20737572726f756e64", + "4530": "20436c617261", + "4531": "2061677265", + "4532": "2073696e6365", + "4533": "20717561636b", + "4534": "2072657665", + "4535": "2066697265706c616365", + "4536": "206170706c617564", + "4537": "6865726f", + "4538": "726f64", + "4539": "20636c617373726f6f6d", + "4540": "2073757065726865726f", + "4541": "6c6f77", + "4542": "57616974", + "4543": "207665696c", + "4544": "20536565", + "4545": "6f72736573", + "4546": "20666c6170706564", + "4547": "4d6f6c6c79", + "4548": "726f7363", + "4549": "2054727565", + "4550": "207365706172617465", + "4551": "207361696c696e67", + "4552": "20696e74726f64", + "4553": "206c61776e", + "4554": "20666f67", + "4555": "2063726f77", + "4556": "7269676874", + "4557": "7461696e", + "4558": "2064616e636572", + "4559": "726f73636f7065", + "4560": "20747567", + "4561": "20686f72736573", + "4562": "204669727374", + "4563": "20666f75676874", + "4564": "69646179", + "4565": "20696e76657374", + "4566": "20776f727468", + "4567": "63617365", + "4568": "2071756965746c79", + "4569": "6d656469", + "4570": "676779", + "4571": "207377616d70", + "4572": "7465", + "4573": "206e6577737061706572", + "4574": "6d6564696174656c79", + "4575": "2073697a6573", + "4576": "737369626c65", + "4577": "766973696f6e", + "4578": "6c6572", + "4579": "206d6963726f73636f7065", + "4580": "20636f6c6c6563746564", + "4581": "6162", + "4582": "206a756e6b", + "4583": "5175", + "4584": "20536164", + "4585": "20696e766573746967", + "4586": "61736973", + "4587": "6f726573", + "4588": "20737472656e67", + "4589": "206d696e6572", + "4590": "42656361757365", + "4591": "20726567726574", + "4592": "696e6b6564", + "4593": "206167726565", + "4594": "20636f6d706c61696e", + "4595": "20736b697274", + "4596": "2062656566", + "4597": "20626f756e63696e67", + "4598": "2070726573656e7473", + "4599": "2070726576656e74", + "4600": "20737472656e677468", + "4601": "206365696c696e67", + "4602": "20666c6f6174", + "4603": "657479", + "4604": "2063656c6c", + "4605": "2063686f696365", + "4606": "20707564646c6573", + "4607": "206f61736973", + "4608": "20636172696e67", + "4609": "2048656c70", + "4610": "20637261776c696e67", + "4611": "207374726169676874", + "4612": "20707572", + "4613": "20747574", + "4614": "446964", + "4615": "2073717565657a6564", + "4616": "20636861697273", + "4617": "206170617274", + "4618": "206d6f70", + "4619": "616e676573", + "4620": "2073636f6c646564", + "4621": "20686f6c", + "4622": "206163636570746564", + "4623": "20686f6c69646179", + "4624": "2063616e646c6573", + "4625": "20617474616368", + "4626": "20616c61726d", + "4627": "2074656c65766973696f6e", + "4628": "69766573", + "4629": "2074656c657068", + "4630": "206a6577656c7279", + "4631": "2077686973706572", + "4632": "20796f6761", + "4633": "63686965", + "4634": "20636f617473", + "4635": "4973", + "4636": "2074727573746564", + "4637": "207969656c64", + "4638": "207469636b6574", + "4639": "20696e66", + "4640": "72696e67", + "4641": "20636173746c6573", + "4642": "20726f6f6d73", + "4643": "206c69636b73", + "4644": "2064656c6179", + "4645": "206d6167617a696e65", + "4646": "20696e766573746967617465", + "4647": "2064656d616e64", + "4648": "2077617264", + "4649": "6f6d6573", + "4650": "20756e6966", + "4651": "20536e6f7779", + "4652": "2074656c6570686f6e65", + "4653": "2077656570", + "4654": "686f757365", + "4655": "6374696f6e", + "4656": "206b6973736573", + "4657": "20736e6170", + "4658": "206c616e6775616765", + "4659": "2077617264726f6265", + "4660": "20617070726f", + "4661": "2067796d", + "4662": "2068617262", + "4663": "2072616b65", + "4664": "204c75636b", + "4665": "20737465616d", + "4666": "207365727665", + "4667": "2064726573736564", + "4668": "20756e69666f726d", + "4669": "20737562", + "4670": "2073696768", + "4671": "206d75737461636865", + "4672": "2072656163686573", + "4673": "20707261", + "4674": "20737461626c65", + "4675": "20636162696e6574", + "4676": "2057686f", + "4677": "5468616e6b73", + "4678": "206d617272696564", + "4679": "20626c6f6f64", + "4680": "2068757274696e67", + "4681": "206368657373", + "4682": "20537465", + "4683": "2073717565657a65", + "4684": "206372617368", + "4685": "204c75636b696c79", + "4686": "20656e7465726564", + "4687": "20726164697368", + "4688": "20626c61636b626f617264", + "4689": "6976616c", + "4690": "2077696767", + "4691": "20626c696e6b6564", + "4692": "206c657474696e67", + "4693": "20667265657a6572", + "4694": "206d617463686573", + "4695": "696374696f6e", + "4696": "2077616e6465726564", + "4697": "206e657773", + "4698": "20467265646479", + "4699": "757073", + "4700": "20656173696c79", + "4701": "207468756d62", + "4702": "2072656c61786564", + "4703": "2073756666", + "4704": "696374696f6e617279", + "4705": "207374726f", + "4706": "20626f6f6b63617365", + "4707": "206b656570696e67", + "4708": "207975636b79", + "4709": "207370656172", + "4710": "20636c617773", + "4711": "2073706c617368696e67", + "4712": "2072657175", + "4713": "2054696e61", + "4714": "2045616368", + "4715": "206f70657261", + "4716": "2072697365", + "4717": "206269726463", + "4718": "20616d617a656d656e74", + "4719": "2068656c706572", + "4720": "2064696374696f6e617279", + "4721": "66756d65", + "4722": "206269726463616765", + "4723": "20776166666c65", + "4724": "20636c696666", + "4725": "20626c756562657272696573", + "4726": "2063726f73736564", + "4727": "20746f6e67", + "4728": "20646973617070656172", + "4729": "616c6c6579", + "4730": "63616b6573", + "4731": "2062696c6c", + "4732": "206d7573696369616e", + "4733": "2070657266756d65", + "4734": "2073616e64636173746c65", + "4735": "20686172626f72", + "4736": "2067726f776c6564", + "4737": "20486f7765766572", + "4738": "20736872696e6b", + "4739": "736974", + "4740": "207261696c", + "4741": "2072617a", + "4742": "6f6d65646179", + "4743": "20696e637265", + "4744": "2042757a7a", + "4745": "207475636b6564", + "4746": "6168", + "4747": "207472616465", + "4748": "6973696e67", + "4749": "7261676564", + "4750": "2076616c6c6579", + "4751": "206472696564", + "4752": "63686f72", + "4753": "7374657273", + "4754": "20706c616e6573", + "4755": "206265727279", + "4756": "20746f726e61646f", + "4757": "2070757a7a6c6573", + "4758": "204f6c6c6965", + "4759": "207761737465", + "4760": "2072657175657374", + "4761": "20696d70726f7665", + "4762": "206261746865", + "4763": "20726962626f6e", + "4764": "2070656172", + "4765": "20776f726d73", + "4766": "2066657374", + "4767": "206b6e65656c", + "4768": "207475746f72", + "4769": "20636f6e66", + "4770": "686963", + "4771": "20706c61746573", + "4772": "2062656c6f77", + "4773": "206d696e74", + "4774": "20686561727473", + "4775": "206170726f6e", + "4776": "20626c656564", + "4777": "207369", + "4778": "207265667265", + "4779": "206669726566", + "4780": "207665686963", + "4781": "756368", + "4782": "20626c6f757365", + "4783": "2056", + "4784": "2070696e6368", + "4785": "206c756d626572", + "4786": "2073706f7473", + "4787": "2069736e", + "4788": "20736361747465726564", + "4789": "6d656e64", + "4790": "616368656c", + "4791": "20766163", + "4792": "2064726573736573", + "4793": "2052756279", + "4794": "204d65", + "4795": "20706f6b6564", + "4796": "2066616c6c656e", + "4797": "206772696c6c", + "4798": "616d706f6f", + "4799": "69766564", + "4800": "20746f6573", + "4801": "206675656c", + "4802": "20616e63686f72", + "4803": "20617574", + "4804": "2061736874", + "4805": "2063616c65", + "4806": "20736f6d65646179", + "4807": "2061736874726179", + "4808": "2063616c656e64", + "4809": "2072617a6f72", + "4810": "666f7274756e6174656c79", + "4811": "2077696e65", + "4812": "2063616c656e646172", + "4813": "6f6d6d656e64", + "4814": "20737572707269736573", + "4815": "5975636b", + "4816": "20626567676564", + "4817": "20737570706c79", + "4818": "207265636f6d6d656e64", + "4819": "206d7973656c66", + "4820": "20626f6f74", + "4821": "20666f6c6c6f77696e67", + "4822": "206c6963", + "4823": "2062756d706564", + "4824": "20696d6d6564696174656c79", + "4825": "2052616368656c", + "4826": "20696e737472", + "4827": "206964656173", + "4828": "2074616c6c6572", + "4829": "20466f", + "4830": "206c656d6f6e616465", + "4831": "20736b696c6c73", + "4832": "20637562", + "4833": "20737472617762657272696573", + "4834": "206361707461696e", + "4835": "20736e6170706564", + "4836": "20646966666572656e", + "4837": "207368616d706f6f", + "4838": "617269", + "4839": "207370656e64696e67", + "4840": "6c696572", + "4841": "2064657374726f796564", + "4842": "2072656d6f7665", + "4843": "20637564646c6564", + "4844": "207761726d7468", + "4845": "20736e65657a65", + "4846": "2070726f70", + "4847": "206c6175676874", + "4848": "2072696e67696e67", + "4849": "20657870657269656e6365", + "4850": "20736166657479", + "4851": "206d617373", + "4852": "20756e6465727374616e64696e67", + "4853": "696e6b79", + "4854": "756374757265", + "4855": "2066696c65", + "4856": "57656c636f6d65", + "4857": "206d6f6d656e7473", + "4858": "20737472756374757265", + "4859": "207261697365", + "4860": "207961776e6564", + "4861": "6f72616c", + "4862": "277665", + "4863": "206c61756768746572", + "4864": "2072657665727365", + "4865": "204265747379", + "4866": "20627572696564", + "4867": "20656e746572", + "4868": "206265616e73", + "4869": "2072616262697473", + "4870": "20706f7274", + "4871": "204c697a", + "4872": "2062726176656c79", + "4873": "20726f77", + "4874": "6963696e67", + "4875": "207370616465", + "4876": "2077656170", + "4877": "62626965", + "4878": "766574", + "4879": "61706564", + "4880": "206b6e6f776e", + "4881": "7269676572", + "4882": "206d756666696e", + "4883": "207265667269676572", + "4884": "2050656f706c65", + "4885": "20726566726967657261746f72", + "4886": "20666c7574746572", + "4887": "20666573746976616c", + "4888": "20736176696e67", + "4889": "20436f", + "4890": "616d626572", + "4891": "2073697473", + "4892": "2069676e6f7265", + "4893": "206a6577656c73", + "4894": "20626f6f6b736865", + "4895": "206c756767", + "4896": "207061676573", + "4897": "20726f6f74", + "4898": "697373696f6e", + "4899": "2064697265", + "4900": "20736f636b73", + "4901": "2062616c616e", + "4902": "2066617374657374", + "4903": "207065726d697373696f6e", + "4904": "272e", + "4905": "20506f707079", + "4906": "2077617368696e67", + "4907": "20686172646c79", + "4908": "20636c696d6273", + "4909": "206c756767616765", + "4910": "20706c617973", + "4911": "20737061726b6c696e67", + "4912": "206d697373696c65", + "4913": "2068697473", + "4914": "204b61746965", + "4915": "2062726561746865", + "4916": "2068616e676572", + "4917": "205370696b65", + "4918": "2061707072656369617465", + "4919": "2063686f726573", + "4920": "206265616d6564", + "4921": "206465636f726174696f6e73", + "4922": "2070726570617265", + "4923": "20737061726b6c65", + "4924": "6f6d6f62", + "4925": "20726879", + "4926": "206561726c696572", + "4927": "207465616368696e67", + "4928": "2048617665", + "4929": "206c6963656e7365", + "4930": "20686f6d6573", + "4931": "20616972706c", + "4932": "20746f6e677565", + "4933": "20497473", + "4934": "2064657374696e6174696f6e", + "4935": "20626f6f6b7368656c66", + "4936": "205a6f65", + "4937": "206175746f6d6f62", + "4938": "20746f7265", + "4939": "2050656e6e79", + "4940": "20736574746c6564", + "4941": "207265706c616365", + "4942": "20696e76697465", + "4943": "204a6f79", + "4944": "2076616e6973686564", + "4945": "6f736564", + "4946": "20647261696e", + "4947": "20736f6c766564", + "4948": "2073746f6d616368", + "4949": "207175697a", + "4950": "20646565706572", + "4951": "616e6e61", + "4952": "206461697379", + "4953": "2072656d61696e", + "4954": "2074696465", + "4955": "2061646d6972", + "4956": "4f6b", + "4957": "2062616e67", + "4958": "206d656e74696f6e", + "4959": "2073636f6f706564", + "4960": "207661636174696f6e", + "4961": "2067656172", + "4962": "204d696c6c79", + "4963": "2068696c6c73", + "4964": "20756e697665727365", + "4965": "20616c7269676874", + "4966": "2070726169736564", + "4967": "726d", + "4968": "2070617469656e746c79", + "4969": "20696e66616e74", + "4970": "20737562776179", + "4971": "204368697270", + "4972": "2046726f", + "4973": "20636865727279", + "4974": "20706f69736f6e", + "4975": "5370", + "4976": "2070756c6c696e67", + "4977": "206372756e6368", + "4978": "207369676e73", + "4979": "7874757265", + "4980": "2073686564", + "4981": "204576657279646179", + "4982": "206578616d696e65", + "4983": "6f7073", + "4984": "20746865726d", + "4985": "207570737461697273", + "4986": "206c6162656c", + "4987": "20726573746f7265", + "4988": "207072696465", + "4989": "2062616b", + "4990": "2064696767", + "4991": "546f", + "4992": "656e7473", + "4993": "2073636f7265", + "4994": "65726f", + "4995": "2064696767696e67", + "4996": "206f72616e676573", + "4997": "20737061636573", + "4998": "20666c6f61746564", + "4999": "2062656c6965766564", + "5000": "206163686965", + "5001": "20656e636f757261676564", + "5002": "206368616d626572", + "5003": "2073686f6f74696e67", + "5004": "6e696573", + "5005": "20737061726b6c6573", + "5006": "20696e736973746564", + "5007": "2073637261746368", + "5008": "206d61696c6d616e", + "5009": "206275636b6c65", + "5010": "20476f6f64", + "5011": "2062757a7a696e67", + "5012": "20646966666572656e6365", + "5013": "204d6174", + "5014": "20746865726d6f6d6574", + "5015": "20736577", + "5016": "20737479", + "5017": "2070616c6d", + "5018": "2070756e697368", + "5019": "626579", + "5020": "206769667473", + "5021": "20726573706f6e73", + "5022": "207265636569766564", + "5023": "2073617573616765", + "5024": "204e7574", + "5025": "2073696d70", + "5026": "2072616e6368", + "5027": "2064617265", + "5028": "617370", + "5029": "74686d", + "5030": "616e746c79", + "5031": "20677579", + "5032": "206368697270696e67", + "5033": "2068657273", + "5034": "207374727567676c696e67", + "5035": "206974656d", + "5036": "2070616e6963", + "5037": "206578657263697365", + "5038": "206a61696c", + "5039": "616c6c656e", + "5040": "73697479", + "5041": "204973", + "5042": "20657175", + "5043": "67696e67", + "5044": "696761696c", + "5045": "20746865726d6f6d65746572", + "5046": "6f73697479", + "5047": "2073686f74", + "5048": "20666c6f6174696e67", + "5049": "207465617365", + "5050": "2062696c6c626f617264", + "5051": "2077697a", + "5052": "5468657365", + "5053": "62626c6573", + "5054": "206d616c6c", + "5055": "20617070726f6163686564", + "5056": "2070726f76696465", + "5057": "206368616c6c656e", + "5058": "7879", + "5059": "206d656f776564", + "5060": "20636f6c6c656374696f6e", + "5061": "2072687974686d", + "5062": "69756d", + "5063": "2062617468726f6265", + "5064": "616c6c6f77", + "5065": "20696e766974", + "5066": "72697365", + "5067": "2068617465", + "5068": "206e6f6e65", + "5069": "20627261696e", + "5070": "206269746573", + "5071": "20736e65657a6564", + "5072": "20737570706f736564", + "5073": "20696e636c7564", + "5074": "20736f7265", + "5075": "20726570656174", + "5076": "20756e697465", + "5077": "20656c657068616e7473", + "5078": "206d6f64656c", + "5079": "206d65617375726564", + "5080": "4d6d6d", + "5081": "20737072696e6b6c6573", + "5082": "61746368", + "5083": "63746c79", + "5084": "20666c6176", + "5085": "2064697363757373", + "5086": "207061747465726e73", + "5087": "204162696761696c", + "5088": "6779", + "5089": "20776f727365", + "5090": "20627269636b73", + "5091": "207761726e696e67", + "5092": "2077697a617264", + "5093": "207765656b73", + "5094": "2070696c6c6f7773", + "5095": "2073756e72697365", + "5096": "20676c6f", + "5097": "756465", + "5098": "4d756d6d79", + "5099": "206175746f6d6f62696c65", + "5100": "2073657474696e67", + "5101": "2074776973746564", + "5102": "206d6172626c6573", + "5103": "66696564", + "5104": "20676c6f776564", + "5105": "206c65616b", + "5106": "2077617463686573", + "5107": "206e65696768626f72686f6f64", + "5108": "20756e6976657273697479", + "5109": "636c6573", + "5110": "2067726f776c", + "5111": "206a617a7a", + "5112": "205374657665", + "5113": "206c69636b", + "5114": "2067726579", + "5115": "206d617373616765", + "5116": "206a6f6767696e67", + "5117": "7368696e67", + "5118": "207061727473", + "5119": "206865736974", + "5120": "206f706572", + "5121": "20636f6c6c656374696e67", + "5122": "20706f707079", + "5123": "206761696e", + "5124": "20736970", + "5125": "204b696e67", + "5126": "69657273", + "5127": "206c657473", + "5128": "20666978696e67", + "5129": "61736573", + "5130": "2068656174", + "5131": "2073706974", + "5132": "20737566666572", + "5133": "2073746164", + "5134": "2073706f7274", + "5135": "207374616469756d", + "5136": "2077686970", + "5137": "20617765", + "5138": "206e6174696f6e", + "5139": "20536f6d656f6e65", + "5140": "2070696773", + "5141": "206769726c73", + "5142": "2067757374", + "5143": "207365617473", + "5144": "2063616b6573", + "5145": "206272696e6773", + "5146": "20737061636573686970", + "5147": "206d6973746572", + "5148": "206772696e6e6564", + "5149": "20667574", + "5150": "20747261636b73", + "5151": "2075726765", + "5152": "20646973636f76657279", + "5153": "206f757273", + "5154": "4e6f74", + "5155": "206d656c746564", + "5156": "20667574757265", + "5157": "204c756e61", + "5158": "617363", + "5159": "20436174", + "5160": "206772617065", + "5161": "2076656869636c65", + "5162": "20616972706c616e65", + "5163": "2063686972", + "5164": "2061637475616c6c79", + "5165": "6173706f6f6e", + "5166": "2070726f626c656d73", + "5167": "2074656173706f6f6e", + "5168": "2063686972706564", + "5169": "4f756368", + "5170": "576f756c64", + "5171": "20706175736564", + "5172": "20556e666f7274756e6174656c79", + "5173": "206576656e74", + "5174": "4a696d6d79", + "5175": "206275726e696e67", + "5176": "2072757368", + "5177": "206d616e616765", + "5178": "20736c696365", + "5179": "2127", + "5180": "20626567", + "5181": "756d626c65", + "5182": "20737472616e67", + "5183": "656564", + "5184": "656d656d626572", + "5185": "2072656d61696e6564", + "5186": "2053686172696e67", + "5187": "20776561706f6e", + "5188": "20736166617269", + "5189": "20746872696c6c6564", + "5190": "20746f7572", + "5191": "206d61726368696e67", + "5192": "2073637265616d73", + "5193": "20736f6c6469657273", + "5194": "62657273", + "5195": "697368696e67", + "5196": "206c616964", + "5197": "207a6970706564", + "5198": "2072757374", + "5199": "20737572726f756e646564", + "5200": "20736372617463686564", + "5201": "646f", + "5202": "204a65", + "5203": "207768657265766572", + "5204": "20776166666c6573", + "5205": "206472696e6b696e67", + "5206": "2076656c", + "5207": "206465636973696f6e", + "5208": "2067726f63657279", + "5209": "206d656d6f7279", + "5210": "20647572696e67", + "5211": "2072656365697665", + "5212": "204a616d", + "5213": "20706574746564", + "5214": "20636865776564", + "5215": "206f726465726564", + "5216": "206472696e6b73", + "5217": "20436869727079", + "5218": "20706f737369626c65", + "5219": "2076656c766574", + "5220": "6f766564", + "5221": "2073706f6f6e73", + "5222": "2073637572", + "5223": "5265616479", + "5224": "2073696e6773", + "5225": "21222c", + "5226": "206d697474656e", + "5227": "20666c6f6f64", + "5228": "2065617274687175", + "5229": "20636172746f6f6e73", + "5230": "20726163696e67", + "5231": "20776f726b73", + "5232": "2062656174", + "5233": "204e75747479", + "5234": "4a696c6c", + "5235": "20456c", + "5236": "2077686973746c6564", + "5237": "426f62", + "5238": "206f627365727665", + "5239": "20737472616e67657273", + "5240": "2070757070", + "5241": "20656666", + "5242": "20646973616772656564", + "5243": "20636972637573", + "5244": "59756d", + "5245": "207061696e7473", + "5246": "20726169736564", + "5247": "2077617465726564", + "5248": "2053756e64", + "5249": "20746f65", + "5250": "206c656374", + "5251": "20736c6964696e67", + "5252": "206375726c6564", + "5253": "2053756e646179", + "5254": "206c696c79", + "5255": "20687572726963", + "5256": "206372756e636879", + "5257": "207370696c6c", + "5258": "20707261796564", + "5259": "2062656473", + "5260": "206c616e6473", + "5261": "207374726970", + "5262": "2078", + "5263": "20696e7669746174696f6e", + "5264": "2063726173686564", + "5265": "206174746163686564", + "5266": "5368", + "5267": "20706f7461746f6573", + "5268": "2066617363", + "5269": "6679", + "5270": "206c656374757265", + "5271": "204e6f74", + "5272": "63696f7573", + "5273": "20636f6e7461696e", + "5274": "207761726e", + "5275": "207477696773", + "5276": "204a616e", + "5277": "20416e6e65", + "5278": "207761697473", + "5279": "20776f6e64", + "5280": "2077686973746c696e67", + "5281": "77616c6b", + "5282": "2052656d79", + "5283": "2066697265776f726b73", + "5284": "5265616c6c79", + "5285": "206f6c697665", + "5286": "20687572726963616e65", + "5287": "206861746564", + "5288": "6974696573", + "5289": "207377696d73", + "5290": "68657374", + "5291": "2073756e6c69676874", + "5292": "69706d656e74", + "5293": "207361746973", + "5294": "20646f676779", + "5295": "206f7065726174696f6e", + "5296": "20556e636c65", + "5297": "2065717569706d656e74", + "5298": "2053756e6e79", + "5299": "206d6f75746873", + "5300": "2072656d656d", + "5301": "20736b697070696e67", + "5302": "207265636f676e697a6564", + "5303": "2072656d656d62657273", + "5304": "207065616e757473", + "5305": "69646564", + "5306": "20726566726573686564", + "5307": "62626c6564", + "5308": "205361646c79", + "5309": "61766964", + "5310": "57696c6c", + "5311": "206d75646479", + "5312": "20746f6d61746f6573", + "5313": "20636f6e74696e7565", + "5314": "206d6f756e7461696e73", + "5315": "20777265", + "5316": "2068697374", + "5317": "20636f6d6679", + "5318": "207072696365", + "5319": "207369646577616c6b", + "5320": "7365656b", + "5321": "20626f6e6573", + "5322": "20666574", + "5323": "20617070656172", + "5324": "616e6e79", + "5325": "2074616c656e74", + "5326": "206275696c64696e6773", + "5327": "20626c75656265727279", + "5328": "206d696e6572616c", + "5329": "2061646d6972696e67", + "5330": "2066617363696e", + "5331": "20686973746f7279", + "5332": "20676c6974746572", + "5333": "206b696e646c79", + "5334": "205175", + "5335": "2065736361706564", + "5336": "20636c656172696e67", + "5337": "6363", + "5338": "20666f6c646564", + "5339": "2067617a", + "5340": "20626573", + "5341": "20666c6170", + "5342": "6d6f74686572", + "5343": "206d6561737572696e67", + "5344": "2061706f6c6f67697a65", + "5345": "20736978", + "5346": "222c", + "5347": "20666564", + "5348": "20627965", + "5349": "206c656174686572", + "5350": "20706c616e746564", + "5351": "207761697374", + "5352": "20656173696572", + "5353": "206f626579", + "5354": "2065617274687175616b65", + "5355": "2073656372657473", + "5356": "206d656f77", + "5357": "2073656c6c696e67", + "5358": "207072657061726564", + "5359": "20736c6970", + "5360": "207475726e696e67", + "5361": "206e617679", + "5362": "207472656d62", + "5363": "416c", + "5364": "206f6e7761726473", + "5365": "206d6f6e7374657273", + "5366": "205072", + "5367": "20626c656564696e67", + "5368": "206c697073", + "5369": "20627269636b", + "5370": "736f6e", + "5371": "7a6167", + "5372": "2074726170706564", + "5373": "206272696e67696e67", + "5374": "204c75636b79", + "5375": "20436872", + "5376": "206566666f7274", + "5377": "206d656e75", + "5378": "2077616e74696e67", + "5379": "20776572656e", + "5380": "204d6f72616c", + "5381": "2061636f726e73", + "5382": "20676f7474656e", + "5383": "2062726964", + "5384": "20776569676874", + "5385": "206c61647962", + "5386": "206861707069657374", + "5387": "7273", + "5388": "61646564", + "5389": "2067756573736564", + "5390": "20616c6c6f77", + "5391": "206f617473", + "5392": "204461766964", + "5393": "20626573696465", + "5394": "20706564616c", + "5395": "20676c6f766573", + "5396": "20637564646c65", + "5397": "20636f6d70616e79", + "5398": "2072657665616c6564", + "5399": "20616e7974696d65", + "5400": "617368696f6e", + "5401": "20736e6f7762616c6c", + "5402": "53706f74", + "5403": "20776f6e64657273", + "5404": "20776964", + "5405": "6a6f796564", + "5406": "205261", + "5407": "20416e", + "5408": "206761696e6564", + "5409": "2064696e6f7361757273", + "5410": "20776973646f6d", + "5411": "206f666669636572", + "5412": "2062726964676573", + "5413": "207261696e6564", + "5414": "207374796c65", + "5415": "20726573746564", + "5416": "20636f70706572", + "5417": "20737563636565646564", + "5418": "206f7665726a6f796564", + "5419": "6e6965", + "5420": "2070726563696f7573", + "5421": "20736574746c65", + "5422": "2070657473", + "5423": "69677a6167", + "5424": "2053757a79", + "5425": "206576657279646179", + "5426": "204f6c", + "5427": "656e7479", + "5428": "2042657468", + "5429": "206b6565706572", + "5430": "20636f6d666f72746564", + "5431": "20736861646f7773", + "5432": "20737465656c", + "5433": "207072696e746572", + "5434": "2073696d706c65", + "5435": "204672616e6b", + "5436": "2073747269706573", + "5437": "69646573", + "5438": "20736572766564", + "5439": "206272616e64", + "5440": "206361746368696e67", + "5441": "207072756e65", + "5442": "20706565726564", + "5443": "20696d616765", + "5444": "20667265657a65", + "5445": "616d62", + "5446": "2063757474696e67", + "5447": "20777269746573", + "5448": "2068757272696564", + "5449": "204a6f7368", + "5450": "20726970706564", + "5451": "2072656d696e646572", + "5452": "2053757265", + "5453": "20466c6f707079", + "5454": "20737465616c696e67", + "5455": "207768617465766572", + "5456": "2066617368696f6e", + "5457": "204279", + "5458": "20626f776564", + "5459": "2073636f6f70", + "5460": "204a616d6965", + "5461": "206372657074", + "5462": "207065726d6974", + "5463": "20776f6f64656e", + "5464": "207069746368", + "5465": "20636f7374756d6573", + "5466": "2072616e67", + "5467": "20736f61726564", + "5468": "20766f69636573", + "5469": "207669736974696e67", + "5470": "20726573706f6e7369626c65", + "5471": "2065617473", + "5472": "69636f726e", + "5473": "20756e69636f726e", + "5474": "2070696573", + "5475": "2073686f7070696e67", + "5476": "206d696365", + "5477": "2070657263686564", + "5478": "6c6f65", + "5479": "20707570696c", + "5480": "20416d", + "5481": "20746173746573", + "5482": "20616e7377657273", + "5483": "2070616e63616b6573", + "5484": "20657861", + "5485": "20627265616b696e67", + "5486": "616e7461", + "5487": "2062727573686564", + "5488": "2076696c6c6167", + "5489": "206e6574776f726b", + "5490": "67656e", + "5491": "206561676c65", + "5492": "206465636f7261746564", + "5493": "20646f776e737461697273", + "5494": "20537061726b", + "5495": "2061677265656d656e74", + "5496": "69676779", + "5497": "20666972656669676874", + "5498": "697261", + "5499": "20656d657267", + "5500": "204b6179", + "5501": "20637572696f73697479", + "5502": "207361646e657373", + "5503": "2054696e79", + "5504": "207065656c", + "5505": "204b65", + "5506": "206578616d696e6564", + "5507": "69656e", + "5508": "206372656174696f6e", + "5509": "20536f6d657468696e67", + "5510": "6f6d696e67", + "5511": "2064726167", + "5512": "204a756c6965", + "5513": "787967656e", + "5514": "20656d657267656e", + "5515": "2062756e", + "5516": "2064656369646573", + "5517": "2062656e65617468", + "5518": "206f6c69766573", + "5519": "2043686c6f65", + "5520": "206e61696c73", + "5521": "206f776e6564", + "5522": "20636f6d706c65", + "5523": "206163636f6d", + "5524": "205468657365", + "5525": "2063686566", + "5526": "6973686573", + "5527": "20747769726c696e67", + "5528": "2070616464", + "5529": "20736e61696c", + "5530": "2066616d696c", + "5531": "42696c6c79", + "5532": "20617267756564", + "5533": "697a696e67", + "5534": "20746970", + "5535": "2073706964657273", + "5536": "207374727567676c6564", + "5537": "64696e67", + "5538": "20426967", + "5539": "6d626c65", + "5540": "20646f63746f7273", + "5541": "206163636964656e7473", + "5542": "20756e64657267726f756e64", + "5543": "20737469746368", + "5544": "204c756c", + "5545": "2076696374", + "5546": "4461646479", + "5547": "20646973747572626564", + "5548": "7272", + "5549": "726f6c", + "5550": "6e65617468", + "5551": "536865", + "5552": "6c6579", + "5553": "207374696e67", + "5554": "20636c617073", + "5555": "20506970", + "5556": "207265616c697a65", + "5557": "2077656967686564", + "5558": "2073697475", + "5559": "2070726163746963696e67", + "5560": "2076696c6c6167657273", + "5561": "7661", + "5562": "6c61697265", + "5563": "746572646179", + "5564": "206c656d6f6e73", + "5565": "2053616e7461", + "5566": "2062656e64", + "5567": "2064696564", + "5568": "207269646573", + "5569": "206163746976", + "5570": "617973", + "5571": "2063617065", + "5572": "206c616479627567", + "5573": "6c656e", + "5574": "20736f616b", + "5575": "2079656c6c696e67", + "5576": "2070617374656c", + "5577": "20777265636b", + "5578": "206163636f6d706c", + "5579": "616c6573", + "5580": "207275696e", + "5581": "207a6f6f6d696e67", + "5582": "20626567696e", + "5583": "2061726561", + "5584": "6d617265", + "5585": "20427562626c6573", + "5586": "20666972656d656e", + "5587": "2072656c65617365", + "5588": "204c696b65", + "5589": "207472656d626c65", + "5590": "207265636f676e697a65", + "5591": "207a69677a6167", + "5592": "5965", + "5593": "2072656c65", + "5594": "697374656e", + "5595": "20656e636f7572616765", + "5596": "206b65657073", + "5597": "2061676f", + "5598": "66666963", + "5599": "2048616e6e61", + "5600": "2077616767696e67", + "5601": "20426972", + "5602": "20736e65616b6564", + "5603": "2053746f70", + "5604": "2065786163746c79", + "5605": "2062656861", + "5606": "206d697373696f6e", + "5607": "206b657973", + "5608": "20747567676564", + "5609": "206e696768746d617265", + "5610": "206b6e656c74", + "5611": "2074726166666963", + "5612": "206f787967656e", + "5613": "206772616e646d6f74686572", + "5614": "2054696c6c79", + "5615": "206368616c6c656e6765", + "5616": "2077696e6e696e67", + "5617": "6962", + "5618": "20466f72", + "5619": "20736c6170706564", + "5620": "2070756e6973686564", + "5621": "206c6f6f7365", + "5622": "4f72", + "5623": "67657273", + "5624": "20526f62", + "5625": "204c75", + "5626": "2062756e6e696573", + "5627": "206f7574646f", + "5628": "20636f6e74726f6c", + "5629": "2e27", + "5630": "20736869766572696e67", + "5631": "206e69636572", + "5632": "20776f6e646572696e67", + "5633": "20706c616e6e6564", + "5634": "20436c61697265", + "5635": "206a6f67676564", + "5636": "2062617265", + "5637": "2064656e74", + "5638": "6173686564", + "5639": "476f6f64627965", + "5640": "2070696e63686564", + "5641": "2067756e73", + "5642": "2066617363696e61746564", + "5643": "6c6f6f6b696e67", + "5644": "206974656d73", + "5645": "2073747265746368696e67", + "5646": "206472617773", + "5647": "2063617574", + "5648": "20736972656e", + "5649": "206e65696768626f7273", + "5650": "207061726b6564", + "5651": "206c616e647363", + "5652": "204c756c75", + "5653": "206c696e6573", + "5654": "20736f6c7574", + "5655": "2073747564696564", + "5656": "20436f636f", + "5657": "20676174686572696e67", + "5658": "20706f636b657473", + "5659": "206275636b657473", + "5660": "20756e6c6f636b6564", + "5661": "20736174697366696564", + "5662": "477261", + "5663": "20436172", + "5664": "207a65726f", + "5665": "20776562", + "5666": "2070696c6573", + "5667": "2074756e65", + "5668": "206d7573636c6573", + "5669": "7269616765", + "5670": "2065786974", + "5671": "20636f6e6573", + "5672": "20737570706c696573", + "5673": "7a656e", + "5674": "2073697374657273", + "5675": "20636f6d7065746974696f6e", + "5676": "20486f707079", + "5677": "20756e6465726e65617468", + "5678": "6262696e67", + "5679": "20686f6e6b", + "5680": "2070726f7065726c79", + "5681": "207469636b657473", + "5682": "696e6479", + "5683": "206472696674", + "5684": "20696e737472756374", + "5685": "20736f6172696e67", + "5686": "20656d657267656e6379", + "5687": "206569676874", + "5688": "207374616972", + "5689": "206c6f76696e67", + "5690": "20627572726f77", + "5691": "204164", + "5692": "206d697474656e73", + "5693": "20746f756368696e67", + "5694": "207374697272696e67", + "5695": "616b79", + "5696": "20696e6368", + "5697": "206c616e647363617065", + "5698": "20706561726c", + "5699": "696361", + "5700": "2041667465727761726473", + "5701": "c3a2e282ac2e", + "5702": "2073686f756c64657273", + "5703": "206665746368", + "5704": "696b", + "5705": "206d61726b73", + "5706": "204861", + "5707": "2054686f6d", + "5708": "207261696e626f7773", + "5709": "204361726c", + "5710": "20616e6e", + "5711": "20696e637265617365", + "5712": "206f757273656c766573", + "5713": "206c696564", + "5714": "20526162626974", + "5715": "2064616d", + "5716": "206465736572", + "5717": "204265747479", + "5718": "204d6164", + "5719": "204a6573736965", + "5720": "206d756666696e73", + "5721": "2074726176656c6c", + "5722": "204d696c", + "5723": "2062616b6564", + "5724": "2070726f746563746564", + "5725": "7875", + "5726": "206d6978696e67", + "5727": "4861707079", + "5728": "6578", + "5729": "6f68", + "5730": "20666162", + "5731": "20666162726963", + "5732": "204e696e61", + "5733": "20666f6f6473", + "5734": "20537573616e", + "5735": "206e65696768626f7572686f6f64", + "5736": "20696e74726f647563", + "5737": "20446964", + "5738": "20746f6e69676874", + "5739": "206372756d62", + "5740": "206c6f61646564", + "5741": "204d617474", + "5742": "206d656e74696f6e6564", + "5743": "206f70696e", + "5744": "2066726f7a656e", + "5745": "20676967676c696e67", + "5746": "20707572726564", + "5747": "20696e737472756374696f6e73", + "5748": "696e6461", + "5749": "204d617274", + "5750": "2062756e6368", + "5751": "2070656163686573", + "5752": "2072756262696e67", + "5753": "207469636b6c65", + "5754": "756573646179", + "5755": "20696e74726f64756365", + "5756": "207369646573", + "5757": "206275646765", + "5758": "62696573", + "5759": "206f7574646f6f7273", + "5760": "20737472696e6773", + "5761": "20736967687473", + "5762": "2062726f6f6d", + "5763": "20706572666f72", + "5764": "2066726f7374696e67", + "5765": "2074656173696e67", + "5766": "6167756c6c", + "5767": "207072696e746564", + "5768": "20626f756e6379", + "5769": "2062727573686573", + "5770": "2064656d616e646564", + "5771": "2072656c6561736564", + "5772": "4c6974746c65", + "5773": "20736f616b6564", + "5774": "207377656574696573", + "5775": "206d616368696e6573", + "5776": "20766963746f7279", + "5777": "2041626279", + "5778": "206170706c6175646564", + "5779": "20706f6b696e67", + "5780": "2070726f6a656374", + "5781": "204f7574", + "5782": "2072756c6572", + "5783": "646965", + "5784": "2054756573646179", + "5785": "2065617374", + "5786": "6b696e73", + "5787": "2064726167676564", + "5788": "757875", + "5789": "20626162696573", + "5790": "2073747275636b", + "5791": "20636861736573", + "5792": "6f6564", + "5793": "20706c616e657473", + "5794": "20626f74746c6573", + "5795": "2073656167756c6c", + "5796": "20486f70", + "5797": "2062616e6b", + "5798": "2073746f6c65", + "5799": "2073696c656e6365", + "5800": "20706f7572696e67", + "5801": "2048616e6e6168", + "5802": "4578", + "5803": "20636f6d706c65746564", + "5804": "206d61726b657273", + "5805": "2076656e74", + "5806": "20726f636b6564", + "5807": "204772", + "5808": "207374656570", + "5809": "20766f776564", + "5810": "20686f726e73", + "5811": "206d617374", + "5812": "20707265747469", + "5813": "2074657874757265", + "5814": "20707265747469657374", + "5815": "204c6f75", + "5816": "2077696e6e6572", + "5817": "20647261676f6e73", + "5818": "206c757875", + "5819": "207363757272696564", + "5820": "206461", + "5821": "204a75", + "5822": "206265617264", + "5823": "69636b6564", + "5824": "2074696d", + "5825": "656464696e67", + "5826": "2073757266616365", + "5827": "20736f72746564", + "5828": "2054686f6d6173", + "5829": "20736561736f6e", + "5830": "207065617273", + "5831": "6e6573", + "5832": "696d79", + "5833": "20626c6f6f6d6564", + "5834": "204a656e6e61", + "5835": "c3a2e282ace2809d", + "5836": "20626172656c79", + "5837": "2063726f7764", + "5838": "4168", + "5839": "756374616e746c79", + "5840": "20676f616c73", + "5841": "2063616c", + "5842": "6e6573646179", + "5843": "207572676564", + "5844": "20536c6f77", + "5845": "207369676873", + "5846": "65646e6573646179", + "5847": "207374616d7073", + "5848": "2077657374", + "5849": "20647269766573", + "5850": "6a616d", + "5851": "207072696e7473", + "5852": "2070726574656e6473", + "5853": "2063617463686573", + "5854": "206d6f76696573", + "5855": "2067757973", + "5856": "2073637265616d696e67", + "5857": "20706f6c6963656d616e", + "5858": "7a6965", + "5859": "20706f77657273", + "5860": "65726e", + "5861": "20636f6e666964656e74", + "5862": "20426f7468", + "5863": "207361757361676573", + "5864": "2031", + "5865": "205765646e6573646179", + "5866": "206865726273", + "5867": "206d61746572", + "5868": "204a6f6e6573", + "5869": "2063616d70696e67", + "5870": "206d6174657269", + "5871": "204b61796c61", + "5872": "656e61", + "5873": "20646973636f766572696e67", + "5874": "207374756479696e67", + "5875": "20696e746572657374", + "5876": "20726f6172696e67", + "5877": "20636f6e74657374", + "5878": "2073696e676572", + "5879": "20657870656374", + "5880": "2070656e6775696e73", + "5881": "2063757473", + "5882": "207469", + "5883": "206b69636b696e67", + "5884": "666974", + "5885": "206469736167726565", + "5886": "2068696768657374", + "5887": "20726573756c74", + "5888": "2068756767696e67", + "5889": "2067617a6564", + "5890": "2063686174", + "5891": "204a6f6e", + "5892": "617669", + "5893": "206368697073", + "5894": "20776f756e64", + "5895": "20706f7574", + "5896": "69727473", + "5897": "2070617469656e6365", + "5898": "6f7261", + "5899": "206475737479", + "5900": "207368697665726564", + "5901": "20676f6f646e69676874", + "5902": "2072656c756374616e746c79", + "5903": "2073686f636b", + "5904": "74686f756768", + "5905": "204d697261", + "5906": "204a656666", + "5907": "206f7574666974", + "5908": "20706562626c6573", + "5909": "206375726c79", + "5910": "61736f6e", + "5911": "6c65646765", + "5912": "206772616e74", + "5913": "6976657279", + "5914": "20736974756174696f6e", + "5915": "207072756e6573", + "5916": "206d617463686564", + "5917": "20726962626f6e73", + "5918": "206765", + "5919": "68696e73", + "5920": "20416c77617973", + "5921": "206375727461696e73", + "5922": "20736f6c7574696f6e", + "5923": "20746865697273", + "5924": "20796f757468", + "5925": "20666f726769", + "5926": "2063617574696f7573", + "5927": "207265706561746564", + "5928": "205a6f6f6d", + "5929": "207375697463", + "5930": "20737472696b65", + "5931": "207368656c766573", + "5932": "20696e636c756465", + "5933": "2063686577696e67", + "5934": "206b6e6f776c65646765", + "5935": "20736c696d79", + "5936": "7468616e6b", + "5937": "2070617373696e67", + "5938": "536f6d6574", + "5939": "20736e756767", + "5940": "20656e74697265", + "5941": "657870", + "5942": "206163746564", + "5943": "2077656464696e67", + "5944": "696e74", + "5945": "2061636869657665", + "5946": "2062616c616e63696e67", + "5947": "20596573", + "5948": "62657274", + "5949": "59656168", + "5950": "20666c6174", + "5951": "2050697070", + "5952": "796f75", + "5953": "204c696e6461", + "5954": "206d61727269616765", + "5955": "20746f77656c73", + "5956": "20726f6262", + "5957": "20666c616773", + "5958": "2064656e74697374", + "5959": "20737765657473", + "5960": "52656d656d626572", + "5961": "207265636f72646564", + "5962": "65736f6d65", + "5963": "20736f7274696e67", + "5964": "2041756e74", + "5965": "65657a696e67", + "5966": "5568", + "5967": "20746872656174", + "5968": "6578706563746564", + "5969": "206775657374", + "5970": "206661756c74", + "5971": "20626c696e6b", + "5972": "20776169746572", + "5973": "20696e636c75646564", + "5974": "20616772656573", + "5975": "2072616e676572", + "5976": "2075736573", + "5977": "20686f6d65776f726b", + "5978": "2064656c6976657279", + "5979": "207477696e6b6c696e67", + "5980": "20696e74726f6475636564", + "5981": "207374617265", + "5982": "697a7a6965", + "5983": "207a6f6f6b6565706572", + "5984": "206d6f74696f6e", + "5985": "20756e6578706563746564", + "5986": "20427965", + "5987": "20726f616473", + "5988": "2073776179", + "5989": "204d6f6e", + "5990": "756c79", + "5991": "207370726179696e67", + "5992": "2072657363756564", + "5993": "206c7578757279", + "5994": "206865656c", + "5995": "207061696e74696e6773", + "5996": "206d61746368696e67", + "5997": "20726f636b696e67", + "5998": "20617374", + "5999": "206c696272617269616e", + "6000": "2066696775726564", + "6001": "2066617274", + "6002": "20536c6f776c79", + "6003": "6a616d6173", + "6004": "207368696e6564", + "6005": "20706c656e7479", + "6006": "207472756c79", + "6007": "206c6561726e73", + "6008": "7573746564", + "6009": "69636b79", + "6010": "20737469636b696e67", + "6011": "207772697474656e", + "6012": "63757365", + "6013": "206b6e65656c6564", + "6014": "20617765736f6d65", + "6015": "204c697a7a79", + "6016": "20646f6c7068696e73", + "6017": "20736e616b6573", + "6018": "626c", + "6019": "206c6170", + "6020": "2073637261", + "6021": "20736c6170", + "6022": "6f7973", + "6023": "20696d706f7274616e6365", + "6024": "2062726176657279", + "6025": "2054657373", + "6026": "20626c6f6f6d696e67", + "6027": "20517561636b", + "6028": "756c7473", + "6029": "204578", + "6030": "20726f6f7473", + "6031": "20617272697665", + "6032": "20636f6d706c6574656c79", + "6033": "2e2e2e", + "6034": "204c756b65", + "6035": "20726567726574746564", + "6036": "2077696e6b", + "6037": "206368656572696e67", + "6038": "457863757365", + "6039": "2073746172696e67", + "6040": "207265616368696e67", + "6041": "206865616c6564", + "6042": "546f646179", + "6043": "204c697a7a6965", + "6044": "206c6561646572", + "6045": "4368", + "6046": "696172", + "6047": "2077616e646572696e67", + "6048": "206372756d6273", + "6049": "204164616d", + "6050": "204c65617665", + "6051": "207361666572", + "6052": "6e6a6f79", + "6053": "206772656574", + "6054": "2062656c6c79", + "6055": "20776f726b657273", + "6056": "20646972656374696f6e", + "6057": "207363616c6573", + "6058": "676f", + "6059": "6d6173", + "6060": "20726f6c6c73", + "6061": "696d6d6572", + "6062": "20436872697374", + "6063": "68656c6c", + "6064": "76657273", + "6065": "2067726f776c696e67", + "6066": "6974656c79", + "6067": "20626f6f6d", + "6068": "20637261636b657273", + "6069": "2062656164", + "6070": "20636c6f736573", + "6071": "207468726f", + "6072": "2070656e63696c73", + "6073": "20757375616c", + "6074": "207468726f6174", + "6075": "207265706c6163", + "6076": "68656c6c6f", + "6077": "207472656d626c696e67", + "6078": "2043696e6479", + "6079": "76696e", + "6080": "204d656c", + "6081": "5768656e", + "6082": "61736d", + "6083": "2061726775", + "6084": "2053757a6965", + "6085": "2065786861", + "6086": "20666c61766f72", + "6087": "2070616a616d6173", + "6088": "6f706573", + "6089": "2074726176656c6564", + "6090": "20756e69746564", + "6091": "2059", + "6092": "204368726973746d6173", + "6093": "20636f6d6d", + "6094": "2061726775696e67", + "6095": "206372656174696e67", + "6096": "206578706563746564", + "6097": "20636175736564", + "6098": "20706172746963", + "6099": "756e65", + "6100": "6c6f77657273", + "6101": "2062616b6572", + "6102": "20776f726b6572", + "6103": "207061636b696e67", + "6104": "204861707079", + "6105": "696473", + "6106": "656e636573", + "6107": "204a6f686e736f6e", + "6108": "20666972", + "6109": "206265636f6d696e67", + "6110": "20726f636b657473", + "6111": "696769", + "6112": "2076656e7475726564", + "6113": "2076616e697368", + "6114": "207374727567676c65", + "6115": "20626f6f", + "6116": "206d616e65", + "6117": "206c616d62", + "6118": "20706174746564", + "6119": "207072696e636573736573", + "6120": "206f62736572766564", + "6121": "616e676c6564", + "6122": "20776f7272696573", + "6123": "2073746172746c6564", + "6124": "2072657374696e67", + "6125": "206461726564", + "6126": "2072656d6f766564", + "6127": "20666c616d6573", + "6128": "204a657373696361", + "6129": "204d6f6d6f", + "6130": "206f776e657273", + "6131": "20737465726e", + "6132": "206f7267616e697a696e67", + "6133": "204d696c6f", + "6134": "20657868617573746564", + "6135": "206669726d", + "6136": "206372616273", + "6137": "20526f626279", + "6138": "206d6973736573", + "6139": "207769726573", + "6140": "20696e766974696e67", + "6141": "207472697073", + "6142": "206b6e6967687473", + "6143": "2068656176", + "6144": "20636f7474", + "6145": "426164", + "6146": "7265656e", + "6147": "2066616972696573", + "6148": "20467269656e64", + "6149": "206c656170", + "6150": "207365656d73", + "6151": "207261646973686573", + "6152": "20707261796572", + "6153": "2042757a7a79", + "6154": "2074696d6572", + "6155": "2063616e73", + "6156": "20736565736177", + "6157": "20726573746f726564", + "6158": "6374696f6e73", + "6159": "6c6565", + "6160": "2074726163746f72", + "6161": "626f6174", + "6162": "20637261636b6564", + "6163": "2066757a7a79", + "6164": "20706f6c697368696e67", + "6165": "206f70656e696e67", + "6166": "2063656e74", + "6167": "4a757374", + "6168": "20636875636b", + "6169": "207061737472696573", + "6170": "2070617374656c73", + "6171": "2072756273", + "6172": "206d656c74696e67", + "6173": "64696573", + "6174": "207269736b", + "6175": "20706f7473", + "6176": "536f6d6574696d6573", + "6177": "2072656c696566", + "6178": "2063617368", + "6179": "207375697463617365", + "6180": "207477696e6b6c65", + "6181": "2077696e6b6564", + "6182": "207065656b696e67", + "6183": "20436865", + "6184": "207772697374", + "6185": "2072657665616c", + "6186": "20706172746963756c6172", + "6187": "20736e65616b79", + "6188": "204a617a7a", + "6189": "617a65", + "6190": "20756e66", + "6191": "20507574", + "6192": "20696e7465726573746564", + "6193": "617261776179", + "6194": "20636f6c6f75726564", + "6195": "2073746f72656b6565706572", + "6196": "20776173746564", + "6197": "2047696769", + "6198": "206576657279626f6479", + "6199": "206974636879", + "6200": "2066617261776179", + "6201": "204265696e67", + "6202": "206661636564", + "6203": "20446f676779", + "6204": "2054617261", + "6205": "20636f756768", + "6206": "20757375616c6c79", + "6207": "2076616c756564", + "6208": "20646573657274", + "6209": "2073696c656e74", + "6210": "20736e6f7762616c6c73", + "6211": "526f", + "6212": "20466f6c6c6f77", + "6213": "46696e65", + "6214": "2050616e6479", + "6215": "2073637275", + "6216": "207374656d", + "6217": "6f6c696e65", + "6218": "2064656e", + "6219": "20636f7573", + "6220": "20446f6c6c79", + "6221": "206c6f6c6c69706f7073", + "6222": "2067726f636572696573", + "6223": "20717561636b6564", + "6224": "6973696573", + "6225": "206d61676e657473", + "6226": "20776967676c6564", + "6227": "20706164646c6564", + "6228": "2072656d6f7465", + "6229": "206c756e67", + "6230": "207369676e6564", + "6231": "206361757365", + "6232": "207469636b6c6564", + "6233": "20636f756e74727973696465", + "6234": "2064616973696573", + "6235": "2062616b657279", + "6236": "20686973736564", + "6237": "72656174", + "6238": "20776f77", + "6239": "5468656e", + "6240": "2073746179696e67", + "6241": "2074656163686573", + "6242": "204a756c6961", + "6243": "20736c616d", + "6244": "2046696e6e", + "6245": "206f70696e696f6e", + "6246": "206c696e6564", + "6247": "204f77", + "6248": "2062616c616e636564", + "6249": "2061637469766974696573", + "6250": "20736c6564", + "6251": "20466f78", + "6252": "2073776f6f706564", + "6253": "204d6f", + "6254": "2070686f746f73", + "6255": "20686f7765766572", + "6256": "2054776f", + "6257": "207265706c79", + "6258": "206e617073", + "6259": "20737769726c", + "6260": "20616464696e67", + "6261": "20646970706564", + "6262": "20706f656d73", + "6263": "206265616473", + "6264": "204d616e79", + "6265": "52756e", + "6266": "207765617273", + "6267": "206869", + "6268": "206e656374", + "6269": "206177616b65", + "6270": "206461726b6e657373", + "6271": "206e6563746172", + "6272": "2067617a65", + "6273": "20736e6174", + "6274": "61636f62", + "6275": "6f726473", + "6276": "207365656b6572", + "6277": "20636f76657273", + "6278": "2050696e6b79", + "6279": "207365617368656c6c", + "6280": "20616666", + "6281": "206869646572", + "6282": "6272757368", + "6283": "2072696e6773", + "6284": "756d6d", + "6285": "20626c6f7773", + "6286": "206163636f6d706c697368", + "6287": "70697465", + "6288": "20636f6e6e", + "6289": "2062756d7079", + "6290": "20456e6a6f79", + "6291": "2070726f7669646564", + "6292": "2044617973", + "6293": "20706f6c6973686564", + "6294": "204e6f7468696e67", + "6295": "20747269636b6564", + "6296": "20626f776c73", + "6297": "204d696b6579", + "6298": "2074697074", + "6299": "20646972656374696f6e73", + "6300": "204d61", + "6301": "20736576656e", + "6302": "2073696e676c65", + "6303": "20636f7079", + "6304": "207265706c61636564", + "6305": "20736f62626564", + "6306": "2073616e64636173746c6573", + "6307": "20676c6173736573", + "6308": "204b656570", + "6309": "206c6f6773", + "6310": "206d61726b6572", + "6311": "207265666c65", + "6312": "2063617574696f75736c79", + "6313": "2068656c6d657473", + "6314": "205761746368", + "6315": "20457863", + "6316": "2066696c6c696e67", + "6317": "2066617274686572", + "6318": "20666f726568656164", + "6319": "207377656174", + "6320": "20726f736573", + "6321": "72616e6365", + "6322": "204d69636b", + "6323": "2042657373", + "6324": "6166666573", + "6325": "20706564616c73", + "6326": "206d61726b6564", + "6327": "204561727468", + "6328": "206261646765", + "6329": "20456d6d79", + "6330": "20576974686f7574", + "6331": "20536f727279", + "6332": "20636875636b6c6564", + "6333": "20706c656173757265", + "6334": "2053717565", + "6335": "206578706c6f72657273", + "6336": "20666c75747465726564", + "6337": "204a61636b79", + "6338": "207374617973", + "6339": "2063617368696572", + "6340": "20627579696e67", + "6341": "2045786369746564", + "6342": "20666972656669676874657273", + "6343": "206769726166666573", + "6344": "206b69746573", + "6345": "737479", + "6346": "20426f", + "6347": "2062756d7073", + "6348": "2073706c6173686573", + "6349": "697463686573", + "6350": "206672656564", + "6351": "20796f756e676572", + "6352": "204c656e61", + "6353": "207479", + "6354": "207265616473", + "6355": "2073636172766573", + "6356": "2042697264", + "6357": "206d61736b73", + "6358": "204f6c697665", + "6359": "2062617273", + "6360": "636f6174", + "6361": "68696c65", + "6362": "206c697175", + "6363": "206465616c", + "6364": "204a61636f62", + "6365": "62626c79", + "6366": "2070726f74656374696e67", + "6367": "20636f6c6f726564", + "6368": "207370696b65", + "6369": "20476574", + "6370": "204e616e", + "6371": "206865616c7468", + "6372": "2063656c6562726174696f6e", + "6373": "2073746f6c656e", + "6374": "207069676779", + "6375": "2073717565616c6564", + "6376": "2074616c656e746564", + "6377": "696b69", + "6378": "2072697065", + "6379": "2074616c6c657374", + "6380": "2064657465726d696e6174696f6e", + "6381": "686565", + "6382": "626f6f", + "6383": "2067656d73", + "6384": "207265636f7264696e67", + "6385": "207374726f6c6c", + "6386": "206375626573", + "6387": "64646c6564", + "6388": "207769706573", + "6389": "20796f757273656c766573", + "6390": "20696e7365637473", + "6391": "204576657279626f6479", + "6392": "204e69636b", + "6393": "2061747461636b", + "6394": "20796573746572646179", + "6395": "666c", + "6396": "206a6f6b696e67", + "6397": "2079656c6c73", + "6398": "20636c6f776e73", + "6399": "20616374696f6e73", + "6400": "206261636b6564", + "6401": "204c61727279", + "6402": "206b6e656573", + "6403": "207261696e636f6174", + "6404": "75727279", + "6405": "206d757368726f6f6d73", + "6406": "20646f7473", + "6407": "20626579", + "6408": "6964696e67", + "6409": "206f7070", + "6410": "20766964656f73", + "6411": "20747269636b79", + "6412": "206265796f6e64", + "6413": "6162656c", + "6414": "20636f7762", + "6415": "207477696e", + "6416": "206b6e6f636b73", + "6417": "206472756d73", + "6418": "20626568617669", + "6419": "20696e737472756d", + "6420": "2070617274696573", + "6421": "696c6573", + "6422": "cb9c", + "6423": "2d2d", + "6424": "20666f6c64696e67", + "6425": "2077656570696e67", + "6426": "7070657273", + "6427": "205761746572", + "6428": "20736f6d65626f6479", + "6429": "2073656e7365", + "6430": "206d696e6572616c73", + "6431": "20526f62657274", + "6432": "c3a2e282accb9c", + "6433": "746c65", + "6434": "206d6573", + "6435": "206d65736d6572", + "6436": "20626567696e6e696e67", + "6437": "6e657273", + "6438": "206c697374656e73", + "6439": "206d696c6573", + "6440": "207461736b73", + "6441": "207361696c626f6174", + "6442": "20736d696c6579", + "6443": "20736563757265", + "6444": "2074686f726e", + "6445": "2066726f7a65", + "6446": "6172746572", + "6447": "20537175", + "6448": "2068616e6466756c", + "6449": "2063616c6d6564", + "6450": "20656e7472616e6365", + "6451": "62616e64", + "6452": "20736869727473", + "6453": "207061757365", + "6454": "20506f", + "6455": "2073636172696e67", + "6456": "6f756368", + "6457": "206c6971756964", + "6458": "206b69636b73", + "6459": "2074616e676c6564", + "6460": "204c6175", + "6461": "2066616d696c696573", + "6462": "204d617961", + "6463": "526578", + "6464": "20706164646c65", + "6465": "20706f75746564", + "6466": "20736d6172746572", + "6467": "2063616e64696573", + "6468": "20536f70686961", + "6469": "2042696e6b79", + "6470": "206c696768746572", + "6471": "20736e617463686564", + "6472": "757a7a", + "6473": "746873", + "6474": "62656c6c", + "6475": "20737065616b696e67", + "6476": "204a6179", + "6477": "2077696e73", + "6478": "20676c6565", + "6479": "2061667465727761726473", + "6480": "2063757374", + "6481": "20636c69636b", + "6482": "206f626a65637473", + "6483": "4772616e646d61", + "6484": "20537061726b79", + "6485": "696e6472", + "6486": "206172726f7773", + "6487": "63657273", + "6488": "54686f7365", + "6489": "207261696e6472", + "6490": "20696d70726f766564", + "6491": "207275737479", + "6492": "2063686f6f", + "6493": "656b65", + "6494": "72696f7573", + "6495": "2062656474696d65", + "6496": "207275696e696e67", + "6497": "2073616c7479", + "6498": "206c656164696e67", + "6499": "2063686f69636573", + "6500": "20636865656b73", + "6501": "6967657273", + "6502": "616b6572", + "6503": "20726970", + "6504": "207374656572", + "6505": "206c6f776572", + "6506": "20616374696e67", + "6507": "20506c", + "6508": "2063757063616b6573", + "6509": "2062656861766564", + "6510": "20637573746f6d", + "6511": "726967", + "6512": "20646976696e67", + "6513": "20417368", + "6514": "206d69", + "6515": "616c7068", + "6516": "207374726f6b6564", + "6517": "6f6579", + "6518": "20736e75636b", + "6519": "61726c696e67", + "6520": "20426c756579", + "6521": "20696e7370", + "6522": "20526f766572", + "6523": "20636f6d706c61696e696e67", + "6524": "20626174746572696573", + "6525": "20486f6f74", + "6526": "20686573697461746564", + "6527": "20736e6f776d656e", + "6528": "4e657874", + "6529": "2072756c65", + "6530": "205072696e", + "6531": "206d697874757265", + "6532": "206a61636b657473", + "6533": "204c61757261", + "6534": "20706179696e67", + "6535": "2072756d626c65", + "6536": "2073717561726573", + "6537": "696c697479", + "6538": "3f222c", + "6539": "206665766572", + "6540": "696c6564", + "6541": "73657175", + "6542": "204375", + "6543": "206172656e", + "6544": "20776964656e6564", + "6545": "20626173656d656e74", + "6546": "20617374726f6e", + "6547": "206665617273", + "6548": "20666c617070696e67", + "6549": "20736372617463686573", + "6550": "20617374726f6e617574", + "6551": "207377696d73756974", + "6552": "206461726b6572", + "6553": "206c756e", + "6554": "616e73", + "6555": "20637265616d79", + "6556": "207265636f6d6d656e646564", + "6557": "20636f6e73657175", + "6558": "6976697479", + "6559": "2070757070696573", + "6560": "204d6172746861", + "6561": "205a6f6579", + "6562": "76657374", + "6563": "204472", + "6564": "697463686564", + "6565": "207261696e64726f7073", + "6566": "20696d706f", + "6567": "20756e7061636b6564", + "6568": "2046726f737479", + "6569": "20696d706f737369626c65", + "6570": "6e6f", + "6571": "2064726966746564", + "6572": "2068616c6c776179", + "6573": "2064656c697665726564", + "6574": "2073746f6d706564", + "6575": "204162626965", + "6576": "206c6f73696e67", + "6577": "206f726e616d656e7473", + "6578": "206e6f6f6e", + "6579": "536868", + "6580": "2063656c6562726174696e67", + "6581": "20636f6d706c61696e6564", + "6582": "3f27", + "6583": "20646f6c6c686f757365", + "6584": "2074656d7074", + "6585": "69676e696e67", + "6586": "20687573", + "6587": "20706f696e7479", + "6588": "206368656673", + "6589": "61747572", + "6590": "69736879", + "6591": "20616e6e6f79", + "6592": "2072656675", + "6593": "20646566", + "6594": "20746561736564", + "6595": "207772617073", + "6596": "2063726561746976697479", + "6597": "206772616262696e67", + "6598": "20677261636566756c6c79", + "6599": "20706572666f726d616e", + "6600": "657074", + "6601": "20746967657273", + "6602": "2064616e636573", + "6603": "205072696e63657373", + "6604": "2070617373656e", + "6605": "206163686965766564", + "6606": "6e65", + "6607": "206e6565646c6573", + "6608": "2073616c65", + "6609": "6d616c6c6f77", + "6610": "6e6470617265", + "6611": "206e6f736573", + "6612": "20696e64656564", + "6613": "204b696b69", + "6614": "617273686d616c6c6f77", + "6615": "207369626c", + "6616": "2074726176656c6c6564", + "6617": "20617274697374", + "6618": "20666f7267657473", + "6619": "2070616e73", + "6620": "207369626c696e6773", + "6621": "206275727374", + "6622": "20526f62626965", + "6623": "20736372657773", + "6624": "636c", + "6625": "63686f6f", + "6626": "6974746564", + "6627": "73746570", + "6628": "206869646573", + "6629": "637573", + "6630": "206e656974686572", + "6631": "207772617070696e67", + "6632": "636f7065", + "6633": "207468697264", + "6634": "207374756e67", + "6635": "206d6174657269616c73", + "6636": "207363727562626564", + "6637": "20636f6e73657175656e636573", + "6638": "726967756564", + "6639": "206265616d", + "6640": "616e6573", + "6641": "20476f6f64627965", + "6642": "2050616d", + "6643": "20657870657269656e", + "6644": "205361646965", + "6645": "206772616e6470617265", + "6646": "77656574", + "6647": "206c657474", + "6648": "20746970746f6564", + "6649": "204a756e65", + "6650": "206c657474756365", + "6651": "2064657365727665", + "6652": "207377657074", + "6653": "20416d656c", + "6654": "2063686970", + "6655": "20706c6179657273", + "6656": "2064616473", + "6657": "6172656e", + "6658": "20466172", + "6659": "20666c7574746572696e67", + "6660": "2074656d706572", + "6661": "656e6479", + "6662": "206f6e696f6e73", + "6663": "20646f6c6c617273", + "6664": "20447572696e67", + "6665": "206c696d6573", + "6666": "206d7573636c65", + "6667": "4d656f77", + "6668": "204c69", + "6669": "2063686f70", + "6670": "20416269", + "6671": "20696e636c7564696e67", + "6672": "206163636f6d706c6973686d656e74", + "6673": "4577", + "6674": "205468616e6b73", + "6675": "20726976657273", + "6676": "20646973637573736564", + "6677": "696d6d6572696e67", + "6678": "2068757362616e64", + "6679": "6b79", + "6680": "77656564", + "6681": "756d626c696e67", + "6682": "4772656174", + "6683": "48617665", + "6684": "20426565", + "6685": "20437261", + "6686": "206d65616c73", + "6687": "207472616d70", + "6688": "20636f6e636572", + "6689": "2067726f776c73", + "6690": "20636f72", + "6691": "2068616e64736f6d65", + "6692": "2067726f7373", + "6693": "20736f75726365", + "6694": "20636f7570", + "6695": "20456c73", + "6696": "204d6162656c", + "6697": "206166666f7264", + "6698": "646564", + "6699": "2047696e61", + "6700": "616e67696e67", + "6701": "204c6961", + "6702": "2068656172696e67", + "6703": "2066616374", + "6704": "206d65727279", + "6705": "6174756c", + "6706": "207765656b65", + "6707": "207765656b656e64", + "6708": "20416c6f6e67", + "6709": "2051", + "6710": "206d6572", + "6711": "2053696d", + "6712": "2061636865", + "6713": "2061766f6964", + "6714": "20617564", + "6715": "2073656177656564", + "6716": "206261636b7761726473", + "6717": "20706561636566756c6c79", + "6718": "207265776172646564", + "6719": "20706572666f726d616e6365", + "6720": "2062656c6f766564", + "6721": "206e6561746c79", + "6722": "20416c74686f756768", + "6723": "207472616d706f6c696e65", + "6724": "206d6167696369616e", + "6725": "526f6172", + "6726": "206d617273686d616c6c6f77", + "6727": "20737461696e", + "6728": "20656368", + "6729": "20726973696e67", + "6730": "7565656e", + "6731": "20546f6464", + "6732": "206465636f726174696e67", + "6733": "20677261737379", + "6734": "20737072696e6b6c6564", + "6735": "20736872696e6b696e67", + "6736": "20636f7573696e", + "6737": "20736e6f77666c", + "6738": "20736e7567676c65", + "6739": "4a616e65", + "6740": "20677269", + "6741": "2068697474696e67", + "6742": "2057616c6c79", + "6743": "20706c616e73", + "6744": "7567676c6573", + "6745": "20626f64696573", + "6746": "20736c616d6d6564", + "6747": "2052616c7068", + "6748": "205363", + "6749": "2068656c70657273", + "6750": "2077696c6c696e67", + "6751": "7269766572", + "6752": "61646573", + "6753": "2074756e6e656c73", + "6754": "20636f75706c65", + "6755": "20626f7773", + "6756": "204e6f626f6479", + "6757": "207468726f776e", + "6758": "4c756379", + "6759": "57686565", + "6760": "2070726f6d69736573", + "6761": "4c65", + "6762": "2074656d7065726174757265", + "6763": "20536e7567676c6573", + "6764": "20636f6c6f72696e67", + "6765": "20736570617261746564", + "6766": "6f6e69", + "6767": "20646f64", + "6768": "20726563697065", + "6769": "2063656e746572", + "6770": "207365636f6e6473", + "6771": "2073746f6f6c", + "6772": "2077696e6479", + "6773": "2066656564696e67", + "6774": "20637564646c79", + "6775": "207265617373", + "6776": "20756e73", + "6777": "54696d65", + "6778": "21c3a2e282ac2e", + "6779": "20647265616d74", + "6780": "20686f72697a", + "6781": "205374656c6c61", + "6782": "206c65616e696e67", + "6783": "2070726573736573", + "6784": "206d617474657273", + "6785": "6e74696c", + "6786": "20656e7468757369", + "6787": "204a616e6965", + "6788": "204a756479", + "6789": "20626c6f636b696e67", + "6790": "4f6f7073", + "6791": "2062756c6c79", + "6792": "207461696c73", + "6793": "2054686f7365", + "6794": "2070726179696e67", + "6795": "2064616d616765", + "6796": "2062757a7a6564", + "6797": "20656e746875736961736d", + "6798": "63636f6f6e", + "6799": "20726571756573746564", + "6800": "206372656174696f6e73", + "6801": "207a697070696e67", + "6802": "20686f72697a6f6e", + "6803": "20646574", + "6804": "207065636b6564", + "6805": "576f6f66", + "6806": "20556e74696c", + "6807": "2073747564656e7473", + "6808": "20726163636f6f6e", + "6809": "2052656d656d626572", + "6810": "2061756469656e6365", + "6811": "7073", + "6812": "207477696365", + "6813": "667573", + "6814": "20656d62617272617373", + "6815": "206461726c696e67", + "6816": "20616d6f6e67", + "6817": "206e65696768626f757273", + "6818": "61646f73", + "6819": "20617272", + "6820": "207072657474", + "6821": "206e61706b696e73", + "6822": "2054756e61", + "6823": "6974756465", + "6824": "20616c69656e", + "6825": "20726f62626572", + "6826": "2068656164696e67", + "6827": "207761726e696e6773", + "6828": "20666f72676976656e", + "6829": "204b6172656e", + "6830": "2064657369676e6564", + "6831": "20727573746c696e67", + "6832": "2074726565686f757365", + "6833": "20736b756e6b", + "6834": "206164756c74", + "6835": "20737572726f756e64696e6773", + "6836": "2077697368696e67", + "6837": "204f776c", + "6838": "20676f62626c6564", + "6839": "20517565656e", + "6840": "20746f7563", + "6841": "2073757065726865726f6573", + "6842": "2073706f6e67", + "6843": "20736c69636573", + "6844": "6974696f6e", + "6845": "20776970696e67", + "6846": "2072657061", + "6847": "205a6970", + "6848": "2057656e6479", + "6849": "206272656174686564", + "6850": "2053717565616b79", + "6851": "204661726d6572", + "6852": "204269727468646179", + "6853": "207072657474696572", + "6854": "20736c6f776572", + "6855": "20776f6f66", + "6856": "206e696e65", + "6857": "207072696e74696e67", + "6858": "6172726f77", + "6859": "207265706169726564", + "6860": "20726f706573", + "6861": "656473", + "6862": "20776167", + "6863": "205279", + "6864": "2074687564", + "6865": "6f6e6772", + "6866": "207265666c656374696f6e", + "6867": "20747269616e676c6573", + "6868": "20666f6375", + "6869": "203130", + "6870": "20696e74726967756564", + "6871": "6c6c656e", + "6872": "20416d656c6961", + "6873": "69656365", + "6874": "204c697374656e", + "6875": "6d656e7473", + "6876": "20706f7073", + "6877": "20706f74696f6e", + "6878": "207377616e73", + "6879": "2053696e6365", + "6880": "206865736974616e74", + "6881": "20626f72726f776564", + "6882": "776b", + "6883": "20637572696f75736c79", + "6884": "207374617266697368", + "6885": "206c756e6773", + "6886": "2061766f6361646f73", + "6887": "204d69636b6579", + "6888": "2073706f727473", + "6889": "20706c616e6e696e67", + "6890": "6f6e67726174756c", + "6891": "207a6562726173", + "6892": "2068656176696572", + "6893": "7072", + "6894": "2046697265", + "6895": "66616c6c", + "6896": "20666c6173", + "6897": "20636f756e7473", + "6898": "20696d6167696e696e67", + "6899": "2064656c69766572696e67", + "6900": "206e696262", + "6901": "207265736973746564", + "6902": "20646965", + "6903": "205a617261", + "6904": "20686f7572", + "6905": "2077657074", + "6906": "20686f6e6b6564", + "6907": "205061747479", + "6908": "20666f726b73", + "6909": "2073637261706564", + "6910": "2076616e696c6c61", + "6911": "206261746873", + "6912": "206861776b", + "6913": "617573696e67", + "6914": "206d617273686d616c6c6f7773", + "6915": "20426573736965", + "6916": "20636f726e657273", + "6917": "206e6f6f64", + "6918": "6f7474656e", + "6919": "20426f756e", + "6920": "20746f7563686573", + "6921": "20666f6375736564", + "6922": "61626c79", + "6923": "69636861", + "6924": "2073706f6e6765", + "6925": "20636f6e64", + "6926": "2073706f6f6e66756c", + "6927": "207361696c6f7273", + "6928": "20666f7267657474696e67", + "6929": "2046726f67", + "6930": "20776f62626c6564", + "6931": "69636861656c", + "6932": "2073616e", + "6933": "686f67", + "6934": "2073747265657473", + "6935": "2064616e63657273", + "6936": "697272656c", + "6937": "204f7572", + "6938": "59756d6d79", + "6939": "2065787072657373", + "6940": "207374616e6473", + "6941": "426f6f", + "6942": "20647261776e", + "6943": "656e746c79", + "6944": "2053696d6f6e", + "6945": "20616765", + "6946": "207370696b6573", + "6947": "737469636b", + "6948": "20737061726b73", + "6949": "20676c756564", + "6950": "20457661", + "6951": "206772616e64706172656e7473", + "6952": "20736372657764", + "6953": "207772696e6b", + "6954": "20706f73746572", + "6955": "20746f6464", + "6956": "206e7572", + "6957": "20706c61696e", + "6958": "20456c6c79", + "6959": "4c696b65", + "6960": "2070656e73", + "6961": "20737072656164696e67", + "6962": "207768697370657273", + "6963": "4a656e6e79", + "6964": "2073616e6b", + "6965": "207468726f7567686f7574", + "6966": "204261636b", + "6967": "20726f7474656e", + "6968": "20736861706564", + "6969": "20626c617374", + "6970": "206a6f7966756c", + "6971": "206d756e6368", + "6972": "53616d6d79", + "6973": "20746573746564", + "6974": "206865726f6573", + "6975": "20467269656e6473", + "6976": "2073717565616b", + "6977": "2050697070696e", + "6978": "2066616d696c696172", + "6979": "207379", + "6980": "20736e69666673", + "6981": "2053756e", + "6982": "2064726f7070696e67", + "6983": "20736f66746572", + "6984": "206c65617374", + "6985": "2074616c6b73", + "6986": "204772616e6e79", + "6987": "2067726565746564", + "6988": "207363726577647269766572", + "6989": "207065636b", + "6990": "204f6e6c79", + "6991": "207761726d6572", + "6992": "207370696e73", + "6993": "206765657365", + "6994": "69646f", + "6995": "746865", + "6996": "206173696465", + "6997": "204d696e61", + "6998": "2070657070657273", + "6999": "204d69636861656c", + "7000": "20737465616479", + "7001": "20746170706564", + "7002": "20706f73746d616e", + "7003": "65617273", + "7004": "207761727269", + "7005": "20706f757273", + "7006": "2053746179", + "7007": "206469766564", + "7008": "206d65736d6572697a6564", + "7009": "206772616e746564", + "7010": "205279616e", + "7011": "20686f706566756c", + "7012": "76696c", + "7013": "4f7572", + "7014": "206e65617265", + "7015": "20706f6c6172", + "7016": "2070756666", + "7017": "206d696e7473", + "7018": "204269726479", + "7019": "20776967676c65", + "7020": "20696e63726561736564", + "7021": "2054616b65", + "7022": "20706f7373", + "7023": "65616c", + "7024": "2073717569736879", + "7025": "2063686f73", + "7026": "2072656d656d626572696e67", + "7027": "65657a6573", + "7028": "2063686f73656e", + "7029": "204461766579", + "7030": "20736c697073", + "7031": "204d6567", + "7032": "206368616e67696e67", + "7033": "20726f7574", + "7034": "206d6f6f", + "7035": "206865646765686f67", + "7036": "74657273", + "7037": "2053706c", + "7038": "20726f617273", + "7039": "2074697373", + "7040": "2064726573736572", + "7041": "2070696e73", + "7042": "207363616d70", + "7043": "6f756e636564", + "7044": "206c69667473", + "7045": "20666c69676874", + "7046": "2064726177657273", + "7047": "536f6d65", + "7048": "20706c75636b6564", + "7049": "20416e67", + "7050": "206372616674", + "7051": "204669646f", + "7052": "61726163", + "7053": "20456c697a", + "7054": "207377616c6c", + "7055": "207377616c6c6f776564", + "7056": "20466c6f70", + "7057": "537565", + "7058": "2073656e64696e67", + "7059": "2072757368696e67", + "7060": "206e65636b6c61636573", + "7061": "20707570706574", + "7062": "2073776f726473", + "7063": "20677565737473", + "7064": "63616b65", + "7065": "2077616b6573", + "7066": "7965617273", + "7067": "65727461696e", + "7068": "53616c6c79", + "7069": "636b79", + "7070": "6d65616c", + "7071": "206a6f696e696e67", + "7072": "20636f63", + "7073": "6173746572", + "7074": "696c6579", + "7075": "2072656672657368696e67", + "7076": "204c6f7665", + "7077": "6d61746573", + "7078": "6a6f", + "7079": "796573", + "7080": "20636f6e666c", + "7081": "20646970", + "7082": "20737069636573", + "7083": "20726561736f6e", + "7084": "20626974696e67", + "7085": "2073686f7073", + "7086": "20736577696e67", + "7087": "20626c696e6b696e67", + "7088": "2e2e2e22", + "7089": "206f6363", + "7090": "2063726577", + "7091": "2073706174", + "7092": "756e63686564", + "7093": "2064656665", + "7094": "20746f73736564", + "7095": "20736e65657a696e67", + "7096": "206665617374", + "7097": "206669726566696768746572", + "7098": "2070616e69636b6564", + "7099": "206f61746d65616c", + "7100": "20537175697272656c", + "7101": "206f66666963", + "7102": "20666c61766f7273", + "7103": "416c696365", + "7104": "756773", + "7105": "2074726f74", + "7106": "2074696573", + "7107": "20746f64646c6572", + "7108": "61747465726564", + "7109": "6c766573", + "7110": "20737461636b6564", + "7111": "2073756e666c6f77657273", + "7112": "20746f6173746572", + "7113": "20636f727265", + "7114": "2053746172", + "7115": "2032", + "7116": "206f68", + "7117": "207472656d626c6564", + "7118": "207374726f6e67657374", + "7119": "20617070726563696174696f6e", + "7120": "206578706c6f", + "7121": "206265686176696e67", + "7122": "204d61646479", + "7123": "20646f6f72776179", + "7124": "2073616464", + "7125": "4576657279", + "7126": "204665", + "7127": "207761746572696e67", + "7128": "20706967656f6e73", + "7129": "757074", + "7130": "20616d6f756e74", + "7131": "20706c6179726f6f6d", + "7132": "66697265", + "7133": "2064726976657273", + "7134": "20726573706563746564", + "7135": "20656e636f75726167", + "7136": "2070726f706572", + "7137": "206576696c", + "7138": "69707079", + "7139": "206f797374657273", + "7140": "207175617272656c696e67", + "7141": "2061696d6564", + "7142": "20636861726163", + "7143": "207772696767", + "7144": "2063617573696e67", + "7145": "20756e73757265", + "7146": "20776164646c6564", + "7147": "727272", + "7148": "20776f62626c79", + "7149": "2067617264656e6572", + "7150": "206372617368696e67", + "7151": "2046726f676779", + "7152": "206561726e", + "7153": "204a61636b6965", + "7154": "206d6173746572", + "7155": "206e6f6f646c6573", + "7156": "20636f6e76657273", + "7157": "6179696e67", + "7158": "2057656c6c", + "7159": "616e746861", + "7160": "20526f636b", + "7161": "726f6b656e", + "7162": "4a6f686e6e79", + "7163": "7768696c65", + "7164": "466f", + "7165": "206d617465", + "7166": "2067657374", + "7167": "206b697474656e73", + "7168": "4d696e65", + "7169": "2062656c6c73", + "7170": "205374696c6c", + "7171": "20486f6e6579", + "7172": "696d656e74", + "7173": "2070656e6e696573", + "7174": "20556e646572", + "7175": "2073636f726564", + "7176": "20746973737565", + "7177": "206e617070696e67", + "7178": "20776f727279696e67", + "7179": "20526179", + "7180": "75727072697365", + "7181": "207461626c6573", + "7182": "20747769737479", + "7183": "20636f72616c", + "7184": "207365617368656c6c73", + "7185": "686577", + "7186": "2070617373656e67657273", + "7187": "207363656e", + "7188": "207261696e79", + "7189": "20726573706f6e64", + "7190": "6f6e6e61", + "7191": "206e6572766f75736c79", + "7192": "206d6f6e746873", + "7193": "205570", + "7194": "2064657369676e73", + "7195": "6472657373", + "7196": "2070726f7665", + "7197": "206c6f6164696e67", + "7198": "205061746368", + "7199": "666c79", + "7200": "2067726f7773", + "7201": "20706572666f726d696e67", + "7202": "2073757272656e6465726564", + "7203": "204368726973", + "7204": "697363", + "7205": "76616e74", + "7206": "20426f6e67", + "7207": "7265747479", + "7208": "206f6e77617264", + "7209": "206c65617074", + "7210": "20706c617974696d65", + "7211": "206c65616473", + "7212": "2066697265666c79", + "7213": "206669726d6c79", + "7214": "69737361", + "7215": "20646172696e67", + "7216": "204d6f6d6d61", + "7217": "20686f7073", + "7218": "20616374696f6e", + "7219": "56657279", + "7220": "206661646564", + "7221": "204c6f75697365", + "7222": "207374697463686573", + "7223": "207265756e", + "7224": "20626174746c65", + "7225": "206c6573736f6e73", + "7226": "2073747265616d657273", + "7227": "206475636b6c696e67", + "7228": "2073796d626f6c73", + "7229": "204173686c6579", + "7230": "517569636b", + "7231": "206d616e6167", + "7232": "207361696c73", + "7233": "206e757a7a", + "7234": "20666c6f776e", + "7235": "20437572696f7573", + "7236": "20686169726472657373", + "7237": "20736c6974686572", + "7238": "20706f6c6974656c79", + "7239": "20536e6f776d616e", + "7240": "2042656361757365", + "7241": "207363616d7065726564", + "7242": "206164756c7473", + "7243": "204a616e6579", + "7244": "2074686f7567687473", + "7245": "2063757368696f6e73", + "7246": "2063726f7373696e67", + "7247": "20706f756e64", + "7248": "696f6e61", + "7249": "204d65616e", + "7250": "20636f7379", + "7251": "206c696674696e67", + "7252": "20426f6e676f", + "7253": "206265686176696f72", + "7254": "656e696e67", + "7255": "62626c696e67", + "7256": "204675", + "7257": "204761", + "7258": "206e6561726c79", + "7259": "20676f6e6e61", + "7260": "2062656c7473", + "7261": "776f726d", + "7262": "206f766572636f6d65", + "7263": "207969656c646564", + "7264": "2062726176657374", + "7265": "206861697264726573736572", + "7266": "2073757276", + "7267": "6962626974", + "7268": "20776f726e", + "7269": "62697264", + "7270": "2066697473", + "7271": "20736d616c6c657374", + "7272": "20426161", + "7273": "207375697473", + "7274": "20517561636b657273", + "7275": "20656e636f75726167696e67", + "7276": "7279696e67", + "7277": "20616973", + "7278": "7070696573", + "7279": "204d6f757365", + "7280": "2066696c6c73", + "7281": "20636c6f756479", + "7282": "2071756965746572", + "7283": "205374616e", + "7284": "20616e67656c73", + "7285": "2068756d616e73", + "7286": "20466c6f707379", + "7287": "20726573756c7473", + "7288": "2053616d616e746861", + "7289": "2073686f76656c73", + "7290": "20446573", + "7291": "2073686f776e", + "7292": "62726f6b656e", + "7293": "706c65617365", + "7294": "204b656c6c79", + "7295": "20537469746368", + "7296": "20636f6277656273", + "7297": "20736369656e74", + "7298": "616c6c6f7765656e", + "7299": "626261", + "7300": "2044696e", + "7301": "207368616479", + "7302": "656d6d61", + "7303": "20636f6d6d756e", + "7304": "2066696e697368696e67", + "7305": "20636c756573", + "7306": "204d65616e7768696c65", + "7307": "20636f6e6e6563746564", + "7308": "20736369656e74697374", + "7309": "20626c6f636b6564", + "7310": "2074696768746572", + "7311": "6f6479", + "7312": "2066726565646f6d", + "7313": "20646566696e", + "7314": "20627265617468696e67", + "7315": "206272757368696e67", + "7316": "2044616e69", + "7317": "2073717565657a696e67", + "7318": "206e69", + "7319": "707061", + "7320": "20756e64", + "7321": "207769646572", + "7322": "207475636b", + "7323": "206d65726d61", + "7324": "2048616c6c6f7765656e", + "7325": "42656570", + "7326": "207477696e6b6c6564", + "7327": "2070756e6973686d656e74", + "7328": "204a6f616e", + "7329": "207261696e73", + "7330": "20536f66", + "7331": "656c696e67", + "7332": "20776f62626c65", + "7333": "466f6c6c6f77", + "7334": "2064726f776e", + "7335": "20416e61", + "7336": "616e746173746963", + "7337": "2046696f6e61", + "7338": "20736f75", + "7339": "206564676573", + "7340": "2072656d696e64696e67", + "7341": "20627275", + "7342": "20676967676c6573", + "7343": "20517569636b", + "7344": "2064697374757262696e67", + "7345": "2069676e6f72696e67", + "7346": "2067656e6965", + "7347": "207479696e67", + "7348": "20666c656173", + "7349": "2070617373706f727473", + "7350": "5761746368", + "7351": "20737566666572696e67", + "7352": "206f66666963657273", + "7353": "207269736b73", + "7354": "20696e6a", + "7355": "2070756d706b696e73", + "7356": "2079656c", + "7357": "61746879", + "7358": "2065617365", + "7359": "2066696e616c", + "7360": "2066726f776e696e67", + "7361": "20707265706172696e67", + "7362": "204a65616e", + "7363": "20736e6f77666c616b6573", + "7364": "20776174657266616c6c", + "7365": "6c657473", + "7366": "20496d", + "7367": "6f6e696573", + "7368": "6d696573", + "7369": "206c6179696e67", + "7370": "20636f6c646572", + "7371": "2070617973", + "7372": "20746970706564", + "7373": "206c69636b696e67", + "7374": "20636f74746f6e", + "7375": "2079656c706564", + "7376": "20636f6e666c696374", + "7377": "20706f7070696573", + "7378": "207061746873", + "7379": "2064616d61676564", + "7380": "6368696c64", + "7381": "20706572666f726d6564", + "7382": "206e6f746573", + "7383": "206f63636173", + "7384": "2074726176656c6c696e67", + "7385": "2063686f70706564", + "7386": "206d616b6572", + "7387": "20646566696e6974656c79", + "7388": "206c6f6f7073", + "7389": "20636c65616e6572", + "7390": "206469737472616374", + "7391": "2066616e746173746963", + "7392": "2074686f726e73", + "7393": "726579", + "7394": "20536b", + "7395": "416d79", + "7396": "206365727461696e", + "7397": "20686569676874", + "7398": "207365766572", + "7399": "2068616e64696e67", + "7400": "207261636573", + "7401": "2073757266696e67", + "7402": "206865616c696e67", + "7403": "2063616d7066697265", + "7404": "2062616e6461676573", + "7405": "496e", + "7406": "207365766572616c", + "7407": "206772616e646368696c64", + "7408": "206765617273", + "7409": "206775696465", + "7410": "205a6f6f6d79", + "7411": "756e64726564", + "7412": "206563686f6564", + "7413": "2073746f7070696e67", + "7414": "20417661", + "7415": "5375727072697365", + "7416": "206d61676e6966", + "7417": "2062757279696e67", + "7418": "206566666f727473", + "7419": "207061696e746272757368", + "7420": "4a656e", + "7421": "2058", + "7422": "436c", + "7423": "20616e676572", + "7424": "207265766572736564", + "7425": "7573686564", + "7426": "76726f6f6d", + "7427": "63656e74", + "7428": "61737479", + "7429": "2062697473", + "7430": "4c697374656e", + "7431": "20506570706572", + "7432": "20686561727462726f6b656e", + "7433": "206772616e6464", + "7434": "205a65726f", + "7435": "20706f6e696573", + "7436": "20636f6e67726174756c", + "7437": "636879", + "7438": "2074696c", + "7439": "20736e696666696e67", + "7440": "6f6f6b79", + "7441": "20756e77", + "7442": "20736861726b73", + "7443": "20676170", + "7444": "2070757272", + "7445": "20666f7265737473", + "7446": "20416c69", + "7447": "2073756767657374696f6e", + "7448": "205069707061", + "7449": "2074726176656c73", + "7450": "207472696379636c65", + "7451": "2077617272696f72", + "7452": "6164696e67", + "7453": "2073757270726973696e67", + "7454": "2070757a7a6c6564", + "7455": "20636f7474616765", + "7456": "726965", + "7457": "20426f77", + "7458": "20547279", + "7459": "7574746572", + "7460": "206275736573", + "7461": "206b6e6f7473", + "7462": "20477265656e", + "7463": "2046757a7a79", + "7464": "3f2122", + "7465": "72696572", + "7466": "20636f6c6c6170", + "7467": "206163636f6d706c6973686564", + "7468": "61636c6573", + "7469": "20736e6f7274", + "7470": "20636865657273", + "7471": "20736d656c6c696e67", + "7472": "207461626c6574", + "7473": "206170707265636961746564", + "7474": "20626f74686572696e67", + "7475": "206c656e64696e67", + "7476": "207370697473", + "7477": "2067657374757265", + "7478": "20696e737472756d656e7473", + "7479": "65726d", + "7480": "69676874696e67", + "7481": "20636f7567", + "7482": "204f6b6179", + "7483": "20627261766572", + "7484": "205374617274", + "7485": "20666973686573", + "7486": "2076696e6573", + "7487": "63686572", + "7488": "206b696e6773", + "7489": "2077686f7365", + "7490": "204d6164646965", + "7491": "204a6178", + "7492": "206578706c6f726572", + "7493": "20736c697070696e67", + "7494": "20656c66", + "7495": "4a6f65", + "7496": "4c697361", + "7497": "20736f616b696e67", + "7498": "2066726f776e73", + "7499": "207375666665726564", + "7500": "7374657073", + "7501": "2074726f74746564", + "7502": "207265756e69746564", + "7503": "2044657370697465", + "7504": "20726174686572", + "7505": "20726573706f6e", + "7506": "20666f72676976696e67", + "7507": "20616e746963", + "7508": "20507265747479", + "7509": "20726573706f6e7365", + "7510": "2073656c6c6572", + "7511": "2067617262", + "7512": "207365776564", + "7513": "206d656d62", + "7514": "2073706f6f6b79", + "7515": "20636f6e7175", + "7516": "20486f6c6c79", + "7517": "20756e6465727374616e6473", + "7518": "207465616368657273", + "7519": "20617272616e676564", + "7520": "2046726564646965", + "7521": "20536f666961", + "7522": "204665656c696e67", + "7523": "2067617262616765", + "7524": "206d656d62657273", + "7525": "627574", + "7526": "20726570656174696e67", + "7527": "54616b65", + "7528": "2070726f62", + "7529": "20707570696c73", + "7530": "20726573706f6e736962", + "7531": "20636f77626f79", + "7532": "20666f6f747374657073", + "7533": "20636f616368", + "7534": "20706c616e74696e67", + "7535": "207370696573", + "7536": "4d617279", + "7537": "207377656574657374", + "7538": "20657863657074", + "7539": "6973696f6e73", + "7540": "2070726f6261626c79", + "7541": "20726f626f7473", + "7542": "207368697073", + "7543": "204c69616d", + "7544": "204275626261", + "7545": "2067726970", + "7546": "20416c6c79", + "7547": "61696c73", + "7548": "206c656e74", + "7549": "2077617374696e67", + "7550": "2062656c696576696e67", + "7551": "206261736b657473", + "7552": "727570", + "7553": "2073717565616b6564", + "7554": "206578616d696e696e67", + "7555": "207765656473", + "7556": "20637564646c6573", + "7557": "2073757266626f617264", + "7558": "20497a7a79", + "7559": "2072617973", + "7560": "2053706c617368", + "7561": "205479", + "7562": "456d", + "7563": "476574", + "7564": "20726566757365", + "7565": "2061776f", + "7566": "2063616c6d6c79", + "7567": "6c696e65", + "7568": "56726f6f6d", + "7569": "696c6461", + "7570": "20636861726765", + "7571": "207370656c6c73", + "7572": "697363756974", + "7573": "207065656c6564", + "7574": "207379727570", + "7575": "206c617074", + "7576": "206472697665776179", + "7577": "666f756e64", + "7578": "2073686f6f", + "7579": "20666f637573", + "7580": "2066696e73", + "7581": "2077686973706572696e67", + "7582": "207374756d626c65", + "7583": "2068756e64726564", + "7584": "205261696e", + "7585": "2068756d6d696e67", + "7586": "2066696775726573", + "7587": "2062616e676564", + "7588": "20737570706f72746564", + "7589": "206368616e676573", + "7590": "20457863697465646c79", + "7591": "206465636973696f6e73", + "7592": "207477656574", + "7593": "2064697363757373696e67", + "7594": "61747572646179", + "7595": "42756e", + "7596": "2070656173", + "7597": "2077697468696e", + "7598": "206e61737479", + "7599": "206e7572736573", + "7600": "20646f74", + "7601": "2064756d70", + "7602": "2061666172", + "7603": "20726561737375726564", + "7604": "206461726c", + "7605": "207477697374696e67", + "7606": "207061636b61676573", + "7607": "206461726c696e6773", + "7608": "75737479", + "7609": "69656e73", + "7610": "20616c69656e73", + "7611": "20696d61676573", + "7612": "20656c626f77", + "7613": "206368696c6c79", + "7614": "2076656869636c6573", + "7615": "2063726f776e73", + "7616": "20656e636f756e74", + "7617": "207475626573", + "7618": "207065726d6974746564", + "7619": "205361747572646179", + "7620": "6361", + "7621": "466c", + "7622": "206c696573", + "7623": "206c75636b696c79", + "7624": "2050696e", + "7625": "696e6b6c65", + "7626": "206d61696c6564", + "7627": "416c7269676874", + "7628": "20726f7474696e67", + "7629": "2048656c656e", + "7630": "20636c6173736d61746573", + "7631": "20776973656c79", + "7632": "726577", + "7633": "706572617465", + "7634": "7269616e", + "7635": "6f6e6e6965", + "7636": "206d6f6f64", + "7637": "20676f6f646279", + "7638": "63726f77", + "7639": "206d65726d616964", + "7640": "20636f6e63657274", + "7641": "20616e6e6f796564", + "7642": "206c6170746f70", + "7643": "206177617265", + "7644": "6464696e67", + "7645": "206d617374657270", + "7646": "2073776f", + "7647": "6d65726564", + "7648": "207265617070", + "7649": "207370656564696e67", + "7650": "4a6f6579", + "7651": "207265736375", + "7652": "206163746976697479", + "7653": "20676f6f6462796573", + "7654": "20626561", + "7655": "6176656e", + "7656": "207374616c6c", + "7657": "64646965", + "7658": "20666c6970706564", + "7659": "20746f6f6c626f78", + "7660": "61756c61", + "7661": "2073776f6c6c656e", + "7662": "207765616c7468", + "7663": "204a6f6e6e79", + "7664": "206765726d", + "7665": "20466c6f7261", + "7666": "205275667573", + "7667": "72657374", + "7668": "2074617070696e67", + "7669": "696768696e67", + "7670": "2072696e", + "7671": "61636c65", + "7672": "20646570", + "7673": "204c696e61", + "7674": "20436c65", + "7675": "205065707061", + "7676": "756767696e67", + "7677": "206e6962626c6564", + "7678": "2062656c6f6e67696e6773", + "7679": "206f7474657273", + "7680": "2064656c61796564", + "7681": "206e6577666f756e64", + "7682": "20726f6262657273", + "7683": "20646f6f7262656c6c", + "7684": "2077696e646f7773696c6c", + "7685": "20746f77617264", + "7686": "206775657373696e67", + "7687": "69646579", + "7688": "2073696c6c", + "7689": "20636f6e766572736174696f6e", + "7690": "20636f6d666f7274696e67", + "7691": "20536565696e67", + "7692": "2062697264686f757365", + "7693": "206d6174657269616c", + "7694": "2044616e69656c", + "7695": "6161", + "7696": "207363656e74", + "7697": "206465636c", + "7698": "696e676572", + "7699": "2061696d", + "7700": "616d6d79", + "7701": "206a617273", + "7702": "20616c74686f756768", + "7703": "20416e79", + "7704": "204b6576696e", + "7705": "206578706c61696e73", + "7706": "20736f6c76696e67", + "7707": "2063686f636f6c61746573", + "7708": "476f6f646e69676874", + "7709": "2064657370697465", + "7710": "2062726163656c657473", + "7711": "204d6f74686572", + "7712": "20746172", + "7713": "2061776f6b65", + "7714": "20656d6572", + "7715": "2065786368", + "7716": "204669676874696e67", + "7717": "205068", + "7718": "207069636e696373", + "7719": "20706c6561646564", + "7720": "206c6162656c73", + "7721": "776865", + "7722": "20537069646579", + "7723": "2054696d65", + "7724": "2074726f75626c6573", + "7725": "2073696d706c79", + "7726": "666174686572", + "7727": "2074726561746564", + "7728": "206275646479", + "7729": "20707269636b", + "7730": "204d6567616e", + "7731": "204d61726965", + "7732": "20696e73697374696e67", + "7733": "207461706564", + "7734": "20706564616c6564", + "7735": "2073747564656e74", + "7736": "206f70706f73", + "7737": "20416e67656c", + "7738": "20636f72726563746c79", + "7739": "20656e636f75726167656d656e74", + "7740": "616279", + "7741": "2053746172746c6564", + "7742": "204265636b79", + "7743": "2073706564", + "7744": "6f6f6b696e67", + "7745": "2064616e67657273", + "7746": "6173657273", + "7747": "205075", + "7748": "20746964696564", + "7749": "20627573696e657373", + "7750": "206e6962626c65", + "7751": "207477656e7479", + "7752": "20637573746f6d657273", + "7753": "69646572", + "7754": "205377", + "7755": "206a6f696e73", + "7756": "20756e74696564", + "7757": "204261727279", + "7758": "77697365", + "7759": "206c6f736573", + "7760": "66697665", + "7761": "69706573", + "7762": "207370696c6c73", + "7763": "205465737361", + "7764": "2061646472657373", + "7765": "20776967676c79", + "7766": "20526f676572", + "7767": "206d6167617a696e6573", + "7768": "20696e6368776f726d", + "7769": "617473", + "7770": "6c6573636f7065", + "7771": "2076656767", + "7772": "20736375727279696e67", + "7773": "20737765617479", + "7774": "206375726c", + "7775": "207468616e6b696e67", + "7776": "20666f726d", + "7777": "20686f6e6b696e67", + "7778": "20746172676574", + "7779": "206f70706f73697465", + "7780": "20776f756e6473", + "7781": "4d616e6479", + "7782": "206d61727665", + "7783": "6f72646572", + "7784": "206772616e646461756768746572", + "7785": "20666f786573", + "7786": "20747572746c6573", + "7787": "20756e666f7274756e6174656c79", + "7788": "2074656c6573636f7065", + "7789": "20636f6e67726174756c61746564", + "7790": "206d61737465727069656365", + "7791": "207371756177", + "7792": "20696e737069726564", + "7793": "20616e7469636970", + "7794": "205365", + "7795": "736f6c", + "7796": "206c6970737469636b", + "7797": "20736d6173686564", + "7798": "2076696272", + "7799": "20617274776f726b", + "7800": "2073757276697665", + "7801": "626563", + "7802": "207472616e73", + "7803": "2062656175746966756c6c79", + "7804": "206a7564", + "7805": "6d756e6b", + "7806": "20657865726369736573", + "7807": "20707572706f7365", + "7808": "204f757473696465", + "7809": "5768", + "7810": "204564646965", + "7811": "70656e", + "7812": "20537065", + "7813": "2070726179657273", + "7814": "204d616e", + "7815": "6f72746572", + "7816": "20646973706c61796564", + "7817": "204475636b", + "7818": "617468616e", + "7819": "2061706f6c6f6769736564", + "7820": "206372797374616c73", + "7821": "204f74686572", + "7822": "2077686973746c6573", + "7823": "20736372616d", + "7824": "20706f736974", + "7825": "20736176", + "7826": "206e6f76656c73", + "7827": "2079756d", + "7828": "20656d6572676564", + "7829": "204c696f6e", + "7830": "2050696e67", + "7831": "2073746f726564", + "7832": "205465727279", + "7833": "616379", + "7834": "416c77617973", + "7835": "20636f766572696e67", + "7836": "6d616e6461", + "7837": "206d616e6e657273", + "7838": "206a6f6b6564", + "7839": "7765", + "7840": "2073746174756573", + "7841": "697473", + "7842": "20666c697073", + "7843": "2057696c", + "7844": "2076656767696573", + "7845": "207761726d6c79", + "7846": "20506f6f72", + "7847": "20626565706564", + "7848": "204d6165", + "7849": "20526f626f", + "7850": "77686174", + "7851": "206169736c65", + "7852": "207075727273", + "7853": "206e656172657374", + "7854": "2052757468", + "7855": "20426568", + "7856": "2073656167756c6c73", + "7857": "204e616e61", + "7858": "2067686f737473", + "7859": "2070616e646173", + "7860": "2067617a696e67", + "7861": "2064657369676e696e67", + "7862": "206265616d696e67", + "7863": "20736361726563726f77", + "7864": "20636869706d756e6b", + "7865": "207772696e6b6c6564", + "7866": "72696c79", + "7867": "2072656c79", + "7868": "20696e746572", + "7869": "20536d6f", + "7870": "206772616e64666174686572", + "7871": "20717565", + "7872": "206e757a7a6c6564", + "7873": "2074617374696e67", + "7874": "20466c6f", + "7875": "2062656568", + "7876": "2070726573656e746564", + "7877": "20416d616e6461", + "7878": "20446f7279", + "7879": "2074797065", + "7880": "20616e74696369706174696f6e", + "7881": "207472616e7366", + "7882": "617573", + "7883": "20456c7361", + "7884": "65726b", + "7885": "2064697363", + "7886": "20636c6561726564", + "7887": "205a6565", + "7888": "207377656570", + "7889": "204b617479", + "7890": "426f7973", + "7891": "6c74", + "7892": "5477", + "7893": "2068656e", + "7894": "20666f726365", + "7895": "6f6c70", + "7896": "20666c617073", + "7897": "75646f6c70", + "7898": "2076697369746f7273", + "7899": "205061756c61", + "7900": "20676c6974746572696e67", + "7901": "20737465726e6c79", + "7902": "20726f7574696e65", + "7903": "2064726f776e6564", + "7904": "75646f6c7068", + "7905": "2072617473", + "7906": "20737079", + "7907": "6761696e", + "7908": "75656c", + "7909": "20646f64676564", + "7910": "2057656c636f6d65", + "7911": "656c6965", + "7912": "20756e746965", + "7913": "20656e636f756e7465726564", + "7914": "20636f6e76696e", + "7915": "206669726573", + "7916": "5465646479", + "7917": "20646973636f766572696573", + "7918": "206d6f73717569746f6573", + "7919": "20736372616d626c6564", + "7920": "204d656c69737361", + "7921": "207265736375696e67", + "7922": "20626561766572", + "7923": "61627261", + "7924": "69766961", + "7925": "20637261776c73", + "7926": "2077686f6f", + "7927": "206465636c61726564", + "7928": "726f6d697365", + "7929": "2070656570", + "7930": "207371756973686564", + "7931": "20626c6f6e64", + "7932": "206469736f", + "7933": "2062697474656e", + "7934": "207469726573", + "7935": "73637265656e", + "7936": "20717565656e73", + "7937": "20636c65726b", + "7938": "2073657276616e74", + "7939": "20666c69636b", + "7940": "20746f7373", + "7941": "696c6c69616d", + "7942": "656c6c65", + "7943": "2067726174", + "7944": "2047617279", + "7945": "6964656e6365", + "7946": "20426568696e64", + "7947": "20636f6e6e656374", + "7948": "20736865657473", + "7949": "207368696d6d6572696e67", + "7950": "6f7264696e617279", + "7951": "205065676779", + "7952": "20617374726f6e61757473", + "7953": "2061747461636b6564", + "7954": "20636f636f6f6e", + "7955": "20637261636b73", + "7956": "205469676572", + "7957": "204672616e6b6965", + "7958": "206f6265796564", + "7959": "2052756479", + "7960": "206d6f6d6d696573", + "7961": "2045766572", + "7962": "207075636b", + "7963": "20677261746974756465", + "7964": "616d6d61", + "7965": "20746572726966696564", + "7966": "42616e67", + "7967": "20766973697473", + "7968": "20437261626279", + "7969": "206e65737473", + "7970": "20736872756e6b", + "7971": "756d6d696573", + "7972": "206d696c6c696f6e", + "7973": "205275646f6c7068", + "7974": "6461", + "7975": "20696e6372656173696e67", + "7976": "206f70706f7274", + "7977": "206469616d6f6e6473", + "7978": "2073686f72746572", + "7979": "2061646d69726174696f6e", + "7980": "20656e6473", + "7981": "5361726168", + "7982": "206e6f6464696e67", + "7983": "207377696d6d6572", + "7984": "206f70706f7274756e", + "7985": "2067656e65726f73697479", + "7986": "20456c6c65", + "7987": "2062726561746873", + "7988": "2073756e73637265656e", + "7989": "6c6f", + "7990": "61746f7273", + "7991": "206c6f63", + "7992": "2057696e", + "7993": "20736e7567", + "7994": "44656172", + "7995": "2047726574", + "7996": "20446f67", + "7997": "207a6f6f6d73", + "7998": "20436869636b", + "7999": "206261636b7061636b73", + "8000": "4c65617665", + "8001": "696e746d656e74", + "8002": "20657870657269656e636564", + "8003": "206d61746579", + "8004": "2068656176656e", + "8005": "2067616c6c", + "8006": "536f", + "8007": "6c6f72", + "8008": "2073656e73", + "8009": "2073776170", + "8010": "20736164646572", + "8011": "2041756e746965", + "8012": "6374726963", + "8013": "20636c69636b6564", + "8014": "20696e63686573", + "8015": "20726f776564", + "8016": "20756e69666f726d73", + "8017": "20636f77626f7973", + "8018": "736d", + "8019": "6c696564", + "8020": "206170706c61757365", + "8021": "206f7264657273", + "8022": "20666f72676976656e657373", + "8023": "20686f6f", + "8024": "756d656e74", + "8025": "206472657373696e67", + "8026": "204c7563", + "8027": "20627269676874657374", + "8028": "204d61697379", + "8029": "206d656d626572", + "8030": "20486f6c64", + "8031": "206963696e67", + "8032": "2073637261746368696e67", + "8033": "656c766574", + "8034": "2073776179696e67", + "8035": "2065787472616f7264696e617279", + "8036": "206772616e64736f6e", + "8037": "416c6c", + "8038": "204d696e", + "8039": "2062617468696e67", + "8040": "20676f6f64696573", + "8041": "2062656174696e67", + "8042": "207175657374696f6e6564", + "8043": "207374656572696e67", + "8044": "206d616e61676572", + "8045": "2074696c746564", + "8046": "20636f6c6c6170736564", + "8047": "20726f62696e", + "8048": "706974", + "8049": "2042616e67", + "8050": "2074696e67", + "8051": "207368616c6c", + "8052": "6f726174696f6e", + "8053": "2050656e", + "8054": "2053616e64", + "8055": "2057696c6c69616d", + "8056": "206472697070696e67", + "8057": "20737761796564", + "8058": "206661696c6564", + "8059": "2074756d6d696573", + "8060": "2054616b696e67", + "8061": "20736c6f776564", + "8062": "207377697463686564", + "8063": "20427269616e", + "8064": "6f6f6b73", + "8065": "206361726573", + "8066": "20616d75", + "8067": "20666c6f6f7273", + "8068": "206465617273", + "8069": "20676f617473", + "8070": "626f61726473", + "8071": "20726573706f6e646564", + "8072": "20647261676f6e666c79", + "8073": "2072616d70", + "8074": "207370726f7574", + "8075": "207363727562", + "8076": "20666c6f617473", + "8077": "20537072696e67", + "8078": "20636f6e64756374", + "8079": "2062656568697665", + "8080": "204d6174696c6461", + "8081": "204c697a61", + "8082": "2054796c6572", + "8083": "636965", + "8084": "4d72", + "8085": "466c75666679", + "8086": "206765726d73", + "8087": "2043617261", + "8088": "207072697a", + "8089": "2053746570", + "8090": "206465636f726174696f6e", + "8091": "2067617370", + "8092": "206d6963726f77", + "8093": "2074756d626c6564", + "8094": "20656e7669726f6e", + "8095": "207265636f72646572", + "8096": "20696e737472756d656e74", + "8097": "20666c617368696e67", + "8098": "73746f726d", + "8099": "2063686172616374657273", + "8100": "204c79", + "8101": "726f6c6c", + "8102": "4d6167", + "8103": "207374726f6b65", + "8104": "656d7074", + "8105": "616374", + "8106": "20656e7669726f6e6d656e74", + "8107": "20706172746564", + "8108": "206c6f7765726564", + "8109": "207472757374696e67", + "8110": "2061646473", + "8111": "47656f726765", + "8112": "20537065656479", + "8113": "204a6f6a6f", + "8114": "20656c656374726963", + "8115": "20737570706c696564", + "8116": "546f6f", + "8117": "2073636f6f74657273", + "8118": "20416c66", + "8119": "43686f6f", + "8120": "20426f6e6e6965", + "8121": "2073636f6c6473", + "8122": "20466f7274", + "8123": "2073657473", + "8124": "205068696c", + "8125": "2067726f616e", + "8126": "20736e617073", + "8127": "2074696479696e67", + "8128": "206d696e6473", + "8129": "41727272", + "8130": "20617474656d7074", + "8131": "206d6963726f77617665", + "8132": "7761746572", + "8133": "2043686f6f", + "8134": "6964656e7473", + "8135": "2072656d696e6473", + "8136": "206b6e6f636b696e67", + "8137": "20746f6f74686272757368", + "8138": "2056656c766574", + "8139": "204f6c69766961", + "8140": "206d656f7773", + "8141": "2064657374726f79696e67", + "8142": "20626164676572", + "8143": "20666f6e64", + "8144": "20666c6970", + "8145": "63636964656e7473", + "8146": "616e676c65", + "8147": "2073746f6d70", + "8148": "2074657374696e67", + "8149": "62656c6c65", + "8150": "206f66666572696e67", + "8151": "20717561636b696e67", + "8152": "206170706c617564696e67", + "8153": "204d6f6e6b6579", + "8154": "20637573746f6d6572", + "8155": "20636f6e647563746f72", + "8156": "20536179", + "8157": "20466f7274756e6174656c79", + "8158": "616e61", + "8159": "20626c6f776e", + "8160": "207465616d73", + "8161": "63686965766f7573", + "8162": "6174697665", + "8163": "207265616c697a6573", + "8164": "64646c6573", + "8165": "20636865636b73", + "8166": "206d756e6368696e67", + "8167": "2068756e74657273", + "8168": "2070726576656e746564", + "8169": "20776561706f6e73", + "8170": "696c6974696573", + "8171": "204e6f7261", + "8172": "206d697363686965766f7573", + "8173": "207368696e", + "8174": "20657870657274", + "8175": "6e6f77", + "8176": "206f757466697473", + "8177": "2068617264657374", + "8178": "76696e6365", + "8179": "206d6f64656c73", + "8180": "2073637269", + "8181": "72696e6b", + "8182": "20636f6e76696e6365", + "8183": "20666966", + "8184": "20686176656e", + "8185": "73686970", + "8186": "2073696e67657273", + "8187": "20426f756e6365", + "8188": "20637265616d73", + "8189": "20486f7065", + "8190": "205072696e6365", + "8191": "20796f756e67657374", + "8192": "3c73797374656d3e", + "8193": "3c2f73797374656d3e", + "8194": "3c757365723e", + "8195": "3c2f757365723e", + "8196": "3c617373697374616e743e", + "8197": "3c2f617373697374616e743e" + }, + "vocab_size": 8198, + "merges": [ + [104, 101, 258], + [32, 116, 259], + [32, 97, 260], + [32, 115, 261], + [32, 119, 262], + [110, 100, 263], + [259, 258, 264], + [101, 100, 265], + [260, 263, 266], + [259, 111, 267], + [32, 98, 268], + [105, 110, 269], + [32, 104, 270], + [114, 101, 271], + [262, 97, 272], + [111, 117, 273], + [32, 102, 274], + [105, 116, 275], + [32, 84, 276], + [32, 108, 277], + [32, 99, 278], + [32, 100, 279], + [101, 114, 280], + [32, 258, 281], + [32, 109, 282], + [32, 112, 283], + [272, 115, 284], + [111, 109, 285], + [32, 111, 286], + [97, 121, 287], + [276, 258, 288], + [97, 114, 289], + [105, 115, 290], + [269, 103, 291], + [105, 108, 292], + [32, 103, 293], + [105, 100, 294], + [32, 83, 295], + [97, 116, 296], + [101, 110, 297], + [32, 110, 298], + [261, 97, 299], + [270, 97, 300], + [97, 110, 301], + [105, 109, 302], + [111, 114, 303], + [111, 110, 304], + [32, 275, 305], + [32, 72, 306], + [259, 104, 307], + [108, 108, 308], + [108, 101, 309], + [111, 116, 310], + [295, 258, 311], + [281, 114, 312], + [101, 116, 313], + [105, 114, 314], + [306, 101, 315], + [288, 121, 316], + [101, 115, 317], + [32, 269, 318], + [118, 280, 319], + [117, 116, 320], + [111, 119, 321], + [99, 107, 322], + [32, 76, 323], + [32, 101, 324], + [32, 121, 325], + [32, 117, 326], + [108, 100, 327], + [111, 111, 328], + [292, 121, 329], + [299, 294, 330], + [105, 103, 331], + [32, 114, 332], + [32, 73, 333], + [268, 101, 334], + [97, 109, 335], + [261, 116, 336], + [107, 101, 337], + [99, 101, 338], + [118, 101, 339], + [275, 104, 340], + [261, 258, 341], + [112, 112, 342], + [262, 340, 343], + [323, 329, 344], + [79, 110, 345], + [286, 110, 346], + [325, 273, 347], + [107, 265, 348], + [114, 105, 349], + [286, 102, 350], + [283, 108, 351], + [110, 116, 352], + [270, 290, 353], + [32, 66, 354], + [97, 100, 355], + [261, 111, 356], + [319, 121, 357], + [279, 287, 358], + [307, 296, 359], + [326, 112, 360], + [351, 287, 361], + [264, 121, 362], + [300, 100, 363], + [115, 116, 364], + [262, 101, 365], + [282, 285, 366], + [274, 303, 367], + [32, 77, 368], + [101, 108, 369], + [117, 110, 370], + [273, 327, 371], + [39, 115, 372], + [109, 121, 373], + [277, 105, 374], + [13, 10, 375], + [99, 104, 376], + [300, 342, 377], + [272, 352, 378], + [297, 116, 379], + [261, 104, 380], + [275, 116, 381], + [258, 114, 382], + [273, 116, 383], + [298, 310, 384], + [279, 111, 385], + [285, 101, 386], + [115, 101, 387], + [32, 357, 388], + [101, 263, 389], + [108, 121, 390], + [97, 308, 391], + [273, 263, 392], + [262, 104, 393], + [32, 107, 394], + [302, 101, 395], + [381, 309, 396], + [259, 395, 397], + [277, 396, 398], + [97, 108, 399], + [298, 101, 400], + [264, 271, 401], + [32, 290, 402], + [104, 116, 403], + [333, 116, 404], + [377, 121, 405], + [268, 331, 406], + [32, 65, 407], + [32, 271, 408], + [261, 109, 409], + [268, 320, 410], + [299, 119, 411], + [46, 34, 412], + [260, 110, 413], + [114, 121, 414], + [114, 97, 415], + [349, 389, 416], + [274, 416, 417], + [260, 115, 418], + [97, 322, 419], + [345, 101, 420], + [294, 101, 421], + [97, 337, 422], + [39, 116, 423], + [117, 103, 424], + [276, 302, 425], + [33, 34, 426], + [277, 328, 427], + [365, 271, 428], + [116, 280, 429], + [118, 265, 430], + [277, 111, 431], + [111, 271, 432], + [345, 338, 433], + [32, 420, 434], + [261, 101, 435], + [267, 111, 436], + [270, 302, 437], + [268, 111, 438], + [293, 111, 439], + [260, 271, 440], + [101, 99, 441], + [32, 74, 442], + [32, 106, 443], + [378, 265, 444], + [289, 100, 445], + [292, 108, 446], + [314, 108, 447], + [264, 109, 448], + [360, 304, 449], + [264, 314, 450], + [293, 447, 451], + [32, 383, 452], + [354, 297, 453], + [260, 116, 454], + [279, 294, 455], + [117, 114, 456], + [105, 263, 457], + [119, 287, 458], + [102, 117, 459], + [276, 285, 460], + [261, 112, 461], + [278, 371, 462], + [409, 292, 463], + [300, 339, 464], + [258, 110, 465], + [97, 269, 466], + [258, 100, 467], + [289, 116, 468], + [114, 285, 469], + [278, 301, 470], + [262, 379, 471], + [324, 120, 472], + [105, 99, 473], + [32, 87, 474], + [260, 114, 475], + [108, 112, 476], + [63, 34, 477], + [328, 100, 478], + [354, 320, 479], + [459, 108, 480], + [298, 335, 481], + [417, 115, 482], + [394, 110, 483], + [281, 476, 484], + [104, 291, 485], + [261, 287, 486], + [331, 403, 487], + [260, 308, 488], + [89, 273, 489], + [117, 109, 490], + [289, 107, 491], + [304, 101, 492], + [268, 419, 493], + [274, 370, 494], + [278, 108, 495], + [97, 271, 496], + [298, 111, 497], + [277, 101, 498], + [425, 373, 499], + [111, 112, 500], + [481, 265, 501], + [261, 99, 502], + [260, 108, 503], + [369, 116, 504], + [431, 430, 505], + [267, 121, 506], + [273, 103, 507], + [282, 301, 508], + [115, 421, 509], + [463, 265, 510], + [374, 337, 511], + [105, 322, 512], + [274, 504, 513], + [418, 348, 514], + [274, 101, 515], + [435, 101, 516], + [282, 101, 517], + [261, 386, 518], + [427, 348, 519], + [335, 101, 520], + [332, 111, 521], + [475, 392, 522], + [285, 313, 523], + [268, 114, 524], + [117, 271, 525], + [268, 314, 526], + [103, 313, 527], + [105, 338, 528], + [438, 121, 529], + [262, 371, 530], + [304, 103, 531], + [336, 468, 532], + [97, 115, 533], + [526, 100, 534], + [261, 523, 535], + [274, 97, 536], + [291, 115, 537], + [44, 34, 538], + [282, 422, 539], + [355, 101, 540], + [260, 103, 541], + [32, 489, 542], + [393, 296, 543], + [262, 303, 544], + [100, 100, 545], + [486, 115, 546], + [368, 285, 547], + [107, 291, 548], + [332, 301, 549], + [321, 110, 550], + [260, 458, 551], + [527, 382, 552], + [259, 271, 553], + [162, 226, 554], + [130, 172, 555], + [195, 554, 556], + [556, 555, 557], + [105, 101, 558], + [97, 103, 559], + [289, 265, 560], + [273, 100, 561], + [282, 540, 562], + [278, 289, 563], + [532, 265, 564], + [105, 102, 565], + [267, 552, 566], + [116, 382, 567], + [283, 491, 568], + [278, 111, 569], + [535, 485, 570], + [299, 100, 571], + [286, 567, 572], + [275, 265, 573], + [400, 119, 574], + [283, 320, 575], + [274, 469, 576], + [101, 308, 577], + [270, 424, 578], + [328, 107, 579], + [541, 466, 580], + [98, 309, 581], + [97, 376, 582], + [283, 114, 583], + [32, 69, 584], + [293, 478, 585], + [472, 99, 586], + [282, 117, 587], + [270, 386, 588], + [277, 97, 589], + [32, 70, 590], + [115, 115, 591], + [349, 265, 592], + [274, 108, 593], + [262, 465, 594], + [101, 112, 595], + [393, 111, 596], + [317, 116, 597], + [278, 104, 598], + [274, 392, 599], + [264, 110, 600], + [279, 441, 601], + [104, 296, 602], + [342, 265, 603], + [361, 291, 604], + [293, 313, 605], + [110, 110, 606], + [272, 108, 607], + [113, 117, 608], + [261, 119, 609], + [507, 403, 610], + [32, 68, 611], + [374, 348, 612], + [117, 322, 613], + [307, 537, 614], + [112, 108, 615], + [391, 121, 616], + [295, 117, 617], + [273, 115, 618], + [293, 310, 619], + [295, 335, 620], + [268, 108, 621], + [114, 111, 622], + [586, 573, 623], + [347, 114, 624], + [278, 520, 625], + [385, 103, 626], + [601, 294, 627], + [97, 339, 628], + [289, 97, 629], + [324, 357, 630], + [627, 265, 631], + [97, 120, 632], + [502, 560, 633], + [606, 97, 634], + [279, 550, 635], + [117, 364, 636], + [32, 118, 637], + [346, 101, 638], + [278, 496, 639], + [274, 457, 640], + [273, 387, 641], + [334, 99, 642], + [97, 112, 643], + [290, 116, 644], + [110, 121, 645], + [307, 301, 646], + [515, 369, 647], + [279, 355, 648], + [282, 121, 649], + [262, 446, 650], + [260, 98, 651], + [289, 110, 652], + [427, 107, 653], + [273, 114, 654], + [293, 114, 655], + [368, 632, 656], + [280, 115, 657], + [455, 110, 658], + [506, 115, 659], + [293, 415, 660], + [458, 115, 661], + [32, 337, 662], + [317, 115, 663], + [263, 280, 664], + [102, 101, 665], + [283, 111, 666], + [436, 107, 667], + [651, 383, 668], + [366, 373, 669], + [503, 661, 670], + [268, 391, 671], + [301, 116, 672], + [277, 310, 673], + [295, 629, 674], + [483, 101, 675], + [407, 634, 676], + [98, 98, 677], + [553, 101, 678], + [268, 117, 679], + [290, 101, 680], + [452, 509, 681], + [307, 610, 682], + [498, 652, 683], + [275, 101, 684], + [259, 97, 685], + [282, 432, 686], + [271, 116, 687], + [297, 265, 688], + [270, 111, 689], + [295, 111, 690], + [483, 321, 691], + [283, 117, 692], + [99, 467, 693], + [261, 303, 694], + [103, 265, 695], + [441, 105, 696], + [318, 116, 697], + [278, 296, 698], + [474, 465, 699], + [286, 327, 700], + [259, 422, 701], + [105, 97, 702], + [589, 424, 703], + [105, 339, 704], + [675, 119, 705], + [696, 399, 706], + [461, 706, 707], + [413, 121, 708], + [306, 280, 709], + [283, 101, 710], + [545, 297, 711], + [102, 429, 712], + [97, 117, 713], + [694, 414, 714], + [270, 321, 715], + [259, 114, 716], + [269, 107, 717], + [380, 321, 718], + [109, 97, 719], + [332, 370, 720], + [290, 104, 721], + [711, 390, 722], + [261, 108, 723], + [587, 376, 724], + [97, 263, 725], + [300, 263, 726], + [286, 114, 727], + [259, 119, 728], + [270, 641, 729], + [442, 111, 730], + [584, 357, 731], + [286, 112, 732], + [259, 592, 733], + [272, 429, 734], + [282, 111, 735], + [118, 297, 736], + [261, 107, 737], + [442, 419, 738], + [301, 107, 739], + [697, 111, 740], + [261, 370, 741], + [318, 509, 742], + [296, 101, 743], + [267, 327, 744], + [495, 111, 745], + [490, 112, 746], + [258, 271, 747], + [293, 628, 748], + [116, 121, 749], + [474, 101, 750], + [324, 582, 751], + [32, 565, 752], + [521, 285, 753], + [324, 296, 754], + [583, 561, 755], + [32, 608, 756], + [286, 319, 757], + [281, 445, 758], + [32, 297, 759], + [462, 110, 760], + [307, 290, 761], + [97, 338, 762], + [283, 687, 763], + [310, 382, 764], + [400, 319, 765], + [368, 702, 766], + [100, 121, 767], + [713, 387, 768], + [361, 265, 769], + [97, 107, 770], + [283, 512, 771], + [98, 121, 772], + [105, 304, 773], + [115, 265, 774], + [313, 429, 775], + [578, 695, 776], + [103, 101, 777], + [617, 722, 778], + [472, 615, 779], + [443, 636, 780], + [278, 285, 781], + [293, 271, 782], + [288, 110, 783], + [590, 469, 784], + [438, 120, 785], + [298, 528, 786], + [508, 121, 787], + [400, 265, 788], + [310, 104, 789], + [117, 101, 790], + [642, 768, 791], + [272, 116, 792], + [407, 263, 793], + [350, 102, 794], + [39, 109, 795], + [336, 114, 796], + [292, 101, 797], + [103, 414, 798], + [409, 391, 799], + [306, 290, 800], + [105, 122, 801], + [118, 291, 802], + [261, 110, 803], + [274, 111, 804], + [277, 531, 805], + [269, 101, 806], + [443, 111, 807], + [302, 399, 808], + [413, 808, 809], + [334, 289, 810], + [456, 116, 811], + [259, 414, 812], + [303, 116, 813], + [683, 265, 814], + [117, 99, 815], + [507, 104, 816], + [115, 369, 817], + [431, 339, 818], + [326, 352, 819], + [292, 100, 820], + [32, 67, 821], + [819, 292, 822], + [394, 457, 823], + [102, 116, 824], + [268, 775, 825], + [593, 321, 826], + [32, 302, 827], + [279, 304, 828], + [456, 110, 829], + [259, 101, 830], + [278, 309, 831], + [324, 263, 832], + [118, 317, 833], + [268, 597, 834], + [278, 391, 835], + [355, 121, 836], + [89, 317, 837], + [279, 271, 838], + [289, 115, 839], + [639, 480, 840], + [97, 320, 841], + [456, 112, 842], + [536, 364, 843], + [260, 100, 844], + [118, 379, 845], + [445, 297, 846], + [293, 846, 847], + [98, 280, 848], + [274, 105, 849], + [334, 841, 850], + [278, 117, 851], + [105, 317, 852], + [268, 415, 853], + [97, 387, 854], + [443, 746, 855], + [277, 561, 856], + [646, 348, 857], + [737, 121, 858], + [101, 109, 859], + [278, 258, 860], + [324, 736, 861], + [268, 121, 862], + [703, 467, 863], + [112, 116, 864], + [269, 121, 865], + [84, 104, 866], + [277, 644, 867], + [817, 102, 868], + [308, 121, 869], + [119, 110, 870], + [293, 108, 871], + [270, 811, 872], + [278, 386, 873], + [336, 446, 874], + [270, 445, 875], + [809, 115, 876], + [850, 105, 877], + [877, 480, 878], + [309, 119, 879], + [272, 121, 880], + [262, 111, 881], + [557, 197, 882], + [882, 147, 883], + [673, 115, 884], + [274, 335, 885], + [326, 664, 886], + [332, 97, 887], + [277, 313, 888], + [98, 101, 889], + [258, 115, 890], + [336, 287, 891], + [268, 355, 892], + [441, 116, 893], + [299, 665, 894], + [99, 116, 895], + [110, 291, 896], + [328, 108, 897], + [115, 104, 898], + [268, 789, 899], + [569, 108, 900], + [73, 116, 901], + [32, 80, 902], + [111, 322, 903], + [97, 364, 904], + [107, 287, 905], + [500, 309, 906], + [268, 579, 907], + [408, 109, 908], + [413, 798, 909], + [827, 112, 910], + [290, 467, 911], + [374, 430, 912], + [407, 115, 913], + [305, 115, 914], + [78, 111, 915], + [380, 371, 916], + [271, 101, 917], + [728, 111, 918], + [853, 339, 919], + [332, 265, 920], + [710, 906, 921], + [301, 101, 922], + [885, 329, 923], + [607, 348, 924], + [842, 114, 925], + [407, 712, 926], + [331, 104, 927], + [380, 496, 928], + [336, 303, 929], + [274, 269, 930], + [261, 925, 931], + [279, 301, 932], + [815, 121, 933], + [261, 117, 934], + [274, 879, 935], + [356, 304, 936], + [274, 390, 937], + [323, 933, 938], + [385, 308, 939], + [76, 313, 940], + [120, 116, 941], + [102, 432, 942], + [105, 112, 943], + [111, 98, 944], + [283, 473, 945], + [835, 265, 946], + [108, 265, 947], + [831, 301, 948], + [274, 432, 949], + [763, 749, 950], + [521, 322, 951], + [545, 265, 952], + [662, 864, 953], + [290, 265, 954], + [385, 303, 955], + [380, 865, 956], + [730, 104, 957], + [439, 291, 958], + [279, 415, 959], + [662, 595, 960], + [497, 680, 961], + [408, 399, 962], + [298, 321, 963], + [611, 355, 964], + [32, 71, 965], + [262, 457, 966], + [278, 414, 967], + [278, 271, 968], + [259, 829, 969], + [547, 373, 970], + [115, 111, 971], + [732, 688, 972], + [490, 373, 973], + [32, 421, 974], + [289, 121, 975], + [334, 942, 976], + [278, 304, 977], + [503, 971, 978], + [844, 845, 979], + [336, 111, 980], + [655, 392, 981], + [559, 101, 982], + [309, 115, 983], + [557, 226, 984], + [105, 265, 985], + [292, 390, 986], + [324, 121, 987], + [300, 115, 988], + [900, 303, 989], + [779, 432, 990], + [974, 97, 991], + [274, 289, 992], + [87, 602, 993], + [274, 114, 994], + [295, 112, 995], + [107, 317, 996], + [302, 98, 997], + [497, 952, 998], + [326, 110, 999], + [272, 114, 1000], + [287, 889, 1001], + [116, 525, 1002], + [400, 941, 1003], + [771, 265, 1004], + [495, 997, 1005], + [116, 485, 1006], + [116, 104, 1007], + [393, 797, 1008], + [463, 101, 1009], + [268, 265, 1010], + [647, 291, 1011], + [716, 613, 1012], + [111, 121, 1013], + [307, 114, 1014], + [132, 162, 1015], + [984, 1015, 1016], + [607, 548, 1017], + [101, 313, 1018], + [866, 739, 1019], + [262, 114, 1020], + [103, 280, 1021], + [979, 525, 1022], + [334, 291, 1023], + [945, 1002, 1024], + [102, 102, 1025], + [957, 110, 1026], + [307, 717, 1027], + [293, 704, 1028], + [354, 944, 1029], + [282, 97, 1030], + [274, 478, 1031], + [259, 415, 1032], + [84, 602, 1033], + [97, 99, 1034], + [524, 111, 1035], + [279, 565, 1036], + [473, 265, 1037], + [267, 119, 1038], + [867, 297, 1039], + [268, 271, 1040], + [116, 265, 1041], + [745, 115, 1042], + [272, 275, 1043], + [106, 1013, 1044], + [987, 317, 1045], + [105, 618, 1046], + [286, 870, 1047], + [424, 403, 1048], + [980, 603, 1049], + [260, 712, 1050], + [32, 78, 1051], + [274, 721, 1052], + [995, 310, 1053], + [826, 657, 1054], + [427, 548, 1055], + [408, 615, 1056], + [859, 848, 1057], + [279, 369, 1058], + [116, 291, 1059], + [617, 101, 1060], + [726, 115, 1061], + [723, 421, 1062], + [607, 107, 1063], + [380, 383, 1064], + [801, 265, 1065], + [76, 329, 1066], + [621, 790, 1067], + [32, 82, 1068], + [281, 355, 1069], + [782, 296, 1070], + [637, 111, 1071], + [324, 319, 1072], + [517, 301, 1073], + [314, 364, 1074], + [910, 813, 1075], + [97, 581, 1076], + [326, 115, 1077], + [759, 1044, 1078], + [418, 107, 1079], + [261, 392, 1080], + [400, 289, 1081], + [908, 1057, 1082], + [949, 364, 1083], + [286, 905, 1084], + [279, 290, 1085], + [309, 854, 1086], + [260, 342, 1087], + [332, 466, 1088], + [384, 1037, 1089], + [308, 321, 1090], + [677, 275, 1091], + [756, 512, 1092], + [332, 117, 1093], + [1075, 672, 1094], + [336, 512, 1095], + [881, 664, 1096], + [279, 101, 1097], + [792, 376, 1098], + [718, 265, 1099], + [111, 304, 1100], + [267, 117, 1101], + [544, 107, 1102], + [259, 399, 1103], + [719, 122, 1104], + [32, 654, 1105], + [87, 101, 1106], + [260, 1104, 1107], + [260, 99, 1108], + [438, 296, 1109], + [1056, 985, 1110], + [677, 265, 1111], + [796, 531, 1112], + [665, 271, 1113], + [772, 101, 1114], + [261, 608, 1115], + [1036, 1113, 1116], + [100, 287, 1117], + [332, 487, 1118], + [262, 747, 1119], + [804, 1090, 1120], + [103, 301, 1121], + [630, 492, 1122], + [385, 317, 1123], + [524, 487, 1124], + [887, 1091, 1125], + [326, 387, 1126], + [99, 111, 1127], + [792, 693, 1128], + [325, 577, 1129], + [270, 500, 1130], + [76, 579, 1131], + [838, 591, 1132], + [292, 97, 1133], + [536, 338, 1134], + [282, 303, 1135], + [296, 104, 1136], + [1092, 390, 1137], + [280, 265, 1138], + [642, 520, 1139], + [276, 104, 1140], + [336, 432, 1141], + [301, 777, 1142], + [261, 525, 1143], + [1116, 352, 1144], + [674, 104, 1145], + [851, 349, 1146], + [334, 297, 1147], + [260, 109, 1148], + [351, 762, 1149], + [110, 103, 1150], + [97, 1048, 1151], + [1071, 528, 1152], + [598, 820, 1153], + [1000, 109, 1154], + [325, 101, 1155], + [585, 1114, 1156], + [282, 559, 1157], + [621, 903, 1158], + [270, 927, 1159], + [367, 103, 1160], + [278, 422, 1161], + [309, 595, 1162], + [278, 579, 1163], + [259, 577, 1164], + [109, 432, 1165], + [263, 719, 1166], + [708, 1165, 1167], + [1042, 280, 1168], + [1146, 618, 1169], + [275, 99, 1170], + [442, 922, 1171], + [536, 118, 1172], + [298, 487, 1173], + [692, 308, 1174], + [273, 352, 1175], + [302, 317, 1176], + [732, 297, 1177], + [334, 108, 1178], + [101, 121, 1179], + [274, 1074, 1180], + [314, 265, 1181], + [515, 308, 1182], + [290, 115, 1183], + [498, 97, 1184], + [281, 271, 1185], + [67, 301, 1186], + [32, 122, 1187], + [97, 693, 1188], + [370, 645, 1189], + [410, 429, 1190], + [283, 466, 1191], + [435, 317, 1192], + [303, 110, 1193], + [117, 308, 1194], + [394, 294, 1195], + [660, 1111, 1196], + [1170, 465, 1197], + [394, 1197, 1198], + [609, 1018, 1199], + [593, 111, 1200], + [346, 390, 1201], + [484, 265, 1202], + [299, 263, 1203], + [307, 917, 1204], + [336, 500, 1205], + [333, 110, 1206], + [534, 115, 1207], + [282, 304, 1208], + [977, 116, 1209], + [77, 285, 1210], + [268, 275, 1211], + [377, 688, 1212], + [855, 265, 1213], + [278, 415, 1214], + [731, 492, 1215], + [276, 602, 1216], + [871, 355, 1217], + [268, 97, 1218], + [279, 114, 1219], + [849, 271, 1220], + [611, 111, 1221], + [323, 1133, 1222], + [660, 591, 1223], + [278, 114, 1224], + [342, 121, 1225], + [524, 764, 1226], + [259, 349, 1227], + [334, 104, 1228], + [290, 121, 1229], + [261, 291, 1230], + [962, 1065, 1231], + [323, 313, 1232], + [97, 314, 1233], + [1064, 265, 1234], + [282, 663, 1235], + [283, 280, 1236], + [770, 317, 1237], + [269, 115, 1238], + [929, 121, 1239], + [277, 836, 1240], + [356, 824, 1241], + [352, 115, 1242], + [325, 973, 1243], + [270, 296, 1244], + [533, 115, 1245], + [408, 1188, 1246], + [959, 119, 1247], + [39, 271, 1248], + [293, 520, 1249], + [454, 101, 1250], + [300, 802, 1251], + [1172, 303, 1252], + [763, 389, 1253], + [886, 364, 1254], + [1252, 684, 1255], + [122, 121, 1256], + [283, 643, 1257], + [408, 364, 1258], + [583, 269, 1259], + [277, 663, 1260], + [296, 280, 1261], + [119, 747, 1262], + [1014, 816, 1263], + [653, 115, 1264], + [408, 836, 1265], + [821, 301, 1266], + [107, 297, 1267], + [307, 291, 1268], + [563, 115, 1269], + [1030, 548, 1270], + [283, 496, 1271], + [279, 613, 1272], + [268, 1189, 1273], + [408, 616, 1274], + [87, 321, 1275], + [1157, 473, 1276], + [296, 382, 1277], + [849, 120, 1278], + [259, 1181, 1279], + [278, 320, 1280], + [1195, 115, 1281], + [282, 490, 1282], + [544, 327, 1283], + [666, 263, 1284], + [730, 101, 1285], + [437, 868, 1286], + [98, 111, 1287], + [279, 492, 1288], + [1101, 376, 1289], + [544, 414, 1290], + [1190, 102, 1291], + [278, 592, 1292], + [781, 291, 1293], + [334, 1121, 1294], + [698, 376, 1295], + [1158, 115, 1296], + [272, 430, 1297], + [300, 314, 1298], + [283, 469, 1299], + [494, 645, 1300], + [393, 121, 1301], + [459, 869, 1302], + [481, 101, 1303], + [84, 285, 1304], + [689, 327, 1305], + [413, 764, 1306], + [326, 774, 1307], + [826, 280, 1308], + [283, 468, 1309], + [336, 101, 1310], + [331, 110, 1311], + [274, 622, 1312], + [278, 897, 1313], + [271, 108, 1314], + [274, 446, 1315], + [590, 269, 1316], + [492, 121, 1317], + [364, 280, 1318], + [685, 364, 1319], + [435, 297, 1320], + [267, 112, 1321], + [265, 767, 1322], + [262, 911, 1323], + [1200, 303, 1324], + [105, 379, 1325], + [105, 319, 1326], + [374, 996, 1327], + [609, 302, 1328], + [782, 297, 1329], + [1058, 473, 1330], + [646, 107, 1331], + [282, 641, 1332], + [268, 559, 1333], + [277, 487, 1334], + [293, 105, 1335], + [1257, 280, 1336], + [1271, 1242, 1337], + [68, 304, 1338], + [104, 121, 1339], + [111, 869, 1340], + [992, 109, 1341], + [385, 291, 1342], + [409, 577, 1343], + [1228, 457, 1344], + [296, 773, 1345], + [460, 373, 1346], + [279, 349, 1347], + [1260, 304, 1348], + [445, 115, 1349], + [268, 321, 1350], + [261, 1162, 1351], + [689, 309, 1352], + [295, 616, 1353], + [279, 622, 1354], + [117, 1025, 1355], + [931, 680, 1356], + [453, 645, 1357], + [689, 603, 1358], + [966, 321, 1359], + [725, 121, 1360], + [544, 348, 1361], + [279, 269, 1362], + [97, 1229, 1363], + [1129, 321, 1364], + [279, 491, 1365], + [408, 355, 1366], + [553, 317, 1367], + [720, 896, 1368], + [105, 337, 1369], + [609, 291, 1370], + [380, 111, 1371], + [703, 104, 1372], + [117, 265, 1373], + [274, 1194, 1374], + [34, 46, 1375], + [261, 296, 1376], + [271, 110, 1377], + [369, 121, 1378], + [262, 478, 1379], + [461, 310, 1380], + [117, 115, 1381], + [569, 327, 1382], + [1082, 265, 1383], + [289, 291, 1384], + [723, 321, 1385], + [931, 954, 1386], + [377, 329, 1387], + [431, 364, 1388], + [1291, 390, 1389], + [117, 108, 1390], + [32, 528, 1391], + [72, 577, 1392], + [1392, 111, 1393], + [788, 265, 1394], + [261, 313, 1395], + [679, 121, 1396], + [304, 115, 1397], + [1155, 839, 1398], + [281, 327, 1399], + [1312, 103, 1400], + [1020, 531, 1401], + [297, 115, 1402], + [735, 339, 1403], + [708, 1006, 1404], + [281, 289, 1405], + [407, 116, 1406], + [639, 1302, 1407], + [803, 321, 1408], + [498, 824, 1409], + [270, 370, 1410], + [536, 314, 1411], + [32, 75, 1412], + [279, 314, 1413], + [259, 391, 1414], + [584, 109, 1415], + [838, 335, 1416], + [1120, 265, 1417], + [324, 108, 1418], + [271, 335, 1419], + [685, 581, 1420], + [907, 115, 1421], + [1051, 321, 1422], + [1236, 102, 1423], + [930, 911, 1424], + [611, 1363, 1425], + [268, 424, 1426], + [1153, 1377, 1427], + [270, 294, 1428], + [270, 310, 1429], + [260, 314, 1430], + [99, 663, 1431], + [679, 115, 1432], + [1078, 265, 1433], + [278, 904, 1434], + [271, 265, 1435], + [283, 1086, 1436], + [312, 868, 1437], + [1254, 725, 1438], + [269, 265, 1439], + [417, 390, 1440], + [475, 349, 1441], + [100, 115, 1442], + [1209, 269, 1443], + [1316, 616, 1444], + [270, 446, 1445], + [745, 387, 1446], + [282, 487, 1447], + [276, 111, 1448], + [622, 591, 1449], + [475, 109, 1450], + [1160, 310, 1451], + [354, 986, 1452], + [1418, 387, 1453], + [1035, 1267, 1454], + [115, 112, 1455], + [108, 102, 1456], + [989, 115, 1457], + [283, 271, 1458], + [679, 820, 1459], + [1330, 1046, 1460], + [378, 115, 1461], + [965, 415, 1462], + [442, 297, 1463], + [533, 525, 1464], + [533, 104, 1465], + [293, 379, 1466], + [79, 75, 1467], + [296, 429, 1468], + [408, 108, 1469], + [558, 327, 1470], + [293, 492, 1471], + [622, 285, 1472], + [259, 487, 1473], + [1441, 430, 1474], + [272, 308, 1475], + [1115, 314, 1476], + [1218, 772, 1477], + [1184, 833, 1478], + [1096, 480, 1479], + [110, 280, 1480], + [968, 296, 1481], + [288, 271, 1482], + [261, 531, 1483], + [260, 581, 1484], + [97, 118, 1485], + [518, 492, 1486], + [325, 317, 1487], + [33, 557, 1488], + [442, 302, 1489], + [293, 117, 1490], + [533, 107, 1491], + [1476, 1314, 1492], + [1443, 1373, 1493], + [1140, 290, 1494], + [105, 376, 1495], + [1315, 265, 1496], + [278, 1175, 1497], + [1040, 770, 1498], + [374, 304, 1499], + [368, 1001, 1500], + [541, 1435, 1501], + [101, 101, 1502], + [283, 717, 1503], + [666, 269, 1504], + [274, 391, 1505], + [502, 975, 1506], + [407, 373, 1507], + [660, 1166, 1508], + [1163, 852, 1509], + [115, 467, 1510], + [637, 290, 1511], + [39, 308, 1512], + [349, 317, 1513], + [282, 1237, 1514], + [1410, 798, 1515], + [443, 117, 1516], + [109, 301, 1517], + [759, 816, 1518], + [366, 379, 1519], + [463, 317, 1520], + [535, 1176, 1521], + [1434, 309, 1522], + [891, 265, 1523], + [274, 487, 1524], + [930, 616, 1525], + [969, 265, 1526], + [1135, 896, 1527], + [473, 121, 1528], + [114, 321, 1529], + [262, 313, 1530], + [46, 557, 1531], + [283, 105, 1532], + [605, 1059, 1533], + [968, 335, 1534], + [572, 115, 1535], + [368, 1340, 1536], + [781, 112, 1537], + [103, 304, 1538], + [97, 342, 1539], + [1024, 115, 1540], + [1005, 265, 1541], + [334, 582, 1542], + [273, 112, 1543], + [474, 602, 1544], + [1103, 107, 1545], + [1362, 1480, 1546], + [867, 688, 1547], + [114, 310, 1548], + [812, 291, 1549], + [282, 313, 1550], + [111, 285, 1551], + [335, 97, 1552], + [685, 292, 1553], + [495, 310, 1554], + [101, 271, 1555], + [1554, 890, 1556], + [282, 764, 1557], + [307, 271, 1558], + [587, 115, 1559], + [108, 99, 1560], + [99, 265, 1561], + [301, 338, 1562], + [515, 119, 1563], + [1107, 291, 1564], + [97, 603, 1565], + [692, 1225, 1566], + [282, 289, 1567], + [337, 121, 1568], + [1079, 115, 1569], + [648, 767, 1570], + [297, 338, 1571], + [1355, 121, 1572], + [114, 265, 1573], + [671, 1100, 1574], + [1423, 893, 1575], + [393, 684, 1576], + [461, 108, 1577], + [303, 291, 1578], + [503, 492, 1579], + [274, 456, 1580], + [283, 313, 1581], + [261, 275, 1582], + [299, 292, 1583], + [895, 303, 1584], + [79, 104, 1585], + [335, 112, 1586], + [84, 258, 1587], + [613, 121, 1588], + [524, 610, 1589], + [290, 429, 1590], + [666, 108, 1591], + [119, 108, 1592], + [655, 321, 1593], + [630, 1006, 1594], + [310, 116, 1595], + [284, 110, 1596], + [1174, 265, 1597], + [100, 280, 1598], + [796, 1142, 1599], + [435, 289, 1600], + [332, 421, 1601], + [272, 573, 1602], + [66, 320, 1603], + [932, 1021, 1604], + [283, 558, 1605], + [296, 265, 1606], + [807, 121, 1607], + [461, 491, 1608], + [521, 308, 1609], + [1097, 595, 1610], + [262, 304, 1611], + [442, 446, 1612], + [1379, 115, 1613], + [720, 115, 1614], + [270, 421, 1615], + [866, 290, 1616], + [351, 301, 1617], + [1511, 275, 1618], + [321, 280, 1619], + [498, 628, 1620], + [1027, 115, 1621], + [1107, 265, 1622], + [1559, 473, 1623], + [122, 122, 1624], + [77, 1001, 1625], + [109, 101, 1626], + [111, 115, 1627], + [259, 1322, 1628], + [374, 802, 1629], + [301, 103, 1630], + [283, 296, 1631], + [283, 297, 1632], + [1038, 1349, 1633], + [804, 120, 1634], + [643, 101, 1635], + [313, 280, 1636], + [261, 986, 1637], + [937, 291, 1638], + [495, 561, 1639], + [79, 905, 1640], + [690, 304, 1641], + [1047, 280, 1642], + [101, 319, 1643], + [424, 103, 1644], + [324, 109, 1645], + [103, 115, 1646], + [335, 317, 1647], + [109, 265, 1648], + [553, 296, 1649], + [290, 317, 1650], + [500, 101, 1651], + [756, 105, 1652], + [1007, 121, 1653], + [380, 560, 1654], + [589, 263, 1655], + [655, 743, 1656], + [466, 265, 1657], + [1299, 954, 1658], + [591, 265, 1659], + [331, 103, 1660], + [282, 1317, 1661], + [274, 1470, 1662], + [322, 313, 1663], + [277, 422, 1664], + [261, 421, 1665], + [407, 308, 1666], + [590, 108, 1667], + [1191, 116, 1668], + [261, 1590, 1669], + [1035, 337, 1670], + [1656, 480, 1671], + [105, 120, 1672], + [1135, 399, 1673], + [1032, 269, 1674], + [268, 1595, 1675], + [843, 280, 1676], + [332, 762, 1677], + [1485, 121, 1678], + [99, 309, 1679], + [278, 1360, 1680], + [441, 317, 1681], + [1309, 121, 1682], + [807, 98, 1683], + [1652, 313, 1684], + [370, 376, 1685], + [932, 338, 1686], + [524, 301, 1687], + [1178, 558, 1688], + [1532, 1681, 1689], + [553, 1464, 1690], + [365, 289, 1691], + [544, 592, 1692], + [939, 115, 1693], + [851, 112, 1694], + [115, 119, 1695], + [621, 739, 1696], + [989, 480, 1697], + [413, 1695, 1698], + [1385, 390, 1699], + [324, 309, 1700], + [293, 565, 1701], + [587, 364, 1702], + [365, 1560, 1703], + [32, 339, 1704], + [408, 582, 1705], + [964, 767, 1706], + [112, 104, 1707], + [1696, 313, 1708], + [383, 104, 1709], + [406, 1021, 1710], + [117, 275, 1711], + [111, 108, 1712], + [122, 101, 1713], + [268, 1369, 1714], + [368, 114, 1715], + [598, 111, 1716], + [117, 616, 1717], + [1210, 373, 1718], + [621, 419, 1719], + [282, 1709, 1720], + [502, 1419, 1721], + [303, 109, 1722], + [102, 813, 1723], + [803, 419, 1724], + [409, 468, 1725], + [630, 1262, 1726], + [533, 265, 1727], + [332, 1326, 1728], + [583, 111, 1729], + [108, 291, 1730], + [268, 289, 1731], + [1604, 618, 1732], + [105, 271, 1733], + [934, 376, 1734], + [807, 269, 1735], + [336, 117, 1736], + [351, 672, 1737], + [322, 265, 1738], + [484, 291, 1739], + [298, 643, 1740], + [309, 591, 1741], + [443, 101, 1742], + [1558, 119, 1743], + [442, 636, 1744], + [1413, 749, 1745], + [1259, 1431, 1746], + [83, 525, 1747], + [262, 269, 1748], + [283, 97, 1749], + [115, 313, 1750], + [563, 1548, 1751], + [261, 1311, 1752], + [967, 291, 1753], + [283, 310, 1754], + [728, 1238, 1755], + [1208, 1568, 1756], + [1038, 280, 1757], + [1350, 108, 1758], + [294, 379, 1759], + [959, 1538, 1760], + [1208, 1318, 1761], + [105, 603, 1762], + [1276, 399, 1763], + [109, 291, 1764], + [1130, 265, 1765], + [334, 101, 1766], + [1108, 99, 1767], + [303, 265, 1768], + [281, 1678, 1769], + [268, 684, 1770], + [385, 1584, 1771], + [544, 1442, 1772], + [461, 114, 1773], + [100, 101, 1774], + [735, 364, 1775], + [368, 975, 1776], + [101, 289, 1777], + [299, 1626, 1778], + [72, 105, 1779], + [98, 581, 1780], + [830, 1034, 1781], + [1341, 280, 1782], + [87, 1339, 1783], + [66, 297, 1784], + [281, 399, 1785], + [1108, 1449, 1786], + [994, 1711, 1787], + [268, 1136, 1788], + [268, 613, 1789], + [368, 1552, 1790], + [1085, 1539, 1791], + [336, 271, 1792], + [336, 613, 1793], + [745, 774, 1794], + [1310, 603, 1795], + [569, 308, 1796], + [270, 331, 1797], + [101, 560, 1798], + [647, 115, 1799], + [282, 1672, 1800], + [267, 870, 1801], + [530, 110, 1802], + [376, 897, 1803], + [281, 468, 1804], + [410, 116, 1805], + [1491, 313, 1806], + [278, 654, 1807], + [268, 1806, 1808], + [1688, 339, 1809], + [1224, 287, 1810], + [454, 116, 1811], + [754, 291, 1812], + [598, 1233, 1813], + [1462, 1166, 1814], + [1781, 382, 1815], + [524, 550, 1816], + [261, 1803, 1817], + [270, 275, 1818], + [1807, 387, 1819], + [692, 1510, 1820], + [290, 774, 1821], + [325, 445, 1822], + [609, 537, 1823], + [275, 291, 1824], + [1411, 121, 1825], + [343, 383, 1826], + [283, 419, 1827], + [116, 115, 1828], + [65, 634, 1829], + [1466, 309, 1830], + [303, 100, 1831], + [1767, 1759, 1832], + [1415, 719, 1833], + [409, 111, 1834], + [282, 269, 1835], + [283, 657, 1836], + [1703, 386, 1837], + [495, 1565, 1838], + [303, 1529, 1839], + [400, 927, 1840], + [342, 291, 1841], + [267, 1117, 1842], + [1797, 382, 1843], + [293, 1647, 1844], + [497, 1650, 1845], + [98, 114, 1846], + [105, 280, 1847], + [490, 98, 1848], + [435, 99, 1849], + [969, 115, 1850], + [1840, 98, 1851], + [336, 725, 1852], + [303, 115, 1853], + [1014, 321, 1854], + [1354, 339, 1855], + [268, 297, 1856], + [1206, 509, 1857], + [339, 108, 1858], + [279, 414, 1859], + [104, 111, 1860], + [666, 1663, 1861], + [845, 1717, 1862], + [934, 722, 1863], + [288, 314, 1864], + [1605, 338, 1865], + [578, 115, 1866], + [277, 304, 1867], + [274, 917, 1868], + [367, 527, 1869], + [400, 322, 1870], + [654, 110, 1871], + [443, 1871, 1872], + [860, 1138, 1873], + [377, 297, 1874], + [262, 537, 1875], + [1867, 1378, 1876], + [274, 1571, 1877], + [415, 895, 1878], + [101, 355, 1879], + [779, 1657, 1880], + [1667, 1572, 1881], + [99, 121, 1882], + [1707, 672, 1883], + [365, 308, 1884], + [435, 97, 1885], + [277, 743, 1886], + [105, 581, 1887], + [360, 1750, 1888], + [503, 531, 1889], + [381, 291, 1890], + [282, 296, 1891], + [929, 852, 1892], + [805, 280, 1893], + [318, 364, 1894], + [299, 1150, 1895], + [1700, 1883, 1896], + [262, 258, 1897], + [1371, 317, 1898], + [278, 1193, 1899], + [282, 1183, 1900], + [99, 890, 1901], + [278, 1151, 1902], + [716, 273, 1903], + [1022, 115, 1904], + [277, 1685, 1905], + [261, 1543, 1906], + [951, 115, 1907], + [380, 643, 1908], + [903, 265, 1909], + [380, 579, 1910], + [262, 721, 1911], + [1849, 687, 1912], + [336, 839, 1913], + [63, 557, 1914], + [336, 289, 1915], + [524, 291, 1916], + [32, 79, 1917], + [1354, 603, 1918], + [283, 842, 1919], + [620, 373, 1920], + [1789, 313, 1921], + [283, 104, 1922], + [1140, 739, 1923], + [32, 345, 1924], + [273, 110, 1925], + [109, 379, 1926], + [347, 1150, 1927], + [609, 335, 1928], + [1230, 291, 1929], + [279, 644, 1930], + [735, 430, 1931], + [121, 445, 1932], + [443, 289, 1933], + [282, 806, 1934], + [99, 317, 1935], + [283, 904, 1936], + [368, 490, 1937], + [101, 107, 1938], + [1178, 531, 1939], + [493, 1932, 1940], + [1489, 373, 1941], + [286, 1592, 1942], + [351, 1034, 1943], + [677, 121, 1944], + [267, 109, 1945], + [1151, 121, 1946], + [1175, 466, 1947], + [308, 111, 1948], + [298, 1946, 1949], + [1792, 313, 1950], + [1219, 717, 1951], + [1087, 309, 1952], + [459, 774, 1953], + [685, 548, 1954], + [685, 996, 1955], + [68, 111, 1956], + [1908, 317, 1957], + [1919, 309, 1958], + [323, 1261, 1959], + [727, 1142, 1960], + [262, 680, 1961], + [472, 112, 1962], + [662, 121, 1963], + [1516, 528, 1964], + [97, 292, 1965], + [98, 321, 1966], + [497, 387, 1967], + [408, 1455, 1968], + [276, 119, 1969], + [278, 321, 1970], + [779, 1578, 1971], + [1081, 772, 1972], + [1096, 265, 1973], + [1856, 376, 1974], + [119, 445, 1975], + [1080, 115, 1976], + [781, 317, 1977], + [32, 317, 1978], + [278, 273, 1979], + [1127, 108, 1980], + [881, 337, 1981], + [370, 103, 1982], + [260, 102, 1983], + [380, 500, 1984], + [261, 512, 1985], + [279, 317, 1986], + [1432, 121, 1987], + [282, 355, 1988], + [384, 485, 1989], + [283, 331, 1990], + [381, 1402, 1991], + [115, 121, 1992], + [516, 1648, 1993], + [83, 629, 1994], + [307, 816, 1995], + [1030, 292, 1996], + [355, 321, 1997], + [1469, 558, 1998], + [1151, 280, 1999], + [322, 115, 2000], + [318, 118, 2001], + [1305, 291, 2002], + [99, 291, 2003], + [773, 115, 2004], + [102, 721, 2005], + [259, 865, 2006], + [317, 101, 2007], + [1450, 115, 2008], + [298, 597, 2009], + [261, 1890, 2010], + [1687, 376, 2011], + [282, 491, 2012], + [1007, 1117, 2013], + [282, 292, 2014], + [1155, 289, 2015], + [269, 291, 2016], + [342, 280, 2017], + [1088, 1966, 2018], + [275, 121, 2019], + [1347, 319, 2020], + [111, 387, 2021], + [286, 98, 2022], + [415, 294, 2023], + [394, 291, 2024], + [97, 309, 2025], + [1983, 2023, 2026], + [474, 104, 2027], + [32, 51, 2028], + [332, 392, 2029], + [1701, 116, 2030], + [278, 314, 2031], + [563, 414, 2032], + [1280, 101, 2033], + [84, 302, 2034], + [281, 1948, 2035], + [274, 275, 2036], + [116, 111, 2037], + [394, 684, 2038], + [821, 386, 2039], + [279, 1999, 2040], + [1631, 1325, 2041], + [949, 319, 2042], + [374, 824, 2043], + [1287, 445, 2044], + [79, 102, 2045], + [294, 777, 2046], + [319, 265, 2047], + [710, 762, 2048], + [341, 308, 2049], + [313, 104, 2050], + [1101, 693, 2051], + [1040, 355, 2052], + [268, 1925, 2053], + [584, 308, 2054], + [461, 1100, 2055], + [569, 296, 2056], + [268, 725, 2057], + [309, 116, 2058], + [578, 101, 2059], + [474, 340, 2060], + [1832, 616, 2061], + [526, 2013, 2062], + [277, 775, 2063], + [660, 98, 2064], + [1299, 680, 2065], + [99, 280, 2066], + [1328, 1764, 2067], + [72, 1179, 2068], + [1130, 101, 2069], + [1998, 430, 2070], + [1335, 833, 2071], + [575, 115, 2072], + [1675, 309, 2073], + [364, 291, 2074], + [654, 115, 2075], + [860, 2007, 2076], + [2014, 107, 2077], + [393, 644, 2078], + [262, 415, 2079], + [1969, 1018, 2080], + [293, 1660, 2081], + [898, 806, 2082], + [274, 291, 2083], + [679, 898, 2084], + [569, 1256, 2085], + [697, 1555, 2086], + [346, 2037, 2087], + [394, 512, 2088], + [103, 103, 2089], + [260, 895, 2090], + [282, 644, 2091], + [103, 695, 2092], + [1481, 525, 2093], + [374, 665, 2094], + [283, 1619, 2095], + [399, 109, 2096], + [358, 115, 2097], + [111, 103, 2098], + [1872, 1179, 2099], + [703, 485, 2100], + [586, 1824, 2101], + [277, 1588, 2102], + [781, 1723, 2103], + [1796, 893, 2104], + [117, 352, 2105], + [461, 269, 2106], + [741, 2082, 2107], + [1894, 1879, 2108], + [274, 301, 2109], + [408, 116, 2110], + [112, 265, 2111], + [741, 645, 2112], + [324, 289, 2113], + [278, 628, 2114], + [407, 108, 2115], + [1504, 1041, 2116], + [289, 112, 2117], + [827, 559, 2118], + [318, 99, 2119], + [692, 545, 2120], + [521, 355, 2121], + [275, 376, 2122], + [1085, 1127, 2123], + [116, 297, 2124], + [369, 115, 2125], + [2110, 829, 2126], + [110, 663, 2127], + [98, 391, 2128], + [270, 1317, 2129], + [380, 2117, 2130], + [801, 101, 2131], + [461, 111, 2132], + [463, 291, 2133], + [324, 533, 2134], + [262, 285, 2135], + [1600, 693, 2136], + [117, 98, 2137], + [2135, 301, 2138], + [1203, 119, 2139], + [277, 1261, 2140], + [1463, 645, 2141], + [708, 492, 2142], + [268, 610, 2143], + [1098, 291, 2144], + [1103, 348, 2145], + [270, 303, 2146], + [581, 109, 2147], + [293, 1277, 2148], + [1729, 2147, 2149], + [261, 815, 2150], + [368, 1991, 2151], + [314, 116, 2152], + [109, 280, 2153], + [71, 478, 2154], + [1272, 115, 2155], + [1408, 1517, 2156], + [1805, 304, 2157], + [1836, 304, 2158], + [517, 313, 2159], + [1785, 1653, 2160], + [1810, 1397, 2161], + [283, 500, 2162], + [1903, 581, 2163], + [282, 582, 2164], + [332, 1651, 2165], + [1087, 983, 2166], + [1319, 121, 2167], + [431, 833, 2168], + [394, 1183, 2169], + [283, 797, 2170], + [1343, 265, 2171], + [114, 392, 2172], + [278, 101, 2173], + [929, 109, 2174], + [1187, 1551, 2175], + [351, 922, 2176], + [295, 523, 2177], + [306, 321, 2178], + [821, 104, 2179], + [283, 289, 2180], + [461, 294, 2181], + [381, 297, 2182], + [101, 297, 2183], + [32, 379, 2184], + [380, 2016, 2185], + [335, 265, 2186], + [1415, 329, 2187], + [2086, 2074, 2188], + [831, 319, 2189], + [1639, 115, 2190], + [111, 339, 2191], + [408, 115, 2192], + [262, 421, 2193], + [263, 112, 2194], + [321, 265, 2195], + [1068, 101, 2196], + [727, 1121, 2197], + [1458, 115, 2198], + [282, 1468, 2199], + [1187, 328, 2200], + [1968, 893, 2201], + [268, 491, 2202], + [278, 445, 2203], + [304, 116, 2204], + [844, 109, 2205], + [515, 1277, 2206], + [1721, 265, 2207], + [587, 100, 2208], + [270, 304, 2209], + [97, 456, 2210], + [283, 446, 2211], + [595, 389, 2212], + [324, 559, 2213], + [260, 112, 2214], + [101, 98, 2215], + [439, 317, 2216], + [994, 2204, 2217], + [803, 422, 2218], + [261, 456, 2219], + [283, 1878, 2220], + [583, 310, 2221], + [418, 1162, 2222], + [299, 102, 2223], + [435, 296, 2224], + [1851, 303, 2225], + [692, 1624, 2226], + [1698, 280, 2227], + [1227, 112, 2228], + [756, 597, 2229], + [932, 1561, 2230], + [67, 386, 2231], + [2221, 893, 2232], + [830, 582, 2233], + [2146, 387, 2234], + [117, 112, 2235], + [328, 387, 2236], + [830, 2050, 2237], + [380, 114, 2238], + [1227, 317, 2239], + [283, 1136, 2240], + [637, 446, 2241], + [290, 97, 2242], + [2198, 379, 2243], + [365, 1384, 2244], + [1087, 1798, 2245], + [1335, 672, 2246], + [1380, 1041, 2247], + [723, 294, 2248], + [1160, 704, 2249], + [818, 390, 2250], + [407, 606, 2251], + [1466, 390, 2252], + [262, 820, 2253], + [1943, 317, 2254], + [860, 280, 2255], + [323, 101, 2256], + [295, 116, 2257], + [1922, 492, 2258], + [280, 390, 2259], + [596, 309, 2260], + [393, 1495, 2261], + [376, 291, 2262], + [649, 1318, 2263], + [1413, 116, 2264], + [1448, 552, 2265], + [965, 111, 2266], + [2194, 97, 2267], + [1835, 320, 2268], + [307, 269, 2269], + [637, 399, 2270], + [910, 271, 2271], + [279, 105, 2272], + [381, 121, 2273], + [304, 317, 2274], + [374, 339, 2275], + [1093, 1774, 2276], + [2164, 806, 2277], + [442, 422, 2278], + [1390, 289, 2279], + [280, 116, 2280], + [977, 1953, 2281], + [418, 104, 2282], + [1609, 265, 2283], + [283, 1245, 2284], + [536, 1935, 2285], + [1253, 265, 2286], + [282, 457, 2287], + [278, 2096, 2288], + [259, 370, 2289], + [338, 301, 2290], + [286, 2290, 2291], + [1319, 265, 2292], + [1979, 376, 2293], + [299, 322, 2294], + [110, 473, 2295], + [801, 122, 2296], + [544, 109, 2297], + [283, 301, 2298], + [282, 265, 2299], + [1577, 533, 2300], + [283, 654, 2301], + [945, 2295, 2302], + [934, 342, 2303], + [261, 280, 2304], + [1347, 339, 2305], + [515, 313, 2306], + [268, 301, 2307], + [1716, 1980, 2308], + [461, 379, 2309], + [108, 97, 2310], + [278, 97, 2311], + [569, 269, 2312], + [299, 108, 2313], + [74, 111, 2314], + [1163, 558, 2315], + [277, 355, 2316], + [2177, 1176, 2317], + [1899, 280, 2318], + [490, 848, 2319], + [370, 107, 2320], + [1426, 115, 2321], + [272, 339, 2322], + [277, 287, 2323], + [323, 2242, 2324], + [2181, 280, 2325], + [1945, 1839, 2326], + [332, 291, 2327], + [1645, 98, 2328], + [1459, 291, 2329], + [692, 898, 2330], + [286, 736, 2331], + [1834, 789, 2332], + [521, 98, 2333], + [1580, 414, 2334], + [1020, 684, 2335], + [1428, 291, 2336], + [262, 943, 2337], + [289, 777, 2338], + [296, 303, 2339], + [32, 1311, 2340], + [354, 101, 2341], + [399, 115, 2342], + [589, 364, 2343], + [408, 99, 2344], + [679, 1780, 2345], + [598, 512, 2346], + [675, 101, 2347], + [484, 480, 2348], + [259, 294, 2349], + [261, 369, 2350], + [908, 457, 2351], + [296, 111, 2352], + [271, 355, 2353], + [804, 327, 2354], + [1184, 102, 2355], + [278, 468, 2356], + [282, 1947, 2357], + [470, 110, 2358], + [268, 879, 2359], + [1058, 487, 2360], + [2316, 1598, 2361], + [1103, 548, 2362], + [1749, 119, 2363], + [283, 897, 2364], + [289, 415, 2365], + [2308, 743, 2366], + [1524, 291, 2367], + [334, 308, 2368], + [930, 721, 2369], + [354, 117, 2370], + [1870, 108, 2371], + [804, 310, 2372], + [2358, 310, 2373], + [2328, 2365, 2374], + [282, 1001, 2375], + [461, 2353, 2376], + [336, 687, 2377], + [262, 432, 2378], + [1483, 115, 2379], + [331, 467, 2380], + [436, 108, 2381], + [598, 1727, 2382], + [1043, 291, 2383], + [1773, 287, 2384], + [309, 120, 2385], + [2120, 309, 2386], + [565, 101, 2387], + [2307, 301, 2388], + [380, 121, 2389], + [2333, 310, 2390], + [109, 111, 2391], + [408, 112, 2392], + [1278, 265, 2393], + [324, 1862, 2394], + [2350, 2005, 2395], + [298, 313, 2396], + [284, 104, 2397], + [356, 108, 2398], + [1731, 348, 2399], + [268, 296, 2400], + [277, 2338, 2401], + [83, 116, 2402], + [902, 1086, 2403], + [1319, 101, 2404], + [594, 1643, 2405], + [278, 1449, 2406], + [278, 328, 2407], + [295, 386, 2408], + [746, 121, 2409], + [1390, 116, 2410], + [112, 280, 2411], + [500, 104, 2412], + [495, 789, 2413], + [2048, 480, 2414], + [796, 291, 2415], + [2027, 797, 2416], + [336, 595, 2417], + [2349, 121, 2418], + [282, 1821, 2419], + [77, 121, 2420], + [483, 1909, 2421], + [1481, 704, 2422], + [260, 352, 2423], + [336, 492, 2424], + [351, 743, 2425], + [1287, 120, 2426], + [276, 1322, 2427], + [281, 839, 2428], + [731, 1262, 2429], + [80, 1086, 2430], + [336, 314, 2431], + [309, 1846, 2432], + [461, 101, 2433], + [502, 289, 2434], + [2371, 762, 2435], + [2173, 2432, 2436], + [601, 303, 2437], + [1233, 115, 2438], + [456, 387, 2439], + [1751, 115, 2440], + [293, 297, 2441], + [341, 1456, 2442], + [1649, 115, 2443], + [303, 414, 2444], + [2241, 982, 2445], + [902, 1636, 2446], + [32, 90, 2447], + [270, 490, 2448], + [336, 415, 2449], + [716, 636, 2450], + [577, 121, 2451], + [527, 1076, 2452], + [735, 304, 2453], + [2374, 1659, 2454], + [1704, 2452, 2455], + [380, 2152, 2456], + [1982, 309, 2457], + [900, 654, 2458], + [1095, 115, 2459], + [1164, 115, 2460], + [443, 2457, 2461], + [1214, 98, 2462], + [331, 280, 2463], + [111, 327, 2464], + [830, 97, 2465], + [380, 1384, 2466], + [951, 313, 2467], + [117, 898, 2468], + [2078, 309, 2469], + [393, 290, 2470], + [1577, 1465, 2471], + [100, 777, 2472], + [855, 291, 2473], + [289, 109, 2474], + [1645, 112, 2475], + [367, 1975, 2476], + [281, 108, 2477], + [2475, 749, 2478], + [299, 339, 2479], + [655, 287, 2480], + [498, 1646, 2481], + [314, 100, 2482], + [1655, 265, 2483], + [283, 309, 2484], + [461, 1528, 2485], + [276, 86, 2486], + [269, 1041, 2487], + [497, 1442, 2488], + [301, 111, 2489], + [2282, 2186, 2490], + [293, 370, 2491], + [1473, 390, 2492], + [2391, 364, 2493], + [121, 291, 2494], + [517, 672, 2495], + [2196, 120, 2496], + [298, 320, 2497], + [279, 1636, 2498], + [606, 121, 2499], + [2256, 111, 2500], + [2211, 321, 2501], + [1742, 119, 2502], + [782, 119, 2503], + [2498, 109, 2504], + [97, 548, 2505], + [2209, 597, 2506], + [1591, 528, 2507], + [611, 304, 2508], + [503, 2493, 2509], + [1254, 478, 2510], + [474, 1339, 2511], + [268, 280, 2512], + [2314, 104, 2513], + [261, 654, 2514], + [341, 595, 2515], + [83, 335, 2516], + [502, 271, 2517], + [283, 287, 2518], + [617, 115, 2519], + [268, 399, 2520], + [2441, 280, 2521], + [264, 387, 2522], + [435, 265, 2523], + [1791, 111, 2524], + [110, 265, 2525], + [563, 592, 2526], + [336, 478, 2527], + [637, 301, 2528], + [275, 399, 2529], + [584, 736, 2530], + [103, 2172, 2531], + [1040, 1136, 2532], + [100, 297, 2533], + [516, 107, 2534], + [2402, 500, 2535], + [1129, 265, 2536], + [280, 112, 2537], + [293, 533, 2538], + [100, 309, 2539], + [105, 301, 2540], + [268, 1768, 2541], + [517, 1997, 2542], + [689, 1455, 2543], + [439, 327, 2544], + [838, 119, 2545], + [2543, 2529, 2546], + [1724, 115, 2547], + [380, 432, 2548], + [104, 269, 2549], + [277, 806, 2550], + [283, 2296, 2551], + [2150, 1431, 2552], + [498, 109, 2553], + [394, 2182, 2554], + [303, 107, 2555], + [323, 396, 2556], + [515, 289, 2557], + [326, 103, 2558], + [1235, 121, 2559], + [260, 118, 2560], + [713, 108, 2561], + [2226, 309, 2562], + [118, 618, 2563], + [2524, 2487, 2564], + [1259, 338, 2565], + [1428, 2533, 2566], + [2558, 390, 2567], + [1608, 390, 2568], + [495, 1245, 2569], + [655, 1543, 2570], + [636, 114, 2571], + [111, 1944, 2572], + [565, 102, 2573], + [354, 108, 2574], + [621, 321, 2575], + [278, 1513, 2576], + [107, 269, 2577], + [272, 833, 2578], + [304, 121, 2579], + [1010, 1472, 2580], + [328, 112, 2581], + [637, 854, 2582], + [515, 265, 2583], + [270, 2075, 2584], + [268, 829, 2585], + [2434, 102, 2586], + [691, 291, 2587], + [957, 2499, 2588], + [1737, 115, 2589], + [261, 2380, 2590], + [87, 747, 2591], + [871, 1245, 2592], + [2139, 1495, 2593], + [381, 280, 2594], + [443, 2451, 2595], + [1351, 121, 2596], + [2134, 121, 2597], + [283, 349, 2598], + [368, 973, 2599], + [2521, 618, 2600], + [666, 303, 2601], + [2095, 480, 2602], + [1042, 313, 2603], + [1852, 291, 2604], + [108, 1619, 2605], + [354, 114, 2606], + [282, 310, 2607], + [2126, 265, 2608], + [1224, 550, 2609], + [2551, 97, 2610], + [394, 1821, 2611], + [259, 597, 2612], + [259, 512, 2613], + [994, 2571, 2614], + [83, 2444, 2615], + [685, 1048, 2616], + [2115, 528, 2617], + [259, 379, 2618], + [2537, 446, 2619], + [698, 2619, 2620], + [298, 280, 2621], + [380, 943, 2622], + [283, 269, 2623], + [2109, 1882, 2624], + [2031, 99, 2625], + [283, 314, 2626], + [830, 839, 2627], + [2513, 110, 2628], + [1287, 767, 2629], + [2132, 292, 2630], + [356, 327, 2631], + [65, 271, 2632], + [119, 2555, 2633], + [300, 1150, 2634], + [112, 419, 2635], + [624, 868, 2636], + [881, 1456, 2637], + [655, 2409, 2638], + [1591, 684, 2639], + [332, 355, 2640], + [2621, 2563, 2641], + [39, 100, 2642], + [97, 102, 2643], + [593, 559, 2644], + [310, 101, 2645], + [268, 1578, 2646], + [328, 102, 2647], + [1331, 480, 2648], + [269, 97, 2649], + [1402, 704, 2650], + [860, 119, 2651], + [533, 774, 2652], + [498, 103, 2653], + [283, 2652, 2654], + [289, 108, 2655], + [728, 447, 2656], + [368, 491, 2657], + [1962, 2650, 2658], + [2268, 317, 2659], + [502, 114, 2660], + [2504, 1439, 2661], + [1219, 500, 2662], + [860, 364, 2663], + [932, 2003, 2664], + [1183, 1853, 2665], + [2614, 1606, 2666], + [350, 2124, 2667], + [270, 456, 2668], + [817, 833, 2669], + [108, 487, 2670], + [293, 743, 2671], + [605, 115, 2672], + [2620, 289, 2673], + [1097, 280, 2674], + [502, 2665, 2675], + [495, 550, 2676], + [1978, 99, 2677], + [2263, 1046, 2678], + [280, 97, 2679], + [524, 2046, 2680], + [313, 115, 2681], + [408, 103, 2682], + [771, 115, 2683], + [272, 2092, 2684], + [1627, 2210, 2685], + [2083, 280, 2686], + [109, 98, 2687], + [32, 1467, 2688], + [586, 684, 2689], + [274, 292, 2690], + [446, 265, 2691], + [272, 664, 2692], + [2148, 265, 2693], + [274, 331, 2694], + [1412, 2273, 2695], + [117, 399, 2696], + [431, 322, 2697], + [294, 2539, 2698], + [299, 430, 2699], + [278, 643, 2700], + [801, 1256, 2701], + [1362, 2685, 2702], + [2091, 422, 2703], + [296, 317, 2704], + [544, 548, 2705], + [1698, 1138, 2706], + [2455, 115, 2707], + [689, 1841, 2708], + [32, 915, 2709], + [2103, 1076, 2710], + [593, 320, 2711], + [1827, 265, 2712], + [87, 1860, 2713], + [310, 115, 2714], + [438, 591, 2715], + [111, 263, 2716], + [626, 115, 2717], + [282, 314, 2718], + [608, 101, 2719], + [716, 790, 2720], + [756, 2183, 2721], + [1219, 490, 2722], + [2081, 947, 2723], + [483, 487, 2724], + [298, 2319, 2725], + [1897, 369, 2726], + [336, 1355, 2727], + [692, 115, 2728], + [277, 275, 2729], + [685, 1267, 2730], + [431, 387, 2731], + [2689, 1926, 2732], + [117, 338, 2733], + [268, 492, 2734], + [103, 1646, 2735], + [1247, 291, 2736], + [324, 2735, 2737], + [1787, 115, 2738], + [278, 725, 2739], + [1448, 1117, 2740], + [259, 1635, 2741], + [1093, 1439, 2742], + [282, 973, 2743], + [2299, 473, 2744], + [84, 747, 2745], + [346, 317, 2746], + [262, 290, 2747], + [1214, 1592, 2748], + [278, 349, 2749], + [418, 548, 2750], + [279, 597, 2751], + [2574, 790, 2752], + [282, 2698, 2753], + [461, 389, 2754], + [696, 616, 2755], + [1199, 558, 2756], + [1343, 121, 2757], + [374, 1738, 2758], + [1063, 115, 2759], + [114, 303, 2760], + [2519, 558, 2761], + [589, 1256, 2762], + [2206, 115, 2763], + [577, 97, 2764], + [274, 422, 2765], + [2744, 806, 2766], + [260, 545, 2767], + [1788, 1472, 2768], + [112, 2755, 2769], + [1978, 2769, 2770], + [871, 790, 2771], + [424, 289, 2772], + [472, 335, 2773], + [495, 643, 2774], + [1891, 376, 2775], + [369, 112, 2776], + [521, 289, 2777], + [333, 102, 2778], + [598, 1142, 2779], + [336, 2274, 2780], + [269, 663, 2781], + [524, 2468, 2782], + [438, 767, 2783], + [293, 289, 2784], + [277, 2494, 2785], + [671, 115, 2786], + [497, 100, 2787], + [835, 291, 2788], + [1930, 1562, 2789], + [261, 487, 2790], + [261, 2772, 2791], + [735, 802, 2792], + [624, 115, 2793], + [2300, 467, 2794], + [830, 335, 2795], + [533, 280, 2796], + [497, 1229, 2797], + [268, 422, 2798], + [259, 558, 2799], + [1219, 739, 2800], + [365, 770, 2801], + [584, 1862, 2802], + [2484, 1727, 2803], + [735, 118, 2804], + [593, 1572, 2805], + [498, 355, 2806], + [2718, 2760, 2807], + [601, 421, 2808], + [286, 107, 2809], + [104, 97, 2810], + [282, 473, 2811], + [270, 1193, 2812], + [366, 115, 2813], + [583, 287, 2814], + [78, 321, 2815], + [642, 386, 2816], + [278, 982, 2817], + [274, 290, 2818], + [72, 2776, 2819], + [104, 943, 2820], + [559, 265, 2821], + [365, 1938, 2822], + [1791, 1798, 2823], + [569, 2047, 2824], + [1644, 597, 2825], + [332, 294, 2826], + [2553, 304, 2827], + [32, 85, 2828], + [361, 2531, 2829], + [1469, 632, 2830], + [261, 2825, 2831], + [296, 291, 2832], + [435, 308, 2833], + [718, 115, 2834], + [1583, 265, 2835], + [438, 445, 2836], + [1502, 122, 2837], + [871, 321, 2838], + [1020, 2645, 2839], + [2119, 1573, 2840], + [313, 101, 2841], + [1038, 369, 2842], + [399, 618, 2843], + [354, 2572, 2844], + [1371, 310, 2845], + [1742, 2843, 2846], + [529, 115, 2847], + [831, 289, 2848], + [2229, 2004, 2849], + [269, 1345, 2850], + [1516, 1528, 2851], + [860, 322, 2852], + [517, 1464, 2853], + [2213, 280, 2854], + [356, 643, 2855], + [279, 331, 2856], + [303, 121, 2857], + [727, 100, 2858], + [728, 644, 2859], + [1068, 2021, 2860], + [274, 806, 2861], + [1209, 379, 2862], + [788, 115, 2863], + [332, 496, 2864], + [737, 1762, 2865], + [32, 265, 2866], + [121, 399, 2867], + [948, 265, 2868], + [32, 457, 2869], + [807, 337, 2870], + [324, 327, 2871], + [119, 947, 2872], + [1214, 2872, 2873], + [446, 105, 2874], + [2607, 303, 2875], + [2054, 558, 2876], + [2840, 1887, 2877], + [66, 101, 2878], + [332, 1495, 2879], + [2054, 97, 2880], + [2345, 115, 2881], + [885, 618, 2882], + [2784, 982, 2883], + [302, 112, 2884], + [544, 100, 2885], + [274, 1277, 2886], + [472, 116, 2887], + [2606, 550, 2888], + [584, 263, 2889], + [1012, 115, 2890], + [1687, 1901, 2891], + [279, 721, 2892], + [872, 115, 2893], + [2388, 97, 2894], + [77, 101, 2895], + [105, 342, 2896], + [2197, 1065, 2897], + [611, 301, 2898], + [116, 105, 2899], + [1874, 291, 2900], + [268, 1839, 2901], + [259, 559, 2902], + [1372, 115, 2903], + [987, 101, 2904], + [545, 121, 2905], + [1715, 115, 2906], + [1848, 947, 2907], + [99, 292, 2908], + [105, 2017, 2909], + [272, 263, 2910], + [280, 121, 2911], + [1632, 2908, 2912], + [569, 319, 2913], + [994, 550, 2914], + [103, 117, 2915], + [431, 2867, 2916], + [302, 265, 2917], + [999, 1381, 2918], + [102, 473, 2919], + [282, 643, 2920], + [277, 1586, 2921], + [2918, 2696, 2922], + [593, 1465, 2923], + [962, 954, 2924], + [856, 280, 2925], + [2502, 369, 2926], + [307, 512, 2927], + [609, 301, 2928], + [1490, 663, 2929], + [2346, 297, 2930], + [609, 1831, 2931], + [108, 852, 2932], + [393, 2025, 2933], + [105, 2719, 2934], + [385, 476, 2935], + [261, 292, 2936], + [569, 364, 2937], + [259, 2463, 2938], + [1036, 2919, 2939], + [260, 119, 2940], + [598, 854, 2941], + [533, 291, 2942], + [2887, 415, 2943], + [735, 100, 2944], + [794, 1138, 2945], + [689, 387, 2946], + [1537, 313, 2947], + [274, 654, 2948], + [569, 1238, 2949], + [2123, 2047, 2950], + [338, 864, 2951], + [999, 2934, 2952], + [2694, 525, 2953], + [102, 904, 2954], + [268, 652, 2955], + [112, 813, 2956], + [477, 46, 2957], + [483, 2387, 2958], + [1675, 285, 2959], + [270, 2105, 2960], + [2939, 2410, 2961], + [259, 811, 2962], + [300, 109, 2963], + [380, 422, 2964], + [2520, 1562, 2965], + [296, 1378, 2966], + [121, 101, 2967], + [337, 595, 2968], + [2458, 480, 2969], + [346, 338, 2970], + [76, 1133, 2971], + [902, 2841, 2972], + [267, 424, 2973], + [72, 1555, 2974], + [1498, 2954, 2975], + [979, 456, 2976], + [413, 103, 2977], + [2043, 265, 2978], + [2871, 2259, 2979], + [87, 577, 2980], + [1505, 115, 2981], + [1335, 802, 2982], + [461, 370, 2983], + [2512, 1513, 2984], + [260, 952, 2985], + [415, 103, 2986], + [365, 2482, 2987], + [2449, 119, 2988], + [312, 111, 2989], + [274, 897, 2990], + [44, 557, 2991], + [268, 1468, 2992], + [1574, 115, 2993], + [108, 725, 2994], + [368, 121, 2995], + [2976, 618, 2996], + [559, 317, 2997], + [785, 317, 2998], + [431, 103, 2999], + [823, 2127, 3000], + [280, 414, 3001], + [484, 115, 3002], + [2079, 603, 3003], + [859, 111, 3004], + [121, 109, 3005], + [1218, 327, 3006], + [1504, 1059, 3007], + [1834, 337, 3008], + [737, 269, 3009], + [1051, 3004, 3010], + [710, 101, 3011], + [283, 746, 3012], + [106, 893, 3013], + [2031, 1679, 3014], + [1244, 115, 3015], + [279, 2701, 3016], + [380, 1635, 3017], + [1712, 2098, 3018], + [534, 558, 3019], + [456, 265, 3020], + [1416, 265, 3021], + [1447, 121, 3022], + [875, 280, 3023], + [1644, 947, 3024], + [2947, 275, 3025], + [2271, 1659, 3026], + [399, 107, 3027], + [1226, 115, 3028], + [621, 1551, 3029], + [655, 550, 3030], + [618, 317, 3031], + [590, 1573, 3032], + [324, 839, 3033], + [508, 2821, 3034], + [298, 1722, 3035], + [368, 289, 3036], + [2973, 104, 3037], + [2214, 3018, 3038], + [279, 2212, 3039], + [258, 355, 3040], + [856, 390, 3041], + [598, 2236, 3042], + [502, 496, 3043], + [1939, 265, 3044], + [2012, 313, 3045], + [2344, 1831, 3046], + [368, 1360, 3047], + [2866, 777, 3048], + [105, 118, 3049], + [2968, 280, 3050], + [2079, 112, 3051], + [439, 296, 3052], + [282, 504, 3053], + [296, 525, 3054], + [1172, 654, 3055], + [332, 816, 3056], + [611, 1588, 3057], + [1073, 115, 3058], + [270, 3031, 3059], + [580, 364, 3060], + [443, 419, 3061], + [292, 116, 3062], + [408, 102, 3063], + [354, 2764, 3064], + [278, 382, 3065], + [2470, 112, 3066], + [2715, 121, 3067], + [271, 836, 3068], + [1078, 291, 3069], + [524, 2874, 3070], + [660, 338, 3071], + [994, 487, 3072], + [3035, 399, 3073], + [274, 296, 3074], + [723, 1762, 3075], + [1736, 677, 3076], + [683, 291, 3077], + [32, 1528, 3078], + [3055, 684, 3079], + [3070, 672, 3080], + [270, 2474, 3081], + [1600, 376, 3082], + [691, 115, 3083], + [97, 119, 3084], + [2990, 721, 3085], + [259, 280, 3086], + [368, 1369, 3087], + [272, 1538, 3088], + [503, 3068, 3089], + [994, 2046, 3090], + [332, 296, 3091], + [325, 2098, 3092], + [301, 695, 3093], + [2049, 115, 3094], + [1805, 1397, 3095], + [3076, 1193, 3096], + [2311, 98, 3097], + [2223, 1378, 3098], + [2962, 309, 3099], + [336, 349, 3100], + [495, 287, 3101], + [1490, 275, 3102], + [1539, 291, 3103], + [112, 317, 3104], + [3097, 269, 3105], + [461, 762, 3106], + [2625, 983, 3107], + [851, 545, 3108], + [1475, 115, 3109], + [1191, 1059, 3110], + [111, 1225, 3111], + [794, 528, 3112], + [589, 119, 3113], + [586, 2310, 3114], + [402, 2994, 3115], + [267, 904, 3116], + [285, 265, 3117], + [1080, 265, 3118], + [272, 337, 3119], + [1416, 115, 3120], + [374, 1846, 3121], + [1029, 111, 3122], + [2988, 848, 3123], + [384, 528, 3124], + [3114, 2917, 3125], + [265, 1325, 3126], + [377, 2781, 3127], + [486, 291, 3128], + [280, 103, 3129], + [278, 622, 3130], + [468, 104, 3131], + [1800, 265, 3132], + [2255, 480, 3133], + [415, 777, 3134], + [2677, 1635, 3135], + [1810, 304, 3136], + [2270, 117, 3137], + [2804, 558, 3138], + [716, 2320, 3139], + [2301, 265, 3140], + [1736, 767, 3141], + [1034, 258, 3142], + [1214, 322, 3143], + [1108, 2951, 3144], + [332, 2647, 3145], + [799, 280, 3146], + [268, 2594, 3147], + [2080, 121, 3148], + [3039, 1076, 3149], + [1333, 115, 3150], + [3086, 349, 3151], + [101, 941, 3152], + [521, 387, 3153], + [355, 111, 3154], + [314, 112, 3155], + [849, 280, 3156], + [3102, 289, 3157], + [2022, 3126, 3158], + [999, 108, 3159], + [380, 491, 3160], + [259, 1176, 3161], + [99, 1193, 3162], + [755, 390, 3163], + [1097, 355, 3164], + [786, 390, 3165], + [1157, 110, 3166], + [497, 1992, 3167], + [278, 500, 3168], + [702, 581, 3169], + [498, 301, 3170], + [307, 117, 3171], + [99, 1325, 3172], + [2630, 265, 3173], + [1432, 890, 3174], + [2001, 573, 3175], + [115, 99, 3176], + [2831, 265, 3177], + [2869, 2212, 3178], + [3156, 338, 3179], + [277, 321, 3180], + [443, 335, 3181], + [2289, 110, 3182], + [270, 2581, 3183], + [1330, 743, 3184], + [1497, 265, 3185], + [379, 773, 3186], + [3072, 688, 3187], + [324, 2089, 3188], + [1126, 480, 3189], + [1334, 115, 3190], + [431, 1738, 3191], + [283, 292, 3192], + [365, 399, 3193], + [980, 339, 3194], + [718, 291, 3195], + [2083, 657, 3196], + [2337, 265, 3197], + [611, 628, 3198], + [1112, 280, 3199], + [759, 103, 3200], + [2180, 1548, 3201], + [119, 1349, 3202], + [830, 309, 3203], + [2660, 296, 3204], + [279, 816, 3205], + [107, 110, 3206], + [1412, 302, 3207], + [413, 3172, 3208], + [1986, 1311, 3209], + [110, 97, 3210], + [278, 2019, 3211], + [274, 2986, 3212], + [97, 122, 3213], + [380, 1997, 3214], + [1469, 3169, 3215], + [336, 1419, 3216], + [943, 101, 3217], + [807, 103, 3218], + [99, 117, 3219], + [2381, 115, 3220], + [521, 1774, 3221], + [448, 2669, 3222], + [1462, 2267, 3223], + [259, 973, 3224], + [651, 2191, 3225], + [3137, 1076, 3226], + [336, 2907, 3227], + [278, 335, 3228], + [948, 291, 3229], + [1712, 97, 3230], + [2088, 265, 3231], + [779, 1768, 3232], + [516, 109, 3233], + [623, 390, 3234], + [2106, 896, 3235], + [609, 101, 3236], + [517, 109, 3237], + [325, 652, 3238], + [303, 777, 3239], + [807, 1439, 3240], + [542, 114, 3241], + [271, 591, 3242], + [2162, 3162, 3243], + [293, 859, 3244], + [3178, 379, 3245], + [268, 746, 3246], + [320, 104, 3247], + [334, 770, 3248], + [289, 414, 3249], + [728, 331, 3250], + [682, 480, 3251], + [274, 1947, 3252], + [1305, 115, 3253], + [335, 2716, 3254], + [274, 271, 3255], + [275, 382, 3256], + [2552, 480, 3257], + [295, 2412, 3258], + [1187, 943, 3259], + [1077, 291, 3260], + [2923, 2670, 3261], + [269, 399, 3262], + [880, 115, 3263], + [286, 119, 3264], + [853, 338, 3265], + [2351, 265, 3266], + [1310, 399, 3267], + [679, 3062, 3268], + [1567, 581, 3269], + [111, 310, 3270], + [3212, 797, 3271], + [1124, 390, 3272], + [823, 115, 3273], + [300, 1456, 3274], + [3206, 550, 3275], + [114, 287, 3276], + [660, 2267, 3277], + [2398, 339, 3278], + [727, 331, 3279], + [759, 1722, 3280], + [2272, 3254, 3281], + [803, 2573, 3282], + [3280, 618, 3283], + [589, 98, 3284], + [699, 1643, 3285], + [3066, 1138, 3286], + [271, 308, 3287], + [1109, 115, 3288], + [590, 114, 3289], + [336, 296, 3290], + [259, 739, 3291], + [385, 339, 3292], + [278, 492, 3293], + [1502, 1713, 3294], + [495, 490, 3295], + [301, 320, 3296], + [1191, 1041, 3297], + [2220, 528, 3298], + [999, 3275, 3299], + [1550, 399, 3300], + [1093, 677, 3301], + [2935, 2549, 3302], + [2053, 338, 3303], + [2523, 115, 3304], + [112, 294, 3305], + [1108, 1193, 3306], + [3012, 2577, 3307], + [2937, 490, 3308], + [307, 1074, 3309], + [3279, 3262, 3310], + [2304, 1046, 3311], + [334, 317, 3312], + [759, 1858, 3313], + [475, 103, 3314], + [2205, 1181, 3315], + [101, 3239, 3316], + [1848, 657, 3317], + [759, 99, 3318], + [284, 467, 3319], + [269, 975, 3320], + [741, 102, 3321], + [640, 291, 3322], + [323, 3230, 3323], + [334, 313, 3324], + [336, 2438, 3325], + [295, 110, 3326], + [860, 643, 3327], + [807, 869, 3328], + [666, 364, 3329], + [361, 480, 3330], + [1736, 3305, 3331], + [598, 3093, 3332], + [1177, 115, 3333], + [260, 2105, 3334], + [380, 492, 3335], + [72, 321, 3336], + [488, 2195, 3337], + [1123, 110, 3338], + [292, 749, 3339], + [1490, 3339, 3340], + [710, 1780, 3341], + [2687, 3287, 3342], + [1945, 2352, 3343], + [2360, 265, 3344], + [334, 301, 3345], + [325, 313, 3346], + [326, 3342, 3347], + [2313, 355, 3348], + [2711, 101, 3349], + [3295, 1992, 3350], + [456, 291, 3351], + [771, 291, 3352], + [1351, 291, 3353], + [745, 322, 3354], + [269, 317, 3355], + [1127, 100, 3356], + [279, 1194, 3357], + [2477, 109, 3358], + [473, 116, 3359], + [118, 528, 3360], + [1205, 115, 3361], + [367, 1007, 3362], + [1490, 445, 3363], + [424, 695, 3364], + [3200, 806, 3365], + [934, 275, 3366], + [100, 285, 3367], + [308, 943, 3368], + [3368, 500, 3369], + [735, 1598, 3370], + [3309, 121, 3371], + [267, 114, 3372], + [1990, 101, 3373], + [301, 1021, 3374], + [446, 97, 3375], + [1184, 802, 3376], + [1900, 291, 3377], + [279, 636, 3378], + [3193, 1653, 3379], + [426, 46, 3380], + [270, 287, 3381], + [99, 285, 3382], + [2977, 369, 3383], + [431, 3369, 3384], + [290, 107, 3385], + [475, 1529, 3386], + [887, 1561, 3387], + [261, 304, 3388], + [3130, 3356, 3389], + [3228, 2679, 3390], + [105, 1571, 3391], + [3151, 581, 3392], + [3255, 898, 3393], + [710, 3296, 3394], + [298, 3317, 3395], + [258, 116, 3396], + [282, 290, 3397], + [730, 1179, 3398], + [336, 1586, 3399], + [2963, 2153, 3400], + [261, 717, 3401], + [558, 119, 3402], + [1093, 98, 3403], + [307, 2021, 3404], + [1754, 2352, 3405], + [1164, 291, 3406], + [1716, 387, 3407], + [1093, 1510, 3408], + [2271, 591, 3409], + [902, 101, 3410], + [1345, 115, 3411], + [1537, 1245, 3412], + [781, 98, 3413], + [1448, 772, 3414], + [274, 2385, 3415], + [1298, 121, 3416], + [2751, 622, 3417], + [1731, 848, 3418], + [380, 1909, 3419], + [3370, 110, 3420], + [278, 1555, 3421], + [2837, 265, 3422], + [1551, 121, 3423], + [637, 3402, 3424], + [32, 433, 3425], + [82, 101, 3426], + [3011, 348, 3427], + [103, 597, 3428], + [821, 108, 3429], + [2192, 99, 3430], + [830, 109, 3431], + [461, 2691, 3432], + [71, 111, 3433], + [726, 265, 3434], + [559, 3396, 3435], + [3435, 2899, 3436], + [2162, 2279, 3437], + [341, 108, 3438], + [336, 419, 3439], + [267, 789, 3440], + [1632, 2915, 3441], + [83, 1502, 3442], + [1600, 2262, 3443], + [1038, 657, 3444], + [293, 2236, 3445], + [1567, 376, 3446], + [1259, 116, 3447], + [598, 289, 3448], + [735, 309, 3449], + [443, 424, 3450], + [1093, 1111, 3451], + [1537, 320, 3452], + [2626, 743, 3453], + [3061, 313, 3454], + [622, 889, 3455], + [832, 291, 3456], + [803, 101, 3457], + [299, 2733, 3458], + [298, 2439, 3459], + [902, 296, 3460], + [406, 3428, 3461], + [1986, 107, 3462], + [3324, 309, 3463], + [1347, 802, 3464], + [32, 291, 3465], + [483, 903, 3466], + [461, 3436, 3467], + [262, 291, 3468], + [639, 1741, 3469], + [482, 2820, 3470], + [282, 1491, 3471], + [431, 355, 3472], + [336, 982, 3473], + [1416, 291, 3474], + [3265, 2058, 3475], + [105, 111, 3476], + [298, 3054, 3477], + [3171, 664, 3478], + [716, 3247, 3479], + [3071, 480, 3480], + [2032, 291, 3481], + [2598, 1713, 3482], + [3415, 1887, 3483], + [32, 280, 3484], + [116, 395, 3485], + [2560, 111, 3486], + [1573, 1325, 3487], + [2248, 317, 3488], + [3465, 3487, 3489], + [839, 104, 3490], + [262, 2122, 3491], + [283, 1468, 3492], + [407, 2385, 3493], + [278, 456, 3494], + [1224, 922, 3495], + [502, 1468, 3496], + [290, 773, 3497], + [1458, 112, 3498], + [3290, 790, 3499], + [502, 2025, 3500], + [587, 898, 3501], + [2682, 2279, 3502], + [2538, 2111, 3503], + [283, 672, 3504], + [102, 97, 3505], + [3313, 1651, 3506], + [1897, 2125, 3507], + [965, 3316, 3508], + [3182, 369, 3509], + [1058, 1326, 3510], + [262, 1733, 3511], + [1115, 496, 3512], + [3492, 110, 3513], + [259, 985, 3514], + [380, 1326, 3515], + [1836, 644, 3516], + [3412, 773, 3517], + [283, 321, 3518], + [1032, 898, 3519], + [1214, 1256, 3520], + [374, 122, 3521], + [679, 308, 3522], + [324, 3131, 3523], + [855, 115, 3524], + [1051, 3152, 3525], + [3421, 399, 3526], + [408, 1975, 3527], + [2517, 297, 3528], + [902, 1340, 3529], + [844, 3360, 3530], + [726, 309, 3531], + [3347, 97, 3532], + [2113, 390, 3533], + [2269, 548, 3534], + [3372, 110, 3535], + [1860, 364, 3536], + [517, 399, 3537], + [3409, 704, 3538], + [72, 101, 3539], + [282, 369, 3540], + [1359, 115, 3541], + [1587, 121, 3542], + [79, 119, 3543], + [759, 118, 3544], + [473, 890, 3545], + [323, 579, 3546], + [1236, 109, 3547], + [577, 331, 3548], + [796, 3374, 3549], + [108, 328, 3550], + [354, 1777, 3551], + [2118, 1439, 3552], + [332, 424, 3553], + [1095, 657, 3554], + [697, 3548, 3555], + [1632, 645, 3556], + [293, 3536, 3557], + [968, 743, 3558], + [3389, 797, 3559], + [1532, 2489, 3560], + [692, 112, 3561], + [73, 102, 3562], + [268, 379, 3563], + [3489, 115, 3564], + [859, 379, 3565], + [984, 128, 3566], + [384, 101, 3567], + [1000, 2525, 3568], + [272, 1025, 3569], + [660, 263, 3570], + [3382, 1723, 3571], + [1032, 1858, 3572], + [105, 996, 3573], + [315, 271, 3574], + [3452, 280, 3575], + [105, 677, 3576], + [999, 1679, 3577], + [2132, 337, 3578], + [1996, 2426, 3579], + [97, 870, 3580], + [2180, 540, 3581], + [3501, 1472, 3582], + [2219, 102, 3583], + [1608, 947, 3584], + [116, 429, 3585], + [1773, 291, 3586], + [446, 558, 3587], + [1749, 309, 3588], + [3571, 1076, 3589], + [260, 3040, 3590], + [3121, 975, 3591], + [367, 107, 3592], + [3417, 121, 3593], + [1593, 291, 3594], + [1796, 289, 3595], + [2436, 743, 3596], + [1423, 1722, 3597], + [2229, 773, 3598], + [727, 1598, 3599], + [999, 3589, 3600], + [2184, 104, 3601], + [270, 579, 3602], + [481, 317, 3603], + [3321, 2605, 3604], + [892, 390, 3605], + [1034, 890, 3606], + [2123, 319, 3607], + [1537, 2058, 3608], + [1497, 291, 3609], + [334, 595, 3610], + [1127, 97, 3611], + [2215, 415, 3612], + [2022, 3013, 3613], + [694, 1828, 3614], + [1032, 112, 3615], + [3601, 1381, 3616], + [2139, 3545, 3617], + [793, 121, 3618], + [1462, 338, 3619], + [3025, 704, 3620], + [1366, 291, 3621], + [1291, 2932, 3622], + [282, 446, 3623], + [269, 338, 3624], + [2448, 581, 3625], + [934, 711, 3626], + [3521, 445, 3627], + [97, 294, 3628], + [850, 121, 3629], + [259, 1491, 3630], + [1067, 848, 3631], + [356, 3505, 3632], + [3081, 1741, 3633], + [593, 654, 3634], + [943, 115, 3635], + [1631, 376, 3636], + [282, 287, 3637], + [1410, 103, 3638], + [571, 390, 3639], + [1694, 2044, 3640], + [3318, 273, 3641], + [2407, 548, 3642], + [1922, 310, 3643], + [3516, 379, 3644], + [2749, 1663, 3645], + [3258, 558, 3646], + [286, 3585, 3647], + [408, 338, 3648], + [1253, 291, 3649], + [803, 3024, 3650], + [278, 1586, 3651], + [1187, 3612, 3652], + [89, 287, 3653], + [3373, 304, 3654], + [303, 672, 3655], + [1093, 983, 3656], + [2063, 115, 3657], + [336, 1345, 3658], + [380, 320, 3659], + [999, 1326, 3660], + [2960, 280, 3661], + [400, 296, 3662], + [278, 1349, 3663], + [270, 2896, 3664], + [99, 320, 3665], + [2080, 558, 3666], + [2858, 3320, 3667], + [115, 317, 3668], + [293, 314, 3669], + [1943, 265, 3670], + [569, 3611, 3671], + [117, 100, 3672], + [2888, 558, 3673], + [2497, 115, 3674], + [349, 99, 3675], + [279, 424, 3676], + [408, 1953, 3677], + [1560, 2489, 3678], + [3448, 1764, 3679], + [3555, 379, 3680], + [902, 2561, 3681], + [871, 3423, 3682], + [354, 446, 3683], + [413, 120, 3684], + [965, 704, 3685], + [295, 104, 3686], + [298, 1965, 3687], + [293, 490, 3688], + [700, 280, 3689], + [103, 636, 3690], + [498, 1465, 3691], + [2057, 982, 3692], + [2340, 3655, 3693], + [934, 2411, 3694], + [1032, 322, 3695], + [2656, 265, 3696], + [354, 97, 3697], + [2433, 265, 3698], + [1085, 3690, 3699], + [598, 1562, 3700], + [1701, 1041, 3701], + [2192, 644, 3702], + [98, 947, 3703], + [97, 1025, 3704], + [2828, 110, 3705], + [111, 101, 3706], + [473, 317, 3707], + [532, 115, 3708], + [325, 3580, 3709], + [268, 854, 3710], + [278, 811, 3711], + [3684, 1046, 3712], + [293, 303, 3713], + [442, 3001, 3714], + [598, 2942, 3715], + [3711, 466, 3716], + [277, 265, 3717], + [283, 2579, 3718], + [558, 102, 3719], + [708, 1262, 3720], + [1227, 1630, 3721], + [495, 2137, 3722], + [895, 500, 3723], + [3723, 1381, 3724], + [1936, 97, 3725], + [2093, 115, 3726], + [1395, 116, 3727], + [403, 2137, 3728], + [1247, 280, 3729], + [1071, 3678, 3730], + [1227, 322, 3731], + [71, 704, 3732], + [2400, 3728, 3733], + [1064, 115, 3734], + [2270, 790, 3735], + [261, 389, 3736], + [77, 632, 3737], + [283, 2439, 3738], + [1811, 1878, 3739], + [710, 2017, 3740], + [1115, 1465, 3741], + [380, 2505, 3742], + [2557, 480, 3743], + [2631, 1847, 3744], + [286, 3724, 3745], + [502, 3270, 3746], + [259, 456, 3747], + [503, 2280, 3748], + [338, 265, 3749], + [781, 615, 3750], + [2437, 743, 3751], + [1097, 2643, 3752], + [2356, 1100, 3753], + [488, 331, 3754], + [967, 364, 3755], + [282, 424, 3756], + [274, 117, 3757], + [443, 313, 3758], + [289, 2540, 3759], + [99, 1419, 3760], + [117, 1944, 3761], + [910, 296, 3762], + [3504, 115, 3763], + [407, 271, 3764], + [563, 112, 3765], + [380, 806, 3766], + [295, 109, 3767], + [1095, 280, 3768], + [1124, 280, 3769], + [280, 1076, 3770], + [2668, 414, 3771], + [364, 349, 3772], + [117, 263, 3773], + [66, 2967, 3774], + [1304, 373, 3775], + [97, 348, 3776], + [2690, 1653, 3777], + [2053, 1561, 3778], + [3762, 1325, 3779], + [259, 1733, 3780], + [474, 747, 3781], + [1458, 1659, 3782], + [2220, 1037, 3783], + [1979, 3134, 3784], + [3397, 3770, 3785], + [1174, 115, 3786], + [3699, 291, 3787], + [2275, 390, 3788], + [2303, 813, 3789], + [456, 98, 3790], + [278, 854, 3791], + [3747, 1568, 3792], + [1930, 3790, 3793], + [3664, 111, 3794], + [105, 597, 3795], + [716, 287, 3796], + [3544, 1046, 3797], + [3517, 743, 3798], + [2640, 3476, 3799], + [788, 309, 3800], + [2377, 693, 3801], + [716, 746, 3802], + [598, 3761, 3803], + [2202, 291, 3804], + [1258, 1741, 3805], + [461, 577, 3806], + [305, 868, 3807], + [268, 1528, 3808], + [2064, 115, 3809], + [3358, 313, 3810], + [3713, 3375, 3811], + [259, 297, 3812], + [493, 2635, 3813], + [323, 1502, 3814], + [484, 1741, 3815], + [368, 3587, 3816], + [1050, 110, 3817], + [367, 116, 3818], + [1577, 275, 3819], + [3817, 1100, 3820], + [2150, 3749, 3821], + [518, 1262, 3822], + [268, 313, 3823], + [2341, 942, 3824], + [268, 349, 3825], + [1970, 115, 3826], + [737, 943, 3827], + [2313, 116, 3828], + [3765, 313, 3829], + [497, 2629, 3830], + [278, 523, 3831], + [473, 500, 3832], + [3832, 429, 3833], + [2417, 115, 3834], + [2034, 373, 3835], + [698, 115, 3836], + [2089, 291, 3837], + [2370, 2905, 3838], + [637, 105, 3839], + [1742, 595, 3840], + [1068, 111, 3841], + [3540, 304, 3842], + [1811, 473, 3843], + [708, 458, 3844], + [115, 280, 3845], + [1558, 355, 3846], + [1203, 2426, 3847], + [2363, 115, 3848], + [666, 859, 3849], + [279, 663, 3850], + [283, 725, 3851], + [963, 747, 3852], + [2201, 480, 3853], + [2477, 3833, 3854], + [517, 533, 3855], + [3739, 704, 3856], + [274, 704, 3857], + [3166, 313, 3858], + [637, 806, 3859], + [637, 421, 3860], + [2210, 672, 3861], + [1220, 2633, 3862], + [2407, 348, 3863], + [2747, 890, 3864], + [3802, 313, 3865], + [490, 2153, 3866], + [103, 110, 3867], + [1258, 3861, 3868], + [354, 1189, 3869], + [2197, 2131, 3870], + [2205, 1733, 3871], + [2081, 309, 3872], + [2392, 101, 3873], + [694, 116, 3874], + [779, 466, 3875], + [374, 1397, 3876], + [999, 2635, 3877], + [283, 370, 3878], + [364, 2438, 3879], + [295, 1360, 3880], + [1731, 1314, 3881], + [2728, 890, 3882], + [1756, 115, 3883], + [286, 3772, 3884], + [260, 1242, 3885], + [277, 294, 3886], + [2179, 2655, 3887], + [1068, 265, 3888], + [1371, 101, 3889], + [408, 1127, 3890], + [2412, 492, 3891], + [380, 1470, 3892], + [3754, 2339, 3893], + [297, 414, 3894], + [259, 643, 3895], + [303, 1076, 3896], + [1930, 672, 3897], + [1827, 982, 3898], + [332, 528, 3899], + [270, 3490, 3900], + [2377, 376, 3901], + [887, 736, 3902], + [1298, 3665, 3903], + [2517, 119, 3904], + [2215, 579, 3905], + [435, 399, 3906], + [3036, 702, 3907], + [384, 3905, 3908], + [276, 114, 3909], + [587, 1025, 3910], + [112, 289, 3911], + [274, 309, 3912], + [640, 115, 3913], + [731, 1006, 3914], + [1700, 118, 3915], + [435, 3911, 3916], + [498, 500, 3917], + [2431, 1573, 3918], + [354, 313, 3919], + [851, 898, 3920], + [307, 3719, 3921], + [904, 473, 3922], + [1032, 1238, 3923], + [3385, 657, 3924], + [258, 468, 3925], + [289, 328, 3926], + [1630, 3926, 3927], + [332, 3576, 3928], + [356, 289, 3929], + [282, 1136, 3930], + [281, 2472, 3931], + [844, 3896, 3932], + [1580, 567, 3933], + [3236, 1261, 3934], + [689, 983, 3935], + [3915, 2339, 3936], + [1475, 313, 3937], + [2914, 265, 3938], + [1040, 296, 3939], + [1085, 890, 3940], + [1220, 1517, 3941], + [2944, 597, 3942], + [269, 548, 3943], + [261, 379, 3944], + [756, 289, 3945], + [609, 2122, 3946], + [2027, 3924, 3947], + [297, 645, 3948], + [3326, 321, 3949], + [261, 309, 3950], + [377, 1847, 3951], + [439, 399, 3952], + [394, 3927, 3953], + [110, 487, 3954], + [300, 308, 3955], + [341, 313, 3956], + [1227, 603, 3957], + [326, 114, 3958], + [99, 386, 3959], + [2448, 301, 3960], + [336, 560, 3961], + [2251, 558, 3962], + [3808, 1679, 3963], + [364, 473, 3964], + [807, 996, 3965], + [810, 115, 3966], + [2739, 309, 3967], + [356, 322, 3968], + [1903, 3703, 3969], + [3850, 2280, 3970], + [1039, 291, 3971], + [1882, 1679, 3972], + [1897, 296, 3973], + [369, 2911, 3974], + [759, 3129, 3975], + [117, 889, 3976], + [261, 2131, 3977], + [3486, 99, 3978], + [268, 3573, 3979], + [910, 622, 3980], + [2811, 114, 3981], + [3755, 399, 3982], + [598, 466, 3983], + [103, 558, 3984], + [702, 3964, 3985], + [2392, 1233, 3986], + [3616, 3985, 3987], + [1370, 291, 3988], + [1247, 537, 3989], + [374, 112, 3990], + [3884, 376, 3991], + [1068, 1627, 3992], + [324, 3256, 3993], + [474, 446, 3994], + [517, 296, 3995], + [1962, 280, 3996], + [1334, 896, 3997], + [99, 105, 3998], + [1160, 628, 3999], + [105, 1841, 4000], + [3767, 340, 4001], + [2826, 291, 4002], + [270, 704, 4003], + [2433, 770, 4004], + [860, 1738, 4005], + [427, 112, 4006], + [598, 3027, 4007], + [2203, 2044, 4008], + [1127, 263, 4009], + [3430, 790, 4010], + [101, 280, 4011], + [3981, 3891, 4012], + [99, 1980, 4013], + [3917, 445, 4014], + [521, 560, 4015], + [435, 4009, 4016], + [4013, 105, 4017], + [815, 116, 4018], + [1917, 114, 4019], + [887, 824, 4020], + [1228, 628, 4021], + [2936, 319, 4022], + [2561, 565, 4023], + [3890, 3867, 4024], + [1257, 657, 4025], + [1749, 294, 4026], + [336, 285, 4027], + [259, 3976, 4028], + [261, 3866, 4029], + [283, 399, 4030], + [851, 889, 4031], + [1729, 118, 4032], + [380, 1237, 4033], + [756, 684, 4034], + [524, 1381, 4035], + [271, 3998, 4036], + [2022, 3845, 4037], + [314, 304, 4038], + [104, 478, 4039], + [1371, 1858, 4040], + [1583, 303, 4041], + [502, 2464, 4042], + [1712, 269, 4043], + [438, 2714, 4044], + [1235, 982, 4045], + [99, 904, 4046], + [679, 1624, 4047], + [3746, 280, 4048], + [3438, 429, 4049], + [1087, 4036, 4050], + [1241, 390, 4051], + [278, 3974, 4052], + [286, 296, 4053], + [500, 1339, 4054], + [647, 537, 4055], + [3484, 2796, 4056], + [2311, 895, 4057], + [282, 1552, 4058], + [503, 704, 4059], + [2810, 1225, 4060], + [3887, 558, 4061], + [2595, 2005, 4062], + [1497, 414, 4063], + [283, 3217, 4064], + [1203, 4046, 4065], + [286, 121, 4066], + [1567, 414, 4067], + [710, 1938, 4068], + [283, 265, 4069], + [2289, 97, 4070], + [2560, 1965, 4071], + [318, 107, 4072], + [354, 370, 4073], + [474, 371, 4074], + [368, 559, 4075], + [268, 269, 4076], + [278, 4023, 4077], + [3441, 269, 4078], + [314, 275, 4079], + [318, 387, 4080], + [716, 4054, 4081], + [283, 303, 4082], + [271, 664, 4083], + [575, 1059, 4084], + [1808, 2128, 4085], + [2202, 115, 4086], + [665, 101, 4087], + [325, 582, 4088], + [615, 287, 4089], + [1498, 115, 4090], + [2728, 485, 4091], + [98, 1390, 4092], + [2238, 2884, 4093], + [1178, 116, 4094], + [114, 304, 4095], + [666, 309, 4096], + [613, 265, 4097], + [521, 100, 4098], + [1505, 291, 4099], + [495, 3103, 4100], + [532, 291, 4101], + [306, 3894, 4102], + [1035, 4017, 4103], + [1504, 116, 4104], + [4071, 1076, 4105], + [102, 4087, 4106], + [121, 1777, 4107], + [1199, 3925, 4108], + [4030, 762, 4109], + [3721, 309, 4110], + [2175, 265, 4111], + [1690, 115, 4112], + [299, 1381, 4113], + [286, 108, 4114], + [293, 622, 4115], + [408, 457, 4116], + [3975, 121, 4117], + [1412, 743, 4118], + [265, 121, 4119], + [1335, 736, 4120], + [1068, 117, 4121], + [4057, 1381, 4122], + [1511, 573, 4123], + [593, 852, 4124], + [1984, 3050, 4125], + [3851, 97, 4126], + [832, 265, 4127], + [589, 3773, 4128], + [900, 2075, 4129], + [999, 4060, 4130], + [4128, 414, 4131], + [1085, 559, 4132], + [280, 500, 4133], + [1773, 717, 4134], + [4133, 108, 4135], + [278, 1831, 4136], + [282, 820, 4137], + [498, 289, 4138], + [3308, 101, 4139], + [260, 4135, 4140], + [4138, 352, 4141], + [3301, 280, 4142], + [439, 1456, 4143], + [4077, 2605, 4144], + [1872, 399, 4145], + [441, 768, 4146], + [475, 116, 4147], + [380, 540, 4148], + [1085, 4089, 4149], + [660, 3104, 4150], + [328, 2111, 4151], + [1516, 2472, 4152], + [902, 643, 4153], + [268, 456, 4154], + [4088, 116, 4155], + [1617, 313, 4156], + [297, 387, 4157], + [1088, 291, 4158], + [442, 663, 4159], + [332, 559, 4160], + [259, 632, 4161], + [101, 102, 4162], + [679, 108, 4163], + [306, 3249, 4164], + [279, 704, 4165], + [4116, 4011, 4166], + [97, 275, 4167], + [1005, 291, 4168], + [387, 490, 4169], + [1130, 291, 4170], + [277, 613, 4171], + [399, 265, 4172], + [1227, 2000, 4173], + [1087, 108, 4174], + [282, 297, 4175], + [77, 702, 4176], + [2643, 665, 4177], + [483, 310, 4178], + [1343, 115, 4179], + [1704, 263, 4180], + [461, 4079, 4181], + [72, 97, 4182], + [2372, 2128, 4183], + [1591, 721, 4184], + [587, 4169, 4185], + [380, 813, 4186], + [3192, 310, 4187], + [323, 986, 4188], + [2388, 533, 4189], + [1429, 369, 4190], + [2354, 280, 4191], + [2119, 108, 4192], + [3159, 903, 4193], + [261, 3005, 4194], + [756, 275, 4195], + [277, 2796, 4196], + [4163, 98, 4197], + [1313, 280, 4198], + [796, 1644, 4199], + [1854, 115, 4200], + [473, 280, 4201], + [3697, 772, 4202], + [115, 644, 4203], + [1458, 591, 4204], + [99, 2066, 4205], + [346, 773, 4206], + [97, 1624, 4207], + [262, 897, 4208], + [363, 110, 4209], + [283, 355, 4210], + [917, 1713, 4211], + [1206, 364, 4212], + [2727, 265, 4213], + [569, 4106, 4214], + [1567, 693, 4215], + [3839, 4043, 4216], + [1694, 115, 4217], + [994, 520, 4218], + [1851, 654, 4219], + [394, 287, 4220], + [103, 317, 4221], + [356, 4205, 4222], + [2662, 115, 4223], + [4153, 97, 4224], + [1148, 4092, 4225], + [4161, 105, 4226], + [3113, 121, 4227], + [286, 292, 4228], + [1331, 115, 4229], + [536, 815, 4230], + [719, 263, 4231], + [368, 1183, 4232], + [3170, 265, 4233], + [1287, 108, 4234], + [817, 121, 4235], + [110, 1021, 4236], + [277, 389, 4237], + [283, 1233, 4238], + [4194, 4234, 4239], + [122, 1256, 4240], + [68, 355, 4241], + [3669, 4177, 4242], + [374, 101, 4243], + [916, 110, 4244], + [1310, 770, 4245], + [2219, 4083, 4246], + [278, 1076, 4247], + [1609, 291, 4248], + [4225, 1562, 4249], + [801, 317, 4250], + [1800, 280, 4251], + [2118, 806, 4252], + [292, 291, 4253], + [737, 446, 4254], + [112, 101, 4255], + [3945, 1314, 4256], + [609, 1982, 4257], + [4230, 313, 4258], + [2818, 382, 4259], + [1497, 280, 4260], + [1939, 115, 4261], + [3123, 414, 4262], + [3608, 101, 4263], + [275, 317, 4264], + [110, 335, 4265], + [1662, 115, 4266], + [1504, 1828, 4267], + [745, 4235, 4268], + [861, 291, 4269], + [1069, 115, 4270], + [1430, 2956, 4271], + [4180, 303, 4272], + [361, 280, 4273], + [374, 833, 4274], + [302, 105, 4275], + [46, 46, 4276], + [259, 2137, 4277], + [3643, 111, 4278], + [356, 292, 4279], + [112, 309, 4280], + [272, 802, 4281], + [3518, 1598, 4282], + [2690, 109, 4283], + [2118, 2850, 4284], + [2549, 111, 4285], + [3038, 1065, 4286], + [119, 2183, 4287], + [587, 309, 4288], + [1400, 115, 4289], + [318, 4203, 4290], + [4082, 376, 4291], + [3823, 4287, 4292], + [1034, 317, 4293], + [3920, 773, 4294], + [710, 582, 4295], + [408, 4162, 4296], + [1191, 429, 4297], + [1032, 292, 4298], + [101, 1136, 4299], + [351, 3922, 4300], + [3992, 558, 4301], + [264, 1261, 4302], + [689, 322, 4303], + [4303, 1179, 4304], + [110, 1179, 4305], + [4080, 895, 4306], + [32, 4038, 4307], + [3950, 864, 4308], + [893, 265, 4309], + [268, 309, 4310], + [1218, 548, 4311], + [860, 1938, 4312], + [332, 4285, 4313], + [337, 2058, 4314], + [1451, 2124, 4315], + [3912, 97, 4316], + [442, 1647, 4317], + [2272, 975, 4318], + [1481, 265, 4319], + [1584, 121, 4320], + [598, 3155, 4321], + [3566, 156, 4322], + [262, 2387, 4323], + [1157, 3213, 4324], + [408, 319, 4325], + [2626, 2704, 4326], + [456, 376, 4327], + [2213, 2259, 4328], + [260, 120, 4329], + [4265, 379, 4330], + [1811, 3186, 4331], + [261, 314, 4332], + [283, 982, 4333], + [1455, 643, 4334], + [1575, 390, 4335], + [276, 265, 4336], + [3065, 1513, 4337], + [3431, 4280, 4338], + [536, 4320, 4339], + [608, 275, 4340], + [2337, 101, 4341], + [1748, 429, 4342], + [489, 114, 4343], + [4212, 1879, 4344], + [574, 4334, 4345], + [117, 317, 4346], + [713, 774, 4347], + [2268, 101, 4348], + [727, 4330, 4349], + [2875, 3972, 4350], + [277, 395, 4351], + [2898, 645, 4352], + [4140, 922, 4353], + [1064, 291, 4354], + [74, 302, 4355], + [3314, 790, 4356], + [368, 4275, 4357], + [3457, 770, 4358], + [637, 313, 4359], + [4227, 280, 4360], + [741, 1750, 4361], + [583, 370, 4362], + [332, 310, 4363], + [516, 291, 4364], + [4314, 304, 4365], + [2818, 485, 4366], + [4340, 111, 4367], + [3092, 811, 4368], + [2575, 291, 4369], + [4259, 1517, 4370], + [301, 121, 4371], + [303, 852, 4372], + [4075, 3984, 4373], + [2909, 121, 4374], + [2838, 291, 4375], + [312, 98, 4376], + [666, 337, 4377], + [2384, 265, 4378], + [276, 2579, 4379], + [483, 944, 4380], + [723, 4374, 4381], + [283, 275, 4382], + [270, 320, 4383], + [4220, 770, 4384], + [1708, 115, 4385], + [735, 115, 4386], + [336, 3704, 4387], + [741, 103, 4388], + [1581, 2342, 4389], + [1187, 2909, 4390], + [4329, 101, 4391], + [1130, 317, 4392], + [108, 1245, 4393], + [2159, 291, 4394], + [737, 1194, 4395], + [290, 304, 4396], + [32, 331, 4397], + [301, 473, 4398], + [4115, 2066, 4399], + [4397, 3550, 4400], + [728, 717, 4401], + [679, 414, 4402], + [1492, 115, 4403], + [308, 558, 4404], + [718, 280, 4405], + [782, 4119, 4406], + [376, 1233, 4407], + [275, 525, 4408], + [1736, 100, 4409], + [829, 4408, 4410], + [794, 280, 4411], + [1271, 352, 4412], + [407, 98, 4413], + [435, 533, 4414], + [1630, 117, 4415], + [2346, 1402, 4416], + [376, 4398, 4417], + [280, 99, 4418], + [274, 4410, 4419], + [497, 1858, 4420], + [4386, 4367, 4421], + [277, 4415, 4422], + [65, 263, 4423], + [2992, 121, 4424], + [593, 520, 4425], + [955, 115, 4426], + [365, 1277, 4427], + [268, 4211, 4428], + [1069, 265, 4429], + [394, 2273, 4430], + [261, 4365, 4431], + [517, 4417, 4432], + [272, 1646, 4433], + [637, 597, 4434], + [2263, 121, 4435], + [735, 833, 4436], + [3978, 3154, 4437], + [1450, 4407, 4438], + [1868, 122, 4439], + [270, 289, 4440], + [109, 297, 4441], + [377, 1402, 4442], + [74, 419, 4443], + [569, 399, 4444], + [830, 289, 4445], + [3860, 111, 4446], + [2544, 297, 4447], + [835, 115, 4448], + [470, 101, 4449], + [2634, 291, 4450], + [615, 762, 4451], + [917, 107, 4452], + [2436, 1606, 4453], + [369, 3959, 4454], + [278, 4452, 4455], + [569, 98, 4456], + [2299, 399, 4457], + [368, 287, 4458], + [637, 1472, 4459], + [115, 291, 4460], + [2284, 2956, 4461], + [267, 292, 4462], + [283, 806, 4463], + [472, 4418, 4464], + [2266, 100, 4465], + [871, 2191, 4466], + [1020, 1824, 4467], + [1740, 2577, 4468], + [2024, 3367, 4469], + [2066, 1627, 4470], + [3282, 265, 4471], + [109, 109, 4472], + [1095, 121, 4473], + [1703, 3117, 4474], + [119, 533, 4475], + [77, 1552, 4476], + [2749, 98, 4477], + [916, 280, 4478], + [666, 603, 4479], + [1030, 1713, 4480], + [4066, 1318, 4481], + [305, 376, 4482], + [302, 4305, 4483], + [1085, 3219, 4484], + [1097, 289, 4485], + [1412, 296, 4486], + [119, 2215, 4487], + [4393, 317, 4488], + [713, 100, 4489], + [1120, 115, 4490], + [269, 3759, 4491], + [1636, 4491, 4492], + [4475, 382, 4493], + [4462, 313, 4494], + [116, 3242, 4495], + [3710, 2128, 4496], + [598, 4483, 4497], + [4388, 4488, 4498], + [1891, 4495, 4499], + [3084, 115, 4500], + [77, 490, 4501], + [1936, 414, 4502], + [2585, 265, 4503], + [3237, 4372, 4504], + [598, 4327, 4505], + [637, 4492, 4506], + [655, 269, 4507], + [2892, 4493, 4508], + [283, 644, 4509], + [365, 927, 4510], + [4313, 4470, 4511], + [754, 297, 4512], + [1032, 339, 4513], + [2238, 3364, 4514], + [673, 773, 4515], + [1115, 101, 4516], + [442, 1390, 4517], + [1854, 291, 4518], + [4509, 1712, 4519], + [2340, 1768, 4520], + [2091, 1237, 4521], + [539, 2235, 4522], + [4456, 4487, 4523], + [370, 97, 4524], + [268, 764, 4525], + [278, 560, 4526], + [286, 770, 4527], + [370, 2966, 4528], + [2219, 2172, 4529], + [3429, 629, 4530], + [541, 271, 4531], + [261, 3624, 4532], + [756, 419, 4533], + [408, 339, 4534], + [1220, 4451, 4535], + [4174, 4489, 4536], + [382, 111, 4537], + [622, 100, 4538], + [2569, 1472, 4539], + [3694, 4537, 4540], + [108, 321, 4541], + [87, 4167, 4542], + [1704, 292, 4543], + [295, 1502, 4544], + [1853, 317, 4545], + [593, 1565, 4546], + [77, 1340, 4547], + [622, 3176, 4548], + [3909, 790, 4549], + [3916, 743, 4550], + [1583, 291, 4551], + [697, 4538, 4552], + [589, 870, 4553], + [804, 103, 4554], + [1224, 321, 4555], + [114, 487, 4556], + [116, 466, 4557], + [932, 2066, 4558], + [4548, 1651, 4559], + [259, 424, 4560], + [270, 4545, 4561], + [590, 1074, 4562], + [274, 610, 4563], + [294, 287, 4564], + [2001, 597, 4565], + [544, 1007, 4566], + [99, 854, 4567], + [1684, 390, 4568], + [1648, 105, 4569], + [2089, 121, 4570], + [609, 1586, 4571], + [116, 101, 4572], + [4345, 280, 4573], + [4569, 2966, 4574], + [261, 4250, 4575], + [591, 1887, 4576], + [118, 3497, 4577], + [108, 280, 4578], + [2811, 4559, 4579], + [2104, 265, 4580], + [97, 98, 4581], + [443, 2320, 4582], + [81, 117, 4583], + [295, 355, 4584], + [4565, 331, 4585], + [533, 290, 4586], + [432, 115, 4587], + [1792, 1150, 4588], + [1835, 280, 4589], + [66, 4146, 4590], + [2682, 687, 4591], + [269, 348, 4592], + [541, 917, 4593], + [3750, 466, 4594], + [737, 2152, 4595], + [1766, 102, 4596], + [2053, 2003, 4597], + [2243, 115, 4598], + [1458, 845, 4599], + [4588, 1007, 4600], + [2173, 4253, 4601], + [1200, 296, 4602], + [313, 121, 4603], + [278, 577, 4604], + [1716, 528, 4605], + [2120, 983, 4606], + [286, 4586, 4607], + [563, 291, 4608], + [315, 476, 4609], + [2748, 291, 4610], + [2449, 487, 4611], + [283, 456, 4612], + [259, 320, 4613], + [68, 294, 4614], + [1115, 3422, 4615], + [1813, 115, 4616], + [2214, 468, 4617], + [282, 500, 4618], + [1630, 317, 4619], + [4042, 265, 4620], + [689, 108, 4621], + [3144, 265, 4622], + [4621, 4564, 4623], + [2739, 983, 4624], + [1811, 582, 4625], + [503, 2474, 4626], + [3203, 4577, 4627], + [105, 833, 4628], + [3203, 1707, 4629], + [2926, 414, 4630], + [2470, 2411, 4631], + [3092, 97, 4632], + [376, 558, 4633], + [2056, 115, 4634], + [73, 115, 4635], + [2450, 265, 4636], + [325, 1470, 4637], + [2613, 313, 4638], + [318, 102, 4639], + [114, 291, 4640], + [1434, 983, 4641], + [753, 115, 4642], + [374, 2000, 4643], + [1058, 287, 4644], + [4324, 806, 4645], + [4585, 743, 4646], + [1097, 4231, 4647], + [1000, 100, 4648], + [285, 317, 4649], + [999, 565, 4650], + [3949, 121, 4651], + [4629, 492, 4652], + [365, 595, 4653], + [104, 641, 4654], + [895, 773, 4655], + [2169, 317, 4656], + [803, 643, 4657], + [4422, 982, 4658], + [4648, 3455, 4659], + [1087, 622, 4660], + [293, 3005, 4661], + [4440, 98, 4662], + [332, 422, 4663], + [323, 613, 4664], + [1310, 335, 4665], + [2304, 339, 4666], + [1132, 265, 4667], + [4650, 1722, 4668], + [934, 98, 4669], + [261, 927, 4670], + [1702, 3142, 4671], + [408, 3606, 4672], + [283, 415, 4673], + [336, 1076, 4674], + [3105, 313, 4675], + [474, 1860, 4676], + [1019, 115, 4677], + [1567, 592, 4678], + [621, 478, 4679], + [872, 291, 4680], + [860, 591, 4681], + [2257, 101, 4682], + [1115, 3294, 4683], + [1214, 898, 4684], + [4664, 329, 4685], + [2184, 1138, 4686], + [2640, 721, 4687], + [1719, 2044, 4688], + [3049, 399, 4689], + [262, 1660, 4690], + [621, 4592, 4691], + [888, 1059, 4692], + [4439, 280, 4693], + [1891, 1901, 4694], + [3359, 773, 4695], + [2692, 265, 4696], + [574, 115, 4697], + [3289, 1322, 4698], + [2235, 115, 4699], + [2134, 329, 4700], + [307, 1848, 4701], + [2830, 265, 4702], + [934, 1025, 4703], + [4695, 975, 4704], + [336, 622, 4705], + [907, 4567, 4706], + [960, 291, 4707], + [325, 1588, 4708], + [461, 1777, 4709], + [495, 4500, 4710], + [2300, 485, 4711], + [408, 608, 4712], + [276, 2649, 4713], + [584, 582, 4714], + [732, 2679, 4715], + [332, 680, 4716], + [534, 99, 4717], + [1107, 3565, 4718], + [484, 280, 4719], + [279, 4704, 4720], + [459, 1626, 4721], + [4717, 982, 4722], + [3569, 309, 4723], + [495, 2573, 4724], + [3631, 1513, 4725], + [2406, 265, 4726], + [267, 1150, 4727], + [1791, 1777, 4728], + [391, 1179, 4729], + [99, 1237, 4730], + [268, 446, 4731], + [1623, 2540, 4732], + [1236, 4721, 4733], + [4065, 309, 4734], + [4662, 303, 4735], + [1593, 947, 4736], + [2178, 1643, 4737], + [2238, 717, 4738], + [115, 275, 4739], + [887, 292, 4740], + [887, 122, 4741], + [3117, 287, 4742], + [2119, 271, 4743], + [2370, 1624, 4744], + [259, 4097, 4745], + [97, 104, 4746], + [716, 540, 4747], + [290, 291, 4748], + [415, 695, 4749], + [637, 4729, 4750], + [279, 592, 4751], + [376, 303, 4752], + [364, 657, 4753], + [1617, 317, 4754], + [2512, 414, 4755], + [3535, 3154, 4756], + [2226, 983, 4757], + [1917, 4404, 4758], + [284, 4572, 4759], + [4712, 597, 4760], + [3980, 339, 4761], + [2400, 258, 4762], + [3928, 304, 4763], + [710, 289, 4764], + [2297, 115, 4765], + [274, 597, 4766], + [675, 369, 4767], + [4613, 303, 4768], + [977, 102, 4769], + [104, 473, 4770], + [351, 2704, 4771], + [1178, 321, 4772], + [1835, 116, 4773], + [1804, 115, 4774], + [2214, 4095, 4775], + [4310, 265, 4776], + [261, 105, 4777], + [3063, 271, 4778], + [1220, 102, 4779], + [1704, 4770, 4780], + [117, 376, 4781], + [621, 641, 4782], + [32, 86, 4783], + [2623, 376, 4784], + [277, 2319, 4785], + [1380, 115, 4786], + [402, 110, 4787], + [3496, 265, 4788], + [109, 389, 4789], + [3142, 108, 4790], + [637, 1034, 4791], + [1132, 317, 4792], + [4121, 772, 4793], + [368, 101, 4794], + [666, 348, 4795], + [1505, 297, 4796], + [655, 446, 4797], + [1586, 328, 4798], + [105, 430, 4799], + [267, 317, 4800], + [3757, 369, 4801], + [413, 4752, 4802], + [260, 320, 4803], + [418, 403, 4804], + [278, 2025, 4805], + [261, 4742, 4806], + [4804, 3276, 4807], + [4805, 263, 4808], + [4741, 303, 4809], + [1723, 4528, 4810], + [262, 806, 4811], + [4808, 289, 4812], + [285, 4789, 4813], + [931, 1650, 4814], + [89, 613, 4815], + [334, 2092, 4816], + [2303, 390, 4817], + [2344, 4813, 4818], + [649, 868, 4819], + [438, 310, 4820], + [1120, 291, 4821], + [374, 99, 4822], + [3246, 265, 4823], + [827, 4574, 4824], + [1068, 4790, 4825], + [1894, 114, 4826], + [974, 533, 4827], + [1414, 280, 4828], + [590, 111, 4829], + [2827, 540, 4830], + [4254, 115, 4831], + [851, 98, 4832], + [3123, 1513, 4833], + [2700, 4557, 4834], + [803, 1565, 4835], + [1116, 110, 4836], + [380, 4798, 4837], + [289, 105, 4838], + [2754, 291, 4839], + [108, 1847, 4840], + [3593, 265, 4841], + [908, 2191, 4842], + [3108, 947, 4843], + [1154, 1007, 4844], + [803, 3294, 4845], + [583, 500, 4846], + [703, 403, 4847], + [2327, 291, 4848], + [3996, 3391, 4849], + [2223, 4603, 4850], + [282, 1245, 4851], + [1438, 291, 4852], + [717, 121, 4853], + [815, 1002, 4854], + [274, 797, 4855], + [87, 4454, 4856], + [1519, 115, 4857], + [796, 4854, 4858], + [887, 680, 4859], + [3709, 265, 4860], + [303, 399, 4861], + [39, 339, 4862], + [4847, 280, 4863], + [4325, 387, 4864], + [3919, 1992, 4865], + [679, 592, 4866], + [2184, 280, 4867], + [3345, 115, 4868], + [1125, 115, 4869], + [283, 813, 4870], + [323, 801, 4871], + [919, 390, 4872], + [332, 321, 4873], + [473, 291, 4874], + [461, 540, 4875], + [365, 643, 4876], + [677, 558, 4877], + [118, 313, 4878], + [643, 265, 4879], + [483, 550, 4880], + [114, 2463, 4881], + [3910, 269, 4882], + [3063, 4881, 4883], + [3410, 906, 4884], + [4883, 2339, 4885], + [2711, 429, 4886], + [4766, 4689, 4887], + [299, 802, 4888], + [821, 111, 4889], + [335, 848, 4890], + [1582, 115, 4891], + [2340, 432, 4892], + [2502, 2125, 4893], + [1421, 258, 4894], + [277, 1644, 4895], + [283, 2997, 4896], + [521, 310, 4897], + [1183, 773, 4898], + [279, 1733, 4899], + [356, 2000, 4900], + [2520, 301, 4901], + [843, 597, 4902], + [3547, 4898, 4903], + [39, 46, 4904], + [902, 3111, 4905], + [284, 485, 4906], + [875, 390, 4907], + [1005, 115, 4908], + [4895, 982, 4909], + [361, 115, 4910], + [1608, 1730, 4911], + [1900, 797, 4912], + [1818, 115, 4913], + [4486, 558, 4914], + [3939, 258, 4915], + [300, 4236, 4916], + [995, 1369, 4917], + [4050, 743, 4918], + [598, 4587, 4919], + [334, 2186, 4920], + [2437, 3411, 4921], + [3498, 496, 4922], + [1608, 309, 4923], + [285, 944, 4924], + [332, 1339, 4925], + [2113, 4840, 4926], + [2233, 291, 4927], + [306, 628, 4928], + [4822, 4157, 4929], + [270, 4649, 4930], + [1430, 615, 4931], + [4727, 790, 4932], + [404, 115, 4933], + [2751, 2850, 4934], + [4894, 1456, 4935], + [2447, 3706, 4936], + [4803, 4924, 4937], + [267, 271, 4938], + [902, 3948, 4939], + [3727, 947, 4940], + [1056, 762, 4941], + [2001, 684, 4942], + [730, 121, 4943], + [2528, 911, 4944], + [111, 774, 4945], + [959, 269, 4946], + [2398, 430, 4947], + [4027, 582, 4948], + [756, 801, 4949], + [1610, 280, 4950], + [301, 3210, 4951], + [279, 1363, 4952], + [908, 466, 4953], + [259, 421, 4954], + [2205, 314, 4955], + [79, 107, 4956], + [268, 1630, 4957], + [282, 3186, 4958], + [502, 4151, 4959], + [4791, 1345, 4960], + [293, 1777, 4961], + [368, 986, 4962], + [1445, 115, 4963], + [3660, 387, 4964], + [503, 4556, 4965], + [4673, 954, 4966], + [114, 109, 4967], + [2041, 390, 4968], + [4639, 672, 4969], + [4669, 458, 4970], + [2179, 3155, 4971], + [590, 622, 4972], + [3065, 414, 4973], + [666, 4396, 4974], + [83, 112, 4975], + [1174, 291, 4976], + [1224, 1685, 4977], + [1752, 115, 4978], + [941, 525, 4979], + [341, 100, 4980], + [731, 1117, 4981], + [2773, 806, 4982], + [500, 115, 4983], + [264, 4967, 4984], + [360, 3879, 4985], + [3284, 369, 4986], + [1258, 432, 4987], + [583, 421, 4988], + [268, 770, 4989], + [279, 1660, 4990], + [84, 111, 4991], + [379, 115, 4992], + [502, 432, 4993], + [280, 111, 4994], + [4990, 291, 4995], + [727, 4619, 4996], + [461, 4293, 4997], + [1200, 1606, 4998], + [1688, 430, 4999], + [260, 4633, 5000], + [3641, 4749, 5001], + [598, 4890, 5002], + [2845, 291, 5003], + [110, 852, 5004], + [1608, 983, 5005], + [4290, 265, 5006], + [3204, 376, 5007], + [1996, 1517, 5008], + [1789, 309, 5009], + [965, 478, 5010], + [4047, 291, 5011], + [4836, 338, 5012], + [368, 296, 5013], + [4984, 523, 5014], + [435, 119, 5015], + [336, 121, 5016], + [283, 2096, 5017], + [3878, 721, 5018], + [889, 121, 5019], + [1701, 1828, 5020], + [1968, 1397, 5021], + [3648, 4799, 5022], + [4113, 982, 5023], + [1051, 320, 5024], + [261, 2884, 5025], + [549, 376, 5026], + [279, 496, 5027], + [533, 112, 5028], + [1007, 109, 5029], + [672, 390, 5030], + [1490, 121, 5031], + [4321, 291, 5032], + [312, 115, 5033], + [4199, 1730, 5034], + [305, 859, 5035], + [2298, 473, 5036], + [4464, 680, 5037], + [443, 1965, 5038], + [391, 297, 5039], + [115, 2019, 5040], + [333, 115, 5041], + [324, 608, 5042], + [103, 291, 5043], + [331, 1965, 5044], + [5014, 280, 5045], + [1627, 2019, 5046], + [380, 310, 5047], + [1200, 2832, 5048], + [830, 854, 5049], + [4731, 2044, 5050], + [262, 801, 5051], + [1587, 387, 5052], + [1780, 115, 5053], + [282, 391, 5054], + [4660, 1188, 5055], + [4032, 421, 5056], + [598, 5039, 5057], + [120, 121, 5058], + [517, 2195, 5059], + [2104, 773, 5060], + [4925, 5029, 5061], + [105, 490, 5062], + [1788, 3455, 5063], + [391, 321, 5064], + [2001, 275, 5065], + [114, 680, 5066], + [270, 743, 5067], + [298, 492, 5068], + [853, 269, 5069], + [1211, 317, 5070], + [803, 3422, 5071], + [2303, 4945, 5072], + [4192, 3672, 5073], + [356, 271, 5074], + [3873, 296, 5075], + [999, 684, 5076], + [1896, 115, 5077], + [2944, 369, 5078], + [3855, 3020, 5079], + [77, 4472, 5080], + [4134, 983, 5081], + [296, 376, 5082], + [895, 390, 5083], + [593, 1485, 5084], + [4484, 591, 5085], + [3513, 115, 5086], + [4413, 5044, 5087], + [103, 121, 5088], + [544, 387, 5089], + [3825, 2000, 5090], + [1000, 896, 5091], + [5051, 445, 5092], + [2822, 115, 5093], + [2501, 115, 5094], + [741, 5066, 5095], + [871, 111, 5096], + [117, 1774, 5097], + [77, 973, 5098], + [4937, 797, 5099], + [1395, 1059, 5100], + [2859, 265, 5101], + [3269, 115, 5102], + [102, 985, 5103], + [871, 2195, 5104], + [498, 770, 5105], + [792, 1901, 5106], + [2225, 4039, 5107], + [3660, 5040, 5108], + [99, 983, 5109], + [1593, 108, 5110], + [443, 4207, 5111], + [4682, 339, 5112], + [374, 322, 5113], + [782, 121, 5114], + [4851, 982, 5115], + [807, 3837, 5116], + [115, 485, 5117], + [1309, 115, 5118], + [281, 4739, 5119], + [732, 280, 5120], + [2104, 291, 5121], + [666, 1225, 5122], + [293, 466, 5123], + [261, 943, 5124], + [1412, 291, 5125], + [105, 657, 5126], + [888, 115, 5127], + [1278, 291, 5128], + [533, 317, 5129], + [281, 296, 5130], + [461, 275, 5131], + [4703, 280, 5132], + [336, 355, 5133], + [461, 813, 5134], + [5133, 5062, 5135], + [393, 943, 5136], + [2940, 101, 5137], + [298, 1345, 5138], + [2408, 492, 5139], + [1990, 115, 5140], + [451, 115, 5141], + [293, 636, 5142], + [2224, 115, 5143], + [278, 1237, 5144], + [524, 537, 5145], + [4997, 2820, 5146], + [282, 1590, 5147], + [4507, 2525, 5148], + [274, 320, 5149], + [1032, 2000, 5150], + [3958, 777, 5151], + [2123, 357, 5152], + [1105, 115, 5153], + [78, 310, 5154], + [3053, 265, 5155], + [5149, 525, 5156], + [323, 4524, 5157], + [533, 99, 5158], + [821, 296, 5159], + [660, 4255, 5160], + [4780, 309, 5161], + [4931, 922, 5162], + [598, 314, 5163], + [2090, 1717, 5164], + [5028, 1100, 5165], + [2149, 115, 5166], + [830, 5165, 5167], + [5163, 2111, 5168], + [79, 4781, 5169], + [87, 371, 5170], + [283, 4347, 5171], + [3705, 4810, 5172], + [324, 845, 5173], + [4355, 373, 5174], + [2585, 291, 5175], + [1093, 898, 5176], + [508, 982, 5177], + [723, 528, 5178], + [33, 39, 5179], + [334, 103, 5180], + [490, 581, 5181], + [796, 1630, 5182], + [101, 265, 5183], + [859, 1057, 5184], + [908, 1657, 5185], + [3686, 1384, 5186], + [4876, 304, 5187], + [2223, 4838, 5188], + [1014, 2691, 5189], + [267, 456, 5190], + [1567, 2262, 5191], + [1721, 115, 5192], + [2631, 5126, 5193], + [98, 657, 5194], + [290, 485, 5195], + [589, 294, 5196], + [1187, 1762, 5197], + [332, 636, 5198], + [4529, 265, 5199], + [3204, 693, 5200], + [100, 111, 5201], + [442, 101, 5202], + [1119, 319, 5203], + [3569, 983, 5204], + [1219, 3943, 5205], + [1704, 108, 5206], + [601, 3497, 5207], + [4399, 121, 5208], + [3237, 2857, 5209], + [279, 3351, 5210], + [3648, 704, 5211], + [442, 335, 5212], + [1581, 1041, 5213], + [2651, 265, 5214], + [2858, 1138, 5215], + [1951, 115, 5216], + [4971, 121, 5217], + [666, 4576, 5218], + [5206, 4878, 5219], + [111, 430, 5220], + [2055, 115, 5221], + [502, 456, 5222], + [3426, 836, 5223], + [261, 537, 5224], + [426, 44, 5225], + [282, 2182, 5226], + [593, 478, 5227], + [3523, 608, 5228], + [3753, 115, 5229], + [887, 2003, 5230], + [1102, 115, 5231], + [334, 296, 5232], + [5024, 749, 5233], + [74, 446, 5234], + [584, 108, 5235], + [2078, 947, 5236], + [66, 944, 5237], + [4037, 339, 5238], + [5182, 657, 5239], + [692, 342, 5240], + [324, 1025, 5241], + [4132, 1435, 5242], + [2625, 1381, 5243], + [89, 490, 5244], + [1668, 115, 5245], + [887, 954, 5246], + [734, 265, 5247], + [617, 263, 5248], + [267, 101, 5249], + [277, 893, 5250], + [2248, 291, 5251], + [3494, 947, 5252], + [5248, 287, 5253], + [277, 329, 5254], + [2668, 3675, 5255], + [4977, 121, 5256], + [461, 446, 5257], + [2814, 265, 5258], + [1010, 115, 5259], + [1655, 115, 5260], + [3100, 112, 5261], + [32, 120, 5262], + [5065, 1345, 5263], + [1214, 1510, 5264], + [1811, 1188, 5265], + [83, 104, 5266], + [3405, 317, 5267], + [274, 5158, 5268], + [102, 121, 5269], + [5250, 525, 5270], + [1051, 310, 5271], + [99, 1046, 5272], + [1209, 466, 5273], + [1000, 110, 5274], + [3250, 115, 5275], + [442, 301, 5276], + [2251, 101, 5277], + [1043, 115, 5278], + [881, 263, 5279], + [2078, 1730, 5280], + [119, 3027, 5281], + [2196, 373, 5282], + [3862, 115, 5283], + [3426, 616, 5284], + [4114, 704, 5285], + [5255, 922, 5286], + [1244, 265, 5287], + [275, 852, 5288], + [1328, 115, 5289], + [258, 364, 5290], + [741, 2670, 5291], + [943, 1926, 5292], + [1376, 290, 5293], + [626, 5088, 5294], + [5120, 1345, 5295], + [3705, 1679, 5296], + [5042, 5292, 5297], + [295, 1189, 5298], + [1720, 115, 5299], + [908, 859, 5300], + [737, 4000, 5301], + [4024, 1065, 5302], + [5300, 5194, 5303], + [3394, 115, 5304], + [294, 265, 5305], + [4778, 1510, 5306], + [677, 947, 5307], + [4584, 390, 5308], + [1485, 294, 5309], + [87, 446, 5310], + [587, 2905, 5311], + [3343, 317, 5312], + [1443, 790, 5313], + [2357, 115, 5314], + [262, 271, 5315], + [353, 116, 5316], + [781, 5269, 5317], + [2598, 338, 5318], + [1665, 5281, 5319], + [387, 1938, 5320], + [268, 2274, 5321], + [274, 313, 5322], + [1087, 1777, 5323], + [301, 645, 5324], + [1103, 379, 5325], + [1459, 537, 5326], + [3631, 414, 5327], + [4589, 399, 5328], + [4955, 291, 5329], + [5268, 269, 5330], + [5316, 2857, 5331], + [871, 2594, 5332], + [823, 390, 5333], + [32, 4583, 5334], + [2677, 4879, 5335], + [831, 1384, 5336], + [99, 99, 5337], + [2354, 265, 5338], + [293, 3213, 5339], + [268, 317, 5340], + [593, 643, 5341], + [109, 764, 5342], + [3855, 3351, 5343], + [3038, 2131, 5344], + [261, 1672, 5345], + [34, 44, 5346], + [274, 265, 5347], + [862, 101, 5348], + [498, 1277, 5349], + [1737, 265, 5350], + [272, 644, 5351], + [2134, 1847, 5352], + [286, 5019, 5353], + [5228, 422, 5354], + [1912, 115, 5355], + [517, 321, 5356], + [2833, 291, 5357], + [3498, 560, 5358], + [723, 943, 5359], + [969, 291, 5360], + [298, 1678, 5361], + [553, 2687, 5362], + [65, 108, 5363], + [346, 3202, 5364], + [1208, 4753, 5365], + [902, 114, 5366], + [4776, 291, 5367], + [3990, 115, 5368], + [3825, 322, 5369], + [115, 304, 5370], + [122, 559, 5371], + [1032, 603, 5372], + [1916, 291, 5373], + [323, 1588, 5374], + [2179, 114, 5375], + [5241, 813, 5376], + [4175, 117, 5377], + [378, 291, 5378], + [428, 110, 5379], + [368, 4861, 5380], + [3306, 115, 5381], + [619, 2124, 5382], + [524, 294, 5383], + [365, 487, 5384], + [1240, 98, 5385], + [377, 3795, 5386], + [114, 115, 5387], + [355, 265, 5388], + [2929, 265, 5389], + [488, 321, 5390], + [4053, 115, 5391], + [611, 5309, 5392], + [5340, 421, 5393], + [4069, 399, 5394], + [5096, 833, 5395], + [3108, 309, 5396], + [1537, 4371, 5397], + [4534, 4172, 5398], + [708, 3485, 5399], + [1465, 773, 5400], + [1408, 2128, 5401], + [4975, 310, 5402], + [5279, 657, 5403], + [262, 294, 5404], + [1044, 265, 5405], + [1068, 97, 5406], + [407, 110, 5407], + [293, 1657, 5408], + [2702, 115, 5409], + [2747, 3367, 5410], + [794, 4201, 5411], + [5383, 4221, 5412], + [1088, 265, 5413], + [5016, 309, 5414], + [1258, 265, 5415], + [569, 2017, 5416], + [3821, 265, 5417], + [757, 5405, 5418], + [110, 558, 5419], + [1458, 5272, 5420], + [3727, 309, 5421], + [1581, 115, 5422], + [331, 5371, 5423], + [617, 1256, 5424], + [630, 1117, 5425], + [1917, 108, 5426], + [379, 121, 5427], + [354, 2050, 5428], + [960, 280, 5429], + [2103, 265, 5430], + [3214, 115, 5431], + [1310, 369, 5432], + [1259, 429, 5433], + [5025, 309, 5434], + [3289, 739, 5435], + [3100, 3104, 5436], + [294, 317, 5437], + [2304, 430, 5438], + [853, 263, 5439], + [1295, 291, 5440], + [4362, 101, 5441], + [710, 1138, 5442], + [827, 982, 5443], + [1868, 1713, 5444], + [335, 98, 5445], + [1280, 1059, 5446], + [1020, 4264, 5447], + [2668, 592, 5448], + [730, 898, 5449], + [332, 1762, 5450], + [2351, 280, 5451], + [295, 525, 5452], + [1667, 3111, 5453], + [3267, 291, 5454], + [543, 1643, 5455], + [274, 5400, 5456], + [354, 121, 5457], + [1350, 265, 5458], + [502, 2581, 5459], + [5212, 558, 5460], + [968, 864, 5461], + [3547, 275, 5462], + [1379, 297, 5463], + [283, 2122, 5464], + [3308, 317, 5465], + [549, 103, 5466], + [356, 560, 5467], + [1071, 3707, 5468], + [1618, 291, 5469], + [5021, 1887, 5470], + [754, 115, 5471], + [473, 1193, 5472], + [999, 5472, 5473], + [283, 852, 5474], + [1371, 1841, 5475], + [282, 528, 5476], + [1236, 693, 5477], + [108, 3706, 5478], + [3561, 292, 5479], + [407, 109, 5480], + [1319, 317, 5481], + [1698, 657, 5482], + [2298, 4730, 5483], + [472, 97, 5484], + [1040, 2505, 5485], + [672, 97, 5486], + [4035, 467, 5487], + [2241, 559, 5488], + [2396, 2633, 5489], + [103, 297, 5490], + [2213, 309, 5491], + [2437, 1606, 5492], + [635, 3879, 5493], + [995, 491, 5494], + [4531, 3565, 5495], + [1660, 121, 5496], + [4779, 487, 5497], + [314, 97, 5498], + [1645, 3129, 5499], + [1412, 287, 5500], + [1146, 5046, 5501], + [571, 2127, 5502], + [276, 865, 5503], + [710, 369, 5504], + [1412, 101, 5505], + [2773, 1439, 5506], + [105, 297, 5507], + [968, 1345, 5508], + [2177, 485, 5509], + [285, 291, 5510], + [959, 103, 5511], + [4517, 558, 5512], + [5058, 5490, 5513], + [5499, 297, 5514], + [268, 370, 5515], + [627, 317, 5516], + [1856, 4299, 5517], + [4114, 4628, 5518], + [2179, 5478, 5519], + [3687, 115, 5520], + [1047, 265, 5521], + [1537, 309, 5522], + [1767, 285, 5523], + [288, 387, 5524], + [860, 102, 5525], + [290, 890, 5526], + [2656, 291, 5527], + [4210, 100, 5528], + [803, 1965, 5529], + [885, 292, 5530], + [66, 986, 5531], + [3314, 1373, 5532], + [801, 291, 5533], + [259, 943, 5534], + [2181, 657, 5535], + [796, 3024, 5536], + [100, 291, 5537], + [354, 331, 5538], + [109, 581, 5539], + [1771, 115, 5540], + [1832, 115, 5541], + [886, 2531, 5542], + [336, 2122, 5543], + [323, 1390, 5544], + [637, 3359, 5545], + [4241, 767, 5546], + [3793, 265, 5547], + [114, 114, 5548], + [622, 108, 5549], + [110, 4299, 5550], + [83, 258, 5551], + [309, 121, 5552], + [336, 291, 5553], + [2774, 115, 5554], + [902, 943, 5555], + [962, 2131, 5556], + [365, 2380, 5557], + [1582, 117, 5558], + [2220, 4874, 5559], + [5488, 657, 5560], + [118, 97, 5561], + [2310, 1733, 5562], + [429, 1117, 5563], + [2553, 1397, 5564], + [295, 5486, 5565], + [334, 263, 5566], + [279, 985, 5567], + [2826, 317, 5568], + [2090, 3049, 5569], + [287, 115, 5570], + [278, 1635, 5571], + [5385, 424, 5572], + [108, 297, 5573], + [356, 770, 5574], + [1129, 291, 5575], + [1936, 369, 5576], + [5315, 322, 5577], + [5523, 615, 5578], + [97, 983, 5579], + [1093, 269, 5580], + [2175, 291, 5581], + [5180, 269, 5582], + [440, 97, 5583], + [109, 496, 5584], + [2370, 5053, 5585], + [1220, 4441, 5586], + [408, 1086, 5587], + [323, 1369, 5588], + [553, 5539, 5589], + [4024, 2131, 5590], + [1187, 5423, 5591], + [89, 101, 5592], + [408, 309, 5593], + [644, 297, 5594], + [3641, 3134, 5595], + [960, 115, 5596], + [541, 111, 5597], + [1025, 473, 5598], + [306, 4951, 5599], + [272, 3837, 5600], + [354, 314, 5601], + [3457, 3776, 5602], + [2257, 500, 5603], + [5484, 5083, 5604], + [1228, 97, 5605], + [1900, 773, 5606], + [1963, 115, 5607], + [259, 3364, 5608], + [1173, 5584, 5609], + [483, 504, 5610], + [1032, 5598, 5611], + [286, 5513, 5612], + [3570, 5342, 5613], + [276, 986, 5614], + [5057, 777, 5615], + [1748, 896, 5616], + [105, 98, 5617], + [590, 303, 5618], + [723, 1565, 5619], + [3878, 911, 5620], + [427, 387, 5621], + [79, 114, 5622], + [103, 657, 5623], + [1068, 944, 5624], + [323, 117, 5625], + [5515, 5004, 5626], + [452, 5201, 5627], + [1209, 5549, 5628], + [46, 39, 5629], + [3515, 291, 5630], + [298, 4201, 5631], + [1096, 291, 5632], + [1617, 2525, 5633], + [821, 5562, 5634], + [807, 2092, 5635], + [268, 496, 5636], + [279, 379, 5637], + [533, 467, 5638], + [2154, 1114, 5639], + [2623, 693, 5640], + [2491, 115, 5641], + [5330, 1606, 5642], + [3550, 548, 5643], + [5035, 115, 5644], + [2377, 2262, 5645], + [1247, 115, 5646], + [278, 841, 5647], + [4777, 1377, 5648], + [1851, 1853, 5649], + [2180, 348, 5650], + [1655, 3176, 5651], + [5544, 117, 5652], + [277, 3355, 5653], + [2398, 320, 5654], + [4409, 985, 5655], + [4889, 1127, 5656], + [2148, 291, 5657], + [1861, 115, 5658], + [1921, 115, 5659], + [3159, 1909, 5660], + [5293, 5103, 5661], + [71, 415, 5662], + [821, 289, 5663], + [1187, 4994, 5664], + [365, 98, 5665], + [3192, 317, 5666], + [2289, 101, 5667], + [1559, 5109, 5668], + [349, 982, 5669], + [472, 275, 5670], + [977, 317, 5671], + [2303, 2932, 5672], + [122, 297, 5673], + [1669, 115, 5674], + [3025, 773, 5675], + [306, 3111, 5676], + [886, 5550, 5677], + [677, 291, 5678], + [2209, 107, 5679], + [4846, 2259, 5680], + [2613, 2681, 5681], + [457, 121, 5682], + [1347, 824, 5683], + [4826, 4018, 5684], + [356, 1384, 5685], + [5514, 1882, 5686], + [324, 487, 5687], + [336, 1233, 5688], + [431, 802, 5689], + [4154, 1529, 5690], + [407, 100, 5691], + [282, 1991, 5692], + [1289, 291, 5693], + [2431, 4640, 5694], + [770, 121, 5695], + [318, 376, 5696], + [5651, 1635, 5697], + [710, 2655, 5698], + [473, 97, 5699], + [926, 3202, 5700], + [557, 46, 5701], + [916, 657, 5702], + [5322, 376, 5703], + [105, 107, 5704], + [2012, 115, 5705], + [306, 97, 5706], + [1140, 285, 5707], + [2018, 115, 5708], + [821, 2655, 5709], + [413, 110, 5710], + [4743, 854, 5711], + [1105, 2669, 5712], + [374, 265, 5713], + [5406, 1091, 5714], + [279, 335, 5715], + [1986, 280, 5716], + [3919, 749, 5717], + [368, 355, 5718], + [4159, 558, 5719], + [3910, 1238, 5720], + [4513, 308, 5721], + [368, 292, 5722], + [1218, 348, 5723], + [2232, 265, 5724], + [120, 117, 5725], + [1800, 291, 5726], + [4182, 1225, 5727], + [101, 120, 5728], + [111, 104, 5729], + [536, 98, 5730], + [5730, 3675, 5731], + [1051, 2649, 5732], + [1031, 115, 5733], + [2519, 301, 5734], + [4219, 4039, 5735], + [4552, 815, 5736], + [611, 294, 5737], + [267, 3954, 5738], + [1224, 1848, 5739], + [3472, 265, 5740], + [5013, 116, 5741], + [4958, 265, 5742], + [732, 269, 5743], + [1312, 5673, 5744], + [2081, 1730, 5745], + [4612, 1573, 5746], + [5684, 2004, 5747], + [457, 97, 5748], + [368, 468, 5749], + [268, 1685, 5750], + [710, 3606, 5751], + [3301, 291, 5752], + [2613, 309, 5753], + [4346, 1117, 5754], + [4552, 2733, 5755], + [261, 5437, 5756], + [679, 2472, 5757], + [98, 852, 5758], + [5627, 1853, 5759], + [796, 537, 5760], + [2790, 115, 5761], + [1035, 285, 5762], + [1423, 303, 5763], + [1312, 2074, 5764], + [830, 2942, 5765], + [559, 1194, 5766], + [1259, 1041, 5767], + [2053, 1882, 5768], + [4035, 890, 5769], + [4647, 265, 5770], + [5593, 1727, 5771], + [76, 396, 5772], + [356, 3776, 5773], + [1199, 852, 5774], + [2164, 3355, 5775], + [5545, 2857, 5776], + [407, 1944, 5777], + [4536, 265, 5778], + [666, 548, 5779], + [1729, 3013, 5780], + [1917, 320, 5781], + [1093, 4578, 5782], + [100, 558, 5783], + [276, 5754, 5784], + [324, 904, 5785], + [107, 1238, 5786], + [959, 2092, 5787], + [117, 5725, 5788], + [1218, 5758, 5789], + [796, 613, 5790], + [598, 5129, 5791], + [111, 265, 5792], + [1617, 2681, 5793], + [1675, 983, 5794], + [435, 5766, 5795], + [306, 500, 5796], + [268, 739, 5797], + [980, 309, 5798], + [2936, 1571, 5799], + [2301, 291, 5800], + [5599, 104, 5801], + [69, 120, 5802], + [5522, 1041, 5803], + [2012, 657, 5804], + [637, 379, 5805], + [951, 265, 5806], + [965, 114, 5807], + [1310, 595, 5808], + [637, 2195, 5809], + [2812, 115, 5810], + [282, 904, 5811], + [763, 2899, 5812], + [830, 4979, 5813], + [5812, 597, 5814], + [323, 273, 5815], + [1748, 1480, 5816], + [5511, 1397, 5817], + [277, 5788, 5818], + [5222, 592, 5819], + [279, 97, 5820], + [442, 117, 5821], + [334, 445, 5822], + [512, 265, 5823], + [259, 302, 5824], + [265, 5537, 5825], + [3583, 762, 5826], + [694, 1041, 5827], + [5707, 533, 5828], + [4414, 304, 5829], + [710, 839, 5830], + [110, 317, 5831], + [302, 121, 5832], + [3029, 265, 5833], + [1463, 3210, 5834], + [3566, 157, 5835], + [5636, 390, 5836], + [4555, 100, 5837], + [65, 104, 5838], + [4018, 5030, 5839], + [439, 2342, 5840], + [278, 399, 5841], + [5831, 1117, 5842], + [3958, 695, 5843], + [295, 4541, 5844], + [4670, 115, 5845], + [265, 5842, 5846], + [3399, 115, 5847], + [262, 597, 5848], + [1347, 833, 5849], + [106, 335, 5850], + [1259, 1828, 5851], + [1253, 115, 5852], + [698, 1901, 5853], + [2804, 852, 5854], + [5031, 115, 5855], + [1721, 291, 5856], + [2507, 1517, 5857], + [122, 558, 5858], + [3518, 657, 5859], + [280, 110, 5860], + [4769, 1759, 5861], + [354, 789, 5862], + [4113, 2997, 5863], + [32, 49, 5864], + [474, 5846, 5865], + [4376, 115, 5866], + [282, 1261, 5867], + [442, 2274, 5868], + [3651, 291, 5869], + [5867, 105, 5870], + [5500, 2310, 5871], + [297, 97, 5872], + [3607, 291, 5873], + [3141, 291, 5874], + [2086, 364, 5875], + [521, 1384, 5876], + [1209, 597, 5877], + [1230, 280, 5878], + [1962, 893, 5879], + [3441, 1238, 5880], + [1280, 115, 5881], + [259, 105, 5882], + [2088, 291, 5883], + [102, 275, 5884], + [4132, 917, 5885], + [1797, 5290, 5886], + [2192, 2410, 5887], + [578, 5043, 5888], + [5339, 265, 5889], + [598, 296, 5890], + [442, 304, 5891], + [1485, 105, 5892], + [598, 3635, 5893], + [262, 392, 5894], + [283, 383, 5895], + [314, 1828, 5896], + [1631, 3391, 5897], + [303, 97, 5898], + [3378, 121, 5899], + [3515, 265, 5900], + [585, 3954, 5901], + [1469, 5839, 5902], + [380, 903, 5903], + [1007, 816, 5904], + [368, 5498, 5905], + [5202, 1025, 5906], + [452, 5884, 5907], + [3341, 115, 5908], + [3494, 390, 5909], + [533, 304, 5910], + [947, 777, 5911], + [655, 672, 5912], + [105, 357, 5913], + [5558, 1345, 5914], + [4362, 317, 5915], + [1891, 693, 5916], + [3928, 1397, 5917], + [293, 101, 5918], + [104, 1238, 5919], + [2115, 661, 5920], + [3716, 115, 5921], + [5654, 773, 5922], + [450, 115, 5923], + [347, 1007, 5924], + [1160, 105, 5925], + [5647, 1046, 5926], + [3873, 1606, 5927], + [2447, 1551, 5928], + [934, 1170, 5929], + [3100, 337, 5930], + [3438, 833, 5931], + [4192, 5097, 5932], + [2651, 291, 5933], + [691, 5911, 5934], + [723, 5832, 5935], + [1007, 739, 5936], + [2284, 291, 5937], + [83, 523, 5938], + [803, 1644, 5939], + [2184, 1733, 5940], + [5728, 112, 5941], + [2090, 265, 5942], + [262, 5825, 5943], + [269, 116, 5944], + [5000, 339, 5945], + [4901, 2003, 5946], + [32, 837, 5947], + [848, 116, 5948], + [5592, 4746, 5949], + [593, 296, 5950], + [902, 2896, 5951], + [121, 273, 5952], + [323, 5748, 5953], + [1567, 5669, 5954], + [1038, 2125, 5955], + [521, 677, 5956], + [2644, 115, 5957], + [5637, 644, 5958], + [1199, 115, 5959], + [82, 5184, 5960], + [3046, 265, 5961], + [317, 386, 5962], + [694, 1059, 5963], + [407, 2105, 5964], + [2837, 291, 5965], + [85, 104, 5966], + [1558, 296, 5967], + [5941, 4309, 5968], + [1490, 597, 5969], + [536, 2410, 5970], + [621, 717, 5971], + [1043, 280, 5972], + [5073, 265, 5973], + [4531, 317, 5974], + [549, 1021, 5975], + [1077, 317, 5976], + [588, 2633, 5977], + [1058, 5913, 5978], + [4401, 1730, 5979], + [5736, 265, 5980], + [336, 496, 5981], + [2296, 558, 5982], + [2200, 3050, 5983], + [2607, 773, 5984], + [999, 5968, 5985], + [354, 2967, 5986], + [2121, 115, 5987], + [261, 458, 5988], + [368, 304, 5989], + [117, 390, 5990], + [2384, 291, 5991], + [3430, 1373, 5992], + [5818, 414, 5993], + [281, 369, 5994], + [1668, 537, 5995], + [1891, 2262, 5996], + [951, 291, 5997], + [260, 364, 5998], + [3121, 3759, 5999], + [2694, 3020, 6000], + [274, 468, 6001], + [5844, 390, 6002], + [5850, 533, 6003], + [380, 1439, 6004], + [351, 5427, 6005], + [716, 5990, 6006], + [683, 115, 6007], + [636, 265, 6008], + [512, 121, 6009], + [1095, 291, 6010], + [1020, 2182, 6011], + [3219, 387, 6012], + [4767, 265, 6013], + [2940, 5962, 6014], + [323, 2701, 6015], + [2935, 5919, 6016], + [803, 1237, 6017], + [98, 108, 6018], + [589, 112, 6019], + [502, 415, 6020], + [723, 643, 6021], + [1013, 115, 6022], + [1075, 1562, 6023], + [853, 357, 6024], + [276, 663, 6025], + [3029, 291, 6026], + [5334, 419, 6027], + [1390, 1828, 6028], + [584, 120, 6029], + [521, 2714, 6030], + [1441, 339, 6031], + [3608, 1378, 6032], + [4276, 46, 6033], + [5625, 337, 6034], + [4591, 1041, 6035], + [262, 717, 6036], + [2255, 291, 6037], + [5802, 6012, 6038], + [336, 1384, 6039], + [1705, 291, 6040], + [1785, 265, 6041], + [4991, 1117, 6042], + [323, 5982, 6043], + [2806, 280, 6044], + [67, 104, 6045], + [105, 289, 6046], + [2692, 291, 6047], + [5739, 115, 6048], + [5691, 335, 6049], + [2256, 628, 6050], + [2223, 280, 6051], + [110, 1044, 6052], + [782, 313, 6053], + [334, 869, 6054], + [1102, 657, 6055], + [4899, 4655, 6056], + [502, 5579, 6057], + [103, 111, 6058], + [109, 533, 6059], + [1609, 115, 6060], + [302, 2153, 6061], + [5375, 644, 6062], + [258, 308, 6063], + [319, 115, 6064], + [1593, 1730, 6065], + [275, 1378, 6066], + [438, 285, 6067], + [3143, 657, 6068], + [334, 355, 6069], + [1042, 317, 6070], + [307, 622, 6071], + [2912, 115, 6072], + [1077, 2696, 6073], + [6071, 296, 6074], + [1056, 1034, 6075], + [258, 1948, 6076], + [5362, 1730, 6077], + [821, 5682, 6078], + [118, 269, 6079], + [368, 369, 6080], + [87, 465, 6081], + [533, 109, 6082], + [475, 2915, 6083], + [617, 5858, 6084], + [472, 2810, 6085], + [5084, 303, 6086], + [1749, 6003, 6087], + [500, 317, 6088], + [4513, 947, 6089], + [999, 573, 6090], + [32, 89, 6091], + [6062, 6059, 6092], + [781, 109, 6093], + [6083, 291, 6094], + [1481, 291, 6095], + [1962, 4309, 6096], + [278, 4347, 6097], + [1309, 473, 6098], + [370, 101, 6099], + [4541, 657, 6100], + [4989, 280, 6101], + [1102, 280, 6102], + [1827, 291, 6103], + [5706, 1225, 6104], + [294, 115, 6105], + [297, 1935, 6106], + [1026, 5370, 6107], + [274, 314, 6108], + [642, 5510, 6109], + [2467, 115, 6110], + [331, 105, 6111], + [5805, 3020, 6112], + [2528, 721, 6113], + [4199, 309, 6114], + [268, 328, 6115], + [508, 101, 6116], + [277, 5445, 6117], + [1631, 1041, 6118], + [1746, 317, 6119], + [4037, 430, 6120], + [1630, 947, 6121], + [544, 1513, 6122], + [532, 947, 6123], + [1258, 291, 6124], + [279, 560, 6125], + [908, 5220, 6126], + [593, 1647, 6127], + [4159, 5699, 6128], + [547, 111, 6129], + [1047, 657, 6130], + [336, 5860, 6131], + [2197, 5533, 6132], + [5722, 111, 6133], + [6085, 6008, 6134], + [6108, 109, 6135], + [2462, 115, 6136], + [1068, 2572, 6137], + [1900, 317, 6138], + [3511, 115, 6139], + [2001, 1824, 6140], + [2228, 115, 6141], + [2724, 115, 6142], + [281, 1485, 6143], + [278, 1595, 6144], + [66, 355, 6145], + [271, 297, 6146], + [1411, 852, 6147], + [590, 416, 6148], + [498, 643, 6149], + [3233, 115, 6150], + [2640, 5526, 6151], + [2814, 280, 6152], + [2370, 4240, 6153], + [5824, 280, 6154], + [470, 115, 6155], + [1192, 3084, 6156], + [1258, 1768, 6157], + [895, 2004, 6158], + [309, 101, 6159], + [1032, 1584, 6160], + [1287, 296, 6161], + [1214, 1738, 6162], + [3757, 4240, 6163], + [1591, 5195, 6164], + [1177, 291, 6165], + [278, 379, 6166], + [74, 636, 6167], + [598, 613, 6168], + [1936, 1513, 6169], + [1936, 2125, 6170], + [3403, 115, 6171], + [3053, 291, 6172], + [100, 852, 6173], + [332, 3385, 6174], + [1754, 115, 6175], + [5938, 1176, 6176], + [1998, 102, 6177], + [278, 1465, 6178], + [5929, 854, 6179], + [4401, 309, 6180], + [1748, 348, 6181], + [3011, 548, 6182], + [821, 258, 6183], + [1020, 644, 6184], + [4534, 399, 6185], + [6098, 2279, 6186], + [4358, 121, 6187], + [442, 4207, 6188], + [97, 1713, 6189], + [999, 102, 6190], + [902, 320, 6191], + [5875, 265, 6192], + [629, 458, 6193], + [2458, 265, 6194], + [1141, 3050, 6195], + [284, 1041, 6196], + [965, 6111, 6197], + [630, 2629, 6198], + [4482, 121, 6199], + [274, 6193, 6200], + [2341, 291, 6201], + [536, 1561, 6202], + [1221, 4570, 6203], + [276, 629, 6204], + [278, 816, 6205], + [1077, 1717, 6206], + [2270, 1373, 6207], + [1986, 2280, 6208], + [2936, 379, 6209], + [5401, 115, 6210], + [82, 111, 6211], + [4829, 1090, 6212], + [70, 806, 6213], + [902, 1360, 6214], + [2660, 117, 6215], + [336, 859, 6216], + [1712, 806, 6217], + [279, 297, 6218], + [278, 618, 6219], + [1221, 869, 6220], + [3384, 115, 6221], + [4399, 852, 6222], + [4533, 265, 6223], + [290, 852, 6224], + [3166, 2681, 6225], + [4690, 947, 6226], + [5528, 947, 6227], + [908, 2645, 6228], + [277, 1982, 6229], + [1752, 265, 6230], + [278, 768, 6231], + [2613, 947, 6232], + [4063, 509, 6233], + [5820, 6224, 6234], + [4989, 2911, 6235], + [353, 774, 6236], + [271, 296, 6237], + [262, 321, 6238], + [84, 465, 6239], + [891, 291, 6240], + [1781, 890, 6241], + [4517, 702, 6242], + [723, 335, 6243], + [1316, 110, 6244], + [5743, 773, 6245], + [277, 1439, 6246], + [1917, 119, 6247], + [4901, 1561, 6248], + [5569, 5288, 6249], + [723, 265, 6250], + [4829, 120, 6251], + [609, 4151, 6252], + [368, 111, 6253], + [3643, 1627, 6254], + [715, 1643, 6255], + [1969, 111, 6256], + [2392, 390, 6257], + [1740, 115, 6258], + [609, 447, 6259], + [2767, 291, 6260], + [279, 1762, 6261], + [3849, 115, 6262], + [6069, 115, 6263], + [368, 4371, 6264], + [82, 370, 6265], + [365, 839, 6266], + [270, 105, 6267], + [400, 895, 6268], + [2940, 422, 6269], + [1365, 2127, 6270], + [6268, 289, 6271], + [293, 6189, 6272], + [803, 296, 6273], + [1034, 944, 6274], + [303, 1442, 6275], + [2534, 280, 6276], + [2913, 115, 6277], + [902, 4853, 6278], + [4414, 6063, 6279], + [260, 1025, 6280], + [1428, 280, 6281], + [1846, 2468, 6282], + [332, 537, 6283], + [490, 109, 6284], + [2575, 115, 6285], + [5578, 721, 6286], + [112, 684, 6287], + [977, 110, 6288], + [268, 2409, 6289], + [584, 6052, 6290], + [4032, 5305, 6291], + [611, 5570, 6292], + [1591, 911, 6293], + [5271, 485, 6294], + [1227, 1738, 6295], + [1758, 115, 6296], + [3087, 121, 6297], + [5882, 864, 6298], + [4899, 6158, 6299], + [368, 97, 6300], + [435, 736, 6301], + [1230, 309, 6302], + [3168, 121, 6303], + [6075, 265, 6304], + [356, 1111, 6305], + [4065, 983, 6306], + [2592, 317, 6307], + [5505, 595, 6308], + [431, 1646, 6309], + [2012, 280, 6310], + [3063, 309, 6311], + [5926, 390, 6312], + [3358, 2681, 6313], + [474, 5082, 6314], + [6029, 99, 6315], + [1315, 291, 6316], + [6001, 382, 6317], + [949, 3040, 6318], + [3236, 296, 6319], + [521, 3668, 6320], + [114, 1562, 6321], + [368, 512, 6322], + [354, 663, 6323], + [3704, 317, 6324], + [4069, 2342, 6325], + [1567, 348, 6326], + [584, 3131, 6327], + [892, 777, 6328], + [1415, 373, 6329], + [2060, 383, 6330], + [295, 2444, 6331], + [6168, 947, 6332], + [2484, 1464, 6333], + [295, 2719, 6334], + [990, 5387, 6335], + [4886, 265, 6336], + [738, 121, 6337], + [891, 115, 6338], + [6178, 1847, 6339], + [1396, 291, 6340], + [6315, 573, 6341], + [5497, 657, 6342], + [3669, 6324, 6343], + [394, 4264, 6344], + [364, 121, 6345], + [354, 111, 6346], + [3246, 115, 6347], + [2300, 890, 6348], + [1170, 890, 6349], + [274, 1435, 6350], + [347, 4236, 6351], + [323, 5872, 6352], + [259, 121, 6353], + [1366, 115, 6354], + [2434, 833, 6355], + [354, 2482, 6356], + [3471, 115, 6357], + [5426, 704, 6358], + [268, 839, 6359], + [1127, 296, 6360], + [104, 797, 6361], + [374, 608, 6362], + [1097, 399, 6363], + [442, 6274, 6364], + [677, 390, 6365], + [2232, 291, 6366], + [989, 265, 6367], + [461, 1369, 6368], + [965, 313, 6369], + [1051, 301, 6370], + [1785, 1007, 6371], + [2436, 1345, 6372], + [980, 5573, 6373], + [283, 5496, 6374], + [4516, 4172, 6375], + [5325, 265, 6376], + [5704, 105, 6377], + [332, 3217, 6378], + [1414, 597, 6379], + [2504, 2850, 6380], + [258, 101, 6381], + [98, 328, 6382], + [3244, 115, 6383], + [3046, 291, 6384], + [4705, 308, 6385], + [4832, 317, 6386], + [545, 947, 6387], + [2337, 317, 6388], + [624, 2669, 6389], + [4306, 115, 6390], + [731, 2629, 6391], + [1051, 512, 6392], + [1811, 419, 6393], + [1487, 5563, 6394], + [102, 108, 6395], + [807, 548, 6396], + [1129, 115, 6397], + [2676, 115, 6398], + [2090, 2004, 6399], + [493, 265, 6400], + [323, 3249, 6401], + [675, 317, 6402], + [1088, 6360, 6403], + [456, 414, 6404], + [3582, 115, 6405], + [279, 2714, 6406], + [334, 121, 6407], + [294, 291, 6408], + [286, 342, 6409], + [3860, 1627, 6410], + [3731, 121, 6411], + [6407, 2716, 6412], + [4581, 369, 6413], + [1970, 98, 6414], + [728, 269, 6415], + [3466, 115, 6416], + [2722, 115, 6417], + [1228, 5892, 6418], + [4826, 490, 6419], + [1309, 852, 6420], + [292, 317, 6421], + [203, 156, 6422], + [45, 45, 6423], + [2354, 291, 6424], + [4653, 291, 6425], + [342, 657, 6426], + [474, 1261, 6427], + [518, 2629, 6428], + [261, 4157, 6429], + [4589, 2342, 6430], + [3841, 5948, 6431], + [557, 6422, 6432], + [116, 309, 6433], + [282, 317, 6434], + [6434, 2153, 6435], + [5582, 896, 6436], + [110, 657, 6437], + [1039, 115, 6438], + [2014, 317, 6439], + [3630, 115, 6440], + [1583, 6161, 6441], + [1009, 121, 6442], + [1849, 525, 6443], + [307, 1193, 6444], + [1312, 1713, 6445], + [289, 429, 6446], + [295, 608, 6447], + [726, 480, 6448], + [5841, 1648, 6449], + [2184, 6321, 6450], + [98, 725, 6451], + [380, 5896, 6452], + [283, 768, 6453], + [902, 111, 6454], + [502, 1384, 6455], + [273, 376, 6456], + [6362, 294, 6457], + [2088, 115, 6458], + [259, 6121, 6459], + [323, 713, 6460], + [5530, 852, 6461], + [4458, 97, 6462], + [3426, 120, 6463], + [4210, 2539, 6464], + [5895, 265, 6465], + [409, 6446, 6466], + [2739, 852, 6467], + [3258, 702, 6468], + [354, 4853, 6469], + [1334, 280, 6470], + [6273, 693, 6471], + [117, 1624, 6472], + [1007, 115, 6473], + [98, 577, 6474], + [2433, 2505, 6475], + [442, 287, 6476], + [262, 1238, 6477], + [293, 6159, 6478], + [1050, 3202, 6479], + [278, 636, 6480], + [495, 512, 6481], + [3613, 115, 6482], + [5662, 1166, 6483], + [5494, 121, 6484], + [457, 114, 6485], + [3386, 115, 6486], + [99, 657, 6487], + [866, 2021, 6488], + [887, 6485, 6489], + [3980, 430, 6490], + [5198, 121, 6491], + [598, 328, 6492], + [101, 337, 6493], + [349, 618, 6494], + [1010, 3485, 6495], + [1093, 2016, 6496], + [2313, 749, 6497], + [2806, 291, 6498], + [1716, 3707, 6499], + [4312, 115, 6500], + [331, 657, 6501], + [770, 280, 6502], + [332, 943, 6503], + [1310, 280, 6504], + [277, 1619, 6505], + [2090, 291, 6506], + [902, 108, 6507], + [1694, 4730, 6508], + [5605, 430, 6509], + [6480, 285, 6510], + [114, 331, 6511], + [2272, 802, 6512], + [407, 898, 6513], + [282, 105, 6514], + [399, 1707, 6515], + [4705, 348, 6516], + [111, 1179, 6517], + [803, 613, 6518], + [289, 1730, 6519], + [2752, 121, 6520], + [318, 1455, 6521], + [3841, 319, 6522], + [4594, 291, 6523], + [2992, 852, 6524], + [306, 3270, 6525], + [5119, 1606, 6526], + [1408, 4441, 6527], + [78, 3152, 6528], + [1093, 309, 6529], + [5366, 269, 6530], + [6514, 4979, 6531], + [3061, 2681, 6532], + [6460, 415, 6533], + [2518, 291, 6534], + [332, 5181, 6535], + [3512, 115, 6536], + [292, 2019, 6537], + [477, 44, 6538], + [515, 319, 6539], + [292, 265, 6540], + [387, 608, 6541], + [821, 117, 6542], + [440, 110, 6543], + [5404, 688, 6544], + [3710, 1926, 6545], + [5998, 4095, 6546], + [515, 839, 6547], + [593, 3103, 6548], + [3204, 1901, 6549], + [6546, 841, 6550], + [5289, 1711, 6551], + [1365, 280, 6552], + [277, 370, 6553], + [301, 115, 6554], + [1534, 121, 6555], + [4818, 265, 6556], + [977, 6541, 6557], + [3049, 2019, 6558], + [5240, 852, 6559], + [5749, 2810, 6560], + [2447, 6517, 6561], + [118, 597, 6562], + [611, 114, 6563], + [275, 693, 6564], + [6489, 4983, 6565], + [910, 111, 6566], + [3877, 265, 6567], + [4972, 6345, 6568], + [6566, 4576, 6569], + [110, 111, 6570], + [5683, 265, 6571], + [3955, 458, 6572], + [3510, 265, 6573], + [4027, 2111, 6574], + [407, 4877, 6575], + [431, 4460, 6576], + [4349, 115, 6577], + [497, 304, 6578], + [5266, 104, 6579], + [2436, 2832, 6580], + [3750, 1657, 6581], + [63, 39, 6582], + [939, 4654, 6583], + [3431, 864, 6584], + [331, 896, 6585], + [270, 1381, 6586], + [1504, 749, 6587], + [5525, 115, 6588], + [296, 456, 6589], + [721, 121, 6590], + [5710, 1013, 6591], + [408, 459, 6592], + [1097, 102, 6593], + [830, 1727, 6594], + [3051, 115, 6595], + [1481, 6558, 6596], + [660, 5678, 6597], + [3071, 1302, 6598], + [5763, 1517, 6599], + [595, 116, 6600], + [259, 6501, 6601], + [932, 1935, 6602], + [6530, 1431, 6603], + [2284, 297, 6604], + [5000, 430, 6605], + [110, 101, 6606], + [788, 983, 6607], + [299, 309, 6608], + [109, 5064, 6609], + [2194, 496, 6610], + [497, 3668, 6611], + [2869, 5183, 6612], + [1412, 6377, 6613], + [3490, 6609, 6614], + [4777, 6018, 6615], + [5721, 265, 6616], + [4147, 644, 6617], + [1869, 115, 6618], + [2298, 115, 6619], + [6615, 537, 6620], + [4154, 364, 6621], + [3841, 4877, 6622], + [3904, 115, 6623], + [99, 108, 6624], + [376, 328, 6625], + [381, 265, 6626], + [364, 595, 6627], + [1428, 317, 6628], + [99, 1381, 6629], + [400, 3256, 6630], + [2079, 1841, 6631], + [99, 1651, 6632], + [307, 2482, 6633], + [336, 1982, 6634], + [5870, 2342, 6635], + [6215, 1111, 6636], + [6557, 6106, 6637], + [6511, 1373, 6638], + [334, 335, 6639], + [301, 317, 6640], + [5010, 1114, 6641], + [902, 335, 6642], + [3996, 5507, 6643], + [4584, 558, 6644], + [660, 6610, 6645], + [119, 1018, 6646], + [888, 116, 6647], + [6298, 5792, 6648], + [442, 6099, 6649], + [6647, 2733, 6650], + [5716, 339, 6651], + [609, 6600, 6652], + [5480, 369, 6653], + [598, 943, 6654], + [361, 657, 6655], + [648, 115, 6656], + [496, 110, 6657], + [590, 289, 6658], + [4886, 291, 6659], + [3431, 2411, 6660], + [389, 121, 6661], + [346, 2004, 6662], + [939, 839, 6663], + [611, 3351, 6664], + [277, 1176, 6665], + [1559, 1679, 6666], + [2895, 321, 6667], + [323, 105, 6668], + [598, 500, 6669], + [4413, 105, 6670], + [5073, 291, 6671], + [6286, 1926, 6672], + [69, 119, 6673], + [1923, 115, 6674], + [1728, 115, 6675], + [4484, 1659, 6676], + [6061, 291, 6677], + [6586, 6451, 6678], + [107, 121, 6679], + [119, 5183, 6680], + [1848, 1730, 6681], + [71, 6237, 6682], + [72, 628, 6683], + [354, 1502, 6684], + [821, 415, 6685], + [517, 2342, 6686], + [716, 1586, 6687], + [977, 2066, 6688], + [5110, 115, 6689], + [278, 303, 6690], + [1061, 386, 6691], + [293, 1449, 6692], + [2514, 338, 6693], + [278, 1543, 6694], + [5235, 115, 6695], + [368, 6413, 6696], + [6280, 1831, 6697], + [100, 265, 6698], + [965, 2649, 6699], + [1630, 291, 6700], + [323, 702, 6701], + [281, 1384, 6702], + [536, 895, 6703], + [282, 3001, 6704], + [296, 1390, 6705], + [365, 6493, 6706], + [6706, 263, 6707], + [2115, 531, 6708], + [32, 81, 6709], + [282, 280, 6710], + [295, 302, 6711], + [1108, 258, 6712], + [3486, 294, 6713], + [260, 3672, 6714], + [1885, 6680, 6715], + [493, 3202, 6716], + [2048, 1302, 6717], + [3527, 265, 6718], + [6599, 338, 6719], + [1178, 5220, 6720], + [3662, 390, 6721], + [2115, 5904, 6722], + [6687, 6217, 6723], + [1276, 2540, 6724], + [6211, 289, 6725], + [282, 6614, 6726], + [336, 466, 6727], + [324, 376, 6728], + [332, 4748, 6729], + [790, 297, 6730], + [1448, 545, 6731], + [2437, 2832, 6732], + [1223, 121, 6733], + [4134, 947, 6734], + [2238, 3943, 6735], + [6219, 269, 6736], + [1408, 6395, 6737], + [5939, 309, 6738], + [74, 922, 6739], + [293, 349, 6740], + [270, 1890, 6741], + [474, 616, 6742], + [1617, 115, 6743], + [1644, 983, 6744], + [438, 6173, 6745], + [6243, 1648, 6746], + [1068, 6515, 6747], + [295, 99, 6748], + [484, 657, 6749], + [650, 291, 6750], + [349, 319, 6751], + [355, 317, 6752], + [3182, 2125, 6753], + [6694, 309, 6754], + [1350, 115, 6755], + [2709, 2629, 6756], + [1014, 550, 6757], + [76, 933, 6758], + [87, 6381, 6759], + [1299, 1650, 6760], + [76, 101, 6761], + [6660, 3054, 6762], + [3326, 6744, 6763], + [989, 291, 6764], + [3916, 1606, 6765], + [304, 105, 6766], + [385, 100, 6767], + [2344, 3217, 6768], + [6166, 280, 6769], + [4016, 115, 6770], + [336, 897, 6771], + [966, 121, 6772], + [2583, 291, 6773], + [3108, 390, 6774], + [408, 1245, 6775], + [999, 115, 6776], + [84, 395, 6777], + [1488, 46, 6778], + [1416, 116, 6779], + [2146, 801, 6780], + [2257, 2764, 6781], + [3170, 291, 6782], + [4204, 317, 6783], + [2199, 115, 6784], + [352, 292, 6785], + [3616, 105, 6786], + [5276, 558, 6787], + [5821, 767, 6788], + [1158, 291, 6789], + [79, 4983, 6790], + [679, 869, 6791], + [1553, 115, 6792], + [1140, 2021, 6793], + [2814, 291, 6794], + [5715, 982, 6795], + [4047, 265, 6796], + [6786, 6082, 6797], + [5337, 1100, 6798], + [4760, 265, 6799], + [968, 3411, 6800], + [1187, 4000, 6801], + [6780, 304, 6802], + [279, 313, 6803], + [710, 1738, 6804], + [87, 2647, 6805], + [2828, 6785, 6806], + [4409, 4992, 6807], + [887, 6798, 6808], + [1068, 5184, 6809], + [6714, 3391, 6810], + [112, 115, 6811], + [728, 528, 6812], + [459, 115, 6813], + [2374, 591, 6814], + [279, 6519, 6815], + [1148, 531, 6816], + [1851, 2075, 6817], + [355, 1627, 6818], + [475, 114, 6819], + [763, 116, 6820], + [1740, 5786, 6821], + [276, 4524, 6822], + [275, 5097, 6823], + [503, 5507, 6824], + [5956, 280, 6825], + [1069, 291, 6826], + [5274, 537, 6827], + [5925, 736, 6828], + [1412, 6657, 6829], + [3209, 265, 6830], + [5198, 1730, 6831], + [678, 4654, 6832], + [737, 2320, 6833], + [844, 2410, 6834], + [4529, 537, 6835], + [2747, 485, 6836], + [1917, 1592, 6837], + [439, 5307, 6838], + [6709, 6730, 6839], + [267, 815, 6840], + [4540, 317, 6841], + [461, 531, 6842], + [723, 3707, 6843], + [275, 773, 6844], + [2337, 291, 6845], + [2392, 97, 6846], + [2447, 943, 6847], + [474, 6661, 6848], + [3939, 467, 6849], + [6334, 5695, 6850], + [6658, 2153, 6851], + [5601, 2013, 6852], + [6820, 1847, 6853], + [1385, 280, 6854], + [262, 2647, 6855], + [298, 806, 6856], + [1259, 1059, 6857], + [289, 1529, 6858], + [6846, 1181, 6859], + [332, 6088, 6860], + [265, 115, 6861], + [272, 103, 6862], + [1068, 121, 6863], + [3171, 100, 6864], + [531, 114, 6865], + [6311, 4655, 6866], + [3721, 983, 6867], + [804, 3219, 6868], + [5864, 48, 6869], + [697, 6638, 6870], + [308, 297, 6871], + [6653, 702, 6872], + [558, 338, 6873], + [323, 5594, 6874], + [1926, 115, 6875], + [2162, 115, 6876], + [1754, 773, 6877], + [2928, 115, 6878], + [295, 3624, 6879], + [5119, 672, 6880], + [2901, 265, 6881], + [119, 107, 6882], + [1169, 390, 6883], + [1915, 2005, 6884], + [6553, 1646, 6885], + [3978, 6818, 6886], + [6322, 1179, 6887], + [5134, 115, 6888], + [1617, 896, 6889], + [6865, 6705, 6890], + [3652, 115, 6891], + [6143, 1847, 6892], + [112, 114, 6893], + [590, 1733, 6894], + [102, 391, 6895], + [593, 533, 6896], + [1497, 115, 6897], + [2118, 2016, 6898], + [3510, 291, 6899], + [298, 3576, 6900], + [3702, 265, 6901], + [279, 558, 6902], + [2447, 629, 6903], + [270, 654, 6904], + [365, 864, 6905], + [2209, 348, 6906], + [3460, 749, 6907], + [3592, 115, 6908], + [6020, 2111, 6909], + [2528, 3375, 6910], + [1788, 115, 6911], + [300, 6882, 6912], + [713, 4460, 6913], + [6726, 115, 6914], + [6323, 558, 6915], + [1899, 657, 6916], + [298, 478, 6917], + [1595, 297, 6918], + [354, 1925, 6919], + [6840, 890, 6920], + [6868, 774, 6921], + [4581, 390, 6922], + [1495, 97, 6923], + [6842, 101, 6924], + [569, 263, 6925], + [2055, 480, 6926], + [1583, 1853, 6927], + [1869, 1059, 6928], + [4972, 103, 6929], + [881, 5307, 6930], + [6923, 369, 6931], + [299, 110, 6932], + [1860, 103, 6933], + [1950, 115, 6934], + [932, 6487, 6935], + [314, 1314, 6936], + [1917, 456, 6937], + [89, 973, 6938], + [1962, 3242, 6939], + [1852, 115, 6940], + [66, 328, 6941], + [959, 870, 6942], + [379, 390, 6943], + [6711, 304, 6944], + [541, 101, 6945], + [461, 3573, 6946], + [364, 512, 6947], + [1608, 115, 6948], + [871, 1373, 6949], + [584, 5561, 6950], + [6645, 1242, 6951], + [3904, 100, 6952], + [1020, 717, 6953], + [666, 1318, 6954], + [267, 545, 6955], + [298, 456, 6956], + [351, 466, 6957], + [584, 869, 6958], + [76, 1369, 6959], + [283, 1402, 6960], + [2376, 291, 6961], + [3066, 657, 6962], + [74, 3948, 6963], + [6932, 107, 6964], + [1263, 383, 6965], + [354, 419, 6966], + [332, 6918, 6967], + [1908, 265, 6968], + [621, 904, 6969], + [1607, 480, 6970], + [282, 1685, 6971], + [2516, 373, 6972], + [2612, 265, 6973], + [2989, 317, 6974], + [6148, 115, 6975], + [4516, 770, 6976], + [5951, 269, 6977], + [5530, 6046, 6978], + [261, 121, 6979], + [3282, 115, 6980], + [295, 370, 6981], + [1354, 1841, 6982], + [356, 712, 6983], + [498, 904, 6984], + [1545, 115, 6985], + [5807, 5324, 6986], + [6053, 265, 6987], + [6952, 6751, 6988], + [710, 322, 6989], + [1924, 390, 6990], + [1154, 280, 6991], + [461, 1238, 6992], + [5918, 2007, 6993], + [294, 111, 6994], + [116, 258, 6995], + [418, 421, 6996], + [368, 2649, 6997], + [710, 6426, 6998], + [368, 6931, 6999], + [1310, 836, 7000], + [685, 603, 7001], + [3329, 1517, 7002], + [101, 839, 7003], + [1000, 349, 7004], + [283, 2075, 7005], + [2257, 287, 7006], + [2272, 430, 7007], + [6435, 1065, 7008], + [5912, 265, 7009], + [6863, 301, 7010], + [2069, 480, 7011], + [118, 292, 7012], + [79, 456, 7013], + [400, 496, 7014], + [1591, 289, 7015], + [692, 1025, 7016], + [1835, 1828, 7017], + [5601, 767, 7018], + [4690, 309, 7019], + [4743, 1727, 7020], + [276, 422, 7021], + [666, 591, 7022], + [101, 399, 7023], + [1115, 6590, 7024], + [598, 1627, 7025], + [1082, 291, 7026], + [2837, 317, 7027], + [7025, 297, 7028], + [3198, 121, 7029], + [723, 3635, 7030], + [4794, 103, 7031], + [598, 6700, 7032], + [332, 383, 7033], + [282, 328, 7034], + [3931, 6933, 7035], + [429, 115, 7036], + [295, 615, 7037], + [521, 839, 7038], + [259, 1183, 7039], + [1132, 280, 7040], + [283, 1238, 7041], + [502, 1586, 7042], + [1925, 1561, 7043], + [2043, 115, 7044], + [593, 487, 7045], + [1247, 657, 7046], + [83, 386, 7047], + [351, 4097, 7048], + [407, 1150, 7049], + [1214, 824, 7050], + [590, 6994, 7051], + [629, 99, 7052], + [5235, 801, 7053], + [609, 391, 7054], + [7054, 2195, 7055], + [1667, 500, 7056], + [83, 790, 7057], + [3736, 291, 7058], + [1093, 5117, 7059], + [2371, 4293, 7060], + [5240, 313, 7061], + [609, 6275, 7062], + [5969, 115, 7063], + [99, 422, 7064], + [272, 996, 7065], + [2967, 839, 7066], + [2280, 466, 7067], + [83, 616, 7068], + [322, 121, 7069], + [1626, 399, 7070], + [1735, 291, 7071], + [569, 99, 7072], + [904, 280, 7073], + [797, 121, 7074], + [4778, 5117, 7075], + [323, 2191, 7076], + [109, 2704, 7077], + [106, 111, 7078], + [121, 317, 7079], + [4769, 108, 7080], + [279, 943, 7081], + [461, 3707, 7082], + [408, 5910, 7083], + [1211, 291, 7084], + [1984, 115, 7085], + [5015, 291, 7086], + [621, 3943, 7087], + [4276, 412, 7088], + [286, 5337, 7089], + [968, 119, 7090], + [461, 296, 7091], + [370, 693, 7092], + [1097, 665, 7093], + [267, 1659, 7094], + [803, 5965, 7095], + [515, 904, 7096], + [5497, 280, 7097], + [2298, 5823, 7098], + [4053, 7070, 7099], + [6447, 6936, 7100], + [794, 473, 7101], + [5084, 1853, 7102], + [5363, 528, 7103], + [424, 115, 7104], + [716, 310, 7105], + [259, 852, 7106], + [6955, 4578, 7107], + [1468, 265, 7108], + [108, 833, 7109], + [3439, 265, 7110], + [3321, 6100, 7111], + [3116, 280, 7112], + [6690, 271, 7113], + [2257, 289, 7114], + [32, 50, 7115], + [286, 104, 7116], + [5362, 947, 7117], + [1112, 597, 7118], + [4050, 1345, 7119], + [779, 111, 7120], + [5605, 802, 7121], + [5718, 767, 7122], + [955, 458, 7123], + [299, 545, 7124], + [69, 357, 7125], + [590, 101, 7126], + [734, 291, 7127], + [3373, 1397, 7128], + [117, 864, 7129], + [1148, 1175, 7130], + [361, 1472, 7131], + [102, 1733, 7132], + [2020, 115, 7133], + [2201, 265, 7134], + [3641, 2986, 7135], + [4846, 280, 7136], + [324, 7012, 7137], + [105, 1225, 7138], + [4066, 4753, 7139], + [4256, 291, 7140], + [260, 2917, 7141], + [598, 7052, 7142], + [1020, 1660, 7143], + [278, 6913, 7144], + [6776, 525, 7145], + [272, 6387, 7146], + [5548, 114, 7147], + [881, 6365, 7148], + [847, 280, 7149], + [1214, 5117, 7150], + [4972, 4570, 7151], + [324, 652, 7152], + [738, 558, 7153], + [5811, 280, 7154], + [6917, 983, 7155], + [977, 6064, 7156], + [287, 291, 7157], + [474, 577, 7158], + [672, 2810, 7159], + [1068, 903, 7160], + [622, 1267, 7161], + [2513, 2499, 7162], + [119, 6361, 7163], + [70, 111, 7164], + [282, 743, 7165], + [293, 597, 7166], + [394, 1991, 7167], + [77, 806, 7168], + [2368, 115, 7169], + [2257, 446, 7170], + [306, 1317, 7171], + [302, 379, 7172], + [1632, 5004, 7173], + [2828, 664, 7174], + [502, 1768, 7175], + [7039, 790, 7176], + [298, 3103, 7177], + [1290, 291, 7178], + [1068, 287, 7179], + [925, 680, 7180], + [1420, 115, 7181], + [2859, 121, 7182], + [278, 4861, 7183], + [6279, 115, 7184], + [258, 119, 7185], + [6604, 5623, 7186], + [502, 297, 7187], + [1088, 121, 7188], + [1968, 2716, 7189], + [304, 3210, 7190], + [2641, 390, 7191], + [1208, 6473, 7192], + [2828, 112, 7193], + [3209, 115, 7194], + [100, 3242, 7195], + [1729, 339, 7196], + [3472, 291, 7197], + [3460, 376, 7198], + [102, 390, 7199], + [1593, 115, 7200], + [3597, 291, 7201], + [4246, 265, 7202], + [5375, 290, 7203], + [290, 99, 7204], + [118, 672, 7205], + [354, 531, 7206], + [687, 749, 7207], + [346, 1975, 7208], + [6149, 116, 7209], + [361, 3485, 7210], + [2806, 115, 7211], + [4779, 390, 7212], + [6135, 390, 7213], + [1183, 97, 7214], + [279, 1384, 7215], + [547, 719, 7216], + [1130, 115, 7217], + [2090, 773, 7218], + [86, 2911, 7219], + [274, 5388, 7220], + [5815, 680, 7221], + [336, 6349, 7222], + [408, 370, 7223], + [2400, 6433, 7224], + [1348, 115, 7225], + [3216, 657, 7226], + [1272, 1730, 7227], + [4239, 115, 7228], + [6513, 5552, 7229], + [4583, 512, 7230], + [508, 559, 7231], + [1583, 115, 7232], + [298, 6472, 7233], + [593, 550, 7234], + [6542, 6494, 7235], + [1298, 7195, 7236], + [723, 3256, 7237], + [1591, 6066, 7238], + [3949, 1517, 7239], + [354, 4146, 7240], + [7042, 1138, 7241], + [844, 6028, 7242], + [1171, 121, 7243], + [682, 115, 7244], + [3920, 2004, 7245], + [2406, 291, 7246], + [283, 392, 7247], + [773, 97, 7248], + [4794, 301, 7249], + [569, 1992, 7250], + [2043, 291, 7251], + [7206, 111, 7252], + [6418, 303, 7253], + [297, 291, 7254], + [677, 1730, 7255], + [590, 117, 7256], + [965, 97, 7257], + [1081, 390, 7258], + [293, 7190, 7259], + [1178, 1828, 7260], + [119, 1722, 7261], + [757, 3959, 7262], + [4637, 265, 7263], + [853, 6562, 7264], + [7236, 280, 7265], + [2219, 118, 7266], + [105, 1091, 7267], + [544, 110, 7268], + [98, 2482, 7269], + [2036, 115, 7270], + [799, 597, 7271], + [3697, 97, 7272], + [3366, 115, 7273], + [6027, 657, 7274], + [7135, 291, 7275], + [414, 291, 7276], + [260, 290, 7277], + [342, 852, 7278], + [368, 641, 7279], + [1315, 115, 7280], + [1639, 121, 7281], + [1652, 1636, 7282], + [2257, 301, 7283], + [2977, 2125, 7284], + [3960, 115, 7285], + [7056, 1992, 7286], + [2192, 6028, 7287], + [620, 7159, 7288], + [4040, 115, 7289], + [611, 317, 7290], + [380, 550, 7291], + [98, 7161, 7292], + [112, 1086, 7293], + [1412, 2451, 7294], + [2257, 2122, 7295], + [4523, 115, 7296], + [502, 1325, 7297], + [5064, 2183, 7298], + [677, 97, 7299], + [611, 269, 7300], + [380, 836, 7301], + [859, 719, 7302], + [6093, 370, 7303], + [930, 5195, 7304], + [495, 4346, 7305], + [7249, 7163, 7306], + [6288, 4309, 7307], + [7297, 644, 7308], + [1158, 265, 7309], + [1473, 280, 7310], + [111, 767, 7311], + [6350, 285, 7312], + [6593, 269, 7313], + [3939, 485, 7314], + [4035, 485, 7315], + [2898, 105, 7316], + [1115, 5965, 7317], + [298, 105, 7318], + [342, 97, 7319], + [326, 263, 7320], + [5404, 280, 7321], + [259, 613, 7322], + [6710, 719, 7323], + [306, 7298, 7324], + [2878, 595, 7325], + [4401, 947, 7326], + [5018, 1926, 7327], + [730, 301, 7328], + [1088, 115, 7329], + [690, 102, 7330], + [369, 291, 7331], + [881, 1780, 7332], + [7164, 1090, 7333], + [1219, 550, 7334], + [407, 3210, 7335], + [672, 3922, 7336], + [590, 7248, 7337], + [261, 273, 7338], + [2866, 4221, 7339], + [2351, 291, 7340], + [524, 117, 7341], + [2081, 983, 7342], + [5334, 512, 7343], + [3793, 291, 7344], + [2340, 1578, 7345], + [2441, 558, 7346], + [259, 2494, 7347], + [3912, 533, 7348], + [4461, 115, 7349], + [87, 5082, 7350], + [5132, 291, 7351], + [7101, 657, 7352], + [6174, 115, 7353], + [318, 106, 7354], + [3012, 5786, 7355], + [325, 369, 7356], + [1136, 121, 7357], + [324, 854, 7358], + [930, 399, 7359], + [2914, 291, 7360], + [3498, 1384, 7361], + [5202, 301, 7362], + [6737, 1237, 7363], + [734, 6895, 7364], + [309, 1828, 7365], + [333, 109, 7366], + [304, 852, 7367], + [109, 852, 7368], + [2323, 291, 7369], + [1382, 280, 7370], + [2518, 115, 7371], + [259, 1762, 7372], + [5113, 291, 7373], + [6144, 304, 7374], + [7356, 2111, 7375], + [7080, 3359, 7376], + [666, 7278, 7377], + [2240, 115, 7378], + [5715, 2821, 7379], + [376, 820, 7380], + [5763, 1648, 7381], + [384, 317, 7382], + [7089, 533, 7383], + [5721, 291, 7384], + [1716, 603, 7385], + [282, 6502, 7386], + [7313, 6066, 7387], + [4006, 115, 7388], + [948, 280, 7389], + [1930, 1878, 7390], + [274, 7336, 7391], + [6444, 115, 7392], + [271, 121, 7393], + [295, 107, 7394], + [65, 373, 7395], + [278, 7067, 7396], + [281, 487, 7397], + [435, 319, 7398], + [726, 291, 7399], + [887, 1935, 7400], + [3583, 291, 7401], + [1785, 291, 7402], + [3651, 7132, 7403], + [2057, 2997, 7404], + [73, 110, 7405], + [7398, 399, 7406], + [3570, 7380, 7407], + [5918, 839, 7408], + [1490, 421, 7409], + [2447, 3423, 7410], + [3773, 1573, 7411], + [6728, 5792, 7412], + [980, 1841, 7413], + [407, 5561, 7414], + [83, 7180, 7415], + [3166, 565, 7416], + [4402, 291, 7417], + [5376, 115, 7418], + [1668, 6282, 7419], + [74, 297, 7420], + [32, 88, 7421], + [67, 108, 7422], + [413, 1021, 7423], + [4325, 774, 7424], + [1381, 467, 7425], + [118, 1472, 7426], + [99, 379, 7427], + [904, 121, 7428], + [1211, 115, 7429], + [76, 5594, 7430], + [3410, 2017, 7431], + [1804, 7292, 7432], + [3570, 100, 7433], + [2447, 4994, 7434], + [283, 7367, 7435], + [278, 6890, 7436], + [376, 121, 7437], + [259, 292, 7438], + [3282, 291, 7439], + [579, 121, 7440], + [999, 119, 7441], + [3160, 115, 7442], + [293, 643, 7443], + [4612, 114, 7444], + [1083, 115, 7445], + [2115, 105, 7446], + [2831, 773, 7447], + [5951, 97, 7448], + [3572, 115, 7449], + [1227, 3972, 7450], + [7004, 303, 7451], + [355, 291, 7452], + [931, 4748, 7453], + [2226, 947, 7454], + [6144, 982, 7455], + [349, 101, 7456], + [354, 321, 7457], + [276, 414, 7458], + [320, 429, 7459], + [1432, 317, 7460], + [483, 2714, 7461], + [965, 6146, 7462], + [7256, 4240, 7463], + [63, 426, 7464], + [349, 280, 7465], + [1796, 643, 7466], + [5578, 911, 7467], + [1034, 983, 7468], + [803, 813, 7469], + [860, 657, 7470], + [1343, 291, 7471], + [1420, 116, 7472], + [4050, 1606, 7473], + [4525, 291, 7474], + [4237, 291, 7475], + [5131, 115, 7476], + [7166, 525, 7477], + [6419, 4992, 7478], + [280, 109, 7479], + [487, 291, 7480], + [278, 507, 7481], + [32, 1640, 7482], + [853, 319, 7483], + [2257, 468, 7484], + [2818, 890, 7485], + [637, 3355, 7486], + [99, 382, 7487], + [394, 537, 7488], + [596, 387, 7489], + [5718, 5783, 7490], + [442, 632, 7491], + [990, 114, 7492], + [723, 4000, 7493], + [1418, 102, 7494], + [2314, 101, 7495], + [76, 2242, 7496], + [356, 2505, 7497], + [2914, 115, 7498], + [4703, 1138, 7499], + [6627, 115, 7500], + [7105, 1041, 7501], + [7223, 573, 7502], + [7290, 6287, 7503], + [332, 1277, 7504], + [1968, 304, 7505], + [5925, 802, 7506], + [2423, 473, 7507], + [902, 7207, 7508], + [7505, 387, 7509], + [2833, 280, 7510], + [2784, 98, 7511], + [5015, 265, 7512], + [517, 2687, 7513], + [461, 7440, 7514], + [977, 608, 7515], + [306, 1340, 7516], + [1438, 115, 7517], + [1815, 115, 7518], + [6819, 3093, 7519], + [3032, 5783, 7520], + [7330, 702, 7521], + [7126, 7331, 7522], + [7511, 982, 7523], + [7513, 657, 7524], + [98, 320, 7525], + [3873, 2832, 7526], + [84, 422, 7527], + [583, 944, 7528], + [5479, 115, 7529], + [5021, 5617, 7530], + [6414, 1013, 7531], + [2372, 7500, 7532], + [569, 582, 7533], + [1737, 291, 7534], + [461, 852, 7535], + [77, 975, 7536], + [1199, 597, 7537], + [472, 2951, 7538], + [290, 2004, 7539], + [7528, 6922, 7540], + [2390, 115, 7541], + [2622, 115, 7542], + [6668, 335, 7543], + [2370, 7299, 7544], + [6740, 112, 7545], + [407, 869, 7546], + [1965, 115, 7547], + [277, 379, 7548], + [284, 1059, 7549], + [1688, 802, 7550], + [1808, 115, 7551], + [114, 2235, 7552], + [4516, 3776, 7553], + [2773, 2016, 7554], + [365, 6861, 7555], + [3108, 983, 7556], + [3583, 2044, 7557], + [333, 4240, 7558], + [332, 5570, 7559], + [7037, 1465, 7560], + [276, 121, 7561], + [69, 109, 7562], + [71, 313, 7563], + [6592, 387, 7564], + [2940, 111, 7565], + [2288, 390, 7566], + [108, 806, 7567], + [86, 1472, 7568], + [820, 97, 7569], + [598, 2338, 7570], + [3806, 115, 7571], + [7204, 1711, 7572], + [5504, 265, 7573], + [6979, 7552, 7574], + [589, 864, 7575], + [2305, 458, 7576], + [102, 392, 7577], + [380, 328, 7578], + [804, 6629, 7579], + [930, 115, 7580], + [4631, 291, 7581], + [336, 5181, 7582], + [270, 7411, 7583], + [1068, 466, 7584], + [2448, 1764, 7585], + [2953, 115, 7586], + [2307, 695, 7587], + [3789, 265, 7588], + [598, 4619, 7589], + [6341, 390, 7590], + [601, 7539, 7591], + [728, 1018, 7592], + [5085, 291, 7593], + [6589, 1117, 7594], + [66, 370, 7595], + [710, 533, 7596], + [343, 269, 7597], + [298, 7428, 7598], + [6956, 3668, 7599], + [279, 310, 7600], + [279, 746, 7601], + [1983, 289, 7602], + [6775, 3020, 7603], + [279, 2655, 7604], + [2859, 291, 7605], + [1827, 2997, 7606], + [7604, 537, 7607], + [636, 121, 7608], + [105, 1402, 7609], + [503, 7609, 7610], + [2118, 317, 7611], + [1418, 1966, 7612], + [598, 986, 7613], + [4780, 983, 7614], + [2609, 115, 7615], + [3318, 1175, 7616], + [4277, 317, 7617], + [3547, 6626, 7618], + [295, 7594, 7619], + [99, 97, 7620], + [70, 108, 7621], + [374, 317, 7622], + [4171, 329, 7623], + [902, 269, 7624], + [717, 309, 7625], + [1996, 265, 7626], + [5363, 4556, 7627], + [4363, 1059, 7628], + [315, 5573, 7629], + [2569, 7077, 7630], + [2747, 1378, 7631], + [271, 119, 7632], + [2411, 743, 7633], + [349, 301, 7634], + [304, 5419, 7635], + [282, 478, 7636], + [585, 772, 7637], + [99, 1529, 7638], + [7323, 294, 7639], + [6688, 116, 7640], + [6591, 265, 7641], + [7575, 500, 7642], + [2940, 496, 7643], + [545, 291, 7644], + [5811, 2537, 7645], + [609, 111, 7646], + [109, 1138, 7647], + [408, 1539, 7648], + [3698, 291, 7649], + [2314, 1179, 7650], + [2192, 3219, 7651], + [5569, 2019, 7652], + [7637, 317, 7653], + [334, 97, 7654], + [97, 736, 7655], + [336, 391, 7656], + [545, 558, 7657], + [593, 1762, 7658], + [2381, 2426, 7659], + [713, 2310, 7660], + [7646, 6871, 7661], + [3193, 1007, 7662], + [5891, 645, 7663], + [293, 7479, 7664], + [1667, 5898, 7665], + [4121, 6813, 7666], + [271, 364, 7667], + [685, 1841, 7668], + [331, 485, 7669], + [332, 269, 7670], + [1034, 309, 7671], + [279, 595, 7672], + [323, 2649, 7673], + [821, 309, 7674], + [3410, 7319, 7675], + [1644, 291, 7676], + [6900, 947, 7677], + [1939, 537, 7678], + [3647, 115, 7679], + [4644, 265, 7680], + [574, 7577, 7681], + [5956, 657, 7682], + [955, 6474, 7683], + [3541, 446, 7684], + [1038, 445, 7685], + [2929, 291, 7686], + [421, 121, 7687], + [261, 446, 7688], + [7156, 1345, 7689], + [2103, 291, 7690], + [4544, 291, 7691], + [534, 4654, 7692], + [5870, 399, 7693], + [7316, 369, 7694], + [97, 97, 7695], + [502, 379, 7696], + [601, 108, 7697], + [291, 280, 7698], + [260, 302, 7699], + [335, 373, 7700], + [443, 839, 7701], + [503, 5904, 7702], + [407, 645, 7703], + [5505, 6079, 7704], + [3875, 115, 7705], + [2398, 802, 7706], + [2308, 2704, 7707], + [2154, 3954, 7708], + [1986, 6287, 7709], + [3265, 7365, 7710], + [368, 764, 7711], + [259, 289, 7712], + [7565, 337, 7713], + [1645, 280, 7714], + [472, 376, 7715], + [590, 7480, 7716], + [902, 104, 7717], + [2302, 115, 7718], + [2484, 5388, 7719], + [3284, 2125, 7720], + [119, 258, 7721], + [995, 7687, 7722], + [276, 395, 7723], + [2163, 115, 7724], + [5025, 390, 7725], + [102, 1277, 7726], + [553, 1606, 7727], + [679, 2905, 7728], + [2598, 322, 7729], + [4794, 1121, 7730], + [3036, 558, 7731], + [4290, 291, 7732], + [3895, 265, 7733], + [4069, 4172, 7734], + [4409, 379, 7735], + [6409, 1627, 7736], + [7049, 369, 7737], + [7113, 5083, 7738], + [5595, 1926, 7739], + [97, 772, 7740], + [7484, 947, 7741], + [2341, 7069, 7742], + [461, 265, 7743], + [328, 548, 7744], + [932, 5623, 7745], + [533, 657, 7746], + [902, 117, 7747], + [2349, 985, 7748], + [1432, 2781, 7749], + [7318, 1780, 7750], + [728, 5427, 7751], + [6510, 657, 7752], + [294, 280, 7753], + [295, 119, 7754], + [807, 1238, 7755], + [819, 985, 7756], + [354, 3249, 7757], + [119, 680, 7758], + [431, 3668, 7759], + [102, 704, 7760], + [943, 317, 7761], + [5257, 115, 7762], + [6025, 97, 7763], + [2767, 3242, 7764], + [4690, 390, 7765], + [3841, 1021, 7766], + [4324, 3355, 7767], + [5696, 7261, 7768], + [296, 115, 7769], + [983, 6632, 7770], + [1704, 2089, 7771], + [5222, 7276, 7772], + [6319, 121, 7773], + [3494, 108, 7774], + [646, 548, 7775], + [367, 109, 7776], + [2209, 548, 7777], + [7712, 527, 7778], + [7736, 684, 7779], + [5894, 115, 7780], + [77, 1360, 7781], + [1567, 339, 7782], + [303, 1598, 7783], + [7433, 1999, 7784], + [1634, 317, 7785], + [2962, 983, 7786], + [999, 4810, 7787], + [830, 7770, 7788], + [7436, 1606, 7789], + [7645, 6873, 7790], + [1115, 3084, 7791], + [6521, 1181, 7792], + [7507, 943, 7793], + [295, 101, 7794], + [971, 108, 7795], + [3990, 6947, 7796], + [409, 5638, 7797], + [3839, 1846, 7798], + [4147, 2633, 7799], + [7266, 704, 7800], + [98, 441, 7801], + [716, 6554, 7802], + [877, 1302, 7803], + [1516, 100, 7804], + [109, 2320, 7805], + [4464, 1650, 7806], + [1919, 2021, 7807], + [5781, 509, 7808], + [87, 104, 7809], + [584, 7657, 7810], + [112, 297, 7811], + [995, 101, 7812], + [2814, 657, 7813], + [368, 301, 7814], + [303, 429, 7815], + [4149, 265, 7816], + [611, 613, 7817], + [1136, 301, 7818], + [3038, 954, 7819], + [3755, 2342, 7820], + [1917, 567, 7821], + [2078, 983, 7822], + [2660, 335, 7823], + [666, 4739, 7824], + [299, 118, 7825], + [4420, 115, 7826], + [325, 490, 7827], + [7714, 695, 7828], + [323, 773, 7829], + [902, 291, 7830], + [929, 265, 7831], + [276, 3001, 7832], + [1034, 121, 7833], + [5363, 661, 7834], + [2913, 291, 7835], + [4231, 97, 7836], + [508, 6437, 7837], + [807, 348, 7838], + [119, 101, 7839], + [3290, 4346, 7840], + [275, 115, 7841], + [593, 3635, 7842], + [474, 292, 7843], + [7771, 852, 7844], + [1154, 390, 7845], + [6454, 303, 7846], + [3610, 265, 7847], + [6300, 101, 7848], + [5624, 111, 7849], + [119, 602, 7850], + [7277, 309, 7851], + [4612, 5387, 7852], + [7014, 364, 7853], + [1068, 3247, 7854], + [2341, 104, 7855], + [5795, 115, 7856], + [6370, 97, 7857], + [3557, 115, 7858], + [3851, 533, 7859], + [5339, 291, 7860], + [1986, 6585, 7861], + [6639, 291, 7862], + [3043, 7638, 7863], + [6654, 7805, 7864], + [6953, 947, 7865], + [114, 329, 7866], + [408, 390, 7867], + [318, 429, 7868], + [295, 2391, 7869], + [3570, 7726, 7870], + [756, 101, 7871], + [7233, 947, 7872], + [1319, 291, 7873], + [1667, 111, 7874], + [1766, 104, 7875], + [2243, 265, 7876], + [407, 7836, 7877], + [611, 2857, 7878], + [6353, 4255, 7879], + [7793, 1345, 7880], + [7802, 102, 7881], + [713, 115, 7882], + [6695, 97, 7883], + [280, 107, 7884], + [1085, 99, 7885], + [831, 560, 7886], + [2447, 1502, 7887], + [3236, 595, 7888], + [4486, 121, 7889], + [66, 6022, 7890], + [108, 116, 7891], + [84, 119, 7892], + [281, 110, 7893], + [367, 338, 7894], + [111, 476, 7895], + [5341, 115, 7896], + [3672, 7895, 7897], + [1618, 1853, 7898], + [902, 7660, 7899], + [5332, 291, 7900], + [6131, 390, 7901], + [7033, 806, 7902], + [7334, 265, 7903], + [7897, 104, 7904], + [3091, 115, 7905], + [461, 121, 7906], + [103, 466, 7907], + [117, 369, 7908], + [6767, 695, 7909], + [474, 4454, 7910], + [369, 558, 7911], + [819, 558, 7912], + [7616, 1138, 7913], + [977, 6079, 7914], + [1220, 115, 7915], + [84, 1322, 7916], + [3607, 852, 7917], + [4421, 317, 7918], + [7823, 3703, 7919], + [6080, 7214, 7920], + [7651, 291, 7921], + [7654, 319, 7922], + [4581, 415, 7923], + [3049, 702, 7924], + [2748, 115, 7925], + [393, 328, 7926], + [7697, 560, 7927], + [469, 680, 7928], + [710, 595, 7929], + [1115, 911, 7930], + [621, 2716, 7931], + [1085, 111, 7932], + [268, 2182, 7933], + [3780, 115, 7934], + [3176, 6146, 7935], + [7871, 1402, 7936], + [495, 7884, 7937], + [2304, 7205, 7938], + [593, 512, 7939], + [267, 591, 7940], + [2874, 335, 7941], + [577, 101, 7942], + [655, 296, 7943], + [965, 975, 7944], + [294, 1571, 7945], + [7855, 457, 7946], + [6288, 893, 7947], + [341, 2681, 7948], + [380, 6677, 7949], + [1831, 3320, 7950], + [3410, 4570, 7951], + [6550, 115, 7952], + [6393, 265, 7953], + [7072, 1100, 7954], + [1214, 2000, 7955], + [276, 2463, 7956], + [5435, 558, 7957], + [5353, 265, 7958], + [4121, 767, 7959], + [366, 7368, 7960], + [584, 319, 7961], + [283, 613, 7962], + [7943, 6823, 7963], + [335, 719, 7964], + [3151, 5103, 7965], + [66, 1630, 7966], + [1618, 115, 7967], + [6685, 1944, 7968], + [2009, 115, 7969], + [2238, 2320, 7970], + [6284, 852, 7971], + [3623, 773, 7972], + [1068, 7904, 7973], + [100, 97, 7974], + [4743, 2942, 7975], + [6409, 813, 7976], + [3281, 115, 7977], + [380, 7815, 7978], + [4955, 1345, 7979], + [832, 115, 7980], + [1994, 104, 7981], + [497, 7644, 7982], + [1328, 2153, 7983], + [7976, 370, 7984], + [2521, 5046, 7985], + [2054, 101, 7986], + [2532, 115, 7987], + [741, 7935, 7988], + [108, 111, 7989], + [296, 1853, 7990], + [431, 99, 7991], + [474, 269, 7992], + [803, 424, 7993], + [68, 1777, 7994], + [965, 687, 7995], + [1221, 103, 7996], + [2175, 115, 7997], + [2179, 512, 7998], + [3813, 115, 7999], + [6761, 628, 8000], + [5944, 1926, 8001], + [6643, 1561, 8002], + [7165, 121, 8003], + [281, 7655, 8004], + [293, 391, 8005], + [83, 111, 8006], + [108, 303, 8007], + [261, 1402, 8008], + [609, 643, 8009], + [7124, 280, 8010], + [5964, 558, 8011], + [895, 3675, 8012], + [495, 5823, 8013], + [318, 1901, 8014], + [332, 2195, 8015], + [4668, 115, 8016], + [6414, 6022, 8017], + [115, 109, 8018], + [108, 985, 8019], + [4174, 768, 8020], + [2858, 657, 8021], + [6828, 663, 8022], + [270, 328, 8023], + [490, 379, 8024], + [1132, 291, 8025], + [323, 815, 8026], + [1124, 597, 8027], + [368, 1363, 8028], + [3237, 848, 8029], + [306, 2464, 8030], + [32, 4874, 8031], + [3204, 2262, 8032], + [369, 4878, 8033], + [5988, 291, 8034], + [2943, 7950, 8035], + [3570, 5370, 8036], + [65, 308, 8037], + [368, 269, 8038], + [2400, 485, 8039], + [585, 852, 8040], + [334, 2832, 8041], + [3598, 265, 8042], + [6504, 291, 8043], + [7231, 280, 8044], + [7438, 1041, 8045], + [7466, 774, 8046], + [2333, 269, 8047], + [112, 275, 8048], + [354, 1630, 8049], + [259, 291, 8050], + [380, 391, 8051], + [303, 1345, 8052], + [902, 297, 8053], + [295, 725, 8054], + [474, 7941, 8055], + [1347, 1841, 8056], + [5988, 265, 8057], + [536, 6540, 8058], + [259, 7971, 8059], + [276, 2505, 8060], + [1385, 265, 8061], + [609, 6564, 8062], + [354, 7634, 8063], + [579, 115, 8064], + [639, 115, 8065], + [1148, 117, 8066], + [1324, 115, 8067], + [1097, 839, 8068], + [3052, 115, 8069], + [1287, 1349, 8070], + [7189, 265, 8071], + [1760, 7199, 8072], + [332, 1586, 8073], + [1773, 383, 8074], + [2660, 2137, 8075], + [4602, 115, 8076], + [995, 4640, 8077], + [6925, 4018, 8078], + [7875, 704, 8079], + [5013, 7569, 8080], + [4871, 97, 8081], + [7561, 4578, 8082], + [99, 558, 8083], + [77, 114, 8084], + [7621, 1572, 8085], + [7664, 115, 8086], + [821, 629, 8087], + [2598, 122, 8088], + [2257, 595, 8089], + [2437, 1345, 8090], + [2538, 112, 8091], + [2811, 1529, 8092], + [259, 2907, 8093], + [3544, 4038, 8094], + [2344, 7783, 8095], + [6419, 379, 8096], + [6896, 485, 8097], + [364, 1722, 8098], + [7142, 7036, 8099], + [323, 121, 8100], + [622, 308, 8101], + [77, 559, 8102], + [4705, 337, 8103], + [859, 864, 8104], + [97, 895, 8105], + [8094, 1926, 8106], + [1309, 265, 8107], + [3180, 1138, 8108], + [2450, 291, 8109], + [2767, 115, 8110], + [71, 3316, 8111], + [7812, 4119, 8112], + [730, 7078, 8113], + [1700, 8012, 8114], + [2303, 8019, 8115], + [84, 328, 8116], + [3746, 657, 8117], + [407, 1456, 8118], + [6045, 328, 8119], + [354, 7635, 8120], + [4042, 115, 8121], + [590, 813, 8122], + [1395, 115, 8123], + [7717, 292, 8124], + [4115, 301, 8125], + [4657, 115, 8126], + [2418, 291, 8127], + [2287, 115, 8128], + [65, 7147, 8129], + [1811, 8104, 8130], + [8092, 628, 8131], + [119, 1261, 8132], + [2179, 328, 8133], + [1759, 115, 8134], + [2351, 115, 8135], + [3466, 291, 8136], + [3440, 6282, 8137], + [4783, 8033, 8138], + [5426, 7924, 8139], + [5356, 115, 8140], + [3417, 2494, 8141], + [892, 1021, 8142], + [804, 263, 8143], + [593, 943, 8144], + [5337, 8134, 8145], + [1630, 309, 8146], + [4027, 112, 8147], + [2612, 291, 8148], + [6474, 101, 8149], + [4411, 291, 8150], + [4533, 291, 8151], + [4536, 291, 8152], + [5989, 1568, 8153], + [6510, 280, 8154], + [8078, 303, 8155], + [295, 287, 8156], + [8122, 4528, 8157], + [301, 97, 8158], + [621, 550, 8159], + [2795, 115, 8160], + [4633, 2563, 8161], + [296, 704, 8162], + [962, 4250, 8163], + [545, 983, 8164], + [860, 2000, 8165], + [6971, 291, 8166], + [2960, 657, 8167], + [4599, 265, 8168], + [4876, 1397, 8169], + [292, 5288, 8170], + [1051, 5898, 8171], + [3397, 8161, 8172], + [380, 269, 8173], + [1962, 2280, 8174], + [110, 321, 8175], + [5907, 115, 8176], + [875, 597, 8177], + [118, 3624, 8178], + [2944, 2125, 8179], + [502, 349, 8180], + [114, 717, 8181], + [977, 8178, 8182], + [274, 565, 8183], + [300, 736, 8184], + [898, 943, 8185], + [1230, 657, 8186], + [6919, 338, 8187], + [1534, 115, 8188], + [306, 1651, 8189], + [5366, 3624, 8190], + [1927, 597, 8191] + ] +} diff --git a/tinystories_bpe.json b/tinystories_bpe.json new file mode 100644 index 00000000..dc634d4c --- /dev/null +++ b/tinystories_bpe.json @@ -0,0 +1,7948 @@ +{ + "type": "bpe_tokenizer", + "vocab": { + "0": "00", + "1": "01", + "2": "02", + "3": "03", + "4": "04", + "5": "05", + "6": "06", + "7": "07", + "8": "08", + "9": "09", + "10": "0a", + "11": "0b", + "12": "0c", + "13": "0d", + "14": "0e", + "15": "0f", + "16": "10", + "17": "11", + "18": "12", + "19": "13", + "20": "14", + "21": "15", + "22": "16", + "23": "17", + "24": "18", + "25": "19", + "26": "1a", + "27": "1b", + "28": "1c", + "29": "1d", + "30": "1e", + "31": "1f", + "32": "20", + "33": "21", + "34": "22", + "35": "23", + "36": "24", + "37": "25", + "38": "26", + "39": "27", + "40": "28", + "41": "29", + "42": "2a", + "43": "2b", + "44": "2c", + "45": "2d", + "46": "2e", + "47": "2f", + "48": "30", + "49": "31", + "50": "32", + "51": "33", + "52": "34", + "53": "35", + "54": "36", + "55": "37", + "56": "38", + "57": "39", + "58": "3a", + "59": "3b", + "60": "3c", + "61": "3d", + "62": "3e", + "63": "3f", + "64": "40", + "65": "41", + "66": "42", + "67": "43", + "68": "44", + "69": "45", + "70": "46", + "71": "47", + "72": "48", + "73": "49", + "74": "4a", + "75": "4b", + "76": "4c", + "77": "4d", + "78": "4e", + "79": "4f", + "80": "50", + "81": "51", + "82": "52", + "83": "53", + "84": "54", + "85": "55", + "86": "56", + "87": "57", + "88": "58", + "89": "59", + "90": "5a", + "91": "5b", + "92": "5c", + "93": "5d", + "94": "5e", + "95": "5f", + "96": "60", + "97": "61", + "98": "62", + "99": "63", + "100": "64", + "101": "65", + "102": "66", + "103": "67", + "104": "68", + "105": "69", + "106": "6a", + "107": "6b", + "108": "6c", + "109": "6d", + "110": "6e", + "111": "6f", + "112": "70", + "113": "71", + "114": "72", + "115": "73", + "116": "74", + "117": "75", + "118": "76", + "119": "77", + "120": "78", + "121": "79", + "122": "7a", + "123": "7b", + "124": "7c", + "125": "7d", + "126": "7e", + "127": "7f", + "128": "80", + "129": "81", + "130": "82", + "131": "83", + "132": "84", + "133": "85", + "134": "86", + "135": "87", + "136": "88", + "137": "89", + "138": "8a", + "139": "8b", + "140": "8c", + "141": "8d", + "142": "8e", + "143": "8f", + "144": "90", + "145": "91", + "146": "92", + "147": "93", + "148": "94", + "149": "95", + "150": "96", + "151": "97", + "152": "98", + "153": "99", + "154": "9a", + "155": "9b", + "156": "9c", + "157": "9d", + "158": "9e", + "159": "9f", + "160": "a0", + "161": "a1", + "162": "a2", + "163": "a3", + "164": "a4", + "165": "a5", + "166": "a6", + "167": "a7", + "168": "a8", + "169": "a9", + "170": "aa", + "171": "ab", + "172": "ac", + "173": "ad", + "174": "ae", + "175": "af", + "176": "b0", + "177": "b1", + "178": "b2", + "179": "b3", + "180": "b4", + "181": "b5", + "182": "b6", + "183": "b7", + "184": "b8", + "185": "b9", + "186": "ba", + "187": "bb", + "188": "bc", + "189": "bd", + "190": "be", + "191": "bf", + "192": "c0", + "193": "c1", + "194": "c2", + "195": "c3", + "196": "c4", + "197": "c5", + "198": "c6", + "199": "c7", + "200": "c8", + "201": "c9", + "202": "ca", + "203": "cb", + "204": "cc", + "205": "cd", + "206": "ce", + "207": "cf", + "208": "d0", + "209": "d1", + "210": "d2", + "211": "d3", + "212": "d4", + "213": "d5", + "214": "d6", + "215": "d7", + "216": "d8", + "217": "d9", + "218": "da", + "219": "db", + "220": "dc", + "221": "dd", + "222": "de", + "223": "df", + "224": "e0", + "225": "e1", + "226": "e2", + "227": "e3", + "228": "e4", + "229": "e5", + "230": "e6", + "231": "e7", + "232": "e8", + "233": "e9", + "234": "ea", + "235": "eb", + "236": "ec", + "237": "ed", + "238": "ee", + "239": "ef", + "240": "f0", + "241": "f1", + "242": "f2", + "243": "f3", + "244": "f4", + "245": "f5", + "246": "f6", + "247": "f7", + "248": "f8", + "249": "f9", + "250": "fa", + "251": "fb", + "252": "fc", + "253": "fd", + "254": "fe", + "255": "ff", + "256": "3c626f733e", + "257": "3c656f733e", + "258": "6865", + "259": "2074", + "260": "2061", + "261": "2073", + "262": "2077", + "263": "6e64", + "264": "20746865", + "265": "6564", + "266": "20616e64", + "267": "20746f", + "268": "2062", + "269": "696e", + "270": "2068", + "271": "207761", + "272": "7265", + "273": "2066", + "274": "6f75", + "275": "6974", + "276": "2054", + "277": "2063", + "278": "206c", + "279": "6572", + "280": "2064", + "281": "206865", + "282": "206d", + "283": "20776173", + "284": "2070", + "285": "6f6d", + "286": "206f", + "287": "6179", + "288": "20546865", + "289": "6172", + "290": "6973", + "291": "696e67", + "292": "696c", + "293": "2067", + "294": "6964", + "295": "2053", + "296": "6174", + "297": "656e", + "298": "206e", + "299": "207361", + "300": "206861", + "301": "616e", + "302": "696d", + "303": "6f72", + "304": "6f6e", + "305": "206974", + "306": "2048", + "307": "207468", + "308": "6c6c", + "309": "6c65", + "310": "20686572", + "311": "6574", + "312": "6f74", + "313": "20536865", + "314": "6972", + "315": "204865", + "316": "2054686579", + "317": "6573", + "318": "20696e", + "319": "766572", + "320": "7574", + "321": "6f77", + "322": "636b", + "323": "2065", + "324": "2075", + "325": "6c64", + "326": "204c", + "327": "2079", + "328": "6f6f", + "329": "2073616964", + "330": "6967", + "331": "696c79", + "332": "2072", + "333": "206265", + "334": "616d", + "335": "207374", + "336": "2049", + "337": "6365", + "338": "20736865", + "339": "6b65", + "340": "7070", + "341": "7665", + "342": "697468", + "343": "2077697468", + "344": "4f6e", + "345": "206f6e", + "346": "206f66", + "347": "6b6564", + "348": "7269", + "349": "20736f", + "350": "204c696c79", + "351": "20796f75", + "352": "20686973", + "353": "6e74", + "354": "20706c", + "355": "6164", + "356": "76657279", + "357": "2042", + "358": "20646179", + "359": "2074686174", + "360": "207570", + "361": "20686164", + "362": "7374", + "363": "20706c6179", + "364": "2074686579", + "365": "207765", + "366": "206d6f6d", + "367": "20666f72", + "368": "656c", + "369": "6f756c64", + "370": "756e", + "371": "6d79", + "372": "204d", + "373": "2773", + "374": "0d0a", + "375": "206c69", + "376": "6368", + "377": "656e74", + "378": "2068617070", + "379": "697474", + "380": "2077616e74", + "381": "207368", + "382": "6f7574", + "383": "686572", + "384": "6c79", + "385": "6f6d65", + "386": "2076657279", + "387": "6f756e64", + "388": "206e6f74", + "389": "20646f", + "390": "7365", + "391": "616c6c", + "392": "207768", + "393": "656e64", + "394": "696d65", + "395": "6974746c65", + "396": "206b", + "397": "2074696d65", + "398": "206c6974746c65", + "399": "207468657265", + "400": "616c", + "401": "6874", + "402": "206e65", + "403": "207265", + "404": "206861707079", + "405": "204974", + "406": "20626967", + "407": "2041", + "408": "20736d", + "409": "206973", + "410": "20627574", + "411": "61636b", + "412": "20736177", + "413": "20616e", + "414": "7269656e64", + "415": "20667269656e64", + "416": "7261", + "417": "206173", + "418": "7279", + "419": "2e22", + "420": "696465", + "421": "4f6e65", + "422": "616b65", + "423": "2774", + "424": "7567", + "425": "766564", + "426": "206c6f6f", + "427": "746572", + "428": "2077657265", + "429": "2054696d", + "430": "4f6e6365", + "431": "6f7265", + "432": "2122", + "433": "6563", + "434": "206c6f", + "435": "204f6e65", + "436": "2068696d", + "437": "20626f", + "438": "204a", + "439": "20746f6f", + "440": "20676f", + "441": "207365", + "442": "69726c", + "443": "2077616e746564", + "444": "617264", + "445": "206a", + "446": "206769726c", + "447": "2075706f6e", + "448": "696c6c", + "449": "206f7574", + "450": "20617265", + "451": "207468656d", + "452": "6675", + "453": "207370", + "454": "206174", + "455": "776179", + "456": "696e64", + "457": "7572", + "458": "207468656972", + "459": "20636f756c64", + "460": "68656e", + "461": "20646964", + "462": "20736d696c", + "463": "2042656e", + "464": "61696e", + "465": "20546f6d", + "466": "2068617665", + "467": "206578", + "468": "686564", + "469": "617274", + "470": "206172", + "471": "2077656e74", + "472": "726f6d", + "473": "6963", + "474": "2057", + "475": "66756c", + "476": "68696e67", + "477": "2063616e", + "478": "6c70", + "479": "20667269656e6473", + "480": "69676874", + "481": "20616c6c", + "482": "206b6e", + "483": "206e616d", + "484": "20427574", + "485": "6f6f64", + "486": "2068656c70", + "487": "756d", + "488": "3f22", + "489": "6f6e65", + "490": "206261636b", + "491": "61726b", + "492": "2066756e", + "493": "20636c", + "494": "617265", + "495": "206c65", + "496": "2054696d6d79", + "497": "20736179", + "498": "596f75", + "499": "206e6f", + "500": "6f70", + "501": "207363", + "502": "206e616d6564", + "503": "6f7567", + "504": "656c74", + "505": "20616c", + "506": "206c6f766564", + "507": "20736d696c6564", + "508": "73696465", + "509": "69636b", + "510": "2066656c74", + "511": "2061736b6564", + "512": "206d616e", + "513": "206665", + "514": "20746f79", + "515": "2061726f756e64", + "516": "206c6f6f6b6564", + "517": "6f6d6574", + "518": "206c696b65", + "519": "616d65", + "520": "20736f6d65", + "521": "20736565", + "522": "20726f", + "523": "757265", + "524": "206d65", + "525": "206272", + "526": "20626f79", + "527": "20776f756c64", + "528": "676574", + "529": "20626972", + "530": "207374617274", + "531": "696365", + "532": "6f6e67", + "533": "20736f6d6574", + "534": "7373", + "535": "2062697264", + "536": "20776f72", + "537": "206661", + "538": "696e6773", + "539": "616465", + "540": "206167", + "541": "6464", + "542": "6b696e67", + "543": "2077686174", + "544": "c3a2", + "545": "82ac", + "546": "c3a2e2", + "547": "c3a2e282ac", + "548": "206d616b65", + "549": "6f776e", + "550": "20747265", + "551": "2061776179", + "552": "2072616e", + "553": "676574686572", + "554": "6965", + "555": "2073746172746564", + "556": "6167", + "557": "2c22", + "558": "6f7564", + "559": "61726564", + "560": "206d616465", + "561": "20736f6d657468696e67", + "562": "204d6f6d", + "563": "6966", + "564": "20596f75", + "565": "20636f", + "566": "20746f676574686572", + "567": "69746564", + "568": "207061726b", + "569": "74686572", + "570": "20636172", + "571": "2073617973", + "572": "2045", + "573": "20657863", + "574": "20736164", + "575": "20707574", + "576": "206f74686572", + "577": "207072", + "578": "206e6577", + "579": "626c65", + "580": "2066726f6d", + "581": "206c61", + "582": "206d75", + "583": "20616761696e", + "584": "2046", + "585": "20686f6d65", + "586": "20687567", + "587": "656c6c", + "588": "207768656e", + "589": "657374", + "590": "2077686f", + "591": "207468656e", + "592": "616368", + "593": "20646563", + "594": "6f6f6b", + "595": "6570", + "596": "20676f6f64", + "597": "72696564", + "598": "20666f756e64", + "599": "70706564", + "600": "206368", + "601": "20666c", + "602": "6173", + "603": "2077616c", + "604": "20676574", + "605": "7175", + "606": "6f75676874", + "607": "686174", + "608": "20706c6179696e67", + "609": "616c6c79", + "610": "207377", + "611": "706c", + "612": "207468696e6773", + "613": "2065786369746564", + "614": "20676f74", + "615": "6f7573", + "616": "2044", + "617": "75636b", + "618": "69646564", + "619": "205375", + "620": "206c696b6564", + "621": "6e6e", + "622": "2064656369646564", + "623": "726f", + "624": "206576657279", + "625": "617665", + "626": "2063616d65", + "627": "20626c", + "628": "2053616d", + "629": "6178", + "630": "20646f776e", + "631": "206f6e65", + "632": "20736361726564", + "633": "20796f7572", + "634": "2063617265", + "635": "2076", + "636": "757374", + "637": "20626563", + "638": "2066696e64", + "639": "207468616e", + "640": "6f757365", + "641": "697374", + "642": "617261", + "643": "20646164", + "644": "206665656c", + "645": "20646f67", + "646": "6e79", + "647": "206162", + "648": "6170", + "649": "20677261", + "650": "61726e", + "651": "77617973", + "652": "206b65", + "653": "206469646e", + "654": "206772", + "655": "6665", + "656": "657273", + "657": "6e6e61", + "658": "6f7572", + "659": "20616c77617973", + "660": "20746f6f6b", + "661": "616e74", + "662": "657373", + "663": "206d79", + "664": "206c6f74", + "665": "20706f", + "666": "2061626f7574", + "667": "206c6f6f6b", + "668": "206d6f6d6d79", + "669": "204d6178", + "670": "6e646572", + "671": "2077696c6c", + "672": "20536f", + "673": "206b6e65", + "674": "656369", + "675": "2074726565", + "676": "6262", + "677": "206f757473696465", + "678": "206275", + "679": "2062616c6c", + "680": "20746f7973", + "681": "63686564", + "682": "2074686f75676874", + "683": "697365", + "684": "697465", + "685": "205768656e", + "686": "656e6564", + "687": "20696e74", + "688": "207461", + "689": "206d6f7265", + "690": "206c6561726e", + "691": "20686f", + "692": "206b6e6f77", + "693": "656369616c", + "694": "207370656369616c", + "695": "20707265", + "696": "206f6c64", + "697": "676564", + "698": "207075", + "699": "2053617261", + "700": "2074616b65", + "701": "206c617567", + "702": "20416e6e61", + "703": "206b6e6577", + "704": "20486572", + "705": "6464656e", + "706": "207065", + "707": "20736f72", + "708": "697665", + "709": "20616e79", + "710": "6175", + "711": "20686f77", + "712": "6d61", + "713": "66746572", + "714": "20636174", + "715": "207472", + "716": "2073686f77", + "717": "6464656e6c79", + "718": "204a6f", + "719": "6961", + "720": "204576657279", + "721": "206d756368", + "722": "207477", + "723": "20736c", + "724": "204a61636b", + "725": "696e6b", + "726": "2072756e", + "727": "697368", + "728": "616e64", + "729": "20686f757365", + "730": "76656e", + "731": "20736f727279", + "732": "2068616e64", + "733": "20736b", + "734": "206d6f", + "735": "206f70", + "736": "20696e746f", + "737": "20696e73696465", + "738": "2073756e", + "739": "20636c6f", + "740": "20746f6c64", + "741": "206f72", + "742": "207175", + "743": "2070726f7564", + "744": "207472696564", + "745": "20636f756c646e", + "746": "207761746572", + "747": "2067617665", + "748": "68657265", + "749": "617465", + "750": "206e65766572", + "751": "20656e", + "752": "756d70", + "753": "206f766572", + "754": "616e6b", + "755": "6f74686572", + "756": "206578706c", + "757": "206865617264", + "758": "7479", + "759": "206966", + "760": "6279", + "761": "61757365", + "762": "616365", + "763": "20656174", + "764": "20726f6f6d", + "765": "6765", + "766": "205765", + "767": "2074686973", + "768": "2053756464656e6c79", + "769": "6479", + "770": "207069636b", + "771": "696f6e", + "772": "2065616368", + "773": "6574746572", + "774": "20706c61796564", + "775": "205468656e", + "776": "20737472", + "777": "2046726f6d", + "778": "2070726574", + "779": "20687567676564", + "780": "20626f78", + "781": "20677265", + "782": "20636f6d", + "783": "616b", + "784": "20776174", + "785": "206a757374", + "786": "2062656361757365", + "787": "6f7468", + "788": "206f6666", + "789": "206e696365", + "790": "696c65", + "791": "20486973", + "792": "7565", + "793": "206e656564", + "794": "206a6f", + "795": "697a", + "796": "20736d616c6c", + "797": "2062656172", + "798": "206c6f6e67", + "799": "736564", + "800": "204d6961", + "801": "206d616e79", + "802": "20416e64", + "803": "76696e67", + "804": "7563", + "805": "20666f", + "806": "20756e74", + "807": "276d", + "808": "6f756768", + "809": "6f7274", + "810": "20736e", + "811": "20756e74696c", + "812": "20747279", + "813": "696e65", + "814": "696c64", + "815": "73656c", + "816": "696d616c", + "817": "206c6561726e6564", + "818": "20616e696d616c", + "819": "677279", + "820": "20626574746572", + "821": "2043", + "822": "656d", + "823": "757274", + "824": "20696d", + "825": "2063616c6c", + "826": "6674", + "827": "20656e64", + "828": "206c6f7665", + "829": "617574", + "830": "626572", + "831": "20647265", + "832": "206b696e64", + "833": "617273", + "834": "206164", + "835": "20666c6f77", + "836": "616479", + "837": "75726e", + "838": "2062657374", + "839": "20636c65", + "840": "76656e74", + "841": "206265617574", + "842": "207468616e6b6564", + "843": "757270", + "844": "617264656e", + "845": "2067617264656e", + "846": "20627261", + "847": "20736b79", + "848": "206361726566756c", + "849": "7074", + "850": "2066617374", + "851": "206669", + "852": "6c6c79", + "853": "766573", + "854": "206576656e", + "855": "20646f6e", + "856": "596573", + "857": "206375", + "858": "206c617567686564", + "859": "206279", + "860": "73656c66", + "861": "20636865", + "862": "207465", + "863": "206c6f7564", + "864": "206c6f7473", + "865": "20626561757469", + "866": "206a756d70", + "867": "696e79", + "868": "2062656175746966756c", + "869": "c3a2e282acc5", + "870": "c3a2e282acc593", + "871": "6c6577", + "872": "2066616d", + "873": "20776179", + "874": "20676c", + "875": "776e", + "876": "696573", + "877": "207374696c6c", + "878": "617365", + "879": "20636f6d65", + "880": "207261", + "881": "2068617264", + "882": "206c697374", + "883": "20776f", + "884": "5468", + "885": "617374", + "886": "2068757274", + "887": "20616e696d616c73", + "888": "2073746179", + "889": "6374", + "890": "20636f6c", + "891": "656374", + "892": "6e696e67", + "893": "2073616665", + "894": "7368", + "895": "2050", + "896": "20756e646572", + "897": "2072656d", + "898": "686573", + "899": "20626f7468", + "900": "6265", + "901": "206c6574", + "902": "206272617665", + "903": "204173", + "904": "2074776f", + "905": "726565", + "906": "6f6f6c", + "907": "6f706c65", + "908": "616e65", + "909": "206c69766564", + "910": "20696d70", + "911": "20697473", + "912": "20626164", + "913": "2066616d696c79", + "914": "204166746572", + "915": "6973686564", + "916": "4974", + "917": "2066696e", + "918": "756379", + "919": "207375", + "920": "75727072", + "921": "6b6179", + "922": "6f62", + "923": "20666c6577", + "924": "6f636b", + "925": "204a6f68", + "926": "204c756379", + "927": "2063616c6c6564", + "928": "2073746f72", + "929": "2070656f706c65", + "930": "2077616c6b6564", + "931": "207375727072", + "932": "696768", + "933": "2073686f756c64", + "934": "2064616e", + "935": "20726564", + "936": "20666c79", + "937": "206b657074", + "938": "666f7265", + "939": "6c6564", + "940": "7874", + "941": "20626f6f6b", + "942": "20736f6f6e", + "943": "20666f7265", + "944": "20616e677279", + "945": "69736564", + "946": "4c6574", + "947": "4e6f", + "948": "20706963", + "949": "20646f6c6c", + "950": "20726f636b", + "951": "20636c65616e", + "952": "207368617265", + "953": "6970", + "954": "20676f696e67", + "955": "64646564", + "956": "206578706c6f7265", + "957": "20636f6e", + "958": "756d6d79", + "959": "206e6f77", + "960": "c3a2e282ace2", + "961": "20647261", + "962": "20707265747479", + "963": "20616476656e74", + "964": "2077696e64", + "965": "616765", + "966": "206f70656e6564", + "967": "206b656570", + "968": "207368696e79", + "969": "206265666f7265", + "970": "20646f6f72", + "971": "696c6c79", + "972": "2047", + "973": "2067726f756e64", + "974": "2073746f", + "975": "207265616c", + "976": "20637279", + "977": "20637265", + "978": "206e6f697365", + "979": "617279", + "980": "20696465", + "981": "206672", + "982": "20666172", + "983": "207475726e", + "984": "20446164", + "985": "20636f6c6f72", + "986": "696564", + "987": "206579", + "988": "204d6f6d6d79", + "989": "6c6573", + "990": "736f", + "991": "20776172", + "992": "206665656c696e67", + "993": "207069636b6564", + "994": "20616c736f", + "995": "696d62", + "996": "2069646561", + "997": "206e657874", + "998": "7468696e67", + "999": "204a6f686e", + "1000": "20756e", + "1001": "2077616c6b696e67", + "1002": "84a2", + "1003": "c3a2e282ace284a2", + "1004": "20636c696d62", + "1005": "57686174", + "1006": "207768696c65", + "1007": "20746872", + "1008": "6f79", + "1009": "20736d696c65", + "1010": "7468", + "1011": "20626564", + "1012": "6666", + "1013": "206e6f64646564", + "1014": "20426f62", + "1015": "69636564", + "1016": "74757265", + "1017": "656574", + "1018": "207772", + "1019": "206d61", + "1020": "20616476656e74757265", + "1021": "676572", + "1022": "20686173", + "1023": "6163", + "1024": "206265696e67", + "1025": "696f7573", + "1026": "61796265", + "1027": "746564", + "1028": "20636c6f73", + "1029": "20646966", + "1030": "206c6f6f6b696e67", + "1031": "656d626572", + "1032": "6a6f79", + "1033": "204e", + "1034": "20747275636b", + "1035": "74696e67", + "1036": "2067697665", + "1037": "2073746f70706564", + "1038": "2070696374757265", + "1039": "54686174", + "1040": "205370", + "1041": "2077616974", + "1042": "20666f6f64", + "1043": "5468616e6b", + "1044": "2065796573", + "1045": "75676874", + "1046": "6b6573", + "1047": "2052", + "1048": "206e6f7469636564", + "1049": "2064656c", + "1050": "207468696e6b", + "1051": "20746f77", + "1052": "206c697374656e", + "1053": "61626c65", + "1054": "6d617a", + "1055": "2068616e6473", + "1056": "2072656d656d626572", + "1057": "2077616c6b", + "1058": "20717569636b", + "1059": "20627265", + "1060": "20616d617a", + "1061": "206772656174", + "1062": "2065766572", + "1063": "206166746572", + "1064": "2062726f", + "1065": "20626c7565", + "1066": "20766f", + "1067": "206f776e", + "1068": "697a6564", + "1069": "20656e6a6f79", + "1070": "20736c696465", + "1071": "207265706c", + "1072": "20646973", + "1073": "20617070", + "1074": "69727374", + "1075": "20666f72657374", + "1076": "62626974", + "1077": "20696d706f7274", + "1078": "6f6f6e", + "1079": "20776f726b", + "1080": "20666c6f77657273", + "1081": "6c6c6f77", + "1082": "20776f6e646572", + "1083": "207261696e", + "1084": "2073686f776564", + "1085": "2065766572796f6e65", + "1086": "206e656172", + "1087": "2073686f7574", + "1088": "67616e", + "1089": "2066697368", + "1090": "206163", + "1091": "207374726f6e67", + "1092": "2068656164", + "1093": "20696d706f7274616e74", + "1094": "207761746368", + "1095": "20726162626974", + "1096": "62626564", + "1097": "206d65616e", + "1098": "4c696c79", + "1099": "66657265", + "1100": "20717569636b6c79", + "1101": "2074616c", + "1102": "207573", + "1103": "20627269676874", + "1104": "20626f6174", + "1105": "2064696666657265", + "1106": "206f6b6179", + "1107": "206465", + "1108": "627965", + "1109": "20736f756e64", + "1110": "207269676874", + "1111": "2077617463686564", + "1112": "207265706c696564", + "1113": "646179", + "1114": "20746f75", + "1115": "20537565", + "1116": "20737469636b", + "1117": "20666f6c6c6f77", + "1118": "6c65617365", + "1119": "20757365", + "1120": "65726564", + "1121": "207768657265", + "1122": "20706c616365", + "1123": "2061736b", + "1124": "206472657373", + "1125": "2073757265", + "1126": "207965", + "1127": "207275", + "1128": "616e6765", + "1129": "206d6f72", + "1130": "2073746f7265", + "1131": "617468", + "1132": "5765", + "1133": "2079656c6c", + "1134": "204a616e65", + "1135": "206265656e", + "1136": "2063757269", + "1137": "20646966666572656e74", + "1138": "2053706f74", + "1139": "2066616365", + "1140": "20626563616d65", + "1141": "20737175", + "1142": "20766f696365", + "1143": "20636c6f736572", + "1144": "205361726168", + "1145": "206368696c64", + "1146": "20747261", + "1147": "636f", + "1148": "2068696768", + "1149": "20676f6f64627965", + "1150": "206f7572", + "1151": "6e67", + "1152": "20686f70", + "1153": "6e646d61", + "1154": "206372", + "1155": "6175676874", + "1156": "207761726d", + "1157": "6f756e74", + "1158": "206d6167", + "1159": "20637572696f7573", + "1160": "20646f6573", + "1161": "205468", + "1162": "206e69676874", + "1163": "6c656570", + "1164": "6163686564", + "1165": "4c6f6f6b", + "1166": "2074656c6c", + "1167": "20666f7267", + "1168": "2062656c", + "1169": "696c61", + "1170": "20636f6f6b", + "1171": "207061696e", + "1172": "6d6f7265", + "1173": "20616e796d6f7265", + "1174": "207468726565", + "1175": "2063616b65", + "1176": "2045766572796f6e65", + "1177": "6579", + "1178": "756c6c", + "1179": "69726564", + "1180": "20616d", + "1181": "206f70656e", + "1182": "696d6573", + "1183": "206669727374", + "1184": "697463", + "1185": "617373", + "1186": "2067726162626564", + "1187": "206c6561", + "1188": "20666176", + "1189": "20636f6e74", + "1190": "20627574746572", + "1191": "2070756c6c", + "1192": "20626c6f636b", + "1193": "207377656574", + "1194": "206f6e6c79", + "1195": "207a", + "1196": "20626974", + "1197": "20496e", + "1198": "206269726473", + "1199": "2068657265", + "1200": "2066697265", + "1201": "2068656c706564", + "1202": "206b6964", + "1203": "2066656c6c", + "1204": "6f726e", + "1205": "2068617070656e6564", + "1206": "697373", + "1207": "756e6e79", + "1208": "43616e", + "1209": "206261", + "1210": "206a756d706564", + "1211": "206472", + "1212": "20706572", + "1213": "69746368656e", + "1214": "206772617373", + "1215": "206b69746368656e", + "1216": "697379", + "1217": "206d6f6e", + "1218": "2072656163686564", + "1219": "20666c6f", + "1220": "2073616e64", + "1221": "20676c6164", + "1222": "2073656573", + "1223": "2073746f7279", + "1224": "2062726f74686572", + "1225": "207265616c697a6564", + "1226": "2073696e67", + "1227": "707079", + "1228": "2054686174", + "1229": "2073746f70", + "1230": "20686176696e67", + "1231": "20626568", + "1232": "6e7473", + "1233": "4d6f6d", + "1234": "207468726f756768", + "1235": "2072657374", + "1236": "207072696e", + "1237": "207265616479", + "1238": "206c657373", + "1239": "7a79", + "1240": "61746572", + "1241": "7768657265", + "1242": "20626567616e", + "1243": "206d756d", + "1244": "616972", + "1245": "2073686f75746564", + "1246": "206c616479", + "1247": "2079756d6d79", + "1248": "20736f6674", + "1249": "20776f726c64", + "1250": "2067616d65", + "1251": "20446f", + "1252": "20747269", + "1253": "20617465", + "1254": "696e73", + "1255": "204c6574", + "1256": "206d657373", + "1257": "2070617265", + "1258": "204c696c61", + "1259": "2068696d73656c66", + "1260": "207265616c6c79", + "1261": "2064726177", + "1262": "204a6f65", + "1263": "66756c6c79", + "1264": "20756e6465727374", + "1265": "277265", + "1266": "6174686572", + "1267": "206475636b", + "1268": "20637574", + "1269": "20646f6e65", + "1270": "20666978", + "1271": "20686174", + "1272": "6b656e", + "1273": "207469726564", + "1274": "206d616b696e67", + "1275": "616b6573", + "1276": "20706170", + "1277": "206d61676963", + "1278": "2075736564", + "1279": "73746572", + "1280": "207365656e", + "1281": "207468696e67", + "1282": "206b696473", + "1283": "20636f6d696e67", + "1284": "2066696c6c", + "1285": "206661766f72", + "1286": "20746f70", + "1287": "2070726574656e64", + "1288": "206661766f72697465", + "1289": "2046696e", + "1290": "69676e", + "1291": "20776f727279", + "1292": "207761766564", + "1293": "6f6c6c79", + "1294": "2074617374", + "1295": "2066756e6e79", + "1296": "20776879", + "1297": "20706f6e64", + "1298": "2062757474657266", + "1299": "2070726f6d", + "1300": "626f", + "1301": "206361746368", + "1302": "576f77", + "1303": "20737465", + "1304": "69656e74", + "1305": "206e616d65", + "1306": "206c6f6f6b73", + "1307": "65646479", + "1308": "2043616e", + "1309": "20666c6f776572", + "1310": "20706172656e7473", + "1311": "20616e6f74686572", + "1312": "2068616972", + "1313": "20746f756368", + "1314": "2066726f", + "1315": "2070617274", + "1316": "2062756e6e79", + "1317": "2053616c6c79", + "1318": "206372696564", + "1319": "207377696d", + "1320": "2064656c6963", + "1321": "756666", + "1322": "6d6572", + "1323": "2063617273", + "1324": "61697379", + "1325": "206c6573736f6e", + "1326": "2064726f", + "1327": "207375727072697365", + "1328": "206c69676874", + "1329": "20636f6f6c", + "1330": "2064696e", + "1331": "2066756c6c", + "1332": "446f6e", + "1333": "2073747265", + "1334": "20626167", + "1335": "61726473", + "1336": "206769", + "1337": "20776f726b6564", + "1338": "206d6f757365", + "1339": "69766572", + "1340": "6174696f6e", + "1341": "7573", + "1342": "72656c", + "1343": "6879", + "1344": "207472656573", + "1345": "20686f70706564", + "1346": "2072656d656d6265726564", + "1347": "20626568696e64", + "1348": "6f6e6579", + "1349": "20686f6c64", + "1350": "546f6d", + "1351": "20626c6f636b73", + "1352": "20677265656e", + "1353": "20647269", + "1354": "207965617273", + "1355": "2073706f74", + "1356": "207468616e6b", + "1357": "20736c656570", + "1358": "20546f6d6d79", + "1359": "20686f6c65", + "1360": "2073686f", + "1361": "20736d656c6c", + "1362": "20646f696e67", + "1363": "2072756e6e696e67", + "1364": "20736574", + "1365": "2079656c6c6f77", + "1366": "756564", + "1367": "20666c6f6f72", + "1368": "20776f6f64", + "1369": "616e6479", + "1370": "20776973686564", + "1371": "20637261", + "1372": "656c79", + "1373": "207061706572", + "1374": "20736c6f77", + "1375": "206461726b", + "1376": "206361726566756c6c79", + "1377": "696b65", + "1378": "2077696e646f77", + "1379": "222e", + "1380": "2068617070696c79", + "1381": "20636f6c64", + "1382": "206e6565646564", + "1383": "2042656e6e79", + "1384": "20736174", + "1385": "2074616c6c", + "1386": "2072656164", + "1387": "756c", + "1388": "20647265616d", + "1389": "2068656c64", + "1390": "6172696e67", + "1391": "20737572707269736564", + "1392": "207377696e67", + "1393": "204174", + "1394": "72656e", + "1395": "206c61756768", + "1396": "2066696e6973686564", + "1397": "20456d", + "1398": "206c6f7374", + "1399": "20626f77", + "1400": "2070657266", + "1401": "2066726f67", + "1402": "206d6f7665", + "1403": "206c696f6e", + "1404": "206c656674", + "1405": "20627574746572666c79", + "1406": "20656e6a6f796564", + "1407": "204461697379", + "1408": "2068696c6c", + "1409": "2077726f6e67", + "1410": "20666f6c6c6f776564", + "1411": "204b", + "1412": "2063617374", + "1413": "20616e797468696e67", + "1414": "2061727269", + "1415": "20627579", + "1416": "2046696e616c6c79", + "1417": "20616972", + "1418": "20696365", + "1419": "2072656c", + "1420": "20736e6f77", + "1421": "696e6564", + "1422": "206c696b6573", + "1423": "2066616972", + "1424": "72656564", + "1425": "206275696c64", + "1426": "2068656172", + "1427": "204e6f77", + "1428": "2068657273656c66", + "1429": "656e73", + "1430": "20546f", + "1431": "20686964", + "1432": "69656c64", + "1433": "20636c6f7365", + "1434": "63657373", + "1435": "20636f6e74696e", + "1436": "726f7373", + "1437": "2064656c6963696f7573", + "1438": "206368696c6472656e", + "1439": "20646972", + "1440": "2068756e", + "1441": "20667269656e646c79", + "1442": "7379", + "1443": "2042696c6c79", + "1444": "2061727269766564", + "1445": "48656c6c", + "1446": "48656c6c6f", + "1447": "20776f6e64657266756c", + "1448": "20656c", + "1449": "6173757265", + "1450": "2061726d", + "1451": "206d69676874", + "1452": "20756e6465727374616e64", + "1453": "20666f72676f74", + "1454": "207461626c65", + "1455": "2062616279", + "1456": "20686f74", + "1457": "20676f6e65", + "1458": "2061626c65", + "1459": "207469676874", + "1460": "20636f6c6f7273", + "1461": "204a696d", + "1462": "20706c65617365", + "1463": "20627567", + "1464": "204a656e", + "1465": "6e6572", + "1466": "6473", + "1467": "206372656174", + "1468": "205468657265", + "1469": "20736f6e67", + "1470": "2066696c6c6564", + "1471": "20477261", + "1472": "6c66", + "1473": "21c3a2e282ac", + "1474": "20626f6f6b73", + "1475": "20656c7365", + "1476": "20736f6d656f6e65", + "1477": "206d6174", + "1478": "2073637265", + "1479": "206c6561766573", + "1480": "2062726f6b656e", + "1481": "2067656e74", + "1482": "20636f6e74696e756564", + "1483": "61736b", + "1484": "20636f756e74", + "1485": "2054686973", + "1486": "676f6e", + "1487": "206772616e646d61", + "1488": "20616772656564", + "1489": "20706174", + "1490": "7370", + "1491": "20796573", + "1492": "696368", + "1493": "2077616c6c", + "1494": "20636173746c65", + "1495": "6176", + "1496": "204d6f6c6c79", + "1497": "20627265616b", + "1498": "20737461796564", + "1499": "276c6c", + "1500": "2ec3a2e282ac", + "1501": "726f6f6d", + "1502": "2077616e7473", + "1503": "73686564", + "1504": "206775", + "1505": "20656e6f756768", + "1506": "6d616e", + "1507": "20616d617a696e67", + "1508": "2067657474696e67", + "1509": "2070696e6b", + "1510": "20706f696e", + "1511": "20766973", + "1512": "617070", + "1513": "206d6f726e696e67", + "1514": "726f77", + "1515": "72696573", + "1516": "617368", + "1517": "697272656c", + "1518": "20416d79", + "1519": "206d6f6d656e74", + "1520": "4f4b", + "1521": "2066696e616c6c79", + "1522": "206d6172", + "1523": "206d7573", + "1524": "206d6f74686572", + "1525": "207363617279", + "1526": "6f7570", + "1527": "207768697465", + "1528": "20636f6d70", + "1529": "2064696e6e6572", + "1530": "20747279696e67", + "1531": "696379", + "1532": "206a75", + "1533": "207069", + "1534": "20666577", + "1535": "20667572", + "1536": "204d61796265", + "1537": "616d61", + "1538": "20737175697272656c", + "1539": "657265", + "1540": "206d6574", + "1541": "75666679", + "1542": "20636c696d626564", + "1543": "2066616c6c", + "1544": "20736f6d6574696d6573", + "1545": "726564", + "1546": "2062616c6c6f6f6e", + "1547": "656e6365", + "1548": "616e6365", + "1549": "2070657266656374", + "1550": "636564", + "1551": "207069637475726573", + "1552": "206669676874", + "1553": "6170706564", + "1554": "207475726e6564", + "1555": "20726f6c6c", + "1556": "20737472616e6765", + "1557": "6f6e73", + "1558": "2068756e677279", + "1559": "2062726f75676874", + "1560": "20776574", + "1561": "20636f6f6b696573", + "1562": "204a696c6c", + "1563": "6f6f6d", + "1564": "207361696c", + "1565": "206265616368", + "1566": "6f72696e67", + "1567": "726f74", + "1568": "2073706c", + "1569": "6c63", + "1570": "206a6f79", + "1571": "206f7468657273", + "1572": "206461646479", + "1573": "20736d696c6573", + "1574": "20636c6f74", + "1575": "206c697374656e6564", + "1576": "207761736e", + "1577": "2057686174", + "1578": "2074616c6b", + "1579": "20636c6f74686573", + "1580": "207075707079", + "1581": "61746564", + "1582": "20706f6c", + "1583": "646572", + "1584": "6565", + "1585": "2065766572797468696e67", + "1586": "206d616b6573", + "1587": "20637265616d", + "1588": "2073656172", + "1589": "616d70", + "1590": "20737061726b", + "1591": "65766572", + "1592": "546865", + "1593": "2072696465", + "1594": "20776f6f6473", + "1595": "776c", + "1596": "207461696c", + "1597": "20616d617a6564", + "1598": "206d75736963", + "1599": "63746f72", + "1600": "20776169746564", + "1601": "6973746572", + "1602": "20536f6f6e", + "1603": "20736974", + "1604": "20616c6f6e65", + "1605": "7a7a", + "1606": "6f776572", + "1607": "2074687265", + "1608": "6f73", + "1609": "65746572", + "1610": "2064616e676572", + "1611": "6f7474", + "1612": "75636b79", + "1613": "20416c6c", + "1614": "2073696c6c79", + "1615": "2064656570", + "1616": "20706574", + "1617": "616d6564", + "1618": "20706965", + "1619": "4f68", + "1620": "207669736974", + "1621": "206c616b65", + "1622": "6d6564", + "1623": "207465646479", + "1624": "2070756c6c6564", + "1625": "206c65617665", + "1626": "6b6579", + "1627": "636b6574", + "1628": "2067726f77", + "1629": "20776f6e", + "1630": "2068696465", + "1631": "6d65", + "1632": "6f7065", + "1633": "756767", + "1634": "616e67", + "1635": "73736564", + "1636": "20666c79696e67", + "1637": "54686973", + "1638": "20466c", + "1639": "2061736b73", + "1640": "20746f7761726473", + "1641": "206669656c64", + "1642": "20636c6f7564", + "1643": "2073696465", + "1644": "206c616e64", + "1645": "616d6573", + "1646": "20706c616e", + "1647": "6978", + "1648": "207472656174", + "1649": "20666f78", + "1650": "2070726f6d69736564", + "1651": "61696e6564", + "1652": "427574", + "1653": "20626967676572", + "1654": "206272616e", + "1655": "65636573", + "1656": "20736563", + "1657": "207061696e74", + "1658": "696767", + "1659": "746879", + "1660": "20706965636573", + "1661": "69736573", + "1662": "636c65", + "1663": "207472656173757265", + "1664": "20736973746572", + "1665": "6773", + "1666": "20776f7272696564", + "1667": "2072756e73", + "1668": "206c6976696e67", + "1669": "20736c6f776c79", + "1670": "20626c616e6b", + "1671": "4d61796265", + "1672": "20656d", + "1673": "206d6f72616c", + "1674": "2062656c6965", + "1675": "20736861726564", + "1676": "696574", + "1677": "2070656e", + "1678": "207468696e6b73", + "1679": "2073686170", + "1680": "4f6b6179", + "1681": "207265616368", + "1682": "206772617465", + "1683": "75616c6c79", + "1684": "206a6f62", + "1685": "20646f6c6c73", + "1686": "20647261676f6e", + "1687": "20626c616e6b6574", + "1688": "7377", + "1689": "206d6f6e6579", + "1690": "2064616e6365", + "1691": "2077656172", + "1692": "20666173746572", + "1693": "666f7274", + "1694": "617679", + "1695": "6c696e67", + "1696": "2072616365", + "1697": "206f776e6572", + "1698": "20616e7377", + "1699": "207061727479", + "1700": "20706c616e74", + "1701": "2073756368", + "1702": "20637570", + "1703": "7068", + "1704": "756e6368", + "1705": "20706f74", + "1706": "6d696e67", + "1707": "207175696574", + "1708": "20626f7474", + "1709": "20677261746566756c", + "1710": "20736e61636b", + "1711": "207665", + "1712": "2065766572797768657265", + "1713": "726574", + "1714": "20636f6c6f7266756c", + "1715": "6f6c", + "1716": "756974", + "1717": "61736564", + "1718": "2077656c63", + "1719": "206d757374", + "1720": "206d6f7374", + "1721": "206d61676963616c", + "1722": "6c657373", + "1723": "697265", + "1724": "206a6f696e", + "1725": "206d6f6e73746572", + "1726": "4d6f6d6d79", + "1727": "20656c65", + "1728": "2063616e6479", + "1729": "20626172", + "1730": "2063686f", + "1731": "6f757468", + "1732": "736574", + "1733": "206d6f757468", + "1734": "206e6170", + "1735": "2070726f", + "1736": "20676966", + "1737": "207072696e63657373", + "1738": "204461646479", + "1739": "20626c61636b", + "1740": "7a65", + "1741": "2062726f6b65", + "1742": "206661726d6572", + "1743": "2062696b65", + "1744": "2073746570706564", + "1745": "656172", + "1746": "204a757374", + "1747": "207269766572", + "1748": "6970706564", + "1749": "6f726d", + "1750": "2064616e6765726f7573", + "1751": "204d616d61", + "1752": "20637279696e67", + "1753": "2068656c70696e67", + "1754": "204d72", + "1755": "20686967", + "1756": "206163726f7373", + "1757": "207061", + "1758": "206a65", + "1759": "2073616d65", + "1760": "20646973617070", + "1761": "62626c65", + "1762": "207369676e", + "1763": "20617474", + "1764": "20736d617274", + "1765": "20646f63746f72", + "1766": "6964656e74", + "1767": "20616363", + "1768": "53757265", + "1769": "206d6978", + "1770": "6f726564", + "1771": "20737475", + "1772": "206865617679", + "1773": "207477696e73", + "1774": "206672756974", + "1775": "2074656163", + "1776": "2062757474", + "1777": "6561726564", + "1778": "204d617279", + "1779": "2077696e", + "1780": "20776f756c646e", + "1781": "206469727479", + "1782": "20636172726f74", + "1783": "20626565", + "1784": "20776f726473", + "1785": "20737475636b", + "1786": "20746f776572", + "1787": "576879", + "1788": "20636c6f736564", + "1789": "20626f776c", + "1790": "20737072", + "1791": "207468726577", + "1792": "20456d6d61", + "1793": "61736b6574", + "1794": "2070617373", + "1795": "2062656c69657665", + "1796": "20656174696e67", + "1797": "6974696e67", + "1798": "2062617468", + "1799": "206865616c", + "1800": "20747261696e", + "1801": "206261736b6574", + "1802": "20686967686572", + "1803": "20636f6c6c", + "1804": "206275636b", + "1805": "63686f6f6c", + "1806": "20746f776e", + "1807": "2062697465", + "1808": "206865617274", + "1809": "6272", + "1810": "20686f706564", + "1811": "20686974", + "1812": "20636c6170706564", + "1813": "2070617374", + "1814": "207061636b", + "1815": "204772616e646d61", + "1816": "4869", + "1817": "204a696d6d79", + "1818": "206d6f6e6b6579", + "1819": "2074656163686572", + "1820": "207363686f6f6c", + "1821": "20776974686f7574", + "1822": "2063686565726564", + "1823": "6f7264", + "1824": "20746f646179", + "1825": "2064726f7665", + "1826": "206665656c73", + "1827": "636b6564", + "1828": "6465", + "1829": "2070657273", + "1830": "20707573686564", + "1831": "20636f7572", + "1832": "76656e7475616c6c79", + "1833": "2063726179", + "1834": "20706f636b6574", + "1835": "207374616e64", + "1836": "756d62", + "1837": "206e65696768", + "1838": "2073756464656e6c79", + "1839": "206163636964656e74", + "1840": "7070696e67", + "1841": "206661697279", + "1842": "20466c75666679", + "1843": "20666f72676574", + "1844": "20736d6f", + "1845": "20496e73696465", + "1846": "6f72726f77", + "1847": "206d696e", + "1848": "2062726f776e", + "1849": "206c6f6e676572", + "1850": "6e6579", + "1851": "72616374", + "1852": "206e6569676862", + "1853": "2079617264", + "1854": "696572", + "1855": "205768", + "1856": "2066656e6365", + "1857": "2067656e746c65", + "1858": "6f7273", + "1859": "2077696e6773", + "1860": "20636f75727365", + "1861": "204d756d", + "1862": "2066726565", + "1863": "206578706c61696e6564", + "1864": "42656e", + "1865": "6379", + "1866": "2068617070656e", + "1867": "6f756e7461696e", + "1868": "206c6f6e", + "1869": "206368616972", + "1870": "20636f726e", + "1871": "206a6f7572", + "1872": "20647279", + "1873": "207377696e6773", + "1874": "206e6f69736573", + "1875": "6973736564", + "1876": "2077656c636f6d65", + "1877": "656164", + "1878": "206c6f6e656c79", + "1879": "204f6e", + "1880": "2073616e67", + "1881": "7473", + "1882": "207475726e73", + "1883": "20736f7570", + "1884": "20616476656e7475726573", + "1885": "3fc3a2e282ac", + "1886": "20616c6f6e67", + "1887": "2067616d6573", + "1888": "7068616e74", + "1889": "2074726f75", + "1890": "20736563726574", + "1891": "207570736574", + "1892": "20696e7374", + "1893": "20706c6163", + "1894": "206c617465", + "1895": "76656c", + "1896": "69626c65", + "1897": "20726f636b73", + "1898": "2073686f6f6b", + "1899": "206272696e67", + "1900": "207069656365", + "1901": "207468726f77", + "1902": "6d656e74", + "1903": "2064726f70706564", + "1904": "79617264", + "1905": "6675736564", + "1906": "2062656e", + "1907": "20656c657068616e74", + "1908": "2073686f70", + "1909": "2074616b696e67", + "1910": "206261636b79617264", + "1911": "206170706c65", + "1912": "206578706c6f72696e67", + "1913": "207377616d", + "1914": "416e6e61", + "1915": "20657870", + "1916": "206472696e6b", + "1917": "206f776c", + "1918": "697474696e67", + "1919": "205468656972", + "1920": "63686573", + "1921": "207374617273", + "1922": "20736561", + "1923": "2068756773", + "1924": "2073686f6573", + "1925": "2073746f72696573", + "1926": "61696c", + "1927": "206d6f766564", + "1928": "205477", + "1929": "2073696e67696e67", + "1930": "206e65636b", + "1931": "20796f756e67", + "1932": "2062656c6f6e67", + "1933": "2064697374", + "1934": "20776865", + "1935": "20636175676874", + "1936": "2073746172", + "1937": "636573", + "1938": "206c756e6368", + "1939": "206275636b6574", + "1940": "206a6172", + "1941": "77617264", + "1942": "2074696e79", + "1943": "2072656c6965", + "1944": "207365656d6564", + "1945": "206e6175676874", + "1946": "2077697365", + "1947": "6f636b6564", + "1948": "20636f75", + "1949": "206e617567687479", + "1950": "2077697368", + "1951": "626f77", + "1952": "656b", + "1953": "206d61696c", + "1954": "696f6e73", + "1955": "2077656c6c", + "1956": "626279", + "1957": "206e6561726279", + "1958": "206166", + "1959": "206573", + "1960": "6c6c6f", + "1961": "617065", + "1962": "204c61746572", + "1963": "2053616d6d79", + "1964": "6f756e", + "1965": "206f72616e6765", + "1966": "20646573", + "1967": "206b6579", + "1968": "20776f6b65", + "1969": "206361", + "1970": "2074686f756768", + "1971": "756e67", + "1972": "20737472656574", + "1973": "207068", + "1974": "206d697373", + "1975": "206a75696365", + "1976": "206272616e6368", + "1977": "204f", + "1978": "20696e76", + "1979": "636f6c", + "1980": "70706572", + "1981": "696e696e67", + "1982": "20736861706573", + "1983": "2033", + "1984": "20627573", + "1985": "206e6f7365", + "1986": "2062757379", + "1987": "72616964", + "1988": "205468616e6b", + "1989": "206e657374", + "1990": "20616672616964", + "1991": "7665726564", + "1992": "20746f6d", + "1993": "207261696e626f77", + "1994": "2072657370", + "1995": "206f62", + "1996": "697479", + "1997": "61756768746572", + "1998": "207369636b", + "1999": "446f", + "2000": "20626174", + "2001": "2061726d73", + "2002": "2070757270", + "2003": "63696e67", + "2004": "20706967", + "2005": "61646f77", + "2006": "20636972", + "2007": "2079656172", + "2008": "20776f6e6465726564", + "2009": "20666f7265766572", + "2010": "636572", + "2011": "206e6f7468696e67", + "2012": "2072656c6965766564", + "2013": "746f", + "2014": "206b697465", + "2015": "20726f756e64", + "2016": "206361727279", + "2017": "686f", + "2018": "20416c", + "2019": "206661726d", + "2020": "206d6164", + "2021": "2068756765", + "2022": "2062656e6368", + "2023": "207768697374", + "2024": "69646765", + "2025": "2073697474696e67", + "2026": "2074616b6573", + "2027": "206b696e67", + "2028": "2063757465", + "2029": "6f757273", + "2030": "205477656574", + "2031": "206461756768746572", + "2032": "7374696e67", + "2033": "207377696d6d696e67", + "2034": "20707572706c65", + "2035": "2068656c6c6f", + "2036": "206d61726b", + "2037": "7468646179", + "2038": "20647269766572", + "2039": "207065616365", + "2040": "206d696e65", + "2041": "20736f756e6473", + "2042": "6c6574", + "2043": "206578636974696e67", + "2044": "20636f77", + "2045": "20636f6d6573", + "2046": "6f7365", + "2047": "2070617469656e74", + "2048": "20696e74657265", + "2049": "69746368", + "2050": "206567", + "2051": "206a6f75726e6579", + "2052": "697474656e73", + "2053": "20746f7563686564", + "2054": "20666974", + "2055": "2073706f6f6e", + "2056": "206c696674", + "2057": "206c696665", + "2058": "2067696674", + "2059": "20636f6174", + "2060": "20646973636f", + "2061": "206d696c", + "2062": "657365", + "2063": "7368696e65", + "2064": "206c61756768696e67", + "2065": "2057697468", + "2066": "20686f6c64696e67", + "2067": "2062616e64", + "2068": "206f6e746f", + "2069": "697274", + "2070": "2064617973", + "2071": "206c6574746572", + "2072": "20696d6167", + "2073": "66697368", + "2074": "206372656174757265", + "2075": "20777261", + "2076": "626f617264", + "2077": "206163636964656e74616c6c79", + "2078": "20626f756e", + "2079": "2066696e67", + "2080": "20636f7a79", + "2081": "53617261", + "2082": "20616374", + "2083": "2073756e7368696e65", + "2084": "20726574", + "2085": "20636f6d666f7274", + "2086": "20696e63", + "2087": "54696d", + "2088": "706564", + "2089": "697474656e", + "2090": "6e657373", + "2091": "2067696767", + "2092": "20706f776572", + "2093": "2070726f6d697365", + "2094": "2066616e", + "2095": "206269727468646179", + "2096": "206d697374", + "2097": "2073756e6e79", + "2098": "2072657475726e", + "2099": "2063617665", + "2100": "2070726573", + "2101": "20696e7374656164", + "2102": "206b69636b", + "2103": "206272656164", + "2104": "486579", + "2105": "206365", + "2106": "4f66", + "2107": "20656172", + "2108": "62616c6c", + "2109": "20706f696e746564", + "2110": "20616e796f6e65", + "2111": "207368656c6c", + "2112": "67676564", + "2113": "2067726162", + "2114": "20686f7065", + "2115": "616c6d", + "2116": "20436f6d65", + "2117": "206c75636b79", + "2118": "20636f6c6c656374", + "2119": "206c61746572", + "2120": "207461737479", + "2121": "20686f72", + "2122": "6f67", + "2123": "726f756e64", + "2124": "207365617263686564", + "2125": "207761746368696e67", + "2126": "657468", + "2127": "2073746f726d", + "2128": "20736d696c696e67", + "2129": "20627574746f6e", + "2130": "20626f75676874", + "2131": "20676174686572", + "2132": "7363", + "2133": "6f7665", + "2134": "626c656d", + "2135": "697a65", + "2136": "2070756464", + "2137": "2070726f626c656d", + "2138": "617270", + "2139": "206d696c6b", + "2140": "20636865657365", + "2141": "20656173", + "2142": "20696e746572657374696e67", + "2143": "20726f7065", + "2144": "74656e", + "2145": "2074726f75626c65", + "2146": "207368617270", + "2147": "2073706964", + "2148": "204a656e6e79", + "2149": "207368696e696e67", + "2150": "206769766573", + "2151": "617572", + "2152": "2073706f", + "2153": "20456c6c", + "2154": "20656e74", + "2155": "204368", + "2156": "2061646d", + "2157": "206475636b73", + "2158": "2074616c6b6564", + "2159": "2062757368", + "2160": "2070726573656e74", + "2161": "20706572736f6e", + "2162": "2073616e6477", + "2163": "20626f74746c65", + "2164": "207370696e", + "2165": "207175657374", + "2166": "756e74", + "2167": "2077696465", + "2168": "20736e6f776d616e", + "2169": "656c73", + "2170": "20536f6d6574", + "2171": "206170", + "2172": "7562", + "2173": "20706c616e65", + "2174": "207a6f6f6d", + "2175": "206d656574", + "2176": "20736d656c6c6564", + "2177": "20737563", + "2178": "206c6f76656c79", + "2179": "6f6e74", + "2180": "20726f6164", + "2181": "2070617468", + "2182": "206d6174746572", + "2183": "207072616374", + "2184": "206b697373", + "2185": "697361", + "2186": "20636c65766572", + "2187": "206170706c6573", + "2188": "6f6d616e", + "2189": "756d626572", + "2190": "6570656e64", + "2191": "20636c6f756473", + "2192": "20456d696c79", + "2193": "20776f6d616e", + "2194": "65656e", + "2195": "2063617264", + "2196": "20726f6c6c6564", + "2197": "206865616c746879", + "2198": "20736166", + "2199": "2070757473", + "2200": "2064616e636564", + "2201": "2073706f74746564", + "2202": "6f6e64", + "2203": "2066726f6e74", + "2204": "20686f6e6579", + "2205": "20737572", + "2206": "206769616e74", + "2207": "20706172", + "2208": "2070696c65", + "2209": "6f776564", + "2210": "207361636b", + "2211": "2076696c6c", + "2212": "2070726f74", + "2213": "2074756e", + "2214": "206368656572", + "2215": "476f6f64", + "2216": "20736872", + "2217": "20486f77", + "2218": "20637261796f6e73", + "2219": "6562", + "2220": "20686f6e", + "2221": "2070696c6c", + "2222": "206d7973746572", + "2223": "206772616e64", + "2224": "207768696368", + "2225": "2070757a7a", + "2226": "204d697474656e73", + "2227": "2066656174686572", + "2228": "7061", + "2229": "2070726f74656374", + "2230": "206c6179", + "2231": "20686f727365", + "2232": "6368696e67", + "2233": "206c6f766573", + "2234": "20706c61636573", + "2235": "207a6f6f", + "2236": "205374", + "2237": "2077696c64", + "2238": "6f6f7365", + "2239": "20746173746564", + "2240": "20636f696e", + "2241": "206170706561726564", + "2242": "206275696c64696e67", + "2243": "2077686f6c65", + "2244": "206d6564", + "2245": "206c6164", + "2246": "20706f70", + "2247": "206f7267616e", + "2248": "657274", + "2249": "69747479", + "2250": "20546f676574686572", + "2251": "2074726970", + "2252": "20656167", + "2253": "4a6f", + "2254": "2072657370656374", + "2255": "616c65", + "2256": "2061736c656570", + "2257": "20656d62", + "2258": "2073637265616d6564", + "2259": "2077656172696e67", + "2260": "207772697465", + "2261": "20736c6964", + "2262": "206d7564", + "2263": "207468696e", + "2264": "65726c79", + "2265": "756c6172", + "2266": "20616e73776572", + "2267": "6365616e", + "2268": "206d6f756e7461696e", + "2269": "206f6365616e", + "2270": "207370656e74", + "2271": "436f6d65", + "2272": "2073656174", + "2273": "2064697274", + "2274": "20696d707265", + "2275": "20636f6e6675736564", + "2276": "20776f726d", + "2277": "20636f6f6b6965", + "2278": "204c697361", + "2279": "20736e616b65", + "2280": "206675727279", + "2281": "7570", + "2282": "206261726b", + "2283": "636b73", + "2284": "206d696e7574", + "2285": "20636f756368", + "2286": "2069676e", + "2287": "206d616368", + "2288": "2076616c", + "2289": "2073757070", + "2290": "20636f726e6572", + "2291": "2073706c6173", + "2292": "206c6164646572", + "2293": "20666f6f74", + "2294": "204a616b65", + "2295": "20737069646572", + "2296": "206469", + "2297": "20656d626172", + "2298": "61726765", + "2299": "20416e6e", + "2300": "2067656e746c79", + "2301": "206e65696768626f72", + "2302": "6e6963", + "2303": "206d696e64", + "2304": "206661636573", + "2305": "20656d6261727261", + "2306": "207069636e6963", + "2307": "206f76656e", + "2308": "2070726574656e646564", + "2309": "2068656c7066756c", + "2310": "207465616368", + "2311": "2070616e", + "2312": "20676f6573", + "2313": "6c61", + "2314": "696665", + "2315": "697a7a", + "2316": "2064656c69676874", + "2317": "2074616c6b696e67", + "2318": "206c697665", + "2319": "2062616e", + "2320": "6c6578", + "2321": "206c617374", + "2322": "2066656574", + "2323": "2063617274", + "2324": "20536f6d6574696d6573", + "2325": "207768656e65766572", + "2326": "2070686f6e65", + "2327": "206472697665", + "2328": "20736572", + "2329": "6f6c64", + "2330": "756e6b", + "2331": "2073616c", + "2332": "20626c6577", + "2333": "6f6e6573", + "2334": "204c65", + "2335": "6e6e79", + "2336": "72656164", + "2337": "206576656e7475616c6c79", + "2338": "204275", + "2339": "20746f6f6c", + "2340": "62626c6573", + "2341": "2077616974696e67", + "2342": "205065746572", + "2343": "2077617665", + "2344": "20737072656164", + "2345": "204265", + "2346": "2072656d696e64", + "2347": "6d6f", + "2348": "20636f6c6f7572", + "2349": "207472696573", + "2350": "2070757368", + "2351": "6c656272", + "2352": "616c73", + "2353": "20616e74", + "2354": "2063616c6d", + "2355": "20746964", + "2356": "20617368", + "2357": "206c61726765", + "2358": "2063686f636f6c", + "2359": "2063656c656272", + "2360": "20726563", + "2361": "2066696e697368", + "2362": "20666f7267697665", + "2363": "206b6e6565", + "2364": "20476f", + "2365": "20746f6d6f72726f77", + "2366": "206669786564", + "2367": "20736d6f6f7468", + "2368": "207368697274", + "2369": "2062616e616e", + "2370": "2073747261", + "2371": "20656d626172726173736564", + "2372": "20706f7572", + "2373": "20726f62", + "2374": "61746f", + "2375": "20706c65", + "2376": "20706177", + "2377": "207465657468", + "2378": "206669676874696e67", + "2379": "205265", + "2380": "20736879", + "2381": "20736f6e6773", + "2382": "20686964696e67", + "2383": "20666f6c64", + "2384": "2062656c6c", + "2385": "207370726179", + "2386": "2072696e67", + "2387": "75727365", + "2388": "2063616e6e", + "2389": "2068756d", + "2390": "205465646479", + "2391": "2045766572797768657265", + "2392": "20636c6f7468", + "2393": "20746561", + "2394": "20706f6f6c", + "2395": "20726f636b6574", + "2396": "205768696c65", + "2397": "206372656174697665", + "2398": "204a6f686e6e79", + "2399": "207374726574", + "2400": "20636869636b", + "2401": "2076696c6c616765", + "2402": "6967686564", + "2403": "20636f6f", + "2404": "756e676c65", + "2405": "2073656c", + "2406": "2063616e6e6f74", + "2407": "20656d70", + "2408": "2063686f636f6c617465", + "2409": "61756c", + "2410": "206b697474656e", + "2411": "206e6574", + "2412": "756c74", + "2413": "20656d707479", + "2414": "646765", + "2415": "2063726f7373", + "2416": "2072756465", + "2417": "20537573", + "2418": "2077686973746c65", + "2419": "207461737465", + "2420": "206a756e676c65", + "2421": "2067656e", + "2422": "20736f6c", + "2423": "696e746564", + "2424": "2062756773", + "2425": "20737065", + "2426": "20707564646c65", + "2427": "2067726577", + "2428": "206d616368696e65", + "2429": "4a6f68", + "2430": "2073746570", + "2431": "67657461626c65", + "2432": "20636861736564", + "2433": "64656e", + "2434": "2073617665", + "2435": "20536f6d65", + "2436": "20766567657461626c65", + "2437": "706572", + "2438": "2074696768746c79", + "2439": "206d6561646f77", + "2440": "5374", + "2441": "20776970", + "2442": "4d79", + "2443": "2073746972", + "2444": "207370696379", + "2445": "20726f626f74", + "2446": "206261726b6564", + "2447": "206c656166", + "2448": "20666f7277617264", + "2449": "61697273", + "2450": "206d6f6f6e", + "2451": "2074696479", + "2452": "207472757374", + "2453": "20747265617473", + "2454": "626f78", + "2455": "206d61796265", + "2456": "69676572", + "2457": "20706561636566756c", + "2458": "20737472696e67", + "2459": "2077617368", + "2460": "20776f7265", + "2461": "756d7079", + "2462": "6f7068", + "2463": "2073636172", + "2464": "20506c65617365", + "2465": "20636172726f7473", + "2466": "206469736170706f", + "2467": "61726d", + "2468": "2073656c66697368", + "2469": "206b6e6f636b6564", + "2470": "205456", + "2471": "206865617273", + "2472": "206176", + "2473": "506c65617365", + "2474": "206e65636b6c", + "2475": "207072696e6365", + "2476": "2068696464656e", + "2477": "20736f7572", + "2478": "2073706c617368", + "2479": "6e6564", + "2480": "757368", + "2481": "20706c617465", + "2482": "204d756d6d79", + "2483": "20737061726b6c79", + "2484": "67726f756e64", + "2485": "2070696e", + "2486": "657465726d", + "2487": "6d6f7374", + "2488": "697264", + "2489": "2064657465726d", + "2490": "205a", + "2491": "207368656c66", + "2492": "2067756e", + "2493": "206d6973736564", + "2494": "204576656e", + "2495": "75737472", + "2496": "2063726162", + "2497": "676c79", + "2498": "2062616c", + "2499": "206d65616e74", + "2500": "2073686f7265", + "2501": "646c65", + "2502": "206a756d70696e67", + "2503": "79696e67", + "2504": "61746f72", + "2505": "206469736170706f696e746564", + "2506": "20616c6d6f7374", + "2507": "204c6974746c65", + "2508": "20736e61636b73", + "2509": "2075676c79", + "2510": "206c656d", + "2511": "2074656c6c73", + "2512": "206b6e6f77696e67", + "2513": "6c6f776572", + "2514": "206c616e646564", + "2515": "20707269", + "2516": "206d6f74", + "2517": "20676f6c64", + "2518": "2073686172696e67", + "2519": "20667275737472", + "2520": "69616e", + "2521": "20726570", + "2522": "2073746f6f64", + "2523": "2067656e6572", + "2524": "4a6f686e", + "2525": "657261", + "2526": "2066656172", + "2527": "2077686973", + "2528": "2063617272696564", + "2529": "2067726f7570", + "2530": "206c696e65", + "2531": "206465636f72", + "2532": "20576879", + "2533": "206c656773", + "2534": "206a6577", + "2535": "20686f6e657374", + "2536": "207374616e64696e67", + "2537": "2063697263", + "2538": "20686f757273", + "2539": "616b696e67", + "2540": "6f727279", + "2541": "2079656c6c6564", + "2542": "2074657374", + "2543": "206275726e", + "2544": "2070757a7a6c65", + "2545": "20756e64657273746f6f64", + "2546": "616e6f", + "2547": "2073746f6e65", + "2548": "6974746572", + "2549": "20706f6c696365", + "2550": "2764", + "2551": "20626c6f77", + "2552": "206e756d626572", + "2553": "204c656f", + "2554": "206e65636b6c616365", + "2555": "6779", + "2556": "2076616e", + "2557": "204272", + "2558": "6b696e", + "2559": "656c6c79", + "2560": "20737469636b73", + "2561": "2066696c", + "2562": "20736c65657079", + "2563": "20706179", + "2564": "2066727573747261746564", + "2565": "2067726179", + "2566": "2066616e6379", + "2567": "206d756d6d79", + "2568": "20637269", + "2569": "20676173", + "2570": "2073637265616d", + "2571": "68696e", + "2572": "2068656c", + "2573": "20706972", + "2574": "207468657365", + "2575": "206d7973746572696f7573", + "2576": "20626572", + "2577": "20627265617468", + "2578": "2070696c6c6f77", + "2579": "2070697a7a", + "2580": "207368656570", + "2581": "766f7573", + "2582": "20746175676874", + "2583": "53746f70", + "2584": "696666", + "2585": "20747769726c", + "2586": "20706c616e7473", + "2587": "20617368616d6564", + "2588": "205375736965", + "2589": "20676172", + "2590": "2073656564", + "2591": "7061636b", + "2592": "6974616c", + "2593": "206372696573", + "2594": "206e6f6473", + "2595": "207761766573", + "2596": "20706f6c697465", + "2597": "20416c696365", + "2598": "20706173736564", + "2599": "20636f6d666f727461626c65", + "2600": "2064726577", + "2601": "2072657475726e6564", + "2602": "2073756363657373", + "2603": "6166", + "2604": "657270", + "2605": "20706f77657266756c", + "2606": "20686f7370", + "2607": "73697374", + "2608": "207468616e6b66756c", + "2609": "20706f6f72", + "2610": "696e61", + "2611": "20446f6e", + "2612": "206d65737379", + "2613": "20686f73706974616c", + "2614": "6f726b", + "2615": "207363617266", + "2616": "53616d", + "2617": "206e6572", + "2618": "2067656e65726f7573", + "2619": "2073616e6477696368", + "2620": "20626564726f6f6d", + "2621": "20726567", + "2622": "6f73617572", + "2623": "20666c7574", + "2624": "2066656564", + "2625": "2074616b656e", + "2626": "75616c", + "2627": "697a7a79", + "2628": "207365656b", + "2629": "2073686970", + "2630": "206761746865726564", + "2631": "2064657465726d696e6564", + "2632": "206e6572766f7573", + "2633": "20736967686564", + "2634": "2065617379", + "2635": "206c6567", + "2636": "656e73697665", + "2637": "2067617465", + "2638": "6f7465", + "2639": "207061636b6564", + "2640": "204b69747479", + "2641": "20736372", + "2642": "2066696e676572", + "2643": "6f626279", + "2644": "2070697a7a61", + "2645": "20657870656e73697665", + "2646": "736572", + "2647": "696c6c6564", + "2648": "756365", + "2649": "206d696e75746573", + "2650": "20657863697465", + "2651": "20636c6f776e", + "2652": "206a6f67", + "2653": "6964646c65", + "2654": "20666967", + "2655": "206e7574", + "2656": "6f6e79", + "2657": "20636c617373", + "2658": "2064696e6f73617572", + "2659": "206c6974", + "2660": "2074656e74", + "2661": "20706c6561736564", + "2662": "776f726b", + "2663": "2073706f696c", + "2664": "20736f6c64", + "2665": "206578616d", + "2666": "206472756d", + "2667": "20636c6f736574", + "2668": "20726164", + "2669": "20776f6c66", + "2670": "20657363", + "2671": "206d65646963", + "2672": "2064616e63696e67", + "2673": "20766567657461626c6573", + "2674": "2077616e646572", + "2675": "20626f726564", + "2676": "20676c617373", + "2677": "696e657373", + "2678": "20616e737765726564", + "2679": "20687572", + "2680": "61726c", + "2681": "6973736f7273", + "2682": "2063686577", + "2683": "206f6674656e", + "2684": "2061736b696e67", + "2685": "206578636974656d656e74", + "2686": "20426c", + "2687": "20796f757273656c66", + "2688": "206d65646963696e65", + "2689": "206b6e69676874", + "2690": "6c69676874", + "2691": "2073636973736f7273", + "2692": "204d61726b", + "2693": "6f6f66", + "2694": "207469636b", + "2695": "20626f6e65", + "2696": "656369616c6c79", + "2697": "207361766564", + "2698": "20686f7070696e67", + "2699": "206b6973736564", + "2700": "2076617365", + "2701": "5768657265", + "2702": "20776973", + "2703": "2064656572", + "2704": "20636f6e74656e74", + "2705": "20776f726b696e67", + "2706": "61746573", + "2707": "2074727565", + "2708": "5468657265", + "2709": "20636174657270", + "2710": "20636174657270696c6c", + "2711": "207370656e64", + "2712": "2064726f70", + "2713": "20676172616765", + "2714": "20676967676c6564", + "2715": "20626f72696e67", + "2716": "20776167676564", + "2717": "657473", + "2718": "20546f646179", + "2719": "2063616e64", + "2720": "207465617273", + "2721": "206265636f6d65", + "2722": "2064697374616e6365", + "2723": "206d6964646c65", + "2724": "2065676773", + "2725": "206368657374", + "2726": "206772756d7079", + "2727": "70656369616c6c79", + "2728": "657465", + "2729": "20707573", + "2730": "207374756666", + "2731": "20657370656369616c6c79", + "2732": "2072656c6178", + "2733": "2073686f6f74", + "2734": "206d697374616b65", + "2735": "206f6e6573", + "2736": "207175657374696f6e73", + "2737": "206c6f636b", + "2738": "207765616b", + "2739": "2062617468726f6f6d", + "2740": "2070726179", + "2741": "20526578", + "2742": "20776865656c", + "2743": "6f6f70", + "2744": "2062616c6c73", + "2745": "2073746f6e6573", + "2746": "20666c6167", + "2747": "20717565656e", + "2748": "206c656164", + "2749": "20726964", + "2750": "20706c617967726f756e64", + "2751": "717565", + "2752": "207275696e6564", + "2753": "206a656c6c79", + "2754": "207361696c6564", + "2755": "2068616e67", + "2756": "536f727279", + "2757": "204f4b", + "2758": "207369676874", + "2759": "20667275697473", + "2760": "626f6479", + "2761": "73656c766573", + "2762": "2077616c6b73", + "2763": "2064657374", + "2764": "206d6f746f72", + "2765": "20636174657270696c6c6172", + "2766": "61676564", + "2767": "796d", + "2768": "6174696e67", + "2769": "696d70", + "2770": "206c656d6f6e", + "2771": "2066616b65", + "2772": "206d6f76", + "2773": "20636f7665726564", + "2774": "206272757368", + "2775": "206c79696e67", + "2776": "20627269646765", + "2777": "57686f", + "2778": "20626f7373", + "2779": "207069636b73", + "2780": "63657074", + "2781": "206d6972", + "2782": "417265", + "2783": "2067657473", + "2784": "20636c6170", + "2785": "206d61746368", + "2786": "2073706c6173686564", + "2787": "2064726177696e67", + "2788": "2063726f776e", + "2789": "20637261776c", + "2790": "6d62", + "2791": "20456e64", + "2792": "20637261636b", + "2793": "20736d656c6c79", + "2794": "20726f6172", + "2795": "20616464", + "2796": "206368616e6765", + "2797": "20666c75666679", + "2798": "20736b6970706564", + "2799": "2063616c6c696e67", + "2800": "20626f7973", + "2801": "207465616d", + "2802": "204576656e7475616c6c79", + "2803": "206469736170706561726564", + "2804": "2063616765", + "2805": "20696e63726564", + "2806": "20746965", + "2807": "2077726f7465", + "2808": "206665617468657273", + "2809": "726f72", + "2810": "20626f617264", + "2811": "2066616d6f7573", + "2812": "20696e64", + "2813": "656c70", + "2814": "2073776565746965", + "2815": "6b656570", + "2816": "2074617065", + "2817": "206d6967687479", + "2818": "206b6e696665", + "2819": "20426c7565", + "2820": "20676c7565", + "2821": "206c617a79", + "2822": "696c6c69", + "2823": "2055", + "2824": "206f6b", + "2825": "20676c6f77", + "2826": "696e6174696f6e", + "2827": "206d6f76696e67", + "2828": "20656c64", + "2829": "20666973", + "2830": "2073696c", + "2831": "20666174686572", + "2832": "2062616b65", + "2833": "204966", + "2834": "206d656173757265", + "2835": "2073686f7773", + "2836": "2050657465", + "2837": "20676f6174", + "2838": "206f7264", + "2839": "20636170", + "2840": "20636865636b", + "2841": "206d6170", + "2842": "206564", + "2843": "20626f72726f77", + "2844": "20686f726e", + "2845": "204d6172", + "2846": "2062616e616e61", + "2847": "2042726f776e", + "2848": "20646973636f7665726564", + "2849": "656c6c61", + "2850": "206c6f7365", + "2851": "20636f766572", + "2852": "20636c656172", + "2853": "4e6f77", + "2854": "20746167", + "2855": "20456c6c61", + "2856": "61727468", + "2857": "6f7473", + "2858": "20737567", + "2859": "20756e7573", + "2860": "20756e757375616c", + "2861": "20657874", + "2862": "20636f7374", + "2863": "207265616c69736564", + "2864": "206d6963", + "2865": "3f222e", + "2866": "204e6f", + "2867": "6173696e67", + "2868": "206a75696379", + "2869": "2064697368", + "2870": "20696e6372656469626c65", + "2871": "7469", + "2872": "20646967", + "2873": "20646563696465", + "2874": "206472616e6b", + "2875": "206f666665726564", + "2876": "207469676572", + "2877": "2046726564", + "2878": "20667269646765", + "2879": "206272616e63686573", + "2880": "20626f74746f6d", + "2881": "206f6e6365", + "2882": "20646f6773", + "2883": "206561676572", + "2884": "2063686572", + "2885": "20736f6170", + "2886": "207375676172", + "2887": "48656c70", + "2888": "2066696e65", + "2889": "20657965", + "2890": "696d6564", + "2891": "20636f6c6f757266756c", + "2892": "20426f626279", + "2893": "20776f7264", + "2894": "207374726177", + "2895": "207765656b", + "2896": "20636c65616e6564", + "2897": "20746869636b", + "2898": "756767657374", + "2899": "2063726177", + "2900": "206c69636b6564", + "2901": "20637261776c6564", + "2902": "2073756767657374", + "2903": "206e6f64", + "2904": "206d6972726f72", + "2905": "206c6f75646572", + "2906": "66617374", + "2907": "6b6565706572", + "2908": "67696e67", + "2909": "20796f757273", + "2910": "206177", + "2911": "204d7273", + "2912": "2068617070656e696e67", + "2913": "75726564", + "2914": "2073656c6c", + "2915": "20627562626c6573", + "2916": "206b696e646e657373", + "2917": "6c616e64", + "2918": "616c6f7573", + "2919": "206c616d70", + "2920": "206578747261", + "2921": "207477697374", + "2922": "20627265616b66617374", + "2923": "686970", + "2924": "206a65616c6f7573", + "2925": "2074756e6e", + "2926": "20626f6479", + "2927": "206661766f7572", + "2928": "6f6c6f67", + "2929": "20656c6465726c79", + "2930": "20746f77656c", + "2931": "20526f7365", + "2932": "206a6f6b65", + "2933": "656469656e74", + "2934": "20616476656e747572", + "2935": "69717565", + "2936": "756d626c6564", + "2937": "206865726f", + "2938": "2072617265", + "2939": "20626f786573", + "2940": "2061706f6c6f67", + "2941": "206661766f7572697465", + "2942": "206875727473", + "2943": "2077616e64", + "2944": "206e6f697379", + "2945": "206d616e61676564", + "2946": "697070", + "2947": "706f7274", + "2948": "2062616c616e6365", + "2949": "79616c", + "2950": "646479", + "2951": "20756e69717565", + "2952": "6970706572", + "2953": "6174656c79", + "2954": "206a6577656c", + "2955": "20616476656e7475726f7573", + "2956": "2cc3a2e282ac", + "2957": "20706565", + "2958": "666963", + "2959": "2063616d", + "2960": "20616363657074", + "2961": "20616e67", + "2962": "20696d70726573736564", + "2963": "20656767", + "2964": "207768616c65", + "2965": "696c74", + "2966": "2062616c6c6f6f6e73", + "2967": "2066726f776e", + "2968": "206f7267616e697a6564", + "2969": "20746f7567", + "2970": "20636f696e73", + "2971": "204d616e6479", + "2972": "617070696e67", + "2973": "2064697a7a79", + "2974": "20616761696e7374", + "2975": "20666f6f6c", + "2976": "206164646564", + "2977": "61707079", + "2978": "20646966666963", + "2979": "20636972636c6573", + "2980": "20666967757265", + "2981": "2077726170706564", + "2982": "206f62656469656e74", + "2983": "7567676c6564", + "2984": "206861726d", + "2985": "4d65", + "2986": "7265616479", + "2987": "57656c6c", + "2988": "2070756d70", + "2989": "206d6f64", + "2990": "206772616365", + "2991": "20736d6f6b65", + "2992": "20706f696e74696e67", + "2993": "20636c6179", + "2994": "20746572", + "2995": "20617267", + "2996": "2067726f776e", + "2997": "6c696573", + "2998": "63696c", + "2999": "20646966666963756c74", + "3000": "2070656e63696c", + "3001": "61636865", + "3002": "2073747265616d", + "3003": "207377616e", + "3004": "2073776f7264", + "3005": "20616c7265616479", + "3006": "4265", + "3007": "2068616d", + "3008": "207761676f6e", + "3009": "206d61726b6574", + "3010": "61676573", + "3011": "20736861726b", + "3012": "206775657373", + "3013": "205768656e65766572", + "3014": "206c6966746564", + "3015": "206c6f79616c", + "3016": "6f6d6564", + "3017": "20636162", + "3018": "20626974746572", + "3019": "2063686f6f7365", + "3020": "20636162696e", + "3021": "207374726177626572", + "3022": "207061696e74696e67", + "3023": "20726f756768", + "3024": "72616765", + "3025": "20627574746f6e73", + "3026": "20746f756768", + "3027": "2068617070696e657373", + "3028": "206272696c6c69", + "3029": "2070696c", + "3030": "657279", + "3031": "20647265616d6564", + "3032": "20736c6970706564", + "3033": "20646570656e64", + "3034": "20626c6f6f6d", + "3035": "206c6561726e696e67", + "3036": "6f7279", + "3037": "2044616e", + "3038": "206c6f75646c79", + "3039": "6177", + "3040": "68656164", + "3041": "20666f7572", + "3042": "206d656c74", + "3043": "206d65616e73", + "3044": "20666f6f6c697368", + "3045": "206272696c6c69616e74", + "3046": "20676976696e67", + "3047": "20686f7365", + "3048": "2069736c616e64", + "3049": "6f6f74", + "3050": "20747275636b73", + "3051": "2077726170", + "3052": "2068756e74", + "3053": "206e65656473", + "3054": "20666c617368", + "3055": "2065646765", + "3056": "207370756e", + "3057": "65657a", + "3058": "206368617365", + "3059": "2062656c6f6e676564", + "3060": "206a616d", + "3061": "616e676564", + "3062": "656d6f", + "3063": "2065617273", + "3064": "2062726f7468657273", + "3065": "6c61696d6564", + "3066": "20656e6a6f79696e67", + "3067": "2077616c6c73", + "3068": "204e656d6f", + "3069": "20736166656c79", + "3070": "20667269676874", + "3071": "2066616c6c73", + "3072": "6976", + "3073": "206578636c61696d6564", + "3074": "206d6f6d73", + "3075": "207765697264", + "3076": "2072696368", + "3077": "206a61636b", + "3078": "20536e", + "3079": "206d69786564", + "3080": "20736561726368", + "3081": "20756e6c", + "3082": "207374726f6e676572", + "3083": "2074757274", + "3084": "207368616b65", + "3085": "2044617665", + "3086": "207472756e6b", + "3087": "20726566", + "3088": "206e6f726d", + "3089": "2063726f", + "3090": "207363617265", + "3091": "2074686f7567687466756c", + "3092": "6961626c65", + "3093": "20636865657266756c", + "3094": "206261726e", + "3095": "20706172726f74", + "3096": "2062657272696573", + "3097": "65727279", + "3098": "206c696272", + "3099": "20646f6c70", + "3100": "20736179696e67", + "3101": "207765616c", + "3102": "7368696f6e", + "3103": "20636f6d706574", + "3104": "20647265616d73", + "3105": "206d6f766965", + "3106": "207465727269", + "3107": "2072656c6961626c65", + "3108": "6174757265", + "3109": "20746f617374", + "3110": "20746f6f6c73", + "3111": "2063697479", + "3112": "2062756c6c", + "3113": "4c696c61", + "3114": "20636972636c65", + "3115": "20746875", + "3116": "2068617473", + "3117": "2074756e6e656c", + "3118": "206e6f726d616c", + "3119": "20696e646570656e64", + "3120": "6174746572", + "3121": "2073756974", + "3122": "20547765657479", + "3123": "2074776967", + "3124": "697065", + "3125": "20637261796f6e", + "3126": "207573696e67", + "3127": "2063616d657261", + "3128": "616c6b", + "3129": "657874", + "3130": "7761726473", + "3131": "2066696572", + "3132": "207265636f7264", + "3133": "20626f737379", + "3134": "7965", + "3135": "206c6175676873", + "3136": "2062616c64", + "3137": "20726f6f66", + "3138": "20736f756e646564", + "3139": "2067756974", + "3140": "616d6f6e64", + "3141": "20646570656e6461626c65", + "3142": "2074696d6573", + "3143": "2076616c75", + "3144": "206e6f74696365", + "3145": "2042656c6c61", + "3146": "2062616773", + "3147": "20636f6d7065746974", + "3148": "2064657369676e", + "3149": "706573", + "3150": "206c65616e", + "3151": "2070726f75646c79", + "3152": "72657373", + "3153": "206c6177", + "3154": "20736b696e", + "3155": "48657265", + "3156": "20726174", + "3157": "204d79", + "3158": "206d6172626c65", + "3159": "20696e7669746564", + "3160": "726167", + "3161": "20696379", + "3162": "206772616e647061", + "3163": "20666965726365", + "3164": "206469616d6f6e64", + "3165": "20636c65616e696e67", + "3166": "20666f756e7461696e", + "3167": "20657863697465646c79", + "3168": "206d6f6c65", + "3169": "20436c", + "3170": "206172726f77", + "3171": "206c6f67", + "3172": "2074616e6b", + "3173": "20667265", + "3174": "20636f70", + "3175": "20657363617065", + "3176": "6f726765", + "3177": "20696e646570656e64656e74", + "3178": "6f75736573", + "3179": "206b696e6473", + "3180": "207961726e", + "3181": "20626972646965", + "3182": "20736d616c6c6572", + "3183": "2068656c7073", + "3184": "2063657265", + "3185": "61646f", + "3186": "207368656c6c73", + "3187": "6a656374", + "3188": "206e6f7379", + "3189": "2061626f7665", + "3190": "20416c6578", + "3191": "206578706c6f726564", + "3192": "207768697370", + "3193": "656e74696f6e", + "3194": "20686172646572", + "3195": "207368617065", + "3196": "20747261636b", + "3197": "7572696e67", + "3198": "206f6666696365", + "3199": "2075736566756c", + "3200": "6f707079", + "3201": "20667269676874656e6564", + "3202": "204475636b79", + "3203": "207061696e746564", + "3204": "204d696b65", + "3205": "20636f756e746564", + "3206": "207374756262", + "3207": "20747572746c65", + "3208": "20677569746172", + "3209": "206d61676e", + "3210": "2067656d", + "3211": "2063756464", + "3212": "2070616e74", + "3213": "6f6c61", + "3214": "2077616b65", + "3215": "2066726167", + "3216": "20686f75736573", + "3217": "20656e6f726d", + "3218": "20426f626f", + "3219": "696e617279", + "3220": "20656e6f726d6f7573", + "3221": "20737567676573746564", + "3222": "206a6f696e6564", + "3223": "20737765", + "3224": "20627573686573", + "3225": "6e61", + "3226": "206d656d", + "3227": "2076616c7561626c65", + "3228": "697270", + "3229": "206c6967687473", + "3230": "756d62657273", + "3231": "2073686f6e65", + "3232": "20666174", + "3233": "206275696c74", + "3234": "2073747562626f726e", + "3235": "204f6e6365", + "3236": "20636869636b656e", + "3237": "20636f7374756d", + "3238": "2064656c696768746564", + "3239": "2074756d6d79", + "3240": "2073747269", + "3241": "207765616c746879", + "3242": "2072656d696e646564", + "3243": "2073686f77696e67", + "3244": "61727279", + "3245": "206b6e6f7773", + "3246": "2061646d69726564", + "3247": "20666c757465", + "3248": "206e75727365", + "3249": "646f6d", + "3250": "206b69636b6564", + "3251": "20726f6465", + "3252": "2063686f7365", + "3253": "207374756d626c6564", + "3254": "2066726167696c65", + "3255": "696c6c6965", + "3256": "2066696e67657273", + "3257": "656f726765", + "3258": "2074656c65", + "3259": "2073756e66", + "3260": "2077617973", + "3261": "206d61726368", + "3262": "206d6574616c", + "3263": "207365656d", + "3264": "204672", + "3265": "207472756d70", + "3266": "206c697a", + "3267": "207072616374696365", + "3268": "706964", + "3269": "20736f6e", + "3270": "206d656c", + "3271": "207375636365737366756c", + "3272": "6369656e74", + "3273": "2073706f696c6564", + "3274": "20706f75726564", + "3275": "20796f67", + "3276": "2063657265616c", + "3277": "20636f6e65", + "3278": "69656e6365", + "3279": "20646f7665", + "3280": "206272696768746c79", + "3281": "20737465616c", + "3282": "207361756365", + "3283": "20736f6c7665", + "3284": "617a", + "3285": "20737475706964", + "3286": "616e7574", + "3287": "20706f7374", + "3288": "207261636564", + "3289": "204772616e64", + "3290": "20626f756e6365", + "3291": "2062656574", + "3292": "2070726570", + "3293": "20736561726368696e67", + "3294": "206f77", + "3295": "20796574", + "3296": "207374756479", + "3297": "20706972617465", + "3298": "696577", + "3299": "20746869727374", + "3300": "206861697279", + "3301": "207363726174", + "3302": "206672657368", + "3303": "20616e6369656e74", + "3304": "204b696d", + "3305": "207069636b696e67", + "3306": "207465727269626c65", + "3307": "2064756c6c", + "3308": "2072657363", + "3309": "20636c756d", + "3310": "20616e67656c", + "3311": "206e756d62657273", + "3312": "2073746f7665", + "3313": "2063616374", + "3314": "20666f727468", + "3315": "20706f7461746f", + "3316": "2061766f", + "3317": "6b6e", + "3318": "204c6f6c61", + "3319": "20686179", + "3320": "206265616b", + "3321": "207374616765", + "3322": "616e676572", + "3323": "2073616c6164", + "3324": "636f6d", + "3325": "696e616c", + "3326": "2076696577", + "3327": "206f726967", + "3328": "20727573686564", + "3329": "20647265616d696e67", + "3330": "20737461697273", + "3331": "207370616365", + "3332": "696e6573", + "3333": "206c6f77", + "3334": "67657374", + "3335": "207065656b6564", + "3336": "206368616e676564", + "3337": "20636f6d70617373", + "3338": "207072697a65", + "3339": "2062756d70", + "3340": "2062696767657374", + "3341": "636f64", + "3342": "206c6162", + "3343": "207468756e646572", + "3344": "207472756d706574", + "3345": "757468", + "3346": "206c6f636b6564", + "3347": "206f726967696e616c", + "3348": "6974686572", + "3349": "2063686172", + "3350": "206368656170", + "3351": "20636163747573", + "3352": "2074656c6c696e67", + "3353": "20656e63", + "3354": "207374616d70", + "3355": "20656e76656c", + "3356": "2068616c66", + "3357": "206572", + "3358": "20706f77", + "3359": "20546f6279", + "3360": "207769706564", + "3361": "20646f756768", + "3362": "206c656176696e67", + "3363": "20636172656c657373", + "3364": "20736572696f7573", + "3365": "20726567756c6172", + "3366": "2074686972737479", + "3367": "20656e76", + "3368": "20746f6f7468", + "3369": "656d656e74", + "3370": "20706c617966756c", + "3371": "21222e", + "3372": "20456c6c6965", + "3373": "20636c756d7379", + "3374": "20706570706572", + "3375": "207468656d73656c766573", + "3376": "636f726e", + "3377": "20696d6167696e6564", + "3378": "205061756c", + "3379": "2070756d706b696e", + "3380": "6b6e6f776e", + "3381": "2074726170", + "3382": "20626f617473", + "3383": "20696e67", + "3384": "2066696e64696e67", + "3385": "2068616e646c65", + "3386": "2063726f636f64", + "3387": "20677261636566756c", + "3388": "207065616e7574", + "3389": "2077697265", + "3390": "69736b", + "3391": "7465726e", + "3392": "20737061726b6c6564", + "3393": "20626174746572", + "3394": "20637269636b6574", + "3395": "20736e696666", + "3396": "204e657874", + "3397": "20626172626572", + "3398": "76696365", + "3399": "72656469656e74", + "3400": "20656e64696e67", + "3401": "61727368", + "3402": "20696e6772656469656e74", + "3403": "206d6973", + "3404": "20776173686564", + "3405": "20676f6f7365", + "3406": "20536f7068", + "3407": "74696d65", + "3408": "20636f6d707574", + "3409": "20666c6173686c69676874", + "3410": "2073746174", + "3411": "206c697a617264", + "3412": "696c7479", + "3413": "207370696e6e696e67", + "3414": "207061737461", + "3415": "206775696c7479", + "3416": "20646f65736e", + "3417": "206561727468", + "3418": "2070657273697374", + "3419": "20776869737065726564", + "3420": "696c6c61", + "3421": "20706f70636f726e", + "3422": "2064656164", + "3423": "20416e6479", + "3424": "206465736b", + "3425": "6375", + "3426": "2068656c6d", + "3427": "20636f6d62", + "3428": "2073746f7073", + "3429": "2074686f7365", + "3430": "20636f6d70617373696f6e", + "3431": "20756e6b6e6f776e", + "3432": "2073637265656e", + "3433": "2047656f726765", + "3434": "20736e65", + "3435": "206e6963656c79", + "3436": "20727567", + "3437": "2073636f6f74", + "3438": "20626565746c65", + "3439": "7564", + "3440": "686574", + "3441": "726f6265", + "3442": "206265616e", + "3443": "20506f6c6c79", + "3444": "2061636f726e", + "3445": "61776e", + "3446": "486f77", + "3447": "20626c7565626572", + "3448": "20616c6c6f776564", + "3449": "2062656573", + "3450": "2073696e6b", + "3451": "207370696c6c6564", + "3452": "207769746368", + "3453": "20736869766572", + "3454": "207061747465726e", + "3455": "7567676564", + "3456": "207072696e74", + "3457": "2064656c6963617465", + "3458": "2064726976696e67", + "3459": "2072656675736564", + "3460": "20676173706564", + "3461": "207365656473", + "3462": "207368656c", + "3463": "20726f7365", + "3464": "656c6c6967", + "3465": "2064656c69766572", + "3466": "c3a2e282ace280", + "3467": "20667269656e6473686970", + "3468": "207761726e6564", + "3469": "20696e6772656469656e7473", + "3470": "206a61636b6574", + "3471": "6865747469", + "3472": "61676865747469", + "3473": "20696e74656c6c6967", + "3474": "20656e76656c6f7065", + "3475": "2070616e7473", + "3476": "20736c69646573", + "3477": "20666c6578", + "3478": "20736f727473", + "3479": "206775617264", + "3480": "2073756464656e", + "3481": "20756e636c65", + "3482": "20637572", + "3483": "2074726176656c", + "3484": "20726577617264", + "3485": "206d697373696e67", + "3486": "20737061676865747469", + "3487": "206a6f6c6c79", + "3488": "20666f726b", + "3489": "2070656e6e79", + "3490": "2068616d6d6572", + "3491": "204772616e647061", + "3492": "636f6d666f7274", + "3493": "2074696564", + "3494": "2063656c656272617465", + "3495": "206b6e6f636b", + "3496": "20706f70756c6172", + "3497": "2073757266", + "3498": "204772616365", + "3499": "206d65616c", + "3500": "20696d7072657373", + "3501": "20756e636f6d666f7274", + "3502": "206361727279696e67", + "3503": "20616e78", + "3504": "20736869656c64", + "3505": "20636f756e74696e67", + "3506": "206561726c79", + "3507": "20737472616e676572", + "3508": "20756e636f6d666f727461626c65", + "3509": "65657a65", + "3510": "2064657374726f", + "3511": "206d6f646572", + "3512": "204d696c6c6965", + "3513": "616964", + "3514": "20596f7572", + "3515": "20737175617265", + "3516": "6174696f6e73", + "3517": "20727562626564", + "3518": "204a6f6579", + "3519": "20646973636f766572", + "3520": "2061756e74", + "3521": "20626561757479", + "3522": "2073636174746572", + "3523": "20706c61636564", + "3524": "20686f6f70", + "3525": "2072756262", + "3526": "20737469636b657273", + "3527": "20526564", + "3528": "20636f6d706c6574", + "3529": "206d656c6f6e", + "3530": "726e", + "3531": "2064757374", + "3532": "20746170", + "3533": "6973696f6e", + "3534": "2063686173696e67", + "3535": "2073756e666c6f776572", + "3536": "2077696e67", + "3537": "206272616365", + "3538": "20636861726d696e67", + "3539": "20686f6c6473", + "3540": "20616e78696f7573", + "3541": "20727562", + "3542": "207175657374696f6e", + "3543": "20747769726c6564", + "3544": "20666c657869626c65", + "3545": "6c6f6f", + "3546": "20646f6c7068696e", + "3547": "20506174", + "3548": "207065726d", + "3549": "20676972", + "3550": "206f70656e73", + "3551": "6f6f6d79", + "3552": "20637265617465", + "3553": "206372616e65", + "3554": "2073686f636b6564", + "3555": "20736c656570696e67", + "3556": "20616476696365", + "3557": "20656e67", + "3558": "2068756d626c65", + "3559": "2070657273697374656e74", + "3560": "6661", + "3561": "206c6f6164", + "3562": "6f72616e74", + "3563": "2070616c65", + "3564": "206174746963", + "3565": "206d6f6465726e", + "3566": "205065", + "3567": "207461736b", + "3568": "20746f6d61746f", + "3569": "2068616e646564", + "3570": "206c696272617279", + "3571": "20636f6c6c6172", + "3572": "207a6970", + "3573": "206368616e6365", + "3574": "207468696e6b696e67", + "3575": "5265", + "3576": "726179", + "3577": "2054776565746965", + "3578": "20736861646f77", + "3579": "206861726d6c657373", + "3580": "7265616d", + "3581": "72656c6c", + "3582": "2063726f636f64696c65", + "3583": "2062656e74", + "3584": "206e6174757265", + "3585": "206168656164", + "3586": "2073657474", + "3587": "206e656174", + "3588": "20636f6d7075746572", + "3589": "20636f6f6b696e67", + "3590": "20656e636f75", + "3591": "6775", + "3592": "20726573697374", + "3593": "686f7374", + "3594": "204c6f6f6b", + "3595": "206f72646572", + "3596": "20636c6f636b", + "3597": "626c6564", + "3598": "20746f77657273", + "3599": "2067726f77696e67", + "3600": "206d61736b", + "3601": "20696e74656c6c6967656e74", + "3602": "2069676e6f72616e74", + "3603": "207961776e", + "3604": "206d696c6c", + "3605": "206f7264696e617279", + "3606": "2073706f6b65", + "3607": "20627574746572666c696573", + "3608": "696e6365", + "3609": "207363616c65", + "3610": "2070696765", + "3611": "207061746368", + "3612": "54686579", + "3613": "207370656564", + "3614": "20696d7072657373697665", + "3615": "4f77", + "3616": "6c6c6970", + "3617": "20616e797768657265", + "3618": "6c6c69706f70", + "3619": "20736e7567676c6564", + "3620": "2067686f7374", + "3621": "207472757468", + "3622": "20707570", + "3623": "2077696e646f7773", + "3624": "616666", + "3625": "204d61726961", + "3626": "206a7567", + "3627": "20737175617368", + "3628": "20637265617475726573", + "3629": "476f", + "3630": "696566", + "3631": "206d656173", + "3632": "6c63616e6f", + "3633": "20776865656c73", + "3634": "6963686573", + "3635": "2063617270", + "3636": "6163686573", + "3637": "206d61696c626f78", + "3638": "20636f7572616765", + "3639": "4865", + "3640": "2066757274686572", + "3641": "2048657265", + "3642": "207361646c79", + "3643": "67757374", + "3644": "2077616666", + "3645": "636f61", + "3646": "20656e76696f7573", + "3647": "206c6f6c6c69706f70", + "3648": "2064697367757374", + "3649": "20737461747565", + "3650": "206e6f7465", + "3651": "20636f636f61", + "3652": "206c6564", + "3653": "726963", + "3654": "616279", + "3655": "20756d62", + "3656": "20756d6272656c6c", + "3657": "20647261776572", + "3658": "2073656e64", + "3659": "2061747472616374", + "3660": "2042656172", + "3661": "2074656d", + "3662": "20647567", + "3663": "20696d706174", + "3664": "2063616d70", + "3665": "617572616e74", + "3666": "65657a6564", + "3667": "2072656164696e67", + "3668": "2073686f65", + "3669": "20737472657463686564", + "3670": "2072657374617572616e74", + "3671": "4966", + "3672": "2064657374726f79", + "3673": "536565", + "3674": "206361726473", + "3675": "20526f", + "3676": "2067756d", + "3677": "2073706c6974", + "3678": "20747269636b", + "3679": "20737570706f7274", + "3680": "206e616d6573", + "3681": "2062617365", + "3682": "206665617266756c", + "3683": "20686f6f6b", + "3684": "20696d70617469656e74", + "3685": "20616c657274", + "3686": "206865646765", + "3687": "74746572", + "3688": "20627269", + "3689": "206261746874", + "3690": "206261646c79", + "3691": "20737469636b6572", + "3692": "2068756e67", + "3693": "2068697070", + "3694": "20636f6d70617373696f6e617465", + "3695": "207370656c6c", + "3696": "20726573746c657373", + "3697": "2062617468747562", + "3698": "696262", + "3699": "204b617465", + "3700": "20697473656c66", + "3701": "206a756d7073", + "3702": "697073", + "3703": "20637570626f617264", + "3704": "2073687574", + "3705": "20737072696e67", + "3706": "2061667465726e", + "3707": "2061667465726e6f6f6e", + "3708": "206c657474657273", + "3709": "2073616e64776963686573", + "3710": "205368", + "3711": "61726f6f", + "3712": "206d617468", + "3713": "616e6761726f6f", + "3714": "696b6573", + "3715": "65627261", + "3716": "20736f6661", + "3717": "206d6179", + "3718": "20746f726e", + "3719": "2070656e6775", + "3720": "206f6174", + "3721": "696374", + "3722": "63746f70", + "3723": "63746f707573", + "3724": "206d756666", + "3725": "20676966746564", + "3726": "20636172746f6f6e", + "3727": "20756e7061636b", + "3728": "20656e67696e65", + "3729": "2045766572797468696e67", + "3730": "69636573", + "3731": "206272696768746572", + "3732": "2064656166", + "3733": "20676f72", + "3734": "2053616e6479", + "3735": "206b616e6761726f6f", + "3736": "79656172", + "3737": "2072656365", + "3738": "206a6574", + "3739": "207472617368", + "3740": "20666972656d616e", + "3741": "20766f6c63616e6f", + "3742": "205472", + "3743": "69726f6e", + "3744": "2073636f6f746572", + "3745": "657261626c65", + "3746": "207368696e65", + "3747": "2063616e646c65", + "3748": "207a65627261", + "3749": "206f63746f707573", + "3750": "2073796d", + "3751": "20636172706574", + "3752": "206d6973657261626c65", + "3753": "656572", + "3754": "2042696c6c", + "3755": "2062616e64616765", + "3756": "2076616c7565", + "3757": "6f65", + "3758": "206372617a79", + "3759": "20636f6d7065746974697665", + "3760": "20696d70726f", + "3761": "206e61696c", + "3762": "20556e", + "3763": "207061636b616765", + "3764": "2070686f74", + "3765": "20656e7468", + "3766": "637574", + "3767": "756265", + "3768": "20737461727473", + "3769": "617269616e", + "3770": "206e6f7768657265", + "3771": "2068756e746572", + "3772": "20737461636b", + "3773": "2070726574656e64696e67", + "3774": "756374", + "3775": "206372797374", + "3776": "626f6c", + "3777": "2061646d697265", + "3778": "206f74746572", + "3779": "207265696e64", + "3780": "2062726163656c6574", + "3781": "20707261637469636564", + "3782": "20656e6572", + "3783": "2064697367757374696e67", + "3784": "2070726573736564", + "3785": "204265666f7265", + "3786": "2066696c746879", + "3787": "2073696c766572", + "3788": "206675", + "3789": "20706562626c65", + "3790": "206c656f70", + "3791": "2064697374616e74", + "3792": "61756c6966", + "3793": "2073796d626f6c", + "3794": "2076696e65", + "3795": "207375706572", + "3796": "2061766f63", + "3797": "20616e79776179", + "3798": "6379636c65", + "3799": "20656e74687573", + "3800": "596179", + "3801": "207069616e6f", + "3802": "736573", + "3803": "20726f61726564", + "3804": "206a656570", + "3805": "206f626a656374", + "3806": "2062617272656c", + "3807": "207468696566", + "3808": "207075727365", + "3809": "204a65727279", + "3810": "2070696c6f74", + "3811": "2064657373", + "3812": "20756d6272656c6c61", + "3813": "207061696e746572", + "3814": "63656564", + "3815": "63636f6c", + "3816": "20737461726564", + "3817": "61736572", + "3818": "2068756d616e", + "3819": "204275646479", + "3820": "63636f6c69", + "3821": "206875727279", + "3822": "2072696365", + "3823": "20736f6172", + "3824": "686f6f64", + "3825": "69657374", + "3826": "205275", + "3827": "207265696e64656572", + "3828": "2066697265776f726b", + "3829": "206861727368", + "3830": "2061747472616374697665", + "3831": "206361756c6966", + "3832": "2062696e", + "3833": "2042756e6e79", + "3834": "206d757368", + "3835": "2072756c6573", + "3836": "20636f6f6c6572", + "3837": "20676f72696c6c61", + "3838": "206d657373616765", + "3839": "206578706572", + "3840": "54696d6d79", + "3841": "205768657265", + "3842": "20636f6d706c", + "3843": "2042726f776e6965", + "3844": "20636f756e747279", + "3845": "20536e6f77", + "3846": "20706967656f6e", + "3847": "20686970706f", + "3848": "2074756265", + "3849": "206178", + "3850": "616e6461", + "3851": "2068656c6d6574", + "3852": "20736b6970", + "3853": "2066696c6d", + "3854": "20626574", + "3855": "2073746174696f6e", + "3856": "206f6c", + "3857": "2073686f706b6565706572", + "3858": "637265616d", + "3859": "6566", + "3860": "207365616c", + "3861": "2057696c6c", + "3862": "20706f6e79", + "3863": "20747265617375726573", + "3864": "20536f70686965", + "3865": "4d6178", + "3866": "20536d", + "3867": "20676967676c65", + "3868": "20737769746368", + "3869": "20706572666f726d", + "3870": "206261636b7061636b", + "3871": "206d61676e6574", + "3872": "737461697273", + "3873": "20636f7773", + "3874": "20666c6f7572", + "3875": "20726f64", + "3876": "206f6c646572", + "3877": "20676f6c66", + "3878": "2074656e", + "3879": "206e6565646c65", + "3880": "20636861696e", + "3881": "20726176656e", + "3882": "20736f6c64696572", + "3883": "2073776561746572", + "3884": "696f", + "3885": "20726964696e67", + "3886": "6f70686f6e65", + "3887": "2064657373657274", + "3888": "2068656c706c657373", + "3889": "20686f6c6573", + "3890": "546f6d6d79", + "3891": "204a616d6573", + "3892": "206e65696768626f7572", + "3893": "20676c6f6f6d79", + "3894": "2073756363656564", + "3895": "20636f7264", + "3896": "6865617274", + "3897": "20726f6c6c696e67", + "3898": "73747269", + "3899": "61696c61626c65", + "3900": "2073636f6c64", + "3901": "206465636f72617465", + "3902": "706172", + "3903": "20617274", + "3904": "207365706172", + "3905": "207368616b696e67", + "3906": "206e6f626f6479", + "3907": "6b656c6574", + "3908": "2047697665", + "3909": "75626279", + "3910": "207365636f6e64", + "3911": "2062726f63636f6c69", + "3912": "756e64", + "3913": "69736b657273", + "3914": "666665", + "3915": "2068697665", + "3916": "2074726f70", + "3917": "2062696379", + "3918": "20657261736572", + "3919": "2074726f75626c6564", + "3920": "20617865", + "3921": "66666565", + "3922": "207265636f", + "3923": "2074697265", + "3924": "20636f6f6b6564", + "3925": "20436861726c", + "3926": "2063757368696f6e", + "3927": "2068616c6c", + "3928": "206578706c61696e", + "3929": "207377696e67696e67", + "3930": "206c656f70617264", + "3931": "206361756c69666c6f776572", + "3932": "20617661696c61626c65", + "3933": "6369", + "3934": "20417265", + "3935": "2063617473", + "3936": "20636c617070696e67", + "3937": "2073686f757473", + "3938": "206c6976656c79", + "3939": "20576869736b657273", + "3940": "20746872656164", + "3941": "20736872696d70", + "3942": "2062656570", + "3943": "6f7261626c65", + "3944": "206e757473", + "3945": "20776973686573", + "3946": "206c6970", + "3947": "2073746972726564", + "3948": "6173746963", + "3949": "207265737065637466756c", + "3950": "20706173747279", + "3951": "6b656c65746f6e", + "3952": "2074726f706879", + "3953": "676e", + "3954": "2073656e74", + "3955": "2073616e64626f78", + "3956": "20736c65", + "3957": "61726e74", + "3958": "206861646e", + "3959": "20736f6d657768657265", + "3960": "20676976656e", + "3961": "206c6561726e74", + "3962": "2070617773", + "3963": "20706164", + "3964": "7a7a79", + "3965": "2072657065", + "3966": "206b6179", + "3967": "2070756c6c73", + "3968": "756d6d6572", + "3969": "207374617274696e67", + "3970": "20636f6c6f757273", + "3971": "207374657073", + "3972": "206d757368726f6f6d", + "3973": "616c6564", + "3974": "20706f6c697368", + "3975": "616b6564", + "3976": "656e6e79", + "3977": "2061646f7261626c65", + "3978": "72656369", + "3979": "20666f7267617665", + "3980": "65626f6f6b", + "3981": "206e6f7465626f6f6b", + "3982": "207175697465", + "3983": "20666f6f7462616c6c", + "3984": "206c61736572", + "3985": "2071756172", + "3986": "2070757368696e67", + "3987": "617a7a", + "3988": "20526f73", + "3989": "6e69676874", + "3990": "207669", + "3991": "206372797374616c", + "3992": "20636f66666565", + "3993": "2066697665", + "3994": "20676f616c", + "3995": "2063757274", + "3996": "2066726f776e6564", + "3997": "206375727461696e", + "3998": "2061707072656369", + "3999": "207572", + "4000": "20706172616465", + "4001": "47697665", + "4002": "206c697374656e696e67", + "4003": "206d617263686564", + "4004": "2073697a65", + "4005": "20636875626279", + "4006": "2073686f7274", + "4007": "20747269616e67", + "4008": "20736b656c65746f6e", + "4009": "207768656174", + "4010": "676965", + "4011": "20416e6e6965", + "4012": "20646973686573", + "4013": "20707573686573", + "4014": "2069726f6e", + "4015": "2074686561746572", + "4016": "2073616c74", + "4017": "206f62736572", + "4018": "2079616368", + "4019": "2070756e", + "4020": "20706f696e74", + "4021": "2072656566", + "4022": "20636c7562", + "4023": "206f7267616e697a65", + "4024": "68696e6f", + "4025": "2074726970706564", + "4026": "204c696c6c79", + "4027": "206d6f6e6b657973", + "4028": "697070696e67", + "4029": "2070697065", + "4030": "737472696368", + "4031": "206261726b696e67", + "4032": "206d756c65", + "4033": "207265636f676e", + "4034": "20706170657273", + "4035": "2076657374", + "4036": "2076697369746564", + "4037": "2068616972637574", + "4038": "206d7567", + "4039": "6f6f706564", + "4040": "20747572", + "4041": "696e6b696e67", + "4042": "20636865636b6564", + "4043": "206d6f64657374", + "4044": "2074726179", + "4045": "20746178", + "4046": "2064726177696e6773", + "4047": "2073706972", + "4048": "656e7279", + "4049": "20636c696d62696e67", + "4050": "20726573637565", + "4051": "75636b6564", + "4052": "206c75636b", + "4053": "656c657279", + "4054": "206d696372", + "4055": "207961636874", + "4056": "20696e636c", + "4057": "2073756e736574", + "4058": "20756e6c6f636b", + "4059": "20677261706573", + "4060": "20636f7374756d65", + "4061": "2062696379636c65", + "4062": "636b6579", + "4063": "427965", + "4064": "636f6d65", + "4065": "20686f636b6579", + "4066": "206a6f6b6573", + "4067": "206c6964", + "4068": "20656974686572", + "4069": "2068617070696572", + "4070": "61666665", + "4071": "206d6f746f726379636c65", + "4072": "20666f7274", + "4073": "207268696e6f", + "4074": "20737472756767", + "4075": "207368656c746572", + "4076": "2062757a7a", + "4077": "2073746f6d", + "4078": "2062727573", + "4079": "20747269636b73", + "4080": "2042616279", + "4081": "20706f72", + "4082": "2063756265", + "4083": "20747261696e73", + "4084": "207475726b6579", + "4085": "2072696262", + "4086": "2070616e6461", + "4087": "2074617869", + "4088": "206d6963726f70686f6e65", + "4089": "20737065616b", + "4090": "20746f696c", + "4091": "20756e68", + "4092": "6f6c696e", + "4093": "2063616e65", + "4094": "2066696e6473", + "4095": "20626568617665", + "4096": "3c73797374656d3e", + "4097": "3c2f73797374656d3e", + "4098": "3c757365723e", + "4099": "3c2f757365723e", + "4100": "3c617373697374616e743e", + "4101": "3c2f617373697374616e743e" + }, + "vocab_size": 4102, + "merges": [ + [104, 101, 258], + [32, 116, 259], + [32, 97, 260], + [32, 115, 261], + [32, 119, 262], + [110, 100, 263], + [259, 258, 264], + [101, 100, 265], + [260, 263, 266], + [259, 111, 267], + [32, 98, 268], + [105, 110, 269], + [32, 104, 270], + [262, 97, 271], + [114, 101, 272], + [32, 102, 273], + [111, 117, 274], + [105, 116, 275], + [32, 84, 276], + [32, 99, 277], + [32, 108, 278], + [101, 114, 279], + [32, 100, 280], + [32, 258, 281], + [32, 109, 282], + [271, 115, 283], + [32, 112, 284], + [111, 109, 285], + [32, 111, 286], + [97, 121, 287], + [276, 258, 288], + [97, 114, 289], + [105, 115, 290], + [269, 103, 291], + [105, 108, 292], + [32, 103, 293], + [105, 100, 294], + [32, 83, 295], + [97, 116, 296], + [101, 110, 297], + [32, 110, 298], + [261, 97, 299], + [270, 97, 300], + [97, 110, 301], + [105, 109, 302], + [111, 114, 303], + [111, 110, 304], + [32, 275, 305], + [32, 72, 306], + [259, 104, 307], + [108, 108, 308], + [108, 101, 309], + [281, 114, 310], + [101, 116, 311], + [111, 116, 312], + [295, 258, 313], + [105, 114, 314], + [306, 101, 315], + [288, 121, 316], + [101, 115, 317], + [32, 269, 318], + [118, 279, 319], + [117, 116, 320], + [111, 119, 321], + [99, 107, 322], + [32, 101, 323], + [32, 117, 324], + [108, 100, 325], + [32, 76, 326], + [32, 121, 327], + [111, 111, 328], + [299, 294, 329], + [105, 103, 330], + [292, 121, 331], + [32, 114, 332], + [268, 101, 333], + [97, 109, 334], + [261, 116, 335], + [32, 73, 336], + [99, 101, 337], + [261, 258, 338], + [107, 101, 339], + [112, 112, 340], + [118, 101, 341], + [275, 104, 342], + [262, 342, 343], + [79, 110, 344], + [286, 110, 345], + [286, 102, 346], + [107, 265, 347], + [114, 105, 348], + [261, 111, 349], + [326, 331, 350], + [327, 274, 351], + [270, 290, 352], + [110, 116, 353], + [284, 108, 354], + [97, 100, 355], + [319, 121, 356], + [32, 66, 357], + [280, 287, 358], + [307, 296, 359], + [324, 112, 360], + [300, 100, 361], + [115, 116, 362], + [354, 287, 363], + [264, 121, 364], + [262, 101, 365], + [282, 285, 366], + [273, 303, 367], + [101, 108, 368], + [274, 325, 369], + [117, 110, 370], + [109, 121, 371], + [32, 77, 372], + [39, 115, 373], + [13, 10, 374], + [278, 105, 375], + [99, 104, 376], + [297, 116, 377], + [300, 340, 378], + [275, 116, 379], + [271, 353, 380], + [261, 104, 381], + [274, 116, 382], + [258, 114, 383], + [108, 121, 384], + [285, 101, 385], + [32, 356, 386], + [274, 263, 387], + [298, 312, 388], + [280, 111, 389], + [115, 101, 390], + [97, 308, 391], + [262, 104, 392], + [101, 263, 393], + [302, 101, 394], + [379, 309, 395], + [32, 107, 396], + [259, 394, 397], + [278, 395, 398], + [264, 272, 399], + [97, 108, 400], + [104, 116, 401], + [298, 101, 402], + [32, 272, 403], + [378, 121, 404], + [336, 116, 405], + [268, 330, 406], + [32, 65, 407], + [261, 109, 408], + [32, 290, 409], + [268, 320, 410], + [97, 322, 411], + [299, 119, 412], + [260, 110, 413], + [348, 393, 414], + [273, 414, 415], + [114, 97, 416], + [260, 115, 417], + [114, 121, 418], + [46, 34, 419], + [294, 101, 420], + [344, 101, 421], + [97, 339, 422], + [39, 116, 423], + [117, 103, 424], + [118, 265, 425], + [278, 328, 426], + [116, 279, 427], + [365, 272, 428], + [276, 302, 429], + [344, 337, 430], + [111, 272, 431], + [33, 34, 432], + [101, 99, 433], + [278, 111, 434], + [32, 421, 435], + [270, 302, 436], + [268, 111, 437], + [32, 74, 438], + [267, 111, 439], + [293, 111, 440], + [261, 101, 441], + [314, 108, 442], + [380, 265, 443], + [289, 100, 444], + [32, 106, 445], + [293, 442, 446], + [360, 304, 447], + [292, 108, 448], + [32, 382, 449], + [260, 272, 450], + [264, 109, 451], + [102, 117, 452], + [261, 112, 453], + [260, 116, 454], + [119, 287, 455], + [105, 263, 456], + [117, 114, 457], + [264, 314, 458], + [277, 369, 459], + [258, 110, 460], + [280, 294, 461], + [408, 292, 462], + [357, 297, 463], + [97, 269, 464], + [276, 285, 465], + [300, 341, 466], + [323, 120, 467], + [258, 100, 468], + [289, 116, 469], + [260, 114, 470], + [262, 377, 471], + [114, 285, 472], + [105, 99, 473], + [32, 87, 474], + [452, 108, 475], + [104, 291, 476], + [277, 301, 477], + [108, 112, 478], + [415, 115, 479], + [330, 401, 480], + [260, 308, 481], + [396, 110, 482], + [298, 334, 483], + [357, 320, 484], + [328, 100, 485], + [281, 478, 486], + [117, 109, 487], + [63, 34, 488], + [304, 101, 489], + [268, 411, 490], + [289, 107, 491], + [273, 370, 492], + [277, 108, 493], + [97, 272, 494], + [278, 101, 495], + [429, 371, 496], + [261, 287, 497], + [89, 274, 498], + [298, 111, 499], + [111, 112, 500], + [261, 99, 501], + [483, 265, 502], + [274, 103, 503], + [368, 116, 504], + [260, 108, 505], + [434, 425, 506], + [462, 265, 507], + [115, 420, 508], + [105, 322, 509], + [273, 504, 510], + [417, 347, 511], + [282, 301, 512], + [273, 101, 513], + [267, 121, 514], + [470, 387, 515], + [426, 347, 516], + [285, 311, 517], + [375, 339, 518], + [334, 101, 519], + [261, 385, 520], + [441, 101, 521], + [332, 111, 522], + [117, 272, 523], + [282, 101, 524], + [268, 114, 525], + [437, 121, 526], + [262, 369, 527], + [103, 311, 528], + [268, 314, 529], + [335, 469, 530], + [105, 337, 531], + [304, 103, 532], + [261, 517, 533], + [115, 115, 534], + [529, 100, 535], + [262, 303, 536], + [273, 97, 537], + [291, 115, 538], + [355, 101, 539], + [260, 103, 540], + [100, 100, 541], + [107, 291, 542], + [392, 296, 543], + [195, 162, 544], + [130, 172, 545], + [544, 226, 546], + [546, 545, 547], + [282, 422, 548], + [321, 110, 549], + [259, 272, 550], + [260, 455, 551], + [332, 301, 552], + [528, 383, 553], + [105, 101, 554], + [530, 265, 555], + [97, 103, 556], + [44, 34, 557], + [274, 100, 558], + [289, 265, 559], + [282, 539, 560], + [533, 476, 561], + [372, 285, 562], + [105, 102, 563], + [32, 498, 564], + [277, 111, 565], + [267, 553, 566], + [275, 265, 567], + [284, 491, 568], + [116, 383, 569], + [277, 289, 570], + [497, 115, 571], + [32, 69, 572], + [467, 99, 573], + [299, 100, 574], + [284, 320, 575], + [286, 569, 576], + [284, 114, 577], + [402, 119, 578], + [98, 309, 579], + [273, 472, 580], + [278, 97, 581], + [282, 117, 582], + [540, 464, 583], + [32, 70, 584], + [270, 385, 585], + [270, 424, 586], + [101, 308, 587], + [262, 460, 588], + [317, 116, 589], + [392, 111, 590], + [264, 110, 591], + [97, 376, 592], + [280, 433, 593], + [328, 107, 594], + [101, 112, 595], + [293, 485, 596], + [348, 265, 597], + [273, 387, 598], + [340, 265, 599], + [277, 104, 600], + [273, 108, 601], + [97, 115, 602], + [271, 108, 603], + [293, 311, 604], + [113, 117, 605], + [503, 401, 606], + [104, 296, 607], + [363, 291, 608], + [391, 121, 609], + [261, 119, 610], + [112, 108, 611], + [307, 538, 612], + [573, 567, 613], + [293, 312, 614], + [274, 115, 615], + [32, 68, 616], + [117, 322, 617], + [294, 265, 618], + [295, 117, 619], + [375, 347, 620], + [110, 110, 621], + [593, 618, 622], + [114, 111, 623], + [323, 356, 624], + [97, 341, 625], + [277, 519, 626], + [268, 108, 627], + [295, 334, 628], + [97, 120, 629], + [280, 549, 630], + [345, 101, 631], + [501, 559, 632], + [351, 114, 633], + [277, 494, 634], + [32, 118, 635], + [117, 362, 636], + [333, 99, 637], + [273, 456, 638], + [307, 301, 639], + [274, 390, 640], + [290, 116, 641], + [289, 97, 642], + [280, 355, 643], + [513, 368, 644], + [389, 103, 645], + [110, 121, 646], + [260, 98, 647], + [97, 112, 648], + [293, 416, 649], + [289, 110, 650], + [455, 115, 651], + [32, 339, 652], + [461, 110, 653], + [293, 114, 654], + [102, 101, 655], + [279, 115, 656], + [621, 97, 657], + [274, 114, 658], + [505, 651, 659], + [439, 107, 660], + [301, 116, 661], + [317, 115, 662], + [282, 121, 663], + [278, 312, 664], + [284, 111, 665], + [647, 382, 666], + [426, 107, 667], + [366, 371, 668], + [372, 629, 669], + [263, 279, 670], + [262, 448, 671], + [295, 111, 672], + [482, 101, 673], + [433, 105, 674], + [550, 101, 675], + [98, 98, 676], + [449, 508, 677], + [268, 117, 678], + [268, 391, 679], + [514, 115, 680], + [99, 468, 681], + [307, 606, 682], + [290, 101, 683], + [275, 101, 684], + [474, 460, 685], + [297, 265, 686], + [318, 116, 687], + [259, 97, 688], + [282, 431, 689], + [495, 650, 690], + [270, 111, 691], + [482, 321, 692], + [674, 400, 693], + [453, 693, 694], + [284, 272, 695], + [286, 325, 696], + [103, 265, 697], + [284, 117, 698], + [295, 642, 699], + [259, 422, 700], + [581, 424, 701], + [407, 657, 702], + [673, 119, 703], + [306, 279, 704], + [541, 297, 705], + [284, 101, 706], + [261, 303, 707], + [105, 341, 708], + [413, 121, 709], + [97, 117, 710], + [270, 321, 711], + [109, 97, 712], + [102, 427, 713], + [277, 296, 714], + [259, 114, 715], + [381, 321, 716], + [705, 384, 717], + [438, 111, 718], + [105, 97, 719], + [572, 356, 720], + [582, 376, 721], + [259, 119, 722], + [261, 108, 723], + [438, 411, 724], + [269, 107, 725], + [332, 370, 726], + [290, 104, 727], + [97, 263, 728], + [270, 640, 729], + [118, 297, 730], + [707, 418, 731], + [300, 263, 732], + [261, 107, 733], + [282, 111, 734], + [286, 112, 735], + [687, 111, 736], + [318, 508, 737], + [261, 370, 738], + [493, 111, 739], + [267, 325, 740], + [286, 114, 741], + [32, 605, 742], + [577, 558, 743], + [259, 597, 744], + [459, 110, 745], + [271, 427, 746], + [293, 625, 747], + [258, 272, 748], + [296, 101, 749], + [402, 319, 750], + [32, 297, 751], + [487, 112, 752], + [286, 319, 753], + [301, 107, 754], + [312, 383, 755], + [467, 611, 756], + [281, 444, 757], + [116, 121, 758], + [32, 563, 759], + [98, 121, 760], + [710, 390, 761], + [97, 337, 762], + [323, 296, 763], + [522, 285, 764], + [103, 101, 765], + [474, 101, 766], + [307, 290, 767], + [619, 717, 768], + [100, 121, 769], + [284, 509, 770], + [105, 304, 771], + [323, 592, 772], + [311, 427, 773], + [363, 265, 774], + [288, 110, 775], + [335, 114, 776], + [584, 472, 777], + [695, 116, 778], + [586, 697, 779], + [437, 120, 780], + [293, 272, 781], + [277, 285, 782], + [97, 107, 783], + [271, 116, 784], + [445, 636, 785], + [637, 761, 786], + [312, 104, 787], + [346, 102, 788], + [298, 531, 789], + [292, 101, 790], + [306, 290, 791], + [117, 101, 792], + [402, 265, 793], + [445, 111, 794], + [105, 122, 795], + [408, 391, 796], + [333, 289, 797], + [278, 532, 798], + [115, 265, 799], + [372, 719, 800], + [512, 121, 801], + [407, 263, 802], + [118, 291, 803], + [117, 99, 804], + [273, 111, 805], + [324, 353, 806], + [39, 109, 807], + [503, 104, 808], + [303, 116, 809], + [261, 110, 810], + [806, 292, 811], + [259, 418, 812], + [269, 101, 813], + [292, 100, 814], + [115, 368, 815], + [302, 400, 816], + [690, 265, 817], + [413, 816, 818], + [103, 418, 819], + [268, 773, 820], + [32, 67, 821], + [101, 109, 822], + [457, 116, 823], + [32, 302, 824], + [277, 391, 825], + [102, 116, 826], + [323, 263, 827], + [434, 341, 828], + [97, 320, 829], + [98, 279, 830], + [280, 272, 831], + [396, 456, 832], + [289, 115, 833], + [260, 100, 834], + [601, 321, 835], + [355, 121, 836], + [457, 110, 837], + [268, 589, 838], + [277, 309, 839], + [118, 377, 840], + [333, 829, 841], + [639, 347, 842], + [457, 112, 843], + [444, 297, 844], + [293, 844, 845], + [268, 416, 846], + [733, 121, 847], + [634, 475, 848], + [112, 116, 849], + [537, 362, 850], + [273, 105, 851], + [308, 121, 852], + [118, 317, 853], + [323, 730, 854], + [280, 304, 855], + [89, 317, 856], + [277, 117, 857], + [701, 468, 858], + [268, 121, 859], + [815, 102, 860], + [277, 258, 861], + [259, 101, 862], + [278, 558, 863], + [664, 115, 864], + [841, 105, 865], + [445, 752, 866], + [269, 121, 867], + [865, 475, 868], + [547, 197, 869], + [869, 147, 870], + [309, 119, 871], + [273, 334, 872], + [271, 121, 873], + [293, 108, 874], + [119, 110, 875], + [105, 317, 876], + [335, 448, 877], + [97, 390, 878], + [277, 385, 879], + [332, 97, 880], + [270, 444, 881], + [278, 641, 882], + [262, 111, 883], + [84, 104, 884], + [97, 362, 885], + [270, 823, 886], + [818, 115, 887], + [335, 287, 888], + [99, 116, 889], + [565, 108, 890], + [433, 116, 891], + [110, 291, 892], + [299, 655, 893], + [115, 104, 894], + [32, 80, 895], + [324, 670, 896], + [403, 109, 897], + [258, 115, 898], + [268, 787, 899], + [98, 101, 900], + [278, 311, 901], + [846, 341, 902], + [407, 115, 903], + [722, 111, 904], + [272, 101, 905], + [328, 108, 906], + [500, 309, 907], + [301, 101, 908], + [375, 425, 909], + [824, 112, 910], + [305, 115, 911], + [268, 355, 912], + [872, 331, 913], + [407, 713, 914], + [290, 468, 915], + [73, 116, 916], + [273, 269, 917], + [804, 121, 918], + [261, 117, 919], + [843, 114, 920], + [107, 287, 921], + [111, 98, 922], + [273, 871, 923], + [111, 322, 924], + [718, 104, 925], + [326, 918, 926], + [825, 265, 927], + [335, 303, 928], + [706, 907, 929], + [603, 347, 930], + [261, 920, 931], + [330, 104, 932], + [381, 369, 933], + [280, 301, 934], + [332, 265, 935], + [273, 384, 936], + [652, 849, 937], + [102, 431, 938], + [108, 265, 939], + [120, 116, 940], + [268, 594, 941], + [349, 304, 942], + [273, 431, 943], + [413, 819, 944], + [290, 265, 945], + [76, 311, 946], + [78, 111, 947], + [284, 473, 948], + [389, 308, 949], + [522, 322, 950], + [839, 301, 951], + [381, 494, 952], + [105, 112, 953], + [440, 291, 954], + [541, 265, 955], + [756, 431, 956], + [277, 304, 957], + [487, 371, 958], + [298, 321, 959], + [547, 226, 960], + [280, 416, 961], + [778, 758, 962], + [834, 840, 963], + [262, 456, 964], + [556, 101, 965], + [735, 686, 966], + [652, 595, 967], + [381, 867, 968], + [333, 938, 969], + [389, 303, 970], + [292, 384, 971], + [32, 71, 972], + [654, 387, 973], + [335, 111, 974], + [403, 400, 975], + [277, 418, 976], + [277, 272, 977], + [499, 683, 978], + [289, 121, 979], + [32, 420, 980], + [273, 114, 981], + [273, 289, 982], + [259, 837, 983], + [616, 355, 984], + [890, 303, 985], + [105, 265, 986], + [323, 121, 987], + [562, 371, 988], + [309, 115, 989], + [115, 111, 990], + [271, 114, 991], + [644, 291, 992], + [770, 265, 993], + [505, 990, 994], + [302, 98, 995], + [980, 97, 996], + [402, 940, 997], + [116, 476, 998], + [925, 110, 999], + [324, 110, 1000], + [603, 542, 1001], + [132, 162, 1002], + [960, 1002, 1003], + [493, 995, 1004], + [87, 607, 1005], + [392, 790, 1006], + [307, 114, 1007], + [111, 121, 1008], + [462, 101, 1009], + [116, 104, 1010], + [268, 265, 1011], + [102, 102, 1012], + [499, 955, 1013], + [357, 922, 1014], + [473, 265, 1015], + [116, 523, 1016], + [101, 311, 1017], + [262, 114, 1018], + [282, 97, 1019], + [963, 523, 1020], + [103, 279, 1021], + [300, 115, 1022], + [97, 99, 1023], + [333, 291, 1024], + [105, 615, 1025], + [287, 900, 1026], + [116, 265, 1027], + [739, 115, 1028], + [280, 563, 1029], + [426, 542, 1030], + [822, 830, 1031], + [106, 1008, 1032], + [32, 78, 1033], + [715, 617, 1034], + [116, 291, 1035], + [293, 708, 1036], + [974, 599, 1037], + [948, 1016, 1038], + [84, 607, 1039], + [295, 112, 1040], + [271, 275, 1041], + [273, 485, 1042], + [884, 754, 1043], + [987, 317, 1044], + [424, 401, 1045], + [107, 317, 1046], + [32, 82, 1047], + [388, 1015, 1048], + [280, 368, 1049], + [307, 725, 1050], + [267, 119, 1051], + [882, 297, 1052], + [97, 579, 1053], + [712, 122, 1054], + [732, 115, 1055], + [897, 1031, 1056], + [603, 107, 1057], + [742, 509, 1058], + [268, 272, 1059], + [260, 1054, 1060], + [781, 296, 1061], + [323, 319, 1062], + [260, 713, 1063], + [525, 111, 1064], + [627, 792, 1065], + [635, 111, 1066], + [286, 875, 1067], + [795, 265, 1068], + [751, 1032, 1069], + [723, 420, 1070], + [403, 611, 1071], + [280, 290, 1072], + [260, 340, 1073], + [314, 362, 1074], + [943, 362, 1075], + [676, 275, 1076], + [910, 809, 1077], + [111, 304, 1078], + [536, 107, 1079], + [835, 656, 1080], + [308, 321, 1081], + [883, 670, 1082], + [332, 464, 1083], + [716, 265, 1084], + [624, 489, 1085], + [402, 289, 1086], + [381, 382, 1087], + [103, 301, 1088], + [273, 727, 1089], + [260, 99, 1090], + [776, 532, 1091], + [281, 355, 1092], + [1077, 661, 1093], + [784, 376, 1094], + [880, 1076, 1095], + [676, 265, 1096], + [524, 301, 1097], + [76, 331, 1098], + [655, 272, 1099], + [1058, 384, 1100], + [259, 400, 1101], + [324, 115, 1102], + [525, 480, 1103], + [437, 296, 1104], + [1029, 1099, 1105], + [286, 921, 1106], + [280, 101, 1107], + [760, 101, 1108], + [261, 387, 1109], + [332, 480, 1110], + [784, 681, 1111], + [1071, 986, 1112], + [100, 287, 1113], + [267, 117, 1114], + [619, 101, 1115], + [335, 509, 1116], + [805, 1081, 1117], + [309, 878, 1118], + [324, 390, 1119], + [279, 265, 1120], + [262, 748, 1121], + [354, 762, 1122], + [417, 107, 1123], + [831, 534, 1124], + [261, 523, 1125], + [327, 101, 1126], + [332, 117, 1127], + [301, 765, 1128], + [282, 303, 1129], + [335, 431, 1130], + [296, 104, 1131], + [87, 101, 1132], + [327, 587, 1133], + [438, 908, 1134], + [333, 297, 1135], + [857, 348, 1136], + [1105, 353, 1137], + [1040, 312, 1138], + [537, 337, 1139], + [637, 519, 1140], + [261, 605, 1141], + [1066, 531, 1142], + [1028, 279, 1143], + [699, 104, 1144], + [600, 814, 1145], + [259, 416, 1146], + [99, 111, 1147], + [270, 932, 1148], + [596, 1108, 1149], + [32, 658, 1150], + [110, 103, 1151], + [270, 500, 1152], + [263, 712, 1153], + [277, 114, 1154], + [97, 1045, 1155], + [991, 109, 1156], + [274, 353, 1157], + [282, 556, 1158], + [1136, 615, 1159], + [389, 317, 1160], + [276, 104, 1161], + [298, 480, 1162], + [309, 595, 1163], + [97, 681, 1164], + [76, 594, 1165], + [259, 587, 1166], + [367, 103, 1167], + [333, 108, 1168], + [292, 97, 1169], + [277, 594, 1170], + [284, 464, 1171], + [109, 431, 1172], + [709, 1172, 1173], + [307, 905, 1174], + [277, 422, 1175], + [720, 489, 1176], + [101, 121, 1177], + [117, 308, 1178], + [314, 265, 1179], + [260, 109, 1180], + [735, 297, 1181], + [302, 317, 1182], + [273, 1074, 1183], + [275, 99, 1184], + [97, 534, 1185], + [649, 1096, 1186], + [495, 97, 1187], + [537, 118, 1188], + [957, 116, 1189], + [410, 427, 1190], + [698, 308, 1191], + [627, 924, 1192], + [610, 1017, 1193], + [345, 384, 1194], + [32, 122, 1195], + [268, 275, 1196], + [336, 110, 1197], + [535, 115, 1198], + [281, 272, 1199], + [851, 272, 1200], + [486, 265, 1201], + [396, 294, 1202], + [513, 308, 1203], + [303, 110, 1204], + [378, 686, 1205], + [290, 115, 1206], + [370, 646, 1207], + [67, 301, 1208], + [268, 97, 1209], + [866, 265, 1210], + [280, 114, 1211], + [284, 279, 1212], + [1184, 460, 1213], + [649, 534, 1214], + [396, 1213, 1215], + [290, 121, 1216], + [282, 304, 1217], + [403, 1164, 1218], + [601, 111, 1219], + [299, 263, 1220], + [874, 355, 1221], + [441, 317, 1222], + [928, 121, 1223], + [525, 755, 1224], + [975, 1068, 1225], + [261, 291, 1226], + [340, 121, 1227], + [276, 607, 1228], + [335, 500, 1229], + [300, 803, 1230], + [333, 104, 1231], + [353, 115, 1232], + [77, 285, 1233], + [1007, 808, 1234], + [403, 362, 1235], + [577, 269, 1236], + [403, 836, 1237], + [278, 662, 1238], + [122, 121, 1239], + [296, 279, 1240], + [119, 748, 1241], + [333, 1088, 1242], + [282, 487, 1243], + [97, 314, 1244], + [1087, 265, 1245], + [278, 836, 1246], + [327, 958, 1247], + [349, 826, 1248], + [536, 325, 1249], + [293, 519, 1250], + [616, 111, 1251], + [259, 348, 1252], + [454, 101, 1253], + [269, 115, 1254], + [326, 311, 1255], + [282, 662, 1256], + [284, 494, 1257], + [326, 1169, 1258], + [436, 860, 1259], + [403, 609, 1260], + [961, 119, 1261], + [718, 101, 1262], + [452, 852, 1263], + [896, 362, 1264], + [39, 272, 1265], + [296, 383, 1266], + [280, 617, 1267], + [277, 320, 1268], + [280, 489, 1269], + [851, 120, 1270], + [270, 296, 1271], + [107, 297, 1272], + [259, 1179, 1273], + [1019, 542, 1274], + [783, 317, 1275], + [284, 648, 1276], + [1158, 473, 1277], + [324, 799, 1278], + [362, 279, 1279], + [441, 297, 1280], + [307, 291, 1281], + [1202, 115, 1282], + [782, 291, 1283], + [273, 448, 1284], + [1188, 303, 1285], + [267, 112, 1286], + [778, 393, 1287], + [1285, 684, 1288], + [584, 269, 1289], + [330, 110, 1290], + [536, 418, 1291], + [271, 425, 1292], + [111, 852, 1293], + [688, 362, 1294], + [492, 646, 1295], + [392, 121, 1296], + [665, 263, 1297], + [1190, 102, 1298], + [284, 472, 1299], + [98, 111, 1300], + [714, 376, 1301], + [87, 321, 1302], + [335, 101, 1303], + [105, 377, 1304], + [483, 101, 1305], + [667, 115, 1306], + [265, 769, 1307], + [821, 301, 1308], + [835, 279, 1309], + [1257, 1232, 1310], + [413, 755, 1311], + [300, 314, 1312], + [1114, 376, 1313], + [273, 623, 1314], + [284, 469, 1315], + [268, 1207, 1316], + [295, 609, 1317], + [277, 597, 1318], + [610, 302, 1319], + [1049, 473, 1320], + [117, 1012, 1321], + [109, 279, 1322], + [570, 115, 1323], + [97, 1216, 1324], + [1238, 304, 1325], + [280, 623, 1326], + [931, 683, 1327], + [278, 480, 1328], + [277, 906, 1329], + [280, 269, 1330], + [273, 1178, 1331], + [68, 304, 1332], + [335, 272, 1333], + [268, 556, 1334], + [444, 115, 1335], + [293, 105, 1336], + [536, 347, 1337], + [282, 640, 1338], + [105, 319, 1339], + [296, 771, 1340], + [117, 115, 1341], + [272, 108, 1342], + [104, 121, 1343], + [550, 317, 1344], + [691, 599, 1345], + [1056, 265, 1346], + [1231, 456, 1347], + [489, 121, 1348], + [691, 325, 1349], + [84, 285, 1350], + [1192, 115, 1351], + [781, 297, 1352], + [280, 348, 1353], + [1126, 833, 1354], + [453, 312, 1355], + [639, 107, 1356], + [261, 1163, 1357], + [465, 371, 1358], + [691, 309, 1359], + [381, 111, 1360], + [408, 587, 1361], + [389, 291, 1362], + [726, 892, 1363], + [261, 311, 1364], + [1133, 321, 1365], + [117, 265, 1366], + [1219, 303, 1367], + [262, 485, 1368], + [728, 121, 1369], + [262, 915, 1370], + [277, 416, 1371], + [368, 121, 1372], + [1276, 279, 1373], + [723, 321, 1374], + [280, 491, 1375], + [634, 1263, 1376], + [105, 339, 1377], + [964, 321, 1378], + [34, 46, 1379], + [378, 331, 1380], + [565, 325, 1381], + [793, 265, 1382], + [463, 646, 1383], + [261, 296, 1384], + [259, 391, 1385], + [403, 355, 1386], + [117, 108, 1387], + [831, 334, 1388], + [281, 325, 1389], + [289, 291, 1390], + [931, 945, 1391], + [610, 291, 1392], + [407, 116, 1393], + [272, 110, 1394], + [701, 104, 1395], + [917, 915, 1396], + [572, 109, 1397], + [434, 362, 1398], + [268, 321, 1399], + [1212, 102, 1400], + [1314, 103, 1401], + [734, 341, 1402], + [375, 304, 1403], + [495, 826, 1404], + [1298, 384, 1405], + [1069, 265, 1406], + [616, 1324, 1407], + [270, 448, 1408], + [1018, 532, 1409], + [1117, 265, 1410], + [32, 75, 1411], + [277, 885, 1412], + [709, 998, 1413], + [470, 348, 1414], + [678, 121, 1415], + [1289, 609, 1416], + [260, 314, 1417], + [32, 531, 1418], + [403, 108, 1419], + [810, 321, 1420], + [269, 265, 1421], + [375, 1046, 1422], + [537, 314, 1423], + [272, 265, 1424], + [678, 814, 1425], + [281, 289, 1426], + [1033, 321, 1427], + [310, 860, 1428], + [297, 115, 1429], + [276, 111, 1430], + [270, 294, 1431], + [554, 325, 1432], + [739, 390, 1433], + [99, 662, 1434], + [1189, 269, 1435], + [623, 534, 1436], + [1320, 1025, 1437], + [1145, 1394, 1438], + [280, 314, 1439], + [270, 370, 1440], + [415, 384, 1441], + [115, 121, 1442], + [357, 971, 1443], + [1414, 425, 1444], + [72, 587, 1445], + [1445, 111, 1446], + [1082, 475, 1447], + [323, 108, 1448], + [602, 523, 1449], + [470, 109, 1450], + [282, 480, 1451], + [1264, 728, 1452], + [1167, 312, 1453], + [688, 579, 1454], + [1209, 760, 1455], + [270, 312, 1456], + [293, 489, 1457], + [260, 579, 1458], + [259, 480, 1459], + [985, 115, 1460], + [438, 302, 1461], + [284, 1118, 1462], + [268, 424, 1463], + [438, 297, 1464], + [110, 279, 1465], + [100, 115, 1466], + [977, 296, 1467], + [288, 272, 1468], + [261, 532, 1469], + [1284, 265, 1470], + [972, 416, 1471], + [108, 102, 1472], + [33, 547, 1473], + [941, 115, 1474], + [1448, 390, 1475], + [520, 489, 1476], + [282, 296, 1477], + [501, 272, 1478], + [1187, 853, 1479], + [1064, 1272, 1480], + [293, 377, 1481], + [1435, 1366, 1482], + [602, 107, 1483], + [277, 1157, 1484], + [1161, 290, 1485], + [103, 304, 1486], + [649, 1153, 1487], + [540, 1424, 1488], + [284, 296, 1489], + [115, 112, 1490], + [327, 317, 1491], + [105, 376, 1492], + [271, 308, 1493], + [1412, 309, 1494], + [97, 118, 1495], + [372, 1293, 1496], + [1059, 783, 1497], + [888, 265, 1498], + [39, 308, 1499], + [46, 547, 1500], + [623, 285, 1501], + [380, 115, 1502], + [115, 468, 1503], + [293, 117, 1504], + [751, 808, 1505], + [109, 301, 1506], + [1060, 291, 1507], + [604, 1035, 1508], + [284, 725, 1509], + [665, 269, 1510], + [635, 290, 1511], + [97, 340, 1512], + [1129, 892, 1513], + [114, 321, 1514], + [348, 317, 1515], + [602, 104, 1516], + [314, 1342, 1517], + [407, 371, 1518], + [366, 377, 1519], + [79, 75, 1520], + [917, 609, 1521], + [282, 289, 1522], + [582, 115, 1523], + [282, 755, 1524], + [501, 979, 1525], + [274, 112, 1526], + [392, 684, 1527], + [782, 112, 1528], + [1330, 1465, 1529], + [812, 291, 1530], + [473, 121, 1531], + [445, 117, 1532], + [284, 105, 1533], + [513, 119, 1534], + [273, 457, 1535], + [372, 1026, 1536], + [334, 97, 1537], + [1141, 1517, 1538], + [101, 272, 1539], + [282, 311, 1540], + [1321, 121, 1541], + [1004, 265, 1542], + [273, 391, 1543], + [533, 1182, 1544], + [114, 265, 1545], + [679, 1078, 1546], + [297, 337, 1547], + [301, 337, 1548], + [1400, 891, 1549], + [99, 265, 1550], + [1038, 115, 1551], + [273, 480, 1552], + [97, 599, 1553], + [983, 265, 1554], + [522, 308, 1555], + [776, 1128, 1556], + [304, 115, 1557], + [1440, 819, 1558], + [525, 606, 1559], + [262, 311, 1560], + [1170, 876, 1561], + [438, 448, 1562], + [111, 285, 1563], + [299, 292, 1564], + [333, 592, 1565], + [303, 291, 1566], + [114, 312, 1567], + [453, 108, 1568], + [108, 99, 1569], + [794, 121, 1570], + [576, 115, 1571], + [643, 769, 1572], + [462, 317, 1573], + [493, 312, 1574], + [882, 686, 1575], + [283, 110, 1576], + [474, 607, 1577], + [1101, 107, 1578], + [1574, 898, 1579], + [698, 1227, 1580], + [296, 265, 1581], + [665, 108, 1582], + [100, 279, 1583], + [101, 101, 1584], + [624, 998, 1585], + [282, 1275, 1586], + [977, 334, 1587], + [441, 289, 1588], + [334, 112, 1589], + [453, 491, 1590], + [101, 319, 1591], + [84, 258, 1592], + [332, 420, 1593], + [1368, 115, 1594], + [119, 108, 1595], + [688, 292, 1596], + [1060, 265, 1597], + [1523, 473, 1598], + [889, 303, 1599], + [271, 567, 1600], + [290, 427, 1601], + [672, 304, 1602], + [261, 275, 1603], + [505, 489, 1604], + [122, 122, 1605], + [321, 279, 1606], + [307, 272, 1607], + [111, 115, 1608], + [311, 279, 1609], + [934, 1021, 1610], + [312, 116, 1611], + [617, 121, 1612], + [407, 308, 1613], + [261, 971, 1614], + [1107, 595, 1615], + [284, 311, 1616], + [334, 265, 1617], + [284, 554, 1618], + [79, 104, 1619], + [1511, 275, 1620], + [278, 422, 1621], + [109, 265, 1622], + [259, 1307, 1623], + [1191, 265, 1624], + [495, 625, 1625], + [339, 121, 1626], + [322, 311, 1627], + [654, 321, 1628], + [262, 304, 1629], + [270, 420, 1630], + [109, 101, 1631], + [500, 101, 1632], + [424, 103, 1633], + [301, 103, 1634], + [534, 265, 1635], + [936, 291, 1636], + [884, 290, 1637], + [584, 108, 1638], + [1123, 115, 1639], + [1051, 1335, 1640], + [273, 1432, 1641], + [493, 558, 1642], + [261, 420, 1643], + [581, 263, 1644], + [334, 317, 1645], + [354, 301, 1646], + [105, 120, 1647], + [550, 296, 1648], + [805, 120, 1649], + [1299, 945, 1650], + [464, 265, 1651], + [66, 320, 1652], + [406, 1021, 1653], + [525, 301, 1654], + [433, 317, 1655], + [261, 433, 1656], + [1171, 116, 1657], + [330, 103, 1658], + [1010, 121, 1659], + [1533, 1655, 1660], + [290, 317, 1661], + [99, 309, 1662], + [550, 1449, 1663], + [261, 1601, 1664], + [103, 115, 1665], + [536, 597, 1666], + [726, 115, 1667], + [375, 803, 1668], + [1374, 384, 1669], + [627, 754, 1670], + [77, 1026, 1671], + [323, 109, 1672], + [1129, 400, 1673], + [1168, 554, 1674], + [381, 559, 1675], + [105, 311, 1676], + [284, 297, 1677], + [1050, 115, 1678], + [381, 648, 1679], + [79, 921, 1680], + [403, 592, 1681], + [654, 749, 1682], + [117, 609, 1683], + [794, 98, 1684], + [949, 115, 1685], + [961, 1486, 1686], + [1670, 311, 1687], + [115, 119, 1688], + [282, 1348, 1689], + [934, 337, 1690], + [365, 289, 1691], + [850, 279, 1692], + [102, 809, 1693], + [1495, 121, 1694], + [108, 291, 1695], + [332, 762, 1696], + [1067, 279, 1697], + [413, 1688, 1698], + [1315, 121, 1699], + [354, 661, 1700], + [919, 376, 1701], + [857, 112, 1702], + [112, 104, 1703], + [370, 376, 1704], + [284, 312, 1705], + [109, 291, 1706], + [742, 1676, 1707], + [268, 1611, 1708], + [1682, 475, 1709], + [810, 411, 1710], + [32, 341, 1711], + [624, 1241, 1712], + [272, 116, 1713], + [985, 475, 1714], + [111, 108, 1715], + [117, 275, 1716], + [602, 265, 1717], + [365, 1569, 1718], + [582, 362, 1719], + [734, 362, 1720], + [1277, 400, 1721], + [309, 534, 1722], + [105, 272, 1723], + [794, 269, 1724], + [1217, 1279, 1725], + [1233, 371, 1726], + [323, 309, 1727], + [277, 1369, 1728], + [268, 289, 1729], + [600, 111, 1730], + [382, 104, 1731], + [115, 311, 1732], + [282, 1731, 1733], + [298, 648, 1734], + [577, 111, 1735], + [293, 563, 1736], + [1236, 1434, 1737], + [984, 769, 1738], + [627, 411, 1739], + [122, 101, 1740], + [1064, 339, 1741], + [982, 1322, 1742], + [268, 1377, 1743], + [1303, 599, 1744], + [101, 289, 1745], + [438, 636, 1746], + [332, 1339, 1747], + [105, 599, 1748], + [303, 109, 1749], + [1610, 615, 1750], + [372, 1537, 1751], + [976, 291, 1752], + [486, 291, 1753], + [372, 114, 1754], + [270, 330, 1755], + [1090, 1436, 1756], + [284, 97, 1757], + [445, 101, 1758], + [299, 1631, 1759], + [1072, 1512, 1760], + [98, 579, 1761], + [261, 1290, 1762], + [454, 116, 1763], + [408, 469, 1764], + [389, 1599, 1765], + [294, 377, 1766], + [1090, 99, 1767], + [83, 523, 1768], + [282, 1647, 1769], + [303, 265, 1770], + [335, 117, 1771], + [281, 1694, 1772], + [722, 1254, 1773], + [981, 1716, 1774], + [862, 1023, 1775], + [410, 116, 1776], + [101, 559, 1777], + [372, 979, 1778], + [262, 269, 1779], + [527, 110, 1780], + [1439, 758, 1781], + [570, 1567, 1782], + [333, 101, 1783], + [536, 1466, 1784], + [335, 617, 1785], + [1051, 279, 1786], + [87, 1343, 1787], + [739, 799, 1788], + [1399, 108, 1789], + [453, 114, 1790], + [1607, 119, 1791], + [1397, 712, 1792], + [1483, 311, 1793], + [284, 1185, 1794], + [1674, 341, 1795], + [763, 291, 1796], + [275, 291, 1797], + [268, 1131, 1798], + [281, 400, 1799], + [1146, 269, 1800], + [268, 1793, 1801], + [1755, 383, 1802], + [565, 308, 1803], + [268, 617, 1804], + [376, 906, 1805], + [267, 875, 1806], + [268, 684, 1807], + [281, 469, 1808], + [98, 114, 1809], + [1152, 265, 1810], + [270, 275, 1811], + [493, 1553, 1812], + [284, 885, 1813], + [284, 411, 1814], + [1471, 1153, 1815], + [72, 105, 1816], + [1461, 371, 1817], + [1217, 1626, 1818], + [1775, 383, 1819], + [261, 1805, 1820], + [343, 382, 1821], + [861, 1120, 1822], + [303, 100, 1823], + [267, 1113, 1824], + [1326, 341, 1825], + [644, 115, 1826], + [322, 265, 1827], + [100, 101, 1828], + [284, 656, 1829], + [698, 1503, 1830], + [277, 658, 1831], + [840, 1683, 1832], + [1154, 287, 1833], + [665, 1627, 1834], + [335, 728, 1835], + [487, 98, 1836], + [402, 932, 1837], + [919, 717, 1838], + [1767, 1766, 1839], + [340, 291, 1840], + [1423, 121, 1841], + [1638, 1541, 1842], + [367, 528, 1843], + [408, 111, 1844], + [1197, 508, 1845], + [303, 1514, 1846], + [282, 269, 1847], + [525, 549, 1848], + [798, 279, 1849], + [110, 1177, 1850], + [416, 889, 1851], + [1837, 98, 1852], + [327, 444, 1853], + [105, 279, 1854], + [474, 104, 1855], + [273, 1547, 1856], + [1481, 309, 1857], + [303, 115, 1858], + [262, 538, 1859], + [1831, 390, 1860], + [372, 487, 1861], + [273, 905, 1862], + [756, 1651, 1863], + [66, 297, 1864], + [99, 121, 1865], + [378, 297, 1866], + [1157, 464, 1867], + [278, 304, 1868], + [600, 1244, 1869], + [277, 1204, 1870], + [445, 658, 1871], + [280, 418, 1872], + [610, 538, 1873], + [499, 1661, 1874], + [290, 799, 1875], + [1718, 385, 1876], + [101, 355, 1877], + [1868, 1372, 1878], + [32, 344, 1879], + [299, 1151, 1880], + [116, 115, 1881], + [983, 115, 1882], + [261, 1526, 1883], + [1020, 115, 1884], + [63, 547, 1885], + [505, 532, 1886], + [293, 1645, 1887], + [1703, 661, 1888], + [715, 274, 1889], + [1656, 1713, 1890], + [360, 1732, 1891], + [318, 362, 1892], + [354, 1023, 1893], + [278, 749, 1894], + [341, 108, 1895], + [105, 579, 1896], + [950, 115, 1897], + [381, 594, 1898], + [525, 291, 1899], + [1618, 337, 1900], + [1007, 321, 1901], + [109, 377, 1902], + [1326, 599, 1903], + [121, 444, 1904], + [452, 799, 1905], + [268, 297, 1906], + [1727, 1888, 1907], + [381, 500, 1908], + [688, 542, 1909], + [490, 1904, 1910], + [1073, 309, 1911], + [756, 1566, 1912], + [610, 334, 1913], + [65, 657, 1914], + [467, 112, 1915], + [1211, 725, 1916], + [286, 1595, 1917], + [379, 291, 1918], + [288, 314, 1919], + [99, 898, 1920], + [335, 833, 1921], + [441, 97, 1922], + [586, 115, 1923], + [1360, 317, 1924], + [928, 876, 1925], + [97, 292, 1926], + [734, 425, 1927], + [276, 119, 1928], + [1226, 291, 1929], + [402, 322, 1930], + [351, 1151, 1931], + [1168, 532, 1932], + [280, 641, 1933], + [262, 258, 1934], + [277, 1155, 1935], + [335, 289, 1936], + [99, 317, 1937], + [278, 1704, 1938], + [1804, 311, 1939], + [445, 289, 1940], + [119, 444, 1941], + [259, 867, 1942], + [1419, 554, 1943], + [521, 1622, 1944], + [298, 1155, 1945], + [262, 683, 1946], + [924, 265, 1947], + [277, 274, 1948], + [1945, 121, 1949], + [262, 727, 1950], + [98, 321, 1951], + [101, 107, 1952], + [1019, 292, 1953], + [771, 115, 1954], + [365, 308, 1955], + [676, 121, 1956], + [1086, 760, 1957], + [260, 102, 1958], + [32, 317, 1959], + [308, 111, 1960], + [648, 101, 1961], + [326, 1240, 1962], + [628, 371, 1963], + [274, 110, 1964], + [741, 1128, 1965], + [280, 317, 1966], + [652, 121, 1967], + [883, 339, 1968], + [277, 97, 1969], + [307, 808, 1970], + [370, 103, 1971], + [1333, 311, 1972], + [284, 104, 1973], + [282, 1206, 1974], + [1532, 531, 1975], + [1654, 376, 1976], + [32, 79, 1977], + [318, 118, 1978], + [1147, 108, 1979], + [340, 279, 1980], + [269, 291, 1981], + [1679, 317, 1982], + [32, 51, 1983], + [678, 115, 1984], + [499, 390, 1985], + [678, 1442, 1986], + [416, 294, 1987], + [1161, 754, 1988], + [298, 589, 1989], + [1958, 1987, 1990], + [319, 265, 1991], + [267, 109, 1992], + [1083, 1951, 1993], + [403, 1490, 1994], + [286, 98, 1995], + [275, 121, 1996], + [1155, 279, 1997], + [261, 509, 1998], + [68, 111, 1999], + [268, 296, 2000], + [1450, 115, 2001], + [284, 843, 2002], + [99, 291, 2003], + [284, 330, 2004], + [355, 321, 2005], + [277, 314, 2006], + [1126, 289, 2007], + [1082, 265, 2008], + [943, 319, 2009], + [99, 279, 2010], + [388, 476, 2011], + [1943, 425, 2012], + [116, 111, 2013], + [396, 684, 2014], + [332, 387, 2015], + [570, 418, 2016], + [104, 111, 2017], + [407, 108, 2018], + [982, 109, 2019], + [282, 355, 2020], + [586, 101, 2021], + [1906, 376, 2022], + [392, 641, 2023], + [294, 765, 2024], + [261, 1918, 2025], + [688, 1046, 2026], + [396, 291, 2027], + [1268, 101, 2028], + [658, 115, 2029], + [1928, 1017, 2030], + [280, 1997, 2031], + [362, 291, 2032], + [1319, 1706, 2033], + [2002, 309, 2034], + [281, 1960, 2035], + [282, 491, 2036], + [1010, 1113, 2037], + [1353, 319, 2038], + [706, 762, 2039], + [282, 813, 2040], + [1109, 115, 2041], + [309, 116, 2042], + [573, 1797, 2043], + [277, 321, 2044], + [782, 317, 2045], + [111, 390, 2046], + [1489, 1304, 2047], + [687, 1539, 2048], + [275, 376, 2049], + [323, 103, 2050], + [1871, 1850, 2051], + [379, 1429, 2052], + [1114, 681, 2053], + [273, 275, 2054], + [453, 1078, 2055], + [375, 826, 2056], + [375, 655, 2057], + [1736, 116, 2058], + [565, 296, 2059], + [1072, 1147, 2060], + [282, 292, 2061], + [317, 101, 2062], + [894, 813, 2063], + [701, 476, 2064], + [474, 342, 2065], + [1349, 291, 2066], + [268, 728, 2067], + [345, 2013, 2068], + [314, 116, 2069], + [358, 115, 2070], + [278, 773, 2071], + [824, 556, 2072], + [102, 727, 2073], + [1467, 523, 2074], + [262, 416, 2075], + [1300, 444, 2076], + [1839, 609, 2077], + [268, 1964, 2078], + [273, 291, 2079], + [565, 1239, 2080], + [83, 642, 2081], + [260, 889, 2082], + [738, 2063, 2083], + [403, 116, 2084], + [782, 1693, 2085], + [318, 99, 2086], + [84, 302, 2087], + [112, 265, 2088], + [379, 297, 2089], + [110, 662, 2090], + [293, 1658, 2091], + [284, 1606, 2092], + [1299, 683, 2093], + [273, 301, 2094], + [529, 2037, 2095], + [282, 641, 2096], + [738, 646, 2097], + [2084, 837, 2098], + [277, 625, 2099], + [695, 115, 2100], + [1892, 1877, 2101], + [396, 509, 2102], + [1059, 355, 2103], + [72, 1177, 2104], + [277, 101, 2105], + [79, 102, 2106], + [323, 289, 2107], + [98, 391, 2108], + [1510, 1027, 2109], + [709, 489, 2110], + [338, 308, 2111], + [103, 697, 2112], + [649, 98, 2113], + [1152, 101, 2114], + [400, 109, 2115], + [821, 385, 2116], + [278, 1612, 2117], + [1803, 891, 2118], + [278, 1240, 2119], + [1294, 121, 2120], + [270, 303, 2121], + [111, 103, 2122], + [114, 387, 2123], + [1588, 681, 2124], + [1094, 291, 2125], + [311, 104, 2126], + [928, 109, 2127], + [462, 291, 2128], + [1776, 304, 2129], + [268, 606, 2130], + [293, 1266, 2131], + [115, 99, 2132], + [111, 341, 2133], + [579, 109, 2134], + [795, 101, 2135], + [698, 541, 2136], + [1735, 2134, 2137], + [289, 112, 2138], + [2061, 107, 2139], + [861, 2062, 2140], + [323, 602, 2141], + [2048, 2032, 2142], + [332, 1632, 2143], + [116, 297, 2144], + [1889, 579, 2145], + [381, 2138, 2146], + [453, 294, 2147], + [1464, 646, 2148], + [381, 1981, 2149], + [1336, 853, 2150], + [97, 457, 2151], + [453, 111, 2152], + [572, 308, 2153], + [32, 377, 2154], + [821, 104, 2155], + [834, 109, 2156], + [1267, 115, 2157], + [1101, 347, 2158], + [678, 894, 2159], + [2100, 377, 2160], + [1829, 304, 2161], + [1220, 119, 2162], + [1708, 309, 2163], + [453, 269, 2164], + [742, 589, 2165], + [117, 353, 2166], + [262, 420, 2167], + [1420, 1506, 2168], + [368, 115, 2169], + [295, 517, 2170], + [260, 112, 2171], + [117, 98, 2172], + [354, 908, 2173], + [1195, 1563, 2174], + [524, 311, 2175], + [1361, 265, 2176], + [261, 804, 2177], + [828, 384, 2178], + [304, 116, 2179], + [522, 355, 2180], + [284, 1131, 2181], + [1477, 427, 2182], + [284, 1851, 2183], + [396, 1206, 2184], + [290, 97, 2185], + [839, 319, 2186], + [1073, 989, 2187], + [285, 301, 2188], + [487, 830, 2189], + [595, 393, 2190], + [1642, 115, 2191], + [1397, 331, 2192], + [262, 2188, 2193], + [101, 297, 2194], + [277, 444, 2195], + [1555, 265, 2196], + [1799, 1659, 2197], + [299, 102, 2198], + [575, 115, 2199], + [934, 1550, 2200], + [1355, 1027, 2201], + [111, 263, 2202], + [981, 2179, 2203], + [270, 1348, 2204], + [261, 457, 2205], + [1336, 661, 2206], + [284, 289, 2207], + [284, 790, 2208], + [321, 265, 2209], + [299, 322, 2210], + [635, 448, 2211], + [577, 312, 2212], + [259, 370, 2213], + [861, 279, 2214], + [71, 485, 2215], + [381, 114, 2216], + [306, 321, 2217], + [1833, 1557, 2218], + [101, 98, 2219], + [270, 304, 2220], + [284, 448, 2221], + [663, 1279, 2222], + [649, 263, 2223], + [392, 1492, 2224], + [698, 1605, 2225], + [372, 2052, 2226], + [513, 1266, 2227], + [112, 97, 2228], + [2212, 891, 2229], + [278, 287, 2230], + [2121, 390, 2231], + [376, 291, 2232], + [434, 853, 2233], + [1893, 317, 2234], + [1195, 328, 2235], + [295, 116, 2236], + [262, 814, 2237], + [328, 390, 2238], + [1294, 265, 2239], + [565, 269, 2240], + [1073, 1777, 2241], + [1425, 291, 2242], + [590, 309, 2243], + [282, 265, 2244], + [278, 355, 2245], + [284, 500, 2246], + [741, 1088, 2247], + [279, 116, 2248], + [379, 121, 2249], + [1430, 553, 2250], + [1252, 112, 2251], + [323, 556, 2252], + [74, 111, 2253], + [1994, 891, 2254], + [97, 309, 2255], + [417, 1163, 2256], + [1672, 98, 2257], + [1478, 1617, 2258], + [365, 1390, 2259], + [1018, 684, 2260], + [723, 294, 2261], + [582, 100, 2262], + [307, 269, 2263], + [279, 384, 2264], + [1387, 289, 2265], + [1698, 279, 2266], + [337, 301, 2267], + [282, 1867, 2268], + [286, 2267, 2269], + [453, 377, 2270], + [67, 385, 2271], + [441, 296, 2272], + [1439, 116, 2273], + [910, 272, 2274], + [957, 1905, 2275], + [536, 109, 2276], + [1170, 554, 2277], + [326, 2185, 2278], + [810, 422, 2279], + [1535, 418, 2280], + [117, 112, 2281], + [268, 491, 2282], + [322, 115, 2283], + [1847, 320, 2284], + [1948, 376, 2285], + [32, 1290, 2286], + [282, 592, 2287], + [635, 400, 2288], + [919, 340, 2289], + [1870, 279, 2290], + [1568, 602, 2291], + [2245, 1583, 2292], + [805, 312, 2293], + [438, 422, 2294], + [2147, 279, 2295], + [280, 105, 2296], + [2257, 289, 2297], + [289, 765, 2298], + [407, 621, 2299], + [1481, 384, 2300], + [1852, 303, 2301], + [110, 473, 2302], + [282, 456, 2303], + [537, 1937, 2304], + [2297, 416, 2305], + [948, 2302, 2306], + [286, 730, 2307], + [1287, 265, 2308], + [486, 475, 2309], + [862, 592, 2310], + [284, 301, 2311], + [440, 317, 2312], + [108, 97, 2313], + [563, 101, 2314], + [795, 122, 2315], + [1049, 480, 2316], + [1101, 542, 2317], + [375, 341, 2318], + [268, 301, 2319], + [309, 120, 2320], + [581, 362, 2321], + [513, 311, 2322], + [277, 469, 2323], + [2170, 1182, 2324], + [588, 1591, 2325], + [1973, 489, 2326], + [1353, 341, 2327], + [261, 279, 2328], + [111, 325, 2329], + [370, 107, 2330], + [299, 108, 2331], + [268, 871, 2332], + [304, 317, 2333], + [326, 101, 2334], + [621, 121, 2335], + [272, 355, 2336], + [323, 1832, 2337], + [357, 117, 2338], + [439, 108, 2339], + [1761, 115, 2340], + [1041, 291, 2341], + [895, 1609, 2342], + [271, 341, 2343], + [453, 2336, 2344], + [357, 101, 2345], + [897, 456, 2346], + [109, 111, 2347], + [890, 658, 2348], + [1252, 317, 2349], + [698, 894, 2350], + [309, 1809, 2351], + [400, 115, 2352], + [260, 353, 2353], + [277, 2115, 2354], + [259, 294, 2355], + [417, 104, 2356], + [278, 2298, 2357], + [1730, 1979, 2358], + [2105, 2351, 2359], + [403, 99, 2360], + [917, 727, 2361], + [1167, 708, 2362], + [673, 101, 2363], + [972, 111, 2364], + [1992, 1846, 2365], + [1270, 265, 2366], + [1844, 787, 2367], + [381, 2069, 2368], + [2319, 301, 2369], + [335, 416, 2370], + [2305, 1635, 2371], + [284, 658, 2372], + [522, 98, 2373], + [296, 111, 2374], + [284, 309, 2375], + [1757, 119, 2376], + [862, 2126, 2377], + [1552, 291, 2378], + [1047, 101, 2379], + [381, 121, 2380], + [1469, 115, 2381], + [1431, 291, 2382], + [805, 325, 2383], + [333, 308, 2384], + [1790, 287, 2385], + [332, 291, 2386], + [457, 390, 2387], + [477, 110, 2388], + [270, 487, 2389], + [276, 1307, 2390], + [720, 1241, 2391], + [493, 787, 2392], + [862, 97, 2393], + [284, 906, 2394], + [950, 311, 2395], + [1855, 790, 2396], + [1467, 708, 2397], + [925, 2335, 2398], + [1333, 116, 2399], + [600, 509, 2400], + [2211, 965, 2401], + [330, 468, 2402], + [277, 328, 2403], + [1971, 309, 2404], + [261, 368, 2405], + [2388, 312, 2406], + [1672, 112, 2407], + [2358, 749, 2408], + [710, 108, 2409], + [396, 2089, 2410], + [298, 311, 2411], + [1387, 116, 2412], + [2407, 758, 2413], + [100, 765, 2414], + [277, 1436, 2415], + [1127, 1828, 2416], + [619, 115, 2417], + [2023, 309, 2418], + [1294, 101, 2419], + [445, 2404, 2420], + [293, 297, 2421], + [349, 108, 2422], + [269, 1027, 2423], + [1463, 115, 2424], + [453, 101, 2425], + [2136, 309, 2426], + [781, 119, 2427], + [2287, 813, 2428], + [2253, 104, 2429], + [335, 595, 2430], + [528, 1053, 2431], + [600, 1717, 2432], + [100, 297, 2433], + [299, 341, 2434], + [295, 385, 2435], + [1711, 2431, 2436], + [112, 279, 2437], + [1459, 384, 2438], + [524, 2005, 2439], + [83, 116, 2440], + [262, 953, 2441], + [77, 121, 2442], + [335, 314, 2443], + [453, 1531, 2444], + [2373, 312, 2445], + [1729, 347, 2446], + [1187, 102, 2447], + [367, 1941, 2448], + [1244, 115, 2449], + [734, 304, 2450], + [2355, 121, 2451], + [715, 636, 2452], + [1648, 115, 2453], + [1300, 120, 2454], + [282, 1026, 2455], + [330, 279, 2456], + [2039, 475, 2457], + [776, 291, 2458], + [283, 104, 2459], + [262, 431, 2460], + [752, 121, 2461], + [500, 104, 2462], + [501, 289, 2463], + [895, 1118, 2464], + [1782, 115, 2465], + [1760, 111, 2466], + [289, 109, 2467], + [2405, 2073, 2468], + [482, 1947, 2469], + [276, 86, 2470], + [281, 833, 2471], + [260, 118, 2472], + [80, 1118, 2473], + [1930, 108, 2474], + [1236, 337, 2475], + [1431, 2433, 2476], + [261, 658, 2477], + [1568, 1516, 2478], + [110, 265, 2479], + [117, 894, 2480], + [354, 749, 2481], + [372, 958, 2482], + [1590, 384, 2483], + [103, 2123, 2484], + [284, 269, 2485], + [1609, 109, 2486], + [2347, 362, 2487], + [314, 100, 2488], + [280, 2486, 2489], + [32, 90, 2490], + [338, 1472, 2491], + [293, 370, 2492], + [282, 1875, 2493], + [572, 730, 2494], + [636, 114, 2495], + [1371, 98, 2496], + [103, 384, 2497], + [268, 400, 2498], + [524, 661, 2499], + [381, 431, 2500], + [100, 309, 2501], + [866, 291, 2502], + [121, 291, 2503], + [296, 303, 2504], + [2466, 2423, 2505], + [505, 2487, 2506], + [326, 395, 2507], + [1710, 115, 2508], + [324, 2497, 2509], + [495, 109, 2510], + [1166, 115, 2511], + [692, 291, 2512], + [108, 1606, 2513], + [1644, 265, 2514], + [284, 348, 2515], + [282, 312, 2516], + [440, 325, 2517], + [381, 1390, 2518], + [981, 2495, 2519], + [105, 301, 2520], + [403, 112, 2521], + [335, 485, 2522], + [2421, 279, 2523], + [2429, 110, 2524], + [279, 97, 2525], + [513, 289, 2526], + [392, 290, 2527], + [570, 597, 2528], + [654, 1526, 2529], + [278, 813, 2530], + [593, 303, 2531], + [474, 1343, 2532], + [495, 1665, 2533], + [1758, 119, 2534], + [2220, 589, 2535], + [1835, 291, 2536], + [2006, 99, 2537], + [270, 2029, 2538], + [97, 542, 2539], + [303, 418, 2540], + [1133, 265, 2541], + [259, 589, 2542], + [268, 837, 2543], + [2225, 309, 2544], + [1264, 485, 2545], + [301, 111, 2546], + [335, 489, 2547], + [379, 279, 2548], + [1582, 531, 2549], + [39, 100, 2550], + [627, 321, 2551], + [298, 2189, 2552], + [2334, 111, 2553], + [2474, 762, 2554], + [103, 121, 2555], + [635, 301, 2556], + [357, 114, 2557], + [107, 269, 2558], + [587, 121, 2559], + [1116, 115, 2560], + [273, 292, 2561], + [1357, 121, 2562], + [284, 287, 2563], + [2519, 1581, 2564], + [654, 287, 2565], + [2094, 1865, 2566], + [282, 958, 2567], + [277, 348, 2568], + [293, 602, 2569], + [1478, 334, 2570], + [104, 269, 2571], + [281, 108, 2572], + [284, 314, 2573], + [264, 390, 2574], + [2222, 1025, 2575], + [268, 279, 2576], + [1059, 1131, 2577], + [2221, 321, 2578], + [284, 2315, 2579], + [338, 595, 2580], + [118, 615, 2581], + [688, 1045, 2582], + [2440, 500, 2583], + [563, 102, 2584], + [722, 442, 2585], + [1700, 115, 2586], + [2356, 1617, 2587], + [2417, 554, 2588], + [293, 289, 2589], + [441, 265, 2590], + [112, 411, 2591], + [275, 400, 2592], + [277, 1515, 2593], + [499, 1466, 2594], + [271, 853, 2595], + [1582, 684, 2596], + [2018, 531, 2597], + [1794, 265, 2598], + [2085, 1053, 2599], + [831, 119, 2600], + [2098, 265, 2601], + [2177, 1434, 2602], + [97, 102, 2603], + [279, 112, 2604], + [2092, 475, 2605], + [691, 1490, 2606], + [115, 641, 2607], + [1356, 475, 2608], + [665, 303, 2609], + [269, 97, 2610], + [616, 304, 2611], + [1256, 121, 2612], + [2606, 2592, 2613], + [303, 107, 2614], + [2463, 102, 2615], + [83, 334, 2616], + [298, 279, 2617], + [2523, 615, 2618], + [2162, 1492, 2619], + [1011, 1501, 2620], + [403, 103, 2621], + [1608, 2151, 2622], + [601, 320, 2623], + [513, 265, 2624], + [688, 1272, 2625], + [117, 400, 2626], + [795, 1239, 2627], + [521, 107, 2628], + [381, 953, 2629], + [2131, 265, 2630], + [2489, 1421, 2631], + [2617, 2581, 2632], + [261, 2402, 2633], + [2141, 121, 2634], + [495, 103, 2635], + [1429, 708, 2636], + [293, 749, 2637], + [312, 101, 2638], + [1814, 265, 2639], + [1411, 2249, 2640], + [501, 114, 2641], + [2079, 279, 2642], + [111, 1956, 2643], + [2579, 97, 2644], + [1915, 2636, 2645], + [115, 279, 2646], + [448, 265, 2647], + [117, 337, 2648], + [2284, 317, 2649], + [573, 684, 2650], + [493, 549, 2651], + [794, 103, 2652], + [294, 2501, 2653], + [273, 330, 2654], + [298, 320, 2655], + [304, 121, 2656], + [493, 1185, 2657], + [1330, 2622, 2658], + [278, 275, 2659], + [259, 377, 2660], + [2375, 1717, 2661], + [119, 2614, 2662], + [2152, 292, 2663], + [349, 325, 2664], + [467, 334, 2665], + [1211, 487, 2666], + [1028, 311, 2667], + [332, 355, 2668], + [883, 1472, 2669], + [1959, 99, 2670], + [2244, 473, 2671], + [934, 2003, 2672], + [2436, 115, 2673], + [271, 670, 2674], + [268, 1770, 2675], + [874, 1185, 2676], + [269, 662, 2677], + [1698, 1120, 2678], + [270, 457, 2679], + [289, 108, 2680], + [1206, 1858, 2681], + [861, 119, 2682], + [346, 2144, 2683], + [417, 542, 2684], + [2650, 1902, 2685], + [357, 108, 2686], + [633, 860, 2687], + [2671, 813, 2688], + [482, 480, 2689], + [108, 480, 2690], + [501, 2681, 2691], + [372, 491, 2692], + [328, 102, 2693], + [259, 509, 2694], + [268, 489, 2695], + [674, 609, 2696], + [299, 425, 2697], + [691, 1840, 2698], + [396, 1875, 2699], + [635, 878, 2700], + [87, 748, 2701], + [262, 290, 2702], + [1107, 279, 2703], + [1189, 377, 2704], + [536, 542, 2705], + [296, 317, 2706], + [715, 792, 2707], + [84, 748, 2708], + [714, 2604, 2709], + [2709, 448, 2710], + [453, 393, 2711], + [1211, 500, 2712], + [2589, 965, 2713], + [2091, 939, 2714], + [268, 1566, 2715], + [271, 2112, 2716], + [311, 115, 2717], + [1430, 1113, 2718], + [277, 728, 2719], + [862, 833, 2720], + [637, 385, 2721], + [1933, 1548, 2722], + [282, 2653, 2723], + [2050, 1665, 2724], + [861, 362, 2725], + [654, 2461, 2726], + [112, 2696, 2727], + [311, 101, 2728], + [698, 115, 2729], + [335, 1321, 2730], + [1959, 2727, 2731], + [1419, 629, 2732], + [1360, 312, 2733], + [2096, 422, 2734], + [345, 317, 2735], + [2165, 1954, 2736], + [434, 322, 2737], + [365, 783, 2738], + [1798, 1501, 2739], + [577, 287, 2740], + [2379, 120, 2741], + [1934, 368, 2742], + [328, 112, 2743], + [679, 115, 2744], + [335, 2333, 2745], + [601, 556, 2746], + [742, 2194, 2747], + [495, 355, 2748], + [332, 294, 2749], + [363, 2484, 2750], + [605, 101, 2751], + [1127, 1421, 2752], + [445, 2559, 2753], + [1564, 265, 2754], + [300, 1151, 2755], + [83, 2540, 2756], + [32, 1520, 2757], + [261, 480, 2758], + [1774, 115, 2759], + [1300, 769, 2760], + [815, 853, 2761], + [1057, 115, 2762], + [280, 589, 2763], + [2516, 303, 2764], + [2710, 289, 2765], + [556, 265, 2766], + [121, 109, 2767], + [296, 291, 2768], + [302, 112, 2769], + [2510, 304, 2770], + [273, 422, 2771], + [734, 118, 2772], + [565, 1991, 2773], + [525, 2480, 2774], + [278, 2503, 2775], + [525, 2024, 2776], + [87, 2017, 2777], + [437, 534, 2778], + [770, 115, 2779], + [337, 849, 2780], + [282, 314, 2781], + [65, 272, 2782], + [604, 115, 2783], + [493, 648, 2784], + [1477, 376, 2785], + [2291, 468, 2786], + [1261, 291, 2787], + [1154, 549, 2788], + [1371, 1595, 2789], + [109, 98, 2790], + [572, 263, 2791], + [1154, 411, 2792], + [1361, 121, 2793], + [522, 289, 2794], + [260, 541, 2795], + [600, 1128, 2796], + [601, 1541, 2797], + [733, 1748, 2798], + [825, 291, 2799], + [526, 115, 2800], + [862, 334, 2801], + [572, 1832, 2802], + [1760, 1777, 2803], + [277, 965, 2804], + [2086, 1545, 2805], + [259, 554, 2806], + [1018, 2638, 2807], + [2227, 115, 2808], + [114, 303, 2809], + [437, 444, 2810], + [872, 615, 2811], + [32, 456, 2812], + [368, 112, 2813], + [1193, 554, 2814], + [339, 595, 2815], + [259, 1961, 2816], + [1451, 121, 2817], + [482, 2314, 2818], + [2686, 792, 2819], + [874, 792, 2820], + [581, 1239, 2821], + [448, 105, 2822], + [32, 85, 2823], + [286, 107, 2824], + [874, 321, 2825], + [269, 1340, 2826], + [734, 803, 2827], + [323, 325, 2828], + [273, 290, 2829], + [261, 292, 2830], + [273, 1266, 2831], + [268, 422, 2832], + [336, 102, 2833], + [524, 1449, 2834], + [716, 115, 2835], + [895, 2728, 2836], + [440, 296, 2837], + [741, 100, 2838], + [277, 648, 2839], + [861, 322, 2840], + [282, 648, 2841], + [32, 265, 2842], + [268, 1846, 2843], + [270, 1204, 2844], + [372, 289, 2845], + [2369, 97, 2846], + [2557, 549, 2847], + [2060, 1991, 2848], + [587, 97, 2849], + [434, 390, 2850], + [565, 319, 2851], + [839, 289, 2852], + [78, 321, 2853], + [259, 556, 2854], + [2153, 97, 2855], + [469, 104, 2856], + [312, 115, 2857], + [261, 424, 2858], + [1000, 1341, 2859], + [2859, 2626, 2860], + [467, 116, 2861], + [565, 362, 2862], + [975, 945, 2863], + [282, 473, 2864], + [488, 46, 2865], + [32, 947, 2866], + [602, 291, 2867], + [1532, 1531, 2868], + [280, 727, 2869], + [2805, 1896, 2870], + [116, 105, 2871], + [280, 330, 2872], + [593, 420, 2873], + [1211, 754, 2874], + [788, 1120, 2875], + [259, 2456, 2876], + [584, 1545, 2877], + [981, 2024, 2878], + [1654, 1920, 2879], + [1708, 285, 2880], + [345, 337, 2881], + [645, 115, 2882], + [2252, 279, 2883], + [277, 383, 2884], + [349, 648, 2885], + [2858, 289, 2886], + [72, 2813, 2887], + [273, 813, 2888], + [987, 101, 2889], + [302, 265, 2890], + [2348, 475, 2891], + [357, 2643, 2892], + [536, 100, 2893], + [2370, 119, 2894], + [365, 1952, 2895], + [951, 265, 2896], + [307, 509, 2897], + [1633, 589, 2898], + [1371, 119, 2899], + [375, 1827, 2900], + [2899, 939, 2901], + [261, 2898, 2902], + [499, 100, 2903], + [2781, 2809, 2904], + [863, 279, 2905], + [102, 885, 2906], + [2815, 279, 2907], + [103, 291, 2908], + [633, 115, 2909], + [260, 119, 2910], + [1754, 115, 2911], + [1866, 291, 2912], + [457, 265, 2913], + [441, 308, 2914], + [678, 2340, 2915], + [832, 2090, 2916], + [108, 728, 2917], + [400, 615, 2918], + [278, 1589, 2919], + [2861, 416, 2920], + [722, 641, 2921], + [1497, 2906, 2922], + [104, 953, 2923], + [1758, 2918, 2924], + [2213, 110, 2925], + [437, 769, 2926], + [1188, 658, 2927], + [1715, 2122, 2928], + [2828, 2264, 2929], + [1051, 368, 2930], + [1047, 2046, 2931], + [794, 339, 2932], + [265, 1304, 2933], + [963, 457, 2934], + [105, 2751, 2935], + [1836, 939, 2936], + [310, 111, 2937], + [332, 494, 2938], + [780, 317, 2939], + [2171, 2928, 2940], + [2927, 684, 2941], + [886, 115, 2942], + [271, 263, 2943], + [499, 1216, 2944], + [512, 2766, 2945], + [105, 340, 2946], + [112, 809, 2947], + [2498, 1548, 2948], + [121, 400, 2949], + [541, 121, 2950], + [1000, 2935, 2951], + [105, 1980, 2952], + [296, 1372, 2953], + [2534, 368, 2954], + [2934, 615, 2955], + [44, 547, 2956], + [706, 101, 2957], + [102, 473, 2958], + [277, 334, 2959], + [1090, 2780, 2960], + [413, 103, 2961], + [2274, 1635, 2962], + [2050, 103, 2963], + [392, 2255, 2964], + [292, 116, 2965], + [1546, 115, 2966], + [981, 549, 2967], + [2247, 1068, 2968], + [267, 424, 2969], + [565, 1254, 2970], + [372, 1369, 2971], + [1512, 291, 2972], + [280, 2627, 2973], + [583, 362, 2974], + [273, 906, 2975], + [260, 955, 2976], + [97, 1227, 2977], + [1029, 2958, 2978], + [2537, 989, 2979], + [2654, 523, 2980], + [2075, 599, 2981], + [1995, 2933, 2982], + [1633, 939, 2983], + [270, 2467, 2984], + [77, 101, 2985], + [272, 836, 2986], + [87, 587, 2987], + [284, 752, 2988], + [734, 100, 2989], + [649, 337, 2990], + [1844, 339, 2991], + [1510, 1035, 2992], + [493, 287, 2993], + [259, 279, 2994], + [470, 103, 2995], + [654, 549, 2996], + [108, 876, 2997], + [99, 292, 2998], + [2978, 2412, 2999], + [1677, 2998, 3000], + [1023, 258, 3001], + [1333, 334, 3002], + [610, 301, 3003], + [610, 1823, 3004], + [505, 2986, 3005], + [66, 101, 3006], + [300, 109, 3007], + [271, 1486, 3008], + [2036, 311, 3009], + [556, 317, 3010], + [381, 491, 3011], + [1504, 662, 3012], + [685, 1591, 3013], + [2056, 265, 3014], + [434, 2949, 3015], + [285, 265, 3016], + [1969, 98, 3017], + [268, 2548, 3018], + [600, 2238, 3019], + [3017, 269, 3020], + [2894, 830, 3021], + [1171, 1035, 3022], + [332, 808, 3023], + [416, 765, 3024], + [1776, 1557, 3025], + [2969, 104, 3026], + [378, 2677, 3027], + [525, 2822, 3028], + [284, 292, 3029], + [279, 121, 3030], + [1388, 265, 3031], + [723, 1748, 3032], + [280, 2190, 3033], + [627, 1563, 3034], + [690, 291, 3035], + [303, 121, 3036], + [616, 301, 3037], + [863, 384, 3038], + [97, 119, 3039], + [258, 355, 3040], + [273, 658, 3041], + [282, 504, 3042], + [1097, 115, 3043], + [2975, 727, 3044], + [3028, 661, 3045], + [1336, 803, 3046], + [691, 390, 3047], + [409, 2917, 3048], + [111, 312, 3049], + [1034, 115, 3050], + [2075, 112, 3051], + [270, 2166, 3052], + [793, 115, 3053], + [601, 1516, 3054], + [2842, 765, 3055], + [453, 370, 3056], + [1584, 122, 3057], + [600, 878, 3058], + [1932, 265, 3059], + [445, 334, 3060], + [301, 697, 3061], + [822, 111, 3062], + [323, 833, 3063], + [1224, 115, 3064], + [2313, 2890, 3065], + [1069, 291, 3066], + [1493, 115, 3067], + [1033, 3062, 3068], + [2198, 1372, 3069], + [981, 480, 3070], + [1543, 115, 3071], + [105, 118, 3072], + [573, 3065, 3073], + [366, 115, 3074], + [365, 2488, 3075], + [332, 1492, 3076], + [445, 411, 3077], + [295, 110, 3078], + [1769, 265, 3079], + [1588, 376, 3080], + [1000, 108, 3081], + [1091, 279, 3082], + [259, 823, 3083], + [381, 422, 3084], + [616, 625, 3085], + [715, 2330, 3086], + [403, 102, 3087], + [298, 1749, 3088], + [277, 623, 3089], + [501, 494, 3090], + [682, 475, 3091], + [719, 579, 3092], + [2214, 475, 3093], + [268, 650, 3094], + [2207, 1567, 3095], + [2576, 1515, 3096], + [279, 418, 3097], + [375, 1809, 3098], + [389, 478, 3099], + [497, 291, 3100], + [365, 400, 3101], + [894, 771, 3102], + [1528, 311, 3103], + [1388, 115, 3104], + [2772, 554, 3105], + [2994, 348, 3106], + [1419, 3092, 3107], + [296, 523, 3108], + [267, 885, 3109], + [2339, 115, 3110], + [277, 1996, 3111], + [678, 308, 3112], + [76, 1169, 3113], + [2006, 1662, 3114], + [307, 117, 3115], + [1271, 115, 3116], + [2925, 368, 3117], + [3088, 400, 3118], + [2812, 2190, 3119], + [296, 427, 3120], + [919, 275, 3121], + [2030, 121, 3122], + [722, 330, 3123], + [953, 101, 3124], + [1833, 304, 3125], + [1102, 291, 3126], + [2959, 2525, 3127], + [400, 107, 3128], + [101, 940, 3129], + [119, 1335, 3130], + [851, 279, 3131], + [2360, 1823, 3132], + [2778, 121, 3133], + [121, 101, 3134], + [1395, 115, 3135], + [1209, 325, 3136], + [332, 2693, 3137], + [1109, 265, 3138], + [1504, 275, 3139], + [334, 2202, 3140], + [3033, 1053, 3141], + [259, 1182, 3142], + [2288, 117, 3143], + [388, 531, 3144], + [357, 2849, 3145], + [1334, 115, 3146], + [3103, 275, 3147], + [1966, 1290, 3148], + [112, 317, 3149], + [495, 301, 3150], + [743, 384, 3151], + [272, 534, 3152], + [581, 119, 3153], + [733, 269, 3154], + [72, 1539, 3155], + [332, 296, 3156], + [372, 121, 3157], + [1522, 579, 3158], + [1978, 567, 3159], + [416, 103, 3160], + [32, 1531, 3161], + [2223, 2228, 3162], + [3131, 337, 3163], + [2296, 3140, 3164], + [951, 291, 3165], + [273, 1867, 3166], + [613, 384, 3167], + [734, 309, 3168], + [821, 108, 3169], + [470, 1514, 3170], + [434, 103, 3171], + [259, 754, 3172], + [273, 272, 3173], + [277, 500, 3174], + [2670, 1961, 3175], + [303, 765, 3176], + [3119, 377, 3177], + [615, 317, 3178], + [832, 115, 3179], + [327, 650, 3180], + [535, 554, 3181], + [796, 279, 3182], + [486, 115, 3183], + [277, 1539, 3184], + [355, 111, 3185], + [2111, 115, 3186], + [106, 891, 3187], + [499, 1442, 3188], + [647, 2133, 3189], + [407, 2320, 3190], + [756, 1770, 3191], + [2527, 112, 3192], + [377, 771, 3193], + [881, 279, 3194], + [1679, 101, 3195], + [715, 411, 3196], + [457, 291, 3197], + [788, 531, 3198], + [1119, 475, 3199], + [111, 1227, 3200], + [3070, 686, 3201], + [616, 1612, 3202], + [1171, 1027, 3203], + [372, 1377, 3204], + [1484, 265, 3205], + [1771, 676, 3206], + [3083, 309, 3207], + [3139, 289, 3208], + [1158, 110, 3209], + [293, 822, 3210], + [857, 541, 3211], + [284, 661, 3212], + [1715, 97, 3213], + [271, 339, 3214], + [273, 3160, 3215], + [270, 3178, 3216], + [751, 1749, 3217], + [1014, 111, 3218], + [269, 979, 3219], + [3217, 615, 3220], + [2902, 265, 3221], + [794, 1421, 3222], + [610, 101, 3223], + [1984, 898, 3224], + [110, 97, 3225], + [524, 109, 3226], + [3143, 1053, 3227], + [314, 112, 3228], + [1328, 115, 3229], + [1836, 656, 3230], + [381, 489, 3231], + [273, 296, 3232], + [678, 2965, 3233], + [3206, 1204, 3234], + [32, 430, 3235], + [2400, 297, 3236], + [2862, 487, 3237], + [2316, 265, 3238], + [259, 958, 3239], + [335, 348, 3240], + [3101, 1659, 3241], + [2346, 265, 3242], + [716, 291, 3243], + [289, 418, 3244], + [692, 115, 3245], + [2156, 1179, 3246], + [2623, 101, 3247], + [298, 2387, 3248], + [100, 285, 3249], + [2102, 265, 3250], + [522, 1828, 3251], + [1730, 390, 3252], + [335, 2936, 3253], + [3215, 790, 3254], + [448, 554, 3255], + [2079, 656, 3256], + [101, 3176, 3257], + [862, 309, 3258], + [738, 102, 3259], + [873, 115, 3260], + [1522, 376, 3261], + [1540, 400, 3262], + [521, 109, 3263], + [584, 114, 3264], + [715, 752, 3265], + [375, 122, 3266], + [2183, 531, 3267], + [112, 294, 3268], + [261, 304, 3269], + [282, 368, 3270], + [2602, 475, 3271], + [99, 1304, 3272], + [2663, 265, 3273], + [2372, 265, 3274], + [327, 2122, 3275], + [3184, 400, 3276], + [277, 489, 3277], + [105, 1547, 3278], + [389, 341, 3279], + [1103, 384, 3280], + [1303, 400, 3281], + [299, 2648, 3282], + [2422, 341, 3283], + [97, 122, 3284], + [1771, 3268, 3285], + [301, 320, 3286], + [665, 362, 3287], + [880, 1550, 3288], + [1471, 263, 3289], + [2078, 337, 3290], + [333, 311, 3291], + [695, 112, 3292], + [1588, 2232, 3293], + [286, 119, 3294], + [327, 311, 3295], + [1771, 769, 3296], + [2573, 749, 3297], + [554, 119, 3298], + [307, 1074, 3299], + [1312, 121, 3300], + [2641, 296, 3301], + [3173, 894, 3302], + [413, 3272, 3303], + [1411, 302, 3304], + [770, 291, 3305], + [3106, 579, 3306], + [280, 1178, 3307], + [403, 2132, 3308], + [493, 487, 3309], + [2961, 368, 3310], + [298, 3230, 3311], + [974, 341, 3312], + [1969, 889, 3313], + [367, 1010, 3314], + [1705, 2374, 3315], + [2472, 111, 3316], + [107, 110, 3317], + [326, 3213, 3318], + [270, 287, 3319], + [333, 783, 3320], + [335, 965, 3321], + [301, 1021, 3322], + [2331, 355, 3323], + [99, 285, 3324], + [269, 400, 3325], + [635, 3298, 3326], + [741, 330, 3327], + [1127, 1503, 3328], + [1388, 291, 3329], + [335, 2449, 3330], + [453, 762, 3331], + [269, 317, 3332], + [278, 321, 3333], + [103, 589, 3334], + [2957, 347, 3335], + [600, 3061, 3336], + [1528, 1185, 3337], + [2515, 1740, 3338], + [268, 752, 3339], + [406, 3334, 3340], + [1147, 100, 3341], + [581, 98, 3342], + [3115, 670, 3343], + [3265, 311, 3344], + [320, 104, 3345], + [434, 1827, 3346], + [3327, 3325, 3347], + [275, 383, 3348], + [600, 289, 3349], + [861, 648, 3350], + [3313, 1341, 3351], + [1166, 291, 3352], + [751, 99, 3353], + [335, 1589, 3354], + [751, 1895, 3355], + [300, 1472, 3356], + [32, 279, 3357], + [284, 321, 3358], + [1430, 760, 3359], + [2441, 265, 3360], + [280, 808, 3361], + [1187, 803, 3362], + [634, 1722, 3363], + [2328, 1025, 3364], + [2621, 2265, 3365], + [3299, 121, 3366], + [751, 118, 3367], + [267, 787, 3368], + [822, 377, 3369], + [363, 475, 3370], + [432, 46, 3371], + [2153, 554, 3372], + [3309, 1442, 3373], + [706, 1980, 3374], + [451, 2761, 3375], + [99, 1204, 3376], + [2072, 1421, 3377], + [895, 2409, 3378], + [2988, 2558, 3379], + [3317, 549, 3380], + [1146, 112, 3381], + [1104, 115, 3382], + [32, 291, 3383], + [638, 291, 3384], + [732, 309, 3385], + [3089, 3341, 3386], + [2990, 475, 3387], + [706, 3286, 3388], + [262, 1723, 3389], + [290, 107, 3390], + [427, 110, 3391], + [1590, 939, 3392], + [2000, 427, 3393], + [2568, 1627, 3394], + [810, 2584, 3395], + [1033, 3129, 3396], + [1729, 830, 3397], + [118, 531, 3398], + [1545, 1304, 3399], + [827, 291, 3400], + [833, 104, 3401], + [3383, 3399, 3402], + [282, 290, 3403], + [283, 468, 3404], + [293, 2238, 3405], + [295, 2462, 3406], + [116, 394, 3407], + [1528, 320, 3408], + [3054, 2690, 3409], + [335, 296, 3410], + [3266, 444, 3411], + [292, 758, 3412], + [2164, 892, 3413], + [1813, 97, 3414], + [1504, 3412, 3415], + [1160, 110, 3416], + [323, 2856, 3417], + [1829, 641, 3418], + [3192, 1120, 3419], + [448, 97, 3420], + [2246, 3376, 3421], + [1107, 355, 3422], + [802, 121, 3423], + [1966, 107, 3424], + [99, 117, 3425], + [2572, 109, 3426], + [782, 98, 3427], + [1229, 115, 3428], + [307, 2046, 3429], + [3337, 771, 3430], + [1000, 3380, 3431], + [1478, 297, 3432], + [972, 3257, 3433], + [810, 101, 3434], + [789, 384, 3435], + [332, 424, 3436], + [501, 3049, 3437], + [3291, 309, 3438], + [117, 100, 3439], + [258, 116, 3440], + [623, 900, 3441], + [333, 301, 3442], + [895, 1293, 3443], + [1090, 1204, 3444], + [97, 875, 3445], + [72, 321, 3446], + [1065, 830, 3447], + [481, 2209, 3448], + [333, 317, 3449], + [261, 725, 3450], + [453, 2647, 3451], + [262, 2049, 3452], + [381, 1339, 3453], + [1489, 3391, 3454], + [424, 697, 3455], + [1236, 116, 3456], + [1320, 749, 3457], + [1353, 803, 3458], + [403, 1905, 3459], + [2569, 2088, 3460], + [2590, 115, 3461], + [338, 108, 3462], + [522, 390, 3463], + [587, 330, 3464], + [1049, 1339, 3465], + [960, 128, 3466], + [479, 2923, 3467], + [991, 2479, 3468], + [3402, 115, 3469], + [3077, 311, 3470], + [3440, 2871, 3471], + [556, 3471, 3472], + [687, 3464, 3473], + [3355, 1632, 3474], + [3212, 115, 3475], + [2261, 317, 3476], + [273, 2320, 3477], + [707, 1881, 3478], + [1504, 444, 3479], + [919, 705, 3480], + [1000, 1662, 3481], + [277, 457, 3482], + [1146, 1895, 3483], + [403, 1941, 3484], + [1974, 291, 3485], + [453, 3472, 3486], + [794, 852, 3487], + [367, 107, 3488], + [1677, 646, 3489], + [3007, 1322, 3490], + [3289, 2228, 3491], + [3324, 1693, 3492], + [259, 986, 3493], + [2359, 749, 3494], + [482, 924, 3495], + [2246, 2265, 3496], + [2205, 102, 3497], + [1471, 337, 3498], + [524, 400, 3499], + [2274, 534, 3500], + [1000, 3492, 3501], + [2016, 291, 3502], + [413, 120, 3503], + [381, 1432, 3504], + [1484, 291, 3505], + [2107, 384, 3506], + [776, 3322, 3507], + [3501, 1053, 3508], + [1584, 1740, 3509], + [2763, 623, 3510], + [734, 1583, 3511], + [372, 3255, 3512], + [97, 294, 3513], + [564, 114, 3514], + [1141, 494, 3515], + [1340, 115, 3516], + [1127, 1096, 3517], + [718, 1177, 3518], + [2060, 319, 3519], + [260, 2166, 3520], + [841, 121, 3521], + [501, 3120, 3522], + [1893, 265, 3523], + [270, 2743, 3524], + [1127, 676, 3525], + [1116, 656, 3526], + [1047, 265, 3527], + [1528, 2042, 3528], + [3270, 304, 3529], + [114, 110, 3530], + [280, 636, 3531], + [259, 648, 3532], + [290, 771, 3533], + [600, 2867, 3534], + [3259, 2513, 3535], + [262, 291, 3536], + [846, 337, 3537], + [3349, 1706, 3538], + [1349, 115, 3539], + [3503, 1025, 3540], + [1127, 98, 3541], + [2165, 771, 3542], + [2585, 265, 3543], + [3477, 1896, 3544], + [108, 328, 3545], + [3099, 2571, 3546], + [895, 296, 3547], + [1212, 109, 3548], + [293, 314, 3549], + [1181, 115, 3550], + [1563, 121, 3551], + [977, 749, 3552], + [1154, 908, 3553], + [381, 1947, 3554], + [1357, 291, 3555], + [834, 3398, 3556], + [751, 103, 3557], + [2389, 579, 3558], + [3418, 377, 3559], + [102, 97, 3560], + [434, 355, 3561], + [303, 661, 3562], + [1757, 309, 3563], + [1763, 473, 3564], + [3511, 110, 3565], + [895, 101, 3566], + [259, 1483, 3567], + [1992, 2374, 3568], + [732, 265, 3569], + [3098, 979, 3570], + [1803, 289, 3571], + [1195, 953, 3572], + [600, 1548, 3573], + [2263, 542, 3574], + [82, 101, 3575], + [114, 287, 3576], + [2030, 554, 3577], + [381, 2005, 3578], + [2984, 1722, 3579], + [272, 334, 3580], + [272, 308, 3581], + [3386, 790, 3582], + [268, 377, 3583], + [298, 3108, 3584], + [260, 3040, 3585], + [1364, 116, 3586], + [402, 296, 3587], + [3408, 279, 3588], + [2403, 542, 3589], + [3353, 274, 3590], + [103, 117, 3591], + [403, 2607, 3592], + [2017, 362, 3593], + [326, 594, 3594], + [741, 1583, 3595], + [739, 322, 3596], + [98, 939, 3597], + [1051, 656, 3598], + [1628, 291, 3599], + [282, 1483, 3600], + [3473, 377, 3601], + [2286, 3562, 3602], + [327, 3445, 3603], + [282, 448, 3604], + [2838, 3219, 3605], + [2152, 339, 3606], + [1298, 2997, 3607], + [269, 337, 3608], + [501, 2255, 3609], + [2004, 101, 3610], + [1489, 376, 3611], + [1592, 121, 3612], + [2425, 265, 3613], + [3500, 708, 3614], + [79, 119, 3615], + [308, 953, 3616], + [709, 1241, 3617], + [3616, 500, 3618], + [810, 2983, 3619], + [293, 3593, 3620], + [715, 3345, 3621], + [698, 112, 3622], + [1378, 115, 3623], + [97, 1012, 3624], + [2845, 719, 3625], + [445, 424, 3626], + [1141, 1516, 3627], + [2074, 115, 3628], + [71, 111, 3629], + [554, 102, 3630], + [524, 602, 3631], + [1569, 2546, 3632], + [1934, 2169, 3633], + [473, 898, 3634], + [570, 112, 3635], + [1023, 898, 3636], + [1953, 2454, 3637], + [1948, 3024, 3638], + [72, 101, 3639], + [1535, 569, 3640], + [315, 272, 3641], + [574, 384, 3642], + [103, 636, 3643], + [271, 1012, 3644], + [1147, 97, 3645], + [3367, 1025, 3646], + [434, 3618, 3647], + [1072, 3643, 3648], + [3410, 792, 3649], + [388, 101, 3650], + [565, 3645, 3651], + [278, 265, 3652], + [348, 99, 3653], + [97, 760, 3654], + [324, 2790, 3655], + [3655, 3581, 3656], + [1261, 279, 3657], + [261, 393, 3658], + [1763, 1851, 3659], + [357, 1745, 3660], + [259, 822, 3661], + [280, 424, 3662], + [910, 296, 3663], + [277, 1589, 3664], + [2151, 661, 3665], + [3057, 265, 3666], + [1386, 291, 3667], + [1360, 101, 3668], + [2399, 681, 3669], + [1235, 3665, 3670], + [73, 102, 3671], + [3510, 121, 3672], + [83, 1584, 3673], + [277, 1335, 3674], + [1047, 111, 3675], + [293, 487, 3676], + [1568, 275, 3677], + [1252, 322, 3678], + [2289, 809, 3679], + [483, 317, 3680], + [268, 878, 3681], + [2526, 475, 3682], + [270, 594, 3683], + [3663, 1304, 3684], + [505, 2248, 3685], + [281, 2414, 3686], + [116, 427, 3687], + [268, 348, 3688], + [2000, 401, 3689], + [912, 384, 3690], + [1116, 279, 3691], + [1440, 103, 3692], + [270, 2946, 3693], + [3430, 749, 3694], + [453, 587, 3695], + [1235, 1722, 3696], + [3689, 2172, 3697], + [105, 676, 3698], + [1411, 749, 3699], + [305, 860, 3700], + [866, 115, 3701], + [953, 115, 3702], + [1702, 2076, 3703], + [381, 320, 3704], + [1790, 291, 3705], + [1063, 110, 3706], + [3706, 1078, 3707], + [2071, 115, 3708], + [2162, 3634, 3709], + [295, 104, 3710], + [289, 328, 3711], + [282, 1131, 3712], + [1634, 3711, 3713], + [105, 1046, 3714], + [2219, 416, 3715], + [349, 3560, 3716], + [282, 287, 3717], + [267, 3530, 3718], + [1677, 3591, 3719], + [286, 296, 3720], + [473, 116, 3721], + [889, 500, 3722], + [3722, 1341, 3723], + [582, 1012, 3724], + [1736, 1027, 3725], + [2323, 1078, 3726], + [1000, 2591, 3727], + [3557, 813, 3728], + [720, 998, 3729], + [473, 317, 3730], + [1103, 279, 3731], + [1107, 2603, 3732], + [293, 303, 3733], + [295, 1369, 3734], + [396, 3713, 3735], + [121, 1745, 3736], + [403, 337, 3737], + [445, 311, 3738], + [1146, 894, 3739], + [1200, 1506, 3740], + [1066, 3632, 3741], + [276, 114, 3742], + [314, 304, 3743], + [3437, 279, 3744], + [279, 1053, 3745], + [381, 813, 3746], + [2719, 309, 3747], + [1195, 3715, 3748], + [286, 3723, 3749], + [261, 2767, 3750], + [3635, 311, 3751], + [3403, 3745, 3752], + [101, 279, 3753], + [357, 448, 3754], + [2067, 965, 3755], + [2288, 792, 3756], + [111, 101, 3757], + [1371, 1239, 3758], + [3147, 708, 3759], + [910, 623, 3760], + [298, 1926, 3761], + [2823, 110, 3762], + [1814, 965, 3763], + [1973, 312, 3764], + [2154, 104, 3765], + [99, 320, 3766], + [117, 900, 3767], + [530, 115, 3768], + [289, 2520, 3769], + [959, 748, 3770], + [3052, 279, 3771], + [335, 411, 3772], + [1287, 291, 3773], + [804, 116, 3774], + [976, 362, 3775], + [1300, 108, 3776], + [2156, 1723, 3777], + [286, 3687, 3778], + [403, 456, 3779], + [3537, 2042, 3780], + [2183, 1015, 3781], + [751, 279, 3782], + [3648, 291, 3783], + [695, 1635, 3784], + [2345, 938, 3785], + [2561, 1659, 3786], + [2830, 319, 3787], + [273, 117, 3788], + [706, 1761, 3789], + [495, 500, 3790], + [1933, 661, 3791], + [2409, 563, 3792], + [3750, 3776, 3793], + [635, 813, 3794], + [919, 2437, 3795], + [3316, 99, 3796], + [709, 455, 3797], + [1865, 1662, 3798], + [3765, 1341, 3799], + [89, 287, 3800], + [1533, 2546, 3801], + [115, 317, 3802], + [522, 559, 3803], + [1758, 595, 3804], + [1995, 3187, 3805], + [1729, 1342, 3806], + [307, 3630, 3807], + [284, 2387, 3808], + [438, 3097, 3809], + [3029, 312, 3810], + [280, 662, 3811], + [3656, 97, 3812], + [1171, 427, 3813], + [337, 265, 3814], + [99, 1979, 3815], + [335, 559, 3816], + [602, 279, 3817], + [2389, 301, 3818], + [2338, 2950, 3819], + [3815, 105, 3820], + [2679, 418, 3821], + [332, 531, 3822], + [349, 289, 3823], + [104, 485, 3824], + [105, 589, 3825], + [1047, 117, 3826], + [3779, 3753, 3827], + [1200, 2662, 3828], + [270, 3401, 3829], + [3659, 708, 3830], + [277, 3792, 3831], + [268, 269, 3832], + [357, 1207, 3833], + [582, 894, 3834], + [1127, 989, 3835], + [1329, 279, 3836], + [3733, 3420, 3837], + [1256, 965, 3838], + [1915, 279, 3839], + [2087, 371, 3840], + [474, 748, 3841], + [782, 611, 3842], + [2847, 554, 3843], + [1484, 418, 3844], + [3078, 321, 3845], + [3610, 304, 3846], + [3693, 111, 3847], + [259, 3767, 3848], + [260, 120, 3849], + [728, 97, 3850], + [3426, 311, 3851], + [733, 953, 3852], + [2561, 109, 3853], + [268, 311, 3854], + [335, 1340, 3855], + [286, 108, 3856], + [1908, 2907, 3857], + [99, 3580, 3858], + [101, 102, 3859], + [441, 400, 3860], + [474, 448, 3861], + [284, 2656, 3862], + [1663, 115, 3863], + [3406, 554, 3864], + [77, 629, 3865], + [295, 109, 3866], + [2091, 309, 3867], + [610, 2049, 3868], + [1400, 1749, 3869], + [490, 2591, 3870], + [3209, 311, 3871], + [362, 2449, 3872], + [2044, 115, 3873], + [601, 658, 3874], + [522, 100, 3875], + [696, 279, 3876], + [440, 1472, 3877], + [259, 297, 3878], + [793, 309, 3879], + [600, 464, 3880], + [880, 730, 3881], + [2664, 1854, 3882], + [3223, 1240, 3883], + [105, 111, 3884], + [2749, 291, 3885], + [2462, 489, 3886], + [3811, 2248, 3887], + [486, 1722, 3888], + [691, 989, 3889], + [1350, 371, 3890], + [438, 1645, 3891], + [1852, 658, 3892], + [874, 3551, 3893], + [2177, 3814, 3894], + [277, 1823, 3895], + [258, 469, 3896], + [1555, 291, 3897], + [362, 348, 3898], + [1926, 1053, 3899], + [501, 2329, 3900], + [2531, 749, 3901], + [112, 289, 3902], + [470, 116, 3903], + [441, 3902, 3904], + [381, 2539, 3905], + [499, 2760, 3906], + [339, 2042, 3907], + [972, 708, 3908], + [117, 1956, 3909], + [1656, 2202, 3910], + [1064, 3820, 3911], + [117, 263, 3912], + [3390, 656, 3913], + [102, 655, 3914], + [270, 708, 3915], + [715, 500, 3916], + [268, 1531, 3917], + [3357, 3817, 3918], + [1889, 3597, 3919], + [3849, 101, 3920], + [3914, 101, 3921], + [403, 1147, 3922], + [259, 1723, 3923], + [2403, 347, 3924], + [2155, 2680, 3925], + [857, 3102, 3926], + [300, 308, 3927], + [756, 464, 3928], + [1392, 291, 3929], + [3790, 444, 3930], + [3831, 2513, 3931], + [2472, 3899, 3932], + [99, 105, 3933], + [407, 272, 3934], + [714, 115, 3935], + [493, 2972, 3936], + [1087, 115, 3937], + [2318, 384, 3938], + [1855, 3913, 3939], + [1607, 355, 3940], + [2216, 2769, 3941], + [333, 595, 3942], + [303, 1053, 3943], + [2655, 115, 3944], + [2702, 898, 3945], + [375, 112, 3946], + [2443, 1545, 3947], + [885, 473, 3948], + [2254, 475, 3949], + [1813, 418, 3950], + [3907, 304, 3951], + [3916, 1343, 3952], + [103, 110, 3953], + [261, 377, 3954], + [1220, 2454, 3955], + [261, 309, 3956], + [289, 353, 3957], + [361, 110, 3958], + [520, 1241, 3959], + [1336, 730, 3960], + [495, 3957, 3961], + [2376, 115, 3962], + [284, 355, 3963], + [122, 1239, 3964], + [2521, 101, 3965], + [396, 287, 3966], + [1191, 115, 3967], + [487, 1322, 3968], + [530, 291, 3969], + [890, 2029, 3970], + [2430, 115, 3971], + [3834, 1501, 3972], + [400, 265, 3973], + [1582, 727, 3974], + [97, 347, 3975], + [297, 646, 3976], + [834, 3943, 3977], + [272, 3933, 3978], + [1167, 625, 3979], + [2219, 594, 3980], + [388, 3980, 3981], + [742, 684, 3982], + [2293, 2108, 3983], + [581, 2646, 3984], + [742, 289, 3985], + [2729, 476, 3986], + [97, 1605, 3987], + [1047, 1608, 3988], + [110, 480, 3989], + [635, 105, 3990], + [3775, 400, 3991], + [565, 3921, 3992], + [273, 708, 3993], + [440, 400, 3994], + [277, 823, 3995], + [2967, 265, 3996], + [3995, 464, 3997], + [1073, 3978, 3998], + [324, 114, 3999], + [2207, 539, 4000], + [71, 708, 4001], + [1052, 291, 4002], + [1522, 681, 4003], + [261, 2135, 4004], + [600, 3909, 4005], + [381, 809, 4006], + [1252, 1634, 4007], + [261, 3951, 4008], + [1934, 296, 4009], + [103, 554, 4010], + [2299, 554, 4011], + [1072, 898, 4012], + [2729, 898, 4013], + [32, 3743, 4014], + [264, 1240, 4015], + [2331, 116, 4016], + [1995, 2646, 4017], + [327, 592, 4018], + [284, 370, 4019], + [1510, 116, 4020], + [403, 3859, 4021], + [493, 2172, 4022], + [2247, 2135, 4023], + [2571, 111, 4024], + [1252, 599, 4025], + [326, 971, 4026], + [1818, 115, 4027], + [105, 1840, 4028], + [284, 3124, 4029], + [3898, 376, 4030], + [2282, 291, 4031], + [582, 309, 4032], + [3922, 3953, 4033], + [1276, 656, 4034], + [635, 589, 4035], + [1511, 567, 4036], + [1312, 3766, 4037], + [282, 424, 4038], + [328, 2088, 4039], + [259, 457, 4040], + [269, 542, 4041], + [861, 1827, 4042], + [2989, 589, 4043], + [715, 287, 4044], + [259, 629, 4045], + [1261, 538, 4046], + [453, 314, 4047], + [297, 418, 4048], + [1004, 291, 4049], + [3308, 792, 4050], + [617, 265, 4051], + [278, 617, 4052], + [368, 3030, 4053], + [2864, 114, 4054], + [4018, 116, 4055], + [2086, 108, 4056], + [738, 1732, 4057], + [3081, 924, 4058], + [649, 3149, 4059], + [3237, 101, 4060], + [3917, 1662, 4061], + [322, 1177, 4062], + [66, 3134, 4063], + [99, 385, 4064], + [691, 4062, 4065], + [794, 1046, 4066], + [278, 294, 4067], + [323, 3348, 4068], + [378, 1854, 4069], + [2603, 655, 4070], + [2764, 3798, 4071], + [367, 116, 4072], + [332, 4024, 4073], + [776, 1633, 4074], + [3462, 427, 4075], + [678, 1605, 4076], + [335, 285, 4077], + [525, 1341, 4078], + [1252, 2283, 4079], + [357, 3654, 4080], + [284, 303, 4081], + [857, 900, 4082], + [1146, 1254, 4083], + [4040, 1626, 4084], + [332, 3698, 4085], + [284, 3850, 4086], + [4045, 105, 4087], + [4054, 3886, 4088], + [2425, 783, 4089], + [267, 292, 4090], + [1000, 104, 4091], + [1715, 269, 4092], + [477, 101, 4093], + [638, 115, 4094], + [1231, 625, 4095] + ] +} diff --git a/vocab_bpe.txt b/vocab_bpe.txt deleted file mode 100644 index 98aad701..00000000 --- a/vocab_bpe.txt +++ /dev/null @@ -1,9740 +0,0 @@ -260 whi -261 pp -262 W -263 said -264 whe -265 ge -266 B -267 sh -268 fr -269 ust -270 ab -271 very -272 âĢĶ -273 ne -274 our -275 by -276 which -277 qu -278 no -279 kn -280 il -281 all -282 ind -283 see -284 lf -285 pe -286 and -287 out -288 fe -289 th -290 con -291 they -292 He -293 from -294 were -295 ard -296 there -297 would -298 un -299 this -300 ra -301 C -302 ul -303 ood -304 -- -305 man -306 could -307 ook -308 ong -309 ex -310 ound -311 res -312 L -313 est -314 ation -315 them -316 li -317 ?âĢĿ -318 out -319 al -320 ,âĢĿ -321 one -322 ant -323 when -324 D -325 or -326 us -327 one -328 ic -329 han -330 been -331 int -332 Mr -333 own -334 v -335 ough -336 ry -337 ab -338 ous -339 art -340 _ -341 ast -342 self -343 ell -344 your -345 ak -346 who -347 itt -348 j -349 any -350 ep -351 ol -352 ag -353 if -354 what -355 un -356 wor -357 com -358 know -359 ers -360 tim -361 ate -362 os -363 J -364 le -365 ame -366 am -367 look -368 ig -369 some -370 are -371 can -372 go -373 la -374 ist -375 say -376 hing -377 ake -378 way -379 very -380 will -381 um -382 ther -383 then -384 did -385 too -386 more -387 af -388 It -389 ven -390 time -391 ide -392 ort -393 hed -394 ive -395 qu -396 ost -397 their -398 od -399 !âĢĿ -400 reat -401 down -402 sha -403 able -404 over -405 us -406 nt -407 fa -408 E -409 fore -410 ure -411 ink -412 ar -413 sp -414 ved -415 au -416 ro -417 into -418 end -419 urn -420 now -421 again -422 en -423 P -424 hand -425 ect -426 ittle -427 pt -428 nder -429 ence -430 little -431 about -432 ine -433 wn -434 ack -435 ity -436 ag -437 like -438 em -439 tr -440 ble -441 other -442 el -443 You -444 than -445 ie -446 ite -447 br -448 ass -449 lo -450 before -451 ance -452 tw -453 -454 how -455 ought -456 im -457 mad -458 ark -459 uch -460 ings -461 day -462 But -463 pro -464 other -465 must -466 ather -467 off -468 ered -469 upon -470 age -471 good -472 cc -473 way -474 dis -475 come -476 ng -477 ress -478 pr -479 G -480 ear -481 po -482 should -483 are -484 think -485 every -486 ud -487 got -488 only -489 F -490 ful -491 co -492 such -493 iss -494 has -495 med -496 ice -497 much -498 ness -499 gr -500 N -501 may -502 ch -503 lo -504 ick -505 ish -506 ad -507 per -508 our -509 led -510 get -511 ree -512 cy -513 went -514 ount -515 after -516 back -517 O -518 ally -519 ho -520 long -521 never -522 ire -523 ment -524 ff -525 ious -526 str -527 rem -528 thr -529 ne -530 ked -531 made -532 might -533 ward -534 bl -535 ady -536 ple -537 ving -538 ps -539 let -540 hen -541 ey -542 ife -543 shall -544 Tom -545 bo -546 cl -547 two -548 She -549 great -550 pre -551 night -552 ried -553 app -554 away -555 ere -556 came -557 uck -558 ning -559 ies -560 ose -561 under -562 ty -563 just -564 here -565 sc -566 beg -567 ouse -568 bet -569 own -570 ise -571 old -572 per -573 sm -574 ever -575 even -576 And -577 ft -578 thought -579 eth -580 ull -581 where -582 fir -583 ary -584 ody -585 pla -586 ke -587 well -588 ened -589 omet -590 ue -591 oll -592 its -593 riend -594 bec -595 ave -596 arn -597 me -598 ond -599 tell -600 pe -601 head -602 cha -603 ves -604 turn -605 des -606 ber -607 want -608 nothing -609 friend -610 You -611 ily -612 op -613 first -614 ild -615 don -616 took -617 âĢĺ -618 ted -619 spe -620 door -621 pass -622 ob -623 right -624 ac -625 ap -626 himself -627 thing -628 through -629 last -630 ated -631 make -632 ach -633 ang -634 ions -635 eyes -636 att -637 liz -638 looked -639 part -640 There -641 ause -642 res -643 ied -644 gl -645 somet -646 room -647 Eliz -648 ord -649 ever -650 We -651 says -652 saw -653 * -654 imp -655 most -656 pl -657 abeth -658 R -659 Elizabeth -660 we -661 ced -662 wr -663 pect -664 Th -665 found -666 -667 face -668 ner -669 ting -670 wat -671 Miss -672 done -673 They -674 happ -675 comp -676 alk -677 these -678 cons -679 ert -680 dear -681 though -682 feel -683 ath -684 har -685 word -686 ces -687 being -688 lie -689 row -690 ged -691 ared -692 house -693 take -694 oss -695 The -696 pped -697 put -698 soon -699 seemed -700 ls -701 less -702 rep -703 body -704 ent -705 hes -706 without -707 ase -708 te -709 iness -710 yet -711 place -712 still -713 myself -714 wind -715 ret -716 mind -717 atter -718 ways -719 âĢĻ -720 life -721 V -722 Then -723 mo -724 ind -725 owed -726 light -727 ign -728 sat -729 end -730 exp -731 fl -732 heard -733 leep -734 kind -735 father -736 iff -737 cont -738 man -739 acc -740 cour -741 heart -742 hy -743 ile -744 When -745 In -746 ible -747 young -748 dre -749 ched -750 kne -751 ss -752 many -753 ank -754 set -755 rest -756 hands -757 get -758 pres -759 St -760 aking -761 always -762 stre -763 work -764 mor -765 ars -766 call -767 something -768 wa -769 once -770 igh -771 going -772 ently -773 mom -774 moment -775 ons -776 thing -777 another -778 enough -779 sw -780 things -781 les -782 left -783 told -784 find -785 So -786 belie -787 ction -788 pa -789 -790 same -791 Mrs -792 wom -793 ip -794 men -795 ra -796 ling -797 dist -798 .. -799 ross -800 wards -801 mean -802 ister -803 rec -804 hor -805 sl -806 give -807 lt -808 ject -809 ple -810 ctor -811 ears -812 hear -813 the -814 answ -815 chan -816 Lu -817 dark -818 ac -819 ertain -820 car -821 lice -822 It -823 oth -824 seen -825 ms -826 not -827 K -828 three -829 better -830 :-- -831 arge -832 while -833 bre -834 ished -835 ane -836 int -837 far -838 inst -839 ix -840 hour -841 ople -842 mes -843 orry -844 hard -845 ef -846 air -847 oke -848 ens -849 sec -850 around -851 boy -852 began -853 Hol -854 fell -855 min -856 ined -857 supp -858 didn -859 poor -860 half -861 along -862 morning -863 bus -864 anything -865 sw -866 till -867 fect -868 ri -869 herself -870 new -871 sleep -872 oub -873 As -874 return -875 ret -876 asked -877 Well -878 Yes -879 iz -880 lic -881 child -882 count -883 This -884 iet -885 la -886 because -887 em -888 few -889 Holmes -890 side -891 ste -892 keep -893 dd -894 sil -895 those -896 people -897 quite -898 ised -899 ins -900 vo -901 turned -902 knew -903 hope -904 ince -905 ear -906 window -907 .âĢĻ -908 av -909 for -910 felt -911 ER -912 sur -913 ute -914 conf -915 ory -916 ev -917 ain -918 sh -919 arcy -920 foll -921 ass -922 mar -923 Darcy -924 occ -925 ison -926 ger -927 talk -928 home -929 sto -930 Do -931 love -932 lp -933 sister -934 ru -935 ingle -936 inc -937 gone -938 If -939 aught -940 ove -941 ained -942 âĢĺ -943 ither -944 open -945 ool -946 gent -947 aw -948 wait -949 disc -950 round -951 near -952 add -953 pleas -954 cor -955 lay -956 poss -957 Alice -958 comm -959 erv -960 diff -961 looking -962 xt -963 read -964 HA -965 enn -966 pers -967 letter -968 ery -969 Oh -970 gether -971 sun -972 TER -973 That -974 What -975 bed -976 Jim -977 help -978 nat -979 âĢľâĢĺ -980 fam -981 however -982 ations -983 whole -984 His -985 CHA -986 tra -987 PTER -988 matter -989 CHAPTER -990 fo -991 sure -992 ition -993 My -994 mis -995 unt -996 icul -997 gentle -998 eed -999 cur -1000 What -1001 be -1002 dead -1003 ._ -1004 fort -1005 cle -1006 ention -1007 ired -1008 most -1009 that -1010 ial -1011 gu -1012 certain -1013 start -1014 Lorry -1015 ren -1016 unc -1017 ower -1018 both -1019 ter -1020 care -1021 dr -1022 light -1023 world -1024 iv -1025 pur -1026 stood -1027 ressed -1028 che -1029 voice -1030 ek -1031 ref -1032 does -1033 believe -1034 ho -1035 tre -1036 Huck -1037 Jo -1038 oud -1039 exc -1040 And -1041 aut -1042 wo -1043 Jane -1044 adv -1045 ten -1046 ully -1047 No -1048 Benn -1049 God -1050 day -1051 Bennet -1052 ract -1053 cannot -1054 pret -1055 oy -1056 towards -1057 dra -1058 so -1059 und -1060 sub -1061 osed -1062 among -1063 ured -1064 reg -1065 here -1066 ember -1067 woman -1068 warn -1069 ange -1070 words -1071 business -1072 ug -1073 wish -1074 coming -1075 run -1076 ween -1077 ener -1078 ents -1079 haps -1080 ,âĢĻ -1081 mother -1082 cept -1083 fear -1084 erest -1085 ped -1086 sound -1087 ense -1088 doub -1089 unt -1090 against -1091 course -1092 next -1093 lock -1094 passed -1095 prison -1096 whom -1097 der -1098 together -1099 ins -1100 white -1101 aken -1102 ub -1103 ail -1104 pr -1105 ( -1106 underst -1107 cried -1108 ask -1109 mer -1110 name -1111 gi -1112 ia -1113 try -1114 sn -1115 cr -1116 imes -1117 vers -1118 town -1119 Van -1120 dden -1121 hind -1122 best -1123 ason -1124 Hels -1125 Helsing -1126 sho -1127 between -1128 rew -1129 each -1130 ingley -1131 owing -1132 ual -1133 air -1134 gir -1135 Bingley -1136 days -1137 el -1138 years -1139 quest -1140 since -1141 At -1142 high -1143 behind -1144 reck -1145 Def -1146 ating -1147 hus -1148 sel -1149 death -1150 land -1151 bro -1152 case -1153 alked -1154 sudden -1155 hur -1156 up -1157 small -1158 wood -1159 dro -1160 nor -1161 gave -1162 amp -1163 aint -1164 interest -1165 vis -1166 water -1167 by -1168 har -1169 alone -1170 ide -1171 its -1172 Defarge -1173 use -1174 ures -1175 ina -1176 det -1177 ider -1178 night -1179 manner -1180 present -1181 until -1182 cei -1183 anger -1184 having -1185 family -1186 ouble -1187 wonder -1188 less -1189 leave -1190 speak -1191 expect -1192 river -1193 doubt -1194 ans -1195 wouldn -1196 Lucy -1197 mon -1198 list -1199 lack -1200 strong -1201 For -1202 Be -1203 son -1204 gra -1205 couldn -1206 brought -1207 No -1208 indeed -1209 fore -1210 ult -1211 ately -1212 but -1213 sir -1214 ken -1215 nce -1216 ying -1217 to -1218 te -1219 hum -1220 ater -1221 ows -1222 But -1223 no -1224 orrow -1225 conc -1226 almost -1227 band -1228 inter -1229 reckon -1230 fully -1231 ham -1232 suff -1233 ience -1234 ued -1235 sk -1236 Ar -1237 deep -1238 obs -1239 ited -1240 dec -1241 ung -1242 ney -1243 act -1244 quick -1245 yd -1246 X -1247 clo -1248 How -1249 pretty -1250 husband -1251 you -1252 pat -1253 riage -1254 ib -1255 ont -1256 watch -1257 stand -1258 called -1259 ient -1260 wife -1261 full -1262 lady -1263 ield -1264 answer -1265 ming -1266 fire -1267 lied -1268 ext -1269 fess -1270 sou -1271 gener -1272 oney -1273 country -1274 understand -1275 ott -1276 rather -1277 eas -1278 Well -1279 aw -1280 daught -1281 beaut -1282 consider -1283 sort -1284 eff -1285 short -1286 others -1287 Why -1288 head -1289 swe -1290 really -1291 reason -1292 og -1293 everything -1294 irc -1295 also -1296 lost -1297 ets -1298 remark -1299 ror -1300 within -1301 quiet -1302 Doctor -1303 eared -1304 uff -1305 isf -1306 dire -1307 need -1308 black -1309 uring -1310 idea -1311 close -1312 ill -1313 surpr -1314 ca -1315 happy -1316 often -1317 used -1318 selves -1319 sign -1320 fall -1321 red -1322 why -1323 ably -1324 road -1325 wanted -1326 ace -1327 spoke -1328 mb -1329 ash -1330 power -1331 whis -1332 Eng -1333 compan -1334 ches -1335 pi -1336 opened -1337 bed -1338 four -1339 inf -1340 inv -1341 live -1342 times -1343 strange -1344 perhaps -1345 ready -1346 ster -1347 Now -1348 break -1349 op -1350 circ -1351 ceed -1352 cked -1353 All -1354 brother -1355 cat -1356 whose -1357 prom -1358 ceived -1359 taken -1360 itting -1361 money -1362 fin -1363 abs -1364 My -1365 bour -1366 opp -1367 heav -1368 low -1369 trouble -1370 contin -1371 walk -1372 lau -1373 uth -1374 yourself -1375 Fr -1376 person -1377 body -1378 spir -1379 cond -1380 kept -1381 answered -1382 cher -1383 tri -1384 affect -1385 else -1386 gs -1387 Mad -1388 laid -1389 step -1390 point -1391 ground -1392 ready -1393 least -1394 blood -1395 appear -1396 bo -1397 week -1398 ude -1399 Y -1400 tried -1401 already -1402 hold -1403 satisf -1404 sight -1405 pain -1406 returned -1407 II -1408 nig -1409 instant -1410 hardly -1411 [ -1412 ene -1413 hair -1414 stay -1415 arri -1416 app -1417 mur -1418 hon -1419 sim -1420 ering -1421 bit -1422 rible -1423 room -1424 silence -1425 ern -1426 ade -1427 adm -1428 cess -1429 rel -1430 Se -1431 gentleman -1432 won -1433 book -1434 aster -1435 umb -1436 dread -1437 mid -1438 ime -1439 airs -1440 differe -1441 flo -1442 iled -1443 play -1444 Luc -1445 ually -1446 bur -1447 fur -1448 real -1449 stra -1450 akes -1451 cious -1452 tou -1453 creat -1454 utes -1455 either -1456 Mina -1457 feet -1458 became -1459 express -1460 fellow -1461 sent -1462 whether -1463 jo -1464 friends -1465 fter -1466 ident -1467 glad -1468 foot -1469 girl -1470 ph -1471 Lond -1472 Jon -1473 saying -1474 aff -1475 London -1476 That -1477 ration -1478 war -1479 dep -1480 shad -1481 eng -1482 cold -1483 replied -1484 held -1485 ility -1486 Count -1487 used -1488 evening -1489 stopped -1490 daughter -1491 ier -1492 wall -1493 ser -1494 big -1495 breat -1496 -1497 fort -1498 Her -1499 minute -1500 wild -1501 land -1502 dev -1503 struck -1504 earth -1505 He -1506 eye -1507 minutes -1508 question -1509 Char -1510 vill -1511 arm -1512 imag -1513 On -1514 object -1515 comfort -1516 empt -1517 es -1518 athan -1519 enty -1520 Man -1521 Pro -1522 ickham -1523 sometimes -1524 Q -1525 Lady -1526 act -1527 mist -1528 yes -1529 Coll -1530 ended -1531 gr -1532 dri -1533 bel -1534 ides -1535 idence -1536 ird -1537 saf -1538 begin -1539 hours -1540 _, -1541 cing -1542 omp -1543 urt -1544 true -1545 char -1546 Collins -1547 oun -1548 sus -1549 Wickham -1550 obser -1551 ination -1552 ical -1553 table -1554 pap -1555 ances -1556 sum -1557 clear -1558 U -1559 bad -1560 di -1561 iful -1562 remember -1563 dream -1564 boys -1565 subject -1566 given -1567 jud -1568 possible -1569 fessor -1570 1 -1571 lad -1572 feelings -1573 expl -1574 visit -1575 free -1576 fix -1577 danger -1578 acqu -1579 cri -1580 ead -1581 ses -1582 cry -1583 late -1584 With -1585 seem -1586 account -1587 second -1588 prisoner -1589 serv -1590 anx -1591 aged -1592 resol -1593 draw -1594 suddenly -1595 ught -1596 meet -1597 followed -1598 king -1599 men -1600 tered -1601 dest -1602 appeared -1603 slow -1604 oured -1605 asion -1606 ette -1607 âĢĶâĢĿ -1608 comes -1609 fact -1610 write -1611 women -1612 eas -1613 mon -1614 walked -1615 bes -1616 ledge -1617 One -1618 perfect -1619 Jonathan -1620 ?âĢĻ -1621 itself -1622 like -1623 Dr -1624 iece -1625 Oh -1626 sweet -1627 ten -1628 inqu -1629 lect -1630 king -1631 known -1632 outh -1633 met -1634 erable -1635 large -1636 raid -1637 itted -1638 getting -1639 pose -1640 mark -1641 Why -1642 particul -1643 thous -1644 pleasure -1645 side -1646 sit -1647 ager -1648 ached -1649 paper -1650 across -1651 added -1652 Joe -1653 state -1654 illed -1655 ._-- -1656 undred -1657 year -1658 forg -1659 stud -1660 lips -1661 feeling -1662 entered -1663 change -1664 suppose -1665 ydia -1666 ided -1667 Mar -1668 Mary -1669 illi -1670 ains -1671 Lydia -1672 iety -1673 ,-- -1674 nigger -1675 bank -1676 ising -1677 cre -1678 To -1679 ran -1680 able -1681 thousand -1682 hn -1683 sens -1684 five -1685 cra -1686 Wh -1687 show -1688 vi -1689 chance -1690 started -1691 soul -1692 Professor -1693 fast -1694 ian -1695 ks -1696 âĢľ_ -1697 delight -1698 means -1699 ingly -1700 ustration -1701 thur -1702 ments -1703 natur -1704 bit -1705 use -1706 sy -1707 six -1708 erry -1709 learn -1710 trust -1711 disapp -1712 attention -1713 nature -1714 month -1715 lish -1716 ors -1717 ote -1718 After -1719 shut -1720 box -1721 human -1722 heavy -1723 itude -1724 Hark -1725 sea -1726 Lord -1727 ago -1728 afraid -1729 rose -1730 bu -1731 ots -1732 lea -1733 ages -1734 grave -1735 Ill -1736 az -1737 ining -1738 erm -1739 fan -1740 ush -1741 anybody -1742 charact -1743 John -1744 Illustration -1745 du -1746 uke -1747 none -1748 sever -1749 front -1750 fav -1751 arms -1752 prev -1753 certainly -1754 Manette -1755 making -1756 pri -1757 hat -1758 exam -1759 Pross -1760 secret -1761 Arthur -1762 yer -1763 mine -1764 cal -1765 ates -1766 gre -1767 urch -1768 An -1769 occasion -1770 crow -1771 during -1772 bear -1773 therefore -1774 ushed -1775 disp -1776 desp -1777 form -1778 corner -1779 miss -1780 pping -1781 seems -1782 laim -1783 chair -1784 difficul -1785 forward -1786 stri -1787 als -1788 arton -1789 respect -1790 children -1791 Harker -1792 How -1793 ging -1794 where -1795 sense -1796 eng -1797 ching -1798 By -1799 appro -1800 ublic -1801 affection -1802 several -1803 how -1804 ring -1805 iting -1806 fig -1807 hundred -1808 usual -1809 moved -1810 different -1811 inted -1812 fing -1813 orn -1814 irt -1815 ang -1816 doing -1817 ular -1818 bright -1819 happened -1820 village -1821 dle -1822 ich -1823 ir -1824 bar -1825 col -1826 seeing -1827 Carton -1828 zy -1829 fast -1830 ift -1831 conv -1832 jour -1833 bring -1834 happiness -1835 breakfast -1836 crowd -1837 ci -1838 aunt -1839 convers -1840 everybody -1841 .... -1842 We -1843 Cather -1844 above -1845 comple -1846 ump -1847 direct -1848 Catherine -1849 ton -1850 what -1851 orm -1852 repe -1853 self -1854 une -1855 fair -1856 terrible -1857 circum -1858 his -1859 pic -1860 necess -1861 worth -1862 Do -1863 run -1864 wit -1865 cause -1866 form -1867 themselves -1868 Darn -1869 Not -1870 iously -1871 talking -1872 character -1873 Darnay -1874 ey -1875 mouth -1876 shook -1877 overed -1878 particular -1879 illiam -1880 favour -1881 taking -1882 ening -1883 gri -1884 oose -1885 remained -1886 esc -1887 ats -1888 though -1889 ards -1890 There -1891 br -1892 ut -1893 othes -1894 medi -1895 determ -1896 effect -1897 wh -1898 Here -1899 osition -1900 spo -1901 carriage -1902 Now -1903 aven -1904 uz -1905 ed -1906 duke -1907 received -1908 ific -1909 beautiful -1910 circumst -1911 tears -1912 soft -1913 knowledge -1914 King -1915 ale -1916 ec -1917 son -1918 cap -1919 hid -1920 ution -1921 mere -1922 sole -1923 ified -1924 deal -1925 laugh -1926 .âĢĶ -1927 If -1928 wis -1929 fun -1930 fool -1931 cut -1932 eness -1933 anced -1934 chool -1935 sol -1936 iles -1937 enc -1938 iew -1939 loom -1940 longer -1941 carried -1942 silent -1943 uncle -1944 nded -1945 ury -1946 exact -1947 Every -1948 immedi -1949 general -1950 easy -1951 continued -1952 safe -1953 !âĢĻ -1954 ili -1955 old -1956 ast -1957 ought -1958 cross -1959 except -1960 Let -1961 raft -1962 wine -1963 dut -1964 lled -1965 cks -1966 alth -1967 itten -1968 lead -1969 prob -1970 opin -1971 moon -1972 fixed -1973 yond -1974 sitting -1975 fine -1976 sett -1977 _. -1978 over -1979 ve -1980 read -1981 pray -1982 prep -1983 drew -1984 horror -1985 purpose -1986 gg -1987 mn -1988 tor -1989 die -1990 ented -1991 mem -1992 fright -1993 iling -1994 ants -1995 iced -1996 truth -1997 straight -1998 field -1999 onse -2000 past -2001 asleep -2002 oof -2003 cky -2004 expected -2005 Madame -2006 honour -2007 floor -2008 Then -2009 ray -2010 sad -2011 eat -2012 weak -2013 ief -2014 -2015 conversation -2016 clock -2017 morrow -2018 mot -2019 mention -2020 fre -2021 fet -2022 died -2023 ative -2024 requ -2025 neigh -2026 church -2027 closed -2028 clothes -2029 lay -2030 est -2031 ines -2032 wid -2033 Tell -2034 tern -2035 key -2036 law -2037 awful -2038 purs -2039 ities -2040 meant -2041 showed -2042 scar -2043 thoughts -2044 horses -2045 common -2046 dropped -2047 XX -2048 ap -2049 wished -2050 sick -2051 reet -2052 essed -2053 live -2054 dog -2055 exper -2056 nearly -2057 woods -2058 surprise -2059 master -2060 dress -2061 beyond -2062 equ -2063 aded -2064 save -2065 ieur -2066 become -2067 occur -2068 letters -2069 Seward -2070 papers -2071 ock -2072 ben -2073 nobody -2074 joy -2075 ept -2076 ected -2077 remain -2078 married -2079 marriage -2080 waiting -2081 Lucie -2082 opinion -2083 ee -2084 ex -2085 ma -2086 def -2087 Crun -2088 usion -2089 afterwards -2090 smile -2091 court -2092 news -2093 ship -2094 trees -2095 English -2096 ners -2097 ping -2098 den -2099 post -2100 hang -2101 thinking -2102 slight -2103 trave -2104 journey -2105 eav -2106 ele -2107 ames -2108 Long -2109 proper -2110 move -2111 streng -2112 breath -2113 urtz -2114 Cruncher -2115 uct -2116 tone -2117 cand -2118 hill -2119 ph -2120 stone -2121 Just -2122 quar -2123 mighty -2124 erence -2125 endeav -2126 raised -2127 rupt -2128 perfectly -2129 natural -2130 inc -2131 sli -2132 ask -2133 att -2134 thus -2135 resh -2136 order -2137 var -2138 prop -2139 offic -2140 cloud -2141 glass -2142 import -2143 early -2144 spirits -2145 creature -2146 Not -2147 talked -2148 din -2149 lot -2150 aring -2151 par -2152 notice -2153 Sher -2154 conn -2155 ord -2156 attend -2157 wrong -2158 Kurtz -2159 trying -2160 broken -2161 observed -2162 lst -2163 pes -2164 itation -2165 earn -2166 ceive -2167 send -2168 ustom -2169 pered -2170 De -2171 ccess -2172 coff -2173 blue -2174 emp -2175 guard -2176 satisfied -2177 den -2178 now -2179 sle -2180 threw -2181 whilst -2182 likely -2183 onsieur -2184 breast -2185 steam -2186 regard -2187 appearance -2188 months -2189 hood -2190 yed -2191 iment -2192 story -2193 knows -2194 peace -2195 distance -2196 England -2197 steps -2198 ladies -2199 inary -2200 cho -2201 hadn -2202 outside -2203 follow -2204 occup -2205 waited -2206 bout -2207 ones -2208 da -2209 kiss -2210 shar -2211 miserable -2212 promise -2213 forgot -2214 wo -2215 pull -2216 Mor -2217 Some -2218 shore -2219 uss -2220 gold -2221 umber -2222 ortun -2223 begun -2224 descri -2225 ugg -2226 listened -2227 pict -2228 dinner -2229 av -2230 sorrow -2231 hung -2232 leng -2233 eager -2234 Wat -2235 unk -2236 onel -2237 proceed -2238 prot -2239 street -2240 Qu -2241 below -2242 ford -2243 nar -2244 forget -2245 super -2246 olute -2247 Ge -2248 obli -2249 imposs -2250 broke -2251 standing -2252 surprised -2253 solemn -2254 neighbour -2255 strength -2256 bound -2257 led -2258 hast -2259 igne -2260 ively -2261 twenty -2262 Gard -2263 pock -2264 scene -2265 hurried -2266 spirit -2267 further -2268 figure -2269 wret -2270 makes -2271 fif -2272 sake -2273 doctor -2274 attempt -2275 darkness -2276 cheer -2277 Your -2278 sym -2279 repeated -2280 onseigne -2281 Watson -2282 onseigneur -2283 bb -2284 squ -2285 piece -2286 grew -2287 sear -2288 view -2289 ryver -2290 ection -2291 Stryver -2292 arrived -2293 aston -2294 In -2295 inct -2296 mile -2297 cou -2298 hot -2299 line -2300 station -2301 success -2302 William -2303 pend -2304 faces -2305 coll -2306 streets -2307 Geor -2308 cent -2309 well -2310 ze -2311 Is -2312 Aunt -2313 Br -2314 ils -2315 lean -2316 placed -2317 izzy -2318 Becky -2319 immediately -2320 length -2321 ges -2322 set -2323 iner -2324 sorry -2325 cott -2326 atch -2327 gloom -2328 relie -2329 neither -2330 maybe -2331 wrote -2332 awyer -2333 dreadful -2334 duty -2335 school -2336 thin -2337 neck -2338 entire -2339 windows -2340 amongst -2341 expression -2342 Un -2343 slowly -2344 sharp -2345 house -2346 joy -2347 path -2348 uc -2349 val -2350 mount -2351 bott -2352 ancing -2353 lie -2354 arent -2355 assist -2356 viol -2357 necessary -2358 impossible -2359 lot -2360 rang -2361 sing -2362 cab -2363 cast -2364 ley -2365 idently -2366 ously -2367 Jerry -2368 Ne -2369 They -2370 hurry -2371 opportun -2372 middle -2373 Charles -2374 earnest -2375 pl -2376 wise -2377 âĢľ -2378 mer -2379 bent -2380 loud -2381 public -2382 allowed -2383 Sid -2384 lived -2385 knowed -2386 osing -2387 remem -2388 written -2389 ibly -2390 engaged -2391 fancy -2392 bs -2393 come -2394 tem -2395 mus -2396 bra -2397 ony -2398 lit -2399 pity -2400 former -2401 note -2402 living -2403 ways -2404 turning -2405 desire -2406 speaking -2407 compl -2408 pale -2409 carry -2410 sky -2411 inquir -2412 determined -2413 exactly -2414 ), -2415 do -2416 lling -2417 lab -2418 gard -2419 owers -2420 yell -2421 deg -2422 ices -2423 aker -2424 aked -2425 thank -2426 heads -2427 plain -2428 drink -2429 warm -2430 She -2431 jun -2432 ears -2433 amed -2434 ship -2435 quis -2436 accomp -2437 marry -2438 quickly -2439 suspic -2440 anxious -2441 was -2442 er -2443 wide -2444 cous -2445 Har -2446 ality -2447 enjoy -2448 ement -2449 girls -2450 considerable -2451 absolute -2452 uli -2453 vi -2454 civ -2455 dare -2456 terday -2457 although -2458 shoulder -2459 blind -2460 places -2461 angu -2462 plan -2463 stret -2464 horse -2465 ized -2466 reflect -2467 snow -2468 concl -2469 effort -2470 remarked -2471 companion -2472 situ -2473 Good -2474 ives -2475 my -2476 top -2477 lif -2478 thick -2479 stir -2480 noticed -2481 ru -2482 anim -2483 Monseigneur -2484 van -2485 emb -2486 company -2487 bitter -2488 destro -2489 priv -2490 This -2491 Your -2492 af -2493 cr -2494 ned -2495 fresh -2496 thro -2497 sequ -2498 rid -2499 Monsieur -2500 kill -2501 shout -2502 worse -2503 osity -2504 somebody -2505 imm -2506 party -2507 persu -2508 concern -2509 whisper -2510 whispered -2511 circumstances -2512 Sherlock -2513 Gardiner -2514 opportunity -2515 lu -2516 look -2517 she -2518 2 -2519 age -2520 miles -2521 cent -2522 lying -2523 ature -2524 und -2525 ters -2526 ilt -2527 ested -2528 looks -2529 art -2530 distur -2531 misery -2532 shadow -2533 beginning -2534 ues -2535 cell -2536 cely -2537 reached -2538 doll -2539 arer -2540 arest -2541 loved -2542 accept -2543 despair -2544 complete -2545 ape -2546 kn -2547 ris -2548 tend -2549 wand -2550 itiz -2551 inform -2552 seat -2553 noble -2554 exist -2555 arted -2556 Par -2557 trans -2558 disg -2559 coach -2560 rooms -2561 evidently -2562 shire -2563 sisters -2564 running -2565 beauty -2566 Charlot -2567 spoken -2568 Tellson -2569 Charlotte -2570 go -2571 oe -2572 mod -2573 bird -2574 later -2575 inno -2576 pick -2577 These -2578 -2579 agree -2580 whatever -2581 ishment -2582 strugg -2583 counten -2584 empt -2585 rench -2586 touch -2587 study -2588 calm -2589 George -2590 entirely -2591 ghed -2592 under -2593 tall -2594 med -2595 fro -2596 caught -2597 dat -2598 lim -2599 langu -2600 pause -2601 any -2602 beat -2603 thir -2604 const -2605 Col -2606 uneas -2607 prov -2608 pride -2609 grat -2610 stop -2611 extre -2612 extra -2613 Madam -2614 yesterday -2615 com -2616 kes -2617 par -2618 ign -2619 hun -2620 ased -2621 ases -2622 asure -2623 luck -2624 latter -2625 pay -2626 beha -2627 bell -2628 alming -2629 sold -2630 Sawyer -2631 greater -2632 impro -2633 pleasant -2634 Godalming -2635 noon -2636 ashion -2637 bourn -2638 admir -2639 beside -2640 Longbourn -2641 garden -2642 fat -2643 cru -2644 health -2645 hall -2646 ored -2647 ourn -2648 conce -2649 unate -2650 ounded -2651 spot -2652 Pres -2653 troub -2654 smoke -2655 Street -2656 believed -2657 questions -2658 boat -2659 weeks -2660 ler -2661 vol -2662 dim -2663 rag -2664 Cha -2665 val -2666 enem -2667 Good -2668 harm -2669 enter -2670 address -2671 advant -2672 instantly -2673 emplo -2674 When -2675 ding -2676 burn -2677 bag -2678 ishing -2679 stairs -2680 reach -2681 Ch -2682 ouses -2683 Of -2684 deser -2685 presently -2686 incey -2687 interrupt -2688 capt -2689 Que -2690 laimed -2691 ob -2692 fly -2693 dust -2694 path -2695 gun -2696 eight -2697 Lizzy -2698 unha -2699 goes -2700 grass -2701 lose -2702 stranger -2703 passing -2704 incl -2705 gentlemen -2706 patient -2707 easily -2708 witness -2709 cousin -2710 Queen -2711 Ah -2712 pter -2713 hered -2714 hide -2715 pen -2716 rain -2717 unf -2718 orted -2719 grow -2720 acqu -2721 evil -2722 teeth -2723 direction -2724 influ -2725 walking -2726 imagine -2727 resolved -2728 difficult -2729 colour -2730 settled -2731 memory -2732 slept -2733 pocket -2734 Is -2735 whe -2736 mess -2737 soc -2738 Who -2739 shot -2740 Cl -2741 manners -2742 pros -2743 produ -2744 watched -2745 supper -2746 confidence -2747 drove -2748 books -2749 acquaint -2750 which -2751 widow -2752 Quincey -2753 sympath -2754 ax -2755 rid -2756 nice -2757 ring -2758 consequ -2759 alar -2760 Don -2761 els -2762 police -2763 bless -2764 pec -2765 passage -2766 writing -2767 recoll -2768 please -2769 discovered -2770 curious -2771 cheek -2772 reckoned -2773 suffer -2774 finished -2775 laughed -2776 touched -2777 fortune -2778 disappo -2779 scarcely -2780 Colonel -2781 Come -2782 Mr -2783 iable -2784 ky -2785 ny -2786 win -2787 port -2788 atur -2789 gro -2790 vered -2791 rushed -2792 ainted -2793 usted -2794 twice -2795 cho -2796 afternoon -2797 olly -2798 changed -2799 ize -2800 forehead -2801 fingers -2802 Morris -2803 throat -2804 cellent -2805 bered -2806 gged -2807 iosity -2808 las -2809 ply -2810 tear -2811 filled -2812 Somet -2813 knowing -2814 boat -2815 greatest -2816 smo -2817 spee -2818 teen -2819 they -2820 terror -2821 drawn -2822 fallen -2823 murd -2824 burst -2825 devil -2826 leaving -2827 angry -2828 occurred -2829 couple -2830 uliar -2831 situation -2832 Paris -2833 cha -2834 cul -2835 faint -2836 forth -2837 delic -2838 ppy -2839 French -2840 ourselves -2841 prec -2842 kindness -2843 stream -2844 pleased -2845 possess -2846 hold -2847 simple -2848 candle -2849 Chapter -2850 peculiar -2851 Yes -2852 single -2853 itz -2854 inj -2855 ether -2856 intro -2857 worked -2858 presence -2859 following -2860 commun -2861 watching -2862 conduct -2863 Yet -2864 iron -2865 countenance -2866 empty -2867 jo -2868 pse -2869 with -2870 tired -2871 mass -2872 enge -2873 dressed -2874 pol -2875 youth -2876 ading -2877 allow -2878 sugg -2879 dogs -2880 nose -2881 train -2882 remo -2883 Injun -2884 curiosity -2885 unday -2886 wonderful -2887 ydney -2888 France -2889 belong -2890 explain -2891 drawing -2892 difficulty -2893 wishes -2894 extreme -2895 ignor -2896 All -2897 To -2898 ts -2899 will -2900 alive -2901 str -2902 ety -2903 semb -2904 lives -2905 ression -2906 locked -2907 erer -2908 reply -2909 there -2910 rich -2911 ails -2912 Marquis -2913 probably -2914 wretched -2915 acquaintance -2916 co -2917 auth -2918 erhaps -2919 fut -2920 furn -2921 tow -2922 coun -2923 dig -2924 pract -2925 hain -2926 vern -2927 tering -2928 ille -2929 Sp -2930 Sunday -2931 delay -2932 itty -2933 spent -2934 ague -2935 proud -2936 uded -2937 udge -2938 hopes -2939 knees -2940 handsome -2941 facts -2942 edge -2943 fetch -2944 acques -2945 speech -2946 gy -2947 mber -2948 lamp -2949 vel -2950 seven -2951 island -2952 Sydney -2953 antly -2954 intell -2955 ission -2956 Nether -2957 result -2958 satisfa -2959 murder -2960 endeavour -2961 coffin -2962 influence -2963 introdu -2964 Netherfield -2965 oor -2966 uly -2967 wor -2968 rent -2969 ball -2970 ises -2971 custom -2972 ences -2973 hy -2974 pie -2975 press -2976 habit -2977 ga -2978 rig -2979 Car -2980 icient -2981 idered -2982 throw -2983 advent -2984 quietly -2985 serious -2986 experience -2987 various -2988 AR -2989 ah -2990 ior -2991 é -2992 treat -2993 mag -2994 heaven -2995 idly -2996 stery -2997 stairs -2998 aled -2999 Sir -3000 Ag -3001 West -3002 Come -3003 excellent -3004 vent -3005 umped -3006 Pet -3007 covered -3008 passion -3009 putting -3010 formed -3011 famili -3012 clean -3013 Lucas -3014 charge -3015 judged -3016 eased -3017 besides -3018 finger -3019 merely -3020 frequ -3021 engeance -3022 Who -3023 min -3024 uous -3025 task -3026 inced -3027 ream -3028 hig -3029 hist -3030 rust -3031 ries -3032 Jacques -3033 faith -3034 profess -3035 pointed -3036 pected -3037 contr -3038 meaning -3039 Kitty -3040 broad -3041 learned -3042 escape -3043 avo -3044 degree -3045 information -3046 society -3047 communic -3048 Don -3049 Let -3050 acy -3051 lan -3052 long -3053 ty -3054 ndeed -3055 gall -3056 oted -3057 wasn -3058 rap -3059 separ -3060 illy -3061 depend -3062 seek -3063 raph -3064 isted -3065 Even -3066 ption -3067 prof -3068 coat -3069 telling -3070 glance -3071 accord -3072 instead -3073 assured -3074 incre -3075 marked -3076 search -3077 dollars -3078 action -3079 iny -3080 reed -3081 mut -3082 fla -3083 fashion -3084 pal -3085 names -3086 Sept -3087 dey -3088 neg -3089 race -3090 vain -3091 anyway -3092 spread -3093 Polly -3094 thrown -3095 blame -3096 nights -3097 attent -3098 happen -3099 courage -3100 assure -3101 board -3102 simply -3103 joined -3104 approach -3105 utter -3106 absolutely -3107 advantage -3108 author -3109 September -3110 HE -3111 So -3112 ew -3113 number -3114 eld -3115 Two -3116 rub -3117 noise -3118 knife -3119 unch -3120 lock -3121 smiled -3122 spec -3123 considered -3124 expressed -3125 record -3126 uncom -3127 exclaimed -3128 interesting -3129 tea -3130 ibility -3131 roads -3132 From -3133 convinced -3134 remembered -3135 !_ -3136 eal -3137 phe -3138 pla -3139 right -3140 sed -3141 tree -3142 wound -3143 itions -3144 fight -3145 camp -3146 ency -3147 lan -3148 loose -3149 let -3150 formed -3151 stick -3152 mystery -3153 redit -3154 feared -3155 --âĢĿ -3156 vol -3157 yours -3158 spring -3159 Nor -3160 hoped -3161 blow -3162 paid -3163 bread -3164 laying -3165 shock -3166 However -3167 comfortable -3168 summer -3169 grief -3170 cheerful -3171 itizen -3172 host -3173 wake -3174 bene -3175 fal -3176 tong -3177 third -3178 Their -3179 ric -3180 canoe -3181 otherwise -3182 madame -3183 chief -3184 firm -3185 doors -3186 impat -3187 moments -3188 stepped -3189 surr -3190 lovely -3191 tremb -3192 crime -3193 18 -3194 build -3195 leaves -3196 pulled -3197 narrow -3198 looking -3199 agreeable -3200 future -3201 Tom -3202 range -3203 uish -3204 mort -3205 fi -3206 pet -3207 attered -3208 gar -3209 gate -3210 rising -3211 rib -3212 exec -3213 eping -3214 arg -3215 horrible -3216 hearing -3217 Thatcher -3218 regular -3219 listen -3220 service -3221 mentioned -3222 hanging -3223 Here -3224 gers -3225 bow -3226 fli -3227 city -3228 houses -3229 iding -3230 velo -3231 weather -3232 rough -3233 ann -3234 dem -3235 thy -3236 esty -3237 ateful -3238 aped -3239 ordinary -3240 entreat -3241 reading -3242 matters -3243 hurt -3244 together -3245 remarkable -3246 condition -3247 holding -3248 III -3249 murm -3250 judge -3251 grey -3252 temper -3253 innocent -3254 unhappy -3255 yard -3256 nine -3257 hate -3258 atic -3259 ardon -3260 exceed -3261 altogether -3262 wants -3263 speed -3264 indiff -3265 belief -3266 misfort -3267 giving -3268 particularly -3269 extraordinary -3270 che -3271 zen -3272 id -3273 iter -3274 lest -3275 omm -3276 rejo -3277 seized -3278 abbit -3279 Look -3280 ume -3281 ument -3282 elve -3283 brain -3284 Fin -3285 grand -3286 board -3287 intment -3288 supposed -3289 confess -3290 opening -3291 signs -3292 catch -3293 finally -3294 urtle -3295 servant -3296 encour -3297 motion -3298 obliged -3299 existence -3300 Presently -3301 enemy -3302 For -3303 aid -3304 chan -3305 cceed -3306 ron -3307 wore -3308 sal -3309 sank -3310 bare -3311 itated -3312 ceased -3313 lam -3314 pit -3315 gaz -3316 otter -3317 force -3318 stro -3319 storm -3320 Saint -3321 suit -3322 succeed -3323 dearest -3324 lig -3325 shud -3326 Did -3327 eps -3328 unless -3329 amaz -3330 somewhat -3331 growing -3332 glanced -3333 sleeping -3334 events -3335 fog -3336 understood -3337 shoes -3338 promised -3339 arrival -3340 diary -3341 crying -3342 anxiety -3343 picture -3344 neighbourhood -3345 fifty -3346 astonishment -3347 suspicion -3348 viour -3349 Sometimes -3350 history -3351 gar -3352 hor -3353 uit -3354 tick -3355 tomb -3356 asc -3357 lew -3358 ones -3359 Turtle -3360 myster -3361 Ant -3362 somewhere -3363 somehow -3364 ode -3365 clos -3366 compass -3367 moving -3368 younger -3369 shop -3370 erval -3371 ailed -3372 boxes -3373 private -3374 known -3375 Clerval -3376 sympathy -3377 inal -3378 oup -3379 caut -3380 dance -3381 cend -3382 cted -3383 pper -3384 Where -3385 this -3386 exer -3387 liked -3388 jury -3389 olation -3390 worst -3391 leg -3392 amiable -3393 Ev -3394 spite -3395 twelve -3396 offer -3397 icked -3398 appre -3399 spect -3400 resp -3401 ... -3402 evidence -3403 drunk -3404 regret -3405 suffered -3406 midnight -3407 difference -3408 walls -3409 frightened -3410 apart -3411 important -3412 bottom -3413 brave -3414 lifted -3415 behaviour -3416 ?âĢĶ -3417 As -3418 aving -3419 iod -3420 just -3421 mit -3422 ver -3423 tied -3424 inting -3425 fit -3426 field -3427 chat -3428 cell -3429 div -3430 ashed -3431 lines -3432 pil -3433 forced -3434 May -3435 Mery -3436 Mock -3437 Heaven -3438 astle -3439 fail -3440 Potter -3441 goodness -3442 shoulders -3443 paused -3444 keeping -3445 generally -3446 cially -3447 uzz -3448 quarter -3449 officers -3450 kissed -3451 relief -3452 intellig -3453 Meryton -3454 eli -3455 ker -3456 mond -3457 zed -3458 ice -3459 tast -3460 sla -3461 mud -3462 mix -3463 erve -3464 credit -3465 nerv -3466 otion -3467 yield -3468 THE -3469 roof -3470 ratt -3471 Hun -3472 Sc -3473 killed -3474 gel -3475 fears -3476 hank -3477 olog -3478 rob -3479 rock -3480 dism -3481 log -3482 cler -3483 repres -3484 event -3485 ruction -3486 awake -3487 command -3488 guess -3489 excited -3490 asking -3491 phon -3492 moonlight -3493 occupied -3494 removed -3495 cle -3496 wy -3497 your -3498 wond -3499 sigh -3500 ndered -3501 fier -3502 dry -3503 hes -3504 loss -3505 git -3506 gently -3507 ghost -3508 ims -3509 ades -3510 Hatter -3511 Can -3512 ryphon -3513 jumped -3514 ols -3515 worn -3516 Pember -3517 bran -3518 Gryphon -3519 Our -3520 hens -3521 conscious -3522 raise -3523 distress -3524 recall -3525 steal -3526 inside -3527 monster -3528 forest -3529 decided -3530 join -3531 afford -3532 ounced -3533 meeting -3534 crossed -3535 prepared -3536 forgotten -3537 Pemberley -3538 bing -3539 iver -3540 kins -3541 of -3542 oine -3543 po -3544 rade -3545 hest -3546 main -3547 ital -3548 fate -3549 cli -3550 arse -3551 echo -3552 thunder -3553 stair -3554 stern -3555 sail -3556 Such -3557 pecially -3558 unfort -3559 umed -3560 Ex -3561 inking -3562 spy -3563 sprang -3564 grim -3565 undert -3566 scr -3567 period -3568 rows -3569 lightning -3570 accident -3571 discover -3572 sunset -3573 decla -3574 ogn -3575 pipe -3576 green -3577 property -3578 slipped -3579 importance -3580 dawn -3581 cottage -3582 mountains -3583 yellow -3584 tender -3585 !âĢĶ -3586 did -3587 wr -3588 tale -3589 ack -3590 oh -3591 odd -3592 orig -3593 mater -3594 food -3595 claim -3596 enger -3597 lower -3598 pack -3599 uther -3600 threat -3601 cended -3602 reve -3603 wel -3604 wear -3605 rate -3606 Had -3607 seated -3608 Sally -3609 shown -3610 Le -3611 intention -3612 virt -3613 unable -3614 venge -3615 imper -3616 daylight -3617 precious -3618 design -3619 Three -3620 swear -3621 trem -3622 sounds -3623 interested -3624 daughters -3625 affair -3626 engage -3627 drive -3628 resolution -3629 craw -3630 Justine -3631 thirty -3632 admiration -3633 lantern -3634 acknow -3635 bl -3636 lor -3637 overy -3638 ral -3639 tic -3640 wed -3641 aid -3642 wig -3643 cave -3644 dull -3645 chief -3646 aside -3647 weight -3648 ris -3649 aining -3650 Ad -3651 Was -3652 anyone -3653 Judge -3654 ending -3655 acher -3656 attack -3657 attended -3658 lights -3659 swift -3660 carefully -3661 deeply -3662 honest -3663 straw -3664 disposition -3665 laughing -3666 maid -3667 steamboat -3668 proceeded -3669 cabin -3670 music -3671 labour -3672 uneasy -3673 cheeks -3674 avoid -3675 Antoine -3676 acc -3677 bre -3678 him -3679 iently -3680 nes -3681 ober -3682 then -3683 time -3684 va -3685 yl -3686 tight -3687 sides -3688 rel -3689 match -3690 cases -3691 onc -3692 oner -3693 dared -3694 pir -3695 pun -3696 position -3697 yo -3698 imate -3699 vey -3700 ste -3701 knitting -3702 ilar -3703 lies -3704 using -3705 agony -3706 urely -3707 arrange -3708 chin -3709 Ros -3710 flowers -3711 racted -3712 bye -3713 falling -3714 softly -3715 equal -3716 ordered -3717 ignorant -3718 profession -3719 false -3720 IN -3721 gment -3722 iest -3723 ré -3724 aud -3725 wed -3726 reck -3727 hand -3728 band -3729 ce -3730 lly -3731 ories -3732 anqu -3733 omin -3734 gen -3735 receive -3736 deck -3737 nearer -3738 nephe -3739 nodd -3740 --âĢľ -3741 manager -3742 uns -3743 legs -3744 fault -3745 trif -3746 holl -3747 scra -3748 smell -3749 desper -3750 glim -3751 Rep -3752 report -3753 finding -3754 -3755 possibly -3756 fold -3757 crim -3758 listening -3759 observ -3760 niggers -3761 depart -3762 especially -3763 dreams -3764 acquainted -3765 Whit -3766 disappeared -3767 prevent -3768 strike -3769 tort -3770 clouds -3771 stretched -3772 cruel -3773 place -3774 compassion -3775 elix -3776 nervous -3777 nephew -3778 On -3779 Very -3780 cise -3781 ev -3782 gress -3783 ips -3784 iction -3785 pet -3786 uce -3787 wood -3788 zing -3789 inite -3790 hall -3791 oul -3792 eral -3793 fish -3794 fount -3795 oned -3796 double -3797 asp -3798 asons -3799 gained -3800 ur -3801 rise -3802 sav -3803 saved -3804 antic -3805 ushing -3806 Perhaps -3807 offered -3808 office -3809 blank -3810 justice -3811 science -3812 perceived -3813 keen -3814 impr -3815 hys -3816 recovered -3817 swept -3818 voices -3819 advice -3820 drop -3821 ograph -3822 absence -3823 Frank -3824 relations -3825 breathing -3826 imagination -3827 mistaken -3828 examined -3829 favourite -3830 golden -3831 birds -3832 language -3833 tongue -3834 wyne -3835 Rosings -3836 Look -3837 iary -3838 rate -3839 some -3840 ahead -3841 mill -3842 bri -3843 bore -3844 cla -3845 enry -3846 enfield -3847 dying -3848 asped -3849 pad -3850 pressed -3851 puzz -3852 oming -3853 ease -3854 Tr -3855 rang -3856 oop -3857 ooping -3858 House -3859 Mc -3860 Sw -3861 Am -3862 shining -3863 shirt -3864 que -3865 olves -3866 agreed -3867 quit -3868 shape -3869 ener -3870 grad -3871 grateful -3872 tyard -3873 Renfield -3874 setting -3875 pair -3876 distinct -3877 recogn -3878 surely -3879 current -3880 careful -3881 highly -3882 ideas -3883 sufficient -3884 companions -3885 heavily -3886 visitor -3887 naturally -3888 directed -3889 Nothing -3890 capable -3891 -3892 phil -3893 incip -3894 choly -3895 familiar -3896 Of -3897 aff -3898 jesty -3899 leness -3900 ope -3901 pir -3902 fol -3903 inging -3904 yards -3905 vere -3906 vert -3907 aspect -3908 sand -3909 soot -3910 ify -3911 ourse -3912 ounds -3913 liber -3914 astic -3915 jack -3916 osp -3917 osure -3918 osop -3919 Ph -3920 Poor -3921 handed -3922 brown -3923 prove -3924 proved -3925 greatly -3926 apparent -3927 Rabbit -3928 compre -3929 courtyard -3930 hearts -3931 onscious -3932 ctory -3933 instr -3934 steady -3935 evident -3936 possessed -3937 curt -3938 ugust -3939 elb -3940 presented -3941 headed -3942 trial -3943 buried -3944 creatures -3945 mistake -3946 dangerous -3947 March -3948 XXX -3949 endeavoured -3950 singular -3951 gratitude -3952 burning -3953 interrupted -3954 produced -3955 satisfaction -3956 exceedingly -3957 wedding -3958 Republic -3959 takes -3960 sought -3961 had -3962 erved -3963 fond -3964 cow -3965 enra -3966 enced -3967 lod -3968 powers -3969 atched -3970 athered -3971 game -3972 lend -3973 iments -3974 urd -3975 melan -3976 Ma -3977 key -3978 Say -3979 Al -3980 shri -3981 Du -3982 vast -3983 unre -3984 osit -3985 osite -3986 Jack -3987 tranqu -3988 hole -3989 scared -3990 impression -3991 calling -3992 recomm -3993 addressed -3994 track -3995 cleared -3996 Before -3997 interv -3998 actually -3999 affairs -4000 shadows -4001 directly -4002 foolish -4003 pursued -4004 connected -4005 civility -4006 shouted -4007 immense -4008 disturbed -4009 pieces -4010 Westenra -4011 type -4012 fiend -4013 Evré -4014 monde -4015 melancholy -4016 Evrémonde -4017 ., -4018 Be -4019 Dr -4020 Did -4021 Where -4022 ires -4023 lished -4024 tin -4025 tempt -4026 wicked -4027 itary -4028 ision -4029 fart -4030 cook -4031 arily -4032 inha -4033 uted -4034 imon -4035 stared -4036 mender -4037 anger -4038 Henry -4039 Bank -4040 fee -4041 intent -4042 unknown -4043 osom -4044 shaking -4045 arrang -4046 pron -4047 princip -4048 pointing -4049 Felix -4050 loving -4051 pref -4052 smiling -4053 chain -4054 glancing -4055 working -4056 steep -4057 unconscious -4058 chess -4059 safety -4060 vict -4061 approached -4062 fetched -4063 leaning -4064 vanished -4065 conceal -4066 winter -4067 possession -4068 unfortunate -4069 philosop -4070 By -4071 Dead -4072 book -4073 nal -4074 tumb -4075 wet -4076 wra -4077 wise -4078 wolves -4079 sne -4080 meth -4081 bod -4082 bosom -4083 lately -4084 esides -4085 phys -4086 atures -4087 gathered -4088 ye -4089 reasons -4090 forb -4091 rage -4092 rank -4093 rush -4094 keep -4095 ctober -4096 shr -4097 seeming -4098 fever -4099 ational -4100 whenever -4101 Dra -4102 vague -4103 spare -4104 process -4105 proof -4106 group -4107 turns -4108 obt -4109 Thus -4110 consc -4111 retired -4112 arsad -4113 distant -4114 stock -4115 terms -4116 ception -4117 details -4118 opposite -4119 painful -4120 servants -4121 funeral -4122 required -4123 benef -4124 eagerly -4125 fordshire -4126 violent -4127 inquiries -4128 troubles -4129 prospect -4130 demand -4131 fountain -4132 Duchess -4133 method -4134 One -4135 ws -4136 held -4137 size -4138 hamed -4139 bab -4140 cro -4141 cool -4142 clock -4143 hint -4144 asant -4145 pill -4146 nur -4147 bend -4148 gang -4149 ashamed -4150 forty -4151 Satur -4152 deli -4153 Wil -4154 Baker -4155 âĢĶâĢĻ -4156 uses -4157 oused -4158 unex -4159 que -4160 ard -4161 splend -4162 rolled -4163 elt -4164 progress -4165 udden -4166 fulness -4167 grown -4168 October -4169 perm -4170 parted -4171 contempt -4172 busy -4173 stead -4174 stole -4175 aware -4176 discovery -4177 north -4178 interf -4179 XV -4180 caused -4181 similar -4182 explan -4183 While -4184 delighted -4185 figures -4186 estate -4187 connection -4188 castle -4189 suggest -4190 elder -4191 fierce -4192 material -4193 Saturday -4194 _.âĢĿ -4195 cause -4196 iers -4197 3 -4198 erning -4199 bill -4200 ism -4201 fence -4202 cart -4203 dining -4204 hell -4205 arling -4206 arthy -4207 pardon -4208 ancy -4209 steps -4210 rendered -4211 revenge -4212 urst -4213 aming -4214 Sh -4215 August -4216 delive -4217 Bill -4218 showing -4219 ulse -4220 manif -4221 Diary -4222 ics -4223 ically -4224 vac -4225 loved -4226 diss -4227 acity -4228 attach -4229 rested -4230 instinct -4231 excuse -4232 meric -4233 ceiving -4234 powerful -4235 personal -4236 shade -4237 Ladyship -4238 driver -4239 driven -4240 conviction -4241 circumstance -4242 equally -4243 hungry -4244 protect -4245 superior -4246 Harper -4247 conclusion -4248 destroyed -4249 concerned -4250 wandered -4251 consequence -4252 alarm -4253 disappointment -4254 hyp -4255 frequently -4256 McC -4257 comprehe -4258 00 -4259 Are -4260 Huck -4261 Indeed -4262 away -4263 cence -4264 ds -4265 hum -4266 oot -4267 vent -4268 rest -4269 mourn -4270 citizen -4271 due -4272 dwell -4273 height -4274 atul -4275 et -4276 idge -4277 uries -4278 rats -4279 Hert -4280 amb -4281 Aust -4282 rose -4283 liest -4284 Bour -4285 Buck -4286 ability -4287 ourt -4288 intended -4289 lear -4290 willing -4291 shame -4292 rope -4293 envelo -4294 assed -4295 scream -4296 spend -4297 plenty -4298 Re -4299 Roy -4300 Ruc -4301 Vi -4302 igned -4303 returning -4304 native -4305 drif -4306 purcha -4307 treasure -4308 advanced -4309 gives -4310 patri -4311 points -4312 played -4313 warning -4314 fortunate -4315 observe -4316 missed -4317 solid -4318 weakness -4319 freed -4320 pictures -4321 destroy -4322 employ -4323 coat -4324 counter -4325 building -4326 mysterious -4327 gradually -4328 conscience -4329 uddenly -4330 Hertfordshire -4331 Bourgh -4332 At -4333 Have -4334 Qu -4335 Say -4336 gl -4337 rist -4338 tle -4339 uge -4340 £ -4341 arose -4342 oper -4343 mal -4344 bowed -4345 fill -4346 orough -4347 inn -4348 pin -4349 parent -4350 ham -4351 beloved -4352 gwyne -4353 elo -4354 Turn -4355 ash -4356 whir -4357 stones -4358 irst -4359 Have -4360 sheet -4361 About -4362 Americ -4363 dozen -4364 Besides -4365 features -4366 resp -4367 Dorm -4368 olate -4369 uments -4370 failed -4371 empl -4372 choose -4373 blessed -4374 sche -4375 chamber -4376 desired -4377 aced -4378 lessly -4379 Vengeance -4380 kindly -4381 restr -4382 Still -4383 disting -4384 swall -4385 clever -4386 excit -4387 useful -4388 strongly -4389 suffering -4390 decl -4391 understanding -4392 consideration -4393 shortly -4394 box -4395 midst -4396 explained -4397 sensible -4398 buy -4399 fanc -4400 request -4401 movement -4402 coffee -4403 describe -4404 ruin -4405 struggle -4406 ournal -4407 murderer -4408 surrounded -4409 189 -4410 shudder -4411 Whitby -4412 farther -4413 McCarthy -4414 Dormouse -4415 Never -4416 ON -4417 can -4418 case -4419 dy -4420 ium -4421 oard -4422 yle -4423 tide -4424 rees -4425 foo -4426 cup -4427 darling -4428 hol -4429 orne -4430 poured -4431 gather -4432 stone -4433 rene -4434 weary -4435 weap -4436 whist -4437 store -4438 sof -4439 Simon -4440 deb -4441 upstairs -4442 Will -4443 Ben -4444 Barsad -4445 shone -4446 shove -4447 ables -4448 noon -4449 libr -4450 jump -4451 Journal -4452 ists -4453 umph -4454 share -4455 shake -4456 Gu -4457 pop -4458 Or -4459 Old -4460 Once -4461 clim -4462 obv -4463 acle -4464 angers -4465 sooner -4466 entertain -4467 indign -4468 flung -4469 emer -4470 corre -4471 trace -4472 excess -4473 inspect -4474 Forster -4475 tele -4476 concerning -4477 acted -4478 fortnight -4479 marks -4480 ustomed -4481 fifteen -4482 groan -4483 adventure -4484 higher -4485 neglect -4486 flies -4487 pilgr -4488 taste -4489 client -4490 nodded -4491 keeper -4492 .) -4493 An -4494 May -4495 Miss -4496 cl -4497 rom -4498 uced -4499 would -4500 4 -4501 £ -4502 woke -4503 sin -4504 erred -4505 bla -4506 born -4507 itate -4508 tore -4509 Ind -4510 hit -4511 hunt -4512 lun -4513 lake -4514 pra -4515 patter -4516 âĢĿ-- -4517 ldom -4518 rele -4519 assemb -4520 stuff -4521 irth -4522 ooped -4523 ribut -4524 kit -4525 alley -4526 Lest -4527 hant -4528 wholly -4529 agitation -4530 trade -4531 howling -4532 proc -4533 Gr -4534 remains -4535 appears -4536 berry -4537 ache -4538 race -4539 ertainly -4540 secure -4541 fellows -4542 swell -4543 persons -4544 bedroom -4545 fearful -4546 south -4547 opy -4548 arrive -4549 See -4550 devoted -4551 Only -4552 gratul -4553 summon -4554 judgment -4555 anguish -4556 bringing -4557 probable -4558 compliment -4559 disturb -4560 Peter -4561 Finn -4562 acknowled -4563 Lestrade -4564 ald -4565 chen -4566 eem -4567 ta -4568 bal -4569 bull -4570 ded -4571 lar -4572 lone -4573 beheld -4574 gle -4575 lection -4576 iday -4577 thread -4578 thorough -4579 reward -4580 resemb -4581 star -4582 Hill -4583 Hare -4584 seldom -4585 Bar -4586 knock -4587 unes -4588 Court -4589 anywhere -4590 comb -4591 amid -4592 usually -4593 spark -4594 thanks -4595 Go -4596 thinks -4597 coast -4598 chim -4599 remind -4600 Young -4601 consent -4602 lighted -4603 contempl -4604 accustomed -4605 moral -4606 waves -4607 licit -4608 confir -4609 incon -4610 awoke -4611 sunk -4612 rence -4613 prisoners -4614 skiff -4615 XI -4616 invitation -4617 admire -4618 sentence -4619 gram -4620 charming -4621 forgive -4622 crept -4623 sensations -4624 gravely -4625 respectable -4626 selfish -4627 utmost -4628 experi -4629 attendant -4630 leaned -4631 gloomy -4632 Next -4633 carrying -4634 persuaded -4635 going -4636 troubled -4637 Christ -4638 magn -4639 misfortune -4640 welcome -4641 delivered -4642 Rucastle -4643 library -4644 opyright -4645 ?_ -4646 bar -4647 hile -4648 ma -4649 oc -4650 pers -4651 sk -4652 wing -4653 william -4654 æ -4655 test -4656 song -4657 mat -4658 mil -4659 iture -4660 flew -4661 informed -4662 omed -4663 gest -4664 goose -4665 âĢĻ, -4666 stri -4667 ental -4668 revol -4669 Tem -4670 adi -4671 ads -4672 stuck -4673 notion -4674 cking -4675 anc -4676 rier -4677 Suddenly -4678 shel -4679 conject -4680 congratul -4681 unted -4682 uls -4683 exert -4684 owned -4685 oses -4686 amuse -4687 madam -4688 Fitz -4689 Never -4690 boots -4691 clue -4692 closing -4693 desert -4694 plans -4695 consolation -4696 enth -4697 stillness -4698 instru -4699 emotion -4700 sunsh -4701 startled -4702 refuse -4703 advance -4704 wooden -4705 souls -4706 lively -4707 circle -4708 Allen -4709 affected -4710 eneva -4711 driving -4712 cript -4713 destruction -4714 delightful -4715 affectionate -4716 grie -4717 ilips -4718 settle -4719 apolog -4720 hastily -4721 mountain -4722 civil -4723 persuade -4724 value -4725 captain -4726 choice -4727 murdered -4728 polit -4729 extremely -4730 attentions -4731 exercise -4732 apprehens -4733 Philips -4734 attachment -4735 1894 -4736 Fitzwilliam -4737 Copyright -4738 UR -4739 ]] -4740 de -4741 fl -4742 fall -4743 ivers -4744 nel -4745 off -4746 pa -4747 rance -4748 sing -4749 ] -4750 sour -4751 rene -4752 ounce -4753 mour -4754 bund -4755 tools -4756 dancing -4757 pounds -4758 bew -4759 leberry -4760 urance -4761 style -4762 hers -4763 shed -4764 wheel -4765 aloud -4766 agin -4767 amount -4768 overcome -4769 armed -4770 space -4771 aging -4772 strain -4773 appo -4774 erent -4775 betray -4776 spell -4777 -4778 compar -4779 composed -4780 endured -4781 restored -4782 lication -4783 discuss -4784 useless -4785 expectation -4786 condem -4787 triumph -4788 [_ -4789 departed -4790 occasionally -4791 bearing -4792 engers -4793 fairly -4794 vex -4795 eleg -4796 Death -4797 bottle -4798 inquiry -4799 soldiers -4800 employed -4801 recollect -4802 delicate -4803 furniture -4804 increased -4805 beneath -4806 intelligence -4807 yonder -4808 folks -4809 Majesty -4810 explanation -4811 hypnot -4812 etc -4813 freedom -4814 obvious -4815 lonely -4816 sunshine -4817 Pray -4818 back -4819 don -4820 jud -4821 kind -4822 ole -4823 port -4824 rant -4825 uing -4826 uished -4827 hearted -4828 som -4829 hast -4830 oug -4831 ferry -4832 toss -4833 onn -4834 oring -4835 asing -4836 lid -4837 posit -4838 âĢĿâĢĶ -4839 withd -4840 forms -4841 stars -4842 irty -4843 hero -4844 Having -4845 Hurst -4846 illar -4847 Sil -4848 Ah -4849 exha -4850 Li -4851 abhor -4852 ighed -4853 quitted -4854 env -4855 brass -4856 gets -4857 perform -4858 resent -4859 mostly -4860 entering -4861 flat -4862 flame -4863 raising -4864 horrid -4865 licity -4866 corn -4867 possibility -4868 readily -4869 ._]] -4870 doesn -4871 visited -4872 gracious -4873 finish -4874 stayed -4875 relation -4876 playing -4877 reality -4878 clearly -4879 example -4880 necessity -4881 torn -4882 churchyard -4883 estab -4884 slip -4885 Nothing -4886 bbing -4887 Again -4888 special -4889 indifferent -4890 Scot -4891 hollow -4892 desperate -4893 energy -4894 typewr -4895 victim -4896 Victor -4897 respond -4898 excitement -4899 kitchen -4900 EN -4901 Ex -4902 St -4903 cred -4904 eau -4905 gu -4906 oci -4907 rise -4908 sc -4909 uable -4910 why -4911 wag -4912 wreck -4913 erman -4914 bother -4915 itating -4916 cost -4917 cere -4918 orous -4919 heap -4920 niece -4921 omes -4922 beings -4923 gay -4924 yard -4925 verse -4926 Tur -4927 stup -4928 rat -4929 row -4930 measure -4931 sacr -4932 sell -4933 severe -4934 frame -4935 ouring -4936 Con -4937 managed -4938 During -4939 aboard -4940 unus -4941 umn -4942 agers -4943 proble -4944 dish -4945 Geneva -4946 German -4947 grind -4948 advert -4949 class -4950 appeal -4951 obst -4952 content -4953 ensive -4954 secre -4955 correspond -4956 deadly -4957 ivid -4958 excl -4959 snake -4960 shoved -4961 sincere -4962 visible -4963 stronger -4964 sorts -4965 sweat -4966 pier -4967 umbled -4968 job -4969 wilder -4970 charm -4971 banks -4972 dressing -4973 travelling -4974 propos -4975 narr -4976 wretch -4977 Uncle -4978 earnestly -4979 remembr -4980 accompan -4981 hunger -4982 flying -4983 suggested -4984 gaol -4985 rapid -4986 profound -4987 North -4988 kerchief -4989 wondered -4990 observation -4991 Dracul -4992 baby -4993 comprehend -4994 sofa -4995 Th -4996 ians -4997 master -4998 iv -4999 tal -5000 west -5001 mum -5002 lane -5003 pan -5004 beating -5005 stand -5006 sec -5007 staring -5008 notes -5009 rum -5010 sacred -5011 season -5012 deed -5013 road -5014 geon -5015 Bo -5016 alled -5017 andy -5018 usi -5019 aband -5020 leis -5021 eling -5022 scen -5023 friendly -5024 waist -5025 slid -5026 breaking -5027 sleepy -5028 silver -5029 confused -5030 associ -5031 commence -5032 helped -5033 terr -5034 Huckleberry -5035 eneral -5036 wondering -5037 patience -5038 invest -5039 touching -5040 eyeb -5041 particulars -5042 built -5043 escaped -5044 educ -5045 hidden -5046 Everybody -5047 immediate -5048 motive -5049 bench -5050 described -5051 description -5052 drinking -5053 centre -5054 limbs -5055 thirst -5056 fatig -5057 tearing -5058 work -5059 heavens -5060 trembled -5061 closer -5062 represent -5063 echoes -5064 declared -5065 folded -5066 elbow -5067 arranged -5068 pronounced -5069 queer -5070 correct -5071 pattern -5072 enthusi -5073 bundle -5074 Dracula -5075 !-- -5076 Certainly -5077 Go -5078 Sh -5079 Thank -5080 With -5081 Will -5082 een -5083 five -5084 father -5085 pre -5086 ulf -5087 water -5088 & -5089 wuz -5090 ouis -5091 erves -5092 borrow -5093 bloom -5094 fully -5095 toward -5096 card -5097 cig -5098 ashing -5099 lift -5100 ese -5101 esh -5102 pist -5103 ator -5104 atory -5105 glo -5106 ghast -5107 utation -5108 roused -5109 alities -5110 Mars -5111 ctions -5112 ifying -5113 roach -5114 upper -5115 upset -5116 lier -5117 conven -5118 Cla -5119 ulous -5120 vengeance -5121 Pray -5122 handkerchief -5123 truly -5124 lover -5125 letting -5126 preced -5127 desir -5128 descend -5129 attached -5130 ords -5131 Ro -5132 harbour -5133 inde -5134 endure -5135 flash -5136 onsense -5137 instance -5138 hopeless -5139 aughter -5140 awhile -5141 reflection -5142 ugged -5143 obsc -5144 needed -5145 brothers -5146 related -5147 footsteps -5148 depth -5149 objects -5150 objection -5151 examination -5152 Another -5153 display -5154 crowded -5155 solitary -5156 lawyer -5157 solemnly -5158 square -5159 astonished -5160 merc -5161 enjoyment -5162 reflections -5163 animal -5164 disgrace -5165 inclination -5166 precip -5167 highest -5168 rapidly -5169 uncomfort -5170 uncommon -5171 ribly -5172 succeeded -5173 mixed -5174 branches -5175 virtue -5176 stein -5177 puzzled -5178 unexpected -5179 ashore -5180 pilgrims -5181 ivory -5182 ** -5183 Wh -5184 aly -5185 bly -5186 iven -5187 ville -5188 who -5189 waked -5190 has -5191 have -5192 mood -5193 ble -5194 bark -5195 fishing -5196 toler -5197 cuss -5198 hither -5199 orse -5200 heels -5201 pall -5202 omest -5203 behold -5204 gla -5205 east -5206 eting -5207 Take -5208 rot -5209 Sol -5210 rod -5211 Wind -5212 pel -5213 feather -5214 Louis -5215 ves -5216 vig -5217 elle -5218 ums -5219 disag -5220 grab -5221 psed -5222 ucks -5223 onday -5224 tells -5225 peep -5226 chap -5227 aches -5228 waters -5229 indul -5230 ignity -5231 oubted -5232 surf -5233 confident -5234 confusion -5235 pleasing -5236 guilt -5237 starting -5238 doubtless -5239 grace -5240 aterp -5241 Art -5242 continually -5243 grounds -5244 admit -5245 landed -5246 White -5247 prevented -5248 striking -5249 fastened -5250 ........ -5251 worthy -5252 leading -5253 veins -5254 prepar -5255 frightful -5256 pursuit -5257 remaining -5258 hills -5259 assistance -5260 picked -5261 struggled -5262 rags -5263 deserted -5264 wheres -5265 treated -5266 according -5267 trembling -5268 origin -5269 urged -5270 savage -5271 bodies -5272 lunatic -5273 chimney -5274 wilderness -5275 ghastly -5276 Bec -5277 His -5278 bits -5279 bacc -5280 eep -5281 even -5282 ially -5283 natur -5284 pon -5285 pent -5286 tie -5287 tail -5288 wha -5289 mach -5290 mail -5291 mult -5292 tol -5293 only -5294 der -5295 heat -5296 por -5297 beast -5298 ec -5299 idents -5300 veyed -5301 street -5302 weep -5303 wealth -5304 urs -5305 urope -5306 meas -5307 More -5308 Mouse -5309 Monday -5310 Bet -5311 abelle -5312 neat -5313 pended -5314 Heading -5315 unning -5316 Could -5317 exh -5318 ounding -5319 alter -5320 intr -5321 valley -5322 aks -5323 agent -5324 unwill -5325 osen -5326 fac -5327 Europe -5328 arrest -5329 spr -5330 tramp -5331 trades -5332 thanked -5333 brood -5334 bride -5335 cover -5336 chill -5337 chosen -5338 blast -5339 boast -5340 scattered -5341 smiles -5342 passenger -5343 obey -5344 glory -5345 impress -5346 winding -5347 flight -5348 kinds -5349 accur -5350 bushes -5351 swim -5352 addition -5353 sunrise -5354 because -5355 dragged -5356 runaway -5357 locked -5358 ampire -5359 skin -5360 generous -5361 reasonable -5362 uffled -5363 infinite -5364 invol -5365 absurd -5366 continue -5367 imagined -5368 suspected -5369 previous -5370 Angel -5371 missing -5372 hiding -5373 solitude -5374 official -5375 orders -5376 collected -5377 violence -5378 marrying -5379 embra -5380 constant -5381 uneasiness -5382 improve -5383 fatal -5384 guns -5385 message -5386 disappointed -5387 practice -5388 pressing -5389 contrary -5390 approaching -5391 authority -5392 benevol -5393 execution -5394 departure -5395 apparently -5396 instrument -5397 inhabit -5398 Austen -5399 Turner -5400 summoned -5401 Temple -5402 paper -5403 Silas -5404 remembrance -5405 aterpillar -5406 Because -5407 Can -5408 ait -5409 apt -5410 ipped -5411 kward -5412 nat -5413 sm -5414 school -5415 two -5416 slo -5417 rem -5418 erby -5419 bat -5420 bust -5421 birth -5422 ites -5423 isc -5424 fled -5425 fulf -5426 cared -5427 enstein -5428 onet -5429 Indeed -5430 asions -5431 pushed -5432 omon -5433 âĢĻ. -5434 etern -5435 stead -5436 ential -5437 rela -5438 urg -5439 ride -5440 rived -5441 sust -5442 Alas -5443 deceived -5444 rop -5445 Cat -5446 Comp -5447 Caterpillar -5448 ---- -5449 manage -5450 La -5451 icle -5452 akened -5453 jaw -5454 jew -5455 unlike -5456 James -5457 July -5458 downstairs -5459 dislike -5460 Far -5461 First -5462 Out -5463 thrust -5464 plex -5465 begged -5466 owner -5467 angled -5468 partial -5469 Though -5470 entrance -5471 moan -5472 flesh -5473 knee -5474 awkward -5475 wolf -5476 ailing -5477 bestowed -5478 shop -5479 deeper -5480 absor -5481 satisfy -5482 dreaded -5483 ladder -5484 mony -5485 inquire -5486 statement -5487 senses -5488 uttered -5489 ification -5490 Everything -5491 pursue -5492 slightly -5493 travelled -5494 phot -5495 temp -5496 reflected -5497 inclined -5498 prosper -5499 recollection -5500 Span -5501 Peters -5502 indifference -5503 encourage -5504 lamps -5505 cellar -5506 staircase -5507 engagement -5508 glimpse -5509 criminal -5510 padd -5511 philosophy -5512 manifest -5513 envelope -5514 scheme -5515 larger -5516 uncomfortable -5517 Solomon -5518 tolerable -5519 Take -5520 ius -5521 lance -5522 ocked -5523 rated -5524 tin -5525 vis -5526 ess -5527 tit -5528 hem -5529 ax -5530 reg -5531 fasc -5532 foul -5533 tobacc -5534 drow -5535 dirty -5536 hut -5537 asm -5538 ling -5539 laughter -5540 arr -5541 inex -5542 pool -5543 gain -5544 lease -5545 waste -5546 seen -5547 receiving -5548 wept -5549 stake -5550 ray -5551 rude -5552 rode -5553 antic -5554 sees -5555 ifice -5556 illot -5557 ament -5558 Are -5559 doan -5560 frau -5561 âĢĶ_ -5562 Co -5563 Last -5564 Derby -5565 olt -5566 overwhe -5567 using -5568 spl -5569 Pr -5570 twink -5571 Five -5572 coarse -5573 chose -5574 throwing -5575 clay -5576 scre -5577 perplex -5578 becoming -5579 desk -5580 descended -5581 wanting -5582 passengers -5583 partner -5584 impulse -5585 Rob -5586 Those -5587 composure -5588 retreat -5589 minds -5590 accused -5591 restless -5592 rail -5593 recover -5594 support -5595 lawn -5596 voy -5597 .âĢĻâĢĿ -5598 corpse -5599 committed -5600 helpless -5601 uncertain -5602 refused -5603 regiment -5604 considering -5605 cats -5606 trick -5607 freely -5608 seconds -5609 anxiously -5610 resolute -5611 sensation -5612 crack -5613 Any -5614 complex -5615 funny -5616 esteem -5617 XXV -5618 travell -5619 pockets -5620 squee -5621 relieved -5622 thankful -5623 plainly -5624 stirred -5625 privile -5626 delicacy -5627 frequent -5628 murmured -5629 chateau -5630 Huns -5631 liberty -5632 principal -5633 ancient -5634 secrecy -5635 eyebrows -5636 Clair -5637 overwhel -5638 God -5639 Tell -5640 _âĢĶ -5641 _,âĢĿ -5642 could -5643 got -5644 oh -5645 ule -5646 uously -5647 wenty -5648 tuck -5649 taught -5650 wave -5651 san -5652 bree -5653 borne -5654 itable -5655 fret -5656 tones -5657 cri -5658 cents -5659 danced -5660 pure -5661 haven -5662 hated -5663 atha -5664 ge -5665 gas -5666 idable -5667 cean -5668 enth -5669 del -5670 âĢĶâĢľ -5671 usions -5672 intense -5673 someone -5674 govern -5675 umps -5676 ivered -5677 Pe -5678 handy -5679 eless -5680 brandy -5681 Gabelle -5682 loc -5683 wherever -5684 acon -5685 aces -5686 glor -5687 impl -5688 wrist -5689 lieve -5690 distressed -5691 carri -5692 retty -5693 incred -5694 corrid -5695 miser -5696 certainty -5697 chest -5698 exchan -5699 advise -5700 skill -5701 illness -5702 absent -5703 Jones -5704 inquired -5705 journal -5706 wishing -5707 duties -5708 prepare -5709 meantime -5710 elev -5711 candles -5712 connect -5713 ordinary -5714 eagerness -5715 warmth -5716 concluded -5717 vanity -5718 embarr -5719 completely -5720 valuable -5721 hideous -5722 chair -5723 injured -5724 polite -5725 !_âĢĿ -5726 gazed -5727 fields -5728 torture -5729 Frankenstein -5730 curtain -5731 seemingly -5732 purchase -5733 parents -5734 eloqu -5735 telegram -5736 praise -5737 released -5738 acknowledged -5739 magnific -5740 Scotland -5741 investig -5742 vessel -5743 natural -5744 photograph -5745 tobacco -5746 anticip -5747 illotine -5748 Later -5749 Perhaps -5750 cad -5751 dom -5752 fax -5753 ict -5754 mas -5755 pired -5756 rs -5757 rust -5758 say -5759 somet -5760 when -5761 tum -5762 aven -5763 alike -5764 sink -5765 silly -5766 od -5767 ole -5768 ouch -5769 mir -5770 bars -5771 isp -5772 cards -5773 dug -5774 damp -5775 III -5776 hook -5777 oration -5778 lunch -5779 pant -5780 park -5781 pile -5782 nerves -5783 âĢľâĢĻ -5784 iders -5785 stem -5786 thy -5787 chie -5788 render -5789 Thank -5790 stag -5791 stiff -5792 meal -5793 Him -5794 Haw -5795 Hold -5796 ate -5797 sore -5798 amped -5799 South -5800 Saf -5801 kings -5802 nearest -5803 ilty -5804 ilities -5805 peak -5806 raw -5807 ulation -5808 intimate -5809 abroad -5810 arts -5811 univers -5812 worry -5813 level -5814 En -5815 East -5816 roll -5817 ens -5818 likewise -5819 cop -5820 grin -5821 Nove -5822 loves -5823 loaf -5824 hoarse -5825 longed -5826 never -5827 bold -5828 cliff -5829 application -5830 perceive -5831 perpet -5832 friendship -5833 attract -5834 attitude -5835 parts -5836 resumed -5837 consult -5838 Very -5839 horiz -5840 changes -5841 ints -5842 bush -5843 keeps -5844 silently -5845 coronet -5846 commission -5847 careless -5848 sounded -5849 insens -5850 bestow -5851 shoem -5852 humour -5853 remarks -5854 honoured -5855 suspect -5856 observing -5857 visitors -5858 Without -5859 creation -5860 disappear -5861 torment -5862 XXI -5863 travel -5864 eleven -5865 guarded -5866 daily -5867 Brit -5868 degrees -5869 accomplished -5870 rule -5871 accepted -5872 wandering -5873 innocence -5874 enemies -5875 alarmed -5876 smooth -5877 Agatha -5878 rusty -5879 communication -5880 muttered -5881 utterly -5882 impatient -5883 argument -5884 murmur -5885 misfortunes -5886 cautiously -5887 ceiling -5888 trifle -5889 recommend -5890 interview -5891 procession -5892 learnt -5893 apprehension -5894 bewild -5895 unusual -5896 problem -5897 narrative -5898 accompanied -5899 scenes -5900 pistol -5901 disagree -5902 oubtedly -5903 exhib -5904 Derbyshire -5905 Hawkins -5906 November -5907 Hold -5908 aves -5909 action -5910 burn -5911 down -5912 low -5913 ns -5914 past -5915 pense -5916 upp -5917 ing -5918 wink -5919 waking -5920 thee -5921 rep -5922 mild -5923 erous -5924 bid -5925 eding -5926 faster -5927 cord -5928 cunning -5929 date -5930 dine -5931 asy -5932 aries -5933 pow -5934 nonsense -5935 atisf -5936 leet -5937 rev -5938 urb -5939 urity -5940 stage -5941 rout -5942 rare -5943 Hot -5944 Muff -5945 ighton -5946 lict -5947 Wal -5948 frown -5949 knocked -5950 fem -5951 exting -5952 icing -5953 ician -5954 iccad -5955 amb -5956 isten -5957 quant -5958 spiders -5959 rolling -5960 eled -5961 brilli -5962 madness -5963 udence -5964 begins -5965 arna -5966 partly -5967 resist -5968 glit -5969 impressed -5970 reproach -5971 indic -5972 individ -5973 exped -5974 flatter -5975 arson -5976 jected -5977 theless -5978 farm -5979 minister -5980 folly -5981 traces -5982 drank -5983 humble -5984 oppress -5985 relate -5986 fury -5987 suspense -5988 subjects -5989 17 -5990 studied -5991 system -5992 sixty -5993 examine -5994 softened -5995 keys -5996 occupation -5997 sorrowful -5998 attempted -5999 Brighton -6000 cottagers -6001 New -6002 disgu -6003 improved -6004 concealed -6005 penet -6006 ignorance -6007 satisfactory -6008 Carfax -6009 adventures -6010 ventured -6011 separate -6012 eldest -6013 motioned -6014 robber -6015 soothing -6016 feeble -6017 permission -6018 dwelling -6019 inspector -6020 thoroughly -6021 experienced -6022 gesture -6023 shelter -6024 standing -6025 abandoned -6026 Hunsford -6027 iccadilly -6028 ;âĢĿ -6029 Just -6030 aps -6031 ease -6032 fe -6033 liness -6034 lott -6035 may -6036 oes -6037 pher -6038 rous -6039 tree -6040 tack -6041 hel -6042 wings -6043 bought -6044 fitted -6045 citiz -6046 Im -6047 hosp -6048 hel -6049 hence -6050 lones -6051 pig -6052 pave -6053 oting -6054 wasted -6055 etry -6056 thou -6057 asc -6058 rushing -6059 sob -6060 iss -6061 rily -6062 ams -6063 sung -6064 ppers -6065 Whe -6066 Wed -6067 Would -6068 Wel -6069 shru -6070 about -6071 unal -6072 raction -6073 Life -6074 Day -6075 org -6076 anyhow -6077 oloured -6078 osmer -6079 law -6080 Its -6081 sport -6082 spared -6083 Pass -6084 Piccadilly -6085 howl -6086 daytime -6087 prolong -6088 Great -6089 grasp -6090 remorse -6091 clut -6092 applied -6093 score -6094 smart -6095 chains -6096 passages -6097 lasted -6098 parting -6099 parties -6100 residence -6101 glow -6102 Varna -6103 flut -6104 contained -6105 restore -6106 Soon -6107 rapt -6108 assurance -6109 occasions -6110 incident -6111 guest -6112 drops -6113 monstr -6114 Beau -6115 forever -6116 foreign -6117 illust -6118 signal -6119 final -6120 Friday -6121 umbling -6122 served -6123 Under -6124 price -6125 examining -6126 secrets -6127 occasional -6128 disposed -6129 barred -6130 capital -6131 crossing -6132 laws -6133 regarding -6134 hotel -6135 efforts -6136 stirring -6137 ruined -6138 beaten -6139 gratified -6140 coming -6141 hunted -6142 hunting -6143 deserve -6144 blessing -6145 portion -6146 separated -6147 impatience -6148 idle -6149 rejoiced -6150 respons -6151 original -6152 jacket -6153 shriek -6154 forbid -6155 nurse -6156 splendid -6157 steadily -6158 Roylott -6159 patriot -6160 America -6161 weapon -6162 Guillotine -6163 inconven -6164 stupid -6165 iveness -6166 brooding -6167 voyage -6168 Safie -6169 quantity -6170 ,âĢĶ -6171 General -6172 OR -6173 Re -6174 awn -6175 after -6176 cif -6177 cast -6178 dge -6179 dered -6180 gra -6181 gone -6182 give -6183 mel -6184 rough -6185 tell -6186 ux -6187 uction -6188 vity -6189 zz -6190 ident -6191 achie -6192 ino -6193 inction -6194 sway -6195 theory -6196 ming -6197 bas -6198 bul -6199 batt -6200 blew -6201 ferv -6202 cries -6203 dumb -6204 dirt -6205 loo -6206 num -6207 omas -6208 oman -6209 âĢĿ; -6210 eter -6211 thumb -6212 verty -6213 ashes -6214 radi -6215 meat -6216 angel -6217 Hosmer -6218 rious -6219 alley -6220 rob -6221 rol -6222 Bast -6223 shiver -6224 frag -6225 quest -6226 felicity -6227 ulated -6228 lip -6229 intim -6230 jest -6231 unple -6232 leather -6233 isto -6234 umin -6235 arch -6236 Pa -6237 poison -6238 poverty -6239 uding -6240 udent -6241 gray -6242 Op -6243 strangers -6244 clasped -6245 apply -6246 performed -6247 desolate -6248 speck -6249 Eliza -6250 consul -6251 harsh -6252 flour -6253 heartily -6254 rivers -6255 emig -6256 earlier -6257 surm -6258 confined -6259 shman -6260 guilty -6261 ivity -6262 checked -6263 subd -6264 subm -6265 crimes -6266 detail -6267 expecting -6268 teach -6269 stands -6270 needn -6271 closely -6272 invited -6273 prompt -6274 condes -6275 created -6276 sentiment -6277 series -6278 chairs -6279 favourable -6280 spoon -6281 prayer -6282 motives -6283 phr -6284 asket -6285 proport -6286 superst -6287 haste -6288 hastened -6289 hurriedly -6290 bbers -6291 collect -6292 mercy -6293 loudly -6294 Harvey -6295 stretch -6296 bitterly -6297 riding -6298 centuries -6299 transf -6300 disgust -6301 provided -6302 lucky -6303 President -6304 ragged -6305 belonged -6306 families -6307 contrast -6308 gallant -6309 rubbed -6310 amazement -6311 yielded -6312 consciousness -6313 recalled -6314 risk -6315 swiftly -6316 labours -6317 nesday -6318 pirates -6319 tranquill -6320 prefer -6321 mournful -6322 drifting -6323 whistle -6324 ributed -6325 tavern -6326 resemblance -6327 ?_âĢĿ -6328 amusement -6329 elegant -6330 sombre -6331 resentment -6332 Turk -6333 grindstone -6334 advertis -6335 leisure -6336 Which -6337 peeped -6338 unwilling -6339 drownded -6340 corridor -6341 disagreeable -6342 Welshman -6343 monstrous -6344 Bastille -6345 30 -6346 AL -6347 Dear -6348 every -6349 equ -6350 eff -6351 faced -6352 ipt -6353 line -6354 nest -6355 oon -6356 ture -6357 win -6358 tid -6359 tax -6360 haw -6361 oust -6362 mate -6363 mental -6364 ban -6365 fain -6366 tot -6367 dus -6368 domest -6369 Ill -6370 llable -6371 ashes -6372 lord -6373 lux -6374 ink -6375 page -6376 habits -6377 ators -6378 oms -6379 âĢľâĢĶ -6380 gown -6381 gift -6382 yawn -6383 wash -6384 ere -6385 ascended -6386 urre -6387 stare -6388 rom -6389 Ha -6390 Slo -6391 sheep -6392 develo -6393 wheels -6394 Both -6395 Black -6396 shows -6397 ilst -6398 alling -6399 andal -6400 outer -6401 feature -6402 conveyed -6403 Cr -6404 ----âĢĿ -6405 altered -6406 joke -6407 agon -6408 agitated -6409 united -6410 umption -6411 quality -6412 Pretty -6413 bris -6414 offended -6415 prin -6416 Give -6417 everywhere -6418 Four -6419 adj -6420 Other -6421 blot -6422 scru -6423 attracted -6424 reserved -6425 plant -6426 plank -6427 Think -6428 household -6429 housekeeper -6430 repose -6431 induce -6432 ends -6433 ended -6434 State -6435 recent -6436 Kn -6437 izing -6438 counted -6439 stopping -6440 inches -6441 awakened -6442 guine -6443 chem -6444 pretend -6445 drag -6446 subst -6447 shoe -6448 shocked -6449 lesson -6450 decide -6451 active -6452 patron -6453 extent -6454 cares -6455 causes -6456 circus -6457 stepfather -6458 dreadfully -6459 floating -6460 sentiments -6461 serve -6462 studying -6463 creep -6464 creeping -6465 Whilst -6466 shutters -6467 humanity -6468 irr -6469 determination -6470 while -6471 memories -6472 sadly -6473 deny -6474 steamer -6475 protest -6476 views -6477 assisted -6478 assistant -6479 singing -6480 formerly -6481 suspicious -6482 lifting -6483 modern -6484 messenger -6485 sufferings -6486 precaut -6487 counsel -6488 endeavouring -6489 introduce -6490 introduced -6491 accordingly -6492 ealous -6493 annoy -6494 Finally -6495 caution -6496 apartment -6497 represented -6498 declare -6499 tremble -6500 crawled -6501 wigw -6502 pirate -6503 imprison -6504 intervals -6505 bending -6506 deliber -6507 Wilson -6508 Quite -6509 hammer -6510 indignation -6511 emerged -6512 politeness -6513 positively -6514 exhausted -6515 sacrifice -6516 exclam -6517 education -6518 depths -6519 hitherto -6520 surface -6521 Company -6522 complexion -6523 People -6524 female -6525 ascertain -6526 Wednesday -6527 Slov -6528 .-- -6529 Jim -6530 Mina -6531 Some -6532 See -6533 aile -6534 ault -6535 boy -6536 cil -6537 eard -6538 kers -6539 lin -6540 phan -6541 sp -6542 table -6543 van -6544 wl -6545 ys -6546 aut -6547 sings -6548 ocean -6549 bully -6550 bones -6551 busted -6552 itive -6553 fired -6554 fren -6555 tooth -6556 dignity -6557 II -6558 llen -6559 lap -6560 ann -6561 omy -6562 aying -6563 rent -6564 Time -6565 stol -6566 aling -6567 ales -6568 alat -6569 sex -6570 Sm -6571 Sus -6572 Ab -6573 deceive -6574 Wid -6575 Baile -6576 ours -6577 nostri -6578 knot -6579 knitted -6580 seeking -6581 ander -6582 convey -6583 arded -6584 unately -6585 mankind -6586 alas -6587 whence -6588 Doug -6589 intol -6590 vul -6591 jis -6592 ages -6593 union -6594 ositions -6595 istr -6596 qualities -6597 arise -6598 roar -6599 rocks -6600 handle -6601 ems -6602 trance -6603 bridge -6604 perhaps -6605 prejud -6606 peril -6607 pew -6608 aping -6609 apable -6610 plate -6611 happily -6612 winds -6613 resting -6614 preserve -6615 rav -6616 ensity -6617 supply -6618 confessed -6619 assert -6620 incapable -6621 dischar -6622 sunlight -6623 trap -6624 tracks -6625 terribly -6626 reference -6627 refuge -6628 referred -6629 prise -6630 (_ -6631 snuff -6632 snatched -6633 crash -6634 Att -6635 hush -6636 Nobody -6637 actions -6638 XL -6639 ibank -6640 ontin -6641 answering -6642 easier -6643 heads -6644 cavern -6645 infam -6646 painted -6647 admired -6648 admitted -6649 burnt -6650 seriously -6651 breathe -6652 devils -6653 summ -6654 accounts -6655 destiny -6656 Maria -6657 thousands -6658 calcul -6659 despond -6660 completed -6661 directions -6662 repeat -6663 effects -6664 veil -6665 maj -6666 variety -6667 glasses -6668 Dev -6669 neighbours -6670 violently -6671 labor -6672 warmly -6673 accomplish -6674 shouting -6675 medic -6676 frost -6677 burned -6678 witnesses -6679 unfold -6680 produce -6681 smoking -6682 massive -6683 Letter -6684 spreading -6685 attentive -6686 cheerfulness -6687 mortal -6688 garret -6689 garlic -6690 flick -6691 slave -6692 rattle -6693 dismal -6694 branch -6695 impertin -6696 professional -6697 temptation -6698 wrapped -6699 obtained -6700 pillow -6701 popular -6702 Christmas -6703 exertion -6704 mourning -6705 condemned -6706 tossed -6707 withdrew -6708 proposal -6709 rotten -6710 Windibank -6711 machine -6712 axe -6713 breeze -6714 connections -6715 magnificent -6716 something -6717 pavement -6718 cifix -6719 unpleasant -6720 subdued -6721 advertisement -6722 wigwam -6723 Bailey -6724 Douglas -6725 :âĢĶ -6726 ;-- -6727 AN -6728 AT -6729 Bl -6730 ES -6731 Would -6732 bank -6733 dig -6734 gly -6735 good -6736 gings -6737 houses -6738 know -6739 luck -6740 ov -6741 pri -6742 rim -6743 such -6744 spect -6745 6 -6746 tiny -6747 tipt -6748 hetic -6749 wounded -6750 sane -6751 sinking -6752 omin -6753 rels -6754 hable -6755 haired -6756 mam -6757 bord -6758 eds -6759 fid -6760 inger -6761 inged -6762 daz -6763 dæ -6764 IV -6765 orney -6766 lots -6767 arly -6768 arvis -6769 pred -6770 gates -6771 yer -6772 ids -6773 etition -6774 verted -6775 reception -6776 Tw -6777 Tues -6778 forces -6779 stool -6780 stump -6781 atm -6782 Mont -6783 ighted -6784 iller -6785 butter -6786 sugar -6787 kin -6788 Winc -6789 gered -6790 ships -6791 âĢĶâĢĺ -6792 pelled -6793 outra -6794 conceive -6795 rap -6796 City -6797 Cross -6798 uled -6799 exal -6800 lick -6801 asting -6802 oline -6803 June -6804 Jarvis -6805 leaped -6806 afeard -6807 shaken -6808 robbers -6809 emed -6810 trous -6811 truck -6812 brains -6813 disease -6814 ressing -6815 chile -6816 loaded -6817 ours -6818 OF -6819 hoping -6820 string -6821 clung -6822 Shepher -6823 prey -6824 preacher -6825 appet -6826 scandal -6827 older -6828 becomes -6829 mean -6830 peasant -6831 doorway -6832 plun -6833 Rog -6834 alking -6835 reputation -6836 VI -6837 Stoner -6838 streak -6839 works -6840 Soho -6841 ipes -6842 carpet -6843 supported -6844 swar -6845 swung -6846 follows -6847 income -6848 openly -6849 enny -6850 travers -6851 curse -6852 drowned -6853 tread -6854 doubted -6855 insuff -6856 inspir -6857 snag -6858 lessons -6859 doubts -6860 Arab -6861 actual -6862 triump -6863 pains -6864 expressions -6865 engine -6866 sermon -6867 summit -6868 sweetness -6869 metal -6870 trusted -6871 shutter -6872 leaf -6873 characterist -6874 determine -6875 exception -6876 horrors -6877 memor -6878 quarrel -6879 propri -6880 attending -6881 regarded -6882 sympt -6883 successful -6884 relieve -6885 err -6886 error -6887 animated -6888 whispering -6889 undoubtedly -6890 admirable -6891 crucifix -6892 furnished -6893 introduction -6894 venture -6895 faithful -6896 speedily -6897 encouraged -6898 shuddered -6899 Hunter -6900 clerk -6901 clergy -6902 break -6903 arrangements -6904 audience -6905 genius -6906 Trans -6907 recognised -6908 drifted -6909 employment -6910 operation -6911 fancied -6912 fools -6913 holes -6914 sinister -6915 reminded -6916 apology -6917 comparison -6918 dishon -6919 gaoler -6920 grabbed -6921 multitude -6922 inhabitants -6923 natured -6924 eternal -6925 glorious -6926 tumult -6927 individual -6928 arsonage -6929 mingled -6930 istocr -6931 tranquillity -6932 Widow -6933 intolerable -6934 Tuesday -6935 Rogers -6936 ED -6937 Lord -6938 OUR -6939 Was -6940 bird -6941 came -6942 course -6943 dam -6944 fty -6945 hine -6946 mark -6947 said -6948 upt -6949 ye -6950 ---- -6951 wi -6952 sch -6953 shan -6954 rer -6955 reast -6956 muffled -6957 erate -6958 basket -6959 ishes -6960 faded -6961 cas -6962 dashed -6963 domin -6964 ork -6965 lin -6966 link -6967 init -6968 inher -6969 nation -6970 naked -6971 haunted -6972 otions -6973 leans -6974 etch -6975 thither -6976 entle -6977 Trib -6978 whither -6979 stout -6980 stamp -6981 rans -6982 Hather -6983 HAR -6984 Men -6985 solic -6986 serene -6987 iform -6988 rive -6989 Sister -6990 upward -6991 whip -6992 pple -6993 Wood -6994 Bistr -6995 frank -6996 knives -6997 inder -6998 Conc -6999 Lod -7000 abomin -7001 abreast -7002 jealous -7003 unb -7004 unnatural -7005 uniform -7006 gover -7007 moreover -7008 overboard -7009 overfl -7010 Park -7011 acked -7012 emember -7013 madman -7014 goods -7015 Galat -7016 Foul -7017 Nay -7018 Nature -7019 chuck -7020 logs -7021 lofty -7022 remove -7023 psies -7024 boot -7025 scorn -7026 chapel -7027 obed -7028 acles -7029 resour -7030 Russ -7031 writ -7032 gedy -7033 reported -7034 retire -7035 endless -7036 flames -7037 contents -7038 youngest -7039 dreary -7040 waved -7041 distinction -7042 intend -7043 secured -7044 swore -7045 emotions -7046 Dover -7047 discour -7048 mischief -7049 untarily -7050 guide -7051 check -7052 hole -7053 subs -7054 runs -7055 verses -7056 detect -7057 Force -7058 fores -7059 ulties -7060 sufferer -7061 extended -7062 sweep -7063 reasoning -7064 catched -7065 oppressed -7066 walks -7067 staying -7068 bury -7069 creator -7070 expressive -7071 landl -7072 landing -7073 imaginary -7074 mistress -7075 ground -7076 dipped -7077 studies -7078 iana -7079 sixteen -7080 prevailed -7081 hatred -7082 hundreds -7083 angrily -7084 mouths -7085 escort -7086 probability -7087 eaten -7088 neighb -7089 denied -7090 properly -7091 proposed -7092 Sherburn -7093 slee -7094 avail -7095 sorrows -7096 oblig -7097 cheerfully -7098 Neville -7099 hurrying -7100 inquiring -7101 animals -7102 bitterness -7103 wander -7104 calmly -7105 constantly -7106 raging -7107 advantages -7108 Church -7109 recollected -7110 Something -7111 wretchedness -7112 practical -7113 delayed -7114 results -7115 Caroline -7116 ARD -7117 idi -7118 lament -7119 tickets -7120 revealed -7121 risen -7122 ylum -7123 arrangement -7124 Only -7125 keenly -7126 distinctly -7127 instruments -7128 tranquil -7129 tumbri -7130 deliver -7131 permitted -7132 interference -7133 distinguish -7134 distinguished -7135 renewed -7136 neglected -7137 milk -7138 grieved -7139 onnet -7140 ceremony -7141 sincerely -7142 fatigue -7143 enthusiasm -7144 Road -7145 tolerably -7146 faculties -7147 tradesman -7148 improvement -7149 frauds -7150 privilege -7151 geese -7152 carriages -7153 Holder -7154 horizon -7155 atisfied -7156 lonesome -7157 domestic -7158 brisk -7159 nostrils -7160 vulgar -7161 dæmon -7162 atmosp -7163 Hatherley -7164 Bistritz -7165 Lodge -7166 Galatz -7167 Foulon -7168 Cz -7169 EW -7170 From -7171 Jon -7172 KER -7173 Keep -7174 None -7175 bage -7176 ded -7177 fo -7178 gany -7179 hip -7180 iance -7181 mind -7182 maid -7183 mwood -7184 nic -7185 olic -7186 part -7187 post -7188 twenty -7189 uits -7190 vag -7191 7 -7192 heard -7193 inations -7194 wot -7195 wreat -7196 wire -7197 sly -7198 sighed -7199 owe -7200 reen -7201 oung -7202 mos -7203 mouse -7204 erted -7205 bite -7206 isher -7207 isible -7208 isons -7209 fer -7210 fil -7211 fant -7212 fainted -7213 das -7214 dam -7215 dried -7216 dined -7217 dangers -7218 huge -7219 orner -7220 heed -7221 lat -7222 lur -7223 loung -7224 arina -7225 inner -7226 pup -7227 puff -7228 anc -7229 gn -7230 import -7231 stad -7232 see -7233 um -7234 relig -7235 stable -7236 stooped -7237 rip -7238 rasc -7239 ant -7240 Mur -7241 sober -7242 solicit -7243 sobbing -7244 seam -7245 isn -7246 dece -7247 licted -7248 doom -7249 geous -7250 Bos -7251 quet -7252 quare -7253 nois -7254 novel -7255 ilight -7256 thought -7257 cong -7258 Little -7259 Law -7260 likes -7261 orn -7262 usal -7263 vice -7264 yourselves -7265 jer -7266 unp -7267 unjust -7268 timid -7269 Jury -7270 lect -7271 amused -7272 lookout -7273 somewheres -7274 uming -7275 shapes -7276 overp -7277 overhead -7278 Er -7279 Ever -7280 Each -7281 trad -7282 brows -7283 brush -7284 twisted -7285 twilight -7286 image -7287 project -7288 disd -7289 earth -7290 chink -7291 hogs -7292 hovel -7293 nevertheless -7294 necess -7295 blowing -7296 cloth -7297 oller -7298 peas -7299 desolation -7300 passions -7301 rights -7302 everal -7303 Rome -7304 Thomas -7305 nery -7306 comply -7307 exposed -7308 flow -7309 presume -7310 pace -7311 Kent -7312 ension -7313 Holmwood -7314 sword -7315 ribb -7316 county -7317 counting -7318 incidents -7319 gentleness -7320 discourse -7321 ials -7322 guessed -7323 ival -7324 woe -7325 draught -7326 subl -7327 subtle -7328 snakes -7329 smallest -7330 cloak -7331 considerably -7332 remarkably -7333 whispers -7334 fourth -7335 heavier -7336 elsewhere -7337 floated -7338 questioning -7339 objections -7340 dreaming -7341 Within -7342 ciple -7343 capacity -7344 -7345 definite -7346 Englishman -7347 parish -7348 peaceful -7349 protection -7350 protested -7351 neighbouring -7352 searched -7353 collar -7354 Georg -7355 louder -7356 compliments -7357 suspicions -7358 enjoyed -7359 Yours -7360 coachman -7361 medical -7362 frock -7363 extravag -7364 deserved -7365 interruption -7366 injury -7367 digging -7368 worth -7369 customers -7370 ventil -7371 communicate -7372 communicated -7373 increasing -7374 species -7375 wounds -7376 impatiently -7377 executed -7378 chant -7379 gazing -7380 amazed -7381 response -7382 verd -7383 divine -7384 hesitated -7385 ghosts -7386 Exeter -7387 tenderly -7388 tenderness -7389 ylvan -7390 punishment -7391 brick -7392 Among -7393 lodgings -7394 physician -7395 demanded -7396 interfere -7397 vacant -7398 humour -7399 Royal -7400 patriots -7401 restraint -7402 swallowed -7403 renew -7404 assembled -7405 procured -7406 skin -7407 conjecture -7408 instructions -7409 source -7410 establish -7411 established -7412 arrested -7413 benevolent -7414 fulfil -7415 absorbed -7416 screamed -7417 shoemaker -7418 Hotel -7419 ambition -7420 disguise -7421 helps -7422 Whether -7423 inois -7424 battle -7425 numerous -7426 proportion -7427 dusk -7428 Illinois -7429 romantic -7430 principle -7431 trousers -7432 clergyman -7433 Tribunal -7434 HARKER -7435 atmosphere -7436 Czarina -7437 Jonathan -7438 fantastic -7439 stadt -7440 Every -7441 Please -7442 bag -7443 bles -7444 eked -7445 fashion -7446 finger -7447 gn -7448 mor -7449 rified -7450 tr -7451 vial -7452 wa -7453 zard -7454 è -7455 ê -7456 inate -7457 inning -7458 oath -7459 mas -7460 monsieur -7461 erity -7462 bath -7463 base -7464 baff -7465 itor -7466 itations -7467 topic -7468 cher -7469 onograph -7470 Ing -7471 host -7472 hears -7473 hedge -7474 asping -7475 lion -7476 arch -7477 nigh -7478 named -7479 anu -7480 omen -7481 beck -7482 otic -7483 leg -7484 eating -7485 elect -7486 idance -7487 ayed -7488 sever -7489 Thing -7490 ascend -7491 asylum -7492 whit -7493 rational -7494 irm -7495 iring -7496 mechan -7497 cken -7498 Hon -7499 Hood -7500 Hundred -7501 sobs -7502 seas -7503 select -7504 butt -7505 Sha -7506 Shall -7507 Square -7508 sheets -7509 Acc -7510 defect -7511 demon -7512 lies -7513 aller -7514 three -7515 ooked -7516 Light -7517 Lane -7518 Duke -7519 ico -7520 icles -7521 intend -7522 abode -7523 oles -7524 olstadt -7525 unlocked -7526 unluck -7527 worried -7528 cann -7529 istrate -7530 quiver -7531 overt -7532 overlook -7533 endous -7534 trot -7535 brute -7536 lose -7537 aret -7538 Night -7539 loat -7540 reek -7541 clam -7542 prem -7543 appar -7544 chase -7545 descent -7546 acing -7547 Therefore -7548 reserve -7549 plane -7550 Red -7551 happens -7552 happier -7553 console -7554 feels -7555 mock -7556 expense -7557 contented -7558 hearty -7559 believing -7560 slim -7561 changing -7562 enshaw -7563 security -7564 suppressed -7565 newly -7566 sleeps -7567 keeper -7568 surv -7569 assumed -7570 awa -7571 corners -7572 guidance -7573 term -7574 drift -7575 drooping -7576 refresh -7577 insol -7578 whitewas -7579 version -7580 dropping -7581 wonderfully -7582 top -7583 concent -7584 acting -7585 XII -7586 patches -7587 oggs -7588 falls -7589 whisky -7590 strangely -7591 conditions -7592 trip -7593 affections -7594 armchair -7595 charged -7596 clearing -7597 Upon -7598 dict -7599 acquies -7600 sweetly -7601 market -7602 cracked -7603 crazy -7604 Which -7605 characters -7606 strict -7607 respects -7608 angle -7609 barbar -7610 colonel -7611 column -7612 witches -7613 formal -7614 formidable -7615 eyed -7616 function -7617 prayed -7618 praying -7619 prayers -7620 frighten -7621 apiece -7622 hanged -7623 quarters -7624 attendants -7625 occupations -7626 pulling -7627 solemnity -7628 cotton -7629 vived -7630 conclusions -7631 ridicul -7632 knife -7633 limb -7634 dusty -7635 consequently -7636 consequences -7637 Mrs -7638 smoked -7639 conducted -7640 belonging -7641 intellect -7642 treatment -7643 magistrate -7644 volume -7645 chiefly -7646 announce -7647 gaze -7648 stroke -7649 spectacle -7650 pilot -7651 afforded -7652 undertaking -7653 undertaker -7654 tremendous -7655 punct -7656 ING -7657 audible -7658 trifling -7659 blanket -7660 clapped -7661 Says -7662 unreason -7663 Jackson -7664 wickedness -7665 physical -7666 ardour -7667 learning -7668 declined -7669 fancies -7670 gathering -7671 weapons -7672 debts -7673 Orleans -7674 groaned -7675 blazing -7676 Indian -7677 amidst -7678 confirm -7679 confirmed -7680 appointed -7681 appointment -7682 hypnotic -7683 Like -7684 gloves -7685 displayed -7686 Between -7687 jaws -7688 partiality -7689 lingering -7690 traveller -7691 delir -7692 government -7693 miseries -7694 exchanged -7695 embarrass -7696 panting -7697 British -7698 bewildered -7699 ingen -7700 glittering -7701 tackle -7702 Openshaw -7703 emigrant -7704 superstition -7705 precaution -7706 ominous -7707 fidget -7708 appetite -7709 triumphant -7710 Transylvan -7711 Russian -7712 dasn -7713 Ernest -7714 peasants -7715 Ingolstadt -7716 After -7717 LE -7718 Shall -7719 Tr -7720 Un -7721 aired -7722 bur -7723 bug -7724 bited -7725 dling -7726 ez -7727 eur -7728 four -7729 fire -7730 gest -7731 gue -7732 iqu -7733 iate -7734 iving -7735 kin -7736 lk -7737 lest -7738 mo -7739 nings -7740 pap -7741 piece -7742 sha -7743 spe -7744 uk -7745 yes -7746 yal -7747 verses -7748 tour -7749 inished -7750 shat -7751 sons -7752 ther -7753 oars -7754 ouched -7755 buck -7756 bang -7757 bonnet -7758 itches -7759 itality -7760 toil -7761 crew -7762 cyl -7763 dru -7764 darted -7765 IX -7766 hired -7767 honey -7768 heah -7769 paint -7770 punch -7771 pouring -7772 pipes -7773 nerve -7774 nuther -7775 national -7776 hagg -7777 anter -7778 atives -7779 belt -7780 âĢĻ.âĢĿ -7781 othing -7782 othered -7783 lex -7784 uts -7785 utable -7786 ep -7787 ela -7788 sters -7789 thres -7790 Toller -7791 weeping -7792 weighed -7793 adv -7794 stained -7795 stately -7796 noticing -7797 rud -7798 Has -7799 sah -7800 sailing -7801 sails -7802 seize -7803 sealed -7804 sevent -7805 ifted -7806 ifully -7807 ammed -7808 Ste -7809 Several -7810 sheer -7811 sund -7812 abs -7813 iler -7814 conceived -7815 Cal -7816 Citizen -7817 Dun -7818 icious -7819 intently -7820 vest -7821 vile -7822 viv -7823 vines -7824 vampire -7825 abrupt -7826 unw -7827 unnecess -7828 Juli -7829 lace -7830 afar -7831 theirs -7832 odo -7833 nted -7834 inkled -7835 Pap -7836 Phelps -7837 trim -7838 ieces -7839 brow -7840 professor -7841 prosec -7842 disple -7843 disord -7844 Green -7845 poetry -7846 shouldn -7847 Father -7848 Fleet -7849 grate -7850 chick -7851 backs -7852 longing -7853 remote -7854 clumb -7855 scound -7856 scare -7857 scale -7858 plac -7859 plat -7860 plot -7861 wrung -7862 happiest -7863 compla -7864 conspic -7865 These -7866 entry -7867 VII -7868 VIII -7869 mob -7870 flood -7871 flowed -7872 kinder -7873 control -7874 momentary -7875 meanwhile -7876 slide -7877 pleaded -7878 chances -7879 acute -7880 nothing -7881 supposing -7882 fectly -7883 silk -7884 conference -7885 evils -7886 Does -7887 refres -7888 treacher -7889 wisht -7890 soundly -7891 inspired -7892 crown -7893 upon -7894 visits -7895 hard -7896 monot -7897 gravity -7898 forefinger -7899 interval -7900 suffic -7901 fireplace -7902 illumin -7903 signific -7904 reduced -7905 mbers -7906 whiskers -7907 infer -7908 invent -7909 tricks -7910 reluct -7911 float -7912 Lucases -7913 burden -7914 warmed -7915 deprived -7916 villain -7917 Many -7918 11 -7919 inquest -7920 banker -7921 irres -7922 barrier -7923 toned -7924 witch -7925 grip -7926 encl -7927 honourable -7928 atively -7929 require -7930 breathless -7931 phonograph -7932 officer -7933 dares -7934 proceedings -7935 searching -7936 Neither -7937 complaint -7938 quisite -7939 snowy -7940 transp -7941 mode -7942 struggling -7943 combe -7944 enterprise -7945 wheel -7946 ringing -7947 towers -7948 islands -7949 endeavours -7950 flag -7951 rubbing -7952 arguments -7953 rejoice -7954 motionless -7955 dismiss -7956 clerks -7957 hesitation -7958 scratch -7959 threatened -7960 acknowledge -7961 Adler -7962 cease -7963 scrap -7964 cruelty -7965 brief -7966 paddle -7967 XXXI -7968 XXXV -7969 coward -7970 shrieked -7971 recommended -7972 intentions -7973 shrill -7974 ardent -7975 elderly -7976 instinctively -7977 000 -7978 glass -7979 American -7980 swallow -7981 climb -7982 deduce -7983 vexation -7984 Liber -7985 envy -7986 wagon -7987 correspondence -7988 Bohem -7989 terrified -7990 cigar -7991 preceding -7992 desirable -7993 independ -7994 ecst -7995 embraced -7996 aptain -7997 relapse -7998 tempest -7999 Spani -8000 encouragement -8001 paddled -8002 essential -8003 rays -8004 local -8005 eloquence -8006 investigation -8007 staggered -8008 frowning -8009 prolonged -8010 tidings -8011 autumn -8012 mamma -8013 exalted -8014 insufficient -8015 Arabian -8016 linen -8017 tumbrils -8018 wreath -8019 Murray -8020 sublime -8021 Georgiana -8022 Transylvania -8023 shattered -8024 elapsed -8025 threshold -8026 Any -8027 Hell -8028 IT -8029 Monseigneur -8030 Pre -8031 Poor -8032 Two -8033 _!âĢĿ -8034 bable -8035 cap -8036 coloured -8037 found -8038 gain -8039 hire -8040 ipp -8041 iated -8042 ipping -8043 lar -8044 might -8045 more -8046 mate -8047 pile -8048 rot -8049 vation -8050 yn -8051 5 -8052 ted -8053 hester -8054 ais -8055 awe -8056 sich -8057 oak -8058 moust -8059 ber -8060 bol -8061 bats -8062 bunch -8063 bache -8064 bald -8065 itness -8066 itory -8067 itical -8068 edy -8069 toe -8070 cough -8071 dun -8072 Irene -8073 oran -8074 lumber -8075 losing -8076 injun -8077 pot -8078 pic -8079 pine -8080 push -8081 numb -8082 hal -8083 anny -8084 beet -8085 beard -8086 gy -8087 gig -8088 gau -8089 goss -8090 gems -8091 imble -8092 imme -8093 elig -8094 stream -8095 ult -8096 ugly -8097 redd -8098 Tow -8099 Tower -8100 weigh -8101 forc -8102 stom -8103 stooping -8104 rif -8105 rack -8106 range -8107 rabbit -8108 mel -8109 mend -8110 ools -8111 anch -8112 hern -8113 sack -8114 allusion -8115 Since -8116 Sav -8117 Said -8118 suited -8119 sullen -8120 kick -8121 Amy -8122 degr -8123 deaf -8124 depended -8125 doomed -8126 Bl -8127 Bible -8128 Bob -8129 Boggs -8130 knelt -8131 alls -8132 outward -8133 think -8134 conqu -8135 unting -8136 Cor -8137 Chan -8138 Cons -8139 Castle -8140 ulp -8141 exult -8142 exquisite -8143 alt -8144 altern -8145 DI -8146 DR -8147 orange -8148 ornery -8149 intending -8150 ellow -8151 ittered -8152 willingly -8153 ivets -8154 faced -8155 aristocr -8156 nowhere -8157 enab -8158 Pi -8159 Parsonage -8160 agger -8161 brings -8162 arkness -8163 propose -8164 prodig -8165 prisons -8166 pole -8167 Foot -8168 grant -8169 ado -8170 holy -8171 thrill -8172 blows -8173 bluff -8174 lethar -8175 eyel -8176 boats -8177 justified -8178 scree -8179 persever -8180 chaw -8181 turnkey -8182 acious -8183 throughout -8184 aching -8185 glided -8186 glaring -8187 harness -8188 moist -8189 flushed -8190 flattered -8191 accent -8192 finds -8193 distrust -8194 slate -8195 them -8196 these -8197 interest -8198 swinging -8199 steel -8200 stealth -8201 earliest -8202 forth -8203 marve -8204 discl -8205 persist -8206 beds -8207 traced -8208 bey -8209 startling -8210 refer -8211 advis -8212 pretended -8213 pretending -8214 register -8215 ceptible -8216 insisted -8217 interests -8218 inah -8219 deepened -8220 patiently -8221 ograp -8222 invar -8223 catching -8224 Sea -8225 Sec -8226 furious -8227 realise -8228 gladness -8229 earthly -8230 Prov -8231 dilig -8232 drawer -8233 banking -8234 graveyard -8235 brightly -8236 auntie -8237 repeating -8238 encoun -8239 safely -8240 leader -8241 preparing -8242 commonplace -8243 except -8244 deference -8245 courtes -8246 slightest -8247 clouded -8248 parce -8249 dah -8250 proceeding -8251 hasten -8252 Brad -8253 musk -8254 door -8255 accompany -8256 blinds -8257 lifeless -8258 animation -8259 century -8260 tendency -8261 modest -8262 dimly -8263 unhapp -8264 difficulties -8265 portra -8266 choked -8267 faintly -8268 peculiarly -8269 belongs -8270 passionate -8271 separation -8272 increase -8273 volupt -8274 tongues -8275 firmly -8276 surround -8277 grandeur -8278 strolled -8279 amazing -8280 horse -8281 mysteries -8282 divided -8283 failure -8284 mixture -8285 sternly -8286 packed -8287 engagements -8288 crawl -8289 soothed -8290 jackal -8291 elbows -8292 tempted -8293 inhabited -8294 philosopher -8295 benefit -8296 Wilks -8297 malice -8298 holiday -8299 entertainment -8300 excessive -8301 Christian -8302 congratulations -8303 performance -8304 typewriter -8305 ENT -8306 guard -8307 scape -8308 terrific -8309 convenient -8310 obscure -8311 glare -8312 preparation -8313 title -8314 fascin -8315 squeezed -8316 implored -8317 incredible -8318 avenue -8319 insensible -8320 exhibited -8321 routine -8322 extinguished -8323 brilliant -8324 expedition -8325 clutched -8326 inconvenience -8327 achieve -8328 basin -8329 intimacy -8330 imprisonment -8331 exclamation -8332 Slovaks -8333 stolen -8334 Attorney -8335 traversed -8336 solicitor -8337 Boscombe -8338 disdain -8339 ribbon -8340 ventilator -8341 establishment -8342 fashioned -8343 haggard -8344 monoton -8345 sufficiently -8346 Liberty -8347 tedious -8348 ography -8349 voluptuous -8350 .] -8351 Al -8352 Ay -8353 Alas -8354 Friend -8355 Give -8356 IL -8357 Lizzy -8358 Tis -8359 _âĢĻ -8360 apped -8361 actor -8362 bted -8363 bby -8364 car -8365 call -8366 eat -8367 iation -8368 ients -8369 iably -8370 must -8371 make -8372 mine -8373 mered -8374 nation -8375 pan -8376 pot -8377 pass -8378 roof -8379 sawyer -8380 uper -8381 ucy -8382 vest -8383 venty -8384 want -8385 yr -8386 zgany -8387 tan -8388 tig -8389 aim -8390 wre -8391 ward -8392 wip -8393 wives -8394 sed -8395 sledge -8396 ost -8397 ouri -8398 map -8399 mush -8400 bone -8401 blo -8402 bacon -8403 itch -8404 isation -8405 fought -8406 tops -8407 cert -8408 cult -8409 enet -8410 ington -8411 onstr -8412 dad -8413 dated -8414 dances -8415 Int -8416 lust -8417 lens -8418 pens -8419 nin -8420 nieces -8421 hail -8422 bef -8423 gasped -8424 seven -8425 church -8426 rever -8427 recei -8428 weird -8429 adding -8430 urred -8431 urious -8432 urances -8433 stile -8434 irwin -8435 analy -8436 Ho -8437 Horner -8438 always -8439 Most -8440 seventy -8441 alluded -8442 butcher -8443 Soup -8444 SEW -8445 Szgany -8446 suits -8447 Ass -8448 Were -8449 Water -8450 shill -8451 fruit -8452 frantic -8453 iless -8454 inders -8455 anded -8456 rave -8457 Cly -8458 ulate -8459 --_ -8460 outhern -8461 antial -8462 Dear -8463 Dodo -8464 orphan -8465 usiness -8466 vault -8467 abst -8468 selfish -8469 elly -8470 jun -8471 jug -8472 unav -8473 unselfish -8474 timidly -8475 Jud -8476 leag -8477 thern -8478 quivered -8479 Ed -8480 splas -8481 ends -8482 Pre -8483 mustn -8484 offence -8485 prime -8486 Grace -8487 arewell -8488 everyone -8489 coinc -8490 grasped -8491 choo -8492 load -8493 adorn -8494 backward -8495 remonstr -8496 neath -8497 blush -8498 blamed -8499 plete -8500 lets -8501 boards -8502 clang -8503 appalling -8504 underneath -8505 screw -8506 perish -8507 perfor -8508 everl -8509 thoughtful -8510 placing -8511 chaise -8512 acr -8513 resigned -8514 plates -8515 Ry -8516 Res -8517 Ross -8518 Remember -8519 wrought -8520 Through -8521 Missouri -8522 consented -8523 consisted -8524 lessness -8525 repuls -8526 repetition -8527 Vampire -8528 flower -8529 flaming -8530 contain -8531 accum -8532 kneeling -8533 calls -8534 wax -8535 pausing -8536 recess -8537 hearth -8538 Kirwin -8539 airo -8540 alongside -8541 swords -8542 ieties -8543 pleasures -8544 coroner -8545 tragedy -8546 curled -8547 cursed -8548 worldly -8549 ivable -8550 purp -8551 refusal -8552 pretence -8553 submit -8554 region -8555 inscri -8556 asks -8557 smaller -8558 detest -8559 lists -8560 Because -8561 graves -8562 teacher -8563 intercourse -8564 hamber -8565 deepest -8566 decent -8567 patients -8568 beauties -8569 sterdam -8570 appearing -8571 simpl -8572 relative -8573 stragg -8574 phia -8575 engaging -8576 coldly -8577 breathed -8578 devotion -8579 Up -8580 diab -8581 12 -8582 explore -8583 prize -8584 ciation -8585 edges -8586 solved -8587 solution -8588 widder -8589 denial -8590 element -8591 sending -8592 Moran -8593 Quick -8594 doctors -8595 couch -8596 colle -8597 sharply -8598 merchant -8599 liter -8600 yelling -8601 cousins -8602 situated -8603 article -8604 dollar -8605 picking -8606 countenances -8607 provo -8608 comed -8609 soldier -8610 healthy -8611 harmless -8612 addressing -8613 pencil -8614 nicely -8615 winning -8616 bursting -8617 without -8618 masses -8619 towel -8620 hyster -8621 avoided -8622 mutual -8623 flapping -8624 authorities -8625 places -8626 breadth -8627 firmness -8628 petition -8629 announced -8630 entreaty -8631 entreated -8632 entreaties -8633 closet -8634 legal -8635 spectators -8636 respond -8637 wonders -8638 hesitating -8639 ghostly -8640 stealing -8641 decidedly -8642 sailors -8643 packet -8644 welcomed -8645 League -8646 virtues -8647 attacked -8648 maids -8649 million -8650 millions -8651 Amsterdam -8652 preference -8653 unconsciously -8654 feverish -8655 obtain -8656 methods -8657 hellish -8658 protectors -8659 picturesque -8660 Quick -8661 whirl -8662 fooling -8663 jumping -8664 climbed -8665 Another -8666 swelled -8667 knocking -8668 contemplated -8669 XIV -8670 conjectured -8671 congratulate -8672 vexed -8673 positive -8674 abhorred -8675 abhorrence -8676 proposals -8677 enthusiastic -8678 indebted -8679 cussing -8680 confidential -8681 preparations -8682 alteration -8683 obeyed -8684 sustained -8685 prosperous -8686 Spanish -8687 perplexity -8688 Robin -8689 disappearance -8690 orgies -8691 illustrious -8692 fragment -8693 surmise -8694 phrase -8695 gallantry -8696 urrection -8697 guinea -8698 tiptoe -8699 Winchester -8700 plunged -8701 symptoms -8702 entlemen -8703 solicitude -8704 governess -8705 intendent -8706 minded -8707 important -8708 ridiculous -8709 embarrassment -8710 placid -8711 Spaniard -8712 Hello -8713 bachelor -8714 beetle -8715 eligible -8716 stomach -8717 SEWARD -8718 leagues -8719 ,) -8720 25 -8721 ;_ -8722 AD -8723 CK -8724 De -8725 Ha -8726 Hush -8727 Lady -8728 Lucy -8729 ND -8730 NAL -8731 ST -8732 Such -8733 _! -8734 _; -8735 born -8736 dri -8737 done -8738 eches -8739 gin -8740 gent -8741 gic -8742 gown -8743 gate -8744 gence -8745 iliz -8746 lers -8747 lined -8748 nin -8749 nay -8750 nine -8751 pat -8752 press -8753 take -8754 tel -8755 ties -8756 ugh -8757 uity -8758 vment -8759 zes -8760 ... -8761 auction -8762 inter -8763 wing -8764 wits -8765 slack -8766 owing -8767 rehens -8768 hal -8769 merry -8770 mask -8771 erland -8772 bure -8773 budge -8774 bolt -8775 itled -8776 isance -8777 fed -8778 fits -8779 cure -8780 claimed -8781 enc -8782 dates -8783 llow -8784 hyd -8785 orgo -8786 lack -8787 lonel -8788 pink -8789 pang -8790 net -8791 hazard -8792 beats -8793 befall -8794 owl -8795 owling -8796 âĢĻâĢĶ -8797 legged -8798 yaller -8799 washed -8800 eg -8801 ecc -8802 lds -8803 stones -8804 onto -8805 rear -8806 rely -8807 reli -8808 reject -8809 TO -8810 Test -8811 iony -8812 whoop -8813 urry -8814 stain -8815 stated -8816 stony -8817 stature -8818 noth -8819 rows -8820 anew -8821 sailed -8822 Mag -8823 Merry -8824 Miller -8825 ills -8826 amond -8827 Ser -8828 Sam -8829 Six -8830 Sax -8831 Skins -8832 kicked -8833 doc -8834 Bless -8835 shucks -8836 feed -8837 rack -8838 Captain -8839 Cairo -8840 mantel -8841 Lasc -8842 Des -8843 Down -8844 Denny -8845 Doran -8846 usive -8847 ici -8848 icial -8849 vag -8850 elling -8851 ung -8852 ungr -8853 comrade -8854 Jenn -8855 JOUR -8856 cane -8857 canv -8858 govment -8859 laced -8860 afore -8861 downward -8862 shared -8863 Early -8864 spots -8865 roared -8866 roots -8867 rocky -8868 Pl -8869 Per -8870 Pool -8871 agged -8872 trunk -8873 ieu -8874 brushed -8875 Get -8876 Gaz -8877 earing -8878 ponder -8879 Friend -8880 Flor -8881 chron -8882 hove -8883 strang -8884 blowed -8885 clump -8886 clinging -8887 apple -8888 hereafter -8889 beggar -8890 thoughtfully -8891 peop -8892 opium -8893 rightly -8894 sawed -8895 pluck -8896 Right -8897 Read -8898 weather -8899 Thurs -8900 foundations -8901 watches -8902 compared -8903 rowing -8904 repent -8905 Str -8906 States -8907 morrow -8908 wail -8909 distracted -8910 reconc -8911 their -8912 busily -8913 retion -8914 lazy -8915 stepping -8916 inson -8917 confide -8918 assented -8919 assurances -8920 marble -8921 homeward -8922 xton -8923 helping -8924 traps -8925 Whatever -8926 rences -8927 uncles -8928 drum -8929 cheap -8930 towns -8931 shoot -8932 highway -8933 vision -8934 watermel -8935 angerford -8936 expectations -8937 humili -8938 XIII -8939 shorthand -8940 fourteen -8941 infant -8942 invisible -8943 promising -8944 tries -8945 trivial -8946 burgl -8947 realised -8948 wallpaper -8949 devote -8950 villages -8951 Ladybird -8952 suspended -8953 10 -8954 lads -8955 acquired -8956 create -8957 trusting -8958 gravel -8959 striving -8960 respected -8961 approve -8962 irre -8963 griev -8964 wisest -8965 wisdom -8966 encomp -8967 _.-- -8968 veget -8969 saddle -8970 sickness -8971 majesty -8972 newspap -8973 pictured -8974 obliv -8975 attempts -8976 astonishing -8977 Island -8978 does -8979 stretching -8980 concerns -8981 articles -8982 modesty -8983 calmness -8984 paying -8985 improbable -8986 captivity -8987 alarming -8988 policeman -8989 grove -8990 terrors -8991 precise -8992 itzerland -8993 withstanding -8994 removal -8995 explaining -8996 introducing -8997 ARY -8998 contradi -8999 gallery -9000 negro -9001 attentively -9002 recorded -9003 fighting -9004 voluntarily -9005 roughly -9006 rejoined -9007 Looking -9008 encouraging -9009 pitch -9010 pitiful -9011 cautious -9012 exerc -9013 spectacles -9014 slavery -9015 rattles -9016 rattling -9017 commanded -9018 undertake -9019 discovering -9020 didn -9021 crawling -9022 Advent -9023 vais -9024 INA -9025 faults -9026 crimin -9027 tortured -9028 compassionate -9029 cisely -9030 imprudence -9031 Switzerland -9032 recognise -9033 folk -9034 Republican -9035 foolishness -9036 arranging -9037 concealment -9038 snee -9039 shrink -9040 ardently -9041 XVI -9042 dissip -9043 screaming -9044 elope -9045 shovel -9046 excessively -9047 blazed -9048 Grangerford -9049 disturbance -9050 selfishness -9051 experiences -9052 betrayed -9053 hypnotise -9054 relationship -9055 pierced -9056 waistcoat -9057 commencement -9058 borrowed -9059 preceded -9060 desirous -9061 uncommonly -9062 pallor -9063 ................ -9064 additional -9065 jewel -9066 twinkling -9067 perplexed -9068 Robinson -9069 railway -9070 tucked -9071 implore -9072 elevated -9073 politely -9074 universal -9075 copy -9076 hoarsely -9077 citizens -9078 Beauvais -9079 Really -9080 banished -9081 totally -9082 frenzy -9083 infamous -9084 majest -9085 Devil -9086 inspiration -9087 dishonour -9088 resource -9089 obligation -9090 verdict -9091 baffled -9092 unlucky -9093 cannon -9094 whitewashed -9095 delirium -9096 burgh -9097 advant -9098 unnecessary -9099 gratefully -9100 platform -9101 conspicuous -9102 Precisely -9103 gigantic -9104 enabled -9105 invariably -9106 Providence -9107 courtesy -9108 Bradstreet -9109 cultiv -9110 Ryder -9111 diabolic -9112 bureau -9113 loneliness -9114 Merryweather -9115 Skinsky -9116 Lascar -9117 Jennie -9118 JOURNAL -9119 Gazette -9120 Thursday -9121 newspapers -9122 ). -9123 Ar -9124 About -9125 Listen -9126 OL -9127 Off -9128 Sid -9129 Were -9130 Witness -9131 ais -9132 bet -9133 bro -9134 bled -9135 bling -9136 bury -9137 breed -9138 burg -9139 cut -9140 colate -9141 chamber -9142 dress -9143 dled -9144 dles -9145 dson -9146 ele -9147 eenth -9148 fit -9149 fold -9150 gage -9151 graph -9152 ike -9153 ived -9154 jac -9155 lour -9156 lorn -9157 liter -9158 mare -9159 nor -9160 nces -9161 ples -9162 rank -9163 rants -9164 rusted -9165 than -9166 ulic -9167 wind -9168 8 -9169 9 -9170 z -9171 ting -9172 tut -9173 tails -9174 trod -9175 aver -9176 inch -9177 iners -9178 wil -9179 whar -9180 oil -9181 reement -9182 bowing -9183 bade -9184 toes -9185 toils -9186 cul -9187 cows -9188 ingham -9189 du -9190 ding -9191 dil -9192 dang -9193 dense -9194 daring -9195 drows -9196 hom -9197 hind -9198 hug -9199 hugged -9200 orter -9201 orable -9202 heaving -9203 arter -9204 estern -9205 inch -9206 inward -9207 pha -9208 pages -9209 pushing -9210 pulse -9211 nap -9212 nails -9213 hailed -9214 beer -9215 beach -9216 bears -9217 gor -9218 gap -9219 gaining -9220 âĢĻ; -9221 otes -9222 yarn -9223 washing -9224 ef -9225 ejac -9226 idences -9227 sey -9228 sect -9229 entric -9230 rend -9231 reass -9232 remed -9233 Ter -9234 Till -9235 weaken -9236 forming -9237 forlorn -9238 ural -9239 urous -9240 sting -9241 irting -9242 oops -9243 ange -9244 Happ -9245 Master -9246 seats -9247 aments -9248 allowing -9249 Suther -9250 Surely -9251 AD -9252 lip -9253 gewater -9254 Bal -9255 Borgo -9256 shine -9257 ustice -9258 conception -9259 converse -9260 raulic -9261 ulses -9262 Later -9263 liken -9264 alacr -9265 Dead -9266 Dey -9267 Darkness -9268 Dinah -9269 intensity -9270 vital -9271 abund -9272 whoever -9273 ittles -9274 je -9275 erson -9276 Jeff -9277 igning -9278 thence -9279 quitting -9280 overcame -9281 farew -9282 Em -9283 arter -9284 Pat -9285 Please -9286 brid -9287 disadvant -9288 earance -9289 cocked -9290 locks -9291 icks -9292 icker -9293 Off -9294 irect -9295 throwed -9296 clad -9297 nightmare -9298 justify -9299 scient -9300 owned -9301 oldest -9302 permit -9303 becuz -9304 chaps -9305 chalk -9306 bery -9307 âĢĺ_ -9308 spelling -9309 attributed -9310 impenet -9311 Ra -9312 happening -9313 consist -9314 dearly -9315 retained -9316 lighter -9317 lighting -9318 lightly -9319 accidents -9320 preserved -9321 Station -9322 Stoke -9323 waving -9324 reco -9325 horribly -9326 Keep -9327 sworn -9328 izes -9329 laz -9330 ruly -9331 discern -9332 discretion -9333 commend -9334 commenced -9335 bedside -9336 untary -9337 owering -9338 career -9339 drun -9340 soever -9341 insist -9342 giant -9343 crushed -9344 crooked -9345 hurled -9346 northern -9347 speaks -9348 Being -9349 patch -9350 ibilities -9351 watchful -9352 watchman -9353 generly -9354 awing -9355 ogue -9356 needs -9357 needle -9358 needless -9359 signed -9360 promises -9361 opposed -9362 opposition -9363 tribe -9364 bloody -9365 appearances -9366 Yard -9367 painfully -9368 apping -9369 ernal -9370 furt -9371 expressing -9372 joining -9373 shaded -9374 bigger -9375 fortunes -9376 wildly -9377 devouring -9378 papa -9379 acquire -9380 services -9381 resolving -9382 destined -9383 destination -9384 student -9385 creaking -9386 victory -9387 symb -9388 syllable -9389 buz -9390 favoured -9391 hats -9392 barrel -9393 fasten -9394 Does -9395 Doctor -9396 escorted -9397 brain -9398 bridge -9399 solely -9400 enchan -9401 opinions -9402 settling -9403 weaker -9404 mentioning -9405 pursuing -9406 sickly -9407 experiment -9408 occurrences -9409 defence -9410 posting -9411 postil -9412 strengthen -9413 quarre -9414 ruption -9415 varied -9416 offices -9417 emphas -9418 guardian -9419 forgetting -9420 attempting -9421 repeatedly -9422 succession -9423 collection -9424 Briony -9425 merit -9426 dozen -9427 vanish -9428 ridge -9429 persuasion -9430 22 -9431 undo -9432 arers -9433 moderate -9434 data -9435 languid -9436 extrem -9437 behaved -9438 bells -9439 pleasantly -9440 admiring -9441 cruelly -9442 valued -9443 addresses -9444 reaching -9445 China -9446 Cliff -9447 chocolate -9448 injustice -9449 Spy -9450 worthy -9451 balls -9452 Carpath -9453 gallows -9454 records -9455 sticks -9456 buildings -9457 argue -9458 hateful -9459 pitiable -9460 rattled -9461 robbed -9462 sailor -9463 claims -9464 designed -9465 genial -9466 nodding -9467 unspeak -9468 -9469 Looky -9470 bricks -9471 True -9472 lodge -9473 feebly -9474 principally -9475 tumbling -9476 benefactor -9477 bills -9478 vacancy -9479 dissatisfied -9480 humoured -9481 pinch -9482 restrained -9483 stores -9484 entertained -9485 procure -9486 bullet -9487 gleam -9488 combined -9489 sparkled -9490 inconce -9491 matches -9492 revolver -9493 grieve -9494 discussion -9495 elegance -9496 recollections -9497 withdra -9498 sacrific -9499 talent -9500 deeds -9501 fatigued -9502 representing -9503 Shucks -9504 cussed -9505 whack -9506 beasts -9507 econ -9508 measured -9509 measures -9510 accurate -9511 benevolence -9512 slope -9513 Peterson -9514 manifestly -9515 fascination -9516 split -9517 travellers -9518 overwhelmed -9519 couldn -9520 sanct -9521 eloquent -9522 mirror -9523 indicated -9524 oppressive -9525 robbery -9526 citizeness -9527 hospitality -9528 abouts -9529 plantation -9530 chemical -9531 annoyed -9532 calculated -9533 Twice -9534 propriety -9535 abominable -9536 extravagant -9537 beckoned -9538 electric -9539 seventeen -9540 chicken -9541 scoundrel -9542 daresay -9543 Bohemia -9544 ecstasy -9545 picnic -9546 numbers -9547 gossip -9548 Towards -9549 forcibly -9550 anchor -9551 altar -9552 DIARY -9553 Pigeon -9554 adopt -9555 eyelids -9556 advisable -9557 encounter -9558 portrait -9559 tiger -9560 receipt -9561 everlasting -9562 purposes -9563 rehensible -9564 hydraulic -9565 eggs -9566 eccentric -9567 Testament -9568 criminals -9569 elopement -9570 Sutherland -9571 likeness -9572 alacrity -9573 farewell -9574 impenetr -9575 !... -9576 15 -9577 17 -9578 ?-- -9579 Aunt -9580 ET -9581 Her -9582 Has -9583 Had -9584 IS -9585 Nay -9586 Or -9587 Sir -9588 Somet -9589 Surely -9590 Upon -9591 aver -9592 bound -9593 bation -9594 bish -9595 contin -9596 dro -9597 dged -9598 dog -9599 gled -9600 groom -9601 gging -9602 garet -9603 giving -9604 hot -9605 iac -9606 lim -9607 lings -9608 much -9609 mates -9610 nake -9611 ox -9612 ops -9613 oft -9614 pone -9615 poss -9616 ruded -9617 sis -9618 sized -9619 tis -9620 times -9621 wow -9622 wart -9623 wix -9624 wash -9625 walking -9626 xford -9627 ze -9628 tar -9629 tires -9630 tables -9631 tangled -9632 aug -9633 wos -9634 sang -9635 theories -9636 handed -9637 mirth -9638 bred -9639 bind -9640 itched -9641 itute -9642 itching -9643 isions -9644 fri -9645 fest -9646 toile -9647 cust -9648 cip -9649 cash -9650 ceas -9651 creek -9652 enched -9653 dw -9654 duck -9655 dial -9656 hing -9657 homes -9658 orus -9659 lame -9660 lined -9661 lyn -9662 arters -9663 inev -9664 paced -9665 phant -9666 nay -9667 nav -9668 hare -9669 ano -9670 âĢľ-- -9671 lee -9672 lebr -9673 utter -9674 utions -9675 veying -9676 ett -9677 thimble -9678 sell -9679 reb -9680 rese -9681 reaction -9682 Tha -9683 wearing -9684 urbed -9685 ster -9686 stor -9687 stal -9688 stab -9689 ric -9690 rig -9691 ruther -9692 ramb -9693 roses -9694 rivets -9695 iron -9696 herry -9697 HE -9698 Hors -9699 alf -9700 alid -9701 saving -9702 sakes -9703 seal -9704 ctive -9705 ifty -9706 rice -9707 alle -9708 suic -9709 Alp -9710 demean -9711 rolled -9712 dozing -9713 ppled -9714 Bri -9715 Bil -9716 Ball -9717 Bud -9718 Boy -9719 shiny -9720 shivered -9721 fringed -9722 Hear -9723 unned -9724 unciation -9725 Com -9726 Cave -9727 Camp -9728 ulted -9729 ulations -9730 ulace -9731 respect -9732 Land -9733 Lamb -9734 liv -9735 licked -9736 Danu -9737 ica -9738 icule -9739 vary -9740 vale -9741 vicious -9742 asted -9743 jist -9744 jolly -9745 olated -9746 aggra -9747 unacc -9748 unequ -9749 worm -9750 osy -9751 leak -9752 looky -9753 istered -9754 umes -9755 umerable -9756 hedral -9757 sharing -9758 failing -9759 End -9760 arrow -9761 spi -9762 spider -9763 spraw -9764 auld -9765 ender -9766 enorm -9767 Padding -9768 handful -9769 agon -9770 trading -9771 elcome -9772 thanking -9773 beforehand -9774 twig -9775 imitate -9776 disli -9777 printed -9778 Fortun -9779 cock -9780 hasn -9781 Nones -9782 lob -9783 loath -9784 ishly -9785 advers -9786 ountable -9787 strained -9788 struct -9789 throng -9790 throws -9791 new -9792 bliss -9793 blade -9794 blunder -9795 pscall -9796 bond -9797 booming -9798 boiler -9799 clatter -9800 club -9801 greatness -9802 appla -9803 scent -9804 scatter -9805 scold -9806 pert -9807 perceiving -9808 peering -9809 peeping -9810 cham -9811 chained -9812 desires -9813 specul -9814 obliter -9815 partners -9816 resign -9817 *** -9818 Rey -9819 wrinkled -9820 founded -9821 compose -9822 consol -9823 entail -9824 entitled -9825 teenth -9826 induced -9827 indisp -9828 ending -9829 flirt -9830 flashed -9831 contrived -9832 access -9833 sets -9834 getic -9835 presents -9836 presumption -9837 swer -9838 pauses -9839 rapscall -9840 distorted -9841 slouch -9842 ltry -9843 channel -9844 darkly -9845 darkened -9846 breeding -9847 enses -9848 supposition -9849 returns -9850 childhood -9851 emin -9852 confine -9853 confron -9854 gerie -9855 incons -9856 disconc -9857 famous -9858 traitor -9859 foam -9860 foli -9861 misunder -9862 bec -9863 before -9864 purple -9865 cheered -9866 refr -9867 treas -9868 excite -9869 advancing -9870 Bennets -9871 regions -9872 fearing -9873 (âĢľ -9874 shops -9875 shooting -9876 airs -9877 detected -9878 speaker -9879 doubtful -9880 Believe -9881 butter -9882 text -9883 nolds -9884 quicker -9885 extract -9886 extensive -9887 soup -9888 beautifully -9889 quieted -9890 pips -9891 infernal -9892 lowest -9893 laud -9894 chers -9895 sights -9896 hairs -9897 relating -9898 relatives -9899 strait -9900 expressly -9901 afflicted -9902 affirm -9903 depressed -9904 engross -9905 iergerie -9906 serpent -9907 fortunately -9908 minutely -9909 landscape -9910 questioned -9911 charms -9912 badly -9913 diamond -9914 judges -9915 visiting -9916 cribed -9917 resolve -9918 ughter -9919 inquis -9920 kingdom -9921 Marlow -9922 Margaret -9923 crev -9924 sensitive -9925 craving -9926 craft -9927 ksome -9928 shutting -9929 previously -9930 calico -9931 occasioned -9932 mission -9933 strikes -9934 stricken -9935 senseless -9936 approbation -9937 affectionately -9938 differently -9939 irts -9940 brighter -9941 brightened -9942 irksome -9943 coloured -9944 complet -9945 umpet -9946 solve -9947 rafts -9948 veh -9949 estim -9950 XXII -9951 joyful -9952 remainder -9953 defend -9954 incerity -9955 slippers -9956 occupy -9957 bounds -9958 spiritual -9959 bottles -9960 casting -9961 earnestness -9962 merits -9963 complain -9964 yelled -9965 enjoying -9966 planned -9967 embrace -9968 immort -9969 arts -9970 despairing -9971 limit -9972 languages -9973 constr -9974 constru -9975 provide -9976 spotted -9977 paths -9978 ridge -9979 blessings -9980 groans -9981 Frenchman -9982 streams -9983 injuries -9984 Sundays -9985 pressure -9986 gaiety -9987 magic -9988 rustic -9989 gallop -9990 palm -9991 palp -9992 palace -9993 rubbage -9994 sticking -9995 springing -9996 surrounding -9997 fires -9998 ribute -9999 confession diff --git a/vocab_bytezip.txt b/vocab_bytezip.txt deleted file mode 100644 index a635f8c0..00000000 --- a/vocab_bytezip.txt +++ /dev/null @@ -1,12385 +0,0 @@ -260 the -261 th -262 the -263 - - -264 t -265 he -266 th -267 a -268 and -269 - - - -270 and -271 in -272 an -273 - - - - -274 er -275 -276 w -277 h -278 s -279 to -280 ing -281 nd -282 -283 re -284 � -285 o -286 of -287 ou -288 that -289 ha -290 he -291 i -292 her -293 at -294 -295 ed -296 on -297 en -298 was -299 in -300 -301 to -302 m -303 ha -304 wa -305 ng -306 b -307 hi -308 it -309 hat -310 you -311 as -312 be -313 with -314 is -315 “ -316 ’ -317 or -318 hi -319 tha -320 ” -321 ar -322 f -323 st -324 ther -325 ve -326 es -327 te -328 c -329 was -330 ere -331 me -332 wh -333 se -334 of -335 his -336 -337 with -338 no -339 it -340 wi -341 his -342 d -343 for -344 ea -345 le -346 co -347 you -348 wit -349 ne -350 for -351 ll -352 ould -353 her -354 I -355 wa -356 yo -357 ver -358 not -359 had -360 nt -361 fo -362 here -363 be -364 ent -365 al -366 so -367 on -368 l -369 sh -370 ti -371 all -372 no -373 n -374 we -375 ter -376 have -377 thi -378 de -379 tion -380 p -381 ight -382 sa -383 .” -384 not -385 ro -386 om -387 ur -388 ho -389 re -390 ad -391 as -392 him -393 ut -394 thin -395 thi -396 el -397 li -398 ow -399 co -400 ma -401 me -402 ion -403 ght -404 ot -405 which -406 do -407 ee -408 said -409 had -410 thing -411 r -412 ri -413 wh -414 our -415 ce -416 wi -417 ch -418 sh -419 there -420 ear -421 al -422 would -423 ever -424 et -425 fo -426 se -427 ave -428 st -429 hin -430 ai -431 eve -432 ome -433 so -434 e -435 g -436 all -437 we -438 but -439 oo -440 she -441 could -442 whi -443 wo -444 us -445 my -446 one -447 there -448 - - - - - -449 hen -450 ca -451 The -452 y -453 out -454 would -455 ough -456 ly -457 im -458 lo -459 un -460 ma -461 li -462 ld -463 ul -464 they -465 she -466 cou -467 - - - - - - -468 from -469 il -470 id -471 The -472 ess -473 were -474 could -475 at -476 other -477 nce -478 ra -479 gh -480 hou -481 ther -482 mo -483 this -484 bu -485 woul -486 rea -487 si -488 is -489 ation -490 la -491 sa -492 pe -493 ain -494 de -495 see -496 ke -497 whe -498 ss -499 coul -500 very -501 ay -502 some -503 lo -504 ic -505 ill -506 ted -507 but -508 them -509 di -510 whi -511 rs -512 res -513 ho -514 ge -515 oun -516 ca -517 Th -518 go -519 do -520 when -521 been -522 com -523 fr -524 ta -525 io -526 cou -527 di -528 red -529 some -530 thin -531 ore -532 oth -533 this -534 su -535 though -536 u -537 aid -538 ac -539 though -540 eve -541 - - - - - - - -542 out -543 hea -544 know -545 ate -546 ell -547 em -548 ers -549 your -550 und -551 han -552 con -553 now -554 ir -555 ig -556 mo -557 wo -558 little -559 ie -560 man -561 tr -562 wer -563 are -564 ou -565 com -566 est -567 ound -568 what -569 nc -570 thou -571 pr -572 av -573 atio -574 them -575 int -576 whe -577 ry -578 their -579 fa -580 ove -581 ati -582 ?” -583 hey -584 ever -585 ol -586 rt -587 up -588 man -589 ble -590 ec -591 som -592 am -593 ,” -594 - - - - - - - - -595 Th -596 when -597 ive -598 self -599 sho -600 ind -601 ht -602 time -603 ting -604 con -605 before -606 see -607 own -608 een -609 look -610 who -611 time -612 know -613 ns -614 int -615 one -616 mi -617 la -618 ook -619 again -620 ust -621 pa -622 hea -623 tt -624 tte -625 ste -626 ard -627 si -628 ine -629 your -630 men -631 very -632 then -633 about -634 what -635 will -636 over -637 der -638 nde -639 ey -640 fe -641 mi -642 rou -643 my -644 ar -645 rom -646 ment -647 — -648 le -649 ead -650 mor -651 ning -652 ned -653 ous -654 kno -655 tho -656 king -657 more -658 should -659 sta -660 os -661 tter -662 efore -663 su -664 other -665 ni -666 fi -667 po -668 tin -669 down -670 ev -671 pr -672 hear -673 eat -674 ne -675 T -676 tho -677 str -678 bu -679 way -680 over -681 sel -682 ck -683 look -684 should -685 wha -686 by -687 any -688 per -689 kin -690 ide -691 bl -692 wor -693 ness -694 into -695 fe -696 any -697 bo -698 ab -699 Elizabeth -700 ame -701 end -702 pa -703 sho -704 ought -705 kn -706 ast -707 ring -708 tur -709 ect -710 will -711 able -712 rd -713 like -714 ught -715 ake -716 ance -717 loo -718 can -719 wil -720 ence -721 ons -722 art -723 thought -724 nder -725 - - - - - - - - - -726 ti -727 tim -728 nte -729 ure -730 use -731 mu -732 ong -733 think -734 tr -735 da -736 ered -737 uch -738 hand -739 str -740 ten -741 sed -742 mor -743 go -744 ood -745 are -746 min -747 He -748 un -749 thought -750 tle -751 ct -752 shoul -753 ttle -754 ant -755 nes -756 every -757 loo -758 hear -759 fi -760 rie -761 come -762 fa -763 up -764 say -765 ling -766 lin -767 gr -768 than -769 ather -770 who -771 reat -772 ch -773 too -774 nothing -775 fr -776 like -777 ag -778 down -779 ked -780 come -781 othe -782 night -783 ug -784 long -785 did -786 uc -787 gain -788 !” -789 H -790 en -791 nothing -792 never -793 ath -794 pre -795 thr -796 rin -797 if -798 M -799 elf -800 after -801 ose -802 ist -803 than -804 dow -805 through -806 rough -807 think -808 thing -809 every -810 k -811 himself -812 anc -813 od -814 ort -815 pl -816 rn -817 into -818 sin -819 round -820 must -821 Elizabeth -822 old -823 bo -824 hand -825 too -826 sta -827 pro -828 might -829 ace -830 dow -831 or -832 led -833 sha -834 itt -835 nin -836 right -837 eas -838 lit -839 enc -840 light -841 ex -842 shall -843 ove -844 upon -845 tra -846 good -847 something -848 wor -849 lea -850 tl -851 iv -852 br -853 han -854 sti -855 ts -856 po -857 eth -858 ok -859 tu -860 dis -861 ding -862 after -863 ven -864 can -865 ep -866 great -867 dea -868 lon -869 nothin -870 friend -871 rea -872 pre -873 abl -874 ta -875 only -876 - - -877 something -878 now -879 night -880 such -881 da -882 where -883 ap -884 ba -885 pro -886 ei -887 sp -888 ab -889 noth -890 lik -891 ak -892 don -893 na -894 much -895 - - -896 nge -897 looked -898 ro -899 sing -900 side -901 rr -902 ction -903 Mr -904 app -905 ov -906 might -907 sp -908 pe -909 under -910 ef -911 friend -912 ice -913 back -914 bou -915 under -916 went -917 les -918 op -919 ene -920 dea -921 age -922 long -923 low -924 if -925 without -926 ga -927 ag -928 te -929 where -930 unt -931 ite -932 even -933 v -934 ack -935 ful -936 ied -937 sto -938 afte -939 ster -940 gr -941 pla -942 mus -943 ner -944 ings -945 turn -946 urn -947 read -948 great -949 ree -950 ran -951 ything -952 ett -953 must -954 din -955 stan -956 thro -957 ile -958 how -959 pl -960 inte -961 che -962 thr -963 say -964 pres -965 made -966 atte -967 vi -968 place -969 migh -970 etter -971 good -972 ty -973 seem -974 pla -975 dr -976 ish -977 cha -978 nly -979 ays -980 went -981 don -982 first -983 pp -984 how -985 rse -986 us -987 nothi -988 even -989 place -990 ass -991 frien -992 nti -993 dis -994 S -995 ang -996 mp -997 eme -998 sha -999 abo -1000 ress -1001 am -1002 rat -1003 res -1004 has -1005 There -1006 ple -1007 came -1008 day -1009 nter -1010 away -1011 ded -1012 right -1013 grea -1014 ge -1015 ins -1016 ese -1017 mad -1018 unde -1019 ise -1020 nigh -1021 came -1022 spe -1023 cr -1024 day -1025 till -1026 He -1027 goo -1028 hed -1029 tions -1030 sto -1031 ssi -1032 ost -1033 by -1034 war -1035 owe -1036 ions -1037 kn -1038 just -1039 oke -1040 ette -1041 seemed -1042 take -1043 ange -1044 sen -1045 here -1046 fte -1047 off -1048 ount -1049 tly -1050 myself -1051 rec -1052 min -1053 way -1054 sion -1055 ds -1056 ppe -1057 pos -1058 hall -1059 gre -1060 per -1061 A -1062 eli -1063 house -1064 take -1065 eri -1066 son -1067 ope -1068 mu -1069 back -1070 got -1071 ade -1072 rest -1073 par -1074 ff -1075 unde -1076 make -1077 ours -1078 another -1079 hal -1080 lly -1081 sse -1082 turned -1083 til -1084 cl -1085 lea -1086 gre -1087 tan -1088 hou -1089 thro -1090 bet -1091 get -1092 eep -1093 ture -1094 most -1095 ex -1096 somet -1097 llow -1098 ord -1099 tre -1100 qui -1101 att -1102 rning -1103 house -1104 anything -1105 less -1106 nig -1107 cha -1108 head -1109 med -1110 ally -1111 ien -1112 well -1113 lle -1114 ious -1115 found -1116 pi -1117 ssed -1118 den -1119 ra -1120 fir -1121 aw -1122 being -1123 father -1124 ward -1125 ink -1126 lon -1127 may -1128 wher -1129 hes -1130 You -1131 ouse -1132 ely -1133 anything -1134 bac -1135 af -1136 ye -1137 But -1138 - - - - - - - - - - -1139 ying -1140 win -1141 ret -1142 ved -1143 ba -1144 cr -1145 rne -1146 qu -1147 ri -1148 get -1149 ys -1150 tell -1151 ger -1152 father -1153 ried -1154 stra -1155 these -1156 seem -1157 But -1158 plac -1159 head -1160 our -1161 well -1162 hous -1163 oin -1164 always -1165 entl -1166 comp -1167 rri -1168 turn -1169 face -1170 lf -1171 qu -1172 found -1173 nig -1174 ving -1175 har -1176 arr -1177 urs -1178 tak -1179 hil -1180 pen -1181 mes -1182 sur -1183 want -1184 ire -1185 br -1186 morning -1187 qui -1188 There -1189 just -1190 oor -1191 comp -1192 took -1193 inte -1194 ection -1195 W -1196 tel -1197 ci -1198 ru -1199 mak -1200 door -1201 tor -1202 pon -1203 still -1204 ngs -1205 iste -1206 rst -1207 cons -1208 rec -1209 mus -1210 B -1211 nother -1212 fac -1213 room -1214 air -1215 cause -1216 fin -1217 cti -1218 It -1219 last -1220 cam -1221 its -1222 count -1223 tain -1224 nger -1225 des -1226 ea -1227 righ -1228 ity -1229 because -1230 ft -1231 pass -1232 believe -1233 cl -1234 fou -1235 things -1236 ond -1237 rem -1238 enough -1239 dear -1240 sure -1241 has -1242 lan -1243 moment -1244 ourse -1245 eyes -1246 mon -1247 gi -1248 herself -1249 appe -1250 ether -1251 atter -1252 nat -1253 cons -1254 tell -1255 off -1256 part -1257 rem -1258 ew -1259 dr -1260 let -1261 lig -1262 urned -1263 says -1264 msel -1265 looking -1266 suc -1267 room -1268 make -1269 ris -1270 foun -1271 pu -1272 two -1273 car -1274 fu -1275 alon -1276 eye -1277 resent -1278 rself -1279 ye -1280 give -1281 ration -1282 heard -1283 most -1284 pea -1285 au -1286 nk -1287 together -1288 -- -1289 part -1290 believe -1291 moment -1292 und -1293 tch -1294 answer -1295 ause -1296 par -1297 fathe -1298 inter -1299 present -1300 hap -1301 ined -1302 nds -1303 fri -1304 its -1305 heart -1306 des -1307 lac -1308 turne -1309 eing -1310 pass -1311 young -1312 lie -1313 once -1314 appear -1315 light -1316 dre -1317 sent -1318 ened -1319 exp -1320 still -1321 face -1322 tai -1323 oll -1324 tting -1325 hous -1326 done -1327 jus -1328 ac -1329 better -1330 omet -1331 ated -1332 eart -1333 ick -1334 ering -1335 mad -1336 orr -1337 wel -1338 She -1339 urne -1340 ui -1341 body -1342 ecause -1343 bl -1344 fin -1345 thes -1346 ark -1347 thre -1348 own -1349 fel -1350 let -1351 oked -1352 You -1353 aw -1354 press -1355 las -1356 looking -1357 hers -1358 arri -1359 row -1360 ser -1361 heart -1362 rned -1363 car -1364 beg -1365 nted -1366 believ -1367 ren -1368 going -1369 hands -1370 iness -1371 around -1372 app -1373 old -1374 pas -1375 last -1376 answer -1377 wan -1378 bro -1379 erin -1380 want -1381 iss -1382 fee -1383 happ -1384 Miss -1385 inc -1386 hu -1387 ies -1388 tw -1389 fir -1390 oi -1391 people -1392 certain -1393 leas -1394 ming -1395 oc -1396 era -1397 inst -1398 bet -1399 return -1400 win -1401 presen -1402 word -1403 ntl -1404 And -1405 ju -1406 tel -1407 cu -1408 ces -1409 give -1410 sl -1411 nev -1412 turned -1413 Tom -1414 roo -1415 sc -1416 Holmes -1417 enti -1418 ssion -1419 aking -1420 ouldn -1421 fort -1422 door -1423 inter -1424 stil -1425 act -1426 fac -1427 ‘ -1428 beca -1429 pres -1430 hese -1431 ooking -1432 sti -1433 ently -1434 oss -1435 hu -1436 mean -1437 rm -1438 Ther -1439 lat -1440 present -1441 ls -1442 business -1443 They -1444 understand -1445 hout -1446 ible -1447 however -1448 soon -1449 ntly -1450 len -1451 ectio -1452 rese -1453 rds -1454 bel -1455 ue -1456 She -1457 shed -1458 spe -1459 ia -1460 momen -1461 nne -1462 oom -1463 appear -1464 feel -1465 everything -1466 open -1467 chi -1468 L -1469 pri -1470 af -1471 sleep -1472 Wh -1473 lookin -1474 tic -1475 eed -1476 tur -1477 oing -1478 bec -1479 business -1480 Wh -1481 ogether -1482 irs -1483 gi -1484 work -1485 - - -1486 window -1487 hor -1488 awa -1489 iti -1490 eath -1491 ines -1492 togethe -1493 happ -1494 life -1495 brea -1496 rit -1497 may -1498 consider -1499 people -1500 tation -1501 hose -1502 pect -1503 eet -1504 doo -1505 ars -1506 young -1507 tru -1508 ert -1509 mos -1510 gl -1511 certain -1512 dy -1513 What -1514 iou -1515 ped -1516 return -1517 mind -1518 stra -1519 tere -1520 im -1521 nst -1522 CHAPTER -1523 prese -1524 while -1525 lli -1526 read -1527 clo -1528 cal -1529 mis -1530 atch -1531 dear -1532 owever -1533 ffe -1534 hands -1535 las -1536 kin -1537 wind -1538 sit -1539 rati -1540 consider -1541 nn -1542 answe -1543 three -1544 sister -1545 atur -1546 lic -1547 omen -1548 ched -1549 llin -1550 strange -1551 whil -1552 eal -1553 feeling -1554 giv -1555 stand -1556 It -1557 fou -1558 lled -1559 met -1560 ach -1561 ask -1562 ende -1563 Miss -1564 ecti -1565 understand -1566 ratio -1567 Then -1568 sur -1569 interest -1570 owed -1571 rth -1572 And -1573 nding -1574 thre -1575 cour -1576 tal -1577 stre -1578 towards -1579 letter -1580 appea -1581 ole -1582 oment -1583 ins -1584 feeling -1585 rl -1586 gin -1587 ki -1588 began -1589 yin -1590 nl -1591 They -1592 kind -1593 exp -1594 count -1595 rai -1596 gra -1597 brea -1598 D -1599 ways -1600 asked -1601 everything -1602 together -1603 minute -1604 Well -1605 ock -1606 tro -1607 ase -1608 ears -1609 mea -1610 ared -1611 rk -1612 wards -1613 hope -1614 ont -1615 - - - - -1616 rtain -1617 ngl -1618 oking -1619 imp -1620 ater -1621 rep -1622 cour -1623 youn -1624 follow -1625 strange -1626 orn -1627 ligh -1628 lar -1629 eye -1630 pleas -1631 wish -1632 love -1633 ected -1634 ga -1635 ingl -1636 ention -1637 once -1638 Tom -1639 rive -1640 ife -1641 saw -1642 tri -1643 gentle -1644 ister -1645 mil -1646 along -1647 esse -1648 cont -1649 question -1650 pleas -1651 lling -1652 many -1653 imp -1654 men -1655 ung -1656 sister -1657 dre -1658 ste -1659 tow -1660 augh -1661 interest -1662 rry -1663 peopl -1664 those -1665 possibl -1666 rest -1667 beca -1668 call -1669 tra -1670 times -1671 gra -1672 possible -1673 sleep -1674 retur -1675 feel -1676 tance -1677 pec -1678 ton -1679 manner -1680 essi -1681 bro -1682 minute -1683 gen -1684 lif -1685 C -1686 oon -1687 rep -1688 ean -1689 Helsing -1690 cted -1691 rc -1692 quite -1693 against -1694 mber -1695 siste -1696 two -1697 elie -1698 matter -1699 clo -1700 ged -1701 table -1702 ady -1703 To -1704 fri -1705 close -1706 call -1707 fat -1708 underst -1709 um -1710 ntere -1711 stre -1712 open -1713 alk -1714 essed -1715 range -1716 ards -1717 sl -1718 ina -1719 ad -1720 ein -1721 mind -1722 When -1723 work -1724 fl -1725 hol -1726 ans -1727 gentl -1728 gentle -1729 rema -1730 left -1731 fee -1732 ber -1733 pin -1734 question -1735 find -1736 los -1737 rge -1738 j -1739 ature -1740 tat -1741 pers -1742 nded -1743 mean -1744 same -1745 ations -1746 child -1747 follow -1748 quit -1749 told -1750 dar -1751 cre -1752 mis -1753 ement -1754 understan -1755 gen -1756 sist -1757 conside -1758 between -1759 ms -1760 certai -1761 strang -1762 eav -1763 talk -1764 fer -1765 yes -1766 ndi -1767 tal -1768 shi -1769 Darcy -1770 serv -1771 cal -1772 fel -1773 ap -1774 coming -1775 eak -1776 togeth -1777 differen -1778 pped -1779 star -1780 “‘ -1781 love -1782 op -1783 uck -1784 arn -1785 new -1786 att -1787 aving -1788 tten -1789 esti -1790 chi -1791 wind -1792 ished -1793 underst -1794 wan -1795 lu -1796 trouble -1797 Bingley -1798 Mis -1799 pi -1800 arge -1801 pris -1802 Then -1803 sc -1804 oa -1805 course -1806 na -1807 ens -1808 bre -1809 mar -1810 riv -1811 hap -1812 life -1813 kind -1814 cont -1815 tatio -1816 Sh -1817 eg -1818 reas -1819 ema -1820 - - - - -1821 late -1822 pri -1823 ud -1824 remember -1825 trouble -1826 pose -1827 form -1828 emen -1829 Lo -1830 wat -1831 ometimes -1832 siste -1833 erv -1834 remember -1835 ope -1836 bod -1837 rac -1838 toward -1839 Bennet -1840 bel -1841 akin -1842 each -1843 round -1844 point -1845 ember -1846 ibl -1847 feelin -1848 ses -1849 ghte -1850 rang -1851 We -1852 course -1853 cannot -1854 Defarge -1855 wing -1856 ente -1857 fect -1858 daughter -1859 close -1860 side -1861 ening -1862 ready -1863 ves -1864 Defarge -1865 beg -1866 nsi -1867 ey -1868 ann -1869 whole -1870 ession -1871 aught -1872 ffer -1873 nts -1874 sou -1875 har -1876 seen -1877 tu -1878 swe -1879 voice -1880 prison -1881 dist -1882 child -1883 bea -1884 hel -1885 tche -1886 rel -1887 hile -1888 trange -1889 coun -1890 uldn -1891 chan -1892 ask -1893 arm -1894 pu -1895 Illustration -1896 coming -1897 passe -1898 interes -1899 gentlem -1900 cau -1901 spec -1902 sen -1903 ow -1904 eel -1905 tw -1906 put -1907 ody -1908 ari -1909 appe -1910 An -1911 wis -1912 prison -1913 aughter -1914 passed -1915 ros -1916 chil -1917 ced -1918 bec -1919 speak -1920 lt -1921 ents -1922 nou -1923 vi -1924 ishe -1925 wonder -1926 hos -1927 ull -1928 Mi -1929 minut -1930 dark -1931 year -1932 rta -1933 far -1934 That -1935 erse -1936 full -1937 ron -1938 cas -1939 chan -1940 Ma -1941 ingle -1942 gs -1943 asse -1944 yet -1945 ressed -1946 ossible -1947 sible -1948 follo -1949 silen -1950 near -1951 ition -1952 sat -1953 mother -1954 Ho -1955 ary -1956 We -1957 plea -1958 clos -1959 hour -1960 acc -1961 ained -1962 mother -1963 spect -1964 hope -1965 questio -1966 tered -1967 hel -1968 ob -1969 ent -1970 sol -1971 alo -1972 pt -1973 alle -1974 ear -1975 ters -1976 iver -1977 poor -1978 rance -1979 ept -1980 live -1981 ieve -1982 J -1983 slee -1984 fell -1985 didn -1986 que -1987 gentleman -1988 wed -1989 nation -1990 hard -1991 supp -1992 atte -1993 inst -1994 compa -1995 Yo -1996 wr -1997 mat -1998 Lorry -1999 express -2000 rema -2001 eca -2002 ims -2003 arriage -2004 ping -2005 wish -2006 stance -2007 quit -2008 tran -2009 owing -2010 till -2011 brought -2012 hop -2013 lov -2014 ece -2015 va -2016 rable -2017 nse -2018 pened -2019 real -2020 brought -2021 keep -2022 wal -2023 cte -2024 wee -2025 wonder -2026 stood -2027 gl -2028 husband -2029 half -2030 atten -2031 mpl -2032 iden -2033 tabl -2034 ily -2035 mb -2036 passe -2037 ral -2038 onde -2039 hree -2040 entle -2041 tow -2042 atten -2043 cke -2044 dn -2045 erta -2046 troubl -2047 tention -2048 imes -2049 Alice -2050 either -2051 world -2052 ither -2053 oming -2054 gentleman -2055 ret -2056 ank -2057 sm -2058 surprise -2059 eare -2060 ctor -2061 star -2062 tree -2063 ke -2064 talk -2065 exc -2066 erhaps -2067 behind -2068 rious -2069 ains -2070 mar -2071 sin -2072 appeared -2073 watch -2074 answered -2075 knew -2076 expe -2077 arl -2078 usi -2079 orm -2080 remain -2081 stin -2082 ince -2083 tent -2084 remembe -2085 express -2086 country -2087 fear -2088 ua -2089 stood -2090 end -2091 sten -2092 This -2093 attention -2094 kne -2095 ind -2096 nner -2097 usin -2098 ju -2099 start -2100 oursel -2101 white -2102 Alice -2103 mea -2104 liev -2105 sometimes -2106 surprise -2107 prisoner -2108 returned -2109 leave -2110 famil -2111 ndo -2112 hom -2113 ute -2114 arc -2115 change -2116 yourself -2117 fl -2118 atu -2119 particular -2120 dist -2121 gent -2122 comin -2123 ffere -2124 cle -2125 husband -2126 sat -2127 kne -2128 yourself -2129 speak -2130 ild -2131 bject -2132 When -2133 rate -2134 Dar -2135 ane -2136 rtai -2137 home -2138 expect -2139 reason -2140 compa -2141 sudden -2142 pos -2143 onder -2144 felt -2145 leav -2146 wis -2147 onsi -2148 eave -2149 tati -2150 sup -2151 ors -2152 ru -2153 country -2154 indeed -2155 family -2156 gone -2157 sle -2158 port -2159 war -2160 cri -2161 tru -2162 nside -2163 natur -2164 rried -2165 ern -2166 find -2167 set -2168 ders -2169 howe -2170 ecte -2171 rg -2172 ntr -2173 try -2174 llowed -2175 eco -2176 ured -2177 aying -2178 quest -2179 uit -2180 entlem -2181 tanc -2182 poss -2183 fere -2184 tea -2185 fat -2186 words -2187 wouldn -2188 reason -2189 omin -2190 sor -2191 eate -2192 point -2193 live -2194 uri -2195 cer -2196 dde -2197 land -2198 eady -2199 gu -2200 having -2201 lice -2202 sted -2203 omi -2204 ai -2205 among -2206 lay -2207 ging -2208 resse -2209 cover -2210 reckon -2211 hal -2212 ower -2213 show -2214 toget -2215 ressi -2216 ross -2217 aste -2218 saw -2219 perhaps -2220 spoke -2221 cried -2222 pressi -2223 ffect -2224 won -2225 ken -2226 sometimes -2227 carr -2228 emb -2229 eac -2230 unc -2231 easure -2232 natur -2233 couldn -2234 strong -2235 sibl -2236 sle -2237 fu -2238 dar -2239 mer -2240 remain -2241 priso -2242 sati -2243 lowe -2244 tive -2245 ouble -2246 ntin -2247 spo -2248 q -2249 enter -2250 expect -2251 drea -2252 expe -2253 ranc -2254 sudden -2255 eck -2256 mys -2257 perhaps -2258 comm -2259 anger -2260 tes -2261 lai -2262 almost -2263 sound -2264 dark -2265 ease -2266 gro -2267 far -2268 ject -2269 conversation -2270 ortun -2271 roun -2272 dly -2273 stru -2274 feren -2275 tar -2276 attention -2277 rou -2278 particular -2279 different -2280 woman -2281 An -2282 circumstance -2283 care -2284 Co -2285 fea -2286 mat -2287 ived -2288 hop -2289 suppose -2290 change -2291 cri -2292 err -2293 ple -2294 orry -2295 keep -2296 walk -2297 octor -2298 avin -2299 uite -2300 oub -2301 wonde -2302 others -2303 knowledge -2304 watch -2305 Lor -2306 rro -2307 pretty -2308 start -2309 writ -2310 both -2311 near -2312 ani -2313 possibl -2314 dge -2315 els -2316 stro -2317 name -2318 eared -2319 tched -2320 wait -2321 ybody -2322 friends -2323 sig -2324 etting -2325 water -2326 erable -2327 assed -2328 spea -2329 gla -2330 liv -2331 ami -2332 tol -2333 acc -2334 mothe -2335 hard -2336 oice -2337 used -2338 surpris -2339 water -2340 pers -2341 mon -2342 ini -2343 called -2344 conc -2345 hole -2346 sou -2347 opped -2348 stoo -2349 rabl -2350 lad -2351 cen -2352 ip -2353 real -2354 sea -2355 differen -2356 help -2357 conf -2358 brother -2359 carri -2360 strong -2361 aint -2362 year -2363 lowed -2364 cur -2365 gla -2366 ign -2367 nten -2368 hour -2369 compan -2370 ub -2371 countr -2372 compl -2373 itte -2374 roa -2375 use -2376 spo -2377 person -2378 husban -2379 eral -2380 hang -2381 bi -2382 sk -2383 isten -2384 ssing -2385 stant -2386 rte -2387 fol -2388 tant -2389 ref -2390 ib -2391 brother -2392 called -2393 comfort -2394 rted -2395 half -2396 care -2397 lis -2398 since -2399 supp -2400 world -2401 lock -2402 nea -2403 tc -2404 river -2405 dress -2406 river -2407 du -2408 du -2409 silence -2410 dee -2411 comm -2412 No -2413 urt -2414 cor -2415 sor -2416 poor -2417 ndeed -2418 whit -2419 white -2420 ceive -2421 wal -2422 bea -2423 aug -2424 ob -2425 fell -2426 fear -2427 dered -2428 ual -2429 lone -2430 kee -2431 suppo -2432 years -2433 instant -2434 continu -2435 appeare -2436 ppeared -2437 poo -2438 Whe -2439 dent -2440 dd -2441 small -2442 nea -2443 diff -2444 gle -2445 rather -2446 instan -2447 bot -2448 put -2449 cc -2450 doubt -2451 reckon -2452 serve -2453 wom -2454 expres -2455 tend -2456 liv -2457 rti -2458 oved -2459 Luc -2460 really -2461 tri -2462 enin -2463 opp -2464 sure -2465 rde -2466 ertainly -2467 already -2468 ext -2469 nstant -2470 Professor -2471 diff -2472 woman -2473 warn -2474 enter -2475 sid -2476 pat -2477 ica -2478 set -2479 inde -2480 conc -2481 To -2482 continu -2483 pression -2484 Jonathan -2485 paper -2486 yours -2487 dead -2488 ps -2489 fellow -2490 cu -2491 ona -2492 death -2493 pene -2494 ega -2495 dl -2496 pleasure -2497 affection -2498 others -2499 sound -2500 poss -2501 doubt -2502 natio -2503 mise -2504 bus -2505 nati -2506 nta -2507 rio -2508 lear -2509 hor -2510 paper -2511 ail -2512 eng -2513 within -2514 remai -2515 doub -2516 erabl -2517 replied -2518 alone -2519 Mrs -2520 rly -2521 evening -2522 spoke -2523 Be -2524 bri -2525 bes -2526 particula -2527 possible -2528 uddenly -2529 aring -2530 ident -2531 disc -2532 writ -2533 cas -2534 aine -2535 tre -2536 high -2537 spi -2538 remark -2539 silence -2540 east -2541 arried -2542 shor -2543 reme -2544 ossi -2545 anne -2546 verybody -2547 Doctor -2548 avi -2549 person -2550 mark -2551 lent -2552 boy -2553 yea -2554 followed -2555 worl -2556 does -2557 What -2558 ments -2559 treet -2560 ues -2561 sig -2562 erf -2563 vo -2564 certainly -2565 affection -2566 P -2567 Sh -2568 eithe -2569 vil -2570 sudde -2571 conce -2572 conv -2573 case -2574 This -2575 vent -2576 ords -2577 rop -2578 esi -2579 sw -2580 inu -2581 happen -2582 direct -2583 instant -2584 considera -2585 ce -2586 eni -2587 ili -2588 ised -2589 circumstance -2590 themselves -2591 watc -2592 home -2593 ison -2594 ape -2595 feelings -2596 rofessor -2597 chang -2598 arte -2599 tely -2600 eren -2601 .’ -2602 struc -2603 reco -2604 mpa -2605 new -2606 suppos -2607 conce -2608 hur -2609 tie -2610 suddenly -2611 rave -2612 conf -2613 ano -2614 conversation -2615 opened -2616 observ -2617 reaso -2618 ellow -2619 form -2620 uring -2621 wait -2622 lett -2623 ffic -2624 drea -2625 tened -2626 fortun -2627 whether -2628 sec -2629 bla -2630 pleasure -2631 suddenly -2632 character -2633 thers -2634 nch -2635 sil -2636 until -2637 power -2638 bed -2639 whether -2640 ating -2641 mile -2642 show -2643 cond -2644 erve -2645 No -2646 exc -2647 rofess -2648 recko -2649 este -2650 orro -2651 fully -2652 ended -2653 discover -2654 covered -2655 owin -2656 alled -2657 ience -2658 gro -2659 nal -2660 cked -2661 vel -2662 appearance -2663 appeared -2664 therefore -2665 llowe -2666 stron -2667 dden -2668 walk -2669 carri -2670 cat -2671 net -2672 E -2673 name -2674 crea -2675 sw -2676 stopped -2677 occasion -2678 ppo -2679 hair -2680 fic -2681 expec -2682 compl -2683 That -2684 rat -2685 wri -2686 nish -2687 iz -2688 ban -2689 few -2690 tood -2691 arrie -2692 considera -2693 cross -2694 acco -2695 direct -2696 carr -2697 lad -2698 bi -2699 feelings -2700 difficult -2701 happiness -2702 listen -2703 wanted -2704 compan -2705 ighte -2706 marr -2707 help -2708 rong -2709 difficult -2710 cre -2711 pret -2712 whisper -2713 comfort -2714 ination -2715 conv -2716 remark -2717 Jonathan -2718 poin -2719 aki -2720 arly -2721 shor -2722 mot -2723 isi -2724 run -2725 character -2726 general -2727 brothe -2728 char -2729 stro -2730 fre -2731 cul -2732 case -2733 stat -2734 ass -2735 col -2736 disc -2737 oes -2738 happen -2739 ground -2740 wai -2741 rise -2742 late -2743 rward -2744 dere -2745 alking -2746 eld -2747 everybody -2748 spok -2749 stat -2750 udde -2751 gan -2752 minutes -2753 continue -2754 hai -2755 hte -2756 observ -2757 lat -2758 comi -2759 beh -2760 cess -2761 anno -2762 happy -2763 contin -2764 silenc -2765 whisper -2766 both -2767 rder -2768 So -2769 sation -2770 listen -2771 selves -2772 street -2773 emain -2774 hom -2775 chin -2776 tate -2777 Al -2778 eno -2779 death -2780 ente -2781 fai -2782 quick -2783 satisf -2784 therefore -2785 ener -2786 tone -2787 breakfast -2788 soun -2789 nging -2790 emp -2791 ................ -2792 themselves -2793 spirit -2794 spirit -2795 eceive -2796 stor -2797 mine -2798 rrow -2799 col -2800 power -2801 sil -2802 arti -2803 rain -2804 mou -2805 sup -2806 thousand -2807 receive -2808 standing -2809 Collins -2810 beautiful -2811 occasion -2812 nigger -2813 alked -2814 whom -2815 lity -2816 ete -2817 ayin -2818 arted -2819 small -2820 fie -2821 ain -2822 woma -2823 Jane -2824 uth -2825 perso -2826 acco -2827 sun -2828 pressed -2829 iting -2830 evi -2831 Dar -2832 creature -2833 stand -2834 foo -2835 Catherine -2836 early -2837 sens -2838 oti -2839 different -2840 happiness -2841 hardly -2842 affect -2843 receiv -2844 Ho -2845 ettin -2846 ching -2847 rive -2848 hoo -2849 qua -2850 Da -2851 natu -2852 unti -2853 continue -2854 affectio -2855 Wickham -2856 next -2857 iff -2858 ivi -2859 Catherine -2860 shoulder -2861 char -2862 clea -2863 eti -2864 ait -2865 affect -2866 natural -2867 ground -2868 tene -2869 sens -2870 cro -2871 thousand -2872 stop -2873 wate -2874 lence -2875 break -2876 sm -2877 oppe -2878 Wickham -2879 gener -2880 short -2881 everybody -2882 perfect -2883 street -2884 rying -2885 taken -2886 lect -2887 die -2888 fea -2889 tab -2890 sib -2891 beaut -2892 ceived -2893 rent -2894 cle -2895 trea -2896 shoulder -2897 creature -2898 continued -2899 calle -2900 iving -2901 heav -2902 subject -2903 account -2904 general -2905 account -2906 assi -2907 Huck -2908 hy -2909 itting -2910 vening -2911 avour -2912 rab -2913 ows -2914 breakfast -2915 blo -2916 stea -2917 fol -2918 sting -2919 each -2920 days -2921 terr -2922 rel -2923 ubl -2924 sea -2925 break -2926 toge -2927 icul -2928 mm -2929 vered -2930 rre -2931 og -2932 eature -2933 nat -2934 ited -2935 elt -2936 discover -2937 orrow -2938 town -2939 cor -2940 warn -2941 nf -2942 parti -2943 receive -2944 best -2945 cious -2946 trai -2947 cent -2948 won -2949 yi -2950 bli -2951 nis -2952 aro -2953 los -2954 carriage -2955 companion -2956 deli -2957 ote -2958 getting -2959 notice -2960 subject -2961 became -2962 vers -2963 aken -2964 mot -2965 Ha -2966 reate -2967 Huck -2968 ffi -2969 lation -2970 circumstances -2971 genera -2972 promise -2973 breath -2974 repl -2975 high -2976 dead -2977 edi -2978 considerabl -2979 happy -2980 rus -2981 orth -2982 arke -2983 ante -2984 ient -2985 lay -2986 aff -2987 getting -2988 expected -2989 eam -2990 bri -2991 either -2992 satisf -2993 trou -2994 ract -2995 inc -2996 ourn -2997 eres -2998 hit -2999 Do -3000 clear -3001 lve -3002 position -3003 promise -3004 London -3005 ference -3006 fort -3007 smal -3008 parti -3009 main -3010 arm -3011 ula -3012 gge -3013 illi -3014 reco -3015 ately -3016 girl -3017 ining -3018 step -3019 remar -3020 times -3021 expression -3022 expressi -3023 rch -3024 Ca -3025 san -3026 fai -3027 arin -3028 ser -3029 marri -3030 lou -3031 cy -3032 Lo -3033 difficul -3034 neral -3035 wood -3036 often -3037 atche -3038 nam -3039 ense -3040 shin -3041 orse -3042 hun -3043 ali -3044 Lor -3045 children -3046 village -3047 black -3048 idea -3049 idn -3050 gon -3051 bus -3052 lm -3053 characte -3054 quiet -3055 exce -3056 ott -3057 ybod -3058 nsta -3059 rous -3060 reck -3061 esp -3062 rv -3063 G -3064 afterwards -3065 stru -3066 allow -3067 deat -3068 quick -3069 ired -3070 etch -3071 happe -3072 direc -3073 perfect -3074 wate -3075 add -3076 conve -3077 —� -3078 gener -3079 ithin -3080 considerabl -3081 appearance -3082 urning -3083 creat -3084 often -3085 overed -3086 ream -3087 sol -3088 ara -3089 ske -3090 continued -3091 immediate -3092 companion -3093 vere -3094 ash -3095 rrible -3096 disco -3097 struck -3098 del -3099 line -3100 nu -3101 Wi -3102 rve -3103 Jo -3104 ks -3105 occ -3106 deli -3107 whose -3108 obser -3109 rot -3110 saying -3111 eviden -3112 danger -3113 object -3114 Hol -3115 dle -3116 cie -3117 happened -3118 forg -3119 plain -3120 eside -3121 lde -3122 ele -3123 taken -3124 covere -3125 havi -3126 pat -3127 lem -3128 entered -3129 anted -3130 move -3131 mme -3132 Monseigneur -3133 notice -3134 laugh -3135 dres -3136 band -3137 ously -3138 disco -3139 ney -3140 ici -3141 whispe -3142 comfor -3143 suffer -3144 tern -3145 va -3146 observe -3147 oe -3148 fire -3149 full -3150 dness -3151 asi -3152 add -3153 eigh -3154 lan -3155 Ma -3156 rtu -3157 - - -3158 - -3159 knowledge -3160 satisfied -3161 nished -3162 seve -3163 gave -3164 sign -3165 considerable -3166 sec -3167 eig -3168 step -3169 ello -3170 dence -3171 nning -3172 sigh -3173 el -3174 twe -3175 laugh -3176 alm -3177 Mi -3178 occasio -3179 osit -3180 mple -3181 gree -3182 oned -3183 Mar -3184 marr -3185 expression -3186 pected -3187 money -3188 urr -3189 rav -3190 dia -3191 oy -3192 considerable -3193 compani -3194 state -3195 rece -3196 Lucy -3197 dec -3198 eryt -3199 oat -3200 ita -3201 bla -3202 uti -3203 conve -3204 earing -3205 lder -3206 truc -3207 natural -3208 Manette -3209 less -3210 imagin -3211 village -3212 tro -3213 edge -3214 dro -3215 ppos -3216 uff -3217 lot -3218 immediately -3219 liste -3220 danger -3221 short -3222 cur -3223 hol -3224 earl -3225 pur -3226 ial -3227 terrible -3228 remained -3229 tanding -3230 acti -3231 ches -3232 oner -3233 renc -3234 peri -3235 swe -3236 inatio -3237 fre -3238 cro -3239 dif -3240 wood -3241 dra -3242 beaut -3243 creat -3244 bot -3245 pli -3246 thousan -3247 standin -3248 inking -3249 patien -3250 earn -3251 ully -3252 fess -3253 ura -3254 cho -3255 children -3256 expressi -3257 powe -3258 In -3259 tem -3260 Luc -3261 adv -3262 flo -3263 osed -3264 road -3265 osi -3266 rew -3267 breath -3268 broke -3269 least -3270 visit -3271 blood -3272 promis -3273 posit -3274 bed -3275 temp -3276 ool -3277 started -3278 Co -3279 ular -3280 fast -3281 received -3282 idence -3283 presse -3284 enera -3285 quiet -3286 move -3287 St -3288 delight -3289 erab -3290 shoulde -3291 list -3292 ler -3293 lme -3294 notic -3295 conti -3296 ista -3297 visi -3298 ston -3299 immediately -3300 bit -3301 nit -3302 ker -3303 ult -3304 ria -3305 hus -3306 wr -3307 chance -3308 observe -3309 spi -3310 bes -3311 tried -3312 ready -3313 miser -3314 Y -3315 resently -3316 pap -3317 wea -3318 Pro -3319 ually -3320 utter -3321 visi -3322 pai -3323 wea -3324 walked -3325 nno -3326 spiri -3327 perfectly -3328 alkin -3329 plied -3330 marri -3331 stu -3332 loc -3333 hee -3334 memb -3335 hig -3336 discove -3337 ........... -3338 second -3339 forward -3340 eceived -3341 anybody -3342 sight -3343 elling -3344 suffer -3345 elves -3346 inted -3347 suff -3348 eful -3349 rke -3350 oot -3351 als -3352 rce -3353 emark -3354 lking -3355 wered -3356 lady -3357 inten -3358 lla -3359 aff -3360 determin -3361 begin -3362 arriv -3363 subjec -3364 accoun -3365 ppin -3366 ley -3367 strai -3368 eated -3369 top -3370 eith -3371 afterwards -3372 large -3373 nature -3374 object -3375 adi -3376 experience -3377 respect -3378 exce -3379 atin -3380 clea -3381 Jim -3382 smile -3383 served -3384 affec -3385 che -3386 God -3387 noti -3388 clear -3389 black -3390 hind -3391 nces -3392 arr -3393 vo -3394 voi -3395 dan -3396 respect -3397 hundred -3398 terribl -3399 ctions -3400 groun -3401 imagin -3402 reach -3403 mal -3404 iet -3405 jo -3406 cept -3407 ,’ -3408 heav -3409 rett -3410 sca -3411 hildren -3412 stea -3413 stop -3414 ries -3415 pai -3416 positi -3417 action -3418 horse -3419 erence -3420 forward -3421 oli -3422 arth -3423 seve -3424 rece -3425 lease -3426 whis -3427 ainly -3428 smil -3429 terr -3430 stan -3431 grou -3432 selve -3433 stree -3434 igger -3435 Pr -3436 chance -3437 state -3438 rect -3439 ntil -3440 urin -3441 adam -3442 sort -3443 quic -3444 wife -3445 boy -3446 rely -3447 alt -3448 terrible -3449 companio -3450 across -3451 deep -3452 crea -3453 hun -3454 tice -3455 dri -3456 indi -3457 cau -3458 dan -3459 anybody -3460 acquaint -3461 ingly -3462 forme -3463 F -3464 corner -3465 positio -3466 osition -3467 scen -3468 org -3469 sight -3470 draw -3471 visit -3472 del -3473 dou -3474 circumstances -3475 elve -3476 ivin -3477 cla -3478 :-- -3479 several -3480 horse -3481 les -3482 mou -3483 tea -3484 repl -3485 nced -3486 write -3487 erl -3488 nion -3489 determin -3490 endeavour -3491 shing -3492 tende -3493 ory -3494 ges -3495 Go -3496 ham -3497 ush -3498 delight -3499 childre -3500 villag -3501 que -3502 sper -3503 iona -3504 rust -3505 equ -3506 nking -3507 need -3508 money -3509 bra -3510 mai -3511 best -3512 nor -3513 aps -3514 beh -3515 rag -3516 comfortable -3517 determine -3518 itself -3519 breat -3520 inten -3521 known -3522 erat -3523 ctly -3524 appy -3525 inat -3526 rme -3527 inn -3528 Ja -3529 spir -3530 outh -3531 inf -3532 ittin -3533 earth -3534 fore -3535 rence -3536 bloo -3537 isc -3538 second -3539 satis -3540 hundred -3541 perfec -3542 nature -3543 etti -3544 unc -3545 ref -3546 eas -3547 ghts -3548 rib -3549 satisfie -3550 atisfied -3551 beau -3552 gene -3553 miser -3554 pan -3555 Count -3556 ntered -3557 ferenc -3558 cei -3559 order -3560 eased -3561 eived -3562 inati -3563 list -3564 sigh -3565 lish -3566 mov -3567 De -3568 So -3569 N -3570 erfectly -3571 girl -3572 alke -3573 Her -3574 fire -3575 Cruncher -3576 ength -3577 event -3578 overe -3579 fas -3580 Lu -3581 fortune -3582 ends -3583 ocke -3584 noti -3585 sli -3586 lanc -3587 dro -3588 erc -3589 opportunity -3590 wishe -3591 strat -3592 stopp -3593 nting -3594 aster -3595 act -3596 teri -3597 rai -3598 rdin -3599 straight -3600 perfectly -3601 necessary -3602 istance -3603 chai -3604 blood -3605 cause -3606 touch -3607 dread -3608 sir -3609 shi -3610 ter -3611 ledge -3612 lost -3613 truck -3614 erful -3615 passi -3616 reath -3617 also -3618 mem -3619 foo -3620 patient -3621 learn -3622 order -3623 rite -3624 expected -3625 miserable -3626 Mo -3627 Her -3628 onf -3629 impossible -3630 satisfied -3631 cher -3632 table -3633 confi -3634 talking -3635 impo -3636 latio -3637 ryin -3638 onse -3639 sub -3640 run -3641 scr -3642 lked -3643 His -3644 straight -3645 thoughts -3646 eading -3647 —” -3648 etched -3649 counte -3650 tters -3651 lings -3652 prop -3653 ollin -3654 endeavour -3655 stu -3656 itin -3657 ache -3658 Yes -3659 thinking -3660 carried -3661 purpose -3662 amo -3663 O -3664 corner -3665 hours -3666 eeping -3667 therin -3668 dange -3669 objec -3670 Madam -3671 ures -3672 forg -3673 uick -3674 unte -3675 lati -3676 owl -3677 mov -3678 natura -3679 secret -3680 sign -3681 ones -3682 grou -3683 miserable -3684 determined -3685 hold -3686 lack -3687 viden -3688 fall -3689 istan -3690 horr -3691 confi -3692 Madam -3693 sco -3694 blo -3695 afraid -3696 during -3697 exa -3698 glan -3699 making -3700 suffe -3701 urnin -3702 four -3703 usual -3704 artic -3705 efu -3706 importan -3707 necessar -3708 ide -3709 agin -3710 etty -3711 oma -3712 rp -3713 gn -3714 countenance -3715 determine -3716 making -3717 silent -3718 please -3719 espec -3720 blac -3721 road -3722 idea -3723 ben -3724 veni -3725 stri -3726 eman -3727 aut -3728 vis -3729 empt -3730 suff -3731 surprised -3732 perfectl -3733 write -3734 quie -3735 pain -3736 desi -3737 van -3738 dress -3739 secre -3740 given -3741 happi -3742 purpose -3743 Lon -3744 ille -3745 ful -3746 dest -3747 bour -3748 epar -3749 raw -3750 Be -3751 begin -3752 ading -3753 ale -3754 sun -3755 dee -3756 woo -3757 cula -3758 ecei -3759 impossible -3760 experience -3761 usual -3762 marriage -3763 neighbour -3764 large -3765 ofte -3766 arriv -3767 elle -3768 epe -3769 amp -3770 rid -3771 gu -3772 erm -3773 earth -3774 taking -3775 imagi -3776 denc -3777 urni -3778 sses -3779 smile -3780 serv -3781 owi -3782 inf -3783 gg -3784 dreadful -3785 itation -3786 eration -3787 nate -3788 prov -3789 onat -3790 elli -3791 rev -3792 opportunity -3793 acquaintance -3794 attend -3795 standi -3796 ender -3797 perf -3798 ested -3799 ci -3800 nci -3801 air -3802 fact -3803 isit -3804 dnes -3805 eek -3806 nishe -3807 inkin -3808 earin -3809 happi -3810 inin -3811 orta -3812 Van -3813 sma -3814 dra -3815 complete -3816 evident -3817 dream -3818 sever -3819 honour -3820 sense -3821 occ -3822 ran -3823 try -3824 nkin -3825 itat -3826 gger -3827 marriage -3828 sense -3829 laug -3830 moved -3831 retch -3832 ances -3833 body -3834 Why -3835 eo -3836 turning -3837 beside -3838 everal -3839 allow -3840 sit -3841 Al -3842 sever -3843 dread -3844 raise -3845 racte -3846 nor -3847 bur -3848 gir -3849 hai -3850 rove -3851 olle -3852 Huc -3853 terri -3854 chanc -3855 strength -3856 distan -3857 collect -3858 Harker -3859 greate -3860 stor -3861 ties -3862 nee -3863 gar -3864 oft -3865 sitting -3866 concern -3867 atel -3868 fal -3869 fall -3870 orne -3871 whis -3872 nged -3873 rown -3874 prop -3875 reg -3876 finger -3877 reach -3878 iously -3879 comes -3880 stone -3881 pir -3882 ric -3883 finger -3884 respec -3885 please -3886 hundre -3887 tory -3888 rty -3889 stl -3890 Do -3891 distance -3892 dreadful -3893 observed -3894 tures -3895 mone -3896 How -3897 ude -3898 bly -3899 isp -3900 surprised -3901 - - -3902 essing -3903 stened -3904 desir -3905 vide -3906 stai -3907 impo -3908 ngly -3909 horr -3910 ws -3911 urp -3912 cit -3913 attend -3914 honour -3915 forwar -3916 concern -3917 occu -3918 Engl -3919 touch -3920 free -3921 laid -3922 andin -3923 Count -3924 draw -3925 obs -3926 oad -3927 secon -3928 ima -3929 espe -3930 rful -3931 fortunate -3932 thinking -3933 emo -3934 determi -3935 bright -3936 taking -3937 seeing -3938 arrive -3939 desire -3940 pare -3941 tog -3942 dec -3943 disapp -3944 beauti -3945 ttered -3946 women -3947 Arthur -3948 learn -3949 else -3950 belo -3951 hold -3952 kept -3953 prom -3954 solut -3955 thu -3956 lam -3957 esc -3958 eh -3959 deep -3960 mil -3961 ten -3962 lied -3963 aces -3964 eter -3965 determined -3966 opinion -3967 engage -3968 Carton -3969 conver -3970 secret -3971 stair -3972 appin -3973 icke -3974 gers -3975 ift -3976 enn -3977 direction -3978 chai -3979 enly -3980 lady -3981 orme -3982 ship -3983 kness -3984 pur -3985 fam -3986 severa -3987 marked -3988 added -3989 ountain -3990 inquir -3991 fied -3992 pap -3993 cap -3994 Hu -3995 His -3996 pleasant -3997 mention -3998 complete -3999 lustr -4000 longe -4001 inger -4002 ellin -4003 lia -4004 flo -4005 mpo -4006 roa -4007 vely -4008 hort -4009 Har -4010 erva -4011 known -4012 favour -4013 necess -4014 assur -4015 rov -4016 deligh -4017 passion -4018 confide -4019 Arthur -4020 week -4021 ility -4022 nche -4023 sca -4024 In -4025 confiden -4026 utt -4027 adv -4028 utiful -4029 throw -4030 onge -4031 otte -4032 dir -4033 appi -4034 ourt -4035 trac -4036 trati -4037 terri -4038 mise -4039 ppi -4040 Jim -4041 disapp -4042 clothes -4043 bra -4044 stic -4045 iled -4046 evi -4047 nec -4048 stay -4049 rf -4050 address -4051 approach -4052 ecessary -4053 letters -4054 miss -4055 low -4056 rked -4057 perf -4058 repeated -4059 listened -4060 conscious -4061 endeavou -4062 conver -4063 waiting -4064 figure -4065 begi -4066 impr -4067 Charl -4068 tire -4069 ousl -4070 usly -4071 reve -4072 pati -4073 ouch -4074 sir -4075 acqu -4076 ncl -4077 agi -4078 strai -4079 corne -4080 ayi -4081 une -4082 hur -4083 otic -4084 inge -4085 arri -4086 dri -4087 nex -4088 neighbour -4089 experienc -4090 favour -4091 eaking -4092 raise -4093 resol -4094 Charl -4095 etermined -4096 larg -4097 sorr -4098 ositi -4099 trust -4100 pic -4101 irl -4102 epl -4103 iri -4104 Mar -4105 dam -4106 boo -4107 sort -4108 race -4109 roke -4110 ical -4111 em -4112 straigh -4113 married -4114 childr -4115 ending -4116 common -4117 comple -4118 master -4119 piness -4120 sweet -4121 patien -4122 comple -4123 possess -4124 tear -4125 eless -4126 circ -4127 Mina -4128 icat -4129 iven -4130 boa -4131 vou -4132 hir -4133 difference -4134 ande -4135 rmin -4136 approach -4137 acquaint -4138 carrie -4139 cation -4140 grave -4141 glanc -4142 court -4143 ntent -4144 econd -4145 play -4146 chee -4147 tempt -4148 ict -4149 avo -4150 retched -4151 uall -4152 ield -4153 resp -4154 quir -4155 vis -4156 Cou -4157 dream -4158 throw -4159 undre -4160 actio -4161 hors -4162 adame -4163 nve -4164 bur -4165 dering -4166 passi -4167 means -4168 figure -4169 engage -4170 fla -4171 plie -4172 fair -4173 wee -4174 erio -4175 umb -4176 beginning -4177 importan -4178 stranger -4179 repeat -4180 address -4181 cture -4182 field -4183 cond -4184 ution -4185 ilit -4186 igge -4187 thank -4188 month -4189 sli -4190 Ben -4191 Ha -4192 airs -4193 acte -4194 posed -4195 admi -4196 tif -4197 eci -4198 rmi -4199 become -4200 concer -4201 purpos -4202 illa -4203 siti -4204 term -4205 sone -4206 duc -4207 tis -4208 longe -4209 bright -4210 easant -4211 bs -4212 advantage -4213 darkness -4214 confide -4215 reflect -4216 clothes -4217 book -4218 ented -4219 train -4220 ative -4221 assur -4222 pie -4223 discovered -4224 book -4225 trie -4226 offi -4227 oc -4228 nv -4229 mention -4230 effect -4231 murder -4232 stairs -4233 relat -4234 Lydia -4235 resp -4236 hair -4237 used -4238 caus -4239 loud -4240 lis -4241 rner -4242 For -4243 oach -4244 rdi -4245 elo -4246 exa -4247 tty -4248 waiting -4249 complet -4250 auti -4251 glad -4252 cold -4253 stead -4254 rais -4255 Coun -4256 assu -4257 lus -4258 eit -4259 murder -4260 proach -4261 human -4262 Pross -4263 dressed -4264 dropped -4265 counte -4266 Darnay -4267 four -4268 tted -4269 bar -4270 rmed -4271 ensi -4272 lls -4273 Pro -4274 hurr -4275 feet -4276 ntain -4277 held -4278 eight -4279 lance -4280 king -4281 tness -4282 ccu -4283 Hi -4284 ased -4285 test -4286 Monseigneur -4287 circu -4288 inquir -4289 arrive -4290 trust -4291 forme -4292 uni -4293 ets -4294 ybo -4295 reg -4296 Gardiner -4297 longer -4298 allowed -4299 reflect -4300 foot -4301 court -4302 shad -4303 Well -4304 eadi -4305 wall -4306 ates -4307 dep -4308 die -4309 det -4310 ela -4311 If -4312 mpe -4313 nco -4314 lso -4315 advan -4316 engag -4317 public -4318 esso -4319 erfe -4320 suf -4321 drop -4322 cra -4323 dropped -4324 nstantly -4325 anxious -4326 courage -4327 assure -4328 Col -4329 Now -4330 darkness -4331 ourselves -4332 confiden -4333 except -4334 simpl -4335 dition -4336 true -4337 sent -4338 piece -4339 grave -4340 ectly -4341 orde -4342 land -4343 ndl -4344 Sa -4345 countenance -4346 cla -4347 need -4348 ext -4349 rror -4350 confidence -4351 English -4352 outside -4353 astonish -4354 English -4355 ections -4356 journey -4357 listene -4358 probabl -4359 circu -4360 itati -4361 ainl -4362 prom -4363 hors -4364 orri -4365 lood -4366 tir -4367 dresse -4368 lighte -4369 doing -4370 heavy -4371 uar -4372 wled -4373 hum -4374 pain -4375 evidently -4376 whispered -4377 standing -4378 advan -4379 appr -4380 hon -4381 jo -4382 atched -4383 front -4384 tended -4385 desire -4386 held -4387 For -4388 ier -4389 rna -4390 God -4391 ... -4392 anxious -4393 journey -4394 oured -4395 touc -4396 ights -4397 swee -4398 soul -4399 lip -4400 ils -4401 Godalming -4402 llen -4403 play -4404 imme -4405 omes -4406 olo -4407 dne -4408 Wi -4409 bring -4410 effect -4411 extraordinary -4412 lear -4413 marke -4414 eepin -4415 tiona -4416 wall -4417 nity -4418 inly -4419 rdly -4420 bles -4421 selv -4422 mpt -4423 cio -4424 patient -4425 imagine -4426 neighbo -4427 gard -4428 fur -4429 ph -4430 seems -4431 tching -4432 inqui -4433 shadow -4434 chair -4435 church -4436 assure -4437 tabl -4438 ravel -4439 sing -4440 front -4441 chair -4442 gri -4443 whispered -4444 fortun -4445 station -4446 lection -4447 shadow -4448 connect -4449 occu -4450 arch -4451 fas -4452 ermi -4453 estr -4454 vill -4455 Bi -4456 mbl -4457 wonderful -4458 eathe -4459 sweet -4460 nable -4461 human -4462 tee -4463 explain -4464 England -4465 illiam -4466 itatio -4467 consci -4468 eratio -4469 rnes -4470 plan -4471 ndon -4472 gri -4473 nown -4474 circ -4475 eces -4476 rage -4477 obs -4478 boo -4479 venture -4480 ressing -4481 glance -4482 closed -4483 earne -4484 wild -4485 prove -4486 eping -4487 eward -4488 mpos -4489 ardl -4490 akes -4491 amin -4492 ably -4493 Netherfield -4494 offic -4495 disappoint -4496 wonderful -4497 nest -4498 iece -4499 rose -4500 impr -4501 astonish -4502 Charlotte -4503 mile -4504 After -4505 reac -4506 otion -4507 tree -4508 mist -4509 somewh -4510 wished -4511 horror -4512 spirits -4513 ban -4514 pra -4515 curio -4516 repeat -4517 lowing -4518 succe -4519 erious -4520 latter -4521 board -4522 bear -4523 eness -4524 desir -4525 hum -4526 magi -4527 nee -4528 cing -4529 ncer -4530 cien -4531 olli -4532 fal -4533 yed -4534 bar -4535 lsi -4536 val -4537 spirits -4538 possess -4539 neighbourhood -4540 evidently -4541 neighbou -4542 fright -4543 asleep -4544 Seward -4545 church -4546 moved -4547 worth -4548 itted -4549 story -4550 termi -4551 boys -4552 itti -4553 arde -4554 red -4555 itu -4556 tud -4557 bit -4558 hat -4559 reso -4560 hank -4561 evil -4562 slow -4563 dres -4564 ached -4565 Ca -4566 finge -4567 morrow -4568 stone -4569 proceed -4570 appoint -4571 explain -4572 mentio -4573 confid -4574 beyond -4575 ishment -4576 corn -4577 ceed -4578 udge -4579 eari -4580 utio -4581 eque -4582 esol -4583 hip -4584 yesterday -4585 remarked -4586 discovere -4587 agree -4588 proc -4589 imme -4590 offic -4591 fle -4592 necessary -4593 sand -4594 coa -4595 hate -4596 free -4597 enge -4598 school -4599 tering -4600 proce -4601 honou -4602 piece -4603 reache -4604 ais -4605 except -4606 rection -4607 nobody -4608 servant -4609 probab -4610 necess -4611 illed -4612 eart -4613 disa -4614 fact -4615 allo -4616 expl -4617 safe -4618 ecret -4619 scar -4620 impl -4621 alth -4622 appr -4623 grat -4624 cus -4625 sly -4626 ightened -4627 speaking -4628 probabl -4629 suspicio -4630 Monsieur -4631 ssa -4632 sua -4633 brigh -4634 above -4635 adin -4636 ecom -4637 interrupt -4638 Longbourn -4639 pping -4640 ttere -4641 antly -4642 month -4643 ket -4644 oud -4645 mas -4646 yesterday -4647 pointed -4648 addres -4649 showed -4650 inform -4651 eff -4652 onou -4653 interesting -4654 information -4655 William -4656 approac -4657 absolute -4658 acquain -4659 worth -4660 illing -4661 crowd -4662 pause -4663 expla -4664 office -4665 uncle -4666 sides -4667 onger -4668 desp -4669 ising -4670 ption -4671 becom -4672 thank -4673 stri -4674 ermin -4675 pit -4676 sub -4677 ras -4678 ray -4679 Da -4680 How -4681 adde -4682 orge -4683 onal -4684 etw -4685 acquaintance -4686 speaking -4687 questions -4688 agreeable -4689 iscovered -4690 common -4691 papers -4692 grav -4693 line -4694 Lady -4695 dded -4696 mer -4697 ndered -4698 waitin -4699 fright -4700 conte -4701 eau -4702 roc -4703 ben -4704 As -4705 particularly -4706 Netherfield -4707 questions -4708 sufficient -4709 Longbourn -4710 picture -4711 somebody -4712 although -4713 cket -4714 wild -4715 chee -4716 why -4717 ives -4718 arked -4719 plan -4720 ional -4721 clock -4722 hment -4723 serva -4724 pra -4725 - - - - -4726 - - - -4727 ication -4728 elieved -4729 company -4730 aiting -4731 eather -4732 shook -4733 glance -4734 mation -4735 solute -4736 nqui -4737 rope -4738 ill -4739 lse -4740 ori -4741 spa -4742 bse -4743 instantly -4744 gentlemen -4745 Monsieur -4746 itude -4747 iousl -4748 iling -4749 prov -4750 equen -4751 appro -4752 cloth -4753 eso -4754 belo -4755 nec -4756 foot -4757 mist -4758 hurried -4759 arrived -4760 rap -4761 Va -4762 disap -4763 figur -4764 breast -4765 formed -4766 cloth -4767 difficulty -4768 advantage -4769 picture -4770 written -4771 nearly -4772 horses -4773 grow -4774 meet -4775 conte -4776 treat -4777 erati -4778 stle -4779 week -4780 heri -4781 je -4782 ems -4783 sum -4784 drop -4785 cap -4786 eser -4787 presently -4788 appro -4789 admir -4790 versa -4791 simpl -4792 rim -4793 servant -4794 Stryver -4795 settle -4796 slight -4797 examin -4798 office -4799 osse -4800 oken -4801 tage -4802 unch -4803 andi -4804 ecre -4805 Cha -4806 La -4807 xc -4808 Go -4809 entur -4810 ocked -4811 bring -4812 rose -4813 ief -4814 presently -4815 gentlemen -4816 inging -4817 favou -4818 inform -4819 scene -4820 aged -4821 shu -4822 opin -4823 offe -4824 medi -4825 istr -4826 gue -4827 rof -4828 somebody -4829 although -4830 absolute -4831 consequen -4832 amongst -4833 allowed -4834 likely -4835 proceed -4836 travel -4837 success -4838 reflec -4839 lding -4840 anged -4841 dete -4842 Char -4843 hurr -4844 ndle -4845 tear -4846 stay -4847 oar -4848 rso -4849 disappoint -4850 ighten -4851 Madame -4852 cat -4853 sel -4854 hund -4855 rther -4856 anxi -4857 iment -4858 una -4859 ehi -4860 On -4861 ifi -4862 instantly -4863 further -4864 hatever -4865 arne -4866 utes -4867 cold -4868 mble -4869 Some -4870 rote -4871 fine -4872 .... -4873 mit -4874 attempt -4875 neither -4876 ourselve -4877 urselves -4878 streets -4879 connect -4880 Sherlock -4881 inning -4882 eserve -4883 journ -4884 pleasant -4885 afternoon -4886 situation -4887 crow -4888 estin -4889 ushed -4890 sett -4891 ordin -4892 ality -4893 ngu -4894 mag -4895 ppy -4896 disa -4897 sav -4898 easi -4899 ago -4900 aded -4901 enes -4902 shad -4903 Coun -4904 fra -4905 My -4906 nfi -4907 - - - - -4908 interesting -4909 comfortable -4910 astonishment -4911 information -4912 confidence -4913 master -4914 prepar -4915 mighty -4916 Charles -4917 repea -4918 ounded -4919 examin -4920 judg -4921 save -4922 proba -4923 lin -4924 ctu -4925 sso -4926 Fo -4927 wif -4928 ze -4929 De -4930 erry -4931 cide -4932 inki -4933 With -4934 sorr -4935 nob -4936 bear -4937 admi -4938 Cha -4939 distr -4940 trave -4941 bility -4942 compli -4943 trying -4944 attempt -4945 nstantl -4946 stantly -4947 proper -4948 llus -4949 ang -4950 abs -4951 lips -4952 vant -4953 fice -4954 pani -4955 nni -4956 ump -4957 itter -4958 written -4959 station -4960 engaged -4961 success -4962 Car -4963 desp -4964 spr -4965 imm -4966 onst -4967 desc -4968 sive -4969 difficulty -4970 tears -4971 effec -4972 mouth -4973 settle -4974 murde -4975 mpr -4976 sn -4977 recollect -4978 communicat -4979 whatever -4980 recollect -4981 condition -4982 finish -4983 Englis -4984 raised -4985 myster -4986 endin -4987 eakin -4988 ively -4989 ounds -4990 Some -4991 fool -4992 uence -4993 slo -4994 mmon -4995 ager -4996 anti -4997 aven -4998 olu -4999 kes -5000 rad -5001 Ch -5002 Jo -5003 ought -5004 alling -5005 agree -5006 prepar -5007 blin -5008 alt -5009 endi -5010 trem -5011 rush -5012 erie -5013 windows -5014 urder -5015 inqu -5016 ngers -5017 desc -5018 erous -5019 coa -5020 uis -5021 gui -5022 rob -5023 Pr -5024 atching -5025 journe -5026 talked -5027 months -5028 elig -5029 eles -5030 erly -5031 inv -5032 scar -5033 rela -5034 ctl -5035 uic -5036 Van -5037 appointment -5038 discovered -5039 remarkabl -5040 entirely -5041 satisfact -5042 finish -5043 judge -5044 allowe -5045 regard -5046 able -5047 catio -5048 repe -5049 bank -5050 dest -5051 imm -5052 cce -5053 ono -5054 ppr -5055 isf -5056 glo -5057 iate -5058 expl -5059 lean -5060 lut -5061 passage -5062 further -5063 exactly -5064 agreeabl -5065 astonis -5066 Charles -5067 waited -5068 broken -5069 pport -5070 driv -5071 fast -5072 claim -5073 mount -5074 roach -5075 bank -5076 glad -5077 aced -5078 ushe -5079 Mons -5080 fla -5081 eer -5082 nda -5083 twenty -5084 eaning -5085 drive -5086 proper -5087 refu -5088 cra -5089 nfor -5090 urch -5091 Lady -5092 somewhere -5093 proce -5094 steps -5095 lips -5096 nally -5097 mont -5098 uncle -5099 Se -5100 dir -5101 uns -5102 R -5103 remembered -5104 considered -5105 quickly -5106 mountain -5107 passage -5108 explai -5109 whilst -5110 ladies -5111 urious -5112 floor -5113 slight -5114 rested -5115 tained -5116 true -5117 met -5118 hon -5119 arry -5120 wid -5121 Str -5122 aunt -5123 Mon -5124 gli -5125 smi -5126 Har -5127 tment -5128 tairs -5129 obody -5130 ioned -5131 Mary -5132 roduc -5133 arms -5134 osing -5135 indi -5136 susp -5137 Mons -5138 ize -5139 oni -5140 St -5141 rred -5142 path -5143 knes -5144 wro -5145 mas -5146 quen -5147 soul -5148 es -5149 ees -5150 All -5151 ian -5152 motion -5153 exact -5154 unning -5155 sn -5156 neighbourhood -5157 disappear -5158 influence -5159 forgot -5160 oppo -5161 lying -5162 offe -5163 pause -5164 bilit -5165 choo -5166 iful -5167 ants -5168 ton -5169 rus -5170 pie -5171 tles -5172 mina -5173 requ -5174 egi -5175 finished -5176 ordinary -5177 mistake -5178 ats -5179 somewhere -5180 interrupt -5181 following -5182 absolutely -5183 pper -5184 urio -5185 entr -5186 yard -5187 olve -5188 settl -5189 travel -5190 uction -5191 curre -5192 simp -5193 shar -5194 brin -5195 extr -5196 disp -5197 ston -5198 proac -5199 huma -5200 ora -5201 tom -5202 dep -5203 dit -5204 Cou -5205 twenty -5206 posses -5207 grow -5208 hid -5209 Here -5210 ners -5211 prob -5212 mid -5213 mountain -5214 suspicio -5215 suspicion -5216 earnest -5217 turning -5218 five -5219 Every -5220 shore -5221 immed -5222 none -5223 trus -5224 stud -5225 mined -5226 tit -5227 ryi -5228 contr -5229 lectio -5230 shado -5231 scr -5232 alit -5233 Cart -5234 ips -5235 lue -5236 My -5237 astonishment -5238 particularly -5239 remarkabl -5240 afternoon -5241 situation -5242 Charlotte -5243 earnest -5244 procee -5245 lighted -5246 stretch -5247 shut -5248 mark -5249 crowd -5250 prob -5251 five -5252 eans -5253 vil -5254 remembered -5255 disposition -5256 communicat -5257 cross -5258 fixed -5259 ressin -5260 careful -5261 believed -5262 greeable -5263 rthe -5264 Your -5265 qua -5266 curiosity -5267 advantag -5268 reckoned -5269 paus -5270 judge -5271 drive -5272 cin -5273 school -5274 ightene -5275 ghtened -5276 changed -5277 dinner -5278 mistake -5279 writing -5280 Watson -5281 opp -5282 easy -5283 oura -5284 nei -5285 dev -5286 enl -5287 det -5288 dently -5289 stinct -5290 force -5291 colour -5292 eeding -5293 ndere -5294 tchin -5295 mouth -5296 contr -5297 lecti -5298 lim -5299 gat -5300 disappoin -5301 influence -5302 meet -5303 safe -5304 cult -5305 noticed -5306 stretch -5307 company -5308 cel -5309 consequence -5310 excellent -5311 slowly -5312 manage -5313 length -5314 istress -5315 regard -5316 Madame -5317 titude -5318 frigh -5319 awful -5320 woods -5321 struct -5322 descri -5323 churc -5324 mine -5325 usion -5326 earl -5327 fron -5328 scen -5329 acqu -5330 enty -5331 ides -5332 cho -5333 ncle -5334 irt -5335 rue -5336 lame -5337 ooks -5338 wake -5339 adm -5340 chool -5341 miss -5342 trea -5343 sions -5344 au -5345 cheerful -5346 reached -5347 destroy -5348 pocket -5349 ranger -5350 commen -5351 Lucie -5352 pictur -5353 forget -5354 llowing -5355 impress -5356 disturb -5357 breast -5358 llar -5359 fiel -5360 inis -5361 Now -5362 refo -5363 fig -5364 usti -5365 dete -5366 ply -5367 iat -5368 pti -5369 mur -5370 uen -5371 distress -5372 schoo -5373 wors -5374 sente -5375 lonel -5376 guard -5377 grat -5378 admir -5379 nar -5380 ecu -5381 gett -5382 curi -5383 thu -5384 sim -5385 proc -5386 ages -5387 disp -5388 erce -5389 anging -5390 excep -5391 hanged -5392 esides -5393 trees -5394 meant -5395 vic -5396 Col -5397 occurre -5398 disturb -5399 solutely -5400 intent -5401 relati -5402 dare -5403 reli -5404 seat -5405 trave -5406 din -5407 judg -5408 tful -5409 cret -5410 isti -5411 ativ -5412 susp -5413 mate -5414 lor -5415 igg -5416 abi -5417 extraordinary -5418 curiosity -5419 understood -5420 condition -5421 manage -5422 commo -5423 dinner -5424 tioned -5425 distin -5426 relati -5427 infor -5428 destr -5429 sett -5430 extr -5431 finished -5432 cheerful -5433 believed -5434 struction -5435 pecta -5436 yster -5437 fair -5438 lowin -5439 anced -5440 exam -5441 Lord -5442 fur -5443 wl -5444 satisfaction -5445 reading -5446 sorrow -5447 solemn -5448 despair -5449 descri -5450 knowed -5451 emp -5452 mur -5453 coll -5454 udg -5455 eff -5456 rle -5457 pleasan -5458 handsome -5459 sisters -5460 impress -5461 nishment -5462 relation -5463 resolved -5464 sufficient -5465 ritten -5466 impos -5467 placed -5468 conse -5469 nguish -5470 arden -5471 quire -5472 distr -5473 John -5474 tua -5475 ads -5476 Si -5477 gged -5478 ench -5479 eng -5480 remo -5481 tnes -5482 uct -5483 pte -5484 Mo -5485 Oh -5486 excellent -5487 disappear -5488 sensation -5489 recollec -5490 scarcely -5491 inished -5492 candle -5493 showe -5494 awful -5495 rived -5496 ardin -5497 scene -5498 ighty -5499 unche -5500 tran -5501 harm -5502 crow -5503 enta -5504 prec -5505 nste -5506 quar -5507 eepi -5508 cord -5509 All -5510 iety -5511 acr -5512 rms -5513 truth -5514 rowing -5515 icatio -5516 lieved -5517 search -5518 intent -5519 pursu -5520 witness -5521 tou -5522 ddle -5523 bled -5524 oure -5525 isin -5526 rehe -5527 duce -5528 disappointment -5529 ssage -5530 morro -5531 fine -5532 apers -5533 force -5534 nform -5535 uncl -5536 considered -5537 running -5538 attemp -5539 wretch -5540 ourselv -5541 destroy -5542 connec -5543 lations -5544 occur -5545 writte -5546 cheer -5547 ighted -5548 cloud -5549 pic -5550 urre -5551 slow -5552 mid -5553 bad -5554 piec -5555 big -5556 nqu -5557 Hu -5558 occurred -5559 important -5560 relation -5561 stai -5562 terin -5563 ounde -5564 join -5565 rdere -5566 rush -5567 sence -5568 lemen -5569 aunt -5570 Car -5571 obl -5572 remarkable -5573 altogether -5574 consequen -5575 connection -5576 September -5577 indifferen -5578 passion -5579 distress -5580 kindness -5581 Cat -5582 tude -5583 unta -5584 hill -5585 ordi -5586 inal -5587 qual -5588 boat -5589 ued -5590 inding -5591 candle -5592 forget -5593 roduce -5594 leased -5595 prope -5596 ishing -5597 sligh -5598 memor -5599 exami -5600 Kurtz -5601 consideration -5602 pocket -5603 former -5604 doctor -5605 science -5606 readi -5607 andle -5608 ester -5609 conse -5610 istin -5611 ormed -5612 oard -5613 elat -5614 scri -5615 nty -5616 isa -5617 clu -5618 ago -5619 het -5620 frai -5621 non -5622 ubli -5623 pend -5624 weet -5625 absolutely -5626 frighten -5627 presented -5628 considere -5629 onsidered -5630 presence -5631 resolved -5632 running -5633 forehead -5634 despair -5635 sympath -5636 travell -5637 quarter -5638 fortune -5639 fetch -5640 sorrow -5641 length -5642 threw -5643 aining -5644 early -5645 doctor -5646 wort -5647 kill -5648 likel -5649 thir -5650 floo -5651 prev -5652 ieved -5653 fresh -5654 spec -5655 lness -5656 ified -5657 awyer -5658 opi -5659 ssu -5660 Ye -5661 interested -5662 reflection -5663 suspicion -5664 misfortune -5665 walking -5666 furthe -5667 escape -5668 statio -5669 placed -5670 mistak -5671 succes -5672 public -5673 olution -5674 fini -5675 ilin -5676 anci -5677 antl -5678 exact -5679 hung -5680 sters -5681 King -5682 duke -5683 leg -5684 why -5685 iar -5686 Ja -5687 stair -5688 righte -5689 custom -5690 solemn -5691 esting -5692 alli -5693 hook -5694 essa -5695 blu -5696 tled -5697 Man -5698 sy -5699 If -5700 ckl -5701 manners -5702 serious -5703 greatest -5704 innocen -5705 rightened -5706 handsome -5707 interrup -5708 hanging -5709 eatures -5710 resting -5711 serious -5712 watche -5713 living -5714 special -5715 quarter -5716 eckoned -5717 Tellson -5718 lower -5719 telli -5720 forc -5721 hilst -5722 prof -5723 habit -5724 clot -5725 yes -5726 shes -5727 dog -5728 igne -5729 Lord -5730 altogether -5731 understood -5732 profession -5733 misfortune -5734 sterda -5735 atchin -5736 forge -5737 maste -5738 guard -5739 ffered -5740 prepa -5741 bless -5742 pal -5743 satisfact -5744 boar -5745 kill -5746 raid -5747 lies -5748 prin -5749 pal -5750 sole -5751 floor -5752 prec -5753 tless -5754 Fr -5755 oce -5756 ndu -5757 dventure -5758 curious -5759 settled -5760 watching -5761 disappea -5762 isappear -5763 recover -5764 peculiar -5765 struggl -5766 passag -5767 ordinar -5768 ceeding -5769 oppo -5770 lass -5771 mmed -5772 smo -5773 oper -5774 ?’ -5775 repa -5776 Mon -5777 V -5778 knows -5779 press -5780 deser -5781 places -5782 cident -5783 tlemen -5784 intention -5785 greatest -5786 kindness -5787 expressed -5788 truth -5789 lock -5790 impos -5791 waite -5792 cting -5793 ingin -5794 poli -5795 tione -5796 oving -5797 remo -5798 loved -5799 ealth -5800 llu -5801 cli -5802 vio -5803 sic -5804 Good -5805 Will -5806 dies -5807 fool -5808 son -5809 cia -5810 ix -5811 unfortunate -5812 imagination -5813 sensation -5814 ourselves -5815 profession -5816 compliment -5817 middle -5818 curious -5819 righten -5820 search -5821 bitter -5822 assist -5823 degree -5824 finis -5825 laying -5826 wrong -5827 glass -5828 erate -5829 cheer -5830 umber -5831 tone -5832 dance -5833 summ -5834 wort -5835 erne -5836 olde -5837 ivel -5838 mee -5839 ione -5840 sum -5841 para -5842 rev -5843 unge -5844 ipp -5845 Not -5846 rip -5847 urd -5848 rlo -5849 yb -5850 frighten -5851 resented -5852 watched -5853 mountai -5854 profess -5855 perceive -5856 ei -5857 shar -5858 aris -5859 repe -5860 llia -5861 moti -5862 rity -5863 reli -5864 cast -5865 stud -5866 boa -5867 exclaimed -5868 forehead -5869 probably -5870 describe -5871 struggle -5872 influenc -5873 convinced -5874 innin -5875 stake -5876 As -5877 begun -5878 story -5879 gement -5880 steps -5881 convi -5882 garden -5883 finishe -5884 rdinary -5885 places -5886 fashion -5887 resolut -5888 resolv -5889 embl -5890 join -5891 save -5892 succ -5893 ouri -5894 arna -5895 abs -5896 wretched -5897 prevent -5898 writing -5899 passing -5900 greater -5901 describe -5902 Quincey -5903 ipped -5904 Will -5905 soft -5906 cient -5907 tress -5908 icate -5909 eigne -5910 eaven -5911 Ba -5912 frightened -5913 mentioned -5914 exclaimed -5915 ided -5916 ller -5917 wful -5918 saf -5919 rker -5920 eign -5921 remarkable -5922 frightened -5923 reflection -5924 indifferen -5925 readin -5926 ghtful -5927 strain -5928 horri -5929 equent -5930 unfortunate -5931 wretched -5932 watching -5933 adventure -5934 earnes -5935 ndering -5936 excite -5937 ssenger -5938 sympath -5939 structi -5940 cottage -5941 rmation -5942 unded -5943 self -5944 trac -5945 iding -5946 arra -5947 wear -5948 oods -5949 uil -5950 .. -5951 lp -5952 ume -5953 nob -5954 afe -5955 nth -5956 Ar -5957 distinct -5958 struggle -5959 frequent -5960 schoo -5961 escape -5962 offer -5963 ancing -5964 attac -5965 erness -5966 prehen -5967 cry -5968 pair -5969 cati -5970 fix -5971 mmen -5972 ells -5973 ride -5974 Lad -5975 Hi -5976 allin -5977 deal -5978 easy -5979 anded -5980 reven -5981 trav -5982 bor -5983 sci -5984 On -5985 conclusion -5986 occasional -5987 importance -5988 convinced -5989 conduct -5990 effort -5991 Ever -5992 Eve -5993 riti -5994 ardi -5995 cks -5996 cut -5997 lu -5998 otherwise -5999 Sometimes -6000 conditio -6001 locked -6002 escap -6003 sorry -6004 entire -6005 assist -6006 peace -6007 racti -6008 heads -6009 temp -6010 mout -6011 stren -6012 aised -6013 ainte -6014 missi -6015 arter -6016 shu -6017 spr -6018 oac -6019 vice -6020 ixed -6021 olen -6022 Wil -6023 shoo -6024 conduct -6025 leaving -6026 distres -6027 fingers -6028 touched -6029 mination -6030 persuade -6031 Tellson -6032 rdl -6033 stretc -6034 recover -6035 witness -6036 profess -6037 ough -6038 aited -6039 icati -6040 unnin -6041 With -6042 agre -6043 gular -6044 bound -6045 taine -6046 ailed -6047 pursu -6048 itud -6049 asti -6050 mati -6051 danc -6052 yond -6053 pul -6054 eager -6055 After -6056 rushed -6057 charge -6058 isters -6059 lected -6060 bitter -6061 eparat -6062 excite -6063 solved -6064 xe -6065 erw -6066 sk -6067 ewa -6068 intention -6069 resolution -6070 acquainted -6071 stly -6072 gest -6073 rses -6074 ulat -6075 acke -6076 mpli -6077 disappea -6078 tence -6079 spee -6080 fetch -6081 mere -6082 note -6083 ities -6084 tinct -6085 servi -6086 nfo -6087 eur -6088 cheerfu -6089 introduc -6090 resolut -6091 solution -6092 cottage -6093 py -6094 conclusion -6095 disappeared -6096 connection -6097 knowing -6098 pleased -6099 claimed -6100 willing -6101 affair -6102 distin -6103 beauty -6104 prepare -6105 mission -6106 attered -6107 destro -6108 cousin -6109 sented -6110 faint -6111 watche -6112 parent -6113 ruth -6114 andl -6115 fan -6116 fish -6117 xpla -6118 eein -6119 tine -6120 ble -6121 rton -6122 issi -6123 nstr -6124 roba -6125 ety -6126 miles -6127 fanc -6128 siona -6129 sole -6130 drew -6131 weak -6132 ncing -6133 minat -6134 pray -6135 forma -6136 reque -6137 pil -6138 mel -6139 urc -6140 egr -6141 lead -6142 cate -6143 ific -6144 spen -6145 affectionate -6146 inv -6147 ntment -6148 manag -6149 utting -6150 makes -6151 dinne -6152 below -6153 steam -6154 dispos -6155 stream -6156 breas -6157 compos -6158 equal -6159 understanding -6160 peculiar -6161 persuade -6162 mportant -6163 arrange -6164 prevent -6165 George -6166 distur -6167 indness -6168 misery -6169 Colonel -6170 tired -6171 cking -6172 occur -6173 unco -6174 tract -6175 sieur -6176 ..... -6177 wise -6178 jud -6179 nall -6180 ayed -6181 ril -6182 sch -6183 Min -6184 lth -6185 dur -6186 wors -6187 emor -6188 mpre -6189 interrupted -6190 difference -6191 resolution -6192 heads -6193 sharp -6194 shore -6195 lengt -6196 relie -6197 bound -6198 regar -6199 vement -6200 accomp -6201 Every -6202 ainted -6203 event -6204 rally -6205 exac -6206 ende -6207 glo -6208 sus -6209 reserve -6210 latter -6211 impres -6212 nishmen -6213 Sawyer -6214 curr -6215 Why -6216 sof -6217 oose -6218 rees -6219 wre -6220 ford -6221 Art -6222 rts -6223 rod -6224 ths -6225 changed -6226 prepare -6227 describ -6228 engeance -6229 arrange -6230 drawing -6231 tinguish -6232 Sa -6233 Presently -6234 existence -6235 suffering -6236 relations -6237 Sherlock -6238 past -6239 eeded -6240 wine -6241 moon -6242 memor -6243 strea -6244 parat -6245 convi -6246 solve -6247 raft -6248 nic -6249 pol -6250 tac -6251 disappeared -6252 disposition -6253 consequence -6254 adventure -6255 professio -6256 agreeable -6257 ewhere -6258 rdered -6259 llowin -6260 smoke -6261 depend -6262 excit -6263 ficien -6264 study -6265 aves -6266 spa -6267 poli -6268 rter -6269 colo -6270 ervi -6271 consideration -6272 Eve -6273 yon -6274 affectionate -6275 Hertfordshire -6276 conclu -6277 charge -6278 contemp -6279 caught -6280 witnes -6281 Nothing -6282 accomp -6283 beat -6284 cloud -6285 purs -6286 iles -6287 twe -6288 lica -6289 mili -6290 unds -6291 pol -6292 prev -6293 eavi -6294 empl -6295 hot -6296 nvi -6297 disappointment -6298 compliment -6299 frighte -6300 presente -6301 walking -6302 confess -6303 grateful -6304 Colonel -6305 violen -6306 ements -6307 occup -6308 nel -6309 ef -6310 emi -6311 ippe -6312 dare -6313 dine -6314 adventur -6315 passenger -6316 frequent -6317 ompliment -6318 soli -6319 cros -6320 offer -6321 willi -6322 pered -6323 elief -6324 danc -6325 olved -6326 bless -6327 rated -6328 speci -6329 trade -6330 unk -6331 yer -6332 existence -6333 acknowledg -6334 expressed -6335 suffering -6336 wander -6337 passio -6338 laughed -6339 extreme -6340 supper -6341 stream -6342 confess -6343 arrang -6344 twi -6345 tual -6346 sity -6347 nage -6348 thir -6349 eaki -6350 inne -6351 Som -6352 arre -6353 astl -6354 rack -6355 ero -6356 ehe -6357 uin -6358 ugg -6359 sorro -6360 solem -6361 descen -6362 castle -6363 strea -6364 confe -6365 descr -6366 rw -6367 preci -6368 dentl -6369 quar -6370 died -6371 stinc -6372 ssive -6373 horri -6374 knowing -6375 handsom -6376 advance -6377 impressi -6378 service -6379 perceiv -6380 Marquis -6381 ccur -6382 itch -6383 pea -6384 erou -6385 cial -6386 rade -6387 Joe -6388 eq -6389 usa -6390 sav -6391 ngr -6392 satisfaction -6393 importance -6394 expectation -6395 sensatio -6396 impressi -6397 complimen -6398 perceive -6399 vidence -6400 simple -6401 violen -6402 despai -6403 spoken -6404 advance -6405 sympat -6406 quarte -6407 persua -6408 readi -6409 candl -6410 glass -6411 ouched -6412 maybe -6413 eaving -6414 madam -6415 titud -6416 ander -6417 lead -6418 laced -6419 gras -6420 awfu -6421 smok -6422 steam -6423 crim -6424 escen -6425 super -6426 equal -6427 ffici -6428 past -6429 hore -6430 pit -6431 trad -6432 seat -6433 dows -6434 Pa -6435 rif -6436 tta -6437 wro -6438 Str -6439 tne -6440 possession -6441 admiration -6442 eepe -6443 tmen -6444 loc -6445 ropp -6446 heer -6447 died -6448 raft -6449 laughed -6450 extreme -6451 understo -6452 horribl -6453 suffere -6454 expresse -6455 xpressed -6456 owers -6457 wrong -6458 knee -6459 At -6460 aye -6461 garden -6462 seven -6463 faces -6464 bitte -6465 super -6466 cover -6467 scribe -6468 publi -6469 positi -6470 seriou -6471 imagina -6472 colour -6473 instru -6474 murdere -6475 eads -6476 lyin -6477 six -6478 san -6479 unco -6480 edly -6481 mpat -6482 mper -6483 teps -6484 osin -6485 post -6486 unn -6487 wid -6488 imagination -6489 lightning -6490 represent -6491 steamboat -6492 innocent -6493 urrie -6494 colo -6495 occup -6496 ship -6497 lun -6498 rink -6499 ashi -6500 ares -6501 shir -6502 summ -6503 prof -6504 occasional -6505 interested -6506 endeavoured -6507 exceedingly -6508 plain -6509 carry -6510 docto -6511 slo -6512 ala -6513 hid -6514 ovi -6515 bt -6516 dressed -6517 wretche -6518 collecti -6519 tolerabl -6520 suggest -6521 Sometimes -6522 collection -6523 curiou -6524 conduc -6525 meaning -6526 easily -6527 content -6528 sappear -6529 recove -6530 ightful -6531 Street -6532 health -6533 iration -6534 employ -6535 Chapter -6536 rowin -6537 rting -6538 cessi -6539 chief -6540 obli -6541 upper -6542 Here -6543 reve -6544 peace -6545 drew -6546 uste -6547 tall -6548 resu -6549 mour -6550 city -6551 thus -6552 tor -6553 ldi -6554 non -6555 asy -6556 epi -6557 Ju -6558 lag -6559 filled -6560 lived -6561 assing -6562 oblig -6563 noble -6564 blind -6565 hain -6566 omew -6567 sons -6568 Come -6569 Tell -6570 outs -6571 lour -6572 egre -6573 Fo -6574 evidence -6575 behaviour -6576 garde -6577 ishin -6578 sick -6579 acti -6580 understandin -6581 nderstanding -6582 meaning -6583 impatien -6584 stretche -6585 contemp -6586 besides -6587 accident -6588 horrible -6589 examine -6590 resente -6591 esented -6592 couple -6593 advanc -6594 examine -6595 dispos -6596 consci -6597 service -6598 profes -6599 kni -6600 box -6601 age -6602 ris -6603 rney -6604 bor -6605 mend -6606 ski -6607 sla -6608 fig -6609 Ki -6610 lusion -6611 accept -6612 restin -6613 gloom -6614 dance -6615 intend -6616 Jerry -6617 trut -6618 sharp -6619 inary -6620 news -6621 nance -6622 dic -6623 yar -6624 il -6625 rsu -6626 important -6627 nterrupted -6628 relations -6629 behaviour -6630 ngli -6631 mere -6632 moo -6633 oble -6634 K -6635 Ev -6636 sim -6637 dev -6638 shoulders -6639 generally -6640 introduc -6641 intentio -6642 horrible -6643 surround -6644 acquainte -6645 cquainted -6646 ignoran -6647 ttering -6648 trembl -6649 throat -6650 ringing -6651 preven -6652 memory -6653 suggest -6654 writt -6655 indin -6656 ghted -6657 faint -6658 blue -6659 memo -6660 clou -6661 bles -6662 thus -6663 Ever -6664 unat -6665 hung -6666 sch -6667 amed -6668 tchi -6669 ongs -6670 rass -6671 ffor -6672 ghty -6673 fici -6674 rank -6675 joy -6676 fli -6677 apt -6678 rh -6679 ignoran -6680 promised -6681 content -6682 pleased -6683 imagina -6684 surround -6685 assured -6686 equently -6687 murdere -6688 persuad -6689 innocent -6690 wander -6691 trembl -6692 prove -6693 touche -6694 earest -6695 utter -6696 itizen -6697 Becky -6698 rb -6699 Wa -6700 gar -6701 isto -6702 deal -6703 uche -6704 edia -6705 cell -6706 One -6707 gat -6708 olly -6709 perat -6710 entr -6711 ainin -6712 laces -6713 ickly -6714 ollar -6715 fered -6716 nstru -6717 Becky -6718 rsa -6719 imi -6720 rer -6721 Wil -6722 rma -6723 One -6724 unconscious -6725 shoulders -6726 interrupte -6727 impression -6728 especially -6729 compassion -6730 rushed -6731 extrem -6732 regular -6733 gather -6734 sibilit -6735 weather -6736 compos -6737 action -6738 dollar -6739 tening -6740 rrying -6741 fancy -6742 searc -6743 wrote -6744 tional -6745 assis -6746 inally -6747 devil -6748 health -6749 diat -6750 reci -6751 note -6752 inci -6753 ucti -6754 Le -6755 rtl -6756 bil -6757 forgotten -6758 stretche -6759 distinct -6760 moonlight -6761 Thatcher -6762 putting -6763 various -6764 quietly -6765 instead -6766 covered -6767 conceal -6768 sibility -6769 eager -6770 fetc -6771 mbled -6772 requ -6773 confe -6774 tenan -6775 rra -6776 oys -6777 joi -6778 tole -6779 hool -6780 soft -6781 eade -6782 chu -6783 ssen -6784 stur -6785 aile -6786 Ch -6787 ana -6788 ncy -6789 nif -6790 ode -6791 bin -6792 cts -6793 La -6794 garde -6795 ceeded -6796 respon -6797 marry -6798 Perhaps -6799 contra -6800 minatio -6801 Jacques -6802 hang -6803 iddle -6804 relie -6805 guar -6806 rding -6807 astle -6808 scend -6809 espon -6810 unit -6811 ovel -6812 nary -6813 orte -6814 lie -6815 olic -6816 viou -6817 resi -6818 Som -6819 rol -6820 moo -6821 otherwise -6822 complimen -6823 telling -6824 courage -6825 peculia -6826 intellig -6827 Jacques -6828 roll -6829 titu -6830 soli -6831 cand -6832 hast -6833 lest -6834 ntra -6835 gine -6836 bou -6837 moon -6838 interrupted -6839 endeavoured -6840 exceedingly -6841 admiration -6842 impatien -6843 encourag -6844 accident -6845 comprehen -6846 Pemberley -6847 fallen -6848 hurry -6849 simple -6850 drink -6851 ecting -6852 shout -6853 ecover -6854 employ -6855 fancy -6856 layin -6857 arrow -6858 atchi -6859 rushe -6860 enti -6861 Georg -6862 ctive -6863 unger -6864 lid -6865 Ro -6866 amb -6867 siv -6868 belong -6869 accept -6870 produc -6871 depend -6872 officer -6873 single -6874 cottag -6875 pursue -6876 visitor -6877 law -6878 vert -6879 lowi -6880 sona -6881 arle -6882 eedi -6883 mons -6884 xed -6885 neck -6886 send -6887 Tell -6888 rench -6889 smoke -6890 accu -6891 lated -6892 moti -6893 grati -6894 abso -6895 perce -6896 fan -6897 fus -6898 orc -6899 eps -6900 rvi -6901 rto -6902 lying -6903 pening -6904 eating -6905 excla -6906 coach -6907 fresh -6908 ateful -6909 scarce -6910 eat -6911 spee -6912 arme -6913 xact -6914 oine -6915 eeks -6916 ckle -6917 anta -6918 rtin -6919 inio -6920 King -6921 chie -6922 eali -6923 fie -6924 riva -6925 guis -6926 bb -6927 produce -6928 fashion -6929 ccording -6930 increas -6931 stepped -6932 private -6933 attendan -6934 society -6935 dollars -6936 pee -6937 rut -6938 agr -6939 awn -6940 recollection -6941 generally -6942 companions -6943 mountains -6944 moonlight -6945 accomplish -6946 represent -6947 daughters -6948 listening -6949 stretched -6950 scription -6951 attendant -6952 connected -6953 convenien -6954 compass -6955 Long -6956 cent -6957 dding -6958 wron -6959 icked -6960 ttled -6961 boun -6962 hill -6963 coff -6964 peac -6965 spen -6966 iced -6967 tick -6968 Just -6969 morr -6970 loom -6971 Kin -6972 crim -6973 sick -6974 esto -6975 nie -6976 Dr -6977 thy -6978 explanation -6979 impression -6980 acknowledge -6981 specta -6982 worse -6983 nounce -6984 thick -6985 enjoy -6986 remov -6987 grati -6988 looks -6989 arp -6990 smo -6991 aci -6992 understanding -6993 Hertfordshire -6994 expectation -6995 ancy -6996 rall -6997 tail -6998 eded -6999 senc -7000 ifie -7001 def -7002 aver -7003 earn -7004 locke -7005 hadn -7006 trem -7007 duced -7008 issed -7009 Fran -7010 warm -7011 cepti -7012 ousin -7013 dog -7014 Su -7015 Man -7016 hanging -7017 learned -7018 officer -7019 separat -7020 somewhat -7021 supposed -7022 frequen -7023 mpliment -7024 fallen -7025 meeting -7026 conten -7027 descend -7028 ovement -7029 depart -7030 fil -7031 joi -7032 vari -7033 dged -7034 weak -7035 erti -7036 ppre -7037 reed -7038 warm -7039 pray -7040 aun -7041 fix -7042 mbling -7043 later -7044 olding -7045 ionate -7046 rising -7047 efully -7048 couple -7049 girls -7050 claime -7051 laimed -7052 gather -7053 strik -7054 essive -7055 rooms -7056 attere -7057 preci -7058 depart -7059 reasonabl -7060 supposed -7061 sympathy -7062 suffered -7063 gratitude -7064 entre -7065 hurry -7066 gold -7067 blow -7068 pale -7069 civil -7070 gloom -7071 sake -7072 monst -7073 spar -7074 egu -7075 bad -7076 Bi -7077 forgotten -7078 passenger -7079 proceeded -7080 instrument -7081 intelligen -7082 urth -7083 ctin -7084 eved -7085 iner -7086 lige -7087 icio -7088 tore -7089 sf -7090 Cat -7091 possession -7092 distinguish -7093 understandi -7094 pointed -7095 sleeping -7096 entreat -7097 decide -7098 number -7099 produce -7100 conceal -7101 confes -7102 coffin -7103 Morris -7104 police -7105 worked -7106 Sunday -7107 miles -7108 catch -7109 French -7110 party -7111 mitted -7112 charg -7113 quired -7114 supper -7115 nlight -7116 trans -7117 dingly -7118 elled -7119 drink -7120 slip -7121 acce -7122 easi -7123 vari -7124 fain -7125 resu -7126 These -7127 lines -7128 ancin -7129 noble -7130 post -7131 kil -7132 umbl -7133 cli -7134 pens -7135 oldi -7136 rday -7137 une -7138 ickl -7139 itor -7140 idd -7141 agg -7142 ado -7143 Com -7144 ots -7145 idi -7146 ods -7147 big -7148 loa -7149 ashe -7150 pera -7151 rmer -7152 clin -7153 liar -7154 .— -7155 exe -7156 inds -7157 ores -7158 creatures -7159 position -7160 promised -7161 carefully -7162 favourite -7163 according -7164 somewhat -7165 disturbed -7166 goodness -7167 perceived -7168 fill -7169 bling -7170 esca -7171 insi -7172 shire -7173 error -7174 lift -7175 rese -7176 devi -7177 rooms -7178 Po -7179 ech -7180 lte -7181 gua -7182 iva -7183 tively -7184 joined -7185 arious -7186 stead -7187 sional -7188 nstead -7189 incre -7190 minati -7191 single -7192 habit -7193 pride -7194 sensib -7195 nishme -7196 melancholy -7197 favourite -7198 possibilit -7199 steamboat -7200 holding -7201 roduced -7202 remove -7203 servic -7204 fulness -7205 rant -7206 ifte -7207 ingi -7208 olit -7209 suit -7210 beat -7211 ivil -7212 Fran -7213 mmer -7214 olla -7215 cure -7216 onis -7217 atta -7218 view -7219 solv -7220 lott -7221 irat -7222 box -7223 zed -7224 esh -7225 dat -7226 ountains -7227 matters -7228 language -7229 prepared -7230 occupied -7231 uttin -7232 rever -7233 eage -7234 pull -7235 Where -7236 shak -7237 const -7238 unted -7239 prote -7240 assin -7241 adve -7242 ffair -7243 kiss -7244 perio -7245 refer -7246 sensi -7247 equa -7248 Let -7249 sag -7250 hut -7251 cum -7252 igu -7253 cea -7254 assa -7255 rist -7256 !’ -7257 anch -7258 loor -7259 angu -7260 lter -7261 eati -7262 pted -7263 itie -7264 ski -7265 mysterious -7266 acknowledg -7267 instrument -7268 compassion -7269 tually -7270 houses -7271 giving -7272 contra -7273 const -7274 ouring -7275 quence -7276 suppe -7277 loved -7278 parted -7279 propo -7280 kle -7281 naturally -7282 dangerous -7283 occupied -7284 attendan -7285 stening -7286 However -7287 French -7288 speech -7289 unhapp -7290 terror -7291 counter -7292 admira -7293 capable -7294 tured -7295 arran -7296 itate -7297 easin -7298 extra -7299 anyw -7300 usted -7301 urely -7302 inni -7303 scap -7304 uses -7305 sult -7306 oped -7307 epen -7308 lant -7309 spar -7310 herl -7311 raf -7312 twi -7313 law -7314 bab -7315 nct -7316 plo -7317 However -7318 holding -7319 hearing -7320 persona -7321 obliged -7322 officers -7323 midnight -7324 solation -7325 anxiety -7326 visitor -7327 blu -7328 listening -7329 stretched -7330 recollecti -7331 approached -7332 dangerous -7333 breathing -7334 frequently -7335 lightning -7336 wande -7337 spread -7338 exist -7339 number -7340 shock -7341 train -7342 pursue -7343 fill -7344 wers -7345 acce -7346 smok -7347 asin -7348 rime -7349 mai -7350 idin -7351 catch -7352 fift -7353 ument -7354 eting -7355 grew -7356 treme -7357 wide -7358 wear -7359 trai -7360 coach -7361 Geor -7362 utely -7363 ird -7364 ghe -7365 nju -7366 filled -7367 tending -7368 opening -7369 violent -7370 merely -7371 agement -7372 spring -7373 wishes -7374 mystery -7375 restor -7376 sugges -7377 ensible -7378 island -7379 citizen -7380 cut -7381 duct -7382 dese -7383 edit -7384 abso -7385 cred -7386 ram -7387 wre -7388 eta -7389 uty -7390 lum -7391 vat -7392 Se -7393 melancholy -7394 companions -7395 especially -7396 possibility -7397 intelligen -7398 addresse -7399 daylight -7400 grateful -7401 possessed -7402 Queen -7403 decide -7404 proved -7405 extra -7406 cautio -7407 ppress -7408 fores -7409 render -7410 suspe -7411 singl -7412 coffin -7413 impro -7414 erely -7415 achin -7416 olent -7417 refu -7418 aused -7419 eaded -7420 fail -7421 flow -7422 intr -7423 awing -7424 corr -7425 cove -7426 Your -7427 view -7428 devil -7429 store -7430 eeper -7431 blin -7432 ede -7433 istening -7434 introdu -7435 laughing -7436 inquiri -7437 surroun -7438 dearest -7439 moments -7440 finally -7441 bell -7442 flow -7443 orgo -7444 ibly -7445 Wit -7446 tric -7447 den -7448 uper -7449 olon -7450 perc -7451 kl -7452 gun -7453 oks -7454 gag -7455 ibe -7456 unconscious -7457 intelligence -7458 recovered -7459 according -7460 courtyard -7461 conviction -7462 Pemberley -7463 locked -7464 contem -7465 persona -7466 arrying -7467 keeping -7468 practi -7469 custom -7470 sensibl -7471 ception -7472 perform -7473 prospe -7474 ctual -7475 plai -7476 ishes -7477 resti -7478 seri -7479 llowi -7480 llows -7481 exci -7482 venge -7483 acted -7484 lam -7485 eply -7486 esc -7487 ulle -7488 rial -7489 scre -7490 inn -7491 rpo -7492 asp -7493 ira -7494 practi -7495 refus -7496 speech -7497 terror -7498 civil -7499 uneas -7500 prote -7501 ightly -7502 colou -7503 spring -7504 fering -7505 intell -7506 ppl -7507 ky -7508 burn -7509 rand -7510 oran -7511 teen -7512 quis -7513 unni -7514 knee -7515 sna -7516 isse -7517 --� -7518 elic -7519 personal -7520 sleeping -7521 travelle -7522 intellig -7523 reading -7524 precious -7525 attended -7526 distinc -7527 building -7528 citizen -7529 enjoy -7530 propo -7531 oblig -7532 isted -7533 rtion -7534 duty -7535 symp -7536 fores -7537 eater -7538 epres -7539 tches -7540 solat -7541 blind -7542 arcel -7543 cry -7544 flu -7545 tfu -7546 invitation -7547 acquainted -7548 uttered -7549 custome -7550 romised -7551 joined -7552 result -7553 protect -7554 nights -7555 rations -7556 judged -7557 anyway -7558 Paris -7559 condu -7560 weeks -7561 earned -7562 eight -7563 detai -7564 awake -7565 healt -7566 final -7567 younge -7568 emplo -7569 nsible -7570 ularly -7571 Lucas -7572 gues -7573 accu -7574 tem -7575 ames -7576 seri -7577 gras -7578 unf -7579 aybe -7580 aker -7581 rnal -7582 rans -7583 noon -7584 sic -7585 unda -7586 arlo -7587 Li -7588 fil -7589 edl -7590 exe -7591 xi -7592 Fr -7593 burn -7594 bott -7595 shoe -7596 worse -7597 anish -7598 lean -7599 alter -7600 uched -7601 itche -7602 nless -7603 castl -7604 unter -7605 grie -7606 erval -7607 ersua -7608 Lizzy -7609 Te -7610 Whi -7611 evo -7612 fortunate -7613 reasonable -7614 conscious -7615 impressio -7616 attendant -7617 gratitude -7618 cape -7619 inti -7620 lot -7621 ntim -7622 spic -7623 hie -7624 uts -7625 mir -7626 Re -7627 occasionally -7628 listenin -7629 history -7630 patience -7631 unhappy -7632 personal -7633 acciden -7634 aturally -7635 mystery -7636 monster -7637 deliver -7638 perform -7639 sensible -7640 belong -7641 grass -7642 coupl -7643 ringin -7644 dispo -7645 compo -7646 ociety -7647 admira -7648 encourage -7649 officers -7650 separate -7651 language -7652 prepared -7653 servation -7654 tolerable -7655 assistan -7656 property -7657 objection -7658 represen -7659 author -7660 descen -7661 require -7662 concea -7663 illness -7664 jection -7665 compli -7666 deliver -7667 younge -7668 glas -7669 licat -7670 shout -7671 thick -7672 ences -7673 pared -7674 broad -7675 tcher -7676 orted -7677 iable -7678 Aunt -7679 Whi -7680 Par -7681 ncom -7682 dat -7683 eets -7684 gina -7685 ren -7686 ums -7687 nga -7688 sei -7689 vol -7690 tia -7691 eb -7692 Long -7693 coat -7694 fun -7695 pul -7696 harg -7697 dles -7698 ects -7699 dut -7700 adie -7701 ncon -7702 tial -7703 lars -7704 ucas -7705 approached -7706 observation -7707 communicati -7708 empty -7709 tterin -7710 tremb -7711 eresti -7712 France -7713 wound -7714 vation -7715 script -7716 fferin -7717 crime -7718 Sydney -7719 hast -7720 narr -7721 anima -7722 armed -7723 nobl -7724 supe -7725 kni -7726 cta -7727 fid -7728 nsc -7729 carefully -7730 encourage -7731 naturally -7732 comprehend -7733 conscience -7734 engagement -7735 violent -7736 opening -7737 separate -7738 scriptio -7739 cription -7740 weather -7741 feature -7742 arrival -7743 connecte -7744 onnected -7745 amiable -7746 slippe -7747 France -7748 thrown -7749 ulation -7750 detail -7751 entreat -7752 oodness -7753 heaven -7754 intell -7755 princip -7756 render -7757 ene -7758 unce -7759 bran -7760 inar -7761 hire -7762 clu -7763 pere -7764 fli -7765 rag -7766 sie -7767 sad -7768 sty -7769 fet -7770 ena -7771 laughing -7772 collected -7773 concerned -7774 requently -7775 sentiment -7776 knitting -7777 osity -7778 flowe -7779 exist -7780 bent -7781 ccept -7782 branc -7783 remov -7784 licit -7785 bene -7786 estra -7787 arest -7788 ounge -7789 Pe -7790 pet -7791 nan -7792 Ge -7793 remove -7794 avouri -7795 custo -7796 cheek -7797 gathe -7798 drove -7799 cease -7800 actio -7801 monst -7802 widow -7803 restor -7804 trade -7805 tende -7806 Lizzy -7807 arty -7808 send -7809 ices -7810 ele -7811 pent -7812 intr -7813 unl -7814 obl -7815 obli -7816 efer -7817 erlo -7818 acknowledge -7819 pes -7820 sts -7821 ury -7822 tn -7823 intelligence -7824 communication -7825 stairs -7826 growing -7827 simply -7828 regula -7829 summer -7830 fashio -7831 command -7832 laying -7833 endure -7834 madame -7835 fille -7836 allen -7837 lamp -7838 pity -7839 umble -7840 using -7841 pposi -7842 aming -7843 calm -7844 onabl -7845 tical -7846 ways -7847 cease -7848 perc -7849 enc -7850 len -7851 eede -7852 livi -7853 icti -7854 sof -7855 somehow -7856 pressing -7857 regular -7858 mysteri -7859 require -7860 disturbe -7861 command -7862 restore -7863 prehensi -7864 gratitud -7865 ratitude -7866 respectable -7867 destruction -7868 conviction -7869 concerning -7870 inclination -7871 arrangement -7872 congratulat -7873 author -7874 tempe -7875 temper -7876 flowe -7877 langu -7878 arning -7879 ywhere -7880 onable -7881 reali -7882 slept -7883 ysteri -7884 charm -7885 repres -7886 threa -7887 burst -7888 heave -7889 erp -7890 ois -7891 yst -7892 cru -7893 delightful -7894 disturbed -7895 Presently -7896 objection -7897 comprehen -7898 achi -7899 thor -7900 nduc -7901 ghed -7902 rock -7903 gui -7904 ndly -7905 repr -7906 bli -7907 soci -7908 addressed -7909 increase -7910 treasure -7911 appointed -7912 urred -7913 rryin -7914 nerv -7915 ectin -7916 fish -7917 mitte -7918 accep -7919 loud -7920 stern -7921 brain -7922 verse -7923 gloo -7924 ifted -7925 justi -7926 shock -7927 forth -7928 wake -7929 snow -7930 inco -7931 rcely -7932 sal -7933 Si -7934 lovely -7935 spread -7936 narrow -7937 rising -7938 ordered -7939 chamber -7940 olation -7941 propos -7942 commen -7943 feature -7944 desert -7945 pliment -7946 arts -7947 sad -7948 blow -7949 ngry -7950 Mor -7951 hick -7952 rick -7953 blue -7954 shak -7955 veri -7956 rows -7957 alls -7958 corr -7959 olat -7960 Aunt -7961 doi -7962 ige -7963 nke -7964 efe -7965 belon -7966 angry -7967 famili -7968 inting -7969 accep -7970 produ -7971 reated -7972 depen -7973 chain -7974 accus -7975 drawn -7976 attach -7977 ignorant -7978 meeting -7979 growing -7980 succeed -7981 glanced -7982 thunder -7983 inquire -7984 isturbed -7985 improve -7986 erceived -7987 tenin -7988 nches -7989 angr -7990 pros -7991 appl -7992 trad -7993 cast -7994 esses -7995 awake -7996 threa -7997 sured -7998 trans -7999 eares -8000 utte -8001 repr -8002 Goo -8003 respectable -8004 explanation -8005 inclination -8006 urst -8007 enev -8008 rid -8009 cru -8010 rupt -8011 esta -8012 ingu -8013 nsu -8014 xam -8015 surrounded -8016 unconsciou -8017 consolation -8018 comprehend -8019 engagement -8020 Fitzwilliam -8021 concerned -8022 ossibility -8023 occupation -8024 ignorant -8025 directed -8026 midnight -8027 philosoph -8028 eparation -8029 frequentl -8030 famili -8031 temper -8032 tongue -8033 flower -8034 succeed -8035 justice -8036 strike -8037 nstruct -8038 shudder -8039 formed -8040 future -8041 intende -8042 report -8043 younger -8044 entrea -8045 twice -8046 loose -8047 encou -8048 alarm -8049 propos -8050 ctiona -8051 books -8052 forth -8053 thered -8054 delica -8055 sprin -8056 atcher -8057 jump -8058 Just -8059 spot -8060 shee -8061 inall -8062 peri -8063 infl -8064 impro -8065 key -8066 nife -8067 espa -8068 sla -8069 kiss -8070 supe -8071 agge -8072 ugge -8073 turb -8074 cit -8075 fra -8076 gate -8077 pil -8078 Sher -8079 pac -8080 chu -8081 ews -8082 Qu -8083 Ne -8084 nfu -8085 uci -8086 Ga -8087 Ar -8088 possibly -8089 removed -8090 collect -8091 Gryphon -8092 naturall -8093 pressive -8094 increase -8095 shudder -8096 treasure -8097 probable -8098 princip -8099 tentions -8100 hist -8101 prea -8102 pull -8103 luck -8104 dder -8105 orti -8106 mplo -8107 erty -8108 hove -8109 equ -8110 ears -8111 advi -8112 earer -8113 emble -8114 tempe -8115 orked -8116 langu -8117 ceede -8118 eatin -8119 coll -8120 yell -8121 hered -8122 Somet -8123 steri -8124 evil -8125 acked -8126 asten -8127 wound -8128 fres -8129 imate -8130 path -8131 tend -8132 sul -8133 epr -8134 nfe -8135 decid -8136 preten -8137 inclin -8138 noise -8139 brain -8140 teeth -8141 alive -8142 oblige -8143 onceal -8144 facts -8145 heavi -8146 consu -8147 onster -8148 hopes -8149 weigh -8150 heaven -8151 ulness -8152 depar -8153 blame -8154 actual -8155 practic -8156 leaves -8157 refuse -8158 inclin -8159 pulled -8160 smiled -8161 accord -8162 repared -8163 improv -8164 Clerval -8165 commend -8166 fuse -8167 hone -8168 weat -8169 degr -8170 ast -8171 itua -8172 nsib -8173 rda -8174 phe -8175 handkerchief -8176 arrangement -8177 congratulat -8178 desperat -8179 delighted -8180 inquiries -8181 civility -8182 accompan -8183 falle -8184 drin -8185 books -8186 scra -8187 enjo -8188 reali -8189 drawn -8190 rming -8191 victi -8192 epart -8193 six -8194 bow -8195 creatures -8196 addressed -8197 cen -8198 gran -8199 mous -8200 roof -8201 hirt -8202 pale -8203 aved -8204 alte -8205 oyed -8206 sno -8207 ishi -8208 hood -8209 nanc -8210 stee -8211 hil -8212 vell -8213 firm -8214 priv -8215 grie -8216 lind -8217 At -8218 frequently -8219 accomplish -8220 Huckleberry -8221 declare -8222 familiar -8223 pointing -8224 directly -8225 protect -8226 keeping -8227 descend -8228 excited -8229 compass -8230 mistaken -8231 commence -8232 examined -8233 gratefu -8234 tired -8235 flower -8236 histor -8237 holdin -8238 numbe -8239 lives -8240 hatche -8241 accord -8242 lities -8243 tement -8244 detail -8245 ailing -8246 coffi -8247 curren -8248 cup -8249 Au -8250 key -8251 bey -8252 Le -8253 nj -8254 uncomfortable -8255 apprehension -8256 decided -8257 preten -8258 houses -8259 avoured -8260 confus -8261 belief -8262 record -8263 requir -8264 motion -8265 perfor -8266 attach -8267 solatio -8268 murmur -8269 relate -8270 ashed -8271 tains -8272 oping -8273 bered -8274 penin -8275 hate -8276 efull -8277 rsona -8278 arned -8279 coac -8280 charm -8281 swea -8282 menta -8283 sail -8284 weigh -8285 rospe -8286 ven -8287 top -8288 bow -8289 ntro -8290 ckly -8291 joy -8292 unse -8293 devi -8294 tifi -8295 doc -8296 lson -8297 Cr -8298 nu -8299 ,-- -8300 cky -8301 ya -8302 tas -8303 hoe -8304 rum -8305 shm -8306 marke -8307 Indeed -8308 reply -8309 feared -8310 istant -8311 pposit -8312 unless -8313 justi -8314 summer -8315 record -8316 secur -8317 knees -8318 iction -8319 vering -8320 reside -8321 ounter -8322 trace -8323 seize -8324 espond -8325 report -8326 fulnes -8327 Tur -8328 rtle -8329 lamp -8330 neck -8331 shr -8332 nclu -8333 var -8334 uted -8335 heel -8336 barr -8337 insi -8338 etai -8339 tuat -8340 tena -8341 possibly -8342 pointing -8343 impatient -8344 produced -8345 movement -8346 mistaken -8347 material -8348 building -8349 discovery -8350 undertak -8351 deci -8352 onsen -8353 shame -8354 burst -8355 gues -8356 ouses -8357 harm -8358 refus -8359 accus -8360 eetin -8361 thic -8362 wling -8363 stir -8364 rings -8365 vious -8366 ially -8367 udged -8368 heave -8369 senti -8370 sland -8371 erd -8372 etr -8373 apa -8374 actually -8375 fountain -8376 nervous -8377 slipped -8378 dressing -8379 constant -8380 triumph -8381 opposit -8382 material -8383 eference -8384 forgive -8385 hesitat -8386 younger -8387 niggers -8388 accustomed -8389 explained -8390 entertain -8391 connected -8392 perceived -8393 presented -8394 declar -8395 anished -8396 eserved -8397 thunder -8398 unless -8399 delica -8400 lantern -8401 tations -8402 nitting -8403 admire -8404 Meryton -8405 Sydney -8406 Let -8407 gold -8408 slip -8409 eech -8410 lang -8411 wide -8412 arli -8413 adn -8414 def -8415 consolation -8416 communicati -8417 observation -8418 indifferent -8419 delightful -8420 communicate -8421 remembrance -8422 conscience -8423 aching -8424 omehow -8425 Frenc -8426 speec -8427 moving -8428 lanced -8429 easing -8430 invit -8431 terro -8432 clean -8433 keepin -8434 ournal -8435 roads -8436 orgive -8437 behav -8438 desert -8439 spent -8440 nction -8441 lights -8442 consol -8443 interv -8444 board -8445 civili -8446 capabl -8447 apable -8448 graph -8449 shin -8450 decl -8451 oined -8452 blame -8453 roll -8454 resi -8455 cript -8456 grate -8457 polic -8458 final -8459 sings -8460 larly -8461 oning -8462 fit -8463 sui -8464 nay -8465 ddin -8466 fail -8467 bal -8468 aini -8469 lift -8470 uni -8471 equi -8472 tors -8473 nden -8474 enan -8475 eu -8476 communication -8477 gni -8478 lap -8479 erg -8480 lst -8481 uke -8482 occasionally -8483 actually -8484 fountain -8485 familiar -8486 introduce -8487 Westenra -8488 precious -8489 destroyed -8490 apparent -8491 examined -8492 recommend -8493 decided -8494 measure -8495 tremble -8496 practic -8497 olerable -8498 Renfield -8499 recommen -8500 Meryton -8501 crossed -8502 atience -8503 giving -8504 attende -8505 ttended -8506 painful -8507 scripti -8508 improve -8509 support -8510 civilit -8511 citize -8512 nice -8513 catc -8514 ossed -8515 feat -8516 grass -8517 asted -8518 ontra -8519 tall -8520 uired -8521 secur -8522 boat -8523 priv -8524 behav -8525 repo -8526 dingl -8527 fina -8528 skin -8529 midd -8530 nsen -8531 swi -8532 grac -8533 wak -8534 rugg -8535 val -8536 imat -8537 ecia -8538 bene -8539 Oh -8540 log -8541 hab -8542 doors -8543 tendin -8544 guess -8545 narrow -8546 ssiona -8547 oppos -8548 rolled -8549 Somet -8550 tified -8551 resto -8552 dig -8553 ph -8554 desperate -8555 inquiries -8556 xamination -8557 possessed -8558 sentiment -8559 contemplat -8560 sensations -8561 attachment -8562 Vengeance -8563 cus -8564 appl -8565 ash -8566 est -8567 tas -8568 secu -8569 ases -8570 gath -8571 orre -8572 rren -8573 mand -8574 knife -8575 otten -8576 tivel -8577 nois -8578 joine -8579 ariou -8580 edge -8581 scre -8582 invi -8583 port -8584 agged -8585 provi -8586 exer -8587 sness -8588 habi -8589 buil -8590 gment -8591 nishm -8592 parte -8593 nvinc -8594 vince -8595 ustin -8596 Sher -8597 eag -8598 sna -8599 noc -8600 mst -8601 unl -8602 vie -8603 communicate -8604 description -8605 sensations -8606 marked -8607 nervous -8608 welcome -8609 measure -8610 patienc -8611 lighte -8612 intend -8613 agitat -8614 seized -8615 civili -8616 delive -8617 Justine -8618 uttere -8619 neare -8620 youth -8621 absen -8622 itions -8623 sisted -8624 pract -8625 secure -8626 shove -8627 hither -8628 pectat -8629 erform -8630 canoe -8631 Polly -8632 layi -8633 shoe -8634 ooth -8635 nois -8636 excl -8637 anke -8638 pen -8639 sequ -8640 urel -8641 Not -8642 iabl -8643 cott -8644 Fi -8645 lab -8646 oup -8647 vit -8648 falling -8649 authori -8650 ordered -8651 soldier -8652 crossed -8653 painful -8654 powerful -8655 lantern -8656 request -8657 singular -8658 Republic -8659 returning -8660 directly -8661 attended -8662 destroye -8663 gradually -8664 Renfield -8665 objectio -8666 tolerabl -8667 prospect -8668 recommen -8669 Republic -8670 tuall -8671 eaten -8672 sembl -8673 bird -8674 lose -8675 suit -8676 hunt -8677 clean -8678 sold -8679 ental -8680 ourin -8681 ulate -8682 seek -8683 iring -8684 rture -8685 nyway -8686 ewh -8687 nks -8688 Bo -8689 fift -8690 swa -8691 brow -8692 sily -8693 vani -8694 Jac -8695 Look -8696 site -8697 torm -8698 por -8699 emin -8700 affa -8701 rded -8702 infl -8703 tme -8704 Bec -8705 athing -8706 highe -8707 mourn -8708 delay -8709 summe -8710 arting -8711 ceased -8712 These -8713 concei -8714 ntreat -8715 comme -8716 spare -8717 knitt -8718 returning -8719 destroyed -8720 gradually -8721 philosoph -8722 apprehensi -8723 attentions -8724 asking -8725 crying -8726 declare -8727 authori -8728 calling -8729 moving -8730 feared -8731 Hatter -8732 triumph -8733 inquire -8734 lifted -8735 kissed -8736 secure -8737 endered -8738 ounding -8739 astened -8740 emotion -8741 suspect -8742 interv -8743 featur -8744 manager -8745 chain -8746 hall -8747 exis -8748 asse -8749 impa -8750 rking -8751 roved -8752 eries -8753 alarm -8754 asing -8755 cheek -8756 gain -8757 shoc -8758 ulati -8759 ivers -8760 assen -8761 suns -8762 requi -8763 erfor -8764 bably -8765 rain -8766 esign -8767 bird -8768 shop -8769 lou -8770 itne -8771 iary -8772 lici -8773 oler -8774 Who -8775 Gr -8776 udi -8777 uca -8778 cip -8779 sar -8780 authorit -8781 pressed -8782 flowers -8783 consent -8784 justice -8785 instruct -8786 servants -8787 willing -8788 treasur -8789 working -8790 emotion -8791 appointe -8792 ppointed -8793 restrain -8794 tranquil -8795 apprehen -8796 lections -8797 glancing -8798 pot -8799 sem -8800 * * -8801 * * * -8802 * * -8803 - * * -8804 - * -8805 and says: - - - -“ -8806 - - - - - - - -[Illustra -8807 - - - -“I don’t -8808 - - - - - - - -[Illustr -8809 - - - - - - -[Illustra -8810 . - - - - - -[Illustra -8811 said Mr. Lorry, -8812 said Mr. Lorry, -8813 said Mr. Lorry, -8814 � - - - -“I don’t -8815 opyright 1894 by Geor -8816 opyright 1894 by Geo -8817 pyright 1894 by Geor -8818 � - - - -“I don’ -8819 opyright 1894 by Ge -8820 pyright 1894 by Geo -8821 yright 1894 by Geor -8822 he says: - - - -“ -8823 opyright 1894 by G -8824 pyright 1894 by Ge -8825 yright 1894 by Geo -8826 right 1894 by Geor -8827 * * * -8828 stration: - - - - -8829 opyright 1894 by -8830 pyright 1894 by G -8831 yright 1894 by Ge -8832 right 1894 by Geo -8833 ight 1894 by Geor -8834 I. - - - - - -[Illustra -8835 don’t know wh -8836 * * -8837 * * * -8838 don’t want to -8839 stration: - - - - -8840 tration: - - - - -8841 opyright 1894 by -8842 pyright 1894 by -8843 yright 1894 by G -8844 right 1894 by Ge -8845 ight 1894 by Geo -8846 ght 1894 by Geor -8847 I. - - - - - -[Illustr -8848 - - - -“I do not -8849 * * * -8850 ge Allen._]] - - - - -8851 arker’s Journal -8852 * * * -8853 arker’s Journa -8854 rker’s Journal -8855 - - - - - -_Dr. Seward� -8856 * * * -8857 ” - - - -[_Copyrigh -8858 and said: - - - -“ -8859 stration] - - - - - - - - -8860 I don’t want to -8861 I don’t know wh -8862 - - - - - -_Dr. Seward -8863 - - - - -_Dr. Seward� -8864 I don’t want t -8865 ” - - - -[_Copyrig -8866 �� - - - -[_Copyrigh -8867 * * - - - -_Lat -8868 eward’s Diary._ -8869 said Mr. Crunche -8870 stration] - - - - - - - -8871 tration] - - - - - - - - -8872 I don’t know w -8873 � - - - -“I think -8874 - - - - - -_Dr. Sewar -8875 - - - - -_Dr. Seward -8876 - - - -_Dr. Seward� -8877 -“Well, then, -8878 stration: - - - -“ -8879 � - - - -“I do not -8880 � - - - -“Certainly -8881 - - - -“Certainly -8882 * * - - - -_La -8883 * - - - -_Lat -8884 eward’s Diary. -8885 ward’s Diary._ -8886 ” - - - -Mr. Lorry -8887 said Mr. Crunch -8888 said Mr. Crunche -8889 * - - - -_Later._- -8890 lonel Fitzwilliam -8891 id Alice. - - - -“ -8892 � - - - -“I do not -8893 � - - - -“Certainl -8894 - - - -“I cannot -8895 ” - - - -Mr. Lorry -8896 * - - - -_Later._ -8897 * - - - -_Later._- -8898 lonel Fitzwillia -8899 onel Fitzwilliam -8900 - - - -“Of course -8901 - - - -“I should -8902 �� - - - -Mr. Lorry -8903 I asked. - - - -“ -8904 I don’t know. -8905 he asked. - - - -“ -8906 � - - - -“I cannot -8907 didn’t want t -8908 didn’t want to -8909 � - - - -“I cannot -8910 d Holmes. - - - -“ -8911 Tom said: - - - -“ -8912 didn’t know wh -8913 and says: - - - -“I -8914 didn’t know wh -8915 He says: - - - -“ -8916 said Mr. Lorry. -8917 ge Allen._]] - - - - - - - - -8918 didn’t want to -8919 and says: - - - -“W -8920 .” - - - -Mr. Lorry -8921 didn’t know w -8922 _Czarina Catheri -8923 warn’t no use -8924 nd says: - - - -“I -8925 he said: - - - -“ -8926 ge Allen._]] - - - - - - - -8927 e Allen._]] - - - - - - - - -8928 � - - - -“I should -8929 � - - - -“I shall -8930 - - - -“I believe -8931 - - - -“I have no -8932 - - - -“I am not -8933 nd says: - - - -“W -8934 - - - -“I reckon -8935 .” - - - -Mr. Lorr -8936 ge Allen._]] - - - - - - -8937 e Allen._]] - - - - - - - -8938 Allen._]] - - - - - - - - -8939 �� - - - - - -_Dr. Seward� -8940 - - - -“I thought -8941 - - - -“I can’t -8942 � - - - -“I should -8943 � - - - -“I never -8944 _Czarina Catheri -8945 � - - - -“I reckon -8946 ge Allen._]] - - - - - -8947 e Allen._]] - - - - - - -8948 Allen._]] - - - - - - - -8949 Allen._]] - - - - - - - - -8950 and Mrs. Gardine -8951 .” - - - -Mr. Lorry -8952 �� - - - - - -_Dr. Seward -8953 � - - - - - -_Dr. Seward� -8954 . - - - - - -_Dr. Seward� -8955 - - - -“I didn’ -8956 with an air of -8957 � - - - -“Quite so -8958 _Czarina Cather -8959 ain’t going t -8960 � - - - -“I reckon -8961 e Allen._]] - - - - - -8962 Allen._]] - - - - - - -8963 Allen._]] - - - - - - - -8964 llen._]] - - - - - - - - -8965 and Mrs. Gardin -8966 and Mrs. Gardine -8967 Defarge. - - - -“ -8968 Attorney-General -8969 - - - -_Mina Harker� -8970 -“Well, that’ -8971 �� - - - - - -_Dr. Sewar -8972 � - - - - - -_Dr. Seward -8973 . - - - - - -_Dr. Seward -8974 - - -DR. SEWARD’S DIA -8975 the door. - - - -“ -8976 - - - -_Mina Harker -8977 - - -_Mina Harker� -8978 he says: - - - -“W -8979 -“Well, that� -8980 “Well, that’ -8981 - - - -Mrs. Bennet -8982 said he. - - - -“ -8983 Neville St. Cla -8984 r. Lorry. - - - -“ -8985 - - - -“Of course -8986 � - - - -“I believe -8987 �� - - - - - -_Dr. Sewa -8988 � - - - - - -_Dr. Sewar -8989 . - - - - - -_Dr. Sewar -8990 -“Well, I don� -8991 stration] - - - -Eliz -8992 - - -DR. SEWARD’S DI -8993 - -DR. SEWARD’S DIA -8994 - - -DR. SEWARD’S -8995 lonel Fitzwilliam -8996 - - -DR. SEWARD’S D -8997 - -DR. SEWARD’S DI -8998 -DR. SEWARD’S DIA -8999 - - -“Of course -9000 � - - - -“I believ -9001 - -“Don’t you -9002 �� - - - - - -_Dr. Sew -9003 � - - - - - -_Dr. Sewa -9004 . - - - - - -_Dr. Sewa -9005 the _Czarina Ca -9006 -“Well, I don� -9007 “Well, I don� -9008 � I says. - - - -“ -9009 - -I says: - - - -“ -9010 uke says: - - - -“ -9011 - - - -“I am sure -9012 ” - - - -[Illustra -9013 said Mrs. Bennet -9014 stration] - - - -Eli -9015 tration] - - - -Eliz -9016 - - - -Mrs. Gardine -9017 . - - - -“I don’t -9018 � - - - -“Of course -9019 ouldn’t help it -9020 - - -“Don’t you -9021 warn’t nothing -9022 - - -DR. SEWARD’S -9023 - -DR. SEWARD’S -9024 said he, “that -9025 � - - - -“I am not -9026 .” - - - -[Illustra -9027 V. - - - - - -[Illustra -9028 onel Fitzwilliam -9029 Attorney-General -9030 - -DR. SEWARD’S D -9031 -DR. SEWARD’S DI -9032 DR. SEWARD’S DIA -9033 said he, “that -9034 don’t believe -9035 I don’t like t -9036 . - - - -“I don’ -9037 didn’t think -9038 � - - - -“Of cours -9039 ouldn’t help i -9040 uldn’t help it -9041 I don’t think -9042 - - -“Don’t yo -9043 warn’t nothin -9044 warn’t nothing -9045 - - -DR. SEWARD’ -9046 - -DR. SEWARD’S -9047 -DR. SEWARD’S -9048 the room. - - - -“ -9049 - - - -“I did not -9050 I don’t know h -9051 . - - - - - -_Mina Har -9052 - - - -“Quite so. -9053 - - - -“I have be -9054 his hand. - - - -“ -9055 said he, “tha -9056 said he, “that -9057 � - - - -“I am not -9058 ouldn’t see no -9059 ain’t nothing -9060 ” - - - -I didn’ -9061 ain’t got no -9062 .” - - - -[Illustr -9063 V. - - - - - -[Illustr -9064 nel Fitzwilliam -9065 ttorney-General -9066 . - - - -“Of course -9067 � - - - -“I can’t -9068 - - -“Oh, don’t -9069 eward’s Diary - - -9070 ARKER’S JOURNAL -9071 -DR. SEWARD’S D -9072 DR. SEWARD’S DI -9073 R. SEWARD’S DIA -9074 � - - - -“Quite so. -9075 said he, “that -9076 stration: - - - -“T -9077 X. - - - - - -[Illustra -9078 said Mr. Lorry. -9079 ***** - - - - -9080 * * * - -9081 ” - - - -Mr. Crunche -9082 ***** - - - -“ -9083 . - - - -“Of cours -9084 � - - - -“I can’ -9085 - - -“Oh, don’ -9086 - -“Oh, don’t -9087 remarked. - - - -“ -9088 eward’s Diary - -9089 ward’s Diary - - -9090 ARKER’S JOURNA -9091 RKER’S JOURNAL -9092 d for an instant -9093 �s Diary._ - - - -_2 -9094 � - - - -“I have n -9095 DR. SEWARD’S D -9096 R. SEWARD’S DI -9097 . SEWARD’S DIA -9098 don’t know ho -9099 - - - -“I think t -9100 aid he, “that -9101 ouldn’t hardly -9102 Tom says: - - - -“ -9103 n’t a-going to -9104 ” - - - -I says: - - -9105 stration] - - - -The -9106 tration: - - - -“T -9107 X. - - - - - -[Illustr -9108 . - - - -Mrs. Bennet -9109 Lord St. Simon -9110 said Mr. Lorry. -9111 ***** - - - -9112 ***** - - - - -9113 hispered: - - - -“ -9114 - - - -JONATHAN HARKER -9115 ***** - - - -� -9116 ***** - - - -“ -9117 * * * - -9118 eptember._ - - - -“ -9119 � - - - -“I have no -9120 ouldn’t see no -9121 e ain’t nothing -9122 ouldn’t seem to -9123 and says: - - - -“T -9124 and says: - - - -“D -9125 in’t a-going to -9126 and says: - - - -“H -9127 -“Well, then, w -9128 said Mrs. Bennet, -9129 stration] - - - -The -9130 . - - - -Mrs. Gardine -9131 ” - - - -Mr. Crunch -9132 �� - - - -Mr. Crunche -9133 said Mr. Stryver -9134 NATHAN HARKER’S JOU -9135 - - - -JONATHAN HARKE -9136 - - -JONATHAN HARKER -9137 e” - - - -[_Copyrigh -9138 ***** - - - -� -9139 ***** - - - -� -9140 ***** - - - -“ -9141 * * * - -9142 don’t underst -9143 he said. - - - -“ -9144 e shadows of the -9145 - - - -“I thank y -9146 �s Diary._ - - - -_1 -9147 eptember._ - - - -� -9148 ptember._ - - - -“ -9149 - - - -“Very well -9150 uldn’t see no -9151 e ain’t nothin -9152 didn’t see no -9153 ouldn’t seem t -9154 uldn’t seem to -9155 nd says: - - - -“T -9156 don’t make no -9157 . I says: - - - -“ -9158 nd says: - - - -“D -9159 ain’t a-going -9160 in’t a-going t -9161 nd says: - - - -“H -9162 “Well, then, w -9163 warn’t no use -9164 he says: - - - -“I -9165 and Mrs. Bennet -9166 said Mr. Bennet -9167 aid Mrs. Bennet, -9168 tration] - - - -The -9169 - - - -“I am afra -9170 . - - - -Mrs. Gardin -9171 � said I. - - - -“ -9172 ” - - - -Mr. Crunc -9173 �� - - - -Mr. Crunch -9174 � - - - -Mr. Crunche -9175 said Mr. Stryve -9176 said Mr. Stryver -9177 - - - -JONATHAN HARKER� -9178 NATHAN HARKER’S JO -9179 ATHAN HARKER’S JOU -9180 . THE ADVENTURE OF THE -9181 I don’t underst -9182 urray’s Journal -9183 - - - -JONATHAN HARK -9184 - - -JONATHAN HARKE -9185 - -JONATHAN HARKER -9186 e shadows of the -9187 I don’t know ho -9188 - - -“Oh, my dear -9189 I don’t care fo -9190 he says: - - - -“W -9191 ouldn’t hardly -9192 and - -says: - - - -“ -9193 t don’t make no -9194 warn’t no use -9195 - -“Why, it’s -9196 ” - - - -Mr. Bennet -9197 e” - - - -[_Copyrig -9198 � said I. - - - -“� -9199 . Neville St. Cla -9200 said Mr. Lorry. - -9201 ***** - - - -� -9202 ***** - - - -� -9203 ***** - - - -“ -9204 . THE ADVENTURE OF THE -9205 THE ADVENTURE OF THE -9206 - - -JONATHAN HARKER� -9207 NATHAN HARKER’S J -9208 ATHAN HARKER’S JO -9209 THAN HARKER’S JOU -9210 ’S JOURNAL - - - - - -_ -9211 ge Allen._]] - - - -“ -9212 . THE ADVENTURE OF TH -9213 THE ADVENTURE OF THE -9214 THE ADVENTURE OF THE -9215 - - - -“I wish I -9216 - - - -“I won’t -9217 I don’t unders -9218 - - - -“Would you -9219 - - - -“I think I -9220 - - - -“I never s -9221 ill you, won’t -9222 , won’t you, w -9223 “Well, I never -9224 didn’t know h -9225 silence. - - - -“ -9226 urray’s Journa -9227 rray’s Journal -9228 - - - -JONATHAN HAR -9229 - - -JONATHAN HARK -9230 - -JONATHAN HARKE -9231 -JONATHAN HARKER -9232 ’S JOURNAL - - - - -9233 shadows of the -9234 - - - -“I am glad -9235 e don’t want t -9236 answered. - - - -“ -9237 I - - - -DR. SEWARD� -9238 - - -“Oh, my dea -9239 - -“Oh, my dear -9240 . - - - - - -_Jonathan -9241 � - - - -“Did you -9242 I don’t care f -9243 don’t care fo -9244 minute, and the -9245 don’t know no -9246 ouldn’t stand -9247 uldn’t hardly -9248 and - -says: - - - -� -9249 nd - -says: - - - -“ -9250 t don’t make n -9251 he says. - - - -“ -9252 she says: - - - -“ -9253 ain’t anythin -9254 - - - -“Blame it, -9255 ings out: - - - -“ -9256 - -“Why, it’s -9257 -“Why, it’s -9258 ” - - - -Mr. Benne -9259 �� - - - -Mr. Bennet -9260 e” - - - -[_Copyri -9261 ” - - - -Mr. Darcy -9262 lizabeth. - - - -“ -9263 and Mr. Collins -9264 I think, Watson, -9265 said I. - - - -“� -9266 Holmes, laughin -9267 . Neville St. Cl -9268 said Mr. Lorry. - -9269 that Mr. Lorry -9270 ***** - - - -� -9271 ***** - - - -� -9272 ***** - - - -“ -9273 - -JONATHAN HARKER� -9274 NATHAN HARKER’S -9275 ATHAN HARKER’S J -9276 THAN HARKER’S JO -9277 HAN HARKER’S JOU -9278 ’S JOURNAL - - - - - - -9279 ��S JOURNAL - - - - - -_ -9280 eptember._ - - - -“M -9281 ” - - - -Mr. Bennet -9282 ge Allen._]] - - - -� -9283 e Allen._]] - - - -“ -9284 . THE ADVENTURE OF T -9285 THE ADVENTURE OF TH -9286 THE ADVENTURE OF THE -9287 HE ADVENTURE OF THE -9288 � - - - -“I won’t -9289 I don’t believe -9290 the door, and the -9291 -“Well, it’s -9292 terpillar. - - - -“ -9293 arch Hare. - - - -“ -9294 of them, and the -9295 � - - - -“I never s -9296 -“Well, I never -9297 -JONATHAN HARKER� -9298 NATHAN HARKER’S -9299 ATHAN HARKER’S -9300 THAN HARKER’S J -9301 HAN HARKER’S JO -9302 AN HARKER’S JOU -9303 ’S JOURNAL - - - - - -9304 ��S JOURNAL - - - - - - -9305 �S JOURNAL - - - - - -_ -9306 ptember._ - - - -“M -9307 � - - - -“Very well -9308 t ain’t anythin -9309 - -“Yes, sir.” -9310 and says: - - - -“M -9311 he says: - - - -“I -9312 � - - - -“I thought -9313 �� - - - -Mr. Bennet -9314 stration] - - - -Mr. -9315 .” - - - -Mrs. Benn -9316 ge Allen._]] - - - -� -9317 e Allen._]] - - - -� -9318 Allen._]] - - - -“ -9319 � - - - -“I am sure -9320 � - - - -“I think t -9321 and Five, North T -9322 eretofore the Mar -9323 RD’S DIARY - - - - - -_ -9324 - - - - -“Dr. Van Hel -9325 . THE ADVENTURE OF -9326 THE ADVENTURE OF T -9327 THE ADVENTURE OF TH -9328 HE ADVENTURE OF THE -9329 E ADVENTURE OF THE -9330 “Well, I’ll -9331 � - - - -“I won’ -9332 - - - -“I beg you -9333 don’t know th -9334 I don’t believ -9335 the door, and th -9336 he door, and the -9337 -“Well, it’s -9338 “Well, it’s -9339 terpillar. - - - -� -9340 erpillar. - - - -“ -9341 - - - -“I suppose -9342 arch Hare. - - - -� -9343 rch Hare. - - - -“ -9344 � - - - -“Is that -9345 of them, and th -9346 of them, and the -9347 � - - - -“I’ll t -9348 - - - -“I’ll te -9349 Gryphon. - - - -“ -9350 -“Well, I neve -9351 - - - -“Never min -9352 JONATHAN HARKER� -9353 NATHAN HARKER’ -9354 ATHAN HARKER’S -9355 THAN HARKER’S -9356 HAN HARKER’S J -9357 AN HARKER’S JO -9358 N HARKER’S JOU -9359 . - - - - - -_Letter, -9360 ��S JOURNAL - - - - - -9361 �S JOURNAL - - - - - - -9362 S JOURNAL - - - - - -_ -9363 moment, and the -9364 he said, with a -9365 * * - - - -On -9366 tember._ - - - -“M -9367 - - - -“I promise -9368 What’s the mat -9369 “Friend John, -9370 - - - -“I underst -9371 � - - - -“Very wel -9372 - - - -“I know th -9373 don’t like to -9374 didn’t believe -9375 warn’t going t -9376 He said: - - - -“ -9377 - - - -“I hain’ -9378 n I says: - - - -“ -9379 a runaway nigger -9380 he says: - - - -“H -9381 t ain’t anythi -9382 - -“Yes, sir.� -9383 -“Yes, sir.” -9384 nd says: - - - -“M -9385 don’t know.� -9386 ing says: - - - -“ -9387 � - - - -“I though -9388 don’t think I -9389 � - - - -Mr. Bennet -9390 stration] - - - -Mr. -9391 tration] - - - -Mr. -9392 .” - - - -Mrs. Ben -9393 ” - - - -Mrs. Benn -9394 � - - - -Mrs. Bennet -9395 that Mr. Bingley -9396 � - - - -“I would -9397 e Allen._]] - - - -� -9398 Allen._]] - - - -� -9399 Allen._]] - - - -“ -9400 � - - - -“Upon my -9401 - - - -“Upon my w -9402 that Mr. Darcy -9403 lizabeth; “but -9404 � - - - -“I am sur -9405 and Five, North -9406 nd Five, North T -9407 I don’t know, -9408 eretofore the Ma -9409 retofore the Mar -9410 rémonde, called -9411 tizen Evrémonde -9412 - -* * * -9413 . - - - -“Of course -9414 ER’S JOURNAL - - - - -9415 RD’S DIARY - - - - - - -9416 D’S DIARY - - - - - -_ -9417 - - - - -“Dr. Van He -9418 - - - -“Dr. Van Hel -9419 lonel Fitzwilliam, -9420 . THE ADVENTURE OF -9421 THE ADVENTURE OF -9422 THE ADVENTURE OF T -9423 HE ADVENTURE OF TH -9424 E ADVENTURE OF THE -9425 ADVENTURE OF THE -9426 !” - - - -Mr. Lorry -9427 ER’S JOURNAL - - - - - -_ -9428 - -* * * -9429 -* * * -9430 . - - - -“I thought -9431 � - - - -“I didn’ -9432 didn’t know th -9433 id Alice. - - - -“W -9434 -“Well, then,� -9435 u don’t know wh -9436 , won’t you, wi -9437 ’t you, will yo -9438 , with an air of -9439 ER’S JOURNAL - - - -9440 R’S JOURNAL - - - - -9441 �� - - - - - -_Letter, -9442 a moment, and the -9443 e don’t want to -9444 * * - - - -On 1 -9445 ed for an instant -9446 EWARD’S DIARY - - -9447 RD’S DIARY - - - - - -9448 D’S DIARY - - - - - - -9449 ’S DIARY - - - - - -_ -9450 - - - - -“Dr. Van H -9451 - - - -“Dr. Van He -9452 - - -“Dr. Van Hel -9453 � - - - -“I thank y -9454 I don’t want an -9455 . - - - -“I do not -9456 � - - - -“I know th -9457 I don’t like to -9458 ouldn’t make no -9459 warn’t going t -9460 -“Well, I’ll -9461 He says: - - - -“I -9462 .” - - - -I didn’ -9463 I ain’t going t -9464 I don’t know.� -9465 � - - - -Mrs. Bennet -9466 that Mr. Bingley -9467 � - - - -“Upon my w -9468 ” - - - -Mr. Darcy -9469 .” - - - -Mr. Darcy -9470 � - - - -“I have be -9471 � - - - -“I am afra -9472 . - - - -Mrs. Bennet -9473 said he, “but -9474 lizabeth; “but -9475 onel Fitzwilliam, -9476 � - - - -“I underst -9477 Holmes, “that -9478 � - - - -“Very good -9479 d Holmes, laughin -9480 he asked. - - - -“� -9481 . THE ADVENTURE O -9482 THE ADVENTURE OF -9483 THE ADVENTURE OF -9484 HE ADVENTURE OF T -9485 E ADVENTURE OF TH -9486 ADVENTURE OF THE -9487 ADVENTURE OF THE -9488 � - - - -“I will do -9489 !” - - - -Mr. Lorry -9490 � croaked Jacques -9491 arles Evrémonde, -9492 rémonde, called -9493 ! Ting-a-ling-lin -9494 ER’S JOURNAL - - - - - - -9495 R’S JOURNAL - - - - - -_ -9496 I. THE ADVENTURE OF THE -9497 MINA HARKER.” - - - - -9498 ER’S JOURNAL - - - - - -9499 R’S JOURNAL - - - - - - -9500 .” - - - -Mr. Crunche -9501 arles Evrémonde, c -9502 - -* * -9503 -* * * -9504 don’t care wh -9505 don’t like th -9506 . - - - -“I though -9507 � - - - -“I didn� -9508 � - - - -“I wish I -9509 hind him. - - - -“ -9510 didn’t know t -9511 didn’t know th -9512 d Alice. - - - -“W -9513 -“Well, then,� -9514 “Well, then,� -9515 ed Alice. - - - -“ -9516 he Queen. - - - -“ -9517 don’t seem to -9518 u don’t know w -9519 k Turtle. - - - -“ -9520 - - -“No, indeed -9521 won’t you, wi -9522 ’t you, will y -9523 ��t you, will yo -9524 � - - - -“I could -9525 , with an air of -9526 ER’S JOURNAL - - -9527 R’S JOURNAL - - - -9528 �� - - - - - -_Letter, -9529 � - - - - - -_Letter, -9530 appiness. - - - -“ -9531 a moment, and th -9532 Journal._ - - - -_2 -9533 here’s somethi -9534 * - - - -On 1 -9535 * * - - - -_2 -9536 ed for an instan -9537 August._ - - - -“ -9538 - - - -“I shall b -9539 - - - - - -_Telegram, -9540 I don’t know; -9541 to him, and he -9542 EWARD’S DIARY - -9543 WARD’S DIARY - - -9544 RD’S DIARY - - - - -9545 D’S DIARY - - - - - -9546 ’S DIARY - - - - - - -9547 ��S DIARY - - - - - -_ -9548 - - - - -“Dr. Van -9549 - - - -“Dr. Van H -9550 - - -“Dr. Van He -9551 - -“Dr. Van Hel -9552 � - - - -“I thank -9553 - - -“Oh, it is -9554 and Mr. Morris -9555 Journal._ - - - -_3 -9556 I don’t want a -9557 don’t want an -9558 � - - - -“A little -9559 a minute, and th -9560 . - - - -“I do not -9561 he names of the -9562 � - - - -“I know t -9563 don’t you see -9564 What’s that?� -9565 that’s the way -9566 ouldn’t make n -9567 uldn’t make no -9568 t ain’t no use -9569 ain’t no use -9570 warn’t going -9571 -“Well, I’ll -9572 He says: - - - -“I -9573 , and by-and-by -9574 ckson’s Island -9575 .” - - - -I didn� -9576 Jim says: - - - -“ -9577 he - -says: - - - -“ -9578 didn’t seem to -9579 I ain’t going -9580 o I says: - - - -“ -9581 don’t.” - - - - -9582 I don’t know.� -9583 t I says: - - - -“ -9584 - - - -“Mr. Darcy -9585 dear Mr. Bennet -9586 o answer. - - - -“ -9587 that Mr. Bingle -9588 �� - - - -Mr. Darcy -9589 .” - - - -Mr. Darc -9590 nishment. - - - -“ -9591 � - - - -“I have b -9592 � - - - -“I am afr -9593 . - - - -Mr. Collins -9594 said he, “but -9595 said he, “but -9596 izabeth; “but -9597 nel Fitzwilliam, -9598 � - - - -“I unders -9599 Mr. Jabez Wilson -9600 Holmes, “that -9601 Holmes, “that -9602 � - - - -“Very goo -9603 - - - -“Very good -9604 xe-Coburg Square -9605 d Holmes, laughi -9606 shoulder. - - - -“ -9607 e asked. - - - -“� -9608 . THE ADVENTURE -9609 THE ADVENTURE O -9610 THE ADVENTURE OF -9611 HE ADVENTURE OF -9612 E ADVENTURE OF T -9613 ADVENTURE OF TH -9614 ADVENTURE OF THE -9615 DVENTURE OF THE -9616 � - - - -“I will d -9617 - - - -“I will do -9618 Lord St. Simon, -9619 !” - - - -Mr. Lorr -9620 . - - - -Mr. Crunche -9621 ss Pross. - - - -“ -9622 ss Pross, “and -9623 � croaked Jacque -9624 croaked Jacques -9625 arles Evrémonde -9626 rles Evrémonde, -9627 émonde, called -9628 ! Ting-a-ling-li -9629 Ting-a-ling-lin -9630 said Tom. - - - -“ -9631 I. THE ADVENTURE OF THE -9632 � - - - -“Of course -9633 MINA HARKER.” - - - -9634 INA HARKER.” - - - - -9635 R’S JOURNAL - - - - - -9636 .” - - - -Mr. Bennet -9637 d Holmes, “that -9638 .” - - - -Mr. Crunch -9639 arles Evrémonde, -9640 rles Evrémonde, c -9641 I. THE ADVENTURE OF TH -9642 I - - - -JONATHAN HARKER� -9643 ��MINA HARKER.” - - - - -9644 I. THE ADVENTURE OF T -9645 ***** - - - -“I -9646 I don’t know th -9647 - - - -“Of course, -9648 � - - - -“I beg you -9649 � - - - -“I wish I -9650 - - - -“Is that al -9651 I don’t like th -9652 was gone, and the -9653 - -“Yes, that’ -9654 � - - - -“I think I -9655 ould not, could n -9656 didn’t know ho -9657 [Illustra -9658 � I said, “you -9659 �� - - - - - -_Telegram -9660 he said, with a -9661 I don’t wonder -9662 MINA HARKER.” - - -9663 INA HARKER.” - - - -9664 NA HARKER.” - - - - -9665 - -“But, my dear -9666 II - - - -DR. SEWARD� -9667 that Mrs. Westenr -9668 - - - -“Is it poss -9669 a minute, and the -9670 the names of the -9671 , don’t you see -9672 I don’t know. I -9673 ouldn’t make ou -9674 ouldn’t stand i -9675 - - -“Oh, certain -9676 - - -“Oh, that’ -9677 didn’t - -want t -9678 didn’t believe -9679 � - - - -“I hain’ -9680 � - - - -“Never you -9681 didn’t seem to -9682 ��Yes’m.” - - - - -9683 didn’t want no -9684 : - - - -“I don’t -9685 he says: - - - -“H -9686 ?” - - - -I says: - - -9687 he says, kind of -9688 e ain’t got no -9689 ?” - - - -I didn’ -9690 what’s the mat -9691 � - - - -“Never min -9692 � I says. - - - -“W -9693 warn’t nobody -9694 didn’t have no -9695 ouldn’t a done -9696 and says: - - - -“O -9697 I don’t think I -9698 there’s one thi -9699 and Mrs. Bennet -9700 dear Mr. Bennet, -9701 .” - - - -Mr. Benne -9702 said Mr. Bennet; -9703 � - - - -“I shall b -9704 . - - - -“I have no -9705 � - - - -“I think y -9706 lizabeth, though -9707 said he, smiling -9708 t length, however -9709 I asked. - - - -“� -9710 d Holmes, “that -9711 shrugged his sho -9712 ville St. Clair, -9713 � - - - -“Never.” -9714 I think, Watson, -9715 and Mrs. Rucastl -9716 r. Lorry. - - - -“I -9717 p for it, weep fo -9718 it, weep for it! -9719 .” - - - -Mr. Crunc -9720 cried Mr. Lorry, -9721 farge’s wine-sh -9722 Equality, Fratern -9723 aternity, or Deat -9724 rles Evrémonde, -9725 les Evrémonde, c -9726 tizen Evrémonde, -9727 ss Pross, “and -9728 I - - - -JONATHAN HARKER -9729 ��MINA HARKER.” - - - -9730 �MINA HARKER.” - - - - -9731 RD’S DIARY--_conti -9732 I. THE ADVENTURE OF -9733 ***** - - - -“I -9734 together. - - - -“ -9735 herself. - - - -“ -9736 can’t remembe -9737 -“I’m afraid -9738 I don’t know t -9739 - - -“Of course, -9740 � - - - -“I beg yo -9741 don’t like it -9742 - - - -“Is that a -9743 - - -“Is that al -9744 � - - - -“I have a -9745 was gone, and th -9746 as gone, and the -9747 ersation. - - - -“ -9748 . - - - -“Did you -9749 e house, and the -9750 a little, and th -9751 e Hatter. - - - -“ -9752 - -“Yes, that� -9753 -“Yes, that’ -9754 you’d better -9755 appeared. - - - -“ -9756 don’t remembe -9757 -“Well, there -9758 ould not, could -9759 uld not, could n -9760 ouldn’t get th -9761 surprise. - - - -“ -9762 didn’t know ho -9763 the King. - - - -“ -9764 [Illustr -9765 [Illustra -9766 passed, and the -9767 the Borgo Pass. -9768 � I said, “you -9769 I said, “you -9770 �� - - - - - -_Telegra -9771 � - - - - - -_Telegram -9772 * * - - - -_1 -9773 .” - - - -I could -9774 he said, with a -9775 I don’t wonder -9776 don’t wonder -9777 station-master -9778 strait-waistcoa -9779 MINA HARKER.” - -9780 INA HARKER.” - - -9781 NA HARKER.” - - - -9782 A HARKER.” - - - - -9783 - - - -“I want yo -9784 - -“But, my dea -9785 -“But, my dear -9786 before, and tha -9787 window, and the -9788 II - - - -DR. SEWARD -9789 that Mrs. Westen -9790 hat Mrs. Westenr -9791 the time, and th -9792 attracted by the -9793 shoulder, and sa -9794 “Well, I have -9795 - - -“No, no, no -9796 - - - -“I know it -9797 he said, “the -9798 ” - - - -Our visit -9799 his face. - - - -“ -9800 - - -“Oh, no, no -9801 and Dr. Seward -9802 - - - -“Is it pos -9803 - - -“Is it poss -9804 Don’t you kno -9805 ” - - - -I thought -9806 - - - -“Excuse me -9807 ain’t no sens -9808 didn’t think o -9809 husband. - - - -“ -9810 ed in an instant -9811 morning, however -9812 the names of the -9813 fellow-creature -9814 the door, and I -9815 . Nothing could -9816 r God’s sake, -9817 - - - -“I am sorr -9818 , don’t you se -9819 darkness. - - - -“ -9820 that’s enough -9821 don’t know. I -9822 ouldn’t make o -9823 uldn’t make ou -9824 uldn’t stand i -9825 - - -“Oh, certai -9826 - -“Oh, certain -9827 - - -“Oh, that� -9828 - -“Oh, that’ -9829 didn’t - -want -9830 didn’t - -want t -9831 didn’t believ -9832 warn’t anythin -9833 a Sunday-school -9834 didn’t see no -9835 don’t reckon -9836 He says: - - - -“W -9837 � - - - -“I hain� -9838 sung out: - - - -“ -9839 ouldn’t let me -9840 � - - - -“Never yo -9841 - - - -“Never you -9842 -“Well, I reck -9843 runaway nigger -9844 didn’t seem t -9845 ��Yes’m.” - - - -9846 �Yes’m.” - - - - -9847 didn’t want n -9848 didn’t want no -9849 the wigwam and -9850 : - - - -“I don’ -9851 ?” - - - -I says: - -9852 he says, kind o -9853 he says, kind of -9854 e ain’t got no -9855 ?” - - - -I didn� -9856 , I says: - - - -“ -9857 what’s the ma -9858 what’s the mat -9859 what’s the use -9860 � - - - -“Never mi -9861 I says. - - - -“W -9862 warn’t nobody -9863 warn’t nobody -9864 didn’t have n -9865 didn’t have no -9866 awhile, and the -9867 ; and by-and-by -9868 � - - - -“I ain’ -9869 ouldn’t a done -9870 uldn’t a done -9871 runaway - -nigger -9872 nd says: - - - -“O -9873 - - -“Oh, shucks -9874 She says: - - - -“ -9875 runaway nigger, -9876 e runaway nigger -9877 there’s one th -9878 here’s one thi -9879 thirty-seven ye -9880 morning, and th -9881 however, that th -9882 divided between -9883 s of Mr. Collins -9884 and Mrs. Bennet -9885 dear Mr. Bennet, -9886 .” - - - -Mr. Benn -9887 said Mr. Bennet; -9888 smiling. - - - -“ -9889 to dine with th -9890 . - - - -“I have n -9891 - - - -“I think y -9892 lizabeth, though -9893 izabeth, though -9894 said he, smilin -9895 said he, smiling -9896 t length, howeve -9897 length, however -9898 - -“Very well. -9899 to Mrs. Collins -9900 � - - - -“I have h -9901 � - - - -“I have s -9902 nce more. - - - -“ -9903 I asked. - - - -“� -9904 d Holmes, “tha -9905 - -“Very good. -9906 -“Well, it is -9907 shrugged his sh -9908 shrugged his sho -9909 . - - - -“I have s -9910 - - - -“I have se -9911 ville St. Clair, -9912 ille St. Clair, -9913 � - - - -“Never.� -9914 - - - -“Never.” -9915 I think, Watson -9916 Mr. Henry Baker -9917 uller’s-earth -9918 -Lord St. Simon -9919 Lord St. Simon, -9920 and Mrs. Rucast -9921 and Mrs. Rucastl -9922 - - - -“Is there -9923 - - - -“I am goin -9924 . Lorry. - - - -“I -9925 p for it, weep f -9926 for it, weep fo -9927 it, weep for it -9928 it, weep for it! -9929 .” - - - -Mr. Crun -9930 cried Mr. Lorry, -9931 ried Mr. Lorry, -9932 farge’s wine-s -9933 arge’s wine-sh -9934 Liberty, Equalit -9935 Equality, Frater -9936 quality, Fratern -9937 aternity, or Dea -9938 ternity, or Deat -9939 les Evrémonde, -9940 es Evrémonde, c -9941 izen Evrémonde, -9942 s Pross, “and -9943 “showed off” -9944 xclaimed: - - - -“ -9945 aid Huck. - - - -“ -9946 I - - - -JONATHAN HARKE -9947 - - - -“MINA HARKER. -9948 ��MINA HARKER.” - - -9949 �MINA HARKER.” - - - -9950 VAN HELSING.” - - - - -9951 RD’S DIARY--_cont -9952 D’S DIARY--_conti -9953 � - - - -“_I_ don’t -9954 .” - - - -Mr. Bennet -9955 I. THE ADVENTURE OF -9956 ***** - - - -“I -9957 terpillar. - - - -“W -9958 I - - - -JONATHAN HARK -9959 Holmwood._ - - - -“_ -9960 - - - -“MINA HARKER -9961 - - -“MINA HARKER. -9962 ��MINA HARKER.” - -9963 �MINA HARKER.” - - -9964 �� - - - - - -_Telegram, -9965 VAN HELSING.” - - - -9966 AN HELSING.” - - - - -9967 RD’S DIARY--_con -9968 D’S DIARY--_cont -9969 ’S DIARY--_conti -9970 Journal._ - - - -_30 -9971 of the _Czarina Ca -9972 . Saville, England -9973 � - - - -“_I_ don’ -9974 - - - -“_I_ don’t -9975 t” - - - -[_Copyrigh -9976 .” - - - -Mr. Darcy -9977 .” - - - -[_Copyrigh -9978 I. THE ADVENTURE O -9979 ” - - - -Mr. Lorry w -9980 ***** - - - -“I -9981 - -through the doo -9982 I can’t remembe -9983 -“I’m afraid -9984 erpillar. - - - -“W -9985 , won’t you?” -9986 secondly, because -9987 he house, and the -9988 a little, and the -9989 about it, and the -9990 - - -“No, I didn� -9991 - - -“Don’t be -9992 I don’t remembe -9993 � - - - -“I’ll te -9994 you needn’t be -9995 - - -“No, indeed, -9996 Will you, won’t -9997 will you, won’t -9998 . - - - -“I should -9999 tiful Soo—oop! -