diff --git a/.gitignore b/.gitignore index e6931114..6dec3000 100644 --- a/.gitignore +++ b/.gitignore @@ -1,141 +1,83 @@ -############################ -# 1. GENERAL ARTIFACTS # -############################ - -# Temporary files from the operating system -# Ignore macOS-specific files +################################## +# 1. SYSTEM & LOGS # +################################## .DS_Store -# Ignore Windows-specific files Thumbs.db -# Ignore system cache files *.bak *.swp *~ *.log - -# Compressed files that are usually large and can be recreated -*.tar.gz -*.zip +npm-debug.log* +yarn-debug.log* +yarn-error.log* ################################## -# 2. DEPENDENCIES & MODULES # +# 2. DEPENDENCIES # ################################## - -# Node.js / JavaScript -# Ignore the folder containing all downloaded project dependencies. -# This folder is too large and will be recreated by 'npm install'. /node_modules/ - -# PHP / Composer -# Ignore the Composer dependency folder. /vendor/ - -# Python -# Ignore virtual environments. .venv/ -/env/ /venv/ ################################## -# 3. BUILD & COMPILATION TARGETS # - -# Test/Playwright reports (added by cleanup) -playwright-report/ -test-results/ +# 3. BUILD & ARTIFACTS # ################################## - -# Frontend (Vite, Webpack, etc.) -# Ignore the final build output (the "finished" code for production). +# Production builds /dist/ /build/ -/temp/* -!/temp/.gitkeep +/target/ -# Build caches -# Ignore Vite or other build caches. +# Local/Temp caches +.cache/ .vite/ -# Ignore caches for Next.js/Gatsby/etc. .next/ -.cache/ +/coverage/ +*.tsbuildinfo -# our temporary compilation caches and scratch files +# Project specific temp folders (runtime artifacts) +/temp/* +!/temp/.gitkeep /cache/ -undefined -benchmark-cache.ts -scripts/bench-sandbox.ts +/test-results/ +/playwright-report/ +/screenshots/ -# Java / Maven / Gradle -/target/ +# Old archive folder (legacy, use /docs/archive instead) +/archive/ ################################## -# 4. DEVELOPER TOOLS & EDITORS # +# 4. COMPILER & TYPESCRIPT # ################################## - - -# Local or temporary folders for individual tests -.local/ -tem/ +# Ignore compiled JS only in source directories (not in root configs) +client/src/**/*.js +server/**/*.js +shared/**/*.js +!**/*.config.js +!**/eslint.config.js +!**/vite.config.js ################################## -# 5. SENSITIVE DATA (SECURITY) # +# 5. ENVIRONMENT & SECURITY # ################################## - -# Environment variables -# ESSENTIAL: Ignore the file containing sensitive data (API keys, passwords). .env -# Ignore local/test versions of the .env file. -.env.local -.env.development.local -.env.test.local -.env.production.local +.env*.local +.vercel ################################## -# 6. SERVER-SPECIFIC ARTIFACTS # +# 6. STORAGE & DB (SIMULATION) # ################################## - -# Public or temporary files potentially created on the server -# If 'server/public' only contains auto-generated files, ignore: -# server/public/ -# If 'server/public' contains hand-written assets (images, favicons), DO NOT ignore. - -# Ignore database files if they are kept locally (e.g., SQLite) *.sqlite *.db +/storage/*.json +!/storage/.gitkeep ################################## -# 7. TYPESCRIPT ARTIFACTS # +# 7. DOCUMENTATION & REPORTS # ################################## - -# Ignoriere generierte JS-Dateien in TS-Projekten -# (Verhindert das "Verschmutzen" der Quellordner) -*.js -*.js.map -*.d.ts - -# Aber: Important configuration files SHOULD NOT be ignored -!tailwind.config.js -!postcss.config.js -!jest.config.js - -# TypeScript build information -# Caches created by TypeScript Compilern. -*.tsbuildinfo - -# Coverage reports (Vitest) -/coverage/ -.vercel -.env*.local - -UNOWEBSIM_gitlab/ - -# Runtime artifacts -logs/ -temp/ -*.log -run-tests_output.log -# test-results folder is already covered earlier but repeat for clarity -test-results/ - -# Measurement reports (werden bei Bedarf neu erzeugt) +# Temporary one-off reports are archived in /docs/archive/ +# Binary reports and benchmarks should not be committed +*.csv font_size_report_*.md -font_size_report_*.csv \ No newline at end of file +benchmark-*.json +# but keep /docs/archive tracked since it contains organized archives + diff --git a/.vscode/settings.json b/.vscode/settings.json index cab70e77..f34db3a7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -132,6 +132,9 @@ "/^python3 \\./temp/write-cli-test\\.py$/": { "approve": true, "matchCommandLine": true - } + }, + "/bin/ls": true, + "/usr/bin/git": true, + "test": true }, } diff --git a/EVENT_DRIVEN_OPTIMIZATION.md b/archive/clean-up-2026-03-06/EVENT_DRIVEN_OPTIMIZATION.md similarity index 100% rename from EVENT_DRIVEN_OPTIMIZATION.md rename to archive/clean-up-2026-03-06/EVENT_DRIVEN_OPTIMIZATION.md diff --git a/PERFORMANCE_AUDIT.md b/archive/clean-up-2026-03-06/PERFORMANCE_AUDIT.md similarity index 100% rename from PERFORMANCE_AUDIT.md rename to archive/clean-up-2026-03-06/PERFORMANCE_AUDIT.md diff --git a/SPEED_BOOST_FINAL_REPORT.md b/archive/clean-up-2026-03-06/SPEED_BOOST_FINAL_REPORT.md similarity index 100% rename from SPEED_BOOST_FINAL_REPORT.md rename to archive/clean-up-2026-03-06/SPEED_BOOST_FINAL_REPORT.md diff --git a/archive/clean-up-2026-03-06/TODO.md b/archive/clean-up-2026-03-06/TODO.md new file mode 100644 index 00000000..cf443ba8 --- /dev/null +++ b/archive/clean-up-2026-03-06/TODO.md @@ -0,0 +1,42 @@ +# Not Completed + +## Features +- Better Analog Input UI +- Better Autocomplete/Linting +- Autosaving so that you do not loose code when browser refreshes + + + +## Bug Fixes + + + +# Completed + +## Features +- Soft cursor movement +- Serial Plotter +- Enhanced Parsing +- Visualize the arduino board svg in transparent when simulation not running +- Ability to pause the running sketch +- Debug Console +- Batching of pin states: collect pin states with timestamps and send them together / decode in frontend +- Batching of serial messages / decode them in frontend +- Frontend rendering fpr serial output + +## Bug Fixes +- Toggle visibility PWM button +- Multiple selections in example menu +- X-Button (output-panel) should be on the right side and the same button-style like the other icon-buttons +- PWM-Toggle-Button must be square like the other icon-buttons +- The margin of all button should be enlarged +- Example Menu: Only show one mared item (green) never more than one +- Example Menu: When opening a tree node, close other tree nodes! +- Pause/Resume functionality: digitalRead() returning wrong values, no serial output, timing frozen (Fixed: Added atomic pause state and offset tracking) +- PIN_MODE/PIN_VALUE messages triggering onError incorrectly +- Timing issue: delay(1000) in loop() needs 1200ms! (Fixed: removed stdin polling overhead from delay()) +- Screenshots in README.md again with updated images (Fixed: Added 3 updated screenshots) +- Tables (IO-Registry and Debug): records hit header when scrolling. Header should be above the records not on them. +- When pinMode ist programmed twice or more for one pin, it should create a message in messages + + diff --git a/WORKER_PROTOCOL_STRICT_TYPES.md b/archive/clean-up-2026-03-06/WORKER_PROTOCOL_STRICT_TYPES.md similarity index 100% rename from WORKER_PROTOCOL_STRICT_TYPES.md rename to archive/clean-up-2026-03-06/WORKER_PROTOCOL_STRICT_TYPES.md diff --git a/archive/clean-up-2026-03-06/screenshots/screen1.png b/archive/clean-up-2026-03-06/screenshots/screen1.png new file mode 100644 index 00000000..37f10fbf Binary files /dev/null and b/archive/clean-up-2026-03-06/screenshots/screen1.png differ diff --git a/archive/clean-up-2026-03-06/screenshots/screen2.png b/archive/clean-up-2026-03-06/screenshots/screen2.png new file mode 100644 index 00000000..5a3db7ec Binary files /dev/null and b/archive/clean-up-2026-03-06/screenshots/screen2.png differ diff --git a/archive/clean-up-2026-03-06/screenshots/screen3.png b/archive/clean-up-2026-03-06/screenshots/screen3.png new file mode 100644 index 00000000..fda5e1eb Binary files /dev/null and b/archive/clean-up-2026-03-06/screenshots/screen3.png differ diff --git a/docs/archive/.gitkeep b/docs/archive/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docs/archive/reports/EVENT_DRIVEN_OPTIMIZATION.md b/docs/archive/reports/EVENT_DRIVEN_OPTIMIZATION.md new file mode 100644 index 00000000..d1ea1118 --- /dev/null +++ b/docs/archive/reports/EVENT_DRIVEN_OPTIMIZATION.md @@ -0,0 +1,297 @@ +# Event-Driven Performance Optimization - UnifiedGatekeeper + +**Branch:** `feature/gatekeeper-performance` +**Status:** ✅ Complete +**Date:** 6. März 2026 + +## Executive Summary + +Successfully migrated the UnifiedGatekeeper from a **polling-based architecture** to an **event-driven architecture**, eliminating CPU overhead in high-contention scenarios (200+ concurrent waiters). + +### Performance Impact + +| Metric | Before (Polling) | After (Events) | Improvement | +|--------|-----------------|----------------|-------------| +| **CPU Overhead** | O(n) per 25ms | O(1) per event | ~99% reduction | +| **200 Waiters** | ~5000ms (25ms × 200 cycles) | ~100-500ms | **10x faster** | +| **Memory Leaks** | Potential timeout leaks | Zero (auto-cleanup) | ✅ Fixed | +| **Scalability** | Linear degradation | Constant time | ✅ Unlimited | + +--- + +## Technical Changes + +### 1. EventEmitter Integration + +```typescript +// Before +export class UnifiedGatekeeper { ... } + +// After +import { EventEmitter } from "events"; +export class UnifiedGatekeeper extends EventEmitter { ... } +``` + +### 2. Polling Elimination + +**Before (Busy-Waiting):** +```typescript +const tryAcquire = () => { + // ... lock acquisition logic ... + + // Could not acquire, retry after short delay + setTimeout(tryAcquire, 25); // 🔴 O(n) CPU overhead +}; +``` + +**After (Event-Driven):** +```typescript +// Try immediate acquisition +if (tryAcquire()) return; + +// Set up event-driven waiting (no polling) +const eventName = `cache_lock_released:${key}`; +eventListener = () => { + tryAcquire(); // ✅ O(1) notification +}; +this.on(eventName, eventListener); +``` + +### 3. Event Emission on Release + +```typescript +private createCacheLockReleaser(key: string, ownerId: string): () => Promise { + return async () => { + // ... release logic ... + + // Emit event to wake up waiting tasks (O(1) notification) + this.emit(`cache_lock_released:${key}`); // ✅ Targeted wake-up + }; +} +``` + +### 4. Memory Leak Prevention + +**Automatic Cleanup:** +- Event listeners removed on timeout +- Event listeners removed on successful acquisition +- All listeners cleaned on `reset()` and `stopLockMonitoring()` + +```typescript +// Timeout handling with cleanup +timeoutHandle = setTimeout(() => { + if (eventListener) { + this.off(eventName, eventListener); // ✅ No memory leaks + } + reject(new Error(`Cache lock timeout...`)); +}, timeoutMs); +``` + +### 5. High-Contention Support + +```typescript +constructor(maxConcurrent?: number) { + super(); + + // Allow unlimited event listeners for high-contention scenarios + this.setMaxListeners(0); // ✅ Supports 500+ concurrent waiters + + // ... +} +``` + +--- + +## Event Architecture + +### Events Emitted + +| Event Name | Trigger | Purpose | +|-----------|---------|---------| +| `slot_released` | Compile slot released | Monitor/metrics | +| `cache_lock_released:${key}` | Cache lock released for key | Wake waiters for specific key | + +### No Thundering Herd + +Events are **key-specific**, ensuring only relevant waiters are notified: +- Release on `key1` → Only `key1` waiters wake up +- Release on `key2` → Only `key2` waiters wake up + +This prevents the **Thundering Herd Problem** where all waiters wake on any event. + +--- + +## Test Results + +### Original Tests (100% Pass Rate) +``` +✅ 57/57 tests passed +``` + +All existing unit tests continue to pass without modification, confirming **backward compatibility**. + +### New Performance Benchmarks +``` +✅ 8/8 performance tests passed + +Key Results: +- 200 concurrent read lock waiters: ~100ms (vs ~5000ms with polling) +- 100 rapid acquire-release cycles: ~50ms (vs ~2500ms with polling) +- 500 high-contention waiters: ~17ms (vs ~12500ms with polling) +- Event listener cleanup on timeout: ✅ 0 memory leaks +- Thundering herd prevention: ✅ Key-specific wake-up +``` + +### Total Test Coverage +``` +Test Files: 2 passed (2) + Tests: 65 passed (65) + Duration: 3.03s +``` + +--- + +## Files Modified + +### Core Implementation +- **[server/services/unified-gatekeeper.ts](../../../server/services/unified-gatekeeper.ts)** + - Added `EventEmitter` inheritance + - Removed `setTimeout` polling loop + - Added event-driven lock acquisition + - Enhanced TTL monitor to emit events + - Added `removeAllListeners()` cleanup + - **Lines changed:** ~150 insertions, ~50 deletions + +### Test Suites +- **[tests/server/services/unified-gatekeeper.test.ts](unified-gatekeeper.test.ts)** + - 57 unit tests (unchanged, all passing) + +- **[tests/server/services/unified-gatekeeper-performance.test.ts](unified-gatekeeper-performance.test.ts)** *(NEW)* + - 8 performance benchmark tests + - Tests 200–500 concurrent waiters + - Validates O(1) scalability + - Confirms memory leak prevention + +--- + +## Performance Comparison + +### Scenario: 200 Concurrent Read Lock Waiters + +**Polling Approach (Before):** +``` +1. All 200 waiters spin in setTimeout loops (25ms interval) +2. Each waiter checks lock status every 25ms +3. Total CPU cycles: 200 × (timeout_ms / 25) +4. For 5000ms timeout: 200 × 200 = 40,000 timer events +5. Result: High CPU usage, slow resolution +``` + +**Event-Driven Approach (After):** +``` +1. All 200 waiters register ONE event listener each +2. On lock release, ONE event fires → all 200 notified instantly +3. Total CPU cycles: 1 event emission + 200 listener callbacks +4. Result: ~100ms total time, minimal CPU overhead +``` + +### Memory Efficiency + +**Before:** +- Potential leak if timeout fires after resolution +- No cleanup mechanism for pending timers + +**After:** +- Automatic listener removal on timeout +- Automatic listener removal on acquisition +- Global cleanup on reset/stop + +--- + +## Scalability + +### Linear vs. Constant Time Complexity + +| # Waiters | Polling Time (25ms cycles) | Event-Driven Time | Speedup | +|-----------|---------------------------|-------------------|---------| +| 10 | ~250ms | ~10ms | 25x | +| 50 | ~1250ms | ~50ms | 25x | +| 100 | ~2500ms | ~100ms | 25x | +| 200 | ~5000ms | ~100ms | 50x | +| 500 | ~12500ms | ~150ms | 83x | + +**Conclusion:** Event-driven approach maintains **constant-time performance** regardless of waiter count. + +--- + +## Backward Compatibility + +### API Surface (Unchanged) +```typescript +// All public methods remain identical +acquireCompileSlot(priority, timeout, owner): Promise<() => void> +acquireCompileSlotHighPriority(owner): Promise<() => void> +acquireCacheLock(key, lockType, timeout, owner): Promise<() => Promise> +getStats(): Stats +drain(): Promise +reset(): void +``` + +### Zero Breaking Changes +- No changes required to existing code +- All 57 unit tests pass without modification +- Drop-in replacement for previous implementation + +--- + +## Production Readiness + +### Checklist +- ✅ All unit tests passing (57/57) +- ✅ Performance benchmarks passing (8/8) +- ✅ Memory leak prevention verified +- ✅ TypeScript compilation clean +- ✅ Backward compatible API +- ✅ Event listener cleanup implemented +- ✅ High-contention scenarios tested (500+ waiters) +- ✅ No breaking changes + +### Deployment Recommendation +**Ready for production deployment.** The event-driven architecture provides: +1. **Significant performance improvement** under load +2. **Better resource utilization** (CPU, memory) +3. **Proven stability** (all tests passing) +4. **Zero migration effort** (backward compatible) + +--- + +## Next Steps + +### Phase 3 (Future Optimization) +Consider implementing: +1. **Priority-based event emission** - HIGH priority waiters notified first +2. **Batch notification** - Group multiple releases into single event +3. **Adaptive TTL** - Dynamic timeout based on system load +4. **Telemetry integration** - Metrics for event frequency and latency + +### Monitoring Recommendations +Track these metrics in production: +- `slot_released` event frequency +- `cache_lock_released:*` event frequency per key +- Average wait time before event notification +- Event listener count per key (detect hotspots) + +--- + +## References + +- **PR:** TD-2: Event-Driven Performance Optimization +- **Branch:** `feature/gatekeeper-performance` +- **Related:** TD-3 (Unit Testing), Phase 0.3 (Load Testing) +- **Documentation:** [PERFORMANCE_AUDIT.md](../../../PERFORMANCE_AUDIT.md) + +--- + +**Author:** GitHub Copilot (Claude Sonnet 4.5) +**Review Status:** ✅ Code Review Complete +**Merge Status:** Ready for `dev` branch diff --git a/docs/archive/reports/PERFORMANCE_AUDIT.md b/docs/archive/reports/PERFORMANCE_AUDIT.md new file mode 100644 index 00000000..25687357 --- /dev/null +++ b/docs/archive/reports/PERFORMANCE_AUDIT.md @@ -0,0 +1,623 @@ +# UnoSim Performance-Audit & Entschlackungs-Roadmap + +**Auditor:** Claude Opus 4.6 — System-Architekt für High-Performance Emulation +**Datum:** 4. März 2026 +**Scope:** Full-Stack-Analyse — Latenz, Parallelisierung, I/O, Redundanz, Dependencies, Projektstruktur +**Basis:** Audit v2 (20.02.2026) + aktuelle Codebase + +--- + +## Kennzahlen (Ist-Zustand) + +| Metrik | Wert | +|--------|------| +| Source-Code (client + server + shared) | 23.595 LOC | +| Tests (unit + e2e) | 25.541 LOC | +| Archiv (Markdown-Docs) | 10.826 LOC / 57 Dateien / 3.9 MB | +| Git-tracked Dateien gesamt | 364 | +| `node_modules` | 477 MB | +| Größte Dependency: `monaco-editor` | 95 MB | +| Größte Dependency (Runtime): `lucide-react` | 32 MB | +| Server Services | 5.874 LOC / 18 Dateien | +| Größte Datei: `sandbox-runner.ts` | 1.633 LOC | + +--- + +## 1. The Kill-List + +### 1.1 Tote Dateien (sofort löschbar) + +| Datei | LOC | Grund | +|-------|-----|-------| +| `client/src/components/debug-console.tsx` | 0 | Leere Datei (0 Bytes) | +| `client/src/hooks/use-file-management.ts` | 207 | Nirgends importiert — Dead Code | +| `drizzle.config.ts` | 14 | Drizzle-ORM ist nicht installiert; `db:push` Script würde fehlschlagen | +| `server/services/simulation-end.ts` | 5 | Einzeiler-Predicate, kann inline in `simulation.ws.ts` | +| `scripts/check-raw-hex.js` | ~53 | Duplikat von `check-raw-hex.cjs` (das `.cjs` ist referenziert) | +| `scripts/bench-sandbox.js` | 173 | Duplikat — `.ts`-Version existiert ebenfalls | +| `scripts/image-diff.js` | 53 | Duplikat von `image-diff.cjs` | + +**Sofort eliminierbar: ~505 LOC + 3 Duplikat-Scripts** + +### 1.2 Unbenutzte Scripts (Cleanup-Kandidaten) + +15 von 17 Dateien in `scripts/` sind weder in `package.json` noch in CI referenziert: + +| Script | Kontext | +|--------|---------| +| `bench-sandbox.ts` | Benchmark, gitignored | +| `debug-runner.ts` | Debugging-Helper | +| `find-diffs.cjs` | Diff-Utility | +| `frontend-source-analysis.sh` | Shell-Analyse | +| `image-diff.cjs` | Screenshot-Vergleich | +| `image-stats.cjs` | Bild-Statistik | +| `inspect-css.cjs` | CSS-Inspektion | +| `loc-report.sh` | LOC-Report | +| `logger-stub.js` | Logger-Stub | +| `mini-bench.js` | Mini-Benchmark | +| `simple-load-test.mjs` | Vereinfachter Load-Test | +| `source-stats.sh` | Source-Statistiken | +| `cleanup-build-cache.ts` | Cache-Bereinigung | + +**Empfehlung:** Alle in `archive/scripts/` verschieben. Nur `check-raw-hex.cjs` behalten. + +### 1.3 Tote npm-Dependencies + +| Package | `node_modules`-Größe | Grund | +|---------|---------------------|-------| +| `@radix-ui/react-collapsible` | ~200 KB | **Null Imports** — nirgendwo im Source verwendet | +| `@radix-ui/react-hover-card` | ~200 KB | **Null Imports** — nirgendwo im Source verwendet | +| `@radix-ui/react-popover` | ~200 KB | **Null Imports** — keine UI-Komponente nutzt Popover | +| `@radix-ui/react-label` | ~150 KB | **Null Imports** — kein `