diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts b/apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts new file mode 100644 index 000000000..6fa651773 --- /dev/null +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts @@ -0,0 +1,357 @@ +import type { JarvisCommand, JarvisSceneState } from '@accomplish/shared'; + +const DEFAULT_OBJECT = 'reactor core'; + +function normalize(input: string): string { + return input.trim().replace(/\s+/g, ' ').toLowerCase(); +} + +function stripCommandPrefix(text: string): string { + return text + .replace(/^(please\s+)?(show me|load|render|summarize|summarise|explain|inspect|highlight|rotate|turn on|power on|assemble|explode|switch to|go to|open)\s+/i, '') + .trim(); +} + +function extractTarget(input: string): string | undefined { + const cleaned = stripCommandPrefix(input); + if (!cleaned) return undefined; + + const blocked = new Set(['it', 'that', 'this', 'the', 'scene', 'view']); + if (blocked.has(cleaned.toLowerCase())) return undefined; + + return cleaned.replace(/^the\s+/i, '').trim(); +} + +function extractDirection(input: string): JarvisCommand['direction'] { + if (/\bleft\b/i.test(input)) return 'left'; + if (/\bright\b/i.test(input)) return 'right'; + if (/\bup\b/i.test(input)) return 'up'; + if (/\bdown\b/i.test(input)) return 'down'; + return undefined; +} + +export function parseJarvisCommand(raw: string): JarvisCommand { + const input = normalize(raw); + const target = extractTarget(raw); + + if (!input) { + return { + intent: 'explain_scene', + target: DEFAULT_OBJECT, + raw, + }; + } + + if (/\b(create|generate|make|build|design|render)\b.{0,40}\b(3d|scene|model|sphere|cube|box|torus|ring|orb|shape|object|structure|object|rendering|composition)\b/i.test(input) + || /\b(3d|three.?d)\s+(render|scene|model|object|composition|version)\b/i.test(input)) { + return { + intent: 'create_scene', + target: target ?? 'custom scene', + query: raw.trim(), + raw, + }; + } + + if (/\b(brand|identity|palette|logo|visual identity|brand kit)\b/.test(input)) { + return { + intent: 'switch_to_brand', + target: target ?? DEFAULT_OBJECT, + mode: 'brand', + raw, + }; + } + + if (/\b(content|social|post|cards|feed|media)\b/.test(input)) { + return { + intent: 'switch_to_content', + target: target ?? DEFAULT_OBJECT, + mode: 'content', + raw, + }; + } + + if (/\b(analytics|metrics|data|chart|stats|kpi|numbers)\b/.test(input)) { + return { + intent: 'switch_to_analytics', + target: target ?? DEFAULT_OBJECT, + mode: 'analytics', + raw, + }; + } + + if (/\b(map|route|terrain|location|locations|nav|navigate)\b/.test(input)) { + return { + intent: 'switch_to_map', + target: target ?? DEFAULT_OBJECT, + mode: 'map', + raw, + }; + } + + if (/\bexplode|exploded\b/.test(input)) { + return { + intent: 'explode_part', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\bassemble|assembled|rebuild|collapse\b/.test(input)) { + return { + intent: 'assemble_part', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\b(turn on|power on|enable|activate|ignite|start)\b/.test(input)) { + return { + intent: 'power_on', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\b(turn off|power off|disable|deactivate|shutdown|shut down|stop)\b/.test(input)) { + return { + intent: 'power_off', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\bhighlight|label|tag\b/.test(input)) { + return { + intent: 'highlight_part', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\brotate|spin|orbit\b/.test(input)) { + return { + intent: 'rotate_view', + target: target ?? DEFAULT_OBJECT, + direction: extractDirection(raw), + raw, + }; + } + + if (/\b(summarize|summarise|what is|what's this|explain|describe|inspect)\b/.test(input)) { + return { + intent: 'summarize_object', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\b(load|show me|render|open)\b/.test(input)) { + return { + intent: 'load_object', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + return { + intent: 'explain_scene', + target: target ?? DEFAULT_OBJECT, + query: raw.trim(), + raw, + }; +} + +function catalogSummary(target: string): string { + const lookup = target.toLowerCase(); + + if (lookup.includes('reactor')) { + return 'High-energy core assembly with concentric containment rings, a bright central emitter, and service modules that can be exploded or reassembled on command.'; + } + + if (lookup.includes('engine')) { + return 'Mechanical power unit with layered housings, visible support geometry, and a state model that supports inspection, explosion, and reassembly.'; + } + + if (lookup.includes('map') || lookup.includes('city') || lookup.includes('route')) { + return 'Navigational surface with markers, route overlays, and contextual labels for quick orientation.'; + } + + return `Interactive 3D object for ${target}, ready for labels, state changes, and HUD-style annotations.`; +} + +export function applyJarvisCommand(state: JarvisSceneState, command: JarvisCommand): JarvisSceneState { + const target = command.target?.trim() || state.activeTarget || DEFAULT_OBJECT; + + switch (command.intent) { + case 'load_object': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + exploded: false, + powerOn: false, + highlightedPart: undefined, + mapFocus: undefined, + }; + case 'summarize_object': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + }; + case 'explode_part': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + exploded: true, + cameraDistance: 8.5, + }; + case 'assemble_part': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + exploded: false, + cameraDistance: 6.2, + }; + case 'power_on': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + powerOn: true, + }; + case 'power_off': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + powerOn: false, + }; + case 'highlight_part': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + highlightedPart: target, + }; + case 'rotate_view': + return { + ...state, + activeTarget: target, + rotationSpeed: command.direction === 'left' ? -0.015 : command.direction === 'right' ? 0.015 : state.rotationSpeed, + cameraDistance: typeof command.zoom === 'number' ? command.zoom : state.cameraDistance, + }; + case 'switch_to_brand': + return { + ...state, + mode: 'brand', + activeTarget: 'brand system', + summary: 'Visual identity elements — mark, palette, and type scale — rendered in 3D space.', + exploded: false, + }; + case 'switch_to_content': + return { + ...state, + mode: 'content', + activeTarget: 'content stack', + summary: 'Social and digital content cards layered in a depth composition.', + exploded: false, + }; + case 'switch_to_analytics': + return { + ...state, + mode: 'analytics', + activeTarget: 'analytics view', + summary: 'Performance metrics visualized as a 3D bar chart with trend line overlay.', + exploded: false, + }; + case 'create_scene': + return { + ...state, + mode: 'scene', + activeTarget: target || 'custom scene', + summary: 'AI-generated 3D composition — objects rendered from your description.', + exploded: false, + }; + case 'switch_to_map': + return { + ...state, + mode: 'map', + activeTarget: target, + summary: `Map mode focused on ${target}. Use annotate and route commands to refine the view.`, + mapFocus: target, + exploded: false, + }; + case 'annotate_map': + return { + ...state, + mode: 'map', + activeTarget: target, + mapFocus: target, + }; + case 'explain_scene': + default: + return { + ...state, + activeTarget: target, + summary: catalogSummary(target), + }; + } +} + +export function createInitialJarvisState(): JarvisSceneState { + return { + mode: 'object', + activeTarget: DEFAULT_OBJECT, + summary: catalogSummary(DEFAULT_OBJECT), + powerOn: false, + exploded: false, + rotationSpeed: 0.01, + cameraDistance: 6.2, + }; +} + +export function describeJarvisCommand(command: JarvisCommand, nextState: JarvisSceneState): string { + switch (command.intent) { + case 'load_object': + return `Loaded ${nextState.activeTarget}. ${nextState.summary}`; + case 'summarize_object': + return `${nextState.activeTarget}: ${nextState.summary}`; + case 'explode_part': + return `Exploding ${nextState.activeTarget}. Bringing the assembly into separated layers.`; + case 'assemble_part': + return `Assembling ${nextState.activeTarget}. Returning all layers to the compact state.`; + case 'power_on': + return `Powering on ${nextState.activeTarget}. Emissive state and energy effects are active.`; + case 'power_off': + return `Shutting down ${nextState.activeTarget}. All emissive output and energy effects offline.`; + case 'highlight_part': + return `Highlighting ${nextState.highlightedPart ?? nextState.activeTarget}.`; + case 'rotate_view': + return `Rotating the view${command.direction ? ` ${command.direction}` : ''}.`; + case 'create_scene': + return `Generating 3D scene for ${nextState.activeTarget}. Rendering objects from your description...`; + case 'switch_to_brand': + return 'Brand system loaded. Mark, palette, and type hierarchy are live.'; + case 'switch_to_content': + return 'Content stack loaded. Social and digital card formats are in view.'; + case 'switch_to_analytics': + return 'Analytics view loaded. Performance metrics and trend line active.'; + case 'switch_to_map': + return `Switched to map mode for ${nextState.mapFocus ?? nextState.activeTarget}.`; + case 'annotate_map': + return `Annotating the map around ${nextState.mapFocus ?? nextState.activeTarget}.`; + case 'explain_scene': + default: + return nextState.summary; + } +} diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx new file mode 100644 index 000000000..b8c4d95ea --- /dev/null +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx @@ -0,0 +1,1484 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { motion } from 'framer-motion'; +import * as THREE from 'three'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { Badge } from '@/components/ui/badge'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { + ArrowRight, + BarChart2, + Bot, + Box, + Clipboard, + Compass, + FileText, + Hash, + Mail, + Mic, + Palette, + RadioTower, + RotateCw, + Sparkles, + Telescope, + TrendingUp, + Wand2, + Zap, +} from 'lucide-react'; +import type { JarvisCommand, JarvisSceneState, JarvisTranscriptEntry } from '@accomplish/shared'; +import { + applyJarvisCommand, + createInitialJarvisState, + describeJarvisCommand, + parseJarvisCommand, +} from '@/lib/jarvis/command-parser'; +import { getAccomplish, isRunningInElectron } from '@/lib/accomplish'; + +type JarvisChip = { label: string; prompt: string; icon: React.ElementType; action?: () => void }; + +const OBJECT_FACTS: Record = { + 'reactor core': { + title: 'Reactor Core', + summary: 'Layered containment assembly — bright central emitter, concentric rings, and service modules.', + bullets: ['Containment rings', 'Central emitter', 'Service pods'], + }, + 'engine block': { + title: 'Engine Block', + summary: 'Structural power unit with layered housings and visible subassemblies.', + bullets: ['Cylinder bank', 'Cooling loops', 'Access panels'], + }, + 'city map': { + title: 'City Map', + summary: 'Navigational surface with route overlays, markers, and district labels.', + bullets: ['Route line', 'Marker pins', 'District labels'], + }, + 'brand system': { + title: 'Brand System', + summary: 'Visual identity in 3D — brand mark, color palette swatches, and type hierarchy.', + bullets: ['Brand mark', 'Color palette', 'Type scale'], + }, + 'content stack': { + title: 'Content Stack', + summary: 'Social and digital content cards floating in a depth-layered composition.', + bullets: ['Social cards', 'Post previews', 'Content grid'], + }, + 'analytics view': { + title: 'Analytics View', + summary: 'Performance metrics as a 3D bar chart with trend line overlay and KPI baseline.', + bullets: ['Bar chart', 'Trend line', 'KPI baseline'], + }, + 'custom scene': { + title: 'Generated Scene', + summary: 'Rendered by local Ollama — describe any scene and it renders live in the viewport.', + bullets: ['Custom objects', 'Ollama LLM', 'Live render'], + }, +}; + +function getFactForTarget(target: string) { + const key = target.toLowerCase(); + if (key.includes('reactor')) return OBJECT_FACTS['reactor core']; + if (key.includes('engine')) return OBJECT_FACTS['engine block']; + if (key.includes('map') || key.includes('city') || key.includes('route')) return OBJECT_FACTS['city map']; + if (key.includes('brand') || key.includes('identity') || key.includes('palette')) return OBJECT_FACTS['brand system']; + if (key.includes('content') || key.includes('social') || key.includes('card')) return OBJECT_FACTS['content stack']; + if (key.includes('analytics') || key.includes('metrics') || key.includes('chart')) return OBJECT_FACTS['analytics view']; + if (key.includes('scene') || key.includes('generated') || key.includes('custom')) return OBJECT_FACTS['custom scene']; + return { + title: target, + summary: `Interactive 3D object: ${target}. Supports load, inspect, explode, assemble, annotate.`, + bullets: ['Load', 'Inspect', 'Animate'], + }; +} + +function makeTranscriptId() { + return `jarvis_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; +} + +async function callAnthropicHaiku(apiKey: string, userText: string): Promise { + const response = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + 'content-type': 'application/json', + }, + body: JSON.stringify({ + model: 'claude-haiku-4-5-20251001', + max_tokens: 220, + system: + 'You are JARVIS, a creative HUD assistant with two modes. For 3D visual commands (explode, power on, load, rotate, switch to map/brand/content/analytics mode, etc.): 1-2 crisp sentences narrating the action — precise, slightly technical. For marketing or creative requests (write copy, taglines, headlines, hooks, captions, social posts, ad copy, email subject lines, product descriptions): respond with the requested content directly — no preamble, no attribution, punchy and commercial-grade. No markdown, no em-dashes.', + messages: [{ role: 'user', content: userText }], + }), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const data = await response.json(); + const text: string | undefined = data.content?.[0]?.text; + if (!text) throw new Error('empty response'); + return text; +} + +// ─── Scene generation (local LLM via Ollama) ───────────────────────────────── + +interface SceneObject { + type: 'sphere' | 'box' | 'torus' | 'cylinder' | 'cone' | 'icosahedron' | 'ring' | 'plane'; + size?: number | [number, number, number]; + color?: string; + emissive?: string; + emissiveIntensity?: number; + metalness?: number; + roughness?: number; + transparent?: boolean; + opacity?: number; + position?: [number, number, number]; + rotation?: [number, number, number]; + wireframe?: boolean; +} + +interface SceneDescriptor { + objects?: SceneObject[]; +} + +const SCENE_GEN_SYSTEM = `You are a Three.js 3D scene generator. Return ONLY a JSON object, no prose. Structure: +{"objects":[{"type":"sphere|box|torus|cylinder|cone|icosahedron|ring|plane","size":1.0,"color":"#6366f1","emissive":"#4f46e5","emissiveIntensity":0.5,"metalness":0.3,"roughness":0.3,"position":[0,0,0],"rotation":[0,0,0],"opacity":1.0,"wireframe":false}]} +Rules: 3 to 8 objects. "type" must be one of the listed values. emissiveIntensity 0.3-0.8 for a glowing look. size 0.3-3.5. position values between -3.5 and 3.5 on each axis. rotation in degrees. Colors as #RRGGBB hex. Use a harmonious palette and combine shapes creatively to match the description.`; + +// Ordered by JSON-instruction-following quality. Embedding models are excluded at runtime. +const OLLAMA_PREFERRED = ['qwen2.5', 'qwen2', 'llama3.3', 'llama3.2', 'llama3.1', 'llama3', 'mistral', 'phi4', 'phi3.5', 'phi3', 'gemma3', 'gemma2', 'gemma', 'mixtral', 'qwen']; +const OLLAMA_BASE = 'http://localhost:11434'; + +interface OllamaTag { + name: string; + details?: { families?: string[] }; +} + +function isChatModel(m: OllamaTag): boolean { + if (/embed|minilm|bert/i.test(m.name)) return false; + const families = m.details?.families ?? []; + if (families.some((f) => /bert/i.test(f))) return false; + return true; +} + +async function resolveOllamaModel(): Promise { + let res: Response; + try { + res = await fetch(`${OLLAMA_BASE}/api/tags`); + } catch { + throw new Error('ollama_down'); + } + if (!res.ok) throw new Error('ollama_down'); + const data = await res.json(); + const all: OllamaTag[] = data.models ?? []; + const chat = all.filter(isChatModel); + if (chat.length === 0) throw new Error('ollama_no_models'); + const names = chat.map((m) => m.name); + for (const pref of OLLAMA_PREFERRED) { + const match = names.find((n) => n.startsWith(pref)); + if (match) return match; + } + return names[0]; +} + +async function callLLMForScene(prompt: string): Promise { + const model = await resolveOllamaModel(); + let res: Response; + try { + res = await fetch(`${OLLAMA_BASE}/api/chat`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + model, + format: 'json', // constrained decoding — guarantees parseable JSON + options: { temperature: 0.5 }, + messages: [ + { role: 'system', content: SCENE_GEN_SYSTEM }, + { role: 'user', content: prompt }, + ], + stream: false, + }), + }); + } catch { + throw new Error('ollama_down'); + } + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = await res.json(); + const text: string = (data.message?.content ?? '').trim(); + const stripped = text.replace(/```json\n?|\n?```|```\n?/g, ''); + const jsonMatch = stripped.match(/\{[\s\S]*\}/); + if (!jsonMatch) throw new Error('no_json'); + let parsed: unknown; + try { + parsed = JSON.parse(jsonMatch[0]); + } catch { + // Defensive repair: smart quotes, single-quoted keys, trailing commas + const repaired = jsonMatch[0] + .replace(/[‘’]/g, "'") + .replace(/[“”]/g, '"') + .replace(/'/g, '"') + .replace(/,\s*([}\]])/g, '$1'); + parsed = JSON.parse(repaired); + } + return normalizeDescriptor(parsed); +} + +// ── Validation: coerce/clamp model output so one bad value never breaks a scene + +const VALID_TYPES = new Set(['sphere', 'box', 'torus', 'cylinder', 'cone', 'icosahedron', 'ring', 'plane']); +const TYPE_ALIASES: Record = { + cube: 'box', ball: 'sphere', orb: 'sphere', donut: 'torus', tube: 'cylinder', + pyramid: 'cone', circle: 'ring', flat: 'plane', disc: 'ring', +}; + +function num(v: unknown, fallback: number, min: number, max: number): number { + const n = typeof v === 'string' ? parseFloat(v) : typeof v === 'number' ? v : NaN; + if (!Number.isFinite(n)) return fallback; + return Math.min(max, Math.max(min, n)); +} + +function vec3(v: unknown, fallback: [number, number, number], min: number, max: number): [number, number, number] { + if (!Array.isArray(v)) return fallback; + return [num(v[0], fallback[0], min, max), num(v[1], fallback[1], min, max), num(v[2], fallback[2], min, max)]; +} + +function normalizeColor(v: unknown, fallback: string): string { + if (typeof v !== 'string') return fallback; + const s = v.trim(); + if (/^#[0-9a-fA-F]{6}$/.test(s)) return s; + if (/^#[0-9a-fA-F]{3}$/.test(s)) return s; + if (/^[a-zA-Z]+$/.test(s)) return s; // CSS color name — THREE.Color accepts these + return fallback; +} + +function normalizeDescriptor(raw: unknown): SceneDescriptor { + const rawObjs = (raw as { objects?: unknown })?.objects; + if (!Array.isArray(rawObjs)) return { objects: [] }; + const objects: SceneObject[] = []; + for (const item of rawObjs.slice(0, 12)) { + if (!item || typeof item !== 'object') continue; + const o = item as Record; + let type = String(o.type ?? '').toLowerCase().trim() as SceneObject['type']; + if (!VALID_TYPES.has(type)) type = TYPE_ALIASES[type] ?? 'sphere'; + const rawSize = Array.isArray(o.size) + ? (vec3(o.size, [1, 1, 1], 0.1, 4) as SceneObject['size']) + : num(o.size, 1, 0.1, 4); + const opacity = num(o.opacity, 1, 0.05, 1); + objects.push({ + type, + size: rawSize, + color: normalizeColor(o.color, '#7dd3fc'), + emissive: normalizeColor(o.emissive, '#000000'), + emissiveIntensity: num(o.emissiveIntensity, 0.4, 0, 1), + metalness: num(o.metalness, 0.2, 0, 1), + roughness: num(o.roughness, 0.5, 0, 1), + opacity, + transparent: o.transparent === true || opacity < 1, + wireframe: o.wireframe === true, + position: vec3(o.position, [0, 0, 0], -6, 6), + rotation: vec3(o.rotation, [0, 0, 0], -360, 360), + }); + } + return { objects }; +} + +function buildSceneFromDescriptor(descriptor: SceneDescriptor, group: THREE.Group): void { + [...group.children].forEach((child) => { + const mesh = child as THREE.Mesh; + if (mesh.isMesh) { + mesh.geometry.dispose(); + const mat = mesh.material; + if (Array.isArray(mat)) mat.forEach((m) => m.dispose()); + else (mat as THREE.Material).dispose(); + } + group.remove(child); + }); + + (descriptor.objects ?? []).forEach((obj) => { + try { + const raw = Array.isArray(obj.size) ? obj.size : [obj.size ?? 1, obj.size ?? 1, obj.size ?? 1]; + const [sw, sh, sd] = raw.map((v) => Math.max(v, 0.1)); + + let geo: THREE.BufferGeometry; + switch (obj.type) { + case 'box': geo = new THREE.BoxGeometry(sw, sh, sd); break; + case 'sphere': geo = new THREE.SphereGeometry(sw * 0.5, 32, 32); break; + case 'torus': geo = new THREE.TorusGeometry(sw * 0.65, Math.max(sw * 0.08, 0.04), 14, 80); break; + case 'cylinder': geo = new THREE.CylinderGeometry(sw * 0.4, sw * 0.4, sh, 32); break; + case 'cone': geo = new THREE.ConeGeometry(sw * 0.5, sh, 32); break; + case 'icosahedron': geo = new THREE.IcosahedronGeometry(sw * 0.5, 1); break; + case 'ring': geo = new THREE.TorusGeometry(sw * 0.65, sw * 0.03, 8, 80); break; + case 'plane': geo = new THREE.PlaneGeometry(sw, sh); break; + default: geo = new THREE.SphereGeometry(0.5, 32, 32); + } + + const opacity = obj.opacity ?? 1; + const mat = new THREE.MeshStandardMaterial({ + color: new THREE.Color(obj.color ?? '#7dd3fc'), + emissive: new THREE.Color(obj.emissive ?? obj.color ?? '#000000'), + emissiveIntensity: obj.emissiveIntensity ?? 0, + metalness: obj.metalness ?? 0.2, + roughness: obj.roughness ?? 0.5, + transparent: obj.transparent ?? opacity < 1, + opacity, + wireframe: obj.wireframe ?? false, + }); + + const mesh = new THREE.Mesh(geo, mat); + if (obj.position) mesh.position.set(...(obj.position as [number, number, number])); + if (obj.rotation) { + const [rx, ry, rz] = obj.rotation; + mesh.rotation.set( + THREE.MathUtils.degToRad(rx), + THREE.MathUtils.degToRad(ry), + THREE.MathUtils.degToRad(rz) + ); + } + group.add(mesh); + } catch (e) { + console.warn('Scene object build error:', e); + } + }); +} + +// ─── Multi-format 3D model loading ─────────────────────────────────────────── + +export const SUPPORTED_MODEL_FORMATS = ['glb', 'gltf', 'stl', 'obj', 'fbx', 'ply', 'dae', '3mf'] as const; + +function defaultModelMaterial(): THREE.MeshStandardMaterial { + // Geometry-only formats (STL/PLY) arrive without materials — give them a + // clean metallic look that fits the HUD. + return new THREE.MeshStandardMaterial({ + color: 0x9fb4d8, + metalness: 0.4, + roughness: 0.45, + emissive: 0x0a1c33, + emissiveIntensity: 0.25, + }); +} + +// Loaders are dynamically imported so only the one for the chosen format is +// pulled into the bundle at runtime. +async function loadModelObject(url: string, ext: string): Promise { + switch (ext) { + case 'glb': + case 'gltf': { + const { GLTFLoader } = await import('three/examples/jsm/loaders/GLTFLoader.js'); + const gltf = await new GLTFLoader().loadAsync(url); + return gltf.scene; + } + case 'stl': { + const { STLLoader } = await import('three/examples/jsm/loaders/STLLoader.js'); + const geo = await new STLLoader().loadAsync(url); + geo.computeVertexNormals(); + return new THREE.Mesh(geo, defaultModelMaterial()); + } + case 'obj': { + const { OBJLoader } = await import('three/examples/jsm/loaders/OBJLoader.js'); + return await new OBJLoader().loadAsync(url); + } + case 'fbx': { + const { FBXLoader } = await import('three/examples/jsm/loaders/FBXLoader.js'); + return await new FBXLoader().loadAsync(url); + } + case 'ply': { + const { PLYLoader } = await import('three/examples/jsm/loaders/PLYLoader.js'); + const geo = await new PLYLoader().loadAsync(url); + geo.computeVertexNormals(); + const mat = defaultModelMaterial(); + if (geo.hasAttribute('color')) mat.vertexColors = true; + return new THREE.Mesh(geo, mat); + } + case 'dae': { + const { ColladaLoader } = await import('three/examples/jsm/loaders/ColladaLoader.js'); + const res = await new ColladaLoader().loadAsync(url); + if (!res?.scene) throw new Error('unsupported_format'); + return res.scene; + } + case '3mf': { + const { ThreeMFLoader } = await import('three/examples/jsm/loaders/3MFLoader.js'); + return await new ThreeMFLoader().loadAsync(url); + } + default: + throw new Error('unsupported_format'); + } +} + +interface ModelSource { + url: string; + ext: string; +} + +// ─── Viewport ──────────────────────────────────────────────────────────────── + +function JarvisViewport({ + state, + isThinking, + modelSource, + onModelResult, + sceneDescriptor, +}: { + state: JarvisSceneState; + isThinking: boolean; + modelSource?: ModelSource; + onModelResult?: (ok: boolean, msg?: string) => void; + sceneDescriptor?: SceneDescriptor; +}) { + const mountRef = useRef(null); + const stateRef = useRef(state); + const thinkingRef = useRef(isThinking); + const speakingUntilRef = useRef(0); + const prevThinkingRef = useRef(isThinking); + const objectGroupRef = useRef(null); + const sceneGroupRef = useRef(null); + + useEffect(() => { + stateRef.current = state; + }, [state]); + + useEffect(() => { + if (!isThinking && prevThinkingRef.current) { + speakingUntilRef.current = performance.now() + 1100; + } + prevThinkingRef.current = isThinking; + thinkingRef.current = isThinking; + }, [isThinking]); + + useEffect(() => { + const group = objectGroupRef.current; + if (!modelSource || !group) return; + let cancelled = false; + + const disposeChildren = () => { + [...group.children].forEach((child) => { + group.remove(child); + child.traverse((node) => { + const mesh = node as THREE.Mesh; + if (mesh.isMesh) { + mesh.geometry?.dispose(); + const mat = mesh.material; + if (Array.isArray(mat)) mat.forEach((m) => m.dispose()); + else (mat as THREE.Material)?.dispose(); + } + }); + }); + }; + + const fit = (object: THREE.Object3D) => { + const box = new THREE.Box3().setFromObject(object); + const size = box.getSize(new THREE.Vector3()).length(); + const center = box.getCenter(new THREE.Vector3()); + const scale = 2.4 / Math.max(size, 0.001); + object.scale.setScalar(scale); + object.position.set(-center.x * scale, -center.y * scale, -center.z * scale); + }; + + loadModelObject(modelSource.url, modelSource.ext) + .then((object) => { + if (cancelled) return; + disposeChildren(); + fit(object); + group.add(object); + onModelResult?.(true, `${modelSource.ext.toUpperCase()} model loaded.`); + }) + .catch((err) => { + if (cancelled) return; + console.error('Model load error:', err); + const msg = + err instanceof Error && err.message === 'unsupported_format' + ? 'Unsupported 3D format.' + : 'Could not load that model. The file may be corrupt or reference external assets.'; + onModelResult?.(false, msg); + }); + + return () => { + cancelled = true; + }; + }, [modelSource, onModelResult]); + + useEffect(() => { + const group = sceneGroupRef.current; + if (!group || !sceneDescriptor) return; + buildSceneFromDescriptor(sceneDescriptor, group); + }, [sceneDescriptor]); + + useEffect(() => { + if (!mountRef.current) return; + + const mount = mountRef.current; + const scene = new THREE.Scene(); + scene.background = new THREE.Color('#040816'); + scene.fog = new THREE.Fog('#040816', 12, 28); + + const camera = new THREE.PerspectiveCamera(38, 1, 0.1, 100); + camera.position.set(0, 1.55, 6.1); + + const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true }); + renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); + renderer.setSize(mount.clientWidth, mount.clientHeight); + renderer.setClearColor(0x040816, 1); + mount.appendChild(renderer.domElement); + + const resizeObserver = new ResizeObserver(() => { + const width = mount.clientWidth; + const height = mount.clientHeight; + renderer.setSize(width, height); + camera.aspect = width / height; + camera.updateProjectionMatrix(); + }); + resizeObserver.observe(mount); + + const ambient = new THREE.AmbientLight(0x7ad8ff, 0.25); + scene.add(ambient); + const keyLight = new THREE.DirectionalLight(0x9ee7ff, 1.3); + keyLight.position.set(6, 8, 8); + scene.add(keyLight); + const fillLight = new THREE.PointLight(0x00d5ff, 2.5, 22); + fillLight.position.set(-4, 2, 4); + scene.add(fillLight); + const rimLight = new THREE.PointLight(0x7c3aed, 1.4, 18); + rimLight.position.set(4, 1, -5); + scene.add(rimLight); + + const starGeometry = new THREE.BufferGeometry(); + const starCount = 220; + const starPositions = new Float32Array(starCount * 3); + for (let i = 0; i < starCount; i += 1) { + const radius = 12 + Math.random() * 12; + const theta = Math.random() * Math.PI * 2; + const phi = Math.acos(THREE.MathUtils.randFloatSpread(2)); + starPositions[i * 3] = Math.cos(theta) * Math.sin(phi) * radius; + starPositions[i * 3 + 1] = THREE.MathUtils.randFloatSpread(10) + 1.5; + starPositions[i * 3 + 2] = Math.sin(theta) * Math.sin(phi) * radius - 4; + } + starGeometry.setAttribute('position', new THREE.BufferAttribute(starPositions, 3)); + const starMaterial = new THREE.PointsMaterial({ + color: 0x8be9ff, + size: 0.045, + transparent: true, + opacity: 0.75, + depthWrite: false, + }); + const stars = new THREE.Points(starGeometry, starMaterial); + scene.add(stars); + + const halo = new THREE.Mesh( + new THREE.TorusGeometry(3.35, 0.06, 10, 120), + new THREE.MeshBasicMaterial({ color: 0x2dd4bf, transparent: true, opacity: 0.14 }) + ); + halo.rotation.x = Math.PI / 2; + halo.position.set(0, 0.7, -1.4); + scene.add(halo); + + const objectGroup = new THREE.Group(); + objectGroup.position.y = 0.85; + objectGroup.scale.setScalar(1.18); + scene.add(objectGroup); + objectGroupRef.current = objectGroup; + + const mapGroup = new THREE.Group(); + mapGroup.visible = false; + scene.add(mapGroup); + + const ground = new THREE.Mesh( + new THREE.CircleGeometry(7.8, 64), + new THREE.MeshStandardMaterial({ + color: 0x091225, + metalness: 0.15, + roughness: 0.75, + transparent: true, + opacity: 0.9, + }) + ); + ground.rotation.x = -Math.PI / 2; + ground.position.y = -2.1; + scene.add(ground); + + const grid = new THREE.GridHelper(18, 36, 0x2bd9ff, 0x16314f); + (grid.material as THREE.Material).transparent = true; + (grid.material as THREE.Material).opacity = 0.35; + grid.position.y = -2.08; + scene.add(grid); + + const centralCore = new THREE.Mesh( + new THREE.SphereGeometry(0.78, 40, 40), + new THREE.MeshStandardMaterial({ + color: 0x0ff0ff, + emissive: 0x19d9ff, + emissiveIntensity: 0.8, + metalness: 0.45, + roughness: 0.25, + }) + ); + objectGroup.add(centralCore); + + const emitterRing = new THREE.Mesh( + new THREE.TorusGeometry(1.28, 0.08, 12, 80), + new THREE.MeshStandardMaterial({ + color: 0x61dbfb, + emissive: 0x0ea5e9, + emissiveIntensity: 0.75, + transparent: true, + opacity: 0.95, + }) + ); + emitterRing.rotation.x = Math.PI / 2; + objectGroup.add(emitterRing); + + const shellRing = new THREE.Mesh( + new THREE.TorusGeometry(1.9, 0.06, 12, 90), + new THREE.MeshStandardMaterial({ + color: 0x8b5cf6, + emissive: 0x7c3aed, + emissiveIntensity: 0.35, + transparent: true, + opacity: 0.8, + }) + ); + shellRing.rotation.x = Math.PI / 2; + shellRing.rotation.z = Math.PI / 7; + objectGroup.add(shellRing); + + const moduleMaterial = new THREE.MeshStandardMaterial({ + color: 0x14294b, + metalness: 0.65, + roughness: 0.28, + emissive: 0x081422, + emissiveIntensity: 0.25, + }); + const accentMaterial = new THREE.MeshStandardMaterial({ + color: 0x1ca9ff, + metalness: 0.3, + roughness: 0.3, + emissive: 0x19d9ff, + emissiveIntensity: 0.45, + }); + + type PartDef = { mesh: THREE.Mesh; direction: THREE.Vector3; basePosition: THREE.Vector3 }; + const partDefs: PartDef[] = []; + const directions = [ + new THREE.Vector3(1, 0.18, 0.1), + new THREE.Vector3(-1, 0.08, 0.15), + new THREE.Vector3(0.1, 0.28, 1), + new THREE.Vector3(-0.12, -0.14, -1), + new THREE.Vector3(0.85, -0.2, -0.5), + new THREE.Vector3(-0.8, 0.25, 0.55), + ]; + + directions.forEach((direction, index) => { + const module = new THREE.Mesh( + new THREE.BoxGeometry(0.78, 0.35, 0.78), + index % 2 === 0 ? moduleMaterial.clone() : accentMaterial.clone() + ); + module.position.copy(direction.clone().multiplyScalar(1.7)); + module.scale.setScalar(index === 2 ? 1.05 : 1); + objectGroup.add(module); + + const cap = new THREE.Mesh( + new THREE.CylinderGeometry(0.18, 0.18, 0.9, 24), + new THREE.MeshStandardMaterial({ + color: 0x8be9ff, + emissive: 0x2bd9ff, + emissiveIntensity: 0.5, + metalness: 0.7, + roughness: 0.2, + }) + ); + cap.rotation.z = Math.PI / 2; + cap.position.copy(direction.clone().multiplyScalar(1.7)).add(new THREE.Vector3(0, 0.24, 0)); + objectGroup.add(cap); + + partDefs.push({ mesh: module, direction, basePosition: module.position.clone() }); + partDefs.push({ mesh: cap, direction, basePosition: cap.position.clone() }); + }); + + const orbitRings = [1.1, 1.65, 2.35].map((radius, index) => { + const ring = new THREE.Mesh( + new THREE.TorusGeometry(radius, 0.03, 12, 96), + new THREE.MeshBasicMaterial({ + color: index === 0 ? 0x61dbfb : index === 1 ? 0x8b5cf6 : 0x16a34a, + transparent: true, + opacity: 0.45, + }) + ); + ring.rotation.x = Math.PI / (2 + index * 0.45); + ring.rotation.y = index * 0.65; + objectGroup.add(ring); + return ring; + }); + + const mapSurface = new THREE.Mesh( + new THREE.CircleGeometry(4.2, 64), + new THREE.MeshStandardMaterial({ + color: 0x08111f, + emissive: 0x08111f, + metalness: 0.15, + roughness: 0.82, + transparent: true, + opacity: 0.96, + }) + ); + mapSurface.rotation.x = -Math.PI / 2; + mapGroup.add(mapSurface); + + const mapGrid = new THREE.GridHelper(9, 18, 0x38bdf8, 0x13324d); + (mapGrid.material as THREE.Material).transparent = true; + (mapGrid.material as THREE.Material).opacity = 0.3; + mapGrid.position.y = 0.02; + mapGroup.add(mapGrid); + + const marker = new THREE.Mesh( + new THREE.SphereGeometry(0.18, 24, 24), + new THREE.MeshStandardMaterial({ + color: 0xffd166, + emissive: 0xffb703, + emissiveIntensity: 0.85, + metalness: 0.2, + roughness: 0.3, + }) + ); + marker.position.set(0.7, 0.3, -0.1); + mapGroup.add(marker); + + const route = new THREE.Mesh( + new THREE.CylinderGeometry(0.035, 0.035, 5.8, 12), + new THREE.MeshStandardMaterial({ + color: 0x22c55e, + emissive: 0x22c55e, + emissiveIntensity: 0.45, + }) + ); + route.position.set(-0.8, 0.18, -0.1); + route.rotation.z = Math.PI / 2.5; + route.rotation.y = Math.PI / 6; + mapGroup.add(route); + + const pulseRing = new THREE.Mesh( + new THREE.TorusGeometry(1.1, 0.06, 8, 80), + new THREE.MeshBasicMaterial({ color: 0x2dd4bf, transparent: true, opacity: 0.55 }) + ); + pulseRing.rotation.x = Math.PI / 2; + mapGroup.add(pulseRing); + + // ── Generated scene group (scene mode) ──────────────────────────────────── + const sceneGroup = new THREE.Group(); + sceneGroup.visible = false; + scene.add(sceneGroup); + sceneGroupRef.current = sceneGroup; + + // ── Neural Graph (brand / content / analytics modes) ────────────────────── + const neuralGroup = new THREE.Group(); + neuralGroup.visible = false; + scene.add(neuralGroup); + + const NODE_COUNT = 52; + const nodePosData = new Float32Array(NODE_COUNT * 3); + const nodeVelData = new Float32Array(NODE_COUNT * 3); + const nodePhase = new Float32Array(NODE_COUNT); + + // Node 0 = center hub; rest scattered in a flat-ish ellipsoid + nodePosData[0] = 0; nodePosData[1] = 0; nodePosData[2] = 0; + nodeVelData[0] = 0; nodeVelData[1] = 0; nodeVelData[2] = 0; + for (let i = 1; i < NODE_COUNT; i++) { + const r = 0.9 + Math.random() * 3.6; + const theta = Math.random() * Math.PI * 2; + nodePosData[i * 3] = r * Math.cos(theta); + nodePosData[i * 3 + 1] = (Math.random() - 0.5) * 2.6; + nodePosData[i * 3 + 2] = r * Math.sin(theta) * 0.28; + nodeVelData[i * 3] = (Math.random() - 0.5) * 0.0045; + nodeVelData[i * 3 + 1] = (Math.random() - 0.5) * 0.003; + nodeVelData[i * 3 + 2] = (Math.random() - 0.5) * 0.001; + nodePhase[i] = Math.random() * Math.PI * 2; + } + + const nodeGeo = new THREE.BufferGeometry(); + const nodePosAttr = new THREE.BufferAttribute(nodePosData, 3); + nodeGeo.setAttribute('position', nodePosAttr); + const nodeMat = new THREE.PointsMaterial({ + color: 0x7dd3fc, + size: 0.07, + transparent: true, + opacity: 0.75, + depthWrite: false, + sizeAttenuation: true, + }); + neuralGroup.add(new THREE.Points(nodeGeo, nodeMat)); + + // Build edges connecting nearby nodes + const edgePairs: number[][] = []; + const rawEdgePos: number[] = []; + const CONNECT_DIST = 2.4; + for (let i = 0; i < NODE_COUNT; i++) { + for (let j = i + 1; j < NODE_COUNT; j++) { + const dx = nodePosData[i * 3] - nodePosData[j * 3]; + const dy = nodePosData[i * 3 + 1] - nodePosData[j * 3 + 1]; + const dz = nodePosData[i * 3 + 2] - nodePosData[j * 3 + 2]; + if (dx * dx + dy * dy + dz * dz < CONNECT_DIST * CONNECT_DIST) { + edgePairs.push([i, j]); + rawEdgePos.push( + nodePosData[i * 3], nodePosData[i * 3 + 1], nodePosData[i * 3 + 2], + nodePosData[j * 3], nodePosData[j * 3 + 1], nodePosData[j * 3 + 2] + ); + } + } + } + const edgeGeo = new THREE.BufferGeometry(); + const edgePosAttr = new THREE.BufferAttribute(new Float32Array(rawEdgePos), 3); + edgePosAttr.setUsage(THREE.DynamicDrawUsage); + edgeGeo.setAttribute('position', edgePosAttr); + const edgeMat = new THREE.LineBasicMaterial({ color: 0x1e3a5f, transparent: true, opacity: 0.38 }); + neuralGroup.add(new THREE.LineSegments(edgeGeo, edgeMat)); + + // Signal particles — dots that travel along edges when firing + const SIGNAL_COUNT = 14; + const sigData: { edgeIdx: number; t: number; speed: number; active: boolean }[] = + Array.from({ length: SIGNAL_COUNT }, () => ({ edgeIdx: 0, t: 0, speed: 0.5, active: false })); + const sigPosArr = new Float32Array(SIGNAL_COUNT * 3); + const sigGeo = new THREE.BufferGeometry(); + const sigPosAttr = new THREE.BufferAttribute(sigPosArr, 3); + sigPosAttr.setUsage(THREE.DynamicDrawUsage); + sigGeo.setAttribute('position', sigPosAttr); + const sigMat = new THREE.PointsMaterial({ + color: 0xffffff, + size: 0.11, + transparent: true, + opacity: 0.92, + depthWrite: false, + sizeAttenuation: true, + }); + neuralGroup.add(new THREE.Points(sigGeo, sigMat)); + + const clock = new THREE.Clock(); + let frameId = 0; + + const animate = () => { + frameId = window.requestAnimationFrame(animate); + const delta = Math.min(clock.getDelta(), 0.05); + const elapsed = clock.elapsedTime; + const current = stateRef.current; + const firing = thinkingRef.current; + + const isObjectMode = current.mode === 'object' || current.mode === 'overview'; + const isNeuralMode = current.mode === 'brand' || current.mode === 'content' || current.mode === 'analytics'; + const isSceneMode = current.mode === 'scene'; + objectGroup.visible = isObjectMode; + mapGroup.visible = current.mode === 'map'; + neuralGroup.visible = isNeuralMode; + sceneGroup.visible = isSceneMode; + grid.visible = isObjectMode; + ground.visible = isObjectMode; + halo.visible = isObjectMode; + + if (isSceneMode) { + sceneGroup.rotation.y += 0.004; + sceneGroup.children.forEach((child, i) => { + child.position.y += Math.sin(elapsed * 0.55 + i * 1.4) * 0.001; + }); + } + + const explodeTarget = current.exploded ? 1 : 0; + const powerTarget = current.powerOn ? 1 : 0; + const camZTarget = isNeuralMode ? 7.2 : isSceneMode ? 6.5 : (current.cameraDistance || 6.2); + const camY = current.mode === 'map' ? 2.1 : isNeuralMode ? 0.2 : isSceneMode ? 1.2 : 2.4; + const lookY = current.mode === 'map' ? 0.2 : isNeuralMode ? 0.0 : isSceneMode ? 0.2 : 0.85; + + camera.position.z += (camZTarget - camera.position.z) * 0.04; + camera.position.y += (camY - camera.position.y) * 0.04; + camera.lookAt(0, lookY, 0); + + objectGroup.rotation.y += current.rotationSpeed; + objectGroup.rotation.x = Math.sin(elapsed * 0.25) * 0.03; + mapGroup.rotation.y = Math.sin(elapsed * 0.2) * 0.05; + stars.rotation.y += 0.0006; + stars.rotation.x = Math.sin(elapsed * 0.08) * 0.01; + if (isObjectMode) halo.rotation.z += 0.0015; + + if (isNeuralMode) { + // Drift nodes + for (let i = 1; i < NODE_COUNT; i++) { + nodePosData[i * 3] += nodeVelData[i * 3]; + nodePosData[i * 3 + 1] += nodeVelData[i * 3 + 1]; + nodePosData[i * 3 + 2] += nodeVelData[i * 3 + 2]; + if (Math.abs(nodePosData[i * 3]) > 5.0) nodeVelData[i * 3] *= -1; + if (Math.abs(nodePosData[i * 3 + 1]) > 2.8) nodeVelData[i * 3 + 1] *= -1; + if (Math.abs(nodePosData[i * 3 + 2]) > 1.3) nodeVelData[i * 3 + 2] *= -1; + } + nodePosAttr.needsUpdate = true; + + // Sync edge endpoints to drifted nodes + const edgeArr = edgePosAttr.array as Float32Array; + for (let i = 0; i < edgePairs.length; i++) { + const a = edgePairs[i][0]; const b = edgePairs[i][1]; + edgeArr[i * 6] = nodePosData[a * 3]; edgeArr[i * 6 + 1] = nodePosData[a * 3 + 1]; edgeArr[i * 6 + 2] = nodePosData[a * 3 + 2]; + edgeArr[i * 6 + 3] = nodePosData[b * 3]; edgeArr[i * 6 + 4] = nodePosData[b * 3 + 1]; edgeArr[i * 6 + 5] = nodePosData[b * 3 + 2]; + } + edgePosAttr.needsUpdate = true; + + // Visual state — calm, firing (thinking), or speaking burst + const speaking = performance.now() < speakingUntilRef.current; + const active = firing || speaking; + const freq = active ? (firing ? 3.2 : 5.5) : 0.55; + nodeMat.opacity = Math.min(0.98, 0.55 + Math.sin(elapsed * freq + 0.5) * 0.18 + (active ? 0.22 : 0)); + edgeMat.opacity = Math.min(0.85, 0.22 + (active ? 0.32 : 0) + Math.sin(elapsed * (active ? 2.8 : 0.7)) * 0.07); + + const nodeIdleColor = current.mode === 'brand' ? 0x818cf8 : current.mode === 'content' ? 0xe879f9 : 0x7dd3fc; + const nodeFireColor = current.mode === 'brand' ? 0xa78bfa : current.mode === 'content' ? 0xf472b6 : 0x22d3ee; + const edgeIdleColor = current.mode === 'brand' ? 0x312e81 : current.mode === 'content' ? 0x4a044e : 0x1e3a5f; + const edgeFireColor = current.mode === 'brand' ? 0x7c3aed : current.mode === 'content' ? 0xbe185d : 0x3b82f6; + nodeMat.color.setHex(active ? nodeFireColor : nodeIdleColor); + edgeMat.color.setHex(active ? edgeFireColor : edgeIdleColor); + + // Signal particles travel along edges + const spawnRate = firing ? 0.07 : speaking ? 0.04 : 0.005; + for (let si = 0; si < SIGNAL_COUNT; si++) { + const sig = sigData[si]; + if (!sig.active && edgePairs.length > 0 && Math.random() < spawnRate) { + sig.active = true; + sig.edgeIdx = Math.floor(Math.random() * edgePairs.length); + sig.t = 0; + sig.speed = (active ? 1.3 : 0.4) + Math.random() * 0.6; + } + if (sig.active) { + sig.t += sig.speed * delta; + if (sig.t >= 1) { sig.active = false; } + const a = edgePairs[sig.edgeIdx][0]; const b = edgePairs[sig.edgeIdx][1]; + sigPosArr[si * 3] = nodePosData[a * 3] + (nodePosData[b * 3] - nodePosData[a * 3]) * sig.t; + sigPosArr[si * 3 + 1] = nodePosData[a * 3 + 1] + (nodePosData[b * 3 + 1] - nodePosData[a * 3 + 1]) * sig.t; + sigPosArr[si * 3 + 2] = nodePosData[a * 3 + 2] + (nodePosData[b * 3 + 2] - nodePosData[a * 3 + 2]) * sig.t; + } else { + sigPosArr[si * 3 + 1] = -1000; + } + } + sigPosAttr.needsUpdate = true; + } + + partDefs.forEach(({ mesh, direction, basePosition }) => { + const offset = direction.clone().multiplyScalar(2.1 * explodeTarget); + mesh.position.lerpVectors(basePosition, basePosition.clone().add(offset), 0.15); + const material = mesh.material as THREE.MeshStandardMaterial; + material.emissiveIntensity = 0.18 + powerTarget * 0.7; + }); + + centralCore.scale.setScalar(1 + powerTarget * 0.12 + Math.sin(elapsed * 2.4) * 0.02); + emitterRing.scale.setScalar(1 + powerTarget * 0.08 + Math.sin(elapsed * 1.5) * 0.01); + shellRing.rotation.z += 0.003 + powerTarget * 0.005; + + orbitRings.forEach((ring, index) => { + ring.rotation.x += 0.0012 + index * 0.0003; + ring.rotation.y += 0.001 + index * 0.00035; + }); + + mapSurface.scale.setScalar(1 + Math.sin(elapsed * 2) * 0.015); + pulseRing.scale.setScalar(1 + Math.sin(elapsed * 2.8) * 0.08); + marker.position.y = 0.28 + Math.sin(elapsed * 3.1) * 0.08; + route.rotation.y += 0.002; + fillLight.intensity = 1.9 + powerTarget * 1.25; + rimLight.intensity = 1.1 + powerTarget * 0.75; + + renderer.render(scene, camera); + }; + + animate(); + + return () => { + window.cancelAnimationFrame(frameId); + resizeObserver.disconnect(); + sceneGroupRef.current = null; + starGeometry.dispose(); + starMaterial.dispose(); + halo.geometry.dispose(); + (halo.material as THREE.Material).dispose(); + nodeGeo.dispose(); + nodeMat.dispose(); + edgeGeo.dispose(); + edgeMat.dispose(); + sigGeo.dispose(); + sigMat.dispose(); + renderer.dispose(); + mount.removeChild(renderer.domElement); + }; + }, []); + + return ( +
+
+
+
+
+
+
+
+
+
+
+ ); +} + +// ─── Page ───────────────────────────────────────────────────────────────────── + +export default function JarvisPage() { + const [input, setInput] = useState(''); + const [sceneState, setSceneState] = useState(() => createInitialJarvisState()); + const [transcript, setTranscript] = useState(() => [ + { + id: makeTranscriptId(), + role: 'system', + text: 'HUD initialized. Issue a command or tap a chip.', + timestamp: new Date().toISOString(), + }, + ]); + const [isThinking, setIsThinking] = useState(false); + const [isListening, setIsListening] = useState(false); + const [modelSource, setModelSource] = useState(undefined); + const [sceneDescriptor, setSceneDescriptor] = useState(undefined); + const recognitionRef = useRef(null); + const transcriptEndRef = useRef(null); + const apiKeyRef = useRef(null); + const fileInputRef = useRef(null); + const modelMsgIdRef = useRef(null); + + const fact = useMemo(() => getFactForTarget(sceneState.activeTarget), [sceneState.activeTarget]); + + useEffect(() => { + if (!isRunningInElectron()) return; + getAccomplish().getApiKey() + .then((key) => { apiKeyRef.current = key; }) + .catch(() => {}); + }, []); + + useEffect(() => { + transcriptEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + }, [transcript]); + + const executeCommand = useCallback( + async (rawInput: string) => { + const trimmed = rawInput.trim(); + if (!trimmed || isThinking) return; + + const command = parseJarvisCommand(trimmed); + const nextState = applyJarvisCommand(sceneState, command); + const localReply = describeJarvisCommand(command, nextState); + + setInput(''); + + const ts = new Date().toISOString(); + const userEntry: JarvisTranscriptEntry = { id: makeTranscriptId(), role: 'user', text: trimmed, timestamp: ts }; + const apiKey = apiKeyRef.current; + + // 3D scene generation via local Ollama — no API key needed. + // Defer the mode switch until the scene is ready so the user keeps + // seeing their current view (not an empty void) during generation. + if (command.intent === 'create_scene') { + const assistantId = makeTranscriptId(); + setTranscript((prev) => [ + ...prev, + userEntry, + { id: assistantId, role: 'assistant', text: 'Generating 3D scene with Ollama...', timestamp: ts }, + ]); + setIsThinking(true); + try { + const descriptor = await callLLMForScene(trimmed); + const n = descriptor.objects?.length ?? 0; + if (n === 0) throw new Error('empty_scene'); + setSceneDescriptor(descriptor); + setSceneState(nextState); // switch to scene mode now that meshes exist + setTranscript((prev) => + prev.map((e) => (e.id === assistantId ? { ...e, text: `Scene rendered — ${n} objects composed.` } : e)) + ); + } catch (err) { + const code = err instanceof Error ? err.message : ''; + const msg = + code === 'ollama_down' ? 'Ollama not running. Start it with: ollama serve' : + code === 'ollama_no_models' ? 'No chat models installed. Run: ollama pull llama3.2' : + code === 'empty_scene' ? 'Could not build a scene from that. Try describing shapes, colors, and a mood.' : + 'Scene generation failed. Make sure Ollama is running.'; + setTranscript((prev) => + prev.map((e) => (e.id === assistantId ? { ...e, text: msg } : e)) + ); + } + setIsThinking(false); + return; + } + + // All other commands apply their scene state immediately. + setSceneState(nextState); + + if (!apiKey) { + setTranscript((prev) => [ + ...prev, + userEntry, + { id: makeTranscriptId(), role: 'assistant', text: localReply, timestamp: ts }, + ]); + return; + } + + const assistantId = makeTranscriptId(); + setTranscript((prev) => [ + ...prev, + userEntry, + { id: assistantId, role: 'assistant', text: '...', timestamp: ts }, + ]); + setIsThinking(true); + + let finalReply = localReply; + try { + finalReply = await callAnthropicHaiku(apiKey, trimmed); + } catch { + // fall back to local reply + } + + setTranscript((prev) => + prev.map((e) => (e.id === assistantId ? { ...e, text: finalReply } : e)) + ); + setIsThinking(false); + }, + [sceneState, isThinking] + ); + + const onSubmit = useCallback(() => { + void executeCommand(input); + }, [executeCommand, input]); + + const onPreset = useCallback( + (prompt: string) => { + setInput(prompt); + void executeCommand(prompt); + }, + [executeCommand] + ); + + const toggleListening = useCallback(() => { + if (isListening) { + recognitionRef.current?.stop(); + setIsListening(false); + return; + } + + const SpeechRecognition = + (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition; + if (!SpeechRecognition) return; + + const recognition = new SpeechRecognition(); + recognition.continuous = false; + recognition.interimResults = false; + recognition.lang = 'en-US'; + + recognition.onresult = (event: any) => { + const heard: string = event.results[0][0].transcript; + setInput(heard); + void executeCommand(heard); + }; + recognition.onend = () => setIsListening(false); + recognition.onerror = () => setIsListening(false); + + recognition.start(); + recognitionRef.current = recognition; + setIsListening(true); + }, [isListening, executeCommand]); + + const handleFileSelect = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + e.target.value = ''; + if (!file) return; + + const ext = file.name.split('.').pop()?.toLowerCase() ?? ''; + const ts = new Date().toISOString(); + + if (!(SUPPORTED_MODEL_FORMATS as readonly string[]).includes(ext)) { + setTranscript((prev) => [ + ...prev, + { id: makeTranscriptId(), role: 'user', text: `Load ${file.name}`, timestamp: ts }, + { + id: makeTranscriptId(), + role: 'assistant', + text: `Unsupported format ".${ext}". Supported: ${SUPPORTED_MODEL_FORMATS.map((f) => f.toUpperCase()).join(', ')}.`, + timestamp: ts, + }, + ]); + return; + } + + const prevUrl = modelSource?.url; + const url = URL.createObjectURL(file); + const msgId = makeTranscriptId(); + modelMsgIdRef.current = msgId; + + setTranscript((prev) => [ + ...prev, + { id: makeTranscriptId(), role: 'user', text: `Load ${file.name}`, timestamp: ts }, + { id: msgId, role: 'assistant', text: `Loading ${file.name}...`, timestamp: ts }, + ]); + + // Models render in the object group, which is only shown in object mode. + setSceneState((s) => ({ + ...s, + mode: 'object', + activeTarget: file.name.replace(/\.[^.]+$/, ''), + summary: `Imported 3D model: ${file.name}`, + exploded: false, + })); + setModelSource({ url, ext }); + if (prevUrl) URL.revokeObjectURL(prevUrl); + }, + [modelSource] + ); + + const handleModelResult = useCallback((ok: boolean, msg?: string) => { + const id = modelMsgIdRef.current; + if (!id) return; + setTranscript((prev) => + prev.map((e) => + e.id === id ? { ...e, text: msg ?? (ok ? 'Model loaded.' : 'Failed to load model.') } : e + ) + ); + }, []); + + const stateChips = useMemo( + () => + [ + sceneState.powerOn ? { label: 'Powered', color: 'text-emerald-400' } : null, + sceneState.exploded ? { label: 'Exploded', color: 'text-amber-400' } : null, + sceneState.mode === 'map' ? { label: 'Map', color: 'text-sky-400' } : null, + sceneState.mode === 'brand' ? { label: 'Brand', color: 'text-violet-400' } : null, + sceneState.mode === 'content' ? { label: 'Content', color: 'text-pink-400' } : null, + sceneState.mode === 'analytics' ? { label: 'Analytics', color: 'text-cyan-400' } : null, + sceneState.mode === 'scene' ? { label: 'Scene', color: 'text-emerald-400' } : null, + ].filter(Boolean) as { label: string; color: string }[], + [sceneState] + ); + + const commandChips = useMemo(() => { + const m = sceneState.mode; + if (m === 'brand') return [ + { label: 'Tagline', prompt: 'Write 3 sharp taglines for a bold digital brand', icon: Sparkles }, + { label: 'Value prop', prompt: 'Write a one-sentence value proposition for a SaaS product', icon: Zap }, + { label: 'Brand voice', prompt: 'Describe a brand voice that is bold, minimal, and modern', icon: Palette }, + { label: 'Ad copy', prompt: 'Write 3 short ad copy lines for a digital product launch', icon: TrendingUp }, + { label: 'Switch scene', prompt: 'Load reactor core', icon: Box }, + ]; + if (m === 'content') return [ + { label: 'IG caption', prompt: 'Write an Instagram caption with a strong hook for a digital product launch', icon: Hash }, + { label: 'Email subject', prompt: 'Write 5 email subject lines for a product launch campaign', icon: Mail }, + { label: 'Twitter thread', prompt: 'Write the first 3 tweets of a thread about building a successful digital product', icon: Sparkles }, + { label: 'Blog hook', prompt: 'Write a compelling opening paragraph for a blog post about digital products', icon: FileText }, + { label: 'Switch scene', prompt: 'Load reactor core', icon: Box }, + ]; + if (m === 'analytics') return [ + { label: 'KPI summary', prompt: 'Write a concise KPI performance summary for a growing digital product', icon: BarChart2 }, + { label: 'Growth post', prompt: 'Write a LinkedIn post announcing strong user growth this quarter', icon: TrendingUp }, + { label: 'Investor update', prompt: 'Write a one-paragraph investor update highlighting strong monthly metrics', icon: Zap }, + { label: 'Data insight', prompt: 'Write 3 data-driven insights for a SaaS product with strong retention', icon: Sparkles }, + { label: 'Switch scene', prompt: 'Load reactor core', icon: Box }, + ]; + // scene mode + if (m === 'scene') return [ + { label: 'Create new', prompt: 'Create a glowing icosahedron with orbiting rings in deep violet and cyan', icon: Wand2 }, + { label: 'Brand 3D', prompt: 'Create a bold geometric logo shape with hexagonal rings and brand violet glow', icon: Palette }, + { label: 'Product 3D', prompt: 'Create a sleek product box with a glowing edge and floating spheres around it', icon: Box }, + { label: 'Brand mode', prompt: 'Switch to brand mode', icon: Sparkles }, + { label: 'Load model', prompt: '', icon: Box, action: () => fileInputRef.current?.click() }, + ]; + return [ + { label: 'Load 3D', prompt: '', icon: Box, action: () => fileInputRef.current?.click() }, + { label: 'Create 3D', prompt: 'Create a glowing orb surrounded by orbiting violet rings and cyan signal particles', icon: Wand2 }, + { label: 'Brand', prompt: 'Switch to brand mode', icon: Palette }, + { label: 'Content', prompt: 'Switch to content mode', icon: FileText }, + { label: 'Analytics', prompt: 'Switch to analytics mode', icon: BarChart2 }, + ]; + }, [sceneState.mode]); + + return ( + + {/* ── Full-bleed 3D viewport ── */} +
+ +
+ + {/* ── Top HUD bar ── */} +
+
+ + + HUD online + + + + {sceneState.mode === 'map' ? 'Map' : sceneState.mode === 'brand' ? 'Brand' : sceneState.mode === 'content' ? 'Content' : sceneState.mode === 'analytics' ? 'Analytics' : sceneState.mode === 'scene' ? 'Scene' : 'Object'} + + {stateChips.map((chip) => ( + + {chip.label} + + ))} + {isThinking && ( + + + Processing + + )} +
+
+

Module

+

{fact.title}

+
+
+ + {/* ── Right panel: scene brief + transcript ── */} +
+ {/* Scene brief */} +
+
+ + Scene brief +
+

{fact.summary}

+
+ {fact.bullets.map((b) => ( + + {b} + + ))} +
+
+ + {/* Transcript feed */} +
+ + + Comms + + +
+ +
+ {transcript.map((entry) => ( +
+ {entry.role === 'user' && ( +

+ > + {entry.text} +

+ )} + {entry.role === 'assistant' && ( +
+ // + {entry.text} + {entry.text !== '...' && ( + + )} +
+ )} + {entry.role === 'system' && ( +

{entry.text}

+ )} +
+ ))} +
+
+ +
+ + {/* ── Bottom command bar ── */} +
+ {/* Quick chips */} +
+ {commandChips.map((chip) => { + const Icon = chip.icon; + return ( + + ); + })} +
+ + {/* Input row */} +
+ + setInput(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + onSubmit(); + } + }} + placeholder="Brand mode · analytics · write 3 Instagram hooks for..." + className="border-cyan-500/20 bg-slate-900/80 text-cyan-50 placeholder:text-slate-600 focus-visible:ring-cyan-500/30 focus-visible:border-cyan-500/40" + /> + +
+
+ + {/* Hidden 3D file picker — accepts all supported model formats */} + + + ); +} diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/vite-env.d.ts b/apps/openwork-memos-integration/apps/desktop/src/renderer/vite-env.d.ts new file mode 100644 index 000000000..0546f921e --- /dev/null +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/vite-env.d.ts @@ -0,0 +1,16 @@ +/// + +import type { AccomplishAPI } from './lib/accomplish'; + +declare global { + interface Window { + accomplish: AccomplishAPI; + accomplishShell?: { + version: string; + platform: string; + isElectron: boolean; + }; + } +} + +export {}; diff --git a/apps/openwork-memos-integration/packages/shared/src/types/jarvis.ts b/apps/openwork-memos-integration/packages/shared/src/types/jarvis.ts new file mode 100644 index 000000000..b6ed62a19 --- /dev/null +++ b/apps/openwork-memos-integration/packages/shared/src/types/jarvis.ts @@ -0,0 +1,48 @@ +export type JarvisMode = 'object' | 'map' | 'overview' | 'brand' | 'content' | 'analytics' | 'scene'; + +export type JarvisCommandType = + | 'load_object' + | 'summarize_object' + | 'explode_part' + | 'assemble_part' + | 'power_on' + | 'power_off' + | 'highlight_part' + | 'rotate_view' + | 'switch_to_map' + | 'annotate_map' + | 'explain_scene' + | 'switch_to_brand' + | 'switch_to_content' + | 'switch_to_analytics' + | 'create_scene'; + +export interface JarvisCommand { + intent: JarvisCommandType; + target?: string; + query?: string; + mode?: JarvisMode; + direction?: 'left' | 'right' | 'up' | 'down'; + speed?: number; + zoom?: number; + raw: string; +} + +export interface JarvisSceneState { + mode: JarvisMode; + activeTarget: string; + summary: string; + powerOn: boolean; + exploded: boolean; + highlightedPart?: string; + mapFocus?: string; + rotationSpeed: number; + cameraDistance: number; +} + +export interface JarvisTranscriptEntry { + id: string; + role: 'user' | 'assistant' | 'system'; + text: string; + timestamp: string; +}