From 69cde2dd69d37ba19bc6ff0e323f004c80d99303 Mon Sep 17 00:00:00 2001 From: Jose Guerra Date: Thu, 11 Jun 2026 22:51:59 -0700 Subject: [PATCH 1/5] feat(jarvis): add AI 3D scene generation via local Ollama LLM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a new 'scene' mode to the Jarvis HUD where users can describe a 3D composition in natural language and see it rendered live in Three.js. The scene generation pipeline calls a local Ollama instance (llama3.2) so no cloud API key is required for this feature. Changes: - shared/types/jarvis.ts: add 'scene' JarvisMode and 'create_scene' command type - command-parser.ts: detect create/generate/make/build + 3D keywords → create_scene intent - Jarvis.tsx: - SceneDescriptor type + callLocalLLMForScene (Ollama /api/chat) - buildSceneFromDescriptor: maps JSON objects to Three.js meshes (sphere/box/torus/ cylinder/cone/icosahedron/ring/plane with full material support) - JarvisViewport: sceneGroup rendered procedurally, gentle rotation + float animation, camera positioned at 6.5z for composed scenes - executeCommand: create_scene branch routes to Ollama before standard Anthropic path - Context-aware chips for scene mode: Create new, Brand 3D, Product 3D - Mode badge, stateChips, and commandChips updated for 'scene' Co-Authored-By: Claude Sonnet 4.6 --- .../src/renderer/lib/jarvis/command-parser.ts | 356 +++++ .../desktop/src/renderer/pages/Jarvis.tsx | 1215 +++++++++++++++++ .../apps/desktop/src/renderer/vite-env.d.ts | 16 + .../packages/shared/src/types/jarvis.ts | 48 + 4 files changed, 1635 insertions(+) create mode 100644 apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts create mode 100644 apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx create mode 100644 apps/openwork-memos-integration/apps/desktop/src/renderer/vite-env.d.ts create mode 100644 apps/openwork-memos-integration/packages/shared/src/types/jarvis.ts diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts b/apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts new file mode 100644 index 000000000..6ad31df99 --- /dev/null +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts @@ -0,0 +1,356 @@ +import type { JarvisCommand, JarvisSceneState } from '@accomplish/shared'; + +const DEFAULT_OBJECT = 'reactor core'; + +function normalize(input: string): string { + return input.trim().replace(/\s+/g, ' ').toLowerCase(); +} + +function stripCommandPrefix(text: string): string { + return text + .replace(/^(please\s+)?(show me|load|render|summarize|summarise|explain|inspect|highlight|rotate|turn on|power on|assemble|explode|switch to|go to|open)\s+/i, '') + .trim(); +} + +function extractTarget(input: string): string | undefined { + const cleaned = stripCommandPrefix(input); + if (!cleaned) return undefined; + + const blocked = new Set(['it', 'that', 'this', 'the', 'scene', 'view']); + if (blocked.has(cleaned.toLowerCase())) return undefined; + + return cleaned.replace(/^the\s+/i, '').trim(); +} + +function extractDirection(input: string): JarvisCommand['direction'] { + if (/\bleft\b/i.test(input)) return 'left'; + if (/\bright\b/i.test(input)) return 'right'; + if (/\bup\b/i.test(input)) return 'up'; + if (/\bdown\b/i.test(input)) return 'down'; + return undefined; +} + +export function parseJarvisCommand(raw: string): JarvisCommand { + const input = normalize(raw); + const target = extractTarget(raw); + + if (!input) { + return { + intent: 'explain_scene', + target: DEFAULT_OBJECT, + raw, + }; + } + + if (/\b(create|generate|make me|build)\b.{0,30}\b(3d|scene|model|sphere|cube|box|torus|ring|orb|shape|object|structure|render me)\b/i.test(input)) { + return { + intent: 'create_scene', + target: target ?? 'custom scene', + query: raw.trim(), + raw, + }; + } + + if (/\b(brand|identity|palette|logo|visual identity|brand kit)\b/.test(input)) { + return { + intent: 'switch_to_brand', + target: target ?? DEFAULT_OBJECT, + mode: 'brand', + raw, + }; + } + + if (/\b(content|social|post|cards|feed|media)\b/.test(input)) { + return { + intent: 'switch_to_content', + target: target ?? DEFAULT_OBJECT, + mode: 'content', + raw, + }; + } + + if (/\b(analytics|metrics|data|chart|stats|kpi|numbers)\b/.test(input)) { + return { + intent: 'switch_to_analytics', + target: target ?? DEFAULT_OBJECT, + mode: 'analytics', + raw, + }; + } + + if (/\b(map|route|terrain|location|locations|nav|navigate)\b/.test(input)) { + return { + intent: 'switch_to_map', + target: target ?? DEFAULT_OBJECT, + mode: 'map', + raw, + }; + } + + if (/\bexplode|exploded\b/.test(input)) { + return { + intent: 'explode_part', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\bassemble|assembled|rebuild|collapse\b/.test(input)) { + return { + intent: 'assemble_part', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\b(turn on|power on|enable|activate|ignite|start)\b/.test(input)) { + return { + intent: 'power_on', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\b(turn off|power off|disable|deactivate|shutdown|shut down|stop)\b/.test(input)) { + return { + intent: 'power_off', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\bhighlight|label|tag\b/.test(input)) { + return { + intent: 'highlight_part', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\brotate|spin|orbit\b/.test(input)) { + return { + intent: 'rotate_view', + target: target ?? DEFAULT_OBJECT, + direction: extractDirection(raw), + raw, + }; + } + + if (/\b(summarize|summarise|what is|what's this|explain|describe|inspect)\b/.test(input)) { + return { + intent: 'summarize_object', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + if (/\b(load|show me|render|open)\b/.test(input)) { + return { + intent: 'load_object', + target: target ?? DEFAULT_OBJECT, + raw, + }; + } + + return { + intent: 'explain_scene', + target: target ?? DEFAULT_OBJECT, + query: raw.trim(), + raw, + }; +} + +function catalogSummary(target: string): string { + const lookup = target.toLowerCase(); + + if (lookup.includes('reactor')) { + return 'High-energy core assembly with concentric containment rings, a bright central emitter, and service modules that can be exploded or reassembled on command.'; + } + + if (lookup.includes('engine')) { + return 'Mechanical power unit with layered housings, visible support geometry, and a state model that supports inspection, explosion, and reassembly.'; + } + + if (lookup.includes('map') || lookup.includes('city') || lookup.includes('route')) { + return 'Navigational surface with markers, route overlays, and contextual labels for quick orientation.'; + } + + return `Interactive 3D object for ${target}, ready for labels, state changes, and HUD-style annotations.`; +} + +export function applyJarvisCommand(state: JarvisSceneState, command: JarvisCommand): JarvisSceneState { + const target = command.target?.trim() || state.activeTarget || DEFAULT_OBJECT; + + switch (command.intent) { + case 'load_object': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + exploded: false, + powerOn: false, + highlightedPart: undefined, + mapFocus: undefined, + }; + case 'summarize_object': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + }; + case 'explode_part': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + exploded: true, + cameraDistance: 8.5, + }; + case 'assemble_part': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + exploded: false, + cameraDistance: 6.2, + }; + case 'power_on': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + powerOn: true, + }; + case 'power_off': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + powerOn: false, + }; + case 'highlight_part': + return { + ...state, + mode: 'object', + activeTarget: target, + summary: catalogSummary(target), + highlightedPart: target, + }; + case 'rotate_view': + return { + ...state, + activeTarget: target, + rotationSpeed: command.direction === 'left' ? -0.015 : command.direction === 'right' ? 0.015 : state.rotationSpeed, + cameraDistance: typeof command.zoom === 'number' ? command.zoom : state.cameraDistance, + }; + case 'switch_to_brand': + return { + ...state, + mode: 'brand', + activeTarget: 'brand system', + summary: 'Visual identity elements — mark, palette, and type scale — rendered in 3D space.', + exploded: false, + }; + case 'switch_to_content': + return { + ...state, + mode: 'content', + activeTarget: 'content stack', + summary: 'Social and digital content cards layered in a depth composition.', + exploded: false, + }; + case 'switch_to_analytics': + return { + ...state, + mode: 'analytics', + activeTarget: 'analytics view', + summary: 'Performance metrics visualized as a 3D bar chart with trend line overlay.', + exploded: false, + }; + case 'create_scene': + return { + ...state, + mode: 'scene', + activeTarget: target || 'custom scene', + summary: 'AI-generated 3D composition — objects rendered from your description.', + exploded: false, + }; + case 'switch_to_map': + return { + ...state, + mode: 'map', + activeTarget: target, + summary: `Map mode focused on ${target}. Use annotate and route commands to refine the view.`, + mapFocus: target, + exploded: false, + }; + case 'annotate_map': + return { + ...state, + mode: 'map', + activeTarget: target, + mapFocus: target, + }; + case 'explain_scene': + default: + return { + ...state, + activeTarget: target, + summary: catalogSummary(target), + }; + } +} + +export function createInitialJarvisState(): JarvisSceneState { + return { + mode: 'object', + activeTarget: DEFAULT_OBJECT, + summary: catalogSummary(DEFAULT_OBJECT), + powerOn: false, + exploded: false, + rotationSpeed: 0.01, + cameraDistance: 6.2, + }; +} + +export function describeJarvisCommand(command: JarvisCommand, nextState: JarvisSceneState): string { + switch (command.intent) { + case 'load_object': + return `Loaded ${nextState.activeTarget}. ${nextState.summary}`; + case 'summarize_object': + return `${nextState.activeTarget}: ${nextState.summary}`; + case 'explode_part': + return `Exploding ${nextState.activeTarget}. Bringing the assembly into separated layers.`; + case 'assemble_part': + return `Assembling ${nextState.activeTarget}. Returning all layers to the compact state.`; + case 'power_on': + return `Powering on ${nextState.activeTarget}. Emissive state and energy effects are active.`; + case 'power_off': + return `Shutting down ${nextState.activeTarget}. All emissive output and energy effects offline.`; + case 'highlight_part': + return `Highlighting ${nextState.highlightedPart ?? nextState.activeTarget}.`; + case 'rotate_view': + return `Rotating the view${command.direction ? ` ${command.direction}` : ''}.`; + case 'create_scene': + return `Generating 3D scene for ${nextState.activeTarget}. Rendering objects from your description...`; + case 'switch_to_brand': + return 'Brand system loaded. Mark, palette, and type hierarchy are live.'; + case 'switch_to_content': + return 'Content stack loaded. Social and digital card formats are in view.'; + case 'switch_to_analytics': + return 'Analytics view loaded. Performance metrics and trend line active.'; + case 'switch_to_map': + return `Switched to map mode for ${nextState.mapFocus ?? nextState.activeTarget}.`; + case 'annotate_map': + return `Annotating the map around ${nextState.mapFocus ?? nextState.activeTarget}.`; + case 'explain_scene': + default: + return nextState.summary; + } +} diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx new file mode 100644 index 000000000..1889760ff --- /dev/null +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx @@ -0,0 +1,1215 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { motion } from 'framer-motion'; +import * as THREE from 'three'; +import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { Badge } from '@/components/ui/badge'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { + ArrowRight, + BarChart2, + Bot, + Box, + Clipboard, + Compass, + FileText, + Hash, + Mail, + Mic, + Palette, + RadioTower, + RotateCw, + Sparkles, + Telescope, + TrendingUp, + Wand2, + Zap, +} from 'lucide-react'; +import type { JarvisCommand, JarvisSceneState, JarvisTranscriptEntry } from '@accomplish/shared'; +import { + applyJarvisCommand, + createInitialJarvisState, + describeJarvisCommand, + parseJarvisCommand, +} from '@/lib/jarvis/command-parser'; +import { getAccomplish, isRunningInElectron } from '@/lib/accomplish'; + +type JarvisChip = { label: string; prompt: string; icon: React.ElementType; action?: () => void }; + +const OBJECT_FACTS: Record = { + 'reactor core': { + title: 'Reactor Core', + summary: 'Layered containment assembly — bright central emitter, concentric rings, and service modules.', + bullets: ['Containment rings', 'Central emitter', 'Service pods'], + }, + 'engine block': { + title: 'Engine Block', + summary: 'Structural power unit with layered housings and visible subassemblies.', + bullets: ['Cylinder bank', 'Cooling loops', 'Access panels'], + }, + 'city map': { + title: 'City Map', + summary: 'Navigational surface with route overlays, markers, and district labels.', + bullets: ['Route line', 'Marker pins', 'District labels'], + }, + 'brand system': { + title: 'Brand System', + summary: 'Visual identity in 3D — brand mark, color palette swatches, and type hierarchy.', + bullets: ['Brand mark', 'Color palette', 'Type scale'], + }, + 'content stack': { + title: 'Content Stack', + summary: 'Social and digital content cards floating in a depth-layered composition.', + bullets: ['Social cards', 'Post previews', 'Content grid'], + }, + 'analytics view': { + title: 'Analytics View', + summary: 'Performance metrics as a 3D bar chart with trend line overlay and KPI baseline.', + bullets: ['Bar chart', 'Trend line', 'KPI baseline'], + }, + 'custom scene': { + title: 'Generated Scene', + summary: 'AI-generated 3D composition rendered from your description via local LLM.', + bullets: ['Custom objects', 'AI-composed', 'Live render'], + }, +}; + +function getFactForTarget(target: string) { + const key = target.toLowerCase(); + if (key.includes('reactor')) return OBJECT_FACTS['reactor core']; + if (key.includes('engine')) return OBJECT_FACTS['engine block']; + if (key.includes('map') || key.includes('city') || key.includes('route')) return OBJECT_FACTS['city map']; + if (key.includes('brand') || key.includes('identity') || key.includes('palette')) return OBJECT_FACTS['brand system']; + if (key.includes('content') || key.includes('social') || key.includes('card')) return OBJECT_FACTS['content stack']; + if (key.includes('analytics') || key.includes('metrics') || key.includes('chart')) return OBJECT_FACTS['analytics view']; + if (key.includes('scene') || key.includes('generated') || key.includes('custom')) return OBJECT_FACTS['custom scene']; + return { + title: target, + summary: `Interactive 3D object: ${target}. Supports load, inspect, explode, assemble, annotate.`, + bullets: ['Load', 'Inspect', 'Animate'], + }; +} + +function makeTranscriptId() { + return `jarvis_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; +} + +async function callAnthropicHaiku(apiKey: string, userText: string): Promise { + const response = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + 'content-type': 'application/json', + }, + body: JSON.stringify({ + model: 'claude-haiku-4-5-20251001', + max_tokens: 220, + system: + 'You are JARVIS, a creative HUD assistant with two modes. For 3D visual commands (explode, power on, load, rotate, switch to map/brand/content/analytics mode, etc.): 1-2 crisp sentences narrating the action — precise, slightly technical. For marketing or creative requests (write copy, taglines, headlines, hooks, captions, social posts, ad copy, email subject lines, product descriptions): respond with the requested content directly — no preamble, no attribution, punchy and commercial-grade. No markdown, no em-dashes.', + messages: [{ role: 'user', content: userText }], + }), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const data = await response.json(); + const text: string | undefined = data.content?.[0]?.text; + if (!text) throw new Error('empty response'); + return text; +} + +// ─── Scene generation (local LLM via Ollama) ───────────────────────────────── + +interface SceneObject { + type: 'sphere' | 'box' | 'torus' | 'cylinder' | 'cone' | 'icosahedron' | 'ring' | 'plane'; + size?: number | [number, number, number]; + color?: string; + emissive?: string; + emissiveIntensity?: number; + metalness?: number; + roughness?: number; + transparent?: boolean; + opacity?: number; + position?: [number, number, number]; + rotation?: [number, number, number]; + wireframe?: boolean; +} + +interface SceneDescriptor { + objects?: SceneObject[]; +} + +const SCENE_GEN_SYSTEM = `You are a Three.js 3D scene generator. Given a description, return ONLY a valid JSON object — no markdown, no code fences, no explanation. Structure: +{"objects":[{"type":"sphere|box|torus|cylinder|cone|icosahedron|ring|plane","size":1.0,"color":"#6366f1","emissive":"#4f46e5","emissiveIntensity":0.5,"metalness":0.3,"roughness":0.3,"position":[0,0,0],"rotation":[0,0,0],"transparent":false,"opacity":1.0,"wireframe":false}]} +Rules: 3-8 objects. Use emissiveIntensity 0.3-0.8 for glowing effects. Size 0.3-3.5. Position -3.5 to 3.5 on each axis. Combine shapes creatively (e.g. icosahedron core with orbiting ring shells). Use harmonious hex color palettes. Return ONLY the JSON object.`; + +async function callLocalLLMForScene(prompt: string): Promise { + const response = await fetch('http://localhost:11434/api/chat', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + model: 'llama3.2', + messages: [ + { role: 'system', content: SCENE_GEN_SYSTEM }, + { role: 'user', content: prompt }, + ], + stream: false, + }), + }); + if (!response.ok) throw new Error(`Ollama HTTP ${response.status}`); + const data = await response.json(); + const text: string = (data.message?.content ?? data.response ?? '').trim(); + const jsonMatch = text.replace(/```json\n?|\n?```|```\n?/g, '').match(/\{[\s\S]*\}/); + if (!jsonMatch) throw new Error('No JSON in response'); + return JSON.parse(jsonMatch[0]) as SceneDescriptor; +} + +function buildSceneFromDescriptor(descriptor: SceneDescriptor, group: THREE.Group): void { + [...group.children].forEach((child) => { + const mesh = child as THREE.Mesh; + if (mesh.isMesh) { + mesh.geometry.dispose(); + const mat = mesh.material; + if (Array.isArray(mat)) mat.forEach((m) => m.dispose()); + else (mat as THREE.Material).dispose(); + } + group.remove(child); + }); + + (descriptor.objects ?? []).forEach((obj) => { + try { + const raw = Array.isArray(obj.size) ? obj.size : [obj.size ?? 1, obj.size ?? 1, obj.size ?? 1]; + const [sw, sh, sd] = raw.map((v) => Math.max(v, 0.1)); + + let geo: THREE.BufferGeometry; + switch (obj.type) { + case 'box': geo = new THREE.BoxGeometry(sw, sh, sd); break; + case 'sphere': geo = new THREE.SphereGeometry(sw * 0.5, 32, 32); break; + case 'torus': geo = new THREE.TorusGeometry(sw * 0.65, Math.max(sw * 0.08, 0.04), 14, 80); break; + case 'cylinder': geo = new THREE.CylinderGeometry(sw * 0.4, sw * 0.4, sh, 32); break; + case 'cone': geo = new THREE.ConeGeometry(sw * 0.5, sh, 32); break; + case 'icosahedron': geo = new THREE.IcosahedronGeometry(sw * 0.5, 1); break; + case 'ring': geo = new THREE.TorusGeometry(sw * 0.65, sw * 0.03, 8, 80); break; + case 'plane': geo = new THREE.PlaneGeometry(sw, sh); break; + default: geo = new THREE.SphereGeometry(0.5, 32, 32); + } + + const opacity = obj.opacity ?? 1; + const mat = new THREE.MeshStandardMaterial({ + color: new THREE.Color(obj.color ?? '#7dd3fc'), + emissive: new THREE.Color(obj.emissive ?? obj.color ?? '#000000'), + emissiveIntensity: obj.emissiveIntensity ?? 0, + metalness: obj.metalness ?? 0.2, + roughness: obj.roughness ?? 0.5, + transparent: obj.transparent ?? opacity < 1, + opacity, + wireframe: obj.wireframe ?? false, + }); + + const mesh = new THREE.Mesh(geo, mat); + if (obj.position) mesh.position.set(...(obj.position as [number, number, number])); + if (obj.rotation) { + const [rx, ry, rz] = obj.rotation; + mesh.rotation.set( + THREE.MathUtils.degToRad(rx), + THREE.MathUtils.degToRad(ry), + THREE.MathUtils.degToRad(rz) + ); + } + group.add(mesh); + } catch (e) { + console.warn('Scene object build error:', e); + } + }); +} + +// ─── Viewport ──────────────────────────────────────────────────────────────── + +function JarvisViewport({ + state, + isThinking, + modelUrl, + sceneDescriptor, +}: { + state: JarvisSceneState; + isThinking: boolean; + modelUrl?: string; + sceneDescriptor?: SceneDescriptor; +}) { + const mountRef = useRef(null); + const stateRef = useRef(state); + const thinkingRef = useRef(isThinking); + const speakingUntilRef = useRef(0); + const prevThinkingRef = useRef(isThinking); + const objectGroupRef = useRef(null); + const sceneGroupRef = useRef(null); + + useEffect(() => { + stateRef.current = state; + }, [state]); + + useEffect(() => { + if (!isThinking && prevThinkingRef.current) { + speakingUntilRef.current = performance.now() + 1100; + } + prevThinkingRef.current = isThinking; + thinkingRef.current = isThinking; + }, [isThinking]); + + useEffect(() => { + if (!modelUrl || !objectGroupRef.current) return; + const group = objectGroupRef.current; + const loader = new GLTFLoader(); + loader.load( + modelUrl, + (gltf) => { + while (group.children.length > 0) group.remove(group.children[0]); + const model = gltf.scene; + const box = new THREE.Box3().setFromObject(model); + const size = box.getSize(new THREE.Vector3()).length(); + const center = box.getCenter(new THREE.Vector3()); + const scale = 2.4 / Math.max(size, 0.001); + model.scale.setScalar(scale); + model.position.set(-center.x * scale, -center.y * scale, -center.z * scale); + group.add(model); + }, + undefined, + (err) => console.error('GLTFLoader:', err) + ); + }, [modelUrl]); + + useEffect(() => { + const group = sceneGroupRef.current; + if (!group || !sceneDescriptor) return; + buildSceneFromDescriptor(sceneDescriptor, group); + }, [sceneDescriptor]); + + useEffect(() => { + if (!mountRef.current) return; + + const mount = mountRef.current; + const scene = new THREE.Scene(); + scene.background = new THREE.Color('#040816'); + scene.fog = new THREE.Fog('#040816', 12, 28); + + const camera = new THREE.PerspectiveCamera(38, 1, 0.1, 100); + camera.position.set(0, 1.55, 6.1); + + const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true }); + renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); + renderer.setSize(mount.clientWidth, mount.clientHeight); + renderer.setClearColor(0x040816, 1); + mount.appendChild(renderer.domElement); + + const resizeObserver = new ResizeObserver(() => { + const width = mount.clientWidth; + const height = mount.clientHeight; + renderer.setSize(width, height); + camera.aspect = width / height; + camera.updateProjectionMatrix(); + }); + resizeObserver.observe(mount); + + const ambient = new THREE.AmbientLight(0x7ad8ff, 0.25); + scene.add(ambient); + const keyLight = new THREE.DirectionalLight(0x9ee7ff, 1.3); + keyLight.position.set(6, 8, 8); + scene.add(keyLight); + const fillLight = new THREE.PointLight(0x00d5ff, 2.5, 22); + fillLight.position.set(-4, 2, 4); + scene.add(fillLight); + const rimLight = new THREE.PointLight(0x7c3aed, 1.4, 18); + rimLight.position.set(4, 1, -5); + scene.add(rimLight); + + const starGeometry = new THREE.BufferGeometry(); + const starCount = 220; + const starPositions = new Float32Array(starCount * 3); + for (let i = 0; i < starCount; i += 1) { + const radius = 12 + Math.random() * 12; + const theta = Math.random() * Math.PI * 2; + const phi = Math.acos(THREE.MathUtils.randFloatSpread(2)); + starPositions[i * 3] = Math.cos(theta) * Math.sin(phi) * radius; + starPositions[i * 3 + 1] = THREE.MathUtils.randFloatSpread(10) + 1.5; + starPositions[i * 3 + 2] = Math.sin(theta) * Math.sin(phi) * radius - 4; + } + starGeometry.setAttribute('position', new THREE.BufferAttribute(starPositions, 3)); + const starMaterial = new THREE.PointsMaterial({ + color: 0x8be9ff, + size: 0.045, + transparent: true, + opacity: 0.75, + depthWrite: false, + }); + const stars = new THREE.Points(starGeometry, starMaterial); + scene.add(stars); + + const halo = new THREE.Mesh( + new THREE.TorusGeometry(3.35, 0.06, 10, 120), + new THREE.MeshBasicMaterial({ color: 0x2dd4bf, transparent: true, opacity: 0.14 }) + ); + halo.rotation.x = Math.PI / 2; + halo.position.set(0, 0.7, -1.4); + scene.add(halo); + + const objectGroup = new THREE.Group(); + objectGroup.position.y = 0.85; + objectGroup.scale.setScalar(1.18); + scene.add(objectGroup); + objectGroupRef.current = objectGroup; + + const mapGroup = new THREE.Group(); + mapGroup.visible = false; + scene.add(mapGroup); + + const ground = new THREE.Mesh( + new THREE.CircleGeometry(7.8, 64), + new THREE.MeshStandardMaterial({ + color: 0x091225, + metalness: 0.15, + roughness: 0.75, + transparent: true, + opacity: 0.9, + }) + ); + ground.rotation.x = -Math.PI / 2; + ground.position.y = -2.1; + scene.add(ground); + + const grid = new THREE.GridHelper(18, 36, 0x2bd9ff, 0x16314f); + (grid.material as THREE.Material).transparent = true; + (grid.material as THREE.Material).opacity = 0.35; + grid.position.y = -2.08; + scene.add(grid); + + const centralCore = new THREE.Mesh( + new THREE.SphereGeometry(0.78, 40, 40), + new THREE.MeshStandardMaterial({ + color: 0x0ff0ff, + emissive: 0x19d9ff, + emissiveIntensity: 0.8, + metalness: 0.45, + roughness: 0.25, + }) + ); + objectGroup.add(centralCore); + + const emitterRing = new THREE.Mesh( + new THREE.TorusGeometry(1.28, 0.08, 12, 80), + new THREE.MeshStandardMaterial({ + color: 0x61dbfb, + emissive: 0x0ea5e9, + emissiveIntensity: 0.75, + transparent: true, + opacity: 0.95, + }) + ); + emitterRing.rotation.x = Math.PI / 2; + objectGroup.add(emitterRing); + + const shellRing = new THREE.Mesh( + new THREE.TorusGeometry(1.9, 0.06, 12, 90), + new THREE.MeshStandardMaterial({ + color: 0x8b5cf6, + emissive: 0x7c3aed, + emissiveIntensity: 0.35, + transparent: true, + opacity: 0.8, + }) + ); + shellRing.rotation.x = Math.PI / 2; + shellRing.rotation.z = Math.PI / 7; + objectGroup.add(shellRing); + + const moduleMaterial = new THREE.MeshStandardMaterial({ + color: 0x14294b, + metalness: 0.65, + roughness: 0.28, + emissive: 0x081422, + emissiveIntensity: 0.25, + }); + const accentMaterial = new THREE.MeshStandardMaterial({ + color: 0x1ca9ff, + metalness: 0.3, + roughness: 0.3, + emissive: 0x19d9ff, + emissiveIntensity: 0.45, + }); + + type PartDef = { mesh: THREE.Mesh; direction: THREE.Vector3; basePosition: THREE.Vector3 }; + const partDefs: PartDef[] = []; + const directions = [ + new THREE.Vector3(1, 0.18, 0.1), + new THREE.Vector3(-1, 0.08, 0.15), + new THREE.Vector3(0.1, 0.28, 1), + new THREE.Vector3(-0.12, -0.14, -1), + new THREE.Vector3(0.85, -0.2, -0.5), + new THREE.Vector3(-0.8, 0.25, 0.55), + ]; + + directions.forEach((direction, index) => { + const module = new THREE.Mesh( + new THREE.BoxGeometry(0.78, 0.35, 0.78), + index % 2 === 0 ? moduleMaterial.clone() : accentMaterial.clone() + ); + module.position.copy(direction.clone().multiplyScalar(1.7)); + module.scale.setScalar(index === 2 ? 1.05 : 1); + objectGroup.add(module); + + const cap = new THREE.Mesh( + new THREE.CylinderGeometry(0.18, 0.18, 0.9, 24), + new THREE.MeshStandardMaterial({ + color: 0x8be9ff, + emissive: 0x2bd9ff, + emissiveIntensity: 0.5, + metalness: 0.7, + roughness: 0.2, + }) + ); + cap.rotation.z = Math.PI / 2; + cap.position.copy(direction.clone().multiplyScalar(1.7)).add(new THREE.Vector3(0, 0.24, 0)); + objectGroup.add(cap); + + partDefs.push({ mesh: module, direction, basePosition: module.position.clone() }); + partDefs.push({ mesh: cap, direction, basePosition: cap.position.clone() }); + }); + + const orbitRings = [1.1, 1.65, 2.35].map((radius, index) => { + const ring = new THREE.Mesh( + new THREE.TorusGeometry(radius, 0.03, 12, 96), + new THREE.MeshBasicMaterial({ + color: index === 0 ? 0x61dbfb : index === 1 ? 0x8b5cf6 : 0x16a34a, + transparent: true, + opacity: 0.45, + }) + ); + ring.rotation.x = Math.PI / (2 + index * 0.45); + ring.rotation.y = index * 0.65; + objectGroup.add(ring); + return ring; + }); + + const mapSurface = new THREE.Mesh( + new THREE.CircleGeometry(4.2, 64), + new THREE.MeshStandardMaterial({ + color: 0x08111f, + emissive: 0x08111f, + metalness: 0.15, + roughness: 0.82, + transparent: true, + opacity: 0.96, + }) + ); + mapSurface.rotation.x = -Math.PI / 2; + mapGroup.add(mapSurface); + + const mapGrid = new THREE.GridHelper(9, 18, 0x38bdf8, 0x13324d); + (mapGrid.material as THREE.Material).transparent = true; + (mapGrid.material as THREE.Material).opacity = 0.3; + mapGrid.position.y = 0.02; + mapGroup.add(mapGrid); + + const marker = new THREE.Mesh( + new THREE.SphereGeometry(0.18, 24, 24), + new THREE.MeshStandardMaterial({ + color: 0xffd166, + emissive: 0xffb703, + emissiveIntensity: 0.85, + metalness: 0.2, + roughness: 0.3, + }) + ); + marker.position.set(0.7, 0.3, -0.1); + mapGroup.add(marker); + + const route = new THREE.Mesh( + new THREE.CylinderGeometry(0.035, 0.035, 5.8, 12), + new THREE.MeshStandardMaterial({ + color: 0x22c55e, + emissive: 0x22c55e, + emissiveIntensity: 0.45, + }) + ); + route.position.set(-0.8, 0.18, -0.1); + route.rotation.z = Math.PI / 2.5; + route.rotation.y = Math.PI / 6; + mapGroup.add(route); + + const pulseRing = new THREE.Mesh( + new THREE.TorusGeometry(1.1, 0.06, 8, 80), + new THREE.MeshBasicMaterial({ color: 0x2dd4bf, transparent: true, opacity: 0.55 }) + ); + pulseRing.rotation.x = Math.PI / 2; + mapGroup.add(pulseRing); + + // ── Generated scene group (scene mode) ──────────────────────────────────── + const sceneGroup = new THREE.Group(); + sceneGroup.visible = false; + scene.add(sceneGroup); + sceneGroupRef.current = sceneGroup; + + // ── Neural Graph (brand / content / analytics modes) ────────────────────── + const neuralGroup = new THREE.Group(); + neuralGroup.visible = false; + scene.add(neuralGroup); + + const NODE_COUNT = 52; + const nodePosData = new Float32Array(NODE_COUNT * 3); + const nodeVelData = new Float32Array(NODE_COUNT * 3); + const nodePhase = new Float32Array(NODE_COUNT); + + // Node 0 = center hub; rest scattered in a flat-ish ellipsoid + nodePosData[0] = 0; nodePosData[1] = 0; nodePosData[2] = 0; + nodeVelData[0] = 0; nodeVelData[1] = 0; nodeVelData[2] = 0; + for (let i = 1; i < NODE_COUNT; i++) { + const r = 0.9 + Math.random() * 3.6; + const theta = Math.random() * Math.PI * 2; + nodePosData[i * 3] = r * Math.cos(theta); + nodePosData[i * 3 + 1] = (Math.random() - 0.5) * 2.6; + nodePosData[i * 3 + 2] = r * Math.sin(theta) * 0.28; + nodeVelData[i * 3] = (Math.random() - 0.5) * 0.0045; + nodeVelData[i * 3 + 1] = (Math.random() - 0.5) * 0.003; + nodeVelData[i * 3 + 2] = (Math.random() - 0.5) * 0.001; + nodePhase[i] = Math.random() * Math.PI * 2; + } + + const nodeGeo = new THREE.BufferGeometry(); + const nodePosAttr = new THREE.BufferAttribute(nodePosData, 3); + nodeGeo.setAttribute('position', nodePosAttr); + const nodeMat = new THREE.PointsMaterial({ + color: 0x7dd3fc, + size: 0.07, + transparent: true, + opacity: 0.75, + depthWrite: false, + sizeAttenuation: true, + }); + neuralGroup.add(new THREE.Points(nodeGeo, nodeMat)); + + // Build edges connecting nearby nodes + const edgePairs: number[][] = []; + const rawEdgePos: number[] = []; + const CONNECT_DIST = 2.4; + for (let i = 0; i < NODE_COUNT; i++) { + for (let j = i + 1; j < NODE_COUNT; j++) { + const dx = nodePosData[i * 3] - nodePosData[j * 3]; + const dy = nodePosData[i * 3 + 1] - nodePosData[j * 3 + 1]; + const dz = nodePosData[i * 3 + 2] - nodePosData[j * 3 + 2]; + if (dx * dx + dy * dy + dz * dz < CONNECT_DIST * CONNECT_DIST) { + edgePairs.push([i, j]); + rawEdgePos.push( + nodePosData[i * 3], nodePosData[i * 3 + 1], nodePosData[i * 3 + 2], + nodePosData[j * 3], nodePosData[j * 3 + 1], nodePosData[j * 3 + 2] + ); + } + } + } + const edgeGeo = new THREE.BufferGeometry(); + const edgePosAttr = new THREE.BufferAttribute(new Float32Array(rawEdgePos), 3); + edgePosAttr.setUsage(THREE.DynamicDrawUsage); + edgeGeo.setAttribute('position', edgePosAttr); + const edgeMat = new THREE.LineBasicMaterial({ color: 0x1e3a5f, transparent: true, opacity: 0.38 }); + neuralGroup.add(new THREE.LineSegments(edgeGeo, edgeMat)); + + // Signal particles — dots that travel along edges when firing + const SIGNAL_COUNT = 14; + const sigData: { edgeIdx: number; t: number; speed: number; active: boolean }[] = + Array.from({ length: SIGNAL_COUNT }, () => ({ edgeIdx: 0, t: 0, speed: 0.5, active: false })); + const sigPosArr = new Float32Array(SIGNAL_COUNT * 3); + const sigGeo = new THREE.BufferGeometry(); + const sigPosAttr = new THREE.BufferAttribute(sigPosArr, 3); + sigPosAttr.setUsage(THREE.DynamicDrawUsage); + sigGeo.setAttribute('position', sigPosAttr); + const sigMat = new THREE.PointsMaterial({ + color: 0xffffff, + size: 0.11, + transparent: true, + opacity: 0.92, + depthWrite: false, + sizeAttenuation: true, + }); + neuralGroup.add(new THREE.Points(sigGeo, sigMat)); + + const clock = new THREE.Clock(); + let frameId = 0; + + const animate = () => { + frameId = window.requestAnimationFrame(animate); + const delta = Math.min(clock.getDelta(), 0.05); + const elapsed = clock.elapsedTime; + const current = stateRef.current; + const firing = thinkingRef.current; + + const isObjectMode = current.mode === 'object' || current.mode === 'overview'; + const isNeuralMode = current.mode === 'brand' || current.mode === 'content' || current.mode === 'analytics'; + const isSceneMode = current.mode === 'scene'; + objectGroup.visible = isObjectMode; + mapGroup.visible = current.mode === 'map'; + neuralGroup.visible = isNeuralMode; + sceneGroup.visible = isSceneMode; + grid.visible = isObjectMode; + ground.visible = isObjectMode; + halo.visible = isObjectMode; + + if (isSceneMode) { + sceneGroup.rotation.y += 0.004; + sceneGroup.children.forEach((child, i) => { + child.position.y += Math.sin(elapsed * 0.55 + i * 1.4) * 0.001; + }); + } + + const explodeTarget = current.exploded ? 1 : 0; + const powerTarget = current.powerOn ? 1 : 0; + const camZTarget = isNeuralMode ? 7.2 : isSceneMode ? 6.5 : (current.cameraDistance || 6.2); + const camY = current.mode === 'map' ? 2.1 : isNeuralMode ? 0.2 : isSceneMode ? 1.2 : 2.4; + const lookY = current.mode === 'map' ? 0.2 : isNeuralMode ? 0.0 : isSceneMode ? 0.2 : 0.85; + + camera.position.z += (camZTarget - camera.position.z) * 0.04; + camera.position.y += (camY - camera.position.y) * 0.04; + camera.lookAt(0, lookY, 0); + + objectGroup.rotation.y += current.rotationSpeed; + objectGroup.rotation.x = Math.sin(elapsed * 0.25) * 0.03; + mapGroup.rotation.y = Math.sin(elapsed * 0.2) * 0.05; + stars.rotation.y += 0.0006; + stars.rotation.x = Math.sin(elapsed * 0.08) * 0.01; + if (isObjectMode) halo.rotation.z += 0.0015; + + if (isNeuralMode) { + // Drift nodes + for (let i = 1; i < NODE_COUNT; i++) { + nodePosData[i * 3] += nodeVelData[i * 3]; + nodePosData[i * 3 + 1] += nodeVelData[i * 3 + 1]; + nodePosData[i * 3 + 2] += nodeVelData[i * 3 + 2]; + if (Math.abs(nodePosData[i * 3]) > 5.0) nodeVelData[i * 3] *= -1; + if (Math.abs(nodePosData[i * 3 + 1]) > 2.8) nodeVelData[i * 3 + 1] *= -1; + if (Math.abs(nodePosData[i * 3 + 2]) > 1.3) nodeVelData[i * 3 + 2] *= -1; + } + nodePosAttr.needsUpdate = true; + + // Sync edge endpoints to drifted nodes + const edgeArr = edgePosAttr.array as Float32Array; + for (let i = 0; i < edgePairs.length; i++) { + const a = edgePairs[i][0]; const b = edgePairs[i][1]; + edgeArr[i * 6] = nodePosData[a * 3]; edgeArr[i * 6 + 1] = nodePosData[a * 3 + 1]; edgeArr[i * 6 + 2] = nodePosData[a * 3 + 2]; + edgeArr[i * 6 + 3] = nodePosData[b * 3]; edgeArr[i * 6 + 4] = nodePosData[b * 3 + 1]; edgeArr[i * 6 + 5] = nodePosData[b * 3 + 2]; + } + edgePosAttr.needsUpdate = true; + + // Visual state — calm, firing (thinking), or speaking burst + const speaking = performance.now() < speakingUntilRef.current; + const active = firing || speaking; + const freq = active ? (firing ? 3.2 : 5.5) : 0.55; + nodeMat.opacity = Math.min(0.98, 0.55 + Math.sin(elapsed * freq + 0.5) * 0.18 + (active ? 0.22 : 0)); + edgeMat.opacity = Math.min(0.85, 0.22 + (active ? 0.32 : 0) + Math.sin(elapsed * (active ? 2.8 : 0.7)) * 0.07); + + const nodeIdleColor = current.mode === 'brand' ? 0x818cf8 : current.mode === 'content' ? 0xe879f9 : 0x7dd3fc; + const nodeFireColor = current.mode === 'brand' ? 0xa78bfa : current.mode === 'content' ? 0xf472b6 : 0x22d3ee; + const edgeIdleColor = current.mode === 'brand' ? 0x312e81 : current.mode === 'content' ? 0x4a044e : 0x1e3a5f; + const edgeFireColor = current.mode === 'brand' ? 0x7c3aed : current.mode === 'content' ? 0xbe185d : 0x3b82f6; + nodeMat.color.setHex(active ? nodeFireColor : nodeIdleColor); + edgeMat.color.setHex(active ? edgeFireColor : edgeIdleColor); + + // Signal particles travel along edges + const spawnRate = firing ? 0.07 : speaking ? 0.04 : 0.005; + for (let si = 0; si < SIGNAL_COUNT; si++) { + const sig = sigData[si]; + if (!sig.active && edgePairs.length > 0 && Math.random() < spawnRate) { + sig.active = true; + sig.edgeIdx = Math.floor(Math.random() * edgePairs.length); + sig.t = 0; + sig.speed = (active ? 1.3 : 0.4) + Math.random() * 0.6; + } + if (sig.active) { + sig.t += sig.speed * delta; + if (sig.t >= 1) { sig.active = false; } + const a = edgePairs[sig.edgeIdx][0]; const b = edgePairs[sig.edgeIdx][1]; + sigPosArr[si * 3] = nodePosData[a * 3] + (nodePosData[b * 3] - nodePosData[a * 3]) * sig.t; + sigPosArr[si * 3 + 1] = nodePosData[a * 3 + 1] + (nodePosData[b * 3 + 1] - nodePosData[a * 3 + 1]) * sig.t; + sigPosArr[si * 3 + 2] = nodePosData[a * 3 + 2] + (nodePosData[b * 3 + 2] - nodePosData[a * 3 + 2]) * sig.t; + } else { + sigPosArr[si * 3 + 1] = -1000; + } + } + sigPosAttr.needsUpdate = true; + } + + partDefs.forEach(({ mesh, direction, basePosition }) => { + const offset = direction.clone().multiplyScalar(2.1 * explodeTarget); + mesh.position.lerpVectors(basePosition, basePosition.clone().add(offset), 0.15); + const material = mesh.material as THREE.MeshStandardMaterial; + material.emissiveIntensity = 0.18 + powerTarget * 0.7; + }); + + centralCore.scale.setScalar(1 + powerTarget * 0.12 + Math.sin(elapsed * 2.4) * 0.02); + emitterRing.scale.setScalar(1 + powerTarget * 0.08 + Math.sin(elapsed * 1.5) * 0.01); + shellRing.rotation.z += 0.003 + powerTarget * 0.005; + + orbitRings.forEach((ring, index) => { + ring.rotation.x += 0.0012 + index * 0.0003; + ring.rotation.y += 0.001 + index * 0.00035; + }); + + mapSurface.scale.setScalar(1 + Math.sin(elapsed * 2) * 0.015); + pulseRing.scale.setScalar(1 + Math.sin(elapsed * 2.8) * 0.08); + marker.position.y = 0.28 + Math.sin(elapsed * 3.1) * 0.08; + route.rotation.y += 0.002; + fillLight.intensity = 1.9 + powerTarget * 1.25; + rimLight.intensity = 1.1 + powerTarget * 0.75; + + renderer.render(scene, camera); + }; + + animate(); + + return () => { + window.cancelAnimationFrame(frameId); + resizeObserver.disconnect(); + sceneGroupRef.current = null; + starGeometry.dispose(); + starMaterial.dispose(); + halo.geometry.dispose(); + (halo.material as THREE.Material).dispose(); + nodeGeo.dispose(); + nodeMat.dispose(); + edgeGeo.dispose(); + edgeMat.dispose(); + sigGeo.dispose(); + sigMat.dispose(); + renderer.dispose(); + mount.removeChild(renderer.domElement); + }; + }, []); + + return ( +
+
+
+
+
+
+
+
+
+
+
+ ); +} + +// ─── Page ───────────────────────────────────────────────────────────────────── + +export default function JarvisPage() { + const [input, setInput] = useState(''); + const [sceneState, setSceneState] = useState(() => createInitialJarvisState()); + const [transcript, setTranscript] = useState(() => [ + { + id: makeTranscriptId(), + role: 'system', + text: 'HUD initialized. Issue a command or tap a chip.', + timestamp: new Date().toISOString(), + }, + ]); + const [isThinking, setIsThinking] = useState(false); + const [isListening, setIsListening] = useState(false); + const [modelUrl, setModelUrl] = useState(undefined); + const [sceneDescriptor, setSceneDescriptor] = useState(undefined); + const recognitionRef = useRef(null); + const transcriptEndRef = useRef(null); + const apiKeyRef = useRef(null); + const fileInputRef = useRef(null); + + const fact = useMemo(() => getFactForTarget(sceneState.activeTarget), [sceneState.activeTarget]); + + useEffect(() => { + if (!isRunningInElectron()) return; + getAccomplish().getApiKey() + .then((key) => { apiKeyRef.current = key; }) + .catch(() => {}); + }, []); + + useEffect(() => { + transcriptEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + }, [transcript]); + + const executeCommand = useCallback( + async (rawInput: string) => { + const trimmed = rawInput.trim(); + if (!trimmed || isThinking) return; + + const command = parseJarvisCommand(trimmed); + const nextState = applyJarvisCommand(sceneState, command); + const localReply = describeJarvisCommand(command, nextState); + + setSceneState(nextState); + setInput(''); + + const ts = new Date().toISOString(); + const userEntry: JarvisTranscriptEntry = { id: makeTranscriptId(), role: 'user', text: trimmed, timestamp: ts }; + const apiKey = apiKeyRef.current; + + // 3D scene generation — calls local Ollama, no API key required + if (command.intent === 'create_scene') { + const assistantId = makeTranscriptId(); + setTranscript((prev) => [ + ...prev, + userEntry, + { id: assistantId, role: 'assistant', text: 'Generating scene via local LLM...', timestamp: ts }, + ]); + setIsThinking(true); + try { + const descriptor = await callLocalLLMForScene(trimmed); + setSceneDescriptor(descriptor); + const n = descriptor.objects?.length ?? 0; + setTranscript((prev) => + prev.map((e) => (e.id === assistantId ? { ...e, text: `Scene rendered — ${n} objects composed.` } : e)) + ); + } catch (err) { + const msg = err instanceof Error && err.message.includes('Ollama') + ? 'Ollama not running. Start Ollama and try again.' + : 'Scene generation failed. Make sure Ollama is running with llama3.2.'; + setTranscript((prev) => + prev.map((e) => (e.id === assistantId ? { ...e, text: msg } : e)) + ); + } + setIsThinking(false); + return; + } + + if (!apiKey) { + setTranscript((prev) => [ + ...prev, + userEntry, + { id: makeTranscriptId(), role: 'assistant', text: localReply, timestamp: ts }, + ]); + return; + } + + const assistantId = makeTranscriptId(); + setTranscript((prev) => [ + ...prev, + userEntry, + { id: assistantId, role: 'assistant', text: '...', timestamp: ts }, + ]); + setIsThinking(true); + + let finalReply = localReply; + try { + finalReply = await callAnthropicHaiku(apiKey, trimmed); + } catch { + // fall back to local reply + } + + setTranscript((prev) => + prev.map((e) => (e.id === assistantId ? { ...e, text: finalReply } : e)) + ); + setIsThinking(false); + }, + [sceneState, isThinking] + ); + + const onSubmit = useCallback(() => { + void executeCommand(input); + }, [executeCommand, input]); + + const onPreset = useCallback( + (prompt: string) => { + setInput(prompt); + void executeCommand(prompt); + }, + [executeCommand] + ); + + const toggleListening = useCallback(() => { + if (isListening) { + recognitionRef.current?.stop(); + setIsListening(false); + return; + } + + const SpeechRecognition = + (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition; + if (!SpeechRecognition) return; + + const recognition = new SpeechRecognition(); + recognition.continuous = false; + recognition.interimResults = false; + recognition.lang = 'en-US'; + + recognition.onresult = (event: any) => { + const heard: string = event.results[0][0].transcript; + setInput(heard); + void executeCommand(heard); + }; + recognition.onend = () => setIsListening(false); + recognition.onerror = () => setIsListening(false); + + recognition.start(); + recognitionRef.current = recognition; + setIsListening(true); + }, [isListening, executeCommand]); + + const handleFileSelect = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (!file) return; + const prev = modelUrl; + const url = URL.createObjectURL(file); + setModelUrl(url); + void executeCommand(`Load ${file.name.replace(/\.[^.]+$/, '')}`); + e.target.value = ''; + if (prev) URL.revokeObjectURL(prev); + }, + [modelUrl, executeCommand] + ); + + const stateChips = useMemo( + () => + [ + sceneState.powerOn ? { label: 'Powered', color: 'text-emerald-400' } : null, + sceneState.exploded ? { label: 'Exploded', color: 'text-amber-400' } : null, + sceneState.mode === 'map' ? { label: 'Map', color: 'text-sky-400' } : null, + sceneState.mode === 'brand' ? { label: 'Brand', color: 'text-violet-400' } : null, + sceneState.mode === 'content' ? { label: 'Content', color: 'text-pink-400' } : null, + sceneState.mode === 'analytics' ? { label: 'Analytics', color: 'text-cyan-400' } : null, + sceneState.mode === 'scene' ? { label: 'Scene', color: 'text-emerald-400' } : null, + ].filter(Boolean) as { label: string; color: string }[], + [sceneState] + ); + + const commandChips = useMemo(() => { + const m = sceneState.mode; + if (m === 'brand') return [ + { label: 'Tagline', prompt: 'Write 3 sharp taglines for a bold digital brand', icon: Sparkles }, + { label: 'Value prop', prompt: 'Write a one-sentence value proposition for a SaaS product', icon: Zap }, + { label: 'Brand voice', prompt: 'Describe a brand voice that is bold, minimal, and modern', icon: Palette }, + { label: 'Ad copy', prompt: 'Write 3 short ad copy lines for a digital product launch', icon: TrendingUp }, + { label: 'Switch scene', prompt: 'Load reactor core', icon: Box }, + ]; + if (m === 'content') return [ + { label: 'IG caption', prompt: 'Write an Instagram caption with a strong hook for a digital product launch', icon: Hash }, + { label: 'Email subject', prompt: 'Write 5 email subject lines for a product launch campaign', icon: Mail }, + { label: 'Twitter thread', prompt: 'Write the first 3 tweets of a thread about building a successful digital product', icon: Sparkles }, + { label: 'Blog hook', prompt: 'Write a compelling opening paragraph for a blog post about digital products', icon: FileText }, + { label: 'Switch scene', prompt: 'Load reactor core', icon: Box }, + ]; + if (m === 'analytics') return [ + { label: 'KPI summary', prompt: 'Write a concise KPI performance summary for a growing digital product', icon: BarChart2 }, + { label: 'Growth post', prompt: 'Write a LinkedIn post announcing strong user growth this quarter', icon: TrendingUp }, + { label: 'Investor update', prompt: 'Write a one-paragraph investor update highlighting strong monthly metrics', icon: Zap }, + { label: 'Data insight', prompt: 'Write 3 data-driven insights for a SaaS product with strong retention', icon: Sparkles }, + { label: 'Switch scene', prompt: 'Load reactor core', icon: Box }, + ]; + // scene mode + if (m === 'scene') return [ + { label: 'Create new', prompt: 'Create a glowing icosahedron with orbiting rings in deep violet and cyan', icon: Wand2 }, + { label: 'Brand 3D', prompt: 'Create a bold geometric logo shape with hexagonal rings and brand violet glow', icon: Palette }, + { label: 'Product 3D', prompt: 'Create a sleek product box with a glowing edge and floating spheres around it', icon: Box }, + { label: 'Brand mode', prompt: 'Switch to brand mode', icon: Sparkles }, + { label: 'Load model', prompt: '', icon: Box, action: () => fileInputRef.current?.click() }, + ]; + return [ + { label: 'Load 3D', prompt: '', icon: Box, action: () => fileInputRef.current?.click() }, + { label: 'Create 3D', prompt: 'Create a glowing orb surrounded by orbiting violet rings and cyan signal particles', icon: Wand2 }, + { label: 'Brand', prompt: 'Switch to brand mode', icon: Palette }, + { label: 'Content', prompt: 'Switch to content mode', icon: FileText }, + { label: 'Analytics', prompt: 'Switch to analytics mode', icon: BarChart2 }, + ]; + }, [sceneState.mode]); + + return ( + + {/* ── Full-bleed 3D viewport ── */} +
+ +
+ + {/* ── Top HUD bar ── */} +
+
+ + + HUD online + + + + {sceneState.mode === 'map' ? 'Map' : sceneState.mode === 'brand' ? 'Brand' : sceneState.mode === 'content' ? 'Content' : sceneState.mode === 'analytics' ? 'Analytics' : sceneState.mode === 'scene' ? 'Scene' : 'Object'} + + {stateChips.map((chip) => ( + + {chip.label} + + ))} + {isThinking && ( + + + Processing + + )} +
+
+

Module

+

{fact.title}

+
+
+ + {/* ── Right panel: scene brief + transcript ── */} +
+ {/* Scene brief */} +
+
+ + Scene brief +
+

{fact.summary}

+
+ {fact.bullets.map((b) => ( + + {b} + + ))} +
+
+ + {/* Transcript feed */} +
+ + + Comms + + +
+ +
+ {transcript.map((entry) => ( +
+ {entry.role === 'user' && ( +

+ > + {entry.text} +

+ )} + {entry.role === 'assistant' && ( +
+ // + {entry.text} + {entry.text !== '...' && ( + + )} +
+ )} + {entry.role === 'system' && ( +

{entry.text}

+ )} +
+ ))} +
+
+ +
+ + {/* ── Bottom command bar ── */} +
+ {/* Quick chips */} +
+ {commandChips.map((chip) => { + const Icon = chip.icon; + return ( + + ); + })} +
+ + {/* Input row */} +
+ + setInput(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + onSubmit(); + } + }} + placeholder="Brand mode · analytics · write 3 Instagram hooks for..." + className="border-cyan-500/20 bg-slate-900/80 text-cyan-50 placeholder:text-slate-600 focus-visible:ring-cyan-500/30 focus-visible:border-cyan-500/40" + /> + +
+
+ + {/* Hidden 3D file picker */} + + + ); +} diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/vite-env.d.ts b/apps/openwork-memos-integration/apps/desktop/src/renderer/vite-env.d.ts new file mode 100644 index 000000000..0546f921e --- /dev/null +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/vite-env.d.ts @@ -0,0 +1,16 @@ +/// + +import type { AccomplishAPI } from './lib/accomplish'; + +declare global { + interface Window { + accomplish: AccomplishAPI; + accomplishShell?: { + version: string; + platform: string; + isElectron: boolean; + }; + } +} + +export {}; diff --git a/apps/openwork-memos-integration/packages/shared/src/types/jarvis.ts b/apps/openwork-memos-integration/packages/shared/src/types/jarvis.ts new file mode 100644 index 000000000..b6ed62a19 --- /dev/null +++ b/apps/openwork-memos-integration/packages/shared/src/types/jarvis.ts @@ -0,0 +1,48 @@ +export type JarvisMode = 'object' | 'map' | 'overview' | 'brand' | 'content' | 'analytics' | 'scene'; + +export type JarvisCommandType = + | 'load_object' + | 'summarize_object' + | 'explode_part' + | 'assemble_part' + | 'power_on' + | 'power_off' + | 'highlight_part' + | 'rotate_view' + | 'switch_to_map' + | 'annotate_map' + | 'explain_scene' + | 'switch_to_brand' + | 'switch_to_content' + | 'switch_to_analytics' + | 'create_scene'; + +export interface JarvisCommand { + intent: JarvisCommandType; + target?: string; + query?: string; + mode?: JarvisMode; + direction?: 'left' | 'right' | 'up' | 'down'; + speed?: number; + zoom?: number; + raw: string; +} + +export interface JarvisSceneState { + mode: JarvisMode; + activeTarget: string; + summary: string; + powerOn: boolean; + exploded: boolean; + highlightedPart?: string; + mapFocus?: string; + rotationSpeed: number; + cameraDistance: number; +} + +export interface JarvisTranscriptEntry { + id: string; + role: 'user' | 'assistant' | 'system'; + text: string; + timestamp: string; +} From 8a4e25d9b96a6773f1100a4a802f050b4dbadc13 Mon Sep 17 00:00:00 2001 From: Jose Guerra Date: Thu, 11 Jun 2026 22:59:00 -0700 Subject: [PATCH 2/5] fix(jarvis): use configured Anthropic API key for scene generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switches 3D scene generation from Ollama (required separate install) to the API key already stored in the app keychain. Users get scene rendering as soon as they have an API key configured — no additional setup required. Also widens the trigger regex to catch more natural phrasings like "design a 3D brand mark" or "3D render of a product". Co-Authored-By: Claude Sonnet 4.6 --- .../src/renderer/lib/jarvis/command-parser.ts | 3 +- .../desktop/src/renderer/pages/Jarvis.tsx | 47 +++++++++++-------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts b/apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts index 6ad31df99..6fa651773 100644 --- a/apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/lib/jarvis/command-parser.ts @@ -42,7 +42,8 @@ export function parseJarvisCommand(raw: string): JarvisCommand { }; } - if (/\b(create|generate|make me|build)\b.{0,30}\b(3d|scene|model|sphere|cube|box|torus|ring|orb|shape|object|structure|render me)\b/i.test(input)) { + if (/\b(create|generate|make|build|design|render)\b.{0,40}\b(3d|scene|model|sphere|cube|box|torus|ring|orb|shape|object|structure|object|rendering|composition)\b/i.test(input) + || /\b(3d|three.?d)\s+(render|scene|model|object|composition|version)\b/i.test(input)) { return { intent: 'create_scene', target: target ?? 'custom scene', diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx index 1889760ff..2d0c73fcd 100644 --- a/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx @@ -72,7 +72,7 @@ const OBJECT_FACTS: Record { - const response = await fetch('http://localhost:11434/api/chat', { +async function callLLMForScene(apiKey: string, prompt: string): Promise { + const response = await fetch('https://api.anthropic.com/v1/messages', { method: 'POST', - headers: { 'content-type': 'application/json' }, + headers: { + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + 'content-type': 'application/json', + }, body: JSON.stringify({ - model: 'llama3.2', - messages: [ - { role: 'system', content: SCENE_GEN_SYSTEM }, - { role: 'user', content: prompt }, - ], - stream: false, + model: 'claude-haiku-4-5-20251001', + max_tokens: 1200, + system: SCENE_GEN_SYSTEM, + messages: [{ role: 'user', content: prompt }], }), }); - if (!response.ok) throw new Error(`Ollama HTTP ${response.status}`); + if (!response.ok) throw new Error(`API HTTP ${response.status}`); const data = await response.json(); - const text: string = (data.message?.content ?? data.response ?? '').trim(); + const text: string = (data.content?.[0]?.text ?? '').trim(); const jsonMatch = text.replace(/```json\n?|\n?```|```\n?/g, '').match(/\{[\s\S]*\}/); if (!jsonMatch) throw new Error('No JSON in response'); return JSON.parse(jsonMatch[0]) as SceneDescriptor; @@ -849,28 +851,33 @@ export default function JarvisPage() { const userEntry: JarvisTranscriptEntry = { id: makeTranscriptId(), role: 'user', text: trimmed, timestamp: ts }; const apiKey = apiKeyRef.current; - // 3D scene generation — calls local Ollama, no API key required + // 3D scene generation — uses the configured API key if (command.intent === 'create_scene') { + if (!apiKey) { + setTranscript((prev) => [ + ...prev, + userEntry, + { id: makeTranscriptId(), role: 'assistant', text: 'Add an API key in Settings to generate 3D scenes.', timestamp: ts }, + ]); + return; + } const assistantId = makeTranscriptId(); setTranscript((prev) => [ ...prev, userEntry, - { id: assistantId, role: 'assistant', text: 'Generating scene via local LLM...', timestamp: ts }, + { id: assistantId, role: 'assistant', text: 'Generating scene...', timestamp: ts }, ]); setIsThinking(true); try { - const descriptor = await callLocalLLMForScene(trimmed); + const descriptor = await callLLMForScene(apiKey, trimmed); setSceneDescriptor(descriptor); const n = descriptor.objects?.length ?? 0; setTranscript((prev) => prev.map((e) => (e.id === assistantId ? { ...e, text: `Scene rendered — ${n} objects composed.` } : e)) ); - } catch (err) { - const msg = err instanceof Error && err.message.includes('Ollama') - ? 'Ollama not running. Start Ollama and try again.' - : 'Scene generation failed. Make sure Ollama is running with llama3.2.'; + } catch { setTranscript((prev) => - prev.map((e) => (e.id === assistantId ? { ...e, text: msg } : e)) + prev.map((e) => (e.id === assistantId ? { ...e, text: 'Scene generation failed. Check your API key.' } : e)) ); } setIsThinking(false); From ff80f56b7b7e76c340395e6513abafa2398e7433 Mon Sep 17 00:00:00 2001 From: Jose Guerra Date: Thu, 11 Jun 2026 23:03:41 -0700 Subject: [PATCH 3/5] feat(jarvis): use Ollama for 3D scene generation with auto model discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scene generation now calls local Ollama instead of the cloud API. Automatically picks the best available model by querying /api/tags and ranking against a preferred list (qwen2.5 > llama3.2 > llama3 > mistral > phi4 > gemma3 > …). Falls back gracefully with actionable error messages if Ollama isn't running or has no models installed. No API key required for 3D scene generation. Co-Authored-By: Claude Sonnet 4.6 --- .../desktop/src/renderer/pages/Jarvis.tsx | 78 +++++++++++-------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx index 2d0c73fcd..223b702bb 100644 --- a/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx @@ -72,8 +72,8 @@ const OBJECT_FACTS: Record { + const res = await fetch(`${OLLAMA_BASE}/api/tags`); + if (!res.ok) throw new Error('ollama_down'); + const data = await res.json(); + const installed: string[] = (data.models ?? []).map((m: { name: string }) => m.name); + if (installed.length === 0) throw new Error('ollama_no_models'); + for (const pref of OLLAMA_PREFERRED) { + const match = installed.find((n) => n.startsWith(pref)); + if (match) return match; + } + return installed[0]; +} -async function callLLMForScene(apiKey: string, prompt: string): Promise { - const response = await fetch('https://api.anthropic.com/v1/messages', { +async function callLLMForScene(prompt: string): Promise { + const model = await resolveOllamaModel(); + const res = await fetch(`${OLLAMA_BASE}/api/chat`, { method: 'POST', - headers: { - 'x-api-key': apiKey, - 'anthropic-version': '2023-06-01', - 'content-type': 'application/json', - }, + headers: { 'content-type': 'application/json' }, body: JSON.stringify({ - model: 'claude-haiku-4-5-20251001', - max_tokens: 1200, - system: SCENE_GEN_SYSTEM, - messages: [{ role: 'user', content: prompt }], + model, + messages: [ + { role: 'system', content: SCENE_GEN_SYSTEM }, + { role: 'user', content: prompt }, + ], + stream: false, }), }); - if (!response.ok) throw new Error(`API HTTP ${response.status}`); - const data = await response.json(); - const text: string = (data.content?.[0]?.text ?? '').trim(); - const jsonMatch = text.replace(/```json\n?|\n?```|```\n?/g, '').match(/\{[\s\S]*\}/); - if (!jsonMatch) throw new Error('No JSON in response'); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = await res.json(); + const text: string = (data.message?.content ?? '').trim(); + const stripped = text.replace(/```json\n?|\n?```|```\n?/g, ''); + const jsonMatch = stripped.match(/\{[\s\S]*\}/); + if (!jsonMatch) throw new Error('no_json'); return JSON.parse(jsonMatch[0]) as SceneDescriptor; } @@ -851,33 +868,30 @@ export default function JarvisPage() { const userEntry: JarvisTranscriptEntry = { id: makeTranscriptId(), role: 'user', text: trimmed, timestamp: ts }; const apiKey = apiKeyRef.current; - // 3D scene generation — uses the configured API key + // 3D scene generation via local Ollama — no API key needed if (command.intent === 'create_scene') { - if (!apiKey) { - setTranscript((prev) => [ - ...prev, - userEntry, - { id: makeTranscriptId(), role: 'assistant', text: 'Add an API key in Settings to generate 3D scenes.', timestamp: ts }, - ]); - return; - } const assistantId = makeTranscriptId(); setTranscript((prev) => [ ...prev, userEntry, - { id: assistantId, role: 'assistant', text: 'Generating scene...', timestamp: ts }, + { id: assistantId, role: 'assistant', text: 'Connecting to Ollama...', timestamp: ts }, ]); setIsThinking(true); try { - const descriptor = await callLLMForScene(apiKey, trimmed); + const descriptor = await callLLMForScene(trimmed); setSceneDescriptor(descriptor); const n = descriptor.objects?.length ?? 0; setTranscript((prev) => prev.map((e) => (e.id === assistantId ? { ...e, text: `Scene rendered — ${n} objects composed.` } : e)) ); - } catch { + } catch (err) { + const code = err instanceof Error ? err.message : ''; + const msg = + code === 'ollama_down' ? 'Ollama not running. Start it with: ollama serve' : + code === 'ollama_no_models' ? 'No models installed. Run: ollama pull llama3.2' : + 'Scene generation failed. Make sure Ollama is running.'; setTranscript((prev) => - prev.map((e) => (e.id === assistantId ? { ...e, text: 'Scene generation failed. Check your API key.' } : e)) + prev.map((e) => (e.id === assistantId ? { ...e, text: msg } : e)) ); } setIsThinking(false); From 5ee1f6a72ec512e16bdb99bd9d696dc97ce41434 Mon Sep 17 00:00:00 2001 From: Jose Guerra Date: Thu, 11 Jun 2026 23:11:20 -0700 Subject: [PATCH 4/5] fix(jarvis): make Ollama 3D scene generation actually robust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of broken scene generation: small local models (llama3.2 etc.) regularly emit malformed JSON — e.g. a mismatched quote like "rotation':[…] — which crashed JSON.parse and produced no scene. Verified against real Ollama. Fixes: - Use Ollama "format":"json" constrained decoding — guarantees parseable JSON output every time. This is the core fix. - Filter out embedding-only models (nomic-embed-text, *-minilm, bert families) so the fallback model picker never selects a model that can't chat. - normalizeDescriptor(): coerce and clamp every field from the model so a single bad value can't break the render — type aliases (cube->box, orb->sphere), numeric range clamps, hex/CSS color validation, and opacity<->transparent coupling. - Defensive JSON repair (smart quotes, single quotes, trailing commas) as a second line of defense. - Wrap fetches in try/catch to surface a clean "ollama_down" instead of an unhandled rejection. - Defer the scene-mode switch until meshes are ready so the user keeps seeing their current view during the multi-second generation instead of an empty void; actionable error messages for each failure mode. Verified: tsc --noEmit clean, vite build succeeds (renderer bundle emitted), and end-to-end generation against local Ollama renders valid scenes for 4 varied prompts. Co-Authored-By: Claude Opus 4.8 --- .../desktop/src/renderer/pages/Jarvis.tsx | 162 ++++++++++++++---- 1 file changed, 133 insertions(+), 29 deletions(-) diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx index 223b702bb..5a9ace52e 100644 --- a/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx @@ -141,48 +141,145 @@ interface SceneDescriptor { objects?: SceneObject[]; } -const SCENE_GEN_SYSTEM = `You are a Three.js 3D scene generator. Given a description, return ONLY a valid JSON object — no markdown, no code fences, no explanation. Structure: -{"objects":[{"type":"sphere|box|torus|cylinder|cone|icosahedron|ring|plane","size":1.0,"color":"#6366f1","emissive":"#4f46e5","emissiveIntensity":0.5,"metalness":0.3,"roughness":0.3,"position":[0,0,0],"rotation":[0,0,0],"transparent":false,"opacity":1.0,"wireframe":false}]} -Rules: 3-8 objects. emissiveIntensity 0.3-0.8 for glow. Size 0.3-3.5. Position -3.5 to 3.5 per axis. Combine shapes creatively. Harmonious hex color palettes. Return ONLY the JSON object, nothing else.`; +const SCENE_GEN_SYSTEM = `You are a Three.js 3D scene generator. Return ONLY a JSON object, no prose. Structure: +{"objects":[{"type":"sphere|box|torus|cylinder|cone|icosahedron|ring|plane","size":1.0,"color":"#6366f1","emissive":"#4f46e5","emissiveIntensity":0.5,"metalness":0.3,"roughness":0.3,"position":[0,0,0],"rotation":[0,0,0],"opacity":1.0,"wireframe":false}]} +Rules: 3 to 8 objects. "type" must be one of the listed values. emissiveIntensity 0.3-0.8 for a glowing look. size 0.3-3.5. position values between -3.5 and 3.5 on each axis. rotation in degrees. Colors as #RRGGBB hex. Use a harmonious palette and combine shapes creatively to match the description.`; -// Ordered by JSON-instruction-following quality -const OLLAMA_PREFERRED = ['qwen2.5', 'qwen2', 'llama3.2', 'llama3.1', 'llama3', 'mistral', 'phi4', 'phi3.5', 'phi3', 'gemma3', 'gemma2', 'mixtral']; +// Ordered by JSON-instruction-following quality. Embedding models are excluded at runtime. +const OLLAMA_PREFERRED = ['qwen2.5', 'qwen2', 'llama3.3', 'llama3.2', 'llama3.1', 'llama3', 'mistral', 'phi4', 'phi3.5', 'phi3', 'gemma3', 'gemma2', 'gemma', 'mixtral', 'qwen']; const OLLAMA_BASE = 'http://localhost:11434'; +interface OllamaTag { + name: string; + details?: { families?: string[] }; +} + +function isChatModel(m: OllamaTag): boolean { + if (/embed|minilm|bert/i.test(m.name)) return false; + const families = m.details?.families ?? []; + if (families.some((f) => /bert/i.test(f))) return false; + return true; +} + async function resolveOllamaModel(): Promise { - const res = await fetch(`${OLLAMA_BASE}/api/tags`); + let res: Response; + try { + res = await fetch(`${OLLAMA_BASE}/api/tags`); + } catch { + throw new Error('ollama_down'); + } if (!res.ok) throw new Error('ollama_down'); const data = await res.json(); - const installed: string[] = (data.models ?? []).map((m: { name: string }) => m.name); - if (installed.length === 0) throw new Error('ollama_no_models'); + const all: OllamaTag[] = data.models ?? []; + const chat = all.filter(isChatModel); + if (chat.length === 0) throw new Error('ollama_no_models'); + const names = chat.map((m) => m.name); for (const pref of OLLAMA_PREFERRED) { - const match = installed.find((n) => n.startsWith(pref)); + const match = names.find((n) => n.startsWith(pref)); if (match) return match; } - return installed[0]; + return names[0]; } async function callLLMForScene(prompt: string): Promise { const model = await resolveOllamaModel(); - const res = await fetch(`${OLLAMA_BASE}/api/chat`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - model, - messages: [ - { role: 'system', content: SCENE_GEN_SYSTEM }, - { role: 'user', content: prompt }, - ], - stream: false, - }), - }); + let res: Response; + try { + res = await fetch(`${OLLAMA_BASE}/api/chat`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + model, + format: 'json', // constrained decoding — guarantees parseable JSON + options: { temperature: 0.5 }, + messages: [ + { role: 'system', content: SCENE_GEN_SYSTEM }, + { role: 'user', content: prompt }, + ], + stream: false, + }), + }); + } catch { + throw new Error('ollama_down'); + } if (!res.ok) throw new Error(`HTTP ${res.status}`); const data = await res.json(); const text: string = (data.message?.content ?? '').trim(); const stripped = text.replace(/```json\n?|\n?```|```\n?/g, ''); const jsonMatch = stripped.match(/\{[\s\S]*\}/); if (!jsonMatch) throw new Error('no_json'); - return JSON.parse(jsonMatch[0]) as SceneDescriptor; + let parsed: unknown; + try { + parsed = JSON.parse(jsonMatch[0]); + } catch { + // Defensive repair: smart quotes, single-quoted keys, trailing commas + const repaired = jsonMatch[0] + .replace(/[‘’]/g, "'") + .replace(/[“”]/g, '"') + .replace(/'/g, '"') + .replace(/,\s*([}\]])/g, '$1'); + parsed = JSON.parse(repaired); + } + return normalizeDescriptor(parsed); +} + +// ── Validation: coerce/clamp model output so one bad value never breaks a scene + +const VALID_TYPES = new Set(['sphere', 'box', 'torus', 'cylinder', 'cone', 'icosahedron', 'ring', 'plane']); +const TYPE_ALIASES: Record = { + cube: 'box', ball: 'sphere', orb: 'sphere', donut: 'torus', tube: 'cylinder', + pyramid: 'cone', circle: 'ring', flat: 'plane', disc: 'ring', +}; + +function num(v: unknown, fallback: number, min: number, max: number): number { + const n = typeof v === 'string' ? parseFloat(v) : typeof v === 'number' ? v : NaN; + if (!Number.isFinite(n)) return fallback; + return Math.min(max, Math.max(min, n)); +} + +function vec3(v: unknown, fallback: [number, number, number], min: number, max: number): [number, number, number] { + if (!Array.isArray(v)) return fallback; + return [num(v[0], fallback[0], min, max), num(v[1], fallback[1], min, max), num(v[2], fallback[2], min, max)]; +} + +function normalizeColor(v: unknown, fallback: string): string { + if (typeof v !== 'string') return fallback; + const s = v.trim(); + if (/^#[0-9a-fA-F]{6}$/.test(s)) return s; + if (/^#[0-9a-fA-F]{3}$/.test(s)) return s; + if (/^[a-zA-Z]+$/.test(s)) return s; // CSS color name — THREE.Color accepts these + return fallback; +} + +function normalizeDescriptor(raw: unknown): SceneDescriptor { + const rawObjs = (raw as { objects?: unknown })?.objects; + if (!Array.isArray(rawObjs)) return { objects: [] }; + const objects: SceneObject[] = []; + for (const item of rawObjs.slice(0, 12)) { + if (!item || typeof item !== 'object') continue; + const o = item as Record; + let type = String(o.type ?? '').toLowerCase().trim() as SceneObject['type']; + if (!VALID_TYPES.has(type)) type = TYPE_ALIASES[type] ?? 'sphere'; + const rawSize = Array.isArray(o.size) + ? (vec3(o.size, [1, 1, 1], 0.1, 4) as SceneObject['size']) + : num(o.size, 1, 0.1, 4); + const opacity = num(o.opacity, 1, 0.05, 1); + objects.push({ + type, + size: rawSize, + color: normalizeColor(o.color, '#7dd3fc'), + emissive: normalizeColor(o.emissive, '#000000'), + emissiveIntensity: num(o.emissiveIntensity, 0.4, 0, 1), + metalness: num(o.metalness, 0.2, 0, 1), + roughness: num(o.roughness, 0.5, 0, 1), + opacity, + transparent: o.transparent === true || opacity < 1, + wireframe: o.wireframe === true, + position: vec3(o.position, [0, 0, 0], -6, 6), + rotation: vec3(o.rotation, [0, 0, 0], -360, 360), + }); + } + return { objects }; } function buildSceneFromDescriptor(descriptor: SceneDescriptor, group: THREE.Group): void { @@ -861,34 +958,38 @@ export default function JarvisPage() { const nextState = applyJarvisCommand(sceneState, command); const localReply = describeJarvisCommand(command, nextState); - setSceneState(nextState); setInput(''); const ts = new Date().toISOString(); const userEntry: JarvisTranscriptEntry = { id: makeTranscriptId(), role: 'user', text: trimmed, timestamp: ts }; const apiKey = apiKeyRef.current; - // 3D scene generation via local Ollama — no API key needed + // 3D scene generation via local Ollama — no API key needed. + // Defer the mode switch until the scene is ready so the user keeps + // seeing their current view (not an empty void) during generation. if (command.intent === 'create_scene') { const assistantId = makeTranscriptId(); setTranscript((prev) => [ ...prev, userEntry, - { id: assistantId, role: 'assistant', text: 'Connecting to Ollama...', timestamp: ts }, + { id: assistantId, role: 'assistant', text: 'Generating 3D scene with Ollama...', timestamp: ts }, ]); setIsThinking(true); try { const descriptor = await callLLMForScene(trimmed); - setSceneDescriptor(descriptor); const n = descriptor.objects?.length ?? 0; + if (n === 0) throw new Error('empty_scene'); + setSceneDescriptor(descriptor); + setSceneState(nextState); // switch to scene mode now that meshes exist setTranscript((prev) => prev.map((e) => (e.id === assistantId ? { ...e, text: `Scene rendered — ${n} objects composed.` } : e)) ); } catch (err) { const code = err instanceof Error ? err.message : ''; const msg = - code === 'ollama_down' ? 'Ollama not running. Start it with: ollama serve' : - code === 'ollama_no_models' ? 'No models installed. Run: ollama pull llama3.2' : + code === 'ollama_down' ? 'Ollama not running. Start it with: ollama serve' : + code === 'ollama_no_models' ? 'No chat models installed. Run: ollama pull llama3.2' : + code === 'empty_scene' ? 'Could not build a scene from that. Try describing shapes, colors, and a mood.' : 'Scene generation failed. Make sure Ollama is running.'; setTranscript((prev) => prev.map((e) => (e.id === assistantId ? { ...e, text: msg } : e)) @@ -898,6 +999,9 @@ export default function JarvisPage() { return; } + // All other commands apply their scene state immediately. + setSceneState(nextState); + if (!apiKey) { setTranscript((prev) => [ ...prev, From 40019cb95403ae97e0e3a8c4ecb850aca57e7114 Mon Sep 17 00:00:00 2001 From: Jose Guerra Date: Fri, 12 Jun 2026 07:20:45 -0700 Subject: [PATCH 5/5] feat(jarvis): load all common 3D model formats, not just GLB/GLTF MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The importer was hardcoded to GLTFLoader with an accept=".glb,.gltf" file picker, so STL (and OBJ/FBX/PLY/DAE/3MF) files silently failed. Adds a format-dispatching loader: - glb/gltf -> GLTFLoader - stl -> STLLoader (geometry-only; wrapped in a metallic standard material) - obj -> OBJLoader - fbx -> FBXLoader - ply -> PLYLoader (geometry-only; enables vertexColors when present) - dae -> ColladaLoader - 3mf -> ThreeMFLoader Each loader is dynamically imported, so it builds into its own lazy chunk and only downloads when that format is actually picked — the main bundle stays lean. Other changes: - File picker now accepts all supported extensions. - Unsupported extensions get a clear "Supported: ..." message instead of a silent failure. - Loads now report progress in the transcript: "Loading file.stl..." -> "STL model loaded." or an actionable error, via an onModelResult callback. - Loading a model auto-switches to object mode (where the model renders), centers and scales it to fit, and disposes the previous model's geometry/materials to avoid leaks. Verified: tsc clean; vite build emits 7 separate loader chunks; and STL/OBJ/PLY parsers confirmed to parse real geometry in a node smoke test. Co-Authored-By: Claude Opus 4.8 --- .../desktop/src/renderer/pages/Jarvis.tsx | 208 +++++++++++++++--- 1 file changed, 176 insertions(+), 32 deletions(-) diff --git a/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx index 5a9ace52e..b8c4d95ea 100644 --- a/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx +++ b/apps/openwork-memos-integration/apps/desktop/src/renderer/pages/Jarvis.tsx @@ -3,7 +3,6 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { motion } from 'framer-motion'; import * as THREE from 'three'; -import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { Badge } from '@/components/ui/badge'; @@ -341,17 +340,87 @@ function buildSceneFromDescriptor(descriptor: SceneDescriptor, group: THREE.Grou }); } +// ─── Multi-format 3D model loading ─────────────────────────────────────────── + +export const SUPPORTED_MODEL_FORMATS = ['glb', 'gltf', 'stl', 'obj', 'fbx', 'ply', 'dae', '3mf'] as const; + +function defaultModelMaterial(): THREE.MeshStandardMaterial { + // Geometry-only formats (STL/PLY) arrive without materials — give them a + // clean metallic look that fits the HUD. + return new THREE.MeshStandardMaterial({ + color: 0x9fb4d8, + metalness: 0.4, + roughness: 0.45, + emissive: 0x0a1c33, + emissiveIntensity: 0.25, + }); +} + +// Loaders are dynamically imported so only the one for the chosen format is +// pulled into the bundle at runtime. +async function loadModelObject(url: string, ext: string): Promise { + switch (ext) { + case 'glb': + case 'gltf': { + const { GLTFLoader } = await import('three/examples/jsm/loaders/GLTFLoader.js'); + const gltf = await new GLTFLoader().loadAsync(url); + return gltf.scene; + } + case 'stl': { + const { STLLoader } = await import('three/examples/jsm/loaders/STLLoader.js'); + const geo = await new STLLoader().loadAsync(url); + geo.computeVertexNormals(); + return new THREE.Mesh(geo, defaultModelMaterial()); + } + case 'obj': { + const { OBJLoader } = await import('three/examples/jsm/loaders/OBJLoader.js'); + return await new OBJLoader().loadAsync(url); + } + case 'fbx': { + const { FBXLoader } = await import('three/examples/jsm/loaders/FBXLoader.js'); + return await new FBXLoader().loadAsync(url); + } + case 'ply': { + const { PLYLoader } = await import('three/examples/jsm/loaders/PLYLoader.js'); + const geo = await new PLYLoader().loadAsync(url); + geo.computeVertexNormals(); + const mat = defaultModelMaterial(); + if (geo.hasAttribute('color')) mat.vertexColors = true; + return new THREE.Mesh(geo, mat); + } + case 'dae': { + const { ColladaLoader } = await import('three/examples/jsm/loaders/ColladaLoader.js'); + const res = await new ColladaLoader().loadAsync(url); + if (!res?.scene) throw new Error('unsupported_format'); + return res.scene; + } + case '3mf': { + const { ThreeMFLoader } = await import('three/examples/jsm/loaders/3MFLoader.js'); + return await new ThreeMFLoader().loadAsync(url); + } + default: + throw new Error('unsupported_format'); + } +} + +interface ModelSource { + url: string; + ext: string; +} + // ─── Viewport ──────────────────────────────────────────────────────────────── function JarvisViewport({ state, isThinking, - modelUrl, + modelSource, + onModelResult, sceneDescriptor, }: { state: JarvisSceneState; isThinking: boolean; - modelUrl?: string; + modelSource?: ModelSource; + onModelResult?: (ok: boolean, msg?: string) => void; sceneDescriptor?: SceneDescriptor; }) { const mountRef = useRef(null); @@ -375,26 +444,56 @@ function JarvisViewport({ }, [isThinking]); useEffect(() => { - if (!modelUrl || !objectGroupRef.current) return; const group = objectGroupRef.current; - const loader = new GLTFLoader(); - loader.load( - modelUrl, - (gltf) => { - while (group.children.length > 0) group.remove(group.children[0]); - const model = gltf.scene; - const box = new THREE.Box3().setFromObject(model); - const size = box.getSize(new THREE.Vector3()).length(); - const center = box.getCenter(new THREE.Vector3()); - const scale = 2.4 / Math.max(size, 0.001); - model.scale.setScalar(scale); - model.position.set(-center.x * scale, -center.y * scale, -center.z * scale); - group.add(model); - }, - undefined, - (err) => console.error('GLTFLoader:', err) - ); - }, [modelUrl]); + if (!modelSource || !group) return; + let cancelled = false; + + const disposeChildren = () => { + [...group.children].forEach((child) => { + group.remove(child); + child.traverse((node) => { + const mesh = node as THREE.Mesh; + if (mesh.isMesh) { + mesh.geometry?.dispose(); + const mat = mesh.material; + if (Array.isArray(mat)) mat.forEach((m) => m.dispose()); + else (mat as THREE.Material)?.dispose(); + } + }); + }); + }; + + const fit = (object: THREE.Object3D) => { + const box = new THREE.Box3().setFromObject(object); + const size = box.getSize(new THREE.Vector3()).length(); + const center = box.getCenter(new THREE.Vector3()); + const scale = 2.4 / Math.max(size, 0.001); + object.scale.setScalar(scale); + object.position.set(-center.x * scale, -center.y * scale, -center.z * scale); + }; + + loadModelObject(modelSource.url, modelSource.ext) + .then((object) => { + if (cancelled) return; + disposeChildren(); + fit(object); + group.add(object); + onModelResult?.(true, `${modelSource.ext.toUpperCase()} model loaded.`); + }) + .catch((err) => { + if (cancelled) return; + console.error('Model load error:', err); + const msg = + err instanceof Error && err.message === 'unsupported_format' + ? 'Unsupported 3D format.' + : 'Could not load that model. The file may be corrupt or reference external assets.'; + onModelResult?.(false, msg); + }); + + return () => { + cancelled = true; + }; + }, [modelSource, onModelResult]); useEffect(() => { const group = sceneGroupRef.current; @@ -929,12 +1028,13 @@ export default function JarvisPage() { ]); const [isThinking, setIsThinking] = useState(false); const [isListening, setIsListening] = useState(false); - const [modelUrl, setModelUrl] = useState(undefined); + const [modelSource, setModelSource] = useState(undefined); const [sceneDescriptor, setSceneDescriptor] = useState(undefined); const recognitionRef = useRef(null); const transcriptEndRef = useRef(null); const apiKeyRef = useRef(null); const fileInputRef = useRef(null); + const modelMsgIdRef = useRef(null); const fact = useMemo(() => getFactForTarget(sceneState.activeTarget), [sceneState.activeTarget]); @@ -1078,17 +1178,61 @@ export default function JarvisPage() { const handleFileSelect = useCallback( (e: React.ChangeEvent) => { const file = e.target.files?.[0]; + e.target.value = ''; if (!file) return; - const prev = modelUrl; + + const ext = file.name.split('.').pop()?.toLowerCase() ?? ''; + const ts = new Date().toISOString(); + + if (!(SUPPORTED_MODEL_FORMATS as readonly string[]).includes(ext)) { + setTranscript((prev) => [ + ...prev, + { id: makeTranscriptId(), role: 'user', text: `Load ${file.name}`, timestamp: ts }, + { + id: makeTranscriptId(), + role: 'assistant', + text: `Unsupported format ".${ext}". Supported: ${SUPPORTED_MODEL_FORMATS.map((f) => f.toUpperCase()).join(', ')}.`, + timestamp: ts, + }, + ]); + return; + } + + const prevUrl = modelSource?.url; const url = URL.createObjectURL(file); - setModelUrl(url); - void executeCommand(`Load ${file.name.replace(/\.[^.]+$/, '')}`); - e.target.value = ''; - if (prev) URL.revokeObjectURL(prev); + const msgId = makeTranscriptId(); + modelMsgIdRef.current = msgId; + + setTranscript((prev) => [ + ...prev, + { id: makeTranscriptId(), role: 'user', text: `Load ${file.name}`, timestamp: ts }, + { id: msgId, role: 'assistant', text: `Loading ${file.name}...`, timestamp: ts }, + ]); + + // Models render in the object group, which is only shown in object mode. + setSceneState((s) => ({ + ...s, + mode: 'object', + activeTarget: file.name.replace(/\.[^.]+$/, ''), + summary: `Imported 3D model: ${file.name}`, + exploded: false, + })); + setModelSource({ url, ext }); + if (prevUrl) URL.revokeObjectURL(prevUrl); }, - [modelUrl, executeCommand] + [modelSource] ); + const handleModelResult = useCallback((ok: boolean, msg?: string) => { + const id = modelMsgIdRef.current; + if (!id) return; + setTranscript((prev) => + prev.map((e) => + e.id === id ? { ...e, text: msg ?? (ok ? 'Model loaded.' : 'Failed to load model.') } : e + ) + ); + }, []); + const stateChips = useMemo( () => [ @@ -1152,7 +1296,7 @@ export default function JarvisPage() { > {/* ── Full-bleed 3D viewport ── */}
- +
{/* ── Top HUD bar ── */} @@ -1327,11 +1471,11 @@ export default function JarvisPage() {
- {/* Hidden 3D file picker */} + {/* Hidden 3D file picker — accepts all supported model formats */}