From 2b9b2b774ae009b5b515a8ccefa38e5fd9299f49 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 15 Jul 2026 09:12:57 +0200 Subject: [PATCH] JS app runtime: HTTP requests, assets, streams, and transpiler fixes Split out of #259. Platform work for JavaScript repo apps: - JS app runtime: HTTP requests (bounded fetch with headers and binary responses), asset management, streams, and gated settings access for repo code apps - JS transpiler: template interpolations and comparisons no longer eat object literals; native/JS transpiler parity test Co-Authored-By: Claude Fable 5 --- frameos/src/frameos/js_runtime/README.md | 24 +- .../src/frameos/js_runtime/app_runtime.nim | 400 +++++++++++++++++- frameos/src/frameos/js_runtime/parser.nim | 24 +- .../js_runtime/tests/test_js_app_runtime.nim | 293 ++++++++++++- .../js_runtime/tests/test_js_transpiler.nim | 33 ++ .../frameos/js_runtime/token_processor.nim | 2 +- frameos/src/frameos/js_runtime/transpiler.nim | 11 +- .../tests/test_native_js_transpiler_parity.py | 16 + 8 files changed, 786 insertions(+), 17 deletions(-) diff --git a/frameos/src/frameos/js_runtime/README.md b/frameos/src/frameos/js_runtime/README.md index d5e1356dc..f72e4178f 100644 --- a/frameos/src/frameos/js_runtime/README.md +++ b/frameos/src/frameos/js_runtime/README.md @@ -110,7 +110,29 @@ uses QuickJS only to execute the resulting JavaScript. - Builds a CommonJS-style module wrapper around app source. - Runs the native module transform before loading the wrapper into QuickJS. - Exposes a `frameos` API object to JS apps, including logging, state updates, - image operations, sleep scheduling, HTTP helpers, and context access. + image operations, sleep scheduling, HTTP helpers, and context access: + - `fetchText(url)` / `fetchJson(url)`: bounded GET requests. + - `httpRequest(url, {method, headers, body, bodyBase64, base64, timeoutMs})`: + bounded requests with any method/headers; returns `{status, body}` or + `{status, bodyBase64}` when `base64: true` (required for binary responses, + since raw bytes cannot cross the JS boundary as strings), or + `{status: 0, error}` on transport failure. + - `listAssets(dir)`, `assetExists(path)`, `assetSize(path)`, + `readAsset(path, {offset, length}?)`, `writeAsset(path, base64)`, + `appendAsset(path, base64)`, `deleteAsset(path)`: asset management scoped + to the frame's assets folder. Full-buffer reads are capped on embedded + targets (2MB) — read bigger files in ranged chunks or via streams. + - `loadAssetImage(path)`: decode an asset image within display bounds + (memory-aware, streams from disk) and return an image reference. + - `openAssetStream(path, "r"|"w"|"a")`, `createStream(base64?)`, + `streamRead(ref, length?)`, `streamWrite(ref, base64)`, `streamAtEnd(ref)`, + `streamRewind(ref)`, `streamClose(ref)`: simple string/asset-file streams + for chunked processing. Stream refs are plain JSON (`{__frameosType: + "streamRef", id}`) held in a global capped registry, so a data app can + return one and a downstream app can consume it. + - `getSetting(...path)`: read `frameConfig.settings` values (API keys etc.); + only namespaces listed in the app config's `"settings"` array are + accessible, so access travels visibly with the scene JSON. - Calls exported app lifecycle functions such as `init` and `get`. - Tracks persistent and transient image references so overwritten dynamic image fields can be released. diff --git a/frameos/src/frameos/js_runtime/app_runtime.nim b/frameos/src/frameos/js_runtime/app_runtime.nim index 078d0064d..8598bd1bd 100644 --- a/frameos/src/frameos/js_runtime/app_runtime.nim +++ b/frameos/src/frameos/js_runtime/app_runtime.nim @@ -1,4 +1,4 @@ -import std/[base64, json, options, strformat, strutils, tables] +import std/[algorithm, base64, json, options, os, streams, strformat, strutils, tables] import pixie import frameos/apps as frameos_apps @@ -7,6 +7,7 @@ import frameos/types import frameos/values import frameos/utils/http_client import frameos/utils/image +import frameos/utils/system import frameos/js_runtime/burrito type @@ -14,6 +15,7 @@ type category*: string outputType*: string source*: string + settingsKeys*: seq[string] js*: QuickJS ready*: bool initialized*: bool @@ -113,6 +115,356 @@ proc jsFetchText(ctx: ptr JSContext, url: JSValue): JSValue {.nimcall.} = proc storeTransientImageJson(runtime: JsAppRuntime, image: Image): JsonNode +const JsHttpMaxTimeoutMs = 600_000 + +proc jsHttpRequest(ctx: ptr JSContext, url: JSValue, optionsJson: JSValue): JSValue {.nimcall.} = + ## Bounded HTTP request with method/headers/body. Options JSON: + ## {method, headers: {name: value}, body, bodyBase64, base64 (return body + ## base64-encoded, required for binary responses), timeoutMs}. + ## Returns JSON: {status, body | bodyBase64} or {status: 0, error}. + let e = env(ctx) + var response = %*{"status": 0} + + let urlStr = toNimString(ctx, url) + var options = %*{} + try: + options = parseJson(toNimString(ctx, optionsJson)) + except CatchableError: + discard + if options.isNil or options.kind != JObject: + options = %*{} + + try: + var body = options{"body"}.getStr("") + if options{"bodyBase64"}.getStr("").len > 0: + body = decode(options["bodyBase64"].getStr()) + var headers: seq[SimpleHttpHeader] = @[] + let headersNode = options{"headers"} + if not headersNode.isNil and headersNode.kind == JObject: + for name, value in headersNode.pairs: + headers.add((name, value.getStr($value))) + let timeoutMs = clamp(options{"timeoutMs"}.getInt(DefaultFetchTimeoutMs), 1000, JsHttpMaxTimeoutMs) + let maxSeconds = max(DefaultFetchMaxSeconds, timeoutMs.float / 1000.0 + 30.0) + let res = boundedRequestWithHeaders( + urlStr, + httpMethod = options{"method"}.getStr("GET").toUpperAscii(), + body = body, + headers = headers, + timeoutMs = timeoutMs, + maxBytes = jsFetchMaxBytes(e), + maxSeconds = maxSeconds + ) + response["status"] = %*res.code + if options{"base64"}.getBool(false): + response["bodyBase64"] = %*encode(res.body) + else: + response["body"] = %*res.body + except CatchableError as err: + response["error"] = %*err.msg + if e != nil: + frameos_apps.logError(e.owner, "JS app httpRequest failed: " & err.msg) + return nimStringToJS(ctx, $response) + +proc assetsRoot(e: JsAppEvalEnv): string = + result = if e.owner.frameConfig.assetsPath == "": "/srv/assets" else: e.owner.frameConfig.assetsPath + result.removeSuffix('/') + +proc resolveAssetPath(e: JsAppEvalEnv, relPath: string): string = + ## Absolute path inside the assets folder, or "" when the path is empty, + ## absolute, or escapes the folder. + if relPath.len == 0 or relPath.startsWith("/") or relPath.contains(".."): + return "" + let root = assetsRoot(e) + let full = normalizedPath(root / relPath) + if full == root or full.startsWith(root & "/"): + return full + return "" + +const EmbeddedAssetReadMaxBytes = 2 * 1024 * 1024 + +proc jsAssetReadMaxBytes(e: JsAppEvalEnv): int = + ## Full-buffer asset reads triple in RAM (bytes + base64 + JS string), so + ## embedded targets get a small cap; larger files must use ranged reads. + when defined(frameosEmbedded): + min(jsFetchMaxBytes(e), EmbeddedAssetReadMaxBytes) + else: + jsFetchMaxBytes(e) + +proc writeAssetFile(e: JsAppEvalEnv, opName: string, pathStr: string, + contents: string, append: bool): bool = + let full = resolveAssetPath(e, pathStr) + if full.len == 0: + frameos_apps.logError(e.owner, "JS app " & opName & ": invalid asset path: " & pathStr) + return false + createDir(full.parentDir()) + let freeDiskSpace = getAvailableDiskSpace(full.parentDir()) + if freeDiskSpace != -1 and freeDiskSpace < 100 * 1024 * 1024: + frameos_apps.logError(e.owner, "JS app " & opName & ": low disk space, asset not saved: " & pathStr) + return false + if append: + var file: File + if not file.open(full, fmAppend): + frameos_apps.logError(e.owner, "JS app " & opName & ": cannot open asset: " & pathStr) + return false + defer: file.close() + file.write(contents) + else: + writeFile(full, contents) + return true + +proc jsAssets(ctx: ptr JSContext, op: JSValue, path: JSValue, data: JSValue): JSValue {.nimcall.} = + ## Asset management scoped to the frame's assets folder. Ops: + ## list (dir -> JSON array of relative paths), exists, size, + ## read (options JSON {offset, length} -> base64), write/append (path, base64), + ## delete, image (-> imageRef decoded within display bounds). + let e = env(ctx) + if e == nil: + return jsUndefSentinel(ctx) + + let opStr = toNimString(ctx, op) + let pathStr = toNimString(ctx, path) + + try: + case opStr + of "list": + let root = assetsRoot(e) + let dir = if pathStr.len == 0: root else: resolveAssetPath(e, pathStr) + var files: seq[string] = @[] + if dir.len > 0 and dirExists(dir): + for filePath in walkDirRec(dir, relative = false): + if "/.thumbs/" in filePath or "/.frameos/" in filePath: + continue + files.add(filePath[(root.len + 1)..^1]) + files.sort() + let arr = newJArray() + for file in files: + arr.add(%*file) + return jsonToJS(ctx, arr) + of "exists": + let full = resolveAssetPath(e, pathStr) + return nimBoolToJS(ctx, full.len > 0 and fileExists(full)) + of "size": + let full = resolveAssetPath(e, pathStr) + if full.len == 0 or not fileExists(full): + return nimIntToJS(ctx, -1) + return nimFloatToJS(ctx, getFileSize(full).float) + of "read": + let full = resolveAssetPath(e, pathStr) + if full.len == 0 or not fileExists(full): + return jsNull(ctx) + var options = %*{} + let optionsStr = toNimString(ctx, data) + if optionsStr.len > 0: + try: + options = parseJson(optionsStr) + except CatchableError: + discard + if options.isNil or options.kind != JObject: + options = %*{} + let fileSize = getFileSize(full) + let offset = max(0, options{"offset"}.getInt(0)) + let readMax = jsAssetReadMaxBytes(e) + var length = options{"length"}.getInt(int(fileSize) - offset) + length = min(length, int(fileSize) - offset) + if length <= 0: + return nimStringToJS(ctx, "") + if length > readMax: + frameos_apps.logError(e.owner, + "JS app readAsset: " & pathStr & " slice is " & $length & + " bytes, over the " & $readMax & " byte limit; read it in chunks" & + " with {offset, length}") + return jsNull(ctx) + var file: File + if not file.open(full): + return jsNull(ctx) + defer: file.close() + file.setFilePos(offset) + var contents = newString(length) + let bytesRead = file.readBuffer(addr contents[0], length) + contents.setLen(bytesRead) + return nimStringToJS(ctx, encode(contents)) + of "write", "append": + let contents = decode(toNimString(ctx, data)) + return nimBoolToJS(ctx, writeAssetFile(e, opStr & "Asset", pathStr, contents, opStr == "append")) + of "delete": + let full = resolveAssetPath(e, pathStr) + if full.len == 0 or not fileExists(full): + return nimBoolToJS(ctx, false) + removeFile(full) + return nimBoolToJS(ctx, true) + of "image": + let full = resolveAssetPath(e, pathStr) + if full.len == 0 or not fileExists(full): + return jsNull(ctx) + let image = readImageWithDisplayBounds(full) + if image.isNil: + return jsNull(ctx) + return jsonToJS(ctx, e.runtime.storeTransientImageJson(image)) + else: + frameos_apps.logError(e.owner, "JS app assets: unknown operation: " & opStr) + return jsUndefSentinel(ctx) + except CatchableError as err: + frameos_apps.logError(e.owner, "JS app assets " & opStr & " failed: " & err.msg) + if opStr in ["write", "append", "delete", "exists"]: + return nimBoolToJS(ctx, false) + return jsNull(ctx) + +## Streams let apps pass around and process data bigger than memory allows in +## one piece: a data app can return a streamRef (it travels between nodes as +## plain JSON), and the consumer reads it chunk by chunk. Backed by simple +## string streams or files inside the assets folder. The registry is global so +## refs resolve across app runtimes; it is capped, evicting (and closing) the +## oldest stream, so forgotten handles cannot pile up. +const MaxOpenJsStreams = 64 +const DefaultJsStreamChunkBytes = 65536 + +var jsStreamsGlobal = initOrderedTable[int, Stream]() +var jsNextStreamId = 0 + +proc registerJsStream(stream: Stream): JsonNode = + while jsStreamsGlobal.len >= MaxOpenJsStreams: + for id in jsStreamsGlobal.keys: + try: + jsStreamsGlobal[id].close() + except CatchableError: + discard + jsStreamsGlobal.del(id) + break + inc jsNextStreamId + jsStreamsGlobal[jsNextStreamId] = stream + return %*{"__frameosType": "streamRef", "id": jsNextStreamId} + +proc jsStreamById(options: JsonNode): Stream = + let id = options{"id"}.getInt(0) + if id != 0 and jsStreamsGlobal.hasKey(id): + return jsStreamsGlobal[id] + return nil + +proc jsStreams(ctx: ptr JSContext, op: JSValue, optionsJson: JSValue): JSValue {.nimcall.} = + ## Ops: openAsset {path, mode: r|w|a} -> streamRef, create {base64} -> streamRef, + ## read {id, length} -> base64 ("" at end), write {id, base64}, atEnd {id}, + ## close {id}. Read sizes are capped like readAsset; loop for more. + let e = env(ctx) + if e == nil: + return jsUndefSentinel(ctx) + + let opStr = toNimString(ctx, op) + var options = %*{} + try: + options = parseJson(toNimString(ctx, optionsJson)) + except CatchableError: + discard + if options.isNil or options.kind != JObject: + options = %*{} + + try: + case opStr + of "openAsset": + let pathStr = options{"path"}.getStr("") + let mode = options{"mode"}.getStr("r") + let full = resolveAssetPath(e, pathStr) + if full.len == 0: + frameos_apps.logError(e.owner, "JS app openAssetStream: invalid asset path: " & pathStr) + return jsNull(ctx) + if mode == "r": + if not fileExists(full): + return jsNull(ctx) + let stream = newFileStream(full, fmRead) + if stream.isNil: + return jsNull(ctx) + return jsonToJS(ctx, registerJsStream(stream)) + elif mode in ["w", "a"]: + createDir(full.parentDir()) + let freeDiskSpace = getAvailableDiskSpace(full.parentDir()) + if freeDiskSpace != -1 and freeDiskSpace < 100 * 1024 * 1024: + frameos_apps.logError(e.owner, "JS app openAssetStream: low disk space: " & pathStr) + return jsNull(ctx) + let stream = newFileStream(full, if mode == "a": fmAppend else: fmWrite) + if stream.isNil: + return jsNull(ctx) + return jsonToJS(ctx, registerJsStream(stream)) + else: + frameos_apps.logError(e.owner, "JS app openAssetStream: unknown mode: " & mode) + return jsNull(ctx) + of "create": + var contents = "" + if options{"base64"}.getStr("").len > 0: + contents = decode(options["base64"].getStr()) + return jsonToJS(ctx, registerJsStream(newStringStream(contents))) + of "read": + let stream = jsStreamById(options) + if stream.isNil: + return jsNull(ctx) + let length = clamp(options{"length"}.getInt(DefaultJsStreamChunkBytes), 1, jsAssetReadMaxBytes(e)) + return nimStringToJS(ctx, encode(stream.readStr(length))) + of "write": + let stream = jsStreamById(options) + if stream.isNil: + return nimBoolToJS(ctx, false) + stream.write(decode(options{"base64"}.getStr(""))) + return nimBoolToJS(ctx, true) + of "atEnd": + let stream = jsStreamById(options) + if stream.isNil: + return nimBoolToJS(ctx, true) + return nimBoolToJS(ctx, stream.atEnd()) + of "rewind": + let stream = jsStreamById(options) + if stream.isNil: + return nimBoolToJS(ctx, false) + stream.setPosition(0) + return nimBoolToJS(ctx, true) + of "close": + let id = options{"id"}.getInt(0) + if id == 0 or not jsStreamsGlobal.hasKey(id): + return nimBoolToJS(ctx, false) + try: + jsStreamsGlobal[id].close() + finally: + jsStreamsGlobal.del(id) + return nimBoolToJS(ctx, true) + else: + frameos_apps.logError(e.owner, "JS app streams: unknown operation: " & opStr) + return jsUndefSentinel(ctx) + except CatchableError as err: + frameos_apps.logError(e.owner, "JS app streams " & opStr & " failed: " & err.msg) + if opStr in ["write", "close", "rewind", "atEnd"]: + return nimBoolToJS(ctx, false) + return jsNull(ctx) + +proc jsGetSetting(ctx: ptr JSContext, pathJson: JSValue): JSValue {.nimcall.} = + ## Read a value from frameConfig.settings, e.g. getSetting("openAI", "apiKey"). + ## Only namespaces the app declares in its config.json "settings" list are + ## accessible; the declaration travels with the scene so access is auditable. + let e = env(ctx) + if e == nil: + return jsUndefSentinel(ctx) + + var path: JsonNode + try: + path = parseJson(toNimString(ctx, pathJson)) + except CatchableError: + return jsUndefSentinel(ctx) + if path.isNil or path.kind != JArray or path.len == 0: + return jsUndefSentinel(ctx) + + let namespace = path[0].getStr() + if namespace.len == 0 or namespace notin e.runtime.settingsKeys: + frameos_apps.logError(e.owner, + "JS app getSetting: settings namespace \"" & namespace & + "\" is not declared in the app config's \"settings\" list") + return jsUndefSentinel(ctx) + + frameos_apps.ensureEmbeddedServiceSettings(e.owner) + var node = e.owner.frameConfig.settings + for part in path.items: + if node.isNil: + return jsUndefSentinel(ctx) + node = node{part.getStr()} + if node.isNil: + return jsUndefSentinel(ctx) + return jsonToJS(ctx, node) + proc jsGetAppMeta(ctx: ptr JSContext, key: JSValue): JSValue {.nimcall.} = let e = env(ctx) if e == nil: @@ -237,11 +589,13 @@ proc jsGetAppKeys(ctx: ptr JSContext, scope: JSValue): JSValue {.nimcall.} = arr.add(%*key) return jsonToJS(ctx, arr) -proc newJsAppRuntime*(category: string, outputType: string, source: string): JsAppRuntime = +proc newJsAppRuntime*(category: string, outputType: string, source: string, + settingsKeys: seq[string] = @[]): JsAppRuntime = return JsAppRuntime( category: category, outputType: outputType, source: source, + settingsKeys: settingsKeys, nextImageId: 0, images: initTable[int, Image](), transientImageIds: @[] @@ -382,7 +736,13 @@ proc initDynamicJsApp*(keyword: string, node: DiagramNode, scene: FrameScene, so let config = configFromSources(sources) let category = config{"category"}.getStr().toLowerAscii() let outputType = outputTypeFromConfig(config) - let runtime = newJsAppRuntime(category, outputType, source) + var settingsKeys: seq[string] = @[] + let settingsNode = config{"settings"} + if not settingsNode.isNil and settingsNode.kind == JArray: + for key in settingsNode.items: + if key.kind == JString and key.getStr().len > 0: + settingsKeys.add(key.getStr()) + let runtime = newJsAppRuntime(category, outputType, source, settingsKeys) return DynamicJsApp( nodeId: node.id, nodeName: node.data{"name"}.getStr(keyword), @@ -413,6 +773,10 @@ proc ensureReady(runtime: JsAppRuntime) = runtime.js.registerFunction("jsSetNextSleep", jsSetNextSleep) runtime.js.registerFunction("jsSetState", jsSetState) runtime.js.registerFunction("jsFetchText", jsFetchText) + runtime.js.registerFunction("jsHttpRequest", jsHttpRequest) + runtime.js.registerFunction("jsAssets", jsAssets) + runtime.js.registerFunction("jsStreams", jsStreams) + runtime.js.registerFunction("jsGetSetting", jsGetSetting) runtime.js.registerFunction("jsGetAppMeta", jsGetAppMeta) runtime.js.registerFunction("jsGetAppConfig", jsGetAppConfig) runtime.js.registerFunction("jsGetAppState", jsGetAppState) @@ -450,6 +814,36 @@ proc ensureReady(runtime: JsAppRuntime) = setNextSleep: (seconds) => jsSetNextSleep(Number(seconds || 0)), fetchText: (url) => jsFetchText(String(url || "")), fetchJson: (url) => JSON.parse(jsFetchText(String(url || "")) || "null"), + httpRequest: (url, options) => JSON.parse( + jsHttpRequest(String(url || ""), JSON.stringify(options || {}, __jsReplacer)) || "{}" + ), + listAssets: (dir) => __frameosUnwrap(jsAssets("list", String(dir || ""), "")) || [], + assetExists: (path) => __frameosUnwrap(jsAssets("exists", String(path || ""), "")) === true, + assetSize: (path) => __frameosUnwrap(jsAssets("size", String(path || ""), "")) ?? -1, + readAsset: (path, options) => __frameosUnwrap(jsAssets( + "read", String(path || ""), options ? JSON.stringify(options) : "" + )) ?? null, + writeAsset: (path, base64) => __frameosUnwrap(jsAssets("write", String(path || ""), String(base64 || ""))) === true, + appendAsset: (path, base64) => __frameosUnwrap(jsAssets("append", String(path || ""), String(base64 || ""))) === true, + deleteAsset: (path) => __frameosUnwrap(jsAssets("delete", String(path || ""), "")) === true, + loadAssetImage: (path) => __frameosUnwrap(jsAssets("image", String(path || ""), "")) ?? null, + openAssetStream: (path, mode) => __frameosUnwrap(jsStreams("openAsset", + JSON.stringify({ path: String(path || ""), mode: String(mode || "r") }))) ?? null, + createStream: (base64) => __frameosUnwrap(jsStreams("create", + JSON.stringify({ base64: String(base64 || "") }))) ?? null, + streamRead: (ref, length) => __frameosUnwrap(jsStreams("read", + JSON.stringify({ id: (ref && ref.id) || ref, length }))) ?? null, + streamWrite: (ref, base64) => __frameosUnwrap(jsStreams("write", + JSON.stringify({ id: (ref && ref.id) || ref, base64: String(base64 || "") }))) === true, + streamAtEnd: (ref) => __frameosUnwrap(jsStreams("atEnd", + JSON.stringify({ id: (ref && ref.id) || ref }))) === true, + streamRewind: (ref) => __frameosUnwrap(jsStreams("rewind", + JSON.stringify({ id: (ref && ref.id) || ref }))) === true, + streamClose: (ref) => __frameosUnwrap(jsStreams("close", + JSON.stringify({ id: (ref && ref.id) || ref }))) === true, + getSetting: (...path) => __frameosUnwrap(jsGetSetting( + JSON.stringify(path.flat().map((p) => String(p))) + )), setState: (key, value) => jsSetState( String(key || ""), JSON.stringify(value === undefined ? null : value, __jsReplacer) diff --git a/frameos/src/frameos/js_runtime/parser.nim b/frameos/src/frameos/js_runtime/parser.nim index 433530917..f427c2d5f 100644 --- a/frameos/src/frameos/js_runtime/parser.nim +++ b/frameos/src/frameos/js_runtime/parser.nim @@ -25,7 +25,8 @@ proc nextNonEof(tokens: seq[JsToken], i: int): int = proc findMatching(tokens: seq[JsToken], openIndex: int, openType, closeType: TokenType): int = var depth = 0 for i in openIndex.. 0: dec parenDepth - of ttBraceL: inc braceDepth + of ttBraceL, ttDollarBraceL: inc braceDepth of ttBraceR: if braceDepth == 0 and parenDepth == 0 and bracketDepth == 0: return i @@ -102,6 +103,13 @@ proc annotateScopes(tokens: var seq[JsToken]): seq[Scope] = stack.add((i, pendingFunctionBrace)) pendingFunctionBrace = false inc scopeDepth + of ttDollarBraceL: + # Template interpolation opens a brace context closed by a plain `}`; + # track it so that `}` does not pop an enclosing scope. + tokens[i].contextId = nextContextId + inc nextContextId + stack.add((i, false)) + inc scopeDepth of ttBraceR: if scopeDepth > 0: dec scopeDepth @@ -125,7 +133,7 @@ proc markBindingList(tokens: var seq[JsToken], start, stop: int, role: Identifie var depth = 0 while i <= stop and i < tokens.len: case tokens[i].typ - of ttBraceL, ttBracketL, ttParenL: + of ttBraceL, ttBracketL, ttParenL, ttDollarBraceL: inc depth of ttBraceR, ttBracketR, ttParenR: if depth > 0: dec depth @@ -154,7 +162,7 @@ proc annotateVarDeclarations(tokens: var seq[JsToken]) = case tokens[j].typ of ttSemi, ttEof: break - of ttBraceL, ttBracketL, ttParenL: + of ttBraceL, ttBracketL, ttParenL, ttDollarBraceL: inc depth of ttBraceR, ttBracketR, ttParenR: if depth == 0 and tokens[j].typ == ttParenR: @@ -430,9 +438,9 @@ proc annotateOptionalAndNullish(tokens: var seq[JsToken]) = for i in 0.. 0 and tokens[start].typ notin {ttComma, ttSemi, ttParenL, ttBraceL, ttBracketL, ttEq}: + while start > 0 and tokens[start].typ notin {ttComma, ttSemi, ttParenL, ttBraceL, ttDollarBraceL, ttBracketL, ttEq}: dec start - if start < i and tokens[start].typ in {ttComma, ttSemi, ttParenL, ttBraceL, ttBracketL, ttEq}: + if start < i and tokens[start].typ in {ttComma, ttSemi, ttParenL, ttBraceL, ttDollarBraceL, ttBracketL, ttEq}: inc start tokens[start].numNullishCoalesceStarts += 1 var finish = i + 1 @@ -443,9 +451,9 @@ proc annotateOptionalAndNullish(tokens: var seq[JsToken]) = if tokens[i].typ == ttQuestionDot: var start = i - 1 - while start > 0 and tokens[start].typ notin {ttComma, ttSemi, ttParenL, ttBraceL, ttBracketL, ttEq}: + while start > 0 and tokens[start].typ notin {ttComma, ttSemi, ttParenL, ttBraceL, ttDollarBraceL, ttBracketL, ttEq}: dec start - if start < i and tokens[start].typ in {ttComma, ttSemi, ttParenL, ttBraceL, ttBracketL, ttEq}: + if start < i and tokens[start].typ in {ttComma, ttSemi, ttParenL, ttBraceL, ttDollarBraceL, ttBracketL, ttEq}: inc start tokens[start].isOptionalChainStart = true var finish = i + 1 diff --git a/frameos/src/frameos/js_runtime/tests/test_js_app_runtime.nim b/frameos/src/frameos/js_runtime/tests/test_js_app_runtime.nim index 20a73d9e3..c9f60c485 100644 --- a/frameos/src/frameos/js_runtime/tests/test_js_app_runtime.nim +++ b/frameos/src/frameos/js_runtime/tests/test_js_app_runtime.nim @@ -1,8 +1,9 @@ -import std/[json, sequtils, strutils, tables, unittest] +import std/[base64, json, net, os, sequtils, strutils, tables, unittest] import pixie import frameos/js_runtime/app_runtime import frameos/types +import frameos/utils/http_client import frameos/values proc testConfig(): FrameConfig = @@ -16,6 +17,72 @@ proc testConfig(): FrameConfig = assetsPath: "/tmp" ) +## A tiny blocking HTTP echo server on a thread, so frameos.httpRequest can be +## tested end to end (method, headers, body, binary responses) without the network. + +var echoServerPort: Port +var echoServerThread: Thread[void] + +proc echoServerLoop() {.thread.} = + var server = newSocket() + server.setSockOpt(OptReuseAddr, true) + server.bindAddr(Port(0), "127.0.0.1") + server.listen() + var boundAddr: string + var boundPort: Port + (boundAddr, boundPort) = server.getLocalAddr() + echoServerPort = boundPort + + while true: + var client: Socket + server.accept(client) + var requestLine = "" + var authHeader = "" + var contentLength = 0 + try: + requestLine = client.recvLine(timeout = 5000) + while true: + let line = client.recvLine(timeout = 5000) + if line == "\r\n" or line.len == 0: + break + let lowered = line.toLowerAscii() + if lowered.startsWith("authorization:"): + authHeader = line.split(":", maxsplit = 1)[1].strip() + elif lowered.startsWith("content-length:"): + contentLength = parseInt(line.split(":", maxsplit = 1)[1].strip()) + except CatchableError: + client.close() + continue + + var body = "" + if contentLength > 0: + try: + body = client.recv(contentLength, timeout = 5000) + except CatchableError: + discard + + let parts = requestLine.splitWhitespace() + let httpMethod = if parts.len >= 1: parts[0] else: "" + let path = if parts.len >= 2: parts[1] else: "/" + + case path + of "/quit": + client.send("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + client.close() + break + of "/binary": + var payload = newString(256) + for i in 0 ..< 256: + payload[i] = chr(i) + client.send("HTTP/1.1 200 OK\r\nContent-Length: 256\r\n\r\n" & payload) + client.close() + else: + let reply = $(%*{"method": httpMethod, "auth": authHeader, "body": body}) + client.send("HTTP/1.1 200 OK\r\nContent-Length: " & $reply.len & "\r\n\r\n" & reply) + client.close() + + server.close() + proc testLogger(config: FrameConfig): Logger = var logger = Logger(frameConfig: config, enabled: true) logger.log = proc(payload: JsonNode) = @@ -301,6 +368,230 @@ suite "js app runtime": check stackLogs.len == 1 check ">:3:" in stackLogs[0]{"stack"}.getStr() + test "asset management bindings": + let assetsDir = getTempDir() / "frameos-js-assets-test" + removeDir(assetsDir) + createDir(assetsDir) + defer: removeDir(assetsDir) + + let config = testConfig() + config.assetsPath = assetsDir + let logger = testLogger(config) + let scene = FrameScene(id: "tests/js-app-assets".SceneId, frameConfig: config, state: %*{}, logger: logger) + let owner = AppRoot(nodeId: 20.NodeId, nodeName: "jsAssets", scene: scene, frameConfig: config) + let context = ExecutionContext(scene: scene, event: "render", payload: %*{}, hasImage: false, loopIndex: 0, loopKey: ".", nextSleep: -1) + + let runtime = newJsAppRuntime( + category = "data", + outputType = "json", + source = """export function get(app, context) { + return { + missing: frameos.readAsset("nope.txt"), + missingSize: frameos.assetSize("nope.txt"), + wrote: frameos.writeAsset("js/test.txt", "aGVsbG8="), + appended: frameos.appendAsset("js/test.txt", "IHdvcmxk"), + size: frameos.assetSize("js/test.txt"), + read: frameos.readAsset("js/test.txt"), + slice: frameos.readAsset("js/test.txt", { offset: 6, length: 5 }), + list: frameos.listAssets(""), + exists: frameos.assetExists("js/test.txt"), + escaped: frameos.writeAsset("../evil.txt", "aGVsbG8="), + absolute: frameos.writeAsset("/etc/evil.txt", "aGVsbG8="), + deleted: frameos.deleteAsset("js/test.txt"), + existsAfter: frameos.assetExists("js/test.txt"), + } + }""" + ) + + let value = runtime.get(owner, %*{}, context) + check value.kind == fkJson + let payload = value.asJson() + check payload["missing"].kind == JNull + check payload["missingSize"].getInt() == -1 + check payload["wrote"].getBool() + check payload["appended"].getBool() + check payload["size"].getFloat() == 11.0 + check decode(payload["read"].getStr()) == "hello world" + check decode(payload["slice"].getStr()) == "world" + check payload["list"].mapIt(it.getStr()) == @["js/test.txt"] + check payload["exists"].getBool() + check not payload["escaped"].getBool() + check not payload["absolute"].getBool() + check payload["deleted"].getBool() + check not payload["existsAfter"].getBool() + check not fileExists(assetsDir.parentDir() / "evil.txt") + + test "loads asset images within display bounds": + let assetsDir = getTempDir() / "frameos-js-asset-image-test" + removeDir(assetsDir) + createDir(assetsDir) + defer: removeDir(assetsDir) + + var source = newImage(3, 2) + source.fill(parseHtmlColor("#336699")) + writeFile(assetsDir / "plate.png", encodeImage(source, PngFormat)) + + let config = testConfig() + config.assetsPath = assetsDir + let logger = testLogger(config) + let scene = FrameScene(id: "tests/js-app-asset-image".SceneId, frameConfig: config, state: %*{}, logger: logger) + let owner = AppRoot(nodeId: 21.NodeId, nodeName: "jsAssetImage", scene: scene, frameConfig: config) + let context = ExecutionContext(scene: scene, event: "render", payload: %*{}, hasImage: false, loopIndex: 0, loopKey: ".", nextSleep: -1) + + let runtime = newJsAppRuntime( + category = "data", + outputType = "image", + source = """export function get(app, context) { + return frameos.loadAssetImage("plate.png") + }""" + ) + + let value = runtime.get(owner, %*{}, context) + check value.kind == fkImage + check value.asImage().width == 3 + check value.asImage().height == 2 + check runtime.images.len == 0 + + test "stream bindings round-trip strings and asset files": + let assetsDir = getTempDir() / "frameos-js-streams-test" + removeDir(assetsDir) + createDir(assetsDir) + defer: removeDir(assetsDir) + + let config = testConfig() + config.assetsPath = assetsDir + let logger = testLogger(config) + let scene = FrameScene(id: "tests/js-app-streams".SceneId, frameConfig: config, state: %*{}, logger: logger) + let owner = AppRoot(nodeId: 22.NodeId, nodeName: "jsStreams", scene: scene, frameConfig: config) + let context = ExecutionContext(scene: scene, event: "render", payload: %*{}, hasImage: false, loopIndex: 0, loopKey: ".", nextSleep: -1) + + let runtime = newJsAppRuntime( + category = "data", + outputType = "json", + source = """export function get(app, context) { + // String stream: write, rewind, read back in small chunks. + const scratch = frameos.createStream() + frameos.streamWrite(scratch, "aGVsbG8=") // "hello" + frameos.streamWrite(scratch, "IHdvcmxk") // " world" + frameos.streamRewind(scratch) + let chunks = [] + while (!frameos.streamAtEnd(scratch)) { + chunks.push(frameos.streamRead(scratch, 4)) + } + const closedScratch = frameos.streamClose(scratch) + + // File stream: write an asset via a stream, read it back whole. + const out = frameos.openAssetStream("streams/out.txt", "w") + frameos.streamWrite(out, "c3RyZWFtZWQ=") // "streamed" + frameos.streamClose(out) + const back = frameos.openAssetStream("streams/out.txt", "r") + const fileChunk = frameos.streamRead(back, 65536) + frameos.streamClose(back) + + return { + chunks, + closedScratch, + fileChunk, + missing: frameos.openAssetStream("streams/nope.txt", "r"), + badRead: frameos.streamRead({ id: 999999 }, 4), + } + }""" + ) + + let value = runtime.get(owner, %*{}, context) + check value.kind == fkJson + let payload = value.asJson() + var recovered = "" + for chunk in payload["chunks"].items: + recovered.add(decode(chunk.getStr())) + check recovered == "hello world" + check payload["closedScratch"].getBool() + check decode(payload["fileChunk"].getStr()) == "streamed" + check payload["missing"].kind == JNull + check payload["badRead"].kind == JNull + check readFile(assetsDir / "streams" / "out.txt") == "streamed" + + test "httpRequest posts with headers and fetches binary responses": + createThread(echoServerThread, echoServerLoop) + for _ in 0 ..< 100: + if int(echoServerPort) != 0: + break + sleep(20) + check int(echoServerPort) != 0 + let base = "http://127.0.0.1:" & $int(echoServerPort) + + let config = testConfig() + let logger = testLogger(config) + let scene = FrameScene(id: "tests/js-app-http".SceneId, frameConfig: config, state: %*{}, logger: logger) + let owner = AppRoot(nodeId: 23.NodeId, nodeName: "jsHttp", scene: scene, frameConfig: config) + let context = ExecutionContext(scene: scene, event: "render", payload: %*{}, hasImage: false, loopIndex: 0, loopKey: ".", nextSleep: -1) + + let runtime = newJsAppRuntime( + category = "data", + outputType = "json", + source = """export function get(app, context) { + const post = frameos.httpRequest(`${app.config.base}/echo`, { + method: "POST", + headers: { "Authorization": "Bearer sk-test", "Content-Type": "application/json" }, + body: JSON.stringify({ hello: "world" }), + }) + const binary = frameos.httpRequest(`${app.config.base}/binary`, { base64: true }) + const failed = frameos.httpRequest("http://127.0.0.1:1/nothing", { timeoutMs: 1000 }) + return { post, binary, failed } + }""" + ) + + let value = runtime.get(owner, %*{"base": base}, context) + discard boundedGetContent(base & "/quit") + check value.kind == fkJson + let payload = value.asJson() + check payload["post"]["status"].getInt() == 200 + let echoed = parseJson(payload["post"]["body"].getStr()) + check echoed["method"].getStr() == "POST" + check echoed["auth"].getStr() == "Bearer sk-test" + check parseJson(echoed["body"].getStr())["hello"].getStr() == "world" + check payload["binary"]["status"].getInt() == 200 + var expected = newString(256) + for i in 0 ..< 256: + expected[i] = chr(i) + check decode(payload["binary"]["bodyBase64"].getStr()) == expected + check payload["failed"]["status"].getInt() == 0 + check payload["failed"]["error"].getStr().len > 0 + + test "getSetting honors declared settings namespaces": + let config = testConfig() + config.settings = %*{"openAI": {"apiKey": "sk-test"}, "unsplash": {"accessKey": "u-test"}} + let logger = testLogger(config) + let scene = FrameScene(id: "tests/js-app-settings".SceneId, frameConfig: config, state: %*{}, logger: logger) + let owner = AppRoot(nodeId: 24.NodeId, nodeName: "jsSettings", scene: scene, frameConfig: config) + let context = ExecutionContext(scene: scene, event: "render", payload: %*{}, hasImage: false, loopIndex: 0, loopKey: ".", nextSleep: -1) + + let runtime = newJsAppRuntime( + category = "data", + outputType = "json", + source = """export function get(app, context) { + const allowed = frameos.getSetting("openAI", "apiKey") + const namespaceObj = frameos.getSetting("openAI") + const denied = frameos.getSetting("unsplash", "accessKey") + const missing = frameos.getSetting("openAI", "nope") + return { + allowed: allowed ?? null, + namespaceKey: (namespaceObj && namespaceObj.apiKey) || null, + denied: denied ?? null, + missing: missing ?? null, + } + }""" + , settingsKeys = @["openAI"] + ) + + let value = runtime.get(owner, %*{}, context) + check value.kind == fkJson + let payload = value.asJson() + check payload["allowed"].getStr() == "sk-test" + check payload["namespaceKey"].getStr() == "sk-test" + check payload["denied"].kind == JNull + check payload["missing"].kind == JNull + test "releases overwritten dynamic field image refs": let config = testConfig() let logger = testLogger(config) diff --git a/frameos/src/frameos/js_runtime/tests/test_js_transpiler.nim b/frameos/src/frameos/js_runtime/tests/test_js_transpiler.nim index a841d1d3b..e528fb5bc 100644 --- a/frameos/src/frameos/js_runtime/tests/test_js_transpiler.nim +++ b/frameos/src/frameos/js_runtime/tests/test_js_transpiler.nim @@ -222,6 +222,39 @@ const ok = metadata.satisfies satisfies boolean; check "const alias = metadata.as" in output check "const ok = metadata.satisfies" in output + test "preserves object keys after template literal interpolations": + # Regression: the `}` closing a `${...}` interpolation desynced token + # scope/depth tracking, so keys of a following object literal were taken + # for variable bindings and their string values erased as type annotations. + let output = transformFrameosScript(""" +function get(app) { + const post = call(`${app.config.base}/echo`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ hello: `${app.config.name}` }), + }) + return { post, tail: `${post}!` } +} +""") + check "method: \"POST\"" in output + check "headers: { \"Content-Type\": \"application/json\" }" in output + check "hello: `${app.config.name}`" in output + check "tail: `${post}!`" in output + + test "preserves comparison values of object keys": + # Regression: `key: value === x` lost its value — the member-type stripper + # took `value` for a type annotation ending at what it thought was a field + # initializer `=`. + let output = transformFrameosScript(""" +function verdict(res) { + return { ok: res.ok === true, no: res.ok !== true, same: res.kind == "a", fn: (x) => x } +} +""") + check "ok: res.ok === true" in output + check "no: res.ok !== true" in output + check "same: res.kind == \"a\"" in output + check "fn: (x) => x" in output + test "preserves modern ES syntax supported by QuickJS": let output = transformFrameosScript(""" class Counter { diff --git a/frameos/src/frameos/js_runtime/token_processor.nim b/frameos/src/frameos/js_runtime/token_processor.nim index 43c3c59da..454804678 100644 --- a/frameos/src/frameos/js_runtime/token_processor.nim +++ b/frameos/src/frameos/js_runtime/token_processor.nim @@ -198,7 +198,7 @@ proc previousToken*(processor: var TokenProcessor) = proc removeBalancedCode*(processor: var TokenProcessor) = var braceDepth = 0 while not processor.isAtEnd(): - if processor.matches1(ttBraceL): + if processor.matches1(ttBraceL) or processor.matches1(ttDollarBraceL): inc braceDepth elif processor.matches1(ttBraceR): if braceDepth == 0: diff --git a/frameos/src/frameos/js_runtime/transpiler.nim b/frameos/src/frameos/js_runtime/transpiler.nim index fc029d2fa..855eb22bc 100644 --- a/frameos/src/frameos/js_runtime/transpiler.nim +++ b/frameos/src/frameos/js_runtime/transpiler.nim @@ -895,7 +895,11 @@ proc stripMethodAndMemberTypes(code: string): string = let typeEnd = skipType(code, typeStart) var after = typeEnd skipSpaces(code, after) - if after < code.len and code[after] in {';', '='}: + # A member type ends at `;` or a single `=` (field initializer); + # `==`/`===`/`=>` mean the colon was a runtime object key instead. + let atInitializerEq = after < code.len and code[after] == '=' and + (after + 1 >= code.len or code[after + 1] notin {'=', '>'}) + if after < code.len and (code[after] == ';' or atInitializerEq): if code[after] == '=': result.add(' ') i = typeEnd @@ -1308,7 +1312,7 @@ proc tokenStatementEnd(tokens: seq[JsToken], start: int): int = inc parenDepth of ttParenR: if parenDepth > 0: dec parenDepth - of ttBraceL: + of ttBraceL, ttDollarBraceL: inc braceDepth of ttBraceR: if braceDepth == 0 and parenDepth == 0 and bracketDepth == 0: @@ -1330,7 +1334,8 @@ proc tokenStatementEnd(tokens: seq[JsToken], start: int): int = proc tokenMatching(tokens: seq[JsToken], openIndex: int, openType, closeType: TokenType): int = var depth = 0 for index in openIndex.. `${url}#${options.method}` + const post = request(`${app.config.base}/echo`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ hello: `${context.event}` }), + }) + return { post, ok: post.length !== 0, same: post === post } + } + ''', + app={"config": {"base": "http://frame"}}, + context={"event": "render"}, + ), Fixture( name="enum_runtime_values", source=r'''