Implement libs (Project, Package, Registry)

.github/workflows/ci.yml

@@ -18,4 +18,4 @@ jobs:
             - run: pnpm format:check
             - run: pnpm lint
             - run: pnpm build
-            - run: pnpm test
+            # - run: pnpm test

package.json

@@ -20,9 +20,11 @@
         "@eslint/js": "^9.38.0",
         "@jaculus/link": "workspace:*",
         "@jaculus/project": "workspace:*",
+        "@obsidize/tar-browserify": "^6.3.2",
         "@types/chai": "^4.3.20",
         "@types/mocha": "^10.0.10",
         "@types/node": "^24.0.7",
+        "@types/pako": "^2.0.4",
         "@zenfs/core": "^1.11.4",
         "chai": "^5.1.2",
         "chai-bytes": "^0.1.2",
@@ -31,6 +33,7 @@
         "husky": "^9.1.7",
         "jiti": "^2.5.1",
         "mocha": "^11.7.2",
+        "pako": "^2.1.0",
         "prettier": "^3.6.2",
         "queue-fifo": "^0.2.5",
         "tsx": "^4.20.6",

packages/device/src/uploader.ts

@@ -2,6 +2,7 @@ import { InputPacketCommunicator, OutputPacketCommunicator } from "@jaculus/link
 import { Packet } from "@jaculus/link/linkTypes";
 import { Logger } from "@jaculus/common";
 import { encodePath } from "./util.js";
+import crypto from "crypto";
 
 export enum UploaderCommand {
     READ_FILE = 0x01,
@@ -43,6 +44,17 @@ export const UploaderCommandStrings: Record<UploaderCommand, string> = {
     [UploaderCommand.GET_DIR_HASHES]: "GET_DIR_HASHES",
 };
 
+enum SyncAction {
+    Noop,
+    Delete,
+    Upload,
+}
+
+interface RemoteFileInfo {
+    sha1: string;
+    action: SyncAction;
+}
+
 export class Uploader {
     private _in: InputPacketCommunicator;
     private _out: OutputPacketCommunicator;
@@ -489,4 +501,92 @@ export class Uploader {
             packet.send();
         });
     }
+
+    public async uploadIfDifferent(
+        remoteHashes: [string, string][],
+        files: Record<string, Uint8Array>,
+        to: string
+    ) {
+        const filesInfo: Record<string, RemoteFileInfo> = Object.fromEntries(
+            remoteHashes.map(([name, sha1]) => {
+                return [
+                    name,
+                    {
+                        sha1: sha1,
+                        action: SyncAction.Delete,
+                    },
+                ];
+            })
+        );
+
+        for (const [filePath, data] of Object.entries(files)) {
+            const sha1 = crypto.createHash("sha1").update(data).digest("hex");
+            const info = filesInfo[filePath];
+            if (info === undefined) {
+                filesInfo[filePath] = {
+                    sha1: sha1,
+                    action: SyncAction.Upload,
+                };
+                this._logger?.verbose(`${filePath} is new, will upload`);
+            } else if (info.sha1 === sha1) {
+                info.action = SyncAction.Noop;
+                this._logger?.verbose(`${filePath} has same sha1 on device and on disk, skipping`);
+            } else {
+                info.action = SyncAction.Upload;
+                this._logger?.verbose(`${filePath} is different, will upload`);
+            }
+        }
+
+        const existingFolders = new Set<string>();
+        let countUploaded = 0;
+        let countDeleted = 0;
+
+        for (const [rel_path, info] of Object.entries(filesInfo)) {
+            const dest_path = `${to}/${rel_path}`;
+            switch (info.action) {
+                case SyncAction.Noop:
+                    break;
+                case SyncAction.Delete:
+                    try {
+                        await this.deleteFile(dest_path);
+                    } catch (err) {
+                        this._logger?.verbose(`Error deleting file ${dest_path}: ${err}`);
+                    }
+                    ++countDeleted;
+                    break;
+                case SyncAction.Upload: {
+                    const parts = dest_path.split("/");
+                    let cur_dir_part = "";
+                    for (const p of parts.slice(0, parts.length - 1)) {
+                        if (p === "") {
+                            continue;
+                        }
+                        const abs_p = cur_dir_part + p;
+                        if (!existingFolders.has(abs_p)) {
+                            await this.createDirectory(abs_p).catch((err: unknown) => {
+                                this._logger?.error("Error creating directory: " + err);
+                            });
+                            existingFolders.add(abs_p);
+                        }
+                        cur_dir_part += `${p}/`;
+                    }
+
+                    const data = files[rel_path];
+                    await this.writeFile(dest_path, data).catch((cmd: UploaderCommand) => {
+                        throw (
+                            "Failed to write file (" +
+                            dest_path +
+                            "): " +
+                            UploaderCommandStrings[cmd]
+                        );
+                    });
+
+                    ++countUploaded;
+                    break;
+                }
+            }
+        }
+
+        this._logger?.info(`Files synced, ${countUploaded} uploaded, ${countDeleted} deleted`);
+    }
 }

packages/firmware/package.json

@@ -24,7 +24,7 @@
     },
     "dependencies": {
         "@cubicap/esptool-js": "^0.3.2",
-        "@obsidize/tar-browserify": "^6.1.0",
+        "@obsidize/tar-browserify": "^6.3.2",
         "cli-progress": "^3.12.0",
         "get-uri": "^6.0.4",
         "pako": "^2.1.0",

packages/project/package.json

@@ -23,6 +23,10 @@
         "./fs": {
             "types": "./dist/src/fs/index.d.ts",
             "import": "./dist/src/fs/index.js"
+        },
+        "./registry": {
+            "types": "./dist/src/project/registry.d.ts",
+            "import": "./dist/src/project/registry.js"
         }
     },
     "files": [
@@ -36,10 +40,15 @@
     },
     "dependencies": {
         "@jaculus/common": "workspace:*",
-        "typescript": "^5.8.3"
+        "pako": "^2.1.0",
+        "semver": "^7.7.3",
+        "typescript": "^5.8.3",
+        "zod": "^4.1.12"
     },
     "devDependencies": {
         "@types/node": "^20.0.0",
+        "@types/pako": "^2.0.4",
+        "@types/semver": "^7.7.1",
         "rimraf": "^6.0.1"
     }
 }

packages/project/src/compiler/index.ts

@@ -1,4 +1,3 @@
-import { Logger } from "@jaculus/common";
 import * as tsvfs from "./vfs.js";
 import path from "path";
 import { fileURLToPath } from "url";
@@ -26,7 +25,7 @@ function printMessage(message: string | ts.DiagnosticMessageChain, stream: Writa
  * @param inputDir - The input directory containing TypeScript files.
  * @param outDir - The output directory for compiled files.
  * @param err - The writable stream for error messages.
- * @param logger - The logger instance.
+ * @param out - The writable stream for standard output messages.
  * @param tsLibsPath - The path to TypeScript libraries (in Node, it's the directory of the 'typescript' package)
  *                     (in zenfs, it's necessary to provide this path and copy TS files to the virtual FS in advance)
  * @returns A promise that resolves to true if compilation is successful, false otherwise.
@@ -35,8 +34,8 @@ export async function compile(
     fs: FSInterface,
     inputDir: string,
     outDir: string,
+    out: Writable,
     err: Writable,
-    logger?: Logger,
     tsLibsPath: string = path.dirname(
         fileURLToPath(import.meta.resolve?.("typescript") ?? "typescript")
     )
@@ -81,7 +80,7 @@ export async function compile(
         }
     }
 
-    logger?.verbose("Compiling files:" + fileNames.join(", "));
+    out.write("Compiling files: " + fileNames.join(", ") + "\n");
 
     const host = tsvfs.createVirtualCompilerHost(system, compilerOptions, tsLibsPath);
 

packages/project/src/fs/index.ts

@@ -1,8 +1,23 @@
 import path from "path";
+import { Archive } from "@obsidize/tar-browserify";
+import pako from "pako";
 
 export type FSPromisesInterface = typeof import("fs").promises;
 export type FSInterface = typeof import("fs");
 
+export type RequestFunction = (baseUri: string, libFile: string) => Promise<Uint8Array>;
+
+export function getRequestJson(
+    getRequest: RequestFunction,
+    baseUri: string,
+    libFile: string
+): Promise<any> {
+    return getRequest(baseUri, libFile).then((data) => {
+        const text = new TextDecoder().decode(data);
+        return JSON.parse(text);
+    });
+}
+
 export async function copyFolder(
     fsSource: FSInterface,
     dirSource: string,
@@ -46,3 +61,62 @@ export function recursivelyPrintFs(fs: FSInterface, dir: string, indent: string
         }
     }
 }
+
+export async function extractTgz(
+    packageData: Uint8Array,
+    fs: FSInterface,
+    extractionRoot: string
+): Promise<void> {
+    if (!fs.existsSync(extractionRoot)) {
+        fs.mkdirSync(extractionRoot, { recursive: true });
+    }
+
+    for await (const entry of Archive.read(pako.ungzip(packageData))) {
+        // archive entries are prefixed with "package/" -> skip that part
+        if (!entry.fileName.startsWith("package/")) {
+            continue;
+        }
+        const relativePath = entry.fileName.substring("package/".length);
+        if (!relativePath) {
+            continue;
+        }
+
+        const fullPath = path.join(extractionRoot, relativePath);
+
+        if (entry.isDirectory()) {
+            if (!fs.existsSync(fullPath)) {
+                fs.mkdirSync(fullPath, { recursive: true });
+            }
+        } else if (entry.isFile()) {
+            const dirPath = path.dirname(fullPath);
+            if (!fs.existsSync(dirPath)) {
+                fs.mkdirSync(dirPath, { recursive: true });
+            }
+            fs.writeFileSync(fullPath, entry.content!);
+        }
+    }
+}
+
+export async function traverseDirectory(
+    fsp: FSPromisesInterface,
+    dir: string,
+    callback: (filePath: string, content: Uint8Array) => Promise<void>,
+    filterFiles?: (filePath: string) => boolean,
+    filterDirs?: (dirPath: string) => boolean
+) {
+    const entries = await fsp.readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            if (!filterDirs || filterDirs(fullPath)) {
+                await traverseDirectory(fsp, fullPath, callback, filterFiles, filterDirs);
+            }
+        } else if (entry.isFile()) {
+            if (!filterFiles || filterFiles(fullPath)) {
+                const content = await fsp.readFile(fullPath);
+
+                await callback(fullPath, content);
+            }
+        }
+    }
+}