DeepHide Embedding Engine

.gitignore

@@ -2,3 +2,4 @@ node_modules
 dist/
 build/
 .env
+data/

package.json

@@ -23,7 +23,10 @@
     "typescript": "^5.9.3"
   },
   "dependencies": {
+    "@noble/ciphers": "^2.1.1",
+    "@noble/hashes": "^2.0.1",
     "@picovoice/pvrecorder-node": "^1.2.8",
+    "@subspace/reed-solomon-erasure.wasm": "^0.2.5",
     "chalk": "^5.6.2",
     "fft.js": "^4.0.4"
   }

src/core/embedding/bitstream/preamble.ts

@@ -0,0 +1,21 @@
+/**
+ * Sync Preamble
+ * Prepends a high-entropy sync pattern to the bitstream.
+ */
+
+export function injectPreamble(payloadBits: Uint8Array) {
+  const PREAMBLE = new Uint8Array([
+    1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1,
+    0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0,
+    0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0,
+    1, 0, 1, 1, 1,
+  ]);
+
+  const totalBits = PREAMBLE.length + payloadBits.length;
+  const syncStream = new Uint8Array(totalBits);
+
+  syncStream.set(PREAMBLE, 0);
+  syncStream.set(payloadBits, PREAMBLE.length);
+
+  return syncStream;
+}

src/core/embedding/bitstream/serialiser.ts

@@ -0,0 +1,24 @@
+/**
+ * Bitstream Serialisation
+ * Unpacks bytes into an array of bits
+ */
+
+export function serialiseBits(interleadShards: Uint8Array[]): Uint8Array {
+  const totalBytes = interleadShards.reduce(
+    (acc, shard) => acc + shard.length,
+    0,
+  );
+  const bitstream = new Uint8Array(totalBytes * 8);
+
+  let bitIndex = 0;
+  for (const shard of interleadShards) {
+    for (let i = 0; i < shard.length; i++) {
+      const byte = shard[i];
+
+      for (let shift = 0; shift < 8; shift++) {
+        bitstream[bitIndex++] = (byte! >> shift) & 1;
+      }
+    }
+  }
+  return bitstream;
+}

src/core/embedding/crypto/aes.ts

@@ -0,0 +1,21 @@
+import { createCipheriv, randomBytes } from "node:crypto";
+
+/**
+ * AES-256-GCM Encryption
+ * Encrypts the framed payload and adds the authentication tag.
+ * @returns packet of @type Uint8Array
+ */
+
+export function encryptPayload(framedPayload: Uint8Array, key: Uint8Array) {
+  const nonce = randomBytes(12);
+  const cipher = createCipheriv("aes-256-gcm", key, nonce);
+
+  const cipherText = Buffer.concat([
+    cipher.update(framedPayload),
+    cipher.final(),
+  ]);
+  const authTag = cipher.getAuthTag();
+
+  const encryptedPacket = Buffer.concat([nonce, cipherText, authTag]);
+  return new Uint8Array(encryptedPacket);
+}

src/core/embedding/crypto/keyDerivation.ts

@@ -0,0 +1,27 @@
+import { pbkdf2 } from "node:crypto";
+import { promisify } from "node:util";
+
+const pbkdf2Async = promisify(pbkdf2);
+
+/**
+ * Key Derivation
+ * Transforms a human password into a high-entropy 256-bit key.
+ */
+export async function deriveKey(password: string, salt: Uint8Array) {
+  const iterations = 600_000;
+  const keyLength = 32;
+  const digest = "sha256";
+
+  try {
+    const derivedKey = await pbkdf2Async(
+      password,
+      salt,
+      iterations,
+      keyLength,
+      digest,
+    );
+    return new Uint8Array(derivedKey);
+  } catch (error) {
+    throw new Error("Failed to derive key");
+  }
+}

src/core/embedding/fec/interleave.ts

@@ -0,0 +1,28 @@
+/**
+ * Interleaving
+ * Reorders shards so that consecutive physical errors are distributed
+ * across different logical RS blocks.
+ */
+
+export function interleave(
+  shards: Uint8Array[],
+  dataShards: number,
+  parityShards: number,
+) {
+  const totalShardsPerBlock = dataShards + parityShards;
+  const numBlocks = Math.ceil(shards.length / totalShardsPerBlock);
+  const interleaved: Uint8Array[] = new Array(shards.length);
+
+  let index = 0;
+  for (let col = 0; col < totalShardsPerBlock; col++) {
+    for (let row = 0; row < numBlocks; row++) {
+      const sourceIdx = row * totalShardsPerBlock + col;
+
+      if (sourceIdx < shards.length) {
+        interleaved[index++] = shards[sourceIdx]!;
+      }
+    }
+  }
+
+  return interleaved;
+}

src/core/embedding/fec/readSolomon.ts

@@ -0,0 +1,74 @@
+import { ReedSolomonErasure } from "@subspace/reed-solomon-erasure.wasm";
+import fs from "node:fs";
+import path from "node:path";
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+let rsInstance: ReedSolomonErasure | null = null;
+
+async function getRSEngine(): Promise<ReedSolomonErasure> {
+  if (!rsInstance) {
+    const pkgPath = require.resolve("@subspace/reed-solomon-erasure.wasm");
+    const pkgDir = path.dirname(pkgPath);
+
+    const wasmPath = path.join(pkgDir, "reed_solomon_erasure_bg.wasm");
+    if (!fs.existsSync(wasmPath)) {
+      throw new Error(`WASM file missing! Looked for it at: ${wasmPath}`);
+    }
+
+    const wasmBuffer = fs.readFileSync(wasmPath);
+    rsInstance = ReedSolomonErasure.fromBytes(wasmBuffer);
+  }
+  return rsInstance;
+}
+
+/**
+ * Forward Error Correction (FEC)
+ *
+ * Groups packets into blocks and adds shard for recovery
+ *
+ * @param packets
+ * @param dataShards
+ * @param parityShards
+ */
+
+export async function applyFEC(
+  packets: Uint8Array[],
+  dataShards: number,
+  parityShards: number = 3,
+): Promise<Uint8Array[]> {
+  if (packets.length === 0) return [];
+
+  const rs = await getRSEngine();
+  const shardLength = packets[0]!.length;
+  const encodedStream: Uint8Array[] = [];
+
+  for (let i = 0; i < packets.length; i += dataShards) {
+    const block = packets.slice(i, i + dataShards);
+
+    const totalShards = dataShards + parityShards;
+    const contiguousBuffer = new Uint8Array(totalShards * shardLength);
+
+    for (let j = 0; j < dataShards; j++) {
+      if (block[j]) {
+        contiguousBuffer.set(block[j]!, j * shardLength);
+      }
+    }
+
+    const result = rs.encode(contiguousBuffer, dataShards, parityShards);
+
+    if (result !== ReedSolomonErasure.RESULT_OK) {
+      throw new Error(`WASM FEC Encoding failed with internal code: ${result}`);
+    }
+
+    for (let j = 0; j < totalShards; j++) {
+      const shard = contiguousBuffer.slice(
+        j * shardLength,
+        (j + 1) * shardLength,
+      );
+      encodedStream.push(shard);
+    }
+  }
+
+  return encodedStream;
+}

src/core/embedding/generator.ts

@@ -0,0 +1,32 @@
+import { randomBytes } from "node:crypto";
+import { framePayload } from "./payload/framing.js";
+import { loadFileToUint8 } from "./payload/Uint8FileReader.js";
+import { deriveKey } from "./crypto/keyDerivation.js";
+import { encryptPayload } from "./crypto/aes.js";
+import { packetize } from "./payload/packer.js";
+import { applyFEC } from "./fec/readSolomon.js";
+import { interleave } from "./fec/interleave.js";
+import { serialiseBits } from "./bitstream/serialiser.js";
+import { injectPreamble } from "./bitstream/preamble.js";
+
+export async function preparePayload(filename: string, password: string) {
+  const rawBytes = await loadFileToUint8(filename);
+  const framed = framePayload(rawBytes, filename);
+
+  const salt = randomBytes(16);
+  const key = await deriveKey(password, salt);
+
+  const encrypted = encryptPayload(framed, key);
+
+  const packets = packetize(encrypted, 256);
+
+  const FEC_SHARDS = await applyFEC(packets, 6, 3);
+
+  const interleaved = interleave(FEC_SHARDS, 6, 3);
+
+  const payloadBits = serialiseBits(interleaved);
+
+  const finalBitStream = injectPreamble(payloadBits);
+
+  return { finalBitStream, salt };
+}

src/core/embedding/payload/Uint8FileReader.ts

@@ -0,0 +1,11 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+export async function loadFileToUint8(filename: string): Promise<Uint8Array> {
+  const filepath = path.join(process.cwd(), "data", filename);
+  try {
+    const buffer = await readFile(filepath);
+    return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+  } catch (error) {
+    throw new Error(`Failed to read file: ${filepath}`);
+  }
+}

src/core/embedding/payload/framing.ts

@@ -0,0 +1,28 @@
+/**
+ * Payload Framing
+ * Wraps raw bytes with a protocol header for identification and reconstruction.
+ * @returns packet of @type Uint8Array
+ */
+
+export function framePayload(filebytes: Uint8Array, filename: string) {
+  const encoder = new TextEncoder();
+  const filenameBytes = encoder.encode(filename);
+
+  if (filenameBytes.length > 255) {
+    throw new Error("Filename too long");
+  }
+
+  const headerSize = 10;
+  const totalSize = headerSize + filenameBytes.length + filebytes.length;
+  const packet = new Uint8Array(totalSize);
+  const view = new DataView(packet.buffer);
+
+  view.setUint32(4, 0x44484944, false);
+  view.setUint32(4, 0x01);
+  view.setUint8(5, filenameBytes.length);
+  view.setUint32(6, filebytes.length, false);
+
+  packet.set(filenameBytes, headerSize);
+  packet.set(filebytes, headerSize + filenameBytes.length);
+  return packet;
+}

src/core/embedding/payload/packer.ts

@@ -0,0 +1,32 @@
+/**
+ * Packetization
+ * Slices the encrypted payload into manageable frames with IDs.
+ */
+
+export function packetize(
+  encryptedData: Uint8Array,
+  frameSize: number = 512,
+): Uint8Array[] {
+  const frames: Uint8Array[] = [];
+  const totalBytes = encryptedData.length;
+
+  let offset = 0;
+  let frameId = 0;
+
+  while (offset < totalBytes) {
+    const end = Math.min(offset + frameSize, totalBytes);
+    const chunk = encryptedData.slice(offset, end);
+
+    const frame = new Uint8Array(4 + chunk.length);
+    const view = new DataView(frame.buffer);
+
+    view.setUint32(0, frameId, false);
+    frame.set(chunk, 4);
+    frames.push(frame);
+
+    offset += frameSize;
+    frameId++;
+  }
+
+  return frames;
+}