Skip to content

Replace convertJSArrayToNumberVector with direct WASM frame buffer #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kalwalt wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Replace convertJSArrayToNumberVector with direct WASM frame buffer #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions build/webarkit_ES6_wasm.js
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions build/webarkit_ES6_wasm.simd.js
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion dist/SpeedyVisionSinkImageData.js
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion dist/WebARKit.js
View file
Open in desktop

Large diffs are not rendered by default.

33 changes: 30 additions & 3 deletions emscripten/WebARKitJS.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,40 @@
#include "WebARKitJS.h"

static int bppForColorSpace(webarkit::ColorSpace cs) {
switch (cs) {
case webarkit::RGBA: return 4;
case webarkit::RGB: return 3;
case webarkit::GRAY: return 1;
default: return 4;
}
}

WebARKit::~WebARKit() {
delete[] m_frameBuffer;
m_frameBuffer = nullptr;
}

void WebARKit::initTrackerGray(emscripten::val data_buffer, int width, int height, webarkit::ColorSpace colorSpace) {
auto u8 = emscripten::convertJSArrayToNumberVector<uint8_t>(data_buffer);
manager.initTracker(u8.data(), width, height, colorSpace);
}

void WebARKit::processFrame(emscripten::val data_buffer, webarkit::ColorSpace colorSpace, webarkit::BLUR_TYPE blurType) {
auto u8 = emscripten::convertJSArrayToNumberVector<uint8_t>(data_buffer);
manager.processFrameData(u8.data(), this->videoWidth, this->videoHeight, colorSpace, blurType);
void WebARKit::initFrameBuffer(webarkit::ColorSpace colorSpace) {
delete[] m_frameBuffer;
m_frameBufferSize = (size_t)this->videoWidth * this->videoHeight * bppForColorSpace(colorSpace);
m_frameBuffer = new uint8_t[m_frameBufferSize];
}

int WebARKit::getFrameBufferPtr() {
return reinterpret_cast<int>(m_frameBuffer);
}

void WebARKit::processFrame(webarkit::ColorSpace colorSpace, webarkit::BLUR_TYPE blurType) {
if (!m_frameBuffer) {
printf("[WebARKit] processFrame called before initFrameBuffer!\n");
return;
}
manager.processFrameData(m_frameBuffer, this->videoWidth, this->videoHeight, colorSpace, blurType);
}

void WebARKit::setLogLevel(int logLevel) { manager.setLogLevel(logLevel); }
Expand Down
8 changes: 7 additions & 1 deletion emscripten/WebARKitJS.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,12 @@ class WebARKit {
manager.initialiseBase(m_trackerType, this->videoWidth, this->videoHeight);
}

~WebARKit();

void initTrackerGray(emscripten::val data_buffer, int width, int height, webarkit::ColorSpace colorSpace);
void processFrame(emscripten::val data_buffer, webarkit::ColorSpace colorSpace, webarkit::BLUR_TYPE blurType);
void initFrameBuffer(webarkit::ColorSpace colorSpace);
int getFrameBufferPtr();
void processFrame(webarkit::ColorSpace colorSpace, webarkit::BLUR_TYPE blurType);
void setLogLevel(int logLevel);
emscripten::val getHomography();
emscripten::val getPoseMatrix();
Expand All @@ -48,4 +52,6 @@ class WebARKit {
int videoHeight;
webarkit::TRACKER_TYPE m_trackerType;
webarkit::WebARKitManager manager;
uint8_t* m_frameBuffer = nullptr;
size_t m_frameBufferSize = 0;
};
2 changes: 2 additions & 0 deletions emscripten/bindings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ EMSCRIPTEN_BINDINGS(constant_bindings) {
.constructor<>()
.constructor<int, int, webarkit::TRACKER_TYPE>()
.function("initTrackerGray", &WebARKit::initTrackerGray)
.function("initFrameBuffer", &WebARKit::initFrameBuffer)
.function("getFrameBufferPtr", &WebARKit::getFrameBufferPtr)
.function("processFrame", &WebARKit::processFrame)
.function("setLogLevel", &WebARKit::setLogLevel)
.function("getHomography", &WebARKit::getHomography)
Expand Down
3 changes: 2 additions & 1 deletion examples/worker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ function initTracker(msg) {
ar = wark;
wark.setLogLevel(WebARKit.WebARKitController.WEBARKIT_LOG_LEVEL_DEBUG);
wark.loadTrackerGrayImage(msg.imageData, msg.imgWidth, msg.imgHeight, WebARKit.WebARKitController.GRAY);

// Allocate the persistent WASM frame buffer once — avoids convertJSArrayToNumberVector on every frame.
wark.initFrameBuffer(WebARKit.WebARKitController.GRAY);
var cameraProjMat = wark.getCameraProjectionMatrix();
console.log("camera proj Mat: ", cameraProjMat);

Expand Down
3 changes: 2 additions & 1 deletion examples/worker_bufferCopy_threejs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ function initTracker(msg) {
//console.log("wark: ", wark)
wark.setLogLevel(WebARKit.WebARKitController.WEBARKIT_LOG_LEVEL_DEBUG);
wark.loadTrackerGrayImage(msg.imageData, msg.imgWidth, msg.imgHeight, WebARKit.WebARKitController.RGBA);

// Allocate the persistent WASM frame buffer once — avoids convertJSArrayToNumberVector on every frame.
wark.initFrameBuffer(WebARKit.WebARKitController.RGBA);
const cameraProjMat = wark.getCameraProjectionMatrix();
console.log("camera proj Mat: ", cameraProjMat);

Expand Down
3 changes: 2 additions & 1 deletion examples/worker_jsfeatNext.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ function initTracker(msg) {
wark.setLogLevel(WebARKit.WebARKitController.WEBARKIT_LOG_LEVEL_DEBUG);
wark.loadTrackerGrayImage(msg.imageData, msg.imgWidth, msg.imgHeight, WebARKit.WebARKitController.GRAY);
//wark.loadTrackerGrayImage(msg.imageData, msg.imgWidth, msg.imgHeight, WebARKit.WebARKitController.RGBA);

// Allocate the persistent WASM frame buffer once — avoids convertJSArrayToNumberVector on every frame.
wark.initFrameBuffer(WebARKit.WebARKitController.GRAY);
var cameraProjMat = wark.getCameraProjectionMatrix();
console.log("camera proj Mat: ", cameraProjMat);

Expand Down
2 changes: 2 additions & 0 deletions examples/worker_threejs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ function initTracker(msg) {
//console.log("wark: ", wark)
wark.setLogLevel(WebARKit.WebARKitController.WEBARKIT_LOG_LEVEL_DEBUG);
wark.loadTrackerGrayImage(msg.imageData, msg.imgWidth, msg.imgHeight, WebARKit.WebARKitController.RGBA);
// Allocate the persistent WASM frame buffer once — avoids convertJSArrayToNumberVector on every frame.
wark.initFrameBuffer(WebARKit.WebARKitController.RGBA);

const cameraProjMat = wark.getCameraProjectionMatrix();
console.log("camera proj Mat: ", cameraProjMat);
Expand Down
16 changes: 14 additions & 2 deletions src/WebARKitController.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ export default class WebARKitController {
// directly init with given width / height
const webARC = new WebARKitController();

return await webARC._initialize_raw( videoWidth, videoHeight, trackerType);
return await webARC._initialize_raw(videoWidth, videoHeight, trackerType);
}

async _initialize_raw(videoWidth, videoHeight, trackerType) {
Expand Down Expand Up @@ -145,8 +145,20 @@ export default class WebARKitController {
return this.webarkit.initTrackerGray(imgData, width, height, trackerType);
}

initFrameBuffer(colorSpace) {
this.webarkit.initFrameBuffer(colorSpace);
this._frameBufferPtr = this.webarkit.getFrameBufferPtr();
}

processFrame(imageData, colorSpace) {
this.webarkit.processFrame(imageData, colorSpace, WebARKitController.NONE_BLUR);
// Lazy-init: allocate buffer on first call if not already done explicitly.
if (!this._frameBufferPtr) {
this.initFrameBuffer(colorSpace);
}
// Write directly into WASM linear memory — no per-frame copy.
// Read HEAPU8 fresh each frame to guard against heap growth invalidation.
this.instance.HEAPU8.set(imageData, this._frameBufferPtr);
this.webarkit.processFrame(colorSpace, WebARKitController.NONE_BLUR);
}

setLogLevel(level) {
Expand Down