In-Browser Computer Vision: WebAssembly, Deep Learning Edge Inference & Canvas Alpha Matting

scalar: out0 = p0 * a;  out1 = p1 * a;  out2 = p2 * a;  out3 = p3 * a;

;; WebAssembly SIMD 128-Bit Vector Multiplication
v128.load       ;; Load 4 float32 pixels into register
v128.load_splat ;; Load alpha broadcasted across all 4 lanes
f32x4.mul       ;; Concurrently multiply all 4 lanes in 1 cycle
v128.store      ;; Store result back to memory buffer

/**
 * INNOXTOOLS CLIENT-SIDE COMPUTER VISION PIPELINE
 * High-performance, zero-server image segmentation and alpha matting engine.
 * Powered by ONNX Runtime Web, WebAssembly SIMD, and HTML5 Canvas.
 */

export interface SegmentationResult {
  alphaMask: Uint8ClampedArray;
  width: number;
  height: number;
  inferenceTimeMs: number;
  compositeBlob: Blob;
}

export class InBrowserMattingEngine {
  private static readonly MODEL_INPUT_SIZE = 512; // Standard 512x512 neural input

  /**
   * 1. Preprocess: Scales input image to tensor dimensions and normalizes RGB to [0.0..1.0]
   */
  private static createInputTensor(
    ctx: CanvasRenderingContext2D,
    width: number,
    height: number
  ): Float32Array {
    const imgData = ctx.getImageData(0, 0, width, height);
    const data = imgData.data;
    const numPixels = width * height;
    
    // Planar Tensor format: [1, 3, H, W] (Channels-First for PyTorch/ONNX models)
    const tensorBuffer = new Float32Array(3 * numPixels);
    const channelSize = numPixels;

    // ImageNet Normalization Constants
    const mean = [0.485, 0.456, 0.406];
    const std = [0.229, 0.224, 0.225];

    for (let i = 0; i < numPixels; i++) {
      const r = data[i * 4] / 255.0;
      const g = data[i * 4 + 1] / 255.0;
      const b = data[i * 4 + 2] / 255.0;

      // Normalize and write to planar channels
      tensorBuffer[i] = (r - mean[0]) / std[0];                   // Red Channel
      tensorBuffer[channelSize + i] = (g - mean[1]) / std[1];     // Green Channel
      tensorBuffer[2 * channelSize + i] = (b - mean[2]) / std[2]; // Blue Channel
    }

    return tensorBuffer;
  }

  /**
   * 2. Postprocess: Upscales the 512x512 neural probability map back to native image resolution
   * and composites the alpha matte directly onto the source image.
   */
  public static async executeMatting(
    imageElement: HTMLImageElement,
    rawModelOutput: Float32Array
  ): Promise<SegmentationResult> {
    const startTime = performance.now();
    const origWidth = imageElement.naturalWidth;
    const origHeight = imageElement.naturalHeight;

    // Offscreen Canvas for native high-res processing
    const canvas = new OffscreenCanvas(origWidth, origHeight);
    const ctx = canvas.getContext('2d') as OffscreenCanvasRenderingContext2D;
    ctx.drawImage(imageElement, 0, 0);

    const originalImage = ctx.getImageData(0, 0, origWidth, origHeight);
    const origData = originalImage.data;

    // Secondary canvas for bilinearly interpolating the 512x512 alpha mask
    const maskCanvas = new OffscreenCanvas(this.MODEL_INPUT_SIZE, this.MODEL_INPUT_SIZE);
    const maskCtx = maskCanvas.getContext('2d') as OffscreenCanvasRenderingContext2D;
    const maskImageData = maskCtx.createImageData(this.MODEL_INPUT_SIZE, this.MODEL_INPUT_SIZE);

    for (let i = 0; i < rawModelOutput.length; i++) {
      // Model outputs continuous probability [0.0..1.0]
      const alphaVal = Math.max(0, Math.min(255, Math.round(rawModelOutput[i] * 255)));
      maskImageData.data[i * 4] = alphaVal;     // R
      maskImageData.data[i * 4 + 1] = alphaVal; // G
      maskImageData.data[i * 4 + 2] = alphaVal; // B
      maskImageData.data[i * 4 + 3] = 255;      // A
    }
    maskCtx.putImageData(maskImageData, 0, 0);

    // Smoothly upscale mask to original dimensions using browser GPU bilinear filtering
    const upscaleCanvas = new OffscreenCanvas(origWidth, origHeight);
    const upscaleCtx = upscaleCanvas.getContext('2d') as OffscreenCanvasRenderingContext2D;
    upscaleCtx.imageSmoothingEnabled = true;
    upscaleCtx.imageSmoothingQuality = 'high';
    upscaleCtx.drawImage(maskCanvas, 0, 0, origWidth, origHeight);

    const upscaledMask = upscaleCtx.getImageData(0, 0, origWidth, origHeight).data;

    // 3. Final Composite: Inject continuous alpha channel into original RGBA buffer
    for (let i = 0; i < origWidth * origHeight; i++) {
      origData[i * 4 + 3] = upscaledMask[i * 4]; // Copy interpolated alpha
    }

    ctx.putImageData(originalImage, 0, 0);
    const compositeBlob = await canvas.convertToBlob({ type: 'image/png' });
    const inferenceTimeMs = performance.now() - startTime;

    return {
      alphaMask: upscaledMask,
      width: origWidth,
      height: origHeight,
      inferenceTimeMs,
      compositeBlob
    };
  }
}