From 7889578ced44cd5c2a2abdc0a48b174d127fae6b Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Wed, 5 Nov 2025 13:01:04 +0100 Subject: [PATCH] :tada: Use textures directly for images --- frontend/src/app/render_wasm/api.cljs | 105 ++++++++++++++++++-------- render-wasm/src/render.rs | 13 ++++ render-wasm/src/render/images.rs | 66 ++++++++++++++++ render-wasm/src/wasm/fills/image.rs | 52 +++++++++++++ 4 files changed, 203 insertions(+), 33 deletions(-) diff --git a/frontend/src/app/render_wasm/api.cljs b/frontend/src/app/render_wasm/api.cljs index 16a78edab0..2472909b5c 100644 --- a/frontend/src/app/render_wasm/api.cljs +++ b/frontend/src/app/render_wasm/api.cljs @@ -34,8 +34,6 @@ [app.render-wasm.wasm :as wasm] [app.util.debug :as dbg] [app.util.functions :as fns] - [app.util.http :as http] - [app.util.webapi :as wapi] [beicon.v2.core :as rx] [promesa.core :as p] [rumext.v2 :as mf])) @@ -234,49 +232,90 @@ [string] (+ (count string) 1)) +(defn- create-webgl-texture-from-image + "Creates a WebGL texture from an HTMLImageElement or ImageBitmap and returns the texture object" + [gl image-element] + (let [texture (.createTexture ^js gl)] + (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture) + (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_WRAP_S ^js gl) (.-CLAMP_TO_EDGE ^js gl)) + (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_WRAP_T ^js gl) (.-CLAMP_TO_EDGE ^js gl)) + (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_MIN_FILTER ^js gl) (.-LINEAR ^js gl)) + (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_MAG_FILTER ^js gl) (.-LINEAR ^js gl)) + (.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) image-element) + (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil) + texture)) + +(defn- get-webgl-context + "Gets the WebGL context from the WASM module" + [] + (when wasm/context-initialized? + (let [gl-obj (unchecked-get wasm/internal-module "GL")] + (when gl-obj + ;; Get the current WebGL context from Emscripten + ;; The GL object has a currentContext property that contains the context handle + (let [current-ctx (.-currentContext ^js gl-obj)] + (when current-ctx + (.-GLctx ^js current-ctx))))))) + +(defn- get-texture-id-for-gl-object + "Registers a WebGL texture with Emscripten's GL object system and returns its ID" + [texture] + (let [gl-obj (unchecked-get wasm/internal-module "GL") + textures (.-textures ^js gl-obj) + new-id (.getNewId ^js gl-obj textures)] + (aset textures new-id texture) + new-id)) (defn- fetch-image + "Loads an image and creates a WebGL texture from it, passing the texture ID to WASM. + This avoids decoding the image twice (once in browser, once in WASM)." [shape-id image-id thumbnail?] (let [url (cf/resolve-file-media {:id image-id} thumbnail?)] {:key url :thumbnail? thumbnail? - :callback #(->> (http/send! {:method :get - :uri url - :response-type :blob}) - (rx/map :body) - (rx/mapcat wapi/read-file-as-array-buffer) - (rx/map (fn [image] - (let [size (.-byteLength image) - padded-size (if (zero? (mod size 4)) size (+ size (- 4 (mod size 4)))) - ;; 36 bytes header (32 for UUIDs + 4 for thumbnail flag) + padded image - total-bytes (+ 36 padded-size) - offset (mem/alloc->offset-32 total-bytes) - heap32 (mem/get-heap-u32) - data (js/Uint8Array. image) - padded (js/Uint8Array. padded-size)] + :callback #(->> (p/create + (fn [resolve reject] + (let [img (js/Image.) + on-load (fn [] + (resolve img)) + on-error (fn [err] + (reject err))] + (set! (.-crossOrigin img) "anonymous") + (.addEventListener img "load" on-load) + (.addEventListener img "error" on-error) + (set! (.-src img) url)))) + (rx/from) + (rx/map (fn [img] + (when-let [gl (get-webgl-context)] + (let [texture (create-webgl-texture-from-image gl img) + texture-id (get-texture-id-for-gl-object texture) + width (.-width ^js img) + height (.-height ^js img) + ;; Header: 32 bytes (2 UUIDs) + 4 bytes (thumbnail) + 4 bytes (texture ID) + 8 bytes (dimensions) + total-bytes 48 + offset (mem/alloc->offset-32 total-bytes) + heap32 (mem/get-heap-u32)] - ;; 1. Set shape id (offset + 0 to offset + 3) - (mem.h32/write-uuid offset heap32 shape-id) + ;; 1. Set shape id (offset + 0 to offset + 3) + (mem.h32/write-uuid offset heap32 shape-id) - ;; 2. Set image id (offset + 4 to offset + 7) - (mem.h32/write-uuid (+ offset 4) heap32 image-id) + ;; 2. Set image id (offset + 4 to offset + 7) + (mem.h32/write-uuid (+ offset 4) heap32 image-id) - ;; 3. Set thumbnail flag as u32 (offset + 8) - (aset heap32 (+ offset 8) thumbnail?) + ;; 3. Set thumbnail flag as u32 (offset + 8) + (aset heap32 (+ offset 8) (if thumbnail? 1 0)) - ;; 4. Adjust padding on image data - (.set padded data) - (when (< size padded-size) - (dotimes [i (- padded-size size)] - (aset padded (+ size i) 0))) + ;; 4. Set texture ID (offset + 9) + (aset heap32 (+ offset 9) texture-id) - ;; 5. Set image data (starting at offset + 9) - (let [u32view (js/Uint32Array. (.-buffer padded)) - image-u32-offset (+ offset 9)] - (.set heap32 u32view image-u32-offset)) + ;; 5. Set width (offset + 10) + (aset heap32 (+ offset 10) width) - (h/call wasm/internal-module "_store_image") - true))))})) + ;; 6. Set height (offset + 11) + (aset heap32 (+ offset 11) height) + + (h/call wasm/internal-module "_store_image_from_texture") + true)))))})) (defn- get-fill-images [leaf] diff --git a/render-wasm/src/render.rs b/render-wasm/src/render.rs index 0c9cea0e6a..a4d6b214d6 100644 --- a/render-wasm/src/render.rs +++ b/render-wasm/src/render.rs @@ -329,6 +329,19 @@ impl RenderState { self.images.add(id, is_thumbnail, image_data) } + /// Adds an image from an existing WebGL texture, avoiding re-decoding + pub fn add_image_from_gl_texture( + &mut self, + id: Uuid, + is_thumbnail: bool, + texture_id: u32, + width: i32, + height: i32, + ) -> Result<(), String> { + self.images + .add_image_from_gl_texture(id, is_thumbnail, texture_id, width, height) + } + pub fn has_image(&self, id: &Uuid, is_thumbnail: bool) -> bool { self.images.contains(id, is_thumbnail) } diff --git a/render-wasm/src/render/images.rs b/render-wasm/src/render/images.rs index 63f06aead7..4faf393895 100644 --- a/render-wasm/src/render/images.rs +++ b/render-wasm/src/render/images.rs @@ -62,6 +62,48 @@ pub struct ImageStore { context: Box, } +/// Creates a Skia image from an existing WebGL texture. +/// This avoids re-decoding the image, as the browser has already decoded +/// and uploaded it to the GPU. +fn create_image_from_gl_texture( + context: &mut Box, + texture_id: u32, + width: i32, + height: i32, +) -> Result { + use skia_safe::gpu; + use skia_safe::gpu::gl::TextureInfo; + + // Create a TextureInfo describing the existing GL texture + let texture_info = TextureInfo { + target: gl::TEXTURE_2D, + id: texture_id, + format: gl::RGBA8, + protected: gpu::Protected::No, + }; + + // Create a backend texture from the GL texture using the new API + let label = format!("shared_texture_{}", texture_id); + let backend_texture = unsafe { + gpu::backend_textures::make_gl((width, height), gpu::Mipmapped::No, texture_info, label) + }; + + // Create a Skia image from the backend texture + // Use TopLeft origin because HTML images have their origin at top-left, + // while WebGL textures traditionally use bottom-left + let image = Image::from_texture( + context.as_mut(), + &backend_texture, + gpu::SurfaceOrigin::TopLeft, + skia::ColorType::RGBA8888, + skia::AlphaType::Premul, + None, + ) + .ok_or("Failed to create Skia image from GL texture")?; + + Ok(image) +} + // Decode and upload to GPU fn decode_image(context: &mut Box, raw_data: &[u8]) -> Option { let data = unsafe { skia::Data::new_bytes(raw_data) }; @@ -122,6 +164,30 @@ impl ImageStore { Ok(()) } + /// Creates a Skia image from an existing WebGL texture, avoiding re-decoding. + /// This is much more efficient as it reuses the texture that was already + /// decoded and uploaded to GPU by the browser. + pub fn add_image_from_gl_texture( + &mut self, + id: Uuid, + is_thumbnail: bool, + texture_id: u32, + width: i32, + height: i32, + ) -> Result<(), String> { + let key = (id, is_thumbnail); + + if self.images.contains_key(&key) { + return Err("Image already exists".to_string()); + } + + // Create a Skia image from the existing GL texture + let image = create_image_from_gl_texture(&mut self.context, texture_id, width, height)?; + self.images.insert(key, StoredImage::Gpu(image)); + + Ok(()) + } + pub fn contains(&self, id: &Uuid, is_thumbnail: bool) -> bool { self.images.contains_key(&(*id, is_thumbnail)) } diff --git a/render-wasm/src/wasm/fills/image.rs b/render-wasm/src/wasm/fills/image.rs index fd3e2e1376..4c1e49b94d 100644 --- a/render-wasm/src/wasm/fills/image.rs +++ b/render-wasm/src/wasm/fills/image.rs @@ -89,3 +89,55 @@ pub extern "C" fn store_image() { mem::free_bytes(); } + +/// Stores an image from an existing WebGL texture, avoiding re-decoding +/// Expected memory layout: +/// - bytes 0-15: shape UUID +/// - bytes 16-31: image UUID +/// - bytes 32-35: is_thumbnail flag (u32) +/// - bytes 36-39: GL texture ID (u32) +/// - bytes 40-43: width (i32) +/// - bytes 44-47: height (i32) +#[no_mangle] +pub extern "C" fn store_image_from_texture() { + let bytes = mem::bytes(); + + if bytes.len() < 48 { + eprintln!("store_image_from_texture: insufficient data"); + mem::free_bytes(); + return; + } + + let ids = ShapeImageIds::try_from(bytes[0..IMAGE_IDS_SIZE].to_vec()).unwrap(); + + // Read is_thumbnail flag (4 bytes as u32) + let is_thumbnail_bytes = &bytes[IMAGE_IDS_SIZE..IMAGE_HEADER_SIZE]; + let is_thumbnail_value = u32::from_le_bytes(is_thumbnail_bytes.try_into().unwrap()); + let is_thumbnail = is_thumbnail_value != 0; + + // Read GL texture ID (4 bytes as u32) + let texture_id_bytes = &bytes[36..40]; + let texture_id = u32::from_le_bytes(texture_id_bytes.try_into().unwrap()); + + // Read width and height (8 bytes as two i32s) + let width_bytes = &bytes[40..44]; + let width = i32::from_le_bytes(width_bytes.try_into().unwrap()); + + let height_bytes = &bytes[44..48]; + let height = i32::from_le_bytes(height_bytes.try_into().unwrap()); + + with_state_mut!(state, { + if let Err(msg) = state.render_state_mut().add_image_from_gl_texture( + ids.image_id, + is_thumbnail, + texture_id, + width, + height, + ) { + eprintln!("store_image_from_texture error: {}", msg); + } + state.touch_shape(ids.shape_id); + }); + + mem::free_bytes(); +}