v0.10.2: Enhance applyEffects with support for FBO, image, and video inputs (image/video support added)

Author: nakednous

README.md

@@ -182,17 +182,17 @@ The `applyShader` function applies a `shader` to a given `scene` and `target`, i
 
 ## Post-effects
 
-Post-effects[^1] play a key role in dynamic visual rendering, allowing for the interactive blending of various shader effects such as _bloom_, _motion blur_, _ambient occlusion_, and _color grading_, into a rendered scene. A user-space array of `effects` may be sequentially applied to a source `layer` with `applyEffects(layer, effects, [uniforms], [flip])`. Example usage:
+Post-effects[^1] play a key role in dynamic visual rendering, allowing for the interactive blending of various shader effects such as _bloom_, _motion blur_, _ambient occlusion_, and _color grading_, into a rendered scene. A user-space array of `effects` may be sequentially applied to a `source` with `applyEffects(source, effects, [uniforms], [flip])`. Example usage:
 
 ```glsl
 // noise_shader
-uniform sampler2D blender; // <- shared layer should be named 'blender'
+uniform sampler2D blender; // <- shared source should be named 'blender'
 uniform float time;
 ```
 
 ```glsl
 // bloom_shader
-uniform sampler2D blender; // <- shared layer should be named 'blender'
+uniform sampler2D blender; // <- shared source should be named 'blender'
 uniform sampler2D depth;
 ```
 
@@ -238,7 +238,7 @@ function keyPressed() {
 }
 ```
 
-1. `applyEffects(layer, effects, [uniforms = {}], [flip = true])`: Sequentially applies all effects (in the order they were added) to the source [p5.Framebuffer](https://p5js.org/reference/#/p5.Framebuffer) `layer`. The `uniforms` param map shader `keys` to their respective uniform values, formatted as `{ uniform_1_name: value_1, ..., uniform_n_name: value_n }`, provided that a `sampler2D uniform blender` variable is declared in each shader effect as a common layer. The `flip` boolean indicates whether the final image should be vertically flipped. This method processes each effect, applying its shader with the corresponding uniforms (with `applyShader`), and returns the final processed layer, now modified by all effects.
+1. `applyEffects(source, effects, [uniforms = {}], [flip = true])`: Sequentially applies all effects (in the order they were added) to the source, which can be a [p5.Framebuffer](https://p5js.org/reference/p5/p5.Framebuffer), [p5.Image](https://p5js.org/reference/p5/p5.Image/), or [p5.MediaElement](https://p5js.org/reference/p5/p5.MediaElement/). The `uniforms` param maps shader `keys` to their respective uniform values, formatted as `{ uniform_1_name: value_1, ..., uniform_n_name: value_n }`, provided that a `sampler2D uniform blender` variable is declared in each shader effect as a common fbo layer. The `flip` boolean indicates whether the final image should be vertically flipped. This method processes each effect, applying its shader with the corresponding uniforms (using `applyShader`), and returns the final processed source, now modified by all effects.
 2. `createBlender(effects, [options={}])`: [Creates](https://p5js.org/reference/#/p5/createFramebuffer) and attaches an fbo layer with specified `options` to each shader in the `effects` array. If `createBlender` is not called, `applyEffects` automatically generates a blender layer for each shader, utilizing default options.
 3. `removeBlender(effects)`: Removes the individual fbo layers associated with each shader in the `effects` array, freeing up resources by invoking [remove](https://p5js.org/reference/#/p5.Framebuffer/remove).
 

p5.treegl.js

@@ -11,7 +11,7 @@
 var Tree = (function (ext) {
   const INFO = {
     LIBRARY: 'p5.treegl',
-    VERSION: '0.10.1',
+    VERSION: '0.10.2',
     HOMEPAGE: 'https://github.com/VisualComputing/p5.treegl'
   };
   Object.freeze(INFO);
@@ -1193,6 +1193,10 @@ void main() {
     }
   }
 
+  // TODO decide what uniforms applyShader and applyEffect should emit (particularly flip stuff )
+  // Define also uniform and function names (should be the same) to those of uniforms,
+  // resolution may be refactored (as uniform with this name is already taken by p5)
+
   /**
    * Applies a shader to a specified rendering `target`, sets shader `uniforms`, and optionally
    * executes a `scene` function with provided `options`. If no `scene` is specified, a default
@@ -1217,6 +1221,8 @@ void main() {
       }
     });
     const { target, uniforms = {}, scene, options = {} } = config;
+    // TODO decide whether to emit this:
+    //uniforms.uMouse = this.mousePosition(options.flip); // TODO needs fine-tuning
     target instanceof p5.Framebuffer && target.begin();
     const context = target instanceof p5.Graphics ? target : this;
     context === this ? context.push() : context._rendererState = context.push();
@@ -1240,13 +1246,13 @@ void main() {
   }
 
   p5.prototype.applyEffects = function (...args) {
-    let layer;
+    let source;
     let effects;
     let uniformsMapping = {};
     let flip = true;
     args.forEach(arg => {
-      if (arg instanceof p5.Framebuffer) {
-        layer = arg;
+      if (arg instanceof p5.Framebuffer || arg instanceof p5.Image || arg instanceof p5.MediaElement) {
+        source = arg;
       } else if (Array.isArray(arg) && arg.every(e => e instanceof p5.Shader)) {
         effects = arg;
       } else if (typeof arg === 'object' && !Array.isArray(arg)) {
@@ -1255,17 +1261,15 @@ void main() {
         flip = arg;
       }
     });
-    if (!(layer instanceof p5.Framebuffer)) {
-      console.log('The layer param should be a p5.Framebuffer in applyEffects(layer, effects).');
-      return layer;
+    if (!(source instanceof p5.Framebuffer || source instanceof p5.Image || source instanceof p5.MediaElement)) {
+      console.log('The source param should be either a p5.Framebuffer, p5.Image, or p5.MediaElement in applyEffects(source, effects).');
+      return source;
     }
     if (!Array.isArray(effects)) {
-      console.log('The effects param should be an array in applyEffects(layer, effects).');
-      return layer;
+      console.log('The effects param should be an array in applyEffects(source, effects).');
+      return source;
     }
-    let blender = layer;
-    // TODO: a duplicate effect produces: WebGL warning: drawElementsInstance
-    // decide whether to include the avoid duplicates using appliedEffects set fix below
+    let blender = source;
     const appliedEffects = new Set();
     effects.forEach((effect, index) => {
       if (!(effect instanceof p5.Shader)) {
@@ -1293,6 +1297,9 @@ void main() {
       }
       !effect._blender && console.log(`Skipping effect '${index}' due to '${effect.key}' shader missed uniform sampler2D blender variable.`);
       uniforms.blender = blender;
+      // TODO decide whether to emit these:
+      // (source instanceof p5.Image || source instanceof p5.MediaElement) &&
+      // (uniforms.uOffset = this.texOffset(source), uniforms.uResolution = this.resolution());
       blender = this.applyShader(effect, { target: effect.blender, scene: () => this.overlay(flip), uniforms });
     });
     return blender;