feat: More Lights! [flame_3d] (#3250)

More Lights! More fun!

I am still trying to get arrays to work (not even the fancy SSBOs, just
plain fixed arrays).

In the meanwhile this puts lights as separate objects.

This supports:

* point lights
* ambient lights
* colors
* intensity


![image](https://github.com/user-attachments/assets/a2f75a8a-9c64-42d1-bbe5-bdf58fa7df69)

---------

Co-authored-by: Jochum van der Ploeg <jochum@vdploeg.net>

.github/.cspell/people_usernames.txt

@@ -20,3 +20,4 @@ tavian # tavianator.com
 videon # github.com/markvideon
 wolfenrain # github.com/wolfenrain
 xaha # github.com/xvrh
+luan # github.com/luanpotter

packages/flame_3d/example/lib/main.dart

@@ -36,8 +36,43 @@ class ExampleGame3D extends FlameGame<World3D>
   @override
   FutureOr<void> onLoad() async {
     world.addAll([
+      LightComponent.ambient(
+        intensity: 1.0,
+      ),
       RotatingLight(),
 
+      LightComponent.point(
+        position: Vector3(0, 0.1, 0),
+        color: const Color(0xFFFF00FF),
+      ),
+      MeshComponent(
+        mesh: SphereMesh(
+          radius: 0.05,
+          material: SpatialMaterial(
+            albedoTexture: ColorTexture(
+              const Color(0xFFFF00FF),
+            ),
+          ),
+        ),
+        position: Vector3(0, 0.1, 0),
+      ),
+
+      LightComponent.point(
+        position: Vector3(-2, 3, 2),
+        color: const Color(0xFFFF2255),
+      ),
+      MeshComponent(
+        mesh: SphereMesh(
+          radius: 0.05,
+          material: SpatialMaterial(
+            albedoTexture: ColorTexture(
+              const Color(0xFFFF2255),
+            ),
+          ),
+        ),
+        position: Vector3(-2, 4, 2),
+      ),
+
       // Add a player box
       PlayerBox(),
 
@@ -50,7 +85,7 @@ class ExampleGame3D extends FlameGame<World3D>
         mesh: SphereMesh(
           radius: 1,
           material: SpatialMaterial(
-            albedoTexture: ColorTexture(Colors.purple),
+            albedoTexture: ColorTexture(Colors.green),
           ),
         ),
       ),

packages/flame_3d/example/lib/rotating_light.dart

@@ -1,12 +1,15 @@
 import 'dart:math';
+import 'dart:ui';
 
 import 'package:flame_3d/components.dart';
 import 'package:flame_3d/game.dart';
 
 class RotatingLight extends LightComponent {
   RotatingLight()
-      : super.spot(
+      : super.point(
           position: Vector3.zero(),
+          color: const Color(0xFF00FF00),
+          intensity: 20.0,
         );
 
   @override

packages/flame_3d/lib/src/camera/world_3d.dart

@@ -69,8 +69,9 @@ class World3D extends flame.World with flame.HasGameReference {
     image.dispose();
   }
 
+  // TODO(luan): consider making this a fixed-size array later
   void _prepareDevice() {
-    device.lights = lights;
+    device.lightingInfo.lights = lights;
   }
 
   // TODO(wolfenrain): this is only here for testing purposes

packages/flame_3d/lib/src/components/light_component.dart

@@ -1,3 +1,5 @@
+import 'dart:ui';
+
 import 'package:flame_3d/camera.dart';
 import 'package:flame_3d/components.dart';
 import 'package:flame_3d/game.dart';
@@ -10,13 +12,28 @@ class LightComponent extends Component3D {
     super.position,
   });
 
-  LightComponent.spot({
+  LightComponent.point({
     Vector3? position,
+    Color color = const Color(0xFFFFFFFF),
+    double intensity = 1.0,
   }) : this(
-          source: SpotLight(),
+          source: PointLight(
+            color: color,
+            intensity: intensity,
+          ),
           position: position,
         );
 
+  LightComponent.ambient({
+    Color color = const Color(0xFFFFFFFF),
+    double intensity = 0.2,
+  }) : this(
+          source: AmbientLight(
+            color: color,
+            intensity: intensity,
+          ),
+        );
+
   final LightSource source;
 
   late final Light _light = Light(

packages/flame_3d/lib/src/extensions/color.dart

@@ -3,6 +3,10 @@ import 'dart:ui';
 
 extension ColorExtension on Color {
   /// Returns a Float32List that represents the color as a vector.
-  Float32List get storage =>
-      Float32List.fromList([red / 255, green / 255, blue / 255, opacity]);
+  Float32List get storage => Float32List.fromList([
+        opacity,
+        red.toDouble() / 255,
+        green.toDouble() / 255,
+        blue.toDouble() / 255,
+      ]);
 }

packages/flame_3d/lib/src/graphics/graphics_device.dart

@@ -50,7 +50,7 @@ class GraphicsDevice {
 
   /// Must be set by the rendering pipeline before elements are bound.
   /// Can be accessed by elements in their bind method.
-  Iterable<Light> lights = [];
+  final LightingInfo lightingInfo = LightingInfo();
 
   /// Begin a new rendering batch.
   ///

packages/flame_3d/lib/src/resources/light.dart

@@ -1,3 +1,5 @@
+export 'light/ambient_light.dart';
 export 'light/light.dart';
 export 'light/light_source.dart';
-export 'light/spot_light.dart';
+export 'light/lighting_info.dart';
+export 'light/point_light.dart';

packages/flame_3d/lib/src/resources/light/ambient_light.dart

@@ -0,0 +1,15 @@
+import 'dart:ui' show Color;
+
+import 'package:flame_3d/resources.dart';
+
+class AmbientLight extends LightSource {
+  AmbientLight({
+    super.color = const Color(0xFFFFFFFF),
+    super.intensity = 0.2,
+  });
+
+  void apply(Shader shader) {
+    shader.setColor('AmbientLight.color', color);
+    shader.setFloat('AmbientLight.intensity', intensity);
+  }
+}

packages/flame_3d/lib/src/resources/light/light.dart

@@ -19,11 +19,9 @@ class Light extends Resource<void> {
     required this.source,
   }) : super(null);
 
-  void apply(Shader shader) {
-    shader.setVector3('Light.position', transform.position);
-    // apply additional parameters
-    source.apply(shader);
+  void apply(int index, Shader shader) {
+    shader.setVector3('Light$index.position', transform.position);
+    shader.setColor('Light$index.color', source.color);
+    shader.setFloat('Light$index.intensity', source.intensity);
   }
-
-  static UniformSlot shaderSlot = UniformSlot.value('Light', {'position'});
 }

packages/flame_3d/lib/src/resources/light/light_source.dart

@@ -1,8 +1,16 @@
+import 'dart:ui' show Color;
+
 import 'package:flame_3d/resources.dart';
 
 /// Describes the properties of a light source.
-/// There are three types of light sources: directional, point, and spot.
-/// Currently only [SpotLight] is implemented.
+/// There are three types of light sources: point, directional, and spot.
+/// Currently only [PointLight] is implemented.
 abstract class LightSource {
-  void apply(Shader shader);
+  final Color color;
+  final double intensity;
+
+  LightSource({
+    required this.color,
+    required this.intensity,
+  });
 }

packages/flame_3d/lib/src/resources/light/lighting_info.dart

@@ -0,0 +1,48 @@
+import 'package:flame_3d/resources.dart';
+
+class LightingInfo {
+  Iterable<Light> lights = [];
+
+  void apply(Shader shader) {
+    _applyAmbientLight(shader);
+    _applyPointLights(shader);
+  }
+
+  void _applyAmbientLight(Shader shader) {
+    final ambient = _extractAmbientLight(lights);
+    ambient.apply(shader);
+  }
+
+  void _applyPointLights(Shader shader) {
+    final pointLights = lights.where((e) => e.source is PointLight);
+    final numLights = pointLights.length;
+    if (numLights > 3) {
+      // temporary, until we support dynamic arrays
+      throw Exception('At most 3 point lights are allowed');
+    }
+
+    shader.setUint('LightsInfo.numLights', numLights);
+    for (final (idx, light) in pointLights.indexed) {
+      light.apply(idx, shader);
+    }
+  }
+
+  AmbientLight _extractAmbientLight(Iterable<Light> lights) {
+    final ambient = lights.where((e) => e.source is AmbientLight);
+    if (ambient.isEmpty) {
+      return AmbientLight();
+    }
+    if (ambient.length > 1) {
+      throw Exception('At most one ambient light is allowed');
+    }
+    return ambient.first.source as AmbientLight;
+  }
+
+  static List<UniformSlot> shaderSlots = [
+    UniformSlot.value('AmbientLight', {'color', 'intensity'}),
+    UniformSlot.value('LightsInfo', {'numLights'}),
+    UniformSlot.value('Light0', {'position', 'color', 'intensity'}),
+    UniformSlot.value('Light1', {'position', 'color', 'intensity'}),
+    UniformSlot.value('Light2', {'position', 'color', 'intensity'}),
+  ];
+}

packages/flame_3d/lib/src/resources/light/point_light.dart

@@ -0,0 +1,9 @@
+import 'package:flame_3d/resources.dart';
+
+/// A point light that emits light in all directions equally.
+class PointLight extends LightSource {
+  PointLight({
+    required super.color,
+    required super.intensity,
+  });
+}

packages/flame_3d/lib/src/resources/light/spot_light.dart

@@ -1,11 +0,0 @@
-import 'package:flame_3d/resources.dart';
-
-/// A point light that emits light in all directions equally.
-class SpotLight extends LightSource {
-  // TODO(luanpotter): add color, intensity, etc
-
-  @override
-  void apply(Shader shader) {
-    //
-  }
-}

packages/flame_3d/lib/src/resources/material/spatial_material.dart

@@ -9,9 +9,8 @@ class SpatialMaterial extends Material {
   SpatialMaterial({
     Texture? albedoTexture,
     Color albedoColor = const Color(0xFFFFFFFF),
-    this.metallic = 0,
-    this.metallicSpecular = 0.5,
-    this.roughness = 1.0,
+    this.metallic = 0.8,
+    this.roughness = 0.6,
   })  : albedoTexture = albedoTexture ?? Texture.standard,
         super(
           vertexShader: Shader(
@@ -27,10 +26,9 @@ class SpatialMaterial extends Material {
               UniformSlot.value('Material', {
                 'albedoColor',
                 'metallic',
-                'metallicSpecular',
                 'roughness',
               }),
-              Light.shaderSlot,
+              ...LightingInfo.shaderSlots,
               UniformSlot.value('Camera', {'position'}),
             ],
           ),
@@ -53,8 +51,6 @@ class SpatialMaterial extends Material {
 
   double metallic;
 
-  double metallicSpecular;
-
   double roughness;
 
   @override
@@ -77,7 +73,6 @@ class SpatialMaterial extends Material {
       ..setTexture('albedoTexture', albedoTexture)
       ..setVector3('Material.albedoColor', _albedoCache)
       ..setFloat('Material.metallic', metallic)
-      ..setFloat('Material.metallicSpecular', metallicSpecular)
       ..setFloat('Material.roughness', roughness);
   }
 
@@ -88,11 +83,7 @@ class SpatialMaterial extends Material {
   }
 
   void _applyLights(GraphicsDevice device) {
-    final light = device.lights.firstOrNull;
-    if (light == null) {
-      return;
-    }
-    light.apply(fragmentShader);
+    device.lightingInfo.apply(fragmentShader);
   }
 
   static final _library = gpu.ShaderLibrary.fromAsset(

packages/flame_3d/lib/src/resources/mesh/vertex.dart

@@ -24,9 +24,8 @@ class Vertex {
         _storage = Float32List.fromList([
           ...position.storage, // 1, 2, 3
           ...texCoord.storage, // 4, 5
-          ...color.storage, // 6,7,8
-          // TODO(wolfenrain): fix normals not working properly
-          ...(normal ?? Vector3.zero()).storage, // 9, 10, 11
+          ...color.storage, // 6, 7, 8, 9
+          ...(normal ?? Vector3.zero()).storage, // 10, 11, 12
         ]);
 
   Float32List get storage => _storage;

packages/flame_3d/lib/src/resources/shader/shader.dart

@@ -1,4 +1,6 @@
 import 'dart:collection';
+import 'dart:typed_data';
+import 'dart:ui';
 
 import 'package:flame_3d/game.dart';
 import 'package:flame_3d/graphics.dart';
@@ -36,8 +38,15 @@ class Shader extends Resource<gpu.Shader> {
   /// Set a [Vector4] at the given [key] on the buffer.
   void setVector4(String key, Vector4 vector) => _setValue(key, vector.storage);
 
+  /// Set an [int] (encoded as uint) at the given [key] on the buffer.
+  void setUint(String key, int value) {
+    _setValue(key, _encodeUint32(value, Endian.little));
+  }
+
   /// Set a [double] at the given [key] on the buffer.
-  void setFloat(String key, double value) => _setValue(key, [value]);
+  void setFloat(String key, double value) {
+    _setValue(key, [value]);
+  }
 
   /// Set a [Matrix2] at the given [key] on the buffer.
   void setMatrix2(String key, Matrix2 matrix) => _setValue(key, matrix.storage);
@@ -48,6 +57,8 @@ class Shader extends Resource<gpu.Shader> {
   /// Set a [Matrix4] at the given [key] on the buffer.
   void setMatrix4(String key, Matrix4 matrix) => _setValue(key, matrix.storage);
 
+  void setColor(String key, Color color) => _setValue(key, color.storage);
+
   void bind(GraphicsDevice device) {
     for (final slot in _slots) {
       _instances[slot.name]?.bind(device);
@@ -91,4 +102,8 @@ class Shader extends Resource<gpu.Shader> {
 
     return (_instances[keys.first], keys.elementAtOrNull(1)) as (T, String?);
   }
+
+  static Float32List _encodeUint32(int value, Endian endian) {
+    return (ByteData(16)..setUint32(0, value, endian)).buffer.asFloat32List();
+  }
 }

packages/flame_3d/shaders/spatial_material.frag

@@ -1,5 +1,11 @@
 #version 460 core
 
+// implementation based on https://learnopengl.com/PBR/Lighting
+
+// #define NUM_LIGHTS 8
+#define PI 3.14159265359
+#define EPSILON 0.0001
+
 in vec2 fragTexCoord;
 in vec4 fragColor;
 in vec3 fragPosition;
@@ -9,48 +15,169 @@ out vec4 outColor;
 
 uniform sampler2D albedoTexture;  // Albedo texture
 
+// material info
+
 uniform Material {
   vec3 albedoColor;
   float metallic;
-  float metallicSpecular;
   float roughness;
 } material;
 
-uniform Light {
+// light info
+
+uniform AmbientLight {
+  vec3 color;
+  float intensity;
+} ambientLight;
+
+uniform LightsInfo {
+  uint numLights;
+} lightsInfo;
+
+// uniform Light {
+//   vec3 position;
+//   vec3 color;
+//   float intensity;
+// } lights[NUM_LIGHTS];
+
+uniform Light0 {
   vec3 position;
-} light;
+  vec3 color;
+  float intensity;
+} light0;
+
+uniform Light1 {
+  vec3 position;
+  vec3 color;
+  float intensity;
+} light1;
+
+uniform Light2 {
+  vec3 position;
+  vec3 color;
+  float intensity;
+} light2;
+
+// camera info
 
 uniform Camera {
   vec3 position;
 } camera;
 
-// Schlick GGX function
-float SchlickGGX(float NdotV, float roughness)
-{
-    float k = (roughness * roughness) / 2.0;
-    float nom = NdotV;
-    float denom = NdotV * (1.0 - k) + k;
-    return nom / denom;
+vec3 fresnelSchlick(float cosTheta, vec3 f0) {
+  return f0 + (1.0 - f0) * pow(clamp(1.0 - cosTheta, 0.0, 1.0), 5.0);
+}
+
+float distributionGGX(vec3 normal, vec3 halfwayDir, float roughness) {
+  float a = roughness * roughness;
+  float a2 = a * a;
+  float num = a2;
+
+  float NdotH = max(dot(normal, halfwayDir), 0.0);
+  float NdotH2 = NdotH * NdotH;
+  float b = (NdotH2 * (a2 - 1.0) + 1.0);
+  float denom = PI * b * b;
+
+  return num / denom;
+}
+
+float geometrySchlickGGX(float NdotV, float roughness) {
+  float r = (roughness + 1.0);
+  float k = (r * r) / 8.0;
+
+  float num = NdotV;
+  float denom = NdotV * (1.0 - k) + k;
+
+  return num / denom;
+}
+
+float geometrySmith(vec3 normal, vec3 viewDir, vec3 lightDir, float roughness) {
+  float NdotV = max(dot(normal, viewDir), 0.0);
+  float NdotL = max(dot(normal, lightDir), 0.0);
+  float ggx2 = geometrySchlickGGX(NdotV, roughness);
+  float ggx1 = geometrySchlickGGX(NdotL, roughness);
+
+  return ggx1 * ggx2;
+}
+
+vec3 processLight(
+  vec3 lightPos,
+  vec3 lightColor,
+  float lightIntensity,
+  vec3 baseColor,
+  vec3 normal,
+  vec3 viewDir,
+  vec3 diffuse
+) {
+  vec3 lightDirVec = lightPos - fragPosition;
+  vec3 lightDir = normalize(lightDirVec);
+  float distance = length(lightDirVec) + EPSILON;
+  vec3 halfwayDir = normalize(viewDir + lightDir);
+
+  float attenuation = lightIntensity / (distance * distance);
+  vec3 radiance = lightColor * attenuation;
+
+  // cook-torrance brdf
+  float ndf = distributionGGX(normal, halfwayDir, material.roughness);
+  float g = geometrySmith(normal, viewDir, lightDir, material.roughness);
+  vec3 f = fresnelSchlick(max(dot(halfwayDir, viewDir), 0.0), diffuse);
+
+  vec3 kS = f; // reflection/specular fraction
+  vec3 kD = (vec3(1.0) - kS) * (1.0 - material.metallic); // refraction/diffuse fraction
+
+  vec3 numerator = ndf * g * f;
+  float denominator = 4.0 * max(dot(normal, viewDir), 0.0) * max(dot(normal, lightDir), 0.0) + EPSILON;
+  vec3 specular = numerator / denominator;
+
+  // add to outgoing radiance Lo
+  float NdotL = max(dot(normal, lightDir), 0.0);
+  return (kD * baseColor / PI + specular) * radiance * NdotL;
 }
 
 void main() {
-    vec3 viewDir = normalize(camera.position - fragPosition);
-    vec3 lightDir = normalize(light.position - fragPosition);
-    vec3 halfwayDir = normalize(viewDir + lightDir);
+  vec3 normal = normalize(fragNormal);
+  vec3 viewDir = normalize(camera.position - fragPosition);
 
-    vec3 normal = normalize(fragNormal);
-    float NdotV = max(dot(normal, viewDir), 0.0);
-    float fresnel = SchlickGGX(NdotV, material.roughness);
+  vec3 baseColor = material.albedoColor;
+  baseColor *= texture(albedoTexture, fragTexCoord).rgb;
 
-    float NdotL = max(dot(normal, lightDir), 0.0);
-    float NdotH = max(dot(normal, halfwayDir), 0.0);
-    float specular = SchlickGGX(NdotL, material.roughness) * SchlickGGX(NdotH, material.roughness);
+  vec3 baseAmbient = vec3(0.03) * baseColor * ambientLight.color * ambientLight.intensity;
+  vec3 ao = vec3(1.0); // white - no ambient occlusion for now
+  vec3 ambient = baseAmbient * baseColor * ao;
 
-    vec3 baseColor = material.albedoColor;
-    baseColor *= texture(albedoTexture, fragTexCoord).rgb;
+  vec3 f0 = vec3(0.04);
+  vec3 diffuse = mix(f0, baseColor, material.metallic);
 
-    vec3 diffuse = mix(baseColor, vec3(0.04, 0.04, 0.04), material.metallic);
-    vec3 finalColor = (diffuse + specular * material.metallicSpecular) * NdotL * fresnel;
+  vec3 lo = vec3(0.0);
 
-    outColor = vec4(finalColor, 1.0);
+  if (lightsInfo.numLights > 0) {
+    vec3 light0Pos = light0.position;
+    vec3 light0Color = light0.color;
+    float light0Intensity = light0.intensity;
+
+    lo += processLight(light0Pos, light0Color, light0Intensity, baseColor, normal, viewDir, diffuse);
+  }
+
+  if (lightsInfo.numLights > 1) {
+    vec3 light1Pos = light1.position;
+    vec3 light1Color = light1.color;
+    float light1Intensity = light1.intensity;
+
+    lo += processLight(light1Pos, light1Color, light1Intensity, baseColor, normal, viewDir, diffuse);
+  }
+
+  if (lightsInfo.numLights > 2) {
+    vec3 light2Pos = light2.position;
+    vec3 light2Color = light2.color;
+    float light2Intensity = light2.intensity;
+
+    lo += processLight(light2Pos, light2Color, light2Intensity, baseColor, normal, viewDir, diffuse);
+  }
+
+  vec3 color = ambient + lo;
+
+  color = color / (color + vec3(1.0));
+  color = pow(color, vec3(1.0 / 2.2));
+
+  outColor = vec4(color, 1.0);
 }