feat: Support skeletal animation basics [flame_3d] (#3291)

Support skeletal animation basics.

Sadly we cannot support arrays yet - see [this
PR](#3282).

Author: luanpotter

packages/flame_3d/assets/shaders/spatial_material.shaderbundle

@@ -3,6 +3,7 @@ import 'dart:ui';
 
 import 'package:flame_3d/game.dart';
 import 'package:flame_3d/resources.dart';
+import 'package:flame_3d/src/graphics/joints_info.dart';
 import 'package:flutter_gpu/gpu.dart' as gpu;
 
 enum BlendState {
@@ -48,6 +49,10 @@ class GraphicsDevice {
 
   Size _previousSize = Size.zero;
 
+  /// Must be set by the rendering pipeline before elements are bound.
+  /// Can be accessed by elements in their bind method.
+  final JointsInfo jointsInfo = JointsInfo();
+
   /// Must be set by the rendering pipeline before elements are bound.
   /// Can be accessed by elements in their bind method.
   final LightingInfo lightingInfo = LightingInfo();

packages/flame_3d/lib/src/graphics/graphics_device.dart

@@ -0,0 +1,13 @@
+import 'package:flame_3d/core.dart';
+
+class JointsInfo {
+  /// Joints per surface idx
+  Map<int, List<Matrix4>> jointTransformsPerSurface = {};
+
+  /// Joints for the current surface
+  List<Matrix4> jointTransforms = [];
+
+  void setSurface(int surfaceIdx) {
+    jointTransforms = jointTransformsPerSurface[surfaceIdx] ?? [];
+  }
+}

packages/flame_3d/lib/src/graphics/joints_info.dart

@@ -16,7 +16,15 @@ class SpatialMaterial extends Material {
           vertexShader: Shader(
             _library['TextureVertex']!,
             slots: [
-              UniformSlot.value('VertexInfo', {'model', 'view', 'projection'}),
+              UniformSlot.value('VertexInfo', {
+                'model',
+                'view',
+                'projection',
+              }),
+              UniformSlot.value(
+                'JointMatrices',
+                List.generate(_maxJoints, (idx) => 'joint$idx').toSet(),
+              ),
             ],
           ),
           fragmentShader: Shader(
@@ -56,6 +64,7 @@ class SpatialMaterial extends Material {
   @override
   void bind(GraphicsDevice device) {
     _bindVertexInfo(device);
+    _bindJointMatrices(device);
     _bindMaterial(device);
     _bindCamera(device);
   }
@@ -67,6 +76,19 @@ class SpatialMaterial extends Material {
       ..setMatrix4('VertexInfo.projection', device.projection);
   }
 
+  void _bindJointMatrices(GraphicsDevice device) {
+    final jointTransforms = device.jointsInfo.jointTransforms;
+    if (jointTransforms.length > _maxJoints) {
+      throw Exception(
+        'At most $_maxJoints joints per surface are supported;'
+        ' found ${jointTransforms.length}',
+      );
+    }
+    for (final (idx, transform) in jointTransforms.indexed) {
+      vertexShader.setMatrix4('JointMatrices.joint$idx', transform);
+    }
+  }
+
   void _bindMaterial(GraphicsDevice device) {
     _applyLights(device);
     fragmentShader
@@ -89,4 +111,6 @@ class SpatialMaterial extends Material {
   static final _library = gpu.ShaderLibrary.fromAsset(
     'packages/flame_3d/assets/shaders/spatial_material.shaderbundle',
   )!;
+
+  static const _maxJoints = 16;
 }

packages/flame_3d/lib/src/resources/material/spatial_material.dart

@@ -26,7 +26,8 @@ class Mesh extends Resource<void> {
   int get vertexCount => _surfaces.fold(0, (p, e) => p + e.vertexCount);
 
   void bind(GraphicsDevice device) {
-    for (final surface in _surfaces) {
+    for (final (idx, surface) in _surfaces.indexed) {
+      device.jointsInfo.setSurface(idx);
       device.bindSurface(surface);
     }
   }

packages/flame_3d/lib/src/resources/mesh/mesh.dart

@@ -18,6 +18,7 @@ class Surface extends Resource<gpu.DeviceBuffer?> {
     required List<Vertex> vertices,
     required List<int> indices,
     this.material,
+    this.jointMap,
     /**
      * If `true`, the normals will be calculated if they are not provided.
      */
@@ -34,15 +35,16 @@ class Surface extends Resource<gpu.DeviceBuffer?> {
     _vertices = Float32List.fromList(
       normalizedVertices.fold([], (p, v) => p..addAll(v.storage)),
     ).buffer;
-    _vertexCount = _vertices.lengthInBytes ~/ (normalizedVertices.length * 9);
+    _vertexCount = normalizedVertices.length;
 
     _indices = Uint16List.fromList(indices).buffer;
-    _indexCount = _indices.lengthInBytes ~/ 2;
+    _indexCount = indices.length;
 
     _calculateAabb(normalizedVertices);
   }
 
   Material? material;
+  Map<int, int>? jointMap;
 
   Aabb3 get aabb => _aabb;
   late Aabb3 _aabb;

packages/flame_3d/lib/src/resources/mesh/surface.dart

@@ -18,14 +18,20 @@ class Vertex {
     required Vector2 texCoord,
     this.color = const Color(0xFFFFFFFF),
     Vector3? normal,
+    Vector4? joints,
+    Vector4? weights,
   })  : position = position.immutable,
         texCoord = texCoord.immutable,
         normal = normal?.immutable,
+        joints = joints?.immutable,
+        weights = weights?.immutable,
         _storage = Float32List.fromList([
           ...position.storage, // 1, 2, 3
           ...texCoord.storage, // 4, 5
           ...color.storage, // 6, 7, 8, 9
           ...(normal ?? Vector3.zero()).storage, // 10, 11, 12
+          ...(joints ?? Vector4.zero()).storage, // 13, 14, 15, 16
+          ...(weights ?? Vector4.zero()).storage, // 17, 18, 19, 20
         ]);
 
   Float32List get storage => _storage;
@@ -40,6 +46,12 @@ class Vertex {
   /// The normal vector of the vertex.
   final ImmutableVector3? normal;
 
+  /// The joints of the vertex.
+  final ImmutableVector4? joints;
+
+  /// The weights of the vertex.
+  final ImmutableVector4? weights;
+
   /// The color on the vertex.
   final Color color;
 
@@ -49,23 +61,36 @@ class Vertex {
       position == other.position &&
       texCoord == other.texCoord &&
       normal == other.normal &&
-      color == other.color;
+      color == other.color &&
+      joints == other.joints &&
+      weights == other.weights;
 
   @override
-  int get hashCode => Object.hashAll([position, texCoord, normal, color]);
+  int get hashCode => Object.hashAll([
+        position,
+        texCoord,
+        normal,
+        color,
+        joints,
+        weights,
+      ]);
 
   Vertex copyWith({
     Vector3? position,
     Vector2? texCoord,
     Vector3? normal,
     Color? color,
+    Vector4? joints,
+    Vector4? weights,
   }) {
     // TODO(wolfenrain): optimize this.
     return Vertex(
       position: position ?? this.position.mutable,
       texCoord: texCoord ?? this.texCoord.mutable,
       normal: normal ?? this.normal?.mutable,
       color: color ?? this.color,
+      joints: joints ?? this.joints?.mutable,
+      weights: weights ?? this.weights?.mutable,
     );
   }
 

packages/flame_3d/lib/src/resources/mesh/vertex.dart

@@ -3,6 +3,7 @@ import 'dart:typed_data';
 
 import 'package:flame_3d/graphics.dart';
 import 'package:flame_3d/resources.dart';
+import 'package:ordered_set/comparing.dart';
 
 /// {@template uniform_value}
 /// Instance of a uniform value. Represented by a [ByteBuffer].
@@ -21,7 +22,9 @@ class UniformValue extends UniformInstance<ByteBuffer> {
     if (super.resource == null) {
       var previousIndex = -1;
 
-      final data = _storage.entries.fold<List<double>>([], (p, e) {
+      final entries = _storage.entries.toList()
+        ..sort(Comparing.on((c) => c.key));
+      final data = entries.fold<List<double>>([], (p, e) {
         if (previousIndex + 1 != e.key) {
           final field =
               slot.fields.indexed.firstWhere((e) => e.$1 == previousIndex + 1);

packages/flame_3d/lib/src/resources/shader/uniform_value.dart

@@ -18,6 +18,7 @@ dependencies:
   flutter_gpu:
     sdk: flutter
   meta: ^1.12.0
+  ordered_set: ^6.0.1
   vector_math: ^2.1.4
 
 dev_dependencies:

packages/flame_3d/pubspec.yaml

@@ -4,6 +4,8 @@ in vec3 vertexPosition;
 in vec2 vertexTexCoord;
 in vec4 vertexColor;
 in vec3 vertexNormal;
+in vec4 vertexJoints;
+in vec4 vertexWeights;
 
 out vec2 fragTexCoord;
 out vec4 fragColor;
@@ -16,18 +18,91 @@ uniform VertexInfo {
   mat4 projection;
 } vertex_info;
 
+uniform JointMatrices {
+  mat4 joint0;
+  mat4 joint1;
+  mat4 joint2;
+  mat4 joint3;
+  mat4 joint4;
+  mat4 joint5;
+  mat4 joint6;
+  mat4 joint7;
+  mat4 joint8;
+  mat4 joint9;
+  mat4 joint10;
+  mat4 joint11;
+  mat4 joint12;
+  mat4 joint13;
+  mat4 joint14;
+  mat4 joint15;
+} joints;
+
+mat4 jointMat(float jointIndex) {
+  if (jointIndex == 0.0) {
+    return joints.joint0;
+  } else if (jointIndex == 1.0) {
+    return joints.joint1;
+  } else if (jointIndex == 2.0) {
+    return joints.joint2;
+  } else if (jointIndex == 3.0) {
+    return joints.joint3;
+  } else if (jointIndex == 4.0) {
+    return joints.joint4;
+  } else if (jointIndex == 5.0) {
+    return joints.joint5;
+  } else if (jointIndex == 6.0) {
+    return joints.joint6;
+  } else if (jointIndex == 7.0) {
+    return joints.joint7;
+  } else if (jointIndex == 8.0) {
+    return joints.joint8;
+  } else if (jointIndex == 9.0) {
+    return joints.joint9;
+  } else if (jointIndex == 10.0) {
+    return joints.joint10;
+  } else if (jointIndex == 11.0) {
+    return joints.joint11;
+  } else if (jointIndex == 12.0) {
+    return joints.joint12;
+  } else if (jointIndex == 13.0) {
+    return joints.joint13;
+  } else if (jointIndex == 14.0) {
+    return joints.joint14;
+  } else if (jointIndex == 15.0) {
+    return joints.joint15;
+  } else {
+    return mat4(0.0);
+  }
+}
+
+mat4 computeSkinMatrix() {
+  if (vertexWeights.x == 0.0 && vertexWeights.y == 0.0 && vertexWeights.z == 0.0 && vertexWeights.w == 0.0) {
+    // no weights, skip skinning
+    return mat4(1.0);
+  }
+
+  return vertexWeights.x * jointMat(vertexJoints.x) +
+    vertexWeights.y * jointMat(vertexJoints.y) +
+    vertexWeights.z * jointMat(vertexJoints.z) +
+    vertexWeights.w * jointMat(vertexJoints.w);
+}
+
 void main() {
+  mat4 skinMatrix = computeSkinMatrix();
+  vec3 position = (skinMatrix * vec4(vertexPosition, 1.0)).xyz;
+  vec3 normal = normalize((skinMatrix * vec4(vertexNormal, 0.0)).xyz);
+
   // Calculate the modelview projection matrix
   mat4 modelViewProjection = vertex_info.projection * vertex_info.view * vertex_info.model;
 
   // Transform the vertex position
-  gl_Position = modelViewProjection * vec4(vertexPosition, 1.0);
+  gl_Position = modelViewProjection * vec4(position, 1.0);
 
   // Pass the interpolated values to the fragment shader
   fragTexCoord = vertexTexCoord;
   fragColor = vertexColor;
 
   // Calculate the world-space position and normal
-  fragPosition = vec3(vertex_info.model * vec4(vertexPosition, 1.0));
-  fragNormal = mat3(transpose(inverse(vertex_info.model))) * vertexNormal;
+  fragPosition = vec3(vertex_info.model * vec4(position, 1.0));
+  fragNormal = mat3(transpose(inverse(vertex_info.model))) * normal;
 }