diff --git a/packages/flame/lib/src/collisions/broadphase/sweep/sweep.dart b/packages/flame/lib/src/collisions/broadphase/sweep/sweep.dart
index f79cc6e33e0..42aabb4bb81 100644
--- a/packages/flame/lib/src/collisions/broadphase/sweep/sweep.dart
+++ b/packages/flame/lib/src/collisions/broadphase/sweep/sweep.dart
@@ -1,3 +1,4 @@
+import 'package:collection/collection.dart';
 import 'package:flame/collisions.dart';
 
 class Sweep<T extends Hitbox<T>> extends Broadphase<T> {
@@ -17,9 +18,17 @@ class Sweep<T extends Hitbox<T>> extends Broadphase<T> {
 
   @override
   void update() {
-    items.sort((a, b) => a.aabb.min.x.compareTo(b.aabb.min.x));
+    // Between two ticks the hitboxes only move a little, so [items] is
+    // always nearly sorted: an insertion sort runs in close to linear time
+    // here, where a general-purpose sort would pay its full O(n log n) on
+    // every tick. The comparator is a static tear-off, so nothing is
+    // allocated per tick.
+    insertionSort(items, compare: _compareMinX);
   }
 
+  static int _compareMinX(Hitbox<dynamic> a, Hitbox<dynamic> b) =>
+      a.aabb.min.x.compareTo(b.aabb.min.x);
+
   @override
   Iterable<CollisionProspect<T>> query() {
     _active.clear();
@@ -44,7 +53,10 @@ class Sweep<T extends Hitbox<T>> extends Broadphase<T> {
             _prospectPool.acquire(item, activeItem);
           }
         } else {
-          _active.remove(activeItem);
+          // The order of the active list does not matter, so the removal can
+          // swap in the last element instead of searching and shifting.
+          _active[i] = _active.last;
+          _active.removeLast();
         }
       }
       _active.add(item);