Anvil trimming optimization for very large servers

paper-server/patches/features/0001-Moonrise-optimisation-patches.patch

@@ -33977,7 +33977,7 @@ index 8324ad114ddd94b9a99ac1b53332e34a39fe079a..7e205616bec7fc7973b81ca36b8f640a
      private final Long2ObjectLinkedOpenHashMap<CompletableFuture<BitSet>> regionCacheForBlender = new Long2ObjectLinkedOpenHashMap<>();
      private static final int REGION_CACHE_SIZE = 1024;
 diff --git a/net/minecraft/world/level/chunk/storage/RegionFile.java b/net/minecraft/world/level/chunk/storage/RegionFile.java
-index e6f83b493edd89fa104ab2e7ef13e0a2a4d3399f..7767e5420e3d2e569008747523d70147ea8d97fb 100644
+index 794504d3fb303ae164c4b4c56762d19dbcfd06a9..4abf1c4cf58a7e81731bb0e39a5f770d9d1f8f11 100644
 --- a/net/minecraft/world/level/chunk/storage/RegionFile.java
 +++ b/net/minecraft/world/level/chunk/storage/RegionFile.java
 @@ -22,7 +22,7 @@ import net.minecraft.world.level.ChunkPos;
@@ -33989,9 +33989,9 @@ index e6f83b493edd89fa104ab2e7ef13e0a2a4d3399f..7767e5420e3d2e569008747523d70147
      private static final Logger LOGGER = LogUtils.getLogger();
      private static final int SECTOR_BYTES = 4096;
      @VisibleForTesting
-@@ -45,6 +45,21 @@ public class RegionFile implements AutoCloseable {
-     @VisibleForTesting
+@@ -46,6 +46,21 @@ public class RegionFile implements AutoCloseable {
      protected final RegionBitmap usedSectors = new RegionBitmap();
+     private boolean needsTruncation; // Paper - truncate unused region file sectors
 
 +    // Paper start - rewrite chunk system
 +    @Override
@@ -34011,7 +34011,7 @@ index e6f83b493edd89fa104ab2e7ef13e0a2a4d3399f..7767e5420e3d2e569008747523d70147
      public RegionFile(final RegionStorageInfo info, final Path path, final Path externalFileDir, final boolean sync) throws IOException {
          this(info, path, externalFileDir, RegionFileVersion.getSelected(), sync);
      }
-@@ -200,6 +215,15 @@ public class RegionFile implements AutoCloseable {
+@@ -204,6 +219,15 @@ public class RegionFile implements AutoCloseable {
      }
 
      private @Nullable DataInputStream createExternalChunkInputStream(final ChunkPos pos, final byte versionId) throws IOException {
@@ -34027,7 +34027,7 @@ index e6f83b493edd89fa104ab2e7ef13e0a2a4d3399f..7767e5420e3d2e569008747523d70147
          Path externalFile = this.getExternalChunkPath(pos);
          if (!Files.isRegularFile(externalFile)) {
              LOGGER.error("External chunk path {} is not file", externalFile);
-@@ -394,9 +418,28 @@ public class RegionFile implements AutoCloseable {
+@@ -415,9 +439,28 @@ public class RegionFile implements AutoCloseable {
          }
      }
 
@@ -34057,7 +34057,7 @@ index e6f83b493edd89fa104ab2e7ef13e0a2a4d3399f..7767e5420e3d2e569008747523d70147
          public ChunkBuffer(final ChunkPos pos) {
              super(8096);
              super.write(0);
-@@ -413,7 +456,7 @@ public class RegionFile implements AutoCloseable {
+@@ -434,7 +477,7 @@ public class RegionFile implements AutoCloseable {
              int streamLength = this.count - 5 + 1;
              JvmProfiler.INSTANCE.onRegionFileWrite(RegionFile.this.info, this.pos, RegionFile.this.version, streamLength);
              result.putInt(0, streamLength);

paper-server/patches/features/0021-Allow-Saving-of-Oversized-Chunks.patch

@@ -31,18 +31,18 @@ this fix, as the data will remain in the oversized file. Once the server returns
 to a jar with this fix, the data will be restored.
 
 diff --git a/net/minecraft/world/level/chunk/storage/RegionFile.java b/net/minecraft/world/level/chunk/storage/RegionFile.java
-index 7767e5420e3d2e569008747523d70147ea8d97fb..b6c6bf9b20fee63b7e2a37d949bb1e55cef40de6 100644
+index 4abf1c4cf58a7e81731bb0e39a5f770d9d1f8f11..5b8f41d3bdbf16bbeec862f9d031c28b035677c8 100644
 --- a/net/minecraft/world/level/chunk/storage/RegionFile.java
 +++ b/net/minecraft/world/level/chunk/storage/RegionFile.java
-@@ -68,6 +68,7 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+@@ -69,6 +69,7 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
          this.info = info;
          this.path = path;
          this.version = version;
 +        this.initOversizedState(); // Paper
          if (!Files.isDirectory(externalFileDir)) {
              throw new IllegalArgumentException("Expected directory, got " + externalFileDir.toAbsolutePath());
          }
-@@ -463,4 +464,75 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+@@ -484,4 +485,75 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
      private interface CommitOp {
          void run() throws IOException;
      }

paper-server/patches/features/0024-Only-write-chunk-data-to-disk-if-it-serializes-witho.patch

@@ -8,7 +8,7 @@ This ensures at least a valid version of the chunk exists
 on disk, even if outdated
 
 diff --git a/net/minecraft/world/level/chunk/storage/RegionFile.java b/net/minecraft/world/level/chunk/storage/RegionFile.java
-index b6c6bf9b20fee63b7e2a37d949bb1e55cef40de6..3ba442ac6b823e1c6a69398abc4f116da07c32c0 100644
+index 5b8f41d3bdbf16bbeec862f9d031c28b035677c8..4eb206cd724d7546c91af5f5bd3ff6e76fcbd555 100644
 --- a/net/minecraft/world/level/chunk/storage/RegionFile.java
 +++ b/net/minecraft/world/level/chunk/storage/RegionFile.java
 @@ -24,6 +24,7 @@ import org.slf4j.Logger;
@@ -19,7 +19,7 @@ index b6c6bf9b20fee63b7e2a37d949bb1e55cef40de6..3ba442ac6b823e1c6a69398abc4f116d
      private static final int SECTOR_BYTES = 4096;
      @VisibleForTesting
      protected static final int SECTOR_INTS = 1024;
-@@ -451,6 +452,24 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+@@ -472,6 +473,24 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
              this.pos = pos;
          }
 

paper-server/patches/features/0026-Attempt-to-recalculate-regionfile-header-if-it-is-co.patch

@@ -23,7 +23,7 @@ index 27eeb842e6b22bd12b582379eaaa76b622dfca5e..d4606e3d95a98e745cd565c0c23b2a65
                  NO_DATA,
                  HAS_DATA,
 diff --git a/net/minecraft/world/level/chunk/storage/RegionBitmap.java b/net/minecraft/world/level/chunk/storage/RegionBitmap.java
-index a6f9435a8e2a6c8cc05bd3873cb6fc0ec98973b8..d44fd9eea0ccb09c894d6431ede8a4c2aae85c5b 100644
+index 071c66f9c5ce7ea3a0e0fdf4bc76ab5a4e5234e9..3898e62e6a2087c4b3fa7d0e8cecabaca166ddf6 100644
 --- a/net/minecraft/world/level/chunk/storage/RegionBitmap.java
 +++ b/net/minecraft/world/level/chunk/storage/RegionBitmap.java
 @@ -9,6 +9,27 @@ import java.util.BitSet;
@@ -55,12 +55,12 @@ index a6f9435a8e2a6c8cc05bd3873cb6fc0ec98973b8..d44fd9eea0ccb09c894d6431ede8a4c2
          this.used.set(position, position + size);
      }
 diff --git a/net/minecraft/world/level/chunk/storage/RegionFile.java b/net/minecraft/world/level/chunk/storage/RegionFile.java
-index 3ba442ac6b823e1c6a69398abc4f116da07c32c0..3de7fd2b084c38e72d7a6bc416880a881f514ad3 100644
+index 4eb206cd724d7546c91af5f5bd3ff6e76fcbd555..7ac742c57ddfc6b3ed5933e5502b9981d5a6437f 100644
 --- a/net/minecraft/world/level/chunk/storage/RegionFile.java
 +++ b/net/minecraft/world/level/chunk/storage/RegionFile.java
-@@ -46,6 +46,363 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
-     @VisibleForTesting
+@@ -47,6 +47,363 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
      protected final RegionBitmap usedSectors = new RegionBitmap();
+     private boolean needsTruncation; // Paper - truncate unused region file sectors
 
 +    // Paper start - Attempt to recalculate regionfile header if it is corrupt
 +    private static long roundToSectors(long bytes) {
@@ -422,15 +422,15 @@ index 3ba442ac6b823e1c6a69398abc4f116da07c32c0..3de7fd2b084c38e72d7a6bc416880a88
      // Paper start - rewrite chunk system
      @Override
      public final ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.WriteData moonrise$startWrite(final net.minecraft.nbt.CompoundTag data, final ChunkPos pos) throws IOException {
-@@ -75,6 +432,7 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+@@ -76,6 +433,7 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
          }
 
          this.externalFileDir = externalFileDir;
 +        this.canRecalcHeader = info.dfuType()[0] == net.minecraft.util.datafix.DataFixTypes.CHUNK; // Paper - add can recalc flag
          this.offsets = this.header.asIntBuffer();
          this.offsets.limit(1024);
          this.header.position(4096);
-@@ -95,11 +453,13 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+@@ -96,11 +454,13 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
 
              long size = Files.size(path);
 
@@ -447,7 +447,7 @@ index 3ba442ac6b823e1c6a69398abc4f116da07c32c0..3de7fd2b084c38e72d7a6bc416880a88
                      // Spigot start
                      if (numSectors == 255) {
                          // We're maxed out, so we need to read the proper length from the section
-@@ -110,18 +470,62 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+@@ -111,18 +471,62 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
                      // Spigot end
                      if (sectorNumber < 2) {
                          LOGGER.warn("Region file {} has invalid sector at index: {}; sector {} overlaps with header", path, i, sectorNumber);
@@ -512,9 +512,9 @@ index 3ba442ac6b823e1c6a69398abc4f116da07c32c0..3de7fd2b084c38e72d7a6bc416880a88
 +            }
 +            // Paper end
          }
-     }
-
-@@ -130,10 +534,34 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+         // Paper start - truncate unused region file sectors
+         this.needsTruncation = this.file.size() > (long)this.usedSectors.getUsedSectorEnd() * SECTOR_BYTES;
+@@ -134,10 +538,34 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
      }
 
      private Path getExternalChunkPath(final ChunkPos pos) {
@@ -550,7 +550,7 @@ index 3ba442ac6b823e1c6a69398abc4f116da07c32c0..3de7fd2b084c38e72d7a6bc416880a88
      public synchronized @Nullable DataInputStream getChunkDataInputStream(final ChunkPos pos) throws IOException {
          int offset = this.getOffset(pos);
          if (offset == 0) {
-@@ -155,6 +583,11 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+@@ -159,6 +587,11 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
          buffer.flip();
          if (buffer.remaining() < 5) {
              LOGGER.error("Chunk {} header is truncated: expected {} but read {}", pos, sectorsLength, buffer.remaining());
@@ -562,7 +562,7 @@ index 3ba442ac6b823e1c6a69398abc4f116da07c32c0..3de7fd2b084c38e72d7a6bc416880a88
              return null;
          }
 
-@@ -162,6 +595,11 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+@@ -166,6 +599,11 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
          byte versionId = buffer.get();
          if (length == 0) {
              LOGGER.warn("Chunk {} is allocated, but stream is missing", pos);
@@ -574,7 +574,7 @@ index 3ba442ac6b823e1c6a69398abc4f116da07c32c0..3de7fd2b084c38e72d7a6bc416880a88
              return null;
          }
 
-@@ -169,18 +607,45 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+@@ -173,18 +611,45 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
          if (isExternalStreamChunk(versionId)) {
              if (streamLength != 0) {
                  LOGGER.warn("Chunk has both internal and external streams");
@@ -622,7 +622,7 @@ index 3ba442ac6b823e1c6a69398abc4f116da07c32c0..3de7fd2b084c38e72d7a6bc416880a88
          }
      }
 
-@@ -357,9 +822,14 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
+@@ -378,9 +843,14 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche
      }
 
      private ByteBuffer createExternalStub() {

paper-server/patches/sources/net/minecraft/world/level/chunk/storage/RegionBitmap.java.patch

@@ -0,0 +1,15 @@
+--- a/net/minecraft/world/level/chunk/storage/RegionBitmap.java
++++ b/net/minecraft/world/level/chunk/storage/RegionBitmap.java
+@@ -17,6 +_,12 @@
+         this.used.clear(position, position + size);
+     }
+
++    // Paper start - truncate unused region file sectors
++    int getUsedSectorEnd() {
++        return this.used.length();
++    }
++    // Paper end - truncate unused region file sectors
++
+     public int allocate(final int size) {
+         int current = 0;
+

paper-server/patches/sources/net/minecraft/world/level/chunk/storage/RegionFile.java.patch

@@ -1,5 +1,13 @@
 --- a/net/minecraft/world/level/chunk/storage/RegionFile.java
 +++ b/net/minecraft/world/level/chunk/storage/RegionFile.java
+@@ -44,6 +_,7 @@
+     private final IntBuffer timestamps;
+     @VisibleForTesting
+     protected final RegionBitmap usedSectors = new RegionBitmap();
++    private boolean needsTruncation; // Paper - truncate unused region file sectors
+
+     public RegionFile(final RegionStorageInfo info, final Path path, final Path externalFileDir, final boolean sync) throws IOException {
+         this(info, path, externalFileDir, RegionFileVersion.getSelected(), sync);
 @@ -83,6 +_,14 @@
                  if (offset != 0) {
                      int sectorNumber = getSectorNumber(offset);
@@ -15,6 +23,16 @@
                      if (sectorNumber < 2) {
                          LOGGER.warn("Region file {} has invalid sector at index: {}; sector {} overlaps with header", path, i, sectorNumber);
                          this.offsets.put(i, 0);
+@@ -98,6 +_,9 @@
+                 }
+             }
+         }
++        // Paper start - truncate unused region file sectors
++        this.needsTruncation = this.file.size() > (long)this.usedSectors.getUsedSectorEnd() * SECTOR_BYTES;
++        // Paper end - truncate unused region file sectors
+     }
+
+     public Path getPath() {
 @@ -117,6 +_,13 @@
 
          int sectorNumber = getSectorNumber(offset);
@@ -37,6 +55,57 @@
              return false;
          }
      }
+@@ -266,11 +_,28 @@
+         return new DataOutputStream(this.version.wrap(new RegionFile.ChunkBuffer(pos)));
+     }
+
+-    public void flush() throws IOException {
++    // Paper start - truncate unused region file sectors
++    public synchronized void flush() throws IOException {
+         this.file.force(true);
+-    }
+-
+-    public void clear(final ChunkPos pos) throws IOException {
++        if (!this.needsTruncation) {
++            return;
++        }
++        final long usedLength = (long)this.usedSectors.getUsedSectorEnd() * SECTOR_BYTES;
++        if (this.file.size() > usedLength) {
++            this.file.truncate(usedLength);
++            this.file.force(true);
++        }
++        this.needsTruncation = false;
++    }
++
++    private void freeSectors(final int position, final int size) {
++        final int previousEnd = this.usedSectors.getUsedSectorEnd();
++        this.usedSectors.free(position, size);
++        this.needsTruncation |= this.usedSectors.getUsedSectorEnd() < previousEnd;
++    }
++    // Paper end - truncate unused region file sectors
++
++    public synchronized void clear(final ChunkPos pos) throws IOException { // Paper - coordinate tail truncation with writes
+         int offsetIndex = getOffsetIndex(pos);
+         int offset = this.offsets.get(offsetIndex);
+         if (offset != 0) {
+@@ -278,7 +_,7 @@
+             this.timestamps.put(offsetIndex, getTimestamp());
+             this.writeHeader();
+             Files.deleteIfExists(this.getExternalChunkPath(pos));
+-            this.usedSectors.free(getSectorNumber(offset), getNumSectors(offset));
++            this.freeSectors(getSectorNumber(offset), getNumSectors(offset)); // Paper - truncate unused region file sectors
+         }
+     }
+
+@@ -310,7 +_,7 @@
+         this.writeHeader();
+         commitOp.run();
+         if (sectorNumber != 0) {
+-            this.usedSectors.free(sectorNumber, currentSectorCount);
++            this.freeSectors(sectorNumber, currentSectorCount); // Paper - truncate unused region file sectors
+         }
+     }
+
 @@ -328,6 +_,11 @@
          try (FileChannel extFile = FileChannel.open(tmpPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
              data.position(5);
@@ -49,3 +118,18 @@
          }
 
          return () -> Files.move(tmpPath, path, StandardCopyOption.REPLACE_EXISTING);
+@@ -351,12 +_,12 @@
+     }
+
+     @Override
+-    public void close() throws IOException {
++    public synchronized void close() throws IOException { // Paper - coordinate tail truncation with writes
+         try {
+             this.padToFullSector();
+         } finally {
+             try {
+-                this.file.force(true);
++                this.flush(); // Paper - truncate unused region file sectors
+             } finally {
+                 this.file.close();
+             }