[file-index] Support V2 file index container format

docs/docs/concepts/spec/fileindex.mdx

@@ -37,15 +37,21 @@ deletion vectors, see [Table Index](./tableindex).
 
 | Encoding | Section |
 | --- | --- |
-| Shared header and column offsets | [Index File](#index-file) |
+| File index container | [Index File](#index-file) |
 | Bloom filter | [BloomFilter](#index-bloomfilter) |
 | Bitmap | [Bitmap](#index-bitmap) |
 | Range bitmap | [Range Bitmap](#index-range-bitmap) |
 | Bit-slice bitmap | [Bit-Slice Index Bitmap](#index-bit-slice-index-bitmap) |
 
 ## Index File
 
-File index file format. Put all column and offset in the header.
+The container version is selected by `file-index.format.version` (default: `1`). Readers support
+both versions. Enable version `2` after upgrading the readers of the table. The container
+version is independent of each index type's payload version.
+
+### V1
+
+V1 puts the index entries in the header before the payloads.
 
 <pre>
  ______________________________________    _____________________
@@ -95,6 +101,64 @@ redundant bytes:                  var bytes (for compatibility with later versio
 BODY:                             column index bytes + column index bytes + column index bytes + .......
 </pre>
 
+### V2
+
+V2 stores the byte-array payloads first, followed by the footer containing the index entries and a
+fixed 12-byte trailer.
+All integers are signed and big-endian; strings use Java `writeUTF` encoding.
+
+<pre>
+ ______________________________________    _____________________
+|     magic    |        version        |          PREFIX
+|--------------------------------------|    ---------------------
+|                BODY                  |
+|                BODY                  |           BODY
+|                BODY                  |
+|--------------------------------------|    ---------------------
+|            column number             |
+|--------------------------------------|
+|   column 1        |  index number    |
+|--------------------------------------|
+|  index name 1 | start pos | length   |
+|--------------------------------------|
+|  index name 2 | start pos | length   |
+|--------------------------------------|
+|   column 2        |  index number    |
+|--------------------------------------|
+|  index name 1 | start pos | length   |
+|--------------------------------------|
+|  index name 2 | start pos | length   |
+|--------------------------------------|
+|                 ...                  |          FOOTER
+|--------------------------------------|
+|                 ...                  |
+|--------------------------------------|    ---------------------
+| footer length |      tail magic      |          TRAILER
+|______________________________________|    _____________________
+
+magic:                            8 bytes long, value is 1493475289347502L, BIG_ENDIAN
+version:                          4 bytes int, BIG_ENDIAN
+BODY:                             column index bytes + column index bytes + .......
+column number:                    4 bytes int, BIG_ENDIAN
+column x name:                    var bytes, Java modified-utf-8
+index number:                     4 bytes int (how many column items below), BIG_ENDIAN
+index name x:                     var bytes, Java modified-utf-8
+start pos:                        8 bytes long, BIG_ENDIAN
+length:                           8 bytes long, BIG_ENDIAN
+footer length:                    4 bytes int, BIG_ENDIAN
+tail magic:                       8 bytes long, value is PAFIDX02
+</pre>
+
+`start pos` is relative to the container start. For a container of length `F`, the footer starts at
+`F - 12 - footer length`.
+`FileIndexFormat.createReader` receives the container length so it can locate the trailer and
+footer.
+
+V2 supports a container larger than 2 GiB by storing payload positions and lengths as 64-bit
+values. Each index still serializes to a `byte[]`. `FileIndexer.createReader` receives a 64-bit
+start position and an int length. Before reading an index, the reader rejects a payload whose
+length exceeds `Integer.MAX_VALUE`.
+
 ## Index: BloomFilter 
 
 Options are:

docs/generated/core_configuration.html

@@ -710,6 +710,12 @@
             <td>String</td>
             <td>Default aggregate function of all fields for partial-update and aggregate merge function.</td>
         </tr>
+        <tr>
+            <td><h5>file-index.format.version</h5></td>
+            <td style="word-wrap: break-word;">1</td>
+            <td>Integer</td>
+            <td>File index container version to write (1 or 2). Version 2 supports 64-bit payload positions.</td>
+        </tr>
         <tr>
             <td><h5>file-index.in-manifest-threshold</h5></td>
             <td style="word-wrap: break-word;">500 bytes</td>

paimon-api/src/main/java/org/apache/paimon/CoreOptions.java

@@ -434,6 +434,13 @@ public InlineElement getDescription() {
                     .defaultValue(MemorySize.parse("500 B"))
                     .withDescription("The threshold to store file index bytes in manifest.");
 
+    public static final ConfigOption<Integer> FILE_INDEX_FORMAT_VERSION =
+            key("file-index.format.version")
+                    .intType()
+                    .defaultValue(1)
+                    .withDescription(
+                            "File index container version to write (1 or 2). Version 2 supports 64-bit payload positions.");
+
     public static final ConfigOption<Boolean> FILE_INDEX_READ_ENABLED =
             key("file-index.read.enabled")
                     .booleanType()
@@ -4457,6 +4464,14 @@ public FileIndexOptions indexColumnsOptions() {
         return new FileIndexOptions(this);
     }
 
+    public int fileIndexFormatVersion() {
+        int version = options.get(FILE_INDEX_FORMAT_VERSION);
+        if (version != 1 && version != 2) {
+            throw new IllegalArgumentException("file-index.format.version must be 1 or 2");
+        }
+        return version;
+    }
+
     public long fileIndexInManifestThreshold() {
         return options.get(FILE_INDEX_IN_MANIFEST_THRESHOLD).getBytes();
     }

paimon-api/src/main/java/org/apache/paimon/fileindex/FileIndexOptions.java

@@ -40,6 +40,7 @@ public class FileIndexOptions {
 
     // if the filter size greater than fileIndexInManifestThreshold, we put it in file
     private final long fileIndexInManifestThreshold;
+    private final int formatVersion;
 
     private final Map<Column, Map<String, Options>> indexTypeOptions;
     private final Map<Column, Map<String, Options>> topLevelMapColumnOptions;
@@ -52,6 +53,7 @@ public FileIndexOptions(CoreOptions coreOptions) {
         this.indexTypeOptions = new HashMap<>();
         this.topLevelMapColumnOptions = new HashMap<>();
         this.fileIndexInManifestThreshold = coreOptions.fileIndexInManifestThreshold();
+        this.formatVersion = coreOptions.fileIndexFormatVersion();
         setupOptions(coreOptions);
     }
 
@@ -186,6 +188,10 @@ public boolean isEmpty() {
         return indexTypeOptions.isEmpty();
     }
 
+    public int formatVersion() {
+        return formatVersion;
+    }
+
     public long fileIndexInManifestThreshold() {
         return fileIndexInManifestThreshold;
     }