feat: FineWeb-10B parquet upload and search examples

README.md

@@ -117,7 +117,7 @@ inline dataset definitions (same fields as
 | `sparse` | CSR matrices (`data.csr`, optional `queries.csr` / `results.gt`) |
 | `npy` | One 2-D float `.npy` — dense vectors only |
 | `multivector` | Directory of `vectors.npy` (flat sub-vectors) + `offsets.npy` (row boundaries per point). Late-interaction, ColBERT-style multivectors only |
-| `parquet` | One parquet file — payload rows only |
+| `parquet` | One parquet file — payload rows, and optionally dense/sparse vectors via `vector_column` / `sparse_column` |
 
 The first three are *bundles*: vectors, payloads, and queries all come out of a
 single artifact. `npy`, `multivector`, and `parquet` are *components*, so a
@@ -134,10 +134,13 @@ collection:
       dataset: { name: meta, format: parquet, path: meta.parquet, exclude: [exif] }
 ```
 
-Parquet sources accept three extra keys: `columns` (keep only these), `exclude`
-(drop these), and `fill_null` (a value substituted for nulls and for NaN/±inf
-floats, which have no JSON form — by default such fields are simply absent).
-See [`examples/upload-laion-part.yaml`](examples/upload-laion-part.yaml).
+Parquet sources accept five extra keys: `columns` (keep only these), `exclude`
+(drop these), `fill_null` (a value substituted for nulls and for NaN/±inf
+floats, which have no JSON form — by default such fields are simply absent),
+`vector_column` (list-of-floats column used as a dense vector source), and
+`sparse_column` (`{indices, values}` struct column used as a sparse vector
+source). See [`examples/upload-laion-part.yaml`](examples/upload-laion-part.yaml)
+and [`examples/upload-fineweb-part.yaml`](examples/upload-fineweb-part.yaml).
 
 #### Multivector (ColBERT-style) datasets
 
@@ -169,7 +172,8 @@ collection:
 
 Corpora published as numbered parts are read as one row space with a `parts:`
 block, so point ids stay global across the whole set. `npy` and `parquet`
-sources support it; `{i}` is substituted with each part's number:
+sources support it; `{i}` (or zero-padded `{i:04d}`) is substituted with each
+part's number:
 
 ```yaml
 source:
@@ -182,6 +186,21 @@ source:
     link: https://deploy.laion.ai/.../img_emb_{i}.npy
 ```
 
+FineWeb-10B shards use zero-padded names — dial the subset with `parts.count`:
+
+```yaml
+source:
+  type: dataset
+  name: fineweb-10b-dense
+  format: parquet
+  vector_column: dense_embedding
+  parts:
+    count: 1                   # first N of train-part0 (raise toward 10000)
+    path: fineweb/train-part0/{i:04d}.parquet
+    link: https://huggingface.co/datasets/Qdrant/FineWeb-10B/resolve/refs%2Fconvert%2Fparquet/default/train-part0/{i:04d}.parquet
+```
+
+See [`examples/upload-fineweb-10b.yaml`](examples/upload-fineweb-10b.yaml).
 Part row counts are **measured, never configured**. Both formats keep their
 shape at a known end of the file — the `.npy` header at the front, the parquet
 footer at the back — so bfb sizes every part with one ranged HTTP request each

examples/search-fineweb.yaml

@@ -0,0 +1,58 @@
+# Search workload for a FineWeb collection uploaded with
+# `upload-fineweb-part` / `upload-fineweb-10b`.
+#
+#   bfb search --example search-fineweb -n 10k -p 8 -t 4 \
+#       --search-limit 10 --uri http://localhost:6334
+#
+# Query vectors are random (MS MARCO query embeddings are not redistributed
+# with the dataset — regenerate them from the HF scripts if you need recall
+# against `queries/gt_*.parquet`). Filters exercise the FineWeb metadata
+# indexes created on upload.
+
+collection:
+  name: fineweb
+
+requests:
+  - kind: dense
+    using: dense
+    size: 768
+    datatype: float16
+    source: random
+
+  - kind: sparse
+    using: sparse
+    source:
+      type: random
+      vocab_size: 250048
+      length: 100
+      distribution: zipf
+
+  - kind: dense
+    using: dense
+    size: 768
+    datatype: float16
+    source: random
+    filters:
+      - name: language
+        type: keyword
+        source: { type: random, cardinality: 1 }
+
+  - kind: dense
+    using: dense
+    size: 768
+    datatype: float16
+    source: random
+    filters:
+      - name: language_score
+        type: float
+        source: { type: random, min: 0.65, max: 1.0 }
+
+  - kind: dense
+    using: dense
+    size: 768
+    datatype: float16
+    source: random
+    filters:
+      - name: token_count
+        type: integer
+        source: { type: random, min: 50, max: 2000 }

examples/upload-fineweb-10b.yaml

@@ -0,0 +1,95 @@
+# Qdrant/FineWeb-10B corpus
+# (https://huggingface.co/datasets/Qdrant/FineWeb-10B) — dense + sparse
+# embeddings and FineWeb metadata in numbered parquet shards.
+#
+#   bfb upload --example upload-fineweb-10b -b 64 -p 8 -t 4 \
+#       --uri http://localhost:6334
+#
+# Dial the subset with `parts.count` (and optional `parts.start`). The Hugging
+# Face convert/parquet layout exposes train-part0 as 10_000 zero-padded shards
+# `0000.parquet` … `9999.parquet` (~10B rows together with train-part1). The
+# default below uploads a single shard — raise `count` toward 10000 for a
+# larger slice (each shard is ~5 GB on disk).
+#
+# On first use bfb sizes every configured part with one ranged request each and
+# caches the result in `datasets/.parts-index/`. Point ids are dataset rows, so
+# `--offset` resumes an interrupted run.
+#
+# Disk: each shard is ~5 GB. Dense, sparse, and payload all read the same files;
+# keep `cache: keep` (default) so one source does not delete a part another still
+# needs. Drop sparse/payload and set `cache: evict` if you want a dense-only
+# stream that holds only a few shards on disk.
+
+collection:
+  name: fineweb
+  on_disk_payload: true
+
+  quantization:
+    type: turbo-2bit
+    always_ram: true
+
+  hnsw:
+    on_disk: false
+
+  optimizers:
+    default_segment_number: 2
+    max_segment_size: 5000000
+
+  vectors:
+    - name: dense
+      size: 768
+      distance: cosine
+      datatype: float16
+      on_disk: true
+      source:
+        type: dataset
+        name: fineweb-10b-dense
+        format: parquet
+        vector_column: dense_embedding
+        parts:
+          # First N of train-part0 (0..9999). Default is one shard; raise for more.
+          count: 1
+          path: fineweb/train-part0/{i:04d}.parquet
+          link: https://huggingface.co/datasets/Qdrant/FineWeb-10B/resolve/refs%2Fconvert%2Fparquet/default/train-part0/{i:04d}.parquet
+
+  sparse_vectors:
+    - name: sparse
+      on_disk: true
+      source:
+        type: dataset
+        dataset:
+          name: fineweb-10b-sparse
+          format: parquet
+          sparse_column: sparse_embedding
+          parts:
+            count: 1
+            path: fineweb/train-part0/{i:04d}.parquet
+            link: https://huggingface.co/datasets/Qdrant/FineWeb-10B/resolve/refs%2Fconvert%2Fparquet/default/train-part0/{i:04d}.parquet
+
+  payload:
+    source:
+      type: dataset
+      dataset:
+        name: fineweb-10b-payload
+        format: parquet
+        exclude: [dense_embedding, sparse_embedding, text]
+        parts:
+          count: 1
+          path: fineweb/train-part0/{i:04d}.parquet
+          link: https://huggingface.co/datasets/Qdrant/FineWeb-10B/resolve/refs%2Fconvert%2Fparquet/default/train-part0/{i:04d}.parquet
+
+  fields:
+    - name: id
+      type: keyword
+    - name: dump
+      type: keyword
+    - name: language
+      type: keyword
+    - name: url
+      type: keyword
+    - name: date
+      type: datetime
+    - name: language_score
+      type: float
+    - name: token_count
+      type: integer

examples/upload-fineweb-part.yaml

@@ -0,0 +1,81 @@
+# One parquet shard of Qdrant/FineWeb-10B
+# (https://huggingface.co/datasets/Qdrant/FineWeb-10B).
+#
+#   bfb upload --example upload-fineweb-part -b 64 -p 8 -t 4 \
+#       --uri http://localhost:6334
+#
+# Dense + sparse embeddings and FineWeb metadata live in the same parquet.
+# `vector_column` / `sparse_column` tell bfb which columns are vectors; the
+# payload source drops those (and the full `text`) so only filterable metadata
+# is uploaded. Omit `-n` to upload the whole shard (~1M points; size varies).
+#
+# The file is downloaded on first use into `./datasets/` (override with
+# BFB_DATASETS_DIR), ~5 GB. For N shards at once, see `upload-fineweb-10b.yaml`.
+
+collection:
+  name: fineweb
+  on_disk_payload: true
+
+  quantization:
+    type: turbo-2bit
+    always_ram: true
+
+  hnsw:
+    on_disk: false
+
+  optimizers:
+    default_segment_number: 2
+    max_segment_size: 2000000
+
+  vectors:
+    - name: dense
+      size: 768
+      distance: cosine
+      datatype: float16
+      on_disk: true
+      source:
+        type: dataset
+        name: fineweb-part0-dense
+        format: parquet
+        path: fineweb/train-part0/0000.parquet
+        link: https://huggingface.co/datasets/Qdrant/FineWeb-10B/resolve/refs%2Fconvert%2Fparquet/default/train-part0/0000.parquet
+        vector_column: dense_embedding
+
+  sparse_vectors:
+    - name: sparse
+      on_disk: true
+      source:
+        type: dataset
+        dataset:
+          name: fineweb-part0-sparse
+          format: parquet
+          path: fineweb/train-part0/0000.parquet
+          link: https://huggingface.co/datasets/Qdrant/FineWeb-10B/resolve/refs%2Fconvert%2Fparquet/default/train-part0/0000.parquet
+          sparse_column: sparse_embedding
+
+  payload:
+    source:
+      type: dataset
+      dataset:
+        name: fineweb-part0-payload
+        format: parquet
+        path: fineweb/train-part0/0000.parquet
+        link: https://huggingface.co/datasets/Qdrant/FineWeb-10B/resolve/refs%2Fconvert%2Fparquet/default/train-part0/0000.parquet
+        # Embeddings are uploaded as vectors; `text` dominates the row size.
+        exclude: [dense_embedding, sparse_embedding, text]
+
+  fields:
+    - name: id
+      type: keyword
+    - name: dump
+      type: keyword
+    - name: language
+      type: keyword
+    - name: url
+      type: keyword
+    - name: date
+      type: datetime
+    - name: language_score
+      type: float
+    - name: token_count
+      type: integer

src/config/examples.rs

@@ -90,6 +90,16 @@ pub static EXAMPLES: &[Example] = &[
         Upload,
         "Full LAION-400M corpus (~410 parts, streamed with cache: evict)"
     ),
+    example!(
+        "upload-fineweb-part",
+        Upload,
+        "One FineWeb-10B parquet shard: dense + sparse + metadata"
+    ),
+    example!(
+        "upload-fineweb-10b",
+        Upload,
+        "FineWeb-10B subset via parts.count (parquet vectors + metadata)"
+    ),
     example!(
         "serverless-upload",
         Upload,
@@ -105,6 +115,11 @@ pub static EXAMPLES: &[Example] = &[
         Search,
         "Measure recall against a dataset query set + ground truth"
     ),
+    example!(
+        "search-fineweb",
+        Search,
+        "Dense/sparse/filtered search against an uploaded FineWeb collection"
+    ),
     example!(
         "scroll-config",
         Scroll,

src/config/schema.rs

@@ -107,23 +107,24 @@ collection:
       #                          #     offsets.npy (row boundaries per point) — ColBERT-style
       #                          #     multivectors; requires `multivector:` above (`count` is
       #                          #     ignored — arity comes from `offsets.npy`)
-      #                          #   parquet one parquet file — payload rows only
+      #                          #   parquet payload rows + optional vector_column/sparse_column
       #   path: glove-25-angular/glove-25-angular.hdf5
       #   link: http://ann-benchmarks.com/glove-25-angular.hdf5
       #   vector_size: 25
       #   distance: cosine
       # A sharded dataset (`npy` / `parquet` only) replaces `path`/`link` with a
-      # `parts` block; the files are read as one row space and `{i}` is
-      # substituted with each part's number. Row counts per part are measured,
-      # not configured — one ranged request per part, cached thereafter.
+      # `parts` block; the files are read as one row space and `{i}` (or
+      # zero-padded `{i:04d}`) is substituted with each part's number. Row
+      # counts per part are measured, not configured — one ranged request per
+      # part, cached thereafter.
       # source:
       #   type: dataset
       #   name: laion-400m-img-emb
       #   format: npy
       #   parts:
       #     count: 410           # uint     required   number of parts
-      #     start: 0             # uint     default=0  index of the first part
-      #     path: laion/img_emb_{i}.npy     # string   required
+      #     start: 0             # uint     default=0  first part index
+      #     path: laion/img_emb_{i}.npy   # string  required  `{i}` or `{i:04d}`
       #     link: https://host/img_emb_{i}.npy  # string  optional
       #   cache: keep          # enum  default=keep  [keep | evict] (sharded only)
       #                        #   evict deletes each downloaded part once the reader
@@ -175,13 +176,16 @@ collection:
     #     path: laion-small-clip/laion-small-clip
     #     link: https://example.com/laion-small-clip.tgz
     # `format: parquet` reads payload rows from a parquet file, and accepts
-    # three extra keys (ignored by every other format):
+    # five extra keys (ignored by every other format):
     #   columns: [url, similarity]   # list  optional  columns to keep (default: all)
     #   exclude: [exif]              # list  default=[]  columns to drop (applied after `columns`)
     #   fill_null: 0                 # any   optional  value substituted for nulls and for
     #                                #   NaN/±inf floats, which have no JSON form. Omitted by
     #                                #   default, leaving the payload field absent.
-
+    #   vector_column: dense_embedding   # optional  list-of-floats dense vector column
+    #   sparse_column: sparse_embedding  # optional  {indices, values} sparse vector column
+    # When `vector_column` / `sparse_column` are set and `columns` is omitted,
+    # only those vector columns are decoded (payload text is skipped).
   # Payload field declarations (optional). Names must be unique. Each entry
   # generates a value and/or declares a field index.
   fields: