RUST-871 Support serializing directly to BSON bytes (#279)

Author: patrickfreed

.evergreen/config.yml

@@ -92,16 +92,6 @@ functions:
           ${PREPARE_SHELL}
           .evergreen/run-tests-u2i.sh
 
-  "run serde tests":
-    - command: shell.exec
-      type: test
-      params:
-        shell: bash
-        working_dir: "src"
-        script: |
-          ${PREPARE_SHELL}
-          .evergreen/run-tests-serde.sh
-
   "run decimal128 tests":
     - command: shell.exec
       type: test
@@ -170,10 +160,6 @@ tasks:
     commands:
       - func: "run u2i tests"
 
-  - name: "test-serde"
-    commands:
-      - func: "run serde tests"
-
   - name: "test-decimal128"
     commands:
       - func: "run decimal128 tests"
@@ -211,7 +197,6 @@ buildvariants:
   tasks:
     - name: "test"
     - name: "test-u2i"
-    - name: "test-serde"
     - name: "test-decimal128"
 
 - matrix_name: "compile only"

.evergreen/run-tests-decimal128.sh

@@ -4,3 +4,6 @@ set -o errexit
 
 . ~/.cargo/env
 RUST_BACKTRACE=1 cargo test --features decimal128
+
+cd serde-tests
+RUST_BACKTRACE=1 cargo test --features decimal128

.evergreen/run-tests-u2i.sh

@@ -4,3 +4,6 @@ set -o errexit
 
 . ~/.cargo/env
 RUST_BACKTRACE=1 cargo test --features u2i
+
+cd serde-tests
+RUST_BACKTRACE=1 cargo test --features u2i

.evergreen/run-tests.sh

@@ -5,3 +5,6 @@ set -o errexit
 . ~/.cargo/env
 RUST_BACKTRACE=1 cargo test
 RUST_BACKTRACE=1 cargo test --features chrono-0_4,uuid-0_8
+
+cd serde-tests
+RUST_BACKTRACE=1 cargo test

serde-tests/Cargo.toml

@@ -4,10 +4,15 @@ version = "0.1.0"
 authors = ["Kevin Yeh <kevinyeah@utexas.edu>"]
 edition = "2018"
 
+[features]
+u2i = ["bson/u2i"]
+decimal128 = ["bson/decimal128"]
+
 [dependencies]
-bson = { path = "..", features = ["decimal128"] }
+bson = { path = ".." }
 serde = { version = "1.0", features = ["derive"] }
 pretty_assertions = "0.6.1"
+hex = "0.4.2"
 
 [lib]
 name = "serde_tests"

serde-tests/test.rs

@@ -14,14 +14,15 @@ use std::{
     collections::{BTreeMap, HashSet},
 };
 
+#[cfg(feature = "decimal128")]
+use bson::Decimal128;
 use bson::{
     doc,
     oid::ObjectId,
     spec::BinarySubtype,
     Binary,
     Bson,
     DateTime,
-    Decimal128,
     Deserializer,
     Document,
     JavaScriptCodeWithScope,
@@ -37,6 +38,7 @@ use bson::{
 ///     - deserializing a `T` from the raw BSON version of `expected_doc` produces `expected_value`
 ///     - deserializing a `Document` from the raw BSON version of `expected_doc` produces
 ///       `expected_doc`
+///   - `bson::to_writer` and `Document::to_writer` produce the same result given the same input
 fn run_test<T>(expected_value: &T, expected_doc: &Document, description: &str)
 where
     T: Serialize + DeserializeOwned + PartialEq + std::fmt::Debug,
@@ -46,6 +48,16 @@ where
         .to_writer(&mut expected_bytes)
         .expect(description);
 
+    let expected_bytes_serde = bson::to_vec(&expected_value).expect(description);
+    assert_eq!(expected_bytes_serde, expected_bytes, "{}", description);
+
+    let expected_bytes_from_doc_serde = bson::to_vec(&expected_doc).expect(description);
+    assert_eq!(
+        expected_bytes_from_doc_serde, expected_bytes,
+        "{}",
+        description
+    );
+
     let serialized_doc = bson::to_document(&expected_value).expect(description);
     assert_eq!(&serialized_doc, expected_doc, "{}", description);
     assert_eq!(
@@ -702,7 +714,7 @@ fn all_types() {
         undefined: Bson,
         code: Bson,
         code_w_scope: JavaScriptCodeWithScope,
-        decimal: Decimal128,
+        decimal: Bson,
         symbol: Bson,
         min_key: Bson,
         max_key: Bson,
@@ -737,6 +749,16 @@ fn all_types() {
     let oid = ObjectId::new();
     let subdoc = doc! { "k": true, "b": { "hello": "world" } };
 
+    #[cfg(not(feature = "decimal128"))]
+    let decimal = {
+        let bytes = hex::decode("18000000136400D0070000000000000000000000003A3000").unwrap();
+        let d = Document::from_reader(bytes.as_slice()).unwrap();
+        d.get("d").unwrap().clone()
+    };
+
+    #[cfg(feature = "decimal128")]
+    let decimal = Bson::Decimal128(Decimal128::from_str("2.000"));
+
     let doc = doc! {
         "x": 1,
         "y": 2_i64,
@@ -758,7 +780,7 @@ fn all_types() {
         "undefined": Bson::Undefined,
         "code": code.clone(),
         "code_w_scope": code_w_scope.clone(),
-        "decimal": Bson::Decimal128(Decimal128::from_i32(5)),
+        "decimal": decimal.clone(),
         "symbol": Bson::Symbol("ok".to_string()),
         "min_key": Bson::MinKey,
         "max_key": Bson::MaxKey,
@@ -789,7 +811,7 @@ fn all_types() {
         undefined: Bson::Undefined,
         code,
         code_w_scope,
-        decimal: Decimal128::from_i32(5),
+        decimal,
         symbol: Bson::Symbol("ok".to_string()),
         min_key: Bson::MinKey,
         max_key: Bson::MaxKey,
@@ -851,3 +873,83 @@ fn borrowed() {
         bson::from_slice(bson.as_slice()).expect("deserialization should succeed");
     assert_eq!(deserialized, v);
 }
+
+#[cfg(feature = "u2i")]
+#[test]
+fn u2i() {
+    #[derive(Serialize, Deserialize, Debug, PartialEq)]
+    struct Foo {
+        u_8: u8,
+        u_16: u16,
+        u_32: u32,
+        u_32_max: u32,
+        u_64: u64,
+        i_64_max: u64,
+    }
+
+    let v = Foo {
+        u_8: 15,
+        u_16: 123,
+        u_32: 1234,
+        u_32_max: u32::MAX,
+        u_64: 12345,
+        i_64_max: i64::MAX as u64,
+    };
+
+    let expected = doc! {
+        "u_8": 15_i32,
+        "u_16": 123_i32,
+        "u_32": 1234_i64,
+        "u_32_max": u32::MAX as i64,
+        "u_64": 12345_i64,
+        "i_64_max": i64::MAX as u64,
+    };
+
+    run_test(&v, &expected, "u2i - valid");
+
+    #[derive(Serialize, Debug)]
+    struct TooBig {
+        u_64: u64,
+    }
+    let v = TooBig {
+        u_64: i64::MAX as u64 + 1,
+    };
+    bson::to_document(&v).unwrap_err();
+    bson::to_vec(&v).unwrap_err();
+}
+
+#[cfg(not(feature = "u2i"))]
+#[test]
+fn unsigned() {
+    #[derive(Serialize, Debug)]
+    struct U8 {
+        v: u8,
+    }
+    let v = U8 { v: 1 };
+    bson::to_document(&v).unwrap_err();
+    bson::to_vec(&v).unwrap_err();
+
+    #[derive(Serialize, Debug)]
+    struct U16 {
+        v: u16,
+    }
+    let v = U16 { v: 1 };
+    bson::to_document(&v).unwrap_err();
+    bson::to_vec(&v).unwrap_err();
+
+    #[derive(Serialize, Debug)]
+    struct U32 {
+        v: u32,
+    }
+    let v = U32 { v: 1 };
+    bson::to_document(&v).unwrap_err();
+    bson::to_vec(&v).unwrap_err();
+
+    #[derive(Serialize, Debug)]
+    struct U64 {
+        v: u64,
+    }
+    let v = U64 { v: 1 };
+    bson::to_document(&v).unwrap_err();
+    bson::to_vec(&v).unwrap_err();
+}

src/bson.rs

@@ -22,7 +22,7 @@
 //! BSON definition
 
 use std::{
-    convert::TryFrom,
+    convert::{TryFrom, TryInto},
     fmt::{self, Debug, Display, Formatter},
 };
 
@@ -692,6 +692,20 @@ impl Bson {
                 }
             }
 
+            ["$numberDecimalBytes"] => {
+                if let Ok(bytes) = doc.get_binary_generic("$numberDecimalBytes") {
+                    if let Ok(b) = bytes.clone().try_into() {
+                        #[cfg(not(feature = "decimal128"))]
+                        return Bson::Decimal128(Decimal128 { bytes: b });
+
+                        #[cfg(feature = "decimal128")]
+                        unsafe {
+                            return Bson::Decimal128(Decimal128::from_raw_bytes_le(b));
+                        }
+                    }
+                }
+            }
+
             ["$binary"] => {
                 if let Some(binary) = Binary::from_extended_doc(&doc) {
                     return Bson::Binary(binary);

src/extjson/models.rs

@@ -4,6 +4,7 @@ use chrono::Utc;
 use serde::{
     de::{Error, Unexpected},
     Deserialize,
+    Serialize,
 };
 
 use crate::{extjson, oid, spec::BinarySubtype, Bson};
@@ -27,7 +28,7 @@ impl Int32 {
     }
 }
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct Int64 {
     #[serde(rename = "$numberLong")]
@@ -72,7 +73,7 @@ impl Double {
     }
 }
 
-#[derive(Deserialize)]
+#[derive(Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct ObjectId {
     #[serde(rename = "$oid")]
@@ -86,6 +87,12 @@ impl ObjectId {
     }
 }
 
+impl From<crate::oid::ObjectId> for ObjectId {
+    fn from(id: crate::oid::ObjectId) -> Self {
+        Self { oid: id.to_hex() }
+    }
+}
+
 #[derive(Deserialize)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct Symbol {
@@ -100,7 +107,7 @@ pub(crate) struct Regex {
     body: RegexBody,
 }
 
-#[derive(Deserialize)]
+#[derive(Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct RegexBody {
     pub(crate) pattern: String,
@@ -127,7 +134,7 @@ pub(crate) struct Binary {
     pub(crate) body: BinaryBody,
 }
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct BinaryBody {
     pub(crate) base64: String,
@@ -207,10 +214,13 @@ pub(crate) struct Timestamp {
     body: TimestampBody,
 }
 
-#[derive(Deserialize)]
+#[derive(Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct TimestampBody {
+    #[serde(serialize_with = "crate::serde_helpers::serialize_u32_as_i64")]
     pub(crate) t: u32,
+
+    #[serde(serialize_with = "crate::serde_helpers::serialize_u32_as_i64")]
     pub(crate) i: u32,
 }
 
@@ -223,20 +233,28 @@ impl Timestamp {
     }
 }
 
-#[derive(Deserialize)]
+#[derive(Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct DateTime {
     #[serde(rename = "$date")]
     pub(crate) body: DateTimeBody,
 }
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize)]
 #[serde(untagged)]
 pub(crate) enum DateTimeBody {
     Canonical(Int64),
     Relaxed(String),
 }
 
+impl DateTimeBody {
+    pub(crate) fn from_millis(m: i64) -> Self {
+        DateTimeBody::Canonical(Int64 {
+            value: m.to_string(),
+        })
+    }
+}
+
 impl DateTime {
     pub(crate) fn parse(self) -> extjson::de::Result<crate::DateTime> {
         match self.body {
@@ -307,7 +325,7 @@ pub(crate) struct DbPointer {
     body: DbPointerBody,
 }
 
-#[derive(Deserialize)]
+#[derive(Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct DbPointerBody {
     #[serde(rename = "$ref")]

src/lib.rs

@@ -198,7 +198,7 @@ pub use self::{
         Deserializer,
     },
     decimal128::Decimal128,
-    ser::{to_bson, to_document, Serializer},
+    ser::{to_bson, to_document, to_vec, Serializer},
 };
 
 #[macro_use]