Add BSON format, JSON.DEBUG & Remove submodule, (#37)

* get_doc: return error on wrong path
* redisjson.rs: turn num_op into a generic value_op
* lib.rs: add support for STRAPPEND, ARRAPPEND, ARRINSERT, ARRPOP
* Add support for JSON.ARRINDEX
* Add support for JSON.ARRTRIM & fix ARRINSERT to accept negative index
* Add support for JSON.OBJKEYS
* add special handling for path=$ (root), can't call replace_with
* fix returned index in case start>0
* Return poped value on arrpop and fix error handle on replace with wrong path
* fix path backward on ARRPOP
* fix ARRPOP for index<0
* Fix commands for backward:
* STRAPPEND & ARRAPPNED return the len after update
* MGET returns nil when path not found for a key
* uncomment more tests
* remove sub-module (set redismodule-rs repo in Cargo.toml)
* inital BSON support
* add debug command (#38)

Author: gkorland

Cargo.lock

@@ -8,6 +8,7 @@ edition = "2018"
 crate-type = ["cdylib"]
 
 [dependencies]
+bson = "0.13.0"
 serde_json = "1.0"
 libc = "0.2"
 jsonpath_lib = { git="https://github.com/gkorland/jsonpath.git", branch="reaplce_with_ownership_parser" }

Cargo.toml

@@ -12,7 +12,7 @@ mod nodevisitor;
 mod redisjson;
 
 use crate::index::Index;
-use crate::redisjson::{Error, RedisJSON, SetOptions};
+use crate::redisjson::{Error, Format, RedisJSON, SetOptions};
 
 static JSON_TYPE_ENCODING_VERSION: i32 = 2;
 static JSON_TYPE_NAME: &str = "ReJSON-RL";
@@ -84,29 +84,37 @@ fn json_set(ctx: &Context, args: Vec<String>) -> RedisResult {
     let path = backwards_compat_path(args.next_string()?);
     let value = args.next_string()?;
 
-    let set_option = args
-        .next()
-        .map(|op| match op.to_uppercase().as_str() {
-            "NX" => Ok(SetOptions::NotExists),
-            "XX" => Ok(SetOptions::AlreadyExists),
-            _ => Err(RedisError::Str("ERR syntax error")),
-        })
-        .transpose()?;
+    let mut format = Format::JSON;
+    let mut set_option = SetOptions::None;
+    loop {
+        if let Some(s) = args.next() {
+            match s.to_uppercase().as_str() {
+                "NX" => set_option = SetOptions::NotExists,
+                "XX" => set_option = SetOptions::AlreadyExists,
+                "FORMAT" => {
+                    format = Format::from_str(args.next_string()?.as_str())?;
+                }
+                _ => break,
+            };
+        } else {
+            break;
+        }
+    }
 
     let key = ctx.open_key_writable(&key);
     let current = key.get_value::<RedisJSON>(&REDIS_JSON_TYPE)?;
 
     match (current, set_option) {
         (Some(ref mut doc), ref op) => {
-            if doc.set_value(&value, &path, op)? {
+            if doc.set_value(&value, &path, op, format)? {
                 REDIS_OK
             } else {
                 Ok(RedisValue::None)
             }
         }
-        (None, Some(SetOptions::AlreadyExists)) => Ok(RedisValue::None),
+        (None, SetOptions::AlreadyExists) => Ok(RedisValue::None),
         (None, _) => {
-            let doc = RedisJSON::from_str(&value)?;
+            let doc = RedisJSON::from_str(&value, format)?;
             if path == "$" {
                 key.set_value(&REDIS_JSON_TYPE, doc)?;
                 REDIS_OK
@@ -128,29 +136,55 @@ fn json_set(ctx: &Context, args: Vec<String>) -> RedisResult {
 /// TODO add support for multi path
 fn json_get(ctx: &Context, args: Vec<String>) -> RedisResult {
     let mut args = args.into_iter().skip(1);
-
     let key = args.next_string()?;
 
-    let mut path = loop {
+    let mut paths: Vec<String> = vec![];
+    let mut first_loop = true;
+    let mut format = Format::JSON;
+    loop {
         let arg = match args.next_string() {
             Ok(s) => s,
-            Err(_) => "$".to_owned(), // path is optional
+            Err(_) => {
+                // path is optional -> no path found on the first loop we use root "$"
+                if first_loop {
+                    paths.push("$".to_owned());
+                }
+                break;
+            }
         };
+        first_loop = false;
 
         match arg.as_str() {
-            "INDENT" => args.next(),  // TODO add support
-            "NEWLINE" => args.next(), // TODO add support
-            "SPACE" => args.next(),   // TODO add support
-            "NOESCAPE" => continue,   // TODO add support
-            _ => break arg,
+            "INDENT" => {
+                args.next();
+            } // TODO add support
+            "NEWLINE" => {
+                args.next();
+            } // TODO add support
+            "SPACE" => {
+                args.next();
+            } // TODO add support
+            "NOESCAPE" => {
+                continue;
+            } // TODO add support
+            "FORMAT" => {
+                format = Format::from_str(args.next_string()?.as_str())?;
+            }
+            _ => {
+                paths.push(backwards_compat_path(arg));
+            }
         };
-    };
-    path = backwards_compat_path(path);
+    }
 
     let key = ctx.open_key_writable(&key);
-
     let value = match key.get_value::<RedisJSON>(&REDIS_JSON_TYPE)? {
-        Some(doc) => doc.to_string(&path)?.into(),
+        Some(doc) => if paths.len() == 1 {
+            doc.to_string(&paths[0], format)?
+        } else {
+            // can't be smaller than 1
+            doc.to_json(&mut paths)?
+        }
+        .into(),
         None => RedisValue::None,
     };
 
@@ -175,7 +209,7 @@ fn json_mget(ctx: &Context, args: Vec<String>) -> RedisResult {
                 let result = ctx
                     .open_key(key)
                     .get_value::<RedisJSON>(&REDIS_JSON_TYPE)?
-                    .map(|doc| doc.to_string(&path))
+                    .map(|doc| doc.to_string(&path, Format::JSON))
                     .transpose()?;
 
                 Ok(result.into())
@@ -572,8 +606,29 @@ fn json_obj_len(ctx: &Context, args: Vec<String>) -> RedisResult {
 /// MEMORY <key> [path]
 /// HELP
 ///
-fn json_debug(_ctx: &Context, _args: Vec<String>) -> RedisResult {
-    Err("Command was not implemented".into())
+fn json_debug(ctx: &Context, args: Vec<String>) -> RedisResult {
+    let mut args = args.into_iter().skip(1);
+    match args.next_string()?.to_uppercase().as_str() {
+        "MEMORY" => {
+            let key = args.next_string()?;
+            let path = backwards_compat_path(args.next_string()?);
+
+            let key = ctx.open_key(&key);
+            let value = match key.get_value::<RedisJSON>(&REDIS_JSON_TYPE)? {
+                Some(doc) => doc.get_memory(&path)?,
+                None => 0,
+            };
+            Ok(value.into())
+        }
+        "HELP" => {
+            let results = vec![
+                "MEMORY <key> [path] - reports memory usage",
+                "HELP                - this message",
+            ];
+            Ok(results.into())
+        }
+        _ => Err("ERR unknown subcommand - try `JSON.DEBUG HELP`".into()),
+    }
 }
 
 ///

redismodule-rs

@@ -3,11 +3,15 @@
 // Translate between JSON and tree of Redis objects:
 // User-provided JSON is converted to a tree. This tree is stored transparently in Redis.
 // It can be operated on (e.g. INCR) and serialized back to JSON.
+
+use bson::decode_document;
 use crate::backward;
 use crate::nodevisitor::NodeVisitorImpl;
 use jsonpath_lib::{JsonPathError, SelectorMut};
 use redismodule::raw;
-use serde_json::Value;
+use serde_json::{Map, Value};
+use std::io::Cursor;
+use std::mem;
 use std::os::raw::{c_int, c_void};
 
 #[derive(Debug)]
@@ -19,6 +23,7 @@ pub struct Error {
 pub enum SetOptions {
     NotExists,
     AlreadyExists,
+    None,
 }
 
 impl From<String> for Error {
@@ -53,12 +58,50 @@ impl From<Error> for redismodule::RedisError {
     }
 }
 
+#[derive(Debug, PartialEq)]
+pub enum Format {
+    JSON,
+    BSON,
+}
+
+impl Format {
+    pub fn from_str(s: &str) -> Result<Format, Error> {
+        match s {
+            "JSON" => Ok(Format::JSON),
+            "BSON" => Ok(Format::BSON),
+            _ => return Err("ERR wrong format".into()),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct RedisJSON {
     data: Value,
 }
 
 impl RedisJSON {
+    pub fn parse_str(data: &str, format: Format) -> Result<Value, Error> {
+        match format {
+            Format::JSON => Ok(serde_json::from_str(data)?),
+            Format::BSON => decode_document(&mut Cursor::new(data.as_bytes()))
+                .map(|docs| {
+                    let v = if docs.len() >= 1 {
+                        docs.iter()
+                            .next()
+                            .map_or_else(|| Value::Null, |(_, b)| b.clone().into())
+                    } else {
+                        Value::Null
+                    };
+                    Ok(v)
+                })
+                .unwrap_or_else(|e| Err(e.to_string().into())),
+        }
+    }
+
+    pub fn from_str(data: &str, format: Format) -> Result<Self, Error> {
+        let value = RedisJSON::parse_str(data, format)?;
+        Ok(Self { data: value })
+    }
     fn add_value(&mut self, path: &str, value: Value) -> Result<bool, Error> {
         if NodeVisitorImpl::check(path)? {
             let mut splits = path.rsplitn(2, '.');
@@ -99,32 +142,24 @@ impl RedisJSON {
         }
     }
 
-    pub fn from_str(data: &str) -> Result<Self, Error> {
-        // Parse the string of data into serde_json::Value.
-        let v: Value = serde_json::from_str(data)?;
-
-        Ok(Self { data: v })
-    }
-
     pub fn set_value(
         &mut self,
         data: &str,
         path: &str,
-        option: &Option<SetOptions>,
+        option: &SetOptions,
+        format: Format
     ) -> Result<bool, Error> {
-        // Parse the string of data into serde_json::Value.
-        let json: Value = serde_json::from_str(data)?;
-
+        let json: Value = RedisJSON::parse_str(data, format)?;
         if path == "$" {
-            if Some(SetOptions::NotExists) == *option {
+            if SetOptions::NotExists == *option {
                 Ok(false)
             } else {
                 self.data = json;
                 Ok(true)
             }
         } else {
             let mut replaced = false;
-            if Some(SetOptions::NotExists) != *option {
+            if SetOptions::NotExists != *option {
                 let current_data = self.data.take();
                 self.data = jsonpath_lib::replace_with(current_data, path, &mut |_v| {
                     replaced = true;
@@ -133,7 +168,7 @@ impl RedisJSON {
             }
             if replaced {
                 Ok(true)
-            } else if Some(SetOptions::AlreadyExists) != *option {
+            } else if SetOptions::AlreadyExists != *option {
                 self.add_value(path, json)
             } else {
                 Ok(false)
@@ -154,9 +189,37 @@ impl RedisJSON {
         Ok(deleted)
     }
 
-    pub fn to_string(&self, path: &str) -> Result<String, Error> {
+    pub fn to_string(&self, path: &str, format: Format) -> Result<String, Error> {
         let results = self.get_doc(path)?;
-        Ok(serde_json::to_string(&results)?)
+        let res = match format {
+            Format::JSON => serde_json::to_string(&results)?,
+            Format::BSON => return Err("Soon to come...".into()) //results.into() as Bson,
+        };
+        Ok(res)
+    }
+
+    pub fn to_json(&self, paths: &mut Vec<String>) -> Result<String, Error> {
+        let mut selector = jsonpath_lib::selector(&self.data);
+        let mut result = paths.drain(..).fold(String::from("{"), |mut acc, path| {
+            let value = match selector(&path) {
+                Ok(s) => match s.first() {
+                    Some(v) => v,
+                    None => &Value::Null,
+                },
+                Err(_) => &Value::Null,
+            };
+            acc.push('\"');
+            acc.push_str(&path);
+            acc.push_str("\":");
+            acc.push_str(value.to_string().as_str());
+            acc.push(',');
+            acc
+        });
+        if result.ends_with(",") {
+            result.pop();
+        }
+        result.push('}');
+        Ok(result.into())
     }
 
     pub fn str_len(&self, path: &str) -> Result<usize, Error> {
@@ -277,6 +340,19 @@ impl RedisJSON {
         }
     }
 
+    pub fn get_memory<'a>(&'a self, path: &'a str) -> Result<usize, Error> {
+        // TODO add better calculation, handle wrappers, internals and length
+        let res = match self.get_doc(path)? {
+            Value::Null => 0,
+            Value::Bool(v) => mem::size_of_val(v),
+            Value::Number(v) => mem::size_of_val(v),
+            Value::String(v) => mem::size_of_val(v), 
+            Value::Array(v) => mem::size_of_val(v), 
+            Value::Object(v) => mem::size_of_val(v),
+        };
+        Ok(res.into())
+    }
+
     pub fn get_doc<'a>(&'a self, path: &'a str) -> Result<&'a Value, Error> {
         let results = jsonpath_lib::select(&self.data, path)?;
         match results.first() {
@@ -292,7 +368,7 @@ pub unsafe extern "C" fn json_rdb_load(rdb: *mut raw::RedisModuleIO, encver: c_i
         0 => RedisJSON {
             data: backward::json_rdb_load(rdb),
         },
-        2 => RedisJSON::from_str(&raw::load_string(rdb)).unwrap(),
+        2 => RedisJSON::from_str(&raw::load_string(rdb), Format::JSON).unwrap(),
         _ => panic!("Can't load old RedisJSON RDB"),
     };
     Box::into_raw(Box::new(json)) as *mut c_void