Buggy unfinished ver

rfc doc

Author: pajgo

.gitignore

@@ -27,3 +27,7 @@ lib
 # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
 node_modules
 test/dump
+
+# So o files
+*.so
+*.o

csrc/.clang-format

@@ -0,0 +1,64 @@
+---
+Language:        Cpp
+AccessModifierOffset: -1
+AlignAfterOpenBracket: true
+AlignConsecutiveAssignments: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+ColumnLimit:     100
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: true
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        4
+UseTab:          Never
+SortIncludes: false
+...
+

csrc/.gitignore

@@ -0,0 +1,6 @@
+*.o
+*.a
+*.so
+*.db
+.vscode
+rmutil/test_vector

csrc/Makefile

@@ -0,0 +1,20 @@
+all: build-host
+
+docker-alpine-build:
+	docker build --rm . -f ./build/Dockerfile-build-image -t alpine_build
+
+docker-redis-build: redis-alpine-module
+	docker build --rm $(CURDIR) -f ./build/Dockerfile-with-module -t redis_custom
+
+redis-alpine-module:
+	docker run -it --rm -v ${PWD}:/src alpine_build make build-host
+
+build-host:
+	cd redis-filtered-sort && $(MAKE) all
+
+test_run: docker-redis-build
+	docker run --rm --name custom_redis redis_custom 
+test_cli:
+	docker exec -it custom_redis redis-cli
+clean:
+	cd redis-filtered-sort && $(MAKE) $@

csrc/build/Dockerfile-build-image

@@ -0,0 +1,16 @@
+FROM alpine
+# RUN apk update \
+#     && apk add --virtual build-dependencies \
+#         build-base \
+#         gcc \
+RUN apk add --no-cache --virtual .build-deps \
+		coreutils \
+		gcc \
+		linux-headers \
+		make \
+		musl-dev \
+    && apk add \
+        bash
+RUN mkdir /src
+VOLUME ["/src"]
+WORKDIR /src

csrc/build/Dockerfile-with-module

@@ -0,0 +1,5 @@
+FROM redis:5.0.5-alpine
+COPY ./redis-filtered-sort/module.so ./filtered_sort.so
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["redis-server", "--loadmodule", "./filtered_sort.so"]
+EXPOSE 6379

csrc/build/docker-compose.yml

@@ -0,0 +1,13 @@
+version: "3"
+services:
+  redismodule:
+    build:
+      context: .
+      dockerfile: Dockerfile-build-image
+    hostname: redismodule
+    volumes:
+      - ${PWD}:/src
+  testredis:
+    build:
+      context: .
+      dockerfile: Dockerfile-with-module

csrc/doc/rfc/00_design.md

@@ -0,0 +1,117 @@
+# Proposal
+As described in Redis Module API modules have more benefints than general LUA scripts. In current environment, lua script locking db for a long period. The main reason of this module: provide non db-locking access for time-consuming operations.
+
+# Design
+* Projects consists from one redis module `filtered_sort` and node.js wrapper `redis-filtered-sort`.
+
+* By default, module going to have 2 build options: Docker based build for redis:5.0-alpine and local system `gcc`
+
+* Exported functions for dev purpose prefixed with "c" prefix
+
+* Current module behaviour copied from existing scripts for compat with lua.
+
+* in Redis Cluster environment: all keys and sets MUST use {prefix} notation, to be stored on same node. Otherwise module won't work correctly.
+
+
+## Expected API and behaviour
+Module will export into Redis command namespace these commands:
+#### cfsort 
+Sorts, filters and paginates provided SET `idSetKey` using data from `metaKeyPattern`.`hashKey` using `sortOrder`,`filter` and stores processed data for `expire` seconds
+
+* **Method usage**: `cfsort idSetKey metaKeyPattern hashKey sortOrder filter(jsonFormattedString) curTime:unixtime [offset:int] [limit:int] [expire:int - ttl in seconds] [keyOnly]`;
+    - **idSetKey**: Redis SET containing id values
+    - **metaKeyPattern**: Pattern to find HASHes of record, "*" is replaced by id from `idSetKey`, eg: _*-metakey_ produce  _id-231-metakey_
+    - **hashKey**: field name from HASHkey formed from `metakeyPattern`
+    - **filter**: Json string with filters
+* **General behaviour**: During work process function going to store postprocessed data into 2 temporary sets(sorted and filterd) in format `{idSetKey}:{order}:{metaKey}:{hasKey}:{filterString}` and reuse them while `ttl` not past. 
+
+    Creates index of this sets in ZSET `tempKeysSet`.
+    > _Definitely I don't understand why this needed, but we have cacheBuster_ in lua. #SeeLater
+
+    - If `metaKeyPattern` and `hashKey` not provided, function going to use `idSetKey` values for sorting, otherwise fetch data from `metaKeyPattern.gsub("*", idSetMemberValue).providedHashKey` and use them for sort.
+    
+    - Same as prev behaviour going to be used in filtering process.
+
+    - If more than one filter provided: `filterString` will contain only "#" filter;
+
+
+### cfsortBust :
+Checking caches for specified `idSetKey` that was created in result of `cfsort` method work. Deletes outdated tempKeys from ZSET `tempKeysSet`
+
+**Method usage**: `cfsort idSetKey curTime:unixtime [expire:int]`
+
+### cfsortAggregate:
+Groups elements by `aggregateParam` from set `idSet` using meta available under `metaKeyPattern` and reports count of each type of criteria.
+
+Currently supports only `sum`;
+
+* **Method usage**: `cfsortAggregate idSetKey metaKeyPattern aggregateParam`
+    - **idSetKey**: Redis SET containing id values
+    - **metaKeyPattern**: Pattern to find HASHes of record, "*" is replaced by id from `idSetKey`, eg: _*-metakey_ produce  _id-231-metakey_
+    - **aggregateParam** : JsonString containing aggregate criteria.
+        ```Javascript
+            {
+                "fieldName": "sum"
+            }
+        ```
+
+
+## Filter Format `filter` parameter:
+Accepting `json` formated object:
+```Javascript
+    {
+        "fieldToFiler": {
+            //Filter params...
+        }
+        "fieldToFilter": "String to find"
+    }
+```
+    
+- **Available criteria**:
+    - `lte`,`gte` - (int) value comparison
+    - `eq`,`ne` - equal or not
+    - `any` - Object containing criteria list to Exclude
+    - `some` - Object containing criteria list to Include
+    - `match` - String contains 
+    - `exists` - if value exists
+
+- **MultiField(#multi)**: Reserved field name for assigning criterias to multiple fields from `meta` record(currently supports only _String contains check_)
+    ```Javascript
+        {
+            "#multi": {
+                "fields": [ 
+                    //Field list 
+                    ...
+                ],
+                "match": "criteria"
+            }
+        }
+    ```
+
+
+**Example**
+```Javascript
+    {
+        "metaFieldName": "contained string",
+        "metaFieldName2": { //Filter params object
+            "gte": 12,
+            "lte": 13,
+        },
+        "metaFieldAnyParams": {
+            "any": {
+                "0": {
+                    "gte": 12,
+                    "lte": 13,
+                },
+                "1": {
+                    "gte": 28,
+                    "lte": 30,
+                }
+            }
+        },
+
+
+    }
+    ```
+        
+

csrc/doc/rfc/01_bench.md

@@ -0,0 +1,2 @@
+# Benchmarking
+...TODO

csrc/redis-filtered-sort/Makefile

@@ -0,0 +1,39 @@
+#set environment variable RM_INCLUDE_DIR to the location of redismodule.h
+ifndef RM_INCLUDE_DIR
+	RM_INCLUDE_DIR=../lib/
+endif
+
+ifndef RMUTIL_LIBDIR
+	RMUTIL_LIBDIR=../lib/rmutil
+endif
+
+# find the OS
+uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
+
+# Compile flags for linux / osx
+ifeq ($(uname_S),Linux)
+	SHOBJ_CFLAGS ?=  -fno-common -g -ggdb
+	SHOBJ_LDFLAGS ?= -shared -Bsymbolic
+else
+	SHOBJ_CFLAGS ?= -dynamic -fno-common -g -ggdb
+	SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup
+endif
+CFLAGS = -I$(RM_INCLUDE_DIR) -Wall -g -O0 -fPIC -lc -lm -std=gnu99  
+CC=gcc
+
+all: rmutil module.so
+
+rmutil: FORCE
+	$(MAKE) -C $(RMUTIL_LIBDIR)
+
+module.so: module.o
+	$(LD) -o $@ module.o $(SHOBJ_LDFLAGS) $(LIBS) -L$(RMUTIL_LIBDIR) -lrmutil -ljansson -lc 
+
+module_sample.so: module_sample.o
+	$(LD) -o $@ module.o $(SHOBJ_LDFLAGS) $(LIBS) -L$(RMUTIL_LIBDIR) -lrmutil -ljansson -lc 
+
+clean:
+	rm -rf *.xo *.so *.o
+	cd $(RMUTIL_LIBDIR) && make clean
+
+FORCE: