Skip to content

Draft: Query V2 serialization support #140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
slowcheetahzzz wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Draft: Query V2 serialization support #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion builtin-adapter/BuiltinAdapter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,9 @@ JNIEXPORT jobject JNICALL Java_ru_rt_restream_reindexer_binding_builtin_BuiltinA
reindexer_string dsn = rx_string(env, path);
reindexer_string vers = rx_string(env, version);
reindexer_error error = reindexer_connect(rx, dsn, ConnectOpts(), vers, BindingCapabilities(
kBindingCapabilityResultsWithShardIDs | kBindingCapabilityComplexRank));
kBindingCapabilityResultsWithShardIDs
| kBindingCapabilityComplexRank
| kBindingCapabilityQueryFormatV2));
env->ReleaseStringUTFChars(path, reinterpret_cast<const char *>(dsn.p));
env->ReleaseStringUTFChars(version, reinterpret_cast<const char *>(vers.p));
return j_res(env, error);
Expand Down
125 changes: 78 additions & 47 deletions src/main/java/ru/rt/restream/reindexer/Query.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ public class Query<T> {
private static final int OP_AND = 2;
private static final int OP_NOT = 3;

private static final int QUERY_FORMAT_V2_MARKER = 2;

private static final int AGG_SUM = 0;
private static final int AGG_AVG = 1;
private static final int AGG_FACET = 2;
Expand Down Expand Up @@ -215,6 +217,7 @@ public enum Condition {
this.reindexer = reindexer;
this.namespace = namespace;
this.transactionContext = transactionContext;
buffer.putUInt8(QUERY_FORMAT_V2_MARKER);
buffer.putVString(namespace.getName());
}

Expand Down Expand Up @@ -437,7 +440,7 @@ public Query<T> where(Query<?> subquery, Condition condition, Object... values)
logBuilder.where(nextOperation, subquery, condition.code, values);
buffer.putVarUInt32(QUERY_SUB_QUERY_CONDITION)
.putVarUInt32(nextOperation)
.putVBytes(subquery.buffer.bytes())
.putVBytes(subquery.bytes())
.putVarUInt32(condition.code);

this.nextOperation = OP_AND;
Expand Down Expand Up @@ -465,7 +468,7 @@ public Query<T> where(String indexName, Condition condition, Query<?> subquery)
.putVarUInt32(nextOperation)
.putVString(indexName)
.putVarUInt32(condition.code)
.putVBytes(subquery.buffer.bytes());
.putVBytes(subquery.bytes());

this.nextOperation = OP_AND;
this.queryCount++;
Expand Down Expand Up @@ -967,11 +970,12 @@ public ResultIterator<T> execute() {
* @return an iterator over a query result
*/
public <S> ResultIterator<S> execute(Class<S> itemClass) {
long[] ptVersions = prepareQueryAndGetPayloadTypesVersions();
byte[] queryData = buildSelectQueryBytes();
long[] payloadTypeVersions = getPayloadTypeVersions();

RequestContext requestContext = transactionContext != null
? transactionContext.selectQuery(buffer.bytes(), fetchCount, ptVersions, false)
: reindexer.getBinding().selectQuery(buffer.bytes(), fetchCount, ptVersions, false);
? transactionContext.selectQuery(queryData, fetchCount, payloadTypeVersions, false)
: reindexer.getBinding().selectQuery(queryData, fetchCount, payloadTypeVersions, false);

updatePayloadTypes(requestContext.getQueryResult());

Expand All @@ -984,11 +988,12 @@ public <S> ResultIterator<S> execute(Class<S> itemClass) {
* @return an iterator over a query result
*/
public QueryResultJsonIterator executeToJson() {
long[] ptVersions = prepareQueryAndGetPayloadTypesVersions();
byte[] queryData = buildSelectQueryBytes();
long[] payloadTypeVersions = getPayloadTypeVersions();

RequestContext requestContext = transactionContext != null
? transactionContext.selectQuery(buffer.bytes(), fetchCount, ptVersions, true)
: reindexer.getBinding().selectQuery(buffer.bytes(), fetchCount, ptVersions, true);
? transactionContext.selectQuery(queryData, fetchCount, payloadTypeVersions, true)
: reindexer.getBinding().selectQuery(queryData, fetchCount, payloadTypeVersions, true);

QueryResult queryResult = requestContext.getQueryResult();

Expand Down Expand Up @@ -1026,49 +1031,29 @@ private void updatePayloadTypes(QueryResult queryResult) {
}
}

private long[] prepareQueryAndGetPayloadTypesVersions() {
private byte[] buildSelectQueryBytes() {
logBuilder.type(SELECT);
if (LOGGER.isDebugEnabled()) {
debug();
LOGGER.debug(logBuilder.getSql());
}

namespaces.clear();
namespaces.add(namespace);

for (Query<?> mergeQuery : mergeQueries) {
namespaces.add(mergeQuery.namespace);
}

for (Query<?> joinQuery : joinQueries) {
namespaces.add(joinQuery.namespace);
}

for (Query<?> mergeQuery : mergeQueries) {
for (Query<?> joinQuery : mergeQuery.joinQueries) {
namespaces.add(joinQuery.namespace);
}
}

buffer.putVarUInt32(QUERY_END);

for (Query<?> joinQuery : joinQueries) {
buffer.putVarUInt32(joinQuery.joinType);
buffer.writeBytes(joinQuery.buffer.bytes());
buffer.putVarUInt32(QUERY_END);
}
ByteBuffer queryBuffer = new ByteBuffer(buffer.bytes());
queryBuffer.putVarUInt32(QUERY_END);
appendJoinQueries(queryBuffer, namespaces);
appendMergeQueries(queryBuffer, namespaces);

for (Query<?> mergeQuery : mergeQueries) {
buffer.putVarUInt32(MERGE);
buffer.writeBytes(mergeQuery.buffer.bytes());
buffer.putVarUInt32(QUERY_END);
List<Query<?>> joinQueries = mergeQuery.getJoinQueries();
for (Query<?> joinQuery : joinQueries) {
buffer.putVarUInt32(joinQuery.joinType);
buffer.writeBytes(joinQuery.buffer.bytes());
buffer.putVarUInt32(QUERY_END);
}
}
return queryBuffer.bytes();
}

private long[] getPayloadTypeVersions() {
return namespaces.stream()
.map(ReindexerNamespace::getPayloadType)
.mapToLong(pt -> pt == null ? 0 : (pt.getVersion() ^ pt.getStateToken()))
Expand All @@ -1085,9 +1070,9 @@ public void delete() {
LOGGER.debug(logBuilder.getSql());
}
if (transactionContext != null) {
transactionContext.deleteQuery(buffer.bytes());
transactionContext.deleteQuery(toExecutableBytes());
} else {
reindexer.getBinding().deleteQuery(buffer.bytes());
reindexer.getBinding().deleteQuery(toExecutableBytes());
}
}

Expand Down Expand Up @@ -1263,13 +1248,13 @@ public void update() {
LOGGER.debug(logBuilder.getSql());
}
if (transactionContext != null) {
transactionContext.updateQuery(buffer.bytes());
transactionContext.updateQuery(toExecutableBytes());
} else {
// There are no support for inner joins for update-queries in Java binding,
// so we are using single pt version
PayloadType pt = namespace.getPayloadType();
long tmVersion = pt == null ? 0 : (pt.getVersion() ^ pt.getStateToken());
reindexer.getBinding().updateQuery(buffer.bytes(), new long[]{tmVersion});
reindexer.getBinding().updateQuery(toExecutableBytes(), new long[]{tmVersion});
}
}

Expand All @@ -1287,6 +1272,13 @@ public List<Query<?>> getMergeQueries() {
return mergeQueries;
}

/**
* Return query namespace.
*/
ReindexerNamespace<T> getNamespace() {
return namespace;
}

/**
* Get the namespace names that are used in the current query.
*/
Expand All @@ -1310,13 +1302,52 @@ String getSql() {
return logBuilder.getSql();
}

/**
* Returns all used bytes from the {@link ByteBuffer}.
*
* @return all used bytes from the {@code ByteBuffer}
*/
public byte[] bytes() {
return buffer.bytes();
return toSubQueryBytes();
}

private byte[] toSubQueryBytes() {
ByteBuffer copy = new ByteBuffer(buffer.bytes());
copy.putVarUInt32(QUERY_END);
copy.putVarUInt32(0);
copy.putVarUInt32(0);
return copy.bytes();
}

private byte[] toExecutableBytes() {
ByteBuffer queryBuffer = new ByteBuffer(buffer.bytes());
queryBuffer.putVarUInt32(QUERY_END);
appendJoinQueries(queryBuffer, new ArrayList<>());
appendMergeQueries(queryBuffer, new ArrayList<>());
return queryBuffer.bytes();
}

private void appendJoinQueries(ByteBuffer target, List<ReindexerNamespace<?>> targetNamespaces) {
target.putVarUInt32(joinQueries.size());
for (Query<?> joinQuery : joinQueries) {
appendQuery(target, joinQuery, joinQuery.joinType, targetNamespaces);
}
}

private void appendMergeQueries(ByteBuffer target, List<ReindexerNamespace<?>> targetNamespaces) {
target.putVarUInt32(mergeQueries.size());
for (Query<?> mergeQuery : mergeQueries) {
appendQuery(target, mergeQuery, MERGE, targetNamespaces);
}
}

private void appendQuery(ByteBuffer target, Query<?> query, int queryJoinType,
List<ReindexerNamespace<?>> targetNamespaces) {
if (queryJoinType != MERGE) {
targetNamespaces.add(query.namespace);
}

target.putVarUInt32(queryJoinType);
target.writeBytes(query.buffer.bytes());
target.putVarUInt32(QUERY_END);

query.appendJoinQueries(target, targetNamespaces);
query.appendMergeQueries(target, targetNamespaces);
}

/**
Expand Down
Loading
Loading