diff --git a/builtin-adapter/BuiltinAdapter.cpp b/builtin-adapter/BuiltinAdapter.cpp index 4bcd914..eac3712 100644 --- a/builtin-adapter/BuiltinAdapter.cpp +++ b/builtin-adapter/BuiltinAdapter.cpp @@ -102,7 +102,9 @@ JNIEXPORT jobject JNICALL Java_ru_rt_restream_reindexer_binding_builtin_BuiltinA reindexer_string dsn = rx_string(env, path); reindexer_string vers = rx_string(env, version); reindexer_error error = reindexer_connect(rx, dsn, ConnectOpts(), vers, BindingCapabilities( - kBindingCapabilityResultsWithShardIDs | kBindingCapabilityComplexRank)); + kBindingCapabilityResultsWithShardIDs + | kBindingCapabilityComplexRank + | kBindingCapabilityQueryFormatV2)); env->ReleaseStringUTFChars(path, reinterpret_cast(dsn.p)); env->ReleaseStringUTFChars(version, reinterpret_cast(vers.p)); return j_res(env, error); diff --git a/src/main/java/ru/rt/restream/reindexer/Query.java b/src/main/java/ru/rt/restream/reindexer/Query.java index 45fdabe..bf55f09 100644 --- a/src/main/java/ru/rt/restream/reindexer/Query.java +++ b/src/main/java/ru/rt/restream/reindexer/Query.java @@ -72,6 +72,8 @@ public class Query { private static final int OP_AND = 2; private static final int OP_NOT = 3; + private static final int QUERY_FORMAT_V2_MARKER = 2; + private static final int AGG_SUM = 0; private static final int AGG_AVG = 1; private static final int AGG_FACET = 2; @@ -215,6 +217,7 @@ public enum Condition { this.reindexer = reindexer; this.namespace = namespace; this.transactionContext = transactionContext; + buffer.putUInt8(QUERY_FORMAT_V2_MARKER); buffer.putVString(namespace.getName()); } @@ -437,7 +440,7 @@ public Query where(Query subquery, Condition condition, Object... values) logBuilder.where(nextOperation, subquery, condition.code, values); buffer.putVarUInt32(QUERY_SUB_QUERY_CONDITION) .putVarUInt32(nextOperation) - .putVBytes(subquery.buffer.bytes()) + .putVBytes(subquery.bytes()) .putVarUInt32(condition.code); this.nextOperation = OP_AND; @@ -465,7 +468,7 @@ public Query where(String indexName, Condition condition, Query subquery) .putVarUInt32(nextOperation) .putVString(indexName) .putVarUInt32(condition.code) - .putVBytes(subquery.buffer.bytes()); + .putVBytes(subquery.bytes()); this.nextOperation = OP_AND; this.queryCount++; @@ -967,11 +970,12 @@ public ResultIterator execute() { * @return an iterator over a query result */ public ResultIterator execute(Class itemClass) { - long[] ptVersions = prepareQueryAndGetPayloadTypesVersions(); + byte[] queryData = buildSelectQueryBytes(); + long[] payloadTypeVersions = getPayloadTypeVersions(); RequestContext requestContext = transactionContext != null - ? transactionContext.selectQuery(buffer.bytes(), fetchCount, ptVersions, false) - : reindexer.getBinding().selectQuery(buffer.bytes(), fetchCount, ptVersions, false); + ? transactionContext.selectQuery(queryData, fetchCount, payloadTypeVersions, false) + : reindexer.getBinding().selectQuery(queryData, fetchCount, payloadTypeVersions, false); updatePayloadTypes(requestContext.getQueryResult()); @@ -984,11 +988,12 @@ public ResultIterator execute(Class itemClass) { * @return an iterator over a query result */ public QueryResultJsonIterator executeToJson() { - long[] ptVersions = prepareQueryAndGetPayloadTypesVersions(); + byte[] queryData = buildSelectQueryBytes(); + long[] payloadTypeVersions = getPayloadTypeVersions(); RequestContext requestContext = transactionContext != null - ? transactionContext.selectQuery(buffer.bytes(), fetchCount, ptVersions, true) - : reindexer.getBinding().selectQuery(buffer.bytes(), fetchCount, ptVersions, true); + ? transactionContext.selectQuery(queryData, fetchCount, payloadTypeVersions, true) + : reindexer.getBinding().selectQuery(queryData, fetchCount, payloadTypeVersions, true); QueryResult queryResult = requestContext.getQueryResult(); @@ -1026,49 +1031,29 @@ private void updatePayloadTypes(QueryResult queryResult) { } } - private long[] prepareQueryAndGetPayloadTypesVersions() { + private byte[] buildSelectQueryBytes() { logBuilder.type(SELECT); if (LOGGER.isDebugEnabled()) { debug(); LOGGER.debug(logBuilder.getSql()); } + namespaces.clear(); namespaces.add(namespace); for (Query mergeQuery : mergeQueries) { namespaces.add(mergeQuery.namespace); } - for (Query joinQuery : joinQueries) { - namespaces.add(joinQuery.namespace); - } - - for (Query mergeQuery : mergeQueries) { - for (Query joinQuery : mergeQuery.joinQueries) { - namespaces.add(joinQuery.namespace); - } - } - - buffer.putVarUInt32(QUERY_END); - - for (Query joinQuery : joinQueries) { - buffer.putVarUInt32(joinQuery.joinType); - buffer.writeBytes(joinQuery.buffer.bytes()); - buffer.putVarUInt32(QUERY_END); - } + ByteBuffer queryBuffer = new ByteBuffer(buffer.bytes()); + queryBuffer.putVarUInt32(QUERY_END); + appendJoinQueries(queryBuffer, namespaces); + appendMergeQueries(queryBuffer, namespaces); - for (Query mergeQuery : mergeQueries) { - buffer.putVarUInt32(MERGE); - buffer.writeBytes(mergeQuery.buffer.bytes()); - buffer.putVarUInt32(QUERY_END); - List> joinQueries = mergeQuery.getJoinQueries(); - for (Query joinQuery : joinQueries) { - buffer.putVarUInt32(joinQuery.joinType); - buffer.writeBytes(joinQuery.buffer.bytes()); - buffer.putVarUInt32(QUERY_END); - } - } + return queryBuffer.bytes(); + } + private long[] getPayloadTypeVersions() { return namespaces.stream() .map(ReindexerNamespace::getPayloadType) .mapToLong(pt -> pt == null ? 0 : (pt.getVersion() ^ pt.getStateToken())) @@ -1085,9 +1070,9 @@ public void delete() { LOGGER.debug(logBuilder.getSql()); } if (transactionContext != null) { - transactionContext.deleteQuery(buffer.bytes()); + transactionContext.deleteQuery(toExecutableBytes()); } else { - reindexer.getBinding().deleteQuery(buffer.bytes()); + reindexer.getBinding().deleteQuery(toExecutableBytes()); } } @@ -1263,13 +1248,13 @@ public void update() { LOGGER.debug(logBuilder.getSql()); } if (transactionContext != null) { - transactionContext.updateQuery(buffer.bytes()); + transactionContext.updateQuery(toExecutableBytes()); } else { // There are no support for inner joins for update-queries in Java binding, // so we are using single pt version PayloadType pt = namespace.getPayloadType(); long tmVersion = pt == null ? 0 : (pt.getVersion() ^ pt.getStateToken()); - reindexer.getBinding().updateQuery(buffer.bytes(), new long[]{tmVersion}); + reindexer.getBinding().updateQuery(toExecutableBytes(), new long[]{tmVersion}); } } @@ -1287,6 +1272,13 @@ public List> getMergeQueries() { return mergeQueries; } + /** + * Return query namespace. + */ + ReindexerNamespace getNamespace() { + return namespace; + } + /** * Get the namespace names that are used in the current query. */ @@ -1310,13 +1302,52 @@ String getSql() { return logBuilder.getSql(); } - /** - * Returns all used bytes from the {@link ByteBuffer}. - * - * @return all used bytes from the {@code ByteBuffer} - */ public byte[] bytes() { - return buffer.bytes(); + return toSubQueryBytes(); + } + + private byte[] toSubQueryBytes() { + ByteBuffer copy = new ByteBuffer(buffer.bytes()); + copy.putVarUInt32(QUERY_END); + copy.putVarUInt32(0); + copy.putVarUInt32(0); + return copy.bytes(); + } + + private byte[] toExecutableBytes() { + ByteBuffer queryBuffer = new ByteBuffer(buffer.bytes()); + queryBuffer.putVarUInt32(QUERY_END); + appendJoinQueries(queryBuffer, new ArrayList<>()); + appendMergeQueries(queryBuffer, new ArrayList<>()); + return queryBuffer.bytes(); + } + + private void appendJoinQueries(ByteBuffer target, List> targetNamespaces) { + target.putVarUInt32(joinQueries.size()); + for (Query joinQuery : joinQueries) { + appendQuery(target, joinQuery, joinQuery.joinType, targetNamespaces); + } + } + + private void appendMergeQueries(ByteBuffer target, List> targetNamespaces) { + target.putVarUInt32(mergeQueries.size()); + for (Query mergeQuery : mergeQueries) { + appendQuery(target, mergeQuery, MERGE, targetNamespaces); + } + } + + private void appendQuery(ByteBuffer target, Query query, int queryJoinType, + List> targetNamespaces) { + if (queryJoinType != MERGE) { + targetNamespaces.add(query.namespace); + } + + target.putVarUInt32(queryJoinType); + target.writeBytes(query.buffer.bytes()); + target.putVarUInt32(QUERY_END); + + query.appendJoinQueries(target, targetNamespaces); + query.appendMergeQueries(target, targetNamespaces); } /** diff --git a/src/main/java/ru/rt/restream/reindexer/QueryResultIterator.java b/src/main/java/ru/rt/restream/reindexer/QueryResultIterator.java index 41eb100..8d15927 100644 --- a/src/main/java/ru/rt/restream/reindexer/QueryResultIterator.java +++ b/src/main/java/ru/rt/restream/reindexer/QueryResultIterator.java @@ -123,79 +123,96 @@ public T next() { fetchResults(); } + T item = itemClass.cast(readItem(namespace, itemReader, query)); + position++; + return item; + + } + + private S readItem(ReindexerNamespace expectedNamespace, ItemReader reader, Query queryContext) { ItemParams params = readItemParams(); - T item; - if (params.cptr != 0) { - ByteBuffer buffer = NativeUtils.getNativeBuffer(queryResult.getResultsPtr(), params.cptr, params.nsId); - item = itemReader.readItem(buffer); - } else { - int length = (int) buffer.getUInt32(); - item = itemReader.readItem(new ByteBuffer(buffer.getBytes(length)).rewind()); - } + Query itemQueryContext = getItemQueryContext(queryContext, params.nsId); - long subNsRes = -1L; - if (queryResult.isWithJoined()) { - subNsRes = buffer.getVarUInt(); + ReindexerNamespace itemNamespace = expectedNamespace; + if (query != null && params.nsId < query.getNamespaces().size()) { + itemNamespace = query.getNamespaces().get(params.nsId); } - int nsIndexOffset = getJoinedNsIndexOffset(params.nsId); + S item = readItemData(params, reader, itemNamespace); + readJoinedItems(item, itemQueryContext); + return item; + } - Map> subItemsMap = new HashMap<>(); - for (int nsIndex = 0; nsIndex < subNsRes; nsIndex++) { - if (query == null) { - skipSubItems(); - } else { - readSubItems(nsIndexOffset, subItemsMap, nsIndex); - } + private Query getItemQueryContext(Query defaultQueryContext, int nsId) { + if (query == null || nsId == 0) { + return defaultQueryContext; } - subItemsMap.forEach((key, value) -> writeJoinResult(item, key, value)); + List> mergeQueries = query.getMergeQueries(); + if (nsId <= mergeQueries.size()) { + return mergeQueries.get(nsId - 1); + } - position++; - return item; + return defaultQueryContext; + } + private S readItemData(ItemParams params, ItemReader reader, ReindexerNamespace itemNamespace) { + if (params.cptr != 0) { + ByteBuffer nativeBuffer = NativeUtils.getNativeBuffer(queryResult.getResultsPtr(), params.cptr, + params.nsId); + return reader.readItem(nativeBuffer); + } + + int length = (int) buffer.getUInt32(); + return reader.readItem(new ByteBuffer(buffer.getBytes(length)).rewind()); } - private void readSubItems(int nsIndexOffset, Map> subItemsMap, int nsIndex) { - List> namespaces = query.getNamespaces(); - int nsId = nsIndex + nsIndexOffset; - ReindexerNamespace subItemNamespace = namespaces.get(nsId); - PayloadType subItemPayloadType = subItemNamespace.getPayloadType(); - CtagMatcher ctagMatcher = new CtagMatcher(); - ctagMatcher.read(subItemPayloadType); - Class siClass = subItemNamespace.getItemClass(); - CjsonItemReader subItemItemReader = new CjsonItemReader<>(siClass, ctagMatcher); - String joinField = query.getJoinFields().get(nsIndex); - List subItems = subItemsMap.computeIfAbsent(joinField, s -> new ArrayList<>()); - - int siRes = (int) buffer.getVarUInt(); - for (int i = 0; i < siRes; i++) { - ItemParams subItemParams = readItemParams(); - Object subItem; - if (subItemParams.cptr != 0) { - ByteBuffer buffer = NativeUtils.getNativeBuffer(queryResult.getResultsPtr(), subItemParams.cptr, - nsId); - subItem = subItemItemReader.readItem(buffer); - } else { - int subItemLength = (int) buffer.getUInt32(); - subItem = subItemItemReader.readItem(new ByteBuffer(buffer.getBytes(subItemLength)).rewind()); + private void readJoinedItems(Object item, Query queryContext) { + if (!queryResult.isWithJoined()) { + return; + } + + int joinedFields = (int) buffer.getVarUInt(); + Map> subItemsMap = new HashMap<>(); + for (int joinedField = 0; joinedField < joinedFields; joinedField++) { + int itemsCount = (int) buffer.getVarUInt(); + if (queryContext == null) { + skipJoinedItems(itemsCount); + continue; } - subItems.add(subItem); + + Query joinQuery = queryContext.getJoinQueries().get(joinedField); + ReindexerNamespace joinedNamespace = joinQuery.getNamespace(); + CjsonItemReader joinedItemReader = newItemReader(joinedNamespace); + List subItems = new ArrayList<>(itemsCount); + for (int i = 0; i < itemsCount; i++) { + subItems.add(readItem(joinedNamespace, joinedItemReader, joinQuery)); + } + subItemsMap.computeIfAbsent(queryContext.getJoinFields().get(joinedField), field -> new ArrayList<>()) + .addAll(subItems); } + subItemsMap.forEach((key, value) -> writeJoinResult(item, key, value)); } - private void skipSubItems() { - int siRes = (int) buffer.getVarUInt(); - for (int i = 0; i < siRes; i++) { - ItemParams subItemParams = readItemParams(); - if (subItemParams.cptr == 0) { - int subItemLength = (int) buffer.getUInt32(); - buffer.skip(subItemLength); + private void skipJoinedItems(int itemsCount) { + for (int i = 0; i < itemsCount; i++) { + ItemParams itemParams = readItemParams(); + if (itemParams.cptr == 0) { + int length = (int) buffer.getUInt32(); + buffer.skip(length); } + readJoinedItems(null, null); } } - private void writeJoinResult(T item, String fieldName, List subItems) { + private CjsonItemReader newItemReader(ReindexerNamespace itemNamespace) { + PayloadType payloadType = itemNamespace.getPayloadType(); + CtagMatcher ctagMatcher = new CtagMatcher(); + ctagMatcher.read(payloadType); + return new CjsonItemReader<>(itemNamespace.getItemClass(), ctagMatcher); + } + + private void writeJoinResult(Object item, String fieldName, List subItems) { Field field = FieldUtils.getField(item.getClass(), fieldName, true); if (field == null || !field.isAnnotationPresent(Transient.class)) { @@ -218,26 +235,6 @@ private void writeJoinResult(T item, String fieldName, List subItems) { } } - private int getJoinedNsIndexOffset(int nsId) { - if (query == null) { - return 1; - } - - int offset = 1 + query.getMergeQueries().size(); - int mergedNsIdx = nsId; - - if (mergedNsIdx > 0) { - offset += query.getJoinQueries().size(); - mergedNsIdx--; - } - - for (int i = 0; i < mergedNsIdx; i++) { - offset += query.getMergeQueries().get(i).getJoinQueries().size(); - } - - return offset; - } - private ItemParams readItemParams() { ItemParams params = new ItemParams(); diff --git a/src/main/java/ru/rt/restream/reindexer/binding/Consts.java b/src/main/java/ru/rt/restream/reindexer/binding/Consts.java index 56044f6..bd54440 100644 --- a/src/main/java/ru/rt/restream/reindexer/binding/Consts.java +++ b/src/main/java/ru/rt/restream/reindexer/binding/Consts.java @@ -120,6 +120,7 @@ public final class Consts { // incarnation tags are not supported in java connector public static final long BINDING_CAPABILITY_NAMESPACE_INCARNATIONS = 1 << 2; public static final long BINDING_CAPABILITY_COMPLEX_RANK = 1 << 3; + public static final long BINDING_CAPABILITY_QUERY_FORMAT_V2 = 1 << 5; public static final int RANK_FORMAT_SINGLE_FLOAT = 0; public static final float EMPTY_RANK = Float.NEGATIVE_INFINITY; diff --git a/src/main/java/ru/rt/restream/reindexer/binding/cproto/PhysicalConnection.java b/src/main/java/ru/rt/restream/reindexer/binding/cproto/PhysicalConnection.java index e6ad5bd..1377a98 100644 --- a/src/main/java/ru/rt/restream/reindexer/binding/cproto/PhysicalConnection.java +++ b/src/main/java/ru/rt/restream/reindexer/binding/cproto/PhysicalConnection.java @@ -54,6 +54,7 @@ import static ru.rt.restream.reindexer.binding.Consts.BINDING_CAPABILITY_NAMESPACE_INCARNATIONS; import static ru.rt.restream.reindexer.binding.Consts.BINDING_CAPABILITY_RESULTS_WITH_SHARD_IDS; import static ru.rt.restream.reindexer.binding.Consts.DEF_APP_NAME; +import static ru.rt.restream.reindexer.binding.Consts.BINDING_CAPABILITY_QUERY_FORMAT_V2; import static ru.rt.restream.reindexer.binding.Consts.REINDEXER_VERSION; /** @@ -72,7 +73,7 @@ public class PhysicalConnection implements Connection { static final long CPROTO_MAGIC = 0xEEDD1132L; - static final int CPROTO_VERSION = 0x104; + static final int CPROTO_VERSION = 0x105; static final int CPROTO_HDR_LEN = 16; @@ -139,7 +140,8 @@ public PhysicalConnection(String host, int port, String user, String password, S getAppName(), BINDING_CAPABILITY_RESULTS_WITH_SHARD_IDS | BINDING_CAPABILITY_COMPLEX_RANK - | BINDING_CAPABILITY_NAMESPACE_INCARNATIONS); + | BINDING_CAPABILITY_NAMESPACE_INCARNATIONS + | BINDING_CAPABILITY_QUERY_FORMAT_V2); } catch (Exception e) { onError(e); throw new NetworkException(e);