Docs unified community detection (#807)

* Louvain - unified write result fields

* LPA - unified write result fields

* add write properties to louvain output

* uniform ordering of results

* pull 'write' up to the abstract class because all of them have that

* add write properties for triangle count

* fix the percentiles for triangle count - the result turned by the function is the size rather than needing to be computed via another computation

* store write property for SCC

* tidy up louvain order

* add write properties to SCC docs

* tidy up union find code

* union find docs

* triangle count docs

* update label prop algo

* lpa docs

* Balanced Triads - use a default weight if no weight property passed in

* Balanced Triads - compute percentiles for balanced triads

Author: mneedham

algo/src/main/java/org/neo4j/graphalgo/BalancedTriadsProc.java

@@ -18,6 +18,8 @@
  */
 package org.neo4j.graphalgo;
 
+import com.carrotsearch.hppc.LongLongMap;
+import org.HdrHistogram.Histogram;
 import org.neo4j.graphalgo.api.HugeGraph;
 import org.neo4j.graphalgo.core.GraphLoader;
 import org.neo4j.graphalgo.core.ProcedureConfiguration;
@@ -26,7 +28,7 @@
 import org.neo4j.graphalgo.core.utils.paged.PagedAtomicIntegerArray;
 import org.neo4j.graphalgo.core.write.Exporter;
 import org.neo4j.graphalgo.impl.triangle.*;
-import org.neo4j.graphalgo.results.AbstractResultBuilder;
+import org.neo4j.graphalgo.results.AbstractCommunityResultBuilder;
 import org.neo4j.kernel.api.KernelTransaction;
 import org.neo4j.kernel.internal.GraphDatabaseAPI;
 import org.neo4j.logging.Log;
@@ -68,7 +70,7 @@ public Stream<HugeBalancedTriads.Result> balancedTriadsStream(
         final HugeGraph graph = (HugeGraph) new GraphLoader(api, Pools.DEFAULT)
                 .withOptionalLabel(configuration.getNodeLabelOrQuery())
                 .withOptionalRelationshipType(configuration.getRelationshipOrQuery())
-                .withRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
+                .withOptionalRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
                 .withoutNodeWeights()
                 .withSort(true)
                 .withLog(log)
@@ -113,7 +115,7 @@ public Stream<Result> balancedTriads(
             graph = (HugeGraph) new GraphLoader(api, Pools.DEFAULT)
                     .withOptionalLabel(configuration.getNodeLabelOrQuery())
                     .withOptionalRelationshipType(configuration.getRelationshipOrQuery())
-                    .withRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
+                    .withOptionalRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
                     .withoutNodeWeights()
                     .withSort(true)
                     .withLog(log)
@@ -132,26 +134,33 @@ public Stream<Result> balancedTriads(
 
         // write
         if (configuration.isWriteFlag()) {
+            builder.withWrite(true);
+
+            String balancedProperty = configuration.get("balancedProperty", DEFAULT_BALANCED_PROPERTY);
+            builder.withBalancedProperty(balancedProperty);
+
+            String unbalancedProperty = configuration.get("unbalancedProperty", DEFAULT_UNBALANCED_PROPERTY);
+            builder.withUnbalancedProperty(unbalancedProperty);
+
             try (ProgressTimer timer = builder.timeWrite()) {
                 Exporter.of(api, graph)
                         .withLog(log)
                         .parallel(Pools.DEFAULT, configuration.getConcurrency(), terminationFlag)
                         .build()
                         .write(
-                                configuration.get("balancedProperty", DEFAULT_BALANCED_PROPERTY),
+                                balancedProperty,
                                 balancedTriads.getBalancedTriangles(),
                                 PagedAtomicIntegerArray.Translator.INSTANCE,
-                                configuration.get("unbalancedProperty", DEFAULT_UNBALANCED_PROPERTY),
+                                unbalancedProperty,
                                 balancedTriads.getUnbalancedTriangles(),
                                 PagedAtomicIntegerArray.Translator.INSTANCE);
             }
         }
 
         // result
-        return Stream.of(builder.withNodeCount(graph.nodeCount())
-                .withBalancedTriadCount(balancedTriads.getBalancedTriangleCount())
-                .withUnbalancedTriadCount(balancedTriads.getUnbalancedTriangleCount())
-                .build());
+        builder.withBalancedTriadCount(balancedTriads.getBalancedTriangleCount())
+                .withUnbalancedTriadCount(balancedTriads.getUnbalancedTriangleCount());
+        return Stream.of(builder.buildII(graph.nodeCount(), balancedTriads.getBalancedTriangles()::get));
     }
 
     /**
@@ -162,54 +171,105 @@ public static class Result {
         public final long loadMillis;
         public final long computeMillis;
         public final long writeMillis;
+        public final long postProcessingMillis;
+
         public final long nodeCount;
         public final long balancedTriadCount;
         public final long unbalancedTriadCount;
 
+        public final long p1;
+        public final long p5;
+        public final long p10;
+        public final long p25;
+        public final long p50;
+        public final long p75;
+        public final long p90;
+        public final long p95;
+        public final long p99;
+        public final long p100;
+
+        public final boolean write;
+        public final String balancedProperty;
+        public final String unbalancedProperty;
+
+
         public Result(
                 long loadMillis,
                 long computeMillis,
                 long writeMillis,
-                long nodeCount, long balancedTriadCount, long unbalancedTriadCount) {
+                long postProcessingMillis,
+                long nodeCount, long balancedTriadCount,
+                long unbalancedTriadCount,
+                long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p5, long p1, boolean write, String balancedProperty, String unbalancedProperty) {
             this.loadMillis = loadMillis;
             this.computeMillis = computeMillis;
             this.writeMillis = writeMillis;
+            this.postProcessingMillis = postProcessingMillis;
             this.nodeCount = nodeCount;
             this.balancedTriadCount = balancedTriadCount;
             this.unbalancedTriadCount = unbalancedTriadCount;
+            this.p100 = p100;
+            this.p99 = p99;
+            this.p95 = p95;
+            this.p90 = p90;
+            this.p75 = p75;
+            this.p50 = p50;
+            this.p25 = p25;
+            this.p10 = p10;
+            this.p5 = p5;
+            this.p1 = p1;
+            this.write = write;
+            this.balancedProperty = balancedProperty;
+            this.unbalancedProperty = unbalancedProperty;
         }
     }
 
-    public class BalancedTriadsResultBuilder extends AbstractResultBuilder<Result> {
+    public class BalancedTriadsResultBuilder extends AbstractCommunityResultBuilder<Result> {
 
-        private long nodeCount = 0;
         private long balancedTriadCount = 0;
         private long unbalancedTriadCount = 0;
+        private String balancedProperty;
+        private String unbalancedProperty;
 
-        public BalancedTriadsResultBuilder withNodeCount(long nodeCount) {
-            this.nodeCount = nodeCount;
-            return this;
-        }
 
         public BalancedTriadsResultBuilder withBalancedTriadCount(long balancedTriadCount) {
             this.balancedTriadCount = balancedTriadCount;
             return this;
         }
 
+        public BalancedTriadsResultBuilder withBalancedProperty(String property) {
+            this.balancedProperty = property;
+            return this;
+        }
+
+        public BalancedTriadsResultBuilder withUnbalancedProperty(String property) {
+            this.unbalancedProperty = property;
+            return this;
+        }
+
         public BalancedTriadsResultBuilder withUnbalancedTriadCount(long unbalancedTriadCount) {
             this.unbalancedTriadCount = unbalancedTriadCount;
             return this;
         }
 
         @Override
-        public Result build() {
+        protected Result build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, boolean write) {
             return new Result(
-                    loadDuration,
-                    evalDuration,
-                    writeDuration,
-                    nodeCount,
-                    balancedTriadCount,
-                    unbalancedTriadCount);
+                    loadMillis, computeMillis, writeMillis, postProcessingMillis,  nodeCount, balancedTriadCount, unbalancedTriadCount,
+                    communityHistogram.getValueAtPercentile(100),
+                    communityHistogram.getValueAtPercentile(99),
+                    communityHistogram.getValueAtPercentile(95),
+                    communityHistogram.getValueAtPercentile(90),
+                    communityHistogram.getValueAtPercentile(75),
+                    communityHistogram.getValueAtPercentile(50),
+                    communityHistogram.getValueAtPercentile(25),
+                    communityHistogram.getValueAtPercentile(10),
+                    communityHistogram.getValueAtPercentile(5),
+                    communityHistogram.getValueAtPercentile(1),
+                    write,
+                    balancedProperty,
+                    unbalancedProperty
+            );
         }
     }
 

algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java

@@ -110,6 +110,7 @@ public Stream<LabelPropagationStats> labelPropagation(
 
         final int[] labels = compute(direction, iterations, batchSize, concurrency, graph, stats);
         if (configuration.isWriteFlag(DEFAULT_WRITE) && partitionProperty != null) {
+            stats.withWrite(true);
             write(concurrency, partitionProperty, graph, labels, stats);
         }
 
@@ -218,7 +219,6 @@ private void write(
             HeavyGraph graph,
             int[] labels,
             LabelPropagationStats.Builder stats) {
-        stats.write(true);
         try (ProgressTimer timer = stats.timeWrite()) {
             Exporter.of(dbAPI, graph)
                     .withLog(log)

algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java

@@ -105,7 +105,18 @@ public Stream<LouvainResult> louvain(
         }
 
         if (configuration.isWriteFlag()) {
-            builder.timeWrite(() -> write(graph, louvain.getDendrogram(), louvain.getCommunityIds(), configuration));
+            builder.timeWrite(() -> {
+                String writeProperty = configuration.getWriteProperty("community");
+                boolean includeIntermediateCommunities = configuration.get(INCLUDE_INTERMEDIATE_COMMUNITIES, false);
+                String intermediateCommunitiesWriteProperty = configuration.get(INTERMEDIATE_COMMUNITIES_WRITE_PROPERTY, "communities");
+
+                builder.withWrite(true);
+                builder.withWriteProperty(writeProperty);
+                builder.withIntermediateCommunities(includeIntermediateCommunities);
+                builder.withIntermediateCommunitiesWriteProperty(intermediateCommunitiesWriteProperty);
+
+                write(graph, louvain.getDendrogram(), louvain.getCommunityIds(), configuration, writeProperty, includeIntermediateCommunities, intermediateCommunitiesWriteProperty);
+            });
         }
 
         builder.withIterations(louvain.getLevel());
@@ -170,18 +181,17 @@ public Graph graph(String label, String relationship, ProcedureConfiguration con
                 .load(config.getGraphImpl());
     }
 
-    private void write(Graph graph, int[][] allCommunities, int[] finalCommunities, ProcedureConfiguration configuration) {
+    private void write(Graph graph, int[][] allCommunities, int[] finalCommunities, ProcedureConfiguration configuration, String writeProperty, boolean includeIntermediateCommunities, String intermediateCommunitiesPropertyName) {
         log.debug("Writing results");
-        boolean includeIntermediateCommunities = configuration.get(INCLUDE_INTERMEDIATE_COMMUNITIES, false);
 
         new LouvainCommunityExporter(
                 api,
                 Pools.DEFAULT,
                 configuration.getConcurrency(),
                 graph,
                 finalCommunities.length,
-                configuration.getWriteProperty("community"),
-                configuration.get(INTERMEDIATE_COMMUNITIES_WRITE_PROPERTY, "communities"))
+                writeProperty,
+                intermediateCommunitiesPropertyName)
                 .export(allCommunities, finalCommunities, includeIntermediateCommunities);
     }
 
@@ -205,29 +215,37 @@ public static class LouvainResult {
                 -1,
                 -1,
                 0,
-                new double[] {}, -1);
+                new double[] {}, -1, false, null, false, null);
 
         public final long loadMillis;
         public final long computeMillis;
-        public final long postProcessingMillis;
         public final long writeMillis;
+        public final long postProcessingMillis;
         public final long nodes;
         public final long communityCount;
-        public final long p100;
-        public final long p99;
-        public final long p95;
-        public final long p90;
-        public final long p75;
-        public final long p50;
-        public final long p25;
-        public final long p10;
-        public final long p05;
-        public final long p01;
         public final long iterations;
         public final List<Double> modularities;
         public final double modularity;
-
-        public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations, double[] modularities, double finalModularity) {
+        public final long p1;
+        public final long p5;
+        public final long p10;
+        public final long p25;
+        public final long p50;
+        public final long p75;
+        public final long p90;
+        public final long p95;
+        public final long p99;
+        public final long p100;
+        public final boolean write;
+        public final String writeProperty;
+        public final boolean includeIntermediateCommunities;
+        public final String intermediateCommunitiesWriteProperty;
+
+        public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes,
+                             long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p5, long p1,
+                             long iterations, double[] modularities, double finalModularity,
+                             boolean write, String writeProperty,
+                             boolean includeIntermediateCommunities, String intermediateCommunitiesWriteProperty) {
             this.loadMillis = loadMillis;
             this.computeMillis = computeMillis;
             this.postProcessingMillis = postProcessingMillis;
@@ -242,12 +260,16 @@ public LouvainResult(long loadMillis, long computeMillis, long postProcessingMil
             this.p50 = p50;
             this.p25 = p25;
             this.p10 = p10;
-            this.p05 = p05;
-            this.p01 = p01;
+            this.p5 = p5;
+            this.p1 = p1;
             this.iterations = iterations;
             this.modularities = new ArrayList<>(modularities.length);
+            this.write = write;
+            this.includeIntermediateCommunities = includeIntermediateCommunities;
             for (double mod : modularities) this.modularities.add(mod);
             this.modularity = finalModularity;
+            this.writeProperty = writeProperty;
+            this.intermediateCommunitiesWriteProperty = intermediateCommunitiesWriteProperty;
         }
     }
 
@@ -256,14 +278,22 @@ public static class Builder extends AbstractCommunityResultBuilder<LouvainResult
         private long iterations = -1;
         private double[] modularities = new double[] {};
         private double finalModularity = -1;
+        private String writeProperty;
+        private String intermediateCommunitiesWriteProperty;
+        private boolean includeIntermediateCommunities;
+
+        public Builder withWriteProperty(String writeProperty) {
+            this.writeProperty = writeProperty;
+            return this;
+        }
 
         public Builder withIterations(long iterations) {
             this.iterations = iterations;
             return this;
         }
 
         @Override
-        protected LouvainResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) {
+        protected LouvainResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, boolean write) {
             return new LouvainResult(
                     loadMillis,
                     computeMillis,
@@ -281,7 +311,8 @@ protected LouvainResult build(long loadMillis, long computeMillis, long writeMil
                     communityHistogram.getValueAtPercentile(10),
                     communityHistogram.getValueAtPercentile(5),
                     communityHistogram.getValueAtPercentile(1),
-                    iterations, modularities, finalModularity
+                    iterations, modularities, finalModularity,
+                    write, writeProperty, includeIntermediateCommunities, intermediateCommunitiesWriteProperty
             );
         }
 
@@ -294,6 +325,16 @@ public Builder withFinalModularity(double finalModularity) {
             this.finalModularity = finalModularity;
             return null;
         }
+
+        public Builder withIntermediateCommunitiesWriteProperty(String intermediateCommunitiesWriteProperty) {
+            this.intermediateCommunitiesWriteProperty = intermediateCommunitiesWriteProperty;
+            return null;
+        }
+
+        public Builder withIntermediateCommunities(boolean includeIntermediateCommunities) {
+            this.includeIntermediateCommunities = includeIntermediateCommunities;
+            return this;
+        }
     }