Skip to content

Commit 7b83fdd

Browse files
JohTJohT
committed
Add graph visualizations to anomaly detection
1 parent ebf9aed commit 7b83fdd

File tree

5 files changed

+180
-2
lines changed

5 files changed

+180
-2
lines changed

‎domains/anomaly-detection/graphs/TopHub.cypher‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Anomaly Detection Graphs: Find top nodes marked as "hub" including their incoming dependencies and output them in Graphviz format.
2+
3+
// Step 1: Query overall statistics, e.g. min/max weight for later normalization
4+
MATCH (sourceForStatistics)-[dependencyForStatistics:DEPENDS_ON]->(targetForStatistics)
5+
WHERE $projection_node_label IN labels(sourceForStatistics)
6+
AND $projection_node_label IN labels(targetForStatistics)
7+
WITH min(coalesce(dependencyForStatistics.weight25PercentInterfaces, dependencyForStatistics.weight)) AS minWeight
8+
,max(coalesce(dependencyForStatistics.weight25PercentInterfaces, dependencyForStatistics.weight)) AS maxWeight
9+
// Step 2: Query direct dependencies to the target
10+
MATCH (source)-[directDependency:DEPENDS_ON]->(target)
11+
WHERE $projection_node_label IN labels(source)
12+
AND $projection_node_label IN labels(target)
13+
AND target.anomalyScore > 0
14+
AND target.anomalyHubRank = toInteger($projection_node_rank)
15+
ORDER BY directDependency.weight DESC
16+
WITH minWeight
17+
,maxWeight
18+
,target
19+
,collect(source)[0..60] AS sources
20+
,collect(directDependency)[0..60] AS directDependencies
21+
// Step 3: Query dependencies among sources
22+
UNWIND sources AS source1
23+
UNWIND sources AS source2
24+
MATCH (source1)-[indirectDependency:DEPENDS_ON]->(source2)
25+
WITH minWeight
26+
,maxWeight
27+
,target
28+
,directDependencies
29+
,collect(indirectDependency) AS indirectDependencies
30+
WITH *, directDependencies + indirectDependencies AS allDependencies
31+
// Step 4: Prepare results in GraphViz format for all dependencies
32+
UNWIND allDependencies AS dependency
33+
WITH *, (endNode(dependency) = target) AS isTargetEndNode
34+
WITH *, CASE WHEN isTargetEndNode THEN endNode(dependency) ELSE null END AS targetEndNodeOrNull
35+
WITH *, CASE WHEN isTargetEndNode THEN null ELSE endNode(dependency) END AS nonTargetEndNodeOrNull
36+
WITH *, coalesce(dependency.weight25PercentInterfaces, dependency.weight) AS weight
37+
WITH *, toFloat(weight - minWeight) / toFloat(maxWeight - minWeight) AS normalizedWeight
38+
WITH *, round((normalizedWeight * 2) + 0.4, 1) AS penWidth
39+
WITH *, coalesce(target.fqn, target.globalFqn, target.fileName, target.signature, target.name) AS targetName
40+
WITH *, replace(replace(targetName, '.', '.\\n'), '/', '/\\n') AS targetNameSplit
41+
WITH *, "\\n(hub #" + targetEndNodeOrNull.anomalyHubRank + ")" AS hubSubLabel
42+
WITH *, "\"" + targetNameSplit + hubSubLabel + "\"" AS hubLabel
43+
WITH *, coalesce("\"hub\" [label=" + hubLabel+ ";]; ", "") AS hubNode
44+
WITH *, "\"" + startNode(dependency).name + "\"" AS sourceNode
45+
WITH *, coalesce("\"" + nonTargetEndNodeOrNull.name + "\"", "\"hub\"") AS targetNode
46+
WITH *, " -> " + targetNode
47+
+ " [label = " + weight + ";"
48+
+ " penwidth = " + penWidth + ";"
49+
+ " ];" AS graphVizDotNotationEdge
50+
WITH *, hubNode + sourceNode + coalesce(graphVizDotNotationEdge, " [];") AS graphVizDotNotationLine
51+
ORDER BY target.anomalyHubRank DESC, target.name ASC
52+
RETURN DISTINCT graphVizDotNotationLine
53+
//Debugging
54+
//,startNode(dependency).name AS sourceName
55+
//,endNode(dependency).name AS targetName
56+
//,hubNode
57+
//,penWidth
58+
//,normalizedWeight
59+
//,dependency.weight AS weight
60+
//,minWeight
61+
//,maxWeight
62+
LIMIT 100

‎domains/anomaly-detection/graphs/TopHub.template.gv‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// This is a GraphViz dot template file for the visualization of a anomaly archetype graph.
2+
// The main part of the template is marked by the comments "Begin-Template" and "End-Template".
3+
// It also contains a simple example graph.
4+
//
5+
strict digraph top_hub_template {
6+
//Begin-Template
7+
graph [layout = "fdp"; start = "7", fontname = "Helvetica,Arial,sans-serif";];
8+
node [fontsize = 8;];
9+
edge [fontsize = 4;];
10+
node [style = "filled"; color = "0.58 0.75 0.75"; fillcolor = "0.58 0.15 0.99"; margin = "0.00001,0.00001";];
11+
edge [color = "0.58 0.75 0.85"; arrowsize = "0.4";];
12+
"hub" [shape = "doublecircle";];
13+
"hub" [fontsize = 10;];
14+
"hub" [color = "0.52 0.7 0.7"; fillcolor = "0.52 0.4 0.9"; penwidth = 3;];
15+
"limit_hint" [color = "0.52 0.7 0.7"; fillcolor = "0.52 0.4 0.9";]
16+
"limit_hint" [shape = "note"; penwidth = 2; fontsize = 10]
17+
"limit_hint" [label = "limited to\n50 nodes...";]
18+
"limit_hint" -> "hub" // Signals that the number of edges might have been limited
19+
//End-Template
20+
"A" -> "hub" [penwidth = 1.0; label = 1;];
21+
"A" -> "B" [penwidth = 3.0; label = 4;];
22+
"B" -> "hub" [penwidth = 2.0; label = 2;];
23+
}

‎domains/anomaly-detection/graphs/anomalyDetectionGraphVisualization.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/usr/bin/env bash
2+
3+
# Executes selected anomaly detection Cypher queries for GraphViz visualization.
4+
# Visualizes top ranked anomaly archetypes.
5+
# Requires an already running Neo4j graph database with already scanned and analyzed artifacts.
6+
# The reports (csv, dot and svg files) will be written into the sub directory reports/anomaly-detection/{language}_{codeUnit}.
7+
8+
# Requires executeQueryFunctions.sh, visualizeQueryResults.sh, cleanupAfterReportGeneration.sh
9+
10+
# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
11+
set -o errexit -o pipefail
12+
13+
# Overrideable Constants (defaults also defined in sub scripts)
14+
REPORTS_DIRECTORY=${REPORTS_DIRECTORY:-"reports"}
15+
16+
## Get this "scripts/reports" directory if not already set
17+
# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution.
18+
# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
19+
# This way non-standard tools like readlink aren't needed.
20+
ANOMALY_DETECTION_GRAPHS_DIR=${REPORTS_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )}
21+
#echo "anomalyDetectionGraphVisualization: ANOMALY_DETECTION_GRAPHS_DIR=${ANOMALY_DETECTION_GRAPHS_DIR}"
22+
23+
# Get the "scripts" directory by taking the path of this script and going one directory up.
24+
SCRIPTS_DIR=${SCRIPTS_DIR:-"${ANOMALY_DETECTION_GRAPHS_DIR}/../../../scripts"} # Repository directory containing the shell scripts
25+
# echo "anomalyDetectionGraphVisualization: SCRIPTS_DIR=${SCRIPTS_DIR}"
26+
27+
# Get the "scripts/visualization" directory.
28+
VISUALIZATION_SCRIPTS_DIR=${VISUALIZATION_SCRIPTS_DIR:-"${SCRIPTS_DIR}/visualization"} # Repository directory containing the shell scripts for visualization
29+
# echo "anomalyDetectionGraphVisualization: VISUALIZATION_SCRIPTS_DIR=${VISUALIZATION_SCRIPTS_DIR}"
30+
31+
# Define functions to execute cypher queries from within a given file
32+
source "${SCRIPTS_DIR}/executeQueryFunctions.sh"
33+
34+
# Run queries, outputs their results in GraphViz format and create Graph visualizations.
35+
#
36+
# Required Parameters:
37+
# - projection_node_label=...
38+
# Label of the nodes that will be used for the projection. Example: "Package"
39+
# - projection_language=...
40+
# Name of the associated programming language. Examples: "Java", "Typescript"
41+
anomaly_detection_graph_visualization() {
42+
local nodeLabel
43+
nodeLabel=$( extractQueryParameter "projection_node_label" "${@}" )
44+
45+
local language
46+
language=$( extractQueryParameter "projection_language" "${@}" )
47+
48+
echo "anomalyDetectionGraphVisualization: $(date +'%Y-%m-%dT%H:%M:%S%z') Creating ${language} ${nodeLabel} anomaly graph visualizations..."
49+
50+
local report_name="TopHub"
51+
local detail_report_directory_name="${language}_${nodeLabel}"
52+
local detail_report_directory="${FULL_REPORT_DIRECTORY}/${detail_report_directory_name}/GraphVisualizations"
53+
54+
for index in {1..3}; do
55+
echo "anomalyDetectionGraphVisualization: Generating ${report_name} ${index}..."
56+
mkdir -p "${detail_report_directory}"
57+
58+
local queryResultFile="${detail_report_directory}/${report_name}${index}.csv"
59+
execute_cypher "${ANOMALY_DETECTION_GRAPHS_DIR}/${report_name}.cypher" "${@}" "projection_node_rank=${index}" > "${queryResultFile}" || true
60+
61+
source "${SCRIPTS_DIR}/cleanupAfterReportGeneration.sh" "${detail_report_directory}" # Remove empty files
62+
63+
if [ -f "${queryResultFile}" ] ; then
64+
source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${queryResultFile}" --template "${ANOMALY_DETECTION_GRAPHS_DIR}/TopHub.template.gv"
65+
else
66+
break;
67+
fi
68+
done
69+
}
70+
71+
72+
# Create report directory
73+
REPORT_NAME="anomaly-detection"
74+
FULL_REPORT_DIRECTORY="${REPORTS_DIRECTORY}/${REPORT_NAME}"
75+
mkdir -p "${FULL_REPORT_DIRECTORY}"
76+
77+
# Query Parameter key pairs for projection and algorithm side
78+
QUERY_NODE="projection_node_label"
79+
QUERY_LANGUAGE="projection_language"
80+
81+
# -- Detail Reports for each code type -------------------------------
82+
83+
anomaly_detection_graph_visualization "${QUERY_NODE}=Artifact" "${QUERY_LANGUAGE}=Java"
84+
anomaly_detection_graph_visualization "${QUERY_NODE}=Package" "${QUERY_LANGUAGE}=Java"
85+
anomaly_detection_graph_visualization "${QUERY_NODE}=Type" "${QUERY_LANGUAGE}=Java"
86+
anomaly_detection_graph_visualization "${QUERY_NODE}=Module" "${QUERY_LANGUAGE}=Typescript"
87+
88+
# ---------------------------------------------------------------
89+
90+
echo "anomalyDetectionGraphVisualization: $(date +'%Y-%m-%dT%H:%M:%S%z') Successfully finished."

‎scripts/visualization/convertQueryResultCsvToGraphVizDotFile.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,10 @@ outputFilename="${inputFilePath}/${graphName}.gv"
105105
echo "strict digraph ${graphName} {"
106106
# Extract the template content from the template file and remove the begin and end markers
107107
sed -n '/\/\/Begin-Template/,/\/\/End-Template/{//!p;}' "${templateFile}"
108-
# Remove the first (header) line of the CSV file, remove the enclosing double quotes and replace the escaped double quotes by double quotes
108+
# Remove the first (header) line of the CSV file,
109+
# print the first column prefixed with a tab,
110+
# remove the enclosing double quotes and
111+
# replace the escaped double quotes by double quotes
109112
awk -F ',' 'NR>1 {print "\t" $1}' "${inputFilename}" \
110113
| sed 's/^\t\"\"\"/\t"/' \
111114
| sed 's/^\t\"\\\"\"/\t"/' \

‎scripts/visualization/visualizeQueryResults.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ echo "visualizeQueryResults: VISUALIZATION_SCRIPTS_DIR=${VISUALIZATION_SCRIPTS_D
1717
# Read the first unnamed input argument containing the version of the project
1818
inputCsvFileName=""
1919
case "${1}" in
20-
"--"*) ;; # Skipping named command line options to forward them later to the "analyze" command
20+
"--"*) ;; # Skipping named command line options to forward them later to the "convertQueryResultCsvToGraphVizDotFile" command
2121
*)
2222
inputCsvFileName="${1}"
2323
shift || true

0 commit comments

Comments
 (0)