Skip to content

Commit bc5beaf

Browse files
hillary-mutisyahillary-mutisya
authored
Update graph layout and relationship calculation (#1751)
- Enable graph layout calcualtion server-side. This allows re-use of layout info and makes UI more responsive. - Use graphology for the graph metrics calculations - Explore more lateral relationships in the topic graph
1 parent d9a2d16 commit bc5beaf

23 files changed

+3317
-359
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -446,3 +446,6 @@ android/samples/mobile/.idea/deploymentTargetSelector.xml
446446

447447
# Mac OS files
448448
.DS_Store
449+
/ts/.claude
450+
/ts/packages/cli/.collection-backups
451+
/ts/packages/cli/.import-states

‎ts/packages/agents/browser/package.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,11 @@
6666
"dompurify": "^3.2.5",
6767
"express": "^4.18.2",
6868
"express-rate-limit": "^7.5.0",
69+
"graphology": "^0.25.4",
70+
"graphology-communities-louvain": "^2.0.1",
71+
"graphology-layout": "^0.6.1",
72+
"graphology-layout-forceatlas2": "^0.10.1",
73+
"graphology-layout-noverlap": "^0.4.1",
6974
"html-to-text": "^9.0.5",
7075
"jsdom": "^26.1.0",
7176
"jsonpath": "^1.1.1",

‎ts/packages/agents/browser/src/agent/knowledge/actions/graphActions.mts‎

Lines changed: 295 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,19 @@ import { searchByEntities } from "../../searchWebMemories.mjs";
77
import { GraphCache, TopicGraphCache } from "../types/knowledgeTypes.mjs";
88
import { calculateTopicImportance } from "../utils/topicMetricsCalculator.mjs";
99
import { getPerformanceTracker } from "../utils/performanceInstrumentation.mjs";
10+
import {
11+
buildGraphologyGraph,
12+
convertToCytoscapeElements,
13+
calculateLayoutQualityMetrics,
14+
type GraphNode,
15+
type GraphEdge,
16+
} from "../utils/graphologyLayoutEngine.mjs";
17+
import {
18+
getGraphologyCache,
19+
setGraphologyCache,
20+
createGraphologyCache,
21+
invalidateAllGraphologyCaches,
22+
} from "../utils/graphologyCache.mjs";
1023
import registerDebug from "debug";
1124
import { openai as ai } from "aiclient";
1225
import { createJsonTranslator } from "typechat";
@@ -130,7 +143,7 @@ export async function getKnowledgeGraphStatus(
130143
if (websiteCollection.knowledgeEntities) {
131144
entityCount = (
132145
websiteCollection.knowledgeEntities as any
133-
).getTotalEntityCount();
146+
).getUniqueEntityCount();
134147
}
135148
} catch (error) {
136149
console.warn("Failed to get entity count:", error);
@@ -956,7 +969,80 @@ export async function getEntityNeighborhood(
956969
},
957970
};
958971

959-
return optimizedResult;
972+
const cacheKey = `entity_neighborhood_${entityId}_${depth}_${maxNodes}`;
973+
let cachedGraph = getGraphologyCache(cacheKey);
974+
975+
if (!cachedGraph) {
976+
debug("[Graphology] Building layout for entity neighborhood...");
977+
const layoutStart = performance.now();
978+
979+
const allEntities = [
980+
optimizedResult.centerEntity,
981+
...optimizedResult.neighbors,
982+
].filter((e) => e !== null);
983+
984+
const graphNodes: GraphNode[] = allEntities.map((entity: any) => ({
985+
id: entity.id,
986+
name: entity.name,
987+
type: entity.type,
988+
confidence: entity.confidence || 0.5,
989+
count: entity.count || 1,
990+
importance: entity.importance || entity.degree || 0,
991+
}));
992+
993+
const graphEdges: GraphEdge[] = optimizedResult.relationships.map(
994+
(rel: any) => ({
995+
from: rel.fromEntity,
996+
to: rel.toEntity,
997+
type: rel.relationshipType,
998+
confidence: rel.confidence || 0.5,
999+
strength: rel.confidence || 0.5,
1000+
}),
1001+
);
1002+
1003+
const graph = buildGraphologyGraph(graphNodes, graphEdges, {
1004+
nodeLimit: maxNodes * 2,
1005+
minEdgeConfidence: 0.2,
1006+
denseClusterThreshold: 50,
1007+
forceAtlas2Iterations: 100,
1008+
noverlapIterations: 300,
1009+
});
1010+
1011+
const cytoscapeElements = convertToCytoscapeElements(graph, 1500);
1012+
const layoutMetrics = calculateLayoutQualityMetrics(graph);
1013+
const layoutDuration = performance.now() - layoutStart;
1014+
1015+
cachedGraph = createGraphologyCache(
1016+
graph,
1017+
cytoscapeElements,
1018+
layoutDuration,
1019+
layoutMetrics.avgSpacing,
1020+
);
1021+
1022+
setGraphologyCache(cacheKey, cachedGraph);
1023+
1024+
debug(
1025+
`[Graphology] Layout complete in ${layoutDuration.toFixed(2)}ms`,
1026+
);
1027+
debug(
1028+
`[Graphology] Average node spacing: ${layoutMetrics.avgSpacing.toFixed(2)}`,
1029+
);
1030+
} else {
1031+
debug("[Graphology] Using cached layout");
1032+
}
1033+
1034+
return {
1035+
...optimizedResult,
1036+
metadata: {
1037+
...optimizedResult.metadata,
1038+
graphologyLayout: {
1039+
elements: cachedGraph.cytoscapeElements,
1040+
layoutDuration: cachedGraph.metadata.layoutDuration,
1041+
avgSpacing: cachedGraph.metadata.avgSpacing,
1042+
communityCount: cachedGraph.metadata.communityCount,
1043+
},
1044+
},
1045+
};
9601046
} catch (error) {
9611047
console.error("Error getting entity neighborhood:", error);
9621048
return {
@@ -1373,10 +1459,95 @@ export async function getGlobalImportanceLayer(
13731459
size: entity.size,
13741460
}));
13751461

1462+
// Build graphology layout for entities
1463+
const cacheKey = `entity_importance_${maxNodes}`;
1464+
let cachedGraph = getGraphologyCache(cacheKey);
1465+
1466+
if (!cachedGraph) {
1467+
debug(
1468+
"[Graphology] Building layout for entity importance layer...",
1469+
);
1470+
const layoutStart = performance.now();
1471+
1472+
const graphNodes: GraphNode[] = optimizedEntities.map(
1473+
(entity: any) => ({
1474+
id: entity.id || entity.name,
1475+
name: entity.name,
1476+
type: entity.type || "entity",
1477+
confidence: entity.confidence || 0.5,
1478+
count: entity.count || 1,
1479+
importance: entity.importance || 0,
1480+
}),
1481+
);
1482+
1483+
const graphEdges: GraphEdge[] = optimizedRelationships.map(
1484+
(rel: any) => ({
1485+
from: rel.fromEntity,
1486+
to: rel.toEntity,
1487+
type: rel.relationshipType,
1488+
confidence: rel.confidence || 0.5,
1489+
strength: rel.confidence || 0.5,
1490+
}),
1491+
);
1492+
1493+
const graph = buildGraphologyGraph(graphNodes, graphEdges, {
1494+
nodeLimit: maxNodes * 2,
1495+
minEdgeConfidence: 0.3,
1496+
denseClusterThreshold: 100,
1497+
});
1498+
1499+
const cytoscapeElements = convertToCytoscapeElements(graph, 2000);
1500+
const layoutMetrics = calculateLayoutQualityMetrics(graph);
1501+
const layoutDuration = performance.now() - layoutStart;
1502+
1503+
cachedGraph = createGraphologyCache(
1504+
graph,
1505+
cytoscapeElements,
1506+
layoutDuration,
1507+
layoutMetrics.avgSpacing,
1508+
);
1509+
1510+
setGraphologyCache(cacheKey, cachedGraph);
1511+
1512+
debug(
1513+
`[Graphology] Entity layout complete in ${layoutDuration.toFixed(2)}ms`,
1514+
);
1515+
debug(
1516+
`[Graphology] Average node spacing: ${layoutMetrics.avgSpacing.toFixed(2)}`,
1517+
);
1518+
} else {
1519+
debug("[Graphology] Using cached entity layout");
1520+
}
1521+
1522+
// Enrich entities with graphology colors and sizes
1523+
const enrichedEntities = optimizedEntities.map((entity: any) => {
1524+
const graphElement = cachedGraph!.cytoscapeElements.find(
1525+
(el: any) =>
1526+
el.data?.id === entity.id || el.data?.label === entity.name,
1527+
);
1528+
if (graphElement?.data) {
1529+
return {
1530+
...entity,
1531+
color: graphElement.data.color,
1532+
size: graphElement.data.size,
1533+
community: graphElement.data.community,
1534+
};
1535+
}
1536+
return entity;
1537+
});
1538+
13761539
return {
1377-
entities: optimizedEntities,
1540+
entities: enrichedEntities,
13781541
relationships: optimizedRelationships,
1379-
metadata: metadata,
1542+
metadata: {
1543+
...metadata,
1544+
graphologyLayout: {
1545+
elements: cachedGraph.cytoscapeElements,
1546+
layoutDuration: cachedGraph.metadata.layoutDuration,
1547+
avgSpacing: cachedGraph.metadata.avgSpacing,
1548+
communityCount: cachedGraph.metadata.communityCount,
1549+
},
1550+
},
13801551
};
13811552
} catch (error) {
13821553
console.error("Error getting global importance layer:", error);
@@ -1790,6 +1961,8 @@ export function invalidateTopicCache(websiteCollection: any): void {
17901961
lastUpdated: 0,
17911962
isValid: false,
17921963
});
1964+
// Also clear the graphology layout cache
1965+
invalidateAllGraphologyCaches();
17931966
}
17941967

17951968
// Ensure topic graph data is cached for fast access
@@ -1854,6 +2027,28 @@ async function ensureTopicGraphCache(websiteCollection: any): Promise<void> {
18542027
);
18552028
}
18562029

2030+
// Calculate childCount for each topic
2031+
tracker.startOperation("ensureTopicGraphCache.calculateChildCounts");
2032+
const childCountMap = new Map<string, number>();
2033+
for (const topic of topics) {
2034+
childCountMap.set(topic.topicId, 0);
2035+
}
2036+
for (const topic of topics) {
2037+
if (topic.parentTopicId) {
2038+
const currentCount =
2039+
childCountMap.get(topic.parentTopicId) || 0;
2040+
childCountMap.set(topic.parentTopicId, currentCount + 1);
2041+
}
2042+
}
2043+
for (const topic of topics) {
2044+
topic.childCount = childCountMap.get(topic.topicId) || 0;
2045+
}
2046+
tracker.endOperation(
2047+
"ensureTopicGraphCache.calculateChildCounts",
2048+
topics.length,
2049+
topics.length,
2050+
);
2051+
18572052
// Build relationships from parent-child structure
18582053
tracker.startOperation("ensureTopicGraphCache.buildTopicRelationships");
18592054
let relationships = buildTopicRelationships(topics);
@@ -2630,18 +2825,39 @@ export async function getTopicImportanceLayer(
26302825
let lateralRelationships: any[] = [];
26312826

26322827
if (websiteCollection.topicRelationships) {
2828+
// Fetch ALL lateral relationships (all types, min strength 0.3)
26332829
const lateralRels =
26342830
websiteCollection.topicRelationships.getRelationshipsForTopicsOptimized(
26352831
selectedTopicIdsArray,
26362832
0.3,
26372833
);
26382834

2639-
lateralRelationships = lateralRels.map((rel: any) => ({
2640-
from: rel.fromTopic,
2641-
to: rel.toTopic,
2642-
type: rel.relationshipType,
2643-
strength: rel.strength,
2644-
}));
2835+
// Build parent map to filter out sibling relationships
2836+
const parentMap = new Map<string, string>();
2837+
for (const topic of selectedTopics) {
2838+
if (topic.parentTopicId) {
2839+
parentMap.set(topic.topicId, topic.parentTopicId);
2840+
}
2841+
}
2842+
2843+
// Filter out sibling relationships (topics with same parent)
2844+
lateralRelationships = lateralRels
2845+
.filter((rel: any) => {
2846+
const parentA = parentMap.get(rel.fromTopic);
2847+
const parentB = parentMap.get(rel.toTopic);
2848+
// Skip if both have same parent (siblings)
2849+
return !(parentA && parentB && parentA === parentB);
2850+
})
2851+
.map((rel: any) => ({
2852+
from: rel.fromTopic,
2853+
to: rel.toTopic,
2854+
type: rel.relationshipType,
2855+
strength: rel.strength,
2856+
}));
2857+
2858+
debug(
2859+
`[Topic Graph] Fetched ${lateralRels.length} lateral relationships, kept ${lateralRelationships.length} after filtering siblings`,
2860+
);
26452861
}
26462862

26472863
const selectedRelationships = [
@@ -2672,10 +2888,78 @@ export async function getTopicImportanceLayer(
26722888
layer: "topic_importance",
26732889
};
26742890

2891+
const cacheKey = `topic_importance_${maxNodes}`;
2892+
let cachedGraph = getGraphologyCache(cacheKey);
2893+
2894+
if (!cachedGraph) {
2895+
debug("[Graphology] Building layout for topic importance layer...");
2896+
const layoutStart = performance.now();
2897+
2898+
const graphNodes: GraphNode[] = topicsWithMetrics.map(
2899+
(topic: any) => ({
2900+
id: topic.topicId,
2901+
name: topic.topicName,
2902+
type: "topic",
2903+
confidence: topic.confidence || 0.5,
2904+
count: topic.descendantCount || 1,
2905+
importance: topic.importance || 0,
2906+
level: topic.level || 0,
2907+
parentId: topic.parentTopicId,
2908+
childCount: topic.childCount || 0,
2909+
}),
2910+
);
2911+
2912+
const graphEdges: GraphEdge[] = selectedRelationships.map(
2913+
(rel: any) => ({
2914+
from: rel.from,
2915+
to: rel.to,
2916+
type: rel.type,
2917+
confidence: rel.strength || rel.confidence || 0.5,
2918+
strength: rel.strength || 0.5,
2919+
}),
2920+
);
2921+
2922+
const graph = buildGraphologyGraph(graphNodes, graphEdges, {
2923+
nodeLimit: maxNodes * 2,
2924+
minEdgeConfidence: 0.3,
2925+
denseClusterThreshold: 100,
2926+
});
2927+
2928+
const cytoscapeElements = convertToCytoscapeElements(graph, 2000);
2929+
const layoutMetrics = calculateLayoutQualityMetrics(graph);
2930+
const layoutDuration = performance.now() - layoutStart;
2931+
2932+
cachedGraph = createGraphologyCache(
2933+
graph,
2934+
cytoscapeElements,
2935+
layoutDuration,
2936+
layoutMetrics.avgSpacing,
2937+
);
2938+
2939+
setGraphologyCache(cacheKey, cachedGraph);
2940+
2941+
debug(
2942+
`[Graphology] Layout complete in ${layoutDuration.toFixed(2)}ms`,
2943+
);
2944+
debug(
2945+
`[Graphology] Average node spacing: ${layoutMetrics.avgSpacing.toFixed(2)}`,
2946+
);
2947+
} else {
2948+
debug("[Graphology] Using cached layout");
2949+
}
2950+
26752951
return {
26762952
topics: topicsWithMetrics,
26772953
relationships: selectedRelationships,
2678-
metadata,
2954+
metadata: {
2955+
...metadata,
2956+
graphologyLayout: {
2957+
elements: cachedGraph.cytoscapeElements,
2958+
layoutDuration: cachedGraph.metadata.layoutDuration,
2959+
avgSpacing: cachedGraph.metadata.avgSpacing,
2960+
communityCount: cachedGraph.metadata.communityCount,
2961+
},
2962+
},
26792963
};
26802964
} catch (error) {
26812965
console.error("Error getting topic importance layer:", error);

0 commit comments

Comments
 (0)