Feature/testing and docs (#36)

* This commit introduces a significant number of improvements to the testing suite and documentation for the agents and parsers modules.

Key changes include:
- Fixed 9 pre-existing failing unit tests in the parsers module.
- Added new unit tests to improve test coverage for `deepagent.py` and `database/utils.py`.
- Created a new `ParserTool` to integrate the agents and parsers modules.
- Added a new integration test for the `ParserTool`.
- Added a new end-to-end test to verify the agent-parser workflow.
- Updated the root README.md with overviews of the agents and parsers modules.
- Added coverage files to .gitignore.

The entire test suite is now passing.

* This commit introduces a significant number of improvements to the testing suite and documentation for the agents and parsers modules.

Key changes include:
- Fixed 9 pre-existing failing unit tests in the parsers module.
- Added new unit tests to improve test coverage for `deepagent.py` and `database/utils.py`.
- Created a new `ParserTool` to integrate the agents and parsers modules.
- Added a new integration test for the `ParserTool`.
- Added a new end-to-end test to verify the agent-parser workflow.
- Updated the root README.md with overviews of the agents and parsers modules.
- Added coverage files to .gitignore.

The entire test suite is now passing.

* Update src/parsers/plantuml_parser.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/parsers/mermaid_parser.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

.gitignore

@@ -289,3 +289,7 @@ profiles.json
 # MSBuildCache
 /MSBuildCacheLogs/
 *.DS_Store
+
+# Coverage reports
+.coverage
+coverage.xml

README.md

@@ -78,6 +78,18 @@ Documentation](https://github.com/SoftwareDevLabs) repository.
 ```
 ---
 
+## 🚀 Modules
+
+### Agents
+
+The `agents` module provides the core components for creating AI agents. It includes a flexible `SDLCFlexibleAgent` that can be configured to use different LLM providers (like OpenAI, Gemini, and Ollama) and a set of tools. The module is designed to be extensible, allowing for the creation of custom agents with specialized skills. Key components include a planner and an executor (currently placeholders for future development) and a `MockAgent` for testing and CI.
+
+### Parsers
+
+The `parsers` module is a powerful utility for parsing various diagram-as-code formats, including PlantUML, Mermaid, and DrawIO. It extracts structured information from diagram files, such as elements, relationships, and metadata, and stores it in a local SQLite database. This allows for complex querying, analysis, and export of diagram data. The module is built on a base parser abstraction, making it easy to extend with new diagram formats. It also includes a suite of utility functions for working with the diagram database, such as exporting to JSON/CSV, finding orphaned elements, and detecting circular dependencies.
+
+---
+
 ## ⚡ Best Practices
 
 - Track prompt versions and results  

src/agents/deepagent.py

@@ -1,6 +1,7 @@
 """LangChain agent integration using OpenAI LLM and standard tools."""
 
 import os
+import re
 import yaml
 from typing import Any, Optional, List
 
@@ -80,7 +81,7 @@ def __init__(
 
         if self.dry_run:
             self.tools = tools or [EchoTool()]
-            self.agent = MockAgent()
+            self.agent = MockAgent(tools=self.tools)
             return
 
         # Configure agent from YAML
@@ -141,13 +142,24 @@ def run(self, input_data: str, session_id: str = "default"):
 
 
 class MockAgent:
-    """A trivial agent used for dry-run and CI that only echoes input."""
-    def __init__(self):
+    """A mock agent for dry-run and CI that can echo or use tools."""
+    def __init__(self, tools: Optional[List[BaseTool]] = None):
         self.last_input = None
+        self.tools = tools or []
+
+    def invoke(self, input_data: dict, config: dict):
+        self.last_input = input_data["input"]
+
+        # Simple logic to simulate tool use for testing
+        if "parse" in self.last_input.lower():
+            for tool in self.tools:
+                if tool.name == "DiagramParserTool":
+                    # Extract file path from prompt (simple parsing)
+                    match = re.search(r"\'(.*?)\'", self.last_input)
+                    if match:
+                        file_path = match.group(1)
+                        return {"output": tool._run(file_path)}
 
-    def invoke(self, input_dict: dict, config: dict):
-    def invoke(self, input: dict, config: dict):
-        self.last_input = input["input"]
         return {"output": f"dry-run-echo:{self.last_input}"}
 
 
@@ -183,7 +195,8 @@ def main():
     except (ValueError, RuntimeError) as e:
         print(f"Error: {e}")
 
+import argparse
+from dotenv import load_dotenv
+
 if __name__ == "__main__":
-    import argparse
-    from dotenv import load_dotenv
     main()

src/parsers/database/models.py

@@ -346,5 +346,7 @@ def get_all_diagrams(self) -> List[DiagramRecord]:
     def delete_diagram(self, diagram_id: int) -> bool:
         """Delete a diagram and all its related records."""
         with sqlite3.connect(self.db_path) as conn:
+            conn.execute("PRAGMA foreign_keys = ON")
             cursor = conn.execute('DELETE FROM diagrams WHERE id = ?', (diagram_id,))
+            conn.commit()
             return cursor.rowcount > 0

src/parsers/drawio_parser.py

@@ -234,22 +234,26 @@ def _determine_element_type(self, style: str, value: str) -> ElementType:
 
     def _determine_relationship_type(self, style: str, value: str) -> str:
         """Determine relationship type based on style and content."""
-        style_lower = style.lower()
-        value_lower = value.lower() if value else ''
-        
-        # Check arrow types and line styles
-        if 'inheritance' in style_lower or 'extends' in value_lower:
-            return 'inheritance'
-        elif 'composition' in style_lower or 'filled' in style_lower:
-            return 'composition'
-        elif 'aggregation' in style_lower:
-            return 'aggregation'
-        elif 'dashed' in style_lower or 'dotted' in style_lower:
-            return 'dependency'
-        elif 'implements' in value_lower:
-            return 'realization'
-        else:
-            return 'association'
+        style_props = self._parse_style(style)
+        value_lower = value.lower() if value else ""
+
+        end_arrow = style_props.get("endArrow")
+        end_fill = style_props.get("endFill")
+
+        if end_arrow == "block" and end_fill == "0":
+            return "inheritance"
+        if end_arrow == "diamond" and end_fill == "1":
+            return "composition"
+        if end_arrow == "diamond" and end_fill == "0":
+            return "aggregation"
+        if style_props.get("dashed") == "1":
+            return "dependency"
+        if "extends" in value_lower:
+            return "inheritance"
+        if "implements" in value_lower:
+            return "realization"
+        
+        return "association"
 
     def _parse_style(self, style: str) -> Dict[str, str]:
         """Parse DrawIO style string into properties."""

src/parsers/mermaid_parser.py

@@ -175,76 +175,84 @@ def _parse_class_relationships(self, line: str, diagram: ParsedDiagram):
     def _parse_flowchart(self, content: str, diagram: ParsedDiagram):
         """Parse flowchart/graph diagram."""
         lines = content.split('\n')[1:]  # Skip diagram type line
-        
-        # Track created nodes to avoid duplicates
         created_nodes = set()
-        
-        for line in lines:
-            line = line.strip()
-            if not line:
-                continue
-            
-            # Node definitions with labels: A[Label] or A(Label) or A{Label}
+
+        def parse_and_create_node(node_str: str):
+            """Parse a node string and create a DiagramElement if it doesn't exist."""
+            node_str = node_str.strip()
             node_patterns = [
-                (r'(\w+)\[([^\]]+)\]', 'rectangular'),
-                (r'(\w+)\(([^)]+)\)', 'rounded'),
-                (r'(\w+)\{([^}]+)\}', 'diamond'),
-                (r'(\w+)\(\(([^)]+)\)\)', 'circle'),
+                (r'^(\w+)\s*\(\((.*)\)\)$', 'circle'),
+                (r'^(\w+)\s*\[(.*)\]$', 'rectangular'),
+                (r'^(\w+)\s*\((.*)\)$', 'rounded'),
+                (r'^(\w+)\s*\{(.*)\}$', 'diamond'),
             ]
-            
             for pattern, shape in node_patterns:
-                match = re.search(pattern, line)
+                match = re.match(pattern, node_str)
                 if match:
-                    node_id = match.group(1)
-                    label = match.group(2)
-                    
+                    node_id, label = match.groups()
                     if node_id not in created_nodes:
                         element = DiagramElement(
-                            id=node_id,
-                            element_type=ElementType.COMPONENT,
-                            name=label,
-                            properties={'shape': shape},
-                            tags=[]
+                            id=node_id, element_type=ElementType.COMPONENT,
+                            name=label, properties={'shape': shape}, tags=[]
                         )
                         diagram.elements.append(element)
                         created_nodes.add(node_id)
+                    return node_id
 
-            # Connection patterns: A --> B or A --- B
+            node_id = node_str
+            if node_id and node_id not in created_nodes:
+                element = DiagramElement(
+                    id=node_id, element_type=ElementType.COMPONENT,
+                    name=node_id, properties={'shape': 'simple'}, tags=[]
+                )
+                diagram.elements.append(element)
+                created_nodes.add(node_id)
+            return node_id
+
+        for line in lines:
+            line = line.strip()
+            if not line:
+                continue
+
             connection_patterns = [
-                (r'(\w+)\s*-->\s*(\w+)', 'directed'),
-                (r'(\w+)\s*---\s*(\w+)', 'undirected'),
-                (r'(\w+)\s*-\.->\s*(\w+)', 'dotted'),
-                (r'(\w+)\s*==>\s*(\w+)', 'thick'),
+                (r'-->', 'directed'), (r'---', 'undirected'),
+                (r'-.->', 'dotted'), (r'==>', 'thick')
             ]
 
-            for pattern, style in connection_patterns:
-                match = re.search(pattern, line)
-                if match:
-                    source = match.group(1)
-                    target = match.group(2)
+            found_connection = False
+            for arrow, style in connection_patterns:
+                if arrow in line:
+                    parts = line.split(arrow, 1)
+                    source_str = parts[0]
+                    target_and_label_str = parts[1]
+
+                    label_match = re.match(r'\s*\|(.*?)\|(.*)', target_and_label_str)
+                    if label_match:
+                        label = label_match.group(1)
+                        target_str = label_match.group(2).strip()
+                    else:
+                        label = None
+                        target_str = target_and_label_str.strip()
 
-                    # Create nodes if they don't exist (simple node without labels)
-                    for node_id in [source, target]:
-                        if node_id not in created_nodes:
-                            element = DiagramElement(
-                                id=node_id,
-                                element_type=ElementType.COMPONENT,
-                                name=node_id,
-                                properties={'shape': 'simple'},
-                                tags=[]
-                            )
-                            diagram.elements.append(element)
-                            created_nodes.add(node_id)
+                    source_id = parse_and_create_node(source_str)
+                    target_id = parse_and_create_node(target_str)
 
-                    relationship = DiagramRelationship(
-                        id=f"rel_{len(diagram.relationships) + 1}",
-                        source_id=source,
-                        target_id=target,
-                        relationship_type='connection',
-                        properties={'style': style},
-                        tags=[]
-                    )
-                    diagram.relationships.append(relationship)
+                    if source_id and target_id:
+                        properties = {'style': style}
+                        if label:
+                            properties['label'] = label
+
+                        relationship = DiagramRelationship(
+                            id=f"rel_{len(diagram.relationships) + 1}",
+                            source_id=source_id, target_id=target_id,
+                            relationship_type='connection', properties=properties, tags=[]
+                        )
+                        diagram.relationships.append(relationship)
+                        found_connection = True
+                        break
+
+            if not found_connection:
+                parse_and_create_node(line)
 
     def _parse_sequence_diagram(self, content: str, diagram: ParsedDiagram):
         """Parse sequence diagram."""
@@ -314,54 +322,37 @@ def _parse_sequence_diagram(self, content: str, diagram: ParsedDiagram):
 
     def _parse_er_diagram(self, content: str, diagram: ParsedDiagram):
         """Parse entity-relationship diagram."""
-        lines = content.split('\n')[1:]  # Skip diagram type line
+        # Parse entities first, handling multiline blocks
+        entity_pattern = r'(\w+)\s*\{([^}]*)\}'
+        entities_found = re.findall(entity_pattern, content, re.DOTALL)
 
+        for entity_name, attributes_text in entities_found:
+            attributes = []
+            if attributes_text:
+                attr_lines = [attr.strip() for attr in attributes_text.split('\n') if attr.strip()]
+                for attr_line in attr_lines:
+                    if attr_line:
+                        attributes.append(attr_line)
+
+            element = DiagramElement(
+                id=entity_name,
+                element_type=ElementType.ENTITY,
+                name=entity_name,
+                properties={'attributes': attributes},
+                tags=[]
+            )
+            diagram.elements.append(element)
+
+        # Remove entity blocks from content to parse relationships
+        content_after_entities = re.sub(entity_pattern, '', content, flags=re.DOTALL)
+        lines = content_after_entities.split('\n')
+
         for line in lines:
             line = line.strip()
             if not line:
                 continue
-            
-            # Entity definition with attributes: ENTITY { attr1 attr2 }
-            entity_match = re.match(r'(\w+)\s*\{([^}]*)\}', line)
-            if entity_match:
-                entity_name = entity_match.group(1)
-                attributes_text = entity_match.group(2)
-                
-                attributes = []
-                if attributes_text:
-                    attr_lines = [attr.strip() for attr in attributes_text.split('\n') if attr.strip()]
-                    for attr_line in attr_lines:
-                        if attr_line:  # Skip empty lines
-                            attributes.append(attr_line)
-                
-                element = DiagramElement(
-                    id=entity_name,
-                    element_type=ElementType.ENTITY,
-                    name=entity_name,
-                    properties={'attributes': attributes},
-                    tags=[]
-                )
-                diagram.elements.append(element)
-                continue
-            
-            # Entity definition without attributes: ENTITY
-            simple_entity_match = re.match(r'^(\w+)$', line)
-            if simple_entity_match and not any(rel_pattern in line for rel_pattern in ['||', '}o', 'o{', '--']):
-                entity_name = simple_entity_match.group(1)
 
-                # Check if entity already exists
-                if not any(elem.id == entity_name for elem in diagram.elements):
-                    element = DiagramElement(
-                        id=entity_name,
-                        element_type=ElementType.ENTITY,
-                        name=entity_name,
-                        properties={'attributes': []},
-                        tags=[]
-                    )
-                    diagram.elements.append(element)
-                continue
-            
-            # Relationship patterns: A ||--o{ B
+            # Relationship patterns
             rel_patterns = [
                 (r'(\w+)\s*\|\|--o\{\s*(\w+)', 'one_to_many'),
                 (r'(\w+)\s*\}o--\|\|\s*(\w+)', 'many_to_one'),
@@ -370,10 +361,9 @@ def _parse_er_diagram(self, content: str, diagram: ParsedDiagram):
             ]
 
             for pattern, rel_type in rel_patterns:
-                match = re.match(pattern, line)
+                match = re.search(pattern, line)
                 if match:
-                    source = match.group(1)
-                    target = match.group(2)
+                    source, target = match.groups()
 
                     relationship = DiagramRelationship(
                         id=f"rel_{len(diagram.relationships) + 1}",

src/parsers/plantuml_parser.py

@@ -58,10 +58,9 @@ def _clean_content(self, content: str) -> str:
         # Remove single-line comments
         content = re.sub(r"'.*$", "", content, flags=re.MULTILINE)
 
-        # Normalize whitespace
-        content = re.sub(r'\s+', ' ', content)
-        
-        return content.strip()
+        # Normalize whitespace but preserve line structure
+        lines = [line.strip() for line in content.split('\n') if line.strip()]
+        return '\n'.join(lines)
 
     def _extract_metadata(self, content: str) -> Dict[str, Any]:
         """Extract metadata like title, skinparam, etc."""
@@ -205,7 +204,7 @@ def _extract_relationships(self, content: str) -> List[DiagramRelationship]:
             # Association: A -- B, A --> B
             (r'(\w+)\s*-->\s*(\w+)', 'association', 'normal'),
             (r'(\w+)\s*<--\s*(\w+)', 'association', 'reverse'),
-            (r'(\w+)\s*--\s*(\w+)(?!\*|o|\|)', 'association', 'normal'),
+            (r'(\w+)\s*(?<!o)(?<!\*)--\s*(\w+)', 'association', 'normal'),
 
             # Dependency: A ..> B, A <.. B
             (r'(\w+)\s*\.\.>\s*(\w+)', 'dependency', 'normal'),