Fix \introspect for Snowflake with foreign key foreign table schemas … (#87)

* Fix \introspect for Snowflake with foreign key foreign table schemas being described as None. Err on side of the the referencing table's schema then.

Author: James Robinson

noteable_magics/sql/meta_commands.py

@@ -526,7 +526,7 @@ def run(self, invoked_as: str, args: List[str]) -> None:
         """
 
         try:
-            ds_id = self.get_datasource_id()
+            ds_id: UUID = self.get_datasource_id()
         except ValueError:
             # Tried to introspect a datasource whose kernel-side handle in the connections dict
             # wasn't coercable back into a UUID. This could be the case for the pre-datasource
@@ -563,7 +563,6 @@ def delta() -> float:
         with ThreadPoolExecutor(
             max_workers=self.MAX_INTROSPECTION_THREADS
         ) as executor, RelationStructureMessager(ds_id) as messenger:
-
             future_to_relation = {
                 executor.submit(
                     self.fully_introspect, inspector, schema_name, relation_name, kind
@@ -622,9 +621,8 @@ def all_table_and_views(self, inspector) -> List[Tuple[str, str, str]]:
         return results
 
     def fully_introspect(
-        self, inspector: 'SchemaStrippingInspector', schema_name: str, relation_name: str, kind: str
+        self, inspector: SchemaStrippingInspector, schema_name: str, relation_name: str, kind: str
     ) -> RelationStructureDescription:
-
         """Drive introspecting into this single relation, making all the necessary Introspector API
         calls to learn all of the relation's sub-structures.
 
@@ -676,19 +674,28 @@ def fully_introspect(
         )
 
     def introspect_foreign_keys(
-        self, inspector, schema_name, relation_name
+        self, inspector: SchemaStrippingInspector, schema_name: str, relation_name: str
     ) -> List[ForeignKeysModel]:
         """Introspect all foreign keys for a table, describing the results as a List[ForeignKeysModel]"""
 
         fkeys: List[ForeignKeysModel] = []
 
         fkey_dicts = inspector.get_foreign_keys(relation_name, schema_name)
 
+        # Convert from SQLA foreign key dicts to our ForeignKeysModel. But beware,
+        # Snowflake driver reports FKs with None for the target table's schema
+        # (at least sometimes?) so in case then err on the referencing table's schema, because
+        # ForeignKeysModel shared between us and Gate hate None for referred_schema.
+
         for fkey in sorted(fkey_dicts, key=lambda d: d['name']):
             fkeys.append(
                 ForeignKeysModel(
                     name=fkey['name'],
-                    referenced_schema=fkey['referred_schema'],
+                    referenced_schema=(
+                        fkey['referred_schema']
+                        if fkey['referred_schema'] is not None
+                        else schema_name
+                    ),
                     referenced_relation=fkey['referred_table'],
                     columns=fkey['constrained_columns'],
                     referenced_columns=fkey['referred_columns'],
@@ -740,7 +747,6 @@ def introspect_indexes(self, inspector, schema_name, relation_name) -> List[Inde
         index_dicts = inspector.get_indexes(relation_name, schema_name)
 
         for index_dict in sorted(index_dicts, key=lambda d: d['name']):
-
             indexes.append(
                 IndexModel(
                     name=index_dict['name'],
@@ -752,7 +758,7 @@ def introspect_indexes(self, inspector, schema_name, relation_name) -> List[Inde
         return indexes
 
     def introspect_primary_key(
-        self, inspector, relation_name, schema_name
+        self, inspector: SchemaStrippingInspector, relation_name: str, schema_name: str
     ) -> Tuple[Optional[str], List[str]]:
         """Introspect the primary key of a table, returning the pkey name and list of columns in the primary key (if any).
 
@@ -770,9 +776,8 @@ def introspect_primary_key(
             return None, []
 
     def introspect_columns(
-        self, inspector: 'SchemaStrippingInspector', schema_name: str, relation_name: str
+        self, inspector: SchemaStrippingInspector, schema_name: str, relation_name: str
     ) -> List[ColumnModel]:
-
         column_dicts = inspector.get_columns(relation_name, schema=schema_name)
 
         retlist = []
@@ -798,38 +803,6 @@ def introspect_columns(
 
         return retlist
 
-    def inform_gate_relation(
-        self,
-        session: requests.Session,
-        datasource_id,
-        relation_description: RelationStructureDescription,
-    ):
-        """POST this `relation_description` up to Gate for storage, relating to `datasource_id`"""
-
-        resp = session.post(
-            f"http://gate.default/api/v1/datasources/{datasource_id}/schema/relation",
-            json=relation_description.dict(),
-        )
-
-        if resp.status_code == 204:
-            print(
-                f'Stored structure of {relation_description.schema_name}.{relation_description.relation_name}'
-            )
-        else:
-            print(
-                f'Failed storing structure of {relation_description.schema_name}.{relation_description.relation_name}: {resp.status_code}, {resp.text}'
-            )
-
-    def inform_gate_completed(
-        self, session: requests.Session, datasource_id: UUID, started_at: datetime
-    ):
-        """Tell gate to forget about any structures known for this datasource older than when we
-        started this introspection run."""
-
-        session.delete(
-            f"http://gate.default/api/v1/datasources/{datasource_id}/schema/relations?older_than={started_at.isoformat()}"
-        )
-
     def get_datasource_id(self) -> UUID:
         """Convert a noteable_magics.sql.connection.Connection's name to the original
         UUID Gate knew it as.
@@ -894,7 +867,6 @@ def __exit__(self, exc_type, exc_value, exc_traceback):
         self._message_gate(completed_introspection=True)
 
     def _message_gate(self, completed_introspection: bool = False):
-
         base_url = f"http://gate.default/api/v1/datasources/{self._datasource_id}/schema/relations"
 
         if self._relations: