Match existence queries exactly by default, and add an existence_match option

doc/api_reference.rst

@@ -83,6 +83,9 @@ Queries
 .. autoclass:: fairgraph.queries.Regex
    :show-inheritance:
 
+.. autoclass:: fairgraph.queries.Equals
+   :show-inheritance:
+
 Utility classes and functions
 =============================
 

doc/creatingupdating.rst

@@ -40,6 +40,13 @@ attribute, e.g.::
     >>> SoftwareVersion.existence_query_properties
     ('short_name', 'version_identifier')
 
+String properties must match exactly (ignoring case).
+If you want a value in the Knowledge Graph to match when it merely *contains* the local value,
+pass ``existence_match="contains"`` to :meth:`save()` or :meth:`exists()`. 
+Use this with care: the object you save will overwrite any that is found::
+
+    >>> dataset.save(client, space="myspace", existence_match="contains")
+
 
 Saving child nodes
 ==================

doc/queries.rst

@@ -143,7 +143,19 @@ For example, to see only datasets whose name contain the phrase 'patch-clamp'::
              e.g., ``DatasetVersion.property_names``
              or consult the inline help (``help(omcore.DatasetVersion)``).
 
-For a more precise search, pass a :class:`Regex` instead of a plain string::
+A plain string matches any property value that *contains* it.
+To require the whole value to match, pass an :class:`Equals` instead::
+
+    from fairgraph import Equals
+
+    datasets = Dataset.list(client, short_name=Equals("FOO"))
+
+This finds datasets whose short name is "FOO", but not those named "FOO-BAR".
+Case is ignored, but whitespace is not, so "FOO " would not match.
+For a case-sensitive search on a property such as ``name``, see :meth:`by_name` below.
+Only a single value is supported.
+
+For a more flexible search, pass a :class:`Regex` instead of a plain string::
 
     from fairgraph import Regex
 

doc/release_notes.rst

@@ -6,13 +6,41 @@ Release notes
 Version 0.16.0
 ==============
 
+New features
+------------
+
+- Filters may now be given as an :class:`~fairgraph.queries.Equals` instead of a plain string,
+  which selects the KG's "EQUALS" operator in place of "CONTAINS", so that the whole property value
+  must match. For example, ``Dataset.list(client, short_name=Equals("FOO"))`` does not return
+  a dataset called "FOO-BAR". As with :class:`~fairgraph.queries.Regex`, matching ignores case.
+
 Changes in behaviour
 --------------------
 
 - Leading and trailing whitespace is now removed from text properties before they are saved
   Text loaded from the Knowledge Graph is left as it is until saved, so saving a fetched object corrects
   any untrimmed text that is stored.
 
+- The existence query that :meth:`~fairgraph.kgobject.KGObject.exists` and
+  :meth:`~fairgraph.kgobject.KGObject.save` use to decide whether an object is already in the KG
+  now matches string properties exactly, ignoring case, where it previously matched any value that
+  merely *contained* the local one. Saving a :class:`Dataset` with the short name "FOO" therefore
+  no longer finds, and overwrites, an existing one called "FOO-BAR"; likewise, a
+  :class:`DatasetVersion` with the version identifier "1.0" no longer matches "1.0.1"
+  (`#149 <https://github.com/HumanBrainProject/fairgraph/issues/149>`_).
+
+  Whitespace is significant in the match, so an existing object whose name has a leading or trailing space
+  will not be found by a local object whose name lacks it, and a new object will be created.
+  Since :meth:`~fairgraph.kgobject.KGObject.save` now removes leading and trailing whitespace from text properties
+  (see above), this applies to any existing object whose text has stray whitespace.
+  Pipelines that re-save objects and previously relied on such near-matches to update
+  existing nodes may therefore now create new ones.
+
+  To restore the previous behaviour for a call, pass ``existence_match="contains"`` to
+  :meth:`~fairgraph.kgobject.KGObject.exists` or :meth:`~fairgraph.kgobject.KGObject.save`
+  (the default is ``existence_match="equals"``). With ``recursive=True`` the setting applies to
+  child objects as well.
+
 Bug fixes
 ---------
 

fairgraph/__init__.py

@@ -25,7 +25,7 @@
 from .embedded import KGEmbedded, EmbeddedMetadata  # EmbeddedMetadata is a deprecated alias
 from .kgproxy import KGProxy
 from .kgquery import KGQuery
-from .queries import Regex
+from .queries import Regex, Equals
 from .collection import Collection
 from . import client, errors, openminds, utility
 

fairgraph/embedded.py

@@ -89,6 +89,7 @@ def save(
         activity_log: Optional[ActivityLog] = None,
         replace: bool = False,
         ignore_duplicates: bool = False,
+        existence_match: str = "equals",
     ):
         """
         Save to the KG any sub-components of the metadata object that are KGObjects.
@@ -101,7 +102,7 @@ def save(
                         target_space = value.space
                     elif (
                         value.__class__.default_space == "controlled"
-                        and value.exists(client, ignore_duplicates=ignore_duplicates)
+                        and value.exists(client, ignore_duplicates=ignore_duplicates, existence_match=existence_match)
                         and value.space == "controlled"
                     ):
                         continue
@@ -111,7 +112,10 @@ def save(
                         assert space is not None  # for type checking
                         target_space = space
                     if target_space == "controlled":
-                        if value.exists(client, ignore_duplicates=ignore_duplicates) and value.space == "controlled":
+                        if (
+                            value.exists(client, ignore_duplicates=ignore_duplicates, existence_match=existence_match)
+                            and value.space == "controlled"
+                        ):
                             continue
                         else:
                             raise Exception("Cannot write to controlled space")
@@ -121,6 +125,7 @@ def save(
                         recursive=recursive,
                         activity_log=activity_log,
                         ignore_duplicates=ignore_duplicates,
+                        existence_match=existence_match,
                     )
 
 

fairgraph/kgobject.py

@@ -44,7 +44,7 @@
 from .caching import object_cache, save_cache, generate_cache_key
 from .base import ErrorHandling, Releasable, JSONdict
 from .utility.name_matching import KG_NAMELIKE_PROPERTIES, MATCH_TYPES, build_name_regex, matches_name
-from .node import KGNode
+from .node import KGNode, check_existence_match
 from .kgproxy import KGProxy
 from .kgquery import KGQuery
 
@@ -558,7 +558,13 @@ def diff(self, other):
                     differences["properties"][prop.name] = (val_self, val_other)
         return differences
 
-    def exists(self, client: KGClient, ignore_duplicates: bool = False, in_spaces: Optional[List[str]] = None) -> bool:
+    def exists(
+        self,
+        client: KGClient,
+        ignore_duplicates: bool = False,
+        in_spaces: Optional[List[str]] = None,
+        existence_match: str = "equals",
+    ) -> bool:
         """
         Check if this object already exists in the KnowledgeGraph.
 
@@ -567,17 +573,34 @@ def exists(self, client: KGClient, ignore_duplicates: bool = False, in_spaces: O
             ignore_duplicates (bool, optional): Whether to ignore the existence of multiple objects with the same properties
                 (and consider only the first in the list), or to raise an Exception. Defaults to False.
             in_spaces (list of str, optional): If provided, only look for the object in these spaces.
+            existence_match (str, optional): How string properties are compared when looking for the object.
+                Either "equals" (the value in the KG must be the same as the local value, ignoring case - the default),
+                or "contains" (it is enough for the value in the KG to contain the local value,
+                so an object with the name "FOO" will match an existing object with the name "FOO-BAR").
         """
-        obj_exists = self._exists_without_query(client)
+        check_existence_match(existence_match)
+        obj_exists = self._exists_without_query(client, existence_match)
         if obj_exists is not None:
             return obj_exists
-        instances = self._query_matching_instances(client, in_spaces=in_spaces)
+        instances = self._query_matching_instances(client, in_spaces=in_spaces, existence_match=existence_match)
         if not instances:
             return False
-        self._check_for_duplicates(instances, ignore_duplicates)
-        return self._use_matching_instance(client, instances[0])
+        self._check_for_duplicates(instances, ignore_duplicates, existence_match)
+        return self._use_matching_instance(client, instances[0], existence_match)
+
+    def _existence_cache_keys(self, existence_match: str) -> List[Tuple]:
+        """
+        The keys under which the save cache may hold the ID of an object matching this one.
 
-    def _exists_without_query(self, client: KGClient) -> Optional[bool]:
+        An object that matches exactly also matches "contains", but not the other way round,
+        so a "contains" lookup may use either key while an "equals" lookup may use only the first.
+        """
+        equals_key = generate_cache_key(self._build_existence_query("equals"))
+        if existence_match == "equals":
+            return [equals_key]
+        return [equals_key, equals_key + (existence_match,)]
+
+    def _exists_without_query(self, client: KGClient, existence_match: str = "equals") -> Optional[bool]:
         """
         Check if this object exists in the KG, where this can be determined without an existence query,
         i.e. if the object has an ID, if there is no existence query, or if the object is found in the save cache.
@@ -598,16 +621,18 @@ def _exists_without_query(self, client: KGClient) -> Optional[bool]:
             return obj_exists
 
         try:
-            query_filter = self._build_existence_query()
+            query_filter = self._build_existence_query(existence_match)
         except CannotBuildExistenceQuery:
             return False
         if query_filter is None:
             # if there's no existence query and no ID, we allow
             # duplicate entries
             return False
 
-        query_cache_key = generate_cache_key(query_filter)
-        if query_cache_key in save_cache[self.__class__]:
+        query_cache_key = next(
+            (key for key in self._existence_cache_keys(existence_match) if key in save_cache[self.__class__]), None
+        )
+        if query_cache_key is not None:
             # Because the KnowledgeGraph is only eventually consistent, an instance
             # that has just been written to the KG may not appear in the query.
             # Therefore we cache the query when creating an instance and
@@ -624,14 +649,16 @@ def _exists_without_query(self, client: KGClient) -> Optional[bool]:
             return True
         return None
 
-    def _query_matching_instances(self, client: KGClient, in_spaces: Optional[List[str]] = None) -> List[JSONdict]:
+    def _query_matching_instances(
+        self, client: KGClient, in_spaces: Optional[List[str]] = None, existence_match: str = "equals"
+    ) -> List[JSONdict]:
         """
         Run the existence query for this object, and return all matching instances
         (their "@id" and space only).
 
         If the connection is lost while querying, an empty list is returned, with a warning.
         """
-        query_filter = self._build_existence_query()
+        query_filter = self._build_existence_query(existence_match)
         query = self.__class__.generate_minimal_query(client=client, filters=query_filter)
         try:
             response = client.query(
@@ -653,15 +680,16 @@ def _query_matching_instances(self, client: KGClient, in_spaces: Optional[List[s
             raise
         return instances
 
-    def _check_for_duplicates(self, instances: List[JSONdict], ignore_duplicates: bool):
+    def _check_for_duplicates(
+        self, instances: List[JSONdict], ignore_duplicates: bool, existence_match: str = "equals"
+    ):
         if len(instances) > 1 and not ignore_duplicates:
-            # we might want to consider running a second query with "equals" rather than "contains"
             raise Exception(
                 f"Existence query is not specific enough. Type: {self.__class__.__name__}; "
-                f"filters: {self._build_existence_query()}"
+                f"filters: {self._build_existence_query(existence_match)}"
             )
 
-    def _use_matching_instance(self, client: KGClient, match: JSONdict) -> bool:
+    def _use_matching_instance(self, client: KGClient, match: JSONdict, existence_match: str = "equals") -> bool:
         """
         Identify this object with an instance found by the existence query.
 
@@ -679,7 +707,7 @@ def _use_matching_instance(self, client: KGClient, match: JSONdict) -> bool:
         # the instance's actual location takes precedence over any space set locally
         if "https://schema.hbp.eu/myQuery/space" in match:
             self._space = match["https://schema.hbp.eu/myQuery/space"]
-        save_cache[self.__class__][generate_cache_key(self._build_existence_query())] = self.id
+        save_cache[self.__class__][self._existence_cache_keys(existence_match)[-1]] = self.id
         self._update_empty_properties(instance)  # also updates `remote_data`
         return True
 
@@ -728,6 +756,7 @@ def save(
         replace: bool = False,
         ignore_auth_errors: bool = False,
         ignore_duplicates: bool = False,
+        existence_match: str = "equals",
     ):
         """
         Store the current object in the Knowledge Graph, either updating an existing instance
@@ -744,11 +773,16 @@ def save(
             ignore_auth_errors (bool, optional): Whether to continue silently when encountering authentication errors. Defaults to False.
             ignore_duplicates (bool, optional): Whether to ignore the existence of multiple objects with the same properties
                 (and consider only the first in the list), or to raise an Exception. Defaults to False.
+            existence_match (str, optional): How string properties are compared when looking for an existing object
+                to update. Either "equals" (the value in the KG must be the same as the local value, ignoring case - the default),
+                or "contains" (it is enough for the value in the KG to contain the local value,
+                so an object with the name "FOO" will overwrite an existing object with the name "FOO-BAR").
 
         Raises:
             - An `AuthorizationError` if the current user is not authorized to perform the requested operation.
 
         """
+        check_existence_match(existence_match)
         # Done first, so that the existence query, the data that is sent and the comparison with
         # the remote data all use the same values. The object is changed in place on purpose:
         # afterwards it is identical to what is stored in the KG.
@@ -765,7 +799,9 @@ def save(
                         if (
                             isinstance(value, KGObject)
                             and value.__class__.default_space == "controlled"
-                            and value.exists(client, ignore_duplicates=ignore_duplicates)
+                            and value.exists(
+                                client, ignore_duplicates=ignore_duplicates, existence_match=existence_match
+                            )
                             and value.space == "controlled"
                         ):
                             continue
@@ -778,7 +814,9 @@ def save(
                         if target_space == "controlled":
                             assert isinstance(value, KGObject)  # for type checking
                             if (
-                                value.exists(client, ignore_duplicates=ignore_duplicates)
+                                value.exists(
+                                    client, ignore_duplicates=ignore_duplicates, existence_match=existence_match
+                                )
                                 and value.space == "controlled"
                             ):
                                 continue
@@ -790,24 +828,25 @@ def save(
                             recursive=True,
                             activity_log=activity_log,
                             ignore_duplicates=ignore_duplicates,
+                            existence_match=existence_match,
                         )
         if space is None:
             if self.space is None:
                 space = self.__class__.default_space
             else:
                 space = self.space
         logger.info(f"Saving a {self.__class__.__name__} in space {space}")
-        found = self._exists_without_query(client)
+        found = self._exists_without_query(client, existence_match)
         if found is None:
             # We look for the object in all spaces, not only the one we are saving to, to avoid creating duplicates,
             # but if it exists both in the target space and elsewhere, we use the instance in the target space.
-            instances = self._query_matching_instances(client)
+            instances = self._query_matching_instances(client, existence_match=existence_match)
             candidates = [
                 instance for instance in instances if instance.get("https://schema.hbp.eu/myQuery/space") == space
             ] or instances
             if candidates:
-                self._check_for_duplicates(candidates, ignore_duplicates)
-                found = self._use_matching_instance(client, candidates[0])
+                self._check_for_duplicates(candidates, ignore_duplicates, existence_match)
+                found = self._use_matching_instance(client, candidates[0], existence_match)
             else:
                 found = False
         if found and self.space is not None and self.space != space: