Fix/231 variable options, fix/232 fetch status (#233)

Author: mvanbrab

CHANGELOG.md

@@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Screencast for the Onto-DESIDE use case (#228).
 
+### Fixed
+
+- Avoided "Error getting variable options..." in templated queries with indirect sources to which the user has no read access (#231).
+- Corrected fetch status in templated queries with indirect sources to which the user has no read access (#232).
+
 ## [2.0.0] - 2025-05-29
 
 ### Added

main/configs/test/config.json

@@ -444,6 +444,38 @@
         "queryLocation": "/sourceQueries/index_example_common_lt.rq"
       }
     },
+    {
+      "id": "9081",
+      "queryGroupId": "gr-test",
+      "queryLocation": "component_material_one_variable.rq",
+      "name": "Component and materials - 1 variable (indirect source & indirect variables; one unauthorized source)",
+      "description": "Query components (including details about materials) with the sources obtained from index files and variables from the sources. One unauthorized indirect source, to check lenient while getting variable options.",
+      "indirectVariables": {
+        "queryLocations": [
+          "variableQueries/components_name_variable.rq"
+        ]
+      },
+      "sourcesIndex": {
+        "url": "http://localhost:8080/example/index-example-with-unauthorized-source-lt",
+        "queryLocation": "/sourceQueries/index_example_common_lt.rq"
+      }
+    },
+    {
+      "id": "9082",
+      "queryGroupId": "gr-test",
+      "queryLocation": "component_material_one_variable.rq",
+      "name": "Component and materials - 1 variable (indirect source & indirect variables; no indirect sources found)",
+      "description": "Query components (including details about materials) with zero sources obtained from index files and variables from the sources.",
+      "indirectVariables": {
+        "queryLocations": [
+          "variableQueries/components_name_variable.rq"
+        ]
+      },
+      "sourcesIndex": {
+        "url": "http://localhost:8080/example/index-example-texon-only-lt",
+        "queryLocation": "/sourceQueries/index_example_common_lt_bad.rq"
+      }
+    },
     {
       "id": "9090",
       "queryGroupId": "gr-test",

main/configs/test/public/queries/sourceQueries/index_example_common_lt_bad.rq

@@ -0,0 +1,5 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+
+SELECT DISTINCT ?source WHERE {
+  ?s rdfs:seeAlso_hihihahahoho ?source.
+}

main/src/components/ActionBar/SourceFetchStatusIcon/SourceFetchStatusIcon.jsx

@@ -1,3 +1,4 @@
+import PendingIcon from '@mui/icons-material/Pending';
 import CheckIcon from '@mui/icons-material/Check';
 import CancelIcon from "@mui/icons-material/Cancel";
 import RemoveCircleOutlineIcon from '@mui/icons-material/RemoveCircleOutline';
@@ -14,7 +15,13 @@ import comunicaEngineWrapper from '../../../comunicaEngineWrapper/comunicaEngine
 function SourceFetchStatusIcon({ source }) {
   const status = comunicaEngineWrapper.getFetchStatusNumber(source);
 
-  if (comunicaEngineWrapper.getFetchSuccess(source)) {
+  if (comunicaEngineWrapper.getFetchSuccess(source) === undefined) {
+    return (
+      <Tooltip title="Not fetched">
+        <PendingIcon size="small" />
+      </Tooltip>
+    );
+  } else if (comunicaEngineWrapper.getFetchSuccess(source)) {
     return (
       <Tooltip title="Fetch was successful">
         <CheckIcon size="small" />

main/src/comunicaEngineWrapper/comunicaEngineWrapper.js

@@ -36,6 +36,7 @@ class ComunicaEngineWrapper {
     this._fetchSuccess = {};
     this._fetchStatusNumber = {};
     this._underlyingFetchFunction = undefined;
+    // LOG console.log(`Comunica engines reset`);
   }
 
   getFetchSuccess(arg) {
@@ -168,13 +169,14 @@ class ComunicaEngineWrapper {
    * @param {array} httpProxies - array of httpProxy definitions
    */
   _prepareQuery(context, httpProxies) {
-    // avoid faulty fetch status for sources cached in Comunica
-    for (const source of context.sources) {
-      this._fetchSuccess[source] = true;
-    }
-    this._underlyingFetchFunction = fetch;
+    // note: there is no need to preset this._fetchSuccess[source] here;
+    // if Comunica caches, we still have the previous value
     if (getDefaultSession().info.isLoggedIn) {
       this._underlyingFetchFunction = authFetch;
+      // LOG console.log(`Using authFetch as underlying fetch function`);
+    } else {
+      this._underlyingFetchFunction = fetch;
+      // LOG console.log(`Using fetch as underlying fetch function`);
     }
     context.fetch = ComunicaEngineWrapper._getWrappedFetchFunction(this._underlyingFetchFunction, httpProxies, this);
   }

main/src/dataProvider/SparqlDataProvider.js

@@ -293,22 +293,19 @@ async function getSourcesFromSourcesIndex(sourcesIndex, httpProxies) {
 
     const bindingsStream = await comunicaEngineWrapper.queryBindings(queryStringIndexSource,
       { lenient: true, sources: [sourcesIndex.url] }, httpProxies, { engine: "link-traversal" });
-    await new Promise((resolve, reject) => {
-      bindingsStream.on('data', (bindings) => {
-        // LOG console.log(`getSourcesFromSourcesIndex bindings: ${bindings.toString()}`);
-        for (const term of bindings.values()) {  // check for 1st value
-          const source = term.value;
-          if (!sourcesList.includes(source)) {
-            // LOG console.log(`getSourcesFromSourcesIndex adding source: ${source}`);
-            sourcesList.push(source);
-          }
-          // we only want the first term, whatever the variable's name is (note: a for ... of loop seems the only way to access it)
-          break;
+    const bindingsArray = await bindingsStream.toArray();
+    for (const bindings of bindingsArray) {
+      // LOG console.log(`getSourcesFromSourcesIndex bindings: ${bindings.toString()}`);
+      for (const term of bindings.values()) {  // check for 1st value
+        const source = term.value;
+        if (!sourcesList.includes(source)) {
+          // LOG console.log(`getSourcesFromSourcesIndex adding source: ${source}`);
+          sourcesList.push(source);
         }
-      });
-      bindingsStream.on('end', resolve);
-      bindingsStream.on('error', reject);
-    });
+        // we only want the first term, whatever the variable's name is (note: a for ... of loop seems the only way to access it)
+        break;
+      }
+    }
   }
   catch (error) {
     throw new Error(`Error adding sources from index: ${error.message}`);
@@ -380,6 +377,9 @@ async function getVariableOptions(query) {
   }
   // END duplicated chunk of code
 
+  if (query.comunicaContext.sources.length === 0) {
+    throw new Error(`Error getting variable options... No sources found.`);
+  }
 
   let variableOptions;
   let queryStringList = [];
@@ -417,35 +417,35 @@ async function getVariableOptions(query) {
 
   try {
     for (const queryString of queryStringList) {
+      // queryBindings with lenient true to avoid errors with unauthorized sources
       const bindingsStream = await comunicaEngineWrapper.queryBindings(queryString,
-        { sources: query.comunicaContext.sources }, query.httpProxies);
-      await new Promise((resolve, reject) => {
-        bindingsStream.on('data', (bindings) => {
-          // LOG console.log(`getVariableOptions bindings: ${bindings.toString()}`);
-          for (const [variable, term] of bindings) {
-            const name = variable.value;
-            if (!variableOptions[name]) {
-              variableOptions[name] = [];
-            }
-            const variableValue = termToSparqlCompatibleString(term);
-            if (variableValue && !variableOptions[name].includes(variableValue)) {
-              // LOG console.log(`getVariableOptions adding variable option for '${name}': ${variableValue}`);
-              variableOptions[name].push(variableValue);
-            }
+        { lenient: true, sources: query.comunicaContext.sources }, query.httpProxies);
+      // convert stream to array (works when no bindings found - handling events 'data', 'end' and 'error' does not work when no bindints found)
+      const bindingsArray = await bindingsStream.toArray();
+      for (const bindings of bindingsArray) {
+        // LOG console.log(`getVariableOptions bindings: ${bindings.toString()}`);
+        for (const [variable, term] of bindings) {
+          const name = variable.value;
+          if (!variableOptions[name]) {
+            variableOptions[name] = [];
           }
-        });
-        bindingsStream.on('end', resolve);
-        bindingsStream.on('error', reject);
-      });
+          const variableValue = termToSparqlCompatibleString(term);
+          if (variableValue && !variableOptions[name].includes(variableValue)) {
+            // LOG console.log(`getVariableOptions adding variable option for '${name}': ${variableValue}`);
+            variableOptions[name].push(variableValue);
+          }
+        }
+      }
     }
   }
   catch (error) {
     throw new Error(`Error getting variable options... ${error.message}`);
   }
 
-  if (variableOptions == {}) {
-    throw new Error(`Error getting variable options... The variable options are empty`);
+  if (Object.keys(variableOptions).length === 0) {
+    throw new Error(`Error getting variable options... No variable options found.`);
   }
+
   return variableOptions;
 }
 

test/cypress/e2e/fetch-status.cy.js

@@ -11,16 +11,16 @@ describe("Fetch Status", () => {
         // Check if the public and restricted sources appear
         cy.get('[aria-label="Sources info"]').click();
 
-        cy.contains("http://localhost:8080/example/favourite-books");
-        cy.contains("http://localhost:8080/example/wish-list");
-
         // Check if the correct icons appear
-        cy.get('[aria-label="Authentication required"]').should("exist");
-        cy.get('[aria-label="Unauthorized"]').should("exist");
-
-        cy.get('[aria-label="No authentication required"]').should("exist");
-        cy.get('[aria-label="Fetch was successful"]').should("exist");
-        
+        cy.contains("http://localhost:8080/example/wish-list").parent().within(() => {
+            cy.get('[aria-label="No authentication required"]').should("exist");
+            cy.get('[aria-label="Fetch was successful"]').should("exist");
+        });
+        cy.contains("http://localhost:8080/example/favourite-books").parent().within(() => {
+            cy.get('[aria-label="Authentication required"]').should("exist");
+            cy.get('[aria-label="Unauthorized"]').should("exist");
+        });
+        cy.get('[aria-label="Not fetched"]').should("not.exist");
 
         // Checking that a non-authorized book is not appearing 
         cy.contains("It Ends With Us").should("not.exist");
@@ -58,16 +58,16 @@ describe("Fetch Status", () => {
         // Check if the public and restricted sources appear
         cy.get('[aria-label="Sources info"]').click();
 
-        cy.contains("http://localhost:8080/example/favourite-books");
-        cy.contains("http://localhost:8080/example/wish-list");
-
         // Check if the correct icons appear
-        cy.get('[aria-label="Authentication required"]').should("exist");
-        cy.get('[aria-label="Fetch Failed"]').should("not.exist");
-        cy.get('[aria-label="Unauthorized"]').should("not.exist");
-
-        cy.get('[aria-label="No authentication required"]').should("exist");
-        cy.get('[aria-label="Fetch was successful"]').should("exist");
+        cy.contains("http://localhost:8080/example/wish-list").parent().within(() => {
+            cy.get('[aria-label="No authentication required"]').should("exist");
+            cy.get('[aria-label="Fetch was successful"]').should("exist");
+        });
+        cy.contains("http://localhost:8080/example/favourite-books").parent().within(() => {
+            cy.get('[aria-label="Authentication required"]').should("exist");
+            cy.get('[aria-label="Fetch was successful"]').should("exist");
+        });
+        cy.get('[aria-label="Not fetched"]').should("not.exist");
 
         // Checking that you see authorized books
         cy.contains("It Ends With Us");
@@ -85,15 +85,45 @@ describe("Fetch Status", () => {
         // Check if the good and bad sources appear
         cy.get('[aria-label="Sources info"]').click();
 
-        // First fetch should be a success
-        cy.contains("http://localhost:8080/example/favourite-musicians");
-        cy.get('[aria-label="No authentication required"]').should("exist");
-        cy.get('[aria-label="Unauthorized"]').should("not.exist");
-        cy.get('[aria-label="Fetch was successful"]').should("exist");
-
-        // the bad source should fail to fetch
-        cy.contains("http://www.example.com/fetch-failure-but-query-success");
-        cy.get('[aria-label="Fetch failed"]').should("exist");
+        // Check if the correct icons appear
+        cy.contains("http://localhost:8080/example/favourite-musicians").parent().within(() => {
+            cy.get('[aria-label="No authentication required"]').should("exist");
+            cy.get('[aria-label="Fetch was successful"]').should("exist");
+        });
+        cy.contains("http://www.example.com/fetch-failure-but-query-success").parent().within(() => {
+            cy.get('[aria-label="Uncertain if authentication is required"]').should("exist");
+            cy.get('[aria-label="Fetch failed"]').should("exist");
+        });
+        cy.get('[aria-label="Not fetched"]').should("not.exist");
+
+    });
+
+    it("Fetch data with no authenticated user, indirect source & indirect variables and one unauthorized source", () => {
+
+        cy.visit("/");
+        cy.contains("For testing only").click();
+        cy.contains("Component and materials - 1 variable (indirect source & indirect variables; one unauthorized source)").click();
+
+        // Fill in the form
+        cy.get('.ra-input-componentName').click();
+        cy.get('li').contains('Component 1').click();
+
+        // Comfirm query
+        cy.get('button[type="submit"]').click();
+
+        // Check if the public and restricted sources appear
+        cy.get('[aria-label="Sources info"]').click();
+
+        // Check if the correct icons appear
+        cy.contains("http://localhost:8080/example/boms").parent().within(() => {
+            cy.get('[aria-label="No authentication required"]').should("exist");
+            cy.get('[aria-label="Fetch was successful"]').should("exist");
+        });
+        cy.contains("http://localhost:8080/example/favourite-books").parent().within(() => {
+            cy.get('[aria-label="Authentication required"]').should("exist");
+            cy.get('[aria-label="Unauthorized"]').should("exist");
+        });
+        cy.get('[aria-label="Not fetched"]').should("not.exist");
 
     });
 

test/cypress/e2e/templated-query.cy.js

@@ -265,6 +265,41 @@ describe("Templated query", () => {
 
   });
 
+  it("Indirect with 1 variable and sources from indexfile, with one unauthorized source", () => {
+
+    cy.visit("/");
+    cy.contains("For testing only").click();
+    cy.contains("Component and materials - 1 variable (indirect source & indirect variables; one unauthorized source)").click();
+
+    // Fill in the form
+    cy.get('.ra-input-componentName').click();
+    cy.get('li').contains('Component 1').click();
+
+    // Comfirm query
+    cy.get('button[type="submit"]').click();
+
+    // Check that it is correctly loaded with and only the correct data appears
+    cy.contains("Finished in:");
+
+    cy.get('.column-componentName').find('span').contains("Component 1").should("exist");
+    cy.get('.column-materialName').find('span').contains("Material 2").should("exist");
+    cy.get('.column-materialName').find('span').contains("Material 1").should("exist");
+
+    cy.get('.column-componentName').find('span').contains("Component 2").should("not.exist");
+    cy.get('.column-componentName').find('span').contains("Component 3").should("not.exist");
+    cy.get('.column-materialName').find('span').contains("Material 6").should("not.exist");
+
+  });
+
+  it("Indirect with 1 variable and sources from indexfile, no indirect sources found", () => {
+
+    cy.visit("/");
+    cy.contains("For testing only").click();
+    cy.contains("Component and materials - 1 variable (indirect source & indirect variables; no indirect sources found)").click();
+
+    cy.contains("Error getting variable options...").should("exist");;
+  });
+
   it("Indirect with 2 variables and sources from indexfile", () => {
     cy.visit("/");
     cy.contains("For testing only").click();