Skip to content

External Controlled Vocabularies: Support vocab specific retrieval filtering #12331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
qqmyers wants to merge 8 commits into
base: develop
Choose a base branch
from
+113 −83
Open

External Controlled Vocabularies: Support vocab specific retrieval filtering #12331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3240448
remove logging
qqmyers Jan 7, 2026
06b130e
per vocab retrieval filtering
qqmyers Feb 24, 2026
a56c90a
Merge remote-tracking branch 'IQSS/develop' into DNO-supportORCIDandR…
qqmyers Feb 27, 2026
dea0e78
allow null top-level retrieval-uri
qqmyers Mar 27, 2026
6e0054f
Merge remote-tracking branch 'IQSS/develop' into DNO-supportORCIDandR…
qqmyers Apr 15, 2026
375a1ee
release note
qqmyers Apr 15, 2026
8908851
fix multi-vocab retrieval filtering
qqmyers May 4, 2026
263cd74
Merge remote-tracking branch 'IQSS/develop' into DNO-supportORCIDandR…
qqmyers May 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions doc/release-notes/12331-multi-PID-support-in-external-vocabs.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Dataverse now allows use of multiple PIDs/PID services per field when using external vocabulary scripts.
One example of this is allowing ORCID and ROR values in the author or contact field.
194 changes: 111 additions & 83 deletions src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,10 +406,11 @@ public void registerExternalVocabValues(DatasetField df) {
public Set<String> getIndexableStringsByTermUri(String termUri, JsonObject cvocEntry, String indexingField) {
Set<String> strings = new HashSet<>();
JsonObject jo = getExternalVocabularyValue(termUri);
JsonObject filtering = cvocEntry.getJsonObject("retrieval-filtering");
String termUriField = cvocEntry.getJsonString("term-uri-field").getString();

if (jo != null) {
JsonObject filtering = getRetrievalFilteringObject(cvocEntry, termUri);
String termUriField = cvocEntry.getJsonString("term-uri-field").getString();

try {
for (String key : jo.keySet()) {
String indexIn = filtering.getJsonObject(key).getString("indexIn", null);
Expand Down Expand Up @@ -467,6 +468,24 @@ public Set<String> getIndexableStringsByTermUri(String termUri, JsonObject cvocE
return strings;
}

private JsonObject getRetrievalFilteringObject(JsonObject cvocEntry, String termUri) {
JsonObject filtering = cvocEntry.getJsonObject("retrieval-filtering");
//Check for per-vocab filtering
JsonObject vocabs = cvocEntry.getJsonObject("vocabs");
for (String key : vocabs.keySet()) {
JsonObject vocab = vocabs.getJsonObject(key);
if (vocab.containsKey("uriSpace")) {
if (termUri.startsWith(vocab.getString("uriSpace"))) {
if (vocab.containsKey("retrieval-filtering")) {
filtering = vocab.getJsonObject("retrieval-filtering");
}
break;
}
}
}
return filtering;
}

/**
* Perform a query to retrieve a cached value from the externalvocabularvalue table
* @param termUri
Expand Down Expand Up @@ -499,7 +518,7 @@ public JsonObject getExternalVocabularyValue(String termUri) {
* @param relatedDatasetFields - siblings or childs of the term
*/
public void registerExternalTerm(JsonObject cvocEntry, String term, List<DatasetField> relatedDatasetFields) {
String retrievalUri = cvocEntry.getString("retrieval-uri");
String retrievalUri = cvocEntry.getString("retrieval-uri", null);
String termUriFieldName = cvocEntry.getString("term-uri-field");
String prefix = cvocEntry.getString("prefix", null);
if(StringUtils.isBlank(term)) {
Expand All @@ -514,6 +533,13 @@ public void registerExternalTerm(JsonObject cvocEntry, String term, List<Dataset
if (vocab.containsKey("uriSpace")) {
if (term.startsWith(vocab.getString("uriSpace"))) {
isExternal = true;
// A specific vocabulary can override the top-level retrieval-uri and prefix
if (vocab.containsKey("retrieval-uri")) {
retrievalUri = vocab.getString("retrieval-uri");
}
if (vocab.containsKey("prefix")) {
prefix = vocab.getString("prefix");
}
break;
}
}
Expand All @@ -522,6 +548,11 @@ public void registerExternalTerm(JsonObject cvocEntry, String term, List<Dataset
logger.fine("Ignoring free text entry: " + term);
return;
}
if (isExternal && (retrievalUri == null)) {
logger.warning("No retrieval-uri found for term " + term);
return;
}

logger.fine("Registering term: " + term);
try {
//Assure the term is in URI form - should be if the uriSpace entry was correct
Expand Down Expand Up @@ -646,96 +677,93 @@ private String tryToReplaceRetrievalUriParam(String retrievalUri, String paramNa
private JsonObject filterResponse(JsonObject cvocEntry, JsonObject readObject, String termUri) {

JsonObjectBuilder job = Json.createObjectBuilder();
JsonObject filtering = cvocEntry.getJsonObject("retrieval-filtering");
logger.fine("RF: " + filtering.toString());
JsonObject managedFields = cvocEntry.getJsonObject("managed-fields");
logger.fine("MF: " + managedFields.toString());
int nrOfNotFound = 0;
for (String filterKey : filtering.keySet()) {
if (!filterKey.equals("@context")) {
try {
JsonObject filter = filtering.getJsonObject(filterKey);
logger.fine("F: " + filter.toString());
JsonArray params = filter.getJsonArray("params");
if (params == null) {
params = Json.createArrayBuilder().build();
}
logger.fine("Params: " + params.toString());
List<Object> vals = new ArrayList<Object>();
for (int i = 0; i < params.size(); i++) {
String param = params.getString(i);
if (param.startsWith("/")) {
// Remove leading /
param = param.substring(1);
String[] pathParts = param.split("/");
logger.fine("PP: " + String.join(", ", pathParts));
var foundPart = processPathSegment(0, pathParts, readObject, termUri);
if (foundPart == null) {
nrOfNotFound ++ ;
logger.warning("External Vocabulary: no value found for %s - %s".formatted(filterKey, param));
} else {
vals.add(i, foundPart);
logger.fine("Added param value: " + i + ": " + vals.get(i));
}
} else {
logger.fine("Param is: " + param);
// param is not a path - either a reference to the term URI
if (param.equals("@id")) {
logger.fine("Adding id param: " + termUri);
vals.add(i, termUri);
} else {
// or a hardcoded value
logger.fine("Adding hardcoded param: " + param);
vals.add(i, param);
}
JsonObject filtering = getRetrievalFilteringObject(cvocEntry, termUri);
if(filtering != null) {
logger.fine("RF: " + filtering.toString());
int nrOfNotFound = 0;
for (String filterKey : filtering.keySet()) {
if (!filterKey.equals("@context")) {
try {
JsonObject filter = filtering.getJsonObject(filterKey);
logger.fine("F: " + filter.toString());
JsonArray params = filter.getJsonArray("params");
if (params == null) {
params = Json.createArrayBuilder().build();
}
}
// Shortcut: nominally using a pattern of {0} and a param that is @id or
// hardcoded value allows the same options as letting the pattern itself be @id
// or a hardcoded value
String pattern = filter.getString("pattern");
logger.fine("Pattern: " + pattern);
if (pattern.equals("@id")) {
logger.fine("Added #id pattern: " + filterKey + ": " + termUri);
job.add(filterKey, termUri);
} else if (pattern.contains("{")) {
if (vals.isEmpty()) {
if (nrOfNotFound == 0) {
logger.warning("External Vocabulary: " + termUri + " - No value found for " + filterKey);
logger.fine("Params: " + params.toString());
List<Object> vals = new ArrayList<Object>();
for (int i = 0; i < params.size(); i++) {
String param = params.getString(i);
if (param.startsWith("/")) {
// Remove leading /
param = param.substring(1);
String[] pathParts = param.split("/");
logger.fine("PP: " + String.join(", ", pathParts));
var foundPart = processPathSegment(0, pathParts, readObject, termUri);
if (foundPart == null) {
nrOfNotFound++;
logger.warning("External Vocabulary: no value found for %s - %s".formatted(filterKey, param));
} else {
vals.add(i, foundPart);
logger.fine("Added param value: " + i + ": " + vals.get(i));
}
} else {
logger.fine("Param is: " + param);
// param is not a path - either a reference to the term URI
if (param.equals("@id")) {
logger.fine("Adding id param: " + termUri);
vals.add(i, termUri);
} else {
// or a hardcoded value
logger.fine("Adding hardcoded param: " + param);
vals.add(i, param);
}
}
}
else {
if (pattern.equals("{0}")) {
if (vals.get(0) instanceof JsonArray) {
job.add(filterKey, (JsonArray) vals.get(0));
// Shortcut: nominally using a pattern of {0} and a param that is @id or
// hardcoded value allows the same options as letting the pattern itself be @id
// or a hardcoded value
String pattern = filter.getString("pattern");
logger.fine("Pattern: " + pattern);
if (pattern.equals("@id")) {
logger.fine("Added #id pattern: " + filterKey + ": " + termUri);
job.add(filterKey, termUri);
} else if (pattern.contains("{")) {
if (vals.isEmpty()) {
if (nrOfNotFound == 0) {
logger.warning("External Vocabulary: " + termUri + " - No value found for " + filterKey);
}
else if (vals.get(0) instanceof JsonObject) {
job.add(filterKey, (JsonObject) vals.get(0));
}
else {
job.add(filterKey, (String) vals.get(0));
} else {
if (pattern.equals("{0}")) {
if (vals.get(0) instanceof JsonArray) {
job.add(filterKey, (JsonArray) vals.get(0));
} else if (vals.get(0) instanceof JsonObject) {
job.add(filterKey, (JsonObject) vals.get(0));
} else {
job.add(filterKey, (String) vals.get(0));
}
} else {
String result = MessageFormat.format(pattern, vals.toArray());
logger.fine("Result: " + result);
job.add(filterKey, result);
logger.fine("Added : " + filterKey + ": " + result);
}
}
else {
String result = MessageFormat.format(pattern, vals.toArray());
logger.fine("Result: " + result);
job.add(filterKey, result);
logger.fine("Added : " + filterKey + ": " + result);
}
} else {
logger.fine("Added hardcoded pattern: " + filterKey + ": " + pattern);
job.add(filterKey, pattern);
}
} else {
logger.fine("Added hardcoded pattern: " + filterKey + ": " + pattern);
job.add(filterKey, pattern);
} catch (Exception e) {
logger.warning("External Vocabulary: " + termUri + " - Failed to find value for " + filterKey + ": "
+ e.getMessage());
e.printStackTrace();
}
} catch (Exception e) {
logger.warning("External Vocabulary: " + termUri + " - Failed to find value for " + filterKey + ": "
+ e.getMessage());
e.printStackTrace();
}
}
}
if(nrOfNotFound>0) {
logger.warning("External Vocabulary: " + termUri + " - Failed to find value(s) reported above in " +readObject);

if (nrOfNotFound > 0) {
logger.warning("External Vocabulary: " + termUri + " - Failed to find value(s) reported above in " + readObject);
}
}
JsonObject filteredResponse = job.build();
if(filteredResponse.isEmpty()) {
Expand Down
Loading