Skip to content

Review and tidy solr/modules/langid code #4835

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
epugh wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Review and tidy solr/modules/langid code #4835

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions ...rc/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ protected List<DetectedLanguage> detectLanguage(Reader solrDocReader) {
List<Language> langlist = orchestrator.detectAll(text);
ArrayList<DetectedLanguage> solrLangList = new ArrayList<>();
for (Language l : langlist) {
solrLangList.add(
new DetectedLanguage(l.getIsoCode639_1().toString(), (double) l.getProbability()));
solrLangList.add(new DetectedLanguage(l.getIsoCode639_1(), (double) l.getProbability()));
}
if (solrLangList.isEmpty()) {
log.debug("Could not determine language, returning empty list");
Expand Down
28 changes: 11 additions & 17 deletions ...s/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ public abstract class LanguageIdentifierUpdateProcessor extends UpdateRequestPro
protected int maxTotalChars;

// Regex patterns
protected final Pattern tikaSimilarityPattern = Pattern.compile(".*\\((.*?)\\)");
protected final Pattern langPattern = Pattern.compile("\\{lang\\}");

public LanguageIdentifierUpdateProcessor(
Expand All @@ -95,7 +94,7 @@ private void initParams(SolrParams params) {
if (params != null) {
// Document-centric langId params
setEnabled(params.getBool(LANGUAGE_ID, true));
if (params.get(FIELDS_PARAM, "").length() > 0) {
if (!params.get(FIELDS_PARAM, "").isEmpty()) {
inputFields = params.get(FIELDS_PARAM, "").split(",");
}
langField = params.get(LANG_FIELD, DOCID_LANGFIELD_DEFAULT);
Expand All @@ -105,7 +104,7 @@ private void initParams(SolrParams params) {
params.get(
DOCID_PARAM, uniqueKeyField == null ? DOCID_FIELD_DEFAULT : uniqueKeyField.getName());
fallbackValue = params.get(FALLBACK);
if (params.get(FALLBACK_FIELDS, "").length() > 0) {
if (!params.get(FALLBACK_FIELDS, "").isEmpty()) {
fallbackFields = params.get(FALLBACK_FIELDS).split(",");
}
overwrite = params.getBool(OVERWRITE, false);
Expand All @@ -119,7 +118,7 @@ private void initParams(SolrParams params) {

// Mapping params (field centric)
enableMapping = params.getBool(MAP_ENABLE, false);
if (params.get(MAP_FL, "").length() > 0) {
if (!params.get(MAP_FL, "").isEmpty()) {
mapFields = params.get(MAP_FL, "").split(",");
} else {
mapFields = inputFields;
Expand All @@ -129,8 +128,8 @@ private void initParams(SolrParams params) {
mapIndividual = params.getBool(MAP_INDIVIDUAL, false);

// Process individual fields
String[] mapIndividualFields = {};
if (params.get(MAP_INDIVIDUAL_FL, "").length() > 0) {
String[] mapIndividualFields;
if (!params.get(MAP_INDIVIDUAL_FL, "").isEmpty()) {
mapIndividualFields = params.get(MAP_INDIVIDUAL_FL, "").split(",");
} else {
mapIndividualFields = mapFields;
Expand Down Expand Up @@ -225,7 +224,7 @@ public void processAdd(AddUpdateCommand cmd) throws IOException {
* @param doc the SolrInputDocument to modify
*/
protected void process(SolrInputDocument doc) {
String docLang = null;
String docLang;
HashSet<String> docLangs = new HashSet<>();
String fallbackLang = getFallbackLang(doc, fallbackFields, fallbackValue);

Expand All @@ -247,7 +246,7 @@ protected void process(SolrInputDocument doc) {
log.debug("Overwritten old value {}", doc.getFieldValue(langField));
}
}
if (langField != null && langField.length() != 0) {
if (langField != null && !langField.isEmpty()) {
doc.setField(langField, docLang);
}
} else {
Expand Down Expand Up @@ -297,7 +296,7 @@ protected void process(SolrInputDocument doc) {
}

// Set the languages field to an array of all detected languages
if (langsField != null && langsField.length() != 0) {
if (langsField != null && !langsField.isEmpty()) {
doc.setField(langsField, docLangs.toArray());
}
}
Expand Down Expand Up @@ -367,7 +366,7 @@ protected String resolveLanguage(String language, String fallbackLang) {
*/
protected String resolveLanguage(List<DetectedLanguage> languages, String fallbackLang) {
String langStr;
if (languages.size() == 0) {
if (languages.isEmpty()) {
log.debug("No language detected, using fallback {}", fallbackLang);
langStr = fallbackLang;
} else {
Expand Down Expand Up @@ -395,7 +394,7 @@ protected String resolveLanguage(List<DetectedLanguage> languages, String fallba
}
}

if (langStr == null || langStr.length() == 0) {
if (langStr == null || langStr.isEmpty()) {
log.warn("Language resolved to null or empty string. Fallback not configured?");
langStr = "";
}
Expand Down Expand Up @@ -429,7 +428,7 @@ protected String normalizeLangCode(String langCode) {
* @return The new schema field name, based on pattern and replace, or null if illegal
*/
protected String getMappedField(String currentField, String language) {
String lc = mapLcMap.containsKey(language) ? mapLcMap.get(language) : language;
String lc = mapLcMap.getOrDefault(language, language);
String newFieldName =
langPattern
.matcher(mapPattern.matcher(currentField).replaceFirst(mapReplaceStr))
Expand Down Expand Up @@ -474,9 +473,4 @@ public void setEnabled(boolean enabled) {
protected SolrInputDocumentReader solrDocReader(SolrInputDocument doc, String[] fields) {
return new SolrInputDocumentReader(doc, fields, maxTotalChars, maxFieldValueChars, " ");
}

/** Concatenates content from input fields defined in langid.fl. For test purposes only */
protected String concatFields(SolrInputDocument doc) {
return SolrInputDocumentReader.asString(solrDocReader(doc, inputFields));
}
}
2 changes: 1 addition & 1 deletion ...es/langid/src/java/org/apache/solr/update/processor/OpenNLPLangDetectUpdateProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public OpenNLPLangDetectUpdateProcessor(
protected List<DetectedLanguage> detectLanguage(Reader solrDocReader) {
List<DetectedLanguage> languages = new ArrayList<>();
String content = SolrInputDocumentReader.asString(solrDocReader);
if (content.length() != 0) {
if (!content.isEmpty()) {
LanguageDetectorME ldme = new LanguageDetectorME(model);
Language[] langs = ldme.predictLanguages(content);
for (Language language : langs) {
Expand Down
14 changes: 7 additions & 7 deletions solr/modules/langid/src/java/org/apache/solr/update/processor/SolrInputDocumentReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ private int fillBuffer(StringBuilder sb, int targetLen) {
nextDocChunk(sb, targetLen);
}

if (sb.length() == 0) {
if (sb.isEmpty()) {
eod = true;
return eodReturnValue;
} else {
Expand All @@ -133,7 +133,7 @@ private int nextDocChunk(StringBuilder sb, int maxChunkLength) {
do {
SolrInputField f = doc.getField(fields[currentFieldIdx]);
if (f == null) {
log.debug("Field with name {} did not exist on docuemnt.", fields[currentFieldIdx]);
log.debug("Field with name {} did not exist on document.", fields[currentFieldIdx]);
incField(sb);
continue;
}
Expand All @@ -144,9 +144,9 @@ private int nextDocChunk(StringBuilder sb, int maxChunkLength) {
String fvStr = String.valueOf(fvIt.next());
if (currentFieldValueIdx < startFieldValueIdx) continue;
startFieldValueIdx = 0;
if (sb.length() > 0) {
if (!sb.isEmpty()) {
if (maxChunkLength - sb.length() < fieldValueSep.length()) {
sb.append(fieldValueSep.substring(0, maxChunkLength - sb.length()));
sb.append(fieldValueSep, 0, maxChunkLength - sb.length());
} else {
sb.append(fieldValueSep);
}
Expand All @@ -161,7 +161,7 @@ private int nextDocChunk(StringBuilder sb, int maxChunkLength) {
if (endOffset - currentFieldValueOffset > maxCharsPerFieldValue) {
endOffset = maxCharsPerFieldValue - currentFieldValueOffset;
}
sb.append(fvStr.substring(currentFieldValueOffset, endOffset));
sb.append(fvStr, currentFieldValueOffset, endOffset);
currentFieldValueOffset = endOffset == fvStr.length() ? 0 : endOffset;
}
if (sb.length() >= maxChunkLength) {
Expand All @@ -170,7 +170,7 @@ private int nextDocChunk(StringBuilder sb, int maxChunkLength) {
incField(sb);
}
} while (currentFieldIdx <= fields.length - 1 && sb.length() < maxChunkLength);
return sb.length() == 0 ? eodReturnValue : sb.length();
return sb.isEmpty() ? eodReturnValue : sb.length();
}

private int returnEod() {
Expand All @@ -179,7 +179,7 @@ private int returnEod() {
}

private int returnValue(StringBuilder sb) {
if (sb.length() == 0) {
if (sb.isEmpty()) {
return returnEod();
} else {
return sb.length();
Expand Down
2 changes: 1 addition & 1 deletion ...rc/test/org/apache/solr/update/processor/OpenNLPLangDetectUpdateProcessorFactoryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import org.apache.solr.request.SolrQueryRequest;
import org.junit.Test;

@ThreadLeakLingering(linger = 0)
@ThreadLeakLingering()
public class OpenNLPLangDetectUpdateProcessorFactoryTest
extends LanguageIdentifierUpdateProcessorFactoryTestCase {
private static final String TEST_MODEL = "opennlp-langdetect.eng-swe-spa-rus-deu.bin";
Expand Down
2 changes: 1 addition & 1 deletion ...modules/langid/src/test/org/apache/solr/update/processor/SolrInputDocumentReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public void allStrFields() throws Exception {
}

@Test
public void testGetStringFields() throws Exception {
public void testGetStringFields() {
String[] expected = new String[] {"f1", "f2", "f4"};
assertArrayEquals(expected, SolrInputDocumentReader.getStringFields(doc));
}
Expand Down
Loading