US1107290: Build libcld2.so

.gitignore

@@ -16,4 +16,15 @@ pom - Copy.xml
 *.ear
 
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
-hs_err_pid*
+hs_err_pid*
+
+*.log
+
+# Eclipse
+.settings/
+.settings
+.project
+.classpath
+bin
+
+**/.gitignore

cld2/README.md

@@ -0,0 +1,26 @@
+### Build Compact Language Detector 2
+
+Build in the `libcld2.so` from [CLD2Owners/cld2: Compact Language Detector 2](https://github.com/CLD2Owners/cld2) source code.
+Download [commit]((https://github.com/CLD2Owners/cld2/commit/b56fa78a2fe44ac2851bae5bf4f4693a0644da7b)) as zip file, unpack its contents and compile the code with gcc.
+
+
+### Finding the correct mangled function name to use in `Cld2Library.java`
+```
+OTX-44M2N13:/mnt/d/Projects/github/worker-languagedetection-agg # docker container run -it --rm --network=host -e HTTP_PROXY -e HTTPS_PROXY -e NO_PROXY -v ~/.m2:/root/.m2 -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/wd -w /wd dev/cafapi/buildenv-cpp:8.3.0-SNAPSHOT
+
+OTX-44M2N13:/wd # nm -D cld2/target/libcld2.so | grep -i "ExtDetectLanguageSummary"
+0000000000025050 T _ZN4CLD224ExtDetectLanguageSummaryEPKcibPKNS_8CLDHintsEiPNS_8LanguageEPiPdPSt6vectorINS_11ResultChunkESaISA_EES7_Pb
+0000000000024e90 T _ZN4CLD224ExtDetectLanguageSummaryEPKcibPNS_8LanguageEPiS4_Pb
+0000000000024f60 T _ZN4CLD224ExtDetectLanguageSummaryEPKcibS1_iNS_8LanguageEPS2_PiPdS4_Pb
+0000000000024f00 T _ZN4CLD224ExtDetectLanguageSummaryEPKcibS1_iNS_8LanguageEPS2_PiS4_Pb
+0000000000024fb0 T _ZN4CLD233ExtDetectLanguageSummaryCheckUTF8EPKcibPKNS_8CLDHintsEiPNS_8LanguageEPiPdPSt6vectorINS_11ResultChunkESaISA_EES7_PbS7_
+
+nm -DC cld2/target/libcld2.so | grep "CLD2::ExtDetectLanguageSummary"
+0000000000025050 T CLD2::ExtDetectLanguageSummary(char const*, int, bool, CLD2::CLDHints const*, int, CLD2::Language*, int*, double*, std::vector<CLD2::ResultChunk, std::allocator<CLD2::ResultChunk> >*, int*, bool*)
+0000000000024e90 T CLD2::ExtDetectLanguageSummary(char const*, int, bool, CLD2::Language*, int*, int*, bool*)
+0000000000024f60 T CLD2::ExtDetectLanguageSummary(char const*, int, bool, char const*, int, CLD2::Language, CLD2::Language*, int*, double*, int*, bool*)
+0000000000024f00 T CLD2::ExtDetectLanguageSummary(char const*, int, bool, char const*, int, CLD2::Language, CLD2::Language*, int*, int*, bool*)
+0000000000024fb0 T CLD2::ExtDetectLanguageSummaryCheckUTF8(char const*, int, bool, CLD2::CLDHints const*, int, CLD2::Language*, int*, double*, std::vector<CLD2::ResultChunk, std::allocator<CLD2::ResultChunk> >*, int*, bool*, int*)
+
+nm -D cld2/target/libcld2.so | grep "0000000000024fb0"
+0000000000024fb0 T _ZN4CLD233ExtDetectLanguageSummaryCheckUTF8EPKcibPKNS_8CLDHintsEiPNS_8LanguageEPiPdPSt6vectorINS_11ResultChunkESaISA_EES7_PbS7_

cld2/pom.xml

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright 2015-2026 Open Text.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>cld2</artifactId>
+    <name>cld2</name>
+    <packaging>pom</packaging>
+
+    <parent>
+        <groupId>com.github.cafdataprocessing.workers.languagedetection</groupId>
+        <artifactId>worker-languagedetection-aggregator</artifactId>
+        <version>7.0.0-SNAPSHOT</version>
+    </parent>
+
+    <properties>
+        <cld2.commit>b56fa78a2fe44ac2851bae5bf4f4693a0644da7b</cld2.commit>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>io.github.download-maven-plugin</groupId>
+                <artifactId>download-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>download-cld2-source</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>wget</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <url>https://github.com/CLD2Owners/cld2/archive/${cld2.commit}.zip</url>
+                    <unpack>true</unpack>
+                    <outputDirectory>${project.build.directory}/external-source</outputDirectory>
+                    <outputFileName>cld2.zip</outputFileName>
+                    <sha256>6d8681eadbb64d8fcd3c6c620e81bed16d99ff75627324d66ee2d54bf2b7d749</sha256>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>compile-cld2-native</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                        <configuration>
+                            <executable>bash</executable>
+                            <workingDirectory>${project.build.directory}/external-source/cld2-${cld2.commit}/internal</workingDirectory>
+                            <arguments>
+                                <argument>-c</argument>
+                                <argument>./compile_libs.sh &amp;&amp; mv ./libcld2.so ${project.build.directory}</argument>
+                            </arguments>
+                            <environmentVariables>
+                                <CFLAGS>-O3 -Wno-narrowing</CFLAGS>
+                            </environmentVariables>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+              <groupId>org.codehaus.mojo</groupId>
+              <artifactId>build-helper-maven-plugin</artifactId>
+              <executions>
+                <execution>
+                  <id>attach-cld2-native-binary</id>
+                  <phase>package</phase>
+                  <goals>
+                    <goal>attach-artifact</goal>
+                  </goals>
+                  <configuration>
+                    <artifacts>
+                      <artifact>
+                        <file>${project.build.directory}/libcld2.so</file>
+                        <type>so</type>
+                        <classifier>native</classifier>
+                      </artifact>
+                    </artifacts>
+                  </configuration>
+                </execution>
+              </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

language-detection-cld2/pom.xml

@@ -27,11 +27,38 @@
     <parent>
         <groupId>com.github.cafdataprocessing.workers.languagedetection</groupId>
         <artifactId>worker-languagedetection-aggregator</artifactId>
-        <version>6.1.5-SNAPSHOT</version>
+        <version>7.0.0-SNAPSHOT</version>
     </parent>
 
     <build>
         <plugins>
+            <plugin>
+              <groupId>org.apache.maven.plugins</groupId>
+              <artifactId>maven-dependency-plugin</artifactId>
+              <executions>
+                <execution>
+                  <id>copy-cld2-library</id>
+                  <phase>process-test-resources</phase>
+                  <goals>
+                    <goal>copy</goal>
+                  </goals>
+                  <configuration>
+                    <artifactItems>
+                      <artifactItem>
+                        <groupId>com.github.cafdataprocessing.workers.languagedetection</groupId>
+                        <artifactId>cld2</artifactId>
+                        <version>${project.version}</version>
+                        <type>so</type>
+                        <classifier>native</classifier>
+                        <overWrite>true</overWrite>
+                        <outputDirectory>${project.build.directory}/cld2native/linux</outputDirectory>
+                        <destFileName>libcld2.so</destFileName>
+                      </artifactItem>
+                    </artifactItems>
+                  </configuration>
+                </execution>
+              </executions>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-failsafe-plugin</artifactId>
@@ -45,7 +72,7 @@
                 </executions>
                 <configuration>
                     <systemPropertyVariables>
-                        <cld2.location>${project.basedir}/cld2native</cld2.location>
+                        <cld2.location>${project.build.directory}/cld2native</cld2.location>
                     </systemPropertyVariables>
                 </configuration>
             </plugin>
@@ -97,6 +124,11 @@
             <artifactId>junit-jupiter-api</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>

language-detection-cld2/src/main/java/com/github/cafdataprocessing/workers/languagedetection/cld2/CLDHints.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2015-2026 Open Text.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.cafdataprocessing.workers.languagedetection.cld2;
+
+import com.sun.jna.Structure;
+import java.util.Arrays;
+import java.util.List;
+
+public final class CLDHints extends Structure {
+    public String content_language_hint;
+    public String tld_hint;
+    public int encoding_hint;
+    public int language_hint;
+
+    public CLDHints() {
+        super();
+    }
+
+    @Override
+    protected List<String> getFieldOrder() {
+        return Arrays.asList("content_language_hint", "tld_hint", "encoding_hint", "language_hint");
+    }
+}

language-detection-cld2/src/main/java/com/github/cafdataprocessing/workers/languagedetection/cld2/Cld2Detector.java

@@ -23,6 +23,8 @@
 import com.github.cafdataprocessing.workers.languagedetection.LanguageDetectorStatus;
 import org.apache.commons.io.IOUtils;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -33,6 +35,8 @@
  */
 public class Cld2Detector implements LanguageDetector
 {
+    private static final Logger LOGGER = LoggerFactory.getLogger(Cld2Detector.class);
+
     public Cld2Detector()
     {
     }
@@ -77,6 +81,7 @@ public LanguageDetectorResult detectLanguage(byte[] textBytes, LanguageDetectorS
             }
 
         } catch (LanguageDetectorException e) {
+            LOGGER.error("LanguageDetectorException", e);
             languageDetectorResult.setLanguageDetectorStatus(LanguageDetectorStatus.FAILED);
         }
         return languageDetectorResult;

language-detection-cld2/src/main/java/com/github/cafdataprocessing/workers/languagedetection/cld2/Cld2Library.java

@@ -16,46 +16,50 @@
 package com.github.cafdataprocessing.workers.languagedetection.cld2;
 
 import com.sun.jna.Library;
+import com.sun.jna.FunctionMapper;
+import com.sun.jna.Pointer;
 
 /**
  * JNA Interface to access the C++ CLD2 methods
  */
 public interface Cld2Library extends Library
 {
+    final static FunctionMapper NAME_MAPPER = (library, method) -> {
+        if (method.getName().equals("DetectLanguageSummaryWithHints")) {
+            return "_ZN4CLD224ExtDetectLanguageSummaryEPKcibPKNS_8CLDHintsEiPNS_8LanguageEPiPdPSt6vectorINS_11ResultChunkESaISA_EES7_Pb";
+        }
+        return method.getName();
+    };
+
     /**
-     * this is the method used in CLD2 to detect the language, passing in the hints and references (as java arrays) of the text buffer,
-     * language3, percent3, text_bytes and is_reliable fields
-     *
-     * @param buffer - Bytes of text to be queried
-     * @param buffer_length - The length of buffer
-     * @param is_plain_text - States whether the text passed in is plain-text or not
-     * @param tld_hint - Encoding hint, it is from an encoding detector applied to an input
-     * @param encoding_hint - Detector hint, such as "en" "en,it", ENGLISH
-     * @param language_hint - The language code specifying a possible language that may be in the text.
-     * @param language3 - Contains the top three languages found
-     * @param percent3 - The confidence percentage for each of the top three languages found
-     * @param text_bytes - The amount of non-tag/letters-only text found
-     * @param is_reliable - Determines the confidence in the findings - True if the returned language is some amount more probable than
-     * the second best.
+     * Calls into CLD2 to detect the language of the given text buffer, using the provided hints.
      *
-     * @return integer value for language corresponding to value in Cld2Language.
+     * @param buffer - Bytes of UTF-8 text to be analyzed
+     * @param buffer_length - The length of the buffer in bytes
+     * @param is_plain_text - True if the text is plain text, false if it contains HTML markup
+     * @param hints - A CLDHints structure containing detection hints (TLD, content language, encoding, and language hints)
+     * @param flags - Detection flags (use 0 for default behavior, or kCLDFlagBestEffort = 0x4000 for short text)
+     * @param language3 - Output array of size 3; receives the top three detected languages as CLD2 Language enum values
+     * @param percent3 - Output array of size 3; receives the confidence percentage for each of the top three detected languages
+     * @param normalized_score3 - Output array of size 3; receives the normalized score for each of the top three detected languages
+     * @param result_chunks - Pointer to a std::vector of ResultChunk; must be passed as null to avoid JNA/C++ interop crashes
+     * @param text_bytes - Output array of size 1; receives the number of non-tag/letters-only text bytes found
+     * @param is_reliable - Output array of size 1; set to non-zero if the top language is significantly more probable than the second best
      *
-     * language3 array contains the top three languages found percent3 array contains the confidence for each of the top three languages
-     * found, that they are correct text_bytes array contains the amount of non-tag/letters-only text found is_reliable, which is set true
-     * if the returned language is some amount more probable than the second best language. Calculation is a complex function of the
-     * length of the text and the different-script runs of text.
+     * @return Integer value corresponding to the top detected language in the CLD2 Language enum (see Cld2Language)
      */
     int DetectLanguageSummaryWithHints(
-        byte[] buffer,
-        int buffer_length,
-        boolean is_plain_text,
-        String tld_hint,
-        int encoding_hint,
-        int language_hint,
-        int[] language3,
-        int[] percent3,
-        int[] text_bytes,
-        boolean[] is_reliable
+        byte[] buffer,           // Arg 1: char*
+        int buffer_length,       // Arg 2: int
+        byte is_plain_text,      // Arg 3: bool (use byte to match C++ 1-byte bool)
+        CLDHints hints,          // Arg 4: CLDHints const*
+        int flags,               // Arg 5: int (Use 0)
+        int[] language3,         // Arg 6: CLD2::Language*
+        int[] percent3,          // Arg 7: int*
+        double[] normalized_score3, // Arg 8: double* (Can pass null)
+        Pointer result_chunks,   // Arg 9: std::vector* (MUST PASS NULL)
+        int[] text_bytes,        // Arg 10: int*
+        byte[] is_reliable       // Arg 11: bool*
     );
 
     /**