[ml service] Add Flare as a new nnfw type

- Implemented support for 'flare' nnfw type in ML service API
- Included test cases to validate flare functionality

Signed-off-by: hyunil park <hyunil46.park@samsung.com>

Author: songgot

c/include/nnstreamer-tizen-internal.h

@@ -19,6 +19,8 @@
 extern "C" {
 #endif /* __cplusplus */
 
+#define ML_NNFW_TYPE_FLARE 23 /**< FLARE framework */
+
 /**
  * @brief Constructs the pipeline (GStreamer + NNStreamer).
  * @details This function is to construct the pipeline without checking the permission in platform internally. See ml_pipeline_construct() for the details.

c/src/ml-api-inference-single.c

@@ -114,6 +114,7 @@ static const char *ml_nnfw_subplugin_name[] = {
   [ML_NNFW_TYPE_QNN] = "qnn",
   [ML_NNFW_TYPE_LLAMACPP] = "llamacpp",
   [ML_NNFW_TYPE_TIZEN_HAL] = "tizen-hal",
+  [ML_NNFW_TYPE_FLARE] = "flare",
   NULL
 };
 
@@ -979,16 +980,24 @@ ml_single_open_custom (ml_single_h * single, ml_single_preset * info)
   for (i = 0; i < num_models; i++)
     g_strstrip (list_models[i]);
 
-  status = _ml_validate_model_file ((const char **) list_models, num_models,
-      &nnfw);
-  if (status != ML_ERROR_NONE) {
-    _ml_error_report_continue
-        ("Cannot validate the model (1st model: %s. # models: %d). Error code: %d",
-        list_models[0], num_models, status);
-    g_strfreev (list_models);
-    return status;
+  /* Note : OpenVINO and flare use the bin extension.
+   * _ml_validate_model_file() infers nnfw based on the file extension.
+   * The .bin extension is recognized as OpenVINO (ML_NNFW_TYPE_OPENVINO) by default
+   * If "flare" is specified, it forces ML_NNFW_TYPE_FLARE.
+   */
+  if (info->fw_name && strcasecmp (info->fw_name, "flare") == 0) {
+    nnfw = ML_NNFW_TYPE_FLARE;
+  } else {
+    status = _ml_validate_model_file ((const char **) list_models, num_models,
+        &nnfw);
+    if (status != ML_ERROR_NONE) {
+      _ml_error_report_continue
+          ("Cannot validate the model (1st model: %s. # models: %d). Error code: %d",
+          list_models[0], num_models, status);
+      g_strfreev (list_models);
+      return status;
+    }
   }
-
   g_strfreev (list_models);
 
   /**
@@ -1089,14 +1098,14 @@ ml_single_open_custom (ml_single_h * single, ml_single_preset * info)
   }
 
   if (single_h->klass && info->invoke_async) {
-    if (info->invoke_async_cb != NULL && info->invoke_async_data!= NULL) {
+    if (info->invoke_async_cb != NULL && info->invoke_async_data != NULL) {
       NNSFilterInvokeAsyncCallback invoke_async_cb =
           (NNSFilterInvokeAsyncCallback) info->invoke_async_cb;
       single_h->klass->set_invoke_async_callback (single_h->filter,
           invoke_async_cb, info->invoke_async_data);
     } else {
       _ml_error_report
-        ("The parameters invoke_async_cb and invoke_async_data in the info argument are invalid");
+          ("The parameters invoke_async_cb and invoke_async_data in the info argument are invalid");
       status = ML_ERROR_INVALID_PARAMETER;
       goto error;
     }

tests/capi/unittest_capi_service_extension.cc

@@ -9,7 +9,7 @@
 
 #include <gtest/gtest.h>
 #include <glib.h>
-
+#include <iostream>
 #include <ml-api-service-private.h>
 #include <ml-api-service.h>
 #include "ml-api-service-extension.h"
@@ -394,8 +394,7 @@ _extension_test_imgclf (ml_service_h handle, gboolean is_pipeline)
  * @brief Callback function for scenario test.
  */
 static void
-_extension_test_llamacpp_cb (
-    ml_service_event_e event, ml_information_h event_data, void *user_data)
+_extension_test_llm_cb (ml_service_event_e event, ml_information_h event_data, void *user_data)
 {
   extension_test_data_s *tdata = (extension_test_data_s *) user_data;
   ml_tensors_data_h data = NULL;
@@ -413,7 +412,8 @@ _extension_test_llamacpp_cb (
       status = ml_tensors_data_get_tensor_data (data, 0U, &_raw, &_size);
       EXPECT_EQ (status, ML_ERROR_NONE);
 
-      g_print ("%s", (char *) _raw);
+      std::cout.write (static_cast<const char *> (_raw), _size); /* korean output */
+      std::cout.flush ();
 
       if (tdata)
         tdata->received++;
@@ -427,7 +427,7 @@ _extension_test_llamacpp_cb (
  * @brief Internal function to run test with ml-service extension handle.
  */
 static inline void
-_extension_test_llamacpp (ml_service_h handle, gboolean is_pipeline)
+_extension_test_llm (ml_service_h handle, gboolean is_pipeline, gchar *file_name, guint sleep_us)
 {
   extension_test_data_s *tdata;
   ml_tensors_info_h info;
@@ -436,14 +436,14 @@ _extension_test_llamacpp (ml_service_h handle, gboolean is_pipeline)
   gsize len = 0;
   gchar *contents = NULL;
 
-  g_autofree gchar *data_file = _get_data_path ("input.txt");
+  g_autofree gchar *data_file = _get_data_path (file_name);
   ASSERT_TRUE (g_file_test (data_file, G_FILE_TEST_EXISTS));
   ASSERT_TRUE (g_file_get_contents (data_file, &contents, &len, NULL));
 
   tdata = _create_test_data (is_pipeline);
   ASSERT_TRUE (tdata != NULL);
 
-  status = ml_service_set_event_cb (handle, _extension_test_llamacpp_cb, tdata);
+  status = ml_service_set_event_cb (handle, _extension_test_llm_cb, tdata);
   EXPECT_EQ (status, ML_ERROR_NONE);
 
   /* Create and push input data. */
@@ -457,7 +457,7 @@ _extension_test_llamacpp (ml_service_h handle, gboolean is_pipeline)
   status = ml_service_request (handle, NULL, input);
   EXPECT_EQ (status, ML_ERROR_NONE);
 
-  g_usleep (5000000U);
+  g_usleep (sleep_us);
   EXPECT_GT (tdata->received, 0);
 
   /* Clear callback before releasing tdata. */
@@ -477,8 +477,9 @@ TEST_REQUIRE_TFLITE (MLServiceExtension, scenarioConfigLlamacpp)
 {
   ml_service_h handle;
   int status;
-
+  g_autofree gchar *input_file = g_strdup ("input.txt");
   g_autofree gchar *model_file = _get_model_path ("llama-2-7b-chat.Q2_K.gguf");
+
   if (!g_file_test (model_file, G_FILE_TEST_EXISTS)) {
     g_critical ("Skipping scenarioConfigLlamacpp test due to missing model file. "
                 "Please download model file from https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF");
@@ -490,7 +491,7 @@ TEST_REQUIRE_TFLITE (MLServiceExtension, scenarioConfigLlamacpp)
   status = ml_service_new (config, &handle);
   ASSERT_EQ (status, ML_ERROR_NONE);
 
-  _extension_test_llamacpp (handle, FALSE);
+  _extension_test_llm (handle, FALSE, input_file, 5000000U);
 
   status = ml_service_destroy (handle);
   EXPECT_EQ (status, ML_ERROR_NONE);
@@ -503,8 +504,9 @@ TEST_REQUIRE_TFLITE (MLServiceExtension, scenarioConfigLlamacppAsync)
 {
   ml_service_h handle;
   int status;
-
+  g_autofree gchar *input_file = g_strdup ("input.txt");
   g_autofree gchar *model_file = _get_model_path ("llama-2-7b-chat.Q2_K.gguf");
+
   if (!g_file_test (model_file, G_FILE_TEST_EXISTS)) {
     g_critical ("Skipping scenarioConfigLlamacppAsync test due to missing model file. "
                 "Please download model file from https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF");
@@ -516,7 +518,33 @@ TEST_REQUIRE_TFLITE (MLServiceExtension, scenarioConfigLlamacppAsync)
   status = ml_service_new (config, &handle);
   ASSERT_EQ (status, ML_ERROR_NONE);
 
-  _extension_test_llamacpp (handle, FALSE);
+  _extension_test_llm (handle, FALSE, input_file, 5000000U);
+
+  status = ml_service_destroy (handle);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+}
+
+/**
+ * @brief Usage of ml-service extension API.
+ */
+TEST_REQUIRE_TFLITE (MLServiceExtension, scenarioConfigFlare)
+{
+  ml_service_h handle;
+  int status;
+  g_autofree gchar *input_file = g_strdup ("flare_input.txt");
+  g_autofree gchar *model_file = _get_model_path ("sflare_if_4bit_3b.bin");
+
+  if (!g_file_test (model_file, G_FILE_TEST_EXISTS)) {
+    g_critical ("Skipping scenarioConfigFlare test due to missing model file.Please download model file");
+    return;
+  }
+
+  g_autofree gchar *config = get_config_path ("config_single_flare.conf");
+
+  status = ml_service_new (config, &handle);
+  ASSERT_EQ (status, ML_ERROR_NONE);
+
+  _extension_test_llm (handle, FALSE, input_file, 40000000U);
 
   status = ml_service_destroy (handle);
   EXPECT_EQ (status, ML_ERROR_NONE);

tests/test_models/config/config_single_flare.conf

@@ -0,0 +1,11 @@
+{
+    "single" :
+    {
+        "framework" : "flare",
+        "model" : ["../tests/test_models/models/sflare_if_4bit_3b.bin"],
+        "adapter" : ["../tests/test_models/models/history_lora.bin"],
+        "custom" : "tokenizer_path:../tests/test_models/data/tokenizer.json,backend:CPU,output_size:1024,model_type:3B,data_type:W4A32",
+        "invoke_dynamic" : "true",
+        "invoke_async" : "false"
+    }
+}