From 1fee66927b21da3ce46168fa53883926f1d4cf52 Mon Sep 17 00:00:00 2001 From: hyunil park Date: Wed, 23 Jul 2025 14:38:59 +0900 Subject: [PATCH 1/2] Add On Device LXM Service API and LLM example - Add On Device LXM Service API - Add LLM example Signed-off-by: hyunil park --- Tizen.platform/lxm_service/README.md | 5 + Tizen.platform/lxm_service/config.conf | 11 + Tizen.platform/lxm_service/example/main.cc | 336 ++++++++++++++++++ .../lxm_service/gen-ml-lxm-service-rpm.sh | 7 + .../lxm_service/include/ml-lxm-service.h | 135 +++++++ Tizen.platform/lxm_service/meson.build | 48 +++ .../packaging/ml-lxm-service.manifest | 5 + .../lxm_service/packaging/ml-lxm-service.spec | 46 +++ .../lxm_service/src/ml-lxm-service.c | 262 ++++++++++++++ 9 files changed, 855 insertions(+) create mode 100644 Tizen.platform/lxm_service/README.md create mode 100644 Tizen.platform/lxm_service/config.conf create mode 100644 Tizen.platform/lxm_service/example/main.cc create mode 100755 Tizen.platform/lxm_service/gen-ml-lxm-service-rpm.sh create mode 100644 Tizen.platform/lxm_service/include/ml-lxm-service.h create mode 100644 Tizen.platform/lxm_service/meson.build create mode 100644 Tizen.platform/lxm_service/packaging/ml-lxm-service.manifest create mode 100644 Tizen.platform/lxm_service/packaging/ml-lxm-service.spec create mode 100644 Tizen.platform/lxm_service/src/ml-lxm-service.c diff --git a/Tizen.platform/lxm_service/README.md b/Tizen.platform/lxm_service/README.md new file mode 100644 index 00000000..3e043d0a --- /dev/null +++ b/Tizen.platform/lxm_service/README.md @@ -0,0 +1,5 @@ +--- +title: On device Machine LXM(LLM, LVM, etc.) Service API example +... + +## On device Machine LXM(LLM, LVM, etc.) Service API example diff --git a/Tizen.platform/lxm_service/config.conf b/Tizen.platform/lxm_service/config.conf new file mode 100644 index 00000000..000c57ff --- /dev/null +++ b/Tizen.platform/lxm_service/config.conf @@ -0,0 +1,11 @@ +{ + "single" : + { + "framework" : "flare", + "model" : ["sflare_if_4bit_3b.bin"], + "adapter" : ["history_lora.bin"], + "custom" : "tokenizer_path:tokenizer.json,backend:CPU,output_size:1024,model_type:3B,data_type:W4A32", + "invoke_dynamic" : "true", + "invoke_async" : "false" + } +} diff --git a/Tizen.platform/lxm_service/example/main.cc b/Tizen.platform/lxm_service/example/main.cc new file mode 100644 index 00000000..1cec3314 --- /dev/null +++ b/Tizen.platform/lxm_service/example/main.cc @@ -0,0 +1,336 @@ +/** + * @file main.cc + * @date 23 JULY 2025 + * @brief Large Language Model service example using ML LXM Service API + * @see https://github.com/nnstreamer/nnstreamer + * @author + * @bug No known bugs. + */ +#include +#include +#include +#include +#include +#include +#include "ml-lxm-service.h" + +#define MAX_STRING_LEN 4096 + +enum { + CURRENT_STATUS_MAINMENU, + CURRENT_STATUS_SESSION_CREATE, + CURRENT_STATUS_SESSION_DESTROY, + CURRENT_STATUS_SESSION_SET_INSTRUCTION, + CURRENT_STATUS_SESSION_RESPOND, + CURRENT_STATUS_SESSION_RESPOND_ASYNC, + CURRENT_STATUS_PROMPT_CREATE, + CURRENT_STATUS_PROMPT_APPEND_TEXT, + CURRENT_STATUS_PROMPT_APPEND_INSTRUCTION, + CURRENT_STATUS_PROMPT_DESTROY +}; + +int g_menu_state = CURRENT_STATUS_MAINMENU; +GMainLoop *loop; +ml_lxm_session_h g_session = NULL; // Global session handle +ml_lxm_prompt_h g_prompt = NULL; // Global prompt handle +char g_instructions[MAX_STRING_LEN]; // Buffer for instructions +char g_prompt_text[MAX_STRING_LEN]; // Buffer for prompt text + +/** + * @brief Token callback for async response + */ +static void token_handler(ml_service_event_e event, ml_information_h event_data, + void *user_data) { + ml_tensors_data_h data = NULL; + void *_raw = NULL; + size_t _size = 0; + int ret; + + switch (event) { + case ML_SERVICE_EVENT_NEW_DATA: + if (event_data != NULL) { + + ret = ml_information_get(event_data, "data", &data); + if (ret != ML_ERROR_NONE) { + g_print("Failed to get data from event_data\n"); + return; + } + + ret = ml_tensors_data_get_tensor_data(data, 0U, &_raw, &_size); + if (ret != ML_ERROR_NONE) { + g_print("Failed to get tensor data\n"); + return; + } + + std::cout.write(static_cast(_raw), + _size); /* korean output */ + std::cout.flush(); + } + default: + break; + } +} + +/** + * @brief Main menu display + */ +void main_menu() { + g_print("\n"); + g_print("================================================================\n"); + g_print(" ML LLM Service Test (press q to quit) \n"); + g_print("----------------------------------------------------------------\n"); + g_print("a. Session Create \n"); + g_print("b. Session Destroy \n"); + g_print("c. Session Set Instruction \n"); + g_print("d. Session Respond \n"); + g_print("e. Prompt Create \n"); + g_print("f. Prompt Destroy \n"); + g_print("g. Prompt Append Text \n"); + g_print("h. Prompt Append Instruction \n"); + g_print("q. Quit \n"); + g_print("================================================================\n"); +} + +/** + * @brief Quit program + */ +static void quit_program() { + // Cleanup before exit + if (g_prompt) { + ml_lxm_prompt_destroy(g_prompt); + g_prompt = NULL; + } + if (g_session) { + ml_lxm_session_destroy(g_session); + g_session = NULL; + } + g_main_loop_quit(loop); +} + +/** + * @brief Reset menu status + */ +void reset_menu_status(void) { g_menu_state = CURRENT_STATUS_MAINMENU; } + +/** + * @brief Display current menu + */ +static void display_menu() { + if (g_menu_state == CURRENT_STATUS_MAINMENU) { + main_menu(); + } else if (g_menu_state == CURRENT_STATUS_SESSION_CREATE) { + g_print("*** Enter config path: "); + } else if (g_menu_state == CURRENT_STATUS_SESSION_SET_INSTRUCTION) { + g_print("*** Enter new instructions: "); + } else if (g_menu_state == CURRENT_STATUS_PROMPT_APPEND_TEXT) { + g_print("*** Enter text to append: "); + } else if (g_menu_state == CURRENT_STATUS_PROMPT_APPEND_INSTRUCTION) { + g_print("*** Enter instruction to append: "); + } else { + g_print("*** Unknown status. \n"); + reset_menu_status(); + } +} + +/** + * @brief Interpret main menu commands + */ +static void interpret_main_menu(char cmd) { + int ret = ML_ERROR_NONE; + + ml_lxm_generation_options_s options = {.temperature = 1.2, .max_tokens = 128}; + + switch (cmd) { + case 'a': + if (g_session) { + g_print("Session already exists!\n"); + break; + } + g_menu_state = CURRENT_STATUS_SESSION_CREATE; + break; + case 'b': + if (!g_session) { + g_print("No active session!\n"); + break; + } + ret = ml_lxm_session_destroy(g_session); + if (ret == 0) { + g_session = NULL; + g_print("Session destroyed.\n"); + } else { + g_print("Session destruction failed: %d\n", ret); + } + break; + case 'c': + if (!g_session) { + g_print("Create a session first!\n"); + break; + } + g_menu_state = CURRENT_STATUS_SESSION_SET_INSTRUCTION; + break; + + case 'd': + if (!g_session || !g_prompt) { + g_print("Create session and prompt first!\n"); + break; + } + + ret = ml_lxm_session_respond(g_session, g_prompt, &options, + token_handler, NULL); + if (ret == 0) { + g_print("\nAsync response started...\n"); + } else { + g_print("Async response failed: %d\n", ret); + } + break; + case 'e': + if (g_prompt) { + g_print("Prompt already exists! Destroy first.\n"); + break; + } + ret = ml_lxm_prompt_create(&g_prompt); + if (ret == 0) { + g_print("Prompt created.\n"); + } else { + g_print("Prompt creation failed: %d\n", ret); + } + break; + case 'f': + if (!g_prompt) { + g_print("No active prompt!\n"); + break; + } + ret = ml_lxm_prompt_destroy(g_prompt); + if (ret == 0) { + g_prompt = NULL; + g_print("Prompt destroyed.\n"); + } else { + g_print("Prompt destruction failed: %d\n", ret); + } + break; + case 'g': + if (!g_prompt) { + g_print("Create a prompt first!\n"); + break; + } + g_menu_state = CURRENT_STATUS_PROMPT_APPEND_TEXT; + break; + + case 'h': + if (!g_prompt) { + g_print("Create a prompt first!\n"); + break; + } + g_menu_state = CURRENT_STATUS_PROMPT_APPEND_INSTRUCTION; + break; + case 'q': + quit_program(); + return; + default: + g_print("Invalid option!\n"); + } + + if (ret != ML_ERROR_NONE && cmd != 'q') { + g_print("Operation failed with error: %d\n", ret); + } +} + +/** + * @brief Handle input for non-main-menu states + */ +static void handle_special_input(const char *input) { + int ret = ML_ERROR_NONE; + + switch (g_menu_state) { + case CURRENT_STATUS_SESSION_CREATE: + g_critical("Input: %s\n", input); + ret = ml_lxm_session_create(&g_session, input, "Default instructions."); + if (ret == 0) { + g_print("Session created successfully.\n"); + } else { + g_print("Session creation failed: %d\n", ret); + } + break; + case CURRENT_STATUS_SESSION_SET_INSTRUCTION: + ret = ml_lxm_session_set_instructions(g_session, input); + if (ret == 0) { + snprintf(g_instructions, MAX_STRING_LEN, "%s", input); + g_print("Instructions updated.\n"); + } else { + g_print("Failed to set instructions: %d\n", ret); + } + break; + case CURRENT_STATUS_PROMPT_APPEND_TEXT: + ret = ml_lxm_prompt_append_text(g_prompt, input); + if (ret == 0) { + snprintf(g_prompt_text, MAX_STRING_LEN, "%s", input); + g_print("Text appended to prompt.\n"); + } else { + g_print("Failed to append text: %d\n", ret); + } + break; + case CURRENT_STATUS_PROMPT_APPEND_INSTRUCTION: + ret = ml_lxm_prompt_append_instruction(g_prompt, input); + if (ret == 0) { + snprintf(g_prompt_text, MAX_STRING_LEN, "%s", input); + g_print("Instruction appended to prompt.\n"); + } else { + g_print("Failed to append instruction: %d\n", ret); + } + break; + + default: + break; + } + + reset_menu_status(); +} + +/** + * @brief Input handler + */ +gboolean input_handler(GIOChannel *channel, GIOCondition condition, + gpointer data) { + char buf[MAX_STRING_LEN]; + ssize_t cnt = read(STDIN_FILENO, buf, sizeof(buf) - 1); + + if (cnt <= 0) + return TRUE; /* Keep watching */ + + buf[cnt] = '\0'; + + if (g_menu_state == CURRENT_STATUS_MAINMENU && cnt == 2) { + interpret_main_menu(buf[0]); + } else if (g_menu_state != CURRENT_STATUS_MAINMENU) { + /* Remove newline if present */ + if (buf[cnt - 1] == '\n') + buf[cnt - 1] = '\0'; + handle_special_input(buf); + } + + display_menu(); + return TRUE; +} + +/** + * @brief Main function + */ +int main(int argc, char **argv) { + loop = g_main_loop_new(NULL, FALSE); + GIOChannel *stdin_channel = g_io_channel_unix_new(STDIN_FILENO); + guint watch_id = g_io_add_watch(stdin_channel, static_cast(G_IO_IN | G_IO_HUP), input_handler, NULL); + + display_menu(); + g_main_loop_run(loop); + + if (g_prompt) + ml_lxm_prompt_destroy(g_prompt); + if (g_session) + ml_lxm_session_destroy(g_session); + if (watch_id > 0) + g_source_remove(watch_id); + if (stdin_channel) + g_io_channel_unref(stdin_channel); + + return 0; +} diff --git a/Tizen.platform/lxm_service/gen-ml-lxm-service-rpm.sh b/Tizen.platform/lxm_service/gen-ml-lxm-service-rpm.sh new file mode 100755 index 00000000..b84d9866 --- /dev/null +++ b/Tizen.platform/lxm_service/gen-ml-lxm-service-rpm.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +git init +git add * +git commit -m 'Initial commit' +gbs build -A armv7l --include-all --clean +rm -rf .git diff --git a/Tizen.platform/lxm_service/include/ml-lxm-service.h b/Tizen.platform/lxm_service/include/ml-lxm-service.h new file mode 100644 index 00000000..e01cd4b0 --- /dev/null +++ b/Tizen.platform/lxm_service/include/ml-lxm-service.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/** + * @file ml-lxm-service.h + * @date 23 JULY 2025 + * @brief Machine Learning LXM(LLM, LVM, etc.) Service API + * @see https://github.com/nnstreamer/api + * @author Hyunil Park + * @bug No known bugs except for NYI items + */ + +#ifndef __ML_LXM_SERVICE_H__ +#define __ML_LXM_SERVICE_H__ + +#include +#include +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * @brief Enumeration for LXM service availability status. + */ +typedef enum +{ + ML_LXM_AVAILABILITY_AVAILABLE = 0, + ML_LXM_AVAILABILITY_DEVICE_NOT_ELIGIBLE, + ML_LXM_AVAILABILITY_SERVICE_DISABLED, + ML_LXM_AVAILABILITY_MODEL_NOT_READY, + ML_LXM_AVAILABILITY_UNKNOWN +} ml_lxm_availability_e; + +/** + * @brief Checks LXM service availability. + * @param[out] status Current availability status. + * @return ML_ERROR_NONE on success, error code otherwise. + */ +int ml_lxm_check_availability (ml_lxm_availability_e * status); + +/** + * @brief A handle for lxm session. + */ +typedef void *ml_lxm_session_h; + +/** + * @brief Creates an LXM session. + * @param[out] session Session handle. + * @param[in] config_path Path to configuration file. + * @param[in] instructions Initial instructions (optional). + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_session_create (ml_lxm_session_h * session, const char *config_path, const char *instructions); + +/** + * @brief Destroys an LXM session. + * @param[in] session Session handle. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_session_destroy (ml_lxm_session_h session); + +/** + * @brief A handle for lxm prompt. + */ +typedef void *ml_lxm_prompt_h; + +/** + * @brief Creates a prompt object. + * @param[out] prompt Prompt handle. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_prompt_create (ml_lxm_prompt_h * prompt); + +/** + * @brief Appends text to a prompt. + * @param[in] prompt Prompt handle. + * @param[in] text Text to append. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_prompt_append_text (ml_lxm_prompt_h prompt, const char *text); + +/** + * @brief Appends an instruction to a prompt. + * @param[in] prompt Prompt handle. + * @param[in] instruction Instruction to append. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_prompt_append_instruction (ml_lxm_prompt_h prompt, const char *instruction); + +/** + * @brief Destroys a prompt object. + * @param[in] prompt Prompt handle. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_prompt_destroy (ml_lxm_prompt_h prompt); + +/** + * @brief Sets runtime instructions for a session. + * @param[in] session Session handle. + * @param[in] instructions New instructions. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_session_set_instructions (ml_lxm_session_h session, const char *instructions); + +/** + * @brief Generation options for LXM responses. + */ +typedef struct +{ + double temperature; /* < Creativity control (0.0~2.0) */ + size_t max_tokens; /* < Maximum tokens to generate */ +} ml_lxm_generation_options_s; + +/** + * @brief Token streaming callback type. + * @param token Generated token string. + * @param user_data User-defined context. + */ +typedef void (*ml_lxm_token_cb) (ml_service_event_e event, ml_information_h event_data, void *user_data); + +/** + * @brief Generates an token-streamed response. + * @param[in] session Session handle. + * @param[in] prompt Prompt handle. + * @param[in] options Generation parameters. + * @param[in] token_callback Callback for each generated token. + * @param[in] user_data User context passed to callback. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_session_respond (ml_lxm_session_h session, ml_lxm_prompt_h prompt, const ml_lxm_generation_options_s * options, ml_lxm_token_cb token_callback, void *user_data); + +#ifdef __cplusplus +} +#endif +#endif +/* __ML_LXM_SERVICE_H__ */ diff --git a/Tizen.platform/lxm_service/meson.build b/Tizen.platform/lxm_service/meson.build new file mode 100644 index 00000000..e03997e5 --- /dev/null +++ b/Tizen.platform/lxm_service/meson.build @@ -0,0 +1,48 @@ +project('ml-lxm-service', 'c', 'cpp', + version: '0.1.0', + license: ['LGPL-2.1'], + meson_version: '>=0.50.0', + default_options: [ + 'werror=true', + 'warning_level=1', + 'c_std=gnu89', + 'cpp_std=c++11' + ] +) + +# Install paths +prefix = get_option('prefix') +bindir = join_paths(prefix, get_option('bindir')) +libdir = join_paths(prefix, get_option('libdir')) +includedir = join_paths(prefix, get_option('includedir')) + +project_inc = include_directories('include') + +# Dependencies +glib_dep = dependency('glib-2.0') +nns_capi_common_dep = dependency('capi-ml-common', required: true) +ml_service_dep = dependency('capi-ml-service', required: true) + +lxm_service_sources = [ + 'src/ml-lxm-service.c' +] + +lxm_service_lib = shared_library( + 'ml-lxm-service', + sources: lxm_service_sources, + dependencies: [glib_dep, nns_capi_common_dep, ml_service_dep], + include_directories: project_inc, + install: true, + install_dir: libdir +) + +install_headers('include/ml-lxm-service.h', subdir: 'ml-lxm-service') + +executable('ml-lxm-service-example', + 'example/main.cc', + dependencies: [glib_dep, nns_capi_common_dep, ml_service_dep], + link_with: lxm_service_lib, + include_directories: project_inc, + install: true, + install_dir: bindir +) diff --git a/Tizen.platform/lxm_service/packaging/ml-lxm-service.manifest b/Tizen.platform/lxm_service/packaging/ml-lxm-service.manifest new file mode 100644 index 00000000..017d22d3 --- /dev/null +++ b/Tizen.platform/lxm_service/packaging/ml-lxm-service.manifest @@ -0,0 +1,5 @@ + + + + + diff --git a/Tizen.platform/lxm_service/packaging/ml-lxm-service.spec b/Tizen.platform/lxm_service/packaging/ml-lxm-service.spec new file mode 100644 index 00000000..ce4717ff --- /dev/null +++ b/Tizen.platform/lxm_service/packaging/ml-lxm-service.spec @@ -0,0 +1,46 @@ +%define nnstexampledir /usr/lib/nnstreamer/bin + +Name: ml-lxm-service +Summary: ML LXM Service API +Version: 1.0.0 +Release: 0 +Group: Machine Learning/ML Framework +Packager: Hyunil Park +License: LGPL-2.1 +Source0: %{name}-%{version}.tar.gz +Source1001: %{name}.manifest + +Requires: nnstreamer +BuildRequires: meson +BuildRequires: pkgconfig(glib-2.0) +BuildRequires: pkgconfig(capi-ml-service) + + +%description +ML LXM Service API + +%prep +%setup -q +cp %{SOURCE1001} . + +%build +mkdir -p build + +meson --buildtype=plain --prefix=%{_prefix} --libdir=%{_libdir} --bindir=%{nnstexampledir} --includedir=%{_includedir} build +ninja -C build %{?_smp_mflags} + +%install +DESTDIR=%{buildroot} ninja -C build install + +find %{buildroot}%{_libdir} -name 'libml-lxm-service.so*' -print + +install -d %{buildroot}%{_includedir}/ml-lxm-service + +%files +%manifest %{name}.manifest +%defattr(-,root,root,-) +%{nnstexampledir}/* +%dir %{_includedir}/ml-lxm-service +%{_libdir}/libml-lxm-service.so* +%{_includedir}/ml-lxm-service/ml-lxm-service.h +%{nnstexampledir}/* diff --git a/Tizen.platform/lxm_service/src/ml-lxm-service.c b/Tizen.platform/lxm_service/src/ml-lxm-service.c new file mode 100644 index 00000000..9b04e895 --- /dev/null +++ b/Tizen.platform/lxm_service/src/ml-lxm-service.c @@ -0,0 +1,262 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/** + * @file ml-lxm-service.c + * @date 23 JULY 2025 + * @brief Machine Learning LXM(LLM, LVM, etc.) Service API + * @see https://github.com/nnstreamer/api + * @author Hyunil Park + * @bug No known bugs except for NYI items + */ +#include +#include +#include +#include +#include "ml-lxm-service.h" + +/** + * @brief Internal structure for the session. + */ +typedef struct +{ + ml_service_h service_handle; + char *config_path; + char *instructions; +} ml_lxm_session_internal; + +/** + * @brief Internal structure for the prompt. + */ +typedef struct +{ + GString *text; +} ml_lxm_prompt_internal; + +/** + * @brief Checks LXM service availability. + * @param[out] status Current availability status. + * @return ML_ERROR_NONE on success, error code otherwise. + */ +int +ml_lxm_check_availability (ml_lxm_availability_e * status) +{ + *status = ML_LXM_AVAILABILITY_AVAILABLE; + + return ML_ERROR_NONE; +} + +/** + * @brief Creates an LXM session. + * @param[out] session Session handle. + * @param[in] config_path Path to configuration file. + * @param[in] instructions Initial instructions (optional). + * @return ML_ERROR_NONE on success. + */ +int +ml_lxm_session_create (ml_lxm_session_h * session, const char *config_path, + const char *instructions) +{ + if (!session || !config_path) + return ML_ERROR_INVALID_PARAMETER; + + ml_lxm_session_internal *s = g_malloc0 (sizeof (ml_lxm_session_internal)); + if (!s) + return ML_ERROR_OUT_OF_MEMORY; + + s->config_path = g_strdup (config_path); + s->instructions = instructions ? g_strdup (instructions) : NULL; + + int ret = ml_service_new (s->config_path, &s->service_handle); + if (ret != ML_ERROR_NONE) { + g_free (s->config_path); + g_free (s->instructions); + g_free (s); + return ret; + } + + *session = s; + + return ML_ERROR_NONE; +} + +/** + * @brief Destroys an LXM session. + * @param[in] session Session handle. + * @return ML_ERROR_NONE on success. + */ +int +ml_lxm_session_destroy (ml_lxm_session_h session) +{ + if (!session) + return ML_ERROR_INVALID_PARAMETER; + + ml_lxm_session_internal *s = (ml_lxm_session_internal *) session; + + ml_service_destroy (s->service_handle); + + g_free (s->config_path); + g_free (s->instructions); + g_free (s); + + return ML_ERROR_NONE; +} + +/** + * @brief Creates a prompt object. + * @param[out] prompt Prompt handle. + * @return ML_ERROR_NONE on success. + */ +int +ml_lxm_prompt_create (ml_lxm_prompt_h * prompt) +{ + if (!prompt) + return ML_ERROR_INVALID_PARAMETER; + + ml_lxm_prompt_internal *p = g_malloc0 (sizeof (ml_lxm_prompt_internal)); + if (!p) + return ML_ERROR_OUT_OF_MEMORY; + + p->text = g_string_new (""); + *prompt = p; + + return ML_ERROR_NONE; +} + +/** + * @brief Destroys a prompt object. + * @param[in] prompt Prompt handle. + * @return ML_ERROR_NONE on success. + */ +int +ml_lxm_prompt_destroy (ml_lxm_prompt_h prompt) +{ + if (!prompt) + return ML_ERROR_INVALID_PARAMETER; + + ml_lxm_prompt_internal *p = (ml_lxm_prompt_internal *) prompt; + g_string_free (p->text, TRUE); + g_free (p); + + return ML_ERROR_NONE; +} + +/** + * @brief Appends text to a prompt. + * @param[in] prompt Prompt handle. + * @param[in] text Text to append. + * @return ML_ERROR_NONE on success. + */ +int +ml_lxm_prompt_append_text (ml_lxm_prompt_h prompt, const char *text) +{ + if (!prompt || !text) + return ML_ERROR_INVALID_PARAMETER; + + ml_lxm_prompt_internal *p = (ml_lxm_prompt_internal *) prompt; + g_string_append (p->text, text); + + return ML_ERROR_NONE; +} + +/** + * @brief Appends an instruction to a prompt. + * @param[in] prompt Prompt handle. + * @param[in] instruction Instruction to append. + * @return ML_ERROR_NONE on success. + */ +int +ml_lxm_prompt_append_instruction (ml_lxm_prompt_h prompt, + const char *instruction) +{ + return ml_lxm_prompt_append_text (prompt, instruction); +} + +/** + * @brief Sets runtime instructions for a session. + * @param[in] session Session handle. + * @param[in] instructions New instructions. + * @return ML_ERROR_NONE on success. + */ +int +ml_lxm_session_set_instructions (ml_lxm_session_h session, + const char *instructions) +{ + if (!session) + return ML_ERROR_INVALID_PARAMETER; + + ml_lxm_session_internal *s = (ml_lxm_session_internal *) session; + g_free (s->instructions); + s->instructions = instructions ? g_strdup (instructions) : NULL; + + return ML_ERROR_NONE; +} + +/** + * @brief Generates an token-streamed response. + * @param[in] session Session handle. + * @param[in] prompt Prompt handle. + * @param[in] options Generation parameters. + * @param[in] token_callback Callback for each generated token. + * @param[in] user_data User context passed to callback. + * @return ML_ERROR_NONE on success. + */ +int +ml_lxm_session_respond (ml_lxm_session_h session, + ml_lxm_prompt_h prompt, + const ml_lxm_generation_options_s * options, + ml_lxm_token_cb token_callback, void *user_data) +{ + int ret = ML_ERROR_NONE; + + if (!session || !prompt || !token_callback) + return ML_ERROR_INVALID_PARAMETER; + + ml_lxm_session_internal *s = (ml_lxm_session_internal *) session; + ml_lxm_prompt_internal *p = (ml_lxm_prompt_internal *) prompt; + + GString *full_input = g_string_new (""); + if (s->instructions && s->instructions[0] != '\0') { + g_string_append (full_input, s->instructions); + g_string_append (full_input, "\n"); + } + g_string_append (full_input, p->text->str); + + /* Prepare input data */ + ml_tensors_info_h input_info = NULL; + ml_tensors_data_h input_data = NULL; + + ret = ml_service_set_event_cb (s->service_handle, token_callback, user_data); + if (ret != ML_ERROR_NONE) + goto error; + + /* Get input information */ + ret = ml_service_get_input_information (s->service_handle, NULL, &input_info); + if (ret != ML_ERROR_NONE) + goto error; + + /* Create input data */ + ret = ml_tensors_data_create (input_info, &input_data); + if (ret != ML_ERROR_NONE) { + goto error; + } + + /* Set tensor data */ + ret = + ml_tensors_data_set_tensor_data (input_data, 0U, full_input->str, + full_input->len); + if (ret != ML_ERROR_NONE) { + goto error; + } + + /* Send request */ + ret = ml_service_request (s->service_handle, NULL, input_data); + +error: + if (input_info) + ml_tensors_info_destroy (input_info); + if (input_data) + ml_tensors_data_destroy (input_data); + if (full_input) + g_string_free (full_input, TRUE); + + return ret; +} From 64b8d1a43ea63a163f719fe1d4341986b336f0f6 Mon Sep 17 00:00:00 2001 From: hyunil park Date: Fri, 25 Jul 2025 14:47:34 +0900 Subject: [PATCH 2/2] [Doc] Add README.md for LXM Service API and example Add LXM Service API README.md Signed-off-by: hyunil park --- Tizen.platform/lxm_service/README.md | 287 +++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) diff --git a/Tizen.platform/lxm_service/README.md b/Tizen.platform/lxm_service/README.md index 3e043d0a..f43daa1d 100644 --- a/Tizen.platform/lxm_service/README.md +++ b/Tizen.platform/lxm_service/README.md @@ -3,3 +3,290 @@ title: On device Machine LXM(LLM, LVM, etc.) Service API example ... ## On device Machine LXM(LLM, LVM, etc.) Service API example + +This example demonstrates how to implement a Large Language Model (LLM) service using `ml-lxm-service`(unreleased). + +## Overview +The sample application provides an interactive CLI(Command Line Interface) to: +- Create/destroy LXM sessions +- Set instructions +- Create/destroy prompts +- Append text/instructions to prompts +- Generate token-streamed responses + + + +## Prerequisites +- `ml-api-service` and `flare` development packages installed on your target device + +## Building the Example +### Build +``` +meson setup build --prefix=${NNST_ROOT} --libdir=lib --bindir=bin --includedir=include +ninja -C build install +``` +### For rpm +``` +./gen-ml-lxm-service-rpm.sh +``` +### Run +``` +cp config.conf tokenlizer.json sflare_if_4bit_3b.bin ${PROJECT}/ +./build/ml-lxm-service-example +``` + + +## API Reference + +#### LXM service availability status. +```cpp +typedef enum +{ + ML_LXM_AVAILABILITY_AVAILABLE = 0, + ML_LXM_AVAILABILITY_DEVICE_NOT_ELIGIBLE, + ML_LXM_AVAILABILITY_SERVICE_DISABLED, + ML_LXM_AVAILABILITY_MODEL_NOT_READY, + ML_LXM_AVAILABILITY_UNKNOWN +} ml_lxm_availability_e; +``` +#### Availability Check +```cpp +/** + * @brief Checks LXM service availability. + * @param[out] status Current availability status. + * @return ML_ERROR_NONE on success, error code otherwise. + */ +int ml_lxm_check_availability (ml_lxm_availability_e * status); +``` + +### Data Type +```cpp +/** + * @brief Token streaming callback type. + * @param token Generated token string. + * @param user_data User-defined context. + */ +typedef void (*ml_lxm_token_cb)(ml_service_event_e event, ml_information_h event_data, void *user_data); + +/** + * @brief Generation options for LXM responses. + */ +typedef struct { + double temperature; + size_t max_tokens; +} ml_lxm_generation_options_s; +``` +#### Session Management +```cpp +/** + * @brief Creates an LXM session. + * @param[out] session Session handle. + * @param[in] config_path Path to configuration file. + * @param[in] instructions Initial instructions (optional). + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_session_create(ml_lxm_session_h *session, const char *config_path, const char *instructions); + +/** + * @brief Destroys an LXM session. + * @param[in] session Session handle. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_session_destroy(ml_lxm_session_h session); + +/** + * @brief Sets runtime instructions for a session. + * @param[in] session Session handle. + * @param[in] instructions New instructions. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_session_set_instructions(ml_lxm_session_h session, const char *instructions); +``` + +#### Prompt Handling +```cpp +/** + * @brief Creates a prompt object. + * @param[out] prompt Prompt handle. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_prompt_create(ml_lxm_prompt_h *prompt); + +/** + * @brief Destroys a prompt object. + * @param[in] prompt Prompt handle. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_prompt_destroy(ml_lxm_prompt_h prompt); + +/** + * @brief Appends text to a prompt. + * @param[in] prompt Prompt handle. + * @param[in] text Text to append. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_prompt_append_text(ml_lxm_prompt_h prompt, const char *text); + +/** + * @brief Appends an instruction to a prompt. + * @param[in] prompt Prompt handle. + * @param[in] instruction Instruction to append. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_prompt_append_instruction(ml_lxm_prompt_h prompt, const char *instruction); +``` + +#### Response Generation +```cpp +/** + * @brief Generates an token-streamed response. + * @param[in] session Session handle. + * @param[in] prompt Prompt handle. + * @param[in] options Generation parameters. + * @param[in] token_callback Callback for each generated token. + * @param[in] user_data User context passed to callback. + * @return ML_ERROR_NONE on success. + */ +int ml_lxm_session_respond( + ml_lxm_session_h session, + ml_lxm_prompt_h prompt, + const ml_lxm_generation_options_s *options, + ml_lxm_token_cb token_cb, + void *user_data +); +``` +#### Error Codes +- ML_ERROR_NONE: Operation successful +- ML_ERROR_INVALID_PARAMETER: Invalid parameters detected +- ML_ERROR_OUT_OF_MEMORY: Memory allocation failure +- ML_ERROR_IO_ERROR: File/DB operation failure + + +## Sample Code Explanation + +### Configuration file +``` +{ + "single" : + { + "framework" : "flare", + "model" : ["sflare_if_4bit_3b.bin"], + "adapter" : ["history_lora.bin"], + "custom" : "tokenizer_path:tokenizer.json,backend:CPU,output_size:1024,model_type:3B,data_type:W4A32", + "invoke_dynamic" : "true", + } +} +``` + +### Key Components +```cpp +#include +#include "ml-lxm-service.h" + +// Global handles +ml_lxm_session_h g_session = NULL; +ml_lxm_prompt_h g_prompt = NULL; + +``` +### Main Workflow +1. Session Creation +```cpp +ret = ml_lxm_session_create(&g_session, config_path, "Default instructions"); +``` +2. Prompt Handling +```cpp +ml_lxm_prompt_create(&g_prompt); +ml_lxm_prompt_append_text(g_prompt, "Explain quantum computing"); +``` +3. Response Generation +```cpp +ml_lxm_generation_options_s options = { + .temperature = 1.2, + .max_tokens = 128 +}; + +ml_lxm_session_respond( + g_session, + g_prompt, + &options, + token_handler, + NULL +); +``` +4. Token Callback +```cpp +static void token_handler( + ml_service_event_e event, + ml_information_h event_data, + void *user_data +) { + /* Process tokens here */ +} +``` +5. Cleanup +```cpp +ml_lxm_prompt_destroy(g_prompt); +ml_lxm_session_destroy(g_session); +``` +### Full Example Snippet +```cpp +#include +#include "ml-lxm-service.h" + +static void token_handler(ml_service_event_e event, + ml_information_h event_data, + void *user_data); +int main() { + ml_lxm_session_h session; + ml_lxm_prompt_h prompt; + + // 1. Create session + ml_lxm_session_create(&session, "/path/to/config", NULL); + + // 2. Create prompt + ml_lxm_prompt_create(&prompt); + ml_lxm_prompt_append_text(prompt, "Hello AI"); + + // 3. Generate response + ml_lxm_generation_options_s options = {1.0, 50}; + ml_lxm_session_respond(session, prompt, &options, token_handler, NULL); + + // 4. Cleanup + ml_lxm_prompt_destroy(prompt); + ml_lxm_session_destroy(session); + + return 0; +} + +static void token_handler(ml_service_event_e event, + ml_information_h event_data, + void *user_data) { + ml_tensors_data_h data = NULL; + void *_raw = NULL; + size_t _size = 0; + int ret; + + switch (event) { + case ML_SERVICE_EVENT_NEW_DATA: + if (event_data != NULL) { + + ret = ml_information_get(event_data, "data", &data); + if (ret != ML_ERROR_NONE) { + g_print("Failed to get data from event_data\n"); + return; + } + + ret = ml_tensors_data_get_tensor_data(data, 0U, &_raw, &_size); + if (ret != ML_ERROR_NONE) { + g_print("Failed to get tensor data\n"); + return; + } + + std::cout.write(static_cast(_raw), _size); + std::cout.flush(); + } + default: + break; + } +} +```