diff --git a/.gitignore b/.gitignore index 935538441..8bd6b31fc 100644 --- a/.gitignore +++ b/.gitignore @@ -355,3 +355,5 @@ site/ /Llama.Mobile/Resources/Raw /nuget_pack.bat /temp +LLama.Web/wwwroot/lib/ +LLama.Web/Models/ diff --git a/LLama.Web/App.razor b/LLama.Web/App.razor new file mode 100644 index 000000000..a50a5f9d1 --- /dev/null +++ b/LLama.Web/App.razor @@ -0,0 +1,13 @@ + + + + + + +
+

Page not found

+

The page you requested does not exist.

+
+ + + diff --git a/LLama.Web/Async/AsyncGuard.cs b/LLama.Web/Async/AsyncGuard.cs index ff6b6c43a..9dd0d425d 100644 --- a/LLama.Web/Async/AsyncGuard.cs +++ b/LLama.Web/Async/AsyncGuard.cs @@ -4,7 +4,7 @@ namespace LLama.Web.Async { /// - /// Creates a async/thread-safe guard helper + /// Creates an async/thread-safe guard helper. /// /// public class AsyncGuard : AsyncGuard @@ -26,7 +26,7 @@ public AsyncGuard() /// /// Guards this instance. /// - /// true if able to enter an guard, false if already guarded + /// true if able to enter a guard; false if already guarded. public bool Guard() { return _lockData.TryAdd(_key, true); @@ -74,7 +74,7 @@ public AsyncGuard() /// Guards the specified value. /// /// The value. - /// true if able to enter a guard for this value, false if this value is already guarded + /// true if able to enter a guard for this value; false if this value is already guarded. public bool Guard(T value) { return _lockData.TryAdd(value, true); diff --git a/LLama.Web/Async/AsyncLock.cs b/LLama.Web/Async/AsyncLock.cs index df294bf8b..6d839c1c1 100644 --- a/LLama.Web/Async/AsyncLock.cs +++ b/LLama.Web/Async/AsyncLock.cs @@ -1,7 +1,7 @@ namespace LLama.Web.Async { /// - /// Create an Async locking using statement + /// Creates an async lock for a using block. /// public sealed class AsyncLock { @@ -20,7 +20,7 @@ public AsyncLock() /// - /// Locks the using statement asynchronously. + /// Acquires the lock asynchronously. /// /// public Task LockAsync() @@ -34,7 +34,7 @@ public Task LockAsync() /// - /// IDisposable wrapper class to release the lock on dispose + /// IDisposable wrapper to release the lock on dispose. /// /// private sealed class Releaser : IDisposable diff --git a/LLama.Web/Common/ModelLoadType.cs b/LLama.Web/Common/ModelLoadType.cs index 9e1c77b7b..d4664637c 100644 --- a/LLama.Web/Common/ModelLoadType.cs +++ b/LLama.Web/Common/ModelLoadType.cs @@ -1,29 +1,29 @@ namespace LLama.Web.Common { /// - /// The type of model load caching to use + /// The type of model load caching to use. /// public enum ModelLoadType { /// - /// Only one model will be loaded into memory at a time, any other models will be unloaded before the new one is loaded + /// Only one model is loaded into memory at a time; any other models are unloaded before the new one is loaded. /// Single = 0, /// - /// Multiple models will be loaded into memory, ensure you use the ModelConfigs to split the hardware resources + /// Multiple models are loaded into memory; ensure you use the ModelConfigs to split hardware resources. /// Multiple = 1, /// - /// The first model in the appsettings.json list will be preloaded into memory at app startup + /// The first model in the appsettings.json list is preloaded into memory at app startup. /// PreloadSingle = 2, /// - /// All models in the appsettings.json list will be preloaded into memory at app startup, ensure you use the ModelConfigs to split the hardware resources + /// All models in the appsettings.json list are preloaded into memory at app startup; ensure you use the ModelConfigs to split hardware resources. /// PreloadMultiple = 3, } diff --git a/LLama.Web/Common/ModelOptions.cs b/LLama.Web/Common/ModelOptions.cs index 586db5611..da12b06f7 100644 --- a/LLama.Web/Common/ModelOptions.cs +++ b/LLama.Web/Common/ModelOptions.cs @@ -8,12 +8,12 @@ public class ModelOptions : ILLamaParams { /// - /// Model friendly name + /// Model-friendly name. /// public string Name { get; set; } /// - /// Max context insta=nces allowed per model + /// Max context instances allowed per model. /// public int MaxInstances { get; set; } @@ -47,6 +47,21 @@ public class ModelOptions /// public string ModelPath { get; set; } + /// + /// Download URL for the model file (GGUF). + /// + public string ModelDownloadUrl { get; set; } + + /// + /// Optional local path to the mmproj (multimodal projection) file. + /// + public string MmprojPath { get; set; } + + /// + /// Download URL for the mmproj file, if required. + /// + public string MmprojDownloadUrl { get; set; } + /// public int? Threads { get; set; } @@ -128,4 +143,4 @@ public class ModelOptions /// public LLamaAttentionType AttentionType { get; set; } = LLamaAttentionType.Unspecified; } -} \ No newline at end of file +} diff --git a/LLama.Web/Controllers/AttachmentController.cs b/LLama.Web/Controllers/AttachmentController.cs new file mode 100644 index 000000000..fe57dc6a2 --- /dev/null +++ b/LLama.Web/Controllers/AttachmentController.cs @@ -0,0 +1,58 @@ +using LLama.Web.Models; +using LLama.Web.Services; +using Microsoft.AspNetCore.Mvc; + +namespace LLama.Web.Controllers; + +[ApiController] +[Route("api/attachments")] +public class AttachmentController : ControllerBase +{ + private readonly IAttachmentService _attachmentService; + + public AttachmentController(IAttachmentService attachmentService) + { + _attachmentService = attachmentService; + } + + [HttpPost] + [RequestSizeLimit(256_000_000)] + public async Task> Upload([FromForm] string connectionId, [FromForm] List files, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(connectionId)) + return BadRequest("Missing connectionId."); + + if (files is null || files.Count == 0) + return BadRequest("No files provided."); + + try + { + var result = await _attachmentService.SaveAsync(connectionId, files, cancellationToken); + return Ok(result); + } + catch (InvalidOperationException ex) + { + return BadRequest(ex.Message); + } + } + + [HttpGet("{connectionId}/{id}")] + public ActionResult Download(string connectionId, string id) + { + if (string.IsNullOrWhiteSpace(connectionId) || string.IsNullOrWhiteSpace(id)) + return BadRequest("Missing connectionId or attachment id."); + + var attachment = _attachmentService.GetAttachment(connectionId, id); + if (attachment == null || string.IsNullOrWhiteSpace(attachment.FilePath)) + return NotFound(); + + if (!System.IO.File.Exists(attachment.FilePath)) + return NotFound(); + + var contentType = string.IsNullOrWhiteSpace(attachment.ContentType) + ? "application/octet-stream" + : attachment.ContentType; + + return PhysicalFile(attachment.FilePath, contentType, attachment.FileName); + } +} diff --git a/LLama.Web/Extensions.cs b/LLama.Web/Extensions.cs index 1fe08f6eb..d8995f082 100644 --- a/LLama.Web/Extensions.cs +++ b/LLama.Web/Extensions.cs @@ -2,10 +2,10 @@ namespace LLama.Web; -public static class Extensions +public static class Extensions { /// - /// Combines the AntiPrompts list and AntiPrompt csv + /// Combines the AntiPrompts list and AntiPrompt CSV. /// /// The session configuration. /// Combined AntiPrompts with duplicates removed @@ -15,7 +15,7 @@ public static List GetAntiPrompts(this ISessionConfig sessionConfig) } /// - /// Combines the OutputFilters list and OutputFilter csv + /// Combines the OutputFilters list and OutputFilter CSV. /// /// The session configuration. /// Combined OutputFilters with duplicates removed @@ -25,7 +25,7 @@ public static List GetOutputFilters(this ISessionConfig sessionConfig) } /// - /// Combines a string list and a csv and removes duplicates + /// Combines a string list and a CSV, and removes duplicates. /// /// The list. /// The CSV. diff --git a/LLama.Web/Hubs/ISessionClient.cs b/LLama.Web/Hubs/ISessionClient.cs index 92302b219..552fa6133 100644 --- a/LLama.Web/Hubs/ISessionClient.cs +++ b/LLama.Web/Hubs/ISessionClient.cs @@ -7,5 +7,8 @@ public interface ISessionClient { Task OnStatus(string connectionId, SessionConnectionStatus status); Task OnError(string error); + Task OnModelDownloadSnapshot(IReadOnlyList snapshots); + Task OnModelDownloadProgress(ModelDownloadProgress progress); + Task OnStorageInfo(StorageInfo info); } } diff --git a/LLama.Web/Hubs/SessionConnectionHub.cs b/LLama.Web/Hubs/SessionConnectionHub.cs index b6a977e36..425aaf8e1 100644 --- a/LLama.Web/Hubs/SessionConnectionHub.cs +++ b/LLama.Web/Hubs/SessionConnectionHub.cs @@ -9,19 +9,30 @@ public class SessionConnectionHub : Hub { private readonly ILogger _logger; private readonly IModelSessionService _modelSessionService; + private readonly IModelDownloadService _modelDownloadService; + private readonly IAttachmentService _attachmentService; - public SessionConnectionHub(ILogger logger, IModelSessionService modelSessionService) + public SessionConnectionHub(ILogger logger, IModelSessionService modelSessionService, IModelDownloadService modelDownloadService, IAttachmentService attachmentService) { _logger = logger; _modelSessionService = modelSessionService; + _modelDownloadService = modelDownloadService; + _attachmentService = attachmentService; } public override async Task OnConnectedAsync() { _logger.Log(LogLevel.Information, "[OnConnectedAsync], Id: {0}", Context.ConnectionId); - // Notify client of successful connection + // Notify the client of a successful connection. await Clients.Caller.OnStatus(Context.ConnectionId, SessionConnectionStatus.Connected); + await Clients.Caller.OnModelDownloadSnapshot(_modelDownloadService.GetSnapshots()); + await Clients.Caller.OnStorageInfo(new StorageInfo + { + ModelsPath = _modelDownloadService.ModelsRoot, + DownloadsPath = _modelDownloadService.DownloadsRoot, + UploadsPath = _attachmentService.UploadsRoot + }); await base.OnConnectedAsync(); } @@ -29,8 +40,9 @@ public override async Task OnDisconnectedAsync(Exception exception) { _logger.Log(LogLevel.Information, "[OnDisconnectedAsync], Id: {0}", Context.ConnectionId); - // Remove connections session on disconnect + // Remove the connection session on disconnect. await _modelSessionService.CloseAsync(Context.ConnectionId); + await _attachmentService.CleanupAsync(Context.ConnectionId); await base.OnDisconnectedAsync(exception); } @@ -40,24 +52,38 @@ public async Task OnLoadModel(SessionConfig sessionConfig, InferenceOptions infe _logger.Log(LogLevel.Information, "[OnLoadModel] - Load new model, Connection: {0}", Context.ConnectionId); await _modelSessionService.CloseAsync(Context.ConnectionId); - // Create model session - var modelSession = await _modelSessionService.CreateAsync(Context.ConnectionId, sessionConfig, inferenceConfig); - if (modelSession is null) + try + { + // Create model session. + var modelSession = await _modelSessionService.CreateAsync(Context.ConnectionId, sessionConfig, inferenceConfig); + if (modelSession is null) + { + await Clients.Caller.OnError("Failed to create model session"); + return; + } + } + catch (Exception ex) { - await Clients.Caller.OnError("Failed to create model session"); + await Clients.Caller.OnError(ex.Message); return; } - // Notify client + // Notify client. await Clients.Caller.OnStatus(Context.ConnectionId, SessionConnectionStatus.Loaded); } [HubMethodName("SendPrompt")] - public IAsyncEnumerable OnSendPrompt(string prompt, InferenceOptions inferConfig, CancellationToken cancellationToken) + public IAsyncEnumerable OnSendPrompt(PromptRequest request, InferenceOptions inferConfig, CancellationToken cancellationToken) { _logger.Log(LogLevel.Information, "[OnSendPrompt] - New prompt received, Connection: {0}", Context.ConnectionId); var linkedCancelationToken = CancellationTokenSource.CreateLinkedTokenSource(Context.ConnectionAborted, cancellationToken); - return _modelSessionService.InferAsync(Context.ConnectionId, prompt, inferConfig, linkedCancelationToken.Token); + return _modelSessionService.InferAsync(Context.ConnectionId, request, inferConfig, linkedCancelationToken.Token); + } + + [HubMethodName("GetModelStatuses")] + public Task> GetModelStatuses() + { + return Task.FromResult(_modelDownloadService.GetSnapshots()); } } diff --git a/LLama.Web/LLama.Web.csproj b/LLama.Web/LLama.Web.csproj index 06311bd07..eb9354d40 100644 --- a/LLama.Web/LLama.Web.csproj +++ b/LLama.Web/LLama.Web.csproj @@ -17,6 +17,7 @@ + diff --git a/LLama.Web/Models/AttachmentInfo.cs b/LLama.Web/Models/AttachmentInfo.cs new file mode 100644 index 000000000..65dbd436b --- /dev/null +++ b/LLama.Web/Models/AttachmentInfo.cs @@ -0,0 +1,32 @@ +namespace LLama.Web.Models; + +public enum AttachmentKind +{ + Unknown = 0, + Pdf = 1, + Image = 2, + Audio = 3, + Word = 4 +} + +public class AttachmentInfo +{ + public string Id { get; set; } + [System.Text.Json.Serialization.JsonIgnore] + public string ConnectionId { get; set; } + public string FileName { get; set; } + public string ContentType { get; set; } + [System.Text.Json.Serialization.JsonIgnore] + public string FilePath { get; set; } + public AttachmentKind Kind { get; set; } + public long SizeBytes { get; set; } + [System.Text.Json.Serialization.JsonIgnore] + public string ExtractedText { get; set; } + [System.Text.Json.Serialization.JsonIgnore] + public bool ExtractedTextTruncated { get; set; } +} + +public class AttachmentUploadResult +{ + public List Attachments { get; set; } = new(); +} diff --git a/LLama.Web/Models/LLamaModel.cs b/LLama.Web/Models/LLamaModel.cs index af228bd4c..ba9aa4c8d 100644 --- a/LLama.Web/Models/LLamaModel.cs +++ b/LLama.Web/Models/LLamaModel.cs @@ -1,4 +1,6 @@ +using LLama; using LLama.Abstractions; +using LLama.Native; using LLama.Web.Common; using System.Collections.Concurrent; @@ -13,6 +15,7 @@ public class LLamaModel : IDisposable private readonly ILogger _llamaLogger; private readonly ModelOptions _config; private readonly LLamaWeights _weights; + private readonly MtmdWeights _mtmdWeights; private readonly ConcurrentDictionary _contexts; /// @@ -21,11 +24,12 @@ public class LLamaModel : IDisposable /// The model parameters. /// A logger class. /// Model weights. - private LLamaModel(ModelOptions modelParams, ILogger llamaLogger, LLamaWeights weights) + private LLamaModel(ModelOptions modelParams, ILogger llamaLogger, LLamaWeights weights, MtmdWeights mtmdWeights) { _config = modelParams; _llamaLogger = llamaLogger; _weights = weights; + _mtmdWeights = mtmdWeights; _contexts = new ConcurrentDictionary(); } @@ -37,7 +41,21 @@ private LLamaModel(ModelOptions modelParams, ILogger llamaLogger, LLamaWeights w public static async Task CreateAsync(ModelOptions modelParams, ILogger llamaLogger) { var weights = await LLamaWeights.LoadFromFileAsync(modelParams); - return new LLamaModel(modelParams, llamaLogger, weights); + + MtmdWeights mtmdWeights = null; + if (!string.IsNullOrWhiteSpace(modelParams.MmprojPath)) + { + if (!File.Exists(modelParams.MmprojPath)) + throw new FileNotFoundException($"Mmproj file not found: {modelParams.MmprojPath}"); + + var mtmdParams = MtmdContextParams.Default(); + mtmdParams.UseGpu = false; + if (modelParams.Threads.HasValue) + mtmdParams.NThreads = modelParams.Threads.Value; + mtmdWeights = await MtmdWeights.LoadFromFileAsync(modelParams.MmprojPath, weights, mtmdParams); + } + + return new LLamaModel(modelParams, llamaLogger, weights, mtmdWeights); } /// @@ -50,6 +68,11 @@ public static async Task CreateAsync(ModelOptions modelParams, ILogg /// public LLamaWeights LLamaWeights => _weights; + /// + /// Gets the multimodal weights, if configured. + /// + public MtmdWeights MtmdWeights => _mtmdWeights; + /// /// Gets the context count. /// @@ -119,5 +142,6 @@ public void Dispose() context?.Dispose(); } _weights.Dispose(); + _mtmdWeights?.Dispose(); } } diff --git a/LLama.Web/Models/MemoryBrowserFile.cs b/LLama.Web/Models/MemoryBrowserFile.cs new file mode 100644 index 000000000..a79938acc --- /dev/null +++ b/LLama.Web/Models/MemoryBrowserFile.cs @@ -0,0 +1,34 @@ +using Microsoft.AspNetCore.Components.Forms; + +namespace LLama.Web.Models; + +public sealed class MemoryBrowserFile : IBrowserFile +{ + private readonly byte[] _data; + + public MemoryBrowserFile(string name, string contentType, byte[] data, DateTimeOffset? lastModified = null) + { + if (string.IsNullOrWhiteSpace(name)) + throw new ArgumentException("File name is required.", nameof(name)); + if (string.IsNullOrWhiteSpace(contentType)) + throw new ArgumentException("Content type is required.", nameof(contentType)); + _data = data ?? throw new ArgumentNullException(nameof(data)); + + Name = name; + ContentType = contentType; + LastModified = lastModified ?? DateTimeOffset.UtcNow; + } + + public string Name { get; } + public DateTimeOffset LastModified { get; } + public long Size => _data.LongLength; + public string ContentType { get; } + + public Stream OpenReadStream(long maxAllowedSize = 512L * 1024 * 1024, CancellationToken cancellationToken = default) + { + if (Size > maxAllowedSize) + throw new IOException($"File {Name} exceeds the maximum allowed size of {maxAllowedSize} bytes."); + + return new MemoryStream(_data, writable: false); + } +} diff --git a/LLama.Web/Models/ModelCapabilities.cs b/LLama.Web/Models/ModelCapabilities.cs new file mode 100644 index 000000000..86a449b98 --- /dev/null +++ b/LLama.Web/Models/ModelCapabilities.cs @@ -0,0 +1,8 @@ +namespace LLama.Web.Models; + +public class ModelCapabilities +{ + public bool SupportsText { get; set; } + public bool SupportsVision { get; set; } + public bool SupportsAudio { get; set; } +} diff --git a/LLama.Web/Models/ModelDownloadStatus.cs b/LLama.Web/Models/ModelDownloadStatus.cs new file mode 100644 index 000000000..785178f4f --- /dev/null +++ b/LLama.Web/Models/ModelDownloadStatus.cs @@ -0,0 +1,45 @@ +namespace LLama.Web.Models; + +public enum ModelAssetKind +{ + Model = 0, + Mmproj = 1 +} + +public enum ModelDownloadState +{ + Queued = 0, + Downloading = 1, + Completed = 2, + Failed = 3, + MissingUrl = 4 +} + +public class ModelAssetSnapshot +{ + public string ModelName { get; set; } + public string FileName { get; set; } + public ModelAssetKind AssetKind { get; set; } + public ModelDownloadState State { get; set; } + public long? TotalBytes { get; set; } + public long BytesReceived { get; set; } + public string Error { get; set; } +} + +public class ModelDownloadSnapshot +{ + public string ModelName { get; set; } + public bool Ready { get; set; } + public List Assets { get; set; } = new(); +} + +public class ModelDownloadProgress +{ + public string ModelName { get; set; } + public string FileName { get; set; } + public ModelAssetKind AssetKind { get; set; } + public ModelDownloadState State { get; set; } + public long? TotalBytes { get; set; } + public long BytesReceived { get; set; } + public string Error { get; set; } +} diff --git a/LLama.Web/Models/ModelSession.cs b/LLama.Web/Models/ModelSession.cs index 71e799ca5..53f76d8c7 100644 --- a/LLama.Web/Models/ModelSession.cs +++ b/LLama.Web/Models/ModelSession.cs @@ -1,4 +1,7 @@ +using LLama; using LLama.Abstractions; +using LLama.Common; +using LLama.Native; using LLama.Sampling; using LLama.Web.Common; @@ -13,6 +16,10 @@ public class ModelSession private readonly ISessionConfig _sessionConfig; private readonly ITextStreamTransform _outputTransform; private readonly InferenceOptions _defaultInferenceConfig; + private readonly ChatHistory _chatHistory = new(); + private readonly string? _systemPrompt; + private readonly string? _templateUserMarker; + private readonly object _historyLock = new(); private CancellationTokenSource _cancellationTokenSource; @@ -28,6 +35,11 @@ public ModelSession(LLamaModel model, LLamaContext context, string sessionId, IS }; _outputTransform = CreateOutputFilter(); _executor = CreateExecutor(); + _systemPrompt = _sessionConfig.Prompt; + _templateUserMarker = ResolveTemplateUserMarker(); + + if (_sessionConfig.ExecutorType != LLamaExecutorType.Stateless && !string.IsNullOrWhiteSpace(_systemPrompt)) + _chatHistory.AddMessage(AuthorRole.System, _systemPrompt); } /// @@ -55,6 +67,43 @@ public ModelSession(LLamaModel model, LLamaContext context, string sessionId, IS /// public InferenceOptions InferenceParams => _defaultInferenceConfig; + /// + /// Returns true if the executor has multimodal support enabled. + /// + public bool IsMultiModal => _executor.IsMultiModal; + + /// + /// Returns true if the multimodal model supports audio inputs. + /// + public bool SupportsAudio => _executor.ClipModel?.SupportsAudio ?? false; + + /// + /// Returns true if the multimodal model supports vision inputs. + /// + public bool SupportsVision => _executor.ClipModel?.SupportsVision ?? false; + + /// + /// Gets the MTMD clip model, if available. + /// + public MtmdWeights ClipModel => _executor.ClipModel; + + /// + /// Returns true if the session should preserve chat history. + /// + public bool StoresHistory => _sessionConfig.ExecutorType != LLamaExecutorType.Stateless; + + /// + /// Queue media embeddings for the next inference call. + /// + public void QueueEmbeds(IEnumerable embeds) + { + if (embeds is null) + return; + + foreach (var embed in embeds) + _executor.Embeds.Add(embed); + } + /// /// Initializes the prompt. /// @@ -65,17 +114,15 @@ internal async Task InitializePrompt(InferenceOptions inferenceConfig = null, Ca if (_sessionConfig.ExecutorType == LLamaExecutorType.Stateless) return; - if (string.IsNullOrEmpty(_sessionConfig.Prompt)) + if (_chatHistory.Messages.Count == 0) return; - // Run Initial prompt - var inferenceParams = ConfigureInferenceParams(inferenceConfig); - _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - await foreach (var _ in _executor.InferAsync(_sessionConfig.Prompt, inferenceParams, _cancellationTokenSource.Token)) + if (_executor is StatefulExecutorBase statefulExecutor) { - // We dont really need the response of the initial prompt, so exit on first token - break; - }; + var prompt = FormatChatHistory(_chatHistory, addAssistant: false); + if (!string.IsNullOrWhiteSpace(prompt)) + await statefulExecutor.PrefillPromptAsync(prompt, cancellationToken); + } } /// @@ -107,6 +154,36 @@ public bool IsInferCanceled() return _cancellationTokenSource.IsCancellationRequested; } + /// + /// Builds a prompt for the current chat session using the model template. + /// + /// The user content to include in the prompt. + /// The formatted prompt to send to the model. + internal string BuildPrompt(string userContent) + { + userContent ??= string.Empty; + + if (_sessionConfig.ExecutorType == LLamaExecutorType.Stateless) + return BuildStatelessPrompt(userContent); + + return BuildChatDelta(userContent); + } + + /// + /// Adds the assistant response to the chat history. + /// + /// Assistant response content. + internal void AddAssistantMessage(string assistantContent) + { + if (_sessionConfig.ExecutorType == LLamaExecutorType.Stateless) + return; + + lock (_historyLock) + { + _chatHistory.AddMessage(AuthorRole.Assistant, assistantContent ?? string.Empty); + } + } + /// /// Configures the inference parameters. /// @@ -114,7 +191,18 @@ public bool IsInferCanceled() private IInferenceParams ConfigureInferenceParams(InferenceOptions inferenceConfig) { var inferenceParams = inferenceConfig ?? _defaultInferenceConfig; - inferenceParams.AntiPrompts = _sessionConfig.GetAntiPrompts(); + var antiPrompts = _sessionConfig.GetAntiPrompts(); + if (!string.IsNullOrWhiteSpace(_templateUserMarker)) + { + if (!antiPrompts.Contains(_templateUserMarker)) + antiPrompts.Add(_templateUserMarker); + + var trimmedMarker = _templateUserMarker.Trim(); + if (!string.IsNullOrWhiteSpace(trimmedMarker) && !antiPrompts.Contains(trimmedMarker)) + antiPrompts.Add(trimmedMarker); + } + + inferenceParams.AntiPrompts = antiPrompts; return inferenceParams; } @@ -131,10 +219,87 @@ private ILLamaExecutor CreateExecutor() { return _sessionConfig.ExecutorType switch { - LLamaExecutorType.Interactive => new InteractiveExecutor(_context), - LLamaExecutorType.Instruct => new InstructExecutor(_context), + LLamaExecutorType.Interactive => _model.MtmdWeights is null + ? new InteractiveExecutor(_context) + : new InteractiveExecutor(_context, _model.MtmdWeights), + LLamaExecutorType.Instruct => _model.MtmdWeights is null + ? new InstructExecutor(_context) + : new InstructExecutor(_context, _model.MtmdWeights), LLamaExecutorType.Stateless => new StatelessExecutor(_model.LLamaWeights, _context.Params), _ => default }; } + + private string BuildChatDelta(string userContent) + { + lock (_historyLock) + { + var pastPrompt = FormatChatHistory(_chatHistory, addAssistant: false); + _chatHistory.AddMessage(AuthorRole.User, userContent); + var fullPrompt = FormatChatHistory(_chatHistory, addAssistant: true); + + if (!fullPrompt.StartsWith(pastPrompt, StringComparison.Ordinal)) + return fullPrompt; + + var delta = fullPrompt[pastPrompt.Length..]; + if (pastPrompt.Length > 0 && pastPrompt.EndsWith('\n') && (delta.Length == 0 || delta[0] != '\n')) + delta = "\n" + delta; + + return delta; + } + } + + private string BuildStatelessPrompt(string userContent) + { + var history = new ChatHistory(); + if (!string.IsNullOrWhiteSpace(_systemPrompt)) + history.AddMessage(AuthorRole.System, _systemPrompt); + history.AddMessage(AuthorRole.User, userContent); + + return FormatChatHistory(history, addAssistant: true); + } + + private string FormatChatHistory(ChatHistory history, bool addAssistant) + { + var template = new LLamaTemplate(_model.LLamaWeights.NativeHandle) + { + AddAssistant = addAssistant + }; + + foreach (var message in history.Messages) + template.Add(message.AuthorRole.ToString().ToLowerInvariant(), message.Content); + + return LLamaTemplate.Encoding.GetString(template.Apply()); + } + + private string? ResolveTemplateUserMarker() + { + const string userMarkerA = "__LLAMA_USER_A__"; + const string assistantMarkerA = "__LLAMA_ASSISTANT_A__"; + const string userMarkerB = "__LLAMA_USER_B__"; + + try + { + var template = new LLamaTemplate(_model.LLamaWeights.NativeHandle) + { + AddAssistant = false + }; + template.Add("user", userMarkerA); + template.Add("assistant", assistantMarkerA); + template.Add("user", userMarkerB); + + var rendered = LLamaTemplate.Encoding.GetString(template.Apply()); + var assistantIndex = rendered.IndexOf(assistantMarkerA, StringComparison.Ordinal); + var userIndex = rendered.IndexOf(userMarkerB, StringComparison.Ordinal); + if (assistantIndex < 0 || userIndex <= assistantIndex) + return null; + + var between = rendered.Substring(assistantIndex + assistantMarkerA.Length, userIndex - (assistantIndex + assistantMarkerA.Length)); + return string.IsNullOrWhiteSpace(between) ? null : between; + } + catch + { + return null; + } + } } diff --git a/LLama.Web/Models/PromptRequest.cs b/LLama.Web/Models/PromptRequest.cs new file mode 100644 index 000000000..cc807eea1 --- /dev/null +++ b/LLama.Web/Models/PromptRequest.cs @@ -0,0 +1,7 @@ +namespace LLama.Web.Models; + +public class PromptRequest +{ + public string Prompt { get; set; } + public List AttachmentIds { get; set; } = new(); +} diff --git a/LLama.Web/Models/StorageInfo.cs b/LLama.Web/Models/StorageInfo.cs new file mode 100644 index 000000000..60d9e36ce --- /dev/null +++ b/LLama.Web/Models/StorageInfo.cs @@ -0,0 +1,8 @@ +namespace LLama.Web.Models; + +public class StorageInfo +{ + public string ModelsPath { get; set; } + public string DownloadsPath { get; set; } + public string UploadsPath { get; set; } +} diff --git a/LLama.Web/Pages/Chat.razor b/LLama.Web/Pages/Chat.razor new file mode 100644 index 000000000..ec41f2a39 --- /dev/null +++ b/LLama.Web/Pages/Chat.razor @@ -0,0 +1,941 @@ +@page "/" +@implements IAsyncDisposable +@inject IModelDownloadService ModelDownloadService +@inject IModelSessionService ModelSessionService +@inject IAttachmentService AttachmentService +@inject IOptions OptionsAccessor +@inject IJSRuntime JS + +
+ + +
+
+
+

Chat Playground

+ Streaming, multimodal, and markdown-ready. +
+
+ @_statusText +
+
+ +
+ @foreach (var message in _messages) + { +
+
@message.Avatar
+
+
+ @if (message.Attachments.Any(attachment => attachment.Kind == AttachmentKind.Image)) + { +
+ @foreach (var attachment in message.Attachments.Where(attachment => attachment.Kind == AttachmentKind.Image)) + { + @attachment.FileName + } +
+ } + @if (message.Attachments.Any(attachment => attachment.Kind == AttachmentKind.Audio)) + { +
+ @foreach (var attachment in message.Attachments.Where(attachment => attachment.Kind == AttachmentKind.Audio)) + { +
+
@attachment.FileName
+ +
+ } +
+ } + @if (message.Attachments.Any()) + { +
+ @foreach (var attachment in message.Attachments) + { + @attachment.FileName + } +
+ } + @if (!string.IsNullOrWhiteSpace(message.Meta)) + { +
@message.Meta
+ } +
+
+ } +
+ + @if (!string.IsNullOrWhiteSpace(_inputText)) + { +
+
+
U
+
+
@_inputText
+
+
+
+ } + +
+
+
+ + @if (_modelCapabilities.SupportsAudio && _audioRecordingSupported) + { + + } + +
+
@_composerHint
+
+ @if (_pendingFiles.Count > 0) + { +
+ @foreach (var file in _pendingFiles) + { + @file.Name + } +
+ } + @if (_hasRecordedAudio) + { +
+
+
@_recordedAudioName
+ +
+
+
+
+ + +
+
+ } + + @if (!string.IsNullOrWhiteSpace(_composerError)) + { +
@_composerError
+ } +
+
+ + +
+
+
+
+
+ +@code { + private readonly List _messages = new(); + private readonly HashSet _pendingRenders = new(); + private readonly List _pendingFiles = new(); + private readonly List _downloadSnapshots = new(); + + private ElementReference _threadRef; + private bool _initialized; + private bool _sessionLoaded; + private bool _isSending; + private bool _scrollToBottom; + private string _inputText = string.Empty; + private string _statusText = "Model not loaded"; + private string _composerHint = "Select a model and start a session."; + private string _composerError = string.Empty; + private string _acceptedUploadTypes = "application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,image/*"; + private SessionConfig _sessionConfig = new(); + private InferenceOptions _inferenceOptions = new(); + private SamplingSettings _samplingSettings = new(); + private string _sessionId = Guid.NewGuid().ToString("N"); + private IReadOnlyList _models = Array.Empty(); + private StorageInfo _storageInfo; + private PeriodicTimer _downloadTimer; + private CancellationTokenSource _downloadCts; + private SidebarTab _activeTab = SidebarTab.Models; + private ModelCapabilities _modelCapabilities = new ModelCapabilities { SupportsText = true }; + private string _capabilitiesHint = "Load a model session to inspect capabilities."; + private bool _audioRecordingSupported; + private bool _isRecordingAudio; + private bool _hasRecordedAudio; + private string _recordedAudioName = string.Empty; + private string _recordedAudioPreviewUrl = string.Empty; + private MemoryBrowserFile _recordedAudioFile; + + private bool _canLoad => !_sessionLoaded && IsModelReady(_sessionConfig.Model); + + protected override Task OnInitializedAsync() + { + _models = OptionsAccessor.Value.Models ?? new List(); + _sessionConfig = new SessionConfig + { + Model = _models.FirstOrDefault()?.Name, + ExecutorType = LLamaExecutorType.Interactive, + Prompt = "Below is an instruction that describes a task. Write a response that appropriately completes the request.", + AntiPrompt = "User:,<|im_end|>,<|eot_id|>,<|endoftext|>", + OutputFilter = "User:, Assistant:,<|im_end|>,<|eot_id|>,<|endoftext|>" + }; + + _samplingSettings = SamplingSettings.CreateDefault(); + _inferenceOptions = BuildInferenceOptions(); + + _storageInfo = new StorageInfo + { + ModelsPath = ModelDownloadService.ModelsRoot, + DownloadsPath = ModelDownloadService.DownloadsRoot, + UploadsPath = AttachmentService.UploadsRoot + }; + + _downloadCts = new CancellationTokenSource(); + _downloadTimer = new PeriodicTimer(TimeSpan.FromMilliseconds(800)); + _ = PollDownloadsAsync(_downloadCts.Token); + + _acceptedUploadTypes = BuildAcceptedUploadTypes(); + return Task.CompletedTask; + } + + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (firstRender) + { + await JS.InvokeVoidAsync("llamaChat.initialize"); + _audioRecordingSupported = await JS.InvokeAsync("llamaChat.isAudioRecordingSupported"); + _initialized = true; + var storedTab = await JS.InvokeAsync("llamaChat.getSidebarTab"); + if (!string.IsNullOrWhiteSpace(storedTab) && Enum.TryParse(storedTab, out var tab)) + { + _activeTab = tab; + } + } + + if (_initialized && _pendingRenders.Count > 0) + { + var toRender = _messages.Where(message => _pendingRenders.Contains(message.Id)).ToList(); + foreach (var message in toRender) + { + await JS.InvokeVoidAsync("llamaChat.renderMarkdown", message.ContentRef, message.Content); + _pendingRenders.Remove(message.Id); + } + } + + if (_scrollToBottom) + { + await JS.InvokeVoidAsync("llamaChat.scrollToBottom", _threadRef); + _scrollToBottom = false; + } + } + + private async Task PollDownloadsAsync(CancellationToken cancellationToken) + { + while (await _downloadTimer.WaitForNextTickAsync(cancellationToken)) + { + _downloadSnapshots.Clear(); + _downloadSnapshots.AddRange(ModelDownloadService.GetSnapshots()); + _composerHint = IsModelReady(_sessionConfig.Model) + ? "Ready to chat." + : "Model is downloading..."; + await InvokeAsync(StateHasChanged); + } + } + + private async Task LoadSessionAsync() + { + if (!IsModelReady(_sessionConfig.Model)) + { + AddSystemMessage("Selected model is still downloading.", MessageKind.Error); + return; + } + + await UnloadSessionAsync(); + + try + { + _inferenceOptions = BuildInferenceOptions(); + await ModelSessionService.CreateAsync(_sessionId, _sessionConfig, _inferenceOptions); + _sessionLoaded = true; + _statusText = $"Model loaded: {_sessionConfig.Model}"; + _composerHint = "Ask about images, PDFs, or Word documents."; + await RefreshCapabilitiesAsync(); + } + catch (Exception ex) + { + AddSystemMessage($"Failed to start session: {ex.Message}", MessageKind.Error); + _sessionLoaded = false; + _statusText = "Model not loaded"; + } + } + + private async Task UnloadSessionAsync() + { + if (!_sessionLoaded) + return; + + await ModelSessionService.CloseAsync(_sessionId); + await AttachmentService.CleanupAsync(_sessionId); + _sessionLoaded = false; + _statusText = "Model not loaded"; + _modelCapabilities = new ModelCapabilities { SupportsText = true }; + _capabilitiesHint = "Load a model session to inspect capabilities."; + } + + private async Task SendAsync() + { + if (!_sessionLoaded || _isSending) + return; + + if (string.IsNullOrWhiteSpace(_inputText)) + return; + + if (HasImageAttachments() && !_modelCapabilities.SupportsVision) + { + _composerError = "This model does not support images. Remove image attachments or choose a multimodal model."; + return; + } + + if (HasAudioAttachments() && !_modelCapabilities.SupportsAudio) + { + _composerError = "This model does not support audio. Remove audio attachments or choose an audio-capable model."; + return; + } + + var prompt = _inputText.Trim(); + _inputText = string.Empty; + _isSending = true; + _composerError = string.Empty; + + var attachments = await UploadAttachmentsAsync(); + + var userMessage = new ChatMessageView(MessageKind.User, prompt) + { + Attachments = attachments + }; + AddMessage(userMessage); + + var assistantMessage = new ChatMessageView(MessageKind.Assistant, string.Empty); + AddMessage(assistantMessage); + + var request = new PromptRequest + { + Prompt = prompt, + AttachmentIds = attachments.Select(a => a.Id).ToList() + }; + + try + { + _inferenceOptions = BuildInferenceOptions(); + await foreach (var token in ModelSessionService.InferAsync(_sessionId, request, _inferenceOptions)) + { + if (token.TokenType == TokenType.Content) + { + assistantMessage.Content += token.Content; + MarkForRender(assistantMessage); + } + else if (token.TokenType == TokenType.End || token.TokenType == TokenType.Cancel) + { + assistantMessage.Meta = token.Content; + } + + _scrollToBottom = true; + await InvokeAsync(StateHasChanged); + } + } + catch (Exception ex) + { + AddSystemMessage($"Failed to invoke prompt: {ex.Message}", MessageKind.Error); + } + finally + { + _isSending = false; + } + } + + private async Task CancelAsync() + { + if (!_isSending) + return; + + await ModelSessionService.CancelAsync(_sessionId); + _isSending = false; + AddSystemMessage("Generation cancelled.", MessageKind.Info); + } + + private async Task> UploadAttachmentsAsync() + { + if (_pendingFiles.Count == 0) + return new List(); + + try + { + AddSystemMessage("Uploading attachments...", MessageKind.Info); + var result = await AttachmentService.SaveAsync(_sessionId, _pendingFiles, CancellationToken.None); + _pendingFiles.Clear(); + return result.Attachments.ToList(); + } + catch (Exception ex) + { + AddSystemMessage($"Attachment upload failed: {ex.Message}", MessageKind.Error); + return new List(); + } + } + + private void OnFilesSelected(InputFileChangeEventArgs args) + { + _pendingFiles.Clear(); + var invalidFiles = new List(); + foreach (var file in args.GetMultipleFiles()) + { + if (IsAllowedUpload(file)) + { + _pendingFiles.Add(file); + } + else + { + invalidFiles.Add(file.Name); + } + } + + if (invalidFiles.Count > 0) + { + var capabilityNote = BuildCapabilitiesHint(); + AddSystemMessage($"Unsupported files ignored: {string.Join(", ", invalidFiles)}. {capabilityNote}", MessageKind.Info); + } + _composerError = string.Empty; + } + + private void ClearAttachments() + { + _pendingFiles.Clear(); + ClearRecordedAudio(); + _composerError = string.Empty; + } + + private bool IsAllowedUpload(IBrowserFile file) + { + var extension = Path.GetExtension(file.Name).ToLowerInvariant(); + var contentType = file.ContentType?.ToLowerInvariant() ?? string.Empty; + + if (extension == ".doc") + return false; + + if (extension == ".docx" || contentType.Contains("officedocument.wordprocessingml")) + return true; + + if (extension == ".pdf" || contentType.Contains("pdf")) + return true; + + if (contentType.StartsWith("image/")) + return true; + + if (contentType.StartsWith("audio/") || extension is ".wav" or ".mp3" or ".m4a" or ".ogg" or ".flac" or ".webm") + return true; + + return false; + } + + private bool HasImageAttachments() + { + foreach (var file in _pendingFiles) + { + var contentType = file.ContentType?.ToLowerInvariant() ?? string.Empty; + var extension = Path.GetExtension(file.Name).ToLowerInvariant(); + if (contentType.StartsWith("image/") || extension is ".png" or ".jpg" or ".jpeg" or ".webp" or ".bmp" or ".gif") + return true; + } + + return false; + } + + private bool HasAudioAttachments() + { + foreach (var file in _pendingFiles) + { + var contentType = file.ContentType?.ToLowerInvariant() ?? string.Empty; + var extension = Path.GetExtension(file.Name).ToLowerInvariant(); + if (contentType.StartsWith("audio/") || extension is ".wav" or ".mp3" or ".m4a" or ".ogg" or ".flac" or ".webm") + return true; + } + + return false; + } + + private async Task RefreshCapabilitiesAsync() + { + if (!_sessionLoaded) + return; + + _modelCapabilities = await ModelSessionService.GetCapabilitiesAsync(_sessionId); + _capabilitiesHint = _modelCapabilities.SupportsVision || _modelCapabilities.SupportsAudio + ? "Multimodal inputs enabled for this model." + : "Text-only model loaded."; + _acceptedUploadTypes = BuildAcceptedUploadTypes(); + } + + private string BuildAcceptedUploadTypes() + { + var types = new List + { + "application/pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + }; + + if (_modelCapabilities.SupportsVision) + types.Add("image/*"); + + types.Add("audio/*"); + + return string.Join(",", types); + } + + private string BuildCapabilitiesHint() + { + var parts = new List { "Use PDF or DOCX" }; + + if (_modelCapabilities.SupportsVision) + parts.Add("images"); + + if (_modelCapabilities.SupportsAudio) + parts.Add("audio"); + + return $"{string.Join(", ", parts)}."; + } + + private async Task HandleKeyPress(KeyboardEventArgs args) + { + if (args.Key == "Enter" && !args.ShiftKey) + { + await SendAsync(); + } + } + + private async Task ToggleAudioRecordingAsync() + { + if (_isRecordingAudio) + { + await StopAudioRecordingAsync(); + return; + } + + if (!_modelCapabilities.SupportsAudio || !_audioRecordingSupported) + return; + + var result = await JS.InvokeAsync("llamaChat.startAudioRecording"); + if (!result.Started) + { + AddSystemMessage(result.Error ?? "Unable to start audio recording.", MessageKind.Error); + return; + } + + _isRecordingAudio = true; + _composerError = string.Empty; + } + + private async Task StopAudioRecordingAsync() + { + if (!_isRecordingAudio) + return; + + var result = await JS.InvokeAsync("llamaChat.stopAudioRecording"); + _isRecordingAudio = false; + + if (result == null || string.IsNullOrWhiteSpace(result.Base64)) + { + AddSystemMessage("No audio captured.", MessageKind.Info); + return; + } + + var bytes = Convert.FromBase64String(result.Base64); + var mimeType = string.IsNullOrWhiteSpace(result.MimeType) ? "audio/webm" : result.MimeType; + var extension = GetAudioExtension(mimeType); + var fileName = $"recording-{DateTimeOffset.Now:yyyyMMdd-HHmmss}{extension}"; + + _recordedAudioFile = new MemoryBrowserFile(fileName, mimeType, bytes, DateTimeOffset.Now); + _recordedAudioName = fileName; + _recordedAudioPreviewUrl = $"data:{mimeType};base64,{result.Base64}"; + _hasRecordedAudio = true; + _composerError = string.Empty; + } + + private void AcceptRecordedAudio() + { + if (_recordedAudioFile == null) + return; + + _pendingFiles.Add(_recordedAudioFile); + ClearRecordedAudio(); + _composerError = string.Empty; + } + + private void DiscardRecordedAudio() + { + ClearRecordedAudio(); + } + + private void ClearRecordedAudio() + { + _recordedAudioFile = null; + _recordedAudioName = string.Empty; + _recordedAudioPreviewUrl = string.Empty; + _hasRecordedAudio = false; + } + + private static string GetAudioExtension(string contentType) + { + var normalized = contentType.ToLowerInvariant(); + if (normalized.Contains("wav")) + return ".wav"; + if (normalized.Contains("mpeg") || normalized.Contains("mp3")) + return ".mp3"; + if (normalized.Contains("ogg")) + return ".ogg"; + if (normalized.Contains("mp4") || normalized.Contains("m4a")) + return ".m4a"; + if (normalized.Contains("webm")) + return ".webm"; + + return ".audio"; + } + + private bool IsModelReady(string modelName) + { + if (string.IsNullOrWhiteSpace(modelName)) + return false; + + return ModelDownloadService.IsModelReady(modelName); + } + + private void AddSystemMessage(string message, MessageKind kind) + { + AddMessage(new ChatMessageView(kind, message)); + } + + private void AddMessage(ChatMessageView message) + { + _messages.Add(message); + MarkForRender(message); + _scrollToBottom = true; + } + + private void MarkForRender(ChatMessageView message) + { + _pendingRenders.Add(message.Id); + } + + private static string FormatBytes(long? value) + { + if (!value.HasValue) + return "-"; + + var bytes = value.Value; + if (bytes <= 0) + return "0 B"; + + var sizes = new[] { "B", "KB", "MB", "GB", "TB" }; + var order = (int)Math.Floor(Math.Log(bytes, 1024)); + var adjusted = bytes / Math.Pow(1024, order); + return $"{adjusted:0.0} {sizes[order]}"; + } + + private InferenceOptions BuildInferenceOptions() + { + return new InferenceOptions + { + MaxTokens = _samplingSettings.MaxTokens, + DecodeSpecialTokens = true, + SamplingPipeline = new DefaultSamplingPipeline + { + Temperature = _samplingSettings.Temperature, + TopP = _samplingSettings.TopP, + TopK = _samplingSettings.TopK, + RepeatPenalty = _samplingSettings.RepeatPenalty, + PresencePenalty = _samplingSettings.PresencePenalty, + FrequencyPenalty = _samplingSettings.FrequencyPenalty, + PenaltyCount = _samplingSettings.PenaltyCount, + TypicalP = _samplingSettings.TypicalP, + MinP = _samplingSettings.MinP, + PreventEOS = _samplingSettings.PreventEos, + PenalizeNewline = _samplingSettings.PenalizeNewline + } + }; + } + + public async ValueTask DisposeAsync() + { + _downloadCts?.Cancel(); + _downloadTimer?.Dispose(); + await ModelSessionService.CloseAsync(_sessionId); + await AttachmentService.CleanupAsync(_sessionId); + } + + private sealed class ChatMessageView + { + public ChatMessageView(MessageKind kind, string content) + { + Id = Guid.NewGuid().ToString("N"); + Kind = kind; + Content = content; + Attachments = new List(); + } + + public string Id { get; } + public MessageKind Kind { get; } + public string Content { get; set; } + public string Meta { get; set; } + public ElementReference ContentRef { get; set; } + public List Attachments { get; set; } + + public string RoleClass => Kind switch + { + MessageKind.User => "user", + MessageKind.Assistant => "assistant", + MessageKind.Info => "info", + MessageKind.Error => "error", + _ => "assistant" + }; + + public string Avatar => Kind switch + { + MessageKind.User => "U", + MessageKind.Assistant => "AI", + MessageKind.Info => "i", + MessageKind.Error => "!", + _ => "•" + }; + } + + private sealed class SamplingSettings + { + public int MaxTokens { get; set; } + public float Temperature { get; set; } + public float TopP { get; set; } + public int TopK { get; set; } + public float RepeatPenalty { get; set; } + public float PresencePenalty { get; set; } + public float FrequencyPenalty { get; set; } + public int PenaltyCount { get; set; } + public float TypicalP { get; set; } + public float MinP { get; set; } + public bool PreventEos { get; set; } + public bool PenalizeNewline { get; set; } + + public static SamplingSettings CreateDefault() + { + return new SamplingSettings + { + MaxTokens = 512, + Temperature = 0.7f, + TopP = 0.9f, + TopK = 40, + RepeatPenalty = 1.0f, + PresencePenalty = 0.0f, + FrequencyPenalty = 0.0f, + PenaltyCount = 64, + TypicalP = 1.0f, + MinP = 0.1f, + PreventEos = false, + PenalizeNewline = false + }; + } + } + + private enum MessageKind + { + User, + Assistant, + Info, + Error + } + + private enum SidebarTab + { + Models, + Settings + } + + private sealed class AudioRecordingStartResult + { + public bool Started { get; set; } + public string Error { get; set; } + public string MimeType { get; set; } + } + + private sealed class AudioRecordingStopResult + { + public string Base64 { get; set; } + public string MimeType { get; set; } + public long Size { get; set; } + } + + private void SetTab(SidebarTab tab) + { + _activeTab = tab; + _ = JS.InvokeVoidAsync("llamaChat.setSidebarTab", tab.ToString()); + } + + private string GetTabClass(SidebarTab tab) + { + return _activeTab == tab ? "tab-button active" : "tab-button"; + } + + private string GetCapabilityClass(bool enabled) + { + return enabled ? "capability-pill enabled" : "capability-pill disabled"; + } +} diff --git a/LLama.Web/Pages/Index.cshtml b/LLama.Web/Pages/Index.cshtml deleted file mode 100644 index 52f87b62c..000000000 --- a/LLama.Web/Pages/Index.cshtml +++ /dev/null @@ -1,110 +0,0 @@ -@page -@using LLama.Web.Common; - -@model IndexModel -@{ - ViewData["Title"] = "Home"; -} - -@Html.AntiForgeryToken() -
- - - -
-
-
- -
-
-
-
- -
- - -
- -
-
- -@{ - await Html.RenderPartialAsync("_ChatTemplates"); -} - -@section Scripts { - - -} \ No newline at end of file diff --git a/LLama.Web/Pages/Index.cshtml.cs b/LLama.Web/Pages/Index.cshtml.cs deleted file mode 100644 index b2c65aa90..000000000 --- a/LLama.Web/Pages/Index.cshtml.cs +++ /dev/null @@ -1,45 +0,0 @@ -using LLama.Sampling; -using LLama.Web.Common; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Mvc.RazorPages; -using Microsoft.Extensions.Options; - -namespace LLama.Web.Pages -{ - public class IndexModel : PageModel - { - private readonly ILogger _logger; - - public IndexModel(ILogger logger, IOptions options) - { - _logger = logger; - Options = options.Value; - } - - public LLamaOptions Options { get; set; } - - [BindProperty] - public ISessionConfig SessionConfig { get; set; } - - [BindProperty] - public InferenceOptions InferenceOptions { get; set; } - - public void OnGet() - { - SessionConfig = new SessionConfig - { - Prompt = "Below is an instruction that describes a task. Write a response that appropriately completes the request.", - AntiPrompt = "User:", - OutputFilter = "User:, Assistant: " - }; - - InferenceOptions = new InferenceOptions - { - SamplingPipeline = new DefaultSamplingPipeline - { - Temperature = 0.8f - }, - }; - } - } -} \ No newline at end of file diff --git a/LLama.Web/Pages/Shared/_ChatTemplates.cshtml b/LLama.Web/Pages/Shared/_ChatTemplates.cshtml index 843c58607..f70b7230b 100644 --- a/LLama.Web/Pages/Shared/_ChatTemplates.cshtml +++ b/LLama.Web/Pages/Shared/_ChatTemplates.cshtml @@ -17,7 +17,13 @@
{{date}}
-
{{text}}
+
{{text}}
+ {{#attachments}} +
+ + {{fileName}} +
+ {{/attachments}} @@ -34,9 +40,9 @@
-
+
- \ No newline at end of file + diff --git a/LLama.Web/Pages/Shared/_Layout.cshtml b/LLama.Web/Pages/Shared/_Layout.cshtml index bba9c6831..80585255f 100644 --- a/LLama.Web/Pages/Shared/_Layout.cshtml +++ b/LLama.Web/Pages/Shared/_Layout.cshtml @@ -5,83 +5,33 @@ @ViewData["Title"] - LLamaSharp Web + -
- -
- -
+
@RenderBody()
- - - - - + + + + + + + + + + + + + @await RenderSectionAsync("Scripts", required: false) - \ No newline at end of file + diff --git a/LLama.Web/Pages/Shared/_Parameters.cshtml b/LLama.Web/Pages/Shared/_Parameters.cshtml index 009f1fe83..780313f96 100644 --- a/LLama.Web/Pages/Shared/_Parameters.cshtml +++ b/LLama.Web/Pages/Shared/_Parameters.cshtml @@ -1,6 +1,10 @@ @page -@using LLama.Common; -@model LLama.Abstractions.IInferenceParams +@using LLama.Sampling; +@model LLama.Web.Common.InferenceOptions + +@{ + var pipeline = Model?.SamplingPipeline as DefaultSamplingPipeline ?? new DefaultSamplingPipeline(); +}
@@ -20,12 +24,11 @@
-@* todo: update for new sampling system
TopK
- @Html.TextBoxFor(m => m.TopK, new { @type = "range", @class = "slider", min = "-1", max = "100", step = "1" }) + @Html.TextBox("SamplingPipeline.TopK", pipeline.TopK, new { @type = "range", @class = "slider", min = "1", max = "100", step = "1" })
@@ -33,7 +36,7 @@
TopP
- @Html.TextBoxFor(m => m.TopP, new { @type = "range", @class = "slider", min = "0.0", max = "1.0", step = "0.01" }) + @Html.TextBox("SamplingPipeline.TopP", pipeline.TopP, new { @type = "range", @class = "slider", min = "0.0", max = "1.0", step = "0.01" })
@@ -43,7 +46,7 @@
TypicalP
- @Html.TextBoxFor(m => m.TypicalP, new { @type = "range", @class = "slider", min = "0.0", max = "1.0", step = "0.01" }) + @Html.TextBox("SamplingPipeline.TypicalP", pipeline.TypicalP, new { @type = "range", @class = "slider", min = "0.0", max = "1.0", step = "0.01" })
@@ -51,7 +54,7 @@
Temperature
- @Html.TextBoxFor(m => m.Temperature, new { @type = "range", @class = "slider", min = "0.0", max = "1.5", step = "0.01" }) + @Html.TextBox("SamplingPipeline.Temperature", pipeline.Temperature, new { @type = "range", @class = "slider", min = "0.0", max = "1.5", step = "0.01" })
@@ -61,15 +64,15 @@
RepeatPenalty
- @Html.TextBoxFor(m => m.RepeatPenalty, new { @type = "range", @class = "slider", min = "0.0", max = "2.0", step = "0.01" }) + @Html.TextBox("SamplingPipeline.RepeatPenalty", pipeline.RepeatPenalty, new { @type = "range", @class = "slider", min = "0.0", max = "2.0", step = "0.01" })
- RepeatLastTokensCount + PenaltyCount
- @Html.TextBoxFor(m => m.RepeatLastTokensCount, new { @type = "range", @class = "slider", min = "0", max = "2048", step = "1" }) + @Html.TextBox("SamplingPipeline.PenaltyCount", pipeline.PenaltyCount, new { @type = "range", @class = "slider", min = "0", max = "2048", step = "1" })
@@ -79,7 +82,7 @@
FrequencyPenalty
- @Html.TextBoxFor(m => m.FrequencyPenalty, new { @type = "range", @class = "slider", min = "0.0", max = "1.0", step = "0.01" }) + @Html.TextBox("SamplingPipeline.FrequencyPenalty", pipeline.FrequencyPenalty, new { @type = "range", @class = "slider", min = "-2.0", max = "2.0", step = "0.01" })
@@ -87,7 +90,7 @@
PresencePenalty
- @Html.TextBoxFor(m => m.PresencePenalty, new { @type = "range", @class = "slider", min = "0.0", max = "1.0", step = "0.01" }) + @Html.TextBox("SamplingPipeline.PresencePenalty", pipeline.PresencePenalty, new { @type = "range", @class = "slider", min = "-2.0", max = "2.0", step = "0.01" })
@@ -95,45 +98,33 @@
- TfsZ + MinP
- @Html.TextBoxFor(m => m.TfsZ, new { @type = "range", @class = "slider", min = "0.0", max = "1.0", step = "0.01" }) + @Html.TextBox("SamplingPipeline.MinP", pipeline.MinP, new { @type = "range", @class = "slider", min = "0.0", max = "1.0", step = "0.01" })
- - + MinKeep
- - + @Html.TextBox("SamplingPipeline.MinKeep", pipeline.MinKeep, new { @type = "range", @class = "slider", min = "1", max = "64", step = "1" }) +
- -
-
- @Html.DropDownListFor(m => m.Mirostat, Html.GetEnumSelectList(), new { @class = "form-control form-select" }) - @Html.LabelFor(m => m.Mirostat) -
-
-
- MirostatTau -
- @Html.TextBoxFor(m => m.MirostatTau, new { @type = "range", @class = "slider", min = "0.0", max = "10.0", step = "0.01" }) - + PreventEOS +
+ @Html.CheckBox("SamplingPipeline.PreventEOS", pipeline.PreventEOS, new { @class = "form-check-input" })
- MirostatEta -
- @Html.TextBoxFor(m => m.MirostatEta, new { @type = "range", @class = "slider", min = "0.0", max = "1.0", step = "0.01" }) - + PenalizeNewline +
+ @Html.CheckBox("SamplingPipeline.PenalizeNewline", pipeline.PenalizeNewline, new { @class = "form-check-input" })
- -*@ \ No newline at end of file diff --git a/LLama.Web/Pages/_Host.cshtml b/LLama.Web/Pages/_Host.cshtml new file mode 100644 index 000000000..906a6c180 --- /dev/null +++ b/LLama.Web/Pages/_Host.cshtml @@ -0,0 +1,8 @@ +@page "/" +@namespace LLama.Web.Pages +@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers +@{ + Layout = "_Layout"; +} + + diff --git a/LLama.Web/Program.cs b/LLama.Web/Program.cs index 7ab301507..3cd4cb4d7 100644 --- a/LLama.Web/Program.cs +++ b/LLama.Web/Program.cs @@ -6,6 +6,8 @@ // Add services to the container. var mvcBuilder = builder.Services.AddRazorPages(); +builder.Services.AddServerSideBlazor(); +builder.Services.AddControllers(); if (builder.Environment.IsDevelopment()) { @@ -16,18 +18,25 @@ builder.Services.AddSignalR(); builder.Logging.ClearProviders(); builder.Services.AddLogging((loggingBuilder) => loggingBuilder.SetMinimumLevel(LogLevel.Trace).AddConsole()); +builder.Services.AddHttpClient(nameof(ModelDownloadService), client => client.Timeout = Timeout.InfiniteTimeSpan); -// Load InteractiveOptions +// Load interactive options. builder.Services.AddOptions() .BindConfiguration(nameof(LLamaOptions)); -// Services DI +// Register DI services. builder.Services.AddHostedService(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); var app = builder.Build(); +app.Logger.LogInformation("Model storage path: {Path}", app.Services.GetRequiredService().ModelsRoot); +app.Logger.LogInformation("Download staging path: {Path}", app.Services.GetRequiredService().DownloadsRoot); +app.Logger.LogInformation("Uploads path: {Path}", app.Services.GetRequiredService().UploadsRoot); + // Configure the HTTP request pipeline. if (!app.Environment.IsDevelopment()) { @@ -44,6 +53,9 @@ app.UseAuthorization(); app.MapRazorPages(); +app.MapControllers(); +app.MapBlazorHub(); +app.MapFallbackToPage("/_Host"); app.MapHub(nameof(SessionConnectionHub)); diff --git a/LLama.Web/README.md b/LLama.Web/README.md index e14562599..101b7fd35 100644 --- a/LLama.Web/README.md +++ b/LLama.Web/README.md @@ -9,6 +9,17 @@ Models, Prompts and Inference parameters can be added to `appsettings.json`. If you would like to add your own local model files then it's best to create an `appSettings.Local.json` file and add them there. The `appSettings.Local.json` file will be ignored by Git. +Local assets downloaded at runtime (models in your AppData `LLama.Web/Models`, uploaded files in your AppData `LLama.Web/Uploads`, and offline web libs in `LLama.Web/wwwroot/lib`) are intentionally git-ignored. + +To restore offline web assets (markdown/diagram libraries), install LibMan if needed (`dotnet tool install -g Microsoft.Web.LibraryManager.Cli`), ensure the dotnet tools folder is on your PATH (typically `~/.dotnet/tools`), and run LibMan from the `LLama.Web` folder: +```sh +libman restore +``` + +Attachments uploaded through the web UI are stored under your AppData `LLama.Web/Uploads/` and are cleaned up when the SignalR connection closes. + +Model downloads are staged under your AppData `LLama.Web/Downloads` and finalized in AppData `LLama.Web/Models`. + **Models** You can add multiple models to the options for quick selection in the UI, options are based on ModelParams so its fully configurable. diff --git a/LLama.Web/Services/AttachmentService.cs b/LLama.Web/Services/AttachmentService.cs new file mode 100644 index 000000000..4990bf845 --- /dev/null +++ b/LLama.Web/Services/AttachmentService.cs @@ -0,0 +1,359 @@ +using System.Collections.Concurrent; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Xml.Linq; +using LLama.Web.Models; +using Microsoft.AspNetCore.Components.Forms; +using UglyToad.PdfPig; + +namespace LLama.Web.Services; + +public class AttachmentService : IAttachmentService +{ + private const int MaxExtractedCharacters = 12000; + private const long MaxUploadSize = 512L * 1024 * 1024; + private readonly IWebHostEnvironment _environment; + private readonly string _uploadsRoot; + private readonly ConcurrentDictionary> _attachments = new(); + + public AttachmentService(IWebHostEnvironment environment) + { + _environment = environment; + var appDataRoot = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData); + _uploadsRoot = Path.Combine(appDataRoot, "LLama.Web", "Uploads"); + Directory.CreateDirectory(_uploadsRoot); + } + + public string UploadsRoot => _uploadsRoot; + + public async Task SaveAsync(string connectionId, IEnumerable files, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(connectionId)) + throw new ArgumentException("Connection id is required.", nameof(connectionId)); + + ValidateUploads(files); + + var result = new AttachmentUploadResult(); + var storage = _attachments.GetOrAdd(connectionId, _ => new ConcurrentDictionary()); + var root = Path.Combine(_uploadsRoot, connectionId); + Directory.CreateDirectory(root); + + foreach (var file in files) + { + if (file == null || file.Length == 0) + continue; + + var id = Guid.NewGuid().ToString("N"); + var safeName = Path.GetFileName(file.FileName); + var filePath = Path.Combine(root, $"{id}-{safeName}"); + + await using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, useAsync: true)) + { + await file.CopyToAsync(stream, cancellationToken); + } + + var info = new AttachmentInfo + { + Id = id, + ConnectionId = connectionId, + FileName = safeName, + FilePath = filePath, + ContentType = file.ContentType, + SizeBytes = file.Length, + Kind = GetKind(file) + }; + + if (info.Kind == AttachmentKind.Pdf) + ExtractPdfText(info); + if (info.Kind == AttachmentKind.Word) + ExtractWordText(info); + + storage[id] = info; + result.Attachments.Add(info); + } + + return result; + } + + public async Task SaveAsync(string connectionId, IEnumerable files, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(connectionId)) + throw new ArgumentException("Connection id is required.", nameof(connectionId)); + + ValidateUploads(files); + + var result = new AttachmentUploadResult(); + var storage = _attachments.GetOrAdd(connectionId, _ => new ConcurrentDictionary()); + var root = Path.Combine(_uploadsRoot, connectionId); + Directory.CreateDirectory(root); + + foreach (var file in files) + { + if (file == null || file.Size == 0) + continue; + + var id = Guid.NewGuid().ToString("N"); + var safeName = Path.GetFileName(file.Name); + var filePath = Path.Combine(root, $"{id}-{safeName}"); + + await using (var input = file.OpenReadStream(maxAllowedSize: MaxUploadSize, cancellationToken)) + await using (var output = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, useAsync: true)) + { + await input.CopyToAsync(output, cancellationToken); + } + + var info = new AttachmentInfo + { + Id = id, + ConnectionId = connectionId, + FileName = safeName, + FilePath = filePath, + ContentType = file.ContentType, + SizeBytes = file.Size, + Kind = GetKind(file) + }; + + if (info.Kind == AttachmentKind.Pdf) + ExtractPdfText(info); + if (info.Kind == AttachmentKind.Word) + ExtractWordText(info); + + storage[id] = info; + result.Attachments.Add(info); + } + + return result; + } + + public IReadOnlyList GetAttachments(string connectionId, IEnumerable ids) + { + if (string.IsNullOrWhiteSpace(connectionId) || ids is null) + return Array.Empty(); + + if (!_attachments.TryGetValue(connectionId, out var storage)) + return Array.Empty(); + + var list = new List(); + foreach (var id in ids) + { + if (storage.TryGetValue(id, out var info)) + list.Add(info); + } + + return list; + } + + public AttachmentInfo GetAttachment(string connectionId, string id) + { + if (string.IsNullOrWhiteSpace(connectionId) || string.IsNullOrWhiteSpace(id)) + return null; + + if (!_attachments.TryGetValue(connectionId, out var storage)) + return null; + + return storage.TryGetValue(id, out var info) ? info : null; + } + + public Task CleanupAsync(string connectionId) + { + if (string.IsNullOrWhiteSpace(connectionId)) + return Task.CompletedTask; + + if (_attachments.TryRemove(connectionId, out _)) + { + var root = Path.Combine(_uploadsRoot, connectionId); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; + } + + private static AttachmentKind GetKind(IFormFile file) + { + var contentType = file.ContentType?.ToLowerInvariant() ?? string.Empty; + var extension = Path.GetExtension(file.FileName).ToLowerInvariant(); + + if (contentType.Contains("pdf") || extension == ".pdf") + return AttachmentKind.Pdf; + if (IsWordDocument(contentType, extension)) + return AttachmentKind.Word; + if (contentType.StartsWith("image/")) + return AttachmentKind.Image; + if (contentType.StartsWith("audio/") || extension is ".wav" or ".mp3" or ".m4a" or ".ogg" or ".flac" or ".webm") + return AttachmentKind.Audio; + + return AttachmentKind.Unknown; + } + + private static AttachmentKind GetKind(IBrowserFile file) + { + var contentType = file.ContentType?.ToLowerInvariant() ?? string.Empty; + var extension = Path.GetExtension(file.Name).ToLowerInvariant(); + + if (contentType.Contains("pdf") || extension == ".pdf") + return AttachmentKind.Pdf; + if (IsWordDocument(contentType, extension)) + return AttachmentKind.Word; + if (contentType.StartsWith("image/")) + return AttachmentKind.Image; + if (contentType.StartsWith("audio/") || extension is ".wav" or ".mp3" or ".m4a" or ".ogg" or ".flac" or ".webm") + return AttachmentKind.Audio; + + return AttachmentKind.Unknown; + } + + private static bool IsWordDocument(string contentType, string extension) + { + if (extension == ".docx") + return true; + + if (string.IsNullOrWhiteSpace(contentType)) + return false; + + return contentType.Contains("word") || contentType.Contains("officedocument.wordprocessingml"); + } + + private static bool IsAllowedUpload(string contentType, string extension) + { + if (extension == ".doc") + return false; + + if (extension == ".docx" || contentType.Contains("officedocument.wordprocessingml") || contentType.Contains("word")) + return true; + + if (extension == ".pdf" || contentType.Contains("pdf")) + return true; + + if (contentType.StartsWith("image/")) + return true; + + if (contentType.StartsWith("audio/")) + return true; + + if (extension is ".wav" or ".mp3" or ".m4a" or ".ogg" or ".flac" or ".webm") + return true; + + return false; + } + + private static void ValidateUploads(IEnumerable files) + { + var invalid = files + .Where(file => file != null) + .Where(file => !IsAllowedUpload(file.ContentType?.ToLowerInvariant() ?? string.Empty, Path.GetExtension(file.FileName).ToLowerInvariant())) + .Select(file => file.FileName) + .ToList(); + + if (invalid.Count == 0) + return; + + throw new InvalidOperationException($"Unsupported files: {string.Join(", ", invalid)}. Use PDF, DOCX, or images."); + } + + private static void ValidateUploads(IEnumerable files) + { + var invalid = files + .Where(file => file != null) + .Where(file => !IsAllowedUpload(file.ContentType?.ToLowerInvariant() ?? string.Empty, Path.GetExtension(file.Name).ToLowerInvariant())) + .Select(file => file.Name) + .ToList(); + + if (invalid.Count == 0) + return; + + throw new InvalidOperationException($"Unsupported files: {string.Join(", ", invalid)}. Use PDF, DOCX, or images."); + } + + private static void ExtractPdfText(AttachmentInfo info) + { + if (!File.Exists(info.FilePath)) + return; + + var builder = new StringBuilder(); + try + { + using var document = PdfDocument.Open(info.FilePath); + foreach (var page in document.GetPages()) + { + if (builder.Length >= MaxExtractedCharacters) + break; + + builder.AppendLine(page.Text); + } + + var text = builder.ToString(); + if (text.Length > MaxExtractedCharacters) + { + info.ExtractedText = text.Substring(0, MaxExtractedCharacters); + info.ExtractedTextTruncated = true; + } + else + { + info.ExtractedText = text; + info.ExtractedTextTruncated = false; + } + } + catch + { + info.ExtractedText = string.Empty; + info.ExtractedTextTruncated = false; + } + } + + private static void ExtractWordText(AttachmentInfo info) + { + if (!File.Exists(info.FilePath)) + return; + + var extension = Path.GetExtension(info.FilePath).ToLowerInvariant(); + if (extension != ".docx") + { + info.ExtractedText = string.Empty; + info.ExtractedTextTruncated = false; + return; + } + + try + { + using var archive = ZipFile.OpenRead(info.FilePath); + var entry = archive.GetEntry("word/document.xml"); + if (entry == null) + return; + + using var stream = entry.Open(); + var document = XDocument.Load(stream); + var textNodes = document.Descendants() + .Where(node => node.Name.LocalName == "t") + .Select(node => node.Value); + + var builder = new StringBuilder(); + foreach (var text in textNodes) + { + if (builder.Length >= MaxExtractedCharacters) + break; + + builder.Append(text); + builder.AppendLine(); + } + + var resultText = builder.ToString(); + if (resultText.Length > MaxExtractedCharacters) + { + info.ExtractedText = resultText.Substring(0, MaxExtractedCharacters); + info.ExtractedTextTruncated = true; + } + else + { + info.ExtractedText = resultText; + info.ExtractedTextTruncated = false; + } + } + catch + { + info.ExtractedText = string.Empty; + info.ExtractedTextTruncated = false; + } + } +} diff --git a/LLama.Web/Services/IAttachmentService.cs b/LLama.Web/Services/IAttachmentService.cs new file mode 100644 index 000000000..31a2789da --- /dev/null +++ b/LLama.Web/Services/IAttachmentService.cs @@ -0,0 +1,14 @@ +using LLama.Web.Models; +using Microsoft.AspNetCore.Components.Forms; + +namespace LLama.Web.Services; + +public interface IAttachmentService +{ + string UploadsRoot { get; } + Task SaveAsync(string connectionId, IEnumerable files, CancellationToken cancellationToken); + Task SaveAsync(string connectionId, IEnumerable files, CancellationToken cancellationToken); + IReadOnlyList GetAttachments(string connectionId, IEnumerable ids); + AttachmentInfo GetAttachment(string connectionId, string id); + Task CleanupAsync(string connectionId); +} diff --git a/LLama.Web/Services/IModelDownloadService.cs b/LLama.Web/Services/IModelDownloadService.cs new file mode 100644 index 000000000..904ac13ba --- /dev/null +++ b/LLama.Web/Services/IModelDownloadService.cs @@ -0,0 +1,13 @@ +using LLama.Web.Models; + +namespace LLama.Web.Services; + +public interface IModelDownloadService +{ + string ModelsRoot { get; } + string DownloadsRoot { get; } + void StartDownloads(CancellationToken cancellationToken); + Task WaitForDownloadsAsync(CancellationToken cancellationToken); + IReadOnlyList GetSnapshots(); + bool IsModelReady(string modelName); +} diff --git a/LLama.Web/Services/IModelService.cs b/LLama.Web/Services/IModelService.cs index 0500a0b29..57796417b 100644 --- a/LLama.Web/Services/IModelService.cs +++ b/LLama.Web/Services/IModelService.cs @@ -4,7 +4,7 @@ namespace LLama.Web.Services; /// -/// Service for managing language Models +/// Service for managing language models. /// public interface IModelService { @@ -37,7 +37,7 @@ public interface IModelService Task UnloadModels(); /// - /// Gets a context with the specified identifier + /// Gets a context with the specified identifier. /// /// Name of the model. /// The context identifier. @@ -58,11 +58,11 @@ public interface IModelService Task CreateContext(string modelName, string contextName); /// - /// Gets the or create model and context. - /// This will load a model from disk if not already loaded, and also create the context + /// Gets or creates a model and context. + /// This loads a model from disk if it is not already loaded and creates the context. /// /// Name of the model. /// The context identifier. - /// Both loaded Model and Context + /// The loaded model and context. Task<(LLamaModel, LLamaContext)> GetOrCreateModelAndContext(string modelName, string contextName); -} \ No newline at end of file +} diff --git a/LLama.Web/Services/IModelSessionService.cs b/LLama.Web/Services/IModelSessionService.cs index 974aebd2f..392403379 100644 --- a/LLama.Web/Services/IModelSessionService.cs +++ b/LLama.Web/Services/IModelSessionService.cs @@ -6,67 +6,67 @@ namespace LLama.Web.Services; public interface IModelSessionService { /// - /// Gets the ModelSession with the specified Id. + /// Gets the ModelSession with the specified ID. /// /// The session identifier. - /// The ModelSession if exists, otherwise null + /// The ModelSession if it exists; otherwise, null. Task GetAsync(string sessionId); /// - /// Gets all ModelSessions + /// Gets all model sessions. /// - /// A collection oa all Model instances + /// A collection of all model sessions. Task> GetAllAsync(); /// - /// Creates a new ModelSession + /// Creates a new model session. /// /// The session identifier. /// The session configuration. - /// The default inference configuration, will be used for all inference where no infer configuration is supplied. + /// The default inference configuration, used when no inference configuration is supplied. /// The cancellation token. /// /// - /// Session with id {sessionId} already exists + /// Session with ID {sessionId} already exists /// or /// Failed to create model session /// Task CreateAsync(string sessionId, ISessionConfig sessionConfig, InferenceOptions inferenceOptions = null, CancellationToken cancellationToken = default); /// - /// Closes the session + /// Closes the session. /// /// The session identifier. /// Task CloseAsync(string sessionId); /// - /// Runs inference on the current ModelSession + /// Runs inference on the current model session. /// /// The session identifier. - /// The prompt. - /// The inference configuration, if null session default is used + /// The prompt request. + /// The inference configuration; if null, the session default is used. /// The cancellation token. /// Inference is already running for this session - IAsyncEnumerable InferAsync(string sessionId, string prompt, InferenceOptions inferenceConfig = null, CancellationToken cancellationToken = default); + IAsyncEnumerable InferAsync(string sessionId, PromptRequest request, InferenceOptions inferenceConfig = null, CancellationToken cancellationToken = default); /// - /// Runs inference on the current ModelSession + /// Runs inference on the current model session. /// /// The session identifier. /// The prompt. - /// The inference configuration, if null session default is used + /// The inference configuration; if null, the session default is used. /// The cancellation token. /// Streaming async result of /// Inference is already running for this session IAsyncEnumerable InferTextAsync(string sessionId, string prompt, InferenceOptions inferenceOptions = null, CancellationToken cancellationToken = default); /// - /// Queues inference on the current ModelSession + /// Queues inference on the current model session. /// /// The session identifier. /// The prompt. - /// The inference configuration, if null session default is used + /// The inference configuration; if null, the session default is used. /// The cancellation token. /// Completed inference result as string /// Inference is already running for this session @@ -78,4 +78,11 @@ public interface IModelSessionService /// The session identifier. /// Task CancelAsync(string sessionId); + + /// + /// Gets the model capabilities for the specified session. + /// + /// The session identifier. + /// The model capabilities. + Task GetCapabilitiesAsync(string sessionId); } diff --git a/LLama.Web/Services/ModelDownloadService.cs b/LLama.Web/Services/ModelDownloadService.cs new file mode 100644 index 000000000..df2c3bc7a --- /dev/null +++ b/LLama.Web/Services/ModelDownloadService.cs @@ -0,0 +1,301 @@ +using System.Collections.Concurrent; +using LLama.Web.Common; +using LLama.Web.Hubs; +using LLama.Web.Models; +using Microsoft.AspNetCore.SignalR; +using Microsoft.Extensions.Options; + +namespace LLama.Web.Services; + +public class ModelDownloadService : IModelDownloadService +{ + private readonly IWebHostEnvironment _environment; + private readonly HttpClient _httpClient; + private readonly LLamaOptions _options; + private readonly IHubContext _hubContext; + private readonly string _modelsRoot; + private readonly string _downloadsRoot; + private readonly object _syncLock = new(); + private readonly ConcurrentDictionary> _assetsByModel = new(); + + private Task _downloadTask; + private bool _started; + private bool _initialized; + + public ModelDownloadService(IWebHostEnvironment environment, IHttpClientFactory httpClientFactory, IOptions options, IHubContext hubContext) + { + _environment = environment; + _httpClient = httpClientFactory.CreateClient(nameof(ModelDownloadService)); + _options = options.Value; + _hubContext = hubContext; + + var appDataRoot = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData); + var appRoot = Path.Combine(appDataRoot, "LLama.Web"); + _modelsRoot = Path.Combine(appRoot, "Models"); + _downloadsRoot = Path.Combine(appRoot, "Downloads"); + } + + public string ModelsRoot => _modelsRoot; + public string DownloadsRoot => _downloadsRoot; + + public void StartDownloads(CancellationToken cancellationToken) + { + lock (_syncLock) + { + if (_started) + return; + + _started = true; + EnsureInitialized(); + _downloadTask = Task.Run(() => DownloadAllAsync(cancellationToken), cancellationToken); + } + } + + public Task WaitForDownloadsAsync(CancellationToken cancellationToken) + { + lock (_syncLock) + { + return _downloadTask ?? Task.CompletedTask; + } + } + + public bool IsModelReady(string modelName) + { + if (string.IsNullOrWhiteSpace(modelName)) + return false; + + if (!_assetsByModel.TryGetValue(modelName, out var assets)) + return false; + + return assets.All(asset => asset.State == ModelDownloadState.Completed); + } + + public IReadOnlyList GetSnapshots() + { + EnsureInitialized(); + var snapshots = new List(); + foreach (var kvp in _assetsByModel) + { + var assets = kvp.Value; + var snapshot = new ModelDownloadSnapshot + { + ModelName = kvp.Key, + Ready = assets.All(asset => asset.State == ModelDownloadState.Completed) + }; + + foreach (var asset in assets) + snapshot.Assets.Add(ToSnapshot(asset)); + + snapshots.Add(snapshot); + } + + return snapshots; + } + + private void EnsureInitialized() + { + lock (_syncLock) + { + if (_initialized) + return; + + InitializeAssets(); + _initialized = true; + } + + _ = BroadcastSnapshotAsync(); + } + + private void InitializeAssets() + { + Directory.CreateDirectory(_modelsRoot); + Directory.CreateDirectory(_downloadsRoot); + foreach (var model in _options.Models ?? new List()) + { + var assets = new List(); + + var modelPath = ResolvePath(_modelsRoot, model.ModelPath); + model.ModelPath = modelPath; + assets.Add(CreateAssetStatus(model.Name, ModelAssetKind.Model, modelPath, model.ModelDownloadUrl)); + + if (!string.IsNullOrWhiteSpace(model.MmprojPath)) + { + var mmprojPath = ResolvePath(_modelsRoot, model.MmprojPath); + model.MmprojPath = mmprojPath; + assets.Add(CreateAssetStatus(model.Name, ModelAssetKind.Mmproj, mmprojPath, model.MmprojDownloadUrl)); + } + + _assetsByModel[model.Name] = assets; + } + } + + private ModelAssetStatus CreateAssetStatus(string modelName, ModelAssetKind kind, string localPath, string downloadUrl) + { + var status = new ModelAssetStatus + { + ModelName = modelName, + AssetKind = kind, + LocalPath = localPath, + DownloadUrl = downloadUrl, + FileName = Path.GetFileName(localPath), + State = ModelDownloadState.Queued + }; + + if (File.Exists(localPath)) + { + var info = new FileInfo(localPath); + status.TotalBytes = info.Length; + status.BytesReceived = info.Length; + status.State = ModelDownloadState.Completed; + } + else if (string.IsNullOrWhiteSpace(downloadUrl)) + { + status.State = ModelDownloadState.MissingUrl; + status.Error = "Download URL is missing."; + } + + return status; + } + + private async Task DownloadAllAsync(CancellationToken cancellationToken) + { + foreach (var assets in _assetsByModel.Values) + { + foreach (var asset in assets) + { + if (asset.State == ModelDownloadState.Completed) + continue; + + if (asset.State == ModelDownloadState.MissingUrl) + { + await BroadcastProgressAsync(asset); + continue; + } + + await DownloadAssetAsync(asset, cancellationToken); + } + } + } + + private async Task DownloadAssetAsync(ModelAssetStatus asset, CancellationToken cancellationToken) + { + asset.State = ModelDownloadState.Downloading; + asset.BytesReceived = 0; + asset.Error = null; + + await BroadcastProgressAsync(asset); + + var directory = Path.GetDirectoryName(asset.LocalPath); + if (string.IsNullOrWhiteSpace(directory)) + directory = _environment.ContentRootPath; + Directory.CreateDirectory(directory); + + var tempFileName = $"{asset.ModelName}-{asset.AssetKind}-{asset.FileName}.download"; + var tempPath = Path.Combine(_downloadsRoot, tempFileName); + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, asset.DownloadUrl); + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + response.EnsureSuccessStatusCode(); + + asset.TotalBytes = response.Content.Headers.ContentLength; + + var lastReport = DateTimeOffset.UtcNow; + + await using var fileStream = new FileStream(tempPath, FileMode.Create, FileAccess.Write, FileShare.None, 1024 * 64, useAsync: true); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + + var buffer = new byte[1024 * 64]; + int read; + while ((read = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0) + { + await fileStream.WriteAsync(buffer.AsMemory(0, read), cancellationToken); + asset.BytesReceived += read; + + if (DateTimeOffset.UtcNow - lastReport > TimeSpan.FromMilliseconds(250)) + { + lastReport = DateTimeOffset.UtcNow; + await BroadcastProgressAsync(asset); + } + } + + if (File.Exists(asset.LocalPath)) + File.Delete(asset.LocalPath); + + File.Move(tempPath, asset.LocalPath); + + asset.State = ModelDownloadState.Completed; + asset.BytesReceived = asset.TotalBytes ?? asset.BytesReceived; + asset.Error = null; + } + catch (Exception ex) + { + asset.State = ModelDownloadState.Failed; + asset.Error = ex.Message; + if (File.Exists(tempPath)) + File.Delete(tempPath); + } + finally + { + await BroadcastProgressAsync(asset); + } + } + + private static string ResolvePath(string contentRoot, string path) + { + if (string.IsNullOrWhiteSpace(path)) + return string.Empty; + + return Path.IsPathRooted(path) + ? path + : Path.GetFullPath(Path.Combine(contentRoot, path)); + } + + private Task BroadcastProgressAsync(ModelAssetStatus asset) + { + var progress = new ModelDownloadProgress + { + ModelName = asset.ModelName, + FileName = asset.FileName, + AssetKind = asset.AssetKind, + State = asset.State, + TotalBytes = asset.TotalBytes, + BytesReceived = asset.BytesReceived, + Error = asset.Error + }; + + return _hubContext.Clients.All.OnModelDownloadProgress(progress); + } + + private Task BroadcastSnapshotAsync() + { + return _hubContext.Clients.All.OnModelDownloadSnapshot(GetSnapshots()); + } + + private static ModelAssetSnapshot ToSnapshot(ModelAssetStatus asset) + { + return new ModelAssetSnapshot + { + ModelName = asset.ModelName, + FileName = asset.FileName, + AssetKind = asset.AssetKind, + State = asset.State, + TotalBytes = asset.TotalBytes, + BytesReceived = asset.BytesReceived, + Error = asset.Error + }; + } + + private sealed class ModelAssetStatus + { + public string ModelName { get; set; } + public string FileName { get; set; } + public string LocalPath { get; set; } + public string DownloadUrl { get; set; } + public ModelAssetKind AssetKind { get; set; } + public ModelDownloadState State { get; set; } + public long? TotalBytes { get; set; } + public long BytesReceived { get; set; } + public string Error { get; set; } + } +} diff --git a/LLama.Web/Services/ModelLoaderService.cs b/LLama.Web/Services/ModelLoaderService.cs index 1ff3e9250..69ffd8f02 100644 --- a/LLama.Web/Services/ModelLoaderService.cs +++ b/LLama.Web/Services/ModelLoaderService.cs @@ -3,28 +3,37 @@ namespace LLama.Web.Services; /// /// Service for managing loading/preloading of models at app startup /// -/// Type used to identify contexts /// public class ModelLoaderService : IHostedService { private readonly IModelService _modelService; + private readonly IModelDownloadService _modelDownloadService; /// /// Initializes a new instance of the class. /// /// The model service. - public ModelLoaderService(IModelService modelService) + public ModelLoaderService(IModelService modelService, IModelDownloadService modelDownloadService) { _modelService = modelService; + _modelDownloadService = modelDownloadService; } /// /// Triggered when the application host is ready to start the service. /// /// Indicates that the start process has been aborted. - public async Task StartAsync(CancellationToken cancellationToken) + public Task StartAsync(CancellationToken cancellationToken) { - await _modelService.LoadModels(); + _modelDownloadService.StartDownloads(cancellationToken); + + _ = Task.Run(async () => + { + await _modelDownloadService.WaitForDownloadsAsync(cancellationToken); + await _modelService.LoadModels(); + }, cancellationToken); + + return Task.CompletedTask; } /// diff --git a/LLama.Web/Services/ModelService.cs b/LLama.Web/Services/ModelService.cs index db95beca7..f5351387a 100644 --- a/LLama.Web/Services/ModelService.cs +++ b/LLama.Web/Services/ModelService.cs @@ -7,7 +7,7 @@ namespace LLama.Web.Services; /// -/// Service for handling Models,Weights & Contexts +/// Service for handling models, weights, and contexts. /// public class ModelService : IModelService { @@ -15,19 +15,22 @@ public class ModelService : IModelService private readonly AsyncLock _contextLock; private readonly LLamaOptions _configuration; private readonly ILogger _llamaLogger; + private readonly IModelDownloadService _modelDownloadService; private readonly ConcurrentDictionary _modelInstances; /// /// Initializes a new instance of the class. /// - /// The logger. - /// The options. - public ModelService(IOptions configuration, ILogger llamaLogger) + /// The configuration. + /// The logger. + /// The model download service. + public ModelService(IOptions configuration, ILogger llamaLogger, IModelDownloadService modelDownloadService) { _llamaLogger = llamaLogger; _modelLock = new AsyncLock(); _contextLock = new AsyncLock(); _configuration = configuration.Value; + _modelDownloadService = modelDownloadService; _modelInstances = new ConcurrentDictionary(); } @@ -46,11 +49,17 @@ public async Task LoadModel(ModelOptions modelOptions) if (_modelInstances.TryGetValue(modelOptions.Name, out var model)) return model; - // If in single mode unload any other models + if (!_modelDownloadService.IsModelReady(modelOptions.Name)) + throw new Exception($"Model '{modelOptions.Name}' is still downloading."); + + // If in single mode, unload any other models. if (_configuration.ModelLoadType == ModelLoadType.Single || _configuration.ModelLoadType == ModelLoadType.PreloadSingle) await UnloadModels(); + if (modelOptions.UBatchSize < modelOptions.BatchSize) + modelOptions.UBatchSize = modelOptions.BatchSize; + model = await LLamaModel.CreateAsync(modelOptions, _llamaLogger); _modelInstances.TryAdd(modelOptions.Name, model); return model; @@ -70,7 +79,7 @@ public async Task LoadModels() { await LoadModel(modelConfig); - //Only preload first model if in SinglePreload mode + // Only preload the first model in PreloadSingle mode. if (_configuration.ModelLoadType == ModelLoadType.PreloadSingle) break; } @@ -103,7 +112,7 @@ public async Task UnloadModels() } /// - /// Gets a model ny name. + /// Gets a model by name. /// /// Name of the model. /// @@ -165,7 +174,7 @@ public async Task RemoveContext(string modelName, string contextName) } /// - /// Loads, Gets,Creates a Model and a Context + /// Loads, gets, or creates a model and a context. /// /// Name of the model. /// The contextName. @@ -176,15 +185,15 @@ public async Task RemoveContext(string modelName, string contextName) if (_modelInstances.TryGetValue(modelName, out var model)) return (model, await model.GetContext(contextName) ?? await model.CreateContext(contextName)); - // Get model configuration + // Get model configuration. var modelConfig = _configuration.Models.FirstOrDefault(x => x.Name == modelName); if (modelConfig is null) throw new Exception($"Model option '{modelName}' not found"); - // Load Model + // Load model. model = await LoadModel(modelConfig); - // Get or Create Context + // Get or create context. return (model, await model.GetContext(contextName) ?? await model.CreateContext(contextName)); } } diff --git a/LLama.Web/Services/ModelSessionService.cs b/LLama.Web/Services/ModelSessionService.cs index 3fc95328a..a316c8299 100644 --- a/LLama.Web/Services/ModelSessionService.cs +++ b/LLama.Web/Services/ModelSessionService.cs @@ -1,63 +1,67 @@ using LLama.Web.Async; using LLama.Web.Common; using LLama.Web.Models; +using LLama.Native; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Text; namespace LLama.Web.Services; /// -/// Example Service for handling a model session for a websockets connection lifetime -/// Each websocket connection will create its own unique session and context allowing you to use multiple tabs to compare prompts etc +/// Example service for handling a model session for the lifetime of a WebSocket connection. +/// Each WebSocket connection creates its own session and context, allowing you to use multiple tabs to compare prompts and results. /// public class ModelSessionService : IModelSessionService { private readonly AsyncGuard _sessionGuard; private readonly IModelService _modelService; + private readonly IAttachmentService _attachmentService; private readonly ConcurrentDictionary _modelSessions; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The model service. - /// The model session state service. - public ModelSessionService(IModelService modelService) + /// The attachment service. + public ModelSessionService(IModelService modelService, IAttachmentService attachmentService) { _modelService = modelService; + _attachmentService = attachmentService; _sessionGuard = new AsyncGuard(); _modelSessions = new ConcurrentDictionary(); } /// - /// Gets the ModelSession with the specified Id. + /// Gets the ModelSession with the specified ID. /// /// The session identifier. - /// The ModelSession if exists, otherwise null + /// The ModelSession if it exists; otherwise, null. public Task GetAsync(string sessionId) { return Task.FromResult(_modelSessions.TryGetValue(sessionId, out var session) ? session : null); } /// - /// Gets all ModelSessions + /// Gets all model sessions. /// - /// A collection oa all Model instances + /// A collection of all model sessions. public Task> GetAllAsync() { return Task.FromResult>(_modelSessions.Values); } /// - /// Creates a new ModelSession + /// Creates a new model session. /// /// The session identifier. /// The session configuration. - /// The default inference configuration, will be used for all inference where no infer configuration is supplied. + /// The default inference configuration, used when no inference configuration is supplied. /// The cancellation token. /// /// - /// Session with id {sessionId} already exists + /// Session with ID {sessionId} already exists /// or /// Failed to create model session /// @@ -66,22 +70,22 @@ public async Task CreateAsync(string sessionId, ISessionConfig ses if (_modelSessions.TryGetValue(sessionId, out _)) throw new Exception($"Session with id {sessionId} already exists"); - // Create context + // Create context. var (model, context) = await _modelService.GetOrCreateModelAndContext(sessionConfig.Model, sessionId); - // Create session + // Create session. var modelSession = new ModelSession(model, context, sessionId, sessionConfig, inferenceConfig); if (!_modelSessions.TryAdd(sessionId, modelSession)) throw new Exception($"Failed to create model session"); - // Run initial Prompt + // Run initial prompt. await modelSession.InitializePrompt(inferenceConfig, cancellationToken); return modelSession; } /// - /// Closes the session + /// Closes the session. /// /// The session identifier. /// @@ -96,14 +100,14 @@ public async Task CloseAsync(string sessionId) } /// - /// Runs inference on the current ModelSession + /// Runs inference on the current model session. /// /// The session identifier. - /// The prompt. - /// The inference configuration, if null session default is used + /// The prompt request. + /// The inference configuration; if null, the session default is used. /// The cancellation token. /// Inference is already running for this session - public async IAsyncEnumerable InferAsync(string sessionId, string prompt, InferenceOptions inferenceConfig = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InferAsync(string sessionId, PromptRequest request, InferenceOptions inferenceConfig = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { if (!_sessionGuard.Guard(sessionId)) throw new Exception($"Inference is already running for this session"); @@ -113,17 +117,26 @@ public async IAsyncEnumerable InferAsync(string sessionId, string pr if (!_modelSessions.TryGetValue(sessionId, out var modelSession)) yield break; - // Send begin of response + var preparedPrompt = await PreparePromptAsync(sessionId, modelSession, request, cancellationToken); + var formattedPrompt = modelSession.BuildPrompt(preparedPrompt.Prompt); + + // Send start of response. var stopwatch = Stopwatch.GetTimestamp(); yield return new TokenModel(default, default, TokenType.Begin); - // Send content of response - await foreach (var token in modelSession.InferAsync(prompt, inferenceConfig, cancellationToken).ConfigureAwait(false)) + var responseBuilder = new StringBuilder(); + + // Send response content. + await foreach (var token in modelSession.InferAsync(formattedPrompt, inferenceConfig, cancellationToken).ConfigureAwait(false)) { + responseBuilder.Append(token); yield return new TokenModel(default, token); } - // Send end of response + if (!modelSession.IsInferCanceled() && modelSession.StoresHistory) + modelSession.AddAssistantMessage(responseBuilder.ToString()); + + // Send end of response. var elapsedTime = GetElapsed(stopwatch); var endTokenType = modelSession.IsInferCanceled() ? TokenType.Cancel : TokenType.End; var signature = endTokenType == TokenType.Cancel @@ -138,11 +151,11 @@ public async IAsyncEnumerable InferAsync(string sessionId, string pr } /// - /// Runs inference on the current ModelSession + /// Runs inference on the current model session. /// /// The session identifier. /// The prompt. - /// The inference configuration, if null session default is used + /// The inference configuration; if null, the session default is used. /// The cancellation token. /// Streaming async result of /// Inference is already running for this session @@ -150,7 +163,7 @@ public IAsyncEnumerable InferTextAsync(string sessionId, string prompt, { async IAsyncEnumerable InferTextInternal() { - await foreach (var token in InferAsync(sessionId, prompt, inferenceConfig, cancellationToken).ConfigureAwait(false)) + await foreach (var token in InferAsync(sessionId, new PromptRequest { Prompt = prompt }, inferenceConfig, cancellationToken).ConfigureAwait(false)) { if (token.TokenType == TokenType.Content) yield return token.Content; @@ -160,17 +173,17 @@ async IAsyncEnumerable InferTextInternal() } /// - /// Runs inference on the current ModelSession + /// Runs inference on the current model session. /// /// The session identifier. /// The prompt. - /// The inference configuration, if null session default is used + /// The inference configuration; if null, the session default is used. /// The cancellation token. /// Completed inference result as string /// Inference is already running for this session public async Task InferTextCompleteAsync(string sessionId, string prompt, InferenceOptions inferenceConfig = null, CancellationToken cancellationToken = default) { - var inferResult = await InferAsync(sessionId, prompt, inferenceConfig, cancellationToken) + var inferResult = await InferAsync(sessionId, new PromptRequest { Prompt = prompt }, inferenceConfig, cancellationToken) .Where(x => x.TokenType == TokenType.Content) .Select(x => x.Content) .ToListAsync(cancellationToken: cancellationToken); @@ -193,6 +206,31 @@ public Task CancelAsync(string sessionId) return Task.FromResult(false); } + /// + /// Gets the model capabilities for the specified session. + /// + /// The session identifier. + /// The model capabilities. + public Task GetCapabilitiesAsync(string sessionId) + { + if (_modelSessions.TryGetValue(sessionId, out var modelSession)) + { + return Task.FromResult(new ModelCapabilities + { + SupportsText = true, + SupportsVision = modelSession.SupportsVision, + SupportsAudio = modelSession.SupportsAudio + }); + } + + return Task.FromResult(new ModelCapabilities + { + SupportsText = true, + SupportsVision = false, + SupportsAudio = false + }); + } + /// /// Gets the elapsed time in milliseconds. /// @@ -202,4 +240,121 @@ private static int GetElapsed(long timestamp) { return (int)Stopwatch.GetElapsedTime(timestamp).TotalMilliseconds; } + + private async Task PreparePromptAsync(string sessionId, ModelSession modelSession, PromptRequest request, CancellationToken cancellationToken) + { + if (request == null) + return new PreparedPrompt(string.Empty); + + var promptBuilder = new StringBuilder(request.Prompt ?? string.Empty); + var embeds = new List(); + + try + { + if (request.AttachmentIds != null && request.AttachmentIds.Count > 0) + { + var mediaMarker = GetMediaMarker(); + var attachments = _attachmentService.GetAttachments(sessionId, request.AttachmentIds); + + foreach (var attachment in attachments) + { + switch (attachment.Kind) + { + case AttachmentKind.Pdf: + AppendPdf(promptBuilder, attachment); + break; + case AttachmentKind.Word: + AppendWord(promptBuilder, attachment); + break; + case AttachmentKind.Image: + if (!modelSession.IsMultiModal) + throw new Exception("This model does not support multimodal inputs."); + + embeds.Add(await LoadEmbedAsync(modelSession, attachment, cancellationToken)); + AppendMedia(promptBuilder, attachment, "Image", mediaMarker); + break; + case AttachmentKind.Audio: + if (!modelSession.IsMultiModal || !modelSession.SupportsAudio) + throw new Exception("This model does not support audio inputs."); + + embeds.Add(await LoadEmbedAsync(modelSession, attachment, cancellationToken)); + AppendMedia(promptBuilder, attachment, "Audio", mediaMarker); + break; + } + } + } + + if (embeds.Count > 0) + modelSession.QueueEmbeds(embeds); + + return new PreparedPrompt(promptBuilder.ToString()); + } + catch + { + foreach (var embed in embeds) + embed.Dispose(); + + throw; + } + } + + private static void AppendPdf(StringBuilder builder, AttachmentInfo attachment) + { + if (string.IsNullOrWhiteSpace(attachment.ExtractedText)) + return; + + builder.AppendLine(); + builder.AppendLine($"[PDF: {attachment.FileName}]"); + builder.AppendLine(attachment.ExtractedText.Trim()); + if (attachment.ExtractedTextTruncated) + builder.AppendLine("[PDF text truncated]"); + } + + private static void AppendMedia(StringBuilder builder, AttachmentInfo attachment, string mediaLabel, string mediaMarker) + { + builder.AppendLine(); + builder.AppendLine($"[{mediaLabel}: {attachment.FileName}]"); + builder.AppendLine(mediaMarker); + } + + private static string GetMediaMarker() + { + var mtmdParameters = MtmdContextParams.Default(); + return mtmdParameters.MediaMarker ?? NativeApi.MtmdDefaultMarker() ?? ""; + } + + private static void AppendWord(StringBuilder builder, AttachmentInfo attachment) + { + builder.AppendLine(); + builder.AppendLine($"[Word: {attachment.FileName}]"); + + if (string.IsNullOrWhiteSpace(attachment.ExtractedText)) + { + builder.AppendLine("[Word text could not be extracted]"); + return; + } + + builder.AppendLine(attachment.ExtractedText.Trim()); + if (attachment.ExtractedTextTruncated) + builder.AppendLine("[Word text truncated]"); + } + + private static async Task LoadEmbedAsync(ModelSession modelSession, AttachmentInfo attachment, CancellationToken cancellationToken) + { + if (!File.Exists(attachment.FilePath)) + throw new FileNotFoundException("Attachment not found.", attachment.FilePath); + + var data = await File.ReadAllBytesAsync(attachment.FilePath, cancellationToken); + var clipModel = modelSession.ClipModel; + if (clipModel is null) + throw new Exception("Multimodal model is not available for this session."); + + var embed = clipModel.LoadMedia(data); + if (embed == null) + throw new Exception("Failed to prepare multimodal embedding."); + + return embed; + } + + private sealed record PreparedPrompt(string Prompt); } diff --git a/LLama.Web/Shared/MainLayout.razor b/LLama.Web/Shared/MainLayout.razor new file mode 100644 index 000000000..224e0204e --- /dev/null +++ b/LLama.Web/Shared/MainLayout.razor @@ -0,0 +1,5 @@ +@inherits LayoutComponentBase + +
+ @Body +
diff --git a/LLama.Web/_Imports.razor b/LLama.Web/_Imports.razor new file mode 100644 index 000000000..143d6c514 --- /dev/null +++ b/LLama.Web/_Imports.razor @@ -0,0 +1,13 @@ +@using System.Net.Http +@using System.Net.Http.Json +@using System.IO +@using Microsoft.AspNetCore.Components +@using Microsoft.AspNetCore.Components.Forms +@using Microsoft.AspNetCore.Components.Routing +@using Microsoft.AspNetCore.Components.Web +@using Microsoft.Extensions.Options +@using Microsoft.JSInterop +@using LLama.Web.Common +@using LLama.Web.Models +@using LLama.Web.Services +@using LLama.Sampling diff --git a/LLama.Web/appsettings.json b/LLama.Web/appsettings.json index caa27cc64..41465acff 100644 --- a/LLama.Web/appsettings.json +++ b/LLama.Web/appsettings.json @@ -10,11 +10,15 @@ "ModelLoadType": 0, "Models": [ { - "Name": "Example LLava-v1.6-mistral", + "Name": "Qwen3-VL-2B-Instruct Q4_K_M", "MaxInstances": 20, - "ModelPath": "..\\LLama.Unittest\\Models\\llava-v1.6-mistral-7b.Q3_K_XS.gguf", - "ContextSize": 2048, - "BatchSize": 2048, + "ModelPath": "Models/Qwen3VL-2B-Instruct-Q4_K_M.gguf", + "ModelDownloadUrl": "https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-GGUF/resolve/main/Qwen3VL-2B-Instruct-Q4_K_M.gguf", + "MmprojPath": "Models/mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf", + "MmprojDownloadUrl": "https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-GGUF/resolve/main/mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf", + "ContextSize": 4096, + "BatchSize": 512, + "UBatchSize": 512, "Threads": 4, "GpuLayerCount": 32, "UseMemorymap": true, @@ -22,7 +26,67 @@ "MainGpu": 0, "LowVram": false, "Seed": 1686349486, - "UseFp16Memory": true, + "UseFp16Memory": false, + "Perplexity": false, + "LoraAdapter": "", + "LoraBase": "", + "EmbeddingMode": false, + "TensorSplits": null, + "GroupedQueryAttention": 1, + "RmsNormEpsilon": 0.000005, + "RopeFrequencyBase": 10000.0, + "RopeFrequencyScale": 1.0, + "MulMatQ": false, + "Encoding": "UTF-8" + }, + { + "Name": "Gemma-3-4B-IT Q4_K_M", + "MaxInstances": 20, + "ModelPath": "Models/google_gemma-3-4b-it-q4_k_m.gguf", + "ModelDownloadUrl": "https://huggingface.co/ggml-org/gemma-3-4b-it-GGUF/resolve/main/gemma-3-4b-it-Q4_K_M.gguf", + "MmprojPath": "Models/google_gemma-3-4b-it-mmproj-f16.gguf", + "MmprojDownloadUrl": "https://huggingface.co/ggml-org/gemma-3-4b-it-GGUF/resolve/main/mmproj-model-f16.gguf", + "ContextSize": 4096, + "BatchSize": 512, + "UBatchSize": 512, + "Threads": 4, + "GpuLayerCount": 32, + "UseMemorymap": true, + "UseMemoryLock": false, + "MainGpu": 0, + "LowVram": false, + "Seed": 1686349486, + "UseFp16Memory": false, + "Perplexity": false, + "LoraAdapter": "", + "LoraBase": "", + "EmbeddingMode": false, + "TensorSplits": null, + "GroupedQueryAttention": 1, + "RmsNormEpsilon": 0.000005, + "RopeFrequencyBase": 10000.0, + "RopeFrequencyScale": 1.0, + "MulMatQ": false, + "Encoding": "UTF-8" + }, + { + "Name": "Qwen2.5-Omni-3B Q4_K_M (Audio)", + "MaxInstances": 20, + "ModelPath": "Models/Qwen2.5-Omni-3B-Q4_K_M.gguf", + "ModelDownloadUrl": "https://huggingface.co/ggml-org/Qwen2.5-Omni-3B-GGUF/resolve/main/Qwen2.5-Omni-3B-Q4_K_M.gguf", + "MmprojPath": "Models/mmproj-Qwen2.5-Omni-3B-Q8_0.gguf", + "MmprojDownloadUrl": "https://huggingface.co/ggml-org/Qwen2.5-Omni-3B-GGUF/resolve/main/mmproj-Qwen2.5-Omni-3B-Q8_0.gguf", + "ContextSize": 4096, + "BatchSize": 512, + "UBatchSize": 512, + "Threads": 4, + "GpuLayerCount": 32, + "UseMemorymap": true, + "UseMemoryLock": false, + "MainGpu": 0, + "LowVram": false, + "Seed": 1686349486, + "UseFp16Memory": false, "Perplexity": false, "LoraAdapter": "", "LoraBase": "", diff --git a/LLama.Web/libman.json b/LLama.Web/libman.json index bb737e097..8a7b77c9b 100644 --- a/LLama.Web/libman.json +++ b/LLama.Web/libman.json @@ -5,6 +5,71 @@ { "library": "jquery@3.7.1", "destination": "wwwroot/lib/jquery/" + }, + { + "library": "katex@0.16.11", + "provider": "unpkg", + "destination": "wwwroot/lib/katex/", + "files": [ + "dist/katex.min.css", + "dist/katex.min.js" + ] + }, + { + "library": "markdown-it@14.1.0", + "provider": "unpkg", + "destination": "wwwroot/lib/markdown-it/", + "files": [ + "dist/markdown-it.min.js" + ] + }, + { + "library": "markdown-it-katex@2.0.3", + "provider": "unpkg", + "destination": "wwwroot/lib/markdown-it-katex/" + }, + { + "library": "markdown-it-task-lists@2.1.1", + "provider": "unpkg", + "destination": "wwwroot/lib/markdown-it-task-lists/" + }, + { + "library": "markdown-it-footnote@4.0.0", + "provider": "unpkg", + "destination": "wwwroot/lib/markdown-it-footnote/" + }, + { + "library": "markdown-it-deflist@2.1.0", + "provider": "unpkg", + "destination": "wwwroot/lib/markdown-it-deflist/" + }, + { + "library": "markdown-it-sub@2.0.0", + "provider": "unpkg", + "destination": "wwwroot/lib/markdown-it-sub/" + }, + { + "library": "markdown-it-sup@2.0.0", + "provider": "unpkg", + "destination": "wwwroot/lib/markdown-it-sup/" + }, + { + "library": "markdown-it-mark@4.0.0", + "provider": "unpkg", + "destination": "wwwroot/lib/markdown-it-mark/" + }, + { + "library": "markdown-it-emoji@3.0.0", + "provider": "unpkg", + "destination": "wwwroot/lib/markdown-it-emoji/" + }, + { + "library": "mermaid@10.9.1", + "provider": "unpkg", + "destination": "wwwroot/lib/mermaid/", + "files": [ + "dist/mermaid.min.js" + ] } ] -} \ No newline at end of file +} diff --git a/LLama.Web/wwwroot/css/site.css b/LLama.Web/wwwroot/css/site.css index f10d5e78c..217617950 100644 --- a/LLama.Web/wwwroot/css/site.css +++ b/LLama.Web/wwwroot/css/site.css @@ -1,106 +1,570 @@ @font-face { - font-family: 'Roboto'; + font-family: "Roboto"; src: url('/fonts/Roboto-Regular.ttf'); font-display: swap; } +:root { + color-scheme: dark; + --bg-0: #0b0d10; + --bg-1: #0f1218; + --bg-2: #141926; + --panel: #111722; + --panel-2: #161c2a; + --panel-3: #1b2233; + --border: rgba(255, 255, 255, 0.08); + --border-strong: rgba(255, 255, 255, 0.16); + --text-primary: #e7ecf3; + --text-muted: #9aa6b2; + --accent: #6ae3ff; + --accent-2: #66d1a3; + --warning: #ffb86b; + --danger: #ff7a7a; + --shadow: 0 18px 40px rgba(0, 0, 0, 0.35); +} + +* { + box-sizing: border-box; +} + html, body { - font-size: 14px; height: 100%; + margin: 0; + font-family: "Roboto", "Segoe UI", sans-serif; + background: radial-gradient(circle at top, #121826 0%, #0b0d10 45%, #090b0f 100%); + color: var(--text-primary); +} + +main { + min-height: 100%; +} + +.app-shell { + min-height: 100vh; + padding: 16px; +} + +.app-grid { + display: grid; + grid-template-columns: minmax(260px, 320px) 1fr; + gap: 16px; + min-height: calc(100vh - 32px); +} + +.sidebar { + background: linear-gradient(180deg, #0f1420 0%, #0b0d12 100%); + border: 1px solid var(--border); + border-radius: 18px; + padding: 18px; display: flex; flex-direction: column; - font-family: 'Roboto'; + gap: 18px; + box-shadow: var(--shadow); } -main { +.sidebar-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.brand { + display: flex; + align-items: center; + gap: 10px; +} + +.brand img { + width: 32px; + height: 32px; +} + +.brand h1 { + font-size: 1.1rem; + margin: 0; + letter-spacing: 0.02em; +} + +.sidebar-section { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 14px; + padding: 12px; + display: flex; + flex-direction: column; + gap: 10px; +} + +.capability-row { + display: flex; + gap: 8px; + flex-wrap: wrap; +} + +.capability-pill { + padding: 6px 10px; + border-radius: 999px; + font-size: 0.75rem; + font-weight: 600; + border: 1px solid var(--border); +} + +.capability-pill.enabled { + background: rgba(106, 227, 255, 0.18); + color: var(--text-primary); + border-color: rgba(106, 227, 255, 0.3); +} + +.capability-pill.disabled { + background: rgba(255, 255, 255, 0.04); + color: var(--text-muted); +} + +.sidebar-tabs { + display: flex; + gap: 8px; + background: var(--panel); + border: 1px solid var(--border); + border-radius: 999px; + padding: 4px; +} + +.tab-button { flex: 1; - overflow: auto; + border: none; + background: transparent; + color: var(--text-muted); + padding: 8px 10px; + border-radius: 999px; + font-weight: 600; + cursor: pointer; } -footer { - bottom: 0; +.tab-button.active { + background: rgba(106, 227, 255, 0.18); + color: var(--text-primary); +} + +.section-title { + font-size: 0.9rem; + color: var(--text-muted); + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.select, .text-input, .textarea { width: 100%; - white-space: nowrap; - line-height: 60px; + background: var(--panel-2); + border: 1px solid var(--border-strong); + color: var(--text-primary); + border-radius: 10px; + padding: 10px 12px; } +.textarea { + min-height: 80px; + resize: vertical; +} -@media (min-width: 768px) { - html { - font-size: 16px; - } +.button-row { + display: flex; + gap: 8px; +} + +.btn-primary, .btn-secondary, .btn-danger { + border: none; + padding: 10px 14px; + border-radius: 10px; + font-weight: 600; + cursor: pointer; +} + +.btn-primary { + background: linear-gradient(135deg, #4f8cff 0%, #6ae3ff 100%); + color: #0b0d10; +} + +.btn-secondary { + background: var(--panel-3); + color: var(--text-primary); + border: 1px solid var(--border); +} + +.btn-danger { + background: linear-gradient(135deg, #ff7a7a 0%, #ffb86b 100%); + color: #1b0c0c; +} + +.btn-primary:disabled, +.btn-secondary:disabled, +.btn-danger:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.sidebar-meta { + font-size: 0.75rem; + color: var(--text-muted); + line-height: 1.4; +} + +.download-card { + border-radius: 12px; + border: 1px solid var(--border); + padding: 10px; + background: var(--panel-2); + display: flex; + flex-direction: column; + gap: 6px; +} + +.download-card header { + display: flex; + justify-content: space-between; + font-weight: 600; +} + +.progress-track { + width: 100%; + height: 6px; + border-radius: 999px; + background: rgba(255, 255, 255, 0.06); + overflow: hidden; +} + +.progress-fill { + height: 100%; + background: linear-gradient(90deg, var(--accent) 0%, var(--accent-2) 100%); + border-radius: 999px; +} + +.download-meta { + font-size: 0.75rem; + color: var(--text-muted); +} + +.download-asset { + border-radius: 10px; + padding: 8px; + background: rgba(255, 255, 255, 0.03); + border: 1px solid rgba(255, 255, 255, 0.06); + display: flex; + flex-direction: column; + gap: 4px; +} + +.download-asset-title { + display: flex; + justify-content: space-between; + font-size: 0.8rem; +} + +.chat-column { + display: flex; + flex-direction: column; + gap: 12px; + background: linear-gradient(180deg, rgba(20, 25, 38, 0.92) 0%, rgba(12, 15, 22, 0.96) 100%); + border-radius: 22px; + border: 1px solid var(--border); + padding: 18px; + box-shadow: var(--shadow); + min-height: 100%; + height: calc(100vh - 32px); +} + +.chat-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.chat-title { + display: flex; + flex-direction: column; + gap: 4px; +} + +.chat-title h2 { + margin: 0; + font-size: 1.3rem; +} + +.chat-title span { + color: var(--text-muted); + font-size: 0.85rem; +} + +.status-pill { + padding: 6px 10px; + border-radius: 999px; + font-size: 0.75rem; + border: 1px solid var(--border); + background: var(--panel-2); } -#scroll-container { +.status-pill.live { + border-color: rgba(106, 227, 255, 0.5); + color: var(--accent); +} + +.status-pill.warn { + border-color: rgba(255, 184, 107, 0.5); + color: var(--warning); +} + +.chat-thread { flex: 1; overflow-y: auto; - border-radius: var(--bs-border-radius); + padding-right: 6px; + display: flex; + flex-direction: column; + gap: 18px; + min-height: 0; +} + +.chat-message { + display: flex; + gap: 12px; + align-items: flex-start; +} + +.chat-message .avatar { + width: 36px; + height: 36px; + border-radius: 12px; + background: var(--panel-3); + display: grid; + place-items: center; + font-weight: 700; + color: var(--text-muted); +} + +.chat-message.user .avatar { + background: rgba(106, 227, 255, 0.16); + color: var(--accent); +} + +.chat-message.assistant .avatar { + background: rgba(102, 209, 163, 0.16); + color: var(--accent-2); +} + +.chat-message.error .avatar { + background: rgba(255, 122, 122, 0.2); + color: var(--danger); +} + +.chat-message.info .avatar { + background: rgba(106, 227, 255, 0.12); + color: var(--accent); +} + +.chat-bubble { + background: var(--panel); + border-radius: 16px; + padding: 12px 14px; + border: 1px solid var(--border); + width: 100%; +} + +.chat-message.user .chat-bubble { + background: linear-gradient(135deg, rgba(79, 140, 255, 0.16), rgba(106, 227, 255, 0.12)); +} + +.chat-message.assistant .chat-bubble { + background: rgba(22, 28, 42, 0.85); +} + +.chat-message.info .chat-bubble { + border-color: rgba(106, 227, 255, 0.2); + background: rgba(20, 30, 45, 0.7); +} + +.message-meta { + margin-top: 8px; + font-size: 0.75rem; + color: var(--text-muted); +} + +.attachment-tags { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 8px; +} + +.attachment-images { + display: flex; + flex-wrap: wrap; + gap: 10px; + margin-top: 10px; } -.slider-container > .slider { +.attachment-image { + max-width: min(360px, 100%); + border-radius: 12px; + border: 1px solid rgba(255, 255, 255, 0.12); + box-shadow: 0 8px 18px rgba(0, 0, 0, 0.25); +} + +.attachment-audio { + display: flex; + flex-direction: column; + gap: 8px; + margin-top: 10px; +} + +.attachment-audio-item { + display: flex; + flex-direction: column; + gap: 6px; +} + +.attachment-audio-name { + font-size: 0.8rem; + color: var(--text-muted); +} + +.attachment-audio audio { width: 100%; } -.slider-container > label { - width: 50px; - text-align: center; +.attachment-tag { + padding: 4px 8px; + border-radius: 999px; + background: rgba(255, 255, 255, 0.08); + font-size: 0.75rem; + color: var(--text-muted); } -.form-floating > textarea.form-control { - height: 100px; +.composer { + border-radius: 16px; + border: 1px solid var(--border); + padding: 12px; + background: var(--panel); + display: flex; + flex-direction: column; + gap: 10px; + position: sticky; + bottom: 0; } -.navbar-image { - height: 30px; - margin-left: 5px; - margin-right: 10px; +.draft-preview { + padding: 0 4px; } -.nav-item.theme { - margin-top: 2px; +.draft-bubble { + border-color: rgba(106, 227, 255, 0.35); + background: rgba(79, 140, 255, 0.12); } -/* Dark theme values */ -[data-bs-theme=dark] #scroll-container { - background-color: #202020; +.draft-text { + white-space: pre-wrap; + color: var(--text-primary); } -[data-bs-theme=dark] html, -[data-bs-theme=dark] body, -[data-bs-theme=dark] #sidebar, -[data-bs-theme=dark] header { - background-color: #191919; +.composer-error { + color: var(--danger); + font-size: 0.85rem; + padding: 4px 6px; + border-radius: 8px; + background: rgba(255, 122, 122, 0.08); + border: 1px solid rgba(255, 122, 122, 0.3); +} + +.composer textarea { + width: 100%; + background: transparent; + border: none; + resize: none; + color: var(--text-primary); + min-height: 72px; } -.entry { - padding-top: 10px; - padding-bottom: 20px; - padding-right: 10px; +.composer textarea:focus { + outline: none; } - .entry .signature, .entry .entry-date { - color: #808080; - font-size: 0.9em; - font-style: italic; +.composer-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; +} + +.composer-actions { + display: flex; + gap: 8px; +} + +.file-button { + position: relative; + overflow: hidden; +} + +.file-input { + position: absolute; + inset: 0; + opacity: 0; + cursor: pointer; +} + +.file-pill { + padding: 6px 10px; + border-radius: 999px; + font-size: 0.75rem; + background: rgba(255, 255, 255, 0.08); + color: var(--text-muted); +} + +.markdown-body pre { + padding: 12px; + border-radius: 10px; + background-color: rgba(0, 0, 0, 0.25); + overflow-x: auto; +} + +.markdown-body code { + padding: 2px 4px; + border-radius: 4px; + background-color: rgba(0, 0, 0, 0.25); +} + +.markdown-body table { + width: 100%; + border-collapse: collapse; + margin-top: 8px; +} + +.markdown-body th, +.markdown-body td { + padding: 6px 8px; + border: 1px solid rgba(255, 255, 255, 0.1); +} + +.markdown-body img { + max-width: 100%; + border-radius: 10px; +} + +.markdown-body blockquote { + border-left: 3px solid var(--accent); + padding-left: 12px; + color: var(--text-muted); +} + +@media (max-width: 960px) { + .app-grid { + grid-template-columns: 1fr; } - .entry .content, .entry .signature { - padding-left: 5px; - margin-left: 50px; + .sidebar { + order: 2; } - .entry .entry-header .participant { - font-weight: bold; - font-size: 1.4em; - padding-bottom: 5px; - } - - .entry .entry-header .participant .participant-icon { - display: inline-block; - width: 50px; - text-align: center - } - - .entry .entry-header .participant .participant-name { - display: inline-block; - } + .chat-column { + order: 1; + } +} diff --git a/LLama.Web/wwwroot/js/blazorChat.js b/LLama.Web/wwwroot/js/blazorChat.js new file mode 100644 index 000000000..5099e04b0 --- /dev/null +++ b/LLama.Web/wwwroot/js/blazorChat.js @@ -0,0 +1,193 @@ +window.llamaChat = (() => { + let markdown = null; + let audioRecorder = null; + let audioChunks = []; + let audioStream = null; + + const ensureMarkdown = () => { + if (markdown) { + return markdown; + } + + markdown = window.markdownit({ + html: false, + linkify: true, + breaks: true + }); + + if (window.markdownitKatex) { + markdown.use(window.markdownitKatex); + } + if (window.markdownitTaskLists) { + markdown.use(window.markdownitTaskLists); + } + if (window.markdownitFootnote) { + markdown.use(window.markdownitFootnote); + } + if (window.markdownitDeflist) { + markdown.use(window.markdownitDeflist); + } + if (window.markdownitSub) { + markdown.use(window.markdownitSub); + } + if (window.markdownitSup) { + markdown.use(window.markdownitSup); + } + if (window.markdownitMark) { + markdown.use(window.markdownitMark); + } + if (window.markdownitEmoji) { + markdown.use(window.markdownitEmoji); + } + + if (window.mermaid) { + window.mermaid.initialize({ startOnLoad: false, theme: "dark" }); + } + + return markdown; + }; + + const renderMermaid = (container) => { + if (!window.mermaid || !container) { + return; + } + + const codeBlocks = container.querySelectorAll("pre > code.language-mermaid, pre > code.lang-mermaid, code.language-mermaid"); + if (!codeBlocks.length) { + return; + } + + codeBlocks.forEach((code) => { + const parent = code.parentElement; + const mermaidDiv = document.createElement("div"); + mermaidDiv.className = "mermaid"; + mermaidDiv.textContent = code.textContent || ""; + parent.replaceWith(mermaidDiv); + }); + + window.mermaid.run({ nodes: container.querySelectorAll('.mermaid') }); + }; + + const isAudioRecordingSupported = () => { + return !!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia && window.MediaRecorder); + }; + + const getSupportedAudioMimeType = () => { + if (!window.MediaRecorder || !MediaRecorder.isTypeSupported) { + return ""; + } + + const candidates = [ + "audio/webm;codecs=opus", + "audio/webm", + "audio/ogg;codecs=opus", + "audio/ogg", + "audio/mp4", + "audio/mpeg" + ]; + + return candidates.find((candidate) => MediaRecorder.isTypeSupported(candidate)) || ""; + }; + + const cleanupAudioStream = () => { + if (audioStream) { + audioStream.getTracks().forEach((track) => track.stop()); + audioStream = null; + } + }; + + const bufferToBase64 = (buffer) => { + const bytes = new Uint8Array(buffer); + let binary = ""; + const chunkSize = 0x8000; + for (let i = 0; i < bytes.length; i += chunkSize) { + binary += String.fromCharCode.apply(null, bytes.subarray(i, i + chunkSize)); + } + return btoa(binary); + }; + + return { + initialize: () => { + ensureMarkdown(); + }, + renderMarkdown: (element, text) => { + if (!element) { + return; + } + + const md = ensureMarkdown(); + element.innerHTML = md.render(text || ""); + renderMermaid(element); + }, + scrollToBottom: (element) => { + if (!element) { + return; + } + element.scrollTop = element.scrollHeight; + }, + getSidebarTab: () => { + return window.localStorage.getItem("llama.sidebar.tab"); + }, + setSidebarTab: (value) => { + if (!value) { + return; + } + window.localStorage.setItem("llama.sidebar.tab", value); + }, + isAudioRecordingSupported: () => { + return isAudioRecordingSupported(); + }, + startAudioRecording: async () => { + if (!isAudioRecordingSupported()) { + return { started: false, error: "Audio recording is not supported in this browser." }; + } + + if (audioRecorder && audioRecorder.state === "recording") { + return { started: false, error: "Audio recording is already in progress." }; + } + + try { + audioStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const mimeType = getSupportedAudioMimeType(); + const options = mimeType ? { mimeType } : undefined; + audioRecorder = new MediaRecorder(audioStream, options); + audioChunks = []; + + audioRecorder.ondataavailable = (event) => { + if (event.data && event.data.size > 0) { + audioChunks.push(event.data); + } + }; + + audioRecorder.start(); + return { started: true, mimeType: audioRecorder.mimeType || mimeType || "" }; + } catch (err) { + cleanupAudioStream(); + return { started: false, error: err?.message || "Unable to access the microphone." }; + } + }, + stopAudioRecording: () => { + if (!audioRecorder || audioRecorder.state !== "recording") { + cleanupAudioStream(); + audioRecorder = null; + audioChunks = []; + return Promise.resolve({ base64: "", mimeType: "", size: 0 }); + } + + return new Promise((resolve) => { + audioRecorder.onstop = async () => { + const blobType = audioRecorder.mimeType || "audio/webm"; + const blob = new Blob(audioChunks, { type: blobType }); + const buffer = await blob.arrayBuffer(); + const base64 = bufferToBase64(buffer); + cleanupAudioStream(); + audioRecorder = null; + audioChunks = []; + resolve({ base64, mimeType: blob.type, size: blob.size }); + }; + + audioRecorder.stop(); + }); + } + }; +})(); diff --git a/LLama.Web/wwwroot/js/sessionConnectionChat.js b/LLama.Web/wwwroot/js/sessionConnectionChat.js index 8ecf0177a..0afd1f35b 100644 --- a/LLama.Web/wwwroot/js/sessionConnectionChat.js +++ b/LLama.Web/wwwroot/js/sessionConnectionChat.js @@ -6,13 +6,30 @@ const createConnectionSessionChat = () => { const signatureTemplate = $("#signatureTemplate").html(); let inferenceSession; + let connectionId; const connection = new signalR.HubConnectionBuilder().withUrl("/SessionConnectionHub").build(); const scrollContainer = $("#scroll-container"); const outputContainer = $("#output-container"); const chatInput = $("#input"); - - const onStatus = (connection, status) => { + const modelSelect = $("#model-select"); + const downloadList = $("#download-list"); + const downloadEmpty = $("#download-empty"); + const attachmentInput = $("#attachments"); + const attachmentList = $("#attachment-list"); + const clearAttachmentsButton = $("#clear-attachments"); + const modelsPath = $("#models-path"); + const downloadsPath = $("#downloads-path"); + const uploadsPath = $("#uploads-path"); + + const downloadState = {}; + let pendingAttachments = []; + + const renderQueue = new Map(); + let renderScheduled = false; + + const onStatus = (id, status) => { + connectionId = id; if (status == Enums.SessionConnectionStatus.Disconnected) { onError("Socket not connected") } @@ -24,7 +41,6 @@ const createConnectionSessionChat = () => { enableControls(); $("#load").hide(); $("#unload").show(); - onInfo(`New model session successfully started`) } } @@ -37,9 +53,18 @@ const createConnectionSessionChat = () => { outputContainer.append(Mustache.render(outputInfoTemplate, { text: message, date: getDateTime() })); } + const onStorageInfo = (info) => { + if (!info) + return; + + modelsPath.text(info.modelsPath || "-"); + downloadsPath.text(info.downloadsPath || "-"); + uploadsPath.text(info.uploadsPath || "-"); + } + let responseContent; let responseContainer; - let responseFirstToken; + let responseRaw = ""; const onResponse = (response) => { if (!response) @@ -50,7 +75,8 @@ const createConnectionSessionChat = () => { outputContainer.append(Mustache.render(outputBotTemplate, { uniqueId: uniqueId, ...response })); responseContainer = $(`#${uniqueId}`); responseContent = responseContainer.find(".content"); - responseFirstToken = true; + responseRaw = ""; + responseContent.text("..."); scrollToBottom(true); return; } @@ -61,34 +87,53 @@ const createConnectionSessionChat = () => { scrollToBottom(); } else { - if (responseFirstToken) { - responseContent.empty(); - responseFirstToken = false; - responseContainer.find(".date").append(getDateTime()); - responseContent.append(response.content.trim()); - } - else { - responseContent.append(response.content); + if (responseContent) { + if (responseRaw.length === 0) { + responseContainer.find(".date").append(getDateTime()); + } + responseRaw += response.content; + scheduleRender(responseContent, responseRaw); + scrollToBottom(); } - scrollToBottom(); } } const sendPrompt = async () => { const text = chatInput.val(); - if (text) { - chatInput.val(null); - disableControls(); - outputContainer.append(Mustache.render(outputUserTemplate, { text: text, date: getDateTime() })); - inferenceSession = await connection - .stream("SendPrompt", text, serializeFormToJson('SessionParameters')) - .subscribe({ - next: onResponse, - complete: onResponse, - error: onError, - }); - scrollToBottom(true); + if (!text) + return; + + if (!isSelectedModelReady()) { + onError("Selected model is still downloading."); + return; } + + chatInput.val(null); + disableControls(); + + let attachments = []; + if (pendingAttachments.length > 0) { + try { + attachments = await uploadAttachments(); + } catch (error) { + onError(error.message || "Failed to upload attachments."); + enableControls(); + return; + } + } + + outputContainer.append(Mustache.render(outputUserTemplate, { text: text, date: getDateTime(), attachments: attachments })); + renderMarkdownAsync(outputContainer.find(".entry:last .content"), text); + + const request = { prompt: text, attachmentIds: attachments.map(a => a.id) }; + inferenceSession = await connection + .stream("SendPrompt", request, serializeFormToJson('SessionParameters')) + .subscribe({ + next: onResponse, + complete: onResponse, + error: onError, + }); + scrollToBottom(true); } const cancelPrompt = async () => { @@ -97,6 +142,11 @@ const createConnectionSessionChat = () => { } const loadModel = async () => { + if (!isSelectedModelReady()) { + onError("Selected model is still downloading."); + return; + } + const sessionParams = serializeFormToJson('SessionParameters'); loaderShow(); disableControls(); @@ -114,6 +164,31 @@ const createConnectionSessionChat = () => { $("#load").removeAttr("disabled"); } + const uploadAttachments = async () => { + if (pendingAttachments.length === 0) + return []; + + if (!connectionId) + throw new Error("Connection not ready for uploads."); + + const formData = new FormData(); + formData.append("connectionId", connectionId); + pendingAttachments.forEach(file => formData.append("files", file)); + + onInfo("Uploading attachments..."); + const response = await fetch("/api/attachments", { method: "POST", body: formData }); + if (!response.ok) { + const errorText = await response.text(); + throw new Error(errorText || "Attachment upload failed."); + } + + const result = await response.json(); + pendingAttachments = []; + attachmentInput.val(""); + renderAttachmentList(); + return result.attachments || []; + } + const serializeFormToJson = (form) => { const formDataJson = {}; const formData = new FormData(document.getElementById(form)); @@ -150,6 +225,7 @@ const createConnectionSessionChat = () => { $("#unload").hide(); $(".prompt-control").removeAttr("disabled"); activatePromptTab(); + updateLoadButtonState(); } const disablePromptControls = () => { @@ -205,6 +281,254 @@ const createConnectionSessionChat = () => { $(".spinner").hide(); } + const createMarkdownRenderer = () => { + if (typeof window.markdownit === "undefined") + return null; + + const md = window.markdownit({ + html: false, + linkify: true, + breaks: true, + typographer: true + }); + + if (window.markdownitKatex) + md.use(window.markdownitKatex); + if (window.markdownitTaskLists) + md.use(window.markdownitTaskLists, { enabled: true, label: true }); + if (window.markdownitFootnote) + md.use(window.markdownitFootnote); + if (window.markdownitDeflist) + md.use(window.markdownitDeflist); + if (window.markdownitSub) + md.use(window.markdownitSub); + if (window.markdownitSup) + md.use(window.markdownitSup); + if (window.markdownitMark) + md.use(window.markdownitMark); + if (window.markdownitEmoji) + md.use(window.markdownitEmoji); + + if (window.mermaid) { + window.mermaid.initialize({ startOnLoad: false }); + } + + return md; + } + + const markdown = createMarkdownRenderer(); + + const scheduleRender = (container, raw) => { + if (!markdown) { + container.text(raw); + return; + } + + renderQueue.set(container[0], { container, raw }); + if (renderScheduled) + return; + + renderScheduled = true; + requestAnimationFrame(() => { + renderScheduled = false; + renderQueue.forEach((value) => { + renderMarkdown(value.container, value.raw); + }); + renderQueue.clear(); + }); + } + + const renderMarkdownAsync = (container, raw) => { + scheduleRender(container, raw); + } + + const renderMarkdown = (container, raw) => { + try { + container.html(markdown.render(raw || "")); + renderMermaid(container); + } catch (error) { + container.text(raw); + } + } + + const renderMermaid = (container) => { + if (!window.mermaid) + return; + + const blocks = container.find("pre code.language-mermaid"); + if (!blocks.length) + return; + + blocks.each((index, block) => { + const code = $(block); + const parent = code.parent(); + const mermaidDiv = $("
"); + mermaidDiv.text(code.text()); + parent.replaceWith(mermaidDiv); + }); + + window.mermaid.run({ nodes: container[0].querySelectorAll('.mermaid') }); + } + + const renderAttachmentList = () => { + attachmentList.empty(); + if (pendingAttachments.length === 0) + return; + + pendingAttachments.forEach(file => { + attachmentList.append(`
${file.name}
`); + }); + } + + const formatBytes = (value) => { + if (value === null || value === undefined) + return ""; + if (value === 0) + return "0 B"; + + const k = 1024; + const sizes = ["B", "KB", "MB", "GB", "TB"]; + const i = Math.floor(Math.log(value) / Math.log(k)); + return `${(value / Math.pow(k, i)).toFixed(1)} ${sizes[i]}`; + } + + const updateDownloadState = (progress) => { + if (!progress || !progress.modelName) + return; + + if (!downloadState[progress.modelName]) + downloadState[progress.modelName] = { assets: {}, ready: false }; + + const assetKey = `${progress.assetKind}-${progress.fileName}`; + downloadState[progress.modelName].assets[assetKey] = progress; + refreshDownloadUI(); + } + + const applyDownloadSnapshot = (snapshots) => { + if (!snapshots) + return; + + snapshots.forEach(snapshot => { + downloadState[snapshot.modelName] = { assets: {}, ready: snapshot.ready }; + snapshot.assets.forEach(asset => { + const assetKey = `${asset.assetKind}-${asset.fileName}`; + downloadState[snapshot.modelName].assets[assetKey] = asset; + }); + }); + + refreshDownloadUI(); + } + + const refreshDownloadUI = () => { + downloadList.empty(); + let anyActive = false; + + Object.keys(downloadState).forEach(modelName => { + const modelEntry = downloadState[modelName]; + const assets = Object.values(modelEntry.assets); + if (assets.length === 0) + return; + + const ready = assets.every(asset => asset.state === 2); + modelEntry.ready = ready; + + const totalBytes = assets.reduce((sum, asset) => sum + (asset.totalBytes || 0), 0); + const receivedBytes = assets.reduce((sum, asset) => sum + (asset.bytesReceived || 0), 0); + const hasUnknownTotal = assets.some(asset => asset.totalBytes === null || asset.totalBytes === undefined); + const hasFailure = assets.some(asset => asset.state === 3 || asset.state === 4); + const isDownloading = assets.some(asset => asset.state === 1); + + const statusText = ready + ? "Ready" + : hasFailure + ? "Error" + : isDownloading + ? "Downloading" + : "Queued"; + + const progressPercent = totalBytes > 0 ? Math.min(100, Math.round((receivedBytes / totalBytes) * 100)) : 0; + const remainingText = hasUnknownTotal + ? "Calculating size..." + : `${formatBytes(Math.max(0, totalBytes - receivedBytes))} remaining`; + + if (!ready) + anyActive = true; + + const assetRows = assets.map(asset => { + const assetTotal = asset.totalBytes || 0; + const assetPercent = assetTotal > 0 ? Math.min(100, Math.round((asset.bytesReceived / assetTotal) * 100)) : 0; + const assetStatus = asset.state === 2 + ? "Ready" + : asset.state === 3 + ? "Failed" + : asset.state === 4 + ? "Missing URL" + : asset.state === 1 + ? "Downloading" + : "Queued"; + const assetMeta = assetTotal > 0 + ? `${formatBytes(asset.bytesReceived)} / ${formatBytes(assetTotal)}` + : asset.bytesReceived > 0 + ? formatBytes(asset.bytesReceived) + : "-"; + + return `
+
+
${asset.fileName}
+
${assetStatus}
+
+
+
+
+
${assetMeta}
+
`; + }).join(""); + + const item = $( + `
+
+
${modelName}
+
${statusText}
+
+
+
+
+
${remainingText}
+
${assetRows}
+
` + ); + + downloadList.append(item); + updateModelOptionState(modelName, ready); + }); + + downloadEmpty.toggle(!anyActive); + updateLoadButtonState(); + } + + const updateModelOptionState = (modelName, ready) => { + const option = modelSelect.find(`option[value='${modelName}']`); + option.prop("disabled", !ready); + } + + const isSelectedModelReady = () => { + const selected = modelSelect.val(); + if (!selected) + return false; + + const entry = downloadState[selected]; + return entry ? entry.ready : false; + } + + const updateLoadButtonState = () => { + const ready = isSelectedModelReady(); + if (ready) { + $("#load").removeAttr("disabled"); + } else { + $("#load").attr("disabled", "disabled"); + } + } + // Map UI functions $("#load").on("click", loadModel); $("#unload").on("click", unloadModel); @@ -212,6 +536,16 @@ const createConnectionSessionChat = () => { $("#clear").on("click", clearOutput); $("#cancel").on("click", cancelPrompt); $("#Prompt").on("change", updatePrompt); + modelSelect.on("change", updateLoadButtonState); + attachmentInput.on("change", () => { + pendingAttachments = Array.from(attachmentInput[0].files || []); + renderAttachmentList(); + }); + clearAttachmentsButton.on("click", () => { + pendingAttachments = []; + attachmentInput.val(""); + renderAttachmentList(); + }); chatInput.on('keydown', function (event) { if (event.key === 'Enter' && !event.shiftKey) { event.preventDefault(); @@ -228,5 +562,13 @@ const createConnectionSessionChat = () => { connection.on("OnStatus", onStatus); connection.on("OnError", onError); connection.on("OnResponse", onResponse); - connection.start(); -} \ No newline at end of file + connection.on("OnModelDownloadSnapshot", applyDownloadSnapshot); + connection.on("OnModelDownloadProgress", updateDownloadState); + connection.on("OnStorageInfo", onStorageInfo); + connection.start().then(() => { + connection.invoke("GetModelStatuses").then(applyDownloadSnapshot); + setInterval(() => { + connection.invoke("GetModelStatuses").then(applyDownloadSnapshot); + }, 2000); + }); +}