diff --git a/Samples/AppContentSearch/README.md b/Samples/AppContentSearch/README.md index ba7edfef1..73a132aab 100644 --- a/Samples/AppContentSearch/README.md +++ b/Samples/AppContentSearch/README.md @@ -16,9 +16,17 @@ extendedZipContent: # AppContentSearch Sample Application -This sample demonstrates how to use App Content Search's **AppContentIndex APIs** in a **WinUI3** notes application. It shows how to create, manage, and semantically search through the index that includes both text content and images. It also shows how to use use the search results to enable retrieval augmented genaration (RAG) scenarios with language models. +This sample demonstrates how to use App Content Search's +**AppContentIndex APIs** in a **WinUI3** notes application. +It shows how to create, manage, and semantically search +through the index that includes both text content and images. +It also shows how to use the search results to enable +retrieval augmented generation (RAG) scenarios with language +models. -> **Note**: This sample is targeted and tested for **Windows App SDK 2.0 Experimental2** and **Visual Studio 2022**. The AppContentSearch APIs are experimental and available in Windows App SDK 2.0 experimental2. +> **Note**: This sample is targeted and tested for +> **Windows App SDK 2.0 Preview1** and +> **Visual Studio 2022**. ## Features @@ -42,7 +50,7 @@ This sample demonstrates: ## Building and Running the Sample -* Open the solution file (`AppContentSearch.sln`) in Visual Studio. +* Open the solution file (`NotesApp.sln`) in Visual Studio. * Press Ctrl+Shift+B, or select **Build** \> **Build Solution**. * Run the application to see the Notes app with integrated search functionality. diff --git a/Samples/AppContentSearch/cs-winui/AI/Provider/FoundryAIProvider.cs b/Samples/AppContentSearch/cs-winui/AI/Provider/FoundryAIProvider.cs index f3a449905..bd3949cb3 100644 --- a/Samples/AppContentSearch/cs-winui/AI/Provider/FoundryAIProvider.cs +++ b/Samples/AppContentSearch/cs-winui/AI/Provider/FoundryAIProvider.cs @@ -1,10 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -using Microsoft.AI.Foundry.Local; -using Microsoft.Extensions.AI; -using Notes.ViewModels; -using OpenAI; -using OpenAI.Chat; using System; using System.ClientModel; using System.Collections.Generic; @@ -13,6 +8,11 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.AI.Foundry.Local; +using Microsoft.Extensions.AI; +using Notes.ViewModels; +using OpenAI; +using OpenAI.Chat; using Windows.Storage; using AppChatRole = Notes.ViewModels.ChatRole; using ExtChatRole = Microsoft.Extensions.AI.ChatRole; @@ -143,7 +143,8 @@ public async IAsyncEnumerable SendStreamingRequestAsync( break; } - if (!hasNext) break; + if (!hasNext) + break; if (cancellationToken.IsCancellationRequested) { @@ -175,7 +176,7 @@ private bool HasCachedModels() { try { - var foundryManager = new FoundryLocalManager(); + using var foundryManager = new FoundryLocalManager(); var cached = foundryManager.ListCachedModelsAsync().GetAwaiter().GetResult(); return cached.Count > 0; } @@ -187,7 +188,8 @@ private bool HasCachedModels() private async Task EnsureFoundryClientAsync(CancellationToken ct) { - if (_foundryClient != null && _foundryChatClient != null) return; + if (_foundryClient != null && _foundryChatClient != null) + return; var settings = ApplicationData.Current.LocalSettings; string alias = (settings.Values[FoundryModelKey] as string)?.Trim() ?? ""; @@ -195,7 +197,7 @@ private async Task EnsureFoundryClientAsync(CancellationToken ct) { try { - var foundryManager = new FoundryLocalManager(); + using var foundryManager = new FoundryLocalManager(); var cached = await foundryManager.ListCachedModelsAsync(); if (cached.Count != 0) @@ -269,4 +271,4 @@ private ChatResponseUpdate Append(SessionEntryViewModel entry, string text, Chat return list; } -} \ No newline at end of file +} diff --git a/Samples/AppContentSearch/cs-winui/Controls/SearchView.xaml b/Samples/AppContentSearch/cs-winui/Controls/SearchView.xaml index 567630d56..a774d94dd 100644 --- a/Samples/AppContentSearch/cs-winui/Controls/SearchView.xaml +++ b/Samples/AppContentSearch/cs-winui/Controls/SearchView.xaml @@ -100,6 +100,20 @@ + + + + OCR Text Results + + + + @@ -127,7 +141,8 @@ Height="32" Width="400" PlaceholderText="Search for anything..." - QuerySubmitted="SearchBox_QuerySubmitted"> + QuerySubmitted="SearchBox_QuerySubmitted" + TextChanged="SearchBox_TextChanged"> ViewModel.Dispose(); } private void HideResultsPanel() @@ -31,7 +32,6 @@ private void HideResultsPanel() private async void ResultsItemsView_ItemInvoked(ItemsView sender, ItemsViewItemInvokedEventArgs e) { - var context = await AppDataContext.GetCurrentAsync(); var item = e.InvokedItem as SearchResult; if (MainWindow.Instance != null && item != null) @@ -40,8 +40,24 @@ private async void ResultsItemsView_ItemInvoked(ItemsView sender, ItemsViewItemI { await MainWindow.Instance.SelectNoteById(item.SourceId); } + else if (item.ContentType == ContentType.OcrText) + { + if (item.AttachmentId is int attachmentId) + { + await MainWindow.Instance.SelectNoteById( + item.SourceId, + attachmentId, + item.MostRelevantSentence, + item.BoundingBox); + } + else + { + await MainWindow.Instance.SelectNoteById(item.SourceId); + } + } else { + var context = await AppDataContext.GetCurrentAsync(); var attachment = context.Attachments.Where(a => a.Id == item.SourceId).FirstOrDefault(); if (attachment != null) @@ -138,6 +154,23 @@ private void SearchBox_QuerySubmitted(AutoSuggestBox sender, AutoSuggestBoxQuery ViewModel.HandleQuerySubmitted(sender.Text); } + private void SearchBox_TextChanged(AutoSuggestBox sender, AutoSuggestBoxTextChangedEventArgs args) + { + // Only react to user input, not programmatic changes + if (args.Reason == AutoSuggestionBoxTextChangeReason.UserInput) + { + ViewModel.HandleTextChanged(sender.Text); + } + } + + public void InitializeQuerySessions() + { + if (MainWindow.AppContentIndexer != null) + { + ViewModel.InitializeQuerySessions(MainWindow.AppContentIndexer, DispatcherQueue); + } + } + private void ItemContainer_BringIntoViewRequested(UIElement sender, BringIntoViewRequestedEventArgs args) { // When the popup is being hidden we get a spurious BringIntoView request which the Repeater doesn't handle well. diff --git a/Samples/AppContentSearch/cs-winui/MainWindow.xaml.cs b/Samples/AppContentSearch/cs-winui/MainWindow.xaml.cs index 14dce1859..e47bacb99 100644 --- a/Samples/AppContentSearch/cs-winui/MainWindow.xaml.cs +++ b/Samples/AppContentSearch/cs-winui/MainWindow.xaml.cs @@ -2,7 +2,7 @@ using Microsoft.UI.Xaml; using Microsoft.UI.Xaml.Controls; -using Microsoft.Windows.AI.Search.Experimental.AppContentIndex; +using Microsoft.Windows.Search.AppContentIndex; using Notes.Controls; using Notes.Pages; using Notes.ViewModels; @@ -11,6 +11,7 @@ using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using Windows.Foundation.Collections; namespace Notes { @@ -20,11 +21,13 @@ public sealed partial class MainWindow : Window private static SearchView? _searchView; private static MainWindow? _instance; private static AppContentIndexer? _appContentIndexer; + private static IndexCapabilitiesOfCurrentSystem? _systemCapabilities; public static ChatSessionView? ChatSessionView => _chatSessionView; public static SearchView? SearchView => _searchView; public static MainWindow? Instance => _instance; public static AppContentIndexer? AppContentIndexer => _appContentIndexer; + public static IndexCapabilitiesOfCurrentSystem? SystemCapabilities => _systemCapabilities; public ViewModel VM; @@ -47,6 +50,12 @@ public MainWindow() VM.Notes.CollectionChanged += Notes_CollectionChanged; + this.Closed += (_, _) => + { + _appContentIndexer?.Dispose(); + _appContentIndexer = null; + }; + _initializeAppContentIndexerTask = InitializeAppContentIndexerAsync(); DispatcherQueue.TryEnqueue(async () => @@ -93,15 +102,123 @@ public async Task SelectNoteById(int id, int? attachmentId = null, string? attac } } + private static void CheckSystemCapabilities() + { + _systemCapabilities = AppContentIndexer.GetIndexCapabilitiesOfCurrentSystem(); + + var capabilities = new[] + { + IndexCapability.TextLexical, + IndexCapability.TextSemantic, + IndexCapability.ImageOcr, + IndexCapability.ImageSemantic + }; + + foreach (var capability in capabilities) + { + var status = _systemCapabilities.GetIndexCapabilityStatus(capability); + Debug.WriteLine($"System capability {capability}: {status}"); + + if (status == IndexCapabilityOfCurrentSystemStatus.DisabledByPolicy) + { + Debug.WriteLine($"Warning: {capability} is disabled by policy."); + } + else if (status == IndexCapabilityOfCurrentSystemStatus.NotSupported) + { + Debug.WriteLine($"Warning: {capability} is not supported on this system."); + } + } + } + + private void SubscribeToIndexerListener() + { + if (_appContentIndexer is null) + { + return; + } + + var listener = _appContentIndexer.Listener; + + listener.IndexCapabilitiesChanged += (sender, capabilities) => + { + if (capabilities.HasCapabilitiesWithErrors) + { + var errors = capabilities.GetCapabilitiesWithErrors(); + foreach (var cap in errors) + { + var state = capabilities.GetCapabilityState(cap); + Debug.WriteLine($"Index capability error — {cap}: {state.InitializationStatus}, {state.ErrorMessage}"); + } + } + else + { + Debug.WriteLine("All index capabilities are healthy."); + } + }; + + listener.IndexStatisticsChanged += (sender, stats) => + { + Debug.WriteLine($"Index statistics changed — Items: {stats.ItemCount}, " + + $"Completed: {stats.CompletedCount}, InProgress: {stats.InProgressCount}, " + + $"NotStarted: {stats.NotStartedCount}, Errors: {stats.ErrorsCount}, " + + $"PendingDeletion: {stats.PendingDeletionCount}, " + + $"RequiringReindexing: {stats.RequiringReindexingCount}"); + + DispatcherQueue.TryEnqueue(() => + { + if (stats.IndexingInProgress) + { + int total = stats.ItemCount; + int completed = stats.CompletedCount; + if (total > 0) + { + double percent = (double)completed / total * 100; + _searchView?.SetIndexProgressBar(percent); + } + } + else + { + _searchView?.SetSearchBoxIndexingCompleted(); + } + }); + }; + + listener.ContentItemStatusChanged += (sender, statusMap) => + { + foreach (var kvp in statusMap) + { + Debug.WriteLine($"Content item '{kvp.Key}' status: {kvp.Value.Status}, Error: {kvp.Value.ErrorDetail}"); + } + }; + } + + private static void LogIndexStatistics() + { + if (_appContentIndexer is null) + { + return; + } + + var stats = _appContentIndexer.GetIndexStatistics(); + Debug.WriteLine($"Index statistics — Items: {stats.ItemCount}, " + + $"Completed: {stats.CompletedCount}, InProgress: {stats.InProgressCount}, " + + $"NotStarted: {stats.NotStartedCount}, Errors: {stats.ErrorsCount}, " + + $"PendingDeletion: {stats.PendingDeletionCount}, " + + $"RequiringReindexing: {stats.RequiringReindexingCount}"); + } + private async Task InitializeAppContentIndexerAsync() { GetOrCreateIndexResult? getOrCreateResult = null; await Task.Run(() => { - getOrCreateResult = Microsoft.Windows.AI.Search.Experimental.AppContentIndex.AppContentIndexer.GetOrCreateIndex("NotesIndex"); + // Pre-flight: check system-level ACI capabilities before creating an index. + CheckSystemCapabilities(); + + getOrCreateResult = AppContentIndexer.GetOrCreateIndex("NotesIndex"); if (getOrCreateResult == null) { - throw new Exception("GetOrCreateIndexResult is null"); + throw new InvalidOperationException("GetOrCreateIndexResult is null"); } if (!getOrCreateResult.Succeeded) { @@ -109,13 +226,21 @@ await Task.Run(() => } _appContentIndexer = getOrCreateResult.Indexer; + + LogIndexStatistics(); }); + // Subscribe to live status updates from the indexer. + SubscribeToIndexerListener(); + DispatcherQueue.TryEnqueue(() => { ChatPaneToggleButton.IsEnabled = true; AppSearchView.SetSearchBoxInitializingCompleted(); + // Initialize query sessions for search-as-you-type. + AppSearchView.InitializeQuerySessions(); + var status = getOrCreateResult?.Status; switch (status) diff --git a/Samples/AppContentSearch/cs-winui/Notes.csproj b/Samples/AppContentSearch/cs-winui/Notes.csproj index d43c76f47..94cc38571 100644 --- a/Samples/AppContentSearch/cs-winui/Notes.csproj +++ b/Samples/AppContentSearch/cs-winui/Notes.csproj @@ -29,11 +29,6 @@ - - @@ -41,10 +36,6 @@ - - @@ -70,13 +61,12 @@ - - + @@ -119,30 +109,6 @@ MSBuild:Compile - diff --git a/Samples/AppContentSearch/cs-winui/Package.appxmanifest b/Samples/AppContentSearch/cs-winui/Package.appxmanifest index c740d6551..32dd743b4 100644 --- a/Samples/AppContentSearch/cs-winui/Package.appxmanifest +++ b/Samples/AppContentSearch/cs-winui/Package.appxmanifest @@ -5,19 +5,17 @@ xmlns:mp="http://schemas.microsoft.com/appx/2014/phone/manifest" xmlns:uap="http://schemas.microsoft.com/appx/manifest/uap/windows10" xmlns:rescap="http://schemas.microsoft.com/appx/manifest/foundation/windows10/restrictedcapabilities" - xmlns:systemai="http://schemas.microsoft.com/appx/manifest/systemai/windows10" - IgnorableNamespaces="uap rescap systemai"> - + IgnorableNamespaces="uap rescap systemai" + xmlns:systemai="http://schemas.microsoft.com/appx/manifest/systemai/windows10"> + - - + Name="ai-powered-notes-winui3-sample" + Publisher="CN=AppContentSearchSample" + Version="1.0.0.0" /> Notes - nikol + AppContentSearchSample Assets\StoreLogo.png diff --git a/Samples/AppContentSearch/cs-winui/Pages/SettingsPage.xaml b/Samples/AppContentSearch/cs-winui/Pages/SettingsPage.xaml index 073d35a48..b543d5746 100644 --- a/Samples/AppContentSearch/cs-winui/Pages/SettingsPage.xaml +++ b/Samples/AppContentSearch/cs-winui/Pages/SettingsPage.xaml @@ -2,8 +2,7 @@ x:Class="Notes.Pages.SettingsPage" xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:local="using:Notes.Pages" - xmlns:toolkit="using:CommunityToolkit.WinUI.Controls"> + xmlns:local="using:Notes.Pages"> 4 @@ -29,27 +28,35 @@ - - - - - - + + + + + + + - - - + + - + - - + + @@ -84,24 +91,28 @@ - - - + + + - - + + + + + OnContent="On"/> - +