Regular expressions to limit the indexed pages. Implements #5

Both regular expressions (when set) need to apply at the same time. Eg a
page MUST match the matchRegex and MUST NOT match the skipRegex to be
applied.

The regular expressions are applied when running the `embed` command
line command. Pages no longer adhering to a changed regex setup will be
removed from the vector store.

For the sqlite storage it is recommended to re-cluster the index when
the reges are changed by running the `maintenance` command.

Author: splitbrain

Embeddings.php

@@ -77,11 +77,12 @@ public function getTokenEncoder()
      * Update the embeddings storage
      *
      * @param string $skipRE Regular expression to filter out pages (full RE with delimiters)
+     * @param string $matchRE Regular expression pages have to match to be included (full RE with delimiters)
      * @param bool $clear Should any existing storage be cleared before updating?
      * @return void
      * @throws \Exception
      */
-    public function createNewIndex($skipRE = '', $clear = false)
+    public function createNewIndex($skipRE = '', $matchRE = '', $clear = false)
     {
         $indexer = new Indexer();
         $pages = $indexer->getPages();
@@ -94,7 +95,8 @@ public function createNewIndex($skipRE = '', $clear = false)
                 !page_exists($page) ||
                 isHiddenPage($page) ||
                 filesize(wikiFN($page)) < 150 || // skip very small pages
-                ($skipRE && preg_match($skipRE, (string) $page))
+                ($skipRE && preg_match($skipRE, (string) $page)) ||
+                ($matchRE && !preg_match($matchRE, ":$page"))
             ) {
                 // this page should not be in the index (anymore)
                 $this->storage->deletePageChunks($page, $chunkID);

cli.php

@@ -265,12 +265,13 @@ protected function runMaintenance()
      * Recreate chunks and embeddings for all pages
      *
      * @return void
-     * @todo make skip regex configurable
      */
     protected function createEmbeddings($clear)
     {
+        [$skipRE, $matchRE] = $this->getRegexps();
+
         $start = time();
-        $this->helper->getEmbeddings()->createNewIndex('/(^|:)(playground|sandbox)(:|$)/', $clear);
+        $this->helper->getEmbeddings()->createNewIndex($skipRE, $matchRE, $clear);
         $this->notice('Peak memory used: {memory}', ['memory' => filesize_h(memory_get_peak_usage(true))]);
         $this->notice('Spent time: {time}min', ['time' => round((time() - $start) / 60, 2)]);
     }
@@ -354,4 +355,42 @@ protected function readLine($prompt)
 
         return $value;
     }
+
+    /**
+     * Read the skip and match regex from the config
+     *
+     * Ensures the regular expressions are valid
+     *
+     * @return string[] [$skipRE, $matchRE]
+     */
+    protected function getRegexps()
+    {
+        $skip = $this->getConf('skipRegex');
+        $skipRE = '';
+        $match = $this->getConf('matchRegex');
+        $matchRE = '';
+
+        if ($skip) {
+            $skipRE = '/' . $skip . '/';
+            if (@preg_match($skipRE, null) === false) {
+                $this->error(preg_last_error_msg());
+                $this->error('Invalid regular expression in $conf[\'skipRegex\']. Ignored.');
+                $skipRE = '';
+            } else {
+                $this->success('Skipping pages matching ' . $skipRE);
+            }
+        }
+
+        if ($match) {
+            $matchRE = '/' . $match . '/';
+            if (@preg_match($matchRE, null) === false) {
+                $this->error(preg_last_error_msg());
+                $this->error('Invalid regular expression in $conf[\'matchRegex\']. Ignored.');
+                $matchRE = '';
+            } else {
+                $this->success('Only indexing pages matching ' . $matchRE);
+            }
+        }
+        return [$skipRE, $matchRE];
+    }
 }

conf/default.php

@@ -25,4 +25,6 @@
 
 $conf['logging'] = 0;
 $conf['restrict'] = '';
+$conf['skipRegex'] = ':(playground|sandbox)(:|$)';
+$conf['matchRegex'] = '';
 $conf['preferUIlanguage'] = 0;

conf/metadata.php

@@ -32,6 +32,8 @@
 
 $meta['logging'] = array('onoff');
 $meta['restrict'] = array('string');
+$meta['skipRegex'] = array('string');
+$meta['matchRegex'] = array('string');
 $meta['preferUIlanguage'] = array('multichoice', '_choices' => array(
     \dokuwiki\plugin\aichat\AIChat::LANG_AUTO_ALL,
     \dokuwiki\plugin\aichat\AIChat::LANG_UI_ALL,

lang/en/settings.php

@@ -24,6 +24,8 @@
 
 $lang['logging'] = 'Log all questions and answers. Use the <a href="?do=admin&page=logviewer&facility=aichat">Log Viewer</a> to access.';
 $lang['restrict'] = 'Restrict access to these users and groups (comma separated). Leave empty to allow all users.';
+$lang['skipRegex'] = 'Skip indexing pages matching this regular expression (no delimiters).';
+$lang['matchRegex'] = 'Only index pages matching this regular expression (no delimiters).';
 $lang['preferUIlanguage'] = 'How to work with multilingual wikis? (Requires the translation plugin)';
 
 $lang['preferUIlanguage_o_0'] = 'Guess language, use all sources';