fix: consider full symbol as index

Author: rafaeldelboni

.clj-kondo/config.edn

@@ -0,0 +1 @@
+{:lint-as {datalevin.interpret/inter-fn clojure.core/fn}}

.lsp/config.edn

@@ -5,4 +5,5 @@
                     flow                    [[:block 1]]
                     flow-with-defaults      [[:block 1]]
                     flow-as-of              [[:block 1]]
-                    flow-without-validation [[:block 1]]}}}
+                    flow-without-validation [[:block 1]]
+                    inter-fn                [[:inner 0] [:inner 1]]}}}

dev/playground.clj

@@ -58,7 +58,7 @@
         db (d/db conn)
 
         datoms (->> (d/fulltext-datoms db
-                                       "ass"
+                                       "->"
                                        {:top 30
                                         :domains ["definition-name"
                                                   "namespace-name"
@@ -143,15 +143,19 @@
 
   ; tests with fulltext and analyzer
   (let [query-analyzer (su/create-analyzer
-                        {:tokenizer (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+")
+                        {:tokenizer (datalevin/merge-tokenizers
+                                     (su/create-regexp-tokenizer #"[.]+")
+                                     (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+"))
                          :token-filters [su/lower-case-token-filter]})
 
         analyzer (su/create-analyzer
-                  {:tokenizer (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+")
+                  {:tokenizer (datalevin/merge-tokenizers
+                               (su/create-regexp-tokenizer #"[.]+")
+                               (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+"))
                    :token-filters [su/lower-case-token-filter
                                    su/prefix-token-filter]})
 
-        dir  "/tmp/mydb"
+        dir  (str "/tmp/mydb-" (random-uuid))
         conn (d/create-conn dir
                             {:text {:db/valueType :db.type/string
                                     :db/fulltext  true
@@ -169,35 +173,45 @@
               {:text "associative?"}
               {:text "b"}
               {:text "ba"}
-              {:text "bas"}]
+              {:text "bas"}
+              {:text "*"}
+              {:text "/"}
+              {:text "->"}
+              {:text "->>"}
+              {:text "as->"}
+              {:text "as->banana"}]
 
         _transact (d/transact! conn data)
 
-        result (->> (d/q '[:find ?i
+        result (->> (d/q '[:find ?e ?v
                            :in $ ?q
                            :where
-                           [(fulltext $ ?q {:top 20}) [[?e]]]
-                           [?e :text ?i]]
+                           [(fulltext $ ?q {:top 20}) [[?e ?a ?v]]]]
                          (d/db conn)
-                         "assoc-me")
+                         "as->")
                     doall)]
 
     (d/close conn)
     (util/delete-files dir)
 
     result)
 
-; tests with fulltext and analyzer on a raw query
+  ; tests with fulltext and analyzer on a raw query
   (let [query-analyzer (su/create-analyzer
-                        {:tokenizer (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+")
+                        {:tokenizer (datalevin/merge-tokenizers
+                                     (su/create-regexp-tokenizer #"[.*]+")
+                                     (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+"))
                          :token-filters [su/lower-case-token-filter]})
 
         analyzer (su/create-analyzer
-                  {:tokenizer (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+")
+                  {:tokenizer (datalevin/merge-tokenizers
+                               (su/create-regexp-tokenizer #"[.*]+")
+                               (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+"))
                    :token-filters [su/lower-case-token-filter
                                    su/prefix-token-filter]})
 
-        lmdb (d/open-kv "/tmp/mydb")
+        dir  (str "/tmp/lmdb-" (random-uuid))
+        lmdb (d/open-kv dir)
 
         engine (d/new-search-engine lmdb {:query-analyzer query-analyzer
                                           :analyzer analyzer
@@ -213,13 +227,18 @@
                7 "associative?"
                8 "b"
                9 "ba"
-               10 "bas"}
+               10 "bas"
+               11 "->"
+               12 "->>"
+               13 "as->"
+               14 "as->banana"}
 
         _transact (doseq [[k v] input]
                     (d/add-doc engine k v))
 
-        result (doall (d/search engine "assoc-m" {:top 20 :display :texts}))]
+        result (doall (d/search engine "as->" {:top 20 :display :texts}))]
 
     (d/close-kv lmdb)
+    (util/delete-files dir)
 
     result))

src/codes/clj/docs/extractor/datalevin.clj

@@ -1,5 +1,6 @@
 (ns codes.clj.docs.extractor.datalevin
   (:require [datalevin.core :as d]
+            [datalevin.interpret :refer [inter-fn]]
             [datalevin.search-utils :as su]))
 
 ;; TODO: add id :db.unique/identity and ref :db.type/ref
@@ -78,12 +79,24 @@
 (def db-schemas
   (merge project-schema namespace-schema definition-schema))
 
+(defn merge-tokenizers
+  "Merges the results of tokenizer a and b into one sequence."
+  [tokenizer-a tokenizer-b]
+  (inter-fn [^String s]
+    (into (sequence (tokenizer-a s))
+      (sequence (tokenizer-b s)))))
+
 (defn bulk-transact! [datoms config]
   (let [query-analyzer (su/create-analyzer
-                        {:tokenizer (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+")
+                        {:tokenizer (merge-tokenizers
+                                     (su/create-regexp-tokenizer #"[.*]+")
+                                     (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+"))
                          :token-filters [su/lower-case-token-filter]})
+
         analyzer (su/create-analyzer
-                  {:tokenizer (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+")
+                  {:tokenizer (merge-tokenizers
+                               (su/create-regexp-tokenizer #"[.*]+")
+                               (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+"))
                    :token-filters [su/lower-case-token-filter
                                    su/prefix-token-filter]})
         conn (-> config :db :dir