Merge pull request #13 from clj-codes/feat/new-fulltext-index-n-extra-fields

feat: new fulltext index and extra fields

Author: rafaeldelboni

deps.edn

@@ -1,10 +1,10 @@
 {:paths ["src" "resources"]
  :deps {org.clojure/clojure {:mvn/version "1.11.1"}
         org.clojure/tools.deps {:mvn/version "0.18.1398"}
-        clj-kondo/clj-kondo {:mvn/version "2023.12.15"}
-        datalevin/datalevin {:mvn/version "0.8.25"}
+        clj-kondo/clj-kondo {:mvn/version "2024.02.12"}
+        datalevin/datalevin {:mvn/version "0.8.26"}
         com.cognitect/transit-clj {:mvn/version "1.0.333"}
-        org.slf4j/slf4j-nop {:mvn/version "2.0.11"}}
+        org.slf4j/slf4j-nop {:mvn/version "2.0.12"}}
 
  :aliases
  {:extract {:ns-default codes.clj.docs.extractor.core
@@ -22,7 +22,7 @@
                       lambdaisland/kaocha           {:mvn/version "1.87.1366"}
                       lambdaisland/kaocha-cloverage {:mvn/version "1.1.89"}
                       nubank/mockfn                 {:mvn/version "0.7.0"}
-                      nubank/matcher-combinators    {:mvn/version "3.8.8"}}
+                      nubank/matcher-combinators    {:mvn/version "3.9.1"}}
         ; needed for datalevin on jvm > 11
         :jvm-opts ["--add-opens=java.base/java.nio=ALL-UNNAMED"
                    "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"]}

dev/playground.clj

@@ -54,12 +54,24 @@
         result (doall (d/q '[:find (pull ?e [* {:namespace/project [*]}]) ?a ?v
                              :in $ ?q
                              :where
-                             [(fulltext $ ?q) [[?e ?a ?v]]]]
+                             [(fulltext $ ?q {:domains ["definition"]}) [[?e ?a ?v]]]]
                            db
                            "assoc"))]
     (d/close conn)
     result)
 
+  ; simple query definition by name
+  (let [conn (d/get-conn "target/docs-db" datalevin/db-schemas)
+        db (d/db conn)
+        result (doall (d/q '[:find (pull ?e [* {:definition/namespace [* {:namespace/project [*]}]}])
+                             :in $ ?q
+                             :where
+                             [?e :definition/name ?q]]
+                           db
+                           "file-position"))]
+    (d/close conn)
+    result)
+
   ; regex searching
   (let [conn (d/get-conn "target/docs-db" datalevin/db-schemas)
         db (d/db conn)

src/codes/clj/docs/extractor/adapters.clj

@@ -77,8 +77,8 @@
                  [group artifact] (-> project :project-name (str/split #"/"))]
              (->> namespaces
                   group-multi-langs
-                  (mapv (fn [{:keys [end-row meta name-end-col name-end-row name-row added
-                                     name author filename col name-col end-col doc row]}]
+                  (mapv (fn [{:keys [end-row meta name-end-col name-end-row name-row deprecated added
+                                     name author filename col name-col end-col doc no-doc row]}]
                           (let [trim-filename (str/replace filename root "")]
                             (assoc-some
                              {:namespace/id (str/join "/" [group artifact name])
@@ -91,6 +91,7 @@
                              :namespace/name-end-col name-end-col
                              :namespace/name-end-row name-end-row
                              :namespace/name-row name-row
+                             :namespace/deprecated deprecated
                              :namespace/added added
                              :namespace/author author
                              :namespace/filename trim-filename
@@ -99,6 +100,7 @@
                              :namespace/name-col name-col
                              :namespace/end-col end-col
                              :namespace/doc doc
+                             :namespace/no-doc (some-> no-doc boolean)
                              :namespace/row row))))))))
    []
    analysis))
@@ -117,7 +119,7 @@
                   (->> definitions
                        group-multi-langs
                        (mapv (fn [{:keys [fixed-arities end-row meta name-end-col
-                                          name-end-row name-row added ns name author
+                                          name-end-row name-row added deprecated ns name author
                                           defined-by filename macro col name-col end-col
                                           arglist-strs varargs-min-arity doc row
                                           private protocol-ns protocol-name]}]
@@ -138,6 +140,7 @@
                                   :definition/name-end-row name-end-row
                                   :definition/name-row name-row
                                   :definition/added added
+                                  :definition/deprecated deprecated
                                   :definition/author author
                                   :definition/filename trim-filename
                                   :definition/git-source (when trim-filename

src/codes/clj/docs/extractor/datalevin.clj

@@ -7,8 +7,14 @@
   {:project/id       {:db/valueType :db.type/string
                       :unique :db.unique/identity}
    :project/name     {:db/valueType :db.type/string}
-   :project/group    {:db/valueType :db.type/string}
-   :project/artifact {:db/valueType :db.type/string}
+   :project/group    {:db/valueType :db.type/string
+                      :db/fulltext  true
+                      :db.fulltext/domains ["project"
+                                            "project-group"]}
+   :project/artifact {:db/valueType :db.type/string
+                      :db/fulltext  true
+                      :db.fulltext/domains ["project"
+                                            "project-name"]}
    :project/paths    {:db/valueType :db.type/string
                       :db/cardinality :db.cardinality/many}
    :project/url      {:db/valueType :db.type/string}
@@ -19,33 +25,47 @@
 (def namespace-schema
   {:namespace/id           {:db/valueType :db.type/string
                             :unique :db.unique/identity}
-   :namespace/name         {:db/valueType :db.type/string}
+   :namespace/name         {:db/valueType :db.type/string
+                            :db/fulltext  true
+                            :db.fulltext/domains ["namespace"
+                                                  "namespace-name"]}
    :namespace/project      {:db/valueType :db.type/ref}
    :namespace/group        {:db/valueType :db.type/string}
    :namespace/artifact     {:db/valueType :db.type/string}
+   :namespace/no-doc       {:db/valueType :db.type/boolean}
    :namespace/doc          {:db/valueType :db.type/string
-                            :db/fulltext  true}
+                            :db/fulltext  true
+                            :db.fulltext/autoDomain true
+                            :db.fulltext/domains ["namespace"
+                                                  "namespace-doc"]}
    :namespace/author       {:db/valueType :db.type/string}
    :namespace/filename     {:db/valueType :db.type/string}
    :namespace/git-source   {:db/valueType :db.type/string}
+   :namespace/deprecated   {:db/valueType :db.type/string}
    :namespace/added        {:db/valueType :db.type/string}
    :namespace/row          {:db/valueType :db.type/long}
    :namespace/col          {:db/valueType :db.type/long}})
 
 (def definition-schema
   {:definition/id                {:db/valueType :db.type/string
                                   :unique :db.unique/identity}
-   :definition/name              {:db/valueType :db.type/string}
+   :definition/name              {:db/valueType :db.type/string
+                                  :db/fulltext  true
+                                  :db.fulltext/domains ["definition"
+                                                        "definition-name"]}
    :definition/namespace         {:db/valueType :db.type/ref}
    :definition/group             {:db/valueType :db.type/string}
    :definition/artifact          {:db/valueType :db.type/string}
    :definition/doc               {:db/valueType :db.type/string
-                                  :db/fulltext  true}
+                                  :db/fulltext  true
+                                  :db.fulltext/domains ["definition"
+                                                        "definition-doc"]}
    :definition/filename          {:db/valueType :db.type/string}
    :definition/git-source        {:db/valueType :db.type/string}
    :definition/arglist-strs      {:db/valueType :db.type/string
                                   :db/cardinality :db.cardinality/many}
    :definition/varargs-min-arity {:db/valueType :db.type/long}
+   :definition/deprecated        {:db/valueType :db.type/string}
    :definition/added             {:db/valueType :db.type/string}
    :definition/macro             {:db/valueType :db.type/boolean}
    :definition/private           {:db/valueType :db.type/boolean}