Merge pull request #1 from clj-codes/feat/extract-and-adapt-data

rafaeldelboni · web-flow · commit de20f737929a · 2023-04-29T17:28:00.000-03:00
feat: adds extraction and adapters for documents
diff --git a/.clj-kondo/babashka/fs/config.edn b/.clj-kondo/babashka/fs/config.edn
@@ -0,0 +1 @@
+{:lint-as {babashka.fs/with-temp-dir clojure.core/let}}
diff --git a/.clj-kondo/babashka/sci/config.edn b/.clj-kondo/babashka/sci/config.edn
@@ -0,0 +1 @@
+{:hooks {:macroexpand {sci.core/copy-ns sci.core/copy-ns}}}
diff --git a/.clj-kondo/babashka/sci/sci/core.clj b/.clj-kondo/babashka/sci/sci/core.clj
@@ -0,0 +1,9 @@
+(ns sci.core)
+
+(defmacro copy-ns
+  ([ns-sym sci-ns]
+   `(copy-ns ~ns-sym ~sci-ns nil))
+  ([ns-sym sci-ns opts]
+   `[(quote ~ns-sym)
+     ~sci-ns
+     (quote ~opts)]))
diff --git a/.clj-kondo/nubank/matcher-combinators/config.edn b/.clj-kondo/nubank/matcher-combinators/config.edn
@@ -0,0 +1,4 @@
+{:linters
+ {:unresolved-symbol
+  {:exclude [(cljs.test/is [match? thrown-match?])
+             (clojure.test/is [match? thrown-match?])]}}}
diff --git a/.clj-kondo/rewrite-clj/rewrite-clj/config.edn b/.clj-kondo/rewrite-clj/rewrite-clj/config.edn
@@ -0,0 +1,5 @@
+{:lint-as
+ {rewrite-clj.zip/subedit-> clojure.core/->
+  rewrite-clj.zip/subedit->> clojure.core/->>
+  rewrite-clj.zip/edit-> clojure.core/->
+  rewrite-clj.zip/edit->> clojure.core/->>}}
diff --git a/.clj-kondo/taoensso/encore/config.edn b/.clj-kondo/taoensso/encore/config.edn
@@ -0,0 +1 @@
+{:hooks {:analyze-call {taoensso.encore/defalias taoensso.encore/defalias}}}
diff --git a/.clj-kondo/taoensso/encore/taoensso/encore.clj b/.clj-kondo/taoensso/encore/taoensso/encore.clj
@@ -0,0 +1,16 @@
+(ns taoensso.encore
+  (:require
+   [clj-kondo.hooks-api :as hooks]))
+
+(defn defalias [{:keys [node]}]
+  (let [[sym-raw src-raw] (rest (:children node))
+        src (if src-raw src-raw sym-raw)
+        sym (if src-raw
+              sym-raw
+              (symbol (name (hooks/sexpr src))))]
+    {:node (with-meta
+             (hooks/list-node
+               [(hooks/token-node 'def)
+                (hooks/token-node (hooks/sexpr sym))
+                (hooks/token-node (hooks/sexpr src))])
+             (meta src))}))
diff --git a/.lsp/config.edn b/.lsp/config.edn
@@ -0,0 +1,8 @@
+{:clean {:ns-inner-blocks-indentation :same-line}
+ :cljfmt {:indents {mlet                    [[:block 1]]
+                    facts                   [[:block 1]]
+                    fact                    [[:block 1]]
+                    flow                    [[:block 1]]
+                    flow-with-defaults      [[:block 1]]
+                    flow-as-of              [[:block 1]]
+                    flow-without-validation [[:block 1]]}}}
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # codes.clj.docs/extractor
 
-Tool to extract namespace/functions documentation from Clojure projects into indexed Datalevin file.
+Tool to extract namespace/functions documentation from Clojure projects into indexed [datalevin](https://github.com/juji-io/datalevin) file.
 
 # CLI
 
@@ -9,11 +9,6 @@ Tool to extract namespace/functions documentation from Clojure projects into ind
 clojure -X:extract
 ```
 
-## Update version of the configured projects in the config file
-```bash
-clojure -X:update
-```
-
 # Developing
 
 ## Repl
@@ -25,3 +20,8 @@ clojure -M:dev:nrepl
 ```bash
 clojure -M:dev:test
 ```
+
+## Build
+```bash
+clojure -T:build uberjar
+```
diff --git a/build.clj b/build.clj
@@ -3,7 +3,7 @@
   (:require [clojure.tools.build.api :as b]))
 
 (def default-lib 'codes.clj.docs/extractor)
-(def default-main 'codes.clj.docs.extractor)
+(def default-main 'codes.clj.docs.core)
 (def default-version "0.0.1-SNAPSHOT")
 (def class-dir "target/classes")
 
diff --git a/deps.edn b/deps.edn
@@ -5,10 +5,8 @@
         datalevin/datalevin {:mvn/version "0.8.14"}
         com.cognitect/transit-clj {:mvn/version "1.0.333"}}
  :aliases
- {:extract {:ns-default codes.clj.docs.extractor
+ {:extract {:ns-default codes.clj.docs.core
             :exec-fn extract!}
-  :update {:ns-default codes.clj.docs.extractor
-           :exec-fn update!}
 
   :build {:deps {io.github.clojure/tools.build {:mvn/version "0.9.4"}}
           :ns-default build
diff --git a/resources/config.edn b/resources/config.edn
@@ -0,0 +1,8 @@
+{:deps {org.clojure/clojure
+        {:git/url "https://github.com/clojure/clojure"
+         :git/tag "clojure-1.11.1"
+         :git/sha "ce55092f2b2f5481d25cff6205470c1335760ef6"}
+        org.clojure/clojurescript
+        {:git/url "https://github.com/clojure/clojurescript"
+         :git/tag "r1.11.60"
+         :git/sha "e7cdc70d0371a26e07e394ea9cd72d5c43e5e363"}}}
diff --git a/src/codes/clj/docs/adapters.clj b/src/codes/clj/docs/adapters.clj
@@ -0,0 +1,112 @@
+(ns codes.clj.docs.adapters
+  (:require [clojure.string :as str]))
+
+(defn ^:private assoc-some-transient! [m k v]
+  (if (nil? v) m (assoc! m k v)))
+
+(defn ^:private assoc-some
+  "Associates a key k, with a value v in a map m, if and only if v is not nil."
+  ([m k v]
+   (if (nil? v) m (assoc m k v)))
+  ([m k v & kvs]
+   (loop [acc (assoc-some-transient! (transient (or m {})) k v)
+          kvs kvs]
+     (if (next kvs)
+       (recur (assoc-some-transient! acc (first kvs) (second kvs)) (nnext kvs))
+       (if (zero? (count acc))
+         m
+         (persistent! acc))))))
+
+(defn analysis->projects
+  [analysis]
+  (mapv
+   (fn [{:keys [project]}]
+     (let [[group artifact] (-> project :project-name (str/split #"/"))]
+       (assoc-some {:project/name (:project-name project)}
+                   :project/group group
+                   :project/artifact artifact
+                   :project/paths (mapv #(str/replace % (:deps/root project) "")
+                                        (:paths project))
+                   :project/url (:git/url project)
+                   :project/tag (:git/tag project)
+                   :project/sha (:git/sha project)
+                   :project/manifest (:deps/manifest project))))
+   analysis))
+
+(defn analysis->libraries
+  [analysis]
+  (reduce
+   (fn [accum {:keys [project libraries]}]
+     (into accum
+           (let [{:git/keys [url tag] :deps/keys [root]} project
+                 [group artifact] (-> project :project-name (str/split #"/"))]
+             (mapv (fn [{:keys [end-row meta name-end-col name-end-row name-row added
+                                name author filename col name-col end-col doc row]}]
+                     (let [trim-filename (str/replace filename root "")]
+                       (assoc-some
+                        {:library/project (:project-name project)
+                         :library/group group
+                         :library/artifact artifact
+                         :library/name (str name)}
+                        :library/end-row end-row
+                        :library/meta meta
+                        :library/name-end-col name-end-col
+                        :library/name-end-row name-end-row
+                        :library/name-row name-row
+                        :library/added added
+                        :library/author author
+                        :library/filename trim-filename
+                        :library/git-source (str url "/blob/" tag trim-filename "#L" row)
+                        :library/col col
+                        :library/name-col name-col
+                        :library/end-col end-col
+                        :library/doc doc
+                        :library/row row)))
+                   libraries))))
+   []
+   analysis))
+
+(defn ^:private inrelevant-definitions [{:definition/keys [defined-by]}]
+  (contains? #{"clojure.core/declare"} defined-by))
+
+(defn analysis->definitions
+  [analysis]
+  (->> analysis
+       (reduce
+        (fn [accum {:keys [project definitions]}]
+          (into accum
+                (let [{:git/keys [url tag] :deps/keys [root]} project
+                      [group artifact] (-> project :project-name (str/split #"/"))]
+                  (mapv (fn [{:keys [fixed-arities end-row meta name-end-col
+                                     name-end-row name-row added ns name author
+                                     defined-by filename macro col name-col end-col
+                                     arglist-strs varargs-min-arity doc row]}]
+                          (let [trim-filename (str/replace filename root "")]
+                            (assoc-some
+                             {:definition/project (:project-name project)
+                              :definition/group group
+                              :definition/artifact artifact
+                              :definition/name (str name)}
+                             :definition/defined-by (some-> defined-by str)
+                             :definition/library (some-> ns str)
+                             :definition/fixed-arities fixed-arities
+                             :definition/arglist-strs arglist-strs
+                             :definition/end-row end-row
+                             :definition/meta meta
+                             :definition/name-end-col name-end-col
+                             :definition/name-end-row name-end-row
+                             :definition/name-row name-row
+                             :definition/added added
+                             :definition/author author
+                             :definition/filename trim-filename
+                             :definition/git-source (str url "/blob/" tag trim-filename "#L" row)
+                             :definition/col col
+                             :definition/name-col name-col
+                             :definition/end-col end-col
+                             :definition/doc doc
+                             :definition/row row
+                             :definition/macro macro
+                             :definition/varargs-min-arity varargs-min-arity)))
+                        definitions))))
+        [])
+       (remove inrelevant-definitions)))
diff --git a/src/codes/clj/docs/analysis.clj b/src/codes/clj/docs/analysis.clj
@@ -0,0 +1,46 @@
+(ns codes.clj.docs.analysis
+  (:require [clj-kondo.core :as kondo]
+            [clojure.edn :as edn]
+            [clojure.java.io :as io]
+            [clojure.tools.deps :as deps]))
+
+(defn download-project!
+  [project git]
+  (-> (deps/resolve-deps
+       {:deps {project git}
+        :mvn/repos {"central" {:url "https://repo1.maven.org/maven2/"},
+                    "clojars" {:url "https://repo.clojars.org/"}}}
+       nil)
+      (get project)
+      (assoc :project-name (str project))))
+
+(defn kondo-run!
+  [paths]
+  (-> {:lint paths
+       :config {:output {:format :edn}
+                :analysis {:arglists true
+                           :var-definitions {:meta [:no-doc :skip-wiki :arglists]}
+                           :namespace-definitions {:meta [:no-doc :skip-wiki]}}}}
+      kondo/run!
+      :analysis
+      (dissoc :namespace-usages
+              :var-usages)))
+
+(defn extract-analysis!
+  [project git]
+  (let [{:keys [paths] :as project-meta} (download-project! project git)
+        {:keys [var-definitions namespace-definitions]} (kondo-run! paths)]
+    {:project project-meta
+     :libraries namespace-definitions
+     :definitions var-definitions}))
+
+(defn extract!
+  ([]
+   (extract! (io/resource "config.edn")))
+  ([config-file]
+   (->> config-file
+        slurp
+        edn/read-string
+        :deps
+        (map (fn [[project git]]
+               (extract-analysis! project git))))))
diff --git a/src/codes/clj/docs/core.clj b/src/codes/clj/docs/core.clj
@@ -0,0 +1,19 @@
+(ns codes.clj.docs.core
+  (:require [codes.clj.docs.adapters :as adapters]
+            [codes.clj.docs.analysis :as analysis])
+  (:gen-class))
+
+(defn extract!
+  "Extract data from configured projects and generate Datalevin file."
+  [_data]
+  (println "extract!")
+  ;; TODO create database
+  (let [analysis-raw (analysis/extract!)]
+    {:project (adapters/analysis->projects analysis-raw)
+     :libraries (adapters/analysis->libraries analysis-raw)
+     :definitions (adapters/analysis->definitions analysis-raw)}))
+
+(defn -main
+  "The entry-point for 'gen-class'"
+  [& _args]
+  (extract! {}))
diff --git a/src/codes/clj/docs/extractor.clj b/src/codes/clj/docs/extractor.clj
diff --git a/test/codes/clj/docs/adapters_test.clj b/test/codes/clj/docs/adapters_test.clj
diff --git a/test/codes/clj/docs/extractor_test.clj b/test/codes/clj/docs/extractor_test.clj

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{:lint-as {babashka.fs/with-temp-dir clojure.core/let}}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{:hooks {:macroexpand {sci.core/copy-ns sci.core/copy-ns}}}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{:hooks {:analyze-call {taoensso.encore/defalias taoensso.encore/defalias}}}`