Skip to content

Commit 686ba17

Browse files
rafaeldelbonirafaeldelboni
authored
Merge pull request #3 from clj-codes/feat/adds-datalevin-transact
feat: adds datalevin schema and test transaction
2 parents 018f525 + 92449dd commit 686ba17

File tree

11 files changed

+365
-173
lines changed

11 files changed

+365
-173
lines changed

‎deps.edn‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
org.clojure/tools.deps {:mvn/version "0.18.1335"}
44
clj-kondo/clj-kondo {:mvn/version "2023.04.14"}
55
datalevin/datalevin {:mvn/version "0.8.14"}
6-
com.cognitect/transit-clj {:mvn/version "1.0.333"}}
6+
com.cognitect/transit-clj {:mvn/version "1.0.333"}
7+
org.slf4j/slf4j-nop {:mvn/version "2.0.7"}}
78
:aliases
89
{:extract {:ns-default codes.clj.docs.extractor.core
910
:exec-fn extract!}

‎resources/config.edn‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
1-
{:deps {org.clojure/clojure
1+
{:db {:dir "target/docs-db"}
2+
:deps {org.clojure/clojure
23
{:git/url "https://github.com/clojure/clojure"
34
:git/tag "clojure-1.11.1"
45
:git/sha "ce55092f2b2f5481d25cff6205470c1335760ef6"}
6+
org.clojure/core.logic
7+
{:git/url "https://github.com/clojure/core.logic"
8+
:git/tag "v1.0.1"
9+
:git/sha "d854548a1eb0706150bd5f5d939c7bca162c07fb"}
510
org.clojure/clojurescript
611
{:git/url "https://github.com/clojure/clojurescript"
712
:git/tag "r1.11.60"

‎src/codes/clj/docs/extractor/adapters.clj‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
(ns codes.clj.docs.extractor.adapters
22
(:require [clojure.string :as str]))
33

4-
(defn ^:private assoc-some-transient! [m k v]
4+
(defn ^:private assoc-some-transient!
5+
[m k v]
56
(if (nil? v) m (assoc! m k v)))
67

78
(defn ^:private assoc-some
@@ -110,3 +111,9 @@
110111
definitions))))
111112
[])
112113
(remove inrelevant-definitions)))
114+
115+
(defn analysis->datoms
116+
[analysis]
117+
(concat (analysis->projects analysis)
118+
(analysis->libraries analysis)
119+
(analysis->definitions analysis)))

‎src/codes/clj/docs/extractor/analysis.clj‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
(ns codes.clj.docs.extractor.analysis
22
(:require [clj-kondo.core :as kondo]
3-
[clojure.edn :as edn]
4-
[clojure.java.io :as io]
53
[clojure.tools.deps :as deps]))
64

75
(defn download-project!
@@ -35,12 +33,8 @@
3533
:definitions var-definitions}))
3634

3735
(defn extract!
38-
([]
39-
(extract! (io/resource "config.edn")))
40-
([config-file]
41-
(->> config-file
42-
slurp
43-
edn/read-string
44-
:deps
45-
(map (fn [[project git]]
46-
(extract-analysis! project git))))))
36+
[config]
37+
(->> config
38+
:deps
39+
(mapv (fn [[project git]]
40+
(extract-analysis! project git)))))

‎src/codes/clj/docs/extractor/config.clj‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
(ns codes.clj.docs.extractor.config
2+
(:require [clojure.edn :as edn]
3+
[clojure.java.io :as io]))
4+
5+
(defn read! [config-file]
6+
(->> config-file
7+
io/file
8+
slurp
9+
edn/read-string))

‎src/codes/clj/docs/extractor/core.clj‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
(ns codes.clj.docs.extractor.core
2-
(:require [codes.clj.docs.adapters :as adapters]
3-
[codes.clj.docs.analysis :as analysis])
2+
(:require [codes.clj.docs.extractor.adapters :as adapters]
3+
[codes.clj.docs.extractor.analysis :as analysis]
4+
[codes.clj.docs.extractor.config :as config]
5+
[codes.clj.docs.extractor.datalevin :as datalevin]
6+
[codes.clj.docs.extractor.log :refer [log->fn]])
47
(:gen-class))
58

69
(defn extract!
710
"Extract data from configured projects and generate Datalevin file."
811
[_data]
9-
(println "extract!")
10-
;; TODO create database
11-
(let [analysis-raw (analysis/extract!)]
12-
{:project (adapters/analysis->projects analysis-raw)
13-
:libraries (adapters/analysis->libraries analysis-raw)
14-
:definitions (adapters/analysis->definitions analysis-raw)}))
12+
(let [config (config/read! "resources/config.edn")
13+
analysis-raw (log->fn (analysis/extract! config))
14+
datoms (adapters/analysis->datoms analysis-raw)]
15+
(log->fn (datalevin/bulk-transact! datoms config))))
1516

1617
(defn -main
1718
"The entry-point for 'gen-class'"

‎src/codes/clj/docs/extractor/datalevin.clj‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
(ns codes.clj.docs.extractor.datalevin
2+
(:require [datalevin.core :as d]))
3+
4+
;; TODO: add id :db.unique/identity and ref :db.type/ref
5+
6+
(def project-schema
7+
{;:project/id {:db/valueType :db.type/string :unique :db.unique/identity}
8+
:project/name {:db/valueType :db.type/string
9+
:db/fulltext true}
10+
:project/group {:db/valueType :db.type/string}
11+
:project/artifact {:db/valueType :db.type/string}
12+
:project/paths {:db/valueType :db.type/string
13+
:db/cardinality :db.cardinality/many}
14+
:project/url {:db/valueType :db.type/string}
15+
:project/tag {:db/valueType :db.type/string}
16+
:project/sha {:db/valueType :db.type/string}
17+
:project/manifest {:db/valueType :db.type/keyword}})
18+
19+
(def library-schema
20+
{;:library/id {:db/valueType :db.type/string :unique :db.unique/identity}
21+
:library/name {:db/valueType :db.type/string
22+
:db/fulltext true}
23+
:library/project {:db/valueType :db.type/string} ;todo change to ref
24+
;:library/project {:db/valueType :db.type/ref}
25+
:library/group {:db/valueType :db.type/string}
26+
:library/artifact {:db/valueType :db.type/string}
27+
:library/doc {:db/valueType :db.type/string
28+
:db/fulltext true}
29+
:library/author {:db/valueType :db.type/string}
30+
:library/filename {:db/valueType :db.type/string}
31+
:library/git-source {:db/valueType :db.type/string}
32+
:library/added {:db/valueType :db.type/string}
33+
:library/row {:db/valueType :db.type/long}
34+
:library/col {:db/valueType :db.type/long}})
35+
36+
(def definition-schema
37+
{;:definition/id {:db/valueType :db.type/string :unique :db.unique/identity}
38+
:definition/name {:db/valueType :db.type/string
39+
:db/fulltext true}
40+
:definition/library {:db/valueType :db.type/string} ;todo change to ref
41+
;:definition/library {:db/valueType :db.type/ref}
42+
:definition/project {:db/valueType :db.type/string} ;todo change to ref
43+
;:definition/project {:db/valueType :db.type/ref}
44+
:definition/group {:db/valueType :db.type/string}
45+
:definition/artifact {:db/valueType :db.type/string}
46+
:definition/doc {:db/valueType :db.type/string
47+
:db/fulltext true}
48+
:definition/filename {:db/valueType :db.type/string}
49+
:definition/git-source {:db/valueType :db.type/string}
50+
:definition/arglist-strs {:db/valueType :db.type/string
51+
:db/cardinality :db.cardinality/many}
52+
:definition/varargs-min-arity {:db/valueType :db.type/long}
53+
:definition/added {:db/valueType :db.type/string}
54+
:definition/macro {:db/valueType :db.type/boolean}
55+
:definition/row {:db/valueType :db.type/long}
56+
:definition/col {:db/valueType :db.type/long}})
57+
58+
(def db-schemas
59+
(merge project-schema library-schema definition-schema))
60+
61+
(defn bulk-transact! [datoms config]
62+
(let [conn (-> config :db :dir (d/get-conn db-schemas))]
63+
(d/transact! conn datoms)
64+
(d/close conn)))

‎src/codes/clj/docs/extractor/log.clj‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
(ns codes.clj.docs.extractor.log
2+
(:import [java.time Duration LocalDateTime]))
3+
4+
(defmacro log->fn [& body]
5+
`(do
6+
(let [executed-form# ~(str (second &form))
7+
start-time# (LocalDateTime/now)]
8+
(println executed-form# "started at" (.toString start-time#))
9+
(let [result# (do ~@body)
10+
end-time# (LocalDateTime/now)]
11+
(println executed-form#
12+
" ended at"
13+
(.toString end-time#)
14+
"took"
15+
(->> (Duration/between start-time# end-time#)
16+
(.getSeconds))
17+
"secs")
18+
result#))))

‎test/codes/clj/docs/extractor/adapters_test.clj‎

Lines changed: 12 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -1,164 +1,25 @@
11
(ns codes.clj.docs.extractor.adapters-test
22
(:require [clojure.test :refer [deftest is testing]]
33
[codes.clj.docs.extractor.adapters :as adapters]
4+
[codes.clj.docs.extractor.fixtures.analysis :as fixtures.analysis]
45
[matcher-combinators.test :refer [match?]]))
56

6-
(def analysis-fixture
7-
[{:project {:git/url "https://github.com/clojure/clojure"
8-
:git/tag "clojure-1.11.1"
9-
:git/sha "ce55092f2b2f5481d25cff6205470c1335760ef6"
10-
:deps/manifest :pom
11-
:deps/root "/Users/username/.gitlibs/libs/org.clojure/clojure/ce55092f2b2f5481d25cff6205470c1335760ef6"
12-
:parents #{[]}
13-
:paths ["/Users/username/.gitlibs/libs/org.clojure/clojure/ce55092f2b2f5481d25cff6205470c1335760ef6/src/main/java"
14-
"/Users/username/.gitlibs/libs/org.clojure/clojure/ce55092f2b2f5481d25cff6205470c1335760ef6/src/main/clojure"
15-
"/Users/username/.gitlibs/libs/org.clojure/clojure/ce55092f2b2f5481d25cff6205470c1335760ef6/src/resources"
16-
"/Users/username/.gitlibs/libs/org.clojure/clojure/ce55092f2b2f5481d25cff6205470c1335760ef6/src/clj"]
17-
:project-name "org.clojure/clojure"}
18-
:libraries [{:end-row 39
19-
:meta {}
20-
:name-end-col 19
21-
:name-end-row 37
22-
:name-row 37
23-
:added "1.2"
24-
:name 'clojure.pprint
25-
:author "Tom Faulhaber"
26-
:filename "/Users/username/.gitlibs/libs/org.clojure/clojure/ce55092f2b2f5481d25cff6205470c1335760ef6/src/clj/clojure/pprint.clj"
27-
:col 1
28-
:name-col 5
29-
:end-col 40
30-
:doc "A Pretty Printer for Clojure\n\nclojure.pprint implements a flexible system for printing structured data\nin a pleasing easy-to-understand format. Basic use of the pretty printer is \nsimple just call pprint instead of println. More advanced users can use \nthe building blocks provided to create custom output formats. \n\nOut of the box pprint supports a simple structured format for basic data \nand a specialized format for Clojure source code. More advanced formats \nincluding formats that don't look like Clojure data at all like XML and \nJSON can be rendered by creating custom dispatch functions. \n\nIn addition to the pprint function this module contains cl-format a text \nformatting function which is fully compatible with the format function in \nCommon Lisp. Because pretty printing directives are directly integrated with\ncl-format it supports very concise custom dispatch. It also provides\na more powerful alternative to Clojure's standard format function.\n\nSee documentation for pprint and cl-format for more information or \ncomplete documentation on the Clojure web site on GitHub."
31-
:row 14}]
32-
:definitions [{:end-row 109
33-
:meta {}
34-
:name-end-col 30
35-
:name-end-row 109
36-
:name-row 109
37-
:ns 'clojure.pprint
38-
:name 'format-simple-number
39-
:defined-by 'clojure.core/declare
40-
:filename "/Users/username/.gitlibs/libs/org.clojure/clojure/ce55092f2b2f5481d25cff6205470c1335760ef6/src/clj/clojure/pprint/pprint_base.clj"
41-
:col 1
42-
:name-col 10
43-
:end-col 31
44-
:row 109}
45-
{:end-row 327
46-
:meta {:arglists '[[options* body]]}
47-
:name-end-col 31
48-
:name-end-row 302
49-
:name-row 302
50-
:added "1.2"
51-
:ns 'clojure.pprint
52-
:name 'pprint-logical-block
53-
:defined-by 'clojure.core/defmacro
54-
:filename "/Users/username/.gitlibs/libs/org.clojure/clojure/ce55092f2b2f5481d25cff6205470c1335760ef6/src/clj/clojure/pprint/pprint_base.clj"
55-
:macro true
56-
:col 1
57-
:name-col 11
58-
:end-col 16
59-
:arglist-strs ["[& args]"]
60-
:varargs-min-arity 0
61-
:doc
62-
"Execute the body as a pretty printing logical block with output to *out* which \nmust be a pretty printing writer. When used from pprint or cl-format this can be \nassumed. \n\nThis function is intended for use when writing custom dispatch functions.\n\nBefore the body the caller can optionally specify options: :prefix :per-line-prefix \nand :suffix."
63-
:row 302}
64-
{:fixed-arities #{1 2}
65-
:end-row 35
66-
:meta {}
67-
:name-end-col 18
68-
:name-end-row 11
69-
:name-row 11
70-
:added "1.3"
71-
:ns 'clojure.pprint
72-
:name 'print-table
73-
:defined-by 'clojure.core/defn
74-
:filename "/Users/username/.gitlibs/libs/org.clojure/clojure/ce55092f2b2f5481d25cff6205470c1335760ef6/src/clj/clojure/pprint/print_table.clj"
75-
:col 1
76-
:name-col 7
77-
:end-col 51
78-
:arglist-strs ["[ks rows]" "[rows]"]
79-
:doc "Prints a collection of maps in a textual table. Prints table headings\n ks and then a line of output for each row corresponding to the keys\n in ks. If ks are not specified use the keys of the first item in rows."
80-
:row 11}]}])
81-
827
(deftest analysis->projects-test
838
(testing "analysis -> project"
84-
(is (match? [#:project{:name "org.clojure/clojure"
85-
:group "org.clojure"
86-
:artifact "clojure"
87-
:paths ["/src/main/java"
88-
"/src/main/clojure"
89-
"/src/resources"
90-
"/src/clj"]
91-
:url "https://github.com/clojure/clojure"
92-
:tag "clojure-1.11.1"
93-
:sha "ce55092f2b2f5481d25cff6205470c1335760ef6"
94-
:manifest :pom}]
95-
(adapters/analysis->projects analysis-fixture)))))
9+
(is (match? fixtures.analysis/projects-adapted
10+
(adapters/analysis->projects fixtures.analysis/raw)))))
9611

9712
(deftest analysis->libraries-test
9813
(testing "analysis -> libraries"
99-
(is (match? [#:library{:project "org.clojure/clojure"
100-
:artifact "clojure"
101-
:name-end-col 19
102-
:added "1.2"
103-
:group "org.clojure"
104-
:end-col 40
105-
:end-row 39
106-
:git-source "https://github.com/clojure/clojure/blob/clojure-1.11.1/src/clj/clojure/pprint.clj#L14"
107-
:name-row 37
108-
:meta {}
109-
:row 14
110-
:name-col 5
111-
:author "Tom Faulhaber"
112-
:col 1
113-
:name "clojure.pprint"
114-
:doc "A Pretty Printer for Clojure\n\nclojure.pprint implements a flexible system for printing structured data\nin a pleasing easy-to-understand format. Basic use of the pretty printer is \nsimple just call pprint instead of println. More advanced users can use \nthe building blocks provided to create custom output formats. \n\nOut of the box pprint supports a simple structured format for basic data \nand a specialized format for Clojure source code. More advanced formats \nincluding formats that don't look like Clojure data at all like XML and \nJSON can be rendered by creating custom dispatch functions. \n\nIn addition to the pprint function this module contains cl-format a text \nformatting function which is fully compatible with the format function in \nCommon Lisp. Because pretty printing directives are directly integrated with\ncl-format it supports very concise custom dispatch. It also provides\na more powerful alternative to Clojure's standard format function.\n\nSee documentation for pprint and cl-format for more information or \ncomplete documentation on the Clojure web site on GitHub."
115-
:name-end-row 37
116-
:filename "/src/clj/clojure/pprint.clj"}]
117-
(adapters/analysis->libraries analysis-fixture)))))
14+
(is (match? fixtures.analysis/libraries-adapted
15+
(adapters/analysis->libraries fixtures.analysis/raw)))))
11816

11917
(deftest analysis->definitions-test
12018
(testing "analysis -> definitions"
121-
(is (match? [#:definition{:defined-by "clojure.core/defmacro"
122-
:library "clojure.pprint"
123-
:filename "/src/clj/clojure/pprint/pprint_base.clj"
124-
:macro true
125-
:project "org.clojure/clojure"
126-
:row 302
127-
:varargs-min-arity 0
128-
:added "1.2"
129-
:arglist-strs ["[& args]"]
130-
:col 1
131-
:name-col 11
132-
:end-col 16
133-
:doc "Execute the body as a pretty printing logical block with output to *out* which \nmust be a pretty printing writer. When used from pprint or cl-format this can be \nassumed. \n\nThis function is intended for use when writing custom dispatch functions.\n\nBefore the body the caller can optionally specify options: :prefix :per-line-prefix \nand :suffix."
134-
:git-source "https://github.com/clojure/clojure/blob/clojure-1.11.1/src/clj/clojure/pprint/pprint_base.clj#L302"
135-
:name-end-row 302
136-
:name-row 302
137-
:group "org.clojure"
138-
:meta {:arglists '[[options* body]]}
139-
:artifact "clojure"
140-
:name-end-col 31
141-
:end-row 327
142-
:name "pprint-logical-block"}
143-
#:definition{:defined-by "clojure.core/defn"
144-
:library "clojure.pprint"
145-
:filename "/src/clj/clojure/pprint/print_table.clj"
146-
:project "org.clojure/clojure"
147-
:row 11
148-
:added "1.3"
149-
:arglist-strs ["[ks rows]" "[rows]"]
150-
:col 1
151-
:name-col 7
152-
:end-col 51
153-
:doc "Prints a collection of maps in a textual table. Prints table headings\n ks and then a line of output for each row corresponding to the keys\n in ks. If ks are not specified use the keys of the first item in rows."
154-
:git-source "https://github.com/clojure/clojure/blob/clojure-1.11.1/src/clj/clojure/pprint/print_table.clj#L11"
155-
:fixed-arities #{1 2}
156-
:name-end-row 11
157-
:name-row 11
158-
:group "org.clojure"
159-
:meta {}
160-
:artifact "clojure"
161-
:name-end-col 18
162-
:end-row 35
163-
:name "print-table"}]
164-
(adapters/analysis->definitions analysis-fixture)))))
19+
(is (match? fixtures.analysis/definitions-adapted
20+
(adapters/analysis->definitions fixtures.analysis/raw)))))
21+
22+
(deftest analysis->datoms-test
23+
(testing "analysis -> datoms"
24+
(is (= 4
25+
(count (adapters/analysis->datoms fixtures.analysis/raw))))))

0 commit comments

Comments
 (0)