Skip to content

Commit 92b84f2

Browse files
rafaeldelbonirafaeldelboni
committed
feat: adds bulk-transaction to db
1 parent a15258f commit 92b84f2

File tree

10 files changed

+109
-35
lines changed

10 files changed

+109
-35
lines changed

‎deps.edn‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
org.clojure/tools.deps {:mvn/version "0.18.1335"}
44
clj-kondo/clj-kondo {:mvn/version "2023.04.14"}
55
datalevin/datalevin {:mvn/version "0.8.14"}
6-
com.cognitect/transit-clj {:mvn/version "1.0.333"}}
6+
com.cognitect/transit-clj {:mvn/version "1.0.333"}
7+
org.slf4j/slf4j-nop {:mvn/version "2.0.7"}}
78
:aliases
89
{:extract {:ns-default codes.clj.docs.extractor.core
910
:exec-fn extract!}

‎resources/config.edn‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
{:deps {org.clojure/clojure
1+
{:db {:dir "target/docs-db"}
2+
:deps {org.clojure/clojure
23
{:git/url "https://github.com/clojure/clojure"
34
:git/tag "clojure-1.11.1"
45
:git/sha "ce55092f2b2f5481d25cff6205470c1335760ef6"}

‎src/codes/clj/docs/extractor/adapters.clj‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
(ns codes.clj.docs.extractor.adapters
22
(:require [clojure.string :as str]))
33

4-
(defn ^:private assoc-some-transient! [m k v]
4+
(defn ^:private assoc-some-transient!
5+
[m k v]
56
(if (nil? v) m (assoc! m k v)))
67

78
(defn ^:private assoc-some
@@ -110,3 +111,9 @@
110111
definitions))))
111112
[])
112113
(remove inrelevant-definitions)))
114+
115+
(defn analysis->datoms
116+
[analysis]
117+
(concat (analysis->projects analysis)
118+
(analysis->libraries analysis)
119+
(analysis->definitions analysis)))

‎src/codes/clj/docs/extractor/analysis.clj‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
(ns codes.clj.docs.extractor.analysis
22
(:require [clj-kondo.core :as kondo]
3-
[clojure.edn :as edn]
4-
[clojure.java.io :as io]
53
[clojure.tools.deps :as deps]))
64

75
(defn download-project!
@@ -35,12 +33,8 @@
3533
:definitions var-definitions}))
3634

3735
(defn extract!
38-
([]
39-
(extract! (io/resource "config.edn")))
40-
([config-file]
41-
(->> config-file
42-
slurp
43-
edn/read-string
44-
:deps
45-
(map (fn [[project git]]
46-
(extract-analysis! project git))))))
36+
[config]
37+
(->> config
38+
:deps
39+
(mapv (fn [[project git]]
40+
(extract-analysis! project git)))))

‎src/codes/clj/docs/extractor/config.clj‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
(ns codes.clj.docs.extractor.config
2+
(:require [clojure.edn :as edn]
3+
[clojure.java.io :as io]))
4+
5+
(defn read! [config-file]
6+
(->> config-file
7+
io/file
8+
slurp
9+
edn/read-string))

‎src/codes/clj/docs/extractor/core.clj‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
(ns codes.clj.docs.extractor.core
2-
(:require [codes.clj.docs.adapters :as adapters]
3-
[codes.clj.docs.analysis :as analysis])
2+
(:require [codes.clj.docs.extractor.adapters :as adapters]
3+
[codes.clj.docs.extractor.analysis :as analysis]
4+
[codes.clj.docs.extractor.config :as config]
5+
[codes.clj.docs.extractor.datalevin :as datalevin]
6+
[codes.clj.docs.extractor.log :refer [log->fn]])
47
(:gen-class))
58

69
(defn extract!
710
"Extract data from configured projects and generate Datalevin file."
811
[_data]
9-
(println "extract!")
10-
;; TODO create database
11-
(let [analysis-raw (analysis/extract!)]
12-
{:project (adapters/analysis->projects analysis-raw)
13-
:libraries (adapters/analysis->libraries analysis-raw)
14-
:definitions (adapters/analysis->definitions analysis-raw)}))
12+
(let [config (config/read! "resources/config.edn")
13+
analysis-raw (log->fn (analysis/extract! config))
14+
datoms (adapters/analysis->datoms analysis-raw)]
15+
(log->fn (datalevin/bulk-transact! datoms config))))
1516

1617
(defn -main
1718
"The entry-point for 'gen-class'"

‎src/codes/clj/docs/extractor/datalevin.clj‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -59,17 +59,7 @@
5959
(def db-schemas
6060
(merge project-schema library-schema definition-schema))
6161

62-
(comment
63-
(require '[codes.clj.docs.extractor.fixtures.analysis :as fix])
64-
65-
(let [conn (d/get-conn "/tmp/mydb" db-schemas)
66-
db (d/db conn)]
67-
(d/transact! conn (concat fix/projects-adapted
68-
fix/libraries-adapted
69-
fix/definitions-adapted))
70-
(pprint/pprint (d/q '[:find (pull ?e [*])
71-
:in $ ?q
72-
:where [?e :project/name ?q]]
73-
db
74-
"org.clojure/clojure"))
62+
(defn bulk-transact! [datoms config]
63+
(let [conn (-> config :db :dir (d/get-conn db-schemas))]
64+
(d/transact! conn datoms)
7565
(d/close conn)))

‎src/codes/clj/docs/extractor/log.clj‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
(ns codes.clj.docs.extractor.log
2+
(:import [java.time Duration LocalDateTime]))
3+
4+
(defmacro log->fn [& body]
5+
`(do
6+
(let [executed-form# ~(str (second &form))
7+
start-time# (LocalDateTime/now)]
8+
(println executed-form# "started at" (.toString start-time#))
9+
(let [result# (do ~@body)
10+
end-time# (LocalDateTime/now)]
11+
(println executed-form#
12+
" ended at"
13+
(.toString end-time#)
14+
"took"
15+
(->> (Duration/between start-time# end-time#)
16+
(.getSeconds))
17+
"secs")
18+
result#))))

‎test/codes/clj/docs/extractor/adapters_test.clj‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,8 @@
1818
(testing "analysis -> definitions"
1919
(is (match? fixtures.analysis/definitions-adapted
2020
(adapters/analysis->definitions fixtures.analysis/raw)))))
21+
22+
(deftest analysis->datoms-test
23+
(testing "analysis -> datoms"
24+
(is (= 4
25+
(count (adapters/analysis->datoms fixtures.analysis/raw))))))

‎test/codes/clj/docs/extractor/datalevin_test.clj‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
(ns codes.clj.docs.extractor.datalevin-test
2+
(:require [clojure.test :refer [deftest is testing]]
3+
[codes.clj.docs.extractor.adapters :as adapters]
4+
[codes.clj.docs.extractor.datalevin :as datalevin]
5+
[codes.clj.docs.extractor.fixtures.analysis :as fixtures.analysis]
6+
[datalevin.core :as d]
7+
[datalevin.util :as util]
8+
[matcher-combinators.test :refer [match?]])
9+
(:import [java.util UUID]))
10+
11+
(deftest analysis->datoms-test
12+
(let [dir (util/tmp-dir (str "lmdb-test-" (UUID/randomUUID)))]
13+
(datalevin/bulk-transact!
14+
(adapters/analysis->datoms fixtures.analysis/raw)
15+
{:db {:dir dir}})
16+
17+
(testing "check data exists in database"
18+
(let [conn (d/get-conn dir datalevin/db-schemas)
19+
db (d/db conn)]
20+
21+
(is (= 4
22+
(-> (d/q '[:find (count ?e)
23+
:where [?e]]
24+
db)
25+
ffirst)))
26+
27+
(is (match? [{:project/manifest :pom
28+
:project/tag "clojure-1.11.1"
29+
:project/sha "ce55092f2b2f5481d25cff6205470c1335760ef6"
30+
:project/url "https://github.com/clojure/clojure"
31+
:project/artifact "clojure"
32+
:project/paths ["/src/clj"
33+
"/src/main/clojure"
34+
"/src/main/java"
35+
"/src/resources"]
36+
:db/id 1
37+
:project/name "org.clojure/clojure"
38+
:project/group "org.clojure"}]
39+
(-> (d/q '[:find (pull ?e [*])
40+
:in $ ?q
41+
:where [?e :project/name ?q]]
42+
db
43+
"org.clojure/clojure")
44+
first)))
45+
46+
(d/close conn)))
47+
48+
(util/delete-files dir)))

0 commit comments

Comments
 (0)