Skip to content

Commit b83e0c6

Browse files
RamblurrRamblurr
committed
Add SQLite benchmark suite and report
1 parent 77fed64 commit b83e0c6

7 files changed

Lines changed: 633 additions & 4 deletions

File tree

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,8 @@ pom.xml.asc
2020
*.sqlite-shm
2121
*.sqlite-wal
2222
extra/
23-
bench/
23+
bench/tmp/
24+
bench/results/
2425
.direnv/
26+
prompts/
27+
*.md

‎bb.edn‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,15 @@
1111
test (shell "./bin/run-integrationtests")
1212
deps-tree (clojure "-X:deps tree :aliases '[:dev]'")
1313
dev {:doc "Start a local development server" :task (clojure "-M:dev:repl/dev")}
14-
fmt (shell "cljfmt -v fix src test")
15-
fmt.check (shell "cljfmt -v check src test")
14+
fmt (shell "cljfmt -v fix src test bench")
15+
fmt.check (shell "cljfmt -v check src test bench")
1616
lint.copy-configs (let [cp (-> (shell {:out :string} "clojure -Spath -M:test:dev") :out str/trim)]
1717
(shell (str "clj-kondo --lint " cp " --dependencies --copy-configs --skip-lint")))
1818
lint.kondo (shell "clj-kondo" *command-line-args*)
1919
lint (shell (str "clj-kondo" " --fail-level error"
2020
(if (seq *command-line-args*)
2121
(str " --lint " (first *command-line-args*))
22-
" --lint src test")))
22+
" --lint src test bench")))
2323
jar (clojure "-T:build jar")
2424
qa {:depends [lint test fmt]}
2525
ci {:depends [prep lint.copy-configs test lint fmt.check jar]}

‎bench/datahike_sqlite/datahike_benchmark.clj‎

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
;; Copyright © 2026 Casey Link <casey@outskirtslabs.com>
2+
;; SPDX-License-Identifier: MIT
3+
(ns datahike-sqlite.datahike-benchmark
4+
"SQLite Datahike benchmark entry point aligned with the datahike-lmdb benchmark.
5+
6+
Comparable rows:
7+
- SQLite-backed Datahike
8+
- Datalevin
9+
10+
Run with:
11+
clojure -M:dev:bench -m datahike-sqlite.datahike-benchmark
12+
clojure -M:dev:bench -m datahike-sqlite.datahike-benchmark '{:mode :smoke}'
13+
clojure -M:dev:bench -m datahike-sqlite.datahike-benchmark '{:sizes [100000 500000] :output \"bench/results/sqlite-datahike.txt\"}'"
14+
(:require [clojure.edn :as edn]
15+
[clojure.java.io :as io]
16+
[datahike-sqlite.core]
17+
[datahike.api :as d]
18+
[datalevin.core :as dl])
19+
(:import [java.io File]
20+
[java.util UUID]))
21+
22+
(set! *warn-on-reflection* true)
23+
24+
(def default-sizes [100000 500000 1000000])
25+
(def smoke-sizes [1000])
26+
(def warmup-size 1000)
27+
(def base-path "bench/tmp/sqlite-datahike")
28+
29+
(def schema
30+
[{:db/ident :person/name
31+
:db/valueType :db.type/string
32+
:db/cardinality :db.cardinality/one}
33+
{:db/ident :person/age
34+
:db/valueType :db.type/long
35+
:db/cardinality :db.cardinality/one}
36+
{:db/ident :person/email
37+
:db/valueType :db.type/string
38+
:db/cardinality :db.cardinality/one}])
39+
40+
(def datalevin-schema
41+
{:person/name {:db/valueType :db.type/string}
42+
:person/age {:db/valueType :db.type/long}
43+
:person/email {:db/valueType :db.type/string}})
44+
45+
(def benchmark-query
46+
'[:find ?n ?a
47+
:where
48+
[?e :person/name ?n]
49+
[?e :person/age ?a]
50+
[(> ?a 40)]])
51+
52+
(defn generate-entities [n]
53+
(mapv (fn [i]
54+
{:person/name (str "Person-" i)
55+
:person/age (+ 20 (mod i 50))
56+
:person/email (str "person" i "@example.com")})
57+
(range n)))
58+
59+
(defn delete-path [^File path]
60+
(when (.exists path)
61+
(doseq [^File child (reverse (file-seq path))]
62+
(.delete child))))
63+
64+
(defn delete-sqlite-files! [path]
65+
(doseq [suffix ["" "-shm" "-wal"]]
66+
(delete-path (io/file (str path suffix)))))
67+
68+
(defn stable-id [label path]
69+
(UUID/nameUUIDFromBytes (.getBytes (str "benchmark:" label ":" path) "UTF-8")))
70+
71+
(defn ensure-parent-dir! [path]
72+
(when-let [parent (.getParentFile (io/file path))]
73+
(.mkdirs parent)))
74+
75+
(defn sqlite-config [path label extra-config]
76+
(merge {:store {:backend :sqlite
77+
:dbname path
78+
:id (stable-id label path)}
79+
:schema-flexibility :write
80+
:keep-history? false}
81+
extra-config))
82+
83+
(defn datahike-row
84+
[backend operation n elapsed & {:as extras}]
85+
(merge {:benchmark :sqlite-datahike
86+
:backend backend
87+
:operation operation
88+
:entities n
89+
:time-sec elapsed}
90+
extras))
91+
92+
(defn cleanup-sqlite-db! [config]
93+
(let [path (get-in config [:store :dbname])]
94+
(ensure-parent-dir! path)
95+
(delete-sqlite-files! path))
96+
(try
97+
(when (d/database-exists? config)
98+
(d/delete-database config))
99+
(catch Exception _))
100+
(delete-sqlite-files! (get-in config [:store :dbname])))
101+
102+
(defn bench-sqlite-write [n path]
103+
(let [config (sqlite-config path :write {:attribute-refs? true})
104+
entities (generate-entities n)]
105+
(ensure-parent-dir! path)
106+
(cleanup-sqlite-db! config)
107+
(try
108+
(d/create-database config)
109+
(let [conn (d/connect config)]
110+
(try
111+
(d/transact conn schema)
112+
(let [start (System/nanoTime)]
113+
(d/transact conn entities)
114+
(let [elapsed (/ (- (System/nanoTime) start) 1e9)]
115+
(datahike-row :sqlite-datahike :write n elapsed
116+
:entities-per-sec (/ n elapsed))))
117+
(finally
118+
(d/release conn))))
119+
(finally
120+
(cleanup-sqlite-db! config)))))
121+
122+
(defn bench-sqlite-query [n path]
123+
(let [config (sqlite-config path :query {})
124+
entities (generate-entities n)]
125+
(ensure-parent-dir! path)
126+
(cleanup-sqlite-db! config)
127+
(try
128+
(d/create-database config)
129+
(let [conn (d/connect config)]
130+
(try
131+
(d/transact conn schema)
132+
(d/transact conn entities)
133+
(let [start (System/nanoTime)
134+
result (d/q benchmark-query @conn)
135+
elapsed (/ (- (System/nanoTime) start) 1e9)]
136+
(datahike-row :sqlite-datahike :query n elapsed
137+
:results (count result)))
138+
(finally
139+
(d/release conn))))
140+
(finally
141+
(cleanup-sqlite-db! config)))))
142+
143+
(defn cleanup-dir! [path]
144+
(.mkdirs (io/file path))
145+
(delete-path (io/file path)))
146+
147+
(defn bench-datalevin-write [n path]
148+
(cleanup-dir! path)
149+
(let [conn (dl/get-conn path datalevin-schema)
150+
entities (generate-entities n)]
151+
(try
152+
(let [start (System/nanoTime)]
153+
(dl/transact! conn entities)
154+
(let [elapsed (/ (- (System/nanoTime) start) 1e9)]
155+
(datahike-row :datalevin :write n elapsed
156+
:entities-per-sec (/ n elapsed))))
157+
(finally
158+
(dl/close conn)
159+
(cleanup-dir! path)))))
160+
161+
(defn bench-datalevin-query [n path]
162+
(cleanup-dir! path)
163+
(let [conn (dl/get-conn path datalevin-schema)
164+
entities (generate-entities n)]
165+
(try
166+
(dl/transact! conn entities)
167+
(let [db (dl/db conn)
168+
start (System/nanoTime)
169+
result (dl/q benchmark-query db)
170+
elapsed (/ (- (System/nanoTime) start) 1e9)]
171+
(datahike-row :datalevin :query n elapsed
172+
:results (count result)))
173+
(finally
174+
(dl/close conn)
175+
(cleanup-dir! path)))))
176+
177+
(defn format-throughput [n elapsed]
178+
(format "%.0f" (/ n elapsed)))
179+
180+
(defn print-row [row]
181+
(let [{:keys [backend operation entities time-sec results entities-per-sec]} row]
182+
(println
183+
(format "| %-15s | %-6s | %8d | %10.3f | %8s | %12s |"
184+
(name backend)
185+
(name operation)
186+
entities
187+
time-sec
188+
(or results "-")
189+
(if entities-per-sec
190+
(format-throughput entities time-sec)
191+
"-")))))
192+
193+
(defn warmup! []
194+
(println "Warming up SQLite Datahike benchmark family...")
195+
(bench-sqlite-write warmup-size (str base-path "/warmup.sqlite"))
196+
(bench-datalevin-write warmup-size (str base-path "/warmup-datalevin"))
197+
(println "Warmup complete.")
198+
(println))
199+
200+
(defn collect-results
201+
[& {:keys [sizes mode]
202+
:or {sizes default-sizes
203+
mode :full}}]
204+
(let [sizes (if (= mode :smoke) smoke-sizes sizes)]
205+
(warmup!)
206+
{:benchmark :sqlite-datahike
207+
:mode mode
208+
:sizes sizes
209+
:rows (vec
210+
(mapcat
211+
(fn [n]
212+
[(bench-datalevin-write n (str base-path "/datalevin-write-" n))
213+
(bench-sqlite-write n (str base-path "/sqlite-write-" n ".sqlite"))
214+
(bench-datalevin-query n (str base-path "/datalevin-query-" n))
215+
(bench-sqlite-query n (str base-path "/sqlite-query-" n ".sqlite"))])
216+
sizes))}))
217+
218+
(defn render-results [{:keys [mode rows sizes]}]
219+
(println)
220+
(println "╔════════════════════════════════════════════════════════════╗")
221+
(println "║ SQLite Datahike Benchmark Suite ║")
222+
(println "╚════════════════════════════════════════════════════════════╝")
223+
(println)
224+
(println (format "Sizes: %s" sizes))
225+
(println (format "Mode: %s" (name mode)))
226+
(println "Comparable rows: SQLite Datahike and Datalevin")
227+
(println)
228+
(doseq [n sizes]
229+
(println (format "--- %d entities ---" n)))
230+
(println)
231+
(println "| Backend | Op | Entities | Time (sec) | Results | Entities/sec |")
232+
(println "|-----------------|--------|----------|------------|----------|--------------|")
233+
(doseq [row rows]
234+
(print-row row))
235+
(println)
236+
(println "Result rows (EDN)")
237+
(doseq [row rows]
238+
(prn row)))
239+
240+
(defn parse-args [args]
241+
(cond
242+
(empty? args) {}
243+
(= 1 (count args)) (edn/read-string (first args))
244+
:else (edn/read-string (apply str args))))
245+
246+
(defn write-output! [path text]
247+
(when path
248+
(let [file (io/file path)]
249+
(when-let [parent (.getParentFile file)]
250+
(.mkdirs parent))
251+
(spit file text))))
252+
253+
(defn -main [& args]
254+
(let [{:keys [output] :as opts} (parse-args args)
255+
results (apply collect-results (mapcat identity (dissoc opts :output)))
256+
report (with-out-str (render-results results))]
257+
(print report)
258+
(write-output! output report)
259+
(shutdown-agents)))

0 commit comments

Comments
 (0)