The Arroyo System Cache (aka arroyo-db
)
collects metadata from org-roam files
stored in the page as #+KEYWORD: value
keyword properties. These are inserted in to a SQLite database which can
then be queried by system generators.
Most of this exists in one form or another, I plan to take heavy inspiration from vulpea and vino when re-assembling the parts i have in my Shared CCE Helpers in a library with a more normalized schema than the simple kv pairs I was choosing to work with.
headmatter & db functions
"arroyo.db" user-emacs-directory)
(defcustom arroyo-db-location (expand-file-name "Arroyo System Cache location"
'arroyo)
:group
defvar arroyo-db--schemata nil
("A list of emacsql table schemas")
defvar arroyo-db--connection (make-hash-table :test #'equal)
("Database connection to `arroyo database.")
defvar arroyo-db-update-functions ()
("list of functions to run in the save hooks to update database tables")
Arroyo-db is largely cribbed from org-roam-db
as that has proven to be a capable
Hypermedia
platform for org-mode and to some
extent vino.
require 'emacsql-sqlite)
(require 'dash)
(
defun arroyo-db ()
("Entrypoint to the `arroyo' sqlite database.
Initializes and stores the database, and the database connection.
Performs a database upgrade when required."
unless (and (arroyo-db--get-connection)
(
(emacsql-live-p (arroyo-db--get-connection)))let ((init-db (not (file-exists-p arroyo-db-location))))
(t)
(make-directory (file-name-directory arroyo-db-location) let ((conn (emacsql-sqlite arroyo-db-location)))
(nil)
(set-process-query-on-exit-flag (emacsql-process conn)
(puthash (expand-file-name arroyo-db-location)
conn
arroyo-db--connection)when init-db
(
(arroyo-db--init conn)))))
(arroyo-db--get-connection))
defun arroyo-db--init (db)
("Initialize DB with the correct schema and user version."
"Initializing Arroyo Database")
(message
(emacsql-with-transaction db
(pcase-dolist (`(,table . ,schema) arroyo-db--schemata)
(condition-case err
(emacsql db [:create-table $i1 $S2] table schema)t nil)))
(format "PRAGMA user_version = \"%s\""
(emacsql db (
(arroyo-db--version)))))
defun arroyo-db--get-connection ()
("Return the database connection, if any."
gethash (expand-file-name arroyo-db-location)
(
arroyo-db--connection))
defun arroyo-db--close-all ()
("Closes all database connections made by Org-roam."
dolist (conn (hash-table-values arroyo-db--connection))
(
(arroyo-db--close conn)))
defun arroyo-db--close (&optional db)
("Closes the database connection for database DB."
unless db
(setq db (arroyo-db--get-connection)))
(when (and db (emacsql-live-p db))
(
(emacsql-close db)))
defun arroyo-db--upgrade-maybe (db version)
("Upgrades the database schema for DB, if VERSION is old."
(emacsql-with-transaction db'ignore
if (not (string-equal (format "%s" version) (arroyo-db--version)))
(progn
(
(arroyo-db--close)delete-file arroyo-db-location)
(
(arroyo-db)))) version)
to build the DB on save, we use an after-save-hook
:
defun arroyo-db--update-on-save-h ()
("Locally setup file update for `arroyo-db' system cache."
when (org-roam-file-p)
('after-save-hook #'arroyo-db-update-file nil 'local)))
(add-hook
'find-file-hook #'arroyo-db--update-on-save-h) (add-hook
defun arroyo-db-file-updated-p (file)
("Returns t if the file's hash does not match the one recorded in the Arroyo DB."
not (equal (caar (arroyo-db-query [:select hash :from files :where (= file $s1)] file))
(
(org-roam-db--file-hash file))))
defun arroyo-db-update-file-maybe (&optional file _no-require)
("Update cache for FILE. it will skip if this file's recorded hash matches the disk hash."
setq file (or file (buffer-file-name (buffer-base-buffer))))
(when (arroyo-db-file-updated-p file)
(
(arroyo-db-update-file file _no-require)))
defun arroyo-db-update-file (&optional file _no-require)
("Update cache for FILE."
setq file (or file (buffer-file-name (buffer-base-buffer))))
(nil
(org-roam-with-file file dolist (func arroyo-db-update-functions)
(funcall func))))
(
;(add-function :after (symbol-function 'org-roam-db-update-file) #'arroyo-db-update-file)
symbol-function 'org-roam-db-sync) #'arroyo-db-update-all-roam-files)
(add-function :after (
defun arroyo-db-update-all-roam-files (&optional force)
("P")
(interactive let ((gc-cons-threshold org-roam-db-gc-threshold))
(;; Force a reconnect
(arroyo-db--close) when force (delete-file arroyo-db-location))
(;; To initialize the database, no-op if already initialized
(arroyo-db)
(dolist-with-progress-reporter (file (org-roam-list-files))"Processing all files..."
(arroyo-db-update-file-maybe file))
(arroyo-db-cull-deleted-files)))
defun arroyo-db-cull-deleted-files ()
(
(interactive)
(->>
(arroyo-db-query [:select file :from keywords])#'car)
(-map #'file-exists-p)
(-remove apply #'vector)
( (arroyo-db-query [:delete :from keywords :where (in file $v1)])))
query interface is just emacsql, higher-level things will be built on that.
'emacsql-constraint "SQL constraint violation")
(define-error defun arroyo-db-query (sql &rest args)
("Run SQL query on arroyo database with ARGS.
SQL can be either the emacsql vector representation, or a string."
apply #'emacsql (arroyo-db) sql args))
(
defun arroyo-db-get (keyword &optional file)
("arroyo KEYWORD value, optionally for FILE"
if file
(#'car (arroyo-db-query [:select value :from keywords
(-map = keyword $s1)
:where (= file $s2)]
:and (keyword file))
(arroyo-db-query [:select [file value] :from keywords= keyword $s1)]
:where (keyword)))
defun arroyo-db-by-keyword (keyword value)
("arroyo files by KEYWORD+VALUE search"
#'car (arroyo-db-query [:select file :from keywords
(-map = keyword $s1)
:where (= value $s2)]
:and (keyword value)))
defun arroyo--files-for-role (role)
(
(->> role"ARROYO_NIXOS_ROLE")
(arroyo-db-by-keyword "ARROYO_NIXOS_MODULE" it))
(--filter (arroyo-db-get cons it (arroyo-db--get-file-title-from-org-roam it)))))
(--map (
defun arroyo-db--get-file-title-from-org-roam (file)
(
(->> file= level 0) :and (= file $s1)])
(org-roam-db-query [:select title :from nodes :where (caar))) (
CANCELLED Arroyo DB uses hashes, not incremental versioning
This list can be extended with other functions which take an
accumulator and hash their own state in to it. See arroyo-db--hash
below for example, or arroyo-db--keywords-hash
, etc. The goal with
this methodology is to be able to invalidate the cache when the
configuration changes. This might be a bad idea, in the case that the
configuration is invalid and i need the working configuration to restore
it. Secure Backup
Infrastructure or at least ZFS maybe saves me, I need to take
advantage of that more often!!
defvar arroyo-db-hash-input-functions
(
'(arroyo-db--hash)"A list of functions which take an accumulator and hash their
own configuraiton in to the accumulator")
defun arroyo-db--version ()
(lambda (first second)
(-reduce (funcall second first))
(;; starts with the accumulator
append '("") arroyo-db-hash-input-functions)))
(
defun arroyo-db--hash (acc)
("A function which mixes the databases schemata in to the arroyo version hash, used to invalidate the DB on change"
'sha256 (format "%s" (append acc arroyo-db-keywords)))) (secure-hash
NEXT still need a hash/cache-key system
auto-tangle files that are modified?
NEXT
the PRAGMA user_version
is an integer,
need to put the hash string in a better place.
NEXT this should probably have a minor mode eventually to manage the hooks. also need rename-file and delete-file adviceā¦
NEXT
deferred upgrade on schema change in (arroyo-db)
should block future ops
org-roam #+PROPERTY keyword caching
This keywords
table provides a simple
key-value store which can be further derived in to tables for smarter
applications like the Arroyo Emacs
Generator, etc. Included here is a hash function.
"BIRTHDAY" "PRONOUNS" "LOCATION")
(defcustom arroyo-db-keywords '("Keyword properties which will be stored in the org-roam db keywords table."
:type '(repeat string)
'arroyo)
:group
defun arroyo-db--keywords-hash (acc)
('sha256 (format "%s" (append acc arroyo-db-keywords))))
(secure-hash 'arroyo-db-hash-input-functions #'arroyo-db--keywords-hash) (add-to-list
It's a simple file-level-properties key/value table shaped like:
'arroyo-db--schemata
(add-to-list
'(keywords
[(file :not-null)keyword :not-null)
( (value :not-null)]))
And we cache the files' hash as well to compare against the org-roam database.
'arroyo-db--schemata
(add-to-list
'(files
[(file :not-null) (hash :not-null)]))
defun arroyo-db--record-file-hash (&optional update-p)
(
(pcase-let* ((file (buffer-file-name))= file $s1)] file))
(pair (org-roam-db-query [:select [file hash] :from files :where (first pair)))
(`(,file ,hash) (= file $s1)]
(arroyo-db-query [:delete :from files :where (
file)
(arroyo-db-query [:insert :into files :values $v1]apply #'vector it) pair))))
(--map ('arroyo-db-update-functions #'arroyo-db--record-file-hash)
(add-to-list
defun arroyo-db--extract-keywords (&optional file-path)
("Extract props specified in [`arroyo-db-keywords'] from current buffer and return the type and the key of the ref."
setq file-path (or file-path
(
(buffer-file-name)))
(save-excursionnil
(org-roam-with-file file-path when arroyo-db-keywords
(
(-filter#'cdr
(arroyo-db--extract-global-props arroyo-db-keywords))))))
defun arroyo-db--insert-keywords (&optional update-p)
("Insert KEYWORDS for current buffer into the Org-roam cache.
If UPDATE-P is non-nil, first remove keywords for the file in the database.
Return the number of rows inserted."
let* ((file (buffer-file-name))
(
(keywords (arroyo-db--extract-keywords file)))
(arroyo-db-query [:delete :from keywords= file $s1)]
:where (
file)if keywords
(progn
(
(arroyo-db-query
[:insert :into keywords
:values $v1]maplist (lambda (keyword)
(vector file (caar keyword) (cdar keyword)))
(
keywords))1)
0)))
'arroyo-db-update-functions #'arroyo-db--insert-keywords) (add-to-list
functions from org-roam v1 to extract file-level properties
defun arroyo-db--collect-keywords (keywords)
("Collect all Org KEYWORDS in the current buffer."
if (functionp 'org-collect-keywords)
(
(org-collect-keywords keywords)let ((buf (org-element-parse-buffer))
(
res)dolist (k keywords)
(let ((p (org-element-map buf 'keyword
(lambda (kw)
(when (string-equal (org-element-property :key kw) k)
(
(org-element-property :value kw)))nil)))
:first-match push (cons k p) res)))
(
res)))
defun arroyo-db--extract-global-props-keyword (keywords)
("Extract KEYWORDS from the current Org buffer."
let (ret)
(values) (arroyo-db--collect-keywords keywords))
(pcase-dolist (`(,key . ,dolist (value values)
(push (cons key value) ret)))
(
ret))
defun arroyo-db--extract-global-props-drawer (props)
("Extract PROPS from the file-level property drawer in Org."
let (ret)
(1
(org-with-point-at dolist (prop props ret)
(
(when-let ((v (org-entry-get (point) prop)))push (cons prop v) ret))))))
(
defun arroyo-db--extract-global-props (props)
("Extract PROPS from the current Org buffer.
Props are extracted from both the file-level property drawer (if
any), and Org keywords. Org keywords take precedence."
append
(
(arroyo-db--extract-global-props-keyword props) (arroyo-db--extract-global-props-drawer props)))
Footmatter
This is arroyo-db.el
and can be (require 'arroyo-db)
'd
provide 'arroyo-db) (