├── .clj-kondo ├── babashka │ ├── fs │ │ └── config.edn │ └── sci │ │ ├── config.edn │ │ └── sci │ │ └── core.clj ├── http-kit │ └── http-kit │ │ ├── config.edn │ │ └── httpkit │ │ └── with_channel.clj ├── rewrite-clj │ └── rewrite-clj │ │ └── config.edn └── taoensso │ └── encore │ ├── config.edn │ └── taoensso │ └── encore.clj ├── .gitignore ├── ARCHITECTURE.md ├── CHANGELOG.md ├── LICENSE ├── README.md ├── demos ├── clojuredocs │ └── README.md ├── code-analysis │ └── README.md ├── email │ └── README.md ├── max-temps │ └── README.md ├── multiplayer │ └── README.md ├── nutrition │ ├── FoodData_Central_foundation_food_csv_2023-10-26 │ │ ├── acquisition_samples.csv │ │ ├── agricultural_samples.csv │ │ ├── food.csv │ │ ├── food_attribute.csv │ │ ├── food_attribute_type.csv │ │ ├── food_calorie_conversion_factor.csv │ │ ├── food_component.csv │ │ ├── food_nutrient.csv │ │ ├── food_nutrient_conversion_factor.csv │ │ ├── food_portion.csv │ │ ├── food_protein_conversion_factor.csv │ │ ├── food_update_log_entry.csv │ │ ├── foundation_food.csv │ │ ├── input_food.csv │ │ ├── lab_method.csv │ │ ├── lab_method_code.csv │ │ ├── lab_method_nutrient.csv │ │ ├── market_acquisition.csv │ │ ├── measure_unit.csv │ │ ├── nutrient.csv │ │ ├── sample_food.csv │ │ ├── sub_sample_food.csv │ │ └── sub_sample_result.csv │ ├── README.md │ ├── dataDictionary.pdf │ ├── query.fdb.edn │ └── repl.fdb.clj └── reference │ ├── call-arg.edn │ ├── call-spec.edn │ ├── doc.md │ ├── doc.md.meta.edn │ ├── fdbconfig.edn │ ├── pattern-glob-match.md │ ├── query.fdb.edn │ ├── query.fdb.md │ ├── ref-one.md.meta.edn │ ├── ref-three.md.meta.edn │ ├── ref-two.md.meta.edn │ ├── repl.fdb.clj │ ├── repl.fdb.md │ ├── todo.md │ └── todo.md.meta.edn ├── deps.edn ├── resources ├── email │ └── sample-crlf.mbox ├── eml │ ├── sample-crlf │ │ ├── 1970-01-01T00.00.00Z 8d247ee6 Sample message 1.eml │ │ └── 1970-01-01T00.00.00Z c2dfc80c Sample message 2.eml │ └── sample.eml ├── file.txt └── md │ ├── another file.md │ ├── file.md │ ├── inbox │ └── another file.md │ ├── md link.md │ └── other file.md ├── src └── fdb │ ├── autoloader.clj │ ├── bb │ ├── bb.edn │ └── cli.clj │ ├── call.clj │ ├── config.clj │ ├── core.clj │ ├── db.clj │ ├── email.clj │ ├── http.clj │ ├── mac.clj │ ├── metadata.clj │ ├── readers.clj │ ├── readers │ ├── edn.clj │ ├── eml.clj │ ├── json.clj │ └── md.clj │ ├── triggers.clj │ ├── utils.clj │ └── watcher.clj ├── symlink-fdb.sh ├── talks └── 2024-05-london-clojurians.iapresenter │ ├── info.json │ ├── text.md │ └── thumb.png └── test └── fdb ├── call_test.clj ├── core_test.clj ├── db_test.clj ├── email_test.clj ├── http_test.clj ├── metadata_test.clj ├── readers ├── edn_test.clj ├── eml_test.clj └── md_test.clj ├── readers_test.clj ├── triggers_test.clj ├── utils_test.clj └── watcher_test.clj /.clj-kondo/babashka/fs/config.edn: -------------------------------------------------------------------------------- 1 | {:lint-as {babashka.fs/with-temp-dir clojure.core/let}} 2 | -------------------------------------------------------------------------------- /.clj-kondo/babashka/sci/config.edn: -------------------------------------------------------------------------------- 1 | {:hooks {:macroexpand {sci.core/copy-ns sci.core/copy-ns}}} 2 | -------------------------------------------------------------------------------- /.clj-kondo/babashka/sci/sci/core.clj: -------------------------------------------------------------------------------- 1 | (ns sci.core) 2 | 3 | (defmacro copy-ns 4 | ([ns-sym sci-ns] 5 | `(copy-ns ~ns-sym ~sci-ns nil)) 6 | ([ns-sym sci-ns opts] 7 | `[(quote ~ns-sym) 8 | ~sci-ns 9 | (quote ~opts)])) 10 | -------------------------------------------------------------------------------- /.clj-kondo/http-kit/http-kit/config.edn: -------------------------------------------------------------------------------- 1 | 2 | {:hooks 3 | {:analyze-call {org.httpkit.server/with-channel httpkit.with-channel/with-channel}}} 4 | -------------------------------------------------------------------------------- /.clj-kondo/http-kit/http-kit/httpkit/with_channel.clj: -------------------------------------------------------------------------------- 1 | (ns httpkit.with-channel 2 | (:require [clj-kondo.hooks-api :as api])) 3 | 4 | (defn with-channel [{node :node}] 5 | (let [[request channel & body] (rest (:children node))] 6 | (when-not (and request channel) (throw (ex-info "No request or channel provided" {}))) 7 | (when-not (api/token-node? channel) (throw (ex-info "Missing channel argument" {}))) 8 | (let [new-node 9 | (api/list-node 10 | (list* 11 | (api/token-node 'let) 12 | (api/vector-node [channel (api/vector-node [])]) 13 | request 14 | body))] 15 | 16 | {:node new-node}))) 17 | -------------------------------------------------------------------------------- /.clj-kondo/rewrite-clj/rewrite-clj/config.edn: -------------------------------------------------------------------------------- 1 | {:lint-as 2 | {rewrite-clj.zip/subedit-> clojure.core/-> 3 | rewrite-clj.zip/subedit->> clojure.core/->> 4 | rewrite-clj.zip/edit-> clojure.core/-> 5 | rewrite-clj.zip/edit->> clojure.core/->>}} 6 | -------------------------------------------------------------------------------- /.clj-kondo/taoensso/encore/config.edn: -------------------------------------------------------------------------------- 1 | {:hooks {:analyze-call {taoensso.encore/defalias taoensso.encore/defalias}}} 2 | -------------------------------------------------------------------------------- /.clj-kondo/taoensso/encore/taoensso/encore.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.encore 2 | (:require 3 | [clj-kondo.hooks-api :as hooks])) 4 | 5 | (defn defalias [{:keys [node]}] 6 | (let [[sym-raw src-raw] (rest (:children node)) 7 | src (if src-raw src-raw sym-raw) 8 | sym (if src-raw 9 | sym-raw 10 | (symbol (name (hooks/sexpr src))))] 11 | {:node (with-meta 12 | (hooks/list-node 13 | [(hooks/token-node 'def) 14 | (hooks/token-node (hooks/sexpr sym)) 15 | (hooks/token-node (hooks/sexpr src))]) 16 | (meta src))})) 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | *.log 4 | tmp/ 5 | 6 | /target 7 | /classes 8 | /checkouts 9 | profiles.clj 10 | pom.xml 11 | pom.xml.asc 12 | *.jar 13 | *.class 14 | /.lein-* 15 | /.nrepl-port 16 | /.prepl-port 17 | /.cpcache 18 | /.clj-kondo/.* 19 | /.clj-kondo/inline-configs 20 | /tmp 21 | -------------------------------------------------------------------------------- /ARCHITECTURE.md: -------------------------------------------------------------------------------- 1 | # Architecture 2 | 3 | TODO 4 | 5 | https://matklad.github.io/2021/02/06/ARCHITECTURE.md.html 6 | 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # unreleased 2 | 3 | # 1.1.0 - 2024-05-28 4 | 5 | - simpler first demo, moved current ones to more-demos 6 | - hopefully makes you feel like you're getting superpowers 7 | - thanks to @escherize for the feedback! 8 | - added default `load-repl.fdb.clj` and `server-repl.fdb.clj` to load vec 9 | - felt it would be nice to have a place for loaded and server stuff 10 | - email send/sync/split-mbox 11 | - mac notification 12 | - removed mentions of first class `:fdb/tags` 13 | - nothing really used them, and you can add whatever key you want for it 14 | - shell triggers run on parent dir 15 | 16 | # 1.0.0 - 2024-04-03 17 | 18 | - First version 19 | - It's whats on the README 20 | 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [year] [fullname] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /demos/clojuredocs/README.md: -------------------------------------------------------------------------------- 1 | # ClojureDocs 2 | 3 | This demo uses a `fdb.on/modify` [metadata trigger](../../README.md#metadata) to search in [ClojureDocs](https://clojuredocs.org) whenever a file changes, and writes the scraped results to another file: 4 | 5 | Start by making `~/fdb/user/clojuredocs.txt`. 6 | This is where you'll write the query in. 7 | 8 | ```sh 9 | echo "reduce" > ~/fdb/user/clojuredocs.txt 10 | ``` 11 | 12 | Then add metadata with the trigger. 13 | This trigger will: 14 | - call https://clojuredocs.org/search with the file contents in the `q` param 15 | - scrape the response for `li` elements with the `arglist` class 16 | - write them to `clojuredocs-out.txt` 17 | 18 | ```sh 19 | echo ' 20 | {:fdb.on/modify 21 | (fn [{:keys [self-path]}] 22 | (-> "https://clojuredocs.org/search" 23 | (fdb.http/add-params {:q (slurp self-path)}) 24 | (fdb.http/scrape [:li.arglist]) 25 | (->> (mapcat :content) 26 | (clojure.string/join "\n") 27 | (spit (fdb.utils/sibling-path self-path "clojuredocs-out.txt")))))} 28 | ' > ~/fdb/user/clojuredocs.txt.meta.edn 29 | ``` 30 | 31 | Read the results 32 | ```sh 33 | cat ~/fdb/user/clojuredocs-out.txt 34 | ``` 35 | 36 | ```sh 37 | (reduce f coll) 38 | (reduce f val coll) 39 | (reduced x) 40 | (reduce f init ch) 41 | (reducer coll xf) 42 | (reduce f coll) 43 | (reduce f init coll) 44 | (reduced? x) 45 | (reduce-kv f init coll) 46 | (kv-reduce amap f init) 47 | (coll-reduce coll f) 48 | (coll-reduce coll f val) 49 | (ensure-reduced x) 50 | ``` 51 | 52 | Search for something else. 53 | 54 | ```sh 55 | # search for something else 56 | echo "map" > ~/fdb/user/clojuredocs.txt 57 | cat ~/fdb/user/clojuredocs-out.txt 58 | ``` -------------------------------------------------------------------------------- /demos/code-analysis/README.md: -------------------------------------------------------------------------------- 1 | # Code Analysis 2 | 3 | TODO: 4 | - reader for clj files that gets their deps by fn 5 | - profit 6 | -------------------------------------------------------------------------------- /demos/email/README.md: -------------------------------------------------------------------------------- 1 | # Email 2 | 3 | TODO: 4 | - mbox split should be on change or called from repl, not on read 5 | - eml reader 6 | - gmail sync 7 | -------------------------------------------------------------------------------- /demos/max-temps/README.md: -------------------------------------------------------------------------------- 1 | # Temps 2 | 3 | This demo uses a `fdb.on/schedule` [metadata trigger](../../README.md#metadata) to: 4 | - each day 5 | - look up temperatures in Lisbon for the past and future 7 days 6 | - load them into FileDB by writing them to `.edn` files on disk that we have [readers](../../README.md#readers) for 7 | - then query the db using a [query file](../../README.md#repl-and-query-files) that calls a function we just defined in a [repl file](../../README.md#repl-and-query-files) 8 | 9 | Start by adding the schedule trigger. 10 | 11 | ``` sh 12 | # temperature tracker 13 | echo ' 14 | {:fdb.on/schedule 15 | {:every [1 :days] 16 | :call (fn [{:keys [self-path]}] 17 | (let [lisbon (-> "https://nominatim.openstreetmap.org/search" 18 | (fdb.http/add-params {:q "Lisbon" :limit 1 :format "json"}) 19 | fdb.http/json 20 | first) 21 | forecast (-> "https://api.open-meteo.com/v1/forecast" 22 | (fdb.http/add-params {:daily ["temperature_2m_max", "temperature_2m_min"] , 23 | :past_days 7 24 | :latitude (:lat lisbon) 25 | :longitude (:lon lisbon)}) 26 | fdb.http/json 27 | :daily) 28 | temps (map (fn [day max min] 29 | {:day day 30 | :max max 31 | :min min}) 32 | (:time forecast) 33 | (:temperature_2m_max forecast) 34 | (:temperature_2m_min forecast))] 35 | (run! (fn [temp] 36 | (fdb.utils/spit-edn 37 | (fdb.utils/sibling-path self-path (str "weather/" (:day temp) ".edn")) 38 | temp)) 39 | temps)))}} 40 | ' > ~/fdb/user/weather.edn 41 | ``` 42 | 43 | This will create edn files in `~/fdb/user/weather` for min/max temp, updated every day, for previous and next 7 days 44 | 45 | ```sh 46 | ll ~/fdb/user/weather 47 | ``` 48 | 49 | ```sh 50 | total 112 51 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-11.edn 52 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-12.edn 53 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-13.edn 54 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-14.edn 55 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-15.edn 56 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-16.edn 57 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-17.edn 58 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-18.edn 59 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-19.edn 60 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-20.edn 61 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-21.edn 62 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-22.edn 63 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-23.edn 64 | -rw-r--r-- 1 filipesilva staff 40B May 18 21:44 2024-05-24.edn 65 | ``` 66 | 67 | These files are loaded into FileDB automatically because they are edn files. 68 | Their data looks like `{:day "2024-05-11" :max 23.6 :min 15.7}`. 69 | 70 | Now we can query for it. 71 | The [tick](https://github.com/juxt/tick) library is already included in FileDB. 72 | 73 | What are the max temperatures like the week around today? 74 | Lets make a date that checks if a given date is in the current week, then use it in a query file. 75 | 76 | ```sh 77 | echo ' 78 | (require \'[tick.core :as t]) 79 | (defn this-week? [date] 80 | (let [today (t/date)] 81 | (t/<= (t/<< today (t/of-days 3)) 82 | (t/date date) 83 | (t/>> today (t/of-days 3))))) 84 | ' >> ~/fdb/user/load-repl.fdb.clj 85 | echo ' 86 | {:find [?day ?max] 87 | :where [[?e :fdb/parent "/user/weather"] 88 | [?e :day ?day] 89 | [(user/this-week? ?day)] 90 | [?e :max ?max]]} 91 | ' > ~/fdb/user/week-max-temp.query.fdb.edn 92 | 93 | cat ~/fdb/user/week-max-temp.query-out.fdb.edn 94 | ``` 95 | 96 | ```clojure 97 | #{["2024-05-15" 19.2] 98 | ["2024-05-16" 19.9] 99 | ["2024-05-17" 20.6] 100 | ["2024-05-18" 20.4] 101 | ["2024-05-19" 20.6] 102 | ["2024-05-20" 21.0] 103 | ["2024-05-21" 20.4]} 104 | ``` 105 | 106 | Just `touch ~/fdb/user/week-max-temp.query.fdb.edn` to run the query again. 107 | Or add a schedule to do it automatically. 108 | 109 | ```sh 110 | echo ' 111 | {:fdb.on/schedule {:every [1 :days] 112 | :call [:sh "touch" "week-max-temp.query.fdb.edn"]}} 113 | ' > ~/fdb/user/touch-query.edn 114 | ``` 115 | 116 | Fun fact: if you added this trigger on `touch ~/fdb/user/week-max-temp.query.fdb.edn.meta.edn` you'd make a infinite loop. 117 | Touch would load both query and metatada, and schedule changes are immediately triggered, which would touch again, etc. 118 | So maybe don't. 119 | But if you do, just stop the watch process, remove the problematic files, and then start it again. 120 | -------------------------------------------------------------------------------- /demos/multiplayer/README.md: -------------------------------------------------------------------------------- 1 | 2 | TODO: 3 | - Multiplayer fdb probably works well through sync thing or git. 4 | - Easy to schedule sync pulls. But unlike normal pulls, fdb will sync db and env state. Because it's a live env. 5 | - Make a multiplayer demo for this! 6 | -------------------------------------------------------------------------------- /demos/nutrition/FoodData_Central_foundation_food_csv_2023-10-26/food_attribute_type.csv: -------------------------------------------------------------------------------- 1 | "id","name","description" 2 | "998","Update Log","Changes that were made to this food" 3 | "999","Attribute","Generic attributes" 4 | "1000","Common Name","Common names associated with a food." 5 | "1001","Additional Description","Additional descriptions for the food." 6 | "1002","Adjustments","Adjustments made to foods, including moisture changes" 7 | -------------------------------------------------------------------------------- /demos/nutrition/FoodData_Central_foundation_food_csv_2023-10-26/food_calorie_conversion_factor.csv: -------------------------------------------------------------------------------- 1 | "food_nutrient_conversion_factor_id","protein_value","fat_value","carbohydrate_value" 2 | "22503","3.47","8.37","4.07" 3 | "22505","4.27","8.79","3.87" 4 | "22507","2.44","8.37","3.57" 5 | "22510","2.44","8.37","3.57" 6 | "22512","2.44","8.37","3.57" 7 | "22514","4.27","8.79","3.87" 8 | "22516","4.27","8.79","3.87" 9 | "22518","4.27","8.79","3.87" 10 | "22520","4.27","9.02","3.87" 11 | "22522","3.47","8.37","4.07" 12 | "22524","4.27","8.79","3.87" 13 | "22526","2.44","8.37","3.57" 14 | "22528","4.36","9.02","3.68" 15 | "22530","4.36","9.02","3.68" 16 | "22532","4.36","9.02","3.68" 17 | "22534","2.44","8.87","3.57" 18 | "22536","4.27","9.02","3.87" 19 | "22538","3.7","8.8","4.0" 20 | "22540","2.44","8.37","3.57" 21 | "22542","3.47","8.37","4.07" 22 | "22544","4.27","8.79","3.87" 23 | "22546","4.27","8.79","3.87" 24 | "22548","3.36","8.37","3.92" 25 | "22550","3.36","8.37","3.6" 26 | "22552","3.47","8.37","4.07" 27 | "22554","4.27","9.02","3.87" 28 | "22556","3.9","8.7","4.1" 29 | "22558","4.27","9.02","3.87" 30 | "22560","4.27","8.79","3.87" 31 | "22562","2.44","8.37","3.57" 32 | "22564","2.78","8.37","3.84" 33 | "22567","3.47","8.37","3.34" 34 | "22568","3.36","8.37","3.6" 35 | "22570","3.36","8.37","3.6" 36 | "22572","3.36","8.37","3.6" 37 | "22574","3.36","8.37","3.6" 38 | "22576","3.36","8.37","3.6" 39 | "22578","3.36","8.37","3.6" 40 | "22580","2.44","8.37","3.57" 41 | "22582","4.27","8.79","3.87" 42 | "22584","4.27","8.79","3.87" 43 | "22586","4.27","8.79","3.87" 44 | "22588","4.36","9.02","3.68" 45 | "22590","4.36","9.02","3.68" 46 | "22592","4.36","9.02","3.68" 47 | "22594","4.27","8.79","3.87" 48 | "22596","4.27","8.79","3.87" 49 | "22598","4.27","8.79","3.87" 50 | "22600","3.47","8.37","4.07" 51 | "22602","4.27","9.02","3.87" 52 | "22604","4.27","9.02","3.87" 53 | "22606","4.27","9.02","3.87" 54 | "22608","2.44","8.37","3.57" 55 | "22610","4.27","9.02","3.87" 56 | "22612","3.36","8.37","3.6" 57 | "22614","3.36","8.37","3.6" 58 | "22616","4.27","9.02","3.87" 59 | "22618","4.0","9.0","4.0" 60 | "22620","2.44","8.37","3.57" 61 | "22622","4.27","9.02","3.87" 62 | "22624","4.27","9.02","3.87" 63 | "22626","4.27","9.02","3.87" 64 | "22628","0.0","0.0","3.87" 65 | "22631","4.0","9.0","4.0" 66 | "22632","4.0","9.0","4.0" 67 | "22634","4.0","9.0","4.0" 68 | "22636","4.0","9.0","4.0" 69 | "22638","4.27","9.02","3.87" 70 | "22640","4.27","9.02","3.87" 71 | "22642","4.27","9.02","3.87" 72 | "22644","4.27","9.02","3.87" 73 | "22646","4.27","9.02","3.87" 74 | "22648","4.27","9.02","3.87" 75 | "22650","4.0","9.0","4.0" 76 | "22653","4.27","9.02","3.87" 77 | "22655","4.27","9.02","3.87" 78 | "22657","4.27","9.02","3.87" 79 | "22659","4.27","9.02","3.87" 80 | "22661","4.27","9.02","3.87" 81 | "22663","4.27","9.02","3.87" 82 | "22665","2.78","8.37","3.84" 83 | "22667","4.27","8.79","3.87" 84 | "22669","4.27","8.79","3.87" 85 | "22671","4.27","8.79","3.87" 86 | "22673","3.36","8.37","3.6" 87 | "22675","2.44","8.37","3.57" 88 | "22677","3.36","8.37","3.6" 89 | "22679","3.36","8.37","3.6" 90 | "22681","4.27","8.79","3.87" 91 | "22683","3.36","8.37","3.6" 92 | "22686","4.0","9.0","4.0" 93 | "22688","4.27","8.79","3.87" 94 | "22690","2.44","8.87","3.57" 95 | "22692","4.27","8.79","3.87" 96 | "22694","4.27","9.02","3.87" 97 | "22696","4.27","9.02","3.87" 98 | "22698","4.27","9.02","3.87" 99 | "22700","4.27","8.79","3.87" 100 | "22702","4.27","9.02","3.87" 101 | "22704","0.0","0.0","3.87" 102 | "22706","4.27","9.02","3.87" 103 | "22721","4.27","9.02","3.87" 104 | "22722","4.27","8.79","3.87" 105 | "22709","2.44","8.37","3.57" 106 | "22711","3.36","8.37","3.6" 107 | "22723","2.44","8.37","3.57" 108 | "22713","4.36","9.02","3.68" 109 | "22715","4.36","9.02","3.68" 110 | "22719","4.36","9.02","3.68" 111 | "22724","4.27","9.02","3.87" 112 | "22725","4.05","8.37","4.12" 113 | "22726","4.05","8.37","4.12" 114 | "22727","4.05","8.37","4.12" 115 | "22728","3.59","8.37","3.78" 116 | "22729","4.05","8.37","4.12" 117 | "22730","3.82","8.37","4.16" 118 | "22731","3.46","8.37","4.16" 119 | "22740","2.78","8.37","3.84" 120 | "22737","3.47","8.37","4.07" 121 | "22738","3.47","8.37","4.07" 122 | "22735","3.41","8.37","4.12" 123 | "22736","3.82","8.37","4.16" 124 | "22734","4.05","8.37","4.12" 125 | "22739","2.78","8.37","3.84" 126 | "22732","3.36","8.37","3.6" 127 | "22733","3.36","8.37","3.6" 128 | "22741","3.36","8.37","3.6" 129 | "22745","3.36","8.37","3.6" 130 | "22744","3.36","8.37","3.6" 131 | "22743","3.36","8.37","3.6" 132 | "22742","3.36","8.37","3.6" 133 | "22764","4.0","9.0","4.0" 134 | "22765","4.0","9.0","4.0" 135 | "22766","3.36","8.37","3.6" 136 | "22767","3.36","8.37","3.6" 137 | "22768","3.36","8.37","3.6" 138 | "22769","3.36","8.37","3.6" 139 | "22770","3.36","8.37","3.6" 140 | "22771","2.62","8.37","3.48" 141 | "22772","2.62","8.37","3.48" 142 | "22773","2.62","8.37","3.48" 143 | "22774","2.62","8.37","3.48" 144 | "22775","4.0","8.84","4.0" 145 | "22776","4.0","8.84","4.0" 146 | "22777","4.0","8.84","4.0" 147 | "22778","4.0","8.84","4.0" 148 | "22779","2.44","8.37","3.57" 149 | "22780","2.44","8.37","3.57" 150 | "22781","2.44","8.37","3.57" 151 | "22796","2.62","8.37","3.48" 152 | "22797","2.62","8.37","3.48" 153 | "22798","2.62","8.37","3.48" 154 | "22799","2.62","8.37","3.48" 155 | "22800","4.0","9.0","4.0" 156 | "22801","4.0","9.0","4.0" 157 | "22802","2.44","8.37","3.57" 158 | "22803","2.44","8.37","3.57" 159 | "22804","2.44","8.37","3.57" 160 | "22805","4.05","8.37","4.12" 161 | "22806","3.59","8.37","3.78" 162 | "22807","4.05","8.37","4.12" 163 | "22808","4.05","8.37","4.12" 164 | "22809","3.36","8.37","3.92" 165 | "22810","3.36","8.37","3.92" 166 | "22811","3.36","8.37","3.92" 167 | "22812","3.36","8.37","3.92" 168 | "22813","3.36","8.37","3.92" 169 | "22814","3.36","8.37","3.92" 170 | "22815","2.44","8.37","3.57" 171 | "22816","3.36","8.37","3.92" 172 | "22817","2.62","8.37","3.48" 173 | "22818","2.62","8.37","3.48" 174 | "22819","2.62","8.37","3.48" 175 | "22820","2.62","8.37","3.48" 176 | "22821","2.62","8.37","3.48" 177 | "22822","2.62","8.37","3.48" 178 | "22823","2.62","8.37","3.48" 179 | "22852","0.0","0.0","0.0" 180 | "22853","0.0","0.0","0.0" 181 | "22854","0.0","0.0","0.0" 182 | "22858","2.78","8.37","3.84" 183 | "22859","2.78","8.37","3.84" 184 | "22860","2.44","8.37","3.57" 185 | "22861","2.44","8.37","3.57" 186 | "22862","2.44","8.37","3.57" 187 | "22863","2.44","8.37","3.57" 188 | "22870","4.27","8.79","3.87" 189 | "22871","4.27","8.79","3.87" 190 | "22872","4.27","8.79","3.87" 191 | "22873","4.27","8.79","3.87" 192 | "22874","4.27","8.79","3.87" 193 | "22880","3.47","8.37","4.07" 194 | "22881","3.46","8.37","4.12" 195 | "22882","2.78","8.37","4.03" 196 | "22886","3.47","8.37","4.07" 197 | "22887","3.47","8.37","4.07" 198 | "22888","3.47","8.37","4.07" 199 | "22889","3.47","8.37","4.07" 200 | "22894","3.36","8.37","3.6" 201 | "22895","3.36","8.37","3.6" 202 | "22896","3.36","8.37","3.6" 203 | "22897","3.36","8.37","3.6" 204 | "22898","3.36","8.37","3.6" 205 | "22899","3.36","8.37","3.6" 206 | "22906","4.27","8.79","3.87" 207 | "22907","4.27","8.79","3.87" 208 | "22908","4.27","8.79","3.87" 209 | "22909","4.27","8.79","3.87" 210 | "22910","2.44","8.37","3.57" 211 | "22911","2.44","8.37","3.57" 212 | "22912","2.44","8.37","3.57" 213 | "22913","2.44","8.37","3.57" 214 | "22914","3.47","8.37","4.07" 215 | "22915","3.47","8.37","4.07" 216 | "22916","3.47","8.37","4.07" 217 | "22917","3.47","8.37","4.07" 218 | "22918","3.46","8.37","4.12" 219 | "22919","3.46","8.37","4.12" 220 | "22920","3.36","8.37","3.6" 221 | "22921","3.36","8.37","3.6" 222 | "22922","2.44","8.37","3.57" 223 | "22923","2.78","8.37","4.03" 224 | "22924","2.78","8.37","4.03" 225 | "22925","2.78","8.37","4.03" 226 | "22926","2.78","8.37","4.03" 227 | "22927","2.44","8.37","3.7" 228 | "22928","2.44","8.37","3.7" 229 | "22929","2.44","8.37","3.7" 230 | "22930","2.44","8.37","3.7" 231 | "22931","3.36","8.37","3.6" 232 | "22932","3.36","8.37","3.6" 233 | "22933","3.36","8.37","3.6" 234 | "22934","3.36","8.37","3.6" 235 | "22935","3.36","8.37","3.6" 236 | "22936","3.36","8.37","3.6" 237 | "22968","3.47","8.37","4.07" 238 | "22969","3.37","8.37","4.07" 239 | "22970","2.28","8.37","4.07" 240 | "22971","3.37","8.37","3.78" 241 | "22972","3.23","8.37","3.99" 242 | "22973","3.55","8.37","3.95" 243 | "22974","2.78","8.37","4.03" 244 | "22975","3.37","8.37","3.78" 245 | "22976","3.87","8.37","4.12" 246 | "22977","3.41","8.37","4.12" 247 | "22978","3.82","8.37","4.16" 248 | "22990","4.27","9.02","3.87" 249 | "22991","4.27","9.02","3.87" 250 | "22992","4.27","9.02","3.87" 251 | "22993","4.27","9.02","3.87" 252 | "22994","4.27","9.02","3.87" 253 | "23000","3.47","8.37","4.07" 254 | "23001","3.47","8.37","4.07" 255 | "23002","3.47","8.37","4.07" 256 | "23003","3.47","8.37","4.07" 257 | "23004","3.47","8.37","4.07" 258 | "23005","3.47","8.37","4.07" 259 | "23006","3.47","8.37","4.07" 260 | "23007","3.47","8.37","4.07" 261 | "23008","3.47","8.37","4.07" 262 | "23009","3.47","8.37","4.07" 263 | "23020","3.47","8.37","4.07" 264 | "23021","3.47","8.37","4.07" 265 | "23022","3.47","8.37","4.07" 266 | "23023","3.47","8.37","4.07" 267 | "23024","3.47","8.37","4.07" 268 | "23025","3.47","8.37","4.07" 269 | "23026","3.47","8.37","4.07" 270 | "23027","3.47","8.37","4.07" 271 | "23028","3.47","8.37","4.07" 272 | "23029","3.47","8.37","4.07" 273 | "23030","3.47","8.37","4.07" 274 | "23031","3.47","8.37","4.07" 275 | "23032","3.47","8.37","4.07" 276 | "23033","3.47","8.37","4.07" 277 | "23048","4.27","9.02","3.87" 278 | "23049","4.27","9.02","3.87" 279 | "23050","4.27","9.02","3.87" 280 | "23051","4.27","9.02","3.87" 281 | "23052","4.27","9.02","3.87" 282 | "23053","4.27","9.02","3.87" 283 | "23054","4.27","9.02","3.87" 284 | "23055","4.27","9.02","3.87" 285 | "23064","4.27","8.79","3.87" 286 | "23065","4.27","8.79","3.87" 287 | "23066","4.27","8.79","3.87" 288 | "23067","4.27","8.79","3.87" 289 | "23068","4.27","8.79","3.87" 290 | "23069","4.27","8.79","3.87" 291 | "23070","4.27","8.79","3.87" 292 | -------------------------------------------------------------------------------- /demos/nutrition/FoodData_Central_foundation_food_csv_2023-10-26/food_protein_conversion_factor.csv: -------------------------------------------------------------------------------- 1 | "food_nutrient_conversion_factor_id","value" 2 | "22504","6.25" 3 | "22506","6.38" 4 | "22508","6.25" 5 | "22509","0.0" 6 | "22511","6.25" 7 | "22513","6.25" 8 | "22515","6.38" 9 | "22517","6.38" 10 | "22519","6.38" 11 | "22521","6.25" 12 | "22523","5.18" 13 | "22525","6.38" 14 | "22527","6.25" 15 | "22529","6.25" 16 | "22531","6.25" 17 | "22533","6.25" 18 | "22535","6.25" 19 | "22537","6.25" 20 | "22539","6.07" 21 | "22541","6.25" 22 | "22543","5.46" 23 | "22545","6.38" 24 | "22547","6.38" 25 | "22549","6.25" 26 | "22551","6.25" 27 | "22553","5.3" 28 | "22555","6.25" 29 | "22557","6.25" 30 | "22559","6.25" 31 | "22561","6.38" 32 | "22563","6.25" 33 | "22565","6.25" 34 | "22566","6.25" 35 | "22569","6.25" 36 | "22571","6.25" 37 | "22573","6.25" 38 | "22575","6.25" 39 | "22577","6.25" 40 | "22579","6.25" 41 | "22581","6.25" 42 | "22583","6.38" 43 | "22585","6.38" 44 | "22587","6.38" 45 | "22589","6.25" 46 | "22591","6.25" 47 | "22593","6.25" 48 | "22595","6.38" 49 | "22597","6.38" 50 | "22599","6.38" 51 | "22601","6.25" 52 | "22603","6.25" 53 | "22605","6.25" 54 | "22607","6.25" 55 | "22609","6.25" 56 | "22611","6.25" 57 | "22613","6.25" 58 | "22615","6.25" 59 | "22617","6.25" 60 | "22619","6.25" 61 | "22621","6.25" 62 | "22623","6.25" 63 | "22625","6.25" 64 | "22627","6.25" 65 | "22629","6.25" 66 | "22630","6.25" 67 | "22633","6.25" 68 | "22635","6.25" 69 | "22637","6.25" 70 | "22639","6.25" 71 | "22641","6.25" 72 | "22643","6.25" 73 | "22645","6.25" 74 | "22647","6.25" 75 | "22649","6.25" 76 | "22651","6.25" 77 | "22654","6.25" 78 | "22656","6.25" 79 | "22658","6.25" 80 | "22660","6.25" 81 | "22662","6.25" 82 | "22664","6.25" 83 | "22666","6.25" 84 | "22668","6.38" 85 | "22670","6.38" 86 | "22672","6.38" 87 | "22674","6.25" 88 | "22676","6.25" 89 | "22678","6.25" 90 | "22680","6.25" 91 | "22682","6.38" 92 | "22684","6.25" 93 | "22685","6.25" 94 | "22687","0.0" 95 | "22689","6.38" 96 | "22691","6.25" 97 | "22693","6.38" 98 | "22695","6.25" 99 | "22697","6.25" 100 | "22699","6.25" 101 | "22701","6.38" 102 | "22703","6.25" 103 | "22705","6.25" 104 | "22707","6.25" 105 | "22708","6.38" 106 | "22710","6.25" 107 | "22712","6.25" 108 | "22714","6.25" 109 | "22716","6.25" 110 | "22717","0.0" 111 | "22718","0.0" 112 | "22720","6.25" 113 | "22782","6.25" 114 | "22783","6.25" 115 | "22784","6.25" 116 | "22785","6.25" 117 | "22786","6.25" 118 | "22787","6.25" 119 | "22788","6.25" 120 | "22789","6.25" 121 | "22790","6.25" 122 | "22791","6.25" 123 | "22792","6.25" 124 | "22793","6.25" 125 | "22794","6.25" 126 | "22795","6.25" 127 | "22824","6.25" 128 | "22825","6.25" 129 | "22826","6.25" 130 | "22827","6.25" 131 | "22828","6.25" 132 | "22829","6.25" 133 | "22830","6.25" 134 | "22831","6.25" 135 | "22832","6.25" 136 | "22833","6.25" 137 | "22834","6.25" 138 | "22835","6.25" 139 | "22836","6.25" 140 | "22837","6.25" 141 | "22838","6.25" 142 | "22839","6.25" 143 | "22840","6.25" 144 | "22841","6.25" 145 | "22842","6.25" 146 | "22843","6.25" 147 | "22844","6.25" 148 | "22845","6.25" 149 | "22846","6.25" 150 | "22847","6.25" 151 | "22848","6.25" 152 | "22849","6.25" 153 | "22850","6.25" 154 | "22851","6.25" 155 | "22855","6.25" 156 | "22856","6.25" 157 | "22857","6.25" 158 | "22864","6.25" 159 | "22865","6.25" 160 | "22866","6.25" 161 | "22867","6.25" 162 | "22868","6.25" 163 | "22869","6.25" 164 | "22875","6.38" 165 | "22876","6.38" 166 | "22877","6.38" 167 | "22878","6.38" 168 | "22879","6.38" 169 | "22883","6.25" 170 | "22884","6.25" 171 | "22885","6.25" 172 | "22890","5.46" 173 | "22891","5.3" 174 | "22892","5.18" 175 | "22893","5.3" 176 | "22900","6.25" 177 | "22901","6.25" 178 | "22902","6.25" 179 | "22903","6.25" 180 | "22904","6.25" 181 | "22905","6.25" 182 | "22937","6.38" 183 | "22938","6.38" 184 | "22939","6.38" 185 | "22940","6.38" 186 | "22941","6.25" 187 | "22942","6.25" 188 | "22943","6.25" 189 | "22944","6.25" 190 | "22945","5.3" 191 | "22946","5.18" 192 | "22947","5.3" 193 | "22948","5.3" 194 | "22949","5.83" 195 | "22950","5.83" 196 | "22951","6.25" 197 | "22952","6.25" 198 | "22953","6.25" 199 | "22954","6.25" 200 | "22955","6.25" 201 | "22956","6.25" 202 | "22957","6.25" 203 | "22958","6.25" 204 | "22959","6.25" 205 | "22960","6.25" 206 | "22961","6.25" 207 | "22962","6.25" 208 | "22963","6.25" 209 | "22964","6.25" 210 | "22965","6.25" 211 | "22966","6.25" 212 | "22967","6.25" 213 | "22979","5.85" 214 | "22980","5.83" 215 | "22981","5.83" 216 | "22982","5.83" 217 | "22983","5.83" 218 | "22984","5.83" 219 | "22985","5.83" 220 | "22986","5.83" 221 | "22988","5.95" 222 | "22987","5.83" 223 | "22989","5.95" 224 | "22995","6.25" 225 | "22997","6.25" 226 | "22996","6.25" 227 | "22998","6.25" 228 | "22999","6.25" 229 | "23010","5.46" 230 | "23016","5.3" 231 | "23011","5.3" 232 | "23012","5.3" 233 | "23013","5.46" 234 | "23014","5.3" 235 | "23015","5.3" 236 | "23017","5.3" 237 | "23018","5.3" 238 | "23019","5.3" 239 | "23034","6.25" 240 | "23035","6.25" 241 | "23036","6.25" 242 | "23037","6.25" 243 | "23038","6.25" 244 | "23039","6.25" 245 | "23040","6.25" 246 | "23041","6.25" 247 | "23042","6.25" 248 | "23043","6.25" 249 | "23044","6.25" 250 | "23045","6.25" 251 | "23046","6.25" 252 | "23047","6.25" 253 | "23056","6.25" 254 | "23057","6.25" 255 | "23058","6.25" 256 | "23059","6.25" 257 | "23060","6.25" 258 | "23061","6.25" 259 | "23062","6.25" 260 | "23063","6.25" 261 | "23071","6.38" 262 | "23072","6.38" 263 | "23073","6.38" 264 | "23074","6.38" 265 | "23075","6.38" 266 | "23076","6.38" 267 | "23077","6.38" 268 | -------------------------------------------------------------------------------- /demos/nutrition/FoodData_Central_foundation_food_csv_2023-10-26/foundation_food.csv: -------------------------------------------------------------------------------- 1 | "fdc_id","NDB_number","footnote" 2 | "321358","16158","" 3 | "321360","100147","" 4 | "321611","11056","" 5 | "323121","7022","" 6 | "323294","12563"," Other phytosterols = 34.67 mg/100g" 7 | "323505","11233","" 8 | "323604","1171","" 9 | "323697","1172","" 10 | "323793","1173","" 11 | "324317","11296","" 12 | "324653","11937","" 13 | "325036","1032","" 14 | "325198","1042","" 15 | "325287","9123","" 16 | "325430","9236","" 17 | "325524","12537","" 18 | "325871","18069","" 19 | "326196","11236","" 20 | "326698","2046","" 21 | "327046","9148","" 22 | "327357","9191","" 23 | "328637","1009","" 24 | "328841","1015","" 25 | "329370","1029","" 26 | "329490","1133","" 27 | "329596","1126","" 28 | "329716","1137","" 29 | "330137","1256","" 30 | "330415","1285","" 31 | "330458","4047","" 32 | "331897","5671","" 33 | "331960","5746","" 34 | "332282","6931","" 35 | "332397","7028","" 36 | "332791","9533","" 37 | "333008","100192","" 38 | "333281","100193","" 39 | "333374","15033","May contain additives to retain moisture" 40 | "333476","15066","May contain additives to retain moisture" 41 | "334194","15121","" 42 | "334536","36602","" 43 | "334628","36412","" 44 | "334720","36408","" 45 | "335240","18075","" 46 | "746758","23377","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 47 | "746759","23385","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 48 | "746760","23362","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 49 | "746761","23359","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 50 | "746762","13468","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 51 | "746763","13236","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 52 | "746764","11130","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 53 | "746765","1227","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 54 | "746766","1036","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 55 | "746767","1040","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 56 | "746768","9094","Mission variety. Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 57 | "746769","11251","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 58 | "746770","9181","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 59 | "746771","9202","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 60 | "746772","1082","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 61 | "746773","9412","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 62 | "746774","36622","Rice was not included in analyses. Ingredients and amount of breading and sauce vary by restaurant. Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the value" 63 | "746775","2047","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 64 | "746776","1085","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 65 | "746777","6164","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 66 | "746778","1079","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 67 | "746779","7954","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 68 | "746780","7089","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 69 | "746781","100173","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 70 | "746782","1077","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 71 | "746783","7919","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 72 | "746784","19335","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 73 | "746785","5666","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 74 | "746952","7129","" 75 | "747429","1062","" 76 | "747430","16500","" 77 | "747431","16501","" 78 | "747432","16502","" 79 | "747433","16503","" 80 | "747434","16504","" 81 | "747435","16505","" 82 | "747436","16506","" 83 | "747437","16507","" 84 | "747438","16508","" 85 | "747439","16509","" 86 | "747440","16510","" 87 | "747441","16511","" 88 | "747442","16512","" 89 | "747443","16513","" 90 | "747444","16514","" 91 | "747445","16515","" 92 | "747446","16516","" 93 | "747447","11090","Source number reflects the actual number of samples analyzed for a nutrient. Repeat nutrient analyses may have been done on the same sample with the values shown." 94 | "747693","11966","" 95 | "747997","1124","" 96 | "748236","1125","" 97 | "748278","4582","" 98 | "748323","4518","" 99 | "748366","4044","" 100 | "748608","4063","" 101 | "748967","1123","" 102 | "749420","10896","" 103 | "789828","1145","" 104 | "789890","20081","" 105 | "789951","20581","" 106 | "790018","20481","" 107 | "790085","20080","" 108 | "790146","20083","" 109 | "790214","20061","" 110 | "790276","100251","" 111 | "790508","1001","" 112 | "790577","100252","" 113 | "790646","100253","" 114 | "1104647","11215","" 115 | "1104705","16117","" 116 | "1104766","16115","" 117 | "1104812","20090","" 118 | "1104867","100257","" 119 | "1104913","100256","" 120 | "1104962","100255","" 121 | "1105073","100254","" 122 | "1105314","9040","" 123 | "1750339","9500","" 124 | "1750340","9504","" 125 | "1750341","9503","" 126 | "1750342","9502","" 127 | "1750343","9501","" 128 | "1750348","4042","" 129 | "1750349","100262","" 130 | "1750350","4511","" 131 | "1750351","100258","" 132 | "1999626","100259","" 133 | "1999627","11987","" 134 | "1999628","11238","" 135 | "1999629","11260","" 136 | "1999630","16222","" 137 | "1999631","14091","" 138 | "1999632","100260","" 139 | "1999633","11457","" 140 | "1999634","100261","" 141 | "2003586","100268","" 142 | "2003587","20140","" 143 | "2003588","100269","" 144 | "2003589","100270","" 145 | "2003590","9400","" 146 | "2003591","9209","" 147 | "2003592","9130","" 148 | "2003593","100266","" 149 | "2003594","43382","" 150 | "2003595","100267","" 151 | "2003596","11540","" 152 | "2003597","9206","" 153 | "2003598","11243","" 154 | "2003599","100263","" 155 | "2003600","11950","" 156 | "2003601","11266","" 157 | "2003602","11993","" 158 | "2003603","100264","" 159 | "2003604","100265","" 160 | "2257044","100275","This food was incorrectly described as unsweetened. The description has been updated to correctly identify this food as sweetened." 161 | "2257045","100276","" 162 | "2257046","100277","" 163 | "2258586","11124","" 164 | "2258587","11960","" 165 | "2258588","11333","" 166 | "2258589","11951","" 167 | "2258590","11821","" 168 | "2258591","100278","" 169 | "2259792","1088","" 170 | "2259793","1116","" 171 | "2259794","1293","" 172 | "2259795","100272","" 173 | "2259796","1019","" 174 | "2261420","100273","" 175 | "2261421","100274","" 176 | "2261422","11413","" 177 | "2262072","16098","" 178 | "2262073","100271","" 179 | "2262074","12195","" 180 | "2262075","12220","" 181 | "2346384","1012","" 182 | "2346385","1017","" 183 | "2346386","1053","" 184 | "2346387","1056","" 185 | "2346388","11252","" 186 | "2346389","100281","" 187 | "2346390","11257","" 188 | "2346391","11253","" 189 | "2346392","12147","" 190 | "2346393","12061","" 191 | "2346394","12155","" 192 | "2346395","12142","" 193 | "2346396","20038","" 194 | "2346397","100282","" 195 | "2346398","9266","" 196 | "2346399","9070","" 197 | "2346400","11052","" 198 | "2346401","100283","" 199 | "2346402","100284","" 200 | "2346403","100285","" 201 | "2346404","100286","" 202 | "2346405","11143","" 203 | "2346406","11205","" 204 | "2346407","11109","" 205 | "2346408","11112","" 206 | "2346409","9316","" 207 | "2346410","9302","" 208 | "2346411","9050","" 209 | "2346412","100279","" 210 | "2346413","100280","" 211 | "2346414","9401","" 212 | "2512371","100287","" 213 | "2512372","100288","" 214 | "2512373","100289","" 215 | "2512374","20011","" 216 | "2512375","100290","" 217 | "2512376","20130","This food was previously listed with a measurement of zero for all Beta-glucan values. The food and its samples have been updated with the correct Beta-glucan values." 218 | "2512377","100291","" 219 | "2512378","20008","" 220 | "2512379","20031","" 221 | "2512380","20036","" 222 | "2512381","20444","" 223 | "2514743","23562","" 224 | "2514744","23572","" 225 | "2514745","10219","" 226 | "2514746","5332","" 227 | "2514747","5665","" 228 | "2515373","100294","" 229 | "2515374","12087","" 230 | "2515375","12120","" 231 | "2515376","16087","" 232 | "2515377","100292","" 233 | "2515378","12131","" 234 | "2515379","12151","" 235 | "2515380","12014","" 236 | "2515381","12036","" 237 | "2515382","100293","" 238 | "2644281","100310","" 239 | "2644282","100311","" 240 | "2644283","100312","" 241 | "2644284","100313","" 242 | "2644285","100314","" 243 | "2644286","100315","" 244 | "2644287","100316","" 245 | "2644288","100317","" 246 | "2644289","100318","" 247 | "2644290","100319","" 248 | "2644291","100320","" 249 | "2644292","100321","" 250 | "2644293","100322","" 251 | "2644294","100323","" 252 | "2646168","100302","" 253 | "2646169","100303","" 254 | "2646170","100304","" 255 | "2646171","100305","" 256 | "2646172","100306","" 257 | "2646173","100307","" 258 | "2646174","100308","" 259 | "2646175","100309","" 260 | "2647437","100295","" 261 | "2647438","100296","" 262 | "2647439","100297","" 263 | "2647440","100298","" 264 | "2647441","100299","" 265 | "2647442","100300","" 266 | "2647443","100301","" 267 | -------------------------------------------------------------------------------- /demos/nutrition/FoodData_Central_foundation_food_csv_2023-10-26/lab_method.csv: -------------------------------------------------------------------------------- 1 | "id","description","technique" 2 | "1000","NIST Handbook 133","Gravimetric" 3 | "1001","AOAC 968.06 + 992.15","Combustion" 4 | "1002","AOAC 960.39 39.1","Extraction" 5 | "1003","AOAC 922.06","Acid hydrolysis" 6 | "1004","AOAC 923.03","Gravimetric" 7 | "1005","doi.10.1021/jf60175a006","GC" 8 | "1006","doi.org/10.1093/jat/4.1.43","GC" 9 | "1007","AOAC 934.06 mod","Vacuum oven" 10 | "1008","AOAC 991.43","Enzymatic-gravimetric" 11 | "1009","AOAC 985.01 + 984.27","ICP" 12 | "1010","AOAC 2011.19","ICP-MS" 13 | "1011","AOAC 942.23 + 953.17 + 957.17","Fluorometric" 14 | "1012","In-house HPLC thiamin","HPLC" 15 | "1013","AOAC 940.33","Microbiological" 16 | "1014","AOAC 944.13 + 960.46 + 985.34","Microbiological" 17 | "1015","AOAC 945.74 + 960.46","Microbiological" 18 | "1016","AOAC 961.15","Microbiological" 19 | "1017","AOAC 992.05","Microbiological" 20 | "1018","AOAC 991.20","Kjeldahl" 21 | "1019","AOAC 989.05 mod","Acid hydrolysis" 22 | "1021","AOAC 963.15","Extraction" 23 | "1022","AOAC 945.46","Gravimetric" 24 | "1023","AOAC 979.10","Enzymatic-spectrometric" 25 | "1024","AOAC 982.14","LC" 26 | "1025","In-house pH","ISE" 27 | "1026","AOAC 936.13","Gravimetric" 28 | "1027","AOAC 950.46","Forced air" 29 | "1028","AOAC 964.22","Vacuum oven" 30 | "1029","AOAC 980.14","LC" 31 | "1030","doi.org/10.1016/S0021-9673(01)95818-2","HPLC" 32 | "1032","doi 10.1039/AN9840900489","HPLC" 33 | "1033","AOAC 967.22","Microfluorometric" 34 | "1034","AOAC 942.23","Fluorometric" 35 | "1035","AOAC 970.65","Fluorometric" 36 | "1036","AOAC 944.13","Microbiological" 37 | "1037","AOAC 945.74","Microbiological" 38 | "1038","AOAC 961.15","Microbiological" 39 | "1039","AOAC 952.20","Microbiological" 40 | "1040","USDA Guidebook 1986","HPLC" 41 | "1041","AOAC 994.10","GC/Direct saponification" 42 | "1042","AOAC 996.06","GLC" 43 | "1043","doi.10.1021/ac025624x","LC/ESI/IDMS" 44 | "1044","Blauch & Tarka J Food Sci 48 (1983) p745","HPLC" 45 | "1045","Huang et al. JAOAC (2009) p1327","HPLC/MS/MS" 46 | "1046","AOAC 986.13","HPLC" 47 | "1047","AOAC 996.11","Enzymatic-colorimetric" 48 | "1048","doi.org/10.1016/S0076-6879(97)82128-3","HPLC" 49 | "1049","doi.org/10.1016/S0889-1575(03)00029-2","HPLC" 50 | "1050","In-house HPLC retinol","HPLC" 51 | "1051","doi.org/10.1002/0471142913.faf0203s00","HPLC" 52 | "1052","In-house HPLC tocopherols","HPLC" 53 | "1053","AOAC 954.02","Acid hydrolysis" 54 | "1055","doi.org/10.1111/j.1745-4522.2005.00011.x","GC" 55 | "1056","doi.10.1021/jf049398z","HPLC" 56 | "1057","doi.org/10.1007/s11746-997-0158-1","Extraction" 57 | "1058","AOAC 925.09 + 926.08","Vacuum oven" 58 | "1059","AOAC 926.08","Vacuum oven" 59 | "1060","AOAC 982.14 mod","HPLC" 60 | "1061","doi.10.1021/jf051505h","GC" 61 | "1062","AOAC 2002.02","Enzymatic-gravimetric" 62 | "1063","AOAC 2011.25","Enzymatic-Gravimetric-LC" 63 | "1064","AOAC 2009.01","Enzymatic-Gravimetric-LC" 64 | "1065","AOAC 984.27","ICP" 65 | "1066","AOAC 982.14 mod","HPLC" 66 | "1067","AOAC 2011.25","Enzymatic-gravimetric-LC" 67 | "1068","AOAC 925.09","Vacuum oven" 68 | "1069","AOAC 942.05","Gravimetric" 69 | "1070","AOAC 990.03","Combustion" 70 | "1071","AOAC 2012.15","ICP-MS" 71 | "1072","Mann et al. JAOAC (2005) p30","LC" 72 | "1073","doi.org/10.1016/j.jfca.2008.02.001","GC/Direct saponification" 73 | "1074","Morrison et al., J Lipid Res, 5 (1964)","GLC" 74 | "1075","Parks & Goins, J. Food Sci. (1994)59, p1262","GLC" 75 | "1076","AOAC 985.35","AAS" 76 | "1077","AOAC 985.35 + 990.23","Emission spectrometry" 77 | "1078","AOAC 13th Ed. 2.019, 2.095, 7.098","Colorimetric" 78 | "1079","doi.org/10.1007/s12161-010-9123-y","GLC" 79 | "1080","AOAC 952.20","Microbiological" 80 | "1081","Folch et al. J Bio Chem 1957","Extraction" 81 | "1082","AOAC 992.15 (39.1.16)","Combustion" 82 | "1083","AOAC 950.46 +934.06","Forced air" 83 | "1084","AOAC 920.15","Gravimetric" 84 | "1085","AOAC 992.05","Microbiological" 85 | "1086","AOAC 933.08","GC/Direct saponification" 86 | "1087","AOAC 941.15","HPLC" 87 | "1088","AOAC 945.74 B5","Microbiological" 88 | "1089","AOAC 948.22","Gravimetric" 89 | "1090","AOAC 963.15","Extraction" 90 | "1091","AOAC 968.06","Combustion" 91 | "1092","AOAC 982.30 HCl","Acid Hydrolysis-HPLC" 92 | "1093","AOAC 983.13","GC" 93 | "1094","AOAC 985.29","Enzymatic/gravimetric" 94 | "1095","AOAC 985.35","AAS" 95 | "1096","AOAC 989.05 + 932.06","Base hydrolysis" 96 | "1097","AOAC 990.26","Colorimetric" 97 | "1098","doi.10.1021/jf301374t","HPLC" 98 | "1099","AOAC 933.05","Extraction" 99 | "1100","AOAC 934.06","Pressure drying" 100 | "1101","AOAC 992.05 + 960.46b","Microbiological" 101 | "1102","In-house HPLC Vit D","HPLC" 102 | "1103","doi.org/10.1016/j.jfca.2005.07.001","Tri-enzyme Extraction" 103 | "1104","EN 14164:2008","HPLC-fluorimetric" 104 | "1105","Fecher et al., J Anal At Spectro (1998) mod","ICP" 105 | "1106","doi.org/10.1111/j.1365-2141.1974.tb06816.x","Radio-Dilution" 106 | "1107","In-house Vit D HPLC","HPLC" 107 | "1108","In-house Vit D HPLC","HPLC" 108 | "1109","doi.org/10.1016/j.jfca.2013.04.003","ICP" 109 | "1110","doi.org/10.1093/jn/130.10.2568","GLC" 110 | "1111","doi.org/10.1016/j.foodchem.2009.07.058","GLC" 111 | "1112","Starch_polarimetric","Polarimetric" 112 | "1113","Moisture_KarlFischer","Titrimetric" 113 | "1114","AACC 08-01","Gravimetric" 114 | "1115","AACC 46-30","Combustion" 115 | "1116","AOAC 934.01","Vacuum oven" 116 | "1117","AOAC 971.30","HPLC" 117 | "1118","AOAC 974.29 Mod","HPLC" 118 | "1119","AOAC 974.29","Colorimetric" 119 | "1120","AOAC 985.35 50.1","AAS" 120 | "1121","AOAC 988.15","Alk. hydrolysis-HPLC" 121 | "1122","AOAC 992.15","Combustion" 122 | "1123","AOAC 992.23","Combustion" 123 | "1124","AOAC 990.03 2","Combustion" 124 | "1125","AOAC 944.13","Microbiological" 125 | "1126","AOAC 974.29 mod","HPLC" 126 | "1127","AOAC 982.30","Alk. hydrolysis-HPLC" 127 | "1128","AOAC 982.30","HCl hydrolysis -HPLC" 128 | "1129","AOAC 996.06","GLC" 129 | "1130","doi.10.1021/jf00120a045","GC" 130 | "1131","Davidek et al. J Micro Anal 1985 p39","HPLC" 131 | "1132","EN 14122:2003","HPLC" 132 | "1133","EN 15652:2009","HPLC-fluorimetric" 133 | "1134","AOAC 992.26 + 2002.05","HPLC" 134 | "1135","Fecher et al., J Anal At Spectro (1998)","ICP" 135 | "1136","doi.10.1021/jf960497p_mod","Single Enzyme" 136 | "1137","doi.10.1021/jf9029546","HPLC" 137 | "1138","FSIS_hydroyproline","HPLC" 138 | "1139","Ash_gravimetric","Gravimetric" 139 | "1140","Niacin_SIDMS","HPLC-SIDMS" 140 | "1141","AOAC 2002.05","HPLC-UV" 141 | "1142","AACC 44-15A","Oven" 142 | "1143","AOAC 923.03","Gravimetric" 143 | "1144","AOAC 933.05","Extraction" 144 | "1145","AOAC 982.30 Ninhy","Acid Hydrolysis-HPLC" 145 | "1146","AOAC 983.23","Extraction" 146 | "1147","AOAC 989.05","Extraction" 147 | "1148","AOAC 990.03","Combustion" 148 | "1149","AOAC 994.12 HCl","Acid hydrolysis-HPLC" 149 | "1150","AOCS Ce1-62 2-66","GLC" 150 | "1151","AOCS Ce1c-89","GLC" 151 | "1152","doi.org/10.1016/0003-2697(89)90059-6","Acid hydrolysis-HPLC" 152 | "1153","AOAC 967.22","Microfluorometric" 153 | "1154","AOAC 982.29","LC" 154 | "1155","EN 14152:2003","HPLC-fluorimetric" 155 | "1156","Carot_HPLC_rev_phase","HPLC" 156 | "1157","In-house Vit D RIA","RIA" 157 | "1158","doi.org/10.1016/j.foodres.2012.11.025","Enzymatic-spectrometric" 158 | "1159","doi.org/10.1006/jfca.1999.0845","HPLC" 159 | "1160","Martin et al., JAOAC 73 (1990)","Tri-enzyme Extraction" 160 | "1161","doi.10.1021/ac00237a004","ID-GC-MS" 161 | "1162","doi.10.1021/jf00084a019","GLC" 162 | "1163","AOAC 925.09","Vacuum oven" 163 | "1164","AOAC 925.40","Oven" 164 | "1165","AOAC 961.15","Microbiological" 165 | "1166","AOAC 982.30 Perf","Performic oxidation-HPLC" 166 | "1167","AOAC 984.27","ICP" 167 | "1168","AOAC 985.01","ICP" 168 | "1169","AOAC 986.15 1","Hydride generation" 169 | "1170","AOAC 991.43 3","Enzymatic/chemical" 170 | "1171","AOAC 994.12 Perf","Performic oxidation-HPLC" 171 | "1172","AOAC 996.06","Hydrolytic extraction/GC" 172 | "1173","doi.org/10.1016/j.chroma.2006.01.049","GLC" 173 | "1174","AOAC Olestra","GC" 174 | "1175","AOCS Ce1d-91","GLC" 175 | "1176","AOAC 968.08 + 985.35 + 965.05","AAS" 176 | "1177","AOAC 994.10","GC/Direct saponification" 177 | "1178","doi.10.1021/jf0259056","Tri-enzyme /LC-MS" 178 | "1179","Hill et al. Anal. Chem. (1986) 58","AAS" 179 | "1180","Moisture_air 100C","Lyophilization" 180 | "1181","doi.org/10.1016/j.foodchem.2004.08.007","Tri-enzyme Extraction" 181 | "1182","doi.10.1021/jf102150n","GC" 182 | "1183","Sullivan & Zywicki, J. AOAC (2012)","ICP-MS" 183 | "1184","doi.10.1021/jf802307h","Hydride generation" 184 | "1185","AOAC 900.02","Gravimetric" 185 | "1186","AOAC 920.15","Gravimetric" 186 | "1187","AOAC 920.39","Extraction" 187 | "1188","AOAC 920.50","Gravimetric" 188 | "1189","AOAC 985.32","Microbiological" 189 | "1190","AOAC 950.46b","Vacuum oven" 190 | "1191","AOAC 960.46","Microbiological" 191 | "1192","AOAC 961.14","Colorimetric" 192 | "1193","AOAC 986.15","Hydride generation" 193 | "1194","AOAC 990.23","Emission spectrometry" 194 | "1195","AOAC 991.43 2","Enzymatic-gravimetric" 195 | "1196","AOAC 991.43 4","Enzymatic-gravimetric" 196 | "1197","AOAC 992.07","Microbiological" 197 | "1198","AOAC 994.12 HPLC","HPLC" 198 | "1199","AOAC 995.05","LC" 199 | "1200","Baker et al., At Spect 20 (1999)","ICP-MS" 200 | "1201","Beckman_AA","Acid hydrolysis/IC" 201 | "1202","Moisture_air","Forced air" 202 | "1203","B_Glucan_Megazyme","Enzymatic-spectrophotometric" 203 | "1204","Choles_GC/Direct","GC/Direct saponification" 204 | "1205","AOAC 43.283 - 43.291","GC" 205 | "1206","SpGrav_gravimetric","Gravimetric" 206 | "1207","AOAC 985.01 + 984.27","ICP" 207 | "1208","AOAC 942.23","Fluorometric" 208 | "1209","AOAC 944.13 B5","Microbiological" 209 | "1210","AOAC 944.13 B2","Microbiological" 210 | "1211","AOAC 960.47","Microbiological" 211 | "1212","AOAC 975.03 3.2","AAS" 212 | "1213","AOAC 991.42","Enzymatic-gravimetric" 213 | "1214","AOAC 993.19","Enzymatic-gravimetric" 214 | "1215","Bui and Cooper, J. AOAC, 70 (1987)","HPLC" 215 | "1216","Carpenter et al., Biochem J. 1960, p. 604","Chemical" 216 | "1217","AOCS Ce1-62","GLC" 217 | "1218","AOAC 925.45","Vacuum oven" 218 | "1219","AOAC 958.01 + 965.17","Spectrophotometric" 219 | "1220","AOAC 980.13 + 982.14 + 984.17","LC" 220 | "1221","AOAC 985.35 + 975.03","AAS" 221 | "1222","Carot_FCL_HPLC","HPLC" 222 | "1223","Hill et al. Anal. Chem. (1986) 58","AES" 223 | "1224","Tryptophan_USDA","Chemical" 224 | "1225","doi.10.1021/jf60175a006","GC" 225 | "1226","AOAC 961.15 + 960.46","Microbiological" 226 | "1227","SpGrav_pyncnometer","Pycnometer" 227 | "1228","doi.org/10.1080/10826079808006596","HPLC" 228 | "1229","AOAC 955.04 N","Kjeldahl" 229 | "1230","AOAC 966.11","Enzymatic-spectrometric" 230 | "1231","AOAC 975.03","AAS" 231 | "1232","AOAC 981.10 39.1","Kjeldahl" 232 | "1233","AOCS Ce-1e-91(01)","GLC" 233 | "1234","AOAC 994.10 mod","GC/Direct saponification" 234 | "1235","doi.org/10.1016/0308-8146(95)00149-2","HPLC" 235 | "1236","doi.org/10.1016/S0308-8146(98)00182-4","HPLC" 236 | "1237","Hurst et al., J. AOAC, 67 (1984)","HPLC" 237 | "1238","doi.10.1021/jf960497p","Tri-enzyme Extraction" 238 | "1239","doi.10.1021/jf950616l","ICP" 239 | "1240","VanWinkle et al., Pediatr. Dent., 17 (1995)","ISE-MD-HMDS" 240 | "1241","AACC 76-11","Enzymatic-colorimetric" 241 | "1242","AOAC 925.51","Gravimetric" 242 | "1243","AOAC 930.15","Oven" 243 | "1244","AOAC 940.33","Microbiological" 244 | "1245","AOAC 950.46","Vacuum oven" 245 | "1246","AOAC 967.21","Titrimetric" 246 | "1247","AOAC 970.65","Fluorometric" 247 | "1248","AOAC 982.29","LC" 248 | "1249","AOAC 985.32","Microbiological" 249 | "1250","AOAC 991.43 1","Enzymatic/gravimetric" 250 | "1251","AOCS Am 3 -96","Extraction" 251 | "1252","AOAC 944.13 + 960.46 + 985.34","Microbiological" 252 | "1253","AOAC 982.30","Acid hydrolysis/ Performic acid" 253 | "1254","AOAC 992.07","Microbiological" 254 | "1255","FDA Meth #2, 1975","GLC" 255 | "1256","Moisture_air 60C","Oven" 256 | "1257","Sawar, et. al., JAOAC 1988","HPLC" 257 | "1258","Se_GFAAS","AAS" 258 | "1259","Fluoride_ISE","ISE-direct" 259 | "1260","J Ag Food Chem, 23: 1157 & other Pubs","Microbiological" 260 | "1261","AOAC 994.10 mod","GC/Direct sponification" 261 | "1262","doi.org/10.3402/fnr.v56i0.18931","GC" 262 | "1263","AOAC 2009.01 + 2011.25","LC" 263 | "1264","AOAC 971.30 with HPLC quantification mod","LC-FLD ()" 264 | "1265","AOAC 971.30 with HPLC quantification mod","LC-FLD ()" 265 | "1266","doi.org/10.1002/jps.2600560249","Nephelometry" 266 | "1267","In-house HPLC retinol","HPLC" 267 | "1268","doi.org/10.1002/0471142913.faf0203s00","HPLC" 268 | "1269","AOAC 934.06 mod","Vacuum oven" 269 | "1270","AOAC 900.02","Gravimetric" 270 | "1271","Publication forthcoming - validation data sent to MAFCL","Direct alkaline saponification, SPE clean up, HPLC-UV" 271 | "1272","AOAC 2001.10","HPLC" 272 | "1273","P-143 Ergothioneine by HPLC","reverse-phase HPLC" 273 | "1274","Phyto- & Fungisterols","Direct alkaline saponification, BSTFA/TMCS derivatization; GC-FID" 274 | "1275","HPAE-PAD"," High-Performance Anion-Exchange Chromatography with Pulsed Amperometric Detection" 275 | "1276","Mushroom and Yeast Beta-Glucan assay","UV-VIS" 276 | "1277","Cereal grains, milling fractions, and wort beer Beta-Glucan assay","UV-VIS" 277 | "1278","AOAC 2009.01 +2011.25","LC" 278 | "1279","AOAC 948.22","Fat by Soxhlet extraction" 279 | "1280","Ethanol extraction along with dithiothreitol (DTT) reduction, followed by LC-MS/MS analysis","LC-QqQ MS" 280 | "1281","AOAC 982.30 mod.","LC-UV/VIS" 281 | -------------------------------------------------------------------------------- /demos/nutrition/FoodData_Central_foundation_food_csv_2023-10-26/lab_method_code.csv: -------------------------------------------------------------------------------- 1 | "lab_method_id","code" 2 | "1001","DGEN_S" 3 | "1003","FAT-MJ-FLT-CHG" 4 | "1003","FAT_AH_S" 5 | "1004","ASHM_S" 6 | "1005","SUGN_S" 7 | "1007","M100T100_S" 8 | "1007","VPI_Moisture" 9 | "1008","TDFL_S" 10 | "1008","FBR-SR3-CHG" 11 | "1009","ICP_S" 12 | "1010","SEICPMS_S" 13 | "1010","SEMSPLUS_S" 14 | "1011","BIDE_S" 15 | "1012","B1LC_S" 16 | "1013","B2FV_S" 17 | "1014","NIAP_S" 18 | "1015","PANN_S" 19 | "1016","B6A_S" 20 | "1017","FOAN_S" 21 | "1018","NITR-BORIC-CHG" 22 | "1019","FAT-MOJO-CHG" 23 | "1019","FAT-MOJOAH-CHG" 24 | "1021","FSOX_S" 25 | "1022","ASH-CHG" 26 | "1023","STARCH-ENZ-MRK" 27 | "1023","STARCH-EWR-MIN" 28 | "1023","STARCH-EWR-CHG" 29 | "1024","SUGR-DP1-2-CHG" 30 | "1026","SPEC-GRAV-CHG" 31 | "1027","MO-FAO-100-CHG" 32 | "1027","MO-FAO-125-CHG" 33 | "1027","MUDA_S" 34 | "1028","MO-VO-100-CHG" 35 | "1028","MO-VO-65-CHG" 36 | "1030","SUGR-GALAC-CHG" 37 | "1030","SUGR-GALAC" 38 | "1033","VITAMIN-C-CHG" 39 | "1034","QD0IU" 40 | "1034","THIAMINE-CHG" 41 | "1035","RIBOFLAVIN-CHG" 42 | "1035","QD0IV" 43 | "1036","VIT-B3-C-MRK" 44 | "1037","PANTO-C-MRK" 45 | "1037","QD0IW" 46 | "1038","VIT-B6-C-MRK" 47 | "1039","QD0IY" 48 | "1039","VIT-B12-C-MRK" 49 | "1040","AMINO-CMPL-CHG" 50 | "1041","CHOLESTRO-CHG" 51 | "1042","FAT-BY-FAP-CHG" 52 | "1043","CHOLINE_UNC" 53 | "1045","VDMS_S" 54 | "1045","MISC_6008" 55 | "1045","VitD_HPLC_MS" 56 | "1046","ORG2_S" 57 | "1047","STCH_S" 58 | "1048","Tufts-VitK" 59 | "1049","Tufts-VitKHydro" 60 | "1049","Tufts-VitKHydrox" 61 | "1050","Craft_Retinol" 62 | "1051","Carot_HPLC" 63 | "1052","Craft_VitE" 64 | "1053","QD0IC" 65 | "1053","FAT_AH_P" 66 | "1055","Choles_VPI" 67 | "1056","VitC_VPI" 68 | "1057","Lipid_VPI" 69 | "1058","M100T100_V" 70 | "1059","MO-VO-100C-CHG" 71 | "1060","Cov_Sug" 72 | "1061","VPI-Choles" 73 | "1065","QD0IQ" 74 | "1065","QD0IL" 75 | "1065","QD0IJ" 76 | "1065","QD0IK" 77 | "1065","QD0IS" 78 | "1065","QD0IM" 79 | "1065","QD0IR" 80 | "1065","QD0IP" 81 | "1065","QD0IN" 82 | "1066","QD0ID" 83 | "1068","QD0IA" 84 | "1069","QD0IE" 85 | "1070","QD0IB" 86 | "1071","IODICPMS_S" 87 | "1072","QD0IX" 88 | "1073","Chol_Dinh" 89 | "1074","TAMU_FA" 90 | "1075","CSU_FA" 91 | "1076","TTU_AAS" 92 | "1077","TTU_AES" 93 | "1078","TTU_Phos" 94 | "1080","B12_MCB" 95 | "1081","Fat-Folch" 96 | "1082","Prot_Dumus" 97 | "1083","Moist_Oven" 98 | "1084","Ash_923" 99 | "1085","QD0J3" 100 | "1087","AOAC_941.15" 101 | "1088","AOAC_945.74_B5" 102 | "1091","AOAC_968.06" 103 | "1094","AOAC_985.29" 104 | "1096","AOAC_989.05_932.06" 105 | "1097","AOAC_990.26" 106 | "1099","Cov_933.05" 107 | "1101","Cov_Folate" 108 | "1103","UGa_Folate" 109 | "1107","Ames_D2" 110 | "1112","Starch_polar" 111 | "1116","AOAC_934.01" 112 | "1121","AOAC_988.15" 113 | "1122","AOAC_992.15" 114 | "1125","Cov_B3Mod" 115 | "1127","Cov_AA_Alk" 116 | "1128","Cov_AA_HCl" 117 | "1134","FCMDL_VitD" 118 | "1136","SingleFolate" 119 | "1138","FSIS_hydroy" 120 | "1143","AOAC_923.03" 121 | "1144","AOAC_933.05" 122 | "1145","AOAC_982.30_Ninhy" 123 | "1146","AOAC_983.23" 124 | "1147","AOAC_989.05" 125 | "1148","AOAC_990.03" 126 | "1153","Cov_VitC" 127 | "1159","UGa_VitE_2" 128 | "1160","TriFolate_Martin" 129 | "1161","Se_IDGC_MS" 130 | "1165","AOAC_961.15" 131 | "1167","AOAC_984.27" 132 | "1169","AOAC_986.15_1" 133 | "1171","AOAC_994.12_Perf" 134 | "1177","Cov_Choles" 135 | "1181","TriFolate_VPI" 136 | "1185","AOAC_900.02" 137 | "1193","AOAC_986.15" 138 | "1195","AOAC_991.43_2" 139 | "1196","AOAC_991.43_4" 140 | "1198","AOAC_994.12_HPLC" 141 | "1207","AAOAC_985.01_984.27" 142 | "1208","AOAC_942.23" 143 | "1211","AOAC_960.47" 144 | "1213","AOAC_991.42" 145 | "1214","AOAC_993.19" 146 | "1216","Lysine_Chem" 147 | "1217","AOCS_Ce1-62" 148 | "1220","EN_Sug" 149 | "1222","Carot_FCL" 150 | "1224","USDA_Trypto" 151 | "1229","AOAC_955.04_N" 152 | "1230","AOAC_966.11" 153 | "1232","AOAC_981.10_39.1" 154 | "1235","B1_B2_HPLC" 155 | "1238","TriFolate" 156 | "1242","AOAC_925.51" 157 | "1243","AOAC_930.15" 158 | "1244","AOAC_940.33" 159 | "1246","AOAC_967.21" 160 | "1249","AOAC_985.32" 161 | "1250","AOAC_991.43_1" 162 | "1253","Cov_AA_Perfor" 163 | "1254","Cov_Panto" 164 | "1255","FA_FDA" 165 | "1256","Moisture_60" 166 | "1257","STR_AA" 167 | "1260","BIOM_S" 168 | "1261","QD0IZ" 169 | "1262","Phytosterols1_VPI" 170 | "1263","TDFM_S" 171 | "1264","QD0GI" 172 | "1265","QD06V" 173 | "1266","QQ022" 174 | "1267","YI00A" 175 | "1268","YI0A4" 176 | "1269","MoistureVacuumDrying_VPI" 177 | "1270","Ash_VPI" 178 | "1271","TotalVitaminD_VPI" 179 | "1272","AOAC_ISOF_HPLC" 180 | "1272","DaidzeinPlaceHolder" 181 | "1272","" 182 | "1273","Ergothioneine via HPLC" 183 | "1274","SterolsDSP_VPI" 184 | "1275","HPAE-PAD" 185 | "1276","Megazyme-Enzymatic Yeast Beta-Glucan Kit Assay Kit" 186 | "1277","beta-Glucan Assay Kit (Mixed Linkage) AOAC 995.16" 187 | "1278","CODEX-FBR-CHG" 188 | "1279","QD062" 189 | "1280","QD062" 190 | -------------------------------------------------------------------------------- /demos/nutrition/FoodData_Central_foundation_food_csv_2023-10-26/lab_method_nutrient.csv: -------------------------------------------------------------------------------- 1 | "lab_method_id","nutrient_id" 2 | "1001","1002" 3 | "1001","1003" 4 | "1003","1004" 5 | "1004","1007" 6 | "1005","1011" 7 | "1005","1014" 8 | "1005","1010" 9 | "1005","1075" 10 | "1005","1013" 11 | "1005","1012" 12 | "1007","1051" 13 | "1008","1079" 14 | "1008","1175" 15 | "1009","1090" 16 | "1009","1089" 17 | "1009","1087" 18 | "1009","1101" 19 | "1009","1095" 20 | "1009","1093" 21 | "1009","1092" 22 | "1009","1091" 23 | "1009","1098" 24 | "1010","1103" 25 | "1010","1102" 26 | "1011","1165" 27 | "1012","1165" 28 | "1013","1166" 29 | "1014","1167" 30 | "1014","1175" 31 | "1015","1170" 32 | "1016","1175" 33 | "1016","1079" 34 | "1016","1093" 35 | "1016","1103" 36 | "1016","1095" 37 | "1017","1177" 38 | "1017","1091" 39 | "1017","1092" 40 | "1017","1167" 41 | "1018","1002" 42 | "1019","1004" 43 | "1021","1004" 44 | "1022","1007" 45 | "1023","1009" 46 | "1024","1011" 47 | "1024","1012" 48 | "1024","1014" 49 | "1024","1010" 50 | "1024","1013" 51 | "1026","1024" 52 | "1027","1051" 53 | "1028","1051" 54 | "1030","1075" 55 | "1033","1162" 56 | "1034","1165" 57 | "1035","1166" 58 | "1036","1167" 59 | "1037","1170" 60 | "1038","1175" 61 | "1039","1178" 62 | "1040","1225" 63 | "1040","1215" 64 | "1040","1213" 65 | "1040","1228" 66 | "1040","1210" 67 | "1040","1219" 68 | "1040","1211" 69 | "1040","1217" 70 | "1040","1222" 71 | "1040","1212" 72 | "1040","1218" 73 | "1040","1232" 74 | "1040","1214" 75 | "1040","1223" 76 | "1040","1220" 77 | "1040","1227" 78 | "1040","1224" 79 | "1040","1226" 80 | "1040","1221" 81 | "1040","1216" 82 | "1041","1253" 83 | "1042","1269" 84 | "1042","1264" 85 | "1042","1263" 86 | "1042","1258" 87 | "1042","1292" 88 | "1042","1266" 89 | "1042","1265" 90 | "1042","1293" 91 | "1042","1257" 92 | "1042","1004" 93 | "1042","1316" 94 | "1042","1301" 95 | "1042","1273" 96 | "1042","2012" 97 | "1042","1267" 98 | "1042","1315" 99 | "1042","2014" 100 | "1042","1404" 101 | "1042","1314" 102 | "1042","1262" 103 | "1042","1259" 104 | "1042","1260" 105 | "1042","1261" 106 | "1042","2013" 107 | "1042","2019" 108 | "1042","1405" 109 | "1042","1406" 110 | "1042","1411" 111 | "1042","1414" 112 | "1042","1304" 113 | "1042","1303" 114 | "1042","1305" 115 | "1042","1306" 116 | "1042","1311" 117 | "1042","1312" 118 | "1042","1313" 119 | "1042","2005" 120 | "1042","2004" 121 | "1042","2008" 122 | "1042","1299" 123 | "1042","1300" 124 | "1042","1333" 125 | "1042","1323" 126 | "1042","1334" 127 | "1042","1335" 128 | "1042","1281" 129 | "1042","2009" 130 | "1042","2015" 131 | "1042","2006" 132 | "1042","2007" 133 | "1042","2021" 134 | "1042","1321" 135 | "1042","1271" 136 | "1042","1272" 137 | "1042","1276" 138 | "1042","1278" 139 | "1042","1280" 140 | "1042","2003" 141 | "1042","1277" 142 | "1042","1317" 143 | "1042","1409" 144 | "1043","1195" 145 | "1043","1199" 146 | "1043","1198" 147 | "1043","1196" 148 | "1043","1197" 149 | "1043","1194" 150 | "1043","1180" 151 | "1045","1112" 152 | "1045","1111" 153 | "1045","1113" 154 | "1045","1110" 155 | "1046","1041" 156 | "1046","1043" 157 | "1046","1032" 158 | "1046","1044" 159 | "1046","1039" 160 | "1047","1009" 161 | "1048","1185" 162 | "1048","1183" 163 | "1049","1184" 164 | "1050","1105" 165 | "1051","2028" 166 | "1051","1121" 167 | "1051","1107" 168 | "1051","1108" 169 | "1051","2029" 170 | "1051","1122" 171 | "1051","1123" 172 | "1051","1120" 173 | "1051","1159" 174 | "1051","1161" 175 | "1051","1160" 176 | "1051","1119" 177 | "1051","2032" 178 | "1052","1129" 179 | "1052","1125" 180 | "1052","1128" 181 | "1052","1109" 182 | "1052","1130" 183 | "1052","1126" 184 | "1052","1127" 185 | "1052","1131" 186 | "1053","1004" 187 | "1055","1253" 188 | "1056","1162" 189 | "1057","1004" 190 | "1058","1051" 191 | "1059","1051" 192 | "1060","1014" 193 | "1060","1012" 194 | "1060","1013" 195 | "1060","1010" 196 | "1060","1011" 197 | "1060","1075" 198 | "1061","1253" 199 | "1065","1092" 200 | "1065","1089" 201 | "1065","1087" 202 | "1065","1098" 203 | "1065","1095" 204 | "1065","1090" 205 | "1065","1093" 206 | "1065","1091" 207 | "1065","1101" 208 | "1066","1011" 209 | "1066","2000" 210 | "1066","1010" 211 | "1066","1013" 212 | "1066","1012" 213 | "1066","1014" 214 | "1066","1075" 215 | "1068","1051" 216 | "1069","1007" 217 | "1070","1002" 218 | "1071","1100" 219 | "1072","1175" 220 | "1073","1253" 221 | "1074","1315" 222 | "1074","1316" 223 | "1074","1277" 224 | "1074","1404" 225 | "1074","1264" 226 | "1074","1272" 227 | "1074","1314" 228 | "1074","1267" 229 | "1074","1265" 230 | "1074","1311" 231 | "1074","1323" 232 | "1074","1300" 233 | "1074","1303" 234 | "1074","1299" 235 | "1074","1313" 236 | "1074","1278" 237 | "1074","1301" 238 | "1074","1304" 239 | "1074","1271" 240 | "1074","1266" 241 | "1075","1264" 242 | "1075","1315" 243 | "1075","1271" 244 | "1075","1267" 245 | "1075","1300" 246 | "1075","1263" 247 | "1075","1265" 248 | "1075","1311" 249 | "1075","1323" 250 | "1075","1278" 251 | "1075","1314" 252 | "1075","1304" 253 | "1075","1266" 254 | "1075","1277" 255 | "1075","1301" 256 | "1075","1262" 257 | "1075","1316" 258 | "1075","1313" 259 | "1076","1087" 260 | "1076","1090" 261 | "1076","1095" 262 | "1076","1101" 263 | "1076","1089" 264 | "1076","1098" 265 | "1077","1092" 266 | "1077","1093" 267 | "1078","1091" 268 | "1080","1178" 269 | "1081","1004" 270 | "1082","1003" 271 | "1083","1051" 272 | "1084","1007" 273 | "1085","1177" 274 | "1087","1108" 275 | "1087","1122" 276 | "1087","1120" 277 | "1087","1107" 278 | "1088","1170" 279 | "1091","1002" 280 | "1094","1079" 281 | "1096","1004" 282 | "1097","1228" 283 | "1099","1004" 284 | "1101","1177" 285 | "1103","1177" 286 | "1107","1110" 287 | "1107","1112" 288 | "1112","1009" 289 | "1116","1051" 290 | "1121","1210" 291 | "1122","1002" 292 | "1122","1003" 293 | "1125","1167" 294 | "1127","1210" 295 | "1128","1218" 296 | "1134","1112" 297 | "1134","1110" 298 | "1136","1177" 299 | "1138","1228" 300 | "1143","1007" 301 | "1144","1004" 302 | "1145","1224" 303 | "1145","1226" 304 | "1145","1221" 305 | "1145","1211" 306 | "1145","1214" 307 | "1145","1217" 308 | "1145","1223" 309 | "1145","1212" 310 | "1145","1220" 311 | "1145","1219" 312 | "1145","1225" 313 | "1145","1222" 314 | "1145","1218" 315 | "1145","1227" 316 | "1145","1213" 317 | "1146","1004" 318 | "1147","1004" 319 | "1148","1002" 320 | "1153","1162" 321 | "1159","1109" 322 | "1159","1127" 323 | "1159","1128" 324 | "1159","1126" 325 | "1159","1129" 326 | "1159","1130" 327 | "1159","1125" 328 | "1159","1131" 329 | "1160","1177" 330 | "1161","1103" 331 | "1165","1175" 332 | "1167","1095" 333 | "1167","1092" 334 | "1167","1089" 335 | "1167","1087" 336 | "1167","1091" 337 | "1167","1098" 338 | "1167","1090" 339 | "1167","1101" 340 | "1167","1093" 341 | "1169","1103" 342 | "1171","1216" 343 | "1171","1215" 344 | "1177","1253" 345 | "1181","1191" 346 | "1181","1188" 347 | "1181","1192" 348 | "1185","1007" 349 | "1193","1103" 350 | "1195","1084" 351 | "1196","1082" 352 | "1198","1216" 353 | "1198","1215" 354 | "1207","1093" 355 | "1207","1092" 356 | "1207","1101" 357 | "1207","1087" 358 | "1207","1098" 359 | "1207","1090" 360 | "1207","1091" 361 | "1207","1095" 362 | "1207","1089" 363 | "1208","1165" 364 | "1211","1225" 365 | "1211","1211" 366 | "1211","1221" 367 | "1211","1216" 368 | "1211","1226" 369 | "1211","1217" 370 | "1211","1213" 371 | "1211","1222" 372 | "1211","1223" 373 | "1211","1212" 374 | "1211","1227" 375 | "1211","1224" 376 | "1211","1220" 377 | "1211","1218" 378 | "1211","1219" 379 | "1211","1215" 380 | "1213","1084" 381 | "1214","1082" 382 | "1214","1084" 383 | "1216","1214" 384 | "1217","1300" 385 | "1217","1299" 386 | "1217","1262" 387 | "1217","1267" 388 | "1217","1333" 389 | "1217","1271" 390 | "1217","1270" 391 | "1217","1273" 392 | "1217","1321" 393 | "1217","1264" 394 | "1217","1323" 395 | "1217","1313" 396 | "1217","1277" 397 | "1217","1261" 398 | "1217","1265" 399 | "1217","1268" 400 | "1217","1266" 401 | "1217","1269" 402 | "1217","1325" 403 | "1217","1263" 404 | "1220","1011" 405 | "1220","1075" 406 | "1220","1014" 407 | "1220","1012" 408 | "1220","1010" 409 | "1220","1013" 410 | "1222","1123" 411 | "1222","1120" 412 | "1222","1107" 413 | "1222","1116" 414 | "1222","1117" 415 | "1222","1122" 416 | "1222","1108" 417 | "1224","1210" 418 | "1229","1002" 419 | "1230","1009" 420 | "1232","1002" 421 | "1235","1166" 422 | "1238","1177" 423 | "1242","1007" 424 | "1243","1051" 425 | "1244","1166" 426 | "1246","1162" 427 | "1249","1175" 428 | "1250","1082" 429 | "1253","1212" 430 | "1253","1227" 431 | "1253","1221" 432 | "1253","1225" 433 | "1253","1216" 434 | "1253","1213" 435 | "1253","1214" 436 | "1253","1219" 437 | "1253","1215" 438 | "1253","1224" 439 | "1253","1222" 440 | "1253","1226" 441 | "1253","1211" 442 | "1253","1217" 443 | "1253","1223" 444 | "1253","1220" 445 | "1254","1170" 446 | "1255","1280" 447 | "1255","1299" 448 | "1255","1264" 449 | "1255","1334" 450 | "1255","1268" 451 | "1255","1273" 452 | "1255","1300" 453 | "1255","1278" 454 | "1255","1301" 455 | "1255","1260" 456 | "1255","1270" 457 | "1255","1271" 458 | "1255","1266" 459 | "1255","1267" 460 | "1255","1269" 461 | "1255","1272" 462 | "1255","1265" 463 | "1255","1261" 464 | "1255","1279" 465 | "1255","1277" 466 | "1255","1262" 467 | "1255","1259" 468 | "1255","1276" 469 | "1255","1263" 470 | "1256","1051" 471 | "1257","1222" 472 | "1257","1214" 473 | "1257","1218" 474 | "1257","1213" 475 | "1257","1219" 476 | "1257","1210" 477 | "1257","1212" 478 | "1257","1211" 479 | "1257","1220" 480 | "1257","1224" 481 | "1257","1223" 482 | "1257","1217" 483 | "1257","1227" 484 | "1257","1226" 485 | "1257","1221" 486 | "1257","1225" 487 | "1260","1176" 488 | "1261","1253" 489 | "1262","1296" 490 | "1262","1288" 491 | "1262","1286" 492 | "1262","1294" 493 | "1262","2052" 494 | "1262","1287" 495 | "1262","1289" 496 | "1262","2053" 497 | "1262","1285" 498 | "1262","1298" 499 | "1263","2038" 500 | "1263","2033" 501 | "1263","2065" 502 | "1264","1126" 503 | "1264","1109" 504 | "1264","1125" 505 | "1264","1127" 506 | "1265","1129" 507 | "1265","1128" 508 | "1265","1130" 509 | "1265","1131" 510 | "1266","1176" 511 | "1267","1109" 512 | "1267","1105" 513 | "1267","1129" 514 | "1267","1128" 515 | "1267","1130" 516 | "1267","1131" 517 | "1267","1125" 518 | "1267","1126" 519 | "1267","1127" 520 | "1268","1118" 521 | "1268","2032" 522 | "1268","1119" 523 | "1268","1122" 524 | "1268","1108" 525 | "1268","1120" 526 | "1268","1107" 527 | "1268","1161" 528 | "1268","1121" 529 | "1268","1159" 530 | "1268","2028" 531 | "1268","2029" 532 | "1268","1160" 533 | "1269","1051" 534 | "1270","1007" 535 | "1271","2059" 536 | "1271","1111" 537 | "1272","1340" 538 | "1272","2051" 539 | "1272","2050" 540 | "1272","1341" 541 | "1272","2049" 542 | "1273","2057" 543 | "1274","2062" 544 | "1274","2061" 545 | "1274","1286" 546 | "1274","2060" 547 | "1274","1284" 548 | "1274","1285" 549 | "1274","1288" 550 | "1274","1294" 551 | "1274","1296" 552 | "1274","2052" 553 | "1275","1076" 554 | "1275","1077" 555 | "1275","2063" 556 | "1276","2058" 557 | "1277","2058" 558 | "1278","2033" 559 | "1279","1004" 560 | "1280","2069" 561 | -------------------------------------------------------------------------------- /demos/nutrition/FoodData_Central_foundation_food_csv_2023-10-26/measure_unit.csv: -------------------------------------------------------------------------------- 1 | "id","name" 2 | "1000","cup" 3 | "1001","tablespoon" 4 | "1002","teaspoon" 5 | "1003","liter" 6 | "1004","milliliter" 7 | "1005","cubic inch" 8 | "1006","cubic centimeter" 9 | "1007","gallon" 10 | "1008","pint" 11 | "1009","fl oz" 12 | "1010","paired cooked w" 13 | "1011","paired raw w" 14 | "1012","dripping w" 15 | "1013","bar" 16 | "1014","bird" 17 | "1015","biscuit" 18 | "1016","bottle" 19 | "1017","box" 20 | "1018","breast" 21 | "1019","can" 22 | "1020","chicken" 23 | "1021","chop" 24 | "1022","cookie" 25 | "1023","container" 26 | "1024","cracker" 27 | "1025","drink" 28 | "1026","drumstick" 29 | "1027","fillet" 30 | "1028","fruit" 31 | "1029","large" 32 | "1030","lb" 33 | "1031","leaf" 34 | "1032","leg" 35 | "1033","link" 36 | "1034","links" 37 | "1035","loaf" 38 | "1036","medium" 39 | "1037","muffin" 40 | "1038","oz" 41 | "1039","package" 42 | "1040","packet" 43 | "1041","patty" 44 | "1042","patties" 45 | "1043","piece" 46 | "1044","pieces" 47 | "1045","quart" 48 | "1046","roast" 49 | "1047","sausage" 50 | "1048","scoop" 51 | "1049","serving" 52 | "1050","slice" 53 | "1051","slices" 54 | "1052","small" 55 | "1053","stalk" 56 | "1054","steak" 57 | "1055","stick" 58 | "1056","strip" 59 | "1057","tablet" 60 | "1058","thigh" 61 | "1059","unit" 62 | "1060","wedge" 63 | "1061","orig ckd g" 64 | "1062","orig rw g" 65 | "1063","medallion" 66 | "1064","pie" 67 | "1065","wing" 68 | "1066","back" 69 | "1067","olive" 70 | "1068","pocket" 71 | "1069","order" 72 | "1070","shrimp" 73 | "1071","each" 74 | "1072","filet" 75 | "1073","plantain" 76 | "1074","nugget" 77 | "1075","pretzel" 78 | "1076","corndog" 79 | "1077","spear" 80 | "1078","sandwich" 81 | "1079","tortilla" 82 | "1080","burrito" 83 | "1081","taco" 84 | "1082","tomatoes" 85 | "1083","chips" 86 | "1084","shell" 87 | "1085","bun" 88 | "1086","crust" 89 | "1087","sheet" 90 | "1088","bag" 91 | "1089","bagel" 92 | "1090","bowl" 93 | "1091","breadstick" 94 | "1092","bulb" 95 | "1093","cake" 96 | "1094","carton" 97 | "1095","chunk" 98 | "1096","contents" 99 | "1097","cutlet" 100 | "1098","doughnut" 101 | "1099","egg" 102 | "1100","fish" 103 | "1101","foreshank" 104 | "1102","frankfurter" 105 | "1103","fries" 106 | "1104","head" 107 | "1105","jar" 108 | "1106","loin" 109 | "1107","pancake" 110 | "1108","pizza" 111 | "1109","rack" 112 | "1110","ribs" 113 | "1111","roll" 114 | "1112","shank" 115 | "1113","shoulder" 116 | "1114","skin" 117 | "1115","wafers" 118 | "1116","wrap" 119 | "1117","bunch" 120 | "1118","Tablespoons" 121 | "1119","Banana" 122 | "1120","Onion" 123 | "9999","undetermined" 124 | -------------------------------------------------------------------------------- /demos/nutrition/README.md: -------------------------------------------------------------------------------- 1 | # Nutrition 2 | 3 | Following up from [But why?](../../README.md#but-why) lets look at the USDA foundation foods [24MB CSV](https://fdc.nal.usda.gov/download-datasets.html), together with the PDF description also mentioned in the website. 4 | I've added them both here in the demo folder. 5 | 6 | 7 | ## Look around 8 | 9 | I looked around with Apple Numbers. A couple of observations: 10 | - looks like db tables with ids to other tables 11 | - the main tables seem to be food.csv, nutrient.csv, and food_nutrient.csv 12 | - in nutrient.csv the most interesting ids seem to be 1003 (protein), 1004 (fat), 1005 (carbohydrates), 1063 (sugars total), 1235 (added sugars), 1079 (fibre), 1008 (energy), but there's lots of entries and some like energy and carbohydrates seem to be repeated 13 | - in food.csv, the important ones seem to be data_type=foundation_food 14 | - food_nutrient.csv links food.csv and nutrient.csv and the PDF says amount of nutrient is per 100g 15 | - filtering food_nutrient.csv by food freezes my Numbers app 16 | 17 | At this point I think it'd be pretty easy to load up data from these CSVs into memory and do some basic filtering to only get foundation food nutrient for the ones I identified. 18 | 19 | But I don't really know if the nutrients I picked are the right ones. 20 | And I'm a bit resentful of Numbers crashing on some basic filtering over a 8mb CSV, which meant I had to eyeball stuff and scroll back and forth a lot. 21 | Surely I can do better than scrolling and cmd+f with my high powered laptop. 22 | 23 | Lets load this data up into a in-memory database and take a look around. 24 | 25 | First thing we'll need is a CSV parsing lib, and clojure has a first party one. 26 | Add it to `fdbconfig.edn` under `extra-deps`. 27 | There's no need to restart `fdb watch`, it will see that `fdbconfig.edn` changed and reload itself. 28 | 29 | 30 | ``` edn 31 | :extra-deps {org.clojure/data.csv {:mvn/version "1.1.0"}} 32 | ``` 33 | 34 | Lets take it for a spin in `~/fdb/demos/reference/nutrition/fdb.repl.edn`: 35 | 36 | ``` clojure 37 | (require '[babashka.fs :as fs] 38 | '[clojure.data.csv :as csv] 39 | '[clojure.java.io :as io] 40 | '[fdb.call :as call]) 41 | 42 | ;; From https://github.com/clojure/data.csv?tab=readme-ov-file#parsing-into-maps 43 | (defn csv-data->maps [csv-data] 44 | (map zipmap 45 | (->> (first csv-data) ;; First row is the header 46 | (map keyword) ;; Drop if you want string keys instead 47 | repeat) 48 | (rest csv-data))) 49 | 50 | (def csv-dir (-> (call/arg) 51 | :self-path 52 | fs/parent 53 | (fs/path "FoodData_Central_foundation_food_csv_2023-10-26"))) 54 | 55 | (defn read-csv [filename] 56 | (with-open [reader (->> filename (fs/path csv-dir) str io/reader)] 57 | (doall 58 | (csv-data->maps (csv/read-csv reader))))) 59 | 60 | ;; defonce so we don't do this again each time we eval 61 | ;; if you need to reset them, just change it to def 62 | (defonce food (read-csv "food.csv")) 63 | (defonce nutrient (read-csv "nutrient.csv")) 64 | (defonce food-nutrient (read-csv "food-nutrient.csv")) 65 | 66 | ;; Take a peek at the data we got 67 | (take 5 food) 68 | ``` 69 | 70 | You should see a some data printed in `~/fdb/demos/reference/nutrition/repl-out.fdb.clj`. 71 | If you have a code editor configured for Clojure development you can connect to the `fdb watch` nREPL server on port 2525 and eval the `fdb.repl.edn` there instead. 72 | 73 | ``` clojure 74 | ;; => ({:fdc_id "319874", 75 | ;; :data_type "sample_food", 76 | ;; :description "HUMMUS, SABRA CLASSIC", 77 | ;; :food_category_id "16", 78 | ;; :publication_date "2019-04-01"} 79 | ;; {:fdc_id "319875", 80 | ;; :data_type "market_acquisition", 81 | ;; :description "HUMMUS, SABRA CLASSIC", 82 | ;; :food_category_id "16", 83 | ;; :publication_date "2019-04-01"} 84 | ;; {:fdc_id "319876", 85 | ;; :data_type "market_acquisition", 86 | ;; :description "HUMMUS, SABRA CLASSIC", 87 | ;; :food_category_id "16", 88 | ;; :publication_date "2019-04-01"} 89 | ;; {:fdc_id "319877", 90 | ;; :data_type "sub_sample_food", 91 | ;; :description "Hummus", 92 | ;; :food_category_id "16", 93 | ;; :publication_date "2019-04-01"} 94 | ;; {:fdc_id "319878", 95 | ;; :data_type "sub_sample_food", 96 | ;; :description "Hummus", 97 | ;; :food_category_id "16", 98 | ;; :publication_date "2019-04-01"}) 99 | ``` 100 | 101 | Results look good enough to shove into a db and query. 102 | Let's make a in-memory XTDB node and just push everything there. 103 | This isn't going to be in the FileDB database itself - it's just a temporary db we're making to explore the data. 104 | Making temporary in-memory databases in Clojure is pretty easy. 105 | 106 | We're going to have to pay some attention to ids. 107 | We want to map them to `:xt/id` so we can use [pull](https://v1-docs.xtdb.com/language-reference/datalog-queries/#pull) to scoop up referenced data. 108 | `food` and `food-nutrient` have their own ids and they don't seem to collide, so we can use those directly. 109 | `food` has fdc_id, and also doesn't doesn't seem to collide with the other ids. 110 | 111 | Add this to the end of the repl file: 112 | 113 | ``` clojure 114 | (require '[xtdb.api :as xt]) 115 | 116 | (defonce node (xt/start-node {})) 117 | 118 | (defn add-xtid-and-submit 119 | [coll from-k] 120 | (->> coll 121 | (mapv (fn [m] 122 | [::xt/put (assoc m :xt/id (get m from-k))])) 123 | (xt/submit-tx node))) 124 | 125 | (defonce food-tx (add-xtid-and-submit food :fdc_id)) 126 | (defonce nutrient-tx (add-xtid-and-submit nutrient :id)) 127 | (defonce food-nutrient-tx (add-xtid-and-submit food-nutrient :id)) 128 | 129 | ;; Wait for the db to catch up to the txs before querying 130 | (xt/sync node) 131 | 132 | (xt/q (xt/db node) 133 | '{:find [(pull ?e [*])] 134 | :where [[?e :data_type "foundation_food"]] 135 | :limit 3}) 136 | ``` 137 | 138 | And in the outputs you should see: 139 | 140 | ``` clojure 141 | ;; => [[{:fdc_id "1104647", 142 | ;; :data_type "foundation_food", 143 | ;; :description "Garlic, raw", 144 | ;; :food_category_id "11", 145 | ;; :publication_date "2020-10-30", 146 | ;; :xt/id "1104647"}] 147 | ;; [{:fdc_id "1104705", 148 | ;; :data_type "foundation_food", 149 | ;; :description "Flour, soy, defatted", 150 | ;; :food_category_id "16", 151 | ;; :publication_date "2020-10-30", 152 | ;; :xt/id "1104705"}] 153 | ;; [{:fdc_id "1104766", 154 | ;; :data_type "foundation_food", 155 | ;; :description "Flour, soy, full-fat", 156 | ;; :food_category_id "16", 157 | ;; :publication_date "2020-10-30", 158 | ;; :xt/id "1104766"}]] 159 | ``` 160 | 161 | Let's now get all the nutrients too. 162 | XTDB is pretty good at this with the pull projection. 163 | 164 | ``` clojure 165 | (xt/q (xt/db node) 166 | '{:find [(pull ?e [:description 167 | {:_fdc_id [:amount 168 | {:nutrient_id [:name 169 | :unit_name]}]}])] 170 | :where [[?e :data_type "foundation_food"]] 171 | :limit 1}) 172 | ``` 173 | 174 | Which gets us: 175 | 176 | ``` clojure 177 | ;; => [[{:description "Garlic, raw", 178 | ;; :_fdc_id 179 | ;; ({} 180 | ;; {:amount "2.7", 181 | ;; :nutrient_id {:name "Fiber, total dietary", :unit_name "G"}} 182 | ;; {:amount "63.1", :nutrient_id {:name "Water", :unit_name "G"}} 183 | ;; {:amount "9.8", 184 | ;; :nutrient_id {:name "Selenium, Se", :unit_name "UG"}} 185 | ;; {:amount "1.06", :nutrient_id {:name "Nitrogen", :unit_name "G"}} 186 | ;; {:amount "1.71", :nutrient_id {:name "Ash", :unit_name "G"}} 187 | ;; {:amount "0.38", 188 | ;; :nutrient_id {:name "Total lipid (fat)", :unit_name "G"}} 189 | ;; {:amount "10.0", 190 | ;; :nutrient_id 191 | ;; {:name "Vitamin C, total ascorbic acid", :unit_name "MG"}} 192 | ;; {:amount "6.62", :nutrient_id {:name "Protein", :unit_name "G"}} 193 | ;; {:amount "28.2", 194 | ;; :nutrient_id 195 | ;; {:name "Carbohydrate, by difference", :unit_name "G"}} 196 | ;; {:amount "143.0", :nutrient_id {:name "Energy", :unit_name "KCAL"}} 197 | ;; {:amount "597.0", :nutrient_id {:name "Energy", :unit_name "kJ"}} 198 | ;; {:amount "130", 199 | ;; :nutrient_id 200 | ;; {:name "Energy (Atwater Specific Factors)", :unit_name "KCAL"}} 201 | ;; {:amount "143", 202 | ;; :nutrient_id 203 | ;; {:name "Energy (Atwater General Factors)", :unit_name "KCAL"}})}]] 204 | ``` 205 | 206 | The pull projection is a bit gnarly: 207 | 208 | ``` edn 209 | (pull ?e [:description 210 | {:_fdc_id [:amount 211 | {:nutrient_id [:name 212 | :unit_name]}]}]) 213 | ``` 214 | 215 | It means: 216 | - for the entity (each foundation food) get `:description` and what's in the `{}` 217 | - `:_fdc_id` pulls all references back to this entity through the `:fdc_id` key (you can do this with any key by prefixing it with `_`) 218 | - for those we're getting `:amount` what's in the `{}` 219 | - `:nutrient_id` pulls what's referenced in this entity in the `:nutrient_id` key 220 | - for those we're getting `:name` and `:amount` 221 | 222 | It's a bit easier if we look at it as built iteratively, each time replacing one of the pulled keys by `{:key pull-pattern}`: 223 | 224 | ``` edn 225 | ;; pull description and all the id of all the entities that references ?e (backrefs) via the :fdc_id key 226 | ;; NB: this won't actually return the backrefs ids, it's just easier to think about 227 | (pull ?e [:description 228 | :_fdc_id]) 229 | 230 | ;; now for those entities, pull :amount and :nutrient_id keys 231 | (pull ?e [:description 232 | {:_fdc_id [:amount 233 | :nutrient_id]}]) 234 | 235 | ;; for the entity referenced in :nutrient_id, pull :name and :unit_name 236 | (pull ?e [:description 237 | {:_fdc_id [:amount 238 | {:nutrient_id [:name 239 | :unit_name]}]}]) 240 | ``` 241 | 242 | It might feel daunting but at the end of the day it's an impressive amount of power and expressiveness in 4 lines. 243 | This is exactly what I'm looking for when hacking together my own tools. 244 | 245 | At this point we've ascertained we can get the data we want from the CSV sources, in about 60 lines of code and pretty fast feedback cycles. 246 | I didn't make the code I put here right on the first try, but I did iterate and debug it with these fast feedback cycles until it was doing what I wanted, and it didn't take a long time. 247 | 248 | 249 | ## Chart a course 250 | 251 | Now I feel pretty confident about what I want to extract from these CSVs. 252 | What do I do with the data after extracting it though? 253 | 254 | I want to: 255 | 1. look it up offline on both desktop and mobile 256 | 2. reference it 257 | 3. calculate meal, daily, and weekly nutrition totals 258 | 259 | I've been able to do some of these things from apps but never all of these things. 260 | Point 2 is particularly hard since apps are so isolated and I don't really own my data in most of them. 261 | 262 | I'm a heavy [Obsidian](https://obsidian.md) user so my thinking right now is that I should make a markdown file for each foundation food with its nutrition data in [YML properties](https://help.obsidian.md/Editing+and+formatting/Properties#Property%20format). 263 | Then I can reference it easily (point 1), and I have a little food database on desktop and mobile (point 2). 264 | 265 | We could transact this data directly into the fdb node instead of putting it in a file. 266 | But that would mean we couldn't do point 1 or 2 as easily, and it wouldn't survive a database wipe or file sync to another machine. 267 | 268 | I don't see any obvious way of calculating stuff (point 3) in Obsidian from YML properties though, nor from any data stored in markdown. 269 | But we have triggers so we should be able to somehow trigger the calculation and write back to the file. 270 | I'm a bit fuzzy on this part right now, but it seems doable, and it's a starting point. 271 | 272 | You don't have to use Obsidian if you don't want to. 273 | You can use just markdown files on disk. 274 | Or something else. 275 | You do you. 276 | Even the most budget computer or phone around right now is more than capable of working through all of the personal data you can accumulate during your whole lifetime. 277 | Make it work for you. 278 | 279 | 280 | ## Make it happen 281 | 282 | Obsidian with YML is pretty straightforward: shove the yml into `---` fences at the start of the doc. 283 | Clojure has a first party yml lib we can use so we don't have to think too much about it, add it to `fdbconfig.edn` under `extra-deps`: 284 | 285 | ``` edn 286 | clj-commons/clj-yaml {:mvn/version "1.0.27"} 287 | ``` 288 | 289 | And add this code to make a markdown file with yml: 290 | 291 | ``` clojure 292 | (require '[clj-yaml.core :as yaml]) 293 | 294 | (defonce foods 295 | (->> '{:find [(pull ?e [:description 296 | {:_fdc_id [:amount 297 | {:nutrient_id [:xt/id 298 | :name 299 | :unit_name]}]}])] 300 | :where [[?e :data_type "foundation_food"]]} 301 | (xt/q (xt/db node)) 302 | (map first))) 303 | 304 | (defonce nutrients #{"1003" "1004" "1005" "1008" "1063" "1079"}) 305 | 306 | (defn to-markdown-yml [food] 307 | (str "---\n" 308 | (yaml/generate-string 309 | (->> (:_fdc_id food) 310 | (filter #(-> % :nutrient_id :xt/id nutrients)) 311 | (map (fn [{:keys [amount] {:keys [name unit_name]} :nutrient_id}] 312 | [(format "%s (%s)" name unit_name) (parse-double amount)])) 313 | sort 314 | (into {})) 315 | :dumper-options {:flow-style :block}) 316 | "---\n")) 317 | 318 | (println (->> foods 319 | first 320 | to-markdown-yml)) 321 | ``` 322 | 323 | This should print: 324 | 325 | ``` clojure 326 | ;; --- 327 | ;; Carbohydrate, by difference (G): 4.91 328 | ;; Energy (KCAL): 50.0 329 | ;; Protein (G): 3.35 330 | ;; Sugars, Total (G): 4.89 331 | ;; Total lipid (fat) (G): 1.9 332 | ;; --- 333 | ``` 334 | 335 | Which looks about right, considering the `;; ` are just from the output format. 336 | 337 | Now we write all these markdown files to disk. 338 | 339 | ``` clojure 340 | (require '[fdb.utils :as u]) 341 | 342 | (defonce food-folder (u/sibling-path (:self-path (call/arg)) "foods/")) 343 | (fs/create-dirs food-folder) 344 | 345 | (defn write-to-md 346 | [{:keys [description] :as food}] 347 | (spit (str food-folder "/" (u/filename-str description) ".md") 348 | (to-markdown-yml food))) 349 | 350 | (run! write-to-md foods) 351 | ``` 352 | 353 | 354 | TODO: not all these foods have energy, but they seem to have protein/fat/carbs, maybe add a step to verify? 355 | think I need a fallback to the atwater ones 356 | 357 | 358 | 359 | TODO: lean into obsidian, put something that can be used as vault in demo, but work over md to allow everyone to follow 360 | TODO: put in md, reference, trigger compute 361 | TODO: u/file-replace, for regex replacement on files 362 | TODO: some cool datalog queries for nutrition data, like foods with low carbs or high protein (hard without md reader) 363 | -------------------------------------------------------------------------------- /demos/nutrition/dataDictionary.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filipesilva/fdb/ef58e652c70edfe37337201f22193290cd2adc00/demos/nutrition/dataDictionary.pdf -------------------------------------------------------------------------------- /demos/nutrition/query.fdb.edn: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /demos/nutrition/repl.fdb.clj: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filipesilva/fdb/ef58e652c70edfe37337201f22193290cd2adc00/demos/nutrition/repl.fdb.clj -------------------------------------------------------------------------------- /demos/reference/call-arg.edn: -------------------------------------------------------------------------------- 1 | ;; This is the format for call arg, which the function resolved for call-spec is called with. 2 | ;; It's also acessible in (fdb.call/arg). 3 | {:config {,,,} ;; fdb config value 4 | :config-path "~/fdb/fdbconfig.json" ;; on-disk path to config 5 | :node {,,,} ;; xtdb database node 6 | :db {,,,} ;; xtdb db value at the time of the tx 7 | :tx {:xtdb.api/id 1 ,,,} ;; the tx 8 | :on println ;; the trigger being called 9 | :on-path [:fdb.on/modify] ;; get-in path inside self for trigger 10 | :self {:xt/id "/mount/foo.md" ,,,} ;; the doc that has the trigger being called 11 | :self-path "/path/foo.md" ;; on-disk path for self 12 | :target {:xt/id "/mount/bar.md" ,,,} ;; the doc the trigger is being called over, if any 13 | :target-path "/path/bar.md" ;; on-disk path for doc, if any 14 | :results {,,,} ;; query results, if any 15 | :timestamp "2024-03-22T16:52:20.995717Z" ;; schedule timestamp, if any 16 | :req {:path-params ["42"] ,,,} ;; http request, if any 17 | } 18 | -------------------------------------------------------------------------------- /demos/reference/call-spec.edn: -------------------------------------------------------------------------------- 1 | ;; These are the different formats supported for call-spec, used in readers and triggers. 2 | 3 | ;; A function name will be required and resolved under the user ns, then called with call-arg. 4 | println 5 | clojure.core/println 6 | 7 | ;; A sexp containing a function, evaluated then called with call-arg. 8 | (fn [{:keys [self-path]}] 9 | (println self-path)) 10 | 11 | ;; A vector uses the first kw element to decide what to do. 12 | ;; The only built-in resolution is :sh, that calls a shell command and can use a few bindings from call-arg. 13 | ;; You can add your own with (defmethod fdb.call/to-fn :my-thing ...) 14 | [:sh "echo" config-path target-path self-path] 15 | 16 | ;; A map containing :call, which is any of the above 17 | ;; You can put more data in this map, and since call-arg has the trigger iself in :on, you can use 18 | ;; this data to parametrize the call. 19 | {:call (fn [{:keys [self-path on]}] 20 | (println self-path (:my-data on))) 21 | :my-data 42} 22 | 23 | ;; Call-specs can always be one or many, and are called in sequence. 24 | [println 25 | {:call println} 26 | [:sh "echo" self-path]] 27 | -------------------------------------------------------------------------------- /demos/reference/doc.md: -------------------------------------------------------------------------------- 1 | Just your friendly neighborhood document. 2 | My mount path, modified date, and parent folder will be loaded onto fdb together with metadata in doc.md.meta.edn. 3 | -------------------------------------------------------------------------------- /demos/reference/doc.md.meta.edn: -------------------------------------------------------------------------------- 1 | ;; This both the format of db and on-disk metadata files. 2 | {;; ID is /mount/ followed by relative path on mount. 3 | ;; It's the unique id for XTDB. 4 | ;; Added automatically. 5 | :xt/id "/demos/reference/doc.md" 6 | 7 | ;; Modified is the most recent between doc.md and doc.md.meta.edn. 8 | ;; Added automatically. 9 | :fdb/modified "2021-03-21T20:00:00.000-00:00" 10 | 11 | ;; The ID of the parent of this ID, useful for recursive queries 12 | ;; Added automatically. 13 | :fdb/parent "/demos/reference" 14 | 15 | ;; ID references are useful enough in relating docs that they're first class. 16 | :fdb/refs #{"/demos/reference/todo.md" 17 | "/demos/reference/ref-one.md"} 18 | 19 | ;; Called when this file, or its metadata, is modified. 20 | ;; The fn will be called with the call-arg. 21 | ;; print-call-arg is a function that we added in repl.fdb.edn, so we can use it here. 22 | :fdb.on/modify print-call-arg 23 | 24 | ;; Called when any file that matches the glob changes. 25 | ;; It should match ./pattern-glob-match.md. 26 | :fdb.on/pattern {:glob "/demos/reference/*glob*.md" 27 | :call print-call-arg} 28 | 29 | ;; Called when the files referenced in :fdb/refs change. 30 | ;; Refs will be resolved recursively and you can have cycles, so this triggers 31 | ;; when ./ref-two.md or ./ref-three are modified too. 32 | :fdb.on/refs print-call-arg 33 | 34 | ;; Called when the query results change. 35 | ;; The latest results will be in important-files.edn, specified in the :path key. 36 | ;; You can add triggers to path metadata, or use it as a ref to other triggers. 37 | :fdb.on/query {:q [:find ?e 38 | :where [?e :tags "important"]] 39 | :path "./important-files.edn" 40 | :call print-call-arg} 41 | 42 | ;; Called every 1 hours. 43 | ;; The :every syntax supports :seconds :hours :days and more, see (keys tick.core/unit-map). 44 | ;; You can also use a cron schedule, use https://crontab.guru/ to make your cron schedules. 45 | :fdb.on/schedule {:every [1 :hours] 46 | ;; or :cron "0 * * * *" 47 | :call print-call-arg} 48 | 49 | ;; Called once on watch startup/shutdown, including restarts. 50 | :fdb.on/startup print-call-arg 51 | :fdb.on/shutdown print-call-arg 52 | 53 | ;; Called on every db transaction via https://v1-docs.xtdb.com/clients/clojure/#_listen 54 | ;; This is how every other trigger is made, so you can make your own triggers. 55 | :fdb.on/tx print-call-arg 56 | } 57 | -------------------------------------------------------------------------------- /demos/reference/fdbconfig.edn: -------------------------------------------------------------------------------- 1 | ;; This is the format of the fdb config file. Your real one is probably on your ~/fdb/fdbconfig.edn. 2 | {;; Where the xtdb db files will be saved. 3 | ;; You can delete this at any time, and the latest state will be recreated from the mount files. 4 | ;; You'll lose time-travel data if you delete it though. 5 | ;; See more about xtdb time travel in https://v1-docs.xtdb.com/concepts/bitemporality/. 6 | :db-path "./xtdb" 7 | 8 | ;; These paths that will be mounted on the db. 9 | ;; If you have ~/fdb/user mounted as :user, and you have ~/fdb/user/repl.fdb.clj, 10 | ;; its id in the db will be /user/repl.fdb.clj. 11 | :mounts {;; "~/fdb/user is the same as {:path "~/fdb/user"} 12 | :user "~/fdb/user"} 13 | 14 | ;; Readers are fns that will read some data from a file as edn when it changes 15 | ;; and add it to the db together with the metadata. 16 | ;; The key is the file extension. 17 | ;; They are called with the call-arg (see below) just like triggers. 18 | ;; Call `fdb read glob-pattern` if you change readers and want to force a re-read. 19 | ;; Defaults to :edn, :md, and :eml readers in fdb/src/readers but can be overwritten 20 | ;; with :readers instead of :extra-readers. 21 | ;; You can also add :extra-readers to a single mount in the map notation. 22 | :extra-readers {:txt user/read-txt} 23 | 24 | ;; Mount or real paths of clj files to be loaded at the start. 25 | ;; Usually repl files where you added fns to use in triggers, or that load namespaces 26 | ;; you want to use without requiring them, or server handlers. 27 | :load ["/user/load-repl.fdb.clj" 28 | "/user/server-repl.fdb.clj"] 29 | 30 | ;; These are Clojure deps loaded dynamically at the start, and reloaded when config changes. 31 | ;; You can add your local deps here too, and use them in triggers. 32 | ;; See https://clojure.org/guides/deps_and_cli for more about deps. 33 | :extra-deps {org.clojure/data.csv {:mvn/version "1.1.0"} 34 | org.clojure/data.json {:git/url "https://github.com/clojure/data.json" 35 | :git/sha "e9e57296e12750512788b723e49ba7f9abb323f9"} 36 | my-local-lib {:local/root "/path/to/lib"}} 37 | 38 | ;; Serve call-specs from fdb. 39 | ;; Use with https://ngrok.com or https://github.com/localtunnel/localtunnel to make a public server. 40 | :serve {;; Map from route to call-spec, req will be within call-arg as :req. 41 | ;; Route format is from https://github.com/tonsky/clj-simple-router. 42 | :routes {"GET /" user/get-root 43 | "GET /stuff/*" user/get-stuff} 44 | ;; Server options for https://github.com/http-kit/http-kit. 45 | ;; Defaults to {:port 80}. 46 | :opts {:port 8081}} 47 | 48 | ;; Mail configuration. 49 | ;; Fetches through IMAP and sends through SMTP. 50 | ;; To get a gmail app password see https://support.google.com/mail/answer/185833?hl=en 51 | ;; IMAP and SMTP defaults to gmail, but you can configure it under the following keys: 52 | ;; {:imap {:host "imap.gmail.com"} 53 | ;; :smtp {:host "smtp.gmail.com" :port 587}} 54 | :email {:email "your@email.com" 55 | :password "password123" 56 | ;; or if you prefer to fetch them from env vars 57 | ;; :email-env "FDB_GMAIL_EMAIL" 58 | ;; :password-env "FDB_GMAIL_PASSWORD" 59 | } 60 | 61 | ;; Files and folders to ignore when watching for changes. 62 | ;; default is [".DS_Store" ".git" ".gitignore" ".obsidian" ".vscode" "node_modules" "target" ".cpcache"] 63 | ;; You can add to the defaults with :extra-ignore, or overwrite it with :ignore. 64 | ;; You can also use :ignore and :extra-ignore on the mount map definition. 65 | :extra-ignore [".noisy-folder"] 66 | 67 | ;; nRepl options, port defaults to 2525. 68 | ;; Started automatically on watch, lets you connect directly from your editor to the fdb process. 69 | ;; Also used by the fdb cli to connnect to the background clojure process. 70 | ;; See https://nrepl.org/nrepl/usage/server.html#server-options for more. 71 | :repl {} 72 | 73 | ;; You can add your own stuff here, and since the call-arg gets the config you will 74 | ;; be able to look up your config items on triggers and readers. 75 | :my-stuff "personal config data I want to use in fns"} 76 | -------------------------------------------------------------------------------- /demos/reference/pattern-glob-match.md: -------------------------------------------------------------------------------- 1 | Changes to this file will trigger fdb.on/pattern in doc.md.meta.edn 2 | -------------------------------------------------------------------------------- /demos/reference/query.fdb.edn: -------------------------------------------------------------------------------- 1 | ;; When you save this file it will be be used as a query, and the result will 2 | ;; be printed to query-out.fdb.edn 3 | ;; See https://v1-docs.xtdb.com/language-reference/datalog-queries/ for query syntax 4 | 5 | ;; All files marked as important. 6 | {:find [?e] 7 | :where [[?e :tags "important"]]} 8 | 9 | ;; Some other common queries. 10 | ;; #_ means they are commented out, so uncomment the one you want to use and leave 11 | ;; all others commented. 12 | 13 | ;; Pull all data for files in this directory 14 | #_{:find [(pull e [*])] 15 | :where [[e :fdb/parent "/demos/reference"]]} 16 | -------------------------------------------------------------------------------- /demos/reference/query.fdb.md: -------------------------------------------------------------------------------- 1 | ```edn 2 | ;; works like query.fdb.edn 3 | [:find ?e 4 | :where [?e :tags "important"]] 5 | ``` 6 | -------------------------------------------------------------------------------- /demos/reference/ref-one.md.meta.edn: -------------------------------------------------------------------------------- 1 | {:fdb/refs #{"/demos/reference/ref-two.md"}} 2 | -------------------------------------------------------------------------------- /demos/reference/ref-three.md.meta.edn: -------------------------------------------------------------------------------- 1 | {:fdb/refs #{"/demos/reference/ref-one.md"}} 2 | -------------------------------------------------------------------------------- /demos/reference/ref-two.md.meta.edn: -------------------------------------------------------------------------------- 1 | {:fdb/refs #{"/demos/reference/ref-three.md"}} 2 | -------------------------------------------------------------------------------- /demos/reference/repl.fdb.clj: -------------------------------------------------------------------------------- 1 | ;; We'll use this fn later in triggers. 2 | ;; Add to the load vector if adding reference as a mount 3 | ;; so it's acessible on first load. 4 | (defn print-call-arg 5 | "Simple fn to see which triggers are called." 6 | [{:keys [on-path]}] 7 | (println "=== called" (first on-path) "===")) 8 | -------------------------------------------------------------------------------- /demos/reference/repl.fdb.md: -------------------------------------------------------------------------------- 1 | ```clojure 2 | ;; works like repl.fdb.edn 3 | (inc 1) 4 | ``` 5 | -------------------------------------------------------------------------------- /demos/reference/todo.md: -------------------------------------------------------------------------------- 1 | - do taxes on the 15th 2 | - take out garbage 3 | -------------------------------------------------------------------------------- /demos/reference/todo.md.meta.edn: -------------------------------------------------------------------------------- 1 | {:tags #{"important"}} 2 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src" "test" "resources"] 2 | :deps {;; alpha6 and up suffer from https://github.com/xtdb/xtdb/issues/3169 3 | ;; keep in sync with the clojure version in fdb.bb.cli 4 | org.clojure/clojure {:mvn/version "1.12.0-alpha5"} 5 | org.clojure/core.async {:mvn/version "1.6.681"} 6 | hashp/hashp {:mvn/version "0.2.2"} 7 | com.xtdb/xtdb-core {:mvn/version "1.24.0"} 8 | com.xtdb/xtdb-rocksdb {:mvn/version "1.24.0"} 9 | com.xtdb/xtdb-lucene {:mvn/version "1.24.0"} 10 | com.nextjournal/beholder {;; https://github.com/nextjournal/beholder/pull/8 11 | :git/url "https://github.com/filipesilva/beholder.git" 12 | :git/sha "b0ca87540a28aac823791782d9f12d098ca71ed3"} 13 | babashka/fs {:mvn/version "0.4.19"} 14 | babashka/process {:mvn/version "0.5.21"} 15 | org.babashka/cli {:mvn/version "0.8.54"} 16 | com.taoensso/timbre {:mvn/version "6.3.1"} 17 | tick/tick {:mvn/version "0.7.5"} 18 | tortue/spy {:mvn/version "2.14.0"} 19 | cronstar/cronstar {:mvn/version "1.0.2"} 20 | jarohen/chime {:mvn/version "0.3.3"} 21 | nrepl/nrepl {:mvn/version "1.1.0"} 22 | cider/cider-nrepl {:mvn/version "0.45.0"} 23 | babashka/nrepl-client {:git/url "https://github.com/babashka/nrepl-client" 24 | :git/sha "f560a68bbad7b625b07b4025b240c2d0b5b610ad"} 25 | mvxcvi/puget {:mvn/version "1.3.4"} 26 | ;; for xtdb, I guess, I don't care, I just want the startup error to go away 27 | org.slf4j/slf4j-nop {:mvn/version "2.0.9"} 28 | io.github.tonsky/clj-reload {:mvn/version "0.4.3"} 29 | http-kit/http-kit {:mvn/version "2.8.0"} 30 | io.github.tonsky/clj-simple-router {:mvn/version "0.1.0"} 31 | metosin/muuntaja {:mvn/version "0.6.10"} 32 | io.github.escherize/huff {:mvn/version "0.2.12"} 33 | org.clojure/data.json {:mvn/version "2.5.0"} 34 | enlive/enlive {:mvn/version "1.1.6"} 35 | ;; built-in reader deps 36 | clj-commons/clj-yaml {:mvn/version "1.0.27"} 37 | io.forward/clojure-mail {:mvn/version "1.0.8"} 38 | com.sun.mail/gimap {:mvn/version "1.6.7"} 39 | ;; Don't really use medley directly, but clojure-mail does, and it always shows 40 | ;; a warning about abs being overwritten because it's on an old medley version 41 | dev.weavejester/medley {:mvn/version "1.7.0"}} 42 | 43 | :aliases 44 | {:neil {:project {:name fdb/fdb}}}} 45 | -------------------------------------------------------------------------------- /resources/email/sample-crlf.mbox: -------------------------------------------------------------------------------- 1 | From MAILER-DAEMON Fri Jul 8 12:08:34 2011 2 | From: Author 3 | To: Recipient 4 | Message-ID: 123 5 | Subject: Sample message 1 6 | 7 | This is the body. 8 | There are 2 lines. 9 | 10 | From MAILER-DAEMON Fri Jul 8 12:08:34 2011 11 | From: Author 12 | To: Recipient 13 | Message-ID: 456 14 | Subject: Sample message 2 15 | 16 | This is the second body. 17 | -------------------------------------------------------------------------------- /resources/eml/sample-crlf/1970-01-01T00.00.00Z 8d247ee6 Sample message 1.eml: -------------------------------------------------------------------------------- 1 | From: Author 2 | To: Recipient 3 | Message-ID: 123 4 | Subject: Sample message 1 5 | 6 | This is the body. 7 | There are 2 lines. 8 | -------------------------------------------------------------------------------- /resources/eml/sample-crlf/1970-01-01T00.00.00Z c2dfc80c Sample message 2.eml: -------------------------------------------------------------------------------- 1 | From: Author 2 | To: Recipient 3 | Message-ID: 456 4 | Subject: Sample message 2 5 | 6 | This is the second body. -------------------------------------------------------------------------------- /resources/eml/sample.eml: -------------------------------------------------------------------------------- 1 | From: Author 2 | To: Recipient 3 | Message-ID: 123 4 | Subject: Sample message 1 5 | 6 | This is the body. 7 | There are 2 lines. 8 | -------------------------------------------------------------------------------- /resources/file.txt: -------------------------------------------------------------------------------- 1 | Used by fdb.util/src-dir. 2 | -------------------------------------------------------------------------------- /resources/md/another file.md: -------------------------------------------------------------------------------- 1 | # Document Title 2 | 3 | -------------------------------------------------------------------------------- /resources/md/file.md: -------------------------------------------------------------------------------- 1 | --- 2 | tags: 3 | - accepted 4 | - applied 5 | - action-item 6 | aliases: 7 | - something 8 | - else 9 | - another 10 | cssclasses: 11 | - .foo 12 | - .bar 13 | text: one two three 14 | checkbox: true 15 | list: 16 | - one 17 | - two 18 | - three 19 | number: 14 20 | date: 1986-04-25 21 | datetime: 1986-04-25T16:00:00 22 | text-link: "[[another file]]" 23 | list-links: 24 | - "[[another file]]" 25 | - "[[other file]]" 26 | fdb/k: '{:foo "bar"}' 27 | fdb.a/ks: 28 | - "n.s/sym" 29 | - "{:call n.s/another-sym}" 30 | - '[:sh "echo"]' 31 | --- 32 | markdown body 33 | 34 | some tags #learning #accepted #1bâc #aAa 35 | 36 | a non-valid tag #123 37 | 38 | some links [[another file]] [[other file]] [a md link](./md%20link.md) 39 | 40 | link with path [[inbox/another file]] 41 | 42 | a block link [[another file#^b86803]] 43 | 44 | an alias link [[another file|foo]] 45 | -------------------------------------------------------------------------------- /resources/md/inbox/another file.md: -------------------------------------------------------------------------------- 1 | # Document Title 2 | 3 | -------------------------------------------------------------------------------- /resources/md/md link.md: -------------------------------------------------------------------------------- 1 | # Document Title 2 | 3 | -------------------------------------------------------------------------------- /resources/md/other file.md: -------------------------------------------------------------------------------- 1 | # Document Title 2 | 3 | -------------------------------------------------------------------------------- /src/fdb/autoloader.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.autoloader) 2 | 3 | ;; TODO: 4 | ;; - catch dep errors, load the lib, reeval 5 | ;; - process lib coords from https://github.com/phronmophobic/dewey 6 | -------------------------------------------------------------------------------- /src/fdb/bb/bb.edn: -------------------------------------------------------------------------------- 1 | ;; bb config for cli.clj, in this dir to allow auto-discovery when its called as executable. 2 | ;; Keep deps in sync in ../../../deps.edn, ignore babashka libs as they are included. 3 | {:paths ["../../../src" "../../../resources"] 4 | :deps {mvxcvi/puget {:mvn/version "1.3.4"} 5 | tick/tick {:mvn/version "0.7.5"} 6 | babashka/nrepl-client {:git/url "https://github.com/babashka/nrepl-client" 7 | :git/sha "f560a68bbad7b625b07b4025b240c2d0b5b610ad"}}} 8 | -------------------------------------------------------------------------------- /src/fdb/bb/cli.clj: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bb 2 | 3 | (ns fdb.bb.cli 4 | "CLI commands for fdb. 5 | Runs in a babashka environment for startup speed, deps in ./bb.edn. 6 | Creates or connects to an existing fdb process to run commands. 7 | Symlink this file to /usr/local/bin/fdb to be able to run it from anywhere 8 | ./symlink-fdb.sh 9 | or 10 | ln -s \"$(pwd)/src/fdb/bb/cli.clj\" /usr/local/bin/fdb 11 | " 12 | (:refer-clojure :exclude [sync read]) 13 | (:require 14 | [babashka.cli :as cli] 15 | [babashka.fs :as fs] 16 | [babashka.nrepl-client :as nrepl] 17 | [babashka.process :as process] 18 | [clojure.string :as str] 19 | [fdb.config :as config] 20 | [fdb.utils :as u] 21 | [taoensso.timbre :as log])) 22 | 23 | (defn escape-quotes 24 | [s] 25 | (str/escape (str s) {\" "\\\""})) 26 | 27 | (defn repl-port [config-path] 28 | (some-> config-path (u/sibling-path ".nrepl-port") u/slurp parse-long)) 29 | 30 | ;; don't attempt to start if there's a repl already 31 | (defn start-fdb-repl 32 | "Start fdb.core/repl in another process and wait until repl is up. 33 | Returns future blocking on proces. Deref future to block until process finishes." 34 | [config-path debug] 35 | (log/merge-config! {:min-level (if debug :debug :info)}) 36 | (log/info "starting fdb repl process") 37 | (let [deps {:deps {'org.clojure/clojure {:mvn/version "1.12.0-alpha5"} 38 | 'filipesilva/fdb {:local/root (u/fdb-root)}}} 39 | opts {:config-path config-path 40 | :debug debug} 41 | cmd (format "clojure -Sdeps \"%s\" -X fdb.core/repl \"%s\"" 42 | (escape-quotes deps) (escape-quotes opts)) 43 | _ (log/debug "running shell cmd" cmd) 44 | fut (future (process/shell cmd))] 45 | (while (not (repl-port config-path)) 46 | (Thread/sleep 50)) 47 | fut)) 48 | 49 | (defn eval-in-fdb [config-path sym & args] 50 | (if-let [port (repl-port config-path)] 51 | (nrepl/eval-expr {:port port :expr (format "(apply %s %s)" sym (pr-str (vec args)))}) 52 | (log/error "no fdb repl server running"))) 53 | 54 | (defn find-config-path [config] 55 | (let [config-path (config/path config)] 56 | (log/info "config found at" config-path) 57 | config-path)) 58 | 59 | (defn init [{{:keys [dir]} :opts}] 60 | (let [path (config/new-path dir)] 61 | (if (fs/exists? path) 62 | (do 63 | (log/error path "already exists!") 64 | (System/exit 1)) 65 | (let [src-demos-path (fs/path (u/fdb-root) "demos") 66 | user-path (u/sibling-path path "user") 67 | demos-path (u/sibling-path path "demos")] 68 | (fs/create-dirs user-path) 69 | (spit (str (fs/path user-path "repl.fdb.clj")) 70 | (str ";; Clojure code added here will be evaluated, output will show up in ./repl-out.fdb.clj\n" 71 | ";; Quick help: https://clojuredocs.org https://github.com/filipesilva/fdb#call-spec-and-call-arg\n")) 72 | (spit (str (fs/path user-path "load-repl.fdb.clj")) 73 | ";; Like repl.fdb.clj, but loaded at startup. Put functions you want to always have loaded here.\n") 74 | (spit (str (fs/path user-path "server-repl.fdb.clj")) 75 | ";; Like load-repl.fdb.clj, but handy to have all server handlers in one place.\n") 76 | (spit (str (fs/path user-path "query.fdb.edn")) 77 | (str ";; XTDB queries added here will be evaluated, output will show up in ./query-out.fdb.edn\n" 78 | ";; Quick help: https://v1-docs.xtdb.com/language-reference/datalog-queries/ https://www.learndatalogtoday.org\n")) 79 | (log/info "created user folder at" user-path) 80 | (fs/copy-tree src-demos-path demos-path {:replace-existing true}) 81 | (log/info "created demos folder at" demos-path) 82 | (u/spit-edn path {:db-path "./xtdb" 83 | :mounts {:user user-path} 84 | :extra-deps {} 85 | :extra-readers {} 86 | :load ["/user/load-repl.fdb.clj" 87 | "/user/server-repl.fdb.clj"] 88 | :serve {:routes {}}}) 89 | (log/info "created new config at" path))))) 90 | 91 | (defn watch [{{:keys [config debug]} :opts}] 92 | (let [config-path (find-config-path config) 93 | repl (start-fdb-repl config-path debug)] 94 | (eval-in-fdb config-path 'fdb.core/watch-config! config-path) 95 | @repl)) 96 | 97 | (defn sync [{{:keys [config debug]} :opts}] 98 | (let [config-path (find-config-path config)] 99 | (start-fdb-repl config-path debug) 100 | (eval-in-fdb config-path 'fdb.core/sync config-path))) 101 | 102 | (defn read [{{:keys [config pattern]} :opts}] 103 | (let [config-path (find-config-path config)] 104 | (eval-in-fdb config-path 'fdb.core/read config-path (str (fs/cwd)) pattern))) 105 | 106 | (def spec {:config {:desc "The FileDB config file. Defaults to ./fdbconfig.edn or ~/fdb/fdbconfig.edn." 107 | :alias :c} 108 | :debug {:desc "Print debug info." 109 | :alias :d 110 | :default false 111 | :coerce :boolean}}) 112 | 113 | (defn help [_ms] 114 | (println (format 115 | "FileDB is a hackable database environment for your file library. 116 | Local docs at %s/README.md. 117 | Repo at https://github.com/filipesilva/fdb. 118 | 119 | Available commands: 120 | fdb init Add a empty fdbconfig.edn at path-to-dir or ~/fdb/ if omitted. 121 | fdb watch Start fdb in watch mode, reacting to file changes as they happen. 122 | fdb sync Run fdb once, updating db to current file state and calling any triggers. 123 | fdb read Read all files matched by glob-pattern. Use after changing readers. 124 | 125 | All commands take the following options: 126 | %s " 127 | (u/fdb-root) 128 | (cli/format-opts {:spec spec})))) 129 | 130 | (def table 131 | [{:cmds [] :fn help :spec spec} 132 | {:cmds ["init"] :fn init :args->opts [:dir]} 133 | {:cmds ["watch"] :fn watch} 134 | {:cmds ["sync"] :fn sync} 135 | {:cmds ["read"] :fn read :args->opts [:pattern]}]) 136 | 137 | (defn -main [& args] 138 | (cli/dispatch table args)) 139 | 140 | ;; Call main when bb is called over this file. 141 | (when (= *file* (System/getProperty "babashka.file")) 142 | (apply -main *command-line-args*)) 143 | 144 | ;; TODO: 145 | ;; - can I start clojure on fdb-root instead of trying making a new deps? 146 | ;; - could enable using aliases, improve repl setup 147 | ;; - atm I'm auto-adding cider stuff 148 | ;; - maybe I can just add them in my own fdbconfig.json 149 | ;; - maybe I can do something fancy with deps alias and config merging from fdbconfig.edn 150 | ;; - actually, global clojure deps should work for defining aliases 151 | ;; - so just a way to chose an alias for the project, I guess --aliases flag on all args 152 | -------------------------------------------------------------------------------- /src/fdb/call.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.call 2 | (:refer-clojure :exclude [apply]) 3 | (:require 4 | [babashka.fs :as fs] 5 | [babashka.process :refer [shell]] 6 | [fdb.utils :as u] 7 | [taoensso.timbre :as log])) 8 | 9 | (defn specs 10 | "Returns a seq of call-specs from x." 11 | [x] 12 | (cond 13 | ;; looks like a sexp call-spec 14 | (list? x) 15 | [x] 16 | 17 | ;; looks like a vector call-spec 18 | (and (vector? x) 19 | (keyword? (first x))) 20 | [x] 21 | 22 | :else 23 | (u/one-or-many x))) 24 | 25 | (defmulti to-fn 26 | "Takes call-spec and returns a function that takes a call-arg. 27 | call-specs are dispatched by type of x, or by keyword if it's 28 | a vector with a keyword first element: 29 | - map Use :call key to resolve fn 30 | - symbol Resolves and returns the sym 31 | - var Calls var-get on var 32 | - list Evaluates and returns the result 33 | - :sh Runs shell command via babashka.process/shell 34 | You can use the shell option map, and the config-path, 35 | target-path and self-path bindings." 36 | (fn [call-spec] 37 | (if (and (vector? call-spec) 38 | (keyword? (first call-spec))) 39 | (first call-spec) 40 | (type call-spec)))) 41 | 42 | (defmethod to-fn :default 43 | [call-spec] 44 | (fn [_call-arg] 45 | (log/error "Unknown call-spec" call-spec))) 46 | 47 | (defmethod to-fn clojure.lang.PersistentArrayMap 48 | [{:keys [call]}] 49 | (to-fn call)) 50 | 51 | (defmethod to-fn clojure.lang.Symbol 52 | [sym] 53 | (binding [*ns* (create-ns 'user)] 54 | (if (qualified-symbol? sym) 55 | (requiring-resolve sym) 56 | (resolve sym)))) 57 | 58 | (defmethod to-fn clojure.lang.Var 59 | [var] 60 | (var-get var)) 61 | 62 | (defmethod to-fn clojure.lang.PersistentList 63 | [sexp] 64 | (binding [*ns* (create-ns 'user)] 65 | (eval sexp))) 66 | 67 | (defn eval-under-call-arg 68 | "Evaluates form under common call-arg bindings, i.e. 69 | (fn [{:keys [config-path target-path self-path] :as call-arg}] 70 |
) 71 | Useful to transform call-args from CLI." 72 | [call-arg form] 73 | (let [bind-args-fn (eval (list 'fn '[{:keys [config-path target-path self-path] :as call-arg}] form))] 74 | (bind-args-fn call-arg))) 75 | 76 | (defmethod to-fn :sh 77 | [[_ & shell-args]] 78 | (fn [call-arg] 79 | (let [[opts & rest :as all] (eval-under-call-arg call-arg (vec shell-args)) 80 | shell-opts {:dir (-> call-arg :self-path fs/parent str) 81 | :out *out* 82 | :err *err*} 83 | shell-args' (if (map? opts) 84 | (into [(merge shell-opts opts)] rest) 85 | (into [shell-opts] all))] 86 | (clojure.core/apply shell shell-args')))) 87 | 88 | (defmethod to-fn clojure.lang.Fn 89 | [f] 90 | f) 91 | 92 | ;; Set by fdb during triggered calls. Nil during repl sessions, but that's what (arg) below is for. 93 | (def ^:dynamic *arg* nil) 94 | ;; Set by watch so repl sessions can also get a call-arg, and for restarts after code reload. 95 | (defonce *arg (atom nil)) 96 | 97 | (defmacro with-arg 98 | "Run body with m merged into current *arg*." 99 | [m & body] 100 | `(binding [*arg* (merge *arg* ~m)] 101 | ~@body)) 102 | 103 | (defmacro arg 104 | "The argument sent into trigger and reader calls. 105 | Also available as a dynamic binding for triggers, readers, repl files, and 106 | files loaded in fdbconfig.edn." 107 | [] 108 | `(or 109 | ;; Calls from triggers, readers, and repl files should have this set. 110 | *arg* 111 | ;; If it's not set, it must be a nrepl session. 112 | ;; Watch should be running so we can get state from there. 113 | ;; *file* should work from a repl session when it evals a file. 114 | (merge {:self-path *file*} @*arg))) 115 | 116 | (defn apply 117 | "Applies call-spec fn to call-arg, defaulting to current *call*. " 118 | ([call-spec] 119 | (apply call-spec *arg*)) 120 | ([call-spec call-arg] 121 | ((to-fn call-spec) call-arg))) 122 | 123 | ;; TODO: 124 | ;; - maybe get rid of eval-under-call-args and just replace bindings with kws 125 | ;; - ref call-spec 126 | ;; - can ref nested keys 127 | ;; - good for shared stuff, say the handler for this trigger is that other one 128 | ;; - would load it from db 129 | ;; - usable for other things too, db would have a resolver for it 130 | ;; - maybe easier is [:ref "user/folder/whatever" :key :another-key] 131 | ;; - no ambiguity 132 | ;; - should arg just be state? 133 | ;; - do it backwards compat if you wanna change it 134 | -------------------------------------------------------------------------------- /src/fdb/config.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.config 2 | (:require 3 | [babashka.fs :as fs] 4 | [fdb.utils :as u])) 5 | 6 | (def filename "fdbconfig.edn") 7 | 8 | (defn new-path 9 | "Returns file path for a new config file." 10 | [path] 11 | (-> (if (u/catch-nil (fs/directory? path)) 12 | path 13 | (fs/path (fs/home) "fdb")) 14 | (fs/path filename) 15 | fs/absolutize 16 | fs/normalize 17 | str)) 18 | 19 | (defn path 20 | "Returns path if it's a file, otherwise looks for the config file in 21 | path, current dir, and ~/fdb/. Returns nil if none was found" 22 | [path] 23 | (some->> [path 24 | (str (fs/file path filename)) 25 | (str (fs/file (fs/cwd) filename)) 26 | (str (fs/file (fs/home) "fdb" filename))] 27 | (filter #(u/catch-nil (fs/regular-file? %))) 28 | first 29 | fs/absolutize 30 | str)) 31 | 32 | ;; TODO: 33 | ;; - move all the config fns scattered around here instead 34 | -------------------------------------------------------------------------------- /src/fdb/db.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.db 2 | (:refer-clojure :exclude [get sync]) 3 | (:require 4 | [babashka.fs :as fs] 5 | [fdb.call :as call] 6 | [xtdb.api :as xt])) 7 | 8 | (defn start-node 9 | [db-path] 10 | (let [cfg {:xtdb/index-store {:kv-store {:xtdb/module 'xtdb.rocksdb/->kv-store 11 | :db-dir (fs/file db-path "rocksdb/index")}} 12 | :xtdb/document-store {:kv-store {:xtdb/module 'xtdb.rocksdb/->kv-store 13 | :db-dir (fs/file db-path "rocksdb/document")}} 14 | :xtdb/tx-log {:kv-store {:xtdb/module 'xtdb.rocksdb/->kv-store 15 | :db-dir (fs/file db-path "rocksdb/tx-log")}} 16 | :xtdb.lucene/lucene-store {:db-dir (fs/file db-path "lucene")}}] 17 | (xt/start-node cfg))) 18 | 19 | (defn put 20 | [node id data] 21 | (xt/submit-tx node [[::xt/put (merge {:xt/id id} data)]])) 22 | 23 | (defn all 24 | [node] 25 | (->> (xt/q (xt/db node) 26 | '{:find [(pull e [*])] 27 | :where [[e :xt/id]]}) 28 | (map first) 29 | set)) 30 | 31 | (defn xtdb-id->xt-id 32 | "Get :xt/id for the xtdb-id provided for :xtdb.api/delete operations in a tx. 33 | From https://github.com/xtdb/xtdb/issues/1769." 34 | [node eid] 35 | (with-open [i (xt/open-entity-history (xt/db node) 36 | eid 37 | :asc 38 | {:with-docs? true 39 | :with-corrections? true})] 40 | (:xt/id (some :xtdb.api/doc (iterator-seq i))))) 41 | 42 | (defn tx-with-ops 43 | "Returns the tx with ops for the given tx without ops." 44 | [node {::xt/keys [tx-id] :as _tx-without-ops}] 45 | (first (iterator-seq (xt/open-tx-log node (dec tx-id) true)))) 46 | 47 | ;; Convenience fns 48 | 49 | (defn node 50 | "Returns current node." 51 | [] 52 | (:node (call/arg))) 53 | 54 | (defn db 55 | "Returns current database." 56 | [] 57 | (xt/db (node))) 58 | 59 | (defn q 60 | "Same as xtdb.api/q, but uses the current database." 61 | [q & args] 62 | (apply xt/q (db) q args)) 63 | 64 | (defn pull 65 | "Same as xtdb.api/pull, but uses the current database." 66 | [q eid] 67 | (xt/pull (db) q eid)) 68 | 69 | (defn pull-many 70 | "Same as xtdb.api/pull-many, but uses the current database." 71 | [q eids] 72 | (xt/pull-many (db) q eids)) 73 | 74 | (defn entity 75 | "Same as xtdb.api/entity, but uses the current database." 76 | [eid] 77 | (xt/entity (db) eid)) 78 | 79 | (defn entity-history 80 | "Same as xtdb.api/entity-history, but uses the current database." 81 | [eid sort-order & {:as opts}] 82 | (xt/entity-history (db) eid sort-order opts)) 83 | -------------------------------------------------------------------------------- /src/fdb/email.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.email 2 | (:refer-clojure :exclude [send sync]) 3 | (:require 4 | [babashka.fs :as fs] 5 | [clojure-mail.core :as mail] 6 | [clojure-mail.folder :as folder] 7 | [clojure-mail.gmail :as gmail] 8 | [clojure-mail.message :as message] 9 | [clojure.java.io :as io] 10 | [clojure.string :as str] 11 | [fdb.db :as db] 12 | [fdb.metadata :as metadata] 13 | [fdb.readers.eml :as eml] 14 | [fdb.utils :as u] 15 | [taoensso.timbre :as log] 16 | [tick.core :as t]) 17 | (:import 18 | [com.sun.mail.gimap GmailMessage] 19 | [java.util Properties] 20 | [javax.mail Message$RecipientType Session Transport] 21 | [javax.mail.internet InternetAddress MimeMessage])) 22 | 23 | (defn str->message 24 | "Like clojure-mail.core/file->message, but doesn't read the file from disk." 25 | [str] 26 | (let [props (Session/getDefaultInstance (Properties.))] 27 | (MimeMessage. props (io/input-stream (.getBytes str))))) 28 | 29 | (defn filename 30 | "Returns file name for a eml message in the following format: 31 | timestamp-or-epoch message-id-or-random-uuid-8-char-hex-hash subject-up-to-80-chars.eml 32 | The message-id hash is there to avoid overwriting emails with same timestamp and subject." 33 | ([str-or-msg] 34 | (let [msg (cond-> str-or-msg 35 | (string? str-or-msg) str->message)] 36 | (filename (or (message/date-sent msg) 37 | (message/date-received msg)) 38 | (message/subject msg) 39 | (message/id msg)))) 40 | ([date subject message-id] 41 | (str 42 | (u/filename-inst (or date (t/epoch))) 43 | " " 44 | (->> (or message-id (str "")) 45 | hash 46 | (format "%08x")) 47 | " " 48 | (u/filename-str (u/ellipsis (or subject "") 80)) 49 | ".eml"))) 50 | 51 | (defn email-config 52 | "Returns email config from call-arg." 53 | [call-arg] 54 | (let [{:keys [email email-env password password-env imap smtp gmail] :as cfg} (-> call-arg :config :email) 55 | cfg' (cond-> cfg 56 | (and email-env (not email)) (assoc :email (System/getenv email-env)) 57 | (and password-env (not password)) (assoc :password (System/getenv password-env)) 58 | (nil? imap) (assoc :imap {:host "imap.gmail.com"}) 59 | (nil? smtp) (assoc :smtp {:host "smtp.gmail.com" :port 587}))] 60 | (when (or (nil? (:password cfg')) 61 | (nil? (:email cfg'))) 62 | (throw (ex-info "Couldn't get email and password from config or env" {}))) 63 | (assoc cfg' :gmail (or gmail (str/ends-with? (:email cfg') "@gmail.com"))))) 64 | 65 | ;; smtp 66 | 67 | (defn smtp-session 68 | [{:keys [host port]}] 69 | (Session/getInstance (mail/as-properties 70 | {"mail.smtp.host" host 71 | "mail.smtp.port" port 72 | "mail.smtp.auth" true 73 | "mail.smtp.starttls.enable" true}))) 74 | 75 | (defn send 76 | "Send mail." 77 | [call-arg {:keys [to subject text]}] 78 | (let [{:keys [password smtp] from-email :email} (email-config call-arg)] 79 | (Transport/send (doto (MimeMessage. (smtp-session smtp)) 80 | (.setFrom (InternetAddress. from-email)) 81 | (.setRecipient Message$RecipientType/TO 82 | (InternetAddress. (if (= to :self) from-email to))) 83 | (.setSubject subject) 84 | (.setText text)) 85 | from-email password))) 86 | 87 | ;; imap 88 | 89 | (defn next-uid 90 | "Returns next uid for `since`, which can be a uid or a inst." 91 | [store folder since] 92 | (inc (if (int? since) 93 | since ;; already looks like uid 94 | (let [msg (-> (mail/open-folder store folder :readonly) 95 | (folder/search :received-after since) 96 | first)] 97 | (if msg 98 | (message/uid msg) 99 | 0))))) 100 | 101 | (defn store 102 | [{:keys [email password imap gmail]}] 103 | ;; Using gimaps to get access to gmail labels and thread-id. 104 | ;; https://github.com/javaee/javamail/blob/master/gimap/src/main/java/com/sun/mail/gimap/package.html 105 | (mail/store (if gmail "gimaps" "imaps") (:host imap) email password)) 106 | 107 | (defn add-gmail-headers 108 | "Return new MimeMessage thread-id and labels from GMail APIs. 109 | You won't be able to call message/uid on this message anymore since that needs folder information." 110 | [^GmailMessage message] 111 | (let [labels (->> message 112 | .getLabels 113 | (map str) 114 | (map #(cond-> % 115 | (str/starts-with? % "\\") (subs 1)))) 116 | seen? (-> message 117 | message/flags 118 | str 119 | ;; Bit hacky, but couldn't find another way 120 | (str/includes? "\\Seen")) 121 | archived? (not (contains? (set labels) "Inbox")) 122 | ;; Got these rules from adding a label to a few messages and exporting 123 | ;; them via google takeout, then seeing what labels they had. 124 | ;; Not complete, there seems to be some "Category ..." labels that's I 125 | ;; don't know how to get, and also once I saw a mail with both "Unread" 126 | ;; "Opened" at the same time, which is weird, and only happens on Takeout. 127 | labels' (cond-> labels 128 | seen? (conj "Opened") 129 | (not seen?) (conj "Unread") 130 | archived? (conj "Archived"))] 131 | (doto (MimeMessage. message) 132 | (.setHeader "X-GM-THRID" (str (.getThrId message))) 133 | (.setHeader "X-Gmail-Labels" (str/join "," labels'))))) 134 | 135 | (defn fetch 136 | [{:keys [gmail] :as cfg} {:keys [folder since order] :or {order :asc}}] 137 | (let [store (store cfg) 138 | folder (if gmail (gmail/folder->folder-name folder) folder) 139 | uid (next-uid store folder since)] 140 | (cond->> (mail/all-messages store folder {:since-uid uid}) 141 | (= order :asc) reverse))) 142 | 143 | (defn sync 144 | "Sync :folder in trigger to :self-path since :since. 145 | Self-updates trigger to last :since written." 146 | [{:keys [self self-path on on-path] :as call-arg}] 147 | (if (fs/exists? self-path) 148 | (assert (fs/directory? self-path) "Sync for email path must be a folder if it exists.") 149 | (fs/create-dirs self-path)) 150 | (let [{:keys [take-n self-update folder] :or {take-n 50 self-update true}} on 151 | {:keys [gmail] :as cfg} (email-config call-arg) 152 | on' (cond-> on 153 | self-update (assoc :since (-> (db/entity (:xt/id self)) 154 | (get-in on-path) 155 | :since)))] 156 | (run! 157 | (fn [^MimeMessage msg] 158 | (let [path (str (fs/path self-path (filename msg))) 159 | uid (message/uid msg) 160 | msg' (cond-> msg 161 | gmail add-gmail-headers)] 162 | (log/debug "syncing" folder uid "to" path) 163 | (.writeTo msg' (io/output-stream path)) 164 | (when self-update 165 | (metadata/swap! self-path 166 | #(-> % 167 | (assoc :fdb.on/ignore true) 168 | (update-in on-path assoc :since uid)))))) 169 | (take take-n (fetch cfg on'))))) 170 | 171 | 172 | ;; mbox 173 | 174 | ;; Permissive version of mime4j mbox regex 175 | ;; org.apache.james.mime4j.mboxiterator.FromLinePatterns/DEFAULT2 176 | @(def line-re #"^From \S+.*\d{4}$") 177 | 178 | (defn from-line? 179 | [line] 180 | (->> line (re-matches line-re) boolean)) 181 | 182 | (defn write-message 183 | [path message message-number] 184 | (let [filename (->> message filename (fs/file path) str)] 185 | (log/info "writing" (str "#" message-number) filename) 186 | (spit filename message))) 187 | 188 | ;; MboxIterator from mime4j blows up on big mboxes, rolling my own 189 | (defn split-mbox 190 | [mbox & {:keys [to drop-n] 191 | :or {to (u/sibling-path mbox (u/filename-without-extension mbox "mbox")) 192 | drop-n 0}}] 193 | (when-not (fs/exists? to) 194 | (fs/create-dir to)) 195 | (with-open [rdr (io/reader mbox)] 196 | (let [counter (volatile! drop-n)] 197 | (doseq [message (sequence 198 | ;; potentially big file, worth it to use transducer 199 | (comp 200 | (partition-by from-line?) 201 | (partition-all 2) 202 | (drop drop-n) 203 | (map second) 204 | (map (partial str/join "\n"))) 205 | (line-seq rdr))] 206 | (write-message to message (vswap! counter inc)))))) 207 | 208 | ;; TODO: 209 | ;; - support multiple mail accounts, make mail config one-or-map 210 | ;; - support muliple to, cc, and bcc in send 211 | ;; - support headers in send 212 | ;; - default to INBOX sync? 213 | 214 | (comment 215 | (def call-arg {:config {:email {:email-env "FDB_GMAIL_DEMO_PASSWORD" 216 | :password-env "FDB_GMAIL_DEMO_PASSWORD" 217 | ;; :email-env "FDB_GMAIL_EMAIL" 218 | ;; :password-env "FDB_GMAIL_PASSWORD" 219 | }} 220 | :on {:folder :all 221 | :since #inst "2024-05-15" #_ 1058154 222 | :take-n 5 223 | :self-update false} 224 | :self-path "tmp/email-sync"}) 225 | @(def cfg (email-config call-arg)) 226 | 227 | (send call-arg {:to :self :subject "test subject" :text "test body"}) 228 | 229 | (def store' (store cfg)) 230 | (def folder (gmail/folder->folder-name :all)) 231 | (next-uid store' folder 0) 232 | (next-uid store' folder #inst "2024-05-15") 233 | (->> (fetch cfg {:folder :all 234 | :since #inst "2024-05-15"}) 235 | (take 2) 236 | (map eml/read-message)) 237 | (sync call-arg) 238 | 239 | @(def mbox (fs/file (io/resource "email/sample-crlf.mbox"))) 240 | @(def to (fs/file (fs/home) "fdb/user/email")) 241 | (fs/delete-tree to) 242 | (split-mbox mbox :to to) 243 | 244 | ) 245 | -------------------------------------------------------------------------------- /src/fdb/http.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.http 2 | (:require 3 | [net.cgrand.enlive-html :as enlive-html] 4 | [clojure.data.json :as json] 5 | [clojure.string :as str] 6 | [huff2.core :as h]) 7 | (:import 8 | (java.net URL URLDecoder URLEncoder))) 9 | 10 | (defn encode-url [s] 11 | (URLEncoder/encode s)) 12 | 13 | (defn decode-url [s] 14 | (URLDecoder/decode s)) 15 | 16 | (defn encode-uri 17 | "Like JS encodeURI https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURI" 18 | [s] 19 | (-> s 20 | str 21 | (URLEncoder/encode) 22 | (.replace "+" "%20") 23 | (.replace "%2F" "/"))) 24 | 25 | (defn decode-uri 26 | "Like JS decodeURI https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURI" 27 | [s] 28 | (decode-url s)) 29 | 30 | (defn add-params 31 | [url param-map] 32 | (->> param-map 33 | (mapcat (fn [[k v]] 34 | (if (sequential? v) 35 | (map vector (repeat k) v) 36 | [[k v]]))) 37 | (map (fn [[k v]] 38 | (str (encode-uri (name k)) "=" (encode-uri v)))) 39 | (str/join "&") 40 | (str url "?"))) 41 | 42 | (defn json 43 | "Get url and parse it as json." 44 | [url] 45 | (some-> url slurp (json/read-str :key-fn keyword))) 46 | 47 | (defn scrape 48 | "Get url and scrap it to edn, using selector if any. 49 | Uses Enlive selectors https://github.com/cgrand/enlive?tab=readme-ov-file#selectors" 50 | ([url] 51 | (-> url URL. enlive-html/html-resource)) 52 | ([url selector] 53 | (-> url scrape (enlive-html/select selector)))) 54 | 55 | (defn render 56 | "Same as huff2.core/html followed by str." 57 | [x] 58 | (str (h/html x))) 59 | 60 | ;; TODO: 61 | ;; - could sync a page to client when it changes, via SSE 62 | ;; - client can ask for static page, or self-updating page 63 | ;; - you change the page on disk, new version is pushed to client 64 | ;; - cool for sync in general, maybe even reified sessions 65 | ;; - where does that headers-in-the-metadata idea fit in now? 66 | ;; - I guess it's just a namespaced k, for headers 67 | ;; - have to make it easier for third party libs to add routes 68 | ;; - blog is a cool example of a subset of your file library you might want to expose online 69 | ;; - Is rss just a folder file listing ordered by date and content, and content negotiated to be rss format? 70 | ;; - The folder could be real, or be full of symlinks, or even be a synthetic listing from the db. 71 | ;; - can I make some auto-PWA thing work? 72 | ;; - that'd make these real personal apps 73 | ;; - need to be able to serve an existing ref 74 | ;; - e.g. serve trigger query result 75 | ;; - data needed: fn to render it, ref id, watch semantics 76 | ;; - watch semantics needs a model of server syncing a component first 77 | ;; - watch both the ref id, and the fn, or the file the fn is in 78 | ;; - probably always watch really... turning it off is the special case 79 | ;; - reload on handler change 80 | ;; - maybe only relevant for the htmx live env case... 81 | ;; - normally you don't want to force-reload endpoint results when editing 82 | ;; - A bit of disconnect between the notion of "server handler is fn" and "live edit the file for a server fn". 83 | ;; - The link between the two is not obvious. 84 | ;; - But handler is deff a fn, not a file. 85 | ;; - Does the eval for the fn preserve the file? 86 | ;; - first class htmx components 87 | ;; - demo should be like http://reagent-project.github.io demos 88 | ;; - turn the htmx problem into a content negotiation problem 89 | ;; - always return the plain data from endpoints, then render it 90 | ;; - edn, json, http for htmx, partial http for htmx partials 91 | ;; - maybe full html can be nested routes? 92 | ;; - /api/email/1 is just the email partial 93 | ;; - /inbox/email/1 is the inbox with the email partial in 94 | ;; 95 | -------------------------------------------------------------------------------- /src/fdb/mac.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.mac 2 | (:require [babashka.process :as process])) 3 | 4 | (defn notification 5 | [title message] 6 | (process/shell (format "osascript -e 'display notification \"%s\" with title \"%s\"'" message title))) 7 | 8 | 9 | (comment 10 | (notification "you got" "mail") 11 | 12 | ) 13 | -------------------------------------------------------------------------------- /src/fdb/metadata.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.metadata 2 | (:refer-clojure :exclude [read swap!]) 3 | (:require 4 | [babashka.fs :as fs] 5 | [clojure.string :as str] 6 | [fdb.call :as call] 7 | [fdb.utils :as u] 8 | [tick.core :as t])) 9 | 10 | (def default-metadata-ext "meta.edn") 11 | 12 | (defn metadata-path? [f] 13 | (-> f (fs/split-ext {:ext default-metadata-ext}) second)) 14 | 15 | (defn content-path->metadata-path 16 | [path] 17 | {:pre [(not (metadata-path? path))]} 18 | (str path "." default-metadata-ext)) 19 | 20 | (defn metadata-path 21 | [id] 22 | (content-path->metadata-path id)) 23 | 24 | (defn metadata-path->content-path 25 | [path] 26 | {:pre [(metadata-path? path)]} 27 | (first (fs/split-ext path {:ext default-metadata-ext}))) 28 | 29 | (defn content-and-metadata-paths 30 | [& paths] 31 | (let [path (str (apply fs/path paths))] 32 | (if (metadata-path? path) 33 | [(metadata-path->content-path path) path] 34 | [path (content-path->metadata-path path)]))) 35 | 36 | (defn in-mount? 37 | [id mount-id] 38 | (str/starts-with? id (str "/" (name mount-id) "/"))) 39 | 40 | (defn id->mount 41 | [{:keys [mounts]} id] 42 | (some (fn [[mount-id mount-spec]] 43 | (when (in-mount? id mount-id) 44 | [mount-id mount-spec])) 45 | mounts)) 46 | 47 | (defn id->mount-spec 48 | [config id] 49 | (second (id->mount config id))) 50 | 51 | (defn mount-path 52 | [config-path mount-spec] 53 | (str (u/sibling-path config-path (cond-> mount-spec 54 | (map? mount-spec) :path)))) 55 | 56 | (defn id 57 | [mount-id path] 58 | (str "/" 59 | (name mount-id) 60 | (when-not (str/starts-with? path "/") "/") 61 | (if (metadata-path? path) 62 | (metadata-path->content-path path) 63 | path))) 64 | 65 | (defn id->path 66 | ([id] 67 | (id->path (:config-path call/*arg*) (:config call/*arg*) id)) 68 | ([config-path {:keys [mounts]} id] 69 | (when-some [[_ mount-id path] (re-find #"^/([^/]+)/(.*)$" id)] 70 | (when-some [mount-spec (or (get mounts mount-id) 71 | (get mounts (keyword mount-id)))] 72 | (str (fs/path (mount-path config-path mount-spec) path)))))) 73 | 74 | (defn path->id 75 | [config-path {:keys [mounts]} path] 76 | (some (fn [[mount-id mount-spec]] 77 | (let [mount-path' (mount-path config-path mount-spec)] 78 | (cond 79 | ;; Path looks like it's already an id. 80 | (str/starts-with? path (str "/" (name mount-id) "/")) 81 | path 82 | ;; Path is an absolute path under a mount. 83 | (str/starts-with? path mount-path') 84 | (id mount-id (fs/relativize mount-path' path))))) 85 | mounts)) 86 | 87 | (defn modified [& paths] 88 | (try 89 | (-> (apply fs/path paths) fs/last-modified-time fs/file-time->instant) 90 | (catch java.nio.file.NoSuchFileException _ nil))) 91 | 92 | (defn read 93 | [& paths] 94 | (let [[content-path metadata-path] (apply content-and-metadata-paths paths) 95 | modifieds (remove nil? [(modified content-path) 96 | (modified metadata-path)])] 97 | (when (seq modifieds) 98 | (merge (u/slurp-edn metadata-path) 99 | {:fdb/modified (apply t/max modifieds)})))) 100 | 101 | (defn swap! 102 | "Like clojure.core/swap! but over metadata file for path. 103 | See fdb.utils/swap-edn-file! docstring for more." 104 | [path f & args] 105 | (let [[_ metadata-path] (content-and-metadata-paths path)] 106 | (apply u/swap-edn-file! metadata-path f args))) 107 | 108 | ;; TODO: 109 | ;; - lots of stuff here, mount stuff, should be in a config ns 110 | ;; - cleaned up too, mount vs mount-spec 111 | ;; - could probably do silent-swap with a tx-id on a :ignore-next k 112 | ;; - next time doc is tx'd, ignore and cache that you shouldn't ignore again 113 | ;; - weird for users to have that key tho 114 | ;; - could set on db directly 115 | -------------------------------------------------------------------------------- /src/fdb/readers.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.readers 2 | (:refer-clojure :exclude [read]) 3 | (:require 4 | [babashka.fs :as fs] 5 | [fdb.call :as call] 6 | [fdb.metadata :as metadata] 7 | [fdb.readers.edn :as readers-edn] 8 | [fdb.readers.eml :as readers-eml] 9 | [fdb.readers.json :as readers-json] 10 | [fdb.readers.md :as readers-md] 11 | [fdb.utils :as u])) 12 | 13 | (def default-readers 14 | {:edn #'readers-edn/read 15 | :eml #'readers-eml/read 16 | :json #'readers-json/read 17 | :md #'readers-md/read}) 18 | 19 | (defn id->readers 20 | [config id] 21 | (let [ext-k (-> id fs/split-ext second keyword) 22 | mount-spec (metadata/id->mount-spec config id) 23 | readers (->> [(or (:readers mount-spec) 24 | (:readers config) 25 | default-readers) 26 | (:extra-readers config) 27 | (:extra-readers mount-spec)] 28 | (map #(update-vals % call/specs)) 29 | (apply merge-with into))] 30 | (get readers ext-k))) 31 | 32 | (defn read 33 | [config id] 34 | (->> (id->readers config id) 35 | (map (fn [call-spec] 36 | (call/with-arg {:on [:fdb.on/read call-spec]} 37 | (u/catch-log 38 | (call/apply call-spec))))) 39 | (remove (comp not map?)) 40 | (reduce merge {}))) 41 | 42 | ;; TODO: 43 | ;; - glob reader ks 44 | ;; - when id matches ks+globs, what happens? 45 | ;; - would be easier if readers were vectors of [k-or-glob f-or-fns], order matters then 46 | ;; - even nicer: support maps and vecs, use vecs when order matters only 47 | ;; - content reader 48 | ;; - puts file content on :content k 49 | ;; - metadata could be a reader too... but that's going a bit meta atm 50 | ;; - would make it easy to have json and other formats tho 51 | ;; - would have to distinguish between edn reader and our edn built-in reader 52 | ;; - should the shell to-fn work as a reader? 53 | ;; - a bit up in the air how the output would be processed... parse edn I guess 54 | ;; - support ext like .foo.bar, fs/split-ext doesn't work for that 55 | ;; - fdb.on/read on metadata can add more readers, used for testing 56 | ;; - default collection readers too? csv, json array, mbox 57 | -------------------------------------------------------------------------------- /src/fdb/readers/edn.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.readers.edn 2 | (:refer-clojure :exclude [read]) 3 | (:require 4 | [fdb.utils :as u])) 5 | 6 | (defn read 7 | [{:keys [self-path]}] 8 | (u/slurp-edn self-path)) 9 | -------------------------------------------------------------------------------- /src/fdb/readers/eml.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.readers.eml 2 | "Read .eml MIME message files, see https://en.wikipedia.org/wiki/MIME. 3 | Also splits .mbox files into .eml, see https://en.wikipedia.org/wiki/Mbox." 4 | (:refer-clojure :exclude [read]) 5 | (:require 6 | [clojure-mail.core :as mail] 7 | [clojure-mail.message :as message] 8 | [clojure-mail.parser :as parser] 9 | [clojure.java.io :as io] 10 | [clojure.set :as set] 11 | [clojure.string :as str] 12 | [fdb.utils :as u] 13 | [fdb.readers.eml :as eml] 14 | [babashka.fs :as fs])) 15 | 16 | (defn- try-msg->map 17 | [msg] 18 | {:content-type (.getContentType msg) 19 | :body (try (.getContent msg) 20 | (catch java.io.UnsupportedEncodingException _ 21 | "") 22 | (catch Throwable _ 23 | ""))}) 24 | 25 | (defn read-message 26 | "Like clojure-mail.message/read-message, but catches unsupported/unknown 27 | encoding exceptions while reading content." 28 | [message] 29 | (with-redefs [message/msg->map try-msg->map] 30 | (let [ret (message/read-message message)] 31 | (doall (:body ret)) 32 | ret))) 33 | 34 | (defn match-type 35 | ([part type] 36 | (some-> part :content-type str/lower-case (str/starts-with? type))) 37 | ([part type ignore] 38 | (when-some [content-type (some-> part :content-type str/lower-case)] 39 | (and (str/starts-with? content-type type) 40 | (not (str/includes? content-type ignore)))))) 41 | 42 | (defn prefer-text 43 | [parts] 44 | (if (and (seq? parts) 45 | (>= (count parts) 2)) 46 | (let [maybe-plain (some #(when (match-type % "text/plain" "; name=") %) parts) 47 | maybe-html (some #(when (match-type % "text/html") %) parts)] 48 | (if (and maybe-plain maybe-html) 49 | ;; looks like an alternative between plain and html, prefer plain 50 | (list maybe-plain) 51 | parts)) 52 | parts)) 53 | 54 | (defn part->text 55 | [{:keys [content-type body] :as part}] 56 | (cond 57 | (nil? part) "" 58 | (str/includes? content-type "; name=") (str "Attachment: " content-type "\n") 59 | (match-type part "text/plain") body 60 | ;; With a nice html parser we could make it the preferred option. 61 | (match-type part "text/html") (parser/html->text body) 62 | :else (str "Attachment: " content-type "\n"))) 63 | 64 | ;; TODO: use multipart/alternative in content-type to figure out which to take 65 | (defn message-content 66 | [message-edn] 67 | (try (->> (:body message-edn) 68 | ;; for each list, prefer text if it looks like an alternative between plain and html 69 | (tree-seq seq? prefer-text) 70 | ;; lists are branches, we only care about the part leaves 71 | (remove seq?) 72 | (map part->text) 73 | (str/join "\n") 74 | u/unix-line-separators) 75 | (catch Throwable _ 76 | ""))) 77 | 78 | (defn to-references-vector 79 | [references-str] 80 | (some->> references-str 81 | u/unix-line-separators 82 | str/trim 83 | str/split-lines 84 | (mapv str/trim))) 85 | 86 | (defn read 87 | [{:keys [self-path]}] 88 | (let [message-edn (-> self-path mail/file->message read-message) 89 | from-message (-> message-edn 90 | (update :from #(mapv :address %)) 91 | (update :to #(mapv :address %)) 92 | (update :cc #(mapv :address %)) 93 | (update :bcc #(mapv :address %)) 94 | (update :sender :address) 95 | (dissoc :multipart? :content-type :headers 96 | :body ;; writing it to content separately 97 | :id ;; getting message-id from headers instead 98 | ) 99 | (set/rename-keys {:date-sent :date})) ;; match headers better 100 | headers (->> message-edn 101 | :headers 102 | (apply merge) 103 | (map (fn [[k v]] 104 | [(-> k str/lower-case keyword) v])) 105 | (into {})) 106 | from-headers (-> headers 107 | (select-keys [:message-id :references :in-reply-to :reply-to]) 108 | (update :references to-references-vector)) 109 | from-body {:text (message-content message-edn)} 110 | from-gmail (-> headers 111 | (select-keys [:x-gm-thrid :x-gmail-labels]) 112 | (set/rename-keys {:x-gm-thrid :thread-id 113 | :x-gmail-labels :labels}) 114 | (update :labels #(when % (str/split % #","))))] 115 | (u/strip-nil-empty 116 | (merge from-message from-headers from-body from-gmail)))) 117 | 118 | (comment 119 | (read {:self-path "./resources/eml/sample.eml"}) 120 | ;; 121 | ) 122 | -------------------------------------------------------------------------------- /src/fdb/readers/json.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.readers.json 2 | (:refer-clojure :exclude [read]) 3 | (:require 4 | [clojure.data.json :as json] 5 | [fdb.utils :as u])) 6 | 7 | (defn read 8 | [{:keys [self-path]}] 9 | (u/catch-nil 10 | (-> self-path 11 | slurp 12 | (json/read-str :key-fn keyword)))) 13 | -------------------------------------------------------------------------------- /src/fdb/readers/md.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.readers.md 2 | (:refer-clojure :exclude [read]) 3 | (:require 4 | [babashka.fs :as fs] 5 | [clj-yaml.core :as yaml] 6 | [clojure.java.io :as io] 7 | [clojure.string :as str] 8 | [fdb.http :as http] 9 | [fdb.metadata :as metadata] 10 | [fdb.utils :as u] 11 | [fdb.watcher :as watcher])) 12 | 13 | ;; https://help.obsidian.md/Editing+and+formatting/Tags#Tag+format 14 | (defn tags [md] 15 | (->> md 16 | (re-seq #"(?<=#)[\p{IsAlphabetic}\d-_/]+") 17 | ;; need to have at least one letter 18 | (filter (partial re-find #"[\p{IsAlphabetic}]")) 19 | (map str/lower-case))) 20 | 21 | ;; https://help.obsidian.md/Linking+notes+and+files/Internal+links#Supported+formats+for+internal+links 22 | (defn mdlinks [md self mount] 23 | (->> md 24 | ;; [] without [ inside followed by () 25 | (re-seq #"\[[^\[]+\]\(([^)]+)\)") 26 | (map second) 27 | (map http/decode-uri) 28 | (map #(->> % 29 | (fs/path (fs/parent self)) 30 | (fs/relativize mount) 31 | fs/normalize 32 | str)))) 33 | 34 | ;; https://help.obsidian.md/Linking+notes+and+files/Internal+links#Supported+formats+for+internal+links 35 | (defn wikilinks [md] 36 | ;; Obsidian doesn't support links with ^#[]| in the filename proper, if you make a new file and input 37 | ;; those characters, it will warn you. 38 | (->> md 39 | ;; not [] surrounded by [[]] via lookbehind and lookahead 40 | (re-seq #"(?<=\[\[)[^\[\]]+(?=\]\])") 41 | ;; cut off alias and anchor 42 | (map (partial re-find #"[^#|]*")))) 43 | 44 | (defn shorter? 45 | [x y] 46 | (> (count y) (count x))) 47 | 48 | ;; Obsidian wikilinks only contain full path when ambiguous, and omit .md by default 49 | (defn link->id 50 | [vault-files mount-id wikilink] 51 | (some #(when (or (str/ends-with? % wikilink) 52 | (str/ends-with? % (str wikilink ".md"))) 53 | (metadata/id mount-id %)) 54 | vault-files)) 55 | 56 | (defn refs 57 | [{:keys [config config-path self self-path]} md] 58 | (let [id (:xt/id self) 59 | [mount-id mount-spec] (metadata/id->mount config id) 60 | mount-path (metadata/mount-path config-path mount-spec) 61 | vault-files (->> mount-path 62 | (watcher/glob config) 63 | ;; ambiguous long-form paths at root don't have folder 64 | (sort shorter?))] 65 | ;; use md to get links in body and front-matter 66 | (disj (->> (mdlinks md self-path mount-path) 67 | (concat (wikilinks md)) 68 | (into #{}) 69 | (map (partial link->id vault-files mount-id)) 70 | (into #{})) 71 | nil))) 72 | 73 | ;; https://help.obsidian.md/Editing+and+formatting/Properties#Property+format 74 | (defn front-matter [yml] 75 | (some-> yml 76 | yaml/parse-string 77 | (update-vals #(if (seq? %) (vec %) %)))) 78 | 79 | (defn read-edn-for-fdb-keys 80 | [m] 81 | (let [ks (filter #(and (qualified-keyword? %) 82 | (or (= "fdb" (namespace %)) 83 | (str/starts-with? (namespace %) "fdb."))) 84 | (keys m))] 85 | (merge m 86 | (-> (select-keys m ks) 87 | (update-vals #(cond 88 | (string? %) (u/read-edn %) 89 | (vector? %) (mapv u/read-edn %))) 90 | u/strip-nil-empty)))) 91 | 92 | (defn read 93 | [{:keys [self-path] :as call-arg}] 94 | (let [md (slurp self-path) 95 | [_ yml] (re-find #"(?s)^(?:---\n(.*?)\n---\n)?" md) 96 | front-matter' (front-matter yml) 97 | refs' (refs call-arg md)] 98 | (merge (read-edn-for-fdb-keys front-matter') 99 | (when (seq refs') 100 | {:fdb/refs refs'})))) 101 | 102 | (comment 103 | @(def file (slurp (io/resource "md/file.md"))) 104 | 105 | (wikilinks file) 106 | 107 | (mdlinks file "foo/bar/file.md" "foo") 108 | ) 109 | 110 | ;; TODO: 111 | ;; - make vault path configurable within a mount, via reader call-spec k 112 | ;; - markdown with yaml is a pretty good catch-all viz format 113 | ;; - would be nice to ouptut it for stuff 114 | ;; - don't have a great way to show things aside from query results, maybe server 115 | -------------------------------------------------------------------------------- /src/fdb/triggers.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.triggers 2 | (:require 3 | [babashka.fs :as fs] 4 | [chime.core :as chime] 5 | [clojure.string :as str] 6 | [cronstar.core :as cron] 7 | [fdb.call :as call] 8 | [fdb.db :as db] 9 | [fdb.metadata :as metadata] 10 | [fdb.utils :as u] 11 | [taoensso.timbre :as log] 12 | [tick.core :as t] 13 | [xtdb.api :as xt]) 14 | (:import 15 | (java.nio.file FileSystems))) 16 | 17 | ;; Helpers 18 | 19 | ;; Whether to call triggers synchronously or asynchronously. 20 | (def ^:dynamic *sync* false) 21 | 22 | (defn call 23 | "Call trigger with call-arg. 24 | Merges call-arg with {:self self, :on-k on-k} and any extra args." 25 | [self on-k trigger trigger-idx & more] 26 | (let [log-str (str (:xt/id self) " " on-k " " (u/ellipsis (str trigger)) 27 | (if-some [target-id (-> call/*arg* :target :xt/id)] 28 | (str " over " target-id) 29 | "")) 30 | f (fn [] 31 | (call/with-arg (apply 32 | merge 33 | {:self self 34 | :self-path (metadata/id->path (:xt/id self)) 35 | :on trigger 36 | ;; trigger-idx might be 0, but on-k might not be a vec because 37 | ;; call-all-triggers and update-schedules do map-indexed over one-or-many 38 | :on-path (if (vector? (get self on-k)) 39 | [on-k trigger-idx] 40 | [on-k])} 41 | more) 42 | (u/maybe-timeout (:timeout trigger) 43 | (fn [] 44 | (u/with-time [t-ms #(log/debug "call" log-str "took" (t-ms) "ms")] 45 | (u/catch-log 46 | (call/apply trigger)))))))] 47 | 48 | (log/info "calling" log-str) 49 | (if *sync* 50 | (f) 51 | (future-call f)))) 52 | 53 | (defn call-all-triggers 54 | "Call all on-k triggers in self if should-trigger? returns truthy. 55 | Merges call-arg with {:self self, :on-k on-k :target target} and return from should-trigger?." 56 | ([target self on-k] 57 | (call-all-triggers target self on-k (constantly true))) 58 | ([target self on-k should-trigger?] 59 | (run! (fn [[trigger-idx trigger]] 60 | (when-let [maybe-map (u/catch-log (should-trigger? trigger))] 61 | (call self on-k trigger trigger-idx 62 | (when target 63 | {:target target 64 | :target-path (metadata/id->path (:xt/id target))}) 65 | (when (map? maybe-map) 66 | maybe-map)))) 67 | (->> self on-k call/specs (map-indexed vector))))) 68 | 69 | (defn docs-with-k 70 | "Get all docs with k in db. 71 | Optionally filter with more filters." 72 | [db k] 73 | ;; quote isn't in its normal place so we can use variable k 74 | (->> {:find '[(pull ?e [*])] 75 | :where [['?e k]]} 76 | (xt/q db) 77 | (map first))) 78 | 79 | (defn call-all-k 80 | "Call all existing k triggers. 81 | Mainly for :fdb.on/startup and :fdb.on/shutdown." 82 | [node k] 83 | (u/with-time [t-ms #(log/debug "call all" k "took" (t-ms) "ms")] 84 | (let [db (xt/db node)] 85 | (call/with-arg {:db db} 86 | (run! #(call-all-triggers % % k) 87 | (docs-with-k db k)))))) 88 | 89 | (defn massage-ops 90 | "Process ops to pass in to call handlers. 91 | Drop ops at a time, drop anything but puts and deletes, 92 | put :xt/id on second element for puts, and convert xtdb-id to xt-id for deletes." 93 | [node tx-ops] 94 | (->> tx-ops 95 | ;; Ignore ops with valid-time 96 | (filter #(-> % count (= 2))) 97 | ;; We only care about puts and deletes 98 | (filter #(-> % first #{::xt/put ::xt/delete})) 99 | ;; Convert xtdb-id to xt-id for deletes, and pull out id 100 | (map (fn [[op id-or-doc]] 101 | (if (= ::xt/delete op) 102 | [op (db/xtdb-id->xt-id node id-or-doc)] 103 | [op (:xt/id id-or-doc) id-or-doc]))))) 104 | 105 | 106 | ;; Schedules 107 | 108 | ;; Reset to {} during watch start. 109 | (defonce *schedules (atom {})) 110 | 111 | (defn update-schedules 112 | "Updates schedules for doc, scoped under config-path." 113 | [[op id doc]] 114 | (swap! *schedules 115 | (fn [schedules] 116 | ;; Start by stopping all schedules for this id, if any. 117 | (run! u/close (get schedules id)) 118 | (cond 119 | ;; Remove schedule if id was deleted. 120 | (and (= op ::xt/delete) 121 | (get schedules id)) 122 | (do (log/debug "removing schedules for" id) 123 | (dissoc schedules id)) 124 | 125 | ;; Add new schedules for this id if any. 126 | (:fdb.on/schedule doc) 127 | (do (log/debug "adding schedules for" id) 128 | (assoc schedules id 129 | (doall 130 | (map-indexed 131 | (fn [trigger-idx {:keys [cron every] :as trigger}] 132 | (when-some [time-seq (u/catch-log 133 | (cond 134 | cron (cron/times cron) 135 | every (chime/periodic-seq 136 | (t/now) (-> every u/duration-ms t/of-millis))))] 137 | (chime/chime-at time-seq 138 | (fn [timestamp] 139 | (call/with-arg {:timestamp (str timestamp)} 140 | (call doc :fdb.on/schedule trigger trigger-idx)) 141 | ;; Never cancel schedule from fn. 142 | true)))) 143 | (-> doc :fdb.on/schedule call/specs))))) 144 | 145 | ;; There's no schedules for this doc, nothing to do. 146 | :else 147 | schedules)))) 148 | 149 | (defn start-schedules! 150 | [node] 151 | (let [db (xt/db node)] 152 | (call/with-arg {:db db} 153 | (run! #(update-schedules [nil (:xt/id %) %]) 154 | (docs-with-k db :fdb.on/schedule))))) 155 | 156 | (defn stop-schedules! 157 | [] 158 | (swap! *schedules 159 | (fn [schedules] 160 | (run! u/close (mapcat identity (vals schedules))) 161 | {}))) 162 | 163 | 164 | ;; Triggers 165 | 166 | (defn out-file 167 | [id in ext] 168 | (when-some [[_ prefix] (re-matches 169 | (re-pattern (str ".*/([^/]*)" in "\\.fdb\\." ext "$")) 170 | id)] 171 | (str prefix in "-out" ".fdb." ext))) 172 | 173 | (defn unwrap-md-codeblock 174 | [lang s] 175 | (->> s 176 | (str/trim) 177 | (re-find (re-pattern (str "(?s)^```" lang "\\n(.*)\\n```$"))) 178 | second)) 179 | 180 | (defn wrap-md-codeblock 181 | [lang s] 182 | (str "```" lang "\n" (str/trim s) "\n```\n\n")) 183 | 184 | (def ext->codeblock-lang 185 | {"clj" "clojure" 186 | "edn" "edn"}) 187 | 188 | (defn rep-ext-or-codeblock 189 | "Read eval print helper for query and repl files." 190 | [id in ext log-f f] 191 | (let [codeblock? (str/ends-with? id ".md")] 192 | (when-some [out-file' (out-file id in (if codeblock? "md" ext))] 193 | (log-f out-file') 194 | (let [in-path (metadata/id->path id) 195 | out-path (u/sibling-path in-path out-file') 196 | content (u/slurp in-path) 197 | lang (ext->codeblock-lang ext)] 198 | (when-not (-> content str/trim empty?) 199 | (let [ret (if codeblock? 200 | (some->> content 201 | (unwrap-md-codeblock lang) 202 | f 203 | (wrap-md-codeblock lang)) 204 | (f content))] 205 | (when (and codeblock? (not ret)) 206 | (log/info "no solo" lang "codeblock found in" id)) 207 | (spit out-path (or ret "")))))))) 208 | 209 | (defn call-on-query-file 210 | "If id matches in /*query.fdb.edn, query with content and output 211 | results to sibling /*results.fdb.edn file." 212 | [[op id]] 213 | (when (= op ::xt/put) 214 | (rep-ext-or-codeblock 215 | id "query" "edn" 216 | #(log/info "querying" id "to" %) 217 | #(try (some->> % u/read-edn (xt/q (:db call/*arg*)) u/edn-str) 218 | (catch Exception e 219 | {:error (ex-message e)}))))) 220 | 221 | (defn call-on-repl-file 222 | "If id matches in /*repl.fdb.clj, call repl with content and print 223 | output to sibling /*outputs.fdb.clj file." 224 | [[op id]] 225 | (when (= op ::xt/put) 226 | (rep-ext-or-codeblock 227 | id "repl" "clj" 228 | #(log/info "sending" id "to repl, outputs in" %) 229 | #(str % "\n" 230 | (binding [*ns* (create-ns 'user)] 231 | (call/with-arg {:self-path (metadata/id->path id)} 232 | (u/eval-to-comment %))) 233 | "\n")))) 234 | 235 | (defn call-on-modify 236 | "Call all :fdb.on/modify triggers in doc." 237 | [[_op _id {:fdb.on/keys [modify] :as doc}]] 238 | (when modify 239 | (call-all-triggers doc doc :fdb.on/modify))) 240 | 241 | (defn recursive-pull-k 242 | "Recursively pull k from doc. 243 | Returns all pulled docs." 244 | [db id k] 245 | (let [pulled (xt/pull db 246 | [:xt/id 247 | ;; No result limit 248 | {(list k {:limit ##Inf}) 249 | ;; Unbounded recursion 250 | '...}] 251 | id)] 252 | (->> (dissoc pulled :xt/id) ;; don't include the root doc 253 | (tree-seq map? k) 254 | (map :xt/id) 255 | (remove nil?) 256 | (into #{}) 257 | (xt/pull-many db '[*])))) 258 | 259 | (defn call-on-refs 260 | "Call all :fdb.on/refs triggers in docs that have doc in :fdb.on/refs." 261 | [db [_op _id doc]] 262 | (->> (recursive-pull-k db (:xt/id doc) :fdb/_refs) 263 | (filter :fdb.on/refs) 264 | (run! #(call-all-triggers doc % :fdb.on/refs)))) 265 | 266 | (defn matches-glob? 267 | "Returns true if id matches glob." 268 | [id glob] 269 | (-> (FileSystems/getDefault) 270 | (.getPathMatcher (str "glob:" glob)) 271 | (.matches (fs/path id)))) 272 | 273 | (defn call-on-pattern 274 | "Call all existing :fdb.on/pattern triggers that match id." 275 | [db [_op id doc]] 276 | (run! #(call-all-triggers doc % :fdb.on/pattern 277 | (fn [trigger] 278 | (matches-glob? id (:glob trigger)))) 279 | (docs-with-k db :fdb.on/pattern))) 280 | 281 | (defn call-on-startup 282 | "Call all :fdb.on/startup triggers in doc. 283 | These run on startup but also must be ran when the doc changes." 284 | [[_op _id {:fdb.on/keys [startup] :as doc}]] 285 | (when startup 286 | (call-all-triggers doc doc :fdb.on/startup))) 287 | 288 | (defn query-results-changed? 289 | "Returns {:results ...} if query results changed compared to file at path." 290 | [db id {:keys [q path call]}] 291 | (let [target-path (metadata/id->path id) 292 | results-path (u/sibling-path target-path path)] 293 | (if (= target-path results-path) 294 | (log/warn "skipping query on" id "because path is the same as file, which would cause an infinite loop") 295 | (let [;; note: comparing these two as edn shows different dates?... 296 | new-results (u/edn-str (u/catch-log (xt/q db q))) 297 | old-results (u/catch-log (u/slurp results-path))] 298 | (when (not= new-results old-results) 299 | (spit results-path new-results) 300 | (when call 301 | ;; don't return the results if there's nothing to call 302 | {:results new-results})))))) 303 | 304 | (defn call-all-on-query 305 | "Call all existing :fdb.on/query triggers, updating their results if changed." 306 | [db] 307 | (run! #(call-all-triggers nil % :fdb.on/query 308 | (partial query-results-changed? db (:xt/id %))) 309 | (docs-with-k db :fdb.on/query))) 310 | 311 | (defn call-all-on-tx 312 | "Call all existing :fdb.on/tx triggers." 313 | [db] 314 | (run! #(call-all-triggers nil % :fdb.on/tx) 315 | (docs-with-k db :fdb.on/tx))) 316 | 317 | ;; tx listener 318 | 319 | (defn on-tx 320 | "Call all applicable triggers over tx. 321 | Triggers will be called with a map arg containing: 322 | {:config fdb config value 323 | :config-path on-disk path to config 324 | :node xtdb database node 325 | :db xtdb db value at the time of the tx 326 | :tx the tx 327 | :on the trigger being called as [fdb.on/k trigger] 328 | :on-path get-in path inside self for trigger as [fdb.on/k 1] 329 | :self the doc that has the trigger being called 330 | :self-path on-disk path for self 331 | :target the doc the trigger is being called over, if any 332 | :target-path on-disk path for doc, if any 333 | :results query results, if any 334 | :timestamp schedule timestamp, if any}" 335 | [call-arg node tx] 336 | (u/catch-log 337 | (when-not (false? (:committed? tx)) ;; can be nil for txs retried from log directly 338 | (u/with-time [time-ms] 339 | (log/debug "processing tx" (::xt/tx-id tx)) 340 | (let [db (xt/db node {::xt/tx tx}) 341 | ops (->> tx 342 | ::xt/tx-ops 343 | (massage-ops node) 344 | (remove (fn [[_ _ doc]] (:fdb.on/ignore doc))))] 345 | (call/with-arg (merge call-arg 346 | {:db db 347 | :tx tx}) 348 | 349 | ;; Update schedules 350 | (run! update-schedules ops) 351 | 352 | ;; Call triggers in order of "closeness" 353 | (run! call-on-repl-file ops) ;; content 354 | (run! call-on-query-file ops) ;; content 355 | (run! call-on-modify ops) ;; self metadata 356 | (run! (partial call-on-refs db) ops) ;; direct ref 357 | (run! (partial call-on-pattern db) ops) ;; pattern 358 | (run! call-on-startup ops) ;; application lifecycle 359 | 360 | ;; Don't need ops, just needs to be called after every tx 361 | (call-all-on-query db) 362 | (call-all-on-tx db))) 363 | (log/debug "processed tx-id" (::xt/tx-id tx) "in" (time-ms) "ms"))))) 364 | 365 | ;; TODO: 366 | ;; - make schedules play nice with sync 367 | ;; - every runs once immediately 368 | ;; - cron saves last execution and runs immediately if missed using cron/times arity 2 369 | ;; - need to make sure to wait on all listeners before exiting 370 | ;; - or don't try to wait, load all files in a big tx, then just call trigger on tx one by one 371 | ;; - this batch load mode is probably better anyway for stale check 372 | ;; - would make tests much easier 373 | ;; - fdb.on/modify receives nil for delete, or dedicated fdb.on/delete 374 | ;; - is *sync* important enough for callers that it should be part of call-arg? 375 | ;; - probably not, as they are ran async by default 376 | ;; - some way to replay the log for repl filesq 377 | ;; - would be cool to render ids in markdown for markdown query 378 | -------------------------------------------------------------------------------- /src/fdb/utils.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.utils 2 | "Grab bag of utilities." 3 | (:refer-clojure :exclude [spit slurp]) 4 | (:require 5 | [babashka.fs :as fs] 6 | [clojure.core.async :refer [timeout alts!! > 27 | "Call f with x (presumably for side effects), then return x." 28 | [f x] 29 | (f x) 30 | x) 31 | 32 | (defmacro catch-log 33 | "Wraps expr in a try/catch that logs to err any exceptions messages, without stack trace." 34 | [expr] 35 | `(try ~expr 36 | (catch Exception e# 37 | (log/error (str (str/replace-first (type e#) "class " "") ":") 38 | (or (ex-message e#) "") 39 | (or (:cause (Throwable->map e#)) "")) 40 | nil))) 41 | 42 | (defmacro catch-nil 43 | "Wraps expr in a try/catch that returns nil on err." 44 | [expr] 45 | `(try ~expr 46 | (catch Exception _#))) 47 | 48 | (defn spit 49 | "Writes content to a file and returns file path, creating parent directories if necessary. 50 | Accepts any number of path fragments followed by content value." 51 | [& args] 52 | (let [f (apply fs/file (butlast args)) 53 | content (last args)] 54 | (-> f fs/parent fs/create-dirs) 55 | (clojure.core/spit f content) 56 | (str f))) 57 | 58 | (defn edn-str 59 | [edn] 60 | (with-out-str (puget/pprint edn {:map-delimiter ""}))) 61 | 62 | (defn spit-edn 63 | "Same as spit but writes it pretty printed as edn." 64 | [& args] 65 | (apply spit (concat (butlast args) (list (edn-str (last args)))))) 66 | 67 | (defn slurp 68 | "Reads content from a file and returns it as a string. Returns nil instead of erroring out. 69 | Accepts any number of path fragments." 70 | [& paths] 71 | (catch-nil 72 | (-> (apply fs/path paths) 73 | str 74 | clojure.core/slurp))) 75 | 76 | (defn read-edn 77 | [s] 78 | ;; xt has a bunch of readers, especially around time. 79 | (catch-nil (edn/read-string {:readers *data-readers*} s))) 80 | 81 | (defn slurp-edn 82 | "Same as slurp but reads it as edn, using current *data-readers*." 83 | [& paths] 84 | (->> (apply slurp paths) 85 | read-edn)) 86 | 87 | (defn sibling-path 88 | "Returns normalized sibling-path relative to file-paths parent. 89 | If sibling-path is absolute, returns it." 90 | [file-path sibling-path] 91 | (if (fs/absolute? sibling-path) 92 | (str sibling-path) 93 | (-> file-path fs/parent (fs/file sibling-path) fs/normalize str))) 94 | 95 | (defn closeable 96 | "From https://medium.com/@maciekszajna/reloaded-workflow-out-of-the-box-be6b5f38ea98 97 | Used to manage state using with-open, for values that do not implement closeable." 98 | ([value] (closeable value identity)) 99 | ([value close] (reify 100 | clojure.lang.IDeref 101 | (deref [_] value) 102 | java.io.Closeable 103 | (close [_] (close value))))) 104 | 105 | (defn close 106 | "Close a value if it is not nil." 107 | [x] 108 | (when x 109 | (.close x))) 110 | 111 | (defn closeable-seq 112 | "Returns a closeable, which closes each value in the seq when closed." 113 | [coll] 114 | (closeable coll #(run! close %))) 115 | 116 | (defn closeable-atom 117 | "Returns a closeable, which resets an atom to `value` on open and to `back` (defaults to `to`) on close." 118 | [atom value & {:keys [back]}] 119 | (closeable (reset! atom value) (fn [_] (reset! atom (or back value))))) 120 | 121 | (defn do-eventually 122 | "Repeatedly calls f ever interval-ms until it returns a truthy value, or timeout-ms has passed. 123 | timeout-ms defaults to 1000, interval-ms defaults to 50." 124 | ([f] 125 | (do-eventually f 1000 50)) 126 | ([f timeout-ms] 127 | (do-eventually f timeout-ms 50)) 128 | ([f timeout-ms interval-ms] 129 | (let [timeout-ch (timeout timeout-ms)] 130 | (loop [] 131 | (or (f) 132 | (when-not (-> (alts!! [timeout-ch (timeout interval-ms)]) 133 | second 134 | (= timeout-ch)) 135 | (recur))))))) 136 | 137 | (defmacro eventually 138 | "Repeatedly evaluates body until it returns a truthy value, or timeout-ms has passed. 139 | See do-eventually for defaults." 140 | [& body] 141 | `(do-eventually (fn [] ~@body))) 142 | 143 | (defn- high-surrogate? [char-code] 144 | (<= 0xD800 char-code 0xDBFF)) 145 | 146 | (defn- char-code-at [^String str pos] 147 | (long ^Character (.charAt str pos))) 148 | 149 | (defn char-seq 150 | "Return a seq of the characters in a string, making sure not to split up 151 | UCS-2 (or is it UTF-16?) surrogate pairs. Because JavaScript. And Java. 152 | From https://lambdaisland.com/blog/2017-06-12-clojure-gotchas-surrogate-pairs" 153 | ([str] 154 | (char-seq str 0)) 155 | ([str offset] 156 | (loop [offset offset 157 | res []] 158 | (if (>= offset (count str)) 159 | res 160 | (let [code (char-code-at str offset) 161 | width (if (high-surrogate? code) 2 1) 162 | next-offset (+ offset width) 163 | cur-char (subs str offset next-offset)] 164 | (recur next-offset 165 | (conj res cur-char))))))) 166 | 167 | (defn ellipsis 168 | "Truncates a string to max-len (default 60), adding ellipsis if necessary." 169 | ([s] (ellipsis s 60)) 170 | ([s max-len] 171 | (if (> (count s) max-len) 172 | (str (apply str (take (- max-len 3) (char-seq s))) 173 | "...") 174 | s))) 175 | 176 | (defn sleep 177 | "Sleeps for ms milliseconds." 178 | [ms] 179 | ( 2023-11-30T14.20.23Z 218 | Parse it back to inst by replacing . with : again." 219 | [inst] 220 | (-> inst 221 | t/instant 222 | str 223 | (str/replace ":" "."))) 224 | 225 | (defn filename-str 226 | "Returns a filename friendly version of s. 227 | Banned characters from https://stackoverflow.com/a/35352640 228 | Control & unused category from https://www.regular-expressions.info/unicode.html#category" 229 | [s] 230 | (-> s 231 | (str/replace #"\p{C}" "") 232 | (str/replace #"[\\/:*?\"<>|]" " ") 233 | str/trim)) 234 | 235 | (defn unix-line-separators 236 | "Converts line separators to \n." 237 | [s] 238 | (-> s 239 | (str/replace #"\r\n" "\n") 240 | (str/replace #"\r" "\n"))) 241 | 242 | (defn duration-ms 243 | [duration] 244 | (cond 245 | (number? duration) duration 246 | (and (vector? duration) 247 | (= (count duration) 2)) (t/millis (apply t/new-duration duration)) 248 | :else nil)) 249 | 250 | (defn maybe-timeout 251 | [timeout f] 252 | (let [timeout-ms (duration-ms timeout) 253 | fut (future (f)) 254 | ret (try (if timeout-ms 255 | (deref fut timeout-ms ::timeout) 256 | (deref fut)) 257 | (catch InterruptedException _ 258 | ;; The future can still be interrupted from outside this fn. 259 | ::interrupted))] 260 | (when (and timeout-ms 261 | (#{::timeout ::interrupted} ret)) 262 | (catch-nil (future-cancel fut))) 263 | ret)) 264 | 265 | (defn filename-without-extension 266 | [path ext] 267 | (-> path 268 | fs/file-name 269 | (fs/split-ext {:ext ext}) 270 | first)) 271 | 272 | (defn strip-nil-empty 273 | "Removes nil and empty values from a map." 274 | [m] 275 | (->> m 276 | (filter (fn [[_ v]] 277 | (and (not (nil? v)) 278 | (or (not (coll? v)) 279 | (seq v))))) 280 | (into {}))) 281 | 282 | (defn fdb-root 283 | "Returns fdb root path." 284 | [] 285 | (-> (io/resource "file.txt") 286 | (fs/path "../../") 287 | fs/normalize 288 | str)) 289 | 290 | (defn as-comments 291 | "Return s as a clojure comment." 292 | [s & {:keys [prefix]}] 293 | (let [first-line (str ";; " prefix) 294 | other-lines (str ";; " (apply str (repeat (count prefix) " ")))] 295 | (str first-line (str/replace s #"\n(?!\Z)" (str "\n" other-lines))))) 296 | 297 | (defn eval-to-comment 298 | "Eval form to a comment string." 299 | [form] 300 | (try 301 | (let [*val (atom nil) 302 | out-and-err (with-out-str 303 | (binding [*err* *out*] 304 | (reset! *val (load-string form))))] 305 | (str 306 | (when-not (empty? out-and-err) (as-comments out-and-err)) 307 | (as-comments (with-out-str (pprint/pprint @*val)) :prefix "=> "))) 308 | (catch Exception e 309 | (as-comments (with-out-str (pprint/pprint e)))))) 310 | 311 | (defn do-without-random-uuid 312 | "Call f when random-uuid returning increasing ints." 313 | [f] 314 | (let [counter (atom 0)] 315 | (with-redefs [random-uuid #(swap! counter inc)] 316 | (f)))) 317 | 318 | (defmacro without-random-uuid 319 | [& body] 320 | `(do-without-random-uuid (fn [] ~@body))) 321 | 322 | (defn flatten-maps 323 | "Flatten m into {uuid m'}, where every nested map in m' has been replaced 324 | with a reference to its uuid. Root is under the \"root\" key instead of uuid. 325 | Provide :xform-uuid or :xform-ref to transform them." 326 | [m & {:keys [xform-uuid xform-ref] :or {xform-uuid identity xform-ref identity}}] 327 | (let [maps (atom [])] 328 | (walk/postwalk 329 | (fn [x] 330 | (if (map? x) 331 | (let [k (random-uuid)] 332 | (swap! maps conj [(xform-uuid k) x]) 333 | (xform-ref k)) 334 | x)) 335 | m) 336 | (-> (into {} (butlast @maps)) 337 | (assoc (xform-uuid "root") (-> @maps last second)) 338 | ))) 339 | 340 | (defn coerce-to-map 341 | [x] 342 | (cond 343 | (map? x) x 344 | (set? x) (into {} (map #(vector (-> (random-uuid) str keyword) %) x)) 345 | (sequential? x) (into {} (map-indexed #(vector (-> %1 str keyword) %2) x)) 346 | :else (throw (ex-info "Can't coerce to map" {:type (type x)})))) 347 | 348 | (defn explode-id 349 | "EXPERIMENTAL: Explode nested maps in edn into a new folder .explode/.edn. 350 | Root will be at .explode/root.edn. Folder will be deleted if it already exists. 351 | If edn is a set, it will be converted into a {uid v} map first. 352 | If edn is sequential, it will be converted into a {idx v} map first. 353 | Does nothing on non-colls or if there are no nested maps. 354 | Returns map of written files." 355 | [id path edn] 356 | (when (coll? edn) 357 | (let [explode-path (str path ".explode") 358 | flattened (flatten-maps (coerce-to-map edn) 359 | :xform-ref #(str id ".explode/" % ".edn") 360 | :xform-uuid #(str (fs/path explode-path (str % ".edn"))))] 361 | (fs/delete-tree explode-path) 362 | (when (> (count flattened) 1) 363 | (run! (fn [[p m]] (spit-edn p m)) flattened) 364 | flattened)))) 365 | 366 | ;; TODO: 367 | ;; - str-path fn 368 | ;; - the watch-config and watch-and-block loop are very similar 369 | ;; - can probably capure the "stop and wait" returns somehow 370 | ;; - maybe a closeable-go ? 371 | ;; - it's a bit more complex... it can be restarted/stopped 372 | ;; - all the string converstions for paths are starting to piss me off 373 | ;; - maybe it's just adding a u/path that returns a str 374 | ;; - puget/pprint is a bit different than pprint/pprint for large stuff 375 | ;; - e.g. printing call-arg in a repl file 376 | ;; - figure out how and if I can just use one of them 377 | ;; - use https://github.com/borkdude/rewrite-edn for swap-edn-file! 378 | -------------------------------------------------------------------------------- /src/fdb/watcher.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.watcher 2 | "File watching fns." 3 | (:require 4 | [clojure.string :as str] 5 | [babashka.fs :as fs] 6 | [nextjournal.beholder :as beholder] 7 | [taoensso.timbre :as log])) 8 | 9 | (def default-ignore-list 10 | [".DS_Store" ".git" ".gitignore" ".obsidian" ".vscode" "node_modules" "target" ".cpcache"]) 11 | 12 | (defn config->ignore-list 13 | [{:keys [ignore extra-ignore]}] 14 | (into (or ignore default-ignore-list) extra-ignore)) 15 | 16 | (def re-escape-cmap 17 | (->> "()&^%$#!?*." 18 | (map (fn [c] [c (str \\ c)])) 19 | (into {}))) 20 | 21 | (defn ignore-re 22 | [ignore-list] 23 | (let [ignore-ors (->> ignore-list 24 | (map #(str/escape % re-escape-cmap)) 25 | (str/join "|"))] 26 | (re-pattern (str "(?:^|\\/)(?:" ignore-ors ")(?:\\/|$)")))) 27 | 28 | (def memo-ignore-re (memoize ignore-re)) 29 | 30 | (defn ignore? 31 | [ignore-list path] 32 | (boolean (re-find (memo-ignore-re ignore-list) path))) 33 | 34 | (defn file-dir-relative 35 | [file-or-dir] 36 | (let [[file dir] (if (fs/directory? file-or-dir) 37 | [nil file-or-dir] 38 | [file-or-dir (fs/parent file-or-dir)]) 39 | relative #(-> dir (fs/relativize %) str)] 40 | [file dir relative])) 41 | 42 | (defn watch 43 | "Watch a file or directory for changes and call update-fn with their relative paths. 44 | File move shows up as two update calls, in no deterministic order. 45 | Returns a closeable that stops watching when closed." 46 | [config file-or-dir update-fn] 47 | (let [[_ _ relative] (file-dir-relative file-or-dir) 48 | ignore-list (config->ignore-list config)] 49 | (beholder/watch (fn [{:keys [path type]}] 50 | (let [path' (str (relative path))] 51 | (when-not (ignore? ignore-list path') 52 | (case type 53 | (:create :modify :delete) (update-fn path') 54 | :overflow (log/error "overflow" path'))))) 55 | file-or-dir))) 56 | 57 | (defn watch-many 58 | [watch-spec] 59 | (->> watch-spec (map (partial apply watch)) doall)) 60 | 61 | (defn glob 62 | "Returns all files in file-or-dir matching glob pattern." 63 | [config file-or-dir & {:keys [pattern] :or {pattern "**"}}] 64 | (when (fs/exists? file-or-dir) 65 | (let [[file dir relative] (file-dir-relative file-or-dir) 66 | ignore-list (config->ignore-list config)] 67 | (->> (if file 68 | [file] 69 | (fs/glob dir pattern)) 70 | (mapv relative) 71 | (filterv (complement (partial ignore? ignore-list))))))) 72 | -------------------------------------------------------------------------------- /symlink-fdb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ln -s "$(pwd)/src/fdb/bb/cli.clj" /usr/local/bin/fdb 4 | -------------------------------------------------------------------------------- /talks/2024-05-london-clojurians.iapresenter/info.json: -------------------------------------------------------------------------------- 1 | { 2 | "creatorIdentifier" : "net.ia.presenter", 3 | "net.ia.presenter" : { 4 | "preset" : "Default", 5 | "template" : "san francisco" 6 | }, 7 | "transient" : false, 8 | "type" : "net.daringfireball.markdown", 9 | "version" : 2 10 | } -------------------------------------------------------------------------------- /talks/2024-05-london-clojurians.iapresenter/thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filipesilva/fdb/ef58e652c70edfe37337201f22193290cd2adc00/talks/2024-05-london-clojurians.iapresenter/thumb.png -------------------------------------------------------------------------------- /test/fdb/call_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.call-test 2 | (:require 3 | [clojure.test :refer [deftest is]] 4 | [fdb.call :as sut])) 5 | 6 | (deftest symbol-test 7 | (is (= 43 ((sut/to-fn 'clojure.core/inc) 42))) 8 | (is (= 43 ((sut/to-fn 'inc) 42)))) 9 | 10 | (deftest map-test 11 | (is (= 43 ((sut/to-fn {:call 'inc}) 42)))) 12 | 13 | (deftest sexp-test 14 | (is (= 43 ((sut/to-fn '(fn [x] (inc x))) 42))) 15 | (is (= "foo" ((sut/to-fn 16 | '(fn [x] 17 | (clojure.string/lower-case x))) 18 | "FOO")))) 19 | 20 | (deftest eval-under-call-arg 21 | (is (= [1] (sut/eval-under-call-arg {:self-path 1} '[self-path]))) 22 | (is (= [1 2] (sut/eval-under-call-arg {:self-path 1} '[(:self-path call-arg) 2]))) 23 | (is (= [1 1] (sut/eval-under-call-arg {:self-path 1} '[1 1])))) 24 | 25 | (deftest shell-test 26 | (is (= "1 2 3\n" 27 | (with-out-str 28 | ((sut/to-fn '[:sh "echo" config-path target-path self-path]) 29 | {:config-path "1" :target-path "2" :self-path "3"}))))) 30 | 31 | (deftest apply-test 32 | (is (= 43 (sut/apply 'inc 42)))) 33 | -------------------------------------------------------------------------------- /test/fdb/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.core-test 2 | (:require 3 | [babashka.fs :as fs :refer [with-temp-dir]] 4 | [clojure.core.async :refer [ call-arg :on-path first))) 67 | 68 | (deftest make-me-a-reactive-fdb 69 | (reset! calls []) 70 | (with-temp-fdb-config [config-path mount-path] 71 | (fdb/with-watch [config-path node] 72 | (u/spit mount-path "file.txt.meta.edn" 73 | {:fdb/refs #{"/test/one.md" 74 | "/test/folder/two.md"} 75 | :fdb.on/modify 'fdb.core-test/log-call 76 | :fdb.on/refs 'fdb.core-test/log-call 77 | :fdb.on/pattern {:glob "**/*.md" 78 | :call 'fdb.core-test/log-call} 79 | :fdb.on/query {:q '[:find ?e :where [?e :xt/id]] 80 | :path "./query-out.edn" 81 | :call 'fdb.core-test/log-call} 82 | :fdb.on/tx 'fdb.core-test/log-call 83 | :fdb.on/schedule [{:every [50 :millis] 84 | :call 'fdb.core-test/log-call} 85 | {:every [50 :millis] 86 | :call '(fn [call-arg] 87 | (fdb.utils/sleep 100) 88 | (fdb.core-test/log-call {:on [:fdb.on/schedule-timeout]})) 89 | :timeout [50 :millis]}] 90 | :fdb.on/startup 'fdb.core-test/log-call 91 | :fdb.on/shutdown 'fdb.core-test/log-call}) 92 | (is (u/eventually (db/entity "/test/file.txt"))) 93 | (db/entity "/test/file.txt") 94 | (u/spit mount-path "one.md" "") 95 | (u/spit mount-path "folder/two.md" "") 96 | (is (u/eventually (db/entity "/test/folder/two.md"))) 97 | ;; Check query ran by waiting for query result. 98 | (is (u/eventually (= #{["/test/file.txt"] 99 | ["/test/folder"] 100 | ["/test/folder/two.md"] 101 | ["/test/one.md"] 102 | ["/test/query-out.edn"]} 103 | (u/slurp-edn mount-path "query-out.edn")))) 104 | (is (u/eventually (some #{:fdb.on/schedule} @calls)))) 105 | 106 | ;; restart for startup/shutdown 107 | (fdb/with-watch [config-path node]) 108 | 109 | (is (= {:fdb.on/modify 1 ;; one for each modify 110 | :fdb.on/refs 2 ;; one.txt folder/two.txt 111 | :fdb.on/pattern 2 ;; one.md folder/two.md 112 | :fdb.on/query 5 ;; one for each tx, but sometimes I see 6, I guess from watcher stuff 113 | :fdb.on/tx 5 ;; one for each tx 114 | :fdb.on/schedule true ;; positive, hard to know how many 115 | :fdb.on/startup 2 ;; one for each startup or write with trigger 116 | :fdb.on/shutdown 2 ;; one for each shutdown 117 | } 118 | (-> @calls 119 | frequencies 120 | (update :fdb.on/schedule #(when % true)) 121 | (update :fdb.on/tx #(min 5 %))))))) 122 | 123 | (deftest make-me-a-query-fdb 124 | (with-temp-fdb-config [config-path mount-path] 125 | (fdb/with-watch [config-path node] 126 | (u/spit mount-path "one.txt" "") 127 | (u/spit mount-path "folder/two.txt" "") 128 | (u/spit mount-path "all-modified.query.fdb.edn" 129 | '[:find ?e ?modified 130 | :where [?e :fdb/modified ?modified]]) 131 | (u/eventually (u/slurp mount-path "all-modified.query-out.fdb.edn")) 132 | (is (= #{"/test/one.txt" 133 | "/test/folder" 134 | "/test/folder/two.txt" 135 | "/test/all-modified.query.fdb.edn"} 136 | (->> (u/slurp-edn mount-path "all-modified.query-out.fdb.edn") 137 | (map first) 138 | set))) 139 | (u/spit mount-path "all-modified.query.fdb.edn" "foo") 140 | (u/eventually (:error (u/slurp-edn mount-path "all-modified.query-out.fdb.edn"))) 141 | (is (= "Query didn't match expected structure" 142 | (:error (u/slurp-edn mount-path "all-modified.query-out.fdb.edn")))) 143 | (u/spit mount-path "all-modified.query.fdb.md" " 144 | ```edn 145 | [:find ?e ?modified 146 | :where [?e :fdb/modified ?modified]] 147 | ```") 148 | (u/eventually (u/slurp mount-path "all-modified.query-out.fdb.md")) 149 | (is (= #{"/test/one.txt" 150 | "/test/folder" 151 | "/test/folder/two.txt" 152 | "/test/all-modified.query.fdb.edn" 153 | "/test/all-modified.query-out.fdb.edn" 154 | "/test/all-modified.query.fdb.md"} 155 | (->> (u/slurp mount-path "all-modified.query-out.fdb.md") 156 | (triggers/unwrap-md-codeblock "edn") 157 | u/read-edn 158 | (map first) 159 | set)))))) 160 | 161 | (deftest make-me-a-repl-fdb 162 | (with-temp-fdb-config [config-path mount-path] 163 | (fdb/with-watch [config-path node] 164 | (u/spit mount-path "repl.fdb.clj" "(inc 1)") 165 | (u/eventually (u/slurp mount-path "repl-out.fdb.clj")) 166 | (is (= "(inc 1)\n;; => 2\n\n" 167 | (u/slurp mount-path "repl-out.fdb.clj"))) 168 | (let [form-with-out-and-err " 169 | (println 1) 170 | (binding [*out* *err*] 171 | (println 2)) 172 | (inc 2)"] 173 | (u/spit mount-path "2repl.fdb.clj" form-with-out-and-err) 174 | (u/eventually (u/slurp mount-path "2repl-out.fdb.clj")) 175 | (is (= (str form-with-out-and-err "\n;; 1\n;; 2\n;; => 3\n\n") 176 | (u/slurp mount-path "2repl-out.fdb.clj"))))))) 177 | 178 | (def ignore-calls (atom 0)) 179 | 180 | (defn ignore-log-call [_] 181 | (swap! ignore-calls inc)) 182 | 183 | (deftest ignore-me-a-change 184 | (reset! ignore-calls 0) 185 | (with-temp-fdb-config [config-path mount-path] 186 | (fdb/with-watch [config-path node] 187 | (let [f (fs/path mount-path "one.meta.edn")] 188 | (u/spit-edn f {:fdb.on/modify {:call 'fdb.core-test/ignore-log-call 189 | :count 1}}) 190 | (is (u/eventually (= 1 @ignore-calls))) 191 | 192 | (metadata/swap! f assoc :fdb.on/ignore true) 193 | (metadata/swap! f update-in [:fdb.on/modify :count] inc) 194 | (metadata/swap! f update-in [:fdb.on/modify :count] inc) 195 | (metadata/swap! f update-in [:fdb.on/modify :count] inc) 196 | 197 | ;; wait to see if it gets there 198 | (u/eventually (= 4 @ignore-calls)) 199 | 200 | (is (= 1 @ignore-calls)) 201 | (is (-> (u/slurp-edn f) 202 | (get-in [:fdb.on/modify :count]) 203 | (= 4))))))) 204 | 205 | (def blocking-ch nil) 206 | 207 | (defn blocking-fn [_] 208 | (> node db/all (map :xt/id) set))] 216 | 217 | ;; starts empty 218 | (fdb/sync config-path) 219 | (is (empty (all-ids))) 220 | 221 | ;; updates 222 | (u/spit f {}) 223 | (fdb/sync config-path) 224 | (is (= #{f-id} (all-ids))) 225 | 226 | ;; blocks on sync calls 227 | (with-redefs [blocking-ch (chan)] 228 | (u/spit-edn f {:fdb.on/modify {:call 'fdb.core-test/blocking-fn}}) 229 | (let [sync-fut (future (fdb/sync config-path))] 230 | (is (not (future-done? sync-fut))) 231 | (u/sleep 100) 232 | (is (not (future-done? sync-fut))) 233 | (close! blocking-ch) 234 | (is (u/eventually (future-done? sync-fut))))) 235 | 236 | ;; deletes 237 | (fs/delete f) 238 | (is (empty (all-ids)))))) 239 | 240 | (deftest make-me-a-reader-db 241 | (with-temp-fdb-config [config-path mount-path] 242 | (let [f (str (fs/path mount-path "one.my-edn")) 243 | f-md (str (fs/path mount-path "one.my-edn.meta.edn")) 244 | f-id "/test/one.my-edn" 245 | get-f (fn [] 246 | (fdb/sync config-path) 247 | (fdb/with-fdb [config-path _ _] 248 | (db/entity f-id)))] 249 | (u/swap-edn-file! config-path assoc 250 | :readers {:my-edn '(fn [x#] (-> x# :self-path fdb.utils/slurp-edn))}) 251 | (is (empty? (get-f))) 252 | (u/spit-edn f {:one 1}) 253 | (is (= {:xt/id f-id 254 | :fdb/modified (metadata/modified f) 255 | :fdb/parent "/test" 256 | :one 1} 257 | (get-f))) 258 | (u/spit-edn f-md {:two 2}) 259 | (is (= {:xt/id f-id 260 | :fdb/modified (metadata/modified f-md) 261 | :fdb/parent "/test" 262 | :one 1 263 | :two 2} 264 | (get-f))) 265 | (u/spit-edn f-md {:one 2}) 266 | (is (= {:xt/id f-id 267 | :fdb/modified (metadata/modified f-md) 268 | :fdb/parent "/test" 269 | :one 2} 270 | (get-f)))))) 271 | 272 | (deftest make-me-a-loader-db 273 | (with-temp-fdb-config [config-path mount-path] 274 | (let [script (str (fs/path mount-path "script.clj")) 275 | f (str (fs/path mount-path "f.meta.edn"))] 276 | (u/swap-edn-file! config-path assoc :load [script]) 277 | (u/spit-edn f {:fdb.on/modify 'foo}) 278 | (spit script " 279 | (def *load-test (atom nil)) 280 | (defn foo [{:keys [self-path]}] (reset! *load-test self-path))") 281 | (fdb/sync config-path) 282 | (is (= (metadata/metadata-path->content-path f) 283 | (-> 'user/*load-test resolve deref deref)))))) 284 | 285 | (deftest make-me-a-server 286 | (with-temp-fdb-config [config-path mount-path] 287 | (let [script (str (fs/path mount-path "script.clj"))] 288 | (u/swap-edn-file! config-path assoc :load [script] :serve {:routes {"GET /" 'user/endpoint}}) 289 | (spit script "(defn endpoint [_] {:status 200 :body {:a 1}})") 290 | (fdb/with-fdb [config-path _ node] 291 | (is (= {:status 200 :body "{\"a\":1}"} 292 | (select-keys @(http/get "http://localhost:80/") [:status :body]))))))) 293 | 294 | ;; TODO 295 | ;; - speed up tests if I can, it's 15s now because of the watch stuff 296 | ;; - on tests that don't start/stop fdb I can use in-memory xtdb, but that doesn't persist 297 | ;; - might be able to stop it from closing during fdb usage and just leave it open all the time 298 | ;; - got it open all the time but couldn't prevent it from being closed in fdb while 299 | ;; still being able to use it as a xtdb node 300 | -------------------------------------------------------------------------------- /test/fdb/db_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.db-test 2 | (:require 3 | [babashka.fs :refer [with-temp-dir]] 4 | [clojure.test :refer [deftest is]] 5 | [fdb.db :as sut] 6 | [xtdb.api :as xt])) 7 | 8 | (defmacro with-db 9 | {:clj-kondo/ignore [:unresolved-symbol]} 10 | [[node] & body] 11 | `(with-temp-dir [db-path# {}] 12 | (with-open [~node (sut/node db-path#)] 13 | ~@body))) 14 | 15 | (deftest make-me-a-db 16 | (with-db [node] 17 | (sut/put node :foo {:bar "bar"}) 18 | (xt/sync node) 19 | (is (= {:xt/id :foo :bar "bar"} 20 | (sut/pull node :foo))))) 21 | -------------------------------------------------------------------------------- /test/fdb/email_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.email-test 2 | (:require 3 | [babashka.fs :as fs :refer [with-temp-dir]] 4 | [clojure.test :refer [deftest is]] 5 | [fdb.email :as sut] 6 | [clojure.java.io :as io])) 7 | 8 | (deftest split-mbox-test 9 | (with-temp-dir [path {}] 10 | (fs/copy (io/resource "email/sample-crlf.mbox") 11 | (fs/file path "sample-crlf.mbox")) 12 | (sut/split-mbox (fs/file path "sample-crlf.mbox")) 13 | (let [f1 "sample-crlf/1970-01-01T00.00.00Z 8d247ee6 Sample message 1.eml" 14 | f2 "sample-crlf/1970-01-01T00.00.00Z c2dfc80c Sample message 2.eml"] 15 | (is (= (slurp (io/resource (str "eml/" f1))) 16 | (slurp (fs/file path f1)))) 17 | (is (= (slurp (io/resource (str "eml/" f2))) 18 | (slurp (fs/file path f2))))))) 19 | -------------------------------------------------------------------------------- /test/fdb/http_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.http-test 2 | (:require 3 | [clojure.test :refer [deftest is]] 4 | [fdb.http :as sut] 5 | )) 6 | 7 | (deftest encode-and-decode-uri 8 | (is (= "foo%20bar" (sut/encode-uri "foo bar"))) 9 | (is (= "../foo.bar" (sut/encode-uri "../foo.bar"))) 10 | (is (= "foo bar" (-> "foo bar" sut/encode-uri sut/decode-uri)))) 11 | 12 | (deftest add-params-test 13 | (is (= "foo?q=Lisbon&limit=1&format=json" 14 | (sut/add-params "foo" {:q "Lisbon" :limit 1 :format "json"}))) 15 | (is (= "foo?latitude=38.7077507&longitude=-9.1365919&daily=temperature_2m_max&daily=temperature_2m_min&past_days=7" 16 | (sut/add-params "foo" {:latitude "38.7077507" 17 | :longitude "-9.1365919" 18 | :daily ["temperature_2m_max", "temperature_2m_min"] , 19 | :past_days 7})))) 20 | -------------------------------------------------------------------------------- /test/fdb/metadata_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.metadata-test 2 | (:refer-clojure :exclude [read]) 3 | (:require 4 | [babashka.fs :refer [with-temp-dir] :as fs] 5 | [clojure.test :refer [deftest is]] 6 | [fdb.metadata :as sut] 7 | [fdb.utils :as u])) 8 | 9 | (deftest id->mount-spec 10 | (let [config {:mounts {:test "./test" 11 | "test2" {:path "./test2"}}}] 12 | (is (= "./test" (sut/id->mount-spec config "/test/foo.txt"))) 13 | (is (= {:path "./test2"} (sut/id->mount-spec config "/test2/foo.txt"))))) 14 | 15 | (defn- as-md 16 | [s] 17 | (str s "." sut/default-metadata-ext)) 18 | 19 | (deftest id 20 | (is (= "/test/foo.txt" 21 | (sut/id :test "foo.txt"))) 22 | (is (= "/test/foo.txt" 23 | (sut/id "test" "foo.txt"))) 24 | (is (= "/test/foo.txt" 25 | (sut/id :test "/foo.txt"))) 26 | (is (= "/test/foo.txt" 27 | (sut/id :test (as-md "foo.txt"))))) 28 | 29 | (deftest id->path 30 | (let [config-path "/root/foo/config.edn" 31 | config {:mounts {:not-test "./not-test" 32 | :test "./test"} }] 33 | (is (= "/root/foo/test/folder/foo.txt" 34 | (sut/id->path config-path config "/test/folder/foo.txt"))) 35 | (is (nil? (sut/id->path config-path config "not-/test/folder/foo.txt"))) 36 | (is (nil? (sut/id->path config-path config "/just-mount-id"))) 37 | (is (nil? (sut/id->path config-path config "/missing-mount-id/foo.txt"))))) 38 | 39 | (deftest path->id 40 | (let [config-path "/root/foo/config.edn" 41 | config {:mounts {:test "./test"} }] 42 | (is (= "/test/foo.txt" (sut/path->id config-path config "/root/foo/test/foo.txt"))) 43 | (is (= "/test/foo.txt" (sut/path->id config-path config "/test/foo.txt"))) 44 | (is (nil? (sut/path->id config-path config "/root/foo/not-test/foo.txt"))))) 45 | 46 | (deftest content-path->metadata-path 47 | (is (= (as-md "foo.txt") 48 | (sut/content-path->metadata-path "foo.txt"))) 49 | (is (thrown? AssertionError 50 | (sut/content-path->metadata-path (as-md "foo.txt"))))) 51 | 52 | (deftest metadata-path->content-path 53 | (is (= "foo.txt" 54 | (sut/metadata-path->content-path (as-md "foo.txt")))) 55 | (is (thrown? AssertionError 56 | (sut/metadata-path->content-path "foo.txt")))) 57 | 58 | (deftest read 59 | (with-temp-dir [dir {}] 60 | (is (nil? (sut/read (str dir "/foo.txt")))) 61 | (let [f (u/spit dir "f.txt" "") 62 | edn {:bar "bar"} 63 | fmd (u/spit (as-md f) edn)] 64 | (is (= (merge {:fdb/modified (sut/modified fmd)} edn) 65 | (sut/read f) 66 | (sut/read fmd)))))) 67 | -------------------------------------------------------------------------------- /test/fdb/readers/edn_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.readers.edn-test 2 | (:require 3 | [babashka.fs :as fs :refer [with-temp-dir]] 4 | [clojure.test :refer [deftest is]] 5 | [fdb.readers.edn :as sut] 6 | [fdb.utils :as u])) 7 | 8 | (deftest read-test 9 | (with-temp-dir [dir {}] 10 | (u/spit-edn dir "foo.edn" {:a 1}) 11 | (is (= {:a 1} (sut/read {:self-path (fs/path dir "foo.edn")}))) 12 | (is (= nil (sut/read {:self-path (fs/path dir "bar.edn")}))))) 13 | -------------------------------------------------------------------------------- /test/fdb/readers/eml_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.readers.eml-test 2 | (:require 3 | [babashka.fs :as fs] 4 | [clojure.test :refer [deftest is]] 5 | [fdb.readers.eml :as sut] 6 | [clojure.java.io :as io])) 7 | 8 | (deftest read-test 9 | (is (= (sut/read {:self-path (-> "eml/sample.eml" io/resource fs/file)}) 10 | (sut/read {:self-path (-> "eml/sample-crlf/1970-01-01T00.00.00Z 8d247ee6 Sample message 1.eml" 11 | io/resource fs/file)}) 12 | {:message-id "123", 13 | :from ["author@example.com"], 14 | :text "This is the body.\nThere are 2 lines.\n", 15 | :subject "Sample message 1", 16 | :to ["recipient@example.com"]}))) 17 | 18 | -------------------------------------------------------------------------------- /test/fdb/readers/md_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.readers.md-test 2 | (:require 3 | [babashka.fs :as fs] 4 | [clojure.java.io :as io] 5 | [clojure.test :refer [deftest is]] 6 | [fdb.readers.md :as sut] 7 | [fdb.utils :as u])) 8 | 9 | (deftest metadata-test 10 | (let [config-path (-> "md" io/resource io/file fs/parent (fs/path "fdbconfig.edn") str)] 11 | (is (= {:aliases ["something" "else" "another"], 12 | :checkbox true, 13 | :cssclasses [".foo" ".bar"], 14 | :date #inst "1986-04-25T00:00:00.000-00:00", 15 | :datetime #inst "1986-04-25T16:00:00.000-00:00", 16 | :list ["one" "two" "three"], 17 | :list-links ["[[another file]]" "[[other file]]"], 18 | :number 14, 19 | :tags ["accepted" "applied" "action-item"], 20 | :text "one two three", 21 | :text-link "[[another file]]", 22 | :fdb/refs #{"/vault/other file.md" 23 | "/vault/inbox/another file.md" 24 | "/vault/another file.md" 25 | "/vault/md link.md"}, 26 | :fdb/k {:foo "bar"}, 27 | :fdb.a/ks ['n.s/sym {:call 'n.s/another-sym} [:sh "echo"]],} 28 | (sut/read {:config-path config-path 29 | :config {:mounts {:vault "./md"}} 30 | :self-path (u/sibling-path config-path "./md/file.md") 31 | :self {:xt/id "/vault/file.md"}}))))) 32 | -------------------------------------------------------------------------------- /test/fdb/readers_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.readers-test 2 | (:refer-clojure :exclude [read]) 3 | (:require 4 | [babashka.fs :as fs :refer [with-temp-dir]] 5 | [clojure.test :refer [deftest is]] 6 | [fdb.call :as call] 7 | [fdb.readers :as sut] 8 | [fdb.utils :as u])) 9 | 10 | (deftest id->readers 11 | (with-redefs [sut/default-readers {}] 12 | (let [fn1 #() 13 | fn2 #() 14 | fn3 #() 15 | id "/test/f.md" 16 | id->rdrs #(sut/id->readers % id)] 17 | (is (nil? (id->rdrs {}))) 18 | (is (= [fn1] (id->rdrs {:readers {:md fn1}}))) 19 | (is (= [fn1 fn2] (id->rdrs {:readers {:md [fn1 fn2]}}))) 20 | (is (= [fn2] (id->rdrs {:readers {:md [fn1]} 21 | :mounts {:test {:readers {:md fn2}} 22 | :not-test {:readers {:md fn3}}}}))) 23 | (is (= [fn1 fn2] (id->rdrs {:readers {:md [fn1]} 24 | :mounts {:test {:extra-readers {:md fn2}}}}))) 25 | (is (= [fn1] (id->rdrs {:mounts {:test {:extra-readers {:md fn1}}}})))))) 26 | 27 | (deftest read 28 | (with-temp-dir [dir {}] 29 | (let [f (fs/file dir "./test/f.edn") 30 | config {:mounts {:test {:path "./test" 31 | :readers {:edn [#(-> % :self-path u/slurp-edn) 32 | {:call (fn [x] {:bar (-> x :on second :str)}) 33 | :str "baz"}]}}}} 34 | call-arg {:config-path (fs/file dir "fdbconfig.edn") 35 | :config config 36 | :self {:xt/id "/test/f.edn"} 37 | :self-path f}] 38 | (u/spit-edn f {:foo "bar"}) 39 | (is (= {:foo "bar" 40 | :bar "baz"} 41 | (call/with-arg call-arg 42 | (sut/read config "/test/f.edn"))))))) 43 | -------------------------------------------------------------------------------- /test/fdb/triggers_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.triggers-test 2 | (:require 3 | [clojure.test :refer [deftest is]] 4 | [fdb.call :as call] 5 | [fdb.db :as db] 6 | [fdb.db-test :as db-test] 7 | [fdb.metadata :as metadata] 8 | [fdb.triggers :as sut] 9 | [fdb.utils :as u] 10 | [spy.core :as spy] 11 | [xtdb.api :as xt])) 12 | 13 | (deftest call-all-triggers-test 14 | (let [should-trigger? (spy/spy (fn [trigger] 15 | (when (:doit trigger) 16 | {:didit true}))) 17 | call-spy (spy/spy) 18 | self {:on-foo [{:call 1} 19 | {:doit true 20 | :call 2} 21 | 3] 22 | :on-bar 4}] 23 | (with-redefs [call/to-fn (fn [_] call-spy) 24 | metadata/id->path (spy/stub "root/folder/foo.txt")] 25 | (binding [sut/*sync* true] 26 | (call/with-arg {:call-arg true} 27 | (sut/call-all-triggers :adoc 28 | self 29 | :on-foo 30 | should-trigger?) 31 | (sut/call-all-triggers :adoc 32 | self 33 | :on-bar))) 34 | (is (= [[{:call 1}] [{:doit true :call 2}] [3]] 35 | (spy/calls should-trigger?))) 36 | (is (= [[{:call-arg true 37 | :self self 38 | :self-path "root/folder/foo.txt" 39 | :target :adoc 40 | :target-path "root/folder/foo.txt" 41 | :on {:doit true :call 2} 42 | :on-path [:on-foo 1] 43 | :didit true}] 44 | [{:call-arg true 45 | :self self 46 | :self-path "root/folder/foo.txt" 47 | :target :adoc 48 | :target-path "root/folder/foo.txt" 49 | :on 4 50 | :on-path [:on-bar]}]] 51 | (spy/calls call-spy)))))) 52 | 53 | (deftest docs-with-k-test 54 | (db-test/with-db [node] 55 | (db/put node :one {:foo 1}) 56 | (db/put node :two {:bar 2}) 57 | (db/put node :three {:baz 3}) 58 | (db/put node :four {:bar 4}) 59 | (xt/sync node) 60 | (is (= [{:xt/id :two :bar 2} 61 | {:xt/id :four :bar 4}] 62 | (sut/docs-with-k (xt/db node) :bar))))) 63 | 64 | (deftest out-file-test 65 | (is (= "query-out.fdb.edn" 66 | (sut/out-file "/test/folder/query.fdb.edn" "query" "edn"))) 67 | (is (= "foo.query-out.fdb.edn" 68 | (sut/out-file "/test/folder/foo.query.fdb.edn" "query" "edn"))) 69 | (is (nil? (sut/out-file "/test/folder/foo.fdb.edn" "query" "edn")))) 70 | 71 | (deftest recursive-pull-k 72 | (db-test/with-db [node] 73 | (db/put node :1 {:foo 1 :fdb/refs #{:4}}) 74 | (db/put node :2 {:foo 2 :fdb/refs #{:1}}) 75 | (db/put node :3 {:foo 3 :fdb/refs #{:1}}) 76 | (db/put node :4 {:foo 4 :fdb/refs #{:2 :3}}) 77 | (db/put node :5 {:foo 5}) 78 | (db/put node :6 {:foo 6 :fdb/refs #{:5}}) 79 | (xt/sync node) 80 | (is (= (set [{:xt/id :1 :foo 1 :fdb/refs #{:4}} 81 | {:xt/id :2 :foo 2 :fdb/refs #{:1}} 82 | {:xt/id :3 :foo 3 :fdb/refs #{:1}} 83 | {:xt/id :4 :foo 4 :fdb/refs #{:2 :3}}]) 84 | (set (sut/recursive-pull-k (xt/db node) :1 :fdb/_refs)))))) 85 | 86 | (deftest matches-glob?-test 87 | (is (sut/matches-glob? "foo.txt" "foo.txt")) 88 | (is (sut/matches-glob? "foo.txt" "foo.*")) 89 | (is (sut/matches-glob? "foo.txt" "*.*")) 90 | (is (sut/matches-glob? "foo.txt" "*")) 91 | (is (not (sut/matches-glob? "foo.txt" "*.log"))) 92 | (is (sut/matches-glob? "/root/folder/foo.txt" "**")) 93 | (is (sut/matches-glob? "/root/folder/foo.txt" "/root/*/*")) 94 | (is (sut/matches-glob? "/root/folder/foo.txt" "/root/**")) 95 | (is (sut/matches-glob? "/root/folder/foo.txt" "**.txt")) 96 | (is (not (sut/matches-glob? "/root/folder/foo.txt" "**.log")))) 97 | 98 | (deftest query-results-changed?-test 99 | (let [id "/test/folder/foo.txt"] 100 | (with-redefs [xt/q (spy/spy (fn [_ q] (if (= q :gimme-new) 101 | {:v 2} 102 | {:v 1}))) 103 | u/slurp-edn (spy/stub {:v 1}) 104 | spit (spy/spy)] 105 | (call/with-arg {:config-path "/root/one/two/config.edn" 106 | :config {:mounts {:test "test"}}} 107 | (is (= {:results {:v 2}} 108 | (sut/query-results-changed? nil id 109 | {:q :gimme-new 110 | :path "results.edn"}))) 111 | (is (nil? (sut/query-results-changed? nil id 112 | {:q :foo 113 | :path "results.edn"}))) 114 | 115 | (is (= [["/root/one/two/test/folder/results.edn" "{:v 2}"]] 116 | (spy/calls spit))))))) 117 | 118 | (deftest massage-ops-test 119 | (with-redefs [db/xtdb-id->xt-id (spy/stub "/test/deleted.txt")] 120 | (is (= [[:xtdb.api/put "/test/one.txt" {:foo 1, :xt/id "/test/one.txt"}] 121 | [:xtdb.api/delete "/test/deleted.txt"]] 122 | (sut/massage-ops nil [[:xtdb.api/put {:foo 1, :xt/id "/test/one.txt"}] 123 | ;; that thing is actually a #xtdb/id 124 | [:xtdb.api/delete 'c8d0e9aa0ad6ad1b22c4b232a822615a263d8099] 125 | [:xtdb.api/put {:foo 2, :xt/id "/test/one.txt"} 126 | #inst "1115-02-13T18:00:00.000-00:00"]]))))) 127 | -------------------------------------------------------------------------------- /test/fdb/utils_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.utils-test 2 | (:require 3 | [babashka.fs :refer [with-temp-dir] :as fs] 4 | [fdb.utils :as sut] 5 | [clojure.test :refer [deftest is]])) 6 | 7 | (deftest eventually-early 8 | (let [foo (atom 0) 9 | foo-fn (fn [] 10 | (let [v (swap! foo inc)] 11 | (= v 3)))] 12 | (sut/eventually (foo-fn)) 13 | (is (= 3 @foo)))) 14 | 15 | (deftest eventually-late 16 | (let [foo (atom 0) 17 | foo-fn (fn [] 18 | (let [v (swap! foo inc)] 19 | (= v 150)))] 20 | (sut/eventually (foo-fn)) 21 | ;; not guaranteed to hit 100 before timeout 22 | (is (>= 100 @foo)))) 23 | 24 | (deftest eventually-never 25 | (let [foo (atom 0) 26 | foo-fn (fn [] 27 | (let [_v (swap! foo inc)] 28 | false))] 29 | (sut/eventually (foo-fn)) 30 | (is (>= 100 @foo)))) 31 | 32 | (deftest lockfile-test 33 | (with-temp-dir [dir {}] 34 | (with-open [lock (sut/lockfile dir "lockfile-test") 35 | lock2 (sut/lockfile dir "lockfile-test")] 36 | (is @lock) 37 | (is (nil? @lock2))) 38 | (let [lock3 (sut/lockfile dir "lockfile-test")] 39 | (is @lock3) 40 | (.close lock3)))) 41 | 42 | (deftest swap-edn-file-test 43 | (with-temp-dir [dir {}] 44 | (let [f (fs/file dir "foo.edn")] 45 | (is (thrown? Exception (sut/swap-edn-file! f :foo inc))) 46 | (sut/spit-edn f {:foo 1}) 47 | (sut/swap-edn-file! f update :foo inc) 48 | (is (= {:foo 2} (sut/slurp-edn f)))))) 49 | 50 | (deftest filename-inst-test 51 | (is (= "2023-11-30T14.20.23Z" 52 | (sut/filename-inst #inst "2023-11-30T14:20:23.000-00:00")))) 53 | 54 | (deftest filename-str-test 55 | (is (= "foo bar-_,% !!àè(baz)[foo]{bar}" 56 | (sut/filename-str " foo:bar-_,%:?!!àè(baz)[foo]{bar} ")))) 57 | 58 | (deftest maybe-timeout-test 59 | (let [f #(sut/sleep 100)] 60 | (is (= ::sut/timeout (sut/maybe-timeout 0 f))) 61 | (is (= ::sut/timeout (sut/maybe-timeout [0 :seconds] f))) 62 | (is (= nil (sut/maybe-timeout nil f))) 63 | (is (= nil (sut/maybe-timeout 200 f))))) 64 | 65 | (deftest without-random-uuid-test 66 | (is (= [1 2 3] (sut/without-random-uuid [(random-uuid) (random-uuid) (random-uuid)])))) 67 | 68 | (deftest flatten-maps-test 69 | (is (= {"root" {:a 1}} (sut/flatten-maps {:a 1}))) 70 | (is (= {"1" {:e "e" 71 | :f "f"} 72 | "2" {:c "c" 73 | :d :1} 74 | "root" {:a "a" 75 | :b :2}} 76 | (sut/without-random-uuid 77 | (sut/flatten-maps 78 | {:a "a" 79 | :b {:c "c" 80 | :d {:e "e" 81 | :f "f"}}} 82 | :xform-uuid str 83 | :xform-ref (comp keyword str)))))) 84 | 85 | (deftest explode-id-test 86 | (with-temp-dir [dir {}] 87 | (let [id "/user/foo.edn" 88 | path (fs/file dir "foo.edn") 89 | path-k #(str path ".explode/" % ".edn") 90 | id-k #(str id ".explode/" % ".edn")] 91 | (is (nil? (sut/explode-id id path 1))) 92 | (is (nil? (sut/explode-id id path {:a "a"}))) 93 | (is (= {(path-k 1) {:d "d"}, 94 | (path-k 2) {:b "b" :c (id-k 1)}, 95 | (path-k "root") {:a (id-k 2)}} 96 | (sut/without-random-uuid 97 | (sut/explode-id id path {:a {:b "b" :c {:d "d"}}})))) 98 | (is (= {(path-k 1) {:a "a"}, 99 | (path-k 2) {:b "b"} 100 | (path-k "root") {:0 (id-k 1) :1 (id-k 2)}} 101 | (sut/without-random-uuid 102 | (sut/explode-id id path [{:a "a"} {:b "b"}])))) 103 | (is (= {(path-k 4) {:a "a"}, 104 | (path-k 3) {:b "b"} 105 | (path-k "root") {:1 (id-k 3) :2 (id-k 4)}} 106 | (sut/without-random-uuid 107 | (sut/explode-id id path #{{:a "a"} {:b "b"}}))))))) 108 | -------------------------------------------------------------------------------- /test/fdb/watcher_test.clj: -------------------------------------------------------------------------------- 1 | (ns fdb.watcher-test 2 | (:require 3 | [babashka.fs :refer [with-temp-dir] :as fs] 4 | [clojure.test :refer [deftest is]] 5 | [fdb.utils :as u] 6 | [fdb.watcher :as sut] 7 | [spy.core :as spy])) 8 | 9 | (deftest ignore?-test 10 | (is (not (sut/ignore? sut/default-ignore-list "foo"))) 11 | (is (sut/ignore? sut/default-ignore-list ".git")) 12 | (is (sut/ignore? sut/default-ignore-list ".git/a")) 13 | (is (sut/ignore? sut/default-ignore-list "a/.git/a"))) 14 | 15 | (deftest watch-me-a-dir 16 | (with-temp-dir [dir {}] 17 | (let [f1 (u/spit dir "f1.txt" "") 18 | f2 (u/spit dir "f2.txt" "") 19 | f3 (u/spit dir "folder/f3.txt" "") 20 | update-fn (spy/spy)] 21 | (with-open [_watcher (sut/watch {} (str dir) update-fn)] 22 | (u/spit f1 "1") 23 | (is (u/eventually (spy/called-n-times? update-fn 1)) 24 | "calls update for file changes") 25 | (u/spit f1 "11") 26 | (u/spit f2 "2") 27 | (u/spit f3 "3") 28 | (is (u/eventually (spy/called-n-times? update-fn 4)) 29 | "calls update multiple times") 30 | (fs/delete f1) 31 | (is (u/eventually (spy/called-n-times? update-fn 5)) 32 | "calls update on delete") 33 | (u/spit dir "folder2/f4.txt" "") 34 | (is (u/eventually (spy/called-n-times? update-fn 7)) 35 | "calls update for new files and folders"))))) 36 | 37 | (deftest watch-me-a-file 38 | (with-temp-dir [dir {}] 39 | (let [f1 (u/spit dir "f1.txt" "") 40 | update-fn (spy/spy)] 41 | (with-open [_watcher (sut/watch {} (str dir) update-fn)] 42 | (u/spit f1 "1") 43 | (is (u/eventually (spy/called-with? update-fn "f1.txt"))) 44 | (is (u/eventually (spy/called-n-times? update-fn 1))))))) 45 | 46 | (deftest glob 47 | (with-temp-dir [dir {}] 48 | (u/spit dir "f1.txt" "") 49 | (u/spit dir "f2.txt" "") 50 | (is (= ["f1.txt" "f2.txt"] (sut/glob {} dir))) 51 | (is (= ["f1.txt"] (sut/glob {} dir :pattern "*1.txt"))))) 52 | --------------------------------------------------------------------------------