├── test-resources ├── folder.clj │ └── .gitkeep └── foo.cljc ├── github ├── poedit.png └── poedit_translator_notes.png ├── test └── pottery │ ├── _fixtures │ ├── single_block.po │ ├── plural_block.po │ ├── multiline_single_block.po │ └── multiline_plural_block.po │ ├── scan_test.clj │ └── po_test.clj ├── deps.edn ├── .gitignore ├── src └── pottery │ ├── utils.clj │ ├── core.clj │ ├── po.clj │ └── scan.clj ├── examples └── simple │ ├── gettext │ ├── template.pot │ ├── nl.po │ └── fr.po │ └── example.clj ├── project.clj ├── CHANGELOG.md ├── LICENSE └── README.md /test-resources/folder.clj/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /github/poedit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brightin/pottery/HEAD/github/poedit.png -------------------------------------------------------------------------------- /test/pottery/_fixtures/single_block.po: -------------------------------------------------------------------------------- 1 | #: reference.clj 2 | msgid "Hello!" 3 | msgstr "Hoi!" 4 | -------------------------------------------------------------------------------- /github/poedit_translator_notes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brightin/pottery/HEAD/github/poedit_translator_notes.png -------------------------------------------------------------------------------- /test/pottery/_fixtures/plural_block.po: -------------------------------------------------------------------------------- 1 | #: reference.cljs 2 | msgid "one mouse" 3 | msgid_plural "%1 mice" 4 | msgstr[0] "een muis" 5 | msgstr[1] "muizen" 6 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:deps {org.clojure/core.match {:mvn/version "1.0.0"} 2 | borkdude/edamame {:mvn/version "1.4.24"}} 3 | :aliases {:test {:extra-paths ["test-resources"]}}} 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | profiles.clj 5 | pom.xml 6 | pom.xml.asc 7 | *.jar 8 | *.class 9 | /.lein-* 10 | /.nrepl-port 11 | .hgignore 12 | .hg/ 13 | -------------------------------------------------------------------------------- /test/pottery/_fixtures/multiline_single_block.po: -------------------------------------------------------------------------------- 1 | #: reference.cljs 2 | msgid "" 3 | "First-line" 4 | "\n" 5 | "Second-line" 6 | msgstr "" 7 | "Eerste regel" 8 | "\n" 9 | "Tweede regel" 10 | -------------------------------------------------------------------------------- /test/pottery/_fixtures/multiline_plural_block.po: -------------------------------------------------------------------------------- 1 | #: reference.cljs 2 | msgid "" 3 | "Some" 4 | "\n" 5 | "Long " 6 | "Message id" 7 | msgid_plural "" 8 | "Some" 9 | "\n" 10 | " plural id" 11 | msgstr[0] "" 12 | "First" 13 | "\n" 14 | "Second line [s]" 15 | msgstr[1] "" 16 | "First" 17 | "\n" 18 | "Second Line [p]" 19 | -------------------------------------------------------------------------------- /src/pottery/utils.clj: -------------------------------------------------------------------------------- 1 | (ns pottery.utils) 2 | 3 | (defn vectorize 4 | "Will return any data in vector form. Vectors return themselves, 5 | sequences are transformed to vectors and anything else will be 6 | wrapped in a vector." 7 | [x] 8 | (cond 9 | (nil? x) [] 10 | (vector? x) x 11 | (set? x) (vec x) 12 | (seq? x) (vec x) 13 | :else (vector x))) 14 | -------------------------------------------------------------------------------- /examples/simple/gettext/template.pot: -------------------------------------------------------------------------------- 1 | #: examples/simple/example.clj 2 | msgid "Greetings" 3 | msgstr "" 4 | 5 | #: examples/simple/example.clj 6 | msgid "Please confirm your email" 7 | msgstr "" 8 | 9 | #: examples/simple/example.clj 10 | msgid "Welcome, %s!" 11 | msgstr "" 12 | 13 | #: examples/simple/example.clj 14 | msgid "product" 15 | msgid_plural "%s products" 16 | msgstr[0] "" 17 | msgstr[1] "" 18 | -------------------------------------------------------------------------------- /test-resources/foo.cljc: -------------------------------------------------------------------------------- 1 | (ns foo 2 | (:require [clojure.set :as set])) 3 | 4 | (defn square [x] x) 5 | (defn tr [& _args]) 6 | 7 | (defn render-thing [{:keys [some-arg] ::keys [some-other-arg] ::set/keys [yet-another-arg]}] 8 | [:div 9 | (tr "Both Clojure and ClojureScript") 10 | #?(:clj (tr "Clojure text %1 %2 %3" some-arg some-other-arg yet-another-arg) 11 | :cljs (tr "ClojureScript text %1 %2 %3 %4" some-arg some-other-arg yet-another-arg #js [1 2 3]))]) 12 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject brightin/pottery "1.0.2" 2 | :description "A clojure library to interact with gettext and PO/POT files" 3 | :url "https://www.github.com/brightin/pottery" 4 | :license {:name "Hippocratic License" 5 | :url "https://firstdonoharm.dev/"} 6 | :dependencies [[org.clojure/clojure "1.10.0" :scope "provided"] 7 | [org.clojure/core.match "1.0.0"] 8 | [borkdude/edamame "1.4.24"]] 9 | :repl-options {:init-ns pottery.core}) 10 | -------------------------------------------------------------------------------- /examples/simple/gettext/nl.po: -------------------------------------------------------------------------------- 1 | msgid "" 2 | msgstr "" 3 | "Project-Id-Version: \n" 4 | "POT-Creation-Date: \n" 5 | "PO-Revision-Date: \n" 6 | "Last-Translator: \n" 7 | "Language-Team: \n" 8 | "Language: nl\n" 9 | "MIME-Version: 1.0\n" 10 | "Content-Type: text/plain; charset=UTF-8\n" 11 | "Content-Transfer-Encoding: 8bit\n" 12 | "X-Generator: Poedit 2.2\n" 13 | "Plural-Forms: nplurals=2; plural=(n != 1);\n" 14 | 15 | #: examples/simple/example.clj 16 | msgid "Greetings" 17 | msgstr "Groeten" 18 | 19 | #: examples/simple/example.clj 20 | msgid "Please confirm your email" 21 | msgstr "Bevestig uw e-mail" 22 | 23 | #: examples/simple/example.clj 24 | msgid "Welcome, %s!" 25 | msgstr "Welkom %s!" 26 | 27 | #: examples/simple/example.clj 28 | msgid "product" 29 | msgid_plural "%s products" 30 | msgstr[0] "product" 31 | msgstr[1] "%s producten" 32 | -------------------------------------------------------------------------------- /examples/simple/gettext/fr.po: -------------------------------------------------------------------------------- 1 | msgid "" 2 | msgstr "" 3 | "Project-Id-Version: \n" 4 | "POT-Creation-Date: \n" 5 | "PO-Revision-Date: \n" 6 | "Last-Translator: \n" 7 | "Language-Team: \n" 8 | "Language: fr\n" 9 | "MIME-Version: 1.0\n" 10 | "Content-Type: text/plain; charset=UTF-8\n" 11 | "Content-Transfer-Encoding: 8bit\n" 12 | "X-Generator: Poedit 2.2\n" 13 | "Plural-Forms: nplurals=2; plural=(n > 1);\n" 14 | 15 | #: examples/simple/example.clj 16 | msgid "Greetings" 17 | msgstr "Bonjour" 18 | 19 | #: examples/simple/example.clj 20 | msgid "Please confirm your email" 21 | msgstr "Veuillez confirmer votre email" 22 | 23 | #: examples/simple/example.clj 24 | msgid "Welcome, %s!" 25 | msgstr "Bienvenue, %s!" 26 | 27 | #: examples/simple/example.clj 28 | msgid "product" 29 | msgid_plural "%s products" 30 | msgstr[0] "produit" 31 | msgstr[1] "%s produits" 32 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). 4 | 5 | ## [1.0.2] 6 | 7 | - Bugfix: Handle multiline notes correctly in .pot file 8 | - Bugfix: Escape double quotes in msgid entries 9 | 10 | ## [1.0.1] 11 | 12 | - Bugfix: Avoid slurping directories. https://github.com/brightin/pottery/pull/10 13 | 14 | ## [1.0.0] 15 | 16 | - Bugfix: parse keys destructuring with auto-aliasing 17 | - Bugfix: some expressions in `.cljc` go undetected 18 | 19 | ## [0.0.5] 20 | 21 | - Bugfix: Allow multiline message ids. https://github.com/brightin/pottery/pull/2 22 | 23 | ## [0.0.4] 24 | 25 | ### Changed 26 | 27 | - Added default-data-reader-fn binding to read-string so it can handle unknown tags 28 | 29 | ## [0.0.3] 30 | 31 | ### Added 32 | 33 | - Added `read-po-str` to public interface of `pottery.core` 34 | 35 | ### Changed 36 | 37 | - Updated license for Clojars 38 | 39 | ## [0.0.2] 40 | 41 | ### Added 42 | 43 | - Added `pottery.po/read-po-str`. https://github.com/brightin/pottery/pull/1 44 | 45 | ### Changed 46 | 47 | - License from EPL to Hippocratic License 48 | - Default extractor now has string guards. Will warn when calling translation functions with the correct pattern but wrong types. 49 | 50 | ## [0.0.1] 51 | 52 | ### Added 53 | 54 | - First version of Pottery. 55 | -------------------------------------------------------------------------------- /src/pottery/core.clj: -------------------------------------------------------------------------------- 1 | (ns pottery.core 2 | (:require [pottery.po :as po] 3 | [pottery.scan :as scan] 4 | [clojure.java.io :as io])) 5 | 6 | (defn- default-scan-options [] 7 | {:dir "src" 8 | :extract-fn #'scan/default-extractor 9 | :template-file (io/file "resources/gettext/template.pot")}) 10 | 11 | (defmacro make-extractor 12 | "Returns an extraction function using the core.match pattern syntax. 13 | 14 | Example: 15 | 16 | (make-extractor 17 | ['tr s] s 18 | ['trn [s1 s2] _] [s1 s2]) " 19 | [& args] 20 | `(scan/make-extractor ~@args)) 21 | 22 | (defn scan-codebase! 23 | "Recursively reads the code in dir, scans all strings and outputs a 24 | .pot file according to the gettext format with all the translatable 25 | strings. 26 | 27 | Opts is a map which accepts: 28 | :dir - The source dir to be scanned. 29 | :extract-fn - The extraction function that gets called with every 30 | expression in the codebase 31 | :template-file - The POT file where the results are to be written. 32 | 33 | All of these options have defaults." 34 | ([] (scan-codebase! {})) 35 | ([opts] 36 | (let [{:keys [template-file] :as opts} (merge (default-scan-options) opts)] 37 | (io/make-parents template-file) 38 | (->> (scan/scan-files opts) 39 | (po/gen-template) 40 | (spit template-file))))) 41 | 42 | (def read-po-file #'po/read-po-file) 43 | (def read-po-str #'po/read-po-str) 44 | -------------------------------------------------------------------------------- /examples/simple/example.clj: -------------------------------------------------------------------------------- 1 | (ns simple.example 2 | (:require [pottery.core :as pottery] 3 | [clojure.java.io :as io])) 4 | 5 | (def DICT 6 | {:nl (pottery/read-po-file (io/file "examples/simple/gettext/nl.po")) 7 | :fr (pottery/read-po-file (io/file "examples/simple/gettext/fr.po"))}) 8 | 9 | (defn tr [lang s & args] 10 | (let [string (or (get-in DICT [lang s]) s)] 11 | (apply format string args))) 12 | 13 | (defn trn [lang strings count & args] 14 | (let [[singular plural] (or (get-in DICT [lang strings]) strings)] 15 | (if (= 1 count) 16 | (apply format singular (conj args count)) 17 | (apply format plural (conj args count))))) 18 | 19 | ;; Useful to have this in a dev user namespace 20 | (defn gettext-do-scan! [] 21 | (pottery/scan-codebase! 22 | {:dir "examples/simple" 23 | :template-file (io/file "examples/simple/gettext/template.pot") 24 | :extract-fn (pottery/make-extractor ;; We use the lang as first argument to tr and trn. 25 | ['tr _ (s :guard string?) & _] s 26 | ['trn _ [(s1 :guard string?) (s2 :guard string?)] & _] [s1 s2] 27 | [(:or 'tr 'trn) & _] (pottery.scan/extraction-warning 28 | "Could not extract strings for the form:"))})) 29 | 30 | ;; After calling the scan functions and translating the PO files, 31 | ;; re-eval DICT and this should be the results: 32 | (tr :en "Greetings") ;; => "Greetings" 33 | (tr :nl "Greetings") ;; => "Groeten" 34 | (tr :fr "Please confirm your email") ;; => "Veillez confirmer votre email" 35 | (tr :nl "Welcome, %s!" "John") ;; => "Welkom, John! 36 | 37 | (trn :nl ["product" "%s products"] 3) ;; => "3 producten" 38 | (trn :fr ["product" "%s products"] 1) ;; => "produit" 39 | -------------------------------------------------------------------------------- /test/pottery/scan_test.clj: -------------------------------------------------------------------------------- 1 | (ns pottery.scan-test 2 | (:require [pottery.scan :as sut] 3 | [clojure.test :refer [deftest testing is are]])) 4 | 5 | (set! *print-namespace-maps* false) 6 | 7 | (deftest extract-test 8 | (testing "Default extractor" 9 | (are [expr result] (= result (::sut/value (sut/extract sut/default-extractor expr))) 10 | '(tr "Hello") "Hello" 11 | '(tr "Hello %s" arg1) "Hello %s" 12 | '(tr "Hello\nthere") "Hello\nthere" 13 | '(tr "Hello:\n%s" arg1) "Hello:\n%s" 14 | '(trn ["One" "Many"] 1) ["One" "Many"] 15 | '(trn ["One %s" "Many %s" arg1] 3) ["One %s" "Many %s"]) 16 | 17 | (testing "Default extractor string guards" 18 | (is (nil? (sut/extract sut/default-extractor '(tr :key)))) 19 | (is (nil? (sut/extract sut/default-extractor '(trn [:key1 :key2] 1)))))) 20 | 21 | (testing "Advanced extractor" 22 | 23 | (let [extractor (sut/make-extractor 24 | ['tr _ [s & _]] s 25 | ['tr [s & _]] s 26 | ['trn _ [s1 s2 & _] _] [s1 s2] 27 | ['trn [s1 s2 & _] _] [s1 s2]) 28 | extract (partial sut/extract extractor)] 29 | 30 | (are [expr result] (= result (::sut/value (extract expr))) 31 | '(tr ["Hello"]) "Hello" 32 | '(tr ["Hello %1!" arg1 arg2]) "Hello %1!" 33 | '(tr ["Hello\n%1" arg1 arg2]) "Hello\n%1" 34 | '(trn ["Item" "Items"] 2) ["Item" "Items"] 35 | '(trn ["It\nem" "It\nems"] 2) ["It\nem" "It\nems"] 36 | '(inc 6) nil 37 | "foo" nil) 38 | 39 | (is (vector? (::sut/value (extract '(trn i18n ["one" "many"] 2))))) 40 | 41 | (is (= ["Some note"] 42 | (::sut/notes 43 | (extract (read-string "^{:notes \"Some note\"} (tr i18n [\"This is text\"])"))))) 44 | 45 | (is (= ["note 1" "note 2"] 46 | (::sut/notes 47 | (extract (read-string "^{:notes [\"note 1\" \"note 2\"]} (tr i18n [\"This is text\"])"))))) 48 | 49 | (is (= "Text" (::sut/value (extract (read-string "(tr (get-i18n) [\"Text\"])")))))))) 50 | 51 | (deftest find-tr-strings-test 52 | (is (= {::sut/filename "foo.cljs" 53 | ::sut/expressions [{::sut/value "Some text %1"}]} 54 | (sut/find-tr-strings 55 | sut/default-extractor 56 | {::sut/filename "foo.cljs" 57 | ::sut/expressions '((ns foo (:require [x :as b])) 58 | (defn square [x] x) 59 | (defn render-thing [{:keys [some-arg]}] 60 | [:div (tr "Some text %1" some-arg)]))})))) 61 | 62 | (deftest scan-files-test 63 | (is (= '({::sut/filename "test-resources/foo.cljc", 64 | ::sut/expressions 65 | ({::sut/value "Both Clojure and ClojureScript"} 66 | {::sut/value "Clojure text %1 %2 %3"} 67 | {::sut/value "ClojureScript text %1 %2 %3 %4"})}) 68 | (sut/scan-files {:dir "test-resources" 69 | :extract-fn sut/default-extractor})))) 70 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020 Brightin 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | 6 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | * The software may not be used by individuals, corporations, governments, or other groups for systems or activities that actively and knowingly endanger, harm, or otherwise threaten the physical, mental, economic, or general well-being of other individuals or groups. 9 | 10 | 11 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 12 | 13 | This license is derived from the MIT License, as amended to limit the impact of the unethical use of open source software. 14 | 15 | Copyright 2020 Brightin 16 | 17 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 18 | 19 | 20 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 21 | 22 | * No Harm: The software may not be used by anyone for systems or activities that actively and knowingly endanger, harm, or otherwise threaten the physical, mental, economic, or general well-being of other individuals or groups, in violation of the United Nations Universal Declaration of Human Rights (https://www.un.org/en/universal-declaration-human-rights/). 23 | 24 | * Services: If the Software is used to provide a service to others, the licensee shall, as a condition of use, require those others not to use the service in any way that violates the No Harm clause above. 25 | 26 | * Enforceability: If any portion or provision of this License shall to any extent be declared illegal or unenforceable by a court of competent jurisdiction, then the remainder of this License, or the application of such portion or provision in circumstances other than those as to which it is so declared illegal or unenforceable, shall not be affected thereby, and each portion and provision of this Agreement shall be valid and enforceable to the fullest extent permitted by law. 27 | 28 | 29 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 30 | 31 | This Hippocratic License is an Ethical Source license (https://ethicalsource.dev) derived from the MIT License, amended to limit the impact of the unethical use of open source software. 32 | -------------------------------------------------------------------------------- /src/pottery/po.clj: -------------------------------------------------------------------------------- 1 | (ns pottery.po 2 | (:require [clojure.string :as str] 3 | [pottery.scan :as scan])) 4 | 5 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 6 | ;; Generating PO Template file 7 | 8 | (def join (partial str/join "\n")) 9 | 10 | (defn- ->blocks 11 | "Converts a scan result block (filename + expressions) and assoc's 12 | all expressions (msg-ids) into the result map, mergine the filenames 13 | and note's of the scan result as values." 14 | [result {::scan/keys [filename expressions]}] 15 | (reduce (fn [acc {::scan/keys [value notes]}] 16 | (update acc value #(merge-with concat % {:filenames [filename] 17 | :notes notes}))) 18 | result expressions)) 19 | 20 | (defn- format-notes [notes] 21 | (when (seq notes) 22 | (str (str/join "\n" (mapcat (fn [note] (map #(str "#. " %) (str/split-lines note))) notes)) "\n"))) 23 | 24 | (defn- create-sort-index 25 | "Creates a map from every scanned value to an occurence 26 | position. Used to retain scan order when generating the PO 27 | template." 28 | [scan-results] 29 | (zipmap (map ::scan/value (mapcat ::scan/expressions scan-results)) 30 | (range))) 31 | 32 | (defn- fmt-msg-id [s] 33 | (let [lines (str/split-lines s) 34 | q #(str \" (str/escape % {\" "\\\""}) \")] 35 | (if (next lines) 36 | (->> (concat [(q "")] 37 | (map #(q (str % "\\n")) (butlast lines)) 38 | [(q (last lines))]) 39 | (str/join "\n")) 40 | (q s)))) 41 | 42 | (defn gen-template 43 | "Takes in a list of scan results (filename + msg-ids), groups 44 | multiple appearances of the same msgid together and returns a ready 45 | to spit PO template file." 46 | [scan-results] 47 | (println "Generating POT file...") 48 | (join 49 | (for [[msg-id {:keys [filenames notes]}] (sort-by (comp (create-sort-index scan-results) key) 50 | (reduce ->blocks {} scan-results))] 51 | (str 52 | (format-notes notes) 53 | (if (vector? msg-id) 54 | (format "#: %s\nmsgid %s\nmsgid_plural %s\nmsgstr[0] \"\"\nmsgstr[1] \"\"\n" 55 | (str/join " " filenames) (fmt-msg-id (first msg-id)) (fmt-msg-id (second msg-id))) 56 | (format "#: %s\nmsgid %s\nmsgstr \"\"\n" (str/join " " filenames) (fmt-msg-id msg-id))))))) 57 | 58 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 59 | ;; Reading PO files 60 | 61 | (defn- quoted-string? [line] 62 | (and (str/starts-with? line "\"") (str/ends-with? line "\""))) 63 | 64 | (defn- read-quoted-string [line] 65 | (str/replace 66 | (subs line 1 (dec (count line))) 67 | #"\\n" "\n")) 68 | 69 | (defn- tag-parser 70 | "Creates a fn that takes remaining lines and returns a tuple with 71 | the key, value and and the remaining unparsed lines." 72 | [key] 73 | (fn [[line & rest]] 74 | (let [[multiline-values rest] (split-with quoted-string? rest) 75 | values (map read-quoted-string (concat [line] multiline-values))] 76 | [key (str/join values) rest]))) 77 | 78 | (defn default-parser [lines] 79 | [nil nil (drop 1 lines)]) 80 | 81 | (def PO_TAGS 82 | {"msgid" (tag-parser ::msgid) 83 | "msgid_plural" (tag-parser ::msgid-plural) 84 | "msgstr" (tag-parser ::msgstr) 85 | "msgstr[0]" (tag-parser ::msgstr) 86 | "msgstr[1]" (tag-parser ::msgstr-plural)}) 87 | 88 | (defn parse-block 89 | "Parses a block in a PO file, and returns an object with the msgid, 90 | msgstr and possible plural versions of the strings." 91 | [block-str] 92 | (loop [lines (map str/trim (str/split-lines block-str)) 93 | result {}] 94 | (if (empty? lines) 95 | result 96 | (let [[tag remainder] (str/split (first lines) #"\s" 2) 97 | lines (concat [remainder] (rest lines)) 98 | parser (get PO_TAGS tag default-parser) 99 | [k v rest] (parser lines)] 100 | (recur rest (if v (assoc result k v) result)))))) 101 | 102 | (defn- ->kv [block] 103 | (if (contains? block ::msgid-plural) 104 | [[(::msgid block) (::msgid-plural block)] 105 | [(::msgstr block) (::msgstr-plural block)]] 106 | [(::msgid block) (::msgstr block)])) 107 | 108 | (defn read-po-str [s] 109 | (->> (str/split s #"\n\n") 110 | (drop 1) ;; Header meta data 111 | (map parse-block) 112 | (map ->kv) 113 | (into {}))) 114 | 115 | (defn read-po-file [file] 116 | (read-po-str (slurp file))) 117 | -------------------------------------------------------------------------------- /test/pottery/po_test.clj: -------------------------------------------------------------------------------- 1 | (ns pottery.po-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer [deftest is testing]] 4 | [pottery.po :as sut] 5 | [pottery.scan :as scan])) 6 | 7 | (deftest gen-template-test 8 | (testing "Single and plural expressions" 9 | (is (= (str "#: a.cljs\n" 10 | "msgid \"Hello %s!\"\n" 11 | "msgstr \"\"\n" 12 | "\n" 13 | "#: b.cljs\n" 14 | "msgid \"Item\"\n" 15 | "msgid_plural \"Items\"\n" 16 | "msgstr[0] \"\"\n" 17 | "msgstr[1] \"\"\n") 18 | (sut/gen-template [{::scan/filename "a.cljs" 19 | ::scan/expressions [{::scan/value "Hello %s!"}]} 20 | {::scan/filename "b.cljs" 21 | ::scan/expressions [{::scan/value ["Item" "Items"]}]}])))) 22 | 23 | (testing "Grouping multiple appearances of same string" 24 | (is (= (str "#: foo.cljs bar.cljs\n" 25 | "msgid \"Hello %s!\"\n" 26 | "msgstr \"\"\n") 27 | (sut/gen-template [{::scan/filename "foo.cljs" 28 | ::scan/expressions [{::scan/value "Hello %s!"}]} 29 | {::scan/filename "bar.cljs" 30 | ::scan/expressions [{::scan/value "Hello %s!"}]}])))) 31 | 32 | (testing "Escape double-quotes in msgids" 33 | (is (= (str "#: foo.cljs\n" 34 | "msgid \"Hello \\\"%s\\\"!\"\n" 35 | "msgstr \"\"\n") 36 | (sut/gen-template [{::scan/filename "foo.cljs" 37 | ::scan/expressions [{::scan/value "Hello \"%s\"!"}]}])))) 38 | 39 | (testing "Outputting translator notes" 40 | (is (= (str "#. note 1\n" 41 | "#. note 2\n" 42 | "#: file.cljs\n" 43 | "msgid \"id\"\n" 44 | "msgstr \"\"\n") 45 | (sut/gen-template [{::scan/filename "file.cljs" 46 | ::scan/expressions [{::scan/value "id" 47 | ::scan/notes ["note 1" "note 2"]}]}])))) 48 | 49 | (testing "Outputting translator of multiple appearances" 50 | (is (= (str "#. note 1\n" 51 | "#. note 2\n" 52 | "#. note 3\n" 53 | "#. note 4\n" 54 | "#: file.cljs file2.cljs\n" 55 | "msgid \"id\"\n" 56 | "msgstr \"\"\n") 57 | (sut/gen-template [{::scan/filename "file.cljs" 58 | ::scan/expressions [{::scan/value "id" 59 | ::scan/notes ["note 1" "note 2"]}]} 60 | {::scan/filename "file2.cljs" 61 | ::scan/expressions [{::scan/value "id" 62 | ::scan/notes ["note 3" "note 4"]}]}])))) 63 | 64 | (testing "Notes with newlines" 65 | (is (= (str "#. note 1\n" 66 | "#. note 2\n" 67 | "#. second line\n" 68 | "#: file.cljs\n" 69 | "msgid \"id\"\n" 70 | "msgstr \"\"\n") 71 | (sut/gen-template [{::scan/filename "file.cljs" 72 | ::scan/expressions [{::scan/value "id" 73 | ::scan/notes ["note 1" "note 2\nsecond line"]}]}]))))) 74 | 75 | (defn read-fixture [name] 76 | (slurp (io/file "test/pottery/_fixtures/" name))) 77 | 78 | (def single-block (read-fixture "single_block.po")) 79 | (def plural-block (read-fixture "plural_block.po")) 80 | (def multi-line-single-block (read-fixture "multiline_single_block.po")) 81 | (def multi-line-plural-block (read-fixture "multiline_plural_block.po")) 82 | 83 | (deftest parse-block-test 84 | (is (= {::sut/msgid "Hello!" 85 | ::sut/msgstr "Hoi!"} 86 | (sut/parse-block single-block))) 87 | 88 | (is (= {::sut/msgid "one mouse" 89 | ::sut/msgid-plural "%1 mice" 90 | ::sut/msgstr "een muis" 91 | ::sut/msgstr-plural "muizen"} 92 | (sut/parse-block plural-block))) 93 | 94 | (is (= {::sut/msgid "First-line\nSecond-line" 95 | ::sut/msgstr "Eerste regel\nTweede regel"} 96 | (sut/parse-block multi-line-single-block))) 97 | 98 | (is (= {::sut/msgid "Some\nLong Message id", 99 | ::sut/msgid-plural "Some\n plural id", 100 | ::sut/msgstr "First\nSecond line [s]", 101 | ::sut/msgstr-plural "First\nSecond Line [p]"} 102 | (sut/parse-block multi-line-plural-block)))) 103 | -------------------------------------------------------------------------------- /src/pottery/scan.clj: -------------------------------------------------------------------------------- 1 | (ns pottery.scan 2 | (:require [pottery.utils :refer [vectorize]] 3 | [clojure.core.match :refer [match]] 4 | [clojure.java.io :as io] 5 | [edamame.core :as e] 6 | [clojure.string :as str]) 7 | (:refer-clojure :exclude [*file*])) 8 | 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | ;; Files 11 | 12 | (defn- source-file? [file] 13 | (and (.isFile file) 14 | (some #(re-find (re-pattern (str "." % "$")) (.getName file)) 15 | ["clj" "cljc" "cljs"]))) 16 | 17 | (defn- get-files [dir] 18 | (filter source-file? (file-seq (io/file dir)))) 19 | 20 | (defn- parse-string-all [s ext opts] 21 | (let [features (if (= :cljc ext) 22 | (:features opts) 23 | #{ext})] 24 | (distinct 25 | (mapcat 26 | (fn [feature] 27 | (e/parse-string-all s 28 | (merge {:all true 29 | :syntax-quote {:resolve-symbol symbol} 30 | :readers (fn [sym] 31 | (or (get *data-readers* sym) 32 | (get default-data-readers sym) 33 | (when-let [dr *default-data-reader-fn*] 34 | (dr sym)) 35 | identity)) 36 | :read-cond :allow 37 | :regex #(list `re-pattern %) 38 | :features #{feature} 39 | :end-location false 40 | :row-key :line 41 | :col-key :column 42 | :auto-resolve symbol} 43 | (dissoc opts :features)))) 44 | features)))) 45 | 46 | (defn extension [file] 47 | (some-> 48 | (str file) 49 | (str/split #"\.") 50 | last 51 | keyword)) 52 | 53 | (defn- read-file [file opts] 54 | {::filename (io/as-relative-path file) 55 | ::expressions 56 | (doall (parse-string-all (slurp file) (extension file) opts))}) 57 | 58 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 59 | ;; Extraction 60 | 61 | (defmacro make-extractor [& match-patterns] 62 | `(fn extract-fn# [expr#] 63 | (match (vec expr#) 64 | ~@match-patterns 65 | :else nil))) 66 | 67 | (defn extraction-warning [msg] 68 | {::warning msg}) 69 | 70 | (def default-extractor 71 | (make-extractor 72 | ['tr (s :guard string?) & _] s 73 | ['trn [(s1 :guard string?) (s2 :guard string?) & _] _] [s1 s2] 74 | [(:or 'tr 'trn) & _] (extraction-warning 75 | "Could not extrapolate translation string for the form:"))) 76 | 77 | (defn- with-comment [expression text] 78 | (if-let [notes (:notes (meta expression))] 79 | {::value text ::notes (vectorize notes)} 80 | {::value text})) 81 | 82 | (defn extract 83 | "Extracts strings from either 84 | `(tr i18n [\"Hello!\" arg])` or 85 | `(tr [\"Hello!\" arg])` or 86 | 87 | and the plural forms; 88 | 89 | `(trn i18n [\"One item\" \"%1 items\"] n)` or 90 | `(trn [\"One item\" \"%1 items\"] n)`" 91 | ([extract-fn expr] 92 | (extract nil extract-fn expr)) 93 | ([file extract-fn expr] 94 | (when-let [val (and (seq? expr) (extract-fn expr))] 95 | (if-let [warning (::warning val)] 96 | (println warning expr (str file) (meta expr)) 97 | (with-comment expr val))))) 98 | 99 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 100 | ;; Scanning 101 | 102 | (defn- find-tr-strings* [file extract-fn expressions] 103 | (->> (map #(extract file extract-fn %) (tree-seq coll? identity expressions)) 104 | (remove nil?) 105 | distinct)) 106 | 107 | (defn find-tr-strings 108 | [extract-fn expr-by-file] 109 | (update expr-by-file ::expressions #(find-tr-strings* (::filename expr-by-file) extract-fn %))) 110 | 111 | (defn scan-files 112 | "Walk the given directory and for every clj, cljc or cljs file 113 | extract the strings for which the extractor returns a value. " 114 | [{:keys [dir extract-fn features] 115 | :or {features #{:clj :cljs}}}] 116 | (println "Scanning files...") 117 | (->> 118 | (get-files (java.io.File. dir)) 119 | (map #(read-file % {:features features})) 120 | (map #(find-tr-strings extract-fn %)) 121 | (filter (comp seq ::expressions)) 122 | (sort-by ::filename))) 123 | 124 | ;;;; Scratch 125 | 126 | (comment 127 | (parse-string-all "#js [1 2 3] #inst \"2004\"" {:features #{:clj :cljs}}) 128 | (scan-files {:dir "test-resources" 129 | :extract-fn default-extractor}) 130 | 131 | (find-tr-strings {::filename "foo.clj" 132 | ::expressions '[(tr "dude")]} 133 | default-extractor) 134 | ) 135 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pottery 2 | 3 | [![Clojars Project](https://img.shields.io/clojars/v/brightin/pottery.svg)](https://clojars.org/brightin/pottery) 4 | 5 | A library to use [GNU Gettext](https://www.gnu.org/software/gettext/) as translation solution for clojure projects. This library is meant to extract translatable strings from your codebase (clj, cljs and cljc), generate a PO Template file and parsing PO (translation) files. 6 | 7 | ## Why Gettext 8 | 9 | Gettext is an old but great solution for managing translatable codebases. In comparison to the i18n or similar ways: 10 | 11 | - No need to invent unique keys anymore for every piece of text that needs translations. The string itself is the key. 12 | - Codebase and translation files stay in sync via tooling, not developers. Since the string is the key, if the string changes, translations must follow. 13 | - Defered development time vs translation time. When developing you don't want to have to manage localisation files. Gettext defers this process to a more appropriate time, like before releases for example. The developer just has to write text in the codebase in whatever language the team is comfortable with. 14 | - Outsource translations easily to professional translators. PO and POT files are an industry standard for translation bureaus. 15 | 16 | ## Why make this library 17 | 18 | There are some libraries in the wild that do some parts of what Pottery tries to achieve. However most of them either had an incomplete API, or not a functional one. Pottery is a functional approach to using gettext, leaving the developer in the front seat when translating his or her application. 19 | 20 | ## Installation 21 | 22 | Add this dependency to your project: 23 | 24 | ```clojure 25 | [brightin/pottery "0.0.4"] 26 | ``` 27 | 28 | And require it: 29 | 30 | ```clojure 31 | (require '[pottery.core :as pottery]) 32 | ``` 33 | 34 | ## Basic usage 35 | 36 | ### 0. Define your translation function 37 | 38 | This library is only meant to generate and read PO files. Translating in and of itself is up to you. There are plenty of great libraries out there for this with which Pottery has no intention to compete. There is a [full example](https://github.com/brightin/pottery/tree/master/examples/simple/example.clj) to demonstrate how this could work. 39 | 40 | ```clojure 41 | (defn tr [s & args] 42 | (translate-string-to-current-language s args)) 43 | ``` 44 | 45 | Throughout your code this `tr` function will be used, and now Pottery will manage the gettext part. 46 | 47 | ### 1. Scan your codebase 48 | 49 | Call the scan function from the REPL: 50 | 51 | ```clojure 52 | (pottery/scan-codebase!) 53 | ``` 54 | 55 | This will scan the codebase for any translatable strings and write the PO template file to `resources/gettext/template.pot`. This is not limited to clojure only files, strings in .cljc and .cljs files are also extracted. 56 | 57 | See [Configuration / Extra features](#configuration--extra-features) for more configurations of the scanner. 58 | 59 | ### 2. Use tools to generate translation files 60 | 61 | For every string in the codebase there needs to be a translation to other languages. These get written in `.po` files. There are various tools available to achieve this. The most popular and recomended one would the free GUI tool [Poedit](https://poedit.net/). Use Poedit to load the PO template file, generate the translation files and interactively translate all the strings. 62 | 63 | ![Poedit](/github/poedit.png?raw=true "Poedit GUI") 64 | 65 | ### 3. Parse the translation files 66 | 67 | Pottery parses the translation files from step 2 into a dictionary with the form: `{"Original String" "Translated string"}`. 68 | 69 | ```clojure 70 | (pottery/read-po-file (io/resource "gettext/es.po")) 71 | => {"Hello" "Hola"} 72 | ``` 73 | 74 | These can act as dictionaries for your translation function in step 0. 75 | 76 | ### 4. Repeat! 77 | 78 | Whenever strings change in the codebase, re-scan at will as in step 1. This will replace the old template file, ensuring changed strings are changed, and removed strings are removed. Next merge that new template file in the translation files. In Poedit you can achieve this by going to _Catalog -> Update from POT file_. After saving in Poedit you can also purge deleted strings from the po file via _Catalog -> Purge Deleted Translations_. You translation files are now in sync! 79 | 80 | ## Gettext features 81 | 82 | This library supports most features that gettext supports. More will be added at request, feel free to create an issue for them. The following examples use the default scanner: 83 | 84 | ### Simple string 85 | 86 | In your code: 87 | 88 | ```clojure 89 | (tr "Hello %s" name) 90 | ``` 91 | 92 | The output after parsing a PO file: 93 | 94 | ```clojure 95 | {"Hello %s" "Hoi %s") 96 | ``` 97 | 98 | ### Pluralisation 99 | 100 | In your code: 101 | 102 | ```clojure 103 | (trn ["One horse" "%s horses"] horse-count) 104 | ``` 105 | 106 | The output after parsing a PO file: 107 | 108 | ```clojure 109 | {["One horse" "%s horses"] ["Een paard" "%s paarden"]} 110 | ``` 111 | 112 | ### Context 113 | 114 | _Not implemented yet_ 115 | 116 | Sometimes the same string can have multiple translations according to context. Gettext has support for this as a translation context. This has not been implemented yet, please submit an issue if the need arises. 117 | 118 | ### Translator notes 119 | 120 | Translator notes are great to provide context for the string to be translated. Since we defer development time and translation time, there might be confusion due to the lack of context. You can add notes to strings as metadata: 121 | 122 | ```clojure 123 | ^{:notes "Abbreviation of horsepower, not healthpoints"} 124 | (tr "Hp") 125 | ``` 126 | 127 | These translations will be extracted and added to the template file. At translation time these will be visible in, for example, Poedit: 128 | 129 | ![Poedit translator notes](/github/poedit_translator_notes.png?raw=true "Poedit translator notes") 130 | 131 | ## Configuration / Extra features 132 | 133 | ### 1. Scanning 134 | 135 | The scan function takes a few options: 136 | 137 | | Key | Description | Default | 138 | | :------------- | :--------------------------------------------------------------------------- | ---------------------------------- | 139 | | :dir | The source directory which needs to be scanned | `"src"` | 140 | | :template-file | The output template file | `"resources/gettext/template.pot"` | 141 | | :extract-fn | The function which maps any expression found to a string, strings or nothing | `pottery.scan/default-extractor` | 142 | 143 | #### extract-fn 144 | 145 | A function which takes a clojure expression as data, and returns a string (single) or a vector of strings (plural). 146 | 147 | A simple extractor would look like this: 148 | 149 | ```clojure 150 | (defn my-extract-fn [expr] 151 | (when (and (list? expr) 152 | (= 'tr (first expr)) 153 | (string? (second expr))) 154 | (second expr))) 155 | 156 | (my-extract-fn '(tr "Some string")) => "Some string" 157 | (my-extract-fn '(inc 12)) => nil 158 | ``` 159 | 160 | It may get tedious to write a good function when you have multiple translation functions that have multiple arities. Pottery offers a shorthand using [core.match](https://github.com/clojure/core.match) to declare patterns in which translation functions are called. 161 | 162 | ```clojure 163 | (pottery/make-extractor 164 | ['tr (s :guard string?) & _] s 165 | ['tr [s & _]] s 166 | ['trn [s1 s2 & _]] [s1 s2] 167 | ...) 168 | ``` 169 | 170 | It's a good idea to also warn when extraction did not pass any of the patterns, as a safe guard. As last pattern of the match sequence, you can provide: 171 | 172 | ```clojure 173 | (pottery/make-extractor 174 | ... patterns 175 | [(:or 'tr 'trn) & _] (pottery.scan/extraction-warning 176 | "Translation function called but no string could be extracted:")) 177 | ``` 178 | 179 | When the expression starts with the familiar function call, but did not match any pattern the warning will be printed with the failing expression. It's a good idea to write patterns for common "mistakes". 180 | 181 | The default extractor is defined as such: 182 | 183 | ```clojure 184 | (make-extractor 185 | ['tr (s :guard string?) & _] s 186 | ['trn [(s1 :guard string?) (s2 :guard string?) & _] _] [s1 s2] 187 | [(:or 'tr 'trn) & _] (extraction-warning 188 | "Could not extrapolate translation string for the form:")) 189 | ``` 190 | 191 | It's a minimal extractor, which will match these clojure forms: 192 | 193 | ```clojure 194 | (tr "My string" & args) => "My string" 195 | (trn ["Singular" "Plural" & args] count) => ["Singular" "Plural"] 196 | (tr some-var) => nil ;; And a warning is printed 197 | (trn ["String"]) => nil ;; And a warning is printed 198 | ``` 199 | 200 | See [the full example](https://github.com/brightin/pottery/tree/master/examples/simple/example.clj) for a simple but complete example of integrating Pottery in your application. 201 | 202 | ## Clojurescript usage example 203 | 204 | This next example shows an approach for compiling dictionaries into the 205 | clojurescript source. We use a macro to read the po file and compile the 206 | translation dictionary at compile time, which will result in the dictionary 207 | being hardcoded in the source. 208 | 209 | ```clojure 210 | ;; src/app/tr.clj 211 | (ns app.tr 212 | (:require [pottery.core :as pottery])) 213 | 214 | (defmacro inline-dict [filename] 215 | (pottery/read-po-file filename)) 216 | 217 | ;; src/app/frontend/tr.cljs 218 | (ns app.frontend.tr 219 | (:require-macros [app.tr :refer [inline-dict]])) 220 | 221 | (def DICT 222 | {:en (inline-dict "gettext/en.po") 223 | :it (inline-dict "gettext/it.po")}) 224 | ``` 225 | 226 | ### Automatic recompilation with shadow-cljs 227 | 228 | If you're using [shadow-cljs](https://github.com/thheller/shadow-cljs) you can 229 | leverage `shadow.resource/slurp-resource` to read the contents of the PO file 230 | and also record the file as a recompilation dependency. 231 | 232 | ```clojure 233 | ;; src/app/tr.clj 234 | (ns app.tr 235 | (:require [pottery.core :as pottery] 236 | [shadow.resource :as res])) 237 | 238 | (defmacro inline-dict [filename] 239 | (pottery/read-po-str (res/slurp-resource &env filename))) 240 | ``` 241 | 242 | ## Gotchas 243 | 244 | With Pottery, translation is done as a function but the function call should be regarded as data. The scanner reads source code with the clojure reader to figure out which strings are to be translated. For example: 245 | 246 | ```clojure 247 | ;; Bad 248 | (let [msg (if available? "Available" "Unavailable")] 249 | (tr msg)) 250 | 251 | ;; Good 252 | (if available? 253 | (tr "Available") 254 | (tr "Unavailable")) 255 | ``` 256 | 257 | In the first case only "msg" or nothing would get extracted. In the second case all strings are extracted. 258 | 259 | ## Credits 260 | 261 | Thanks to [Carlo Sciolla](https://github.com/skuro) for coming up with the name of this project! 262 | 263 | ## License 264 | 265 | Copyright © 2025 Brightin 266 | 267 | This project uses the [Hippocratic License](https://firstdonoharm.dev/), and is 268 | thus freely available to use for purposes that do not infringe on the United 269 | Nations Universal Declaration of Human Rights. 270 | --------------------------------------------------------------------------------