├── version.lisp-expr ├── package.lisp ├── README.md ├── weblocks-cms-import-export-data.asd ├── weblocks-cms-import-export-data.lisp ├── serialization-testing.lisp ├── serialization.lisp ├── import-export-data-ui.lisp └── license.txt /version.lisp-expr: -------------------------------------------------------------------------------- 1 | "0.3.3" 2 | -------------------------------------------------------------------------------- /package.lisp: -------------------------------------------------------------------------------- 1 | ;;;; package.lisp 2 | 3 | (defpackage #:weblocks-cms-import-export-data 4 | (:use #:cl) 5 | (:export #:import-export-data-ui) 6 | (:import-from :weblocks #:lambda/cc) 7 | (:import-from :weblocks-utils #:all-of #:first-by-values)) 8 | 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Weblocks CMS Import/Export Data plugin 2 | 3 | Contains import/export functionality for Weblocks CMS. 4 | You can import/export any separate model data and all models data at once. 5 | Include package and new button should appear in standard Weblocks CMS admin interface. 6 | 7 | ## Compatibility 8 | 9 | Currently Weblocks CMS Import/Export Data plugin should work ok with latest Weblocks and Weblocks CMS, though it was tested using weblocks-routes branch of https://github.com/html/weblocks repository 10 | -------------------------------------------------------------------------------- /weblocks-cms-import-export-data.asd: -------------------------------------------------------------------------------- 1 | ;;;; weblocks-cms-import-export-data.asd 2 | 3 | (asdf:defsystem #:weblocks-cms-import-export-data 4 | :serial t 5 | :description "Import/export separate model data, all models data" 6 | :author "Olexiy Zamkoviy " 7 | :version (:read-from-file "version.lisp-expr") 8 | :license "LLGPL" 9 | :depends-on (#:weblocks 10 | #:weblocks-cms 11 | #:weblocks-prevalence 12 | #:weblocks-stores 13 | #:weblocks-utils 14 | #:closer-mop 15 | #:cl-json 16 | #:clos-diff) 17 | :components 18 | ((:file "package") 19 | (:file "serialization") 20 | (:file "serialization-testing") 21 | (:file "weblocks-cms-import-export-data") 22 | (:file "import-export-data-ui"))) 23 | 24 | -------------------------------------------------------------------------------- /weblocks-cms-import-export-data.lisp: -------------------------------------------------------------------------------- 1 | ;;;; weblocks-cms-import-export-data.lisp 2 | 3 | (in-package #:weblocks-cms-import-export-data) 4 | 5 | ;;; "weblocks-cms-import-export-data" goes here. Hacks and glory await! 6 | 7 | (defmacro with-yaclml (&body body) 8 | "A wrapper around cl-yaclml with-yaclml-stream macro." 9 | `(yaclml:with-yaclml-stream weblocks:*weblocks-output-stream* 10 | ,@body)) 11 | 12 | (defun get-store-type (store) 13 | (if (equal "HU.DWIM.PEREC" (package-name (symbol-package (type-of store))) ) 14 | (return-from get-store-type :perec)) 15 | 16 | (loop for i in (weblocks-stores:list-store-types) do 17 | (when (find-symbol (string-upcase (type-of store)) 18 | (find-package (alexandria:make-keyword (format nil "WEBLOCKS-~A" (string-upcase i))))) 19 | (return-from get-store-type i)))) 20 | 21 | (defun write-model-export-data (model s) 22 | (write-string (weblocks-stores:serialize (all-of model) :format :json) s)) 23 | 24 | (defun get-model-export-data (model) 25 | (with-output-to-string (s) 26 | (write-model-export-data model s))) 27 | 28 | (defun get-models-export-data (model-classes) 29 | (with-output-to-string (s) 30 | (json:with-object (s) 31 | (loop for i in model-classes do 32 | (json:as-object-member (i s) 33 | (write-model-export-data i s)))))) 34 | 35 | (defvar *update-meta-deferred* nil) 36 | (defvar *update-meta-callbacks* nil) 37 | 38 | (defun update-model-id-counter (store model) 39 | (let ((max (apply #'max (list* -1 (mapcar #'weblocks-stores:object-id (all-of model :store store)))))) 40 | (ignore-errors (setf (slot-value (weblocks-prevalence::get-root-object store model) 'weblocks-prevalence::next-id) max)))) 41 | 42 | (defun import-model-data (store model json-string &key (update-meta t) (testp nil)) 43 | (unless testp 44 | (weblocks-utils:delete-all model :store store)) 45 | 46 | (let* ((data (json:decode-json-from-string json-string)) 47 | (result-records (loop for i in data collect 48 | (weblocks-stores:unserialize (json:encode-json-alist-to-string i) :format :json)))) 49 | (unless testp 50 | (loop for i in result-records do 51 | (weblocks-stores:persist-object store i))) 52 | 53 | (update-model-id-counter store model) 54 | 55 | result-records)) 56 | 57 | (defun run-update-meta-callbacks () 58 | (declare (special *update-meta-callbacks*)) 59 | (mapcar #'funcall (reverse *update-meta-callbacks*))) 60 | 61 | (defun import-model-data-with-meta-deferred (store model data &key (testp nil)) 62 | (let ((*update-meta-callbacks* nil) 63 | (*update-meta-deferred* t)) 64 | (declare (special *update-meta-callbacks*)) 65 | (import-model-data store model data :update-meta nil :testp testp) 66 | (run-update-meta-callbacks))) 67 | 68 | (defun import-models-data (store data &optional data-package) 69 | "Imports all data specified for store." 70 | (unless data-package 71 | (setf data-package (get-store-data-package store))) 72 | (let ((*update-meta-callbacks* nil)) 73 | (declare (special *update-meta-callbacks*)) 74 | 75 | (loop for (key . value) in data do 76 | (import-model-data store (intern (string key) data-package) 77 | value 78 | :update-meta nil)) 79 | 80 | (run-update-meta-callbacks))) 81 | 82 | (defun get-store-data-package (store) 83 | (let ((packages (remove-duplicates (mapcar #'symbol-package (weblocks-stores:list-model-classes store))))) 84 | (when (second packages) 85 | (error "There are several packages possible for store ~A" store)) 86 | (intern (package-name (first packages))))) 87 | 88 | -------------------------------------------------------------------------------- /serialization-testing.lisp: -------------------------------------------------------------------------------- 1 | (in-package :weblocks-cms-import-export-data) 2 | 3 | (defun reserialized-copy (object) 4 | "Serializes object and than unserializes it" 5 | (ignore-errors 6 | (weblocks-stores:unserialize 7 | (weblocks-stores:serialize object :format :json) 8 | :format :json))) 9 | 10 | (defun equalp-improved (obj1 obj2) 11 | (cond 12 | ((typep obj1 'standard-object) 13 | (equal (weblocks-stores:object-id obj1) (weblocks-stores:object-id obj2))) 14 | ((consp obj1) 15 | (progn 16 | (and 17 | (equalp-improved (car obj1) (car obj2)) 18 | (equalp-improved (cdr obj1) (cdr obj2))))) 19 | ((hash-table-p obj1) 20 | (equalp-improved 21 | (sort (alexandria:hash-table-alist obj1) #'string> 22 | :key #'car) 23 | (sort (alexandria:hash-table-alist obj2) #'string> 24 | :key #'car))) 25 | (t (equal obj1 obj2)))) 26 | 27 | 28 | (defun serialization-error-p (object) 29 | "T if reserialized copy is not equal to original" 30 | (flet ((objects-differ-p (obj1 obj2) 31 | (cdr (clos-diff:diff obj1 obj2 :test #'equalp-improved)))) 32 | (objects-differ-p 33 | (reserialized-copy object) 34 | object))) 35 | 36 | (defun smart-describe-to-string (obj) 37 | (with-output-to-string (s) 38 | (cond 39 | ((hash-table-p obj) 40 | (format s "~A" (sort (alexandria:hash-table-alist obj) #'string> :key #'car))) 41 | (t (describe obj s))))) 42 | 43 | (defun print-objects-diff (obj1 obj2) 44 | (if (equal (type-of obj1) (type-of obj2)) 45 | (let ((difference (cdr (clos-diff:diff obj1 obj2 :test #'equalp-improved)))) 46 | (when difference 47 | (loop for (dummy slot value) in difference do 48 | (format t "Slot ~A is different~%~%Object 1 value - ~%~%~A~%Object 2 value - ~%~%~A~%~%" 49 | (prin1-to-string slot) 50 | (smart-describe-to-string (slot-value obj1 slot)) 51 | (smart-describe-to-string (slot-value obj2 slot)))))) 52 | (format t "Objects types not equal - ~A ~A~%" (type-of obj1) (type-of obj2)))) 53 | 54 | (defun print-records-serialized-with-errors (model) 55 | "Reserializes all records for specific `model`, if any reserialized record is different from original, shows a message" 56 | (loop for i in (all-of model) 57 | if (serialization-error-p i) 58 | do 59 | (describe i) 60 | (print-objects-diff i (reserialized-copy i)))) 61 | 62 | (defun model-reserialization-ok-p (model) 63 | "Reserializes all records for specific `model`, returns true if all records reserialized correctly, nil in other case" 64 | 65 | (loop for i in (all-of model) 66 | if (serialization-error-p i) 67 | do 68 | (return-from model-reserialization-ok-p nil)) 69 | 70 | t) 71 | 72 | (defun store-reserialization-ok-p (store) 73 | "Reserializes all records for all models of store. If all records reserialized correctly, returns true, nil in other case" 74 | (loop for model in (weblocks-stores:list-model-classes store) 75 | if (not (model-reserialization-ok-p model)) 76 | do (return-from store-reserialization-ok-p nil)) 77 | t) 78 | 79 | (defun all-stores-reserialization-ok-p () 80 | "Reserializes all records in all models of all stores. If all records reserialized correctly, returns true, nil in other case" 81 | (loop for i in weblocks-stores:*store-names* 82 | if (not (store-reserialization-ok-p (symbol-value i))) 83 | do 84 | (return-from all-stores-reserialization-ok-p nil)) 85 | t) 86 | 87 | (defun records-with-reserialization-errors (model) 88 | "Reserializes all records for specific `model`, if any reserialized record is different from original, shows a message" 89 | (loop for i in (all-of model) 90 | if (serialization-error-p i) 91 | collect i)) 92 | -------------------------------------------------------------------------------- /serialization.lisp: -------------------------------------------------------------------------------- 1 | (in-package :weblocks-cms-import-export-data) 2 | 3 | (defgeneric serialization-link-to-data-object (object) 4 | (:method ((obj standard-object)) 5 | `(:eval 6 | (first-by-id ',(type-of obj) 7 | ,(weblocks-stores:object-id obj))))) 8 | 9 | (defmethod weblocks-stores::serialize-impl (obj &key (format (eql :json))) 10 | (labels ((lisp-code (value inside-lisp-code-p) 11 | (if inside-lisp-code-p 12 | value 13 | (format nil "#+lisp~A" (prin1-to-string value)))) 14 | (serialize-value (value &optional inside-lisp-code-p) 15 | (cond 16 | ((null value) 17 | nil) 18 | ((typep value 'standard-object) 19 | (lisp-code 20 | (serialization-link-to-data-object value) 21 | inside-lisp-code-p)) 22 | ((typep value 'hash-table) 23 | (lisp-code 24 | `(:eval 25 | (alexandria:alist-hash-table 26 | ',(loop for (key . val) in (alexandria:hash-table-alist value) 27 | collect 28 | (cons (serialize-value key t) (serialize-value val t))))) 29 | inside-lisp-code-p)) 30 | ((consp value) 31 | (lisp-code 32 | (cons 33 | (serialize-value (car value) t) 34 | (serialize-value (cdr value) t)) 35 | inside-lisp-code-p)) 36 | ((stringp value) 37 | value) 38 | (t (lisp-code value inside-lisp-code-p))))) 39 | (let ((model (type-of obj))) 40 | (json:with-object () 41 | (json:encode-object-member :cl-type (prin1-to-string model)) 42 | (loop for j in (weblocks-stores:class-visible-slots model) 43 | ;if (equal (c2mop:slot-definition-name j) 'weblocks-cms::social-networks) 44 | do 45 | (let ((value (slot-value obj (c2mop:slot-definition-name j)))) 46 | (json:encode-object-member 47 | (alexandria:make-keyword (c2mop:slot-definition-name j)) 48 | (serialize-value value)))))))) 49 | 50 | (defmethod weblocks-stores:serialize ((obj standard-object) &key (format (eql :json))) 51 | (with-output-to-string (json:*json-output*) 52 | (weblocks-stores::serialize-impl obj :format format))) 53 | 54 | (defmethod weblocks-stores:serialize ((list list) &key (format (eql :json))) 55 | (with-output-to-string (json:*json-output*) 56 | (json:with-array 57 | (json:*json-output*) 58 | (loop for i in list 59 | do 60 | (json:as-array-member (json:*json-output*) 61 | (format json:*json-output* "~A" (weblocks-stores:serialize i :format :json))))))) 62 | 63 | (defun first-by-id (model id) 64 | (first-by-values model :id id)) 65 | 66 | (defmethod weblocks-stores:unserialize (obj &key (format (eql :json))) 67 | (let* ((item-data (json:decode-json-from-string obj)) 68 | (model (read-from-string (cdr (assoc :cl-type item-data)))) 69 | (item (make-instance model)) 70 | (slot-name)) 71 | 72 | (loop for i in (weblocks-stores:class-visible-slots model) 73 | do 74 | (setf slot-name (c2mop:slot-definition-name i)) 75 | (setf (slot-value item slot-name) 76 | (cdr (assoc (alexandria:make-keyword slot-name) item-data)))) 77 | 78 | (labels ((unserialize-value (item slot-name slot-value) 79 | (cond 80 | ((and (stringp slot-value) (string= "f" slot-value)) 81 | t) 82 | ((stringp slot-value) 83 | (ppcre:register-groups-bind 84 | (data ) 85 | ((ppcre:create-scanner "#\\+lisp(.*)$" :single-line-mode t) slot-value) 86 | (setf data (read-from-string data)) 87 | (when (and (consp data) (equal :eval (car data))) 88 | (if *update-meta-deferred* 89 | (progn 90 | (push 91 | (lambda () 92 | (setf (slot-value item slot-name) 93 | (eval (second data)))) 94 | *update-meta-callbacks*) 95 | (return-from unserialize-value :deferred-value)) 96 | (setf data (eval (second data))))) 97 | 98 | (return-from unserialize-value data)) 99 | slot-value) 100 | (t slot-value)))) 101 | (flet ((update-meta () 102 | (loop for i in (weblocks-stores:class-visible-slots model) 103 | do 104 | (let* ((slot-name (c2mop:slot-definition-name i)) 105 | (slot-value (slot-value item slot-name))) 106 | (setf (slot-value item slot-name) 107 | (unserialize-value item slot-name slot-value)))))) 108 | 109 | (update-meta))) 110 | 111 | item)) 112 | 113 | -------------------------------------------------------------------------------- /import-export-data-ui.lisp: -------------------------------------------------------------------------------- 1 | (in-package :weblocks-cms-import-export-data) 2 | 3 | (defun import-callback (test-callback callback caption) 4 | (ps:ps-inline* 5 | `(initiate-action 6 | ,(weblocks:make-action 7 | (lambda (&rest args) 8 | (weblocks:do-page 9 | (weblocks:make-quickform 10 | (weblocks:defview nil (:type weblocks:form :persistp nil :caption caption) 11 | (data :requiredp t 12 | :present-as textarea)) 13 | :satisfies (lambda (form data) 14 | (let ((ret (handler-case 15 | (progn 16 | (funcall test-callback (slot-value data 'data)) 17 | t) 18 | (t (var) (with-output-to-string (s) 19 | (let ((*print-escape* nil)) 20 | (print-object var s))))))) 21 | (if (equal ret t) 22 | t 23 | (values nil `((data . ,(format nil "Importing data failed - ~A" ret))))))) 24 | :on-success (lambda/cc (form data) 25 | (progn 26 | (funcall callback (slot-value data 'data)) 27 | (weblocks:do-information "Import finished") 28 | (weblocks:answer form t))) 29 | :on-cancel (lambda (form) 30 | (weblocks:answer form t)) 31 | :answerp nil)))) 32 | ,(weblocks:session-name-string-pair)))) 33 | 34 | (defun stores-check-html(continuation) 35 | (with-yaclml 36 | (if (all-stores-reserialization-ok-p) 37 | (<:as-is "All stores are ok !") 38 | (