├── .github └── workflows │ └── ci.yml ├── .gitignore ├── MANIFEST.in ├── README.md ├── event_streaming ├── __init__.py ├── config │ ├── __init__.py │ ├── desktop.py │ └── docs.py ├── event_streaming │ ├── __init__.py │ └── doctype │ │ ├── __init__.py │ │ ├── document_type_field_mapping │ │ ├── __init__.py │ │ ├── document_type_field_mapping.json │ │ └── document_type_field_mapping.py │ │ ├── document_type_mapping │ │ ├── __init__.py │ │ ├── document_type_mapping.js │ │ ├── document_type_mapping.json │ │ ├── document_type_mapping.py │ │ └── test_document_type_mapping.py │ │ ├── event_consumer │ │ ├── __init__.py │ │ ├── event_consumer.js │ │ ├── event_consumer.json │ │ ├── event_consumer.py │ │ └── test_event_consumer.py │ │ ├── event_consumer_document_type │ │ ├── __init__.py │ │ ├── event_consumer_document_type.json │ │ └── event_consumer_document_type.py │ │ ├── event_producer │ │ ├── __init__.py │ │ ├── event_producer.js │ │ ├── event_producer.json │ │ ├── event_producer.py │ │ └── test_event_producer.py │ │ ├── event_producer_document_type │ │ ├── __init__.py │ │ ├── event_producer_document_type.json │ │ └── event_producer_document_type.py │ │ ├── event_producer_last_update │ │ ├── __init__.py │ │ ├── event_producer_last_update.js │ │ ├── event_producer_last_update.json │ │ ├── event_producer_last_update.py │ │ └── test_event_producer_last_update.py │ │ ├── event_sync_log │ │ ├── __init__.py │ │ ├── event_sync_log.js │ │ ├── event_sync_log.json │ │ ├── event_sync_log.py │ │ ├── event_sync_log_list.js │ │ └── test_event_sync_log.py │ │ ├── event_update_log │ │ ├── __init__.py │ │ ├── event_update_log.js │ │ ├── event_update_log.json │ │ ├── event_update_log.py │ │ └── test_event_update_log.py │ │ └── event_update_log_consumer │ │ ├── __init__.py │ │ ├── event_update_log_consumer.json │ │ └── event_update_log_consumer.py ├── hooks.py ├── modules.txt ├── patches.txt ├── public │ └── .gitkeep └── templates │ ├── __init__.py │ └── pages │ └── __init__.py ├── license.txt ├── requirements.txt └── setup.py /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | 2 | name: CI 3 | 4 | on: 5 | push: 6 | branches: 7 | - develop 8 | pull_request: 9 | 10 | concurrency: 11 | group: develop-event_streaming-${{ github.event.number }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | tests: 16 | runs-on: ubuntu-latest 17 | strategy: 18 | fail-fast: false 19 | name: Server 20 | 21 | services: 22 | mariadb: 23 | image: mariadb:10.6 24 | env: 25 | MYSQL_ROOT_PASSWORD: root 26 | ports: 27 | - 3306:3306 28 | options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 29 | 30 | steps: 31 | - name: Clone 32 | uses: actions/checkout@v2 33 | 34 | - name: Setup Python 35 | uses: actions/setup-python@v2 36 | with: 37 | python-version: '3.10' 38 | 39 | - name: Setup Node 40 | uses: actions/setup-node@v2 41 | with: 42 | node-version: 18 43 | check-latest: true 44 | 45 | - name: Cache pip 46 | uses: actions/cache@v2 47 | with: 48 | path: ~/.cache/pip 49 | key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py', '**/setup.cfg') }} 50 | restore-keys: | 51 | ${{ runner.os }}-pip- 52 | ${{ runner.os }}- 53 | 54 | - name: Get yarn cache directory path 55 | id: yarn-cache-dir-path 56 | run: 'echo "::set-output name=dir::$(yarn cache dir)"' 57 | 58 | - uses: actions/cache@v2 59 | id: yarn-cache 60 | with: 61 | path: ${{ steps.yarn-cache-dir-path.outputs.dir }} 62 | key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} 63 | restore-keys: | 64 | ${{ runner.os }}-yarn- 65 | 66 | - name: Setup 67 | run: | 68 | pip install frappe-bench 69 | bench init --skip-redis-config-generation --skip-assets --python "$(which python)" ~/frappe-bench 70 | mysql --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'" 71 | mysql --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'" 72 | 73 | - name: Install 74 | working-directory: /home/runner/frappe-bench 75 | run: | 76 | bench get-app event_streaming $GITHUB_WORKSPACE 77 | bench setup requirements --dev 78 | bench new-site --db-root-password root --admin-password admin test_site 79 | bench --site test_site install-app event_streaming 80 | bench build 81 | env: 82 | CI: 'Yes' 83 | 84 | - name: Run Tests 85 | working-directory: /home/runner/frappe-bench 86 | run: | 87 | bench --site test_site set-config allow_tests true 88 | bench --site test_site run-tests --app event_streaming 89 | env: 90 | TYPE: server 91 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *.egg-info 4 | *.swp 5 | tags 6 | event_streaming/docs/current 7 | node_modules/ -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include MANIFEST.in 2 | include requirements.txt 3 | include *.json 4 | include *.md 5 | include *.py 6 | include *.txt 7 | recursive-include event_streaming *.css 8 | recursive-include event_streaming *.csv 9 | recursive-include event_streaming *.html 10 | recursive-include event_streaming *.ico 11 | recursive-include event_streaming *.js 12 | recursive-include event_streaming *.json 13 | recursive-include event_streaming *.md 14 | recursive-include event_streaming *.png 15 | recursive-include event_streaming *.py 16 | recursive-include event_streaming *.svg 17 | recursive-include event_streaming *.txt 18 | recursive-exclude event_streaming *.pyc -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Event Streaming 2 | 3 | Event Streaming for frappe 4 | 5 | #### License 6 | 7 | MIT -------------------------------------------------------------------------------- /event_streaming/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | __version__ = '0.0.1' 3 | 4 | -------------------------------------------------------------------------------- /event_streaming/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/config/__init__.py -------------------------------------------------------------------------------- /event_streaming/config/desktop.py: -------------------------------------------------------------------------------- 1 | from frappe import _ 2 | 3 | def get_data(): 4 | return [ 5 | { 6 | "module_name": "Event Streaming", 7 | "type": "module", 8 | "label": _("Event Streaming") 9 | } 10 | ] 11 | -------------------------------------------------------------------------------- /event_streaming/config/docs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration for docs 3 | """ 4 | 5 | # source_link = "https://github.com/[org_name]/event_streaming" 6 | # headline = "App that does everything" 7 | # sub_heading = "Yes, you got that right the first time, everything" 8 | 9 | def get_context(context): 10 | context.brand_html = "Event Streaming" 11 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/document_type_field_mapping/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/document_type_field_mapping/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [], 3 | "creation": "2019-09-27 12:46:50.165135", 4 | "doctype": "DocType", 5 | "editable_grid": 1, 6 | "engine": "InnoDB", 7 | "field_order": [ 8 | "local_fieldname", 9 | "mapping_type", 10 | "mapping", 11 | "remote_value_filters", 12 | "column_break_5", 13 | "remote_fieldname", 14 | "default_value" 15 | ], 16 | "fields": [ 17 | { 18 | "fieldname": "remote_fieldname", 19 | "fieldtype": "Data", 20 | "in_list_view": 1, 21 | "label": "Remote Fieldname" 22 | }, 23 | { 24 | "fieldname": "local_fieldname", 25 | "fieldtype": "Data", 26 | "in_list_view": 1, 27 | "label": "Local Fieldname", 28 | "reqd": 1 29 | }, 30 | { 31 | "fieldname": "column_break_5", 32 | "fieldtype": "Column Break" 33 | }, 34 | { 35 | "fieldname": "default_value", 36 | "fieldtype": "Data", 37 | "label": "Default Value" 38 | }, 39 | { 40 | "fieldname": "mapping_type", 41 | "fieldtype": "Select", 42 | "label": "Mapping Type", 43 | "options": "\nChild Table\nDocument" 44 | }, 45 | { 46 | "depends_on": "eval:doc.mapping_type;", 47 | "fieldname": "mapping", 48 | "fieldtype": "Link", 49 | "label": "Mapping", 50 | "options": "Document Type Mapping" 51 | }, 52 | { 53 | "depends_on": "eval:doc.mapping_type==\"Document\";", 54 | "fieldname": "remote_value_filters", 55 | "fieldtype": "Code", 56 | "label": "Remote Value Filters", 57 | "mandatory_depends_on": "eval:doc.mapping_type===\"Document\";", 58 | "options": "JSON" 59 | } 60 | ], 61 | "istable": 1, 62 | "links": [], 63 | "modified": "2020-03-19 13:56:36.223799", 64 | "modified_by": "Administrator", 65 | "module": "Event Streaming", 66 | "name": "Document Type Field Mapping", 67 | "owner": "Administrator", 68 | "permissions": [], 69 | "quick_entry": 1, 70 | "sort_field": "modified", 71 | "sort_order": "DESC", 72 | "track_changes": 1 73 | } -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and contributors 2 | # License: MIT. See LICENSE 3 | 4 | # import frappe 5 | from frappe.model.document import Document 6 | 7 | 8 | class DocumentTypeFieldMapping(Document): 9 | pass 10 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/document_type_mapping/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/document_type_mapping/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/document_type_mapping/document_type_mapping.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019, Frappe Technologies and contributors 2 | // For license information, please see license.txt 3 | 4 | frappe.ui.form.on("Document Type Mapping", { 5 | local_doctype: function (frm) { 6 | if (frm.doc.local_doctype) { 7 | frappe.model.clear_table(frm.doc, "field_mapping"); 8 | let fields = frm.events.get_fields(frm); 9 | $.each(fields, function (i, data) { 10 | let row = frappe.model.add_child( 11 | frm.doc, 12 | "Document Type Field Mapping", 13 | "field_mapping" 14 | ); 15 | row.local_fieldname = data; 16 | }); 17 | refresh_field("field_mapping"); 18 | } 19 | }, 20 | 21 | get_fields: function (frm) { 22 | let filtered_fields = []; 23 | frappe.model.with_doctype(frm.doc.local_doctype, () => { 24 | frappe.get_meta(frm.doc.local_doctype).fields.map((field) => { 25 | if ( 26 | field.fieldname !== "remote_docname" && 27 | field.fieldname !== "remote_site_name" && 28 | frappe.model.is_value_type(field) && 29 | !field.hidden 30 | ) { 31 | filtered_fields.push(field.fieldname); 32 | } 33 | }); 34 | }); 35 | return filtered_fields; 36 | }, 37 | }); 38 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/document_type_mapping/document_type_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "autoname": "field:mapping_name", 3 | "creation": "2019-09-27 12:45:56.529124", 4 | "doctype": "DocType", 5 | "editable_grid": 1, 6 | "engine": "InnoDB", 7 | "field_order": [ 8 | "mapping_name", 9 | "local_doctype", 10 | "remote_doctype", 11 | "section_break_3", 12 | "field_mapping" 13 | ], 14 | "fields": [ 15 | { 16 | "fieldname": "local_doctype", 17 | "fieldtype": "Link", 18 | "in_list_view": 1, 19 | "label": "Local Document Type", 20 | "options": "DocType", 21 | "reqd": 1 22 | }, 23 | { 24 | "fieldname": "remote_doctype", 25 | "fieldtype": "Data", 26 | "in_list_view": 1, 27 | "label": "Remote Document Type", 28 | "reqd": 1 29 | }, 30 | { 31 | "fieldname": "section_break_3", 32 | "fieldtype": "Section Break" 33 | }, 34 | { 35 | "fieldname": "field_mapping", 36 | "fieldtype": "Table", 37 | "label": "Field Mapping", 38 | "options": "Document Type Field Mapping" 39 | }, 40 | { 41 | "fieldname": "mapping_name", 42 | "fieldtype": "Data", 43 | "label": "Mapping Name", 44 | "reqd": 1, 45 | "unique": 1 46 | } 47 | ], 48 | "modified": "2019-10-09 08:36:04.621397", 49 | "modified_by": "Administrator", 50 | "module": "Event Streaming", 51 | "name": "Document Type Mapping", 52 | "owner": "Administrator", 53 | "permissions": [ 54 | { 55 | "create": 1, 56 | "delete": 1, 57 | "email": 1, 58 | "export": 1, 59 | "print": 1, 60 | "read": 1, 61 | "report": 1, 62 | "role": "System Manager", 63 | "share": 1, 64 | "write": 1 65 | } 66 | ], 67 | "quick_entry": 1, 68 | "sort_field": "modified", 69 | "sort_order": "DESC", 70 | "track_changes": 1 71 | } -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/document_type_mapping/document_type_mapping.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and contributors 2 | # License: MIT. See LICENSE 3 | import json 4 | 5 | import frappe 6 | from frappe import _ 7 | from frappe.model import child_table_fields, default_fields 8 | from frappe.model.document import Document 9 | 10 | 11 | class DocumentTypeMapping(Document): 12 | def validate(self): 13 | self.validate_inner_mapping() 14 | 15 | def validate_inner_mapping(self): 16 | meta = frappe.get_meta(self.local_doctype) 17 | for field_map in self.field_mapping: 18 | if field_map.local_fieldname not in (default_fields + child_table_fields): 19 | field = meta.get_field(field_map.local_fieldname) 20 | if not field: 21 | frappe.throw(_("Row #{0}: Invalid Local Fieldname").format(field_map.idx)) 22 | 23 | fieldtype = field.get("fieldtype") 24 | if fieldtype in ["Link", "Dynamic Link", "Table"]: 25 | if not field_map.mapping and not field_map.default_value: 26 | msg = _( 27 | "Row #{0}: Please set Mapping or Default Value for the field {1} since its a dependency field" 28 | ).format(field_map.idx, frappe.bold(field_map.local_fieldname)) 29 | frappe.throw(msg, title="Inner Mapping Missing") 30 | 31 | if field_map.mapping_type == "Document" and not field_map.remote_value_filters: 32 | msg = _( 33 | "Row #{0}: Please set remote value filters for the field {1} to fetch the unique remote dependency document" 34 | ).format(field_map.idx, frappe.bold(field_map.remote_fieldname)) 35 | frappe.throw(msg, title="Remote Value Filters Missing") 36 | 37 | def get_mapping(self, doc, producer_site, update_type): 38 | remote_fields = [] 39 | # list of tuples (local_fieldname, dependent_doc) 40 | dependencies = [] 41 | 42 | for mapping in self.field_mapping: 43 | if doc.get(mapping.remote_fieldname): 44 | if mapping.mapping_type == "Document": 45 | if not mapping.default_value: 46 | dependency = self.get_mapped_dependency(mapping, producer_site, doc) 47 | if dependency: 48 | dependencies.append((mapping.local_fieldname, dependency)) 49 | else: 50 | doc[mapping.local_fieldname] = mapping.default_value 51 | 52 | if mapping.mapping_type == "Child Table" and update_type != "Update": 53 | doc[mapping.local_fieldname] = get_mapped_child_table_docs( 54 | mapping.mapping, doc[mapping.remote_fieldname], producer_site 55 | ) 56 | else: 57 | # copy value into local fieldname key and remove remote fieldname key 58 | doc[mapping.local_fieldname] = doc[mapping.remote_fieldname] 59 | 60 | if mapping.local_fieldname != mapping.remote_fieldname: 61 | remote_fields.append(mapping.remote_fieldname) 62 | 63 | if not doc.get(mapping.remote_fieldname) and mapping.default_value and update_type != "Update": 64 | doc[mapping.local_fieldname] = mapping.default_value 65 | 66 | # remove the remote fieldnames 67 | for field in remote_fields: 68 | doc.pop(field, None) 69 | 70 | if update_type != "Update": 71 | doc["doctype"] = self.local_doctype 72 | 73 | mapping = {"doc": frappe.as_json(doc)} 74 | if len(dependencies): 75 | mapping["dependencies"] = dependencies 76 | return mapping 77 | 78 | def get_mapped_update(self, update, producer_site): 79 | update_diff = frappe._dict(json.loads(update.data)) 80 | mapping = update_diff 81 | dependencies = [] 82 | if update_diff.changed: 83 | doc_map = self.get_mapping(update_diff.changed, producer_site, "Update") 84 | mapped_doc = doc_map.get("doc") 85 | mapping.changed = json.loads(mapped_doc) 86 | if doc_map.get("dependencies"): 87 | dependencies += doc_map.get("dependencies") 88 | 89 | if update_diff.removed: 90 | mapping = self.map_rows_removed(update_diff, mapping) 91 | if update_diff.added: 92 | mapping = self.map_rows(update_diff, mapping, producer_site, operation="added") 93 | if update_diff.row_changed: 94 | mapping = self.map_rows(update_diff, mapping, producer_site, operation="row_changed") 95 | 96 | update = {"doc": frappe.as_json(mapping)} 97 | if len(dependencies): 98 | update["dependencies"] = dependencies 99 | return update 100 | 101 | def get_mapped_dependency(self, mapping, producer_site, doc): 102 | inner_mapping = frappe.get_doc("Document Type Mapping", mapping.mapping) 103 | filters = json.loads(mapping.remote_value_filters) 104 | for key, value in filters.items(): 105 | if value.startswith("eval:"): 106 | val = frappe.safe_eval(value[5:], None, dict(doc=doc)) 107 | filters[key] = val 108 | if doc.get(value): 109 | filters[key] = doc.get(value) 110 | matching_docs = producer_site.get_doc(inner_mapping.remote_doctype, filters=filters) 111 | if len(matching_docs): 112 | remote_docname = matching_docs[0].get("name") 113 | remote_doc = producer_site.get_doc(inner_mapping.remote_doctype, remote_docname) 114 | doc = inner_mapping.get_mapping(remote_doc, producer_site, "Insert").get("doc") 115 | return doc 116 | return 117 | 118 | def map_rows_removed(self, update_diff, mapping): 119 | removed = [] 120 | mapping["removed"] = update_diff.removed 121 | for key, value in update_diff.removed.copy().items(): 122 | local_table_name = frappe.db.get_value( 123 | "Document Type Field Mapping", 124 | {"remote_fieldname": key, "parent": self.name}, 125 | "local_fieldname", 126 | ) 127 | mapping.removed[local_table_name] = value 128 | if local_table_name != key: 129 | removed.append(key) 130 | 131 | # remove the remote fieldnames 132 | for field in removed: 133 | mapping.removed.pop(field, None) 134 | return mapping 135 | 136 | def map_rows(self, update_diff, mapping, producer_site, operation): 137 | remote_fields = [] 138 | for tablename, entries in update_diff.get(operation).copy().items(): 139 | local_table_name = frappe.db.get_value( 140 | "Document Type Field Mapping", {"remote_fieldname": tablename}, "local_fieldname" 141 | ) 142 | table_map = frappe.db.get_value( 143 | "Document Type Field Mapping", 144 | {"local_fieldname": local_table_name, "parent": self.name}, 145 | "mapping", 146 | ) 147 | table_map = frappe.get_doc("Document Type Mapping", table_map) 148 | docs = [] 149 | for entry in entries: 150 | mapped_doc = table_map.get_mapping(entry, producer_site, "Update").get("doc") 151 | docs.append(json.loads(mapped_doc)) 152 | mapping.get(operation)[local_table_name] = docs 153 | if local_table_name != tablename: 154 | remote_fields.append(tablename) 155 | 156 | # remove the remote fieldnames 157 | for field in remote_fields: 158 | mapping.get(operation).pop(field, None) 159 | 160 | return mapping 161 | 162 | 163 | def get_mapped_child_table_docs(child_map, table_entries, producer_site): 164 | """Get mapping for child doctypes""" 165 | child_map = frappe.get_doc("Document Type Mapping", child_map) 166 | mapped_entries = [] 167 | remote_fields = [] 168 | for child_doc in table_entries: 169 | for mapping in child_map.field_mapping: 170 | if child_doc.get(mapping.remote_fieldname): 171 | child_doc[mapping.local_fieldname] = child_doc[mapping.remote_fieldname] 172 | if mapping.local_fieldname != mapping.remote_fieldname: 173 | child_doc.pop(mapping.remote_fieldname, None) 174 | mapped_entries.append(child_doc) 175 | 176 | # remove the remote fieldnames 177 | for field in remote_fields: 178 | child_doc.pop(field, None) 179 | 180 | child_doc["doctype"] = child_map.local_doctype 181 | return mapped_entries 182 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and Contributors 2 | # License: MIT. See LICENSE 3 | # import frappe 4 | from frappe.tests.utils import FrappeTestCase 5 | 6 | 7 | class TestDocumentTypeMapping(FrappeTestCase): 8 | pass 9 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_consumer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/event_consumer/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_consumer/event_consumer.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019, Frappe Technologies and contributors 2 | // For license information, please see license.txt 3 | 4 | frappe.ui.form.on("Event Consumer", { 5 | refresh: function (frm) { 6 | // formatter for subscribed doctype approval status 7 | frm.set_indicator_formatter("status", function (doc) { 8 | let indicator = "orange"; 9 | if (doc.status == "Approved") { 10 | indicator = "green"; 11 | } else if (doc.status == "Rejected") { 12 | indicator = "red"; 13 | } 14 | return indicator; 15 | }); 16 | }, 17 | }); 18 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_consumer/event_consumer.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [], 3 | "autoname": "field:callback_url", 4 | "creation": "2019-08-26 17:45:15.479530", 5 | "doctype": "DocType", 6 | "editable_grid": 1, 7 | "engine": "InnoDB", 8 | "field_order": [ 9 | "consumer_doctypes", 10 | "callback_url", 11 | "section_break_3", 12 | "api_key", 13 | "api_secret", 14 | "column_break_6", 15 | "user", 16 | "incoming_change" 17 | ], 18 | "fields": [ 19 | { 20 | "fieldname": "callback_url", 21 | "fieldtype": "Data", 22 | "in_list_view": 1, 23 | "label": "Callback URL", 24 | "read_only": 1, 25 | "reqd": 1, 26 | "unique": 1 27 | }, 28 | { 29 | "fieldname": "api_key", 30 | "fieldtype": "Data", 31 | "label": "API Key", 32 | "reqd": 1 33 | }, 34 | { 35 | "fieldname": "api_secret", 36 | "fieldtype": "Password", 37 | "label": "API Secret", 38 | "reqd": 1 39 | }, 40 | { 41 | "fieldname": "user", 42 | "fieldtype": "Link", 43 | "label": "Event Subscriber", 44 | "options": "User", 45 | "read_only": 1, 46 | "reqd": 1 47 | }, 48 | { 49 | "fieldname": "section_break_3", 50 | "fieldtype": "Section Break" 51 | }, 52 | { 53 | "fieldname": "column_break_6", 54 | "fieldtype": "Column Break" 55 | }, 56 | { 57 | "default": "0", 58 | "fieldname": "incoming_change", 59 | "fieldtype": "Check", 60 | "hidden": 1, 61 | "label": "Incoming Change", 62 | "read_only": 1 63 | }, 64 | { 65 | "fieldname": "consumer_doctypes", 66 | "fieldtype": "Table", 67 | "label": "Event Consumer Document Types", 68 | "options": "Event Consumer Document Type", 69 | "reqd": 1 70 | } 71 | ], 72 | "in_create": 1, 73 | "links": [], 74 | "modified": "2020-09-08 16:42:39.828085", 75 | "modified_by": "Administrator", 76 | "module": "Event Streaming", 77 | "name": "Event Consumer", 78 | "owner": "Administrator", 79 | "permissions": [ 80 | { 81 | "create": 1, 82 | "delete": 1, 83 | "email": 1, 84 | "export": 1, 85 | "print": 1, 86 | "read": 1, 87 | "report": 1, 88 | "role": "System Manager", 89 | "share": 1, 90 | "write": 1 91 | } 92 | ], 93 | "quick_entry": 1, 94 | "sort_field": "modified", 95 | "sort_order": "DESC", 96 | "track_changes": 1 97 | } -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_consumer/event_consumer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and contributors 2 | # License: MIT. See LICENSE 3 | 4 | import json 5 | import os 6 | 7 | import requests 8 | 9 | import frappe 10 | from frappe import _ 11 | from frappe.frappeclient import FrappeClient 12 | from frappe.model.document import Document 13 | from frappe.utils.background_jobs import get_jobs 14 | from frappe.utils.data import get_url 15 | 16 | 17 | class EventConsumer(Document): 18 | def validate(self): 19 | # approve subscribed doctypes for tests 20 | # frappe.flags.in_test won't work here as tests are running on the consumer site 21 | if os.environ.get("CI"): 22 | for entry in self.consumer_doctypes: 23 | entry.status = "Approved" 24 | 25 | def on_update(self): 26 | if not self.incoming_change: 27 | doc_before_save = self.get_doc_before_save() 28 | if doc_before_save.api_key != self.api_key or doc_before_save.api_secret != self.api_secret: 29 | return 30 | 31 | self.update_consumer_status() 32 | else: 33 | frappe.db.set_value(self.doctype, self.name, "incoming_change", 0) 34 | 35 | def clear_cache(self): 36 | from event_streaming.event_streaming.doctype.event_update_log.event_update_log import ( 37 | ENABLED_DOCTYPES_CACHE_KEY, 38 | ) 39 | 40 | frappe.cache().delete_value(ENABLED_DOCTYPES_CACHE_KEY) 41 | return super().clear_cache() 42 | 43 | def on_trash(self): 44 | for i in frappe.get_all("Event Update Log Consumer", {"consumer": self.name}): 45 | frappe.delete_doc("Event Update Log Consumer", i.name) 46 | 47 | def update_consumer_status(self): 48 | consumer_site = get_consumer_site(self.callback_url) 49 | event_producer = consumer_site.get_doc("Event Producer", get_url()) 50 | event_producer = frappe._dict(event_producer) 51 | config = event_producer.producer_doctypes 52 | event_producer.producer_doctypes = [] 53 | for entry in config: 54 | if entry.get("has_mapping"): 55 | ref_doctype = consumer_site.get_value( 56 | "Document Type Mapping", "remote_doctype", entry.get("mapping") 57 | ).get("remote_doctype") 58 | else: 59 | ref_doctype = entry.get("ref_doctype") 60 | 61 | entry["status"] = frappe.db.get_value( 62 | "Event Consumer Document Type", {"parent": self.name, "ref_doctype": ref_doctype}, "status" 63 | ) 64 | 65 | event_producer.producer_doctypes = config 66 | # when producer doc is updated it updates the consumer doc 67 | # set flag to avoid deadlock 68 | event_producer.incoming_change = True 69 | consumer_site.update(event_producer) 70 | 71 | def get_consumer_status(self): 72 | response = requests.get(self.callback_url) 73 | if response.status_code != 200: 74 | return "offline" 75 | return "online" 76 | 77 | 78 | @frappe.whitelist() 79 | def register_consumer(data): 80 | """create an event consumer document for registering a consumer""" 81 | data = json.loads(data) 82 | # to ensure that consumer is created only once 83 | if frappe.db.exists("Event Consumer", data["event_consumer"]): 84 | return None 85 | 86 | user = data["user"] 87 | if not frappe.db.exists("User", user): 88 | frappe.throw(_("User {0} not found on the producer site").format(user)) 89 | 90 | if "System Manager" not in frappe.get_roles(user): 91 | frappe.throw(_("Event Subscriber has to be a System Manager.")) 92 | 93 | consumer = frappe.new_doc("Event Consumer") 94 | consumer.callback_url = data["event_consumer"] 95 | consumer.user = data["user"] 96 | consumer.api_key = data["api_key"] 97 | consumer.api_secret = data["api_secret"] 98 | consumer.incoming_change = True 99 | consumer_doctypes = json.loads(data["consumer_doctypes"]) 100 | 101 | for entry in consumer_doctypes: 102 | consumer.append( 103 | "consumer_doctypes", 104 | {"ref_doctype": entry.get("doctype"), "status": "Pending", "condition": entry.get("condition")}, 105 | ) 106 | 107 | consumer.insert() 108 | 109 | # consumer's 'last_update' field should point to the latest update 110 | # in producer's update log when subscribing 111 | # so that, updates after subscribing are consumed and not the old ones. 112 | last_update = str(get_last_update()) 113 | return json.dumps({"last_update": last_update}) 114 | 115 | 116 | def get_consumer_site(consumer_url): 117 | """create a FrappeClient object for event consumer site""" 118 | consumer_doc = frappe.get_doc("Event Consumer", consumer_url) 119 | consumer_site = FrappeClient( 120 | url=consumer_url, 121 | api_key=consumer_doc.api_key, 122 | api_secret=consumer_doc.get_password("api_secret"), 123 | ) 124 | return consumer_site 125 | 126 | 127 | def get_last_update(): 128 | """get the creation timestamp of last update consumed""" 129 | updates = frappe.get_list( 130 | "Event Update Log", "creation", ignore_permissions=True, limit=1, order_by="creation desc" 131 | ) 132 | if updates: 133 | return updates[0].creation 134 | return frappe.utils.now_datetime() 135 | 136 | 137 | @frappe.whitelist() 138 | def notify_event_consumers(doctype): 139 | """get all event consumers and set flag for notification status""" 140 | event_consumers = frappe.get_all( 141 | "Event Consumer Document Type", ["parent"], {"ref_doctype": doctype, "status": "Approved"} 142 | ) 143 | for entry in event_consumers: 144 | consumer = frappe.get_doc("Event Consumer", entry.parent) 145 | consumer.flags.notified = False 146 | notify(consumer) 147 | 148 | 149 | @frappe.whitelist() 150 | def notify(consumer): 151 | """notify individual event consumers about a new update""" 152 | consumer_status = consumer.get_consumer_status() 153 | if consumer_status == "online": 154 | try: 155 | client = get_consumer_site(consumer.callback_url) 156 | client.post_request( 157 | { 158 | "cmd": "event_streaming.event_streaming.doctype.event_producer.event_producer.new_event_notification", 159 | "producer_url": get_url(), 160 | } 161 | ) 162 | consumer.flags.notified = True 163 | except Exception: 164 | consumer.flags.notified = False 165 | else: 166 | consumer.flags.notified = False 167 | 168 | # enqueue another job if the site was not notified 169 | if not consumer.flags.notified: 170 | enqueued_method = "event_streaming.event_streaming.doctype.event_consumer.event_consumer.notify" 171 | jobs = get_jobs() 172 | if not jobs or enqueued_method not in jobs[frappe.local.site] and not consumer.flags.notifed: 173 | frappe.enqueue( 174 | enqueued_method, queue="long", enqueue_after_commit=True, **{"consumer": consumer} 175 | ) 176 | 177 | 178 | def has_consumer_access(consumer, update_log): 179 | """Checks if consumer has completely satisfied all the conditions on the doc""" 180 | 181 | if isinstance(consumer, str): 182 | consumer = frappe.get_doc("Event Consumer", consumer) 183 | 184 | if not frappe.db.exists(update_log.ref_doctype, update_log.docname): 185 | # Delete Log 186 | # Check if the last Update Log of this document was read by this consumer 187 | last_update_log = frappe.get_all( 188 | "Event Update Log", 189 | filters={ 190 | "ref_doctype": update_log.ref_doctype, 191 | "docname": update_log.docname, 192 | "creation": ["<", update_log.creation], 193 | }, 194 | order_by="creation desc", 195 | limit_page_length=1, 196 | ) 197 | if not len(last_update_log): 198 | return False 199 | 200 | last_update_log = frappe.get_doc("Event Update Log", last_update_log[0].name) 201 | return len([x for x in last_update_log.consumers if x.consumer == consumer.name]) 202 | 203 | doc = frappe.get_doc(update_log.ref_doctype, update_log.docname) 204 | try: 205 | for dt_entry in consumer.consumer_doctypes: 206 | if dt_entry.ref_doctype != update_log.ref_doctype: 207 | continue 208 | 209 | if not dt_entry.condition: 210 | return True 211 | 212 | condition: str = dt_entry.condition 213 | if condition.startswith("cmd:"): 214 | cmd = condition.split("cmd:")[1].strip() 215 | args = {"consumer": consumer, "doc": doc, "update_log": update_log} 216 | return frappe.call(cmd, **args) 217 | else: 218 | return frappe.safe_eval(condition, frappe._dict(doc=doc)) 219 | except Exception as e: 220 | consumer.log_error("has_consumer_access error") 221 | return False 222 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_consumer/test_event_consumer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and Contributors 2 | # License: MIT. See LICENSE 3 | # import frappe 4 | from frappe.tests.utils import FrappeTestCase 5 | 6 | 7 | class TestEventConsumer(FrappeTestCase): 8 | pass 9 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_consumer_document_type/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/event_consumer_document_type/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [], 3 | "creation": "2019-10-03 21:10:54.754651", 4 | "doctype": "DocType", 5 | "editable_grid": 1, 6 | "engine": "InnoDB", 7 | "field_order": [ 8 | "ref_doctype", 9 | "status", 10 | "unsubscribed", 11 | "condition" 12 | ], 13 | "fields": [ 14 | { 15 | "columns": 4, 16 | "fieldname": "ref_doctype", 17 | "fieldtype": "Link", 18 | "in_list_view": 1, 19 | "label": "Document Type", 20 | "options": "DocType", 21 | "read_only": 1, 22 | "reqd": 1 23 | }, 24 | { 25 | "columns": 4, 26 | "default": "Pending", 27 | "fieldname": "status", 28 | "fieldtype": "Select", 29 | "in_list_view": 1, 30 | "label": "Approval Status", 31 | "options": "Pending\nApproved\nRejected" 32 | }, 33 | { 34 | "columns": 2, 35 | "default": "0", 36 | "fieldname": "unsubscribed", 37 | "fieldtype": "Check", 38 | "in_list_view": 1, 39 | "label": "Unsubscribed", 40 | "read_only": 1 41 | }, 42 | { 43 | "fieldname": "condition", 44 | "fieldtype": "Code", 45 | "label": "Condition", 46 | "read_only": 1 47 | } 48 | ], 49 | "istable": 1, 50 | "links": [], 51 | "modified": "2020-11-07 09:26:49.894294", 52 | "modified_by": "Administrator", 53 | "module": "Event Streaming", 54 | "name": "Event Consumer Document Type", 55 | "owner": "Administrator", 56 | "permissions": [], 57 | "quick_entry": 1, 58 | "sort_field": "modified", 59 | "sort_order": "DESC", 60 | "track_changes": 1 61 | } -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and contributors 2 | # License: MIT. See LICENSE 3 | 4 | # import frappe 5 | from frappe.model.document import Document 6 | 7 | 8 | class EventConsumerDocumentType(Document): 9 | pass 10 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/event_producer/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer/event_producer.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019, Frappe Technologies and contributors 2 | // For license information, please see license.txt 3 | 4 | frappe.ui.form.on("Event Producer", { 5 | refresh: function (frm) { 6 | frm.set_query("ref_doctype", "producer_doctypes", function () { 7 | return { 8 | filters: { 9 | issingle: 0, 10 | istable: 0, 11 | }, 12 | }; 13 | }); 14 | 15 | frm.set_indicator_formatter("status", function (doc) { 16 | let indicator = "orange"; 17 | if (doc.status == "Approved") { 18 | indicator = "green"; 19 | } else if (doc.status == "Rejected") { 20 | indicator = "red"; 21 | } 22 | return indicator; 23 | }); 24 | }, 25 | }); 26 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer/event_producer.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [], 3 | "autoname": "field:producer_url", 4 | "creation": "2019-08-26 19:17:24.919196", 5 | "doctype": "DocType", 6 | "editable_grid": 1, 7 | "engine": "InnoDB", 8 | "field_order": [ 9 | "producer_url", 10 | "producer_doctypes", 11 | "section_break_3", 12 | "api_key", 13 | "api_secret", 14 | "column_break_6", 15 | "user", 16 | "incoming_change" 17 | ], 18 | "fields": [ 19 | { 20 | "fieldname": "producer_url", 21 | "fieldtype": "Data", 22 | "in_list_view": 1, 23 | "label": "Producer URL", 24 | "reqd": 1, 25 | "unique": 1 26 | }, 27 | { 28 | "description": "API Key of the user(Event Subscriber) on the producer site", 29 | "fieldname": "api_key", 30 | "fieldtype": "Data", 31 | "label": "API Key", 32 | "reqd": 1 33 | }, 34 | { 35 | "description": "API Secret of the user(Event Subscriber) on the producer site", 36 | "fieldname": "api_secret", 37 | "fieldtype": "Password", 38 | "label": "API Secret", 39 | "reqd": 1 40 | }, 41 | { 42 | "fieldname": "user", 43 | "fieldtype": "Link", 44 | "label": "Event Subscriber", 45 | "options": "User", 46 | "reqd": 1, 47 | "set_only_once": 1 48 | }, 49 | { 50 | "fieldname": "column_break_6", 51 | "fieldtype": "Column Break" 52 | }, 53 | { 54 | "fieldname": "section_break_3", 55 | "fieldtype": "Section Break" 56 | }, 57 | { 58 | "default": "0", 59 | "fieldname": "incoming_change", 60 | "fieldtype": "Check", 61 | "hidden": 1, 62 | "label": "Incoming Change" 63 | }, 64 | { 65 | "fieldname": "producer_doctypes", 66 | "fieldtype": "Table", 67 | "label": "Event Producer Document Types", 68 | "options": "Event Producer Document Type", 69 | "reqd": 1 70 | } 71 | ], 72 | "links": [], 73 | "modified": "2020-10-26 13:00:15.361316", 74 | "modified_by": "Administrator", 75 | "module": "Event Streaming", 76 | "name": "Event Producer", 77 | "owner": "Administrator", 78 | "permissions": [ 79 | { 80 | "create": 1, 81 | "delete": 1, 82 | "email": 1, 83 | "export": 1, 84 | "print": 1, 85 | "read": 1, 86 | "report": 1, 87 | "role": "System Manager", 88 | "share": 1, 89 | "write": 1 90 | } 91 | ], 92 | "quick_entry": 1, 93 | "sort_field": "modified", 94 | "sort_order": "DESC", 95 | "track_changes": 1 96 | } -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer/event_producer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and contributors 2 | # License: MIT. See LICENSE 3 | 4 | import json 5 | import time 6 | 7 | import requests 8 | 9 | import frappe 10 | from frappe import _ 11 | from frappe.custom.doctype.custom_field.custom_field import create_custom_field 12 | from frappe.frappeclient import FrappeClient 13 | from frappe.model.document import Document 14 | from frappe.utils.background_jobs import get_jobs 15 | from frappe.utils.data import get_link_to_form, get_url 16 | from frappe.utils.password import get_decrypted_password 17 | 18 | 19 | class EventProducer(Document): 20 | def before_insert(self): 21 | self.check_url() 22 | self.validate_event_subscriber() 23 | self.incoming_change = True 24 | self.create_event_consumer() 25 | self.create_custom_fields() 26 | 27 | def validate(self): 28 | self.validate_event_subscriber() 29 | if frappe.flags.in_test: 30 | for entry in self.producer_doctypes: 31 | entry.status = "Approved" 32 | 33 | def validate_event_subscriber(self): 34 | if not frappe.db.get_value("User", self.user, "api_key"): 35 | frappe.throw( 36 | _("Please generate keys for the Event Subscriber User {0} first.").format( 37 | frappe.bold(get_link_to_form("User", self.user)) 38 | ) 39 | ) 40 | 41 | def on_update(self): 42 | if not self.incoming_change: 43 | if frappe.db.exists("Event Producer", self.name): 44 | if not self.api_key or not self.api_secret: 45 | frappe.throw(_("Please set API Key and Secret on the producer and consumer sites first.")) 46 | else: 47 | doc_before_save = self.get_doc_before_save() 48 | if doc_before_save.api_key != self.api_key or doc_before_save.api_secret != self.api_secret: 49 | return 50 | 51 | self.update_event_consumer() 52 | self.create_custom_fields() 53 | else: 54 | # when producer doc is updated it updates the consumer doc, set flag to avoid deadlock 55 | self.db_set("incoming_change", 0) 56 | self.reload() 57 | 58 | def on_trash(self): 59 | last_update = frappe.db.get_value("Event Producer Last Update", dict(event_producer=self.name)) 60 | if last_update: 61 | frappe.delete_doc("Event Producer Last Update", last_update) 62 | 63 | def check_url(self): 64 | valid_url_schemes = ("http", "https") 65 | frappe.utils.validate_url(self.producer_url, throw=True, valid_schemes=valid_url_schemes) 66 | 67 | # remove '/' from the end of the url like http://test_site.com/ 68 | # to prevent mismatch in get_url() results 69 | if self.producer_url.endswith("/"): 70 | self.producer_url = self.producer_url[:-1] 71 | 72 | def create_event_consumer(self): 73 | """register event consumer on the producer site""" 74 | if self.is_producer_online(): 75 | producer_site = FrappeClient( 76 | url=self.producer_url, api_key=self.api_key, api_secret=self.get_password("api_secret") 77 | ) 78 | 79 | response = producer_site.post_api( 80 | "event_streaming.event_streaming.doctype.event_consumer.event_consumer.register_consumer", 81 | params={"data": json.dumps(self.get_request_data())}, 82 | ) 83 | if response: 84 | response = json.loads(response) 85 | self.set_last_update(response["last_update"]) 86 | else: 87 | frappe.throw( 88 | _( 89 | "Failed to create an Event Consumer or an Event Consumer for the current site is already registered." 90 | ) 91 | ) 92 | 93 | def set_last_update(self, last_update): 94 | last_update_doc_name = frappe.db.get_value( 95 | "Event Producer Last Update", dict(event_producer=self.name) 96 | ) 97 | if not last_update_doc_name: 98 | frappe.get_doc( 99 | dict( 100 | doctype="Event Producer Last Update", 101 | event_producer=self.producer_url, 102 | last_update=last_update, 103 | ) 104 | ).insert(ignore_permissions=True) 105 | else: 106 | frappe.db.set_value( 107 | "Event Producer Last Update", last_update_doc_name, "last_update", last_update 108 | ) 109 | 110 | def get_last_update(self): 111 | return frappe.db.get_value( 112 | "Event Producer Last Update", dict(event_producer=self.name), "last_update" 113 | ) 114 | 115 | def get_request_data(self): 116 | consumer_doctypes = [] 117 | for entry in self.producer_doctypes: 118 | if entry.has_mapping: 119 | # if mapping, subscribe to remote doctype on consumer's site 120 | dt = frappe.db.get_value("Document Type Mapping", entry.mapping, "remote_doctype") 121 | else: 122 | dt = entry.ref_doctype 123 | consumer_doctypes.append({"doctype": dt, "condition": entry.condition}) 124 | 125 | user_key = frappe.db.get_value("User", self.user, "api_key") 126 | user_secret = get_decrypted_password("User", self.user, "api_secret") 127 | return { 128 | "event_consumer": get_url(), 129 | "consumer_doctypes": json.dumps(consumer_doctypes), 130 | "user": self.user, 131 | "api_key": user_key, 132 | "api_secret": user_secret, 133 | } 134 | 135 | def create_custom_fields(self): 136 | """create custom field to store remote docname and remote site url""" 137 | for entry in self.producer_doctypes: 138 | if not entry.use_same_name: 139 | if not frappe.db.exists( 140 | "Custom Field", {"fieldname": "remote_docname", "dt": entry.ref_doctype} 141 | ): 142 | df = dict( 143 | fieldname="remote_docname", 144 | label="Remote Document Name", 145 | fieldtype="Data", 146 | read_only=1, 147 | print_hide=1, 148 | ) 149 | create_custom_field(entry.ref_doctype, df) 150 | if not frappe.db.exists( 151 | "Custom Field", {"fieldname": "remote_site_name", "dt": entry.ref_doctype} 152 | ): 153 | df = dict( 154 | fieldname="remote_site_name", 155 | label="Remote Site", 156 | fieldtype="Data", 157 | read_only=1, 158 | print_hide=1, 159 | ) 160 | create_custom_field(entry.ref_doctype, df) 161 | 162 | def update_event_consumer(self): 163 | if self.is_producer_online(): 164 | producer_site = get_producer_site(self.producer_url) 165 | event_consumer = producer_site.get_doc("Event Consumer", get_url()) 166 | event_consumer = frappe._dict(event_consumer) 167 | if event_consumer: 168 | config = event_consumer.consumer_doctypes 169 | event_consumer.consumer_doctypes = [] 170 | for entry in self.producer_doctypes: 171 | if entry.has_mapping: 172 | # if mapping, subscribe to remote doctype on consumer's site 173 | ref_doctype = frappe.db.get_value("Document Type Mapping", entry.mapping, "remote_doctype") 174 | else: 175 | ref_doctype = entry.ref_doctype 176 | 177 | event_consumer.consumer_doctypes.append( 178 | { 179 | "ref_doctype": ref_doctype, 180 | "status": get_approval_status(config, ref_doctype), 181 | "unsubscribed": entry.unsubscribe, 182 | "condition": entry.condition, 183 | } 184 | ) 185 | event_consumer.user = self.user 186 | event_consumer.incoming_change = True 187 | producer_site.update(event_consumer) 188 | 189 | def is_producer_online(self): 190 | """check connection status for the Event Producer site""" 191 | retry = 3 192 | while retry > 0: 193 | res = requests.get(self.producer_url) 194 | if res.status_code == 200: 195 | return True 196 | retry -= 1 197 | time.sleep(5) 198 | frappe.throw(_("Failed to connect to the Event Producer site. Retry after some time.")) 199 | 200 | 201 | def get_producer_site(producer_url): 202 | """create a FrappeClient object for event producer site""" 203 | producer_doc = frappe.get_doc("Event Producer", producer_url) 204 | producer_site = FrappeClient( 205 | url=producer_url, 206 | api_key=producer_doc.api_key, 207 | api_secret=producer_doc.get_password("api_secret"), 208 | ) 209 | return producer_site 210 | 211 | 212 | def get_approval_status(config, ref_doctype): 213 | """check the approval status for consumption""" 214 | for entry in config: 215 | if entry.get("ref_doctype") == ref_doctype: 216 | return entry.get("status") 217 | return "Pending" 218 | 219 | 220 | @frappe.whitelist() 221 | def pull_producer_data(): 222 | """Fetch data from producer node.""" 223 | response = requests.get(get_url()) 224 | if response.status_code == 200: 225 | for event_producer in frappe.get_all("Event Producer"): 226 | pull_from_node(event_producer.name) 227 | return "success" 228 | return None 229 | 230 | 231 | @frappe.whitelist() 232 | def pull_from_node(event_producer): 233 | """pull all updates after the last update timestamp from event producer site""" 234 | event_producer = frappe.get_doc("Event Producer", event_producer) 235 | producer_site = get_producer_site(event_producer.producer_url) 236 | last_update = event_producer.get_last_update() 237 | 238 | (doctypes, mapping_config, naming_config) = get_config(event_producer.producer_doctypes) 239 | 240 | updates = get_updates(producer_site, last_update, doctypes) 241 | 242 | for update in updates: 243 | update.use_same_name = naming_config.get(update.ref_doctype) 244 | mapping = mapping_config.get(update.ref_doctype) 245 | if mapping: 246 | update.mapping = mapping 247 | update = get_mapped_update(update, producer_site) 248 | if not update.update_type == "Delete": 249 | update.data = json.loads(update.data) 250 | 251 | sync(update, producer_site, event_producer) 252 | 253 | 254 | def get_config(event_config): 255 | """get the doctype mapping and naming configurations for consumption""" 256 | doctypes, mapping_config, naming_config = [], {}, {} 257 | 258 | for entry in event_config: 259 | if entry.status == "Approved": 260 | if entry.has_mapping: 261 | (mapped_doctype, mapping) = frappe.db.get_value( 262 | "Document Type Mapping", entry.mapping, ["remote_doctype", "name"] 263 | ) 264 | mapping_config[mapped_doctype] = mapping 265 | naming_config[mapped_doctype] = entry.use_same_name 266 | doctypes.append(mapped_doctype) 267 | else: 268 | naming_config[entry.ref_doctype] = entry.use_same_name 269 | doctypes.append(entry.ref_doctype) 270 | return (doctypes, mapping_config, naming_config) 271 | 272 | 273 | def sync(update, producer_site, event_producer, in_retry=False): 274 | """Sync the individual update""" 275 | try: 276 | if update.update_type == "Create": 277 | set_insert(update, producer_site, event_producer.name) 278 | if update.update_type == "Update": 279 | set_update(update, producer_site) 280 | if update.update_type == "Delete": 281 | set_delete(update) 282 | if in_retry: 283 | return "Synced" 284 | log_event_sync(update, event_producer.name, "Synced") 285 | 286 | except Exception: 287 | if in_retry: 288 | if frappe.flags.in_test: 289 | print(frappe.get_traceback()) 290 | return "Failed" 291 | log_event_sync(update, event_producer.name, "Failed", frappe.get_traceback()) 292 | 293 | event_producer.set_last_update(update.creation) 294 | frappe.db.commit() 295 | 296 | 297 | def set_insert(update, producer_site, event_producer): 298 | """Sync insert type update""" 299 | if frappe.db.get_value(update.ref_doctype, update.docname): 300 | # doc already created 301 | return 302 | doc = frappe.get_doc(update.data) 303 | 304 | if update.mapping: 305 | if update.get("dependencies"): 306 | dependencies_created = sync_mapped_dependencies(update.dependencies, producer_site) 307 | for fieldname, value in dependencies_created.items(): 308 | doc.update({fieldname: value}) 309 | else: 310 | sync_dependencies(doc, producer_site) 311 | 312 | if update.use_same_name: 313 | doc.insert(set_name=update.docname, set_child_names=False) 314 | else: 315 | # if event consumer is not saving documents with the same name as the producer 316 | # store the remote docname in a custom field for future updates 317 | doc.remote_docname = update.docname 318 | doc.remote_site_name = event_producer 319 | doc.insert(set_child_names=False) 320 | 321 | 322 | def set_update(update, producer_site): 323 | """Sync update type update""" 324 | local_doc = get_local_doc(update) 325 | if local_doc: 326 | data = frappe._dict(update.data) 327 | 328 | if data.changed: 329 | local_doc.update(data.changed) 330 | if data.removed: 331 | local_doc = update_row_removed(local_doc, data.removed) 332 | if data.row_changed: 333 | update_row_changed(local_doc, data.row_changed) 334 | if data.added: 335 | local_doc = update_row_added(local_doc, data.added) 336 | 337 | if update.mapping: 338 | if update.get("dependencies"): 339 | dependencies_created = sync_mapped_dependencies(update.dependencies, producer_site) 340 | for fieldname, value in dependencies_created.items(): 341 | local_doc.update({fieldname: value}) 342 | else: 343 | sync_dependencies(local_doc, producer_site) 344 | 345 | local_doc.save() 346 | local_doc.db_update_all() 347 | 348 | 349 | def update_row_removed(local_doc, removed): 350 | """Sync child table row deletion type update""" 351 | for tablename, rownames in removed.items(): 352 | table = local_doc.get_table_field_doctype(tablename) 353 | for row in rownames: 354 | table_rows = local_doc.get(tablename) 355 | child_table_row = get_child_table_row(table_rows, row) 356 | table_rows.remove(child_table_row) 357 | local_doc.set(tablename, table_rows) 358 | return local_doc 359 | 360 | 361 | def get_child_table_row(table_rows, row): 362 | for entry in table_rows: 363 | if entry.get("name") == row: 364 | return entry 365 | 366 | 367 | def update_row_changed(local_doc, changed): 368 | """Sync child table row updation type update""" 369 | for tablename, rows in changed.items(): 370 | old = local_doc.get(tablename) 371 | for doc in old: 372 | for row in rows: 373 | if row["name"] == doc.get("name"): 374 | doc.update(row) 375 | 376 | 377 | def update_row_added(local_doc, added): 378 | """Sync child table row addition type update""" 379 | for tablename, rows in added.items(): 380 | local_doc.extend(tablename, rows) 381 | for child in rows: 382 | child_doc = frappe.get_doc(child) 383 | child_doc.parent = local_doc.name 384 | child_doc.parenttype = local_doc.doctype 385 | child_doc.insert(set_name=child_doc.name) 386 | return local_doc 387 | 388 | 389 | def set_delete(update): 390 | """Sync delete type update""" 391 | local_doc = get_local_doc(update) 392 | if local_doc: 393 | local_doc.delete() 394 | 395 | 396 | def get_updates(producer_site, last_update, doctypes): 397 | """Get all updates generated after the last update timestamp""" 398 | docs = producer_site.post_request( 399 | { 400 | "cmd": "event_streaming.event_streaming.doctype.event_update_log.event_update_log.get_update_logs_for_consumer", 401 | "event_consumer": get_url(), 402 | "doctypes": frappe.as_json(doctypes), 403 | "last_update": last_update, 404 | } 405 | ) 406 | return [frappe._dict(d) for d in (docs or [])] 407 | 408 | 409 | def get_local_doc(update): 410 | """Get the local document if created with a different name""" 411 | try: 412 | if not update.use_same_name: 413 | return frappe.get_doc(update.ref_doctype, {"remote_docname": update.docname}) 414 | return frappe.get_doc(update.ref_doctype, update.docname) 415 | except frappe.DoesNotExistError: 416 | return None 417 | 418 | 419 | def sync_dependencies(document, producer_site): 420 | """ 421 | dependencies is a dictionary to store all the docs 422 | having dependencies and their sync status, 423 | which is shared among all nested functions. 424 | """ 425 | dependencies = {document: True} 426 | 427 | def check_doc_has_dependencies(doc, producer_site): 428 | """Sync child table link fields first, 429 | then sync link fields, 430 | then dynamic links""" 431 | meta = frappe.get_meta(doc.doctype) 432 | table_fields = meta.get_table_fields() 433 | link_fields = meta.get_link_fields() 434 | dl_fields = meta.get_dynamic_link_fields() 435 | if table_fields: 436 | sync_child_table_dependencies(doc, table_fields, producer_site) 437 | if link_fields: 438 | sync_link_dependencies(doc, link_fields, producer_site) 439 | if dl_fields: 440 | sync_dynamic_link_dependencies(doc, dl_fields, producer_site) 441 | 442 | def sync_child_table_dependencies(doc, table_fields, producer_site): 443 | for df in table_fields: 444 | child_table = doc.get(df.fieldname) 445 | for entry in child_table: 446 | child_doc = producer_site.get_doc(entry.doctype, entry.name) 447 | if child_doc: 448 | child_doc = frappe._dict(child_doc) 449 | set_dependencies(child_doc, frappe.get_meta(entry.doctype).get_link_fields(), producer_site) 450 | 451 | def sync_link_dependencies(doc, link_fields, producer_site): 452 | set_dependencies(doc, link_fields, producer_site) 453 | 454 | def sync_dynamic_link_dependencies(doc, dl_fields, producer_site): 455 | for df in dl_fields: 456 | docname = doc.get(df.fieldname) 457 | linked_doctype = doc.get(df.options) 458 | if docname and not check_dependency_fulfilled(linked_doctype, docname): 459 | master_doc = producer_site.get_doc(linked_doctype, docname) 460 | frappe.get_doc(master_doc).insert(set_name=docname) 461 | 462 | def set_dependencies(doc, link_fields, producer_site): 463 | for df in link_fields: 464 | docname = doc.get(df.fieldname) 465 | linked_doctype = df.get_link_doctype() 466 | if docname and not check_dependency_fulfilled(linked_doctype, docname): 467 | master_doc = producer_site.get_doc(linked_doctype, docname) 468 | try: 469 | master_doc = frappe.get_doc(master_doc) 470 | master_doc.insert(set_name=docname) 471 | frappe.db.commit() 472 | 473 | # for dependency inside a dependency 474 | except Exception: 475 | dependencies[master_doc] = True 476 | 477 | def check_dependency_fulfilled(linked_doctype, docname): 478 | return frappe.db.exists(linked_doctype, docname) 479 | 480 | while dependencies[document]: 481 | # find the first non synced dependency 482 | for item in reversed(list(dependencies.keys())): 483 | if dependencies[item]: 484 | dependency = item 485 | break 486 | 487 | check_doc_has_dependencies(dependency, producer_site) 488 | 489 | # mark synced for nested dependency 490 | if dependency != document: 491 | dependencies[dependency] = False 492 | dependency.insert() 493 | 494 | # no more dependencies left to be synced, the main doc is ready to be synced 495 | # end the dependency loop 496 | if not any(list(dependencies.values())[1:]): 497 | dependencies[document] = False 498 | 499 | 500 | def sync_mapped_dependencies(dependencies, producer_site): 501 | dependencies_created = {} 502 | for entry in dependencies: 503 | doc = frappe._dict(json.loads(entry[1])) 504 | docname = frappe.db.exists(doc.doctype, doc.name) 505 | if not docname: 506 | doc = frappe.get_doc(doc).insert(set_child_names=False) 507 | dependencies_created[entry[0]] = doc.name 508 | else: 509 | dependencies_created[entry[0]] = docname 510 | 511 | return dependencies_created 512 | 513 | 514 | def log_event_sync(update, event_producer, sync_status, error=None): 515 | """Log event update received with the sync_status as Synced or Failed""" 516 | doc = frappe.new_doc("Event Sync Log") 517 | doc.update_type = update.update_type 518 | doc.ref_doctype = update.ref_doctype 519 | doc.status = sync_status 520 | doc.event_producer = event_producer 521 | doc.producer_doc = update.docname 522 | doc.data = frappe.as_json(update.data) 523 | doc.use_same_name = update.use_same_name 524 | doc.mapping = update.mapping if update.mapping else None 525 | if update.use_same_name: 526 | doc.docname = update.docname 527 | else: 528 | doc.docname = frappe.db.get_value(update.ref_doctype, {"remote_docname": update.docname}, "name") 529 | if error: 530 | doc.error = error 531 | doc.insert() 532 | 533 | 534 | def get_mapped_update(update, producer_site): 535 | """get the new update document with mapped fields""" 536 | mapping = frappe.get_doc("Document Type Mapping", update.mapping) 537 | if update.update_type == "Create": 538 | doc = frappe._dict(json.loads(update.data)) 539 | mapped_update = mapping.get_mapping(doc, producer_site, update.update_type) 540 | update.data = mapped_update.get("doc") 541 | update.dependencies = mapped_update.get("dependencies", None) 542 | elif update.update_type == "Update": 543 | mapped_update = mapping.get_mapped_update(update, producer_site) 544 | update.data = mapped_update.get("doc") 545 | update.dependencies = mapped_update.get("dependencies", None) 546 | 547 | update["ref_doctype"] = mapping.local_doctype 548 | return update 549 | 550 | 551 | @frappe.whitelist() 552 | def new_event_notification(producer_url): 553 | """Pull data from producer when notified""" 554 | enqueued_method = "event_streaming.event_streaming.doctype.event_producer.event_producer.pull_from_node" 555 | jobs = get_jobs() 556 | if not jobs or enqueued_method not in jobs[frappe.local.site]: 557 | frappe.enqueue(enqueued_method, queue="default", **{"event_producer": producer_url}) 558 | 559 | 560 | @frappe.whitelist() 561 | def resync(update): 562 | """Retry syncing update if failed""" 563 | update = frappe._dict(json.loads(update)) 564 | producer_site = get_producer_site(update.event_producer) 565 | event_producer = frappe.get_doc("Event Producer", update.event_producer) 566 | if update.mapping: 567 | update = get_mapped_update(update, producer_site) 568 | update.data = json.loads(update.data) 569 | return sync(update, producer_site, event_producer, in_retry=True) 570 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer/test_event_producer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and Contributors 2 | # License: MIT. See LICENSE 3 | import json 4 | 5 | from event_streaming.event_streaming.doctype.event_producer.event_producer import pull_from_node 6 | 7 | import frappe 8 | from frappe.core.doctype.user.user import generate_keys 9 | from frappe.frappeclient import FrappeClient 10 | from frappe.query_builder.utils import db_type_is 11 | from frappe.tests.test_query_builder import run_only_if 12 | from frappe.tests.utils import FrappeTestCase 13 | 14 | producer_url = "http://test_site_producer:8000" 15 | 16 | 17 | class TestEventProducer(FrappeTestCase): 18 | def setUp(self): 19 | create_event_producer(producer_url) 20 | 21 | def tearDown(self): 22 | unsubscribe_doctypes(producer_url) 23 | 24 | def test_insert(self): 25 | producer = get_remote_site() 26 | producer_doc = insert_into_producer(producer, "test creation 1 sync") 27 | self.pull_producer_data() 28 | self.assertTrue(frappe.db.exists("ToDo", producer_doc.name)) 29 | 30 | def test_update(self): 31 | producer = get_remote_site() 32 | producer_doc = insert_into_producer(producer, "test update 1") 33 | producer_doc["description"] = "test update 2" 34 | producer_doc = producer.update(producer_doc) 35 | self.pull_producer_data() 36 | local_doc = frappe.get_doc(producer_doc.doctype, producer_doc.name) 37 | self.assertEqual(local_doc.description, producer_doc.description) 38 | 39 | def test_delete(self): 40 | producer = get_remote_site() 41 | producer_doc = insert_into_producer(producer, "test delete sync") 42 | self.pull_producer_data() 43 | self.assertTrue(frappe.db.exists("ToDo", producer_doc.name)) 44 | producer.delete("ToDo", producer_doc.name) 45 | self.pull_producer_data() 46 | self.assertFalse(frappe.db.exists("ToDo", producer_doc.name)) 47 | 48 | @run_only_if(db_type_is.MARIADB) 49 | def test_multiple_doctypes_sync(self): 50 | # TODO: This test is extremely flaky with Postgres. Rewrite this! 51 | producer = get_remote_site() 52 | 53 | # insert todo and note in producer 54 | producer_todo = insert_into_producer(producer, "test multiple doc sync") 55 | producer_note1 = frappe._dict(doctype="Note", title="test multiple doc sync 1") 56 | delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]}) 57 | frappe.db.delete("Note", {"title": producer_note1["title"]}) 58 | producer_note1 = producer.insert(producer_note1) 59 | producer_note2 = frappe._dict(doctype="Note", title="test multiple doc sync 2") 60 | delete_on_remote_if_exists(producer, "Note", {"title": producer_note2["title"]}) 61 | frappe.db.delete("Note", {"title": producer_note2["title"]}) 62 | producer_note2 = producer.insert(producer_note2) 63 | 64 | # update in producer 65 | producer_todo["description"] = "test multiple doc update sync" 66 | producer_todo = producer.update(producer_todo) 67 | producer_note1["content"] = "testing update sync" 68 | producer_note1 = producer.update(producer_note1) 69 | 70 | producer.delete("Note", producer_note2.name) 71 | 72 | self.pull_producer_data() 73 | 74 | # check inserted 75 | self.assertTrue(frappe.db.exists("ToDo", producer_todo.name)) 76 | 77 | # check update 78 | local_todo = frappe.get_doc("ToDo", producer_todo.name) 79 | self.assertEqual(local_todo.description, producer_todo.description) 80 | local_note1 = frappe.get_doc("Note", producer_note1.name) 81 | self.assertEqual(local_note1.content, producer_note1.content) 82 | 83 | # check delete 84 | self.assertFalse(frappe.db.exists("Note", producer_note2.name)) 85 | 86 | def test_child_table_sync_with_dependencies(self): 87 | producer = get_remote_site() 88 | producer_user = frappe._dict( 89 | doctype="User", 90 | email="test_user@sync.com", 91 | send_welcome_email=0, 92 | first_name="Test Sync User", 93 | enabled=1, 94 | roles=[{"role": "System Manager"}], 95 | ) 96 | delete_on_remote_if_exists(producer, "User", {"email": producer_user.email}) 97 | frappe.db.delete("User", {"email": producer_user.email}) 98 | producer_user = producer.insert(producer_user) 99 | 100 | producer_note = frappe._dict( 101 | doctype="Note", title="test child table dependency sync", seen_by=[{"user": producer_user.name}] 102 | ) 103 | delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title}) 104 | frappe.db.delete("Note", {"title": producer_note.title}) 105 | producer_note = producer.insert(producer_note) 106 | 107 | self.pull_producer_data() 108 | self.assertTrue(frappe.db.exists("User", producer_user.name)) 109 | if self.assertTrue(frappe.db.exists("Note", producer_note.name)): 110 | local_note = frappe.get_doc("Note", producer_note.name) 111 | self.assertEqual(len(local_note.seen_by), 1) 112 | 113 | def test_dynamic_link_dependencies_synced(self): 114 | producer = get_remote_site() 115 | # unsubscribe for Note to check whether dependency is fulfilled 116 | event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) 117 | event_producer.producer_doctypes = [] 118 | event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1}) 119 | event_producer.save() 120 | 121 | producer_link_doc = frappe._dict(doctype="Note", title="Test Dynamic Link 1") 122 | 123 | delete_on_remote_if_exists(producer, "Note", {"title": producer_link_doc.title}) 124 | frappe.db.delete("Note", {"title": producer_link_doc.title}) 125 | producer_link_doc = producer.insert(producer_link_doc) 126 | producer_doc = frappe._dict( 127 | doctype="ToDo", 128 | description="Test Dynamic Link 2", 129 | assigned_by="Administrator", 130 | reference_type="Note", 131 | reference_name=producer_link_doc.name, 132 | ) 133 | producer_doc = producer.insert(producer_doc) 134 | 135 | self.pull_producer_data() 136 | 137 | # check dynamic link dependency created 138 | self.assertTrue(frappe.db.exists("Note", producer_link_doc.name)) 139 | self.assertEqual( 140 | producer_link_doc.name, frappe.db.get_value("ToDo", producer_doc.name, "reference_name") 141 | ) 142 | 143 | reset_configuration(producer_url) 144 | 145 | def test_naming_configuration(self): 146 | # test with use_same_name = 0 147 | producer = get_remote_site() 148 | event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) 149 | event_producer.producer_doctypes = [] 150 | event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 0}) 151 | event_producer.save() 152 | 153 | producer_doc = insert_into_producer(producer, "test different name sync") 154 | self.pull_producer_data() 155 | self.assertTrue( 156 | frappe.db.exists( 157 | "ToDo", {"remote_docname": producer_doc.name, "remote_site_name": producer_url} 158 | ) 159 | ) 160 | 161 | reset_configuration(producer_url) 162 | 163 | def test_conditional_events(self): 164 | producer = get_remote_site() 165 | 166 | # Add Condition 167 | event_producer = frappe.get_doc("Event Producer", producer_url) 168 | note_producer_entry = [x for x in event_producer.producer_doctypes if x.ref_doctype == "Note"][0] 169 | note_producer_entry.condition = "doc.public == 1" 170 | event_producer.save() 171 | 172 | # Make test doc 173 | producer_note1 = frappe._dict(doctype="Note", public=0, title="test conditional sync") 174 | delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]}) 175 | producer_note1 = producer.insert(producer_note1) 176 | 177 | # Make Update 178 | producer_note1["content"] = "Test Conditional Sync Content" 179 | producer_note1 = producer.update(producer_note1) 180 | 181 | self.pull_producer_data() 182 | 183 | # Check if synced here 184 | self.assertFalse(frappe.db.exists("Note", producer_note1.name)) 185 | 186 | # Lets satisfy the condition 187 | producer_note1["public"] = 1 188 | producer_note1 = producer.update(producer_note1) 189 | 190 | self.pull_producer_data() 191 | 192 | # it should sync now 193 | self.assertTrue(frappe.db.exists("Note", producer_note1.name)) 194 | local_note = frappe.get_doc("Note", producer_note1.name) 195 | self.assertEqual(local_note.content, producer_note1.content) 196 | 197 | reset_configuration(producer_url) 198 | 199 | def test_conditional_events_with_cmd(self): 200 | producer = get_remote_site() 201 | 202 | # Add Condition 203 | event_producer = frappe.get_doc("Event Producer", producer_url) 204 | note_producer_entry = [x for x in event_producer.producer_doctypes if x.ref_doctype == "Note"][0] 205 | note_producer_entry.condition = ( 206 | "cmd: event_streaming.event_streaming.doctype.event_producer.test_event_producer.can_sync_note" 207 | ) 208 | event_producer.save() 209 | 210 | # Make test doc 211 | producer_note1 = frappe._dict(doctype="Note", public=0, title="test conditional sync cmd") 212 | delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]}) 213 | producer_note1 = producer.insert(producer_note1) 214 | 215 | # Make Update 216 | producer_note1["content"] = "Test Conditional Sync Content" 217 | producer_note1 = producer.update(producer_note1) 218 | 219 | self.pull_producer_data() 220 | 221 | # Check if synced here 222 | self.assertFalse(frappe.db.exists("Note", producer_note1.name)) 223 | 224 | # Lets satisfy the condition 225 | producer_note1["public"] = 1 226 | producer_note1 = producer.update(producer_note1) 227 | 228 | self.pull_producer_data() 229 | 230 | # it should sync now 231 | self.assertTrue(frappe.db.exists("Note", producer_note1.name)) 232 | local_note = frappe.get_doc("Note", producer_note1.name) 233 | self.assertEqual(local_note.content, producer_note1.content) 234 | 235 | reset_configuration(producer_url) 236 | 237 | def test_update_log(self): 238 | producer = get_remote_site() 239 | producer_doc = insert_into_producer(producer, "test update log") 240 | update_log_doc = producer.get_value( 241 | "Event Update Log", "docname", {"docname": producer_doc.get("name")} 242 | ) 243 | self.assertEqual(update_log_doc.get("docname"), producer_doc.get("name")) 244 | 245 | def test_event_sync_log(self): 246 | producer = get_remote_site() 247 | producer_doc = insert_into_producer(producer, "test event sync log") 248 | self.pull_producer_data() 249 | self.assertTrue(frappe.db.exists("Event Sync Log", {"docname": producer_doc.name})) 250 | 251 | def pull_producer_data(self): 252 | pull_from_node(producer_url) 253 | 254 | def test_mapping(self): 255 | producer = get_remote_site() 256 | event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) 257 | event_producer.producer_doctypes = [] 258 | mapping = [{"local_fieldname": "description", "remote_fieldname": "content"}] 259 | event_producer.append( 260 | "producer_doctypes", 261 | { 262 | "ref_doctype": "ToDo", 263 | "use_same_name": 1, 264 | "has_mapping": 1, 265 | "mapping": get_mapping("ToDo to Note", "ToDo", "Note", mapping), 266 | }, 267 | ) 268 | event_producer.save() 269 | 270 | producer_note = frappe._dict(doctype="Note", title="Test Mapping", content="Test Mapping") 271 | delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title}) 272 | producer_note = producer.insert(producer_note) 273 | self.pull_producer_data() 274 | # check inserted 275 | self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note.content})) 276 | 277 | # update in producer 278 | producer_note["content"] = "test mapped doc update sync" 279 | producer_note = producer.update(producer_note) 280 | self.pull_producer_data() 281 | 282 | # check updated 283 | self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note["content"]})) 284 | 285 | producer.delete("Note", producer_note.name) 286 | self.pull_producer_data() 287 | # check delete 288 | self.assertFalse(frappe.db.exists("ToDo", {"description": producer_note.content})) 289 | 290 | reset_configuration(producer_url) 291 | 292 | def test_inner_mapping(self): 293 | producer = get_remote_site() 294 | 295 | setup_event_producer_for_inner_mapping() 296 | producer_note = frappe._dict( 297 | doctype="Note", title="Inner Mapping Tester", content="Test Inner Mapping" 298 | ) 299 | delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title}) 300 | producer_note = producer.insert(producer_note) 301 | self.pull_producer_data() 302 | 303 | # check dependency inserted 304 | self.assertTrue(frappe.db.exists("Role", {"role_name": producer_note.title})) 305 | # check doc inserted 306 | self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note.content})) 307 | 308 | reset_configuration(producer_url) 309 | 310 | 311 | def can_sync_note(consumer, doc, update_log): 312 | return doc.public == 1 313 | 314 | 315 | def setup_event_producer_for_inner_mapping(): 316 | event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) 317 | event_producer.producer_doctypes = [] 318 | inner_mapping = [{"local_fieldname": "role_name", "remote_fieldname": "title"}] 319 | inner_map = get_mapping("Role to Note Dependency Creation", "Role", "Note", inner_mapping) 320 | mapping = [ 321 | { 322 | "local_fieldname": "description", 323 | "remote_fieldname": "content", 324 | }, 325 | { 326 | "local_fieldname": "role", 327 | "remote_fieldname": "title", 328 | "mapping_type": "Document", 329 | "mapping": inner_map, 330 | "remote_value_filters": json.dumps({"title": "title"}), 331 | }, 332 | ] 333 | event_producer.append( 334 | "producer_doctypes", 335 | { 336 | "ref_doctype": "ToDo", 337 | "use_same_name": 1, 338 | "has_mapping": 1, 339 | "mapping": get_mapping("ToDo to Note Mapping", "ToDo", "Note", mapping), 340 | }, 341 | ) 342 | event_producer.save() 343 | return event_producer 344 | 345 | 346 | def insert_into_producer(producer, description): 347 | # create and insert todo on remote site 348 | todo = dict(doctype="ToDo", description=description, assigned_by="Administrator") 349 | return producer.insert(todo) 350 | 351 | 352 | def delete_on_remote_if_exists(producer, doctype, filters): 353 | remote_doc = producer.get_value(doctype, "name", filters) 354 | if remote_doc: 355 | producer.delete(doctype, remote_doc.get("name")) 356 | 357 | 358 | def get_mapping(mapping_name, local, remote, field_map): 359 | name = frappe.db.exists("Document Type Mapping", mapping_name) 360 | if name: 361 | doc = frappe.get_doc("Document Type Mapping", name) 362 | else: 363 | doc = frappe.new_doc("Document Type Mapping") 364 | 365 | doc.mapping_name = mapping_name 366 | doc.local_doctype = local 367 | doc.remote_doctype = remote 368 | for entry in field_map: 369 | doc.append("field_mapping", entry) 370 | doc.save() 371 | return doc.name 372 | 373 | 374 | def create_event_producer(producer_url): 375 | if frappe.db.exists("Event Producer", producer_url): 376 | event_producer = frappe.get_doc("Event Producer", producer_url) 377 | for entry in event_producer.producer_doctypes: 378 | entry.unsubscribe = 0 379 | event_producer.save() 380 | return 381 | 382 | generate_keys("Administrator") 383 | 384 | producer_site = connect() 385 | 386 | response = producer_site.post_api( 387 | "frappe.core.doctype.user.user.generate_keys", params={"user": "Administrator"} 388 | ) 389 | 390 | api_secret = response.get("api_secret") 391 | 392 | response = producer_site.get_value("User", "api_key", {"name": "Administrator"}) 393 | api_key = response.get("api_key") 394 | 395 | event_producer = frappe.new_doc("Event Producer") 396 | event_producer.producer_doctypes = [] 397 | event_producer.producer_url = producer_url 398 | event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1}) 399 | event_producer.append("producer_doctypes", {"ref_doctype": "Note", "use_same_name": 1}) 400 | event_producer.user = "Administrator" 401 | event_producer.api_key = api_key 402 | event_producer.api_secret = api_secret 403 | event_producer.save() 404 | 405 | 406 | def reset_configuration(producer_url): 407 | event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) 408 | event_producer.producer_doctypes = [] 409 | event_producer.conditions = [] 410 | event_producer.producer_url = producer_url 411 | event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1}) 412 | event_producer.append("producer_doctypes", {"ref_doctype": "Note", "use_same_name": 1}) 413 | event_producer.user = "Administrator" 414 | event_producer.save() 415 | 416 | 417 | def get_remote_site(): 418 | producer_doc = frappe.get_doc("Event Producer", producer_url) 419 | producer_site = FrappeClient( 420 | url=producer_doc.producer_url, username="Administrator", password="admin", verify=False 421 | ) 422 | return producer_site 423 | 424 | 425 | def unsubscribe_doctypes(producer_url): 426 | event_producer = frappe.get_doc("Event Producer", producer_url) 427 | for entry in event_producer.producer_doctypes: 428 | entry.unsubscribe = 1 429 | event_producer.save() 430 | 431 | 432 | def connect(): 433 | def _connect(): 434 | return FrappeClient(url=producer_url, username="Administrator", password="admin", verify=False) 435 | 436 | try: 437 | return _connect() 438 | except Exception: 439 | return _connect() 440 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer_document_type/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/event_producer_document_type/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [], 3 | "creation": "2019-10-03 21:08:25.890352", 4 | "doctype": "DocType", 5 | "editable_grid": 1, 6 | "engine": "InnoDB", 7 | "field_order": [ 8 | "ref_doctype", 9 | "status", 10 | "use_same_name", 11 | "unsubscribe", 12 | "has_mapping", 13 | "mapping", 14 | "condition" 15 | ], 16 | "fields": [ 17 | { 18 | "columns": 3, 19 | "fieldname": "ref_doctype", 20 | "fieldtype": "Link", 21 | "in_list_view": 1, 22 | "label": "Document Type", 23 | "options": "DocType", 24 | "reqd": 1, 25 | "set_only_once": 1 26 | }, 27 | { 28 | "default": "0", 29 | "description": "If the document has different field names on the Producer and Consumer's end check this and set up the Mapping", 30 | "fieldname": "has_mapping", 31 | "fieldtype": "Check", 32 | "label": "Has Mapping" 33 | }, 34 | { 35 | "depends_on": "eval: doc.has_mapping", 36 | "fieldname": "mapping", 37 | "fieldtype": "Link", 38 | "label": "Mapping", 39 | "options": "Document Type Mapping" 40 | }, 41 | { 42 | "columns": 2, 43 | "default": "0", 44 | "description": "If this is checked the documents will have the same name as they have on the Event Producer's site", 45 | "fieldname": "use_same_name", 46 | "fieldtype": "Check", 47 | "in_list_view": 1, 48 | "label": "Use Same Name" 49 | }, 50 | { 51 | "columns": 3, 52 | "default": "Pending", 53 | "fieldname": "status", 54 | "fieldtype": "Select", 55 | "in_list_view": 1, 56 | "label": "Approval Status", 57 | "options": "Pending\nApproved\nRejected", 58 | "read_only": 1 59 | }, 60 | { 61 | "columns": 2, 62 | "default": "0", 63 | "fieldname": "unsubscribe", 64 | "fieldtype": "Check", 65 | "in_list_view": 1, 66 | "label": "Unsubscribe" 67 | }, 68 | { 69 | "fieldname": "condition", 70 | "fieldtype": "Code", 71 | "label": "Condition" 72 | } 73 | ], 74 | "istable": 1, 75 | "links": [], 76 | "modified": "2020-11-07 09:26:58.463868", 77 | "modified_by": "Administrator", 78 | "module": "Event Streaming", 79 | "name": "Event Producer Document Type", 80 | "owner": "Administrator", 81 | "permissions": [], 82 | "quick_entry": 1, 83 | "sort_field": "modified", 84 | "sort_order": "DESC", 85 | "track_changes": 1 86 | } -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and contributors 2 | # License: MIT. See LICENSE 3 | 4 | # import frappe 5 | from frappe.model.document import Document 6 | 7 | 8 | class EventProducerDocumentType(Document): 9 | pass 10 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer_last_update/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/event_producer_last_update/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020, Frappe Technologies and contributors 2 | // For license information, please see license.txt 3 | 4 | frappe.ui.form.on("Event Producer Last Update", { 5 | // refresh: function(frm) { 6 | // } 7 | }); 8 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer_last_update/event_producer_last_update.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [], 3 | "autoname": "field:event_producer", 4 | "creation": "2020-10-26 12:53:11.940177", 5 | "doctype": "DocType", 6 | "editable_grid": 1, 7 | "engine": "InnoDB", 8 | "field_order": [ 9 | "event_producer", 10 | "last_update" 11 | ], 12 | "fields": [ 13 | { 14 | "fieldname": "event_producer", 15 | "fieldtype": "Data", 16 | "in_list_view": 1, 17 | "label": "Event Producer", 18 | "reqd": 1, 19 | "unique": 1 20 | }, 21 | { 22 | "fieldname": "last_update", 23 | "fieldtype": "Data", 24 | "label": "Last Update" 25 | } 26 | ], 27 | "in_create": 1, 28 | "index_web_pages_for_search": 1, 29 | "links": [], 30 | "modified": "2020-10-26 13:22:27.056599", 31 | "modified_by": "Administrator", 32 | "module": "Event Streaming", 33 | "name": "Event Producer Last Update", 34 | "owner": "Administrator", 35 | "permissions": [ 36 | { 37 | "create": 1, 38 | "delete": 1, 39 | "email": 1, 40 | "export": 1, 41 | "print": 1, 42 | "read": 1, 43 | "report": 1, 44 | "role": "System Manager", 45 | "share": 1, 46 | "write": 1 47 | } 48 | ], 49 | "read_only": 1, 50 | "sort_field": "modified", 51 | "sort_order": "DESC", 52 | "track_changes": 1 53 | } -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, Frappe Technologies and contributors 2 | # License: MIT. See LICENSE 3 | 4 | # import frappe 5 | from frappe.model.document import Document 6 | 7 | 8 | class EventProducerLastUpdate(Document): 9 | pass 10 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, Frappe Technologies and Contributors 2 | # License: MIT. See LICENSE 3 | # import frappe 4 | from frappe.tests.utils import FrappeTestCase 5 | 6 | 7 | class TestEventProducerLastUpdate(FrappeTestCase): 8 | pass 9 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_sync_log/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/event_sync_log/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_sync_log/event_sync_log.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019, Frappe Technologies and contributors 2 | // For license information, please see license.txt 3 | 4 | frappe.ui.form.on("Event Sync Log", { 5 | refresh: function (frm) { 6 | if (frm.doc.status == "Failed") { 7 | frm.add_custom_button(__("Resync"), function () { 8 | frappe.call({ 9 | method: "event_streaming.event_streaming.doctype.event_producer.event_producer.resync", 10 | args: { 11 | update: frm.doc, 12 | }, 13 | callback: function (r) { 14 | if (r.message) { 15 | frappe.msgprint(r.message); 16 | frm.set_value("status", r.message); 17 | frm.save(); 18 | } 19 | }, 20 | }); 21 | }); 22 | } 23 | }, 24 | }); 25 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_sync_log/event_sync_log.json: -------------------------------------------------------------------------------- 1 | { 2 | "creation": "2019-09-24 22:22:05.845089", 3 | "doctype": "DocType", 4 | "editable_grid": 1, 5 | "engine": "InnoDB", 6 | "field_order": [ 7 | "update_type", 8 | "ref_doctype", 9 | "docname", 10 | "column_break_4", 11 | "status", 12 | "event_producer", 13 | "producer_doc", 14 | "event_configurations_section", 15 | "use_same_name", 16 | "column_break_9", 17 | "mapping", 18 | "section_break_8", 19 | "data", 20 | "error" 21 | ], 22 | "fields": [ 23 | { 24 | "fieldname": "update_type", 25 | "fieldtype": "Select", 26 | "in_list_view": 1, 27 | "label": "Update Type", 28 | "options": "Create\nUpdate\nDelete", 29 | "read_only": 1 30 | }, 31 | { 32 | "fieldname": "ref_doctype", 33 | "fieldtype": "Link", 34 | "label": "Doctype", 35 | "options": "DocType", 36 | "read_only": 1 37 | }, 38 | { 39 | "fieldname": "docname", 40 | "fieldtype": "Data", 41 | "in_list_view": 1, 42 | "label": "Document Name", 43 | "options": "ref_doctype", 44 | "read_only": 1 45 | }, 46 | { 47 | "fieldname": "column_break_4", 48 | "fieldtype": "Column Break" 49 | }, 50 | { 51 | "fieldname": "status", 52 | "fieldtype": "Select", 53 | "in_list_view": 1, 54 | "label": "Status", 55 | "options": "\nSynced\nFailed", 56 | "read_only": 1 57 | }, 58 | { 59 | "fieldname": "event_producer", 60 | "fieldtype": "Data", 61 | "in_list_view": 1, 62 | "label": "Event Producer", 63 | "options": "Event Producer", 64 | "read_only": 1 65 | }, 66 | { 67 | "fieldname": "section_break_8", 68 | "fieldtype": "Section Break", 69 | "label": "Data" 70 | }, 71 | { 72 | "fieldname": "data", 73 | "fieldtype": "Code", 74 | "label": "Data", 75 | "read_only": 1 76 | }, 77 | { 78 | "fieldname": "producer_doc", 79 | "fieldtype": "Data", 80 | "label": "Producer Document Name", 81 | "read_only": 1 82 | }, 83 | { 84 | "depends_on": "eval:doc.status=='Failed'", 85 | "fieldname": "error", 86 | "fieldtype": "Code", 87 | "label": "Error", 88 | "read_only": 1 89 | }, 90 | { 91 | "fieldname": "event_configurations_section", 92 | "fieldtype": "Section Break", 93 | "label": "Event Configurations" 94 | }, 95 | { 96 | "default": "0", 97 | "fieldname": "use_same_name", 98 | "fieldtype": "Data", 99 | "label": "Use Same Name", 100 | "read_only": 1 101 | }, 102 | { 103 | "fieldname": "column_break_9", 104 | "fieldtype": "Column Break" 105 | }, 106 | { 107 | "fieldname": "mapping", 108 | "fieldtype": "Data", 109 | "label": "Mapping", 110 | "read_only": 1 111 | } 112 | ], 113 | "in_create": 1, 114 | "modified": "2019-10-07 13:22:10.401479", 115 | "modified_by": "Administrator", 116 | "module": "Event Streaming", 117 | "name": "Event Sync Log", 118 | "owner": "Administrator", 119 | "permissions": [ 120 | { 121 | "create": 1, 122 | "delete": 1, 123 | "email": 1, 124 | "export": 1, 125 | "print": 1, 126 | "read": 1, 127 | "report": 1, 128 | "role": "System Manager", 129 | "share": 1, 130 | "write": 1 131 | } 132 | ], 133 | "quick_entry": 1, 134 | "sort_field": "modified", 135 | "sort_order": "DESC", 136 | "track_changes": 1 137 | } -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_sync_log/event_sync_log.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and contributors 2 | # License: MIT. See LICENSE 3 | 4 | # import frappe 5 | from frappe.model.document import Document 6 | 7 | 8 | class EventSyncLog(Document): 9 | pass 10 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_sync_log/event_sync_log_list.js: -------------------------------------------------------------------------------- 1 | frappe.listview_settings["Event Sync Log"] = { 2 | get_indicator: function (doc) { 3 | var colors = { 4 | Failed: "red", 5 | Synced: "green", 6 | }; 7 | return [__(doc.status), colors[doc.status], "status,=," + doc.status]; 8 | }, 9 | }; 10 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_sync_log/test_event_sync_log.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies and Contributors 2 | # License: MIT. See LICENSE 3 | # import frappe 4 | from frappe.tests.utils import FrappeTestCase 5 | 6 | 7 | class TestEventSyncLog(FrappeTestCase): 8 | pass 9 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_update_log/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/event_update_log/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_update_log/event_update_log.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors 2 | // For license information, please see license.txt 3 | 4 | frappe.ui.form.on("Event Update Log", { 5 | // refresh: function(frm) { 6 | // } 7 | }); 8 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_update_log/event_update_log.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [], 3 | "creation": "2019-07-30 15:31:26.352527", 4 | "doctype": "DocType", 5 | "editable_grid": 1, 6 | "engine": "InnoDB", 7 | "field_order": [ 8 | "update_type", 9 | "ref_doctype", 10 | "docname", 11 | "data", 12 | "consumers" 13 | ], 14 | "fields": [ 15 | { 16 | "fieldname": "update_type", 17 | "fieldtype": "Select", 18 | "in_list_view": 1, 19 | "label": "Update Type", 20 | "options": "Create\nUpdate\nDelete", 21 | "read_only": 1 22 | }, 23 | { 24 | "fieldname": "ref_doctype", 25 | "fieldtype": "Link", 26 | "in_list_view": 1, 27 | "label": "DocType", 28 | "options": "DocType", 29 | "read_only": 1 30 | }, 31 | { 32 | "fieldname": "docname", 33 | "fieldtype": "Data", 34 | "in_list_view": 1, 35 | "label": "Document Name", 36 | "read_only": 1 37 | }, 38 | { 39 | "fieldname": "data", 40 | "fieldtype": "Code", 41 | "label": "Data", 42 | "read_only": 1 43 | }, 44 | { 45 | "fieldname": "consumers", 46 | "fieldtype": "Table MultiSelect", 47 | "label": "Consumers", 48 | "options": "Event Update Log Consumer", 49 | "read_only": 1 50 | } 51 | ], 52 | "in_create": 1, 53 | "links": [], 54 | "modified": "2020-09-04 07:31:52.599804", 55 | "modified_by": "Administrator", 56 | "module": "Event Streaming", 57 | "name": "Event Update Log", 58 | "owner": "Administrator", 59 | "permissions": [ 60 | { 61 | "create": 1, 62 | "delete": 1, 63 | "email": 1, 64 | "export": 1, 65 | "print": 1, 66 | "read": 1, 67 | "report": 1, 68 | "role": "System Manager", 69 | "share": 1, 70 | "write": 1 71 | } 72 | ], 73 | "quick_entry": 1, 74 | "sort_field": "modified", 75 | "sort_order": "DESC", 76 | "track_changes": 1 77 | } -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_update_log/event_update_log.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors 2 | # License: MIT. See LICENSE 3 | 4 | import frappe 5 | from frappe.model import no_value_fields, table_fields 6 | from frappe.model.document import Document 7 | from frappe.utils.background_jobs import get_jobs 8 | 9 | 10 | class EventUpdateLog(Document): 11 | def after_insert(self): 12 | """Send update notification updates to event consumers 13 | whenever update log is generated""" 14 | enqueued_method = ( 15 | "event_streaming.event_streaming.doctype.event_consumer.event_consumer.notify_event_consumers" 16 | ) 17 | jobs = get_jobs() 18 | if not jobs or enqueued_method not in jobs[frappe.local.site]: 19 | frappe.enqueue( 20 | enqueued_method, doctype=self.ref_doctype, queue="long", enqueue_after_commit=True 21 | ) 22 | 23 | 24 | def notify_consumers(doc, event): 25 | """called via hooks""" 26 | # make event update log for doctypes having event consumers 27 | if frappe.flags.in_install or frappe.flags.in_migrate: 28 | return 29 | 30 | consumers = check_doctype_has_consumers(doc.doctype) 31 | if consumers: 32 | if event == "after_insert": 33 | doc.flags.event_update_log = make_event_update_log(doc, update_type="Create") 34 | elif event == "on_trash": 35 | make_event_update_log(doc, update_type="Delete") 36 | else: 37 | # on_update 38 | # called after saving 39 | if not doc.flags.event_update_log: # if not already inserted 40 | diff = get_update(doc.get_doc_before_save(), doc) 41 | if diff: 42 | doc.diff = diff 43 | make_event_update_log(doc, update_type="Update") 44 | 45 | ENABLED_DOCTYPES_CACHE_KEY = "event_streaming_enabled_doctypes" 46 | 47 | def check_doctype_has_consumers(doctype: str) -> bool: 48 | """Check if doctype has event consumers for event streaming""" 49 | def fetch_from_db(): 50 | return frappe.get_all( 51 | "Event Consumer Document Type", 52 | filters={"ref_doctype": doctype, "status": "Approved", "unsubscribed": 0}, 53 | ignore_ddl=True, 54 | ) 55 | 56 | return bool(frappe.cache().hget(ENABLED_DOCTYPES_CACHE_KEY, doctype, fetch_from_db)) 57 | 58 | 59 | def get_update(old, new, for_child=False): 60 | """ 61 | Get document objects with updates only 62 | If there is a change, then returns a dict like: 63 | { 64 | "changed" : {fieldname1: new_value1, fieldname2: new_value2, }, 65 | "added" : {table_fieldname1: [{row_dict1}, {row_dict2}], }, 66 | "removed" : {table_fieldname1: [row_name1, row_name2], }, 67 | "row_changed" : {table_fieldname1: 68 | { 69 | child_fieldname1: new_val, 70 | child_fieldname2: new_val 71 | }, 72 | }, 73 | } 74 | """ 75 | if not new: 76 | return None 77 | 78 | out = frappe._dict(changed={}, added={}, removed={}, row_changed={}) 79 | for df in new.meta.fields: 80 | if df.fieldtype in no_value_fields and df.fieldtype not in table_fields: 81 | continue 82 | 83 | old_value, new_value = old.get(df.fieldname), new.get(df.fieldname) 84 | 85 | if df.fieldtype in table_fields: 86 | old_row_by_name, new_row_by_name = make_maps(old_value, new_value) 87 | out = check_for_additions(out, df, new_value, old_row_by_name) 88 | out = check_for_deletions(out, df, old_value, new_row_by_name) 89 | 90 | elif old_value != new_value: 91 | out.changed[df.fieldname] = new_value 92 | 93 | out = check_docstatus(out, old, new, for_child) 94 | if any((out.changed, out.added, out.removed, out.row_changed)): 95 | return out 96 | return None 97 | 98 | 99 | def make_event_update_log(doc, update_type): 100 | """Save update info for doctypes that have event consumers""" 101 | if update_type != "Delete": 102 | # diff for update type, doc for create type 103 | data = frappe.as_json(doc) if not doc.get("diff") else frappe.as_json(doc.diff) 104 | else: 105 | data = None 106 | return frappe.get_doc( 107 | { 108 | "doctype": "Event Update Log", 109 | "update_type": update_type, 110 | "ref_doctype": doc.doctype, 111 | "docname": doc.name, 112 | "data": data, 113 | } 114 | ).insert(ignore_permissions=True) 115 | 116 | 117 | def make_maps(old_value, new_value): 118 | """make maps""" 119 | old_row_by_name, new_row_by_name = {}, {} 120 | for d in old_value: 121 | old_row_by_name[d.name] = d 122 | for d in new_value: 123 | new_row_by_name[d.name] = d 124 | return old_row_by_name, new_row_by_name 125 | 126 | 127 | def check_for_additions(out, df, new_value, old_row_by_name): 128 | """check rows for additions, changes""" 129 | for _i, d in enumerate(new_value): 130 | if d.name in old_row_by_name: 131 | diff = get_update(old_row_by_name[d.name], d, for_child=True) 132 | if diff and diff.changed: 133 | if not out.row_changed.get(df.fieldname): 134 | out.row_changed[df.fieldname] = [] 135 | diff.changed["name"] = d.name 136 | out.row_changed[df.fieldname].append(diff.changed) 137 | else: 138 | if not out.added.get(df.fieldname): 139 | out.added[df.fieldname] = [] 140 | out.added[df.fieldname].append(d.as_dict()) 141 | return out 142 | 143 | 144 | def check_for_deletions(out, df, old_value, new_row_by_name): 145 | """check for deletions""" 146 | for d in old_value: 147 | if d.name not in new_row_by_name: 148 | if not out.removed.get(df.fieldname): 149 | out.removed[df.fieldname] = [] 150 | out.removed[df.fieldname].append(d.name) 151 | return out 152 | 153 | 154 | def check_docstatus(out, old, new, for_child): 155 | """docstatus changes""" 156 | if not for_child and old.docstatus != new.docstatus: 157 | out.changed["docstatus"] = new.docstatus 158 | return out 159 | 160 | 161 | def is_consumer_uptodate(update_log, consumer): 162 | """ 163 | Checks if Consumer has read all the UpdateLogs before the specified update_log 164 | :param update_log: The UpdateLog Doc in context 165 | :param consumer: The EventConsumer doc 166 | """ 167 | if update_log.update_type == "Create": 168 | # consumer is obviously up to date 169 | return True 170 | 171 | prev_logs = frappe.get_all( 172 | "Event Update Log", 173 | filters={ 174 | "ref_doctype": update_log.ref_doctype, 175 | "docname": update_log.docname, 176 | "creation": ["<", update_log.creation], 177 | }, 178 | order_by="creation desc", 179 | limit_page_length=1, 180 | ) 181 | 182 | if not len(prev_logs): 183 | return False 184 | 185 | prev_log_consumers = frappe.get_all( 186 | "Event Update Log Consumer", 187 | fields=["consumer"], 188 | filters={ 189 | "parent": prev_logs[0].name, 190 | "parenttype": "Event Update Log", 191 | "consumer": consumer.name, 192 | }, 193 | ) 194 | 195 | return len(prev_log_consumers) > 0 196 | 197 | 198 | def mark_consumer_read(update_log_name, consumer_name): 199 | """ 200 | This function appends the Consumer to the list of Consumers that has 'read' an Update Log 201 | """ 202 | update_log = frappe.get_doc("Event Update Log", update_log_name) 203 | if len([x for x in update_log.consumers if x.consumer == consumer_name]): 204 | return 205 | 206 | frappe.get_doc( 207 | frappe._dict( 208 | doctype="Event Update Log Consumer", 209 | consumer=consumer_name, 210 | parent=update_log_name, 211 | parenttype="Event Update Log", 212 | parentfield="consumers", 213 | ) 214 | ).insert(ignore_permissions=True) 215 | 216 | 217 | def get_unread_update_logs(consumer_name, dt, dn): 218 | """ 219 | Get old logs unread by the consumer on a particular document 220 | """ 221 | already_consumed = [ 222 | x[0] 223 | for x in frappe.db.sql( 224 | """ 225 | SELECT 226 | update_log.name 227 | FROM `tabEvent Update Log` update_log 228 | JOIN `tabEvent Update Log Consumer` consumer ON consumer.parent = %(log_name)s 229 | WHERE 230 | consumer.consumer = %(consumer)s 231 | AND update_log.ref_doctype = %(dt)s 232 | AND update_log.docname = %(dn)s 233 | """, 234 | { 235 | "consumer": consumer_name, 236 | "dt": dt, 237 | "dn": dn, 238 | "log_name": "update_log.name" 239 | if frappe.conf.db_type == "mariadb" 240 | else "CAST(update_log.name AS VARCHAR)", 241 | }, 242 | as_dict=0, 243 | ) 244 | ] 245 | 246 | logs = frappe.get_all( 247 | "Event Update Log", 248 | fields=["update_type", "ref_doctype", "docname", "data", "name", "creation"], 249 | filters={"ref_doctype": dt, "docname": dn, "name": ["not in", already_consumed]}, 250 | order_by="creation", 251 | ) 252 | 253 | return logs 254 | 255 | 256 | @frappe.whitelist() 257 | def get_update_logs_for_consumer(event_consumer, doctypes, last_update): 258 | """ 259 | Fetches all the UpdateLogs for the consumer 260 | It will inject old un-consumed Update Logs if a doc was just found to be accessible to the Consumer 261 | """ 262 | 263 | if isinstance(doctypes, str): 264 | doctypes = frappe.parse_json(doctypes) 265 | 266 | from event_streaming.event_streaming.doctype.event_consumer.event_consumer import has_consumer_access 267 | 268 | consumer = frappe.get_doc("Event Consumer", event_consumer) 269 | docs = frappe.get_list( 270 | doctype="Event Update Log", 271 | filters={"ref_doctype": ("in", doctypes), "creation": (">", last_update)}, 272 | fields=["update_type", "ref_doctype", "docname", "data", "name", "creation"], 273 | order_by="creation desc", 274 | ) 275 | 276 | result = [] 277 | to_update_history = [] 278 | for d in docs: 279 | if (d.ref_doctype, d.docname) in to_update_history: 280 | # will be notified by background jobs 281 | continue 282 | 283 | if not has_consumer_access(consumer=consumer, update_log=d): 284 | continue 285 | 286 | if not is_consumer_uptodate(d, consumer): 287 | to_update_history.append((d.ref_doctype, d.docname)) 288 | # get_unread_update_logs will have the current log 289 | old_logs = get_unread_update_logs(consumer.name, d.ref_doctype, d.docname) 290 | if old_logs: 291 | old_logs.reverse() 292 | result.extend(old_logs) 293 | else: 294 | result.append(d) 295 | 296 | for d in result: 297 | mark_consumer_read(update_log_name=d.name, consumer_name=consumer.name) 298 | 299 | result.reverse() 300 | return result 301 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_update_log/test_event_update_log.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors 2 | # License: MIT. See LICENSE 3 | # import frappe 4 | from frappe.tests.utils import FrappeTestCase 5 | 6 | 7 | class TestEventUpdateLog(FrappeTestCase): 8 | pass 9 | -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_update_log_consumer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/event_streaming/doctype/event_update_log_consumer/__init__.py -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [], 3 | "creation": "2020-06-30 10:54:53.301787", 4 | "doctype": "DocType", 5 | "editable_grid": 1, 6 | "engine": "InnoDB", 7 | "field_order": [ 8 | "consumer" 9 | ], 10 | "fields": [ 11 | { 12 | "fieldname": "consumer", 13 | "fieldtype": "Link", 14 | "in_list_view": 1, 15 | "label": "Consumer", 16 | "options": "Event Consumer", 17 | "reqd": 1 18 | } 19 | ], 20 | "istable": 1, 21 | "links": [], 22 | "modified": "2020-06-30 10:54:53.301787", 23 | "modified_by": "Administrator", 24 | "module": "Event Streaming", 25 | "name": "Event Update Log Consumer", 26 | "owner": "Administrator", 27 | "permissions": [], 28 | "quick_entry": 1, 29 | "sort_field": "modified", 30 | "sort_order": "DESC", 31 | "track_changes": 1 32 | } -------------------------------------------------------------------------------- /event_streaming/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, Frappe Technologies and contributors 2 | # License: MIT. See LICENSE 3 | 4 | # import frappe 5 | from frappe.model.document import Document 6 | 7 | 8 | class EventUpdateLogConsumer(Document): 9 | pass 10 | -------------------------------------------------------------------------------- /event_streaming/hooks.py: -------------------------------------------------------------------------------- 1 | from . import __version__ as app_version 2 | 3 | app_name = "event_streaming" 4 | app_title = "Event Streaming" 5 | app_publisher = "Frappe Technologies" 6 | app_description = "Event Streaming for frappe" 7 | app_email = "hello@frappe.io" 8 | app_license = "MIT" 9 | 10 | # Includes in 11 | # ------------------ 12 | 13 | # include js, css files in header of desk.html 14 | # app_include_css = "/assets/event_streaming/css/event_streaming.css" 15 | # app_include_js = "/assets/event_streaming/js/event_streaming.js" 16 | 17 | # include js, css files in header of web template 18 | # web_include_css = "/assets/event_streaming/css/event_streaming.css" 19 | # web_include_js = "/assets/event_streaming/js/event_streaming.js" 20 | 21 | # include custom scss in every website theme (without file extension ".scss") 22 | # website_theme_scss = "event_streaming/public/scss/website" 23 | 24 | # include js, css files in header of web form 25 | # webform_include_js = {"doctype": "public/js/doctype.js"} 26 | # webform_include_css = {"doctype": "public/css/doctype.css"} 27 | 28 | # include js in page 29 | # page_js = {"page" : "public/js/file.js"} 30 | 31 | # include js in doctype views 32 | # doctype_js = {"doctype" : "public/js/doctype.js"} 33 | # doctype_list_js = {"doctype" : "public/js/doctype_list.js"} 34 | # doctype_tree_js = {"doctype" : "public/js/doctype_tree.js"} 35 | # doctype_calendar_js = {"doctype" : "public/js/doctype_calendar.js"} 36 | 37 | # Home Pages 38 | # ---------- 39 | 40 | # application home page (will override Website Settings) 41 | # home_page = "login" 42 | 43 | # website user home page (by Role) 44 | # role_home_page = { 45 | # "Role": "home_page" 46 | # } 47 | 48 | # Generators 49 | # ---------- 50 | 51 | # automatically create page for each record of this doctype 52 | # website_generators = ["Web Page"] 53 | 54 | # Jinja 55 | # ---------- 56 | 57 | # add methods and filters to jinja environment 58 | # jinja = { 59 | # "methods": "event_streaming.utils.jinja_methods", 60 | # "filters": "event_streaming.utils.jinja_filters" 61 | # } 62 | 63 | # Installation 64 | # ------------ 65 | 66 | # before_install = "event_streaming.install.before_install" 67 | # after_install = "event_streaming.install.after_install" 68 | 69 | # Uninstallation 70 | # ------------ 71 | 72 | # before_uninstall = "event_streaming.uninstall.before_uninstall" 73 | # after_uninstall = "event_streaming.uninstall.after_uninstall" 74 | 75 | # Desk Notifications 76 | # ------------------ 77 | # See frappe.core.notifications.get_notification_config 78 | 79 | # notification_config = "event_streaming.notifications.get_notification_config" 80 | 81 | # Permissions 82 | # ----------- 83 | # Permissions evaluated in scripted ways 84 | 85 | # permission_query_conditions = { 86 | # "Event": "frappe.desk.doctype.event.event.get_permission_query_conditions", 87 | # } 88 | # 89 | # has_permission = { 90 | # "Event": "frappe.desk.doctype.event.event.has_permission", 91 | # } 92 | 93 | # DocType Class 94 | # --------------- 95 | # Override standard doctype classes 96 | 97 | # override_doctype_class = { 98 | # "ToDo": "custom_app.overrides.CustomToDo" 99 | # } 100 | 101 | # Document Events 102 | # --------------- 103 | # Hook on document methods and events 104 | 105 | doc_events = { 106 | "*": { 107 | "after_insert": "event_streaming.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", 108 | "on_update": "event_streaming.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", 109 | "on_cancel": "event_streaming.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", 110 | "on_trash": "event_streaming.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" 111 | } 112 | } 113 | 114 | # Scheduled Tasks 115 | # --------------- 116 | 117 | # scheduler_events = { 118 | # "all": [ 119 | # "event_streaming.tasks.all" 120 | # ], 121 | # "daily": [ 122 | # "event_streaming.tasks.daily" 123 | # ], 124 | # "hourly": [ 125 | # "event_streaming.tasks.hourly" 126 | # ], 127 | # "weekly": [ 128 | # "event_streaming.tasks.weekly" 129 | # ], 130 | # "monthly": [ 131 | # "event_streaming.tasks.monthly" 132 | # ], 133 | # } 134 | 135 | # Testing 136 | # ------- 137 | 138 | # before_tests = "event_streaming.install.before_tests" 139 | 140 | # Overriding Methods 141 | # ------------------------------ 142 | # 143 | # override_whitelisted_methods = { 144 | # "frappe.desk.doctype.event.event.get_events": "event_streaming.event.get_events" 145 | # } 146 | # 147 | # each overriding function accepts a `data` argument; 148 | # generated from the base implementation of the doctype dashboard, 149 | # along with any modifications made in other Frappe apps 150 | # override_doctype_dashboards = { 151 | # "Task": "event_streaming.task.get_dashboard_data" 152 | # } 153 | 154 | # exempt linked doctypes from being automatically cancelled 155 | # 156 | # auto_cancel_exempted_doctypes = ["Auto Repeat"] 157 | 158 | 159 | # User Data Protection 160 | # -------------------- 161 | 162 | # user_data_fields = [ 163 | # { 164 | # "doctype": "{doctype_1}", 165 | # "filter_by": "{filter_by}", 166 | # "redact_fields": ["{field_1}", "{field_2}"], 167 | # "partial": 1, 168 | # }, 169 | # { 170 | # "doctype": "{doctype_2}", 171 | # "filter_by": "{filter_by}", 172 | # "partial": 1, 173 | # }, 174 | # { 175 | # "doctype": "{doctype_3}", 176 | # "strict": False, 177 | # }, 178 | # { 179 | # "doctype": "{doctype_4}" 180 | # } 181 | # ] 182 | 183 | # Authentication and authorization 184 | # -------------------------------- 185 | 186 | # auth_hooks = [ 187 | # "event_streaming.auth.validate" 188 | # ] 189 | -------------------------------------------------------------------------------- /event_streaming/modules.txt: -------------------------------------------------------------------------------- 1 | Event Streaming -------------------------------------------------------------------------------- /event_streaming/patches.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/patches.txt -------------------------------------------------------------------------------- /event_streaming/public/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/public/.gitkeep -------------------------------------------------------------------------------- /event_streaming/templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/templates/__init__.py -------------------------------------------------------------------------------- /event_streaming/templates/pages/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frappe/event_streaming/2dddfbb74d44c956be6481e0abb4f59fae4f4787/event_streaming/templates/pages/__init__.py -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | License: MIT -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # frappe -- https://github.com/frappe/frappe is installed via 'bench init' -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open("requirements.txt") as f: 4 | install_requires = f.read().strip().split("\n") 5 | 6 | # get version from __version__ variable in event_streaming/__init__.py 7 | from event_streaming import __version__ as version 8 | 9 | setup( 10 | name="event_streaming", 11 | version=version, 12 | description="Event Streaming for frappe", 13 | author="Frappe Technologies", 14 | author_email="hello@frappe.io", 15 | packages=find_packages(), 16 | zip_safe=False, 17 | include_package_data=True, 18 | install_requires=install_requires 19 | ) 20 | --------------------------------------------------------------------------------