├── .gitignore ├── ckanext ├── __init__.py └── drupal │ ├── __init__.py │ ├── tests.py │ └── plugin.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /ckanext/__init__.py: -------------------------------------------------------------------------------- 1 | # this is a namespace package 2 | try: 3 | import pkg_resources 4 | pkg_resources.declare_namespace(__name__) 5 | except ImportError: 6 | import pkgutil 7 | __path__ = pkgutil.extend_path(__path__, __name__) 8 | -------------------------------------------------------------------------------- /ckanext/drupal/__init__.py: -------------------------------------------------------------------------------- 1 | # this is a namespace package 2 | try: 3 | import pkg_resources 4 | pkg_resources.declare_namespace(__name__) 5 | except ImportError: 6 | import pkgutil 7 | __path__ = pkgutil.extend_path(__path__, __name__) 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import sys, os 3 | 4 | version = '0.1' 5 | 6 | setup( 7 | name='ckanext-drupal', 8 | version=version, 9 | description="drupal integration", 10 | long_description="""\ 11 | """, 12 | classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers 13 | keywords='', 14 | author='David Raznick', 15 | author_email='kindly@gmail.com', 16 | url='', 17 | license='agpl', 18 | packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), 19 | namespace_packages=['ckanext', 'ckanext.drupal'], 20 | include_package_data=True, 21 | zip_safe=False, 22 | install_requires=[ 23 | # -*- Extra requirements: -*- 24 | ], 25 | entry_points=\ 26 | """ 27 | [ckan.plugins] 28 | # Add plugins here, eg 29 | drupal=ckanext.drupal.plugin:Drupal 30 | """, 31 | ) 32 | -------------------------------------------------------------------------------- /ckanext/drupal/tests.py: -------------------------------------------------------------------------------- 1 | from ckan.lib.create_test_data import CreateTestData 2 | import ckan.model as model 3 | from ckan.tests import WsgiAppCase 4 | import json 5 | from ckan import plugins 6 | from pprint import pprint, pformat 7 | from pylons import config 8 | from sqlalchemy import MetaData, create_engine, Table 9 | 10 | class TestAction(WsgiAppCase): 11 | 12 | @classmethod 13 | def setup_class(cls): 14 | model.repo.rebuild_db() 15 | CreateTestData.create() 16 | plugins.load('drupal') 17 | from ckan.plugins import PluginImplementations 18 | from ckan.plugins.interfaces import IConfigurer 19 | for plugin in PluginImplementations(IConfigurer): 20 | plugin.update_config(config) 21 | 22 | url = config['drupal.db_url'] 23 | cls.engine = create_engine(url, echo=True) 24 | cls.metadata = MetaData(cls.engine) 25 | cls.package_table = Table('ckan_package', 26 | cls.metadata, 27 | autoload = True) 28 | cls.extra_table = Table('ckan_package_extra', 29 | cls.metadata, 30 | autoload = True) 31 | cls.resource_table = Table('ckan_package_extra', 32 | cls.metadata, 33 | autoload = True) 34 | 35 | 36 | def test_01_create_update_package(self): 37 | 38 | package = { 39 | 'nid': 1, 40 | 'vid': 1, 41 | 'author': None, 42 | 'author_email': None, 43 | 'extras': [{'key': u'original media','value': u'"book"'}], 44 | 'license_id': u'other-open', 45 | 'maintainer': None, 46 | 'maintainer_email': None, 47 | 'name': u'moo1', 48 | 'notes': u'Some test now', 49 | 'resources': [{'alt_url': u'alt123', 50 | 'description': u'Full text.', 51 | 'extras': {u'alt_url': u'alt123', u'size': u'123'}, 52 | 'format': u'plain text', 53 | 'hash': u'abc123', 54 | 'position': 0, 55 | 'url': u'http://www.annakarenina.com/download/'}, 56 | {'alt_url': u'alt345', 57 | 'description': u'Index of the novel', 58 | 'extras': {u'alt_url': u'alt345', u'size': u'345'}, 59 | 'format': u'json', 60 | 'hash': u'def456', 61 | 'position': 1, 62 | 'url': u'http://www.annakarenina.com/index.json'}], 63 | 'tags': [{'name': u'russian'}, {'name': u'tolstoy'}], 64 | 'title': u'A Novel By Tolstoy', 65 | 'url': u'http://www.annakarenina.com', 66 | 'version': u'0.7a' 67 | } 68 | 69 | wee = json.dumps(package) 70 | postparams = '%s=1' % json.dumps(package) 71 | res = self.app.post('/api/action/drupal_package_create', params=postparams, 72 | extra_environ={'Authorization': 'tester'}) 73 | package_created = json.loads(res.body)['result'] 74 | 75 | 76 | package_created['name'] = 'moo2' 77 | postparams = '%s=1' % json.dumps(package_created) 78 | res = self.app.post('/api/action/drupal_package_update', params=postparams, 79 | extra_environ={'Authorization': 'tester'}) 80 | 81 | package_updated = json.loads(res.body)['result'] 82 | package_updated.pop('revision_id') 83 | package_updated.pop('revision_timestamp') 84 | package_updated.pop('revision_message') 85 | package_created.pop('revision_id') 86 | package_created.pop('revision_timestamp') 87 | package_created.pop('revision_message') 88 | pprint(package_updated) 89 | pprint(package_created) 90 | assert package_updated == package_created#, (pformat(json.loads(res.body)), pformat(package_created['result'])) 91 | 92 | -------------------------------------------------------------------------------- /ckanext/drupal/plugin.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from sqlalchemy import types, Column, Table 3 | from sqlalchemy.sql import select, and_ 4 | from sqlalchemy import MetaData, create_engine 5 | import json 6 | import time 7 | 8 | from ckan.plugins import IConfigurer, ISession 9 | from ckan.plugins import implements, SingletonPlugin 10 | import ckan.model as model 11 | from ckan.logic.action import create, update 12 | from ckan.controllers.api import ApiController 13 | 14 | def drupal_package_create(context, data_dict): 15 | 16 | session = context['model'].Session 17 | context['nid'] = data_dict.pop('nid') 18 | context['vid'] = data_dict.pop('vid') 19 | package_create = create.package_create(context, data_dict) 20 | package_create['nid'] = context['nid'] 21 | package_create['vid'] = context['vid'] 22 | package_create['revision_message'] = '%s-%s'%(session.revision.id,session.revision.message) 23 | return package_create 24 | 25 | def drupal_package_update(context, data_dict): 26 | session = context['model'].Session 27 | context['nid'] = data_dict.pop('nid') 28 | context['vid'] = data_dict.pop('vid') 29 | package_create = update.package_update(context, data_dict) 30 | package_create['nid'] = context['nid'] 31 | package_create['vid'] = context['vid'] 32 | package_create['revision_message'] = '%s-%s'%(session.revision.id,session.revision.message) 33 | return package_create 34 | 35 | class Drupal(SingletonPlugin): 36 | '''initial test of plugin''' 37 | implements(IConfigurer) 38 | implements(ISession, inherit=True) 39 | 40 | def get_package_row(self, conn, package_id): 41 | return conn.execute( 42 | select( 43 | [self.package_table], 44 | self.package_table.c.id == package_id 45 | ) 46 | ).fetchone() 47 | 48 | def update_drupal(self, session, conn): 49 | 50 | obj_cache = session._object_cache 51 | new = obj_cache['new'] 52 | changed = obj_cache['changed'] 53 | deleted = obj_cache['deleted'] 54 | nid = session._context['nid'] 55 | vid = session._context['vid'] 56 | 57 | try: 58 | update_date = int( 59 | time.mktime(session.revision.timestamp.timetuple()) 60 | ) 61 | except AttributeError: 62 | update_date = int(time.time()) 63 | 64 | package_rows = {} 65 | package_tags = {} 66 | 67 | inserts = [] 68 | updates = [] 69 | deletes = [] 70 | 71 | for obj in new: 72 | if hasattr(obj, 'state') and 'pending' in obj.state: 73 | continue 74 | if hasattr(obj, 'current') and obj.current <> '1': 75 | continue 76 | if isinstance(obj, (model.Package, model.PackageRevision)): 77 | insert = self.add_insert(obj, self.package_table) 78 | package_rows[insert['id']] = insert 79 | inserts.append(insert) 80 | if isinstance(obj, (model.Resource, model.ResourceRevision)): 81 | inserts.append(self.add_insert(obj, self.resource_table)) 82 | if isinstance(obj, (model.PackageExtra, model.PackageExtraRevision)): 83 | inserts.append(self.add_insert(obj, self.package_extra_table)) 84 | 85 | for obj in changed: 86 | if hasattr(obj, 'state') and 'pending' in obj.state: 87 | continue 88 | if hasattr(obj, 'current') and obj.current <> '1': 89 | continue 90 | if isinstance(obj, (model.Package, model.PackageRevision)): 91 | update = self.add_update(obj, self.package_table) 92 | package_row = package_rows.get( 93 | obj.id, 94 | self.get_package_row(conn, obj.id) 95 | ) 96 | package_rows[update['id']] = update 97 | updates.append(update) 98 | if isinstance(obj, (model.Resource, model.ResourceRevision)): 99 | if obj.state == 'deleted': 100 | deletes.append(self.add_delete(obj, self.resource_table, conn)) 101 | else: 102 | updates.append(self.add_update(obj, self.resource_table)) 103 | if isinstance(obj, (model.PackageExtra, model.PackageExtraRevision)): 104 | if obj.state == 'deleted': 105 | deletes.append(self.add_delete(obj, self.package_extra_table, conn)) 106 | else: 107 | updates.append(self.add_update(obj, self.package_extra_table)) 108 | 109 | for obj in deleted: 110 | if hasattr(obj, 'state') and 'pending' in obj.state: 111 | continue 112 | if hasattr(obj, 'current') and obj.current <> '1': 113 | continue 114 | if isinstance(obj, (model.Package, model.PackageRevision)): 115 | delete = self.add_delete(obj, self.package_table) 116 | package_row = self.get_package_row(conn, obj.id) 117 | delete['nid'] = delete['nid'] 118 | package_rows[delete['id']] = delete 119 | deletes.append(delete) 120 | if isinstance(obj, (model.PackageExtra, model.PackageExtraRevision)): 121 | deletes.append(self.add_delete(obj, self.package_extra_table, conn)) 122 | if isinstance(obj, (model.Resource, model.ResourceRevision)): 123 | deletes.append(self.add_delete(obj, self.resource_table, conn)) 124 | 125 | for row in inserts + updates + deletes: 126 | if 'package_id' in row: 127 | package_id = row['package_id'] 128 | else: 129 | package_id = row['id'] 130 | if package_id in package_rows: 131 | package_rows[package_id]['update_date'] = update_date 132 | else: 133 | package_row = self.get_package_row(conn, package_id) 134 | update = {'__table': self.package_table, 135 | 'id': package_id, 136 | 'update_date': update_date} 137 | updates.append(update) 138 | package_rows[package_id] = update 139 | 140 | for row in inserts: 141 | table = row.pop('__table') 142 | row.update({'nid':nid, 'vid':vid}) 143 | conn.execute(table.insert().values(**row)) 144 | 145 | for row in updates: 146 | table = row.pop('__table') 147 | id = row.pop('id') 148 | row.update({'nid':nid, 'vid':vid}) 149 | conn.execute(table.update().where(table.c.id==id).values(**row)) 150 | 151 | for row in deletes: 152 | table = row.pop('__table') 153 | conn.execute(table.delete().where(table.c.id==id)) 154 | 155 | 156 | def add_insert(self, obj, table): 157 | 158 | insert = {'__table': table} 159 | for column in table.c: 160 | value = getattr(obj, column.name, None) 161 | if value is not None: 162 | insert[column.name] = value 163 | if isinstance(obj, model.Resource): 164 | insert['package_id'] = obj.resource_group.package_id 165 | insert['extras'] = json.dumps(insert['extras']) 166 | if isinstance(obj, model.ResourceRevision): 167 | insert['package_id'] = obj.coninuity.resource_group.package_id 168 | insert['extras'] = json.dumps(insert['extras']) 169 | return insert 170 | 171 | def add_update(self, obj, table): 172 | 173 | update = {'__table': table} 174 | for column in table.c: 175 | value = getattr(obj, column.name, None) 176 | if value is not None: 177 | update[column.name] = value 178 | if isinstance(obj, model.Resource): 179 | update['package_id'] = obj.resource_group.package_id 180 | update['extras'] = json.dumps(update['extras']) 181 | if isinstance(obj, model.ResourceRevision): 182 | update['package_id'] = obj.continuity.resource_group.package_id 183 | update['extras'] = json.dumps(update['extras']) 184 | return update 185 | 186 | def add_delete(self, obj, table, conn): 187 | 188 | delete = {'__table': table} 189 | for column in table.c: 190 | value = getattr(obj, column.name, None) 191 | if value is not None: 192 | delete[column.name] = value 193 | if isinstance(obj, (model.Resource, model.ResourceRevision)): 194 | package_id = conn.execute( 195 | select( 196 | [self.resource_table], 197 | self.resource_table.c.id == obj.id 198 | ) 199 | ).fetchone()["package_id"] 200 | delete["package_id"] = package_id 201 | return delete 202 | 203 | def before_commit(self, session): 204 | session.flush() 205 | if not hasattr(session, '_object_cache'): 206 | return 207 | conn = self.engine.connect() 208 | trans = conn.begin() 209 | try: 210 | self.update_drupal(session, conn) 211 | trans.commit() 212 | except: 213 | trans.rollback() 214 | session.rollback() 215 | raise 216 | finally: 217 | conn.close() 218 | 219 | def update_config(self, config): 220 | config['ckan.site_title'] = 'CKAN-Drupal' 221 | 222 | url = config['drupal.db_url'] 223 | 224 | self.engine = create_engine(url, echo=True) 225 | self.metadata = MetaData(self.engine) 226 | 227 | PACKAGE_NAME_MAX_LENGTH = 100 228 | PACKAGE_VERSION_MAX_LENGTH = 100 229 | 230 | ApiController.register_action('drupal_package_create', 231 | drupal_package_create) 232 | ApiController.register_action('drupal_package_update', 233 | drupal_package_update) 234 | 235 | self.package_table = Table('ckan_package', self.metadata, 236 | Column('nid', types.Integer, unique=True), 237 | Column('vid', types.Integer, unique=True), 238 | Column('id', types.Unicode(100), primary_key=True), 239 | Column('name', types.Unicode(PACKAGE_NAME_MAX_LENGTH), 240 | nullable=False, unique=True), 241 | Column('title', types.UnicodeText), 242 | Column('version', types.Unicode(PACKAGE_VERSION_MAX_LENGTH)), 243 | Column('url', types.UnicodeText), 244 | Column('author', types.UnicodeText), 245 | Column('author_email', types.UnicodeText), 246 | Column('maintainer', types.UnicodeText), 247 | Column('maintainer_email', types.UnicodeText), 248 | Column('notes', types.UnicodeText), 249 | Column('license_id', types.UnicodeText), 250 | Column('update_date', types.Integer), 251 | Column('state', types.UnicodeText), 252 | Column('completed', types.Boolean), 253 | ) 254 | 255 | self.resource_table = Table( 256 | 'ckan_resource', self.metadata, 257 | Column('nid', types.Integer), 258 | Column('vid', types.Integer), 259 | ## cache of package id to make things easier 260 | Column('package_id', types.UnicodeText), 261 | ## 262 | Column('id', types.Unicode(100), primary_key=True), 263 | Column('resource_group_id', types.UnicodeText), 264 | Column('package_id', types.UnicodeText), 265 | Column('url', types.UnicodeText, nullable=False), 266 | Column('format', types.UnicodeText), 267 | Column('description', types.UnicodeText), 268 | Column('hash', types.UnicodeText), 269 | Column('position', types.Integer), 270 | Column('extras', types.UnicodeText), 271 | ) 272 | 273 | self.package_extra_table = Table('ckan_package_extra', self.metadata, 274 | Column('nid', types.Integer), 275 | Column('vid', types.Integer), 276 | Column('id', types.Unicode(100), primary_key=True), 277 | Column('package_id', types.UnicodeText), 278 | Column('key', types.UnicodeText), 279 | Column('value', types.UnicodeText), 280 | ) 281 | 282 | self.metadata.create_all(self.engine) 283 | 284 | --------------------------------------------------------------------------------