├── .gitignore ├── emptyNVivo10Win.nvp ├── emptyNVivo11Win.nvp ├── emptyNVivo10Mac.nvpx ├── emptyNVivo11Mac.nvpx ├── requirements.txt ├── helpers ├── sqlanysrv.sh ├── mssqlSave.bat ├── mssqlCreate.bat ├── mssqlAttach.bat └── sqlanyenv.sh ├── DeleteAllData.py ├── tests └── testnodes.csv ├── DropItemForeignKeys.py ├── mssqlList.py ├── editUser.py ├── mssqlCreate.py ├── ACKNOWLEDGEMENTS.md ├── mssqlAttach.py ├── mssqlSave.py ├── mssqlDrop.py ├── NVivo database structure notes.txt ├── Translate.py ├── DataTypes.py ├── DenormaliseDB.py ├── Norm2RQDA.py ├── NormaliseDB.py ├── extractTagging.py ├── RQDA2Norm.py ├── NVivo2RQDADB.py ├── saveSources.py ├── RQDA2NVivoDB.py ├── regressionTest.sh ├── editSourceAttribute.py ├── editNodeAttribute.py ├── AdjustDate.py ├── editProject.py ├── NVP2RQDA.py ├── RQDA2NVP.py ├── Subtract.py ├── querySource.py ├── NVPX2RQDA.py ├── nvpn2bqda.py ├── mssqlTools.py ├── RQDA2NVPX.py ├── NormaliseNVP.py ├── nvpn2nvp.py ├── editNodeCategory.py ├── editSourceCategory.py ├── NormaliseNVPX.py ├── tagSpeakers.py ├── queryTagging.py ├── nvpn2nvpx.py ├── tagNounPhrases.py └── NVivoNorm.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /emptyNVivo10Win.nvp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BarraQDA/nvivotools/HEAD/emptyNVivo10Win.nvp -------------------------------------------------------------------------------- /emptyNVivo11Win.nvp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BarraQDA/nvivotools/HEAD/emptyNVivo11Win.nvp -------------------------------------------------------------------------------- /emptyNVivo10Mac.nvpx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BarraQDA/nvivotools/HEAD/emptyNVivo10Mac.nvpx -------------------------------------------------------------------------------- /emptyNVivo11Mac.nvpx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BarraQDA/nvivotools/HEAD/emptyNVivo11Mac.nvpx -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | dateutils 2 | future 3 | csv 4 | pdfminer 5 | Pillow 6 | pymssql 7 | python-dateutil 8 | SQLAlchemy 9 | git+git://github.com/BarraQDA/sqlalchemy-sqlany 10 | -------------------------------------------------------------------------------- /helpers/sqlanysrv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2019 Jonathan Schultz 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | if test -f "$binpath"/sa_config.sh; then 19 | . "$binpath"/sa_config.sh 20 | fi 21 | "$dbeng" "$@" 22 | -------------------------------------------------------------------------------- /DeleteAllData.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from sqlalchemy import exc, TypeDecorator, CHAR, String, create_engine, MetaData, bindparam 5 | from sqlalchemy.engine import reflection 6 | import warnings 7 | import sys 8 | import os 9 | import argparse 10 | import uuid 11 | 12 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 13 | 14 | try: 15 | parser = argparse.ArgumentParser(description='Delete all data leaving only database structure.') 16 | 17 | parser.add_argument('database', type=str, 18 | help='Path of database from which to delete data.') 19 | 20 | args = parser.parse_args() 21 | 22 | mssql.ischema_names['xml'] = UUID 23 | 24 | databasedb = create_engine(args.database) 25 | databasemd = MetaData(bind=databasedb) 26 | databasemd.reflect(databasedb) 27 | 28 | for table in reversed(databasemd.sorted_tables): 29 | databasedb.execute(table.delete()) 30 | 31 | except exc.SQLAlchemyError: 32 | raise 33 | -------------------------------------------------------------------------------- /tests/testnodes.csv: -------------------------------------------------------------------------------- 1 | Name,Parent,Description,Category,Aggregate,Color,integer attribute,Decimal attribute,Boolean attribute,Date attribute,Time attribute,Datetime attribute,Text attribute 2 | With attributes,,Node with all attribute types,All attribute types,,,123,123.456,TRUE,1967-12-10,16:20:00,01/03/16 09:15,Now is the time… 3 | Top level no category,,,,,,,,,,,, 4 | Second level not aggregating no category,Top level no category,,,,,,,,,,, 5 | Third level not aggregating no category,Second level not aggregating no category,,,,,,,,,,, 6 | Second level aggregating no category,Top level no category,,,TRUE,,,,,,,, 7 | Third level aggregating no category,Second level aggregating no category,,,TRUE,,,,,,,, 8 | Top level with category,,,Random category,,,,,,,,, 9 | Second level not aggregating with category,Top level with category,,Random category,,,,,,,,, 10 | Third level not aggregating with category,Second level not aggregating with category,,Random category,,,,,,,,, 11 | Second level aggregating with category,Top level with category,,Random category,TRUE,,,,,,,, 12 | Third level aggregating with category,Second level aggregating with category,,Random category,TRUE,,,,,,,, 13 | -------------------------------------------------------------------------------- /helpers/mssqlSave.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem Copyright 2016 Jonathan Schultz 4 | rem 5 | rem This program is free software: you can redistribute it and/or modify 6 | rem it under the terms of the GNU General Public License as published by 7 | rem the Free Software Foundation, either version 3 of the License, or 8 | rem (at your option) any later version. 9 | rem 10 | rem This program is distributed in the hope that it will be useful, 11 | rem but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | rem MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | rem GNU General Public License for more details. 14 | rem 15 | rem You should have received a copy of the GNU General Public License 16 | rem along with this program. If not, see . 17 | 18 | set DB=%2 19 | IF "%DB%"=="" ( 20 | set DB=nvivo 21 | ) 22 | set INSTANCE=%3 23 | IF "%INSTANCE%"=="" ( 24 | set INSTANCE=QSRNVIVO10 25 | ) 26 | set server=%COMPUTERNAME%\%INSTANCE% 27 | 28 | FOR /F "tokens=* USEBACKQ" %%F IN (`sqlcmd -W -S %server% -h -1 -Q "SET NOCOUNT ON; SELECT filename FROM master.dbo.sysdatabases where name='%DB%'"`) DO ( 29 | SET FILENAME=%%F 30 | ) 31 | sqlcmd -S %server% -Q "EXEC sp_detach_db %DB%" 32 | copy "%FILENAME%" "%~1" >nul 33 | del "%FILENAME%" -------------------------------------------------------------------------------- /helpers/mssqlCreate.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem Copyright 2016 Jonathan Schultz 4 | rem 5 | rem This program is free software: you can redistribute it and/or modify 6 | rem it under the terms of the GNU General Public License as published by 7 | rem the Free Software Foundation, either version 3 of the License, or 8 | rem (at your option) any later version. 9 | rem 10 | rem This program is distributed in the hope that it will be useful, 11 | rem but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | rem MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | rem GNU General Public License for more details. 14 | rem 15 | rem You should have received a copy of the GNU General Public License 16 | rem along with this program. If not, see . 17 | 18 | set TEMPDIR=%TMP%\SQL 19 | 20 | IF NOT EXIST "%TEMPDIR%\" ( 21 | mkdir "%TEMPDIR%" 22 | ) 23 | icacls "%TEMPDIR%" /grant Everyone:(OI)(CI)F 24 | 25 | set DB=%1 26 | IF "%DB%"=="" ( 27 | set DB=nvivo 28 | ) 29 | 30 | set INSTANCE=%2 31 | IF "%INSTANCE%"=="" ( 32 | set INSTANCE=QSRNVIVO10 33 | ) 34 | set server=%COMPUTERNAME%\%INSTANCE% 35 | 36 | sqlcmd -S %server% -Q "CREATE DATABASE %DB%" 37 | sqlcmd -S %server% -Q "CREATE LOGIN nvivotools WITH PASSWORD='nvivotools'" 38 | sqlcmd -S %server% -Q "use %DB%; IF EXISTS (SELECT * FROM sys.database_principals WHERE name = N'nvivotools') DROP USER nvivotools" >nul 39 | sqlcmd -S %server% -Q "use %DB%; CREATE USER nvivotools FROM LOGIN nvivotools" >nul 40 | sqlcmd -S %server% -Q "use %DB%; GRANT CONTROL TO nvivotools" >nul 41 | -------------------------------------------------------------------------------- /DropItemForeignKeys.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from sqlalchemy import exc, TypeDecorator, CHAR, String, create_engine, MetaData, bindparam 5 | from sqlalchemy.engine import reflection 6 | import warnings 7 | import sys 8 | import os 9 | import argparse 10 | import uuid 11 | from sqlalchemy.schema import ( 12 | MetaData, 13 | Table, 14 | DropTable, 15 | ForeignKeyConstraint, 16 | DropConstraint, 17 | ) 18 | 19 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 20 | 21 | try: 22 | parser = argparse.ArgumentParser(description='Drop certain foreign keys.') 23 | parser.add_argument('database', type=str) 24 | args = parser.parse_args() 25 | 26 | databasedb = create_engine(args.database) 27 | databaseconn = databasedb.connect() 28 | databasetrans = databaseconn.begin() 29 | 30 | inspector = reflection.Inspector.from_engine(databasedb) 31 | databasemd = MetaData() 32 | 33 | all_fks = [] 34 | 35 | for table_name in inspector.get_table_names(): 36 | #if table_name != 'Item' and table_name != 'Project': 37 | #continue 38 | 39 | fks = [] 40 | for fk in inspector.get_foreign_keys(table_name): 41 | if not fk['name']: 42 | continue 43 | fks.append( 44 | ForeignKeyConstraint((),(),name=fk['name']) 45 | ) 46 | t = Table(table_name,databasemd,*fks) 47 | all_fks.extend(fks) 48 | 49 | for fkc in all_fks: 50 | databaseconn.execute(DropConstraint(fkc)) 51 | 52 | databasetrans.commit() 53 | 54 | except exc.SQLAlchemyError: 55 | raise 56 | -------------------------------------------------------------------------------- /helpers/mssqlAttach.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem Copyright 2016 Jonathan Schultz 4 | rem 5 | rem This program is free software: you can redistribute it and/or modify 6 | rem it under the terms of the GNU General Public License as published by 7 | rem the Free Software Foundation, either version 3 of the License, or 8 | rem (at your option) any later version. 9 | rem 10 | rem This program is distributed in the hope that it will be useful, 11 | rem but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | rem MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | rem GNU General Public License for more details. 14 | rem 15 | rem You should have received a copy of the GNU General Public License 16 | rem along with this program. If not, see . 17 | 18 | set TEMPDIR=%TMP%\SQL 19 | 20 | IF NOT EXIST "%TEMPDIR%\" ( 21 | mkdir "%TEMPDIR%" 22 | ) 23 | icacls "%TEMPDIR%" /grant Everyone:(OI)(CI)F >nul 24 | 25 | set DB=%2 26 | IF "%DB%"=="" ( 27 | set DB=nvivo 28 | ) 29 | 30 | set INSTANCE=%3 31 | IF "%INSTANCE%"=="" ( 32 | set INSTANCE=QSRNVIVO10 33 | ) 34 | set server=%COMPUTERNAME%\%INSTANCE% 35 | 36 | del "%TEMPDIR%\%~n1*" >nul 2>&1 37 | copy "%~1" %TEMPDIR% >nul 38 | icacls "%TEMPDIR%\%~n1%~x1" /grant Everyone:(OI)(CI)F >nul 39 | sqlcmd -S %server% -Q "CREATE DATABASE %DB% ON (FILENAME='%TEMPDIR%\%~n1%~x1') FOR ATTACH" 40 | sqlcmd -S %server% -Q "CREATE LOGIN nvivotools WITH PASSWORD='nvivotools'" 41 | sqlcmd -S %server% -Q "use %DB%; IF EXISTS (SELECT * FROM sys.database_principals WHERE name = N'nvivotools') DROP USER nvivotools" >nul 42 | sqlcmd -S %server% -Q "use %DB%; CREATE USER nvivotools FROM LOGIN nvivotools" >nul 43 | sqlcmd -S %server% -Q "use %DB%; GRANT CONTROL TO nvivotools" >nul 44 | -------------------------------------------------------------------------------- /mssqlList.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2017 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | from mssqlTools import mssqlAPI 22 | 23 | def mssqlList(arglist): 24 | parser = argparse.ArgumentParser(description='List databases attached to an MS SQL Server instance.') 25 | 26 | parser.add_argument('-v', '--verbosity', type=int, default=1) 27 | 28 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 29 | help='NVivo version (10 or 11)') 30 | 31 | parser.add_argument('-S', '--server', type=str, 32 | help="IP address/name of Microsoft SQL Server") 33 | parser.add_argument('-P', '--port', type=int, 34 | help="Port of Microsoft SQL Server") 35 | parser.add_argument('-i', '--instance', type=str, 36 | help="Microsoft SQL Server instance") 37 | 38 | args = parser.parse_args(arglist) 39 | 40 | api = mssqlAPI(args.server, 41 | args.port, 42 | args.instance, 43 | version = ('MSSQL12' if args.nvivoversion == '11' else 'MSSQL10_50'), 44 | verbosity = args.verbosity) 45 | 46 | print(api.list()) 47 | 48 | if __name__ == '__main__': 49 | mssqlList(None) 50 | -------------------------------------------------------------------------------- /editUser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | import os 20 | import sys 21 | import argparse 22 | from sqlalchemy import * 23 | from sqlalchemy import exc 24 | import re 25 | from datetime import datetime 26 | import uuid 27 | 28 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 29 | 30 | 31 | parser = argparse.ArgumentParser(description='Insert user into normalised file.') 32 | 33 | parser.add_argument('-v', '--verbosity', type=int, default=1) 34 | 35 | parser.add_argument('-n', '--name', type = str) 36 | 37 | parser.add_argument('normFile', type=str) 38 | 39 | args = parser.parse_args() 40 | 41 | try: 42 | normdb = create_engine('sqlite:///' + args.normFile) 43 | normmd = MetaData(bind=normdb) 44 | normcon = normdb.connect() 45 | normtr = normcon.begin() 46 | 47 | normUser = Table('User', normmd, autoload=True) 48 | 49 | Id = uuid.uuid4() 50 | 51 | datetimeNow = datetime.utcnow() 52 | 53 | userColumns = { 54 | 'Id': Id, 55 | 'Name': args.name, 56 | } 57 | normcon.execute(normUser.insert(), userColumns) 58 | 59 | normtr.commit() 60 | normtr = None 61 | normcon.close() 62 | normdb.dispose() 63 | 64 | except: 65 | raise 66 | if not normtr is None: 67 | normtr.rollback() 68 | normdb.dispose() 69 | -------------------------------------------------------------------------------- /mssqlCreate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2017 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | from mssqlTools import mssqlAPI 22 | 23 | def mssqlAttach(arglist): 24 | parser = argparse.ArgumentParser(description='Create a database on an MS SQL Server instance.') 25 | 26 | parser.add_argument('-v', '--verbosity', type=int, default=1) 27 | 28 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 29 | help='NVivo version (10 or 11)') 30 | 31 | parser.add_argument('-S', '--server', type=str, 32 | help="IP address/name of Microsoft SQL Server") 33 | parser.add_argument('-P', '--port', type=int, 34 | help="Port of Microsoft SQL Server") 35 | parser.add_argument('-i', '--instance', type=str, 36 | help="Microsoft SQL Server instance") 37 | 38 | parser.add_argument('dbname', type=str, 39 | help="Name to assign database") 40 | 41 | args = parser.parse_args(arglist) 42 | 43 | api = mssqlAPI(args.server, 44 | args.port, 45 | args.instance, 46 | version = ('MSSQL12' if args.nvivoversion == '11' else 'MSSQL10_50'), 47 | verbosity = args.verbosity) 48 | 49 | api.create(args.dbname) 50 | 51 | if __name__ == '__main__': 52 | mssqlAttach(None) 53 | -------------------------------------------------------------------------------- /ACKNOWLEDGEMENTS.md: -------------------------------------------------------------------------------- 1 | # Acknowledgements 2 | 3 | NVivotools includes the following licenced software: 4 | 5 | ## [unoconv](https://github.com/dagwieers/unoconv) 6 | 7 | Copyright 2007-2010 Dag Wieers 8 | 9 | This program is free software; you can redistribute it and/or modify 10 | it under the terms of the GNU General Public License as published by 11 | the Free Software Foundation; version 2 only 12 | 13 | This program is distributed in the hope that it will be useful, 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | GNU General Public License for more details. 17 | 18 | You should have received a copy of the GNU General Public License 19 | along with this program; if not, write to the Free Software 20 | Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 21 | 22 | ## [sqlalchemy](https://github.com/zzzeek/sqlalchemy) 23 | 24 | This is the MIT license: http://www.opensource.org/licenses/mit-license.php 25 | 26 | Copyright (c) 2005-2016 the SQLAlchemy authors and contributors . 27 | SQLAlchemy is a trademark of Michael Bayer. 28 | 29 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 30 | software and associated documentation files (the "Software"), to deal in the Software 31 | without restriction, including without limitation the rights to use, copy, modify, merge, 32 | publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons 33 | to whom the Software is furnished to do so, subject to the following conditions: 34 | 35 | The above copyright notice and this permission notice shall be included in all copies or 36 | substantial portions of the Software. 37 | 38 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 39 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 40 | PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE 41 | FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 42 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 43 | DEALINGS IN THE SOFTWARE. 44 | 45 | -------------------------------------------------------------------------------- /mssqlAttach.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2017 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | from mssqlTools import mssqlAPI 22 | 23 | def mssqlAttach(arglist): 24 | parser = argparse.ArgumentParser(description='Attach database file to an MS SQL Server instance.') 25 | 26 | parser.add_argument('-v', '--verbosity', type=int, default=1) 27 | 28 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 29 | help='NVivo version (10 or 11)') 30 | 31 | parser.add_argument('-S', '--server', type=str, 32 | help="IP address/name of Microsoft SQL Server") 33 | parser.add_argument('-P', '--port', type=int, 34 | help="Port of Microsoft SQL Server") 35 | parser.add_argument('-i', '--instance', type=str, 36 | help="Microsoft SQL Server instance") 37 | 38 | parser.add_argument('filename', type=str, 39 | help="File to attach") 40 | parser.add_argument('dbname', type=str, 41 | help="Name to assign database") 42 | 43 | args = parser.parse_args(arglist) 44 | 45 | api = mssqlAPI(args.server, 46 | args.port, 47 | args.instance, 48 | version = ('MSSQL12' if args.nvivoversion == '11' else 'MSSQL10_50'), 49 | verbosity = args.verbosity) 50 | 51 | api.attach(args.filename, args.dbname) 52 | 53 | if __name__ == '__main__': 54 | mssqlAttach(None) 55 | -------------------------------------------------------------------------------- /mssqlSave.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2017 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | from mssqlTools import mssqlAPI 22 | 23 | def mssqlSave(arglist): 24 | parser = argparse.ArgumentParser(description='Detach a database and save database file from an MS SQL Server instance.') 25 | 26 | parser.add_argument('-v', '--verbosity', type=int, default=1) 27 | 28 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 29 | help='NVivo version (10 or 11)') 30 | 31 | parser.add_argument('-S', '--server', type=str, 32 | help="IP address/name of Microsoft SQL Server") 33 | parser.add_argument('-P', '--port', type=int, 34 | help="Port of Microsoft SQL Server") 35 | parser.add_argument('-i', '--instance', type=str, 36 | help="Microsoft SQL Server instance") 37 | 38 | parser.add_argument('filename', type=str, 39 | help="File to save as") 40 | parser.add_argument('dbname', type=str, 41 | help="Name to assign database") 42 | 43 | args = parser.parse_args(arglist) 44 | 45 | api = mssqlAPI(args.server, 46 | args.port, 47 | args.instance, 48 | version = ('MSSQL12' if args.nvivoversion == '11' else 'MSSQL10_50'), 49 | verbosity = args.verbosity) 50 | 51 | api.save(args.filename, args.dbname) 52 | 53 | if __name__ == '__main__': 54 | mssqlSave(None) 55 | -------------------------------------------------------------------------------- /mssqlDrop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2017 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | import sys 22 | import fnmatch 23 | from mssqlTools import mssqlAPI 24 | 25 | def mssqlDrop(arglist): 26 | parser = argparse.ArgumentParser(description='Drop database from an MS SQL Server instance.') 27 | 28 | parser.add_argument('-v', '--verbosity', type=int, default=1) 29 | 30 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 31 | help='NVivo version (10 or 11)') 32 | 33 | parser.add_argument('-S', '--server', type=str, 34 | help="IP address/name of Microsoft SQL Server") 35 | parser.add_argument('-P', '--port', type=int, 36 | help="Port of Microsoft SQL Server") 37 | parser.add_argument('-i', '--instance', type=str, 38 | help="Microsoft SQL Server instance") 39 | 40 | parser.add_argument('dbname', type=str, 41 | help="Name or regular expression of database to drop") 42 | 43 | args = parser.parse_args(arglist) 44 | 45 | api = mssqlAPI(args.server, 46 | args.port, 47 | args.instance, 48 | version = ('MSSQL12' if args.nvivoversion == '11' else 'MSSQL10_50'), 49 | verbosity = args.verbosity) 50 | 51 | dbnames = api.list() 52 | for dbname in dbnames: 53 | if fnmatch.fnmatch(dbname, args.dbname): 54 | if args.verbosity >= 1: 55 | print("Dropping database: " + dbname, file=sys.stderr) 56 | 57 | api.drop(dbname) 58 | 59 | if __name__ == '__main__': 60 | mssqlDrop(None) 61 | -------------------------------------------------------------------------------- /NVivo database structure notes.txt: -------------------------------------------------------------------------------- 1 | NVivo database structure notes 2 | 3 | Item.TypeID 4 | 5 | 0 - Folders 6 | 2 - Source (internal) 7 | 3 - Source (Memo) 8 | 4 - Source (external) 9 | 33 - Source - ? 10 | 34 - Source - Online? 11 | 48 - Source - ? 12 | 14 - Connected with node matrix 13 | 16 - Node (NVivo 10), : NVivo11 also uses 62? 14 | 17 - Node Matrix 15 | 24 - Matrix Coding Query 16 | 21 - Attribute value 17 | 51 - Source classifications 18 | 52 - Node classifications 19 | 20 - Attribute name 20 | 21 | 62 - NVivo11 only - Cases 22 | 66 - NVivo11 only - Sentiment 23 | 24 | Role.TypeId 25 | 0 - Item2 belongs in Item1 (eg source classifications,?) 26 | 1 - Item2 is Node ID 27 | 28 | 2 - Attribute parent-child - Item1 is parent, Item2 is child 29 | 6 - Attribute name - Item1 is attribute name, Item2 Attribute value, Tag is value ordinal 30 | 7 - Source/node attribute value - Item1 is source/node, Item2 Attribute value 31 | 13 - Attribute - Item1 is attribute name, Item2 is Source/Node classification, Tag is attribute index 32 | 33 | 34 | 14 - Source/Node Category - Item1 is Source/Node Item2 is Category 35 | 36 | 15 - Item2 is Node, Item1 is Parent Node ? Nodes seem to be linked to themselves? 37 | 38 | ExtendedItem 39 | - XML contains attribute type: 0-Text, 1=Integer, 2-Decimal, 3-Date/time, 4-Date, 5-Time, 6-Boolean 40 | 41 | Node attribute structures 42 | 43 | Node <-> Item1 7 Item2 <-> Value <-> Item2 6 Item1 <-> Attribute 44 | 45 | Node classifications 46 | 47 | Node <-> Item1 14 Item2 <-> Category <-> Item2 13 Item1 <-> Attribute 48 | 49 | Nodes: 50 | 51 | Aggregate is stored in Item table. 52 | 53 | All have type 0 link to node folder or base node 'Nodes' (F1450EED-D162-4CC9-B45D-6724156F7220) 54 | All have type 15 link to self or to parent if aggregating 55 | Non top-level nodes have type 1 link to parent 56 | Top-level nodes have type 2 link to self with incrementing tag beginning zero 57 | Top-level nodes have type 2 link to their childen with incrementing tag beginning 65536 * depth 58 | 59 | Sources: 60 | 61 | TypeID: 0=???; 62 | 1 = sound recording; 63 | 11 = PDF; 64 | 5=video; 65 | 8=image; 66 | 14=dataset (spreadsheet); 67 | 18=Twitter feed 68 | 69 | SourceType: 2,3,4 - DOC 70 | 31 - MP3 71 | 32 - WMV 72 | 33 - JPEG 73 | 34 - PDF 74 | 48 - ? 75 | 34 - PDF 76 | 77 | CompoundSourceRegion: 78 | 79 | ItemId refers to source 80 | 81 | NodeReference 82 | 83 | TypeId: 0 for text 84 | 2 for image 85 | 86 | Noderefernce.startX/lengthX refer to transcript (?), compoundsourceregion.start{X,Y}/length{X,Y} 87 | refer to media. 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /Translate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from sqlalchemy import * 5 | from sqlalchemy import exc 6 | from sqlalchemy.engine import reflection 7 | import warnings 8 | import sys 9 | import os 10 | import argparse 11 | import uuid 12 | 13 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 14 | 15 | try: 16 | parser = argparse.ArgumentParser(description='Translate NVivo encoded strings.') 17 | 18 | parser.add_argument('-r', '--reverse', action='store_true', 19 | help='Reverse translate, that is from clear text to NVivo encoded strings.') 20 | parser.add_argument('infile', type=str, 21 | help='SQLAlchemy path of NVivo database.') 22 | 23 | args = parser.parse_args() 24 | 25 | nvivodb = create_engine(args.infile) 26 | nvivomd = MetaData(bind=nvivodb) 27 | nvivomd.reflect(nvivodb) 28 | 29 | if args.reverse: 30 | charoffset = +0x377 31 | else: 32 | charoffset = -0x377 33 | 34 | nvivoProject = nvivomd.tables.get('Project') 35 | projectSel = select([ 36 | nvivoProject.c.Id.label('b_Id'), # For some reason SQLAlchemy won't allow 'Id' 37 | nvivoProject.c.Title, 38 | nvivoProject.c.Description 39 | ]) 40 | projectRows = [ dict(row) for row in nvivodb.execute(projectSel) ] 41 | updateSql = update(nvivoProject) \ 42 | .where(nvivoProject.c.Id == bindparam('b_Id')) \ 43 | .values(Title = bindparam('Title')) \ 44 | .values(Description = bindparam('Description')) 45 | for project in projectRows: 46 | project['Title'] = u''.join(map(lambda ch: unichr(ord(ch) + charoffset), project['Title'])) 47 | project['Description'] = u''.join(map(lambda ch: unichr(ord(ch) + charoffset), project['Description'])) 48 | nvivodb.execute(updateSql, project) 49 | 50 | nvivoItem = nvivomd.tables.get('Item') 51 | itemSel = select([ 52 | nvivoItem.c.Id.label('b_Id'), # For some reason SQLAlchemy won't allow 'Id' 53 | nvivoItem.c.Name, 54 | nvivoItem.c.Description 55 | ]) 56 | itemRows = [ dict(row) for row in nvivodb.execute(itemSel) ] 57 | updateSql = update(nvivoItem) \ 58 | .where(nvivoItem.c.Id == bindparam('b_Id')) \ 59 | .values(Name = bindparam('Name')) \ 60 | .values(Description = bindparam('Description')) 61 | for item in itemRows: 62 | item['Name'] = u''.join(map(lambda ch: unichr(ord(ch) + charoffset), item['Name'])) 63 | item['Description'] = u''.join(map(lambda ch: unichr(ord(ch) + charoffset), item['Description'])) 64 | nvivodb.execute(updateSql, item) 65 | 66 | except exc.SQLAlchemyError: 67 | raise 68 | -------------------------------------------------------------------------------- /DataTypes.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 Jonathan Schultz 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program. If not, see . 16 | 17 | 18 | try: 19 | import sqlalchemy_sqlany 20 | sqlany = True 21 | except: 22 | sqlany = False 23 | 24 | from sqlalchemy.databases import mssql, sqlite 25 | from sqlalchemy.ext.compiler import compiles 26 | from sqlalchemy.dialects.mssql import UNIQUEIDENTIFIER 27 | from sqlalchemy import TypeDecorator, BINARY, TEXT, String 28 | import uuid 29 | 30 | class UUID(TypeDecorator): 31 | """Platform-independent UUID type. 32 | 33 | Uses Postgresql's UUID type, otherwise uses 34 | CHAR(36), storing as stringified hex values. 35 | 36 | """ 37 | #impl = CHAR 38 | #http://stackoverflow.com/questions/5849389/storing-uuids-in-sqlite-using-pylons-and-sqlalchemy 39 | impl = BINARY 40 | 41 | def load_dialect_impl(self, dialect): 42 | if dialect.name == 'postgresql': 43 | return dialect.type_descriptor(UUID()) 44 | else: 45 | return dialect.type_descriptor(CHAR(36)) 46 | 47 | def process_bind_param(self, value, dialect): 48 | if value is None: 49 | return value 50 | elif dialect.name == 'postgresql': 51 | return str(value) 52 | else: 53 | if not isinstance(value, uuid.UUID): 54 | return str(uuid.UUID(value)).upper() 55 | else: 56 | return str(value).upper() 57 | 58 | def process_result_value(self, value, dialect): 59 | if value is None: 60 | return value 61 | elif not isinstance(value, uuid.UUID): 62 | return uuid.UUID(value) 63 | else: 64 | return value 65 | 66 | @compiles(UUID, 'sqlite') 67 | def compile_UUID_mssql_sqlite(element, compiler, **kw): 68 | """ SQLite doesn't care too much about type names, UNIQUEIDENTIFIER is fine. """ 69 | return 'UNIQUEIDENTIFIER' 70 | 71 | mssql.ischema_names['xml'] = String 72 | mssql.ischema_names['uniqueidentifier'] = UUID 73 | 74 | sqlite.ischema_names['UNIQUEIDENTIFIER'] = UUID 75 | sqlite.ischema_names['UUIDTEXT'] = UUID 76 | 77 | if sqlany: 78 | sqlalchemy_sqlany.dialect.ischema_names['xml'] = String 79 | sqlalchemy_sqlany.dialect.ischema_names['long nvarchar'] = TEXT 80 | sqlalchemy_sqlany.dialect.ischema_names['uniqueidentifier'] = UUID 81 | -------------------------------------------------------------------------------- /DenormaliseDB.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | import argparse 20 | import NVivo 21 | 22 | parser = argparse.ArgumentParser(description='Denormalise a normalised NVivo project.') 23 | parser.add_argument('-w', '--windows', action='store_true', 24 | help='Correct NVivo for Windows string coding. Use if offloaded file will be used with Windows version of NVivo.') 25 | parser.add_argument('-m', '--mac', action='store_true', 26 | help='Use NVivo for Mac database format.') 27 | 28 | parser.add_argument('-v', '--verbosity', type=int, default=1) 29 | 30 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 31 | help='NVivo version (10 or 11)') 32 | 33 | parser.add_argument('-u', '--users', choices=["skip", "merge", "overwrite", "replace"], default="merge", 34 | help='User action.') 35 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 36 | help='Project action.') 37 | parser.add_argument('-nc', '--node-categories', choices=["skip", "merge", "overwrite"], default="merge", 38 | help='Node category action.') 39 | parser.add_argument('-n', '--nodes', choices=["skip", "merge"], default="merge", 40 | help='Node action.') 41 | parser.add_argument('-na', '--node-attributes', choices=["skip", "merge", "overwrite"], default="merge", 42 | help='Node attribute table action.') 43 | parser.add_argument('-sc', '--source-categories', choices=["skip", "merge", "overwrite"], default="merge", 44 | help='Source category action.') 45 | parser.add_argument('--sources', choices=["skip", "merge", "overwrite"], default="merge", 46 | help='Source action.') 47 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "merge", "overwrite"], default="merge", 48 | help='Source attribute action.') 49 | parser.add_argument('-t', '--taggings', choices=["skip", "merge"], default="merge", 50 | help='Tagging action.') 51 | parser.add_argument('-a', '--annotations', choices=["skip", "merge"], default="merge", 52 | help='Annotation action.') 53 | 54 | parser.add_argument('indb', type=str, 55 | help='SQLAlchemy path of input normalised database.') 56 | parser.add_argument('outdb', type=str, nargs='?', 57 | help='SQLAlchemy path of input output NVivo database.') 58 | 59 | args = parser.parse_args() 60 | 61 | NVivo.Denormalise(args) 62 | -------------------------------------------------------------------------------- /Norm2RQDA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | import RQDA 22 | import os 23 | import shutil 24 | import tempfile 25 | 26 | parser = argparse.ArgumentParser(description='Convert a normalised NVivo project to RQDA.') 27 | 28 | parser.add_argument('-v', '--verbosity', type=int, default=1) 29 | 30 | parser.add_argument('-u', '--users', choices=["skip", "overwrite"], default="merge", 31 | help='User action.') 32 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 33 | help='Project action.') 34 | parser.add_argument('-nc', '--node-categories', choices=["skip", "overwrite"], default="merge", 35 | help='Node category action.') 36 | parser.add_argument('-n', '--nodes', choices=["skip", "overwrite"], default="merge", 37 | help='Node action.') 38 | parser.add_argument('-na', '--node-attributes', choices=["skip", "overwrite"], default="merge", 39 | help='Node attribute table action.') 40 | parser.add_argument('-sc', '--source-categories', choices=["skip", "overwrite"], default="merge", 41 | help='Source category action.') 42 | parser.add_argument('-s', '--sources', choices=["skip", "overwrite"], default="merge", 43 | help='Source action.') 44 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "overwrite"], default="merge", 45 | help='Source attribute action.') 46 | parser.add_argument('-t', '--taggings', choices=["skip", "overwrite"], default="merge", 47 | help='Tagging action.') 48 | parser.add_argument('-a', '--annotations', choices=["skip", "overwrite"], default="merge", 49 | help='Annotation action.') 50 | 51 | parser.add_argument('infile', type=argparse.FileType('rb'), 52 | help="Input normalised (.norm) file") 53 | parser.add_argument('outfilename', type=str, nargs='?', 54 | help="Output RQDA file") 55 | 56 | args = parser.parse_args() 57 | 58 | tmpinfilename = tempfile.mktemp() 59 | tmpinfileptr = open(tmpinfilename, 'wb') 60 | tmpinfileptr.write(args.infile.read()) 61 | args.infile.close() 62 | tmpinfileptr.close() 63 | 64 | tmpoutfilename = tempfile.mktemp() 65 | 66 | if args.outfilename is None: 67 | args.outfilename = args.infile.name.rsplit('.',1)[0] + '.rqda' 68 | 69 | args.indb = 'sqlite:///' + tmpinfilename 70 | args.outdb = 'sqlite:///' + tmpoutfilename 71 | 72 | RQDA.Norm2RQDA(args) 73 | 74 | shutil.move(tmpoutfilename, os.path.basename(args.outfilename)) 75 | os.remove(tmpinfilename) 76 | -------------------------------------------------------------------------------- /NormaliseDB.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | import argparse 20 | import NVivo 21 | 22 | parser = argparse.ArgumentParser(description='Normalise an offloaded NVivo project.') 23 | parser.add_argument('-w', '--windows', action='store_true', 24 | help='Correct NVivo for Windows string coding. Use if names and descriptions appear wierd.') 25 | parser.add_argument('-m', '--mac', action='store_true', 26 | help='Use NVivo for Mac database format.') 27 | 28 | parser.add_argument('-v', '--verbosity', type=int, default=1) 29 | 30 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 31 | help='NVivo version (10 or 11)') 32 | 33 | parser.add_argument('-u', '--users', choices=["skip", "merge", "overwrite", "replace"], default="merge", 34 | help='User action.') 35 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 36 | help='Project action.') 37 | parser.add_argument('-nc', '--node-categories', choices=["skip", "merge", "overwrite", "replace"], default="merge", 38 | help='Node category action.') 39 | parser.add_argument('-n', '--nodes', choices=["skip", "merge", "overwrite", "replace"], default="merge", 40 | help='Node action.') 41 | parser.add_argument('-na', '--node-attributes', choices=["skip", "merge", "overwrite", "replace"], default="merge", 42 | help='Node attribute table action.') 43 | parser.add_argument('-sc', '--source-categories', choices=["skip", "merge", "overwrite", "replace"], default="merge", 44 | help='Source category action.') 45 | parser.add_argument('--sources', choices=["skip", "merge", "overwrite", "replace"], default="merge", 46 | help='Source action.') 47 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "merge", "overwrite", "replace"], default="merge", 48 | help='Source attribute action.') 49 | parser.add_argument('-t', '--taggings', choices=["skip", "merge", "overwrite", "replace"], default="merge", 50 | help='Tagging action.') 51 | parser.add_argument('-a', '--annotations', choices=["skip", "merge", "overwrite", "replace"], default="merge", 52 | help='Annotation action.') 53 | 54 | parser.add_argument('indb', type=str, 55 | help='SQLAlchemy path of input NVivo database or "-" to create empty project.') 56 | parser.add_argument('outdb', type=str, nargs='?', 57 | help='SQLAlchemy path of output normalised database.') 58 | 59 | args = parser.parse_args() 60 | 61 | NVivo.Normalise(args) 62 | -------------------------------------------------------------------------------- /extractTagging.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import os 21 | import sys 22 | import argparse 23 | from sqlalchemy import * 24 | from sqlalchemy import exc 25 | import re 26 | 27 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 28 | 29 | 30 | parser = argparse.ArgumentParser(description='Extract tagging from normalised file.') 31 | 32 | parser.add_argument('-v', '--verbosity', type=int, default=1) 33 | parser.add_argument('-l', '--limit', type=int, default=0, 34 | help="Limit number of sources to process") 35 | parser.add_argument('-n', '--node', type=str, nargs='?', 36 | help="Name of node") 37 | parser.add_argument('-s', '--source', type=str, nargs='?', 38 | help="Name of source") 39 | 40 | parser.add_argument('infile', type=str, 41 | help="Normalised project file to analyse") 42 | 43 | args = parser.parse_args() 44 | 45 | from textblob import TextBlob 46 | 47 | try: 48 | normdb = create_engine('sqlite:///' + args.infile) 49 | normmd = MetaData(bind=normdb) 50 | 51 | normNode = Table('Node', normmd, autoload=True) 52 | normSource = Table('Source', normmd, autoload=True) 53 | normTagging = Table('Tagging', normmd, autoload=True) 54 | 55 | sel = select([ 56 | normTagging.c.Fragment, 57 | normNode.c.Name.label('NodeName'), 58 | normSource.c.Name.label('SourceName'), 59 | normSource.c.Content 60 | ]).where(and_( 61 | normTagging.c.Node == normNode.c.Id, 62 | normSource.c.Id == normTagging.c.Source 63 | )) 64 | if args.node is not None: 65 | sel = sel.where( 66 | normNode.c.Name == bindparam('NodeName') 67 | ) 68 | if args.source is not None: 69 | sel = sel.where( 70 | normSource.c.Name == bindparam('SourceName') 71 | ) 72 | 73 | taggings = normdb.execute(sel, { 74 | 'NodeName': args.node, 75 | 'SourceName': args.source 76 | }) 77 | 78 | for tagging in taggings: 79 | print("Node: " + tagging['NodeName'] + " Source: " + tagging['SourceName'] + "[" + tagging['Fragment'] + "]", file=sys.stderr) 80 | 81 | matchfragment = re.match("([0-9]+):([0-9]+)(?:,([0-9]+)(?::([0-9]+))?)?", tagging['Fragment']) 82 | if matchfragment is None: 83 | print("WARNING: Unrecognised tagging fragment", file=sys.stderr) 84 | else: 85 | print(tagging['Content'][int(matchfragment.group(1)):int(matchfragment.group(2))+1], file=sys.stderr) 86 | 87 | print("", file=sys.stderr) 88 | 89 | normdb.dispose() 90 | 91 | 92 | except: 93 | raise 94 | -------------------------------------------------------------------------------- /RQDA2Norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | import RQDA 22 | import os 23 | import shutil 24 | import tempfile 25 | 26 | parser = argparse.ArgumentParser(description='Convert an RQDA project to normalised NVivo format.') 27 | 28 | parser.add_argument('-v', '--verbosity', type=int, default=1) 29 | 30 | parser.add_argument('-u', '--users', choices=["skip", "overwrite"], default="merge", 31 | help='User action.') 32 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 33 | help='Project action.') 34 | parser.add_argument('-nc', '--node-categories', choices=["skip", "overwrite"], default="merge", 35 | help='Node category action.') 36 | parser.add_argument('-n', '--nodes', choices=["skip", "overwrite"], default="merge", 37 | help='Node action.') 38 | parser.add_argument('-c', '--cases', choices=["skip", "overwrite"], default="merge", 39 | help='case action.') 40 | parser.add_argument('-ca', '--case-attributes', choices=["skip", "overwrite"], default="merge", 41 | help='Case attribute table action.') 42 | parser.add_argument('-sc', '--source-categories', choices=["skip", "overwrite"], default="merge", 43 | help='Source category action.') 44 | parser.add_argument('-s', '--sources', choices=["skip", "overwrite"], default="merge", 45 | help='Source action.') 46 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "overwrite"], default="merge", 47 | help='Source attribute action.') 48 | parser.add_argument('-t', '--taggings', choices=["skip", "overwrite"], default="merge", 49 | help='Tagging action.') 50 | parser.add_argument('-a', '--annotations', choices=["skip", "overwrite"], default="merge", 51 | help='Annotation action.') 52 | 53 | parser.add_argument('infile', type=argparse.FileType('rb'), 54 | help="Input RQDA file") 55 | parser.add_argument('outfilename', type=str, nargs='?', 56 | help="Output normalised (.norm) file") 57 | 58 | args = parser.parse_args() 59 | 60 | tmpinfilename = tempfile.mktemp() 61 | tmpinfileptr = open(tmpinfilename, 'wb') 62 | tmpinfileptr.write(args.infile.read()) 63 | args.infile.close() 64 | tmpinfileptr.close() 65 | 66 | tmpoutfilename = tempfile.mktemp() 67 | 68 | if args.outfilename is None: 69 | args.outfilename = args.infile.name.rsplit('.',1)[0] + '.rqda' 70 | 71 | args.indb = 'sqlite:///' + tmpinfilename 72 | args.outdb = 'sqlite:///' + tmpoutfilename 73 | 74 | RQDA.RQDA2Norm(args) 75 | 76 | shutil.move(tmpoutfilename, os.path.basename(args.outfilename)) 77 | os.remove(tmpinfilename) 78 | -------------------------------------------------------------------------------- /helpers/sqlanyenv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2017 Jonathan Schultz 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | set > /tmp/sqlanyenv.1 19 | 20 | if [ "$(uname)" = "Linux" ]; then 21 | sqlanywhere=${sqlanywhere:-/opt} 22 | elif [ "$(uname)" = "Darwin" ]; then 23 | sqlanywhere=${sqlanywhere:-/Applications} 24 | fi 25 | 26 | # Hacky way of choosing which SQLAnywhere installation to use: sort in 27 | # reverse order so SQLAnywhere installations take precedence over NVivo 28 | # installations and 64 bit over 32 bit. But only stop when we find a 29 | # dynamic executable. 30 | IFS=$'\n' 31 | for dbeng in `find $sqlanywhere -name dbeng[0-9][0-9] | sort -r`; do 32 | source=$(echo $dbeng | awk -F/ '{print $3}') 33 | version=$(echo $dbeng | awk -F/ '{print $NF}' | sed 's/dbeng\(..\)/\1/') 34 | arch=$(echo $dbeng | awk -F/ '{print $(NF-1)}') 35 | bits=$(echo $arch | sed 's/bin\([0-9]*\).*/\1/') 36 | binpath=`dirname $dbeng` 37 | libpath=`dirname $binpath`/lib$bits 38 | [[ "${arch: -1}" == "s" ]] && static=1 || static=0 39 | [[ $source =~ NVivo ]] && nvivo=1 || nvivo=0 40 | [[ $nvivo ]] && nvivoversion=$(echo $source | sed 's/NVivo\(.*\)\.app/\1/' | xargs) 41 | [[ $static == 0 ]] && break 42 | done 43 | unset IFS 44 | if [[ "$source" != "" ]]; then 45 | echo "Found SQLAnywhere installation at `dirname "$dbeng"`" > /dev/stderr 46 | else 47 | echo "Could not find SQLAnywhere installation" > /dev/stderr 48 | exit 1 49 | fi 50 | 51 | # The version of SQLAnywhere bundled with NVivo is very difficult to work with 52 | # as its libraries contain references to @rpath. In addition, dlopen() doesn't 53 | # seem to respect DYLD_LIBRARY_PATH and friends. But it will find a library 54 | # in the current working directory. So this hack makes a copy of 55 | # libdbcapi_r.dylib, modifies its embedded rpath, tells sqlanydb to use 56 | # the modified library, and flags that the current working directory much 57 | # be changed prior to loading sqlanydb. Whew! 58 | if [[ $nvivo == 1 && $static == 0 ]]; then 59 | if test -f "$libpath"/libdbcapi_r.dylib; then 60 | cp -p "$libpath"/libdbcapi_r.dylib "$libpath"/libdbcapi_r.rpath.dylib 61 | chmod +w "$libpath"/libdbcapi_r.rpath.dylib 62 | install_name_tool -add_rpath "$libpath"/ "$libpath"/libdbcapi_r.rpath.dylib 63 | SQLANY_API_DLL=libdbcapi_r.rpath.dylib 64 | CHDIR="$libpath" 65 | fi 66 | else 67 | # As of High Sierra we have another problem - changes to DYLD_LIBRARY_PATH are dumped by the 68 | # System Integrity Protection system. We hack our way around this problem by changing the current 69 | # working directory to the one containing the dynamic libraries. 70 | CHDIR=$libpath 71 | fi 72 | 73 | if test -f "$binpath"/sa_config.sh; then 74 | . "$binpath"/sa_config.sh 75 | fi 76 | 77 | set > /tmp/sqlanyenv.2 78 | comm -13 <(sort /tmp/sqlanyenv.1) <(sort /tmp/sqlanyenv.2) 79 | -------------------------------------------------------------------------------- /NVivo2RQDADB.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | import NVivo 22 | import RQDA 23 | import os 24 | import shutil 25 | import tempfile 26 | 27 | parser = argparse.ArgumentParser(description='Convert an NVivo project to an RQDA project.') 28 | 29 | parser.add_argument('-v', '--verbosity', type=int, default=1) 30 | 31 | parser.add_argument('-w', '--windows', action='store_true', 32 | help='Correct NVivo for Windows string coding. Use if offloaded file will be used with Windows version of NVivo.') 33 | parser.add_argument('-m', '--mac', action='store_true', 34 | help='Use NVivo for Mac database format.') 35 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 36 | help='NVivo version (10 or 11)') 37 | 38 | parser.add_argument('-u', '--users', choices=["skip", "overwrite"], default="merge", 39 | help='User action.') 40 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 41 | help='Project action.') 42 | parser.add_argument('-nc', '--node-categories', choices=["skip", "overwrite"], default="merge", 43 | help='Node category action.') 44 | parser.add_argument('-n', '--nodes', choices=["skip", "overwrite"], default="merge", 45 | help='Node action.') 46 | parser.add_argument('-c', '--cases', choices=["skip", "overwrite"], default="merge", 47 | help='case action.') 48 | parser.add_argument('-ca', '--case-attributes', choices=["skip", "overwrite"], default="merge", 49 | help='Case attribute table action.') 50 | parser.add_argument('-sc', '--source-categories', choices=["skip", "overwrite"], default="merge", 51 | help='Source category action.') 52 | parser.add_argument('-s', '--sources', choices=["skip", "overwrite"], default="merge", 53 | help='Source action.') 54 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "overwrite"], default="merge", 55 | help='Source attribute action.') 56 | parser.add_argument('-t', '--taggings', choices=["skip", "overwrite"], default="merge", 57 | help='Tagging action.') 58 | parser.add_argument('-a', '--annotations', choices=["skip", "overwrite"], default="merge", 59 | help='Annotation action.') 60 | 61 | parser.add_argument('innvivodb', type=str, 62 | help="Input database") 63 | parser.add_argument('outrqdadb', type=str, nargs='?', 64 | help="Output database") 65 | 66 | args = parser.parse_args() 67 | 68 | if args.outrqdadb is None: 69 | args.outrqdadb = args.innvivodb.rsplit('.',1)[0] + '.rqda' 70 | 71 | tmpnormfilename = tempfile.mktemp() 72 | 73 | args.indb = args.innvivodb 74 | args.outdb = 'sqlite:///' + tmpnormfilename 75 | 76 | NVivo.Normalise(args) 77 | 78 | args.node_attributes = args.case_attributes 79 | 80 | args.indb = 'sqlite:///' + tmpnormfilename 81 | args.outdb = args.outrqdadb 82 | RQDA.Norm2RQDA(args) 83 | 84 | os.remove(tmpnormfilename) 85 | -------------------------------------------------------------------------------- /saveSources.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2017 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import os 21 | import sys 22 | import argparse 23 | import fnmatch 24 | from NVivoNorm import NVivoNorm 25 | from sqlalchemy import * 26 | 27 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 28 | 29 | def saveSources(arglist): 30 | 31 | parser = argparse.ArgumentParser(description='Save sources from a normalised NVivo file', 32 | fromfile_prefix_chars='@') 33 | 34 | parser.add_argument('-v', '--verbosity', type=int, default=1) 35 | parser.add_argument('--no-comments', action='store_true', help='Do not produce a comments logfile') 36 | 37 | parser.add_argument('-s', '--source', type=str, default = '%', 38 | help='Source or name or pattern') 39 | 40 | parser.add_argument('-p', '--path', type=str, default='.', 41 | help='Output file directory') 42 | parser.add_argument('infile', type=str, 43 | help='Input normalised file') 44 | 45 | 46 | args = parser.parse_args() 47 | hiddenargs = ['verbosity'] 48 | 49 | try: 50 | if not args.no_comments: 51 | logfilename = os.path.join(args.path, 'saveSources.log') 52 | 53 | comments = (' ' + args.path + ' ').center(80, '#') + '\n' 54 | comments += '# ' + os.path.basename(sys.argv[0]) + '\n' 55 | arglist = args.__dict__.keys() 56 | for arg in arglist: 57 | if arg not in hiddenargs: 58 | val = getattr(args, arg) 59 | if type(val) == str: 60 | comments += '# --' + arg + '="' + val + '"\n' 61 | elif type(val) == bool: 62 | if val: 63 | comments += '# --' + arg + '\n' 64 | elif type(val) == list: 65 | for valitem in val: 66 | if type(valitem) == str: 67 | comments += '# --' + arg + '="' + valitem + '"\n' 68 | else: 69 | comments += '# --' + arg + '=' + str(valitem) + '\n' 70 | elif val is not None: 71 | comments += '# --' + arg + '=' + str(val) + '\n' 72 | 73 | with open(logfilename, 'w') as logfile: 74 | logfile.write(comments) 75 | 76 | norm = NVivoNorm(args.infile) 77 | norm.begin() 78 | 79 | query = select([norm.Source.c.Name, norm.Source.c.Object, norm.Source.c.ObjectType]).where( 80 | norm.Source.c.Name.like(literal(args.source))) 81 | for row in norm.con.execute(query): 82 | outfile = open(os.path.join(args.path, row.Name + '.' + row.ObjectType.lower()), 'wb') 83 | outfile.write(row.Object) 84 | outfile.close() 85 | 86 | except: 87 | raise 88 | norm.rollback() 89 | del norm 90 | 91 | if __name__ == '__main__': 92 | saveSources(None) 93 | -------------------------------------------------------------------------------- /RQDA2NVivoDB.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | import NVivo 22 | import RQDA 23 | import os 24 | import shutil 25 | import tempfile 26 | 27 | parser = argparse.ArgumentParser(description='Convert an RQDA project to NVivo format.') 28 | 29 | parser.add_argument('-v', '--verbosity', type=int, default=1) 30 | 31 | parser.add_argument('-w', '--windows', action='store_true', 32 | help='Correct NVivo for Windows string coding. Use if offloaded file will be used with Windows version of NVivo.') 33 | parser.add_argument('-m', '--mac', action='store_true', 34 | help='Use NVivo for Mac database format.') 35 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 36 | help='NVivo version (10 or 11)') 37 | 38 | parser.add_argument('-u', '--users', choices=["skip", "overwrite"], default="merge", 39 | help='User action.') 40 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 41 | help='Project action.') 42 | parser.add_argument('-nc', '--node-categories', choices=["skip", "overwrite"], default="merge", 43 | help='Node category action.') 44 | parser.add_argument('-n', '--nodes', choices=["skip", "overwrite"], default="merge", 45 | help='Node action.') 46 | parser.add_argument('-c', '--cases', choices=["skip", "overwrite"], default="merge", 47 | help='case action.') 48 | parser.add_argument('-ca', '--case-attributes', choices=["skip", "overwrite"], default="merge", 49 | help='Case attribute table action.') 50 | parser.add_argument('-sc', '--source-categories', choices=["skip", "overwrite"], default="merge", 51 | help='Source category action.') 52 | parser.add_argument('-s', '--sources', choices=["skip", "overwrite"], default="merge", 53 | help='Source action.') 54 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "overwrite"], default="merge", 55 | help='Source attribute action.') 56 | parser.add_argument('-t', '--taggings', choices=["skip", "overwrite"], default="merge", 57 | help='Tagging action.') 58 | parser.add_argument('-a', '--annotations', choices=["skip", "overwrite"], default="merge", 59 | help='Annotation action.') 60 | 61 | parser.add_argument('inrqdadb', type=str, 62 | help="Input database") 63 | parser.add_argument('outnvivodb', type=str, nargs='?', 64 | help="Output database, structure must already exist") 65 | 66 | args = parser.parse_args() 67 | 68 | if args.outnvivodb is None: 69 | args.outnvivodb = args.inrqdadb.rsplit('.',1)[0] + '.nvivo' 70 | 71 | tmpnormfilename = tempfile.mktemp() 72 | 73 | args.indb = args.inrqdadb 74 | args.outdb = 'sqlite:///' + tmpnormfilename 75 | RQDA.RQDA2Norm(args) 76 | 77 | args.node_attributes = args.case_attributes 78 | 79 | args.indb = 'sqlite:///' + tmpnormfilename 80 | args.outdb = args.outnvivodb 81 | NVivo.Denormalise(args) 82 | 83 | os.remove(tmpnormfilename) 84 | -------------------------------------------------------------------------------- /regressionTest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2016 Jonathan Schultz 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | dir=`dirname $0` 19 | 20 | $dir/NormaliseDB.py - sqlite:///$1 21 | 22 | $dir/editProject.py --title "Test project" --description "This test project is for regression testing NVivotools" --user "Regression Tester" $1 23 | 24 | $dir/editNodeCategory.py --name "Node cat one" --description "First of two node categories" $1 25 | $dir/editNodeCategory.py --name "Node cat two" --description "Second of two node categories" $1 26 | 27 | $dir/editNodeAttribute.py --name "String attribute" --type text --length 64 $1 28 | $dir/editNodeAttribute.py --name "Integer attribute" --type integer $1 29 | $dir/editNodeAttribute.py --name "Decimal attribute" --type decimal $1 30 | $dir/editNodeAttribute.py --name "DateTime attribute" --type datetime $1 31 | $dir/editNodeAttribute.py --name "Date attribute" --type date $1 32 | $dir/editNodeAttribute.py --name "Time attribute" --type time $1 33 | $dir/editNodeAttribute.py --name "Boolean attribute" --type boolean $1 34 | 35 | $dir/editNode.py --name "Node with attributes" --description "Node in first category testing attributes" --category "Node cat one" --attribute "String attribute:string value" --attribute "Integer attribute:17" --attribute "Decimal attribute:3.1415" --attribute "DateTime attribute:2000-01-01 00:01" --attribute "Date attribute:10 Dec 1967" --attribute "Time attribute:16:20" --attribute "Boolean attribute:false" $1 36 | 37 | $dir/editNode.py --name "Top level node no aggregate" $1 38 | $dir/editNode.py --name "Second level node" --description "Child of no aggregate top node" --parent "Top level node no aggregate" $1 39 | $dir/editNode.py --name "Second second level node" --description "Second child of no aggregate top node" --parent "Top level node no aggregate" $1 40 | $dir/editNode.py --name "Third level node" --parent "Second second level node" $1 41 | 42 | $dir/editNode.py --name "Top level node with aggregate" --aggregate $1 43 | $dir/editNode.py --name "Another second level node" --description "Child of with aggregate top node" --parent "Top level node with aggregate" $1 44 | 45 | $dir/editSourceCategory.py --name "Source cat one" --description "First of two source categories" $1 46 | $dir/editSourceCategory.py --name "Source cat two" --description "Second of two source categories" $1 47 | 48 | $dir/editSourceAttribute.py --name "String attribute" --type text --length 64 $1 49 | $dir/editSourceAttribute.py --name "Integer attribute" --type integer $1 50 | $dir/editSourceAttribute.py --name "Decimal attribute" --type decimal $1 51 | $dir/editSourceAttribute.py --name "DateTime attribute" --type datetime $1 52 | $dir/editSourceAttribute.py --name "Date attribute" --type date $1 53 | $dir/editSourceAttribute.py --name "Time attribute" --type time $1 54 | $dir/editSourceAttribute.py --name "Boolean attribute" --type boolean $1 55 | 56 | echo "Hello world" > source.txt 57 | 58 | $dir/editSource.py --name "Source with attributes" --description "Source in first category testing attributes" --category "Source cat one" --attribute "String attribute:string value" --attribute "Integer attribute:17" --attribute "Decimal attribute:3.1415" --attribute "DateTime attribute:2000-01-01 00:01" --attribute "Date attribute:10 Dec 1967" --attribute "Time attribute:16:20" --attribute "Boolean attribute:false" --source source.txt $1 59 | 60 | $dir/editTagging.py --node "Third level node" --source "Source with attributes" --fragment "0:4" $1 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /editSourceAttribute.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | import os 20 | import sys 21 | import argparse 22 | from sqlalchemy import * 23 | from sqlalchemy import exc 24 | import re 25 | from datetime import date, time, datetime 26 | from distutils import util 27 | import uuid 28 | 29 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 30 | 31 | 32 | parser = argparse.ArgumentParser(description='Insert or update source attribute in normalised file.') 33 | 34 | parser.add_argument('-v', '--verbosity', type=int, default=1) 35 | 36 | parser.add_argument('-n', '--name', type = str) 37 | parser.add_argument('-d', '--description', type = str) 38 | parser.add_argument('-t', '--type', choices=["text", "integer", "decimal", "datetime", "date", "time", "boolean"]) 39 | parser.add_argument('-l', '--length', type = int) 40 | parser.add_argument('-u', '--user', type = str, 41 | help = 'User name, default is project "modified by".') 42 | 43 | parser.add_argument('normFile', type=str) 44 | 45 | args = parser.parse_args() 46 | 47 | try: 48 | normdb = create_engine('sqlite:///' + args.normFile) 49 | normmd = MetaData(bind=normdb) 50 | normcon = normdb.connect() 51 | normtr = normcon.begin() 52 | 53 | normUser = Table('User', normmd, autoload=True) 54 | normProject = Table('Project', normmd, autoload=True) 55 | normSourceAttribute = Table('SourceAttribute', normmd, autoload=True) 56 | 57 | if args.user is not None: 58 | user = normcon.execute(select([ 59 | normUser.c.Id 60 | ]).where( 61 | normUser.c.Name == bindparam('Name') 62 | ), { 63 | 'Name': args.user 64 | }).first() 65 | if user is not None: 66 | userId = user['Id'] 67 | else: 68 | userId = uuid.uuid4() 69 | normocon.execute(normUser.insert(), { 70 | 'Id': userId, 71 | 'Name': args.user 72 | }) 73 | else: 74 | project = normcon.execute(select([ 75 | normProject.c.ModifiedBy 76 | ])).first() 77 | userId = project['ModifiedBy'] 78 | 79 | att = normcon.execute(select([ 80 | normSourceAttribute.c.Id 81 | ]).where( 82 | normSourceAttribute.c.Name == bindparam('Name') 83 | ), { 84 | 'Name': args.name 85 | }).first() 86 | Id = uuid.uuid4() if att is None else att['Id'] 87 | 88 | datetimeNow = datetime.utcnow() 89 | 90 | attColumns = { 91 | 'Id': Id, 92 | '_Id': Id, 93 | 'Name': args.name, 94 | 'Description': args.description, 95 | 'Type': args.type.title(), 96 | 'Length': args.length, 97 | 'CreatedBy': userId, 98 | 'CreatedDate': datetimeNow, 99 | 'ModifiedBy': userId, 100 | 'ModifiedDate': datetimeNow 101 | } 102 | if att is None: # New category 103 | normcon.execute(normSourceAttribute.insert(), attColumns) 104 | else: 105 | normcon.execute(normSourceAttribute.update( 106 | normSourceAttribute.c.Id == bindparam('_Id')), 107 | attColumns) 108 | 109 | normtr.commit() 110 | normtr = None 111 | normcon.close() 112 | normdb.dispose() 113 | 114 | 115 | except: 116 | raise 117 | if not normtr is None: 118 | normtr.rollback() 119 | normdb.dispose() 120 | -------------------------------------------------------------------------------- /editNodeAttribute.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | import os 20 | import sys 21 | import argparse 22 | from sqlalchemy import * 23 | from sqlalchemy import exc 24 | import re 25 | from dateutil import parser as dateparser 26 | from datetime import date, time, datetime 27 | from distutils import util 28 | import uuid 29 | 30 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 31 | 32 | 33 | parser = argparse.ArgumentParser(description='Insert or update node attribute in normalised file.') 34 | 35 | parser.add_argument('-v', '--verbosity', type=int, default=1) 36 | 37 | parser.add_argument('-n', '--name', type = str) 38 | parser.add_argument('-d', '--description', type = str) 39 | parser.add_argument('-t', '--type', choices=["text", "integer", "decimal", "datetime", "date", "time", "boolean"]) 40 | parser.add_argument('-l', '--length', type = int) 41 | parser.add_argument('-u', '--user', type = str, 42 | help = 'User name, default is project "modified by".') 43 | 44 | parser.add_argument('normFile', type=str) 45 | 46 | args = parser.parse_args() 47 | 48 | try: 49 | normdb = create_engine('sqlite:///' + args.normFile) 50 | normmd = MetaData(bind=normdb) 51 | normcon = normdb.connect() 52 | normtr = normcon.begin() 53 | 54 | normUser = Table('User', normmd, autoload=True) 55 | normProject = Table('Project', normmd, autoload=True) 56 | normNodeAttribute = Table('NodeAttribute', normmd, autoload=True) 57 | 58 | if args.user is not None: 59 | user = normcon.execute(select([ 60 | normUser.c.Id 61 | ]).where( 62 | normUser.c.Name == bindparam('Name') 63 | ), { 64 | 'Name': args.user 65 | }).first() 66 | if user is not None: 67 | userId = user['Id'] 68 | else: 69 | userId = uuid.uuid4() 70 | normocon.execute(normUser.insert(), { 71 | 'Id': userId, 72 | 'Name': args.user 73 | }) 74 | else: 75 | project = normcon.execute(select([ 76 | normProject.c.ModifiedBy 77 | ])).first() 78 | userId = project['ModifiedBy'] 79 | 80 | att = normcon.execute(select([ 81 | normNodeAttribute.c.Id 82 | ]).where( 83 | normNodeAttribute.c.Name == bindparam('Name') 84 | ), { 85 | 'Name': args.name 86 | }).first() 87 | Id = uuid.uuid4() if att is None else att['Id'] 88 | 89 | datetimeNow = datetime.utcnow() 90 | 91 | catColumns = { 92 | 'Id': Id, 93 | '_Id': Id, 94 | 'Name': args.name, 95 | 'Description': args.description, 96 | 'Type': args.type.title(), 97 | 'Length': args.length, 98 | 'CreatedBy': userId, 99 | 'CreatedDate': datetimeNow, 100 | 'ModifiedBy': userId, 101 | 'ModifiedDate': datetimeNow 102 | } 103 | if att is None: # New category 104 | normcon.execute(normNodeAttribute.insert(), catColumns) 105 | else: 106 | normcon.execute(normNodeAttribute.update( 107 | normNodeAttribute.c.Id == bindparam('_Id')), 108 | catColumns) 109 | 110 | normtr.commit() 111 | normtr = None 112 | normcon.close() 113 | normdb.dispose() 114 | 115 | 116 | except: 117 | raise 118 | if not normtr is None: 119 | normtr.rollback() 120 | normdb.dispose() 121 | -------------------------------------------------------------------------------- /AdjustDate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import print_function 5 | import NVivo 6 | from sqlalchemy import * 7 | from sqlalchemy import exc 8 | import warnings 9 | import sys 10 | import os 11 | import argparse 12 | import re 13 | from dateutil import parser as dateparser 14 | from datetime import datetime, timedelta 15 | from pytimeparse.timeparse import timeparse 16 | 17 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 18 | 19 | db = None 20 | con = None 21 | tr = None 22 | try: 23 | parser = argparse.ArgumentParser(description='Adjust all CreatedDate/ModifiedDate columns in a database before a specified date by a given number of days. Simultateouly correct ModifiedDate to be no earlier than CreatedDate.') 24 | 25 | parser.add_argument('db', type=str, 26 | help='SQLAlchemy path of database from which to delete data.') 27 | parser.add_argument('--before', type=str, required=True, 28 | help='Adjust all records with created date before this date.') 29 | parser.add_argument('--adjust', type=str, required=True, 30 | help='Time delta to adjust time forward by, for example "3 days 2 hours"') 31 | 32 | parser.add_argument('--dry-run', action='store_true', help='Print but do not execute command') 33 | 34 | args = parser.parse_args() 35 | 36 | before = dateparser.parse(args.before) 37 | adjust = timedelta(seconds=timeparse(args.adjust)) 38 | 39 | if '://' not in args.db: 40 | args.db = NVivo.mount(args.db) 41 | 42 | db = create_engine(args.db) 43 | md = MetaData(bind=db) 44 | md.reflect(db) 45 | con = db.connect() 46 | tr = con.begin() 47 | 48 | datetimeNow = datetime.utcnow() 49 | 50 | for table in md.sorted_tables: 51 | CreatedDate = table.c.get('CreatedDate') 52 | ModifiedDate = table.c.get('ModifiedDate') 53 | if CreatedDate is not None and ModifiedDate is not None: 54 | # Prepend columns with '_' to avoid bindparam conflict error with reserved names 55 | rows = [{'_'+key:value for key,value in dict(row).items()} for row in con.execute( 56 | select( 57 | table.primary_key.columns + [CreatedDate, ModifiedDate]).where(or_( 58 | table.c.CreatedDate <= bindparam('Before'), 59 | table.c.CreatedDate > table.c.ModifiedDate)), { 60 | 'Before': before 61 | })] 62 | 63 | keycondition = [column == bindparam('_'+column.name) for column in table.primary_key.columns] 64 | 65 | print ("Table " + table.name + " Updating " + str(len(rows)) + " rows.") 66 | for row in rows: 67 | if type(row['_CreatedDate']) == str: # ??? 68 | createdDate = dateparser.parse(row['_CreatedDate']) 69 | modifiedDate = dateparser.parse(row['_ModifiedDate']) 70 | else: 71 | createdDate = row['_CreatedDate'] 72 | modifiedDate = row['_ModifiedDate'] 73 | if createdDate <= before: 74 | createdDate += adjust 75 | if modifiedDate <= before: 76 | modifiedDate += adjust 77 | if createdDate > datetimeNow or modifiedDate > datetimeNow: 78 | print("WARNING: future date", file=sys.stderr) 79 | 80 | if createdDate > modifiedDate: 81 | print("WARNING: created date", createdDate, "later than modified date", modifiedDate, file=sys.stderr) 82 | #row['_ModifiedDate'] = row['_CreatedDate'] 83 | 84 | row['_CreatedDate'] = createdDate 85 | row['_ModifiedDate'] = modifiedDate 86 | 87 | if not args.dry_run: 88 | con.execute(table.update( 89 | and_(*keycondition)).values({ 90 | 'CreatedDate': bindparam('_CreatedDate'), 91 | 'ModifiedDate': bindparam('_ModifiedDate')}), row) 92 | 93 | if not args.dry_run: 94 | tr.commit() 95 | 96 | except: 97 | raise 98 | if tr: 99 | tr.rollback() 100 | 101 | finally: 102 | if con: 103 | con.close() 104 | if db: 105 | db.dispose() 106 | -------------------------------------------------------------------------------- /editProject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from argrecord import ArgumentHelper, ArgumentRecorder 20 | import os 21 | import sys 22 | from NVivoNorm import NVivoNorm 23 | from sqlalchemy import * 24 | import re 25 | from dateutil import parser as dateparser 26 | from datetime import date, time, datetime 27 | from distutils import util 28 | import uuid 29 | 30 | def editProject(arglist=None): 31 | 32 | parser = ArgumentRecorder(description='Insert or update project in normalised file.') 33 | 34 | generalgroup = parser.add_argument_group('General') 35 | generalgroup.add_argument('-o', '--outfile', type=str, required=True, output=True, 36 | help='Output normalised NVivo (.norm) file') 37 | generalgroup.add_argument('-t', '--title', type=str, 38 | required=True) 39 | generalgroup.add_argument('-d', '--description', type=str) 40 | generalgroup.add_argument('-u', '--user', type=str, 41 | help='User, default is first user from user table') 42 | 43 | advancedgroup = parser.add_argument_group('Advanced') 44 | advancedgroup.add_argument('-v', '--verbosity', type=int, default=1, 45 | private=True) 46 | advancedgroup.add_argument('--logfile', type=str, help="Logfile, default is .log", 47 | private=True) 48 | advancedgroup.add_argument('--no-logfile', action='store_true', help='Do not output a logfile') 49 | 50 | args = parser.parse_args(arglist) 51 | 52 | if not args.no_logfile: 53 | logfilename = args.outfile.rsplit('.',1)[0] + '.log' 54 | incomments = ArgumentHelper.read_comments(logfilename) or ArgumentHelper.separator() 55 | logfile = open(logfilename, 'w') 56 | parser.write_comments(args, logfile, incomments=incomments) 57 | logfile.close() 58 | 59 | try: 60 | 61 | norm = NVivoNorm(args.outfile) 62 | norm.begin() 63 | 64 | if args.user: 65 | userRecord = norm.con.execute(select([ 66 | norm.User.c.Id 67 | ]).where( 68 | norm.User.c.Name == bindparam('Name') 69 | ), { 70 | 'Name': args.user 71 | }).first() 72 | if userRecord: 73 | userId = userRecord['Id'] 74 | else: 75 | userId = uuid.uuid4() 76 | norm.con.execute(norm.User.insert(), { 77 | 'Id': userId, 78 | 'Name': args.user 79 | }) 80 | else: 81 | userRecord = norm.con.execute(select([ 82 | norm.User.c.Id 83 | ])).first() 84 | if userRecord is not None: 85 | userId = userRecord['Id'] 86 | else: 87 | raise RuntimeError("No user on command line or user file") 88 | 89 | project = norm.con.execute(select([ 90 | norm.Project.c.Title 91 | ])).first() 92 | 93 | datetimeNow = datetime.utcnow() 94 | 95 | projectColumns = {'Version': u'0.2'} 96 | if args.title: 97 | projectColumns.update({'Title': args.title}) 98 | if args.description: 99 | projectColumns.update({'Description': args.description}) 100 | projectColumns.update({'ModifiedBy': userId, 101 | 'ModifiedDate': datetimeNow}) 102 | 103 | if project is None: # New project 104 | projectColumns.update({'CreatedBy': userId, 105 | 'CreatedDate': datetimeNow}) 106 | norm.con.execute(norm.Project.insert(), projectColumns) 107 | else: 108 | norm.con.execute(norm.Project.update(), projectColumns) 109 | 110 | norm.commit() 111 | 112 | except: 113 | raise 114 | norm.rollback() 115 | 116 | finally: 117 | del norm 118 | 119 | if __name__ == '__main__': 120 | editProject(None) 121 | -------------------------------------------------------------------------------- /NVP2RQDA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | import NVivo 22 | import RQDA 23 | import os 24 | import sys 25 | import shutil 26 | import tempfile 27 | from subprocess import Popen, PIPE 28 | 29 | parser = argparse.ArgumentParser(description='Convert an NVivo for Windows (.nvp) file into an RQDA project.') 30 | 31 | parser.add_argument('-v', '--verbosity', type=int, default=1) 32 | 33 | parser.add_argument('-i', '--instance', type=str, nargs='?', 34 | help="Microsoft SQL Server instance") 35 | 36 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 37 | help='NVivo version (10 or 11)') 38 | 39 | parser.add_argument('-u', '--users', choices=["skip", "overwrite"], default="merge", 40 | help='User action.') 41 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 42 | help='Project action.') 43 | parser.add_argument('-nc', '--node-categories', choices=["skip", "overwrite"], default="merge", 44 | help='Node category action.') 45 | parser.add_argument('-n', '--nodes', choices=["skip", "overwrite"], default="merge", 46 | help='Node action.') 47 | parser.add_argument('-c', '--cases', choices=["skip", "overwrite"], default="merge", 48 | help='case action.') 49 | parser.add_argument('-ca', '--node-attributes', choices=["skip", "overwrite"], default="merge", 50 | help='Case attribute table action.') 51 | parser.add_argument('-sc', '--source-categories', choices=["skip", "overwrite"], default="merge", 52 | help='Source category action.') 53 | parser.add_argument('-s', '--sources', choices=["skip", "overwrite"], default="merge", 54 | help='Source action.') 55 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "overwrite"], default="merge", 56 | help='Source attribute action.') 57 | parser.add_argument('-t', '--taggings', choices=["skip", "overwrite"], default="merge", 58 | help='Tagging action.') 59 | parser.add_argument('-a', '--annotations', choices=["skip", "overwrite"], default="merge", 60 | help='Annotation action.') 61 | 62 | parser.add_argument('infile', type=argparse.FileType('rb'), 63 | help="Input NVivo for Windows (.nvp) file") 64 | parser.add_argument('outfilename', type=str, nargs='?', 65 | help="Output RQDA file") 66 | 67 | args = parser.parse_args() 68 | 69 | # Fill in extra arguments that NVivo module expects 70 | args.mac = False 71 | args.windows = True 72 | 73 | tmpinfilename = tempfile.mktemp() 74 | tmpinfileptr = open(tmpinfilename, 'wb') 75 | tmpinfileptr.write(args.infile.read()) 76 | args.infile.close() 77 | tmpinfileptr.close() 78 | 79 | if args.outfilename is None: 80 | args.outfilename = args.infile.name.rsplit('.',1)[0] + '.rqda' 81 | 82 | tmpnormfilename = tempfile.mktemp() 83 | 84 | helperpath = os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'helpers' + os.path.sep 85 | 86 | if args.instance is None: 87 | proc = Popen([helperpath + 'mssqlInstance.bat'], stdout=PIPE) 88 | args.instance = proc.stdout.readline()[0:-len(os.linesep)] 89 | if args.verbosity > 0: 90 | print("Using MSSQL instance: " + args.instance, file=sys.stderr) 91 | 92 | # Get reasonably distinct yet recognisable DB name 93 | dbname = 'nt' + str(os.getpid()) 94 | 95 | proc = Popen([helperpath + 'mssqlAttach.bat', tmpinfilename, dbname, args.instance]) 96 | proc.wait() 97 | if args.verbosity > 0: 98 | print("Attached database " + dbname, file=sys.stderr) 99 | 100 | args.indb = 'mssql+pymssql://nvivotools:nvivotools@localhost/' + dbname 101 | args.outdb = 'sqlite:///' + tmpnormfilename 102 | 103 | NVivo.Normalise(args) 104 | 105 | proc = Popen([helperpath + 'mssqlDrop.bat', dbname, args.instance]) 106 | proc.wait() 107 | if args.verbosity > 0: 108 | print("Dropped database " + dbname, file=sys.stderr) 109 | os.remove(tmpinfilename) 110 | 111 | tmpoutfilename = tempfile.mktemp() 112 | 113 | args.indb = 'sqlite:///' + tmpnormfilename 114 | args.outdb = 'sqlite:///' + tmpoutfilename 115 | 116 | # Small hack 117 | args.case_attributes = args.node_attributes 118 | 119 | RQDA.Norm2RQDA(args) 120 | 121 | shutil.move(tmpoutfilename, os.path.basename(args.outfilename)) 122 | os.remove(tmpnormfilename) 123 | -------------------------------------------------------------------------------- /RQDA2NVP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | import NVivo 22 | import RQDA 23 | import os 24 | import sys 25 | import shutil 26 | import tempfile 27 | from subprocess import Popen, PIPE 28 | 29 | parser = argparse.ArgumentParser(description='Convert an RQDA project to NVivo for Windows (.nvp) format.') 30 | 31 | parser.add_argument('-v', '--verbosity', type=int, default=1) 32 | 33 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 34 | help='NVivo version (10 or 11)') 35 | 36 | parser.add_argument('-i', '--instance', type=str, nargs='?', 37 | help="Microsoft SQL Server instance") 38 | 39 | parser.add_argument('-u', '--users', choices=["skip", "overwrite"], default="merge", 40 | help='User action.') 41 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 42 | help='Project action.') 43 | parser.add_argument('-nc', '--node-categories', choices=["skip", "overwrite"], default="merge", 44 | help='Node category action.') 45 | parser.add_argument('-n', '--nodes', choices=["skip", "overwrite"], default="merge", 46 | help='Node action.') 47 | parser.add_argument('-c', '--cases', choices=["skip", "overwrite"], default="merge", 48 | help='case action.') 49 | parser.add_argument('-ca', '--case-attributes', choices=["skip", "overwrite"], default="merge", 50 | help='Case attribute table action.') 51 | parser.add_argument('-sc', '--source-categories', choices=["skip", "overwrite"], default="merge", 52 | help='Source category action.') 53 | parser.add_argument('-s', '--sources', choices=["skip", "overwrite"], default="merge", 54 | help='Source action.') 55 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "overwrite"], default="merge", 56 | help='Source attribute action.') 57 | parser.add_argument('-t', '--taggings', choices=["skip", "overwrite"], default="merge", 58 | help='Tagging action.') 59 | parser.add_argument('-a', '--annotations', choices=["skip", "overwrite"], default="merge", 60 | help='Annotation action.') 61 | 62 | parser.add_argument('-b', '--base', dest='basefile', type=argparse.FileType('rb'), nargs='?', 63 | help="Base NVPX file to insert into") 64 | 65 | parser.add_argument('infile', type=argparse.FileType('rb'), 66 | help="Input RQDA file") 67 | parser.add_argument('outfilename', metavar='outfile', type=str, nargs='?', 68 | help="Output NVP file") 69 | 70 | args = parser.parse_args() 71 | 72 | # Fill in extra arguments that NVivo module expects 73 | args.mac = False 74 | args.windows = True 75 | 76 | tmpinfilename = tempfile.mktemp() 77 | tmpinfileptr = open(tmpinfilename, 'wb') 78 | tmpinfileptr.write(args.infile.read()) 79 | args.infile.close() 80 | tmpinfileptr.close() 81 | 82 | if args.outfilename is None: 83 | args.outfilename = args.infile.name.rsplit('.',1)[0] + '.nvpx' 84 | 85 | if args.basefile is None: 86 | args.basefile = open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + ('emptyNVivo10Win.nvp' if args.nvivoversion == '10' else 'emptyNVivo11Win.nvp'), 'rb') 87 | 88 | tmpnormfilename = tempfile.mktemp() 89 | 90 | args.indb = 'sqlite:///' + tmpinfilename 91 | args.outdb = 'sqlite:///' + tmpnormfilename 92 | RQDA.RQDA2Norm(args) 93 | 94 | os.remove(tmpinfilename) 95 | 96 | tmpoutfilename = tempfile.mktemp() 97 | tmpoutfileptr = open(tmpoutfilename, 'wb') 98 | tmpoutfileptr.write(args.basefile.read()) 99 | args.basefile.close() 100 | tmpoutfileptr.close() 101 | 102 | helperpath = os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'helpers' + os.path.sep 103 | 104 | if args.instance is None: 105 | proc = Popen([helperpath + 'mssqlInstance.bat'], stdout=PIPE) 106 | args.instance = proc.stdout.readline()[0:-len(os.linesep)] 107 | if args.verbosity > 0: 108 | print("Using MSSQL instance: " + args.instance, file=sys.stderr) 109 | 110 | # Get reasonably distinct yet recognisable DB name 111 | dbname = 'nt' + str(os.getpid()) 112 | 113 | proc = Popen([helperpath + 'mssqlAttach.bat', tmpoutfilename, dbname, args.instance]) 114 | proc.wait() 115 | if args.verbosity > 0: 116 | print("Attached database " + dbname, file=sys.stderr) 117 | 118 | args.indb = 'sqlite:///' + tmpnormfilename 119 | args.outdb = 'mssql+pymssql://nvivotools:nvivotools@localhost/' + dbname 120 | 121 | # Small hack 122 | args.node_attributes = args.case_attributes 123 | 124 | NVivo.Denormalise(args) 125 | 126 | proc = Popen([helperpath + 'mssqlSave.bat', tmpoutfilename, dbname, args.instance]) 127 | proc.wait() 128 | if args.verbosity > 0: 129 | print("Saved database " + dbname, file=sys.stderr) 130 | 131 | shutil.move(tmpoutfilename, args.outfilename) 132 | os.remove(tmpnormfilename) 133 | -------------------------------------------------------------------------------- /Subtract.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import print_function 5 | from sqlalchemy import exc, TypeDecorator, CHAR, String, create_engine, MetaData, bindparam 6 | from sqlalchemy.engine import reflection 7 | import warnings 8 | import sys 9 | import os 10 | import argparse 11 | import uuid 12 | 13 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 14 | 15 | try: 16 | 17 | parser = argparse.ArgumentParser(description='Subtract the contents of one database from another.') 18 | 19 | parser.add_argument('-r', '--reverse', action='store_true', 20 | help='Reverse the subtraction, that is swap the minuent and subtrahend.') 21 | parser.add_argument('-i', '--ignore', type=str, nargs='?', 22 | help='Comma-separated list of columns to ignore when doing record comparison.') 23 | 24 | parser.add_argument('minuend', type=str, 25 | help='Path of database from which to subtract contents.') 26 | parser.add_argument('subtrahend', type=str, 27 | help='Path of database whose contents are to be subracted from the first database.') 28 | parser.add_argument('difference', type=str, nargs='?', 29 | help='Path of database to be populated with the difference.') 30 | 31 | args = parser.parse_args() 32 | 33 | if args.reverse: 34 | args.minuend, args.subtrahend = args.subtrahend, args.minuend 35 | 36 | if args.ignore != None: 37 | ignorecols = args.ignore.split(",") 38 | else: 39 | ignorecols = [] 40 | 41 | minuenddb = create_engine(args.minuend) 42 | minuendmd = MetaData() 43 | minuendmd.reflect(minuenddb) 44 | 45 | subtrahenddb = create_engine(args.subtrahend) 46 | subtrahendmd = MetaData() 47 | subtrahendmd.reflect(subtrahenddb) 48 | 49 | if args.difference != None: 50 | differencedb = create_engine(args.difference) 51 | differencemd = MetaData() 52 | differencemd.reflect(differencedb) 53 | differenceconn = differencedb.connect() 54 | differencetrans = differenceconn.begin() 55 | inspector = reflection.Inspector.from_engine(differencedb) 56 | 57 | for minuendtable in minuendmd.sorted_tables: 58 | if args.difference != None: 59 | if minuendtable.name not in differencemd.tables.keys(): 60 | print("Creating table: " + minuendtable.name, file=sys.stderr) 61 | minuendtable.create(differenceconn) 62 | 63 | for minuendtable in minuendmd.sorted_tables: 64 | subtrahendtable = subtrahendmd.tables[minuendtable.name] 65 | if subtrahendtable != None: 66 | subtrahendrows = subtrahenddb.execute(subtrahendtable.select()) 67 | subtrahendrows = [dict(row) for row in subtrahendrows] 68 | for row in subtrahendrows: 69 | for ignorecolumn in ignorecols: 70 | row[ignorecolumn] = None 71 | 72 | minuendrows = minuenddb.execute(minuendtable.select()) 73 | minuendrows = [dict(row) for row in minuendrows] 74 | for row in minuendrows: 75 | for ignorecolumn in ignorecols: 76 | row[ignorecolumn] = None 77 | 78 | differencerows = [ x for x in minuendrows if not x in subtrahendrows ] 79 | 80 | if len(differencerows) > 0: 81 | if args.difference != None: 82 | print("Finding foreign key references for table " + minuendtable.name, file=sys.stderr) 83 | for fk in inspector.get_foreign_keys(minuendtable.name): 84 | if not fk['name']: 85 | continue 86 | 87 | #print(" " + fk['name'], file=sys.stderr) 88 | fkreferredtable = minuendmd.tables[fk['referred_table']] 89 | fkselect = fkreferredtable.select() 90 | for referred_column, constrained_column in zip(fk['referred_columns'], fk['constrained_columns']): 91 | fkselect = fkselect.where(fkreferredtable.c[referred_column] == bindparam(constrained_column)) 92 | 93 | #print(fkselect, file=sys.stderr) 94 | 95 | fkrows = [] 96 | fkexists = [] 97 | for differencerow in differencerows: 98 | fkrow = minuenddb.execute(fkselect, differencerow) 99 | fkrows += [dict(row) for row in fkrow if not dict(row) in fkrows] 100 | 101 | fkexist = differenceconn.execute(fkselect, differencerow) 102 | fkexists += [dict(row) for row in fkexist if not dict(row) in fkexists] 103 | 104 | fkinsert = [ x for x in fkrows if not x in fkexists ] 105 | if len(fkinsert) > 0: 106 | differencereferredtable = differencemd.tables[fk['referred_table']] 107 | #print( "fkinsert: " + str(fkinsert)) differenceconn.execute(differencereferredtable.insert(), fkinsert, file=sys.stderr) 108 | 109 | differenceconn.execute(minuendtable.insert(), differencerows) 110 | else: 111 | print("-------------- " + minuendtable.name + " --------------", file=sys.stderr) 112 | for row in differencerows: 113 | print(row, file=sys.stderr) 114 | 115 | # All done. 116 | 117 | if args.difference != None: 118 | differencetrans.commit() 119 | 120 | except exc.SQLAlchemyError: 121 | raise 122 | -------------------------------------------------------------------------------- /querySource.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | import os 20 | import sys 21 | import argparse 22 | from NVivoNorm import NVivoNorm 23 | from sqlalchemy import * 24 | import re 25 | import csv 26 | import shutil 27 | 28 | def add_arguments(parser): 29 | parser.description = "Query sources in a normalised file." 30 | 31 | generalgroup = parser.add_argument_group('General') 32 | generalgroup.add_argument( 'infile', type=str, 33 | help='Input normalised NVivo (.nvpn) file') 34 | generalgroup.add_argument('-o', '--outfile', type=str, 35 | help='Output file') 36 | generalgroup.add_argument('-s', '--source', type=str) 37 | generalgroup.add_argument('-c', '--category', type=str) 38 | 39 | advancedgroup = parser.add_argument_group('Advanced') 40 | advancedgroup.add_argument('-v', '--verbosity', type=int, default=1) 41 | advancedgroup.add_argument('--no-comments', action='store_true', help='Do not produce a comments logfile') 42 | 43 | parser.set_defaults(func=querySource) 44 | parser.set_defaults(build_comments=build_comments) 45 | parser.set_defaults(hiddenargs=['hiddenargs', 'verbosity', 'no_comments']) 46 | 47 | def parse_arguments(): 48 | parser = argparse.ArgumentParser() 49 | add_arguments(parser) 50 | return vars(parser.parse_args()) 51 | 52 | def build_comments(kwargs): 53 | comments = ((' ' + kwargs['outfile'] + ' ') if kwargs['outfile'] else '').center(80, '#') + '\n' 54 | comments += '# ' + os.path.basename(__file__) + '\n' 55 | hiddenargs = kwargs['hiddenargs'] + ['hiddenargs', 'func', 'build_comments'] 56 | for argname, argval in kwargs.items(): 57 | if argname not in hiddenargs: 58 | if type(argval) == str: 59 | comments += '# --' + argname + '="' + argval + '"\n' 60 | elif type(argval) == bool: 61 | if argval: 62 | comments += '# --' + argname + '\n' 63 | elif type(argval) == list: 64 | for valitem in argval: 65 | if type(valitem) == str: 66 | comments += '# --' + argname + '="' + valitem + '"\n' 67 | else: 68 | comments += '# --' + argname + '=' + str(valitem) + '\n' 69 | elif argval is not None: 70 | comments += '# --' + argname + '=' + str(argval) + '\n' 71 | 72 | return comments 73 | 74 | def querySource(infile, outfile, 75 | source, category, 76 | verbosity, no_comments, 77 | comments, **dummy): 78 | 79 | try: 80 | norm = NVivoNorm(infile) 81 | norm.begin() 82 | 83 | sourcesel = select([ 84 | norm.Source.c.Name, 85 | norm.Source.c.Description, 86 | norm.SourceCategory.c.Name.label('Category'), 87 | norm.Source.c.Color, 88 | norm.Source.c.Content 89 | ]).select_from( 90 | norm.Source.outerjoin(norm.SourceCategory, 91 | norm.SourceCategory.c.Id == norm.Source.c.Category) 92 | ) 93 | params = {} 94 | 95 | if source: 96 | sourcesel = sourcesel.where( 97 | norm.Source.c.Name == bindparam('Source') 98 | ) 99 | params.update({'Source': source}) 100 | 101 | if category: 102 | sourcesel = sourcesel.where(and_( 103 | norm.Source.c.Category == norm.SourceCategory.c.Id, 104 | norm.SourceCategory.c.Name == bindparam('SourceCategory') 105 | )) 106 | params.update({'SourceCategory': category}) 107 | 108 | if outfile: 109 | if os.path.exists(outfile): 110 | shutil.move(outfile, outfile + '.bak') 111 | 112 | csvfile = open(outfile, 'w') 113 | else: 114 | csvfile = sys.stdout 115 | 116 | if not no_comments: 117 | csvfile.write(comments) 118 | csvfile.write('#' * 80 + '\n') 119 | 120 | csvwriter = csv.DictWriter(csvfile, 121 | fieldnames=['Name', 'Description', 'Content', 'Category', 'Color'], 122 | extrasaction='ignore', 123 | lineterminator=os.linesep, 124 | quoting=csv.QUOTE_NONNUMERIC) 125 | 126 | csvwriter.writeheader() 127 | 128 | for source in norm.con.execute(sourcesel, params): 129 | csvwriter.writerow(dict(source)) 130 | 131 | csvfile.close() 132 | 133 | except: 134 | raise 135 | 136 | finally: 137 | del norm 138 | 139 | def main(): 140 | kwargs = parse_arguments() 141 | kwargs['comments'] = build_comments(kwargs) 142 | kwargs['func'](**kwargs) 143 | 144 | if __name__ == '__main__': 145 | main() 146 | -------------------------------------------------------------------------------- /NVPX2RQDA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import os 21 | import subprocess 22 | import re 23 | import sys 24 | 25 | # First set up environment for SQL Anywhere server and restart process if necessary 26 | helperpath = os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'helpers' + os.path.sep 27 | 28 | # Set environment variables for SQL Anywhere server 29 | if not os.environ.get('_sqlanywhere'): 30 | envlines = subprocess.check_output(helperpath + 'sqlanyenv.sh', text=True).splitlines() 31 | for envline in envlines: 32 | env = re.match(r"(?P\w+)=(?P['\"]?)(?P.+)(?P=quote)", envline).groupdict() 33 | os.environ[env['name']] = env['value'] 34 | 35 | os.environ['_sqlanywhere'] = 'TRUE' 36 | os.execv(sys.argv[0], sys.argv) 37 | 38 | # Environment is now ready 39 | import argparse 40 | import NVivo 41 | import RQDA 42 | import shutil 43 | import tempfile 44 | 45 | parser = argparse.ArgumentParser(description='Convert an NVivo for Mac (.nvpx) file into an RQDA project.') 46 | 47 | # --cmdline argument means retain full output file path name, otherwise strip directory, 48 | # so that things work under Wooey. 49 | parser.add_argument('--cmdline', action='store_true', 50 | help=argparse.SUPPRESS) 51 | 52 | parser.add_argument('-v', '--verbosity', type=int, default=1) 53 | 54 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 55 | help='NVivo version (10 or 11)') 56 | 57 | parser.add_argument('-u', '--users', choices=["skip", "overwrite"], default="overwrite", 58 | help='User action.') 59 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 60 | help='Project action.') 61 | parser.add_argument('-nc', '--node-categories', choices=["skip", "overwrite"], default="overwrite", 62 | help='Node category action.') 63 | parser.add_argument('-n', '--nodes', choices=["skip", "overwrite"], default="overwrite", 64 | help='Node action.') 65 | parser.add_argument('-c', '--cases', choices=["skip", "overwrite"], default="overwrite", 66 | help='case action.') 67 | parser.add_argument('-ca', '--node-attributes', choices=["skip", "overwrite"], default="overwrite", 68 | help='Case attribute table action.') 69 | parser.add_argument('-sc', '--source-categories', choices=["skip", "overwrite"], default="overwrite", 70 | help='Source category action.') 71 | parser.add_argument('-s', '--sources', choices=["skip", "overwrite"], default="overwrite", 72 | help='Source action.') 73 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "overwrite"], default="overwrite", 74 | help='Source attribute action.') 75 | parser.add_argument('-t', '--taggings', choices=["skip", "overwrite"], default="overwrite", 76 | help='Tagging action.') 77 | parser.add_argument('-a', '--annotations', choices=["skip", "overwrite"], default="overwrite", 78 | help='Annotation action.') 79 | 80 | parser.add_argument('infile', type=argparse.FileType('rb'), 81 | help="Input NVivo for Mac file (extension .nvpx)") 82 | parser.add_argument('outfilename', type=str, nargs='?', 83 | help="Output RQDA file") 84 | 85 | args = parser.parse_args() 86 | 87 | # Fill in extra arguments that NVivo module expects 88 | args.mac = True 89 | args.windows = False 90 | 91 | tmpinfilename = tempfile.mktemp() 92 | tmpinfileptr = open(tmpinfilename, 'wb') 93 | tmpinfileptr.write(args.infile.read()) 94 | args.infile.close() 95 | tmpinfileptr.close() 96 | 97 | if args.outfilename is None: 98 | args.outfilename = args.infile.name.rsplit('.',1)[0] + '.rqda' 99 | 100 | tmpnormfilename = tempfile.mktemp() 101 | 102 | # Find a free sock for SQL Anywhere server to bind to 103 | import socket 104 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 105 | s.bind(("",0)) 106 | freeport = str(s.getsockname()[1]) 107 | s.close() 108 | 109 | DEVNULL = open(os.devnull, 'wb') 110 | dbproc = subprocess.Popen(['sh', helperpath + 'sqlanysrv.sh', '-x TCPIP(port='+freeport+')', '-ga', '-xd', tmpinfilename, '-n', 'NVivo'+freeport], text=True, 111 | stdout=subprocess.PIPE, stdin=DEVNULL) 112 | 113 | # Wait until SQL Anywhere engine starts... 114 | while dbproc.poll() is None: 115 | line = dbproc.stdout.readline() 116 | if line == 'Now accepting requests\n': 117 | break 118 | 119 | if dbproc.poll() is not None: 120 | raise RuntimeError("Failed to start database server") 121 | 122 | if args.verbosity > 0: 123 | print("Started database server on port " + freeport, file=sys.stderr) 124 | 125 | args.indb = 'sqlalchemy_sqlany://wiwalisataob2aaf:iatvmoammgiivaam@localhost:' + freeport + '/NVivo' + freeport 126 | args.outdb = 'sqlite:///' + tmpnormfilename 127 | 128 | NVivo.Normalise(args) 129 | 130 | os.remove(tmpinfilename) 131 | 132 | tmpoutfilename = tempfile.mktemp() 133 | 134 | args.indb = 'sqlite:///' + tmpnormfilename 135 | args.outdb = 'sqlite:///' + tmpoutfilename 136 | 137 | # Small hack 138 | args.case_attributes = args.node_attributes 139 | 140 | RQDA.Norm2RQDA(args) 141 | 142 | if not args.cmdline: 143 | args.outfilename = os.path.basename(args.outfilename) 144 | 145 | shutil.move(tmpoutfilename, args.outfilename) 146 | os.remove(tmpnormfilename) 147 | -------------------------------------------------------------------------------- /nvpn2bqda.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | import os 20 | import sys 21 | from argrecord import ArgumentHelper, ArgumentRecorder 22 | from NVivoNorm import NVivoNorm 23 | from sqlalchemy import * 24 | import re 25 | import csv 26 | import shutil 27 | import re 28 | 29 | def nvpn2bqda(arglist=None): 30 | parser = ArgumentRecorder(description='Export NVPN to BarraQDA project.', 31 | fromfile_prefix_chars='@') 32 | 33 | parser.description = "Export NVPN to BarraQDA project." 34 | 35 | generalgroup = parser.add_argument_group('General') 36 | generalgroup.add_argument('-ob', '--outbase', type=str, help='Output base path') 37 | generalgroup.add_argument('infile', type=str, help='Input NVPN file', input=True) 38 | 39 | advancedgroup = parser.add_argument_group('Advanced') 40 | advancedgroup.add_argument('-v', '--verbosity', type=int, default=1, private=True) 41 | advancedgroup.add_argument('--logfile', type=str, help="Logfile") 42 | 43 | args = parser.parse_args(arglist) 44 | 45 | if args.logfile: 46 | logfile = open(logfilename, 'w') 47 | parser.write_comments(args, logfile, incomments=ArgumentHelper.separator()) 48 | logfile.close() 49 | 50 | try: 51 | norm = NVivoNorm(args.infile) 52 | norm.begin() 53 | 54 | docdata = os.path.join(args.outbase, 'docdata') 55 | if not os.path.exists(docdata): 56 | os.mkdir(docdata) 57 | 58 | sourcesel = select([ 59 | norm.Source.c.Id, 60 | norm.Source.c.Name, 61 | norm.Source.c.ObjectType, 62 | norm.Source.c.Object 63 | ]) 64 | nodesel = select([ 65 | norm.Node.c.Id, 66 | norm.Node.c.Name, 67 | ]) 68 | nodeattrsel = select([ 69 | norm.NodeAttribute.c.Name.label('Attribute'), 70 | norm.NodeValue.c.Value 71 | ]).select_from( 72 | norm.NodeAttribute.join( 73 | norm.NodeValue, 74 | norm.NodeValue.c.Attribute == norm.NodeAttribute.c.Id 75 | )).where(norm.NodeValue.c.Node == bindparam('Node')) 76 | taggingsel = select([ 77 | norm.Tagging.c.Node, 78 | norm.Tagging.c.Fragment, 79 | norm.Tagging.c.Memo, 80 | norm.Tagging.c.ModifiedDate, 81 | norm.Tagging.c.CreatedDate, 82 | norm.User.c.Name.label('User') 83 | ]).select_from( 84 | norm.Tagging.join( 85 | norm.User, 86 | norm.User.c.Id == norm.Tagging.c.ModifiedBy 87 | )).where( 88 | norm.Tagging.c.Source == bindparam('Source'), 89 | ) 90 | fragmentregex = re.compile(r'(?P[0-9]+):(?P[0-9]+)') 91 | 92 | for source in norm.con.execute(sourcesel): 93 | sourcefilename = os.path.join(args.outbase, source['Name'] + '.' + source['ObjectType'].lower()) 94 | sourcefile = open(sourcefilename, 'wb') 95 | sourcefile.write(source['Object']) 96 | sourcefile.close() 97 | docfilename = os.path.join(docdata, str(len(source['Object'])) + '.' + source['Name'] + '.' + source['ObjectType'].lower() + '.xml') 98 | docfile = open(docfilename, 'w') 99 | docfile.write(""" 100 | 101 | 102 | 103 | """) 104 | for node in norm.con.execute(nodesel): 105 | docfile.write(" \n") 111 | docfile.write(" \n") 112 | if anyattr: 113 | docfile.write(" \n") 114 | else: 115 | docfile.write("/>\n") 116 | docfile.write(" \n") 117 | 118 | anytagging = False 119 | for tagging in norm.con.execute(taggingsel, { 'Source': source['Id'] }): 120 | matchfragment = fragmentregex.match(tagging['Fragment']) 121 | start = int(matchfragment.group('start')) - 1 122 | end = int(matchfragment.group('end')) 123 | if not anytagging: 124 | anytagging = True 125 | docfile.write(""" 126 | 127 | 128 | """) 129 | docfile.write(""" 130 | 131 | \n") 132 | docfile.write(" 133 | """) 134 | if anytagging: 135 | docfile.write(""" 136 | 137 | 138 | """) 139 | 140 | docfile.write(""" 141 | """) 142 | 143 | except: 144 | raise 145 | 146 | finally: 147 | del norm 148 | 149 | if __name__ == '__main__': 150 | nvpn2bqda(None) 151 | -------------------------------------------------------------------------------- /mssqlTools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2017 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import os 21 | import sys 22 | import subprocess 23 | import tempfile 24 | import shutil 25 | import random 26 | 27 | class mssqlAPI(object): 28 | 29 | # Function to execute a command either locally or remotely 30 | def executecommand(self, command): 31 | if not self.server: # ie server is on same machine as this script 32 | return subprocess.check_output(command, text=True).strip() 33 | else: 34 | # This quoting of arguments is a bit of a hack but seems to work 35 | return subprocess.check_output(['ssh', self.sshserver] + [('"' + word + '"') if ' ' in word else word for word in command], text=True).strip() 36 | 37 | # Function to execute a helper script either locally or remotely 38 | def executescript(self, script, arglist=None): 39 | if not self.server: # ie server is on same machine as this script 40 | return subprocess.check_output([self.helperpath + script] + (arglist or []), text=True).strip() 41 | else: 42 | subprocess.call(['scp', '-q', self.helperpath + script, self.sshserver + ':' + self.tmpdir]) 43 | return subprocess.check_output(['ssh', self.sshserver, self.tmpdir + '\\' + script] + (arglist or []), text=True).strip() 44 | 45 | def __init__(self, server, user='', port=None, instance=None, version=None, verbosity=1): 46 | self.server = server 47 | self.user = user 48 | self.port = port 49 | self.instance = instance 50 | self.verbosity = verbosity 51 | 52 | self.helperpath = os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'helpers' + os.path.sep 53 | 54 | if self.server is None: # ie MSSQL server is on local machine 55 | if os.name != 'nt': 56 | raise RuntimeError("This does not appear to be a Windows machine so --server must be specified.") 57 | else: 58 | self.sshserver = ((self.user + '@') if self.user else '') + self.server 59 | self.tmpdir = subprocess.check_output(['ssh', self.sshserver, r'echo %tmp%'], text=True).strip() 60 | 61 | 62 | if self.instance is None: 63 | regquery = self.executecommand(['reg', 'query', 'HKLM\\Software\\Microsoft\\Microsoft SQL Server\\Instance Names\\SQL']).splitlines() 64 | for regqueryline in regquery[1:]: 65 | regquerydata = regqueryline.split() 66 | instancename = regquerydata[0] 67 | instanceversion = regquerydata[2].split('.')[0] 68 | if self.verbosity > 1: 69 | print("Found SQL server instance " + instancename + " version " + instanceversion, file=sys.stderr) 70 | if (not version or instanceversion == version): 71 | self.instance = instancename 72 | break 73 | else: 74 | raise RuntimeError('No suitable SQL self.server self.instance found') 75 | 76 | if self.verbosity > 1: 77 | print("Using MSSQL instance: " + self.instance, file=sys.stderr) 78 | 79 | if self.port is None: 80 | regquery = self.executecommand(['reg', 'query', 'HKLM\\SOFTWARE\\Microsoft\\Microsoft SQL Server\\' + self.instance + '\\MSSQLServer\\SuperSocketNetLib\\Tcp']).splitlines() 81 | self.port = int(regquery[1].split()[2]) 82 | 83 | if self.verbosity > 1: 84 | print("Using port: " + str(self.port), file=sys.stderr) 85 | 86 | def attach(self, filename, dbname): 87 | # Generate a filename for the temporary MDB file 88 | if self.server is None: # ie MSSQL server is on local machine 89 | mdbFilename = tempfile.mktemp() 90 | shutil.copy(filename, mdbFilename) 91 | else: 92 | mdbFilename = self.tmpdir + r'\mssqltools' + str(random.randint(0,99999)).zfill(5) 93 | subprocess.call(['scp', '-q', filename, self.sshserver + ':' + mdbFilename]) 94 | 95 | self.executescript('mssqlAttach.bat', [mdbFilename, dbname, self.instance]) 96 | if self.verbosity > 1: 97 | print("Attached database " + dbname, file=sys.stderr) 98 | 99 | def create(self, dbname): 100 | self.executescript('mssqlCreate.bat', [dbname, self.instance]) 101 | if self.verbosity > 1: 102 | print("Created database " + dbname, file=sys.stderr) 103 | 104 | def detach(self, dbname): 105 | serverinstance = 'localhost\\' + self.instance 106 | 107 | self.executecommand(['sqlcmd', '-S', serverinstance, '-Q', 'EXEC sp_detach_db ' + dbname]) 108 | 109 | def drop(self, dbname): 110 | serverinstance = 'localhost\\' + self.instance 111 | 112 | self.executecommand(['sqlcmd', '-S', serverinstance, '-Q', 'DROP DATABASE ' + dbname]) 113 | 114 | def save(self, filename, dbname): 115 | if self.server is None: # ie MSSQL server is on local machine 116 | self.executescript('mssqlSave.bat', [filename, dbname, self.instance]) 117 | else: 118 | mdbFilename = self.tmpdir + r'\mssqltools' + str(random.randint(0,99999)).zfill(5) 119 | self.executescript('mssqlSave.bat', [mdbFilename, dbname, self.instance]) 120 | subprocess.call(['scp', '-qT', self.sshserver + ':' + mdbFilename, filename]) 121 | 122 | def list(self): 123 | serverinstance = 'localhost\\' + self.instance 124 | 125 | dblist = self.executecommand(['sqlcmd', '-W', '-S', serverinstance, '-h', '-1', '-Q', 'SET NOCOUNT ON; SELECT name FROM master.dbo.sysdatabases']).split() 126 | 127 | # Ignore the first four internal databases: master, tempdb, model and msdb 128 | return dblist[4:] 129 | -------------------------------------------------------------------------------- /RQDA2NVPX.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import os 21 | import subprocess 22 | import re 23 | import sys 24 | 25 | # First set up environment for SQL Anywhere server and restart process if necessary 26 | helperpath = os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'helpers' + os.path.sep 27 | 28 | # Set environment variables for SQL Anywhere server 29 | if not os.environ.get('_sqlanywhere'): 30 | envlines = subprocess.check_output(helperpath + 'sqlanyenv.sh', text=True).splitlines() 31 | for envline in envlines: 32 | env = re.match(r"(?P\w+)=(?P['\"]?)(?P.+)(?P=quote)", envline).groupdict() 33 | os.environ[env['name']] = env['value'] 34 | 35 | os.environ['_sqlanywhere'] = 'TRUE' 36 | os.execv(sys.argv[0], sys.argv) 37 | 38 | # Environment is now ready 39 | import argparse 40 | import NVivo 41 | import RQDA 42 | import shutil 43 | import tempfile 44 | 45 | parser = argparse.ArgumentParser(description='Convert an RQDA project to NVivo for Mac (.nvpx) format.') 46 | 47 | # --cmdline argument means retain full output file path name, otherwise strip directory, 48 | # so that things work under Wooey. 49 | parser.add_argument('--cmdline', action='store_true', 50 | help=argparse.SUPPRESS) 51 | 52 | parser.add_argument('-v', '--verbosity', type=int, default=1) 53 | 54 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 55 | help='NVivo version (10 or 11)') 56 | 57 | parser.add_argument('-u', '--users', choices=["skip", "overwrite"], default="overwrite", 58 | help='User action.') 59 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 60 | help='Project action.') 61 | parser.add_argument('-nc', '--node-categories', choices=["skip", "overwrite"], default="overwrite", 62 | help='Node category action.') 63 | parser.add_argument('-n', '--nodes', choices=["skip", "overwrite"], default="overwrite", 64 | help='Node action.') 65 | parser.add_argument('-c', '--cases', choices=["skip", "overwrite"], default="overwrite", 66 | help='case action.') 67 | parser.add_argument('-ca', '--case-attributes', choices=["skip", "overwrite"], default="overwrite", 68 | help='Case attribute table action.') 69 | parser.add_argument('-sc', '--source-categories', choices=["skip", "overwrite"], default="overwrite", 70 | help='Source category action.') 71 | parser.add_argument('-s', '--sources', choices=["skip", "overwrite"], default="overwrite", 72 | help='Source action.') 73 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "overwrite"], default="overwrite", 74 | help='Source attribute action.') 75 | parser.add_argument('-t', '--taggings', choices=["skip", "overwrite"], default="overwrite", 76 | help='Tagging action.') 77 | parser.add_argument('-a', '--annotations', choices=["skip", "overwrite"], default="overwrite", 78 | help='Annotation action.') 79 | 80 | parser.add_argument('-b', '--base', dest='basefile', type=argparse.FileType('rb'), nargs='?', 81 | help="Base NVPX file to insert into") 82 | 83 | parser.add_argument('infile', type=argparse.FileType('rb'), 84 | help="Input RQDA file") 85 | parser.add_argument('outfilename', metavar='outfile', type=str, nargs='?', 86 | help="Output NVPX file") 87 | 88 | args = parser.parse_args() 89 | 90 | # Fill in extra arguments that NVivo module expects 91 | args.mac = True 92 | args.windows = False 93 | 94 | tmpinfilename = tempfile.mktemp() 95 | tmpinfileptr = open(tmpinfilename, 'wb') 96 | tmpinfileptr.write(args.infile.read()) 97 | args.infile.close() 98 | tmpinfileptr.close() 99 | 100 | if args.outfilename is None: 101 | args.outfilename = args.infile.name.rsplit('.',1)[0] + '.nvpx' 102 | 103 | if args.basefile is None: 104 | args.basefile = open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + ('emptyNVivo10Mac.nvpx' if args.nvivoversion == '10' else 'emptyNVivo11Mac.nvpx'), 'rb') 105 | 106 | tmpnormfilename = tempfile.mktemp() 107 | 108 | args.indb = 'sqlite:///' + tmpinfilename 109 | args.outdb = 'sqlite:///' + tmpnormfilename 110 | RQDA.RQDA2Norm(args) 111 | 112 | os.remove(tmpinfilename) 113 | 114 | tmpoutfilename = tempfile.mktemp() 115 | tmpoutfileptr = open(tmpoutfilename, 'wb') 116 | tmpoutfileptr.write(args.basefile.read()) 117 | args.basefile.close() 118 | tmpoutfileptr.close() 119 | 120 | # Find a free sock for SQL Anywhere server to bind to 121 | import socket 122 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 123 | s.bind(("",0)) 124 | freeport = str(s.getsockname()[1]) 125 | s.close() 126 | 127 | DEVNULL = open(os.devnull, 'wb') 128 | dbproc = subprocess.Popen(['sh', helperpath + 'sqlanysrv.sh', '-x TCPIP(port='+freeport+')', '-ga', '-xd', tmpoutfilename, '-n', 'NVivo'+freeport], text=True, 129 | stdout=subprocess.PIPE, stdin=DEVNULL) 130 | 131 | # Wait until SQL Anywhere engine starts... 132 | while dbproc.poll() is None: 133 | line = dbproc.stdout.readline() 134 | if line == 'Now accepting requests\n': 135 | break 136 | 137 | if dbproc.poll() is not None: 138 | raise RuntimeError("Failed to start database server") 139 | 140 | if args.verbosity > 0: 141 | print("Started database server on port " + freeport, file=sys.stderr) 142 | 143 | args.indb = 'sqlite:///' + tmpnormfilename 144 | args.outdb = 'sqlalchemy_sqlany://wiwalisataob2aaf:iatvmoammgiivaam@localhost:' + freeport + '/NVivo' + freeport 145 | 146 | # Small hack 147 | args.node_attributes = args.case_attributes 148 | 149 | chdir = os.environ.get('CHDIR') 150 | if chdir: 151 | cwd = os.getcwd() 152 | os.chdir(chdir) 153 | 154 | NVivo.Denormalise(args) 155 | 156 | if chdir: 157 | os.chdir(cwd) 158 | 159 | if not args.cmdline: 160 | args.outfilename = os.path.basename(args.outfilename) 161 | 162 | shutil.move(tmpoutfilename, args.outfilename) 163 | os.remove(tmpnormfilename) 164 | -------------------------------------------------------------------------------- /NormaliseNVP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import argparse 21 | import NVivo 22 | from mssqlTools import mssqlAPI 23 | import os 24 | import sys 25 | import shutil 26 | import subprocess 27 | import tempfile 28 | 29 | def NormaliseNVP(arglist): 30 | parser = argparse.ArgumentParser(description='Normalise an NVivo for Windows file.') 31 | 32 | parser.add_argument('-v', '--verbosity', type=int, default=1) 33 | 34 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 35 | help='NVivo version (10 or 11)') 36 | 37 | parser.add_argument('-S', '--server', type=str, 38 | help="IP address/name of Microsoft SQL Server") 39 | parser.add_argument('-P', '--port', type=int, 40 | help="Port of Microsoft SQL Server") 41 | parser.add_argument('-i', '--instance', type=str, 42 | help="Microsoft SQL Server instance") 43 | 44 | parser.add_argument('-u', '--users', choices=["skip", "merge", "overwrite", "replace"], default="merge", 45 | help='User action.') 46 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 47 | help='Project action.') 48 | parser.add_argument('-nc', '--node-categories', choices=["skip", "merge", "overwrite", "replace"], default="merge", 49 | help='Node category action.') 50 | parser.add_argument('-n', '--nodes', choices=["skip", "merge", "overwrite", "replace"], default="merge", 51 | help='Node action.') 52 | parser.add_argument('-na', '--node-attributes', choices=["skip", "merge", "overwrite", "replace"], default="merge", 53 | help='Node attribute table action.') 54 | parser.add_argument('-sc', '--source-categories', choices=["skip", "merge", "overwrite", "replace"], default="merge", 55 | help='Source category action.') 56 | parser.add_argument('--sources', choices=["skip", "merge", "overwrite", "replace"], default="merge", 57 | help='Source action.') 58 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "merge", "overwrite", "replace"], default="merge", 59 | help='Source attribute action.') 60 | parser.add_argument('-t', '--taggings', choices=["skip", "merge", "overwrite", "replace"], default="merge", 61 | help='Tagging action.') 62 | parser.add_argument('-a', '--annotations', choices=["skip", "merge", "overwrite", "replace"], default="merge", 63 | help='Annotation action.') 64 | 65 | parser.add_argument('infile', type=str, 66 | help="Input NVivo (.nvp) file") 67 | parser.add_argument('outfilename', type=str, nargs='?', 68 | help="Output normalised SQLite (.norm) file") 69 | 70 | args = parser.parse_args(arglist) 71 | 72 | # Function to execute a command either locally or remotely 73 | def executecommand(command): 74 | if not args.server: # ie server is on same machine as this script 75 | return subprocess.check_output(command, text=True).strip() 76 | else: 77 | # This quoting of arguments is a bit of a hack but seems to work 78 | return subprocess.check_output(['ssh', args.server] + [('"' + word + '"') if ' ' in word else word for word in command], text=True).strip() 79 | 80 | # Fill in extra arguments that NVivo module expects 81 | args.mac = False 82 | args.windows = True 83 | 84 | if args.server is None and os.name != 'nt': 85 | raise RuntimeError("This does not appear to be a Windows machine so --server must be specified.") 86 | 87 | tmpoutfilename = tempfile.mktemp() 88 | 89 | # Generate reasonable default output file name 90 | if args.outfilename is None: 91 | args.outfilename = args.infile.rsplit('.',1)[0] + '.norm' 92 | 93 | if args.instance is None: 94 | regquery = executecommand(['reg', 'query', 'HKLM\\Software\\Microsoft\\Microsoft SQL Server\\Instance Names\\SQL']).splitlines() 95 | for regqueryline in regquery[1:]: 96 | regquerydata = regqueryline.split() 97 | instancename = regquerydata[0] 98 | instanceversion = regquerydata[2].split('.')[0] 99 | if args.verbosity >= 2: 100 | print("Found SQL server instance " + instancename + " version " + instanceversion, file=sys.stderr) 101 | if (args.nvivoversion == '10' and instanceversion == 'MSSQL10_50') or (args.nvivoversion == '11' and instanceversion == 'MSSQL12'): 102 | args.instance = instancename 103 | break 104 | else: 105 | raise RuntimeError('No suitable SQL server instance found') 106 | 107 | if args.verbosity > 0: 108 | print("Using MSSQL instance: " + args.instance, file=sys.stderr) 109 | 110 | if args.port is None: 111 | regquery = executecommand(['reg', 'query', 'HKLM\\SOFTWARE\\Microsoft\\Microsoft SQL Server\\' + args.instance + '\\MSSQLServer\\SuperSocketNetLib\\Tcp']).splitlines() 112 | args.port = int(regquery[1].split()[2]) 113 | 114 | if args.verbosity > 0: 115 | print("Using port: " + str(args.port), file=sys.stderr) 116 | 117 | mssqlapi = mssqlAPI(args.server, 118 | args.port, 119 | args.instance, 120 | version = ('MSSQL12' if args.nvivoversion == '11' else 'MSSQL10_50'), 121 | verbosity = args.verbosity) 122 | 123 | # Get reasonably distinct yet recognisable DB name 124 | dbname = 'nvivo' + str(os.getpid()) 125 | 126 | mssqlapi.attach(args.infile, dbname) 127 | try: 128 | args.indb = 'mssql+pymssql://nvivotools:nvivotools@' + (args.server or 'localhost') + ((':' + str(args.port)) if args.port else '') + '/' + dbname 129 | args.outdb = 'sqlite:///' + tmpoutfilename 130 | 131 | NVivo.Normalise(args) 132 | 133 | shutil.move(tmpoutfilename, args.outfilename) 134 | 135 | except: 136 | raise 137 | 138 | finally: 139 | mssqlapi.drop(dbname) 140 | 141 | if __name__ == '__main__': 142 | NormaliseNVP(None) 143 | -------------------------------------------------------------------------------- /nvpn2nvp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from argrecord import ArgumentHelper, ArgumentRecorder 20 | import argparse 21 | import NVivo 22 | from mssqlTools import mssqlAPI 23 | import os 24 | import sys 25 | import shutil 26 | import subprocess 27 | import tempfile 28 | 29 | def DenormaliseNVP(arglist): 30 | parser = ArgumentRecorder(description='Create an NVivo for Mac file from a normalised SQLite file.') 31 | 32 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 33 | help='NVivo version (10 or 11)') 34 | 35 | parser.add_argument('-S', '--server', type=str, 36 | help="IP address/name of Microsoft SQL Server") 37 | parser.add_argument('-P', '--port', type=int, 38 | help="Port of Microsoft SQL Server") 39 | parser.add_argument('-i', '--instance', type=str, 40 | help="Microsoft SQL Server instance") 41 | parser.add_argument('-U', '--sshuser', type=str, 42 | help="User name for ssh connections to server") 43 | 44 | parser.add_argument('-u', '--users', choices=["skip", "merge", "overwrite", "replace"], default="merge", 45 | help='User action.') 46 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 47 | help='Project action.') 48 | parser.add_argument('-nc', '--node-categories', choices=["skip", "merge", "overwrite"], default="merge", 49 | help='Node category action.') 50 | parser.add_argument('-n', '--nodes', choices=["skip", "merge"], default="merge", 51 | help='Node action.') 52 | parser.add_argument('-na', '--node-attributes', choices=["skip", "merge", "overwrite"], default="merge", 53 | help='Node attribute table action.') 54 | parser.add_argument('-sc', '--source-categories', choices=["skip", "merge", "overwrite"], default="merge", 55 | help='Source category action.') 56 | parser.add_argument('--sources', choices=["skip", "merge", "overwrite"], default="merge", 57 | help='Source action.') 58 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "merge", "overwrite"], default="merge", 59 | help='Source attribute action.') 60 | parser.add_argument('-t', '--taggings', choices=["skip", "merge"], default="merge", 61 | help='Tagging action.') 62 | parser.add_argument('-a', '--annotations', choices=["skip", "merge"], default="merge", 63 | help='Annotation action.') 64 | 65 | parser.add_argument('-b', '--base', dest='basefile', type=argparse.FileType('rb'), nargs='?', 66 | help="Base NVP file to insert into") 67 | 68 | parser.add_argument('-v', '--verbosity', type=int, default=1, private=True) 69 | parser.add_argument('--logfile', type=str, help="Logfile, default is .log", 70 | private=True) 71 | parser.add_argument('--no-logfile', action='store_true', help='Do not output a logfile') 72 | 73 | parser.add_argument('infile', type=str, input=True, 74 | help="Input normalised SQLite (.nvpn) file") 75 | parser.add_argument('outfile', type=str, nargs='?', output=True, 76 | help="Output NVivo for Windows (.nvp) file or directory; default is .nvp") 77 | 78 | args = parser.parse_args(arglist) 79 | 80 | # Function to execute a command either locally or remotely 81 | def executecommand(command): 82 | if not args.server: # ie server is on same machine as this script 83 | return subprocess.check_output(command, text=True).strip() 84 | else: 85 | print(['ssh', ((args.sshuser + '@') if args.sshuser else '') + args.server] + [('"' + word + '"') if ' ' in word else word for word in command]) 86 | # This quoting of arguments is a bit of a hack but seems to work 87 | return subprocess.check_output(['ssh', ((args.sshuser + '@') if args.sshuser else '') + args.server] + [('"' + word + '"') if ' ' in word else word for word in command], text=True).strip() 88 | 89 | if args.outfile is None: 90 | args.outfile = args.infile.rsplit('.',1)[0] + '.nvp' 91 | elif os.path.isdir(args.outfile): 92 | args.outfile = os.path.join(args.outfile, 93 | os.path.basename(args.infile.name.rsplit('.',1)[0] + '.nvp')) 94 | 95 | if not args.no_logfile: 96 | logfilename = args.outfile.rsplit('.',1)[0] + '.log' 97 | incomments = ArgumentHelper.read_comments(logfilename) or ArgumentHelper.separator() 98 | logfile = open(logfilename, 'w') 99 | parser.write_comments(args, logfile, incomments=incomments) 100 | logfile.close() 101 | 102 | # Fill in extra arguments that NVivo module expects 103 | args.mac = False 104 | args.windows = True 105 | 106 | if args.basefile is None: 107 | args.basefile = os.path.dirname(os.path.realpath(__file__)) + os.path.sep + ('emptyNVivo10Win.nvp' if args.nvivoversion == '10' else 'emptyNVivo11Win.nvp') 108 | 109 | if args.server is None: 110 | if os.name != 'nt': 111 | raise RuntimeError("This does not appear to be a Windows machine so --server must be specified.") 112 | 113 | mssqlapi = mssqlAPI(args.server, 114 | user=args.sshuser, 115 | port=args.port, 116 | instance=args.instance, 117 | version = ('MSSQL12' if args.nvivoversion == '11' else 'MSSQL10_50'), 118 | verbosity = args.verbosity) 119 | 120 | # Get reasonably distinct yet recognisable DB name 121 | dbname = 'nvivo' + str(os.getpid()) 122 | 123 | mssqlapi.attach(args.basefile, dbname) 124 | try: 125 | args.indb = 'sqlite:///' + args.infile 126 | args.outdb = 'mssql+pymssql://nvivotools:nvivotools@' + (args.server or 'localhost') + ((':' + str(args.port)) if args.port else '') + '/' + dbname 127 | 128 | NVivo.Denormalise(args) 129 | 130 | mssqlapi.save(args.outfile, dbname) 131 | 132 | except: 133 | raise 134 | 135 | finally: 136 | mssqlapi.drop(dbname) 137 | 138 | if __name__ == '__main__': 139 | DenormaliseNVP(None) 140 | -------------------------------------------------------------------------------- /editNodeCategory.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | import os 20 | import sys 21 | import argparse 22 | from NVivoNorm import NVivoNorm 23 | from sqlalchemy import * 24 | import re 25 | from dateutil import parser as dateparser 26 | from datetime import date, time, datetime 27 | from distutils import util 28 | import uuid 29 | 30 | def add_arguments(parser): 31 | parser.description = "Insert or update node category in normalised file." 32 | 33 | generalgroup = parser.add_argument_group('General') 34 | generalgroup.add_argument('-o', '--outfile', type=str, required=True, 35 | help='Output normalised NVivo (.norm) file') 36 | generalgroup.add_argument('-n', '--name', type=str) 37 | generalgroup.add_argument('-d', '--description', type=str) 38 | generalgroup.add_argument('-u', '--user', type=str, 39 | help = 'User name, default is project "modified by".') 40 | 41 | advancedgroup = parser.add_argument_group('Advanced') 42 | advancedgroup.add_argument('-v', '--verbosity', type=int, default=1) 43 | advancedgroup.add_argument('--no-comments', action='store_true', help='Do not produce a comments logfile') 44 | 45 | parser.set_defaults(func=editNodeCategory) 46 | parser.set_defaults(build_comments=build_comments) 47 | parser.set_defaults(hiddenargs=['hiddenargs', 'verbosity', 'no_comments']) 48 | 49 | 50 | def parse_arguments(): 51 | parser = argparse.ArgumentParser() 52 | add_arguments(parser) 53 | return vars(parser.parse_args()) 54 | 55 | def build_comments(kwargs): 56 | comments = ((' ' + kwargs['outfile'] + ' ') if kwargs['outfile'] else '').center(80, '#') + '\n' 57 | comments += '# ' + os.path.basename(__file__) + '\n' 58 | hiddenargs = kwargs['hiddenargs'] + ['hiddenargs', 'func', 'build_comments'] 59 | for argname, argval in kwargs.items(): 60 | if argname not in hiddenargs: 61 | if type(argval) == str: 62 | comments += '# --' + argname + '="' + argval + '"\n' 63 | elif type(argval) == bool: 64 | if argval: 65 | comments += '# --' + argname + '\n' 66 | elif type(argval) == list: 67 | for valitem in argval: 68 | if type(valitem) == str: 69 | comments += '# --' + argname + '="' + valitem + '"\n' 70 | else: 71 | comments += '# --' + argname + '=' + str(valitem) + '\n' 72 | elif argval is not None: 73 | comments += '# --' + argname + '=' + str(argval) + '\n' 74 | 75 | return comments 76 | 77 | def editNodeCategory(outfile, name, description, user, 78 | verbosity, no_comments, 79 | comments, **dummy): 80 | 81 | try: 82 | if not no_comments: 83 | logfilename = outfile.rsplit('.',1)[0] + '.log' 84 | if os.path.isfile(logfilename): 85 | incomments = open(logfilename, 'r').read() 86 | else: 87 | incomments = '' 88 | logfile = open(logfilename, 'w') 89 | logfile.write(comments) 90 | logfile.write(incomments) 91 | logfile.close() 92 | 93 | norm = NVivoNorm(outfile) 94 | norm.begin() 95 | 96 | if user is not None: 97 | userRecord = norm.con.execute(select([ 98 | norm.User.c.Id 99 | ]).where( 100 | norm.User.c.Name == bindparam('Name') 101 | ), { 102 | 'Name': user 103 | }).first() 104 | if userRecord is not None: 105 | userId = userRecord['Id'] 106 | else: 107 | userId = uuid.uuid4() 108 | norm.con.execute(norm.User.insert(), { 109 | 'Id': userId, 110 | 'Name': user 111 | }) 112 | else: 113 | project = norm.con.execute(select([ 114 | norm.Project.c.ModifiedBy 115 | ])).first() 116 | if project: 117 | userId = project['ModifiedBy'] 118 | else: 119 | userId = uuid.uuid4() 120 | norm.con.execute(norm.User.insert(), { 121 | 'Id': userId, 122 | 'Name': u"Default User" 123 | }) 124 | norm.con.execute(norm.Project.insert(), { 125 | 'Version': u'0.2', 126 | 'Title': '', 127 | 'Description': u"Created by NVivotools http://barraqda.org/nvivotools/", 128 | 'CreatedBy': userId, 129 | 'CreatedDate': datetimeNow, 130 | 'ModifiedBy': userId, 131 | 'ModifiedDate': datetimeNow 132 | }) 133 | 134 | catRecord = norm.con.execute(select([ 135 | norm.NodeCategory.c.Id 136 | ]).where( 137 | norm.NodeCategory.c.Name == bindparam('Name') 138 | ), { 139 | 'Name': name 140 | }).first() 141 | catId = uuid.uuid4() if catRecord is None else catRecord['Id'] 142 | 143 | datetimeNow = datetime.utcnow() 144 | 145 | attColumns = { 146 | 'Id': catId, 147 | '_Id': catId, 148 | 'Name': name, 149 | 'Description': description, 150 | 'ModifiedBy': userId, 151 | 'ModifiedDate': datetimeNow 152 | } 153 | if catRecord: 154 | norm.con.execute(norm.NodeCategory.update( 155 | norm.NodeCategory.c.Id == bindparam('_Id')), 156 | attColumns) 157 | else: 158 | attColumns.update({ 159 | 'CreatedBy': userId, 160 | 'CreatedDate': datetimeNow, 161 | }) 162 | norm.con.execute(norm.NodeCategory.insert(), attColumns) 163 | 164 | norm.commit() 165 | 166 | except: 167 | raise 168 | norm.rollback() 169 | 170 | finally: 171 | del norm 172 | 173 | def main(): 174 | kwargs = parse_arguments() 175 | kwargs['comments'] = build_comments(kwargs) 176 | kwargs['func'](**kwargs) 177 | 178 | if __name__ == '__main__': 179 | main() 180 | -------------------------------------------------------------------------------- /editSourceCategory.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | import os 20 | import sys 21 | import argparse 22 | from NVivoNorm import NVivoNorm 23 | from sqlalchemy import * 24 | import re 25 | from dateutil import parser as dateparser 26 | from datetime import date, time, datetime 27 | from distutils import util 28 | import uuid 29 | 30 | def add_arguments(parser): 31 | parser.description = "Insert or update source category in normalised file." 32 | 33 | generalgroup = parser.add_argument_group('General') 34 | generalgroup.add_argument('-o', '--outfile', type=str, required=True, 35 | help='Output normalised NVivo (.norm) file') 36 | generalgroup.add_argument('-n', '--name', type=str) 37 | generalgroup.add_argument('-d', '--description', type=str) 38 | generalgroup.add_argument('-u', '--user', type=str, 39 | help = 'User name, default is project "modified by".') 40 | 41 | advancedgroup = parser.add_argument_group('Advanced') 42 | advancedgroup.add_argument('-v', '--verbosity', type=int, default=1) 43 | advancedgroup.add_argument('--no-comments', action='store_true', help='Do not produce a comments logfile') 44 | 45 | parser.set_defaults(func=editSourceCategory) 46 | parser.set_defaults(build_comments=build_comments) 47 | parser.set_defaults(hiddenargs=['hiddenargs', 'verbosity', 'no_comments']) 48 | 49 | 50 | def parse_arguments(): 51 | parser = argparse.ArgumentParser() 52 | add_arguments(parser) 53 | return vars(parser.parse_args()) 54 | 55 | def build_comments(kwargs): 56 | comments = ((' ' + kwargs['outfile'] + ' ') if kwargs['outfile'] else '').center(80, '#') + '\n' 57 | comments += '# ' + os.path.basename(__file__) + '\n' 58 | hiddenargs = kwargs['hiddenargs'] + ['hiddenargs', 'func', 'build_comments'] 59 | for argname, argval in kwargs.items(): 60 | if argname not in hiddenargs: 61 | if type(argval) == str: 62 | comments += '# --' + argname + '="' + argval + '"\n' 63 | elif type(argval) == bool: 64 | if argval: 65 | comments += '# --' + argname + '\n' 66 | elif type(argval) == list: 67 | for valitem in argval: 68 | if type(valitem) == str: 69 | comments += '# --' + argname + '="' + valitem + '"\n' 70 | else: 71 | comments += '# --' + argname + '=' + str(valitem) + '\n' 72 | elif argval is not None: 73 | comments += '# --' + argname + '=' + str(argval) + '\n' 74 | 75 | return comments 76 | 77 | def editSourceCategory(outfile, name, description, user, 78 | verbosity, no_comments, 79 | comments, **dummy): 80 | 81 | try: 82 | if not no_comments: 83 | logfilename = outfile.rsplit('.',1)[0] + '.log' 84 | if os.path.isfile(logfilename): 85 | incomments = open(logfilename, 'r').read() 86 | else: 87 | incomments = '' 88 | logfile = open(logfilename, 'w') 89 | logfile.write(comments) 90 | logfile.write(incomments) 91 | logfile.close() 92 | 93 | norm = NVivoNorm(outfile) 94 | norm.begin() 95 | 96 | if user is not None: 97 | userRecord = norm.con.execute(select([ 98 | norm.User.c.Id 99 | ]).where( 100 | norm.User.c.Name == bindparam('Name') 101 | ), { 102 | 'Name': user 103 | }).first() 104 | if userRecord is not None: 105 | userId = userRecord['Id'] 106 | else: 107 | userId = uuid.uuid4() 108 | norm.con.execute(norm.User.insert(), { 109 | 'Id': userId, 110 | 'Name': user 111 | }) 112 | else: 113 | project = norm.con.execute(select([ 114 | norm.Project.c.ModifiedBy 115 | ])).first() 116 | if project: 117 | userId = project['ModifiedBy'] 118 | else: 119 | userId = uuid.uuid4() 120 | norm.con.execute(norm.User.insert(), { 121 | 'Id': userId, 122 | 'Name': u"Default User" 123 | }) 124 | norm.con.execute(norm.Project.insert(), { 125 | 'Version': u'0.2', 126 | 'Title': '', 127 | 'Description': u"Created by NVivotools http://barraqda.org/nvivotools/", 128 | 'CreatedBy': userId, 129 | 'CreatedDate': datetimeNow, 130 | 'ModifiedBy': userId, 131 | 'ModifiedDate': datetimeNow 132 | }) 133 | 134 | catRecord = norm.con.execute(select([ 135 | norm.SourceCategory.c.Id 136 | ]).where( 137 | norm.SourceCategory.c.Name == bindparam('Name') 138 | ), { 139 | 'Name': name 140 | }).first() 141 | catId = uuid.uuid4() if catRecord is None else catRecord['Id'] 142 | 143 | datetimeNow = datetime.utcnow() 144 | 145 | attColumns = { 146 | 'Id': catId, 147 | '_Id': catId, 148 | 'Name': name, 149 | 'Description': description, 150 | 'ModifiedBy': userId, 151 | 'ModifiedDate': datetimeNow 152 | } 153 | if catRecord: 154 | norm.con.execute(norm.SourceCategory.update( 155 | norm.SourceCategory.c.Id == bindparam('_Id')), 156 | attColumns) 157 | else: 158 | attColumns.update({ 159 | 'CreatedBy': userId, 160 | 'CreatedDate': datetimeNow, 161 | }) 162 | norm.con.execute(norm.SourceCategory.insert(), attColumns) 163 | 164 | norm.commit() 165 | 166 | except: 167 | raise 168 | norm.rollback() 169 | 170 | finally: 171 | del norm 172 | 173 | def main(): 174 | kwargs = parse_arguments() 175 | kwargs['comments'] = build_comments(kwargs) 176 | kwargs['func'](**kwargs) 177 | 178 | if __name__ == '__main__': 179 | main() 180 | -------------------------------------------------------------------------------- /NormaliseNVPX.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from __future__ import print_function 20 | import os 21 | import subprocess 22 | import re 23 | import sys 24 | import glob 25 | import argparse 26 | import NVivo 27 | import shutil 28 | import tempfile 29 | 30 | parser = argparse.ArgumentParser(description='Normalise an NVivo for Mac file.') 31 | 32 | # --cmdline argument means retain full output file path name, otherwise strip directory, 33 | # so that things work under Wooey. 34 | parser.add_argument('--cmdline', action='store_true', 35 | help=argparse.SUPPRESS) 36 | 37 | parser.add_argument('-v', '--verbosity', type=int, default=1) 38 | 39 | parser.add_argument('-nv', '--nvivoversion', 40 | choices=["10", "11", "12"], default="10", 41 | help='NVivo version (10, 11 or 12)') 42 | 43 | parser.add_argument('-u', '--users', choices=["skip", "merge", "overwrite", "replace"], default="merge", 44 | help='User action.') 45 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 46 | help='Project action.') 47 | parser.add_argument('-nc', '--node-categories', choices=["skip", "merge", "overwrite", "replace"], default="merge", 48 | help='Node category action.') 49 | parser.add_argument('-n', '--nodes', choices=["skip", "merge", "overwrite", "replace"], default="merge", 50 | help='Node action.') 51 | parser.add_argument('-na', '--node-attributes', choices=["skip", "merge", "overwrite", "replace"], default="merge", 52 | help='Node attribute table action.') 53 | parser.add_argument('-sc', '--source-categories', choices=["skip", "merge", "overwrite", "replace"], default="merge", 54 | help='Source category action.') 55 | parser.add_argument('-s', '--sources', choices=["skip", "merge", "overwrite", "replace"], default="merge", 56 | help='Source action.') 57 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "merge", "overwrite", "replace"], default="merge", 58 | help='Source attribute action.') 59 | parser.add_argument('-t', '--taggings', choices=["skip", "merge", "overwrite", "replace"], default="merge", 60 | help='Tagging action.') 61 | parser.add_argument('-a', '--annotations', choices=["skip", "merge", "overwrite", "replace"], default="merge", 62 | help='Annotation action.') 63 | 64 | parser.add_argument('--sqlanywhere', type=str, 65 | help="Path to SQL Anywhere installation") 66 | 67 | parser.add_argument('infile', type=argparse.FileType('rb'), 68 | help="Input NVivo for Mac file (extension .nvpx)") 69 | parser.add_argument('outfile', type=str, nargs='?', 70 | help="Output normalised SQLite file") 71 | 72 | args = parser.parse_args() 73 | 74 | helperpath = os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'helpers' + os.path.sep 75 | 76 | # On non-Windows OS, need to set up environment for SQL Anywhere server and restart process. 77 | if os.name != 'nt': 78 | # Check if already done 79 | if not os.environ.get('_restart'): 80 | if args.sqlanywhere: 81 | os.environ['sqlanywhere'] = args.sqlanywhere 82 | envlines = subprocess.check_output(helperpath + 'sqlanyenv.sh', text=True).splitlines() 83 | for envline in envlines: 84 | env = re.match(r"(?P\w+)=(?P['\"]?)(?P.*)(?P=quote)", envline, re.MULTILINE | re.DOTALL).groupdict() 85 | os.environ[env['name']] = env['value'] 86 | 87 | os.environ['_restart'] = 'TRUE' 88 | os.execve(sys.argv[0], sys.argv, os.environ) 89 | else: 90 | dbengfile = None 91 | if args.sqlanywhere: 92 | dbengpaths = glob.glob(args.sqlanywhere + '\\dbeng*.exe') 93 | if dbengpaths: 94 | dbengfile = os.path.basename(dbengpaths[0]) 95 | else: 96 | pathlist=os.environ['PATH'].split(';') 97 | for path in pathlist: 98 | dbengpaths = glob.glob(path + '\\dbeng*.exe') 99 | if dbengpaths: 100 | dbengfile = os.path.basename(dbengpaths[0]) 101 | break 102 | 103 | if not dbengfile: 104 | raise RuntimeError("Could not find SQL Anywhere executable") 105 | 106 | # Fill in extra arguments that NVivo module expects 107 | args.mac = True 108 | args.windows = False 109 | 110 | tmpinfilename = tempfile.mktemp() 111 | tmpinfileptr = open(tmpinfilename, 'wb') 112 | tmpinfileptr.write(args.infile.read()) 113 | args.infile.close() 114 | tmpinfileptr.close() 115 | 116 | tmpoutfile = tempfile.mktemp() 117 | 118 | if args.outfile is None: 119 | args.outfile = args.infile.name.rsplit('.',1)[0] + '.norm' 120 | elif os.path.isdir(args.outfile): 121 | args.outfile = os.path.join(args.outfile, 122 | os.path.basename(args.infile.name.rsplit('.',1)[0] + '.norm')) 123 | 124 | # Find a free sock for SQL Anywhere server to bind to 125 | import socket 126 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 127 | s.bind(("",0)) 128 | freeport = str(s.getsockname()[1]) 129 | s.close() 130 | 131 | DEVNULL = open(os.devnull, 'wb') 132 | 133 | if os.name != 'nt': 134 | dbproc = subprocess.Popen(['sh', helperpath + 'sqlanysrv.sh', '-x TCPIP(port='+freeport+')', '-ga', tmpinfilename, '-n', 'NVivo'+freeport], text=True, 135 | stdout=subprocess.PIPE, stdin=DEVNULL) 136 | # Wait until SQL Anywhere engine starts... 137 | while dbproc.poll() is None: 138 | line = dbproc.stdout.readline() 139 | if line == 'Now accepting requests\n': 140 | break 141 | else: 142 | dbproc = subprocess.Popen(['dbspawn', dbengfile, '-x TCPIP(port='+freeport+')', '-ga', tmpinfilename, '-n', 'NVivo'+freeport], text=True, 143 | stdout=subprocess.PIPE, stdin=DEVNULL) 144 | # Wait until SQL Anywhere engine starts... 145 | while dbproc.poll() is None: 146 | line = dbproc.stdout.readline() 147 | if 'SQL Anywhere Start Server In Background Utility' in line: 148 | break 149 | 150 | if dbproc.poll() is not None: 151 | raise RuntimeError("Failed to start database server") 152 | 153 | if args.verbosity > 0: 154 | print("Started database server on port " + freeport, file=sys.stderr) 155 | 156 | args.indb = 'sqlalchemy_sqlany://wiwalisataob2aaf:iatvmoammgiivaam@localhost:' + freeport + '/NVivo' + freeport 157 | args.outdb = 'sqlite:///' + tmpoutfile 158 | 159 | chdir = os.environ.get('CHDIR') 160 | if chdir: 161 | cwd = os.getcwd() 162 | os.chdir(chdir) 163 | 164 | NVivo.Normalise(args) 165 | 166 | if chdir: 167 | os.chdir(cwd) 168 | 169 | if not args.cmdline: 170 | args.outfile = os.path.basename(args.outfile) 171 | 172 | if os.path.exists(args.outfile): 173 | shutil.move(args.outfile, args.outfile + '.bak') 174 | 175 | shutil.move(tmpoutfile, args.outfile) 176 | # Need to change file mode so that delete works under Windows 177 | os.chmod(tmpinfilename, 0o777) 178 | os.remove(tmpinfilename) 179 | -------------------------------------------------------------------------------- /tagSpeakers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from argrecord import ArgumentHelper, ArgumentRecorder 20 | import os 21 | import sys 22 | from NVivoNorm import NVivoNorm 23 | from sqlalchemy import * 24 | from datetime import datetime 25 | import re 26 | import uuid 27 | 28 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 29 | 30 | def tagSpeakers(arglist=None): 31 | 32 | parser = ArgumentRecorder(description='Insert or update source in normalised file.') 33 | 34 | parser.add_argument( 'file', type = str, input=True, output=True, 35 | help = 'Normalised NVivo (.nvpn) project file') 36 | parser.add_argument('-u', '--user', type=str, 37 | help='User name, default is project "modified by".') 38 | 39 | parser.add_argument('-sc', '--source-category', type=str, 40 | help='Category of sources to process') 41 | parser.add_argument('-nc', '--node-category', type=str, 42 | help='Category of nodes to process') 43 | 44 | parser.add_argument('-v', '--verbosity', type=int, default=1, private=True) 45 | parser.add_argument('-l', '--limit', type=int, 46 | help='Limit number of lines from table file') 47 | parser.add_argument('--logfile', type=str, private=True, 48 | help="Logfile, default is .log") 49 | parser.add_argument('--no-logfile', action='store_true', 50 | help='Do not output a logfile') 51 | 52 | args = parser.parse_args(arglist) 53 | 54 | if not args.no_logfile: 55 | logfilename = args.file.rsplit('.',1)[0] + '.log' 56 | incomments = ArgumentHelper.read_comments(logfilename) or ArgumentHelper.separator() 57 | logfile = open(logfilename, 'w') 58 | parser.write_comments(args, logfile, incomments=incomments) 59 | logfile.close() 60 | 61 | try: 62 | 63 | norm = NVivoNorm(args.file) 64 | norm.begin() 65 | 66 | datetimeNow = datetime.utcnow() 67 | 68 | if args.user: 69 | userRecord = norm.con.execute(select([ 70 | norm.User.c.Id 71 | ]).where( 72 | norm.User.c.Name == bindparam('Name') 73 | ), { 74 | 'Name': args.user 75 | }).first() 76 | if userRecord: 77 | userId = userRecord['Id'] 78 | else: 79 | userId = uuid.uuid4() 80 | norm.con.execute(norm.User.insert(), { 81 | 'Id': userId, 82 | 'Name': args.user 83 | }) 84 | else: 85 | projectRecord = norm.con.execute(select([ 86 | norm.Project.c.ModifiedBy 87 | ])).first() 88 | if projectRecord: 89 | userId = projectRecord['ModifiedBy'] 90 | else: 91 | userId = uuid.uuid4() 92 | norm.con.execute(norm.User.insert(), { 93 | 'Id': userId, 94 | 'Name': u"Default User" 95 | }) 96 | norm.con.execute(norm.Project.insert(), { 97 | 'Version': '0.2', 98 | 'Title': "Created by NVivotools http://barraqda.org/nvivotools/", 99 | 'CreatedBy': userId, 100 | 'CreatedDate': datetimeNow, 101 | 'ModifiedBy': userId, 102 | 'ModifiedDate': datetimeNow 103 | }) 104 | 105 | nodeSelect = select([ 106 | norm.Node.c.Name, 107 | norm.Node.c.Id 108 | ]) 109 | if args.node_category: 110 | nodeCategoryId = norm.con.execute( 111 | select([norm.NodeCategory.c.Id]). 112 | where(norm.NodeCategory.c.Name == bindparam('Category')), 113 | { 'Category': args.node_category }).first()['Id'] 114 | 115 | nodeSelect = nodeSelect.where( 116 | norm.Node.c.Category == bindparam('Category') 117 | ) 118 | else: 119 | nodeCategoryId = None 120 | 121 | speakers = { row['Name']: row['Id'] for row in norm.con.execute(nodeSelect, { 'Category': nodeCategoryId }) } 122 | 123 | sourceSelect = select([ 124 | norm.Source.c.Name, 125 | norm.Source.c.Content, 126 | norm.Source.c.Id 127 | ]) 128 | if args.source_category: 129 | sourceCategoryNode = norm.con.execute( 130 | select([norm.SourceCategory.c.Id]). 131 | where(norm.SourceCategory.c.Name == bindparam('Category')), 132 | { 'Category': args.source_category }).first() 133 | if sourceCategoryNode: 134 | sourceCategoryId = sourceCategoryNode['Id'] 135 | else: 136 | raise RuntimeError("Source category: ", args.source_category, " does not exist.") 137 | 138 | sourceSelect = sourceSelect.where( 139 | norm.Source.c.Category == bindparam('Category') 140 | ) 141 | else: 142 | sourceCategoryId = None 143 | 144 | if args.limit: 145 | sourceSelect = sourceSelect.limit(args.limit) 146 | 147 | speakerPattern = re.compile(r'^(.+?):\s*(.*?)$', re.MULTILINE) 148 | taggingRows = [] 149 | for sourceRow in norm.con.execute(sourceSelect, { 'Category': sourceCategoryId }): 150 | if args.verbosity > 1: 151 | print("Source: ", sourceRow['Name']) 152 | speakerMatches = speakerPattern.finditer(sourceRow['Content']) 153 | for speakerMatch in speakerMatches: 154 | speakerName = speakerMatch.group(1) 155 | speakerId = speakers.get(speakerName, None) 156 | if speakerId: 157 | fragment = str(speakerMatch.start(2)) + ':' + str(speakerMatch.end(2)) 158 | taggingRows.append({ 159 | 'Id': uuid.uuid4(), 160 | 'Source': sourceRow['Id'], 161 | 'Node': speakerId, 162 | 'Fragment': fragment, 163 | 'CreatedBy': userId, 164 | 'CreatedDate': datetimeNow, 165 | 'ModifiedBy': userId, 166 | 'ModifiedDate': datetimeNow 167 | }) 168 | 169 | if taggingRows: 170 | norm.con.execute(norm.Tagging.insert(), taggingRows) 171 | 172 | if args.verbosity > 1: 173 | print("Inserted", len(taggingRows), "taggings.", file=sys.stderr) 174 | 175 | norm.commit() 176 | 177 | except: 178 | raise 179 | norm.rollback() 180 | 181 | finally: 182 | del norm 183 | 184 | if __name__ == '__main__': 185 | tagSpeakers(None) 186 | -------------------------------------------------------------------------------- /queryTagging.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2016 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from argrecord import ArgumentHelper, ArgumentRecorder 20 | import os 21 | import sys 22 | import argparse 23 | from NVivoNorm import NVivoNorm 24 | from sqlalchemy import * 25 | import re 26 | import csv 27 | import shutil 28 | 29 | def queryTagging(arglist=None): 30 | 31 | parser = ArgumentRecorder(description="Query taggings in a normalised file.") 32 | 33 | generalgroup = parser.add_argument_group('General') 34 | generalgroup.add_argument( 'file', type=str, 35 | help='Normalised NVivo (.nvpn) file') 36 | generalgroup.add_argument('-o', '--outfile', type=str, 37 | help='Output CSV file') 38 | generalgroup.add_argument('-s', '--source', type=str) 39 | generalgroup.add_argument('-sc', '--source-category', type=str) 40 | generalgroup.add_argument('-n', '--node', nargs='*', type=str) 41 | generalgroup.add_argument('-nc', '--node-category', type=str) 42 | 43 | advancedgroup = parser.add_argument_group('Advanced') 44 | advancedgroup.add_argument('-v', '--verbosity', type=int, default=1, private=True) 45 | advancedgroup.add_argument('-l', '--limit', type=int, default=0, 46 | help="Limit number of sources to process") 47 | advancedgroup.add_argument('--no-comments', action='store_true', 48 | help='Do not output comments in header of output file') 49 | 50 | args = parser.parse_args(arglist) 51 | 52 | try: 53 | norm = NVivoNorm(args.file) 54 | norm.begin() 55 | 56 | sourcesel = select([ 57 | norm.Source.c.Name.label('Source'), 58 | norm.Node.c.Name.label('Node'), 59 | norm.Source.c.Content, 60 | norm.Tagging.c.Fragment, 61 | norm.Tagging.c.Memo 62 | ]).where( 63 | norm.Source.c.Id == norm.Tagging.c.Source, 64 | ).select_from( 65 | norm.Tagging.outerjoin( 66 | norm.Node, 67 | norm.Tagging.c.Node == norm.Node.c.Id) 68 | ) 69 | params = {} 70 | 71 | if args.source: 72 | sourcesel = sourcesel.where( 73 | norm.Source.c.Name == bindparam('Source') 74 | ) 75 | params.update({'Source': args.source}) 76 | 77 | if args.source_category: 78 | sourcesel = sourcesel.where(and_( 79 | norm.Source.c.Category == norm.SourceCategory.c.Id, 80 | norm.SourceCategory.c.Name == bindparam('SourceCategory') 81 | )) 82 | params.update({'SourceCategory': args.source_category}) 83 | 84 | tagginglist = [] 85 | if args.node_category: 86 | sourceselnodecat = sourcesel.where(and_( 87 | norm.Node.c.Category == norm.NodeCategory.c.Id, 88 | norm.NodeCategory.c.Name == bindparam('NodeCategory') 89 | )) 90 | params.update({'NodeCategory': args.node_category}) 91 | tagginglist.append([dict(row) for row in norm.con.execute(sourceselnodecat, params)]) 92 | 93 | if args.node: 94 | sourceselnode = sourcesel.where(and_( 95 | norm.Tagging.c.Node == norm.Node.c.Id, 96 | norm.Node.c.Name == bindparam('Node') 97 | )) 98 | for nodeiter in args.node: 99 | params.update({'Node': nodeiter}) 100 | 101 | tagginglist.append([dict(row) for row in norm.con.execute(sourceselnode, params)]) 102 | elif not args.node_category: 103 | tagginglist = [[dict(row) for row in norm.con.execute(sourcesel, params)]] 104 | 105 | fragmentregex = re.compile(r'(?P[0-9]+):(?P[0-9]+)') 106 | for taggings in tagginglist: 107 | for tagging in taggings: 108 | matchfragment = fragmentregex.match(tagging['Fragment']) 109 | tagging['Start'] = int(matchfragment.group('start')) 110 | tagging['End'] = int(matchfragment.group('end')) 111 | 112 | def sortandmergetagginglist(tagginglist): 113 | tagginglist.sort(key = lambda tagging: (tagging['Source'], tagging['NodeTuple'], tagging['Start'], tagging['End'])) 114 | idx = 0 115 | while idx < len(tagginglist) - 1: 116 | if tagginglist[idx]['Source'] == tagginglist[idx+1]['Source'] \ 117 | and tagginglist[idx]['NodeTuple'] == tagginglist[idx+1]['NodeTuple'] \ 118 | and tagginglist[idx]['End'] >= tagginglist[idx+1]['Start']: 119 | tagginglist[idx]['End'] = max(tagginglist[idx]['End'], tagginglist[idx+1]['End']) 120 | del tagginglist[idx+1] 121 | else: 122 | idx += 1 123 | 124 | intersection = tagginglist[0] 125 | for tagging in intersection: 126 | if tagging.get('Node'): 127 | tagging['NodeTuple'] = (tagging['Node'],) 128 | else: 129 | tagging['NodeTuple'] = () 130 | 131 | sortandmergetagginglist(intersection) 132 | for taggings in tagginglist[1:]: 133 | idx = 0 134 | newintersection = [] 135 | for intagging in intersection: 136 | for tagging in taggings: 137 | if tagging['Source'] == intagging['Source']: 138 | newstart = max(tagging['Start'], intagging['Start']) 139 | newend = min(tagging['End'], intagging['End']) 140 | if newend >= newstart: 141 | newintersection.append({'Source': tagging['Source'], 142 | 'NodeTuple': (tagging['Node'],) + intagging['NodeTuple'], 143 | 'Content': tagging['Content'], 144 | 'Start': newstart, 145 | 'End': newend}) 146 | 147 | intersection = newintersection 148 | sortandmergetagginglist(intersection) 149 | 150 | if args.outfile: 151 | if os.path.exists(args.outfile): 152 | shutil.move(args.outfile, args.outfile + '.bak') 153 | 154 | csvfile = open(args.outfile, 'w') 155 | else: 156 | csvfile = sys.stdout 157 | 158 | if not args.no_comments: 159 | parser.write_comments(args, csvfile, incomments=ArgumentHelper.separator()) 160 | 161 | csvwriter = csv.DictWriter(csvfile, 162 | fieldnames=['Source', 'Node', 'Memo', 'Text', 'Fragment'], 163 | extrasaction='ignore', 164 | lineterminator=os.linesep, 165 | quoting=csv.QUOTE_NONNUMERIC) 166 | 167 | csvwriter.writeheader() 168 | 169 | for tagging in intersection: 170 | tagging['Fragment'] = str(tagging['Start']) + ':' + str(tagging['End']) 171 | tagging['Text'] = tagging['Content'][tagging['Start']-1:tagging['End']] 172 | tagging['Node'] = os.linesep.join(nodeiter for nodeiter in tagging['NodeTuple']) 173 | 174 | csvwriter.writerows(intersection) 175 | csvfile.close() 176 | 177 | except: 178 | raise 179 | 180 | finally: 181 | del norm 182 | 183 | if __name__ == '__main__': 184 | queryTagging(None) 185 | -------------------------------------------------------------------------------- /nvpn2nvpx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from argrecord import ArgumentHelper, ArgumentRecorder 20 | import argparse 21 | import os 22 | import subprocess 23 | import re 24 | import sys 25 | import glob 26 | import NVivo 27 | import shutil 28 | import tempfile 29 | import io # Remove when converted to ArgumentRecord 30 | 31 | def nvpn2nvpx(arglist=None): 32 | 33 | parser = ArgumentRecorder(description='Create an NVivo for Mac (.nvpx) file from a normalised NVivo (.nvpn) file.') 34 | 35 | # --cmdline argument means retain full output file path name, otherwise strip directory, 36 | # so that things work under Wooey. 37 | parser.add_argument('--cmdline', action='store_true', 38 | help=argparse.SUPPRESS) 39 | 40 | parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", 41 | help='NVivo version (10 or 11)') 42 | 43 | parser.add_argument('-u', '--users', choices=["skip", "merge", "overwrite", "replace"], default="merge", 44 | help='User action.') 45 | parser.add_argument('-p', '--project', choices=["skip", "overwrite"], default="overwrite", 46 | help='Project action.') 47 | parser.add_argument('-nc', '--node-categories', choices=["skip", "merge", "overwrite"], default="merge", 48 | help='Node category action.') 49 | parser.add_argument('-n', '--nodes', choices=["skip", "merge"], default="merge", 50 | help='Node action.') 51 | parser.add_argument('-na', '--node-attributes', choices=["skip", "merge", "overwrite"], default="merge", 52 | help='Node attribute table action.') 53 | parser.add_argument('-sc', '--source-categories', choices=["skip", "merge", "overwrite"], default="merge", 54 | help='Source category action.') 55 | parser.add_argument('--sources', choices=["skip", "merge", "overwrite"], default="merge", 56 | help='Source action.') 57 | parser.add_argument('-sa', '--source-attributes', choices=["skip", "merge", "overwrite"], default="merge", 58 | help='Source attribute action.') 59 | parser.add_argument('-t', '--taggings', choices=["skip", "merge"], default="merge", 60 | help='Tagging action.') 61 | parser.add_argument('-a', '--annotations', choices=["skip", "merge"], default="merge", 62 | help='Annotation action.') 63 | 64 | parser.add_argument('-b', '--base', dest='basefile', type=argparse.FileType('rb'), nargs='?', 65 | help="Base NVPX file to insert into") 66 | 67 | parser.add_argument('-v', '--verbosity', type=int, default=1, private=True) 68 | parser.add_argument('--logfile', type=str, help="Logfile, default is .log", 69 | private=True) 70 | parser.add_argument('--no-logfile', action='store_true', help='Do not output a logfile') 71 | 72 | parser.add_argument('--sqlanywhere', type=str, 73 | help="Path to SQL Anywhere installation") 74 | 75 | parser.add_argument('infile', type=argparse.FileType('rb'), input=True, 76 | help="Input normalised SQLite (.nvpn) file") 77 | parser.add_argument('outfile', type=str, nargs='?', output=True, 78 | help="Output NVivo for Mac (.nvpx) file or directory, default is .nvpx") 79 | 80 | args = parser.parse_args(arglist) 81 | 82 | helperpath = os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'helpers' + os.path.sep 83 | 84 | # On non-Windows OS, need to set up environment for SQL Anywhere server and restart process. 85 | if os.name != 'nt': 86 | # Check if already done 87 | if not os.environ.get('_restart'): 88 | if args.sqlanywhere: 89 | os.environ['sqlanywhere'] = args.sqlanywhere 90 | envlines = subprocess.check_output(helperpath + 'sqlanyenv.sh', text=True).splitlines() 91 | for envline in envlines: 92 | env = re.match(r"(?P\w+)=(?P['\"]?)(?P.*)(?P=quote)", envline, re.MULTILINE | re.DOTALL).groupdict() 93 | os.environ[env['name']] = env['value'] 94 | 95 | os.environ['_restart'] = 'TRUE' 96 | os.execve(sys.argv[0], sys.argv, os.environ) 97 | else: 98 | dbengfile = None 99 | if args.sqlanywhere: 100 | dbengpaths = glob.glob(args.sqlanywhere + '\\dbeng*.exe') 101 | if dbengpaths: 102 | dbengfile = os.path.basename(dbengpaths[0]) 103 | else: 104 | pathlist=os.environ['PATH'].split(';') 105 | for path in pathlist: 106 | dbengpaths = glob.glob(path + '\\dbeng*.exe') 107 | if dbengpaths: 108 | dbengfile = os.path.basename(dbengpaths[0]) 109 | break 110 | 111 | if not dbengfile: 112 | raise RuntimeError("Could not find SQL Anywhere executable") 113 | 114 | 115 | if args.outfile is None: 116 | args.outfile = args.infile.name.rsplit('.',1)[0] + '.nvpx' 117 | elif os.path.isdir(args.outfile): 118 | args.outfile = os.path.join(args.outfile, 119 | os.path.basename(args.infile.name.rsplit('.',1)[0] + '.nvpx')) 120 | 121 | if not args.no_logfile: 122 | logfilename = args.outfile.rsplit('.',1)[0] + '.log' 123 | incomments = ArgumentHelper.read_comments(logfilename) or ArgumentHelper.separator() 124 | logfile = open(logfilename, 'w') 125 | parser.write_comments(args, logfile, incomments=incomments) 126 | logfile.close() 127 | 128 | # Fill in extra arguments that NVivo module expects 129 | args.mac = True 130 | args.windows = False 131 | 132 | tmpinfilename = tempfile.mktemp() 133 | tmpinfileptr = open(tmpinfilename, 'wb') 134 | tmpinfileptr.write(args.infile.read()) 135 | args.infile.close() 136 | tmpinfileptr.close() 137 | 138 | if args.basefile is None: 139 | args.basefile = open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + ('emptyNVivo10Mac.nvpx' if args.nvivoversion == '10' else 'emptyNVivo11Mac.nvpx'), 'rb') 140 | 141 | tmpoutfilename = tempfile.mktemp() 142 | tmpoutfileptr = open(tmpoutfilename, 'wb') 143 | tmpoutfileptr.write(args.basefile.read()) 144 | args.basefile.close() 145 | tmpoutfileptr.close() 146 | 147 | # Find a free sock for SQL Anywhere server to bind to 148 | import socket 149 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 150 | s.bind(("",0)) 151 | freeport = str(s.getsockname()[1]) 152 | s.close() 153 | 154 | DEVNULL = open(os.devnull, 'wb') 155 | 156 | if os.name != 'nt': 157 | dbproc = subprocess.Popen(['sh', helperpath + 'sqlanysrv.sh', '-x TCPIP(port='+freeport+')', '-ga', tmpoutfilename, '-n', 'NVivo'+freeport], text=True, stdout=subprocess.PIPE, stdin=DEVNULL) 158 | # Wait until SQL Anywhere engine starts... 159 | while dbproc.poll() is None: 160 | line = dbproc.stdout.readline() 161 | if line == 'Now accepting requests\n': 162 | break 163 | else: 164 | pathlist=os.environ['PATH'].split(';') 165 | for path in pathlist: 166 | dbengpaths = glob.glob(path + '\\dbeng*.exe') 167 | if dbengpaths: 168 | dbengfile = os.path.basename(dbengpaths[0]) 169 | break 170 | else: 171 | raise RuntimeError("Could not find SQL Anywhere executable") 172 | 173 | dbproc = subprocess.Popen(['dbspawn', dbengfile, '-x TCPIP(port='+freeport+')', '-ga', tmpoutfilename, '-n', 'NVivo'+freeport], text=True, stdout=subprocess.PIPE, stdin=DEVNULL) 174 | # Wait until SQL Anywhere engine starts... 175 | while dbproc.poll() is None: 176 | line = dbproc.stdout.readline() 177 | if 'SQL Anywhere Start Server In Background Utility' in line: 178 | break 179 | 180 | if dbproc.poll() is not None: 181 | raise RuntimeError("Failed to start database server") 182 | 183 | if args.verbosity > 0: 184 | print("Started database server on port " + freeport, file=sys.stderr) 185 | 186 | args.indb = 'sqlite:///' + tmpinfilename 187 | args.outdb = 'sqlalchemy_sqlany://wiwalisataob2aaf:iatvmoammgiivaam@localhost:' + freeport + '/NVivo' + freeport 188 | 189 | chdir = os.environ.get('CHDIR') 190 | if chdir: 191 | cwd = os.getcwd() 192 | os.chdir(chdir) 193 | 194 | NVivo.Denormalise(args) 195 | 196 | if chdir: 197 | os.chdir(cwd) 198 | 199 | if not args.cmdline: 200 | args.outfile = os.path.basename(args.outfile) 201 | 202 | if os.path.exists(args.outfile): 203 | shutil.move(args.outfile, args.outfile + '.bak') 204 | 205 | shutil.move(tmpoutfilename, args.outfile) 206 | os.remove(tmpinfilename) 207 | 208 | 209 | if __name__ == '__main__': 210 | nvpn2nvpx(None) 211 | -------------------------------------------------------------------------------- /tagNounPhrases.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | from argrecord import ArgumentHelper, ArgumentRecorder 20 | import os 21 | import sys 22 | from NVivoNorm import NVivoNorm 23 | from sqlalchemy import * 24 | from textblob import TextBlob 25 | import uuid 26 | from datetime import datetime 27 | 28 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 29 | 30 | def tagNounPhrases(arglist=None): 31 | 32 | parser = ArgumentRecorder(description='Analyse source text.') 33 | 34 | parser.add_argument( 'file', type = str, input=True, output=True, 35 | help = 'Normalised NVivo (.nvpn) project file') 36 | parser.add_argument('-u', '--user', type=str, 37 | help='User name, default is project "modified by".') 38 | 39 | parser.add_argument('-sc', '--source-category', type=str, 40 | help='Category of sources to process') 41 | parser.add_argument('-t', '--threshold', type=int, default=0, 42 | help="Miniumum number of occurrences for inclusion") 43 | 44 | parser.add_argument('-v', '--verbosity', type=int, default=1, private=True) 45 | parser.add_argument('-l', '--limit', type=int, default=0, 46 | help="Limit number of sources to process") 47 | parser.add_argument('--logfile', type=str, private=True, 48 | help="Logfile, default is .log") 49 | parser.add_argument('--no-logfile', action='store_true', 50 | help='Do not output a logfile') 51 | 52 | args = parser.parse_args(arglist) 53 | 54 | if not args.no_logfile: 55 | logfilename = args.file.rsplit('.',1)[0] + '.log' 56 | incomments = ArgumentHelper.read_comments(logfilename) or ArgumentHelper.separator() 57 | logfile = open(logfilename, 'w') 58 | parser.write_comments(args, logfile, incomments=incomments) 59 | logfile.close() 60 | 61 | try: 62 | norm = NVivoNorm(args.file) 63 | norm.begin() 64 | 65 | datetimeNow = datetime.utcnow() 66 | 67 | if args.user: 68 | userRecord = norm.con.execute(select([ 69 | norm.User.c.Id 70 | ]).where( 71 | norm.User.c.Name == bindparam('Name') 72 | ), { 73 | 'Name': args.user 74 | }).first() 75 | if userRecord: 76 | userId = userRecord['Id'] 77 | else: 78 | userId = uuid.uuid4() 79 | norm.con.execute(norm.User.insert(), { 80 | 'Id': userId, 81 | 'Name': args.user 82 | }) 83 | else: 84 | projectRecord = norm.con.execute(select([ 85 | norm.Project.c.ModifiedBy 86 | ])).first() 87 | if projectRecord: 88 | userId = projectRecord['ModifiedBy'] 89 | else: 90 | userId = uuid.uuid4() 91 | norm.con.execute(norm.User.insert(), { 92 | 'Id': userId, 93 | 'Name': u"Default User" 94 | }) 95 | norm.con.execute(norm.Project.insert(), { 96 | 'Version': '0.2', 97 | 'Title': "Created by NVivotools http://barraqda.org/nvivotools/", 98 | 'CreatedBy': userId, 99 | 'CreatedDate': datetimeNow, 100 | 'ModifiedBy': userId, 101 | 'ModifiedDate': datetimeNow 102 | }) 103 | 104 | sourceSelect = select([ 105 | norm.Source.c.Name, 106 | norm.Source.c.Content, 107 | norm.Source.c.Id 108 | ]) 109 | if args.source_category: 110 | sourceCategoryNode = norm.con.execute( 111 | select([norm.SourceCategory.c.Id]). 112 | where(norm.SourceCategory.c.Name == bindparam('Category')), 113 | { 'Category': args.source_category }).first() 114 | if sourceCategoryNode: 115 | sourceCategoryId = sourceCategoryNode['Id'] 116 | else: 117 | raise RuntimeError("Source category: ", args.source_category, " does not exist.") 118 | 119 | sourceSelect = sourceSelect.where( 120 | norm.Source.c.Category == bindparam('Category') 121 | ) 122 | else: 123 | sourceCategoryId = None 124 | 125 | if args.limit: 126 | sourceSelect = sourceSelect.limit(args.limit) 127 | 128 | lemmaFrequency = {} 129 | for sourceRow in norm.con.execute(sourceSelect, { 'Category': sourceCategoryId }): 130 | if args.verbosity > 1: 131 | print("Source: ", sourceRow['Name']) 132 | if sourceRow['Content']: 133 | content = TextBlob(sourceRow['Content']) 134 | nounPhrases = content.lower().noun_phrases 135 | lemmas = nounPhrases.lemmatize() 136 | for lemma in lemmas: 137 | lemmaFrequency[lemma] = lemmaFrequency.get(lemma, 0) + 1 138 | 139 | lemmaNode = {} 140 | 141 | nounPhraseNodeCategory = norm.con.execute(select([ 142 | norm.NodeCategory.c.Id 143 | ]).where( 144 | norm.NodeCategory.c.Name == 'Noun Phrases' 145 | )).first() 146 | if nounPhraseNodeCategory is not None: 147 | nounPhraseNodeCategoryId = nounPhraseNodeCategory['Id'] 148 | if args.verbosity > 1: 149 | print("Found Noun Phrases node category ID ", nounPhraseNodeCategoryId) 150 | 151 | for node in norm.con.execute(select([norm.Node.c.Id, norm.Node.c.Name]). 152 | where(norm.Node.c.Category == nounPhraseNodeCategoryId)): 153 | lemmaNode[node['Name']] = node['Id'] 154 | else: 155 | if args.verbosity > 1: 156 | print("Creating Noun Phrases node category") 157 | nounPhraseNodeCategoryId = uuid.uuid4() 158 | norm.con.execute(norm.NodeCategory.insert().values({ 159 | 'Id': nounPhraseNodeCategoryId, 160 | 'Name': 'Noun Phrases', 161 | 'CreatedBy': userId, 162 | 'CreatedDate': datetimeNow, 163 | 'ModifiedBy': userId, 164 | 'ModifiedDate': datetimeNow 165 | })) 166 | 167 | for lemma in lemmaFrequency.keys(): 168 | if (args.threshold == 0 or lemmaFrequency[lemma] >= args.threshold) and not lemmaNode.get(lemma): 169 | if args.verbosity > 1: 170 | print("Creating node: " + lemma) 171 | lemmaNode[lemma] = uuid.uuid4() 172 | norm.con.execute(norm.Node.insert().values({ 173 | 'Id': lemmaNode[lemma], 174 | 'Category': nounPhraseNodeCategoryId, 175 | 'Name': lemma, 176 | 'Aggregate': False, 177 | 'CreatedBy': userId, 178 | 'CreatedDate': datetimeNow, 179 | 'ModifiedBy': userId, 180 | 'ModifiedDate': datetimeNow 181 | })) 182 | 183 | for sourceRow in norm.con.execute(sourceSelect, { 'Category': sourceCategoryId }): 184 | if args.verbosity > 1: 185 | print("Processing source: " + sourceRow['Name']) 186 | if sourceRow['Content']: 187 | content = TextBlob(sourceRow['Content']) 188 | for sentence in content.lower().sentences: 189 | lemmas = sentence.noun_phrases.lemmatize() 190 | for lemma in lemmas: 191 | if lemma in lemmaFrequency.keys() and (args.threshold == 0 or lemmaFrequency[lemma] >= args.threshold): 192 | if args.verbosity > 2: 193 | print(" Inserting tagging: " + str(sentence.start) + ':' + str(sentence.end - 1)) 194 | norm.con.execute(norm.Tagging.insert().values({ 195 | 'Id': uuid.uuid4(), 196 | 'Source': sourceRow['Id'], 197 | 'Node': lemmaNode[lemma], 198 | 'Fragment': str(sentence.start) + ':' + str(sentence.end - 1), 199 | 'CreatedBy': userId, 200 | 'CreatedDate': datetimeNow, 201 | 'ModifiedBy': userId, 202 | 'ModifiedDate': datetimeNow 203 | })) 204 | 205 | norm.commit() 206 | 207 | except: 208 | raise 209 | norm.rollback() 210 | 211 | finally: 212 | del norm 213 | 214 | if __name__ == '__main__': 215 | tagNounPhrases(None) 216 | -------------------------------------------------------------------------------- /NVivoNorm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2017 Jonathan Schultz 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | import os 20 | from sqlalchemy import * 21 | from sqlalchemy import exc 22 | import uuid 23 | 24 | exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) 25 | 26 | class NVivoNorm(object): 27 | 28 | def __init__(self, path): 29 | try: 30 | self.db = create_engine('sqlite:///' + path) 31 | self.md = MetaData(bind=self.db) 32 | self.con = self.db.connect() 33 | self.tr = None 34 | except: 35 | raise 36 | 37 | # Load or create the database 38 | try: 39 | self.User = Table('User', self.md, autoload=True) 40 | except exc.NoSuchTableError: 41 | self.User = Table('User', self.md, 42 | Column('Id', UUID(), primary_key=True), 43 | Column('Name', UnicodeText(256))) 44 | self.User.create(self.db) 45 | 46 | try: 47 | self.Project = Table('Project', self.md, autoload=True) 48 | except exc.NoSuchTableError: 49 | self.Project = Table('Project', self.md, 50 | Column('Version', UnicodeText(16)), 51 | Column('Title', UnicodeText(256), nullable=False), 52 | Column('Description', UnicodeText(2048)), 53 | Column('CreatedBy', UUID(), ForeignKey("User.Id"), nullable=False), 54 | Column('CreatedDate', DateTime, nullable=False), 55 | Column('ModifiedBy', UUID(), ForeignKey("User.Id"), nullable=False), 56 | Column('ModifiedDate', DateTime, nullable=False)) 57 | self.Project.create(self.db) 58 | 59 | try: 60 | self.NodeCategory = Table('NodeCategory', self.md, autoload=True) 61 | except exc.NoSuchTableError: 62 | self.NodeCategory = Table('NodeCategory', self.md, 63 | Column('Id', UUID(), primary_key=True), 64 | Column('Name', UnicodeText(256)), 65 | Column('Description', UnicodeText(512)), 66 | Column('CreatedBy', UUID(), ForeignKey("User.Id")), 67 | Column('CreatedDate', DateTime), 68 | Column('ModifiedBy', UUID(), ForeignKey("User.Id")), 69 | Column('ModifiedDate', DateTime)) 70 | self.NodeCategory.create(self.db) 71 | 72 | try: 73 | self.Node = Table('Node', self.md, autoload=True) 74 | except exc.NoSuchTableError: 75 | self.Node = Table('Node', self.md, 76 | Column('Id', UUID(), primary_key=True), 77 | Column('Parent', UUID(), ForeignKey("Node.Id")), 78 | Column('Category', UUID(), ForeignKey("NodeCategory.Id")), 79 | Column('Name', UnicodeText(256)), 80 | Column('Description', UnicodeText(512)), 81 | Column('Color', Integer), 82 | Column('Aggregate', Boolean), 83 | Column('CreatedBy', UUID(), ForeignKey("User.Id")), 84 | Column('CreatedDate', DateTime), 85 | Column('ModifiedBy', UUID(), ForeignKey("User.Id")), 86 | Column('ModifiedDate', DateTime)) 87 | self.Node.create(self.db) 88 | 89 | try: 90 | self.NodeAttribute = Table('NodeAttribute', self.md, autoload=True) 91 | except exc.NoSuchTableError: 92 | self.NodeAttribute = Table('NodeAttribute', self.md, 93 | Column('Id', UUID(), primary_key=True), 94 | Column('Name', UnicodeText(256)), 95 | Column('Description', UnicodeText(512)), 96 | Column('Type', UnicodeText(16)), 97 | Column('Length', Integer), 98 | Column('CreatedBy', UUID(), ForeignKey("User.Id")), 99 | Column('CreatedDate', DateTime), 100 | Column('ModifiedBy', UUID(), ForeignKey("User.Id")), 101 | Column('ModifiedDate', DateTime)) 102 | self.NodeAttribute.create(self.db) 103 | 104 | try: 105 | self.NodeValue = Table('NodeValue', self.md, autoload=True) 106 | except exc.NoSuchTableError: 107 | self.NodeValue = Table('NodeValue', self.md, 108 | Column('Node', UUID(), ForeignKey("Node.Id"), primary_key=True), 109 | Column('Attribute', UUID(), ForeignKey("NodeAttribute.Id"), 110 | primary_key=True), 111 | Column('Value', UnicodeText(256)), 112 | Column('CreatedBy', UUID(), ForeignKey("User.Id")), 113 | Column('CreatedDate', DateTime), 114 | Column('ModifiedBy', UUID(), ForeignKey("User.Id")), 115 | Column('ModifiedDate', DateTime)) 116 | self.NodeValue.create(self.db) 117 | 118 | try: 119 | self.SourceCategory = Table('SourceCategory', self.md, autoload=True) 120 | except exc.NoSuchTableError: 121 | self.SourceCategory = Table('SourceCategory', self.md, 122 | Column('Id', UUID(), primary_key=True), 123 | Column('Name', UnicodeText(256)), 124 | Column('Description', UnicodeText(512)), 125 | Column('CreatedBy', UUID(), ForeignKey("User.Id")), 126 | Column('CreatedDate', DateTime), 127 | Column('ModifiedBy', UUID(), ForeignKey("User.Id")), 128 | Column('ModifiedDate', DateTime)) 129 | self.SourceCategory.create(self.db) 130 | 131 | try: 132 | self.Source = Table('Source', self.md, autoload=True) 133 | except exc.NoSuchTableError: 134 | self.Source = Table('Source', self.md, 135 | Column('Id', UUID(), primary_key=True), 136 | Column('Category', UUID(), ForeignKey("SourceCategory.Id")), 137 | Column('Name', UnicodeText(256)), 138 | Column('Description', UnicodeText(512)), 139 | Column('Color', Integer), 140 | Column('Content', UnicodeText(16384)), 141 | Column('ObjectType', UnicodeText(256)), 142 | Column('SourceType', Integer), 143 | Column('Object', LargeBinary), 144 | Column('Thumbnail', LargeBinary), 145 | #Column('Waveform', LargeBinary, nullable=False), 146 | Column('CreatedBy', UUID(), ForeignKey("User.Id")), 147 | Column('CreatedDate', DateTime), 148 | Column('ModifiedBy', UUID(), ForeignKey("User.Id")), 149 | Column('ModifiedDate', DateTime)) 150 | self.Source.create(self.db) 151 | 152 | try: 153 | self.SourceAttribute = Table('SourceAttribute', self.md, autoload=True) 154 | except exc.NoSuchTableError: 155 | self.SourceAttribute = Table('SourceAttribute', self.md, 156 | Column('Id', UUID(), primary_key=True), 157 | Column('Name', UnicodeText(256)), 158 | Column('Description', UnicodeText(512)), 159 | Column('Type', UnicodeText(16)), 160 | Column('Length', Integer), 161 | Column('CreatedBy', UUID(), ForeignKey("User.Id")), 162 | Column('CreatedDate', DateTime), 163 | Column('ModifiedBy', UUID(), ForeignKey("User.Id")), 164 | Column('ModifiedDate', DateTime)) 165 | self.SourceAttribute.create(self.db) 166 | 167 | try: 168 | self.SourceValue = Table('SourceValue', self.md, autoload=True) 169 | except exc.NoSuchTableError: 170 | self.SourceValue = Table('SourceValue', self.md, 171 | Column('Source', UUID(), ForeignKey("Source.Id"), primary_key=True), 172 | Column('Attribute', UUID(), ForeignKey("SourceAttribute.Id"), 173 | primary_key=True), 174 | Column('Value', UnicodeText(256)), 175 | Column('CreatedBy', UUID(), ForeignKey("User.Id")), 176 | Column('CreatedDate', DateTime), 177 | Column('ModifiedBy', UUID(), ForeignKey("User.Id")), 178 | Column('ModifiedDate', DateTime)) 179 | self.SourceValue.create(self.db) 180 | 181 | try: 182 | self.Tagging = Table('Tagging', self.md, autoload=True) 183 | except exc.NoSuchTableError: 184 | self.Tagging = Table('Tagging', self.md, 185 | Column('Id', UUID(), primary_key=True), 186 | Column('Source', UUID(), ForeignKey("Source.Id")), 187 | Column('Node', UUID(), ForeignKey("Node.Id")), 188 | Column('Fragment', UnicodeText(256)), 189 | Column('Memo', UnicodeText(256)), 190 | Column('CreatedBy', UUID(), ForeignKey("User.Id")), 191 | Column('CreatedDate', DateTime), 192 | Column('ModifiedBy', UUID(), ForeignKey("User.Id")), 193 | Column('ModifiedDate', DateTime)) 194 | self.Tagging.create(self.db) 195 | 196 | def __del__(self): 197 | if self.tr: 198 | self.tr.rollback() 199 | #self.con.close() 200 | self.db.dispose() 201 | 202 | def begin(self): 203 | self.tr = self.con.begin() 204 | 205 | def commit(self): 206 | if self.tr: 207 | self.tr.commit() 208 | self.tr = None 209 | 210 | def rollback(self): 211 | if self.tr: 212 | self.tr.rollback() 213 | self.tr = None 214 | --------------------------------------------------------------------------------