├── .gitignore ├── img ├── img_time.png └── LVM_GEO-logo-PMS30.PNG ├── GISPython ├── UpgradeLog.htm ├── .vs │ └── GISPython │ │ ├── v14 │ │ └── .suo │ │ └── v15 │ │ └── .suo ├── SysGISToolsSysParams.py ├── PShell │ ├── ClearDir.ps1 │ ├── GetLog.ps1 │ └── BackupOldFiles.ps1 ├── RarHelper.py ├── MyError.py ├── __init__.py ├── SysTools_unittest.py ├── GISPythonTool.py ├── TimerHelper.py ├── SFTPHelper.py ├── xmlParamsHealper.py ├── JsonParamsHelper.py ├── ZipHelper.py ├── CachingHelper.py ├── MailHelper.py ├── SetupDefaultEnvironment.py ├── FTPHleper.py ├── SimpleFileOpsSafe.py ├── GDPSyncroniserHelper.py ├── MXDHelper.py ├── GISPythonModule.py ├── SysGISTools.py ├── AGServerHelper.py ├── SimpleFileOps.py └── GISPythonToolBase.py ├── docs ├── requirements.txt ├── modules.rst ├── Makefile ├── authors.rst ├── make.bat ├── index.rst ├── general.rst ├── main_modules.rst ├── conf.py ├── changelog.rst └── helper_modules.rst ├── .idea └── .gitignore ├── MANIFEST.in ├── .gitattributes ├── setup.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | docs/_build 4 | docs/_static 5 | docs/_templates -------------------------------------------------------------------------------- /img/img_time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvmgeo/GISPython/HEAD/img/img_time.png -------------------------------------------------------------------------------- /GISPython/UpgradeLog.htm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvmgeo/GISPython/HEAD/GISPython/UpgradeLog.htm -------------------------------------------------------------------------------- /img/LVM_GEO-logo-PMS30.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvmgeo/GISPython/HEAD/img/LVM_GEO-logo-PMS30.PNG -------------------------------------------------------------------------------- /GISPython/.vs/GISPython/v14/.suo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvmgeo/GISPython/HEAD/GISPython/.vs/GISPython/v14/.suo -------------------------------------------------------------------------------- /GISPython/.vs/GISPython/v15/.suo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lvmgeo/GISPython/HEAD/GISPython/.vs/GISPython/v15/.suo -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinxcontrib-napoleon>=0.6.1 2 | paramiko>=2.1.2 3 | simplejson>=3.10.0 4 | patool>=1.12 5 | python-ntlm>=1.1.0 6 | lxml>=4.3.1 -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- 1 | Modules and examples 2 | ==================== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | main_modules 8 | helper_modules 9 | -------------------------------------------------------------------------------- /GISPython/SysGISToolsSysParams.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module for storing the scripting parameters 4 | """ 5 | Version = 'GISPython Core 10.x - 1.60.0003' # GISPython full version name 6 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include GISPython/PShell/* 3 | 4 | exclude img/* 5 | exclude docs/* 6 | exclude GISPython/*.pyc 7 | exclude GISPython/GEOToolBoxSetup.isproj 8 | exclude GISPython/GISPython.pyproj 9 | exclude GISPython/GISPython.pyproj.vspscc -------------------------------------------------------------------------------- /GISPython/PShell/ClearDir.ps1: -------------------------------------------------------------------------------- 1 | $Now = Get-Date 2 | $InFolder = $args[0] 3 | $InWhat = $args[1] 4 | 5 | function Remove-Tree($Path,$Include='*') { 6 | (Get-ChildItem $Path -Force -Filter $Include) | 7 | Foreach { 8 | Remove-Item -force -recurse ($Path + "\" + $_) 9 | echo ("`t`tRemovinge file: [" + $Path + "\" + $_ + "]") 10 | } 11 | } 12 | 13 | echo ("") 14 | echo ("--- Removing folder: [" + $InFolder + "]" ) 15 | Remove-Tree $InFolder $InWhat 16 | 17 | exit -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = GISPython 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /GISPython/PShell/GetLog.ps1: -------------------------------------------------------------------------------- 1 | $logs = "system","application" 2 | $ComputerName = $args[0] 3 | $OutputDir = $args[1] 4 | $Days = $args[2] 5 | $current = get-date 6 | $oldDate = $current.adddays(-$Days) 7 | $DateStr = Get-Date -format yyyy-MM-dd_HH-mm 8 | $logs | % { echo (" Beginning export " + $ComputerName + "_" + $_ + "_" + $DateStr + ".xml " + (Get-Date -format "HH:mm:ss")) 9 | Get-WinEvent -LogName $_ -ComputerName $ComputerName -ErrorAction SilentlyContinue | Where-Object { ( ($_.TimeCreated -ge $oldDate) ) } | Export-Clixml ($OutputDir + "\" + $ComputerName + "_" + $_ + "_" + $DateStr + ".xml") 10 | echo (" ... Finishing export " + (Get-Date -format "HH:mm:ss") + " ...") } -------------------------------------------------------------------------------- /GISPython/RarHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Rar file operations module 4 | """ 5 | 6 | import patoolib 7 | 8 | class RarHelper: 9 | """Class for easing the Rar file operations""" 10 | 11 | def __init__(self): 12 | """Class initialization procedure 13 | 14 | Args: 15 | self: The reserved object 'self' 16 | """ 17 | 18 | def ExtractRarFile(self, RarFileName, destPath): 19 | """Rar file extraction procedure 20 | 21 | Args: 22 | self: The reserved object 'self' 23 | RarFileName: Extractable file path + name 24 | destPath: Destination path for extracted files 25 | """ 26 | patoolib.extract_archive(RarFileName, outdir=destPath) 27 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | Authors 2 | ======= 3 | 4 | LVM GEO is a collection of geospatial information technology (GIT) products and services 5 | provided by the Geospatial Information Technologies business unit of the JSC Latvia’s State Forests (LVM). 6 | We have been developing GIT since 2009 to support our business operations, and we offer GIT products 7 | and services not only internally, but also to clients specializing in various industries. 8 | LVM GEO products range from a modular and multifunctional geospatial information technology platform 9 | with interfaces for companies and organizations to open tools for spatial data processing for any GIT user. 10 | 11 | :Website: http://www.lvmgeo.lv/en/ 12 | :E-mail: lvmGEOGit@lvm.lv 13 | 14 | .. toctree:: 15 | :maxdepth: 4 16 | -------------------------------------------------------------------------------- /GISPython/PShell/BackupOldFiles.ps1: -------------------------------------------------------------------------------- 1 | $Now = Get-Date 2 | $Days = $args[2] 3 | $InFolder = $args[0] 4 | $OutFolder = $args[1] 5 | $LastWrite = $Now.AddDays(-$Days) 6 | 7 | echo ("") 8 | echo ("--- Archiving of directory " + $InFolder + " to " + $OutFolder + " for last " + $Days + " days - " + (Get-Date -format "HH:mm:ss")) 9 | $Files = get-childitem -path $InFolder | 10 | Where {$_.psIsContainer -eq $false} | 11 | Where {$_.LastWriteTime -le "$LastWrite"} 12 | 13 | foreach ($File in $Files) 14 | {if ($File -ne $NULL) 15 | { 16 | echo (" Archiving: " + $InFolder + "\" + $File) 17 | Move-Item -Path ($InFolder + "\" + $File) -Destination ($OutFolder + "\" + $File) -Force 18 | } 19 | ELSE 20 | {echo (" No files found") 21 | exit} 22 | } 23 | exit -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=python -msphinx 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=GISPython 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The Sphinx module was not found. Make sure you have Sphinx installed, 20 | echo.then set the SPHINXBUILD environment variable to point to the full 21 | echo.path of the 'sphinx-build' executable. Alternatively you may add the 22 | echo.Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /GISPython/MyError.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Error module 4 | """ 5 | 6 | class MyError(Exception): 7 | """Class for storing an error object""" 8 | def __init__(self, strerror): 9 | self.strerror = strerror 10 | def __str__(self): 11 | try: # string 12 | return str(self.strerror) 13 | except Exception: 14 | pass 15 | 16 | try: # unicode 17 | value = unicode(self.strerror) 18 | return value.encode("ascii", "backslashreplace") 19 | except Exception: 20 | pass 21 | 22 | try: # repr 23 | value = self.strerror.repr() 24 | return value 25 | except Exception: 26 | pass 27 | 28 | return '' % type(value).__name__ 29 | def __repr__(self): 30 | return self.__str__() 31 | def __unicode__(self): 32 | return self.strerror 33 | -------------------------------------------------------------------------------- /GISPython/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module for the package definition 4 | """ 5 | 6 | import os 7 | import SysGISToolsSysParams 8 | 9 | if __name__ == "__main__": 10 | """ 11 | Execution of the module 12 | """ 13 | print(u""" 14 | Hello! 15 | 16 | This is GISPython for ArcGIS 10.3.1. 17 | 18 | Version: """ + SysGISToolsSysParams.Version + u""" 19 | 20 | Author: LVM GEO 21 | 22 | Platform contains following additional modules: 23 | """) 24 | # Get the list of all python modules in GISPython folder 25 | filenames = [f for f in os.listdir(os.path.dirname(os.path.abspath(__file__))) if (os.path.isfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), f)) and not f[0:3].upper() == "SYS" and f[-3:].upper() == '.PY' and f.upper() != 'MYERROR.PY' and f.upper() != 'GISPYTHONTOOL.PY' and f.upper() != 'GISPYTHONMODULE.PY' and f.upper() != '__INIT__.PY')] 26 | for fl in filenames: 27 | print(' ' + fl) 28 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | LVM GEO Python core (*GISPython*) documentation! 2 | ================================================ 3 | 4 | LVM GEO Python Core (*GISPython*) is an open source automation and scripting core developed by the LVM GEO team. 5 | Based on this core any developer can build simple and structured Python programming scripts with a 6 | rich functionality. The programming core allows management of all system's maintenance processes 7 | within a unified environment. 8 | *GISPython* project can be found at `GitHub `_. 9 | 10 | .. Attention:: There is undergoing code arrangement according to `PEP 8 `_, so module and some code object names can be changed. All changes will be documented in `changelog `_. 11 | 12 | Table of contents 13 | ----------------- 14 | 15 | .. toctree:: 16 | :maxdepth: 3 17 | 18 | self 19 | general 20 | modules 21 | changelog 22 | authors 23 | 24 | Package index 25 | ------------- 26 | 27 | * :ref:`genindex` 28 | -------------------------------------------------------------------------------- /GISPython/SysTools_unittest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Test procedure module 4 | """ 5 | 6 | import unittest 7 | import GISPythonModule 8 | 9 | class Pr(object): 10 | """Defines test parameters""" 11 | OutDir = 'C:\\GIS\\Log\\Outlog\\' # Output file directory 12 | OutDirArh = 'C:\\GIS\\Log\\Outlog\\Archive\\' # Output file archive directory 13 | ErrorLogDir = 'C:\\GIS\\Log\\ErrorLog\\' # Error output file directory 14 | ErrorLogDirArh = 'C:\\GIS\\Log\\ErrorLog\\Archive\\' # Error output file archive directory 15 | encodingPrimary = 'cp1257' # Windows charset 16 | encodingSecondary = 'cp775' # Shell charset 17 | 18 | class GISTools_unittest(unittest.TestCase): 19 | """GEOPython unit test class""" 20 | 21 | def setUp(self): 22 | """The Test setting up procedure""" 23 | print u"Preparing for tests..." 24 | self.Module = GISPythonModule.GISPythonModule('Unittest', Pr) 25 | self.Module.initModule() 26 | #don't catch errors in the file 27 | return super(GISTools_unittest, self).setUp() 28 | 29 | def tearDown(self): 30 | """"The Test tear down - cleaning up objects after test""" 31 | print u"Cleaning up after tests..." 32 | self.Module.MyDispose() 33 | return super(GISTools_unittest, self).tearDown() 34 | 35 | def test_Tool_init(self): 36 | """Check if it is possible to get the geoprocessor object""" 37 | print u"Check if it is possible to get the geoprocessor object ... OK" 38 | 39 | def test_shellRun(self): 40 | """Check the shell execution commands""" 41 | self.Module.Tool.runShell("Dir") 42 | print u"Check if it is possible to get the geoprocessor object ... OK" 43 | 44 | if __name__ == "__main__": 45 | print "" 46 | unittest.main() 47 | raw_input("Press Enter to continue...") 48 | -------------------------------------------------------------------------------- /GISPython/GISPythonTool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module defines abstract classes for the ESRI Toolbox tool definition 4 | """ 5 | 6 | class GISPythonTool(object): 7 | """Class which helps to create an ArcGIS Toolbox""" 8 | def __init__(self): 9 | """Define the tool (tool name is the class name)""" 10 | self.canRunInBackground = False 11 | 12 | def getParameterInfo(self): 13 | """Define parameter definitions""" 14 | params = None 15 | return params 16 | 17 | def isLicensed(self): 18 | """Determines if the tool is licenced for execution""" 19 | return True 20 | 21 | def updateParameters(self, parameters): 22 | """This method is called in case the parameters are changed, 23 | and is used for setting up the parameters 24 | """ 25 | validator = getattr(self, 'ToolValidator', None) 26 | if validator: 27 | return validator(parameters).updateParameters() 28 | 29 | def updateMessages(self, parameters): 30 | """This method is called after an inner validation, 31 | and is intended for carrying out an additional validations 32 | """ 33 | validator = getattr(self, 'ToolValidator', None) 34 | if validator: 35 | return validator(parameters).updateMessages() 36 | 37 | def execute(self, parameters, messages): 38 | """Executes a tool""" 39 | raise NotImplementedError 40 | 41 | class ToolValidator(object): 42 | """Class for validating the tool's parameter values and controlling 43 | the behavior of the tool's dialog. 44 | """ 45 | def __init__(self, parameters): 46 | """Setup arcpy and the list of the tool's parameters.""" 47 | self.params = parameters 48 | 49 | def initializeParameters(self): 50 | """Refine properties of the tool's parameters. This method is 51 | called when the tool is opened. 52 | """ 53 | return 54 | 55 | def updateParameters(self): 56 | """Modify the values and properties of parameters before internal 57 | validation is performed. This method is called whenever a parameter 58 | has been changed. 59 | """ 60 | return 61 | 62 | def updateMessages(self): 63 | """Modify the messages created by internal validation for each tool 64 | parameter. This method is called after internal validation. 65 | """ 66 | return 67 | -------------------------------------------------------------------------------- /GISPython/TimerHelper.py: -------------------------------------------------------------------------------- 1 | 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Timing module 5 | """ 6 | 7 | import datetime 8 | 9 | class TimerHelper(object): 10 | """Class for easing the countdown procedures""" 11 | 12 | def __init__(self): 13 | """Class initialization procedure 14 | 15 | Args: 16 | self: The reserved object 'self' 17 | """ 18 | self.StartTime = datetime.datetime.now() 19 | 20 | def TimerReset(self): 21 | """Reset the countdown 22 | 23 | Args: 24 | self: The reserved object 'self' 25 | """ 26 | self.StartTime = datetime.datetime.now() 27 | 28 | 29 | def GetTime(self): 30 | """Get the elapsed time 31 | 32 | Args: 33 | self: The reserved object 'self' 34 | 35 | Returns: 36 | Output text as a string 37 | """ 38 | TD = datetime.datetime.now() - self.StartTime 39 | return str(TD) 40 | 41 | def GetTimeReset(self): 42 | """Reset the elapsed time 43 | 44 | Args: 45 | self: The reserved object 'self' 46 | 47 | Returns: 48 | Output text as a string 49 | """ 50 | TD = self.GetTime() 51 | self.TimerReset() 52 | return TD 53 | 54 | class TimedSubprocess: 55 | """Class for providing a code block with timing capabilities""" 56 | 57 | def __init__(self, Tool, txt, lvl=1): 58 | """Class initialization procedure 59 | Args: 60 | self: The reserved object 'self' 61 | Tool: Reference to the GISTools10 object 62 | txt: Output text 63 | lvl: Sublevel 64 | """ 65 | self.StartTime = datetime.datetime.now() 66 | self.Tool = Tool 67 | self.txt = txt 68 | self.T = TimerHelper() 69 | self.prefix = "" 70 | for i in range(0, lvl-1): 71 | self.prefix += ' ' 72 | def __enter__(self): 73 | """With statement opening procedure 74 | Args: 75 | self: The reserved object 'self' 76 | """ 77 | self.Tool.AddMessage(u'\n\n{0}------------------------\n{0}>>>> Begin {1} - {2}\n{0}------------------------\n'.format(self.prefix, self.txt, self.Tool.MyNow())) 78 | return self 79 | 80 | def __exit__(self, type, value, traceback): 81 | """With statement closing procedure 82 | Args: 83 | self: The reserved object 'self' 84 | """ 85 | self.Tool.AddMessage(u'\n{0}------------------------\n{0}>>>> End {1} - {2}\n{0}------------------------\n'.format(self.prefix, self.txt, self.T.GetTimeReset())) 86 | -------------------------------------------------------------------------------- /GISPython/SFTPHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | SFTP operations module 4 | """ 5 | 6 | import paramiko 7 | 8 | 9 | class SFTPHelper: 10 | """Class for easing the SFTP operations""" 11 | 12 | def __init__(self, userName, password, host, port, pkey_file=None): 13 | """Class initialization procedure 14 | 15 | Args: 16 | self: The reserved object 'self' 17 | username: Username 18 | password: Password (provide None if using pkey_file parameter) 19 | host: IP adress 20 | port: Port number 21 | pkey_file: path to rsa private key file 22 | """ 23 | self.host = host 24 | self.port = port 25 | self.username = userName 26 | self.password = password 27 | 28 | if not pkey_file is None: 29 | pkey = paramiko.RSAKey.from_private_key_file(pkey_file) 30 | 31 | # Transporta objekta izveide 32 | self.transport = paramiko.Transport((self.host, self.port)) 33 | if pkey_file is None: 34 | self.transport.connect(username=self.username, password=self.password) 35 | else: 36 | self.transport.connect(username=self.username, pkey=pkey) 37 | 38 | # SFTP objekta izveide 39 | self.sftp = paramiko.SFTPClient.from_transport(self.transport) 40 | 41 | def list_files(self, remote_path): 42 | """List the files in the remote directory 43 | 44 | Args: 45 | self: The reserved object 'self' 46 | remote_path: Remote directory path 47 | """ 48 | return self.sftp.listdir(remote_path) 49 | 50 | def delete_file(self, remote): 51 | """Deletes the file, using SFTP 52 | 53 | Args: 54 | self: The reserved object 'self' 55 | remote: Remote file path 56 | """ 57 | self.sftp.remove(remote) 58 | 59 | def upload(self, local, remote): 60 | """Upload the file, using SFTP 61 | 62 | Args: 63 | self: The reserved object 'self' 64 | local: Uploadable file local path 65 | remote: Uploadable file path on the server 66 | """ 67 | self.sftp.put(local, remote) 68 | 69 | def download(self, remote, local): 70 | """Download the file, using SFTP 71 | 72 | Args: 73 | self: The reserved object 'self' 74 | remote: Downloadable file path on the server 75 | local: Local download path 76 | """ 77 | self.sftp.get(remote, local) 78 | 79 | def close(self): 80 | """Closes the connection, if it is active""" 81 | if self.transport.is_active(): 82 | self.sftp.close() 83 | self.transport.close() 84 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | import os 3 | import re 4 | 5 | def get_version(): 6 | """Get version number from SysGISToolsSysParams file""" 7 | #versFile = 'GISPython\\SysGISToolsSysParams.py' 8 | versFile = os.path.join('GISPython', 'SysGISToolsSysParams.py') # os.path.join for Unix compatibility 9 | versFDir = os.path.dirname(os.path.abspath(__file__)) 10 | versionpath = os.path.join(versFDir, versFile) 11 | fileRead = open(versionpath, 'rt').readlines() 12 | vsre = r"^Version = ['\"]([^'\"]*)['\"]" 13 | for line in fileRead: 14 | mo = re.search(vsre, line, re.M) 15 | if mo: 16 | return str(mo.group(1)[24:]) 17 | 18 | setup( 19 | name = 'GISPython', 20 | version = get_version(), # version number according to PEP440 https://www.python.org/dev/peps/pep-0440/ 21 | description = 'Additional tools for administering and automating different ArcPy and ArcGIS Server geoprocessing operations. Package is intended for use with ArcGIS 10.2.1 and later (has been tested on ArcGIS 10.3.1, ArcGIS 10.4, ArcGIS 10.5, ArcGIS 10.6.1)', # short package description 22 | long_description = 'For readme see GitHub https://github.com/lvmgeo/GISPython', # if needed (entire documentation) 23 | url = 'https://github.com/lvmgeo/GISPython', # GitHub url 24 | author = 'LVM GEO', # author 25 | author_email = 'lvmGEOGit@lvm.lv', # email 26 | license = 'GPLv3', # license 27 | 28 | # classifiers for PyPi: https://pypi.python.org/pypi?%3Aaction=list_classifiers 29 | classifiers=[ 30 | # Project development status 31 | 'Development Status :: 5 - Production/Stable', 32 | # Intended audience and topic 33 | 'Intended Audience :: Developers', 34 | 'Intended Audience :: System Administrators', 35 | 'Topic :: Scientific/Engineering :: GIS', 36 | 'Topic :: Software Development :: Libraries :: Python Modules', 37 | # License https://choosealicense.com/ 38 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 39 | # Operating system 40 | 'Operating System :: Microsoft :: Windows', 41 | 42 | # Specify the Python versions you support here. 43 | 'Programming Language :: Python', 44 | 'Programming Language :: Python :: 2', 45 | 'Programming Language :: Python :: 2.6', 46 | 'Programming Language :: Python :: 2.7', 47 | ], 48 | 49 | keywords = ['ArcGIS', 'ArcPy', 'ArcGIS Server', 'automation'], # keywords (string or list of strings) 50 | packages = ["GISPython"], # packages to install 51 | include_package_data = True, 52 | install_requires = ['paramiko>=2.1.2', 'simplejson>=3.10.0', 'patool>=1.12', 'python-ntlm>=1.1.0', 'lxml>=4.3.1'], # list of packages required in project (these will be installed by pip) 53 | 54 | # entry_points = {'console_scripts': [ 55 | # 'GISPython = __init__:main' 56 | # ] 57 | # }, 58 | ) 59 | -------------------------------------------------------------------------------- /GISPython/xmlParamsHealper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module for xml parameter file procedures 4 | """ 5 | 6 | import os 7 | import codecs 8 | from lxml import etree 9 | from collections import OrderedDict 10 | 11 | utf8_parser = etree.XMLParser(encoding='utf-8') 12 | 13 | class XMLParams(object): 14 | """xml parameter reading support class""" 15 | def __init__(self, Tool, ConfigFolder, ConfigFile): 16 | """Class initialization procedure 17 | 18 | Args: 19 | self: The reserved object 'self' 20 | Tool: GISPythonTool (Optional) 21 | ConfigFolder: Configuration file storing directory 22 | ConfigFile: Name of the configuration file (without extension) 23 | """ 24 | self.Tool = Tool 25 | self.ConfigFolder = ConfigFolder 26 | self.ConfigFile = ConfigFile 27 | if not Tool == None: 28 | self.ConfigPath = os.path.join(self.Tool.ExecutePatch, ConfigFolder, ConfigFile) 29 | else: 30 | if not ConfigFolder == None: 31 | self.ConfigPath = os.path.join(ConfigFolder, ConfigFile) 32 | else: 33 | self.ConfigPath = ConfigFile 34 | self.Params = [] 35 | 36 | def GetParams(self): 37 | """Get parameters from the parameter file 38 | 39 | Args: 40 | self: The reserved object 'self' 41 | """ 42 | with open(self.ConfigPath, 'r') as xmlfile: 43 | xmllines = xmlfile.readlines() 44 | doc = etree.ElementTree(etree.XML("".join(xmllines))) 45 | self.Params = doc 46 | return self.Params 47 | 48 | def WriteParams(self): 49 | """Save parameters in the parameter file 50 | 51 | Args: 52 | self: The reserved object 'self' 53 | """ 54 | with open(self.ConfigPath, 'w') as xmlfile: 55 | # xmlString = etree.tostring(self.Params, pretty_print=True, method="xml", xml_declaration=True, encoding="utf-8") 56 | self.Params.write(xmlfile, pretty_print=True, method="xml", xml_declaration=True, encoding="utf-8") 57 | 58 | 59 | def UpdateValueByPath(self, path, Value, index = 0, isString = False): 60 | elem = self.Params 61 | if isString: 62 | elem.xpath(path)[index].append(etree.fromstring(Value)) 63 | else: 64 | elem.xpath(path)[index].text = Value 65 | 66 | def AppendValueByPath(self, path, key, Value, attrib, index = 0, isString = False): 67 | elem = self.Params 68 | elem = elem.xpath(path)[index] 69 | if key==None: 70 | node = elem 71 | else: 72 | node = etree.SubElement(elem, key, attrib) 73 | if isString: 74 | node.append(etree.fromstring(Value)) 75 | else: 76 | node.text = Value 77 | 78 | def UpdateAtributeByPath(self, path, atribute, Value, index = 0): 79 | elem = self.Params 80 | elem.xpath(path)[index].attrib[atribute] = Value 81 | 82 | def GetValueByPath(self, path, namespaces='#'): 83 | elem = self.Params 84 | if namespaces == '#': 85 | elem = elem.xpath(path)[0].text 86 | else: 87 | elem = elem.xpath(path, namespaces=namespaces)[0].text 88 | return elem 89 | 90 | def GetAtributeByPath(self, path, atribute, namespaces='#'): 91 | elem = self.Params 92 | if namespaces == '#': 93 | elem = elem.xpath(path)[0].attrib[atribute] 94 | else: 95 | elem = elem.xpath(path, namespaces=namespaces)[0].attrib[atribute] 96 | return elem 97 | -------------------------------------------------------------------------------- /GISPython/JsonParamsHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module for Json parameter file procedures 4 | """ 5 | 6 | import os 7 | import codecs 8 | import simplejson as json 9 | from collections import OrderedDict 10 | 11 | class JsonParams(object): 12 | """Json parameter reading support class""" 13 | def __init__(self, Tool, ConfigFolder, ConfigFile): 14 | """Class initialization procedure 15 | 16 | Args: 17 | self: The reserved object 'self' 18 | Tool: GEOPython tool (optional) 19 | ConfigFolder: Configuration file storing directory 20 | ConfigFile: Name of the configuration file (without extension) 21 | """ 22 | self.Tool = Tool 23 | self.ConfigFolder = ConfigFolder 24 | self.ConfigFile = ConfigFile 25 | if not Tool == None: 26 | self.ConfigPath = os.path.join(self.Tool.ExecutePatch, ConfigFolder, ConfigFile) 27 | else: 28 | if not ConfigFolder == None: 29 | self.ConfigPath = os.path.join(ConfigFolder, ConfigFile) 30 | else: 31 | self.ConfigPath = ConfigFile 32 | if not self.ConfigPath.lower()[-4:] == "json": 33 | self.ConfigPath = self.ConfigPath + '.json' 34 | self.Params = [] 35 | 36 | def GetParams(self): 37 | """Get parameters from the parameter file 38 | 39 | Args: 40 | self: The reserved object 'self' 41 | """ 42 | f = codecs.open(self.ConfigPath, 'r', 'utf-8') 43 | JsonString = f.read() 44 | f.close() 45 | J = json.loads(JsonString, object_pairs_hook=OrderedDict) 46 | self.Params = J 47 | return self.Params 48 | 49 | def WriteParams(self, sort_keys=True): 50 | """Save parameters in the parameter file 51 | 52 | Args: 53 | self: The reserved object 'self' 54 | """ 55 | JsonString = json.dumps(self.Params, sort_keys, indent=4 * ' ') 56 | f = codecs.open(self.ConfigPath, 'w', 'utf-8') 57 | f.write(JsonString) 58 | f.close() 59 | 60 | def UpdateValueByPath(self, path, Value, valueIsStringJson = False): 61 | elem = self.Params 62 | 63 | if valueIsStringJson: 64 | Value = json.loads(Value, object_pairs_hook=OrderedDict) 65 | 66 | pathList = path.strip("\\").split("\\") 67 | for key in pathList[:-1]: 68 | elem = self.__get_sub_element__(elem, key) 69 | LastKey = pathList[-1:][0] 70 | 71 | elem[LastKey] = Value 72 | 73 | def AppendValueByPath(self, path, key, Value, valueIsStringJson = False): 74 | elem = self.Params 75 | 76 | if valueIsStringJson: 77 | Value = json.loads(Value, object_pairs_hook=OrderedDict) 78 | 79 | if not path == '': 80 | pathList = path.strip("\\").split("\\") 81 | for pathkey in pathList: 82 | elem = self.__get_sub_element__(elem, pathkey) 83 | 84 | if isinstance(elem, list): 85 | elem.insert(key, Value) 86 | else: 87 | elem[key] = Value 88 | 89 | def GetValueByPath(self, path): 90 | elem = self.Params 91 | pathList = path.strip("\\").split("\\") 92 | for key in pathList: 93 | elem = self.__get_sub_element__(elem, key) 94 | 95 | return elem 96 | 97 | def __get_sub_element__(self, element, key): 98 | position = 0 99 | if key.find('[')>-1 and key.find(']')>-1: 100 | position = int(key.replace(']','').replace('[','')) 101 | return element[position] 102 | else: 103 | return element[key] -------------------------------------------------------------------------------- /GISPython/ZipHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Zip file operations module 4 | """ 5 | import os 6 | import zipfile 7 | 8 | class ZipHelper: 9 | """Class for easing the Zip file operations""" 10 | 11 | def __init__(self): 12 | """Class initialization procedure 13 | 14 | Args: 15 | self: The reserved object 'self' 16 | """ 17 | 18 | def CompressFile(self, filePath, zipFileName): 19 | """Compress file 20 | 21 | Args: 22 | self: The reserved object 'self' 23 | filePath: Archivable file path + name 24 | zipFileName: New Zip file path + name 25 | """ 26 | zfile = zipfile.ZipFile(zipFileName, "w", zipfile.ZIP_DEFLATED) 27 | zfile.write(filePath, arcname=zfile.write(filePath, arcname=os.path.basename(filePath))) 28 | zfile.close() 29 | 30 | def CompressFileList(self, filePathList, zipFileName, base_dir=None, append=False): 31 | """Zip all files in the list 32 | 33 | Args: 34 | self: The reserved object 'self' 35 | filePathList: List of archivable file paths + names 36 | zipFileName: New Zip file path + name 37 | base_dir: dase dir to strip from paths in file lists 38 | append (Bool): (Optional) True if Zip exists and data is appended to it 39 | """ 40 | if append: 41 | zfile = zipfile.ZipFile(zipFileName, 'a', zipfile.ZIP_DEFLATED, allowZip64=True) 42 | else: 43 | zfile = zipfile.ZipFile(zipFileName, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) 44 | for filePath in filePathList: 45 | if base_dir is None: 46 | arc_name = os.path.basename(filePath) 47 | else: 48 | arc_name = _ireplace(base_dir, '', filePath) 49 | zfile.write(filePath, arcname=arc_name) 50 | zfile.close() 51 | 52 | def CompressDir(self, dirPath, zipFileName, excludeExt=[], append=False, start_path_in_zip=''): 53 | """Zip all files in the directory 54 | 55 | Args: 56 | self: The reserved object 'self' 57 | zipFileName (string): New Zip file path + name 58 | dirPath (string): Directory which contains archivable files 59 | excludeExt (list): File extensions not to include in the archive 60 | append (Bool): (Optional) True if Zip exists and data is appended to it 61 | start_path_in_zip (string): (Optional) used to specify subfolder in zipfile in with data vill be written 62 | """ 63 | if append: 64 | zfile = zipfile.ZipFile(zipFileName, 'a', zipfile.ZIP_DEFLATED, allowZip64=True) 65 | else: 66 | zfile = zipfile.ZipFile(zipFileName, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) 67 | for root, dirs, files in os.walk(dirPath): 68 | for file_path in files: 69 | do_compress = True 70 | for ext in excludeExt: 71 | if file_path.endswith(ext): 72 | do_compress = False 73 | if do_compress: 74 | zfile.write(os.path.join(root, file_path), arcname=os.path.join(root, file_path).replace(dirPath, start_path_in_zip)) 75 | zfile.close() 76 | 77 | def ExtractZipFile(self, zipFileName, destPath): 78 | """Extracts the compressed file 79 | 80 | Args: 81 | self: The reserved object 'self' 82 | zipFileName: Extractable file full path + name 83 | destPath: Destination path in which to extract the files 84 | """ 85 | zfile = zipfile.ZipFile(zipFileName) 86 | zfile.extractall(destPath) 87 | zfile.close() 88 | 89 | def _ireplace(old, new, text): 90 | idx = 0 91 | while idx < len(text): 92 | index_l = text.lower().find(old.lower(), idx) 93 | if index_l == -1: 94 | return text 95 | text = text[:index_l] + new + text[index_l + len(old):] 96 | idx = index_l + len(new) 97 | return text 98 | -------------------------------------------------------------------------------- /docs/general.rst: -------------------------------------------------------------------------------- 1 | General information 2 | =================== 3 | 4 | Installation 5 | ------------ 6 | 7 | Dependencies 8 | ************ 9 | 10 | - ArcGIS 10.x /recommended with newest patches and service packs/ (*GISPython* is currently running on production systems based on ArcGIS 10.2.1 un until 10.9.1) 11 | - Python 2.7 (included in ArcGIS installation) (arcpy and numpy modules included) 12 | - Additional python modules: 13 | 14 | - `PyCrypto `_ (manual installation) 15 | - NTLM: ``pip install python-ntlm`` (included in package setup process) 16 | - Paramiko: ``pip install paramiko`` (included in package setup process) 17 | - patool: ``pip install patool`` (included in package setup process) 18 | - simpleJson: ``pip install simplejson`` (included in package setup process) 19 | 20 | Package installation 21 | ******************** 22 | 23 | *GISPython* package is available on the `Python Package Index `_, so you can get it via pip:: 24 | 25 | pip install GISPython 26 | 27 | .. Note:: If pip isn’t installed, you can get it `here `_! 28 | 29 | Configuration & basic usage 30 | --------------------------- 31 | 32 | Before using *GISPython* modules in custom geoprocessing scripts, you need to set up your scripting environment with `*SetupDefaultEnvironment* `_ module which also includes template for user scripts. 33 | 34 | *SetupDefaultEnvironment* module also includes basic parameters (variable *paramsFileSource*) for parameter file (e.g. SysGISParams.py) which is important, because *GISPython* relies of several 35 | parameters to be present to function successfully: 36 | 37 | * OutDir - directory for storing script output log files ``OutDir = r'C:\GIS\Log\Outlog\'`` 38 | * OutDirArh - directory for storing script output log file archive (all non active files) ``OutDirArh = r'C:\GIS\Log\Outlog\Archive\'`` 39 | * ErrorLogDir - directory for storing script error log files ``ErrorLogDir = r'C:\GIS\Log\ErrorLog\'`` *(Important! This directory can be monitored for non empty files. If this directory has a file that is non empty - this indicates that a script has failed)* 40 | * ErrorLogDirArh - directory for storing script error log files ``ErrorLogDirArh = r'C:\GIS\Log\ErrorLog\Archive'`` 41 | * TmpFolder - Temp folder ``TmpFolder = r'C:\GIS\tmp'`` 42 | * encodingPrimary - encoding of Windows shell ``encodingPrimary = 'cp775'`` 43 | * encodingSecondary - encoding of Windows unicode language used ``encodingSecondary = 'cp1257'`` 44 | * SetLogHistory - enable or disable Geoprocessing history logging ``SetLogHistory = False`` 45 | 46 | .. Note:: It is recommended to define additional script parameters in SysGISParams.py file, to keep the main code clean. Our approach is to define all the parameters that define current system environment be kept in this one file. In case of moving environment (e.g. test system and production system) this one file has the specific connections and can be easily modified without changing the scripts. 47 | 48 | Recommendations 49 | *************** 50 | 51 | Set up the variables at the beggining of the main function, to shorten the main code:: 52 | 53 | Tool = self.Tool 54 | gp = Tool.gp 55 | callGP = Tool.callGP 56 | pj = os.path.join 57 | 58 | Basic operations 59 | **************** 60 | 61 | ArcPy function call:: 62 | 63 | gpCaller = self.Tool.callGP 64 | slay = 'some_layer' 65 | callGP('AddField_management', slay, 'Day_TXT', 'TEXT', '#', '#', 10) 66 | callGP('AddField_management', slay, 'CAR', 'TEXT', '#', '#', 128) 67 | callGP('AddField_management', slay, 'WorkID', 'DOUBLE', 12, 0) 68 | callGP('AddField_management', slay, 'REC_DATE_FROM', 'DATE') 69 | 70 | Tool message output:: 71 | 72 | Tool = self.Tool 73 | self.Tool.AddMessage(u'This is a message') 74 | self.Tool.AddWarning(u'This is a warning') 75 | self.Tool.AddError(u'This is an error') 76 | 77 | .. toctree:: 78 | :maxdepth: 4 79 | -------------------------------------------------------------------------------- /GISPython/CachingHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Carries out service caching on ArcServer providing x3 retry capabilities 4 | """ 5 | 6 | #Modula pamata klase 7 | class CachingHelper: 8 | """Support class which generates the caches""" 9 | def __init__(self, Tool, vServer, vExtent="#", vTerritoryLayer="#"): 10 | """Class initialization procedure 11 | 12 | Args: 13 | self: The reserved object 'self' 14 | Tool: GEOPython tool 15 | vServer: Server 16 | vExtent: Extent 17 | vTerritoryLayer: Territory layer 18 | """ 19 | self.Tool = Tool 20 | self.Server = vServer 21 | self.Extent = vExtent 22 | self.TerritoryLayer = vTerritoryLayer 23 | 24 | def GenerateCache(self, vService, vInstances, vCashScales, vFolder="#", vDeleteScales='#'): 25 | """Base procedure of the tool 26 | 27 | Args: 28 | self: The reserved object 'self' 29 | vService: Cacheable serviss 30 | vInstances: Cacheable instance count 31 | vCashScales: Cacheable scales 32 | vFolder: Cache service directory 33 | vDeleteScales: Scales to delete 34 | """ 35 | 36 | if vFolder == "#": 37 | vFolder = "" 38 | elif vFolder == None: 39 | vFolder = "" 40 | else: 41 | vFolder = str(vFolder) + "\\" 42 | vFolder = "\\" + vFolder 43 | 44 | if vDeleteScales == '#': 45 | vDeleteScales = vCashScales 46 | 47 | input_service = self.Server + vFolder + str(vService) + '.MapServer' 48 | if vDeleteScales != "None": 49 | try: # 1 try to delete 50 | self.Tool.gp.ManageMapServerCacheTiles_server(input_service, vCashScales, "DELETE_TILES", vInstances, self.TerritoryLayer, self.Extent, 'WAIT') 51 | self.Tool.OutputMessages() 52 | except: # 2 try to delete 53 | self.Tool.OutputMessages() 54 | self.Tool.AddMessage("Delete Restart x1 ----------------------------------------------- " + self.Tool.MyNow()) 55 | try: 56 | self.Tool.gp.ManageMapServerCacheTiles_server(input_service, vCashScales, "DELETE_TILES", vInstances, self.TerritoryLayer, self.Extent, 'WAIT') 57 | self.Tool.OutputMessages() 58 | except: # 3 try to delete 59 | self.Tool.OutputMessages() 60 | self.Tool.AddMessage("Delete Restart x2 ----------------------------------------------- " + self.Tool.MyNow()) 61 | try: 62 | self.Tool.gp.ManageMapServerCacheTiles_server(input_service, vCashScales, "DELETE_TILES", vInstances, self.TerritoryLayer, self.Extent, 'WAIT') 63 | self.Tool.OutputMessages() 64 | except: 65 | self.Tool.OutputMessages() 66 | self.Tool.AddWarning("Delete Restart x3 for service {0} - Failed ------------------------------------- {1} ".format(input_service, self.Tool.MyNow())) 67 | 68 | try: # 1 try 69 | self.Tool.gp.ManageMapServerCacheTiles_server(input_service, vCashScales, "RECREATE_ALL_TILES", vInstances, self.TerritoryLayer, self.Extent, 'WAIT') 70 | self.Tool.OutputMessages() 71 | except: # 2 try 72 | self.Tool.OutputMessages() 73 | self.Tool.AddMessage("Restart x1 ----------------------------------------------- " + self.Tool.MyNow()) 74 | try: 75 | self.Tool.gp.ManageMapServerCacheTiles_server(input_service, vCashScales, "RECREATE_EMPTY_TILES", vInstances, self.TerritoryLayer, self.Extent, 'WAIT') 76 | self.Tool.OutputMessages() 77 | except: # 3 try 78 | self.Tool.OutputMessages() 79 | self.Tool.AddMessage("Restart x2 ----------------------------------------------- " + self.Tool.MyNow()) 80 | try: 81 | self.Tool.gp.ManageMapServerCacheTiles_server(input_service, vCashScales, "RECREATE_EMPTY_TILES", vInstances, self.TerritoryLayer, self.Extent, 'WAIT') 82 | self.Tool.OutputMessages() 83 | except: 84 | self.Tool.OutputMessages() 85 | self.Tool.AddWarning("Caching Restart x3 for service {0} - Failed ------------------------------------- {1} ".format(input_service, self.Tool.MyNow())) 86 | -------------------------------------------------------------------------------- /GISPython/MailHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module for e-mail operations 4 | """ 5 | import os 6 | 7 | class MailHelper: 8 | """Class for easing the SMTP operations""" 9 | 10 | def __init__(self, From, recipients, Subject, Text, msg_format='plain'): 11 | """Class initialization procedure 12 | 13 | Args: 14 | self: The reserved object 'self' 15 | From: E-mail adress from which to send 16 | recipients: Recipient array 17 | Subject: E-mail subject 18 | Text: E-mail text 19 | msg_format: E-mail text format (Default = 'plain') 20 | """ 21 | from email.mime.multipart import MIMEMultipart 22 | from email.mime.text import MIMEText 23 | self.msg = MIMEMultipart() 24 | self.msg['Subject'] = Subject 25 | self.msg['To'] = ", ".join(recipients) 26 | self.msg.attach(MIMEText(Text, msg_format, 'utf-8')) 27 | self.From = From 28 | self.recipients = recipients 29 | 30 | def AttachFile(self, FilePath): 31 | """Procedure to add attachments 32 | 33 | Args: 34 | self: The reserved object 'self' 35 | FilePath: Path to the attachment 36 | """ 37 | from email.mime.base import MIMEBase 38 | from email import encoders 39 | part = MIMEBase('application', "octet-stream") 40 | with open(FilePath, "rb") as _file: 41 | part.set_payload(_file.read()) 42 | encoders.encode_base64(part) 43 | part.add_header('Content-Disposition', 'attachment; filename="{0}"'.format(os.path.basename(FilePath))) 44 | self.msg.attach(part) 45 | 46 | def SendMessage(self, Mailserver, port=None, user=None, password=None, useTLS=False, useSSL=False): 47 | """Procedure for sending an e-mail 48 | 49 | Args: 50 | self: The reserved object 'self' 51 | Mailserver: Mailserver name 52 | port: Mailserver port number 53 | user: Username 54 | password: Password 55 | useTLS: Use TLS (Default = False) 56 | useSSL: Use SSL (Default = False) 57 | """ 58 | import smtplib 59 | if useSSL: 60 | mailer = smtplib.SMTP_SSL() 61 | else: 62 | mailer = smtplib.SMTP() 63 | if port != None: 64 | mailer.connect(Mailserver, port) 65 | else: 66 | mailer.connect(Mailserver) 67 | if useTLS: 68 | mailer.starttls() 69 | if user != None: 70 | mailer.login(user, password) 71 | mailer.sendmail(self.From, self.recipients, self.msg.as_string()) 72 | mailer.close() 73 | 74 | class GISPythonMailHelper(MailHelper): 75 | """MailHelper wrapper class, which acquires parameters from the GISPython parameter file""" 76 | 77 | def __init__(self, Pr, recipients, Subject, Text, Files=None, msg_format='plain'): 78 | """Class initialization procedure 79 | 80 | Args: 81 | self: The reserved object 'self' 82 | Pr: Parameter file with corresponding parameters 83 | recipients: Recipient array 84 | Subject: E-mail subject 85 | Text: E-mail text 86 | Files: Array of files to attach 87 | msg_format: E-mail text format (Default = 'plain') 88 | """ 89 | mailer = MailHelper(Pr.MailFromAdress, recipients, Subject, Text, msg_format) 90 | 91 | # Check if attributes exists 92 | if hasattr(Pr, 'MailserverPort'): 93 | port = Pr.MailserverPort 94 | else: 95 | port = None 96 | 97 | if hasattr(Pr, 'MailserverUseTLS'): 98 | useTLS = Pr.MailserverUseTLS 99 | else: 100 | useTLS = None 101 | 102 | if hasattr(Pr, 'MailserverUseSSL'): 103 | useSSL = Pr.MailserverUseSSL 104 | else: 105 | useSSL = None 106 | 107 | if hasattr(Pr, 'MailserverUser'): 108 | user = Pr.MailserverUser 109 | pwd = Pr.MailserverPWD 110 | else: 111 | user = None 112 | pwd = None 113 | 114 | if Files is not None and Files: 115 | for File in Files: 116 | mailer.AttachFile(File) 117 | 118 | # Send a message 119 | mailer.SendMessage(Pr.Mailserver, port, user, pwd, useTLS, useSSL) 120 | -------------------------------------------------------------------------------- /GISPython/SetupDefaultEnvironment.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module for setting up an default environment 4 | """ 5 | 6 | import codecs, os 7 | 8 | paramsFileSource = """# -*- coding: cp1257 -*- 9 | ''' 10 | Standard generated gispython params file 11 | see: http://gispython.readthedocs.io/en/latest/index.html 12 | ''' 13 | 14 | import GISPython.SysGISToolsSysParams 15 | import os 16 | 17 | 18 | # --------------------------------------------------------------------------- 19 | # Basic parameters 20 | # --------------------------------------------------------------------------- 21 | Version = GISPython.SysGISToolsSysParams.Version 22 | OutDir = 'C:\\\\GIS\\\\Log\\\\Outlog\\\\' # Output file directory 23 | OutDirArh = 'C:\\\\GIS\\\\Log\\\\Outlog\\\\Archive\\\\' # Output file directory archive 24 | ErrorLogDir = 'C:\\\\GIS\\\\Log\\\\ErrorLog\\\\' # Error output file directory 25 | ErrorLogDirArh = 'C:\\\\GIS\\\\Log\\\\ErrorLog\\\\Archive\\\\' # Error output file directory archive 26 | TmpFolder = 'C:\\\\GIS\\\\tmp\\\\' # Temp directory 27 | # DataFolder = 'C:\\\\GIS\\\\Data\\\\' # Directory for storing source data (not automaticaly created) 28 | # BackupFolder = 'C:\\\\GIS\\\\Backup\\\\' # Directory for storing data backup (not automaticaly created) 29 | encodingPrimary = 'cp1257' # Windows charset 30 | encodingSecondary = 'cp775' # Windows Shell charset 31 | SetLogHistory = False # enable or disable Geoprocessing history logging 32 | """ 33 | 34 | sampleModule = """# -*- coding: utf-8 -*- 35 | '''This sample code can be used as a template for gispython modules''' 36 | import os 37 | 38 | from GISPython import GISPythonModule 39 | 40 | import SysGISParams as Pr 41 | 42 | 43 | #main class GISPythonModule 44 | class MainModule (GISPythonModule.GISPythonModule): 45 | ''' 46 | Main module 47 | ''' 48 | def __init__(self): 49 | '''Class initialization''' 50 | modulename = os.path.splitext(os.path.basename(__file__))[0] 51 | GISPythonModule.GISPythonModule.__init__(self, modulename, Pr, __file__, licenceLevel = 'arceditor') 52 | 53 | def mainModule(self): 54 | ''' 55 | Main procedure 56 | Args: 57 | self: The reserved object 'self' 58 | ''' 59 | # initialization of default shortcuts - simplifies code usage 60 | Tool = self.Tool # gispython tools 61 | callGP = self.Tool.callGP # ArcPy geoprocesing tools runner 62 | pj = os.path.join # shorter syntax for path joining 63 | arcpy = self.Tool.gp # already initialized arcpy 64 | 65 | Tool.AddMessage('\\nHello World!\\n ') 66 | 67 | 68 | if __name__ == '__main__': 69 | '''default execution''' 70 | Module = MainModule() 71 | Module.DoJob() 72 | """ 73 | 74 | def CheckCreateDir(OutDirName): 75 | """Automation procedure which creates directory, in case it doesn't exist 76 | 77 | Args: 78 | self: The reserved object 'self' 79 | OutDirName: Output directory 80 | """ 81 | if not os.path.exists(OutDirName): 82 | os.makedirs(OutDirName) 83 | print('...Directory "{0}" created'.format(OutDirName)) 84 | else: 85 | print('...Directory "{0}" already exists'.format(OutDirName)) 86 | 87 | if __name__ == "__main__": 88 | '''default execution''' 89 | CheckCreateDir(r'C:\GIS') 90 | CheckCreateDir(r'C:\GIS\Log') 91 | CheckCreateDir(r'C:\GIS\GISPythonScripts') 92 | CheckCreateDir(r'C:\GIS\tmp') 93 | CheckCreateDir(r'C:\GIS\Log\Outlog') 94 | CheckCreateDir(r'C:\GIS\Log\Outlog\Archive') 95 | CheckCreateDir(r'C:\GIS\Log\ErrorLog') 96 | CheckCreateDir(r'C:\GIS\Log\ErrorLog\Archive') 97 | paramsFileName = r'C:\GIS\GISPythonScripts\SysGISParams.py' 98 | if os.path.exists(paramsFileName): 99 | print('...old params file found "{0}" ... will not be owerwriten'.format(paramsFileName)) 100 | else: 101 | with codecs.open(paramsFileName, "w", "utf-8") as prFile: 102 | prFile.write(paramsFileSource) 103 | print('...params file created "{0}"'.format(paramsFileName)) 104 | sampleFileName = r'C:\GIS\GISPythonScripts\sampleGISPythonModule.py' 105 | if os.path.exists(sampleFileName): 106 | print('...old sample file found "{0}" ... will not be owerwriten'.format(sampleFileName)) 107 | else: 108 | with codecs.open(sampleFileName, "w", "utf-8") as smplFile: 109 | smplFile.write(sampleModule) 110 | print('...sample file created "{0}"'.format(sampleFileName)) 111 | -------------------------------------------------------------------------------- /GISPython/FTPHleper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | FTP operations module 4 | """ 5 | 6 | import datetime 7 | import os 8 | import traceback 9 | import sys 10 | import MyError 11 | 12 | class FTPFile: 13 | """Class for describing the FTP file""" 14 | 15 | def __init__(self, file, date, size): 16 | """Class initialization procedure 17 | 18 | Args: 19 | self: The reserved object 'self' 20 | file: Filename 21 | date: File date 22 | size: File size 23 | """ 24 | self.file = file 25 | self.date = date 26 | self.size = size 27 | 28 | class FTPHleper: 29 | """Class for easing the FTP operations""" 30 | 31 | def __init__(self, FTPHost, FTPUser, FTPPwd, FTPDir=None): 32 | """Class initialization procedure 33 | 34 | Args: 35 | self: The reserved object 'self' 36 | FTPHost: FTP server 37 | FTPUser: FTP username 38 | FTPPwd: FTP user password 39 | """ 40 | import ftplib 41 | 42 | try: 43 | self.ftp = ftplib.FTP(FTPHost, FTPUser, FTPPwd) 44 | if FTPDir == None: 45 | self.ftp.cwd('/') 46 | else: 47 | self.ftp.cwd(FTPDir) 48 | except Exception, e: 49 | 50 | tb = sys.exc_info() 51 | txt = "Line %i" % tb[2].tb_lineno + unicode(traceback.format_exc(), errors='ignore') 52 | raise MyError.MyError(u"Error connecting to ftp: " + FTPHost + u'\n Error output: \n\n' + txt) 53 | 54 | 55 | def list_files(self): 56 | """Procedure to retrieve a file description in the specific connection directory 57 | 58 | Args: 59 | self: The reserved object 'self' 60 | """ 61 | self.files = [] 62 | self.ftp.dir(self.dir_callback) 63 | return self.files 64 | 65 | def get_file_date(self, fileName): 66 | """Determines the ftp file modification date 67 | 68 | Args: 69 | self: The reserved object 'self' 70 | fileName: Ftp file name 71 | """ 72 | modifiedTime = self.ftp.sendcmd('MDTM ' + fileName) 73 | modifiedTime = self.ftp.sendcmd('MDTM ' + fileName) 74 | return datetime.datetime.strptime(modifiedTime[4:], "%Y%m%d%H%M%S") 75 | 76 | def get_file_size(self, fileName): 77 | """Determines the ftp file size 78 | 79 | Args: 80 | self: The reserved object 'self' 81 | fileName: Ftp file name 82 | """ 83 | size = self.ftp.sendcmd('SIZE ' + fileName) 84 | size = self.ftp.sendcmd('SIZE ' + fileName) 85 | return int(size[4:]) 86 | 87 | def get_file(self, filename, savePath): 88 | """Retrieves the file from the FTP server 89 | 90 | Args: 91 | self: The reserved object 'self' 92 | filename: Ftp file name 93 | """ 94 | # Open the file for writing in binary mode 95 | print 'Opening local file ' + savePath 96 | file = open(savePath, 'wb') 97 | fSize = [0, 0] 98 | 99 | # Download the file a chunk at a time 100 | # Each chunk is sent to handleDownload 101 | # We append the chunk to the file and then print a '.' for progress 102 | # RETR is an FTP command 103 | 104 | def download_file_callback(data): 105 | """Download file callback""" 106 | file.write(data) 107 | fSize[0] += len(data) 108 | fSize[1] += len(data) 109 | if fSize[1] >= 10485760: 110 | file.flush() 111 | print 'Downloaded ' + str(fSize[0]/1048576) + ' MB' 112 | fSize[1] = 0 113 | 114 | print 'Getting ' + savePath 115 | self.ftp.retrbinary('RETR %s' % filename, download_file_callback, 4096) 116 | 117 | # Clean up time 118 | print 'Closing file ' + savePath 119 | file.close() 120 | 121 | def upload_file(self, fileName, filePath): 122 | """Uploads the binary file, using FTP 123 | 124 | Args: 125 | self: The reserved object 'self' 126 | filePath: Uploadable file local path 127 | fileName: Uploadable file name 128 | """ 129 | uploadFile = open(os.path.join(filePath, fileName), 'rb') 130 | # Store a file in binary transfer mode 131 | self.ftp.storbinary('STOR ' + fileName, uploadFile) 132 | # close file 133 | uploadFile.close() 134 | 135 | def delete_file(self, fileName): 136 | """Deletes the file from the FTP server 137 | 138 | Args: 139 | self: The reserved object 'self' 140 | fileName: Name of the file to delete 141 | """ 142 | self.ftp.delete(fileName) 143 | 144 | def dir_callback(self, line): 145 | """Processes callback from the procedure 'list_files' 146 | 147 | Args: 148 | self: The reserved object 'self' 149 | line: Row with the FTP file description 150 | """ 151 | bits = line.split() 152 | 153 | if 'd' not in bits[0]: 154 | fFile = FTPFile(bits[-1], self.get_file_date(bits[-1]), self.get_file_size(bits[-1])) 155 | self.files.append(fFile) 156 | -------------------------------------------------------------------------------- /docs/main_modules.rst: -------------------------------------------------------------------------------- 1 | Main modules 2 | ============ 3 | 4 | *GISPython* package core modules 5 | 6 | GISPythonModule 7 | --------------- 8 | 9 | *Main module, which contains frame for all GISPython package modules. Module allows the code unification, and ensures the code execution from both the ArcGIS Desktop Python console and the Command Prompt.* 10 | 11 | .. automodule:: GISPythonModule 12 | :members: 13 | :undoc-members: 14 | :show-inheritance: 15 | 16 | Examples 17 | ******** 18 | Executes the tool if it's to be executed within an another Python tool:: 19 | 20 | from GISPython import TimerHelper 21 | import OnlineCompress 22 | 23 | with TimerHelper.TimedSubprocess(Tool, u'Compress DB'): # Adds a message to the tool output 24 | with TimerHelper.TimedSubprocess(Tool, u'disconnect users from DB', 2): # Adds a message to the tool output 25 | self.Tool.RunSQL('KillUsers', Pr.u_sde, Pr.p_sde) # Runs custom SQL script 26 | OnlineCompress.MainModule(None, False).runInsideJob(Tool) # Runs SDE Compression procedure from OnlineCompress module (custom geoprocessing module) 27 | 28 | GISPythonTool 29 | ------------- 30 | 31 | *Module defines abstract classes for the ESRI Toolbox tool definition and contains functions which helps to create an ArcGIS Toolbox, validates the tool's parameter values and controls a behavior of the tool's dialog.* 32 | 33 | .. automodule:: GISPythonTool 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | 38 | Examples 39 | ******** 40 | Define parameters in ESRI Python toolbox:: 41 | 42 | from GISPython import GISPythonTool 43 | 44 | class ToolAtributeValidator(GISPythonTool.GISPythonTool): 45 | def __init__(self): 46 | """Define tool (tool name is the class name)""" 47 | self.label = u"Tool for attribute data validation (test mode)" 48 | self.description = u"Tool for attribute data validation (test mode)" 49 | self.category = 'GEO Maintenance' 50 | 51 | def getParameterInfo(self): 52 | """Define the parameters""" 53 | param_0 = arcpy.Parameter( 54 | displayName=r'Key:', 55 | name=u"key", 56 | datatype=u"String", 57 | parameterType=u"Required", 58 | direction=u"Input") 59 | 60 | ret_val = [param_0] 61 | return ret_val 62 | 63 | def execute(self, parameters, messages): 64 | """Tool execution""" 65 | AtributeValidator.MainModule(parameters[0].valueAsText).DoJob() 66 | return 67 | 68 | GISPythonToolBase 69 | ----------------- 70 | 71 | *Module contains geoprocessing tool operations and automation mechanisms for different operations.* 72 | 73 | .. automodule:: GISPythonToolBase 74 | :members: 75 | :undoc-members: 76 | :show-inheritance: 77 | 78 | Examples 79 | ******** 80 | Run a script from Shell with parameters:: 81 | 82 | Tool = self.Tool 83 | # Executes your custom process script (mainly maintenance scripts) within runShell function, 84 | # which implements _runProcess function (message output in terminal window). 85 | # Function executes script seperately from main process (Detached=True) and indicates, 86 | # that the errors doesn't have to be logged (noErr=True). 87 | Tool.runShell('SomeProcess.py', Detached = True, noErr = True) 88 | time.sleep(10) # after 10 seconds launch another runShell process 89 | 90 | Executes a custom SQL script file (only Oracle sqlplus supported):: 91 | 92 | from GISPython import TimerHelper 93 | 94 | Tool = self.Tool 95 | # Executes process from SQL file 96 | with TimerHelper.TimedSubprocess(self.Tool, u'datu atlasi no nogabaliem'): # Adds a message to the tool output 97 | Tool.RunSQL('LoadSomeDataFromTable') # Runs SQL file within RunSQL function, which implements GetSQL function (gets the SQL file location) 98 | 99 | Duplicates path seperator symbol '\' for external execution compatibility:: 100 | 101 | from GISPython import SimpleFileOps 102 | from GISPython import TimerHelper 103 | 104 | Tool = self.Tool 105 | # Your code 106 | with TimerHelper.TimedSubprocess(Tool, u'prepare environment'): # Adds a message to the tool output 107 | DirHelper = SimpleFileOps.SimpleFileOps(Tool) # Set variable for SimpleFileOps module functions 108 | tmpFolder = os.path.join(Pr.TmpFolder, "Soil") # Set tmp folder path 109 | # Your code 110 | Tool.AddMessage(u'\n ...delete previous tmp data') # Add tool output message 111 | DirHelper.CheckCreateDir(Tool.CorrectStr(tmpFolder)) # Check/create tmp directory (with modified path seperators) 112 | DirHelper.ClearDir(Tool.CorrectStr(tmpFolder)) # Clear tmp directory (with modified path seperators) 113 | 114 | SysGISTools 115 | ----------- 116 | 117 | *Base module which contains GISPython scripting framework, geoprocessing message delivery, logging and error processing. Inherits GISPythonToolBase.* 118 | 119 | .. automodule:: SysGISTools 120 | :members: 121 | :undoc-members: 122 | :show-inheritance: 123 | 124 | SysGISToolsSysParams 125 | -------------------- 126 | 127 | .. automodule:: SysGISToolsSysParams 128 | :members: 129 | :undoc-members: 130 | :show-inheritance: 131 | 132 | SysTools_unittest 133 | ----------------- 134 | 135 | .. automodule:: SysTools_unittest 136 | :members: 137 | :undoc-members: 138 | :show-inheritance: 139 | 140 | .. toctree:: 141 | :maxdepth: 4 -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # GISPython documentation build configuration file, created by 4 | # sphinx-quickstart on Sat Jun 03 14:40:34 2017. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | # 19 | import os 20 | import sys 21 | import sphinx_rtd_theme 22 | 23 | sys.path.insert(0, os.path.abspath('../GISPython')) 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ['sphinx.ext.autodoc', 35 | 'sphinx.ext.githubpages', 36 | 'sphinxcontrib.napoleon'] 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ['_templates'] 40 | 41 | # The suffix(es) of source filenames. 42 | # You can specify multiple suffix as a list of string: 43 | # 44 | # source_suffix = ['.rst', '.md'] 45 | source_suffix = '.rst' 46 | 47 | # The master toctree document. 48 | master_doc = 'index' 49 | 50 | # General information about the project. 51 | project = u'GISPython' 52 | copyright = u'2017, LVM GEO' 53 | author = u'LVM GEO' 54 | 55 | # The version info for the project you're documenting, acts as replacement for 56 | # |version| and |release|, also used in various other places throughout the 57 | # built documents. 58 | # 59 | # The short X.Y version. 60 | version = '1.41.1' 61 | # The full version, including alpha/beta/rc tags. 62 | release = '1.41.1' 63 | 64 | # The language for content autogenerated by Sphinx. Refer to documentation 65 | # for a list of supported languages. 66 | # 67 | # This is also used if you do content translation via gettext catalogs. 68 | # Usually you set "language" from the command line for these cases. 69 | language = None 70 | 71 | # List of patterns, relative to source directory, that match files and 72 | # directories to ignore when looking for source files. 73 | # This patterns also effect to html_static_path and html_extra_path 74 | exclude_patterns = [] 75 | 76 | # The name of the Pygments (syntax highlighting) style to use. 77 | pygments_style = 'sphinx' 78 | 79 | # If true, `todo` and `todoList` produce output, else they produce nothing. 80 | todo_include_todos = False 81 | 82 | # -- Options for HTML output ---------------------------------------------- 83 | 84 | # The theme to use for HTML and HTML Help pages. See the documentation for 85 | # a list of builtin themes. 86 | # 87 | html_theme = 'sphinx_rtd_theme' 88 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 89 | 90 | # Theme options are theme-specific and customize the look and feel of a theme 91 | # further. For a list of options available for each theme, see the 92 | # documentation. 93 | # 94 | html_theme_options = {} 95 | 96 | # Add any paths that contain custom static files (such as style sheets) here, 97 | # relative to this directory. They are copied after the builtin static files, 98 | # so a file named "default.css" will overwrite the builtin "default.css". 99 | html_static_path = ['_static'] 100 | 101 | 102 | # -- Options for HTMLHelp output ------------------------------------------ 103 | 104 | # Output file base name for HTML help builder. 105 | htmlhelp_basename = 'GISPythondoc' 106 | 107 | 108 | # -- Options for LaTeX output --------------------------------------------- 109 | 110 | latex_elements = { 111 | # The paper size ('letterpaper' or 'a4paper'). 112 | # 113 | # 'papersize': 'letterpaper', 114 | 115 | # The font size ('10pt', '11pt' or '12pt'). 116 | # 117 | # 'pointsize': '10pt', 118 | 119 | # Additional stuff for the LaTeX preamble. 120 | # 121 | # 'preamble': '', 122 | 123 | # Latex figure (float) alignment 124 | # 125 | # 'figure_align': 'htbp', 126 | } 127 | 128 | # Grouping the document tree into LaTeX files. List of tuples 129 | # (source start file, target name, title, 130 | # author, documentclass [howto, manual, or own class]). 131 | latex_documents = [ 132 | (master_doc, 'GISPython.tex', u'GISPython Documentation', 133 | u'lvmgeo', 'manual'), 134 | ] 135 | 136 | 137 | # -- Options for manual page output --------------------------------------- 138 | 139 | # One entry per manual page. List of tuples 140 | # (source start file, name, description, authors, manual section). 141 | man_pages = [ 142 | (master_doc, 'gispython', u'GISPython Documentation', 143 | [author], 1) 144 | ] 145 | 146 | 147 | # -- Options for Texinfo output ------------------------------------------- 148 | 149 | # Grouping the document tree into Texinfo files. List of tuples 150 | # (source start file, target name, title, author, 151 | # dir menu entry, description, category) 152 | texinfo_documents = [ 153 | (master_doc, 'GISPython', u'GISPython Documentation', 154 | author, 'GISPython', 'One line description of project.', 155 | 'Miscellaneous'), 156 | ] 157 | 158 | # Napoleon settings 159 | napoleon_google_docstring = True 160 | napoleon_numpy_docstring = True 161 | napoleon_include_init_with_doc = False 162 | napoleon_include_private_with_doc = True 163 | napoleon_include_special_with_doc = True 164 | napoleon_use_admonition_for_examples = False 165 | napoleon_use_admonition_for_notes = False 166 | napoleon_use_admonition_for_references = False 167 | napoleon_use_ivar = False 168 | napoleon_use_param = True 169 | napoleon_use_rtype = True 170 | napoleon_use_keyword = True 171 | -------------------------------------------------------------------------------- /GISPython/SimpleFileOpsSafe.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | File and filesystem operations module 4 | """ 5 | import os 6 | import shutil 7 | from datetime import date, datetime, timedelta 8 | import stat 9 | import fnmatch 10 | import SimpleFileOps 11 | 12 | class SimpleFileOpsSafe(SimpleFileOps.SimpleFileOps): 13 | """Class for easing the most typical file and filesystem operations""" 14 | def __init__(self, _Tool): 15 | """Class initialization procedure 16 | 17 | Args: 18 | self: The reserved object 'self' 19 | Tool: The 'GISTools10' object for data processing 20 | """ 21 | super(SimpleFileOpsSafe, self).__init__(_Tool) 22 | 23 | def ClearDir(self, DirName, searchPatern = '*'): # Overriding 24 | """Directory cleaning automation procedure 25 | 26 | Args: 27 | self: The reserved object 'self' 28 | DirName: The directory to be cleaned 29 | """ 30 | self.Tool.AddMessage(u"\n----------- Cleaning directory [" + DirName + "] -------" + self.Tool.MyNow()) 31 | self._clear_dir(DirName, searchPatern) 32 | self.Tool.AddMessage(u" ... directory [" + DirName + u"] cleaned ") 33 | 34 | def DelClearDir(self, DirName): # Overriding 35 | """Delete non-empty directory 36 | 37 | Args: 38 | self: The reserved object 'self' 39 | DirName: The directory to be deleted 40 | """ 41 | self.Tool.AddMessage(u"\n----------- Cleaning and deleting directory [" + DirName + "] -------" + self.Tool.MyNow()) 42 | self._clear_dir(DirName) 43 | os.rmdir(DirName) 44 | self.Tool.AddMessage(u" ... directory [" + DirName + u"] deleted ") 45 | 46 | 47 | def delFileIfExists(self, fileName): # Overriding 48 | """"Deletes file if file exists 49 | 50 | Args: 51 | self: The reserved object 'self' 52 | DirName: The directory to be cleaned 53 | 54 | """ 55 | if os.path.exists(fileName): 56 | self._force_remove_file_or_symlink(fileName) 57 | 58 | def BackupFiles(self, InDirName, OutDirName, D=0, Ext='*'): # Overriding 59 | """File archiving automation procedure 60 | 61 | Args: 62 | self: The reserved object 'self' 63 | InDirName: Input directory 64 | OutDirName: Output directory 65 | D: How old files to archive (number of days) 66 | Ext: The extension to search for ('*' - search any file) 67 | """ 68 | self.Tool.AddMessage('Executing file, newer than ' + str(D) + ' days, archiving from [' + InDirName + '] to [' + OutDirName + ']') 69 | d = date.today() - timedelta(days=D) 70 | d = datetime(year=d.year, month=d.month, day=d.day) 71 | for file_path in self.FindFileByDate(InDirName, Ext, d, 'Old'): 72 | self.BackupOneFile(file_path, OutDirName) 73 | 74 | def BackupOneFile(self, InFileName, OutDirName): # Overriding 75 | """Specific file archiving automation procedure 76 | 77 | Args: 78 | self: The reserved object 'self' 79 | InFileName: Input file 80 | OutDirName: Output directory 81 | """ 82 | self.Tool.AddMessage("--- Archiving file [" + InFileName + "] to [" + OutDirName + "] - " + self.Tool.MyNow()) 83 | if os.path.exists(InFileName): 84 | shutil.copy(InFileName, OutDirName + '\\' + os.path.split(InFileName)[1].split(".")[-2] + "_" + self.Tool.MyNowFile() + os.path.splitext(InFileName)[1]) 85 | self._force_remove_file_or_symlink(InFileName) 86 | else: 87 | self.Tool.AddMessage(" File " + InFileName + " does not exist ...") 88 | 89 | # http://stackoverflow.com/questions/1889597/deleting-directory-in-python 90 | def _remove_readonly(self, fn, path_, excinfo): 91 | """Remove read-only files and directories 92 | 93 | Args: 94 | self: The reserved object 'self' 95 | fn: Function for removing either a directory or a file 96 | path_: Path to the directory/file 97 | """ 98 | if fn is os.rmdir: 99 | os.chmod(path_, stat.S_IWRITE) 100 | os.rmdir(path_) 101 | elif fn is os.remove: 102 | os.chmod(path_, stat.S_IWRITE) 103 | os.remove(path_) 104 | 105 | 106 | def _force_remove_file_or_symlink(self, path_): 107 | """Force remove files and symlinks 108 | 109 | Args: 110 | self: The reserved object 'self' 111 | path_: Path to the file/symlink 112 | """ 113 | try: 114 | os.remove(path_) 115 | except OSError: 116 | os.chmod(path_, stat.S_IWRITE) 117 | os.remove(path_) 118 | 119 | 120 | def _is_regular_dir(self, path_): 121 | """Check if directory is regular directory 122 | 123 | Args: 124 | self: The reserved object 'self' 125 | path_: Path to the directory 126 | """ 127 | try: 128 | mode = os.lstat(path_).st_mode 129 | except os.error: 130 | mode = 0 131 | return stat.S_ISDIR(mode) 132 | 133 | 134 | def _clear_dir(self, path_, pattern='*'): 135 | """Clear directory contents 136 | 137 | Args: 138 | self: The reserved object 'self' 139 | path_: Path to the directory 140 | pattern: Check if the file matches the given pattern (Default = '*'(Matches everything)) 141 | """ 142 | if self._is_regular_dir(path_): 143 | # Given path is a directory, clear its contents 144 | for name in [filename for filename in os.listdir(path_) if fnmatch.fnmatch(filename, pattern)]: 145 | fullpath = os.path.join(path_, name) 146 | if self._is_regular_dir(fullpath): 147 | shutil.rmtree(fullpath, onerror=self._remove_readonly) 148 | else: 149 | self._force_remove_file_or_symlink(fullpath) 150 | else: 151 | # Given path is a file or a symlink. 152 | # Raise an exception here to avoid accidentally clearing the contents 153 | # of a symbolic linked directory. 154 | raise OSError("Cannot call clear_dir() on a symbolic link") 155 | -------------------------------------------------------------------------------- /GISPython/GDPSyncroniserHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Data synchronization module (synchronizes data between tables) 4 | """ 5 | 6 | class GDPSyncroniserHelper(object): 7 | """ESRI table synchronization class""" 8 | 9 | def __init__(self, gp, Tool=None): 10 | """Class initialization operation 11 | 12 | Args: 13 | self: The reserved object 'self' 14 | gp: ArcPy GP object 15 | """ 16 | self.gp = gp 17 | if Tool != None: 18 | self.isTool = True 19 | else: 20 | self.isTool = False 21 | self.Tool = Tool 22 | 23 | def AddWarning(self, str): 24 | """Wrapper for the 'AddWarning' procedure 25 | 26 | Args: 27 | self: The reserved object 'self' 28 | str: Output string 29 | """ 30 | if self.isTool: 31 | self.Tool.AddWarning(str) 32 | else: 33 | self.gp.AddWarning(str) 34 | 35 | def AddMessage(self, str): 36 | """Wrapper for the 'AddMessage' procedure 37 | 38 | Args: 39 | self: The reserved object 'self' 40 | str: Output string 41 | """ 42 | if self.isTool: 43 | self.Tool.AddMessage(str) 44 | else: 45 | self.gp.AddMessage(str) 46 | 47 | def DoSync(self, definition, workspace=None, startEditing=False, startOperation=False, with_undo=False, multiuser=False): 48 | """Data synchronization procedure 49 | 50 | Args: 51 | self: The reserved object 'self' 52 | definition: 'SyncDefinition' object which describes the synchronization parameters 53 | workspace: DB in which to start data editing 54 | startEditing: Start the edit session in the DB specified in 'workspace' parameter? (Default = False) 55 | startOperation: Start the edit operation in the DB specified in 'workspace' parameter? (Default = False) 56 | with_undo: Sets whether the undo and redo stacks are enabled or disabled for an edit session. (Default = False) 57 | multiuser: Sets whether a DB contains a nonversioned, or versioned dataset. (Default = False) 58 | 59 | Returns: 60 | * output - Returns the report about the process execution 61 | * outputErrors - Returns the error description 62 | * errIDs - list of IDs that had an error 63 | * SyncIDs - list of IDs that was syncronized 64 | """ 65 | outputErrors = u"" 66 | output = u"" 67 | errIDs = [] 68 | SyncIDs = [] 69 | 70 | if definition.inTableQuery != None: 71 | hasInQuery = True 72 | else: 73 | hasInQuery = False 74 | 75 | inFields = () 76 | inFields = (definition.inTableJoinField, ) + definition.inTableFields 77 | outFields = () 78 | outFields = (definition.outTableJoinField, ) + definition.outTableFields 79 | 80 | self.AddMessage(u'>>>>Start the synchronization of the tables {0} and {1}'.format(definition.inTable, definition.outTable)) 81 | self.AddMessage(u'>>>>>>>>Join {0}.{1}={2}.{3}'.format(definition.inTable, definition.inTableJoinField, definition.outTable, definition.outTableJoinField)) 82 | self.AddMessage(u'>>>>>>>>Input fields: [{0}]'.format(", ".join(definition.inTableFields))) 83 | if hasInQuery: 84 | self.AddMessage(u'>>>>>>>>Input query: [{0}]'.format(definition.inTableQuery)) 85 | self.AddMessage(u'>>>>>>>>Output fields: [{0}]'.format(", ".join(definition.outTableFields))) 86 | 87 | if startEditing or startOperation: 88 | edit = self.gp.da.Editor(workspace) 89 | if startEditing: 90 | edit.startEditing(with_undo, multiuser) 91 | if startOperation: 92 | edit.startOperation() 93 | 94 | with self.gp.da.SearchCursor(definition.inTable, inFields, definition.inTableQuery) as cur: 95 | i = 0 96 | j = 0 97 | k = 0 98 | err = 0 99 | for row in cur: 100 | output = self.__DoSyncRow(row, definition.outTable, outFields, definition.outTableJoinField, definition.messageDefinition, definition.idvalueseparator, definition.createNew) 101 | if type(output) is int: 102 | i = i + output 103 | SyncIDs.append(row[0]) 104 | else: 105 | if self.isTool: 106 | self.AddMessage(u' ...{0}'.format(output)) 107 | outputErrors = outputErrors + u'{0}\n'.format(output) 108 | err = err + 1 109 | errIDs.append(row[0]) 110 | j = j + 1 111 | k = k + 1 112 | 113 | if k >= 1000: 114 | if self.isTool: 115 | self.AddMessage(u'>>>> ...Processed [{0}], stored [{1}], faulty [{2}]'.format(j, i, err)) 116 | k = 0 117 | if startOperation: 118 | edit.stopOperation() 119 | if startOperation: 120 | edit.startOperation() 121 | 122 | if startOperation: 123 | edit.stopOperation() 124 | if startEditing: 125 | edit.stopEditing(True) 126 | 127 | if self.isTool: 128 | self.AddMessage(u'>>>> ...Processed [{0}], stored [{1}], faulty [{2}]'.format(j, i, err)) 129 | output = u'Processed [{0}], stored [{1}], faulty [{2}]'.format(j, i, err) 130 | 131 | return output, outputErrors, errIDs, SyncIDs 132 | 133 | def __DoSyncRow(self, inRow, outTable, outTableFields, outTableJoinField, messageString, idvalueseparator, createNew): 134 | """Procedure performs row synchronization 135 | 136 | Args: 137 | self: The reserved object 'self' 138 | inRow: Row to synchronize 139 | outTable: Output table 140 | outTableFields: Output table fields 141 | outTableJoinField: Output table join field 142 | messageString: Output message formatting 143 | createNew: Create new record if needed 144 | 145 | Returns: 146 | * output - 0, If no changes were necessary; 1, If there were any changes 147 | * Error description (in case there were errors) 148 | """ 149 | if inRow[0] == None: 150 | return u"Error: join field is empty " + messageString.format(*inRow) 151 | 152 | with self.gp.da.UpdateCursor(outTable, outTableFields, u'{0} = {2}{1}{2}'.format(outTableJoinField, inRow[0], idvalueseparator)) as cur: 153 | output = 0 154 | i = 0 155 | foundRow = False 156 | for row in cur: 157 | if i == 0: 158 | foundRow = True 159 | output, row = self.__DoSyncFields(inRow, row) 160 | if output == 1: 161 | cur.updateRow(row) 162 | i = i+1 163 | else: 164 | return u"Error: row in the results table is not unique" + messageString.format(*inRow) 165 | 166 | if not foundRow: 167 | if createNew: 168 | with self.gp.da.InsertCursor(outTable, outTableFields) as insCur: 169 | newRow = [None for _ in range(len(inRow))] 170 | output, newRow = self.__DoSyncFields(inRow, newRow) 171 | insCur.insertRow(newRow) 172 | else: 173 | return u"Error: no related records found in the results table " + messageString.format(*inRow) 174 | 175 | return output 176 | 177 | def __DoSyncFields(self, inRow, outRow): 178 | """Procedure performs the field synchronization 179 | 180 | Args: 181 | self: The reserved object 'self' 182 | inRow: Row to synchronize 183 | outRow: Row to which synchronize 184 | 185 | Returns: 186 | * output - 0, If no changes were necessary; 1, If there were any changes 187 | * outRow - Altered row 188 | """ 189 | output = 0 190 | for x in range(0, len(inRow)): 191 | if inRow[x] != outRow[x]: 192 | outRow[x] = inRow[x] 193 | output = 1 194 | return output, outRow 195 | 196 | class SyncDefinition(object): 197 | """Synchronization definition description class""" 198 | 199 | def __init__(self): 200 | """Class initialization procedure 201 | 202 | Args: 203 | self: The reserved object 'self' 204 | """ 205 | self.inTable = "" 206 | self.inTableFields = () 207 | self.inTableQuery = None 208 | self.inTableJoinField = "" 209 | self.outTable = "" 210 | self.outTableFields = () 211 | self.outTableJoinField = "" 212 | self.messageDefinition = "" 213 | self.idvalueseparator = "" 214 | self.createNew = False 215 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Changelog 3 | ========= 4 | v1.60.2 (2024.08.13) 5 | -------------------- 6 | * publish changes 7 | 8 | v1.60.1 (2024.02.11) 9 | -------------------- 10 | * Upgraded GDPSyncroniserHelper2 for better data change detection and geometry compare 11 | 12 | v1.59.2 (2023.06.02) 13 | -------------------- 14 | * Bug fixes for MailHelper and AGSHelper 15 | 16 | v1.59.1 (2023.03.23) 17 | -------------------- 18 | * MailHelper - added ability to send e-mails in HTML format 19 | 20 | v1.58.3 (2022.11.06) 21 | -------------------- 22 | * GISPythonToolBase - added ability to run Detached oracle sqlplus scripts (RunSQL) 23 | 24 | v1.58.1 and (fix) v1.58.2 (2022.08.29) 25 | -------------------- 26 | * AGServerHelperNTLM - added function cleanServicePermisions to remove rights from services 27 | 28 | v1.57.1 (2022.07.15) 29 | -------------------- 30 | * SFTPHelper - added list_files ability 31 | * SFTPHelper - added delete_file ability 32 | 33 | v1.56.1 (2022.04.21) 34 | -------------------- 35 | * MailHelper - added ability to attach files to GISPythonMailHelper 36 | * GISPythonModule - added ability to get log files in error and status e-mails of platform 37 | 38 | v1.55.6 (2022.03.19) 39 | -------------------- 40 | * MXDHelper - fix of bug, in error reporting 41 | 42 | v1.55.5 (2022.03.17) 43 | -------------------- 44 | * GDPSyncroniserHelper2 - fix of bug, when Shape fields wuld not sync if changes are minor 45 | 46 | v1.55.4 (2022.03.16) 47 | -------------------- 48 | * AGServerHelperNTLM - minor bug fixes 49 | 50 | v1.55.3 (2022.02.21) 51 | -------------------- 52 | * GDBHelper - CalculateXY bug fixes 53 | 54 | v1.55.2 (2022.01.23) 55 | -------------------- 56 | * GDBHelper - GetRelations added ability to add only child tables 57 | * minor documentation fixes 58 | * minor bug fixes 59 | 60 | v1.55.1 (2022.01.22) 61 | -------------------- 62 | * combined functionality of AGServerHelperNTLM and AGServerHelperToken - now AGServerHelperNTLM supports token authentication 63 | * AGServerHelperNTLM - added function for publishing MXD files 64 | 65 | v1.54.3 (2021.08.25) 66 | -------------------- 67 | * fix errors in GISPythonToolBase.py 68 | 69 | v1.54.2 (2021.08.10) 70 | -------------------- 71 | * fix errors in AGServerHelperToken and AGServerHelperNTLM 72 | 73 | v1.54.1 (2021.07.13) 74 | -------------------- 75 | * Added new Module AGServerHelperToken - with replaces AGSServerHelper and supports all the methods avalable in AGServerHelperNTLM. Module is used for AGS management with tocken autentification 76 | 77 | v1.53.2 (2021.03.26) 78 | -------------------- 79 | * fix deploy error in PublisherHealper 80 | 81 | v1.53.1 (2021.03.11) 82 | -------------------- 83 | * Optimization of AGServerHelperNTLM 84 | 85 | v1.52.1 (2021.02.12) 86 | -------------------- 87 | * added xmlParamsHelper ability to provide namespaces if searching by XPath 88 | 89 | v1.51.1 (2021.02.11) 90 | -------------------- 91 | * fix errors in PublisherHealper for publishing whole folders with recursive option 92 | 93 | v1.50.2 (2020.11.25) 94 | -------------------- 95 | * fix errors in PublisherHealper 96 | 97 | v1.50.1 (2020.10.08) 98 | -------------------- 99 | * changes to GDBHelper - added ability to get all field names for table 100 | 101 | v1.49.0 (2020.10.08) 102 | -------------------- 103 | * changes to PublisherHealper - added ability to backup only changed files 104 | 105 | v1.48.0 (2020.09.17) 106 | -------------------- 107 | * changes to e-mail notification subcet to be more esaly parsable. 108 | * fix: error with PublisherHealper when publishing to wast directory tree 109 | 110 | v1.47.0 (2020.08.18) 111 | -------------------- 112 | * Added ability for Zip files to append data to existing file 113 | 114 | v1.46.3 (2020.08.12) 115 | -------------------- 116 | * Fixed problems with backup function in SimpleFileOpsSafe 117 | 118 | v1.46.2 (2020.08.09) 119 | -------------------- 120 | * Fixed problems when PublisherHealper not processing json config files 121 | 122 | v1.46.1 (2020.05.12) 123 | -------------------- 124 | * SFTPHelper added ability to autorise SFTP connection with RSA private key file 125 | 126 | v1.45.3 (2020.05.05) 127 | -------------------- 128 | * bug fixes for AGServerHelperNTLM 129 | 130 | v1.45.2 (2020.04.28) 131 | -------------------- 132 | * aditional functionality for PublisherHealper 133 | 134 | v1.45.1 (2019.15.12) 135 | -------------------- 136 | 137 | * refactoring and aditional functionality for AGServerHelperNTLM. New functionality for working with permisions, roles and users 138 | * refactoring and aditional functionality for PublisherHealper 139 | 140 | v1.44.1 (2019.09.11) 141 | -------------------- 142 | 143 | * Mor functionality for PublisherHealper module and coresponding functionality for JsonParamsHelper and xmlParamsHealper modules to support automated soulution publishing. New functionality further develops capabilities of automatic configuration file changing in CI/CD solutions. 144 | 145 | v1.43.3 (2019.08.16) 146 | -------------------- 147 | 148 | * MXDHelper module bug fixes 149 | 150 | v1.43.2 (2019.07.26) 151 | -------------------- 152 | 153 | * PublisherHealper module bug fixes 154 | 155 | v1.43.1 (2019.07.19) 156 | -------------------- 157 | 158 | * Some code cleanup and bug fixes 159 | * PublisherHealper module has added functionality for updating Json parameter files 160 | 161 | v1.42.1 (2019.05.24) 162 | -------------------- 163 | 164 | * Added GISPythonToolBase module for geoprocessing tool operations and automation mechanisms for different operations 165 | * Added PublisherHealper module for deployment operations and xmlParamsHealper for XML parameter file procedures 166 | * AGServerHelperNTLM - added method for ArcGIS dataset name retrieval 167 | * GISPythonModule - additional functionality for tool licence level check 168 | * JsonParamsHelper - additional functionality for value update 169 | * SysGISTools - restructured, multiple methods migrated to new GISPythonToolBase module 170 | * SysTools_unittest - added encoding parameters 171 | * Bug fix for TimerHelper 172 | 173 | v1.41.1 (2019.01.03) 174 | -------------------- 175 | 176 | * GDPSyncroniserHelper2 - added functionality for synchronising geodtabase tables with Python list objects 177 | 178 | v1.40.2 (2018.11.24) 179 | -------------------- 180 | 181 | * AGServerHelperNTLM.py added function to get rights group names for AGS services 182 | * Added capabilities for SimpleFileOps and SimpleFileOpsSafe to CheckCreateClearDir - check if dir exist, creates it and clears in one function 183 | * Added aditional support for outputting unicode text in script output 184 | 185 | v1.40.1 (2018.09.26) 186 | -------------------- 187 | 188 | * AGServerHelperNTLM.py added support for self generated SSL sites (Unverified SSL cases) 189 | * Added capabilities for GDBHelper.py 190 | * Bug fixes and added capabilities for GDPSyncroniserHelper2.py 191 | 192 | v1.39.2 (2018.08.15) 193 | -------------------- 194 | 195 | * Added aditional possibilities for SimpleFileOps.py 196 | 197 | v1.39.1 (2018.07.29) 198 | -------------------- 199 | 200 | * Added aditional possibilities for AGServerHelperNTLM.py 201 | 202 | v1.38.1 (2018.07.20) 203 | -------------------- 204 | 205 | * Bug fixes for GDPSyncroniserHelper2.py 206 | * Added parameter EnvironmentName. Used for Error Email generation, to indicate Environment in with error ocured. 207 | * Added SetupDefaultEnvironment.py module for fast environment setup. 208 | 209 | v1.37.2 (2018.03.29) 210 | -------------------- 211 | 212 | * Bug fixes for AGServerHelperNTLM 213 | 214 | v1.37.1 (2018.03.29) 215 | -------------------- 216 | 217 | * Bug fixes for ZipHelper and GISTools10 218 | 219 | v1.36.2 (2018.03.04) 220 | -------------------- 221 | 222 | * Added additional capabilities for GDBHelper 223 | 224 | v1.36.1 (2018.02.25) 225 | -------------------- 226 | 227 | * Added additional capabilities in shell command running 228 | * Bug fixes in SimpleFileOps file locking functionality 229 | * Added possibility to store time in date time parametrs in Json parameter file 230 | 231 | v1.35.2 (2018.01.13) 232 | -------------------- 233 | 234 | * Additional functionality added to SysGISTools module: 235 | - As there is many performance problems with *arcpy* geoprocessing script history logging, GISPython sets by default for *arcpy* not to log GP history. You can change this behavior by setting ``SetLogHistory = True`` in parameters file 236 | - GISPython Shell command launcher and SQL command launcher has added capabilities to hide passwords that they do not apear in output (logfiles and on screen) 237 | 238 | v1.35.1 (2017.11.09) 239 | -------------------- 240 | 241 | * Added GDPSyncroniserHelper2 module 242 | * Changes and bug fixes in: 243 | - AGServerHelperNTLM 244 | - SimpleFileOps 245 | - SysGISTools 246 | - GISPythonModule 247 | - GDPSyncroniserHelper 248 | * Added functionality in GDBHelper 249 | 250 | v1.34.3 (2017.07.10) 251 | -------------------- 252 | 253 | * Aditional functionality helping of reprinting MyError objects 254 | 255 | v1.34.2 (2017.06.19) 256 | -------------------- 257 | 258 | * Bug fixes in unicode output management 259 | * Added functionality in SimpleFileOps and SimpleFileOpsSafe 260 | 261 | v1.34.1 (2017.06.09) 262 | -------------------- 263 | 264 | * Released documentation on Read The Docs 265 | * Renamed AGSServerHelpaer module and class to AGSServerHelper 266 | * Added AGSServerHelperNTLM module 267 | * Added additional package dependencies (python-ntlm, patool) 268 | * More module examples 269 | * Modified README.md 270 | 271 | v1.33.1 (2017.06.05) 272 | -------------------- 273 | 274 | * Initial release 275 | 276 | .. toctree:: 277 | :maxdepth: 4 278 | -------------------------------------------------------------------------------- /GISPython/MXDHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | MXD hepler functions 4 | """ 5 | 6 | import TimerHelper 7 | import GDBHelper 8 | import os, sys, traceback 9 | 10 | class MXDHelper: 11 | """Class for easing the MXD file operations""" 12 | 13 | def __init__(self, gp=None, Tool=None): 14 | """Class initialization procedure 15 | Args: 16 | self: The reserved object 'self' 17 | gp: ArcPy GP object 18 | Tool: Reference to the GISTools10 object 19 | """ 20 | if Tool == None: 21 | self.gp = gp 22 | self.isTool = False 23 | else: 24 | self.isTool = True 25 | self.gp = Tool.gp 26 | self.Tool = Tool 27 | 28 | def AddWarning(self, str): 29 | """Helper function to unify use of AddWarning with GISPython Tool or without""" 30 | if self.isTool: 31 | self.Tool.AddWarning(str) 32 | else: 33 | self.gp.AddWarning(str) 34 | 35 | def AddMessage(self, str): 36 | """Helper function to unify use of AddMessage with GISPython Tool or without""" 37 | if self.isTool: 38 | self.Tool.AddMessage(str) 39 | else: 40 | self.gp.AddMessage(str) 41 | 42 | def OutputMessages(self, ErrorSeverity=2): 43 | """Helper function to unify use of OutputMessages with GISPython Tool or without""" 44 | if self.isTool: 45 | self.Tool.OutputMessages(str) 46 | else: 47 | maxSeverity = self.gp.GetMaxSeverity() 48 | if maxSeverity > 1: 49 | self.gp.AddMessage('') 50 | self.gp.AddMessage('--------------------------------------------') 51 | for i in range(0, self.gp.GetMessageCount()): 52 | self.gp.AddReturnMessage(i) 53 | self.gp.AddMessage('--------------------------------------------') 54 | 55 | def ChangeMXDSource(self, mxd, conn, difOutDir = False, outDir = "#", silent=False): 56 | """Changes MXD file connection and saves the file in user specified directory 57 | Args: 58 | self: The reserved object 'self' 59 | mxd: path to MXD, 60 | conn: new connection, 61 | difOutDir: is the output dir diferent from source (do the MXD be saved in thesame path ('TRUE')) 62 | outDir: directory to with to copy new MXD, 63 | silent: does the messages will be supressed""" 64 | 65 | T = TimerHelper.TimerHelper() 66 | if not silent == True: 67 | self.AddMessage(u' Processing document [{0}]'.format(mxd)) 68 | if not self.gp.Exists(mxd): 69 | self.AddWarning(u'MXD with path {0} has not been found'.format(mxd)) 70 | else: 71 | if not silent == True: 72 | self.AddMessage(u' Document found') 73 | T.TimerReset() 74 | try: 75 | Doc = self.gp.mapping.MapDocument(mxd) 76 | if not silent == True: 77 | self.AddMessage(u' Document opened - {0}'.format(T.GetTimeReset())) 78 | try: 79 | Doc.replaceWorkspaces('', 'NONE', conn, 'SDE_WORKSPACE', True) 80 | #Doc.findAndReplaceWorkspacePaths("", conn, True) 81 | if not silent == True: 82 | self.AddMessage(u' Document sources changed - {0}'.format(T.GetTimeReset())) 83 | try: 84 | if difOutDir == True: 85 | sourceDir = os.path.basename(mxd) 86 | outPath = mxd.replace(sourceDir, outDir) 87 | outDir = os.path.dirname(outPath) 88 | if not os.path.exists(outDir): 89 | os.makedirs(outDir) 90 | Doc.saveACopy(outPath) 91 | else: 92 | Doc.save() 93 | del Doc 94 | if not silent == True: 95 | self.AddMessage(u' Document saved - {0}'.format(T.GetTimeReset())) 96 | except: 97 | self.AddWarning(u'MXD with path {0} cannot be saved'.format(mxd)) 98 | tb = sys.exc_info() 99 | self.AddWarning("Line %i" % tb[2].tb_lineno) 100 | self.AddWarning(traceback.format_exc()) 101 | except: 102 | self.AddWarning(u'MXD sources with path {0} cannot be replaced'.format(mxd)) 103 | tb = sys.exc_info() 104 | self.AddWarning("Line %i" % tb[2].tb_lineno) 105 | self.AddWarning(traceback.format_exc()) 106 | except: 107 | self.AddWarning(u'MXD with path {0} cannot be opened'.format(mxd)) 108 | tb = sys.exc_info() 109 | self.AddWarning("Line %i" % tb[2].tb_lineno) 110 | self.AddWarning(traceback.format_exc()) 111 | 112 | def AddRealatedTablesToMXD(self, MXD, silent=False): 113 | """ 114 | Procedure adds all mising related tables to MXD 115 | Args: 116 | self: The reserved object 'self' 117 | mxd: path to MXD, 118 | silent: does the messages will be supressed 119 | """ 120 | 121 | self.GDBTools = GDBHelper.GDBHelper(self.gp) 122 | T = TimerHelper.TimerHelper() 123 | if not silent: 124 | self.AddMessage(u' Processing document [{0}]'.format(MXD)) 125 | if not self.gp.Exists(MXD): 126 | if not silent: 127 | self.AddWarning(u'MXD with path {0} has not been found'.format(MXD)) 128 | else: 129 | if not silent: 130 | self.AddMessage(u' Document found') 131 | T.TimerReset() 132 | try: 133 | Doc = self.gp.mapping.MapDocument(MXD) 134 | if not silent: 135 | self.AddMessage(u' Document opened - {0}'.format(T.GetTimeReset())) 136 | try: 137 | Relations = list() 138 | for lyr in self.gp.mapping.ListLayers(Doc): 139 | self.__ProcessLyr(lyr,' ', Relations, silent = silent) 140 | Relations = set(Relations) 141 | 142 | if not silent: 143 | self.AddMessage('\n ' + u'Found tables:') 144 | for tabl in Relations: 145 | self.AddMessage(' > ' + tabl) 146 | 147 | remList = list() 148 | for tab in self.gp.mapping.ListTableViews(Doc): 149 | for rel in Relations: 150 | if tab.datasetName.lower() == rel.lower().split('\\')[-1]: 151 | remList.append(rel) 152 | Relations = [x for x in Relations if x not in remList] 153 | 154 | if not silent == True: 155 | self.AddMessage('\n ' + u'Tables missing from MXD:') 156 | for tabl in Relations: 157 | self.AddMessage(' > ' + tabl) 158 | 159 | for rel in Relations: 160 | addTable = self.gp.mapping.TableView(rel) 161 | self.gp.mapping.AddTableView(Doc.activeDataFrame, addTable) 162 | 163 | if not silent == True: 164 | self.AddMessage(u'\n Document tables processed - {0}'.format(T.GetTimeReset())) 165 | try: 166 | Doc.save() 167 | del Doc 168 | if not silent == True: 169 | self.AddMessage(u' Document saved - {0}'.format(T.GetTimeReset())) 170 | except: 171 | self.AddWarning(u'MXD with path {0} cannot be saved'.format(MXD)) 172 | tb = sys.exc_info() 173 | self.AddWarning("Line %i" % tb[2].tb_lineno) 174 | self.AddWarning(traceback.format_exc()) 175 | except: 176 | self.AddWarning(u'MXD sources with path {0} cannot be processed'.format(MXD)) 177 | tb = sys.exc_info() 178 | self.AddWarning("Line %i" % tb[2].tb_lineno) 179 | self.AddWarning(traceback.format_exc()) 180 | except: 181 | self.AddWarning(u'MXD with path {0} cannot be opened'.format(MXD)) 182 | tb = sys.exc_info() 183 | self.AddWarning("Line %i" % tb[2].tb_lineno) 184 | self.AddWarning(traceback.format_exc()) 185 | 186 | def __ProcessLyr(self, lyr, step, Tables, silent=False): 187 | if lyr.isFeatureLayer == True: 188 | if not silent == True: 189 | self.AddMessage(step + lyr.name) 190 | Relations = list() 191 | Relations = self.GDBTools.GetRelations(lyr.workspacePath, self.gp.Describe(lyr.dataSource), Relations, only_childs=True) 192 | for rel in Relations: 193 | DRel = self.gp.Describe(os.path.join(lyr.workspacePath, rel)) 194 | if DRel.dataElementType == u'DETable': 195 | Tables.append(os.path.join(lyr.workspacePath, rel).lower()) 196 | if not silent == True: 197 | self.AddMessage(step + ' ' + lyr.name + " -> " + rel) 198 | if lyr.isGroupLayer == True: 199 | if not silent == True: 200 | self.AddMessage(step + lyr.name + ':') 201 | for sublyr in self.gp.mapping.ListLayers(lyr): 202 | if not silent == True: 203 | self.__ProcessLyr(sublyr, step + ' ', Tables) -------------------------------------------------------------------------------- /GISPython/GISPythonModule.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module for the GISPython module frame and code unification 4 | """ 5 | 6 | import sys 7 | import os 8 | import traceback 9 | import GISPythonToolBase 10 | import SysGISTools 11 | import MailHelper 12 | import MyError 13 | 14 | class GISPythonModule(object): 15 | """Interface class for all the GISPython modules. Interface class allows 16 | the code unification, and ensures the code execution from both the 17 | ArcGIS Desktop Python console and the Command Prompt. 18 | 19 | Standalone tool execution: 20 | SetName('Tool Name') 21 | DoJob() 22 | 23 | The tool executes within an another tool: 24 | SetTool(reference to SysGISTools.GISTools10) 25 | Get the tool name with the command 'PrintText()' 26 | mainModule() 27 | """ 28 | def __init__(self, ToolName, SysGISParams, ExecutePatch=__file__, statusMailRecipients=[], errorMailRecipients=[], licenceLevel = "arceditor", toollevel = "full"): 29 | """Initialize the tool and the tool parameters 30 | 31 | Args: 32 | self: The reserved object 'self' 33 | ToolName: Tool name 34 | SysGISParams: GISPython parameter module 35 | ExecutePatch: Module execution path 36 | statusMailRecipients: List with status e-mail message recipients 37 | errorMailRecipients: List with error e-mail message recipients 38 | licenceLevel: arcinfo, arceditor, arcview, arcserver, arcenginegeodb, or arcengine 39 | """ 40 | self.ToolName = ToolName 41 | self.licenceLevel = licenceLevel 42 | if toollevel.upper() in ("FULL", "LITE"): 43 | self.toollevel = toollevel.upper() 44 | else: 45 | raise AttributeError('Provided Tool Type {} is not supported!'.format(toollevel)) 46 | self.Pr = SysGISParams 47 | self.ExecutePatch = os.path.dirname(os.path.realpath(ExecutePatch)) 48 | if hasattr(self.Pr, 'EnableStatusMail'): 49 | self.DoMail = self.Pr.EnableStatusMail 50 | else: 51 | self.DoMail = False 52 | if hasattr(self.Pr, 'EnvironmentName'): 53 | self.EnvironmentName = '[{}] '.format(self.Pr.EnvironmentName) 54 | else: 55 | self.EnvironmentName = '' 56 | self.DoMailOutput = False 57 | self.DoMailErrOutput = False 58 | if self.DoMail: 59 | self.statusMailRecipients = statusMailRecipients 60 | if hasattr(self.Pr, 'MailOutputRecipientsAlways'): 61 | self.statusMailRecipients = self.statusMailRecipients + self.Pr.MailOutputRecipientsAlways 62 | self.errorMailRecipients = errorMailRecipients 63 | if hasattr(self.Pr, 'MailErrorRecipientsAlways'): 64 | self.errorMailRecipients = self.errorMailRecipients + self.Pr.MailErrorRecipientsAlways 65 | if self.statusMailRecipients: 66 | self.DoMailOutput = True 67 | if self.errorMailRecipients: 68 | self.DoMailErrOutput = True 69 | 70 | def runInsideJob(self, Tool): 71 | """Procedure executes the tool, if it's to be executed within an another Python tool 72 | 73 | Args: 74 | self: The reserved object 'self' 75 | Tool: The tool name to execute 76 | """ 77 | self.SetTool(Tool) 78 | self.PrintText() 79 | self.mainModule() 80 | 81 | def mainModule(self): 82 | """Rewritable procedure which contains the logic of the module 83 | """ 84 | raise NotImplementedError 85 | 86 | def initModule(self): 87 | """Procedure which initializes the GISPython environment, if the tool runs as a standalone tool 88 | """ 89 | if self.toollevel == "FULL": 90 | self.Tool = SysGISTools.GISTools10(self.ToolName, self.Pr, self.licenceLevel) 91 | sys.stderr = self.Tool.fLog 92 | elif self.toollevel == "LITE": 93 | self.Tool = GISPythonToolBase.GISPythonToolBase(self.ToolName, self.Pr) 94 | self.Tool.ExecutePatch = self.ExecutePatch 95 | 96 | def SetTool(self, Tool): 97 | """Sets up the GISPython environment object, if the tool runs within an another tool 98 | """ 99 | self.Tool = Tool 100 | 101 | def PrintText(self): 102 | """The auxiliary procedure for the tool name output 103 | """ 104 | self.Tool.AddMessage("--------------------------------------------") 105 | self.Tool.AddMessage(self.ToolName) 106 | self.Tool.AddMessage("--------------------------------------------") 107 | 108 | def MyEnd(self): 109 | """Procedure for the tool end message output, if the tool runs as a standalone tool 110 | """ 111 | self.Tool.MyEnd() 112 | 113 | def MyDispose(self): 114 | """Procedure which closes the tool environment after running it (in case the tool runs as a standalone tool) 115 | """ 116 | self.Tool.MyDispose() 117 | 118 | def DoJob(self): 119 | """Procedure which runs the tool with environment preparation and deletion (in case the tool runs as a standalone tool) 120 | """ 121 | try: 122 | self.initModule() 123 | self.mainModule() 124 | self.MyEnd() 125 | if self.DoMailOutput: 126 | MailHelper.GISPythonMailHelper( 127 | self.Pr, 128 | recipients=self.statusMailRecipients, 129 | Subject=unicode.format(u'{}[{}] - {}', self.EnvironmentName, self.ToolName, self.Tool.MyNow()), 130 | Text=self.Tool.OutputStr, 131 | Files=[self.Tool.out_log_path] 132 | ) 133 | if self.DoMailErrOutput: 134 | if not self.Tool.OutputErrStr == u'': 135 | MailHelper.GISPythonMailHelper( 136 | self.Pr, 137 | recipients=self.errorMailRecipients, 138 | Subject=unicode.format(u'{}Error [{}] - {}', self.EnvironmentName, self.ToolName, self.Tool.MyNow()), 139 | Text=self.Tool.OutputErrStr, 140 | Files=[self.Tool.out_log_path, self.Tool.error_log_path]) 141 | self.MyDispose() 142 | except Exception, e: 143 | # If an error occurred, print line number and error message 144 | tb = sys.exc_info() 145 | orgLine = "Line %i" % tb[2].tb_lineno 146 | orgTraceback = unicode(traceback.format_exc(), errors='ignore') 147 | if hasattr(self, 'Tool'): 148 | try: 149 | if not (self.Tool.State == "Disposed" or self.Tool.gp == None): 150 | self.Tool.AddWarning(r'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') 151 | try: 152 | self.Tool.AddWarning(u'Raw tool error {0} - {1}'.format(self.ToolName, self.Tool.MyNow())) 153 | except: 154 | None 155 | self.Tool.AddWarning(orgLine) 156 | self.Tool.AddError(orgTraceback) 157 | self.Tool.AddWarning(r'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') 158 | self.Tool.AddWarning(u'GP tool output') 159 | self.Tool.AddWarning(r'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') 160 | self.Tool.OutputErrors() 161 | if self.DoMailErrOutput: 162 | if self.Tool.OutputErrStr != u'': 163 | MailHelper.GISPythonMailHelper( 164 | self.Pr, 165 | recipients=self.errorMailRecipients, 166 | Subject=unicode.format(u'{}Error [{}] - {}', self.EnvironmentName, self.ToolName, self.Tool.MyNow()), 167 | Text=self.Tool.OutputErrStr, 168 | Files=[self.Tool.out_log_path, self.Tool.error_log_path]) 169 | else: 170 | print(r'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') 171 | print(u'Raw tool error {0}'.format(self.ToolName)) 172 | print(orgLine) 173 | print(orgTraceback) 174 | print(r'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') 175 | if self.DoMailErrOutput: 176 | MailHelper.GISPythonMailHelper( 177 | self.Pr, 178 | recipients=self.errorMailRecipients, 179 | Subject=unicode.format(u'{}Critical error [{}]', self.EnvironmentName, self.ToolName), 180 | Text=orgLine + '\n' + orgTraceback, 181 | Files=[self.Tool.out_log_path, self.Tool.error_log_path]) 182 | raise 183 | except Exception: 184 | print(r'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') 185 | print(u'Raw tool error {0}'.format(self.ToolName)) 186 | print(orgLine) 187 | print(orgTraceback) 188 | print(r'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') 189 | if self.DoMailErrOutput: 190 | MailHelper.GISPythonMailHelper( 191 | self.Pr, 192 | recipients=self.errorMailRecipients, 193 | Subject=unicode.format(u'{}Critical error [{}]', self.EnvironmentName, self.ToolName), 194 | Text=orgLine + '\n' + orgTraceback, 195 | Files=[self.Tool.out_log_path, self.Tool.error_log_path]) 196 | raise tb[1], None, tb[2] 197 | else: 198 | print(r'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') 199 | print(u'Raw tool error {0}'.format(self.ToolName)) 200 | print(orgLine) 201 | print(orgTraceback) 202 | print(r'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') 203 | if self.DoMailErrOutput: 204 | MailHelper.GISPythonMailHelper( 205 | self.Pr, 206 | recipients=self.errorMailRecipients, 207 | Subject=unicode.format(u'{}Critical error [{}]', self.EnvironmentName, self.ToolName), 208 | Text=orgLine + '\n' + orgTraceback, 209 | Files=[self.Tool.out_log_path, self.Tool.error_log_path]) 210 | raise 211 | 212 | class GISPythonModuleArgsHelper(object): 213 | """Class for handling the argument passing for the module in three different places: 214 | 215 | 1. In the class initialization process. 216 | 2. In the arguments. 217 | 3. In the "main" operation call. 218 | """ 219 | def __init__(self, InitValue=None): 220 | self.argumentValue = None 221 | self.initValue = None 222 | self.mainModuleValue = None 223 | self.SetInitValue(InitValue) 224 | 225 | def processArgument(self, argumentNumber=1): 226 | """Procedure processes a parameter acquisition from the argument 227 | 228 | Args: 229 | self: The reserved object 'self' 230 | argumentNumber: Argument from which to read the data 231 | """ 232 | if len(sys.argv) > argumentNumber: 233 | self.argumentValue = sys.argv[argumentNumber] 234 | if self.argumentValue == '#': 235 | self.argumentValue = None 236 | 237 | 238 | def SetInitValue(self, value): 239 | """Procedure processes a parameter setup from the tool initialization procedure 240 | 241 | Args: 242 | self: The reserved object 'self' 243 | value: Setup value 244 | """ 245 | self.initValue = value 246 | if self.initValue == '#': 247 | self.initValue = None 248 | 249 | def SetMainModuleValue(self, value): 250 | """Procedure processes a parameter setup from the base module of the tool 251 | 252 | Args: 253 | self: The reserved object 'self' 254 | value: Setup value 255 | """ 256 | self.mainModuleValue = value 257 | if self.mainModuleValue == '#': 258 | self.mainModuleValue = None 259 | 260 | def GetResultValue(self, asBool=False, Default=None): 261 | """Procedure makes a choice from the given attributes 262 | 263 | Args: 264 | self: The reserved object self 265 | value: Setup value 266 | """ 267 | candidate = Default 268 | 269 | if self.argumentValue != None: 270 | candidate = self.argumentValue 271 | else: 272 | if self.mainModuleValue != None: 273 | candidate = self.mainModuleValue 274 | else: 275 | if self.initValue != None: 276 | candidate = self.initValue 277 | if asBool: 278 | if candidate == None: 279 | candidate = False 280 | elif unicode(candidate).upper() == u'TRUE': 281 | candidate = True 282 | elif unicode(candidate).upper() == u'FALSE': 283 | candidate = False 284 | else: 285 | raise MyError.MyError(u'Cannot convert the value {0} to a boolean'.format(candidate)) 286 | 287 | return candidate 288 | -------------------------------------------------------------------------------- /GISPython/SysGISTools.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | GIS function support module 4 | """ 5 | import GDBHelper 6 | import GISPythonToolBase 7 | import codecs 8 | 9 | class GISTools10(GISPythonToolBase.GISPythonToolBase): 10 | """Class for storing the auxiliary batch processing and GIS geoprocesing functions""" 11 | def __init__(self, ToolName, Params, licenceLevel = "arceditor"): 12 | """Class initialization procedure 13 | 14 | Args: 15 | self: The reserved object 'self' 16 | ToolName: Name of the tool (used for output) 17 | Params: parameter object 18 | licenceLevel: arcinfo, arceditor, arcview, arcserver, arcenginegeodb, or arcengine 19 | """ 20 | super(GISTools10, self).__init__(ToolName, Params) 21 | 22 | if licenceLevel == "arcinfo": 23 | print(u'...Using licence level : ' + licenceLevel) 24 | import arcinfo 25 | elif licenceLevel == "arceditor": 26 | print(u'...Using licence level : ' + licenceLevel) 27 | import arceditor 28 | elif licenceLevel == "arcview": 29 | print(u'...Using licence level : ' + licenceLevel) 30 | import arcview 31 | elif licenceLevel == "arcserver": 32 | print(u'...Using licence level : ' + licenceLevel) 33 | import arcserver 34 | elif licenceLevel == "arcenginegeodb": 35 | print(u'...Using licence level : ' + licenceLevel) 36 | import arcenginegeodb 37 | elif licenceLevel == "arcengine": 38 | print(u'...Using licence level : ' + licenceLevel) 39 | import arcengine 40 | else: 41 | print(u'...Incorect licence suplied - using : arceditor') 42 | import arceditor 43 | 44 | import arcpy # arcpy variables 45 | self.gp = arcpy 46 | if hasattr(Params, 'SetLogHistory'): 47 | arcpy.SetLogHistory(Params.SetLogHistory) 48 | else: 49 | arcpy.SetLogHistory(False) 50 | self.GDBHelper = GDBHelper.GDBHelper(self.gp) 51 | self.SR = MySR() 52 | 53 | # Set up Logging 54 | LogDir = self.Pr.ErrorLogDir 55 | LogDirArh = self.Pr.ErrorLogDirArh 56 | OutDir = self.Pr.OutDir 57 | OutDirArh = self.Pr.OutDirArh 58 | 59 | try: 60 | self.AchiveFiles(LogDir, LogDirArh, ToolName, False) 61 | except Exception, err: 62 | print(u'Error archiving errlog files') 63 | if hasattr(err, 'strerror'): 64 | if hasattr(err, 'strerror'): 65 | print(err.strerror) 66 | else: 67 | print('{}'.format(err)) 68 | else: 69 | print(err.message) 70 | try: 71 | self.AchiveFiles(OutDir, OutDirArh, ToolName, False) 72 | except Exception, err: 73 | print(u'Error archiving outlog files') 74 | if hasattr(err, 'strerror'): 75 | if hasattr(err, 'strerror'): 76 | print(err.strerror) 77 | else: 78 | print('{}'.format(err)) 79 | else: 80 | print(err.message) 81 | self.error_log_path = LogDir + '\\' + self.ToolName + self.MyNowFile() + '.errlog' 82 | self.fLog = codecs.open(self.error_log_path, encoding='utf-8', mode='w') 83 | self.out_log_path = OutDir + '\\' + self.ToolName + self.MyNowFile() + '.outlog' 84 | self.fOut = codecs.open(self.out_log_path, encoding='utf-8', mode='w') 85 | self.fSQL = OutDir + '\\' + self.ToolName + self.MyNowFile() + 'SQL.outlog' 86 | 87 | self.AddMessage(u'\n==================================================') 88 | self.AddMessage(r'//////////////////////////////////////////////////') 89 | self.AddMessage(u'Executing the tool ' + ToolName + u' ' + self.MyNow()) 90 | self.AddMessage(r'\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\') 91 | self.AddMessage(u'==================================================\n') 92 | 93 | def callGP(self, functionName, *args): 94 | """Function to call the arcPy GP functions with automatic output messge display and output result returning 95 | 96 | Args: 97 | self: The reserved object 'self' 98 | functionName: Name of the arcPy GP function 99 | args: arguments as Tuple 100 | """ 101 | gpOutput = self.callGPSilent(functionName, *args) 102 | self.OutputMessages() 103 | return gpOutput 104 | 105 | def callGPSilent(self, functionName, *args): 106 | """Function to call the arcPy GP functions without output 107 | 108 | Args: 109 | self: The reserved object 'self' 110 | functionName: Name of the arcPy GP function 111 | args: arguments as Tuple 112 | """ 113 | gpFunction = getattr(self.gp, functionName) 114 | rezult = gpFunction(*args) 115 | if rezult.outputCount > 0: 116 | gpOutput = rezult.getOutput(0) 117 | else: 118 | gpOutput = None 119 | return gpOutput 120 | 121 | def MyEnd(self): 122 | """End of the process 123 | 124 | Args: 125 | self: The reserved object 'self' 126 | """ 127 | super(GISTools10, self).MyEnd() 128 | self.fOut.close() 129 | self.fLog.close() 130 | 131 | def OutputMessages(self, ErrorSeverity=2): 132 | """Procedure to output messages stored in the GP object 133 | 134 | Args: 135 | self: The reserved object 'self' 136 | ErrorSeverity: Maximum Severity to report as error 137 | """ 138 | self.AddMessage('\n--------------------------------------------') 139 | maxSeverity = self.gp.GetMaxSeverity() 140 | for i in range(0, self.gp.GetMessageCount()): 141 | self.gp.AddReturnMessage(i) 142 | #self.gp.AddMessage(self.gp.GetMessage(i)) 143 | self.fOut.write(self.gp.GetMessage(i)) 144 | self.OutputStr += self.gp.GetMessage(i) 145 | self.fOut.write('\n') 146 | self.OutputStr += '\n' 147 | self.fOut.flush() 148 | if maxSeverity >= ErrorSeverity: 149 | self.fLog.write(self.gp.GetMessage(i)) 150 | self.OutputErrStr += self.gp.GetMessage(i) 151 | self.fLog.write('\n') 152 | self.OutputErrStr += '\n' 153 | self.fLog.flush() 154 | self.AddMessage('--------------------------------------------') 155 | 156 | def OutputErrors(self): 157 | """Procedure to output messages and errors stored in the GP object. 158 | !!! Depricated - Left for backwards compatibility - use OutputMessages with ErrorSeverity 0 !!! 159 | 160 | Args: 161 | self: The reserved object 'self' 162 | """ 163 | self.AddMessage('') 164 | self.AddMessage('--------------------------------------------') 165 | for i in range(0, self.gp.GetMessageCount()): 166 | self.gp.AddReturnMessage(i) 167 | self.fOut.write(self.gp.GetMessage(i)) 168 | self.OutputStr += self.gp.GetMessage(i) 169 | self.fOut.write('\n') 170 | self.OutputStr += '\n' 171 | self.fOut.flush() 172 | self.fLog.write(self.gp.GetMessage(i)) 173 | self.OutputErrStr += self.gp.GetMessage(i) 174 | self.fLog.write('\n') 175 | self.OutputErrStr += '\n' 176 | self.fLog.flush() 177 | self.AddMessage('--------------------------------------------') 178 | 179 | def AddMessage(self, strMessage, newline=True): 180 | """Procedure for a message output (screen, logfile and if necessary e-mail) 181 | 182 | Args: 183 | self: The reserved object 'self' 184 | strMessage: Output message text 185 | newline: Flag that marks that the output must be continued in a new line 186 | """ 187 | if strMessage == None: 188 | strMessage = '' 189 | if not isinstance(strMessage, unicode): 190 | if not isinstance(strMessage, str): 191 | strMessage = str(strMessage) 192 | if strMessage[-1:] == '\n': 193 | strMessage = strMessage[0:-1] 194 | try: 195 | self.fOut.write(strMessage) 196 | self.OutputStr += strMessage 197 | except: 198 | self.fOut.write(self._tryCovertStringEncoding(strMessage)) 199 | self.OutputStr += self._tryCovertStringEncoding(strMessage) 200 | 201 | try: 202 | self.gp.AddMessage(strMessage) 203 | except: 204 | print strMessage 205 | if newline == True: 206 | self.fOut.write('\n') 207 | self.OutputStr += '\n' 208 | self.fOut.flush() 209 | 210 | def AddError(self, strMessage, newline=True): 211 | """Procedure for the GP object error message output (screen, logfile and if necessary e-mail) 212 | 213 | Args: 214 | self: The reserved object 'self' 215 | strMessage: Output message text 216 | """ 217 | if strMessage == None: 218 | strMessage = '' 219 | if not isinstance(strMessage, unicode): 220 | if not isinstance(strMessage, str): 221 | strMessage = str(strMessage) 222 | if strMessage[-1:] == '\n': 223 | strMessage = strMessage[0:-1] 224 | try: 225 | self.gp.AddError(strMessage) 226 | except: 227 | print strMessage 228 | try: 229 | self.fLog.write(strMessage) 230 | self.OutputErrStr += strMessage 231 | except: 232 | self.fLog.write(self._tryCovertStringEncoding(strMessage)) 233 | self.OutputErrStr += self._tryCovertStringEncoding(strMessage) 234 | if newline == True: 235 | self.fLog.write('\n') 236 | self.OutputErrStr += '\n' 237 | self.fLog.flush() 238 | try: 239 | self.fOut.write(strMessage) 240 | self.OutputStr += strMessage 241 | except: 242 | self.fOut.write(self._tryCovertStringEncoding(strMessage)) 243 | self.OutputStr += self._tryCovertStringEncoding(strMessage) 244 | if newline == True: 245 | self.fOut.write('\n') 246 | self.OutputStr += '\n' 247 | self.fOut.flush() 248 | 249 | def AddWarning(self, strMessage): 250 | """Procedure for the GP object warning message output (screen, logfile and if necessary e-mail) 251 | 252 | Args: 253 | self: The reserved object 'self' 254 | strMessage: Output message text 255 | """ 256 | if strMessage == None: 257 | strMessage = '' 258 | if not isinstance(strMessage, unicode): 259 | if not isinstance(strMessage, str): 260 | strMessage = str(strMessage) 261 | if strMessage[-1:] == '\n': 262 | strMessage = strMessage[0:-1] 263 | try: 264 | self.gp.AddWarning(strMessage) 265 | except: 266 | print strMessage 267 | try: 268 | self.fLog.write(strMessage) 269 | self.OutputErrStr += strMessage 270 | except: 271 | self.fLog.write(self._tryCovertStringEncoding(strMessage)) 272 | self.OutputErrStr += self._tryCovertStringEncoding(strMessage) 273 | self.fLog.write('\n') 274 | self.OutputErrStr += '\n' 275 | self.fLog.flush() 276 | try: 277 | self.fOut.write(strMessage) 278 | self.OutputStr += strMessage 279 | except: 280 | self.fOut.write(self._tryCovertStringEncoding(strMessage)) 281 | self.OutputStr += self._tryCovertStringEncoding(strMessage) 282 | self.fOut.write('\n') 283 | self.OutputStr += '\n' 284 | self.fOut.flush() 285 | 286 | 287 | 288 | class MySR(): 289 | """Class for storing often used coordinate system parameters""" 290 | def __init__(self): 291 | """Class initialization procedure 292 | 293 | Args: 294 | self: The reserved object 'self' 295 | """ 296 | self.LKS92_0 = "PROJCS['LKS_1992_Latvia_TM_0',GEOGCS['GCS_LKS_1992',DATUM['D_Latvia_1992',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Transverse_Mercator'],PARAMETER['False_Easting',500000.0],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',24.0],PARAMETER['Scale_Factor',0.9996],PARAMETER['Latitude_Of_Origin',0.0],UNIT['Meter',1.0]]" 297 | self.LKS92 = "PROJCS['LKS_1992_Latvia_TM',GEOGCS['GCS_LKS_1992',DATUM['D_Latvia_1992',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Transverse_Mercator'],PARAMETER['False_Easting',500000.0],PARAMETER['False_Northing',-6000000.0],PARAMETER['Central_Meridian',24.0],PARAMETER['Scale_Factor',0.9996],PARAMETER['Latitude_Of_Origin',0.0],UNIT['Meter',1.0]]" 298 | self.WebMercator = "PROJCS['WGS_1984_Web_Mercator_Auxiliary_Sphere',GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Mercator_Auxiliary_Sphere'],PARAMETER['False_Easting',0.0],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',0.0],PARAMETER['Standard_Parallel_1',0.0],PARAMETER['Auxiliary_Sphere_Type',0.0],UNIT['Meter',1.0]]" 299 | self.WGS84 = "GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]" 300 | -------------------------------------------------------------------------------- /GISPython/AGServerHelper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module for operations with ArcGIS Server services 4 | """ 5 | 6 | # For Http calls 7 | import httplib, urllib, json, urllib2, sys 8 | import MyError 9 | 10 | class AGSServerHelper(object): 11 | 12 | def __init__(self, username, password, serverName, serverPort=6080, Tool=None, https=False): 13 | """Class initialization procedure 14 | 15 | Args: 16 | self: The reserved object 'self' 17 | username: ArcGIS Server administrator username 18 | password: ArcGIS Server administrator password 19 | serverName: Server adress 20 | serverPort: Server port (optional) 21 | Tool: GISPython tool (optional) 22 | """ 23 | if not Tool is None: 24 | Tool.AddMessage('Warning: Tool AGSServerHelper is beeing discontinued please use AGServerHelperNTLM instead!') 25 | else: 26 | print ('Warning: Tool AGSServerHelper is beeing discontinued please use AGServerHelperNTLM instead!') 27 | self.username = username 28 | self.password = password 29 | self.serverName = serverName 30 | self.serverPort = serverPort 31 | self.Tool = Tool 32 | self.https = https 33 | if https: 34 | self.httstring = 'https' 35 | else: 36 | self.httstring = 'http' 37 | # Get a token 38 | self.token = self.genToken(username, password, serverName, serverPort) 39 | if self.token == "": 40 | raise MyError.MyError("Could not generate a token with the username and password provided.") 41 | 42 | def StartService(self, folder, service): 43 | self.StartStopService(folder, service, "START") 44 | 45 | def StopService(self, folder, service): 46 | self.StartStopService(folder, service, "STOP") 47 | 48 | def StartStopService(self, folder, service, action): 49 | # Construct URL to read folder 50 | if folder == None: 51 | folder = "ROOT" 52 | if folder.upper() == "ROOT": 53 | folder = "" 54 | else: 55 | folder += "/" 56 | 57 | # Define service URL with action (START/STOP) 58 | folderURL = "/arcgis/admin/services/" + folder + service + "/" + action 59 | 60 | # This request only needs the token and the response formatting parameter 61 | params = urllib.urlencode({'token': self.token, 'f': 'json'}) 62 | 63 | headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} 64 | 65 | # Connect to URL and post parameters 66 | if self.https: 67 | httpConn = httplib.HTTPSConnection(self.serverName, self.serverPort) 68 | else: 69 | httpConn = httplib.HTTPConnection(self.serverName, self.serverPort) 70 | httpConn.request("POST", folderURL, params, headers) 71 | 72 | # Read response 73 | response = httpConn.getresponse() 74 | if response.status != 200: 75 | httpConn.close() 76 | raise MyError.MyError("Could not read folder information.") 77 | else: 78 | data = response.read() 79 | 80 | # Check that data returned is not an error object 81 | if not self.assertJsonSuccess(data): 82 | raise MyError.MyError("Error when reading folder information. " + str(data)) 83 | else: 84 | if self.Tool != None: 85 | self.Tool.AddMessage("Service " + folder + service + ' ' + action + ' done successfully ...') 86 | 87 | # Deserialize response into Python object 88 | #dataObj = json.loads(data) 89 | httpConn.close() 90 | 91 | def genToken(self, adminUser, adminPass, server, port, expiration=60): 92 | """Create ArcGIS server connection file 93 | 94 | Args: 95 | server: Server name 96 | port: Server port 97 | adminUser: Username 98 | adminPass: Password 99 | """ 100 | query_dict = {'username': adminUser, 101 | 'password': adminPass, 102 | 'expiration': expiration, 103 | 'client': 'requestip'} 104 | 105 | query_string = urllib.urlencode(query_dict) 106 | url = "{}://{}:{}/arcgis/admin/generateToken".format(self.httstring, server, port) 107 | 108 | token = json.loads(urllib.urlopen(url + "?f=json", query_string).read()) 109 | 110 | if "token" not in token: 111 | raise MyError.MyError(token['messages']) 112 | sys.exit() 113 | else: 114 | return token['token'] 115 | 116 | def getServiceList(self, server, port, adminUser, adminPass, token=None): 117 | """Retrieve ArcGIS server services 118 | 119 | Args: 120 | self: The reserved object 'self' 121 | server: Server name 122 | port: Server port 123 | adminUser: Username 124 | adminPass: Password 125 | token: Token (if created) 126 | """ 127 | if token is None: 128 | token = self.genToken(adminUser, adminPass, server, port) 129 | 130 | services = [] 131 | folder = '' 132 | URL = "{}://{}:{}/arcgis/admin/services{}?f=pjson&token={}".format(self.httstring, server, port, folder, token) 133 | 134 | try: 135 | serviceList = json.loads(urllib2.urlopen(URL).read()) 136 | except urllib2.URLError, e: 137 | raise MyError.MyError(e) 138 | 139 | # Build up list of services at the root level 140 | for single in serviceList["services"]: 141 | services.append(single['serviceName'] + '.' + single['type']) 142 | 143 | # Build up list of folders and remove the System and Utilities folder (we dont want anyone playing with them) 144 | folderList = serviceList["folders"] 145 | folderList.remove("Utilities") 146 | folderList.remove("System") 147 | 148 | if len(folderList) > 0: 149 | for folder in folderList: 150 | URL = "{}://{}:{}/arcgis/admin/services/{}?f=pjson&token={}".format(self.httstring, server, port, folder, token) 151 | fList = json.loads(urllib2.urlopen(URL).read()) 152 | 153 | for single in fList["services"]: 154 | services.append(folder + "//" + single['serviceName'] + '.' + single['type']) 155 | 156 | if len(services) == 0: 157 | if self.Tool != None: 158 | self.Tool.AddMessage("No services found") 159 | else: 160 | if self.Tool != None: 161 | self.Tool.AddMessage("Services on " + server +":") 162 | for service in services: 163 | statusURL = "{}://{}:{}/arcgis/admin/services/{}/status?f=pjson&token={}".format(self.httstring, server, port, service, token) 164 | status = json.loads(urllib2.urlopen(statusURL).read()) 165 | if self.Tool != None: 166 | self.Tool.AddMessage(" " + status["realTimeState"] + " > " + service) 167 | 168 | return services 169 | 170 | def GetServerJson(self, token, serverName, serverPort, serverService): 171 | """Retrieve service parameters 172 | 173 | Args: 174 | self: The reserved object 'self' 175 | token: Token 176 | serverName: Server name 177 | serverPort: Server port 178 | serverService: Service which parameter configuration shall be retrieved 179 | """ 180 | # This request only needs the token and the response formatting parameter 181 | params = urllib.urlencode({'token': token, 'f': 'json'}) 182 | 183 | headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} 184 | serviceURL = "/arcgis/admin/services/" + serverService 185 | 186 | # Connect to service to get its current JSON definition 187 | if self.https: 188 | httpConn = httplib.HTTPSConnection(serverName, serverPort) 189 | else: 190 | httpConn = httplib.HTTPConnection(serverName, serverPort) 191 | httpConn.request("POST", serviceURL, params, headers) 192 | 193 | # Read response 194 | response = httpConn.getresponse() 195 | if response.status != 200: 196 | httpConn.close() 197 | raise MyError.MyError(u'...Couldn\'t retrieve service parameter configuration\n') 198 | return 199 | else: 200 | data = response.read() 201 | 202 | # Check that data returned is not an error object 203 | if not self.assertJsonSuccess(data): 204 | raise MyError.MyError(u'...Couldn\'t retrieve service parameter configuration: ' + str(data) + '\n') 205 | else: 206 | if self.Tool != None: 207 | self.Tool.AddMessage(u'...Service parameter configuration successfully retrieved\n') 208 | 209 | dataObj = json.loads(data) 210 | httpConn.close() 211 | return dataObj 212 | 213 | def PublishServerJson(self, service, serverName, dataObj, token, serverPort): 214 | """Publish service parameters to server 215 | 216 | Args: 217 | self: The reserved object 'self' 218 | service: Service which parameter configuration shall be renewed 219 | serverName: Server name 220 | dataObj: Parameter configuration 221 | token: Token 222 | serverPort: Server port 223 | """ 224 | if self.https: 225 | httpConn = httplib.HTTPSConnection(serverName, serverPort) 226 | else: 227 | httpConn = httplib.HTTPConnection(serverName, serverPort) 228 | headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} 229 | 230 | # Serialize back into JSON 231 | updatedSvcJson = json.dumps(dataObj) 232 | 233 | # Call the edit operation on the service. Pass in modified JSON. 234 | editSvcURL = "/arcgis/admin/services/" + service + "/edit" 235 | params = urllib.urlencode({'token': token, 'f': 'json', 'service': updatedSvcJson}) 236 | httpConn.request("POST", editSvcURL, params, headers) 237 | 238 | # Read service edit response 239 | editResponse = httpConn.getresponse() 240 | if editResponse.status != 200: 241 | httpConn.close() 242 | if self.Tool != None: 243 | self.Tool.AddMessage(u'...Service configuration renewal error\n') 244 | return 245 | else: 246 | editData = editResponse.read() 247 | 248 | # Check that data returned is not an error object 249 | if not self.assertJsonSuccess(editData): 250 | if self.Tool != None: 251 | self.Tool.AddMessage(u'...Service configuration renewal error: ' + str(editData) + '\n') 252 | else: 253 | if self.Tool != None: 254 | self.Tool.AddMessage(u'...Service configuration succesfully renewed\n') 255 | 256 | httpConn.close() 257 | 258 | return 259 | 260 | def getServiceFromServer(self, services, service, serviceDir): 261 | """Retrieve the full service name from the server 262 | 263 | Args: 264 | self: The reserved object 'self' 265 | services: List of all services on server 266 | service: Name of the service from which to get corresponding name from the server services list 267 | serviceDir: Name of the service directory which is shown in the configuration of services to be published on the server 268 | """ 269 | if serviceDir == None: 270 | configService = service 271 | else: 272 | configService = serviceDir + "//" + service 273 | for serverService in services: 274 | if serverService.split('.')[0].upper() == configService.upper(): 275 | return serverService 276 | else: 277 | serverService = '' 278 | 279 | return serverService 280 | 281 | def assertJsonSuccess(self, data): 282 | """A function that checks that the input JSON object is not an error object. 283 | 284 | Args: 285 | self: The reserved object 'self' 286 | data: JSON data object 287 | """ 288 | obj = json.loads(data) 289 | if 'status' in obj and obj['status'] == "error": 290 | raise MyError.MyError("Error: JSON object returns an error. " + str(obj)) 291 | return False 292 | else: 293 | return True 294 | 295 | #Check if service is running 296 | def IsServiceRunning(self, folder, service): 297 | """Retrieve the service status from the server 298 | 299 | Args: 300 | self: The reserved object 'self' 301 | folder: Service directory 302 | service: Name of a service 303 | """ 304 | # Construct URL to read folder 305 | if folder == None: 306 | folder = "ROOT" 307 | if folder.upper() == "ROOT": 308 | folder = "" 309 | else: 310 | folder += "/" 311 | 312 | folderURL = "/arcgis/admin/services/" + folder 313 | 314 | # This request only needs the token and the response formatting parameter 315 | params = urllib.urlencode({'token': self.token, 'f': 'json'}) 316 | 317 | headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} 318 | 319 | # Connect to URL and post parameters 320 | if self.https: 321 | httpConn = httplib.HTTPSConnection(self.serverName, self.serverPort) 322 | else: 323 | httpConn = httplib.HTTPConnection(self.serverName, self.serverPort) 324 | 325 | httpConn.request("POST", folderURL, params, headers) 326 | 327 | # Read response 328 | response = httpConn.getresponse() 329 | if response.status != 200: 330 | httpConn.close() 331 | raise MyError.MyError("Could not read folder information.") 332 | else: 333 | data = response.read() 334 | 335 | # Check that data returned is not an error object 336 | if not self.assertJsonSuccess(data): 337 | raise MyError.MyError("Error while reading folder information. " + str(data)) 338 | 339 | # Deserialize response into Python object 340 | dataObj = json.loads(data) 341 | 342 | httpConn.close() 343 | 344 | # Construct URL to stop or start service, then make the request 345 | statusURL = "/arcgis/admin/services/" + folder + service + "/status" 346 | httpConn.request("POST", statusURL, params, headers) 347 | 348 | # Read status response 349 | statusResponse = httpConn.getresponse() 350 | if statusResponse.status != 200: 351 | httpConn.close() 352 | raise MyError.MyError("Error while checking status for " + service) 353 | return 354 | else: 355 | statusData = statusResponse.read() 356 | 357 | # Check that data returned is not an error object 358 | if not self.assertJsonSuccess(statusData): 359 | raise MyError.MyError("Error while retrieving status information for " + service + ".") 360 | else: 361 | # Get service status 362 | statusDataObj = json.loads(statusData) 363 | if statusDataObj['realTimeState'] == "STOPPED": 364 | return False #print "Service " + service + " was detected to be stopped" 365 | else: 366 | return True #print "Service " + service + " is running" 367 | 368 | httpConn.close() 369 | -------------------------------------------------------------------------------- /docs/helper_modules.rst: -------------------------------------------------------------------------------- 1 | Helper modules 2 | ============== 3 | 4 | Additional *GISPython* package modules 5 | 6 | AGServerHelper 7 | -------------- 8 | 9 | *Module contains procedures for typical operations with ArcGIS server. All procedures use token authorization. For procedures with NTLM authorization use AGServerHelperNTLM module.* 10 | 11 | .. automodule:: AGServerHelper 12 | :members: 13 | :undoc-members: 14 | :show-inheritance: 15 | 16 | Examples 17 | ******** 18 | 19 | Check if ArcGIS Server service is running:: 20 | 21 | from GISPython import AGServerHelper 22 | 23 | Tool = self.Tool 24 | JsonParams = JsonParamsHelper.JsonParams(Tool, 'Config', 'ServiceCheck') 25 | AGS = AGServerHelper.AGSServerHelper(Pr.AGS_u, Pr.AGS_p, Pr.AGS_Servers[0], Pr.AGS_Port, self.Tool) 26 | configData = JsonParams.GetParams() 27 | dirs = configData[u'chechServiceList'] 28 | for dir in dirs: 29 | services = dir["services"] 30 | for service in services: 31 | serviceName = service["serviceName"] 32 | type = service["type"] 33 | description = service["description"] 34 | try: 35 | running = AGS.IsServiceRunning(dir["folderName"], serviceName + "." + type) 36 | 37 | if running: 38 | Tool.AddMessage(u"\t{0}\\{1}.{2} ({3}) - OK".format(dir["folderName"], serviceName, type, description)) 39 | else: 40 | txt = u"\t{0}\\{1}.{2} ({3}) - Stopped".format(dir["folderName"], serviceName, type, description) 41 | Tool.AddMessage(txt) 42 | errorTxt += "\n" + txt 43 | 44 | except Exception, e: 45 | tb = sys.exc_info() 46 | orgLine = "Line %i" % tb[2].tb_lineno 47 | orgTraceback = unicode(traceback.format_exc(), errors='ignore') 48 | 49 | txt = u"\t{0}\\{1}.{2} ({3}) - Error - Line:{4} Error: {5} ".format(dir["folderName"], serviceName, type, description, orgLine, orgTraceback) 50 | Tool.AddMessage(txt) 51 | errorTxt += "\n" + txt 52 | 53 | AGServerHelperNTLM 54 | ------------------ 55 | 56 | *Module contains procedures for typical operations with ArcGIS server. All procedures use NTLM authorization. For token authorization use AGServerHelpaer module.* 57 | 58 | .. automodule:: AGServerHelperNTLM 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | CachingHelper 64 | ------------- 65 | 66 | *Module generates and carries out map scale caching in ArcGIS Server services.* 67 | 68 | .. automodule:: CachingHelper 69 | :members: 70 | :undoc-members: 71 | :show-inheritance: 72 | 73 | Examples 74 | ******** 75 | 76 | Daily server caching procedure:: 77 | 78 | from GISPython import CachingHelper 79 | 80 | GCache = CachingHelper.CachingHelper(self.Tool, Pr.ConnAGSCache, self.Tool.gp.Extent(307950, 167920, 767480, 443890), Pr.ConnAuto + "\\SDEOWNER.CashLVBuffer") 81 | GCache.GenerateCache('Nog_Dalplans_cache',8,'30000;20000;15000;10000;5000;2000','CacheDinamic') 82 | GCache.GenerateCache('Nog_MezaudzuPlans_cache',8,'30000;20000;15000;10000;5000;2000','CacheDinamic') 83 | 84 | FTPHleper 85 | --------- 86 | 87 | *Module contains procedures for typical FTP server file operations.* 88 | 89 | .. automodule:: FTPHleper 90 | :members: 91 | :undoc-members: 92 | :show-inheritance: 93 | 94 | Examples 95 | ******** 96 | 97 | Deletes old and uploads new files to FTP server:: 98 | 99 | from GISPython import FTPHleper 100 | 101 | tmpFolder = pj(Pr.TmpFolder, "mobileInfo") 102 | FTP = FTPHleper.FTPHleper(Pr.mobileFTPHost, Pr.mobileFTPuser, Pr.mobileFTPpwd, Pr.mobileFTPmobileinfoFolder) 103 | ftpfiles = FTP.list_files() 104 | 105 | # Delete old files 106 | for file in (f for f in ftpfiles if (f.file == 'file1.geojson' or f.file == 'file2.geojson')): 107 | Tool.AddMessage(u'Delete file ' + file.file + ' from ftp ...') 108 | FTP.delete_file(file.file) 109 | 110 | # Upload new files 111 | FTP.upload_file('file1.geojson', tmpFolder) 112 | Tool.AddMessage(u'\nfile ' + 'file2.geojson' + u' uploaded to ftp ...') 113 | FTP.upload_file('file2.geojson', tmpFolder) 114 | Tool.AddMessage(u'file ' + 'file2.geojson' + u' uploaded to ftp ...') 115 | 116 | GDBHelper 117 | --------- 118 | 119 | *Module for typical GDB (File Geodatabase) operations.* 120 | 121 | .. automodule:: GDBHelper 122 | :members: 123 | :undoc-members: 124 | :show-inheritance: 125 | 126 | Examples 127 | ******** 128 | 129 | Clear old data from a table or a feature class and append new data:: 130 | 131 | from GISPython import GDBHelper 132 | from GISPython import TimerHelper 133 | 134 | GDB = GDBHelper.GDBHelper(self.Tool.gp, self.Tool) 135 | Appender = GDBHelper.SimpleAppend(self.Tool, Pr.ConnAuto, Pr.VAADConn) 136 | t = TimerHelper.TimerHelper() 137 | layer = 'Dataset.Table' 138 | 139 | # Deletes old data 140 | self.Tool.AddMessage(u'\n>>>> Begin the old file deletion procedure {0}'.format(Tool.MyNow())) 141 | GDB.ClearData(Pr.Connection, layer) 142 | self.Tool.AddMessage(u'\n>>>> End the old file deletion procedure {0}'.format(t.GetTimeReset())) 143 | 144 | # Appends new data 145 | self.Tool.AddMessage(u'\n>>>> Begin data appending procedure {0}'.format(Tool.MyNow())) 146 | Appender.Append('SDEOWNER.MKViews\\SDEOWNER.TABLEVIEW1', layer) 147 | self.Tool.AddMessage(u'\n>>>> End data appending procedure {0}'.format(t.GetTimeReset())) 148 | 149 | Validate the rows with the SQL clause:: 150 | 151 | from GISPython import GDBHelper 152 | 153 | RHelper = GDBHelper.RowHelper(self.Tool.gp, self.Tool) 154 | 155 | # Validate rows 156 | rezultList = RHelper.ValidateRowsForSQLClause(fc, validator[u'Fields'], validator[u'Query'], validator[u'OutputPatern']) 157 | 158 | # Output message 159 | if len(rezultList)>0: 160 | self.Tool.AddMessage(u' !!! {0} faulty records found ... '.format(len(rezultList))) 161 | 162 | GDPSyncroniserHelper 163 | -------------------- 164 | 165 | .. automodule:: GDPSyncroniserHelper 166 | :members: 167 | :undoc-members: 168 | :show-inheritance: 169 | 170 | Examples 171 | ******** 172 | 173 | Synchronize data between two tables with internal parameter definition:: 174 | 175 | from GISPython import GDPSyncroniserHelper 176 | 177 | ErrOutput = '' 178 | sync = GDPSyncroniserHelper.GDPSyncroniserHelper(gp, Tool) 179 | deff = GDPSyncroniserHelper.SyncDefinition() 180 | bridges = callGP('MakeFeatureLayer_management', pj(Pr.ConnAuto, 'SDEOWNER.Roads', 'SDEOWNER.Bridge'), '#', 'BRIDGESUBTYPE = 1') 181 | 182 | # Define input table parameters 183 | deff.inTable = bridges 184 | deff.inTableJoinField = 'OBJECTID' # join field 185 | deff.inTableFields = ('OBJECTID', 'BRIDGENUMBER', 'BRIDGENAME', 'LVM_DISTRICT_CODE','MI_SPECIALIST', 'MIREGION', 'SHAPE@XY') # fields 186 | 187 | # Define output table parameters 188 | deff.outTable = pj(Pr.ConnAuto, 'SDEOWNER.Roads', 'SDEOWNER.BridgeInspection') 189 | deff.outTableJoinField = 'BRIDGEOID' # join field 190 | deff.outTableFields = ('BRIDGEOID', 'BRIDGENUMBER', 'BRIDGENAME', 'LVM_DISTRICT_CODE','MI_SPECIALIST', 'MIREGION', 'SHAPE@XY') # fields 191 | deff.createNew = True 192 | deff.messageDefinition = u'Nr: {1} - {2}' # output mask 'inTableJoinField + inTableFields' defines sync order 193 | 194 | # End of synchronization 195 | output, outputErrors = sync.DoSync(deff, Pr.ConnAuto, True, True, True, True) 196 | 197 | # Error message in case there are any error 198 | if not outputErrors == u"": 199 | ErrOutput += u'Found errors in synchronization process:\n' + outputErrors + '\n\n'; 200 | 201 | GDPSyncroniserHelper2 202 | -------------------- 203 | 204 | .. automodule:: GDPSyncroniserHelper2 205 | :members: 206 | :undoc-members: 207 | :show-inheritance: 208 | 209 | 210 | JsonParamsHelper 211 | ---------------- 212 | 213 | .. automodule:: JsonParamsHelper 214 | :members: 215 | :undoc-members: 216 | :show-inheritance: 217 | 218 | Examples 219 | ******** 220 | 221 | Update attributes from JSON file:: 222 | 223 | from GISPython import JsonParamsHelper 224 | 225 | # User defined data update function 226 | def UpdateData(self, key, configData, origKey): 227 | """Executes attribute selection from configuration file and pass the parameters to the attribute calculation tool 228 | 229 | Args: 230 | self: The reserved object 'self' 231 | key: Dictionary key, which corresponds to the 'rightsType' argument 232 | configData: Retrieved data from the configuration file 233 | origKey: Primary rights type 234 | """ 235 | 236 | def mainModule(self, rightsType = '#'): 237 | #Define variables 238 | rightsType = self.rightsType 239 | 240 | JsonParams = JsonParamsHelper.JsonParams(self.Tool, 'ConfigFolder', 'ConfigFile') 241 | configData = JsonParams.GetParams() 242 | 243 | # Call UpdateData function (user defined) with according rights 244 | if (rightsType.upper() == "ALL" or rightsType.upper() == "SOMETYPE"): 245 | for key in configData.keys(): 246 | self.UpdateData(key, configData, rightsType) 247 | else: 248 | self.UpdateData(rightsType, configData, rightsType) 249 | 250 | 251 | 252 | MailHelper 253 | ---------- 254 | 255 | *Module for e-mail operations. Module contains functions for typical SMTP operations, and parameter processing from user parameter file.* 256 | 257 | .. automodule:: MailHelper 258 | :members: 259 | :undoc-members: 260 | :show-inheritance: 261 | 262 | Examples 263 | ******** 264 | Send e-mail using parameters from parameter file:: 265 | 266 | from GISPython import MailHelper 267 | 268 | MailHelper.GISPythonMailHelper(self.Pr, ['***@mail.com'], 'Subject', 'e-mail content') 269 | 270 | This script depends on following parameters which needs to be configured in *SysGISParams.py* file: 271 | 272 | * Mailserver - mail server name ``Mailserver = 'mail.server.com'`` 273 | * MailserverPort - mail server port number ``MailserverPort = 587`` 274 | * MailserverUseTLS - use TLS in mail server? ``MailserverUseTLS = True`` 275 | * MailserverUseSSL - use SSL in mail server? ``MailserverUseSSL = False`` 276 | * MailserverUser - user name ``MailserverUser = 'UserName`` 277 | * MailserverPWD - user password ``MailserverPWD = r'userPassword'`` 278 | * MailFromAdress - e-mail adress from which to send the e-mail ``MailFromAdress = 'userAdress@mail.com'`` 279 | 280 | Send e-mail:: 281 | 282 | from GISPython import MailHelper 283 | 284 | mailSender = MailHelper.MailHelper('mail@from.com', ['***@mail.com'], 'Subject', u'e-mail content') # Set up the e-mail for sending 285 | mailSender.SendMessage('smtp.server.com', 587, 'username', 'password', useTLS=True) # Send e-mail 286 | 287 | MyError 288 | ------- 289 | 290 | *Module contains class for storing an error object.* 291 | 292 | .. automodule:: MyError 293 | :members: 294 | :undoc-members: 295 | :show-inheritance: 296 | 297 | Examples 298 | ******** 299 | Raise error in the tool output:: 300 | 301 | from geopythoncore import MyError 302 | 303 | l = 12 304 | linecount = 10 305 | if linecount != l-2: 306 | raise MyError.MyError(u'Error in line count') # Add the error in the tool output 307 | 308 | PublisherHelper 309 | --------------- 310 | 311 | *Module for deployment operations.* 312 | 313 | .. automodule:: PublisherHealper 314 | :members: 315 | :undoc-members: 316 | :show-inheritance: 317 | 318 | RarHelper 319 | --------- 320 | 321 | .. automodule:: RarHelper 322 | :members: 323 | :undoc-members: 324 | :show-inheritance: 325 | 326 | Examples 327 | ******** 328 | Extract the Rar file:: 329 | 330 | from GISPython import RarHelper 331 | 332 | # File extraction procedure 333 | RH = RarHelper.RarHelper() 334 | rarFile = 'c:\\tmp\fileName.rar' # Rar file full path 335 | workDir = 'c:\\tmp\someDirectory' # Directory in which to extract the Zip file 336 | RH.ExtractRarFile(rarFile, workDir) # Extraction procedure 337 | 338 | 339 | SFTPHelper 340 | ---------- 341 | 342 | .. automodule:: SFTPHelper 343 | :members: 344 | :undoc-members: 345 | :show-inheritance: 346 | 347 | Examples 348 | ******** 349 | 350 | Function which compresses files and sends to SFTP server:: 351 | 352 | from GISPython import SFTPHelper 353 | from GISPython import ZipHelper 354 | 355 | tmpFolder = os.path.join(Pr.TmpFolder, 'DataFolder') 356 | WorkDBName = 'DBName' 357 | WorkDB = callGP('CreateFileGDB_management', tmpFolder, WorkDBName) 358 | zipFileName = os.path.join(tmpFolder, WorkDBName + self.Tool.MyNowFile() + '.zip') 359 | ZIP = ZipHelper.ZipHelper() 360 | ZIP.CompressDir(WorkDB, zipFileName, ['lock']) # Compress directory contents 361 | 362 | # Call parameters from the external parameter file 363 | SFTP = SFTPHelper.SFTPHelper(Pr.SFTPUser, Pr.SFTPPwd, Pr.SFTPHost, Pr.SFTPPort) 364 | SFTP.upload(zipFileName, os.path.basename(zipFileName)) # Upload file to SFTP 365 | 366 | SimpleFileOps 367 | ------------- 368 | 369 | *File and filesystem operations module. Module contains functions for typical file and filesystem operations, and locking control and processing. 370 | This module uses windows PowerShell to address file locking situations. For module with the same functionality without PowerShell script usage use *SimpleFileOpsSafe* module.* 371 | 372 | .. automodule:: SimpleFileOps 373 | :members: 374 | :undoc-members: 375 | :show-inheritance: 376 | 377 | Examples 378 | ******** 379 | Check for the directory, and clear its contents:: 380 | 381 | from GISPython import SimpleFileOps 382 | 383 | FO = SimpleFileOps.SimpleFileOps(self.Tool) 384 | workDir = pj(Pr.TmpFolder, 'WorkingDirectory') # Set the working directory 385 | FO.CheckCreateDir(workDir) # Check if directory exists, if not, create the directory 386 | FO.ClearDir(workDir) # Clear directory contents 387 | 388 | Find the newest file in the directory and create a backup:: 389 | 390 | from GISPython import SimpleFileOps 391 | 392 | FO = SimpleFileOps.SimpleFileOps(self.Tool) 393 | workDir = pj(Pr.TmpFolder, 'WorkingDirectory') # Set the working directory 394 | backupDir = pj(Pr.TmpFolder, 'BackupDirectory') # Set the working directory 395 | newFile = FO.FindNewestFile(workDir, '*') # Find newest file with any extension 396 | FO.BackupOneFile(newFile, backupDir) 397 | 398 | SimpleFileOpsSafe 399 | ----------------- 400 | 401 | *File and filesystem operations module. Module contains SimpleFileOpsSafe class, which contains functions for typical file and filesystem operations. Class inherits SimpleFileOps functionality, but does not relay on Windows Powershell scripts to address locked files.* 402 | 403 | .. automodule:: SimpleFileOpsSafe 404 | :members: 405 | :undoc-members: 406 | :show-inheritance: 407 | 408 | TimerHelper 409 | ----------- 410 | 411 | *Timing module. Module contains functions for countdown procedures in a code block.* 412 | 413 | .. automodule:: TimerHelper 414 | :members: 415 | :undoc-members: 416 | :show-inheritance: 417 | 418 | Examples 419 | ******** 420 | Add a message to the tool output:: 421 | 422 | from GISPython import TimerHelper 423 | 424 | with TimerHelper.TimedSubprocess(self.Tool, u'some process'): 425 | # Your code 426 | self.Tool.AddMessage(u'some action') 427 | 428 | Output:: 429 | 430 | ------------------------ 431 | >>>> Begin some process - yyyy-mm-dd hh:mm:ss 432 | ------------------------ 433 | some action 434 | 435 | ------------------------ 436 | >>>> End some process - h:mm:ss.ssssss 437 | ------------------------ 438 | 439 | xmlParamsHelper 440 | ---------------- 441 | 442 | *Module for XML parameter file procedures*. 443 | 444 | .. automodule:: xmlParamsHealper 445 | :members: 446 | :undoc-members: 447 | :show-inheritance: 448 | 449 | ZipHelper 450 | --------- 451 | 452 | *Zip file operations module. Module contains functions for common Zip file operations.* 453 | 454 | .. automodule:: ZipHelper 455 | :members: 456 | :undoc-members: 457 | :show-inheritance: 458 | 459 | Examples 460 | ******** 461 | Archiving procedure:: 462 | 463 | from GISPython import ZipHelper 464 | 465 | ZH = ZipHelper.ZipHelper() 466 | workDir = 'c:\\tmp\someDirectory' # Directory to archive 467 | zipFile = 'c:\\tmp\fileName' + self.Tool.MyNowFileSafe() + '.zip' # New zip file with formatted date (simple example) 468 | zipFile = 'c:\\tmp\fileName{0}.zip'.format(self.Tool.MyNowFileSafe()) # New zip file with formatted date (good example) 469 | ZH.CompressDir(workDir, zipFile) 470 | 471 | Extraction procedure:: 472 | 473 | from GISPython import ZipHelper 474 | 475 | ZH = ZipHelper.ZipHelper() 476 | workDir = 'c:\\tmp\someDirectory' # Directory in which to extract the Zip file 477 | zipFile = 'c:\\tmp\fileName{0}.zip'.format(self.Tool.MyNowFileSafe()) # Zip file with formatted date 478 | ZH.ExtractZipFile(zipFile, workDir) 479 | 480 | .. toctree:: 481 | :maxdepth: 4 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ![img](http://www.lvmgeo.lv/images/lvmgeo/backgrouds/icons/python_small.svg) LVM GEO Python Core (_GISPython_) 2 | 3 | LVM GEO Python Core (_GISPython_) is an open source automation and scripting core developed by the LVM GEO team. Based on this core any developer can build simple and structured Python programming scripts with a rich functionality. The programming core allows management of all system's maintenance processes within a unified environment. 4 | 5 | There are many automated maintenance operations necessary for every large geospatial information system (GIS). These operations are required for database and server maintenance, data validations and calculations, map preparing and caching, or data exchange with other systems. Within ERSI platform maintenance scripting is done by Python programming language and ArcPy library. The LVM GEO team has worked with ArcPy library for many years and has developed the LVM GEO Python Core complementing and enriching the Platform maintenance possibilities of the ESRI library that are not provided in the ArcPy standard: 6 | * monitoring of automated scripts (for example Zabbix) 7 | * script audit storage 8 | * generation of automated e-mails (script progress status reports as well as automated data validation warning e-mails, etc.) 9 | * data transfer using FTP and SFTP, data compressing and uncompressing 10 | * SQL, PowerShell, Windows Shell file initiation and progress monitoring within an unified environment with ArcPy geoprocessing tools 11 | 12 | The Core also includes tools that simplify usage of ArcPy and Python functions, significantly easing the development process: 13 | * scripting of ArcGIS server administration 14 | * scripting of file operations 15 | * scripting of service caching 16 | * unified script initiation from ArcGIS environment, Python Shell and from other tools 17 | * etc. 18 | 19 | The LVM GEO Python Core is already being used by several companies in Latvia, including JSC Latvia's State Forests for more than 200 automated processes every day. LVM GEO offers courses about LVM GEO Python to support development of an automation platform for companies and organizations. 20 | 21 | ### _GISPython_ package contains following modules: 22 | * [**Main modules**](http://gispython.readthedocs.io/en/latest/main_modules.html): 23 | * [GISPythonModule](http://gispython.readthedocs.io/en/latest/main_modules.html#gispythonmodule) 24 | * [GISPythonTool](http://gispython.readthedocs.io/en/latest/main_modules.html#gispythontool) 25 | * [GISPythonToolBase](http://gispython.readthedocs.io/en/latest/main_modules.html#gispythontoolbase) 26 | * [SysGISTools](http://gispython.readthedocs.io/en/latest/main_modules.html#sysgistools) 27 | * [SysTools_unittest](http://gispython.readthedocs.io/en/latest/main_modules.html#systools_unittest) 28 | * [**Helper modules**](http://gispython.readthedocs.io/en/latest/helper_modules.html): 29 | * [AGServerHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#agserverhelper) 30 | * [AGServerHelperNTLM](http://gispython.readthedocs.io/en/latest/helper_modules.html#agserverhelperntlm) 31 | * [CachingHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#cachinghelper) 32 | * [FTPHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#ftphleper) 33 | * [GDBHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#gdbhelper) 34 | * [GDPSynchroniserHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#module-GISPython.GDPSyncroniserHelper) 35 | * [GDPSynchroniserHelper2](http://gispython.readthedocs.io/en/latest/helper_modules.html#module-GISPython.GDPSyncroniserHelper2) 36 | * [JsonParamsHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#jsonparamshelper) 37 | * [MailHelper](https://github.com/lvmgeo/GISPython#mailhelper) 38 | * [MyError](http://gispython.readthedocs.io/en/latest/helper_modules.html#myerror) 39 | * [PublisherHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#publisherhelper) 40 | * [RarHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#rarhelper) 41 | * [SFTPHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#sftphelper) 42 | * [SimpleFileOps](http://gispython.readthedocs.io/en/latest/helper_modules.html#simplefileops) 43 | * [SimpleFileOpsSafe](http://gispython.readthedocs.io/en/latest/helper_modules.html#simplefileopssafe) 44 | * [TimerHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#timerhelper) 45 | * [xmlParamsHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#xmlparamshelper) 46 | * [ZipHelper](http://gispython.readthedocs.io/en/latest/helper_modules.html#ziphelper) 47 | 48 | ## Dependencies 49 | 50 | * ArcGIS 10.x /recommended with newest patches and service packs/ (**_GISPython_** is currently running on production systems based on ArcGIS 10.2.1, ArcGIS 10.3.1 and has been tested on ArcGIS 10.4) 51 | * Python 2.7 (included in ArcGIS installation) (arcpy and numpy modules included) 52 | * Additional python modules: 53 | * [PyCrypto](http://www.voidspace.org.uk/python/modules.shtml#pycrypto) (manual installation) 54 | * NTLM: `pip install python-ntlm` (included in package setup process) 55 | * paramiko: `pip install paramiko` (included in package setup process) 56 | * patool: `pip install patool` (included in package setup process) 57 | * simpleJson: `pip install simplejson` (included in package setup process) 58 | 59 | >If pip isn’t installed, you can get it [**here**](https://packaging.python.org/installing/#install-pip-setuptools-and-wheel)! 60 | 61 | ## Installation 62 | 63 | **_GISPython_** is available on the [Python Package Index](https://pypi.python.org/pypi/GISPython), so you can get it via pip: `pip install GISPython`. 64 | Before installing it is recomended to upgrade pip to latest version using `pip install --upgrade pip`. 65 | To upgrade **_GISPython_** use `pip install gispython --upgrade`. 66 | 67 | ## Configuration & basic usage 68 | 69 | Before using **_GISPython_** modules in custom geoprocessing scripts, you need to set up your scripting environment with [_SetupDefaultEnvironment_](https://github.com/lvmgeo/GISPython/blob/master/GISPython/SetupDefaultEnvironment.py) module which also includes template for user scripts. 70 | 71 | _SetupDefaultEnvironment_ module also includes basic parameters (variable _paramsFileSource_) for parameter file (e.g. SysGISParams.py) which is important, because **_GISPython_** relies of several parameters to be present to function successfully: 72 | * OutDir - directory for storing script output log files `OutDir = r'C:\GIS\Log\Outlog\' ` 73 | * OutDirArh - directory for storing script output log file archive (all non active files) `OutDirArh = r'C:\GIS\Log\Outlog\Archive\' ` 74 | * ErrorLogDir - directory for storing script error log files `ErrorLogDir = r'C:\GIS\Log\ErrorLog\' `(*Important! This directory can be monitored for non empty files. If this directory has a file that is non empty - this indicates that a script has failed*) 75 | * ErrorLogDirArh - directory for storing script error log files `ErrorLogDirArh = r'C:\GIS\Log\ErrorLog\Archive' ` 76 | * TmpFolder - Temp folder `TmpFolder = r'C:\GIS\tmp'` 77 | * encodingPrimary - encoding of Windows shell `encodingPrimary = 'cp775' ` 78 | * encodingSecondary - encoding of Windows unicode language used `encodingSecondary = 'cp1257' ` 79 | * SetLogHistory - enable or disable Geoprocessing history logging `SetLogHistory = False` 80 | 81 | >It is recommended to define additional script parameters in SysGISParams.py file, to keep the main code clean. Our approach is to define all the parameters that define current system environment be kept in this one file. In case of moving environment (e.g. test system and production system) this one file has the specific connections and can be easily modified without changing the scripts. 82 | 83 | ### Recommendations 84 | Set up the variables at the beggining of the main function, to shorten the main code: 85 | ```python 86 | Tool = self.Tool 87 | gp = Tool.gp 88 | callGP = Tool.callGP 89 | pj = os.path.join 90 | ``` 91 | 92 | ### Basic operations: 93 | ArcPy function call: 94 | ```python 95 | gpCaller = self.Tool.callGP 96 | slay = 'some_layer' 97 | callGP('AddField_management', slay, 'Day_TXT', 'TEXT', '#', '#', 10) 98 | callGP('AddField_management', slay, 'CAR', 'TEXT', '#', '#', 128) 99 | callGP('AddField_management', slay, 'WorkID', 'DOUBLE', 12, 0) 100 | callGP('AddField_management', slay, 'REC_DATE_FROM', 'DATE') 101 | ``` 102 | Tool message output: 103 | ```python 104 | Tool = self.Tool 105 | self.Tool.AddMessage(u'This is a message') 106 | self.Tool.AddWarning(u'This is a warning') 107 | self.Tool.AddError(u'This is an error') 108 | ``` 109 | 110 | ## Documentation 111 | 112 | Documentation includes information about all GISPython modules & examples of use. Documentation can be found on: http://gispython.readthedocs.io. 113 | 114 | ## Examples 115 | 116 | ### Main modules 117 | 118 | #### _GISPythonModule_ 119 | Main module, which contains frame for all GISPython package modules. Module allows the code unification, and ensures the code execution from both the ArcGIS Desktop Python console and the Command Prompt. 120 | 121 | _Example:_ 122 | ###### Executes the tool if it's to be executed within an another Python tool 123 | ```Python 124 | from GISPython import TimerHelper 125 | import OnlineCompress 126 | with TimerHelper.TimedSubprocess(Tool, u'Compress DB'): # Adds a message to the tool output 127 | with TimerHelper.TimedSubprocess(Tool, u'disconnect users from DB', 2): # Adds a message to the tool output 128 | self.Tool.RunSQL('KillUsers', Pr.u_sde, Pr.p_sde) # Runs custom SQL script 129 | OnlineCompress.MainModule(None, False).runInsideJob(Tool) # Runs SDE Compression procedure from OnlineCompress module (custom geoprocessing module) 130 | ``` 131 | 132 | #### _GISPythonTool_ 133 | Module defines abstract classes for the ESRI Toolbox tool definition and contains functions which helps to create an ArcGIS Toolbox, validates the tool's parameter values and controls a behavior of the tool's dialog. 134 | 135 | _Example:_ 136 | ###### Define parameters in ESRI Python toolbox 137 | ```Python 138 | from GISPython import GISPythonTool 139 | 140 | class ToolAtributeValidator(GISPythonTool.GISPythonTool): 141 | def __init__(self): 142 | """Define tool (tool name is the class name)""" 143 | self.label = u"Tool for attribute data validation (test mode)" 144 | self.description = u"Tool for attribute data validation (test mode)" 145 | self.category = 'GEO Maintenance' 146 | 147 | def getParameterInfo(self): 148 | """Define the parameters""" 149 | param_0 = arcpy.Parameter( 150 | displayName=r'Key:', 151 | name=u"key", 152 | datatype=u"String", 153 | parameterType=u"Required", 154 | direction=u"Input") 155 | 156 | ret_val = [param_0] 157 | return ret_val 158 | 159 | def execute(self, parameters, messages): 160 | """Tool execution""" 161 | AtributeValidator.MainModule(parameters[0].valueAsText).DoJob() 162 | return 163 | ``` 164 | 165 | #### _SysGISTools_ 166 | Base module which contains GISPython scripting framework, logging, error processing and automation mechanisms for different operations, and module and interface base classes. 167 | 168 | _Examples:_ 169 | ###### Run a script from Shell with parameters 170 | ```Python 171 | Tool = self.Tool 172 | # Executes your custom process script (mainly maintenance scripts) within runShell function, 173 | # which implements _runProcess function (message output in terminal window). 174 | # Function executes script seperately from main process (Detached=True) and indicates, 175 | # that the errors doesn't have to be logged (noErr=True). 176 | Tool.runShell('SomeProcess.py', Detached = True, noErr = True) 177 | time.sleep(10) # after 10 seconds launch another runShell process 178 | ``` 179 | 180 | ###### Executes a custom SQL script file (only Oracle sqlplus supported) 181 | ```Python 182 | from GISPython import TimerHelper 183 | Tool = self.Tool 184 | # Executes process from SQL file 185 | with TimerHelper.TimedSubprocess(self.Tool, u'datu atlasi no nogabaliem'): # Adds a message to the tool output 186 | Tool.RunSQL('LoadSomeDataFromTable') # Runs SQL file within RunSQL function, which implements GetSQL function (gets the SQL file location) 187 | ``` 188 | 189 | ###### Duplicates path seperator symbol '\' for external execution compatibility 190 | ```Python 191 | from GISPython import SimpleFileOps 192 | from GISPython import TimerHelper 193 | Tool = self.Tool 194 | #... 195 | with TimerHelper.TimedSubprocess(Tool, u'prepare environment'): # Adds a message to the tool output 196 | DirHelper = SimpleFileOps.SimpleFileOps(Tool) # Set variable for SimpleFileOps module functions 197 | tmpFolder = os.path.join(Pr.TmpFolder, "Soil") # Set tmp folder path 198 | #... 199 | Tool.AddMessage(u'\n ...delete previous tmp data') # Add tool output message 200 | DirHelper.CheckCreateDir(Tool.CorrectStr(tmpFolder)) # Check/create tmp directory (with modified path seperators) 201 | DirHelper.ClearDir(Tool.CorrectStr(tmpFolder)) # Clear tmp directory (with modified path seperators) 202 | ``` 203 | 204 | ### Helper modules 205 | 206 | #### _MailHelper_ 207 | Module for e-mail operations. Module contains functions for typical SMTP operations, and parameter processing from user parameter file. 208 | 209 | _Examples:_ 210 | ###### Send e-mail using parameters from parameter file 211 | ```python 212 | from GISPython import MailHelper 213 | MailHelper.GISPythonMailHelper(self.Pr, ['***@mail.com'], 'Subject', 'e-mail content') 214 | ``` 215 | This script depends on following parameters which needs to be configured in _SysGISParams.py_ file: 216 | * Mailserver - mail server name `Mailserver = 'mail.server.com'` 217 | * MailserverPort - mail server port number `MailserverPort = 587` 218 | * MailserverUseTLS - use TLS in mail server? `MailserverUseTLS = True` 219 | * MailserverUseSSL - use SSL in mail server? `MailserverUseSSL = False` 220 | * MailserverUser - user name `MailserverUser = 'UserName` 221 | * MailserverPWD - user password `MailserverPWD = r'userPassword'` 222 | * MailFromAdress - e-mail adress from which to send the e-mail `MailFromAdress = 'userAdress@mail.com'` 223 | 224 | ###### Send e-mail 225 | ```python 226 | from GISPython import MailHelper 227 | mailSender = MailHelper.MailHelper('mail@from.com', ['***@mail.com'], 'Subject', u'e-mail content') # Set up the e-mail for sending 228 | mailSender.SendMessage('smtp.server.com', 587, 'username', 'password', useTLS=True) # Send e-mail 229 | ``` 230 | 231 | #### _SimpleFileOps_ 232 | File and filesystem operations module. Module contains functions for typical file and filesystem operations, and locking control and processing. 233 | This module uses windows PowerShell to address file locking situations. For module with the same functionality without PowerShell script usage use _SimpleFileOpsSafe_. 234 | 235 | _Examples:_ 236 | ###### Check for the directory, and clear its contents 237 | ```python 238 | from GISPython import SimpleFileOps 239 | FO = SimpleFileOps.SimpleFileOps(self.Tool) 240 | workDir = pj(Pr.TmpFolder, 'WorkingDirectory') # Set the working directory 241 | FO.CheckCreateDir(workDir) # Check if directory exists, if not, create the directory 242 | FO.ClearDir(workDir) # Clear directory contents 243 | ``` 244 | 245 | ###### Find the newest file in the directory and create a backup 246 | ```python 247 | from GISPython import SimpleFileOps 248 | FO = SimpleFileOps.SimpleFileOps(self.Tool) 249 | workDir = pj(Pr.TmpFolder, 'WorkingDirectory') # Set the working directory 250 | backupDir = pj(Pr.TmpFolder, 'BackupDirectory') # Set the working directory 251 | newFile = FO.FindNewestFile(workDir, '*') # Find newest file with any extension 252 | FO.BackupOneFile(newFile, backupDir) 253 | ``` 254 | 255 | #### _TimerHelper_ 256 | Timing module. Module contains functions for countdown procedures in a code block. 257 | 258 | _Example:_ 259 | ###### Add a message to the tool output 260 | ```python 261 | from GISPython import TimerHelper 262 | with TimerHelper.TimedSubprocess(self.Tool, u'some process'): 263 | self.Tool.AddMessage(u'some action') 264 | ``` 265 | ![img_time.png](https://github.com/lvmgeo/GISPython/blob/master/img/img_time.png) 266 | 267 | #### _ZipHelper_ 268 | Zip file operations module. Module contains functions for common Zip file operations. 269 | 270 | _Examples:_ 271 | ###### Archiving procedure 272 | ```python 273 | from GISPython import ZipHelper 274 | ZH = ZipHelper.ZipHelper() 275 | workDir = 'c:\\tmp\someDirectory' # Directory to archive 276 | zipFile = 'c:\\tmp\fileName' + self.Tool.MyNowFileSafe() + '.zip' # New zip file with formatted date (simple example) 277 | zipFile = 'c:\\tmp\fileName{0}.zip'.format(self.Tool.MyNowFileSafe()) # New zip file with formatted date (good example) 278 | ZH.CompressDir(workDir, zipFile) 279 | ``` 280 | 281 | ###### Extraction procedure 282 | ```python 283 | from GISPython import ZipHelper 284 | ZH = ZipHelper.ZipHelper() 285 | workDir = 'c:\\tmp\someDirectory' # Directory in which to extract the Zip file 286 | zipFile = 'c:\\tmp\fileName{0}.zip'.format(self.Tool.MyNowFileSafe()) # Zip file with formatted date 287 | ZH.ExtractZipFile(zipFile, workDir) 288 | ``` 289 | 290 | ## Contributing 291 | We encourage developers to use LVM GEO Python Core code and also contribute to the project. LVM GEO team is open for cooperation in developing new solutions, provides LVM GEO Python courses and offers to implement a fully functional automation platform for companies and organizations for their business needs. 292 | 293 | ## Licensing 294 | 295 | [GPL-3.0+](https://choosealicense.com/licenses/gpl-3.0/) 296 | 297 | ## Copyright 298 | 299 | [![LVMGEO](https://github.com/lvmgeo/GISPython/blob/master/img/LVM_GEO-logo-PMS30.PNG)](http://www.lvmgeo.lv/en "Copyright (c) 2017 LVM GEO") 300 | -------------------------------------------------------------------------------- /GISPython/SimpleFileOps.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | File and filesystem operations module 4 | """ 5 | import os 6 | import shutil 7 | import unicodedata 8 | import codecs 9 | from datetime import datetime 10 | import simplejson as json 11 | 12 | class SimpleFileOps(object): 13 | """Class for easing typical file and filesystem operations""" 14 | def __init__(self, _Tool): 15 | """Class initialization procedure 16 | 17 | Args: 18 | self: The reserved object 'self' 19 | Tool: The GISTools10 object for data processing 20 | """ 21 | self.Tool = _Tool 22 | 23 | def ClearDir(self, DirName, searchPatern='*'): # Overriding 24 | """Directory cleaning automation procedure 25 | 26 | Args: 27 | self: The reserved object 'self' 28 | DirName: The directory to be cleaned 29 | """ 30 | self.Tool.AddMessage(u"\n----------- Cleaning directory [" + DirName + u"] searching: {" + searchPatern + "} -------" + self.Tool.MyNow()) 31 | self.Tool.RunPS("ClearDir", '"' + self.Tool.CorrectStr(DirName) + '" ' + searchPatern, os.path.dirname(__file__)) 32 | 33 | def delFileIfExists(self, fileName): # Overriding 34 | """"Deletes file if file exists 35 | 36 | Args: 37 | self: The reserved object 'self' 38 | DirName: The directory to be cleaned 39 | 40 | """ 41 | if os.path.exists(fileName): 42 | os.remove(fileName) 43 | 44 | def DelClearDir(self, DirName, searchPatern='*'): # Overriding 45 | """Delete non-empty directory 46 | 47 | Args: 48 | self: The reserved object 'self' 49 | DirName: The directory to be deleted 50 | """ 51 | self.Tool.AddMessage(u"\n----------- Cleaning and deleting directory [" + DirName + "] -------" + self.Tool.MyNow()) 52 | self.Tool.RunPS("ClearDir", '"' + self.Tool.CorrectStr(DirName) + '" ' + searchPatern, os.path.dirname(__file__)) 53 | os.rmdir(DirName) 54 | self.Tool.AddMessage(u" ... directory [" + DirName + u"] deleted ") 55 | 56 | def BackupFiles(self, InDirName, OutDirName, D): # Overriding 57 | """File archiving automation procedure (Overriding) 58 | 59 | Args: 60 | self: The reserved object 'self' 61 | InDirName: Input directory 62 | OutDirName: Output directory 63 | D: How old files to archive (number of days) 64 | """ 65 | self.Tool.RunPS("BackupOldFiles", '"' + self.Tool.CorrectStr(InDirName) + '" "' + self.Tool.CorrectStr(OutDirName) + '" ' + str(D), os.path.dirname(__file__)) 66 | 67 | def GetLog(self, server, EventLogOutputDir, D): # NoneOverriding 68 | """File archiving automation procedure (None overriding) 69 | 70 | Args: 71 | self: The reserved object 'self' 72 | server: Server which eventlog backup to create 73 | EventLogOutputDir: Event log output directory 74 | D: How old files to archive (number of days) 75 | """ 76 | self.Tool.RunPS("GetLog", '"' + server + '" "' + self.Tool.CorrectStr(EventLogOutputDir) + '" '+ str(D), os.path.dirname(__file__)) 77 | 78 | def BackupOneFile(self, InFileName, OutDirName): # Overriding 79 | """Specific file archiving automation procedure 80 | 81 | Args: 82 | self: The reserved object 'self' 83 | InFileName: Input file 84 | OutDirName: Output directory 85 | """ 86 | self.Tool.AddMessage("") 87 | self.Tool.AddMessage("--- Archiving file " + InFileName + " to " + OutDirName + " - " + self.Tool.MyNow()) 88 | if os.path.exists(InFileName): 89 | shutil.move(InFileName, OutDirName + '\\' + os.path.split(InFileName)[1].split(".")[-2] + "_" + self.Tool.MyNowFile() + os.path.splitext(InFileName)[1]) 90 | else: 91 | self.Tool.AddMessage(" File " + InFileName + ' does not exist ...') 92 | 93 | def CheckCreateDir(self, OutDirName): 94 | """Automation procedure which creates directory, in case it doesn't exist 95 | 96 | Args: 97 | self: The reserved object 'self' 98 | OutDirName: Output directory 99 | """ 100 | if not os.path.exists(OutDirName): 101 | os.makedirs(OutDirName) 102 | 103 | def CheckCreateClearDir(self, DirName): 104 | """Automation procedure which creates directory, in case it doesn't exist and if it exists then clear this dir 105 | 106 | Args: 107 | self: The reserved object 'self' 108 | DirName: Output directory 109 | """ 110 | self.CheckCreateDir(DirName) 111 | self.ClearDir(DirName) 112 | 113 | def GetSafeName(self, text, substituteChar='_', aditionalScaryChars='', aditionalSpaceChars='', noDotsInName=False): 114 | """Modifies the text for use in the filesystem 115 | 116 | Args: 117 | self: The reserved object 'self' 118 | text: Text string which will be transformed 119 | substituteChar: Character to substitute the whitespace 120 | aditionalScaryChars: Additional characters to eliminate 121 | aditionalSpaceChars: Additional characters to replace 122 | noDotsInName: Forbid dots in filename (except the file extension seperator) (Default = False) 123 | 124 | Returns: 125 | * Modified text as string 126 | """ 127 | if type(text) is unicode: 128 | text = unicodedata.normalize('NFD', text).encode('ascii', 'ignore') 129 | checkChars = ''.join(set(r'[]/\;,><&*:%=+@!#^()|?^-' + aditionalScaryChars)) 130 | for c in checkChars: 131 | text = text.replace(c, '') 132 | checkSpaceChars = ''.join(set(' ' + aditionalSpaceChars)) 133 | for c in checkSpaceChars: 134 | text = text.replace(c, substituteChar) 135 | if noDotsInName: 136 | lastDot = text.rfind('.') 137 | if lastDot != -1: 138 | text = (text[:lastDot]).replace('.', substituteChar) + text[lastDot:] 139 | while text.find(substituteChar+substituteChar) > -1: 140 | text = text.replace(substituteChar+substituteChar, substituteChar) 141 | return text 142 | 143 | def FindNewestFile(self, Dir, Ext='*'): 144 | """Finds the newest file in the directory 145 | 146 | Args: 147 | self: The reserved object 'self' 148 | Dir: The directory in which to look for the file 149 | Ext: The extension to search for ('*' - search any file) 150 | """ 151 | if not Ext == '*': 152 | dated_files = [(os.path.getmtime(Dir + "\\" + fn), os.path.basename(Dir + "\\" + fn)) 153 | for fn in os.listdir(Dir) 154 | ] 155 | else: 156 | dated_files = [(os.path.getmtime(Dir + "\\" + fn), os.path.basename(Dir + "\\" + fn)) 157 | for fn in os.listdir(Dir) 158 | if fn.lower().endswith('.' + (Ext.lower())) 159 | ] 160 | dated_files.sort() 161 | dated_files.reverse() 162 | newest = dated_files[0][1] 163 | return Dir + "\\" + newest 164 | 165 | def FindFileByDate(self, Dir, Ext='*', Date=datetime.now(), Mode='New'): 166 | """Find files in the given directory which are newer than the given date 167 | 168 | Args: 169 | self: The reserved object 'self' 170 | Dir: The directory in which to look for the file 171 | Ext: The extension to search for ('*' - search any file) 172 | Date: Find files newer than given date 173 | Mode: File searching modes: New - search newer files; Old - search older files (Default = New) 174 | """ 175 | if Ext == '*': 176 | dated_files = [Dir + "\\" + fn 177 | for fn in os.listdir(Dir) 178 | if ((Mode == 'New' and datetime.fromtimestamp(os.path.getmtime(Dir + "\\" + fn)) >= Date) or 179 | (Mode == 'Old' and datetime.fromtimestamp(os.path.getmtime(Dir + "\\" + fn)) <= Date)) 180 | ] 181 | else: 182 | dated_files = [Dir + "\\" + fn 183 | for fn in os.listdir(Dir) 184 | if ((Mode == 'New' and datetime.fromtimestamp(os.path.getmtime(Dir + "\\" + fn)) >= Date) or 185 | (Mode == 'Old' and datetime.fromtimestamp(os.path.getmtime(Dir + "\\" + fn)) <= Date)) 186 | and fn.lower().endswith('.' + (Ext.lower())) 187 | ] 188 | dated_files.sort() 189 | return dated_files 190 | 191 | def FindFile(self, Dir, Ext): 192 | """Finds files in the directory 193 | 194 | Args: 195 | self: The reserved object 'self' 196 | Dir: The directory in which to look for the file 197 | Ext: The extension to search for 198 | """ 199 | if Ext == '*': 200 | files = [os.path.join(Dir,fn) 201 | for fn in os.listdir(Dir) if os.path.isfile(os.path.join(Dir,fn))] 202 | else: 203 | files = [os.path.join(Dir,fn) 204 | for fn in os.listdir(Dir) if fn.lower().endswith('.' + (Ext.lower()))] 205 | files.sort() 206 | return files 207 | 208 | def FindDirectory(self, Dir, Searchpattern='*'): 209 | """Find subdirectories in the given directory 210 | 211 | Args: 212 | self: The reserved object 'self' 213 | Dir: The directory in which to look for subdirectories 214 | Searchpattern: Searching condition 215 | """ 216 | if Searchpattern == '*': 217 | dirs = [fn for fn in os.listdir(Dir) if os.path.isdir(os.path.join(Dir,fn))] 218 | else: 219 | dirs = [fn for fn in os.listdir(Dir) if os.path.isdir(os.path.join(Dir,fn)) 220 | and not fn.lower().find(Searchpattern.lower()) == -1] 221 | dirs.sort() 222 | return dirs 223 | 224 | def FindFileRecursive(self, Dir, Ext): 225 | """Find files by extension 226 | 227 | Args: 228 | self: The reserved object 'self' 229 | Dir: The directory in which to look for the file 230 | Ext: The extension to search for 231 | """ 232 | dated_files = [] 233 | for root, dirnames, filenames in os.walk(Dir): 234 | for filename in filenames: 235 | if filename.lower().endswith('.' + (Ext.lower())): 236 | dated_files.append(os.path.join(root, filename)) 237 | dated_files.sort() 238 | return dated_files 239 | 240 | def CopyAllFilesInDir(self, SourceDir, DestDir, Searchpattern='*', Ignorepattern=None): 241 | """Copy entire directory tree from one directory to another 242 | 243 | Args: 244 | self: The reserved object 'self' 245 | SourceDir: Source directory 246 | DestDir: Destination directory 247 | Searchpattern: Searching condition 248 | """ 249 | if not os.path.exists(DestDir): 250 | os.makedirs(DestDir) 251 | 252 | if Searchpattern == '*' and Ignorepattern==None: 253 | foundfiles = [fn for fn in os.listdir(SourceDir)] 254 | elif Searchpattern != '*' and Ignorepattern==None: 255 | foundfiles = [fn for fn in os.listdir(SourceDir) if not fn.lower().find(Searchpattern.lower()) == -1] 256 | elif Searchpattern == '*' and Ignorepattern!=None: 257 | foundfiles = [fn for fn in os.listdir(SourceDir) if fn.lower().find(Ignorepattern.lower()) == -1] 258 | elif Searchpattern != '*' and Ignorepattern!=None: 259 | foundfiles = [fn for fn in os.listdir(SourceDir) if not fn.lower().find(Searchpattern.lower()) == -1 and fn.lower().find(Ignorepattern.lower()) == -1] 260 | foundfiles.sort() 261 | 262 | for item in foundfiles: 263 | s = os.path.join(SourceDir, item) 264 | d = os.path.join(DestDir, item) 265 | if os.path.isdir(s): 266 | self.CopyAllFilesInDir(s, d, Searchpattern, None) 267 | else: 268 | shutil.copy2(s, d) 269 | 270 | def printFile(self, filePath): 271 | """Print file content to the screen 272 | 273 | Args: 274 | self: The reserved object 'self' 275 | filePath: File to print 276 | """ 277 | with codecs.open(filePath, 'r', 'utf8') as fin: 278 | self.Tool.AddMessage(fin.read()) 279 | 280 | def GetfileNameWithDate(self, file): 281 | """Add a date at the end of the filename 282 | 283 | Args: 284 | self: The reserved object 'self' 285 | file: Full path to the file to add the date 286 | """ 287 | dirname = os.path.dirname(file) 288 | filename = ('.').join(file.split('.')[:-1]) 289 | fileext = ('.').join(file.split('.')[-1:]) 290 | 291 | return os.path.join(dirname, filename + '_' + self.Tool.MyNowFileSafe() + '.' + fileext) 292 | 293 | def CopareFileLists(self, SourceFiles, DestFiles): 294 | """Compare two file lists to determine changes 295 | 296 | Args: 297 | SourceFiles (List): Sorce file path list 298 | DestFiles (List): Destination file path list 299 | 300 | Returns: 301 | Dictionary: 302 | AbsentSource (List): Files in Source and not in Destination 303 | AbsentDestination (List): Files in Destination and not in Source 304 | """ 305 | AbsentSource = list() 306 | AbsentDestination = list() 307 | 308 | SourceFileDict = dict() 309 | for sFile in SourceFiles: 310 | basename = os.path.basename(sFile) 311 | ext = os.path.splitext(basename)[1] 312 | modifyied = datetime.fromtimestamp(os.path.getmtime(sFile)) 313 | SourceFileDict[basename] = {"path": sFile, "ext": ext, "modifyied": modifyied} 314 | 315 | DestFileDict = dict() 316 | for dFile in DestFiles: 317 | basename = os.path.basename(dFile) 318 | ext = os.path.splitext(basename)[1] 319 | modifyied = datetime.fromtimestamp(os.path.getmtime(dFile)) 320 | DestFileDict[basename] = {"path": sFile, "ext": ext, "modifyied": modifyied} 321 | 322 | for sFile in SourceFileDict.keys(): 323 | if not DestFileDict.has_key(sFile): 324 | AbsentDestination.append(SourceFileDict[sFile]["path"]) 325 | else: 326 | # print SourceFileDict[sFile]["modifyied"] 327 | # print DestFileDict[sFile]["modifyied"] 328 | if SourceFileDict[sFile]["modifyied"].replace(microsecond=0)>DestFileDict[sFile]["modifyied"].replace(microsecond=0): 329 | AbsentDestination.append(SourceFileDict[sFile]["path"]) 330 | 331 | for dFile in DestFileDict.keys(): 332 | if not SourceFileDict.has_key(dFile): 333 | AbsentSource.append(DestFileDict[dFile]["path"]) 334 | 335 | return {"AbsentSource":AbsentSource, "AbsentDestination":AbsentDestination} 336 | 337 | class LockSubprocess(object): 338 | """Class that provides directory locking control and processing""" 339 | def __init__(self, Tool, Dir, processName): 340 | """Class initialization procedure 341 | Args: 342 | self: The reserved object 'self' 343 | Tool: Reference to the GISTools10 object 344 | Dir: Working directory 345 | processName: Name of the process 346 | """ 347 | self.StartTime = datetime.now() 348 | self.Tool = Tool 349 | self.Dir = Dir 350 | self.processName = processName 351 | self.results = LockResults(processName) 352 | self.lockFileName = os.path.join(self.Dir, 'gispython.{0}.lock'.format(self.processName)) 353 | 354 | def __enter__(self): 355 | """With statement opening procedure 356 | Args: 357 | self: The reserved object 'self' 358 | Returns: 359 | LockResults with status: 360 | Locked - The file is locked by another process; 361 | DoneBefore - Process is already done; 362 | NoDir - Directory not found; 363 | Running - Process is running; 364 | """ 365 | # Check if directory exists 366 | if os.path.exists(self.Dir): 367 | if os.path.exists(self.lockFileName) and os.path.isfile(self.lockFileName): 368 | try: 369 | self.f = codecs.open(self.lockFileName, 'r+', 'utf-8') 370 | self.readJson() 371 | except: 372 | self.results.status = "Locked" 373 | return self.results 374 | if self.J['status'] == 'Running': 375 | self.results.status = "Locked" 376 | return self.results 377 | if self.J['status'] == 'Done': 378 | self.results.status = "DoneBefore" 379 | return self.results 380 | else: 381 | self.results.status = "Running" 382 | self.writeJson() 383 | else: 384 | self.results.status = "Running" 385 | self.writeJson() 386 | else: 387 | self.results.status = "NoDir" 388 | return self.results 389 | 390 | def __exit__(self, type, value, traceback): 391 | """With statement closing procedure 392 | Args: 393 | self: The reserved object 'self' 394 | """ 395 | if self.results.status != "Locked": 396 | self.results.status = "Done" 397 | self.writeJson() 398 | if hasattr(self, 'f'): 399 | self.f.close() 400 | 401 | def readJson(self): 402 | """Get the data from the lock file 403 | Args: 404 | self: The reserved object 'self' 405 | """ 406 | 407 | JsonString = self.f.read() 408 | try: 409 | self.J = json.loads(JsonString) 410 | except: 411 | self.J = {'status':'Ok'} 412 | 413 | def writeJson(self): 414 | """Save parameters in the file 415 | Args: 416 | self: The reserved object 'self' 417 | """ 418 | rezult = {'status': self.results.status, 419 | 'processName':self.results.processName, 420 | 'processdate': str(self.results.processdate)} 421 | JsonString = json.dumps(rezult, sort_keys=True, indent=4 * ' ') 422 | if hasattr(self, 'f'): 423 | self.f.close() 424 | self.f = codecs.open(self.lockFileName, 'w', 'utf-8') 425 | self.f.truncate() 426 | self.f.write(JsonString) 427 | self.f.flush() 428 | 429 | class LockResults(object): 430 | """Class for saving the data processing results (for LockSubprocess)""" 431 | 432 | def __init__(self, processName): 433 | """Class initialization procedure 434 | 435 | Args: 436 | self: The reserved object 'self' 437 | processName: Name of process 438 | """ 439 | self.status = "Running" 440 | self.processdate = datetime.now() 441 | self.processName = processName 442 | 443 | def GetStatusTxt(self): 444 | """Get status description 445 | 446 | Args: 447 | self: The reserved object 'self' 448 | """ 449 | if self.status == "Running": 450 | return u'Process execution is running' 451 | if self.status == "Locked": 452 | return u'This process cannot be continued, because another process is already running' 453 | if self.status == "DoneBefore": 454 | return u'Process is already done' 455 | if self.status == "NoDir": 456 | return u'Corresponding directory not found' 457 | -------------------------------------------------------------------------------- /GISPython/GISPythonToolBase.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Base class for GISPython Tool object 3 | """ 4 | 5 | import os 6 | import datetime 7 | import sys 8 | import codecs 9 | import subprocess 10 | import shlex 11 | import shutil 12 | import locale 13 | import time 14 | import string 15 | import unicodedata 16 | import uuid 17 | from multiprocessing import Process, Queue 18 | 19 | class GISPythonToolBase(object): 20 | def __init__(self, ToolName, Params): 21 | self.ExecutePatch = os.path.dirname(os.path.realpath(__file__)) 22 | self.Pr = Params 23 | self.StartTime = datetime.datetime.now() 24 | self.ToolName = ToolName 25 | self.OutputStr = u'' 26 | self.OutputErrStr = u'' 27 | self.State = "Started" 28 | 29 | if hasattr(Params, 'encodingPrimary'): 30 | UnicodeWriter = codecs.getwriter(Params.encodingPrimary) 31 | sys.stdout = UnicodeWriter(sys.stdout) 32 | 33 | def MyEnd(self): 34 | """End of the process 35 | 36 | Args: 37 | self: The reserved object 'self' 38 | """ 39 | self.AddMessage(u'End tool ' + self.ToolName + u' execution ' + self.MyNow()) 40 | TD = datetime.datetime.now()- self.StartTime 41 | self.AddMessage(u' ...Tool execution time: ' + str(TD)) 42 | self.State = "Done" 43 | 44 | def MyDispose(self): 45 | """Disposal of the class 46 | 47 | Args: 48 | self: The reserved object 'self' 49 | """ 50 | if self.State == "Started": 51 | self.MyEnd() 52 | self.State = "Disposed" 53 | 54 | def AddMessage(self, strMessage, newline=True): 55 | """Procedure for a message output (screen, logfile and if necessary e-mail) 56 | 57 | Args: 58 | self: The reserved object 'self' 59 | strMessage: Output message text 60 | newline: Flag that marks that the output must be continued in a new line 61 | """ 62 | if strMessage == None: 63 | strMessage = '' 64 | if not isinstance(strMessage, unicode): 65 | if not isinstance(strMessage, str): 66 | strMessage = str(strMessage) 67 | if strMessage[-1:] == '\n': 68 | strMessage = strMessage[0:-1] 69 | 70 | print strMessage 71 | 72 | self.OutputStr += strMessage 73 | if newline == True: 74 | self.OutputStr += '\n' 75 | 76 | def AddError(self, strMessage, newline=True): 77 | """Procedure for the GP object error message output (screen, logfile and if necessary e-mail) 78 | 79 | Args: 80 | self: The reserved object 'self' 81 | strMessage: Output message text 82 | """ 83 | if strMessage == None: 84 | strMessage = '' 85 | if not isinstance(strMessage, unicode): 86 | if not isinstance(strMessage, str): 87 | strMessage = str(strMessage) 88 | if strMessage[-1:] == '\n': 89 | strMessage = strMessage[0:-1] 90 | 91 | self.AddMessage('ERROR> ' + strMessage, newline) 92 | 93 | self.OutputErrStr += strMessage 94 | if newline == True: 95 | self.OutputErrStr += '\n' 96 | 97 | def AddWarning(self, strMessage, newline=True): 98 | """Procedure for the GP object warning message output (screen, logfile and if necessary e-mail) 99 | 100 | Args: 101 | self: The reserved object 'self' 102 | strMessage: Output message text 103 | """ 104 | if strMessage == None: 105 | strMessage = '' 106 | if not isinstance(strMessage, unicode): 107 | if not isinstance(strMessage, str): 108 | strMessage = str(strMessage) 109 | if strMessage[-1:] == '\n': 110 | strMessage = strMessage[0:-1] 111 | 112 | self.AddMessage('Warning> ' + strMessage, newline) 113 | 114 | self.OutputErrStr += strMessage 115 | if newline == True: 116 | self.OutputErrStr += '\n' 117 | 118 | def _tryCovertStringEncoding(self, txt): 119 | """Function for working with strings in diferent encodings. Converts string from input to string in correct encoding. 120 | 121 | Args: 122 | self: The reserved object 'self' 123 | txt: String to be converted 124 | 125 | Returns: 126 | converted string 127 | """ 128 | if isinstance(txt, str): 129 | try: 130 | txt = txt.decode(self.Pr.encodingPrimary, errors='strict') 131 | except Exception: 132 | try: 133 | txt = txt.decode(self.Pr.encodingSecondary, errors='strict') 134 | except Exception: 135 | try: 136 | txt = txt.decode('utf-8', errors='strict') 137 | except Exception: 138 | txt = txt.decode(self.Pr.encodingPrimary, errors='replace') 139 | return txt 140 | 141 | def CorrectStr(self, Str): 142 | """Function doubles symbol '\' in path for some external execution compatibility 143 | Args: 144 | self: The reserved object 'self' 145 | Str: Input string 146 | 147 | Returns: 148 | string 149 | """ 150 | return Str.replace('\\', '\\\\') 151 | 152 | def AchiveFiles(self, Dir, AchiveDir, FileName, PrintOut=True): 153 | """Function moves log files to the archive 154 | 155 | Args: 156 | self: The reserved object 'self' 157 | Dir: Directory from which to archive 158 | AchiveDir: Archive directory 159 | FileName: Parameter for searching a file (full or partial filename) 160 | """ 161 | # Check if Archive directory exists 162 | if os.path.exists(AchiveDir): 163 | names = os.listdir(Dir) 164 | for name in names: 165 | if name.lower().find(FileName.lower()) == 0: 166 | if PrintOut: 167 | print(" move: " + Dir + "\\" + name + " -> " + AchiveDir + "\\" + name + "\n") 168 | try: 169 | shutil.move(Dir + "\\" + name, AchiveDir + "\\" + name) 170 | except: 171 | print(" archiving failed : " + Dir + "\\" + name + " -> " + AchiveDir + "\\" + name + "\n") 172 | 173 | def MyNowFile(self): 174 | """Function returns formatted date for the filename output 175 | 176 | Args: 177 | self: The reserved object 'self' 178 | 179 | Returns: 180 | Date, formatted as text 181 | """ 182 | return datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d_%H-%M-%S") 183 | 184 | def MyNowOracle(self): 185 | """Function returns formatted date for the data selection in SQL 186 | 187 | Args: 188 | self: The reserved object 'self' 189 | 190 | Returns: 191 | Date, formatted as text 192 | """ 193 | return "TO_DATE('" + datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") + "', 'YYYY-MM-DD HH24:MI:SS')" 194 | 195 | def MyNowFileSafe(self): 196 | """Function returns formatted date for the filename output (additional compatibility) 197 | 198 | Args: 199 | self: The reserved object 'self' 200 | 201 | Returns: 202 | Date, formatted as text 203 | """ 204 | return datetime.datetime.strftime(datetime.datetime.now(), "%Y%m%d_%H%M%S") 205 | 206 | def MyNow(self): 207 | """Function returns formatted date for the output 208 | 209 | Args: 210 | self: The reserved object 'self' 211 | 212 | Returns: 213 | Date, formatted as text 214 | """ 215 | return datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") 216 | 217 | def MyNowUTC(self): 218 | """Function returns formatted date for the UTC date output 219 | 220 | Args: 221 | self: The reserved object 'self' 222 | 223 | Returns: 224 | Date, formatted as text 225 | """ 226 | return datetime.datetime.strftime(datetime.datetime.utcnow(), "%Y-%m-%d %H:%M:%S") 227 | 228 | def MyNowForParam(self, minusdays=0, includeTime = False): 229 | """Function returns formatted date (date now) for the GEO parameter processing 230 | 231 | Args: 232 | self: The reserved object 'self' 233 | minusdays: Number of days to subtract from today 234 | includeTime: Include time in param (True/False) 235 | 236 | Returns: 237 | Date, formatted as text 238 | """ 239 | if includeTime: 240 | DD = datetime.timedelta(days=minusdays) 241 | return datetime.datetime.strftime((datetime.datetime.now() - DD), "%Y-%m-%d %H:%M:%S") 242 | else: 243 | DD = datetime.timedelta(days=minusdays) 244 | return datetime.datetime.strftime((datetime.datetime.now() - DD), "%Y-%m-%d") 245 | 246 | def MyDateFromParam(self, dateString, includeTime = False): 247 | """Function converts date written in the parameter file to a date object 248 | 249 | Args: 250 | self: The reserved object 'self' 251 | includeTime: Include time in param (True/False) 252 | """ 253 | if includeTime: 254 | return datetime.datetime.strptime(dateString, "%Y-%m-%d %H:%M:%S") 255 | else: 256 | return datetime.datetime.strptime(dateString, "%Y-%m-%d") 257 | 258 | def MyDateForParam(self, paramStr, includeTime = False): 259 | """Function returns date from GEO parameter processing saved string 260 | 261 | Args: 262 | self: The reserved object 'self' 263 | paramStr: Parameter value as text 264 | includeTime: Include time in param (True/False) 265 | 266 | Returns: 267 | datetime 268 | """ 269 | if includeTime: 270 | return datetime.datetime.strptime(paramStr, "%Y-%m-%d %H:%M:%S") 271 | else: 272 | return datetime.datetime.strptime(paramStr, "%Y-%m-%d") 273 | 274 | def _runProcess(self, exe, noErr=False, Detached=False, Silent=False, hidenStrings = []): 275 | """Shell command execution support function (see the runShell function) 276 | 277 | Args: 278 | self: The reserved object 'self' 279 | exe: Executable command 280 | 281 | Return: 282 | stdoutdata 283 | """ 284 | lines = list() 285 | if Detached == True: 286 | DETACHED_PROCESS = 0x00000010 287 | p = subprocess.Popen(exe, creationflags=DETACHED_PROCESS, stdout=None, stderr=None, shell=True, universal_newlines=True, close_fds=True) #, env={'LANG':'C'}) 288 | else: 289 | p = subprocess.Popen(exe, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, universal_newlines=True) #, env={'LANG':'C'}) 290 | while True: 291 | retcode = p.poll() #returns None while subprocess is running 292 | str = p.stdout.readline() 293 | for hideString in hidenStrings: 294 | str = str.replace(hideString, '*' * len(hideString)) 295 | if str != '' and Silent != True: 296 | self.AddMessage(u'>>>>' + self._tryCovertStringEncoding(str), True) 297 | lines.append(str) 298 | if retcode is not None: 299 | break 300 | errlines = p.stderr.readlines() 301 | if not errlines == None: 302 | for str in errlines: 303 | line = self._tryCovertStringEncoding(str) 304 | for hideString in hidenStrings: 305 | line = line.replace(hideString, '*' * len(hideString)) 306 | if line != '': 307 | if noErr == False: 308 | self.AddError('>>>>' + line, True) 309 | else: 310 | self.AddMessage('>>>>' + line, True) 311 | return lines 312 | 313 | def runShell(self, exe, noErr=False, ErrorStrings=['ERROR', 'FAILED', u'KĻŪDA', 'EXCEPTION', 'ORA-'], Detached=False, Silent=False, hidenStrings = []): 314 | """Shell command execution procedure. It can detect errors in execution and can output results to screen. 315 | 316 | Args: 317 | self: The reserved object 'self' 318 | exe: Executable command 319 | noErr: 'True' indicates that the errors doesn't have to be logged 320 | ErrorStrings: List of error strings to look for in the output. Found error will be considered as an error in the execution process 321 | Detached: Whether to execute seperately from the main process (Default: False) 322 | hidenStrings: List of strings tha has to be hiden in the output (used for hiding passwords) 323 | """ 324 | if not isinstance(exe, list): 325 | args = shlex.split(exe) 326 | else: 327 | args = exe 328 | exe = " ".join(exe) 329 | _StartTime = datetime.datetime.now() 330 | if not Silent: 331 | command = exe 332 | for hideString in hidenStrings: 333 | command = command.replace(hideString, '*' * len(hideString)) 334 | self.AddMessage(u'>Executing the Shell command> ' + command) 335 | ShellOutput = self._runProcess(args, noErr, Detached, Silent, hidenStrings) 336 | lines = ShellOutput 337 | self._outputLines(lines, False, noErr, ErrorStrings, Silent, hidenStrings = hidenStrings) 338 | _TD = datetime.datetime.now()- _StartTime 339 | if not Silent: 340 | self.AddMessage(u'>Done executing the Shell command. Execution time ' + str(_TD)) 341 | 342 | def outputLogfile(self, file, encoding='utf8', noErr=False, ErrorStrings=['ERROR', 'FAILED', u'KĻŪDA', 'EXCEPTION', 'ORA-'], Silent=False, hidenStrings = []): 343 | """Procedure prints text file to screent - processing error keywords 344 | 345 | Args: 346 | self: The reserved object 'self' 347 | file: path to file to process 348 | noErr: True ir no error logging is necesery 349 | ErrorStrings: List or keywords with will be recognized as errors 350 | Silent: if True no Errors will be rised 351 | hidenStrings: List of strings tha has to be hiden in the output (used for hiding passwords) 352 | """ 353 | with codecs.open(file, 'r', encoding) as fin: 354 | self._outputLines(fin.readlines(), True, noErr, ErrorStrings, Silent, hidenStrings = hidenStrings) 355 | 356 | def _outputLines(self, lines, doMessges, noErr=False, ErrorStrings=['ERROR', 'FAILED', u'KĻŪDA', 'EXCEPTION', 'ORA-'], Silent=False, hidenStrings = []): 357 | """Procedure for outputing set of lines to screen with error key word recognition. (for example for log file output processing) 358 | 359 | Args: 360 | self: The reserved object 'self' 361 | lines: Lines to process 362 | noErr: True ir no error logging is necesery 363 | ErrorStrings: List or keywords with will be recognized as errors 364 | Silent: if True no Errors will be rised 365 | hidenStrings: List of strings tha has to be hiden in the output (used for hiding passwords) 366 | """ 367 | isError = False 368 | for line in lines: 369 | for hideString in hidenStrings: 370 | line = line.replace(hideString, '*' * len(hideString)) 371 | line = self._tryCovertStringEncoding(line) 372 | if line != '': 373 | if doMessges == True and Silent==False: 374 | self.AddMessage(line) 375 | if noErr == False: 376 | for ErStr in ErrorStrings: 377 | if line.upper().find(ErStr.upper()) > -1: 378 | isError = True 379 | for line in lines: 380 | for hideString in hidenStrings: 381 | line = line.replace(hideString, '*' * len(hideString)) 382 | line = self._tryCovertStringEncoding(line) 383 | if line != '': 384 | if isError == True: 385 | self.AddWarning('>>>>' + line) 386 | 387 | def AutorizeNTWLocation(self, Adress, user, pwd): 388 | """Network directory authorization 389 | 390 | Args: 391 | self: The reserved object 'self' 392 | Adress: Network adress 393 | user: Username 394 | pwd: Password 395 | """ 396 | self.runShell(self.CorrectStr(r"net use " + Adress + r" /delete /y"), True, []) 397 | self.runShell(self.CorrectStr(r'net use ' + Adress + r' "' + pwd + r'" /user:' + user), True, []) 398 | 399 | def GetSQL(self, Name): 400 | """Function gets the SQL file location from the installation directory 401 | 402 | Args: 403 | self: The reserved object 'self' 404 | Name: Filename without an extension 405 | 406 | Returns: 407 | SQL file full path 408 | """ 409 | return self.ExecutePatch + '\\SQL\\' + Name + '.sql' 410 | 411 | def RunSQL(self, Name, user="#", pwd="#", SpoolFile='#', ErrorStrings=['ERROR', 'FAILED', u'KĻŪDA', 'EXCEPTION', 'ORA-'], params=[], DBType='Oracle', hidenStrings = [], DBName='#', Detached=False): 412 | """Procedure for SQL file execution (only Oracle sqlplus supported) 413 | Typically used for execution, passing only SQL filename parameter 414 | 415 | Args: 416 | self: The reserved object 'self' 417 | Name: Filename without an extension 418 | u: Username 419 | p: Password 420 | SpoolFile: SQL output 421 | ErrorStrings: A list of keyword error strings that defines an error in the execution 422 | params: aditional parameters 423 | DBType: only Oracle is supported 424 | hidenStrings: List of strings tha has to be hiden in the output (used for hiding passwords) 425 | DBName: Oracle TNS name 426 | Detached: Whether to execute seperately from the main process (Default: False) 427 | """ 428 | if DBType.upper() == 'ORACLE': 429 | if pwd == "#": 430 | pwd = self.Pr.p 431 | if user == "#": 432 | user = self.Pr.u 433 | if DBName == "#": 434 | DBName = self.Pr.TNSName 435 | if SpoolFile == '#': 436 | SpoolFile = self.Pr.OutDir + '\\' + Name + self.MyNowFile() + 'SQL.outlog' 437 | hidenStrings.append(pwd) 438 | self.AchiveFiles(self.Pr.OutDir, self.Pr.OutDirArh, Name, False) 439 | self.runShell( 440 | 'sqlplus -L ' + user + '/' + pwd + '@' + DBName + ' @"' + self.GetSQL(Name) + '" "' + SpoolFile + '" ' + ' '.join(params), 441 | False, ErrorStrings, hidenStrings=hidenStrings, Detached=Detached) 442 | else: 443 | raise AttributeError('Provided DB Type is not supported!') 444 | 445 | def GetPS(self, Name, Path='#'): 446 | """Function gets the PowerShell file location from the installation directory 447 | 448 | Args: 449 | self: The reserved object 'self' 450 | Name: Filename without an extension 451 | """ 452 | if Path == '#': 453 | self.ExecutePatch 454 | return Path + '\\PShell\\' + Name + '.ps1' 455 | 456 | def RunPS(self, Name, Args, Path='#'): 457 | """Procedure for the PowerShell file execution 458 | 459 | Args: 460 | self: The reserved object 'self' 461 | Name: Filename without an extension 462 | Args: Arguments 463 | """ 464 | if Path == '#': 465 | self.ExecutePatch 466 | self.runShell('powershell -File "' + self.GetPS(Name, Path) + '" ' + Args) 467 | 468 | def run_with_limited_time(self, func, args, kwargs, time): 469 | """Runs a function with time limit 470 | 471 | Args: 472 | self: The reserved object 'self' 473 | func: The function to run 474 | args: The functions args, given as a tuple 475 | kwargs: The functions args, given as a tuple 476 | time: The time limit in seconds 477 | 478 | Returns: 479 | True if the function ended successfully. False if it was terminated. 480 | """ 481 | p = Process(target=func, args=args, kwargs=kwargs) 482 | p.start() 483 | p.join(time) 484 | if p.is_alive(): 485 | p.terminate() 486 | return False 487 | return True 488 | --------------------------------------------------------------------------------