├── sc-list.txt ├── isc └── py │ ├── msg │ ├── DebugRequest.cls │ ├── CreateCQRequest.cls │ ├── ExecutionResponse.cls │ ├── StreamExecutionResponse.cls │ ├── DataRequest.cls │ ├── RestoreRequest.cls │ ├── SQLRequest.cls │ ├── QueryRequest.cls │ ├── SaveRequest.cls │ ├── ExecutionRequest.cls │ ├── TableRequest.cls │ ├── ClassRequest.cls │ ├── StreamExecutionRequest.cls │ └── GlobalRequest.cls │ ├── query │ ├── List.cls │ ├── Abstract.cls │ ├── Custom.cls │ └── Generator.cls │ ├── init │ ├── Generator.cls │ ├── Test.cls │ └── Abstract.cls │ ├── util │ ├── JupyterCheckpoints.cls │ ├── Shell.cls │ ├── Storage.cls │ ├── Redirect.cls │ ├── Evaluator.cls │ ├── Matcher.cls │ ├── BPEmulator.cls │ ├── Installer.cls │ ├── Converter.cls │ ├── PMML.cls │ └── Generator.cls │ ├── ens │ ├── ContextSearch.cls │ ├── SQL.cls │ ├── ProcessUtils.cls │ └── Operation.cls │ ├── test │ ├── Production.cls │ ├── Person.cls │ └── Process.cls │ ├── gw │ ├── PyString.cls │ └── DynamicObject.cls │ ├── data │ ├── Function.cls │ ├── Variable.cls │ └── Context.cls │ └── Callout.cls ├── jupyter ├── jupyter_notebook_config.py ├── Dockerfile ├── Readme.md └── MLContentsManager.py ├── .gitignore ├── rtn ├── %ZSTART.xml └── %ZLANGC00.xml ├── c ├── iscpython.h └── Makefile ├── LICENSE ├── Dockerfile-GPU ├── CHANGELOG.md ├── Dockerfile191 ├── Dockerfile ├── RELEASE-NOTES.md ├── DataTransfer.md ├── docs ├── Article II Installation.md ├── Article I.md └── Article IV Interoperability Adapter.md └── Gateway.md /sc-list.txt: -------------------------------------------------------------------------------- 1 | isc.py.pkg 2 | -------------------------------------------------------------------------------- /isc/py/msg/DebugRequest.cls: -------------------------------------------------------------------------------- 1 | Class isc.py.msg.DebugRequest Extends Ens.Request 2 | { 3 | 4 | } 5 | 6 | -------------------------------------------------------------------------------- /jupyter/jupyter_notebook_config.py: -------------------------------------------------------------------------------- 1 | exec(open("MLContentsManager.py").read()) 2 | 3 | c.NotebookApp.contents_manager_class = MLContentsManager 4 | -------------------------------------------------------------------------------- /isc/py/query/List.cls: -------------------------------------------------------------------------------- 1 | /// SELECT $ListToString(Columns) FROM isc_py_query.List('df') 2 | Class isc.py.query.List Extends (isc.py.query.Abstract, %SQL.CustomQuery) 3 | { 4 | 5 | Parameter SQLNAME As String = "List"; 6 | 7 | Property Columns As %List; 8 | 9 | Method %FetchCursor() As %Integer 10 | { 11 | set ..Columns = ..GetRow(.atEnd) 12 | quit 'atEnd 13 | } 14 | 15 | } 16 | 17 | -------------------------------------------------------------------------------- /isc/py/msg/CreateCQRequest.cls: -------------------------------------------------------------------------------- 1 | /// Generate Custom Query based on a Python DataFrame 2 | Class isc.py.msg.CreateCQRequest Extends Ens.Request 3 | { 4 | 5 | /// Python variable with DataFrame 6 | Property Variable As %String [ Required ]; 7 | 8 | /// Classname to generate. 9 | /// Overrides if exists. 10 | Property Classname As %Dictionary.CacheClassname [ Required ]; 11 | 12 | } 13 | 14 | -------------------------------------------------------------------------------- /isc/py/msg/ExecutionResponse.cls: -------------------------------------------------------------------------------- 1 | /// Response from Python environment 2 | Class isc.py.msg.ExecutionResponse Extends Ens.Response 3 | { 4 | 5 | /// Array of Python variables 6 | Property Variables As array Of %VarString; 7 | 8 | Storage Default 9 | { 10 | 11 | Variables 12 | subnode 13 | "isc.py.msg.ExecutionResponse.Variables" 14 | 15 | %Library.CacheStorage 16 | } 17 | 18 | } 19 | 20 | -------------------------------------------------------------------------------- /jupyter/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM intersystemscommunity/irispy:latest 2 | 3 | USER root 4 | 5 | RUN pip install jupyter && pip install $ISC_PACKAGE_INSTALLDIR/dev/python/irisnative-1.0.0-cp34-abi3-linux_x86_64.whl 6 | 7 | USER irisowner 8 | 9 | COPY --chown=irisowner ./jupyter/*.py /jupyter/ 10 | COPY --chown=irisowner ./jupyter/*.sh / 11 | 12 | RUN echo -e "\r\nc.MLContentsManager.namespace = 'PYTHON'" >> /jupyter/jupyter_notebook_config.py 13 | 14 | HEALTHCHECK --interval=5s CMD /irisHealth.sh || exit 1 -------------------------------------------------------------------------------- /isc/py/msg/StreamExecutionResponse.cls: -------------------------------------------------------------------------------- 1 | /// Response from Python environment 2 | Class isc.py.msg.StreamExecutionResponse Extends Ens.Response 3 | { 4 | 5 | /// Array of Python variables 6 | Property Variables As array Of %Stream.GlobalCharacter(XMLPROJECTION = "NONE"); 7 | 8 | Storage Default 9 | { 10 | 11 | Variables 12 | subnode 13 | "isc.py.msg.StreamExecutionResponse.Variables" 14 | 15 | %Library.CacheStorage 16 | } 17 | 18 | } 19 | 20 | -------------------------------------------------------------------------------- /isc/py/msg/DataRequest.cls: -------------------------------------------------------------------------------- 1 | /// Common properties for Data transfer IRIS -> Python 2 | Class isc.py.msg.DataRequest [ Abstract ] 3 | { 4 | 5 | /// Python variable to set 6 | Property Variable As %String [ Required ]; 7 | 8 | /// Variable type (Currently supported: dataframe (pandas dataframe) and list. 9 | /// Pandas automatically imported if required. 10 | Property Type As %String(VALUELIST = ",dataframe,list") [ InitialExpression = "list", Required ]; 11 | 12 | /// Namespace in which to execute the query. 13 | /// 'isc.py' package must be available in this namespace 14 | Property Namespace As %String [ InitialExpression = {$namespace}, Required ]; 15 | 16 | } 17 | 18 | -------------------------------------------------------------------------------- /isc/py/init/Generator.cls: -------------------------------------------------------------------------------- 1 | Class isc.py.init.Generator Extends isc.py.init.Abstract 2 | { 3 | 4 | Parameter Modules = {$lb("types", "json", "inspect", "random")}; 5 | 6 | ClassMethod InitGetMembers() As %Status [ CodeMode = expression ] 7 | { 8 | ##class(isc.py.Main).SimpleString( "def zzzgetmembers(object, predicate):" _ $c(10) _ 9 | " return json.dumps([item[0] for item in inspect.getmembers(object, predicate)])") 10 | } 11 | 12 | ClassMethod InitUncallable() As %Status [ CodeMode = expression ] 13 | { 14 | ##class(isc.py.Main).SimpleString( "def zzzuncallable(object):" _ $c(10) _ 15 | " return not callable(object)") 16 | } 17 | 18 | } 19 | 20 | -------------------------------------------------------------------------------- /isc/py/msg/RestoreRequest.cls: -------------------------------------------------------------------------------- 1 | /// Restore saved Python context 2 | Class isc.py.msg.RestoreRequest Extends Ens.Request 3 | { 4 | 5 | /// Context identifier to restore 6 | Property ContextId As %Integer; 7 | 8 | /// Clear context before restore 9 | Property Clear As %Boolean [ InitialExpression = {$$$NO} ]; 10 | 11 | Storage Default 12 | { 13 | 14 | "RestoreRequest" 15 | 16 | ContextId 17 | 18 | 19 | Clear 20 | 21 | 22 | RestoreRequestDefaultData 23 | %Library.CacheStorage 24 | } 25 | 26 | } 27 | 28 | -------------------------------------------------------------------------------- /isc/py/util/JupyterCheckpoints.cls: -------------------------------------------------------------------------------- 1 | Class isc.py.util.JupyterCheckpoints Extends isc.py.util.Jupyter 2 | { 3 | 4 | ClassMethod CreateFile(content, format, path) 5 | { 6 | do ..Log(,"CreateFile", path, content,, format) 7 | quit $$$YES 8 | } 9 | 10 | ClassMethod CreateNotebook(nb, path) 11 | { 12 | do ..Log(,"CreateNotebook",path,nb) 13 | quit $$$YES 14 | } 15 | 16 | ClassMethod DeleteCheckpoint(checkpointId, path) 17 | { 18 | do ..Log(,"DeleteCheckpoint",path, checkpointId) 19 | quit $$$YES 20 | } 21 | 22 | ClassMethod RenameCheckpoint(checkpointId, oldPath, newPath) 23 | { 24 | do ..Log(,"RenameCheckpoint",oldPath, checkpointId, newPath) 25 | quit $$$YES 26 | } 27 | 28 | } 29 | 30 | -------------------------------------------------------------------------------- /isc/py/msg/SQLRequest.cls: -------------------------------------------------------------------------------- 1 | /// Execute SQL Query in the current process 2 | Class isc.py.msg.SQLRequest Extends Ens.Request 3 | { 4 | 5 | /// Query to execute, in a form of 6 | /// - SQL Query 7 | /// - Class:Query 8 | /// - Class:XData 9 | Property Query As %VarString; 10 | 11 | /// Query arguments, if any 12 | Property Args As list Of %String; 13 | 14 | Storage Default 15 | { 16 | 17 | "SQLRequest" 18 | 19 | Query 20 | 21 | 22 | Args 23 | 24 | 25 | SQLRequestDefaultData 26 | %Library.CacheStorage 27 | } 28 | 29 | } 30 | 31 | -------------------------------------------------------------------------------- /isc/py/msg/QueryRequest.cls: -------------------------------------------------------------------------------- 1 | /// Create pandas dataframe or list form sql. 2 | Class isc.py.msg.QueryRequest Extends (Ens.Request, isc.py.msg.DataRequest) 3 | { 4 | 5 | /// Text of the SQL query 6 | Property Query As %VarString; 7 | 8 | Storage Default 9 | { 10 | 11 | "QueryRequest" 12 | 13 | Query 14 | 15 | 16 | Variable 17 | 18 | 19 | Type 20 | 21 | 22 | Namespace 23 | 24 | 25 | QueryRequestDefaultData 26 | %Library.CacheStorage 27 | } 28 | 29 | } 30 | 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.sh 2 | *.html 3 | *.bat 4 | *.zip 5 | 6 | # Prerequisites 7 | *.d 8 | 9 | # Object files 10 | *.o 11 | *.ko 12 | *.obj 13 | *.elf 14 | 15 | # Linker output 16 | *.ilk 17 | *.map 18 | *.exp 19 | 20 | # Precompiled Headers 21 | *.gch 22 | *.pch 23 | 24 | # Libraries 25 | *.lib 26 | *.a 27 | *.la 28 | *.lo 29 | 30 | # Shared objects (inc. Windows DLLs) 31 | *.dll 32 | *.so 33 | *.so.* 34 | *.dylib 35 | 36 | # Executables 37 | *.exe 38 | *.out 39 | *.app 40 | *.i*86 41 | *.x86_64 42 | *.hex 43 | 44 | # Debug files 45 | *.dSYM/ 46 | *.su 47 | *.idb 48 | *.pdb 49 | 50 | # Kernel Module Compile Results 51 | *.mod* 52 | *.cmd 53 | .tmp_versions/ 54 | modules.order 55 | Module.symvers 56 | Mkfile.old 57 | dkms.conf 58 | -------------------------------------------------------------------------------- /rtn/%ZSTART.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 28 | 29 | -------------------------------------------------------------------------------- /isc/py/ens/ContextSearch.cls: -------------------------------------------------------------------------------- 1 | /// Context search for ensemble/interopeability production hosts 2 | Class isc.py.ens.ContextSearch Extends %ZEN.Portal.ContextSearch 3 | { 4 | 5 | /// Get list of subclasses 6 | ClassMethod SubclassOf(Output caption As %String, Output topResults, Output results, ByRef parms As %String, searchKey As %String = "") As %Status 7 | { 8 | #dim sc AS %Status = $$$OK 9 | kill results, topResults 10 | set caption = "Модели ПУ" 11 | 12 | try { 13 | 14 | /// Really %sqlcq..cls 15 | #dim rs As %SQL.ISelectResult 16 | set rs = ##class(%Dictionary.ClassDefinitionQuery).SubclassOfFunc(parms("class")) 17 | while rs.%Next() { 18 | set results($i(results)) = rs.Name 19 | } 20 | } catch ex { 21 | set sс = ex.AsStatus() 22 | } 23 | 24 | quit sc 25 | } 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /c/iscpython.h: -------------------------------------------------------------------------------- 1 | /* 2 | * iscpython.h 3 | * 4 | * Created on: 16 Dec 2018 5 | * Author: Eduard 6 | */ 7 | 8 | #ifndef ISCPYTHON_H 9 | #define ISCPYTHON_H 10 | 11 | 12 | // Production methods 13 | int Initialize(char *file); 14 | int Finalize(); 15 | int SimpleString(CACHE_EXSTRP command, char *resultVar, int serialization, CACHE_EXSTRP result); 16 | int StreamInit(int length) 17 | int StreamWrite(CACHE_EXSTRP command); 18 | int StreamExecute(); 19 | 20 | int EscapeString(CACHE_EXSTRP string, CACHE_EXSTRP result) 21 | PyTupleObject* ListToTuple(CACHE_EXSTRP result, char* mask, int maskLength, int maskSymbolLength, int key) 22 | int GetGlobalOrder(const char *global, int start, int end, const char* mask, const char* name, CACHE_EXSTRP value) 23 | 24 | // Console test method 25 | int main(int argc, char **argv); 26 | 27 | #endif /* ISCPYTHON_H */ 28 | -------------------------------------------------------------------------------- /isc/py/test/Production.cls: -------------------------------------------------------------------------------- 1 | /// Sample production 2 | Class isc.py.test.Production Extends Ens.Production 3 | { 4 | 5 | /// Register callout lib system-wide 6 | ClassMethod OnStart(pTimeStarted As %String) As %Status 7 | { 8 | set sc = ##class(isc.py.Callout).Setup() 9 | quit sc 10 | } 11 | 12 | XData ProductionDefinition 13 | { 14 | 15 | 16 | 2 17 | 18 | 19 | 20 | 21 | 22 | } 23 | 24 | } 25 | 26 | -------------------------------------------------------------------------------- /rtn/%ZLANGC00.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 25 | 26 | -------------------------------------------------------------------------------- /isc/py/gw/PyString.cls: -------------------------------------------------------------------------------- 1 | /// Python object property 2 | Class isc.py.gw.PyString Extends %String 3 | { 4 | 5 | /// Specified in cases where property name on InterSystems IRIS side is not equal to Python attribute name 6 | Parameter PyName; 7 | 8 | /// Generate Getter 9 | Method Get() As %String [ CodeMode = objectgenerator, NoContext ] 10 | { 11 | #; don't generate any code if it not for a property 12 | quit:%mode'="propertymethod" $$$OK 13 | set property = %parameter("PyName") 14 | set:property="" property = $g(%member) 15 | do %code.WriteLine($c(9) _ "quit ..%DispatchGetProperty(""" _ property _ """)") 16 | quit $$$OK 17 | } 18 | 19 | /// Generate Setter 20 | Method Set(%val) [ CodeMode = objectgenerator, NoContext ] 21 | { 22 | quit:%mode'="propertymethod" $$$OK 23 | set property = %parameter("PyName") 24 | set:property="" property = $g(%member) 25 | do %code.WriteLine($c(9) _ "do ..%DispatchSetProperty(""" _ property _ """, %val)") 26 | quit $$$OK 27 | } 28 | 29 | } 30 | 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Eduard Lebedyuk. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /isc/py/msg/SaveRequest.cls: -------------------------------------------------------------------------------- 1 | /// Save Python context to disk. 2 | Class isc.py.msg.SaveRequest Extends Ens.Request 3 | { 4 | 5 | /// Comma-separated list of variables to save. 6 | /// Only variables that satisfy Mask are saved. 7 | /// Wildcards * and ? are accepted. 8 | /// Example: "Data*,Figure?" 9 | Property Mask As %VarString [ InitialExpression = "*" ]; 10 | 11 | /// Maximum length of saved variable. 12 | /// If veriable serialization is longer than that, it would be ignored. 13 | /// Set to 0 to get them all. 14 | Property MaxLength As %Integer [ InitialExpression = {$$$MaxStringLength} ]; 15 | 16 | /// Short name for the context. 17 | Property Name As %String; 18 | 19 | /// Extended context info. 20 | Property Description As %VarString; 21 | 22 | Storage Default 23 | { 24 | 25 | "SaveRequest" 26 | 27 | Mask 28 | 29 | 30 | Name 31 | 32 | 33 | Description 34 | 35 | 36 | MaxLength 37 | 38 | 39 | SaveRequestDefaultData 40 | %Library.CacheStorage 41 | } 42 | 43 | } 44 | 45 | -------------------------------------------------------------------------------- /isc/py/msg/ExecutionRequest.cls: -------------------------------------------------------------------------------- 1 | /// Request to execute code for python operation 2 | Class isc.py.msg.ExecutionRequest Extends Ens.Request 3 | { 4 | 5 | /// Python code to execute 6 | Property Code As %VarString; 7 | 8 | /// Comma-separated list of variables to get in response message 9 | Property Variables As %VarString; 10 | 11 | /// Separate incoming message into lines for execution. 12 | /// $c(10) is used for line separation 13 | /// Note that it's NOT recommended to process whole message at once, 14 | /// this feature is only for `def` and similar multi-line expressions processing. 15 | Property SeparateLines As %Boolean [ InitialExpression = {$$$YES} ]; 16 | 17 | /// How to serialize variables we want to return 18 | Property Serialization As %String(VALUELIST = ",Str,Repr,JSON,Pickle,Dill") [ InitialExpression = "Str" ]; 19 | 20 | Storage Default 21 | { 22 | 23 | "ExecutionRequest" 24 | 25 | Code 26 | 27 | 28 | Variables 29 | 30 | 31 | SeparateLines 32 | 33 | 34 | Serialization 35 | 36 | 37 | ExecutionRequestDefaultData 38 | %Library.CacheStorage 39 | } 40 | 41 | } 42 | 43 | -------------------------------------------------------------------------------- /isc/py/msg/TableRequest.cls: -------------------------------------------------------------------------------- 1 | /// Transfer table data to python 2 | Class isc.py.msg.TableRequest Extends (Ens.Request, isc.py.msg.DataRequest) 3 | { 4 | 5 | /// Table name 6 | Property Table As %String(MAXLEN = 500) [ Required ]; 7 | 8 | /// Initial row id 9 | Property Start As %Integer [ InitialExpression = 1 ]; 10 | 11 | /// Final row id 12 | Property End As %Integer; 13 | 14 | /// Comma-separated list of class properties to form dataframe from. 15 | /// `*` and `?` wildcards are supported. 16 | /// Defaults to `*` (all properties). 17 | /// `%%CLASSNAME` property is ignored. 18 | /// Only stored properties can be used. 19 | Property Properties As %VarString [ InitialExpression = "*" ]; 20 | 21 | Storage Default 22 | { 23 | 24 | "TableRequest" 25 | 26 | Table 27 | 28 | 29 | Start 30 | 31 | 32 | End 33 | 34 | 35 | Properties 36 | 37 | 38 | Namespace 39 | 40 | 41 | Type 42 | 43 | 44 | Variable 45 | 46 | 47 | TableRequestDefaultData 48 | %Library.CacheStorage 49 | } 50 | 51 | } 52 | 53 | -------------------------------------------------------------------------------- /isc/py/msg/ClassRequest.cls: -------------------------------------------------------------------------------- 1 | /// Transfer class data to python 2 | Class isc.py.msg.ClassRequest Extends (Ens.Request, isc.py.msg.DataRequest) 3 | { 4 | 5 | /// Class name 6 | Property Class As %Dictionary.CacheClassname [ Required ]; 7 | 8 | /// Initial object id 9 | Property Start As %Integer [ InitialExpression = 1 ]; 10 | 11 | /// Final object id 12 | Property End As %Integer; 13 | 14 | /// Comma-separated list of properties to form dataframe from. 15 | /// `*` and `?` wildcards are supported. 16 | /// Defaults to `*` (all properties). 17 | /// `%%CLASSNAME` property is ignored. 18 | /// Only stored properties can be used. 19 | Property Properties As %VarString [ InitialExpression = "*" ]; 20 | 21 | Storage Default 22 | { 23 | 24 | "ClassRequest" 25 | 26 | Class 27 | 28 | 29 | Start 30 | 31 | 32 | End 33 | 34 | 35 | Properties 36 | 37 | 38 | Namespace 39 | 40 | 41 | Type 42 | 43 | 44 | Variable 45 | 46 | 47 | ClassRequestDefaultData 48 | %Library.CacheStorage 49 | } 50 | 51 | } 52 | 53 | -------------------------------------------------------------------------------- /isc/py/msg/StreamExecutionRequest.cls: -------------------------------------------------------------------------------- 1 | /// Same as ExecutionRequest but Code is a stream 2 | Class isc.py.msg.StreamExecutionRequest Extends Ens.Request 3 | { 4 | 5 | /// Python code to execute 6 | Property Code As %Stream.GlobalCharacter; 7 | 8 | /// Comma-separated list of variables to get in response message 9 | Property Variables As %VarString; 10 | 11 | /// Separate incoming message into lines for execution. 12 | /// $c(10) is used for line separation 13 | /// Note that it's NOT recommended to process whole message at once, 14 | /// this feature is only for `def` and similar multi-line expressions processing. 15 | Property SeparateLines As %Boolean [ InitialExpression = {$$$YES} ]; 16 | 17 | /// How to serialize variables we want to return 18 | Property Serialization As %String(VALUELIST = ",Str,Repr,JSON,Pickle,Dill") [ InitialExpression = "Str" ]; 19 | 20 | Storage Default 21 | { 22 | 23 | "StreamExecutionRequest" 24 | 25 | Code 26 | 27 | 28 | Variables 29 | 30 | 31 | SeparateLines 32 | 33 | 34 | Serialization 35 | 36 | 37 | StreamExecutionRequestDefaultData 38 | %Library.CacheStorage 39 | } 40 | 41 | } 42 | 43 | -------------------------------------------------------------------------------- /c/Makefile: -------------------------------------------------------------------------------- 1 | CC := gcc 2 | RM := rm 3 | CD := cd 4 | MAKE := make 5 | LDFLAGS := -shared 6 | 7 | CFLAGS += -Wall -Wextra -fpic -O3 -fno-strict-aliasing -Wno-incompatible-pointer-types 8 | 9 | SYS := $(shell gcc -dumpmachine) 10 | 11 | ifneq (, $(findstring linux, $(SYS))) 12 | SUFFIX := so 13 | 14 | ifndef PYTHONVER 15 | $(error Couldn't find PYTHONVER) 16 | endif 17 | 18 | INCLUDES += $(shell python${PYTHONVER}-config --includes) 19 | LDFLAGS += $(shell python${PYTHONVER}-config --ldflags) 20 | LIBS += $(shell python${PYTHONVER}-config --libs) 21 | else ifneq (, $(findstring mingw, $(SYS))) 22 | SUFFIX := dll 23 | 24 | ifndef PYTHONHOME 25 | $(error Couldn't find PYTHONHOME) 26 | endif 27 | 28 | PYTHONVER := $(shell ${PYTHONHOME}/python -c "import sys; print(str(sys.version_info[0])+str(sys.version_info[1]))") 29 | 30 | INCLUDES += -I${PYTHONHOME}/include 31 | LDFLAGS += -L${PYTHONHOME}/Libs 32 | LIBS = -lpython${PYTHONVER} 33 | else 34 | $(error Unsupported build platform) 35 | endif 36 | 37 | ifndef GLOBALS_HOME 38 | $(error Couldn't find GLOBALS_HOME) 39 | endif 40 | 41 | INCLUDES += -I${GLOBALS_HOME}/dev/cpp/include 42 | CFLAGS += $(INCLUDES) 43 | 44 | .PHONY: all clean 45 | 46 | all: iscpython.$(SUFFIX) 47 | 48 | iscpython.o: iscpython.c 49 | 50 | iscpython.$(SUFFIX): iscpython.o 51 | $(CC) $(LDFLAGS) -o $@ $^ $(LIBS) 52 | 53 | clean: 54 | $(RM) *.$(SUFFIX) *.o -------------------------------------------------------------------------------- /isc/py/query/Abstract.cls: -------------------------------------------------------------------------------- 1 | Class isc.py.query.Abstract [ Abstract ] 2 | { 3 | 4 | Parameter SQLNAME As String = "Abstract"; 5 | 6 | Parameter Separator = 1; 7 | 8 | Parameter Terminator = 2; 9 | 10 | Property %Data As %Stream.GlobalCharacter [ Private ]; 11 | 12 | Property %DTypes As %DynamicObject [ Private ]; 13 | 14 | Method %OpenCursor(variable As %String) As %Status 15 | { 16 | #dim sc As %Status = $$$OK 17 | set sc = ##class(isc.py.Main).SimpleString("zzzcsv=" _ variable _ ".to_csv(index=False, header=False, sep=chr(" _ ..#Separator _ "), line_terminator=chr(" _ ..#Terminator _ "))") 18 | quit:$$$ISERR(sc) sc 19 | 20 | #dim stream As %Stream.GlobalCharacter 21 | set sc = ##class(isc.py.Main).GetVariable("zzzcsv", , .stream) 22 | quit:$$$ISERR(sc) sc 23 | set sc = ##class(isc.py.Main).SimpleString("del zzzcsv") 24 | quit:$$$ISERR(sc) sc 25 | 26 | set stream.LineTerminator = $char(..#Terminator) 27 | 28 | set ..%Data = stream 29 | 30 | set sc = ##class(isc.py.Main).GetVariableJson(variable _ ".dtypes", .dtypes, $$$YES) 31 | quit:$$$ISERR(sc) sc 32 | 33 | set ..%DTypes = {}.%FromJSON(dtypes) 34 | 35 | quit sc 36 | } 37 | 38 | Method GetRow(Output atEnd As %Boolean) As %List 39 | { 40 | if '..%Data.AtEnd { 41 | set list = $lfs(..%Data.ReadLine(), $char(..#Separator)) 42 | set atEnd = $$$NO 43 | } else { 44 | set list = "" 45 | set atEnd = $$$YES 46 | } 47 | return list 48 | } 49 | 50 | } 51 | 52 | -------------------------------------------------------------------------------- /isc/py/util/Shell.cls: -------------------------------------------------------------------------------- 1 | /// Python shell 2 | Class isc.py.util.Shell 3 | { 4 | 5 | /// Start Python Shell. 6 | /// Press enter to quit. 7 | /// do ##class(isc.py.util.Shell).Shell() 8 | ClassMethod Shell() 9 | { 10 | try { 11 | // Multiline mode 12 | #dim multi As %Boolean = $$$NO 13 | for { 14 | write $case(multi, $$$YES:"...", :"PY>") 15 | read command, ! 16 | 17 | if multi = $$$NO { 18 | // enter multi mode if line ends on : 19 | set multi = ($e($zstrip(command,">W"),*)=":") 20 | } elseif command = "" { 21 | // leave multi mode 22 | set multi = $$$NO 23 | set command = $g(multiCommand) _ $c(10) _ command 24 | kill multiCommand 25 | } 26 | 27 | if multi { 28 | set multiCommand = $g(multiCommand) _ $c(10) _ command 29 | } else { 30 | quit:(command="quit()")||(command="") 31 | kill result 32 | if ((command [ "=") || (command [ $c(10)) || ($zcvt(command, "l")["import") || (command [ " ")) { 33 | set sc = ##class(isc.py.Main).SimpleString(command) 34 | } else { 35 | set sc = ##class(isc.py.Main).SimpleString("zzzresult=" _ command,"zzzresult", , .result) 36 | } 37 | if $$$ISOK(sc) { 38 | write $replace($g(result), $c(10), $$$NL) 39 | } else { 40 | write $System.Status.GetErrorText(sc) 41 | } 42 | write ! 43 | } 44 | } 45 | } catch ex { 46 | #dim ex As %Exception.General 47 | write ex.DisplayString(),! 48 | } 49 | } 50 | 51 | } 52 | 53 | -------------------------------------------------------------------------------- /isc/py/query/Custom.cls: -------------------------------------------------------------------------------- 1 | /// Works on Cache and Ensemble (and InterSystems IRIS) 2 | /// SELECT $ListToString(Columns) FROM isc_py_query.DataFrame('df') 3 | Class isc.py.query.Custom 4 | { 5 | 6 | /// SELECT $ListToString(Columns) FROM isc_py_query.DataFrame('df') 7 | Query DataFrame(variable) As %Query(ROWSPEC = "Columns:%List") [ SqlName = DataFrame, SqlProc ] 8 | { 9 | } 10 | 11 | ClassMethod DataFrameExecute(ByRef qHandle As %Binary, variable) As %Status 12 | { 13 | #dim sc As %Status = $$$OK 14 | set sc = ##class(isc.py.Main).SimpleString("zzzcsv=" _ variable _ ".to_csv(index=False, header=False, sep=chr(" _ ##class(isc.py.query.Abstract).#Separator _ "), line_terminator=chr(" _ ##class(isc.py.query.Abstract).#Terminator _ "))") 15 | quit:$$$ISERR(sc) sc 16 | 17 | #dim stream As %Stream.GlobalCharacter 18 | set sc = ##class(isc.py.Main).GetVariable("zzzcsv", , .stream) 19 | quit:$$$ISERR(sc) sc 20 | set sc = ##class(isc.py.Main).SimpleString("del zzzcsv") 21 | quit:$$$ISERR(sc) sc 22 | 23 | set stream.LineTerminator = $char(##class(isc.py.query.Abstract).#Terminator) 24 | 25 | while 'stream.AtEnd { 26 | set str = stream.ReadLine() 27 | set list = $lfs(str, $char(##class(isc.py.query.Abstract).#Separator)) 28 | set qHandle($i(qHandle)) = $lb(list) 29 | } 30 | 31 | set qHandle = $g(qHandle) + 1 32 | 33 | quit $$$OK 34 | } 35 | 36 | ClassMethod DataFrameFetch(ByRef qHandle As %Binary, ByRef Row As %List, ByRef AtEnd As %Integer = 0) As %Status [ PlaceAfter = DataFrameExecute ] 37 | { 38 | if qHandle { 39 | set Row=$g(qHandle($increment(qHandle,-1))) 40 | set:qHandle=0 AtEnd = 1 41 | } else { 42 | set AtEnd=1 43 | } 44 | 45 | quit $$$OK 46 | } 47 | 48 | ClassMethod DataFrameClose(ByRef qHandle As %Binary) As %Status [ CodeMode = expression, PlaceAfter = DataFrameExecute ] 49 | { 50 | $$$OK 51 | } 52 | 53 | } 54 | 55 | -------------------------------------------------------------------------------- /isc/py/util/Storage.cls: -------------------------------------------------------------------------------- 1 | Class isc.py.util.Storage Extends %XML.Adaptor [ Abstract ] 2 | { 3 | 4 | /// Name of XData block that stores our data for population 5 | Parameter XDATA = "Data"; 6 | 7 | ClassMethod Export() As %Status 8 | { 9 | quit:$classmethod($classname(), "%Extends", "%XML.Adaptor")=$$$NO $$$ERROR($$$GeneralError, "Class " _ $classname() _ " should extend %XML.Adaptor") 10 | 11 | if ##class(%Dictionary.XDataDefinition).IDKEYExists($classname(), ..#XDATA) { 12 | set xdata = ##class(%Dictionary.XDataDefinition).IDKEYOpen($classname(), ..#XDATA) 13 | } else { 14 | set xdata = ##class(%Dictionary.XDataDefinition).%New($classname() _ ":" _ ..#XDATA) 15 | } 16 | 17 | #dim stream As %Stream.TmpCharacter 18 | set stream = xdata.Data 19 | 20 | do stream.Clear() 21 | do stream.WriteLine("") 22 | 23 | set rs = ..ExtentFunc() 24 | while rs.%Next() { 25 | set obj = ..%OpenId(rs.ID) 26 | do obj.XMLExportToStream(.stream) 27 | do stream.WriteLine() 28 | } 29 | do stream.WriteLine("") 30 | quit xdata.%Save() 31 | } 32 | 33 | ClassMethod Import() As %Status 34 | { 35 | #dim sc As %Status = $$$OK 36 | set xdata = ##class(%Dictionary.XDataDefinition).IDKEYOpen($classname(), "Data") 37 | 38 | #dim stream As %Stream.TmpCharacter 39 | set stream = xdata.Data 40 | 41 | set reader = ##class(%XML.Reader).%New() 42 | set sc=reader.OpenStream(stream) 43 | quit:$$$ISERR(sc) sc 44 | do reader.Correlate(..GetXMLName(), $classname()) 45 | 46 | do ..%KillExtent() 47 | while reader.Next(.obj, .sc) { 48 | set sc = obj.%Save() 49 | quit:$$$ISERR(sc) 50 | } 51 | 52 | quit sc 53 | } 54 | 55 | /// Get class XMLName 56 | /// w ##class(isc.py.util.Storage).GetXMLName() 57 | ClassMethod GetXMLName(class As %Dictionary.CacheClassname = {$classname()}) As %String 58 | { 59 | set xmlname = ..#XMLNAME 60 | set:xmlname="" xmlname = $$$ClassShortName(class) 61 | quit xmlname 62 | } 63 | 64 | } 65 | 66 | -------------------------------------------------------------------------------- /isc/py/util/Redirect.cls: -------------------------------------------------------------------------------- 1 | /// Utils to redirect stdout 2 | /// set r=##class(isc.py.util.Redirect).%New() 3 | /// w r.Init() 4 | /// w r.Enable(1) 5 | /// w r.%GetString() 6 | Class isc.py.util.Redirect Extends %RegisteredObject 7 | { 8 | 9 | /// Catcher IO 10 | Property IO; 11 | 12 | /// Old IO, whatever it was 13 | Property OldIO; 14 | 15 | /// Alias of the SYS module 16 | Property SYS; 17 | 18 | /// Devive name 19 | Property Device As %String(VALUELIST = "stdout,stderr,stdin") [ InitialExpression = "stdout" ]; 20 | 21 | /// Init devices 22 | Method Init() 23 | { 24 | set sc = ##class(isc.py.Main).ImportModule("io",,.io) 25 | quit:$$$ISERR(sc) sc 26 | 27 | set sc = ##class(isc.py.Main).ImportModule("sys",,.sys) 28 | set ..SYS = sys 29 | quit:$$$ISERR(sc) sc 30 | 31 | set ..OldIO = ##class(isc.py.gw.DynamicObject).%GetNewVar() 32 | set sc = ##class(isc.py.Main).SimpleString(..OldIO _ "=" _ ..SYS _ "." _ ..Device) 33 | quit:$$$ISERR(sc) sc 34 | 35 | set ..IO = ##class(isc.py.gw.DynamicObject).%GetNewVar() 36 | set sc = ##class(isc.py.Main).SimpleString(..IO _ "=" _ io _ ".StringIO()") 37 | quit:$$$ISERR(sc) sc 38 | 39 | 40 | quit sc 41 | } 42 | 43 | /// If new = 1 enables redirect, otherwise disables redirect 44 | Method Enable(new As %Boolean = {$$$YES}) [ CodeMode = expression ] 45 | { 46 | ##class(isc.py.Main).SimpleString(..SYS _ "." _ ..Device _ "=" _ $case(new, $$$YES: ..IO, :..OldIO)) 47 | } 48 | 49 | /// Get current device output. 50 | /// If flush = 1 (default) then device would be cleaned after getting output. 51 | Method GetString(flush As %Boolean = {$$$YES}) As %String 52 | { 53 | set sc = ##class(isc.py.Main).SimpleString("zzzbuffer = " _ ..IO _ ".getvalue()", "zzzbuffer", , .zzzbuffer) 54 | quit:$$$ISERR(sc) sc 55 | 56 | set sc = ##class(isc.py.Main).SimpleString("del zzzbuffer") 57 | quit:$$$ISERR(sc) sc 58 | 59 | set:flush sc = ##class(isc.py.Main).SimpleString(..IO _ ".truncate(0)" _ $c(10) _ ..IO _ ".seek(0)") 60 | quit:$$$ISERR(sc) sc 61 | 62 | quit $replace(zzzbuffer, $c(10), $$$NL) 63 | } 64 | 65 | } 66 | 67 | -------------------------------------------------------------------------------- /isc/py/msg/GlobalRequest.cls: -------------------------------------------------------------------------------- 1 | /// Transfer Global Dat to Python 2 | Class isc.py.msg.GlobalRequest Extends (Ens.Request, isc.py.msg.DataRequest) 3 | { 4 | 5 | /// global name without "^" 6 | Property Global As %String [ Required ]; 7 | 8 | /// Initial global key. 9 | Property Start As %Integer [ InitialExpression = 1 ]; 10 | 11 | /// Final global key. 12 | Property End As %Integer; 13 | 14 | /// Mask for global values. Mask may be shorter than the number of global value fields (in this case fields at the end would be skipped). 15 | /// How to format mask: 16 | /// `+` use field as is 17 | /// `-` skip field 18 | /// `b` - boolean (0 - False, anything else - True) 19 | /// `d` - date (from $horolog, on Windows only from 1970, on Linux from 1900 see notes for details) 20 | /// `t` - time ($horolog, seconds since midnight) 21 | /// `m` - (moment) timestamp string in YEAR-MONTH-DAY HOUR:MINUTE:SECOND format. 22 | Property Mask As %VarString; 23 | 24 | /// - `labels` - %List of column names, first element is key column name. 25 | /// Therefore: List length must be mask symbol length + 1. 26 | /// If comma-separated string is used it would be converted to %List automatically. 27 | Property Labels As %List; 28 | 29 | Method GlobalSet(value) As %Status 30 | { 31 | set:$e(value)="^" value = $e(value, 2, *) 32 | set i%Global = value 33 | quit $$$OK 34 | } 35 | 36 | Method LabelsSet(value) As %Status 37 | { 38 | set:'$lv(value) value = $lts(value) 39 | set i%Labels = value 40 | quit $$$OK 41 | } 42 | 43 | Storage Default 44 | { 45 | 46 | "GlobalRequest" 47 | 48 | Global 49 | 50 | 51 | Start 52 | 53 | 54 | End 55 | 56 | 57 | Mask 58 | 59 | 60 | Labels 61 | 62 | 63 | Namespace 64 | 65 | 66 | Type 67 | 68 | 69 | Variable 70 | 71 | 72 | GlobalRequestDefaultData 73 | %Library.CacheStorage 74 | } 75 | 76 | } 77 | 78 | -------------------------------------------------------------------------------- /isc/py/util/Evaluator.cls: -------------------------------------------------------------------------------- 1 | /// Evaluate strings 2 | Class isc.py.util.Evaluator 3 | { 4 | 5 | /// Evaluate a string with expressions: 6 | /// ${class:method:arg1:...:argN} - execute method 7 | /// #{expr} - execute ObjectScript code 8 | /// write ##class(isc.py.util.Evaluator).Evaluate("123 #{2+3} ${%PopulateUtils:Integer:1:100}") 9 | ClassMethod Evaluate(expr As %String) As %String 10 | { 11 | #dim sc As %Status = $$$OK 12 | set result = expr 13 | 14 | // First, let's evaluate ${} expressions 15 | $$$TOE(sc, ..EvalMethods(expr, .result)) 16 | 17 | // Next, let's evaluate #{} expressions 18 | $$$TOE(sc, ..EvalOS(result, .result)) 19 | 20 | quit result 21 | } 22 | 23 | /// Evaluate ${} expressions 24 | ClassMethod EvalMethods(expr As %String, Output result As %String) As %Status 25 | { 26 | quit ..EvalToken(expr, "$", "EvalMethod", .result) 27 | } 28 | 29 | /// Evaluate #{} expressions 30 | ClassMethod EvalOS(expr As %String, Output result As %String) As %Status 31 | { 32 | quit ..EvalToken(expr, "#", "EvalCode", .result) 33 | } 34 | 35 | /// Evaluate X{} expressions 36 | ClassMethod EvalToken(expr As %String, type As %String, function As %String, Output result As %String) As %Status 37 | { 38 | #dim sc As %Status = $$$OK 39 | set result=expr 40 | while (1) { 41 | set p = $find(result, type _ "{") 42 | quit:'p 43 | 44 | set q = $find(result, "}", p) 45 | quit:'q 46 | 47 | // Grab the token 48 | set token = $extract(result, p, q-($l(type)+1)) 49 | set value = $classmethod(, function, token) 50 | 51 | // Insert the new value 52 | set result = $extract(result, 1, p-3) _ $g(value) _ $extract(result, q, $l(result)) 53 | } 54 | quit sc 55 | } 56 | 57 | /// Evaluate method: class:method:arg1:...:argN 58 | ClassMethod EvalMethod(code As %String = "") As %String 59 | { 60 | set result = "" 61 | set length = $l(code, ":") 62 | 63 | if length<2 { 64 | throw ##class(%Exception.General).%New("", code) 65 | } else { 66 | for i=3:1:length { 67 | set args(i-2) = $p(code, ":", i) 68 | } 69 | set args = length - 2 70 | set result = $classmethod($p(code, ":"), $p(code, ":", 2), args...) 71 | } 72 | 73 | quit result 74 | } 75 | 76 | /// Evaluate an ObjectScript expression 77 | ClassMethod EvalCode(code As %String) As %String 78 | { 79 | quit @code 80 | } 81 | 82 | } 83 | 84 | -------------------------------------------------------------------------------- /Dockerfile-GPU: -------------------------------------------------------------------------------- 1 | FROM intersystemsdc/irispy 2 | 3 | ARG CUDA=11.2 4 | ARG CUDNN=8.1.0.77-1 5 | ARG CUDNN_MAJOR_VERSION=8 6 | ARG LIB_DIR_PREFIX=x86_64 7 | ARG LIBNVINFER=8.0.1-1 8 | ARG LIBNVINFER_MAJOR_VERSION=8 9 | 10 | USER root 11 | 12 | # Needed for string substitution 13 | SHELL ["/bin/bash", "-c"] 14 | # Install Cuda and some TF dependencies 15 | RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb && \ 16 | dpkg -i cuda-repo-ubuntu1804_10.0.130-1_amd64.deb && \ 17 | apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub && \ 18 | rm -f cuda-repo-ubuntu1804_10.0.130-1_amd64.deb && \ 19 | apt-get update && apt-get install -y --no-install-recommends \ 20 | build-essential \ 21 | cuda-command-line-tools-${CUDA/./-} \ 22 | libcublas-${CUDA/./-} \ 23 | cuda-nvrtc-${CUDA/./-} \ 24 | libcufft-${CUDA/./-} \ 25 | libcurand-${CUDA/./-} \ 26 | libcusolver-${CUDA/./-} \ 27 | libcusparse-${CUDA/./-} \ 28 | curl \ 29 | libcudnn8=${CUDNN}+cuda${CUDA} \ 30 | libfreetype6-dev \ 31 | libhdf5-serial-dev \ 32 | libzmq3-dev \ 33 | pkg-config \ 34 | software-properties-common \ 35 | unzip \ 36 | cuda=11.2.2-1 \ 37 | libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.3 \ 38 | libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.3 \ 39 | && apt-get clean \ 40 | && rm -rf /var/lib/apt/lists/* 41 | 42 | # For CUDA profiling, TensorFlow requires CUPTI. 43 | ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH 44 | 45 | # Link the libcuda stub to the location where tensorflow is searching for it and reconfigure 46 | # dynamic linker run-time bindings 47 | RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ 48 | && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ 49 | && ldconfig 50 | 51 | # Options: 52 | # tensorflow 53 | # tensorflow-gpu 54 | # tf-nightly 55 | # tf-nightly-gpu 56 | # Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version. 57 | # Installs the latest version by default. 58 | ARG TF_PACKAGE=tensorflow 59 | ARG TF_PACKAGE_VERSION= 60 | RUN python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}} 61 | 62 | USER irisowner -------------------------------------------------------------------------------- /isc/py/data/Function.cls: -------------------------------------------------------------------------------- 1 | Class isc.py.data.Function Extends %Persistent 2 | { 3 | 4 | /// Function Name 5 | Property Name As %String(MAXLEN = 100); 6 | 7 | /// dill serialization 8 | Property Dill As %Stream.GlobalCharacter; 9 | 10 | /// Function documentation 11 | Property Docs As %VarString; 12 | 13 | /// Function signature 14 | Property Signature As %VarString; 15 | 16 | /// JSON representation of Signature 17 | /// str is 'FullArgSpec(args=[\\'a\\', \\'b\\'], varargs=None, varkw=None, defaults=None, kwonlyargs=[\\'c\\', \\'d\\'], kwonlydefaults={\\'d\\': 10}, annotations={})' 18 | /// json is [args, varargs, varkw, kwonlyargs, kwonlydefaults, annotations] 19 | Property Arguments As %VarString; 20 | 21 | ClassMethod SaveFunction(function As %String = "", Output obj As isc.py.data.Function) As %Status 22 | { 23 | kill obj 24 | quit:function="" $$$ERROR($$$GeneralError, "Variable name can't be empty") 25 | 26 | set sc = ##class(isc.py.Main).GetFunctionInfo(function, .defined, .type, .docs, .signature, .arguments) 27 | quit:$$$ISERR(sc) sc 28 | 29 | set sc = ##class(isc.py.Main).GetVariablePickle(function ,.dill,, $$$YES) 30 | 31 | set obj = ..%New() 32 | set obj.Name = function 33 | set obj.Docs = docs 34 | set obj.Signature = signature 35 | set obj.Arguments = arguments 36 | set obj.Dill = dill 37 | 38 | 39 | quit obj.%Save() 40 | } 41 | 42 | /// Restore variable from disk. 43 | /// context - variable owner Python context object. 44 | Method Restore(context As isc.py.data.Context) As %Status 45 | { 46 | #dim sc As %Status = $$$OK 47 | if ..Dill.Size > 0 { 48 | set stream = ##class(%Stream.TmpCharacter).%New() 49 | do stream.Write(context.%DillAlias _ ".loads(") 50 | do stream.CopyFrom(..Dill) 51 | do stream.Write(")") 52 | set sc = ##class(isc.py.Main).ExecuteCode(stream, ..Name) 53 | } 54 | 55 | quit sc 56 | } 57 | 58 | /// Dispaly function on the current device. 59 | Method Display(indent As %String = "") 60 | { 61 | write indent, ..Name, ..Signature, $case(..Docs'="", $$$YES: " - " _ ..Docs, :""), ! 62 | } 63 | 64 | Storage Default 65 | { 66 | 67 | 68 | %%CLASSNAME 69 | 70 | 71 | Name 72 | 73 | 74 | Dill 75 | 76 | 77 | Docs 78 | 79 | 80 | Signature 81 | 82 | 83 | Arguments 84 | 85 | 86 | ^isc.py.data.FunctionD 87 | FunctionDefaultData 88 | ^isc.py.data.FunctionD 89 | ^isc.py.data.FunctionI 90 | ^isc.py.data.FunctionS 91 | %Library.CacheStorage 92 | } 93 | 94 | } 95 | 96 | -------------------------------------------------------------------------------- /isc/py/ens/SQL.cls: -------------------------------------------------------------------------------- 1 | Include %occErrors 2 | 3 | Class isc.py.ens.SQL 4 | { 5 | 6 | /// Execute arbitrarty query. 7 | /// query - sql statement or class:Query or class:XData 8 | /// args - query arguments if any 9 | /// w $System.Status.GetErrorText(##class(isc.py.ens.SQL).Execute()) 10 | ClassMethod Execute(query, args...) As %Status 11 | { 12 | #dim sc As %Status = $$$OK 13 | set sc = ..GetSQL(query, .sql) 14 | quit:$$$ISERR(sc) sc 15 | 16 | set st = ##class(%SQL.Statement).%New() 17 | set sc = st.%Prepare(sql) 18 | quit:$$$ISERR(sc) sc 19 | 20 | #dim result As %SQL.StatementResult 21 | set result = st.%Execute(args...) 22 | 23 | if result.%SQLCODE'=0 { 24 | set sc = $System.Status.Error($$$SQLError, result.%SQLCODE, result.%Message) 25 | } 26 | 27 | quit sc 28 | } 29 | 30 | /// w ##class(isc.py.ens.SQL).GetSQL("SELECT 1") 31 | /// w ##class(isc.py.ens.SQL).GetSQL("class:Query") 32 | /// w ##class(isc.py.ens.SQL).GetSQL("class:XData") 33 | ClassMethod GetSQL(query, Output sql) As %Status 34 | { 35 | #dim sc As %Status = $$$OK 36 | set sql = "" 37 | 38 | if (($l(query, ":")=2) && (##class(%Dictionary.CompiledClass).%ExistsId($p(query, ":")))) { 39 | set classname = $p(query, ":") 40 | set element = $p(query, ":", 2) 41 | 42 | if ##class(%Dictionary.QueryDefinition).IDKEYExists(classname, element) { 43 | set sql = ..GetClassQuery(classname, element) 44 | } elseif ##class(%Dictionary.XDataDefinition).IDKEYExists(classname, element) { 45 | set sql = ..GetClassXData(classname, element) 46 | } else { 47 | set sc = $$$ERROR($$$GeneralError, $$$FormatText("Class %1 does not have Query or XData named %2", class, queryName)) 48 | } 49 | } else { 50 | set sql = query 51 | } 52 | quit sc 53 | } 54 | 55 | /// Get XData contents from classname 56 | /// w ##class(isc.py.ens.SQL).GetClassXData("isc.py.test.Process", "BPL") 57 | ClassMethod GetClassXData(classname, xdata) As %String 58 | { 59 | set str = "" 60 | for i=1:1:$$$comMemberKeyGet(classname,$$$cCLASSxdata,xdata,$$$cXDATAdata) { 61 | set str = str _ $$$comMemberArrayGet(classname,$$$cCLASSxdata,xdata,$$$cXDATAdata,i) _ $$$NL 62 | } 63 | quit str 64 | } 65 | 66 | /// Get Query contents from classname 67 | /// w ##class(isc.py.ens.SQL).GetClassQuery("", "") 68 | ClassMethod GetClassQuery(classname, query) As %String [ CodeMode = expression ] 69 | { 70 | $$$comMemberKeyGet(classname,$$$cCLASSquery,query,$$$cQUERYsqlquery) 71 | } 72 | 73 | /// do ##class(isc.py.ens.SQL).Test() 74 | ClassMethod Test() 75 | { 76 | set req = ##class(isc.py.msg.SQLRequest).%New() 77 | set req.Query = "INSERT INTO test.p (City, Name, Phone) VALUES (?, ?, ?)" 78 | do req.Args.Insert(4) 79 | do req.Args.Insert(5) 80 | do req.Args.Insert(6) 81 | 82 | set sc = ##class(EnsLib.Testing.Service).SendTestRequest("isc.py.ens.Operation", req, .resp, .sessionId, $$$YES) 83 | w $System.Status.GetErrorText(sc),! 84 | zw resp,sessionId 85 | } 86 | 87 | } 88 | 89 | -------------------------------------------------------------------------------- /isc/py/util/Matcher.cls: -------------------------------------------------------------------------------- 1 | /// Utility to match input against comma-separated string of masks. 2 | Class isc.py.util.Matcher 3 | { 4 | 5 | /// Returns $$$YES if input matches at least one element from the list 6 | /// input - string to match 7 | /// list - comma separated list of masks containig * and ? 8 | /// write ##class(isc.py.util.Matcher).MatchOr() 9 | ClassMethod MatchOr(input As %String, list As %String) As %Boolean 10 | { 11 | set ok = $$$NO 12 | for pie = 1:1:$length(list,",") { 13 | set mask = $piece(list,",",pie) 14 | if mask'="",..Match(input,mask) { 15 | set ok = $$$YES 16 | } 17 | } 18 | quit ok 19 | } 20 | 21 | /// Returns $$$YES if input matches all elements from the list 22 | /// input - string to match 23 | /// list - comma separated list of masks containig * and ? 24 | /// write ##class(isc.py.util.Matcher).MatchAnd() 25 | ClassMethod MatchAnd(input As %String, list As %String) As %Boolean 26 | { 27 | set ok = $$$YES 28 | for pie = 1:1:$length(list,",") { 29 | set mask = $piece(list,",",pie) 30 | if mask'="",'..Match(input,mask) { 31 | set ok = $$$NO 32 | } 33 | } 34 | quit ok 35 | } 36 | 37 | /// Returns $$$YES if input matches the mask 38 | /// write ##class(isc.py.util.Matcher).Match() 39 | ClassMethod Match(input As %String, mask As %String) As %Boolean [ CodeMode = expression ] 40 | { 41 | input?@..MaskToPattern(mask) 42 | } 43 | 44 | /// Translate mask into a pattern 45 | /// write ##class(isc.py.util.Matcher).MaskToPattern() 46 | ClassMethod MaskToPattern(mask As %String) As %String 47 | { 48 | set pattern = "" 49 | set char = "" 50 | for pos = 1:1:$length(mask) { 51 | set curChar = $extract(mask, pos) 52 | if curChar = "*" { 53 | set pattern = pattern _ $select(char="":"", 1:"1"""_char_"""") _ ".E" 54 | set char = "" 55 | } elseif curChar = "?" { 56 | set pattern = pattern _ $select(char="":"", 1:"1"""_char_"""") _ "1E" 57 | set char = "" 58 | } else { 59 | set char = char _ curChar 60 | } 61 | } 62 | set pattern = pattern _ $select(char="":"", 1:"1"""_char_"""") 63 | quit pattern 64 | } 65 | 66 | /// Filter %List against a comma-separated list of masks. 67 | /// List element should both be on a whiteMask and not on blackMask 68 | /// Skip mask to skip that check. 69 | /// zwrite ##class(isc.py.util.Matcher).FilterList($lb("classA","classB","_classC","pack._ClassD","classA_B","clasE_"), "*", "_*,*_") 70 | ClassMethod FilterList(input As %List, whiteMask As %List = "", blackMask As %List = "") As %List 71 | { 72 | set whiteList = $lfs(whiteMask) 73 | set blackList = $lfs(blackMask) 74 | set out = "" 75 | set ptr=0 76 | while $listnext(input, ptr, value) { 77 | if $lf(whiteList, value) { 78 | // Always add exact matches 79 | } elseif $lf(blackList, value) { 80 | continue 81 | } else { 82 | continue:((whiteMask'="")&&('..MatchOr(value, whiteMask))) 83 | continue:((blackMask'="")&&(..MatchOr(value, blackMask))) 84 | } 85 | set out = out _ $lb(value) 86 | } 87 | 88 | quit out 89 | } 90 | 91 | } 92 | 93 | -------------------------------------------------------------------------------- /isc/py/ens/ProcessUtils.cls: -------------------------------------------------------------------------------- 1 | /// Utilities for Python PB 2 | Class isc.py.ens.ProcessUtils 3 | { 4 | 5 | /// List of nodes that may contain ActivityList. 6 | /// Retrieved with: 7 | /// SELECT parent, ID, Name, Type 8 | /// FROM %Dictionary.CompiledProperty 9 | /// WHERE Parent %STARTSWITH 'Ens.BPL' AND Name='Activities' 10 | Parameter ActivityNodes = {$lb("Ens.BPL.Case", "Ens.BPL.CaseNode", "Ens.BPL.Catch", "Ens.BPL.CatchAll", "Ens.BPL.CompensationHandler", "Ens.BPL.Default", "Ens.BPL.Flow", "Ens.BPL.ForEach", "Ens.BPL.Message", "Ens.BPL.Request", "Ens.BPL.Response", "Ens.BPL.Scope", "Ens.BPL.Sequence", "Ens.BPL.Until", "Ens.BPL.While")}; 11 | 12 | /// Get value of activity annatation by name 13 | ClassMethod GetAnnotation(name As %String) As %String [ CodeMode = objectgenerator ] 14 | { 15 | set sc = $$$OK 16 | try { 17 | #; Don't run this method on the base class 18 | quit:%compiledclass.Name="isc.py.ens.AbstractBPLProcess" 19 | quit:##class(%Dictionary.CompiledXData).%ExistsId(%class.Name_"||BPLERROR") 20 | 21 | #; find XDATA block named BPL 22 | if ##class(%Dictionary.CompiledXData).%ExistsId(%class.Name_"||BPL") { 23 | set index = %compiledclass.XDatas.FindObjectId(%class.Name_"||BPL") 24 | 25 | #; get XDATA as stream 26 | set stream = %compiledclass.XDatas.GetAt(index).Data 27 | do stream.Rewind() 28 | 29 | set parser=##class(Ens.BPL.Parser).%New() 30 | 31 | #; Parse the specified stream and obtain the root of the parse tree, 32 | #; Ens.BPL.Process 33 | #dim process As Ens.BPL.Process 34 | set sc = parser.ParseStream(stream,.process) 35 | quit:$$$ISERR(sc) 36 | 37 | #dim activities As Ens.BPL.ActivityList 38 | set activities = process.Sequence.Activities 39 | do ..ParseActivities(activities) 40 | 41 | do %code.WriteLine(" quit """"") 42 | } 43 | } catch ex { 44 | set sc = ex.AsStatus() 45 | } 46 | quit sc 47 | } 48 | 49 | /// Get value of activity annatation by name as stream 50 | ClassMethod GetAnnotationStream(name As %String) As %Stream.GlobalCharacter 51 | { 52 | set stream = ##class(%Stream.GlobalCharacter).%New() 53 | do stream.Write(..GetAnnotation(name)) 54 | quit stream 55 | } 56 | 57 | ClassMethod ParseActivities(activities As Ens.BPL.ActivityList) 58 | { 59 | for i=1:1:activities.Count() { 60 | #dim activity As Ens.BPL.Activity 61 | set activity = activities.GetAt(i) 62 | set class = activity.%ClassName(1) 63 | if $lf(..#ActivityNodes, class) { 64 | do ..ParseActivities(activity.Activities) 65 | } elseif (class="Ens.BPL.If") { 66 | do ..ParseActivities(activity.True) 67 | do ..ParseActivities(activity.False) 68 | } elseif (class ="Ens.BPL.Switch"){ 69 | do ..ParseActivities(activity.Cases) 70 | } else { 71 | set annotationText = $$$quote(activity.Annotation) 72 | do:activity.Annotation'="" %code.WriteLine(" quit:name=""" _ activity.Name _ """ ##class(isc.py.util.Evaluator).Evaluate(" _ annotationText _ ")") 73 | } 74 | } 75 | } 76 | 77 | } 78 | 79 | -------------------------------------------------------------------------------- /isc/py/init/Test.cls: -------------------------------------------------------------------------------- 1 | /// Test initializer 2 | /// do ##class(isc.py.init.Test).Initialize(,1) 3 | Class isc.py.init.Test Extends isc.py.init.Abstract 4 | { 5 | 6 | Parameter Modules = {$lb("types", "json", "inspect", "random")}; 7 | 8 | ClassMethod InitCube() As %Status [ CodeMode = expression ] 9 | { 10 | ##class(isc.py.Main).SimpleString( "def cube(x):" _ $c(10) _ 11 | " return x*x*x") 12 | } 13 | 14 | ClassMethod InitArgs() As %Status [ CodeMode = expression ] 15 | { 16 | ##class(isc.py.Main).SimpleString( "def posargs(parg, *args):" _ $c(10) _ 17 | " print(""first positional arg:"", parg)" _ $c(10) _ 18 | " for arg in args:" _ $c(10) _ 19 | " print(""another arg through *args:"", arg)") 20 | } 21 | 22 | ClassMethod InitKWargs() As %Status [ CodeMode = expression ] 23 | { 24 | ##class(isc.py.Main).SimpleString( "def kwargs(name, **kwargs):" _ $c(10) _ 25 | " for key, value in kwargs.items():" _ $c(10) _ 26 | " print(""{0} = {1}"".format(key, value))") 27 | } 28 | 29 | /// * separates positional and keyword arguments 30 | ClassMethod InitFoo() As %Status [ CodeMode = expression ] 31 | { 32 | ##class(isc.py.Main).SimpleString( "def foo(a, b, *, c, d=10):" _ $c(10) _ 33 | " pass") 34 | } 35 | 36 | ClassMethod InitAllargs() As %Status [ CodeMode = expression ] 37 | { 38 | ##class(isc.py.Main).SimpleString( "def allargs(parg, *args, name=1, **kwargs):" _ $c(10) _ 39 | " print(""first positional arg:"", parg)" _ $c(10) _ 40 | " for arg in args:" _ $c(10) _ 41 | " print(""arg from *argv:"", arg)" _ $c(10) _ 42 | " print(""first keyword arg name:"", name)" _ $c(10) _ 43 | " for key, value in kwargs.items():" _ $c(10) _ 44 | " print(""kwarg: {0} = {1}"".format(key, value))") 45 | } 46 | 47 | ClassMethod InitPerson() As %Status [ CodeMode = expression ] 48 | { 49 | ##class(isc.py.Main).SimpleString( "class Person(object):" _ $c(10) _ 50 | " def __init__(self, name, age, city):" _ $c(10) _ 51 | " self.name = name" _ $c(10) _ 52 | " self.age = age" _ $c(10) _ 53 | " self.city = city" _ $c(10) _ 54 | " def getAge(self):" _ $c(10) _ 55 | " return self.age" _ $c(10) _ 56 | " def getAgePlus(self, add):" _ $c(10) _ 57 | " return self.age + add") 58 | } 59 | 60 | /// https://www.python-course.eu/python3_class_and_instance_attributes.php 61 | ClassMethod InitZEmployee() As %Status [ CodeMode = expression ] 62 | { 63 | ##class(isc.py.Main).SimpleString( "class Employee(Person):" _ $c(10) _ 64 | " def __init__(self, name, age, city, company):" _ $c(10) _ 65 | " super().__init__(name, age, city)" _ $c(10) _ 66 | " self.company = company" _ $c(10) _ 67 | " def getCompany(self):" _ $c(10) _ 68 | " return self.company" _ $c(10) _ 69 | " def getAgePlus(self, add):" _ $c(10) _ 70 | " return self.age + add*2") 71 | } 72 | 73 | } 74 | 75 | -------------------------------------------------------------------------------- /jupyter/Readme.md: -------------------------------------------------------------------------------- 1 | 2 | # Introduction 3 | 4 | The [Jupyter Notebook](https://jupyter.org/) is an open-source web application that allows you to create and share documents that contain live code, equations, visualizations and narrative text. 5 | 6 | This extension allows you to browse and edit InterSystems IRIS BPL processes as jupyter notebooks. 7 | 8 | Note that currently default Python 3 executor is used. 9 | 10 | This extension assumes that annotations contain Python code and uses activities names as preceding read-only headings. 11 | 12 | This is a direct Jupyter<->BPL bridge, at no time file representations of processes or notebooks are created. 13 | 14 | # Screenshots 15 | 16 | ## Process Explorer 17 | ![](https://i.imgur.com/DAW30UZ.png) 18 | 19 | ## Process Editor 20 | 21 | ![](https://i.imgur.com/43qA3B1.png) 22 | 23 | # Installation 24 | 25 | ## Prerequisites 26 | 27 | 1. You'll need [InterSystems IRIS 2019.2+](https://wrc.intersystems.com/wrc/). 28 | 2. Install PythonGateway v0.8+ (only `isc.py.util.Jupyter` and `isc.py.ens.ProcessUtils` are required). 29 | 3. Update ObjectScript code from the repo edge. 30 | 31 | ## Automatic installation 32 | 33 | 1. Run `do ##class(isc.py.util.Jupyter).Install()` and follow the prompt. 34 | 35 | ## Manual installation 36 | 37 | Manual installation contains all the same steps as automatic installation, but you need to execute them manually. 38 | 39 | 1. Install IRISNative for Python 3.6.7 (`cp3` **OR** `py3` should be in a finename, wheel is in `\dev\python\` folder inside InterSystems IRIS installation path): 40 | ``` 41 | pip install \dev\python\irisnative-*cp3*.whl 42 | pip install \dev\python\*py3*.whl 43 | ``` 44 | 45 | 2. Install Jupyter. 46 | ``` 47 | pip install jupyter 48 | ``` 49 | 3. Check [jupyter_notebook_config.py](jupyter_notebook_config.py). It assumes the following defaults for IRIS connection: 50 | - host: `localhost` 51 | - port: `51773` 52 | - namespace: `USER` 53 | - user: `_SYSTEM` 54 | - password: `SYS` 55 | 56 | 4. If you need other connection parameters values, modify `jupyter_notebook_config.py`. 57 | For example to connect to InterSystems IRIS instance on port `51776` you'll need to add this line to the bottom of `jupyter_notebook_config.py`: 58 | ``` 59 | c.MLContentsManager.port = 51776 60 | ``` 61 | 62 | You can configure all settings (host, port, namespace, user, password) this way. Completely customized config looks like this: 63 | ``` 64 | exec(open("MLContentsManager.py").read()) 65 | c.NotebookApp.contents_manager_class = MLContentsManager 66 | c.MLContentsManager.host = 'localhost' 67 | c.MLContentsManager.port = 51773 68 | c.MLContentsManager.namespace = 'USER' 69 | c.MLContentsManager.user = '_SYSTEM' 70 | c.MLContentsManager.password = 'SYS' 71 | ``` 72 | 73 | # Docker 74 | 75 | You can also run PythonGateway with Jupyter in Docker. To run the latest image execute: 76 | ``` 77 | docker run -d \ 78 | -p 14558:52773 \ 79 | -p 8888:8888 \ 80 | --name irisj \ 81 | intersystemsdc/irispyj:latest \ 82 | --log $ISC_PACKAGE_INSTALLDIR/mgr/messages.log \ 83 | --after /start.sh 84 | ``` 85 | 86 | You might be required to start jupyter manually, to do that execute: `docker exec -d irisj sh /start.sh`. 87 | 88 | To build Docker image execute: `docker build --force-rm --tag intersystemsdc/irispyj:latest .` 89 | 90 | 91 | # Runnning 92 | 93 | After completing automatic or manual installation open OS bash in the folder with `jupyter_notebook_config.py` and `MLContentsManager.py` and start jupyter with: 94 | 95 | ``` 96 | jupyter notebook 97 | ``` 98 | 99 | # Notes 100 | 101 | - Ignore `Checkpoint failed` warning on save 102 | 103 | ![](https://i.imgur.com/Vg0H4U8.png) 104 | -------------------------------------------------------------------------------- /isc/py/init/Abstract.cls: -------------------------------------------------------------------------------- 1 | Class isc.py.init.Abstract [ Abstract ] 2 | { 3 | 4 | /// List of modules to import in a form of: 5 | /// $lb($lb("module1", "alias1"), $lb("module2", "alias2"), "module3", ...) 6 | Parameter Modules; 7 | 8 | Query MethodsList(class = {$classname()}) As %SQLQuery 9 | { 10 | SELECT 11 | Name 12 | FROM %Dictionary.CompiledMethod 13 | WHERE ClassMethod=1 14 | AND Name %STARTSWITH 'Init' 15 | AND Name != 'Initialize' 16 | AND parent = :class 17 | ORDER BY Name ASC 18 | } 19 | 20 | /// Initializes Python environment. Idempotent. 21 | /// Extend this or Default class to execute code on Python process startup 22 | /// set sc = ##class(isc.py.init.Abstract).Initialize() 23 | ClassMethod Initialize(file As %String = "", verbose As %Boolean = {$$$NO}) As %Status 24 | { 25 | try { 26 | #dim sc As %Status = $$$OK 27 | 28 | do:verbose ..Log("Initializing callout library, file argument: '" _ file _ "'") 29 | set sc = ##class(isc.py.Callout).Initialize(file) 30 | do:verbose ..LogStatus("Initializing callout library", sc) 31 | quit:$$$ISERR(sc) 32 | 33 | do:verbose ..Log("Start Importing modules") 34 | set scModules = ..ImportModules(verbose) 35 | do:verbose ..Log("Finished Importing modules") 36 | 37 | 38 | do:verbose ..Log("Executing Init methods") 39 | set scMethods = ..ExecuteInitMethods(verbose) 40 | do:verbose ..Log("Finished executing Init methods") 41 | 42 | set sc = $$$ADDSC(scModules, scMethods) 43 | 44 | if verbose { 45 | if $$$ISERR(sc) { 46 | do ..Log("There were errors during initialization") 47 | } else { 48 | do ..Log("Initialization completed successfully") 49 | } 50 | } 51 | 52 | } catch ex { 53 | do:verbose ..LogException("Initialize", ex) 54 | set sc = ex.AsStatus() 55 | } 56 | 57 | quit sc 58 | } 59 | 60 | ClassMethod ImportModules(verbose As %Boolean = {$$$NO}) As %Status 61 | { 62 | #dim sc As %Status = $$$OK 63 | if '$listvalid(..#Modules) { 64 | do:verbose ..Log("Modules parameter must be list") 65 | set sc = $$$ERROR($$$AttributeValueMustBeList, "Modules parameter") 66 | quit sc 67 | } 68 | for i=1:1:$ll(..#Modules) { 69 | set moduleList = $lg(..#Modules, i) 70 | 71 | kill module, alias, sc1 72 | if $listvalid(moduleList) { 73 | set module = $lg(moduleList, 1) 74 | set alias = $lg(moduleList, 2) 75 | } else { 76 | set module = moduleList 77 | set alias = module 78 | } 79 | set sc1 = ##class(isc.py.Main).ImportModule(module, 0, alias) 80 | do:verbose ..LogStatus("Importing module: " _ module _ " as " _ alias, sc1) 81 | 82 | set sc = $$$ADDSC(sc, sc1) 83 | } 84 | 85 | quit sc 86 | } 87 | 88 | ClassMethod ExecuteInitMethods(verbose As %Boolean = {$$$NO}) 89 | { 90 | #dim sc As %Status = $$$OK 91 | 92 | #dim rs As %SQL.ISelectResult 93 | set rs = ..MethodsListFunc() 94 | while rs.%Next() { 95 | set method = rs.Name 96 | set sc1 = $classmethod(, method) 97 | do:verbose ..LogStatus("Executing method: " _ method, sc1) 98 | 99 | set sc = $$$ADDSC(sc, sc1) 100 | } 101 | 102 | quit sc 103 | } 104 | 105 | /// do ##class(isc.py.init.Abstract).LogException() 106 | ClassMethod LogException(msg As %String, ex As %Exception.AbstractException) 107 | { 108 | Do ..Log(msg _ ". Caught exception: " _ ex.DisplayString()) 109 | } 110 | 111 | /// do ##class(isc.py.init.Abstract).LogStatus() 112 | ClassMethod LogStatus(msg As %String, sc As %Status) 113 | { 114 | Do ..Log(msg _ ". Completed with " _ $select($$$ISERR(sc):$System.Status.GetErrorText(sc), 1:"success.")) 115 | } 116 | 117 | /// do ##class(isc.py.init.Abstract).Log() 118 | ClassMethod Log(msg As %String) 119 | { 120 | Write $ZDATETIME($ZTIMESTAMP, 3, 1) _ " " _ msg,! 121 | } 122 | 123 | } 124 | 125 | -------------------------------------------------------------------------------- /isc/py/query/Generator.cls: -------------------------------------------------------------------------------- 1 | /// Generate custom query based on a Python DataFrame 2 | Class isc.py.query.Generator [ Abstract ] 3 | { 4 | 5 | /// Main entry point. 6 | /// variable - Python variable with DataFrame 7 | /// classname - class to generate. 8 | /// 9 | /// w $System.Status.GetErrorText(##class(isc.py.query.Generator).Generate("df", "test.df")) 10 | ClassMethod Generate(variable As %String, classname As %Dictionary.CacheClassname) As %Status 11 | { 12 | set sc = ##class(isc.py.Main).GetVariableDefined(variable, .defined) 13 | quit:$$$ISERR(sc) sc 14 | quit:'defined $$$ERROR($$$GeneralError, "Variable is not defined: " _ variable) 15 | 16 | set sc = ##class(%Dictionary.Classname).IsValid(classname) 17 | quit:$$$ISERR(sc) sc 18 | 19 | set sc = ##class(isc.py.Main).GetVariableJson(variable _ ".dtypes", .dtypes, $$$YES) 20 | quit:$$$ISERR(sc) sc 21 | 22 | #dim class As %Dictionary.ClassDefinition 23 | set class = ..GetInitialClass(classname) 24 | 25 | set columns = {}.%FromJSON(dtypes) 26 | set colNames = "" 27 | set iterator = columns.%GetIterator() 28 | 29 | set sequenceNumber = 1 30 | while iterator.%GetNext(.colName, .colDesc) { 31 | set colNames = colNames _ $lb($case($zname(colName,6), 1: colName, 0: """" _ colName _ """")) 32 | do class.Properties.Insert(..GetColumn(colName, .colDesc, .sequenceNumber)) 33 | } 34 | 35 | do class.Methods.Insert(..GetFetchCursor(colNames)) 36 | do class.Parameters.Insert(..GetSQLNAME(classname)) 37 | 38 | set sc = class.%Save() 39 | quit:$$$ISERR(sc) sc 40 | 41 | set sc = ..Compile(classname) 42 | quit sc 43 | } 44 | 45 | /// Get class object. Populates docs and inheritance. 46 | ClassMethod GetInitialClass(classname As %Dictionary.CacheClassname) As %Dictionary.ClassDefinition 47 | { 48 | if ##class(%Dictionary.ClassDefinition).%ExistsId(classname) { 49 | do ##class(%Dictionary.ClassDefinition).%DeleteId(classname) 50 | } 51 | 52 | set class = ##class(%Dictionary.ClassDefinition).%New(classname, 26) 53 | set class.Super = "isc.py.query.Abstract, %SQL.CustomQuery" 54 | set class.GeneratedBy = $classname() 55 | set class.Description = "Custom Query for Python DataFrame" 56 | quit class 57 | } 58 | 59 | /// Generate property 60 | ClassMethod GetColumn(colName As %String, colDesc As %DynamicObject, ByRef sequenceNumber) As %Dictionary.PropertyDefinition 61 | { 62 | set sequenceNumber = sequenceNumber + 1 63 | set column = ##class(%Dictionary.PropertyDefinition).%New() 64 | set column.Name = colName 65 | set column.Type = "%String" 66 | set column.Description = colDesc.%ToJSON() 67 | set column.SequenceNumber = sequenceNumber 68 | quit column 69 | } 70 | 71 | /// Generate %FetchCursor method 72 | ClassMethod GetFetchCursor(colNames As %List) As %Dictionary.MethodDefinition 73 | { 74 | set method = ##class(%Dictionary.MethodDefinition).%New() 75 | set method.Name = "%FetchCursor" 76 | set method.ReturnType = "%Integer" 77 | do method.Implementation.WriteLine($c(9) _ "set $lb(i%" _ $lts(colNames, ", i%") _ ") = ..GetRow(.atEnd)") 78 | do method.Implementation.WriteLine($c(9) _ "quit 'atEnd") 79 | quit method 80 | } 81 | 82 | /// Generate SQLNAME parameter 83 | ClassMethod GetSQLNAME(classname As %Dictionary.CacheClassname) As %Dictionary.ParameterDefinition 84 | { 85 | set param = ##class(%Dictionary.ParameterDefinition).%New() 86 | set param.Name = "SQLNAME" 87 | set param.Type = "%String" 88 | set param.Default = $p(classname, ".", *) 89 | quit param 90 | } 91 | 92 | /// Compile generated class 93 | ClassMethod Compile(classname) As %Status 94 | { 95 | set displayFlags = "/checkuptodate=expandedonly /displaylog=0 /displayerror=0" 96 | set sc = $system.OBJ.Compile(classname, "ckub" _ displayFlags, .errors) 97 | quit sc 98 | } 99 | 100 | } 101 | 102 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ## [0.8] - 2019-09-12 10 | 11 | ### Added 12 | - Direct global transfer for fast global, class, table transfer. 13 | - Jupyter support. 14 | - Annotation evaluation. 15 | - (Experimental) Proxy Gateway support. 16 | - PMML integration. 17 | - IO redirection utils. 18 | 19 | ### Changed 20 | - Docker now can build from arbitrary image. 21 | - Docker now supports 2019.3+ base images. 22 | - Multiline mode for shell. 23 | - JSON harvesting improvements. 24 | - BPEmulator supports `process` and `context` objects. 25 | 26 | ## [0.7] - 2019-03-21 27 | ### Added 28 | - Docker support. 29 | - (Experimental) `ExecuteFunction` support. 30 | - (Experimental) Proxyless Gateway support. 31 | - `ExecuteQuery` support for types and namespaces. 32 | - JSON serialization for interoperability (also Repr, Pickle and Dill). 33 | - Saving context now saves user functions. 34 | - Python datatypes escaper: string, bool, date, time, timestamp. 35 | - Initializer interface. 36 | - Python shell: `do ##class(isc.py.util.Shell).Shell()` 37 | - Business Process emulator. 38 | - Sample %ZLANGC00 and %ZSTART routines. 39 | 40 | ### Changed 41 | - Dill is now a required module. 42 | - `GetVariableInfo` now returns FQCN as type. 43 | - `GetVariableInfo` support for attributes (methods and the like). 44 | - Test process now uses non-interactive GUI for test business process 45 | - Test process: new data, simplified code. 46 | - Log trace events enabled by default. 47 | 48 | ### Removed 49 | - Test methods GetRandom, GetRandomSimple, SimpleStringFull from callout API 50 | 51 | 52 | ## [0.6] - 2019-01-22 53 | ### Added 54 | - Mac OS X support. 55 | - Linux - ability to load Python library explicitly. 56 | - ExcuteQuery - load data from query without xDBC. 57 | - Exception handling to Callout interface. 58 | 59 | ### Changed 60 | - Test process now works in 3 modes: RAW, ODBC, JDBC. 61 | - Completed Main interface utility methods. 62 | 63 | ## [0.5] - 2019-01-18 64 | ### Added 65 | - Streams up to 2,147,483,647 bytes (~2Gb) now can be passed inside to python context and retrieved (up to IRIS limit) from it. 66 | - Mask and MaxLength conditions added to context persistence. 67 | - New class `isc.py.Main` as a main public interface. 68 | 69 | ### Changed 70 | - Context persistence uses streams. 71 | 72 | ## [0.4.2] - 2019-01-17 73 | ### Changed 74 | - Pass/retrieve limit raised to 3 641 144 characters. 75 | 76 | ## [0.4] - 2018-12-28 77 | ### Added 78 | - Context persistence. 79 | - Serialization options. 80 | - Unit tests. 81 | - Error management. 82 | 83 | ## [0.3] - 2018-12-24 84 | ### Changed 85 | - Transfer limit raised to 3 641 144 characters. 86 | 87 | ### Removed 88 | - Explicit initialization (now it's done implicitly). 89 | 90 | ## [0.2] - 2018-12-20 91 | ### Removed 92 | - Debugging. 93 | 94 | ## [0.1.1] - 2018-12-18 95 | ### Changed 96 | - Retrieving undefined variable does not crash the process now. 97 | 98 | ## [0.1] - 2018-12-18 99 | ### Added 100 | - Initial version. 101 | 102 | [Unreleased]: https://github.com/intersystems-community/PythonGateway/compare/v0.7...HEAD 103 | [0.8]: https://github.com/intersystems-community/PythonGateway/compare/v0.7...v0.8 104 | [0.7]: https://github.com/intersystems-community/PythonGateway/compare/v0.6...v0.7 105 | [0.6]: https://github.com/intersystems-community/PythonGateway/compare/v0.5...v0.6 106 | [0.5]: https://github.com/intersystems-community/PythonGateway/compare/v0.4.2...v0.5 107 | [0.4.2]: https://github.com/intersystems-community/PythonGateway/compare/v0.4...v0.4.2 108 | [0.4]: https://github.com/intersystems-community/PythonGateway/compare/v0.3...v0.4 109 | [0.3]: https://github.com/intersystems-community/PythonGateway/compare/v0.2...v0.3 110 | [0.2]: https://github.com/intersystems-community/PythonGateway/compare/v0.1.1...v0.2 111 | [0.1.1]: https://github.com/intersystems-community/PythonGateway/compare/v0.1...v0.1.1 112 | [0.1]: https://github.com/intersystems-community/PythonGateway/compare/318afe8dba59d6892943512941f9063c91fececa...v0.1 113 | -------------------------------------------------------------------------------- /isc/py/test/Person.cls: -------------------------------------------------------------------------------- 1 | Class isc.py.test.Person Extends (%Persistent, %Populate) 2 | { 3 | 4 | Property Name As %String; 5 | 6 | Property DOB As %Date(MINVAL = 47117); 7 | 8 | Property TS As %TimeStamp(MINVAL = "1970-01-01 00:00:00"); 9 | 10 | Property RandomTime As %Time; 11 | 12 | Property AgeYears As %Integer; 13 | 14 | Property AgeDecimal As %Decimal(SCALE = 2); 15 | 16 | Property AgeDouble As %Double(SCALE = 4); 17 | 18 | Property Bool As %Boolean; 19 | 20 | /// Test data transfer methods 21 | /// do ##class(isc.py.test.Person).Test() 22 | ClassMethod Test(count As %Integer = 100) 23 | { 24 | if count { 25 | do ..%KillExtent() 26 | do ..Populate(count,,,,$$$NO) 27 | } 28 | 29 | write "Global structure:",! 30 | zw ^isc.py.test.PersonD(1) 31 | 32 | // Import now so it does not affect timings 33 | set sc = ##class(isc.py.Main).ImportModule("pandas") 34 | write:$$$ISERR(sc) $System.Status.GetErrorText(sc) 35 | 36 | // All the ways to transfer data 37 | set query = "SELECT * FROM isc_py_test.Person" 38 | set class = "isc.py.test.Person" 39 | set table = "isc_py_test.Person" 40 | set global = "isc.py.test.PersonD" 41 | 42 | // Common arguments 43 | set variable = "df" 44 | set type = "dataframe" 45 | set start = 1 46 | set end = $select(count>0:count, 1: $g(^isc.py.test.PersonD, 1)) 47 | 48 | // Approach 0: ExecuteGlobal without arguments 49 | set t1 = $zh 50 | set sc = ##class(isc.py.Main).ExecuteGlobal(global, variable _ 0, type) 51 | write:$$$ISERR(sc) $System.Status.GetErrorText(sc) 52 | set t2 = $zh 53 | write !,"global auto time: ", t2-t1,! 54 | write ##class(isc.py.util.BPEmulator).DescribeDataframe(variable _ 0),! 55 | 56 | 57 | // Approach 1: ExecuteGlobal with arguments 58 | // for global transfer labels are not calculated automatically 59 | set labels = $lb("globalKey", "Name", "DOB", "TS", "RandomTime", "AgeYears", "AgeDecimal", "AgeDouble", "Bool") 60 | 61 | // mask is 1 element shorter than labels because "globalKey" is global subscript label 62 | // Here we want to skip %%CLASSNAME field 63 | set mask = "-+dmt+++b" 64 | 65 | set t1 = $zh 66 | set sc = ##class(isc.py.Main).ExecuteGlobal(global, variable _ 1, type, start, end, mask, labels) 67 | write:$$$ISERR(sc) $System.Status.GetErrorText(sc) 68 | set t2 = $zh 69 | write !,"global time: ", t2-t1,! 70 | write ##class(isc.py.util.BPEmulator).DescribeDataframe(variable _ 1),! 71 | 72 | // Approach 2: ExecuteClass 73 | set t1 = $zh 74 | set sc = ##class(isc.py.Main).ExecuteClass(class, variable _ 2, type, start, end) 75 | write:$$$ISERR(sc) $System.Status.GetErrorText(sc) 76 | set t2 = $zh 77 | write !,"class time: ", t2-t1,! 78 | write ##class(isc.py.util.BPEmulator).DescribeDataframe(variable _ 2),! 79 | 80 | // Approach 3: ExecuteTable 81 | set t1 = $zh 82 | set sc = ##class(isc.py.Main).ExecuteTable(table, variable _ 3, type, start, end) 83 | write:$$$ISERR(sc) $System.Status.GetErrorText(sc) 84 | set t2 = $zh 85 | write !,"table time: ", t2-t1,! 86 | write ##class(isc.py.util.BPEmulator).DescribeDataframe(variable _ 3),! 87 | 88 | // Approach 4: ExecuteTable 89 | set t1 = $zh 90 | set sc = ##class(isc.py.Main).ExecuteQuery(query, variable _ 4, type) 91 | write:$$$ISERR(sc) $System.Status.GetErrorText(sc) 92 | set t2 = $zh 93 | write !,"query time: ", t2-t1,! 94 | write ##class(isc.py.util.BPEmulator).DescribeDataframe(variable _ 4),! 95 | } 96 | 97 | Storage Default 98 | { 99 | 100 | 101 | %%CLASSNAME 102 | 103 | 104 | Name 105 | 106 | 107 | DOB 108 | 109 | 110 | TS 111 | 112 | 113 | RandomTime 114 | 115 | 116 | AgeYears 117 | 118 | 119 | AgeDecimal 120 | 121 | 122 | AgeDouble 123 | 124 | 125 | Bool 126 | 127 | 128 | ^isc.py.test.PersonD 129 | PersonDefaultData 130 | ^isc.py.test.PersonD 131 | ^isc.py.test.PersonI 132 | ^isc.py.test.PersonS 133 | %Library.CacheStorage 134 | } 135 | 136 | } 137 | 138 | -------------------------------------------------------------------------------- /Dockerfile191: -------------------------------------------------------------------------------- 1 | ARG IMAGE=intersystems/iris:2019.1.0.510.0-1 2 | FROM ${IMAGE} 3 | 4 | # ensure local python is preferred over distribution python 5 | ENV PATH /usr/local/bin:$PATH 6 | 7 | # https://stackoverflow.com/questions/8671308/non-interactive-method-for-dpkg-reconfigure-tzdata 8 | RUN export DEBIAN_FRONTEND=noninteractive; \ 9 | export DEBCONF_NONINTERACTIVE_SEEN=true; \ 10 | echo 'tzdata tzdata/Areas select Etc' | debconf-set-selections; \ 11 | echo 'tzdata tzdata/Zones/Etc select UTC' | debconf-set-selections; \ 12 | apt-get update -qqy \ 13 | && apt-get install -qqy --no-install-recommends \ 14 | tzdata 15 | 16 | # http://bugs.python.org/issue19846 17 | # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. 18 | ENV LANG C.UTF-8 19 | 20 | # extra dependencies (over what buildpack-deps already includes) 21 | RUN apt-get update && apt-get install -y --no-install-recommends \ 22 | tk-dev \ 23 | gcc \ 24 | openssl \ 25 | libssl1.1 \ 26 | libssl-dev \ 27 | libbz2-dev \ 28 | liblzma-dev \ 29 | && rm -rf /var/lib/apt/lists/* 30 | 31 | ENV PYTHON_VERSION 3.6.7 32 | 33 | RUN set -ex \ 34 | \ 35 | && wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ 36 | && mkdir -p /usr/src/python \ 37 | && tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \ 38 | && rm python.tar.xz \ 39 | && cd /usr/src/python \ 40 | && gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \ 41 | && ./configure \ 42 | --build="$gnuArch" \ 43 | --enable-loadable-sqlite-extensions \ 44 | --enable-shared \ 45 | --with-system-expat \ 46 | --with-system-ffi \ 47 | --without-ensurepip \ 48 | && make -j "$(nproc)" \ 49 | && make install \ 50 | && ldconfig \ 51 | \ 52 | && find /usr/local -depth \ 53 | \( \ 54 | \( -type d -a \( -name test -o -name tests \) \) \ 55 | -o \ 56 | \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \ 57 | \) -exec rm -rf '{}' + \ 58 | && rm -rf /usr/src/python \ 59 | \ 60 | && python3 --version 61 | 62 | # make some useful symlinks that are expected to exist 63 | RUN cd /usr/local/bin \ 64 | && ln -s idle3 idle \ 65 | && ln -s pydoc3 pydoc \ 66 | && ln -s python3 python \ 67 | && ln -s python3-config python-config 68 | 69 | # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" 70 | ENV PYTHON_PIP_VERSION 19.0.3 71 | 72 | RUN set -ex; \ 73 | \ 74 | wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \ 75 | \ 76 | python get-pip.py \ 77 | --disable-pip-version-check \ 78 | --no-cache-dir \ 79 | "pip==$PYTHON_PIP_VERSION" \ 80 | ; \ 81 | pip --version; \ 82 | \ 83 | find /usr/local -depth \ 84 | \( \ 85 | \( -type d -a \( -name test -o -name tests \) \) \ 86 | -o \ 87 | \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \ 88 | \) -exec rm -rf '{}' +; \ 89 | rm -f get-pip.py 90 | 91 | RUN pip install pandas matplotlib seaborn numpy dill 92 | 93 | # now for InterSystems IRIS 94 | 95 | ENV SRC_DIR=/tmp/src 96 | 97 | COPY ./isc/ $SRC_DIR/isc 98 | COPY ./rtn/ $SRC_DIR/rtn 99 | COPY iscpython.so $ISC_PACKAGE_INSTALLDIR/bin/ 100 | 101 | RUN iris start $ISC_PACKAGE_INSTANCENAME EmergencyId=sys,sys && \ 102 | /bin/echo -e "sys\nsys\n" \ 103 | " do ##class(Security.Users).UnExpireUserPasswords(\"*\")\n" \ 104 | " do ##class(Security.Users).AddRoles(\"Admin\", \"%ALL\")\n" \ 105 | " do \$system.OBJ.Load(\$system.Util.GetEnviron(\"SRC_DIR\") _ \"/isc/py/util/Installer.cls\",\"ck\")\n" \ 106 | " set sc = ##class(isc.py.util.Installer).Setup(, 3)\n" \ 107 | " if 'sc write !,\$System.Status.GetErrorText(sc),! do \$system.Process.Terminate(, 1)\n" \ 108 | " zn \"%SYS\"" \ 109 | " do INT^JRNSTOP" \ 110 | " kill ^%SYS(\"JOURNAL\")" \ 111 | " kill ^SYS(\"NODE\")" \ 112 | " halt" \ 113 | | iris session $ISC_PACKAGE_INSTANCENAME && \ 114 | /bin/echo -e "sys\nsys\n" \ 115 | | iris stop $ISC_PACKAGE_INSTANCENAME quietly \ 116 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/journal.log \ 117 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/IRIS.WIJ \ 118 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/iris.ids \ 119 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/alerts.log \ 120 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/journal/* \ 121 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/messages.log \ 122 | && rm -rf $SRC_DIR 123 | 124 | HEALTHCHECK --interval=5s CMD /irisHealth.sh || exit 1 -------------------------------------------------------------------------------- /isc/py/data/Variable.cls: -------------------------------------------------------------------------------- 1 | /// Stored Python variable 2 | Class isc.py.data.Variable Extends %Persistent 3 | { 4 | 5 | Parameter DisplayLimit = 100; 6 | 7 | /// Variable Name 8 | Property Name As %String(MAXLEN = 100); 9 | 10 | /// Variable Class 11 | Property Type As %String(MAXLEN = 100); 12 | 13 | /// Variable repr serialization 14 | Property Value As %Stream.GlobalCharacter; 15 | 16 | /// Variable pickle serialization 17 | Property Pickle As %Stream.GlobalCharacter; 18 | 19 | /// Variable JSON serialization 20 | Property JSON As %Stream.GlobalCharacter; 21 | 22 | /// Module used to serialize the string to Pickle 23 | /// empty == pickle 24 | /// dill == dill 25 | Property PickleModule As %String; 26 | 27 | /// Save variable on disk. 28 | /// Do not save if length > maxLength 29 | /// zw ##class(isc.py.data.Variable).SaveVariable() 30 | ClassMethod SaveVariable(variable As %String = "", maxLength As %Integer = {$$$MaxStringLength}, Output obj As isc.py.data.Variable) As %Status 31 | { 32 | #include Ensemble 33 | kill obj 34 | quit:variable="" $$$ERROR($$$GeneralError, "Variable name can't be empty") 35 | 36 | //$$$TRACE(variable) 37 | 38 | set sc = ##class(isc.py.Main).GetVariableInfo(variable, ##class(isc.py.Callout).#SerializationRepr, .defined, .type, .length) 39 | quit:(($$$ISERR(sc)) || ((maxLength>0) && (length>maxLength))) sc 40 | quit:'defined $$$ERROR($$$GeneralError, "Variable '" _ variable _ "' is not defined") 41 | 42 | #dim repr, pickle, json As %Stream.GlobalCharacter 43 | 44 | set sc = ##class(isc.py.Main).GetVariable(variable, ##class(isc.py.Callout).#SerializationRepr, .repr) 45 | quit:$$$ISERR(sc) sc 46 | 47 | // Ignore errors, as some types can't be serialized. 48 | set sc = ##class(isc.py.Main).GetVariableJson(variable, .json) 49 | set sc = ##class(isc.py.Main).GetVariablePickle(variable, .pickle) 50 | 51 | if $$$ISERR(sc) { 52 | kill pickle 53 | set sc = ##class(isc.py.Main).GetVariablePickle(variable, .pickle, , $$$YES) 54 | set:$$$ISOK(sc) pickleModule = "dill" 55 | } 56 | 57 | set obj = ..%New() 58 | set obj.Name = variable 59 | set obj.Value = repr 60 | set obj.Type = type 61 | set obj.Pickle = pickle 62 | set obj.JSON = json 63 | set obj.PickleModule = $g(pickleModule) 64 | set sc = obj.%Save() 65 | 66 | quit sc 67 | } 68 | 69 | /// Get saved JSON stream as dynamic object. 70 | /// set sc = ##class(isc.py.data.Variable).AsDynamicObject(,.obj) 71 | ClassMethod AsDynamicObject(id As %Integer = "", Output dynObj) As %Status 72 | { 73 | kill dynObj 74 | quit:'..%ExistsId(id) $$$ERROR($$$LoadObjectNotFound, $classname(), id) 75 | set stream = ##class(%Stream.GlobalCharacter).%Open($listbuild(..JSONGetStored(id), "%Stream.GlobalCharacter", "^isc.py.data.VariableS"),,.sc) 76 | 77 | try { 78 | set dynObj = {}.%FromJSON(stream) 79 | } catch ex { 80 | set sc = ex.AsStatus() 81 | } 82 | quit sc 83 | } 84 | 85 | /// Dispaly variable on the current device. 86 | Method Display(indent As %String = "") 87 | { 88 | write indent, "Name: ", ..Name, ! 89 | write indent, "Type: ", ..Type, ! 90 | write indent, "Value", $case(..Value.Size<=..#DisplayLimit, $$$YES:"", $$$NO:" (truncated)"), ": ", ..Value.Read(..#DisplayLimit), !, ! 91 | } 92 | 93 | /// Restore variable from disk. 94 | /// context - variable owner Python context object. 95 | Method Restore(context As isc.py.data.Context) As %Status 96 | { 97 | #dim sc As %Status = $$$OK 98 | if ..Pickle.Size > 0 { 99 | set stream = ##class(%Stream.TmpCharacter).%New() 100 | if ..PickleModule = "dill" { 101 | set module = context.%DillAlias 102 | } else { 103 | set module = context.%PickleAlias 104 | } 105 | 106 | do stream.Write(module _ ".loads(") 107 | do stream.CopyFrom(..Pickle) 108 | do stream.Write(")") 109 | set sc = ##class(isc.py.Main).ExecuteCode(stream, ..Name) 110 | } 111 | 112 | quit sc 113 | } 114 | 115 | Storage Default 116 | { 117 | 118 | 119 | %%CLASSNAME 120 | 121 | 122 | Name 123 | 124 | 125 | Type 126 | 127 | 128 | Value 129 | 130 | 131 | Pickle 132 | 133 | 134 | JSON 135 | 136 | 137 | PickleModule 138 | 139 | 140 | ^isc.py.data.VariableD 141 | VariableDefaultData 142 | ^isc.py.data.VariableD 143 | ^isc.py.data.VariableI 144 | ^isc.py.data.VariableS 145 | %Library.CacheStorage 146 | } 147 | 148 | } 149 | 150 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG IMAGE=store/intersystems/iris-community:2019.4.0.383.0 2 | FROM ${IMAGE} 3 | 4 | 5 | USER root 6 | 7 | # ensure local python is preferred over distribution python 8 | ENV PATH /usr/local/bin:$PATH 9 | 10 | # https://stackoverflow.com/questions/8671308/non-interactive-method-for-dpkg-reconfigure-tzdata 11 | RUN export DEBIAN_FRONTEND=noninteractive; \ 12 | export DEBCONF_NONINTERACTIVE_SEEN=true; \ 13 | echo 'tzdata tzdata/Areas select Etc' | debconf-set-selections; \ 14 | echo 'tzdata tzdata/Zones/Etc select UTC' | debconf-set-selections; \ 15 | apt-get update -qqy \ 16 | && apt-get install -qqy --no-install-recommends \ 17 | tzdata 18 | 19 | # http://bugs.python.org/issue19846 20 | # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. 21 | ENV LANG C.UTF-8 22 | 23 | # extra dependencies (over what buildpack-deps already includes) 24 | RUN apt-get update && apt-get install -y --no-install-recommends \ 25 | tk-dev \ 26 | gcc \ 27 | openssl \ 28 | libssl1.1 \ 29 | libssl-dev \ 30 | libbz2-dev \ 31 | liblzma-dev \ 32 | libsqlite3-dev \ 33 | && rm -rf /var/lib/apt/lists/* 34 | 35 | ENV PYTHON_VERSION 3.6.7 36 | 37 | RUN set -ex \ 38 | \ 39 | && wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ 40 | && mkdir -p /usr/src/python \ 41 | && tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \ 42 | && rm python.tar.xz \ 43 | && cd /usr/src/python \ 44 | && gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \ 45 | && ./configure \ 46 | --build="$gnuArch" \ 47 | --enable-loadable-sqlite-extensions \ 48 | --enable-shared \ 49 | --with-system-expat \ 50 | --with-system-ffi \ 51 | --without-ensurepip \ 52 | && make -j "$(nproc)" \ 53 | && make install \ 54 | && ldconfig \ 55 | \ 56 | && find /usr/local -depth \ 57 | \( \ 58 | \( -type d -a \( -name test -o -name tests \) \) \ 59 | -o \ 60 | \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \ 61 | \) -exec rm -rf '{}' + \ 62 | && rm -rf /usr/src/python \ 63 | \ 64 | && python3 --version 65 | 66 | # make some useful symlinks that are expected to exist 67 | RUN cd /usr/local/bin \ 68 | && ln -s idle3 idle \ 69 | && ln -s pydoc3 pydoc \ 70 | && ln -s python3 python \ 71 | && ln -s python3-config python-config 72 | 73 | # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" 74 | ENV PYTHON_PIP_VERSION 19.0.3 75 | 76 | RUN set -ex; \ 77 | \ 78 | wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \ 79 | \ 80 | python get-pip.py \ 81 | --disable-pip-version-check \ 82 | --no-cache-dir \ 83 | "pip==$PYTHON_PIP_VERSION" \ 84 | ; \ 85 | pip --version; \ 86 | \ 87 | find /usr/local -depth \ 88 | \( \ 89 | \( -type d -a \( -name test -o -name tests \) \) \ 90 | -o \ 91 | \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \ 92 | \) -exec rm -rf '{}' +; \ 93 | rm -f get-pip.py 94 | 95 | RUN pip install pandas matplotlib seaborn numpy dill 96 | 97 | # now for InterSystems IRIS 98 | 99 | USER irisowner 100 | 101 | ENV SRC_DIR=/home/irisowner 102 | 103 | COPY --chown=irisowner ./isc/ $SRC_DIR/isc 104 | COPY --chown=irisowner ./rtn/ $SRC_DIR/rtn 105 | COPY --chown=irisowner iscpython.so $ISC_PACKAGE_INSTALLDIR/bin/ 106 | 107 | 108 | RUN iris start $ISC_PACKAGE_INSTANCENAME && \ 109 | /bin/echo -e " do \$system.OBJ.Load(\$system.Util.GetEnviron(\"SRC_DIR\") _ \"/isc/py/util/Installer.cls\",\"ck\")\n" \ 110 | " set sc = ##class(isc.py.util.Installer).Setup(, 3)\n" \ 111 | " if 'sc write !,\$System.Status.GetErrorText(sc),! do \$system.Process.Terminate(, 1)\n" \ 112 | " zn \"%SYS\"" \ 113 | " set sc = ##class(Security.Users).UnExpireUserPasswords(\"*\")\n" \ 114 | " if 'sc write !,\$System.Status.GetErrorText(sc),! do \$system.Process.Terminate(, 1)\n" \ 115 | " do ##class(Security.Users).AddRoles(\"Admin\", \"%ALL\")\n" \ 116 | " do INT^JRNSTOP" \ 117 | " kill ^%SYS(\"JOURNAL\")" \ 118 | " kill ^SYS(\"NODE\")" \ 119 | " halt" \ 120 | | iris session $ISC_PACKAGE_INSTANCENAME && \ 121 | iris stop $ISC_PACKAGE_INSTANCENAME quietly \ 122 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/journal.log \ 123 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/IRIS.WIJ \ 124 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/iris.ids \ 125 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/alerts.log \ 126 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/journal/* \ 127 | && rm -f $ISC_PACKAGE_INSTALLDIR/mgr/messages.log \ 128 | && rm -rf $SRC_DIR/isc $SRC_DIR/rtn \ 129 | && touch $ISC_PACKAGE_INSTALLDIR/mgr/messages.log 130 | 131 | HEALTHCHECK --interval=5s CMD /irisHealth.sh || exit 1 -------------------------------------------------------------------------------- /jupyter/MLContentsManager.py: -------------------------------------------------------------------------------- 1 | import irisnative 2 | from notebook.services.contents.manager import ContentsManager 3 | import json 4 | from traitlets import Unicode, Integer, HasTraits, default 5 | from datetime import datetime 6 | import nbformat 7 | 8 | class MLContentsManager(ContentsManager, HasTraits): 9 | 10 | port = Integer( 11 | default_value=51773, 12 | config=True, 13 | help="InterSystems IRIS SuperServer port number.", 14 | ) 15 | 16 | host = Unicode( 17 | default_value='localhost', 18 | config=True, 19 | help="InterSystems IRIS server address.", 20 | ) 21 | 22 | namespace = Unicode( 23 | default_value='USER', 24 | config=True, 25 | help="InterSystems IRIS namespace.", 26 | ) 27 | 28 | user = Unicode( 29 | default_value='_SYSTEM', 30 | config=True, 31 | help="InterSystems IRIS User", 32 | ) 33 | 34 | password = Unicode( 35 | default_value='SYS', 36 | config=True, 37 | help='InterSystems IRIS Password', 38 | ) 39 | 40 | 41 | @default('checkpoints_class') 42 | def _checkpoints_class_default(self): 43 | return MLCheckpoints 44 | 45 | className = 'isc.py.util.Jupyter' 46 | 47 | def __init__(self, **kwargs): 48 | super().__init__(**kwargs) 49 | 50 | conn = irisnative.createConnection(self.host, self.port, self.namespace, self.user, self.password) 51 | self.iris = irisnative.createIris(conn) 52 | 53 | 54 | def get(self, path, content=True, type=None, format=None): 55 | ''' 56 | Get the file (bpl) or directory (package) at path. 57 | ''' 58 | model = self.iris.classMethodValue(self.className, 'Get', path, content, type, format) 59 | 60 | model = json.loads(model) 61 | model['last_modified'] = datetime.strptime(model['last_modified'],"%Y-%m-%dT%H:%M:%S.%f") 62 | if model['type'] == 'notebook' and content==True: 63 | model['content'] = nbformat.reads(json.dumps(model['content']), 4) 64 | return model 65 | 66 | def save(self, model, path): 67 | ''' 68 | Save a file or directory model to path. 69 | ''' 70 | self.iris.classMethodValue(self.className, 'Save', json.dumps(model), path) 71 | model = self.get(path, content=False) 72 | model['content'] = None 73 | model['format'] = None 74 | return model 75 | 76 | def delete_file(self, path): 77 | ''' 78 | Delete the file or directory at path. 79 | ''' 80 | return self.iris.classMethodValue(self.className, 'Delete', path) 81 | 82 | def rename_file(self, old_path, new_path): 83 | ''' 84 | Rename a file or directory. 85 | ''' 86 | return self.iris.classMethodValue(self.className, 'Rename', old_path, new_path) 87 | 88 | def file_exists(self, path): 89 | ''' 90 | Does a file exist at the given path? 91 | ''' 92 | return self.iris.classMethodValue(self.className, 'Exists', path) 93 | 94 | def dir_exists(self, path): 95 | ''' 96 | Does a directory exist at the given path? 97 | ''' 98 | return self.iris.classMethodValue(self.className, 'ExistsDir', path) 99 | 100 | def is_hidden(self, path): 101 | return False 102 | 103 | from notebook.services.contents.checkpoints import ( 104 | Checkpoints, 105 | GenericCheckpointsMixin, 106 | ) 107 | 108 | class MLCheckpoints(GenericCheckpointsMixin, Checkpoints): 109 | className = 'isc.py.util.JupyterCheckpoints' 110 | 111 | 112 | """requires the following methods:""" 113 | def create_file_checkpoint(self, content, format, path): 114 | """ -> checkpoint model""" 115 | return self.parent.iris.classMethodValue(self.className, 'CreateFile', json.dumps(content), format, path) 116 | 117 | 118 | def create_notebook_checkpoint(self, nb, path): 119 | """ -> checkpoint model""" 120 | return self.parent.iris.classMethodValue(self.className, 'CreateNotebook', json.dumps(nb), path) 121 | 122 | def get_file_checkpoint(self, checkpoint_id, path): 123 | """ -> {'type': 'file', 'content': , 'format': {'text', 'base64'}}""" 124 | #return self.parent.iris.classMethodValue(self.className, 'CreateNotebook', nb, path) 125 | return self.parent.get(path, True, None) 126 | 127 | def get_notebook_checkpoint(self, checkpoint_id, path): 128 | """ -> {'type': 'notebook', 'content': }""" 129 | return self.parent.get(path, True, 'text') 130 | 131 | def delete_checkpoint(self, checkpoint_id, path): 132 | """deletes a checkpoint for a file""" 133 | return self.parent.iris.classMethodValue(self.className, 'DeleteCheckpoint', checkpoint_id, path) 134 | 135 | def list_checkpoints(self, path): 136 | """returns a list of checkpoint models for a given file, 137 | default just does one per file 138 | """ 139 | return [] 140 | 141 | def rename_checkpoint(self, checkpoint_id, old_path, new_path): 142 | """renames checkpoint from old path to new path""" 143 | return self.parent.iris.classMethodValue(self.className, 'RenameCheckpoint', checkpoint_id, old_path, new_path) -------------------------------------------------------------------------------- /isc/py/util/BPEmulator.cls: -------------------------------------------------------------------------------- 1 | /// Execute BP in a current job 2 | Class isc.py.util.BPEmulator 3 | { 4 | 5 | /// Init context and process variables 6 | /// do ##class(isc.py.util.BPEmulator).Init() 7 | ClassMethod Init() [ PublicList = (context, process) ] 8 | { 9 | set context = {} 10 | set process = {"WorkDirectory": (##class(%File).NormalizeDirectory(##class(%SYS.System).TempDirectory()))} 11 | } 12 | 13 | /// Execute sentiment process 14 | /// do ##class(isc.py.util.BPEmulator).ExecuteSentiment() 15 | ClassMethod ExecuteSentiment() 16 | { 17 | set class = "CONVERGENCE.WEB.SENTIMENT" 18 | set activities = $lb("Import Python Libraries", 19 | $lb("Query In Negative Tweets", "data_negative"), 20 | $lb("Query In Positive Tweets", "data_positive"), 21 | $lb("Query In Unknown Tweets", "data_unknown"), 22 | "Preprocess & Split") 23 | do ..RunActivities(class, activities) 24 | } 25 | 26 | /// Execute intra process 27 | /// do ##class(isc.py.util.BPEmulator).ExecuteIntra() 28 | ClassMethod ExecuteIntra() 29 | { 30 | set class = "CONVERGENCE.WEB.INTRA" 31 | set activities = $lb("Import Python Libraries", 32 | $lb("Query In Time Series", "df"), 33 | "Histroric Time Series", 34 | "Decompose Time Series", 35 | "Explore Modeling Space", 36 | "Explore Selected Model", 37 | "Create Forecast") 38 | do ..RunActivities(class, activities) 39 | } 40 | 41 | /// Execute test process 42 | /// do ##class(isc.py.util.BPEmulator).ExecuteTest() 43 | ClassMethod ExecuteTest() 44 | { 45 | set class = "isc.py.test.Process" 46 | set activities = $lb("Import pandas", 47 | $lb("RAW", "Data"), 48 | "Correlation Matrix: Tabular", 49 | "Correlation Matrix: Graph") 50 | do ..RunActivities(class, activities) 51 | } 52 | 53 | /// Execute campaign process 54 | /// do ##class(isc.py.util.BPEmulator).ExecuteCampaign() 55 | ClassMethod ExecuteCampaign() 56 | { 57 | set class = "CONVERGENCE.WEB.CAMPAIGN" 58 | set activities = $lb("Import Python Libraries", 59 | $lb("Query In Testing Set", "Data_test"), 60 | $lb("Query In Training Set", "Data_train"), 61 | "Form Testing Set", 62 | "Form Training Set", 63 | "Split Training Set", 64 | "Initialize LR", 65 | "Fit/Apply GB + LR") 66 | do ..RunActivities(class, activities) 67 | } 68 | 69 | /// Execute activities from class 70 | ClassMethod RunActivities(class As %Dictionary.CacheClassname, activities As %List) 71 | { 72 | do ..Init() 73 | for i=1:1:$ll(activities) { 74 | set activity = $lg(activities, i) 75 | 76 | write "Executing: " 77 | set start = $zh 78 | if $listvalid(activity) { 79 | write $lg(activity)," (query), into variable: ", $lg(activity, 2) 80 | set sc = ..ExecuteQuery(class, $lg(activity), $lg(activity, 2), "RAW") 81 | } else { 82 | write activity 83 | set sc = ..ExecuteCode(class, activity) 84 | } 85 | set end = $zh 86 | 87 | write " result: ", $case($$$ISOK(sc), $$$YES:"OK", $$$NO: $System.Status.GetErrorText(sc)),", Time: ",end-start," sec.",! 88 | break:$$$ISERR(sc) 89 | } 90 | } 91 | 92 | /// Execute code activity 93 | /// w ##class(isc.py.util.BPEmulator).ExecuteCode() 94 | ClassMethod ExecuteCode(class, name) As %Status 95 | { 96 | set code = $classmethod(class, "GetAnnotation", name) 97 | set sc = ##class(isc.py.Main).SimpleString(code) 98 | quit sc 99 | } 100 | 101 | /// Execute Query activity 102 | /// w ##class(isc.py.util.BPEmulator).ExecuteQuery() 103 | ClassMethod ExecuteQuery(class, name, variable, mode As %String = "RAW") As %Status 104 | { 105 | set query = $classmethod(class, "GetAnnotation", name) 106 | if mode = "RAW" { 107 | set sc = ##class(isc.py.Main).ExecuteQuery(query, variable, "dataframe") 108 | } elseif mode = "ODBC" { 109 | set sc = ##class(isc.py.Main).ImportModule("pyodbc", , .pyodbc) 110 | set sc = ##class(isc.py.Main).ImportModule("pandas", , .pandas) 111 | set sc = ##class(isc.py.Main).SimpleString("cnxn=" _ pyodbc _ ".connect(('DSN=ENSEMBLE(PYTHON);UID=dev;PWD=123'),autocommit=True)") 112 | set sc = ##class(isc.py.Main).SimpleString(variable _ "=" _ pandas _".read_sql('"_ query _ "',cnxn)") 113 | //set sc = ##class(isc.py.Main).SimpleString("cnxn.close()") 114 | } elseif mode = "JDBC" { 115 | set sc = ##class(isc.py.Main).ImportModule("jaydebeapi", , .jaydebeapi) 116 | set sc = ##class(isc.py.Main).ImportModule("pandas", , .pandas) 117 | set sc = ##class(isc.py.Main).SimpleString("cnxn=" _ jaydebeapi _ ".connect(""com.intersystems.jdbc.IRISDriver"",""jdbc:IRIS://localhost:51773/Python"", [""dev"", ""123""], ""/InterSystems/IRIS/dev/java/lib/JDK18/intersystems-jdbc-3.0.0.jar"")") 118 | set sc = ##class(isc.py.Main).SimpleString(variable _ "=" _ pandas _".read_sql('"_ query _ "',cnxn)") 119 | set sc = ##class(isc.py.Main).SimpleString("cnxn.close()") 120 | } else { 121 | set sc = $$$ERROR($$$GeneralError, "Unknown query mode: " _ mode) 122 | } 123 | quit sc 124 | } 125 | 126 | /// w ##class(isc.py.util.BPEmulator).DescribeDataframe() 127 | ClassMethod DescribeDataframe(variable) As %Status 128 | { 129 | set sc = ##class(isc.py.Main).SimpleString("zzzinfo=" _ variable _ ".dtypes", "zzzinfo", , .zzzinfo) 130 | quit:$$$ISERR(sc) sc 131 | write $replace(zzzinfo, $c(10), $$$NL) 132 | set sc = ##class(isc.py.Main).SimpleString("del zzzinfo") 133 | quit sc 134 | } 135 | 136 | } 137 | 138 | -------------------------------------------------------------------------------- /RELEASE-NOTES.md: -------------------------------------------------------------------------------- 1 | # Release notes 2 | 3 | See [chagelog](CHANGELOG.md) for a full list of changes. This document marks the most important changes and offers some commentary. 4 | 5 | ## [0.8] - 2019-09-12 6 | 7 | Download new release [from GitHub](https://github.com/intersystems-community/PythonGateway/releases). 8 | 9 | Now for the new features. 10 | 11 | **Fast transfer**. Pass globals, classes and tables from InterSystems IRIS to Python with ease and speed (3x faster than ODBC/JDBC driver and 10x faster than old `QueryExecute`). [Documentation](https://github.com/intersystems-community/PythonGateway/blob/master/DataTransfer.md). 12 | 13 | **Proxy Gateway** allows generation of ObjectScript classes for Python classes. Instantinate objects, call object and class methods using familiar ObjectScript code ([docs](https://github.com/intersystems-community/PythonGateway/blob/master/Gateway.md)). 14 | ``` 15 | set module = "random" 16 | set sc = ##class(isc.py.gw.Generator).Generate(module) 17 | set sc = ##class(isc.py.Main).ImportModule(module) 18 | write ##class(py.random).randint(1,100) 19 | ``` 20 | 21 | **Annotation evaluation** - business processes inheriting from `isc.py.ens.ProcessUtils` now can use activity annotations with variables which would be calculated on ObjectScript side before being passed to Python. 22 | 23 | **Jupyter support** - Jupyter can now be used to browse and edit Interoperability processes. [Documentation](https://github.com/intersystems-community/PythonGateway/tree/master/jupyter). 24 | 25 | Last but not least and thanks to Benjamin De Boe we added **PMML support** via `isc.py.util.PMML` allowing imports of predictive models from Python into InterSystems IRIS via PMML. 26 | 27 | **Installation instructions** are available in [Readme](https://github.com/intersystems-community/PythonGateway). 28 | 29 | **Update instructions** (from v0.7): 30 | 31 | 1. Load new code and library [from GitHub](https://github.com/intersystems-community/PythonGateway/releases). 32 | 33 | ## [0.7] - 2019-03-22 34 | 35 | First of all we're now public! You can download new release [from GitHub](https://github.com/intersystems-community/PythonGateway/releases). 36 | 37 | Now for the new features. 38 | 39 | We added **Docker support** and with that, Python Gateway runs instantly on any machine with Docker. Dockerfile builds upon InterSystems IRIS 2019.1 image. Alternatively, pull the prebuilt image from _intersystemscommunity_ on docker hub (write me a mail and I will add you to the organization). 40 | 41 | Next feature is the ability to execute arbitrary Python functions in a structured and orderly way with the new **Execute Function API**. Here is a small preview on how it looks like: 42 | 43 | ``` 44 | set sc = ##class(isc.py.Main).ImportModule("random") 45 | set sc = ##class(isc.py.Main).ExecuteFunctionArgs("random.randint", , ,.result, 1, 100) 46 | write result, ! 47 | >26 48 | ``` 49 | 50 | **Proxyless Gateway** builds on top of that, offering `DynamicObject` class. Get and set object properties and execute object methods (and get results back): 51 | 52 | ``` 53 | set obj = ##class(isc.py.gw.DynamicObject).%New("Person", "p1", "'Ed'", "25", "'Test'") 54 | write obj.name 55 | >Ed 56 | ``` 57 | 58 | In Python modules and classes are objects too, so this works too: 59 | ``` 60 | set module = "random" 61 | set sc = ##class(isc.py.Main).ImportModule(module) 62 | set random = ##class(isc.py.gw.DynamicObject).%New(,module) 63 | write random.randint(1,100) 64 | >74 65 | ``` 66 | More examples of how you can use Proxyless Gateway and Execute Function API are available [here](https://github.com/intersystems-community/PythonGateway/blob/master/Gateway.md). 67 | 68 | Next, **Execute Query API** (a basis for Interoperability RAW data transfer) saw considerable improvements. Now it can execute queries from any namespace and can transfer data of numeric, boolean and date/time types in addition to strings. 69 | 70 | Interoperability productions now can retrieve variables as a **JSON**. Check test process to see how you can leverage it. 71 | 72 | Saving process context now also saves user-defined functions. 73 | 74 | You can now run **Python shell** from the terminal with either command: 75 | 76 | - `do ##class(isc.py.util.Shell).Shell()` 77 | - `zpy` 78 | 79 | Users now can implement **initializer interface**. It allows you to define modules, functions and classes you want loaded with the start of the python process. Checkisc.py.init.Abstract class for implementation details. This option is available for both Interoperability and Terminal API users. 80 | 81 | **Business Process emulator** from `isc.py.util.BPEmulator` class allows running Python-enabled business processes in a current terminal job, making testing, debugging and profiling intelligent business processes easier than ever before. 82 | 83 | And many other bug fixes and improvements, check full changelog available [here](https://github.com/intersystems-community/PythonGateway/blob/master/Gateway.md). 84 | 85 | **Installation instructions** are available in [Readme](https://github.com/intersystems-community/PythonGateway). 86 | 87 | **Update instructions** (from v0.6): 88 | 89 | 1. Load new code and library [from GitHub](https://github.com/intersystems-community/PythonGateway/releases). 90 | 2. PythonLib setting should be reset again if it was set before the update. 91 | 3. All processes using RAW data ingestion method must be recompiled. 92 | 4. It is now a requirement to install dill module: `pip install dill`. 93 | 5. Reload test data: `do ##class(isc.py.test.CannibalizationData).Import()` 94 | 95 | [0.8]: https://github.com/intersystems-community/PythonGateway/compare/v0.7...v0.8 96 | [0.7]: https://github.com/intersystems-community/PythonGateway/compare/v0.6...v0.7 97 | [0.7]: https://github.com/intersystems-community/PythonGateway/compare/v0.6...v0.7 98 | -------------------------------------------------------------------------------- /DataTransfer.md: -------------------------------------------------------------------------------- 1 | # Data Transfer 2 | 3 | Transfer data into and from Python. All the methods are defined in `isc.py.Main`. All methods return `%Status`. 4 | 5 | # Python -> InterSystems IRIS 6 | 7 | - `GetVariable(variable, serialization, .stream, useString)` - get `serialization` of `variable` in `stream`. If `useString` is 1 and variable serialization can fit into string then string is returned instead of the stream. 8 | - `GetVariableJson(variable, .stream, useString)` - get JSON serialization of variable. 9 | - `GetVariablePickle(variable, .stream, useString, useDill)` - get Pickle (or Dill) serialization of variable. 10 | 11 | # InterSystems IRIS -> Python 12 | 13 | Load data from InterSystems IRIS to Python. 14 | All these methods support data transfer from any local namespace. `isc.py` package must be available in `namespace`. 15 | 16 | ## ExecuteQuery 17 | 18 | `ExecuteQuery(query, variable, type, namespace)` - transfer results from any valid SQL query into Python. It is the slowest method of data transfer. Use it if `ExecuteGlobal` and its wrappers are unavailable. 19 | 20 | Arguments: 21 | - `query` - sql query 22 | - `variable` - target variable on a Python side 23 | - `type` - `list` or Pandas `dataframe` 24 | 25 | 26 | ## ExecuteGlobal 27 | 28 | `ExecuteGlobal(global, variable, type, start, end, mask, labels, namespace)` - transfer global data to Python. 29 | 30 | Arguments: 31 | - `global` - global name without `^` 32 | - `variable` - target variable on a Python side 33 | - `type` - `list` or Pandas `dataframe` 34 | - `start` - initial global key. Must be integer. 35 | - `end` - final global key. Must be integer. 36 | - `mask` - string, mask for global values. Mask may be shorter than the number of global value fields (in this case fields at the end would be skipped). How to format mask: 37 | - `+` use field as is 38 | - `-` skip field 39 | - `b` - boolean (0 - False, anything else - True) 40 | - `d` - date (from $horolog, on Windows only from 1970, on Linux from 1900 see notes for details) 41 | - `t` - time ($horolog, seconds since midnight) 42 | - `m` - (moment) timestamp string in YEAR-MONTH-DAY HOUR:MINUTE:SECOND format. 43 | - `labels` - %List of column names, first element is key column name. Therefore: List length must be mask symbol length + 1. 44 | 45 | ## ExecuteClass 46 | 47 | Wrapper for `ExecuteGlobal`. Effectively it parses compiled class definition, constructs `ExecuteGlobal` arguments and calls it. 48 | 49 | `ExecuteClass(class, variable, type, start, end, properties, namespace)` - transfer class data to Python list of tuples or pandas dataframe. `properties` - comma-separated list of properties to form dataframe from. `*` and `?` wildcards are supported. Defaults to * (all properties). %%CLASSNAME property is ignored. Only stored properties can be used. 50 | 51 | Arguments: 52 | - `class` - class name 53 | - `variable` - target variable on a Python side 54 | - `type` - `list` or Pandas `dataframe` 55 | - `start` - initial object id. Must be integer. 56 | - `end` - final object id. Must be integer. 57 | - `properties` - comma-separated list of properties to form dataframe from. `*` and `?` wildcards are supported. Defaults to `*` (all properties). `%%CLASSNAME` property is ignored. Only stored properties can be used. 58 | 59 | All properties transferred as is except properties of `%Date`, `%Time`, `%Boolean` and `%TimeStamp` types. They are converted to respective Python datatypes. 60 | 61 | ## ExecuteTable 62 | 63 | Wrapper for `ExecuteClass`. Translates table name to class name and calls `ExecuteClass`. Signature: 64 | 65 | `ExecuteTable(table, variable, type, start, end, properties, namespace)` - transfer table data to Python list of tuples or pandas dataframe. 66 | 67 | Arguments: 68 | - `table` - table name. 69 | 70 | Other arguments are passed as is to `ExecuteClass`. 71 | 72 | ## Notes 73 | 74 | - `ExecuteGlobal`, `ExecuteClass` and `ExecuteTable` generally offer the same speed (as the time to parse class definition is negligible). 75 | - `ExecuteGlobal` is 3-5 times faster than ODBC driver and up to 20 times faster than `ExecuteQuery` on measurable workloads (>0.01 second). 76 | - `ExecuteGlobal`, `ExecuteClass` and `ExecuteTable` only work on the globals with this structure: `^global(key) = $lb(prop1, prop2, ..., propN)` where `key` must be an integer. 77 | - For `ExecuteGlobal`, `ExecuteClass` and `ExecuteTable` supported `%Date` range equals `mktime` range ([windows](https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/mktime-mktime32-mktime64?view=vs-2019): 1970-01-01, [linux](https://linux.die.net/man/3/mktime) 1900-01-01, [mac](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/mktime.3.html)). Use `%TimeStamp` to transfer dates outside of this range. 78 | - For `ExecuteGlobal`, `ExecuteClass` and `ExecuteTable` all arguments besides the source (global, class, table) and variable are optional. 79 | 80 | ## Examples 81 | 82 | Let's say we have [isc.py.test.Person](https://github.com/intersystems-community/PythonGateway/blob/master/isc/py/test/Person.cls) class. Here's how we can use all methods of data transfer: 83 | 84 | ``` 85 | // All the ways to transfer data 86 | set global = "isc.py.test.PersonD" 87 | set class = "isc.py.test.Person" 88 | set table = "isc_py_test.Person" 89 | set query = "SELECT * FROM isc_py_test.Person" 90 | 91 | // Common arguments 92 | set variable = "df" 93 | set type = "dataframe" 94 | set start = 1 95 | set end = $g(^isc.py.test.PersonD, start) 96 | 97 | // Approach 0: ExecuteGlobal without arguments 98 | set sc = ##class(isc.py.Main).ExecuteGlobal(global, variable _ 0, type) 99 | 100 | // Approach 1: ExecuteGlobal with arguments 101 | // For global transfer labels are not calculated automatically 102 | // globalKey - is global subscript 103 | set labels = $lb("globalKey", "Name", "DOB", "TS", "RandomTime", "AgeYears", "AgeDecimal", "AgeDouble", "Bool") 104 | 105 | // mask is 1 element shorter than labels because "globalKey" is global subscript label 106 | // Here we want to skip %%CLASSNAME field 107 | set mask = "-+dmt+++b" 108 | 109 | set sc = ##class(isc.py.Main).ExecuteGlobal(global, variable _ 1, type, start, end, mask, labels) 110 | 111 | // Approach 2: ExecuteClass 112 | set sc = ##class(isc.py.Main).ExecuteClass(class, variable _ 2, type, start, end) 113 | 114 | // Approach 3: ExecuteTable 115 | set sc = ##class(isc.py.Main).ExecuteTable(table, variable _ 3, type, start, end) 116 | 117 | // Approach 4: ExecuteQuery 118 | set sc = ##class(isc.py.Main).ExecuteQuery(query, variable _ 4, type) 119 | ``` 120 | 121 | You can call this method: `do ##class(isc.py.test.Person).Test()` to check how these data transfer methods work. 122 | -------------------------------------------------------------------------------- /isc/py/util/Installer.cls: -------------------------------------------------------------------------------- 1 | /// Installer for the container release of PythonGateway 2 | Class isc.py.util.Installer 3 | { 4 | 5 | XData Install [ XMLNamespace = INSTALLER ] 6 | { 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | } 39 | 40 | ClassMethod Setup(ByRef pVars, pLogLevel As %Integer = 3, pInstaller As %Installer.Installer, pLogger As %Installer.AbstractLogger) As %Status [ CodeMode = objectgenerator, Internal ] 41 | { 42 | Quit ##class(%Installer.Manifest).%Generate(%compiledclass, %code, "Install") 43 | } 44 | 45 | /// Set production as default. 46 | /// Load test data 47 | /// Set libpython3.6m.so as PythonLib 48 | ClassMethod ConfigureProduction() As %Status 49 | { 50 | set production = "isc.py.test.Production" 51 | set ^Ens.Configuration("csp","LastProduction") = production 52 | set sc = ##class(isc.py.test.CannibalizationData).Import() 53 | quit:$$$ISERR(sc) sc 54 | 55 | set setting = ##class(Ens.Config.Setting).%New() 56 | set setting.Name = "PythonLib" 57 | set setting.Target = "Host" 58 | set setting.Value = "libpython3.6m.so" 59 | 60 | set exists = ##class(Ens.Config.Item).NameExists(production, "isc.py.ens.Operation", .itemId) 61 | set item = ##class(Ens.Config.Item).%OpenId(itemId) 62 | do item.Settings.Insert(setting) 63 | set sc = item.%Save() 64 | quit:$$$ISERR(sc) sc 65 | 66 | 67 | quit sc 68 | } 69 | 70 | /// Try to configure test Process. 71 | /// set sc = ##class(isc.py.util.Installer).ConfigureTestProcess() 72 | ClassMethod ConfigureTestProcess(user As %String = "Admin", password As %String = "SYS", host As %String = "localhost", port As %Integer = 51773, namespace As %String = {$namespace}) As %Status 73 | { 74 | #include %occJava 75 | // Try to adjust default process 76 | set process = "isc.py.test.Process" 77 | set sc = ##class(%Compiler.UDL.TextServices).GetTextAsString(,process, .code) 78 | quit:$$$ISERR(sc) sc 79 | 80 | // JDBC 81 | 82 | set sc = ##class(%Net.Remote.Service).GetJARDirectory("1.8", .jarDir) 83 | quit:$$$ISERR(sc) sc 84 | set jdbcJar = ##class(%File).NormalizeFilename($$$javaJDBCJAR, jarDir) 85 | set jdbcJar = ##class(%File).NormalizeFilenameWithSpaces(jdbcJar) 86 | 87 | set code = $replace(code, "[""dev"", ""123""]", $$$FormatText("[""%1"", ""%2""]", user, password)) // JDBC access 88 | set code = $replace(code, "/InterSystems/IRIS/dev/java/lib/JDK18/intersystems-jdbc-3.0.0.jar", jdbcJar) // JDBC Jar path 89 | set code = $replace(code, "jdbc:IRIS://localhost:51773/Python", $$$FormatText("jdbc:IRIS://%1:%2/%3", host, port, namespace)) // JDBC connection string 90 | 91 | /// ODBC 92 | if $$$isWINDOWS { 93 | set driver = "InterSystems ODBC35" 94 | } else { 95 | set driver = ##class(%File).NormalizeDirectory($System.Util.BinaryDirectory()) _ "libirisodbcu35.so" 96 | } 97 | set code = $replace(code, "DSN=ENSEMBLE(PYTHON);UID=dev;PWD=123", $$$FormatText("Driver=%1;Server=%2;Port=%3;database=%4;UID=%5;PWD=%6;" ,driver, host, port, namespace, user, password)) 98 | 99 | set sc = ##class(%Compiler.UDL.TextServices).SetTextFromString(,process, .code) 100 | quit:$$$ISERR(sc) sc 101 | 102 | set sc = $system.OBJ.Compile(process, "ck") 103 | 104 | quit sc 105 | } 106 | 107 | /// Enable OS Authentication 108 | ClassMethod OSAuth() As %Status 109 | { 110 | new $namespace 111 | set $namespace = "%SYS" 112 | set sc = ##Class(Security.System).Get($$$SystemSecurityName,.properties) 113 | quit:$$$ISERR(sc) sc 114 | 115 | set autheEnabled = properties("AutheEnabled") 116 | quit:'$data(autheEnabled) $$$ERROR($$$PropertyDoesNotExist, "AutheEnabled") 117 | set properties("AutheEnabled") = $ZBOOLEAN(+autheEnabled, $$$AutheOS, 7) 118 | set sc = ##Class(Security.System).Modify($$$SystemSecurityName, .properties) 119 | 120 | quit sc 121 | } 122 | 123 | ClassMethod LoadZRoutines(dir As %String) As %Status 124 | { 125 | new $namespace 126 | set $namespace = "%SYS" 127 | 128 | set dir = ##class(%File).NormalizeDirectory(dir) 129 | set sc = $system.OBJ.ImportDir(dir, "*.xml", "c") 130 | quit sc 131 | } 132 | 133 | /// Add magnifying glass to Annotation 134 | /// zw ##class(isc.py.util.Installer).PatchBP() 135 | ClassMethod PatchBP() As %Status 136 | { 137 | set class = "EnsPortal.BPLEditor" 138 | set patch = "" 139 | $$$QuitOnError(##class(%Compiler.UDL.TextServices).GetTextAsString(, class, .code)) 140 | quit:$find(code, patch) $$$OK 141 | 142 | set oldNS = $namespace 143 | new $namespace 144 | set $namespace = "%SYS" 145 | 146 | set db = ##class(SYS.Database).%OpenId($system.Util.ManagerDirectory() _ "enslib") 147 | set bk = db.ReadOnly 148 | set db.ReadOnly = 0 149 | 150 | $$$QuitOnError(db.%Save()) 151 | 152 | set $namespace = oldNS 153 | set line = "